Pulpito :: Results Dashboard

PASS

master

smithi

smithi019 smithi131 smithi164

multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}

2366061

FAIL

master

smithi

smithi143 smithi156 smithi175

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}

"2018-04-08 16:05:52.594309 mon.b mon.0 172.21.15.156:6789/0 237 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 86 sec (SLOW_OPS)" in cluster log

2366062

FAIL

master

smithi

smithi043 smithi047 smithi169

multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}

"2018-04-08 15:51:14.529455 mon.a mon.0 172.21.15.47:6789/0 177 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 748 sec (SLOW_OPS)" in cluster log

2366063

FAIL

master

smithi

smithi048 smithi086 smithi109

multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml}

"2018-04-08 15:48:43.564972 mds.b mds.1 172.21.15.109:6817/3347618994 1 : cluster [ERR] replayed stray Session close event for client.5434 172.21.15.48:0/2801371376 from time 2018-04-08 15:43:43.581727, ignoring" in cluster log

2366064

FAIL

master

smithi

smithi005 smithi077 smithi081

multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}

"2018-04-08 15:49:24.803383 mon.a mon.0 172.21.15.5:6789/0 264 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 1074 sec (SLOW_OPS)" in cluster log

2366065

FAIL

master

smithi

smithi045 smithi130 smithi151

multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}

"2018-04-08 15:33:33.299820 mon.a mon.0 172.21.15.45:6789/0 198 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 116 sec (SLOW_OPS)" in cluster log

2366066

PASS

master

smithi

smithi013 smithi057 smithi144

multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}

2366067

FAIL

master

smithi

smithi115 smithi173 smithi174

multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}

"2018-04-08 15:49:57.461998 mon.a mon.0 172.21.15.173:6789/0 358 : cluster [WRN] Health check failed: 17 slow ops, oldest one blocked for 599 sec (SLOW_OPS)" in cluster log

2366068

FAIL

master

smithi

smithi015 smithi067 smithi203

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}

"2018-04-08 15:38:54.201442 mon.a mon.0 172.21.15.15:6789/0 212 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 99 sec (SLOW_OPS)" in cluster log

2366069

FAIL

master

smithi

centos

smithi017 smithi104 smithi200

multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}

"2018-04-08 16:14:56.395731 mon.b mon.0 172.21.15.104:6789/0 186 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366070

FAIL

master

smithi

smithi008 smithi033 smithi036

multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}

"2018-04-08 15:47:02.573768 mon.a mon.0 172.21.15.8:6789/0 263 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 746 sec (SLOW_OPS)" in cluster log

2366071

FAIL

master

smithi

smithi028 smithi078 smithi082

multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}

"2018-04-08 15:55:57.690021 mon.b mon.0 172.21.15.28:6789/0 177 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366072

PASS

master

smithi

smithi004 smithi058 smithi166

multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}

2366073

FAIL

master

smithi

smithi037 smithi149 smithi154

multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}

"2018-04-08 15:45:53.960024 mon.a mon.0 172.21.15.37:6789/0 327 : cluster [WRN] Health check failed: 12 slow ops, oldest one blocked for 412 sec (SLOW_OPS)" in cluster log

2366074

FAIL

master

smithi

smithi041 smithi080 smithi111

"2018-04-08 16:21:19.845402 mon.a mon.0 172.21.15.41:6789/0 170 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 84 sec (SLOW_OPS)" in cluster log

2366075

PASS

master

smithi

smithi063 smithi165 smithi188

2366076

FAIL

master

smithi

smithi026 smithi095 smithi177

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}

"2018-04-08 15:38:28.969890 mon.b mon.0 172.21.15.26:6789/0 139 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366077

PASS

master

smithi

smithi142 smithi157 smithi188

multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}

2366078

FAIL

master

smithi

smithi117 smithi118 smithi195

"2018-04-08 15:43:54.759636 mds.a mds.1 172.21.15.118:6817/2690629090 1 : cluster [ERR] replayed stray Session close event for client.4656 172.21.15.195:0/3539743377 from time 2018-04-08 15:43:14.948329, ignoring" in cluster log

2366079

FAIL

master

smithi

smithi044 smithi087 smithi148

"2018-04-08 16:07:27.418961 mon.b mon.0 172.21.15.44:6789/0 196 : cluster [WRN] Health check failed: 26 slow ops, oldest one blocked for 1297 sec (SLOW_OPS)" in cluster log

2366080

PASS

master

smithi

smithi146 smithi160 smithi185

2366081

FAIL

master

smithi

smithi056 smithi084 smithi199

"2018-04-08 15:50:53.546838 mon.a mon.0 172.21.15.56:6789/0 195 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 84 sec (SLOW_OPS)" in cluster log

2366082

FAIL

master

smithi

smithi067 smithi076 smithi203

"2018-04-08 15:56:56.000880 mon.a mon.0 172.21.15.67:6789/0 155 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366083

PASS

master

smithi

smithi139 smithi171 smithi205

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}

2366084

PASS

master

smithi

centos

smithi064 smithi070 smithi145

multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}

2366085

FAIL

master

smithi

smithi055 smithi189 smithi192

"2018-04-08 16:28:58.388464 mon.b mon.0 172.21.15.55:6789/0 176 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366086

FAIL

master

smithi

smithi003 smithi024 smithi202

"2018-04-08 15:55:46.236704 mon.a mon.0 172.21.15.24:6789/0 242 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366087

FAIL

master

smithi

smithi105 smithi136 smithi168

"2018-04-08 15:51:49.379889 mon.b mon.0 172.21.15.105:6789/0 171 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366088

FAIL

master

smithi

smithi011 smithi031 smithi079

"2018-04-08 15:53:21.834845 mon.b mon.0 172.21.15.11:6789/0 145 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366089

FAIL

master

smithi

smithi088 smithi190 smithi198

"2018-04-08 16:03:18.619245 mon.b mon.0 172.21.15.190:6789/0 167 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366090

PASS

master

smithi

smithi015 smithi083 smithi175

2366091

PASS

master

smithi

smithi146 smithi160 smithi185

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}

2366092

FAIL

master

smithi

smithi026 smithi095 smithi197

multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}

"2018-04-08 15:56:04.489545 mon.a mon.0 172.21.15.95:6789/0 249 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366093

FAIL

master

smithi

smithi015 smithi073 smithi083

"2018-04-08 16:10:06.444782 mds.h mds.1 172.21.15.15:6817/3965153490 1 : cluster [ERR] replayed stray Session close event for client.5417 172.21.15.83:0/3375661632 from time 2018-04-08 16:04:31.544135, ignoring" in cluster log

2366094

PASS

master

smithi

smithi158 smithi181 smithi204

2366095

FAIL

master

smithi

smithi057 smithi129 smithi170

"2018-04-08 16:03:38.181291 mon.a mon.0 172.21.15.129:6789/0 333 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 138 sec (SLOW_OPS)" in cluster log

2366096

PASS

master

smithi

smithi066 smithi072 smithi085

2366097

FAIL

master

smithi

smithi037 smithi067 smithi076

"2018-04-08 16:47:05.420446 mon.a mon.0 172.21.15.37:6789/0 339 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 50 sec (SLOW_OPS)" in cluster log

2366098

FAIL

master

smithi

smithi037 smithi149 smithi182

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}

"2018-04-08 16:04:30.276288 mon.a mon.0 172.21.15.37:6789/0 229 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366099

FAIL

master

smithi

centos

smithi058 smithi098 smithi201

multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}

"2018-04-08 16:43:38.411027 mon.b mon.0 172.21.15.58:6789/0 183 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366100

FAIL

master

smithi

smithi140 smithi172 smithi178

"2018-04-08 17:05:45.614526 mon.b mon.0 172.21.15.140:6789/0 242 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 743 sec (SLOW_OPS)" in cluster log

2366101

PASS

master

smithi

smithi142 smithi157 smithi188

2366102

FAIL

master

smithi

smithi084 smithi191 smithi199

"2018-04-08 16:10:10.404532 mon.b mon.0 172.21.15.191:6789/0 149 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366103

FAIL

master

smithi

smithi049 smithi089 smithi166

"2018-04-08 16:20:55.085341 mon.a mon.0 172.21.15.49:6789/0 205 : cluster [WRN] Health check failed: 12 slow ops, oldest one blocked for 518 sec (SLOW_OPS)" in cluster log

2366104

FAIL

master

smithi

smithi013 smithi018 smithi144

"2018-04-08 16:01:06.009550 mon.b mon.0 172.21.15.13:6789/0 243 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366105

FAIL

master

smithi

smithi071 smithi094 smithi110

"2018-04-08 16:36:59.223946 mon.b mon.0 172.21.15.71:6789/0 270 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366106

FAIL

master

smithi

smithi007 smithi174 smithi196

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}

"2018-04-08 16:15:39.449309 mon.a mon.0 172.21.15.174:6789/0 175 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 74 sec (SLOW_OPS)" in cluster log

2366107

FAIL

master

smithi

smithi146 smithi160 smithi185

multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}

"2018-04-08 17:45:19.803185 mon.a mon.0 172.21.15.146:6789/0 263 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 738 sec (SLOW_OPS)" in cluster log

2366108

FAIL

master

smithi

smithi066 smithi085 smithi093

"2018-04-08 16:36:39.052825 mds.c mds.1 172.21.15.66:6817/620769405 1 : cluster [ERR] replayed stray Session close event for client.4597 172.21.15.93:0/2094354729 from time 2018-04-08 16:36:00.134805, ignoring" in cluster log

2366109

FAIL

master

smithi

smithi041 smithi049 smithi089

"2018-04-08 17:27:31.242132 mon.a mon.0 172.21.15.49:6789/0 215 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 863 sec (SLOW_OPS)" in cluster log

2366110

FAIL

master

smithi

smithi140 smithi172 smithi178

"2018-04-08 16:05:41.465331 mon.b mon.0 172.21.15.140:6789/0 135 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366111

FAIL

master

smithi

smithi048 smithi109 smithi203

"2018-04-08 16:27:50.079676 mon.a mon.0 172.21.15.109:6789/0 202 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366112

FAIL

master

smithi

smithi071 smithi094 smithi134

"2018-04-08 17:15:33.333975 mon.a mon.0 172.21.15.71:6789/0 251 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 53 sec (SLOW_OPS)" in cluster log

2366113

FAIL

master

smithi

smithi048 smithi086 smithi109

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}

"2018-04-08 16:07:52.645236 mon.b mon.0 172.21.15.86:6789/0 185 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366114

FAIL

master

smithi

centos

smithi001 smithi059 smithi176

multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}

"2018-04-08 16:51:23.391427 mon.a mon.0 172.21.15.59:6789/0 193 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366115

FAIL

master

smithi

smithi035 smithi040 smithi090

"2018-04-08 16:45:29.893152 mon.a mon.0 172.21.15.35:6789/0 332 : cluster [WRN] Health check failed: 13 slow ops, oldest one blocked for 802 sec (SLOW_OPS)" in cluster log

2366116

PASS

master

smithi

smithi013 smithi050 smithi144

2366117

FAIL

master

smithi

smithi017 smithi080 smithi096

"2018-04-08 16:40:23.161297 mon.a mon.0 172.21.15.17:6789/0 195 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 46 sec (SLOW_OPS)" in cluster log

2366118

PASS

master

smithi

smithi008 smithi036 smithi143

2366119

FAIL

master

smithi

smithi105 smithi118 smithi195

"2018-04-08 16:11:49.586697 mon.b mon.0 172.21.15.105:6789/0 170 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366120

FAIL

master

smithi

smithi007 smithi031 smithi174

"2018-04-08 16:33:08.856426 mon.a mon.0 172.21.15.31:6789/0 362 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 41 sec (SLOW_OPS)" in cluster log

2366121

FAIL

master

smithi

smithi037 smithi067 smithi076

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}

"2018-04-08 16:26:53.886538 mon.a mon.0 172.21.15.37:6789/0 242 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366122

PASS

master

smithi

smithi020 smithi141 smithi195

multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}

2366123

FAIL

master

smithi

smithi068 smithi069 smithi182

"2018-04-08 16:40:52.090950 mds.d mds.1 172.21.15.68:6817/3119363142 1 : cluster [ERR] replayed stray Session close event for client.5424 172.21.15.69:0/2772985655 from time 2018-04-08 16:35:09.077601, ignoring" in cluster log

2366124

FAIL

master

smithi

smithi015 smithi083 smithi175

"2018-04-08 17:23:11.333839 mon.a mon.0 172.21.15.15:6789/0 260 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 2182 sec (SLOW_OPS)" in cluster log

2366125

FAIL

master

smithi

smithi027 smithi029 smithi167

"2018-04-08 16:22:32.110655 mon.b mon.0 172.21.15.27:6789/0 316 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 180 sec (SLOW_OPS)" in cluster log

2366126

FAIL

master

smithi

smithi088 smithi190 smithi198

"2018-04-08 16:25:20.668520 mon.b mon.0 172.21.15.190:6789/0 135 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 208 sec (SLOW_OPS)" in cluster log

2366127

FAIL

master

smithi

smithi064 smithi070 smithi145

"2018-04-08 17:39:53.280232 mon.b mon.0 172.21.15.64:6789/0 186 : cluster [WRN] Health check failed: 15 slow ops, oldest one blocked for 807 sec (SLOW_OPS)" in cluster log

2366128

FAIL

master

smithi

smithi004 smithi018 smithi102

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}

"2018-04-08 16:23:27.702033 mon.b mon.0 172.21.15.4:6789/0 186 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366129

FAIL

master

smithi

centos

smithi019 smithi131 smithi164

multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}

"2018-04-08 16:51:29.509047 mon.a mon.0 172.21.15.131:6789/0 312 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 150 sec (SLOW_OPS)" in cluster log

2366130

PASS

master

smithi

smithi073 smithi084 smithi199

2366131

FAIL

master

smithi

smithi007 smithi031 smithi174

"2018-04-08 17:30:29.370490 mon.a mon.0 172.21.15.31:6789/0 229 : cluster [WRN] Health check failed: 11 slow ops, oldest one blocked for 422 sec (SLOW_OPS)" in cluster log

2366132

FAIL

master

smithi

smithi064 smithi070 smithi145

"2018-04-08 16:49:27.287727 mon.b mon.0 172.21.15.64:6789/0 139 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366133

FAIL

master

smithi

smithi079 smithi105 smithi118

"2018-04-08 16:32:45.918183 mon.b mon.0 172.21.15.79:6789/0 307 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366134

FAIL

master

smithi

smithi020 smithi152 smithi172

"2018-04-07 16:48:48.978343 mon.b mon.0 172.21.15.20:6789/0 132 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366135

PASS

master

smithi

smithi033 smithi149 smithi156

2366136

FAIL

master

smithi

smithi011 smithi117 smithi177

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}

"2018-04-08 16:36:25.661297 mon.b mon.0 172.21.15.11:6789/0 160 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366137

FAIL

master

smithi

smithi051 smithi173 smithi193

"2018-04-08 17:11:25.704092 mon.a mon.0 172.21.15.51:6789/0 170 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 54 sec (SLOW_OPS)" in cluster log

2366138

FAIL

master

smithi

smithi009 smithi086 smithi186

"2018-04-08 16:30:05.884801 mds.c mds.1 172.21.15.9:6817/2526861913 1 : cluster [ERR] replayed stray Session close event for client.4605 172.21.15.186:0/3806745840 from time 2018-04-08 16:29:24.165941, ignoring" in cluster log

2366139

PASS

master

smithi

smithi094 smithi108 smithi190

2366140

PASS

master

smithi

smithi011 smithi117 smithi177

2366141

FAIL

master

smithi

smithi102 smithi108 smithi135

"2018-04-08 17:04:44.462179 mon.a mon.0 172.21.15.102:6789/0 326 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 68 sec (SLOW_OPS)" in cluster log

2366142

PASS

master

smithi

smithi139 smithi171 smithi205

2366143

FAIL

master

smithi

smithi045 smithi104 smithi130

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}

"2018-04-08 16:38:20.751165 mon.b mon.0 172.21.15.45:6789/0 175 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366144

FAIL

master

smithi

centos

smithi044 smithi087 smithi196

"2018-04-08 17:00:02.010873 mon.a mon.0 172.21.15.44:6789/0 192 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 744 sec (SLOW_OPS)" in cluster log

2366145

FAIL

master

smithi

smithi111 smithi151 smithi200

"2018-04-08 16:39:39.270252 mon.b mon.0 172.21.15.111:6789/0 309 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366146

FAIL

master

smithi

smithi047 smithi169 smithi191

"2018-04-08 16:30:00.829535 mon.a mon.0 172.21.15.169:6789/0 163 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366147

PASS

master

smithi

smithi026 smithi095 smithi156

2366148

FAIL

master

smithi

smithi139 smithi171 smithi205

"2018-04-08 16:57:04.889167 mon.a mon.0 172.21.15.139:6789/0 249 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 128 sec (SLOW_OPS)" in cluster log

2366149

FAIL

master

smithi

smithi032 smithi103 smithi194

"2018-04-08 18:06:19.615996 mon.a mon.0 172.21.15.103:6789/0 343 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 125 sec (SLOW_OPS)" in cluster log

2366150

FAIL

master

smithi

smithi142 smithi157 smithi188

"2018-04-08 16:33:58.570143 mon.a mon.0 172.21.15.157:6789/0 245 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366151

PASS

master

smithi

smithi014 smithi043 smithi060

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}

2366152

FAIL

master

smithi

smithi142 smithi157 smithi188

"2018-04-08 16:52:01.161857 mon.a mon.0 172.21.15.157:6789/0 164 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 53 sec (SLOW_OPS)" in cluster log

2366153

FAIL

master

smithi

smithi041 smithi049 smithi089

"2018-04-08 16:54:27.232939 mds.a mds.1 172.21.15.49:6817/1630334313 1 : cluster [ERR] replayed stray Session close event for client.5442 172.21.15.41:0/3491683820 from time 2018-04-08 16:48:34.669328, ignoring" in cluster log

2366154

PASS

master

smithi

smithi043 smithi060 smithi186

2366155

FAIL

master

smithi

smithi045 smithi104 smithi130

"2018-04-08 16:59:13.497257 mon.b mon.0 172.21.15.45:6789/0 180 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 121 sec (SLOW_OPS)" in cluster log

2366156

FAIL

master

smithi

smithi052 smithi148 smithi159

"2018-04-08 16:34:37.480462 mon.b mon.0 172.21.15.52:6789/0 236 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366157

FAIL

master

smithi

smithi023 smithi065 smithi115

"2018-04-08 16:49:27.283125 mon.a mon.0 172.21.15.23:6789/0 216 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366158

PASS

master

smithi

smithi051 smithi173 smithi193

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}

2366159

FAIL

master

smithi

centos

smithi068 smithi069 smithi182

"2018-04-08 17:19:07.553609 mon.a mon.0 172.21.15.68:6789/0 311 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366160

FAIL

master

smithi

smithi005 smithi077 smithi081

"2018-04-08 16:47:07.120396 mon.a mon.0 172.21.15.5:6789/0 160 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366161

FAIL

master

smithi

smithi061 smithi091 smithi134

"2018-04-08 16:45:11.434806 mon.b mon.0 172.21.15.61:6789/0 304 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366162

FAIL

master

smithi

smithi018 smithi152 smithi197

"2018-04-08 16:42:58.415808 mon.b mon.0 172.21.15.152:6789/0 136 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366163

FAIL

master

smithi

smithi007 smithi031 smithi174

"2018-04-08 16:50:57.944744 mon.a mon.0 172.21.15.31:6789/0 197 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366164

FAIL

master

smithi

smithi004 smithi062 smithi138

"2018-04-08 16:45:17.221031 mon.a mon.0 172.21.15.62:6789/0 256 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366165

FAIL

master

smithi

smithi052 smithi148 smithi159

"2018-04-08 16:57:38.656645 mon.b mon.0 172.21.15.52:6789/0 277 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366166

FAIL

master

smithi

smithi071 smithi094 smithi110

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}

"2018-04-08 16:55:33.082644 mon.b mon.0 172.21.15.71:6789/0 167 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366167

PASS

master

smithi

smithi033 smithi149 smithi166

2366168

FAIL

master

smithi

smithi004 smithi062 smithi138

"2018-04-08 17:06:11.000756 mds.b mds.1 172.21.15.138:6817/3584361485 1 : cluster [ERR] replayed stray Session close event for client.4614 172.21.15.4:0/4144193107 from time 2018-04-08 17:05:26.904319, ignoring" in cluster log

2366169

FAIL

master

smithi

smithi038 smithi072 smithi074

"2018-04-08 17:02:57.931075 mon.a mon.0 172.21.15.38:6789/0 326 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366170

PASS

master

smithi

smithi043 smithi060 smithi179

2366171

FAIL

master

smithi

smithi004 smithi059 smithi135

"2018-04-08 17:41:39.793859 mon.a mon.0 172.21.15.59:6789/0 196 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366172

FAIL

master

smithi

smithi003 smithi024 smithi202

"2018-04-08 16:50:55.785320 mon.a mon.0 172.21.15.24:6789/0 180 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 469 sec (SLOW_OPS)" in cluster log

2366173

FAIL

master

smithi

smithi088 smithi190 smithi198

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}

"2018-04-08 17:06:06.286824 mon.b mon.0 172.21.15.190:6789/0 306 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366174

FAIL

master

smithi

centos

smithi139 smithi171 smithi205

"2018-04-08 18:02:54.470451 mon.a mon.0 172.21.15.139:6789/0 215 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 256 sec (SLOW_OPS)" in cluster log

2366175

FAIL

master

smithi

smithi048 smithi109 smithi203

"2018-04-08 17:01:25.235139 mon.a mon.0 172.21.15.109:6789/0 202 : cluster [WRN] Health check failed: 11 slow ops, oldest one blocked for 828 sec (SLOW_OPS)" in cluster log

2366176

FAIL

master

smithi

smithi036 smithi073 smithi199

"2018-04-08 17:19:52.821009 mon.b mon.0 172.21.15.36:6789/0 143 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366177

FAIL

master

smithi

smithi088 smithi156 smithi198

"2018-04-08 17:24:59.779633 mon.b mon.0 172.21.15.156:6789/0 180 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366178

FAIL

master

smithi

smithi066 smithi088 smithi198

"2018-04-08 18:17:22.264037 mon.b mon.0 172.21.15.88:6789/0 132 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 107 sec (SLOW_OPS)" in cluster log

2366179

FAIL

master

smithi

smithi009 smithi086 smithi186

"2018-04-08 16:57:55.442938 mon.b mon.0 172.21.15.9:6789/0 301 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366180

PASS

master

smithi

smithi051 smithi173 smithi193

2366181

FAIL

master

smithi

smithi008 smithi036 smithi143

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}

"2018-04-08 17:00:44.232738 mon.a mon.0 172.21.15.8:6789/0 174 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366182

FAIL

master

smithi

smithi064 smithi070 smithi145

"2018-04-08 17:08:31.079891 mon.b mon.0 172.21.15.64:6789/0 136 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366183

FAIL

master

smithi

smithi079 smithi105 smithi118

"2018-04-08 17:12:22.679555 mds.h mds.1 172.21.15.79:6817/3216780690 1 : cluster [ERR] replayed stray Session close event for client.15422 172.21.15.105:0/2640058396 from time 2018-04-08 17:06:14.017729, ignoring" in cluster log

2366184

PASS

master

smithi

smithi018 smithi152 smithi197

2366185

FAIL

master

smithi

smithi019 smithi026 smithi095

"2018-04-08 17:41:29.346891 mon.b mon.0 172.21.15.26:6789/0 166 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366186

PASS

master

smithi

smithi056 smithi136 smithi168

2366187

FAIL

master

smithi

smithi028 smithi078 smithi082

"2018-04-08 17:01:34.080390 mon.b mon.0 172.21.15.28:6789/0 174 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366188

PASS

master

smithi

smithi146 smithi160 smithi185

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}

2366189

FAIL

master

smithi

centos

smithi158 smithi181 smithi204

"2018-04-08 17:50:08.434134 mon.a mon.0 172.21.15.158:6789/0 217 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366190

FAIL

master

smithi

smithi033 smithi149 smithi166

"2018-04-08 17:00:08.394374 mon.a mon.0 172.21.15.33:6789/0 164 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366191

FAIL

master

smithi

smithi027 smithi029 smithi167

"2018-04-08 17:13:05.070747 mon.b mon.0 172.21.15.27:6789/0 201 : cluster [WRN] Health check failed: 14 slow ops, oldest one blocked for 695 sec (SLOW_OPS)" in cluster log

2366192

FAIL

master

smithi

smithi026 smithi095 smithi156

"2018-04-08 17:05:21.246245 mon.b mon.0 172.21.15.26:6789/0 240 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366193

FAIL

master

smithi

smithi017 smithi080 smithi187

"2018-04-08 17:11:55.194872 mon.a mon.0 172.21.15.17:6789/0 342 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 748 sec (SLOW_OPS)" in cluster log

2366194

FAIL

master

smithi

smithi058 smithi098 smithi201

"2018-04-08 17:29:29.982264 mon.b mon.0 172.21.15.58:6789/0 141 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366195

PASS

master

smithi

smithi024 smithi057 smithi129

2366196

PASS

master

smithi

smithi024 smithi057 smithi129

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}

2366197

FAIL

master

smithi

smithi058 smithi098 smithi201

"2018-04-08 17:10:54.490632 mon.b mon.0 172.21.15.58:6789/0 139 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 47 sec (SLOW_OPS)" in cluster log

2366198

FAIL

master

smithi

smithi036 smithi065 smithi110

"2018-04-08 18:08:20.852868 mds.b mds.1 172.21.15.36:6816/1158704808 1 : cluster [ERR] replayed stray Session close event for client.4671 172.21.15.65:0/2193581710 from time 2018-04-08 18:07:34.828348, ignoring" in cluster log

2366199

FAIL

master

smithi

smithi023 smithi148 smithi174

"2018-04-08 17:56:05.957143 mon.a mon.0 172.21.15.23:6789/0 188 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366200

PASS

master

smithi

smithi073 smithi170 smithi199

2366201

FAIL

master

smithi

smithi011 smithi117 smithi177

"2018-04-08 17:46:01.386288 mon.b mon.0 172.21.15.11:6789/0 298 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366202

PASS

master

smithi

smithi007 smithi051 smithi193

2366203

FAIL

master

smithi

smithi050 smithi084 smithi176

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}

"2018-04-08 18:19:02.099591 mon.a mon.0 172.21.15.50:6789/0 180 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 38 sec (SLOW_OPS)" in cluster log

2366204

FAIL

master

smithi

centos

smithi006 smithi016 smithi163

"2018-04-08 17:30:37.300005 mon.a mon.0 172.21.15.6:6789/0 304 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366205

FAIL

master

smithi

smithi027 smithi072 smithi085

"2018-04-08 18:27:12.219300 mon.b mon.0 172.21.15.72:6789/0 180 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 749 sec (SLOW_OPS)" in cluster log

2366206

PASS

master

smithi

smithi066 smithi085 smithi093

2366207

FAIL

master

smithi

smithi059 smithi084 smithi176

"2018-04-08 17:22:27.753438 mon.a mon.0 172.21.15.59:6789/0 197 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366208

FAIL

master

smithi

smithi048 smithi109 smithi203

"2018-04-08 17:59:37.928025 mon.a mon.0 172.21.15.109:6789/0 267 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 828 sec (SLOW_OPS)" in cluster log

2366209

PASS

master

smithi

smithi056 smithi136 smithi168

2366210

FAIL

master

smithi

smithi056 smithi136 smithi168

multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}

"2018-04-08 17:37:34.331168 mon.b mon.0 172.21.15.56:6789/0 212 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366211

FAIL

master

smithi

smithi052 smithi148 smithi159

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}

"2018-04-08 17:16:01.484666 mon.b mon.0 172.21.15.52:6789/0 143 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 35 sec (SLOW_OPS)" in cluster log

2366212

FAIL

master

smithi

smithi009 smithi050 smithi144

multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}

"2018-04-08 17:29:16.306210 mon.a mon.0 172.21.15.50:6789/0 159 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 744 sec (SLOW_OPS)" in cluster log

2366213

FAIL

master

smithi

smithi011 smithi032 smithi037

multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml}

"2018-04-08 18:34:19.668738 mds.b mds.1 172.21.15.37:6817/3612390816 1 : cluster [ERR] replayed stray Session close event for client.5324 172.21.15.32:0/4000904984 from time 2018-04-08 18:29:27.046368, ignoring" in cluster log

2366214

FAIL

master

smithi

smithi001 smithi133 smithi188

multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}

"2018-04-08 18:29:42.788683 mon.a mon.0 172.21.15.133:6789/0 255 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366215

FAIL

master

smithi

smithi007 smithi051 smithi193

multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}

"2018-04-08 18:38:19.946662 mon.a mon.0 172.21.15.51:6789/0 336 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 112 sec (SLOW_OPS)" in cluster log

2366216

FAIL

master

smithi

smithi031 smithi077 smithi177

multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}

"2018-04-08 18:37:11.154389 mon.b mon.0 172.21.15.31:6789/0 140 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366217

FAIL

master

smithi

smithi015 smithi050 smithi176

multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}

"2018-04-08 18:45:13.095764 mon.a mon.0 172.21.15.15:6789/0 217 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 469 sec (SLOW_OPS)" in cluster log

2366218

FAIL

master

smithi

smithi057 smithi118 smithi129

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}

"2018-04-08 18:29:33.897382 mon.b mon.0 172.21.15.118:6789/0 218 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 93 sec (SLOW_OPS)" in cluster log

2366219

FAIL

master

smithi

centos

smithi103 smithi145 smithi201

multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}

"2018-04-08 19:13:36.286870 mon.a mon.0 172.21.15.103:6789/0 194 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366220

FAIL

master

smithi

smithi101 smithi166 smithi202

multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}

"2018-04-08 18:42:59.875435 mon.a mon.0 172.21.15.101:6789/0 163 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366221

FAIL

master

smithi

smithi014 smithi102 smithi196

multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}

"2018-04-08 18:34:21.648466 mon.b mon.0 172.21.15.14:6789/0 175 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366222

FAIL

master

smithi

smithi117 smithi139 smithi205

multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}

"2018-04-08 18:33:17.931712 mon.a mon.0 172.21.15.117:6789/0 166 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366223

FAIL

master

smithi

smithi083 smithi108 smithi190

multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}

"2018-04-08 18:44:40.039492 mon.b mon.0 172.21.15.83:6789/0 169 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366224

PASS

master

smithi

smithi004 smithi044 smithi087

2366225

PASS

master

smithi

smithi006 smithi048 smithi076

2366226

PASS

master

smithi

smithi002 smithi081 smithi132

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}

2366227

FAIL

master

smithi

smithi022 smithi039 smithi181

multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}

"2018-04-08 18:49:11.547989 mon.b mon.0 172.21.15.39:6789/0 141 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 743 sec (SLOW_OPS)" in cluster log

2366228

FAIL

master

smithi

smithi062 smithi158 smithi190

"2018-04-08 19:08:08.361052 mds.b mds.1 172.21.15.62:6817/1139219190 1 : cluster [ERR] replayed stray Session close event for client.4658 172.21.15.158:0/940540229 from time 2018-04-08 19:07:25.426511, ignoring" in cluster log

2366229

FAIL

master

smithi

smithi024 smithi057 smithi129

"2018-04-08 19:35:12.231926 mon.a mon.0 172.21.15.24:6789/0 217 : cluster [WRN] Health check failed: 19 slow ops, oldest one blocked for 1657 sec (SLOW_OPS)" in cluster log

2366230

FAIL

master

smithi

smithi036 smithi065 smithi110

"2018-04-08 18:39:27.464699 mon.b mon.0 172.21.15.36:6789/0 136 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 157 sec (SLOW_OPS)" in cluster log

2366231

FAIL

master

smithi

smithi071 smithi074 smithi171

"2018-04-08 18:34:21.270868 mon.b mon.0 172.21.15.71:6789/0 177 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 35 sec (SLOW_OPS)" in cluster log

2366232

FAIL

master

smithi

smithi096 smithi175 smithi187

"2018-04-08 19:08:10.686901 mon.a mon.0 172.21.15.175:6789/0 170 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 58 sec (SLOW_OPS)" in cluster log

2366233

FAIL

master

smithi

smithi074 smithi136 smithi168

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}

"2018-04-08 18:54:21.118117 mon.b mon.0 172.21.15.74:6789/0 207 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366234

FAIL

master

smithi

centos

smithi033 smithi079 smithi149

multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}

reached maximum tries (50) after waiting for 300 seconds

2366235

FAIL

master

smithi

smithi080 smithi091 smithi115

"2018-04-08 18:55:21.222999 mon.a mon.0 172.21.15.80:6789/0 220 : cluster [WRN] Health check failed: 15 slow ops, oldest one blocked for 848 sec (SLOW_OPS)" in cluster log

2366236

FAIL

master

smithi

smithi049 smithi089 smithi163

"2018-04-08 18:55:04.057399 mon.a mon.0 172.21.15.49:6789/0 161 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 1137 sec (SLOW_OPS)" in cluster log

2366237

FAIL

master

smithi

smithi061 smithi109 smithi152

"2018-04-08 18:39:24.383695 mon.b mon.0 172.21.15.61:6789/0 174 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 123 sec (SLOW_OPS)" in cluster log

2366238

FAIL

master

smithi

smithi008 smithi143 smithi169

"2018-04-08 19:07:30.882599 mon.b mon.0 172.21.15.8:6789/0 151 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366239

PASS

master

smithi

smithi061 smithi142 smithi157

2366240

PASS

master

smithi

smithi094 smithi105 smithi203

2366241

FAIL

master

smithi

smithi023 smithi148 smithi174

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}

"2018-04-08 18:41:13.404763 mon.a mon.0 172.21.15.23:6789/0 143 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366242

FAIL

master

smithi

smithi073 smithi170 smithi199

multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}

"2018-04-08 19:04:45.045149 mon.a mon.0 172.21.15.73:6789/0 159 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 747 sec (SLOW_OPS)" in cluster log

2366243

FAIL

master

smithi

smithi028 smithi151 smithi200

"2018-04-08 18:56:45.899453 mds.e mds.1 172.21.15.28:6817/3341173500 1 : cluster [ERR] replayed stray Session close event for client.5443 172.21.15.151:0/2619414993 from time 2018-04-08 18:50:20.786424, ignoring" in cluster log

2366244

FAIL

master

smithi

smithi084 smithi160 smithi185

"2018-04-08 18:39:45.776761 mon.a mon.0 172.21.15.84:6789/0 156 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366245

FAIL

master

smithi

smithi068 smithi069 smithi160

"2018-04-08 19:53:38.578129 mon.a mon.0 172.21.15.68:6789/0 195 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366246

FAIL

master

smithi

smithi052 smithi135 smithi204

"2018-04-08 19:07:34.155085 mon.b mon.0 172.21.15.135:6789/0 136 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366247

FAIL

master

smithi

smithi166 smithi173 smithi202

"2018-04-08 19:43:10.068304 mon.b mon.0 172.21.15.173:6789/0 322 : cluster [WRN] Health check failed: 14 slow ops, oldest one blocked for 684 sec (SLOW_OPS)" in cluster log

2366248

FAIL

master

smithi

smithi152 smithi159 smithi191

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}

"2018-04-08 19:54:22.069554 mon.b mon.0 172.21.15.152:6789/0 182 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 80 sec (SLOW_OPS)" in cluster log

2366249

FAIL

master

smithi

centos

smithi131 smithi140 smithi164

multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}

"2018-04-08 19:24:48.372286 mon.a mon.0 172.21.15.140:6789/0 214 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 82 sec (SLOW_OPS)" in cluster log

2366250

FAIL

master

smithi

smithi023 smithi148 smithi174

"2018-04-08 19:00:50.814198 mon.a mon.0 172.21.15.23:6789/0 267 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366251

PASS

master

smithi

smithi083 smithi108 smithi138

2366252

FAIL

master

smithi

smithi004 smithi044 smithi205

"2018-04-08 19:53:24.024451 mon.b mon.0 172.21.15.4:6789/0 154 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366253

FAIL

master

smithi

smithi035 smithi055 smithi189

"2018-04-08 18:51:54.984121 mon.a mon.0 172.21.15.35:6789/0 203 : cluster [WRN] Health check failed: 12 slow ops, oldest one blocked for 521 sec (SLOW_OPS)" in cluster log

2366254

PASS

master

smithi

smithi011 smithi071 smithi171

2366255

FAIL

master

smithi

smithi068 smithi069 smithi159

"2018-04-08 19:16:04.051910 mon.a mon.0 172.21.15.68:6789/0 303 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366256

FAIL

master

smithi

smithi077 smithi078 smithi098

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}

"2018-04-08 19:16:56.468165 mon.a mon.0 172.21.15.77:6789/0 146 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366257

FAIL

master

smithi

smithi088 smithi118 smithi198

multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}

"2018-04-08 19:00:01.052954 mon.b mon.0 172.21.15.118:6789/0 232 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 742 sec (SLOW_OPS)" in cluster log

2366258

FAIL

master

smithi

smithi005 smithi082 smithi162

"2018-04-08 18:47:01.675929 mds.c mds.1 172.21.15.162:6816/3766530365 1 : cluster [ERR] replayed stray Session close event for client.4637 172.21.15.5:0/3096059129 from time 2018-04-08 18:46:22.719947, ignoring" in cluster log

2366259

FAIL

master

smithi

smithi040 smithi066 smithi090

"2018-04-08 19:13:01.220623 mon.a mon.0 172.21.15.40:6789/0 242 : cluster [WRN] Health check failed: 18 slow ops, oldest one blocked for 1439 sec (SLOW_OPS)" in cluster log

2366260

FAIL

master

smithi

smithi032 smithi037 smithi098

"2018-04-08 19:56:35.655375 mon.a mon.0 172.21.15.37:6789/0 245 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366261

FAIL

master

smithi

smithi024 smithi057 smithi129

"2018-04-08 18:48:02.454502 mon.a mon.0 172.21.15.24:6789/0 331 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 89 sec (SLOW_OPS)" in cluster log

2366262

FAIL

master

smithi

smithi019 smithi026 smithi095

"2018-04-08 19:22:08.189947 mon.b mon.0 172.21.15.26:6789/0 246 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366263

FAIL

master

smithi

smithi006 smithi048 smithi149

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}

"2018-04-08 19:52:04.813079 mon.a mon.0 172.21.15.6:6789/0 206 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366264

PASS

master

smithi

centos

smithi032 smithi037 smithi117

multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}

2366265

FAIL

master

smithi

smithi047 smithi152 smithi191

"2018-04-08 18:56:36.260056 mon.b mon.0 172.21.15.47:6789/0 175 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366266

FAIL

master

smithi

smithi004 smithi044 smithi087

"2018-04-08 18:59:32.121393 mon.b mon.0 172.21.15.4:6789/0 141 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366267

FAIL

master

smithi

smithi077 smithi078 smithi098

"2018-04-08 18:58:05.783248 mon.a mon.0 172.21.15.77:6789/0 327 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 82 sec (SLOW_OPS)" in cluster log

2366268

FAIL

master

smithi

smithi064 smithi070 smithi182

"2018-04-08 19:20:01.528482 mon.b mon.0 172.21.15.64:6789/0 145 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366269

FAIL

master

smithi

smithi086 smithi139 smithi205

Command failed (workunit test suites/pjd.sh) on smithi205 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=master TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/pjd.sh'

2366270

FAIL

master

smithi

smithi068 smithi069 smithi159

"2018-04-08 19:34:23.806175 mon.a mon.0 172.21.15.68:6789/0 253 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366271

FAIL

master

smithi

smithi005 smithi082 smithi140

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}

"2018-04-08 19:53:34.623515 mon.b mon.0 172.21.15.5:6789/0 127 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366272

FAIL

master

smithi

smithi006 smithi048 smithi076

multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}

"2018-04-08 18:55:26.117153 mon.a mon.0 172.21.15.6:6789/0 162 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366273

FAIL

master

smithi

smithi011 smithi071 smithi171

"2018-04-08 19:05:35.933707 mds.i mds.1 172.21.15.11:6816/1104901821 1 : cluster [ERR] replayed stray Session close event for client.5420 172.21.15.171:0/2676684985 from time 2018-04-08 19:00:12.678593, ignoring" in cluster log

2366274

FAIL

master

smithi

smithi094 smithi105 smithi203

"2018-04-08 19:18:04.901104 mon.a mon.0 172.21.15.94:6789/0 172 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366275

FAIL

master

smithi

smithi031 smithi109 smithi177

"2018-04-08 18:57:38.029832 mon.b mon.0 172.21.15.31:6789/0 313 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366276

FAIL

master

smithi

smithi019 smithi026 smithi095

"2018-04-08 18:59:18.252731 mon.b mon.0 172.21.15.26:6789/0 243 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366277

FAIL

master

smithi

smithi041 smithi066 smithi196

"2018-04-08 19:40:26.944172 mon.a mon.0 172.21.15.41:6789/0 198 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366278

FAIL

master

smithi

smithi131 smithi148 smithi164

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}

"2018-04-08 19:51:56.470563 mon.a mon.0 172.21.15.148:6789/0 223 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366279

PASS

master

smithi

centos

smithi061 smithi142 smithi157

multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}

2366280

PASS

master

smithi

smithi058 smithi086 smithi156

2366281

FAIL

master

smithi

smithi050 smithi132 smithi176

"2018-04-08 19:13:48.601193 mon.b mon.0 172.21.15.50:6789/0 176 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366282

FAIL

master

smithi

smithi064 smithi070 smithi182

"2018-04-08 19:01:47.214017 mon.b mon.0 172.21.15.64:6789/0 154 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366283

FAIL

master

smithi

smithi041 smithi130 smithi146

"2018-04-08 19:22:14.535966 mon.a mon.0 172.21.15.41:6789/0 198 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 542 sec (SLOW_OPS)" in cluster log

2366284

FAIL

master

smithi

smithi052 smithi135 smithi204

"2018-04-08 19:30:16.802857 mon.b mon.0 172.21.15.135:6789/0 139 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 152 sec (SLOW_OPS)" in cluster log

2366285

FAIL

master

smithi

smithi017 smithi045 smithi104

"2018-04-08 19:19:59.502851 mon.b mon.0 172.21.15.17:6789/0 449 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366286

FAIL

master

smithi

smithi186 smithi194 smithi205

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}

"2018-04-08 19:13:11.985856 mon.b mon.0 172.21.15.194:6789/0 235 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366287

FAIL

master

smithi

smithi059 smithi088 smithi198

"2018-04-08 19:46:04.795566 mon.b mon.0 172.21.15.59:6789/0 143 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366288

FAIL

master

smithi

smithi008 smithi036 smithi169

"2018-04-08 20:07:00.805094 mds.a mds.1 172.21.15.8:6816/3201724840 1 : cluster [ERR] replayed stray Session close event for client.4673 172.21.15.169:0/2538718308 from time 2018-04-08 20:06:16.788272, ignoring" in cluster log

2366289

FAIL

master

smithi

smithi043 smithi060 smithi081

"2018-04-08 19:35:36.431222 mon.a mon.0 172.21.15.60:6789/0 352 : cluster [WRN] Health check failed: 19 slow ops, oldest one blocked for 1324 sec (SLOW_OPS)" in cluster log

2366290

FAIL

master

smithi

smithi028 smithi151 smithi200

"2018-04-08 19:36:23.782478 mon.b mon.0 172.21.15.28:6789/0 137 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366291

FAIL

master

smithi

smithi175 smithi178 smithi179

"2018-04-08 19:35:09.483598 mon.a mon.0 172.21.15.175:6789/0 322 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 72 sec (SLOW_OPS)" in cluster log

2366292

FAIL

master

smithi

smithi002 smithi018 smithi172

"2018-04-08 19:25:39.776125 mon.a mon.0 172.21.15.2:6789/0 266 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366293

FAIL

master

smithi

smithi016 smithi056 smithi205

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}

"2018-04-08 19:32:52.136695 mon.b mon.0 172.21.15.16:6789/0 160 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366294

FAIL

master

smithi

centos

smithi112 smithi139 smithi192

"2018-04-08 19:50:35.583946 mon.b mon.0 172.21.15.112:6789/0 325 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 40 sec (SLOW_OPS)" in cluster log

2366295

FAIL

master

smithi

smithi033 smithi105 smithi172

"2018-04-08 20:33:42.415045 mon.b mon.0 172.21.15.33:6789/0 166 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366296

FAIL

master

smithi

smithi035 smithi055 smithi063

"2018-04-08 19:12:29.024838 mon.a mon.0 172.21.15.35:6789/0 158 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366297

FAIL

master

smithi

smithi028 smithi151 smithi200

"2018-04-08 19:15:42.335641 mon.b mon.0 172.21.15.28:6789/0 182 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 57 sec (SLOW_OPS)" in cluster log

2366298

FAIL

master

smithi

smithi036 smithi065 smithi110

"2018-04-08 19:29:47.062653 mon.b mon.0 172.21.15.36:6789/0 136 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366299

FAIL

master

smithi

smithi016 smithi067 smithi189

"2018-04-08 19:14:46.784706 mon.a mon.0 172.21.15.67:6789/0 193 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366300

PASS

master

smithi

smithi005 smithi082 smithi162

2366301

FAIL

master

smithi

smithi015 smithi106 smithi113

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}

"2018-04-08 19:16:28.516497 mon.a mon.0 172.21.15.15:6789/0 215 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366302

FAIL

master

smithi

smithi003 smithi029 smithi167

"2018-04-08 20:18:51.037087 mon.b mon.0 172.21.15.3:6789/0 137 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 54 sec (SLOW_OPS)" in cluster log

2366303

FAIL

master

smithi

smithi062 smithi158 smithi190

"2018-04-08 19:47:18.853880 mds.c mds.1 172.21.15.190:6817/1596179823 1 : cluster [ERR] replayed stray Session close event for client.5502 172.21.15.158:0/3794388839 from time 2018-04-08 19:42:05.093636, ignoring" in cluster log

2366304

DEAD

master

smithi

Timeout exceeded. command: /usr/bin/ipmitool args: ['/usr/bin/ipmitool', '-H', 'smithi048.ipmi.sepia.ceph.com', '-I', 'lanplus', '-U', 'inktank', '-P', 'ApGNXcA7', 'power', 'off'] buffer (last 100 chars): '' before (last 100 chars): '' after: match: None match_index: None exitstatus: None flag_eof: False pid: 26045 child_fd: 8 closed: False timeout: 30 delimiter: logfile: None logfile_read: None logfile_send: None maxread: 2000 ignorecase: False searchwindowsize: None delaybeforesend: 0.05 delayafterclose: 0.1 delayafterterminate: 0.1 searcher: searcher_re: 0: re.compile("Chassis Power Control: Down/Off") 1: EOF

2366305

FAIL

master

smithi

smithi013 smithi070 smithi104

"2018-04-08 19:57:28.912923 mon.b mon.0 172.21.15.70:6789/0 307 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366306

PASS

master

smithi

smithi016 smithi056 smithi076

2366307

FAIL

master

smithi

smithi080 smithi091 smithi115

"2018-04-08 19:51:11.576446 mon.a mon.0 172.21.15.80:6789/0 344 : cluster [WRN] Health check failed: 15 slow ops, oldest one blocked for 633 sec (SLOW_OPS)" in cluster log

2366308

FAIL

master

smithi

smithi027 smithi072 smithi085

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}

"2018-04-08 19:16:27.537691 mon.b mon.0 172.21.15.72:6789/0 165 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366309

FAIL

master

smithi

centos

smithi141 smithi186 smithi194

"2018-04-08 19:58:21.228911 mon.b mon.0 172.21.15.141:6789/0 192 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366310

FAIL

master

smithi

smithi022 smithi039 smithi181

"2018-04-08 19:49:36.898663 mon.b mon.0 172.21.15.39:6789/0 247 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 744 sec (SLOW_OPS)" in cluster log

2366311

FAIL

master

smithi

smithi019 smithi028 smithi077

"2018-04-08 21:05:00.205564 mon.b mon.0 172.21.15.19:6789/0 303 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366312

PASS

master

smithi

smithi011 smithi071 smithi171

2366313

FAIL

master

smithi

smithi035 smithi082 smithi140

"2018-04-08 20:28:55.552892 mon.a mon.0 172.21.15.82:6789/0 201 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 734 sec (SLOW_OPS)" in cluster log

2366314

PASS

master

smithi

smithi027 smithi077 smithi087

2366315

PASS

master

smithi

smithi002 smithi020 smithi174

2366316

PASS

master

smithi

smithi091 smithi108 smithi196

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}

2366317

PASS

master

smithi

smithi073 smithi170 smithi199

2366318

FAIL

master

smithi

smithi067 smithi101 smithi189

"2018-04-08 19:36:44.794988 mds.a mds.1 172.21.15.67:6817/196036152 1 : cluster [ERR] replayed stray Session close event for client.14629 172.21.15.101:0/840161021 from time 2018-04-08 19:36:00.375806, ignoring" in cluster log

2366319

FAIL

master

smithi

smithi144 smithi154 smithi165

"2018-04-08 19:43:06.296687 mon.a mon.0 172.21.15.154:6789/0 219 : cluster [WRN] Health check failed: 16 slow ops, oldest one blocked for 1032 sec (SLOW_OPS)" in cluster log

2366320

FAIL

master

smithi

smithi049 smithi089 smithi163

"2018-04-08 19:23:19.767709 mon.a mon.0 172.21.15.49:6789/0 260 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366321

FAIL

master

smithi

smithi083 smithi093 smithi143

"2018-04-08 20:04:14.950731 mon.a mon.0 172.21.15.83:6789/0 326 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366322

FAIL

master

smithi

smithi009 smithi013 smithi111

"2018-04-08 19:28:31.192464 mon.b mon.0 172.21.15.9:6789/0 147 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366323

FAIL

master

smithi

smithi027 smithi077 smithi098

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}

"2018-04-08 19:39:03.674789 mon.b mon.0 172.21.15.27:6789/0 317 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 140 sec (SLOW_OPS)" in cluster log

2366324

FAIL

master

smithi

centos

smithi015 smithi106 smithi113

"2018-04-08 19:49:24.823288 mon.a mon.0 172.21.15.15:6789/0 190 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366325

FAIL

master

smithi

smithi072 smithi078 smithi085

"2018-04-08 19:50:23.092246 mon.b mon.0 172.21.15.72:6789/0 180 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366326

PASS

master

smithi

smithi108 smithi134 smithi138

2366327

FAIL

master

smithi

smithi005 smithi082 smithi162

"2018-04-08 19:34:04.532595 mon.a mon.0 172.21.15.82:6789/0 194 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366328

FAIL

master

smithi

smithi038 smithi083 smithi093

"2018-04-08 19:33:26.538113 mon.a mon.0 172.21.15.38:6789/0 164 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366329

FAIL

master

smithi

smithi008 smithi143 smithi169

"2018-04-08 19:46:25.630415 mon.b mon.0 172.21.15.8:6789/0 180 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 124 sec (SLOW_OPS)" in cluster log

2366330

FAIL

master

smithi

smithi088 smithi189 smithi198

"2018-04-08 20:04:37.447194 mon.b mon.0 172.21.15.88:6789/0 214 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366331

PASS

master

smithi

smithi017 smithi045 smithi104

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}

2366332

FAIL

master

smithi

smithi009 smithi028 smithi061

"2018-04-08 20:45:51.817796 mon.b mon.0 172.21.15.28:6789/0 143 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366333

FAIL

master

smithi

smithi023 smithi095 smithi188

"2018-04-08 20:01:15.118900 mds.g mds.1 172.21.15.23:6816/2352693932 1 : cluster [ERR] replayed stray Session close event for client.5325 172.21.15.188:0/1336561619 from time 2018-04-08 19:56:08.876097, ignoring" in cluster log

2366334

PASS

master

smithi

smithi103 smithi145 smithi201

2366335

FAIL

master

smithi

smithi090 smithi130 smithi146

"2018-04-08 19:44:16.937575 mon.b mon.0 172.21.15.130:6789/0 177 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366336

FAIL

master

smithi

smithi009 smithi080 smithi115

"2018-04-08 21:06:12.645398 mon.b mon.0 172.21.15.9:6789/0 139 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366337

FAIL

master

smithi

smithi079 smithi084 smithi185

"2018-04-08 20:04:15.328717 mon.a mon.0 172.21.15.79:6789/0 344 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 732 sec (SLOW_OPS)" in cluster log

2366338

FAIL

master

smithi

smithi006 smithi044 smithi068

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}

"2018-04-08 20:41:21.321516 mon.b mon.0 172.21.15.44:6789/0 333 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366339

FAIL

master

smithi

centos

smithi040 smithi102 smithi187

"2018-04-08 20:07:03.967989 mon.a mon.0 172.21.15.102:6789/0 208 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 165 sec (SLOW_OPS)" in cluster log

2366340

FAIL

master

smithi

smithi052 smithi172 smithi204

"2018-04-08 19:58:37.623159 mon.a mon.0 172.21.15.172:6789/0 175 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 738 sec (SLOW_OPS)" in cluster log

2366341

FAIL

master

smithi

smithi001 smithi019 smithi133

"2018-04-08 20:00:26.432376 mon.b mon.0 172.21.15.1:6789/0 200 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 699 sec (SLOW_OPS)" in cluster log

2366342

PASS

master

smithi

smithi050 smithi132 smithi176

2366343

PASS

master

smithi

smithi073 smithi170 smithi199

2366344

FAIL

master

smithi

smithi064 smithi076 smithi130

"2018-04-08 20:20:24.705105 mon.a mon.0 172.21.15.64:6789/0 269 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 62 sec (SLOW_OPS)" in cluster log

2366345

PASS

master

smithi

smithi002 smithi020 smithi118

2366346

FAIL

master

smithi

smithi059 smithi081 smithi154

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}

"2018-04-08 20:45:42.808211 mon.b mon.0 172.21.15.59:6789/0 196 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366347

PASS

master

smithi

smithi014 smithi096 smithi174

2366348

FAIL

master

smithi

smithi007 smithi136 smithi193

"2018-04-08 20:01:10.323480 mds.a mds.1 172.21.15.136:6817/2419946301 1 : cluster [ERR] replayed stray Session close event for client.4680 172.21.15.7:0/1856876146 from time 2018-04-08 20:00:23.579540, ignoring" in cluster log

2366349

FAIL

master

smithi

smithi038 smithi101 smithi197

"2018-04-08 20:04:15.774505 mon.b mon.0 172.21.15.101:6789/0 169 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366350

FAIL

master

smithi

smithi017 smithi074 smithi182

"2018-04-08 20:01:12.605618 mon.b mon.0 172.21.15.17:6789/0 236 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 165 sec (SLOW_OPS)" in cluster log

2366351

FAIL

master

smithi

smithi031 smithi109 smithi131

"2018-04-08 20:11:49.942615 mon.b mon.0 172.21.15.109:6789/0 305 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 67 sec (SLOW_OPS)" in cluster log

2366352

FAIL

master

smithi

smithi047 smithi152 smithi159

"2018-04-08 20:11:41.341709 mon.a mon.0 172.21.15.47:6789/0 160 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366353

FAIL

master

smithi

smithi026 smithi049 smithi089

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}

"2018-04-08 20:48:29.924877 mon.b mon.0 172.21.15.26:6789/0 297 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366354

FAIL

master

smithi

centos

smithi041 smithi063 smithi066

"2018-04-08 20:38:27.999098 mon.a mon.0 172.21.15.41:6789/0 319 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 39 sec (SLOW_OPS)" in cluster log

2366355

FAIL

master

smithi

smithi018 smithi026 smithi135

"2018-04-08 20:00:56.449545 mon.a mon.0 172.21.15.26:6789/0 214 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 743 sec (SLOW_OPS)" in cluster log

2366356

FAIL

master

smithi

smithi005 smithi055 smithi191

"2018-04-08 20:35:21.080234 mon.a mon.0 172.21.15.5:6789/0 251 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366357

FAIL

master

smithi

smithi043 smithi059 smithi081

"2018-04-08 20:06:54.920288 mon.a mon.0 172.21.15.59:6789/0 338 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 129 sec (SLOW_OPS)" in cluster log

2366358

FAIL

master

smithi

smithi057 smithi071 smithi171

"2018-04-08 20:14:18.292205 mon.a mon.0 172.21.15.71:6789/0 253 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 822 sec (SLOW_OPS)" in cluster log

2366359

FAIL

master

smithi

smithi003 smithi029 smithi167

"2018-04-08 20:00:02.931000 mon.b mon.0 172.21.15.3:6789/0 173 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366360

FAIL

master

smithi

smithi033 smithi047 smithi162

multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}

"2018-04-08 19:53:58.626953 mon.b mon.0 172.21.15.47:6789/0 452 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366361

FAIL

master

smithi

smithi027 smithi087 smithi134

"2018-04-08 20:16:36.116351 mon.b mon.0 172.21.15.27:6789/0 284 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366362

FAIL

master

smithi

smithi058 smithi086 smithi156

"2018-04-08 20:06:46.619442 mon.b mon.0 172.21.15.58:6789/0 306 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366363

FAIL

master

smithi

smithi094 smithi162 smithi203

multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml}

"2018-04-08 20:23:39.965382 mds.c mds.1 172.21.15.94:6817/2716305134 1 : cluster [ERR] replayed stray Session close event for client.5130 172.21.15.203:0/2856551718 from time 2018-04-08 20:18:19.481067, ignoring" in cluster log

2366364

FAIL

master

smithi

smithi117 smithi175 smithi178

multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}

"2018-04-08 20:11:10.114804 mon.a mon.0 172.21.15.117:6789/0 213 : cluster [WRN] Health check failed: 12 slow ops, oldest one blocked for 850 sec (SLOW_OPS)" in cluster log

2366365

FAIL

master

smithi

smithi135 smithi151 smithi168

multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}

"2018-04-08 21:06:14.613297 mon.b mon.0 172.21.15.135:6789/0 137 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366366

FAIL

master

smithi

smithi022 smithi131 smithi197

multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}

"2018-04-08 21:05:57.824132 mon.b mon.0 172.21.15.22:6789/0 312 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 213 sec (SLOW_OPS)" in cluster log

2366367

FAIL

master

smithi

smithi065 smithi110 smithi195

multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}

"2018-04-08 20:00:30.848312 mon.b mon.0 172.21.15.65:6789/0 148 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366368

FAIL

master

smithi

smithi079 smithi136 smithi193

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}

"2018-04-08 20:30:00.833621 mon.b mon.0 172.21.15.79:6789/0 196 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366369

FAIL

master

smithi

centos

smithi060 smithi118 smithi158

"2018-04-08 20:17:12.111452 mon.a mon.0 172.21.15.60:6789/0 152 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366370

FAIL

master

smithi

smithi045 smithi051 smithi168

multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}

"2018-04-08 20:13:11.788979 mon.b mon.0 172.21.15.45:6789/0 181 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 222 sec (SLOW_OPS)" in cluster log

2366371

PASS

master

smithi

smithi144 smithi177 smithi179

multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}

2366372

FAIL

master

smithi

smithi009 smithi064 smithi111

multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}

"2018-04-08 19:58:17.220363 mon.b mon.0 172.21.15.64:6789/0 318 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366373

PASS

master

smithi

smithi085 smithi087 smithi134

multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}

2366374

PASS

master

smithi

smithi152 smithi159 smithi200

2366375

PASS

master

smithi

smithi001 smithi008 smithi027

2366376

FAIL

master

smithi

smithi043 smithi154 smithi166

"2018-04-08 20:25:01.808741 mon.a mon.0 172.21.15.154:6789/0 343 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366377

FAIL

master

smithi

smithi011 smithi173 smithi202

"2018-04-08 20:07:49.467245 mon.b mon.0 172.21.15.173:6789/0 298 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366378

FAIL

master

smithi

smithi050 smithi132 smithi176

"2018-04-08 20:03:04.952229 mds.h mds.1 172.21.15.50:6816/2294709562 1 : cluster [ERR] replayed stray Session close event for client.4772 172.21.15.132:0/2438574388 from time 2018-04-08 20:02:24.885872, ignoring" in cluster log

2366379

PASS

master

smithi

smithi062 smithi067 smithi190

2366380

FAIL

master

smithi

smithi024 smithi092 smithi129

"2018-04-08 20:03:19.562828 mon.a mon.0 172.21.15.24:6789/0 198 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366381

PASS

master

smithi

smithi033 smithi096 smithi105

2366382

FAIL

master

smithi

smithi006 smithi044 smithi048

"2018-04-08 20:11:20.959731 mon.b mon.0 172.21.15.6:6789/0 178 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366383

PASS

master

smithi

smithi004 smithi014 smithi205

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}

2366384

FAIL

master

smithi

centos

smithi148 smithi149 smithi164

"2018-04-08 20:28:44.167807 mon.a mon.0 172.21.15.148:6789/0 253 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 394 sec (SLOW_OPS)" in cluster log

2366385

FAIL

master

smithi

smithi047 smithi152 smithi159

"2018-04-08 20:37:58.711614 mon.a mon.0 172.21.15.47:6789/0 170 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366386

PASS

master

smithi

smithi077 smithi080 smithi115

2366387

FAIL

master

smithi

smithi154 smithi165 smithi166

"2018-04-08 20:07:19.195885 mon.a mon.0 172.21.15.154:6789/0 154 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366388

FAIL

master

smithi

smithi052 smithi060 smithi104

"2018-04-08 21:08:29.950207 mon.a mon.0 172.21.15.60:6789/0 346 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 979 sec (SLOW_OPS)" in cluster log

2366389

FAIL

master

smithi

smithi056 smithi142 smithi157

"2018-04-08 20:18:05.359741 mon.a mon.0 172.21.15.56:6789/0 164 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 67 sec (SLOW_OPS)" in cluster log

2366390

PASS

master

smithi

smithi049 smithi089 smithi200

2366391

FAIL

master

smithi

smithi018 smithi135 smithi168

"2018-04-08 20:48:15.843587 mon.b mon.0 172.21.15.135:6789/0 202 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366392

FAIL

master

smithi

smithi016 smithi111 smithi138

"2018-04-08 20:32:23.235832 mon.b mon.0 172.21.15.16:6789/0 181 : cluster [WRN] Health check failed: 14 slow ops, oldest one blocked for 957 sec (SLOW_OPS)" in cluster log

2366393

FAIL

master

smithi

smithi008 smithi027 smithi169

"2018-04-08 20:45:11.938536 mds.c mds.1 172.21.15.169:6817/657883060 1 : cluster [ERR] replayed stray Session close event for client.5196 172.21.15.27:0/4184250417 from time 2018-04-08 20:39:46.946647, ignoring" in cluster log

2366394

FAIL

master

smithi

smithi093 smithi118 smithi140

"2018-04-08 21:36:59.344550 mon.a mon.0 172.21.15.118:6789/0 336 : cluster [WRN] Health check failed: 13 slow ops, oldest one blocked for 827 sec (SLOW_OPS)" in cluster log

2366395

PASS

master

smithi

smithi062 smithi067 smithi190

2366396

FAIL

master

smithi

smithi069 smithi082 smithi113

"2018-04-08 21:35:22.686603 mon.b mon.0 172.21.15.69:6789/0 162 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366397

FAIL

master

smithi

smithi078 smithi091 smithi096

"2018-04-08 20:37:12.598514 mon.b mon.0 172.21.15.78:6789/0 150 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 69 sec (SLOW_OPS)" in cluster log

2366398

PASS

master

smithi

smithi011 smithi173 smithi196

2366399

FAIL

master

smithi

centos

smithi133 smithi154 smithi192

"2018-04-08 21:34:20.267574 mon.a mon.0 172.21.15.154:6789/0 251 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366400

FAIL

master

smithi

smithi095 smithi143 smithi198

"2018-04-08 20:23:55.134354 mon.b mon.0 172.21.15.95:6789/0 296 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366401

PASS

master

smithi

smithi015 smithi094 smithi194

2366402

FAIL

master

smithi

smithi009 smithi059 smithi081

"2018-04-08 20:26:53.854164 mon.a mon.0 172.21.15.59:6789/0 343 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 73 sec (SLOW_OPS)" in cluster log

2366403

PASS

master

smithi

smithi093 smithi140 smithi192

2366404

FAIL

master

smithi

smithi083 smithi112 smithi139

"2018-04-08 20:24:23.900007 mon.b mon.0 172.21.15.112:6789/0 306 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366405

PASS

master

smithi

smithi020 smithi163 smithi174

2366406

FAIL

master

smithi

smithi052 smithi104 smithi204

"2018-04-08 20:34:03.329758 mon.a mon.0 172.21.15.104:6789/0 196 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 54 sec (SLOW_OPS)" in cluster log

2366407

FAIL

master

smithi

smithi023 smithi090 smithi146

"2018-04-08 20:35:02.521736 mon.a mon.0 172.21.15.23:6789/0 206 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366408

FAIL

master

smithi

smithi093 smithi165 smithi192

"2018-04-08 20:27:54.498284 mds.a mds.1 172.21.15.165:6817/371813141 1 : cluster [ERR] replayed stray Session close event for client.4771 172.21.15.93:0/2299424938 from time 2018-04-08 20:27:15.965273, ignoring" in cluster log

2366409

PASS

master

smithi

smithi088 smithi188 smithi189

2366410

FAIL

master

smithi

smithi040 smithi083 smithi139

"2018-04-08 20:45:42.348340 mon.b mon.0 172.21.15.83:6789/0 182 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 157 sec (SLOW_OPS)" in cluster log

2366411

PASS

master

smithi

smithi020 smithi186 smithi200

2366412

FAIL

master

smithi

smithi022 smithi039 smithi131

"2018-04-08 20:41:43.081699 mon.b mon.0 172.21.15.39:6789/0 179 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366413

FAIL

master

smithi

smithi031 smithi109 smithi181

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}

"2018-04-08 20:30:06.598736 mon.a mon.0 172.21.15.109:6789/0 263 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 60 sec (SLOW_OPS)" in cluster log

2366414

FAIL

master

smithi

centos

smithi003 smithi029 smithi167

"2018-04-08 20:51:05.411994 mon.b mon.0 172.21.15.3:6789/0 152 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366415

FAIL

master

smithi

smithi013 smithi014 smithi070

"2018-04-08 20:44:14.700727 mon.a mon.0 172.21.15.14:6789/0 152 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366416

FAIL

master

smithi

smithi045 smithi051 smithi136

"2018-04-08 20:48:22.616186 mon.b mon.0 172.21.15.51:6789/0 169 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366417

FAIL

master

smithi

smithi028 smithi061 smithi151

"2018-04-08 20:27:42.489059 mon.b mon.0 172.21.15.28:6789/0 136 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366418

FAIL

master

smithi

smithi142 smithi160 smithi169

"2018-04-08 21:19:23.093237 mon.a mon.0 172.21.15.160:6789/0 209 : cluster [WRN] Health check failed: 13 slow ops, oldest one blocked for 949 sec (SLOW_OPS)" in cluster log

2366419

FAIL

master

smithi

smithi036 smithi108 smithi196

"2018-04-08 20:35:11.696593 mon.a mon.0 172.21.15.36:6789/0 164 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 57 sec (SLOW_OPS)" in cluster log

2366420

PASS

master

smithi

smithi056 smithi072 smithi157

2366421

FAIL

master

smithi

smithi007 smithi084 smithi185

"2018-04-08 20:29:51.290216 mon.a mon.0 172.21.15.84:6789/0 219 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366422

FAIL

master

smithi

smithi110 smithi132 smithi176

"2018-04-08 20:45:10.452971 mon.a mon.0 172.21.15.110:6789/0 335 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 171 sec (SLOW_OPS)" in cluster log

2366423

FAIL

master

smithi

smithi050 smithi065 smithi195

"2018-04-08 20:42:42.550561 mds.b mds.1 172.21.15.50:6817/2348420816 1 : cluster [ERR] replayed stray Session close event for client.5226 172.21.15.195:0/1945650320 from time 2018-04-08 20:37:03.591655, ignoring" in cluster log

2366424

FAIL

master

smithi

smithi002 smithi004 smithi205

"2018-04-08 20:42:52.074863 mon.a mon.0 172.21.15.2:6789/0 323 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 731 sec (SLOW_OPS)" in cluster log

2366425

FAIL

master

smithi

smithi069 smithi106 smithi113

"2018-04-08 20:39:52.494836 mon.a mon.0 172.21.15.69:6789/0 170 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 128 sec (SLOW_OPS)" in cluster log

2366426

FAIL

master

smithi

smithi063 smithi066 smithi117

"2018-04-08 21:16:39.790607 mon.b mon.0 172.21.15.63:6789/0 188 : cluster [WRN] Health check failed: 16 slow ops, oldest one blocked for 246 sec (SLOW_OPS)" in cluster log

2366427

FAIL

master

smithi

smithi085 smithi087 smithi134

"2018-04-08 21:09:42.177516 mon.a mon.0 172.21.15.85:6789/0 165 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 607 sec (SLOW_OPS)" in cluster log

2366428

PASS

master

smithi

smithi057 smithi071 smithi171

2366429

FAIL

master

smithi

centos

smithi035 smithi082 smithi185

"2018-04-08 21:04:43.755703 mon.b mon.0 172.21.15.35:6789/0 144 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366430

FAIL

master

smithi

smithi036 smithi056 smithi072

"2018-04-08 21:07:36.710459 mon.a mon.0 172.21.15.56:6789/0 216 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 745 sec (SLOW_OPS)" in cluster log

2366431

FAIL

master

smithi

smithi108 smithi118 smithi158

"2018-04-08 21:00:14.389306 mon.a mon.0 172.21.15.118:6789/0 254 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366432

FAIL

master

smithi

smithi078 smithi092 smithi182

"2018-04-08 21:09:00.114699 mon.a mon.0 172.21.15.92:6789/0 205 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 49 sec (SLOW_OPS)" in cluster log

2366433

FAIL

master

smithi

smithi141 smithi142 smithi160

"2018-04-08 20:36:23.091236 mon.b mon.0 172.21.15.141:6789/0 148 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366434

PASS

master

smithi

smithi157 smithi164 smithi204

2366435

FAIL

master

smithi

smithi031 smithi072 smithi158

"2018-04-08 23:46:21.729058 mon.a mon.0 172.21.15.72:6789/0 352 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366436

FAIL

master

smithi

smithi087 smithi144 smithi156

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}

"2018-04-08 23:08:08.726776 mon.b mon.0 172.21.15.144:6789/0 185 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366437

FAIL

master

smithi

smithi149 smithi165 smithi198

"2018-04-08 21:13:44.813241 mon.b mon.0 172.21.15.165:6789/0 175 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 39 sec (SLOW_OPS)" in cluster log

2366438

FAIL

master

smithi

smithi162 smithi170 smithi203

"2018-04-08 20:45:05.138856 mds.h mds.1 172.21.15.170:6817/854997060 1 : cluster [ERR] replayed stray Session close event for client.4772 172.21.15.203:0/1539862323 from time 2018-04-08 20:44:23.638787, ignoring" in cluster log

2366439

FAIL

master

smithi

smithi079 smithi084 smithi193

"2018-04-08 21:11:12.230752 mon.a mon.0 172.21.15.79:6789/0 265 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 1339 sec (SLOW_OPS)" in cluster log

2366440

FAIL

master

smithi

smithi060 smithi068 smithi104

"2018-04-08 23:36:13.068802 mon.a mon.0 172.21.15.60:6789/0 201 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 152 sec (SLOW_OPS)" in cluster log

2366441

FAIL

master

smithi

smithi043 smithi112 smithi166

"2018-04-08 20:43:50.899139 mon.a mon.0 172.21.15.112:6789/0 255 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 71 sec (SLOW_OPS)" in cluster log

2366442

FAIL

master

smithi

smithi106 smithi143 smithi148

"2018-04-08 21:15:12.205280 mon.b mon.0 172.21.15.143:6789/0 181 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366443

FAIL

master

smithi

smithi031 smithi109 smithi181

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}

"2018-04-08 20:50:20.878349 mon.a mon.0 172.21.15.109:6789/0 236 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366444

FAIL

master

smithi

centos

smithi026 smithi049 smithi089

"2018-04-08 21:23:58.996826 mon.b mon.0 172.21.15.26:6789/0 156 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366445

PASS

master

smithi

smithi016 smithi061 smithi163

2366446

FAIL

master

smithi

smithi058 smithi112 smithi195

"2018-04-08 21:02:28.769535 mon.b mon.0 172.21.15.58:6789/0 180 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366447

PASS

master

smithi

smithi037 smithi145 smithi201

2366448

FAIL

master

smithi

smithi032 smithi068 smithi098

"2018-04-08 21:10:51.847828 mon.a mon.0 172.21.15.68:6789/0 209 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 716 sec (SLOW_OPS)" in cluster log

2366449

FAIL

master

smithi

smithi151 smithi163 smithi174

"2018-04-08 20:48:47.219207 mon.b mon.0 172.21.15.163:6789/0 146 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366450

PASS

master

smithi

smithi024 smithi169 smithi202

2366451

FAIL

master

smithi

smithi008 smithi059 smithi081

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}

"2018-04-08 21:05:09.503270 mon.b mon.0 172.21.15.8:6789/0 303 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 35 sec (SLOW_OPS)" in cluster log

2366452

FAIL

master

smithi

smithi050 smithi065 smithi086

"2018-04-08 21:01:54.860763 mon.a mon.0 172.21.15.50:6789/0 193 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366453

FAIL

master

smithi

smithi022 smithi059 smithi081

"2018-04-08 21:37:25.809675 mds.b mds.1 172.21.15.81:6817/449659003 1 : cluster [ERR] replayed stray Session close event for client.5208 172.21.15.22:0/3598344049 from time 2018-04-08 21:31:53.835362, ignoring" in cluster log

2366454

FAIL

master

smithi

smithi007 smithi102 smithi187

"2018-04-08 20:54:52.892185 mon.a mon.0 172.21.15.102:6789/0 346 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 372 sec (SLOW_OPS)" in cluster log

2366455

FAIL

master

smithi

smithi016 smithi061 smithi163

"2018-04-08 22:07:15.112086 mon.b mon.0 172.21.15.16:6789/0 133 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366456

FAIL

master

smithi

smithi038 smithi101 smithi156

"2018-04-08 21:05:00.603544 mon.a mon.0 172.21.15.38:6789/0 324 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366457

PASS

master

smithi

smithi006 smithi044 smithi202

2366458

FAIL

master

smithi

smithi069 smithi113 smithi194

"2018-04-08 21:17:09.068350 mon.a mon.0 172.21.15.69:6789/0 179 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 128 sec (SLOW_OPS)" in cluster log

2366459

FAIL

master

smithi

centos

smithi057 smithi071 smithi171

"2018-04-08 21:06:56.866729 mon.a mon.0 172.21.15.71:6789/0 156 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 177 sec (SLOW_OPS)" in cluster log

2366460

FAIL

master

smithi

smithi096 smithi187 smithi190

"2018-04-08 22:23:04.098336 mon.b mon.0 172.21.15.187:6789/0 186 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 749 sec (SLOW_OPS)" in cluster log

2366461

FAIL

master

smithi

smithi057 smithi071 smithi102

"2018-04-08 22:10:25.385405 mon.a mon.0 172.21.15.71:6789/0 160 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366462

FAIL

master

smithi

smithi148 smithi149 smithi165

"2018-04-08 20:56:10.629040 mon.b mon.0 172.21.15.148:6789/0 299 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 38 sec (SLOW_OPS)" in cluster log

2366463

FAIL

master

smithi

smithi090 smithi091 smithi146

"2018-04-08 21:12:02.255178 mon.b mon.0 172.21.15.91:6789/0 151 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366464

FAIL

master

smithi

smithi096 smithi175 smithi178

"2018-04-08 21:13:10.105960 mon.a mon.0 172.21.15.175:6789/0 192 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366465

PASS

master

smithi

smithi052 smithi067 smithi170

2366466

FAIL

master

smithi

smithi023 smithi055 smithi066

"2018-04-08 21:41:11.258704 mon.a mon.0 172.21.15.23:6789/0 265 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 78 sec (SLOW_OPS)" in cluster log

2366467

FAIL

master

smithi

smithi144 smithi170 smithi177

"2018-04-08 21:13:00.133292 mon.b mon.0 172.21.15.144:6789/0 294 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366468

FAIL

master

smithi

smithi024 smithi129 smithi181

"2018-04-08 21:12:32.393470 mds.h mds.1 172.21.15.129:6817/1851554463 1 : cluster [ERR] replayed stray Session close event for client.24745 172.21.15.181:0/4170837431 from time 2018-04-08 21:11:47.276817, ignoring" in cluster log

2366469

FAIL

master

smithi

smithi074 smithi111 smithi138

"2018-04-08 21:29:19.092184 mon.a mon.0 172.21.15.74:6789/0 170 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 1422 sec (SLOW_OPS)" in cluster log

2366470

FAIL

master

smithi

smithi076 smithi109 smithi134

"2018-04-08 22:15:36.418158 mon.a mon.0 172.21.15.109:6789/0 343 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366471

FAIL

master

smithi

smithi007 smithi036 smithi179

"2018-04-08 22:07:37.796580 mon.a mon.0 172.21.15.36:6789/0 158 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 73 sec (SLOW_OPS)" in cluster log

2366472

FAIL

master

smithi

smithi023 smithi031 smithi109

"2018-04-08 21:18:30.526721 mon.a mon.0 172.21.15.23:6789/0 355 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366473

FAIL

master

smithi

smithi001 smithi133 smithi154

"2018-04-08 21:04:03.986797 mon.b mon.0 172.21.15.1:6789/0 137 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366474

FAIL

master

smithi

centos

smithi037 smithi145 smithi201

"2018-04-08 21:25:40.936111 mon.a mon.0 172.21.15.37:6789/0 254 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366475

FAIL

master

smithi

smithi166 smithi171 smithi191

"2018-04-08 21:49:27.298351 mon.b mon.0 172.21.15.171:6789/0 147 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366476

FAIL

master

smithi

smithi043 smithi166 smithi191

"2018-04-08 21:02:16.967000 mon.b mon.0 172.21.15.166:6789/0 169 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366477

FAIL

master

smithi

smithi027 smithi103 smithi141

"2018-04-08 21:05:00.481640 mon.b mon.0 172.21.15.27:6789/0 147 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366478

FAIL

master

smithi

smithi005 smithi039 smithi055

"2018-04-08 21:17:47.125003 mon.a mon.0 172.21.15.5:6789/0 330 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 791 sec (SLOW_OPS)" in cluster log

2366479

FAIL

master

smithi

smithi095 smithi162 smithi203

"2018-04-08 21:13:29.574082 mon.a mon.0 172.21.15.95:6789/0 155 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 54 sec (SLOW_OPS)" in cluster log

2366480

FAIL

master

smithi

smithi058 smithi139 smithi195

"2018-04-08 21:57:23.398952 mon.b mon.0 172.21.15.58:6789/0 452 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366481

FAIL

master

smithi

smithi017 smithi117 smithi202

"2018-04-08 22:56:54.703336 mon.b mon.0 172.21.15.17:6789/0 281 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366482

FAIL

master

smithi

smithi103 smithi144 smithi156

"2018-04-08 22:50:03.533968 mon.a mon.0 172.21.15.103:6789/0 334 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366483

FAIL

master

smithi

smithi088 smithi157 smithi164

"2018-04-08 21:44:51.674730 mds.c mds.1 172.21.15.164:6817/819802745 1 : cluster [ERR] replayed stray Session close event for client.5241 172.21.15.157:0/1351353332 from time 2018-04-08 21:39:41.544596, ignoring" in cluster log

2366484

FAIL

master

smithi

smithi038 smithi070 smithi156

"2018-04-08 22:08:46.279759 mon.a mon.0 172.21.15.38:6789/0 356 : cluster [WRN] Health check failed: 28 slow ops, oldest one blocked for 2299 sec (SLOW_OPS)" in cluster log

2366485

PASS

master

smithi

smithi055 smithi066 smithi168

2366486

FAIL

master

smithi

smithi110 smithi158 smithi179

"2018-04-08 21:26:07.919182 mon.a mon.0 172.21.15.110:6789/0 217 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 224 sec (SLOW_OPS)" in cluster log

2366487

PASS

master

smithi

smithi020 smithi174 smithi200

2366488

FAIL

master

smithi

smithi152 smithi159 smithi200

"2018-04-08 21:56:15.316764 mon.b mon.0 172.21.15.152:6789/0 213 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366489

FAIL

master

smithi

centos

smithi001 smithi006 smithi027

"2018-04-08 21:57:03.311812 mon.b mon.0 172.21.15.1:6789/0 155 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366490

PASS

master

smithi

smithi017 smithi018 smithi186

2366491

PASS

master

smithi

smithi080 smithi115 smithi161

2366492

FAIL

master

smithi

smithi004 smithi064 smithi175

"2018-04-08 22:13:35.048384 mon.b mon.0 172.21.15.64:6789/0 306 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366493

PASS

master

smithi

smithi044 smithi090 smithi202

2366494

PASS

master

smithi

smithi108 smithi132 smithi176

2366495

FAIL

master

smithi

smithi006 smithi044 smithi202

"2018-04-08 21:26:16.414590 mon.b mon.0 172.21.15.6:6789/0 203 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366496

FAIL

master

smithi

smithi033 smithi105 smithi173

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}

"2018-04-08 21:36:46.701015 mon.b mon.0 172.21.15.33:6789/0 171 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 39 sec (SLOW_OPS)" in cluster log

2366497

FAIL

master

smithi

smithi162 smithi190 smithi203

"2018-04-08 21:32:12.506250 mon.b mon.0 172.21.15.162:6789/0 300 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 51 sec (SLOW_OPS)" in cluster log

2366498

FAIL

master

smithi

smithi026 smithi166 smithi197

"2018-04-08 23:15:27.438195 mds.e mds.1 172.21.15.26:6817/2540180194 1 : cluster [ERR] replayed stray Session close event for client.4731 172.21.15.166:0/2241985098 from time 2018-04-08 23:14:45.565031, ignoring" in cluster log

2366499

FAIL

master

smithi

smithi015 smithi041 smithi094

"2018-04-08 21:33:16.753811 mon.a mon.0 172.21.15.15:6789/0 166 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366500

FAIL

master

smithi

smithi011 smithi196 smithi204

"2018-04-08 21:23:15.781830 mon.a mon.0 172.21.15.11:6789/0 220 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 154 sec (SLOW_OPS)" in cluster log

2366501

FAIL

master

smithi

smithi060 smithi068 smithi104

"2018-04-08 21:32:27.390063 mon.a mon.0 172.21.15.60:6789/0 253 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 76 sec (SLOW_OPS)" in cluster log

2366502

FAIL

master

smithi

smithi019 smithi028 smithi077

"2018-04-08 21:35:11.032371 mon.b mon.0 172.21.15.19:6789/0 165 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366503

PASS

master

smithi

smithi088 smithi157 smithi164

2366504

FAIL

master

smithi

centos

smithi040 smithi081 smithi110

"2018-04-08 22:25:01.065220 mon.b mon.0 172.21.15.81:6789/0 145 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366505

FAIL

master

smithi

smithi017 smithi022 smithi033

"2018-04-08 22:07:05.722401 mon.a mon.0 172.21.15.17:6789/0 263 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 744 sec (SLOW_OPS)" in cluster log

2366506

FAIL

master

smithi

smithi078 smithi167 smithi197

"2018-04-08 21:27:15.152246 mon.b mon.0 172.21.15.167:6789/0 166 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366507

FAIL

master

smithi

smithi045 smithi143 smithi146

"2018-04-08 21:45:42.395985 mon.b mon.0 172.21.15.143:6789/0 149 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 50 sec (SLOW_OPS)" in cluster log

2366508

FAIL

master

smithi

smithi050 smithi065 smithi086

"2018-04-08 21:34:54.123161 mon.a mon.0 172.21.15.50:6789/0 343 : cluster [WRN] Health check failed: 12 slow ops, oldest one blocked for 792 sec (SLOW_OPS)" in cluster log

2366509

FAIL

master

smithi

smithi173 smithi188 smithi189

"2018-04-08 21:17:25.456063 mon.b mon.0 172.21.15.173:6789/0 138 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366510

PASS

master

smithi

smithi074 smithi111 smithi138

multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}

2366511

FAIL

master

smithi

smithi041 smithi094 smithi112

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}

"2018-04-08 22:03:58.599816 mon.a mon.0 172.21.15.41:6789/0 448 : cluster [WRN] Health check failed: 30 slow ops, oldest one blocked for 330 sec (SLOW_OPS)" in cluster log

2366512

FAIL

master

smithi

smithi062 smithi162 smithi203

"2018-04-08 22:11:04.690420 mon.b mon.0 172.21.15.62:6789/0 179 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366513

FAIL

master

smithi

smithi106 smithi132 smithi148

multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml}

"2018-04-08 21:55:17.738706 mds.b mds.1 172.21.15.132:6817/3419155987 1 : cluster [ERR] replayed stray Session close event for client.5161 172.21.15.106:0/2105920005 from time 2018-04-08 21:49:22.453264, ignoring" in cluster log

2366514

FAIL

master

smithi

smithi085 smithi149 smithi198

multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}

"2018-04-08 21:47:41.962408 mon.b mon.0 172.21.15.85:6789/0 189 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 849 sec (SLOW_OPS)" in cluster log

2366515

FAIL

master

smithi

smithi003 smithi029 smithi131

multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}

"2018-04-08 21:27:28.654506 mon.b mon.0 172.21.15.3:6789/0 232 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366516

FAIL

master

smithi

smithi113 smithi154 smithi192

multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}

"2018-04-08 23:42:09.590472 mon.a mon.0 172.21.15.154:6789/0 222 : cluster [WRN] Health check failed: 12 slow ops, oldest one blocked for 232 sec (SLOW_OPS)" in cluster log

2366517

FAIL

master

smithi

smithi079 smithi109 smithi129

multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}

"2018-04-08 21:40:13.231030 mon.b mon.0 172.21.15.79:6789/0 232 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366518

FAIL

master

smithi

smithi032 smithi080 smithi115

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}

"2018-04-08 21:30:48.667641 mon.a mon.0 172.21.15.80:6789/0 259 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366519

FAIL

master

smithi

centos

smithi009 smithi014 smithi182

"2018-04-08 21:57:41.791511 mon.b mon.0 172.21.15.14:6789/0 252 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366520

FAIL

master

smithi

smithi013 smithi098 smithi101

multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}

"2018-04-08 21:47:31.337657 mon.b mon.0 172.21.15.13:6789/0 181 : cluster [WRN] Health check failed: 18 slow ops, oldest one blocked for 1066 sec (SLOW_OPS)" in cluster log

2366521

PASS

master

smithi

smithi051 smithi064 smithi163

multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}

2366522

PASS

master

smithi

smithi092 smithi103 smithi141

multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}

2366523

FAIL

master

smithi

smithi003 smithi029 smithi045

multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}

"2018-04-08 22:56:25.886558 mon.a mon.0 172.21.15.3:6789/0 246 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366524

FAIL

master

smithi

smithi134 smithi175 smithi177

"2018-04-08 21:32:10.238808 mon.b mon.0 172.21.15.134:6789/0 183 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366525

PASS

master

smithi

smithi062 smithi067 smithi144

2366526

FAIL

master

smithi

smithi052 smithi095 smithi170

"2018-04-08 21:32:10.093548 mon.a mon.0 172.21.15.95:6789/0 190 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 90 sec (SLOW_OPS)" in cluster log

2366527

FAIL

master

smithi

smithi056 smithi072 smithi158

"2018-04-08 21:59:32.163194 mon.a mon.0 172.21.15.56:6789/0 199 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366528

FAIL

master

smithi

smithi087 smithi096 smithi178

"2018-04-08 21:37:32.768759 mds.a mds.1 172.21.15.178:6818/1069018314 1 : cluster [ERR] replayed stray Session close event for client.4787 172.21.15.87:0/1523911581 from time 2018-04-08 21:36:49.094753, ignoring" in cluster log

2366529

PASS

master

smithi

smithi002 smithi043 smithi205

2366530

FAIL

master

smithi

smithi031 smithi084 smithi193

"2018-04-08 21:41:49.149043 mon.a mon.0 172.21.15.31:6789/0 212 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 157 sec (SLOW_OPS)" in cluster log

2366531

FAIL

master

smithi

smithi018 smithi022 smithi088

"2018-04-08 23:40:30.245465 mon.b mon.0 172.21.15.22:6789/0 237 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366532

FAIL

master

smithi

smithi035 smithi165 smithi185

"2018-04-08 21:33:18.010353 mon.b mon.0 172.21.15.35:6789/0 167 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366533

FAIL

master

smithi

smithi040 smithi083 smithi139

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}

"2018-04-08 21:36:46.538139 mon.b mon.0 172.21.15.83:6789/0 154 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366534

PASS

master

smithi

centos

smithi060 smithi068 smithi104

2366535

FAIL

master

smithi

smithi117 smithi172 smithi194

"2018-04-08 21:45:42.199498 mon.b mon.0 172.21.15.117:6789/0 143 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366536

FAIL

master

smithi

smithi063 smithi188 smithi189

"2018-04-08 21:37:52.715292 mon.b mon.0 172.21.15.188:6789/0 167 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366537

PASS

master

smithi

smithi005 smithi039 smithi047

2366538

FAIL

master

smithi

smithi074 smithi111 smithi138

"2018-04-08 22:10:52.776720 mon.a mon.0 172.21.15.74:6789/0 347 : cluster [WRN] Health check failed: 12 slow ops, oldest one blocked for 954 sec (SLOW_OPS)" in cluster log

2366539

FAIL

master

smithi

smithi020 smithi091 smithi179

"2018-04-08 21:47:12.508315 mon.a mon.0 172.21.15.20:6789/0 148 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366540

FAIL

master

smithi

smithi004 smithi032 smithi175

"2018-04-08 21:52:53.791630 mon.b mon.0 172.21.15.4:6789/0 275 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366541

FAIL

master

smithi

smithi135 smithi151 smithi168

"2018-04-08 21:41:42.101237 mon.b mon.0 172.21.15.135:6789/0 176 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 38 sec (SLOW_OPS)" in cluster log

2366542

FAIL

master

smithi

smithi077 smithi143 smithi146

"2018-04-08 22:18:56.115275 mon.a mon.0 172.21.15.77:6789/0 343 : cluster [WRN] Health check failed: 13 slow ops, oldest one blocked for 942 sec (SLOW_OPS)" in cluster log

2366543

FAIL

master

smithi

smithi142 smithi160 smithi174

"2018-04-08 21:51:53.383278 mds.b mds.1 172.21.15.174:6817/3069710080 1 : cluster [ERR] replayed stray Session close event for client.5245 172.21.15.142:0/3386508979 from time 2018-04-08 21:46:17.188572, ignoring" in cluster log

2366544

FAIL

master

smithi

smithi073 smithi130 smithi199

"2018-04-08 22:04:34.215768 mon.a mon.0 172.21.15.73:6789/0 341 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366545

FAIL

master

smithi

smithi024 smithi169 smithi181

"2018-04-08 21:41:23.364487 mon.a mon.0 172.21.15.24:6789/0 165 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 63 sec (SLOW_OPS)" in cluster log

2366546

FAIL

master

smithi

smithi008 smithi108 smithi176

"2018-04-08 21:43:44.287732 mon.a mon.0 172.21.15.8:6789/0 205 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366547

PASS

master

smithi

smithi015 smithi135 smithi151

2366548

FAIL

master

smithi

smithi004 smithi142 smithi159

"2018-04-08 23:45:26.442902 mon.b mon.0 172.21.15.4:6789/0 179 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 71 sec (SLOW_OPS)" in cluster log

2366549

FAIL

master

smithi

centos

smithi008 smithi061 smithi132

"2018-04-09 00:03:03.920865 mon.a mon.0 172.21.15.8:6789/0 164 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366550

FAIL

master

smithi

smithi049 smithi089 smithi131

"2018-04-08 22:28:43.318858 mon.a mon.0 172.21.15.49:6789/0 354 : cluster [WRN] Health check failed: 20 slow ops, oldest one blocked for 1468 sec (SLOW_OPS)" in cluster log

2366551

PASS

master

smithi

smithi059 smithi086 smithi105

2366552

FAIL

master

smithi

smithi040 smithi081 smithi083

"2018-04-08 21:57:02.736269 mon.b mon.0 172.21.15.81:6789/0 310 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 69 sec (SLOW_OPS)" in cluster log

2366553

FAIL

master

smithi

smithi051 smithi110 smithi136

"2018-04-08 21:55:28.990951 mon.b mon.0 172.21.15.51:6789/0 226 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366554

FAIL

master

smithi

smithi057 smithi071 smithi162

"2018-04-08 21:51:29.129963 mon.b mon.0 172.21.15.71:6789/0 313 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366555

FAIL

master

smithi

smithi036 smithi092 smithi141

"2018-04-08 21:48:50.764527 mon.b mon.0 172.21.15.36:6789/0 204 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 40 sec (SLOW_OPS)" in cluster log

2366556

FAIL

master

smithi

smithi064 smithi076 smithi134

"2018-04-08 21:53:12.583167 mon.b mon.0 172.21.15.64:6789/0 184 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366557

FAIL

master

smithi

smithi097 smithi099 smithi177

"2018-04-08 22:06:46.069639 mon.a mon.0 172.21.15.97:6789/0 202 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366558

FAIL

master

smithi

smithi010 smithi103 smithi150

"2018-04-08 21:56:02.807219 mds.f mds.1 172.21.15.150:6817/3590809924 1 : cluster [ERR] replayed stray Session close event for client.4735 172.21.15.10:0/275773003 from time 2018-04-08 21:55:22.857889, ignoring" in cluster log

2366559

FAIL

master

smithi

smithi051 smithi106 smithi136

"2018-04-08 22:14:33.250795 mon.b mon.0 172.21.15.51:6789/0 152 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366560

FAIL

master

smithi

smithi050 smithi065 smithi173

"2018-04-08 21:59:00.144109 mon.b mon.0 172.21.15.50:6789/0 183 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 148 sec (SLOW_OPS)" in cluster log

2366561

FAIL

master

smithi

smithi062 smithi190 smithi203

"2018-04-08 21:51:04.311405 mon.b mon.0 172.21.15.62:6789/0 150 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366562

FAIL

master

smithi

smithi087 smithi144 smithi156

"2018-04-08 23:28:27.517580 mon.b mon.0 172.21.15.144:6789/0 174 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366563

FAIL

master

smithi

smithi018 smithi026 smithi087

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}

"2018-04-08 22:05:10.291259 mon.b mon.0 172.21.15.18:6789/0 131 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366564

FAIL

master

smithi

centos

smithi048 smithi095 smithi144

"2018-04-08 22:05:42.740253 mon.a mon.0 172.21.15.95:6789/0 158 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366565

FAIL

master

smithi

smithi041 smithi165 smithi185

"2018-04-08 22:45:53.170909 mon.b mon.0 172.21.15.41:6789/0 151 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366566

PASS

master

smithi

smithi083 smithi145 smithi152

2366567

FAIL

master

smithi

smithi069 smithi082 smithi113

"2018-04-08 21:58:51.266136 mon.b mon.0 172.21.15.69:6789/0 141 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366568

FAIL

master

smithi

smithi005 smithi039 smithi047

"2018-04-08 21:57:15.982093 mon.a mon.0 172.21.15.5:6789/0 344 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 63 sec (SLOW_OPS)" in cluster log

2366569

FAIL

master

smithi

smithi019 smithi028 smithi157

"2018-04-08 22:03:55.207840 mon.b mon.0 172.21.15.19:6789/0 142 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366570

PASS

master

smithi

smithi003 smithi029 smithi045

2366571

FAIL

master

smithi

smithi023 smithi088 smithi164

"2018-04-08 22:33:18.783445 mon.a mon.0 172.21.15.23:6789/0 182 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366572

FAIL

master

smithi

smithi008 smithi108 smithi176

"2018-04-08 22:21:55.520393 mon.a mon.0 172.21.15.8:6789/0 212 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 362 sec (SLOW_OPS)" in cluster log

2366573

FAIL

master

smithi

smithi049 smithi131 smithi177

"2018-04-08 23:04:20.143417 mds.b mds.1 172.21.15.131:6816/1791659432 1 : cluster [ERR] replayed stray Session close event for client.35215 172.21.15.177:0/3331530764 from time 2018-04-08 22:59:16.004326, ignoring" in cluster log

2366574

FAIL

master

smithi

smithi032 smithi079 smithi129

"2018-04-08 22:28:39.968368 mon.b mon.0 172.21.15.79:6789/0 304 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366575

FAIL

master

smithi

smithi091 smithi092 smithi182

"2018-04-08 22:26:25.590021 mon.a mon.0 172.21.15.92:6789/0 257 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366576

FAIL

master

smithi

smithi043 smithi172 smithi205

"2018-04-08 22:59:19.051146 mon.b mon.0 172.21.15.43:6789/0 172 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366577

FAIL

master

smithi

smithi052 smithi067 smithi170

"2018-04-08 22:10:29.357388 mon.a mon.0 172.21.15.67:6789/0 252 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366578

PASS

master

smithi

smithi035 smithi165 smithi185

2366579

FAIL

master

smithi

centos

smithi013 smithi094 smithi112

"2018-04-08 22:33:02.989910 mon.b mon.0 172.21.15.94:6789/0 151 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366580

FAIL

master

smithi

smithi058 smithi181 smithi197

"2018-04-08 22:19:15.690211 mon.b mon.0 172.21.15.181:6789/0 181 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366581

PASS

master

smithi

smithi023 smithi088 smithi164

2366582

FAIL

master

smithi

smithi020 smithi091 smithi186

"2018-04-08 22:07:26.887642 mon.a mon.0 172.21.15.20:6789/0 340 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 39 sec (SLOW_OPS)" in cluster log

2366583

PASS

master

smithi

smithi159 smithi200 smithi201

2366584

FAIL

master

smithi

smithi061 smithi132 smithi163

"2018-04-08 22:46:20.429606 mon.a mon.0 172.21.15.61:6789/0 192 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 67 sec (SLOW_OPS)" in cluster log

2366585

FAIL

master

smithi

smithi092 smithi141 smithi178

"2018-04-08 22:07:17.104186 mon.a mon.0 172.21.15.92:6789/0 364 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366586

FAIL

master

smithi

smithi069 smithi078 smithi082

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}

SSH connection to smithi078 was lost: 'rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0'

2366587

FAIL

master

smithi

smithi018 smithi020 smithi186

"2018-04-08 22:27:21.830763 mon.a mon.0 172.21.15.18:6789/0 215 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 149 sec (SLOW_OPS)" in cluster log

2366588

FAIL

master

smithi

smithi113 smithi154 smithi192

"2018-04-08 22:20:42.240159 mds.a mds.1 172.21.15.154:6817/1008127222 1 : cluster [ERR] replayed stray Session close event for client.4681 172.21.15.113:0/2568035857 from time 2018-04-08 22:20:00.912548, ignoring" in cluster log

2366589

FAIL

master

smithi

smithi037 smithi139 smithi195

"2018-04-08 22:46:22.997451 mon.a mon.0 172.21.15.37:6789/0 177 : cluster [WRN] Health check failed: 12 slow ops, oldest one blocked for 1915 sec (SLOW_OPS)" in cluster log

2366590

FAIL

master

smithi

smithi044 smithi090 smithi133

"2018-04-08 22:21:15.046797 mon.a mon.0 172.21.15.44:6789/0 213 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366591

FAIL

master

smithi

smithi024 smithi169 smithi202

"2018-04-08 22:19:03.072430 mon.a mon.0 172.21.15.24:6789/0 256 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 68 sec (SLOW_OPS)" in cluster log

2366592

FAIL

master

smithi

smithi142 smithi160 smithi174

"2018-04-08 22:19:51.453767 mon.a mon.0 172.21.15.160:6789/0 226 : cluster [WRN] Health check failed: 18 slow ops, oldest one blocked for 644 sec (SLOW_OPS)" in cluster log

2366593

FAIL

master

smithi

smithi016 smithi018 smithi141

"2018-04-08 23:22:07.689638 mon.a mon.0 172.21.15.16:6789/0 264 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 88 sec (SLOW_OPS)" in cluster log

2366594

FAIL

master

smithi

centos

smithi055 smithi089 smithi094

"2018-04-08 23:39:47.244994 mon.b mon.0 172.21.15.55:6789/0 251 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 64 sec (SLOW_OPS)" in cluster log

2366595

FAIL

master

smithi

smithi059 smithi086 smithi105

"2018-04-08 22:37:33.539390 mon.a mon.0 172.21.15.59:6789/0 166 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 742 sec (SLOW_OPS)" in cluster log

2366596

PASS

master

smithi

smithi052 smithi067 smithi170

2366597

FAIL

master

smithi

smithi070 smithi084 smithi087

"2018-04-08 22:48:12.081135 mon.a mon.0 172.21.15.70:6789/0 163 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 62 sec (SLOW_OPS)" in cluster log

2366598

FAIL

master

smithi

smithi132 smithi148 smithi167

"2018-04-08 22:25:18.558137 mon.a mon.0 172.21.15.148:6789/0 199 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366599

FAIL

master

smithi

smithi002 smithi057 smithi071

"2018-04-08 23:30:05.624358 mon.a mon.0 172.21.15.2:6789/0 164 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 110 sec (SLOW_OPS)" in cluster log

2366600

DEAD

master

smithi

reached maximum tries (60) after waiting for 900 seconds

2366601

FAIL

master

smithi

smithi026 smithi087 smithi103

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}

"2018-04-08 22:29:09.012207 mon.b mon.0 172.21.15.26:6789/0 227 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 84 sec (SLOW_OPS)" in cluster log

2366602

FAIL

master

smithi

smithi017 smithi033 smithi204

"2018-04-08 23:15:12.781171 mon.b mon.0 172.21.15.17:6789/0 172 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366603

FAIL

master

smithi

smithi113 smithi154 smithi192

"2018-04-08 23:01:25.406154 mds.b mds.1 172.21.15.192:6817/3229976243 1 : cluster [ERR] replayed stray Session close event for client.5195 172.21.15.113:0/1581106527 from time 2018-04-08 22:54:46.415576, ignoring" in cluster log

2366604

FAIL

master

smithi

smithi051 smithi171 smithi191

"2018-04-08 23:48:24.648816 mon.a mon.0 172.21.15.51:6789/0 251 : cluster [WRN] Health check failed: 12 slow ops, oldest one blocked for 2537 sec (SLOW_OPS)" in cluster log

2366605

PASS

master

smithi

smithi009 smithi010 smithi150

2366606

FAIL

master

smithi

smithi106 smithi136 smithi176

"2018-04-08 23:31:07.717055 mon.a mon.0 172.21.15.136:6789/0 347 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 274 sec (SLOW_OPS)" in cluster log

2366607

FAIL

master

smithi

smithi039 smithi047 smithi064

"2018-04-08 22:33:54.519726 mon.a mon.0 172.21.15.47:6789/0 158 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366608

PASS

master

smithi

smithi159 smithi200 smithi201

2366609

FAIL

master

smithi

centos

smithi092 smithi143 smithi188

"2018-04-08 23:25:16.097130 mon.a mon.0 172.21.15.92:6789/0 162 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366610

FAIL

master

smithi

smithi182 smithi190 smithi199

"2018-04-08 23:04:51.370216 mon.b mon.0 172.21.15.182:6789/0 314 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366611

PASS

master

smithi

smithi002 smithi057 smithi071

2366612

FAIL

master

smithi

smithi036 smithi141 smithi178

"2018-04-08 22:25:57.888834 mon.b mon.0 172.21.15.36:6789/0 174 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366613

FAIL

master

smithi

smithi003 smithi029 smithi045

"2018-04-08 22:24:02.112979 mon.a mon.0 172.21.15.3:6789/0 250 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 99 sec (SLOW_OPS)" in cluster log

2366614

FAIL

master

smithi

smithi028 smithi098 smithi101

"2018-04-08 22:24:11.243941 mon.b mon.0 172.21.15.28:6789/0 178 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366615

FAIL

master

smithi

smithi057 smithi071 smithi194

"2018-04-08 22:40:48.138695 mon.a mon.0 172.21.15.71:6789/0 256 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 43 sec (SLOW_OPS)" in cluster log

2366616

FAIL

master

smithi

smithi026 smithi095 smithi166

"2018-04-08 22:48:22.643651 mon.b mon.0 172.21.15.26:6789/0 168 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366617

FAIL

master

smithi

smithi055 smithi066 smithi168

"2018-04-08 22:37:10.788026 mon.a mon.0 172.21.15.55:6789/0 328 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 66 sec (SLOW_OPS)" in cluster log

2366618

FAIL

master

smithi

smithi031 smithi038 smithi193

"2018-04-08 22:51:31.870106 mds.i mds.1 172.21.15.193:6817/2030524270 1 : cluster [ERR] replayed stray Session close event for client.4751 172.21.15.31:0/755935619 from time 2018-04-08 22:50:47.093564, ignoring" in cluster log

2366619

PASS

master

smithi

smithi007 smithi135 smithi151

2366620

FAIL

master

smithi

smithi014 smithi015 smithi179

"2018-04-08 22:29:48.327086 mon.b mon.0 172.21.15.14:6789/0 191 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366621

PASS

master

smithi

smithi142 smithi160 smithi174

2366622

FAIL

master

smithi

smithi063 smithi188 smithi189

"2018-04-08 22:32:48.673923 mon.b mon.0 172.21.15.188:6789/0 177 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366623

FAIL

master

smithi

smithi043 smithi172 smithi205

"2018-04-08 23:38:29.353588 mon.b mon.0 172.21.15.43:6789/0 217 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366624

PASS

master

smithi

centos

smithi027 smithi028 smithi098

2366625

PASS

master

smithi

smithi052 smithi067 smithi170

2366626

FAIL

master

smithi

smithi060 smithi068 smithi104

"2018-04-08 23:06:15.212155 mon.a mon.0 172.21.15.60:6789/0 203 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 1092 sec (SLOW_OPS)" in cluster log

2366627

FAIL

master

smithi

smithi005 smithi196 smithi204

"2018-04-08 22:35:09.401743 mon.a mon.0 172.21.15.5:6789/0 259 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366628

FAIL

master

smithi

smithi056 smithi072 smithi158

"2018-04-08 22:49:43.485756 mon.a mon.0 172.21.15.56:6789/0 330 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 96 sec (SLOW_OPS)" in cluster log

2366629

FAIL

master

smithi

smithi011 smithi117 smithi172

"2018-04-08 22:38:26.221651 mon.b mon.0 172.21.15.117:6789/0 148 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 74 sec (SLOW_OPS)" in cluster log

2366630

PASS

master

smithi

smithi063 smithi115 smithi189

2366631

FAIL

master

smithi

smithi106 smithi136 smithi176

"2018-04-08 23:08:08.552849 mon.a mon.0 172.21.15.136:6789/0 271 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 174 sec (SLOW_OPS)" in cluster log

2366632

FAIL

master

smithi

smithi138 smithi149 smithi198

"2018-04-08 22:52:20.056035 mon.b mon.0 172.21.15.149:6789/0 168 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 67 sec (SLOW_OPS)" in cluster log

2366633

FAIL

master

smithi

smithi093 smithi118 smithi140

"2018-04-08 22:50:51.905307 mds.b mds.1 172.21.15.140:6817/3283567712 1 : cluster [ERR] replayed stray Session close event for client.15181 172.21.15.93:0/3653786500 from time 2018-04-08 22:45:19.310918, ignoring" in cluster log

2366634

FAIL

master

smithi

smithi036 smithi101 smithi175

"2018-04-08 22:46:14.503222 mon.b mon.0 172.21.15.101:6789/0 176 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366635

FAIL

master

smithi

smithi001 smithi004 smithi006

"2018-04-08 22:42:52.591835 mon.b mon.0 172.21.15.1:6789/0 143 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366636

FAIL

master

smithi

smithi102 smithi178 smithi186

"2018-04-08 22:48:04.223328 mon.b mon.0 172.21.15.102:6789/0 179 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366637

FAIL

master

smithi

smithi016 smithi018 smithi020

"2018-04-08 22:44:39.945037 mon.b mon.0 172.21.15.16:6789/0 139 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366638

FAIL

master

smithi

smithi141 smithi148 smithi167

"2018-04-08 22:43:55.122599 mon.a mon.0 172.21.15.148:6789/0 269 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 66 sec (SLOW_OPS)" in cluster log

2366639

FAIL

master

smithi

centos

smithi023 smithi074 smithi085

"2018-04-08 23:17:53.461256 mon.b mon.0 172.21.15.23:6789/0 162 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 36 sec (SLOW_OPS)" in cluster log

2366640

FAIL

master

smithi

smithi022 smithi088 smithi164

"2018-04-08 23:05:07.530817 mon.a mon.0 172.21.15.88:6789/0 328 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 746 sec (SLOW_OPS)" in cluster log

2366641

FAIL

master

smithi

smithi048 smithi070 smithi084

"2018-04-08 23:16:02.459072 mon.b mon.0 172.21.15.70:6789/0 140 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

2366642

FAIL

master

smithi

smithi011 smithi194 smithi196

"2018-04-08 22:59:34.575732 mon.a mon.0 172.21.15.11:6789/0 208 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 77 sec (SLOW_OPS)" in cluster log

2366643

PASS

master

smithi

smithi050 smithi081 smithi110

2366644

FAIL

master

smithi

smithi142 smithi159 smithi200

"2018-04-08 22:59:10.245360 mon.b mon.0 172.21.15.159:6789/0 178 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366645

PASS

master

smithi

smithi048 smithi171 smithi191

2366646

FAIL

master

smithi

smithi005 smithi033 smithi204

multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}

"2018-04-08 22:53:47.298414 mon.a mon.0 172.21.15.33:6789/0 181 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

2366647

FAIL

master

smithi

smithi073 smithi130 smithi141

"2018-04-08 23:03:17.714123 mon.a mon.0 172.21.15.73:6789/0 191 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366648

FAIL

master

smithi

smithi097 smithi099 smithi111

"2018-04-08 22:55:43.996462 mds.c mds.1 172.21.15.97:6816/3634658677 1 : cluster [ERR] replayed stray Session close event for client.4744 172.21.15.99:0/1339769397 from time 2018-04-08 22:54:58.343013, ignoring" in cluster log

2366649

FAIL

master

smithi

smithi148 smithi167 smithi174

"2018-04-08 23:38:57.018114 mon.a mon.0 172.21.15.148:6789/0 162 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

2366650

FAIL

master

smithi

smithi009 smithi010 smithi150

"2018-04-08 23:00:51.634356 mon.b mon.0 172.21.15.9:6789/0 186 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366651

FAIL

master

smithi

smithi050 smithi081 smithi110

"2018-04-08 23:09:22.795075 mon.b mon.0 172.21.15.50:6789/0 237 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

2366652

FAIL

master

smithi

smithi109 smithi118 smithi134

"2018-04-08 23:15:28.948710 mon.b mon.0 172.21.15.109:6789/0 190 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 368 sec (SLOW_OPS)" in cluster log

2366653

FAIL

master

smithi

smithi003 smithi020 smithi045

"2018-04-08 23:40:36.909913 mon.a mon.0 172.21.15.3:6789/0 203 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 158 sec (SLOW_OPS)" in cluster log

2366654

PASS

master

smithi

centos

smithi032 smithi079 smithi129

2366655

FAIL

master

smithi

smithi097 smithi099 smithi111

"2018-04-08 23:43:14.611004 mon.a mon.0 172.21.15.97:6789/0 246 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

2366656

PASS

master

smithi

smithi077 smithi146 smithi201

2366657

PASS

master

smithi

smithi055 smithi089 smithi168

2366658

FAIL

master

smithi

smithi024 smithi066 smithi169

"2018-04-08 23:08:47.178472 mon.a mon.0 172.21.15.24:6789/0 339 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 786 sec (SLOW_OPS)" in cluster log

2366659

FAIL

master

smithi

smithi040 smithi065 smithi093

"2018-04-08 23:11:33.680155 mon.a mon.0 172.21.15.40:6789/0 163 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 126 sec (SLOW_OPS)" in cluster log

pdonnell-2018-04-06_22:40:34-multimds-master-testing-basic-smithi