.. | .. |
---|
19 | 19 | #include <linux/sunrpc/addr.h> |
---|
20 | 20 | #include <linux/sunrpc/xprtmultipath.h> |
---|
21 | 21 | |
---|
22 | | -typedef struct rpc_xprt *(*xprt_switch_find_xprt_t)(struct list_head *head, |
---|
| 22 | +typedef struct rpc_xprt *(*xprt_switch_find_xprt_t)(struct rpc_xprt_switch *xps, |
---|
23 | 23 | const struct rpc_xprt *cur); |
---|
24 | 24 | |
---|
25 | 25 | static const struct rpc_xprt_iter_ops rpc_xprt_iter_singular; |
---|
.. | .. |
---|
36 | 36 | if (xps->xps_nxprts == 0) |
---|
37 | 37 | xps->xps_net = xprt->xprt_net; |
---|
38 | 38 | xps->xps_nxprts++; |
---|
| 39 | + xps->xps_nactive++; |
---|
39 | 40 | } |
---|
40 | 41 | |
---|
41 | 42 | /** |
---|
.. | .. |
---|
51 | 52 | if (xprt == NULL) |
---|
52 | 53 | return; |
---|
53 | 54 | spin_lock(&xps->xps_lock); |
---|
54 | | - if ((xps->xps_net == xprt->xprt_net || xps->xps_net == NULL) && |
---|
55 | | - !rpc_xprt_switch_has_addr(xps, (struct sockaddr *)&xprt->addr)) |
---|
| 55 | + if (xps->xps_net == xprt->xprt_net || xps->xps_net == NULL) |
---|
56 | 56 | xprt_switch_add_xprt_locked(xps, xprt); |
---|
57 | 57 | spin_unlock(&xps->xps_lock); |
---|
58 | 58 | } |
---|
.. | .. |
---|
62 | 62 | { |
---|
63 | 63 | if (unlikely(xprt == NULL)) |
---|
64 | 64 | return; |
---|
| 65 | + xps->xps_nactive--; |
---|
65 | 66 | xps->xps_nxprts--; |
---|
66 | 67 | if (xps->xps_nxprts == 0) |
---|
67 | 68 | xps->xps_net = NULL; |
---|
.. | .. |
---|
102 | 103 | if (xps != NULL) { |
---|
103 | 104 | spin_lock_init(&xps->xps_lock); |
---|
104 | 105 | kref_init(&xps->xps_kref); |
---|
105 | | - xps->xps_nxprts = 0; |
---|
| 106 | + xps->xps_nxprts = xps->xps_nactive = 0; |
---|
| 107 | + atomic_long_set(&xps->xps_queuelen, 0); |
---|
| 108 | + xps->xps_net = NULL; |
---|
106 | 109 | INIT_LIST_HEAD(&xps->xps_xprt_list); |
---|
107 | 110 | xps->xps_iter_ops = &rpc_xprt_iter_singular; |
---|
108 | 111 | xprt_switch_add_xprt_locked(xps, xprt); |
---|
.. | .. |
---|
193 | 196 | } |
---|
194 | 197 | |
---|
195 | 198 | static |
---|
| 199 | +bool xprt_is_active(const struct rpc_xprt *xprt) |
---|
| 200 | +{ |
---|
| 201 | + return kref_read(&xprt->kref) != 0; |
---|
| 202 | +} |
---|
| 203 | + |
---|
| 204 | +static |
---|
196 | 205 | struct rpc_xprt *xprt_switch_find_first_entry(struct list_head *head) |
---|
197 | 206 | { |
---|
198 | | - return list_first_or_null_rcu(head, struct rpc_xprt, xprt_switch); |
---|
| 207 | + struct rpc_xprt *pos; |
---|
| 208 | + |
---|
| 209 | + list_for_each_entry_rcu(pos, head, xprt_switch) { |
---|
| 210 | + if (xprt_is_active(pos)) |
---|
| 211 | + return pos; |
---|
| 212 | + } |
---|
| 213 | + return NULL; |
---|
199 | 214 | } |
---|
200 | 215 | |
---|
201 | 216 | static |
---|
.. | .. |
---|
213 | 228 | const struct rpc_xprt *cur) |
---|
214 | 229 | { |
---|
215 | 230 | struct rpc_xprt *pos; |
---|
| 231 | + bool found = false; |
---|
216 | 232 | |
---|
217 | 233 | list_for_each_entry_rcu(pos, head, xprt_switch) { |
---|
218 | 234 | if (cur == pos) |
---|
| 235 | + found = true; |
---|
| 236 | + if (found && xprt_is_active(pos)) |
---|
219 | 237 | return pos; |
---|
220 | 238 | } |
---|
221 | 239 | return NULL; |
---|
.. | .. |
---|
260 | 278 | const struct rpc_xprt *cur) |
---|
261 | 279 | { |
---|
262 | 280 | struct rpc_xprt *pos, *prev = NULL; |
---|
| 281 | + bool found = false; |
---|
263 | 282 | |
---|
264 | 283 | list_for_each_entry_rcu(pos, head, xprt_switch) { |
---|
265 | 284 | if (cur == prev) |
---|
| 285 | + found = true; |
---|
| 286 | + if (found && xprt_is_active(pos)) |
---|
266 | 287 | return pos; |
---|
267 | 288 | prev = pos; |
---|
268 | 289 | } |
---|
.. | .. |
---|
270 | 291 | } |
---|
271 | 292 | |
---|
272 | 293 | static |
---|
273 | | -struct rpc_xprt *xprt_switch_set_next_cursor(struct list_head *head, |
---|
| 294 | +struct rpc_xprt *xprt_switch_set_next_cursor(struct rpc_xprt_switch *xps, |
---|
274 | 295 | struct rpc_xprt **cursor, |
---|
275 | 296 | xprt_switch_find_xprt_t find_next) |
---|
276 | 297 | { |
---|
277 | | - struct rpc_xprt *cur, *pos, *old; |
---|
| 298 | + struct rpc_xprt *pos, *old; |
---|
278 | 299 | |
---|
279 | | - cur = READ_ONCE(*cursor); |
---|
280 | | - for (;;) { |
---|
281 | | - old = cur; |
---|
282 | | - pos = find_next(head, old); |
---|
283 | | - if (pos == NULL) |
---|
284 | | - break; |
---|
285 | | - cur = cmpxchg_relaxed(cursor, old, pos); |
---|
286 | | - if (cur == old) |
---|
287 | | - break; |
---|
288 | | - } |
---|
| 300 | + old = smp_load_acquire(cursor); |
---|
| 301 | + pos = find_next(xps, old); |
---|
| 302 | + smp_store_release(cursor, pos); |
---|
289 | 303 | return pos; |
---|
290 | 304 | } |
---|
291 | 305 | |
---|
.. | .. |
---|
297 | 311 | |
---|
298 | 312 | if (xps == NULL) |
---|
299 | 313 | return NULL; |
---|
300 | | - return xprt_switch_set_next_cursor(&xps->xps_xprt_list, |
---|
301 | | - &xpi->xpi_cursor, |
---|
302 | | - find_next); |
---|
| 314 | + return xprt_switch_set_next_cursor(xps, &xpi->xpi_cursor, find_next); |
---|
303 | 315 | } |
---|
304 | 316 | |
---|
305 | 317 | static |
---|
306 | | -struct rpc_xprt *xprt_switch_find_next_entry_roundrobin(struct list_head *head, |
---|
| 318 | +struct rpc_xprt *__xprt_switch_find_next_entry_roundrobin(struct list_head *head, |
---|
307 | 319 | const struct rpc_xprt *cur) |
---|
308 | 320 | { |
---|
309 | 321 | struct rpc_xprt *ret; |
---|
.. | .. |
---|
315 | 327 | } |
---|
316 | 328 | |
---|
317 | 329 | static |
---|
| 330 | +struct rpc_xprt *xprt_switch_find_next_entry_roundrobin(struct rpc_xprt_switch *xps, |
---|
| 331 | + const struct rpc_xprt *cur) |
---|
| 332 | +{ |
---|
| 333 | + struct list_head *head = &xps->xps_xprt_list; |
---|
| 334 | + struct rpc_xprt *xprt; |
---|
| 335 | + unsigned int nactive; |
---|
| 336 | + |
---|
| 337 | + for (;;) { |
---|
| 338 | + unsigned long xprt_queuelen, xps_queuelen; |
---|
| 339 | + |
---|
| 340 | + xprt = __xprt_switch_find_next_entry_roundrobin(head, cur); |
---|
| 341 | + if (!xprt) |
---|
| 342 | + break; |
---|
| 343 | + xprt_queuelen = atomic_long_read(&xprt->queuelen); |
---|
| 344 | + xps_queuelen = atomic_long_read(&xps->xps_queuelen); |
---|
| 345 | + nactive = READ_ONCE(xps->xps_nactive); |
---|
| 346 | + /* Exit loop if xprt_queuelen <= average queue length */ |
---|
| 347 | + if (xprt_queuelen * nactive <= xps_queuelen) |
---|
| 348 | + break; |
---|
| 349 | + cur = xprt; |
---|
| 350 | + } |
---|
| 351 | + return xprt; |
---|
| 352 | +} |
---|
| 353 | + |
---|
| 354 | +static |
---|
318 | 355 | struct rpc_xprt *xprt_iter_next_entry_roundrobin(struct rpc_xprt_iter *xpi) |
---|
319 | 356 | { |
---|
320 | 357 | return xprt_iter_next_entry_multiple(xpi, |
---|
.. | .. |
---|
322 | 359 | } |
---|
323 | 360 | |
---|
324 | 361 | static |
---|
| 362 | +struct rpc_xprt *xprt_switch_find_next_entry_all(struct rpc_xprt_switch *xps, |
---|
| 363 | + const struct rpc_xprt *cur) |
---|
| 364 | +{ |
---|
| 365 | + return xprt_switch_find_next_entry(&xps->xps_xprt_list, cur); |
---|
| 366 | +} |
---|
| 367 | + |
---|
| 368 | +static |
---|
325 | 369 | struct rpc_xprt *xprt_iter_next_entry_all(struct rpc_xprt_iter *xpi) |
---|
326 | 370 | { |
---|
327 | | - return xprt_iter_next_entry_multiple(xpi, xprt_switch_find_next_entry); |
---|
| 371 | + return xprt_iter_next_entry_multiple(xpi, |
---|
| 372 | + xprt_switch_find_next_entry_all); |
---|
328 | 373 | } |
---|
329 | 374 | |
---|
330 | 375 | /* |
---|
.. | .. |
---|
383 | 428 | /** |
---|
384 | 429 | * xprt_iter_xchg_switch - Atomically swap out the rpc_xprt_switch |
---|
385 | 430 | * @xpi: pointer to rpc_xprt_iter |
---|
386 | | - * @xps: pointer to a new rpc_xprt_switch or NULL |
---|
| 431 | + * @newswitch: pointer to a new rpc_xprt_switch or NULL |
---|
387 | 432 | * |
---|
388 | 433 | * Swaps out the existing xpi->xpi_xpswitch with a new value. |
---|
389 | 434 | */ |
---|
.. | .. |
---|
401 | 446 | |
---|
402 | 447 | /** |
---|
403 | 448 | * xprt_iter_destroy - Destroys the xprt iterator |
---|
404 | | - * @xpi pointer to rpc_xprt_iter |
---|
| 449 | + * @xpi: pointer to rpc_xprt_iter |
---|
405 | 450 | */ |
---|
406 | 451 | void xprt_iter_destroy(struct rpc_xprt_iter *xpi) |
---|
407 | 452 | { |
---|