hc
2024-10-22 8ac6c7a54ed1b98d142dce24b11c6de6a1e239a5
kernel/net/sunrpc/xprtmultipath.c
....@@ -19,7 +19,7 @@
1919 #include <linux/sunrpc/addr.h>
2020 #include <linux/sunrpc/xprtmultipath.h>
2121
22
-typedef struct rpc_xprt *(*xprt_switch_find_xprt_t)(struct list_head *head,
22
+typedef struct rpc_xprt *(*xprt_switch_find_xprt_t)(struct rpc_xprt_switch *xps,
2323 const struct rpc_xprt *cur);
2424
2525 static const struct rpc_xprt_iter_ops rpc_xprt_iter_singular;
....@@ -36,6 +36,7 @@
3636 if (xps->xps_nxprts == 0)
3737 xps->xps_net = xprt->xprt_net;
3838 xps->xps_nxprts++;
39
+ xps->xps_nactive++;
3940 }
4041
4142 /**
....@@ -51,8 +52,7 @@
5152 if (xprt == NULL)
5253 return;
5354 spin_lock(&xps->xps_lock);
54
- if ((xps->xps_net == xprt->xprt_net || xps->xps_net == NULL) &&
55
- !rpc_xprt_switch_has_addr(xps, (struct sockaddr *)&xprt->addr))
55
+ if (xps->xps_net == xprt->xprt_net || xps->xps_net == NULL)
5656 xprt_switch_add_xprt_locked(xps, xprt);
5757 spin_unlock(&xps->xps_lock);
5858 }
....@@ -62,6 +62,7 @@
6262 {
6363 if (unlikely(xprt == NULL))
6464 return;
65
+ xps->xps_nactive--;
6566 xps->xps_nxprts--;
6667 if (xps->xps_nxprts == 0)
6768 xps->xps_net = NULL;
....@@ -102,7 +103,9 @@
102103 if (xps != NULL) {
103104 spin_lock_init(&xps->xps_lock);
104105 kref_init(&xps->xps_kref);
105
- xps->xps_nxprts = 0;
106
+ xps->xps_nxprts = xps->xps_nactive = 0;
107
+ atomic_long_set(&xps->xps_queuelen, 0);
108
+ xps->xps_net = NULL;
106109 INIT_LIST_HEAD(&xps->xps_xprt_list);
107110 xps->xps_iter_ops = &rpc_xprt_iter_singular;
108111 xprt_switch_add_xprt_locked(xps, xprt);
....@@ -193,9 +196,21 @@
193196 }
194197
195198 static
199
+bool xprt_is_active(const struct rpc_xprt *xprt)
200
+{
201
+ return kref_read(&xprt->kref) != 0;
202
+}
203
+
204
+static
196205 struct rpc_xprt *xprt_switch_find_first_entry(struct list_head *head)
197206 {
198
- return list_first_or_null_rcu(head, struct rpc_xprt, xprt_switch);
207
+ struct rpc_xprt *pos;
208
+
209
+ list_for_each_entry_rcu(pos, head, xprt_switch) {
210
+ if (xprt_is_active(pos))
211
+ return pos;
212
+ }
213
+ return NULL;
199214 }
200215
201216 static
....@@ -213,9 +228,12 @@
213228 const struct rpc_xprt *cur)
214229 {
215230 struct rpc_xprt *pos;
231
+ bool found = false;
216232
217233 list_for_each_entry_rcu(pos, head, xprt_switch) {
218234 if (cur == pos)
235
+ found = true;
236
+ if (found && xprt_is_active(pos))
219237 return pos;
220238 }
221239 return NULL;
....@@ -260,9 +278,12 @@
260278 const struct rpc_xprt *cur)
261279 {
262280 struct rpc_xprt *pos, *prev = NULL;
281
+ bool found = false;
263282
264283 list_for_each_entry_rcu(pos, head, xprt_switch) {
265284 if (cur == prev)
285
+ found = true;
286
+ if (found && xprt_is_active(pos))
266287 return pos;
267288 prev = pos;
268289 }
....@@ -270,22 +291,15 @@
270291 }
271292
272293 static
273
-struct rpc_xprt *xprt_switch_set_next_cursor(struct list_head *head,
294
+struct rpc_xprt *xprt_switch_set_next_cursor(struct rpc_xprt_switch *xps,
274295 struct rpc_xprt **cursor,
275296 xprt_switch_find_xprt_t find_next)
276297 {
277
- struct rpc_xprt *cur, *pos, *old;
298
+ struct rpc_xprt *pos, *old;
278299
279
- cur = READ_ONCE(*cursor);
280
- for (;;) {
281
- old = cur;
282
- pos = find_next(head, old);
283
- if (pos == NULL)
284
- break;
285
- cur = cmpxchg_relaxed(cursor, old, pos);
286
- if (cur == old)
287
- break;
288
- }
300
+ old = smp_load_acquire(cursor);
301
+ pos = find_next(xps, old);
302
+ smp_store_release(cursor, pos);
289303 return pos;
290304 }
291305
....@@ -297,13 +311,11 @@
297311
298312 if (xps == NULL)
299313 return NULL;
300
- return xprt_switch_set_next_cursor(&xps->xps_xprt_list,
301
- &xpi->xpi_cursor,
302
- find_next);
314
+ return xprt_switch_set_next_cursor(xps, &xpi->xpi_cursor, find_next);
303315 }
304316
305317 static
306
-struct rpc_xprt *xprt_switch_find_next_entry_roundrobin(struct list_head *head,
318
+struct rpc_xprt *__xprt_switch_find_next_entry_roundrobin(struct list_head *head,
307319 const struct rpc_xprt *cur)
308320 {
309321 struct rpc_xprt *ret;
....@@ -315,6 +327,31 @@
315327 }
316328
317329 static
330
+struct rpc_xprt *xprt_switch_find_next_entry_roundrobin(struct rpc_xprt_switch *xps,
331
+ const struct rpc_xprt *cur)
332
+{
333
+ struct list_head *head = &xps->xps_xprt_list;
334
+ struct rpc_xprt *xprt;
335
+ unsigned int nactive;
336
+
337
+ for (;;) {
338
+ unsigned long xprt_queuelen, xps_queuelen;
339
+
340
+ xprt = __xprt_switch_find_next_entry_roundrobin(head, cur);
341
+ if (!xprt)
342
+ break;
343
+ xprt_queuelen = atomic_long_read(&xprt->queuelen);
344
+ xps_queuelen = atomic_long_read(&xps->xps_queuelen);
345
+ nactive = READ_ONCE(xps->xps_nactive);
346
+ /* Exit loop if xprt_queuelen <= average queue length */
347
+ if (xprt_queuelen * nactive <= xps_queuelen)
348
+ break;
349
+ cur = xprt;
350
+ }
351
+ return xprt;
352
+}
353
+
354
+static
318355 struct rpc_xprt *xprt_iter_next_entry_roundrobin(struct rpc_xprt_iter *xpi)
319356 {
320357 return xprt_iter_next_entry_multiple(xpi,
....@@ -322,9 +359,17 @@
322359 }
323360
324361 static
362
+struct rpc_xprt *xprt_switch_find_next_entry_all(struct rpc_xprt_switch *xps,
363
+ const struct rpc_xprt *cur)
364
+{
365
+ return xprt_switch_find_next_entry(&xps->xps_xprt_list, cur);
366
+}
367
+
368
+static
325369 struct rpc_xprt *xprt_iter_next_entry_all(struct rpc_xprt_iter *xpi)
326370 {
327
- return xprt_iter_next_entry_multiple(xpi, xprt_switch_find_next_entry);
371
+ return xprt_iter_next_entry_multiple(xpi,
372
+ xprt_switch_find_next_entry_all);
328373 }
329374
330375 /*
....@@ -383,7 +428,7 @@
383428 /**
384429 * xprt_iter_xchg_switch - Atomically swap out the rpc_xprt_switch
385430 * @xpi: pointer to rpc_xprt_iter
386
- * @xps: pointer to a new rpc_xprt_switch or NULL
431
+ * @newswitch: pointer to a new rpc_xprt_switch or NULL
387432 *
388433 * Swaps out the existing xpi->xpi_xpswitch with a new value.
389434 */
....@@ -401,7 +446,7 @@
401446
402447 /**
403448 * xprt_iter_destroy - Destroys the xprt iterator
404
- * @xpi pointer to rpc_xprt_iter
449
+ * @xpi: pointer to rpc_xprt_iter
405450 */
406451 void xprt_iter_destroy(struct rpc_xprt_iter *xpi)
407452 {