write in 30M

hc

2024-02-20 ea08eeccae9297f7aabd2ef7f0c2517ac4549acc

 kernel/net/sunrpc/xprtmultipath.c
..
..
@@ -19,7 +19,7 @@
19
19
 #include <linux/sunrpc/addr.h>
20
20
 #include <linux/sunrpc/xprtmultipath.h>
21
21
 
22
-typedef struct rpc_xprt *(*xprt_switch_find_xprt_t)(struct list_head *head,
22
+typedef struct rpc_xprt *(*xprt_switch_find_xprt_t)(struct rpc_xprt_switch *xps,
23
23
 		const struct rpc_xprt *cur);
24
24
 
25
25
 static const struct rpc_xprt_iter_ops rpc_xprt_iter_singular;
..
..
@@ -36,6 +36,7 @@
36
36
 	if (xps->xps_nxprts == 0)
37
37
 		xps->xps_net = xprt->xprt_net;
38
38
 	xps->xps_nxprts++;
39
+	xps->xps_nactive++;
39
40
 }
40
41
 
41
42
 /**
..
..
@@ -51,8 +52,7 @@
51
52
 	if (xprt == NULL)
52
53
 		return;
53
54
 	spin_lock(&xps->xps_lock);
54
-	if ((xps->xps_net == xprt->xprt_net || xps->xps_net == NULL) &&
55
-	    !rpc_xprt_switch_has_addr(xps, (struct sockaddr *)&xprt->addr))
55
+	if (xps->xps_net == xprt->xprt_net || xps->xps_net == NULL)
56
56
 		xprt_switch_add_xprt_locked(xps, xprt);
57
57
 	spin_unlock(&xps->xps_lock);
58
58
 }
..
..
@@ -62,6 +62,7 @@
62
62
 {
63
63
 	if (unlikely(xprt == NULL))
64
64
 		return;
65
+	xps->xps_nactive--;
65
66
 	xps->xps_nxprts--;
66
67
 	if (xps->xps_nxprts == 0)
67
68
 		xps->xps_net = NULL;
..
..
@@ -102,7 +103,9 @@
102
103
 	if (xps != NULL) {
103
104
 		spin_lock_init(&xps->xps_lock);
104
105
 		kref_init(&xps->xps_kref);
105
-		xps->xps_nxprts = 0;
106
+		xps->xps_nxprts = xps->xps_nactive = 0;
107
+		atomic_long_set(&xps->xps_queuelen, 0);
108
+		xps->xps_net = NULL;
106
109
 		INIT_LIST_HEAD(&xps->xps_xprt_list);
107
110
 		xps->xps_iter_ops = &rpc_xprt_iter_singular;
108
111
 		xprt_switch_add_xprt_locked(xps, xprt);
..
..
@@ -193,9 +196,21 @@
193
196
 }
194
197
 
195
198
 static
199
+bool xprt_is_active(const struct rpc_xprt *xprt)
200
+{
201
+	return kref_read(&xprt->kref) != 0;
202
+}
203
+
204
+static
196
205
 struct rpc_xprt *xprt_switch_find_first_entry(struct list_head *head)
197
206
 {
198
-	return list_first_or_null_rcu(head, struct rpc_xprt, xprt_switch);
207
+	struct rpc_xprt *pos;
208
+
209
+	list_for_each_entry_rcu(pos, head, xprt_switch) {
210
+		if (xprt_is_active(pos))
211
+			return pos;
212
+	}
213
+	return NULL;
199
214
 }
200
215
 
201
216
 static
..
..
@@ -213,9 +228,12 @@
213
228
 		const struct rpc_xprt *cur)
214
229
 {
215
230
 	struct rpc_xprt *pos;
231
+	bool found = false;
216
232
 
217
233
 	list_for_each_entry_rcu(pos, head, xprt_switch) {
218
234
 		if (cur == pos)
235
+			found = true;
236
+		if (found && xprt_is_active(pos))
219
237
 			return pos;
220
238
 	}
221
239
 	return NULL;
..
..
@@ -260,9 +278,12 @@
260
278
 		const struct rpc_xprt *cur)
261
279
 {
262
280
 	struct rpc_xprt *pos, *prev = NULL;
281
+	bool found = false;
263
282
 
264
283
 	list_for_each_entry_rcu(pos, head, xprt_switch) {
265
284
 		if (cur == prev)
285
+			found = true;
286
+		if (found && xprt_is_active(pos))
266
287
 			return pos;
267
288
 		prev = pos;
268
289
 	}
..
..
@@ -270,22 +291,15 @@
270
291
 }
271
292
 
272
293
 static
273
-struct rpc_xprt *xprt_switch_set_next_cursor(struct list_head *head,
294
+struct rpc_xprt *xprt_switch_set_next_cursor(struct rpc_xprt_switch *xps,
274
295
 		struct rpc_xprt **cursor,
275
296
 		xprt_switch_find_xprt_t find_next)
276
297
 {
277
-	struct rpc_xprt *cur, *pos, *old;
298
+	struct rpc_xprt *pos, *old;
278
299
 
279
-	cur = READ_ONCE(*cursor);
280
-	for (;;) {
281
-		old = cur;
282
-		pos = find_next(head, old);
283
-		if (pos == NULL)
284
-			break;
285
-		cur = cmpxchg_relaxed(cursor, old, pos);
286
-		if (cur == old)
287
-			break;
288
-	}
300
+	old = smp_load_acquire(cursor);
301
+	pos = find_next(xps, old);
302
+	smp_store_release(cursor, pos);
289
303
 	return pos;
290
304
 }
291
305
 
..
..
@@ -297,13 +311,11 @@
297
311
 
298
312
 	if (xps == NULL)
299
313
 		return NULL;
300
-	return xprt_switch_set_next_cursor(&xps->xps_xprt_list,
301
-			&xpi->xpi_cursor,
302
-			find_next);
314
+	return xprt_switch_set_next_cursor(xps, &xpi->xpi_cursor, find_next);
303
315
 }
304
316
 
305
317
 static
306
-struct rpc_xprt *xprt_switch_find_next_entry_roundrobin(struct list_head *head,
318
+struct rpc_xprt *__xprt_switch_find_next_entry_roundrobin(struct list_head *head,
307
319
 		const struct rpc_xprt *cur)
308
320
 {
309
321
 	struct rpc_xprt *ret;
..
..
@@ -315,6 +327,31 @@
315
327
 }
316
328
 
317
329
 static
330
+struct rpc_xprt *xprt_switch_find_next_entry_roundrobin(struct rpc_xprt_switch *xps,
331
+		const struct rpc_xprt *cur)
332
+{
333
+	struct list_head *head = &xps->xps_xprt_list;
334
+	struct rpc_xprt *xprt;
335
+	unsigned int nactive;
336
+
337
+	for (;;) {
338
+		unsigned long xprt_queuelen, xps_queuelen;
339
+
340
+		xprt = __xprt_switch_find_next_entry_roundrobin(head, cur);
341
+		if (!xprt)
342
+			break;
343
+		xprt_queuelen = atomic_long_read(&xprt->queuelen);
344
+		xps_queuelen = atomic_long_read(&xps->xps_queuelen);
345
+		nactive = READ_ONCE(xps->xps_nactive);
346
+		/* Exit loop if xprt_queuelen <= average queue length */
347
+		if (xprt_queuelen * nactive <= xps_queuelen)
348
+			break;
349
+		cur = xprt;
350
+	}
351
+	return xprt;
352
+}
353
+
354
+static
318
355
 struct rpc_xprt *xprt_iter_next_entry_roundrobin(struct rpc_xprt_iter *xpi)
319
356
 {
320
357
 	return xprt_iter_next_entry_multiple(xpi,
..
..
@@ -322,9 +359,17 @@
322
359
 }
323
360
 
324
361
 static
362
+struct rpc_xprt *xprt_switch_find_next_entry_all(struct rpc_xprt_switch *xps,
363
+		const struct rpc_xprt *cur)
364
+{
365
+	return xprt_switch_find_next_entry(&xps->xps_xprt_list, cur);
366
+}
367
+
368
+static
325
369
 struct rpc_xprt *xprt_iter_next_entry_all(struct rpc_xprt_iter *xpi)
326
370
 {
327
-	return xprt_iter_next_entry_multiple(xpi, xprt_switch_find_next_entry);
371
+	return xprt_iter_next_entry_multiple(xpi,
372
+			xprt_switch_find_next_entry_all);
328
373
 }
329
374
 
330
375
 /*
..
..
@@ -383,7 +428,7 @@
383
428
 /**
384
429
  * xprt_iter_xchg_switch - Atomically swap out the rpc_xprt_switch
385
430
  * @xpi: pointer to rpc_xprt_iter
386
- * @xps: pointer to a new rpc_xprt_switch or NULL
431
+ * @newswitch: pointer to a new rpc_xprt_switch or NULL
387
432
  *
388
433
  * Swaps out the existing xpi->xpi_xpswitch with a new value.
389
434
  */
..
..
@@ -401,7 +446,7 @@
401
446
 
402
447
 /**
403
448
  * xprt_iter_destroy - Destroys the xprt iterator
404
- * @xpi pointer to rpc_xprt_iter
449
+ * @xpi: pointer to rpc_xprt_iter
405
450
  */
406
451
 void xprt_iter_destroy(struct rpc_xprt_iter *xpi)
407
452
 {