update kernel to 5.10.198

hc

2024-01-03 2f7c68cb55ecb7331f2381deb497c27155f32faf

 kernel/drivers/infiniband/hw/hfi1/mmu_rb.c
..
..
@@ -60,8 +60,7 @@
60
60
 		const struct mmu_notifier_range *);
61
61
 static struct mmu_rb_node *__mmu_rb_search(struct mmu_rb_handler *,
62
62
 					   unsigned long, unsigned long);
63
-static void do_remove(struct mmu_rb_handler *handler,
64
-		      struct list_head *del_list);
63
+static void release_immediate(struct kref *refcount);
65
64
 static void handle_remove(struct work_struct *work);
66
65
 
67
66
 static const struct mmu_notifier_ops mn_opts = {
..
..
@@ -144,7 +143,11 @@
144
143
 	}
145
144
 	spin_unlock_irqrestore(&handler->lock, flags);
146
145
 
147
-	do_remove(handler, &del_list);
146
+	while (!list_empty(&del_list)) {
147
+		rbnode = list_first_entry(&del_list, struct mmu_rb_node, list);
148
+		list_del(&rbnode->list);
149
+		kref_put(&rbnode->refcount, release_immediate);
150
+	}
148
151
 
149
152
 	/* Now the mm may be freed. */
150
153
 	mmdrop(handler->mn.mm);
..
..
@@ -167,21 +170,28 @@
167
170
 	spin_lock_irqsave(&handler->lock, flags);
168
171
 	node = __mmu_rb_search(handler, mnode->addr, mnode->len);
169
172
 	if (node) {
170
-		ret = -EINVAL;
173
+		ret = -EEXIST;
171
174
 		goto unlock;
172
175
 	}
173
176
 	__mmu_int_rb_insert(mnode, &handler->root);
174
-	list_add(&mnode->list, &handler->lru_list);
175
-
176
-	ret = handler->ops->insert(handler->ops_arg, mnode);
177
-	if (ret) {
178
-		__mmu_int_rb_remove(mnode, &handler->root);
179
-		list_del(&mnode->list); /* remove from LRU list */
180
-	}
177
+	list_add_tail(&mnode->list, &handler->lru_list);
181
178
 	mnode->handler = handler;
182
179
 unlock:
183
180
 	spin_unlock_irqrestore(&handler->lock, flags);
184
181
 	return ret;
182
+}
183
+
184
+/* Caller must hold handler lock */
185
+struct mmu_rb_node *hfi1_mmu_rb_get_first(struct mmu_rb_handler *handler,
186
+					  unsigned long addr, unsigned long len)
187
+{
188
+	struct mmu_rb_node *node;
189
+
190
+	trace_hfi1_mmu_rb_search(addr, len);
191
+	node = __mmu_int_rb_iter_first(&handler->root, addr, (addr + len) - 1);
192
+	if (node)
193
+		list_move_tail(&node->list, &handler->lru_list);
194
+	return node;
185
195
 }
186
196
 
187
197
 /* Caller must hold handler lock */
..
..
@@ -208,30 +218,46 @@
208
218
 	return node;
209
219
 }
210
220
 
211
-bool hfi1_mmu_rb_remove_unless_exact(struct mmu_rb_handler *handler,
212
-				     unsigned long addr, unsigned long len,
213
-				     struct mmu_rb_node **rb_node)
221
+/*
222
+ * Must NOT call while holding mnode->handler->lock.
223
+ * mnode->handler->ops->remove() may sleep and mnode->handler->lock is a
224
+ * spinlock.
225
+ */
226
+static void release_immediate(struct kref *refcount)
214
227
 {
215
-	struct mmu_rb_node *node;
216
-	unsigned long flags;
217
-	bool ret = false;
228
+	struct mmu_rb_node *mnode =
229
+		container_of(refcount, struct mmu_rb_node, refcount);
230
+	mnode->handler->ops->remove(mnode->handler->ops_arg, mnode);
231
+}
218
232
 
219
-	if (current->mm != handler->mn.mm)
220
-		return ret;
233
+/* Caller must hold mnode->handler->lock */
234
+static void release_nolock(struct kref *refcount)
235
+{
236
+	struct mmu_rb_node *mnode =
237
+		container_of(refcount, struct mmu_rb_node, refcount);
238
+	list_move(&mnode->list, &mnode->handler->del_list);
239
+	queue_work(mnode->handler->wq, &mnode->handler->del_work);
240
+}
241
+
242
+/*
243
+ * struct mmu_rb_node->refcount kref_put() callback.
244
+ * Adds mmu_rb_node to mmu_rb_node->handler->del_list and queues
245
+ * handler->del_work on handler->wq.
246
+ * Does not remove mmu_rb_node from handler->lru_list or handler->rb_root.
247
+ * Acquires mmu_rb_node->handler->lock; do not call while already holding
248
+ * handler->lock.
249
+ */
250
+void hfi1_mmu_rb_release(struct kref *refcount)
251
+{
252
+	struct mmu_rb_node *mnode =
253
+		container_of(refcount, struct mmu_rb_node, refcount);
254
+	struct mmu_rb_handler *handler = mnode->handler;
255
+	unsigned long flags;
221
256
 
222
257
 	spin_lock_irqsave(&handler->lock, flags);
223
-	node = __mmu_rb_search(handler, addr, len);
224
-	if (node) {
225
-		if (node->addr == addr && node->len == len)
226
-			goto unlock;
227
-		__mmu_int_rb_remove(node, &handler->root);
228
-		list_del(&node->list); /* remove from LRU list */
229
-		ret = true;
230
-	}
231
-unlock:
258
+	list_move(&mnode->list, &mnode->handler->del_list);
232
259
 	spin_unlock_irqrestore(&handler->lock, flags);
233
-	*rb_node = node;
234
-	return ret;
260
+	queue_work(handler->wq, &handler->del_work);
235
261
 }
236
262
 
237
263
 void hfi1_mmu_rb_evict(struct mmu_rb_handler *handler, void *evict_arg)
..
..
@@ -247,8 +273,11 @@
247
273
 	INIT_LIST_HEAD(&del_list);
248
274
 
249
275
 	spin_lock_irqsave(&handler->lock, flags);
250
-	list_for_each_entry_safe_reverse(rbnode, ptr, &handler->lru_list,
251
-					 list) {
276
+	list_for_each_entry_safe(rbnode, ptr, &handler->lru_list, list) {
277
+		/* refcount == 1 implies mmu_rb_handler has only rbnode ref */
278
+		if (kref_read(&rbnode->refcount) > 1)
279
+			continue;
280
+
252
281
 		if (handler->ops->evict(handler->ops_arg, rbnode, evict_arg,
253
282
 					&stop)) {
254
283
 			__mmu_int_rb_remove(rbnode, &handler->root);
..
..
@@ -260,34 +289,9 @@
260
289
 	}
261
290
 	spin_unlock_irqrestore(&handler->lock, flags);
262
291
 
263
-	while (!list_empty(&del_list)) {
264
-		rbnode = list_first_entry(&del_list, struct mmu_rb_node, list);
265
-		list_del(&rbnode->list);
266
-		handler->ops->remove(handler->ops_arg, rbnode);
292
+	list_for_each_entry_safe(rbnode, ptr, &del_list, list) {
293
+		kref_put(&rbnode->refcount, release_immediate);
267
294
 	}
268
-}
269
-
270
-/*
271
- * It is up to the caller to ensure that this function does not race with the
272
- * mmu invalidate notifier which may be calling the users remove callback on
273
- * 'node'.
274
- */
275
-void hfi1_mmu_rb_remove(struct mmu_rb_handler *handler,
276
-			struct mmu_rb_node *node)
277
-{
278
-	unsigned long flags;
279
-
280
-	if (current->mm != handler->mn.mm)
281
-		return;
282
-
283
-	/* Validity of handler and node pointers has been checked by caller. */
284
-	trace_hfi1_mmu_rb_remove(node->addr, node->len);
285
-	spin_lock_irqsave(&handler->lock, flags);
286
-	__mmu_int_rb_remove(node, &handler->root);
287
-	list_del(&node->list); /* remove from LRU list */
288
-	spin_unlock_irqrestore(&handler->lock, flags);
289
-
290
-	handler->ops->remove(handler->ops_arg, node);
291
295
 }
292
296
 
293
297
 static int mmu_notifier_range_start(struct mmu_notifier *mn,
..
..
@@ -298,7 +302,6 @@
298
302
 	struct rb_root_cached *root = &handler->root;
299
303
 	struct mmu_rb_node *node, *ptr = NULL;
300
304
 	unsigned long flags;
301
-	bool added = false;
302
305
 
303
306
 	spin_lock_irqsave(&handler->lock, flags);
304
307
 	for (node = __mmu_int_rb_iter_first(root, range->start, range->end-1);
..
..
@@ -307,36 +310,14 @@
307
310
 		ptr = __mmu_int_rb_iter_next(node, range->start,
308
311
 					     range->end - 1);
309
312
 		trace_hfi1_mmu_mem_invalidate(node->addr, node->len);
310
-		if (handler->ops->invalidate(handler->ops_arg, node)) {
311
-			__mmu_int_rb_remove(node, root);
312
-			/* move from LRU list to delete list */
313
-			list_move(&node->list, &handler->del_list);
314
-			added = true;
315
-		}
313
+		/* Remove from rb tree and lru_list. */
314
+		__mmu_int_rb_remove(node, root);
315
+		list_del_init(&node->list);
316
+		kref_put(&node->refcount, release_nolock);
316
317
 	}
317
318
 	spin_unlock_irqrestore(&handler->lock, flags);
318
319
 
319
-	if (added)
320
-		queue_work(handler->wq, &handler->del_work);
321
-
322
320
 	return 0;
323
-}
324
-
325
-/*
326
- * Call the remove function for the given handler and the list.  This
327
- * is expected to be called with a delete list extracted from handler.
328
- * The caller should not be holding the handler lock.
329
- */
330
-static void do_remove(struct mmu_rb_handler *handler,
331
-		      struct list_head *del_list)
332
-{
333
-	struct mmu_rb_node *node;
334
-
335
-	while (!list_empty(del_list)) {
336
-		node = list_first_entry(del_list, struct mmu_rb_node, list);
337
-		list_del(&node->list);
338
-		handler->ops->remove(handler->ops_arg, node);
339
-	}
340
321
 }
341
322
 
342
323
 /*
..
..
@@ -351,11 +332,16 @@
351
332
 						del_work);
352
333
 	struct list_head del_list;
353
334
 	unsigned long flags;
335
+	struct mmu_rb_node *node;
354
336
 
355
337
 	/* remove anything that is queued to get removed */
356
338
 	spin_lock_irqsave(&handler->lock, flags);
357
339
 	list_replace_init(&handler->del_list, &del_list);
358
340
 	spin_unlock_irqrestore(&handler->lock, flags);
359
341
 
360
-	do_remove(handler, &del_list);
342
+	while (!list_empty(&del_list)) {
343
+		node = list_first_entry(&del_list, struct mmu_rb_node, list);
344
+		list_del(&node->list);
345
+		handler->ops->remove(handler->ops_arg, node);
346
+	}
361
347
 }