forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-08 01573e231f18eb2d99162747186f59511f56b64d 
 kernel/lib/scatterlist.c
....@@ -1,10 +1,8 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * Copyright (C) 2007 Jens Axboe <jens.axboe@oracle.com>
34  *
45  * Scatterlist handling helpers.
5
- *
6
- * This source code is licensed under the GNU General Public License,
7
- * Version 2. See the file COPYING for more details.
86  */
97 #include <linux/export.h>
108 #include <linux/slab.h>
....@@ -181,7 +179,8 @@
181179  * __sg_free_table - Free a previously mapped sg table
182180  * @table:	The sg table header to use
183181  * @max_ents:	The maximum number of entries per single scatterlist
184
- * @skip_first_chunk: don't free the (preallocated) first scatterlist chunk
182
+ * @nents_first_chunk: Number of entries int the (preallocated) first
183
+ * 	scatterlist chunk, 0 means no such preallocated first chunk
185184  * @free_fn:	Free function
186185  *
187186  *  Description:
....@@ -191,9 +190,10 @@
191190  *
192191  **/
193192 void __sg_free_table(struct sg_table *table, unsigned int max_ents,
194
-		     bool skip_first_chunk, sg_free_fn *free_fn)
193
+		     unsigned int nents_first_chunk, sg_free_fn *free_fn)
195194 {
196195 	struct scatterlist *sgl, *next;
196
+	unsigned curr_max_ents = nents_first_chunk ?: max_ents;
197197 
198198 	if (unlikely(!table->sgl))
199199 		return;
....@@ -209,9 +209,9 @@
209209 		 * sg_size is then one less than alloc size, since the last
210210 		 * element is the chain pointer.
211211 		 */
212
-		if (alloc_size > max_ents) {
213
-			next = sg_chain_ptr(&sgl[max_ents - 1]);
214
-			alloc_size = max_ents;
212
+		if (alloc_size > curr_max_ents) {
213
+			next = sg_chain_ptr(&sgl[curr_max_ents - 1]);
214
+			alloc_size = curr_max_ents;
215215 			sg_size = alloc_size - 1;
216216 		} else {
217217 			sg_size = alloc_size;
....@@ -219,11 +219,12 @@
219219 		}
220220 
221221 		table->orig_nents -= sg_size;
222
-		if (skip_first_chunk)
223
-			skip_first_chunk = false;
222
+		if (nents_first_chunk)
223
+			nents_first_chunk = 0;
224224 		else
225225 			free_fn(sgl, alloc_size);
226226 		sgl = next;
227
+		curr_max_ents = max_ents;
227228 	}
228229 
229230 	table->sgl = NULL;
....@@ -246,6 +247,8 @@
246247  * @table:	The sg table header to use
247248  * @nents:	Number of entries in sg list
248249  * @max_ents:	The maximum number of entries the allocator returns per call
250
+ * @nents_first_chunk: Number of entries int the (preallocated) first
251
+ * 	scatterlist chunk, 0 means no such preallocated chunk provided by user
249252  * @gfp_mask:	GFP allocation mask
250253  * @alloc_fn:	Allocator to use
251254  *
....@@ -262,16 +265,19 @@
262265  **/
263266 int __sg_alloc_table(struct sg_table *table, unsigned int nents,
264267 		     unsigned int max_ents, struct scatterlist *first_chunk,
265
-		     gfp_t gfp_mask, sg_alloc_fn *alloc_fn)
268
+		     unsigned int nents_first_chunk, gfp_t gfp_mask,
269
+		     sg_alloc_fn *alloc_fn)
266270 {
267271 	struct scatterlist *sg, *prv;
268272 	unsigned int left;
273
+	unsigned curr_max_ents = nents_first_chunk ?: max_ents;
274
+	unsigned prv_max_ents;
269275 
270276 	memset(table, 0, sizeof(*table));
271277 
272278 	if (nents == 0)
273279 		return -EINVAL;
274
-#ifndef CONFIG_ARCH_HAS_SG_CHAIN
280
+#ifdef CONFIG_ARCH_NO_SG_CHAIN
275281 	if (WARN_ON_ONCE(nents > max_ents))
276282 		return -EINVAL;
277283 #endif
....@@ -281,8 +287,8 @@
281287 	do {
282288 		unsigned int sg_size, alloc_size = left;
283289 
284
-		if (alloc_size > max_ents) {
285
-			alloc_size = max_ents;
290
+		if (alloc_size > curr_max_ents) {
291
+			alloc_size = curr_max_ents;
286292 			sg_size = alloc_size - 1;
287293 		} else
288294 			sg_size = alloc_size;
....@@ -316,7 +322,7 @@
316322 		 * If this is not the first mapping, chain previous part.
317323 		 */
318324 		if (prv)
319
-			sg_chain(prv, max_ents, sg);
325
+			sg_chain(prv, prv_max_ents, sg);
320326 		else
321327 			table->sgl = sg;
322328 
....@@ -327,6 +333,8 @@
327333 			sg_mark_end(&sg[sg_size - 1]);
328334 
329335 		prv = sg;
336
+		prv_max_ents = curr_max_ents;
337
+		curr_max_ents = max_ents;
330338 	} while (left);
331339 
332340 	return 0;
....@@ -349,13 +357,44 @@
349357 	int ret;
350358 
351359 	ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC,
352
-			       NULL, gfp_mask, sg_kmalloc);
360
+			       NULL, 0, gfp_mask, sg_kmalloc);
353361 	if (unlikely(ret))
354
-		__sg_free_table(table, SG_MAX_SINGLE_ALLOC, false, sg_kfree);
362
+		__sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree);
355363 
356364 	return ret;
357365 }
358366 EXPORT_SYMBOL(sg_alloc_table);
367
+
368
+static struct scatterlist *get_next_sg(struct sg_table *table,
369
+				       struct scatterlist *cur,
370
+				       unsigned long needed_sges,
371
+				       gfp_t gfp_mask)
372
+{
373
+	struct scatterlist *new_sg, *next_sg;
374
+	unsigned int alloc_size;
375
+
376
+	if (cur) {
377
+		next_sg = sg_next(cur);
378
+		/* Check if last entry should be keeped for chainning */
379
+		if (!sg_is_last(next_sg) || needed_sges == 1)
380
+			return next_sg;
381
+	}
382
+
383
+	alloc_size = min_t(unsigned long, needed_sges, SG_MAX_SINGLE_ALLOC);
384
+	new_sg = sg_kmalloc(alloc_size, gfp_mask);
385
+	if (!new_sg)
386
+		return ERR_PTR(-ENOMEM);
387
+	sg_init_table(new_sg, alloc_size);
388
+	if (cur) {
389
+		__sg_chain(next_sg, new_sg);
390
+		table->orig_nents += alloc_size - 1;
391
+	} else {
392
+		table->sgl = new_sg;
393
+		table->orig_nents = alloc_size;
394
+		table->nents = 0;
395
+	}
396
+	return new_sg;
397
+}
359398 
360399 /**
361400  * __sg_alloc_table_from_pages - Allocate and initialize an sg table from
....@@ -365,30 +404,69 @@
365404  * @n_pages:	 Number of pages in the pages array
366405  * @offset:      Offset from start of the first page to the start of a buffer
367406  * @size:        Number of valid bytes in the buffer (after offset)
368
- * @max_segment: Maximum size of a scatterlist node in bytes (page aligned)
407
+ * @max_segment: Maximum size of a scatterlist element in bytes
408
+ * @prv:	 Last populated sge in sgt
409
+ * @left_pages:  Left pages caller have to set after this call
369410  * @gfp_mask:	 GFP allocation mask
370411  *
371
- *  Description:
372
- *    Allocate and initialize an sg table from a list of pages. Contiguous
373
- *    ranges of the pages are squashed into a single scatterlist node up to the
374
- *    maximum size specified in @max_segment. An user may provide an offset at a
375
- *    start and a size of valid data in a buffer specified by the page array.
376
- *    The returned sg table is released by sg_free_table.
412
+ * Description:
413
+ *    If @prv is NULL, allocate and initialize an sg table from a list of pages,
414
+ *    else reuse the scatterlist passed in at @prv.
415
+ *    Contiguous ranges of the pages are squashed into a single scatterlist
416
+ *    entry up to the maximum size specified in @max_segment.  A user may
417
+ *    provide an offset at a start and a size of valid data in a buffer
418
+ *    specified by the page array.
377419  *
378420  * Returns:
379
- *   0 on success, negative error on failure
421
+ *   Last SGE in sgt on success, PTR_ERR on otherwise.
422
+ *   The allocation in @sgt must be released by sg_free_table.
423
+ *
424
+ * Notes:
425
+ *   If this function returns non-0 (eg failure), the caller must call
426
+ *   sg_free_table() to cleanup any leftover allocations.
380427  */
381
-int __sg_alloc_table_from_pages(struct sg_table *sgt, struct page **pages,
382
-				unsigned int n_pages, unsigned int offset,
383
-				unsigned long size, unsigned int max_segment,
384
-				gfp_t gfp_mask)
428
+struct scatterlist *__sg_alloc_table_from_pages(struct sg_table *sgt,
429
+		struct page **pages, unsigned int n_pages, unsigned int offset,
430
+		unsigned long size, unsigned int max_segment,
431
+		struct scatterlist *prv, unsigned int left_pages,
432
+		gfp_t gfp_mask)
385433 {
386
-	unsigned int chunks, cur_page, seg_len, i;
387
-	int ret;
388
-	struct scatterlist *s;
434
+	unsigned int chunks, cur_page, seg_len, i, prv_len = 0;
435
+	unsigned int added_nents = 0;
436
+	struct scatterlist *s = prv;
389437 
390
-	if (WARN_ON(!max_segment || offset_in_page(max_segment)))
391
-		return -EINVAL;
438
+	/*
439
+	 * The algorithm below requires max_segment to be aligned to PAGE_SIZE
440
+	 * otherwise it can overshoot.
441
+	 */
442
+	max_segment = ALIGN_DOWN(max_segment, PAGE_SIZE);
443
+	if (WARN_ON(max_segment < PAGE_SIZE))
444
+		return ERR_PTR(-EINVAL);
445
+
446
+	if (IS_ENABLED(CONFIG_ARCH_NO_SG_CHAIN) && prv)
447
+		return ERR_PTR(-EOPNOTSUPP);
448
+
449
+	if (prv) {
450
+		unsigned long paddr = (page_to_pfn(sg_page(prv)) * PAGE_SIZE +
451
+				       prv->offset + prv->length) /
452
+				      PAGE_SIZE;
453
+
454
+		if (WARN_ON(offset))
455
+			return ERR_PTR(-EINVAL);
456
+
457
+		/* Merge contiguous pages into the last SG */
458
+		prv_len = prv->length;
459
+		while (n_pages && page_to_pfn(pages[0]) == paddr) {
460
+			if (prv->length + PAGE_SIZE > max_segment)
461
+				break;
462
+			prv->length += PAGE_SIZE;
463
+			paddr++;
464
+			pages++;
465
+			n_pages--;
466
+		}
467
+		if (!n_pages)
468
+			goto out;
469
+	}
392470 
393471 	/* compute number of contiguous chunks */
394472 	chunks = 1;
....@@ -402,13 +480,9 @@
402480 		}
403481 	}
404482 
405
-	ret = sg_alloc_table(sgt, chunks, gfp_mask);
406
-	if (unlikely(ret))
407
-		return ret;
408
-
409483 	/* merging chunks and putting them into the scatterlist */
410484 	cur_page = 0;
411
-	for_each_sg(sgt->sgl, s, sgt->orig_nents, i) {
485
+	for (i = 0; i < chunks; i++) {
412486 		unsigned int j, chunk_size;
413487 
414488 		/* look for the end of the current chunk */
....@@ -421,15 +495,30 @@
421495 				break;
422496 		}
423497 
498
+		/* Pass how many chunks might be left */
499
+		s = get_next_sg(sgt, s, chunks - i + left_pages, gfp_mask);
500
+		if (IS_ERR(s)) {
501
+			/*
502
+			 * Adjust entry length to be as before function was
503
+			 * called.
504
+			 */
505
+			if (prv)
506
+				prv->length = prv_len;
507
+			return s;
508
+		}
424509 		chunk_size = ((j - cur_page) << PAGE_SHIFT) - offset;
425510 		sg_set_page(s, pages[cur_page],
426511 			    min_t(unsigned long, size, chunk_size), offset);
512
+		added_nents++;
427513 		size -= chunk_size;
428514 		offset = 0;
429515 		cur_page = j;
430516 	}
431
-
432
-	return 0;
517
+	sgt->nents += added_nents;
518
+out:
519
+	if (!left_pages)
520
+		sg_mark_end(s);
521
+	return s;
433522 }
434523 EXPORT_SYMBOL(__sg_alloc_table_from_pages);
435524 
....@@ -457,8 +546,8 @@
457546 			      unsigned int n_pages, unsigned int offset,
458547 			      unsigned long size, gfp_t gfp_mask)
459548 {
460
-	return __sg_alloc_table_from_pages(sgt, pages, n_pages, offset, size,
461
-					   SCATTERLIST_MAX_SEGMENT, gfp_mask);
549
+	return PTR_ERR_OR_ZERO(__sg_alloc_table_from_pages(sgt, pages, n_pages,
550
+			offset, size, UINT_MAX, NULL, 0, gfp_mask));
462551 }
463552 EXPORT_SYMBOL(sg_alloc_table_from_pages);
464553 
....@@ -496,7 +585,7 @@
496585 		nalloc++;
497586 	}
498587 	sgl = kmalloc_array(nalloc, sizeof(struct scatterlist),
499
-			    (gfp & ~GFP_DMA) | __GFP_ZERO);
588
+			    gfp & ~GFP_DMA);
500589 	if (!sgl)
501590 		return NULL;
502591 
....@@ -624,6 +713,32 @@
624713 	return true;
625714 }
626715 EXPORT_SYMBOL(__sg_page_iter_next);
716
+
717
+static int sg_dma_page_count(struct scatterlist *sg)
718
+{
719
+	return PAGE_ALIGN(sg->offset + sg_dma_len(sg)) >> PAGE_SHIFT;
720
+}
721
+
722
+bool __sg_page_iter_dma_next(struct sg_dma_page_iter *dma_iter)
723
+{
724
+	struct sg_page_iter *piter = &dma_iter->base;
725
+
726
+	if (!piter->__nents || !piter->sg)
727
+		return false;
728
+
729
+	piter->sg_pgoffset += piter->__pg_advance;
730
+	piter->__pg_advance = 1;
731
+
732
+	while (piter->sg_pgoffset >= sg_dma_page_count(piter->sg)) {
733
+		piter->sg_pgoffset -= sg_dma_page_count(piter->sg);
734
+		piter->sg = sg_next(piter->sg);
735
+		if (!--piter->__nents || !piter->sg)
736
+			return false;
737
+	}
738
+
739
+	return true;
740
+}
741
+EXPORT_SYMBOL(__sg_page_iter_dma_next);
627742 
628743 /**
629744  * sg_miter_start - start mapping iteration over a sg list
....@@ -777,7 +892,7 @@
777892 			flush_kernel_dcache_page(miter->page);
778893 
779894 		if (miter->__flags & SG_MITER_ATOMIC) {
780
-			WARN_ON_ONCE(preemptible());
895
+			WARN_ON_ONCE(!pagefault_disabled());
781896 			kunmap_atomic(miter->addr);
782897 		} else
783898 			kunmap(miter->page);
....@@ -798,7 +913,7 @@
798913  * @buflen:		 The number of bytes to copy
799914  * @skip:		 Number of bytes to skip before copying
800915  * @to_buffer:		 transfer direction (true == from an sg list to a
801
- *			 buffer, false == from a buffer to an sg list
916
+ *			 buffer, false == from a buffer to an sg list)
802917  *
803918  * Returns the number of copied bytes.
804919  *
....@@ -818,7 +933,7 @@
818933 	sg_miter_start(&miter, sgl, nents, sg_flags);
819934 
820935 	if (!sg_miter_skip(&miter, skip))
821
-		return false;
936
+		return 0;
822937 
823938 	while ((offset < buflen) && sg_miter_next(&miter)) {
824939 		unsigned int len;