hc
2024-10-22 8ac6c7a54ed1b98d142dce24b11c6de6a1e239a5
kernel/include/linux/scatterlist.h
....@@ -165,6 +165,22 @@
165165 #define for_each_sgtable_dma_sg(sgt, sg, i) \
166166 for_each_sg((sgt)->sgl, sg, (sgt)->nents, i)
167167
168
+static inline void __sg_chain(struct scatterlist *chain_sg,
169
+ struct scatterlist *sgl)
170
+{
171
+ /*
172
+ * offset and length are unused for chain entry. Clear them.
173
+ */
174
+ chain_sg->offset = 0;
175
+ chain_sg->length = 0;
176
+
177
+ /*
178
+ * Set lowest bit to indicate a link pointer, and make sure to clear
179
+ * the termination bit if it happens to be set.
180
+ */
181
+ chain_sg->page_link = ((unsigned long) sgl | SG_CHAIN) & ~SG_END;
182
+}
183
+
168184 /**
169185 * sg_chain - Chain two sglists together
170186 * @prv: First scatterlist
....@@ -178,18 +194,7 @@
178194 static inline void sg_chain(struct scatterlist *prv, unsigned int prv_nents,
179195 struct scatterlist *sgl)
180196 {
181
- /*
182
- * offset and length are unused for chain entry. Clear them.
183
- */
184
- prv[prv_nents - 1].offset = 0;
185
- prv[prv_nents - 1].length = 0;
186
-
187
- /*
188
- * Set lowest bit to indicate a link pointer, and make sure to clear
189
- * the termination bit if it happens to be set.
190
- */
191
- prv[prv_nents - 1].page_link = ((unsigned long) sgl | SG_CHAIN)
192
- & ~SG_END;
197
+ __sg_chain(&prv[prv_nents - 1], sgl);
193198 }
194199
195200 /**
....@@ -280,15 +285,17 @@
280285 typedef struct scatterlist *(sg_alloc_fn)(unsigned int, gfp_t);
281286 typedef void (sg_free_fn)(struct scatterlist *, unsigned int);
282287
283
-void __sg_free_table(struct sg_table *, unsigned int, bool, sg_free_fn *);
288
+void __sg_free_table(struct sg_table *, unsigned int, unsigned int,
289
+ sg_free_fn *);
284290 void sg_free_table(struct sg_table *);
285291 int __sg_alloc_table(struct sg_table *, unsigned int, unsigned int,
286
- struct scatterlist *, gfp_t, sg_alloc_fn *);
292
+ struct scatterlist *, unsigned int, gfp_t, sg_alloc_fn *);
287293 int sg_alloc_table(struct sg_table *, unsigned int, gfp_t);
288
-int __sg_alloc_table_from_pages(struct sg_table *sgt, struct page **pages,
289
- unsigned int n_pages, unsigned int offset,
290
- unsigned long size, unsigned int max_segment,
291
- gfp_t gfp_mask);
294
+struct scatterlist *__sg_alloc_table_from_pages(struct sg_table *sgt,
295
+ struct page **pages, unsigned int n_pages, unsigned int offset,
296
+ unsigned long size, unsigned int max_segment,
297
+ struct scatterlist *prv, unsigned int left_pages,
298
+ gfp_t gfp_mask);
292299 int sg_alloc_table_from_pages(struct sg_table *sgt, struct page **pages,
293300 unsigned int n_pages, unsigned int offset,
294301 unsigned long size, gfp_t gfp_mask);
....@@ -338,27 +345,29 @@
338345 * Like SG_CHUNK_SIZE, but for archs that have sg chaining. This limit
339346 * is totally arbitrary, a setting of 2048 will get you at least 8mb ios.
340347 */
341
-#ifdef CONFIG_ARCH_HAS_SG_CHAIN
342
-#define SG_MAX_SEGMENTS 2048
343
-#else
348
+#ifdef CONFIG_ARCH_NO_SG_CHAIN
344349 #define SG_MAX_SEGMENTS SG_CHUNK_SIZE
350
+#else
351
+#define SG_MAX_SEGMENTS 2048
345352 #endif
346353
347354 #ifdef CONFIG_SG_POOL
348
-void sg_free_table_chained(struct sg_table *table, bool first_chunk);
355
+void sg_free_table_chained(struct sg_table *table,
356
+ unsigned nents_first_chunk);
349357 int sg_alloc_table_chained(struct sg_table *table, int nents,
350
- struct scatterlist *first_chunk);
358
+ struct scatterlist *first_chunk,
359
+ unsigned nents_first_chunk);
351360 #endif
352361
353362 /*
354363 * sg page iterator
355364 *
356
- * Iterates over sg entries page-by-page. On each successful iteration,
357
- * you can call sg_page_iter_page(@piter) and sg_page_iter_dma_address(@piter)
358
- * to get the current page and its dma address. @piter->sg will point to the
359
- * sg holding this page and @piter->sg_pgoffset to the page's page offset
360
- * within the sg. The iteration will stop either when a maximum number of sg
361
- * entries was reached or a terminating sg (sg_last(sg) == true) was reached.
365
+ * Iterates over sg entries page-by-page. On each successful iteration, you
366
+ * can call sg_page_iter_page(@piter) to get the current page.
367
+ * @piter->sg will point to the sg holding this page and @piter->sg_pgoffset to
368
+ * the page's page offset within the sg. The iteration will stop either when a
369
+ * maximum number of sg entries was reached or a terminating sg
370
+ * (sg_last(sg) == true) was reached.
362371 */
363372 struct sg_page_iter {
364373 struct scatterlist *sg; /* sg holding the page */
....@@ -370,7 +379,19 @@
370379 * next step */
371380 };
372381
382
+/*
383
+ * sg page iterator for DMA addresses
384
+ *
385
+ * This is the same as sg_page_iter however you can call
386
+ * sg_page_iter_dma_address(@dma_iter) to get the page's DMA
387
+ * address. sg_page_iter_page() cannot be called on this iterator.
388
+ */
389
+struct sg_dma_page_iter {
390
+ struct sg_page_iter base;
391
+};
392
+
373393 bool __sg_page_iter_next(struct sg_page_iter *piter);
394
+bool __sg_page_iter_dma_next(struct sg_dma_page_iter *dma_iter);
374395 void __sg_page_iter_start(struct sg_page_iter *piter,
375396 struct scatterlist *sglist, unsigned int nents,
376397 unsigned long pgoffset);
....@@ -386,11 +407,13 @@
386407 /**
387408 * sg_page_iter_dma_address - get the dma address of the current page held by
388409 * the page iterator.
389
- * @piter: page iterator holding the page
410
+ * @dma_iter: page iterator holding the page
390411 */
391
-static inline dma_addr_t sg_page_iter_dma_address(struct sg_page_iter *piter)
412
+static inline dma_addr_t
413
+sg_page_iter_dma_address(struct sg_dma_page_iter *dma_iter)
392414 {
393
- return sg_dma_address(piter->sg) + (piter->sg_pgoffset << PAGE_SHIFT);
415
+ return sg_dma_address(dma_iter->base.sg) +
416
+ (dma_iter->base.sg_pgoffset << PAGE_SHIFT);
394417 }
395418
396419 /**
....@@ -398,13 +421,32 @@
398421 * @sglist: sglist to iterate over
399422 * @piter: page iterator to hold current page, sg, sg_pgoffset
400423 * @nents: maximum number of sg entries to iterate over
401
- * @pgoffset: starting page offset
424
+ * @pgoffset: starting page offset (in pages)
425
+ *
426
+ * Callers may use sg_page_iter_page() to get each page pointer.
427
+ * In each loop it operates on PAGE_SIZE unit.
402428 */
403429 #define for_each_sg_page(sglist, piter, nents, pgoffset) \
404430 for (__sg_page_iter_start((piter), (sglist), (nents), (pgoffset)); \
405431 __sg_page_iter_next(piter);)
406432
407433 /**
434
+ * for_each_sg_dma_page - iterate over the pages of the given sg list
435
+ * @sglist: sglist to iterate over
436
+ * @dma_iter: DMA page iterator to hold current page
437
+ * @dma_nents: maximum number of sg entries to iterate over, this is the value
438
+ * returned from dma_map_sg
439
+ * @pgoffset: starting page offset (in pages)
440
+ *
441
+ * Callers may use sg_page_iter_dma_address() to get each page's DMA address.
442
+ * In each loop it operates on PAGE_SIZE unit.
443
+ */
444
+#define for_each_sg_dma_page(sglist, dma_iter, dma_nents, pgoffset) \
445
+ for (__sg_page_iter_start(&(dma_iter)->base, sglist, dma_nents, \
446
+ pgoffset); \
447
+ __sg_page_iter_dma_next(dma_iter);)
448
+
449
+/**
408450 * for_each_sgtable_page - iterate over all pages in the sg_table object
409451 * @sgt: sg_table object to iterate over
410452 * @piter: page iterator to hold current page