| .. | .. |
|---|
| 165 | 165 | #define for_each_sgtable_dma_sg(sgt, sg, i) \ |
|---|
| 166 | 166 | for_each_sg((sgt)->sgl, sg, (sgt)->nents, i) |
|---|
| 167 | 167 | |
|---|
| 168 | +static inline void __sg_chain(struct scatterlist *chain_sg, |
|---|
| 169 | + struct scatterlist *sgl) |
|---|
| 170 | +{ |
|---|
| 171 | + /* |
|---|
| 172 | + * offset and length are unused for chain entry. Clear them. |
|---|
| 173 | + */ |
|---|
| 174 | + chain_sg->offset = 0; |
|---|
| 175 | + chain_sg->length = 0; |
|---|
| 176 | + |
|---|
| 177 | + /* |
|---|
| 178 | + * Set lowest bit to indicate a link pointer, and make sure to clear |
|---|
| 179 | + * the termination bit if it happens to be set. |
|---|
| 180 | + */ |
|---|
| 181 | + chain_sg->page_link = ((unsigned long) sgl | SG_CHAIN) & ~SG_END; |
|---|
| 182 | +} |
|---|
| 183 | + |
|---|
| 168 | 184 | /** |
|---|
| 169 | 185 | * sg_chain - Chain two sglists together |
|---|
| 170 | 186 | * @prv: First scatterlist |
|---|
| .. | .. |
|---|
| 178 | 194 | static inline void sg_chain(struct scatterlist *prv, unsigned int prv_nents, |
|---|
| 179 | 195 | struct scatterlist *sgl) |
|---|
| 180 | 196 | { |
|---|
| 181 | | - /* |
|---|
| 182 | | - * offset and length are unused for chain entry. Clear them. |
|---|
| 183 | | - */ |
|---|
| 184 | | - prv[prv_nents - 1].offset = 0; |
|---|
| 185 | | - prv[prv_nents - 1].length = 0; |
|---|
| 186 | | - |
|---|
| 187 | | - /* |
|---|
| 188 | | - * Set lowest bit to indicate a link pointer, and make sure to clear |
|---|
| 189 | | - * the termination bit if it happens to be set. |
|---|
| 190 | | - */ |
|---|
| 191 | | - prv[prv_nents - 1].page_link = ((unsigned long) sgl | SG_CHAIN) |
|---|
| 192 | | - & ~SG_END; |
|---|
| 197 | + __sg_chain(&prv[prv_nents - 1], sgl); |
|---|
| 193 | 198 | } |
|---|
| 194 | 199 | |
|---|
| 195 | 200 | /** |
|---|
| .. | .. |
|---|
| 280 | 285 | typedef struct scatterlist *(sg_alloc_fn)(unsigned int, gfp_t); |
|---|
| 281 | 286 | typedef void (sg_free_fn)(struct scatterlist *, unsigned int); |
|---|
| 282 | 287 | |
|---|
| 283 | | -void __sg_free_table(struct sg_table *, unsigned int, bool, sg_free_fn *); |
|---|
| 288 | +void __sg_free_table(struct sg_table *, unsigned int, unsigned int, |
|---|
| 289 | + sg_free_fn *); |
|---|
| 284 | 290 | void sg_free_table(struct sg_table *); |
|---|
| 285 | 291 | int __sg_alloc_table(struct sg_table *, unsigned int, unsigned int, |
|---|
| 286 | | - struct scatterlist *, gfp_t, sg_alloc_fn *); |
|---|
| 292 | + struct scatterlist *, unsigned int, gfp_t, sg_alloc_fn *); |
|---|
| 287 | 293 | int sg_alloc_table(struct sg_table *, unsigned int, gfp_t); |
|---|
| 288 | | -int __sg_alloc_table_from_pages(struct sg_table *sgt, struct page **pages, |
|---|
| 289 | | - unsigned int n_pages, unsigned int offset, |
|---|
| 290 | | - unsigned long size, unsigned int max_segment, |
|---|
| 291 | | - gfp_t gfp_mask); |
|---|
| 294 | +struct scatterlist *__sg_alloc_table_from_pages(struct sg_table *sgt, |
|---|
| 295 | + struct page **pages, unsigned int n_pages, unsigned int offset, |
|---|
| 296 | + unsigned long size, unsigned int max_segment, |
|---|
| 297 | + struct scatterlist *prv, unsigned int left_pages, |
|---|
| 298 | + gfp_t gfp_mask); |
|---|
| 292 | 299 | int sg_alloc_table_from_pages(struct sg_table *sgt, struct page **pages, |
|---|
| 293 | 300 | unsigned int n_pages, unsigned int offset, |
|---|
| 294 | 301 | unsigned long size, gfp_t gfp_mask); |
|---|
| .. | .. |
|---|
| 338 | 345 | * Like SG_CHUNK_SIZE, but for archs that have sg chaining. This limit |
|---|
| 339 | 346 | * is totally arbitrary, a setting of 2048 will get you at least 8mb ios. |
|---|
| 340 | 347 | */ |
|---|
| 341 | | -#ifdef CONFIG_ARCH_HAS_SG_CHAIN |
|---|
| 342 | | -#define SG_MAX_SEGMENTS 2048 |
|---|
| 343 | | -#else |
|---|
| 348 | +#ifdef CONFIG_ARCH_NO_SG_CHAIN |
|---|
| 344 | 349 | #define SG_MAX_SEGMENTS SG_CHUNK_SIZE |
|---|
| 350 | +#else |
|---|
| 351 | +#define SG_MAX_SEGMENTS 2048 |
|---|
| 345 | 352 | #endif |
|---|
| 346 | 353 | |
|---|
| 347 | 354 | #ifdef CONFIG_SG_POOL |
|---|
| 348 | | -void sg_free_table_chained(struct sg_table *table, bool first_chunk); |
|---|
| 355 | +void sg_free_table_chained(struct sg_table *table, |
|---|
| 356 | + unsigned nents_first_chunk); |
|---|
| 349 | 357 | int sg_alloc_table_chained(struct sg_table *table, int nents, |
|---|
| 350 | | - struct scatterlist *first_chunk); |
|---|
| 358 | + struct scatterlist *first_chunk, |
|---|
| 359 | + unsigned nents_first_chunk); |
|---|
| 351 | 360 | #endif |
|---|
| 352 | 361 | |
|---|
| 353 | 362 | /* |
|---|
| 354 | 363 | * sg page iterator |
|---|
| 355 | 364 | * |
|---|
| 356 | | - * Iterates over sg entries page-by-page. On each successful iteration, |
|---|
| 357 | | - * you can call sg_page_iter_page(@piter) and sg_page_iter_dma_address(@piter) |
|---|
| 358 | | - * to get the current page and its dma address. @piter->sg will point to the |
|---|
| 359 | | - * sg holding this page and @piter->sg_pgoffset to the page's page offset |
|---|
| 360 | | - * within the sg. The iteration will stop either when a maximum number of sg |
|---|
| 361 | | - * entries was reached or a terminating sg (sg_last(sg) == true) was reached. |
|---|
| 365 | + * Iterates over sg entries page-by-page. On each successful iteration, you |
|---|
| 366 | + * can call sg_page_iter_page(@piter) to get the current page. |
|---|
| 367 | + * @piter->sg will point to the sg holding this page and @piter->sg_pgoffset to |
|---|
| 368 | + * the page's page offset within the sg. The iteration will stop either when a |
|---|
| 369 | + * maximum number of sg entries was reached or a terminating sg |
|---|
| 370 | + * (sg_last(sg) == true) was reached. |
|---|
| 362 | 371 | */ |
|---|
| 363 | 372 | struct sg_page_iter { |
|---|
| 364 | 373 | struct scatterlist *sg; /* sg holding the page */ |
|---|
| .. | .. |
|---|
| 370 | 379 | * next step */ |
|---|
| 371 | 380 | }; |
|---|
| 372 | 381 | |
|---|
| 382 | +/* |
|---|
| 383 | + * sg page iterator for DMA addresses |
|---|
| 384 | + * |
|---|
| 385 | + * This is the same as sg_page_iter however you can call |
|---|
| 386 | + * sg_page_iter_dma_address(@dma_iter) to get the page's DMA |
|---|
| 387 | + * address. sg_page_iter_page() cannot be called on this iterator. |
|---|
| 388 | + */ |
|---|
| 389 | +struct sg_dma_page_iter { |
|---|
| 390 | + struct sg_page_iter base; |
|---|
| 391 | +}; |
|---|
| 392 | + |
|---|
| 373 | 393 | bool __sg_page_iter_next(struct sg_page_iter *piter); |
|---|
| 394 | +bool __sg_page_iter_dma_next(struct sg_dma_page_iter *dma_iter); |
|---|
| 374 | 395 | void __sg_page_iter_start(struct sg_page_iter *piter, |
|---|
| 375 | 396 | struct scatterlist *sglist, unsigned int nents, |
|---|
| 376 | 397 | unsigned long pgoffset); |
|---|
| .. | .. |
|---|
| 386 | 407 | /** |
|---|
| 387 | 408 | * sg_page_iter_dma_address - get the dma address of the current page held by |
|---|
| 388 | 409 | * the page iterator. |
|---|
| 389 | | - * @piter: page iterator holding the page |
|---|
| 410 | + * @dma_iter: page iterator holding the page |
|---|
| 390 | 411 | */ |
|---|
| 391 | | -static inline dma_addr_t sg_page_iter_dma_address(struct sg_page_iter *piter) |
|---|
| 412 | +static inline dma_addr_t |
|---|
| 413 | +sg_page_iter_dma_address(struct sg_dma_page_iter *dma_iter) |
|---|
| 392 | 414 | { |
|---|
| 393 | | - return sg_dma_address(piter->sg) + (piter->sg_pgoffset << PAGE_SHIFT); |
|---|
| 415 | + return sg_dma_address(dma_iter->base.sg) + |
|---|
| 416 | + (dma_iter->base.sg_pgoffset << PAGE_SHIFT); |
|---|
| 394 | 417 | } |
|---|
| 395 | 418 | |
|---|
| 396 | 419 | /** |
|---|
| .. | .. |
|---|
| 398 | 421 | * @sglist: sglist to iterate over |
|---|
| 399 | 422 | * @piter: page iterator to hold current page, sg, sg_pgoffset |
|---|
| 400 | 423 | * @nents: maximum number of sg entries to iterate over |
|---|
| 401 | | - * @pgoffset: starting page offset |
|---|
| 424 | + * @pgoffset: starting page offset (in pages) |
|---|
| 425 | + * |
|---|
| 426 | + * Callers may use sg_page_iter_page() to get each page pointer. |
|---|
| 427 | + * In each loop it operates on PAGE_SIZE unit. |
|---|
| 402 | 428 | */ |
|---|
| 403 | 429 | #define for_each_sg_page(sglist, piter, nents, pgoffset) \ |
|---|
| 404 | 430 | for (__sg_page_iter_start((piter), (sglist), (nents), (pgoffset)); \ |
|---|
| 405 | 431 | __sg_page_iter_next(piter);) |
|---|
| 406 | 432 | |
|---|
| 407 | 433 | /** |
|---|
| 434 | + * for_each_sg_dma_page - iterate over the pages of the given sg list |
|---|
| 435 | + * @sglist: sglist to iterate over |
|---|
| 436 | + * @dma_iter: DMA page iterator to hold current page |
|---|
| 437 | + * @dma_nents: maximum number of sg entries to iterate over, this is the value |
|---|
| 438 | + * returned from dma_map_sg |
|---|
| 439 | + * @pgoffset: starting page offset (in pages) |
|---|
| 440 | + * |
|---|
| 441 | + * Callers may use sg_page_iter_dma_address() to get each page's DMA address. |
|---|
| 442 | + * In each loop it operates on PAGE_SIZE unit. |
|---|
| 443 | + */ |
|---|
| 444 | +#define for_each_sg_dma_page(sglist, dma_iter, dma_nents, pgoffset) \ |
|---|
| 445 | + for (__sg_page_iter_start(&(dma_iter)->base, sglist, dma_nents, \ |
|---|
| 446 | + pgoffset); \ |
|---|
| 447 | + __sg_page_iter_dma_next(dma_iter);) |
|---|
| 448 | + |
|---|
| 449 | +/** |
|---|
| 408 | 450 | * for_each_sgtable_page - iterate over all pages in the sg_table object |
|---|
| 409 | 451 | * @sgt: sg_table object to iterate over |
|---|
| 410 | 452 | * @piter: page iterator to hold current page |
|---|