| .. | .. |
|---|
| 10 | 10 | |
|---|
| 11 | 11 | #include <linux/bio.h> |
|---|
| 12 | 12 | #include <linux/blkdev.h> |
|---|
| 13 | +#include <linux/dm-ioctl.h> |
|---|
| 13 | 14 | #include <linux/math64.h> |
|---|
| 14 | 15 | #include <linux/ratelimit.h> |
|---|
| 16 | +#include <linux/android_kabi.h> |
|---|
| 15 | 17 | |
|---|
| 16 | 18 | struct dm_dev; |
|---|
| 17 | 19 | struct dm_target; |
|---|
| 18 | 20 | struct dm_table; |
|---|
| 21 | +struct dm_report_zones_args; |
|---|
| 19 | 22 | struct mapped_device; |
|---|
| 20 | 23 | struct bio_vec; |
|---|
| 21 | 24 | |
|---|
| .. | .. |
|---|
| 26 | 29 | DM_TYPE_NONE = 0, |
|---|
| 27 | 30 | DM_TYPE_BIO_BASED = 1, |
|---|
| 28 | 31 | DM_TYPE_REQUEST_BASED = 2, |
|---|
| 29 | | - DM_TYPE_MQ_REQUEST_BASED = 3, |
|---|
| 30 | | - DM_TYPE_DAX_BIO_BASED = 4, |
|---|
| 31 | | - DM_TYPE_NVME_BIO_BASED = 5, |
|---|
| 32 | + DM_TYPE_DAX_BIO_BASED = 3, |
|---|
| 32 | 33 | }; |
|---|
| 33 | 34 | |
|---|
| 34 | 35 | typedef enum { STATUSTYPE_INFO, STATUSTYPE_TABLE } status_type_t; |
|---|
| .. | .. |
|---|
| 93 | 94 | |
|---|
| 94 | 95 | typedef int (*dm_prepare_ioctl_fn) (struct dm_target *ti, struct block_device **bdev); |
|---|
| 95 | 96 | |
|---|
| 97 | +typedef int (*dm_report_zones_fn) (struct dm_target *ti, |
|---|
| 98 | + struct dm_report_zones_args *args, |
|---|
| 99 | + unsigned int nr_zones); |
|---|
| 100 | + |
|---|
| 96 | 101 | /* |
|---|
| 97 | 102 | * These iteration functions are typically used to check (and combine) |
|---|
| 98 | 103 | * properties of underlying devices. |
|---|
| .. | .. |
|---|
| 136 | 141 | long nr_pages, void **kaddr, pfn_t *pfn); |
|---|
| 137 | 142 | typedef size_t (*dm_dax_copy_iter_fn)(struct dm_target *ti, pgoff_t pgoff, |
|---|
| 138 | 143 | void *addr, size_t bytes, struct iov_iter *i); |
|---|
| 144 | +typedef int (*dm_dax_zero_page_range_fn)(struct dm_target *ti, pgoff_t pgoff, |
|---|
| 145 | + size_t nr_pages); |
|---|
| 139 | 146 | #define PAGE_SECTORS (PAGE_SIZE / 512) |
|---|
| 140 | 147 | |
|---|
| 141 | 148 | void dm_error(const char *message); |
|---|
| .. | .. |
|---|
| 181 | 188 | dm_status_fn status; |
|---|
| 182 | 189 | dm_message_fn message; |
|---|
| 183 | 190 | dm_prepare_ioctl_fn prepare_ioctl; |
|---|
| 191 | +#ifdef CONFIG_BLK_DEV_ZONED |
|---|
| 192 | + dm_report_zones_fn report_zones; |
|---|
| 193 | +#endif |
|---|
| 184 | 194 | dm_busy_fn busy; |
|---|
| 185 | 195 | dm_iterate_devices_fn iterate_devices; |
|---|
| 186 | 196 | dm_io_hints_fn io_hints; |
|---|
| 187 | 197 | dm_dax_direct_access_fn direct_access; |
|---|
| 188 | 198 | dm_dax_copy_iter_fn dax_copy_from_iter; |
|---|
| 189 | 199 | dm_dax_copy_iter_fn dax_copy_to_iter; |
|---|
| 200 | + dm_dax_zero_page_range_fn dax_zero_page_range; |
|---|
| 201 | + |
|---|
| 202 | + ANDROID_KABI_RESERVE(1); |
|---|
| 203 | + ANDROID_KABI_RESERVE(2); |
|---|
| 190 | 204 | |
|---|
| 191 | 205 | /* For internal device-mapper use. */ |
|---|
| 192 | 206 | struct list_head list; |
|---|
| .. | .. |
|---|
| 236 | 250 | #define dm_target_passes_integrity(type) ((type)->features & DM_TARGET_PASSES_INTEGRITY) |
|---|
| 237 | 251 | |
|---|
| 238 | 252 | /* |
|---|
| 239 | | - * Indicates that a target supports host-managed zoned block devices. |
|---|
| 253 | + * Indicates support for zoned block devices: |
|---|
| 254 | + * - DM_TARGET_ZONED_HM: the target also supports host-managed zoned |
|---|
| 255 | + * block devices but does not support combining different zoned models. |
|---|
| 256 | + * - DM_TARGET_MIXED_ZONED_MODEL: the target supports combining multiple |
|---|
| 257 | + * devices with different zoned models. |
|---|
| 240 | 258 | */ |
|---|
| 241 | 259 | #define DM_TARGET_ZONED_HM 0x00000040 |
|---|
| 242 | 260 | #define dm_target_supports_zoned_hm(type) ((type)->features & DM_TARGET_ZONED_HM) |
|---|
| 261 | + |
|---|
| 262 | +/* |
|---|
| 263 | + * A target handles REQ_NOWAIT |
|---|
| 264 | + */ |
|---|
| 265 | +#define DM_TARGET_NOWAIT 0x00000080 |
|---|
| 266 | +#define dm_target_supports_nowait(type) ((type)->features & DM_TARGET_NOWAIT) |
|---|
| 267 | + |
|---|
| 268 | +/* |
|---|
| 269 | + * A target supports passing through inline crypto support. |
|---|
| 270 | + */ |
|---|
| 271 | +#define DM_TARGET_PASSES_CRYPTO 0x00000100 |
|---|
| 272 | +#define dm_target_passes_crypto(type) ((type)->features & DM_TARGET_PASSES_CRYPTO) |
|---|
| 273 | + |
|---|
| 274 | +#ifdef CONFIG_BLK_DEV_ZONED |
|---|
| 275 | +#define DM_TARGET_MIXED_ZONED_MODEL 0x00000200 |
|---|
| 276 | +#define dm_target_supports_mixed_zoned_model(type) \ |
|---|
| 277 | + ((type)->features & DM_TARGET_MIXED_ZONED_MODEL) |
|---|
| 278 | +#else |
|---|
| 279 | +#define DM_TARGET_MIXED_ZONED_MODEL 0x00000000 |
|---|
| 280 | +#define dm_target_supports_mixed_zoned_model(type) (false) |
|---|
| 281 | +#endif |
|---|
| 243 | 282 | |
|---|
| 244 | 283 | struct dm_target { |
|---|
| 245 | 284 | struct dm_table *table; |
|---|
| .. | .. |
|---|
| 311 | 350 | bool discards_supported:1; |
|---|
| 312 | 351 | |
|---|
| 313 | 352 | /* |
|---|
| 314 | | - * Set if the target required discard bios to be split |
|---|
| 315 | | - * on max_io_len boundary. |
|---|
| 353 | + * Set if we need to limit the number of in-flight bios when swapping. |
|---|
| 316 | 354 | */ |
|---|
| 317 | | - bool split_discard_bios:1; |
|---|
| 355 | + bool limit_swap_bios:1; |
|---|
| 318 | 356 | |
|---|
| 319 | | - /* |
|---|
| 320 | | - * Set if inline crypto capabilities from this target's underlying |
|---|
| 321 | | - * device(s) can be exposed via the device-mapper device. |
|---|
| 322 | | - */ |
|---|
| 323 | | - bool may_passthrough_inline_crypto:1; |
|---|
| 324 | | -}; |
|---|
| 325 | | - |
|---|
| 326 | | -/* Each target can link one of these into the table */ |
|---|
| 327 | | -struct dm_target_callbacks { |
|---|
| 328 | | - struct list_head list; |
|---|
| 329 | | - int (*congested_fn) (struct dm_target_callbacks *, int); |
|---|
| 357 | + ANDROID_KABI_RESERVE(1); |
|---|
| 358 | + ANDROID_KABI_RESERVE(2); |
|---|
| 330 | 359 | }; |
|---|
| 331 | 360 | |
|---|
| 332 | 361 | void *dm_per_bio_data(struct bio *bio, size_t data_size); |
|---|
| 333 | 362 | struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size); |
|---|
| 334 | 363 | unsigned dm_bio_get_target_bio_nr(const struct bio *bio); |
|---|
| 364 | + |
|---|
| 365 | +u64 dm_start_time_ns_from_clone(struct bio *bio); |
|---|
| 335 | 366 | |
|---|
| 336 | 367 | int dm_register_target(struct target_type *t); |
|---|
| 337 | 368 | void dm_unregister_target(struct target_type *t); |
|---|
| .. | .. |
|---|
| 428 | 459 | int dm_post_suspending(struct dm_target *ti); |
|---|
| 429 | 460 | int dm_noflush_suspending(struct dm_target *ti); |
|---|
| 430 | 461 | void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors); |
|---|
| 431 | | -void dm_remap_zone_report(struct dm_target *ti, struct bio *bio, |
|---|
| 432 | | - sector_t start); |
|---|
| 433 | 462 | union map_info *dm_get_rq_mapinfo(struct request *rq); |
|---|
| 463 | + |
|---|
| 464 | +#ifdef CONFIG_BLK_DEV_ZONED |
|---|
| 465 | +struct dm_report_zones_args { |
|---|
| 466 | + struct dm_target *tgt; |
|---|
| 467 | + sector_t next_sector; |
|---|
| 468 | + |
|---|
| 469 | + void *orig_data; |
|---|
| 470 | + report_zones_cb orig_cb; |
|---|
| 471 | + unsigned int zone_idx; |
|---|
| 472 | + |
|---|
| 473 | + /* must be filled by ->report_zones before calling dm_report_zones_cb */ |
|---|
| 474 | + sector_t start; |
|---|
| 475 | +}; |
|---|
| 476 | +int dm_report_zones_cb(struct blk_zone *zone, unsigned int idx, void *data); |
|---|
| 477 | +#endif /* CONFIG_BLK_DEV_ZONED */ |
|---|
| 478 | + |
|---|
| 479 | +/* |
|---|
| 480 | + * Device mapper functions to parse and create devices specified by the |
|---|
| 481 | + * parameter "dm-mod.create=" |
|---|
| 482 | + */ |
|---|
| 483 | +int __init dm_early_create(struct dm_ioctl *dmi, |
|---|
| 484 | + struct dm_target_spec **spec_array, |
|---|
| 485 | + char **target_params_array); |
|---|
| 434 | 486 | |
|---|
| 435 | 487 | struct queue_limits *dm_get_queue_limits(struct mapped_device *md); |
|---|
| 436 | 488 | |
|---|
| .. | .. |
|---|
| 455 | 507 | */ |
|---|
| 456 | 508 | int dm_table_add_target(struct dm_table *t, const char *type, |
|---|
| 457 | 509 | sector_t start, sector_t len, char *params); |
|---|
| 458 | | - |
|---|
| 459 | | -/* |
|---|
| 460 | | - * Target_ctr should call this if it needs to add any callbacks. |
|---|
| 461 | | - */ |
|---|
| 462 | | -void dm_table_add_target_callbacks(struct dm_table *t, struct dm_target_callbacks *cb); |
|---|
| 463 | 510 | |
|---|
| 464 | 511 | /* |
|---|
| 465 | 512 | * Target can use this to set the table's type. |
|---|
| .. | .. |
|---|
| 498 | 545 | unsigned int dm_table_get_num_targets(struct dm_table *t); |
|---|
| 499 | 546 | fmode_t dm_table_get_mode(struct dm_table *t); |
|---|
| 500 | 547 | struct mapped_device *dm_table_get_md(struct dm_table *t); |
|---|
| 548 | +const char *dm_table_device_name(struct dm_table *t); |
|---|
| 501 | 549 | |
|---|
| 502 | 550 | /* |
|---|
| 503 | 551 | * Trigger an event. |
|---|
| .. | .. |
|---|
| 517 | 565 | struct dm_table *t); |
|---|
| 518 | 566 | |
|---|
| 519 | 567 | /* |
|---|
| 568 | + * Table keyslot manager functions |
|---|
| 569 | + */ |
|---|
| 570 | +void dm_destroy_keyslot_manager(struct blk_keyslot_manager *ksm); |
|---|
| 571 | + |
|---|
| 572 | +/* |
|---|
| 520 | 573 | * A wrapper around vmalloc. |
|---|
| 521 | 574 | */ |
|---|
| 522 | 575 | void *dm_vcalloc(unsigned long nmemb, unsigned long elem_size); |
|---|
| .. | .. |
|---|
| 526 | 579 | *---------------------------------------------------------------*/ |
|---|
| 527 | 580 | #define DM_NAME "device-mapper" |
|---|
| 528 | 581 | |
|---|
| 529 | | -#define DM_RATELIMIT(pr_func, fmt, ...) \ |
|---|
| 530 | | -do { \ |
|---|
| 531 | | - static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, \ |
|---|
| 532 | | - DEFAULT_RATELIMIT_BURST); \ |
|---|
| 533 | | - \ |
|---|
| 534 | | - if (__ratelimit(&rs)) \ |
|---|
| 535 | | - pr_func(DM_FMT(fmt), ##__VA_ARGS__); \ |
|---|
| 536 | | -} while (0) |
|---|
| 537 | | - |
|---|
| 538 | 582 | #define DM_FMT(fmt) DM_NAME ": " DM_MSG_PREFIX ": " fmt "\n" |
|---|
| 539 | 583 | |
|---|
| 540 | 584 | #define DMCRIT(fmt, ...) pr_crit(DM_FMT(fmt), ##__VA_ARGS__) |
|---|
| 541 | 585 | |
|---|
| 542 | 586 | #define DMERR(fmt, ...) pr_err(DM_FMT(fmt), ##__VA_ARGS__) |
|---|
| 543 | | -#define DMERR_LIMIT(fmt, ...) DM_RATELIMIT(pr_err, fmt, ##__VA_ARGS__) |
|---|
| 587 | +#define DMERR_LIMIT(fmt, ...) pr_err_ratelimited(DM_FMT(fmt), ##__VA_ARGS__) |
|---|
| 544 | 588 | #define DMWARN(fmt, ...) pr_warn(DM_FMT(fmt), ##__VA_ARGS__) |
|---|
| 545 | | -#define DMWARN_LIMIT(fmt, ...) DM_RATELIMIT(pr_warn, fmt, ##__VA_ARGS__) |
|---|
| 589 | +#define DMWARN_LIMIT(fmt, ...) pr_warn_ratelimited(DM_FMT(fmt), ##__VA_ARGS__) |
|---|
| 546 | 590 | #define DMINFO(fmt, ...) pr_info(DM_FMT(fmt), ##__VA_ARGS__) |
|---|
| 547 | | -#define DMINFO_LIMIT(fmt, ...) DM_RATELIMIT(pr_info, fmt, ##__VA_ARGS__) |
|---|
| 591 | +#define DMINFO_LIMIT(fmt, ...) pr_info_ratelimited(DM_FMT(fmt), ##__VA_ARGS__) |
|---|
| 548 | 592 | |
|---|
| 549 | | -#ifdef CONFIG_DM_DEBUG |
|---|
| 550 | | -#define DMDEBUG(fmt, ...) printk(KERN_DEBUG DM_FMT(fmt), ##__VA_ARGS__) |
|---|
| 551 | | -#define DMDEBUG_LIMIT(fmt, ...) DM_RATELIMIT(pr_debug, fmt, ##__VA_ARGS__) |
|---|
| 552 | | -#else |
|---|
| 553 | | -#define DMDEBUG(fmt, ...) no_printk(fmt, ##__VA_ARGS__) |
|---|
| 554 | | -#define DMDEBUG_LIMIT(fmt, ...) no_printk(fmt, ##__VA_ARGS__) |
|---|
| 555 | | -#endif |
|---|
| 593 | +#define DMDEBUG(fmt, ...) pr_debug(DM_FMT(fmt), ##__VA_ARGS__) |
|---|
| 594 | +#define DMDEBUG_LIMIT(fmt, ...) pr_debug_ratelimited(DM_FMT(fmt), ##__VA_ARGS__) |
|---|
| 556 | 595 | |
|---|
| 557 | 596 | #define DMEMIT(x...) sz += ((sz >= maxlen) ? \ |
|---|
| 558 | 597 | 0 : scnprintf(result + sz, maxlen - sz, x)) |
|---|
| .. | .. |
|---|
| 599 | 638 | * ceiling(n / size) * size |
|---|
| 600 | 639 | */ |
|---|
| 601 | 640 | #define dm_round_up(n, sz) (dm_div_up((n), (sz)) * (sz)) |
|---|
| 602 | | - |
|---|
| 603 | | -#define dm_array_too_big(fixed, obj, num) \ |
|---|
| 604 | | - ((num) > (UINT_MAX - (fixed)) / (obj)) |
|---|
| 605 | 641 | |
|---|
| 606 | 642 | /* |
|---|
| 607 | 643 | * Sector offset taken relative to the start of the target instead of |
|---|