forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-09 b22da3d8526a935aa31e086e63f60ff3246cb61c 
 kernel/include/linux/blkdev.h
....@@ -4,13 +4,11 @@
44 
55 #include <linux/sched.h>
66 #include <linux/sched/clock.h>
7
-
8
-#ifdef CONFIG_BLOCK
9
-
107 #include <linux/major.h>
118 #include <linux/genhd.h>
129 #include <linux/list.h>
1310 #include <linux/llist.h>
11
+#include <linux/minmax.h>
1412 #include <linux/timer.h>
1513 #include <linux/workqueue.h>
1614 #include <linux/pagemap.h>
....@@ -27,6 +25,9 @@
2725 #include <linux/percpu-refcount.h>
2826 #include <linux/scatterlist.h>
2927 #include <linux/blkzoned.h>
28
+#include <linux/pm.h>
29
+#include <linux/android_kabi.h>
30
+#include <linux/android_vendor.h>
3031 
3132 struct module;
3233 struct scsi_ioctl_command;
....@@ -43,7 +44,7 @@
4344 struct rq_qos;
4445 struct blk_queue_stats;
4546 struct blk_stat_callback;
46
-struct keyslot_manager;
47
+struct blk_keyslot_manager;
4748 
4849 #define BLKDEV_MIN_RQ	4
4950 #define BLKDEV_MAX_RQ	128	/* Default maximum */
....@@ -51,32 +52,24 @@
5152 /* Must be consistent with blk_mq_poll_stats_bkt() */
5253 #define BLK_MQ_POLL_STATS_BKTS 16
5354 
55
+/* Doing classic polling */
56
+#define BLK_MQ_POLL_CLASSIC -1
57
+
5458 /*
5559  * Maximum number of blkcg policies allowed to be registered concurrently.
5660  * Defined here to simplify include dependency.
5761  */
5862 #define BLKCG_MAX_POLS		5
5963 
64
+static inline int blk_validate_block_size(unsigned int bsize)
65
+{
66
+	if (bsize < 512 || bsize > PAGE_SIZE || !is_power_of_2(bsize))
67
+		return -EINVAL;
68
+
69
+	return 0;
70
+}
71
+
6072 typedef void (rq_end_io_fn)(struct request *, blk_status_t);
61
-
62
-#define BLK_RL_SYNCFULL		(1U << 0)
63
-#define BLK_RL_ASYNCFULL	(1U << 1)
64
-
65
-struct request_list {
66
-	struct request_queue	*q;	/* the queue this rl belongs to */
67
-#ifdef CONFIG_BLK_CGROUP
68
-	struct blkcg_gq		*blkg;	/* blkg this request pool belongs to */
69
-#endif
70
-	/*
71
-	 * count[], starved[], and wait[] are indexed by
72
-	 * BLK_RW_SYNC/BLK_RW_ASYNC
73
-	 */
74
-	int			count[2];
75
-	int			starved[2];
76
-	mempool_t		*rq_pool;
77
-	wait_queue_head_t	wait[2];
78
-	unsigned int		flags;
79
-};
8073 
8174 /*
8275  * request flags */
....@@ -86,8 +79,6 @@
8679 #define RQF_SORTED		((__force req_flags_t)(1 << 0))
8780 /* drive already may have started this one */
8881 #define RQF_STARTED		((__force req_flags_t)(1 << 1))
89
-/* uses tagged queueing */
90
-#define RQF_QUEUED		((__force req_flags_t)(1 << 2))
9182 /* may not be passed by ioscheduler */
9283 #define RQF_SOFTBARRIER		((__force req_flags_t)(1 << 3))
9384 /* request for flush sequence */
....@@ -98,18 +89,13 @@
9889 #define RQF_MQ_INFLIGHT		((__force req_flags_t)(1 << 6))
9990 /* don't call prep for this one */
10091 #define RQF_DONTPREP		((__force req_flags_t)(1 << 7))
101
-/* set for "ide_preempt" requests and also for requests for which the SCSI
102
-   "quiesce" state must be ignored. */
103
-#define RQF_PREEMPT		((__force req_flags_t)(1 << 8))
104
-/* contains copies of user pages */
105
-#define RQF_COPY_USER		((__force req_flags_t)(1 << 9))
10692 /* vaguely specified driver internal error.  Ignored by the block layer */
10793 #define RQF_FAILED		((__force req_flags_t)(1 << 10))
10894 /* don't warn about errors */
10995 #define RQF_QUIET		((__force req_flags_t)(1 << 11))
11096 /* elevator private data attached */
11197 #define RQF_ELVPRIV		((__force req_flags_t)(1 << 12))
112
-/* account I/O stat */
98
+/* account into disk and partition IO statistics */
11399 #define RQF_IO_STAT		((__force req_flags_t)(1 << 13))
114100 /* request came from our alloc pool */
115101 #define RQF_ALLOCED		((__force req_flags_t)(1 << 14))
....@@ -117,7 +103,7 @@
117103 #define RQF_PM			((__force req_flags_t)(1 << 15))
118104 /* on IO scheduler merge hash */
119105 #define RQF_HASHED		((__force req_flags_t)(1 << 16))
120
-/* IO stats tracking on */
106
+/* track IO completion time */
121107 #define RQF_STATS		((__force req_flags_t)(1 << 17))
122108 /* Look at ->special_vec for the actual data payload instead of the
123109    bio chain. */
....@@ -151,16 +137,16 @@
151137 struct request {
152138 	struct request_queue *q;
153139 	struct blk_mq_ctx *mq_ctx;
140
+	struct blk_mq_hw_ctx *mq_hctx;
154141 
155
-	int cpu;
156142 	unsigned int cmd_flags;		/* op and common flags */
157143 	req_flags_t rq_flags;
158144 
145
+	int tag;
159146 	int internal_tag;
160147 
161148 	/* the following two fields are internal, NEVER access directly */
162149 	unsigned int __data_len;	/* total data len */
163
-	int tag;
164150 	sector_t __sector;		/* sector cursor */
165151 
166152 	struct bio *bio;
....@@ -177,7 +163,7 @@
177163 	 */
178164 	union {
179165 		struct hlist_node hash;	/* merge hash */
180
-		struct list_head ipi_list;
166
+		struct llist_node ipi_list;
181167 	};
182168 
183169 	/*
....@@ -213,7 +199,11 @@
213199 
214200 	struct gendisk *rq_disk;
215201 	struct hd_struct *part;
216
-	/* Time that I/O was submitted to the kernel. */
202
+#ifdef CONFIG_BLK_RQ_ALLOC_TIME
203
+	/* Time that the first bio started allocating this request. */
204
+	u64 alloc_time_ns;
205
+#endif
206
+	/* Time that this request was allocated for this IO. */
217207 	u64 start_time_ns;
218208 	/* Time that I/O was submitted to the device. */
219209 	u64 io_start_time_ns;
....@@ -221,9 +211,12 @@
221211 #ifdef CONFIG_BLK_WBT
222212 	unsigned short wbt_flags;
223213 #endif
224
-#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
225
-	unsigned short throtl_size;
226
-#endif
214
+	/*
215
+	 * rq sectors used for blk stats. It has the same value
216
+	 * with blk_rq_sectors(rq), except that it never be zeroed
217
+	 * by completion.
218
+	 */
219
+	unsigned short stats_sectors;
227220 
228221 	/*
229222 	 * Number of scatter-gather DMA addr+len pairs after
....@@ -235,22 +228,19 @@
235228 	unsigned short nr_integrity_segments;
236229 #endif
237230 
231
+#ifdef CONFIG_BLK_INLINE_ENCRYPTION
232
+	struct bio_crypt_ctx *crypt_ctx;
233
+	struct blk_ksm_keyslot *crypt_keyslot;
234
+#endif
235
+
238236 	unsigned short write_hint;
239237 	unsigned short ioprio;
240
-
241
-	void *special;		/* opaque pointer available for LLD use */
242
-
243
-	unsigned int extra_len;	/* length of alignment and padding */
244238 
245239 	enum mq_rq_state state;
246240 	refcount_t ref;
247241 
248242 	unsigned int timeout;
249
-
250
-	/* access through blk_rq_set_deadline, blk_rq_deadline */
251
-	unsigned long __deadline;
252
-
253
-	struct list_head timeout_list;
243
+	unsigned long deadline;
254244 
255245 	union {
256246 		struct __call_single_data csd;
....@@ -263,12 +253,7 @@
263253 	rq_end_io_fn *end_io;
264254 	void *end_io_data;
265255 
266
-	/* for bidi */
267
-	struct request *next_rq;
268
-
269
-#ifdef CONFIG_BLK_CGROUP
270
-	struct request_list *rl;		/* rl this rq is alloced from */
271
-#endif
256
+	ANDROID_KABI_RESERVE(1);
272257 };
273258 
274259 static inline bool blk_op_is_scsi(unsigned int op)
....@@ -312,41 +297,18 @@
312297 
313298 struct blk_queue_ctx;
314299 
315
-typedef void (request_fn_proc) (struct request_queue *q);
316
-typedef blk_qc_t (make_request_fn) (struct request_queue *q, struct bio *bio);
317
-typedef bool (poll_q_fn) (struct request_queue *q, blk_qc_t);
318
-typedef int (prep_rq_fn) (struct request_queue *, struct request *);
319
-typedef void (unprep_rq_fn) (struct request_queue *, struct request *);
320
-
321300 struct bio_vec;
322
-typedef void (softirq_done_fn)(struct request *);
323
-typedef int (dma_drain_needed_fn)(struct request *);
324
-typedef int (lld_busy_fn) (struct request_queue *q);
325
-typedef int (bsg_job_fn) (struct bsg_job *);
326
-typedef int (init_rq_fn)(struct request_queue *, struct request *, gfp_t);
327
-typedef void (exit_rq_fn)(struct request_queue *, struct request *);
328301 
329302 enum blk_eh_timer_return {
330303 	BLK_EH_DONE,		/* drivers has completed the command */
331304 	BLK_EH_RESET_TIMER,	/* reset timer and try again */
332305 };
333306 
334
-typedef enum blk_eh_timer_return (rq_timed_out_fn)(struct request *);
335
-
336307 enum blk_queue_state {
337308 	Queue_down,
338309 	Queue_up,
339310 };
340311 
341
-struct blk_queue_tag {
342
-	struct request **tag_index;	/* map of busy tags */
343
-	unsigned long *tag_map;		/* bit map of free/busy tags */
344
-	int max_depth;			/* what we will send to device */
345
-	int real_max_depth;		/* what the array can hold */
346
-	atomic_t refcnt;		/* map can be shared */
347
-	int alloc_policy;		/* tag allocation policy */
348
-	int next_tag;			/* next tag */
349
-};
350312 #define BLK_TAG_ALLOC_FIFO 0 /* allocate starting from 0 */
351313 #define BLK_TAG_ALLOC_RR 1 /* allocate starting from last allocated tag */
352314 
....@@ -355,11 +317,14 @@
355317 
356318 /*
357319  * Zoned block device models (zoned limit).
320
+ *
321
+ * Note: This needs to be ordered from the least to the most severe
322
+ * restrictions for the inheritance in blk_stack_limits() to work.
358323  */
359324 enum blk_zoned_model {
360
-	BLK_ZONED_NONE,	/* Regular block device */
361
-	BLK_ZONED_HA,	/* Host-aware zoned block device */
362
-	BLK_ZONED_HM,	/* Host-managed zoned block device */
325
+	BLK_ZONED_NONE = 0,	/* Regular block device */
326
+	BLK_ZONED_HA,		/* Host-aware zoned block device */
327
+	BLK_ZONED_HM,		/* Host-managed zoned block device */
363328 };
364329 
365330 struct queue_limits {
....@@ -381,6 +346,7 @@
381346 	unsigned int		max_hw_discard_sectors;
382347 	unsigned int		max_write_same_sectors;
383348 	unsigned int		max_write_zeroes_sectors;
349
+	unsigned int		max_zone_append_sectors;
384350 	unsigned int		discard_granularity;
385351 	unsigned int		discard_alignment;
386352 
....@@ -390,30 +356,40 @@
390356 
391357 	unsigned char		misaligned;
392358 	unsigned char		discard_misaligned;
393
-	unsigned char		cluster;
394359 	unsigned char		raid_partial_stripes_expensive;
395360 	enum blk_zoned_model	zoned;
361
+
362
+	ANDROID_KABI_RESERVE(1);
396363 };
364
+
365
+typedef int (*report_zones_cb)(struct blk_zone *zone, unsigned int idx,
366
+			       void *data);
367
+
368
+void blk_queue_set_zoned(struct gendisk *disk, enum blk_zoned_model model);
397369 
398370 #ifdef CONFIG_BLK_DEV_ZONED
399371 
400
-struct blk_zone_report_hdr {
401
-	unsigned int	nr_zones;
402
-	u8		padding[60];
403
-};
404
-
405
-extern int blkdev_report_zones(struct block_device *bdev,
406
-			       sector_t sector, struct blk_zone *zones,
407
-			       unsigned int *nr_zones, gfp_t gfp_mask);
408
-extern int blkdev_reset_zones(struct block_device *bdev, sector_t sectors,
409
-			      sector_t nr_sectors, gfp_t gfp_mask);
372
+#define BLK_ALL_ZONES  ((unsigned int)-1)
373
+int blkdev_report_zones(struct block_device *bdev, sector_t sector,
374
+			unsigned int nr_zones, report_zones_cb cb, void *data);
375
+unsigned int blkdev_nr_zones(struct gendisk *disk);
376
+extern int blkdev_zone_mgmt(struct block_device *bdev, enum req_opf op,
377
+			    sector_t sectors, sector_t nr_sectors,
378
+			    gfp_t gfp_mask);
379
+int blk_revalidate_disk_zones(struct gendisk *disk,
380
+			      void (*update_driver_data)(struct gendisk *disk));
410381 
411382 extern int blkdev_report_zones_ioctl(struct block_device *bdev, fmode_t mode,
412383 				     unsigned int cmd, unsigned long arg);
413
-extern int blkdev_reset_zones_ioctl(struct block_device *bdev, fmode_t mode,
414
-				    unsigned int cmd, unsigned long arg);
384
+extern int blkdev_zone_mgmt_ioctl(struct block_device *bdev, fmode_t mode,
385
+				  unsigned int cmd, unsigned long arg);
415386 
416387 #else /* CONFIG_BLK_DEV_ZONED */
388
+
389
+static inline unsigned int blkdev_nr_zones(struct gendisk *disk)
390
+{
391
+	return 0;
392
+}
417393 
418394 static inline int blkdev_report_zones_ioctl(struct block_device *bdev,
419395 					    fmode_t mode, unsigned int cmd,
....@@ -422,9 +398,9 @@
422398 	return -ENOTTY;
423399 }
424400 
425
-static inline int blkdev_reset_zones_ioctl(struct block_device *bdev,
426
-					   fmode_t mode, unsigned int cmd,
427
-					   unsigned long arg)
401
+static inline int blkdev_zone_mgmt_ioctl(struct block_device *bdev,
402
+					 fmode_t mode, unsigned int cmd,
403
+					 unsigned long arg)
428404 {
429405 	return -ENOTTY;
430406 }
....@@ -432,66 +408,24 @@
432408 #endif /* CONFIG_BLK_DEV_ZONED */
433409 
434410 struct request_queue {
435
-	/*
436
-	 * Together with queue_head for cacheline sharing
437
-	 */
438
-	struct list_head	queue_head;
439411 	struct request		*last_merge;
440412 	struct elevator_queue	*elevator;
441
-	int			nr_rqs[2];	/* # allocated [a]sync rqs */
442
-	int			nr_rqs_elvpriv;	/* # allocated rqs w/ elvpriv */
413
+
414
+	struct percpu_ref	q_usage_counter;
443415 
444416 	struct blk_queue_stats	*stats;
445417 	struct rq_qos		*rq_qos;
446418 
447
-	/*
448
-	 * If blkcg is not used, @q->root_rl serves all requests.  If blkcg
449
-	 * is used, root blkg allocates from @q->root_rl and all other
450
-	 * blkgs from their own blkg->rl.  Which one to use should be
451
-	 * determined using bio_request_list().
452
-	 */
453
-	struct request_list	root_rl;
454
-
455
-	request_fn_proc		*request_fn;
456
-	make_request_fn		*make_request_fn;
457
-	poll_q_fn		*poll_fn;
458
-	prep_rq_fn		*prep_rq_fn;
459
-	unprep_rq_fn		*unprep_rq_fn;
460
-	softirq_done_fn		*softirq_done_fn;
461
-	rq_timed_out_fn		*rq_timed_out_fn;
462
-	dma_drain_needed_fn	*dma_drain_needed;
463
-	lld_busy_fn		*lld_busy_fn;
464
-	/* Called just after a request is allocated */
465
-	init_rq_fn		*init_rq_fn;
466
-	/* Called just before a request is freed */
467
-	exit_rq_fn		*exit_rq_fn;
468
-	/* Called from inside blk_get_request() */
469
-	void (*initialize_rq_fn)(struct request *rq);
470
-
471419 	const struct blk_mq_ops	*mq_ops;
472
-
473
-	unsigned int		*mq_map;
474420 
475421 	/* sw queues */
476422 	struct blk_mq_ctx __percpu	*queue_ctx;
477
-	unsigned int		nr_queues;
478423 
479424 	unsigned int		queue_depth;
480425 
481426 	/* hw dispatch queues */
482427 	struct blk_mq_hw_ctx	**queue_hw_ctx;
483428 	unsigned int		nr_hw_queues;
484
-
485
-	/*
486
-	 * Dispatch queue sorting
487
-	 */
488
-	sector_t		end_sector;
489
-	struct request		*boundary_rq;
490
-
491
-	/*
492
-	 * Delayed queue handling
493
-	 */
494
-	struct delayed_work	delay_work;
495429 
496430 	struct backing_dev_info	*backing_dev_info;
497431 
....@@ -507,8 +441,7 @@
507441 	unsigned long		queue_flags;
508442 	/*
509443 	 * Number of contexts that have called blk_set_pm_only(). If this
510
-	 * counter is above zero then only RQF_PM and RQF_PREEMPT requests are
511
-	 * processed.
444
+	 * counter is above zero then only RQF_PM requests are processed.
512445 	 */
513446 	atomic_t		pm_only;
514447 
....@@ -523,13 +456,7 @@
523456 	 */
524457 	gfp_t			bounce_gfp;
525458 
526
-	/*
527
-	 * protects queue structures from reentrancy. ->__queue_lock should
528
-	 * _never_ be used directly, it is queue private. always use
529
-	 * ->queue_lock.
530
-	 */
531
-	spinlock_t		__queue_lock;
532
-	spinlock_t		*queue_lock;
459
+	spinlock_t		queue_lock;
533460 
534461 	/*
535462 	 * queue kobject
....@@ -539,7 +466,7 @@
539466 	/*
540467 	 * mq queue kobject
541468 	 */
542
-	struct kobject mq_kobj;
469
+	struct kobject *mq_kobj;
543470 
544471 #ifdef  CONFIG_BLK_DEV_INTEGRITY
545472 	struct blk_integrity integrity;
....@@ -547,7 +474,7 @@
547474 
548475 #ifdef CONFIG_PM
549476 	struct device		*dev;
550
-	int			rpm_status;
477
+	enum rpm_status		rpm_status;
551478 	unsigned int		nr_pending;
552479 #endif
553480 
....@@ -555,29 +482,13 @@
555482 	 * queue settings
556483 	 */
557484 	unsigned long		nr_requests;	/* Max # of requests */
558
-	unsigned int		nr_congestion_on;
559
-	unsigned int		nr_congestion_off;
560
-	unsigned int		nr_batching;
561485 
562
-	unsigned int		dma_drain_size;
563
-	void			*dma_drain_buffer;
564486 	unsigned int		dma_pad_mask;
565487 	unsigned int		dma_alignment;
566488 
567
-	struct blk_queue_tag	*queue_tags;
568
-
569
-	unsigned int		nr_sorted;
570
-	unsigned int		in_flight[2];
571
-
572
-	/*
573
-	 * Number of active block driver functions for which blk_drain_queue()
574
-	 * must wait. Must be incremented around functions that unlock the
575
-	 * queue_lock internally, e.g. scsi_request_fn().
576
-	 */
577
-	unsigned int		request_fn_active;
578489 #ifdef CONFIG_BLK_INLINE_ENCRYPTION
579490 	/* Inline crypto capabilities */
580
-	struct keyslot_manager *ksm;
491
+	struct blk_keyslot_manager *ksm;
581492 #endif
582493 
583494 	unsigned int		rq_timeout;
....@@ -588,7 +499,8 @@
588499 
589500 	struct timer_list	timeout;
590501 	struct work_struct	timeout_work;
591
-	struct list_head	timeout_list;
502
+
503
+	atomic_t		nr_active_requests_shared_sbitmap;
592504 
593505 	struct list_head	icq_list;
594506 #ifdef CONFIG_BLK_CGROUP
....@@ -599,13 +511,15 @@
599511 
600512 	struct queue_limits	limits;
601513 
514
+	unsigned int		required_elevator_features;
515
+
602516 #ifdef CONFIG_BLK_DEV_ZONED
603517 	/*
604518 	 * Zoned block device information for request dispatch control.
605519 	 * nr_zones is the total number of zones of the device. This is always
606
-	 * 0 for regular block devices. seq_zones_bitmap is a bitmap of nr_zones
607
-	 * bits which indicates if a zone is conventional (bit clear) or
608
-	 * sequential (bit set). seq_zones_wlock is a bitmap of nr_zones
520
+	 * 0 for regular block devices. conv_zones_bitmap is a bitmap of nr_zones
521
+	 * bits which indicates if a zone is conventional (bit set) or
522
+	 * sequential (bit clear). seq_zones_wlock is a bitmap of nr_zones
609523 	 * bits which indicates if a zone is write locked, that is, if a write
610524 	 * request targeting the zone was dispatched. All three fields are
611525 	 * initialized by the low level device driver (e.g. scsi/sd.c).
....@@ -618,8 +532,10 @@
618532 	 * blk_mq_unfreeze_queue().
619533 	 */
620534 	unsigned int		nr_zones;
621
-	unsigned long		*seq_zones_bitmap;
535
+	unsigned long		*conv_zones_bitmap;
622536 	unsigned long		*seq_zones_wlock;
537
+	unsigned int		max_open_zones;
538
+	unsigned int		max_active_zones;
623539 #endif /* CONFIG_BLK_DEV_ZONED */
624540 
625541 	/*
....@@ -628,9 +544,9 @@
628544 	unsigned int		sg_timeout;
629545 	unsigned int		sg_reserved_size;
630546 	int			node;
547
+	struct mutex		debugfs_mutex;
631548 #ifdef CONFIG_BLK_DEV_IO_TRACE
632549 	struct blk_trace __rcu	*blk_trace;
633
-	struct mutex		blk_trace_mutex;
634550 #endif
635551 	/*
636552 	 * for flush operations
....@@ -642,12 +558,20 @@
642558 	struct delayed_work	requeue_work;
643559 
644560 	struct mutex		sysfs_lock;
561
+	struct mutex		sysfs_dir_lock;
645562 
646
-	int			bypass_depth;
647
-	atomic_t		mq_freeze_depth;
563
+	/*
564
+	 * for reusing dead hctx instance in case of updating
565
+	 * nr_hw_queues
566
+	 */
567
+	struct list_head	unused_hctx_list;
568
+	spinlock_t		unused_hctx_lock;
648569 
649
-	bsg_job_fn		*bsg_job_fn;
570
+	int			mq_freeze_depth;
571
+
572
+#if defined(CONFIG_BLK_DEV_BSG)
650573 	struct bsg_class_device bsg_dev;
574
+#endif
651575 
652576 #ifdef CONFIG_BLK_DEV_THROTTLING
653577 	/* Throttle data */
....@@ -655,90 +579,105 @@
655579 #endif
656580 	struct rcu_head		rcu_head;
657581 	wait_queue_head_t	mq_freeze_wq;
658
-	struct percpu_ref	q_usage_counter;
659
-	struct list_head	all_q_node;
582
+	/*
583
+	 * Protect concurrent access to q_usage_counter by
584
+	 * percpu_ref_kill() and percpu_ref_reinit().
585
+	 */
586
+	struct mutex		mq_freeze_lock;
660587 
661588 	struct blk_mq_tag_set	*tag_set;
662589 	struct list_head	tag_set_list;
663590 	struct bio_set		bio_split;
664591 
665
-#ifdef CONFIG_BLK_DEBUG_FS
666592 	struct dentry		*debugfs_dir;
593
+
594
+#ifdef CONFIG_BLK_DEBUG_FS
667595 	struct dentry		*sched_debugfs_dir;
596
+	struct dentry		*rqos_debugfs_dir;
668597 #endif
669598 
670599 	bool			mq_sysfs_init_done;
671600 
672601 	size_t			cmd_size;
673
-	void			*rq_alloc_data;
674
-
675
-	struct work_struct	release_work;
676602 
677603 #define BLK_MAX_WRITE_HINTS	5
678604 	u64			write_hints[BLK_MAX_WRITE_HINTS];
605
+
606
+	ANDROID_KABI_RESERVE(1);
607
+	ANDROID_KABI_RESERVE(2);
608
+	ANDROID_KABI_RESERVE(3);
609
+	ANDROID_KABI_RESERVE(4);
610
+	ANDROID_OEM_DATA(1);
679611 };
680612 
681
-#define QUEUE_FLAG_QUEUED	0	/* uses generic tag queueing */
682
-#define QUEUE_FLAG_STOPPED	1	/* queue is stopped */
683
-#define QUEUE_FLAG_DYING	2	/* queue being torn down */
684
-#define QUEUE_FLAG_BYPASS	3	/* act as dumb FIFO queue */
685
-#define QUEUE_FLAG_BIDI		4	/* queue supports bidi requests */
686
-#define QUEUE_FLAG_NOMERGES     5	/* disable merge attempts */
687
-#define QUEUE_FLAG_SAME_COMP	6	/* complete on same CPU-group */
688
-#define QUEUE_FLAG_FAIL_IO	7	/* fake timeout */
689
-#define QUEUE_FLAG_NONROT	9	/* non-rotational device (SSD) */
690
-#define QUEUE_FLAG_VIRT        QUEUE_FLAG_NONROT /* paravirt device */
691
-#define QUEUE_FLAG_IO_STAT     10	/* do IO stats */
692
-#define QUEUE_FLAG_DISCARD     11	/* supports DISCARD */
693
-#define QUEUE_FLAG_NOXMERGES   12	/* No extended merges */
694
-#define QUEUE_FLAG_ADD_RANDOM  13	/* Contributes to random pool */
695
-#define QUEUE_FLAG_SECERASE    14	/* supports secure erase */
696
-#define QUEUE_FLAG_SAME_FORCE  15	/* force complete on same CPU */
697
-#define QUEUE_FLAG_DEAD        16	/* queue tear-down finished */
698
-#define QUEUE_FLAG_INIT_DONE   17	/* queue is initialized */
699
-#define QUEUE_FLAG_NO_SG_MERGE 18	/* don't attempt to merge SG segments*/
700
-#define QUEUE_FLAG_POLL	       19	/* IO polling enabled if set */
701
-#define QUEUE_FLAG_WC	       20	/* Write back caching */
702
-#define QUEUE_FLAG_FUA	       21	/* device supports FUA writes */
703
-#define QUEUE_FLAG_FLUSH_NQ    22	/* flush not queueuable */
704
-#define QUEUE_FLAG_DAX         23	/* device supports DAX */
705
-#define QUEUE_FLAG_STATS       24	/* track rq completion times */
706
-#define QUEUE_FLAG_POLL_STATS  25	/* collecting stats for hybrid polling */
707
-#define QUEUE_FLAG_REGISTERED  26	/* queue has been registered to a disk */
708
-#define QUEUE_FLAG_SCSI_PASSTHROUGH 27	/* queue supports SCSI commands */
709
-#define QUEUE_FLAG_QUIESCED    28	/* queue has been quiesced */
710
-
711
-#define QUEUE_FLAG_DEFAULT	((1 << QUEUE_FLAG_IO_STAT) |		\
712
-				 (1 << QUEUE_FLAG_SAME_COMP)	|	\
713
-				 (1 << QUEUE_FLAG_ADD_RANDOM))
613
+/* Keep blk_queue_flag_name[] in sync with the definitions below */
614
+#define QUEUE_FLAG_STOPPED	0	/* queue is stopped */
615
+#define QUEUE_FLAG_DYING	1	/* queue being torn down */
616
+#define QUEUE_FLAG_NOMERGES     3	/* disable merge attempts */
617
+#define QUEUE_FLAG_SAME_COMP	4	/* complete on same CPU-group */
618
+#define QUEUE_FLAG_FAIL_IO	5	/* fake timeout */
619
+#define QUEUE_FLAG_NONROT	6	/* non-rotational device (SSD) */
620
+#define QUEUE_FLAG_VIRT		QUEUE_FLAG_NONROT /* paravirt device */
621
+#define QUEUE_FLAG_IO_STAT	7	/* do disk/partitions IO accounting */
622
+#define QUEUE_FLAG_DISCARD	8	/* supports DISCARD */
623
+#define QUEUE_FLAG_NOXMERGES	9	/* No extended merges */
624
+#define QUEUE_FLAG_ADD_RANDOM	10	/* Contributes to random pool */
625
+#define QUEUE_FLAG_SECERASE	11	/* supports secure erase */
626
+#define QUEUE_FLAG_SAME_FORCE	12	/* force complete on same CPU */
627
+#define QUEUE_FLAG_DEAD		13	/* queue tear-down finished */
628
+#define QUEUE_FLAG_INIT_DONE	14	/* queue is initialized */
629
+#define QUEUE_FLAG_STABLE_WRITES 15	/* don't modify blks until WB is done */
630
+#define QUEUE_FLAG_POLL		16	/* IO polling enabled if set */
631
+#define QUEUE_FLAG_WC		17	/* Write back caching */
632
+#define QUEUE_FLAG_FUA		18	/* device supports FUA writes */
633
+#define QUEUE_FLAG_DAX		19	/* device supports DAX */
634
+#define QUEUE_FLAG_STATS	20	/* track IO start and completion times */
635
+#define QUEUE_FLAG_POLL_STATS	21	/* collecting stats for hybrid polling */
636
+#define QUEUE_FLAG_REGISTERED	22	/* queue has been registered to a disk */
637
+#define QUEUE_FLAG_SCSI_PASSTHROUGH 23	/* queue supports SCSI commands */
638
+#define QUEUE_FLAG_QUIESCED	24	/* queue has been quiesced */
639
+#define QUEUE_FLAG_PCI_P2PDMA	25	/* device supports PCI p2p requests */
640
+#define QUEUE_FLAG_ZONE_RESETALL 26	/* supports Zone Reset All */
641
+#define QUEUE_FLAG_RQ_ALLOC_TIME 27	/* record rq->alloc_time_ns */
642
+#define QUEUE_FLAG_HCTX_ACTIVE	28	/* at least one blk-mq hctx is active */
643
+#define QUEUE_FLAG_NOWAIT       29	/* device supports NOWAIT */
714644 
715645 #define QUEUE_FLAG_MQ_DEFAULT	((1 << QUEUE_FLAG_IO_STAT) |		\
716
-				 (1 << QUEUE_FLAG_SAME_COMP)	|	\
717
-				 (1 << QUEUE_FLAG_POLL))
646
+				 (1 << QUEUE_FLAG_SAME_COMP) |		\
647
+				 (1 << QUEUE_FLAG_NOWAIT))
718648 
719649 void blk_queue_flag_set(unsigned int flag, struct request_queue *q);
720650 void blk_queue_flag_clear(unsigned int flag, struct request_queue *q);
721651 bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q);
722
-bool blk_queue_flag_test_and_clear(unsigned int flag, struct request_queue *q);
723652 
724
-#define blk_queue_tagged(q)	test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags)
725653 #define blk_queue_stopped(q)	test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags)
726654 #define blk_queue_dying(q)	test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags)
727655 #define blk_queue_dead(q)	test_bit(QUEUE_FLAG_DEAD, &(q)->queue_flags)
728
-#define blk_queue_bypass(q)	test_bit(QUEUE_FLAG_BYPASS, &(q)->queue_flags)
729656 #define blk_queue_init_done(q)	test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags)
730657 #define blk_queue_nomerges(q)	test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags)
731658 #define blk_queue_noxmerges(q)	\
732659 	test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags)
733660 #define blk_queue_nonrot(q)	test_bit(QUEUE_FLAG_NONROT, &(q)->queue_flags)
661
+#define blk_queue_stable_writes(q) \
662
+	test_bit(QUEUE_FLAG_STABLE_WRITES, &(q)->queue_flags)
734663 #define blk_queue_io_stat(q)	test_bit(QUEUE_FLAG_IO_STAT, &(q)->queue_flags)
735664 #define blk_queue_add_random(q)	test_bit(QUEUE_FLAG_ADD_RANDOM, &(q)->queue_flags)
736665 #define blk_queue_discard(q)	test_bit(QUEUE_FLAG_DISCARD, &(q)->queue_flags)
666
+#define blk_queue_zone_resetall(q)	\
667
+	test_bit(QUEUE_FLAG_ZONE_RESETALL, &(q)->queue_flags)
737668 #define blk_queue_secure_erase(q) \
738669 	(test_bit(QUEUE_FLAG_SECERASE, &(q)->queue_flags))
739670 #define blk_queue_dax(q)	test_bit(QUEUE_FLAG_DAX, &(q)->queue_flags)
740671 #define blk_queue_scsi_passthrough(q)	\
741672 	test_bit(QUEUE_FLAG_SCSI_PASSTHROUGH, &(q)->queue_flags)
673
+#define blk_queue_pci_p2pdma(q)	\
674
+	test_bit(QUEUE_FLAG_PCI_P2PDMA, &(q)->queue_flags)
675
+#ifdef CONFIG_BLK_RQ_ALLOC_TIME
676
+#define blk_queue_rq_alloc_time(q)	\
677
+	test_bit(QUEUE_FLAG_RQ_ALLOC_TIME, &(q)->queue_flags)
678
+#else
679
+#define blk_queue_rq_alloc_time(q)	false
680
+#endif
742681 
743682 #define blk_noretry_request(rq) \
744683 	((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \
....@@ -747,47 +686,38 @@
747686 #define blk_queue_pm_only(q)	atomic_read(&(q)->pm_only)
748687 #define blk_queue_fua(q)	test_bit(QUEUE_FLAG_FUA, &(q)->queue_flags)
749688 #define blk_queue_registered(q)	test_bit(QUEUE_FLAG_REGISTERED, &(q)->queue_flags)
689
+#define blk_queue_nowait(q)	test_bit(QUEUE_FLAG_NOWAIT, &(q)->queue_flags)
750690 
751691 extern void blk_set_pm_only(struct request_queue *q);
752692 extern void blk_clear_pm_only(struct request_queue *q);
753
-
754
-static inline int queue_in_flight(struct request_queue *q)
755
-{
756
-	return q->in_flight[0] + q->in_flight[1];
757
-}
758693 
759694 static inline bool blk_account_rq(struct request *rq)
760695 {
761696 	return (rq->rq_flags & RQF_STARTED) && !blk_rq_is_passthrough(rq);
762697 }
763698 
764
-#define blk_rq_cpu_valid(rq)	((rq)->cpu != -1)
765
-#define blk_bidi_rq(rq)		((rq)->next_rq != NULL)
766
-/* rq->queuelist of dequeued request must be list_empty() */
767
-#define blk_queued_rq(rq)	(!list_empty(&(rq)->queuelist))
768
-
769699 #define list_entry_rq(ptr)	list_entry((ptr), struct request, queuelist)
770700 
771701 #define rq_data_dir(rq)		(op_is_write(req_op(rq)) ? WRITE : READ)
772702 
773
-/*
774
- * Driver can handle struct request, if it either has an old style
775
- * request_fn defined, or is blk-mq based.
776
- */
777
-static inline bool queue_is_rq_based(struct request_queue *q)
778
-{
779
-	return q->request_fn || q->mq_ops;
780
-}
703
+#define rq_dma_dir(rq) \
704
+	(op_is_write(req_op(rq)) ? DMA_TO_DEVICE : DMA_FROM_DEVICE)
781705 
782
-static inline unsigned int blk_queue_cluster(struct request_queue *q)
706
+#define dma_map_bvec(dev, bv, dir, attrs) \
707
+	dma_map_page_attrs(dev, (bv)->bv_page, (bv)->bv_offset, (bv)->bv_len, \
708
+	(dir), (attrs))
709
+
710
+static inline bool queue_is_mq(struct request_queue *q)
783711 {
784
-	return q->limits.cluster;
712
+	return q->mq_ops;
785713 }
786714 
787715 static inline enum blk_zoned_model
788716 blk_queue_zoned_model(struct request_queue *q)
789717 {
790
-	return q->limits.zoned;
718
+	if (IS_ENABLED(CONFIG_BLK_DEV_ZONED))
719
+		return q->limits.zoned;
720
+	return BLK_ZONED_NONE;
791721 }
792722 
793723 static inline bool blk_queue_is_zoned(struct request_queue *q)
....@@ -801,12 +731,17 @@
801731 	}
802732 }
803733 
804
-static inline unsigned int blk_queue_zone_sectors(struct request_queue *q)
734
+static inline sector_t blk_queue_zone_sectors(struct request_queue *q)
805735 {
806736 	return blk_queue_is_zoned(q) ? q->limits.chunk_sectors : 0;
807737 }
808738 
809739 #ifdef CONFIG_BLK_DEV_ZONED
740
+static inline unsigned int blk_queue_nr_zones(struct request_queue *q)
741
+{
742
+	return blk_queue_is_zoned(q) ? q->nr_zones : 0;
743
+}
744
+
810745 static inline unsigned int blk_queue_zone_no(struct request_queue *q,
811746 					     sector_t sector)
812747 {
....@@ -818,36 +753,62 @@
818753 static inline bool blk_queue_zone_is_seq(struct request_queue *q,
819754 					 sector_t sector)
820755 {
821
-	if (!blk_queue_is_zoned(q) || !q->seq_zones_bitmap)
756
+	if (!blk_queue_is_zoned(q))
822757 		return false;
823
-	return test_bit(blk_queue_zone_no(q, sector), q->seq_zones_bitmap);
758
+	if (!q->conv_zones_bitmap)
759
+		return true;
760
+	return !test_bit(blk_queue_zone_no(q, sector), q->conv_zones_bitmap);
761
+}
762
+
763
+static inline void blk_queue_max_open_zones(struct request_queue *q,
764
+		unsigned int max_open_zones)
765
+{
766
+	q->max_open_zones = max_open_zones;
767
+}
768
+
769
+static inline unsigned int queue_max_open_zones(const struct request_queue *q)
770
+{
771
+	return q->max_open_zones;
772
+}
773
+
774
+static inline void blk_queue_max_active_zones(struct request_queue *q,
775
+		unsigned int max_active_zones)
776
+{
777
+	q->max_active_zones = max_active_zones;
778
+}
779
+
780
+static inline unsigned int queue_max_active_zones(const struct request_queue *q)
781
+{
782
+	return q->max_active_zones;
783
+}
784
+#else /* CONFIG_BLK_DEV_ZONED */
785
+static inline unsigned int blk_queue_nr_zones(struct request_queue *q)
786
+{
787
+	return 0;
788
+}
789
+static inline bool blk_queue_zone_is_seq(struct request_queue *q,
790
+					 sector_t sector)
791
+{
792
+	return false;
793
+}
794
+static inline unsigned int blk_queue_zone_no(struct request_queue *q,
795
+					     sector_t sector)
796
+{
797
+	return 0;
798
+}
799
+static inline unsigned int queue_max_open_zones(const struct request_queue *q)
800
+{
801
+	return 0;
802
+}
803
+static inline unsigned int queue_max_active_zones(const struct request_queue *q)
804
+{
805
+	return 0;
824806 }
825807 #endif /* CONFIG_BLK_DEV_ZONED */
826808 
827809 static inline bool rq_is_sync(struct request *rq)
828810 {
829811 	return op_is_sync(rq->cmd_flags);
830
-}
831
-
832
-static inline bool blk_rl_full(struct request_list *rl, bool sync)
833
-{
834
-	unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;
835
-
836
-	return rl->flags & flag;
837
-}
838
-
839
-static inline void blk_set_rl_full(struct request_list *rl, bool sync)
840
-{
841
-	unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;
842
-
843
-	rl->flags |= flag;
844
-}
845
-
846
-static inline void blk_clear_rl_full(struct request_list *rl, bool sync)
847
-{
848
-	unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;
849
-
850
-	rl->flags &= ~flag;
851812 }
852813 
853814 static inline bool rq_mergeable(struct request *rq)
....@@ -859,6 +820,9 @@
859820 		return false;
860821 
861822 	if (req_op(rq) == REQ_OP_WRITE_ZEROES)
823
+		return false;
824
+
825
+	if (req_op(rq) == REQ_OP_ZONE_APPEND)
862826 		return false;
863827 
864828 	if (rq->cmd_flags & REQ_NOMERGE_FLAGS)
....@@ -885,16 +849,6 @@
885849 
886850 	return q->nr_requests;
887851 }
888
-
889
-/*
890
- * q->prep_rq_fn return values
891
- */
892
-enum {
893
-	BLKPREP_OK,		/* serve it */
894
-	BLKPREP_KILL,		/* fatal error, kill, return -EIO */
895
-	BLKPREP_DEFER,		/* leave on queue */
896
-	BLKPREP_INVALID,	/* invalid command, kill, return -EREMOTEIO */
897
-};
898852 
899853 extern unsigned long blk_max_low_pfn, blk_max_pfn;
900854 
....@@ -945,6 +899,10 @@
945899 	__rq_for_each_bio(_iter.bio, _rq)			\
946900 		bio_for_each_segment(bvl, _iter.bio, _iter.iter)
947901 
902
+#define rq_for_each_bvec(bvl, _rq, _iter)			\
903
+	__rq_for_each_bio(_iter.bio, _rq)			\
904
+		bio_for_each_bvec(bvl, _iter.bio, _iter.iter)
905
+
948906 #define rq_iter_last(bvec, _iter)				\
949907 		(_iter.bio->bi_next == NULL &&			\
950908 		 bio_iter_last(bvec, _iter.iter))
....@@ -962,15 +920,11 @@
962920 
963921 extern int blk_register_queue(struct gendisk *disk);
964922 extern void blk_unregister_queue(struct gendisk *disk);
965
-extern blk_qc_t generic_make_request(struct bio *bio);
966
-extern blk_qc_t direct_make_request(struct bio *bio);
923
+blk_qc_t submit_bio_noacct(struct bio *bio);
967924 extern void blk_rq_init(struct request_queue *q, struct request *rq);
968
-extern void blk_init_request_from_bio(struct request *req, struct bio *bio);
969925 extern void blk_put_request(struct request *);
970
-extern void __blk_put_request(struct request_queue *, struct request *);
971926 extern struct request *blk_get_request(struct request_queue *, unsigned int op,
972927 				       blk_mq_req_flags_t flags);
973
-extern void blk_requeue_request(struct request_queue *, struct request *);
974928 extern int blk_lld_busy(struct request_queue *q);
975929 extern int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
976930 			     struct bio_set *bs, gfp_t gfp_mask,
....@@ -980,9 +934,7 @@
980934 extern blk_status_t blk_insert_cloned_request(struct request_queue *q,
981935 				     struct request *rq);
982936 extern int blk_rq_append_bio(struct request *rq, struct bio **bio);
983
-extern void blk_delay_queue(struct request_queue *, unsigned long);
984
-extern void blk_queue_split(struct request_queue *, struct bio **);
985
-extern void blk_recount_segments(struct request_queue *, struct bio *);
937
+extern void blk_queue_split(struct bio **);
986938 extern int scsi_verify_blk_ioctl(struct block_device *, unsigned int);
987939 extern int scsi_cmd_blk_ioctl(struct block_device *, fmode_t,
988940 			      unsigned int, void __user *);
....@@ -990,18 +942,12 @@
990942 			  unsigned int, void __user *);
991943 extern int sg_scsi_ioctl(struct request_queue *, struct gendisk *, fmode_t,
992944 			 struct scsi_ioctl_command __user *);
945
+extern int get_sg_io_hdr(struct sg_io_hdr *hdr, const void __user *argp);
946
+extern int put_sg_io_hdr(const struct sg_io_hdr *hdr, void __user *argp);
993947 
994948 extern int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags);
995949 extern void blk_queue_exit(struct request_queue *q);
996
-extern void blk_start_queue(struct request_queue *q);
997
-extern void blk_start_queue_async(struct request_queue *q);
998
-extern void blk_stop_queue(struct request_queue *q);
999950 extern void blk_sync_queue(struct request_queue *q);
1000
-extern void __blk_stop_queue(struct request_queue *q);
1001
-extern void __blk_run_queue(struct request_queue *q);
1002
-extern void __blk_run_queue_uncond(struct request_queue *q);
1003
-extern void blk_run_queue(struct request_queue *);
1004
-extern void blk_run_queue_async(struct request_queue *q);
1005951 extern int blk_rq_map_user(struct request_queue *, struct request *,
1006952 			   struct rq_map_data *, void __user *, unsigned long,
1007953 			   gfp_t);
....@@ -1015,10 +961,13 @@
1015961 extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *,
1016962 				  struct request *, int, rq_end_io_fn *);
1017963 
964
+/* Helper to convert REQ_OP_XXX to its string format XXX */
965
+extern const char *blk_op_str(unsigned int op);
966
+
1018967 int blk_status_to_errno(blk_status_t status);
1019968 blk_status_t errno_to_blk_status(int errno);
1020969 
1021
-bool blk_poll(struct request_queue *q, blk_qc_t cookie);
970
+int blk_poll(struct request_queue *q, blk_qc_t cookie, bool spin);
1022971 
1023972 static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
1024973 {
....@@ -1045,6 +994,7 @@
1045994  * blk_rq_err_bytes()		: bytes left till the next error boundary
1046995  * blk_rq_sectors()		: sectors left in the entire request
1047996  * blk_rq_cur_sectors()		: sectors left in the current segment
997
+ * blk_rq_stats_sectors()	: sectors of the entire request used for stats
1048998  */
1049999 static inline sector_t blk_rq_pos(const struct request *rq)
10501000 {
....@@ -1073,7 +1023,16 @@
10731023 	return blk_rq_cur_bytes(rq) >> SECTOR_SHIFT;
10741024 }
10751025 
1026
+static inline unsigned int blk_rq_stats_sectors(const struct request *rq)
1027
+{
1028
+	return rq->stats_sectors;
1029
+}
1030
+
10761031 #ifdef CONFIG_BLK_DEV_ZONED
1032
+
1033
+/* Helper to convert BLK_ZONE_ZONE_XXX to its string format XXX */
1034
+const char *blk_zone_cond_str(enum blk_zone_cond zone_cond);
1035
+
10771036 static inline unsigned int blk_rq_zone_no(struct request *rq)
10781037 {
10791038 	return blk_queue_zone_no(rq->q, blk_rq_pos(rq));
....@@ -1098,6 +1057,17 @@
10981057 	return blk_rq_bytes(rq);
10991058 }
11001059 
1060
+/*
1061
+ * Return the first full biovec in the request.  The caller needs to check that
1062
+ * there are any bvecs before calling this helper.
1063
+ */
1064
+static inline struct bio_vec req_bvec(struct request *rq)
1065
+{
1066
+	if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
1067
+		return rq->special_vec;
1068
+	return mp_bvec_iter_bvec(rq->bio->bi_io_vec, rq->bio->bi_iter);
1069
+}
1070
+
11011071 static inline unsigned int blk_queue_get_max_sectors(struct request_queue *q,
11021072 						     int op)
11031073 {
....@@ -1119,13 +1089,22 @@
11191089  * file system requests.
11201090  */
11211091 static inline unsigned int blk_max_size_offset(struct request_queue *q,
1122
-					       sector_t offset)
1092
+					       sector_t offset,
1093
+					       unsigned int chunk_sectors)
11231094 {
1124
-	if (!q->limits.chunk_sectors)
1125
-		return q->limits.max_sectors;
1095
+	if (!chunk_sectors) {
1096
+		if (q->limits.chunk_sectors)
1097
+			chunk_sectors = q->limits.chunk_sectors;
1098
+		else
1099
+			return q->limits.max_sectors;
1100
+	}
11261101 
1127
-	return min(q->limits.max_sectors, (unsigned int)(q->limits.chunk_sectors -
1128
-			(offset & (q->limits.chunk_sectors - 1))));
1102
+	if (likely(is_power_of_2(chunk_sectors)))
1103
+		chunk_sectors -= offset & (chunk_sectors - 1);
1104
+	else
1105
+		chunk_sectors -= sector_div(offset, chunk_sectors);
1106
+
1107
+	return min(q->limits.max_sectors, chunk_sectors);
11291108 }
11301109 
11311110 static inline unsigned int blk_rq_get_max_sectors(struct request *rq,
....@@ -1141,7 +1120,7 @@
11411120 	    req_op(rq) == REQ_OP_SECURE_ERASE)
11421121 		return blk_queue_get_max_sectors(q, req_op(rq));
11431122 
1144
-	return min(blk_max_size_offset(q, offset),
1123
+	return min(blk_max_size_offset(q, offset, 0),
11451124 			blk_queue_get_max_sectors(q, req_op(rq)));
11461125 }
11471126 
....@@ -1156,13 +1135,6 @@
11561135 	return nr_bios;
11571136 }
11581137 
1159
-/*
1160
- * Request issue related functions.
1161
- */
1162
-extern struct request *blk_peek_request(struct request_queue *q);
1163
-extern void blk_start_request(struct request *rq);
1164
-extern struct request *blk_fetch_request(struct request_queue *q);
1165
-
11661138 void blk_steal_bios(struct bio_list *list, struct request *rq);
11671139 
11681140 /*
....@@ -1170,39 +1142,16 @@
11701142  *
11711143  * blk_update_request() completes given number of bytes and updates
11721144  * the request without completing it.
1173
- *
1174
- * blk_end_request() and friends.  __blk_end_request() must be called
1175
- * with the request queue spinlock acquired.
1176
- *
1177
- * Several drivers define their own end_request and call
1178
- * blk_end_request() for parts of the original function.
1179
- * This prevents code duplication in drivers.
11801145  */
11811146 extern bool blk_update_request(struct request *rq, blk_status_t error,
11821147 			       unsigned int nr_bytes);
1183
-extern void blk_finish_request(struct request *rq, blk_status_t error);
1184
-extern bool blk_end_request(struct request *rq, blk_status_t error,
1185
-			    unsigned int nr_bytes);
1186
-extern void blk_end_request_all(struct request *rq, blk_status_t error);
1187
-extern bool __blk_end_request(struct request *rq, blk_status_t error,
1188
-			      unsigned int nr_bytes);
1189
-extern void __blk_end_request_all(struct request *rq, blk_status_t error);
1190
-extern bool __blk_end_request_cur(struct request *rq, blk_status_t error);
11911148 
1192
-extern void blk_complete_request(struct request *);
1193
-extern void __blk_complete_request(struct request *);
11941149 extern void blk_abort_request(struct request *);
1195
-extern void blk_unprep_request(struct request *);
11961150 
11971151 /*
11981152  * Access functions for manipulating queue properties
11991153  */
1200
-extern struct request_queue *blk_init_queue_node(request_fn_proc *rfn,
1201
-					spinlock_t *lock, int node_id);
1202
-extern struct request_queue *blk_init_queue(request_fn_proc *, spinlock_t *);
1203
-extern int blk_init_allocated_queue(struct request_queue *);
12041154 extern void blk_cleanup_queue(struct request_queue *);
1205
-extern void blk_queue_make_request(struct request_queue *, make_request_fn *);
12061155 extern void blk_queue_bounce_limit(struct request_queue *, u64);
12071156 extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int);
12081157 extern void blk_queue_chunk_sectors(struct request_queue *, unsigned int);
....@@ -1217,9 +1166,12 @@
12171166 extern void blk_queue_max_write_zeroes_sectors(struct request_queue *q,
12181167 		unsigned int max_write_same_sectors);
12191168 extern void blk_queue_logical_block_size(struct request_queue *, unsigned int);
1169
+extern void blk_queue_max_zone_append_sectors(struct request_queue *q,
1170
+		unsigned int max_zone_append_sectors);
12201171 extern void blk_queue_physical_block_size(struct request_queue *, unsigned int);
12211172 extern void blk_queue_alignment_offset(struct request_queue *q,
12221173 				       unsigned int alignment);
1174
+void blk_queue_update_readahead(struct request_queue *q);
12231175 extern void blk_limits_io_min(struct queue_limits *limits, unsigned int min);
12241176 extern void blk_queue_io_min(struct request_queue *q, unsigned int min);
12251177 extern void blk_limits_io_opt(struct queue_limits *limits, unsigned int opt);
....@@ -1229,28 +1181,19 @@
12291181 extern void blk_set_stacking_limits(struct queue_limits *lim);
12301182 extern int blk_stack_limits(struct queue_limits *t, struct queue_limits *b,
12311183 			    sector_t offset);
1232
-extern int bdev_stack_limits(struct queue_limits *t, struct block_device *bdev,
1233
-			    sector_t offset);
12341184 extern void disk_stack_limits(struct gendisk *disk, struct block_device *bdev,
12351185 			      sector_t offset);
1236
-extern void blk_queue_stack_limits(struct request_queue *t, struct request_queue *b);
1237
-extern void blk_queue_dma_pad(struct request_queue *, unsigned int);
12381186 extern void blk_queue_update_dma_pad(struct request_queue *, unsigned int);
1239
-extern int blk_queue_dma_drain(struct request_queue *q,
1240
-			       dma_drain_needed_fn *dma_drain_needed,
1241
-			       void *buf, unsigned int size);
1242
-extern void blk_queue_lld_busy(struct request_queue *q, lld_busy_fn *fn);
12431187 extern void blk_queue_segment_boundary(struct request_queue *, unsigned long);
12441188 extern void blk_queue_virt_boundary(struct request_queue *, unsigned long);
1245
-extern void blk_queue_prep_rq(struct request_queue *, prep_rq_fn *pfn);
1246
-extern void blk_queue_unprep_rq(struct request_queue *, unprep_rq_fn *ufn);
12471189 extern void blk_queue_dma_alignment(struct request_queue *, int);
12481190 extern void blk_queue_update_dma_alignment(struct request_queue *, int);
1249
-extern void blk_queue_softirq_done(struct request_queue *, softirq_done_fn *);
1250
-extern void blk_queue_rq_timed_out(struct request_queue *, rq_timed_out_fn *);
12511191 extern void blk_queue_rq_timeout(struct request_queue *, unsigned int);
1252
-extern void blk_queue_flush_queueable(struct request_queue *q, bool queueable);
12531192 extern void blk_queue_write_cache(struct request_queue *q, bool enabled, bool fua);
1193
+extern void blk_queue_required_elevator_features(struct request_queue *q,
1194
+						 unsigned int features);
1195
+extern bool blk_queue_can_use_dma_map_merging(struct request_queue *q,
1196
+					      struct device *dev);
12541197 
12551198 /*
12561199  * Number of physical segments as sent to the device.
....@@ -1277,40 +1220,23 @@
12771220 	return max_t(unsigned short, rq->nr_phys_segments, 1);
12781221 }
12791222 
1280
-extern int blk_rq_map_sg(struct request_queue *, struct request *, struct scatterlist *);
1223
+int __blk_rq_map_sg(struct request_queue *q, struct request *rq,
1224
+		struct scatterlist *sglist, struct scatterlist **last_sg);
1225
+static inline int blk_rq_map_sg(struct request_queue *q, struct request *rq,
1226
+		struct scatterlist *sglist)
1227
+{
1228
+	struct scatterlist *last_sg = NULL;
1229
+
1230
+	return __blk_rq_map_sg(q, rq, sglist, &last_sg);
1231
+}
12811232 extern void blk_dump_rq_flags(struct request *, char *);
1282
-extern long nr_blockdev_pages(void);
12831233 
12841234 bool __must_check blk_get_queue(struct request_queue *);
1285
-struct request_queue *blk_alloc_queue(gfp_t);
1286
-struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id,
1287
-					   spinlock_t *lock);
1235
+struct request_queue *blk_alloc_queue(int node_id);
12881236 extern void blk_put_queue(struct request_queue *);
12891237 extern void blk_set_queue_dying(struct request_queue *);
12901238 
1291
-/*
1292
- * block layer runtime pm functions
1293
- */
1294
-#ifdef CONFIG_PM
1295
-extern void blk_pm_runtime_init(struct request_queue *q, struct device *dev);
1296
-extern int blk_pre_runtime_suspend(struct request_queue *q);
1297
-extern void blk_post_runtime_suspend(struct request_queue *q, int err);
1298
-extern void blk_pre_runtime_resume(struct request_queue *q);
1299
-extern void blk_post_runtime_resume(struct request_queue *q, int err);
1300
-extern void blk_set_runtime_active(struct request_queue *q);
1301
-#else
1302
-static inline void blk_pm_runtime_init(struct request_queue *q,
1303
-	struct device *dev) {}
1304
-static inline int blk_pre_runtime_suspend(struct request_queue *q)
1305
-{
1306
-	return -ENOSYS;
1307
-}
1308
-static inline void blk_post_runtime_suspend(struct request_queue *q, int err) {}
1309
-static inline void blk_pre_runtime_resume(struct request_queue *q) {}
1310
-static inline void blk_post_runtime_resume(struct request_queue *q, int err) {}
1311
-static inline void blk_set_runtime_active(struct request_queue *q) {}
1312
-#endif
1313
-
1239
+#ifdef CONFIG_BLOCK
13141240 /*
13151241  * blk_plug permits building a queue of related requests by holding the I/O
13161242  * fragments for a short period. This allows merging of sequential requests
....@@ -1324,12 +1250,12 @@
13241250  * schedule() where blk_schedule_flush_plug() is called.
13251251  */
13261252 struct blk_plug {
1327
-	struct list_head list; /* requests */
13281253 	struct list_head mq_list; /* blk-mq requests */
13291254 	struct list_head cb_list; /* md requires an unplug callback */
1255
+	unsigned short rq_count;
1256
+	bool multiple_queues;
1257
+	bool nowait;
13301258 };
1331
-#define BLK_MAX_REQUEST_COUNT 16
1332
-#define BLK_PLUG_FLUSH_SIZE (128 * 1024)
13331259 
13341260 struct blk_plug_cb;
13351261 typedef void (*blk_plug_cb_fn)(struct blk_plug_cb *, bool);
....@@ -1365,32 +1291,51 @@
13651291 	struct blk_plug *plug = tsk->plug;
13661292 
13671293 	return plug &&
1368
-		(!list_empty(&plug->list) ||
1369
-		 !list_empty(&plug->mq_list) ||
1294
+		 (!list_empty(&plug->mq_list) ||
13701295 		 !list_empty(&plug->cb_list));
13711296 }
13721297 
1373
-/*
1374
- * tag stuff
1375
- */
1376
-extern int blk_queue_start_tag(struct request_queue *, struct request *);
1377
-extern struct request *blk_queue_find_tag(struct request_queue *, int);
1378
-extern void blk_queue_end_tag(struct request_queue *, struct request *);
1379
-extern int blk_queue_init_tags(struct request_queue *, int, struct blk_queue_tag *, int);
1380
-extern void blk_queue_free_tags(struct request_queue *);
1381
-extern int blk_queue_resize_tags(struct request_queue *, int);
1382
-extern struct blk_queue_tag *blk_init_tags(int, int);
1383
-extern void blk_free_tags(struct blk_queue_tag *);
1298
+int blkdev_issue_flush(struct block_device *, gfp_t);
1299
+long nr_blockdev_pages(void);
1300
+#else /* CONFIG_BLOCK */
1301
+struct blk_plug {
1302
+};
13841303 
1385
-static inline struct request *blk_map_queue_find_tag(struct blk_queue_tag *bqt,
1386
-						int tag)
1304
+static inline void blk_start_plug(struct blk_plug *plug)
13871305 {
1388
-	if (unlikely(bqt == NULL || tag >= bqt->real_max_depth))
1389
-		return NULL;
1390
-	return bqt->tag_index[tag];
13911306 }
13921307 
1393
-extern int blkdev_issue_flush(struct block_device *, gfp_t, sector_t *);
1308
+static inline void blk_finish_plug(struct blk_plug *plug)
1309
+{
1310
+}
1311
+
1312
+static inline void blk_flush_plug(struct task_struct *task)
1313
+{
1314
+}
1315
+
1316
+static inline void blk_schedule_flush_plug(struct task_struct *task)
1317
+{
1318
+}
1319
+
1320
+
1321
+static inline bool blk_needs_flush_plug(struct task_struct *tsk)
1322
+{
1323
+	return false;
1324
+}
1325
+
1326
+static inline int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask)
1327
+{
1328
+	return 0;
1329
+}
1330
+
1331
+static inline long nr_blockdev_pages(void)
1332
+{
1333
+	return 0;
1334
+}
1335
+#endif /* CONFIG_BLOCK */
1336
+
1337
+extern void blk_io_schedule(void);
1338
+
13941339 extern int blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
13951340 		sector_t nr_sects, gfp_t gfp_mask, struct page *page);
13961341 
....@@ -1434,6 +1379,11 @@
14341379 
14351380 extern int blk_verify_command(unsigned char *cmd, fmode_t mode);
14361381 
1382
+static inline bool bdev_is_partition(struct block_device *bdev)
1383
+{
1384
+	return bdev->bd_partno;
1385
+}
1386
+
14371387 enum blk_default_limits {
14381388 	BLK_MAX_SEGMENTS	= 128,
14391389 	BLK_SAFE_MAX_SECTORS	= 255,
....@@ -1442,42 +1392,50 @@
14421392 	BLK_SEG_BOUNDARY_MASK	= 0xFFFFFFFFUL,
14431393 };
14441394 
1445
-static inline unsigned long queue_segment_boundary(struct request_queue *q)
1395
+static inline unsigned long queue_segment_boundary(const struct request_queue *q)
14461396 {
14471397 	return q->limits.seg_boundary_mask;
14481398 }
14491399 
1450
-static inline unsigned long queue_virt_boundary(struct request_queue *q)
1400
+static inline unsigned long queue_virt_boundary(const struct request_queue *q)
14511401 {
14521402 	return q->limits.virt_boundary_mask;
14531403 }
14541404 
1455
-static inline unsigned int queue_max_sectors(struct request_queue *q)
1405
+static inline unsigned int queue_max_sectors(const struct request_queue *q)
14561406 {
14571407 	return q->limits.max_sectors;
14581408 }
14591409 
1460
-static inline unsigned int queue_max_hw_sectors(struct request_queue *q)
1410
+static inline unsigned int queue_max_hw_sectors(const struct request_queue *q)
14611411 {
14621412 	return q->limits.max_hw_sectors;
14631413 }
14641414 
1465
-static inline unsigned short queue_max_segments(struct request_queue *q)
1415
+static inline unsigned short queue_max_segments(const struct request_queue *q)
14661416 {
14671417 	return q->limits.max_segments;
14681418 }
14691419 
1470
-static inline unsigned short queue_max_discard_segments(struct request_queue *q)
1420
+static inline unsigned short queue_max_discard_segments(const struct request_queue *q)
14711421 {
14721422 	return q->limits.max_discard_segments;
14731423 }
14741424 
1475
-static inline unsigned int queue_max_segment_size(struct request_queue *q)
1425
+static inline unsigned int queue_max_segment_size(const struct request_queue *q)
14761426 {
14771427 	return q->limits.max_segment_size;
14781428 }
14791429 
1480
-static inline unsigned queue_logical_block_size(struct request_queue *q)
1430
+static inline unsigned int queue_max_zone_append_sectors(const struct request_queue *q)
1431
+{
1432
+
1433
+	const struct queue_limits *l = &q->limits;
1434
+
1435
+	return min(l->max_zone_append_sectors, l->max_sectors);
1436
+}
1437
+
1438
+static inline unsigned queue_logical_block_size(const struct request_queue *q)
14811439 {
14821440 	int retval = 512;
14831441 
....@@ -1492,7 +1450,7 @@
14921450 	return queue_logical_block_size(bdev_get_queue(bdev));
14931451 }
14941452 
1495
-static inline unsigned int queue_physical_block_size(struct request_queue *q)
1453
+static inline unsigned int queue_physical_block_size(const struct request_queue *q)
14961454 {
14971455 	return q->limits.physical_block_size;
14981456 }
....@@ -1502,7 +1460,7 @@
15021460 	return queue_physical_block_size(bdev_get_queue(bdev));
15031461 }
15041462 
1505
-static inline unsigned int queue_io_min(struct request_queue *q)
1463
+static inline unsigned int queue_io_min(const struct request_queue *q)
15061464 {
15071465 	return q->limits.io_min;
15081466 }
....@@ -1512,7 +1470,7 @@
15121470 	return queue_io_min(bdev_get_queue(bdev));
15131471 }
15141472 
1515
-static inline unsigned int queue_io_opt(struct request_queue *q)
1473
+static inline unsigned int queue_io_opt(const struct request_queue *q)
15161474 {
15171475 	return q->limits.io_opt;
15181476 }
....@@ -1522,7 +1480,7 @@
15221480 	return queue_io_opt(bdev_get_queue(bdev));
15231481 }
15241482 
1525
-static inline int queue_alignment_offset(struct request_queue *q)
1483
+static inline int queue_alignment_offset(const struct request_queue *q)
15261484 {
15271485 	if (q->limits.misaligned)
15281486 		return -1;
....@@ -1545,14 +1503,13 @@
15451503 
15461504 	if (q->limits.misaligned)
15471505 		return -1;
1548
-
1549
-	if (bdev != bdev->bd_contains)
1550
-		return bdev->bd_part->alignment_offset;
1551
-
1506
+	if (bdev_is_partition(bdev))
1507
+		return queue_limit_alignment_offset(&q->limits,
1508
+				bdev->bd_part->start_sect);
15521509 	return q->limits.alignment_offset;
15531510 }
15541511 
1555
-static inline int queue_discard_alignment(struct request_queue *q)
1512
+static inline int queue_discard_alignment(const struct request_queue *q)
15561513 {
15571514 	if (q->limits.discard_misaligned)
15581515 		return -1;
....@@ -1583,13 +1540,29 @@
15831540 	return offset << SECTOR_SHIFT;
15841541 }
15851542 
1543
+/*
1544
+ * Two cases of handling DISCARD merge:
1545
+ * If max_discard_segments > 1, the driver takes every bio
1546
+ * as a range and send them to controller together. The ranges
1547
+ * needn't to be contiguous.
1548
+ * Otherwise, the bios/requests will be handled as same as
1549
+ * others which should be contiguous.
1550
+ */
1551
+static inline bool blk_discard_mergable(struct request *req)
1552
+{
1553
+	if (req_op(req) == REQ_OP_DISCARD &&
1554
+	    queue_max_discard_segments(req->q) > 1)
1555
+		return true;
1556
+	return false;
1557
+}
1558
+
15861559 static inline int bdev_discard_alignment(struct block_device *bdev)
15871560 {
15881561 	struct request_queue *q = bdev_get_queue(bdev);
15891562 
1590
-	if (bdev != bdev->bd_contains)
1591
-		return bdev->bd_part->discard_alignment;
1592
-
1563
+	if (bdev_is_partition(bdev))
1564
+		return queue_limit_discard_alignment(&q->limits,
1565
+				bdev->bd_part->start_sect);
15931566 	return q->limits.discard_alignment;
15941567 }
15951568 
....@@ -1633,7 +1606,7 @@
16331606 	return false;
16341607 }
16351608 
1636
-static inline unsigned int bdev_zone_sectors(struct block_device *bdev)
1609
+static inline sector_t bdev_zone_sectors(struct block_device *bdev)
16371610 {
16381611 	struct request_queue *q = bdev_get_queue(bdev);
16391612 
....@@ -1642,7 +1615,25 @@
16421615 	return 0;
16431616 }
16441617 
1645
-static inline int queue_dma_alignment(struct request_queue *q)
1618
+static inline unsigned int bdev_max_open_zones(struct block_device *bdev)
1619
+{
1620
+	struct request_queue *q = bdev_get_queue(bdev);
1621
+
1622
+	if (q)
1623
+		return queue_max_open_zones(q);
1624
+	return 0;
1625
+}
1626
+
1627
+static inline unsigned int bdev_max_active_zones(struct block_device *bdev)
1628
+{
1629
+	struct request_queue *q = bdev_get_queue(bdev);
1630
+
1631
+	if (q)
1632
+		return queue_max_active_zones(q);
1633
+	return 0;
1634
+}
1635
+
1636
+static inline int queue_dma_alignment(const struct request_queue *q)
16461637 {
16471638 	return q ? q->dma_alignment : 511;
16481639 }
....@@ -1667,113 +1658,10 @@
16671658 
16681659 static inline unsigned int block_size(struct block_device *bdev)
16691660 {
1670
-	return bdev->bd_block_size;
1671
-}
1672
-
1673
-static inline bool queue_flush_queueable(struct request_queue *q)
1674
-{
1675
-	return !test_bit(QUEUE_FLAG_FLUSH_NQ, &q->queue_flags);
1676
-}
1677
-
1678
-typedef struct {struct page *v;} Sector;
1679
-
1680
-unsigned char *read_dev_sector(struct block_device *, sector_t, Sector *);
1681
-
1682
-static inline void put_dev_sector(Sector p)
1683
-{
1684
-	put_page(p.v);
1685
-}
1686
-
1687
-static inline bool __bvec_gap_to_prev(struct request_queue *q,
1688
-				struct bio_vec *bprv, unsigned int offset)
1689
-{
1690
-	return offset ||
1691
-		((bprv->bv_offset + bprv->bv_len) & queue_virt_boundary(q));
1692
-}
1693
-
1694
-/*
1695
- * Check if adding a bio_vec after bprv with offset would create a gap in
1696
- * the SG list. Most drivers don't care about this, but some do.
1697
- */
1698
-static inline bool bvec_gap_to_prev(struct request_queue *q,
1699
-				struct bio_vec *bprv, unsigned int offset)
1700
-{
1701
-	if (!queue_virt_boundary(q))
1702
-		return false;
1703
-	return __bvec_gap_to_prev(q, bprv, offset);
1704
-}
1705
-
1706
-/*
1707
- * Check if the two bvecs from two bios can be merged to one segment.
1708
- * If yes, no need to check gap between the two bios since the 1st bio
1709
- * and the 1st bvec in the 2nd bio can be handled in one segment.
1710
- */
1711
-static inline bool bios_segs_mergeable(struct request_queue *q,
1712
-		struct bio *prev, struct bio_vec *prev_last_bv,
1713
-		struct bio_vec *next_first_bv)
1714
-{
1715
-	if (!BIOVEC_PHYS_MERGEABLE(prev_last_bv, next_first_bv))
1716
-		return false;
1717
-	if (!BIOVEC_SEG_BOUNDARY(q, prev_last_bv, next_first_bv))
1718
-		return false;
1719
-	if (prev->bi_seg_back_size + next_first_bv->bv_len >
1720
-			queue_max_segment_size(q))
1721
-		return false;
1722
-	return true;
1723
-}
1724
-
1725
-static inline bool bio_will_gap(struct request_queue *q,
1726
-				struct request *prev_rq,
1727
-				struct bio *prev,
1728
-				struct bio *next)
1729
-{
1730
-	if (bio_has_data(prev) && queue_virt_boundary(q)) {
1731
-		struct bio_vec pb, nb;
1732
-
1733
-		/*
1734
-		 * don't merge if the 1st bio starts with non-zero
1735
-		 * offset, otherwise it is quite difficult to respect
1736
-		 * sg gap limit. We work hard to merge a huge number of small
1737
-		 * single bios in case of mkfs.
1738
-		 */
1739
-		if (prev_rq)
1740
-			bio_get_first_bvec(prev_rq->bio, &pb);
1741
-		else
1742
-			bio_get_first_bvec(prev, &pb);
1743
-		if (pb.bv_offset)
1744
-			return true;
1745
-
1746
-		/*
1747
-		 * We don't need to worry about the situation that the
1748
-		 * merged segment ends in unaligned virt boundary:
1749
-		 *
1750
-		 * - if 'pb' ends aligned, the merged segment ends aligned
1751
-		 * - if 'pb' ends unaligned, the next bio must include
1752
-		 *   one single bvec of 'nb', otherwise the 'nb' can't
1753
-		 *   merge with 'pb'
1754
-		 */
1755
-		bio_get_last_bvec(prev, &pb);
1756
-		bio_get_first_bvec(next, &nb);
1757
-
1758
-		if (!bios_segs_mergeable(q, prev, &pb, &nb))
1759
-			return __bvec_gap_to_prev(q, &pb, nb.bv_offset);
1760
-	}
1761
-
1762
-	return false;
1763
-}
1764
-
1765
-static inline bool req_gap_back_merge(struct request *req, struct bio *bio)
1766
-{
1767
-	return bio_will_gap(req->q, req, req->biotail, bio);
1768
-}
1769
-
1770
-static inline bool req_gap_front_merge(struct request *req, struct bio *bio)
1771
-{
1772
-	return bio_will_gap(req->q, NULL, bio, req->bio);
1661
+	return 1 << bdev->bd_inode->i_blkbits;
17731662 }
17741663 
17751664 int kblockd_schedule_work(struct work_struct *work);
1776
-int kblockd_schedule_work_on(int cpu, struct work_struct *work);
17771665 int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned long delay);
17781666 
17791667 #define MODULE_ALIAS_BLOCKDEV(major,minor) \
....@@ -1800,10 +1688,14 @@
18001688 };
18011689 
18021690 typedef blk_status_t (integrity_processing_fn) (struct blk_integrity_iter *);
1691
+typedef void (integrity_prepare_fn) (struct request *);
1692
+typedef void (integrity_complete_fn) (struct request *, unsigned int);
18031693 
18041694 struct blk_integrity_profile {
18051695 	integrity_processing_fn		*generate_fn;
18061696 	integrity_processing_fn		*verify_fn;
1697
+	integrity_prepare_fn		*prepare_fn;
1698
+	integrity_complete_fn		*complete_fn;
18071699 	const char			*name;
18081700 };
18091701 
....@@ -1813,10 +1705,6 @@
18131705 extern int blk_rq_map_integrity_sg(struct request_queue *, struct bio *,
18141706 				   struct scatterlist *);
18151707 extern int blk_rq_count_integrity_sg(struct request_queue *, struct bio *);
1816
-extern bool blk_integrity_merge_rq(struct request_queue *, struct request *,
1817
-				   struct request *);
1818
-extern bool blk_integrity_merge_bio(struct request_queue *, struct request *,
1819
-				    struct bio *);
18201708 
18211709 static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
18221710 {
....@@ -1834,6 +1722,12 @@
18341722 	return blk_get_integrity(bdev->bd_disk);
18351723 }
18361724 
1725
+static inline bool
1726
+blk_integrity_queue_supports_integrity(struct request_queue *q)
1727
+{
1728
+	return q->integrity.profile;
1729
+}
1730
+
18371731 static inline bool blk_integrity_rq(struct request *rq)
18381732 {
18391733 	return rq->cmd_flags & REQ_INTEGRITY;
....@@ -1846,29 +1740,9 @@
18461740 }
18471741 
18481742 static inline unsigned short
1849
-queue_max_integrity_segments(struct request_queue *q)
1743
+queue_max_integrity_segments(const struct request_queue *q)
18501744 {
18511745 	return q->limits.max_integrity_segments;
1852
-}
1853
-
1854
-static inline bool integrity_req_gap_back_merge(struct request *req,
1855
-						struct bio *next)
1856
-{
1857
-	struct bio_integrity_payload *bip = bio_integrity(req->bio);
1858
-	struct bio_integrity_payload *bip_next = bio_integrity(next);
1859
-
1860
-	return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1],
1861
-				bip_next->bip_vec[0].bv_offset);
1862
-}
1863
-
1864
-static inline bool integrity_req_gap_front_merge(struct request *req,
1865
-						 struct bio *bio)
1866
-{
1867
-	struct bio_integrity_payload *bip = bio_integrity(bio);
1868
-	struct bio_integrity_payload *bip_next = bio_integrity(req->bio);
1869
-
1870
-	return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1],
1871
-				bip_next->bip_vec[0].bv_offset);
18721746 }
18731747 
18741748 /**
....@@ -1891,6 +1765,17 @@
18911765 					       unsigned int sectors)
18921766 {
18931767 	return bio_integrity_intervals(bi, sectors) * bi->tuple_size;
1768
+}
1769
+
1770
+/*
1771
+ * Return the first bvec that contains integrity data.  Only drivers that are
1772
+ * limited to a single integrity segment should use this helper.
1773
+ */
1774
+static inline struct bio_vec *rq_integrity_vec(struct request *rq)
1775
+{
1776
+	if (WARN_ON_ONCE(queue_max_integrity_segments(rq->q) > 1))
1777
+		return NULL;
1778
+	return rq->bio->bi_integrity->bip_vec;
18941779 }
18951780 
18961781 #else /* CONFIG_BLK_DEV_INTEGRITY */
....@@ -1923,6 +1808,11 @@
19231808 {
19241809 	return NULL;
19251810 }
1811
+static inline bool
1812
+blk_integrity_queue_supports_integrity(struct request_queue *q)
1813
+{
1814
+	return false;
1815
+}
19261816 static inline int blk_integrity_compare(struct gendisk *a, struct gendisk *b)
19271817 {
19281818 	return 0;
....@@ -1938,32 +1828,9 @@
19381828 						    unsigned int segs)
19391829 {
19401830 }
1941
-static inline unsigned short queue_max_integrity_segments(struct request_queue *q)
1831
+static inline unsigned short queue_max_integrity_segments(const struct request_queue *q)
19421832 {
19431833 	return 0;
1944
-}
1945
-static inline bool blk_integrity_merge_rq(struct request_queue *rq,
1946
-					  struct request *r1,
1947
-					  struct request *r2)
1948
-{
1949
-	return true;
1950
-}
1951
-static inline bool blk_integrity_merge_bio(struct request_queue *rq,
1952
-					   struct request *r,
1953
-					   struct bio *b)
1954
-{
1955
-	return true;
1956
-}
1957
-
1958
-static inline bool integrity_req_gap_back_merge(struct request *req,
1959
-						struct bio *next)
1960
-{
1961
-	return false;
1962
-}
1963
-static inline bool integrity_req_gap_front_merge(struct request *req,
1964
-						 struct bio *bio)
1965
-{
1966
-	return false;
19671834 }
19681835 
19691836 static inline unsigned int bio_integrity_intervals(struct blk_integrity *bi,
....@@ -1978,9 +1845,34 @@
19781845 	return 0;
19791846 }
19801847 
1848
+static inline struct bio_vec *rq_integrity_vec(struct request *rq)
1849
+{
1850
+	return NULL;
1851
+}
1852
+
19811853 #endif /* CONFIG_BLK_DEV_INTEGRITY */
19821854 
1855
+#ifdef CONFIG_BLK_INLINE_ENCRYPTION
1856
+
1857
+bool blk_ksm_register(struct blk_keyslot_manager *ksm, struct request_queue *q);
1858
+
1859
+void blk_ksm_unregister(struct request_queue *q);
1860
+
1861
+#else /* CONFIG_BLK_INLINE_ENCRYPTION */
1862
+
1863
+static inline bool blk_ksm_register(struct blk_keyslot_manager *ksm,
1864
+				    struct request_queue *q)
1865
+{
1866
+	return true;
1867
+}
1868
+
1869
+static inline void blk_ksm_unregister(struct request_queue *q) { }
1870
+
1871
+#endif /* CONFIG_BLK_INLINE_ENCRYPTION */
1872
+
1873
+
19831874 struct block_device_operations {
1875
+	blk_qc_t (*submit_bio) (struct bio *bio);
19841876 	int (*open) (struct block_device *, fmode_t);
19851877 	void (*release) (struct gendisk *, fmode_t);
19861878 	int (*rw_page)(struct block_device *, sector_t, struct page *, unsigned int);
....@@ -1988,16 +1880,28 @@
19881880 	int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
19891881 	unsigned int (*check_events) (struct gendisk *disk,
19901882 				      unsigned int clearing);
1991
-	/* ->media_changed() is DEPRECATED, use ->check_events() instead */
1992
-	int (*media_changed) (struct gendisk *);
19931883 	void (*unlock_native_capacity) (struct gendisk *);
19941884 	int (*revalidate_disk) (struct gendisk *);
19951885 	int (*getgeo)(struct block_device *, struct hd_geometry *);
19961886 	/* this callback is with swap_lock and sometimes page table lock held */
19971887 	void (*swap_slot_free_notify) (struct block_device *, unsigned long);
1888
+	int (*report_zones)(struct gendisk *, sector_t sector,
1889
+			unsigned int nr_zones, report_zones_cb cb, void *data);
1890
+	char *(*devnode)(struct gendisk *disk, umode_t *mode);
19981891 	struct module *owner;
19991892 	const struct pr_ops *pr_ops;
1893
+
1894
+	ANDROID_KABI_RESERVE(1);
1895
+	ANDROID_KABI_RESERVE(2);
1896
+	ANDROID_OEM_DATA(1);
20001897 };
1898
+
1899
+#ifdef CONFIG_COMPAT
1900
+extern int blkdev_compat_ptr_ioctl(struct block_device *, fmode_t,
1901
+				      unsigned int, unsigned long);
1902
+#else
1903
+#define blkdev_compat_ptr_ioctl NULL
1904
+#endif
20011905 
20021906 extern int __blkdev_driver_ioctl(struct block_device *, fmode_t, unsigned int,
20031907 				 unsigned long);
....@@ -2007,6 +1911,7 @@
20071911 
20081912 #ifdef CONFIG_BLK_DEV_ZONED
20091913 bool blk_req_needs_zone_write_lock(struct request *rq);
1914
+bool blk_req_zone_write_trylock(struct request *rq);
20101915 void __blk_req_zone_write_lock(struct request *rq);
20111916 void __blk_req_zone_write_unlock(struct request *rq);
20121917 
....@@ -2058,51 +1963,102 @@
20581963 }
20591964 #endif /* CONFIG_BLK_DEV_ZONED */
20601965 
2061
-#else /* CONFIG_BLOCK */
1966
+static inline void blk_wake_io_task(struct task_struct *waiter)
1967
+{
1968
+	/*
1969
+	 * If we're polling, the task itself is doing the completions. For
1970
+	 * that case, we don't need to signal a wakeup, it's enough to just
1971
+	 * mark us as RUNNING.
1972
+	 */
1973
+	if (waiter == current)
1974
+		__set_current_state(TASK_RUNNING);
1975
+	else
1976
+		wake_up_process(waiter);
1977
+}
20621978 
2063
-struct block_device;
1979
+unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors,
1980
+		unsigned int op);
1981
+void disk_end_io_acct(struct gendisk *disk, unsigned int op,
1982
+		unsigned long start_time);
20641983 
2065
-/*
2066
- * stubs for when the block layer is configured out
1984
+unsigned long part_start_io_acct(struct gendisk *disk, struct hd_struct **part,
1985
+				 struct bio *bio);
1986
+void part_end_io_acct(struct hd_struct *part, struct bio *bio,
1987
+		      unsigned long start_time);
1988
+
1989
+/**
1990
+ * bio_start_io_acct - start I/O accounting for bio based drivers
1991
+ * @bio:	bio to start account for
1992
+ *
1993
+ * Returns the start time that should be passed back to bio_end_io_acct().
20671994  */
2068
-#define buffer_heads_over_limit 0
2069
-
2070
-static inline long nr_blockdev_pages(void)
1995
+static inline unsigned long bio_start_io_acct(struct bio *bio)
20711996 {
2072
-	return 0;
1997
+	return disk_start_io_acct(bio->bi_disk, bio_sectors(bio), bio_op(bio));
20731998 }
20741999 
2075
-struct blk_plug {
2076
-};
2077
-
2078
-static inline void blk_start_plug(struct blk_plug *plug)
2000
+/**
2001
+ * bio_end_io_acct - end I/O accounting for bio based drivers
2002
+ * @bio:	bio to end account for
2003
+ * @start:	start time returned by bio_start_io_acct()
2004
+ */
2005
+static inline void bio_end_io_acct(struct bio *bio, unsigned long start_time)
20792006 {
2007
+	return disk_end_io_acct(bio->bi_disk, bio_op(bio), start_time);
20802008 }
20812009 
2082
-static inline void blk_finish_plug(struct blk_plug *plug)
2083
-{
2084
-}
2010
+int bdev_read_only(struct block_device *bdev);
2011
+int set_blocksize(struct block_device *bdev, int size);
20852012 
2086
-static inline void blk_flush_plug(struct task_struct *task)
2087
-{
2088
-}
2013
+const char *bdevname(struct block_device *bdev, char *buffer);
2014
+struct block_device *lookup_bdev(const char *);
20892015 
2090
-static inline void blk_schedule_flush_plug(struct task_struct *task)
2091
-{
2092
-}
2016
+void blkdev_show(struct seq_file *seqf, off_t offset);
20932017 
2094
-
2095
-static inline bool blk_needs_flush_plug(struct task_struct *tsk)
2096
-{
2097
-	return false;
2098
-}
2099
-
2100
-static inline int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask,
2101
-				     sector_t *error_sector)
2102
-{
2103
-	return 0;
2104
-}
2105
-
2106
-#endif /* CONFIG_BLOCK */
2107
-
2018
+#define BDEVNAME_SIZE	32	/* Largest string for a blockdev identifier */
2019
+#define BDEVT_SIZE	10	/* Largest string for MAJ:MIN for blkdev */
2020
+#ifdef CONFIG_BLOCK
2021
+#define BLKDEV_MAJOR_MAX	512
2022
+#else
2023
+#define BLKDEV_MAJOR_MAX	0
21082024 #endif
2025
+
2026
+struct block_device *blkdev_get_by_path(const char *path, fmode_t mode,
2027
+		void *holder);
2028
+struct block_device *blkdev_get_by_dev(dev_t dev, fmode_t mode, void *holder);
2029
+int bd_prepare_to_claim(struct block_device *bdev, struct block_device *whole,
2030
+		void *holder);
2031
+void bd_abort_claiming(struct block_device *bdev, struct block_device *whole,
2032
+		void *holder);
2033
+void blkdev_put(struct block_device *bdev, fmode_t mode);
2034
+
2035
+struct block_device *I_BDEV(struct inode *inode);
2036
+struct block_device *bdget_part(struct hd_struct *part);
2037
+struct block_device *bdgrab(struct block_device *bdev);
2038
+void bdput(struct block_device *);
2039
+
2040
+#ifdef CONFIG_BLOCK
2041
+void invalidate_bdev(struct block_device *bdev);
2042
+int truncate_bdev_range(struct block_device *bdev, fmode_t mode, loff_t lstart,
2043
+			loff_t lend);
2044
+int sync_blockdev(struct block_device *bdev);
2045
+#else
2046
+static inline void invalidate_bdev(struct block_device *bdev)
2047
+{
2048
+}
2049
+static inline int truncate_bdev_range(struct block_device *bdev, fmode_t mode,
2050
+				      loff_t lstart, loff_t lend)
2051
+{
2052
+	return 0;
2053
+}
2054
+static inline int sync_blockdev(struct block_device *bdev)
2055
+{
2056
+	return 0;
2057
+}
2058
+#endif
2059
+int fsync_bdev(struct block_device *bdev);
2060
+
2061
+int freeze_bdev(struct block_device *bdev);
2062
+int thaw_bdev(struct block_device *bdev);
2063
+
2064
+#endif /* _LINUX_BLKDEV_H */