~hc/RK356X_SDK_RELEASE.git

..	..	@@ -4,16 +4,13 @@
4	4
5	5	#include <linux/sched.h>
6	6	#include <linux/sched/clock.h>
7		-
8		-#ifdef CONFIG_BLOCK
9		-
10	7	#include <linux/major.h>
11	8	#include <linux/genhd.h>
12	9	#include <linux/list.h>
13	10	#include <linux/llist.h>
	11	+#include <linux/minmax.h>
14	12	#include <linux/timer.h>
15	13	#include <linux/workqueue.h>
16		-#include <linux/kthread.h>
17	14	#include <linux/pagemap.h>
18	15	#include <linux/backing-dev-defs.h>
19	16	#include <linux/wait.h>
..	..	@@ -28,6 +25,9 @@
28	25	#include <linux/percpu-refcount.h>
29	26	#include <linux/scatterlist.h>
30	27	#include <linux/blkzoned.h>
	28	+#include <linux/pm.h>
	29	+#include <linux/android_kabi.h>
	30	+#include <linux/android_vendor.h>
31	31
32	32	struct module;
33	33	struct scsi_ioctl_command;
..	..	@@ -44,7 +44,7 @@
44	44	struct rq_qos;
45	45	struct blk_queue_stats;
46	46	struct blk_stat_callback;
47		-struct keyslot_manager;
	47	+struct blk_keyslot_manager;
48	48
49	49	#define BLKDEV_MIN_RQ 4
50	50	#define BLKDEV_MAX_RQ 128 /* Default maximum */
..	..	@@ -52,32 +52,24 @@
52	52	/* Must be consistent with blk_mq_poll_stats_bkt() */
53	53	#define BLK_MQ_POLL_STATS_BKTS 16
54	54
	55	+/* Doing classic polling */
	56	+#define BLK_MQ_POLL_CLASSIC -1
	57	+
55	58	/*
56	59	* Maximum number of blkcg policies allowed to be registered concurrently.
57	60	* Defined here to simplify include dependency.
58	61	*/
59	62	#define BLKCG_MAX_POLS 5
60	63
	64	+static inline int blk_validate_block_size(unsigned int bsize)
	65	+{
	66	+ if (bsize < 512 \|\| bsize > PAGE_SIZE \|\| !is_power_of_2(bsize))
	67	+ return -EINVAL;
	68	+
	69	+ return 0;
	70	+}
	71	+
61	72	typedef void (rq_end_io_fn)(struct request *, blk_status_t);
62		-
63		-#define BLK_RL_SYNCFULL (1U << 0)
64		-#define BLK_RL_ASYNCFULL (1U << 1)
65		-
66		-struct request_list {
67		- struct request_queue q; / the queue this rl belongs to */
68		-#ifdef CONFIG_BLK_CGROUP
69		- struct blkcg_gq blkg; / blkg this request pool belongs to */
70		-#endif
71		- /*
72		- * count[], starved[], and wait[] are indexed by
73		- * BLK_RW_SYNC/BLK_RW_ASYNC
74		- */
75		- int count[2];
76		- int starved[2];
77		- mempool_t *rq_pool;
78		- wait_queue_head_t wait[2];
79		- unsigned int flags;
80		-};
81	73
82	74	/*
83	75	* request flags */
..	..	@@ -87,8 +79,6 @@
87	79	#define RQF_SORTED ((__force req_flags_t)(1 << 0))
88	80	/* drive already may have started this one */
89	81	#define RQF_STARTED ((__force req_flags_t)(1 << 1))
90		-/* uses tagged queueing */
91		-#define RQF_QUEUED ((__force req_flags_t)(1 << 2))
92	82	/* may not be passed by ioscheduler */
93	83	#define RQF_SOFTBARRIER ((__force req_flags_t)(1 << 3))
94	84	/* request for flush sequence */
..	..	@@ -99,18 +89,13 @@
99	89	#define RQF_MQ_INFLIGHT ((__force req_flags_t)(1 << 6))
100	90	/* don't call prep for this one */
101	91	#define RQF_DONTPREP ((__force req_flags_t)(1 << 7))
102		-/* set for "ide_preempt" requests and also for requests for which the SCSI
103		- "quiesce" state must be ignored. */
104		-#define RQF_PREEMPT ((__force req_flags_t)(1 << 8))
105		-/* contains copies of user pages */
106		-#define RQF_COPY_USER ((__force req_flags_t)(1 << 9))
107	92	/* vaguely specified driver internal error. Ignored by the block layer */
108	93	#define RQF_FAILED ((__force req_flags_t)(1 << 10))
109	94	/* don't warn about errors */
110	95	#define RQF_QUIET ((__force req_flags_t)(1 << 11))
111	96	/* elevator private data attached */
112	97	#define RQF_ELVPRIV ((__force req_flags_t)(1 << 12))
113		-/* account I/O stat */
	98	+/* account into disk and partition IO statistics */
114	99	#define RQF_IO_STAT ((__force req_flags_t)(1 << 13))
115	100	/* request came from our alloc pool */
116	101	#define RQF_ALLOCED ((__force req_flags_t)(1 << 14))
..	..	@@ -118,7 +103,7 @@
118	103	#define RQF_PM ((__force req_flags_t)(1 << 15))
119	104	/* on IO scheduler merge hash */
120	105	#define RQF_HASHED ((__force req_flags_t)(1 << 16))
121		-/* IO stats tracking on */
	106	+/* track IO completion time */
122	107	#define RQF_STATS ((__force req_flags_t)(1 << 17))
123	108	/* Look at ->special_vec for the actual data payload instead of the
124	109	bio chain. */
..	..	@@ -151,20 +136,17 @@
151	136	*/
152	137	struct request {
153	138	struct request_queue *q;
154		-#ifdef CONFIG_PREEMPT_RT_FULL
155		- struct work_struct work;
156		-#endif
157	139	struct blk_mq_ctx *mq_ctx;
	140	+ struct blk_mq_hw_ctx *mq_hctx;
158	141
159		- int cpu;
160	142	unsigned int cmd_flags; /* op and common flags */
161	143	req_flags_t rq_flags;
162	144
	145	+ int tag;
163	146	int internal_tag;
164	147
165	148	/* the following two fields are internal, NEVER access directly */
166	149	unsigned int __data_len; /* total data len */
167		- int tag;
168	150	sector_t __sector; /* sector cursor */
169	151
170	152	struct bio *bio;
..	..	@@ -217,7 +199,11 @@
217	199
218	200	struct gendisk *rq_disk;
219	201	struct hd_struct *part;
220		- /* Time that I/O was submitted to the kernel. */
	202	+#ifdef CONFIG_BLK_RQ_ALLOC_TIME
	203	+ /* Time that the first bio started allocating this request. */
	204	+ u64 alloc_time_ns;
	205	+#endif
	206	+ /* Time that this request was allocated for this IO. */
221	207	u64 start_time_ns;
222	208	/* Time that I/O was submitted to the device. */
223	209	u64 io_start_time_ns;
..	..	@@ -225,9 +211,12 @@
225	211	#ifdef CONFIG_BLK_WBT
226	212	unsigned short wbt_flags;
227	213	#endif
228		-#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
229		- unsigned short throtl_size;
230		-#endif
	214	+ /*
	215	+ * rq sectors used for blk stats. It has the same value
	216	+ * with blk_rq_sectors(rq), except that it never be zeroed
	217	+ * by completion.
	218	+ */
	219	+ unsigned short stats_sectors;
231	220
232	221	/*
233	222	* Number of scatter-gather DMA addr+len pairs after
..	..	@@ -239,22 +228,19 @@
239	228	unsigned short nr_integrity_segments;
240	229	#endif
241	230
	231	+#ifdef CONFIG_BLK_INLINE_ENCRYPTION
	232	+ struct bio_crypt_ctx *crypt_ctx;
	233	+ struct blk_ksm_keyslot *crypt_keyslot;
	234	+#endif
	235	+
242	236	unsigned short write_hint;
243	237	unsigned short ioprio;
244		-
245		- void special; / opaque pointer available for LLD use */
246		-
247		- unsigned int extra_len; /* length of alignment and padding */
248	238
249	239	enum mq_rq_state state;
250	240	refcount_t ref;
251	241
252	242	unsigned int timeout;
253		-
254		- /* access through blk_rq_set_deadline, blk_rq_deadline */
255		- unsigned long __deadline;
256		-
257		- struct list_head timeout_list;
	243	+ unsigned long deadline;
258	244
259	245	union {
260	246	struct __call_single_data csd;
..	..	@@ -267,12 +253,7 @@
267	253	rq_end_io_fn *end_io;
268	254	void *end_io_data;
269	255
270		- /* for bidi */
271		- struct request *next_rq;
272		-
273		-#ifdef CONFIG_BLK_CGROUP
274		- struct request_list rl; / rl this rq is alloced from */
275		-#endif
	256	+ ANDROID_KABI_RESERVE(1);
276	257	};
277	258
278	259	static inline bool blk_op_is_scsi(unsigned int op)
..	..	@@ -316,41 +297,18 @@
316	297
317	298	struct blk_queue_ctx;
318	299
319		-typedef void (request_fn_proc) (struct request_queue *q);
320		-typedef blk_qc_t (make_request_fn) (struct request_queue q, struct bio bio);
321		-typedef bool (poll_q_fn) (struct request_queue *q, blk_qc_t);
322		-typedef int (prep_rq_fn) (struct request_queue , struct request );
323		-typedef void (unprep_rq_fn) (struct request_queue , struct request );
324		-
325	300	struct bio_vec;
326		-typedef void (softirq_done_fn)(struct request *);
327		-typedef int (dma_drain_needed_fn)(struct request *);
328		-typedef int (lld_busy_fn) (struct request_queue *q);
329		-typedef int (bsg_job_fn) (struct bsg_job *);
330		-typedef int (init_rq_fn)(struct request_queue , struct request , gfp_t);
331		-typedef void (exit_rq_fn)(struct request_queue , struct request );
332	301
333	302	enum blk_eh_timer_return {
334	303	BLK_EH_DONE, /* drivers has completed the command */
335	304	BLK_EH_RESET_TIMER, /* reset timer and try again */
336	305	};
337	306
338		-typedef enum blk_eh_timer_return (rq_timed_out_fn)(struct request *);
339		-
340	307	enum blk_queue_state {
341	308	Queue_down,
342	309	Queue_up,
343	310	};
344	311
345		-struct blk_queue_tag {
346		- struct request *tag_index; / map of busy tags */
347		- unsigned long tag_map; / bit map of free/busy tags */
348		- int max_depth; /* what we will send to device */
349		- int real_max_depth; /* what the array can hold */
350		- atomic_t refcnt; /* map can be shared */
351		- int alloc_policy; /* tag allocation policy */
352		- int next_tag; /* next tag */
353		-};
354	312	#define BLK_TAG_ALLOC_FIFO 0 /* allocate starting from 0 */
355	313	#define BLK_TAG_ALLOC_RR 1 /* allocate starting from last allocated tag */
356	314
..	..	@@ -359,11 +317,14 @@
359	317
360	318	/*
361	319	* Zoned block device models (zoned limit).
	320	+ *
	321	+ * Note: This needs to be ordered from the least to the most severe
	322	+ * restrictions for the inheritance in blk_stack_limits() to work.
362	323	*/
363	324	enum blk_zoned_model {
364		- BLK_ZONED_NONE, /* Regular block device */
365		- BLK_ZONED_HA, /* Host-aware zoned block device */
366		- BLK_ZONED_HM, /* Host-managed zoned block device */
	325	+ BLK_ZONED_NONE = 0, /* Regular block device */
	326	+ BLK_ZONED_HA, /* Host-aware zoned block device */
	327	+ BLK_ZONED_HM, /* Host-managed zoned block device */
367	328	};
368	329
369	330	struct queue_limits {
..	..	@@ -385,6 +346,7 @@
385	346	unsigned int max_hw_discard_sectors;
386	347	unsigned int max_write_same_sectors;
387	348	unsigned int max_write_zeroes_sectors;
	349	+ unsigned int max_zone_append_sectors;
388	350	unsigned int discard_granularity;
389	351	unsigned int discard_alignment;
390	352
..	..	@@ -394,30 +356,40 @@
394	356
395	357	unsigned char misaligned;
396	358	unsigned char discard_misaligned;
397		- unsigned char cluster;
398	359	unsigned char raid_partial_stripes_expensive;
399	360	enum blk_zoned_model zoned;
	361	+
	362	+ ANDROID_KABI_RESERVE(1);
400	363	};
	364	+
	365	+typedef int (report_zones_cb)(struct blk_zone zone, unsigned int idx,
	366	+ void *data);
	367	+
	368	+void blk_queue_set_zoned(struct gendisk *disk, enum blk_zoned_model model);
401	369
402	370	#ifdef CONFIG_BLK_DEV_ZONED
403	371
404		-struct blk_zone_report_hdr {
405		- unsigned int nr_zones;
406		- u8 padding[60];
407		-};
408		-
409		-extern int blkdev_report_zones(struct block_device *bdev,
410		- sector_t sector, struct blk_zone *zones,
411		- unsigned int *nr_zones, gfp_t gfp_mask);
412		-extern int blkdev_reset_zones(struct block_device *bdev, sector_t sectors,
413		- sector_t nr_sectors, gfp_t gfp_mask);
	372	+#define BLK_ALL_ZONES ((unsigned int)-1)
	373	+int blkdev_report_zones(struct block_device *bdev, sector_t sector,
	374	+ unsigned int nr_zones, report_zones_cb cb, void *data);
	375	+unsigned int blkdev_nr_zones(struct gendisk *disk);
	376	+extern int blkdev_zone_mgmt(struct block_device *bdev, enum req_opf op,
	377	+ sector_t sectors, sector_t nr_sectors,
	378	+ gfp_t gfp_mask);
	379	+int blk_revalidate_disk_zones(struct gendisk *disk,
	380	+ void (update_driver_data)(struct gendisk disk));
414	381
415	382	extern int blkdev_report_zones_ioctl(struct block_device *bdev, fmode_t mode,
416	383	unsigned int cmd, unsigned long arg);
417		-extern int blkdev_reset_zones_ioctl(struct block_device *bdev, fmode_t mode,
418		- unsigned int cmd, unsigned long arg);
	384	+extern int blkdev_zone_mgmt_ioctl(struct block_device *bdev, fmode_t mode,
	385	+ unsigned int cmd, unsigned long arg);
419	386
420	387	#else /* CONFIG_BLK_DEV_ZONED */
	388	+
	389	+static inline unsigned int blkdev_nr_zones(struct gendisk *disk)
	390	+{
	391	+ return 0;
	392	+}
421	393
422	394	static inline int blkdev_report_zones_ioctl(struct block_device *bdev,
423	395	fmode_t mode, unsigned int cmd,
..	..	@@ -426,9 +398,9 @@
426	398	return -ENOTTY;
427	399	}
428	400
429		-static inline int blkdev_reset_zones_ioctl(struct block_device *bdev,
430		- fmode_t mode, unsigned int cmd,
431		- unsigned long arg)
	401	+static inline int blkdev_zone_mgmt_ioctl(struct block_device *bdev,
	402	+ fmode_t mode, unsigned int cmd,
	403	+ unsigned long arg)
432	404	{
433	405	return -ENOTTY;
434	406	}
..	..	@@ -436,66 +408,24 @@
436	408	#endif /* CONFIG_BLK_DEV_ZONED */
437	409
438	410	struct request_queue {
439		- /*
440		- * Together with queue_head for cacheline sharing
441		- */
442		- struct list_head queue_head;
443	411	struct request *last_merge;
444	412	struct elevator_queue *elevator;
445		- int nr_rqs[2]; /* # allocated [a]sync rqs */
446		- int nr_rqs_elvpriv; /* # allocated rqs w/ elvpriv */
	413	+
	414	+ struct percpu_ref q_usage_counter;
447	415
448	416	struct blk_queue_stats *stats;
449	417	struct rq_qos *rq_qos;
450	418
451		- /*
452		- * If blkcg is not used, @q->root_rl serves all requests. If blkcg
453		- * is used, root blkg allocates from @q->root_rl and all other
454		- * blkgs from their own blkg->rl. Which one to use should be
455		- * determined using bio_request_list().
456		- */
457		- struct request_list root_rl;
458		-
459		- request_fn_proc *request_fn;
460		- make_request_fn *make_request_fn;
461		- poll_q_fn *poll_fn;
462		- prep_rq_fn *prep_rq_fn;
463		- unprep_rq_fn *unprep_rq_fn;
464		- softirq_done_fn *softirq_done_fn;
465		- rq_timed_out_fn *rq_timed_out_fn;
466		- dma_drain_needed_fn *dma_drain_needed;
467		- lld_busy_fn *lld_busy_fn;
468		- /* Called just after a request is allocated */
469		- init_rq_fn *init_rq_fn;
470		- /* Called just before a request is freed */
471		- exit_rq_fn *exit_rq_fn;
472		- /* Called from inside blk_get_request() */
473		- void (initialize_rq_fn)(struct request rq);
474		-
475	419	const struct blk_mq_ops *mq_ops;
476		-
477		- unsigned int *mq_map;
478	420
479	421	/* sw queues */
480	422	struct blk_mq_ctx __percpu *queue_ctx;
481		- unsigned int nr_queues;
482	423
483	424	unsigned int queue_depth;
484	425
485	426	/* hw dispatch queues */
486	427	struct blk_mq_hw_ctx **queue_hw_ctx;
487	428	unsigned int nr_hw_queues;
488		-
489		- /*
490		- * Dispatch queue sorting
491		- */
492		- sector_t end_sector;
493		- struct request *boundary_rq;
494		-
495		- /*
496		- * Delayed queue handling
497		- */
498		- struct delayed_work delay_work;
499	429
500	430	struct backing_dev_info *backing_dev_info;
501	431
..	..	@@ -511,8 +441,7 @@
511	441	unsigned long queue_flags;
512	442	/*
513	443	* Number of contexts that have called blk_set_pm_only(). If this
514		- * counter is above zero then only RQF_PM and RQF_PREEMPT requests are
515		- * processed.
	444	+ * counter is above zero then only RQF_PM requests are processed.
516	445	*/
517	446	atomic_t pm_only;
518	447
..	..	@@ -527,13 +456,7 @@
527	456	*/
528	457	gfp_t bounce_gfp;
529	458
530		- /*
531		- * protects queue structures from reentrancy. ->__queue_lock should
532		- * _never_ be used directly, it is queue private. always use
533		- * ->queue_lock.
534		- */
535		- spinlock_t __queue_lock;
536		- spinlock_t *queue_lock;
	459	+ spinlock_t queue_lock;
537	460
538	461	/*
539	462	* queue kobject
..	..	@@ -543,7 +466,7 @@
543	466	/*
544	467	* mq queue kobject
545	468	*/
546		- struct kobject mq_kobj;
	469	+ struct kobject *mq_kobj;
547	470
548	471	#ifdef CONFIG_BLK_DEV_INTEGRITY
549	472	struct blk_integrity integrity;
..	..	@@ -551,7 +474,7 @@
551	474
552	475	#ifdef CONFIG_PM
553	476	struct device *dev;
554		- int rpm_status;
	477	+ enum rpm_status rpm_status;
555	478	unsigned int nr_pending;
556	479	#endif
557	480
..	..	@@ -559,29 +482,13 @@
559	482	* queue settings
560	483	*/
561	484	unsigned long nr_requests; /* Max # of requests */
562		- unsigned int nr_congestion_on;
563		- unsigned int nr_congestion_off;
564		- unsigned int nr_batching;
565	485
566		- unsigned int dma_drain_size;
567		- void *dma_drain_buffer;
568	486	unsigned int dma_pad_mask;
569	487	unsigned int dma_alignment;
570	488
571		- struct blk_queue_tag *queue_tags;
572		-
573		- unsigned int nr_sorted;
574		- unsigned int in_flight[2];
575		-
576		- /*
577		- * Number of active block driver functions for which blk_drain_queue()
578		- * must wait. Must be incremented around functions that unlock the
579		- * queue_lock internally, e.g. scsi_request_fn().
580		- */
581		- unsigned int request_fn_active;
582	489	#ifdef CONFIG_BLK_INLINE_ENCRYPTION
583	490	/* Inline crypto capabilities */
584		- struct keyslot_manager *ksm;
	491	+ struct blk_keyslot_manager *ksm;
585	492	#endif
586	493
587	494	unsigned int rq_timeout;
..	..	@@ -592,7 +499,8 @@
592	499
593	500	struct timer_list timeout;
594	501	struct work_struct timeout_work;
595		- struct list_head timeout_list;
	502	+
	503	+ atomic_t nr_active_requests_shared_sbitmap;
596	504
597	505	struct list_head icq_list;
598	506	#ifdef CONFIG_BLK_CGROUP
..	..	@@ -603,13 +511,15 @@
603	511
604	512	struct queue_limits limits;
605	513
	514	+ unsigned int required_elevator_features;
	515	+
606	516	#ifdef CONFIG_BLK_DEV_ZONED
607	517	/*
608	518	* Zoned block device information for request dispatch control.
609	519	* nr_zones is the total number of zones of the device. This is always
610		- * 0 for regular block devices. seq_zones_bitmap is a bitmap of nr_zones
611		- * bits which indicates if a zone is conventional (bit clear) or
612		- * sequential (bit set). seq_zones_wlock is a bitmap of nr_zones
	520	+ * 0 for regular block devices. conv_zones_bitmap is a bitmap of nr_zones
	521	+ * bits which indicates if a zone is conventional (bit set) or
	522	+ * sequential (bit clear). seq_zones_wlock is a bitmap of nr_zones
613	523	* bits which indicates if a zone is write locked, that is, if a write
614	524	* request targeting the zone was dispatched. All three fields are
615	525	* initialized by the low level device driver (e.g. scsi/sd.c).
..	..	@@ -622,8 +532,10 @@
622	532	* blk_mq_unfreeze_queue().
623	533	*/
624	534	unsigned int nr_zones;
625		- unsigned long *seq_zones_bitmap;
	535	+ unsigned long *conv_zones_bitmap;
626	536	unsigned long *seq_zones_wlock;
	537	+ unsigned int max_open_zones;
	538	+ unsigned int max_active_zones;
627	539	#endif /* CONFIG_BLK_DEV_ZONED */
628	540
629	541	/*
..	..	@@ -632,9 +544,9 @@
632	544	unsigned int sg_timeout;
633	545	unsigned int sg_reserved_size;
634	546	int node;
	547	+ struct mutex debugfs_mutex;
635	548	#ifdef CONFIG_BLK_DEV_IO_TRACE
636	549	struct blk_trace __rcu *blk_trace;
637		- struct mutex blk_trace_mutex;
638	550	#endif
639	551	/*
640	552	* for flush operations
..	..	@@ -646,12 +558,20 @@
646	558	struct delayed_work requeue_work;
647	559
648	560	struct mutex sysfs_lock;
	561	+ struct mutex sysfs_dir_lock;
649	562
650		- int bypass_depth;
651		- atomic_t mq_freeze_depth;
	563	+ /*
	564	+ * for reusing dead hctx instance in case of updating
	565	+ * nr_hw_queues
	566	+ */
	567	+ struct list_head unused_hctx_list;
	568	+ spinlock_t unused_hctx_lock;
652	569
653		- bsg_job_fn *bsg_job_fn;
	570	+ int mq_freeze_depth;
	571	+
	572	+#if defined(CONFIG_BLK_DEV_BSG)
654	573	struct bsg_class_device bsg_dev;
	574	+#endif
655	575
656	576	#ifdef CONFIG_BLK_DEV_THROTTLING
657	577	/* Throttle data */
..	..	@@ -659,91 +579,105 @@
659	579	#endif
660	580	struct rcu_head rcu_head;
661	581	wait_queue_head_t mq_freeze_wq;
662		- struct work_struct mq_pcpu_wake;
663		- struct percpu_ref q_usage_counter;
664		- struct list_head all_q_node;
	582	+ /*
	583	+ * Protect concurrent access to q_usage_counter by
	584	+ * percpu_ref_kill() and percpu_ref_reinit().
	585	+ */
	586	+ struct mutex mq_freeze_lock;
665	587
666	588	struct blk_mq_tag_set *tag_set;
667	589	struct list_head tag_set_list;
668	590	struct bio_set bio_split;
669	591
670		-#ifdef CONFIG_BLK_DEBUG_FS
671	592	struct dentry *debugfs_dir;
	593	+
	594	+#ifdef CONFIG_BLK_DEBUG_FS
672	595	struct dentry *sched_debugfs_dir;
	596	+ struct dentry *rqos_debugfs_dir;
673	597	#endif
674	598
675	599	bool mq_sysfs_init_done;
676	600
677	601	size_t cmd_size;
678		- void *rq_alloc_data;
679		-
680		- struct work_struct release_work;
681	602
682	603	#define BLK_MAX_WRITE_HINTS 5
683	604	u64 write_hints[BLK_MAX_WRITE_HINTS];
	605	+
	606	+ ANDROID_KABI_RESERVE(1);
	607	+ ANDROID_KABI_RESERVE(2);
	608	+ ANDROID_KABI_RESERVE(3);
	609	+ ANDROID_KABI_RESERVE(4);
	610	+ ANDROID_OEM_DATA(1);
684	611	};
685	612
686		-#define QUEUE_FLAG_QUEUED 0 /* uses generic tag queueing */
687		-#define QUEUE_FLAG_STOPPED 1 /* queue is stopped */
688		-#define QUEUE_FLAG_DYING 2 /* queue being torn down */
689		-#define QUEUE_FLAG_BYPASS 3 /* act as dumb FIFO queue */
690		-#define QUEUE_FLAG_BIDI 4 /* queue supports bidi requests */
691		-#define QUEUE_FLAG_NOMERGES 5 /* disable merge attempts */
692		-#define QUEUE_FLAG_SAME_COMP 6 /* complete on same CPU-group */
693		-#define QUEUE_FLAG_FAIL_IO 7 /* fake timeout */
694		-#define QUEUE_FLAG_NONROT 9 /* non-rotational device (SSD) */
695		-#define QUEUE_FLAG_VIRT QUEUE_FLAG_NONROT /* paravirt device */
696		-#define QUEUE_FLAG_IO_STAT 10 /* do IO stats */
697		-#define QUEUE_FLAG_DISCARD 11 /* supports DISCARD */
698		-#define QUEUE_FLAG_NOXMERGES 12 /* No extended merges */
699		-#define QUEUE_FLAG_ADD_RANDOM 13 /* Contributes to random pool */
700		-#define QUEUE_FLAG_SECERASE 14 /* supports secure erase */
701		-#define QUEUE_FLAG_SAME_FORCE 15 /* force complete on same CPU */
702		-#define QUEUE_FLAG_DEAD 16 /* queue tear-down finished */
703		-#define QUEUE_FLAG_INIT_DONE 17 /* queue is initialized */
704		-#define QUEUE_FLAG_NO_SG_MERGE 18 /* don't attempt to merge SG segments*/
705		-#define QUEUE_FLAG_POLL 19 /* IO polling enabled if set */
706		-#define QUEUE_FLAG_WC 20 /* Write back caching */
707		-#define QUEUE_FLAG_FUA 21 /* device supports FUA writes */
708		-#define QUEUE_FLAG_FLUSH_NQ 22 /* flush not queueuable */
709		-#define QUEUE_FLAG_DAX 23 /* device supports DAX */
710		-#define QUEUE_FLAG_STATS 24 /* track rq completion times */
711		-#define QUEUE_FLAG_POLL_STATS 25 /* collecting stats for hybrid polling */
712		-#define QUEUE_FLAG_REGISTERED 26 /* queue has been registered to a disk */
713		-#define QUEUE_FLAG_SCSI_PASSTHROUGH 27 /* queue supports SCSI commands */
714		-#define QUEUE_FLAG_QUIESCED 28 /* queue has been quiesced */
715		-
716		-#define QUEUE_FLAG_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) \| \
717		- (1 << QUEUE_FLAG_SAME_COMP) \| \
718		- (1 << QUEUE_FLAG_ADD_RANDOM))
	613	+/* Keep blk_queue_flag_name[] in sync with the definitions below */
	614	+#define QUEUE_FLAG_STOPPED 0 /* queue is stopped */
	615	+#define QUEUE_FLAG_DYING 1 /* queue being torn down */
	616	+#define QUEUE_FLAG_NOMERGES 3 /* disable merge attempts */
	617	+#define QUEUE_FLAG_SAME_COMP 4 /* complete on same CPU-group */
	618	+#define QUEUE_FLAG_FAIL_IO 5 /* fake timeout */
	619	+#define QUEUE_FLAG_NONROT 6 /* non-rotational device (SSD) */
	620	+#define QUEUE_FLAG_VIRT QUEUE_FLAG_NONROT /* paravirt device */
	621	+#define QUEUE_FLAG_IO_STAT 7 /* do disk/partitions IO accounting */
	622	+#define QUEUE_FLAG_DISCARD 8 /* supports DISCARD */
	623	+#define QUEUE_FLAG_NOXMERGES 9 /* No extended merges */
	624	+#define QUEUE_FLAG_ADD_RANDOM 10 /* Contributes to random pool */
	625	+#define QUEUE_FLAG_SECERASE 11 /* supports secure erase */
	626	+#define QUEUE_FLAG_SAME_FORCE 12 /* force complete on same CPU */
	627	+#define QUEUE_FLAG_DEAD 13 /* queue tear-down finished */
	628	+#define QUEUE_FLAG_INIT_DONE 14 /* queue is initialized */
	629	+#define QUEUE_FLAG_STABLE_WRITES 15 /* don't modify blks until WB is done */
	630	+#define QUEUE_FLAG_POLL 16 /* IO polling enabled if set */
	631	+#define QUEUE_FLAG_WC 17 /* Write back caching */
	632	+#define QUEUE_FLAG_FUA 18 /* device supports FUA writes */
	633	+#define QUEUE_FLAG_DAX 19 /* device supports DAX */
	634	+#define QUEUE_FLAG_STATS 20 /* track IO start and completion times */
	635	+#define QUEUE_FLAG_POLL_STATS 21 /* collecting stats for hybrid polling */
	636	+#define QUEUE_FLAG_REGISTERED 22 /* queue has been registered to a disk */
	637	+#define QUEUE_FLAG_SCSI_PASSTHROUGH 23 /* queue supports SCSI commands */
	638	+#define QUEUE_FLAG_QUIESCED 24 /* queue has been quiesced */
	639	+#define QUEUE_FLAG_PCI_P2PDMA 25 /* device supports PCI p2p requests */
	640	+#define QUEUE_FLAG_ZONE_RESETALL 26 /* supports Zone Reset All */
	641	+#define QUEUE_FLAG_RQ_ALLOC_TIME 27 /* record rq->alloc_time_ns */
	642	+#define QUEUE_FLAG_HCTX_ACTIVE 28 /* at least one blk-mq hctx is active */
	643	+#define QUEUE_FLAG_NOWAIT 29 /* device supports NOWAIT */
719	644
720	645	#define QUEUE_FLAG_MQ_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) \| \
721		- (1 << QUEUE_FLAG_SAME_COMP) \| \
722		- (1 << QUEUE_FLAG_POLL))
	646	+ (1 << QUEUE_FLAG_SAME_COMP) \| \
	647	+ (1 << QUEUE_FLAG_NOWAIT))
723	648
724	649	void blk_queue_flag_set(unsigned int flag, struct request_queue *q);
725	650	void blk_queue_flag_clear(unsigned int flag, struct request_queue *q);
726	651	bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q);
727		-bool blk_queue_flag_test_and_clear(unsigned int flag, struct request_queue *q);
728	652
729		-#define blk_queue_tagged(q) test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags)
730	653	#define blk_queue_stopped(q) test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags)
731	654	#define blk_queue_dying(q) test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags)
732	655	#define blk_queue_dead(q) test_bit(QUEUE_FLAG_DEAD, &(q)->queue_flags)
733		-#define blk_queue_bypass(q) test_bit(QUEUE_FLAG_BYPASS, &(q)->queue_flags)
734	656	#define blk_queue_init_done(q) test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags)
735	657	#define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags)
736	658	#define blk_queue_noxmerges(q) \
737	659	test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags)
738	660	#define blk_queue_nonrot(q) test_bit(QUEUE_FLAG_NONROT, &(q)->queue_flags)
	661	+#define blk_queue_stable_writes(q) \
	662	+ test_bit(QUEUE_FLAG_STABLE_WRITES, &(q)->queue_flags)
739	663	#define blk_queue_io_stat(q) test_bit(QUEUE_FLAG_IO_STAT, &(q)->queue_flags)
740	664	#define blk_queue_add_random(q) test_bit(QUEUE_FLAG_ADD_RANDOM, &(q)->queue_flags)
741	665	#define blk_queue_discard(q) test_bit(QUEUE_FLAG_DISCARD, &(q)->queue_flags)
	666	+#define blk_queue_zone_resetall(q) \
	667	+ test_bit(QUEUE_FLAG_ZONE_RESETALL, &(q)->queue_flags)
742	668	#define blk_queue_secure_erase(q) \
743	669	(test_bit(QUEUE_FLAG_SECERASE, &(q)->queue_flags))
744	670	#define blk_queue_dax(q) test_bit(QUEUE_FLAG_DAX, &(q)->queue_flags)
745	671	#define blk_queue_scsi_passthrough(q) \
746	672	test_bit(QUEUE_FLAG_SCSI_PASSTHROUGH, &(q)->queue_flags)
	673	+#define blk_queue_pci_p2pdma(q) \
	674	+ test_bit(QUEUE_FLAG_PCI_P2PDMA, &(q)->queue_flags)
	675	+#ifdef CONFIG_BLK_RQ_ALLOC_TIME
	676	+#define blk_queue_rq_alloc_time(q) \
	677	+ test_bit(QUEUE_FLAG_RQ_ALLOC_TIME, &(q)->queue_flags)
	678	+#else
	679	+#define blk_queue_rq_alloc_time(q) false
	680	+#endif
747	681
748	682	#define blk_noretry_request(rq) \
749	683	((rq)->cmd_flags & (REQ_FAILFAST_DEV\|REQ_FAILFAST_TRANSPORT\| \
..	..	@@ -752,47 +686,38 @@
752	686	#define blk_queue_pm_only(q) atomic_read(&(q)->pm_only)
753	687	#define blk_queue_fua(q) test_bit(QUEUE_FLAG_FUA, &(q)->queue_flags)
754	688	#define blk_queue_registered(q) test_bit(QUEUE_FLAG_REGISTERED, &(q)->queue_flags)
	689	+#define blk_queue_nowait(q) test_bit(QUEUE_FLAG_NOWAIT, &(q)->queue_flags)
755	690
756	691	extern void blk_set_pm_only(struct request_queue *q);
757	692	extern void blk_clear_pm_only(struct request_queue *q);
758		-
759		-static inline int queue_in_flight(struct request_queue *q)
760		-{
761		- return q->in_flight[0] + q->in_flight[1];
762		-}
763	693
764	694	static inline bool blk_account_rq(struct request *rq)
765	695	{
766	696	return (rq->rq_flags & RQF_STARTED) && !blk_rq_is_passthrough(rq);
767	697	}
768	698
769		-#define blk_rq_cpu_valid(rq) ((rq)->cpu != -1)
770		-#define blk_bidi_rq(rq) ((rq)->next_rq != NULL)
771		-/* rq->queuelist of dequeued request must be list_empty() */
772		-#define blk_queued_rq(rq) (!list_empty(&(rq)->queuelist))
773		-
774	699	#define list_entry_rq(ptr) list_entry((ptr), struct request, queuelist)
775	700
776	701	#define rq_data_dir(rq) (op_is_write(req_op(rq)) ? WRITE : READ)
777	702
778		-/*
779		- * Driver can handle struct request, if it either has an old style
780		- * request_fn defined, or is blk-mq based.
781		- */
782		-static inline bool queue_is_rq_based(struct request_queue *q)
783		-{
784		- return q->request_fn \|\| q->mq_ops;
785		-}
	703	+#define rq_dma_dir(rq) \
	704	+ (op_is_write(req_op(rq)) ? DMA_TO_DEVICE : DMA_FROM_DEVICE)
786	705
787		-static inline unsigned int blk_queue_cluster(struct request_queue *q)
	706	+#define dma_map_bvec(dev, bv, dir, attrs) \
	707	+ dma_map_page_attrs(dev, (bv)->bv_page, (bv)->bv_offset, (bv)->bv_len, \
	708	+ (dir), (attrs))
	709	+
	710	+static inline bool queue_is_mq(struct request_queue *q)
788	711	{
789		- return q->limits.cluster;
	712	+ return q->mq_ops;
790	713	}
791	714
792	715	static inline enum blk_zoned_model
793	716	blk_queue_zoned_model(struct request_queue *q)
794	717	{
795		- return q->limits.zoned;
	718	+ if (IS_ENABLED(CONFIG_BLK_DEV_ZONED))
	719	+ return q->limits.zoned;
	720	+ return BLK_ZONED_NONE;
796	721	}
797	722
798	723	static inline bool blk_queue_is_zoned(struct request_queue *q)
..	..	@@ -806,12 +731,17 @@
806	731	}
807	732	}
808	733
809		-static inline unsigned int blk_queue_zone_sectors(struct request_queue *q)
	734	+static inline sector_t blk_queue_zone_sectors(struct request_queue *q)
810	735	{
811	736	return blk_queue_is_zoned(q) ? q->limits.chunk_sectors : 0;
812	737	}
813	738
814	739	#ifdef CONFIG_BLK_DEV_ZONED
	740	+static inline unsigned int blk_queue_nr_zones(struct request_queue *q)
	741	+{
	742	+ return blk_queue_is_zoned(q) ? q->nr_zones : 0;
	743	+}
	744	+
815	745	static inline unsigned int blk_queue_zone_no(struct request_queue *q,
816	746	sector_t sector)
817	747	{
..	..	@@ -823,36 +753,62 @@
823	753	static inline bool blk_queue_zone_is_seq(struct request_queue *q,
824	754	sector_t sector)
825	755	{
826		- if (!blk_queue_is_zoned(q) \|\| !q->seq_zones_bitmap)
	756	+ if (!blk_queue_is_zoned(q))
827	757	return false;
828		- return test_bit(blk_queue_zone_no(q, sector), q->seq_zones_bitmap);
	758	+ if (!q->conv_zones_bitmap)
	759	+ return true;
	760	+ return !test_bit(blk_queue_zone_no(q, sector), q->conv_zones_bitmap);
	761	+}
	762	+
	763	+static inline void blk_queue_max_open_zones(struct request_queue *q,
	764	+ unsigned int max_open_zones)
	765	+{
	766	+ q->max_open_zones = max_open_zones;
	767	+}
	768	+
	769	+static inline unsigned int queue_max_open_zones(const struct request_queue *q)
	770	+{
	771	+ return q->max_open_zones;
	772	+}
	773	+
	774	+static inline void blk_queue_max_active_zones(struct request_queue *q,
	775	+ unsigned int max_active_zones)
	776	+{
	777	+ q->max_active_zones = max_active_zones;
	778	+}
	779	+
	780	+static inline unsigned int queue_max_active_zones(const struct request_queue *q)
	781	+{
	782	+ return q->max_active_zones;
	783	+}
	784	+#else /* CONFIG_BLK_DEV_ZONED */
	785	+static inline unsigned int blk_queue_nr_zones(struct request_queue *q)
	786	+{
	787	+ return 0;
	788	+}
	789	+static inline bool blk_queue_zone_is_seq(struct request_queue *q,
	790	+ sector_t sector)
	791	+{
	792	+ return false;
	793	+}
	794	+static inline unsigned int blk_queue_zone_no(struct request_queue *q,
	795	+ sector_t sector)
	796	+{
	797	+ return 0;
	798	+}
	799	+static inline unsigned int queue_max_open_zones(const struct request_queue *q)
	800	+{
	801	+ return 0;
	802	+}
	803	+static inline unsigned int queue_max_active_zones(const struct request_queue *q)
	804	+{
	805	+ return 0;
829	806	}
830	807	#endif /* CONFIG_BLK_DEV_ZONED */
831	808
832	809	static inline bool rq_is_sync(struct request *rq)
833	810	{
834	811	return op_is_sync(rq->cmd_flags);
835		-}
836		-
837		-static inline bool blk_rl_full(struct request_list *rl, bool sync)
838		-{
839		- unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;
840		-
841		- return rl->flags & flag;
842		-}
843		-
844		-static inline void blk_set_rl_full(struct request_list *rl, bool sync)
845		-{
846		- unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;
847		-
848		- rl->flags \|= flag;
849		-}
850		-
851		-static inline void blk_clear_rl_full(struct request_list *rl, bool sync)
852		-{
853		- unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;
854		-
855		- rl->flags &= ~flag;
856	812	}
857	813
858	814	static inline bool rq_mergeable(struct request *rq)
..	..	@@ -864,6 +820,9 @@
864	820	return false;
865	821
866	822	if (req_op(rq) == REQ_OP_WRITE_ZEROES)
	823	+ return false;
	824	+
	825	+ if (req_op(rq) == REQ_OP_ZONE_APPEND)
867	826	return false;
868	827
869	828	if (rq->cmd_flags & REQ_NOMERGE_FLAGS)
..	..	@@ -890,16 +849,6 @@
890	849
891	850	return q->nr_requests;
892	851	}
893		-
894		-/*
895		- * q->prep_rq_fn return values
896		- */
897		-enum {
898		- BLKPREP_OK, /* serve it */
899		- BLKPREP_KILL, /* fatal error, kill, return -EIO */
900		- BLKPREP_DEFER, /* leave on queue */
901		- BLKPREP_INVALID, /* invalid command, kill, return -EREMOTEIO */
902		-};
903	852
904	853	extern unsigned long blk_max_low_pfn, blk_max_pfn;
905	854
..	..	@@ -950,6 +899,10 @@
950	899	__rq_for_each_bio(_iter.bio, _rq) \
951	900	bio_for_each_segment(bvl, _iter.bio, _iter.iter)
952	901
	902	+#define rq_for_each_bvec(bvl, _rq, _iter) \
	903	+ __rq_for_each_bio(_iter.bio, _rq) \
	904	+ bio_for_each_bvec(bvl, _iter.bio, _iter.iter)
	905	+
953	906	#define rq_iter_last(bvec, _iter) \
954	907	(_iter.bio->bi_next == NULL && \
955	908	bio_iter_last(bvec, _iter.iter))
..	..	@@ -967,15 +920,11 @@
967	920
968	921	extern int blk_register_queue(struct gendisk *disk);
969	922	extern void blk_unregister_queue(struct gendisk *disk);
970		-extern blk_qc_t generic_make_request(struct bio *bio);
971		-extern blk_qc_t direct_make_request(struct bio *bio);
	923	+blk_qc_t submit_bio_noacct(struct bio *bio);
972	924	extern void blk_rq_init(struct request_queue q, struct request rq);
973		-extern void blk_init_request_from_bio(struct request req, struct bio bio);
974	925	extern void blk_put_request(struct request *);
975		-extern void __blk_put_request(struct request_queue , struct request );
976	926	extern struct request blk_get_request(struct request_queue , unsigned int op,
977	927	blk_mq_req_flags_t flags);
978		-extern void blk_requeue_request(struct request_queue , struct request );
979	928	extern int blk_lld_busy(struct request_queue *q);
980	929	extern int blk_rq_prep_clone(struct request rq, struct request rq_src,
981	930	struct bio_set *bs, gfp_t gfp_mask,
..	..	@@ -985,9 +934,7 @@
985	934	extern blk_status_t blk_insert_cloned_request(struct request_queue *q,
986	935	struct request *rq);
987	936	extern int blk_rq_append_bio(struct request rq, struct bio *bio);
988		-extern void blk_delay_queue(struct request_queue *, unsigned long);
989		-extern void blk_queue_split(struct request_queue , struct bio *);
990		-extern void blk_recount_segments(struct request_queue , struct bio );
	937	+extern void blk_queue_split(struct bio **);
991	938	extern int scsi_verify_blk_ioctl(struct block_device *, unsigned int);
992	939	extern int scsi_cmd_blk_ioctl(struct block_device *, fmode_t,
993	940	unsigned int, void __user *);
..	..	@@ -995,18 +942,12 @@
995	942	unsigned int, void __user *);
996	943	extern int sg_scsi_ioctl(struct request_queue , struct gendisk , fmode_t,
997	944	struct scsi_ioctl_command __user *);
	945	+extern int get_sg_io_hdr(struct sg_io_hdr hdr, const void __user argp);
	946	+extern int put_sg_io_hdr(const struct sg_io_hdr hdr, void __user argp);
998	947
999	948	extern int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags);
1000	949	extern void blk_queue_exit(struct request_queue *q);
1001		-extern void blk_start_queue(struct request_queue *q);
1002		-extern void blk_start_queue_async(struct request_queue *q);
1003		-extern void blk_stop_queue(struct request_queue *q);
1004	950	extern void blk_sync_queue(struct request_queue *q);
1005		-extern void __blk_stop_queue(struct request_queue *q);
1006		-extern void __blk_run_queue(struct request_queue *q);
1007		-extern void __blk_run_queue_uncond(struct request_queue *q);
1008		-extern void blk_run_queue(struct request_queue *);
1009		-extern void blk_run_queue_async(struct request_queue *q);
1010	951	extern int blk_rq_map_user(struct request_queue , struct request ,
1011	952	struct rq_map_data , void __user , unsigned long,
1012	953	gfp_t);
..	..	@@ -1020,10 +961,13 @@
1020	961	extern void blk_execute_rq_nowait(struct request_queue , struct gendisk ,
1021	962	struct request , int, rq_end_io_fn );
1022	963
	964	+/* Helper to convert REQ_OP_XXX to its string format XXX */
	965	+extern const char *blk_op_str(unsigned int op);
	966	+
1023	967	int blk_status_to_errno(blk_status_t status);
1024	968	blk_status_t errno_to_blk_status(int errno);
1025	969
1026		-bool blk_poll(struct request_queue *q, blk_qc_t cookie);
	970	+int blk_poll(struct request_queue *q, blk_qc_t cookie, bool spin);
1027	971
1028	972	static inline struct request_queue bdev_get_queue(struct block_device bdev)
1029	973	{
..	..	@@ -1050,6 +994,7 @@
1050	994	* blk_rq_err_bytes() : bytes left till the next error boundary
1051	995	* blk_rq_sectors() : sectors left in the entire request
1052	996	* blk_rq_cur_sectors() : sectors left in the current segment
	997	+ * blk_rq_stats_sectors() : sectors of the entire request used for stats
1053	998	*/
1054	999	static inline sector_t blk_rq_pos(const struct request *rq)
1055	1000	{
..	..	@@ -1078,7 +1023,16 @@
1078	1023	return blk_rq_cur_bytes(rq) >> SECTOR_SHIFT;
1079	1024	}
1080	1025
	1026	+static inline unsigned int blk_rq_stats_sectors(const struct request *rq)
	1027	+{
	1028	+ return rq->stats_sectors;
	1029	+}
	1030	+
1081	1031	#ifdef CONFIG_BLK_DEV_ZONED
	1032	+
	1033	+/* Helper to convert BLK_ZONE_ZONE_XXX to its string format XXX */
	1034	+const char *blk_zone_cond_str(enum blk_zone_cond zone_cond);
	1035	+
1082	1036	static inline unsigned int blk_rq_zone_no(struct request *rq)
1083	1037	{
1084	1038	return blk_queue_zone_no(rq->q, blk_rq_pos(rq));
..	..	@@ -1103,6 +1057,17 @@
1103	1057	return blk_rq_bytes(rq);
1104	1058	}
1105	1059
	1060	+/*
	1061	+ * Return the first full biovec in the request. The caller needs to check that
	1062	+ * there are any bvecs before calling this helper.
	1063	+ */
	1064	+static inline struct bio_vec req_bvec(struct request *rq)
	1065	+{
	1066	+ if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
	1067	+ return rq->special_vec;
	1068	+ return mp_bvec_iter_bvec(rq->bio->bi_io_vec, rq->bio->bi_iter);
	1069	+}
	1070	+
1106	1071	static inline unsigned int blk_queue_get_max_sectors(struct request_queue *q,
1107	1072	int op)
1108	1073	{
..	..	@@ -1124,13 +1089,22 @@
1124	1089	* file system requests.
1125	1090	*/
1126	1091	static inline unsigned int blk_max_size_offset(struct request_queue *q,
1127		- sector_t offset)
	1092	+ sector_t offset,
	1093	+ unsigned int chunk_sectors)
1128	1094	{
1129		- if (!q->limits.chunk_sectors)
1130		- return q->limits.max_sectors;
	1095	+ if (!chunk_sectors) {
	1096	+ if (q->limits.chunk_sectors)
	1097	+ chunk_sectors = q->limits.chunk_sectors;
	1098	+ else
	1099	+ return q->limits.max_sectors;
	1100	+ }
1131	1101
1132		- return min(q->limits.max_sectors, (unsigned int)(q->limits.chunk_sectors -
1133		- (offset & (q->limits.chunk_sectors - 1))));
	1102	+ if (likely(is_power_of_2(chunk_sectors)))
	1103	+ chunk_sectors -= offset & (chunk_sectors - 1);
	1104	+ else
	1105	+ chunk_sectors -= sector_div(offset, chunk_sectors);
	1106	+
	1107	+ return min(q->limits.max_sectors, chunk_sectors);
1134	1108	}
1135	1109
1136	1110	static inline unsigned int blk_rq_get_max_sectors(struct request *rq,
..	..	@@ -1146,7 +1120,7 @@
1146	1120	req_op(rq) == REQ_OP_SECURE_ERASE)
1147	1121	return blk_queue_get_max_sectors(q, req_op(rq));
1148	1122
1149		- return min(blk_max_size_offset(q, offset),
	1123	+ return min(blk_max_size_offset(q, offset, 0),
1150	1124	blk_queue_get_max_sectors(q, req_op(rq)));
1151	1125	}
1152	1126
..	..	@@ -1161,13 +1135,6 @@
1161	1135	return nr_bios;
1162	1136	}
1163	1137
1164		-/*
1165		- * Request issue related functions.
1166		- */
1167		-extern struct request blk_peek_request(struct request_queue q);
1168		-extern void blk_start_request(struct request *rq);
1169		-extern struct request blk_fetch_request(struct request_queue q);
1170		-
1171	1138	void blk_steal_bios(struct bio_list list, struct request rq);
1172	1139
1173	1140	/*
..	..	@@ -1175,39 +1142,16 @@
1175	1142	*
1176	1143	* blk_update_request() completes given number of bytes and updates
1177	1144	* the request without completing it.
1178		- *
1179		- * blk_end_request() and friends. __blk_end_request() must be called
1180		- * with the request queue spinlock acquired.
1181		- *
1182		- * Several drivers define their own end_request and call
1183		- * blk_end_request() for parts of the original function.
1184		- * This prevents code duplication in drivers.
1185	1145	*/
1186	1146	extern bool blk_update_request(struct request *rq, blk_status_t error,
1187	1147	unsigned int nr_bytes);
1188		-extern void blk_finish_request(struct request *rq, blk_status_t error);
1189		-extern bool blk_end_request(struct request *rq, blk_status_t error,
1190		- unsigned int nr_bytes);
1191		-extern void blk_end_request_all(struct request *rq, blk_status_t error);
1192		-extern bool __blk_end_request(struct request *rq, blk_status_t error,
1193		- unsigned int nr_bytes);
1194		-extern void __blk_end_request_all(struct request *rq, blk_status_t error);
1195		-extern bool __blk_end_request_cur(struct request *rq, blk_status_t error);
1196	1148
1197		-extern void blk_complete_request(struct request *);
1198		-extern void __blk_complete_request(struct request *);
1199	1149	extern void blk_abort_request(struct request *);
1200		-extern void blk_unprep_request(struct request *);
1201	1150
1202	1151	/*
1203	1152	* Access functions for manipulating queue properties
1204	1153	*/
1205		-extern struct request_queue blk_init_queue_node(request_fn_proc rfn,
1206		- spinlock_t *lock, int node_id);
1207		-extern struct request_queue blk_init_queue(request_fn_proc , spinlock_t *);
1208		-extern int blk_init_allocated_queue(struct request_queue *);
1209	1154	extern void blk_cleanup_queue(struct request_queue *);
1210		-extern void blk_queue_make_request(struct request_queue , make_request_fn );
1211	1155	extern void blk_queue_bounce_limit(struct request_queue *, u64);
1212	1156	extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int);
1213	1157	extern void blk_queue_chunk_sectors(struct request_queue *, unsigned int);
..	..	@@ -1222,9 +1166,12 @@
1222	1166	extern void blk_queue_max_write_zeroes_sectors(struct request_queue *q,
1223	1167	unsigned int max_write_same_sectors);
1224	1168	extern void blk_queue_logical_block_size(struct request_queue *, unsigned int);
	1169	+extern void blk_queue_max_zone_append_sectors(struct request_queue *q,
	1170	+ unsigned int max_zone_append_sectors);
1225	1171	extern void blk_queue_physical_block_size(struct request_queue *, unsigned int);
1226	1172	extern void blk_queue_alignment_offset(struct request_queue *q,
1227	1173	unsigned int alignment);
	1174	+void blk_queue_update_readahead(struct request_queue *q);
1228	1175	extern void blk_limits_io_min(struct queue_limits *limits, unsigned int min);
1229	1176	extern void blk_queue_io_min(struct request_queue *q, unsigned int min);
1230	1177	extern void blk_limits_io_opt(struct queue_limits *limits, unsigned int opt);
..	..	@@ -1234,28 +1181,19 @@
1234	1181	extern void blk_set_stacking_limits(struct queue_limits *lim);
1235	1182	extern int blk_stack_limits(struct queue_limits t, struct queue_limits b,
1236	1183	sector_t offset);
1237		-extern int bdev_stack_limits(struct queue_limits t, struct block_device bdev,
1238		- sector_t offset);
1239	1184	extern void disk_stack_limits(struct gendisk disk, struct block_device bdev,
1240	1185	sector_t offset);
1241		-extern void blk_queue_stack_limits(struct request_queue t, struct request_queue b);
1242		-extern void blk_queue_dma_pad(struct request_queue *, unsigned int);
1243	1186	extern void blk_queue_update_dma_pad(struct request_queue *, unsigned int);
1244		-extern int blk_queue_dma_drain(struct request_queue *q,
1245		- dma_drain_needed_fn *dma_drain_needed,
1246		- void *buf, unsigned int size);
1247		-extern void blk_queue_lld_busy(struct request_queue q, lld_busy_fn fn);
1248	1187	extern void blk_queue_segment_boundary(struct request_queue *, unsigned long);
1249	1188	extern void blk_queue_virt_boundary(struct request_queue *, unsigned long);
1250		-extern void blk_queue_prep_rq(struct request_queue , prep_rq_fn pfn);
1251		-extern void blk_queue_unprep_rq(struct request_queue , unprep_rq_fn ufn);
1252	1189	extern void blk_queue_dma_alignment(struct request_queue *, int);
1253	1190	extern void blk_queue_update_dma_alignment(struct request_queue *, int);
1254		-extern void blk_queue_softirq_done(struct request_queue , softirq_done_fn );
1255		-extern void blk_queue_rq_timed_out(struct request_queue , rq_timed_out_fn );
1256	1191	extern void blk_queue_rq_timeout(struct request_queue *, unsigned int);
1257		-extern void blk_queue_flush_queueable(struct request_queue *q, bool queueable);
1258	1192	extern void blk_queue_write_cache(struct request_queue *q, bool enabled, bool fua);
	1193	+extern void blk_queue_required_elevator_features(struct request_queue *q,
	1194	+ unsigned int features);
	1195	+extern bool blk_queue_can_use_dma_map_merging(struct request_queue *q,
	1196	+ struct device *dev);
1259	1197
1260	1198	/*
1261	1199	* Number of physical segments as sent to the device.
..	..	@@ -1282,40 +1220,23 @@
1282	1220	return max_t(unsigned short, rq->nr_phys_segments, 1);
1283	1221	}
1284	1222
1285		-extern int blk_rq_map_sg(struct request_queue , struct request , struct scatterlist *);
	1223	+int __blk_rq_map_sg(struct request_queue q, struct request rq,
	1224	+ struct scatterlist sglist, struct scatterlist *last_sg);
	1225	+static inline int blk_rq_map_sg(struct request_queue q, struct request rq,
	1226	+ struct scatterlist *sglist)
	1227	+{
	1228	+ struct scatterlist *last_sg = NULL;
	1229	+
	1230	+ return __blk_rq_map_sg(q, rq, sglist, &last_sg);
	1231	+}
1286	1232	extern void blk_dump_rq_flags(struct request , char );
1287		-extern long nr_blockdev_pages(void);
1288	1233
1289	1234	bool __must_check blk_get_queue(struct request_queue *);
1290		-struct request_queue *blk_alloc_queue(gfp_t);
1291		-struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id,
1292		- spinlock_t *lock);
	1235	+struct request_queue *blk_alloc_queue(int node_id);
1293	1236	extern void blk_put_queue(struct request_queue *);
1294	1237	extern void blk_set_queue_dying(struct request_queue *);
1295	1238
1296		-/*
1297		- * block layer runtime pm functions
1298		- */
1299		-#ifdef CONFIG_PM
1300		-extern void blk_pm_runtime_init(struct request_queue q, struct device dev);
1301		-extern int blk_pre_runtime_suspend(struct request_queue *q);
1302		-extern void blk_post_runtime_suspend(struct request_queue *q, int err);
1303		-extern void blk_pre_runtime_resume(struct request_queue *q);
1304		-extern void blk_post_runtime_resume(struct request_queue *q, int err);
1305		-extern void blk_set_runtime_active(struct request_queue *q);
1306		-#else
1307		-static inline void blk_pm_runtime_init(struct request_queue *q,
1308		- struct device *dev) {}
1309		-static inline int blk_pre_runtime_suspend(struct request_queue *q)
1310		-{
1311		- return -ENOSYS;
1312		-}
1313		-static inline void blk_post_runtime_suspend(struct request_queue *q, int err) {}
1314		-static inline void blk_pre_runtime_resume(struct request_queue *q) {}
1315		-static inline void blk_post_runtime_resume(struct request_queue *q, int err) {}
1316		-static inline void blk_set_runtime_active(struct request_queue *q) {}
1317		-#endif
1318		-
	1239	+#ifdef CONFIG_BLOCK
1319	1240	/*
1320	1241	* blk_plug permits building a queue of related requests by holding the I/O
1321	1242	* fragments for a short period. This allows merging of sequential requests
..	..	@@ -1329,12 +1250,12 @@
1329	1250	* schedule() where blk_schedule_flush_plug() is called.
1330	1251	*/
1331	1252	struct blk_plug {
1332		- struct list_head list; /* requests */
1333	1253	struct list_head mq_list; /* blk-mq requests */
1334	1254	struct list_head cb_list; /* md requires an unplug callback */
	1255	+ unsigned short rq_count;
	1256	+ bool multiple_queues;
	1257	+ bool nowait;
1335	1258	};
1336		-#define BLK_MAX_REQUEST_COUNT 16
1337		-#define BLK_PLUG_FLUSH_SIZE (128 * 1024)
1338	1259
1339	1260	struct blk_plug_cb;
1340	1261	typedef void (blk_plug_cb_fn)(struct blk_plug_cb , bool);
..	..	@@ -1370,32 +1291,51 @@
1370	1291	struct blk_plug *plug = tsk->plug;
1371	1292
1372	1293	return plug &&
1373		- (!list_empty(&plug->list) \|\|
1374		- !list_empty(&plug->mq_list) \|\|
	1294	+ (!list_empty(&plug->mq_list) \|\|
1375	1295	!list_empty(&plug->cb_list));
1376	1296	}
1377	1297
1378		-/*
1379		- * tag stuff
1380		- */
1381		-extern int blk_queue_start_tag(struct request_queue , struct request );
1382		-extern struct request blk_queue_find_tag(struct request_queue , int);
1383		-extern void blk_queue_end_tag(struct request_queue , struct request );
1384		-extern int blk_queue_init_tags(struct request_queue , int, struct blk_queue_tag , int);
1385		-extern void blk_queue_free_tags(struct request_queue *);
1386		-extern int blk_queue_resize_tags(struct request_queue *, int);
1387		-extern struct blk_queue_tag *blk_init_tags(int, int);
1388		-extern void blk_free_tags(struct blk_queue_tag *);
	1298	+int blkdev_issue_flush(struct block_device *, gfp_t);
	1299	+long nr_blockdev_pages(void);
	1300	+#else /* CONFIG_BLOCK */
	1301	+struct blk_plug {
	1302	+};
1389	1303
1390		-static inline struct request blk_map_queue_find_tag(struct blk_queue_tag bqt,
1391		- int tag)
	1304	+static inline void blk_start_plug(struct blk_plug *plug)
1392	1305	{
1393		- if (unlikely(bqt == NULL \|\| tag >= bqt->real_max_depth))
1394		- return NULL;
1395		- return bqt->tag_index[tag];
1396	1306	}
1397	1307
1398		-extern int blkdev_issue_flush(struct block_device , gfp_t, sector_t );
	1308	+static inline void blk_finish_plug(struct blk_plug *plug)
	1309	+{
	1310	+}
	1311	+
	1312	+static inline void blk_flush_plug(struct task_struct *task)
	1313	+{
	1314	+}
	1315	+
	1316	+static inline void blk_schedule_flush_plug(struct task_struct *task)
	1317	+{
	1318	+}
	1319	+
	1320	+
	1321	+static inline bool blk_needs_flush_plug(struct task_struct *tsk)
	1322	+{
	1323	+ return false;
	1324	+}
	1325	+
	1326	+static inline int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask)
	1327	+{
	1328	+ return 0;
	1329	+}
	1330	+
	1331	+static inline long nr_blockdev_pages(void)
	1332	+{
	1333	+ return 0;
	1334	+}
	1335	+#endif /* CONFIG_BLOCK */
	1336	+
	1337	+extern void blk_io_schedule(void);
	1338	+
1399	1339	extern int blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
1400	1340	sector_t nr_sects, gfp_t gfp_mask, struct page *page);
1401	1341
..	..	@@ -1439,6 +1379,11 @@
1439	1379
1440	1380	extern int blk_verify_command(unsigned char *cmd, fmode_t mode);
1441	1381
	1382	+static inline bool bdev_is_partition(struct block_device *bdev)
	1383	+{
	1384	+ return bdev->bd_partno;
	1385	+}
	1386	+
1442	1387	enum blk_default_limits {
1443	1388	BLK_MAX_SEGMENTS = 128,
1444	1389	BLK_SAFE_MAX_SECTORS = 255,
..	..	@@ -1447,42 +1392,50 @@
1447	1392	BLK_SEG_BOUNDARY_MASK = 0xFFFFFFFFUL,
1448	1393	};
1449	1394
1450		-static inline unsigned long queue_segment_boundary(struct request_queue *q)
	1395	+static inline unsigned long queue_segment_boundary(const struct request_queue *q)
1451	1396	{
1452	1397	return q->limits.seg_boundary_mask;
1453	1398	}
1454	1399
1455		-static inline unsigned long queue_virt_boundary(struct request_queue *q)
	1400	+static inline unsigned long queue_virt_boundary(const struct request_queue *q)
1456	1401	{
1457	1402	return q->limits.virt_boundary_mask;
1458	1403	}
1459	1404
1460		-static inline unsigned int queue_max_sectors(struct request_queue *q)
	1405	+static inline unsigned int queue_max_sectors(const struct request_queue *q)
1461	1406	{
1462	1407	return q->limits.max_sectors;
1463	1408	}
1464	1409
1465		-static inline unsigned int queue_max_hw_sectors(struct request_queue *q)
	1410	+static inline unsigned int queue_max_hw_sectors(const struct request_queue *q)
1466	1411	{
1467	1412	return q->limits.max_hw_sectors;
1468	1413	}
1469	1414
1470		-static inline unsigned short queue_max_segments(struct request_queue *q)
	1415	+static inline unsigned short queue_max_segments(const struct request_queue *q)
1471	1416	{
1472	1417	return q->limits.max_segments;
1473	1418	}
1474	1419
1475		-static inline unsigned short queue_max_discard_segments(struct request_queue *q)
	1420	+static inline unsigned short queue_max_discard_segments(const struct request_queue *q)
1476	1421	{
1477	1422	return q->limits.max_discard_segments;
1478	1423	}
1479	1424
1480		-static inline unsigned int queue_max_segment_size(struct request_queue *q)
	1425	+static inline unsigned int queue_max_segment_size(const struct request_queue *q)
1481	1426	{
1482	1427	return q->limits.max_segment_size;
1483	1428	}
1484	1429
1485		-static inline unsigned queue_logical_block_size(struct request_queue *q)
	1430	+static inline unsigned int queue_max_zone_append_sectors(const struct request_queue *q)
	1431	+{
	1432	+
	1433	+ const struct queue_limits *l = &q->limits;
	1434	+
	1435	+ return min(l->max_zone_append_sectors, l->max_sectors);
	1436	+}
	1437	+
	1438	+static inline unsigned queue_logical_block_size(const struct request_queue *q)
1486	1439	{
1487	1440	int retval = 512;
1488	1441
..	..	@@ -1497,7 +1450,7 @@
1497	1450	return queue_logical_block_size(bdev_get_queue(bdev));
1498	1451	}
1499	1452
1500		-static inline unsigned int queue_physical_block_size(struct request_queue *q)
	1453	+static inline unsigned int queue_physical_block_size(const struct request_queue *q)
1501	1454	{
1502	1455	return q->limits.physical_block_size;
1503	1456	}
..	..	@@ -1507,7 +1460,7 @@
1507	1460	return queue_physical_block_size(bdev_get_queue(bdev));
1508	1461	}
1509	1462
1510		-static inline unsigned int queue_io_min(struct request_queue *q)
	1463	+static inline unsigned int queue_io_min(const struct request_queue *q)
1511	1464	{
1512	1465	return q->limits.io_min;
1513	1466	}
..	..	@@ -1517,7 +1470,7 @@
1517	1470	return queue_io_min(bdev_get_queue(bdev));
1518	1471	}
1519	1472
1520		-static inline unsigned int queue_io_opt(struct request_queue *q)
	1473	+static inline unsigned int queue_io_opt(const struct request_queue *q)
1521	1474	{
1522	1475	return q->limits.io_opt;
1523	1476	}
..	..	@@ -1527,7 +1480,7 @@
1527	1480	return queue_io_opt(bdev_get_queue(bdev));
1528	1481	}
1529	1482
1530		-static inline int queue_alignment_offset(struct request_queue *q)
	1483	+static inline int queue_alignment_offset(const struct request_queue *q)
1531	1484	{
1532	1485	if (q->limits.misaligned)
1533	1486	return -1;
..	..	@@ -1550,14 +1503,13 @@
1550	1503
1551	1504	if (q->limits.misaligned)
1552	1505	return -1;
1553		-
1554		- if (bdev != bdev->bd_contains)
1555		- return bdev->bd_part->alignment_offset;
1556		-
	1506	+ if (bdev_is_partition(bdev))
	1507	+ return queue_limit_alignment_offset(&q->limits,
	1508	+ bdev->bd_part->start_sect);
1557	1509	return q->limits.alignment_offset;
1558	1510	}
1559	1511
1560		-static inline int queue_discard_alignment(struct request_queue *q)
	1512	+static inline int queue_discard_alignment(const struct request_queue *q)
1561	1513	{
1562	1514	if (q->limits.discard_misaligned)
1563	1515	return -1;
..	..	@@ -1588,13 +1540,29 @@
1588	1540	return offset << SECTOR_SHIFT;
1589	1541	}
1590	1542
	1543	+/*
	1544	+ * Two cases of handling DISCARD merge:
	1545	+ * If max_discard_segments > 1, the driver takes every bio
	1546	+ * as a range and send them to controller together. The ranges
	1547	+ * needn't to be contiguous.
	1548	+ * Otherwise, the bios/requests will be handled as same as
	1549	+ * others which should be contiguous.
	1550	+ */
	1551	+static inline bool blk_discard_mergable(struct request *req)
	1552	+{
	1553	+ if (req_op(req) == REQ_OP_DISCARD &&
	1554	+ queue_max_discard_segments(req->q) > 1)
	1555	+ return true;
	1556	+ return false;
	1557	+}
	1558	+
1591	1559	static inline int bdev_discard_alignment(struct block_device *bdev)
1592	1560	{
1593	1561	struct request_queue *q = bdev_get_queue(bdev);
1594	1562
1595		- if (bdev != bdev->bd_contains)
1596		- return bdev->bd_part->discard_alignment;
1597		-
	1563	+ if (bdev_is_partition(bdev))
	1564	+ return queue_limit_discard_alignment(&q->limits,
	1565	+ bdev->bd_part->start_sect);
1598	1566	return q->limits.discard_alignment;
1599	1567	}
1600	1568
..	..	@@ -1638,7 +1606,7 @@
1638	1606	return false;
1639	1607	}
1640	1608
1641		-static inline unsigned int bdev_zone_sectors(struct block_device *bdev)
	1609	+static inline sector_t bdev_zone_sectors(struct block_device *bdev)
1642	1610	{
1643	1611	struct request_queue *q = bdev_get_queue(bdev);
1644	1612
..	..	@@ -1647,7 +1615,25 @@
1647	1615	return 0;
1648	1616	}
1649	1617
1650		-static inline int queue_dma_alignment(struct request_queue *q)
	1618	+static inline unsigned int bdev_max_open_zones(struct block_device *bdev)
	1619	+{
	1620	+ struct request_queue *q = bdev_get_queue(bdev);
	1621	+
	1622	+ if (q)
	1623	+ return queue_max_open_zones(q);
	1624	+ return 0;
	1625	+}
	1626	+
	1627	+static inline unsigned int bdev_max_active_zones(struct block_device *bdev)
	1628	+{
	1629	+ struct request_queue *q = bdev_get_queue(bdev);
	1630	+
	1631	+ if (q)
	1632	+ return queue_max_active_zones(q);
	1633	+ return 0;
	1634	+}
	1635	+
	1636	+static inline int queue_dma_alignment(const struct request_queue *q)
1651	1637	{
1652	1638	return q ? q->dma_alignment : 511;
1653	1639	}
..	..	@@ -1672,113 +1658,10 @@
1672	1658
1673	1659	static inline unsigned int block_size(struct block_device *bdev)
1674	1660	{
1675		- return bdev->bd_block_size;
1676		-}
1677		-
1678		-static inline bool queue_flush_queueable(struct request_queue *q)
1679		-{
1680		- return !test_bit(QUEUE_FLAG_FLUSH_NQ, &q->queue_flags);
1681		-}
1682		-
1683		-typedef struct {struct page *v;} Sector;
1684		-
1685		-unsigned char read_dev_sector(struct block_device , sector_t, Sector *);
1686		-
1687		-static inline void put_dev_sector(Sector p)
1688		-{
1689		- put_page(p.v);
1690		-}
1691		-
1692		-static inline bool __bvec_gap_to_prev(struct request_queue *q,
1693		- struct bio_vec *bprv, unsigned int offset)
1694		-{
1695		- return offset \|\|
1696		- ((bprv->bv_offset + bprv->bv_len) & queue_virt_boundary(q));
1697		-}
1698		-
1699		-/*
1700		- * Check if adding a bio_vec after bprv with offset would create a gap in
1701		- * the SG list. Most drivers don't care about this, but some do.
1702		- */
1703		-static inline bool bvec_gap_to_prev(struct request_queue *q,
1704		- struct bio_vec *bprv, unsigned int offset)
1705		-{
1706		- if (!queue_virt_boundary(q))
1707		- return false;
1708		- return __bvec_gap_to_prev(q, bprv, offset);
1709		-}
1710		-
1711		-/*
1712		- * Check if the two bvecs from two bios can be merged to one segment.
1713		- * If yes, no need to check gap between the two bios since the 1st bio
1714		- * and the 1st bvec in the 2nd bio can be handled in one segment.
1715		- */
1716		-static inline bool bios_segs_mergeable(struct request_queue *q,
1717		- struct bio prev, struct bio_vec prev_last_bv,
1718		- struct bio_vec *next_first_bv)
1719		-{
1720		- if (!BIOVEC_PHYS_MERGEABLE(prev_last_bv, next_first_bv))
1721		- return false;
1722		- if (!BIOVEC_SEG_BOUNDARY(q, prev_last_bv, next_first_bv))
1723		- return false;
1724		- if (prev->bi_seg_back_size + next_first_bv->bv_len >
1725		- queue_max_segment_size(q))
1726		- return false;
1727		- return true;
1728		-}
1729		-
1730		-static inline bool bio_will_gap(struct request_queue *q,
1731		- struct request *prev_rq,
1732		- struct bio *prev,
1733		- struct bio *next)
1734		-{
1735		- if (bio_has_data(prev) && queue_virt_boundary(q)) {
1736		- struct bio_vec pb, nb;
1737		-
1738		- /*
1739		- * don't merge if the 1st bio starts with non-zero
1740		- * offset, otherwise it is quite difficult to respect
1741		- * sg gap limit. We work hard to merge a huge number of small
1742		- * single bios in case of mkfs.
1743		- */
1744		- if (prev_rq)
1745		- bio_get_first_bvec(prev_rq->bio, &pb);
1746		- else
1747		- bio_get_first_bvec(prev, &pb);
1748		- if (pb.bv_offset)
1749		- return true;
1750		-
1751		- /*
1752		- * We don't need to worry about the situation that the
1753		- * merged segment ends in unaligned virt boundary:
1754		- *
1755		- * - if 'pb' ends aligned, the merged segment ends aligned
1756		- * - if 'pb' ends unaligned, the next bio must include
1757		- * one single bvec of 'nb', otherwise the 'nb' can't
1758		- * merge with 'pb'
1759		- */
1760		- bio_get_last_bvec(prev, &pb);
1761		- bio_get_first_bvec(next, &nb);
1762		-
1763		- if (!bios_segs_mergeable(q, prev, &pb, &nb))
1764		- return __bvec_gap_to_prev(q, &pb, nb.bv_offset);
1765		- }
1766		-
1767		- return false;
1768		-}
1769		-
1770		-static inline bool req_gap_back_merge(struct request req, struct bio bio)
1771		-{
1772		- return bio_will_gap(req->q, req, req->biotail, bio);
1773		-}
1774		-
1775		-static inline bool req_gap_front_merge(struct request req, struct bio bio)
1776		-{
1777		- return bio_will_gap(req->q, NULL, bio, req->bio);
	1661	+ return 1 << bdev->bd_inode->i_blkbits;
1778	1662	}
1779	1663
1780	1664	int kblockd_schedule_work(struct work_struct *work);
1781		-int kblockd_schedule_work_on(int cpu, struct work_struct *work);
1782	1665	int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned long delay);
1783	1666
1784	1667	#define MODULE_ALIAS_BLOCKDEV(major,minor) \
..	..	@@ -1805,10 +1688,14 @@
1805	1688	};
1806	1689
1807	1690	typedef blk_status_t (integrity_processing_fn) (struct blk_integrity_iter *);
	1691	+typedef void (integrity_prepare_fn) (struct request *);
	1692	+typedef void (integrity_complete_fn) (struct request *, unsigned int);
1808	1693
1809	1694	struct blk_integrity_profile {
1810	1695	integrity_processing_fn *generate_fn;
1811	1696	integrity_processing_fn *verify_fn;
	1697	+ integrity_prepare_fn *prepare_fn;
	1698	+ integrity_complete_fn *complete_fn;
1812	1699	const char *name;
1813	1700	};
1814	1701
..	..	@@ -1818,10 +1705,6 @@
1818	1705	extern int blk_rq_map_integrity_sg(struct request_queue , struct bio ,
1819	1706	struct scatterlist *);
1820	1707	extern int blk_rq_count_integrity_sg(struct request_queue , struct bio );
1821		-extern bool blk_integrity_merge_rq(struct request_queue , struct request ,
1822		- struct request *);
1823		-extern bool blk_integrity_merge_bio(struct request_queue , struct request ,
1824		- struct bio *);
1825	1708
1826	1709	static inline struct blk_integrity blk_get_integrity(struct gendisk disk)
1827	1710	{
..	..	@@ -1839,6 +1722,12 @@
1839	1722	return blk_get_integrity(bdev->bd_disk);
1840	1723	}
1841	1724
	1725	+static inline bool
	1726	+blk_integrity_queue_supports_integrity(struct request_queue *q)
	1727	+{
	1728	+ return q->integrity.profile;
	1729	+}
	1730	+
1842	1731	static inline bool blk_integrity_rq(struct request *rq)
1843	1732	{
1844	1733	return rq->cmd_flags & REQ_INTEGRITY;
..	..	@@ -1851,29 +1740,9 @@
1851	1740	}
1852	1741
1853	1742	static inline unsigned short
1854		-queue_max_integrity_segments(struct request_queue *q)
	1743	+queue_max_integrity_segments(const struct request_queue *q)
1855	1744	{
1856	1745	return q->limits.max_integrity_segments;
1857		-}
1858		-
1859		-static inline bool integrity_req_gap_back_merge(struct request *req,
1860		- struct bio *next)
1861		-{
1862		- struct bio_integrity_payload *bip = bio_integrity(req->bio);
1863		- struct bio_integrity_payload *bip_next = bio_integrity(next);
1864		-
1865		- return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1],
1866		- bip_next->bip_vec[0].bv_offset);
1867		-}
1868		-
1869		-static inline bool integrity_req_gap_front_merge(struct request *req,
1870		- struct bio *bio)
1871		-{
1872		- struct bio_integrity_payload *bip = bio_integrity(bio);
1873		- struct bio_integrity_payload *bip_next = bio_integrity(req->bio);
1874		-
1875		- return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1],
1876		- bip_next->bip_vec[0].bv_offset);
1877	1746	}
1878	1747
1879	1748	/**
..	..	@@ -1896,6 +1765,17 @@
1896	1765	unsigned int sectors)
1897	1766	{
1898	1767	return bio_integrity_intervals(bi, sectors) * bi->tuple_size;
	1768	+}
	1769	+
	1770	+/*
	1771	+ * Return the first bvec that contains integrity data. Only drivers that are
	1772	+ * limited to a single integrity segment should use this helper.
	1773	+ */
	1774	+static inline struct bio_vec rq_integrity_vec(struct request rq)
	1775	+{
	1776	+ if (WARN_ON_ONCE(queue_max_integrity_segments(rq->q) > 1))
	1777	+ return NULL;
	1778	+ return rq->bio->bi_integrity->bip_vec;
1899	1779	}
1900	1780
1901	1781	#else /* CONFIG_BLK_DEV_INTEGRITY */
..	..	@@ -1928,6 +1808,11 @@
1928	1808	{
1929	1809	return NULL;
1930	1810	}
	1811	+static inline bool
	1812	+blk_integrity_queue_supports_integrity(struct request_queue *q)
	1813	+{
	1814	+ return false;
	1815	+}
1931	1816	static inline int blk_integrity_compare(struct gendisk a, struct gendisk b)
1932	1817	{
1933	1818	return 0;
..	..	@@ -1943,32 +1828,9 @@
1943	1828	unsigned int segs)
1944	1829	{
1945	1830	}
1946		-static inline unsigned short queue_max_integrity_segments(struct request_queue *q)
	1831	+static inline unsigned short queue_max_integrity_segments(const struct request_queue *q)
1947	1832	{
1948	1833	return 0;
1949		-}
1950		-static inline bool blk_integrity_merge_rq(struct request_queue *rq,
1951		- struct request *r1,
1952		- struct request *r2)
1953		-{
1954		- return true;
1955		-}
1956		-static inline bool blk_integrity_merge_bio(struct request_queue *rq,
1957		- struct request *r,
1958		- struct bio *b)
1959		-{
1960		- return true;
1961		-}
1962		-
1963		-static inline bool integrity_req_gap_back_merge(struct request *req,
1964		- struct bio *next)
1965		-{
1966		- return false;
1967		-}
1968		-static inline bool integrity_req_gap_front_merge(struct request *req,
1969		- struct bio *bio)
1970		-{
1971		- return false;
1972	1834	}
1973	1835
1974	1836	static inline unsigned int bio_integrity_intervals(struct blk_integrity *bi,
..	..	@@ -1983,9 +1845,34 @@
1983	1845	return 0;
1984	1846	}
1985	1847
	1848	+static inline struct bio_vec rq_integrity_vec(struct request rq)
	1849	+{
	1850	+ return NULL;
	1851	+}
	1852	+
1986	1853	#endif /* CONFIG_BLK_DEV_INTEGRITY */
1987	1854
	1855	+#ifdef CONFIG_BLK_INLINE_ENCRYPTION
	1856	+
	1857	+bool blk_ksm_register(struct blk_keyslot_manager ksm, struct request_queue q);
	1858	+
	1859	+void blk_ksm_unregister(struct request_queue *q);
	1860	+
	1861	+#else /* CONFIG_BLK_INLINE_ENCRYPTION */
	1862	+
	1863	+static inline bool blk_ksm_register(struct blk_keyslot_manager *ksm,
	1864	+ struct request_queue *q)
	1865	+{
	1866	+ return true;
	1867	+}
	1868	+
	1869	+static inline void blk_ksm_unregister(struct request_queue *q) { }
	1870	+
	1871	+#endif /* CONFIG_BLK_INLINE_ENCRYPTION */
	1872	+
	1873	+
1988	1874	struct block_device_operations {
	1875	+ blk_qc_t (submit_bio) (struct bio bio);
1989	1876	int (open) (struct block_device , fmode_t);
1990	1877	void (release) (struct gendisk , fmode_t);
1991	1878	int (rw_page)(struct block_device , sector_t, struct page *, unsigned int);
..	..	@@ -1993,16 +1880,28 @@
1993	1880	int (compat_ioctl) (struct block_device , fmode_t, unsigned, unsigned long);
1994	1881	unsigned int (check_events) (struct gendisk disk,
1995	1882	unsigned int clearing);
1996		- /* ->media_changed() is DEPRECATED, use ->check_events() instead */
1997		- int (media_changed) (struct gendisk );
1998	1883	void (unlock_native_capacity) (struct gendisk );
1999	1884	int (revalidate_disk) (struct gendisk );
2000	1885	int (getgeo)(struct block_device , struct hd_geometry *);
2001	1886	/* this callback is with swap_lock and sometimes page table lock held */
2002	1887	void (swap_slot_free_notify) (struct block_device , unsigned long);
	1888	+ int (report_zones)(struct gendisk , sector_t sector,
	1889	+ unsigned int nr_zones, report_zones_cb cb, void *data);
	1890	+ char (devnode)(struct gendisk disk, umode_t mode);
2003	1891	struct module *owner;
2004	1892	const struct pr_ops *pr_ops;
	1893	+
	1894	+ ANDROID_KABI_RESERVE(1);
	1895	+ ANDROID_KABI_RESERVE(2);
	1896	+ ANDROID_OEM_DATA(1);
2005	1897	};
	1898	+
	1899	+#ifdef CONFIG_COMPAT
	1900	+extern int blkdev_compat_ptr_ioctl(struct block_device *, fmode_t,
	1901	+ unsigned int, unsigned long);
	1902	+#else
	1903	+#define blkdev_compat_ptr_ioctl NULL
	1904	+#endif
2006	1905
2007	1906	extern int __blkdev_driver_ioctl(struct block_device *, fmode_t, unsigned int,
2008	1907	unsigned long);
..	..	@@ -2012,6 +1911,7 @@
2012	1911
2013	1912	#ifdef CONFIG_BLK_DEV_ZONED
2014	1913	bool blk_req_needs_zone_write_lock(struct request *rq);
	1914	+bool blk_req_zone_write_trylock(struct request *rq);
2015	1915	void __blk_req_zone_write_lock(struct request *rq);
2016	1916	void __blk_req_zone_write_unlock(struct request *rq);
2017	1917
..	..	@@ -2063,51 +1963,102 @@
2063	1963	}
2064	1964	#endif /* CONFIG_BLK_DEV_ZONED */
2065	1965
2066		-#else /* CONFIG_BLOCK */
	1966	+static inline void blk_wake_io_task(struct task_struct *waiter)
	1967	+{
	1968	+ /*
	1969	+ * If we're polling, the task itself is doing the completions. For
	1970	+ * that case, we don't need to signal a wakeup, it's enough to just
	1971	+ * mark us as RUNNING.
	1972	+ */
	1973	+ if (waiter == current)
	1974	+ __set_current_state(TASK_RUNNING);
	1975	+ else
	1976	+ wake_up_process(waiter);
	1977	+}
2067	1978
2068		-struct block_device;
	1979	+unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors,
	1980	+ unsigned int op);
	1981	+void disk_end_io_acct(struct gendisk *disk, unsigned int op,
	1982	+ unsigned long start_time);
2069	1983
2070		-/*
2071		- * stubs for when the block layer is configured out
	1984	+unsigned long part_start_io_acct(struct gendisk disk, struct hd_struct *part,
	1985	+ struct bio *bio);
	1986	+void part_end_io_acct(struct hd_struct part, struct bio bio,
	1987	+ unsigned long start_time);
	1988	+
	1989	+/**
	1990	+ * bio_start_io_acct - start I/O accounting for bio based drivers
	1991	+ * @bio: bio to start account for
	1992	+ *
	1993	+ * Returns the start time that should be passed back to bio_end_io_acct().
2072	1994	*/
2073		-#define buffer_heads_over_limit 0
2074		-
2075		-static inline long nr_blockdev_pages(void)
	1995	+static inline unsigned long bio_start_io_acct(struct bio *bio)
2076	1996	{
2077		- return 0;
	1997	+ return disk_start_io_acct(bio->bi_disk, bio_sectors(bio), bio_op(bio));
2078	1998	}
2079	1999
2080		-struct blk_plug {
2081		-};
2082		-
2083		-static inline void blk_start_plug(struct blk_plug *plug)
	2000	+/**
	2001	+ * bio_end_io_acct - end I/O accounting for bio based drivers
	2002	+ * @bio: bio to end account for
	2003	+ * @start: start time returned by bio_start_io_acct()
	2004	+ */
	2005	+static inline void bio_end_io_acct(struct bio *bio, unsigned long start_time)
2084	2006	{
	2007	+ return disk_end_io_acct(bio->bi_disk, bio_op(bio), start_time);
2085	2008	}
2086	2009
2087		-static inline void blk_finish_plug(struct blk_plug *plug)
2088		-{
2089		-}
	2010	+int bdev_read_only(struct block_device *bdev);
	2011	+int set_blocksize(struct block_device *bdev, int size);
2090	2012
2091		-static inline void blk_flush_plug(struct task_struct *task)
2092		-{
2093		-}
	2013	+const char bdevname(struct block_device bdev, char *buffer);
	2014	+struct block_device lookup_bdev(const char );
2094	2015
2095		-static inline void blk_schedule_flush_plug(struct task_struct *task)
2096		-{
2097		-}
	2016	+void blkdev_show(struct seq_file *seqf, off_t offset);
2098	2017
2099		-
2100		-static inline bool blk_needs_flush_plug(struct task_struct *tsk)
2101		-{
2102		- return false;
2103		-}
2104		-
2105		-static inline int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask,
2106		- sector_t *error_sector)
2107		-{
2108		- return 0;
2109		-}
2110		-
2111		-#endif /* CONFIG_BLOCK */
2112		-
	2018	+#define BDEVNAME_SIZE 32 /* Largest string for a blockdev identifier */
	2019	+#define BDEVT_SIZE 10 /* Largest string for MAJ:MIN for blkdev */
	2020	+#ifdef CONFIG_BLOCK
	2021	+#define BLKDEV_MAJOR_MAX 512
	2022	+#else
	2023	+#define BLKDEV_MAJOR_MAX 0
2113	2024	#endif
	2025	+
	2026	+struct block_device blkdev_get_by_path(const char path, fmode_t mode,
	2027	+ void *holder);
	2028	+struct block_device blkdev_get_by_dev(dev_t dev, fmode_t mode, void holder);
	2029	+int bd_prepare_to_claim(struct block_device bdev, struct block_device whole,
	2030	+ void *holder);
	2031	+void bd_abort_claiming(struct block_device bdev, struct block_device whole,
	2032	+ void *holder);
	2033	+void blkdev_put(struct block_device *bdev, fmode_t mode);
	2034	+
	2035	+struct block_device I_BDEV(struct inode inode);
	2036	+struct block_device bdget_part(struct hd_struct part);
	2037	+struct block_device bdgrab(struct block_device bdev);
	2038	+void bdput(struct block_device *);
	2039	+
	2040	+#ifdef CONFIG_BLOCK
	2041	+void invalidate_bdev(struct block_device *bdev);
	2042	+int truncate_bdev_range(struct block_device *bdev, fmode_t mode, loff_t lstart,
	2043	+ loff_t lend);
	2044	+int sync_blockdev(struct block_device *bdev);
	2045	+#else
	2046	+static inline void invalidate_bdev(struct block_device *bdev)
	2047	+{
	2048	+}
	2049	+static inline int truncate_bdev_range(struct block_device *bdev, fmode_t mode,
	2050	+ loff_t lstart, loff_t lend)
	2051	+{
	2052	+ return 0;
	2053	+}
	2054	+static inline int sync_blockdev(struct block_device *bdev)
	2055	+{
	2056	+ return 0;
	2057	+}
	2058	+#endif
	2059	+int fsync_bdev(struct block_device *bdev);
	2060	+
	2061	+int freeze_bdev(struct block_device *bdev);
	2062	+int thaw_bdev(struct block_device *bdev);
	2063	+
	2064	+#endif /* _LINUX_BLKDEV_H */