forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-08 01573e231f18eb2d99162747186f59511f56b64d 
 kernel/block/blk-mq.h
....@@ -7,17 +7,23 @@
77 
88 struct blk_mq_tag_set;
99 
10
+struct blk_mq_ctxs {
11
+	struct kobject kobj;
12
+	struct blk_mq_ctx __percpu	*queue_ctx;
13
+};
14
+
1015 /**
1116  * struct blk_mq_ctx - State for a software queue facing the submitting CPUs
1217  */
1318 struct blk_mq_ctx {
1419 	struct {
1520 		spinlock_t		lock;
16
-		struct list_head	rq_list;
17
-	}  ____cacheline_aligned_in_smp;
21
+		struct list_head	rq_lists[HCTX_MAX_TYPES];
22
+	} ____cacheline_aligned_in_smp;
1823 
1924 	unsigned int		cpu;
20
-	unsigned int		index_hw;
25
+	unsigned short		index_hw[HCTX_MAX_TYPES];
26
+	struct blk_mq_hw_ctx 	*hctxs[HCTX_MAX_TYPES];
2127 
2228 	/* incremented at dispatch time */
2329 	unsigned long		rq_dispatched[2];
....@@ -27,29 +33,35 @@
2733 	unsigned long		____cacheline_aligned_in_smp rq_completed[2];
2834 
2935 	struct request_queue	*queue;
36
+	struct blk_mq_ctxs      *ctxs;
3037 	struct kobject		kobj;
38
+
39
+	ANDROID_OEM_DATA_ARRAY(1, 2);
3140 } ____cacheline_aligned_in_smp;
3241 
33
-void blk_mq_freeze_queue(struct request_queue *q);
3442 void blk_mq_exit_queue(struct request_queue *q);
3543 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr);
3644 void blk_mq_wake_waiters(struct request_queue *q);
37
-bool blk_mq_dispatch_rq_list(struct request_queue *, struct list_head *, bool);
45
+bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *,
46
+			     unsigned int);
47
+void blk_mq_add_to_requeue_list(struct request *rq, bool at_head,
48
+				bool kick_requeue_list);
3849 void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list);
39
-bool blk_mq_get_driver_tag(struct request *rq);
4050 struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx,
4151 					struct blk_mq_ctx *start);
52
+void blk_mq_put_rq_ref(struct request *rq);
4253 
4354 /*
4455  * Internal helpers for allocating/freeing the request map
4556  */
4657 void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
4758 		     unsigned int hctx_idx);
48
-void blk_mq_free_rq_map(struct blk_mq_tags *tags);
59
+void blk_mq_free_rq_map(struct blk_mq_tags *tags, unsigned int flags);
4960 struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
5061 					unsigned int hctx_idx,
5162 					unsigned int nr_tags,
52
-					unsigned int reserved_tags);
63
+					unsigned int reserved_tags,
64
+					unsigned int flags);
5365 int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
5466 		     unsigned int hctx_idx, unsigned int depth);
5567 
....@@ -58,24 +70,55 @@
5870  */
5971 void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
6072 				bool at_head);
61
-void blk_mq_request_bypass_insert(struct request *rq, bool run_queue);
73
+void blk_mq_request_bypass_insert(struct request *rq, bool at_head,
74
+				  bool run_queue);
6275 void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
6376 				struct list_head *list);
6477 
6578 /* Used by blk_insert_cloned_request() to issue request directly */
66
-blk_status_t blk_mq_request_issue_directly(struct request *rq);
79
+blk_status_t blk_mq_request_issue_directly(struct request *rq, bool last);
6780 void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
6881 				    struct list_head *list);
6982 
7083 /*
7184  * CPU -> queue mappings
7285  */
73
-extern int blk_mq_hw_queue_to_node(unsigned int *map, unsigned int);
86
+extern int blk_mq_hw_queue_to_node(struct blk_mq_queue_map *qmap, unsigned int);
7487 
75
-static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q,
76
-		int cpu)
88
+/*
89
+ * blk_mq_map_queue_type() - map (hctx_type,cpu) to hardware queue
90
+ * @q: request queue
91
+ * @type: the hctx type index
92
+ * @cpu: CPU
93
+ */
94
+static inline struct blk_mq_hw_ctx *blk_mq_map_queue_type(struct request_queue *q,
95
+							  enum hctx_type type,
96
+							  unsigned int cpu)
7797 {
78
-	return q->queue_hw_ctx[q->mq_map[cpu]];
98
+	return q->queue_hw_ctx[q->tag_set->map[type].mq_map[cpu]];
99
+}
100
+
101
+/*
102
+ * blk_mq_map_queue() - map (cmd_flags,type) to hardware queue
103
+ * @q: request queue
104
+ * @flags: request command flags
105
+ * @cpu: cpu ctx
106
+ */
107
+static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q,
108
+						     unsigned int flags,
109
+						     struct blk_mq_ctx *ctx)
110
+{
111
+	enum hctx_type type = HCTX_TYPE_DEFAULT;
112
+
113
+	/*
114
+	 * The caller ensure that if REQ_HIPRI, poll must be enabled.
115
+	 */
116
+	if (flags & REQ_HIPRI)
117
+		type = HCTX_TYPE_POLL;
118
+	else if ((flags & REQ_OP_MASK) == REQ_OP_READ)
119
+		type = HCTX_TYPE_READ;
120
+	
121
+	return ctx->hctxs[type];
79122 }
80123 
81124 /*
....@@ -89,15 +132,6 @@
89132 extern void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx);
90133 
91134 void blk_mq_release(struct request_queue *q);
92
-
93
-/**
94
- * blk_mq_rq_state() - read the current MQ_RQ_* state of a request
95
- * @rq: target request.
96
- */
97
-static inline enum mq_rq_state blk_mq_rq_state(struct request *rq)
98
-{
99
-	return READ_ONCE(rq->state);
100
-}
101135 
102136 static inline struct blk_mq_ctx *__blk_mq_get_ctx(struct request_queue *q,
103137 					   unsigned int cpu)
....@@ -113,12 +147,7 @@
113147  */
114148 static inline struct blk_mq_ctx *blk_mq_get_ctx(struct request_queue *q)
115149 {
116
-	return __blk_mq_get_ctx(q, get_cpu());
117
-}
118
-
119
-static inline void blk_mq_put_ctx(struct blk_mq_ctx *ctx)
120
-{
121
-	put_cpu();
150
+	return __blk_mq_get_ctx(q, raw_smp_processor_id());
122151 }
123152 
124153 struct blk_mq_alloc_data {
....@@ -126,15 +155,21 @@
126155 	struct request_queue *q;
127156 	blk_mq_req_flags_t flags;
128157 	unsigned int shallow_depth;
158
+	unsigned int cmd_flags;
129159 
130160 	/* input & output parameter */
131161 	struct blk_mq_ctx *ctx;
132162 	struct blk_mq_hw_ctx *hctx;
133163 };
134164 
165
+static inline bool blk_mq_is_sbitmap_shared(unsigned int flags)
166
+{
167
+	return flags & BLK_MQ_F_TAG_HCTX_SHARED;
168
+}
169
+
135170 static inline struct blk_mq_tags *blk_mq_tags_from_data(struct blk_mq_alloc_data *data)
136171 {
137
-	if (data->flags & BLK_MQ_REQ_INTERNAL)
172
+	if (data->q->elevator)
138173 		return data->hctx->sched_tags;
139174 
140175 	return data->hctx->tags;
....@@ -150,66 +185,145 @@
150185 	return hctx->nr_ctx && hctx->tags;
151186 }
152187 
153
-void blk_mq_in_flight(struct request_queue *q, struct hd_struct *part,
154
-		      unsigned int inflight[2]);
188
+unsigned int blk_mq_in_flight(struct request_queue *q, struct hd_struct *part);
155189 void blk_mq_in_flight_rw(struct request_queue *q, struct hd_struct *part,
156190 			 unsigned int inflight[2]);
157191 
158
-static inline void blk_mq_put_dispatch_budget(struct blk_mq_hw_ctx *hctx)
192
+static inline void blk_mq_put_dispatch_budget(struct request_queue *q)
159193 {
160
-	struct request_queue *q = hctx->queue;
161
-
162194 	if (q->mq_ops->put_budget)
163
-		q->mq_ops->put_budget(hctx);
195
+		q->mq_ops->put_budget(q);
164196 }
165197 
166
-static inline bool blk_mq_get_dispatch_budget(struct blk_mq_hw_ctx *hctx)
198
+static inline bool blk_mq_get_dispatch_budget(struct request_queue *q)
167199 {
168
-	struct request_queue *q = hctx->queue;
169
-
170200 	if (q->mq_ops->get_budget)
171
-		return q->mq_ops->get_budget(hctx);
201
+		return q->mq_ops->get_budget(q);
172202 	return true;
173203 }
174204 
205
+static inline void __blk_mq_inc_active_requests(struct blk_mq_hw_ctx *hctx)
206
+{
207
+	if (blk_mq_is_sbitmap_shared(hctx->flags))
208
+		atomic_inc(&hctx->queue->nr_active_requests_shared_sbitmap);
209
+	else
210
+		atomic_inc(&hctx->nr_active);
211
+}
212
+
213
+static inline void __blk_mq_dec_active_requests(struct blk_mq_hw_ctx *hctx)
214
+{
215
+	if (blk_mq_is_sbitmap_shared(hctx->flags))
216
+		atomic_dec(&hctx->queue->nr_active_requests_shared_sbitmap);
217
+	else
218
+		atomic_dec(&hctx->nr_active);
219
+}
220
+
221
+static inline int __blk_mq_active_requests(struct blk_mq_hw_ctx *hctx)
222
+{
223
+	if (blk_mq_is_sbitmap_shared(hctx->flags))
224
+		return atomic_read(&hctx->queue->nr_active_requests_shared_sbitmap);
225
+	return atomic_read(&hctx->nr_active);
226
+}
175227 static inline void __blk_mq_put_driver_tag(struct blk_mq_hw_ctx *hctx,
176228 					   struct request *rq)
177229 {
178
-	blk_mq_put_tag(hctx, hctx->tags, rq->mq_ctx, rq->tag);
179
-	rq->tag = -1;
230
+	blk_mq_put_tag(hctx->tags, rq->mq_ctx, rq->tag);
231
+	rq->tag = BLK_MQ_NO_TAG;
180232 
181233 	if (rq->rq_flags & RQF_MQ_INFLIGHT) {
182234 		rq->rq_flags &= ~RQF_MQ_INFLIGHT;
183
-		atomic_dec(&hctx->nr_active);
235
+		__blk_mq_dec_active_requests(hctx);
184236 	}
185
-}
186
-
187
-static inline void blk_mq_put_driver_tag_hctx(struct blk_mq_hw_ctx *hctx,
188
-				       struct request *rq)
189
-{
190
-	if (rq->tag == -1 || rq->internal_tag == -1)
191
-		return;
192
-
193
-	__blk_mq_put_driver_tag(hctx, rq);
194237 }
195238 
196239 static inline void blk_mq_put_driver_tag(struct request *rq)
197240 {
198
-	struct blk_mq_hw_ctx *hctx;
199
-
200
-	if (rq->tag == -1 || rq->internal_tag == -1)
241
+	if (rq->tag == BLK_MQ_NO_TAG || rq->internal_tag == BLK_MQ_NO_TAG)
201242 		return;
202243 
203
-	hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu);
204
-	__blk_mq_put_driver_tag(hctx, rq);
244
+	__blk_mq_put_driver_tag(rq->mq_hctx, rq);
205245 }
206246 
207
-static inline void blk_mq_clear_mq_map(struct blk_mq_tag_set *set)
247
+static inline void blk_mq_clear_mq_map(struct blk_mq_queue_map *qmap)
208248 {
209249 	int cpu;
210250 
211251 	for_each_possible_cpu(cpu)
212
-		set->mq_map[cpu] = 0;
252
+		qmap->mq_map[cpu] = 0;
213253 }
214254 
255
+/*
256
+ * blk_mq_plug() - Get caller context plug
257
+ * @q: request queue
258
+ * @bio : the bio being submitted by the caller context
259
+ *
260
+ * Plugging, by design, may delay the insertion of BIOs into the elevator in
261
+ * order to increase BIO merging opportunities. This however can cause BIO
262
+ * insertion order to change from the order in which submit_bio() is being
263
+ * executed in the case of multiple contexts concurrently issuing BIOs to a
264
+ * device, even if these context are synchronized to tightly control BIO issuing
265
+ * order. While this is not a problem with regular block devices, this ordering
266
+ * change can cause write BIO failures with zoned block devices as these
267
+ * require sequential write patterns to zones. Prevent this from happening by
268
+ * ignoring the plug state of a BIO issuing context if the target request queue
269
+ * is for a zoned block device and the BIO to plug is a write operation.
270
+ *
271
+ * Return current->plug if the bio can be plugged and NULL otherwise
272
+ */
273
+static inline struct blk_plug *blk_mq_plug(struct request_queue *q,
274
+					   struct bio *bio)
275
+{
276
+	/*
277
+	 * For regular block devices or read operations, use the context plug
278
+	 * which may be NULL if blk_start_plug() was not executed.
279
+	 */
280
+	if (!blk_queue_is_zoned(q) || !op_is_write(bio_op(bio)))
281
+		return current->plug;
282
+
283
+	/* Zoned block device write operation case: do not plug the BIO */
284
+	return NULL;
285
+}
286
+
287
+/*
288
+ * For shared tag users, we track the number of currently active users
289
+ * and attempt to provide a fair share of the tag depth for each of them.
290
+ */
291
+static inline bool hctx_may_queue(struct blk_mq_hw_ctx *hctx,
292
+				  struct sbitmap_queue *bt)
293
+{
294
+	unsigned int depth, users;
295
+
296
+	if (!hctx || !(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED))
297
+		return true;
298
+
299
+	/*
300
+	 * Don't try dividing an ant
301
+	 */
302
+	if (bt->sb.depth == 1)
303
+		return true;
304
+
305
+	if (blk_mq_is_sbitmap_shared(hctx->flags)) {
306
+		struct request_queue *q = hctx->queue;
307
+		struct blk_mq_tag_set *set = q->tag_set;
308
+
309
+		if (!test_bit(QUEUE_FLAG_HCTX_ACTIVE, &q->queue_flags))
310
+			return true;
311
+		users = atomic_read(&set->active_queues_shared_sbitmap);
312
+	} else {
313
+		if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
314
+			return true;
315
+		users = atomic_read(&hctx->tags->active_queues);
316
+	}
317
+
318
+	if (!users)
319
+		return true;
320
+
321
+	/*
322
+	 * Allow at least some tags
323
+	 */
324
+	depth = max((bt->sb.depth + users - 1) / users, 4U);
325
+	return __blk_mq_active_requests(hctx) < depth;
326
+}
327
+
328
+
215329 #endif