forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-11 d2ccde1c8e90d38cee87a1b0309ad2827f3fd30d 
 kernel/drivers/infiniband/hw/hfi1/iowait.h
....@@ -1,7 +1,7 @@
11 #ifndef _HFI1_IOWAIT_H
22 #define _HFI1_IOWAIT_H
33 /*
4
- * Copyright(c) 2015, 2016 Intel Corporation.
4
+ * Copyright(c) 2015 - 2018 Intel Corporation.
55  *
66  * This file is provided under a dual BSD/GPLv2 license.  When using or
77  * redistributing this file, you may do so under either license.
....@@ -49,6 +49,7 @@
4949 
5050 #include <linux/list.h>
5151 #include <linux/workqueue.h>
52
+#include <linux/wait.h>
5253 #include <linux/sched.h>
5354 
5455 #include "sdma_txreq.h"
....@@ -59,16 +60,48 @@
5960  */
6061 typedef void (*restart_t)(struct work_struct *work);
6162 
63
+#define IOWAIT_PENDING_IB  0x0
64
+#define IOWAIT_PENDING_TID 0x1
65
+
66
+/*
67
+ * A QP can have multiple Send Engines (SEs).
68
+ *
69
+ * The current use case is for supporting a TID RDMA
70
+ * packet build/xmit mechanism independent from verbs.
71
+ */
72
+#define IOWAIT_SES 2
73
+#define IOWAIT_IB_SE 0
74
+#define IOWAIT_TID_SE 1
75
+
6276 struct sdma_txreq;
6377 struct sdma_engine;
6478 /**
65
- * struct iowait - linkage for delayed progress/waiting
79
+ * @iowork: the work struct
80
+ * @tx_head: list of prebuilt packets
81
+ * @iow: the parent iowait structure
82
+ *
83
+ * This structure is the work item (process) specific
84
+ * details associated with the each of the two SEs of the
85
+ * QP.
86
+ *
87
+ * The workstruct and the queued TXs are unique to each
88
+ * SE.
89
+ */
90
+struct iowait;
91
+struct iowait_work {
92
+	struct work_struct iowork;
93
+	struct list_head tx_head;
94
+	struct iowait *iow;
95
+};
96
+
97
+/**
6698  * @list: used to add/insert into QP/PQ wait lists
67
- * @lock: uses to record the list head lock
6899  * @tx_head: overflow list of sdma_txreq's
69100  * @sleep: no space callback
70101  * @wakeup: space callback wakeup
71102  * @sdma_drained: sdma count drained
103
+ * @init_priority: callback to manipulate priority
104
+ * @lock: lock protected head of wait queue
72105  * @iowork: workqueue overhead
73106  * @wait_dma: wait for sdma_busy == 0
74107  * @wait_pio: wait for pio_busy == 0
....@@ -76,6 +109,8 @@
76109  * @count: total number of descriptors in tx_head'ed list
77110  * @tx_limit: limit for overflow queuing
78111  * @tx_count: number of tx entry's in tx_head'ed list
112
+ * @flags: wait flags (one per QP)
113
+ * @wait: SE array for multiple legs
79114  *
80115  * This is to be embedded in user's state structure
81116  * (QP or PQ).
....@@ -86,10 +121,13 @@
86121  * are callbacks for the ULP to implement
87122  * what ever queuing/dequeuing of
88123  * the embedded iowait and its containing struct
89
- * when a resource shortage like SDMA ring space is seen.
124
+ * when a resource shortage like SDMA ring space
125
+ * or PIO credit space is seen.
90126  *
91127  * Both potentially have locks help
92
- * so sleeping is not allowed.
128
+ * so sleeping is not allowed and it is not
129
+ * supported to submit txreqs from the wakeup
130
+ * call directly because of lock conflicts.
93131  *
94132  * The wait_dma member along with the iow
95133  *
....@@ -98,21 +136,19 @@
98136  * Waiters explicity know that, but the destroy
99137  * code that unwaits QPs does not.
100138  */
101
-
102139 struct iowait {
103140 	struct list_head list;
104
-	struct list_head tx_head;
105141 	int (*sleep)(
106142 		struct sdma_engine *sde,
107
-		struct iowait *wait,
143
+		struct iowait_work *wait,
108144 		struct sdma_txreq *tx,
109145 		uint seq,
110146 		bool pkts_sent
111147 		);
112148 	void (*wakeup)(struct iowait *wait, int reason);
113149 	void (*sdma_drained)(struct iowait *wait);
150
+	void (*init_priority)(struct iowait *wait);
114151 	seqlock_t *lock;
115
-	struct work_struct iowork;
116152 	wait_queue_head_t wait_dma;
117153 	wait_queue_head_t wait_pio;
118154 	atomic_t sdma_busy;
....@@ -121,63 +157,51 @@
121157 	u32 tx_limit;
122158 	u32 tx_count;
123159 	u8 starved_cnt;
160
+	u8 priority;
161
+	unsigned long flags;
162
+	struct iowait_work wait[IOWAIT_SES];
124163 };
125164 
126165 #define SDMA_AVAIL_REASON 0
127166 
128
-/**
129
- * iowait_init() - initialize wait structure
130
- * @wait: wait struct to initialize
131
- * @tx_limit: limit for overflow queuing
132
- * @func: restart function for workqueue
133
- * @sleep: sleep function for no space
134
- * @resume: wakeup function for no space
135
- *
136
- * This function initializes the iowait
137
- * structure embedded in the QP or PQ.
138
- *
139
- */
167
+void iowait_set_flag(struct iowait *wait, u32 flag);
168
+bool iowait_flag_set(struct iowait *wait, u32 flag);
169
+void iowait_clear_flag(struct iowait *wait, u32 flag);
140170 
141
-static inline void iowait_init(
142
-	struct iowait *wait,
143
-	u32 tx_limit,
144
-	void (*func)(struct work_struct *work),
145
-	int (*sleep)(
146
-		struct sdma_engine *sde,
147
-		struct iowait *wait,
148
-		struct sdma_txreq *tx,
149
-		uint seq,
150
-		bool pkts_sent),
151
-	void (*wakeup)(struct iowait *wait, int reason),
152
-	void (*sdma_drained)(struct iowait *wait))
153
-{
154
-	wait->count = 0;
155
-	wait->lock = NULL;
156
-	INIT_LIST_HEAD(&wait->list);
157
-	INIT_LIST_HEAD(&wait->tx_head);
158
-	INIT_WORK(&wait->iowork, func);
159
-	init_waitqueue_head(&wait->wait_dma);
160
-	init_waitqueue_head(&wait->wait_pio);
161
-	atomic_set(&wait->sdma_busy, 0);
162
-	atomic_set(&wait->pio_busy, 0);
163
-	wait->tx_limit = tx_limit;
164
-	wait->sleep = sleep;
165
-	wait->wakeup = wakeup;
166
-	wait->sdma_drained = sdma_drained;
167
-}
171
+void iowait_init(struct iowait *wait, u32 tx_limit,
172
+		 void (*func)(struct work_struct *work),
173
+		 void (*tidfunc)(struct work_struct *work),
174
+		 int (*sleep)(struct sdma_engine *sde,
175
+			      struct iowait_work *wait,
176
+			      struct sdma_txreq *tx,
177
+			      uint seq,
178
+			      bool pkts_sent),
179
+		 void (*wakeup)(struct iowait *wait, int reason),
180
+		 void (*sdma_drained)(struct iowait *wait),
181
+		 void (*init_priority)(struct iowait *wait));
168182 
169183 /**
170
- * iowait_schedule() - initialize wait structure
184
+ * iowait_schedule() - schedule the default send engine work
171185  * @wait: wait struct to schedule
172186  * @wq: workqueue for schedule
173187  * @cpu: cpu
174188  */
175
-static inline void iowait_schedule(
176
-	struct iowait *wait,
177
-	struct workqueue_struct *wq,
178
-	int cpu)
189
+static inline bool iowait_schedule(struct iowait *wait,
190
+				   struct workqueue_struct *wq, int cpu)
179191 {
180
-	queue_work_on(cpu, wq, &wait->iowork);
192
+	return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_IB_SE].iowork);
193
+}
194
+
195
+/**
196
+ * iowait_tid_schedule - schedule the tid SE
197
+ * @wait: the iowait structure
198
+ * @wq: the work queue
199
+ * @cpu: the cpu
200
+ */
201
+static inline bool iowait_tid_schedule(struct iowait *wait,
202
+				       struct workqueue_struct *wq, int cpu)
203
+{
204
+	return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_TID_SE].iowork);
181205 }
182206 
183207 /**
....@@ -228,6 +252,8 @@
228252  */
229253 static inline int iowait_sdma_dec(struct iowait *wait)
230254 {
255
+	if (!wait)
256
+		return 0;
231257 	return atomic_dec_and_test(&wait->sdma_busy);
232258 }
233259 
....@@ -267,11 +293,13 @@
267293 }
268294 
269295 /**
270
- * iowait_sdma_dec - note pio complete
296
+ * iowait_pio_dec - note pio complete
271297  * @wait: iowait structure
272298  */
273299 static inline int iowait_pio_dec(struct iowait *wait)
274300 {
301
+	if (!wait)
302
+		return 0;
275303 	return atomic_dec_and_test(&wait->pio_busy);
276304 }
277305 
....@@ -293,9 +321,9 @@
293321 /**
294322  * iowait_get_txhead() - get packet off of iowait list
295323  *
296
- * @wait wait struture
324
+ * @wait iowait_work struture
297325  */
298
-static inline struct sdma_txreq *iowait_get_txhead(struct iowait *wait)
326
+static inline struct sdma_txreq *iowait_get_txhead(struct iowait_work *wait)
299327 {
300328 	struct sdma_txreq *tx = NULL;
301329 
....@@ -309,6 +337,61 @@
309337 	return tx;
310338 }
311339 
340
+static inline u16 iowait_get_desc(struct iowait_work *w)
341
+{
342
+	u16 num_desc = 0;
343
+	struct sdma_txreq *tx = NULL;
344
+
345
+	if (!list_empty(&w->tx_head)) {
346
+		tx = list_first_entry(&w->tx_head, struct sdma_txreq,
347
+				      list);
348
+		num_desc = tx->num_desc;
349
+		if (tx->flags & SDMA_TXREQ_F_VIP)
350
+			w->iow->priority++;
351
+	}
352
+	return num_desc;
353
+}
354
+
355
+static inline u32 iowait_get_all_desc(struct iowait *w)
356
+{
357
+	u32 num_desc = 0;
358
+
359
+	num_desc = iowait_get_desc(&w->wait[IOWAIT_IB_SE]);
360
+	num_desc += iowait_get_desc(&w->wait[IOWAIT_TID_SE]);
361
+	return num_desc;
362
+}
363
+
364
+static inline void iowait_update_priority(struct iowait_work *w)
365
+{
366
+	struct sdma_txreq *tx = NULL;
367
+
368
+	if (!list_empty(&w->tx_head)) {
369
+		tx = list_first_entry(&w->tx_head, struct sdma_txreq,
370
+				      list);
371
+		if (tx->flags & SDMA_TXREQ_F_VIP)
372
+			w->iow->priority++;
373
+	}
374
+}
375
+
376
+static inline void iowait_update_all_priority(struct iowait *w)
377
+{
378
+	iowait_update_priority(&w->wait[IOWAIT_IB_SE]);
379
+	iowait_update_priority(&w->wait[IOWAIT_TID_SE]);
380
+}
381
+
382
+static inline void iowait_init_priority(struct iowait *w)
383
+{
384
+	w->priority = 0;
385
+	if (w->init_priority)
386
+		w->init_priority(w);
387
+}
388
+
389
+static inline void iowait_get_priority(struct iowait *w)
390
+{
391
+	iowait_init_priority(w);
392
+	iowait_update_all_priority(w);
393
+}
394
+
312395 /**
313396  * iowait_queue - Put the iowait on a wait queue
314397  * @pkts_sent: have some packets been sent before queuing?
....@@ -316,7 +399,7 @@
316399  * @wait_head: the wait queue
317400  *
318401  * This function is called to insert an iowait struct into a
319
- * wait queue after a resource (eg, sdma decriptor or pio
402
+ * wait queue after a resource (eg, sdma descriptor or pio
320403  * buffer) is run out.
321404  */
322405 static inline void iowait_queue(bool pkts_sent, struct iowait *w,
....@@ -325,14 +408,18 @@
325408 	/*
326409 	 * To play fair, insert the iowait at the tail of the wait queue if it
327410 	 * has already sent some packets; Otherwise, put it at the head.
411
+	 * However, if it has priority packets to send, also put it at the
412
+	 * head.
328413 	 */
329
-	if (pkts_sent) {
330
-		list_add_tail(&w->list, wait_head);
414
+	if (pkts_sent)
331415 		w->starved_cnt = 0;
332
-	} else {
333
-		list_add(&w->list, wait_head);
416
+	else
334417 		w->starved_cnt++;
335
-	}
418
+
419
+	if (w->priority > 0 || !pkts_sent)
420
+		list_add(&w->list, wait_head);
421
+	else
422
+		list_add_tail(&w->list, wait_head);
336423 }
337424 
338425 /**
....@@ -349,35 +436,63 @@
349436 		w->starved_cnt = 0;
350437 }
351438 
352
-/**
353
- * iowait_starve_find_max - Find the maximum of the starve count
354
- * @w: the iowait struct
355
- * @max: a variable containing the max starve count
356
- * @idx: the index of the current iowait in an array
357
- * @max_idx: a variable containing the array index for the
358
- *         iowait entry that has the max starve count
359
- *
360
- * This function is called to compare the starve count of a
361
- * given iowait with the given max starve count. The max starve
362
- * count and the index will be updated if the iowait's start
363
- * count is larger.
364
- */
365
-static inline void iowait_starve_find_max(struct iowait *w, u8 *max,
366
-					  uint idx, uint *max_idx)
367
-{
368
-	if (w->starved_cnt > *max) {
369
-		*max = w->starved_cnt;
370
-		*max_idx = idx;
371
-	}
372
-}
439
+/* Update the top priority index */
440
+uint iowait_priority_update_top(struct iowait *w,
441
+				struct iowait *top,
442
+				uint idx, uint top_idx);
373443 
374444 /**
375
- * iowait_packet_queued() - determine if a packet is already built
376
- * @wait: the wait structure
445
+ * iowait_packet_queued() - determine if a packet is queued
446
+ * @wait: the iowait_work structure
377447  */
378
-static inline bool iowait_packet_queued(struct iowait *wait)
448
+static inline bool iowait_packet_queued(struct iowait_work *wait)
379449 {
380450 	return !list_empty(&wait->tx_head);
381451 }
382452 
453
+/**
454
+ * inc_wait_count - increment wait counts
455
+ * @w: the log work struct
456
+ * @n: the count
457
+ */
458
+static inline void iowait_inc_wait_count(struct iowait_work *w, u16 n)
459
+{
460
+	if (!w)
461
+		return;
462
+	w->iow->tx_count++;
463
+	w->iow->count += n;
464
+}
465
+
466
+/**
467
+ * iowait_get_tid_work - return iowait_work for tid SE
468
+ * @w: the iowait struct
469
+ */
470
+static inline struct iowait_work *iowait_get_tid_work(struct iowait *w)
471
+{
472
+	return &w->wait[IOWAIT_TID_SE];
473
+}
474
+
475
+/**
476
+ * iowait_get_ib_work - return iowait_work for ib SE
477
+ * @w: the iowait struct
478
+ */
479
+static inline struct iowait_work *iowait_get_ib_work(struct iowait *w)
480
+{
481
+	return &w->wait[IOWAIT_IB_SE];
482
+}
483
+
484
+/**
485
+ * iowait_ioww_to_iow - return iowait given iowait_work
486
+ * @w: the iowait_work struct
487
+ */
488
+static inline struct iowait *iowait_ioww_to_iow(struct iowait_work *w)
489
+{
490
+	if (likely(w))
491
+		return w->iow;
492
+	return NULL;
493
+}
494
+
495
+void iowait_cancel_work(struct iowait *w);
496
+int iowait_set_work_flag(struct iowait_work *w);
497
+
383498 #endif