forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-11 d2ccde1c8e90d38cee87a1b0309ad2827f3fd30d 
 kernel/drivers/infiniband/hw/hfi1/verbs.h
....@@ -71,6 +71,8 @@
7171 struct hfi1_packet;
7272 
7373 #include "iowait.h"
74
+#include "tid_rdma.h"
75
+#include "opfn.h"
7476 
7577 #define HFI1_MAX_RDMA_ATOMIC     16
7678 
....@@ -105,9 +107,9 @@
105107 	HFI1_HAS_GRH = (1 << 0),
106108 };
107109 
108
-#define LRH_16B_BYTES (FIELD_SIZEOF(struct hfi1_16b_header, lrh))
110
+#define LRH_16B_BYTES (sizeof_field(struct hfi1_16b_header, lrh))
109111 #define LRH_16B_DWORDS (LRH_16B_BYTES / sizeof(u32))
110
-#define LRH_9B_BYTES (FIELD_SIZEOF(struct ib_header, lrh))
112
+#define LRH_9B_BYTES (sizeof_field(struct ib_header, lrh))
111113 #define LRH_9B_DWORDS (LRH_9B_BYTES / sizeof(u32))
112114 
113115 /* 24Bits for qpn, upper 8Bits reserved */
....@@ -156,21 +158,83 @@
156158 	struct hfi1_ahg_info *s_ahg;              /* ahg info for next header */
157159 	struct sdma_engine *s_sde;                /* current sde */
158160 	struct send_context *s_sendcontext;       /* current sendcontext */
161
+	struct hfi1_ctxtdata *rcd;                /* QP's receive context */
162
+	struct page **pages;                      /* for TID page scan */
163
+	u32 tid_enqueue;                          /* saved when tid waited */
159164 	u8 s_sc;		                  /* SC[0..4] for next packet */
160165 	struct iowait s_iowait;
166
+	struct timer_list s_tid_timer;            /* for timing tid wait */
167
+	struct timer_list s_tid_retry_timer;      /* for timing tid ack */
168
+	struct list_head tid_wait;                /* for queueing tid space */
169
+	struct hfi1_opfn_data opfn;
170
+	struct tid_flow_state flow_state;
171
+	struct tid_rdma_qp_params tid_rdma;
161172 	struct rvt_qp *owner;
173
+	u16 s_running_pkt_size;
162174 	u8 hdr_type; /* 9B or 16B */
175
+	struct rvt_sge_state tid_ss;       /* SGE state pointer for 2nd leg */
176
+	atomic_t n_requests;               /* # of TID RDMA requests in the */
177
+					   /* queue */
178
+	atomic_t n_tid_requests;            /* # of sent TID RDMA requests */
179
+	unsigned long tid_timer_timeout_jiffies;
180
+	unsigned long tid_retry_timeout_jiffies;
181
+
182
+	/* variables for the TID RDMA SE state machine */
183
+	u8 s_state;
184
+	u8 s_retry;
185
+	u8 rnr_nak_state;       /* RNR NAK state */
186
+	u8 s_nak_state;
187
+	u32 s_nak_psn;
188
+	u32 s_flags;
189
+	u32 s_tid_cur;
190
+	u32 s_tid_head;
191
+	u32 s_tid_tail;
192
+	u32 r_tid_head;     /* Most recently added TID RDMA request */
193
+	u32 r_tid_tail;     /* the last completed TID RDMA request */
194
+	u32 r_tid_ack;      /* the TID RDMA request to be ACK'ed */
195
+	u32 r_tid_alloc;    /* Request for which we are allocating resources */
196
+	u32 pending_tid_w_segs; /* Num of pending tid write segments */
197
+	u32 pending_tid_w_resp; /* Num of pending tid write responses */
198
+	u32 alloc_w_segs;       /* Number of segments for which write */
199
+			       /* resources have been allocated for this QP */
200
+
201
+	/* For TID RDMA READ */
202
+	u32 tid_r_reqs;         /* Num of tid reads requested */
203
+	u32 tid_r_comp;         /* Num of tid reads completed */
204
+	u32 pending_tid_r_segs; /* Num of pending tid read segments */
205
+	u16 pkts_ps;            /* packets per segment */
206
+	u8 timeout_shift;       /* account for number of packets per segment */
207
+
208
+	u32 r_next_psn_kdeth;
209
+	u32 r_next_psn_kdeth_save;
210
+	u32 s_resync_psn;
211
+	u8 sync_pt;           /* Set when QP reaches sync point */
212
+	u8 resync;
213
+};
214
+
215
+#define HFI1_QP_WQE_INVALID   ((u32)-1)
216
+
217
+struct hfi1_swqe_priv {
218
+	struct tid_rdma_request tid_req;
219
+	struct rvt_sge_state ss;  /* Used for TID RDMA READ Request */
220
+};
221
+
222
+struct hfi1_ack_priv {
223
+	struct rvt_sge_state ss;               /* used for TID WRITE RESP */
224
+	struct tid_rdma_request tid_req;
163225 };
164226 
165227 /*
166228  * This structure is used to hold commonly lookedup and computed values during
167229  * the send engine progress.
168230  */
231
+struct iowait_work;
169232 struct hfi1_pkt_state {
170233 	struct hfi1_ibdev *dev;
171234 	struct hfi1_ibport *ibp;
172235 	struct hfi1_pportdata *ppd;
173236 	struct verbs_txreq *s_txreq;
237
+	struct iowait_work *wait;
174238 	unsigned long flags;
175239 	unsigned long timeout;
176240 	unsigned long timeout_int;
....@@ -221,6 +285,7 @@
221285 	struct kmem_cache *verbs_txreq_cache;
222286 	u64 n_txwait;
223287 	u64 n_kmem_wait;
288
+	u64 n_tidwait;
224289 
225290 	/* protect iowait lists */
226291 	seqlock_t iowait_lock ____cacheline_aligned_in_smp;
....@@ -247,7 +312,7 @@
247312 	return container_of(rdi, struct hfi1_ibdev, rdi);
248313 }
249314 
250
-static inline struct rvt_qp *iowait_to_qp(struct  iowait *s_iowait)
315
+static inline struct rvt_qp *iowait_to_qp(struct iowait *s_iowait)
251316 {
252317 	struct hfi1_qp_priv *priv;
253318 
....@@ -265,9 +330,8 @@
265330 void hfi1_node_desc_chg(struct hfi1_ibport *ibp);
266331 int hfi1_process_mad(struct ib_device *ibdev, int mad_flags, u8 port,
267332 		     const struct ib_wc *in_wc, const struct ib_grh *in_grh,
268
-		     const struct ib_mad_hdr *in_mad, size_t in_mad_size,
269
-		     struct ib_mad_hdr *out_mad, size_t *out_mad_size,
270
-		     u16 *out_mad_pkey_index);
333
+		     const struct ib_mad *in_mad, struct ib_mad *out_mad,
334
+		     size_t *out_mad_size, u16 *out_mad_pkey_index);
271335 
272336 /*
273337  * The PSN_MASK and PSN_SHIFT allow for
....@@ -308,13 +372,35 @@
308372 	return (((int)a - (int)b) << PSN_SHIFT) >> PSN_SHIFT;
309373 }
310374 
375
+static inline struct tid_rdma_request *wqe_to_tid_req(struct rvt_swqe *wqe)
376
+{
377
+	return &((struct hfi1_swqe_priv *)wqe->priv)->tid_req;
378
+}
379
+
380
+static inline struct tid_rdma_request *ack_to_tid_req(struct rvt_ack_entry *e)
381
+{
382
+	return &((struct hfi1_ack_priv *)e->priv)->tid_req;
383
+}
384
+
385
+/*
386
+ * Look through all the active flows for a TID RDMA request and find
387
+ * the one (if it exists) that contains the specified PSN.
388
+ */
389
+static inline u32 __full_flow_psn(struct flow_state *state, u32 psn)
390
+{
391
+	return mask_psn((state->generation << HFI1_KDETH_BTH_SEQ_SHIFT) |
392
+			(psn & HFI1_KDETH_BTH_SEQ_MASK));
393
+}
394
+
395
+static inline u32 full_flow_psn(struct tid_rdma_flow *flow, u32 psn)
396
+{
397
+	return __full_flow_psn(&flow->flow_state, psn);
398
+}
399
+
311400 struct verbs_txreq;
312401 void hfi1_put_txreq(struct verbs_txreq *tx);
313402 
314403 int hfi1_verbs_send(struct rvt_qp *qp, struct hfi1_pkt_state *ps);
315
-
316
-void hfi1_copy_sge(struct rvt_sge_state *ss, void *data, u32 length,
317
-		   bool release, bool copy_last);
318404 
319405 void hfi1_cnp_rcv(struct hfi1_packet *packet);
320406 
....@@ -329,6 +415,7 @@
329415 
330416 u8 ah_to_sc(struct ib_device *ibdev, struct rdma_ah_attr *ah_attr);
331417 
418
+void hfi1_rc_verbs_aborted(struct rvt_qp *qp, struct hfi1_opa_header *opah);
332419 void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_opa_header *opah);
333420 
334421 void hfi1_ud_rcv(struct hfi1_packet *packet);
....@@ -343,7 +430,8 @@
343430 void hfi1_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr,
344431 		    int attr_mask, struct ib_udata *udata);
345432 void hfi1_restart_rc(struct rvt_qp *qp, u32 psn, int wait);
346
-int hfi1_check_send_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe);
433
+int hfi1_setup_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe,
434
+		   bool *call_send);
347435 
348436 extern const u32 rc_only_opcode;
349437 extern const u32 uc_only_opcode;
....@@ -354,17 +442,17 @@
354442 		  const struct ib_global_route *grh, u32 hwords, u32 nwords);
355443 
356444 void hfi1_make_ruc_header(struct rvt_qp *qp, struct ib_other_headers *ohdr,
357
-			  u32 bth0, u32 bth2, int middle,
445
+			  u32 bth0, u32 bth1, u32 bth2, int middle,
358446 			  struct hfi1_pkt_state *ps);
447
+
448
+bool hfi1_schedule_send_yield(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
449
+			      bool tid);
359450 
360451 void _hfi1_do_send(struct work_struct *work);
361452 
362453 void hfi1_do_send_from_rvt(struct rvt_qp *qp);
363454 
364455 void hfi1_do_send(struct rvt_qp *qp, bool in_thread);
365
-
366
-void hfi1_send_complete(struct rvt_qp *qp, struct rvt_swqe *wqe,
367
-			enum ib_wc_status status);
368456 
369457 void hfi1_send_rc_ack(struct hfi1_packet *packet, bool is_fecn);
370458 
....@@ -378,6 +466,10 @@
378466 
379467 void hfi1_unregister_ib_device(struct hfi1_devdata *);
380468 
469
+void hfi1_kdeth_eager_rcv(struct hfi1_packet *packet);
470
+
471
+void hfi1_kdeth_expected_rcv(struct hfi1_packet *packet);
472
+
381473 void hfi1_ib_rcv(struct hfi1_packet *packet);
382474 
383475 void hfi1_16B_rcv(struct hfi1_packet *packet);
....@@ -390,33 +482,21 @@
390482 int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
391483 			u64 pbc);
392484 
393
-int hfi1_wss_init(void);
394
-void hfi1_wss_exit(void);
395
-
396
-/* platform specific: return the lowest level cache (llc) size, in KiB */
397
-static inline int wss_llc_size(void)
398
-{
399
-	/* assume that the boot CPU value is universal for all CPUs */
400
-	return boot_cpu_data.x86_cache_size;
401
-}
402
-
403
-/* platform specific: cacheless copy */
404
-static inline void cacheless_memcpy(void *dst, void *src, size_t n)
405
-{
406
-	/*
407
-	 * Use the only available X64 cacheless copy.  Add a __user cast
408
-	 * to quiet sparse.  The src agument is already in the kernel so
409
-	 * there are no security issues.  The extra fault recovery machinery
410
-	 * is not invoked.
411
-	 */
412
-	__copy_user_nocache(dst, (void __user *)src, n, 0);
413
-}
414
-
415485 static inline bool opa_bth_is_migration(struct ib_other_headers *ohdr)
416486 {
417487 	return ohdr->bth[1] & cpu_to_be32(OPA_BTH_MIG_REQ);
418488 }
419489 
490
+void hfi1_wait_kmem(struct rvt_qp *qp);
491
+
492
+static inline void hfi1_trdma_send_complete(struct rvt_qp *qp,
493
+					    struct rvt_swqe *wqe,
494
+					    enum ib_wc_status status)
495
+{
496
+	trdma_clean_swqe(qp, wqe);
497
+	rvt_send_complete(qp, wqe, status);
498
+}
499
+
420500 extern const enum ib_wc_opcode ib_hfi1_wc_opcode[];
421501 
422502 extern const u8 hdr_len_by_opcode[];