From 04dd17822334871b23ea2862f7798fb0e0007777 Mon Sep 17 00:00:00 2001
From: hc <hc@nodka.com>
Date: Sat, 11 May 2024 08:53:19 +0000
Subject: [PATCH] change otg to host mode

---
 kernel/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c |  355 ++++++++++++++++++-----------------------------------------
 1 files changed, 108 insertions(+), 247 deletions(-)

diff --git a/kernel/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c b/kernel/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c
index 4a362fc..5826c2c 100644
--- a/kernel/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c
+++ b/kernel/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c
@@ -13,7 +13,6 @@
 #define FDATA_SIZE 32
 /* Base destination port for the solicited requests */
 #define SOLICIT_BASE_DPORT 256
-#define PENDING_SIG	0xFFFFFFFFFFFFFFFFUL
 
 #define REQ_NOT_POSTED 1
 #define REQ_BACKLOG    2
@@ -52,58 +51,26 @@
 	return index;
 }
 
-/**
- * dma_free_sglist - unmap and free the sg lists.
- * @ndev: N5 device
- * @sgtbl: SG table
- */
 static void softreq_unmap_sgbufs(struct nitrox_softreq *sr)
 {
 	struct nitrox_device *ndev = sr->ndev;
 	struct device *dev = DEV(ndev);
-	struct nitrox_sglist *sglist;
 
-	/* unmap in sgbuf */
-	sglist = sr->in.sglist;
-	if (!sglist)
-		goto out_unmap;
 
-	/* unmap iv */
-	dma_unmap_single(dev, sglist->dma, sglist->len, DMA_BIDIRECTIONAL);
-	/* unmpa src sglist */
-	dma_unmap_sg(dev, sr->in.buf, (sr->in.map_bufs_cnt - 1), sr->in.dir);
-	/* unamp gather component */
-	dma_unmap_single(dev, sr->in.dma, sr->in.len, DMA_TO_DEVICE);
-	kfree(sr->in.sglist);
+	dma_unmap_sg(dev, sr->in.sg, sr->in.sgmap_cnt, DMA_BIDIRECTIONAL);
+	dma_unmap_single(dev, sr->in.sgcomp_dma, sr->in.sgcomp_len,
+			 DMA_TO_DEVICE);
 	kfree(sr->in.sgcomp);
-	sr->in.sglist = NULL;
-	sr->in.buf = NULL;
-	sr->in.map_bufs_cnt = 0;
+	sr->in.sg = NULL;
+	sr->in.sgmap_cnt = 0;
 
-out_unmap:
-	/* unmap out sgbuf */
-	sglist = sr->out.sglist;
-	if (!sglist)
-		return;
-
-	/* unmap orh */
-	dma_unmap_single(dev, sr->resp.orh_dma, ORH_HLEN, sr->out.dir);
-
-	/* unmap dst sglist */
-	if (!sr->inplace) {
-		dma_unmap_sg(dev, sr->out.buf, (sr->out.map_bufs_cnt - 3),
-			     sr->out.dir);
-	}
-	/* unmap completion */
-	dma_unmap_single(dev, sr->resp.completion_dma, COMP_HLEN, sr->out.dir);
-
-	/* unmap scatter component */
-	dma_unmap_single(dev, sr->out.dma, sr->out.len, DMA_TO_DEVICE);
-	kfree(sr->out.sglist);
+	dma_unmap_sg(dev, sr->out.sg, sr->out.sgmap_cnt,
+		     DMA_BIDIRECTIONAL);
+	dma_unmap_single(dev, sr->out.sgcomp_dma, sr->out.sgcomp_len,
+			 DMA_TO_DEVICE);
 	kfree(sr->out.sgcomp);
-	sr->out.sglist = NULL;
-	sr->out.buf = NULL;
-	sr->out.map_bufs_cnt = 0;
+	sr->out.sg = NULL;
+	sr->out.sgmap_cnt = 0;
 }
 
 static void softreq_destroy(struct nitrox_softreq *sr)
@@ -116,7 +83,7 @@
  * create_sg_component - create SG componets for N5 device.
  * @sr: Request structure
  * @sgtbl: SG table
- * @nr_comp: total number of components required
+ * @map_nents: number of dma mapped entries
  *
  * Component structure
  *
@@ -140,7 +107,7 @@
 {
 	struct nitrox_device *ndev = sr->ndev;
 	struct nitrox_sgcomp *sgcomp;
-	struct nitrox_sglist *sglist;
+	struct scatterlist *sg;
 	dma_addr_t dma;
 	size_t sz_comp;
 	int i, j, nr_sgcomp;
@@ -154,17 +121,15 @@
 		return -ENOMEM;
 
 	sgtbl->sgcomp = sgcomp;
-	sgtbl->nr_sgcomp = nr_sgcomp;
 
-	sglist = sgtbl->sglist;
+	sg = sgtbl->sg;
 	/* populate device sg component */
 	for (i = 0; i < nr_sgcomp; i++) {
-		for (j = 0; j < 4; j++) {
-			sgcomp->len[j] = cpu_to_be16(sglist->len);
-			sgcomp->dma[j] = cpu_to_be64(sglist->dma);
-			sglist++;
+		for (j = 0; j < 4 && sg; j++) {
+			sgcomp[i].len[j] = cpu_to_be16(sg_dma_len(sg));
+			sgcomp[i].dma[j] = cpu_to_be64(sg_dma_address(sg));
+			sg = sg_next(sg);
 		}
-		sgcomp++;
 	}
 	/* map the device sg component */
 	dma = dma_map_single(DEV(ndev), sgtbl->sgcomp, sz_comp, DMA_TO_DEVICE);
@@ -174,8 +139,8 @@
 		return -ENOMEM;
 	}
 
-	sgtbl->dma = dma;
-	sgtbl->len = sz_comp;
+	sgtbl->sgcomp_dma = dma;
+	sgtbl->sgcomp_len = sz_comp;
 
 	return 0;
 }
@@ -193,66 +158,27 @@
 {
 	struct device *dev = DEV(sr->ndev);
 	struct scatterlist *sg = req->src;
-	struct nitrox_sglist *glist;
 	int i, nents, ret = 0;
-	dma_addr_t dma;
-	size_t sz;
 
-	nents = sg_nents(req->src);
+	nents = dma_map_sg(dev, req->src, sg_nents(req->src),
+			   DMA_BIDIRECTIONAL);
+	if (!nents)
+		return -EINVAL;
 
-	/* creater gather list IV and src entries */
-	sz = roundup((1 + nents), 4) * sizeof(*glist);
-	glist = kzalloc(sz, sr->gfp);
-	if (!glist)
-		return -ENOMEM;
+	for_each_sg(req->src, sg, nents, i)
+		sr->in.total_bytes += sg_dma_len(sg);
 
-	sr->in.sglist = glist;
-	/* map IV */
-	dma = dma_map_single(dev, &req->iv, req->ivsize, DMA_BIDIRECTIONAL);
-	if (dma_mapping_error(dev, dma)) {
-		ret = -EINVAL;
-		goto iv_map_err;
-	}
-
-	sr->in.dir = (req->src == req->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
-	/* map src entries */
-	nents = dma_map_sg(dev, req->src, nents, sr->in.dir);
-	if (!nents) {
-		ret = -EINVAL;
-		goto src_map_err;
-	}
-	sr->in.buf = req->src;
-
-	/* store the mappings */
-	glist->len = req->ivsize;
-	glist->dma = dma;
-	glist++;
-	sr->in.total_bytes += req->ivsize;
-
-	for_each_sg(req->src, sg, nents, i) {
-		glist->len = sg_dma_len(sg);
-		glist->dma = sg_dma_address(sg);
-		sr->in.total_bytes += glist->len;
-		glist++;
-	}
-	/* roundup map count to align with entires in sg component */
-	sr->in.map_bufs_cnt = (1 + nents);
-
-	/* create NITROX gather component */
-	ret = create_sg_component(sr, &sr->in, sr->in.map_bufs_cnt);
+	sr->in.sg = req->src;
+	sr->in.sgmap_cnt = nents;
+	ret = create_sg_component(sr, &sr->in, sr->in.sgmap_cnt);
 	if (ret)
 		goto incomp_err;
 
 	return 0;
 
 incomp_err:
-	dma_unmap_sg(dev, req->src, nents, sr->in.dir);
-	sr->in.map_bufs_cnt = 0;
-src_map_err:
-	dma_unmap_single(dev, dma, req->ivsize, DMA_BIDIRECTIONAL);
-iv_map_err:
-	kfree(sr->in.sglist);
-	sr->in.sglist = NULL;
+	dma_unmap_sg(dev, req->src, nents, DMA_BIDIRECTIONAL);
+	sr->in.sgmap_cnt = 0;
 	return ret;
 }
 
@@ -260,104 +186,25 @@
 			   struct se_crypto_request *req)
 {
 	struct device *dev = DEV(sr->ndev);
-	struct nitrox_sglist *glist = sr->in.sglist;
-	struct nitrox_sglist *slist;
-	struct scatterlist *sg;
-	int i, nents, map_bufs_cnt, ret = 0;
-	size_t sz;
+	int nents, ret = 0;
 
-	nents = sg_nents(req->dst);
+	nents = dma_map_sg(dev, req->dst, sg_nents(req->dst),
+			   DMA_BIDIRECTIONAL);
+	if (!nents)
+		return -EINVAL;
 
-	/* create scatter list ORH, IV, dst entries and Completion header */
-	sz = roundup((3 + nents), 4) * sizeof(*slist);
-	slist = kzalloc(sz, sr->gfp);
-	if (!slist)
-		return -ENOMEM;
-
-	sr->out.sglist = slist;
-	sr->out.dir = DMA_BIDIRECTIONAL;
-	/* map ORH */
-	sr->resp.orh_dma = dma_map_single(dev, &sr->resp.orh, ORH_HLEN,
-					  sr->out.dir);
-	if (dma_mapping_error(dev, sr->resp.orh_dma)) {
-		ret = -EINVAL;
-		goto orh_map_err;
-	}
-
-	/* map completion */
-	sr->resp.completion_dma = dma_map_single(dev, &sr->resp.completion,
-						 COMP_HLEN, sr->out.dir);
-	if (dma_mapping_error(dev, sr->resp.completion_dma)) {
-		ret = -EINVAL;
-		goto compl_map_err;
-	}
-
-	sr->inplace = (req->src == req->dst) ? true : false;
-	/* out place */
-	if (!sr->inplace) {
-		nents = dma_map_sg(dev, req->dst, nents, sr->out.dir);
-		if (!nents) {
-			ret = -EINVAL;
-			goto dst_map_err;
-		}
-	}
-	sr->out.buf = req->dst;
-
-	/* store the mappings */
-	/* orh */
-	slist->len = ORH_HLEN;
-	slist->dma = sr->resp.orh_dma;
-	slist++;
-
-	/* copy the glist mappings */
-	if (sr->inplace) {
-		nents = sr->in.map_bufs_cnt - 1;
-		map_bufs_cnt = sr->in.map_bufs_cnt;
-		while (map_bufs_cnt--) {
-			slist->len = glist->len;
-			slist->dma = glist->dma;
-			slist++;
-			glist++;
-		}
-	} else {
-		/* copy iv mapping */
-		slist->len = glist->len;
-		slist->dma = glist->dma;
-		slist++;
-		/* copy remaining maps */
-		for_each_sg(req->dst, sg, nents, i) {
-			slist->len = sg_dma_len(sg);
-			slist->dma = sg_dma_address(sg);
-			slist++;
-		}
-	}
-
-	/* completion */
-	slist->len = COMP_HLEN;
-	slist->dma = sr->resp.completion_dma;
-
-	sr->out.map_bufs_cnt = (3 + nents);
-
-	ret = create_sg_component(sr, &sr->out, sr->out.map_bufs_cnt);
+	sr->out.sg = req->dst;
+	sr->out.sgmap_cnt = nents;
+	ret = create_sg_component(sr, &sr->out, sr->out.sgmap_cnt);
 	if (ret)
 		goto outcomp_map_err;
 
 	return 0;
 
 outcomp_map_err:
-	if (!sr->inplace)
-		dma_unmap_sg(dev, req->dst, nents, sr->out.dir);
-	sr->out.map_bufs_cnt = 0;
-	sr->out.buf = NULL;
-dst_map_err:
-	dma_unmap_single(dev, sr->resp.completion_dma, COMP_HLEN, sr->out.dir);
-	sr->resp.completion_dma = 0;
-compl_map_err:
-	dma_unmap_single(dev, sr->resp.orh_dma, ORH_HLEN, sr->out.dir);
-	sr->resp.orh_dma = 0;
-orh_map_err:
-	kfree(sr->out.sglist);
-	sr->out.sglist = NULL;
+	dma_unmap_sg(dev, req->dst, nents, DMA_BIDIRECTIONAL);
+	sr->out.sgmap_cnt = 0;
+	sr->out.sg = NULL;
 	return ret;
 }
 
@@ -382,11 +229,11 @@
 {
 	INIT_LIST_HEAD(&sr->backlog);
 
-	spin_lock_bh(&cmdq->backlog_lock);
+	spin_lock_bh(&cmdq->backlog_qlock);
 	list_add_tail(&sr->backlog, &cmdq->backlog_head);
 	atomic_inc(&cmdq->backlog_count);
 	atomic_set(&sr->status, REQ_BACKLOG);
-	spin_unlock_bh(&cmdq->backlog_lock);
+	spin_unlock_bh(&cmdq->backlog_qlock);
 }
 
 static inline void response_list_add(struct nitrox_softreq *sr,
@@ -394,17 +241,17 @@
 {
 	INIT_LIST_HEAD(&sr->response);
 
-	spin_lock_bh(&cmdq->response_lock);
+	spin_lock_bh(&cmdq->resp_qlock);
 	list_add_tail(&sr->response, &cmdq->response_head);
-	spin_unlock_bh(&cmdq->response_lock);
+	spin_unlock_bh(&cmdq->resp_qlock);
 }
 
 static inline void response_list_del(struct nitrox_softreq *sr,
 				     struct nitrox_cmdq *cmdq)
 {
-	spin_lock_bh(&cmdq->response_lock);
+	spin_lock_bh(&cmdq->resp_qlock);
 	list_del(&sr->response);
-	spin_unlock_bh(&cmdq->response_lock);
+	spin_unlock_bh(&cmdq->resp_qlock);
 }
 
 static struct nitrox_softreq *
@@ -422,6 +269,8 @@
 		smp_mb__after_atomic();
 		return true;
 	}
+	/* sync with other cpus */
+	smp_mb__after_atomic();
 	return false;
 }
 
@@ -439,11 +288,11 @@
 	int idx;
 	u8 *ent;
 
-	spin_lock_bh(&cmdq->cmdq_lock);
+	spin_lock_bh(&cmdq->cmd_qlock);
 
 	idx = cmdq->write_idx;
 	/* copy the instruction */
-	ent = cmdq->head + (idx * cmdq->instr_size);
+	ent = cmdq->base + (idx * cmdq->instr_size);
 	memcpy(ent, &sr->instr, cmdq->instr_size);
 
 	atomic_set(&sr->status, REQ_POSTED);
@@ -454,12 +303,13 @@
 
 	/* Ring doorbell with count 1 */
 	writeq(1, cmdq->dbell_csr_addr);
-	/* orders the doorbell rings */
-	mmiowb();
 
 	cmdq->write_idx = incr_index(idx, 1, ndev->qlen);
 
-	spin_unlock_bh(&cmdq->cmdq_lock);
+	spin_unlock_bh(&cmdq->cmd_qlock);
+
+	/* increment the posted command count */
+	atomic64_inc(&ndev->stats.posted);
 }
 
 static int post_backlog_cmds(struct nitrox_cmdq *cmdq)
@@ -471,11 +321,9 @@
 	if (!atomic_read(&cmdq->backlog_count))
 		return 0;
 
-	spin_lock_bh(&cmdq->backlog_lock);
+	spin_lock_bh(&cmdq->backlog_qlock);
 
 	list_for_each_entry_safe(sr, tmp, &cmdq->backlog_head, backlog) {
-		struct skcipher_request *skreq;
-
 		/* submit until space available */
 		if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
 			ret = -ENOSPC;
@@ -487,14 +335,10 @@
 		/* sync with other cpus */
 		smp_mb__after_atomic();
 
-		skreq = sr->skreq;
 		/* post the command */
 		post_se_instr(sr, cmdq);
-
-		/* backlog requests are posted, wakeup with -EINPROGRESS */
-		skcipher_request_complete(skreq, -EINPROGRESS);
 	}
-	spin_unlock_bh(&cmdq->backlog_lock);
+	spin_unlock_bh(&cmdq->backlog_qlock);
 
 	return ret;
 }
@@ -508,11 +352,14 @@
 	post_backlog_cmds(cmdq);
 
 	if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
-		if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
+		if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
+			/* increment drop count */
+			atomic64_inc(&ndev->stats.dropped);
 			return -ENOSPC;
+		}
 		/* add to backlog list */
 		backlog_list_add(sr, cmdq);
-		return -EBUSY;
+		return -EINPROGRESS;
 	}
 	post_se_instr(sr, cmdq);
 
@@ -529,7 +376,7 @@
 int nitrox_process_se_request(struct nitrox_device *ndev,
 			      struct se_crypto_request *req,
 			      completion_t callback,
-			      struct skcipher_request *skreq)
+			      void *cb_arg)
 {
 	struct nitrox_softreq *sr;
 	dma_addr_t ctx_handle = 0;
@@ -546,12 +393,12 @@
 	sr->flags = req->flags;
 	sr->gfp = req->gfp;
 	sr->callback = callback;
-	sr->skreq = skreq;
+	sr->cb_arg = cb_arg;
 
 	atomic_set(&sr->status, REQ_NOT_POSTED);
 
-	WRITE_ONCE(sr->resp.orh, PENDING_SIG);
-	WRITE_ONCE(sr->resp.completion, PENDING_SIG);
+	sr->resp.orh = req->orh;
+	sr->resp.completion = req->comp;
 
 	ret = softreq_map_iobuf(sr, req);
 	if (ret) {
@@ -572,7 +419,7 @@
 	/* select the queue */
 	qno = smp_processor_id() % ndev->nr_queues;
 
-	sr->cmdq = &ndev->pkt_cmdqs[qno];
+	sr->cmdq = &ndev->pkt_inq[qno];
 
 	/*
 	 * 64-Byte Instruction Format
@@ -592,13 +439,13 @@
 
 	/* fill the packet instruction */
 	/* word 0 */
-	sr->instr.dptr0 = cpu_to_be64(sr->in.dma);
+	sr->instr.dptr0 = cpu_to_be64(sr->in.sgcomp_dma);
 
 	/* word 1 */
 	sr->instr.ih.value = 0;
 	sr->instr.ih.s.g = 1;
-	sr->instr.ih.s.gsz = sr->in.map_bufs_cnt;
-	sr->instr.ih.s.ssz = sr->out.map_bufs_cnt;
+	sr->instr.ih.s.gsz = sr->in.sgmap_cnt;
+	sr->instr.ih.s.ssz = sr->out.sgmap_cnt;
 	sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr);
 	sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes;
 	sr->instr.ih.value = cpu_to_be64(sr->instr.ih.value);
@@ -620,11 +467,11 @@
 
 	/* word 4 */
 	sr->instr.slc.value[0] = 0;
-	sr->instr.slc.s.ssz = sr->out.map_bufs_cnt;
+	sr->instr.slc.s.ssz = sr->out.sgmap_cnt;
 	sr->instr.slc.value[0] = cpu_to_be64(sr->instr.slc.value[0]);
 
 	/* word 5 */
-	sr->instr.slc.s.rptr = cpu_to_be64(sr->out.dma);
+	sr->instr.slc.s.rptr = cpu_to_be64(sr->out.sgcomp_dma);
 
 	/*
 	 * No conversion for front data,
@@ -658,6 +505,24 @@
 	post_backlog_cmds(cmdq);
 }
 
+static bool sr_completed(struct nitrox_softreq *sr)
+{
+	u64 orh = READ_ONCE(*sr->resp.orh);
+	unsigned long timeout = jiffies + msecs_to_jiffies(1);
+
+	if ((orh != PENDING_SIG) && (orh & 0xff))
+		return true;
+
+	while (READ_ONCE(*sr->resp.completion) == PENDING_SIG) {
+		if (time_after(jiffies, timeout)) {
+			pr_err("comp not done\n");
+			return false;
+		}
+	}
+
+	return true;
+}
+
 /**
  * process_request_list - process completed requests
  * @ndev: N5 device
@@ -669,9 +534,9 @@
 {
 	struct nitrox_device *ndev = cmdq->ndev;
 	struct nitrox_softreq *sr;
-	struct skcipher_request *skreq;
-	completion_t callback;
 	int req_completed = 0, err = 0, budget;
+	completion_t callback;
+	void *cb_arg;
 
 	/* check all pending requests */
 	budget = atomic_read(&cmdq->pending_count);
@@ -685,47 +550,45 @@
 			break;
 
 		/* check orh and completion bytes updates */
-		if (READ_ONCE(sr->resp.orh) == READ_ONCE(sr->resp.completion)) {
+		if (!sr_completed(sr)) {
 			/* request not completed, check for timeout */
 			if (!cmd_timeout(sr->tstamp, ndev->timeout))
 				break;
 			dev_err_ratelimited(DEV(ndev),
 					    "Request timeout, orh 0x%016llx\n",
-					    READ_ONCE(sr->resp.orh));
+					    READ_ONCE(*sr->resp.orh));
 		}
 		atomic_dec(&cmdq->pending_count);
+		atomic64_inc(&ndev->stats.completed);
 		/* sync with other cpus */
 		smp_mb__after_atomic();
 		/* remove from response list */
 		response_list_del(sr, cmdq);
-
-		callback = sr->callback;
-		skreq = sr->skreq;
-
 		/* ORH error code */
-		err = READ_ONCE(sr->resp.orh) & 0xff;
+		err = READ_ONCE(*sr->resp.orh) & 0xff;
+		callback = sr->callback;
+		cb_arg = sr->cb_arg;
 		softreq_destroy(sr);
-
 		if (callback)
-			callback(skreq, err);
+			callback(cb_arg, err);
 
 		req_completed++;
 	}
 }
 
 /**
- * pkt_slc_resp_handler - post processing of SE responses
+ * pkt_slc_resp_tasklet - post processing of SE responses
  */
-void pkt_slc_resp_handler(unsigned long data)
+void pkt_slc_resp_tasklet(unsigned long data)
 {
-	struct bh_data *bh = (void *)(uintptr_t)(data);
-	struct nitrox_cmdq *cmdq = bh->cmdq;
-	union nps_pkt_slc_cnts pkt_slc_cnts;
+	struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data);
+	struct nitrox_cmdq *cmdq = qvec->cmdq;
+	union nps_pkt_slc_cnts slc_cnts;
 
 	/* read completion count */
-	pkt_slc_cnts.value = readq(bh->completion_cnt_csr_addr);
+	slc_cnts.value = readq(cmdq->compl_cnt_csr_addr);
 	/* resend the interrupt if more work to do */
-	pkt_slc_cnts.s.resend = 1;
+	slc_cnts.s.resend = 1;
 
 	process_response_list(cmdq);
 
@@ -733,9 +596,7 @@
 	 * clear the interrupt with resend bit enabled,
 	 * MSI-X interrupt generates if Completion count > Threshold
 	 */
-	writeq(pkt_slc_cnts.value, bh->completion_cnt_csr_addr);
-	/* order the writes */
-	mmiowb();
+	writeq(slc_cnts.value, cmdq->compl_cnt_csr_addr);
 
 	if (atomic_read(&cmdq->backlog_count))
 		schedule_work(&cmdq->backlog_qflush);

--
Gitblit v1.6.2