forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-02-20 ea08eeccae9297f7aabd2ef7f0c2517ac4549acc 
 kernel/drivers/crypto/caam/qi.c
....@@ -4,13 +4,14 @@
44  * Queue Interface backend functionality
55  *
66  * Copyright 2013-2016 Freescale Semiconductor, Inc.
7
- * Copyright 2016-2017 NXP
7
+ * Copyright 2016-2017, 2019-2020 NXP
88  */
99 
1010 #include <linux/cpumask.h>
1111 #include <linux/kthread.h>
1212 #include <soc/fsl/qman.h>
1313 
14
+#include "debugfs.h"
1415 #include "regs.h"
1516 #include "qi.h"
1617 #include "desc.h"
....@@ -18,6 +19,7 @@
1819 #include "desc_constr.h"
1920 
2021 #define PREHDR_RSLS_SHIFT	31
22
+#define PREHDR_ABS		BIT(25)
2123 
2224 /*
2325  * Use a reasonable backlog of frames (per CPU) as congestion threshold,
....@@ -58,11 +60,9 @@
5860 /*
5961  * caam_qi_priv - CAAM QI backend private params
6062  * @cgr: QMan congestion group
61
- * @qi_pdev: platform device for QI backend
6263  */
6364 struct caam_qi_priv {
6465 	struct qman_cgr cgr;
65
-	struct platform_device *qi_pdev;
6666 };
6767 
6868 static struct caam_qi_priv qipriv ____cacheline_aligned;
....@@ -73,15 +73,6 @@
7373  */
7474 bool caam_congested __read_mostly;
7575 EXPORT_SYMBOL(caam_congested);
76
-
77
-#ifdef CONFIG_DEBUG_FS
78
-/*
79
- * This is a counter for the number of times the congestion group (where all
80
- * the request and response queueus are) reached congestion. Incremented
81
- * each time the congestion callback is called with congested == true.
82
- */
83
-static u64 times_congested;
84
-#endif
8576 
8677 /*
8778  * This is a a cache of buffers, from which the users of CAAM QI driver
....@@ -94,6 +85,16 @@
9485  * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here
9586  */
9687 static struct kmem_cache *qi_cache;
88
+
89
+static void *caam_iova_to_virt(struct iommu_domain *domain,
90
+			       dma_addr_t iova_addr)
91
+{
92
+	phys_addr_t phys_addr;
93
+
94
+	phys_addr = domain ? iommu_iova_to_phys(domain, iova_addr) : iova_addr;
95
+
96
+	return phys_to_virt(phys_addr);
97
+}
9798 
9899 int caam_qi_enqueue(struct device *qidev, struct caam_drv_req *req)
99100 {
....@@ -115,8 +116,10 @@
115116 
116117 	do {
117118 		ret = qman_enqueue(req->drv_ctx->req_fq, &fd);
118
-		if (likely(!ret))
119
+		if (likely(!ret)) {
120
+			refcount_inc(&req->drv_ctx->refcnt);
119121 			return 0;
122
+		}
120123 
121124 		if (ret != -EBUSY)
122125 			break;
....@@ -135,25 +138,31 @@
135138 	const struct qm_fd *fd;
136139 	struct caam_drv_req *drv_req;
137140 	struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev);
141
+	struct caam_drv_private *priv = dev_get_drvdata(qidev);
138142 
139143 	fd = &msg->ern.fd;
140144 
141
-	if (qm_fd_get_format(fd) != qm_fd_compound) {
142
-		dev_err(qidev, "Non-compound FD from CAAM\n");
143
-		return;
144
-	}
145
-
146
-	drv_req = (struct caam_drv_req *)phys_to_virt(qm_fd_addr_get64(fd));
145
+	drv_req = caam_iova_to_virt(priv->domain, qm_fd_addr_get64(fd));
147146 	if (!drv_req) {
148147 		dev_err(qidev,
149148 			"Can't find original request for CAAM response\n");
150149 		return;
151150 	}
152151 
152
+	refcount_dec(&drv_req->drv_ctx->refcnt);
153
+
154
+	if (qm_fd_get_format(fd) != qm_fd_compound) {
155
+		dev_err(qidev, "Non-compound FD from CAAM\n");
156
+		return;
157
+	}
158
+
153159 	dma_unmap_single(drv_req->drv_ctx->qidev, qm_fd_addr(fd),
154160 			 sizeof(drv_req->fd_sgt), DMA_BIDIRECTIONAL);
155161 
156
-	drv_req->cbk(drv_req, -EIO);
162
+	if (fd->status)
163
+		drv_req->cbk(drv_req, be32_to_cpu(fd->status));
164
+	else
165
+		drv_req->cbk(drv_req, JRSTA_SSRC_QI);
157166 }
158167 
159168 static struct qman_fq *create_caam_req_fq(struct device *qidev,
....@@ -274,9 +283,10 @@
274283 	return ret;
275284 }
276285 
277
-static int empty_caam_fq(struct qman_fq *fq)
286
+static int empty_caam_fq(struct qman_fq *fq, struct caam_drv_ctx *drv_ctx)
278287 {
279288 	int ret;
289
+	int retries = 10;
280290 	struct qm_mcr_queryfq_np np;
281291 
282292 	/* Wait till the older CAAM FQ get empty */
....@@ -291,11 +301,18 @@
291301 		msleep(20);
292302 	} while (1);
293303 
294
-	/*
295
-	 * Give extra time for pending jobs from this FQ in holding tanks
296
-	 * to get processed
297
-	 */
298
-	msleep(20);
304
+	/* Wait until pending jobs from this FQ are processed by CAAM */
305
+	do {
306
+		if (refcount_read(&drv_ctx->refcnt) == 1)
307
+			break;
308
+
309
+		msleep(20);
310
+	} while (--retries);
311
+
312
+	if (!retries)
313
+		dev_warn_once(drv_ctx->qidev, "%d frames from FQID %u still pending in CAAM\n",
314
+			      refcount_read(&drv_ctx->refcnt), fq->fqid);
315
+
299316 	return 0;
300317 }
301318 
....@@ -318,7 +335,7 @@
318335 	/* Create a new req FQ in parked state */
319336 	new_fq = create_caam_req_fq(drv_ctx->qidev, drv_ctx->rsp_fq,
320337 				    drv_ctx->context_a, 0);
321
-	if (unlikely(IS_ERR_OR_NULL(new_fq))) {
338
+	if (IS_ERR(new_fq)) {
322339 		dev_err(qidev, "FQ allocation for shdesc update failed\n");
323340 		return PTR_ERR(new_fq);
324341 	}
....@@ -327,7 +344,7 @@
327344 	drv_ctx->req_fq = new_fq;
328345 
329346 	/* Empty and remove the older FQ */
330
-	ret = empty_caam_fq(old_fq);
347
+	ret = empty_caam_fq(old_fq, drv_ctx);
331348 	if (ret) {
332349 		dev_err(qidev, "Old CAAM FQ empty failed: %d\n", ret);
333350 
....@@ -346,6 +363,7 @@
346363 	 */
347364 	drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) |
348365 					   num_words);
366
+	drv_ctx->prehdr[1] = cpu_to_caam32(PREHDR_ABS);
349367 	memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc));
350368 	dma_sync_single_for_device(qidev, drv_ctx->context_a,
351369 				   sizeof(drv_ctx->sh_desc) +
....@@ -401,6 +419,7 @@
401419 	 */
402420 	drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) |
403421 					   num_words);
422
+	drv_ctx->prehdr[1] = cpu_to_caam32(PREHDR_ABS);
404423 	memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc));
405424 	size = sizeof(drv_ctx->prehdr) + sizeof(drv_ctx->sh_desc);
406425 	hwdesc = dma_map_single(qidev, drv_ctx->prehdr, size,
....@@ -431,12 +450,15 @@
431450 	/* Attach request FQ */
432451 	drv_ctx->req_fq = create_caam_req_fq(qidev, drv_ctx->rsp_fq, hwdesc,
433452 					     QMAN_INITFQ_FLAG_SCHED);
434
-	if (unlikely(IS_ERR_OR_NULL(drv_ctx->req_fq))) {
453
+	if (IS_ERR(drv_ctx->req_fq)) {
435454 		dev_err(qidev, "create_caam_req_fq failed\n");
436455 		dma_unmap_single(qidev, hwdesc, size, DMA_BIDIRECTIONAL);
437456 		kfree(drv_ctx);
438457 		return ERR_PTR(-ENOMEM);
439458 	}
459
+
460
+	/* init reference counter used to track references to request FQ */
461
+	refcount_set(&drv_ctx->refcnt, 1);
440462 
441463 	drv_ctx->qidev = qidev;
442464 	return drv_ctx;
....@@ -485,10 +507,11 @@
485507 }
486508 EXPORT_SYMBOL(caam_drv_ctx_rel);
487509 
488
-void caam_qi_shutdown(struct device *qidev)
510
+static void caam_qi_shutdown(void *data)
489511 {
490512 	int i;
491
-	struct caam_qi_priv *priv = dev_get_drvdata(qidev);
513
+	struct device *qidev = data;
514
+	struct caam_qi_priv *priv = &qipriv;
492515 	const cpumask_t *cpus = qman_affine_cpus();
493516 
494517 	for_each_cpu(i, cpus) {
....@@ -506,8 +529,6 @@
506529 	qman_release_cgrid(priv->cgr.cgrid);
507530 
508531 	kmem_cache_destroy(qi_cache);
509
-
510
-	platform_device_unregister(priv->qi_pdev);
511532 }
512533 
513534 static void cgr_cb(struct qman_portal *qm, struct qman_cgr *cgr, int congested)
....@@ -515,9 +536,8 @@
515536 	caam_congested = congested;
516537 
517538 	if (congested) {
518
-#ifdef CONFIG_DEBUG_FS
519
-		times_congested++;
520
-#endif
539
+		caam_debugfs_qi_congested();
540
+
521541 		pr_debug_ratelimited("CAAM entered congestion\n");
522542 
523543 	} else {
....@@ -550,12 +570,23 @@
550570 	struct caam_drv_req *drv_req;
551571 	const struct qm_fd *fd;
552572 	struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev);
573
+	struct caam_drv_private *priv = dev_get_drvdata(qidev);
553574 	u32 status;
554575 
555576 	if (caam_qi_napi_schedule(p, caam_napi))
556577 		return qman_cb_dqrr_stop;
557578 
558579 	fd = &dqrr->fd;
580
+
581
+	drv_req = caam_iova_to_virt(priv->domain, qm_fd_addr_get64(fd));
582
+	if (unlikely(!drv_req)) {
583
+		dev_err(qidev,
584
+			"Can't find original request for caam response\n");
585
+		return qman_cb_dqrr_consume;
586
+	}
587
+
588
+	refcount_dec(&drv_req->drv_ctx->refcnt);
589
+
559590 	status = be32_to_cpu(fd->status);
560591 	if (unlikely(status)) {
561592 		u32 ssrc = status & JRSTA_SSRC_MASK;
....@@ -563,19 +594,13 @@
563594 
564595 		if (ssrc != JRSTA_SSRC_CCB_ERROR ||
565596 		    err_id != JRSTA_CCBERR_ERRID_ICVCHK)
566
-			dev_err(qidev, "Error: %#x in CAAM response FD\n",
567
-				status);
597
+			dev_err_ratelimited(qidev,
598
+					    "Error: %#x in CAAM response FD\n",
599
+					    status);
568600 	}
569601 
570602 	if (unlikely(qm_fd_get_format(fd) != qm_fd_compound)) {
571603 		dev_err(qidev, "Non-compound FD from CAAM\n");
572
-		return qman_cb_dqrr_consume;
573
-	}
574
-
575
-	drv_req = (struct caam_drv_req *)phys_to_virt(qm_fd_addr_get64(fd));
576
-	if (unlikely(!drv_req)) {
577
-		dev_err(qidev,
578
-			"Can't find original request for caam response\n");
579604 		return qman_cb_dqrr_consume;
580605 	}
581606 
....@@ -692,33 +717,17 @@
692717 int caam_qi_init(struct platform_device *caam_pdev)
693718 {
694719 	int err, i;
695
-	struct platform_device *qi_pdev;
696720 	struct device *ctrldev = &caam_pdev->dev, *qidev;
697721 	struct caam_drv_private *ctrlpriv;
698722 	const cpumask_t *cpus = qman_affine_cpus();
699
-	static struct platform_device_info qi_pdev_info = {
700
-		.name = "caam_qi",
701
-		.id = PLATFORM_DEVID_NONE
702
-	};
703
-
704
-	qi_pdev_info.parent = ctrldev;
705
-	qi_pdev_info.dma_mask = dma_get_mask(ctrldev);
706
-	qi_pdev = platform_device_register_full(&qi_pdev_info);
707
-	if (IS_ERR(qi_pdev))
708
-		return PTR_ERR(qi_pdev);
709
-	set_dma_ops(&qi_pdev->dev, get_dma_ops(ctrldev));
710723 
711724 	ctrlpriv = dev_get_drvdata(ctrldev);
712
-	qidev = &qi_pdev->dev;
713
-
714
-	qipriv.qi_pdev = qi_pdev;
715
-	dev_set_drvdata(qidev, &qipriv);
725
+	qidev = ctrldev;
716726 
717727 	/* Initialize the congestion detection */
718728 	err = init_cgr(qidev);
719729 	if (err) {
720730 		dev_err(qidev, "CGR initialization failed: %d\n", err);
721
-		platform_device_unregister(qi_pdev);
722731 		return err;
723732 	}
724733 
....@@ -727,7 +736,6 @@
727736 	if (err) {
728737 		dev_err(qidev, "Can't allocate CAAM response FQs: %d\n", err);
729738 		free_rsp_fqs();
730
-		platform_device_unregister(qi_pdev);
731739 		return err;
732740 	}
733741 
....@@ -750,22 +758,20 @@
750758 		napi_enable(irqtask);
751759 	}
752760 
753
-	/* Hook up QI device to parent controlling caam device */
754
-	ctrlpriv->qidev = qidev;
755
-
756761 	qi_cache = kmem_cache_create("caamqicache", CAAM_QI_MEMCACHE_SIZE, 0,
757762 				     SLAB_CACHE_DMA, NULL);
758763 	if (!qi_cache) {
759764 		dev_err(qidev, "Can't allocate CAAM cache\n");
760765 		free_rsp_fqs();
761
-		platform_device_unregister(qi_pdev);
762766 		return -ENOMEM;
763767 	}
764768 
765
-#ifdef CONFIG_DEBUG_FS
766
-	debugfs_create_file("qi_congested", 0444, ctrlpriv->ctl,
767
-			    &times_congested, &caam_fops_u64_ro);
768
-#endif
769
+	caam_debugfs_qi_init(ctrlpriv);
770
+
771
+	err = devm_add_action_or_reset(qidev, caam_qi_shutdown, ctrlpriv);
772
+	if (err)
773
+		return err;
774
+
769775 	dev_info(qidev, "Linux CAAM Queue I/F driver initialised\n");
770776 	return 0;
771777 }