From 102a0743326a03cd1a1202ceda21e175b7d3575c Mon Sep 17 00:00:00 2001
From: hc <hc@nodka.com>
Date: Tue, 20 Feb 2024 01:20:52 +0000
Subject: [PATCH] add new system file
---
kernel/crypto/crypto_engine.c | 212 +++++++++++++++++++++++++++++++++++-----------------
1 files changed, 143 insertions(+), 69 deletions(-)
diff --git a/kernel/crypto/crypto_engine.c b/kernel/crypto/crypto_engine.c
index 992e8d8..cff21f4 100644
--- a/kernel/crypto/crypto_engine.c
+++ b/kernel/crypto/crypto_engine.c
@@ -1,19 +1,15 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Handle async block request by crypto hardware engine.
*
* Copyright (C) 2016 Linaro, Inc.
*
* Author: Baolin Wang <baolin.wang@linaro.org>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation; either version 2 of the License, or (at your option)
- * any later version.
- *
*/
#include <linux/err.h>
#include <linux/delay.h>
+#include <linux/device.h>
#include <crypto/engine.h>
#include <uapi/linux/sched/types.h>
#include "internal.h"
@@ -27,32 +23,36 @@
* @err: error number
*/
static void crypto_finalize_request(struct crypto_engine *engine,
- struct crypto_async_request *req, int err)
+ struct crypto_async_request *req, int err)
{
unsigned long flags;
- bool finalize_cur_req = false;
+ bool finalize_req = false;
int ret;
struct crypto_engine_ctx *enginectx;
- spin_lock_irqsave(&engine->queue_lock, flags);
- if (engine->cur_req == req)
- finalize_cur_req = true;
- spin_unlock_irqrestore(&engine->queue_lock, flags);
+ /*
+ * If hardware cannot enqueue more requests
+ * and retry mechanism is not supported
+ * make sure we are completing the current request
+ */
+ if (!engine->retry_support) {
+ spin_lock_irqsave(&engine->queue_lock, flags);
+ if (engine->cur_req == req) {
+ finalize_req = true;
+ engine->cur_req = NULL;
+ }
+ spin_unlock_irqrestore(&engine->queue_lock, flags);
+ }
- if (finalize_cur_req) {
+ if (finalize_req || engine->retry_support) {
enginectx = crypto_tfm_ctx(req->tfm);
- if (engine->cur_req_prepared &&
+ if (enginectx->op.prepare_request &&
enginectx->op.unprepare_request) {
ret = enginectx->op.unprepare_request(engine, req);
if (ret)
dev_err(engine->dev, "failed to unprepare request\n");
}
- spin_lock_irqsave(&engine->queue_lock, flags);
- engine->cur_req = NULL;
- engine->cur_req_prepared = false;
- spin_unlock_irqrestore(&engine->queue_lock, flags);
}
-
req->complete(req, err);
kthread_queue_work(engine->kworker, &engine->pump_requests);
@@ -79,7 +79,7 @@
spin_lock_irqsave(&engine->queue_lock, flags);
/* Make sure we are not already running a request */
- if (engine->cur_req)
+ if (!engine->retry_support && engine->cur_req)
goto out;
/* If another context is idling then defer */
@@ -113,13 +113,21 @@
goto out;
}
+start_request:
/* Get the fist request from the engine queue to handle */
backlog = crypto_get_backlog(&engine->queue);
async_req = crypto_dequeue_request(&engine->queue);
if (!async_req)
goto out;
- engine->cur_req = async_req;
+ /*
+ * If hardware doesn't support the retry mechanism,
+ * keep track of the request we are processing now.
+ * We'll need it on completion (crypto_finalize_request).
+ */
+ if (!engine->retry_support)
+ engine->cur_req = async_req;
+
if (backlog)
backlog->complete(backlog, -EINPROGRESS);
@@ -135,7 +143,7 @@
ret = engine->prepare_crypt_hardware(engine);
if (ret) {
dev_err(engine->dev, "failed to prepare crypt hardware\n");
- goto req_err;
+ goto req_err_2;
}
}
@@ -146,28 +154,90 @@
if (ret) {
dev_err(engine->dev, "failed to prepare request: %d\n",
ret);
- goto req_err;
+ goto req_err_2;
}
- engine->cur_req_prepared = true;
}
if (!enginectx->op.do_one_request) {
dev_err(engine->dev, "failed to do request\n");
ret = -EINVAL;
- goto req_err;
+ goto req_err_1;
}
- ret = enginectx->op.do_one_request(engine, async_req);
- if (ret) {
- dev_err(engine->dev, "Failed to do one request from queue: %d\n", ret);
- goto req_err;
- }
- return;
-req_err:
- crypto_finalize_request(engine, async_req, ret);
+ ret = enginectx->op.do_one_request(engine, async_req);
+
+ /* Request unsuccessfully executed by hardware */
+ if (ret < 0) {
+ /*
+ * If hardware queue is full (-ENOSPC), requeue request
+ * regardless of backlog flag.
+ * Otherwise, unprepare and complete the request.
+ */
+ if (!engine->retry_support ||
+ (ret != -ENOSPC)) {
+ dev_err(engine->dev,
+ "Failed to do one request from queue: %d\n",
+ ret);
+ goto req_err_1;
+ }
+ /*
+ * If retry mechanism is supported,
+ * unprepare current request and
+ * enqueue it back into crypto-engine queue.
+ */
+ if (enginectx->op.unprepare_request) {
+ ret = enginectx->op.unprepare_request(engine,
+ async_req);
+ if (ret)
+ dev_err(engine->dev,
+ "failed to unprepare request\n");
+ }
+ spin_lock_irqsave(&engine->queue_lock, flags);
+ /*
+ * If hardware was unable to execute request, enqueue it
+ * back in front of crypto-engine queue, to keep the order
+ * of requests.
+ */
+ crypto_enqueue_request_head(&engine->queue, async_req);
+
+ kthread_queue_work(engine->kworker, &engine->pump_requests);
+ goto out;
+ }
+
+ goto retry;
+
+req_err_1:
+ if (enginectx->op.unprepare_request) {
+ ret = enginectx->op.unprepare_request(engine, async_req);
+ if (ret)
+ dev_err(engine->dev, "failed to unprepare request\n");
+ }
+
+req_err_2:
+ async_req->complete(async_req, ret);
+
+retry:
+ /* If retry mechanism is supported, send new requests to engine */
+ if (engine->retry_support) {
+ spin_lock_irqsave(&engine->queue_lock, flags);
+ goto start_request;
+ }
return;
out:
spin_unlock_irqrestore(&engine->queue_lock, flags);
+
+ /*
+ * Batch requests is possible only if
+ * hardware can enqueue multiple requests
+ */
+ if (engine->do_batch_requests) {
+ ret = engine->do_batch_requests(engine);
+ if (ret)
+ dev_err(engine->dev, "failed to do batch requests: %d\n",
+ ret);
+ }
+
+ return;
}
static void crypto_pump_work(struct kthread_work *work)
@@ -217,20 +287,6 @@
{
return crypto_transfer_request(engine, req, true);
}
-
-/**
- * crypto_transfer_ablkcipher_request_to_engine - transfer one ablkcipher_request
- * to list into the engine queue
- * @engine: the hardware engine
- * @req: the request need to be listed into the engine queue
- * TODO: Remove this function when skcipher conversion is finished
- */
-int crypto_transfer_ablkcipher_request_to_engine(struct crypto_engine *engine,
- struct ablkcipher_request *req)
-{
- return crypto_transfer_request_to_engine(engine, &req->base);
-}
-EXPORT_SYMBOL_GPL(crypto_transfer_ablkcipher_request_to_engine);
/**
* crypto_transfer_aead_request_to_engine - transfer one aead_request
@@ -283,21 +339,6 @@
return crypto_transfer_request_to_engine(engine, &req->base);
}
EXPORT_SYMBOL_GPL(crypto_transfer_skcipher_request_to_engine);
-
-/**
- * crypto_finalize_ablkcipher_request - finalize one ablkcipher_request if
- * the request is done
- * @engine: the hardware engine
- * @req: the request need to be finalized
- * @err: error number
- * TODO: Remove this function when skcipher conversion is finished
- */
-void crypto_finalize_ablkcipher_request(struct crypto_engine *engine,
- struct ablkcipher_request *req, int err)
-{
- return crypto_finalize_request(engine, &req->base, err);
-}
-EXPORT_SYMBOL_GPL(crypto_finalize_ablkcipher_request);
/**
* crypto_finalize_aead_request - finalize one aead_request if
@@ -420,17 +461,28 @@
EXPORT_SYMBOL_GPL(crypto_engine_stop);
/**
- * crypto_engine_alloc_init - allocate crypto hardware engine structure and
- * initialize it.
+ * crypto_engine_alloc_init_and_set - allocate crypto hardware engine structure
+ * and initialize it by setting the maximum number of entries in the software
+ * crypto-engine queue.
* @dev: the device attached with one hardware engine
+ * @retry_support: whether hardware has support for retry mechanism
+ * @cbk_do_batch: pointer to a callback function to be invoked when executing
+ * a batch of requests.
+ * This has the form:
+ * callback(struct crypto_engine *engine)
+ * where:
+ * @engine: the crypto engine structure.
* @rt: whether this queue is set to run as a realtime task
+ * @qlen: maximum size of the crypto-engine queue
*
* This must be called from context that can sleep.
* Return: the crypto engine structure on success, else NULL.
*/
-struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt)
+struct crypto_engine *crypto_engine_alloc_init_and_set(struct device *dev,
+ bool retry_support,
+ int (*cbk_do_batch)(struct crypto_engine *engine),
+ bool rt, int qlen)
{
- struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
struct crypto_engine *engine;
if (!dev)
@@ -445,12 +497,18 @@
engine->running = false;
engine->busy = false;
engine->idling = false;
- engine->cur_req_prepared = false;
+ engine->retry_support = retry_support;
engine->priv_data = dev;
+ /*
+ * Batch requests is possible only if
+ * hardware has support for retry mechanism.
+ */
+ engine->do_batch_requests = retry_support ? cbk_do_batch : NULL;
+
snprintf(engine->name, sizeof(engine->name),
"%s-engine", dev_name(dev));
- crypto_init_queue(&engine->queue, CRYPTO_ENGINE_MAX_QLEN);
+ crypto_init_queue(&engine->queue, qlen);
spin_lock_init(&engine->queue_lock);
engine->kworker = kthread_create_worker(0, "%s", engine->name);
@@ -462,11 +520,27 @@
if (engine->rt) {
dev_info(dev, "will run requests pump with realtime priority\n");
- sched_setscheduler(engine->kworker->task, SCHED_FIFO, ¶m);
+ sched_set_fifo(engine->kworker->task);
}
return engine;
}
+EXPORT_SYMBOL_GPL(crypto_engine_alloc_init_and_set);
+
+/**
+ * crypto_engine_alloc_init - allocate crypto hardware engine structure and
+ * initialize it.
+ * @dev: the device attached with one hardware engine
+ * @rt: whether this queue is set to run as a realtime task
+ *
+ * This must be called from context that can sleep.
+ * Return: the crypto engine structure on success, else NULL.
+ */
+struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt)
+{
+ return crypto_engine_alloc_init_and_set(dev, false, NULL, rt,
+ CRYPTO_ENGINE_MAX_QLEN);
+}
EXPORT_SYMBOL_GPL(crypto_engine_alloc_init);
/**
--
Gitblit v1.6.2