// SPDX-License-Identifier: GPL-2.0+
|
#include <linux/init.h>
|
#include <linux/module.h>
|
#include <linux/types.h>
|
#include <linux/io.h>
|
#include <linux/export.h>
|
#include <linux/slab.h>
|
#include <linux/platform_device.h>
|
#include <linux/fs.h>
|
#include <linux/rwsem.h>
|
#include "kpc_dma_driver.h"
|
|
/********** IRQ Handlers **********/
|
static
|
irqreturn_t ndd_irq_handler(int irq, void *dev_id)
|
{
|
struct kpc_dma_device *ldev = (struct kpc_dma_device *)dev_id;
|
|
if ((GetEngineControl(ldev) & ENG_CTL_IRQ_ACTIVE) || (ldev->desc_completed->MyDMAAddr != GetEngineCompletePtr(ldev)))
|
schedule_work(&ldev->irq_work);
|
|
return IRQ_HANDLED;
|
}
|
|
static
|
void ndd_irq_worker(struct work_struct *ws)
|
{
|
struct kpc_dma_descriptor *cur;
|
struct kpc_dma_device *eng = container_of(ws, struct kpc_dma_device, irq_work);
|
|
lock_engine(eng);
|
|
if (GetEngineCompletePtr(eng) == 0)
|
goto out;
|
|
if (eng->desc_completed->MyDMAAddr == GetEngineCompletePtr(eng))
|
goto out;
|
|
cur = eng->desc_completed;
|
do {
|
cur = cur->Next;
|
dev_dbg(&eng->pldev->dev, "Handling completed descriptor %p (acd = %p)\n", cur, cur->acd);
|
BUG_ON(cur == eng->desc_next); // Ordering failure.
|
|
if (cur->DescControlFlags & DMA_DESC_CTL_SOP) {
|
eng->accumulated_bytes = 0;
|
eng->accumulated_flags = 0;
|
}
|
|
eng->accumulated_bytes += cur->DescByteCount;
|
if (cur->DescStatusFlags & DMA_DESC_STS_ERROR)
|
eng->accumulated_flags |= ACD_FLAG_ENG_ACCUM_ERROR;
|
|
if (cur->DescStatusFlags & DMA_DESC_STS_SHORT)
|
eng->accumulated_flags |= ACD_FLAG_ENG_ACCUM_SHORT;
|
|
if (cur->DescControlFlags & DMA_DESC_CTL_EOP) {
|
if (cur->acd)
|
transfer_complete_cb(cur->acd, eng->accumulated_bytes, eng->accumulated_flags | ACD_FLAG_DONE);
|
}
|
|
eng->desc_completed = cur;
|
} while (cur->MyDMAAddr != GetEngineCompletePtr(eng));
|
|
out:
|
SetClearEngineControl(eng, ENG_CTL_IRQ_ACTIVE, 0);
|
|
unlock_engine(eng);
|
}
|
|
/********** DMA Engine Init/Teardown **********/
|
void start_dma_engine(struct kpc_dma_device *eng)
|
{
|
eng->desc_next = eng->desc_pool_first;
|
eng->desc_completed = eng->desc_pool_last;
|
|
// Setup the engine pointer registers
|
SetEngineNextPtr(eng, eng->desc_pool_first);
|
SetEngineSWPtr(eng, eng->desc_pool_first);
|
ClearEngineCompletePtr(eng);
|
|
WriteEngineControl(eng, ENG_CTL_DMA_ENABLE | ENG_CTL_IRQ_ENABLE);
|
}
|
|
int setup_dma_engine(struct kpc_dma_device *eng, u32 desc_cnt)
|
{
|
u32 caps;
|
struct kpc_dma_descriptor *cur;
|
struct kpc_dma_descriptor *next;
|
dma_addr_t next_handle;
|
dma_addr_t head_handle;
|
unsigned int i;
|
int rv;
|
|
caps = GetEngineCapabilities(eng);
|
|
if (WARN(!(caps & ENG_CAP_PRESENT), "%s() called for DMA Engine at %p which isn't present in hardware!\n", __func__, eng))
|
return -ENXIO;
|
|
if (caps & ENG_CAP_DIRECTION)
|
eng->dir = DMA_FROM_DEVICE;
|
else
|
eng->dir = DMA_TO_DEVICE;
|
|
eng->desc_pool_cnt = desc_cnt;
|
eng->desc_pool = dma_pool_create("KPC DMA Descriptors", &eng->pldev->dev, sizeof(struct kpc_dma_descriptor), DMA_DESC_ALIGNMENT, 4096);
|
|
eng->desc_pool_first = dma_pool_alloc(eng->desc_pool, GFP_KERNEL | GFP_DMA, &head_handle);
|
if (!eng->desc_pool_first) {
|
dev_err(&eng->pldev->dev, "%s: couldn't allocate desc_pool_first!\n", __func__);
|
dma_pool_destroy(eng->desc_pool);
|
return -ENOMEM;
|
}
|
|
eng->desc_pool_first->MyDMAAddr = head_handle;
|
clear_desc(eng->desc_pool_first);
|
|
cur = eng->desc_pool_first;
|
for (i = 1 ; i < eng->desc_pool_cnt ; i++) {
|
next = dma_pool_alloc(eng->desc_pool, GFP_KERNEL | GFP_DMA, &next_handle);
|
if (!next)
|
goto done_alloc;
|
|
clear_desc(next);
|
next->MyDMAAddr = next_handle;
|
|
cur->DescNextDescPtr = next_handle;
|
cur->Next = next;
|
cur = next;
|
}
|
|
done_alloc:
|
// Link the last descriptor back to the first, so it's a circular linked list
|
cur->Next = eng->desc_pool_first;
|
cur->DescNextDescPtr = eng->desc_pool_first->MyDMAAddr;
|
|
eng->desc_pool_last = cur;
|
eng->desc_completed = eng->desc_pool_last;
|
|
// Setup work queue
|
INIT_WORK(&eng->irq_work, ndd_irq_worker);
|
|
// Grab IRQ line
|
rv = request_irq(eng->irq, ndd_irq_handler, IRQF_SHARED, KP_DRIVER_NAME_DMA_CONTROLLER, eng);
|
if (rv) {
|
dev_err(&eng->pldev->dev, "%s: failed to request_irq: %d\n", __func__, rv);
|
return rv;
|
}
|
|
// Turn on the engine!
|
start_dma_engine(eng);
|
unlock_engine(eng);
|
|
return 0;
|
}
|
|
void stop_dma_engine(struct kpc_dma_device *eng)
|
{
|
unsigned long timeout;
|
|
// Disable the descriptor engine
|
WriteEngineControl(eng, 0);
|
|
// Wait for descriptor engine to finish current operaion
|
timeout = jiffies + (HZ / 2);
|
while (GetEngineControl(eng) & ENG_CTL_DMA_RUNNING) {
|
if (time_after(jiffies, timeout)) {
|
dev_crit(&eng->pldev->dev, "DMA_RUNNING still asserted!\n");
|
break;
|
}
|
}
|
|
// Request a reset
|
WriteEngineControl(eng, ENG_CTL_DMA_RESET_REQUEST);
|
|
// Wait for reset request to be processed
|
timeout = jiffies + (HZ / 2);
|
while (GetEngineControl(eng) & (ENG_CTL_DMA_RUNNING | ENG_CTL_DMA_RESET_REQUEST)) {
|
if (time_after(jiffies, timeout)) {
|
dev_crit(&eng->pldev->dev, "ENG_CTL_DMA_RESET_REQUEST still asserted!\n");
|
break;
|
}
|
}
|
|
// Request a reset
|
WriteEngineControl(eng, ENG_CTL_DMA_RESET);
|
|
// And wait for reset to complete
|
timeout = jiffies + (HZ / 2);
|
while (GetEngineControl(eng) & ENG_CTL_DMA_RESET) {
|
if (time_after(jiffies, timeout)) {
|
dev_crit(&eng->pldev->dev, "DMA_RESET still asserted!\n");
|
break;
|
}
|
}
|
|
// Clear any persistent bits just to make sure there is no residue from the reset
|
SetClearEngineControl(eng, (ENG_CTL_IRQ_ACTIVE | ENG_CTL_DESC_COMPLETE | ENG_CTL_DESC_ALIGN_ERR | ENG_CTL_DESC_FETCH_ERR | ENG_CTL_SW_ABORT_ERR | ENG_CTL_DESC_CHAIN_END | ENG_CTL_DMA_WAITING_PERSIST), 0);
|
|
// Reset performance counters
|
|
// Completely disable the engine
|
WriteEngineControl(eng, 0);
|
}
|
|
void destroy_dma_engine(struct kpc_dma_device *eng)
|
{
|
struct kpc_dma_descriptor *cur;
|
dma_addr_t cur_handle;
|
unsigned int i;
|
|
stop_dma_engine(eng);
|
|
cur = eng->desc_pool_first;
|
cur_handle = eng->desc_pool_first->MyDMAAddr;
|
|
for (i = 0 ; i < eng->desc_pool_cnt ; i++) {
|
struct kpc_dma_descriptor *next = cur->Next;
|
dma_addr_t next_handle = cur->DescNextDescPtr;
|
|
dma_pool_free(eng->desc_pool, cur, cur_handle);
|
cur_handle = next_handle;
|
cur = next;
|
}
|
|
dma_pool_destroy(eng->desc_pool);
|
|
free_irq(eng->irq, eng);
|
}
|
|
/********** Helper Functions **********/
|
int count_descriptors_available(struct kpc_dma_device *eng)
|
{
|
u32 count = 0;
|
struct kpc_dma_descriptor *cur = eng->desc_next;
|
|
while (cur != eng->desc_completed) {
|
BUG_ON(!cur);
|
count++;
|
cur = cur->Next;
|
}
|
return count;
|
}
|
|
void clear_desc(struct kpc_dma_descriptor *desc)
|
{
|
if (!desc)
|
return;
|
desc->DescByteCount = 0;
|
desc->DescStatusErrorFlags = 0;
|
desc->DescStatusFlags = 0;
|
desc->DescUserControlLS = 0;
|
desc->DescUserControlMS = 0;
|
desc->DescCardAddrLS = 0;
|
desc->DescBufferByteCount = 0;
|
desc->DescCardAddrMS = 0;
|
desc->DescControlFlags = 0;
|
desc->DescSystemAddrLS = 0;
|
desc->DescSystemAddrMS = 0;
|
desc->acd = NULL;
|
}
|
