/*************************************************************************/ /*!
|
@File
|
@Title Device Memory Management
|
@Copyright Copyright (c) Imagination Technologies Ltd. All Rights Reserved
|
@Description Front End (nominally Client side part, but now invokable
|
from server too) of device memory management
|
@License Dual MIT/GPLv2
|
|
The contents of this file are subject to the MIT license as set out below.
|
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
of this software and associated documentation files (the "Software"), to deal
|
in the Software without restriction, including without limitation the rights
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
copies of the Software, and to permit persons to whom the Software is
|
furnished to do so, subject to the following conditions:
|
|
The above copyright notice and this permission notice shall be included in
|
all copies or substantial portions of the Software.
|
|
Alternatively, the contents of this file may be used under the terms of
|
the GNU General Public License Version 2 ("GPL") in which case the provisions
|
of GPL are applicable instead of those above.
|
|
If you wish to allow use of your version of this file only under the terms of
|
GPL, and not to allow others to use your version of this file under the terms
|
of the MIT license, indicate your decision by deleting the provisions above
|
and replace them with the notice and other provisions required by GPL as set
|
out in the file called "GPL-COPYING" included in this distribution. If you do
|
not delete the provisions above, a recipient may use your version of this file
|
under the terms of either the MIT license or GPL.
|
|
This License is also included in this distribution in the file called
|
"MIT-COPYING".
|
|
EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
|
PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
|
BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
|
PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
|
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
*/ /**************************************************************************/
|
|
|
|
#include "devicemem.h"
|
#include "img_types.h"
|
#include "pvr_debug.h"
|
#include "pvrsrv_error.h"
|
#include "allocmem.h"
|
#include "ra.h"
|
#include "osfunc.h"
|
#include "osmmap.h"
|
#include "devicemem_utils.h"
|
#include "client_mm_bridge.h"
|
#include "client_cache_bridge.h"
|
#include "services_km.h"
|
|
#if defined(PDUMP)
|
#if defined(__KERNEL__)
|
#include "pdump_km.h"
|
#else
|
#include "client_pdump_bridge.h"
|
#endif
|
#include "devicemem_pdump.h"
|
#endif
|
#if defined(PVR_RI_DEBUG)
|
#include "client_ri_bridge.h"
|
#endif
|
#if defined(SUPPORT_PAGE_FAULT_DEBUG)
|
#include "client_devicememhistory_bridge.h"
|
#endif
|
|
#include "rgx_heaps.h"
|
#if defined(__KERNEL__)
|
#include "pvrsrv.h"
|
#include "rgxdefs_km.h"
|
#include "rgx_bvnc_defs_km.h"
|
#if defined(LINUX)
|
#include "linux/kernel.h"
|
#endif
|
#else
|
#include "rgxdefs.h"
|
#endif
|
|
#if defined(__KERNEL__) && defined(PVR_RI_DEBUG)
|
extern PVRSRV_ERROR RIDumpAllKM(void);
|
#endif
|
|
/*****************************************************************************
|
* Sub allocation internals *
|
*****************************************************************************/
|
|
static PVRSRV_ERROR
|
_AllocateDeviceMemory(SHARED_DEV_CONNECTION hDevConnection,
|
IMG_UINT32 uiLog2Quantum,
|
IMG_DEVMEM_SIZE_T uiSize,
|
IMG_DEVMEM_SIZE_T uiChunkSize,
|
IMG_UINT32 ui32NumPhysChunks,
|
IMG_UINT32 ui32NumVirtChunks,
|
IMG_UINT32 *pui32MappingTable,
|
IMG_DEVMEM_ALIGN_T uiAlign,
|
DEVMEM_FLAGS_T uiFlags,
|
IMG_BOOL bExportable,
|
const IMG_CHAR *pszAnnotation,
|
DEVMEM_IMPORT **ppsImport)
|
{
|
DEVMEM_IMPORT *psImport;
|
DEVMEM_FLAGS_T uiPMRFlags;
|
IMG_HANDLE hPMR;
|
PVRSRV_ERROR eError;
|
|
eError = _DevmemImportStructAlloc(hDevConnection,
|
&psImport);
|
if (eError != PVRSRV_OK)
|
{
|
goto failAlloc;
|
}
|
|
/* Check the size is a multiple of the quantum */
|
PVR_ASSERT((uiSize & ((1ULL<<uiLog2Quantum)-1)) == 0);
|
|
/* Pass only the PMR flags down */
|
uiPMRFlags = uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK;
|
eError = BridgePhysmemNewRamBackedPMR(hDevConnection,
|
uiSize,
|
uiChunkSize,
|
ui32NumPhysChunks,
|
ui32NumVirtChunks,
|
pui32MappingTable,
|
uiLog2Quantum,
|
uiPMRFlags,
|
#if defined(PDUMP)
|
OSStringLength(pszAnnotation) + 1,
|
pszAnnotation,
|
&hPMR);
|
#else
|
1,
|
"",
|
&hPMR);
|
|
PVR_UNREFERENCED_PARAMETER(pszAnnotation);
|
#endif
|
|
|
if (eError != PVRSRV_OK)
|
{
|
/* Our check above should have ensured this the "not page
|
multiple" error never happens */
|
PVR_ASSERT(eError != PVRSRV_ERROR_PMR_NOT_PAGE_MULTIPLE);
|
|
goto failPMR;
|
}
|
|
_DevmemImportStructInit(psImport,
|
uiSize,
|
uiAlign,
|
uiFlags,
|
hPMR,
|
bExportable ? DEVMEM_PROPERTIES_EXPORTABLE : 0);
|
|
*ppsImport = psImport;
|
return PVRSRV_OK;
|
|
failPMR:
|
_DevmemImportDiscard(psImport);
|
failAlloc:
|
PVR_ASSERT(eError != PVRSRV_OK);
|
|
return eError;
|
}
|
|
|
/*****************************************************************************
|
* Sub allocation internals *
|
*****************************************************************************/
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DeviceMemChangeSparse(DEVMEM_MEMDESC *psMemDesc,
|
IMG_UINT32 ui32AllocPageCount,
|
IMG_UINT32 *paui32AllocPageIndices,
|
IMG_UINT32 ui32FreePageCount,
|
IMG_UINT32 *pauiFreePageIndices,
|
SPARSE_MEM_RESIZE_FLAGS uiSparseFlags)
|
{
|
PVRSRV_ERROR eError = PVRSRV_ERROR_INVALID_PARAMS;
|
DEVMEM_IMPORT *psImport = psMemDesc->psImport;
|
SHARED_DEV_CONNECTION hDevConnection;
|
IMG_HANDLE hPMR;
|
IMG_HANDLE hSrvDevMemHeap;
|
POS_LOCK hLock;
|
IMG_DEV_VIRTADDR sDevVAddr;
|
IMG_CPU_VIRTADDR sCpuVAddr;
|
|
if (NULL == psImport)
|
{
|
PVR_DPF((PVR_DBG_ERROR, "%s: Invalid Sparse memory import", __func__));
|
goto e0;
|
}
|
|
hDevConnection = psImport->hDevConnection;
|
hPMR = psImport->hPMR;
|
hLock = psImport->hLock;
|
sDevVAddr = psImport->sDeviceImport.sDevVAddr;
|
sCpuVAddr = psImport->sCPUImport.pvCPUVAddr;
|
|
if (NULL == hDevConnection)
|
{
|
PVR_DPF((PVR_DBG_ERROR, "%s: Invalid Bridge handle", __func__));
|
goto e0;
|
}
|
|
if (NULL == hPMR)
|
{
|
PVR_DPF((PVR_DBG_ERROR, "%s: Invalid PMR handle", __func__));
|
goto e0;
|
}
|
|
if ((uiSparseFlags & SPARSE_RESIZE_BOTH) && (0 == sDevVAddr.uiAddr))
|
{
|
PVR_DPF((PVR_DBG_ERROR, "%s: Invalid Device Virtual Map", __func__));
|
goto e0;
|
}
|
|
if ((uiSparseFlags & SPARSE_MAP_CPU_ADDR) && (0 == sCpuVAddr))
|
{
|
PVR_DPF((PVR_DBG_ERROR, "%s: Invalid CPU Virtual Map", __func__));
|
goto e0;
|
}
|
|
if (psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_SECURE)
|
{
|
PVR_DPF((PVR_DBG_ERROR,
|
"%s: Secure buffers currently do not support sparse changes",
|
__func__));
|
eError = PVRSRV_ERROR_INVALID_PARAMS;
|
goto e0;
|
}
|
|
hSrvDevMemHeap = psImport->sDeviceImport.psHeap->hDevMemServerHeap;
|
|
OSLockAcquire(hLock);
|
|
eError = BridgeChangeSparseMem(hDevConnection,
|
hSrvDevMemHeap,
|
hPMR,
|
ui32AllocPageCount,
|
paui32AllocPageIndices,
|
ui32FreePageCount,
|
pauiFreePageIndices,
|
uiSparseFlags,
|
psImport->uiFlags,
|
sDevVAddr,
|
(IMG_UINT64)((uintptr_t)sCpuVAddr));
|
|
OSLockRelease(hLock);
|
|
#if defined(PVR_RI_DEBUG)
|
if(PVRSRVIsBridgeEnabled(psImport->hDevConnection, PVRSRV_BRIDGE_RI))
|
{
|
BridgeRIUpdateMEMDESCBacking(psImport->hDevConnection,
|
psMemDesc->hRIHandle,
|
((IMG_INT32) ui32AllocPageCount - (IMG_INT32) ui32FreePageCount)
|
* (1 << psImport->sDeviceImport.psHeap->uiLog2Quantum));
|
}
|
#endif
|
|
#if defined(SUPPORT_PAGE_FAULT_DEBUG)
|
if(PVRSRVIsBridgeEnabled(psMemDesc->psImport->hDevConnection, PVRSRV_BRIDGE_DEVICEMEMHISTORY))
|
{
|
static IMG_BOOL bHaveNewAPI = IMG_TRUE;
|
PVRSRV_ERROR eError;
|
|
if(bHaveNewAPI)
|
{
|
eError = BridgeDevicememHistorySparseChange(psMemDesc->psImport->hDevConnection,
|
psMemDesc->psImport->hPMR,
|
psMemDesc->uiOffset,
|
psMemDesc->sDeviceMemDesc.sDevVAddr,
|
psMemDesc->uiAllocSize,
|
psMemDesc->sTraceData.szText,
|
DevmemGetHeapLog2PageSize(psImport->sDeviceImport.psHeap),
|
ui32AllocPageCount,
|
paui32AllocPageIndices,
|
ui32FreePageCount,
|
pauiFreePageIndices,
|
psMemDesc->sTraceData.ui32AllocationIndex,
|
&psMemDesc->sTraceData.ui32AllocationIndex);
|
|
if(eError == PVRSRV_ERROR_BRIDGE_CALL_FAILED)
|
{
|
bHaveNewAPI = IMG_FALSE;
|
}
|
}
|
|
/* no fallback required here.
|
* the old version of devicememhistory doesn't have entry
|
* points for SparseChange
|
*/
|
}
|
#endif
|
|
#ifdef PVRSRV_UNMAP_ON_SPARSE_CHANGE
|
if ((PVRSRV_OK == eError) && (psMemDesc->sCPUMemDesc.ui32RefCount))
|
{
|
/*
|
* Release the CPU Virtual mapping here
|
* the caller is supposed to map entire range again
|
*/
|
DevmemReleaseCpuVirtAddr(psMemDesc);
|
}
|
#endif
|
|
e0:
|
return eError;
|
}
|
|
static void
|
_FreeDeviceMemory(DEVMEM_IMPORT *psImport)
|
{
|
_DevmemImportStructRelease(psImport);
|
}
|
|
static PVRSRV_ERROR
|
_SubAllocImportAlloc(RA_PERARENA_HANDLE hArena,
|
RA_LENGTH_T uiSize,
|
RA_FLAGS_T _flags,
|
const IMG_CHAR *pszAnnotation,
|
/* returned data */
|
RA_BASE_T *puiBase,
|
RA_LENGTH_T *puiActualSize,
|
RA_PERISPAN_HANDLE *phImport)
|
{
|
/* When suballocations need a new lump of memory, the RA calls
|
back here. Later, in the kernel, we must construct a new PMR
|
and a pairing between the new lump of virtual memory and the
|
PMR (whether or not such PMR is backed by physical memory) */
|
DEVMEM_HEAP *psHeap;
|
DEVMEM_IMPORT *psImport;
|
IMG_DEVMEM_ALIGN_T uiAlign;
|
PVRSRV_ERROR eError;
|
IMG_UINT32 ui32MappingTable = 0;
|
DEVMEM_FLAGS_T uiFlags = (DEVMEM_FLAGS_T) _flags;
|
IMG_UINT64 ui64OptionalMapAddress = DEVICEMEM_UTILS_NO_ADDRESS;
|
|
/* Per-arena private handle is, for us, the heap */
|
psHeap = hArena;
|
|
/* align to the l.s.b. of the size... e.g. 96kiB aligned to
|
32kiB. NB: There is an argument to say that the RA should never
|
ask us for Non-power-of-2 size anyway, but I don't want to make
|
that restriction arbitrarily now */
|
uiAlign = uiSize & ~(uiSize-1);
|
#if defined(SUPPORT_PVRSRV_GPUVIRT)
|
/* Technically this is only required for guest drivers due to
|
fw heaps being pre-allocated and pre-mapped resulting in
|
a 1:1 (i.e. virtual : physical) offset correlation but we
|
force this behaviour for all drivers to maintain consistency
|
(i.e. heap->VA uiAlign <= heap->PA uiLog2Quantum) */
|
if (uiAlign > (IMG_DEVMEM_ALIGN_T)(1 << psHeap->uiLog2Quantum))
|
{
|
uiAlign = (IMG_DEVMEM_ALIGN_T)(1 << psHeap->uiLog2Quantum);
|
}
|
#endif
|
|
/* The RA should not have invoked us with a size that is not a
|
multiple of the quantum anyway */
|
PVR_ASSERT((uiSize & ((1ULL<<psHeap->uiLog2Quantum)-1)) == 0);
|
|
eError = _AllocateDeviceMemory(psHeap->psCtx->hDevConnection,
|
psHeap->uiLog2Quantum,
|
uiSize,
|
uiSize,
|
1,
|
1,
|
&ui32MappingTable,
|
uiAlign,
|
uiFlags,
|
IMG_FALSE,
|
pszAnnotation,
|
&psImport);
|
if (eError != PVRSRV_OK)
|
{
|
goto failAlloc;
|
}
|
|
#if defined (PDUMP)
|
/* Keep the annotation in the Devmem layer so we know where suballocations were done from*/
|
psImport->pszAnnotation = OSAllocMem(OSStringLength(pszAnnotation)+1);
|
if (psImport->pszAnnotation == NULL)
|
{
|
eError = PVRSRV_ERROR_OUT_OF_MEMORY;
|
goto failAllocMem;
|
}
|
OSStringNCopy(psImport->pszAnnotation, pszAnnotation, OSStringLength(pszAnnotation)+1);
|
#endif
|
|
#if defined(PVR_RI_DEBUG)
|
if(PVRSRVIsBridgeEnabled(psImport->hDevConnection, PVRSRV_BRIDGE_RI))
|
{
|
eError = BridgeRIWritePMREntry (psImport->hDevConnection,
|
psImport->hPMR,
|
sizeof("PMR sub-allocated"),
|
"PMR sub-allocated",
|
psImport->uiSize);
|
if( eError != PVRSRV_OK)
|
{
|
PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIWritePMREntry failed (eError=%d)", __func__, eError));
|
}
|
}
|
#endif
|
|
/*
|
Suballocations always get mapped into the device was we need to
|
key the RA off something and as we can't export suballocations
|
there is no valid reason to request an allocation an not map it
|
*/
|
eError = _DevmemImportStructDevMap(psHeap,
|
IMG_TRUE,
|
psImport,
|
ui64OptionalMapAddress);
|
if (eError != PVRSRV_OK)
|
{
|
goto failMap;
|
}
|
|
/* Mark this import struct as zeroed so we can save some PDump LDBs
|
* and do not have to CPU map + memset()*/
|
if (uiFlags & PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC)
|
{
|
psImport->uiProperties |= DEVMEM_PROPERTIES_IMPORT_IS_ZEROED;
|
}
|
psImport->uiProperties |= DEVMEM_PROPERTIES_IMPORT_IS_CLEAN;
|
|
*puiBase = psImport->sDeviceImport.sDevVAddr.uiAddr;
|
*puiActualSize = uiSize;
|
*phImport = psImport;
|
|
return PVRSRV_OK;
|
|
/*
|
error exit paths follow
|
*/
|
failMap:
|
#if defined(PDUMP)
|
failAllocMem:
|
OSFreeMem(psImport->pszAnnotation);
|
psImport->pszAnnotation = NULL;
|
#endif
|
_FreeDeviceMemory(psImport);
|
failAlloc:
|
|
return eError;
|
}
|
|
static void
|
_SubAllocImportFree(RA_PERARENA_HANDLE hArena,
|
RA_BASE_T uiBase,
|
RA_PERISPAN_HANDLE hImport)
|
{
|
DEVMEM_IMPORT *psImport = hImport;
|
|
PVR_ASSERT(psImport != NULL);
|
PVR_ASSERT(hArena == psImport->sDeviceImport.psHeap);
|
PVR_ASSERT(uiBase == psImport->sDeviceImport.sDevVAddr.uiAddr);
|
|
_DevmemImportStructDevUnmap(psImport);
|
_DevmemImportStructRelease(psImport);
|
}
|
|
/*****************************************************************************
|
* Devmem context internals *
|
*****************************************************************************/
|
|
static PVRSRV_ERROR
|
_PopulateContextFromBlueprint(struct _DEVMEM_CONTEXT_ *psCtx,
|
DEVMEM_HEAPCFGID uiHeapBlueprintID)
|
{
|
PVRSRV_ERROR eError;
|
PVRSRV_ERROR eError2;
|
struct _DEVMEM_HEAP_ **ppsHeapArray;
|
IMG_UINT32 uiNumHeaps;
|
IMG_UINT32 uiHeapsToUnwindOnError;
|
IMG_UINT32 uiHeapIndex;
|
IMG_DEV_VIRTADDR sDevVAddrBase;
|
IMG_CHAR aszHeapName[DEVMEM_HEAPNAME_MAXLENGTH];
|
IMG_DEVMEM_SIZE_T uiHeapLength;
|
IMG_DEVMEM_LOG2ALIGN_T uiLog2DataPageSize;
|
IMG_DEVMEM_LOG2ALIGN_T uiLog2ImportAlignment;
|
IMG_DEVMEM_LOG2ALIGN_T uiLog2TilingStrideFactor;
|
|
eError = DevmemHeapCount(psCtx->hDevConnection,
|
uiHeapBlueprintID,
|
&uiNumHeaps);
|
if (eError != PVRSRV_OK)
|
{
|
goto e0;
|
}
|
|
if (uiNumHeaps == 0)
|
{
|
ppsHeapArray = NULL;
|
}
|
else
|
{
|
ppsHeapArray = OSAllocMem(sizeof(*ppsHeapArray) * uiNumHeaps);
|
if (ppsHeapArray == NULL)
|
{
|
eError = PVRSRV_ERROR_OUT_OF_MEMORY;
|
goto e0;
|
}
|
}
|
|
uiHeapsToUnwindOnError = 0;
|
|
for (uiHeapIndex = 0; uiHeapIndex < uiNumHeaps; uiHeapIndex++)
|
{
|
eError = DevmemHeapDetails(psCtx->hDevConnection,
|
uiHeapBlueprintID,
|
uiHeapIndex,
|
&aszHeapName[0],
|
sizeof(aszHeapName),
|
&sDevVAddrBase,
|
&uiHeapLength,
|
&uiLog2DataPageSize,
|
&uiLog2ImportAlignment,
|
&uiLog2TilingStrideFactor);
|
if (eError != PVRSRV_OK)
|
{
|
goto e1;
|
}
|
|
eError = DevmemCreateHeap(psCtx,
|
sDevVAddrBase,
|
uiHeapLength,
|
uiLog2DataPageSize,
|
uiLog2ImportAlignment,
|
uiLog2TilingStrideFactor,
|
aszHeapName,
|
uiHeapBlueprintID,
|
&ppsHeapArray[uiHeapIndex]);
|
if (eError != PVRSRV_OK)
|
{
|
goto e1;
|
}
|
|
uiHeapsToUnwindOnError = uiHeapIndex + 1;
|
}
|
|
psCtx->uiAutoHeapCount = uiNumHeaps;
|
psCtx->ppsAutoHeapArray = ppsHeapArray;
|
|
PVR_ASSERT(psCtx->uiNumHeaps >= psCtx->uiAutoHeapCount);
|
PVR_ASSERT(psCtx->uiAutoHeapCount == uiNumHeaps);
|
|
return PVRSRV_OK;
|
|
/*
|
error exit paths
|
*/
|
e1:
|
for (uiHeapIndex = 0; uiHeapIndex < uiHeapsToUnwindOnError; uiHeapIndex++)
|
{
|
eError2 = DevmemDestroyHeap(ppsHeapArray[uiHeapIndex]);
|
PVR_ASSERT(eError2 == PVRSRV_OK);
|
}
|
|
if (uiNumHeaps != 0)
|
{
|
OSFreeMem(ppsHeapArray);
|
}
|
|
e0:
|
PVR_ASSERT(eError != PVRSRV_OK);
|
return eError;
|
}
|
|
static PVRSRV_ERROR
|
_UnpopulateContextFromBlueprint(struct _DEVMEM_CONTEXT_ *psCtx)
|
{
|
PVRSRV_ERROR eReturn = PVRSRV_OK;
|
PVRSRV_ERROR eError2;
|
IMG_UINT32 uiHeapIndex;
|
IMG_BOOL bDoCheck = IMG_TRUE;
|
#if defined(__KERNEL__)
|
PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
|
if (psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK)
|
{
|
bDoCheck = IMG_FALSE;
|
}
|
#endif
|
|
for (uiHeapIndex = 0; uiHeapIndex < psCtx->uiAutoHeapCount; uiHeapIndex++)
|
{
|
if (!psCtx->ppsAutoHeapArray[uiHeapIndex])
|
{
|
continue;
|
}
|
|
eError2 = DevmemDestroyHeap(psCtx->ppsAutoHeapArray[uiHeapIndex]);
|
if (eError2 != PVRSRV_OK)
|
{
|
eReturn = eError2;
|
}
|
else
|
{
|
psCtx->ppsAutoHeapArray[uiHeapIndex] = NULL;
|
}
|
}
|
|
if ((!bDoCheck || (eReturn == PVRSRV_OK)) && psCtx->ppsAutoHeapArray)
|
{
|
OSFreeMem(psCtx->ppsAutoHeapArray);
|
psCtx->ppsAutoHeapArray = NULL;
|
psCtx->uiAutoHeapCount = 0;
|
}
|
|
return eReturn;
|
}
|
|
|
/*****************************************************************************
|
* Devmem context functions *
|
*****************************************************************************/
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemCreateContext(SHARED_DEV_CONNECTION hDevConnection,
|
DEVMEM_HEAPCFGID uiHeapBlueprintID,
|
DEVMEM_CONTEXT **ppsCtxPtr)
|
{
|
PVRSRV_ERROR eError;
|
DEVMEM_CONTEXT *psCtx;
|
/* handle to the server-side counterpart of the device memory
|
context (specifically, for handling mapping to device MMU) */
|
IMG_HANDLE hDevMemServerContext;
|
IMG_HANDLE hPrivData;
|
IMG_BOOL bHeapCfgMetaId = (uiHeapBlueprintID == DEVMEM_HEAPCFG_META);
|
|
if (ppsCtxPtr == NULL)
|
{
|
eError = PVRSRV_ERROR_INVALID_PARAMS;
|
goto e0;
|
}
|
|
psCtx = OSAllocMem(sizeof *psCtx);
|
if (psCtx == NULL)
|
{
|
eError = PVRSRV_ERROR_OUT_OF_MEMORY;
|
goto e0;
|
}
|
|
psCtx->uiNumHeaps = 0;
|
|
psCtx->hDevConnection = hDevConnection;
|
|
/* Create (server-side) Device Memory context */
|
eError = BridgeDevmemIntCtxCreate(psCtx->hDevConnection,
|
bHeapCfgMetaId,
|
&hDevMemServerContext,
|
&hPrivData,
|
&psCtx->ui32CPUCacheLineSize);
|
if (eError != PVRSRV_OK)
|
{
|
goto e1;
|
}
|
|
psCtx->hDevMemServerContext = hDevMemServerContext;
|
psCtx->hPrivData = hPrivData;
|
|
/* automagic heap creation */
|
psCtx->uiAutoHeapCount = 0;
|
|
eError = _PopulateContextFromBlueprint(psCtx, uiHeapBlueprintID);
|
if (eError != PVRSRV_OK)
|
{
|
goto e2;
|
}
|
|
|
*ppsCtxPtr = psCtx;
|
|
|
PVR_ASSERT(psCtx->uiNumHeaps == psCtx->uiAutoHeapCount);
|
return PVRSRV_OK;
|
|
/*
|
error exit paths follow
|
*/
|
|
e2:
|
PVR_ASSERT(psCtx->uiAutoHeapCount == 0);
|
PVR_ASSERT(psCtx->uiNumHeaps == 0);
|
BridgeDevmemIntCtxDestroy(psCtx->hDevConnection, hDevMemServerContext);
|
|
e1:
|
OSFreeMem(psCtx);
|
|
e0:
|
PVR_ASSERT(eError != PVRSRV_OK);
|
return eError;
|
}
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemAcquireDevPrivData(DEVMEM_CONTEXT *psCtx,
|
IMG_HANDLE *hPrivData)
|
{
|
PVRSRV_ERROR eError;
|
|
if ((psCtx == NULL) || (hPrivData == NULL))
|
{
|
eError = PVRSRV_ERROR_INVALID_PARAMS;
|
goto e0;
|
}
|
|
*hPrivData = psCtx->hPrivData;
|
return PVRSRV_OK;
|
|
e0:
|
PVR_ASSERT(eError != PVRSRV_OK);
|
return eError;
|
}
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemReleaseDevPrivData(DEVMEM_CONTEXT *psCtx)
|
{
|
PVRSRV_ERROR eError;
|
|
if (psCtx == NULL)
|
{
|
eError = PVRSRV_ERROR_INVALID_PARAMS;
|
goto e0;
|
}
|
return PVRSRV_OK;
|
|
e0:
|
PVR_ASSERT(eError != PVRSRV_OK);
|
return eError;
|
}
|
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemFindHeapByName(const struct _DEVMEM_CONTEXT_ *psCtx,
|
const IMG_CHAR *pszHeapName,
|
struct _DEVMEM_HEAP_ **ppsHeapRet)
|
{
|
IMG_UINT32 uiHeapIndex;
|
|
/* N.B. This func is only useful for finding "automagic" heaps by name */
|
for (uiHeapIndex = 0;
|
uiHeapIndex < psCtx->uiAutoHeapCount;
|
uiHeapIndex++)
|
{
|
if (!OSStringCompare(psCtx->ppsAutoHeapArray[uiHeapIndex]->pszName, pszHeapName))
|
{
|
*ppsHeapRet = psCtx->ppsAutoHeapArray[uiHeapIndex];
|
return PVRSRV_OK;
|
}
|
}
|
|
return PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_INDEX;
|
}
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemDestroyContext(DEVMEM_CONTEXT *psCtx)
|
{
|
PVRSRV_ERROR eError;
|
IMG_BOOL bDoCheck = IMG_TRUE;
|
|
#if defined(__KERNEL__)
|
PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
|
if (psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK)
|
{
|
bDoCheck = IMG_FALSE;
|
}
|
#endif
|
|
if (psCtx == NULL)
|
{
|
return PVRSRV_ERROR_INVALID_PARAMS;
|
}
|
|
eError = _UnpopulateContextFromBlueprint(psCtx);
|
if (bDoCheck && eError != PVRSRV_OK)
|
{
|
PVR_DPF((PVR_DBG_ERROR,
|
"%s: _UnpopulateContextFromBlueprint failed (%d) leaving %d heaps",
|
__func__, eError, psCtx->uiNumHeaps));
|
goto e1;
|
}
|
|
eError = BridgeDevmemIntCtxDestroy(psCtx->hDevConnection,
|
psCtx->hDevMemServerContext);
|
if (bDoCheck && eError != PVRSRV_OK)
|
{
|
PVR_DPF((PVR_DBG_ERROR,
|
"%s: BridgeDevmemIntCtxDestroy failed (%d)",
|
__func__, eError));
|
goto e1;
|
}
|
|
/* should be no more heaps left */
|
if (bDoCheck && psCtx->uiNumHeaps)
|
{
|
PVR_DPF((PVR_DBG_ERROR,
|
"%s: Additional heaps remain in DEVMEM_CONTEXT",
|
__func__));
|
eError = PVRSRV_ERROR_DEVICEMEM_ADDITIONAL_HEAPS_IN_CONTEXT;
|
goto e1;
|
}
|
|
OSDeviceMemSet(psCtx, 0, sizeof(*psCtx));
|
OSFreeMem(psCtx);
|
|
e1:
|
return eError;
|
}
|
|
/*****************************************************************************
|
* Devmem heap query functions *
|
*****************************************************************************/
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemHeapConfigCount(SHARED_DEV_CONNECTION hDevConnection,
|
IMG_UINT32 *puiNumHeapConfigsOut)
|
{
|
PVRSRV_ERROR eError;
|
|
eError = BridgeHeapCfgHeapConfigCount(hDevConnection,
|
puiNumHeapConfigsOut);
|
return eError;
|
}
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemHeapCount(SHARED_DEV_CONNECTION hDevConnection,
|
IMG_UINT32 uiHeapConfigIndex,
|
IMG_UINT32 *puiNumHeapsOut)
|
{
|
PVRSRV_ERROR eError;
|
|
eError = BridgeHeapCfgHeapCount(hDevConnection,
|
uiHeapConfigIndex,
|
puiNumHeapsOut);
|
return eError;
|
}
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemHeapConfigName(SHARED_DEV_CONNECTION hDevConnection,
|
IMG_UINT32 uiHeapConfigIndex,
|
IMG_CHAR *pszConfigNameOut,
|
IMG_UINT32 uiConfigNameBufSz)
|
{
|
PVRSRV_ERROR eError;
|
|
eError = BridgeHeapCfgHeapConfigName(hDevConnection,
|
uiHeapConfigIndex,
|
uiConfigNameBufSz,
|
pszConfigNameOut);
|
return eError;
|
}
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemHeapDetails(SHARED_DEV_CONNECTION hDevConnection,
|
IMG_UINT32 uiHeapConfigIndex,
|
IMG_UINT32 uiHeapIndex,
|
IMG_CHAR *pszHeapNameOut,
|
IMG_UINT32 uiHeapNameBufSz,
|
IMG_DEV_VIRTADDR *psDevVAddrBaseOut,
|
IMG_DEVMEM_SIZE_T *puiHeapLengthOut,
|
IMG_UINT32 *puiLog2DataPageSizeOut,
|
IMG_UINT32 *puiLog2ImportAlignmentOut,
|
IMG_UINT32 *puiLog2TilingStrideFactor)
|
{
|
PVRSRV_ERROR eError;
|
|
eError = BridgeHeapCfgHeapDetails(hDevConnection,
|
uiHeapConfigIndex,
|
uiHeapIndex,
|
uiHeapNameBufSz,
|
pszHeapNameOut,
|
psDevVAddrBaseOut,
|
puiHeapLengthOut,
|
puiLog2DataPageSizeOut,
|
puiLog2ImportAlignmentOut);
|
|
/* REL/1.8 maintain bridge compatibility
|
* 4:0 - uiLog2ImportAlignment (13--20)
|
* 18:16 - uiLog2TilingStrideFactor (3--4)
|
*/
|
*puiLog2TilingStrideFactor = (*puiLog2ImportAlignmentOut >> 16);
|
*puiLog2ImportAlignmentOut &= 0xffff;
|
|
/* NB: *puiLog2TilingStrideFactor is either 3 or 4 (tiling mode 1 or 0).
|
* If reading from an older KM, *puiLog2TilingStrideFactor will not be set.
|
* If so force to 4 (tiling mode 0), which was the original assumption
|
* before puiLog2TilingStrideFactor was queried.
|
*/
|
if (!*puiLog2TilingStrideFactor)
|
{
|
*puiLog2TilingStrideFactor = 4;
|
}
|
|
VG_MARK_INITIALIZED(pszHeapNameOut,uiHeapNameBufSz);
|
|
return eError;
|
}
|
|
/*****************************************************************************
|
* Devmem heap functions *
|
*****************************************************************************/
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemGetHeapInt(DEVMEM_HEAP *psHeap,
|
IMG_HANDLE *phDevmemHeap)
|
{
|
if (psHeap == NULL)
|
{
|
return PVRSRV_ERROR_INVALID_PARAMS;
|
}
|
*phDevmemHeap = psHeap->hDevMemServerHeap;
|
return PVRSRV_OK;
|
}
|
|
/* See devicemem.h for important notes regarding the arguments
|
to this function */
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemCreateHeap(DEVMEM_CONTEXT *psCtx,
|
IMG_DEV_VIRTADDR sBaseAddress,
|
IMG_DEVMEM_SIZE_T uiLength,
|
IMG_UINT32 ui32Log2Quantum,
|
IMG_UINT32 ui32Log2ImportAlignment,
|
IMG_UINT32 ui32Log2TilingStrideFactor,
|
const IMG_CHAR *pszName,
|
DEVMEM_HEAPCFGID uiHeapBlueprintID,
|
DEVMEM_HEAP **ppsHeapPtr)
|
{
|
PVRSRV_ERROR eError = PVRSRV_OK;
|
PVRSRV_ERROR eError2;
|
DEVMEM_HEAP *psHeap;
|
/* handle to the server-side counterpart of the device memory
|
heap (specifically, for handling mapping to device MMU */
|
IMG_HANDLE hDevMemServerHeap;
|
IMG_BOOL bRANoSplit = IMG_FALSE;
|
|
IMG_CHAR aszBuf[100];
|
IMG_CHAR *pszStr;
|
|
if (ppsHeapPtr == NULL)
|
{
|
eError = PVRSRV_ERROR_INVALID_PARAMS;
|
goto e0;
|
}
|
|
psHeap = OSAllocMem(sizeof *psHeap);
|
if (psHeap == NULL)
|
{
|
eError = PVRSRV_ERROR_OUT_OF_MEMORY;
|
goto e0;
|
}
|
|
/* Need to keep local copy of heap name, so caller may free
|
theirs */
|
pszStr = OSAllocMem(OSStringLength(pszName)+1);
|
if (pszStr == NULL)
|
{
|
eError = PVRSRV_ERROR_OUT_OF_MEMORY;
|
goto e1;
|
}
|
OSStringCopy(pszStr, pszName);
|
psHeap->pszName = pszStr;
|
|
psHeap->uiSize = uiLength;
|
psHeap->sBaseAddress = sBaseAddress;
|
OSAtomicWrite(&psHeap->hImportCount,0);
|
|
OSSNPrintf(aszBuf, sizeof(aszBuf),
|
"NDM heap '%s' (suballocs) ctx:%p",
|
pszName, psCtx);
|
pszStr = OSAllocMem(OSStringLength(aszBuf)+1);
|
if (pszStr == NULL)
|
{
|
eError = PVRSRV_ERROR_OUT_OF_MEMORY;
|
goto e2;
|
}
|
OSStringCopy(pszStr, aszBuf);
|
psHeap->pszSubAllocRAName = pszStr;
|
|
#if defined(PDUMP)
|
/* the META heap is shared globally so a single
|
* physical memory import may be used to satisfy
|
* allocations of different processes.
|
* This is problematic when PDumping because the
|
* physical memory import used to satisfy a new allocation
|
* may actually have been imported (and thus the PDump MALLOC
|
* generated) before the PDump client was started, leading to the
|
* MALLOC being missing.
|
* This is solved by disabling splitting of imports for the META physmem
|
* RA, meaning that every firmware allocation gets its own import, thus
|
* ensuring the MALLOC is present for every allocation made within the
|
* pdump capture range
|
*/
|
if(uiHeapBlueprintID == DEVMEM_HEAPCFG_META)
|
{
|
bRANoSplit = IMG_TRUE;
|
}
|
#else
|
PVR_UNREFERENCED_PARAMETER(uiHeapBlueprintID);
|
#endif
|
|
|
psHeap->psSubAllocRA = RA_Create(psHeap->pszSubAllocRAName,
|
/* Subsequent imports: */
|
ui32Log2Quantum,
|
RA_LOCKCLASS_2,
|
_SubAllocImportAlloc,
|
_SubAllocImportFree,
|
(RA_PERARENA_HANDLE) psHeap,
|
bRANoSplit);
|
if (psHeap->psSubAllocRA == NULL)
|
{
|
eError = PVRSRV_ERROR_DEVICEMEM_UNABLE_TO_CREATE_ARENA;
|
goto e3;
|
}
|
|
psHeap->uiLog2ImportAlignment = ui32Log2ImportAlignment;
|
psHeap->uiLog2TilingStrideFactor = ui32Log2TilingStrideFactor;
|
psHeap->uiLog2Quantum = ui32Log2Quantum;
|
|
if (! OSStringCompare(pszName, RGX_GENERAL_SVM_HEAP_IDENT))
|
{
|
/* The SVM heap normally starts out as this type though
|
it may transition to DEVMEM_HEAP_TYPE_USER_MANAGED
|
on platforms with more processor virtual address
|
bits than device virtual address bits */
|
psHeap->eHeapType = DEVMEM_HEAP_TYPE_KERNEL_MANAGED;
|
}
|
else
|
{
|
psHeap->eHeapType = DEVMEM_HEAP_TYPE_UNKNOWN;
|
}
|
|
OSSNPrintf(aszBuf, sizeof(aszBuf),
|
"NDM heap '%s' (QVM) ctx:%p",
|
pszName, psCtx);
|
pszStr = OSAllocMem(OSStringLength(aszBuf)+1);
|
if (pszStr == NULL)
|
{
|
eError = PVRSRV_ERROR_OUT_OF_MEMORY;
|
goto e4;
|
}
|
OSStringCopy(pszStr, aszBuf);
|
psHeap->pszQuantizedVMRAName = pszStr;
|
|
psHeap->psQuantizedVMRA = RA_Create(psHeap->pszQuantizedVMRAName,
|
/* Subsequent import: */
|
0, RA_LOCKCLASS_1, NULL, NULL,
|
(RA_PERARENA_HANDLE) psHeap,
|
IMG_FALSE);
|
if (psHeap->psQuantizedVMRA == NULL)
|
{
|
eError = PVRSRV_ERROR_DEVICEMEM_UNABLE_TO_CREATE_ARENA;
|
goto e5;
|
}
|
|
if (!RA_Add(psHeap->psQuantizedVMRA,
|
(RA_BASE_T)sBaseAddress.uiAddr,
|
(RA_LENGTH_T)uiLength,
|
(RA_FLAGS_T)0, /* This RA doesn't use or need flags */
|
NULL /* per ispan handle */))
|
{
|
RA_Delete(psHeap->psQuantizedVMRA);
|
eError = PVRSRV_ERROR_DEVICEMEM_UNABLE_TO_CREATE_ARENA;
|
goto e5;
|
}
|
|
psHeap->psCtx = psCtx;
|
|
|
/* Create server-side counterpart of Device Memory heap */
|
eError = BridgeDevmemIntHeapCreate(psCtx->hDevConnection,
|
psCtx->hDevMemServerContext,
|
sBaseAddress,
|
uiLength,
|
ui32Log2Quantum,
|
&hDevMemServerHeap);
|
if (eError != PVRSRV_OK)
|
{
|
goto e6;
|
}
|
psHeap->hDevMemServerHeap = hDevMemServerHeap;
|
|
eError = OSLockCreate(&psHeap->hLock, LOCK_TYPE_PASSIVE);
|
if (eError != PVRSRV_OK)
|
{
|
goto e7;
|
}
|
|
psHeap->psCtx->uiNumHeaps ++;
|
*ppsHeapPtr = psHeap;
|
|
#if defined PVRSRV_NEWDEVMEM_SUPPORT_MEM_TRACKING
|
psHeap->psMemDescList = NULL;
|
#endif /* PVRSRV_NEWDEVMEM_SUPPORT_MEM_TRACKING */
|
|
return PVRSRV_OK;
|
|
/*
|
error exit paths
|
*/
|
e7:
|
eError2 = BridgeDevmemIntHeapDestroy(psCtx->hDevConnection,
|
psHeap->hDevMemServerHeap);
|
PVR_ASSERT (eError2 == PVRSRV_OK);
|
e6:
|
if (psHeap->psQuantizedVMRA)
|
RA_Delete(psHeap->psQuantizedVMRA);
|
e5:
|
if (psHeap->pszQuantizedVMRAName)
|
OSFreeMem(psHeap->pszQuantizedVMRAName);
|
e4:
|
RA_Delete(psHeap->psSubAllocRA);
|
e3:
|
OSFreeMem(psHeap->pszSubAllocRAName);
|
e2:
|
OSFreeMem(psHeap->pszName);
|
e1:
|
OSFreeMem(psHeap);
|
e0:
|
PVR_ASSERT(eError != PVRSRV_OK);
|
return eError;
|
}
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemGetHeapBaseDevVAddr(struct _DEVMEM_HEAP_ *psHeap,
|
IMG_DEV_VIRTADDR *pDevVAddr)
|
{
|
if (psHeap == NULL)
|
{
|
return PVRSRV_ERROR_INVALID_PARAMS;
|
}
|
|
*pDevVAddr = psHeap->sBaseAddress;
|
|
return PVRSRV_OK;
|
}
|
|
IMG_INTERNAL void
|
DevmemExportalignAdjustSizeAndAlign(IMG_UINT32 uiLog2Quantum,
|
IMG_DEVMEM_SIZE_T *puiSize,
|
IMG_DEVMEM_ALIGN_T *puiAlign)
|
{
|
IMG_DEVMEM_SIZE_T uiSize = *puiSize;
|
IMG_DEVMEM_ALIGN_T uiAlign = *puiAlign;
|
|
if ((1ULL << uiLog2Quantum) > uiAlign)
|
{
|
uiAlign = 1ULL << uiLog2Quantum;
|
}
|
uiSize = (uiSize + uiAlign - 1) & ~(uiAlign - 1);
|
|
*puiSize = uiSize;
|
*puiAlign = uiAlign;
|
}
|
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemDestroyHeap(DEVMEM_HEAP *psHeap)
|
{
|
PVRSRV_ERROR eError;
|
IMG_INT uiImportCount;
|
#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
|
IMG_BOOL bDoCheck = IMG_TRUE;
|
#if defined(__KERNEL__)
|
if (PVRSRVGetPVRSRVData()->eServicesState != PVRSRV_SERVICES_STATE_OK)
|
{
|
bDoCheck = IMG_FALSE;
|
}
|
#endif
|
#endif
|
|
if (psHeap == NULL)
|
{
|
return PVRSRV_ERROR_INVALID_PARAMS;
|
}
|
|
uiImportCount = OSAtomicRead(&psHeap->hImportCount);
|
if (uiImportCount > 0)
|
{
|
PVR_DPF((PVR_DBG_ERROR, "%d(%s) leaks remain", uiImportCount, psHeap->pszName));
|
#if defined(__KERNEL__)
|
#if defined(PVR_RI_DEBUG)
|
PVR_DPF((PVR_DBG_ERROR, "Details of remaining allocated device memory (for all processes):"));
|
RIDumpAllKM();
|
#else
|
PVR_DPF((PVR_DBG_ERROR, "Compile with PVR_RI_DEBUG=1 to get a full "
|
"list of all driver allocations."));
|
#endif
|
#endif
|
#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
|
if (bDoCheck)
|
#endif
|
{
|
return PVRSRV_ERROR_DEVICEMEM_ALLOCATIONS_REMAIN_IN_HEAP;
|
}
|
}
|
|
eError = BridgeDevmemIntHeapDestroy(psHeap->psCtx->hDevConnection,
|
psHeap->hDevMemServerHeap);
|
#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
|
if (bDoCheck)
|
#endif
|
{
|
if (eError != PVRSRV_OK)
|
{
|
PVR_DPF((PVR_DBG_ERROR,
|
"%s: BridgeDevmemIntHeapDestroy failed (%d)",
|
__func__, eError));
|
return eError;
|
}
|
}
|
|
PVR_ASSERT(psHeap->psCtx->uiNumHeaps > 0);
|
psHeap->psCtx->uiNumHeaps--;
|
|
OSLockDestroy(psHeap->hLock);
|
|
if (psHeap->psQuantizedVMRA)
|
{
|
RA_Delete(psHeap->psQuantizedVMRA);
|
}
|
if (psHeap->pszQuantizedVMRAName)
|
{
|
OSFreeMem(psHeap->pszQuantizedVMRAName);
|
}
|
|
RA_Delete(psHeap->psSubAllocRA);
|
OSFreeMem(psHeap->pszSubAllocRAName);
|
|
OSFreeMem(psHeap->pszName);
|
|
OSDeviceMemSet(psHeap, 0, sizeof(*psHeap));
|
OSFreeMem(psHeap);
|
|
return PVRSRV_OK;
|
}
|
|
/*****************************************************************************
|
* Devmem allocation/free functions *
|
*****************************************************************************/
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemSubAllocate(IMG_UINT8 uiPreAllocMultiplier,
|
DEVMEM_HEAP *psHeap,
|
IMG_DEVMEM_SIZE_T uiSize,
|
IMG_DEVMEM_ALIGN_T uiAlign,
|
DEVMEM_FLAGS_T uiFlags,
|
const IMG_CHAR *pszText,
|
DEVMEM_MEMDESC **ppsMemDescPtr)
|
{
|
RA_BASE_T uiAllocatedAddr;
|
RA_LENGTH_T uiAllocatedSize;
|
RA_PERISPAN_HANDLE hImport; /* the "import" from which this sub-allocation came */
|
PVRSRV_ERROR eError;
|
DEVMEM_MEMDESC *psMemDesc = NULL;
|
IMG_DEVMEM_OFFSET_T uiOffset = 0;
|
DEVMEM_IMPORT *psImport;
|
IMG_UINT32 ui32CPUCacheLineSize;
|
void *pvAddr;
|
|
IMG_BOOL bImportClean;
|
IMG_BOOL bCPUCleanFlag = PVRSRV_CHECK_CPU_CACHE_CLEAN(uiFlags);
|
IMG_BOOL bZero = PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags);
|
IMG_BOOL bCPUCached = (PVRSRV_CHECK_CPU_CACHE_COHERENT(uiFlags) ||
|
PVRSRV_CHECK_CPU_CACHE_INCOHERENT(uiFlags));
|
IMG_BOOL bGPUCached = (PVRSRV_CHECK_GPU_CACHE_COHERENT(uiFlags) ||
|
PVRSRV_CHECK_GPU_CACHE_INCOHERENT(uiFlags));
|
PVRSRV_CACHE_OP eOp = PVRSRV_CACHE_OP_INVALIDATE;
|
IMG_UINT32 ui32CacheLineSize;
|
|
if (uiFlags & PVRSRV_MEMALLOCFLAG_NO_OSPAGES_ON_ALLOC)
|
{
|
/* Deferred Allocation not supported on SubAllocs*/
|
eError = PVRSRV_ERROR_INVALID_PARAMS;
|
goto failParams;
|
}
|
|
if (psHeap == NULL || psHeap->psCtx == NULL ||ppsMemDescPtr == NULL)
|
{
|
eError = PVRSRV_ERROR_INVALID_PARAMS;
|
goto failParams;
|
}
|
|
#if defined(__KERNEL__)
|
{
|
/* The hDevConnection holds two different types of pointers depending on the
|
* address space in which it is used.
|
* In this instance the variable points to the device node in server */
|
PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)psHeap->psCtx->hDevConnection;
|
ui32CacheLineSize = GET_ROGUE_CACHE_LINE_SIZE(psDevNode->pfnGetDeviceFeatureValue(psDevNode, \
|
RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_BIT_MASK));
|
}
|
#else
|
ui32CacheLineSize = ROGUE_CACHE_LINE_SIZE;
|
#endif
|
|
/* The following logic makes sure that any cached memory is aligned to both the CPU and GPU.
|
* To be aligned on both you have to take the Lowest Common Multiple (LCM) of the cache line sizes of each.
|
* As the possibilities are all powers of 2 then simply the largest number can be picked as the LCM.
|
* Therefore this algorithm just picks the highest from the CPU, GPU and given alignments.
|
*/
|
ui32CPUCacheLineSize = psHeap->psCtx->ui32CPUCacheLineSize;
|
/* If the CPU cache line size is larger than the alignment given then it is the lowest common multiple
|
* Also checking if the allocation is going to be cached on the CPU
|
* Currently there is no check for the validity of the cache coherent option.
|
* In this case, the alignment could be applied but the mode could still fall back to uncached.
|
*/
|
if (ui32CPUCacheLineSize > uiAlign && bCPUCached)
|
{
|
uiAlign = ui32CPUCacheLineSize;
|
}
|
|
/* If the GPU cache line size is larger than the alignment given then it is the lowest common multiple
|
* Also checking if the allocation is going to be cached on the GPU via checking for any of the cached options.
|
* Currently there is no check for the validity of the cache coherent option.
|
* In this case, the alignment could be applied but the mode could still fall back to uncached.
|
*/
|
if (ui32CacheLineSize > uiAlign && bGPUCached)
|
{
|
uiAlign = ui32CacheLineSize;
|
}
|
|
eError = _DevmemValidateParams(uiSize,
|
uiAlign,
|
&uiFlags);
|
if (eError != PVRSRV_OK)
|
{
|
goto failParams;
|
}
|
|
eError =_DevmemMemDescAlloc(&psMemDesc);
|
if (eError != PVRSRV_OK)
|
{
|
goto failMemDescAlloc;
|
}
|
|
/* No request for exportable memory so use the RA */
|
eError = RA_Alloc(psHeap->psSubAllocRA,
|
uiSize,
|
uiPreAllocMultiplier,
|
uiFlags,
|
uiAlign,
|
pszText,
|
&uiAllocatedAddr,
|
&uiAllocatedSize,
|
&hImport);
|
if (PVRSRV_OK != eError)
|
{
|
goto failDeviceMemAlloc;
|
}
|
|
psImport = hImport;
|
|
/* This assignment is assuming the RA returns an hImport where suballocations
|
* can be made from if uiSize is NOT a page multiple of the passed heap.
|
*
|
* So we check if uiSize is a page multiple and mark it as exportable
|
* if it is not.
|
* */
|
if (!(uiSize & ((1 << psHeap->uiLog2Quantum) - 1)) &&
|
(uiPreAllocMultiplier == RA_NO_IMPORT_MULTIPLIER) )
|
{
|
psImport->uiProperties |= DEVMEM_PROPERTIES_EXPORTABLE;
|
}
|
psImport->uiProperties |= DEVMEM_PROPERTIES_SUBALLOCATABLE;
|
|
uiOffset = uiAllocatedAddr - psImport->sDeviceImport.sDevVAddr.uiAddr;
|
|
#if defined(PDUMP)
|
#if defined(__KERNEL__)
|
PDumpCommentWithFlags(PDUMP_NONE,
|
"Suballocated %u Byte for \"%s\" from physical allocation \"%s\"",
|
(IMG_UINT32) uiSize, pszText, psImport->pszAnnotation);
|
#else
|
{
|
IMG_CHAR pszComment[PVRSRV_PDUMP_MAX_COMMENT_SIZE];
|
OSSNPrintf(pszComment,
|
PVRSRV_PDUMP_MAX_COMMENT_SIZE,
|
"Suballocated %u Byte for \"%s\" from physical allocation \"%s\"",
|
(IMG_UINT32) uiSize,
|
pszText,
|
psImport->pszAnnotation);
|
|
BridgePVRSRVPDumpComment(psHeap->psCtx->hDevConnection, pszComment, IMG_FALSE);
|
}
|
#endif
|
#endif
|
|
_DevmemMemDescInit(psMemDesc,
|
uiOffset,
|
psImport,
|
uiSize);
|
|
bImportClean = ((psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_IMPORT_IS_CLEAN) != 0);
|
|
/* Zero the memory */
|
if (bZero)
|
{
|
/* Has the import been zeroed on allocation and were no suballocations returned to it so far? */
|
bImportClean = bImportClean && ((psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_IMPORT_IS_ZEROED) != 0);
|
|
if(!bImportClean)
|
{
|
eOp = PVRSRV_CACHE_OP_FLUSH;
|
|
eError = DevmemAcquireCpuVirtAddr(psMemDesc, &pvAddr);
|
if (eError != PVRSRV_OK)
|
{
|
goto failMaintenance;
|
}
|
|
/* uiSize is a 64-bit quantity whereas the 3rd argument
|
* to OSDeviceMemSet is a 32-bit quantity on 32-bit systems
|
* hence a compiler warning of implicit cast and loss of data.
|
* Added explicit cast and assert to remove warning.
|
*/
|
PVR_ASSERT(uiSize < IMG_UINT32_MAX);
|
|
OSDeviceMemSet(pvAddr, 0x0, (size_t) uiSize);
|
|
DevmemReleaseCpuVirtAddr(psMemDesc);
|
|
#if defined(PDUMP)
|
DevmemPDumpLoadZeroMem(psMemDesc, 0, uiSize, PDUMP_FLAGS_CONTINUOUS);
|
#endif
|
}
|
}
|
|
/* Flush or invalidate */
|
if (bCPUCached && !bImportClean && (bZero || bCPUCleanFlag))
|
{
|
/* BridgeCacheOpQueue _may_ be deferred so use BridgeCacheOpExec
|
to ensure this cache maintenance is actioned immediately */
|
eError = BridgeCacheOpExec (psMemDesc->psImport->hDevConnection,
|
psMemDesc->psImport->hPMR,
|
psMemDesc->uiOffset,
|
psMemDesc->uiAllocSize,
|
eOp);
|
if (eError != PVRSRV_OK)
|
{
|
goto failMaintenance;
|
}
|
}
|
|
#if defined(SUPPORT_PAGE_FAULT_DEBUG)
|
if(PVRSRVIsBridgeEnabled(psMemDesc->psImport->hDevConnection, PVRSRV_BRIDGE_DEVICEMEMHISTORY))
|
{
|
/* copy the allocation descriptive name and size so it can be passed to DevicememHistory when
|
* the allocation gets mapped/unmapped
|
*/
|
OSStringNCopy(psMemDesc->sTraceData.szText, pszText, sizeof(psMemDesc->sTraceData.szText) - 1);
|
}
|
#endif
|
|
#if defined(PVR_RI_DEBUG)
|
if(PVRSRVIsBridgeEnabled(psMemDesc->psImport->hDevConnection, PVRSRV_BRIDGE_RI))
|
{
|
/* Attach RI information */
|
eError = BridgeRIWriteMEMDESCEntry (psMemDesc->psImport->hDevConnection,
|
psMemDesc->psImport->hPMR,
|
OSStringNLength(pszText, RI_MAX_TEXT_LEN),
|
pszText,
|
psMemDesc->uiOffset,
|
uiAllocatedSize,
|
uiAllocatedSize,
|
IMG_FALSE,
|
IMG_FALSE,
|
&(psMemDesc->hRIHandle));
|
if( eError != PVRSRV_OK)
|
{
|
PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIWriteMEMDESCEntry failed (eError=%d)", __func__, eError));
|
}
|
}
|
#else /* if defined(PVR_RI_DEBUG) */
|
PVR_UNREFERENCED_PARAMETER (pszText);
|
#endif /* if defined(PVR_RI_DEBUG) */
|
|
*ppsMemDescPtr = psMemDesc;
|
|
return PVRSRV_OK;
|
|
/*
|
error exit paths follow
|
*/
|
|
failMaintenance:
|
_DevmemMemDescRelease(psMemDesc);
|
psMemDesc = NULL; /* Make sure we don't do a discard after the release */
|
failDeviceMemAlloc:
|
if (psMemDesc)
|
{
|
_DevmemMemDescDiscard(psMemDesc);
|
}
|
failMemDescAlloc:
|
failParams:
|
PVR_ASSERT(eError != PVRSRV_OK);
|
PVR_DPF((PVR_DBG_ERROR,
|
"%s: Failed! Error is %s. Allocation size: %#llX",
|
__func__,
|
PVRSRVGETERRORSTRING(eError),
|
(unsigned long long) uiSize));
|
return eError;
|
}
|
|
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemAllocateExportable(SHARED_DEV_CONNECTION hDevConnection,
|
IMG_DEVMEM_SIZE_T uiSize,
|
IMG_DEVMEM_ALIGN_T uiAlign,
|
IMG_UINT32 uiLog2HeapPageSize,
|
DEVMEM_FLAGS_T uiFlags,
|
const IMG_CHAR *pszText,
|
DEVMEM_MEMDESC **ppsMemDescPtr)
|
{
|
PVRSRV_ERROR eError;
|
DEVMEM_MEMDESC *psMemDesc = NULL;
|
DEVMEM_IMPORT *psImport;
|
IMG_UINT32 ui32MappingTable = 0;
|
|
DevmemExportalignAdjustSizeAndAlign(uiLog2HeapPageSize,
|
&uiSize,
|
&uiAlign);
|
|
eError = _DevmemValidateParams(uiSize,
|
uiAlign,
|
&uiFlags);
|
if (eError != PVRSRV_OK)
|
{
|
goto failParams;
|
}
|
|
eError =_DevmemMemDescAlloc(&psMemDesc);
|
if (eError != PVRSRV_OK)
|
{
|
goto failMemDescAlloc;
|
}
|
|
eError = _AllocateDeviceMemory(hDevConnection,
|
uiLog2HeapPageSize,
|
uiSize,
|
uiSize,
|
1,
|
1,
|
&ui32MappingTable,
|
uiAlign,
|
uiFlags,
|
IMG_TRUE,
|
pszText,
|
&psImport);
|
if (eError != PVRSRV_OK)
|
{
|
goto failDeviceMemAlloc;
|
}
|
|
_DevmemMemDescInit(psMemDesc,
|
0,
|
psImport,
|
uiSize);
|
|
*ppsMemDescPtr = psMemDesc;
|
|
#if defined(SUPPORT_PAGE_FAULT_DEBUG)
|
if(PVRSRVIsBridgeEnabled(psImport->hDevConnection, PVRSRV_BRIDGE_DEVICEMEMHISTORY))
|
{
|
/* copy the allocation descriptive name and size so it can be passed to DevicememHistory when
|
* the allocation gets mapped/unmapped
|
*/
|
OSStringNCopy(psMemDesc->sTraceData.szText, pszText, sizeof(psMemDesc->sTraceData.szText) - 1);
|
}
|
#endif
|
|
#if defined(PVR_RI_DEBUG)
|
if(PVRSRVIsBridgeEnabled(psImport->hDevConnection, PVRSRV_BRIDGE_RI))
|
{
|
eError = BridgeRIWritePMREntry (psImport->hDevConnection,
|
psImport->hPMR,
|
OSStringNLength(pszText, RI_MAX_TEXT_LEN),
|
(IMG_CHAR *)pszText,
|
psImport->uiSize);
|
if( eError != PVRSRV_OK)
|
{
|
PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIWritePMREntry failed (eError=%d)", __func__, eError));
|
}
|
|
/* Attach RI information */
|
eError = BridgeRIWriteMEMDESCEntry (psImport->hDevConnection,
|
psImport->hPMR,
|
sizeof("^"),
|
"^",
|
psMemDesc->uiOffset,
|
uiSize,
|
uiSize,
|
IMG_FALSE,
|
IMG_TRUE,
|
&psMemDesc->hRIHandle);
|
if( eError != PVRSRV_OK)
|
{
|
PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIWriteMEMDESCEntry failed (eError=%d)", __func__, eError));
|
}
|
}
|
#else /* if defined(PVR_RI_DEBUG) */
|
PVR_UNREFERENCED_PARAMETER (pszText);
|
#endif /* if defined(PVR_RI_DEBUG) */
|
|
return PVRSRV_OK;
|
|
/*
|
error exit paths follow
|
*/
|
|
failDeviceMemAlloc:
|
_DevmemMemDescDiscard(psMemDesc);
|
|
failMemDescAlloc:
|
failParams:
|
PVR_ASSERT(eError != PVRSRV_OK);
|
PVR_DPF((PVR_DBG_ERROR,
|
"%s: Failed! Error is %s. Allocation size: %#llX",
|
__func__,
|
PVRSRVGETERRORSTRING(eError),
|
(unsigned long long) uiSize));
|
return eError;
|
}
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemAllocateSparse(SHARED_DEV_CONNECTION hDevConnection,
|
IMG_DEVMEM_SIZE_T uiSize,
|
IMG_DEVMEM_SIZE_T uiChunkSize,
|
IMG_UINT32 ui32NumPhysChunks,
|
IMG_UINT32 ui32NumVirtChunks,
|
IMG_UINT32 *pui32MappingTable,
|
IMG_DEVMEM_ALIGN_T uiAlign,
|
IMG_UINT32 uiLog2HeapPageSize,
|
DEVMEM_FLAGS_T uiFlags,
|
const IMG_CHAR *pszText,
|
DEVMEM_MEMDESC **ppsMemDescPtr)
|
{
|
PVRSRV_ERROR eError;
|
DEVMEM_MEMDESC *psMemDesc = NULL;
|
DEVMEM_IMPORT *psImport;
|
IMG_UINT32 i;
|
|
for (i = 0; i < ui32NumPhysChunks; i++)
|
{
|
if (!(pui32MappingTable[i] < ui32NumVirtChunks))
|
{
|
PVR_DPF((PVR_DBG_ERROR,
|
"A mapping table index exceeds the size of the allocation:"
|
" pui32MappingTable[%u] %u, ui32NumVirtChunks %u ",
|
i,
|
pui32MappingTable[i],
|
ui32NumVirtChunks));
|
eError = PVRSRV_ERROR_INVALID_PARAMS;
|
goto failMemDescAlloc;
|
}
|
}
|
|
DevmemExportalignAdjustSizeAndAlign(uiLog2HeapPageSize,
|
&uiSize,
|
&uiAlign);
|
|
eError = _DevmemValidateParams(uiSize,
|
uiAlign,
|
&uiFlags);
|
if (eError != PVRSRV_OK)
|
{
|
goto failParams;
|
}
|
|
eError =_DevmemMemDescAlloc(&psMemDesc);
|
if (eError != PVRSRV_OK)
|
{
|
goto failMemDescAlloc;
|
}
|
|
eError = _AllocateDeviceMemory(hDevConnection,
|
uiLog2HeapPageSize,
|
uiSize,
|
uiChunkSize,
|
ui32NumPhysChunks,
|
ui32NumVirtChunks,
|
pui32MappingTable,
|
uiAlign,
|
uiFlags,
|
IMG_TRUE,
|
pszText,
|
&psImport);
|
if (eError != PVRSRV_OK)
|
{
|
goto failDeviceMemAlloc;
|
}
|
|
_DevmemMemDescInit(psMemDesc,
|
0,
|
psImport,
|
uiSize);
|
|
#if defined(SUPPORT_PAGE_FAULT_DEBUG)
|
if(PVRSRVIsBridgeEnabled(psImport->hDevConnection, PVRSRV_BRIDGE_DEVICEMEMHISTORY))
|
{
|
/* copy the allocation descriptive name and size so it can be passed to DevicememHistory when
|
* the allocation gets mapped/unmapped
|
*/
|
OSStringNCopy(psMemDesc->sTraceData.szText, pszText, sizeof(psMemDesc->sTraceData.szText) - 1);
|
}
|
#endif
|
|
#if defined(PVR_RI_DEBUG)
|
if(PVRSRVIsBridgeEnabled(psImport->hDevConnection, PVRSRV_BRIDGE_RI))
|
{
|
eError = BridgeRIWritePMREntry (psImport->hDevConnection,
|
psImport->hPMR,
|
OSStringNLength(pszText, RI_MAX_TEXT_LEN),
|
(IMG_CHAR *)pszText,
|
psImport->uiSize);
|
if( eError != PVRSRV_OK)
|
{
|
PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIWritePMREntry failed (eError=%d)", __func__, eError));
|
}
|
|
/* Attach RI information */
|
eError = BridgeRIWriteMEMDESCEntry (psMemDesc->psImport->hDevConnection,
|
psMemDesc->psImport->hPMR,
|
sizeof("^"),
|
"^",
|
psMemDesc->uiOffset,
|
uiSize,
|
ui32NumPhysChunks * uiChunkSize,
|
IMG_FALSE,
|
IMG_TRUE,
|
&psMemDesc->hRIHandle);
|
if( eError != PVRSRV_OK)
|
{
|
PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIWriteMEMDESCEntry failed (eError=%d)", __func__, eError));
|
}
|
}
|
#else /* if defined(PVR_RI_DEBUG) */
|
PVR_UNREFERENCED_PARAMETER (pszText);
|
#endif /* if defined(PVR_RI_DEBUG) */
|
|
*ppsMemDescPtr = psMemDesc;
|
|
return PVRSRV_OK;
|
|
/*
|
error exit paths follow
|
*/
|
|
failDeviceMemAlloc:
|
_DevmemMemDescDiscard(psMemDesc);
|
|
failMemDescAlloc:
|
failParams:
|
PVR_ASSERT(eError != PVRSRV_OK);
|
PVR_DPF((PVR_DBG_ERROR,
|
"%s: Failed! Error is %s. Allocation size: %#llX",
|
__func__,
|
PVRSRVGETERRORSTRING(eError),
|
(unsigned long long) uiSize));
|
return eError;
|
}
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemMakeLocalImportHandle(SHARED_DEV_CONNECTION hBridge,
|
IMG_HANDLE hServerHandle,
|
IMG_HANDLE *hLocalImportHandle)
|
{
|
return BridgePMRMakeLocalImportHandle(hBridge,
|
hServerHandle,
|
hLocalImportHandle);
|
}
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemUnmakeLocalImportHandle(SHARED_DEV_CONNECTION hBridge,
|
IMG_HANDLE hLocalImportHandle)
|
{
|
return BridgePMRUnmakeLocalImportHandle(hBridge, hLocalImportHandle);
|
}
|
|
/*****************************************************************************
|
* Devmem unsecure export functions *
|
*****************************************************************************/
|
|
#if defined(SUPPORT_INSECURE_EXPORT)
|
|
static PVRSRV_ERROR
|
_Mapping_Export(DEVMEM_IMPORT *psImport,
|
DEVMEM_EXPORTHANDLE *phPMRExportHandlePtr,
|
DEVMEM_EXPORTKEY *puiExportKeyPtr,
|
DEVMEM_SIZE_T *puiSize,
|
DEVMEM_LOG2ALIGN_T *puiLog2Contig)
|
{
|
/* Gets an export handle and key for the PMR used for this mapping */
|
/* Can only be done if there are no suballocations for this mapping */
|
|
PVRSRV_ERROR eError;
|
DEVMEM_EXPORTHANDLE hPMRExportHandle;
|
DEVMEM_EXPORTKEY uiExportKey;
|
IMG_DEVMEM_SIZE_T uiSize;
|
IMG_DEVMEM_LOG2ALIGN_T uiLog2Contig;
|
|
if (psImport == NULL)
|
{
|
eError = PVRSRV_ERROR_INVALID_PARAMS;
|
goto failParams;
|
}
|
|
if ((psImport->uiProperties & DEVMEM_PROPERTIES_EXPORTABLE) == 0)
|
{
|
eError = PVRSRV_ERROR_DEVICEMEM_CANT_EXPORT_SUBALLOCATION;
|
goto failParams;
|
}
|
|
eError = BridgePMRExportPMR(psImport->hDevConnection,
|
psImport->hPMR,
|
&hPMRExportHandle,
|
&uiSize,
|
&uiLog2Contig,
|
&uiExportKey);
|
if (eError != PVRSRV_OK)
|
{
|
goto failExport;
|
}
|
|
PVR_ASSERT(uiSize == psImport->uiSize);
|
|
*phPMRExportHandlePtr = hPMRExportHandle;
|
*puiExportKeyPtr = uiExportKey;
|
*puiSize = uiSize;
|
*puiLog2Contig = uiLog2Contig;
|
|
return PVRSRV_OK;
|
|
/*
|
error exit paths follow
|
*/
|
|
failExport:
|
failParams:
|
|
PVR_ASSERT(eError != PVRSRV_OK);
|
return eError;
|
|
}
|
|
static void
|
_Mapping_Unexport(DEVMEM_IMPORT *psImport,
|
DEVMEM_EXPORTHANDLE hPMRExportHandle)
|
{
|
PVRSRV_ERROR eError;
|
|
PVR_ASSERT (psImport != NULL);
|
|
eError = BridgePMRUnexportPMR(psImport->hDevConnection,
|
hPMRExportHandle);
|
PVR_ASSERT(eError == PVRSRV_OK);
|
}
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemExport(DEVMEM_MEMDESC *psMemDesc,
|
DEVMEM_EXPORTCOOKIE *psExportCookie)
|
{
|
/* Caller to provide storage for export cookie struct */
|
PVRSRV_ERROR eError;
|
IMG_HANDLE hPMRExportHandle = 0;
|
IMG_UINT64 uiPMRExportPassword = 0;
|
IMG_DEVMEM_SIZE_T uiSize = 0;
|
IMG_DEVMEM_LOG2ALIGN_T uiLog2Contig = 0;
|
|
if (psMemDesc == NULL || psExportCookie == NULL)
|
{
|
eError = PVRSRV_ERROR_INVALID_PARAMS;
|
goto e0;
|
}
|
|
eError = _Mapping_Export(psMemDesc->psImport,
|
&hPMRExportHandle,
|
&uiPMRExportPassword,
|
&uiSize,
|
&uiLog2Contig);
|
if (eError != PVRSRV_OK)
|
{
|
psExportCookie->uiSize = 0;
|
goto e0;
|
}
|
|
psExportCookie->hPMRExportHandle = hPMRExportHandle;
|
psExportCookie->uiPMRExportPassword = uiPMRExportPassword;
|
psExportCookie->uiSize = uiSize;
|
psExportCookie->uiLog2ContiguityGuarantee = uiLog2Contig;
|
|
return PVRSRV_OK;
|
|
/*
|
error exit paths follow
|
*/
|
|
e0:
|
PVR_ASSERT(eError != PVRSRV_OK);
|
return eError;
|
}
|
|
IMG_INTERNAL void
|
DevmemUnexport(DEVMEM_MEMDESC *psMemDesc,
|
DEVMEM_EXPORTCOOKIE *psExportCookie)
|
{
|
_Mapping_Unexport(psMemDesc->psImport,
|
psExportCookie->hPMRExportHandle);
|
|
psExportCookie->uiSize = 0;
|
}
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemImport(SHARED_DEV_CONNECTION hDevConnection,
|
DEVMEM_EXPORTCOOKIE *psCookie,
|
DEVMEM_FLAGS_T uiFlags,
|
DEVMEM_MEMDESC **ppsMemDescPtr)
|
{
|
DEVMEM_MEMDESC *psMemDesc = NULL;
|
DEVMEM_IMPORT *psImport;
|
IMG_HANDLE hPMR;
|
PVRSRV_ERROR eError;
|
|
if (ppsMemDescPtr == NULL)
|
{
|
eError = PVRSRV_ERROR_INVALID_PARAMS;
|
goto failParams;
|
}
|
|
eError =_DevmemMemDescAlloc(&psMemDesc);
|
if (eError != PVRSRV_OK)
|
{
|
goto failMemDescAlloc;
|
}
|
|
eError = _DevmemImportStructAlloc(hDevConnection,
|
&psImport);
|
if (eError != PVRSRV_OK)
|
{
|
eError = PVRSRV_ERROR_OUT_OF_MEMORY;
|
goto failImportAlloc;
|
}
|
|
/* Get a handle to the PMR (inc refcount) */
|
eError = BridgePMRImportPMR(hDevConnection,
|
psCookie->hPMRExportHandle,
|
psCookie->uiPMRExportPassword,
|
psCookie->uiSize, /* not trusted - just for sanity checks */
|
psCookie->uiLog2ContiguityGuarantee, /* not trusted - just for sanity checks */
|
&hPMR);
|
if (eError != PVRSRV_OK)
|
{
|
goto failImport;
|
}
|
|
_DevmemImportStructInit(psImport,
|
psCookie->uiSize,
|
1ULL << psCookie->uiLog2ContiguityGuarantee,
|
uiFlags,
|
hPMR,
|
DEVMEM_PROPERTIES_IMPORTED |
|
DEVMEM_PROPERTIES_EXPORTABLE);
|
|
_DevmemMemDescInit(psMemDesc,
|
0,
|
psImport,
|
psImport->uiSize);
|
|
*ppsMemDescPtr = psMemDesc;
|
|
#if defined(PVR_RI_DEBUG)
|
if(PVRSRVIsBridgeEnabled(psMemDesc->psImport->hDevConnection, PVRSRV_BRIDGE_RI))
|
{
|
/* Attach RI information */
|
eError = BridgeRIWriteMEMDESCEntry (psMemDesc->psImport->hDevConnection,
|
psMemDesc->psImport->hPMR,
|
sizeof("^"),
|
"^",
|
psMemDesc->uiOffset,
|
psMemDesc->psImport->uiSize,
|
psMemDesc->psImport->uiSize,
|
IMG_TRUE,
|
IMG_FALSE,
|
&psMemDesc->hRIHandle);
|
if( eError != PVRSRV_OK)
|
{
|
PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIWriteMEMDESCEntry failed (eError=%d)", __func__, eError));
|
}
|
}
|
#endif /* if defined(PVR_RI_DEBUG) */
|
|
return PVRSRV_OK;
|
|
/*
|
error exit paths follow
|
*/
|
|
failImport:
|
_DevmemImportDiscard(psImport);
|
failImportAlloc:
|
_DevmemMemDescDiscard(psMemDesc);
|
failMemDescAlloc:
|
failParams:
|
PVR_ASSERT(eError != PVRSRV_OK);
|
|
return eError;
|
}
|
|
#endif /* SUPPORT_INSECURE_EXPORT */
|
|
/*****************************************************************************
|
* Common MemDesc functions *
|
*****************************************************************************/
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemUnpin(DEVMEM_MEMDESC *psMemDesc)
|
{
|
PVRSRV_ERROR eError = PVRSRV_OK;
|
DEVMEM_IMPORT *psImport = psMemDesc->psImport;
|
|
/* Stop if the allocation might have suballocations. */
|
if (!(psImport->uiProperties & DEVMEM_PROPERTIES_EXPORTABLE))
|
{
|
eError = PVRSRV_ERROR_INVALID_PARAMS;
|
PVR_DPF((PVR_DBG_ERROR,
|
"%s: The passed allocation is not valid to unpin because "
|
"there might be suballocations on it. Make sure you allocate a page multiple "
|
"of the heap when using PVRSRVAllocDeviceMem()",
|
__FUNCTION__));
|
|
goto e_exit;
|
}
|
|
/* Stop if the Import is still mapped to CPU */
|
if (psImport->sCPUImport.ui32RefCount)
|
{
|
eError = PVRSRV_ERROR_STILL_MAPPED;
|
PVR_DPF((PVR_DBG_ERROR,
|
"%s: There are still %u references on the CPU mapping. "
|
"Please remove all CPU mappings before unpinning.",
|
__FUNCTION__,
|
psImport->sCPUImport.ui32RefCount));
|
|
goto e_exit;
|
}
|
|
/* Only unpin if it is not already unpinned
|
* Return PVRSRV_OK */
|
if (psImport->uiProperties & DEVMEM_PROPERTIES_UNPINNED)
|
{
|
goto e_exit;
|
}
|
|
/* Unpin it and invalidate mapping */
|
if (psImport->sDeviceImport.bMapped == IMG_TRUE)
|
{
|
eError = BridgeDevmemIntUnpinInvalidate(psImport->hDevConnection,
|
psImport->sDeviceImport.hMapping,
|
psImport->hPMR);
|
}
|
else
|
{
|
/* Or just unpin it */
|
eError = BridgeDevmemIntUnpin(psImport->hDevConnection,
|
psImport->hPMR);
|
}
|
|
/* Update flags and RI when call was successful */
|
if (eError == PVRSRV_OK)
|
{
|
psImport->uiProperties |= DEVMEM_PROPERTIES_UNPINNED;
|
#if defined(PVR_RI_DEBUG)
|
if(PVRSRVIsBridgeEnabled(psMemDesc->psImport->hDevConnection, PVRSRV_BRIDGE_RI))
|
{
|
if (psMemDesc->hRIHandle)
|
{
|
PVRSRV_ERROR eError2;
|
|
eError2 = BridgeRIUpdateMEMDESCPinning(psMemDesc->psImport->hDevConnection,
|
psMemDesc->hRIHandle,
|
IMG_FALSE);
|
|
if( eError2 != PVRSRV_OK)
|
{
|
PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIUpdateMEMDESCPinningKM failed (eError=%d)",
|
__func__,
|
eError));
|
}
|
}
|
}
|
#endif
|
}
|
else
|
{
|
/* Or just show what went wrong */
|
PVR_DPF((PVR_DBG_ERROR, "%s: Unpin aborted because of error %d",
|
__func__,
|
eError));
|
}
|
|
e_exit:
|
return eError;
|
}
|
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemPin(DEVMEM_MEMDESC *psMemDesc)
|
{
|
PVRSRV_ERROR eError = PVRSRV_OK;
|
DEVMEM_IMPORT *psImport = psMemDesc->psImport;
|
|
/* Only pin if it is unpinned */
|
if ((psImport->uiProperties & DEVMEM_PROPERTIES_UNPINNED) == 0)
|
{
|
goto e_exit;
|
}
|
|
/* Pin it and make mapping valid */
|
if (psImport->sDeviceImport.bMapped)
|
{
|
eError = BridgeDevmemIntPinValidate(psImport->hDevConnection,
|
psImport->sDeviceImport.hMapping,
|
psImport->hPMR);
|
}
|
else
|
{
|
/* Or just pin it */
|
eError = BridgeDevmemIntPin(psImport->hDevConnection,
|
psImport->hPMR);
|
}
|
|
if ( (eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_PMR_NEW_MEMORY) )
|
{
|
psImport->uiProperties &= ~DEVMEM_PROPERTIES_UNPINNED;
|
#if defined(PVR_RI_DEBUG)
|
if(PVRSRVIsBridgeEnabled(psMemDesc->psImport->hDevConnection, PVRSRV_BRIDGE_RI))
|
{
|
if (psMemDesc->hRIHandle)
|
{
|
PVRSRV_ERROR eError2;
|
|
eError2 = BridgeRIUpdateMEMDESCPinning(psMemDesc->psImport->hDevConnection,
|
psMemDesc->hRIHandle,
|
IMG_TRUE);
|
|
if( eError2 != PVRSRV_OK)
|
{
|
PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIUpdateMEMDESCPinningKM failed (eError=%d)",
|
__func__,
|
eError));
|
}
|
}
|
}
|
#endif
|
}
|
else
|
{
|
/* Or just show what went wrong */
|
PVR_DPF((PVR_DBG_ERROR, "%s: Pin aborted because of error %d",
|
__func__,
|
eError));
|
}
|
|
e_exit:
|
return eError;
|
}
|
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemGetSize(DEVMEM_MEMDESC *psMemDesc, IMG_DEVMEM_SIZE_T* puiSize)
|
{
|
PVRSRV_ERROR eError = PVRSRV_OK;
|
|
*puiSize = psMemDesc->uiAllocSize;
|
|
return eError;
|
}
|
|
/*
|
This function is called for freeing any class of memory
|
*/
|
IMG_INTERNAL void
|
DevmemFree(DEVMEM_MEMDESC *psMemDesc)
|
{
|
if (psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_SECURE)
|
{
|
PVR_DPF((PVR_DBG_ERROR,
|
"%s: Please use methods dedicated to secure buffers.",
|
__func__));
|
return;
|
}
|
|
#if defined(PVR_RI_DEBUG)
|
if(PVRSRVIsBridgeEnabled(psMemDesc->psImport->hDevConnection, PVRSRV_BRIDGE_RI))
|
{
|
if (psMemDesc->hRIHandle)
|
{
|
PVRSRV_ERROR eError;
|
|
eError = BridgeRIDeleteMEMDESCEntry(psMemDesc->psImport->hDevConnection,
|
psMemDesc->hRIHandle);
|
if( eError != PVRSRV_OK)
|
{
|
PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIDeleteMEMDESCEntry failed (eError=%d)", __func__, eError));
|
}
|
}
|
}
|
#endif /* if defined(PVR_RI_DEBUG) */
|
_DevmemMemDescRelease(psMemDesc);
|
}
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemMapToDevice(DEVMEM_MEMDESC *psMemDesc,
|
DEVMEM_HEAP *psHeap,
|
IMG_DEV_VIRTADDR *psDevVirtAddr)
|
{
|
DEVMEM_IMPORT *psImport;
|
IMG_DEV_VIRTADDR sDevVAddr;
|
PVRSRV_ERROR eError;
|
IMG_BOOL bMap = IMG_TRUE;
|
|
/* Do not try to map unpinned memory */
|
if (psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_UNPINNED)
|
{
|
eError = PVRSRV_ERROR_INVALID_MAP_REQUEST;
|
goto failFlags;
|
}
|
|
OSLockAcquire(psMemDesc->sDeviceMemDesc.hLock);
|
if (psHeap == NULL)
|
{
|
eError = PVRSRV_ERROR_INVALID_PARAMS;
|
goto failParams;
|
}
|
|
if (psMemDesc->sDeviceMemDesc.ui32RefCount != 0)
|
{
|
eError = PVRSRV_ERROR_DEVICEMEM_ALREADY_MAPPED;
|
goto failCheck;
|
}
|
|
/* Don't map memory for deferred allocations */
|
if (psMemDesc->psImport->uiFlags & PVRSRV_MEMALLOCFLAG_NO_OSPAGES_ON_ALLOC)
|
{
|
PVR_ASSERT(psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_EXPORTABLE);
|
bMap = IMG_FALSE;
|
}
|
|
DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
|
__FUNCTION__,
|
psMemDesc,
|
psMemDesc->sDeviceMemDesc.ui32RefCount,
|
psMemDesc->sDeviceMemDesc.ui32RefCount+1);
|
|
psImport = psMemDesc->psImport;
|
_DevmemMemDescAcquire(psMemDesc);
|
|
eError = _DevmemImportStructDevMap(psHeap,
|
bMap,
|
psImport,
|
DEVICEMEM_UTILS_NO_ADDRESS);
|
if (eError != PVRSRV_OK)
|
{
|
goto failMap;
|
}
|
|
sDevVAddr.uiAddr = psImport->sDeviceImport.sDevVAddr.uiAddr;
|
sDevVAddr.uiAddr += psMemDesc->uiOffset;
|
psMemDesc->sDeviceMemDesc.sDevVAddr = sDevVAddr;
|
psMemDesc->sDeviceMemDesc.ui32RefCount++;
|
|
*psDevVirtAddr = psMemDesc->sDeviceMemDesc.sDevVAddr;
|
|
OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
|
|
#if defined(SUPPORT_PAGE_FAULT_DEBUG)
|
if(PVRSRVIsBridgeEnabled(psMemDesc->psImport->hDevConnection, PVRSRV_BRIDGE_DEVICEMEMHISTORY))
|
{
|
static IMG_BOOL bHaveNewAPI = IMG_TRUE;
|
PVRSRV_ERROR eError;
|
|
if(bHaveNewAPI)
|
{
|
eError = BridgeDevicememHistoryMapNew(psMemDesc->psImport->hDevConnection,
|
psMemDesc->psImport->hPMR,
|
psMemDesc->uiOffset,
|
psMemDesc->sDeviceMemDesc.sDevVAddr,
|
psMemDesc->uiAllocSize,
|
psMemDesc->sTraceData.szText,
|
DevmemGetHeapLog2PageSize(psHeap),
|
psMemDesc->sTraceData.ui32AllocationIndex,
|
&psMemDesc->sTraceData.ui32AllocationIndex);
|
|
if(eError == PVRSRV_ERROR_BRIDGE_CALL_FAILED)
|
{
|
bHaveNewAPI = IMG_FALSE;
|
}
|
}
|
|
if(!bHaveNewAPI)
|
{
|
BridgeDevicememHistoryMap(psMemDesc->psImport->hDevConnection,
|
psMemDesc->sDeviceMemDesc.sDevVAddr,
|
psMemDesc->uiAllocSize,
|
psMemDesc->sTraceData.szText);
|
}
|
}
|
#endif
|
|
#if defined(PVR_RI_DEBUG)
|
if(PVRSRVIsBridgeEnabled(psImport->hDevConnection, PVRSRV_BRIDGE_RI))
|
{
|
if (psMemDesc->hRIHandle)
|
{
|
eError = BridgeRIUpdateMEMDESCAddr(psImport->hDevConnection,
|
psMemDesc->hRIHandle,
|
psImport->sDeviceImport.sDevVAddr);
|
if( eError != PVRSRV_OK)
|
{
|
PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIUpdateMEMDESCAddr failed (eError=%d)", __func__, eError));
|
}
|
}
|
}
|
#endif
|
|
return PVRSRV_OK;
|
|
failMap:
|
_DevmemMemDescRelease(psMemDesc);
|
failCheck:
|
failParams:
|
OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
|
PVR_ASSERT(eError != PVRSRV_OK);
|
|
failFlags:
|
return eError;
|
}
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemMapToDeviceAddress(DEVMEM_MEMDESC *psMemDesc,
|
DEVMEM_HEAP *psHeap,
|
IMG_DEV_VIRTADDR sDevVirtAddr)
|
{
|
DEVMEM_IMPORT *psImport;
|
IMG_DEV_VIRTADDR sDevVAddr;
|
PVRSRV_ERROR eError;
|
IMG_BOOL bMap = IMG_TRUE;
|
|
/* Do not try to map unpinned memory */
|
if (psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_UNPINNED)
|
{
|
eError = PVRSRV_ERROR_INVALID_MAP_REQUEST;
|
goto failFlags;
|
}
|
|
OSLockAcquire(psMemDesc->sDeviceMemDesc.hLock);
|
if (psHeap == NULL)
|
{
|
eError = PVRSRV_ERROR_INVALID_PARAMS;
|
goto failParams;
|
}
|
|
if (psMemDesc->sDeviceMemDesc.ui32RefCount != 0)
|
{
|
eError = PVRSRV_ERROR_DEVICEMEM_ALREADY_MAPPED;
|
goto failCheck;
|
}
|
|
/* Don't map memory for deferred allocations */
|
if (psMemDesc->psImport->uiFlags & PVRSRV_MEMALLOCFLAG_NO_OSPAGES_ON_ALLOC)
|
{
|
PVR_ASSERT(psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_EXPORTABLE);
|
bMap = IMG_FALSE;
|
}
|
|
DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
|
__FUNCTION__,
|
psMemDesc,
|
psMemDesc->sDeviceMemDesc.ui32RefCount,
|
psMemDesc->sDeviceMemDesc.ui32RefCount+1);
|
|
psImport = psMemDesc->psImport;
|
_DevmemMemDescAcquire(psMemDesc);
|
|
eError = _DevmemImportStructDevMap(psHeap,
|
bMap,
|
psImport,
|
sDevVirtAddr.uiAddr);
|
if (eError != PVRSRV_OK)
|
{
|
goto failMap;
|
}
|
|
sDevVAddr.uiAddr = psImport->sDeviceImport.sDevVAddr.uiAddr;
|
sDevVAddr.uiAddr += psMemDesc->uiOffset;
|
psMemDesc->sDeviceMemDesc.sDevVAddr = sDevVAddr;
|
psMemDesc->sDeviceMemDesc.ui32RefCount++;
|
|
OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
|
|
#if defined(SUPPORT_PAGE_FAULT_DEBUG)
|
if(PVRSRVIsBridgeEnabled(psMemDesc->psImport->hDevConnection, PVRSRV_BRIDGE_DEVICEMEMHISTORY))
|
{
|
static IMG_BOOL bHaveNewAPI = IMG_TRUE;
|
PVRSRV_ERROR eError;
|
|
if(bHaveNewAPI)
|
{
|
eError = BridgeDevicememHistoryMapNew(psMemDesc->psImport->hDevConnection,
|
psMemDesc->psImport->hPMR,
|
psMemDesc->uiOffset,
|
psMemDesc->sDeviceMemDesc.sDevVAddr,
|
psMemDesc->uiAllocSize,
|
psMemDesc->sTraceData.szText,
|
DevmemGetHeapLog2PageSize(psHeap),
|
psMemDesc->sTraceData.ui32AllocationIndex,
|
&psMemDesc->sTraceData.ui32AllocationIndex);
|
|
if(eError == PVRSRV_ERROR_BRIDGE_CALL_FAILED)
|
{
|
bHaveNewAPI = IMG_FALSE;
|
}
|
}
|
|
if(!bHaveNewAPI)
|
{
|
BridgeDevicememHistoryMap(psMemDesc->psImport->hDevConnection,
|
psMemDesc->sDeviceMemDesc.sDevVAddr,
|
psMemDesc->uiAllocSize,
|
psMemDesc->sTraceData.szText);
|
}
|
}
|
#endif
|
|
#if defined(PVR_RI_DEBUG)
|
if(PVRSRVIsBridgeEnabled(psImport->hDevConnection, PVRSRV_BRIDGE_RI))
|
{
|
if (psMemDesc->hRIHandle)
|
{
|
eError = BridgeRIUpdateMEMDESCAddr(psImport->hDevConnection,
|
psMemDesc->hRIHandle,
|
psImport->sDeviceImport.sDevVAddr);
|
if( eError != PVRSRV_OK)
|
{
|
PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIUpdateMEMDESCAddr failed (eError=%d)", __func__, eError));
|
}
|
}
|
}
|
#endif
|
|
return PVRSRV_OK;
|
|
failMap:
|
_DevmemMemDescRelease(psMemDesc);
|
failCheck:
|
failParams:
|
OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
|
PVR_ASSERT(eError != PVRSRV_OK);
|
|
failFlags:
|
return eError;
|
}
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemAcquireDevVirtAddr(DEVMEM_MEMDESC *psMemDesc,
|
IMG_DEV_VIRTADDR *psDevVirtAddr)
|
{
|
PVRSRV_ERROR eError;
|
|
/* Do not try to map unpinned memory */
|
if (psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_UNPINNED)
|
{
|
eError = PVRSRV_ERROR_INVALID_MAP_REQUEST;
|
goto failCheck;
|
}
|
|
OSLockAcquire(psMemDesc->sDeviceMemDesc.hLock);
|
DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
|
__FUNCTION__,
|
psMemDesc,
|
psMemDesc->sDeviceMemDesc.ui32RefCount,
|
psMemDesc->sDeviceMemDesc.ui32RefCount+1);
|
|
if (psMemDesc->sDeviceMemDesc.ui32RefCount == 0)
|
{
|
eError = PVRSRV_ERROR_DEVICEMEM_NO_MAPPING;
|
goto failRelease;
|
}
|
psMemDesc->sDeviceMemDesc.ui32RefCount++;
|
|
*psDevVirtAddr = psMemDesc->sDeviceMemDesc.sDevVAddr;
|
OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
|
|
return PVRSRV_OK;
|
|
failRelease:
|
OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
|
PVR_ASSERT(eError != PVRSRV_OK);
|
failCheck:
|
return eError;
|
}
|
|
IMG_INTERNAL void
|
DevmemReleaseDevVirtAddr(DEVMEM_MEMDESC *psMemDesc)
|
{
|
PVR_ASSERT(psMemDesc != NULL);
|
|
OSLockAcquire(psMemDesc->sDeviceMemDesc.hLock);
|
DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
|
__FUNCTION__,
|
psMemDesc,
|
psMemDesc->sDeviceMemDesc.ui32RefCount,
|
psMemDesc->sDeviceMemDesc.ui32RefCount-1);
|
|
PVR_ASSERT(psMemDesc->sDeviceMemDesc.ui32RefCount != 0);
|
|
if (--psMemDesc->sDeviceMemDesc.ui32RefCount == 0)
|
{
|
#if defined(SUPPORT_PAGE_FAULT_DEBUG)
|
if(PVRSRVIsBridgeEnabled(psMemDesc->psImport->hDevConnection, PVRSRV_BRIDGE_DEVICEMEMHISTORY))
|
{
|
static IMG_BOOL bHaveNewAPI = IMG_TRUE;
|
PVRSRV_ERROR eError;
|
|
if(bHaveNewAPI)
|
{
|
eError = BridgeDevicememHistoryUnmapNew(psMemDesc->psImport->hDevConnection,
|
psMemDesc->psImport->hPMR,
|
psMemDesc->uiOffset,
|
psMemDesc->sDeviceMemDesc.sDevVAddr,
|
psMemDesc->uiAllocSize,
|
psMemDesc->sTraceData.szText,
|
DevmemGetHeapLog2PageSize(psMemDesc->psImport->sDeviceImport.psHeap),
|
psMemDesc->sTraceData.ui32AllocationIndex,
|
&psMemDesc->sTraceData.ui32AllocationIndex);
|
|
if(eError == PVRSRV_ERROR_BRIDGE_CALL_FAILED)
|
{
|
bHaveNewAPI = IMG_FALSE;
|
}
|
}
|
|
if(!bHaveNewAPI)
|
{
|
BridgeDevicememHistoryUnmap(psMemDesc->psImport->hDevConnection,
|
psMemDesc->sDeviceMemDesc.sDevVAddr,
|
psMemDesc->uiAllocSize,
|
psMemDesc->sTraceData.szText);
|
}
|
}
|
#endif
|
_DevmemImportStructDevUnmap(psMemDesc->psImport);
|
OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
|
|
_DevmemMemDescRelease(psMemDesc);
|
}
|
else
|
{
|
OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
|
}
|
}
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemAcquireCpuVirtAddr(DEVMEM_MEMDESC *psMemDesc,
|
void **ppvCpuVirtAddr)
|
{
|
PVRSRV_ERROR eError;
|
|
if ( psMemDesc->psImport->uiProperties &
|
(DEVMEM_PROPERTIES_UNPINNED | DEVMEM_PROPERTIES_SECURE) )
|
{
|
PVR_DPF((PVR_DBG_ERROR,
|
"%s: Allocation is currently unpinned or a secure buffer. "
|
"Not possible to map to CPU!",
|
__func__));
|
eError = PVRSRV_ERROR_INVALID_MAP_REQUEST;
|
goto failFlags;
|
}
|
|
OSLockAcquire(psMemDesc->sCPUMemDesc.hLock);
|
DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
|
__FUNCTION__,
|
psMemDesc,
|
psMemDesc->sCPUMemDesc.ui32RefCount,
|
psMemDesc->sCPUMemDesc.ui32RefCount+1);
|
|
if (psMemDesc->sCPUMemDesc.ui32RefCount++ == 0)
|
{
|
DEVMEM_IMPORT *psImport = psMemDesc->psImport;
|
IMG_UINT8 *pui8CPUVAddr;
|
|
_DevmemMemDescAcquire(psMemDesc);
|
eError = _DevmemImportStructCPUMap(psImport);
|
if (eError != PVRSRV_OK)
|
{
|
goto failMap;
|
}
|
|
pui8CPUVAddr = psImport->sCPUImport.pvCPUVAddr;
|
pui8CPUVAddr += psMemDesc->uiOffset;
|
psMemDesc->sCPUMemDesc.pvCPUVAddr = pui8CPUVAddr;
|
}
|
*ppvCpuVirtAddr = psMemDesc->sCPUMemDesc.pvCPUVAddr;
|
|
VG_MARK_INITIALIZED(*ppvCpuVirtAddr, psMemDesc->psImport->uiSize);
|
|
OSLockRelease(psMemDesc->sCPUMemDesc.hLock);
|
|
return PVRSRV_OK;
|
|
failMap:
|
PVR_ASSERT(eError != PVRSRV_OK);
|
psMemDesc->sCPUMemDesc.ui32RefCount--;
|
_DevmemMemDescRelease(psMemDesc);
|
OSLockRelease(psMemDesc->sCPUMemDesc.hLock);
|
failFlags:
|
return eError;
|
}
|
|
IMG_INTERNAL void
|
DevmemReleaseCpuVirtAddr(DEVMEM_MEMDESC *psMemDesc)
|
{
|
PVR_ASSERT(psMemDesc != NULL);
|
|
OSLockAcquire(psMemDesc->sCPUMemDesc.hLock);
|
DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
|
__FUNCTION__,
|
psMemDesc,
|
psMemDesc->sCPUMemDesc.ui32RefCount,
|
psMemDesc->sCPUMemDesc.ui32RefCount-1);
|
|
PVR_ASSERT(psMemDesc->sCPUMemDesc.ui32RefCount != 0);
|
|
if (--psMemDesc->sCPUMemDesc.ui32RefCount == 0)
|
{
|
OSLockRelease(psMemDesc->sCPUMemDesc.hLock);
|
_DevmemImportStructCPUUnmap(psMemDesc->psImport);
|
_DevmemMemDescRelease(psMemDesc);
|
}
|
else
|
{
|
OSLockRelease(psMemDesc->sCPUMemDesc.hLock);
|
}
|
}
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemLocalGetImportHandle(DEVMEM_MEMDESC *psMemDesc,
|
IMG_HANDLE *phImport)
|
{
|
if ((psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_EXPORTABLE) == 0)
|
{
|
return PVRSRV_ERROR_DEVICEMEM_CANT_EXPORT_SUBALLOCATION;
|
}
|
|
*phImport = psMemDesc->psImport->hPMR;
|
|
return PVRSRV_OK;
|
}
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemGetImportUID(DEVMEM_MEMDESC *psMemDesc,
|
IMG_UINT64 *pui64UID)
|
{
|
DEVMEM_IMPORT *psImport = psMemDesc->psImport;
|
PVRSRV_ERROR eError;
|
|
eError = BridgePMRGetUID(psImport->hDevConnection,
|
psImport->hPMR,
|
pui64UID);
|
|
return eError;
|
}
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemGetReservation(DEVMEM_MEMDESC *psMemDesc,
|
IMG_HANDLE *hReservation)
|
{
|
DEVMEM_IMPORT *psImport;
|
|
PVR_ASSERT(psMemDesc);
|
psImport = psMemDesc->psImport;
|
|
PVR_ASSERT(psImport);
|
*hReservation = psImport->sDeviceImport.hReservation;
|
|
return PVRSRV_OK;
|
}
|
|
PVRSRV_ERROR
|
DevmemGetPMRData(DEVMEM_MEMDESC *psMemDesc,
|
IMG_HANDLE *phPMR,
|
IMG_DEVMEM_OFFSET_T *puiPMROffset)
|
{
|
DEVMEM_IMPORT *psImport;
|
|
PVR_ASSERT(psMemDesc);
|
*puiPMROffset = psMemDesc->uiOffset;
|
psImport = psMemDesc->psImport;
|
|
PVR_ASSERT(psImport);
|
*phPMR = psImport->hPMR;
|
|
return PVRSRV_OK;
|
}
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemGetFlags(DEVMEM_MEMDESC *psMemDesc,
|
DEVMEM_FLAGS_T *puiFlags)
|
{
|
DEVMEM_IMPORT *psImport;
|
|
PVR_ASSERT(psMemDesc);
|
psImport = psMemDesc->psImport;
|
|
PVR_ASSERT(psImport);
|
*puiFlags = psImport->uiFlags;
|
|
return PVRSRV_OK;
|
}
|
|
IMG_INTERNAL IMG_HANDLE
|
DevmemGetConnection(DEVMEM_MEMDESC *psMemDesc)
|
{
|
return psMemDesc->psImport->hDevConnection;
|
}
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemLocalImport(IMG_HANDLE hBridge,
|
IMG_HANDLE hExtHandle,
|
DEVMEM_FLAGS_T uiFlags,
|
DEVMEM_MEMDESC **ppsMemDescPtr,
|
IMG_DEVMEM_SIZE_T *puiSizePtr,
|
const IMG_CHAR *pszAnnotation)
|
{
|
DEVMEM_MEMDESC *psMemDesc = NULL;
|
DEVMEM_IMPORT *psImport;
|
IMG_DEVMEM_SIZE_T uiSize;
|
IMG_DEVMEM_ALIGN_T uiAlign;
|
IMG_HANDLE hPMR;
|
PVRSRV_ERROR eError;
|
|
if (ppsMemDescPtr == NULL)
|
{
|
eError = PVRSRV_ERROR_INVALID_PARAMS;
|
goto failParams;
|
}
|
|
eError =_DevmemMemDescAlloc(&psMemDesc);
|
if (eError != PVRSRV_OK)
|
{
|
goto failMemDescAlloc;
|
}
|
|
eError = _DevmemImportStructAlloc(hBridge,
|
&psImport);
|
if (eError != PVRSRV_OK)
|
{
|
eError = PVRSRV_ERROR_OUT_OF_MEMORY;
|
goto failImportAlloc;
|
}
|
|
/* Get the PMR handle and its size from the server */
|
eError = BridgePMRLocalImportPMR(hBridge,
|
hExtHandle,
|
&hPMR,
|
&uiSize,
|
&uiAlign);
|
if (eError != PVRSRV_OK)
|
{
|
goto failImport;
|
}
|
|
_DevmemImportStructInit(psImport,
|
uiSize,
|
uiAlign,
|
uiFlags,
|
hPMR,
|
DEVMEM_PROPERTIES_IMPORTED |
|
DEVMEM_PROPERTIES_EXPORTABLE);
|
|
_DevmemMemDescInit(psMemDesc,
|
0,
|
psImport,
|
uiSize);
|
|
*ppsMemDescPtr = psMemDesc;
|
if (puiSizePtr)
|
*puiSizePtr = uiSize;
|
|
#if defined(PVR_RI_DEBUG)
|
if(PVRSRVIsBridgeEnabled(psMemDesc->psImport->hDevConnection, PVRSRV_BRIDGE_RI))
|
{
|
/* Attach RI information.
|
* Set backed size to 0 since this allocation has been allocated
|
* by the same process and has been accounted for. */
|
eError = BridgeRIWriteMEMDESCEntry (psMemDesc->psImport->hDevConnection,
|
psMemDesc->psImport->hPMR,
|
sizeof("^"),
|
"^",
|
psMemDesc->uiOffset,
|
psMemDesc->psImport->uiSize,
|
0,
|
IMG_TRUE,
|
IMG_FALSE,
|
&(psMemDesc->hRIHandle));
|
if( eError != PVRSRV_OK)
|
{
|
PVR_DPF((PVR_DBG_ERROR, "%s: call to BridgeRIWriteMEMDESCEntry failed (eError=%d)", __func__, eError));
|
}
|
}
|
#endif /* if defined(PVR_RI_DEBUG) */
|
|
#if defined(SUPPORT_PAGE_FAULT_DEBUG)
|
if(PVRSRVIsBridgeEnabled(psMemDesc->psImport->hDevConnection, PVRSRV_BRIDGE_DEVICEMEMHISTORY))
|
{
|
/* copy the allocation descriptive name and size so it can be passed to DevicememHistory when
|
* the allocation gets mapped/unmapped
|
*/
|
OSStringNCopy(psMemDesc->sTraceData.szText, pszAnnotation, sizeof(psMemDesc->sTraceData.szText) - 1);
|
}
|
#else
|
PVR_UNREFERENCED_PARAMETER(pszAnnotation);
|
#endif
|
|
return PVRSRV_OK;
|
|
failImport:
|
_DevmemImportDiscard(psImport);
|
failImportAlloc:
|
_DevmemMemDescDiscard(psMemDesc);
|
failMemDescAlloc:
|
failParams:
|
PVR_ASSERT(eError != PVRSRV_OK);
|
|
return eError;
|
}
|
|
IMG_INTERNAL PVRSRV_ERROR
|
DevmemIsDevVirtAddrValid(DEVMEM_CONTEXT *psContext,
|
IMG_DEV_VIRTADDR sDevVAddr)
|
{
|
return BridgeDevmemIsVDevAddrValid(psContext->hDevConnection,
|
psContext->hDevMemServerContext,
|
sDevVAddr);
|
}
|
|
IMG_INTERNAL IMG_UINT32
|
DevmemGetHeapLog2PageSize(DEVMEM_HEAP *psHeap)
|
{
|
return psHeap->uiLog2Quantum;
|
}
|
|
IMG_INTERNAL IMG_UINT32
|
DevmemGetHeapTilingProperties(DEVMEM_HEAP *psHeap,
|
IMG_UINT32 *puiLog2ImportAlignment,
|
IMG_UINT32 *puiLog2TilingStrideFactor)
|
{
|
*puiLog2ImportAlignment = psHeap->uiLog2ImportAlignment;
|
*puiLog2TilingStrideFactor = psHeap->uiLog2TilingStrideFactor;
|
return PVRSRV_OK;
|
}
|
|
/**************************************************************************/ /*!
|
@Function RegisterDevMemPFNotify
|
@Description Registers that the application wants to be signaled when a page
|
fault occurs.
|
|
@Input psContext Memory context the process that would like to
|
be notified about.
|
@Input ui32PID The PID of the calling process.
|
@Input bRegister If true, register. If false, de-register.
|
@Return PVRSRV_ERROR: PVRSRV_OK on success. Otherwise, a PVRSRV_
|
error code
|
*/ /***************************************************************************/
|
IMG_INTERNAL PVRSRV_ERROR
|
RegisterDevmemPFNotify(DEVMEM_CONTEXT *psContext,
|
IMG_UINT32 ui32PID,
|
IMG_BOOL bRegister)
|
{
|
PVRSRV_ERROR eError;
|
|
eError = BridgeDevmemIntRegisterPFNotifyKM(psContext->hDevConnection,
|
psContext->hDevMemServerContext,
|
ui32PID,
|
bRegister);
|
if (eError == PVRSRV_ERROR_BRIDGE_CALL_FAILED)
|
{
|
PVR_DPF((PVR_DBG_ERROR,
|
"%s: Bridge Call Failed: This could suggest a UM/KM miss-match (%d)",
|
__func__,
|
(IMG_INT)(eError)));
|
}
|
|
return eError;
|
}
|
