/*************************************************************************/ /*!
|
@File rgxworkest.c
|
@Title RGX Workload Estimation Functionality
|
@Codingstyle IMG
|
@Copyright Copyright (c) Imagination Technologies Ltd. All Rights Reserved
|
@Description Kernel mode workload estimation functionality.
|
@License Dual MIT/GPLv2
|
|
The contents of this file are subject to the MIT license as set out below.
|
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
of this software and associated documentation files (the "Software"), to deal
|
in the Software without restriction, including without limitation the rights
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
copies of the Software, and to permit persons to whom the Software is
|
furnished to do so, subject to the following conditions:
|
|
The above copyright notice and this permission notice shall be included in
|
all copies or substantial portions of the Software.
|
|
Alternatively, the contents of this file may be used under the terms of
|
the GNU General Public License Version 2 ("GPL") in which case the provisions
|
of GPL are applicable instead of those above.
|
|
If you wish to allow use of your version of this file only under the terms of
|
GPL, and not to allow others to use your version of this file under the terms
|
of the MIT license, indicate your decision by deleting the provisions above
|
and replace them with the notice and other provisions required by GPL as set
|
out in the file called "GPL-COPYING" included in this distribution. If you do
|
not delete the provisions above, a recipient may use your version of this file
|
under the terms of either the MIT license or GPL.
|
|
This License is also included in this distribution in the file called
|
"MIT-COPYING".
|
|
EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
|
PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
|
BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
|
PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
|
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
*/ /**************************************************************************/
|
|
#include "rgxworkest.h"
|
#include "rgxfwutils.h"
|
#include "rgxdevice.h"
|
#include "rgxpdvfs.h"
|
#include "device.h"
|
#include "pvr_debug.h"
|
|
#define ROUND_DOWN_TO_NEAREST_1024(number) (((number) >> 10) << 10)
|
|
void WorkEstRCInit(WORKEST_HOST_DATA *psWorkEstData)
|
{
|
/* Create hash tables for workload matching */
|
psWorkEstData->sWorkloadMatchingDataTA.psWorkloadDataHash =
|
HASH_Create_Extended(WORKLOAD_HASH_SIZE,
|
sizeof(RGX_WORKLOAD_TA3D *),
|
&WorkEstHashFuncTA3D,
|
(HASH_KEY_COMP *)&WorkEstHashCompareTA3D);
|
|
/* Create a lock to protect the hash table */
|
WorkEstHashLockCreate(&(psWorkEstData->sWorkloadMatchingDataTA.psWorkEstHashLock));
|
|
psWorkEstData->sWorkloadMatchingData3D.psWorkloadDataHash =
|
HASH_Create_Extended(WORKLOAD_HASH_SIZE,
|
sizeof(RGX_WORKLOAD_TA3D *),
|
&WorkEstHashFuncTA3D,
|
(HASH_KEY_COMP *)&WorkEstHashCompareTA3D);
|
|
/* Create a lock to protect the hash tables */
|
WorkEstHashLockCreate(&(psWorkEstData->sWorkloadMatchingData3D.psWorkEstHashLock));
|
}
|
|
void WorkEstRCDeInit(WORKEST_HOST_DATA *psWorkEstData,
|
PVRSRV_RGXDEV_INFO *psDevInfo)
|
{
|
HASH_TABLE *psWorkloadDataHash;
|
RGX_WORKLOAD_TA3D *pasWorkloadHashKeys;
|
RGX_WORKLOAD_TA3D *psWorkloadHashKey;
|
IMG_UINT32 ui32i;
|
IMG_UINT64 *paui64WorkloadCycleData;
|
|
pasWorkloadHashKeys = psWorkEstData->sWorkloadMatchingDataTA.asWorkloadHashKeys;
|
paui64WorkloadCycleData = psWorkEstData->sWorkloadMatchingDataTA.aui64HashCycleData;
|
psWorkloadDataHash = psWorkEstData->sWorkloadMatchingDataTA.psWorkloadDataHash;
|
|
if(psWorkloadDataHash)
|
{
|
for(ui32i = 0; ui32i < WORKLOAD_HASH_SIZE; ui32i++)
|
{
|
if(paui64WorkloadCycleData[ui32i] > 0)
|
{
|
psWorkloadHashKey = &pasWorkloadHashKeys[ui32i];
|
HASH_Remove_Extended(psWorkloadDataHash,
|
(uintptr_t*)&psWorkloadHashKey);
|
}
|
}
|
|
HASH_Delete(psWorkloadDataHash);
|
}
|
|
/* Remove the hash lock */
|
WorkEstHashLockDestroy(psWorkEstData->sWorkloadMatchingDataTA.psWorkEstHashLock);
|
|
pasWorkloadHashKeys = psWorkEstData->sWorkloadMatchingData3D.asWorkloadHashKeys;
|
paui64WorkloadCycleData = psWorkEstData->sWorkloadMatchingData3D.aui64HashCycleData;
|
psWorkloadDataHash = psWorkEstData->sWorkloadMatchingData3D.psWorkloadDataHash;
|
|
if(psWorkloadDataHash)
|
{
|
for(ui32i = 0; ui32i < WORKLOAD_HASH_SIZE; ui32i++)
|
{
|
if(paui64WorkloadCycleData[ui32i] > 0)
|
{
|
psWorkloadHashKey = &pasWorkloadHashKeys[ui32i];
|
HASH_Remove_Extended(psWorkloadDataHash,
|
(uintptr_t*)&psWorkloadHashKey);
|
}
|
}
|
|
HASH_Delete(psWorkloadDataHash);
|
}
|
|
/* Remove the hash lock */
|
WorkEstHashLockDestroy(psWorkEstData->sWorkloadMatchingData3D.psWorkEstHashLock);
|
|
return;
|
}
|
|
IMG_BOOL WorkEstHashCompareTA3D(size_t uKeySize,
|
void *pKey1,
|
void *pKey2)
|
{
|
RGX_WORKLOAD_TA3D *psWorkload1;
|
RGX_WORKLOAD_TA3D *psWorkload2;
|
|
if(pKey1 && pKey2)
|
{
|
psWorkload1 = *((RGX_WORKLOAD_TA3D **)pKey1);
|
psWorkload2 = *((RGX_WORKLOAD_TA3D **)pKey2);
|
|
PVR_ASSERT(psWorkload1);
|
PVR_ASSERT(psWorkload2);
|
|
if(psWorkload1->ui32RenderTargetSize == psWorkload2->ui32RenderTargetSize
|
&& psWorkload1->ui32NumberOfDrawCalls == psWorkload2->ui32NumberOfDrawCalls
|
&& psWorkload1->ui32NumberOfIndices == psWorkload2->ui32NumberOfIndices
|
&& psWorkload1->ui32NumberOfMRTs == psWorkload2->ui32NumberOfMRTs)
|
{
|
/* This is added to allow this memory to be freed */
|
*(uintptr_t*)pKey2 = *(uintptr_t*)pKey1;
|
return IMG_TRUE;
|
}
|
}
|
return IMG_FALSE;
|
}
|
|
static inline IMG_UINT32 WorkEstDoHash(IMG_UINT32 ui32Input)
|
{
|
IMG_UINT32 ui32HashPart;
|
|
ui32HashPart = ui32Input;
|
ui32HashPart += (ui32HashPart << 12);
|
ui32HashPart ^= (ui32HashPart >> 22);
|
ui32HashPart += (ui32HashPart << 4);
|
ui32HashPart ^= (ui32HashPart >> 9);
|
ui32HashPart += (ui32HashPart << 10);
|
ui32HashPart ^= (ui32HashPart >> 2);
|
ui32HashPart += (ui32HashPart << 7);
|
ui32HashPart ^= (ui32HashPart >> 12);
|
|
return ui32HashPart;
|
}
|
|
IMG_UINT32 WorkEstHashFuncTA3D(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen)
|
{
|
RGX_WORKLOAD_TA3D *psWorkload = *((RGX_WORKLOAD_TA3D**)pKey);
|
IMG_UINT32 ui32HashKey = 0;
|
PVR_UNREFERENCED_PARAMETER(uHashTabLen);
|
PVR_UNREFERENCED_PARAMETER(uKeySize);
|
|
ui32HashKey += WorkEstDoHash(psWorkload->ui32RenderTargetSize);
|
ui32HashKey += WorkEstDoHash(psWorkload->ui32NumberOfDrawCalls);
|
ui32HashKey += WorkEstDoHash(psWorkload->ui32NumberOfIndices);
|
ui32HashKey += WorkEstDoHash(psWorkload->ui32NumberOfMRTs);
|
|
return ui32HashKey;
|
}
|
|
PVRSRV_ERROR WorkEstPrepare(PVRSRV_RGXDEV_INFO *psDevInfo,
|
WORKEST_HOST_DATA *psWorkEstHostData,
|
WORKLOAD_MATCHING_DATA *psWorkloadMatchingData,
|
IMG_UINT32 ui32RenderTargetSize,
|
IMG_UINT32 ui32NumberOfDrawCalls,
|
IMG_UINT32 ui32NumberOfIndices,
|
IMG_UINT32 ui32NumberOfMRTs,
|
IMG_UINT64 ui64DeadlineInus,
|
RGXFWIF_WORKEST_KICK_DATA *psWorkEstKickData)
|
{
|
PVRSRV_ERROR eError;
|
RGX_WORKLOAD_TA3D *psWorkloadCharacteristics;
|
IMG_UINT64 *pui64CyclePrediction;
|
POS_LOCK psWorkEstHashLock;
|
IMG_UINT64 ui64WorkloadDeadlineInus = ui64DeadlineInus;
|
IMG_UINT64 ui64CurrentTime;
|
HASH_TABLE *psWorkloadDataHash;
|
WORKEST_RETURN_DATA *psReturnData;
|
|
if(psDevInfo == NULL)
|
{
|
PVR_DPF((PVR_DBG_ERROR,"WorkEstPrepare: Device Info not available"));
|
return PVRSRV_ERROR_INVALID_PARAMS;
|
}
|
|
if(psDevInfo->bWorkEstEnabled != IMG_TRUE)
|
{
|
/* No error message to avoid excessive messages */
|
return PVRSRV_OK;
|
}
|
|
if(psWorkEstHostData == NULL)
|
{
|
PVR_DPF((PVR_DBG_ERROR,
|
"WorkEstPrepare: Host data not available"));
|
return PVRSRV_ERROR_INVALID_PARAMS;
|
}
|
|
if(psWorkloadMatchingData == NULL)
|
{
|
PVR_DPF((PVR_DBG_ERROR,
|
"WorkEstPrepare: Workload Matching Data not available"));
|
return PVRSRV_ERROR_INVALID_PARAMS;
|
}
|
|
psWorkloadDataHash = psWorkloadMatchingData->psWorkloadDataHash;
|
if(psWorkloadDataHash == NULL)
|
{
|
PVR_DPF((PVR_DBG_ERROR,"WorkEstPrepare: Hash Table not available"));
|
return PVRSRV_ERROR_INVALID_PARAMS;
|
}
|
|
psWorkEstHashLock = psWorkloadMatchingData->psWorkEstHashLock;
|
if(psWorkEstHashLock == NULL)
|
{
|
PVR_DPF((PVR_DBG_ERROR,
|
"WorkEstPrepare: Hash lock not available"
|
));
|
eError = PVRSRV_ERROR_UNABLE_TO_RETRIEVE_HASH_VALUE;
|
return eError;
|
}
|
|
eError = OSClockMonotonicus64(&ui64CurrentTime);
|
if(eError != PVRSRV_OK)
|
{
|
PVR_DPF((PVR_DBG_ERROR,
|
"WorkEstPrepare: Unable to access System Monotonic clock"));
|
PVR_ASSERT(eError == PVRSRV_OK);
|
return eError;
|
}
|
|
#if defined(SUPPORT_PDVFS)
|
psDevInfo->psDeviceNode->psDevConfig->sDVFS.sPDVFSData.bWorkInFrame = IMG_TRUE;
|
#endif
|
|
/* Set up data for the return path to process the workload */
|
|
/* Any host side data needed for the return path is stored in an array and
|
* only the array's index is passed to and from the firmware. This is a
|
* similar abstraction to using handles but is optimised for this case.
|
*/
|
psReturnData =
|
&psDevInfo->asReturnData[psDevInfo->ui32ReturnDataWO];
|
|
/* The index for the specific data is passed to the FW */
|
psWorkEstKickData->ui64ReturnDataIndex = psDevInfo->ui32ReturnDataWO;
|
|
psDevInfo->ui32ReturnDataWO =
|
(psDevInfo->ui32ReturnDataWO + 1) & RETURN_DATA_ARRAY_WRAP_MASK;
|
|
/* The workload characteristics are needed in the return data for the
|
* matching of future workloads via the hash.
|
*/
|
psWorkloadCharacteristics = &psReturnData->sWorkloadCharacteristics;
|
psWorkloadCharacteristics->ui32RenderTargetSize = ui32RenderTargetSize;
|
psWorkloadCharacteristics->ui32NumberOfDrawCalls = ui32NumberOfDrawCalls;
|
psWorkloadCharacteristics->ui32NumberOfIndices = ui32NumberOfIndices;
|
psWorkloadCharacteristics->ui32NumberOfMRTs = ui32NumberOfMRTs;
|
|
/* The matching data is needed as it holds the hash data. */
|
psReturnData->psWorkloadMatchingData = psWorkloadMatchingData;
|
|
/* The host data for the completion updates */
|
psReturnData->psWorkEstHostData = psWorkEstHostData;
|
if(ui64WorkloadDeadlineInus > ui64CurrentTime)
|
{
|
/* This is rounded to reduce multiple deadlines with a minor spread
|
* flooding the fw workload array.
|
*/
|
psWorkEstKickData->ui64DeadlineInus =
|
ROUND_DOWN_TO_NEAREST_1024(ui64WorkloadDeadlineInus);
|
}
|
else
|
{
|
/* If the deadline has already passed assign as zero to suggest full
|
* frequency
|
*/
|
psWorkEstKickData->ui64DeadlineInus = 0;
|
}
|
|
/* Acquire the lock to access hash */
|
OSLockAcquire(psWorkEstHashLock);
|
|
/* Check if there is a prediction for this workload */
|
pui64CyclePrediction =
|
(IMG_UINT64*) HASH_Retrieve(psWorkloadDataHash,
|
(uintptr_t)psWorkloadCharacteristics);
|
|
/* Release lock */
|
OSLockRelease(psWorkEstHashLock);
|
|
if(pui64CyclePrediction != NULL)
|
{
|
/* Cycle prediction is available, store this prediction */
|
psWorkEstKickData->ui64CyclesPrediction = *pui64CyclePrediction;
|
}
|
else
|
{
|
/* There is no prediction */
|
psWorkEstKickData->ui64CyclesPrediction = 0;
|
}
|
|
return PVRSRV_OK;
|
}
|
|
PVRSRV_ERROR WorkEstWorkloadFinished(PVRSRV_RGXDEV_INFO *psDevInfo,
|
RGXFWIF_WORKEST_FWCCB_CMD *psReturnCmd)
|
{
|
RGX_WORKLOAD_TA3D *psWorkloadCharacteristics;
|
RGX_WORKLOAD_TA3D *pasWorkloadHashKeys;
|
IMG_UINT64 *paui64HashCycleData;
|
IMG_UINT32 *pui32HashArrayWO;
|
RGX_WORKLOAD_TA3D *psWorkloadHashKey;
|
IMG_UINT64 *pui64CyclesTaken;
|
HASH_TABLE *psWorkloadHash;
|
WORKLOAD_MATCHING_DATA *psWorkloadMatchingData;
|
POS_LOCK psWorkEstHashLock;
|
IMG_BOOL bHashSucess;
|
WORKEST_RETURN_DATA *psReturnData;
|
WORKEST_HOST_DATA *psWorkEstHostData;
|
PVRSRV_ERROR eError = PVRSRV_OK;
|
|
if(psDevInfo->bWorkEstEnabled != IMG_TRUE)
|
{
|
/* No error message to avoid excessive messages */
|
return PVRSRV_OK;
|
}
|
|
if(psReturnCmd == NULL)
|
{
|
PVR_DPF((PVR_DBG_ERROR,
|
"WorkEstFinished: Missing Return Command"));
|
return PVRSRV_ERROR_INVALID_PARAMS;
|
}
|
|
if(psReturnCmd->ui64ReturnDataIndex >= RETURN_DATA_ARRAY_SIZE)
|
{
|
PVR_DPF((PVR_DBG_ERROR,
|
"WorkEstFinished: Handle Reference Out of Bounds"));
|
return PVRSRV_ERROR_INVALID_PARAMS;
|
}
|
|
/* Retrieve the return data for this workload */
|
psReturnData = &psDevInfo->asReturnData[psReturnCmd->ui64ReturnDataIndex];
|
|
psWorkEstHostData = psReturnData->psWorkEstHostData;
|
|
if(psWorkEstHostData == NULL)
|
{
|
PVR_DPF((PVR_DBG_ERROR,
|
"WorkEstFinished: Missing host data"));
|
eError = PVRSRV_ERROR_INVALID_PARAMS;
|
return eError;
|
}
|
|
psWorkloadCharacteristics = &psReturnData->sWorkloadCharacteristics;
|
|
if(psWorkloadCharacteristics == NULL)
|
{
|
PVR_DPF((PVR_DBG_ERROR,
|
"WorkEstFinished: Missing workload characteristics"));
|
eError = PVRSRV_ERROR_INVALID_PARAMS;
|
goto hasherror;
|
}
|
|
psWorkloadMatchingData = psReturnData->psWorkloadMatchingData;
|
|
psWorkloadHash = psWorkloadMatchingData->psWorkloadDataHash;
|
if(psWorkloadHash == NULL)
|
{
|
PVR_DPF((PVR_DBG_ERROR,
|
"WorkEstFinished: Missing hash"));
|
eError = PVRSRV_ERROR_INVALID_PARAMS;
|
goto hasherror;
|
}
|
|
psWorkEstHashLock = psWorkloadMatchingData->psWorkEstHashLock;
|
if(psWorkEstHashLock == NULL)
|
{
|
PVR_DPF((PVR_DBG_ERROR,
|
"WorkEstFinished: Missing hash lock"));
|
eError = PVRSRV_ERROR_INVALID_PARAMS;
|
goto hasherror;
|
}
|
|
OSLockAcquire(psWorkEstHashLock);
|
|
pui64CyclesTaken =
|
(IMG_UINT64*) HASH_Remove_Extended(psWorkloadHash,
|
(uintptr_t*)&psWorkloadCharacteristics);
|
|
pui32HashArrayWO = &(psWorkloadMatchingData->ui32HashArrayWO);
|
paui64HashCycleData = psWorkloadMatchingData->aui64HashCycleData;
|
pasWorkloadHashKeys = psWorkloadMatchingData->asWorkloadHashKeys;
|
|
/* Remove the oldest Hash data before it becomes overwritten */
|
if(paui64HashCycleData[*pui32HashArrayWO] > 0)
|
{
|
psWorkloadHashKey = &pasWorkloadHashKeys[*pui32HashArrayWO];
|
HASH_Remove_Extended(psWorkloadHash,
|
(uintptr_t*)&psWorkloadHashKey);
|
}
|
|
if(pui64CyclesTaken == NULL)
|
{
|
/* There is no existing entry for these characteristics. */
|
pasWorkloadHashKeys[*pui32HashArrayWO] = *psWorkloadCharacteristics;
|
|
paui64HashCycleData[*pui32HashArrayWO] = psReturnCmd->ui64CyclesTaken;
|
}
|
else
|
{
|
*pui64CyclesTaken =
|
(*pui64CyclesTaken + psReturnCmd->ui64CyclesTaken)/2;
|
|
pasWorkloadHashKeys[*pui32HashArrayWO] = *psWorkloadCharacteristics;
|
|
paui64HashCycleData[*pui32HashArrayWO] = *pui64CyclesTaken;
|
|
/* Set the old value to 0 so it is known to be invalid */
|
*pui64CyclesTaken = 0;
|
}
|
|
|
bHashSucess = HASH_Insert((HASH_TABLE*)(psWorkloadHash),
|
(uintptr_t)&pasWorkloadHashKeys[*pui32HashArrayWO],
|
(uintptr_t)&paui64HashCycleData[*pui32HashArrayWO]);
|
PVR_ASSERT(bHashSucess);
|
|
if(*pui32HashArrayWO == WORKLOAD_HASH_SIZE-1)
|
{
|
*pui32HashArrayWO = 0;
|
}
|
else
|
{
|
(*pui32HashArrayWO)++;
|
}
|
|
OSLockRelease(psWorkEstHashLock);
|
|
hasherror:
|
|
/* Update the received counter so that the FW is able to check as to whether
|
* all the workloads connected to a render context are finished.
|
*/
|
psWorkEstHostData->ui32WorkEstCCBReceived++;
|
return eError;
|
}
|
|
void WorkEstHashLockCreate(POS_LOCK *psWorkEstHashLock)
|
{
|
if(*psWorkEstHashLock == NULL)
|
{
|
OSLockCreate(psWorkEstHashLock, LOCK_TYPE_DISPATCH);
|
}
|
return;
|
}
|
|
void WorkEstHashLockDestroy(POS_LOCK sWorkEstHashLock)
|
{
|
if(sWorkEstHashLock != NULL)
|
{
|
OSLockDestroy(sWorkEstHashLock);
|
sWorkEstHashLock = NULL;
|
}
|
return;
|
}
|
|
void WorkEstCheckFirmwareCCB(PVRSRV_RGXDEV_INFO *psDevInfo)
|
{
|
RGXFWIF_WORKEST_FWCCB_CMD *psFwCCBCmd;
|
|
RGXFWIF_CCB_CTL *psFWCCBCtl = psDevInfo->psWorkEstFirmwareCCBCtl;
|
IMG_UINT8 *psFWCCB = psDevInfo->psWorkEstFirmwareCCB;
|
|
while (psFWCCBCtl->ui32ReadOffset != psFWCCBCtl->ui32WriteOffset)
|
{
|
/* Point to the next command */
|
psFwCCBCmd = ((RGXFWIF_WORKEST_FWCCB_CMD *)psFWCCB) + psFWCCBCtl->ui32ReadOffset;
|
|
WorkEstWorkloadFinished(psDevInfo, psFwCCBCmd);
|
|
/* Update read offset */
|
psFWCCBCtl->ui32ReadOffset = (psFWCCBCtl->ui32ReadOffset + 1) & psFWCCBCtl->ui32WrapMask;
|
}
|
}
|
