/*
|
* Copyright © 2017 Broadcom
|
*
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
* copy of this software and associated documentation files (the "Software"),
|
* to deal in the Software without restriction, including without limitation
|
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
* and/or sell copies of the Software, and to permit persons to whom the
|
* Software is furnished to do so, subject to the following conditions:
|
*
|
* The above copyright notice and this permission notice (including the next
|
* paragraph) shall be included in all copies or substantial portions of the
|
* Software.
|
*
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
* IN THE SOFTWARE.
|
*/
|
|
/** @file vc4_tiling_lt.c
|
*
|
* Helper functions from vc4_tiling.c that will be compiled for using NEON
|
* assembly or not.
|
*
|
* If VC4_BUILD_NEON is set, then the functions will be suffixed with _neon.
|
* They will only use NEON assembly if __ARM_ARCH is also set, to keep the x86
|
* sim build working.
|
*/
|
|
#include <string.h>
|
#include "pipe/p_state.h"
|
#include "vc4_tiling.h"
|
|
#ifdef VC4_BUILD_NEON
|
#define NEON_TAG(x) x ## _neon
|
#else
|
#define NEON_TAG(x) x ## _base
|
#endif
|
|
/** Returns the stride in bytes of a 64-byte microtile. */
|
static uint32_t
|
vc4_utile_stride(int cpp)
|
{
|
switch (cpp) {
|
case 1:
|
return 8;
|
case 2:
|
case 4:
|
case 8:
|
return 16;
|
default:
|
unreachable("bad cpp");
|
}
|
}
|
|
static void
|
vc4_load_utile(void *cpu, void *gpu, uint32_t cpu_stride, uint32_t cpp)
|
{
|
uint32_t gpu_stride = vc4_utile_stride(cpp);
|
#if defined(VC4_BUILD_NEON) && defined(PIPE_ARCH_ARM)
|
if (gpu_stride == 8) {
|
__asm__ volatile (
|
/* Load from the GPU in one shot, no interleave, to
|
* d0-d7.
|
*/
|
"vldm %0, {q0, q1, q2, q3}\n"
|
/* Store each 8-byte line to cpu-side destination,
|
* incrementing it by the stride each time.
|
*/
|
"vst1.8 d0, [%1], %2\n"
|
"vst1.8 d1, [%1], %2\n"
|
"vst1.8 d2, [%1], %2\n"
|
"vst1.8 d3, [%1], %2\n"
|
"vst1.8 d4, [%1], %2\n"
|
"vst1.8 d5, [%1], %2\n"
|
"vst1.8 d6, [%1], %2\n"
|
"vst1.8 d7, [%1]\n"
|
:
|
: "r"(gpu), "r"(cpu), "r"(cpu_stride)
|
: "q0", "q1", "q2", "q3");
|
} else {
|
assert(gpu_stride == 16);
|
__asm__ volatile (
|
/* Load from the GPU in one shot, no interleave, to
|
* d0-d7.
|
*/
|
"vldm %0, {q0, q1, q2, q3};\n"
|
/* Store each 16-byte line in 2 parts to the cpu-side
|
* destination. (vld1 can only store one d-register
|
* at a time).
|
*/
|
"vst1.8 d0, [%1], %3\n"
|
"vst1.8 d1, [%2], %3\n"
|
"vst1.8 d2, [%1], %3\n"
|
"vst1.8 d3, [%2], %3\n"
|
"vst1.8 d4, [%1], %3\n"
|
"vst1.8 d5, [%2], %3\n"
|
"vst1.8 d6, [%1]\n"
|
"vst1.8 d7, [%2]\n"
|
:
|
: "r"(gpu), "r"(cpu), "r"(cpu + 8), "r"(cpu_stride)
|
: "q0", "q1", "q2", "q3");
|
}
|
#elif defined (PIPE_ARCH_AARCH64)
|
if (gpu_stride == 8) {
|
__asm__ volatile (
|
/* Load from the GPU in one shot, no interleave, to
|
* d0-d7.
|
*/
|
"ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%0]\n"
|
/* Store each 8-byte line to cpu-side destination,
|
* incrementing it by the stride each time.
|
*/
|
"st1 {v0.D}[0], [%1], %2\n"
|
"st1 {v0.D}[1], [%1], %2\n"
|
"st1 {v1.D}[0], [%1], %2\n"
|
"st1 {v1.D}[1], [%1], %2\n"
|
"st1 {v2.D}[0], [%1], %2\n"
|
"st1 {v2.D}[1], [%1], %2\n"
|
"st1 {v3.D}[0], [%1], %2\n"
|
"st1 {v3.D}[1], [%1]\n"
|
:
|
: "r"(gpu), "r"(cpu), "r"(cpu_stride)
|
: "v0", "v1", "v2", "v3");
|
} else {
|
assert(gpu_stride == 16);
|
__asm__ volatile (
|
/* Load from the GPU in one shot, no interleave, to
|
* d0-d7.
|
*/
|
"ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%0]\n"
|
/* Store each 16-byte line in 2 parts to the cpu-side
|
* destination. (vld1 can only store one d-register
|
* at a time).
|
*/
|
"st1 {v0.D}[0], [%1], %3\n"
|
"st1 {v0.D}[1], [%2], %3\n"
|
"st1 {v1.D}[0], [%1], %3\n"
|
"st1 {v1.D}[1], [%2], %3\n"
|
"st1 {v2.D}[0], [%1], %3\n"
|
"st1 {v2.D}[1], [%2], %3\n"
|
"st1 {v3.D}[0], [%1]\n"
|
"st1 {v3.D}[1], [%2]\n"
|
:
|
: "r"(gpu), "r"(cpu), "r"(cpu + 8), "r"(cpu_stride)
|
: "v0", "v1", "v2", "v3");
|
}
|
#else
|
for (uint32_t gpu_offset = 0; gpu_offset < 64; gpu_offset += gpu_stride) {
|
memcpy(cpu, gpu + gpu_offset, gpu_stride);
|
cpu += cpu_stride;
|
}
|
#endif
|
}
|
|
static void
|
vc4_store_utile(void *gpu, void *cpu, uint32_t cpu_stride, uint32_t cpp)
|
{
|
uint32_t gpu_stride = vc4_utile_stride(cpp);
|
|
#if defined(VC4_BUILD_NEON) && defined(PIPE_ARCH_ARM)
|
if (gpu_stride == 8) {
|
__asm__ volatile (
|
/* Load each 8-byte line from cpu-side source,
|
* incrementing it by the stride each time.
|
*/
|
"vld1.8 d0, [%1], %2\n"
|
"vld1.8 d1, [%1], %2\n"
|
"vld1.8 d2, [%1], %2\n"
|
"vld1.8 d3, [%1], %2\n"
|
"vld1.8 d4, [%1], %2\n"
|
"vld1.8 d5, [%1], %2\n"
|
"vld1.8 d6, [%1], %2\n"
|
"vld1.8 d7, [%1]\n"
|
/* Load from the GPU in one shot, no interleave, to
|
* d0-d7.
|
*/
|
"vstm %0, {q0, q1, q2, q3}\n"
|
:
|
: "r"(gpu), "r"(cpu), "r"(cpu_stride)
|
: "q0", "q1", "q2", "q3");
|
} else {
|
assert(gpu_stride == 16);
|
__asm__ volatile (
|
/* Load each 16-byte line in 2 parts from the cpu-side
|
* destination. (vld1 can only store one d-register
|
* at a time).
|
*/
|
"vld1.8 d0, [%1], %3\n"
|
"vld1.8 d1, [%2], %3\n"
|
"vld1.8 d2, [%1], %3\n"
|
"vld1.8 d3, [%2], %3\n"
|
"vld1.8 d4, [%1], %3\n"
|
"vld1.8 d5, [%2], %3\n"
|
"vld1.8 d6, [%1]\n"
|
"vld1.8 d7, [%2]\n"
|
/* Store to the GPU in one shot, no interleave. */
|
"vstm %0, {q0, q1, q2, q3}\n"
|
:
|
: "r"(gpu), "r"(cpu), "r"(cpu + 8), "r"(cpu_stride)
|
: "q0", "q1", "q2", "q3");
|
}
|
#elif defined (PIPE_ARCH_AARCH64)
|
if (gpu_stride == 8) {
|
__asm__ volatile (
|
/* Load each 8-byte line from cpu-side source,
|
* incrementing it by the stride each time.
|
*/
|
"ld1 {v0.D}[0], [%1], %2\n"
|
"ld1 {v0.D}[1], [%1], %2\n"
|
"ld1 {v1.D}[0], [%1], %2\n"
|
"ld1 {v1.D}[1], [%1], %2\n"
|
"ld1 {v2.D}[0], [%1], %2\n"
|
"ld1 {v2.D}[1], [%1], %2\n"
|
"ld1 {v3.D}[0], [%1], %2\n"
|
"ld1 {v3.D}[1], [%1]\n"
|
/* Store to the GPU in one shot, no interleave. */
|
"st1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%0]\n"
|
:
|
: "r"(gpu), "r"(cpu), "r"(cpu_stride)
|
: "v0", "v1", "v2", "v3");
|
} else {
|
assert(gpu_stride == 16);
|
__asm__ volatile (
|
/* Load each 16-byte line in 2 parts from the cpu-side
|
* destination. (vld1 can only store one d-register
|
* at a time).
|
*/
|
"ld1 {v0.D}[0], [%1], %3\n"
|
"ld1 {v0.D}[1], [%2], %3\n"
|
"ld1 {v1.D}[0], [%1], %3\n"
|
"ld1 {v1.D}[1], [%2], %3\n"
|
"ld1 {v2.D}[0], [%1], %3\n"
|
"ld1 {v2.D}[1], [%2], %3\n"
|
"ld1 {v3.D}[0], [%1]\n"
|
"ld1 {v3.D}[1], [%2]\n"
|
/* Store to the GPU in one shot, no interleave. */
|
"st1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%0]\n"
|
:
|
: "r"(gpu), "r"(cpu), "r"(cpu + 8), "r"(cpu_stride)
|
: "v0", "v1", "v2", "v3");
|
}
|
#else
|
for (uint32_t gpu_offset = 0; gpu_offset < 64; gpu_offset += gpu_stride) {
|
memcpy(gpu + gpu_offset, cpu, gpu_stride);
|
cpu += cpu_stride;
|
}
|
#endif
|
|
}
|
|
void
|
NEON_TAG(vc4_load_lt_image)(void *dst, uint32_t dst_stride,
|
void *src, uint32_t src_stride,
|
int cpp, const struct pipe_box *box)
|
{
|
uint32_t utile_w = vc4_utile_width(cpp);
|
uint32_t utile_h = vc4_utile_height(cpp);
|
uint32_t xstart = box->x;
|
uint32_t ystart = box->y;
|
|
for (uint32_t y = 0; y < box->height; y += utile_h) {
|
for (int x = 0; x < box->width; x += utile_w) {
|
vc4_load_utile(dst + (dst_stride * y +
|
x * cpp),
|
src + ((ystart + y) * src_stride +
|
(xstart + x) * 64 / utile_w),
|
dst_stride, cpp);
|
}
|
}
|
}
|
|
void
|
NEON_TAG(vc4_store_lt_image)(void *dst, uint32_t dst_stride,
|
void *src, uint32_t src_stride,
|
int cpp, const struct pipe_box *box)
|
{
|
uint32_t utile_w = vc4_utile_width(cpp);
|
uint32_t utile_h = vc4_utile_height(cpp);
|
uint32_t xstart = box->x;
|
uint32_t ystart = box->y;
|
|
for (uint32_t y = 0; y < box->height; y += utile_h) {
|
for (int x = 0; x < box->width; x += utile_w) {
|
vc4_store_utile(dst + ((ystart + y) * dst_stride +
|
(xstart + x) * 64 / utile_w),
|
src + (src_stride * y +
|
x * cpp),
|
src_stride, cpp);
|
}
|
}
|
}
|
