/**************************************************************************
|
*
|
* Copyright 2008 VMware, Inc.
|
* All Rights Reserved.
|
*
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
* copy of this software and associated documentation files (the
|
* "Software"), to deal in the Software without restriction, including
|
* without limitation the rights to use, copy, modify, merge, publish,
|
* distribute, sub license, and/or sell copies of the Software, and to
|
* permit persons to whom the Software is furnished to do so, subject to
|
* the following conditions:
|
*
|
* The above copyright notice and this permission notice (including the
|
* next paragraph) shall be included in all copies or substantial portions
|
* of the Software.
|
*
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
|
* IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
|
* ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
*
|
**************************************************************************/
|
|
/**
|
* @file
|
* SSE intrinsics portability header.
|
*
|
* Although the SSE intrinsics are support by all modern x86 and x86-64
|
* compilers, there are some intrisincs missing in some implementations
|
* (especially older MSVC versions). This header abstracts that away.
|
*/
|
|
#ifndef U_SSE_H_
|
#define U_SSE_H_
|
|
#include "pipe/p_config.h"
|
|
#if defined(PIPE_ARCH_SSE)
|
|
#include <emmintrin.h>
|
|
|
union m128i {
|
__m128i m;
|
ubyte ub[16];
|
ushort us[8];
|
uint ui[4];
|
};
|
|
static inline void u_print_epi8(const char *name, __m128i r)
|
{
|
union { __m128i m; ubyte ub[16]; } u;
|
u.m = r;
|
|
debug_printf("%s: "
|
"%02x/"
|
"%02x/"
|
"%02x/"
|
"%02x/"
|
"%02x/"
|
"%02x/"
|
"%02x/"
|
"%02x/"
|
"%02x/"
|
"%02x/"
|
"%02x/"
|
"%02x/"
|
"%02x/"
|
"%02x/"
|
"%02x/"
|
"%02x\n",
|
name,
|
u.ub[0], u.ub[1], u.ub[2], u.ub[3],
|
u.ub[4], u.ub[5], u.ub[6], u.ub[7],
|
u.ub[8], u.ub[9], u.ub[10], u.ub[11],
|
u.ub[12], u.ub[13], u.ub[14], u.ub[15]);
|
}
|
|
static inline void u_print_epi16(const char *name, __m128i r)
|
{
|
union { __m128i m; ushort us[8]; } u;
|
u.m = r;
|
|
debug_printf("%s: "
|
"%04x/"
|
"%04x/"
|
"%04x/"
|
"%04x/"
|
"%04x/"
|
"%04x/"
|
"%04x/"
|
"%04x\n",
|
name,
|
u.us[0], u.us[1], u.us[2], u.us[3],
|
u.us[4], u.us[5], u.us[6], u.us[7]);
|
}
|
|
static inline void u_print_epi32(const char *name, __m128i r)
|
{
|
union { __m128i m; uint ui[4]; } u;
|
u.m = r;
|
|
debug_printf("%s: "
|
"%08x/"
|
"%08x/"
|
"%08x/"
|
"%08x\n",
|
name,
|
u.ui[0], u.ui[1], u.ui[2], u.ui[3]);
|
}
|
|
static inline void u_print_ps(const char *name, __m128 r)
|
{
|
union { __m128 m; float f[4]; } u;
|
u.m = r;
|
|
debug_printf("%s: "
|
"%f/"
|
"%f/"
|
"%f/"
|
"%f\n",
|
name,
|
u.f[0], u.f[1], u.f[2], u.f[3]);
|
}
|
|
|
#define U_DUMP_EPI32(a) u_print_epi32(#a, a)
|
#define U_DUMP_EPI16(a) u_print_epi16(#a, a)
|
#define U_DUMP_EPI8(a) u_print_epi8(#a, a)
|
#define U_DUMP_PS(a) u_print_ps(#a, a)
|
|
|
|
#if defined(PIPE_ARCH_SSSE3)
|
|
#include <tmmintrin.h>
|
|
#else /* !PIPE_ARCH_SSSE3 */
|
|
/**
|
* Describe _mm_shuffle_epi8() with gcc extended inline assembly, for cases
|
* where -mssse3 is not supported/enabled.
|
*
|
* MSVC will never get in here as its intrinsics support do not rely on
|
* compiler command line options.
|
*/
|
static __inline __m128i
|
#ifdef __clang__
|
__attribute__((__always_inline__, __nodebug__))
|
#else
|
__attribute__((__gnu_inline__, __always_inline__, __artificial__))
|
#endif
|
_mm_shuffle_epi8(__m128i a, __m128i mask)
|
{
|
__m128i result;
|
__asm__("pshufb %1, %0"
|
: "=x" (result)
|
: "xm" (mask), "0" (a));
|
return result;
|
}
|
|
#endif /* !PIPE_ARCH_SSSE3 */
|
|
|
/*
|
* Provide an SSE implementation of _mm_mul_epi32() in terms of
|
* _mm_mul_epu32().
|
*
|
* Basically, albeit surprising at first (and second, and third...) look
|
* if a * b is done signed instead of unsigned, can just
|
* subtract b from the high bits of the result if a is negative
|
* (and the same for a if b is negative). Modular arithmetic at its best!
|
*
|
* So for int32 a,b in crude pseudo-code ("*" here denoting a widening mul)
|
* fixupb = (signmask(b) & a) << 32ULL
|
* fixupa = (signmask(a) & b) << 32ULL
|
* a * b = (unsigned)a * (unsigned)b - fixupb - fixupa
|
* = (unsigned)a * (unsigned)b -(fixupb + fixupa)
|
*
|
* This does both lo (dwords 0/2) and hi parts (1/3) at the same time due
|
* to some optimization potential.
|
*/
|
static inline __m128i
|
mm_mullohi_epi32(const __m128i a, const __m128i b, __m128i *res13)
|
{
|
__m128i a13, b13, mul02, mul13;
|
__m128i anegmask, bnegmask, fixup, fixup02, fixup13;
|
a13 = _mm_shuffle_epi32(a, _MM_SHUFFLE(2,3,0,1));
|
b13 = _mm_shuffle_epi32(b, _MM_SHUFFLE(2,3,0,1));
|
anegmask = _mm_srai_epi32(a, 31);
|
bnegmask = _mm_srai_epi32(b, 31);
|
fixup = _mm_add_epi32(_mm_and_si128(anegmask, b),
|
_mm_and_si128(bnegmask, a));
|
mul02 = _mm_mul_epu32(a, b);
|
mul13 = _mm_mul_epu32(a13, b13);
|
fixup02 = _mm_slli_epi64(fixup, 32);
|
fixup13 = _mm_and_si128(fixup, _mm_set_epi32(-1,0,-1,0));
|
*res13 = _mm_sub_epi64(mul13, fixup13);
|
return _mm_sub_epi64(mul02, fixup02);
|
}
|
|
|
/* Provide an SSE2 implementation of _mm_mullo_epi32() in terms of
|
* _mm_mul_epu32().
|
*
|
* This always works regardless the signs of the operands, since
|
* the high bits (which would be different) aren't used.
|
*
|
* This seems close enough to the speed of SSE4 and the real
|
* _mm_mullo_epi32() intrinsic as to not justify adding an sse4
|
* dependency at this point.
|
*/
|
static inline __m128i mm_mullo_epi32(const __m128i a, const __m128i b)
|
{
|
__m128i a4 = _mm_srli_epi64(a, 32); /* shift by one dword */
|
__m128i b4 = _mm_srli_epi64(b, 32); /* shift by one dword */
|
__m128i ba = _mm_mul_epu32(b, a); /* multply dwords 0, 2 */
|
__m128i b4a4 = _mm_mul_epu32(b4, a4); /* multiply dwords 1, 3 */
|
|
/* Interleave the results, either with shuffles or (slightly
|
* faster) direct bit operations:
|
* XXX: might be only true for some cpus (in particular 65nm
|
* Core 2). On most cpus (including that Core 2, but not Nehalem...)
|
* using _mm_shuffle_ps/_mm_shuffle_epi32 might also be faster
|
* than using the 3 instructions below. But logic should be fine
|
* as well, we can't have optimal solution for all cpus (if anything,
|
* should just use _mm_mullo_epi32() if sse41 is available...).
|
*/
|
#if 0
|
__m128i ba8 = _mm_shuffle_epi32(ba, 8);
|
__m128i b4a48 = _mm_shuffle_epi32(b4a4, 8);
|
__m128i result = _mm_unpacklo_epi32(ba8, b4a48);
|
#else
|
__m128i mask = _mm_setr_epi32(~0,0,~0,0);
|
__m128i ba_mask = _mm_and_si128(ba, mask);
|
__m128i b4a4_mask_shift = _mm_slli_epi64(b4a4, 32);
|
__m128i result = _mm_or_si128(ba_mask, b4a4_mask_shift);
|
#endif
|
|
return result;
|
}
|
|
|
static inline void
|
transpose4_epi32(const __m128i * restrict a,
|
const __m128i * restrict b,
|
const __m128i * restrict c,
|
const __m128i * restrict d,
|
__m128i * restrict o,
|
__m128i * restrict p,
|
__m128i * restrict q,
|
__m128i * restrict r)
|
{
|
__m128i t0 = _mm_unpacklo_epi32(*a, *b);
|
__m128i t1 = _mm_unpacklo_epi32(*c, *d);
|
__m128i t2 = _mm_unpackhi_epi32(*a, *b);
|
__m128i t3 = _mm_unpackhi_epi32(*c, *d);
|
|
*o = _mm_unpacklo_epi64(t0, t1);
|
*p = _mm_unpackhi_epi64(t0, t1);
|
*q = _mm_unpacklo_epi64(t2, t3);
|
*r = _mm_unpackhi_epi64(t2, t3);
|
}
|
|
|
/*
|
* Same as above, except the first two values are already interleaved
|
* (i.e. contain 64bit values).
|
*/
|
static inline void
|
transpose2_64_2_32(const __m128i * restrict a01,
|
const __m128i * restrict a23,
|
const __m128i * restrict c,
|
const __m128i * restrict d,
|
__m128i * restrict o,
|
__m128i * restrict p,
|
__m128i * restrict q,
|
__m128i * restrict r)
|
{
|
__m128i t0 = *a01;
|
__m128i t1 = _mm_unpacklo_epi32(*c, *d);
|
__m128i t2 = *a23;
|
__m128i t3 = _mm_unpackhi_epi32(*c, *d);
|
|
*o = _mm_unpacklo_epi64(t0, t1);
|
*p = _mm_unpackhi_epi64(t0, t1);
|
*q = _mm_unpacklo_epi64(t2, t3);
|
*r = _mm_unpackhi_epi64(t2, t3);
|
}
|
|
|
#define SCALAR_EPI32(m, i) _mm_shuffle_epi32((m), _MM_SHUFFLE(i,i,i,i))
|
|
|
#endif /* PIPE_ARCH_SSE */
|
|
#endif /* U_SSE_H_ */
|
