/**************************************************************************
|
*
|
* Copyright 2009 VMware, Inc.
|
* All Rights Reserved.
|
*
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
* copy of this software and associated documentation files (the
|
* "Software"), to deal in the Software without restriction, including
|
* without limitation the rights to use, copy, modify, merge, publish,
|
* distribute, sub license, and/or sell copies of the Software, and to
|
* permit persons to whom the Software is furnished to do so, subject to
|
* the following conditions:
|
*
|
* The above copyright notice and this permission notice (including the
|
* next paragraph) shall be included in all copies or substantial portions
|
* of the Software.
|
*
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
|
* IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
|
* ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
*
|
**************************************************************************/
|
|
/**
|
* @file
|
* Helper functions for logical operations.
|
*
|
* @author Jose Fonseca <jfonseca@vmware.com>
|
*/
|
|
|
#include "util/u_cpu_detect.h"
|
#include "util/u_memory.h"
|
#include "util/u_debug.h"
|
|
#include "lp_bld_type.h"
|
#include "lp_bld_const.h"
|
#include "lp_bld_swizzle.h"
|
#include "lp_bld_init.h"
|
#include "lp_bld_intr.h"
|
#include "lp_bld_debug.h"
|
#include "lp_bld_logic.h"
|
|
|
/*
|
* XXX
|
*
|
* Selection with vector conditional like
|
*
|
* select <4 x i1> %C, %A, %B
|
*
|
* is valid IR (e.g. llvm/test/Assembler/vector-select.ll), but it is only
|
* supported on some backends (x86) starting with llvm 3.1.
|
*
|
* Expanding the boolean vector to full SIMD register width, as in
|
*
|
* sext <4 x i1> %C to <4 x i32>
|
*
|
* is valid and supported (e.g., llvm/test/CodeGen/X86/vec_compare.ll), but
|
* it causes assertion failures in LLVM 2.6. It appears to work correctly on
|
* LLVM 2.7.
|
*/
|
|
|
/**
|
* Build code to compare two values 'a' and 'b' of 'type' using the given func.
|
* \param func one of PIPE_FUNC_x
|
* If the ordered argument is true the function will use LLVM's ordered
|
* comparisons, otherwise unordered comparisons will be used.
|
* The result values will be 0 for false or ~0 for true.
|
*/
|
static LLVMValueRef
|
lp_build_compare_ext(struct gallivm_state *gallivm,
|
const struct lp_type type,
|
unsigned func,
|
LLVMValueRef a,
|
LLVMValueRef b,
|
boolean ordered)
|
{
|
LLVMBuilderRef builder = gallivm->builder;
|
LLVMTypeRef int_vec_type = lp_build_int_vec_type(gallivm, type);
|
LLVMValueRef zeros = LLVMConstNull(int_vec_type);
|
LLVMValueRef ones = LLVMConstAllOnes(int_vec_type);
|
LLVMValueRef cond;
|
LLVMValueRef res;
|
|
assert(lp_check_value(type, a));
|
assert(lp_check_value(type, b));
|
|
if(func == PIPE_FUNC_NEVER)
|
return zeros;
|
if(func == PIPE_FUNC_ALWAYS)
|
return ones;
|
|
assert(func > PIPE_FUNC_NEVER);
|
assert(func < PIPE_FUNC_ALWAYS);
|
|
if(type.floating) {
|
LLVMRealPredicate op;
|
switch(func) {
|
case PIPE_FUNC_EQUAL:
|
op = ordered ? LLVMRealOEQ : LLVMRealUEQ;
|
break;
|
case PIPE_FUNC_NOTEQUAL:
|
op = ordered ? LLVMRealONE : LLVMRealUNE;
|
break;
|
case PIPE_FUNC_LESS:
|
op = ordered ? LLVMRealOLT : LLVMRealULT;
|
break;
|
case PIPE_FUNC_LEQUAL:
|
op = ordered ? LLVMRealOLE : LLVMRealULE;
|
break;
|
case PIPE_FUNC_GREATER:
|
op = ordered ? LLVMRealOGT : LLVMRealUGT;
|
break;
|
case PIPE_FUNC_GEQUAL:
|
op = ordered ? LLVMRealOGE : LLVMRealUGE;
|
break;
|
default:
|
assert(0);
|
return lp_build_undef(gallivm, type);
|
}
|
|
cond = LLVMBuildFCmp(builder, op, a, b, "");
|
res = LLVMBuildSExt(builder, cond, int_vec_type, "");
|
}
|
else {
|
LLVMIntPredicate op;
|
switch(func) {
|
case PIPE_FUNC_EQUAL:
|
op = LLVMIntEQ;
|
break;
|
case PIPE_FUNC_NOTEQUAL:
|
op = LLVMIntNE;
|
break;
|
case PIPE_FUNC_LESS:
|
op = type.sign ? LLVMIntSLT : LLVMIntULT;
|
break;
|
case PIPE_FUNC_LEQUAL:
|
op = type.sign ? LLVMIntSLE : LLVMIntULE;
|
break;
|
case PIPE_FUNC_GREATER:
|
op = type.sign ? LLVMIntSGT : LLVMIntUGT;
|
break;
|
case PIPE_FUNC_GEQUAL:
|
op = type.sign ? LLVMIntSGE : LLVMIntUGE;
|
break;
|
default:
|
assert(0);
|
return lp_build_undef(gallivm, type);
|
}
|
|
cond = LLVMBuildICmp(builder, op, a, b, "");
|
res = LLVMBuildSExt(builder, cond, int_vec_type, "");
|
}
|
|
return res;
|
}
|
|
/**
|
* Build code to compare two values 'a' and 'b' of 'type' using the given func.
|
* \param func one of PIPE_FUNC_x
|
* The result values will be 0 for false or ~0 for true.
|
*/
|
LLVMValueRef
|
lp_build_compare(struct gallivm_state *gallivm,
|
const struct lp_type type,
|
unsigned func,
|
LLVMValueRef a,
|
LLVMValueRef b)
|
{
|
LLVMTypeRef int_vec_type = lp_build_int_vec_type(gallivm, type);
|
LLVMValueRef zeros = LLVMConstNull(int_vec_type);
|
LLVMValueRef ones = LLVMConstAllOnes(int_vec_type);
|
|
assert(lp_check_value(type, a));
|
assert(lp_check_value(type, b));
|
|
if(func == PIPE_FUNC_NEVER)
|
return zeros;
|
if(func == PIPE_FUNC_ALWAYS)
|
return ones;
|
|
assert(func > PIPE_FUNC_NEVER);
|
assert(func < PIPE_FUNC_ALWAYS);
|
|
#if defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)
|
/*
|
* There are no unsigned integer comparison instructions in SSE.
|
*/
|
|
if (!type.floating && !type.sign &&
|
type.width * type.length == 128 &&
|
util_cpu_caps.has_sse2 &&
|
(func == PIPE_FUNC_LESS ||
|
func == PIPE_FUNC_LEQUAL ||
|
func == PIPE_FUNC_GREATER ||
|
func == PIPE_FUNC_GEQUAL) &&
|
(gallivm_debug & GALLIVM_DEBUG_PERF)) {
|
debug_printf("%s: inefficient <%u x i%u> unsigned comparison\n",
|
__FUNCTION__, type.length, type.width);
|
}
|
#endif
|
|
return lp_build_compare_ext(gallivm, type, func, a, b, FALSE);
|
}
|
|
/**
|
* Build code to compare two values 'a' and 'b' using the given func.
|
* \param func one of PIPE_FUNC_x
|
* If the operands are floating point numbers, the function will use
|
* ordered comparison which means that it will return true if both
|
* operands are not a NaN and the specified condition evaluates to true.
|
* The result values will be 0 for false or ~0 for true.
|
*/
|
LLVMValueRef
|
lp_build_cmp_ordered(struct lp_build_context *bld,
|
unsigned func,
|
LLVMValueRef a,
|
LLVMValueRef b)
|
{
|
return lp_build_compare_ext(bld->gallivm, bld->type, func, a, b, TRUE);
|
}
|
|
/**
|
* Build code to compare two values 'a' and 'b' using the given func.
|
* \param func one of PIPE_FUNC_x
|
* If the operands are floating point numbers, the function will use
|
* unordered comparison which means that it will return true if either
|
* operand is a NaN or the specified condition evaluates to true.
|
* The result values will be 0 for false or ~0 for true.
|
*/
|
LLVMValueRef
|
lp_build_cmp(struct lp_build_context *bld,
|
unsigned func,
|
LLVMValueRef a,
|
LLVMValueRef b)
|
{
|
return lp_build_compare(bld->gallivm, bld->type, func, a, b);
|
}
|
|
|
/**
|
* Return (mask & a) | (~mask & b);
|
*/
|
LLVMValueRef
|
lp_build_select_bitwise(struct lp_build_context *bld,
|
LLVMValueRef mask,
|
LLVMValueRef a,
|
LLVMValueRef b)
|
{
|
LLVMBuilderRef builder = bld->gallivm->builder;
|
struct lp_type type = bld->type;
|
LLVMValueRef res;
|
|
assert(lp_check_value(type, a));
|
assert(lp_check_value(type, b));
|
|
if (a == b) {
|
return a;
|
}
|
|
if(type.floating) {
|
LLVMTypeRef int_vec_type = lp_build_int_vec_type(bld->gallivm, type);
|
a = LLVMBuildBitCast(builder, a, int_vec_type, "");
|
b = LLVMBuildBitCast(builder, b, int_vec_type, "");
|
}
|
|
a = LLVMBuildAnd(builder, a, mask, "");
|
|
/* This often gets translated to PANDN, but sometimes the NOT is
|
* pre-computed and stored in another constant. The best strategy depends
|
* on available registers, so it is not a big deal -- hopefully LLVM does
|
* the right decision attending the rest of the program.
|
*/
|
b = LLVMBuildAnd(builder, b, LLVMBuildNot(builder, mask, ""), "");
|
|
res = LLVMBuildOr(builder, a, b, "");
|
|
if(type.floating) {
|
LLVMTypeRef vec_type = lp_build_vec_type(bld->gallivm, type);
|
res = LLVMBuildBitCast(builder, res, vec_type, "");
|
}
|
|
return res;
|
}
|
|
|
/**
|
* Return mask ? a : b;
|
*
|
* mask is a bitwise mask, composed of 0 or ~0 for each element. Any other value
|
* will yield unpredictable results.
|
*/
|
LLVMValueRef
|
lp_build_select(struct lp_build_context *bld,
|
LLVMValueRef mask,
|
LLVMValueRef a,
|
LLVMValueRef b)
|
{
|
LLVMBuilderRef builder = bld->gallivm->builder;
|
LLVMContextRef lc = bld->gallivm->context;
|
struct lp_type type = bld->type;
|
LLVMValueRef res;
|
|
assert(lp_check_value(type, a));
|
assert(lp_check_value(type, b));
|
|
if(a == b)
|
return a;
|
|
if (type.length == 1) {
|
mask = LLVMBuildTrunc(builder, mask, LLVMInt1TypeInContext(lc), "");
|
res = LLVMBuildSelect(builder, mask, a, b, "");
|
}
|
else if (!(HAVE_LLVM == 0x0307) &&
|
(LLVMIsConstant(mask) ||
|
LLVMGetInstructionOpcode(mask) == LLVMSExt)) {
|
/* Generate a vector select.
|
*
|
* Using vector selects should avoid emitting intrinsics hence avoid
|
* hindering optimization passes, but vector selects weren't properly
|
* supported yet for a long time, and LLVM will generate poor code when
|
* the mask is not the result of a comparison.
|
* Also, llvm 3.7 may miscompile them (bug 94972).
|
* XXX: Even if the instruction was an SExt, this may still produce
|
* terrible code. Try piglit stencil-twoside.
|
*/
|
|
/* Convert the mask to a vector of booleans.
|
*
|
* XXX: In x86 the mask is controlled by the MSB, so if we shifted the
|
* mask by `type.width - 1`, LLVM should realize the mask is ready. Alas
|
* what really happens is that LLVM will emit two shifts back to back.
|
*/
|
if (0) {
|
LLVMValueRef shift = LLVMConstInt(bld->int_elem_type, bld->type.width - 1, 0);
|
shift = lp_build_broadcast(bld->gallivm, bld->int_vec_type, shift);
|
mask = LLVMBuildLShr(builder, mask, shift, "");
|
}
|
LLVMTypeRef bool_vec_type = LLVMVectorType(LLVMInt1TypeInContext(lc), type.length);
|
mask = LLVMBuildTrunc(builder, mask, bool_vec_type, "");
|
|
res = LLVMBuildSelect(builder, mask, a, b, "");
|
}
|
else if (((util_cpu_caps.has_sse4_1 &&
|
type.width * type.length == 128) ||
|
(util_cpu_caps.has_avx &&
|
type.width * type.length == 256 && type.width >= 32) ||
|
(util_cpu_caps.has_avx2 &&
|
type.width * type.length == 256)) &&
|
!LLVMIsConstant(a) &&
|
!LLVMIsConstant(b) &&
|
!LLVMIsConstant(mask)) {
|
const char *intrinsic;
|
LLVMTypeRef arg_type;
|
LLVMValueRef args[3];
|
|
/*
|
* There's only float blend in AVX but can just cast i32/i64
|
* to float.
|
*/
|
if (type.width * type.length == 256) {
|
if (type.width == 64) {
|
intrinsic = "llvm.x86.avx.blendv.pd.256";
|
arg_type = LLVMVectorType(LLVMDoubleTypeInContext(lc), 4);
|
}
|
else if (type.width == 32) {
|
intrinsic = "llvm.x86.avx.blendv.ps.256";
|
arg_type = LLVMVectorType(LLVMFloatTypeInContext(lc), 8);
|
} else {
|
assert(util_cpu_caps.has_avx2);
|
intrinsic = "llvm.x86.avx2.pblendvb";
|
arg_type = LLVMVectorType(LLVMInt8TypeInContext(lc), 32);
|
}
|
}
|
else if (type.floating &&
|
type.width == 64) {
|
intrinsic = "llvm.x86.sse41.blendvpd";
|
arg_type = LLVMVectorType(LLVMDoubleTypeInContext(lc), 2);
|
} else if (type.floating &&
|
type.width == 32) {
|
intrinsic = "llvm.x86.sse41.blendvps";
|
arg_type = LLVMVectorType(LLVMFloatTypeInContext(lc), 4);
|
} else {
|
intrinsic = "llvm.x86.sse41.pblendvb";
|
arg_type = LLVMVectorType(LLVMInt8TypeInContext(lc), 16);
|
}
|
|
if (arg_type != bld->int_vec_type) {
|
mask = LLVMBuildBitCast(builder, mask, arg_type, "");
|
}
|
|
if (arg_type != bld->vec_type) {
|
a = LLVMBuildBitCast(builder, a, arg_type, "");
|
b = LLVMBuildBitCast(builder, b, arg_type, "");
|
}
|
|
args[0] = b;
|
args[1] = a;
|
args[2] = mask;
|
|
res = lp_build_intrinsic(builder, intrinsic,
|
arg_type, args, ARRAY_SIZE(args), 0);
|
|
if (arg_type != bld->vec_type) {
|
res = LLVMBuildBitCast(builder, res, bld->vec_type, "");
|
}
|
}
|
else {
|
res = lp_build_select_bitwise(bld, mask, a, b);
|
}
|
|
return res;
|
}
|
|
|
/**
|
* Return mask ? a : b;
|
*
|
* mask is a TGSI_WRITEMASK_xxx.
|
*/
|
LLVMValueRef
|
lp_build_select_aos(struct lp_build_context *bld,
|
unsigned mask,
|
LLVMValueRef a,
|
LLVMValueRef b,
|
unsigned num_channels)
|
{
|
LLVMBuilderRef builder = bld->gallivm->builder;
|
const struct lp_type type = bld->type;
|
const unsigned n = type.length;
|
unsigned i, j;
|
|
assert((mask & ~0xf) == 0);
|
assert(lp_check_value(type, a));
|
assert(lp_check_value(type, b));
|
|
if(a == b)
|
return a;
|
if((mask & 0xf) == 0xf)
|
return a;
|
if((mask & 0xf) == 0x0)
|
return b;
|
if(a == bld->undef || b == bld->undef)
|
return bld->undef;
|
|
/*
|
* There are two major ways of accomplishing this:
|
* - with a shuffle
|
* - with a select
|
*
|
* The flip between these is empirical and might need to be adjusted.
|
*/
|
if (n <= 4) {
|
/*
|
* Shuffle.
|
*/
|
LLVMTypeRef elem_type = LLVMInt32TypeInContext(bld->gallivm->context);
|
LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH];
|
|
for(j = 0; j < n; j += num_channels)
|
for(i = 0; i < num_channels; ++i)
|
shuffles[j + i] = LLVMConstInt(elem_type,
|
(mask & (1 << i) ? 0 : n) + j + i,
|
0);
|
|
return LLVMBuildShuffleVector(builder, a, b, LLVMConstVector(shuffles, n), "");
|
}
|
else {
|
LLVMValueRef mask_vec = lp_build_const_mask_aos(bld->gallivm, type, mask, num_channels);
|
return lp_build_select(bld, mask_vec, a, b);
|
}
|
}
|
|
|
/**
|
* Return (scalar-cast)val ? true : false;
|
*/
|
LLVMValueRef
|
lp_build_any_true_range(struct lp_build_context *bld,
|
unsigned real_length,
|
LLVMValueRef val)
|
{
|
LLVMBuilderRef builder = bld->gallivm->builder;
|
LLVMTypeRef scalar_type;
|
LLVMTypeRef true_type;
|
|
assert(real_length <= bld->type.length);
|
|
true_type = LLVMIntTypeInContext(bld->gallivm->context,
|
bld->type.width * real_length);
|
scalar_type = LLVMIntTypeInContext(bld->gallivm->context,
|
bld->type.width * bld->type.length);
|
val = LLVMBuildBitCast(builder, val, scalar_type, "");
|
/*
|
* We're using always native types so we can use intrinsics.
|
* However, if we don't do per-element calculations, we must ensure
|
* the excess elements aren't used since they may contain garbage.
|
*/
|
if (real_length < bld->type.length) {
|
val = LLVMBuildTrunc(builder, val, true_type, "");
|
}
|
return LLVMBuildICmp(builder, LLVMIntNE,
|
val, LLVMConstNull(true_type), "");
|
}
|
