/*
|
* Copyright (c) 2017 Imagination Technologies.
|
*
|
* All rights reserved.
|
*
|
* Redistribution and use in source and binary forms, with or without
|
* modification, are permitted provided that the following conditions
|
* are met:
|
*
|
* * Redistributions of source code must retain the above copyright
|
* notice, this list of conditions and the following disclaimer.
|
* * Redistributions in binary form must reproduce the above copyright
|
* notice, this list of conditions and the following disclaimer
|
* in the documentation and/or other materials provided with
|
* the distribution.
|
* * Neither the name of Imagination Technologies nor the names of its
|
* contributors may be used to endorse or promote products derived
|
* from this software without specific prior written permission.
|
*
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
*/
|
#include <string.h>
|
|
#define ENABLE_PREFETCH 1
|
#define op_t unsigned long int
|
#define op_size sizeof (op_t)
|
|
#if ENABLE_PREFETCH
|
#define PREFETCH(addr) __builtin_prefetch (addr, 0, 1);
|
#else
|
#define PREFETCH(addr)
|
#endif
|
|
#if __mips64 || __mips_isa_rev >= 2
|
static inline void * __attribute__ ((always_inline))
|
do_bytes (const op_t* w, op_t inval)
|
{
|
const unsigned char *p = (const unsigned char *) w;
|
op_t outval = 0;
|
#if __mips64
|
__asm__ volatile (
|
"dsbh %1, %0 \n\t"
|
"dshd %0, %1 \n\t"
|
"dclz %1, %0 \n\t"
|
: "+r" (inval), "+r" (outval)
|
);
|
#else
|
__asm__ volatile (
|
"wsbh %1, %0 \n\t"
|
"rotr %0, %1, 16 \n\t"
|
"clz %1, %0 \n\t"
|
: "+r" (inval), "+r" (outval)
|
);
|
#endif
|
p += (outval >> 3);
|
return (void *) p;
|
}
|
|
#define DO_WORD(in, val) { \
|
op_t tmp = ((val - mask_1) & ~val) & mask_128; \
|
if (tmp != 0) \
|
return do_bytes(in, tmp); \
|
}
|
#else
|
static inline void * __attribute__ ((always_inline))
|
do_bytes (const op_t* w, unsigned char ch)
|
{
|
const unsigned char *p = (const unsigned char *) w;
|
for (; *p != ch; ++p);
|
return (void *) p;
|
}
|
|
#define DO_WORD(in, val) { \
|
op_t tmp = ((val - mask_1) & ~val) & mask_128; \
|
if (tmp != 0) \
|
return do_bytes(in, ch); \
|
}
|
#endif
|
|
#define DO_WORDS(w) { \
|
op_t* w1 = (op_t*) w; \
|
op_t val0 = w1[0] ^ mask_c; \
|
op_t val1 = w1[1] ^ mask_c; \
|
op_t val2 = w1[2] ^ mask_c; \
|
op_t val3 = w1[3] ^ mask_c; \
|
DO_WORD(w1, val0) \
|
DO_WORD(w1 + 1, val1) \
|
DO_WORD(w1 + 2, val2) \
|
DO_WORD(w1 + 3, val3) \
|
}
|
|
void *
|
memchr (void const *s, int c_in, size_t n)
|
{
|
if (n != 0) {
|
const unsigned char *p = (const unsigned char *) s;
|
const op_t *w;
|
op_t mask_1, mask_128, mask_c;
|
unsigned char ch = (unsigned char) c_in;
|
|
/*
|
* Check bytewize till initial alignment
|
*/
|
for (; n > 0 && ((size_t) p % op_size) != 0; --n, ++p) {
|
if (*p == ch)
|
return (void *) p;
|
}
|
|
w = (const op_t *) p;
|
|
mask_c = ch | (ch << 8);
|
mask_c |= mask_c << 16;
|
__asm__ volatile (
|
"li %0, 0x01010101 \n\t"
|
: "=r" (mask_1)
|
);
|
#if __mips64
|
mask_1 |= mask_1 << 32;
|
mask_c |= mask_c << 32;
|
#endif
|
mask_128 = mask_1 << 7;
|
|
/*
|
* Check op_size byteswize after initial alignment
|
*/
|
#if ((_MIPS_SIM == _ABIO32) || _MIPS_TUNE_I6400)
|
PREFETCH (w);
|
PREFETCH (w + 8);
|
while (n >= 24 * op_size) {
|
PREFETCH(w + 16);
|
DO_WORDS(w);
|
DO_WORDS(w + 4);
|
w += 8;
|
n -= 8 * op_size;
|
}
|
while (n >= 8 * op_size) {
|
DO_WORDS(w);
|
DO_WORDS(w + 4);
|
w += 8;
|
n -= 8 * op_size;
|
}
|
#else
|
PREFETCH (w);
|
PREFETCH (w + 4);
|
while (n >= 12 * op_size) {
|
PREFETCH(w + 8);
|
DO_WORDS(w);
|
w += 4;
|
n -= 4 * op_size;
|
}
|
while (n >= 4 * op_size) {
|
DO_WORDS(w);
|
w += 4;
|
n -= 4 * op_size;
|
}
|
#endif
|
|
while (n >= op_size) {
|
op_t val = *w ^ mask_c;
|
DO_WORD(w, val);
|
w++;
|
n -= op_size;
|
}
|
|
/*
|
* Check bytewize for remaining bytes
|
*/
|
p = (const unsigned char *) w;
|
for (; n > 0; --n, ++p) {
|
if (*p == ch)
|
return (void *) p;
|
}
|
}
|
return NULL;
|
}
|
