diff --git a/string/aarch64/memchr-scalar.S b/string/aarch64/memchr-scalar.S new file mode 100644 index 00000000..4d6eadac --- /dev/null +++ b/string/aarch64/memchr-scalar.S @@ -0,0 +1,110 @@ +/* + * memchr - find a character in a memory zone + * + * Copyright (c) 2014-2026, Arm Limited. + * SPDX-License-Identifier: MIT OR Apache-2.0 WITH LLVM-exception + */ + +/* Assumptions: + * + * ARMv8-a, AArch64 only + */ + +#include "asmdefs.h" + +#define result x0 +#define src_in x0 +#define chr_in w1 +#define chr_repeated x1 +#define count_in x2 + +#define src x3 +#define zeroones x4 +#define data1 x5 +#define tmp1 x6 +#define tmp2 x7 +#define found1 x8 +#define src_end x9 + +#define REP8_01 0x0101010101010101 +#define REP8_7f 0x7f7f7f7f7f7f7f7f + +/* This function implements the hacker's delight algorithm for searching + matching values. We find them in words by performing an XOR against the + repeated searched char. Then, x being the xor result, we calculate + (x - 0x0101010101010101) & ~(x | 0x7f7f7f7f7f7f7f7f). Words with a matching + byte will produce a non-zero result. */ + +ENTRY_ALIAS (__memchr_scalar_sc) +ENTRY (__memchr_scalar) + cbz count_in, L(no_match) + + /* Replicate the needle byte across the rest of the word. */ + and chr_in, chr_in, #255 + orr chr_in, chr_in, chr_in, lsl 8 + orr chr_in, chr_in, chr_in, lsl 16 + orr chr_repeated, chr_repeated, chr_repeated, lsl 32 + mov zeroones, #REP8_01 + + /* Align back to 8-byte boundary, look for a match and shift the found + result back before checking. */ + bic src, src_in, #7 + ldr data1, [src], #8 + + eor data1, data1, chr_repeated + sub tmp1, data1, zeroones + orr tmp2, data1, REP8_7f + bic found1, tmp1, tmp2 + + lsl tmp1, src_in, #3 + lsr found1, found1, tmp1 + cbz found1, L(loop_start) + + /* Calculate the offset and return + it if offset < count_in. */ + rbit found1, found1 + clz tmp1, found1 + cmp count_in, tmp1, lsr #3 + add result, src_in, tmp1, lsr #3 + csel result, result, xzr, hi + ret + +L(loop_start): + adds src_end, src_in, count_in + /* Deal with count_in being so large that src_end comes before src_in due + to wraparound. When this happens, set src_end to SIZE_MAX so that we do + not reject results based on the address. */ + csinv src_end, src_end, xzr, cc + /* if src >= src_end then we found no matches. */ + cmp src, src_end + b.cs L(no_match) + + .p2align 4 +L(loop): + ldr data1, [src], #8 + eor data1, data1, chr_repeated + sub tmp1, data1, zeroones + orr tmp2, data1, REP8_7f + bic found1, tmp1, tmp2 + + /* if src < src_end, then compare found1 with zero + if src >= src_end, force NZCV to #0, where Z is clear + b.eq L(loop) only happens when src < src_end && found1 == 0 */ + cmp src, src_end + ccmp found1, #0, #0, lo + b.eq L(loop) + +L(found): + sub src, src, #8 + rbit found1, found1 + clz tmp1, found1 + add result, src, tmp1, lsr #3 + cmp src_end, result + csel result, result, xzr, hi + ret + +L(no_match): + mov result, 0 + ret + +END (__memchr_scalar) diff --git a/string/aarch64/memcpy.S b/string/aarch64/memcpy.S index 351f1a11..67eb9276 100644 --- a/string/aarch64/memcpy.S +++ b/string/aarch64/memcpy.S @@ -53,6 +53,8 @@ The loop tail is handled by always copying 64 bytes from the end. */ +ENTRY_ALIAS (__memmove_aarch64_sc) +ENTRY_ALIAS (__memcpy_aarch64_sc) ENTRY_ALIAS (__memmove_aarch64) ENTRY (__memcpy_aarch64) add srcend, src, count diff --git a/string/aarch64/memset-scalar.S b/string/aarch64/memset-scalar.S new file mode 100644 index 00000000..a66373d5 --- /dev/null +++ b/string/aarch64/memset-scalar.S @@ -0,0 +1,165 @@ +/* + * __arm_sc_memset - fill memory with a constant byte + * + * Copyright (c) 2012-2026, Arm Limited. + * SPDX-License-Identifier: MIT OR Apache-2.0 WITH LLVM-exception + */ + +/* Assumptions: + * + * ARMv8-a, AArch64, unaligned accesses. + * + */ + +#include "asmdefs.h" + +#define dstin x0 +#define val x1 +#define valw w1 +#define count x2 +#define dst x3 +#define dstend x4 +#define zva_val x5 +#define off x3 +#define dstend2 x5 + +/* This implementation is similar to __memset_aarch64 but avoids FP/AdvSIMD + register instructions. */ + +ENTRY_ALIAS (__memset_scalar_sc) +ENTRY (__memset_scalar) + /* Replicate the fill byte across the last half of the word. */ + and valw, valw, 255 + orr valw, valw, valw, lsl 8 + orr valw, valw, valw, lsl 16 + + cmp count, 16 + b.lo L(set_lt16) + + /* Replicate the fill byte across the rest of the word. */ + orr val, val, val, lsl 32 + + add dstend, dstin, count + cmp count, 64 + b.hi L(set_gt64) + +L(set_16_64): + /* Calculate an interior store offset based on half the count, This will be + one of 0, 16 or 32 bytes. Then perform four overlapping stores. */ + mov off, 48 + and off, off, count, lsr 1 + sub dstend2, dstend, off + + stp val, val, [dstin] + add off, dstin, off + stp val, val, [off] + stp val, val, [dstend2, -16] + stp val, val, [dstend, -16] + ret + + .p2align 4 +L(set_lt16): + add dstend, dstin, count + cmp count, 4 + b.lo 1f + + /* For sizes 4..15, salculate an interior offset based on the count. Then + perform four overlapping 32 bit stores. */ + lsr off, count, 3 /* off = count >> 3. */ + sub dstend2, dstend, off, lsl 2 + str valw, [dstin] + str valw, [dstin, off, lsl 2] + str valw, [dstend2, -4] + str valw, [dstend, -4] + ret + + /* Potentially overlapping stores for sizes 0..3. */ +1: cbz count, 2f /* Skip stores entirely for a zero-length memset. */ + lsr off, count, 1 + strb valw, [dstin] + strb valw, [dstin, off] + strb valw, [dstend, -1] +2: ret + + .p2align 4 +L(set_gt64): + bic dst, dstin, 15 /* Align dst down to 16 bytes. */ + cmp count, 128 + b.hi L(set_gt128) + + /* For sizes 64..128, perform eight overlapping 16-byte stores. */ + stp val, val, [dstin] + stp val, val, [dstin, 16] + stp val, val, [dstin, 32] + stp val, val, [dstin, 48] + stp val, val, [dstend, -64] + stp val, val, [dstend, -48] + stp val, val, [dstend, -32] + stp val, val, [dstend, -16] + ret + + .p2align 4 +L(set_gt128): + /* For sizes above 128, perform initial stores before entering loop on + aligned addresses. */ + stp val, val, [dstin] + stp val, val, [dst, 16] + + /* We can use DC ZVA to zero memory when the fill byte is zero and the block + size reported by DCZID_EL0 is 64 bytes. */ + cbnz valw, L(no_zva) +#ifndef SKIP_ZVA_CHECK + mrs zva_val, dczid_el0 + and zva_val, zva_val, 31 + cmp zva_val, 4 + b.ne L(no_zva) +#endif + /* Complete stores up to the first 64 byte aligned block. */ + stp val, val, [dst, 32] + stp val, val, [dst, 48] + + bic dst, dstin, 63 /* Align dst down to 64 bytes. */ + /* Compute count, accounting for the stores above and bias for the loop + exit. */ + sub count, dstend, dst + sub count, count, 64 + 64 + + /* Write last bytes before entering ZVA loop. */ + stp val, val, [dstend, -64] + stp val, val, [dstend, -48] + stp val, val, [dstend, -32] + stp val, val, [dstend, -16] + + .p2align 4 +L(zva64_loop): + add dst, dst, 64 + dc zva, dst /* Zero the 64-byte cache block containing dst. */ + subs count, count, 64 + b.hi L(zva64_loop) + ret + + .p2align 3 +L(no_zva): + /* Compute count, accounting for the two stores in set_gt128 and bias for + the loop exit. */ + sub count, dstend, dst + sub count, count, 64 + 32 + +L(no_zva_loop): + /* Perform eight contiguous 16-byte stores per iteration. */ + stp val, val, [dst, 32] + stp val, val, [dst, 48] + stp val, val, [dst, 64] + stp val, val, [dst, 80] + add dst, dst, 64 + subs count, count, 64 + b.hi L(no_zva_loop) + + /* Perform final stores for the tail of the range, potentially overlapping + with previous ones. */ + stp val, val, [dstend, -64] + stp val, val, [dstend, -48] + stp val, val, [dstend, -32] + stp val, val, [dstend, -16] + ret +END (__memset_scalar) diff --git a/string/bench/memset.c b/string/bench/memset.c index ca96a76c..7f88d42c 100644 --- a/string/bench/memset.c +++ b/string/bench/memset.c @@ -26,6 +26,7 @@ static uint8_t a[MAX_SIZE + 4096] __attribute__((__aligned__(4096))); printf (STR); \ RUN (TESTFN, memset); \ RUNA64 (TESTFN, __memset_aarch64); \ + RUNA64 (TESTFN, __memset_scalar); \ RUNSVE (TESTFN, __memset_aarch64_sve); \ RUNMOPS (TESTFN, __memset_aarch64_mops); \ RUNA32 (TESTFN, __memset_arm); \ diff --git a/string/include/stringlib.h b/string/include/stringlib.h index 9ddfc672..dd45dafd 100644 --- a/string/include/stringlib.h +++ b/string/include/stringlib.h @@ -16,7 +16,9 @@ void *__memcpy_aarch64 (void *__restrict, const void *__restrict, size_t); void *__memmove_aarch64 (void *, const void *, size_t); void *__memset_aarch64 (void *, int, size_t); +void *__memset_scalar (void *, int, size_t); void *__memchr_aarch64 (const void *, int, size_t); +void *__memchr_scalar (const void *, int, size_t); void *__memrchr_aarch64 (const void *, int, size_t); int __memcmp_aarch64 (const void *, const void *, size_t); char *__strcpy_aarch64 (char *__restrict, const char *__restrict); @@ -55,6 +57,13 @@ int __strncmp_aarch64_sve (const char *, const char *, size_t); char *__strchr_aarch64_sve2 (const char *, int); char *__strchrnul_aarch64_sve2 (const char *, int ); # endif +# if __ARM_FEATURE_SME +/* These are used to test functionality in streaming mode */ +void *__memchr_scalar_sc (const void *, int, size_t) __arm_streaming_compatible; +void *__memcpy_aarch64_sc (void *, const void *, size_t) __arm_streaming_compatible; +void *__memmove_aarch64_sc (void *, const void *, size_t) __arm_streaming_compatible; +void *__memset_scalar_sc (void *, int, size_t) __arm_streaming_compatible; +# endif # if WANT_MOPS void *__memcpy_aarch64_mops (void *__restrict, const void *__restrict, size_t); void *__memmove_aarch64_mops (void *__restrict, const void *__restrict, size_t); diff --git a/string/test/memchr.c b/string/test/memchr.c index a2a8bcd3..9e41e16d 100644 --- a/string/test/memchr.c +++ b/string/test/memchr.c @@ -26,6 +26,7 @@ static const struct fun F(memchr, 0) #if __aarch64__ F(__memchr_aarch64, 0) + F(__memchr_scalar, 0) F(__memchr_aarch64_mte, 1) # if __ARM_FEATURE_SVE F(__memchr_aarch64_sve, 1) diff --git a/string/test/memset.c b/string/test/memset.c index 86ad2901..8ec04d93 100644 --- a/string/test/memset.c +++ b/string/test/memset.c @@ -25,6 +25,7 @@ static const struct fun F(memset, 0) #if __aarch64__ F(__memset_aarch64, 1) + F(__memset_scalar, 1) # if __ARM_FEATURE_SVE F(__memset_aarch64_sve, 1) # endif