Add basic scalar __arm_sc_* implementations

string/aarch64/memchr-scalar.S

@@ -0,0 +1,110 @@
+/*
+ * memchr - find a character in a memory zone
+ *
+ * Copyright (c) 2014-2026, Arm Limited.
+ * SPDX-License-Identifier: MIT OR Apache-2.0 WITH LLVM-exception
+ */
+
+/* Assumptions:
+ *
+ * ARMv8-a, AArch64 only
+ */
+
+#include "asmdefs.h"
+
+#define result			x0
+#define src_in			x0
+#define chr_in			w1
+#define chr_repeated	x1
+#define count_in		x2
+
+#define src				x3
+#define zeroones		x4
+#define	data1			x5
+#define	tmp1			x6
+#define tmp2			x7
+#define found1			x8
+#define src_end			x9
+
+#define REP8_01 0x0101010101010101
+#define REP8_7f 0x7f7f7f7f7f7f7f7f
+
+/* This function implements the hacker's delight algorithm for searching
+   matching values. We find them in words by performing an XOR against the
+   repeated searched char. Then, x being the xor result, we calculate
+   (x - 0x0101010101010101) & ~(x | 0x7f7f7f7f7f7f7f7f). Words with a matching
+   byte will produce a non-zero result. */
+
+ENTRY_ALIAS (__memchr_scalar_sc)
+ENTRY (__memchr_scalar)
+	cbz	count_in, L(no_match)
+
+	/* Replicate the needle byte across the rest of the word.  */
+	and	chr_in, chr_in, #255
+	orr	chr_in, chr_in, chr_in, lsl 8
+	orr	chr_in, chr_in, chr_in, lsl 16
+	orr	chr_repeated, chr_repeated, chr_repeated, lsl 32
+	mov	zeroones, #REP8_01
+
+	/* Align back to 8-byte boundary, look for a match and shift the found
+	   result back before checking.  */
+	bic	src, src_in, #7
+	ldr	data1, [src], #8
+
+	eor	data1, data1, chr_repeated
+	sub	tmp1, data1, zeroones
+	orr	tmp2, data1, REP8_7f
+	bic	found1, tmp1, tmp2
+
+	lsl	tmp1, src_in, #3
+	lsr	found1, found1, tmp1
+	cbz	found1, L(loop_start)
+
+    /* Calculate the offset and return
+	   it if offset < count_in. */
+	rbit	found1, found1
+	clz	tmp1, found1
+	cmp	count_in, tmp1, lsr #3
+	add	result, src_in, tmp1, lsr #3
+	csel	result, result, xzr, hi
+	ret
+
+L(loop_start):
+	adds	src_end, src_in, count_in
+	/* Deal with count_in being so large that src_end comes before src_in due
+	   to wraparound. When this happens, set src_end to SIZE_MAX so that we do
+	   not reject results based on the address.  */
+	csinv	src_end, src_end, xzr, cc
+	/* if src >= src_end then we found no matches. */
+	cmp src, src_end
+	b.cs L(no_match)
+
+	.p2align 4
+L(loop):
+	ldr	data1, [src], #8
+	eor	data1, data1, chr_repeated
+	sub	tmp1, data1, zeroones
+	orr	tmp2, data1, REP8_7f
+	bic	found1, tmp1, tmp2
+
+	/* if src < src_end, then compare found1 with zero
+	   if src >= src_end, force NZCV to #0, where Z is clear
+	   b.eq L(loop) only happens when src < src_end && found1 == 0 */
+	cmp	src, src_end
+	ccmp	found1, #0, #0, lo
+	b.eq L(loop)
+
+L(found):
+	sub	src, src, #8
+	rbit	found1, found1
+	clz	tmp1, found1
+	add	result, src, tmp1, lsr #3
+	cmp	src_end, result
+	csel	result, result, xzr, hi
+	ret
+
+L(no_match):
+	mov	result, 0
+	ret
+
+END (__memchr_scalar)

string/aarch64/memcpy.S

@@ -53,6 +53,8 @@
    The loop tail is handled by always copying 64 bytes from the end.
 */
 
+ENTRY_ALIAS (__memmove_aarch64_sc)
+ENTRY_ALIAS (__memcpy_aarch64_sc)
 ENTRY_ALIAS (__memmove_aarch64)
 ENTRY (__memcpy_aarch64)
 	add	srcend, src, count

string/aarch64/memset-scalar.S

@@ -0,0 +1,165 @@
+/*
+ * __arm_sc_memset - fill memory with a constant byte
+ *
+ * Copyright (c) 2012-2026, Arm Limited.
+ * SPDX-License-Identifier: MIT OR Apache-2.0 WITH LLVM-exception
+ */
+
+/* Assumptions:
+ *
+ * ARMv8-a, AArch64, unaligned accesses.
+ *
+ */
+
+#include "asmdefs.h"
+
+#define dstin	x0
+#define val	x1
+#define valw	w1
+#define count	x2
+#define dst	x3
+#define dstend	x4
+#define zva_val	x5
+#define off	x3
+#define dstend2	x5
+
+/* This implementation is similar to __memset_aarch64 but avoids FP/AdvSIMD
+   register instructions.  */
+
+ENTRY_ALIAS (__memset_scalar_sc)
+ENTRY (__memset_scalar)
+	/* Replicate the fill byte across the last half of the word.  */
+	and	valw, valw, 255
+	orr	valw, valw, valw, lsl 8
+	orr	valw, valw, valw, lsl 16
+
+	cmp	count, 16
+	b.lo	L(set_lt16)
+
+	/* Replicate the fill byte across the rest of the word.  */
+	orr	val, val, val, lsl 32
+
+	add	dstend, dstin, count
+	cmp	count, 64
+	b.hi	L(set_gt64)
+
+L(set_16_64):
+	/* Calculate an interior store offset based on half the count, This will be
+	  one of 0, 16 or 32 bytes. Then perform four overlapping stores.  */
+	mov	off, 48
+	and	off, off, count, lsr 1
+	sub	dstend2, dstend, off
+
+	stp	val, val, [dstin]
+	add	off, dstin, off
+	stp	val, val, [off]
+	stp	val, val, [dstend2, -16]
+	stp	val, val, [dstend, -16]
+	ret
+
+	.p2align 4
+L(set_lt16):
+	add	dstend, dstin, count
+	cmp	count, 4
+	b.lo	1f
+
+	/* For sizes 4..15, salculate an interior offset based on the count. Then
+	   perform four overlapping 32 bit stores.  */
+	lsr	off, count, 3				/* off = count >> 3.  */
+	sub	dstend2, dstend, off, lsl 2
+	str	valw, [dstin]
+	str	valw, [dstin, off, lsl 2]
+	str	valw, [dstend2, -4]
+	str	valw, [dstend, -4]
+	ret
+
+	/* Potentially overlapping stores for sizes 0..3. */
+1:	cbz	count, 2f		/* Skip stores entirely for a zero-length memset.  */
+	lsr	off, count, 1
+	strb	valw, [dstin]
+	strb	valw, [dstin, off]
+	strb	valw, [dstend, -1]
+2:	ret
+
+	.p2align 4
+L(set_gt64):
+	bic	dst, dstin, 15		/* Align dst down to 16 bytes.  */
+	cmp	count, 128
+	b.hi	L(set_gt128)
+
+	/* For sizes 64..128, perform eight overlapping 16-byte stores.  */
+	stp	val, val, [dstin]
+	stp	val, val, [dstin, 16]
+	stp	val, val, [dstin, 32]
+	stp	val, val, [dstin, 48]
+	stp	val, val, [dstend, -64]
+	stp	val, val, [dstend, -48]
+	stp	val, val, [dstend, -32]
+	stp	val, val, [dstend, -16]
+	ret
+
+	.p2align 4
+L(set_gt128):
+	/* For sizes above 128, perform initial stores before entering loop on
+	   aligned addresses.  */
+	stp	val, val, [dstin]
+	stp	val, val, [dst, 16]
+
+	/* We can use DC ZVA to zero memory when the fill byte is zero and the block
+	   size reported by DCZID_EL0 is 64 bytes.  */
+	cbnz valw, L(no_zva)
+#ifndef SKIP_ZVA_CHECK
+	mrs	zva_val, dczid_el0
+	and	zva_val, zva_val, 31
+	cmp	zva_val, 4
+	b.ne	L(no_zva)
+#endif
+	/* Complete stores up to the first 64 byte aligned block.  */
+	stp	val, val, [dst, 32]
+	stp	val, val, [dst, 48]
+
+	bic	dst, dstin, 63		/* Align dst down to 64 bytes.  */
+	/* Compute count, accounting for the stores above and bias for the loop
+	   exit.  */
+	sub	count, dstend, dst
+	sub	count, count, 64 + 64
+
+	/* Write last bytes before entering ZVA loop.  */
+	stp	val, val, [dstend, -64]
+	stp	val, val, [dstend, -48]
+	stp	val, val, [dstend, -32]
+	stp	val, val, [dstend, -16]
+
+	.p2align 4
+L(zva64_loop):
+	add	dst, dst, 64
+	dc	zva, dst		/* Zero the 64-byte cache block containing dst.  */
+	subs	count, count, 64
+	b.hi	L(zva64_loop)
+	ret
+
+	.p2align 3
+L(no_zva):
+	/* Compute count, accounting for the two stores in set_gt128 and bias for
+	   the loop exit.  */
+	sub	count, dstend, dst
+	sub	count, count, 64 + 32
+
+L(no_zva_loop):
+	/* Perform eight contiguous 16-byte stores per iteration.  */
+	stp	val, val, [dst, 32]
+	stp	val, val, [dst, 48]
+	stp	val, val, [dst, 64]
+	stp	val, val, [dst, 80]
+	add	dst, dst, 64
+	subs	count, count, 64
+	b.hi	L(no_zva_loop)
+
+	/* Perform final stores for the tail of the range, potentially overlapping
+	   with previous ones.  */
+	stp	val, val, [dstend, -64]
+	stp	val, val, [dstend, -48]
+	stp	val, val, [dstend, -32]
+	stp	val, val, [dstend, -16]
+	ret
+END (__memset_scalar)

string/bench/memset.c

@@ -26,6 +26,7 @@ static uint8_t a[MAX_SIZE + 4096] __attribute__((__aligned__(4096)));
   printf (STR);					\
   RUN (TESTFN, memset);				\
   RUNA64 (TESTFN, __memset_aarch64);		\
+  RUNA64 (TESTFN, __memset_scalar);		\
   RUNSVE (TESTFN, __memset_aarch64_sve);	\
   RUNMOPS (TESTFN, __memset_aarch64_mops);	\
   RUNA32 (TESTFN, __memset_arm);		\

string/include/stringlib.h

@@ -16,7 +16,9 @@
 void *__memcpy_aarch64 (void *__restrict, const void *__restrict, size_t);
 void *__memmove_aarch64 (void *, const void *, size_t);
 void *__memset_aarch64 (void *, int, size_t);
+void *__memset_scalar (void *, int, size_t);
 void *__memchr_aarch64 (const void *, int, size_t);
+void *__memchr_scalar (const void *, int, size_t);
 void *__memrchr_aarch64 (const void *, int, size_t);
 int __memcmp_aarch64 (const void *, const void *, size_t);
 char *__strcpy_aarch64 (char *__restrict, const char *__restrict);
@@ -55,6 +57,13 @@ int __strncmp_aarch64_sve (const char *, const char *, size_t);
 char *__strchr_aarch64_sve2 (const char *, int);
 char *__strchrnul_aarch64_sve2 (const char *, int );
 # endif
+# if __ARM_FEATURE_SME
+/* These are used to test functionality in streaming mode */
+void *__memchr_scalar_sc (const void *, int, size_t) __arm_streaming_compatible;
+void *__memcpy_aarch64_sc (void *, const void *, size_t) __arm_streaming_compatible;
+void *__memmove_aarch64_sc (void *, const void *, size_t) __arm_streaming_compatible;
+void *__memset_scalar_sc (void *, int, size_t) __arm_streaming_compatible;
+# endif
 # if WANT_MOPS
 void *__memcpy_aarch64_mops (void *__restrict, const void *__restrict, size_t);
 void *__memmove_aarch64_mops (void *__restrict, const void *__restrict, size_t);

string/test/memchr.c

@@ -26,6 +26,7 @@ static const struct fun
   F(memchr, 0)
 #if __aarch64__
   F(__memchr_aarch64, 0)
+  F(__memchr_scalar, 0)
   F(__memchr_aarch64_mte, 1)
 # if __ARM_FEATURE_SVE
   F(__memchr_aarch64_sve, 1)

string/test/memset.c

@@ -25,6 +25,7 @@ static const struct fun
   F(memset, 0)
 #if __aarch64__
   F(__memset_aarch64, 1)
+  F(__memset_scalar, 1)
 # if __ARM_FEATURE_SVE
   F(__memset_aarch64_sve, 1)
 # endif