mirror of
https://github.com/apache/nuttx.git
synced 2026-08-30 05:40:37 +00:00
0790b52e64
memcmp, strncmp and strcmp reach their word loops only when both pointers
are already on a register boundary:
or t0, a0, a1
andi t0, t0, SZREG-1
That asks more than the loops need. They load from the two pointers at
the same boundary, so what matters is that the two agree about where a
boundary falls, not that either is already on one. A pair offset by the
same amount can be walked up to the boundary a byte at a time and
compared a register at a time from there.
The union also holds far less often than the difference. For arbitrary
pointers on RV64 it is true about one time in 64 against one in eight,
and the case it rejects, two strings carved out of the same buffer, is
the common one.
Test the difference of the pointers, and walk to the boundary first.
arch_strcpy.S and arch_memcpy.S already do this. Keeping every access
aligned is not only faster here: the base ISA does not require misaligned
loads and stores to be supported at all, so a routine in a machine
directory cannot assume one will work, whatever it costs.
Measured on a 1.4 GHz rv64, source and destination misaligned by one:
before after
memcmp 32K 34.4 458.0 MB/s
strncmp 32K 32.4 253.0 MB/s
strcmp 32K 41.0 280.0 MB/s
Each of those was the rate of the byte loop the word loop was meant to
replace. Pointers that genuinely disagree still take the byte loop, and
the aligned rates are unchanged.
The measurements come from the benchmark in apache/nuttx-apps#3706.
Assisted-by: Claude:claude-opus-5
Signed-off-by: Justin Hammond <justin@dynam.ac>
159 lines
3.6 KiB
ArmAsm
159 lines
3.6 KiB
ArmAsm
/****************************************************************************
|
|
* libs/libc/machine/risc-v/arch_memcmp.S
|
|
*
|
|
* SPDX-License-Identifier: Apache-2.0
|
|
*
|
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
|
* contributor license agreements. See the NOTICE file distributed with
|
|
* this work for additional information regarding copyright ownership. The
|
|
* ASF licenses this file to you under the Apache License, Version 2.0 (the
|
|
* "License"); you may not use this file except in compliance with the
|
|
* License. You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
* License for the specific language governing permissions and limitations
|
|
* under the License.
|
|
*
|
|
****************************************************************************/
|
|
|
|
#include "libc.h"
|
|
|
|
#ifdef LIBC_BUILD_MEMCMP
|
|
|
|
#include "asm.h"
|
|
|
|
.text
|
|
.global ARCH_LIBCFUN(memcmp)
|
|
.type ARCH_LIBCFUN(memcmp), @function
|
|
.align 2
|
|
|
|
/************************************************************************************
|
|
* Name: memcmp
|
|
*
|
|
* int memcmp(const void *s1, const void *s2, size_t n)
|
|
*
|
|
* Word-at-a-time comparison with 4-word XOR|OR folding to reduce branch
|
|
* overhead in the main loop.
|
|
************************************************************************************/
|
|
ARCH_LIBCFUN(memcmp):
|
|
.cfi_sections .debug_frame
|
|
.cfi_startproc
|
|
|
|
beqz a2, .Lequal
|
|
|
|
/* The loops below load from both pointers at a boundary, which asks
|
|
* that the two agree about where a boundary falls, not that either
|
|
* is already on one. Test the difference of the pointers, not their
|
|
* union.
|
|
*/
|
|
|
|
xor t0, a0, a1
|
|
andi t0, t0, SZREG-1
|
|
bnez t0, .Lbyte_cmp
|
|
|
|
/* Same offset: walk both up to the boundary a byte at a time,
|
|
* stopping early on a difference.
|
|
*/
|
|
|
|
.Lalign_head:
|
|
andi t0, a0, SZREG-1
|
|
beqz t0, .Laligned
|
|
lbu t1, 0(a0)
|
|
lbu t2, 0(a1)
|
|
bne t1, t2, .Ldiff
|
|
addi a0, a0, 1
|
|
addi a1, a1, 1
|
|
addi a2, a2, -1
|
|
beqz a2, .Lequal
|
|
j .Lalign_head
|
|
|
|
.Laligned:
|
|
li t0, SZREG
|
|
bltu a2, t0, .Lbyte_cmp
|
|
|
|
/* Need at least 4*SZREG for unrolled loop */
|
|
|
|
li a3, 4*SZREG
|
|
bltu a2, a3, .Lword_loop
|
|
|
|
/* 4-word unrolled loop: XOR each pair, OR the results.
|
|
* One branch per 4 words. On difference, rewind and let
|
|
* the single-word loop find the exact mismatch word.
|
|
*/
|
|
|
|
.Lword4_loop:
|
|
REG_L t1, 0*SZREG(a0)
|
|
REG_L t2, 0*SZREG(a1)
|
|
REG_L t3, 1*SZREG(a0)
|
|
REG_L t4, 1*SZREG(a1)
|
|
xor t1, t1, t2
|
|
xor t3, t3, t4
|
|
or t1, t1, t3
|
|
|
|
REG_L t3, 2*SZREG(a0)
|
|
REG_L t4, 2*SZREG(a1)
|
|
REG_L t5, 3*SZREG(a0)
|
|
REG_L t6, 3*SZREG(a1)
|
|
xor t3, t3, t4
|
|
xor t5, t5, t6
|
|
or t3, t3, t5
|
|
or t1, t1, t3
|
|
|
|
bnez t1, .Lword_loop
|
|
addi a0, a0, 4*SZREG
|
|
addi a1, a1, 4*SZREG
|
|
sub a2, a2, a3
|
|
bgeu a2, a3, .Lword4_loop
|
|
|
|
/* Fall through to single-word loop for remainder */
|
|
|
|
.Lword_loop:
|
|
bltu a2, t0, .Lbyte_cmp
|
|
REG_L t1, 0(a0)
|
|
REG_L t2, 0(a1)
|
|
bne t1, t2, .Lfind_diff
|
|
addi a0, a0, SZREG
|
|
addi a1, a1, SZREG
|
|
sub a2, a2, t0
|
|
bgeu a2, t0, .Lword_loop
|
|
beqz a2, .Lequal
|
|
|
|
.Lbyte_cmp:
|
|
beqz a2, .Lequal
|
|
lbu t1, 0(a0)
|
|
lbu t2, 0(a1)
|
|
bne t1, t2, .Ldiff
|
|
addi a0, a0, 1
|
|
addi a1, a1, 1
|
|
addi a2, a2, -1
|
|
bnez a2, .Lbyte_cmp
|
|
|
|
.Lequal:
|
|
li a0, 0
|
|
ret
|
|
|
|
.Ldiff:
|
|
sub a0, t1, t2
|
|
ret
|
|
|
|
.Lfind_diff:
|
|
/* Little-endian: first differing byte is at LSB side */
|
|
|
|
andi t3, t1, 0xff
|
|
andi t4, t2, 0xff
|
|
bne t3, t4, .Lfound
|
|
srli t1, t1, 8
|
|
srli t2, t2, 8
|
|
j .Lfind_diff
|
|
.Lfound:
|
|
sub a0, t3, t4
|
|
ret
|
|
|
|
.cfi_endproc
|
|
.size ARCH_LIBCFUN(memcmp), .-ARCH_LIBCFUN(memcmp)
|
|
|
|
#endif
|