diff --git a/testing/libc/arch_libc/CMakeLists.txt b/testing/libc/arch_libc/CMakeLists.txt index 89e26e784..8c8eeff5a 100644 --- a/testing/libc/arch_libc/CMakeLists.txt +++ b/testing/libc/arch_libc/CMakeLists.txt @@ -21,6 +21,10 @@ # ############################################################################## if(CONFIG_TESTING_ARCH_LIBC) + if(CONFIG_TESTING_ARCH_LIBC_BENCH) + set(bench_srcs arch_libc_bench.c) + endif() + nuttx_add_application( NAME ${CONFIG_TESTING_ARCH_LIBC_PROGNAME} @@ -31,5 +35,6 @@ if(CONFIG_TESTING_ARCH_LIBC) MODULE ${CONFIG_TESTING_ARCH_LIBC} SRCS - arch_libc_test_main.c) + arch_libc_test_main.c + ${bench_srcs}) endif() diff --git a/testing/libc/arch_libc/Kconfig b/testing/libc/arch_libc/Kconfig index 05ccacdd8..0de9563cc 100644 --- a/testing/libc/arch_libc/Kconfig +++ b/testing/libc/arch_libc/Kconfig @@ -75,6 +75,32 @@ config TESTING_ARCH_LIBC_STRCHRNUL bool "test strchrnul" default y +config TESTING_ARCH_LIBC_BENCH + bool "throughput benchmark" + default n + ---help--- + Measure each function across a size sweep and every source and + destination alignment pair, reporting MB/s and cycles per byte. + An implementation usually takes its wide path only when the + pointers meet some alignment condition, so an aligned measurement + alone does not show what the rest of the input space costs. + +if TESTING_ARCH_LIBC_BENCH + +config TESTING_ARCH_LIBC_BENCH_BUFSIZE + int "benchmark buffer size" + default 262144 + ---help--- + Size of each of the two buffers the benchmark allocates. + +config TESTING_ARCH_LIBC_BENCH_MINMS + int "minimum milliseconds per measurement" + default 250 + ---help--- + Each measurement repeats until it has run for at least this long. + +endif + config TESTING_ARCH_LIBC_PROGNAME string "Program name" default "arch_libctest" diff --git a/testing/libc/arch_libc/Makefile b/testing/libc/arch_libc/Makefile index 610e10fa9..918497580 100644 --- a/testing/libc/arch_libc/Makefile +++ b/testing/libc/arch_libc/Makefile @@ -31,6 +31,10 @@ MODULE = $(CONFIG_TESTING_ARCH_LIBC) # arch libc test +ifneq ($(CONFIG_TESTING_ARCH_LIBC_BENCH),) +CSRCS += arch_libc_bench.c +endif + MAINSRC = arch_libc_test_main.c include $(APPDIR)/Application.mk diff --git a/testing/libc/arch_libc/arch_libc_bench.c b/testing/libc/arch_libc/arch_libc_bench.c new file mode 100644 index 000000000..d73fd48f9 --- /dev/null +++ b/testing/libc/arch_libc/arch_libc_bench.c @@ -0,0 +1,420 @@ +/**************************************************************************** + * apps/testing/libc/arch_libc/arch_libc_bench.c + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +/* Throughput for the string and memory functions a machine directory may + * override, across a size sweep and every source/destination alignment + * pair. A machine implementation usually takes its wide path only when + * the pointers satisfy some alignment condition, so a single aligned size + * reports the best case and hides what the rest of the input space costs. + * + * Each measurement reports MB/s, which compares across machines, and + * cycles per byte from perf_gettime(), which does not depend on the + * timebase being calibrated. Both come from one timed loop. + * + * Two ways a benchmark of these functions measures nothing are defeated + * here: the compiler treating a pure call with unchanged arguments as loop + * invariant, countered by laundering the pointers through an asm that + * claims to change them, and comparison inputs an earlier measurement + * scribbled on, countered by preparing the buffers before each timed loop. + */ + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +#include +#include +#include +#include +#include +#include + +#include + +/**************************************************************************** + * Pre-processor Definitions + ****************************************************************************/ + +#define BENCH_BUF (CONFIG_TESTING_ARCH_LIBC_BENCH_BUFSIZE) +#define BENCH_MINMS (CONFIG_TESTING_ARCH_LIBC_BENCH_MINMS) +#define BENCH_GUARD 64 +#define BENCH_BATCH 16 + +/* perf_gettime() reaches an application only where the C library builds + * its own copy, or where the application and the kernel are one image. + * Elsewhere the cycle count is left out and the throughput stands alone. + */ + +#if defined(CONFIG_ARCH_HAVE_PERF_EVENTS_USER_ACCESS) || \ + defined(CONFIG_BUILD_FLAT) +# define BENCH_CYCLES 1 +#endif + +/* Operations. A writer stores through the destination pointer, so its + * destination offset is worth sweeping; a reader only takes one. + */ + +#define OP_MEMCPY 0 +#define OP_MEMMOVE 1 +#define OP_MEMSET 2 +#define OP_MEMCMP 3 +#define OP_MEMCHR 4 +#define OP_STRLEN 5 +#define OP_STRCMP 6 +#define OP_STRNCMP 7 +#define OP_STRCPY 8 +#define OP_STRLCPY 9 +#define OP_STRCHR 10 +#define OP_STRRCHR 11 +#define OP_STRCHRNUL 12 +#define OP_STRNLEN 13 +#define OP_STPCPY 14 +#define OP_STRNCPY 15 +#define OP_STRCAT 16 + +/**************************************************************************** + * Private Types + ****************************************************************************/ + +struct bench_op_s +{ + FAR const char *name; + uint8_t op; + bool dst; /* Touches the destination pointer, so its + * offset is worth sweeping too */ + bool str; /* Operand is a string, so it needs a + * terminator at the measured length */ +}; + +/**************************************************************************** + * Private Data + ****************************************************************************/ + +static FAR char *g_src; +static FAR char *g_dst; +static volatile unsigned long g_sink; + +static const struct bench_op_s g_ops[] = +{ + {"memcpy", OP_MEMCPY, true, false}, + {"memmove", OP_MEMMOVE, true, false}, + {"memset", OP_MEMSET, true, false}, + {"memcmp", OP_MEMCMP, true, false}, + {"memchr", OP_MEMCHR, false, false}, + {"strlen", OP_STRLEN, false, true}, + {"strcmp", OP_STRCMP, true, true}, + {"strncmp", OP_STRNCMP, true, true}, + {"strcpy", OP_STRCPY, true, true}, + {"strlcpy", OP_STRLCPY, true, true}, + {"strchr", OP_STRCHR, false, true}, + {"strrchr", OP_STRRCHR, false, true}, + {"strchrnul", OP_STRCHRNUL, false, true}, + {"strnlen", OP_STRNLEN, false, true}, + {"stpcpy", OP_STPCPY, true, true}, + {"strncpy", OP_STRNCPY, true, true}, + {"strcat", OP_STRCAT, true, true}, +}; + +static const size_t g_sizes[] = +{ + 64, 512, 4096, 32768 +}; + +/* Source and destination offsets within a register. Aligned, equally + * misaligned, misaligned by different amounts, and a source shift against + * an aligned destination. + */ + +static const uint8_t g_aligns[][2] = +{ + { + 0, 0 + }, + { + 1, 1 + }, + { + 1, 2 + }, + { + 3, 0 + } +}; + +#define NOPS (sizeof(g_ops) / sizeof(g_ops[0])) +#define NSIZES (sizeof(g_sizes) / sizeof(g_sizes[0])) +#define NALIGNS (sizeof(g_aligns) / sizeof(g_aligns[0])) + +/**************************************************************************** + * Private Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: bench_now + ****************************************************************************/ + +static double bench_now(void) +{ + struct timespec t; + + clock_gettime(CLOCK_MONOTONIC, &t); + return t.tv_sec + t.tv_nsec / 1e9; +} + +/**************************************************************************** + * Name: bench_seen_src + * + * Description: + * Whether an earlier alignment pair already used this source offset. + * + ****************************************************************************/ + +static bool bench_seen_src(size_t ai) +{ + size_t i; + + for (i = 0; i < ai; i++) + { + if (g_aligns[i][0] == g_aligns[ai][0]) + { + return true; + } + } + + return false; +} + +/**************************************************************************** + * Name: bench_prepare + * + * Description: + * Content for one measurement. The compares need the two operands equal + * for the whole length, or they stop early and time nothing, and the + * absent-character scans need a byte that never appears. + * + ****************************************************************************/ + +static void bench_prepare(FAR const struct bench_op_s *o, size_t n, + int so, int dofs) +{ + FAR char *s = g_src + BENCH_GUARD + so; + FAR char *d = g_dst + BENCH_GUARD + dofs; + size_t i; + + for (i = 0; i < n + BENCH_GUARD; i++) + { + s[i] = 'a' + (i % 23); + } + + memcpy(d, s, n + BENCH_GUARD); + + if (o->str) + { + s[n] = '\0'; + d[n] = '\0'; + } +} + +/**************************************************************************** + * Name: bench_one + * + * Description: + * One operation at one size and one alignment pair, timed until it has + * run for at least the configured interval. + * + ****************************************************************************/ + +static void bench_one(FAR const struct bench_op_s *o, size_t n, + int so, int dofs) +{ +#ifdef BENCH_CYCLES + clock_t c0; + clock_t c1; +#endif + double t0; + double el; + double mbs; + unsigned long reps = 0; + int i; + + bench_prepare(o, n, so, dofs); + + t0 = bench_now(); +#ifdef BENCH_CYCLES + c0 = perf_gettime(); +#endif + + do + { + for (i = 0; i < BENCH_BATCH; i++) + { + FAR char *s = g_src + BENCH_GUARD + so; + FAR char *d = g_dst + BENCH_GUARD + dofs; + + __asm__ volatile("" : "+r"(s), "+r"(d)); + + switch (o->op) + { + case OP_MEMCPY: + memcpy(d, s, n); + break; + case OP_MEMMOVE: + memmove(d + 8, d, n); + break; + case OP_MEMSET: + memset(d, i, n); + break; + case OP_MEMCMP: + g_sink += memcmp(s, d, n); + break; + case OP_MEMCHR: + g_sink += (uintptr_t)memchr(s, '~', n); + break; + case OP_STRLEN: + g_sink += strlen(s); + break; + case OP_STRCMP: + g_sink += strcmp(s, d); + break; + case OP_STRNCMP: + g_sink += strncmp(s, d, n); + break; + case OP_STRCPY: + strcpy(d, s); + break; + case OP_STRLCPY: + g_sink += strlcpy(d, s, n + 1); + break; + case OP_STRCHR: + g_sink += (uintptr_t)strchr(s, '~'); + break; + case OP_STRRCHR: + g_sink += (uintptr_t)strrchr(s, '~'); + break; + case OP_STRCHRNUL: + g_sink += (uintptr_t)strchrnul(s, '~'); + break; + case OP_STRNLEN: + g_sink += strnlen(s, n); + break; + case OP_STPCPY: + g_sink += (uintptr_t)stpcpy(d, s); + break; + case OP_STRNCPY: + strncpy(d, s, n); + break; + case OP_STRCAT: + + /* Appending to what the last turn appended would grow the + * destination without bound, so it starts empty each time. + * The store is inside the measurement. + */ + + d[0] = '\0'; + strcat(d, s); + break; + } + + g_sink += (unsigned char)d[0]; + } + + reps += BENCH_BATCH; + el = bench_now() - t0; + } + while (el * 1000.0 < BENCH_MINMS); + + mbs = (double)reps * n / el / 1048576.0; + +#ifdef BENCH_CYCLES + c1 = perf_gettime(); + printf(" %-8s %6zu B s+%d/d+%d %9.1f MB/s %8.3f cyc/B\n", + o->name, n, so, dofs, mbs, + (double)(uintmax_t)(c1 - c0) / ((double)reps * n)); +#else + printf(" %-8s %6zu B s+%d/d+%d %9.1f MB/s\n", + o->name, n, so, dofs, mbs); +#endif +} + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: arch_libc_bench + * + * Description: + * Run every operation over the size sweep and the alignment matrix. + * + ****************************************************************************/ + +int arch_libc_bench(void) +{ + size_t oi; + size_t si; + size_t ai; + + g_src = malloc(BENCH_BUF); + g_dst = malloc(BENCH_BUF); + if (g_src == NULL || g_dst == NULL) + { + printf("arch_libc bench: cannot allocate 2 x %d\n", BENCH_BUF); + free(g_src); + free(g_dst); + return 1; + } + + printf("== throughput ==\n"); + + for (oi = 0; oi < NOPS; oi++) + { + for (si = 0; si < NSIZES; si++) + { + if (g_sizes[si] + 2 * BENCH_GUARD + 8 > BENCH_BUF) + { + continue; + } + + for (ai = 0; ai < NALIGNS; ai++) + { + /* An operation that never touches the destination is decided + * by the source offset alone, so a pair repeating one already + * measured would report the same number twice. + */ + + if (!g_ops[oi].dst && bench_seen_src(ai)) + { + continue; + } + + bench_one(&g_ops[oi], g_sizes[si], + g_aligns[ai][0], g_aligns[ai][1]); + } + } + } + + free(g_src); + free(g_dst); + return 0; +} diff --git a/testing/libc/arch_libc/arch_libc_test_main.c b/testing/libc/arch_libc/arch_libc_test_main.c index 1a5e8bbf1..9e0a222f1 100644 --- a/testing/libc/arch_libc/arch_libc_test_main.c +++ b/testing/libc/arch_libc/arch_libc_test_main.c @@ -44,6 +44,14 @@ #define TEST_REPEAT 100 #define MAX_ALIGN 16 +/**************************************************************************** + * Public Function Prototypes + ****************************************************************************/ + +#ifdef CONFIG_TESTING_ARCH_LIBC_BENCH +int arch_libc_bench(void); +#endif + /**************************************************************************** * Private Data ****************************************************************************/ @@ -1458,6 +1466,9 @@ int main(int argc, FAR char *argv[]) fail += test_strchrnul(); speed_strchrnul(); #endif +#ifdef CONFIG_TESTING_ARCH_LIBC_BENCH + fail += arch_libc_bench(); +#endif printf("arch_libc_test %s\n", fail ? "Failed" : "Passed"); return fail ? EXIT_FAILURE : EXIT_SUCCESS;