arm64/mte: Add support for arm64 mte

For details, please refer to the kernel's introduction to this at "https://docs.kernel.org/arch/arm64/memory-tagging-extension.html" and Android's introduction to this at "https://source.android.com/docs/security/test/memory-safety/arm-mte"

Of course, there is also the following detailed principle introduction
https://developer.arm.com/-/media/Arm%20Developer%20Community/PDF/Arm_Memory_Tagging_Extension_Whitepaper.pdf

The modification of this patch is only to merge the simplest MTE function support. In the future, the MTE function will be integrated into the kernel to a greater extent, for example, hardware MTE Kasan will be supported in the future.

Signed-off-by: wangmingrong1 <wangmingrong1@xiaomi.com>

Documentation/guides/index.rst

@@ -45,6 +45,7 @@ Guides
   kernel_threads_with_custom_stacks.rst
   versioning_and_task_names.rst
   logging_rambuffer.rst
+  mte.rst
   ipv6.rst
   integrate_newlib.rst
   protected_build.rst

Documentation/guides/mte.rst

@@ -0,0 +1,95 @@
+====================================
+ATM64 MTE extension
+====================================
+
+Introduction
+------------
+
+Arm v8.5 introduced the Arm Memory Tagging Extension (MTE),
+a hardware implementation of tagged memory.
+
+Basically, MTE tags every memory allocation/deallocation
+with additional metadata. It assigns a tag to a memory location,
+which can then be associated with a pointer that references
+that memory location. At runtime, the CPU checks that the pointer
+and metadata tags match with every load and store.
+
+NX OS currently supports deploying MTE on ARM64 QEMU,
+which is supported at the EL1 level of NX OS.
+
+Principle
+---------
+
+The Arm Memory Tagging Extension implements lock and key access to memory.
+Locks can be set on memory and keys provided during memory access. If the key matches
+the lock, the access is permitted. If it does not match, an error is reported.
+
+Memory locations are tagged by adding four bits of metadata to each 16 bytes
+of physical memory. This is the Tag Granule. Tagging memory implements the lock.
+Pointers, and therefore virtual addresses, are modified to contain the key.
+In order to implement the key bits without requiring larger pointers MTE uses the Top Byte
+Ignore (TBI) feature of the Armv8-A Architecture. When TBI is enabled, the top byte of
+a virtual address is ignored when using it as an input for address translation. This allows the
+top byte to store metadata. In MTE four bits of the top byte are used to provide the key
+
+Architectural Details
+---------------------
+
+MTE adds instructions to the Armv8-A Architecture that are outlined below and grouped
+into three different categories [6]:
+Instructions for tag manipulation applicable to stack and heap tagging.
+
+IRG
+In order for the statistical basis of MTE to be valid, a source of random tags is required.
+IRG is defined to provide this in hardware and insert such a tag into a register for use
+by other instructions.
+
+GMI
+This instruction is for manipulating the excluded set of tags for use with the IRG instruction.
+This is intended for cases where software uses specific tag values for special purposes
+while retaining random tag behavior for normal allocations.
+
+LDG, STG, and STZG
+These instructions allow getting or setting tags in memory. They are intended for changing
+tags in memory either without modifying the data or zeroing the data.
+
+ST2G and STZ2G
+These are denser alternatives to STG and STZG which operate on two granules of memory
+when allocation size allows them to be used.
+
+STGP
+This instruction stores both tag and data to memory.
+Instructions Intended for pointer arithmetic and stack tagging:
+
+ADDG and SUBG
+These are variants of the ADD and SUB instructions, intended for arithmetic on addresses.
+They allow both the tag and address to be separately modified by an immediate value.
+These instructions are intended for creating the addresses of objects on the stack.
+
+SUBP(S)
+This instruction provides a 56-bit subtract with optional flag setting which is required
+for pointer arithmetic that ignores the tag in the top byte.
+
+Instructions intended for system use:
+LDGM, STGM, and STZGM
+These are bulk tag manipulation instructions which are UNDEFINED at EL0. These are
+intended for system software to manipulate tags for the purposes of initialization and
+serialization. For example, they can be used to implement swapping of tagged memory
+to a medium which is not tag-aware. The zeroing form can be used for efficient
+initialization of memory.
+
+Currently NX OS supports the execution of the above instructions,
+such as irg, ldg, stg instructions.
+Their test programs are stored in "apps/system/mte" to test whether the current system supports
+
+Usage
+-----
+
+If you want to experience the MTE function of NX OS, you can refer to the following：
+To enable ARM64_MTE, configure the kernel with::
+
+    CONFIG_ARM64_MTE=y
+
+Of course you can also run it with the existing configuration：
+
+    boards/arm64/qemu/qemu-armv8a/configs/mte

arch/arm64/Kconfig

@@ -108,7 +108,7 @@ config ARCH_CHIP_ZYNQ_MPSOC
 	select ARM64_HAVE_PSCI
 	---help---
 		XilinX ZYNQ MPSOC
-		
+
 config ARCH_CHIP_ARM64_CUSTOM
 	bool "Custom ARM64 chip"
 	select ARCH_CHIP_CUSTOM
@@ -127,6 +127,21 @@ config ARCH_ARMV8R
 	default n
 	select ARCH_SINGLE_SECURITY_STATE
 
+config ARCH_AS_HAS_ARMV8_5
+	bool "Support ARMv8.5 assembly"
+	depends on ARCH_ARMV8A
+	---help---
+		Support ARMv8.5 assembly instruction set
+
+menu "ARMv8.5 architectural features"
+	depends on ARCH_AS_HAS_ARMV8_5
+
+config ARM64_MTE
+	bool "Memory Tagging Extension support"
+	default y
+
+endmenu # "ARMv8.5 architectural features"
+
 config ARCH_SINGLE_SECURITY_STATE
 	bool "ARM Single Security State Support"
 	default n

arch/arm64/src/Toolchain.defs

@@ -30,7 +30,19 @@
 #                   reliable code generation.
 #
 
-ifeq ($(CONFIG_ARCH_CORTEX_A53),y)
+ifeq ($(CONFIG_ARCH_ARMV8A),y)
+  ifeq ($(CONFIG_ARCH_AS_HAS_ARMV8_5),y)
+    OPTION_MARCH = -march=armv8.5-a
+  else
+    OPTION_MARCH = -march=armv8-a
+  endif
+  ifeq ($(CONFIG_ARM64_MTE),y)
+    OPTION_MARCH_FEATURE = +memtag
+  endif
+  ARCHCPUFLAGS += $(OPTION_MARCH)$(OPTION_MARCH_FEATURE)
+else ifeq ($(CONFIG_ARCH_ARMV8R),y)
+  ARCHCPUFLAGS += -march=armv8-r
+else ifeq ($(CONFIG_ARCH_CORTEX_A53),y)
   ARCHCPUFLAGS += -mcpu=cortex-a53
 else ifeq ($(CONFIG_ARCH_CORTEX_A55),y)
   ARCHCPUFLAGS += -mcpu=cortex-a55
@@ -40,10 +52,6 @@ else ifeq ($(CONFIG_ARCH_CORTEX_A72),y)
   ARCHCPUFLAGS += -mcpu=cortex-a72
 else ifeq ($(CONFIG_ARCH_CORTEX_R82),y)
   ARCHCPUFLAGS += -mcpu=cortex-r82
-else ifeq ($(CONFIG_ARCH_ARMV8A),y)
-  ARCHCPUFLAGS += -march=armv8-a
-else ifeq ($(CONFIG_ARCH_ARMV8R),y)
-  ARCHCPUFLAGS += -march=armv8-r
 endif
 
 ifeq ($(CONFIG_DEBUG_CUSTOMOPT),y)

arch/arm64/src/common/CMakeLists.txt

@@ -61,6 +61,10 @@ if(CONFIG_ARCH_HAVE_MMU)
   list(APPEND SRCS arm64_mmu.c)
 endif()
 
+if(CONFIG_ARM64_MTE)
+  list(APPEND SRCS arm64_mte.c)
+endif()
+
 if(CONFIG_ARCH_HAVE_MPU)
   list(APPEND SRCS arm64_mpu.c)
 endif()

arch/arm64/src/common/Make.defs

@@ -74,6 +74,10 @@ ifeq ($(CONFIG_ARCH_HAVE_MMU),y)
 CMN_CSRCS += arm64_mmu.c
 endif
 
+ifeq ($(CONFIG_ARM64_MTE),y)
+CMN_CSRCS += arm64_mte.c
+endif
+
 ifeq ($(CONFIG_ARCH_HAVE_MPU),y)
 CMN_CSRCS += arm64_mpu.c
 common/arm64_mpu.c_CFLAGS += -fno-sanitize=kernel-address

arch/arm64/src/common/arm64_arch.h

@@ -96,6 +96,18 @@
 #define SCTLR_SA_BIT        BIT(3)
 #define SCTLR_I_BIT         BIT(12)
 
+/* Controls the impact of tag check faults
+ * due to loads and stores in EL0 EL1.
+ */
+
+#define SCTLR_TCF0_BIT       BIT(38)
+#define SCTLR_TCF1_BIT       BIT(40)
+
+/* Controlling EL0 EL1 access to assigned tags */
+
+#define SCTLR_ATA0_BIT       BIT(42)
+#define SCTLR_ATA_BIT        BIT(43)
+
 #define ACTLR_AUX_BIT        BIT(9)
 #define ACTLR_CLPORTS_BIT    BIT(8)
 #define ACTLR_CLPMU_BIT      BIT(7)
@@ -135,6 +147,12 @@
 #define SPSR_MODE_EL3H      (0xd)
 #define SPSR_MODE_MASK      (0xf)
 
+#define RGSR_EL1_TAG_MASK   0xfUL
+#define RGSR_EL1_SEED_SHIFT 8
+#define RGSR_EL1_SEED_MASK  0xffffUL
+
+#define TTBR_CNP_BIT        BIT(0)
+
 /* CurrentEL: Current Exception Level */
 
 #define MODE_EL_SHIFT       (0x2)
@@ -243,6 +261,7 @@
 #define HCR_IMO_BIT                 BIT(4)
 #define HCR_AMO_BIT                 BIT(5)
 #define HCR_RW_BIT                  BIT(31)
+#define HCR_ATA_BIT                 BIT(56)
 
 /* CNTHCTL_EL2 bits definitions */
 
@@ -485,6 +504,12 @@ uint64_t arm64_get_mpid(int cpu);
 int arm64_get_cpuid(uint64_t mpid);
 #endif
 
+#ifdef CONFIG_ARM64_MTE
+void arm64_enable_mte(void);
+#else
+#define arm64_enable_mte()
+#endif
+
 #endif /* __ASSEMBLY__ */
 
 #endif /* ___ARCH_ARM64_SRC_COMMON_ARM64_ARCH_H */

arch/arm64/src/common/arm64_boot.c

@@ -137,6 +137,11 @@ void arm64_boot_el2_init(void)
 
   reg = read_sysreg(hcr_el2);
   reg |= HCR_RW_BIT;      /* EL1 Execution state is AArch64 */
+
+#ifdef CONFIG_ARM64_MTE
+  reg |= HCR_ATA_BIT;
+#endif
+
   write_sysreg(reg, hcr_el2);
 
   reg = 0U;                   /* RES0 */

arch/arm64/src/common/arm64_mmu.c

@@ -159,6 +159,12 @@
 #define TCR_KASAN_SW_FLAGS 0
 #endif
 
+#ifdef CONFIG_ARM64_MTE
+#define TCR_MTE_FLAGS (TCR_TCMA1 | TCR_TBI0 | TCR_TBI1 | TCR_ASID_8)
+#else
+#define TCR_MTE_FLAGS 0
+#endif
+
 /****************************************************************************
  * Private Data
  ****************************************************************************/
@@ -262,7 +268,7 @@ static uint64_t get_tcr(int el)
    */
 
   tcr |= TCR_TG0_4K | TCR_SHARED_INNER | TCR_ORGN_WBWA |
-         TCR_IRGN_WBWA | TCR_KASAN_SW_FLAGS;
+         TCR_IRGN_WBWA | TCR_KASAN_SW_FLAGS | TCR_MTE_FLAGS;
 
   return tcr;
 }

arch/arm64/src/common/arm64_mmu.h

@@ -49,11 +49,17 @@
 #define MT_NORMAL_NC                3U
 #define MT_NORMAL                   4U
 
+#ifdef CONFIG_ARM64_MTE
+#define MT_NORMAL_VAL               0xf0UL
+#else
+#define MT_NORMAL_VAL               0xffUL
+#endif
+
 #define MEMORY_ATTRIBUTES           ((0x00 << (MT_DEVICE_NGNRNE * 8)) |  \
                                      (0x04 << (MT_DEVICE_NGNRE * 8))   | \
                                      (0x0c << (MT_DEVICE_GRE * 8))     | \
                                      (0x44 << (MT_NORMAL_NC * 8))      | \
-                                     (0xffUL << (MT_NORMAL * 8)))
+                                     (MT_NORMAL_VAL << (MT_NORMAL * 8)))
 
 /* More flags from user's perpective are supported using remaining bits
  * of "attrs" field, i.e. attrs[31:3], underlying code will take care
@@ -155,6 +161,16 @@
 #define TCR_TBI0                    (1ULL << 37)
 #define TCR_TBI1                    (1ULL << 38)
 
+/* TCMA1 (bit [58]) controls whether memory accesses
+ * in the address range [59:55] = 0b11111 are unchecked accesses.
+ *
+ * TCMA0 (bit [57]) controls whether memory accesses
+ * in the address range [59:55] = 0b00000 are unchecked accesses.
+ */
+
+#define TCR_TCMA0                   (1ULL << 57)
+#define TCR_TCMA1                   (1ULL << 58)
+
 #define TCR_PS_BITS_4GB             0x0ULL
 #define TCR_PS_BITS_64GB            0x1ULL
 #define TCR_PS_BITS_1TB             0x2ULL

arch/arm64/src/common/arm64_mte.c

@@ -0,0 +1,107 @@
+/****************************************************************************
+ * arch/arm64/src/common/arm64_mte.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <assert.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include "arm64_arch.h"
+
+/****************************************************************************
+ * Pre-processor Definitions
+ ****************************************************************************/
+
+#define GCR_EL1_VAL     0x10001
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+static int arm64_mte_is_support(void)
+{
+  int supported;
+  __asm__ volatile (
+    "mrs %0, ID_AA64PFR1_EL1\n"
+    "ubfx %0, %0, #8, #3\n"
+    : "=r" (supported)
+    :
+    : "memory"
+  );
+  return supported != 0;
+}
+
+/****************************************************************************
+ * Public Functions
+ ****************************************************************************/
+
+void arm64_enable_mte(void)
+{
+  uint64_t val;
+
+  if (!arm64_mte_is_support())
+    {
+      return;
+    }
+
+  /* CnP must be enabled only after the MAIR_EL1 register has been set
+   * up. Inconsistent MAIR_EL1 between CPUs sharing the same TLB may
+   * lead to the wrong memory type being used for a brief window during
+   * CPU power-up.
+   *
+   * CnP is not a boot feature so MTE gets enabled before CnP, but let's
+   * make sure that is the case.
+   */
+
+  assert(!(read_sysreg(ttbr0_el1) & TTBR_CNP_BIT));
+  assert(!(read_sysreg(ttbr1_el1) & TTBR_CNP_BIT));
+
+  val = read_sysreg(sctlr_el1);
+  val |= SCTLR_ATA_BIT | SCTLR_TCF1_BIT;
+  write_sysreg(val, sctlr_el1);
+
+  write_sysreg(GCR_EL1_VAL, gcr_el1);
+
+  /* If GCR_EL1.RRND=1 is implemented the same way as RRND=0, then
+   * RGSR_EL1.SEED must be non-zero for IRG to produce
+   * pseudorandom numbers. As RGSR_EL1 is UNKNOWN out of reset, we
+   * must initialize it.
+   */
+
+  val = (read_sysreg(CNTVCT_EL0) & RGSR_EL1_SEED_MASK) <<
+         RGSR_EL1_SEED_SHIFT;
+
+  if (0 == val)
+    {
+      val = 1 << RGSR_EL1_SEED_SHIFT;
+    }
+
+  write_sysreg(val, rgsr_el1);
+
+  /* clear any pending tag check faults in TFSR*_EL1 */
+
+  write_sysreg(0, tfsr_el1);
+  write_sysreg(0, tfsre0_el1);
+}

arch/arm64/src/qemu/qemu_boot.c

@@ -159,6 +159,8 @@ void arm64_chip_boot(void)
 
   arm64_mmu_init(true);
 
+  arm64_enable_mte();
+
 #ifdef CONFIG_DEVICE_TREE
   fdt_register((const char *)0x40000000);
 #endif