diff --git a/arch/arm64/include/arch.h b/arch/arm64/include/arch.h index 9782ca9b730..281d7cf504f 100644 --- a/arch/arm64/include/arch.h +++ b/arch/arm64/include/arch.h @@ -33,8 +33,7 @@ #ifndef __ASSEMBLY__ # include -# include -# include +# include #endif /**************************************************************************** @@ -46,6 +45,11 @@ # error Only pages sizes of 4096 are currently supported (CONFIG_ARCH_ADDRENV) #endif +/* All implementations have 3 levels of page tables */ + +#define ARCH_PGT_MAX_LEVELS (3) +#define ARCH_SPGTS (ARCH_PGT_MAX_LEVELS - 1) + #endif /* CONFIG_ARCH_ADDRENV */ /**************************************************************************** @@ -57,40 +61,46 @@ ****************************************************************************/ #ifdef CONFIG_ARCH_ADDRENV -/* The task group resources are retained in a single structure, task_group_s - * that is defined in the header file nuttx/include/nuttx/sched.h. The type - * arch_addrenv_t must be defined by platform specific logic in - * nuttx/arch//include/arch.h. +#ifndef __ASSEMBLY__ + +/* A task group must have its L1 table in memory always, and the rest can + * be dynamically committed to memory (and even swapped). * - * These tables would hold the physical address of the level 2 page tables. - * All would be initially NULL and would not be backed up with physical - * memory until mappings in the level 2 page table are required. + * In this implementation level tables except the final level N are always + * kept in static memory, while the level N tables are always dynamically + * allocated. There is one static page per level in `spgtables[]`. + * + * For the VMSAv8-64 address translation system this means that: + * - A task can not have more than 1GB of memory allocated. This should be + * plenty enough... + * - The minimum amount of memory needed for page tables per task is 12K, + * which gives access to 2MB of memory. This is plenty for many tasks. */ struct arch_addrenv_s { - /* Level 1 page table entries for each group section */ - - uintptr_t *text[ARCH_TEXT_NSECTS]; - uintptr_t *data[ARCH_DATA_NSECTS]; -#ifdef CONFIG_BUILD_KERNEL - uintptr_t *heap[ARCH_HEAP_NSECTS]; -#ifdef CONFIG_ARCH_VMA_MAPPING - uintptr_t *shm[ARCH_SHM_NSECTS]; -#endif - - /* Initial heap allocation (in bytes). This exists only provide an - * indirect path for passing the size of the initial heap to the heap - * initialization logic. These operations are separated in time and - * architecture. REVISIT: I would like a better way to do this. + /* Physical addresses of the static page tables (levels N-1) here, these + * are allocated when a task is created. */ - size_t heapsize; -#endif + uintptr_t spgtables[ARCH_SPGTS]; + + /* The text, data, heap bases and heap size here */ + + uintptr_t textvbase; + uintptr_t datavbase; + uintptr_t heapvbase; + size_t heapsize; + + /* The page directory root (ttbr0) value */ + + uintptr_t ttbr0; }; typedef struct arch_addrenv_s arch_addrenv_t; -#endif + +#endif /* __ASSEMBLY__ */ +#endif /* CONFIG_ARCH_ADDRENV */ /**************************************************************************** * Public Data diff --git a/arch/arm64/include/irq.h b/arch/arm64/include/irq.h index baa5c08ab7d..79cca7308bf 100644 --- a/arch/arm64/include/irq.h +++ b/arch/arm64/include/irq.h @@ -247,6 +247,12 @@ extern "C" EXTERN volatile uint64_t *g_current_regs[CONFIG_SMP_NCPUS]; #define CURRENT_REGS (g_current_regs[up_cpu_index()]) +/**************************************************************************** + * Public Types + ****************************************************************************/ + +#ifndef __ASSEMBLY__ + struct xcptcontext { /* The following function pointer is non-zero if there are pending signals @@ -308,11 +314,12 @@ struct xcptcontext uintptr_t *ustkptr; /* Saved user stack pointer */ uintptr_t *kstack; /* Allocate base of the (aligned) kernel stack */ - uintptr_t *kstkptr; /* Saved kernel stack pointer */ # endif #endif }; +#endif /* __ASSEMBLY__ */ + /* Name: up_irq_save, up_irq_restore, and friends. * * NOTE: This function should never be called from application code and, diff --git a/arch/arm64/include/syscall.h b/arch/arm64/include/syscall.h index 3d019b1f4ad..c4f058d04d8 100644 --- a/arch/arm64/include/syscall.h +++ b/arch/arm64/include/syscall.h @@ -52,7 +52,7 @@ */ #ifndef CONFIG_BUILD_FLAT -# define CONFIG_SYS_RESERVED 8 +# define CONFIG_SYS_RESERVED 6 #else # define CONFIG_SYS_RESERVED 4 #endif @@ -80,32 +80,7 @@ #define SYS_switch_context (2) -#ifdef CONFIG_LIB_SYSCALL -/* SYS call 3: - * - * void arm_syscall_return(void); - */ - -#define SYS_syscall_return (3) -#endif /* CONFIG_LIB_SYSCALL */ - #ifndef CONFIG_BUILD_FLAT -/* SYS call 4: - * - * void up_task_start(main_t taskentry, int argc, char *argv[]) - * noreturn_function; - */ - -#define SYS_task_start (4) - -/* SYS call 5: - * - * void up_pthread_start((pthread_startroutine_t startup, - * pthread_startroutine_t entrypt, pthread_addr_t arg) - * noreturn_function - */ - -#define SYS_pthread_start (5) /* SYS call 6: * @@ -114,14 +89,14 @@ * void *ucontext); */ -#define SYS_signal_handler (6) +#define SYS_signal_handler (4) /* SYS call 7: * * void signal_handler_return(void); */ -#define SYS_signal_handler_return (7) +#define SYS_signal_handler_return (5) #endif /* !CONFIG_BUILD_FLAT */ #define ARM_SMCC_RES_A0 (0) diff --git a/arch/arm64/src/common/CMakeLists.txt b/arch/arm64/src/common/CMakeLists.txt index 9355d26bd10..7f9e95042a3 100644 --- a/arch/arm64/src/common/CMakeLists.txt +++ b/arch/arm64/src/common/CMakeLists.txt @@ -76,17 +76,17 @@ if(CONFIG_BUILD_KERNEL) arm64_signal_dispatch.c) endif() +if(CONFIG_ARCH_KERNEL_STACK) + list(APPEND SRCS arm64_addrenv_kstack.c) +endif() + if(CONFIG_ARCH_ADDRENV) - list(APPEND SRCS arm64_addrenv.c arm64_addrenv_utils.c arm64_pgalloc.c) + list(APPEND SRCS arm64_addrenv.c arm64_pgalloc.c arm64_addrenv_perms.c) + list(APPEND SRCS arm64_addrenv_utils.c arm64_addrenv_shm.c) + list(APPEND SRCS arm64_addrenv_pgmap.c) if(CONFIG_ARCH_STACK_DYNAMIC) list(APPEND SRCS arm64_addrenv_ustack.c) endif() - if(CONFIG_ARCH_KERNEL_STACK) - list(APPEND SRCS arm64_addrenv_kstack.c) - endif() - if(CONFIG_ARCH_VMA_MAPPING) - list(APPEND SRCS arm64_addrenv_shm.c) - endif() endif() if(CONFIG_MM_PGALLOC) diff --git a/arch/arm64/src/common/Make.defs b/arch/arm64/src/common/Make.defs index 8cdc9e99fcb..51540dd31ab 100644 --- a/arch/arm64/src/common/Make.defs +++ b/arch/arm64/src/common/Make.defs @@ -88,23 +88,15 @@ ifeq ($(CONFIG_BUILD_KERNEL),y) CMN_CSRCS += arm64_task_start.c arm64_pthread_start.c arm64_signal_dispatch.c endif -ifeq ($(CONFIG_ARCH_ADDRENV),y) -CMN_CSRCS += arm64_addrenv.c arm64_addrenv_utils.c arm64_pgalloc.c -ifeq ($(CONFIG_ARCH_STACK_DYNAMIC),y) -CMN_CSRCS += arm64_addrenv_ustack.c -endif ifeq ($(CONFIG_ARCH_KERNEL_STACK),y) CMN_CSRCS += arm64_addrenv_kstack.c endif -ifeq ($(CONFIG_ARCH_VMA_MAPPING),y) -CMN_CSRCS += arm64_addrenv_shm.c -endif -endif -ifeq ($(CONFIG_MM_PGALLOC),y) -CMN_CSRCS += arm64_physpgaddr.c -ifeq ($(CONFIG_ARCH_PGPOOL_MAPPING),y) -CMN_CSRCS += arm64_virtpgaddr.c +ifeq ($(CONFIG_ARCH_ADDRENV),y) +CMN_CSRCS += arm64_addrenv.c arm64_pgalloc.c arm64_addrenv_perms.c +CMN_CSRCS += arm64_addrenv_utils.c arm64_addrenv_shm.c arm64_addrenv_pgmap.c +ifeq ($(CONFIG_ARCH_STACK_DYNAMIC),y) +CMN_CSRCS += arm64_addrenv_ustack.c endif endif diff --git a/arch/arm64/src/common/addrenv.h b/arch/arm64/src/common/addrenv.h index 700c089596b..f5236336867 100644 --- a/arch/arm64/src/common/addrenv.h +++ b/arch/arm64/src/common/addrenv.h @@ -41,62 +41,92 @@ /* Aligned size of the kernel stack */ #ifdef CONFIG_ARCH_KERNEL_STACK -# define ARCH_KERNEL_STACKSIZE STACK_ALIGN_UP(CONFIG_ARCH_KERNEL_STACKSIZE) +# define ARCH_KERNEL_STACKSIZE STACK_ALIGN_UP(CONFIG_ARCH_KERNEL_STACKSIZE) #endif -/**************************************************************************** - * Inline Functions - ****************************************************************************/ +/* Base address for address environment */ -/**************************************************************************** - * Public Data - ****************************************************************************/ - -#ifndef __ASSEMBLY__ -#ifdef __cplusplus -#define EXTERN extern "C" -extern "C" -{ +#if CONFIG_ARCH_TEXT_VBASE != 0 +# define ARCH_ADDRENV_VBASE (CONFIG_ARCH_TEXT_VBASE) #else -#define EXTERN extern +# define ARCH_ADDRENV_VBASE (CONFIG_ARCH_DATA_VBASE) #endif +/* Maximum user address environment size */ + +#define ARCH_ADDRENV_MAX_SIZE (0x40000000) + +/* User address environment end */ + +#define ARCH_ADDRENV_VEND (ARCH_ADDRENV_VBASE + ARCH_ADDRENV_MAX_SIZE - 1) + /**************************************************************************** * Public Function Prototypes ****************************************************************************/ /**************************************************************************** - * Name: arm_addrenv_create_region + * Name: arm64_get_pgtable * * Description: - * Create one memory region. + * Get the physical address of the final level page table corresponding to + * 'vaddr'. If one does not exist, it will be allocated. + * + * Input Parameters: + * addrenv - Pointer to a structure describing the address environment + * vaddr - Virtual address to query for * * Returned Value: - * On success, the number of pages allocated is returned. Otherwise, a - * negated errno value is returned. + * The physical address of the corresponding final level page table, or + * NULL if one does not exist, and there is no free memory to allocate one * ****************************************************************************/ -int arm64_addrenv_create_region(uintptr_t **list, size_t listlen, - uintptr_t vaddr, size_t regionsize, - uint32_t mmuflags); +uintptr_t arm64_get_pgtable(arch_addrenv_t *addrenv, uintptr_t vaddr); /**************************************************************************** - * Name: arm_addrenv_destroy_region + * Name: arm64_map_pages * * Description: - * Destroy one memory region. + * Map physical pages into a continuous virtual memory block. + * + * Input Parameters: + * addrenv - Pointer to a structure describing the address environment. + * pages - A pointer to the first element in a array of physical address, + * each corresponding to one page of memory. + * npages - The number of pages in the list of physical pages to be mapped. + * vaddr - The virtual address corresponding to the beginning of the + * (continuous) virtual address region. + * prot - MMU flags to use. + * + * Returned Value: + * Zero (OK) is returned on success; a negated errno value is returned + * on failure. * ****************************************************************************/ -void arm64_addrenv_destroy_region(uintptr_t **list, size_t listlen, - uintptr_t vaddr, bool keep); +int arm64_map_pages(arch_addrenv_t *addrenv, uintptr_t *pages, + unsigned int npages, uintptr_t vaddr, uint64_t prot); -#undef EXTERN -#ifdef __cplusplus -} -#endif -#endif /* __ASSEMBLY__ */ +/**************************************************************************** + * Name: arm64_unmap_pages + * + * Description: + * Unmap a previously mapped virtual memory region. + * + * Input Parameters: + * addrenv - Pointer to a structure describing the address environment. + * vaddr - The virtual address corresponding to the beginning of the + * (continuous) virtual address region. + * npages - The number of pages to be unmapped + * + * Returned Value: + * Zero (OK) is returned on success; a negated errno value is returned + * on failure. + * + ****************************************************************************/ + +int arm64_unmap_pages(arch_addrenv_t *addrenv, uintptr_t vaddr, + unsigned int npages); #endif /* CONFIG_ARCH_ADDRENV */ #endif /* __ARCH_ARM64_SRC_COMMON_ADDRENV_H */ diff --git a/arch/arm64/src/common/arm64_addrenv.c b/arch/arm64/src/common/arm64_addrenv.c new file mode 100644 index 00000000000..2864de4f23e --- /dev/null +++ b/arch/arm64/src/common/arm64_addrenv.c @@ -0,0 +1,818 @@ +/**************************************************************************** + * arch/arm64/src/common/arm64_addrenv.c + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +/**************************************************************************** + * Address Environment Interfaces + * + * Low-level interfaces used in binfmt/ to instantiate tasks with address + * environments. These interfaces all operate on type arch_addrenv_t which + * is an abstract representation of a task group's address environment and + * must be defined in arch/arch.h if CONFIG_ARCH_ADDRENV is defined. + * + * up_addrenv_create - Create an address environment + * up_addrenv_destroy - Destroy an address environment. + * up_addrenv_vtext - Returns the virtual base address of the .text + * address environment + * up_addrenv_vdata - Returns the virtual base address of the .bss/.data + * address environment + * up_addrenv_heapsize - Returns the size of the initial heap allocation. + * up_addrenv_select - Instantiate an address environment + * up_addrenv_clone - Copy an address environment from one location to + * another. + * + * Higher-level interfaces used by the tasking logic. These interfaces are + * used by the functions in sched/ and all operate on the thread which whose + * group been assigned an address environment by up_addrenv_clone(). + * + * up_addrenv_attach - Clone the address environment assigned to one TCB + * to another. This operation is done when a pthread + * is created that share's the same address + * environment. + * up_addrenv_detach - Release the threads reference to an address + * environment when a task/thread exits. + * + ****************************************************************************/ + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "addrenv.h" +#include "barriers.h" +#include "pgalloc.h" +#include "arm64_mmu.h" + +/**************************************************************************** + * Pre-processor Definitions + ****************************************************************************/ + +/* Only CONFIG_BUILD_KERNEL is supported (i.e. tested) */ + +#ifndef CONFIG_BUILD_KERNEL +# error "This module is intended to be used with CONFIG_BUILD_KERNEL" +#endif + +/* Entries per PGT */ + +#define ENTRIES_PER_PGT (MMU_PAGE_ENTRIES) + +/* Make sure the address environment virtual address boundary is valid */ + +static_assert((ARCH_ADDRENV_VBASE & MMU_L2_PAGE_SIZE) == 0, + "Addrenv start address is not aligned to section boundary"); + +/**************************************************************************** + * Public Data + ****************************************************************************/ + +extern uintptr_t g_kernel_mappings; + +/**************************************************************************** + * Private Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: map_spgtables + * + * Description: + * Map vaddr to the static page tables. + * + * Input Parameters: + * addrenv - Describes the address environment + * + ****************************************************************************/ + +static void map_spgtables(arch_addrenv_t *addrenv, uintptr_t vaddr) +{ + int i; + uintptr_t prev; + + /* Start from L1, and connect until max level - 1 */ + + prev = arm64_pgvaddr(addrenv->spgtables[0]); + + /* Check if the mapping already exists */ + + if (mmu_ln_getentry(1, prev, vaddr) != 0) + { + return; + } + + /* No mapping yet, create it */ + + for (i = 0; i < (ARCH_SPGTS - 1); i++) + { + uintptr_t next = addrenv->spgtables[i + 1]; + mmu_ln_setentry(i + 1, prev, next, vaddr, MMU_UPGT_FLAGS); + prev = arm64_pgvaddr(next); + } +} + +/**************************************************************************** + * Name: create_spgtables + * + * Description: + * Create the static page tables. Allocate memory for them and connect them + * together. + * + * Input Parameters: + * addrenv - Describes the address environment + * + * Returned value: + * Amount of pages created on success; a negated errno value on failure + * + ****************************************************************************/ + +static int create_spgtables(arch_addrenv_t *addrenv) +{ + int i; + uintptr_t paddr; + + for (i = 0; i < ARCH_SPGTS; i++) + { + paddr = mm_pgalloc(1); + if (!paddr) + { + return -ENOMEM; + } + + /* Wipe the memory and assign it */ + + arm64_pgwipe(paddr); + addrenv->spgtables[i] = paddr; + } + + /* Synchronize data and instruction pipelines */ + + ARM64_DSB(); + ARM64_ISB(); + + return i; +} + +/**************************************************************************** + * Name: copy_kernel_mappings + * + * Description: + * Copy kernel mappings to address environment. Expects that the user page + * table does not contain any mappings yet (as they will be wiped). + * + * Input Parameters: + * addrenv - Describes the address environment. The page tables must exist + * at this point. + * + * Returned value: + * OK on success, ERROR on failure + * + ****************************************************************************/ + +static int copy_kernel_mappings(arch_addrenv_t *addrenv) +{ + uintptr_t user_mappings = arm64_pgvaddr(addrenv->spgtables[0]); + + /* Copy the L1 references */ + + if (user_mappings == 0) + { + return -EINVAL; + } + + memcpy((void *)user_mappings, (void *)g_kernel_mappings, MMU_PAGE_SIZE); + + return OK; +} + +/**************************************************************************** + * Name: create_region + * + * Description: + * Map a single region of memory to MMU. Assumes that the static page + * tables exist. Allocates the final level page tables and commits the + * region memory to physical memory. + * + * Input Parameters: + * addrenv - Describes the address environment + * vaddr - Base virtual address for the mapping + * size - Size of the region in bytes + * mmuflags - MMU flags to use + * + * Returned value: + * Amount of pages created on success; a negated errno value on failure + * + ****************************************************************************/ + +static int create_region(arch_addrenv_t *addrenv, uintptr_t vaddr, + size_t size, uint64_t mmuflags) +{ + uintptr_t ptlast; + uintptr_t ptprev; + uintptr_t paddr; + uint32_t ptlevel; + int npages; + int nmapped; + int i; + int j; + + nmapped = 0; + npages = MM_NPAGES(size); + ptprev = arm64_pgvaddr(addrenv->spgtables[ARCH_SPGTS - 1]); + ptlevel = ARCH_SPGTS; + + /* Create mappings for the lower level tables */ + + map_spgtables(addrenv, vaddr); + + /* Begin allocating memory for the page tables */ + + for (i = 0; i < npages; i += ENTRIES_PER_PGT) + { + /* Get the current final level entry corresponding to this vaddr */ + + paddr = mmu_pte_to_paddr(mmu_ln_getentry(ptlevel, ptprev, vaddr)); + + if (!paddr) + { + /* Nothing yet, allocate one page for final level page table */ + + paddr = mm_pgalloc(1); + if (!paddr) + { + return -ENOMEM; + } + + /* Map the page table to the prior level */ + + mmu_ln_setentry(ptlevel, ptprev, paddr, vaddr, MMU_UPGT_FLAGS); + + /* This is then used to map the final level */ + + arm64_pgwipe(paddr); + } + + ptlast = arm64_pgvaddr(paddr); + + /* Then allocate memory for the region data */ + + for (j = 0; +#ifdef CONFIG_PAGING + j < 1; +#else + j < ENTRIES_PER_PGT && nmapped < size; +#endif + j++) + { + paddr = mm_pgalloc(1); + if (!paddr) + { + return -ENOMEM; + } + + /* Wipe the physical page memory */ + + arm64_pgwipe(paddr); + + /* Then map the virtual address to the physical address */ + + mmu_ln_setentry(ptlevel + 1, ptlast, paddr, vaddr, mmuflags); + nmapped += MM_PGSIZE; + vaddr += MM_PGSIZE; + } + } + + /* Synchronize data and instruction pipelines */ + + ARM64_DSB(); + ARM64_ISB(); + + return npages; +} + +/**************************************************************************** + * Name: vaddr_is_shm + * + * Description: + * Check if a vaddr is part of the SHM area + * + * Input Parameters: + * vaddr - Virtual address to check + * + * Returned value: + * true if it is; false if not + * + ****************************************************************************/ + +static inline bool vaddr_is_shm(uintptr_t vaddr) +{ +#if defined (CONFIG_ARCH_SHM_VBASE) && defined(ARCH_SHM_VEND) + return vaddr >= CONFIG_ARCH_SHM_VBASE && vaddr < ARCH_SHM_VEND; +#else + UNUSED(vaddr); + return false; +#endif +} + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: up_addrenv_create + * + * Description: + * This function is called when a new task is created in order to + * instantiate an address environment for the new task group. + * up_addrenv_create() is essentially the allocator of the physical + * memory for the new task. + * + * Input Parameters: + * textsize - The size (in bytes) of the .text address environment needed + * by the task. This region may be read/execute only. + * datasize - The size (in bytes) of the .data/.bss address environment + * needed by the task. This region may be read/write only. NOTE: The + * actual size of the data region that is allocated will include a + * OS private reserved region at the beginning. The size of the + * private, reserved region is give by ARCH_DATA_RESERVE_SIZE. + * heapsize - The initial size (in bytes) of the heap address environment + * needed by the task. This region may be read/write only. + * addrenv - The location to return the representation of the task address + * environment. + * + * Returned Value: + * Zero (OK) on success; a negated errno value on failure. + * + ****************************************************************************/ + +int up_addrenv_create(size_t textsize, size_t datasize, size_t heapsize, + arch_addrenv_t *addrenv) +{ + int ret; + uintptr_t resvbase; + uintptr_t resvsize; + uintptr_t textbase; + uintptr_t database; + uintptr_t heapbase; + + DEBUGASSERT(addrenv); + DEBUGASSERT(MM_ISALIGNED(ARCH_ADDRENV_VBASE)); + + /* Make sure the address environment is wiped before doing anything */ + + memset(addrenv, 0, sizeof(arch_addrenv_t)); + + /* Create the static page tables */ + + ret = create_spgtables(addrenv); + + if (ret < 0) + { + serr("ERROR: Failed to create static page tables\n"); + goto errout; + } + + /* Map the kernel memory for the user */ + + ret = copy_kernel_mappings(addrenv); + + if (ret < 0) + { + serr("ERROR: Failed to copy kernel mappings to new environment"); + goto errout; + } + + /* Calculate the base addresses for convenience */ + +#if (CONFIG_ARCH_TEXT_VBASE != 0x0) && (CONFIG_ARCH_HEAP_VBASE != 0x0) + resvbase = CONFIG_ARCH_DATA_VBASE; + resvsize = ARCH_DATA_RESERVE_SIZE; + textbase = CONFIG_ARCH_TEXT_VBASE; + database = resvbase + MM_PGALIGNUP(resvsize); + heapbase = CONFIG_ARCH_HEAP_VBASE; +#else + resvbase = ARCH_ADDRENV_VBASE; + resvsize = ARCH_DATA_RESERVE_SIZE; + textbase = resvbase + MM_PGALIGNUP(resvsize); + database = textbase + MM_PGALIGNUP(textsize); + heapbase = database + MM_PGALIGNUP(datasize); +#endif + + /* Allocate 1 extra page for heap, temporary fix for #5811 */ + + heapsize = heapsize + MM_PGALIGNUP(4*CONFIG_DEFAULT_TASK_STACKSIZE); + + /* Map the reserved area */ + + ret = create_region(addrenv, resvbase, resvsize, MMU_UDATA_FLAGS); + + if (ret < 0) + { + berr("ERROR: Failed to create reserved region: %d\n", ret); + goto errout; + } + + /* Map each region in turn */ + + ret = create_region(addrenv, textbase, textsize, MMU_UTEXT_FLAGS); + + if (ret < 0) + { + berr("ERROR: Failed to create .text region: %d\n", ret); + goto errout; + } + + ret = create_region(addrenv, database, datasize, MMU_UDATA_FLAGS); + + if (ret < 0) + { + berr("ERROR: Failed to create .bss/.data region: %d\n", ret); + goto errout; + } + + ret = create_region(addrenv, heapbase, heapsize, MMU_UDATA_FLAGS); + + if (ret < 0) + { + berr("ERROR: Failed to create heap region: %d\n", ret); + goto errout; + } + + /* Save the heap base and initial size allocated. These will be needed when + * the heap data structures are initialized. + */ + + addrenv->heapvbase = heapbase; + addrenv->heapsize = (size_t)ret << MM_PGSHIFT; + + /* Save the text base */ + + addrenv->textvbase = textbase; + + /* Save the data base */ + + addrenv->datavbase = database; + + /* Provide the ttbr0 value for context switch */ + + addrenv->ttbr0 = mmu_ttbr_reg(addrenv->spgtables[0], 0); + + /* Synchronize data and instruction pipelines */ + + ARM64_DSB(); + ARM64_ISB(); + + return OK; + +errout: + up_addrenv_destroy(addrenv); + return ret; +} + +/**************************************************************************** + * Name: up_addrenv_destroy + * + * Description: + * This function is called when a final thread leaves the task group and + * the task group is destroyed. This function then destroys the defunct + * address environment, releasing the underlying physical memory. + * + * Input Parameters: + * addrenv - The address environment to be destroyed. + * + * Returned Value: + * Zero (OK) on success; a negated errno value on failure. + * + ****************************************************************************/ + +int up_addrenv_destroy(arch_addrenv_t *addrenv) +{ + /* Recursively destroy it all, need to table walk */ + + uintptr_t *ptprev; + uintptr_t *ptlast; + uintptr_t paddr; + uintptr_t vaddr; + size_t pgsize; + int i; + int j; + + DEBUGASSERT(addrenv); + + /* Things start from the beginning of the user virtual memory */ + + vaddr = ARCH_ADDRENV_VBASE; + pgsize = mmu_get_region_size(ARCH_SPGTS); + + /* First destroy the allocated memory and the final level page table */ + + ptprev = (uintptr_t *)arm64_pgvaddr(addrenv->spgtables[ARCH_SPGTS - 1]); + if (ptprev) + { + /* walk user space only */ + + i = (ARCH_SPGTS < 2) ? vaddr / pgsize : 0; + for (; i < ENTRIES_PER_PGT; i++, vaddr += pgsize) + { + ptlast = (uintptr_t *)arm64_pgvaddr(mmu_pte_to_paddr(ptprev[i])); + if (ptlast) + { + if (!vaddr_is_shm(vaddr)) + { + /* Free the allocated pages, but not from SHM area */ + + for (j = 0; j < ENTRIES_PER_PGT; j++) + { + paddr = mmu_pte_to_paddr(ptlast[j]); + if (paddr) + { + mm_pgfree(paddr, 1); + } + } + } + + /* Regardless, free the page table itself */ + + mm_pgfree((uintptr_t)ptlast, 1); + } + } + } + + /* Then destroy the static tables */ + + for (i = 0; i < ARCH_SPGTS; i++) + { + paddr = addrenv->spgtables[i]; + if (paddr) + { + mm_pgfree(paddr, 1); + } + } + + /* Synchronize data and instruction pipelines */ + + ARM64_DSB(); + ARM64_ISB(); + + memset(addrenv, 0, sizeof(arch_addrenv_t)); + return OK; +} + +/**************************************************************************** + * Name: up_addrenv_vtext + * + * Description: + * Return the virtual address associated with the newly create .text + * address environment. This function is used by the binary loaders in + * order get an address that can be used to initialize the new task. + * + * Input Parameters: + * addrenv - The representation of the task address environment previously + * returned by up_addrenv_create. + * vtext - The location to return the virtual address. + * + * Returned Value: + * Zero (OK) on success; a negated errno value on failure. + * + ****************************************************************************/ + +int up_addrenv_vtext(arch_addrenv_t *addrenv, void **vtext) +{ + DEBUGASSERT(addrenv && vtext); + *vtext = (void *)addrenv->textvbase; + return OK; +} + +/**************************************************************************** + * Name: up_addrenv_vdata + * + * Description: + * Return the virtual address associated with the newly create .text + * address environment. This function is used by the binary loaders in + * order get an address that can be used to initialize the new task. + * + * Input Parameters: + * addrenv - The representation of the task address environment previously + * returned by up_addrenv_create. + * textsize - For some implementations, the text and data will be saved + * in the same memory region (read/write/execute) and, in this case, + * the virtual address of the data just lies at this offset into the + * common region. + * vdata - The location to return the virtual address. + * + * Returned Value: + * Zero (OK) on success; a negated errno value on failure. + * + ****************************************************************************/ + +int up_addrenv_vdata(arch_addrenv_t *addrenv, uintptr_t textsize, + void **vdata) +{ + DEBUGASSERT(addrenv && vdata); + *vdata = (void *)addrenv->datavbase; + return OK; +} + +/**************************************************************************** + * Name: up_addrenv_vheap + * + * Description: + * Return the heap virtual address associated with the newly created + * address environment. This function is used by the binary loaders in + * order get an address that can be used to initialize the new task. + * + * Input Parameters: + * addrenv - The representation of the task address environment previously + * returned by up_addrenv_create. + * vheap - The location to return the virtual address. + * + * Returned Value: + * Zero (OK) on success; a negated errno value on failure. + * + ****************************************************************************/ + +#ifdef CONFIG_BUILD_KERNEL +int up_addrenv_vheap(const arch_addrenv_t *addrenv, void **vheap) +{ + DEBUGASSERT(addrenv && vheap); + *vheap = (void *)addrenv->heapvbase; + return OK; +} +#endif + +/**************************************************************************** + * Name: up_addrenv_heapsize + * + * Description: + * Return the initial heap allocation size. That is the amount of memory + * allocated by up_addrenv_create() when the heap memory region was first + * created. This may or may not differ from the heapsize parameter that + * was passed to up_addrenv_create() + * + * Input Parameters: + * addrenv - The representation of the task address environment previously + * returned by up_addrenv_create. + * + * Returned Value: + * The initial heap size allocated is returned on success; a negated + * errno value on failure. + * + ****************************************************************************/ + +#ifdef CONFIG_BUILD_KERNEL +ssize_t up_addrenv_heapsize(const arch_addrenv_t *addrenv) +{ + DEBUGASSERT(addrenv); + return (ssize_t)addrenv->heapsize; +} +#endif + +/**************************************************************************** + * Name: up_addrenv_select + * + * Description: + * After an address environment has been established for a task (via + * up_addrenv_create()), this function may be called to instantiate + * that address environment in the virtual address space. This might be + * necessary, for example, to load the code for the task from a file or + * to access address environment private data. + * + * Input Parameters: + * addrenv - The representation of the task address environment previously + * returned by up_addrenv_create. + * + * Returned Value: + * Zero (OK) on success; a negated errno value on failure. + * + ****************************************************************************/ + +int up_addrenv_select(const arch_addrenv_t *addrenv) +{ + DEBUGASSERT(addrenv && addrenv->ttbr0); + mmu_write_ttbr0(addrenv->ttbr0); + return OK; +} + +/**************************************************************************** + * Name: up_addrenv_coherent + * + * Description: + * Flush D-Cache and invalidate I-Cache in preparation for a change in + * address environments. This should immediately precede a call to + * up_addrenv_select(); + * + * Input Parameters: + * addrenv - Describes the address environment to be made coherent. + * + * Returned Value: + * Zero (OK) on success; a negated errno value on failure. + * + ****************************************************************************/ + +int up_addrenv_coherent(const arch_addrenv_t *addrenv) +{ + /* Nothing needs to be done */ + + return OK; +} + +/**************************************************************************** + * Name: up_addrenv_clone + * + * Description: + * Duplicate an address environment. This does not copy the underlying + * memory, only the representation that can be used to instantiate that + * memory as an address environment. + * + * Input Parameters: + * src - The address environment to be copied. + * dest - The location to receive the copied address environment. + * + * Returned Value: + * Zero (OK) on success; a negated errno value on failure. + * + ****************************************************************************/ + +int up_addrenv_clone(const arch_addrenv_t *src, + arch_addrenv_t *dest) +{ + DEBUGASSERT(src && dest); + memcpy(dest, src, sizeof(arch_addrenv_t)); + return OK; +} + +/**************************************************************************** + * Name: up_addrenv_attach + * + * Description: + * This function is called from the core scheduler logic when a thread + * is created that needs to share the address environment of its task + * group. + * + * Input Parameters: + * ptcb - The tcb of the parent task. + * tcb - The tcb of the thread needing the address environment. + * + * Returned Value: + * Zero (OK) on success; a negated errno value on failure. + * + ****************************************************************************/ + +int up_addrenv_attach(struct tcb_s *ptcb, struct tcb_s *tcb) +{ + /* There is nothing that needs to be done */ + + return OK; +} + +/**************************************************************************** + * Name: up_addrenv_detach + * + * Description: + * This function is called when a task or thread exits in order to release + * its reference to an address environment. The address environment, + * however, should persist until up_addrenv_destroy() is called when the + * task group is itself destroyed. Any resources unique to this thread + * may be destroyed now. + * + * Input Parameters: + * tcb - The TCB of the task or thread whose the address environment will + * be released. + * + * Returned Value: + * Zero (OK) on success; a negated errno value on failure. + * + ****************************************************************************/ + +int up_addrenv_detach(struct tcb_s *tcb) +{ + /* There is nothing that needs to be done */ + + return OK; +} diff --git a/arch/arm64/src/common/arm64_addrenv_kstack.c b/arch/arm64/src/common/arm64_addrenv_kstack.c new file mode 100644 index 00000000000..80e55cff042 --- /dev/null +++ b/arch/arm64/src/common/arm64_addrenv_kstack.c @@ -0,0 +1,110 @@ +/**************************************************************************** + * arch/arm64/src/common/arm64_addrenv_kstack.c + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +#include +#include +#include + +#include +#include +#include +#include + +#include "addrenv.h" +#include "arm64_internal.h" + +#if defined(CONFIG_ARCH_ADDRENV) && defined(CONFIG_ARCH_KERNEL_STACK) + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: up_addrenv_kstackalloc + * + * Description: + * This function is called when a new thread is created to allocate + * the new thread's kernel stack. This function may be called for certain + * terminating threads which have no kernel stack. It must be tolerant of + * that case. + * + * Input Parameters: + * tcb - The TCB of the thread that requires the kernel stack. + * + * Returned Value: + * Zero (OK) on success; a negated errno value on failure. + * + ****************************************************************************/ + +int up_addrenv_kstackalloc(struct tcb_s *tcb) +{ + DEBUGASSERT(tcb && tcb->xcp.kstack == NULL); + + /* Allocate the kernel stack */ + + tcb->xcp.kstack = kmm_memalign(STACK_ALIGNMENT, ARCH_KERNEL_STACKSIZE); + if (!tcb->xcp.kstack) + { + berr("ERROR: Failed to allocate the kernel stack\n"); + return -ENOMEM; + } + + return OK; +} + +/**************************************************************************** + * Name: up_addrenv_kstackfree + * + * Description: + * This function is called when any thread exits. This function frees + * the kernel stack. + * + * Input Parameters: + * tcb - The TCB of the thread that no longer requires the kernel stack. + * + * Returned Value: + * Zero (OK) on success; a negated errno value on failure. + * + ****************************************************************************/ + +int up_addrenv_kstackfree(struct tcb_s *tcb) +{ + DEBUGASSERT(tcb); + + /* Does the exiting thread have a kernel stack? */ + + if (tcb->xcp.kstack) + { + /* Yes.. Free the kernel stack */ + + kmm_free(tcb->xcp.kstack); + tcb->xcp.kstack = NULL; + } + + return OK; +} + +#endif /* CONFIG_ARCH_ADDRENV && CONFIG_ARCH_KERNEL_STACK */ diff --git a/arch/arm64/src/common/arm64_addrenv_perms.c b/arch/arm64/src/common/arm64_addrenv_perms.c new file mode 100644 index 00000000000..f8081d56b7a --- /dev/null +++ b/arch/arm64/src/common/arm64_addrenv_perms.c @@ -0,0 +1,148 @@ +/**************************************************************************** + * arch/arm64/src/common/arm64_addrenv_perms.c + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +#include +#include + +#include +#include +#include + +#include + +#include "barriers.h" +#include "pgalloc.h" +#include "arm64_arch.h" +#include "arm64_mmu.h" + +/**************************************************************************** + * Pre-processor Definitions + ****************************************************************************/ + +#define CLR_MASK (PTE_BLOCK_DESC_AP_MASK | PTE_BLOCK_DESC_UXN) + +/**************************************************************************** + * Private Functions + ****************************************************************************/ + +static int modify_region(uintptr_t vstart, uintptr_t vend, uintptr_t setmask) +{ + uintptr_t l1vaddr; + uintptr_t lnvaddr; + uintptr_t entry; + uintptr_t paddr; + uintptr_t vaddr; + uint32_t ptlevel; + + /* Must perform a reverse table walk */ + + l1vaddr = arm64_pgvaddr(mmu_get_ttbr0_pgbase()); + + if (!l1vaddr) + { + /* Oops, there is no address environment to modify at all ? */ + + return -EINVAL; + } + + for (vaddr = vstart; vaddr < vend; vaddr += MM_PGSIZE) + { + for (ptlevel = 1, lnvaddr = l1vaddr; + ptlevel < MMU_PGT_LEVELS; + ptlevel++) + { + paddr = mmu_pte_to_paddr(mmu_ln_getentry(ptlevel, lnvaddr, vaddr)); + lnvaddr = arm64_pgvaddr(paddr); + if (!lnvaddr) + { + return -EINVAL; + } + } + + /* Get entry and modify the flags */ + + entry = mmu_ln_getentry(ptlevel, lnvaddr, vaddr); + entry &= ~CLR_MASK; + entry |= setmask | PTE_BLOCK_DESC_AP_USER; + + /* Restore the entry */ + + mmu_ln_restore(ptlevel, lnvaddr, vaddr, entry); + } + + return OK; +} + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: up_addrenv_mprot + * + * Description: + * Modify access rights to an address range. + * + * Input Parameters: + * addrenv - The address environment to be modified. + * addr - Base address of the region. + * len - Size of the region. + * prot - Access right flags. + * + * Returned Value: + * Zero (OK) on success; a negated errno value on failure. + * + ****************************************************************************/ + +int up_addrenv_mprot(arch_addrenv_t *addrenv, uintptr_t addr, size_t len, + int prot) +{ + uintptr_t setmask; + + /* addrenv not needed by this implementation */ + + UNUSED(addrenv); + + /* The access is either RO or RW, read access cannot be revoked */ + + if ((prot & PROT_WRITE) == 0) + { + setmask = PTE_BLOCK_DESC_AP_RO; + } + else + { + setmask = PTE_BLOCK_DESC_AP_RW; + } + + /* Execute access is explicitly revoked, implicitly granted */ + + if ((prot & PROT_EXEC) == 0) + { + setmask |= PTE_BLOCK_DESC_UXN; + } + + return modify_region(addr, addr + MM_PGALIGNUP(len), setmask); +} diff --git a/arch/arm64/src/common/arm64_addrenv_pgmap.c b/arch/arm64/src/common/arm64_addrenv_pgmap.c new file mode 100644 index 00000000000..5fa22e62ac2 --- /dev/null +++ b/arch/arm64/src/common/arm64_addrenv_pgmap.c @@ -0,0 +1,312 @@ +/**************************************************************************** + * arch/arm64/src/common/arm64_addrenv_pgmap.c + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +#include +#include +#include +#include +#include + +#include + +#include "barriers.h" +#include "pgalloc.h" +#include "arm64_mmu.h" + +#ifdef CONFIG_BUILD_KERNEL + +/**************************************************************************** + * Private Data + ****************************************************************************/ + +#ifdef CONFIG_MM_KMAP +static struct arch_addrenv_s g_kernel_addrenv; +#endif + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: up_addrenv_find_page + * + * Description: + * Find physical page mapped to user virtual address from the address + * environment page directory. + * + * Input Parameters: + * addrenv - The user address environment. + * vaddr - The user virtual address + * + * Returned Value: + * Page physical address on success; NULL on failure. + * + ****************************************************************************/ + +uintptr_t up_addrenv_find_page(arch_addrenv_t *addrenv, uintptr_t vaddr) +{ + uintptr_t pgdir; + uintptr_t lnvaddr; + uintptr_t paddr; + uint32_t ptlevel; + + /* If vaddr is not user space, get out */ + + if (!arm64_uservaddr(vaddr)) + { + return 0; + } + + /* Get the kernel addressable virtual address of the page directory root */ + + pgdir = arm64_pgvaddr(mmu_ttbr_to_paddr(addrenv->ttbr0)); + if (!pgdir) + { + return 0; + } + + /* Make table walk to find the page */ + + for (ptlevel = 1, lnvaddr = pgdir; ptlevel < MMU_PGT_LEVELS; ptlevel++) + { + paddr = mmu_pte_to_paddr(mmu_ln_getentry(ptlevel, lnvaddr, vaddr)); + lnvaddr = arm64_pgvaddr(paddr); + if (!lnvaddr) + { + return 0; + } + } + + /* Got it, now convert it to physical page */ + + paddr = mmu_ln_getentry(ptlevel, lnvaddr, vaddr); + paddr = mmu_pte_to_paddr(paddr); + + return paddr; +} + +/**************************************************************************** + * Name: up_addrenv_page_vaddr + * + * Description: + * Find the kernel virtual address associated with physical page. + * + * Input Parameters: + * page - The page physical address. + * + * Returned Value: + * Page kernel virtual address on success; NULL on failure. + * + ****************************************************************************/ + +uintptr_t up_addrenv_page_vaddr(uintptr_t page) +{ + return arm64_pgvaddr(page); +} + +/**************************************************************************** + * Name: up_addrenv_user_vaddr + * + * Description: + * Check if a virtual address is in user virtual address space. + * + * Input Parameters: + * vaddr - The virtual address. + * + * Returned Value: + * True if it is; false if it's not + * + ****************************************************************************/ + +bool up_addrenv_user_vaddr(uintptr_t vaddr) +{ + return arm64_uservaddr(vaddr); +} + +/**************************************************************************** + * Name: up_addrenv_page_wipe + * + * Description: + * Wipe a page of physical memory, first mapping it into kernel virtual + * memory. + * + * Input Parameters: + * page - The page physical address. + * + * Returned Value: + * None. + * + ****************************************************************************/ + +void up_addrenv_page_wipe(uintptr_t page) +{ + arm64_pgwipe(page); +} + +#ifdef CONFIG_MM_KMAP + +/**************************************************************************** + * Name: up_addrenv_kmap_init + * + * Description: + * Initialize the architecture specific part of the kernel mapping + * interface. + * + * Input Parameters: + * None. + * + * Returned Value: + * Zero (OK) is returned on success; a negated errno value is returned + * on failure. + * + ****************************************************************************/ + +int up_addrenv_kmap_init(void) +{ + struct arch_addrenv_s *addrenv; + uintptr_t next; + uintptr_t vaddr; + int i; + + /* Populate the static page tables one by one */ + + addrenv = &g_kernel_addrenv; + next = g_kernel_pgt_pbase; + vaddr = CONFIG_ARCH_KMAP_VBASE; + + for (i = 0; i < ARCH_SPGTS; i++) + { + /* Connect the static page tables */ + + uintptr_t lnvaddr = arm64_pgvaddr(next); + addrenv->spgtables[i] = next; + next = mmu_pte_to_paddr(mmu_ln_getentry(i + 1, lnvaddr, vaddr)); + } + + /* Set the page directory root */ + + addrenv->ttbr0 = mmu_ttbr_reg(g_kernel_pgt_pbase, 0); + + /* When all is set and done, flush the data caches */ + + ARM64_DSB(); + + return OK; +} + +/**************************************************************************** + * Name: up_addrenv_kmap_pages + * + * Description: + * Map physical pages into a continuous virtual memory block. + * + * Input Parameters: + * pages - A pointer to the first element in a array of physical address, + * each corresponding to one page of memory. + * npages - The number of pages in the list of physical pages to be mapped. + * vaddr - The virtual address corresponding to the beginning of the + * (continuous) virtual address region. + * prot - Access right flags. + * + * Returned Value: + * Zero (OK) is returned on success; a negated errno value is returned + * on failure. + * + ****************************************************************************/ + +int up_addrenv_kmap_pages(void **pages, unsigned int npages, uintptr_t vaddr, + int prot) +{ + struct arch_addrenv_s *addrenv = &g_kernel_addrenv; + int mask = 0; + + /* Sanity checks */ + + DEBUGASSERT(pages != NULL && npages > 0); + DEBUGASSERT(vaddr >= CONFIG_ARCH_KMAP_VBASE && vaddr < ARCH_KMAP_VEND); + DEBUGASSERT(MM_ISALIGNED(vaddr)); + + /* The access is either RO or RW, read access cannot be revoked */ + + if ((prot & PROT_WRITE) == 0) + { + mask = PTE_BLOCK_DESC_AP_RO; + } + else + { + mask = PTE_BLOCK_DESC_AP_RW; + } + + /* Execute access is explicitly revoked, implicitly granted */ + + if ((prot & PROT_EXEC) == 0) + { + mask |= PTE_BLOCK_DESC_PXN; + } + + /* This is a kernel (global) mapping */ + + mask &= ~PTE_BLOCK_DESC_NG; + + /* Let arm64_map_pages do the work */ + + return arm64_map_pages(addrenv, (uintptr_t *)pages, npages, vaddr, mask); +} + +/**************************************************************************** + * Name: up_addrenv_kunmap_pages + * + * Description: + * Unmap a previously mapped virtual memory region. + * + * Input Parameters: + * vaddr - The virtual address corresponding to the beginning of the + * (continuous) virtual address region. + * npages - The number of pages to be unmapped + * + * Returned Value: + * Zero (OK) is returned on success; a negated errno value is returned + * on failure. + * + ****************************************************************************/ + +int up_addrenv_kunmap_pages(uintptr_t vaddr, unsigned int npages) +{ + struct arch_addrenv_s *addrenv = &g_kernel_addrenv; + + /* Sanity checks */ + + DEBUGASSERT(npages > 0); + DEBUGASSERT(vaddr >= CONFIG_ARCH_KMAP_VBASE && vaddr < ARCH_KMAP_VEND); + DEBUGASSERT(MM_ISALIGNED(vaddr)); + + /* Let arm64_unmap_pages do the work */ + + return arm64_unmap_pages(addrenv, vaddr, npages); +} + +#endif /* CONFIG_MM_KMAP */ +#endif /* CONFIG_BUILD_KERNEL */ diff --git a/arch/arm64/src/common/arm64_addrenv_shm.c b/arch/arm64/src/common/arm64_addrenv_shm.c new file mode 100644 index 00000000000..23405ce3b65 --- /dev/null +++ b/arch/arm64/src/common/arm64_addrenv_shm.c @@ -0,0 +1,117 @@ +/**************************************************************************** + * arch/arm64/src/common/arm64_addrenv_shm.c + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +#include +#include +#include + +#include +#include +#include +#include + +#include "addrenv.h" +#include "barriers.h" +#include "pgalloc.h" +#include "arm64_mmu.h" + +#if defined(CONFIG_BUILD_KERNEL) && defined(CONFIG_ARCH_VMA_MAPPING) + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: up_shmat + * + * Description: + * Attach, i.e, map, on shared memory region to a user virtual address + * + * Input Parameters: + * pages - A pointer to the first element in a array of physical address, + * each corresponding to one page of memory. + * npages - The number of pages in the list of physical pages to be mapped. + * vaddr - The virtual address corresponding to the beginning of the + * (contiguous) virtual address region. + * + * Returned Value: + * Zero (OK) is returned on success; a negated errno value is returned + * on failure. + * + ****************************************************************************/ + +int up_shmat(uintptr_t *pages, unsigned int npages, uintptr_t vaddr) +{ + struct tcb_s *tcb = nxsched_self(); + struct arch_addrenv_s *addrenv = &tcb->addrenv_own->addrenv; + + /* Sanity checks */ + + DEBUGASSERT(tcb && tcb->addrenv_own); + DEBUGASSERT(pages != NULL && npages > 0); + DEBUGASSERT(vaddr >= CONFIG_ARCH_SHM_VBASE && vaddr < ARCH_SHM_VEND); + DEBUGASSERT(MM_ISALIGNED(vaddr)); + + /* Let arm64_map_pages do the work */ + + return arm64_map_pages(addrenv, pages, npages, vaddr, MMU_UDATA_FLAGS); +} + +/**************************************************************************** + * Name: up_shmdt + * + * Description: + * Detach, i.e, unmap, on shared memory region from a user virtual address + * + * Input Parameters: + * vaddr - The virtual address corresponding to the beginning of the + * (contiguous) virtual address region. + * npages - The number of pages to be unmapped + * + * Returned Value: + * Zero (OK) is returned on success; a negated errno value is returned + * on failure. + * + ****************************************************************************/ + +int up_shmdt(uintptr_t vaddr, unsigned int npages) +{ + struct tcb_s *tcb = nxsched_self(); + struct arch_addrenv_s *addrenv = &tcb->addrenv_own->addrenv; + + /* Sanity checks */ + + DEBUGASSERT(tcb && tcb->addrenv_own); + DEBUGASSERT(npages > 0); + DEBUGASSERT(vaddr >= CONFIG_ARCH_SHM_VBASE && vaddr < ARCH_SHM_VEND); + DEBUGASSERT(MM_ISALIGNED(vaddr)); + + /* Let arm64_unmap_pages do the work */ + + return arm64_unmap_pages(addrenv, vaddr, npages); +} + +#endif /* CONFIG_BUILD_KERNEL */ diff --git a/arch/arm64/src/common/arm64_addrenv_utils.c b/arch/arm64/src/common/arm64_addrenv_utils.c new file mode 100644 index 00000000000..9d9e00ef792 --- /dev/null +++ b/arch/arm64/src/common/arm64_addrenv_utils.c @@ -0,0 +1,219 @@ +/**************************************************************************** + * arch/arm64/src/common/arm64_addrenv_utils.c + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +#include +#include +#include +#include +#include + +#include "barriers.h" +#include "pgalloc.h" +#include "arm64_mmu.h" + +#ifdef CONFIG_BUILD_KERNEL + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: arm64_get_pgtable + * + * Description: + * Get the physical address of the final level page table corresponding to + * 'vaddr'. If one does not exist, it will be allocated. + * + * Input Parameters: + * addrenv - Pointer to a structure describing the address environment + * vaddr - Virtual address to query for + * + * Returned Value: + * The physical address of the corresponding final level page table, or + * NULL if one does not exist, and there is no free memory to allocate one + * + ****************************************************************************/ + +uintptr_t arm64_get_pgtable(arch_addrenv_t *addrenv, uintptr_t vaddr) +{ + uintptr_t paddr; + uintptr_t ptprev; + uint32_t ptlevel; + uint32_t flags; + + /* Get the current level MAX_LEVELS-1 entry corresponding to this vaddr */ + + ptlevel = ARCH_SPGTS; + ptprev = arm64_pgvaddr(addrenv->spgtables[ARCH_SPGTS - 1]); + if (!ptprev) + { + /* Something is very wrong */ + + return 0; + } + + /* Find the physical address of the final level page table */ + + paddr = mmu_pte_to_paddr(mmu_ln_getentry(ptlevel, ptprev, vaddr)); + if (!paddr) + { + /* No page table has been allocated... allocate one now */ + + paddr = mm_pgalloc(1); + if (paddr) + { + /* Determine page table flags */ + + if (arm64_uservaddr(vaddr)) + { + flags = MMU_UPGT_FLAGS; + } + else + { + flags = MMU_KPGT_FLAGS; + } + + /* Wipe the page and assign it */ + + arm64_pgwipe(paddr); + mmu_ln_setentry(ptlevel, ptprev, paddr, vaddr, flags); + } + } + + return paddr; +} + +/**************************************************************************** + * Name: arm64_map_pages + * + * Description: + * Map physical pages into a continuous virtual memory block. + * + * Input Parameters: + * addrenv - Pointer to a structure describing the address environment. + * pages - A pointer to the first element in a array of physical address, + * each corresponding to one page of memory. + * npages - The number of pages in the list of physical pages to be mapped. + * vaddr - The virtual address corresponding to the beginning of the + * (continuous) virtual address region. + * prot - MMU flags to use. + * + * Returned Value: + * Zero (OK) is returned on success; a negated errno value is returned + * on failure. + * + ****************************************************************************/ + +int arm64_map_pages(arch_addrenv_t *addrenv, uintptr_t *pages, + unsigned int npages, uintptr_t vaddr, uint64_t prot) +{ + uintptr_t ptlast; + uintptr_t ptlevel; + uintptr_t paddr; + + ptlevel = MMU_PGT_LEVELS; + + /* Add the references to pages[] into the caller's address environment */ + + for (; npages > 0; npages--) + { + /* Get the address of the last level page table */ + + ptlast = arm64_pgvaddr(arm64_get_pgtable(addrenv, vaddr)); + if (!ptlast) + { + return -ENOMEM; + } + + /* Then add the reference */ + + paddr = *pages++; + mmu_ln_setentry(ptlevel, ptlast, paddr, vaddr, prot); + vaddr += MM_PGSIZE; + } + + return OK; +} + +/**************************************************************************** + * Name: arm64_unmap_pages + * + * Description: + * Unmap a previously mapped virtual memory region. + * + * Input Parameters: + * addrenv - Pointer to a structure describing the address environment. + * vaddr - The virtual address corresponding to the beginning of the + * (continuous) virtual address region. + * npages - The number of pages to be unmapped + * + * Returned Value: + * Zero (OK) is returned on success; a negated errno value is returned + * on failure. + * + ****************************************************************************/ + +int arm64_unmap_pages(arch_addrenv_t *addrenv, uintptr_t vaddr, + unsigned int npages) +{ + uintptr_t ptlast; + uintptr_t ptprev; + uintptr_t ptlevel; + uintptr_t paddr; + + ptprev = arm64_pgvaddr(addrenv->spgtables[ARCH_SPGTS - 1]); + if (!ptprev) + { + /* Something is very wrong */ + + return -EFAULT; + } + + ptlevel = ARCH_SPGTS; + + /* Remove the references from the caller's address environment */ + + for (; npages > 0; npages--) + { + /* Get the current final level entry corresponding to this vaddr */ + + paddr = mmu_pte_to_paddr(mmu_ln_getentry(ptlevel, ptprev, vaddr)); + ptlast = arm64_pgvaddr(paddr); + if (!ptlast) + { + return -EFAULT; + } + + /* Then wipe the reference */ + + mmu_ln_clear(ptlevel + 1, ptlast, vaddr); + vaddr += MM_PGSIZE; + } + + return OK; +} + +#endif /* CONFIG_BUILD_KERNEL */ diff --git a/arch/arm64/src/common/arm64_fork.c b/arch/arm64/src/common/arm64_fork.c index 8398058b614..7fdd290063b 100644 --- a/arch/arm64/src/common/arm64_fork.c +++ b/arch/arm64/src/common/arm64_fork.c @@ -238,8 +238,12 @@ pid_t arm64_fork(const struct fork_s *context) pforkctx->elr = (uint64_t)context->lr; pforkctx->exe_depth = 0; - pforkctx->sp_elx = (uint64_t)pforkctx; + pforkctx->sp_elx = (uint64_t)stack_ptr; +#ifdef CONFIG_ARCH_KERNEL_STACK + pforkctx->sp_el0 = (uint64_t)child->cmn.xcp.ustkptr; +#else pforkctx->sp_el0 = (uint64_t)pforkctx; +#endif pforkctx->tpidr_el0 = (uint64_t)(&child->cmn); pforkctx->tpidr_el1 = (uint64_t)(&child->cmn); diff --git a/arch/arm64/src/common/arm64_initialstate.c b/arch/arm64/src/common/arm64_initialstate.c index 1e4d776004e..b61f01b3b26 100644 --- a/arch/arm64/src/common/arm64_initialstate.c +++ b/arch/arm64/src/common/arm64_initialstate.c @@ -41,6 +41,7 @@ #include #include +#include "addrenv.h" #include "arm64_arch.h" #include "arm64_internal.h" #include "chip.h" @@ -59,6 +60,16 @@ void arm64_new_task(struct tcb_s * tcb) uint64_t stack_ptr = (uintptr_t)tcb->stack_base_ptr + tcb->adj_stack_size; struct regs_context *pinitctx; +#ifdef CONFIG_ARCH_KERNEL_STACK + /* Use the process kernel stack to store context for user processes */ + + if (tcb->xcp.kstack) + { + tcb->xcp.ustkptr = (uintptr_t *)stack_ptr; + stack_ptr = (uintptr_t)tcb->xcp.kstack + ARCH_KERNEL_STACKSIZE; + } +#endif + #ifdef CONFIG_ARCH_FPU struct fpu_reg *pfpuctx; pfpuctx = STACK_PTR_TO_FRAME(struct fpu_reg, stack_ptr); @@ -87,7 +98,11 @@ void arm64_new_task(struct tcb_s * tcb) #endif /* CONFIG_SUPPRESS_INTERRUPTS */ pinitctx->sp_elx = (uint64_t)stack_ptr; +#ifdef CONFIG_ARCH_KERNEL_STACK + pinitctx->sp_el0 = (uint64_t)tcb->xcp.ustkptr; +#else pinitctx->sp_el0 = (uint64_t)pinitctx; +#endif pinitctx->exe_depth = 0; pinitctx->tpidr_el0 = (uint64_t)tcb; pinitctx->tpidr_el1 = (uint64_t)tcb; @@ -112,9 +127,16 @@ void arm64_new_task(struct tcb_s * tcb) void up_initial_state(struct tcb_s *tcb) { struct xcptcontext *xcp = &tcb->xcp; +#ifdef CONFIG_ARCH_KERNEL_STACK + uint64_t *kstack = xcp->kstack; +#endif memset(xcp, 0, sizeof(struct xcptcontext)); +#ifdef CONFIG_ARCH_KERNEL_STACK + xcp->kstack = kstack; +#endif + if (tcb->pid == IDLE_PROCESS_ID) { /* Initialize the idle thread stack */ diff --git a/arch/arm64/src/common/arm64_mmu.c b/arch/arm64/src/common/arm64_mmu.c index e16ee23b6d3..ec1bb1b98b5 100644 --- a/arch/arm64/src/common/arm64_mmu.c +++ b/arch/arm64/src/common/arm64_mmu.c @@ -1,4 +1,4 @@ -/*************************************************************************** +/**************************************************************************** * arch/arm64/src/common/arm64_mmu.c * * Licensed to the Apache Software Foundation (ASF) under one or more @@ -16,11 +16,11 @@ * License for the specific language governing permissions and limitations * under the License. * - ***************************************************************************/ + ****************************************************************************/ -/*************************************************************************** +/**************************************************************************** * Included Files - ***************************************************************************/ + ****************************************************************************/ #include #include @@ -36,9 +36,9 @@ #include "arm64_fatal.h" #include "arm64_mmu.h" -/*************************************************************************** +/**************************************************************************** * Pre-processor Definitions - ***************************************************************************/ + ****************************************************************************/ /* MMU debug option * #define CONFIG_MMU_ASSERT 1 @@ -75,14 +75,14 @@ /* We support only 4kB translation granule */ -#define PAGE_SIZE_SHIFT 12U +#define PAGE_SIZE_SHIFT MMU_PAGE_SHIFT #define PAGE_SIZE (1U << PAGE_SIZE_SHIFT) #define XLAT_TABLE_SIZE_SHIFT PAGE_SIZE_SHIFT /* Size of one * complete table */ #define XLAT_TABLE_SIZE (1U << XLAT_TABLE_SIZE_SHIFT) #define XLAT_TABLE_ENTRY_SIZE_SHIFT 3U /* Each table entry is 8 bytes */ -#define XLAT_TABLE_LEVEL_MAX 3U +#define XLAT_TABLE_LEVEL_MAX MMU_PGT_LEVELS #define XLAT_TABLE_ENTRIES_SHIFT \ (XLAT_TABLE_SIZE_SHIFT - XLAT_TABLE_ENTRY_SIZE_SHIFT) @@ -145,9 +145,9 @@ #define TCR_PS_BITS TCR_PS_BITS_4GB #endif -/*************************************************************************** +/**************************************************************************** * Private Data - ***************************************************************************/ + ****************************************************************************/ static uint64_t base_xlat_table[NUM_BASE_LEVEL_ENTRIES] aligned_data( NUM_BASE_LEVEL_ENTRIES * sizeof(uint64_t)); @@ -182,6 +182,13 @@ static const struct arm_mmu_region g_mmu_nxrt_regions[] = (uint64_t)_sdata, (uint64_t)_szdata, MT_NORMAL | MT_RW | MT_SECURE), + +#ifdef CONFIG_BUILD_KERNEL + MMU_REGION_FLAT_ENTRY("nx_pgpool", + (uint64_t)CONFIG_ARCH_PGPOOL_PBASE, + (uint64_t)CONFIG_ARCH_PGPOOL_SIZE, + MT_NORMAL | MT_RW | MT_SECURE), +#endif }; static const struct arm_mmu_config g_mmu_nxrt_config = @@ -190,9 +197,23 @@ static const struct arm_mmu_config g_mmu_nxrt_config = .mmu_regions = g_mmu_nxrt_regions, }; -/*************************************************************************** +static const size_t g_pgt_sizes[] = +{ + MMU_L1_PAGE_SIZE, + MMU_L2_PAGE_SIZE, + MMU_L3_PAGE_SIZE +}; + +/**************************************************************************** + * Public Data + ****************************************************************************/ + +uintptr_t g_kernel_mappings = (uintptr_t)&base_xlat_table; +uintptr_t g_kernel_pgt_pbase = (uintptr_t)&base_xlat_table; + +/**************************************************************************** * Private Functions - ***************************************************************************/ + ****************************************************************************/ /* Translation table control register settings */ @@ -591,9 +612,9 @@ static void enable_mmu_el1(unsigned int flags) } #endif -/*************************************************************************** +/**************************************************************************** * Public Functions - ***************************************************************************/ + ****************************************************************************/ int arm64_mmu_set_memregion(const struct arm_mmu_region *region) { @@ -673,3 +694,80 @@ int arm64_mmu_init(bool is_primary_core) return 0; } + +void mmu_ln_setentry(uint32_t ptlevel, uintptr_t lnvaddr, uintptr_t paddr, + uintptr_t vaddr, uint64_t mmuflags) +{ + uintptr_t *lntable = (uintptr_t *)lnvaddr; + uint32_t index; + + DEBUGASSERT(ptlevel > 0 && ptlevel <= XLAT_TABLE_LEVEL_MAX); + + /* Calculate index for lntable */ + + index = XLAT_TABLE_VA_IDX(vaddr, ptlevel); + + /* Setup the page descriptor and access flag */ + + mmuflags |= PTE_PAGE_DESC | PTE_BLOCK_DESC_AF; + + /* Save it */ + + lntable[index] = (paddr | mmuflags); + + /* Update with memory by flushing the cache */ + + up_flush_dcache((uintptr_t)&lntable[index], + (uintptr_t)&lntable[index] + sizeof(uintptr_t)); + + /* Remove TLB entry for the modified page */ + + mmu_invalidate_tlb_by_vaddr(vaddr); +} + +uintptr_t mmu_ln_getentry(uint32_t ptlevel, uintptr_t lnvaddr, + uintptr_t vaddr) +{ + uintptr_t *lntable = (uintptr_t *)lnvaddr; + uint32_t index; + + DEBUGASSERT(ptlevel > 0 && ptlevel <= XLAT_TABLE_LEVEL_MAX); + + index = XLAT_TABLE_VA_IDX(vaddr, ptlevel); + + /* Invalidate D-Cache so that we read from the physical memory */ + + up_invalidate_dcache((uintptr_t)&lntable[index], + (uintptr_t)&lntable[index] + sizeof(uintptr_t)); + + return lntable[index]; +} + +void mmu_ln_restore(uint32_t ptlevel, uintptr_t lnvaddr, uintptr_t vaddr, + uintptr_t entry) +{ + uintptr_t *lntable = (uintptr_t *)lnvaddr; + uint32_t index; + + DEBUGASSERT(ptlevel > 0 && ptlevel <= XLAT_TABLE_LEVEL_MAX); + + index = XLAT_TABLE_VA_IDX(vaddr, ptlevel); + + lntable[index] = entry; + + /* Update with memory by flushing the cache */ + + up_flush_dcache((uintptr_t)&lntable[index], + (uintptr_t)&lntable[index] + sizeof(uintptr_t)); + + /* Remove TLB entry for the modified page */ + + mmu_invalidate_tlb_by_vaddr(vaddr); +} + +size_t mmu_get_region_size(uint32_t ptlevel) +{ + DEBUGASSERT(ptlevel > 0 && ptlevel <= XLAT_TABLE_LEVEL_MAX); + + return g_pgt_sizes[ptlevel - 1]; +} diff --git a/arch/arm64/src/common/arm64_mmu.h b/arch/arm64/src/common/arm64_mmu.h index 24e2f2fb06c..a5c59791a81 100644 --- a/arch/arm64/src/common/arm64_mmu.h +++ b/arch/arm64/src/common/arm64_mmu.h @@ -40,6 +40,7 @@ * * See Arm® Architecture Reference Manual, ARM DDI 0487E.a, B2.7.2 */ + #define MT_TYPE_MASK 0x7U #define MT_TYPE(attr) ((attr) & MT_TYPE_MASK) #define MT_DEVICE_NGNRNE 0U @@ -97,16 +98,25 @@ /* Block and Page descriptor attributes fields */ #define PTE_BLOCK_DESC_MEMTYPE(x) ((x) << 2) -#define PTE_BLOCK_DESC_NS (1ULL << 5) -#define PTE_BLOCK_DESC_AP_RO (1ULL << 7) -#define PTE_BLOCK_DESC_AP_RW (0ULL << 7) +#define PTE_BLOCK_DESC_NS (1ULL << 5) /* Non-secure */ +#define PTE_BLOCK_DESC_AP_USER (1ULL << 6) /* User */ +#define PTE_BLOCK_DESC_AP_RO (1ULL << 7) /* Read-only */ +#define PTE_BLOCK_DESC_AP_RW (0ULL << 7) /* Read-write */ +#define PTE_BLOCK_DESC_AP_MASK (3ULL << 6) #define PTE_BLOCK_DESC_NON_SHARE (0ULL << 8) #define PTE_BLOCK_DESC_OUTER_SHARE (2ULL << 8) #define PTE_BLOCK_DESC_INNER_SHARE (3ULL << 8) -#define PTE_BLOCK_DESC_AF (1ULL << 10) -#define PTE_BLOCK_DESC_NG (1ULL << 11) -#define PTE_BLOCK_DESC_PXN (1ULL << 53) -#define PTE_BLOCK_DESC_UXN (1ULL << 54) +#define PTE_BLOCK_DESC_AF (1ULL << 10) /* A-flag */ +#define PTE_BLOCK_DESC_NG (1ULL << 11) /* Non-global */ +#define PTE_BLOCK_DESC_DIRTY (1ULL << 51) /* D-flag */ +#define PTE_BLOCK_DESC_PXN (1ULL << 53) /* Kernel execute never */ +#define PTE_BLOCK_DESC_UXN (1ULL << 54) /* User execute never */ + +/* PTE address field */ + +#define PTE_PADDR_SHIFT (12U) +#define PTE_PADDR_WIDTH (36U) +#define PTE_PADDR_MASK (((1UL << PTE_PADDR_WIDTH) - 1) << PTE_PADDR_SHIFT) /* TCR definitions. * @@ -119,7 +129,7 @@ #define TCR_EL3_PS_SHIFT 16U #define TCR_T0SZ_SHIFT 0U -#define TCR_T0SZ(x) ((64 - (x)) << TCR_T0SZ_SHIFT) +#define TCR_T0SZ(x) ((64 - (x)) << TCR_T0SZ_SHIFT) #define TCR_IRGN_NC (0ULL << 8) #define TCR_IRGN_WBWA (1ULL << 8) @@ -167,6 +177,15 @@ #define CCSIDR_EL1_SETS_SHIFT 13 #define CCSIDR_EL1_SETS_MASK BIT_MASK(15) +/* ttbr0/1_el1 */ + +#define TTBR_BADDR_SHIFT (0) +#define TTBR_BADDR_WIDTH (48) +#define TTBR_BADDR_MASK (((1UL << TTBR_BADDR_WIDTH) - 1) << TTBR_BADDR_SHIFT) +#define TTBR_ASID_SHIFT (48) +#define TTBR_ASID_WIDTH (16) +#define TTBR_ASID_MASK (((1UL << TTBR_ASID_WIDTH) - 1) << TTBR_ASID_SHIFT) + /* Convenience macros to represent the ARMv8-A-specific * configuration for memory access permission and * cache-ability attribution. @@ -184,6 +203,50 @@ #define MMU_REGION_FLAT_ENTRY(name, adr, sz, attrs) \ MMU_REGION_ENTRY(name, adr, adr, sz, attrs) +#define MMU_PAGE_SHIFT (12U) +#define MMU_PAGE_SIZE (1U << MMU_PAGE_SHIFT) /* 4K pages */ +#define MMU_PAGE_MASK (MMU_PAGE_SIZE - 1) + +/* Entries per PGT */ + +#define MMU_PAGE_ENTRIES (MMU_PAGE_SIZE / sizeof(uintptr_t)) + +/* Amount of page table levels */ + +#define MMU_PGT_LEVELS (3U) + +/* Page sizes per page table level */ + +#define MMU_L1_PAGE_SIZE (0x40000000) /* 1G */ +#define MMU_L2_PAGE_SIZE (0x200000) /* 2M */ +#define MMU_L3_PAGE_SIZE (0x1000) /* 4K */ + +/* Flags for user page tables */ + +#define MMU_UPGT_FLAGS (0) + +/* Flags for normal memory region */ + +#define MMU_MT_NORMAL_FLAGS (PTE_BLOCK_DESC_INNER_SHARE | PTE_BLOCK_DESC_MEMTYPE(MT_NORMAL)) + +/* Flags for user FLASH (RX) and user RAM (RW) */ + +#define MMU_UTEXT_FLAGS (PTE_BLOCK_DESC_AP_RO | PTE_BLOCK_DESC_PXN | PTE_BLOCK_DESC_AP_USER | PTE_BLOCK_DESC_NG | MMU_MT_NORMAL_FLAGS) +#define MMU_UDATA_FLAGS (PTE_BLOCK_DESC_AP_RW | PTE_BLOCK_DESC_PXN | PTE_BLOCK_DESC_UXN | PTE_BLOCK_DESC_AP_USER | PTE_BLOCK_DESC_NG | MMU_MT_NORMAL_FLAGS) + +/* I/O region flags */ + +#define MMU_IO_FLAGS (PTE_BLOCK_DESC_AP_RW | PTE_BLOCK_DESC_UXN | PTE_BLOCK_DESC_PXN) + +/* Flags for kernel page tables */ + +#define MMU_KPGT_FLAGS (0) + +/* Kernel FLASH and RAM are mapped globally */ + +#define MMU_KTEXT_FLAGS (PTE_BLOCK_DESC_AP_RO | PTE_BLOCK_DESC_UXN | MMU_MT_NORMAL_FLAGS) +#define MMU_KDATA_FLAGS (PTE_BLOCK_DESC_AP_RW | PTE_BLOCK_DESC_UXN | MMU_MT_NORMAL_FLAGS) + #ifndef __ASSEMBLY__ /**************************************************************************** @@ -233,6 +296,10 @@ struct arm_mmu_ptables uint64_t *base_xlat_table; }; +/**************************************************************************** + * Public Data + ****************************************************************************/ + /* Reference to the MMU configuration. * * This struct is defined and populated for each SoC, @@ -242,6 +309,193 @@ struct arm_mmu_ptables extern const struct arm_mmu_config g_mmu_config; +/* Kernel page directory root */ + +extern uintptr_t g_kernel_pgt_pbase; + +/**************************************************************************** + * Inline Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: mmu_ttbr_reg + * + * Description: + * Utility function to build ttbr register value for input parameters + * + * Input Parameters: + * pgbase - The physical base address of the translation table base + * asid - Address space identifier. This can be used to identify different + * address spaces. + * + ****************************************************************************/ + +static inline uintptr_t mmu_ttbr_reg(uintptr_t pgbase, uint16_t asid) +{ + uintptr_t reg; + reg = ((pgbase << TTBR_BADDR_SHIFT) & TTBR_BADDR_MASK); + reg |= (((uintptr_t)asid << TTBR_ASID_SHIFT) & TTBR_ASID_MASK); + return reg; +} + +/**************************************************************************** + * Name: mmu_invalidate_tlb_by_vaddr + * + * Description: + * Flush the TLB for vaddr entry + * + * Input Parameters: + * vaddr - The virtual address to flush + * + ****************************************************************************/ + +static inline void mmu_invalidate_tlb_by_vaddr(uintptr_t vaddr) +{ + __asm__ __volatile__ + ( + "dsb ishst\n" + "tlbi vale1is, %0\n" + "dsb ish\n" + "isb" + : + : "r" (vaddr) + : "memory" + ); +} + +/**************************************************************************** + * Name: mmu_invalidate_tlbs + * + * Description: + * Flush the entire TLB + * + ****************************************************************************/ + +static inline void mmu_invalidate_tlbs(void) +{ + __asm__ __volatile__ + ( + "dsb nshst\n" + "tlbi vmalle1\n" + "dsb nsh\n" + "isb" + : + : + : "memory" + ); +} + +/**************************************************************************** + * Name: mmu_write_ttbr0 + * + * Description: + * Write ttbr0 + * + * Input Parameters: + * reg - ttbr0 value + * + ****************************************************************************/ + +static inline void mmu_write_ttbr0(uintptr_t reg) +{ + write_sysreg(reg, ttbr0_el1); + mmu_invalidate_tlbs(); +} + +/**************************************************************************** + * Name: mmu_read_ttbr0 + * + * Description: + * Read ttbr0 + * + * Returned Value: + * ttbr0 register value + * + ****************************************************************************/ + +static inline uintptr_t mmu_read_ttbr0(void) +{ + return read_sysreg(ttbr0_el1); +} + +/**************************************************************************** + * Name: mmu_enable + * + * Description: + * Enable MMU and set the base page table address + * + * Input Parameters: + * pgbase - The physical base address of the translation table base + * asid - Address space identifier. This can be used to identify different + * address spaces. + * + ****************************************************************************/ + +static inline void mmu_enable(uintptr_t pgbase, uint16_t asid) +{ + uintptr_t reg = mmu_ttbr_reg(pgbase, asid); + + /* Commit to ttbr0 and synchronize */ + + mmu_write_ttbr0(reg); +} + +/**************************************************************************** + * Name: mmu_pte_to_paddr + * + * Description: + * Extract physical address from PTE + * + * Input Parameters: + * pte - Page table entry + * + * Returned Value: + * Physical address from PTE + * + ****************************************************************************/ + +static inline uintptr_t mmu_pte_to_paddr(uintptr_t pte) +{ + uintptr_t paddr = pte; + paddr &= PTE_PADDR_MASK; /* Remove flags */ + return paddr; +} + +/**************************************************************************** + * Name: mmu_ttbr_to_paddr + * + * Description: + * Extract physical address from TTBR0/1 + * + * Returned Value: + * Physical address from TTBR0/1 value + * + ****************************************************************************/ + +static inline uintptr_t mmu_ttbr_to_paddr(uintptr_t ttbr) +{ + uintptr_t baddr; + baddr = ttbr; + baddr = ((baddr >> TTBR_BADDR_SHIFT) & TTBR_BADDR_MASK); + return baddr; +} + +/**************************************************************************** + * Name: mmu_get_ttbr0_pgbase + * + * Description: + * Utility function to read the current base page table physical address + * + * Returned Value: + * Physical address of the current base page table + * + ****************************************************************************/ + +static inline uintptr_t mmu_get_ttbr0_pgbase(void) +{ + return mmu_ttbr_to_paddr(read_sysreg(ttbr0_el1)); +} + /**************************************************************************** * Public Function Prototypes ****************************************************************************/ @@ -249,6 +503,103 @@ extern const struct arm_mmu_config g_mmu_config; int arm64_mmu_init(bool is_primary_core); int arm64_mmu_set_memregion(const struct arm_mmu_region *region); +/**************************************************************************** + * Name: mmu_ln_setentry + * + * Description: + * Set a level n translation table entry. + * + * Input Parameters: + * ptlevel - The translation table level, amount of levels is + * MMU implementation specific + * lnvaddr - The virtual address of the beginning of the page table at + * level n + * paddr - The physical address to be mapped. Must be aligned to a PPN + * address boundary which is dependent on the level of the entry + * vaddr - The virtual address to be mapped. Must be aligned to a PPN + * address boundary which is dependent on the level of the entry + * mmuflags - The MMU flags to use in the mapping. + * + ****************************************************************************/ + +void mmu_ln_setentry(uint32_t ptlevel, uintptr_t lnvaddr, uintptr_t paddr, + uintptr_t vaddr, uint64_t mmuflags); + +/**************************************************************************** + * Name: mmu_ln_getentry + * + * Description: + * Get a level n translation table entry. + * + * Input Parameters: + * ptlevel - The translation table level, amount of levels is + * MMU implementation specific + * lnvaddr - The virtual address of the beginning of the page table at + * level n + * vaddr - The virtual address to get pte for. Must be aligned to a PPN + * address boundary which is dependent on the level of the entry + * + ****************************************************************************/ + +uintptr_t mmu_ln_getentry(uint32_t ptlevel, uintptr_t lnvaddr, + uintptr_t vaddr); + +/**************************************************************************** + * Name: mmu_ln_restore + * + * Description: + * Restore a level n translation table entry. + * + * Input Parameters: + * ptlevel - The translation table level, amount of levels is + * MMU implementation specific + * lnvaddr - The virtual address of the beginning of the page table at + * level n + * vaddr - The virtual address to get pte for. Must be aligned to a PPN + * address boundary which is dependent on the level of the entry + * entry - Entry to restore, previously obtained by mmu_ln_getentry + * + ****************************************************************************/ + +void mmu_ln_restore(uint32_t ptlevel, uintptr_t lnvaddr, uintptr_t vaddr, + uintptr_t entry); + +/**************************************************************************** + * Name: mmu_ln_clear + * + * Description: + * Unmap a level n translation table entry. + * + * Input Parameters: + * ptlevel - The translation table level, amount of levels is + * MMU implementation specific + * lnvaddr - The virtual address of the beginning of the page table at + * level n + * vaddr - The virtual address to get pte for. Must be aligned to a PPN + * address boundary which is dependent on the level of the entry + * + ****************************************************************************/ + +#define mmu_ln_clear(ptlevel, lnvaddr, vaddr) \ + mmu_ln_restore(ptlevel, lnvaddr, vaddr, 0) + +/**************************************************************************** + * Name: mmu_get_region_size + * + * Description: + * Get (giga/mega) page size for level n. + * + * Input Parameters: + * ptlevel - The translation table level, amount of levels is + * MMU implementation specific + * + * Returned Value: + * Region size for one page at level n. + * + ****************************************************************************/ + +size_t mmu_get_region_size(uint32_t ptlevel); + #endif /* __ASSEMBLY__ */ #endif /* __ARCH_ARM64_SRC_COMMON_ARM64_MMU_H */ diff --git a/arch/arm64/src/common/arm64_pgalloc.c b/arch/arm64/src/common/arm64_pgalloc.c new file mode 100644 index 00000000000..22115ffcd41 --- /dev/null +++ b/arch/arm64/src/common/arm64_pgalloc.c @@ -0,0 +1,173 @@ +/**************************************************************************** + * arch/arm64/src/common/arm64_pgalloc.c + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "addrenv.h" +#include "barriers.h" +#include "pgalloc.h" +#include "arm64_mmu.h" + +#ifdef CONFIG_BUILD_KERNEL + +/**************************************************************************** + * Pre-processor Definitions + ****************************************************************************/ + +/**************************************************************************** + * Private Functions + ****************************************************************************/ + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: pgalloc + * + * Description: + * If there is a page allocator in the configuration and if and MMU is + * available to map physical addresses to virtual address, then function + * must be provided by the platform-specific code. This is part of the + * implementation of sbrk(). This function will allocate the requested + * number of pages using the page allocator and map them into consecutive + * virtual addresses beginning with 'brkaddr' + * + * NOTE: This function does not use the up_ naming standard because it + * is indirectly callable from user-space code via a system trap. + * Therefore, it is a system interface and follows a different naming + * convention. + * + * Input Parameters: + * brkaddr - The heap break address. The next page will be allocated and + * mapped to this address. Must be page aligned. If the memory manager + * has not yet been initialized and this is the first block requested for + * the heap, then brkaddr should be zero. pgalloc will then assigned the + * well-known virtual address of the beginning of the heap. + * npages - The number of pages to allocate and map. Mapping of pages + * will be contiguous beginning beginning at 'brkaddr' + * + * Returned Value: + * The (virtual) base address of the mapped page will returned on success. + * Normally this will be the same as the 'brkaddr' input. However, if + * the 'brkaddr' input was zero, this will be the virtual address of the + * beginning of the heap. Zero is returned on any failure. + * + ****************************************************************************/ + +uintptr_t pgalloc(uintptr_t brkaddr, unsigned int npages) +{ + struct tcb_s *tcb = nxsched_self(); + struct arch_addrenv_s *addrenv; + uintptr_t ptlast; + uintptr_t paddr; + uintptr_t vaddr; + + DEBUGASSERT(tcb && tcb->addrenv_own); + addrenv = &tcb->addrenv_own->addrenv; + + /* The current implementation only supports extending the user heap + * region as part of the implementation of user sbrk(). This function + * needs to be expanded to also handle (1) extending the user stack + * space and (2) extending the kernel memory regions as well. + */ + + /* brkaddr = 0 means that no heap has yet been allocated */ + + if (!brkaddr) + { + brkaddr = addrenv->heapvbase; + } + + /* Start mapping from the old heap break address */ + + vaddr = brkaddr; + + /* Sanity checks */ + + DEBUGASSERT(brkaddr >= addrenv->heapvbase); + DEBUGASSERT(MM_ISALIGNED(brkaddr)); + + for (; npages > 0; npages--) + { + /* Get the address of the last level page table */ + + ptlast = arm64_pgvaddr(arm64_get_pgtable(addrenv, vaddr)); + if (!ptlast) + { + return 0; + } + + /* Allocate physical memory for the new heap */ + + paddr = mm_pgalloc(1); + if (!paddr) + { + return 0; + } + + /* Wipe the memory */ + + arm64_pgwipe(paddr); + + /* Then add the reference */ + + mmu_ln_setentry(MMU_PGT_LEVELS, ptlast, paddr, vaddr, MMU_UDATA_FLAGS); + vaddr += MM_PGSIZE; + } + + return brkaddr; +} + +/**************************************************************************** + * Name: up_allocate_pgheap + * + * Description: + * If there is a page allocator in the configuration, then this function + * must be provided by the platform-specific code. The OS initialization + * logic will call this function early in the initialization sequence to + * get the page heap information needed to configure the page allocator. + * + ****************************************************************************/ + +void up_allocate_pgheap(void **heap_start, size_t *heap_size) +{ + DEBUGASSERT(heap_start && heap_size); + + *heap_start = (void *)CONFIG_ARCH_PGPOOL_PBASE; + *heap_size = (size_t)CONFIG_ARCH_PGPOOL_SIZE; +} + +#endif /* CONFIG_BUILD_KERNEL */ diff --git a/arch/arm64/src/common/arm64_pthread_start.c b/arch/arm64/src/common/arm64_pthread_start.c new file mode 100644 index 00000000000..f063c9b2848 --- /dev/null +++ b/arch/arm64/src/common/arm64_pthread_start.c @@ -0,0 +1,104 @@ +/**************************************************************************** + * arch/arm64/src/common/arm64_pthread_start.c + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include +#include +#include +#include + +#include + +#include "arm64_internal.h" + +#include "sched/sched.h" + +#if !defined(CONFIG_BUILD_FLAT) && defined(__KERNEL__) && \ + !defined(CONFIG_DISABLE_PTHREAD) + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: up_pthread_start + * + * Description: + * In this kernel mode build, this function will be called to execute a + * pthread in user-space. When the pthread is first started, a kernel-mode + * stub will first run to perform some housekeeping functions. This + * kernel-mode stub will then be called transfer control to the user-mode + * pthread. + * + * Normally the a user-mode start-up stub will also execute before the + * pthread actually starts. See libc/pthread/pthread_create.c + * + * Input Parameters: + * startup - The user-space pthread startup function + * entrypt - The user-space address of the pthread entry point + * arg - Standard argument for the pthread entry point + * + * Returned Value: + * This function should not return. It should call the user-mode start-up + * stub and that stub should call pthread_exit if/when the user pthread + * terminates. + * + ****************************************************************************/ + +void up_pthread_start(pthread_trampoline_t startup, + pthread_startroutine_t entrypt, pthread_addr_t arg) +{ + struct tcb_s *tcb = this_task(); + uint64_t spsr; + + /* Set up to enter the user-space pthread start-up function in + * unprivileged mode. We need: + * + * R0 = entrypt + * R1 = arg + * ELR = startup + * SPSR = user mode + */ + + tcb->xcp.regs[REG_ELR] = (uint64_t)startup; + tcb->xcp.regs[REG_X0] = (uint64_t)entrypt; + tcb->xcp.regs[REG_X1] = (uint64_t)arg; + spsr = tcb->xcp.regs[REG_SPSR] & ~SPSR_MODE_MASK; + tcb->xcp.regs[REG_SPSR] = spsr | SPSR_MODE_EL0T; + + /* Fully unwind the kernel stack and drop to user space */ + + asm + ( + "mov x0, %0\n" /* Get context registers */ + "mov sp, x0\n" /* Stack pointer = context */ + "b arm64_exit_exception\n" + : + : "r" (tcb->xcp.regs) + : "memory" + ); + + PANIC(); +} + +#endif /* !CONFIG_BUILD_FLAT && __KERNEL__ && !CONFIG_DISABLE_PTHREAD */ diff --git a/arch/arm64/src/common/arm64_schedulesigaction.c b/arch/arm64/src/common/arm64_schedulesigaction.c index eee31cd17f8..47060cdd808 100644 --- a/arch/arm64/src/common/arm64_schedulesigaction.c +++ b/arch/arm64/src/common/arm64_schedulesigaction.c @@ -56,21 +56,21 @@ void arm64_init_signal_process(struct tcb_s *tcb, struct regs_context *regs) struct regs_context *pctx = (regs != NULL) ? regs : (struct regs_context *)tcb->xcp.regs; struct regs_context *psigctx; - char *stack_ptr = (char *)pctx->sp_elx - sizeof(struct regs_context); + char *stack_ptr = (char *)pctx->sp_elx - sizeof(struct regs_context); #ifdef CONFIG_ARCH_FPU struct fpu_reg *pfpuctx; - pfpuctx = STACK_PTR_TO_FRAME(struct fpu_reg, stack_ptr); - tcb->xcp.fpu_regs = (uint64_t *)pfpuctx; + pfpuctx = STACK_PTR_TO_FRAME(struct fpu_reg, stack_ptr); + tcb->xcp.fpu_regs = (uint64_t *)pfpuctx; /* set fpu context */ arm64_init_fpu(tcb); - stack_ptr = (char *)pfpuctx; + stack_ptr = (char *)pfpuctx; #endif - psigctx = STACK_PTR_TO_FRAME(struct regs_context, stack_ptr); + psigctx = STACK_PTR_TO_FRAME(struct regs_context, stack_ptr); memset(psigctx, 0, sizeof(struct regs_context)); - psigctx->elr = (uint64_t)arm64_sigdeliver; + psigctx->elr = (uint64_t)arm64_sigdeliver; /* Keep using SP_EL1 */ @@ -80,7 +80,11 @@ void arm64_init_signal_process(struct tcb_s *tcb, struct regs_context *regs) psigctx->spsr = SPSR_MODE_EL1H | DAIF_FIQ_BIT | DAIF_IRQ_BIT; #endif psigctx->sp_elx = (uint64_t)stack_ptr; +#ifdef CONFIG_ARCH_KERNEL_STACK + psigctx->sp_el0 = (uint64_t)pctx->sp_el0; +#else psigctx->sp_el0 = (uint64_t)psigctx; +#endif psigctx->exe_depth = 1; psigctx->tpidr_el0 = (uint64_t)tcb; psigctx->tpidr_el1 = (uint64_t)tcb; diff --git a/arch/arm64/src/common/arm64_signal_dispatch.c b/arch/arm64/src/common/arm64_signal_dispatch.c new file mode 100644 index 00000000000..c8153536977 --- /dev/null +++ b/arch/arm64/src/common/arm64_signal_dispatch.c @@ -0,0 +1,76 @@ +/**************************************************************************** + * arch/arm64/src/common/arm64_signal_dispatch.c + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include +#include + +#include + +#include "arm64_internal.h" + +#if !defined(CONFIG_BUILD_FLAT) && defined(__KERNEL__) + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: up_signal_dispatch + * + * Description: + * In the kernel mode build, this function will be called to execute a + * a signal handler in user-space. When the signal is delivered, a + * kernel-mode stub will first run to perform some housekeeping functions. + * This kernel-mode stub will then be called transfer control to the user + * mode signal handler by calling this function. + * + * Normally the user-mode signaling handling stub will also execute + * before the ultimate signal handler is called. See + * arch/arm64/src/common/arm64_signal_handler.S. This function is the + * user-space, signal handler trampoline function. It is called from + * up_signal_dispatch() in user-mode. + * + * Input Parameters: + * sighand - The address user-space signal handling function + * signo, info, and ucontext - Standard arguments to be passed to the + * signal handling function. + * + * Returned Value: + * None. This function does not return in the normal sense. It returns + * via an architecture specific system call made by up_signal_handler(). + * However, this will look like a normal return by the caller of + * up_signal_dispatch. + * + ****************************************************************************/ + +void up_signal_dispatch(_sa_sigaction_t sighand, int signo, + siginfo_t *info, void *ucontext) +{ + /* Let sys_call4() do all of the work */ + + sys_call4(SYS_signal_handler, (uintptr_t)sighand, (uintptr_t)signo, + (uintptr_t)info, (uintptr_t)ucontext); +} + +#endif /* !CONFIG_BUILD_FLAT && __KERNEL__ */ diff --git a/arch/arm64/src/common/arm64_syscall.c b/arch/arm64/src/common/arm64_syscall.c index 6a80b645ee3..fa8b51b4e3b 100644 --- a/arch/arm64/src/common/arm64_syscall.c +++ b/arch/arm64/src/common/arm64_syscall.c @@ -43,54 +43,10 @@ #include "signal/signal.h" /**************************************************************************** - * Private Functions + * Private Types ****************************************************************************/ -/**************************************************************************** - * Name: dispatch_syscall - * - * Description: - * Call the stub function corresponding to the system call. NOTE the non- - * standard parameter passing: - * - * x0 = SYS_ call number - * x1 = parm0 - * x2 = parm1 - * x3 = parm2 - * x4 = parm3 - * x5 = parm4 - * x6 = parm5 - * - * The values of X4-X5 may be preserved in the proxy called by the user - * code if they are used (but otherwise will not be). - * - * WARNING: There are hard-coded values in this logic! - * - ****************************************************************************/ - -#ifdef CONFIG_LIB_SYSCALL -static void dispatch_syscall(void) naked_function; -static void dispatch_syscall(void) -{ - __asm__ __volatile__ - ( - " sub sp, sp, #16\n" /* Create a stack frame to hold 3 parms + lr */ - " str r4, [sp, #0]\n" /* Move parameter 4 (if any) into position */ - " str r5, [sp, #4]\n" /* Move parameter 5 (if any) into position */ - " str r6, [sp, #8]\n" /* Move parameter 6 (if any) into position */ - " str lr, [sp, #12]\n" /* Save lr in the stack frame */ - " ldr ip, =g_stublookup\n" /* R12=The base of the stub lookup table */ - " ldr ip, [ip, r0, lsl #2]\n" /* R12=The address of the stub for this SYSCALL */ - " blx ip\n" /* Call the stub (modifies lr) */ - " ldr lr, [sp, #12]\n" /* Restore lr */ - " add sp, sp, #16\n" /* Destroy the stack frame */ - " mov r2, r0\n" /* R2=Save return value in R2 */ - " mov r0, %0\n" /* R0=SYS_syscall_return */ - " svc %1\n"::"i"(SYS_syscall_return), - "i"(SYS_syscall) /* Return from the SYSCALL */ - ); -} -#endif +typedef uintptr_t (*syscall_t)(unsigned int, ...); /**************************************************************************** * Private Functions @@ -111,10 +67,87 @@ static void arm64_dump_syscall(const char *tag, uint64_t cmd, f_regs->regs[REG_X6], f_regs->regs[REG_X7]); } +#ifdef CONFIG_LIB_SYSCALL + /**************************************************************************** * Public Functions ****************************************************************************/ +/**************************************************************************** + * Name: dispatch_syscall + * + * Description: + * Call the stub function corresponding to the system call. NOTE the non- + * standard parameter passing: + * + * R0 = SYS_ call number + * R1 = parm0 + * R2 = parm1 + * R3 = parm2 + * R4 = parm3 + * R5 = parm4 + * R6 = parm5 + * R7 = context (aka SP) + * + ****************************************************************************/ + +uintptr_t dispatch_syscall(unsigned int nbr, uintptr_t parm1, + uintptr_t parm2, uintptr_t parm3, + uintptr_t parm4, uintptr_t parm5, + uintptr_t parm6, void *context) +{ + struct tcb_s *rtcb = this_task(); + register long x0 asm("x0") = (long)(nbr); + register long x1 asm("x1") = (long)(parm1); + register long x2 asm("x2") = (long)(parm2); + register long x3 asm("x3") = (long)(parm3); + register long x4 asm("x4") = (long)(parm4); + register long x5 asm("x5") = (long)(parm5); + register long x6 asm("x6") = (long)(parm6); + syscall_t do_syscall; + uintptr_t ret; + + /* Valid system call ? */ + + if (x0 > SYS_maxsyscall) + { + /* Nope, get out */ + + return -ENOSYS; + } + + /* Set the user register context to TCB */ + + rtcb->xcp.regs = context; + + /* Indicate that we are in a syscall handler */ + + rtcb->flags |= TCB_FLAG_SYSCALL; + + /* Offset a0 to account for the reserved syscalls */ + + x0 -= CONFIG_SYS_RESERVED; + + /* Find the system call from the lookup table */ + + do_syscall = (syscall_t)g_stublookup[x0]; + + /* Run the system call, save return value locally */ + + ret = do_syscall(x0, x1, x2, x3, x4, x5, x6); + + /* System call is now done */ + + rtcb->flags &= ~TCB_FLAG_SYSCALL; + + /* Unmask any pending signals now */ + + nxsig_unmask_pendingsignal(); + + return ret; +} +#endif + /**************************************************************************** * Name: arm64_syscall_switch * @@ -245,6 +278,9 @@ int arm64_syscall(uint64_t *regs) { uint64_t cmd; struct regs_context *f_regs; +#ifdef CONFIG_BUILD_KERNEL + uint64_t spsr; +#endif /* Nested interrupts are not supported */ @@ -260,138 +296,6 @@ int arm64_syscall(uint64_t *regs) switch (cmd) { - /* R0=SYS_syscall_return: This a SYSCALL return command: - * - * void arm_syscall_return(void); - * - * At this point, the following values are saved in context: - * - * R0 = SYS_syscall_return - * - * We need to restore the saved return address and return in - * unprivileged thread mode. - */ - -#ifdef CONFIG_LIB_SYSCALL - case SYS_syscall_return: - { - struct tcb_s *rtcb = nxsched_self(); - int index = (int)rtcb->xcp.nsyscalls - 1; - - /* Make sure that there is a saved SYSCALL return address. */ - - DEBUGASSERT(index >= 0); - - /* Setup to return to the saved SYSCALL return address in - * the original mode. - */ - - regs[REG_PC] = rtcb->xcp.syscall[index].sysreturn; -#ifdef CONFIG_BUILD_KERNEL - regs[REG_CPSR] = rtcb->xcp.syscall[index].cpsr; -#endif - /* The return value must be in R0-R1. dispatch_syscall() - * temporarily moved the value for R0 into R2. - */ - - regs[REG_R0] = regs[REG_R2]; - -#ifdef CONFIG_ARCH_KERNEL_STACK - /* If this is the outermost SYSCALL and if there is a saved user - * stack pointer, then restore the user stack pointer on this - * final return to user code. - */ - - if (index == 0 && rtcb->xcp.ustkptr != NULL) - { - regs[REG_SP] = (uint64_t)rtcb->xcp.ustkptr; - rtcb->xcp.ustkptr = NULL; - } -#endif - - /* Save the new SYSCALL nesting level */ - - rtcb->xcp.nsyscalls = index; - - /* Handle any signal actions that were deferred while processing - * the system call. - */ - - rtcb->flags &= ~TCB_FLAG_SYSCALL; - (void)nxsig_unmask_pendingsignal(); - } - break; -#endif - - /* R0=SYS_task_start: This a user task start - * - * void up_task_start(main_t taskentry, int argc, char *argv[]) - * noreturn_function; - * - * At this point, the following values are saved in context: - * - * R0 = SYS_task_start - * R1 = taskentry - * R2 = argc - * R3 = argv - */ - -#ifdef CONFIG_BUILD_KERNEL - case SYS_task_start: - { - /* Set up to return to the user-space _start function in - * unprivileged mode. We need: - * - * R0 = argc - * R1 = argv - * PC = taskentry - * CSPR = user mode - */ - - regs[REG_PC] = regs[REG_R1]; - regs[REG_R0] = regs[REG_R2]; - regs[REG_R1] = regs[REG_R3]; - - cpsr = regs[REG_CPSR] & ~PSR_MODE_MASK; - regs[REG_CPSR] = cpsr | PSR_MODE_USR; - } - break; -#endif - - /* R0=SYS_pthread_start: This a user pthread start - * - * void up_pthread_start(pthread_startroutine_t entrypt, - * pthread_addr_t arg) noreturn_function; - * - * At this point, the following values are saved in context: - * - * R0 = SYS_pthread_start - * R1 = entrypt - * R2 = arg - */ - -#if !defined(CONFIG_BUILD_FLAT) && !defined(CONFIG_DISABLE_PTHREAD) - case SYS_pthread_start: - { - /* Set up to enter the user-space pthread start-up function in - * unprivileged mode. We need: - * - * R0 = entrypt - * R1 = arg - * PC = startup - * CSPR = user mode - */ - - regs[REG_PC] = regs[REG_R1]; - regs[REG_R0] = regs[REG_R2]; - regs[REG_R1] = regs[REG_R3]; - - cpsr = regs[REG_CPSR] & ~PSR_MODE_MASK; - regs[REG_CPSR] = cpsr | PSR_MODE_USR; - } - break; -#endif - #ifdef CONFIG_BUILD_KERNEL /* R0=SYS_signal_handler: This a user signal handler callback * @@ -414,24 +318,24 @@ int arm64_syscall(uint64_t *regs) /* Remember the caller's return address */ DEBUGASSERT(rtcb->xcp.sigreturn == 0); - rtcb->xcp.sigreturn = regs[REG_PC]; + rtcb->xcp.sigreturn = regs[REG_ELR]; /* Set up to return to the user-space trampoline function in * unprivileged mode. */ - regs[REG_PC] = (uint64_t)ARCH_DATA_RESERVE->ar_sigtramp; - cpsr = regs[REG_CPSR] & ~PSR_MODE_MASK; - regs[REG_CPSR] = cpsr | PSR_MODE_USR; + regs[REG_ELR] = (uint64_t)ARCH_DATA_RESERVE->ar_sigtramp; + spsr = regs[REG_SPSR] & ~SPSR_MODE_MASK; + regs[REG_SPSR] = spsr | SPSR_MODE_EL0T; /* Change the parameter ordering to match the expectation of struct * userpace_s signal_handler. */ - regs[REG_R0] = regs[REG_R1]; /* sighand */ - regs[REG_R1] = regs[REG_R2]; /* signal */ - regs[REG_R2] = regs[REG_R3]; /* info */ - regs[REG_R3] = regs[REG_R4]; /* ucontext */ + regs[REG_X0] = regs[REG_X1]; /* sighand */ + regs[REG_X1] = regs[REG_X2]; /* signal */ + regs[REG_X2] = regs[REG_X3]; /* info */ + regs[REG_X3] = regs[REG_X4]; /* ucontext */ #ifdef CONFIG_ARCH_KERNEL_STACK /* If we are signalling a user process, then we must be operating @@ -445,29 +349,16 @@ int arm64_syscall(uint64_t *regs) { uint64_t usp; - DEBUGASSERT(rtcb->xcp.kstkptr == NULL); - - /* Copy "info" into user stack */ - - if (rtcb->xcp.sigdeliver) - { - usp = rtcb->xcp.saved_regs[REG_SP]; - } - else - { - usp = rtcb->xcp.regs[REG_SP]; - } - /* Create a frame for info and copy the kernel info */ - usp = usp - sizeof(siginfo_t); - memcpy((void *)usp, (void *)regs[REG_R2], sizeof(siginfo_t)); + rtcb->xcp.ustkptr = (uintptr_t *)read_sysreg(sp_el0); + usp = (uintptr_t)rtcb->xcp.ustkptr - sizeof(siginfo_t); + memcpy((void *)usp, (void *)regs[REG_X2], sizeof(siginfo_t)); /* Now set the updated SP and user copy of "info" to R2 */ - rtcb->xcp.kstkptr = (uint64_t *)regs[REG_SP]; - regs[REG_SP] = usp; - regs[REG_R2] = usp; + write_sysreg(usp, sp_el0); + regs[REG_X2] = usp; } #endif } @@ -492,98 +383,24 @@ int arm64_syscall(uint64_t *regs) DEBUGASSERT(rtcb->xcp.sigreturn != 0); - regs[REG_PC] = rtcb->xcp.sigreturn; - cpsr = regs[REG_CPSR] & ~PSR_MODE_MASK; - regs[REG_CPSR] = cpsr | PSR_MODE_SVC; + regs[REG_ELR] = rtcb->xcp.sigreturn; + spsr = regs[REG_SPSR] & ~SPSR_MODE_MASK; + regs[REG_SPSR] = spsr | SPSR_MODE_EL1H; rtcb->xcp.sigreturn = 0; #ifdef CONFIG_ARCH_KERNEL_STACK - /* We must enter here be using the user stack. We need to switch - * to back to the kernel user stack before returning to the kernel - * mode signal trampoline. - */ + /* Restore the original user SP, destroying the signal frame */ - if (rtcb->xcp.kstack != NULL) - { - DEBUGASSERT(rtcb->xcp.kstkptr != NULL); - - regs[REG_SP] = (uint64_t)rtcb->xcp.kstkptr; - rtcb->xcp.kstkptr = NULL; - } + write_sysreg(rtcb->xcp.ustkptr, sp_el0); #endif } break; #endif - /* This is not an architecture-specific system call. If NuttX is built - * as a standalone kernel with a system call interface, then all of the - * additional system calls must be handled as in the default case. - */ - default: - { -#ifdef CONFIG_LIB_SYSCALL - struct tcb_s *rtcb = nxsched_self(); - int index = rtcb->xcp.nsyscalls; - /* Verify that the SYS call number is within range */ - - DEBUGASSERT(cmd >= CONFIG_SYS_RESERVED && cmd < SYS_maxsyscall); - - /* Make sure that there is a no saved SYSCALL return address. We - * cannot yet handle nested system calls. - */ - - DEBUGASSERT(index < CONFIG_SYS_NNEST); - - /* Setup to return to dispatch_syscall in privileged mode. */ - - rtcb->xcp.syscall[index].sysreturn = regs[REG_PC]; -#ifdef CONFIG_BUILD_KERNEL - rtcb->xcp.syscall[index].cpsr = regs[REG_CPSR]; -#endif - - regs[REG_PC] = (uint64_t)dispatch_syscall; -#ifdef CONFIG_BUILD_KERNEL - cpsr = regs[REG_CPSR] & ~PSR_MODE_MASK; - regs[REG_CPSR] = cpsr | PSR_MODE_SVC; -#endif - /* Offset R0 to account for the reserved values */ - - regs[REG_R0] -= CONFIG_SYS_RESERVED; - - /* Indicate that we are in a syscall handler. */ - - rtcb->flags |= TCB_FLAG_SYSCALL; - -#ifdef CONFIG_ARCH_KERNEL_STACK - /* If this is the first SYSCALL and if there is an allocated - * kernel stack, then switch to the kernel stack. - */ - - if (index == 0 && rtcb->xcp.kstack != NULL) - { - rtcb->xcp.ustkptr = (uint64_t *)regs[REG_SP]; - if (rtcb->xcp.kstkptr != NULL) - { - regs[REG_SP] = (uint64_t)rtcb->xcp.kstkptr; - } - else - { - regs[REG_SP] = (uint64_t)rtcb->xcp.kstack + - ARCH_KERNEL_STACKSIZE; - } - } -#endif - - /* Save the new SYSCALL nesting level */ - - rtcb->xcp.nsyscalls = index + 1; -#else - svcerr("ERROR: Bad SYS call: 0x%" PRIx64 "\n", cmd); -#endif - } - break; + DEBUGPANIC(); + break; } return 0; diff --git a/arch/arm64/src/common/arm64_task_start.c b/arch/arm64/src/common/arm64_task_start.c new file mode 100644 index 00000000000..fd91e7b378e --- /dev/null +++ b/arch/arm64/src/common/arm64_task_start.c @@ -0,0 +1,101 @@ +/**************************************************************************** + * arch/arm64/src/common/arm64_task_start.c + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include +#include +#include + +#include + +#include "arm64_internal.h" + +#include "sched/sched.h" + +#ifndef CONFIG_BUILD_FLAT + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: up_task_start + * + * Description: + * In this kernel mode build, this function will be called to execute a + * task in user-space. When the task is first started, a kernel-mode + * stub will first run to perform some housekeeping functions. This + * kernel-mode stub will then be called transfer control to the user-mode + * task. + * + * Normally the a user-mode start-up stub will also execute before the + * task actually starts. See libc/sched/task_startup.c + * + * Input Parameters: + * taskentry - The user-space entry point of the task. + * argc - The number of parameters being passed. + * argv - The parameters being passed. These lie in kernel-space memory + * and will have to be reallocated in user-space memory. + * + * Returned Value: + * This function should not return. It should call the user-mode start-up + * stub and that stub should call exit if/when the user task terminates. + * + ****************************************************************************/ + +void up_task_start(main_t taskentry, int argc, char *argv[]) +{ + struct tcb_s *tcb = this_task(); + uint64_t spsr; + + /* Set up to return to the user-space _start function in + * unprivileged mode. We need: + * + * R0 = argc + * R1 = argv + * ELR = taskentry + * SPSR = user mode + */ + + tcb->xcp.regs[REG_ELR] = (uint64_t)taskentry; + tcb->xcp.regs[REG_X0] = (uint64_t)argc; + tcb->xcp.regs[REG_X1] = (uint64_t)argv; + spsr = tcb->xcp.regs[REG_SPSR] & ~SPSR_MODE_MASK; + tcb->xcp.regs[REG_SPSR] = spsr | SPSR_MODE_EL0T; + + /* Fully unwind the kernel stack and drop to user space */ + + asm + ( + "mov x0, %0\n" /* Get context registers */ + "mov sp, x0\n" /* Stack pointer = context */ + "b arm64_exit_exception\n" + : + : "r" (tcb->xcp.regs) + : "memory" + ); + + PANIC(); +} + +#endif /* !CONFIG_BUILD_FLAT */ diff --git a/arch/arm64/src/common/arm64_vectors.S b/arch/arm64/src/common/arm64_vectors.S index 3a4ce2f74c7..95ab1f13676 100644 --- a/arch/arm64/src/common/arm64_vectors.S +++ b/arch/arm64/src/common/arm64_vectors.S @@ -56,6 +56,22 @@ .endm +.macro enable_irq +#ifdef CONFIG_ARM64_DECODEFIQ + msr daifclr, #3 +#else + msr daifclr, #2 +#endif +.endm + +.macro disable_irq +#ifdef CONFIG_ARM64_DECODEFIQ + msr daifset, #3 +#else + msr daifset, #2 +#endif +.endm + /**************************************************************************** * Public Functions ****************************************************************************/ @@ -132,7 +148,7 @@ SECTION_FUNC(text, arm64_context_switch) ldp x0, x1, [sp], #16 #endif - /* Save the current SP_EL0 */ + /* Save the current SP_ELx */ add x4, sp, #8 * XCPTCONTEXT_GP_REGS str x4, [x1, #8 * REG_SP_ELX] @@ -214,6 +230,30 @@ SECTION_FUNC(text, arm64_sync_exc) bne exc_handle +#ifdef CONFIG_LIB_SYSCALL + /* Handle user system calls separately */ + + cmp x0, #CONFIG_SYS_RESERVED + blt reserved_syscall + + /* Call dispatch_syscall() on the kernel stack with interrupts enabled */ + + enable_irq + mov x7, sp /* x7 = context */ + bl dispatch_syscall + disable_irq + + /* Save the return value into the user context */ + + str x0, [sp, #8 * REG_X0] + + /* Return from exception */ + + b arm64_exit_exception + +reserved_syscall: +#endif + /* x0 = syscall_cmd * if ( x0 <= SYS_switch_context ) { * call context_switch @@ -253,10 +293,6 @@ SECTION_FUNC(text, arm64_sync_exc) bl arm64_syscall /* Call the handler */ - /* Save the return value into the */ - - str x0, [sp, #8 * REG_X0] - /* Return from exception */ b arm64_exit_exception diff --git a/arch/arm64/src/common/crt0.c b/arch/arm64/src/common/crt0.c index 750a5e250b9..74feb54aa1d 100644 --- a/arch/arm64/src/common/crt0.c +++ b/arch/arm64/src/common/crt0.c @@ -27,6 +27,7 @@ #include #include +#include #include #include @@ -71,21 +72,75 @@ static void sig_trampoline(void) { __asm__ __volatile__ ( - " push {lr}\n" /* Save LR on the stack */ - " mov ip, r0\n" /* IP=sighand */ - " mov r0, r1\n" /* R0=signo */ - " mov r1, r2\n" /* R1=info */ - " mov r2, r3\n" /* R2=ucontext */ - " blx ip\n" /* Call the signal handler */ - " pop {r2}\n" /* Recover LR in R2 */ - " mov lr, r2\n" /* Restore LR */ - " mov r0, %0\n" /* SYS_signal_handler_return */ - " svc %1\n" /* Return from the SYSCALL */ - ::"i"(SYS_signal_handler_return), + " sub sp, sp, #8\n" /* Create a stack frame to hold LR */ + " str lr, [sp, #0]\n" /* Save LR on the stack */ + " mov ip0, x0\n" /* IP=sighand */ + " mov x0, x1\n" /* R0=signo */ + " mov x1, x2\n" /* R1=info */ + " mov x2, x3\n" /* R2=ucontext */ + " blr ip0\n" /* Call the signal handler */ + " ldr lr, [sp, #0]\n" /* Recover LR in R2 */ + " add sp, sp, #8\n" /* Destroy the stack frame */ + " mov x0, %0\n" /* SYS_signal_handler_return */ + " svc %1\n" /* Return from the SYSCALL */ + : + : "i"(SYS_signal_handler_return), "i"(SYS_syscall) + : ); } +/**************************************************************************** + * Public Data + ****************************************************************************/ + +/* Linker defined symbols to .ctors and .dtors */ + +extern initializer_t _sctors[]; +extern initializer_t _ectors[]; +extern initializer_t _sdtors[]; +extern initializer_t _edtors[]; + +/**************************************************************************** + * Private Functions + ****************************************************************************/ + +#ifdef CONFIG_HAVE_CXX + +/**************************************************************************** + * Name: exec_ctors + * + * Description: + * Call static constructors + * + ****************************************************************************/ + +static void exec_ctors(void) +{ + for (initializer_t *ctor = _sctors; ctor != _ectors; ctor++) + { + (*ctor)(); + } +} + +/**************************************************************************** + * Name: exec_dtors + * + * Description: + * Call static destructors + * + ****************************************************************************/ + +static void exec_dtors(void) +{ + for (initializer_t *dtor = _sdtors; dtor != _edtors; dtor++) + { + (*dtor)(); + } +} + +#endif + /**************************************************************************** * Public Functions ****************************************************************************/ @@ -120,11 +175,15 @@ void __start(int argc, char *argv[]) ARCH_DATA_RESERVE->ar_sigtramp = (addrenv_sigtramp_t)sig_trampoline; +#ifdef CONFIG_HAVE_CXX /* Call C++ constructors */ + exec_ctors(); + /* Setup so that C++ destructors called on task exit */ - /* REVISIT: Missing logic */ + atexit(exec_dtors); +#endif /* Call the main() entry point passing argc and argv. */ diff --git a/arch/arm64/src/common/pgalloc.h b/arch/arm64/src/common/pgalloc.h new file mode 100644 index 00000000000..34fd861ec7a --- /dev/null +++ b/arch/arm64/src/common/pgalloc.h @@ -0,0 +1,127 @@ +/**************************************************************************** + * arch/arm64/src/common/pgalloc.h + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +#ifndef __ARCH_ARM64_SRC_COMMON_PGALLOC_H +#define __ARCH_ARM64_SRC_COMMON_PGALLOC_H + +#ifdef CONFIG_MM_PGALLOC + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +#include +#include + +#include "addrenv.h" + +/**************************************************************************** + * Pre-processor Definitions + ****************************************************************************/ + +#ifndef CONFIG_ARCH_PGPOOL_MAPPING +# error "ARM64 needs CONFIG_ARCH_PGPOOL_MAPPING" +#endif + +/**************************************************************************** + * Public Functions Prototypes + ****************************************************************************/ + +/**************************************************************************** + * Name: arm64_pgvaddr + * + * Description: + * Get virtual address for pgpool physical address. Note: this function + * is minimalistic and is only usable for kernel mappings and only tests + * if the paddr is in the pgpool. For user mapped addresses this does not + * work. + * + * Note: + * To get it to work with user addresses, a manual table walk needs to be + * implemented. Not too complex, but not needed for anything -> not + * implemented. + * + * Input Parameters: + * paddr - Physical pgpool address + * + * Return: + * vaddr - Virtual address for physical address + * + ****************************************************************************/ + +#ifdef CONFIG_ARCH_PGPOOL_MAPPING +static inline uintptr_t arm64_pgvaddr(uintptr_t paddr) +{ + if (paddr >= CONFIG_ARCH_PGPOOL_PBASE && paddr < CONFIG_ARCH_PGPOOL_PEND) + { + return paddr - CONFIG_ARCH_PGPOOL_PBASE + CONFIG_ARCH_PGPOOL_VBASE; + } + else if (paddr >= CONFIG_RAM_START && paddr < CONFIG_RAM_END) + { + return paddr - CONFIG_RAM_START + CONFIG_RAM_VSTART; + } + + return 0; +} +#endif /* CONFIG_ARCH_PGPOOL_MAPPING */ + +/**************************************************************************** + * Name: arm64_uservaddr + * + * Description: + * Return true if the virtual address, vaddr, lies in the user address + * space. + * + ****************************************************************************/ + +static inline bool arm64_uservaddr(uintptr_t vaddr) +{ + /* Check if this address is within the range of the virtualized .bss/.data, + * heap, or stack regions. + */ + + return ((vaddr >= ARCH_ADDRENV_VBASE && vaddr < ARCH_ADDRENV_VEND) +#ifdef CONFIG_ARCH_VMA_MAPPING + || (vaddr >= CONFIG_ARCH_SHM_VBASE && vaddr < ARCH_SHM_VEND) +#endif + ); +} + +/**************************************************************************** + * Name: arm64_pgwipe + * + * Description: + * Wipe a page of physical memory, first mapping it into virtual memory. + * + * Input Parameters: + * paddr - Physical address of page + * + ****************************************************************************/ + +static inline void arm64_pgwipe(uintptr_t paddr) +{ + uintptr_t vaddr = arm64_pgvaddr(paddr); + memset((void *)vaddr, 0, MM_PGSIZE); +} + +#endif /* CONFIG_MM_PGALLOC */ +#endif /* __ARCH_ARM64_SRC_COMMON_PGALLOC_H */