diff --git a/arch/xtensa/src/esp32s3/Kconfig b/arch/xtensa/src/esp32s3/Kconfig index bb16be86b60..8b941c160a2 100644 --- a/arch/xtensa/src/esp32s3/Kconfig +++ b/arch/xtensa/src/esp32s3/Kconfig @@ -343,6 +343,26 @@ config ESP32S3_RUN_IRAM This loads all of NuttX inside IRAM. Used to test somewhat small images that can fit entirely in IRAM. +config ESP32S3_PGPOOL_SCRATCH + bool + default y + depends on BUILD_KERNEL && ARCH_ADDRENV && MM_PGALLOC + select ARCH_KVMA_MAPPING + ---help--- + The page pool is deliberately not kept mapped into the kernel + address space: it is carved out of the PSRAM the user processes + themselves run from, and the external memory permissions are indexed + by physical address, so a kernel window onto the pool is a window + onto every process's memory that no permission setting can close. + + The kernel reaches a pool page through a small scratch region + instead, mapped for the duration of one operation and invalidated + afterwards. ARCH_KMAP_VBASE and ARCH_KMAP_NPAGES describe it. + + The pool itself is described by ARCH_PGPOOL_PBASE and + ARCH_PGPOOL_SIZE as usual. Only ARCH_PGPOOL_VBASE does not apply, + because there is no permanent virtual mapping to name. + menu "ESP32-S3 Peripheral Selection" source "arch/xtensa/src/common/espressif/Kconfig" @@ -923,12 +943,12 @@ config ESP32S3_WCL bool "World Controller" default n select ARCH_USE_MPU - select XTENSA_HAVE_GENERAL_EXCEPTION_HOOKS if BUILD_PROTECTED + select XTENSA_HAVE_GENERAL_EXCEPTION_HOOKS if !BUILD_FLAT config ESP32S3_PAGEFAULT bool "Recoverable PMS permission faults" default n - depends on BUILD_PROTECTED + depends on !BUILD_FLAT ---help--- Route the precise Load/Store/InstrFetch Prohibited exceptions raised by PMS (memory-protection) permission violations through a diff --git a/arch/xtensa/src/esp32s3/esp32s3_addrenv.c b/arch/xtensa/src/esp32s3/esp32s3_addrenv.c index 42c0bd3d0e5..1b15cf3635e 100644 --- a/arch/xtensa/src/esp32s3/esp32s3_addrenv.c +++ b/arch/xtensa/src/esp32s3/esp32s3_addrenv.c @@ -316,9 +316,16 @@ ssize_t up_addrenv_heapsize(const arch_addrenv_t *addrenv) * is no page-table-base register to load; instead the shared user * cache-MMU windows (.text on the instruction bus, .data/.bss and heap on * the data bus) are reprogrammed to point at this environment's PSRAM - * pages. Only one user environment can be resident at a time, so - * isolation between groups is provided by this remap: while a group runs, - * only its own pages are visible in the windows. + * pages. Only one user environment can be resident at a time, so while a + * group runs only its own pages are visible *in the windows*: the entries + * it uses are pointed at its pages and the rest are invalidated. + * + * That is not by itself isolation between groups. Every user page comes + * from the one page pool, which the kernel keeps permanently mapped at + * CONFIG_ARCH_PGPOOL_VBASE, and the external-memory permissions are + * indexed by physical address, so they cannot deny the unprivileged world + * that window without also denying it its own pages. Closing that is a + * separate piece of work on the kernel's side of the map. * * The remap is skipped when 'addrenv' is already the resident environment * (thread<->thread within a group, ISRs, syscalls), which pays nothing. @@ -355,17 +362,6 @@ int up_addrenv_select(const arch_addrenv_t *addrenv) cache_state = esp32s3_dcache_suspend(false); - /* TODO(Unit F hardening): only the pages this group actually uses are - * remapped below. Window entries beyond ntext/ndata/nheap still point at - * the previously-resident group's pages, so a buggy or malicious task that - * touches its window above its own allocation could reach stale mappings. - * A well-behaved task never does, and the guard-page/SIGSEGV abort - * (CONFIG_ESP32S3_PAGEFAULT_ABORT) is the backstop, but full isolation - * needs the unused window entries invalidated here. Deferred to on-target - * bring-up because invalidating cache-MMU entries has documented sharp - * edges (an invalid in-window entry reads 0 silently, it does not fault). - */ - /* Point the instruction-bus (.text) window at this group's pages */ for (i = 0; i < addrenv->ntext; i++) @@ -393,6 +389,23 @@ int up_addrenv_select(const arch_addrenv_t *addrenv) addrenv->heappages[i], 1); } + /* Take away what this group does not use. Only as many entries as the + * incoming group has pages were rewritten above; the rest of each window + * would otherwise still point at the pages of whoever was resident before, + * which a task that ran off the end of its own allocation could read. + * + * A group's page count only ever grows (the heap window, through + * pgalloc()), so this cannot invalidate an entry that is about to be + * needed again without a select in between. + */ + + esp32s3_mmu_unmap(ESP32S3_TEXT_VBASE + addrenv->ntext * MM_PGSIZE, + CONFIG_ARCH_TEXT_NPAGES - addrenv->ntext); + esp32s3_mmu_unmap(ESP32S3_DATA_VBASE + addrenv->ndata * MM_PGSIZE, + CONFIG_ARCH_DATA_NPAGES - addrenv->ndata); + esp32s3_mmu_unmap(ESP32S3_HEAP_VBASE + addrenv->nheap * MM_PGSIZE, + CONFIG_ARCH_HEAP_NPAGES - addrenv->nheap); + /* Drop stale instruction lines from the previous mapping and resume */ esp32s3_icache_invalidate_all(); diff --git a/arch/xtensa/src/esp32s3/esp32s3_addrenv.h b/arch/xtensa/src/esp32s3/esp32s3_addrenv.h index 5de0eab4649..1242d0de18a 100644 --- a/arch/xtensa/src/esp32s3/esp32s3_addrenv.h +++ b/arch/xtensa/src/esp32s3/esp32s3_addrenv.h @@ -42,10 +42,56 @@ * Pre-processor Definitions ****************************************************************************/ -#ifndef CONFIG_ARCH_PGPOOL_MAPPING -# error "ESP32-S3 address environments need CONFIG_ARCH_PGPOOL_MAPPING" +/* The page pool must NOT be statically mapped into the kernel. It is carved + * out of the same PSRAM the user processes run from, and the external memory + * permissions (APB_CTRL_SRAM_ACEn_*) are indexed by physical address, so a + * permanent kernel window onto the pool is a window onto every process's + * memory that no permission setting can close: a process's own pages are + * pool pages. This was measured, not assumed -- a user task read another + * process's .bss through it. The kernel uses the scratch region below + * instead. + */ + +#ifdef CONFIG_ARCH_PGPOOL_MAPPING +# error "ESP32-S3 must not map the page pool; see esp32s3_pgmap()" #endif +#ifndef CONFIG_ARCH_KMAP_VBASE +# error "ESP32-S3 address environments need CONFIG_ARCH_KMAP_VBASE" +#endif + +#if CONFIG_ARCH_KMAP_NPAGES < 2 +# error "CONFIG_ARCH_KMAP_NPAGES must be at least 2 (fork() copies " \ + "a source and a destination page at once)" +#endif + +/* CONFIG_MM_KMAP cannot work here. kmm_map()'s single-page path is + * up_addrenv_page_vaddr(), which asks for a kernel address that stays valid + * after the call returns -- exactly what a scratch mapping cannot promise. + */ + +#ifdef CONFIG_MM_KMAP +# error "CONFIG_MM_KMAP needs a permanently mapped page pool" +#endif + +/* The kernel's scratch region. ARCH_KMAP_VEND is only defined by + * when CONFIG_MM_KMAP is set, which it is not. + */ + +#define ESP32S3_KMAP_VBASE (CONFIG_ARCH_KMAP_VBASE) +#define ESP32S3_KMAP_NPAGES (CONFIG_ARCH_KMAP_NPAGES) +#define ESP32S3_KMAP_VEND (CONFIG_ARCH_KMAP_VBASE + \ + CONFIG_ARCH_KMAP_NPAGES * MM_PGSIZE) + +/* The page pool, described physically. mm_pgalloc() hands out physical page + * addresses; only the virtual mapping went away, not the pool. + */ + +#define ESP32S3_PGPOOL_PBASE (CONFIG_ARCH_PGPOOL_PBASE) +#define ESP32S3_PGPOOL_SIZE (CONFIG_ARCH_PGPOOL_SIZE) +#define ESP32S3_PGPOOL_PEND (CONFIG_ARCH_PGPOOL_PBASE + \ + CONFIG_ARCH_PGPOOL_SIZE) + /* The user address space is split across two disjoint cache-MMU windows: * .text lives in the instruction-bus window, .data/.bss and the heap in the * data-bus window. Each window is described by its base and page count. @@ -66,45 +112,16 @@ ****************************************************************************/ /**************************************************************************** - * Name: esp32s3_pgvaddr + * Name: esp32s3_pgpool_page * * Description: - * Get the kernel-addressable virtual address of a page-pool physical - * address. The page pool (PSRAM) is permanently mapped into the kernel - * (WORLD0) address space, so a page allocated by mm_pgalloc() can be - * reached directly through this fixed offset. Returns 0 if the physical - * address is not inside the page pool. + * Return true if paddr is a page-pool physical address. * ****************************************************************************/ -static inline uintptr_t esp32s3_pgvaddr(uintptr_t paddr) +static inline bool esp32s3_pgpool_page(uintptr_t paddr) { - if (paddr >= CONFIG_ARCH_PGPOOL_PBASE && paddr < CONFIG_ARCH_PGPOOL_PEND) - { - return paddr - CONFIG_ARCH_PGPOOL_PBASE + CONFIG_ARCH_PGPOOL_VBASE; - } - - return 0; -} - -/**************************************************************************** - * Name: esp32s3_pgpaddr - * - * Description: - * Inverse of esp32s3_pgvaddr(): translate a kernel page-pool virtual - * address back to its physical address. Returns 0 if the virtual address - * is not inside the mapped page pool. - * - ****************************************************************************/ - -static inline uintptr_t esp32s3_pgpaddr(uintptr_t vaddr) -{ - if (vaddr >= CONFIG_ARCH_PGPOOL_VBASE && vaddr < CONFIG_ARCH_PGPOOL_VEND) - { - return vaddr - CONFIG_ARCH_PGPOOL_VBASE + CONFIG_ARCH_PGPOOL_PBASE; - } - - return 0; + return (paddr >= ESP32S3_PGPOOL_PBASE && paddr < ESP32S3_PGPOOL_PEND); } /**************************************************************************** @@ -123,27 +140,86 @@ static inline bool esp32s3_uservaddr(uintptr_t vaddr) (vaddr >= ESP32S3_HEAP_VBASE && vaddr < ESP32S3_HEAP_VEND)); } +/**************************************************************************** + * Public Function Prototypes + ****************************************************************************/ + +/**************************************************************************** + * Name: esp32s3_pgmap + * + * Description: + * Map a page-pool physical page into the kernel's scratch region and + * return the virtual address it can be reached at. + * + * This replaces the arithmetic esp32s3_pgvaddr() the port used while the + * whole pool was mapped, and unlike it the result has a *lifetime*: it is + * valid until the matching esp32s3_pgunmap(), and not one instruction + * longer. It must not be stored anywhere that outlives the operation. + * + * The caller must hold sched_lock() across the whole map/use/unmap + * sequence. A scratch address is ordinary external memory as far as the + * permission control is concerned, so an unprivileged task that ran while + * the mapping was live could read the page through it -- which is the leak + * this whole arrangement exists to close. Interrupts do not need to be + * disabled: no interrupt handler reaches these paths. + * + * Input Parameters: + * paddr - Physical (page pool) address of the page, page-aligned. + * + * Returned Value: + * The kernel virtual address of the page, or 0 if 'paddr' is not a pool + * page or no scratch slot is free. + * + ****************************************************************************/ + +uintptr_t esp32s3_pgmap(uintptr_t paddr); + +/**************************************************************************** + * Name: esp32s3_pgunmap + * + * Description: + * Release a mapping made by esp32s3_pgmap(), writing back anything written + * through it and leaving the scratch entry invalid. + * + * Input Parameters: + * vaddr - The address esp32s3_pgmap() returned. + * + ****************************************************************************/ + +void esp32s3_pgunmap(uintptr_t vaddr); + /**************************************************************************** * Name: esp32s3_pgwipe * * Description: - * Zero a page-pool physical page through its kernel virtual mapping. + * Zero a page-pool physical page, mapping it into the kernel's scratch + * region for as long as that takes. + * + * Input Parameters: + * paddr - Physical (page pool) address of the page. * ****************************************************************************/ -static inline void esp32s3_pgwipe(uintptr_t paddr) -{ - uintptr_t vaddr = esp32s3_pgvaddr(paddr); - if (vaddr) - { - memset((void *)vaddr, 0, MM_PGSIZE); - } -} +void esp32s3_pgwipe(uintptr_t paddr); /**************************************************************************** - * Public Function Prototypes + * Name: esp32s3_pgpool_unmap + * + * Description: + * Tear down the boot-time cache-MMU mapping of the page pool, leaving the + * pool reachable only a page at a time through esp32s3_pgmap(). Called + * once, from up_allocate_pgheap(), after its consistency checks -- which + * have to run first, since they ask the cache MMU where the pool actually + * is rather than deriving it. + * + * Input Parameters: + * vbase - Virtual base the boot-time mapping put the pool at. + * size - Size of the pool in bytes. + * ****************************************************************************/ +void esp32s3_pgpool_unmap(uintptr_t vbase, size_t size); + /**************************************************************************** * Name: esp32s3_addrenv_mapnew * diff --git a/arch/xtensa/src/esp32s3/esp32s3_addrenv_utils.c b/arch/xtensa/src/esp32s3/esp32s3_addrenv_utils.c index 803252c1495..f473835933d 100644 --- a/arch/xtensa/src/esp32s3/esp32s3_addrenv_utils.c +++ b/arch/xtensa/src/esp32s3/esp32s3_addrenv_utils.c @@ -27,19 +27,175 @@ #include #include +#include +#include +#include #include #include +#include #include #include #include #include "esp32s3_addrenv.h" +#include "esp32s3_mmu.h" + +/**************************************************************************** + * Private Data + ****************************************************************************/ + +/* The kernel's scratch region: the only way it can reach a page-pool page, + * one page at a time and only while it is working on it. A slot holds the + * physical page it currently maps, or 0 when it is free. + * + * There is no lock here on purpose. Callers hold sched_lock() for the whole + * map/use/unmap sequence -- which they must anyway, so that a live mapping + * cannot be observed by an unprivileged task -- and that also makes this + * table single-threaded. No interrupt handler reaches these paths. + */ + +static uintptr_t g_scratch[ESP32S3_KMAP_NPAGES]; + +/**************************************************************************** + * Private Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: scratch_slot + * + * Description: + * Return the scratch slot index a scratch virtual address belongs to, or + * -1 if the address is not in the scratch region. + * + ****************************************************************************/ + +static int scratch_slot(uintptr_t vaddr) +{ + if (vaddr < ESP32S3_KMAP_VBASE || vaddr >= ESP32S3_KMAP_VEND) + { + return -1; + } + + return (int)((vaddr - ESP32S3_KMAP_VBASE) >> MM_PGSHIFT); +} /**************************************************************************** * Public Functions ****************************************************************************/ +/**************************************************************************** + * Name: esp32s3_pgmap + ****************************************************************************/ + +uintptr_t esp32s3_pgmap(uintptr_t paddr) +{ + uintptr_t vaddr; + int i; + + DEBUGASSERT(!up_interrupt_context()); + DEBUGASSERT(MM_ISALIGNED(paddr)); + + if (!esp32s3_pgpool_page(paddr)) + { + return 0; + } + + for (i = 0; i < ESP32S3_KMAP_NPAGES; i++) + { + if (g_scratch[i] == 0) + { + break; + } + } + + if (i >= ESP32S3_KMAP_NPAGES) + { + return 0; + } + + g_scratch[i] = paddr; + vaddr = ESP32S3_KMAP_VBASE + i * MM_PGSIZE; + + esp32s3_mmu_scratch_map(vaddr, paddr); + + return vaddr; +} + +/**************************************************************************** + * Name: esp32s3_pgunmap + ****************************************************************************/ + +void esp32s3_pgunmap(uintptr_t vaddr) +{ + int i = scratch_slot(vaddr); + + DEBUGASSERT(i >= 0 && g_scratch[i] != 0); + + esp32s3_mmu_scratch_unmap(ESP32S3_KMAP_VBASE + i * MM_PGSIZE); + + g_scratch[i] = 0; +} + +/**************************************************************************** + * Name: esp32s3_pgwipe + ****************************************************************************/ + +void esp32s3_pgwipe(uintptr_t paddr) +{ + uintptr_t vaddr; + + /* Hold off the scheduler for the whole sequence. Not for mutual exclusion + * against another wipe -- there is none to worry about here -- but because + * a scratch address is reachable by the unprivileged world like any other + * external memory address, so no user task may run while a pool page is + * parked at one. + */ + + sched_lock(); + + vaddr = esp32s3_pgmap(paddr); + if (vaddr == 0) + { + /* Not recoverable, and much too dangerous to let pass: an unwiped page + * carries its previous tenant's data into a new process. This is + * exactly how the CONFIG_ARCH_PGPOOL_PBASE drift stayed hidden. + */ + + _err("ERROR: no scratch mapping for page %08" PRIxPTR "\n", paddr); + PANIC(); + } + + memset((void *)vaddr, 0, MM_PGSIZE); + esp32s3_pgunmap(vaddr); + + sched_unlock(); +} + +/**************************************************************************** + * Name: esp32s3_pgpool_unmap + ****************************************************************************/ + +void esp32s3_pgpool_unmap(uintptr_t vbase, size_t size) +{ + uint32_t cache_state; + + /* Take away the boot-time mapping of the pool in one go. esp32s3_spiram.c + * mapped the whole PSRAM device; only the pool's share of that window is + * withdrawn, because the rest maps no page a process will ever own and is + * the aperture a kernel-side PSRAM allocation (a loaded library's text + * heap, say) would have to come from. + * + * Write back before invalidating the entries: this runs after the PSRAM + * memory test, which leaves the window's lines dirty. + */ + + cache_state = esp32s3_dcache_suspend(true); + esp32s3_mmu_unmap(vbase, size / MM_PGSIZE); + esp32s3_icache_invalidate_all(); + esp32s3_dcache_resume(cache_state); +} + /**************************************************************************** * Name: up_addrenv_find_page * @@ -102,14 +258,22 @@ uintptr_t up_addrenv_find_page(arch_addrenv_t *addrenv, uintptr_t vaddr) * * Description: * Get the kernel virtual address of a physical page allocated for an - * address environment. Since the PSRAM page pool is permanently mapped - * into the kernel, this is a fixed offset translation. + * address environment. + * + * The ESP32-S3 cannot answer this. The contract is a kernel address that + * stays valid after the call returns, and the page pool is deliberately + * not mapped: a pool page is reachable only for the duration of an + * esp32s3_pgmap()/esp32s3_pgunmap() pair. Returning an address that is + * about to stop meaning anything would be worse than refusing, so this + * fails, and CONFIG_MM_KMAP -- whose single-page path is built on this + * function -- is rejected at compile time in esp32s3_addrenv.h. * ****************************************************************************/ uintptr_t up_addrenv_page_vaddr(uintptr_t page) { - return esp32s3_pgvaddr(page); + UNUSED(page); + return 0; } /**************************************************************************** @@ -146,11 +310,35 @@ void up_addrenv_page_wipe(uintptr_t page) * Map a physical page-pool address to the kernel virtual address that * currently maps it. * + * Which is now literally what this does: only a page held in a scratch + * slot is mapped at all, so the answer comes from the slot table rather + * than from arithmetic over a window that no longer exists. A page nobody + * is working on has no kernel virtual address, and 0 says so. + * ****************************************************************************/ void *up_addrenv_pa_to_va(uintptr_t pa) { - return (void *)esp32s3_pgvaddr(pa); + uintptr_t page = MM_PGALIGNDOWN(pa); + int i; + + /* A free slot holds 0, so page 0 could otherwise match one of them */ + + if (!esp32s3_pgpool_page(page)) + { + return NULL; + } + + for (i = 0; i < ESP32S3_KMAP_NPAGES; i++) + { + if (g_scratch[i] == page) + { + return (void *)(ESP32S3_KMAP_VBASE + i * MM_PGSIZE + + (pa & MM_PGMASK)); + } + } + + return NULL; } /**************************************************************************** @@ -158,10 +346,20 @@ void *up_addrenv_pa_to_va(uintptr_t pa) * * Description: * Map a kernel page-pool virtual address back to its physical address. + * The inverse of the above, and just as narrow: scratch addresses are the + * only kernel virtual addresses that name a pool page. * ****************************************************************************/ uintptr_t up_addrenv_va_to_pa(void *va) { - return esp32s3_pgpaddr((uintptr_t)va); + uintptr_t vaddr = (uintptr_t)va; + int i = scratch_slot(vaddr); + + if (i < 0 || g_scratch[i] == 0) + { + return 0; + } + + return g_scratch[i] + (vaddr & MM_PGMASK); } diff --git a/arch/xtensa/src/esp32s3/esp32s3_mmu.c b/arch/xtensa/src/esp32s3/esp32s3_mmu.c index cf54eb0c55d..9e801d28249 100644 --- a/arch/xtensa/src/esp32s3/esp32s3_mmu.c +++ b/arch/xtensa/src/esp32s3/esp32s3_mmu.c @@ -27,11 +27,13 @@ #include #include +#include #include #include "xtensa.h" #include "esp_attr.h" +#include "soc/ext_mem_defs.h" #include "soc/extmem_reg.h" #include "esp32s3_mmu.h" @@ -51,6 +53,35 @@ extern int cache_dbus_mmu_set(uint32_t ext_ram, uint32_t vaddr, extern int cache_ibus_mmu_set(uint32_t ext_ram, uint32_t vaddr, uint32_t paddr, uint32_t psize, uint32_t num, uint32_t fixed); +extern int cache_invalidate_addr(uint32_t addr, uint32_t size); +extern int cache_writeback_addr(uint32_t addr, uint32_t size); + +/**************************************************************************** + * Private Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: dcache_unshut + * + * Description: + * Re-open the data cache bus that cache_suspend_dcache() shut. Taken from + * esp_spiram_map(): the range cache maintenance that follows a remap has + * to run with the bus open but the cache still suspended, so this is the + * half of esp32s3_dcache_resume() that comes before cache_resume_dcache(). + * + ****************************************************************************/ + +static void IRAM_ATTR dcache_unshut(void) +{ + uint32_t regval; + + regval = getreg32(EXTMEM_DCACHE_CTRL1_REG); + regval &= ~EXTMEM_DCACHE_SHUT_CORE0_BUS; +#ifdef CONFIG_SMP + regval &= ~EXTMEM_DCACHE_SHUT_CORE1_BUS; +#endif + putreg32(regval, EXTMEM_DCACHE_CTRL1_REG); +} /**************************************************************************** * Public Functions @@ -130,3 +161,108 @@ int IRAM_ATTR esp32s3_mmu_map_ibus(uint32_t ext_ram, uint32_t vaddr, { return cache_ibus_mmu_set(ext_ram, vaddr, paddr, 64, (int)npages, 0); } + +/**************************************************************************** + * Name: esp32s3_mmu_paddr + ****************************************************************************/ + +bool esp32s3_mmu_paddr(uint32_t vaddr, uint32_t *paddr) +{ + uint32_t entry = FLASH_MMU_TABLE[MMU_ENTRY_OF(vaddr)]; + + if ((entry & MMU_TABLE_INVALID_VAL) != 0) + { + return false; + } + + *paddr = (entry & MMU_ADDRESS_MASK) * MMU_PAGE_SIZE + + (vaddr & (MMU_PAGE_SIZE - 1)); + return true; +} + +/**************************************************************************** + * Name: esp32s3_mmu_unmap + ****************************************************************************/ + +void IRAM_ATTR esp32s3_mmu_unmap(uint32_t vaddr, uint32_t npages) +{ + uint32_t entry = MMU_ENTRY_OF(vaddr); + uint32_t i; + + /* One table, one entry per 64 KB, shared by the instruction and the data + * bus: the IBUS and DBUS linear addresses are asserted equal in + * ext_mem_defs.h, so a single write covers both views of the page. + */ + + for (i = 0; i < npages && entry + i < SOC_MMU_ENTRY_NUM; i++) + { + FLASH_MMU_TABLE[entry + i] = MMU_TABLE_INVALID_VAL; + } +} + +/**************************************************************************** + * Name: esp32s3_mmu_scratch_map + ****************************************************************************/ + +void esp32s3_mmu_scratch_map(uint32_t vaddr, uint32_t paddr) +{ + irqstate_t flags; + uint32_t cache_state; + + flags = enter_critical_section(); + + /* Suspend the cache, point the entry at the page, then invalidate just + * this page's lines rather than the whole cache. The difference matters: + * esp32s3_dcache_suspend() invalidates everything, which would discard the + * resident process's dirty lines, and its write-back variant would put a + * whole-cache write-back inside this critical section. Neither is + * acceptable at the rate this is called -- once per page of every process + * created. + */ + + cache_state = cache_suspend_dcache(); + + esp32s3_mmu_map_dbus(SOC_MMU_ACCESS_SPIRAM, vaddr, paddr, 1); + + dcache_unshut(); + cache_invalidate_addr(vaddr, MMU_PAGE_SIZE); + + cache_resume_dcache(cache_state); + + leave_critical_section(flags); +} + +/**************************************************************************** + * Name: esp32s3_mmu_scratch_unmap + ****************************************************************************/ + +void esp32s3_mmu_scratch_unmap(uint32_t vaddr) +{ + irqstate_t flags; + uint32_t cache_state; + + /* Push whatever was written through this window out to the page it maps, + * while it still maps it. The lines are tagged by virtual address, so + * repointing the entry with them still dirty would write them back to + * whichever page the scratch slot is used for next. + * + * This runs with interrupts enabled on purpose -- it is a write-back of up + * to a page, and the caller holds sched_lock(), so nothing else can reach + * the slot in the meantime. + */ + + cache_writeback_addr(vaddr, MMU_PAGE_SIZE); + + flags = enter_critical_section(); + + cache_state = cache_suspend_dcache(); + + esp32s3_mmu_unmap(vaddr, 1); + + dcache_unshut(); + cache_invalidate_addr(vaddr, MMU_PAGE_SIZE); + + cache_resume_dcache(cache_state); + + leave_critical_section(flags); +} diff --git a/arch/xtensa/src/esp32s3/esp32s3_mmu.h b/arch/xtensa/src/esp32s3/esp32s3_mmu.h index 6553c9e77cf..5dc2f1baf65 100644 --- a/arch/xtensa/src/esp32s3/esp32s3_mmu.h +++ b/arch/xtensa/src/esp32s3/esp32s3_mmu.h @@ -40,6 +40,15 @@ #define MMU_FLASH_MASK (~(MMU_PAGE_SIZE - 1)) +/* The cache MMU entry a virtual address resolves to. There is one table for + * both buses -- ext_mem_defs.h asserts that the IRAM0 and DRAM0 linear + * addresses are equal -- so an instruction-bus and a data-bus address 64 MB + * apart share an entry, and comparing entries is the only way to tell + * whether two windows collide. + */ + +#define MMU_ENTRY_OF(vaddr) (((vaddr) & SOC_MMU_VADDR_MASK) >> 16) + /**************************************************************************** * Public Function Prototypes ****************************************************************************/ @@ -153,4 +162,81 @@ int esp32s3_mmu_map_dbus(uint32_t ext_ram, uint32_t vaddr, uint32_t paddr, int esp32s3_mmu_map_ibus(uint32_t ext_ram, uint32_t vaddr, uint32_t paddr, uint32_t npages); +/**************************************************************************** + * Name: esp32s3_mmu_paddr + * + * Description: + * Ask the cache MMU what a virtual address currently resolves to. This is + * ground truth rather than arithmetic, which matters because where the + * kernel's PSRAM window lands is decided at run time (see + * up_allocate_pgheap()). + * + * Input Parameters: + * vaddr - A virtual address inside one of the external memory windows. + * paddr - Receives the physical/flash address it maps to. + * + * Returned Value: + * True if the entry is valid and *paddr was written; false if the entry + * maps nothing. + * + ****************************************************************************/ + +bool esp32s3_mmu_paddr(uint32_t vaddr, uint32_t *paddr); + +/**************************************************************************** + * Name: esp32s3_mmu_unmap + * + * Description: + * Invalidate a range of 64 KB cache MMU entries, so that the virtual + * addresses they covered map nothing at all. Note that an access to an + * invalidated entry is not necessarily a fault: unless the cache reject + * monitors are armed it reads back as zero and swallows writes. The + * caller is responsible for suspending/resuming the data cache around the + * change and for invalidating the instruction cache afterwards -- entries + * being taken away is exactly when stale instruction lines are left + * behind. + * + * Input Parameters: + * vaddr - 64 KB-aligned virtual base address. + * npages - Number of 64 KB pages to invalidate. + * + ****************************************************************************/ + +void esp32s3_mmu_unmap(uint32_t vaddr, uint32_t npages); + +/**************************************************************************** + * Name: esp32s3_mmu_scratch_map + * + * Description: + * Point one 64 KB data-bus entry at a PSRAM page and make it usable, doing + * the cache maintenance itself and confining it to that page. This is the + * kernel's way of reaching a page-pool page, which is otherwise not mapped + * at all (see esp32s3_pgmap()). + * + * The caller must already hold the scratch slot -- these entries are + * reachable by the unprivileged world like any other external memory + * address, so a mapping must not outlive the operation that needs it. + * + * Input Parameters: + * vaddr - 64 KB-aligned scratch virtual address. + * paddr - 64 KB-aligned PSRAM physical address. + * + ****************************************************************************/ + +void esp32s3_mmu_scratch_map(uint32_t vaddr, uint32_t paddr); + +/**************************************************************************** + * Name: esp32s3_mmu_scratch_unmap + * + * Description: + * Take a scratch mapping away again: write back what was written through + * it, invalidate the entry, and drop the page's cache lines. + * + * Input Parameters: + * vaddr - The scratch virtual address returned when it was mapped. + * + ****************************************************************************/ + +void esp32s3_mmu_scratch_unmap(uint32_t vaddr); + #endif /* __ARCH_XTENSA_SRC_ESP32S3_ESP32S3_MMU_H */ diff --git a/arch/xtensa/src/esp32s3/esp32s3_pgalloc.c b/arch/xtensa/src/esp32s3/esp32s3_pgalloc.c index e3a94f53abb..b52d47ba0e4 100644 --- a/arch/xtensa/src/esp32s3/esp32s3_pgalloc.c +++ b/arch/xtensa/src/esp32s3/esp32s3_pgalloc.c @@ -29,20 +29,146 @@ #include #include #include +#include #include #include +#include #include #include #include "sched/sched.h" #include "esp32s3_addrenv.h" +#include "esp32s3_mmu.h" #ifdef CONFIG_ESP32S3_SPIRAM # include "esp32s3_spiram.h" #endif +#if defined(CONFIG_ESP32S3_SPIRAM) && defined(CONFIG_ARCH_ADDRENV) + +/* The page pool and a PSRAM kernel heap cannot coexist. xtensa_add_region() + * hands the *whole* allocable PSRAM window to the kernel heap, which + * necessarily includes the pool -- and a kernel heap that contains pool + * pages is a permanently mapped view of every process's memory, arriving + * from the other side of the same problem esp32s3_pgmap() exists to solve. + * + * This is not hypothetical: up_textheap_memalign() falls back to the kernel + * heap and derives an instruction-bus alias for anything it finds outside + * internal RAM, which is the path a dlopen()ed shared library would take. + * Kernel-side PSRAM has to come from outside the pool. + */ + +#ifdef CONFIG_ESP32S3_SPIRAM_COMMON_HEAP +# error "the page pool cannot be shared with a PSRAM kernel heap" +#endif + +/**************************************************************************** + * Private Types + ****************************************************************************/ + +struct window_s +{ + FAR const char *name; + uintptr_t vbase; + uint32_t npages; +}; + +/**************************************************************************** + * Private Data + ****************************************************************************/ + +/* Every cache-MMU window this configuration uses, checked against each other + * and against the kernel's PSRAM window at boot. The scratch region is one + * of them: it is a real window now, and the heap window growing into it + * would be as silent as every other mapping mistake in this port. + */ + +static const struct window_s g_windows[] = +{ + { + .name = "text", + .vbase = ESP32S3_TEXT_VBASE, + .npages = CONFIG_ARCH_TEXT_NPAGES + }, + { + .name = "data", + .vbase = ESP32S3_DATA_VBASE, + .npages = CONFIG_ARCH_DATA_NPAGES + }, + { + .name = "heap", + .vbase = ESP32S3_HEAP_VBASE, + .npages = CONFIG_ARCH_HEAP_NPAGES + }, + { + .name = "scratch", + .vbase = ESP32S3_KMAP_VBASE, + .npages = ESP32S3_KMAP_NPAGES + } +}; + +/**************************************************************************** + * Private Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: check_windows_clear + * + * Description: + * Panic if any two cache-MMU windows overlap, or if one of them overlaps + * the kernel's PSRAM window. They would otherwise do it silently: the + * instruction and the data bus share one entry per 64 KB, so two windows + * 64 MB apart are the same entries, and mapping one over another simply + * takes the first one's pages away. + * + * The comparison is by entry, since that is what actually collides, and it + * belongs at run time because the kernel PSRAM window starts wherever the + * flash mappings end and so moves as the image grows. + * + * Input Parameters: + * kfirst - First cache-MMU entry of the kernel's PSRAM window. + * klast - Last cache-MMU entry of the kernel's PSRAM window. + * + ****************************************************************************/ + +static void check_windows_clear(uint32_t kfirst, uint32_t klast) +{ + size_t i; + size_t j; + + for (i = 0; i < nitems(g_windows); i++) + { + uint32_t first = MMU_ENTRY_OF(g_windows[i].vbase); + uint32_t last = first + g_windows[i].npages - 1; + + if (first <= klast && last >= kfirst) + { + _err("ERROR: %s window (MMU entries %" PRIu32 "-%" PRIu32 ") " + "overlaps the kernel PSRAM window (entries %" PRIu32 "-%" + PRIu32 ")\n", g_windows[i].name, first, last, kfirst, klast); + PANIC(); + } + + for (j = 0; j < i; j++) + { + uint32_t ofirst = MMU_ENTRY_OF(g_windows[j].vbase); + uint32_t olast = ofirst + g_windows[j].npages - 1; + + if (first <= olast && last >= ofirst) + { + _err("ERROR: %s window (MMU entries %" PRIu32 "-%" PRIu32 ") " + "overlaps the %s window (entries %" PRIu32 "-%" PRIu32 + ")\n", g_windows[i].name, first, last, g_windows[j].name, + ofirst, olast); + PANIC(); + } + } + } +} +#endif + /**************************************************************************** * Public Functions ****************************************************************************/ @@ -59,10 +185,14 @@ * On the ESP32-S3 the page pool is a slice of the external octal PSRAM. * The pool is described by its *physical* base -- for the cache MMU that * is the zero-based offset into the PSRAM device -- because mm_pgalloc() - * hands out physical page addresses. The whole pool is also permanently - * mapped into the kernel (WORLD0) data-bus window at - * CONFIG_ARCH_PGPOOL_VBASE so the kernel can reach any page it allocates - * (see esp32s3_pgvaddr()). + * hands out physical page addresses. + * + * It is deliberately not mapped into the kernel. esp32s3_spiram.c maps + * the whole PSRAM device at boot, and this function is where the pool's + * share of that mapping is withdrawn again, after the checks that need it. + * The kernel reaches a pool page one at a time through esp32s3_pgmap(); + * see esp32s3_addrenv.h for why a permanent window cannot be allowed to + * stand. * * Input Parameters: * heap_start - Receives the physical base address of the page pool. @@ -74,40 +204,62 @@ void up_allocate_pgheap(void **heap_start, size_t *heap_size) { DEBUGASSERT(heap_start && heap_size); -#ifdef CONFIG_ESP32S3_SPIRAM +#if defined(CONFIG_ESP32S3_SPIRAM) && defined(CONFIG_ARCH_ADDRENV) /* Where the kernel's PSRAM window lands is decided at run time: * esp32s3_spiram.c maps PSRAM immediately after the last cache-MMU entry * the flash mappings occupy, so it moves as the kernel image grows. The - * page pool is described to the OS by compile-time constants, so the two - * have to be checked against each other -- and loudly, because getting it - * wrong is otherwise silent: an unmapped cache window swallows writes and - * reads back as zero without faulting, so a misplaced pool would simply - * lose every page handed out of it. + * pool is described by a compile-time *physical* base, so ask the cache + * MMU where that physical page currently is rather than deriving it from + * a constant -- which is how CONFIG_ARCH_PGPOOL_PBASE went stale twice, + * and the second time by exactly one page, which left one unwiped page in + * every region of every new process. */ - { - uintptr_t ramstart = (uintptr_t)esp_spiram_allocable_vaddr_start(); - uintptr_t ramend = (uintptr_t)esp_spiram_allocable_vaddr_end(); + { + uintptr_t ramstart = (uintptr_t)esp_spiram_allocable_vaddr_start(); + uintptr_t ramend = (uintptr_t)esp_spiram_allocable_vaddr_end(); + uintptr_t poolvbase; + uint32_t rampbase; - _info("PSRAM window %08" PRIxPTR "-%08" PRIxPTR ", " - "page pool %08x-%08x\n", - ramstart, ramend, - CONFIG_ARCH_PGPOOL_VBASE, CONFIG_ARCH_PGPOOL_VEND); + if (!esp32s3_mmu_paddr(ramstart, &rampbase)) + { + _err("ERROR: PSRAM window base %08" PRIxPTR " maps nothing\n", + ramstart); + PANIC(); + } - if ((uintptr_t)CONFIG_ARCH_PGPOOL_VBASE < ramstart || - (uintptr_t)CONFIG_ARCH_PGPOOL_VEND > ramend) - { - _err("ERROR: page pool %08x-%08x is outside the mapped PSRAM " - "window %08" PRIxPTR "-%08" PRIxPTR "\n", - CONFIG_ARCH_PGPOOL_VBASE, CONFIG_ARCH_PGPOOL_VEND, - ramstart, ramend); - PANIC(); - } - } + if (ESP32S3_PGPOOL_PBASE < rampbase || + ESP32S3_PGPOOL_PEND > rampbase + (ramend - ramstart)) + { + _err("ERROR: page pool %08x-%08x is outside the mapped PSRAM " + "%08" PRIx32 "-%08" PRIxPTR "\n", + ESP32S3_PGPOOL_PBASE, ESP32S3_PGPOOL_PEND, + rampbase, rampbase + (ramend - ramstart)); + PANIC(); + } + + poolvbase = ramstart + (ESP32S3_PGPOOL_PBASE - rampbase); + + _info("PSRAM window %08" PRIxPTR "-%08" PRIxPTR " (phys %08" PRIx32 + "), page pool phys %08x-%08x at %08" PRIxPTR "\n", + ramstart, ramend, rampbase, + ESP32S3_PGPOOL_PBASE, ESP32S3_PGPOOL_PEND, poolvbase); + + check_windows_clear(MMU_ENTRY_OF(ramstart), MMU_ENTRY_OF(ramend - 1)); + + /* Everything above has been established while the pool was still + * mapped, which is the only time it can be. Now take that mapping + * away: from here the kernel reaches a pool page only through + * esp32s3_pgmap(), and an unprivileged task reaches one only if it is + * its own. + */ + + esp32s3_pgpool_unmap(poolvbase, ESP32S3_PGPOOL_SIZE); + } #endif - *heap_start = (void *)CONFIG_ARCH_PGPOOL_PBASE; - *heap_size = (size_t)CONFIG_ARCH_PGPOOL_SIZE; + *heap_start = (void *)ESP32S3_PGPOOL_PBASE; + *heap_size = (size_t)ESP32S3_PGPOOL_SIZE; } #ifdef CONFIG_BUILD_KERNEL diff --git a/boards/xtensa/esp32s3/common/scripts/esp32s3_sections.ld b/boards/xtensa/esp32s3/common/scripts/esp32s3_sections.ld index 8fc6d5a484d..5fa66fd9f37 100644 --- a/boards/xtensa/esp32s3/common/scripts/esp32s3_sections.ld +++ b/boards/xtensa/esp32s3/common/scripts/esp32s3_sections.ld @@ -355,7 +355,25 @@ SECTIONS _iram_text = ABSOLUTE(.); } >iram0_0_seg - /* Marks the end of IRAM code segment */ + #ifdef CONFIG_BUILD_KERNEL + /* The vector table the World Controller gives to WORLD1, the unprivileged + * world. It needs the 1 KB alignment a vector base requires, and it is + * kept in a region of its own so that the permission control can make it + * the only instruction memory a user task may fetch. + * + * It has to land in Internal SRAM1, past 0x40378000: the permission control + * splits SRAM0 into two 16 KB blocks and can say nothing finer, while SRAM1 + * is divided by split lines at 256-byte granularity. Following the IRAM + * code puts it there; esp32s3_isolation_permissions() checks that it did. + */ + + .world1.vectors : ALIGN(1024) + { + KEEP (*(.world1_vectors.text)); + } >iram0_0_seg AT>ROM +#endif + + /* Marks the end of IRAM code segment */ .iram0.text_end (NOLOAD) : { diff --git a/boards/xtensa/esp32s3/esp32s3-devkit/configs/kernel_oct/defconfig b/boards/xtensa/esp32s3/esp32s3-devkit/configs/kernel_oct/defconfig index 0e494c08f5c..e7b65090a03 100644 --- a/boards/xtensa/esp32s3/esp32s3-devkit/configs/kernel_oct/defconfig +++ b/boards/xtensa/esp32s3/esp32s3-devkit/configs/kernel_oct/defconfig @@ -23,15 +23,15 @@ CONFIG_ARCH_HEAP_VBASE=0x3d200000 CONFIG_ARCH_INTERRUPTSTACK=2048 CONFIG_ARCH_IRQ_TO_NDX=y CONFIG_ARCH_KERNEL_STACKSIZE=8192 +CONFIG_ARCH_KMAP_NPAGES=2 +CONFIG_ARCH_KMAP_VBASE=0x3d400000 CONFIG_ARCH_MINIMAL_VECTORTABLE_DYNAMIC=y CONFIG_ARCH_NUSER_INTERRUPTS=2 -CONFIG_ARCH_PGPOOL_MAPPING=y -CONFIG_ARCH_PGPOOL_PBASE=0x360000 +CONFIG_ARCH_PGPOOL_PBASE=0x350000 CONFIG_ARCH_PGPOOL_SIZE=4194304 -CONFIG_ARCH_PGPOOL_VBASE=0x3c400000 CONFIG_ARCH_STACKDUMP=y CONFIG_ARCH_TEXT_NPAGES=8 -CONFIG_ARCH_TEXT_VBASE=0x42800000 +CONFIG_ARCH_TEXT_VBASE=0x42c00000 CONFIG_ARCH_USE_MMU=y CONFIG_ARCH_XTENSA=y CONFIG_BINFMT_ELF_EXECUTABLE=y @@ -51,6 +51,7 @@ CONFIG_DEFAULT_TASK_STACKSIZE=8192 CONFIG_ELF=y CONFIG_ESP32S3_FLASH_MODE_OCT=y CONFIG_ESP32S3_FLASH_SAMPLE_MODE_STR=y +CONFIG_ESP32S3_PAGEFAULT=y CONFIG_ESP32S3_SPIFLASH=y CONFIG_ESP32S3_SPIRAM=y CONFIG_ESP32S3_SPIRAM_MODE_OCT=y