mirror of
https://github.com/apache/nuttx.git
synced 2026-10-09 15:25:24 +00:00
530f5cd324
Add dcache and icache support for intel64 Signed-off-by: p-szafonimateusz <p-szafonimateusz@xiaomi.com>
591 lines
14 KiB
C
591 lines
14 KiB
C
/****************************************************************************
|
|
* arch/x86_64/src/intel64/intel64_cache.c
|
|
*
|
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
|
* contributor license agreements. See the NOTICE file distributed with
|
|
* this work for additional information regarding copyright ownership. The
|
|
* ASF licenses this file to you under the Apache License, Version 2.0 (the
|
|
* "License"); you may not use this file except in compliance with the
|
|
* License. You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
* License for the specific language governing permissions and limitations
|
|
* under the License.
|
|
*
|
|
****************************************************************************/
|
|
|
|
/****************************************************************************
|
|
* Included Files
|
|
****************************************************************************/
|
|
|
|
#include <nuttx/config.h>
|
|
#include <nuttx/cache.h>
|
|
|
|
#include <stdint.h>
|
|
|
|
#include "x86_64_internal.h"
|
|
|
|
/****************************************************************************
|
|
* Pre-processor Definitions
|
|
****************************************************************************/
|
|
|
|
/****************************************************************************
|
|
* Inline Functions
|
|
****************************************************************************/
|
|
|
|
/****************************************************************************
|
|
* Name: x86_64_wbindv
|
|
****************************************************************************/
|
|
|
|
static inline void x86_64_wbindv(void)
|
|
{
|
|
asm volatile("wbinvd" : : : "memory");
|
|
}
|
|
|
|
/****************************************************************************
|
|
* Name: x86_64_wbnoinvd
|
|
****************************************************************************/
|
|
|
|
static inline void x86_64_wbnoinvd(void)
|
|
{
|
|
asm volatile("wbnoinvd" : : : "memory");
|
|
}
|
|
|
|
/****************************************************************************
|
|
* Name: x86_64_invd
|
|
****************************************************************************/
|
|
|
|
static inline void x86_64_invd(void)
|
|
{
|
|
asm volatile("invd" : : : "memory");
|
|
}
|
|
|
|
/****************************************************************************
|
|
* Private Functions
|
|
****************************************************************************/
|
|
|
|
/****************************************************************************
|
|
* Name: x86_64_cache_linesize
|
|
****************************************************************************/
|
|
|
|
static size_t x86_64_cache_linesize(void)
|
|
{
|
|
#if CONFIG_ARCH_INTEL64_CACHE_LINESIZE == 0
|
|
unsigned long eax = 0;
|
|
unsigned long ebx = 0;
|
|
|
|
eax = 1;
|
|
asm volatile("cpuid\n\t"
|
|
: "=b" (ebx)
|
|
: "a" (eax));
|
|
|
|
return ((ebx >> 8) & 0xff) * 8;
|
|
#else
|
|
return CONFIG_ARCH_INTEL64_CACHE_LINESIZE;
|
|
#endif
|
|
}
|
|
|
|
/****************************************************************************
|
|
* Name: x86_64_cache_size
|
|
****************************************************************************/
|
|
|
|
static size_t x86_64_cache_size(int leaf)
|
|
{
|
|
unsigned long eax;
|
|
unsigned long ebx;
|
|
unsigned long ecx;
|
|
unsigned long edx;
|
|
|
|
/* The leaf 0 is Data cache */
|
|
|
|
eax = 4;
|
|
ecx = leaf;
|
|
asm volatile("cpuid"
|
|
: "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx)
|
|
: "a"(eax), "c"(ecx));
|
|
|
|
/* (Ways + 1) * (Partitions + 1) * (Line_Size + 1) * (Sets + 1) */
|
|
|
|
return ((((ebx >> 22) & 0x3ff) + 1) *
|
|
(((ebx >> 12) & 0x3ff) + 1) *
|
|
((ebx & 0x7ff) + 1) *
|
|
(ecx + 1));
|
|
}
|
|
|
|
/****************************************************************************
|
|
* Name: x86_64_cache_enable
|
|
****************************************************************************/
|
|
|
|
static void x86_64_cache_enable(void)
|
|
{
|
|
/* Clear "Not-write through" (NW) and "Cache disable" (CD) bits */
|
|
|
|
asm volatile("\t mov %%cr0, %%rax\n"
|
|
"\t mov $0x9fffffff, %%rbx\n"
|
|
"\t and %%rbx, %%rax\n"
|
|
"\t mov %%rax, %%cr0\n"
|
|
::: "memory", "rax", "rbx");
|
|
}
|
|
|
|
/****************************************************************************
|
|
* Name: x86_64_cache_disable
|
|
****************************************************************************/
|
|
|
|
static void x86_64_cache_disable(void)
|
|
{
|
|
/* Set "Not-write through" (NW) and "Cache disable" (CD) bits */
|
|
|
|
asm volatile("\t mov %%cr0, %%rax\n"
|
|
"\t mov $0x9fffffff, %%rbx \n"
|
|
"\t and %%rbx, %%rax \n"
|
|
"\t mov $0x60000000, %%rbx\n"
|
|
"\t or %%rbx, %%rax\n"
|
|
"\t mov %%rax, %%cr0\n"
|
|
:::"memory", "rax", "rbx");
|
|
|
|
/* And flush all caches */
|
|
|
|
up_flush_dcache_all();
|
|
|
|
/* Disable MTRR */
|
|
|
|
write_msr(MSR_MTRR_DEF_TYPE, 0);
|
|
|
|
/* And flush once again */
|
|
|
|
up_flush_dcache_all();
|
|
}
|
|
|
|
/****************************************************************************
|
|
* Public Functions
|
|
****************************************************************************/
|
|
|
|
/****************************************************************************
|
|
* Name: up_get_icache_linesize
|
|
*
|
|
* Description:
|
|
* Get icache linesize
|
|
*
|
|
* Input Parameters:
|
|
* None
|
|
*
|
|
* Returned Value:
|
|
* Cache line size
|
|
*
|
|
****************************************************************************/
|
|
|
|
#ifdef CONFIG_ARCH_ICACHE
|
|
size_t up_get_icache_linesize(void)
|
|
{
|
|
return x86_64_cache_linesize();
|
|
}
|
|
|
|
/****************************************************************************
|
|
* Name: up_get_icache_size
|
|
*
|
|
* Description:
|
|
* Get icache size
|
|
*
|
|
* Input Parameters:
|
|
* None
|
|
*
|
|
* Returned Value:
|
|
* Cache size
|
|
*
|
|
****************************************************************************/
|
|
|
|
size_t up_get_icache_size(void)
|
|
{
|
|
#if CONFIG_ARCH_INTEL64_ICACHE_SIZE == 0
|
|
return x86_64_cache_size(1);
|
|
#else
|
|
return CONFIG_ARCH_INTEL64_ICACHE_SIZE;
|
|
#endif
|
|
}
|
|
#endif
|
|
|
|
/****************************************************************************
|
|
* Name: up_enable_icache
|
|
*
|
|
* Description:
|
|
* Enable the I-Cache
|
|
*
|
|
* Input Parameters:
|
|
* None
|
|
*
|
|
* Returned Value:
|
|
* None
|
|
*
|
|
****************************************************************************/
|
|
|
|
#ifdef CONFIG_ARCH_ICACHE
|
|
void up_enable_icache(void)
|
|
{
|
|
x86_64_cache_enable();
|
|
}
|
|
#endif
|
|
|
|
/****************************************************************************
|
|
* Name: up_disable_icache
|
|
*
|
|
* Description:
|
|
* Disable the I-Cache
|
|
*
|
|
* Input Parameters:
|
|
* None
|
|
*
|
|
* Returned Value:
|
|
* None
|
|
*
|
|
****************************************************************************/
|
|
|
|
#ifdef CONFIG_ARCH_ICACHE
|
|
void up_disable_icache(void)
|
|
{
|
|
x86_64_cache_disable();
|
|
}
|
|
#endif
|
|
|
|
/****************************************************************************
|
|
* Name: up_invalidate_icache
|
|
*
|
|
* Description:
|
|
* Invalidate the instruction cache within the specified region.
|
|
*
|
|
* Input Parameters:
|
|
* start - virtual start address of region
|
|
* end - virtual end address of region + 1
|
|
*
|
|
* Returned Value:
|
|
* None
|
|
*
|
|
****************************************************************************/
|
|
|
|
#ifdef CONFIG_ARCH_ICACHE
|
|
void up_invalidate_icache(uintptr_t start, uintptr_t end)
|
|
{
|
|
/* NOTE: x86 doesn't have separate instructions for I-cache */
|
|
|
|
up_invalidate_dcache(start, end);
|
|
}
|
|
#endif /* CONFIG_ARCH_ICACHE */
|
|
|
|
/****************************************************************************
|
|
* Name: up_invalidate_icache_all
|
|
*
|
|
* Description:
|
|
* Invalidate the entire contents of I cache.
|
|
*
|
|
* Input Parameters:
|
|
* None
|
|
*
|
|
* Returned Value:
|
|
* None
|
|
*
|
|
****************************************************************************/
|
|
|
|
#ifdef CONFIG_ARCH_ICACHE
|
|
void up_invalidate_icache_all(void)
|
|
{
|
|
/* NOTE: x86 doesn't have separate instructions for I-cache */
|
|
|
|
up_invalidate_dcache_all();
|
|
}
|
|
#endif
|
|
|
|
/****************************************************************************
|
|
* Name: up_get_dcache_linesize
|
|
*
|
|
* Description:
|
|
* Get dcache linesize
|
|
*
|
|
* Input Parameters:
|
|
* None
|
|
*
|
|
* Returned Value:
|
|
* Cache line size
|
|
*
|
|
****************************************************************************/
|
|
|
|
#ifdef CONFIG_ARCH_DCACHE
|
|
size_t up_get_dcache_linesize(void)
|
|
{
|
|
return x86_64_cache_linesize();
|
|
}
|
|
|
|
/****************************************************************************
|
|
* Name: up_get_dcache_size
|
|
*
|
|
* Description:
|
|
* Get icache size
|
|
*
|
|
* Input Parameters:
|
|
* None
|
|
*
|
|
* Returned Value:
|
|
* Cache size
|
|
*
|
|
****************************************************************************/
|
|
|
|
size_t up_get_dcache_size(void)
|
|
{
|
|
#if CONFIG_ARCH_INTEL64_DCACHE_SIZE == 0
|
|
return x86_64_cache_size(0);
|
|
#else
|
|
return CONFIG_ARCH_INTEL64_DCACHE_SIZE;
|
|
#endif
|
|
}
|
|
#endif
|
|
|
|
/****************************************************************************
|
|
* Name: up_enable_dcache
|
|
*
|
|
* Description:
|
|
* Enable the D-Cache
|
|
*
|
|
* Input Parameters:
|
|
* None
|
|
*
|
|
* Returned Value:
|
|
* None
|
|
*
|
|
****************************************************************************/
|
|
|
|
#ifdef CONFIG_ARCH_DCACHE
|
|
void up_enable_dcache(void)
|
|
{
|
|
x86_64_cache_enable();
|
|
}
|
|
#endif /* CONFIG_ARCH_DCACHE */
|
|
|
|
/****************************************************************************
|
|
* Name: up_disable_dcache
|
|
*
|
|
* Description:
|
|
* Disable the D-Cache
|
|
*
|
|
* Input Parameters:
|
|
* None
|
|
*
|
|
* Returned Value:
|
|
* None
|
|
*
|
|
****************************************************************************/
|
|
|
|
#ifdef CONFIG_ARCH_DCACHE
|
|
void up_disable_dcache(void)
|
|
{
|
|
x86_64_cache_disable();
|
|
}
|
|
#endif /* CONFIG_ARCH_DCACHE */
|
|
|
|
/****************************************************************************
|
|
* Name: up_invalidae_dcache
|
|
*
|
|
* Description:
|
|
* Invalidate the data cache within the specified region; we will be
|
|
* performing a DMA operation in this region and we want to purge old data
|
|
* in the cache.
|
|
*
|
|
* Input Parameters:
|
|
* start - virtual start address of region
|
|
* end - virtual end address of region + 1
|
|
*
|
|
* Returned Value:
|
|
* None
|
|
*
|
|
****************************************************************************/
|
|
|
|
#ifdef CONFIG_ARCH_DCACHE
|
|
void up_invalidate_dcache(uintptr_t start, uintptr_t end)
|
|
{
|
|
up_flush_dcache(start, end);
|
|
}
|
|
#endif /* CONFIG_ARCH_DCACHE */
|
|
|
|
/****************************************************************************
|
|
* Name: up_invalidate_dcache_all
|
|
*
|
|
* Description:
|
|
* Invalidate the entire contents of D cache.
|
|
*
|
|
* Input Parameters:
|
|
* None
|
|
*
|
|
* Returned Value:
|
|
* None
|
|
*
|
|
****************************************************************************/
|
|
|
|
#ifdef CONFIG_ARCH_DCACHE
|
|
void up_invalidate_dcache_all(void)
|
|
{
|
|
x86_64_invd();
|
|
}
|
|
#endif /* CONFIG_ARCH_DCACHE */
|
|
|
|
/****************************************************************************
|
|
* Name: up_clean_dcache
|
|
*
|
|
* Description:
|
|
* Clean the data cache within the specified region by flushing the
|
|
* contents of the data cache to memory.
|
|
*
|
|
* Input Parameters:
|
|
* start - virtual start address of region
|
|
* end - virtual end address of region + 1
|
|
*
|
|
* Returned Value:
|
|
* None
|
|
*
|
|
****************************************************************************/
|
|
|
|
#ifdef CONFIG_ARCH_DCACHE
|
|
void up_clean_dcache(uintptr_t start, uintptr_t end)
|
|
{
|
|
#ifdef CONFIG_ARCH_INTEL64_HAVE_CLWB
|
|
size_t lsize = up_get_dcache_linesize();
|
|
|
|
start &= ~(lsize - 1);
|
|
|
|
asm volatile("mfence" : : : "memory");
|
|
|
|
do
|
|
{
|
|
asm volatile("\tclwb %0;\n" : "+m" (start));
|
|
|
|
/* Increment the address by the size of one cache line. */
|
|
|
|
start += lsize;
|
|
}
|
|
while (start < end);
|
|
|
|
asm volatile("mfence" : : : "memory");
|
|
#else
|
|
x86_64_wbnoinvd();
|
|
#endif
|
|
}
|
|
#endif /* CONFIG_ARCH_DCACHE */
|
|
|
|
/****************************************************************************
|
|
* Name: up_clean_dcache_all
|
|
*
|
|
* Description:
|
|
* Clean the entire data cache within the specified region by flushing the
|
|
* contents of the data cache to memory.
|
|
*
|
|
* Input Parameters:
|
|
* None
|
|
*
|
|
* Returned Value:
|
|
* None
|
|
*
|
|
****************************************************************************/
|
|
|
|
#ifdef CONFIG_ARCH_DCACHE
|
|
void up_clean_dcache_all(void)
|
|
{
|
|
x86_64_wbnoinvd();
|
|
}
|
|
#endif /* CONFIG_ARCH_DCACHE */
|
|
|
|
/****************************************************************************
|
|
* Name: up_flush_dcache
|
|
*
|
|
* Description:
|
|
* Flush the data cache within the specified region by cleaning and
|
|
* invalidating the D cache.
|
|
*
|
|
* Input Parameters:
|
|
* start - virtual start address of region
|
|
* end - virtual end address of region + 1
|
|
*
|
|
* Returned Value:
|
|
* None
|
|
*
|
|
****************************************************************************/
|
|
|
|
#ifdef CONFIG_ARCH_DCACHE
|
|
void up_flush_dcache(uintptr_t start, uintptr_t end)
|
|
{
|
|
size_t lsize = up_get_dcache_linesize();
|
|
|
|
start &= ~(lsize - 1);
|
|
|
|
asm volatile("mfence" : : : "memory");
|
|
|
|
do
|
|
{
|
|
asm volatile("\tclflush %0;\n" : "+m" (start));
|
|
|
|
/* Increment the address by the size of one cache line. */
|
|
|
|
start += lsize;
|
|
}
|
|
while (start < end);
|
|
|
|
asm volatile("mfence" : : : "memory");
|
|
}
|
|
#endif /* CONFIG_ARCH_DCACHE */
|
|
|
|
/****************************************************************************
|
|
* Name: up_flush_dcache_all
|
|
*
|
|
* Description:
|
|
* Flush the entire data cache by cleaning and invalidating the D cache.
|
|
*
|
|
* Input Parameters:
|
|
* None
|
|
*
|
|
* Returned Value:
|
|
* None
|
|
*
|
|
****************************************************************************/
|
|
|
|
#ifdef CONFIG_ARCH_DCACHE
|
|
void up_flush_dcache_all(void)
|
|
{
|
|
x86_64_wbindv();
|
|
}
|
|
#endif /* CONFIG_ARCH_DCACHE */
|
|
|
|
/****************************************************************************
|
|
* Name: up_coherent_dcache
|
|
*
|
|
* Description:
|
|
* Ensure that the I and D caches are coherent within specified region
|
|
* by cleaning the D cache (i.e., flushing the D cache contents to memory
|
|
* and invalidating the I cache. This is typically used when code has been
|
|
* written to a memory region, and will be executed.
|
|
*
|
|
* Input Parameters:
|
|
* addr - virtual start address of region
|
|
* len - Size of the address region in bytes
|
|
*
|
|
* Returned Value:
|
|
* None
|
|
*
|
|
****************************************************************************/
|
|
|
|
#if defined(CONFIG_ARCH_ICACHE) && defined(CONFIG_ARCH_DCACHE)
|
|
void up_coherent_dcache(uintptr_t addr, size_t len)
|
|
{
|
|
uintptr_t end;
|
|
|
|
if (len > 0)
|
|
{
|
|
/* Flush any dirtcy D-Cache lines to memory */
|
|
|
|
end = addr + len;
|
|
up_clean_dcache(addr, end);
|
|
|
|
/* Invalidate I-Cache lines */
|
|
|
|
up_invalidate_icache(addr, end);
|
|
}
|
|
}
|
|
#endif
|