mirror of
https://github.com/torvalds/linux
synced 2026-07-22 10:10:47 +09:00
4edd70ee6a
section_activate() does not flush TLB after populating new vmemmap pages. On most architectures, this is okay. However it is a problem on RISC-V since there the TLB caching non-present entries is permitted, which causes spurious faults on some hardwares. This seems to be most easily reproduced with DEBUG_VM=y and PAGE_POISONING=y, which causes these newly mapped struct pages to be poisoned i.e. written to immediately after mapping. Extend the RISC-V flush_cache_vmap() to also handle the vmemmap range, and call it after hotplugging vmemmap, which gets the possible spurious fault handled in the exception handler. At least for now, the only other architecture with both SPARSEMEM_VMEMMAP and flush_cache_vmap() is PowerPC, which has a similar problem with newly valid PTEs. But there flush_cache_vmap() is just a ptesync. So it should be safe to do this for generic code while having minimal performance impact. Suggested-by: Muchun Song <muchun.song@linux.dev> Signed-off-by: Vivian Wang <wangruikang@iscas.ac.cn> Reviewed-by: Muchun Song <muchun.song@linux.dev> Acked-by: David Hildenbrand (Arm) <david@kernel.org> Link: https://patch.msgid.link/20260713-mark-after-vmemmap-populate-v6-2-b945ceba29d4@iscas.ac.cn Signed-off-by: Paul Walmsley <pjw@kernel.org>
110 lines
2.9 KiB
C
110 lines
2.9 KiB
C
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
/*
|
|
* Copyright (C) 2015 Regents of the University of California
|
|
*/
|
|
|
|
#ifndef _ASM_RISCV_CACHEFLUSH_H
|
|
#define _ASM_RISCV_CACHEFLUSH_H
|
|
|
|
#include <linux/mm.h>
|
|
|
|
static inline void local_flush_icache_all(void)
|
|
{
|
|
asm volatile ("fence.i" ::: "memory");
|
|
}
|
|
|
|
static inline void local_flush_icache_range(unsigned long start,
|
|
unsigned long end)
|
|
{
|
|
local_flush_icache_all();
|
|
}
|
|
|
|
#define PG_dcache_clean PG_arch_1
|
|
|
|
static inline void flush_dcache_folio(struct folio *folio)
|
|
{
|
|
if (test_bit(PG_dcache_clean, &folio->flags.f))
|
|
clear_bit(PG_dcache_clean, &folio->flags.f);
|
|
}
|
|
#define flush_dcache_folio flush_dcache_folio
|
|
#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1
|
|
|
|
static inline void flush_dcache_page(struct page *page)
|
|
{
|
|
flush_dcache_folio(page_folio(page));
|
|
}
|
|
|
|
#define flush_icache_user_page(vma, pg, addr, len) \
|
|
do { \
|
|
if (vma->vm_flags & VM_EXEC) \
|
|
flush_icache_mm(vma->vm_mm, 0); \
|
|
} while (0)
|
|
|
|
#if defined(CONFIG_64BIT) && defined(CONFIG_MMU)
|
|
/* This is accessed in assembly code. cpumask_var_t would be too complex. */
|
|
extern DECLARE_BITMAP(new_valid_map_cpus, NR_CPUS);
|
|
extern char _end[];
|
|
static inline void mark_new_valid_map(void)
|
|
{
|
|
/*
|
|
* We don't care if concurrently a cpu resets this value since
|
|
* the only place this can happen is in handle_exception() where
|
|
* an sfence.vma is emitted.
|
|
*/
|
|
bitmap_fill(new_valid_map_cpus, NR_CPUS);
|
|
}
|
|
#define flush_cache_vmap flush_cache_vmap
|
|
static inline void flush_cache_vmap(unsigned long start, unsigned long end)
|
|
{
|
|
if (is_vmalloc_or_module_addr((void *)start) ||
|
|
(start >= VMEMMAP_START && end <= VMEMMAP_END))
|
|
mark_new_valid_map();
|
|
}
|
|
#define flush_cache_vmap_early(start, end) local_flush_tlb_kernel_range(start, end)
|
|
#endif
|
|
|
|
#ifndef CONFIG_SMP
|
|
|
|
#define flush_icache_all() local_flush_icache_all()
|
|
#define flush_icache_mm(mm, local) flush_icache_all()
|
|
|
|
#else /* CONFIG_SMP */
|
|
|
|
void flush_icache_all(void);
|
|
void flush_icache_mm(struct mm_struct *mm, bool local);
|
|
|
|
#endif /* CONFIG_SMP */
|
|
|
|
/*
|
|
* RISC-V doesn't have an instruction to flush parts of the instruction cache,
|
|
* so instead we just flush the whole thing.
|
|
*/
|
|
#define flush_icache_range flush_icache_range
|
|
static inline void flush_icache_range(unsigned long start, unsigned long end)
|
|
{
|
|
flush_icache_all();
|
|
}
|
|
|
|
extern unsigned int riscv_cbom_block_size;
|
|
extern unsigned int riscv_cboz_block_size;
|
|
extern unsigned int riscv_cbop_block_size;
|
|
void riscv_init_cbo_blocksizes(void);
|
|
|
|
#ifdef CONFIG_RISCV_DMA_NONCOHERENT
|
|
void riscv_noncoherent_supported(void);
|
|
void __init riscv_set_dma_cache_alignment(void);
|
|
#else
|
|
static inline void riscv_noncoherent_supported(void) {}
|
|
static inline void riscv_set_dma_cache_alignment(void) {}
|
|
#endif
|
|
|
|
/*
|
|
* Bits in sys_riscv_flush_icache()'s flags argument.
|
|
*/
|
|
#define SYS_RISCV_FLUSH_ICACHE_LOCAL 1UL
|
|
#define SYS_RISCV_FLUSH_ICACHE_ALL (SYS_RISCV_FLUSH_ICACHE_LOCAL)
|
|
|
|
#include <asm-generic/cacheflush.h>
|
|
|
|
#endif /* _ASM_RISCV_CACHEFLUSH_H */
|