diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index bff50d7c63b5..76703b2b6f3c 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -620,6 +620,50 @@ static bool is_dabt(u64 esr) return ESR_ELx_EC(esr) == ESR_ELx_EC_DABT_LOW; } +static void host_inject_abort(struct kvm_cpu_context *host_ctxt) +{ + u64 spsr = read_sysreg_el2(SYS_SPSR); + u64 esr = read_sysreg_el2(SYS_ESR); + u64 ventry, ec; + + /* Repaint the ESR to report a same-level fault if taken from EL1 */ + if ((spsr & PSR_MODE_MASK) != PSR_MODE_EL0t) { + ec = ESR_ELx_EC(esr); + if (ec == ESR_ELx_EC_DABT_LOW) + ec = ESR_ELx_EC_DABT_CUR; + else if (ec == ESR_ELx_EC_IABT_LOW) + ec = ESR_ELx_EC_IABT_CUR; + else + WARN_ON(1); + esr &= ~ESR_ELx_EC_MASK; + esr |= ec << ESR_ELx_EC_SHIFT; + } + + /* + * Since S1PTW should only ever be set for stage-2 faults, we're pretty + * much guaranteed that it won't be set in ESR_EL1 by the hardware. So, + * let's use that bit to allow the host abort handler to differentiate + * this abort from normal userspace faults. + * + * Note: although S1PTW is RES0 at EL1, it is guaranteed by the + * architecture to be backed by flops, so it should be safe to use. + */ + esr |= ESR_ELx_S1PTW; + + write_sysreg_el1(esr, SYS_ESR); + write_sysreg_el1(spsr, SYS_SPSR); + write_sysreg_el1(read_sysreg_el2(SYS_ELR), SYS_ELR); + write_sysreg_el1(read_sysreg_el2(SYS_FAR), SYS_FAR); + + ventry = read_sysreg_el1(SYS_VBAR); + ventry += get_except64_offset(spsr, PSR_MODE_EL1h, except_type_sync); + write_sysreg_el2(ventry, SYS_ELR); + + spsr = get_except64_cpsr(spsr, system_supports_mte(), + read_sysreg_el1(SYS_SCTLR), PSR_MODE_EL1h); + write_sysreg_el2(spsr, SYS_SPSR); +} + void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt) { struct kvm_vcpu_fault_info fault; @@ -644,7 +688,11 @@ void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt) ret = host_stage2_idmap(addr); host_unlock_component(); - BUG_ON(ret && ret != -EAGAIN); + + if (ret == -EPERM) + host_inject_abort(host_ctxt); + else + BUG_ON(ret && ret != -EAGAIN); } /* This corresponds to locking order */ diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 540ef3cd0a36..9af359f68356 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -41,6 +41,7 @@ #include #include #include +#include #include @@ -253,6 +254,15 @@ static inline bool is_el1_permission_fault(unsigned long addr, unsigned int esr, return false; } +static bool is_pkvm_stage2_abort(unsigned int esr) +{ + /* + * S1PTW should only ever be set in ESR_EL1 if the pkvm hypervisor + * injected a stage-2 abort -- see host_inject_abort(). + */ + return is_pkvm_initialized() && (esr & ESR_ELx_S1PTW); +} + static bool __kprobes is_spurious_el1_translation_fault(unsigned long addr, unsigned int esr, struct pt_regs *regs) @@ -264,6 +274,9 @@ static bool __kprobes is_spurious_el1_translation_fault(unsigned long addr, (esr & ESR_ELx_FSC_TYPE) != ESR_ELx_FSC_FAULT) return false; + if (is_pkvm_stage2_abort(esr)) + return false; + local_irq_save(flags); asm volatile("at s1e1r, %0" :: "r" (addr)); isb(); @@ -379,6 +392,8 @@ static void __do_kernel_fault(unsigned long addr, unsigned int esr, msg = "read from unreadable memory"; } else if (addr < PAGE_SIZE) { msg = "NULL pointer dereference"; + } else if (is_pkvm_stage2_abort(esr)) { + msg = "access to hypervisor-protected memory"; } else { if (kfence_handle_page_fault(addr, esr & ESR_ELx_WNR, regs)) return; @@ -557,6 +572,13 @@ static int __kprobes do_page_fault(unsigned long far, unsigned int esr, addr, esr, regs); } + if (is_pkvm_stage2_abort(esr)) { + if (!user_mode(regs)) + goto no_context; + arm64_force_sig_fault(SIGSEGV, SEGV_ACCERR, far, "stage-2 fault"); + return 0; + } + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); /*