From 59c0cb06cc0889fbe19a552a4f3f9ae5a3d8e3a7 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Fri, 17 Feb 2023 12:29:42 +0000 Subject: [PATCH] ANDROID: KVM: arm64: Add __pkvm_enable_tracing This newly introduced hypercall allows the host to disable tracing on all CPUs, while keeping the tracing buffers loaded into the hypervisor. This intends to later improve the userspace interface which will be able to turn on and off tracing and reset (teardown for the hyp) the tracing buffers. As disabling buffers will switch the buffer status, rename those status to nonwritable - writable - writing. Another way of identifying buffers which have not been loaded is needed. See rb_cpu_loaded(). Bug: 249050813 Change-Id: I6080aafe71d5628e94b37c432bcd8616e68ddfe8 Signed-off-by: Vincent Donnefort --- arch/arm64/include/asm/kvm_asm.h | 5 +- arch/arm64/kvm/hyp/include/nvhe/trace.h | 28 +++--- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 20 ++-- arch/arm64/kvm/hyp/nvhe/trace.c | 119 ++++++++++++++++-------- arch/arm64/kvm/hyp_trace.c | 10 +- 5 files changed, 117 insertions(+), 65 deletions(-) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 96fecb072570..f4a0b50bf0e1 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -97,8 +97,9 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___pkvm_iommu_register, __KVM_HOST_SMCCC_FUNC___pkvm_iommu_pm_notify, __KVM_HOST_SMCCC_FUNC___pkvm_iommu_finalize, - __KVM_HOST_SMCCC_FUNC___pkvm_start_tracing, - __KVM_HOST_SMCCC_FUNC___pkvm_stop_tracing, + __KVM_HOST_SMCCC_FUNC___pkvm_load_tracing, + __KVM_HOST_SMCCC_FUNC___pkvm_teardown_tracing, + __KVM_HOST_SMCCC_FUNC___pkvm_enable_tracing, __KVM_HOST_SMCCC_FUNC___pkvm_rb_swap_reader_page, __KVM_HOST_SMCCC_FUNC___pkvm_rb_update_footers, __KVM_HOST_SMCCC_FUNC___pkvm_enable_event, diff --git a/arch/arm64/kvm/hyp/include/nvhe/trace.h b/arch/arm64/kvm/hyp/include/nvhe/trace.h index ff5d047e8715..2aec8f755f53 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/trace.h +++ b/arch/arm64/kvm/hyp/include/nvhe/trace.h @@ -19,9 +19,9 @@ struct hyp_buffer_page { atomic_t entries; }; -#define HYP_RB_UNUSED 0 -#define HYP_RB_READY 1 -#define HYP_RB_WRITE 2 +#define HYP_RB_NONWRITABLE 0 +#define HYP_RB_WRITABLE 1 +#define HYP_RB_WRITING 2 struct hyp_rb_per_cpu { struct hyp_buffer_page *tail_page; @@ -38,19 +38,16 @@ struct hyp_rb_per_cpu { static inline bool __start_write_hyp_rb(struct hyp_rb_per_cpu *rb) { - /* - * Paired with rb_cpu_init() - */ - return atomic_cmpxchg_acquire(&rb->status, HYP_RB_READY, HYP_RB_WRITE) - != HYP_RB_UNUSED; + return atomic_cmpxchg(&rb->status, HYP_RB_WRITABLE, HYP_RB_WRITING) + != HYP_RB_NONWRITABLE; } static inline void __stop_write_hyp_rb(struct hyp_rb_per_cpu *rb) { /* - * Paired with rb_cpu_teardown() + * Paired with rb_cpu_disable() */ - atomic_set_release(&rb->status, HYP_RB_READY); + atomic_set_release(&rb->status, HYP_RB_WRITABLE); } struct hyp_rb_per_cpu; @@ -58,8 +55,9 @@ DECLARE_PER_CPU(struct hyp_rb_per_cpu, trace_rb); void *rb_reserve_trace_entry(struct hyp_rb_per_cpu *cpu_buffer, unsigned long length); -int __pkvm_start_tracing(unsigned long pack_va, size_t pack_size); -void __pkvm_stop_tracing(void); +int __pkvm_load_tracing(unsigned long pack_va, size_t pack_size); +void __pkvm_teardown_tracing(void); +int __pkvm_enable_tracing(bool enable); int __pkvm_rb_swap_reader_page(int cpu); int __pkvm_rb_update_footers(int cpu); int __pkvm_enable_event(unsigned short id, bool enable); @@ -87,12 +85,14 @@ int __pkvm_enable_event(unsigned short id, bool enable); /* TODO: atomic_t to static_branch */ #else -static inline int __pkvm_start_tracing(unsigned long pack_va, size_t pack_size) +static inline int __pkvm_load_tracing(unsigned long pack_va, size_t pack_size) { return -ENODEV; } -static inline void __pkvm_stop_tracing(void) { } +static inline void __pkvm_teardown_tracing(void) { } + +static inline int __pkvm_enable_tracing(bool enable) { return -ENODEV; } static inline int __pkvm_rb_swap_reader_page(int cpu) { diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 970c0f94680d..a70b039d5508 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -1220,21 +1220,28 @@ handle___pkvm_close_module_registration(struct kvm_cpu_context *host_ctxt) cpu_reg(host_ctxt, 1) = __pkvm_close_module_registration(); } -static void handle___pkvm_start_tracing(struct kvm_cpu_context *host_ctxt) +static void handle___pkvm_load_tracing(struct kvm_cpu_context *host_ctxt) { DECLARE_REG(unsigned long, pack_hva, host_ctxt, 1); DECLARE_REG(size_t, pack_size, host_ctxt, 2); - cpu_reg(host_ctxt, 1) = __pkvm_start_tracing(pack_hva, pack_size); + cpu_reg(host_ctxt, 1) = __pkvm_load_tracing(pack_hva, pack_size); } -static void handle___pkvm_stop_tracing(struct kvm_cpu_context *host_ctxt) +static void handle___pkvm_teardown_tracing(struct kvm_cpu_context *host_ctxt) { - __pkvm_stop_tracing(); + __pkvm_teardown_tracing(); cpu_reg(host_ctxt, 1) = 0; } +static void handle___pkvm_enable_tracing(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(bool, enable, host_ctxt, 1); + + cpu_reg(host_ctxt, 1) = __pkvm_enable_tracing(enable); +} + static void handle___pkvm_rb_swap_reader_page(struct kvm_cpu_context *host_ctxt) { DECLARE_REG(int, cpu, host_ctxt, 1); @@ -1303,8 +1310,9 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__pkvm_iommu_register), HANDLE_FUNC(__pkvm_iommu_pm_notify), HANDLE_FUNC(__pkvm_iommu_finalize), - HANDLE_FUNC(__pkvm_start_tracing), - HANDLE_FUNC(__pkvm_stop_tracing), + HANDLE_FUNC(__pkvm_load_tracing), + HANDLE_FUNC(__pkvm_teardown_tracing), + HANDLE_FUNC(__pkvm_enable_tracing), HANDLE_FUNC(__pkvm_rb_swap_reader_page), HANDLE_FUNC(__pkvm_rb_update_footers), HANDLE_FUNC(__pkvm_enable_event), diff --git a/arch/arm64/kvm/hyp/nvhe/trace.c b/arch/arm64/kvm/hyp/nvhe/trace.c index 714433b684a0..6f796ec7b826 100644 --- a/arch/arm64/kvm/hyp/nvhe/trace.c +++ b/arch/arm64/kvm/hyp/nvhe/trace.c @@ -308,20 +308,51 @@ static int rb_page_init(struct hyp_buffer_page *bpage, unsigned long hva) return 0; } -static void rb_cpu_teardown(struct hyp_rb_per_cpu *cpu_buffer) +static bool rb_cpu_loaded(struct hyp_rb_per_cpu *cpu_buffer) +{ + return cpu_buffer->bpages; +} + +static void rb_cpu_disable(struct hyp_rb_per_cpu *cpu_buffer) { unsigned int prev_status; - int i; /* Wait for release of the buffer */ do { + /* Paired with __stop_write_hyp_rb */ prev_status = atomic_cmpxchg_acquire(&cpu_buffer->status, - HYP_RB_READY, - HYP_RB_UNUSED); - } while (prev_status == HYP_RB_WRITE); + HYP_RB_WRITABLE, + HYP_RB_NONWRITABLE); + } while (prev_status == HYP_RB_WRITING); - if (prev_status == HYP_RB_READY) + if (prev_status == HYP_RB_WRITABLE) rb_update_footers(cpu_buffer); +} + +static int rb_cpu_enable(struct hyp_rb_per_cpu *cpu_buffer) +{ + unsigned int prev_status; + + if (!rb_cpu_loaded(cpu_buffer)) + return -EINVAL; + + prev_status = atomic_cmpxchg(&cpu_buffer->status, + HYP_RB_NONWRITABLE, HYP_RB_WRITABLE); + + if (prev_status == HYP_RB_NONWRITABLE) + return 0; + + return -EINVAL; +} + +static void rb_cpu_teardown(struct hyp_rb_per_cpu *cpu_buffer) +{ + int i; + + if (!rb_cpu_loaded(cpu_buffer)) + return; + + rb_cpu_disable(cpu_buffer); for (i = 0; i < cpu_buffer->nr_pages; i++) { struct hyp_buffer_page *bpage = &cpu_buffer->bpages[i]; @@ -332,6 +363,8 @@ static void rb_cpu_teardown(struct hyp_rb_per_cpu *cpu_buffer) hyp_unpin_shared_mem((void *)bpage->page, (void *)bpage->page + PAGE_SIZE); } + + cpu_buffer->bpages = NULL; } static bool rb_cpu_fits_backing(unsigned long nr_pages, @@ -405,11 +438,6 @@ static int rb_cpu_init(struct ring_buffer_pack *rb_pack, struct hyp_buffer_page atomic_set(&cpu_buffer->overrun, 0); atomic64_set(&cpu_buffer->write_stamp, 0); - /* - * Paired with __start_write_hyp_rb() - */ - atomic_set_release(&cpu_buffer->status, HYP_RB_READY); - return 0; err: rb_cpu_teardown(cpu_buffer); @@ -462,23 +490,19 @@ int __pkvm_rb_update_footers(int cpu) struct hyp_rb_per_cpu *cpu_buffer; int ret = 0; + if (cpu >= hyp_nr_cpus) + return -EINVAL; + /* TODO: per-CPU lock for */ hyp_spin_lock(&trace_rb_lock); - if (cpu >= hyp_nr_cpus) { - ret = -EINVAL; - goto err; - } - cpu_buffer = per_cpu_ptr(&trace_rb, cpu); - if (atomic_read(&cpu_buffer->status) == HYP_RB_UNUSED) { + if (!rb_cpu_loaded(cpu_buffer)) ret = -ENODEV; - goto err; - } + else + ret = rb_update_footers(cpu_buffer); - ret = rb_update_footers(cpu_buffer); -err: hyp_spin_unlock(&trace_rb_lock); return ret; @@ -489,30 +513,25 @@ int __pkvm_rb_swap_reader_page(int cpu) struct hyp_rb_per_cpu *cpu_buffer = per_cpu_ptr(&trace_rb, cpu); int ret = 0; + if (cpu >= hyp_nr_cpus) + return -EINVAL; + /* TODO: per-CPU lock for */ hyp_spin_lock(&trace_rb_lock); - if (cpu >= hyp_nr_cpus) { - ret = -EINVAL; - goto err; - } cpu_buffer = per_cpu_ptr(&trace_rb, cpu); - if (atomic_read(&cpu_buffer->status) == HYP_RB_UNUSED) { + if (!rb_cpu_loaded(cpu_buffer)) ret = -ENODEV; - goto err; - } + else + ret = rb_swap_reader_page(cpu_buffer); - ret = rb_swap_reader_page(cpu_buffer); - if (ret) - goto err; -err: hyp_spin_unlock(&trace_rb_lock); return ret; } -static void __pkvm_stop_tracing_locked(void) +static void __pkvm_teardown_tracing_locked(void) { int cpu; @@ -521,23 +540,20 @@ static void __pkvm_stop_tracing_locked(void) for (cpu = 0; cpu < hyp_nr_cpus; cpu++) { struct hyp_rb_per_cpu *cpu_buffer = per_cpu_ptr(&trace_rb, cpu); - if (atomic_read(&cpu_buffer->status) == HYP_RB_UNUSED) - continue; - rb_cpu_teardown(cpu_buffer); } rb_teardown_bpage_backing(); } -void __pkvm_stop_tracing(void) +void __pkvm_teardown_tracing(void) { hyp_spin_lock(&trace_rb_lock); - __pkvm_stop_tracing_locked(); + __pkvm_teardown_tracing_locked(); hyp_spin_unlock(&trace_rb_lock); } -int __pkvm_start_tracing(unsigned long pack_hva, size_t pack_size) +int __pkvm_load_tracing(unsigned long pack_hva, size_t pack_size) { struct hyp_trace_pack *pack = (struct hyp_trace_pack *)kern_hyp_va(pack_hva); struct trace_buffer_pack *trace_pack = &pack->trace_buffer_pack; @@ -586,7 +602,7 @@ int __pkvm_start_tracing(unsigned long pack_hva, size_t pack_size) } err: if (ret) - __pkvm_stop_tracing_locked(); + __pkvm_teardown_tracing_locked(); hyp_spin_unlock(&trace_rb_lock); @@ -594,3 +610,26 @@ err: pack_size >> PAGE_SHIFT)); return ret; } + +int __pkvm_enable_tracing(bool enable) +{ + int cpu, ret = enable ? -EINVAL : 0; + + hyp_spin_lock(&trace_rb_lock); + for (cpu = 0; cpu < hyp_nr_cpus; cpu++) { + struct hyp_rb_per_cpu *cpu_buffer = per_cpu_ptr(&trace_rb, cpu); + + if (enable) { + int __ret = rb_cpu_enable(cpu_buffer); + + if (!__ret) + ret = 0; + } else { + rb_cpu_disable(cpu_buffer); + } + + } + hyp_spin_unlock(&trace_rb_lock); + + return ret; +} diff --git a/arch/arm64/kvm/hyp_trace.c b/arch/arm64/kvm/hyp_trace.c index 09e82b05fb17..8c15dd5a04d9 100644 --- a/arch/arm64/kvm/hyp_trace.c +++ b/arch/arm64/kvm/hyp_trace.c @@ -228,12 +228,16 @@ static int hyp_start_tracing(void) if (ret) goto end_buffer_teardown; - ret = kvm_call_hyp_nvhe(__pkvm_start_tracing, (unsigned long)pack, pack_size); + ret = kvm_call_hyp_nvhe(__pkvm_load_tracing, (unsigned long)pack, pack_size); + if (ret) + goto end_backing_teardown; + + ret = kvm_call_hyp_nvhe(__pkvm_enable_tracing, true); if (!ret) { hyp_trace_on = true; goto end_free_pack; } - +end_backing_teardown: bpage_backing_teardown(); end_buffer_teardown: trace_buffer_teardown(&pack->trace_buffer_pack); @@ -250,7 +254,7 @@ static void hyp_stop_tracing(void) if (!hyp_trace_buffer || !hyp_trace_on) return; - ret = kvm_call_hyp_nvhe(__pkvm_stop_tracing); + ret = kvm_call_hyp_nvhe(__pkvm_teardown_tracing); if (ret) { WARN_ON(1); return;