diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 96fecb072570..f4a0b50bf0e1 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -97,8 +97,9 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___pkvm_iommu_register, __KVM_HOST_SMCCC_FUNC___pkvm_iommu_pm_notify, __KVM_HOST_SMCCC_FUNC___pkvm_iommu_finalize, - __KVM_HOST_SMCCC_FUNC___pkvm_start_tracing, - __KVM_HOST_SMCCC_FUNC___pkvm_stop_tracing, + __KVM_HOST_SMCCC_FUNC___pkvm_load_tracing, + __KVM_HOST_SMCCC_FUNC___pkvm_teardown_tracing, + __KVM_HOST_SMCCC_FUNC___pkvm_enable_tracing, __KVM_HOST_SMCCC_FUNC___pkvm_rb_swap_reader_page, __KVM_HOST_SMCCC_FUNC___pkvm_rb_update_footers, __KVM_HOST_SMCCC_FUNC___pkvm_enable_event, diff --git a/arch/arm64/kvm/hyp/include/nvhe/trace.h b/arch/arm64/kvm/hyp/include/nvhe/trace.h index ff5d047e8715..2aec8f755f53 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/trace.h +++ b/arch/arm64/kvm/hyp/include/nvhe/trace.h @@ -19,9 +19,9 @@ struct hyp_buffer_page { atomic_t entries; }; -#define HYP_RB_UNUSED 0 -#define HYP_RB_READY 1 -#define HYP_RB_WRITE 2 +#define HYP_RB_NONWRITABLE 0 +#define HYP_RB_WRITABLE 1 +#define HYP_RB_WRITING 2 struct hyp_rb_per_cpu { struct hyp_buffer_page *tail_page; @@ -38,19 +38,16 @@ struct hyp_rb_per_cpu { static inline bool __start_write_hyp_rb(struct hyp_rb_per_cpu *rb) { - /* - * Paired with rb_cpu_init() - */ - return atomic_cmpxchg_acquire(&rb->status, HYP_RB_READY, HYP_RB_WRITE) - != HYP_RB_UNUSED; + return atomic_cmpxchg(&rb->status, HYP_RB_WRITABLE, HYP_RB_WRITING) + != HYP_RB_NONWRITABLE; } static inline void __stop_write_hyp_rb(struct hyp_rb_per_cpu *rb) { /* - * Paired with rb_cpu_teardown() + * Paired with rb_cpu_disable() */ - atomic_set_release(&rb->status, HYP_RB_READY); + atomic_set_release(&rb->status, HYP_RB_WRITABLE); } struct hyp_rb_per_cpu; @@ -58,8 +55,9 @@ DECLARE_PER_CPU(struct hyp_rb_per_cpu, trace_rb); void *rb_reserve_trace_entry(struct hyp_rb_per_cpu *cpu_buffer, unsigned long length); -int __pkvm_start_tracing(unsigned long pack_va, size_t pack_size); -void __pkvm_stop_tracing(void); +int __pkvm_load_tracing(unsigned long pack_va, size_t pack_size); +void __pkvm_teardown_tracing(void); +int __pkvm_enable_tracing(bool enable); int __pkvm_rb_swap_reader_page(int cpu); int __pkvm_rb_update_footers(int cpu); int __pkvm_enable_event(unsigned short id, bool enable); @@ -87,12 +85,14 @@ int __pkvm_enable_event(unsigned short id, bool enable); /* TODO: atomic_t to static_branch */ #else -static inline int __pkvm_start_tracing(unsigned long pack_va, size_t pack_size) +static inline int __pkvm_load_tracing(unsigned long pack_va, size_t pack_size) { return -ENODEV; } -static inline void __pkvm_stop_tracing(void) { } +static inline void __pkvm_teardown_tracing(void) { } + +static inline int __pkvm_enable_tracing(bool enable) { return -ENODEV; } static inline int __pkvm_rb_swap_reader_page(int cpu) { diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 970c0f94680d..a70b039d5508 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -1220,21 +1220,28 @@ handle___pkvm_close_module_registration(struct kvm_cpu_context *host_ctxt) cpu_reg(host_ctxt, 1) = __pkvm_close_module_registration(); } -static void handle___pkvm_start_tracing(struct kvm_cpu_context *host_ctxt) +static void handle___pkvm_load_tracing(struct kvm_cpu_context *host_ctxt) { DECLARE_REG(unsigned long, pack_hva, host_ctxt, 1); DECLARE_REG(size_t, pack_size, host_ctxt, 2); - cpu_reg(host_ctxt, 1) = __pkvm_start_tracing(pack_hva, pack_size); + cpu_reg(host_ctxt, 1) = __pkvm_load_tracing(pack_hva, pack_size); } -static void handle___pkvm_stop_tracing(struct kvm_cpu_context *host_ctxt) +static void handle___pkvm_teardown_tracing(struct kvm_cpu_context *host_ctxt) { - __pkvm_stop_tracing(); + __pkvm_teardown_tracing(); cpu_reg(host_ctxt, 1) = 0; } +static void handle___pkvm_enable_tracing(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(bool, enable, host_ctxt, 1); + + cpu_reg(host_ctxt, 1) = __pkvm_enable_tracing(enable); +} + static void handle___pkvm_rb_swap_reader_page(struct kvm_cpu_context *host_ctxt) { DECLARE_REG(int, cpu, host_ctxt, 1); @@ -1303,8 +1310,9 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__pkvm_iommu_register), HANDLE_FUNC(__pkvm_iommu_pm_notify), HANDLE_FUNC(__pkvm_iommu_finalize), - HANDLE_FUNC(__pkvm_start_tracing), - HANDLE_FUNC(__pkvm_stop_tracing), + HANDLE_FUNC(__pkvm_load_tracing), + HANDLE_FUNC(__pkvm_teardown_tracing), + HANDLE_FUNC(__pkvm_enable_tracing), HANDLE_FUNC(__pkvm_rb_swap_reader_page), HANDLE_FUNC(__pkvm_rb_update_footers), HANDLE_FUNC(__pkvm_enable_event), diff --git a/arch/arm64/kvm/hyp/nvhe/trace.c b/arch/arm64/kvm/hyp/nvhe/trace.c index 714433b684a0..6f796ec7b826 100644 --- a/arch/arm64/kvm/hyp/nvhe/trace.c +++ b/arch/arm64/kvm/hyp/nvhe/trace.c @@ -308,20 +308,51 @@ static int rb_page_init(struct hyp_buffer_page *bpage, unsigned long hva) return 0; } -static void rb_cpu_teardown(struct hyp_rb_per_cpu *cpu_buffer) +static bool rb_cpu_loaded(struct hyp_rb_per_cpu *cpu_buffer) +{ + return cpu_buffer->bpages; +} + +static void rb_cpu_disable(struct hyp_rb_per_cpu *cpu_buffer) { unsigned int prev_status; - int i; /* Wait for release of the buffer */ do { + /* Paired with __stop_write_hyp_rb */ prev_status = atomic_cmpxchg_acquire(&cpu_buffer->status, - HYP_RB_READY, - HYP_RB_UNUSED); - } while (prev_status == HYP_RB_WRITE); + HYP_RB_WRITABLE, + HYP_RB_NONWRITABLE); + } while (prev_status == HYP_RB_WRITING); - if (prev_status == HYP_RB_READY) + if (prev_status == HYP_RB_WRITABLE) rb_update_footers(cpu_buffer); +} + +static int rb_cpu_enable(struct hyp_rb_per_cpu *cpu_buffer) +{ + unsigned int prev_status; + + if (!rb_cpu_loaded(cpu_buffer)) + return -EINVAL; + + prev_status = atomic_cmpxchg(&cpu_buffer->status, + HYP_RB_NONWRITABLE, HYP_RB_WRITABLE); + + if (prev_status == HYP_RB_NONWRITABLE) + return 0; + + return -EINVAL; +} + +static void rb_cpu_teardown(struct hyp_rb_per_cpu *cpu_buffer) +{ + int i; + + if (!rb_cpu_loaded(cpu_buffer)) + return; + + rb_cpu_disable(cpu_buffer); for (i = 0; i < cpu_buffer->nr_pages; i++) { struct hyp_buffer_page *bpage = &cpu_buffer->bpages[i]; @@ -332,6 +363,8 @@ static void rb_cpu_teardown(struct hyp_rb_per_cpu *cpu_buffer) hyp_unpin_shared_mem((void *)bpage->page, (void *)bpage->page + PAGE_SIZE); } + + cpu_buffer->bpages = NULL; } static bool rb_cpu_fits_backing(unsigned long nr_pages, @@ -405,11 +438,6 @@ static int rb_cpu_init(struct ring_buffer_pack *rb_pack, struct hyp_buffer_page atomic_set(&cpu_buffer->overrun, 0); atomic64_set(&cpu_buffer->write_stamp, 0); - /* - * Paired with __start_write_hyp_rb() - */ - atomic_set_release(&cpu_buffer->status, HYP_RB_READY); - return 0; err: rb_cpu_teardown(cpu_buffer); @@ -462,23 +490,19 @@ int __pkvm_rb_update_footers(int cpu) struct hyp_rb_per_cpu *cpu_buffer; int ret = 0; + if (cpu >= hyp_nr_cpus) + return -EINVAL; + /* TODO: per-CPU lock for */ hyp_spin_lock(&trace_rb_lock); - if (cpu >= hyp_nr_cpus) { - ret = -EINVAL; - goto err; - } - cpu_buffer = per_cpu_ptr(&trace_rb, cpu); - if (atomic_read(&cpu_buffer->status) == HYP_RB_UNUSED) { + if (!rb_cpu_loaded(cpu_buffer)) ret = -ENODEV; - goto err; - } + else + ret = rb_update_footers(cpu_buffer); - ret = rb_update_footers(cpu_buffer); -err: hyp_spin_unlock(&trace_rb_lock); return ret; @@ -489,30 +513,25 @@ int __pkvm_rb_swap_reader_page(int cpu) struct hyp_rb_per_cpu *cpu_buffer = per_cpu_ptr(&trace_rb, cpu); int ret = 0; + if (cpu >= hyp_nr_cpus) + return -EINVAL; + /* TODO: per-CPU lock for */ hyp_spin_lock(&trace_rb_lock); - if (cpu >= hyp_nr_cpus) { - ret = -EINVAL; - goto err; - } cpu_buffer = per_cpu_ptr(&trace_rb, cpu); - if (atomic_read(&cpu_buffer->status) == HYP_RB_UNUSED) { + if (!rb_cpu_loaded(cpu_buffer)) ret = -ENODEV; - goto err; - } + else + ret = rb_swap_reader_page(cpu_buffer); - ret = rb_swap_reader_page(cpu_buffer); - if (ret) - goto err; -err: hyp_spin_unlock(&trace_rb_lock); return ret; } -static void __pkvm_stop_tracing_locked(void) +static void __pkvm_teardown_tracing_locked(void) { int cpu; @@ -521,23 +540,20 @@ static void __pkvm_stop_tracing_locked(void) for (cpu = 0; cpu < hyp_nr_cpus; cpu++) { struct hyp_rb_per_cpu *cpu_buffer = per_cpu_ptr(&trace_rb, cpu); - if (atomic_read(&cpu_buffer->status) == HYP_RB_UNUSED) - continue; - rb_cpu_teardown(cpu_buffer); } rb_teardown_bpage_backing(); } -void __pkvm_stop_tracing(void) +void __pkvm_teardown_tracing(void) { hyp_spin_lock(&trace_rb_lock); - __pkvm_stop_tracing_locked(); + __pkvm_teardown_tracing_locked(); hyp_spin_unlock(&trace_rb_lock); } -int __pkvm_start_tracing(unsigned long pack_hva, size_t pack_size) +int __pkvm_load_tracing(unsigned long pack_hva, size_t pack_size) { struct hyp_trace_pack *pack = (struct hyp_trace_pack *)kern_hyp_va(pack_hva); struct trace_buffer_pack *trace_pack = &pack->trace_buffer_pack; @@ -586,7 +602,7 @@ int __pkvm_start_tracing(unsigned long pack_hva, size_t pack_size) } err: if (ret) - __pkvm_stop_tracing_locked(); + __pkvm_teardown_tracing_locked(); hyp_spin_unlock(&trace_rb_lock); @@ -594,3 +610,26 @@ err: pack_size >> PAGE_SHIFT)); return ret; } + +int __pkvm_enable_tracing(bool enable) +{ + int cpu, ret = enable ? -EINVAL : 0; + + hyp_spin_lock(&trace_rb_lock); + for (cpu = 0; cpu < hyp_nr_cpus; cpu++) { + struct hyp_rb_per_cpu *cpu_buffer = per_cpu_ptr(&trace_rb, cpu); + + if (enable) { + int __ret = rb_cpu_enable(cpu_buffer); + + if (!__ret) + ret = 0; + } else { + rb_cpu_disable(cpu_buffer); + } + + } + hyp_spin_unlock(&trace_rb_lock); + + return ret; +} diff --git a/arch/arm64/kvm/hyp_trace.c b/arch/arm64/kvm/hyp_trace.c index 09e82b05fb17..8c15dd5a04d9 100644 --- a/arch/arm64/kvm/hyp_trace.c +++ b/arch/arm64/kvm/hyp_trace.c @@ -228,12 +228,16 @@ static int hyp_start_tracing(void) if (ret) goto end_buffer_teardown; - ret = kvm_call_hyp_nvhe(__pkvm_start_tracing, (unsigned long)pack, pack_size); + ret = kvm_call_hyp_nvhe(__pkvm_load_tracing, (unsigned long)pack, pack_size); + if (ret) + goto end_backing_teardown; + + ret = kvm_call_hyp_nvhe(__pkvm_enable_tracing, true); if (!ret) { hyp_trace_on = true; goto end_free_pack; } - +end_backing_teardown: bpage_backing_teardown(); end_buffer_teardown: trace_buffer_teardown(&pack->trace_buffer_pack); @@ -250,7 +254,7 @@ static void hyp_stop_tracing(void) if (!hyp_trace_buffer || !hyp_trace_on) return; - ret = kvm_call_hyp_nvhe(__pkvm_stop_tracing); + ret = kvm_call_hyp_nvhe(__pkvm_teardown_tracing); if (ret) { WARN_ON(1); return;