From eab8f11bed73de6ca0cfe5a6897f53e0a1560bfb Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 25 Aug 2022 10:33:32 +0100 Subject: [PATCH] ANDROID: KVM: arm64: Add tracing support for the nVHE hyp Running at EL2, the host has very close to no way to know what's happening in the hypervisor, which is great from a security point of view, a bit less when it turns to debug it. With the introduction of the protected mode, this piece of code is getting more responsibilities, which we would like to debug and profile, hence the need to trace "things" that are happening in the hypervisor. There's no way the hypervisor could log things directly into the host tracing interface. So instead let's use a separated per-CPU ring buffer compliant with the host so the latter can decode the events. The tracing interface is composed of 4 HVCs: __pkvm_start_tracing: Gets a hyp_trace_pack describing the ring buffers, the backing storage that'll support the internal structures and some clock init values. __pkvm_stop_tracing: Disable writing to the ring buffers and teardown. __pkvm_rb_swap_reader_page: Request the writer to swap the head with the reader page. This enables consuming read of the read buffer. See kernel/ring_buffer.c (*ext_writer_swap_reader)() callback. __pkvm_rb_update_footers: Request the writer to update the pages footers. Those informations are then used to update the reader view on the ring buffer. See kernel/ring_buffer.c (*ext_writer_update_footers)() callback. Bug: 229972309 Change-Id: I0e32223795de435aee8546af368672f0e67637b3 Signed-off-by: Vincent Donnefort --- arch/arm64/include/asm/kvm_asm.h | 4 + arch/arm64/include/asm/kvm_hyptrace.h | 21 + arch/arm64/kvm/hyp/include/nvhe/trace.h | 78 ++++ arch/arm64/kvm/hyp/nvhe/Makefile | 2 +- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 34 ++ arch/arm64/kvm/hyp/nvhe/trace.c | 596 ++++++++++++++++++++++++ 6 files changed, 734 insertions(+), 1 deletion(-) create mode 100644 arch/arm64/include/asm/kvm_hyptrace.h create mode 100644 arch/arm64/kvm/hyp/include/nvhe/trace.h create mode 100644 arch/arm64/kvm/hyp/nvhe/trace.c diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 6286bf2a8615..7df0b6ea7e06 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -96,6 +96,10 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___pkvm_iommu_register, __KVM_HOST_SMCCC_FUNC___pkvm_iommu_pm_notify, __KVM_HOST_SMCCC_FUNC___pkvm_iommu_finalize, + __KVM_HOST_SMCCC_FUNC___pkvm_start_tracing, + __KVM_HOST_SMCCC_FUNC___pkvm_stop_tracing, + __KVM_HOST_SMCCC_FUNC___pkvm_rb_swap_reader_page, + __KVM_HOST_SMCCC_FUNC___pkvm_rb_update_footers, /* * Start of the dynamically registered hypercalls. Start a bit diff --git a/arch/arm64/include/asm/kvm_hyptrace.h b/arch/arm64/include/asm/kvm_hyptrace.h new file mode 100644 index 000000000000..d32b445b4cf6 --- /dev/null +++ b/arch/arm64/include/asm/kvm_hyptrace.h @@ -0,0 +1,21 @@ +#ifndef __ARM64_KVM_HYPTRACE_H_ +#define __ARM64_KVM_HYPTRACE_H_ +#include + +#include + +/* + * Host donations to the hypervisor to store the struct hyp_buffer_page. + */ +struct hyp_buffer_pages_backing { + unsigned long start; + size_t size; +}; + +struct hyp_trace_pack { + struct hyp_buffer_pages_backing backing; + struct kvm_nvhe_clock_data trace_clock_data; + struct trace_buffer_pack trace_buffer_pack; + +}; +#endif diff --git a/arch/arm64/kvm/hyp/include/nvhe/trace.h b/arch/arm64/kvm/hyp/include/nvhe/trace.h new file mode 100644 index 000000000000..18d3c404a4a3 --- /dev/null +++ b/arch/arm64/kvm/hyp/include/nvhe/trace.h @@ -0,0 +1,78 @@ +#ifndef __ARM64_KVM_HYP_NVHE_TRACE_H +#define __ARM64_KVM_HYP_NVHE_TRACE_H +#include + +#include +#include + +#ifdef CONFIG_TRACING + +struct hyp_buffer_page { + struct list_head list; + struct buffer_data_page *page; + atomic_t write; + atomic_t entries; +}; + +#define HYP_RB_UNUSED 0 +#define HYP_RB_READY 1 +#define HYP_RB_WRITE 2 + +struct hyp_rb_per_cpu { + struct hyp_buffer_page *tail_page; + struct hyp_buffer_page *reader_page; + struct hyp_buffer_page *head_page; + struct hyp_buffer_page *bpages; + unsigned long nr_pages; + atomic64_t write_stamp; + atomic_t pages_touched; + atomic_t nr_entries; + atomic_t status; + atomic_t overrun; +}; + +static inline bool __start_write_hyp_rb(struct hyp_rb_per_cpu *rb) +{ + /* + * Paired with rb_cpu_init() + */ + return atomic_cmpxchg_acquire(&rb->status, HYP_RB_READY, HYP_RB_WRITE) + != HYP_RB_UNUSED; +} + +static inline void __stop_write_hyp_rb(struct hyp_rb_per_cpu *rb) +{ + /* + * Paired with rb_cpu_teardown() + */ + atomic_set_release(&rb->status, HYP_RB_READY); +} + +struct hyp_rb_per_cpu; +DECLARE_PER_CPU(struct hyp_rb_per_cpu, trace_rb); + +void *rb_reserve_trace_entry(struct hyp_rb_per_cpu *cpu_buffer, unsigned long length); + +int __pkvm_start_tracing(unsigned long pack_va, size_t pack_size); +void __pkvm_stop_tracing(void); +int __pkvm_rb_swap_reader_page(int cpu); +int __pkvm_rb_update_footers(int cpu); +#else +static inline int __pkvm_start_tracing(unsigned long pack_va, size_t pack_size) +{ + return -ENODEV; +} + +static inline void __pkvm_stop_tracing(void) { } + +static inline int __pkvm_rb_swap_reader_page(int cpu) +{ + return -ENODEV; +} + +static inline int __pkvm_rb_update_footers(int cpu) +{ + return -ENODEV; +} +#endif +#endif diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile index bd3ad71d432a..64032a2c2202 100644 --- a/arch/arm64/kvm/hyp/nvhe/Makefile +++ b/arch/arm64/kvm/hyp/nvhe/Makefile @@ -8,7 +8,7 @@ hyp-obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o serial.o hyp-obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \ ../fpsimd.o ../hyp-entry.o ../exception.o ../pgtable.o -hyp-obj-$(CONFIG_TRACING) += clock.o +hyp-obj-$(CONFIG_TRACING) += clock.o trace.o hyp-obj-$(CONFIG_DEBUG_LIST) += list_debug.o hyp-obj-$(CONFIG_MODULES) += modules.o hyp-obj-y += $(lib-objs) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 8a9681874dd6..7bf26f8f082f 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -1208,6 +1209,35 @@ handle___pkvm_close_module_registration(struct kvm_cpu_context *host_ctxt) cpu_reg(host_ctxt, 1) = __pkvm_close_module_registration(); } +static void handle___pkvm_start_tracing(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(unsigned long, pack_hva, host_ctxt, 1); + DECLARE_REG(size_t, pack_size, host_ctxt, 2); + + cpu_reg(host_ctxt, 1) = __pkvm_start_tracing(pack_hva, pack_size); +} + +static void handle___pkvm_stop_tracing(struct kvm_cpu_context *host_ctxt) +{ + __pkvm_stop_tracing(); + + cpu_reg(host_ctxt, 1) = 0; +} + +static void handle___pkvm_rb_swap_reader_page(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(int, cpu, host_ctxt, 1); + + cpu_reg(host_ctxt, 1) = __pkvm_rb_swap_reader_page(cpu); +} + +static void handle___pkvm_rb_update_footers(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(int, cpu, host_ctxt, 1); + + cpu_reg(host_ctxt, 1) = __pkvm_rb_update_footers(cpu); +} + typedef void (*hcall_t)(struct kvm_cpu_context *); #define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x @@ -1253,6 +1283,10 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__pkvm_iommu_register), HANDLE_FUNC(__pkvm_iommu_pm_notify), HANDLE_FUNC(__pkvm_iommu_finalize), + HANDLE_FUNC(__pkvm_start_tracing), + HANDLE_FUNC(__pkvm_stop_tracing), + HANDLE_FUNC(__pkvm_rb_swap_reader_page), + HANDLE_FUNC(__pkvm_rb_update_footers), }; unsigned long pkvm_priv_hcall_limit __ro_after_init = __KVM_HOST_SMCCC_FUNC___pkvm_prot_finalize; diff --git a/arch/arm64/kvm/hyp/nvhe/trace.c b/arch/arm64/kvm/hyp/nvhe/trace.c new file mode 100644 index 000000000000..714433b684a0 --- /dev/null +++ b/arch/arm64/kvm/hyp/nvhe/trace.c @@ -0,0 +1,596 @@ +#include +#include +#include +#include + +#include +#include + +#include + +#define HYP_RB_PAGE_HEAD 1UL +#define HYP_RB_PAGE_UPDATE 2UL +#define HYP_RB_FLAG_MASK 3UL + +static struct hyp_buffer_pages_backing hyp_buffer_pages_backing; +DEFINE_PER_CPU(struct hyp_rb_per_cpu, trace_rb); +DEFINE_HYP_SPINLOCK(trace_rb_lock); + +static bool rb_set_flag(struct hyp_buffer_page *bpage, int new_flag) +{ + unsigned long ret, val = (unsigned long)bpage->list.next; + + ret = cmpxchg((unsigned long *)&bpage->list.next, + val, (val & ~HYP_RB_FLAG_MASK) | new_flag); + + return ret == val; +} + +static void rb_set_footer_status(struct hyp_buffer_page *bpage, + unsigned long status, + bool reader) +{ + struct buffer_data_page *page = bpage->page; + struct rb_ext_page_footer *footer; + + footer = rb_ext_page_get_footer(page); + + if (reader) + atomic_set(&footer->reader_status, status); + else + atomic_set(&footer->writer_status, status); +} + +static void rb_footer_writer_status(struct hyp_buffer_page *bpage, + unsigned long status) +{ + rb_set_footer_status(bpage, status, false); +} + +static void rb_footer_reader_status(struct hyp_buffer_page *bpage, + unsigned long status) +{ + rb_set_footer_status(bpage, status, true); +} + +static struct hyp_buffer_page *rb_hyp_buffer_page(struct list_head *list) +{ + unsigned long ptr = (unsigned long)list & ~HYP_RB_FLAG_MASK; + + return container_of((struct list_head *)ptr, struct hyp_buffer_page, list); +} + +static struct hyp_buffer_page *rb_next_page(struct hyp_buffer_page *bpage) +{ + return rb_hyp_buffer_page(bpage->list.next); +} + +static bool rb_is_head_page(struct hyp_buffer_page *bpage) +{ + return (unsigned long)bpage->list.prev->next & HYP_RB_PAGE_HEAD; +} + +static struct hyp_buffer_page *rb_set_head_page(struct hyp_rb_per_cpu *cpu_buffer) +{ + struct hyp_buffer_page *bpage, *prev_head; + int cnt = 0; +again: + bpage = prev_head = cpu_buffer->head_page; + do { + if (rb_is_head_page(bpage)) { + cpu_buffer->head_page = bpage; + rb_footer_reader_status(prev_head, 0); + rb_footer_reader_status(bpage, RB_PAGE_FT_HEAD); + return bpage; + } + + bpage = rb_next_page(bpage); + } while (bpage != prev_head); + + cnt++; + + /* We might have race with the writer let's try again */ + if (cnt < 3) + goto again; + + return NULL; +} + +static int rb_swap_reader_page(struct hyp_rb_per_cpu *cpu_buffer) +{ + unsigned long *old_head_link, old_link_val, new_link_val, overrun; + struct hyp_buffer_page *head, *reader = cpu_buffer->reader_page; + struct rb_ext_page_footer *footer; + + rb_footer_reader_status(cpu_buffer->reader_page, 0); +spin: + /* Update the cpu_buffer->header_page according to HYP_RB_PAGE_HEAD */ + head = rb_set_head_page(cpu_buffer); + if (!head) + return -ENODEV; + + /* Connect the reader page around the header page */ + reader->list.next = head->list.next; + reader->list.prev = head->list.prev; + + /* The reader page points to the new header page */ + rb_set_flag(reader, HYP_RB_PAGE_HEAD); + + /* + * Paired with the cmpxchg in rb_move_tail(). Order the read of the head + * page and overrun. + */ + smp_mb(); + overrun = atomic_read(&cpu_buffer->overrun); + + /* Try to swap the prev head link to the reader page */ + old_head_link = (unsigned long *)&reader->list.prev->next; + old_link_val = (*old_head_link & ~HYP_RB_FLAG_MASK) | HYP_RB_PAGE_HEAD; + new_link_val = (unsigned long)&reader->list; + if (cmpxchg(old_head_link, old_link_val, new_link_val) + != old_link_val) + goto spin; + + cpu_buffer->head_page = rb_hyp_buffer_page(reader->list.next); + cpu_buffer->head_page->list.prev = &reader->list; + cpu_buffer->reader_page = head; + + rb_footer_reader_status(cpu_buffer->reader_page, RB_PAGE_FT_READER); + rb_footer_reader_status(cpu_buffer->head_page, RB_PAGE_FT_HEAD); + + footer = rb_ext_page_get_footer(cpu_buffer->reader_page->page); + footer->stats.overrun = overrun; + + return 0; +} + +static struct hyp_buffer_page * +rb_move_tail(struct hyp_rb_per_cpu *cpu_buffer) +{ + struct hyp_buffer_page *tail_page, *new_tail, *new_head; + + tail_page = cpu_buffer->tail_page; + new_tail = rb_next_page(tail_page); +again: + /* + * We caught the reader ... Let's try to move the head page. + * The writer can only rely on ->next links to check if this is head. + */ + if ((unsigned long)tail_page->list.next & HYP_RB_PAGE_HEAD) { + /* The reader moved the head in between */ + if (!rb_set_flag(tail_page, HYP_RB_PAGE_UPDATE)) + goto again; + + atomic_add(atomic_read(&new_tail->entries), &cpu_buffer->overrun); + + /* Move the head */ + rb_set_flag(new_tail, HYP_RB_PAGE_HEAD); + + /* The new head is in place, reset the update flag */ + rb_set_flag(tail_page, 0); + + new_head = rb_next_page(new_tail); + } + + rb_footer_writer_status(tail_page, 0); + rb_footer_writer_status(new_tail, RB_PAGE_FT_COMMIT); + + local_set(&new_tail->page->commit, 0); + + atomic_set(&new_tail->write, 0); + atomic_set(&new_tail->entries, 0); + + atomic_inc(&cpu_buffer->pages_touched); + + cpu_buffer->tail_page = new_tail; + + return new_tail; +} + +unsigned long rb_event_size(unsigned long length) +{ + struct ring_buffer_event *event; + + return length + RB_EVNT_HDR_SIZE + sizeof(event->array[0]); +} + +static struct ring_buffer_event * +rb_add_ts_extend(struct ring_buffer_event *event, u64 delta) +{ + event->type_len = RINGBUF_TYPE_TIME_EXTEND; + event->time_delta = delta & TS_MASK; + event->array[0] = delta >> TS_SHIFT; + + return (struct ring_buffer_event *)((unsigned long)event + 8); +} + +static struct ring_buffer_event * +rb_reserve_next(struct hyp_rb_per_cpu *cpu_buffer, unsigned long length) +{ + unsigned long ts_ext_size = 0, event_size = rb_event_size(length); + struct hyp_buffer_page *tail_page = cpu_buffer->tail_page; + struct ring_buffer_event *event; + unsigned long write, prev_write; + u64 ts, time_delta; + + ts = trace_clock(); + + time_delta = ts - atomic64_read(&cpu_buffer->write_stamp); + + if (test_time_stamp(time_delta)) + ts_ext_size = 8; + + prev_write = atomic_read(&tail_page->write); + write = prev_write + event_size + ts_ext_size; + + if (unlikely(write > BUF_EXT_PAGE_SIZE)) + tail_page = rb_move_tail(cpu_buffer); + + if (!atomic_read(&tail_page->entries)) { + tail_page->page->time_stamp = ts; + time_delta = 0; + ts_ext_size = 0; + write = event_size; + prev_write = 0; + } + + atomic_set(&tail_page->write, write); + atomic_inc(&tail_page->entries); + + local_set(&tail_page->page->commit, write); + + atomic_inc(&cpu_buffer->nr_entries); + atomic64_set(&cpu_buffer->write_stamp, ts); + + event = (struct ring_buffer_event *)(tail_page->page->data + + prev_write); + if (ts_ext_size) { + event = rb_add_ts_extend(event, time_delta); + time_delta = 0; + } + + event->type_len = 0; + event->time_delta = time_delta; + event->array[0] = event_size - RB_EVNT_HDR_SIZE; + + return event; +} + +void * +rb_reserve_trace_entry(struct hyp_rb_per_cpu *cpu_buffer, unsigned long length) +{ + struct ring_buffer_event *rb_event; + + rb_event = rb_reserve_next(cpu_buffer, length); + + return &rb_event->array[1]; +} + +static int rb_update_footers(struct hyp_rb_per_cpu *cpu_buffer) +{ + unsigned long entries, pages_touched, overrun; + struct rb_ext_page_footer *footer; + struct buffer_data_page *reader; + + if (!rb_set_head_page(cpu_buffer)) + return -ENODEV; + + reader = cpu_buffer->reader_page->page; + footer = rb_ext_page_get_footer(reader); + + entries = atomic_read(&cpu_buffer->nr_entries); + footer->stats.entries = entries; + pages_touched = atomic_read(&cpu_buffer->pages_touched); + footer->stats.pages_touched = pages_touched; + overrun = atomic_read(&cpu_buffer->overrun); + footer->stats.overrun = overrun; + + return 0; +} + +static int rb_page_init(struct hyp_buffer_page *bpage, unsigned long hva) +{ + void *hyp_va = (void *)kern_hyp_va(hva); + int ret; + + ret = hyp_pin_shared_mem(hyp_va, hyp_va + PAGE_SIZE); + if (ret) + return ret; + + INIT_LIST_HEAD(&bpage->list); + bpage->page = (struct buffer_data_page *)hyp_va; + + atomic_set(&bpage->write, 0); + + rb_footer_reader_status(bpage, 0); + rb_footer_writer_status(bpage, 0); + + return 0; +} + +static void rb_cpu_teardown(struct hyp_rb_per_cpu *cpu_buffer) +{ + unsigned int prev_status; + int i; + + /* Wait for release of the buffer */ + do { + prev_status = atomic_cmpxchg_acquire(&cpu_buffer->status, + HYP_RB_READY, + HYP_RB_UNUSED); + } while (prev_status == HYP_RB_WRITE); + + if (prev_status == HYP_RB_READY) + rb_update_footers(cpu_buffer); + + for (i = 0; i < cpu_buffer->nr_pages; i++) { + struct hyp_buffer_page *bpage = &cpu_buffer->bpages[i]; + + if (!bpage->page) + continue; + + hyp_unpin_shared_mem((void *)bpage->page, + (void *)bpage->page + PAGE_SIZE); + } +} + +static bool rb_cpu_fits_backing(unsigned long nr_pages, + struct hyp_buffer_page *start) +{ + unsigned long max = hyp_buffer_pages_backing.start + + hyp_buffer_pages_backing.size; + struct hyp_buffer_page *end = start + nr_pages; + + return (unsigned long)end <= max; +} + +static bool rb_cpu_fits_pack(struct ring_buffer_pack *rb_pack, + unsigned long pack_end) +{ + unsigned long *end; + + /* Check we can at least read nr_pages */ + if ((unsigned long)&rb_pack->nr_pages >= pack_end) + return false; + + end = &rb_pack->page_va[rb_pack->nr_pages]; + + return (unsigned long)end <= pack_end; +} + +static int rb_cpu_init(struct ring_buffer_pack *rb_pack, struct hyp_buffer_page *start, + struct hyp_rb_per_cpu *cpu_buffer) +{ + struct hyp_buffer_page *bpage = start; + int i, ret; + + if (!rb_pack->nr_pages || + !rb_cpu_fits_backing(rb_pack->nr_pages + 1, start)) + return -EINVAL; + + memset(cpu_buffer, 0, sizeof(*cpu_buffer)); + + cpu_buffer->bpages = start; + cpu_buffer->nr_pages = rb_pack->nr_pages + 1; + + /* The reader page is not part of the ring initially */ + ret = rb_page_init(bpage, rb_pack->reader_page_va); + if (ret) + return ret; + + cpu_buffer->reader_page = bpage; + cpu_buffer->tail_page = bpage + 1; + cpu_buffer->head_page = bpage + 1; + + for (i = 0; i < rb_pack->nr_pages; i++) { + ret = rb_page_init(++bpage, rb_pack->page_va[i]); + if (ret) + goto err; + + bpage->list.next = &(bpage + 1)->list; + bpage->list.prev = &(bpage - 1)->list; + } + + /* Close the ring */ + bpage->list.next = &cpu_buffer->tail_page->list; + cpu_buffer->tail_page->list.prev = &bpage->list; + + /* The last init'ed page points to the head page */ + rb_set_flag(bpage, HYP_RB_PAGE_HEAD); + + rb_footer_reader_status(cpu_buffer->reader_page, RB_PAGE_FT_READER); + rb_footer_reader_status(cpu_buffer->head_page, RB_PAGE_FT_HEAD); + rb_footer_writer_status(cpu_buffer->head_page, RB_PAGE_FT_COMMIT); + + atomic_set(&cpu_buffer->overrun, 0); + atomic64_set(&cpu_buffer->write_stamp, 0); + + /* + * Paired with __start_write_hyp_rb() + */ + atomic_set_release(&cpu_buffer->status, HYP_RB_READY); + + return 0; +err: + rb_cpu_teardown(cpu_buffer); + + return ret; +} + +static int rb_setup_bpage_backing(struct hyp_trace_pack *pack) +{ + unsigned long start = kern_hyp_va(pack->backing.start); + size_t size = pack->backing.size; + int ret; + + if (hyp_buffer_pages_backing.size) + return -EBUSY; + + if (!PAGE_ALIGNED(start) || !PAGE_ALIGNED(size)) + return -EINVAL; + + ret = __pkvm_host_donate_hyp(hyp_virt_to_pfn((void *)start), size >> PAGE_SHIFT); + if (ret) + return ret; + + memset((void *)start, 0, size); + + hyp_buffer_pages_backing.start = start; + hyp_buffer_pages_backing.size = size; + + return 0; +} + +static void rb_teardown_bpage_backing(void) +{ + unsigned long start = hyp_buffer_pages_backing.start; + size_t size = hyp_buffer_pages_backing.size; + + if (!size) + return; + + memset((void *)start, 0, size); + + WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(start), size >> PAGE_SHIFT)); + + hyp_buffer_pages_backing.start = 0; + hyp_buffer_pages_backing.size = 0; +} + +int __pkvm_rb_update_footers(int cpu) +{ + struct hyp_rb_per_cpu *cpu_buffer; + int ret = 0; + + /* TODO: per-CPU lock for */ + hyp_spin_lock(&trace_rb_lock); + + if (cpu >= hyp_nr_cpus) { + ret = -EINVAL; + goto err; + } + + cpu_buffer = per_cpu_ptr(&trace_rb, cpu); + + if (atomic_read(&cpu_buffer->status) == HYP_RB_UNUSED) { + ret = -ENODEV; + goto err; + } + + ret = rb_update_footers(cpu_buffer); +err: + hyp_spin_unlock(&trace_rb_lock); + + return ret; +} + +int __pkvm_rb_swap_reader_page(int cpu) +{ + struct hyp_rb_per_cpu *cpu_buffer = per_cpu_ptr(&trace_rb, cpu); + int ret = 0; + + /* TODO: per-CPU lock for */ + hyp_spin_lock(&trace_rb_lock); + + if (cpu >= hyp_nr_cpus) { + ret = -EINVAL; + goto err; + } + cpu_buffer = per_cpu_ptr(&trace_rb, cpu); + + if (atomic_read(&cpu_buffer->status) == HYP_RB_UNUSED) { + ret = -ENODEV; + goto err; + } + + ret = rb_swap_reader_page(cpu_buffer); + if (ret) + goto err; +err: + hyp_spin_unlock(&trace_rb_lock); + + return ret; +} + +static void __pkvm_stop_tracing_locked(void) +{ + int cpu; + + hyp_assert_lock_held(&trace_rb_lock); + + for (cpu = 0; cpu < hyp_nr_cpus; cpu++) { + struct hyp_rb_per_cpu *cpu_buffer = per_cpu_ptr(&trace_rb, cpu); + + if (atomic_read(&cpu_buffer->status) == HYP_RB_UNUSED) + continue; + + rb_cpu_teardown(cpu_buffer); + } + + rb_teardown_bpage_backing(); +} + +void __pkvm_stop_tracing(void) +{ + hyp_spin_lock(&trace_rb_lock); + __pkvm_stop_tracing_locked(); + hyp_spin_unlock(&trace_rb_lock); +} + +int __pkvm_start_tracing(unsigned long pack_hva, size_t pack_size) +{ + struct hyp_trace_pack *pack = (struct hyp_trace_pack *)kern_hyp_va(pack_hva); + struct trace_buffer_pack *trace_pack = &pack->trace_buffer_pack; + struct hyp_buffer_page *bpage_backing_start; + struct ring_buffer_pack *rb_pack; + int ret, cpu; + + if (!pack_size || !PAGE_ALIGNED(pack_hva) || !PAGE_ALIGNED(pack_size)) + return -EINVAL; + + ret = __pkvm_host_donate_hyp(hyp_virt_to_pfn((void *)pack), + pack_size >> PAGE_SHIFT); + if (ret) + return ret; + + hyp_spin_lock(&trace_rb_lock); + + ret = rb_setup_bpage_backing(pack); + if (ret) + goto err; + + trace_clock_update(&pack->trace_clock_data); + + bpage_backing_start = (struct hyp_buffer_page *)hyp_buffer_pages_backing.start; + + for_each_ring_buffer_pack(rb_pack, cpu, trace_pack) { + struct hyp_rb_per_cpu *cpu_buffer; + int cpu; + + ret = -EINVAL; + if (!rb_cpu_fits_pack(rb_pack, pack_hva + pack_size)) + break; + + cpu = rb_pack->cpu; + if (cpu >= hyp_nr_cpus) + break; + + cpu_buffer = per_cpu_ptr(&trace_rb, cpu); + + ret = rb_cpu_init(rb_pack, bpage_backing_start, cpu_buffer); + if (ret) + break; + + /* reader page + nr pages in rb */ + bpage_backing_start += 1 + rb_pack->nr_pages; + } +err: + if (ret) + __pkvm_stop_tracing_locked(); + + hyp_spin_unlock(&trace_rb_lock); + + WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn((void *)pack), + pack_size >> PAGE_SHIFT)); + return ret; +}