diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 6286bf2a8615..7df0b6ea7e06 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -96,6 +96,10 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___pkvm_iommu_register, __KVM_HOST_SMCCC_FUNC___pkvm_iommu_pm_notify, __KVM_HOST_SMCCC_FUNC___pkvm_iommu_finalize, + __KVM_HOST_SMCCC_FUNC___pkvm_start_tracing, + __KVM_HOST_SMCCC_FUNC___pkvm_stop_tracing, + __KVM_HOST_SMCCC_FUNC___pkvm_rb_swap_reader_page, + __KVM_HOST_SMCCC_FUNC___pkvm_rb_update_footers, /* * Start of the dynamically registered hypercalls. Start a bit diff --git a/arch/arm64/include/asm/kvm_hyptrace.h b/arch/arm64/include/asm/kvm_hyptrace.h new file mode 100644 index 000000000000..d32b445b4cf6 --- /dev/null +++ b/arch/arm64/include/asm/kvm_hyptrace.h @@ -0,0 +1,21 @@ +#ifndef __ARM64_KVM_HYPTRACE_H_ +#define __ARM64_KVM_HYPTRACE_H_ +#include + +#include + +/* + * Host donations to the hypervisor to store the struct hyp_buffer_page. + */ +struct hyp_buffer_pages_backing { + unsigned long start; + size_t size; +}; + +struct hyp_trace_pack { + struct hyp_buffer_pages_backing backing; + struct kvm_nvhe_clock_data trace_clock_data; + struct trace_buffer_pack trace_buffer_pack; + +}; +#endif diff --git a/arch/arm64/kvm/hyp/include/nvhe/trace.h b/arch/arm64/kvm/hyp/include/nvhe/trace.h new file mode 100644 index 000000000000..18d3c404a4a3 --- /dev/null +++ b/arch/arm64/kvm/hyp/include/nvhe/trace.h @@ -0,0 +1,78 @@ +#ifndef __ARM64_KVM_HYP_NVHE_TRACE_H +#define __ARM64_KVM_HYP_NVHE_TRACE_H +#include + +#include +#include + +#ifdef CONFIG_TRACING + +struct hyp_buffer_page { + struct list_head list; + struct buffer_data_page *page; + atomic_t write; + atomic_t entries; +}; + +#define HYP_RB_UNUSED 0 +#define HYP_RB_READY 1 +#define HYP_RB_WRITE 2 + +struct hyp_rb_per_cpu { + struct hyp_buffer_page *tail_page; + struct hyp_buffer_page *reader_page; + struct hyp_buffer_page *head_page; + struct hyp_buffer_page *bpages; + unsigned long nr_pages; + atomic64_t write_stamp; + atomic_t pages_touched; + atomic_t nr_entries; + atomic_t status; + atomic_t overrun; +}; + +static inline bool __start_write_hyp_rb(struct hyp_rb_per_cpu *rb) +{ + /* + * Paired with rb_cpu_init() + */ + return atomic_cmpxchg_acquire(&rb->status, HYP_RB_READY, HYP_RB_WRITE) + != HYP_RB_UNUSED; +} + +static inline void __stop_write_hyp_rb(struct hyp_rb_per_cpu *rb) +{ + /* + * Paired with rb_cpu_teardown() + */ + atomic_set_release(&rb->status, HYP_RB_READY); +} + +struct hyp_rb_per_cpu; +DECLARE_PER_CPU(struct hyp_rb_per_cpu, trace_rb); + +void *rb_reserve_trace_entry(struct hyp_rb_per_cpu *cpu_buffer, unsigned long length); + +int __pkvm_start_tracing(unsigned long pack_va, size_t pack_size); +void __pkvm_stop_tracing(void); +int __pkvm_rb_swap_reader_page(int cpu); +int __pkvm_rb_update_footers(int cpu); +#else +static inline int __pkvm_start_tracing(unsigned long pack_va, size_t pack_size) +{ + return -ENODEV; +} + +static inline void __pkvm_stop_tracing(void) { } + +static inline int __pkvm_rb_swap_reader_page(int cpu) +{ + return -ENODEV; +} + +static inline int __pkvm_rb_update_footers(int cpu) +{ + return -ENODEV; +} +#endif +#endif diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile index bd3ad71d432a..64032a2c2202 100644 --- a/arch/arm64/kvm/hyp/nvhe/Makefile +++ b/arch/arm64/kvm/hyp/nvhe/Makefile @@ -8,7 +8,7 @@ hyp-obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o serial.o hyp-obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \ ../fpsimd.o ../hyp-entry.o ../exception.o ../pgtable.o -hyp-obj-$(CONFIG_TRACING) += clock.o +hyp-obj-$(CONFIG_TRACING) += clock.o trace.o hyp-obj-$(CONFIG_DEBUG_LIST) += list_debug.o hyp-obj-$(CONFIG_MODULES) += modules.o hyp-obj-y += $(lib-objs) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 8a9681874dd6..7bf26f8f082f 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -1208,6 +1209,35 @@ handle___pkvm_close_module_registration(struct kvm_cpu_context *host_ctxt) cpu_reg(host_ctxt, 1) = __pkvm_close_module_registration(); } +static void handle___pkvm_start_tracing(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(unsigned long, pack_hva, host_ctxt, 1); + DECLARE_REG(size_t, pack_size, host_ctxt, 2); + + cpu_reg(host_ctxt, 1) = __pkvm_start_tracing(pack_hva, pack_size); +} + +static void handle___pkvm_stop_tracing(struct kvm_cpu_context *host_ctxt) +{ + __pkvm_stop_tracing(); + + cpu_reg(host_ctxt, 1) = 0; +} + +static void handle___pkvm_rb_swap_reader_page(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(int, cpu, host_ctxt, 1); + + cpu_reg(host_ctxt, 1) = __pkvm_rb_swap_reader_page(cpu); +} + +static void handle___pkvm_rb_update_footers(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(int, cpu, host_ctxt, 1); + + cpu_reg(host_ctxt, 1) = __pkvm_rb_update_footers(cpu); +} + typedef void (*hcall_t)(struct kvm_cpu_context *); #define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x @@ -1253,6 +1283,10 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__pkvm_iommu_register), HANDLE_FUNC(__pkvm_iommu_pm_notify), HANDLE_FUNC(__pkvm_iommu_finalize), + HANDLE_FUNC(__pkvm_start_tracing), + HANDLE_FUNC(__pkvm_stop_tracing), + HANDLE_FUNC(__pkvm_rb_swap_reader_page), + HANDLE_FUNC(__pkvm_rb_update_footers), }; unsigned long pkvm_priv_hcall_limit __ro_after_init = __KVM_HOST_SMCCC_FUNC___pkvm_prot_finalize; diff --git a/arch/arm64/kvm/hyp/nvhe/trace.c b/arch/arm64/kvm/hyp/nvhe/trace.c new file mode 100644 index 000000000000..714433b684a0 --- /dev/null +++ b/arch/arm64/kvm/hyp/nvhe/trace.c @@ -0,0 +1,596 @@ +#include +#include +#include +#include + +#include +#include + +#include + +#define HYP_RB_PAGE_HEAD 1UL +#define HYP_RB_PAGE_UPDATE 2UL +#define HYP_RB_FLAG_MASK 3UL + +static struct hyp_buffer_pages_backing hyp_buffer_pages_backing; +DEFINE_PER_CPU(struct hyp_rb_per_cpu, trace_rb); +DEFINE_HYP_SPINLOCK(trace_rb_lock); + +static bool rb_set_flag(struct hyp_buffer_page *bpage, int new_flag) +{ + unsigned long ret, val = (unsigned long)bpage->list.next; + + ret = cmpxchg((unsigned long *)&bpage->list.next, + val, (val & ~HYP_RB_FLAG_MASK) | new_flag); + + return ret == val; +} + +static void rb_set_footer_status(struct hyp_buffer_page *bpage, + unsigned long status, + bool reader) +{ + struct buffer_data_page *page = bpage->page; + struct rb_ext_page_footer *footer; + + footer = rb_ext_page_get_footer(page); + + if (reader) + atomic_set(&footer->reader_status, status); + else + atomic_set(&footer->writer_status, status); +} + +static void rb_footer_writer_status(struct hyp_buffer_page *bpage, + unsigned long status) +{ + rb_set_footer_status(bpage, status, false); +} + +static void rb_footer_reader_status(struct hyp_buffer_page *bpage, + unsigned long status) +{ + rb_set_footer_status(bpage, status, true); +} + +static struct hyp_buffer_page *rb_hyp_buffer_page(struct list_head *list) +{ + unsigned long ptr = (unsigned long)list & ~HYP_RB_FLAG_MASK; + + return container_of((struct list_head *)ptr, struct hyp_buffer_page, list); +} + +static struct hyp_buffer_page *rb_next_page(struct hyp_buffer_page *bpage) +{ + return rb_hyp_buffer_page(bpage->list.next); +} + +static bool rb_is_head_page(struct hyp_buffer_page *bpage) +{ + return (unsigned long)bpage->list.prev->next & HYP_RB_PAGE_HEAD; +} + +static struct hyp_buffer_page *rb_set_head_page(struct hyp_rb_per_cpu *cpu_buffer) +{ + struct hyp_buffer_page *bpage, *prev_head; + int cnt = 0; +again: + bpage = prev_head = cpu_buffer->head_page; + do { + if (rb_is_head_page(bpage)) { + cpu_buffer->head_page = bpage; + rb_footer_reader_status(prev_head, 0); + rb_footer_reader_status(bpage, RB_PAGE_FT_HEAD); + return bpage; + } + + bpage = rb_next_page(bpage); + } while (bpage != prev_head); + + cnt++; + + /* We might have race with the writer let's try again */ + if (cnt < 3) + goto again; + + return NULL; +} + +static int rb_swap_reader_page(struct hyp_rb_per_cpu *cpu_buffer) +{ + unsigned long *old_head_link, old_link_val, new_link_val, overrun; + struct hyp_buffer_page *head, *reader = cpu_buffer->reader_page; + struct rb_ext_page_footer *footer; + + rb_footer_reader_status(cpu_buffer->reader_page, 0); +spin: + /* Update the cpu_buffer->header_page according to HYP_RB_PAGE_HEAD */ + head = rb_set_head_page(cpu_buffer); + if (!head) + return -ENODEV; + + /* Connect the reader page around the header page */ + reader->list.next = head->list.next; + reader->list.prev = head->list.prev; + + /* The reader page points to the new header page */ + rb_set_flag(reader, HYP_RB_PAGE_HEAD); + + /* + * Paired with the cmpxchg in rb_move_tail(). Order the read of the head + * page and overrun. + */ + smp_mb(); + overrun = atomic_read(&cpu_buffer->overrun); + + /* Try to swap the prev head link to the reader page */ + old_head_link = (unsigned long *)&reader->list.prev->next; + old_link_val = (*old_head_link & ~HYP_RB_FLAG_MASK) | HYP_RB_PAGE_HEAD; + new_link_val = (unsigned long)&reader->list; + if (cmpxchg(old_head_link, old_link_val, new_link_val) + != old_link_val) + goto spin; + + cpu_buffer->head_page = rb_hyp_buffer_page(reader->list.next); + cpu_buffer->head_page->list.prev = &reader->list; + cpu_buffer->reader_page = head; + + rb_footer_reader_status(cpu_buffer->reader_page, RB_PAGE_FT_READER); + rb_footer_reader_status(cpu_buffer->head_page, RB_PAGE_FT_HEAD); + + footer = rb_ext_page_get_footer(cpu_buffer->reader_page->page); + footer->stats.overrun = overrun; + + return 0; +} + +static struct hyp_buffer_page * +rb_move_tail(struct hyp_rb_per_cpu *cpu_buffer) +{ + struct hyp_buffer_page *tail_page, *new_tail, *new_head; + + tail_page = cpu_buffer->tail_page; + new_tail = rb_next_page(tail_page); +again: + /* + * We caught the reader ... Let's try to move the head page. + * The writer can only rely on ->next links to check if this is head. + */ + if ((unsigned long)tail_page->list.next & HYP_RB_PAGE_HEAD) { + /* The reader moved the head in between */ + if (!rb_set_flag(tail_page, HYP_RB_PAGE_UPDATE)) + goto again; + + atomic_add(atomic_read(&new_tail->entries), &cpu_buffer->overrun); + + /* Move the head */ + rb_set_flag(new_tail, HYP_RB_PAGE_HEAD); + + /* The new head is in place, reset the update flag */ + rb_set_flag(tail_page, 0); + + new_head = rb_next_page(new_tail); + } + + rb_footer_writer_status(tail_page, 0); + rb_footer_writer_status(new_tail, RB_PAGE_FT_COMMIT); + + local_set(&new_tail->page->commit, 0); + + atomic_set(&new_tail->write, 0); + atomic_set(&new_tail->entries, 0); + + atomic_inc(&cpu_buffer->pages_touched); + + cpu_buffer->tail_page = new_tail; + + return new_tail; +} + +unsigned long rb_event_size(unsigned long length) +{ + struct ring_buffer_event *event; + + return length + RB_EVNT_HDR_SIZE + sizeof(event->array[0]); +} + +static struct ring_buffer_event * +rb_add_ts_extend(struct ring_buffer_event *event, u64 delta) +{ + event->type_len = RINGBUF_TYPE_TIME_EXTEND; + event->time_delta = delta & TS_MASK; + event->array[0] = delta >> TS_SHIFT; + + return (struct ring_buffer_event *)((unsigned long)event + 8); +} + +static struct ring_buffer_event * +rb_reserve_next(struct hyp_rb_per_cpu *cpu_buffer, unsigned long length) +{ + unsigned long ts_ext_size = 0, event_size = rb_event_size(length); + struct hyp_buffer_page *tail_page = cpu_buffer->tail_page; + struct ring_buffer_event *event; + unsigned long write, prev_write; + u64 ts, time_delta; + + ts = trace_clock(); + + time_delta = ts - atomic64_read(&cpu_buffer->write_stamp); + + if (test_time_stamp(time_delta)) + ts_ext_size = 8; + + prev_write = atomic_read(&tail_page->write); + write = prev_write + event_size + ts_ext_size; + + if (unlikely(write > BUF_EXT_PAGE_SIZE)) + tail_page = rb_move_tail(cpu_buffer); + + if (!atomic_read(&tail_page->entries)) { + tail_page->page->time_stamp = ts; + time_delta = 0; + ts_ext_size = 0; + write = event_size; + prev_write = 0; + } + + atomic_set(&tail_page->write, write); + atomic_inc(&tail_page->entries); + + local_set(&tail_page->page->commit, write); + + atomic_inc(&cpu_buffer->nr_entries); + atomic64_set(&cpu_buffer->write_stamp, ts); + + event = (struct ring_buffer_event *)(tail_page->page->data + + prev_write); + if (ts_ext_size) { + event = rb_add_ts_extend(event, time_delta); + time_delta = 0; + } + + event->type_len = 0; + event->time_delta = time_delta; + event->array[0] = event_size - RB_EVNT_HDR_SIZE; + + return event; +} + +void * +rb_reserve_trace_entry(struct hyp_rb_per_cpu *cpu_buffer, unsigned long length) +{ + struct ring_buffer_event *rb_event; + + rb_event = rb_reserve_next(cpu_buffer, length); + + return &rb_event->array[1]; +} + +static int rb_update_footers(struct hyp_rb_per_cpu *cpu_buffer) +{ + unsigned long entries, pages_touched, overrun; + struct rb_ext_page_footer *footer; + struct buffer_data_page *reader; + + if (!rb_set_head_page(cpu_buffer)) + return -ENODEV; + + reader = cpu_buffer->reader_page->page; + footer = rb_ext_page_get_footer(reader); + + entries = atomic_read(&cpu_buffer->nr_entries); + footer->stats.entries = entries; + pages_touched = atomic_read(&cpu_buffer->pages_touched); + footer->stats.pages_touched = pages_touched; + overrun = atomic_read(&cpu_buffer->overrun); + footer->stats.overrun = overrun; + + return 0; +} + +static int rb_page_init(struct hyp_buffer_page *bpage, unsigned long hva) +{ + void *hyp_va = (void *)kern_hyp_va(hva); + int ret; + + ret = hyp_pin_shared_mem(hyp_va, hyp_va + PAGE_SIZE); + if (ret) + return ret; + + INIT_LIST_HEAD(&bpage->list); + bpage->page = (struct buffer_data_page *)hyp_va; + + atomic_set(&bpage->write, 0); + + rb_footer_reader_status(bpage, 0); + rb_footer_writer_status(bpage, 0); + + return 0; +} + +static void rb_cpu_teardown(struct hyp_rb_per_cpu *cpu_buffer) +{ + unsigned int prev_status; + int i; + + /* Wait for release of the buffer */ + do { + prev_status = atomic_cmpxchg_acquire(&cpu_buffer->status, + HYP_RB_READY, + HYP_RB_UNUSED); + } while (prev_status == HYP_RB_WRITE); + + if (prev_status == HYP_RB_READY) + rb_update_footers(cpu_buffer); + + for (i = 0; i < cpu_buffer->nr_pages; i++) { + struct hyp_buffer_page *bpage = &cpu_buffer->bpages[i]; + + if (!bpage->page) + continue; + + hyp_unpin_shared_mem((void *)bpage->page, + (void *)bpage->page + PAGE_SIZE); + } +} + +static bool rb_cpu_fits_backing(unsigned long nr_pages, + struct hyp_buffer_page *start) +{ + unsigned long max = hyp_buffer_pages_backing.start + + hyp_buffer_pages_backing.size; + struct hyp_buffer_page *end = start + nr_pages; + + return (unsigned long)end <= max; +} + +static bool rb_cpu_fits_pack(struct ring_buffer_pack *rb_pack, + unsigned long pack_end) +{ + unsigned long *end; + + /* Check we can at least read nr_pages */ + if ((unsigned long)&rb_pack->nr_pages >= pack_end) + return false; + + end = &rb_pack->page_va[rb_pack->nr_pages]; + + return (unsigned long)end <= pack_end; +} + +static int rb_cpu_init(struct ring_buffer_pack *rb_pack, struct hyp_buffer_page *start, + struct hyp_rb_per_cpu *cpu_buffer) +{ + struct hyp_buffer_page *bpage = start; + int i, ret; + + if (!rb_pack->nr_pages || + !rb_cpu_fits_backing(rb_pack->nr_pages + 1, start)) + return -EINVAL; + + memset(cpu_buffer, 0, sizeof(*cpu_buffer)); + + cpu_buffer->bpages = start; + cpu_buffer->nr_pages = rb_pack->nr_pages + 1; + + /* The reader page is not part of the ring initially */ + ret = rb_page_init(bpage, rb_pack->reader_page_va); + if (ret) + return ret; + + cpu_buffer->reader_page = bpage; + cpu_buffer->tail_page = bpage + 1; + cpu_buffer->head_page = bpage + 1; + + for (i = 0; i < rb_pack->nr_pages; i++) { + ret = rb_page_init(++bpage, rb_pack->page_va[i]); + if (ret) + goto err; + + bpage->list.next = &(bpage + 1)->list; + bpage->list.prev = &(bpage - 1)->list; + } + + /* Close the ring */ + bpage->list.next = &cpu_buffer->tail_page->list; + cpu_buffer->tail_page->list.prev = &bpage->list; + + /* The last init'ed page points to the head page */ + rb_set_flag(bpage, HYP_RB_PAGE_HEAD); + + rb_footer_reader_status(cpu_buffer->reader_page, RB_PAGE_FT_READER); + rb_footer_reader_status(cpu_buffer->head_page, RB_PAGE_FT_HEAD); + rb_footer_writer_status(cpu_buffer->head_page, RB_PAGE_FT_COMMIT); + + atomic_set(&cpu_buffer->overrun, 0); + atomic64_set(&cpu_buffer->write_stamp, 0); + + /* + * Paired with __start_write_hyp_rb() + */ + atomic_set_release(&cpu_buffer->status, HYP_RB_READY); + + return 0; +err: + rb_cpu_teardown(cpu_buffer); + + return ret; +} + +static int rb_setup_bpage_backing(struct hyp_trace_pack *pack) +{ + unsigned long start = kern_hyp_va(pack->backing.start); + size_t size = pack->backing.size; + int ret; + + if (hyp_buffer_pages_backing.size) + return -EBUSY; + + if (!PAGE_ALIGNED(start) || !PAGE_ALIGNED(size)) + return -EINVAL; + + ret = __pkvm_host_donate_hyp(hyp_virt_to_pfn((void *)start), size >> PAGE_SHIFT); + if (ret) + return ret; + + memset((void *)start, 0, size); + + hyp_buffer_pages_backing.start = start; + hyp_buffer_pages_backing.size = size; + + return 0; +} + +static void rb_teardown_bpage_backing(void) +{ + unsigned long start = hyp_buffer_pages_backing.start; + size_t size = hyp_buffer_pages_backing.size; + + if (!size) + return; + + memset((void *)start, 0, size); + + WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(start), size >> PAGE_SHIFT)); + + hyp_buffer_pages_backing.start = 0; + hyp_buffer_pages_backing.size = 0; +} + +int __pkvm_rb_update_footers(int cpu) +{ + struct hyp_rb_per_cpu *cpu_buffer; + int ret = 0; + + /* TODO: per-CPU lock for */ + hyp_spin_lock(&trace_rb_lock); + + if (cpu >= hyp_nr_cpus) { + ret = -EINVAL; + goto err; + } + + cpu_buffer = per_cpu_ptr(&trace_rb, cpu); + + if (atomic_read(&cpu_buffer->status) == HYP_RB_UNUSED) { + ret = -ENODEV; + goto err; + } + + ret = rb_update_footers(cpu_buffer); +err: + hyp_spin_unlock(&trace_rb_lock); + + return ret; +} + +int __pkvm_rb_swap_reader_page(int cpu) +{ + struct hyp_rb_per_cpu *cpu_buffer = per_cpu_ptr(&trace_rb, cpu); + int ret = 0; + + /* TODO: per-CPU lock for */ + hyp_spin_lock(&trace_rb_lock); + + if (cpu >= hyp_nr_cpus) { + ret = -EINVAL; + goto err; + } + cpu_buffer = per_cpu_ptr(&trace_rb, cpu); + + if (atomic_read(&cpu_buffer->status) == HYP_RB_UNUSED) { + ret = -ENODEV; + goto err; + } + + ret = rb_swap_reader_page(cpu_buffer); + if (ret) + goto err; +err: + hyp_spin_unlock(&trace_rb_lock); + + return ret; +} + +static void __pkvm_stop_tracing_locked(void) +{ + int cpu; + + hyp_assert_lock_held(&trace_rb_lock); + + for (cpu = 0; cpu < hyp_nr_cpus; cpu++) { + struct hyp_rb_per_cpu *cpu_buffer = per_cpu_ptr(&trace_rb, cpu); + + if (atomic_read(&cpu_buffer->status) == HYP_RB_UNUSED) + continue; + + rb_cpu_teardown(cpu_buffer); + } + + rb_teardown_bpage_backing(); +} + +void __pkvm_stop_tracing(void) +{ + hyp_spin_lock(&trace_rb_lock); + __pkvm_stop_tracing_locked(); + hyp_spin_unlock(&trace_rb_lock); +} + +int __pkvm_start_tracing(unsigned long pack_hva, size_t pack_size) +{ + struct hyp_trace_pack *pack = (struct hyp_trace_pack *)kern_hyp_va(pack_hva); + struct trace_buffer_pack *trace_pack = &pack->trace_buffer_pack; + struct hyp_buffer_page *bpage_backing_start; + struct ring_buffer_pack *rb_pack; + int ret, cpu; + + if (!pack_size || !PAGE_ALIGNED(pack_hva) || !PAGE_ALIGNED(pack_size)) + return -EINVAL; + + ret = __pkvm_host_donate_hyp(hyp_virt_to_pfn((void *)pack), + pack_size >> PAGE_SHIFT); + if (ret) + return ret; + + hyp_spin_lock(&trace_rb_lock); + + ret = rb_setup_bpage_backing(pack); + if (ret) + goto err; + + trace_clock_update(&pack->trace_clock_data); + + bpage_backing_start = (struct hyp_buffer_page *)hyp_buffer_pages_backing.start; + + for_each_ring_buffer_pack(rb_pack, cpu, trace_pack) { + struct hyp_rb_per_cpu *cpu_buffer; + int cpu; + + ret = -EINVAL; + if (!rb_cpu_fits_pack(rb_pack, pack_hva + pack_size)) + break; + + cpu = rb_pack->cpu; + if (cpu >= hyp_nr_cpus) + break; + + cpu_buffer = per_cpu_ptr(&trace_rb, cpu); + + ret = rb_cpu_init(rb_pack, bpage_backing_start, cpu_buffer); + if (ret) + break; + + /* reader page + nr pages in rb */ + bpage_backing_start += 1 + rb_pack->nr_pages; + } +err: + if (ret) + __pkvm_stop_tracing_locked(); + + hyp_spin_unlock(&trace_rb_lock); + + WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn((void *)pack), + pack_size >> PAGE_SHIFT)); + return ret; +}