ANDROID: KVM: arm64: Add support for nVHE hyp events

Following the introduction of tracing buffers for the nVHE hypervisor,
add the support to declare in-hypervisor events.

Hypervisor events are declared into kvm_hypevents.h and can be called
with trace_<event_name>().

hyp_enter and hyp_exit events are provided as an example.

Bug: 229972309
Co-authored-By: Nikita Ioffe <ioffe@google.com>
Change-Id: I42d110fece793112d30530154aab49049b7fa520
Signed-off-by: Vincent Donnefort <vdonnefort@google.com>
Signed-off-by: Nikita Ioffe <ioffe@google.com>
This commit is contained in:
Vincent Donnefort
2023-01-26 14:11:49 +00:00
parent 575dd1da64
commit 089c9082c2
16 changed files with 453 additions and 6 deletions

View File

@@ -100,6 +100,7 @@ enum __kvm_host_smccc_func {
__KVM_HOST_SMCCC_FUNC___pkvm_stop_tracing,
__KVM_HOST_SMCCC_FUNC___pkvm_rb_swap_reader_page,
__KVM_HOST_SMCCC_FUNC___pkvm_rb_update_footers,
__KVM_HOST_SMCCC_FUNC___pkvm_enable_event,
/*
* Start of the dynamically registered hypercalls. Start a bit

View File

@@ -0,0 +1,31 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __ARM64_KVM_HYPEVENTS_H_
#define __ARM64_KVM_HYPEVENTS_H_
#ifdef __KVM_NVHE_HYPERVISOR__
#include <nvhe/trace.h>
#endif
/*
* Hypervisor events definitions.
*/
HYP_EVENT(hyp_enter,
HE_PROTO(void),
HE_STRUCT(
),
HE_ASSIGN(
),
HE_PRINTK(" ")
);
HYP_EVENT(hyp_exit,
HE_PROTO(void),
HE_STRUCT(
),
HE_ASSIGN(
),
HE_PRINTK(" ")
);
#endif

View File

@@ -0,0 +1,30 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __ARM64_KVM_HYPEVENTS_DEFS_H
#define __ARM64_KVM_HYPEVENTS_DEFS_H
struct hyp_event_id {
unsigned short id;
void *data;
};
struct hyp_entry_hdr {
unsigned short id;
};
/*
* Hyp events definitions common to the hyp and the host
*/
#define HYP_EVENT_FORMAT(__name, __struct) \
struct trace_hyp_format_##__name { \
struct hyp_entry_hdr hdr; \
__struct \
}
#define HE_PROTO(args...) args
#define HE_STRUCT(args...) args
#define HE_ASSIGN(args...) args
#define HE_PRINTK(args...) args
#define he_field(type, item) type item;
#endif

View File

@@ -126,6 +126,10 @@ KVM_NVHE_ALIAS(__hyp_data_start);
KVM_NVHE_ALIAS(__hyp_data_end);
KVM_NVHE_ALIAS(__hyp_rodata_start);
KVM_NVHE_ALIAS(__hyp_rodata_end);
#ifdef CONFIG_FTRACE
KVM_NVHE_ALIAS(__hyp_event_ids_start);
KVM_NVHE_ALIAS(__hyp_event_ids_end);
#endif
/* pKVM static key */
KVM_NVHE_ALIAS(kvm_protected_mode_initialized);

View File

@@ -13,12 +13,23 @@
*(__kvm_ex_table) \
__stop___kvm_ex_table = .;
#ifdef CONFIG_TRACING
#define HYPERVISOR_EVENT_IDS \
. = ALIGN(PAGE_SIZE); \
__hyp_event_ids_start = .; \
*(HYP_SECTION_NAME(_hyp_event_ids)) \
__hyp_event_ids_end = .;
#else
#define HYPERVISOR_EVENT_IDS
#endif
#define HYPERVISOR_RODATA_SECTIONS \
HYP_SECTION_NAME(.rodata) : { \
. = ALIGN(PAGE_SIZE); \
__hyp_rodata_start = .; \
*(HYP_SECTION_NAME(.data..ro_after_init)) \
*(HYP_SECTION_NAME(.rodata)) \
HYPERVISOR_EVENT_IDS \
. = ALIGN(PAGE_SIZE); \
__hyp_rodata_end = .; \
}
@@ -51,6 +62,17 @@
. = ALIGN(PAGE_SIZE); \
__hyp_bss_end = .;
#ifdef CONFIG_TRACING
#define HYPERVISOR_EVENTS \
.hyp.events : { \
__start_hyp_events = .; \
*(_hyp_events) \
__stop_hyp_events = .; \
}
#else
#define HYPERVISOR_EVENTS
#endif
/*
* We require that __hyp_bss_start and __bss_start are aligned, and enforce it
* with an assertion. But the BSS_SECTION macro places an empty .sbss section
@@ -64,6 +86,7 @@
#define HYPERVISOR_DATA_SECTION
#define HYPERVISOR_PERCPU_SECTION
#define HYPERVISOR_RELOC_SECTION
#define HYPERVISOR_EVENTS
#define SBSS_ALIGN 0
#endif
@@ -191,6 +214,8 @@ SECTIONS
/* everything from this point to __init_begin will be marked RO NX */
RO_DATA(PAGE_SIZE)
HYPERVISOR_EVENTS
HYPERVISOR_RODATA_SECTIONS
idmap_pg_dir = .;

View File

@@ -26,7 +26,7 @@ kvm-y := $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o $(KVM)/eventfd.o \
kvm-$(CONFIG_HW_PERF_EVENTS) += pmu-emul.o pmu.o
kvm-$(CONFIG_TRACING) += hyp_trace.o
kvm-$(CONFIG_TRACING) += hyp_events.o hyp_trace.o
always-y := hyp_constants.h hyp-constants.s

View File

@@ -2022,6 +2022,8 @@ static void kvm_hyp_init_symbols(void)
kvm_nvhe_sym(smccc_trng_available) = smccc_trng_available;
}
int kvm_hyp_init_events(void);
static int kvm_hyp_init_protection(u32 hyp_va_bits)
{
void *addr = phys_to_virt(hyp_mem_base);
@@ -2209,6 +2211,11 @@ static int init_hyp_mode(void)
kvm_hyp_init_symbols();
/* TODO: Real .h interface */
#ifdef CONFIG_TRACING
kvm_hyp_init_events();
#endif
if (is_protected_kvm_enabled()) {
init_cpu_logical_map();

View File

@@ -1,8 +1,13 @@
#ifndef __ARM64_KVM_HYP_NVHE_TRACE_H
#define __ARM64_KVM_HYP_NVHE_TRACE_H
#include <nvhe/trace.h>
#include <linux/trace_events.h>
#include <linux/ring_buffer.h>
#include <asm/kvm_hyptrace.h>
#include <asm/kvm_hypevents_defs.h>
#include <asm/percpu.h>
#ifdef CONFIG_TRACING
@@ -57,6 +62,30 @@ int __pkvm_start_tracing(unsigned long pack_va, size_t pack_size);
void __pkvm_stop_tracing(void);
int __pkvm_rb_swap_reader_page(int cpu);
int __pkvm_rb_update_footers(int cpu);
int __pkvm_enable_event(unsigned short id, bool enable);
#define HYP_EVENT(__name, __proto, __struct, __assign, __printk) \
HYP_EVENT_FORMAT(__name, __struct); \
extern atomic_t __name##_enabled; \
extern unsigned short hyp_event_id_##__name; \
static inline void trace_##__name(__proto) \
{ \
size_t length = sizeof(struct trace_hyp_format_##__name); \
struct hyp_rb_per_cpu *rb = this_cpu_ptr(&trace_rb); \
struct trace_hyp_format_##__name *__entry; \
\
if (!atomic_read(&__name##_enabled)) \
return; \
if (!__start_write_hyp_rb(rb)) \
return; \
__entry = rb_reserve_trace_entry(rb, length); \
__entry->hdr.id = hyp_event_id_##__name; \
__assign \
__stop_write_hyp_rb(rb); \
}
/* TODO: atomic_t to static_branch */
#else
static inline int __pkvm_start_tracing(unsigned long pack_va, size_t pack_size)
{
@@ -74,5 +103,13 @@ static inline int __pkvm_rb_update_footers(int cpu)
{
return -ENODEV;
}
#define HYP_EVENT(__name, __proto, __struct, __assign, __printk) \
static inline void trace_##__name(__proto) {}
static inline int __pkvm_enable_event(unsigned short id, bool enable)
{
return -ENODEV;
}
#endif
#endif

View File

@@ -8,7 +8,7 @@ hyp-obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o
serial.o
hyp-obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \
../fpsimd.o ../hyp-entry.o ../exception.o ../pgtable.o
hyp-obj-$(CONFIG_TRACING) += clock.o trace.o
hyp-obj-$(CONFIG_TRACING) += clock.o events.o trace.o
hyp-obj-$(CONFIG_DEBUG_LIST) += list_debug.o
hyp-obj-$(CONFIG_MODULES) += modules.o
hyp-obj-y += $(lib-objs)

View File

@@ -0,0 +1,41 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2023 Google LLC
*/
#include <nvhe/trace.h>
extern struct hyp_event_id __hyp_event_ids_start[];
extern struct hyp_event_id __hyp_event_ids_end[];
#undef HYP_EVENT
#define HYP_EVENT(__name, __proto, __struct, __assign, __printk) \
atomic_t __name##_enabled = ATOMIC_INIT(0); \
struct hyp_event_id hyp_event_id_##__name __section("_hyp_event_ids") = { \
.data = (void *)&__name##_enabled, \
}
#include <asm/kvm_hypevents.h>
int __pkvm_enable_event(unsigned short id, bool enable)
{
struct hyp_event_id *event_id = __hyp_event_ids_start;
atomic_t *enable_key;
for (; (unsigned long)event_id < (unsigned long)__hyp_event_ids_end;
event_id++) {
if (event_id->id != id)
continue;
enable_key = (atomic_t *)event_id->data;
if (enable)
atomic_set(enable_key, 1);
else
atomic_set(enable_key, 0);
return 0;
}
return -EINVAL;
}

View File

@@ -13,6 +13,7 @@
#include <asm/kvm_emulate.h>
#include <asm/kvm_host.h>
#include <asm/kvm_hyp.h>
#include <asm/kvm_hypevents.h>
#include <asm/kvm_mmu.h>
#include <nvhe/ffa.h>
@@ -1238,6 +1239,14 @@ static void handle___pkvm_rb_update_footers(struct kvm_cpu_context *host_ctxt)
cpu_reg(host_ctxt, 1) = __pkvm_rb_update_footers(cpu);
}
static void handle___pkvm_enable_event(struct kvm_cpu_context *host_ctxt)
{
DECLARE_REG(unsigned short, id, host_ctxt, 1);
DECLARE_REG(bool, enable, host_ctxt, 2);
cpu_reg(host_ctxt, 1) = __pkvm_enable_event(id, enable);
}
typedef void (*hcall_t)(struct kvm_cpu_context *);
#define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x
@@ -1287,6 +1296,7 @@ static const hcall_t host_hcall[] = {
HANDLE_FUNC(__pkvm_stop_tracing),
HANDLE_FUNC(__pkvm_rb_swap_reader_page),
HANDLE_FUNC(__pkvm_rb_update_footers),
HANDLE_FUNC(__pkvm_enable_event),
};
unsigned long pkvm_priv_hcall_limit __ro_after_init = __KVM_HOST_SMCCC_FUNC___pkvm_prot_finalize;
@@ -1368,6 +1378,8 @@ void handle_trap(struct kvm_cpu_context *host_ctxt)
{
u64 esr = read_sysreg_el2(SYS_ESR);
trace_hyp_enter();
switch (ESR_ELx_EC(esr)) {
case ESR_ELx_EC_HVC64:
handle_host_hcall(host_ctxt);
@@ -1386,4 +1398,6 @@ void handle_trap(struct kvm_cpu_context *host_ctxt)
default:
BUG_ON(!READ_ONCE(default_trap_handler) || !default_trap_handler(host_ctxt));
}
trace_hyp_exit();
}

View File

@@ -16,6 +16,10 @@ SECTIONS {
HYP_SECTION(.text)
HYP_SECTION(.data..ro_after_init)
HYP_SECTION(.rodata)
#ifdef CONFIG_TRACING
. = ALIGN(PAGE_SIZE);
HYP_SECTION(_hyp_event_ids)
#endif
/*
* .hyp..data..percpu needs to be page aligned to maintain the same

View File

@@ -6,6 +6,7 @@
#include <asm/kvm_asm.h>
#include <asm/kvm_hyp.h>
#include <asm/kvm_hypevents.h>
#include <asm/kvm_mmu.h>
#include <linux/arm-smccc.h>
#include <linux/kvm_host.h>
@@ -169,6 +170,7 @@ static int psci_cpu_suspend(u64 func_id, struct kvm_cpu_context *host_ctxt)
DECLARE_REG(u64, power_state, host_ctxt, 1);
DECLARE_REG(unsigned long, pc, host_ctxt, 2);
DECLARE_REG(unsigned long, r0, host_ctxt, 3);
int ret;
struct psci_boot_args *boot_args;
struct kvm_nvhe_init_params *init_params;
@@ -184,14 +186,17 @@ static int psci_cpu_suspend(u64 func_id, struct kvm_cpu_context *host_ctxt)
boot_args->r0 = r0;
pkvm_psci_notify(PKVM_PSCI_CPU_SUSPEND, host_ctxt);
trace_hyp_exit();
/*
* Will either return if shallow sleep state, or wake up into the entry
* point if it is a deep sleep state.
*/
return psci_call(func_id, power_state,
__hyp_pa(&kvm_hyp_cpu_resume),
__hyp_pa(init_params));
ret = psci_call(func_id, power_state,
__hyp_pa(&kvm_hyp_cpu_resume),
__hyp_pa(init_params));
trace_hyp_enter();
return ret;
}
static int psci_system_suspend(u64 func_id, struct kvm_cpu_context *host_ctxt)

View File

@@ -21,6 +21,7 @@
#include <asm/kvm_asm.h>
#include <asm/kvm_emulate.h>
#include <asm/kvm_hyp.h>
#include <asm/kvm_hypevents.h>
#include <asm/kvm_mmu.h>
#include <asm/fpsimd.h>
#include <asm/debug-monitors.h>
@@ -295,10 +296,13 @@ int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
__debug_switch_to_guest(vcpu);
do {
trace_hyp_exit();
/* Jump in the fire! */
exit_code = __guest_enter(vcpu);
/* And we're baaack! */
trace_hyp_enter();
} while (fixup_guest_exit(vcpu, &exit_code));
__sysreg_save_state_nvhe(guest_ctxt);

226
arch/arm64/kvm/hyp_events.c Normal file
View File

@@ -0,0 +1,226 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2023 Google LLC
*/
#include <linux/tracefs.h>
#include <linux/trace_events.h>
#include <asm/kvm_host.h>
#include <asm/kvm_hypevents_defs.h>
#include "hyp_trace.h"
struct hyp_event {
struct trace_event_call *call;
char name[32];
bool *enabled;
};
#define HYP_EVENT(__name, __proto, __struct, __assign, __printk) \
HYP_EVENT_FORMAT(__name, __struct); \
enum print_line_t hyp_event_trace_##__name(struct trace_iterator *iter, \
int flags, struct trace_event *event) \
{ \
struct ht_iterator *ht_iter = (struct ht_iterator *)iter; \
struct trace_hyp_format_##__name __maybe_unused *__entry = \
(struct trace_hyp_format_##__name *)ht_iter->ent; \
trace_seq_puts(&ht_iter->seq, #__name); \
trace_seq_putc(&ht_iter->seq, ' '); \
trace_seq_printf(&ht_iter->seq, __printk); \
trace_seq_putc(&ht_iter->seq, '\n'); \
return TRACE_TYPE_HANDLED; \
}
#include <asm/kvm_hypevents.h>
#undef he_field
#define he_field(_type, _item) \
{ \
.type = #_type, .name = #_item, \
.size = sizeof(_type), .align = __alignof__(_type), \
.is_signed = is_signed_type(_type), \
},
#undef HYP_EVENT
#define HYP_EVENT(__name, __proto, __struct, __assign, __printk) \
static struct trace_event_fields hyp_event_fields_##__name[] = { \
__struct \
{} \
}; \
#undef __ARM64_KVM_HYPEVENTS_H_
#include <asm/kvm_hypevents.h>
#undef HYP_EVENT
#define HYP_EVENT(__name, __proto, __struct, __assign, __printk) \
static struct trace_event_functions hyp_event_funcs_##__name = { \
.trace = &hyp_event_trace_##__name, \
}; \
static struct trace_event_class hyp_event_class_##__name = { \
.system = "nvhe-hypervisor", \
.fields_array = hyp_event_fields_##__name, \
.fields = LIST_HEAD_INIT(hyp_event_class_##__name.fields),\
}; \
static struct trace_event_call hyp_event_call_##__name = { \
.class = &hyp_event_class_##__name, \
.event.funcs = &hyp_event_funcs_##__name, \
}; \
static bool hyp_event_enabled_##__name; \
struct hyp_event __section("_hyp_events") hyp_event_##__name = { \
.name = #__name, \
.call = &hyp_event_call_##__name, \
.enabled = &hyp_event_enabled_##__name, \
}
#undef __ARM64_KVM_HYPEVENTS_H_
#include <asm/kvm_hypevents.h>
extern struct hyp_event __start_hyp_events[];
extern struct hyp_event __stop_hyp_events[];
/* hyp_event section used by the hypervisor */
extern struct hyp_event_id __hyp_event_ids_start[];
extern struct hyp_event_id __hyp_event_ids_end[];
static ssize_t
hyp_event_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos)
{
struct seq_file *seq_file = (struct seq_file *)filp->private_data;
struct hyp_event *evt = (struct hyp_event *)seq_file->private;
unsigned short id = evt->call->event.type;
bool enabling;
int ret;
char c;
if (cnt != 2)
return -EINVAL;
if (get_user(c, ubuf))
return -EFAULT;
switch (c) {
case '1':
enabling = true;
break;
case '0':
enabling = false;
break;
default:
return -EINVAL;
}
if (enabling != *evt->enabled) {
ret = kvm_call_hyp_nvhe(__pkvm_enable_event, id, enabling);
if (ret)
return ret;
}
*evt->enabled = enabling;
return cnt;
}
static int hyp_event_show(struct seq_file *m, void *v)
{
struct hyp_event *evt = (struct hyp_event *)m->private;
/* lock ?? Ain't no time for that ! */
seq_printf(m, "%d\n", *evt->enabled);
return 0;
}
static int hyp_event_open(struct inode *inode, struct file *filp)
{
return single_open(filp, hyp_event_show, inode->i_private);
}
static const struct file_operations hyp_event_fops = {
.open = hyp_event_open,
.write = hyp_event_write,
.read = seq_read,
.llseek = seq_lseek,
.release = single_release,
};
static int hyp_event_id_show(struct seq_file *m, void *v)
{
struct hyp_event *evt = (struct hyp_event *)m->private;
seq_printf(m, "%d\n", evt->call->event.type);
return 0;
}
static int hyp_event_id_open(struct inode *inode, struct file *filp)
{
return single_open(filp, hyp_event_id_show, inode->i_private);
}
static const struct file_operations hyp_event_id_fops = {
.open = hyp_event_id_open,
.read = seq_read,
.llseek = seq_lseek,
.release = single_release,
};
void kvm_hyp_init_events_tracefs(struct dentry *parent)
{
struct hyp_event *event = __start_hyp_events;
struct dentry *d, *event_dir;
parent = tracefs_create_dir("events", parent);
if (!parent) {
pr_err("Failed to create tracefs folder for hyp events\n");
return;
}
for (; (unsigned long)event < (unsigned long)__stop_hyp_events; event++) {
event_dir = tracefs_create_dir(event->name, parent);
if (!event_dir) {
pr_err("Failed to create events/hyp/%s\n", event->name);
continue;
}
d = tracefs_create_file("enable", 0700, event_dir, (void *)event,
&hyp_event_fops);
if (!d)
pr_err("Failed to create events/hyp/%s/enable\n", event->name);
d = tracefs_create_file("id", 0400, event_dir, (void *)event,
&hyp_event_id_fops);
if (!d)
pr_err("Failed to create events/hyp/%s/id\n", event->name);
}
}
/*
* Register hyp events and write their id into the hyp section _hyp_event_ids.
*/
int kvm_hyp_init_events(void)
{
struct hyp_event *event = __start_hyp_events;
struct hyp_event_id *hyp_event_id = __hyp_event_ids_start;
int ret, err = -ENODEV;
/* TODO: BUILD_BUG nr events host side / hyp side */
for (; (unsigned long)event < (unsigned long)__stop_hyp_events;
event++, hyp_event_id++) {
event->call->name = event->name;
ret = register_trace_event(&event->call->event);
if (!ret) {
pr_warn("Couldn't register trace event for %s\n", event->name);
continue;
}
/*
* Both the host and the hypervisor relies on the same hyp event
* declarations from kvm_hypevents.h. We have then a 1:1
* mapping.
*/
hyp_event_id->id = ret;
err = 0;
}
return err;
}

View File

@@ -7,10 +7,12 @@
#include <linux/list.h>
#include <linux/percpu-defs.h>
#include <linux/ring_buffer.h>
#include <linux/trace_events.h>
#include <linux/tracefs.h>
#include <asm/kvm_host.h>
#include <asm/kvm_hyptrace.h>
#include <asm/kvm_hypevents_defs.h>
#include "hyp_constants.h"
#include "hyp_trace.h"
@@ -365,8 +367,12 @@ static void ht_print_trace_time(struct ht_iterator *iter)
(unsigned long)ts_ns, usecs_rem);
}
extern struct trace_event *ftrace_find_event(int type);
static void ht_print_trace_fmt(struct ht_iterator *iter)
{
struct trace_event *e;
if (iter->lost_events)
trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
iter->cpu, iter->lost_events);
@@ -374,6 +380,14 @@ static void ht_print_trace_fmt(struct ht_iterator *iter)
/* TODO: format bin/hex/raw */
ht_print_trace_time(iter);
e = ftrace_find_event(iter->ent->id);
if (e) {
e->funcs->trace((struct trace_iterator *)iter, 0, e);
return;
}
trace_seq_printf(&iter->seq, "Unknown event id %d\n", iter->ent->id);
};
static void *ht_next(struct seq_file *m, void *v, loff_t *pos)
@@ -636,6 +650,8 @@ static void hyp_tracefs_create_cpu_file(const char *file_name,
pr_warn("Failed to create tracefs %pd/%s\n", parent, file_name);
}
void kvm_hyp_init_events_tracefs(struct dentry *parent);
int init_hyp_tracefs(void)
{
struct dentry *d, *root_dir, *per_cpu_root_dir;
@@ -686,5 +702,7 @@ int init_hyp_tracefs(void)
&hyp_trace_pipe_fops, dir);
}
kvm_hyp_init_events_tracefs(root_dir);
return 0;
}