ANDROID: KVM: arm64: Add __pkvm_enable_tracing

This newly introduced hypercall allows the host to disable tracing on
all CPUs, while keeping the tracing buffers loaded into the hypervisor.
This intends to later improve the userspace interface which will be able
to turn on and off tracing and reset (teardown for the hyp) the tracing
buffers.

As disabling buffers will switch the buffer status, rename those status
to nonwritable - writable - writing. Another way of identifying buffers
which have not been loaded is needed. See rb_cpu_loaded().

Bug: 249050813
Change-Id: I6080aafe71d5628e94b37c432bcd8616e68ddfe8
Signed-off-by: Vincent Donnefort <vdonnefort@google.com>

arch/arm64/include/asm/kvm_asm.h

@@ -97,8 +97,9 @@ enum __kvm_host_smccc_func {
 	__KVM_HOST_SMCCC_FUNC___pkvm_iommu_register,
 	__KVM_HOST_SMCCC_FUNC___pkvm_iommu_pm_notify,
 	__KVM_HOST_SMCCC_FUNC___pkvm_iommu_finalize,
-	__KVM_HOST_SMCCC_FUNC___pkvm_start_tracing,
-	__KVM_HOST_SMCCC_FUNC___pkvm_stop_tracing,
+	__KVM_HOST_SMCCC_FUNC___pkvm_load_tracing,
+	__KVM_HOST_SMCCC_FUNC___pkvm_teardown_tracing,
+	__KVM_HOST_SMCCC_FUNC___pkvm_enable_tracing,
 	__KVM_HOST_SMCCC_FUNC___pkvm_rb_swap_reader_page,
 	__KVM_HOST_SMCCC_FUNC___pkvm_rb_update_footers,
 	__KVM_HOST_SMCCC_FUNC___pkvm_enable_event,

arch/arm64/kvm/hyp/include/nvhe/trace.h

@@ -19,9 +19,9 @@ struct hyp_buffer_page {
 	atomic_t entries;
 };
 
-#define HYP_RB_UNUSED	0
-#define HYP_RB_READY	1
-#define HYP_RB_WRITE	2
+#define HYP_RB_NONWRITABLE	0
+#define HYP_RB_WRITABLE		1
+#define HYP_RB_WRITING		2
 
 struct hyp_rb_per_cpu {
 	struct hyp_buffer_page *tail_page;
@@ -38,19 +38,16 @@ struct hyp_rb_per_cpu {
 
 static inline bool __start_write_hyp_rb(struct hyp_rb_per_cpu *rb)
 {
-	/*
-	 * Paired with rb_cpu_init()
-	 */
-	return atomic_cmpxchg_acquire(&rb->status, HYP_RB_READY, HYP_RB_WRITE)
-		!= HYP_RB_UNUSED;
+	return atomic_cmpxchg(&rb->status, HYP_RB_WRITABLE, HYP_RB_WRITING)
+		!= HYP_RB_NONWRITABLE;
 }
 
 static inline void __stop_write_hyp_rb(struct hyp_rb_per_cpu *rb)
 {
 	/*
-	 * Paired with rb_cpu_teardown()
+	 * Paired with rb_cpu_disable()
 	 */
-	atomic_set_release(&rb->status, HYP_RB_READY);
+	atomic_set_release(&rb->status, HYP_RB_WRITABLE);
 }
 
 struct hyp_rb_per_cpu;
@@ -58,8 +55,9 @@ DECLARE_PER_CPU(struct hyp_rb_per_cpu, trace_rb);
 
 void *rb_reserve_trace_entry(struct hyp_rb_per_cpu *cpu_buffer, unsigned long length);
 
-int __pkvm_start_tracing(unsigned long pack_va, size_t pack_size);
-void __pkvm_stop_tracing(void);
+int __pkvm_load_tracing(unsigned long pack_va, size_t pack_size);
+void __pkvm_teardown_tracing(void);
+int __pkvm_enable_tracing(bool enable);
 int __pkvm_rb_swap_reader_page(int cpu);
 int __pkvm_rb_update_footers(int cpu);
 int __pkvm_enable_event(unsigned short id, bool enable);
@@ -87,12 +85,14 @@ int __pkvm_enable_event(unsigned short id, bool enable);
 /* TODO: atomic_t to static_branch */
 
 #else
-static inline int __pkvm_start_tracing(unsigned long pack_va, size_t pack_size)
+static inline int __pkvm_load_tracing(unsigned long pack_va, size_t pack_size)
 {
 	return -ENODEV;
 }
 
-static inline void __pkvm_stop_tracing(void) { }
+static inline void __pkvm_teardown_tracing(void) { }
+
+static inline int __pkvm_enable_tracing(bool enable) { return -ENODEV; }
 
 static inline int __pkvm_rb_swap_reader_page(int cpu)
 {

arch/arm64/kvm/hyp/nvhe/hyp-main.c

@@ -1220,21 +1220,28 @@ handle___pkvm_close_module_registration(struct kvm_cpu_context *host_ctxt)
 	cpu_reg(host_ctxt, 1) = __pkvm_close_module_registration();
 }
 
-static void handle___pkvm_start_tracing(struct kvm_cpu_context *host_ctxt)
+static void handle___pkvm_load_tracing(struct kvm_cpu_context *host_ctxt)
 {
 	 DECLARE_REG(unsigned long, pack_hva, host_ctxt, 1);
 	 DECLARE_REG(size_t, pack_size, host_ctxt, 2);
 
-	 cpu_reg(host_ctxt, 1) = __pkvm_start_tracing(pack_hva, pack_size);
+	 cpu_reg(host_ctxt, 1) = __pkvm_load_tracing(pack_hva, pack_size);
 }
 
-static void handle___pkvm_stop_tracing(struct kvm_cpu_context *host_ctxt)
+static void handle___pkvm_teardown_tracing(struct kvm_cpu_context *host_ctxt)
 {
-	__pkvm_stop_tracing();
+	__pkvm_teardown_tracing();
 
 	cpu_reg(host_ctxt, 1) = 0;
 }
 
+static void handle___pkvm_enable_tracing(struct kvm_cpu_context *host_ctxt)
+{
+	DECLARE_REG(bool, enable, host_ctxt, 1);
+
+	cpu_reg(host_ctxt, 1) = __pkvm_enable_tracing(enable);
+}
+
 static void handle___pkvm_rb_swap_reader_page(struct kvm_cpu_context *host_ctxt)
 {
 	DECLARE_REG(int, cpu, host_ctxt, 1);
@@ -1303,8 +1310,9 @@ static const hcall_t host_hcall[] = {
 	HANDLE_FUNC(__pkvm_iommu_register),
 	HANDLE_FUNC(__pkvm_iommu_pm_notify),
 	HANDLE_FUNC(__pkvm_iommu_finalize),
-	HANDLE_FUNC(__pkvm_start_tracing),
-	HANDLE_FUNC(__pkvm_stop_tracing),
+	HANDLE_FUNC(__pkvm_load_tracing),
+	HANDLE_FUNC(__pkvm_teardown_tracing),
+	HANDLE_FUNC(__pkvm_enable_tracing),
 	HANDLE_FUNC(__pkvm_rb_swap_reader_page),
 	HANDLE_FUNC(__pkvm_rb_update_footers),
 	HANDLE_FUNC(__pkvm_enable_event),

arch/arm64/kvm/hyp/nvhe/trace.c

@@ -308,20 +308,51 @@ static int rb_page_init(struct hyp_buffer_page *bpage, unsigned long hva)
 	return 0;
 }
 
-static void rb_cpu_teardown(struct hyp_rb_per_cpu *cpu_buffer)
+static bool rb_cpu_loaded(struct hyp_rb_per_cpu *cpu_buffer)
+{
+	return cpu_buffer->bpages;
+}
+
+static void rb_cpu_disable(struct hyp_rb_per_cpu *cpu_buffer)
 {
 	unsigned int prev_status;
-	int i;
 
 	/* Wait for release of the buffer */
 	do {
+		/* Paired with __stop_write_hyp_rb */
 		prev_status = atomic_cmpxchg_acquire(&cpu_buffer->status,
-						     HYP_RB_READY,
-						     HYP_RB_UNUSED);
-	} while (prev_status == HYP_RB_WRITE);
+						     HYP_RB_WRITABLE,
+						     HYP_RB_NONWRITABLE);
+	} while (prev_status == HYP_RB_WRITING);
 
-	if (prev_status == HYP_RB_READY)
+	if (prev_status == HYP_RB_WRITABLE)
 		rb_update_footers(cpu_buffer);
+}
+
+static int rb_cpu_enable(struct hyp_rb_per_cpu *cpu_buffer)
+{
+	unsigned int prev_status;
+
+	if (!rb_cpu_loaded(cpu_buffer))
+		return -EINVAL;
+
+	prev_status = atomic_cmpxchg(&cpu_buffer->status,
+				     HYP_RB_NONWRITABLE, HYP_RB_WRITABLE);
+
+	if (prev_status == HYP_RB_NONWRITABLE)
+		return 0;
+
+	return -EINVAL;
+}
+
+static void rb_cpu_teardown(struct hyp_rb_per_cpu *cpu_buffer)
+{
+	int i;
+
+	if (!rb_cpu_loaded(cpu_buffer))
+		return;
+
+	rb_cpu_disable(cpu_buffer);
 
 	for (i = 0; i < cpu_buffer->nr_pages; i++) {
 		struct hyp_buffer_page *bpage = &cpu_buffer->bpages[i];
@@ -332,6 +363,8 @@ static void rb_cpu_teardown(struct hyp_rb_per_cpu *cpu_buffer)
 		hyp_unpin_shared_mem((void *)bpage->page,
 				     (void *)bpage->page + PAGE_SIZE);
 	}
+
+	cpu_buffer->bpages = NULL;
 }
 
 static bool rb_cpu_fits_backing(unsigned long nr_pages,
@@ -405,11 +438,6 @@ static int rb_cpu_init(struct ring_buffer_pack *rb_pack, struct hyp_buffer_page
 	atomic_set(&cpu_buffer->overrun, 0);
 	atomic64_set(&cpu_buffer->write_stamp, 0);
 
-	/*
-	 * Paired with __start_write_hyp_rb()
-	 */
-	atomic_set_release(&cpu_buffer->status, HYP_RB_READY);
-
 	return 0;
 err:
 	rb_cpu_teardown(cpu_buffer);
@@ -462,23 +490,19 @@ int __pkvm_rb_update_footers(int cpu)
 	struct hyp_rb_per_cpu *cpu_buffer;
 	int ret = 0;
 
+	if (cpu >= hyp_nr_cpus)
+		return -EINVAL;
+
 	/* TODO: per-CPU lock for */
 	hyp_spin_lock(&trace_rb_lock);
 
-	if (cpu >= hyp_nr_cpus) {
-		ret = -EINVAL;
-		goto err;
-	}
-
 	cpu_buffer = per_cpu_ptr(&trace_rb, cpu);
 
-	if (atomic_read(&cpu_buffer->status) == HYP_RB_UNUSED) {
+	if (!rb_cpu_loaded(cpu_buffer))
 		ret = -ENODEV;
-		goto err;
-	}
+	else
+		ret = rb_update_footers(cpu_buffer);
 
-	ret = rb_update_footers(cpu_buffer);
-err:
 	hyp_spin_unlock(&trace_rb_lock);
 
 	return ret;
@@ -489,30 +513,25 @@ int __pkvm_rb_swap_reader_page(int cpu)
 	struct hyp_rb_per_cpu *cpu_buffer = per_cpu_ptr(&trace_rb, cpu);
 	int ret = 0;
 
+	if (cpu >= hyp_nr_cpus)
+		return -EINVAL;
+
 	/* TODO: per-CPU lock for */
 	hyp_spin_lock(&trace_rb_lock);
 
-	if (cpu >= hyp_nr_cpus) {
-		ret = -EINVAL;
-		goto err;
-	}
 	cpu_buffer = per_cpu_ptr(&trace_rb, cpu);
 
-	if (atomic_read(&cpu_buffer->status) == HYP_RB_UNUSED) {
+	if (!rb_cpu_loaded(cpu_buffer))
 		ret = -ENODEV;
-		goto err;
-	}
+	else
+		ret = rb_swap_reader_page(cpu_buffer);
 
-	ret = rb_swap_reader_page(cpu_buffer);
-	if (ret)
-		goto err;
-err:
 	hyp_spin_unlock(&trace_rb_lock);
 
 	return ret;
 }
 
-static void __pkvm_stop_tracing_locked(void)
+static void __pkvm_teardown_tracing_locked(void)
 {
 	int cpu;
 
@@ -521,23 +540,20 @@ static void __pkvm_stop_tracing_locked(void)
 	for (cpu = 0; cpu < hyp_nr_cpus; cpu++) {
 		struct hyp_rb_per_cpu *cpu_buffer = per_cpu_ptr(&trace_rb, cpu);
 
-		if (atomic_read(&cpu_buffer->status) == HYP_RB_UNUSED)
-			continue;
-
 		rb_cpu_teardown(cpu_buffer);
 	}
 
 	rb_teardown_bpage_backing();
 }
 
-void __pkvm_stop_tracing(void)
+void __pkvm_teardown_tracing(void)
 {
 	hyp_spin_lock(&trace_rb_lock);
-	__pkvm_stop_tracing_locked();
+	__pkvm_teardown_tracing_locked();
 	hyp_spin_unlock(&trace_rb_lock);
 }
 
-int __pkvm_start_tracing(unsigned long pack_hva, size_t pack_size)
+int __pkvm_load_tracing(unsigned long pack_hva, size_t pack_size)
 {
 	struct hyp_trace_pack *pack = (struct hyp_trace_pack *)kern_hyp_va(pack_hva);
 	struct trace_buffer_pack *trace_pack = &pack->trace_buffer_pack;
@@ -586,7 +602,7 @@ int __pkvm_start_tracing(unsigned long pack_hva, size_t pack_size)
 	}
 err:
 	if (ret)
-		__pkvm_stop_tracing_locked();
+		__pkvm_teardown_tracing_locked();
 
 	hyp_spin_unlock(&trace_rb_lock);
 
@@ -594,3 +610,26 @@ err:
 				       pack_size >> PAGE_SHIFT));
 	return ret;
 }
+
+int __pkvm_enable_tracing(bool enable)
+{
+	int cpu, ret = enable ? -EINVAL : 0;
+
+	hyp_spin_lock(&trace_rb_lock);
+	for (cpu = 0; cpu < hyp_nr_cpus; cpu++) {
+		struct hyp_rb_per_cpu *cpu_buffer = per_cpu_ptr(&trace_rb, cpu);
+
+		if (enable) {
+			int __ret = rb_cpu_enable(cpu_buffer);
+
+			if (!__ret)
+				ret = 0;
+		} else {
+			rb_cpu_disable(cpu_buffer);
+		}
+
+	}
+	hyp_spin_unlock(&trace_rb_lock);
+
+	return ret;
+}

arch/arm64/kvm/hyp_trace.c

@@ -228,12 +228,16 @@ static int hyp_start_tracing(void)
 	if (ret)
 		goto end_buffer_teardown;
 
-	ret = kvm_call_hyp_nvhe(__pkvm_start_tracing, (unsigned long)pack, pack_size);
+	ret = kvm_call_hyp_nvhe(__pkvm_load_tracing, (unsigned long)pack, pack_size);
+	if (ret)
+		goto end_backing_teardown;
+
+	ret = kvm_call_hyp_nvhe(__pkvm_enable_tracing, true);
 	if (!ret) {
 		hyp_trace_on = true;
 		goto end_free_pack;
 	}
-
+end_backing_teardown:
 	bpage_backing_teardown();
 end_buffer_teardown:
 	trace_buffer_teardown(&pack->trace_buffer_pack);
@@ -250,7 +254,7 @@ static void hyp_stop_tracing(void)
 	if (!hyp_trace_buffer || !hyp_trace_on)
 		return;
 
-	ret = kvm_call_hyp_nvhe(__pkvm_stop_tracing);
+	ret = kvm_call_hyp_nvhe(__pkvm_teardown_tracing);
 	if (ret) {
 		WARN_ON(1);
 		return;