diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 16abb1e0de08..48e20ad5ff38 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -39,7 +39,12 @@ static bool (*default_trap_handler)(struct kvm_cpu_context *host_ctxt); int __pkvm_register_host_smc_handler(bool (*cb)(struct kvm_cpu_context *)) { - return cmpxchg(&default_host_smc_handler, NULL, cb) ? -EBUSY : 0; + /* + * Paired with smp_load_acquire(&default_host_smc_handler) in + * handle_host_smc(). Ensure memory stores happening during a pKVM module + * init are observed before executing the callback. + */ + return cmpxchg_release(&default_host_smc_handler, NULL, cb) ? -EBUSY : 0; } int __pkvm_register_default_trap_handler(bool (*cb)(struct kvm_cpu_context *)) @@ -1376,7 +1381,7 @@ static void handle_host_smc(struct kvm_cpu_context *host_ctxt) handled = kvm_host_psci_handler(host_ctxt); if (!handled) handled = kvm_host_ffa_handler(host_ctxt); - if (!handled && READ_ONCE(default_host_smc_handler)) + if (!handled && smp_load_acquire(&default_host_smc_handler)) handled = default_host_smc_handler(host_ctxt); if (!handled) __kvm_hyp_host_forward_smc(host_ctxt); diff --git a/arch/arm64/kvm/hyp/nvhe/psci-relay.c b/arch/arm64/kvm/hyp/nvhe/psci-relay.c index d4825b6140ba..f8db5445b530 100644 --- a/arch/arm64/kvm/hyp/nvhe/psci-relay.c +++ b/arch/arm64/kvm/hyp/nvhe/psci-relay.c @@ -28,14 +28,19 @@ struct kvm_host_psci_config __ro_after_init kvm_host_psci_config; static void (*pkvm_psci_notifier)(enum pkvm_psci_notification, struct kvm_cpu_context *); static void pkvm_psci_notify(enum pkvm_psci_notification notif, struct kvm_cpu_context *host_ctxt) { - if (READ_ONCE(pkvm_psci_notifier)) + if (smp_load_acquire(&pkvm_psci_notifier)) pkvm_psci_notifier(notif, host_ctxt); } #ifdef CONFIG_MODULES int __pkvm_register_psci_notifier(void (*cb)(enum pkvm_psci_notification, struct kvm_cpu_context *)) { - return cmpxchg(&pkvm_psci_notifier, NULL, cb) ? -EBUSY : 0; + /* + * Paired with smp_load_acquire(&pkvm_psci_notifier) in + * pkvm_psci_notify(). Ensure memory stores hapenning during a pKVM module + * init are observed before executing the callback. + */ + return cmpxchg_release(&pkvm_psci_notifier, NULL, cb) ? -EBUSY : 0; } #endif diff --git a/arch/arm64/kvm/hyp/nvhe/serial.c b/arch/arm64/kvm/hyp/nvhe/serial.c index 0b2cf3b6d6a5..475ebf4ba7de 100644 --- a/arch/arm64/kvm/hyp/nvhe/serial.c +++ b/arch/arm64/kvm/hyp/nvhe/serial.c @@ -35,7 +35,8 @@ static inline void __hyp_putx4n(unsigned long x, int n) static inline bool hyp_serial_enabled(void) { - return !!READ_ONCE(__hyp_putc); + /* Paired with __pkvm_register_serial_driver()'s cmpxchg */ + return !!smp_load_acquire(&__hyp_putc); } void hyp_puts(const char *s) @@ -64,5 +65,10 @@ void hyp_putc(char c) int __pkvm_register_serial_driver(void (*cb)(char)) { - return cmpxchg(&__hyp_putc, NULL, cb) ? -EBUSY : 0; + /* + * Paired with smp_load_acquire(&__hyp_putc) in + * hyp_serial_enabled(). Ensure memory stores hapenning during a pKVM + * module init are observed before executing the callback. + */ + return cmpxchg_release(&__hyp_putc, NULL, cb) ? -EBUSY : 0; }