shinys000114/linux
1
0
Fork 0
mirror of https://github.com/hardkernel/linux.git synced 2026-06-06 02:50:49 +09:00
Code Issues Packages Projects Releases Wiki Activity

ANDROID: arm64: kvm: s2mpu: Remove S2MPU pKVM driver

Browse Source 
As S2MPU driver is used by a few devices and EL2 loadable modules
code is merged.
S2MPU pKVM driver will be moved to be an EL2 module.
This commit removes all the code and configuration for the driver.

Bug: 261857213
Change-Id: I53da660f8503c4bedf2fc48a90d1e8ba73329c05
Signed-off-by: Mostafa Saleh <smostafa@google.com>
This commit is contained in:
Mostafa Saleh
2022-12-09 14:56:06 +00:00
parent 685a2ade28
commit 12bbef61d8
11 changed files with 1 additions and 1647 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
arch/arm64/configs/gki_defconfig
View File

@@ -85,7 +85,6 @@ CONFIG_ARM_SCPI_CPUFREQ=y
CONFIG_ARM_SCMI_CPUFREQ=y
CONFIG_VIRTUALIZATION=y
CONFIG_KVM=y
CONFIG_KVM_S2MPU=y
CONFIG_CRYPTO_SHA2_ARM64_CE=y
CONFIG_CRYPTO_SHA512_ARM64_CE=y
CONFIG_CRYPTO_POLYVAL_ARM64_CE=y

29
arch/arm64/include/asm/io-mpt-s2mpu.h
View File

@@ -1,29 +0,0 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2022 - Google LLC
*/
#ifndef __IO_MPT_S2MPU_H__
#define __IO_MPT_S2MPU_H__
#include <linux/bitfield.h>
#include <asm/kvm_mmu.h>
#include <asm/kvm_s2mpu.h>
struct s2mpu_mpt_cfg {
enum s2mpu_version version;
};
struct s2mpu_mpt_ops {
u32 (*smpt_size)(void);
void (*init_with_prot)(void *dev_va, enum mpt_prot prot);
void (*init_with_mpt)(void *dev_va, struct mpt *mpt);
void (*apply_range)(void *dev_va, struct mpt *mpt, u32 first_gb, u32 last_gb);
void (*prepare_range)(struct mpt *mpt, phys_addr_t first_byte,
phys_addr_t last_byte, enum mpt_prot prot);
int (*pte_from_addr_smpt)(u32 *smpt, u64 addr);
};
const struct s2mpu_mpt_ops *s2mpu_get_mpt_ops(struct s2mpu_mpt_cfg cfg);
#endif /* __IO_MPT_S2MPU_H__ */

7
arch/arm64/include/asm/kvm_host.h
View File

@@ -395,19 +395,12 @@ struct pkvm_iommu_driver {
atomic_t state;
};
extern struct pkvm_iommu_driver kvm_nvhe_sym(pkvm_s2mpu_driver);
extern struct pkvm_iommu_driver kvm_nvhe_sym(pkvm_sysmmu_sync_driver);
int pkvm_iommu_driver_init(struct pkvm_iommu_driver *drv, void *data, size_t size);
int pkvm_iommu_register(struct device *dev, struct pkvm_iommu_driver *drv,
phys_addr_t pa, size_t size, struct device *parent);
int pkvm_iommu_suspend(struct device *dev);
int pkvm_iommu_resume(struct device *dev);
int pkvm_iommu_s2mpu_init(u32 version);
int pkvm_iommu_s2mpu_register(struct device *dev, phys_addr_t pa);
int pkvm_iommu_sysmmu_sync_register(struct device *dev, phys_addr_t pa,
struct device *parent);
/* Reject future calls to pkvm_iommu_driver_init() and pkvm_iommu_register(). */
int pkvm_iommu_finalize(void);

436
arch/arm64/include/asm/kvm_s2mpu.h
View File

@@ -1,436 +0,0 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2021 - Google LLC
* Author: David Brazdil <dbrazdil@google.com>
*/
#ifndef __ARM64_KVM_S2MPU_H__
#define __ARM64_KVM_S2MPU_H__
#include <linux/bitfield.h>
#include <asm/kvm_mmu.h>
#define S2MPU_MMIO_SIZE SZ_64K
#define SYSMMU_SYNC_MMIO_SIZE SZ_64K
#define SYSMMU_SYNC_S2_OFFSET SZ_32K
#define SYSMMU_SYNC_S2_MMIO_SIZE (SYSMMU_SYNC_MMIO_SIZE - \
SYSMMU_SYNC_S2_OFFSET)
#define NR_VIDS 8
#define NR_CTX_IDS 8
#define ALL_VIDS_BITMAP GENMASK(NR_VIDS - 1, 0)
/*
* S2MPU V9 specific values (some new and some different from old versions)
* to avoid any confusion all names are prefixed with V9.
*/
#define REG_NS_V9_CTRL_PROT_EN_PER_VID_SET 0x50
#define REG_NS_V9_CTRL_ERR_RESP_T_PER_VID_SET 0x70
#define REG_NS_V9_CFG_MPTW_ATTRIBUTE 0x10
#define REG_NS_V9_READ_MPTC 0x3014
#define REG_NS_V9_READ_MPTC_TAG_PPN 0x3018
#define REG_NS_V9_READ_MPTC_TAG_OTHERS 0x301C
#define REG_NS_V9_READ_MPTC_DATA 0x3020
#define REG_NS_V9_READ_PTLB 0x3030
#define REG_NS_V9_READ_PTLB_TAG 0x3034
#define REG_NS_V9_READ_PTLB_DATA_S1_EN_PPN_AP 0x3040
#define REG_NS_V9_READ_PTLB_DATA_S1_DIS_AP_LIST 0x3044
#define REG_NS_V9_PMMU_INDICATOR 0x3050
#define REG_NS_V9_PMMU_INFO 0x3100
#define REG_NS_V9_PMMU_PTLB_INFO(n) (0x3400 + (n)*0x4)
#define REG_NS_V9_SWALKER_INFO 0x3104
#define REG_NS_V9_MPTC_INFO 0x3C00
/* V9 Masks */
#define V9_READ_MPTC_TAG_PPN_VALID_MASK BIT(28)
#define V9_READ_MPTC_TAG_PPN_TPN_PPN_MASK GENMASK(23, 0)
#define V9_READ_MPTC_TAG_PPN_MASK (V9_READ_MPTC_TAG_PPN_VALID_MASK | \
V9_READ_MPTC_TAG_PPN_TPN_PPN_MASK)
#define V9_READ_MPTC_TAG_OTHERS_VID_MASK GENMASK(10, 8)
#define V9_READ_MPTC_TAG_OTHERS_PAGE_GRAN_MASK GENMASK(5, 4)
#define V9_READ_MPTC_TAG_OTHERS_MASK (V9_READ_MPTC_TAG_OTHERS_VID_MASK | \
V9_READ_MPTC_TAG_OTHERS_PAGE_GRAN_MASK)
#define V9_READ_PTLB_WAY_MASK GENMASK(31, 24)
#define V9_READ_PTLB_SET_MASK GENMASK(23, 16)
#define V9_READ_PTLB_PTLB_MASK GENMASK(15, 4)
#define V9_READ_PTLB_PMMU_MASK GENMASK(3, 0)
#define V9_READ_PTLB_MASK (V9_READ_PTLB_WAY_MASK | V9_READ_PTLB_SET_MASK | \
V9_READ_PTLB_PTLB_MASK | V9_READ_PTLB_PMMU_MASK)
#define V9_READ_PTLB_TAG_VALID_MASK BIT(31)
#define V9_READ_PTLB_TAG_PAGE_SIZE_MASK GENMASK(30, 28)
#define V9_READ_PTLB_TAG_STAGE1_ENABLED_MASK BIT(27)
#define V9_READ_PTLB_TAG_VID_MASK GENMASK(26, 24)
#define V9_READ_PTLB_TAG_TPN_MASK GENMASK(23, 0)
#define V9_READ_PTLB_TAG_MASK (V9_READ_PTLB_TAG_VALID_MASK | \
V9_READ_PTLB_TAG_TPN_MASK | \
V9_READ_PTLB_TAG_VID_MASK | \
V9_READ_PTLB_TAG_PAGE_SIZE_MASK | \
V9_READ_PTLB_TAG_STAGE1_ENABLED_MASK)
#define V9_READ_PTLB_DTA_S1_EN_PPN_AP_S2AP_MASK GENMASK(25, 24)
#define V9_READ_PTLB_DTA_S1_EN_PPN_AP_PPN_MASK GENMASK(23, 0)
#define V9_READ_PTLB_DATA_S1_ENABLE_PPN_AP_MASK (V9_READ_PTLB_DTA_S1_EN_PPN_AP_S2AP_MASK | \
V9_READ_PTLB_DTA_S1_EN_PPN_AP_PPN_MASK)
#define V9_READ_MPTC_INFO_NUM_MPTC_SET GENMASK(31, 16)
#define V9_READ_MPTC_INFO_NUM_MPTC_WAY GENMASK(15, 12)
#define V9_READ_MPTC_INFO_MASK (V9_READ_MPTC_INFO_NUM_MPTC_SET | \
V9_READ_MPTC_INFO_NUM_MPTC_SET)
#define V9_READ_PMMU_INFO_NUM_PTLB GENMASK(15, 1)
#define V9_READ_PMMU_INFO_VA_WIDTH BIT(0)
#define V9_READ_PMMU_INFO_NUM_STREAM_TABLE GENMASK(31, 16)
#define V9_READ_PMMU_INFO_MASK (V9_READ_PMMU_INFO_NUM_PTLB | \
V9_READ_PMMU_INFO_VA_WIDTH | \
V9_READ_PMMU_INFO_NUM_STREAM_TABLE)
#define V9_READ_PMMU_PTLB_INFO_NUM_WAY GENMASK(31, 16)
#define V9_READ_PMMU_PTLB_INFO_NUM_SET GENMASK(15, 0)
#define V9_READ_PMMU_PTLB_INFO_MASK (V9_READ_PMMU_PTLB_INFO_NUM_WAY | \
V9_READ_PMMU_PTLB_INFO_NUM_SET)
#define V9_READ_PMMU_INDICATOR_PMMU_NUM GENMASK(3, 0)
#define V9_READ_PMMU_INDICATOR_MASK V9_READ_PMMU_INDICATOR_PMMU_NUM
#define V9_READ_MPTC_WAY_MASK GENMASK(17, 16)
#define V9_READ_MPTC_SET_MASK GENMASK(15, 0)
#define V9_READ_MPTC_MASK (V9_READ_MPTC_WAY_MASK | \
V9_READ_MPTC_SET_MASK)
#define V9_READ_MPTC_WAY(way) FIELD_PREP(V9_READ_MPTC_WAY_MASK, (way))
#define V9_READ_MPTC_SET(set) FIELD_PREP(V9_READ_MPTC_SET_MASK, (set))
#define V9_READ_MPTC(set, way) (V9_READ_MPTC_SET(set) | V9_READ_MPTC_WAY(way))
#define V9_READ_PTLB_WAY(x) FIELD_PREP(V9_READ_PTLB_WAY_MASK, (x))
#define V9_READ_PTLB_SET(x) FIELD_PREP(V9_READ_PTLB_SET_MASK, (x))
#define V9_READ_PTLB_PTLB(x) FIELD_PREP(V9_READ_PTLB_PTLB_MASK, (x))
#define V9_READ_PTLB_PMMU(x) FIELD_PREP(V9_READ_PTLB_PMMU_MASK, (x))
#define V9_READ_PTLB(pu_i, pb_i, s, w) (V9_READ_PTLB_WAY(w) | V9_READ_PTLB_SET(s) | \
V9_READ_PTLB_PTLB(pb_i) | V9_READ_PTLB_PMMU(pu_i))
#define V9_READ_SLTB_INFO_SET_MASK GENMASK(15, 0)
#define V9_READ_SLTB_INFO_WAY_MASK GENMASK(31, 16)
#define V9_READ_SLTB_INFO_MASK (V9_READ_SLTB_INFO_SET_MASK | \
V9_READ_SLTB_INFO_WAY_MASK)
#define V9_SWALKER_INFO_NUM_STLB_MASK GENMASK(31, 16)
#define V9_SWALKER_INFO_NUM_PMMU_MASK GENMASK(15, 0)
#define V9_SWALKER_INFO_MASK (V9_SWALKER_INFO_NUM_STLB_MASK | \
V9_SWALKER_INFO_NUM_PMMU_MASK)
/*
* STLB has 2 types: A,B based on how S2MPU is connected
* registers or masks that vary based on type are suffixed with
* either TYPEA or TYPEB.
*/
#define REG_NS_V9_READ_STLB 0x3000
#define REG_NS_V9_READ_STLB_TPN 0x3004
#define REG_NS_V9_READ_STLB_TAG_PPN 0x3008
#define REG_NS_V9_READ_STLB_TAG_OTHERS 0x300C
#define REG_NS_V9_READ_STLB_DATA 0x3010
#define REG_NS_V9_STLB_INFO(n) (0x3800 + (n)*0x4)
#define V9_READ_STLB_SET_MASK_TYPEA GENMASK(7, 0)
#define V9_READ_STLB_WAY_MASK_TYPEA GENMASK(15, 8)
#define V9_READ_STLB_SUBLINE_MASK_TYPEA GENMASK(31, 20)
#define V9_READ_STLB_STLBID_MASK_TYPEA GENMASK(17, 16)
#define V9_READ_STLB_MASK_TYPEA (V9_READ_STLB_SET_MASK_TYPEA | \
V9_READ_STLB_WAY_MASK_TYPEA | \
V9_READ_STLB_SUBLINE_MASK_TYPEA | \
V9_READ_STLB_STLBID_MASK_TYPEA)
#define V9_READ_STLB_SET_MASK_TYPEB GENMASK(15, 0)
#define V9_READ_STLB_WAY_MASK_TYPEB GENMASK(17, 16)
#define V9_READ_STLB_STLBID_MASK_TYPEB GENMASK(31, 20)
#define V9_READ_STLB_MASK_TYPEB (V9_READ_STLB_SET_MASK_TYPEB | \
V9_READ_STLB_WAY_MASK_TYPEB | \
V9_READ_STLB_STLBID_MASK_TYPEB)
#define V9_READ_STLB_TPN_TPN_MASK GENMASK(23, 0)
#define V9_READ_STLB_TPN_S2VALID_MASK BIT(24)
#define V9_READ_STLB_TPN_STAGE1_ENABLED_MASK BIT(27)
#define V9_READ_STLB_TPN_VALID_MASK BIT(28)
#define V9_READ_STLB_TPN_MASK (V9_READ_STLB_TPN_TPN_MASK | \
V9_READ_STLB_TPN_S2VALID_MASK | \
V9_READ_STLB_TPN_STAGE1_ENABLED_MASK | \
V9_READ_STLB_TPN_VALID_MASK)
#define V9_READ_STLB_TAG_PPN_VALID_MASK_TYPEB BIT(28)
#define V9_READ_STLB_TAG_PPN_PPN_MASK GENMASK(23, 0)
#define V9_READ_STLB_TAG_PPN_MASK (V9_READ_STLB_TAG_PPN_PPN_MASK | \
V9_READ_STLB_TAG_PPN_VALID_MASK_TYPEB)
#define V9_READ_STLB_TAG_OTHERS_S2AP_MASK_TYPEA GENMASK(1, 0)
#define V9_READ_STLB_TAG_OTHERS_PS_MASK GENMASK(10, 8)
#define V9_READ_STLB_TAG_OTHERS_BPS_MASK BIT(12)
#define V9_READ_STLB_TAG_OTHERS_VID_MASK GENMASK(23, 20)
#define V9_READ_STLB_TAG_OTHERS_MASK (V9_READ_STLB_TAG_OTHERS_S2AP_MASK_TYPEA | \
V9_READ_STLB_TAG_OTHERS_PS_MASK | \
V9_READ_STLB_TAG_OTHERS_BPS_MASK | \
V9_READ_STLB_TAG_OTHERS_VID_MASK)
#define V9_READ_STLB_WAY_TYPEA(x) FIELD_PREP(V9_READ_STLB_WAY_MASK_TYPEA, (x))
#define V9_READ_STLB_SET_TYPEA(x) FIELD_PREP(V9_READ_STLB_SET_MASK_TYPEA, (x))
#define V9_READ_STLB_STLBID_TYPEA(x) FIELD_PREP(V9_READ_STLB_STLBID_MASK_TYPEA, (x))
#define V9_READ_STLB_SUBLINE_TYPEA(x) FIELD_PREP(V9_READ_STLB_SUBLINE_MASK_TYPEA, (x))
#define V9_READ_STLB_TYPEA(s_i, sub, s, w) (V9_READ_STLB_WAY_TYPEA(w) | \
V9_READ_STLB_SET_TYPEA(s) | \
V9_READ_STLB_STLBID_TYPEA(s_i) | \
V9_READ_STLB_SUBLINE_TYPEA(sub))
#define V9_READ_STLB_WAY_TYPEB(x) FIELD_PREP(V9_READ_STLB_WAY_MASK_TYPEB, (x))
#define V9_READ_STLB_SET_TYPEB(x) FIELD_PREP(V9_READ_STLB_SET_MASK_TYPEB, (x))
#define V9_READ_STLB_STLBID_TYPEB(x) FIELD_PREP(V9_READ_STLB_STLBID_MASK_TYPEB, (x))
#define V9_READ_STLB_TYPEB(s_i, s, w) (V9_READ_STLB_WAY_TYPEB(w) | \
V9_READ_STLB_SET_TYPEB(s) | \
V9_READ_STLB_STLBID_TYPEB(s_i))
#define V9_MAX_PTLB_NUM 0x100
#define V9_MAX_STLB_NUM 0x100
#define V9_CTRL0_DIS_CHK_S1L1PTW_MASK BIT(0)
#define V9_CTRL0_DIS_CHK_S1L2PTW_MASK BIT(1)
#define V9_CTRL0_DIS_CHK_USR_MARCHED_REQ_MASK BIT(3)
#define V9_CTRL0_FAULT_MODE_MASK BIT(4)
#define V9_CTRL0_ENF_FLT_MODE_S1_NONSEC_MASK BIT(5)
#define V9_CTRL0_DESTRUCTIVE_AP_CHK_MODE_MASK BIT(6)
#define V9_CTRL0_MASK (V9_CTRL0_DIS_CHK_S1L1PTW_MASK | \
V9_CTRL0_DESTRUCTIVE_AP_CHK_MODE_MASK | \
V9_CTRL0_DIS_CHK_USR_MARCHED_REQ_MASK | \
V9_CTRL0_DIS_CHK_S1L2PTW_MASK | \
V9_CTRL0_ENF_FLT_MODE_S1_NONSEC_MASK | \
V9_CTRL0_FAULT_MODE_MASK)
/*
* S2MPU V9 specific values (some new and some different from old versions)
* to avoid any confusion all names are prefixed with V9.
*/
#define V9_L1ENTRY_ATTR_GRAN_MASK BIT(3)
#define V9_MPT_PROT_BITS 4
#define V9_MPT_ACCESS_SHIFT 2
/* V1,V2 variants. */
#define MPT_ACCESS_SHIFT 0
#define L1ENTRY_ATTR_GRAN_MASK GENMASK(5, 4)
#define MPT_PROT_BITS 2
#define REG_NS_CTRL0 0x0
#define REG_NS_CTRL1 0x4
#define REG_NS_CFG 0x10
#define REG_NS_INTERRUPT_ENABLE_PER_VID_SET 0x20
#define REG_NS_INTERRUPT_CLEAR 0x2c
#define REG_NS_VERSION 0x60
#define REG_NS_INFO 0x64
#define REG_NS_STATUS 0x68
#define REG_NS_NUM_CONTEXT 0x100
#define REG_NS_CONTEXT_CFG_VALID_VID 0x104
#define REG_NS_ALL_INVALIDATION 0x1000
#define REG_NS_RANGE_INVALIDATION 0x1020
#define REG_NS_RANGE_INVALIDATION_START_PPN 0x1024
#define REG_NS_RANGE_INVALIDATION_END_PPN 0x1028
#define REG_NS_FAULT_STATUS 0x2000
#define REG_NS_FAULT_PA_LOW(vid) (0x2004 + ((vid) * 0x20))
#define REG_NS_FAULT_PA_HIGH(vid) (0x2008 + ((vid) * 0x20))
#define REG_NS_FAULT_INFO(vid) (0x2010 + ((vid) * 0x20))
#define REG_NS_READ_MPTC 0x3000
#define REG_NS_READ_MPTC_TAG_PPN 0x3004
#define REG_NS_READ_MPTC_TAG_OTHERS 0x3008
#define REG_NS_READ_MPTC_DATA 0x3010
#define REG_NS_L1ENTRY_L2TABLE_ADDR(vid, gb) (0x4000 + ((vid) * 0x200) + ((gb) * 0x8))
#define REG_NS_L1ENTRY_ATTR(vid, gb) (0x4004 + ((vid) * 0x200) + ((gb) * 0x8))
#define CTRL0_ENABLE BIT(0)
#define CTRL0_INTERRUPT_ENABLE BIT(1)
#define CTRL0_FAULT_RESP_TYPE_SLVERR BIT(2) /* for v1 */
#define CTRL0_FAULT_RESP_TYPE_DECERR BIT(2) /* for v2 */
#define CTRL0_MASK (CTRL0_ENABLE | \
CTRL0_INTERRUPT_ENABLE | \
CTRL0_FAULT_RESP_TYPE_SLVERR | \
CTRL0_FAULT_RESP_TYPE_DECERR)
#define CTRL1_DISABLE_CHK_S1L1PTW BIT(0)
#define CTRL1_DISABLE_CHK_S1L2PTW BIT(1)
#define CTRL1_ENABLE_PAGE_SIZE_AWARENESS BIT(2)
#define CTRL1_DISABLE_CHK_USER_MATCHED_REQ BIT(3)
#define CTRL1_MASK (CTRL1_DISABLE_CHK_S1L1PTW | \
CTRL1_DISABLE_CHK_S1L2PTW | \
CTRL1_ENABLE_PAGE_SIZE_AWARENESS | \
CTRL1_DISABLE_CHK_USER_MATCHED_REQ)
#define CFG_MPTW_CACHE_OVERRIDE BIT(0)
#define CFG_MPTW_CACHE_VALUE GENMASK(7, 4)
#define CFG_MPTW_QOS_OVERRIDE BIT(8)
#define CFG_MPTW_QOS_VALUE GENMASK(15, 12)
#define CFG_MPTW_SHAREABLE BIT(16)
#define CFG_MASK (CFG_MPTW_CACHE_OVERRIDE | \
CFG_MPTW_CACHE_VALUE | \
CFG_MPTW_QOS_OVERRIDE | \
CFG_MPTW_QOS_VALUE | \
CFG_MPTW_SHAREABLE)
/* For use with hi_lo_readq_relaxed(). */
#define REG_NS_FAULT_PA_HIGH_LOW(vid) REG_NS_FAULT_PA_LOW(vid)
/* Mask used for extracting VID from FAULT_* register offset. */
#define REG_NS_FAULT_VID_MASK GENMASK(7, 5)
#define VERSION_MAJOR_ARCH_VER_MASK GENMASK(31, 28)
#define VERSION_MINOR_ARCH_VER_MASK GENMASK(27, 24)
#define VERSION_REV_ARCH_VER_MASK GENMASK(23, 16)
#define VERSION_RTL_VER_MASK GENMASK(7, 0)
/* Ignore RTL version in driver version check. */
#define VERSION_CHECK_MASK (VERSION_MAJOR_ARCH_VER_MASK | \
VERSION_MINOR_ARCH_VER_MASK | \
VERSION_REV_ARCH_VER_MASK)
#define INFO_NUM_SET_MASK GENMASK(15, 0)
#define STATUS_BUSY BIT(0)
#define STATUS_ON_INVALIDATING BIT(1)
#define NUM_CONTEXT_MASK GENMASK(3, 0)
#define CONTEXT_CFG_VALID_VID_CTX_VALID(ctx) BIT((4 * (ctx)) + 3)
#define CONTEXT_CFG_VALID_VID_CTX_VID(ctx, vid) \
FIELD_PREP(GENMASK((4 * (ctx) + 2), 4 * (ctx)), (vid))
#define INVALIDATION_INVALIDATE BIT(0)
#define RANGE_INVALIDATION_PPN_SHIFT 12
#define NR_FAULT_INFO_REGS 8
#define FAULT_INFO_VID_MASK GENMASK(26, 24)
#define FAULT_INFO_TYPE_MASK GENMASK(23, 21)
#define FAULT_INFO_TYPE_CONTEXT 0x4 /* v2 only */
#define FAULT_INFO_TYPE_AP 0x2
#define FAULT_INFO_TYPE_MPTW 0x1
#define FAULT_INFO_RW_BIT BIT(20)
#define FAULT_INFO_LEN_MASK GENMASK(19, 16)
#define FAULT_INFO_ID_MASK GENMASK(15, 0)
#define L1ENTRY_L2TABLE_ADDR_SHIFT 4
#define L1ENTRY_L2TABLE_ADDR(pa) ((pa) >> L1ENTRY_L2TABLE_ADDR_SHIFT)
#define READ_MPTC_WAY_MASK GENMASK(18, 16)
#define READ_MPTC_SET_MASK GENMASK(15, 0)
#define READ_MPTC_MASK (READ_MPTC_WAY_MASK | READ_MPTC_SET_MASK)
#define READ_MPTC_WAY(way) FIELD_PREP(READ_MPTC_WAY_MASK, (way))
#define READ_MPTC_SET(set) FIELD_PREP(READ_MPTC_SET_MASK, (set))
#define READ_MPTC(set, way) (READ_MPTC_SET(set) | READ_MPTC_WAY(way))
#define READ_MPTC_TAG_PPN_MASK GENMASK(23, 0)
#define READ_MPTC_TAG_OTHERS_VID_MASK GENMASK(10, 8)
#define READ_MPTC_TAG_OTHERS_GRAN_MASK GENMASK(5, 4)
#define READ_MPTC_TAG_OTHERS_VALID_BIT BIT(0)
#define READ_MPTC_TAG_OTHERS_MASK (READ_MPTC_TAG_OTHERS_VID_MASK | \
READ_MPTC_TAG_OTHERS_GRAN_MASK | \
READ_MPTC_TAG_OTHERS_VALID_BIT)
#define L1ENTRY_ATTR_L2TABLE_EN BIT(0)
#define L1ENTRY_ATTR_GRAN_4K 0x0
#define L1ENTRY_ATTR_GRAN_64K 0x1
#define L1ENTRY_ATTR_GRAN_2M 0x2
#define L1ENTRY_ATTR_GRAN(gran, msk) FIELD_PREP(msk, gran)
#define L1ENTRY_ATTR_PROT_MASK GENMASK(2, 1)
#define L1ENTRY_ATTR_PROT(prot) FIELD_PREP(L1ENTRY_ATTR_PROT_MASK, prot)
#define L1ENTRY_ATTR_1G(prot) L1ENTRY_ATTR_PROT(prot)
#define L1ENTRY_ATTR_L2(gran, msk) (L1ENTRY_ATTR_GRAN(gran, msk) | \
L1ENTRY_ATTR_L2TABLE_EN)
#define NR_GIGABYTES 64
#define RO_GIGABYTES_FIRST 4
#define RO_GIGABYTES_LAST 33
#define NR_RO_GIGABYTES (RO_GIGABYTES_LAST - RO_GIGABYTES_FIRST + 1)
#define NR_RW_GIGABYTES (NR_GIGABYTES - NR_RO_GIGABYTES)
#ifdef CONFIG_ARM64_64K_PAGES
#define SMPT_GRAN SZ_64K
#define SMPT_GRAN_ATTR L1ENTRY_ATTR_GRAN_64K
#else
#define SMPT_GRAN SZ_4K
#define SMPT_GRAN_ATTR L1ENTRY_ATTR_GRAN_4K
#endif
static_assert(SMPT_GRAN <= PAGE_SIZE);
#define SMPT_WORD_SIZE sizeof(u32)
#define SMPT_ELEMS_PER_BYTE(prot_bits) (BITS_PER_BYTE / (prot_bits))
#define SMPT_ELEMS_PER_WORD(prot_bits) (SMPT_WORD_SIZE * SMPT_ELEMS_PER_BYTE(prot_bits))
#define SMPT_WORD_BYTE_RANGE(prot_bits) (SMPT_GRAN * SMPT_ELEMS_PER_WORD(prot_bits))
#define SMPT_NUM_ELEMS (SZ_1G / SMPT_GRAN)
#define SMPT_SIZE(prot_bits) (SMPT_NUM_ELEMS / SMPT_ELEMS_PER_BYTE(prot_bits))
#define SMPT_NUM_WORDS(prot_bits) (SMPT_SIZE(prot_bits) / SMPT_WORD_SIZE)
#define SMPT_NUM_PAGES(prot_bits) (SMPT_SIZE(prot_bits) / PAGE_SIZE)
#define SMPT_ORDER(prot_bits) get_order(SMPT_SIZE(prot_bits))
#define SMPT_GRAN_MASK GENMASK(1, 0)
/* SysMMU_SYNC registers, relative to SYSMMU_SYNC_S2_OFFSET. */
#define REG_NS_SYNC_CMD 0x0
#define REG_NS_SYNC_COMP 0x4
#define SYNC_CMD_SYNC BIT(0)
#define SYNC_COMP_COMPLETE BIT(0)
/*
* Iterate over S2MPU gigabyte regions. Skip those that cannot be modified
* (the MMIO registers are read only, with reset value MPT_PROT_NONE).
*/
#define for_each_gb_in_range(i, first, last) \
for ((i) = (first); (i) <= (last) && (i) < NR_GIGABYTES; \
(i) = (((i) + 1 == RO_GIGABYTES_FIRST) ? RO_GIGABYTES_LAST : (i)) + 1)
#define for_each_gb(i) for_each_gb_in_range(i, 0, NR_GIGABYTES - 1)
#define for_each_vid(i) for ((i) = 0; (i) < NR_VIDS; (i)++)
#define for_each_gb_and_vid(gb, vid) for_each_vid((vid)) for_each_gb((gb))
enum s2mpu_version {
S2MPU_VERSION_1 = 0x11000000,
S2MPU_VERSION_2 = 0x20000000,
S2MPU_VERSION_9 = 0x90000000,
};
static inline int smpt_order_from_version(enum s2mpu_version version)
{
if (version == S2MPU_VERSION_9)
return SMPT_ORDER(V9_MPT_PROT_BITS);
else if ((version == S2MPU_VERSION_1) || (version == S2MPU_VERSION_2))
return SMPT_ORDER(MPT_PROT_BITS);
BUG();
}
enum mpt_prot {
MPT_PROT_NONE = 0,
MPT_PROT_R = BIT(0),
MPT_PROT_W = BIT(1),
MPT_PROT_RW = MPT_PROT_R | MPT_PROT_W,
MPT_PROT_MASK = MPT_PROT_RW,
};
enum mpt_update_flags {
MPT_UPDATE_L1 = BIT(0),
MPT_UPDATE_L2 = BIT(1),
};
struct fmpt {
u32 *smpt;
bool gran_1g;
enum mpt_prot prot;
enum mpt_update_flags flags;
};
struct mpt {
struct fmpt fmpt[NR_GIGABYTES];
enum s2mpu_version version;
};
#endif /* __ARM64_KVM_S2MPU_H__ */

9
arch/arm64/kvm/Kconfig
View File

@@ -68,13 +68,4 @@ config PROTECTED_NVHE_STACKTRACE
If unsure, or not using protected nVHE (pKVM), say N.
config KVM_S2MPU
bool "Stage-2 Memory Protection Unit support"
depends on KVM
help
Support for the Stage-2 Memory Protection Unit (S2MPU) and Stream
Security Mapping Table (SSMT) devices in KVM. This allows the
hypervisor to restrict DMA access to its memory and the memory of
protected guests.
endif # VIRTUALIZATION

2
arch/arm64/kvm/Makefile
View File

@@ -8,7 +8,7 @@ ccflags-y += -I $(srctree)/$(src)
KVM=../../../virt/kvm
obj-$(CONFIG_KVM) += kvm.o
obj-$(CONFIG_KVM) += hyp/ iommu/
obj-$(CONFIG_KVM) += hyp/
kvm-y := $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o $(KVM)/eventfd.o \
$(KVM)/vfio.o $(KVM)/irqchip.o $(KVM)/binary_stats.o iommu.o \

3
arch/arm64/kvm/hyp/nvhe/Makefile
View File

@@ -12,9 +12,6 @@ hyp-obj-$(CONFIG_DEBUG_LIST) += list_debug.o
hyp-obj-$(CONFIG_MODULES) += modules.o
hyp-obj-y += $(lib-objs)
hyp-obj-$(CONFIG_KVM_S2MPU) += iommu/s2mpu.o
hyp-obj-$(CONFIG_KVM_S2MPU) += iommu/io-mpt-s2mpu.o
$(obj)/hyp.lds: $(src)/hyp.lds.S FORCE
$(call if_changed_dep,cpp_lds_S)

321
arch/arm64/kvm/hyp/nvhe/iommu/io-mpt-s2mpu.c
View File

@@ -1,321 +0,0 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2022 - Google LLC
*/
#include <asm/io-mpt-s2mpu.h>
#define GRAN_BYTE(gran) ((gran << V9_MPT_PROT_BITS) | (gran))
#define GRAN_HWORD(gran) ((GRAN_BYTE(gran) << 8) | (GRAN_BYTE(gran)))
#define GRAN_WORD(gran) (((u32)(GRAN_HWORD(gran) << 16) | (GRAN_HWORD(gran))))
#define GRAN_DWORD(gran) ((u64)((u64)GRAN_WORD(gran) << 32) | (u64)(GRAN_WORD(gran)))
#define SMPT_NUM_TO_BYTE(x) ((x) / SMPT_GRAN / SMPT_ELEMS_PER_BYTE(config_prot_bits))
#define BYTE_TO_SMPT_INDEX(x) ((x) / SMPT_WORD_BYTE_RANGE(config_prot_bits))
/*
* MPT table ops can be configured only for one version at runtime,
* these variables will hold version specific data set a run time init, to avoid
* having duplicate code or unnessery check during operations.
*/
static u32 config_prot_bits;
static u32 config_access_shift;
static const u64 *config_lut_prot;
static u32 config_gran_mask;
static u32 this_version;
/*
* page table entries for different protection look up table
* granularity is compile time config, so we can do this also for
* this array without having duplicate arrays
*/
static const u64 v9_mpt_prot_doubleword[] = {
[MPT_PROT_NONE] = 0x0000000000000000 | GRAN_DWORD(SMPT_GRAN_ATTR),
[MPT_PROT_R] = 0x4444444444444444 | GRAN_DWORD(SMPT_GRAN_ATTR),
[MPT_PROT_W] = 0x8888888888888888 | GRAN_DWORD(SMPT_GRAN_ATTR),
[MPT_PROT_RW] = 0xcccccccccccccccc | GRAN_DWORD(SMPT_GRAN_ATTR),
};
static const u64 mpt_prot_doubleword[] = {
[MPT_PROT_NONE] = 0x0000000000000000,
[MPT_PROT_R] = 0x5555555555555555,
[MPT_PROT_W] = 0xaaaaaaaaaaaaaaaa,
[MPT_PROT_RW] = 0xffffffffffffffff,
};
static inline int pte_from_addr_smpt(u32 *smpt, u64 addr)
{
u32 word_idx, idx, pte, val;
word_idx = BYTE_TO_SMPT_INDEX(addr);
val = READ_ONCE(smpt[word_idx]);
idx = (addr / SMPT_GRAN) % SMPT_ELEMS_PER_WORD(config_prot_bits);
pte = (val >> (idx * config_prot_bits)) & ((1 << config_prot_bits)-1);
return pte;
}
static inline int prot_from_addr_smpt(u32 *smpt, u64 addr)
{
int pte = pte_from_addr_smpt(smpt, addr);
return (pte >> config_access_shift);
}
/* Set protection bits of SMPT in a given range without using memset. */
static void __set_smpt_range_slow(u32 *smpt, size_t start_gb_byte,
size_t end_gb_byte, enum mpt_prot prot)
{
size_t i, start_word_byte, end_word_byte, word_idx, first_elem, last_elem;
u32 val;
/* Iterate over u32 words. */
start_word_byte = start_gb_byte;
while (start_word_byte < end_gb_byte) {
/* Determine the range of bytes covered by this word. */
word_idx = BYTE_TO_SMPT_INDEX(start_word_byte);
end_word_byte = min(
ALIGN(start_word_byte + 1, SMPT_WORD_BYTE_RANGE(config_prot_bits)),
end_gb_byte);
/* Identify protection bit offsets within the word. */
first_elem = (start_word_byte / SMPT_GRAN) % SMPT_ELEMS_PER_WORD(config_prot_bits);
last_elem =
((end_word_byte - 1) / SMPT_GRAN) % SMPT_ELEMS_PER_WORD(config_prot_bits);
/* Modify the corresponding word. */
val = READ_ONCE(smpt[word_idx]);
for (i = first_elem; i <= last_elem; i++) {
val &= ~(MPT_PROT_MASK << (i * config_prot_bits + config_access_shift));
val |= prot << (i * config_prot_bits + config_access_shift);
}
WRITE_ONCE(smpt[word_idx], val);
start_word_byte = end_word_byte;
}
}
/* Set protection bits of SMPT in a given range. */
static void __set_smpt_range(u32 *smpt, size_t start_gb_byte,
size_t end_gb_byte, enum mpt_prot prot)
{
size_t interlude_start, interlude_end, interlude_bytes, word_idx;
char prot_byte = (char)config_lut_prot[prot];
if (start_gb_byte >= end_gb_byte)
return;
/* Check if range spans at least one full u32 word. */
interlude_start = ALIGN(start_gb_byte, SMPT_WORD_BYTE_RANGE(config_prot_bits));
interlude_end = ALIGN_DOWN(end_gb_byte, SMPT_WORD_BYTE_RANGE(config_prot_bits));
/*
* If not, fall back to editing bits in the given range.
* sets bit for PTEs that are in less than 32 bits (can't be done by memset)
*/
if (interlude_start >= interlude_end) {
__set_smpt_range_slow(smpt, start_gb_byte, end_gb_byte, prot);
return;
}
/* Use bit-editing for prologue/epilogue, memset for interlude. */
word_idx = BYTE_TO_SMPT_INDEX(interlude_start);
interlude_bytes = SMPT_NUM_TO_BYTE(interlude_end - interlude_start);
/*
* These are pages in the start and at then end that are
* not part of full 32 bit SMPT word.
*/
__set_smpt_range_slow(smpt, start_gb_byte, interlude_start, prot);
memset(&smpt[word_idx], prot_byte, interlude_bytes);
__set_smpt_range_slow(smpt, interlude_end, end_gb_byte, prot);
}
/* Returns true if all SMPT protection bits match 'prot'. */
static bool __is_smpt_uniform(u32 *smpt, enum mpt_prot prot)
{
size_t i;
u64 *doublewords = (u64 *)smpt;
for (i = 0; i < SMPT_NUM_WORDS(config_prot_bits) / 2; i++) {
if (doublewords[i] != config_lut_prot[prot])
return false;
}
return true;
}
/*
* Set protection bits of FMPT/SMPT in a given range.
* Returns flags specifying whether L1/L2 changes need to be made visible
* to the device.
*/
static void __set_fmpt_range(struct fmpt *fmpt, size_t start_gb_byte,
size_t end_gb_byte, enum mpt_prot prot)
{
if (start_gb_byte == 0 && end_gb_byte >= SZ_1G) {
/* Update covers the entire GB region. */
if (fmpt->gran_1g && fmpt->prot == prot) {
fmpt->flags = 0;
return;
}
fmpt->gran_1g = true;
fmpt->prot = prot;
fmpt->flags = MPT_UPDATE_L1;
return;
}
if (fmpt->gran_1g) {
/* GB region currently uses 1G mapping. */
if (fmpt->prot == prot) {
fmpt->flags = 0;
return;
}
/*
* Range has different mapping than the rest of the GB.
* Convert to PAGE_SIZE mapping.
*/
fmpt->gran_1g = false;
__set_smpt_range(fmpt->smpt, 0, start_gb_byte, fmpt->prot);
__set_smpt_range(fmpt->smpt, start_gb_byte, end_gb_byte, prot);
__set_smpt_range(fmpt->smpt, end_gb_byte, SZ_1G, fmpt->prot);
fmpt->flags = MPT_UPDATE_L1 | MPT_UPDATE_L2;
return;
}
/* GB region currently uses PAGE_SIZE mapping. */
__set_smpt_range(fmpt->smpt, start_gb_byte, end_gb_byte, prot);
/* Check if the entire GB region has the same prot bits. */
if (!__is_smpt_uniform(fmpt->smpt, prot)) {
fmpt->flags = MPT_UPDATE_L2;
return;
}
fmpt->gran_1g = true;
fmpt->prot = prot;
fmpt->flags = MPT_UPDATE_L1;
}
static u32 smpt_size(void)
{
return SMPT_SIZE(config_prot_bits);
}
static void __set_l1entry_attr_with_prot(void *dev_va, unsigned int gb,
unsigned int vid, enum mpt_prot prot)
{
writel_relaxed(L1ENTRY_ATTR_1G(prot),
dev_va + REG_NS_L1ENTRY_ATTR(vid, gb));
}
static void __set_l1entry_attr_with_fmpt(void *dev_va, unsigned int gb,
unsigned int vid, struct fmpt *fmpt)
{
if (fmpt->gran_1g) {
__set_l1entry_attr_with_prot(dev_va, gb, vid, fmpt->prot);
} else {
/* Order against writes to the SMPT. */
writel(config_gran_mask | L1ENTRY_ATTR_L2TABLE_EN,
dev_va + REG_NS_L1ENTRY_ATTR(vid, gb));
}
}
static void __set_l1entry_l2table_addr(void *dev_va, unsigned int gb,
unsigned int vid, phys_addr_t addr)
{
/* Order against writes to the SMPT. */
writel(L1ENTRY_L2TABLE_ADDR(addr),
dev_va + REG_NS_L1ENTRY_L2TABLE_ADDR(vid, gb));
}
static void init_with_prot(void *dev_va, enum mpt_prot prot)
{
unsigned int gb, vid;
for_each_gb_and_vid(gb, vid)
__set_l1entry_attr_with_prot(dev_va, gb, vid, prot);
}
static void init_with_mpt(void *dev_va, struct mpt *mpt)
{
unsigned int gb, vid;
struct fmpt *fmpt;
for_each_gb_and_vid(gb, vid) {
fmpt = &mpt->fmpt[gb];
__set_l1entry_l2table_addr(dev_va, gb, vid, __hyp_pa(fmpt->smpt));
__set_l1entry_attr_with_fmpt(dev_va, gb, vid, fmpt);
}
}
static void apply_range(void *dev_va, struct mpt *mpt, u32 first_gb, u32 last_gb)
{
unsigned int gb, vid;
struct fmpt *fmpt;
for_each_gb_in_range(gb, first_gb, last_gb) {
fmpt = &mpt->fmpt[gb];
if (fmpt->flags & MPT_UPDATE_L1) {
for_each_vid(vid)
__set_l1entry_attr_with_fmpt(dev_va, gb, vid, fmpt);
}
}
}
static void prepare_range(struct mpt *mpt, phys_addr_t first_byte,
phys_addr_t last_byte, enum mpt_prot prot)
{
unsigned int first_gb = first_byte / SZ_1G;
unsigned int last_gb = last_byte / SZ_1G;
size_t start_gb_byte, end_gb_byte;
unsigned int gb;
struct fmpt *fmpt;
for_each_gb_in_range(gb, first_gb, last_gb) {
fmpt = &mpt->fmpt[gb];
start_gb_byte = (gb == first_gb) ? first_byte % SZ_1G : 0;
end_gb_byte = (gb == last_gb) ? (last_byte % SZ_1G) + 1 : SZ_1G;
__set_fmpt_range(fmpt, start_gb_byte, end_gb_byte, prot);
if (fmpt->flags & MPT_UPDATE_L2)
kvm_flush_dcache_to_poc(fmpt->smpt, smpt_size());
}
}
static const struct s2mpu_mpt_ops this_ops = {
.smpt_size = smpt_size,
.init_with_prot = init_with_prot,
.init_with_mpt = init_with_mpt,
.apply_range = apply_range,
.prepare_range = prepare_range,
.pte_from_addr_smpt = pte_from_addr_smpt,
};
const struct s2mpu_mpt_ops *s2mpu_get_mpt_ops(struct s2mpu_mpt_cfg cfg)
{
/* If called before with different version return NULL. */
if (WARN_ON(this_version && (this_version != cfg.version)))
return NULL;
/* 2MB granularity not supported in V9 */
if ((cfg.version == S2MPU_VERSION_9) && (SMPT_GRAN_ATTR != L1ENTRY_ATTR_GRAN_2M)) {
config_prot_bits = V9_MPT_PROT_BITS;
config_access_shift = V9_MPT_ACCESS_SHIFT;
config_lut_prot = v9_mpt_prot_doubleword;
config_gran_mask = L1ENTRY_ATTR_GRAN(SMPT_GRAN_ATTR, V9_L1ENTRY_ATTR_GRAN_MASK);
this_version = cfg.version;
return &this_ops;
} else if ((cfg.version == S2MPU_VERSION_2) || (cfg.version == S2MPU_VERSION_1)) {
config_prot_bits = MPT_PROT_BITS;
config_access_shift = MPT_ACCESS_SHIFT;
config_lut_prot = mpt_prot_doubleword;
config_gran_mask = L1ENTRY_ATTR_GRAN(SMPT_GRAN_ATTR, L1ENTRY_ATTR_GRAN_MASK);
this_version = cfg.version;
return &this_ops;
}
return NULL;
}

703
arch/arm64/kvm/hyp/nvhe/iommu/s2mpu.c
View File

@@ -1,703 +0,0 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2021 - Google LLC
* Author: David Brazdil <dbrazdil@google.com>
*/
#include <linux/kvm_host.h>
#include <asm/kvm_asm.h>
#include <asm/kvm_hyp.h>
#include <asm/kvm_mmu.h>
#include <asm/kvm_s2mpu.h>
#include <linux/arm-smccc.h>
#include <nvhe/iommu.h>
#include <nvhe/memory.h>
#include <nvhe/mm.h>
#include <nvhe/spinlock.h>
#include <nvhe/trap_handler.h>
#include <asm/io-mpt-s2mpu.h>
#define SMC_CMD_PREPARE_PD_ONOFF 0x82000410
#define SMC_MODE_POWER_UP 1
#define PA_MAX ((phys_addr_t)SZ_1G * NR_GIGABYTES)
#define SYNC_MAX_RETRIES 5
#define SYNC_TIMEOUT 5
#define SYNC_TIMEOUT_MULTIPLIER 3
#define CTX_CFG_ENTRY(ctxid, nr_ctx, vid) \
(CONTEXT_CFG_VALID_VID_CTX_VID(ctxid, vid) \
| (((ctxid) < (nr_ctx)) ? CONTEXT_CFG_VALID_VID_CTX_VALID(ctxid) : 0))
#define for_each_child(child, dev) \
list_for_each_entry((child), &(dev)->children, siblings)
/* HW version-specific operations. */
struct s2mpu_reg_ops {
int (*init)(struct pkvm_iommu *dev);
void (*set_control_regs)(struct pkvm_iommu *dev);
u32 (*host_mmio_reg_access_mask)(size_t off, bool is_write);
};
struct s2mpu_drv_data {
u32 version;
u32 context_cfg_valid_vid;
};
static const struct s2mpu_mpt_ops *mpt_ops;
static const struct s2mpu_reg_ops *reg_ops;
static struct mpt host_mpt;
const struct pkvm_iommu_ops pkvm_s2mpu_ops;
const struct pkvm_iommu_ops pkvm_sysmmu_sync_ops;
static inline enum mpt_prot prot_to_mpt(enum kvm_pgtable_prot prot)
{
return ((prot & KVM_PGTABLE_PROT_R) ? MPT_PROT_R : 0) |
((prot & KVM_PGTABLE_PROT_W) ? MPT_PROT_W : 0);
}
static bool is_version(struct pkvm_iommu *dev, u32 version)
{
struct s2mpu_drv_data *data = (struct s2mpu_drv_data *)dev->data;
return (data->version & VERSION_CHECK_MASK) == version;
}
static u32 __context_cfg_valid_vid(struct pkvm_iommu *dev, u32 vid_bmap)
{
struct s2mpu_drv_data *data = (struct s2mpu_drv_data *)dev->data;
u8 ctx_vid[NR_CTX_IDS] = { 0 };
unsigned int vid, ctx = 0;
unsigned int num_ctx;
u32 res;
/* Only initialize once. */
if (data->context_cfg_valid_vid)
return data->context_cfg_valid_vid;
num_ctx = readl_relaxed(dev->va + REG_NS_NUM_CONTEXT) & NUM_CONTEXT_MASK;
while (vid_bmap) {
/* Break if we cannot allocate more. */
if (ctx >= num_ctx)
break;
vid = __ffs(vid_bmap);
vid_bmap &= ~BIT(vid);
ctx_vid[ctx++] = vid;
}
/* The following loop was unrolled so bitmasks are constant. */
BUILD_BUG_ON(NR_CTX_IDS != 8);
res = CTX_CFG_ENTRY(0, ctx, ctx_vid[0])
| CTX_CFG_ENTRY(1, ctx, ctx_vid[1])
| CTX_CFG_ENTRY(2, ctx, ctx_vid[2])
| CTX_CFG_ENTRY(3, ctx, ctx_vid[3])
| CTX_CFG_ENTRY(4, ctx, ctx_vid[4])
| CTX_CFG_ENTRY(5, ctx, ctx_vid[5])
| CTX_CFG_ENTRY(6, ctx, ctx_vid[6])
| CTX_CFG_ENTRY(7, ctx, ctx_vid[7]);
data->context_cfg_valid_vid = res;
return res;
}
static int __initialize_v2(struct pkvm_iommu *dev)
{
u32 ssmt_valid_vid_bmap, ctx_cfg;
/* Assume all VIDs may be generated by the connected SSMTs for now. */
ssmt_valid_vid_bmap = ALL_VIDS_BITMAP;
ctx_cfg = __context_cfg_valid_vid(dev, ssmt_valid_vid_bmap);
if (!ctx_cfg)
return -EINVAL;
/*
* Write CONTEXT_CFG_VALID_VID configuration before touching L1ENTRY*
* registers. Writes to those registers are ignored unless there is
* a context ID allocated to the corresponding VID (v2 only).
*/
writel_relaxed(ctx_cfg, dev->va + REG_NS_CONTEXT_CFG_VALID_VID);
return 0;
}
static int __initialize(struct pkvm_iommu *dev)
{
struct s2mpu_drv_data *data = (struct s2mpu_drv_data *)dev->data;
if (!data->version)
data->version = readl_relaxed(dev->va + REG_NS_VERSION);
switch (data->version & VERSION_CHECK_MASK) {
case S2MPU_VERSION_1:
return 0;
case S2MPU_VERSION_2:
return __initialize_v2(dev);
default:
return -EINVAL;
}
}
static void __set_control_regs(struct pkvm_iommu *dev)
{
u32 ctrl0 = 0, irq_vids;
/*
* Note: We set the values of CTRL0, CTRL1 and CFG registers here but we
* still rely on the correctness of their reset values. S2MPUs *must*
* reset to a state where all DMA traffic is blocked until the hypervisor
* writes its configuration to the S2MPU. A malicious EL1 could otherwise
* attempt to bypass the permission checks in the window between powering
* on the S2MPU and this function being called.
*/
/* Enable the S2MPU, otherwise all traffic would be allowed through. */
ctrl0 |= CTRL0_ENABLE;
/*
* Enable interrupts on fault for all VIDs. The IRQ must also be
* specified in DT to get unmasked in the GIC.
*/
ctrl0 |= CTRL0_INTERRUPT_ENABLE;
irq_vids = ALL_VIDS_BITMAP;
/* Return SLVERR/DECERR to device on permission fault. */
ctrl0 |= is_version(dev, S2MPU_VERSION_2) ? CTRL0_FAULT_RESP_TYPE_DECERR
: CTRL0_FAULT_RESP_TYPE_SLVERR;
writel_relaxed(irq_vids, dev->va + REG_NS_INTERRUPT_ENABLE_PER_VID_SET);
writel_relaxed(0, dev->va + REG_NS_CFG);
writel_relaxed(0, dev->va + REG_NS_CTRL1);
writel_relaxed(ctrl0, dev->va + REG_NS_CTRL0);
}
static void __set_control_regs_v9(struct pkvm_iommu *dev)
{
/* Return DECERR to device on permission fault. */
writel_relaxed(ALL_VIDS_BITMAP,
dev->va + REG_NS_V9_CTRL_ERR_RESP_T_PER_VID_SET);
/*
* Enable interrupts on fault for all VIDs. The IRQ must also be
* specified in DT to get unmasked in the GIC.
*/
writel_relaxed(ALL_VIDS_BITMAP,
dev->va + REG_NS_INTERRUPT_ENABLE_PER_VID_SET);
writel_relaxed(0, dev->va + REG_NS_CTRL0);
/* Enable the S2MPU, otherwise all traffic would be allowed through. */
writel_relaxed(ALL_VIDS_BITMAP,
dev->va + REG_NS_V9_CTRL_PROT_EN_PER_VID_SET);
writel_relaxed(0, dev->va + REG_NS_V9_CFG_MPTW_ATTRIBUTE);
}
/*
* Poll the given SFR until its value has all bits of a given mask set.
* Returns true if successful, false if not successful after a given number of
* attempts.
*/
static bool __wait_until(void __iomem *addr, u32 mask, size_t max_attempts)
{
size_t i;
for (i = 0; i < max_attempts; i++) {
if ((readl_relaxed(addr) & mask) == mask)
return true;
}
return false;
}
/* Poll the given SFR as long as its value has all bits of a given mask set. */
static void __wait_while(void __iomem *addr, u32 mask)
{
while ((readl_relaxed(addr) & mask) == mask)
continue;
}
static void __sync_cmd_start(struct pkvm_iommu *sync)
{
writel_relaxed(SYNC_CMD_SYNC, sync->va + REG_NS_SYNC_CMD);
}
static void __invalidation_barrier_slow(struct pkvm_iommu *sync)
{
size_t i, timeout;
/*
* Wait for transactions to drain if SysMMU_SYNCs were registered.
* Assumes that they are in the same power domain as the S2MPU.
*
* The algorithm will try initiating the SYNC if the SYNC_COMP_COMPLETE
* bit has not been set after a given number of attempts, increasing the
* timeout exponentially each time. If this cycle fails a given number
* of times, the algorithm will give up completely to avoid deadlock.
*/
timeout = SYNC_TIMEOUT;
for (i = 0; i < SYNC_MAX_RETRIES; i++) {
__sync_cmd_start(sync);
if (__wait_until(sync->va + REG_NS_SYNC_COMP, SYNC_COMP_COMPLETE, timeout))
break;
timeout *= SYNC_TIMEOUT_MULTIPLIER;
}
}
/* Initiate invalidation barrier. */
static void __invalidation_barrier_init(struct pkvm_iommu *dev)
{
struct pkvm_iommu *sync;
for_each_child(sync, dev)
__sync_cmd_start(sync);
}
/* Wait for invalidation to complete. */
static void __invalidation_barrier_complete(struct pkvm_iommu *dev)
{
struct pkvm_iommu *sync;
/*
* Check if the SYNC_COMP_COMPLETE bit has been set for individual
* devices. If not, fall back to non-parallel invalidation.
*/
for_each_child(sync, dev) {
if (!(readl_relaxed(sync->va + REG_NS_SYNC_COMP) & SYNC_COMP_COMPLETE))
__invalidation_barrier_slow(sync);
}
/* Must not access SFRs while S2MPU is busy invalidating */
if (is_version(dev, S2MPU_VERSION_2) || is_version(dev, S2MPU_VERSION_9)) {
__wait_while(dev->va + REG_NS_STATUS,
STATUS_BUSY | STATUS_ON_INVALIDATING);
}
}
static void __all_invalidation(struct pkvm_iommu *dev)
{
writel_relaxed(INVALIDATION_INVALIDATE, dev->va + REG_NS_ALL_INVALIDATION);
__invalidation_barrier_init(dev);
__invalidation_barrier_complete(dev);
}
static void __range_invalidation_init(struct pkvm_iommu *dev, phys_addr_t first_byte,
phys_addr_t last_byte)
{
u32 start_ppn = first_byte >> RANGE_INVALIDATION_PPN_SHIFT;
u32 end_ppn = last_byte >> RANGE_INVALIDATION_PPN_SHIFT;
writel_relaxed(start_ppn, dev->va + REG_NS_RANGE_INVALIDATION_START_PPN);
writel_relaxed(end_ppn, dev->va + REG_NS_RANGE_INVALIDATION_END_PPN);
writel_relaxed(INVALIDATION_INVALIDATE, dev->va + REG_NS_RANGE_INVALIDATION);
__invalidation_barrier_init(dev);
}
/*
* Initialize S2MPU device and set all GB regions to 1G granularity with
* given protection bits.
*/
static int initialize_with_prot(struct pkvm_iommu *dev, enum mpt_prot prot)
{
int ret;
ret = reg_ops->init(dev);
if (ret)
return ret;
mpt_ops->init_with_prot(dev->va, prot);
__all_invalidation(dev);
/* Set control registers, enable the S2MPU. */
reg_ops->set_control_regs(dev);
return 0;
}
/*
* Initialize S2MPU device, set L2 table addresses and configure L1TABLE_ATTR
* registers according to the given MPT struct.
*/
static int initialize_with_mpt(struct pkvm_iommu *dev, struct mpt *mpt)
{
int ret;
ret = reg_ops->init(dev);
if (ret)
return ret;
mpt_ops->init_with_mpt(dev->va, mpt);
__all_invalidation(dev);
/* Set control registers, enable the S2MPU. */
reg_ops->set_control_regs(dev);
return 0;
}
static bool to_valid_range(phys_addr_t *start, phys_addr_t *end)
{
phys_addr_t new_start = *start;
phys_addr_t new_end = *end;
if (new_end > PA_MAX)
new_end = PA_MAX;
new_start = ALIGN_DOWN(new_start, SMPT_GRAN);
new_end = ALIGN(new_end, SMPT_GRAN);
if (new_start >= new_end)
return false;
*start = new_start;
*end = new_end;
return true;
}
static void __mpt_idmap_prepare(struct mpt *mpt, phys_addr_t first_byte,
phys_addr_t last_byte, enum mpt_prot prot)
{
mpt_ops->prepare_range(mpt, first_byte, last_byte, prot);
}
static void __mpt_idmap_apply(struct pkvm_iommu *dev, struct mpt *mpt,
phys_addr_t first_byte, phys_addr_t last_byte)
{
unsigned int first_gb = first_byte / SZ_1G;
unsigned int last_gb = last_byte / SZ_1G;
mpt_ops->apply_range(dev->va, mpt, first_gb, last_gb);
/* Initiate invalidation, completed in __mdt_idmap_complete. */
__range_invalidation_init(dev, first_byte, last_byte);
}
static void __mpt_idmap_complete(struct pkvm_iommu *dev, struct mpt *mpt)
{
__invalidation_barrier_complete(dev);
}
static void s2mpu_host_stage2_idmap_prepare(phys_addr_t start, phys_addr_t end,
enum kvm_pgtable_prot prot)
{
if (!to_valid_range(&start, &end))
return;
__mpt_idmap_prepare(&host_mpt, start, end - 1, prot_to_mpt(prot));
}
static void s2mpu_host_stage2_idmap_apply(struct pkvm_iommu *dev,
phys_addr_t start, phys_addr_t end)
{
if (!to_valid_range(&start, &end))
return;
__mpt_idmap_apply(dev, &host_mpt, start, end - 1);
}
static void s2mpu_host_stage2_idmap_complete(struct pkvm_iommu *dev)
{
__mpt_idmap_complete(dev, &host_mpt);
}
static int s2mpu_resume(struct pkvm_iommu *dev)
{
/*
* Initialize the S2MPU with the host stage-2 MPT. It is paramount
* that the S2MPU reset state is enabled and blocking all traffic,
* otherwise the host would not be forced to call the resume HVC
* before issuing DMA traffic.
*/
return initialize_with_mpt(dev, &host_mpt);
}
static int s2mpu_suspend(struct pkvm_iommu *dev)
{
/*
* Stop updating the S2MPU when the host informs us about the intention
* to suspend it. Writes to powered-down MMIO registers would trigger
* SErrors in EL1 otherwise. However, hyp must put S2MPU back to
* blocking state first, in case the host does not actually power it
* down and continues issuing DMA traffic.
*/
return initialize_with_prot(dev, MPT_PROT_NONE);
}
static u32 host_mmio_reg_access_mask_v9(size_t off, bool is_write)
{
const u32 no_access = 0;
const u32 read_write = (u32)(-1);
const u32 read_only = is_write ? no_access : read_write;
const u32 write_only = is_write ? read_write : no_access;
switch (off) {
/* Allow reading control registers for debugging. */
case REG_NS_CTRL0:
return read_only & V9_CTRL0_MASK;
case REG_NS_V9_CTRL_ERR_RESP_T_PER_VID_SET:
return read_only & ALL_VIDS_BITMAP;
case REG_NS_V9_CTRL_PROT_EN_PER_VID_SET:
return read_only & ALL_VIDS_BITMAP;
case REG_NS_V9_READ_STLB:
return write_only & (V9_READ_STLB_MASK_TYPEA|V9_READ_STLB_MASK_TYPEB);
case REG_NS_V9_READ_STLB_TPN:
return read_only & V9_READ_STLB_TPN_MASK;
case REG_NS_V9_READ_STLB_TAG_PPN:
return read_only & V9_READ_STLB_TAG_PPN_MASK;
case REG_NS_V9_READ_STLB_TAG_OTHERS:
return read_only & V9_READ_STLB_TAG_OTHERS_MASK;
case REG_NS_V9_READ_STLB_DATA:
return read_only;
case REG_NS_V9_MPTC_INFO:
return read_only & V9_READ_MPTC_INFO_MASK;
case REG_NS_V9_READ_MPTC:
return write_only & V9_READ_MPTC_MASK;
case REG_NS_V9_READ_MPTC_TAG_PPN:
return read_only & V9_READ_MPTC_TAG_PPN_MASK;
case REG_NS_V9_READ_MPTC_TAG_OTHERS:
return read_only & V9_READ_MPTC_TAG_OTHERS_MASK;
case REG_NS_V9_READ_MPTC_DATA:
return read_only;
case REG_NS_V9_PMMU_INFO:
return read_only & V9_READ_PMMU_INFO_MASK;
case REG_NS_V9_READ_PTLB:
return write_only & V9_READ_PTLB_MASK;
case REG_NS_V9_READ_PTLB_TAG:
return read_only & V9_READ_PTLB_TAG_MASK;
case REG_NS_V9_READ_PTLB_DATA_S1_EN_PPN_AP:
return read_only & V9_READ_PTLB_DATA_S1_ENABLE_PPN_AP_MASK;
case REG_NS_V9_READ_PTLB_DATA_S1_DIS_AP_LIST:
return read_only;
case REG_NS_V9_PMMU_INDICATOR:
return read_only & V9_READ_PMMU_INDICATOR_MASK;
case REG_NS_V9_SWALKER_INFO:
return read_only&V9_SWALKER_INFO_MASK;
};
if (off >= REG_NS_V9_PMMU_PTLB_INFO(0) && off < REG_NS_V9_PMMU_PTLB_INFO(V9_MAX_PTLB_NUM))
return read_only&V9_READ_PMMU_PTLB_INFO_MASK;
if (off >= REG_NS_V9_STLB_INFO(0) && off < REG_NS_V9_STLB_INFO(V9_MAX_STLB_NUM))
return read_only&V9_READ_SLTB_INFO_MASK;
return no_access;
}
static u32 host_mmio_reg_access_mask_v1_v2(size_t off, bool is_write)
{
const u32 no_access = 0;
const u32 read_write = (u32)(-1);
const u32 read_only = is_write ? no_access : read_write;
const u32 write_only = is_write ? read_write : no_access;
switch (off) {
/* Allow reading control registers for debugging. */
case REG_NS_CTRL0:
return read_only & CTRL0_MASK;
case REG_NS_CTRL1:
return read_only & CTRL1_MASK;
/* Allow reading MPTC entries for debugging. That involves:
* - writing (set,way) to READ_MPTC
* - reading READ_MPTC_*
*/
case REG_NS_READ_MPTC:
return write_only & READ_MPTC_MASK;
case REG_NS_READ_MPTC_TAG_PPN:
return read_only & READ_MPTC_TAG_PPN_MASK;
case REG_NS_READ_MPTC_TAG_OTHERS:
return read_only & READ_MPTC_TAG_OTHERS_MASK;
case REG_NS_READ_MPTC_DATA:
return read_only;
};
return no_access;
}
static u32 host_mmio_reg_access_mask(size_t off, bool is_write)
{
const u32 no_access = 0;
const u32 read_write = (u32)(-1);
const u32 read_only = is_write ? no_access : read_write;
const u32 write_only = is_write ? read_write : no_access;
u32 masked_off;
switch (off) {
case REG_NS_CFG:
return read_only & CFG_MASK;
/* Allow EL1 IRQ handler to clear interrupts. */
case REG_NS_INTERRUPT_CLEAR:
return write_only & ALL_VIDS_BITMAP;
/* Allow reading number of sets used by MPTC. */
case REG_NS_INFO:
return read_only & INFO_NUM_SET_MASK;
/* Allow EL1 IRQ handler to read bitmap of pending interrupts. */
case REG_NS_FAULT_STATUS:
return read_only & ALL_VIDS_BITMAP;
}
/* Allow reading L1ENTRY registers for debugging. */
if (off >= REG_NS_L1ENTRY_L2TABLE_ADDR(0, 0) &&
off < REG_NS_L1ENTRY_ATTR(NR_VIDS, 0))
return read_only;
/* Allow EL1 IRQ handler to read fault information. */
masked_off = off & ~REG_NS_FAULT_VID_MASK;
if ((masked_off == REG_NS_FAULT_PA_LOW(0)) ||
(masked_off == REG_NS_FAULT_PA_HIGH(0)) ||
(masked_off == REG_NS_FAULT_INFO(0)))
return read_only;
/* Check version-specific registers. */
return reg_ops->host_mmio_reg_access_mask(off, is_write);
}
static bool s2mpu_host_dabt_handler(struct pkvm_iommu *dev,
struct kvm_cpu_context *host_ctxt,
u32 esr, size_t off)
{
bool is_write = esr & ESR_ELx_WNR;
unsigned int len = BIT((esr & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT);
int rd = (esr & ESR_ELx_SRT_MASK) >> ESR_ELx_SRT_SHIFT;
u32 mask;
/* Only handle MMIO access with u32 size and alignment. */
if ((len != sizeof(u32)) || (off & (sizeof(u32) - 1)))
return false;
mask = host_mmio_reg_access_mask(off, is_write);
if (!mask)
return false;
if (is_write)
writel_relaxed(cpu_reg(host_ctxt, rd) & mask, dev->va + off);
else
cpu_reg(host_ctxt, rd) = readl_relaxed(dev->va + off) & mask;
return true;
}
/*
* Operations that differ between versions. We need to maintain
* old behaviour were v1 and v2 can be used together.
*/
const struct s2mpu_reg_ops ops_v1_v2 = {
.init = __initialize,
.host_mmio_reg_access_mask = host_mmio_reg_access_mask_v1_v2,
.set_control_regs = __set_control_regs,
};
const struct s2mpu_reg_ops ops_v9 = {
.init = __initialize_v2,
.host_mmio_reg_access_mask = host_mmio_reg_access_mask_v9,
.set_control_regs = __set_control_regs_v9,
};
static int s2mpu_init(void *data, size_t size)
{
struct mpt in_mpt;
u32 *smpt;
phys_addr_t pa;
unsigned int gb;
int ret = 0;
int smpt_nr_pages, smpt_size;
struct s2mpu_mpt_cfg cfg;
if (size != sizeof(in_mpt))
return -EINVAL;
/* The host can concurrently modify 'data'. Copy it to avoid TOCTOU. */
memcpy(&in_mpt, data, sizeof(in_mpt));
cfg.version = in_mpt.version;
/* Make sure the version sent is supported by the driver. */
if ((cfg.version == S2MPU_VERSION_1) || (cfg.version == S2MPU_VERSION_2))
reg_ops = &ops_v1_v2;
else if (cfg.version == S2MPU_VERSION_9)
reg_ops = &ops_v9;
else
return -ENODEV;
/* Get page table operations for this version. */
mpt_ops = s2mpu_get_mpt_ops(cfg);
/* If version is wrong return. */
if (!mpt_ops)
return -EINVAL;
smpt_size = mpt_ops->smpt_size();
smpt_nr_pages = smpt_size / PAGE_SIZE;
/* Take ownership of all SMPT buffers. This will also map them in. */
for_each_gb(gb) {
smpt = kern_hyp_va(in_mpt.fmpt[gb].smpt);
pa = __hyp_pa(smpt);
if (!IS_ALIGNED(pa, smpt_size)) {
ret = -EINVAL;
break;
}
ret = __pkvm_host_donate_hyp(pa >> PAGE_SHIFT, smpt_nr_pages);
if (ret)
break;
host_mpt.fmpt[gb] = (struct fmpt){
.smpt = smpt,
.gran_1g = true,
.prot = MPT_PROT_RW,
};
}
/* Try to return memory back if there was an error. */
if (ret) {
for_each_gb(gb) {
smpt = host_mpt.fmpt[gb].smpt;
if (!smpt)
break;
WARN_ON(__pkvm_hyp_donate_host(__hyp_pa(smpt) >> PAGE_SHIFT,
smpt_nr_pages));
}
memset(&host_mpt, 0, sizeof(host_mpt));
}
return ret;
}
static int s2mpu_validate(struct pkvm_iommu *dev)
{
if (dev->size != S2MPU_MMIO_SIZE)
return -EINVAL;
return 0;
}
static int s2mpu_validate_child(struct pkvm_iommu *dev, struct pkvm_iommu *child)
{
if (child->ops != &pkvm_sysmmu_sync_ops)
return -EINVAL;
return 0;
}
static int sysmmu_sync_validate(struct pkvm_iommu *dev)
{
if (dev->size != SYSMMU_SYNC_S2_MMIO_SIZE)
return -EINVAL;
if (!dev->parent || dev->parent->ops != &pkvm_s2mpu_ops)
return -EINVAL;
return 0;
}
const struct pkvm_iommu_ops pkvm_s2mpu_ops = (struct pkvm_iommu_ops){
.init = s2mpu_init,
.validate = s2mpu_validate,
.validate_child = s2mpu_validate_child,
.resume = s2mpu_resume,
.suspend = s2mpu_suspend,
.host_stage2_idmap_prepare = s2mpu_host_stage2_idmap_prepare,
.host_stage2_idmap_apply = s2mpu_host_stage2_idmap_apply,
.host_stage2_idmap_complete = s2mpu_host_stage2_idmap_complete,
.host_dabt_handler = s2mpu_host_dabt_handler,
.data_size = sizeof(struct s2mpu_drv_data),
};
const struct pkvm_iommu_ops pkvm_sysmmu_sync_ops = (struct pkvm_iommu_ops){
.validate = sysmmu_sync_validate,
};
struct pkvm_iommu_driver pkvm_s2mpu_driver = (struct pkvm_iommu_driver){
.ops = &pkvm_s2mpu_ops,
};
struct pkvm_iommu_driver pkvm_sysmmu_sync_driver = (struct pkvm_iommu_driver){
.ops = &pkvm_sysmmu_sync_ops,
};

6
arch/arm64/kvm/iommu/Makefile
View File

@@ -1,6 +0,0 @@
# SPDX-License-Identifier: GPL-2.0
#
# Makefile for Kernel-based Virtual Machine module
#
obj-$(CONFIG_KVM_S2MPU) += s2mpu.o

131
arch/arm64/kvm/iommu/s2mpu.c
View File

@@ -1,131 +0,0 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2021 - Google LLC
* Author: David Brazdil <dbrazdil@google.com>
*/
#include <linux/kvm_host.h>
#include <asm/kvm_s2mpu.h>
#include <asm/kvm_host.h>
#include <asm/kvm_asm.h>
/* For an nvhe symbol get the kernel linear address of it. */
#define ksym_ref_addr_nvhe(x) kvm_ksym_ref(&kvm_nvhe_sym(x))
static int init_s2mpu_driver(u32 version)
{
static DEFINE_MUTEX(lock);
static bool init_done;
struct mpt *mpt;
unsigned int gb;
unsigned long addr;
u64 pfn;
int ret = 0;
const int smpt_order = smpt_order_from_version(version);
mutex_lock(&lock);
if (init_done)
goto out;
/* Allocate a page for driver data. Must fit MPT descriptor. */
BUILD_BUG_ON(sizeof(*mpt) > PAGE_SIZE);
addr = __get_free_page(GFP_KERNEL);
if (!addr) {
ret = -ENOMEM;
goto out;
}
mpt = (struct mpt *)addr;
/* Allocate SMPT buffers. */
for_each_gb(gb) {
addr = __get_free_pages(GFP_KERNEL, smpt_order);
if (!addr) {
ret = -ENOMEM;
goto out_free;
}
mpt->fmpt[gb].smpt = (u32 *)addr;
}
mpt->version = version;
/* Share MPT descriptor with hyp. */
pfn = __pa(mpt) >> PAGE_SHIFT;
ret = kvm_call_hyp_nvhe(__pkvm_host_share_hyp, pfn);
if (ret)
goto out_free;
/* Hypercall to initialize EL2 driver. */
ret = pkvm_iommu_driver_init(ksym_ref_addr_nvhe(pkvm_s2mpu_driver),
mpt, sizeof(*mpt));
if (ret)
goto out_unshare;
init_done = true;
out_unshare:
WARN_ON(kvm_call_hyp_nvhe(__pkvm_host_unshare_hyp, pfn));
out_free:
/* TODO - will driver return the memory? */
if (ret) {
for_each_gb(gb)
free_pages((unsigned long)mpt->fmpt[gb].smpt, smpt_order);
free_page((unsigned long)mpt);
}
out:
mutex_unlock(&lock);
return ret;
}
int pkvm_iommu_s2mpu_init(u32 version)
{
if (!is_protected_kvm_enabled())
return -ENODEV;
return init_s2mpu_driver(version);
}
EXPORT_SYMBOL_GPL(pkvm_iommu_s2mpu_init);
int pkvm_iommu_s2mpu_register(struct device *dev, phys_addr_t addr)
{
if (!is_protected_kvm_enabled())
return -ENODEV;
return pkvm_iommu_register(dev, ksym_ref_addr_nvhe(pkvm_s2mpu_driver),
addr, S2MPU_MMIO_SIZE, NULL);
}
EXPORT_SYMBOL_GPL(pkvm_iommu_s2mpu_register);
static int init_sysmmu_sync_driver(void)
{
static DEFINE_MUTEX(lock);
static bool init_done;
int ret = 0;
mutex_lock(&lock);
if (!init_done) {
ret = pkvm_iommu_driver_init(ksym_ref_addr_nvhe(pkvm_sysmmu_sync_driver),
NULL, 0);
init_done = !ret;
}
mutex_unlock(&lock);
return ret;
}
int pkvm_iommu_sysmmu_sync_register(struct device *dev, phys_addr_t addr,
struct device *parent)
{
int ret;
if (!is_protected_kvm_enabled())
return -ENODEV;
ret = init_sysmmu_sync_driver();
if (ret)
return ret;
return pkvm_iommu_register(dev, ksym_ref_addr_nvhe(pkvm_sysmmu_sync_driver),
addr + SYSMMU_SYNC_S2_OFFSET,
SYSMMU_SYNC_S2_MMIO_SIZE, parent);
}
EXPORT_SYMBOL_GPL(pkvm_iommu_sysmmu_sync_register);