diff --git a/arch/arm64/include/asm/kvm_s2mpu.h b/arch/arm64/include/asm/kvm_s2mpu.h index d1ed76c5f066..b8a0cd6701bd 100644 --- a/arch/arm64/include/asm/kvm_s2mpu.h +++ b/arch/arm64/include/asm/kvm_s2mpu.h @@ -9,18 +9,41 @@ #include +#define S2MPU_MMIO_SIZE SZ_64K + #define NR_VIDS 8 #define NR_CTX_IDS 8 #define ALL_VIDS_BITMAP GENMASK(NR_VIDS - 1, 0) +#define REG_NS_CTRL0 0x0 +#define REG_NS_CTRL1 0x4 +#define REG_NS_CFG 0x10 +#define REG_NS_INTERRUPT_ENABLE_PER_VID_SET 0x20 #define REG_NS_INTERRUPT_CLEAR 0x2c #define REG_NS_VERSION 0x60 #define REG_NS_NUM_CONTEXT 0x100 +#define REG_NS_CONTEXT_CFG_VALID_VID 0x104 +#define REG_NS_ALL_INVALIDATION 0x1000 #define REG_NS_FAULT_STATUS 0x2000 #define REG_NS_FAULT_PA_LOW(vid) (0x2004 + ((vid) * 0x20)) #define REG_NS_FAULT_PA_HIGH(vid) (0x2008 + ((vid) * 0x20)) #define REG_NS_FAULT_INFO(vid) (0x2010 + ((vid) * 0x20)) +#define REG_NS_L1ENTRY_ATTR(vid, gb) (0x4004 + ((vid) * 0x200) + ((gb) * 0x8)) + +#define CTRL0_ENABLE BIT(0) +#define CTRL0_INTERRUPT_ENABLE BIT(1) +#define CTRL0_FAULT_RESP_TYPE_SLVERR BIT(2) /* for v8 */ +#define CTRL0_FAULT_RESP_TYPE_DECERR BIT(2) /* for v9 */ + +#define CTRL1_DISABLE_CHK_S1L1PTW BIT(0) +#define CTRL1_DISABLE_CHK_S1L2PTW BIT(1) +#define CTRL1_ENABLE_PAGE_SIZE_AWARENESS BIT(2) +#define CTRL1_DISABLE_CHK_USER_MATCHED_REQ BIT(3) + +#define CFG_MPTW_CACHE_OVERRIDE BIT(0) +#define CFG_MPTW_QOS_OVERRIDE BIT(8) +#define CFG_MPTW_SHAREABLE BIT(16) /* For use with hi_lo_readq_relaxed(). */ #define REG_NS_FAULT_PA_HIGH_LOW(vid) REG_NS_FAULT_PA_LOW(vid) @@ -41,6 +64,8 @@ #define CONTEXT_CFG_VALID_VID_CTX_VID(ctx, vid) \ FIELD_PREP(GENMASK((4 * (ctx) + 2), 4 * (ctx)), (vid)) +#define INVALIDATION_INVALIDATE BIT(0) + #define NR_FAULT_INFO_REGS 8 #define FAULT_INFO_VID_MASK GENMASK(26, 24) #define FAULT_INFO_TYPE_MASK GENMASK(23, 21) @@ -51,6 +76,25 @@ #define FAULT_INFO_LEN_MASK GENMASK(19, 16) #define FAULT_INFO_ID_MASK GENMASK(15, 0) +#define L1ENTRY_ATTR_PROT(prot) FIELD_PREP(GENMASK(2, 1), prot) +#define L1ENTRY_ATTR_1G(prot) L1ENTRY_ATTR_PROT(prot) + +#define NR_GIGABYTES 64 +#define RO_GIGABYTES_FIRST 4 +#define RO_GIGABYTES_LAST 33 + +/* + * Iterate over S2MPU gigabyte regions. Skip those that cannot be modified + * (the MMIO registers are read only, with reset value MPT_PROT_NONE). + */ +#define for_each_gb_in_range(i, first, last) \ + for ((i) = (first); (i) <= (last) && (i) < NR_GIGABYTES; \ + (i) = (((i) + 1 == RO_GIGABYTES_FIRST) ? RO_GIGABYTES_LAST : (i)) + 1) + +#define for_each_gb(i) for_each_gb_in_range(i, 0, NR_GIGABYTES - 1) +#define for_each_vid(i) for ((i) = 0; (i) < NR_VIDS; (i)++) +#define for_each_gb_and_vid(gb, vid) for_each_vid((vid)) for_each_gb((gb)) + enum s2mpu_version { S2MPU_VERSION_8 = 0x11000000, S2MPU_VERSION_9 = 0x20000000, @@ -71,6 +115,14 @@ struct s2mpu { u32 context_cfg_valid_vid; }; +enum mpt_prot { + MPT_PROT_NONE = 0, + MPT_PROT_R = BIT(0), + MPT_PROT_W = BIT(1), + MPT_PROT_RW = MPT_PROT_R | MPT_PROT_W, + MPT_PROT_MASK = MPT_PROT_RW, +}; + extern size_t kvm_nvhe_sym(kvm_hyp_nr_s2mpus); #define kvm_hyp_nr_s2mpus kvm_nvhe_sym(kvm_hyp_nr_s2mpus) diff --git a/arch/arm64/kvm/hyp/nvhe/iommu/s2mpu.c b/arch/arm64/kvm/hyp/nvhe/iommu/s2mpu.c index 43fc1e693aef..67cb98b3309a 100644 --- a/arch/arm64/kvm/hyp/nvhe/iommu/s2mpu.c +++ b/arch/arm64/kvm/hyp/nvhe/iommu/s2mpu.c @@ -6,9 +6,148 @@ #include +#include #include +#include +#include -size_t __ro_after_init kvm_hyp_nr_s2mpus; -struct s2mpu __ro_after_init *kvm_hyp_s2mpus; +#include -const struct kvm_iommu_ops kvm_s2mpu_ops = (struct kvm_iommu_ops){}; +#define for_each_s2mpu(i) \ + for ((i) = &kvm_hyp_s2mpus[0]; (i) != &kvm_hyp_s2mpus[kvm_hyp_nr_s2mpus]; (i)++) + +#define for_each_powered_s2mpu(i) \ + for_each_s2mpu((i)) if (is_powered_on((i))) + +size_t __ro_after_init kvm_hyp_nr_s2mpus; +struct s2mpu __ro_after_init *kvm_hyp_s2mpus; + +static bool is_version(struct s2mpu *dev, u32 version) +{ + return (dev->version & VERSION_CHECK_MASK) == version; +} + +static bool is_powered_on(struct s2mpu *dev) +{ + switch (dev->power_state) { + case S2MPU_POWER_ALWAYS_ON: + case S2MPU_POWER_ON: + return true; + case S2MPU_POWER_OFF: + return false; + default: + BUG(); + } +} + +/* + * Write CONTEXT_CFG_VALID_VID configuration before touching L1ENTRY* registers. + * Writes to those registers are ignored unless there is a context ID allocated + * to the corresponding VID (v9 only). + */ +static void __set_context_ids(struct s2mpu *dev) +{ + if (!is_version(dev, S2MPU_VERSION_9)) + return; + + writel_relaxed(dev->context_cfg_valid_vid, + dev->va + REG_NS_CONTEXT_CFG_VALID_VID); +} + +static void __set_control_regs(struct s2mpu *dev) +{ + u32 ctrl0 = 0, irq_vids; + + /* + * Note: We set the values of CTRL0, CTRL1 and CFG registers here but we + * still rely on the correctness of their reset values. S2MPUs *must* + * reset to a state where all DMA traffic is blocked until the hypervisor + * writes its configuration to the S2MPU. A malicious EL1 could otherwise + * attempt to bypass the permission checks in the window between powering + * on the S2MPU and this function being called. + */ + + /* Enable the S2MPU, otherwise all traffic would be allowed through. */ + ctrl0 |= CTRL0_ENABLE; + + /* + * Enable interrupts on fault for all VIDs. The IRQ must also be + * specified in DT to get unmasked in the GIC. + */ + ctrl0 |= CTRL0_INTERRUPT_ENABLE; + irq_vids = ALL_VIDS_BITMAP; + + /* Return SLVERR/DECERR to device on permission fault. */ + ctrl0 |= is_version(dev, S2MPU_VERSION_9) ? CTRL0_FAULT_RESP_TYPE_DECERR + : CTRL0_FAULT_RESP_TYPE_SLVERR; + + writel_relaxed(irq_vids, dev->va + REG_NS_INTERRUPT_ENABLE_PER_VID_SET); + writel_relaxed(0, dev->va + REG_NS_CFG); + writel_relaxed(0, dev->va + REG_NS_CTRL1); + writel_relaxed(ctrl0, dev->va + REG_NS_CTRL0); +} + +static void __all_invalidation(struct s2mpu *dev) +{ + writel_relaxed(INVALIDATION_INVALIDATE, + dev->va + REG_NS_ALL_INVALIDATION); +} + +static void __set_l1entry_attr_with_prot(struct s2mpu *dev, unsigned int gb, + unsigned int vid, enum mpt_prot prot) +{ + writel_relaxed(L1ENTRY_ATTR_1G(prot), + dev->va + REG_NS_L1ENTRY_ATTR(vid, gb)); +} + +/** + * Initialize S2MPU device and set all GB regions to 1G granularity with + * given protection bits. + */ +static void initialize_with_prot(struct s2mpu *dev, enum mpt_prot prot) +{ + unsigned int gb, vid; + + /* Must write CONTEXT_CFG_VALID_VID before setting L1ENTRY registers. */ + __set_context_ids(dev); + + for_each_gb_and_vid(gb, vid) + __set_l1entry_attr_with_prot(dev, gb, vid, prot); + __all_invalidation(dev); + + /* Set control registers, enable the S2MPU. */ + __set_control_regs(dev); +} + +static int s2mpu_init(void) +{ + struct s2mpu *dev; + int ret; + + /* Map data structures in EL2 stage-1. */ + ret = pkvm_create_mappings(kvm_hyp_s2mpus, + kvm_hyp_s2mpus + kvm_hyp_nr_s2mpus, + PAGE_HYP); + if (ret) + return ret; + + /* Map S2MPU MMIO regions in EL2 stage-1. */ + for_each_s2mpu(dev) { + dev->va = (void __iomem *)__pkvm_create_private_mapping( + dev->pa, S2MPU_MMIO_SIZE, PAGE_HYP_DEVICE); + if (IS_ERR_OR_NULL(dev->va)) + return PTR_ERR(dev->va); + } + + /* + * Program all S2MPUs powered on at boot. Note that they may not be in + * the blocking reset state as the bootloader may have programmed them. + */ + for_each_powered_s2mpu(dev) + initialize_with_prot(dev, MPT_PROT_RW); + return 0; +} + +const struct kvm_iommu_ops kvm_s2mpu_ops = (struct kvm_iommu_ops){ + .init = s2mpu_init, +}; diff --git a/arch/arm64/kvm/iommu/s2mpu.c b/arch/arm64/kvm/iommu/s2mpu.c index 6d990c47146c..9704d7951517 100644 --- a/arch/arm64/kvm/iommu/s2mpu.c +++ b/arch/arm64/kvm/iommu/s2mpu.c @@ -12,8 +12,6 @@ #include #include -#define S2MPU_MMIO_SIZE SZ_64K - #define CTX_CFG_ENTRY(ctxid, nr_ctx, vid) \ (CONTEXT_CFG_VALID_VID_CTX_VID(ctxid, vid) \ | (((ctxid) < (nr_ctx)) ? CONTEXT_CFG_VALID_VID_CTX_VALID(ctxid) : 0))