From 781efc8f25e7cfe233ed45245b509c626e46a13b Mon Sep 17 00:00:00 2001 From: Mostafa Saleh Date: Tue, 15 Nov 2022 11:18:33 +0000 Subject: [PATCH] ANDROID: KVM: arm64: s2mpu: Add SMPT and MPT functions to pgtable abstraction No functional change Move SMPT and MPT functions to io-pgtable-s2mpu.c as they will need to change later when new version of S2MPU is added Bug: 255731794 Change-Id: Ie890bd4e085c1e23a0d033147f955ba8789b8a28 Signed-off-by: Mostafa Saleh Signed-off-by: Quentin Perret --- arch/arm64/include/asm/kvm_s2mpu.h | 135 ------------------- arch/arm64/kvm/hyp/nvhe/iommu/io-mpt-s2mpu.c | 135 +++++++++++++++++++ 2 files changed, 135 insertions(+), 135 deletions(-) diff --git a/arch/arm64/include/asm/kvm_s2mpu.h b/arch/arm64/include/asm/kvm_s2mpu.h index b1075abd604c..2d9cd5509b16 100644 --- a/arch/arm64/include/asm/kvm_s2mpu.h +++ b/arch/arm64/include/asm/kvm_s2mpu.h @@ -203,13 +203,6 @@ enum mpt_prot { MPT_PROT_MASK = MPT_PROT_RW, }; -static const u64 mpt_prot_doubleword[] = { - [MPT_PROT_NONE] = 0x0000000000000000, - [MPT_PROT_R] = 0x5555555555555555, - [MPT_PROT_W] = 0xaaaaaaaaaaaaaaaa, - [MPT_PROT_RW] = 0xffffffffffffffff, -}; - enum mpt_update_flags { MPT_UPDATE_L1 = BIT(0), MPT_UPDATE_L2 = BIT(1), @@ -226,132 +219,4 @@ struct mpt { struct fmpt fmpt[NR_GIGABYTES]; }; -/* Set protection bits of SMPT in a given range without using memset. */ -static inline void __set_smpt_range_slow(u32 *smpt, size_t start_gb_byte, - size_t end_gb_byte, enum mpt_prot prot) -{ - size_t i, start_word_byte, end_word_byte, word_idx, first_elem, last_elem; - u32 val; - - /* Iterate over u32 words. */ - start_word_byte = start_gb_byte; - while (start_word_byte < end_gb_byte) { - /* Determine the range of bytes covered by this word. */ - word_idx = start_word_byte / SMPT_WORD_BYTE_RANGE; - end_word_byte = min( - ALIGN(start_word_byte + 1, SMPT_WORD_BYTE_RANGE), - end_gb_byte); - - /* Identify protection bit offsets within the word. */ - first_elem = (start_word_byte / SMPT_GRAN) % SMPT_ELEMS_PER_WORD; - last_elem = ((end_word_byte - 1) / SMPT_GRAN) % SMPT_ELEMS_PER_WORD; - - /* Modify the corresponding word. */ - val = READ_ONCE(smpt[word_idx]); - for (i = first_elem; i <= last_elem; i++) { - val &= ~(MPT_PROT_MASK << (i * MPT_PROT_BITS)); - val |= prot << (i * MPT_PROT_BITS); - } - WRITE_ONCE(smpt[word_idx], val); - - start_word_byte = end_word_byte; - } -} - -/* Set protection bits of SMPT in a given range. */ -static inline void __set_smpt_range(u32 *smpt, size_t start_gb_byte, - size_t end_gb_byte, enum mpt_prot prot) -{ - size_t interlude_start, interlude_end, interlude_bytes, word_idx; - char prot_byte = (char)mpt_prot_doubleword[prot]; - - if (start_gb_byte >= end_gb_byte) - return; - - /* Check if range spans at least one full u32 word. */ - interlude_start = ALIGN(start_gb_byte, SMPT_WORD_BYTE_RANGE); - interlude_end = ALIGN_DOWN(end_gb_byte, SMPT_WORD_BYTE_RANGE); - - /* If not, fall back to editing bits in the given range. */ - if (interlude_start >= interlude_end) { - __set_smpt_range_slow(smpt, start_gb_byte, end_gb_byte, prot); - return; - } - - /* Use bit-editing for prologue/epilogue, memset for interlude. */ - word_idx = interlude_start / SMPT_WORD_BYTE_RANGE; - interlude_bytes = (interlude_end - interlude_start) / SMPT_GRAN / SMPT_ELEMS_PER_BYTE; - - __set_smpt_range_slow(smpt, start_gb_byte, interlude_start, prot); - memset(&smpt[word_idx], prot_byte, interlude_bytes); - __set_smpt_range_slow(smpt, interlude_end, end_gb_byte, prot); -} - -/* Returns true if all SMPT protection bits match 'prot'. */ -static inline bool __is_smpt_uniform(u32 *smpt, enum mpt_prot prot) -{ - size_t i; - u64 *doublewords = (u64 *)smpt; - - for (i = 0; i < SMPT_NUM_WORDS / 2; i++) { - if (doublewords[i] != mpt_prot_doubleword[prot]) - return false; - } - return true; -} - -/* - * Set protection bits of FMPT/SMPT in a given range. - * Returns flags specifying whether L1/L2 changes need to be made visible - * to the device. - */ -static inline void __set_fmpt_range(struct fmpt *fmpt, size_t start_gb_byte, - size_t end_gb_byte, enum mpt_prot prot) -{ - if (start_gb_byte == 0 && end_gb_byte >= SZ_1G) { - /* Update covers the entire GB region. */ - if (fmpt->gran_1g && fmpt->prot == prot) { - fmpt->flags = 0; - return; - } - - fmpt->gran_1g = true; - fmpt->prot = prot; - fmpt->flags = MPT_UPDATE_L1; - return; - } - - if (fmpt->gran_1g) { - /* GB region currently uses 1G mapping. */ - if (fmpt->prot == prot) { - fmpt->flags = 0; - return; - } - - /* - * Range has different mapping than the rest of the GB. - * Convert to PAGE_SIZE mapping. - */ - fmpt->gran_1g = false; - __set_smpt_range(fmpt->smpt, 0, start_gb_byte, fmpt->prot); - __set_smpt_range(fmpt->smpt, start_gb_byte, end_gb_byte, prot); - __set_smpt_range(fmpt->smpt, end_gb_byte, SZ_1G, fmpt->prot); - fmpt->flags = MPT_UPDATE_L1 | MPT_UPDATE_L2; - return; - } - - /* GB region currently uses PAGE_SIZE mapping. */ - __set_smpt_range(fmpt->smpt, start_gb_byte, end_gb_byte, prot); - - /* Check if the entire GB region has the same prot bits. */ - if (!__is_smpt_uniform(fmpt->smpt, prot)) { - fmpt->flags = MPT_UPDATE_L2; - return; - } - - fmpt->gran_1g = true; - fmpt->prot = prot; - fmpt->flags = MPT_UPDATE_L1; -} - #endif /* __ARM64_KVM_S2MPU_H__ */ diff --git a/arch/arm64/kvm/hyp/nvhe/iommu/io-mpt-s2mpu.c b/arch/arm64/kvm/hyp/nvhe/iommu/io-mpt-s2mpu.c index 5ec941d842a5..a7715ac8f24e 100644 --- a/arch/arm64/kvm/hyp/nvhe/iommu/io-mpt-s2mpu.c +++ b/arch/arm64/kvm/hyp/nvhe/iommu/io-mpt-s2mpu.c @@ -5,6 +5,141 @@ #include +static const u64 mpt_prot_doubleword[] = { + [MPT_PROT_NONE] = 0x0000000000000000, + [MPT_PROT_R] = 0x5555555555555555, + [MPT_PROT_W] = 0xaaaaaaaaaaaaaaaa, + [MPT_PROT_RW] = 0xffffffffffffffff, +}; + +/* Set protection bits of SMPT in a given range without using memset. */ +static void __set_smpt_range_slow(u32 *smpt, size_t start_gb_byte, + size_t end_gb_byte, enum mpt_prot prot) +{ + size_t i, start_word_byte, end_word_byte, word_idx, first_elem, last_elem; + u32 val; + + /* Iterate over u32 words. */ + start_word_byte = start_gb_byte; + while (start_word_byte < end_gb_byte) { + /* Determine the range of bytes covered by this word. */ + word_idx = start_word_byte / SMPT_WORD_BYTE_RANGE; + end_word_byte = min( + ALIGN(start_word_byte + 1, SMPT_WORD_BYTE_RANGE), + end_gb_byte); + + /* Identify protection bit offsets within the word. */ + first_elem = (start_word_byte / SMPT_GRAN) % SMPT_ELEMS_PER_WORD; + last_elem = ((end_word_byte - 1) / SMPT_GRAN) % SMPT_ELEMS_PER_WORD; + + /* Modify the corresponding word. */ + val = READ_ONCE(smpt[word_idx]); + for (i = first_elem; i <= last_elem; i++) { + val &= ~(MPT_PROT_MASK << (i * MPT_PROT_BITS)); + val |= prot << (i * MPT_PROT_BITS); + } + WRITE_ONCE(smpt[word_idx], val); + + start_word_byte = end_word_byte; + } +} + +/* Set protection bits of SMPT in a given range. */ +static void __set_smpt_range(u32 *smpt, size_t start_gb_byte, + size_t end_gb_byte, enum mpt_prot prot) +{ + size_t interlude_start, interlude_end, interlude_bytes, word_idx; + char prot_byte = (char)mpt_prot_doubleword[prot]; + + if (start_gb_byte >= end_gb_byte) + return; + + /* Check if range spans at least one full u32 word. */ + interlude_start = ALIGN(start_gb_byte, SMPT_WORD_BYTE_RANGE); + interlude_end = ALIGN_DOWN(end_gb_byte, SMPT_WORD_BYTE_RANGE); + + /* If not, fall back to editing bits in the given range. */ + if (interlude_start >= interlude_end) { + __set_smpt_range_slow(smpt, start_gb_byte, end_gb_byte, prot); + return; + } + + /* Use bit-editing for prologue/epilogue, memset for interlude. */ + word_idx = interlude_start / SMPT_WORD_BYTE_RANGE; + interlude_bytes = (interlude_end - interlude_start) / SMPT_GRAN / SMPT_ELEMS_PER_BYTE; + + __set_smpt_range_slow(smpt, start_gb_byte, interlude_start, prot); + memset(&smpt[word_idx], prot_byte, interlude_bytes); + __set_smpt_range_slow(smpt, interlude_end, end_gb_byte, prot); +} + +/* Returns true if all SMPT protection bits match 'prot'. */ +static bool __is_smpt_uniform(u32 *smpt, enum mpt_prot prot) +{ + size_t i; + u64 *doublewords = (u64 *)smpt; + + for (i = 0; i < SMPT_NUM_WORDS / 2; i++) { + if (doublewords[i] != mpt_prot_doubleword[prot]) + return false; + } + return true; +} + +/* + * Set protection bits of FMPT/SMPT in a given range. + * Returns flags specifying whether L1/L2 changes need to be made visible + * to the device. + */ +static void __set_fmpt_range(struct fmpt *fmpt, size_t start_gb_byte, + size_t end_gb_byte, enum mpt_prot prot) +{ + if (start_gb_byte == 0 && end_gb_byte >= SZ_1G) { + /* Update covers the entire GB region. */ + if (fmpt->gran_1g && fmpt->prot == prot) { + fmpt->flags = 0; + return; + } + + fmpt->gran_1g = true; + fmpt->prot = prot; + fmpt->flags = MPT_UPDATE_L1; + return; + } + + if (fmpt->gran_1g) { + /* GB region currently uses 1G mapping. */ + if (fmpt->prot == prot) { + fmpt->flags = 0; + return; + } + + /* + * Range has different mapping than the rest of the GB. + * Convert to PAGE_SIZE mapping. + */ + fmpt->gran_1g = false; + __set_smpt_range(fmpt->smpt, 0, start_gb_byte, fmpt->prot); + __set_smpt_range(fmpt->smpt, start_gb_byte, end_gb_byte, prot); + __set_smpt_range(fmpt->smpt, end_gb_byte, SZ_1G, fmpt->prot); + fmpt->flags = MPT_UPDATE_L1 | MPT_UPDATE_L2; + return; + } + + /* GB region currently uses PAGE_SIZE mapping. */ + __set_smpt_range(fmpt->smpt, start_gb_byte, end_gb_byte, prot); + + /* Check if the entire GB region has the same prot bits. */ + if (!__is_smpt_uniform(fmpt->smpt, prot)) { + fmpt->flags = MPT_UPDATE_L2; + return; + } + + fmpt->gran_1g = true; + fmpt->prot = prot; + fmpt->flags = MPT_UPDATE_L1; +} + static void __set_l1entry_attr_with_prot(void *dev_va, unsigned int gb, unsigned int vid, enum mpt_prot prot) {