shinys000114/linux
1
0
Fork 0
mirror of https://github.com/hardkernel/linux.git synced 2026-06-11 05:17:10 +09:00
Code Issues Packages Projects Releases Wiki Activity

Merge tag 'v4.9.310' of git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable into odroidg12-4.9.y

Browse Source 
This is the 4.9.310 stable release

Conflicts:
	arch/arm64/kernel/cpu_errata.c
	arch/arm64/kernel/cpufeature.c
	arch/arm64/kernel/entry.S
	drivers/clocksource/Kconfig
This commit is contained in:
Mauro (mdrjr) Ribeiro
2022-04-27 17:01:22 -03:00
parent 960dcd6fe1 6348ae0783
commit cff110156c
38 changed files with 1510 additions and 326 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Documentation/arm64/silicon-errata.txt
View File

@@ -55,6 +55,7 @@ stable kernels.
| ARM | Cortex-A57 | #834220 | ARM64_ERRATUM_834220 |
| ARM | Cortex-A72 | #853709 | N/A |
| ARM | Cortex-A55 | #1024718 | ARM64_ERRATUM_1024718 |
| ARM | Cortex-A76 | #1188873 | ARM64_ERRATUM_1188873 |
| ARM | MMU-500 | #841119,#826419 | N/A |
| | | | |
| Cavium | ThunderX ITS | #22375, #24313 | CAVIUM_ERRATUM_22375 |

9
Documentation/kernel-parameters.txt
View File

@@ -752,15 +752,6 @@ bytes respectively. Such letter suffixes can also be entirely omitted.
loops can be debugged more effectively on production
systems.
clocksource.arm_arch_timer.fsl-a008585=
[ARM64]
Format: <bool>
Enable/disable the workaround of Freescale/NXP
erratum A-008585. This can be useful for KVM
guests, if the guest device tree doesn't show the
erratum. If unspecified, the workaround is
enabled based on the device tree.
clearcpuid=BITNUM [X86]
Disable CPUID feature X for the kernel. See
arch/x86/include/asm/cpufeatures.h for the valid bit

2
Makefile
View File

@@ -1,6 +1,6 @@
VERSION = 4
PATCHLEVEL = 9
SUBLEVEL = 309
SUBLEVEL = 310
EXTRAVERSION =
NAME = Roaring Lionus

5
arch/arm/include/asm/kvm_host.h
View File

@@ -349,4 +349,9 @@ static inline int kvm_arm_have_ssbd(void)
return KVM_SSBD_UNKNOWN;
}
static inline bool kvm_arm_spectre_bhb_mitigated(void)
{
/* 32bit guests don't need firmware for this */
return false;
}
#endif /* __ARM_KVM_HOST_H__ */

4
arch/arm/kvm/psci.c
View File

@@ -431,6 +431,10 @@ int kvm_hvc_call_handler(struct kvm_vcpu *vcpu)
break;
}
break;
case ARM_SMCCC_ARCH_WORKAROUND_3:
if (kvm_arm_spectre_bhb_mitigated())
val = SMCCC_RET_SUCCESS;
break;
}
break;
default:

24
arch/arm64/Kconfig
View File

@@ -443,6 +443,20 @@ config ARM64_ERRATUM_1024718
If unsure, say Y.
config ARM64_ERRATUM_1188873
bool "Cortex-A76: MRC read following MRRC read of specific Generic Timer in AArch32 might give incorrect result"
default y
depends on COMPAT
select ARM_ARCH_TIMER_OOL_WORKAROUND
help
This option adds work arounds for ARM Cortex-A76 erratum 1188873
Affected Cortex-A76 cores (r0p0, r1p0, r2p0) could cause
register corruption when accessing the timer registers from
AArch32 userspace.
If unsure, say Y.
config CAVIUM_ERRATUM_22375
bool "Cavium erratum 22375, 24313"
default y
@@ -787,6 +801,16 @@ config ARM64_SSBD
If unsure, say Y.
config MITIGATE_SPECTRE_BRANCH_HISTORY
bool "Mitigate Spectre style attacks against branch history" if EXPERT
default y
depends on HARDEN_BRANCH_PREDICTOR || !KVM
help
Speculation attacks against some high-performance processors can
make use of branch history to influence future speculation.
When taking an exception from user-space, a sequence of branches
or a firmware call overwrites the branch history.
menuconfig ARMV8_DEPRECATED
bool "Emulate deprecated/obsolete ARMv8 instructions"
depends on COMPAT

44
arch/arm64/include/asm/arch_timer.h
View File

@@ -29,41 +29,35 @@
#include <clocksource/arm_arch_timer.h>
#if IS_ENABLED(CONFIG_FSL_ERRATUM_A008585)
#if IS_ENABLED(CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND)
extern struct static_key_false arch_timer_read_ool_enabled;
#define needs_fsl_a008585_workaround() \
#define needs_unstable_timer_counter_workaround() \
static_branch_unlikely(&arch_timer_read_ool_enabled)
#else
#define needs_fsl_a008585_workaround() false
#define needs_unstable_timer_counter_workaround() false
#endif
u32 __fsl_a008585_read_cntp_tval_el0(void);
u32 __fsl_a008585_read_cntv_tval_el0(void);
u64 __fsl_a008585_read_cntvct_el0(void);
enum arch_timer_erratum_match_type {
ate_match_dt,
ate_match_local_cap_id,
};
/*
* The number of retries is an arbitrary value well beyond the highest number
* of iterations the loop has been observed to take.
*/
#define __fsl_a008585_read_reg(reg) ({ \
u64 _old, _new; \
int _retries = 200; \
\
do { \
_old = read_sysreg(reg); \
_new = read_sysreg(reg); \
_retries--; \
} while (unlikely(_old != _new) && _retries); \
\
WARN_ON_ONCE(!_retries); \
_new; \
})
struct arch_timer_erratum_workaround {
enum arch_timer_erratum_match_type match_type;
const void *id;
const char *desc;
u32 (*read_cntp_tval_el0)(void);
u32 (*read_cntv_tval_el0)(void);
u64 (*read_cntvct_el0)(void);
};
extern const struct arch_timer_erratum_workaround *timer_unstable_counter_workaround;
#define arch_timer_reg_read_stable(reg) \
({ \
u64 _val; \
if (needs_fsl_a008585_workaround()) \
_val = __fsl_a008585_read_##reg(); \
if (needs_unstable_timer_counter_workaround()) \
_val = timer_unstable_counter_workaround->read_##reg();\
else \
_val = read_sysreg(reg); \
_val; \

34
arch/arm64/include/asm/assembler.h
View File

@@ -103,6 +103,13 @@
hint #20
.endm
/*
* Clear Branch History instruction
*/
.macro clearbhb
hint #22
.endm
/*
* Sanitise a 64-bit bounded index wrt speculation, returning zero if out
* of bounds.
@@ -510,4 +517,31 @@ alternative_endif
.Ldone\@:
.endm
.macro __mitigate_spectre_bhb_loop tmp
#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
alternative_cb spectre_bhb_patch_loop_iter
mov \tmp, #32 // Patched to correct the immediate
alternative_cb_end
.Lspectre_bhb_loop\@:
b . + 4
subs \tmp, \tmp, #1
b.ne .Lspectre_bhb_loop\@
dsb nsh
isb
#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
.endm
/* Save/restores x0-x3 to the stack */
.macro __mitigate_spectre_bhb_fw
#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
stp x0, x1, [sp, #-16]!
stp x2, x3, [sp, #-16]!
mov w0, #ARM_SMCCC_ARCH_WORKAROUND_3
alternative_cb arm64_update_smccc_conduit
nop // Patched to SMC/HVC #0
alternative_cb_end
ldp x2, x3, [sp], #16
ldp x0, x1, [sp], #16
#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
.endm
#endif /* __ASM_ASSEMBLER_H */

1
arch/arm64/include/asm/cpu.h
View File

@@ -36,6 +36,7 @@ struct cpuinfo_arm64 {
u64 reg_id_aa64dfr1;
u64 reg_id_aa64isar0;
u64 reg_id_aa64isar1;
u64 reg_id_aa64isar2;
u64 reg_id_aa64mmfr0;
u64 reg_id_aa64mmfr1;
u64 reg_id_aa64mmfr2;

4
arch/arm64/include/asm/cpucaps.h
View File

@@ -38,7 +38,9 @@
#define ARM64_HARDEN_BRANCH_PREDICTOR 17
#define ARM64_SSBD 18
#define ARM64_MISMATCHED_CACHE_TYPE 19
#define ARM64_WORKAROUND_1188873 20
#define ARM64_SPECTRE_BHB 21
#define ARM64_NCAPS 20
#define ARM64_NCAPS 22
#endif /* __ASM_CPUCAPS_H */

232
arch/arm64/include/asm/cpufeature.h
View File

@@ -10,6 +10,7 @@
#define __ASM_CPUFEATURE_H
#include <asm/cpucaps.h>
#include <asm/cputype.h>
#include <asm/hwcap.h>
#include <asm/sysreg.h>
@@ -66,24 +67,173 @@ struct arm64_ftr_reg {
extern struct arm64_ftr_reg arm64_ftr_reg_ctrel0;
/* scope of capability check */
enum {
SCOPE_SYSTEM,
SCOPE_LOCAL_CPU,
};
/*
* CPU capabilities:
*
* We use arm64_cpu_capabilities to represent system features, errata work
* arounds (both used internally by kernel and tracked in cpu_hwcaps) and
* ELF HWCAPs (which are exposed to user).
*
* To support systems with heterogeneous CPUs, we need to make sure that we
* detect the capabilities correctly on the system and take appropriate
* measures to ensure there are no incompatibilities.
*
* This comment tries to explain how we treat the capabilities.
* Each capability has the following list of attributes :
*
* 1) Scope of Detection : The system detects a given capability by
* performing some checks at runtime. This could be, e.g, checking the
* value of a field in CPU ID feature register or checking the cpu
* model. The capability provides a call back ( @matches() ) to
* perform the check. Scope defines how the checks should be performed.
* There are two cases:
*
* a) SCOPE_LOCAL_CPU: check all the CPUs and "detect" if at least one
* matches. This implies, we have to run the check on all the
* booting CPUs, until the system decides that state of the
* capability is finalised. (See section 2 below)
* Or
* b) SCOPE_SYSTEM: check all the CPUs and "detect" if all the CPUs
* matches. This implies, we run the check only once, when the
* system decides to finalise the state of the capability. If the
* capability relies on a field in one of the CPU ID feature
* registers, we use the sanitised value of the register from the
* CPU feature infrastructure to make the decision.
*
* The process of detection is usually denoted by "update" capability
* state in the code.
*
* 2) Finalise the state : The kernel should finalise the state of a
* capability at some point during its execution and take necessary
* actions if any. Usually, this is done, after all the boot-time
* enabled CPUs are brought up by the kernel, so that it can make
* better decision based on the available set of CPUs. However, there
* are some special cases, where the action is taken during the early
* boot by the primary boot CPU. (e.g, running the kernel at EL2 with
* Virtualisation Host Extensions). The kernel usually disallows any
* changes to the state of a capability once it finalises the capability
* and takes any action, as it may be impossible to execute the actions
* safely. A CPU brought up after a capability is "finalised" is
* referred to as "Late CPU" w.r.t the capability. e.g, all secondary
* CPUs are treated "late CPUs" for capabilities determined by the boot
* CPU.
*
* 3) Verification: When a CPU is brought online (e.g, by user or by the
* kernel), the kernel should make sure that it is safe to use the CPU,
* by verifying that the CPU is compliant with the state of the
* capabilities finalised already. This happens via :
*
* secondary_start_kernel()-> check_local_cpu_capabilities()
*
* As explained in (2) above, capabilities could be finalised at
* different points in the execution. Each CPU is verified against the
* "finalised" capabilities and if there is a conflict, the kernel takes
* an action, based on the severity (e.g, a CPU could be prevented from
* booting or cause a kernel panic). The CPU is allowed to "affect" the
* state of the capability, if it has not been finalised already.
* See section 5 for more details on conflicts.
*
* 4) Action: As mentioned in (2), the kernel can take an action for each
* detected capability, on all CPUs on the system. Appropriate actions
* include, turning on an architectural feature, modifying the control
* registers (e.g, SCTLR, TCR etc.) or patching the kernel via
* alternatives. The kernel patching is batched and performed at later
* point. The actions are always initiated only after the capability
* is finalised. This is usally denoted by "enabling" the capability.
* The actions are initiated as follows :
* a) Action is triggered on all online CPUs, after the capability is
* finalised, invoked within the stop_machine() context from
* enable_cpu_capabilitie().
*
* b) Any late CPU, brought up after (1), the action is triggered via:
*
* check_local_cpu_capabilities() -> verify_local_cpu_capabilities()
*
* 5) Conflicts: Based on the state of the capability on a late CPU vs.
* the system state, we could have the following combinations :
*
* x-----------------------------x
* | Type | System | Late CPU |
* |-----------------------------|
* | a | y | n |
* |-----------------------------|
* | b | n | y |
* x-----------------------------x
*
* Two separate flag bits are defined to indicate whether each kind of
* conflict can be allowed:
* ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU - Case(a) is allowed
* ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU - Case(b) is allowed
*
* Case (a) is not permitted for a capability that the system requires
* all CPUs to have in order for the capability to be enabled. This is
* typical for capabilities that represent enhanced functionality.
*
* Case (b) is not permitted for a capability that must be enabled
* during boot if any CPU in the system requires it in order to run
* safely. This is typical for erratum work arounds that cannot be
* enabled after the corresponding capability is finalised.
*
* In some non-typical cases either both (a) and (b), or neither,
* should be permitted. This can be described by including neither
* or both flags in the capability's type field.
*/
/* Decide how the capability is detected. On a local CPU vs System wide */
#define ARM64_CPUCAP_SCOPE_LOCAL_CPU ((u16)BIT(0))
#define ARM64_CPUCAP_SCOPE_SYSTEM ((u16)BIT(1))
#define ARM64_CPUCAP_SCOPE_MASK \
(ARM64_CPUCAP_SCOPE_SYSTEM | \
ARM64_CPUCAP_SCOPE_LOCAL_CPU)
#define SCOPE_SYSTEM ARM64_CPUCAP_SCOPE_SYSTEM
#define SCOPE_LOCAL_CPU ARM64_CPUCAP_SCOPE_LOCAL_CPU
/*
* Is it permitted for a late CPU to have this capability when system
* hasn't already enabled it ?
*/
#define ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU ((u16)BIT(4))
/* Is it safe for a late CPU to miss this capability when system has it */
#define ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU ((u16)BIT(5))
/*
* CPU errata workarounds that need to be enabled at boot time if one or
* more CPUs in the system requires it. When one of these capabilities
* has been enabled, it is safe to allow any CPU to boot that doesn't
* require the workaround. However, it is not safe if a "late" CPU
* requires a workaround and the system hasn't enabled it already.
*/
#define ARM64_CPUCAP_LOCAL_CPU_ERRATUM \
(ARM64_CPUCAP_SCOPE_LOCAL_CPU | ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU)
/*
* CPU feature detected at boot time based on system-wide value of a
* feature. It is safe for a late CPU to have this feature even though
* the system hasn't enabled it, although the featuer will not be used
* by Linux in this case. If the system has enabled this feature already,
* then every late CPU must have it.
*/
#define ARM64_CPUCAP_SYSTEM_FEATURE \
(ARM64_CPUCAP_SCOPE_SYSTEM | ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU)
struct arm64_cpu_capabilities {
const char *desc;
u16 capability;
int def_scope; /* default scope */
u16 type;
bool (*matches)(const struct arm64_cpu_capabilities *caps, int scope);
int (*enable)(void *); /* Called on all active CPUs */
/*
* Take the appropriate actions to enable this capability for this CPU.
* For each successfully booted CPU, this method is called for each
* globally detected capability.
*/
void (*cpu_enable)(const struct arm64_cpu_capabilities *cap);
union {
struct { /* To be used for erratum handling only */
u32 midr_model;
u32 midr_range_min, midr_range_max;
struct midr_range midr_range;
};
const struct midr_range *midr_range_list;
struct { /* Feature register checking */
u32 sys_reg;
u8 field_pos;
@@ -95,6 +245,23 @@ struct arm64_cpu_capabilities {
};
};
static inline int cpucap_default_scope(const struct arm64_cpu_capabilities *cap)
{
return cap->type & ARM64_CPUCAP_SCOPE_MASK;
}
static inline bool
cpucap_late_cpu_optional(const struct arm64_cpu_capabilities *cap)
{
return !!(cap->type & ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU);
}
static inline bool
cpucap_late_cpu_permitted(const struct arm64_cpu_capabilities *cap)
{
return !!(cap->type & ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU);
}
extern DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
extern struct static_key_false cpu_hwcap_keys[ARM64_NCAPS];
extern struct static_key_false arm64_const_caps_ready;
@@ -195,15 +362,8 @@ static inline bool id_aa64pfr0_32bit_el0(u64 pfr0)
}
void __init setup_cpu_features(void);
void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps,
const char *info);
void enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps);
void check_local_cpu_capabilities(void);
void update_cpu_errata_workarounds(void);
void __init enable_errata_workarounds(void);
void verify_local_cpu_errata_workarounds(void);
u64 read_system_reg(u32 id);
@@ -212,6 +372,34 @@ static inline bool cpu_supports_mixed_endian_el0(void)
return id_aa64mmfr0_mixed_endian_el0(read_cpuid(ID_AA64MMFR0_EL1));
}
static inline bool supports_csv2p3(int scope)
{
u64 pfr0;
u8 csv2_val;
if (scope == SCOPE_LOCAL_CPU)
pfr0 = read_sysreg_s(SYS_ID_AA64PFR0_EL1);
else
pfr0 = read_system_reg(SYS_ID_AA64PFR0_EL1);
csv2_val = cpuid_feature_extract_unsigned_field(pfr0,
ID_AA64PFR0_CSV2_SHIFT);
return csv2_val == 3;
}
static inline bool supports_clearbhb(int scope)
{
u64 isar2;
if (scope == SCOPE_LOCAL_CPU)
isar2 = read_sysreg_s(SYS_ID_AA64ISAR2_EL1);
else
isar2 = read_system_reg(SYS_ID_AA64ISAR2_EL1);
return cpuid_feature_extract_unsigned_field(isar2,
ID_AA64ISAR2_CLEARBHB_SHIFT);
}
static inline bool system_supports_32bit_el0(void)
{
return cpus_have_const_cap(ARM64_HAS_32BIT_EL0);
@@ -250,6 +438,18 @@ void arm64_set_ssbd_mitigation(bool state);
static inline void arm64_set_ssbd_mitigation(bool state) {}
#endif
/* Watch out, ordering is important here. */
enum mitigation_state {
SPECTRE_UNAFFECTED,
SPECTRE_MITIGATED,
SPECTRE_VULNERABLE,
};
enum mitigation_state arm64_get_spectre_bhb_state(void);
bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, int scope);
u8 spectre_bhb_loop_affected(int scope);
void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry);
#endif /* __ASSEMBLY__ */
#endif

63
arch/arm64/include/asm/cputype.h
View File

@@ -83,6 +83,18 @@
#define ARM_CPU_PART_CORTEX_A53 0xD03
#define ARM_CPU_PART_CORTEX_A73 0xD09
#define ARM_CPU_PART_CORTEX_A75 0xD0A
#define ARM_CPU_PART_CORTEX_A35 0xD04
#define ARM_CPU_PART_CORTEX_A55 0xD05
#define ARM_CPU_PART_CORTEX_A76 0xD0B
#define ARM_CPU_PART_NEOVERSE_N1 0xD0C
#define ARM_CPU_PART_CORTEX_A77 0xD0D
#define ARM_CPU_PART_NEOVERSE_V1 0xD40
#define ARM_CPU_PART_CORTEX_A78 0xD41
#define ARM_CPU_PART_CORTEX_X1 0xD44
#define ARM_CPU_PART_CORTEX_A710 0xD47
#define ARM_CPU_PART_CORTEX_X2 0xD48
#define ARM_CPU_PART_NEOVERSE_N2 0xD49
#define ARM_CPU_PART_CORTEX_A78C 0xD4B
#define APM_CPU_PART_POTENZA 0x000
@@ -98,6 +110,18 @@
#define MIDR_CORTEX_A72 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A72)
#define MIDR_CORTEX_A73 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A73)
#define MIDR_CORTEX_A75 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A75)
#define MIDR_CORTEX_A35 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A35)
#define MIDR_CORTEX_A55 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A55)
#define MIDR_CORTEX_A76 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A76)
#define MIDR_NEOVERSE_N1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N1)
#define MIDR_CORTEX_A77 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A77)
#define MIDR_NEOVERSE_V1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_V1)
#define MIDR_CORTEX_A78 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A78)
#define MIDR_CORTEX_X1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X1)
#define MIDR_CORTEX_A710 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A710)
#define MIDR_CORTEX_X2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X2)
#define MIDR_NEOVERSE_N2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N2)
#define MIDR_CORTEX_A78C MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A78C)
#define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX)
#define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX)
#define MIDR_CAVIUM_THUNDERX2 MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX2)
@@ -109,6 +133,45 @@
#define read_cpuid(reg) read_sysreg_s(SYS_ ## reg)
/*
* Represent a range of MIDR values for a given CPU model and a
* range of variant/revision values.
*
* @model - CPU model as defined by MIDR_CPU_MODEL
* @rv_min - Minimum value for the revision/variant as defined by
* MIDR_CPU_VAR_REV
* @rv_max - Maximum value for the variant/revision for the range.
*/
struct midr_range {
u32 model;
u32 rv_min;
u32 rv_max;
};
#define MIDR_RANGE(m, v_min, r_min, v_max, r_max) \
{ \
.model = m, \
.rv_min = MIDR_CPU_VAR_REV(v_min, r_min), \
.rv_max = MIDR_CPU_VAR_REV(v_max, r_max), \
}
#define MIDR_ALL_VERSIONS(m) MIDR_RANGE(m, 0, 0, 0xf, 0xf)
static inline bool is_midr_in_range(u32 midr, struct midr_range const *range)
{
return MIDR_IS_CPU_MODEL_RANGE(midr, range->model,
range->rv_min, range->rv_max);
}
static inline bool
is_midr_in_range_list(u32 midr, struct midr_range const *ranges)
{
while (ranges->model)
if (is_midr_in_range(midr, ranges++))
return true;
return false;
}
/*
* The CPU ID never changes at run time, so we might as well tell the
* compiler that it's constant. Use this function to read the CPU ID

6
arch/arm64/include/asm/fixmap.h
View File

@@ -53,9 +53,11 @@ enum fixed_addresses {
FIX_TEXT_POKE0,
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
FIX_ENTRY_TRAMP_TEXT3,
FIX_ENTRY_TRAMP_TEXT2,
FIX_ENTRY_TRAMP_TEXT1,
FIX_ENTRY_TRAMP_DATA,
FIX_ENTRY_TRAMP_TEXT,
#define TRAMP_VALIAS (__fix_to_virt(FIX_ENTRY_TRAMP_TEXT))
#define TRAMP_VALIAS (__fix_to_virt(FIX_ENTRY_TRAMP_TEXT1))
#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
__end_of_permanent_fixed_addresses,

2
arch/arm64/include/asm/insn.h
View File

@@ -332,6 +332,8 @@ bool aarch64_insn_is_branch(u32 insn);
u64 aarch64_insn_decode_immediate(enum aarch64_insn_imm_type type, u32 insn);
u32 aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
u32 insn, u64 imm);
u32 aarch64_insn_decode_register(enum aarch64_insn_register_type type,
u32 insn);
u32 aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr,
enum aarch64_insn_branch_type type);
u32 aarch64_insn_gen_comp_branch_imm(unsigned long pc, unsigned long addr,

4
arch/arm64/include/asm/kvm_host.h
View File

@@ -452,4 +452,8 @@ static inline int kvm_arm_have_ssbd(void)
}
}
static inline bool kvm_arm_spectre_bhb_mitigated(void)
{
return arm64_get_spectre_bhb_state() == SPECTRE_MITIGATED;
}
#endif /* __ARM64_KVM_HOST_H__ */

2
arch/arm64/include/asm/kvm_mmu.h
View File

@@ -362,7 +362,7 @@ static inline void *kvm_get_hyp_vector(void)
struct bp_hardening_data *data = arm64_get_bp_hardening_data();
void *vect = kvm_ksym_ref(__kvm_hyp_vector);
if (data->fn) {
if (data->template_start) {
vect = __bp_harden_hyp_vecs_start +
data->hyp_vectors_slot * SZ_2K;

8
arch/arm64/include/asm/mmu.h
View File

@@ -36,7 +36,7 @@ typedef struct {
*/
#define ASID(mm) ((mm)->context.id.counter & 0xffff)
static inline bool arm64_kernel_unmapped_at_el0(void)
static __always_inline bool arm64_kernel_unmapped_at_el0(void)
{
return IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0) &&
cpus_have_const_cap(ARM64_UNMAP_KERNEL_AT_EL0);
@@ -47,6 +47,12 @@ typedef void (*bp_hardening_cb_t)(void);
struct bp_hardening_data {
int hyp_vectors_slot;
bp_hardening_cb_t fn;
/*
* template_start is only used by the BHB mitigation to identify the
* hyp_vectors_slot sequence.
*/
const char *template_start;
};
#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR

6
arch/arm64/include/asm/processor.h
View File

@@ -37,6 +37,7 @@
#include <linux/string.h>
#include <asm/alternative.h>
#include <asm/cpufeature.h>
#include <asm/fpsimd.h>
#include <asm/hw_breakpoint.h>
#include <asm/lse.h>
@@ -219,9 +220,8 @@ static inline void spin_lock_prefetch(const void *ptr)
#endif
int cpu_enable_pan(void *__unused);
int cpu_enable_uao(void *__unused);
int cpu_enable_cache_maint_trap(void *__unused);
void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused);
void cpu_enable_cache_maint_trap(const struct arm64_cpu_capabilities *__unused);
#endif /* __ASSEMBLY__ */
#endif /* __ASM_PROCESSOR_H */

6
arch/arm64/include/asm/sections.h
View File

@@ -26,5 +26,11 @@ extern char __hyp_text_start[], __hyp_text_end[];
extern char __idmap_text_start[], __idmap_text_end[];
extern char __irqentry_text_start[], __irqentry_text_end[];
extern char __mmuoff_data_start[], __mmuoff_data_end[];
extern char __entry_tramp_text_start[], __entry_tramp_text_end[];
static inline size_t entry_tramp_text_size(void)
{
return __entry_tramp_text_end - __entry_tramp_text_start;
}
#endif /* __ASM_SECTIONS_H */

5
arch/arm64/include/asm/sysreg.h
View File

@@ -70,6 +70,7 @@
#define SYS_ID_AA64ISAR0_EL1 sys_reg(3, 0, 0, 6, 0)
#define SYS_ID_AA64ISAR1_EL1 sys_reg(3, 0, 0, 6, 1)
#define SYS_ID_AA64ISAR2_EL1 sys_reg(3, 0, 0, 6, 2)
#define SYS_ID_AA64MMFR0_EL1 sys_reg(3, 0, 0, 7, 0)
#define SYS_ID_AA64MMFR1_EL1 sys_reg(3, 0, 0, 7, 1)
@@ -173,6 +174,9 @@
#define ID_AA64ISAR0_SHA1_SHIFT 8
#define ID_AA64ISAR0_AES_SHIFT 4
/* id_aa64isar2 */
#define ID_AA64ISAR2_CLEARBHB_SHIFT 28
/* id_aa64pfr0 */
#define ID_AA64PFR0_CSV3_SHIFT 60
#define ID_AA64PFR0_CSV2_SHIFT 56
@@ -211,6 +215,7 @@
#define ID_AA64MMFR0_TGRAN16_SUPPORTED 0x1
/* id_aa64mmfr1 */
#define ID_AA64MMFR1_ECBHB_SHIFT 60
#define ID_AA64MMFR1_PAN_SHIFT 20
#define ID_AA64MMFR1_LOR_SHIFT 16
#define ID_AA64MMFR1_HPD_SHIFT 12

74
arch/arm64/include/asm/vectors.h Normal file
View File

@@ -0,0 +1,74 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2022 ARM Ltd.
*/
#ifndef __ASM_VECTORS_H
#define __ASM_VECTORS_H
#include <linux/bug.h>
#include <linux/percpu.h>
#include <asm/fixmap.h>
#include <asm/mmu.h>
extern char vectors[];
extern char tramp_vectors[];
extern char __bp_harden_el1_vectors[];
/*
* Note: the order of this enum corresponds to two arrays in entry.S:
* tramp_vecs and __bp_harden_el1_vectors. By default the canonical
* 'full fat' vectors are used directly.
*/
enum arm64_bp_harden_el1_vectors {
#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
/*
* Perform the BHB loop mitigation, before branching to the canonical
* vectors.
*/
EL1_VECTOR_BHB_LOOP,
/*
* Make the SMC call for firmware mitigation, before branching to the
* canonical vectors.
*/
EL1_VECTOR_BHB_FW,
/*
* Use the ClearBHB instruction, before branching to the canonical
* vectors.
*/
EL1_VECTOR_BHB_CLEAR_INSN,
#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
/*
* Remap the kernel before branching to the canonical vectors.
*/
EL1_VECTOR_KPTI,
};
#ifndef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
#define EL1_VECTOR_BHB_LOOP -1
#define EL1_VECTOR_BHB_FW -1
#define EL1_VECTOR_BHB_CLEAR_INSN -1
#endif /* !CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
/* The vectors to use on return from EL0. e.g. to remap the kernel */
DECLARE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector);
#ifndef CONFIG_UNMAP_KERNEL_AT_EL0
#define TRAMP_VALIAS 0
#endif
static inline const char *
arm64_get_bp_hardening_vector(enum arm64_bp_harden_el1_vectors slot)
{
if (arm64_kernel_unmapped_at_el0())
return (char *)TRAMP_VALIAS + SZ_2K * slot;
WARN_ON_ONCE(slot == EL1_VECTOR_KPTI);
return __bp_harden_el1_vectors + SZ_2K * slot;
}
#endif /* __ASM_VECTORS_H */

55
arch/arm64/kernel/bpi.S
View File

@@ -73,3 +73,58 @@ ENTRY(__smccc_workaround_1_smc_end)
ENTRY(__smccc_workaround_1_hvc_start)
smccc_workaround_1 hvc
ENTRY(__smccc_workaround_1_hvc_end)
ENTRY(__smccc_workaround_3_smc_start)
sub sp, sp, #(8 * 4)
stp x2, x3, [sp, #(8 * 0)]
stp x0, x1, [sp, #(8 * 2)]
mov w0, #ARM_SMCCC_ARCH_WORKAROUND_3
smc #0
ldp x2, x3, [sp, #(8 * 0)]
ldp x0, x1, [sp, #(8 * 2)]
add sp, sp, #(8 * 4)
ENTRY(__smccc_workaround_3_smc_end)
ENTRY(__spectre_bhb_loop_k8_start)
sub sp, sp, #(8 * 2)
stp x0, x1, [sp, #(8 * 0)]
mov x0, #8
2: b . + 4
subs x0, x0, #1
b.ne 2b
dsb nsh
isb
ldp x0, x1, [sp, #(8 * 0)]
add sp, sp, #(8 * 2)
ENTRY(__spectre_bhb_loop_k8_end)
ENTRY(__spectre_bhb_loop_k24_start)
sub sp, sp, #(8 * 2)
stp x0, x1, [sp, #(8 * 0)]
mov x0, #24
2: b . + 4
subs x0, x0, #1
b.ne 2b
dsb nsh
isb
ldp x0, x1, [sp, #(8 * 0)]
add sp, sp, #(8 * 2)
ENTRY(__spectre_bhb_loop_k24_end)
ENTRY(__spectre_bhb_loop_k32_start)
sub sp, sp, #(8 * 2)
stp x0, x1, [sp, #(8 * 0)]
mov x0, #32
2: b . + 4
subs x0, x0, #1
b.ne 2b
dsb nsh
isb
ldp x0, x1, [sp, #(8 * 0)]
add sp, sp, #(8 * 2)
ENTRY(__spectre_bhb_loop_k32_end)
ENTRY(__spectre_bhb_clearbhb_start)
hint #22 /* aka clearbhb */
isb
ENTRY(__spectre_bhb_clearbhb_end)

595
arch/arm64/kernel/cpu_errata.c
View File

@@ -25,14 +25,23 @@
#include <asm/cpufeature.h>
#include <linux/psci.h>
#include <linux/arm-smccc.h>
#include <asm/vectors.h>
static bool __maybe_unused
is_affected_midr_range(const struct arm64_cpu_capabilities *entry, int scope)
{
u32 midr = read_cpuid_id();
WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
return MIDR_IS_CPU_MODEL_RANGE(read_cpuid_id(), entry->midr_model,
entry->midr_range_min,
entry->midr_range_max);
return is_midr_in_range(midr, &entry->midr_range);
}
static bool __maybe_unused
is_affected_midr_range_list(const struct arm64_cpu_capabilities *entry,
int scope)
{
WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
return is_midr_in_range_list(read_cpuid_id(), entry->midr_range_list);
}
static bool
@@ -50,17 +59,18 @@ has_mismatched_cache_type(const struct arm64_cpu_capabilities *entry,
(arm64_ftr_reg_ctrel0.sys_val & mask);
}
static int cpu_enable_trap_ctr_access(void *__unused)
static void
cpu_enable_trap_ctr_access(const struct arm64_cpu_capabilities *__unused)
{
/* Clear SCTLR_EL1.UCT */
config_sctlr_el1(SCTLR_EL1_UCT, 0);
return 0;
}
#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
#include <asm/mmu_context.h>
#include <asm/cacheflush.h>
static bool __hardenbp_enab;
DEFINE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data);
#ifdef CONFIG_KVM
@@ -68,6 +78,16 @@ extern char __smccc_workaround_1_smc_start[];
extern char __smccc_workaround_1_smc_end[];
extern char __smccc_workaround_1_hvc_start[];
extern char __smccc_workaround_1_hvc_end[];
extern char __smccc_workaround_3_smc_start[];
extern char __smccc_workaround_3_smc_end[];
extern char __spectre_bhb_loop_k8_start[];
extern char __spectre_bhb_loop_k8_end[];
extern char __spectre_bhb_loop_k24_start[];
extern char __spectre_bhb_loop_k24_end[];
extern char __spectre_bhb_loop_k32_start[];
extern char __spectre_bhb_loop_k32_end[];
extern char __spectre_bhb_clearbhb_start[];
extern char __spectre_bhb_clearbhb_end[];
static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start,
const char *hyp_vecs_end)
@@ -81,12 +101,14 @@ static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start,
flush_icache_range((uintptr_t)dst, (uintptr_t)dst + SZ_2K);
}
static DEFINE_SPINLOCK(bp_lock);
static int last_slot = -1;
static void __install_bp_hardening_cb(bp_hardening_cb_t fn,
const char *hyp_vecs_start,
const char *hyp_vecs_end)
{
static int last_slot = -1;
static DEFINE_SPINLOCK(bp_lock);
int cpu, slot = -1;
spin_lock(&bp_lock);
@@ -107,6 +129,8 @@ static void __install_bp_hardening_cb(bp_hardening_cb_t fn,
__this_cpu_write(bp_hardening_data.hyp_vectors_slot, slot);
__this_cpu_write(bp_hardening_data.fn, fn);
__this_cpu_write(bp_hardening_data.template_start, hyp_vecs_start);
__hardenbp_enab = true;
spin_unlock(&bp_lock);
}
#else
@@ -120,6 +144,7 @@ static void __install_bp_hardening_cb(bp_hardening_cb_t fn,
const char *hyp_vecs_end)
{
__this_cpu_write(bp_hardening_data.fn, fn);
__hardenbp_enab = true;
}
#endif /* CONFIG_KVM */
@@ -154,25 +179,25 @@ static void call_hvc_arch_workaround_1(void)
arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
}
static int enable_smccc_arch_workaround_1(void *data)
static void
enable_smccc_arch_workaround_1(const struct arm64_cpu_capabilities *entry)
{
const struct arm64_cpu_capabilities *entry = data;
bp_hardening_cb_t cb;
void *smccc_start, *smccc_end;
struct arm_smccc_res res;
if (!entry->matches(entry, SCOPE_LOCAL_CPU))
return 0;
return;
if (psci_ops.smccc_version == SMCCC_VERSION_1_0)
return 0;
return;
switch (psci_ops.conduit) {
case PSCI_CONDUIT_HVC:
arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
ARM_SMCCC_ARCH_WORKAROUND_1, &res);
if ((int)res.a0 < 0)
return 0;
return;
cb = call_hvc_arch_workaround_1;
smccc_start = __smccc_workaround_1_hvc_start;
smccc_end = __smccc_workaround_1_hvc_end;
@@ -182,22 +207,44 @@ static int enable_smccc_arch_workaround_1(void *data)
arm_smccc_1_1_smc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
ARM_SMCCC_ARCH_WORKAROUND_1, &res);
if ((int)res.a0 < 0)
return 0;
return;
cb = call_smc_arch_workaround_1;
smccc_start = __smccc_workaround_1_smc_start;
smccc_end = __smccc_workaround_1_smc_end;
break;
default:
return 0;
return;
}
install_bp_hardening_cb(entry, cb, smccc_start, smccc_end);
return 0;
return;
}
#endif /* CONFIG_HARDEN_BRANCH_PREDICTOR */
void __init arm64_update_smccc_conduit(struct alt_instr *alt,
__le32 *origptr, __le32 *updptr,
int nr_inst)
{
u32 insn;
BUG_ON(nr_inst != 1);
switch (psci_ops.conduit) {
case PSCI_CONDUIT_HVC:
insn = aarch64_insn_get_hvc_value();
break;
case PSCI_CONDUIT_SMC:
insn = aarch64_insn_get_smc_value();
break;
default:
return;
}
*updptr = cpu_to_le32(insn);
}
#ifdef CONFIG_ARM64_SSBD
DEFINE_PER_CPU_READ_MOSTLY(u64, arm64_ssbd_callback_required);
@@ -233,28 +280,6 @@ static int __init ssbd_cfg(char *buf)
}
early_param("ssbd", ssbd_cfg);
void __init arm64_update_smccc_conduit(struct alt_instr *alt,
__le32 *origptr, __le32 *updptr,
int nr_inst)
{
u32 insn;
BUG_ON(nr_inst != 1);
switch (psci_ops.conduit) {
case PSCI_CONDUIT_HVC:
insn = aarch64_insn_get_hvc_value();
break;
case PSCI_CONDUIT_SMC:
insn = aarch64_insn_get_smc_value();
break;
default:
return;
}
*updptr = cpu_to_le32(insn);
}
void __init arm64_enable_wa2_handling(struct alt_instr *alt,
__le32 *origptr, __le32 *updptr,
int nr_inst)
@@ -370,19 +395,60 @@ static bool has_ssbd_mitigation(const struct arm64_cpu_capabilities *entry,
}
#endif /* CONFIG_ARM64_SSBD */
#define MIDR_RANGE(model, min, max) \
.def_scope = SCOPE_LOCAL_CPU, \
.matches = is_affected_midr_range, \
.midr_model = model, \
.midr_range_min = min, \
.midr_range_max = max
#define CAP_MIDR_RANGE(model, v_min, r_min, v_max, r_max) \
.matches = is_affected_midr_range, \
.midr_range = MIDR_RANGE(model, v_min, r_min, v_max, r_max)
#define MIDR_ALL_VERSIONS(model) \
.def_scope = SCOPE_LOCAL_CPU, \
.matches = is_affected_midr_range, \
.midr_model = model, \
.midr_range_min = 0, \
.midr_range_max = (MIDR_VARIANT_MASK | MIDR_REVISION_MASK)
#define CAP_MIDR_ALL_VERSIONS(model) \
.matches = is_affected_midr_range, \
.midr_range = MIDR_ALL_VERSIONS(model)
#define MIDR_FIXED(rev, revidr_mask) \
.fixed_revs = (struct arm64_midr_revidr[]){{ (rev), (revidr_mask) }, {}}
#define ERRATA_MIDR_RANGE(model, v_min, r_min, v_max, r_max) \
.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, \
CAP_MIDR_RANGE(model, v_min, r_min, v_max, r_max)
#define CAP_MIDR_RANGE_LIST(list) \
.matches = is_affected_midr_range_list, \
.midr_range_list = list
/* Errata affecting a range of revisions of given model variant */
#define ERRATA_MIDR_REV_RANGE(m, var, r_min, r_max) \
ERRATA_MIDR_RANGE(m, var, r_min, var, r_max)
/* Errata affecting a single variant/revision of a model */
#define ERRATA_MIDR_REV(model, var, rev) \
ERRATA_MIDR_RANGE(model, var, rev, var, rev)
/* Errata affecting all variants/revisions of a given a model */
#define ERRATA_MIDR_ALL_VERSIONS(model) \
.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, \
CAP_MIDR_ALL_VERSIONS(model)
/* Errata affecting a list of midr ranges, with same work around */
#define ERRATA_MIDR_RANGE_LIST(midr_list) \
.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, \
CAP_MIDR_RANGE_LIST(midr_list)
#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
/*
* List of CPUs where we need to issue a psci call to
* harden the branch predictor.
*/
static const struct midr_range arm64_bp_harden_smccc_cpus[] = {
MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
MIDR_ALL_VERSIONS(MIDR_CORTEX_A75),
MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN),
MIDR_ALL_VERSIONS(MIDR_CAVIUM_THUNDERX2),
{},
};
#endif
const struct arm64_cpu_capabilities arm64_errata[] = {
#if defined(CONFIG_ARM64_ERRATUM_826319) || \
@@ -392,8 +458,8 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
/* Cortex-A53 r0p[012] */
.desc = "ARM errata 826319, 827319, 824069",
.capability = ARM64_WORKAROUND_CLEAN_CACHE,
MIDR_RANGE(MIDR_CORTEX_A53, 0x00, 0x02),
.enable = cpu_enable_cache_maint_trap,
ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 2),
.cpu_enable = cpu_enable_cache_maint_trap,
},
#endif
#ifdef CONFIG_ARM64_ERRATUM_819472
@@ -401,8 +467,8 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
/* Cortex-A53 r0p[01] */
.desc = "ARM errata 819472",
.capability = ARM64_WORKAROUND_CLEAN_CACHE,
MIDR_RANGE(MIDR_CORTEX_A53, 0x00, 0x01),
.enable = cpu_enable_cache_maint_trap,
ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 1),
.cpu_enable = cpu_enable_cache_maint_trap,
},
#endif
#ifdef CONFIG_ARM64_ERRATUM_832075
@@ -410,8 +476,9 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
/* Cortex-A57 r0p0 - r1p2 */
.desc = "ARM erratum 832075",
.capability = ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE,
MIDR_RANGE(MIDR_CORTEX_A57, 0x00,
(1 << MIDR_VARIANT_SHIFT) | 2),
ERRATA_MIDR_RANGE(MIDR_CORTEX_A57,
0, 0,
1, 2),
},
#endif
#ifdef CONFIG_ARM64_ERRATUM_834220
@@ -419,8 +486,9 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
/* Cortex-A57 r0p0 - r1p2 */
.desc = "ARM erratum 834220",
.capability = ARM64_WORKAROUND_834220,
MIDR_RANGE(MIDR_CORTEX_A57, 0x00,
(1 << MIDR_VARIANT_SHIFT) | 2),
ERRATA_MIDR_RANGE(MIDR_CORTEX_A57,
0, 0,
1, 2),
},
#endif
#ifdef CONFIG_ARM64_ERRATUM_845719
@@ -428,7 +496,7 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
/* Cortex-A53 r0p[01234] */
.desc = "ARM erratum 845719",
.capability = ARM64_WORKAROUND_845719,
MIDR_RANGE(MIDR_CORTEX_A53, 0x00, 0x04),
ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 4),
},
#endif
#ifdef CONFIG_CAVIUM_ERRATUM_23154
@@ -436,7 +504,7 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
/* Cavium ThunderX, pass 1.x */
.desc = "Cavium erratum 23154",
.capability = ARM64_WORKAROUND_CAVIUM_23154,
MIDR_RANGE(MIDR_THUNDERX, 0x00, 0x01),
ERRATA_MIDR_REV_RANGE(MIDR_THUNDERX, 0, 0, 1),
},
#endif
#ifdef CONFIG_CAVIUM_ERRATUM_27456
@@ -444,103 +512,402 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
/* Cavium ThunderX, T88 pass 1.x - 2.1 */
.desc = "Cavium erratum 27456",
.capability = ARM64_WORKAROUND_CAVIUM_27456,
MIDR_RANGE(MIDR_THUNDERX, 0x00,
(1 << MIDR_VARIANT_SHIFT) | 1),
ERRATA_MIDR_RANGE(MIDR_THUNDERX,
0, 0,
1, 1),
},
{
/* Cavium ThunderX, T81 pass 1.0 */
.desc = "Cavium erratum 27456",
.capability = ARM64_WORKAROUND_CAVIUM_27456,
MIDR_RANGE(MIDR_THUNDERX_81XX, 0x00, 0x00),
ERRATA_MIDR_REV(MIDR_THUNDERX_81XX, 0, 0),
},
#endif
{
.desc = "Mismatched cache line size",
.capability = ARM64_MISMATCHED_CACHE_LINE_SIZE,
.matches = has_mismatched_cache_type,
.def_scope = SCOPE_LOCAL_CPU,
.enable = cpu_enable_trap_ctr_access,
.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
.cpu_enable = cpu_enable_trap_ctr_access,
},
{
.desc = "Mismatched cache type",
.capability = ARM64_MISMATCHED_CACHE_TYPE,
.matches = has_mismatched_cache_type,
.def_scope = SCOPE_LOCAL_CPU,
.enable = cpu_enable_trap_ctr_access,
.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
.cpu_enable = cpu_enable_trap_ctr_access,
},
#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
{
.capability = ARM64_HARDEN_BRANCH_PREDICTOR,
MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
.enable = enable_smccc_arch_workaround_1,
},
{
.capability = ARM64_HARDEN_BRANCH_PREDICTOR,
MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
.enable = enable_smccc_arch_workaround_1,
},
{
.capability = ARM64_HARDEN_BRANCH_PREDICTOR,
MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
.enable = enable_smccc_arch_workaround_1,
},
{
.capability = ARM64_HARDEN_BRANCH_PREDICTOR,
MIDR_ALL_VERSIONS(MIDR_CORTEX_A75),
.enable = enable_smccc_arch_workaround_1,
},
{
.capability = ARM64_HARDEN_BRANCH_PREDICTOR,
MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN),
.enable = enable_smccc_arch_workaround_1,
},
{
.capability = ARM64_HARDEN_BRANCH_PREDICTOR,
MIDR_ALL_VERSIONS(MIDR_CAVIUM_THUNDERX2),
.enable = enable_smccc_arch_workaround_1,
ERRATA_MIDR_RANGE_LIST(arm64_bp_harden_smccc_cpus),
.cpu_enable = enable_smccc_arch_workaround_1,
},
#endif
#ifdef CONFIG_ARM64_SSBD
{
.desc = "Speculative Store Bypass Disable",
.def_scope = SCOPE_LOCAL_CPU,
.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
.capability = ARM64_SSBD,
.matches = has_ssbd_mitigation,
},
#endif
#ifdef CONFIG_ARM64_ERRATUM_1188873
{
/* Cortex-A76 r0p0 to r2p0 */
.desc = "ARM erratum 1188873",
.capability = ARM64_WORKAROUND_1188873,
ERRATA_MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 2, 0),
},
#endif
{
.desc = "Spectre-BHB",
.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
.capability = ARM64_SPECTRE_BHB,
.matches = is_spectre_bhb_affected,
#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
.cpu_enable = spectre_bhb_enable_mitigation,
#endif
},
{
}
};
/*
* The CPU Errata work arounds are detected and applied at boot time
* and the related information is freed soon after. If the new CPU requires
* an errata not detected at boot, fail this CPU.
* We try to ensure that the mitigation state can never change as the result of
* onlining a late CPU.
*/
void verify_local_cpu_errata_workarounds(void)
static void __maybe_unused update_mitigation_state(enum mitigation_state *oldp,
enum mitigation_state new)
{
const struct arm64_cpu_capabilities *caps = arm64_errata;
enum mitigation_state state;
for (; caps->matches; caps++) {
if (cpus_have_cap(caps->capability)) {
if (caps->enable)
caps->enable((void *)caps);
} else if (caps->matches(caps, SCOPE_LOCAL_CPU)) {
pr_crit("CPU%d: Requires work around for %s, not detected"
" at boot time\n",
smp_processor_id(),
caps->desc ? : "an erratum");
cpu_die_early();
}
do {
state = READ_ONCE(*oldp);
if (new <= state)
break;
} while (cmpxchg_relaxed(oldp, state, new) != state);
}
/*
* Spectre BHB.
*
* A CPU is either:
* - Mitigated by a branchy loop a CPU specific number of times, and listed
* in our "loop mitigated list".
* - Mitigated in software by the firmware Spectre v2 call.
* - Has the ClearBHB instruction to perform the mitigation.
* - Has the 'Exception Clears Branch History Buffer' (ECBHB) feature, so no
* software mitigation in the vectors is needed.
* - Has CSV2.3, so is unaffected.
*/
static enum mitigation_state spectre_bhb_state;
enum mitigation_state arm64_get_spectre_bhb_state(void)
{
return spectre_bhb_state;
}
/*
* This must be called with SCOPE_LOCAL_CPU for each type of CPU, before any
* SCOPE_SYSTEM call will give the right answer.
*/
u8 spectre_bhb_loop_affected(int scope)
{
u8 k = 0;
static u8 max_bhb_k;
if (scope == SCOPE_LOCAL_CPU) {
static const struct midr_range spectre_bhb_k32_list[] = {
MIDR_ALL_VERSIONS(MIDR_CORTEX_A78),
MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C),
MIDR_ALL_VERSIONS(MIDR_CORTEX_X1),
MIDR_ALL_VERSIONS(MIDR_CORTEX_A710),
MIDR_ALL_VERSIONS(MIDR_CORTEX_X2),
MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2),
MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1),
{},
};
static const struct midr_range spectre_bhb_k24_list[] = {
MIDR_ALL_VERSIONS(MIDR_CORTEX_A77),
MIDR_ALL_VERSIONS(MIDR_CORTEX_A76),
MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1),
{},
};
static const struct midr_range spectre_bhb_k8_list[] = {
MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
{},
};
if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k32_list))
k = 32;
else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k24_list))
k = 24;
else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k8_list))
k = 8;
max_bhb_k = max(max_bhb_k, k);
} else {
k = max_bhb_k;
}
return k;
}
static enum mitigation_state spectre_bhb_get_cpu_fw_mitigation_state(void)
{
int ret;
struct arm_smccc_res res;
if (psci_ops.smccc_version == SMCCC_VERSION_1_0)
return SPECTRE_VULNERABLE;
switch (psci_ops.conduit) {
case PSCI_CONDUIT_HVC:
arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
ARM_SMCCC_ARCH_WORKAROUND_3, &res);
break;
case PSCI_CONDUIT_SMC:
arm_smccc_1_1_smc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
ARM_SMCCC_ARCH_WORKAROUND_3, &res);
break;
default:
return SPECTRE_VULNERABLE;
}
ret = res.a0;
switch (ret) {
case SMCCC_RET_SUCCESS:
return SPECTRE_MITIGATED;
case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED:
return SPECTRE_UNAFFECTED;
default:
case SMCCC_RET_NOT_SUPPORTED:
return SPECTRE_VULNERABLE;
}
}
void update_cpu_errata_workarounds(void)
static bool is_spectre_bhb_fw_affected(int scope)
{
update_cpu_capabilities(arm64_errata, "enabling workaround for");
static bool system_affected;
enum mitigation_state fw_state;
bool has_smccc = (psci_ops.smccc_version >= SMCCC_VERSION_1_1);
static const struct midr_range spectre_bhb_firmware_mitigated_list[] = {
MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
MIDR_ALL_VERSIONS(MIDR_CORTEX_A75),
{},
};
bool cpu_in_list = is_midr_in_range_list(read_cpuid_id(),
spectre_bhb_firmware_mitigated_list);
if (scope != SCOPE_LOCAL_CPU)
return system_affected;
fw_state = spectre_bhb_get_cpu_fw_mitigation_state();
if (cpu_in_list || (has_smccc && fw_state == SPECTRE_MITIGATED)) {
system_affected = true;
return true;
}
return false;
}
void __init enable_errata_workarounds(void)
static bool __maybe_unused supports_ecbhb(int scope)
{
enable_cpu_capabilities(arm64_errata);
u64 mmfr1;
if (scope == SCOPE_LOCAL_CPU)
mmfr1 = read_sysreg_s(SYS_ID_AA64MMFR1_EL1);
else
mmfr1 = read_system_reg(SYS_ID_AA64MMFR1_EL1);
return cpuid_feature_extract_unsigned_field(mmfr1,
ID_AA64MMFR1_ECBHB_SHIFT);
}
bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry,
int scope)
{
WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
if (supports_csv2p3(scope))
return false;
if (supports_clearbhb(scope))
return true;
if (spectre_bhb_loop_affected(scope))
return true;
if (is_spectre_bhb_fw_affected(scope))
return true;
return false;
}
#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
static void this_cpu_set_vectors(enum arm64_bp_harden_el1_vectors slot)
{
const char *v = arm64_get_bp_hardening_vector(slot);
if (slot < 0)
return;
__this_cpu_write(this_cpu_vector, v);
/*
* When KPTI is in use, the vectors are switched when exiting to
* user-space.
*/
if (arm64_kernel_unmapped_at_el0())
return;
write_sysreg(v, vbar_el1);
isb();
}
#ifdef CONFIG_KVM
static const char *kvm_bhb_get_vecs_end(const char *start)
{
if (start == __smccc_workaround_3_smc_start)
return __smccc_workaround_3_smc_end;
else if (start == __spectre_bhb_loop_k8_start)
return __spectre_bhb_loop_k8_end;
else if (start == __spectre_bhb_loop_k24_start)
return __spectre_bhb_loop_k24_end;
else if (start == __spectre_bhb_loop_k32_start)
return __spectre_bhb_loop_k32_end;
else if (start == __spectre_bhb_clearbhb_start)
return __spectre_bhb_clearbhb_end;
return NULL;
}
static void kvm_setup_bhb_slot(const char *hyp_vecs_start)
{
int cpu, slot = -1;
const char *hyp_vecs_end;
if (!IS_ENABLED(CONFIG_KVM) || !is_hyp_mode_available())
return;
hyp_vecs_end = kvm_bhb_get_vecs_end(hyp_vecs_start);
if (WARN_ON_ONCE(!hyp_vecs_start || !hyp_vecs_end))
return;
spin_lock(&bp_lock);
for_each_possible_cpu(cpu) {
if (per_cpu(bp_hardening_data.template_start, cpu) == hyp_vecs_start) {
slot = per_cpu(bp_hardening_data.hyp_vectors_slot, cpu);
break;
}
}
if (slot == -1) {
last_slot++;
BUG_ON(((__bp_harden_hyp_vecs_end - __bp_harden_hyp_vecs_start)
/ SZ_2K) <= last_slot);
slot = last_slot;
__copy_hyp_vect_bpi(slot, hyp_vecs_start, hyp_vecs_end);
}
__this_cpu_write(bp_hardening_data.hyp_vectors_slot, slot);
__this_cpu_write(bp_hardening_data.template_start, hyp_vecs_start);
spin_unlock(&bp_lock);
}
#else
#define __smccc_workaround_3_smc_start NULL
#define __spectre_bhb_loop_k8_start NULL
#define __spectre_bhb_loop_k24_start NULL
#define __spectre_bhb_loop_k32_start NULL
#define __spectre_bhb_clearbhb_start NULL
static void kvm_setup_bhb_slot(const char *hyp_vecs_start) { };
#endif /* CONFIG_KVM */
static bool is_spectrev2_safe(void)
{
return !is_midr_in_range_list(read_cpuid_id(),
arm64_bp_harden_smccc_cpus);
}
void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry)
{
enum mitigation_state fw_state, state = SPECTRE_VULNERABLE;
if (!is_spectre_bhb_affected(entry, SCOPE_LOCAL_CPU))
return;
if (!is_spectrev2_safe() && !__hardenbp_enab) {
/* No point mitigating Spectre-BHB alone. */
} else if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY)) {
pr_info_once("spectre-bhb mitigation disabled by compile time option\n");
} else if (cpu_mitigations_off()) {
pr_info_once("spectre-bhb mitigation disabled by command line option\n");
} else if (supports_ecbhb(SCOPE_LOCAL_CPU)) {
state = SPECTRE_MITIGATED;
} else if (supports_clearbhb(SCOPE_LOCAL_CPU)) {
kvm_setup_bhb_slot(__spectre_bhb_clearbhb_start);
this_cpu_set_vectors(EL1_VECTOR_BHB_CLEAR_INSN);
state = SPECTRE_MITIGATED;
} else if (spectre_bhb_loop_affected(SCOPE_LOCAL_CPU)) {
switch (spectre_bhb_loop_affected(SCOPE_SYSTEM)) {
case 8:
kvm_setup_bhb_slot(__spectre_bhb_loop_k8_start);
break;
case 24:
kvm_setup_bhb_slot(__spectre_bhb_loop_k24_start);
break;
case 32:
kvm_setup_bhb_slot(__spectre_bhb_loop_k32_start);
break;
default:
WARN_ON_ONCE(1);
}
this_cpu_set_vectors(EL1_VECTOR_BHB_LOOP);
state = SPECTRE_MITIGATED;
} else if (is_spectre_bhb_fw_affected(SCOPE_LOCAL_CPU)) {
fw_state = spectre_bhb_get_cpu_fw_mitigation_state();
if (fw_state == SPECTRE_MITIGATED) {
kvm_setup_bhb_slot(__smccc_workaround_3_smc_start);
this_cpu_set_vectors(EL1_VECTOR_BHB_FW);
/*
* With WA3 in the vectors, the WA1 calls can be
* removed.
*/
__this_cpu_write(bp_hardening_data.fn, NULL);
state = SPECTRE_MITIGATED;
}
}
update_mitigation_state(&spectre_bhb_state, state);
}
/* Patched to correct the immediate */
void __init spectre_bhb_patch_loop_iter(struct alt_instr *alt,
__le32 *origptr, __le32 *updptr, int nr_inst)
{
u8 rd;
u32 insn;
u16 loop_count = spectre_bhb_loop_affected(SCOPE_SYSTEM);
BUG_ON(nr_inst != 1); /* MOV -> MOV */
if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY))
return;
insn = le32_to_cpu(*origptr);
rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, insn);
insn = aarch64_insn_gen_movewide(rd, loop_count, 0,
AARCH64_INSN_VARIANT_64BIT,
AARCH64_INSN_MOVEWIDE_ZERO);
*updptr++ = cpu_to_le32(insn);
}
#endif /* CONFIG_HARDEN_BRANCH_PREDICTOR */

166
arch/arm64/kernel/cpufeature.c
View File

@@ -20,16 +20,19 @@
#include <linux/bsearch.h>
#include <linux/cpumask.h>
#include <linux/percpu.h>
#include <linux/sort.h>
#include <linux/stop_machine.h>
#include <linux/types.h>
#include <linux/mm.h>
#include <asm/cpu.h>
#include <asm/cpufeature.h>
#include <asm/cpu_ops.h>
#include <asm/mmu_context.h>
#include <asm/processor.h>
#include <asm/sysreg.h>
#include <asm/vectors.h>
#include <asm/virt.h>
unsigned long elf_hwcap __read_mostly;
@@ -50,6 +53,8 @@ unsigned int compat_elf_hwcap2 __read_mostly;
DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
EXPORT_SYMBOL(cpu_hwcaps);
DEFINE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector) = vectors;
DEFINE_STATIC_KEY_ARRAY_FALSE(cpu_hwcap_keys, ARM64_NCAPS);
EXPORT_SYMBOL(cpu_hwcap_keys);
@@ -94,6 +99,11 @@ static const struct arm64_ftr_bits ftr_id_aa64isar0[] = {
ARM64_FTR_END,
};
static const struct arm64_ftr_bits ftr_id_aa64isar2[] = {
ARM64_FTR_BITS(FTR_STRICT, FTR_HIGHER_SAFE, ID_AA64ISAR2_CLEARBHB_SHIFT, 4, 0),
ARM64_FTR_END,
};
static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = {
ARM64_FTR_BITS(FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_CSV3_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_CSV2_SHIFT, 4, 0),
@@ -333,6 +343,7 @@ static const struct __ftr_reg_entry {
/* Op1 = 0, CRn = 0, CRm = 6 */
ARM64_FTR_REG(SYS_ID_AA64ISAR0_EL1, ftr_id_aa64isar0),
ARM64_FTR_REG(SYS_ID_AA64ISAR1_EL1, ftr_aa64raz),
ARM64_FTR_REG(SYS_ID_AA64ISAR2_EL1, ftr_id_aa64isar2),
/* Op1 = 0, CRn = 0, CRm = 7 */
ARM64_FTR_REG(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0),
@@ -445,6 +456,9 @@ static void __init init_cpu_ftr_reg(u32 sys_reg, u64 new)
reg->strict_mask = strict_mask;
}
extern const struct arm64_cpu_capabilities arm64_errata[];
static void update_cpu_errata_workarounds(void);
void __init init_cpu_features(struct cpuinfo_arm64 *info)
{
/* Before we start using the tables, make sure it is sorted */
@@ -457,6 +471,7 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info)
init_cpu_ftr_reg(SYS_ID_AA64DFR1_EL1, info->reg_id_aa64dfr1);
init_cpu_ftr_reg(SYS_ID_AA64ISAR0_EL1, info->reg_id_aa64isar0);
init_cpu_ftr_reg(SYS_ID_AA64ISAR1_EL1, info->reg_id_aa64isar1);
init_cpu_ftr_reg(SYS_ID_AA64ISAR2_EL1, info->reg_id_aa64isar2);
init_cpu_ftr_reg(SYS_ID_AA64MMFR0_EL1, info->reg_id_aa64mmfr0);
init_cpu_ftr_reg(SYS_ID_AA64MMFR1_EL1, info->reg_id_aa64mmfr1);
init_cpu_ftr_reg(SYS_ID_AA64MMFR2_EL1, info->reg_id_aa64mmfr2);
@@ -482,6 +497,11 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info)
init_cpu_ftr_reg(SYS_MVFR2_EL1, info->reg_mvfr2);
}
/*
* Run the errata work around checks on the boot CPU, once we have
* initialised the cpu feature infrastructure.
*/
update_cpu_errata_workarounds();
}
static void update_cpu_ftr_reg(struct arm64_ftr_reg *reg, u64 new)
@@ -563,6 +583,8 @@ void update_cpu_features(int cpu,
info->reg_id_aa64isar0, boot->reg_id_aa64isar0);
taint |= check_update_ftr_reg(SYS_ID_AA64ISAR1_EL1, cpu,
info->reg_id_aa64isar1, boot->reg_id_aa64isar1);
taint |= check_update_ftr_reg(SYS_ID_AA64ISAR2_EL1, cpu,
info->reg_id_aa64isar2, boot->reg_id_aa64isar2);
/*
* Differing PARange support is fine as long as all peripherals and
@@ -682,6 +704,7 @@ static u64 __raw_read_system_reg(u32 sys_id)
case SYS_ID_AA64MMFR2_EL1: return read_cpuid(ID_AA64MMFR2_EL1);
case SYS_ID_AA64ISAR0_EL1: return read_cpuid(ID_AA64ISAR0_EL1);
case SYS_ID_AA64ISAR1_EL1: return read_cpuid(ID_AA64ISAR1_EL1);
case SYS_ID_AA64ISAR2_EL1: return read_cpuid(ID_AA64ISAR2_EL1);
case SYS_CNTFRQ_EL0: return read_cpuid(CNTFRQ_EL0);
case SYS_CTR_EL0: return read_cpuid(CTR_EL0);
@@ -734,13 +757,11 @@ static bool has_useable_gicv3_cpuif(const struct arm64_cpu_capabilities *entry,
static bool has_no_hw_prefetch(const struct arm64_cpu_capabilities *entry, int __unused)
{
u32 midr = read_cpuid_id();
u32 rv_min, rv_max;
/* Cavium ThunderX pass 1.x and 2.x */
rv_min = 0;
rv_max = (1 << MIDR_VARIANT_SHIFT) | MIDR_REVISION_MASK;
return MIDR_IS_CPU_MODEL_RANGE(midr, MIDR_THUNDERX, rv_min, rv_max);
return MIDR_IS_CPU_MODEL_RANGE(midr, MIDR_THUNDERX,
MIDR_CPU_VAR_REV(0, 0),
MIDR_CPU_VAR_REV(1, MIDR_REVISION_MASK));
}
static bool runs_at_el2(const struct arm64_cpu_capabilities *entry, int __unused)
@@ -767,6 +788,17 @@ static int __kpti_forced; /* 0: not forced, >0: forced on, <0: forced off */
static bool unmap_kernel_at_el0(const struct arm64_cpu_capabilities *entry,
int __unused)
{
/* List of CPUs that are not vulnerable and don't need KPTI */
static const struct midr_range kpti_safe_list[] = {
MIDR_ALL_VERSIONS(MIDR_CAVIUM_THUNDERX2),
MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN),
MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),
MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
};
char const *str = "command line option";
u64 pfr0 = read_system_reg(SYS_ID_AA64PFR0_EL1);
@@ -792,16 +824,8 @@ static bool unmap_kernel_at_el0(const struct arm64_cpu_capabilities *entry,
return true;
/* Don't force KPTI for CPUs that are not vulnerable */
switch (read_cpuid_id() & MIDR_CPU_MODEL_MASK) {
case MIDR_CAVIUM_THUNDERX2:
case MIDR_BRCM_VULCAN:
case MIDR_CORTEX_A53:
case MIDR_CORTEX_A55:
case MIDR_CORTEX_A57:
case MIDR_CORTEX_A72:
case MIDR_CORTEX_A73:
if (is_midr_in_range_list(read_cpuid_id(), kpti_safe_list))
return false;
}
/* Defer to CPU feature registers */
return !cpuid_feature_extract_unsigned_field(pfr0,
@@ -809,6 +833,8 @@ static bool unmap_kernel_at_el0(const struct arm64_cpu_capabilities *entry,
}
static int __nocfi kpti_install_ng_mappings(void *__unused)
//static void
//kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)
{
typedef void (kpti_remap_fn)(int, int, phys_addr_t);
extern kpti_remap_fn idmap_kpti_install_ng_mappings;
@@ -817,8 +843,14 @@ static int __nocfi kpti_install_ng_mappings(void *__unused)
static bool kpti_applied = false;
int cpu = smp_processor_id();
if (__this_cpu_read(this_cpu_vector) == vectors) {
const char *v = arm64_get_bp_hardening_vector(EL1_VECTOR_KPTI);
__this_cpu_write(this_cpu_vector, v);
}
if (kpti_applied)
return 0;
return;
remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings);
@@ -829,7 +861,7 @@ static int __nocfi kpti_install_ng_mappings(void *__unused)
if (!cpu)
kpti_applied = true;
return 0;
return;
}
static int __init parse_kpti(char *str)
@@ -846,7 +878,7 @@ static int __init parse_kpti(char *str)
early_param("kpti", parse_kpti);
#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
static int cpu_copy_el2regs(void *__unused)
static void cpu_copy_el2regs(const struct arm64_cpu_capabilities *__unused)
{
/*
* Copy register values that aren't redirected by hardware.
@@ -858,15 +890,13 @@ static int cpu_copy_el2regs(void *__unused)
*/
if (!alternatives_applied)
write_sysreg(read_sysreg(tpidr_el1), tpidr_el2);
return 0;
}
static const struct arm64_cpu_capabilities arm64_features[] = {
{
.desc = "GIC system register CPU interface",
.capability = ARM64_HAS_SYSREG_GIC_CPUIF,
.def_scope = SCOPE_SYSTEM,
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
.matches = has_useable_gicv3_cpuif,
.sys_reg = SYS_ID_AA64PFR0_EL1,
.field_pos = ID_AA64PFR0_GIC_SHIFT,
@@ -877,20 +907,20 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
{
.desc = "Privileged Access Never",
.capability = ARM64_HAS_PAN,
.def_scope = SCOPE_SYSTEM,
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
.matches = has_cpuid_feature,
.sys_reg = SYS_ID_AA64MMFR1_EL1,
.field_pos = ID_AA64MMFR1_PAN_SHIFT,
.sign = FTR_UNSIGNED,
.min_field_value = 1,
.enable = cpu_enable_pan,
.cpu_enable = cpu_enable_pan,
},
#endif /* CONFIG_ARM64_PAN */
#if defined(CONFIG_AS_LSE) && defined(CONFIG_ARM64_LSE_ATOMICS)
{
.desc = "LSE atomic instructions",
.capability = ARM64_HAS_LSE_ATOMICS,
.def_scope = SCOPE_SYSTEM,
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
.matches = has_cpuid_feature,
.sys_reg = SYS_ID_AA64ISAR0_EL1,
.field_pos = ID_AA64ISAR0_ATOMICS_SHIFT,
@@ -901,39 +931,42 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
{
.desc = "Software prefetching using PRFM",
.capability = ARM64_HAS_NO_HW_PREFETCH,
.def_scope = SCOPE_SYSTEM,
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
.matches = has_no_hw_prefetch,
},
#ifdef CONFIG_ARM64_UAO
{
.desc = "User Access Override",
.capability = ARM64_HAS_UAO,
.def_scope = SCOPE_SYSTEM,
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
.matches = has_cpuid_feature,
.sys_reg = SYS_ID_AA64MMFR2_EL1,
.field_pos = ID_AA64MMFR2_UAO_SHIFT,
.min_field_value = 1,
.enable = cpu_enable_uao,
/*
* We rely on stop_machine() calling uao_thread_switch() to set
* UAO immediately after patching.
*/
},
#endif /* CONFIG_ARM64_UAO */
#ifdef CONFIG_ARM64_PAN
{
.capability = ARM64_ALT_PAN_NOT_UAO,
.def_scope = SCOPE_SYSTEM,
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
.matches = cpufeature_pan_not_uao,
},
#endif /* CONFIG_ARM64_PAN */
{
.desc = "Virtualization Host Extensions",
.capability = ARM64_HAS_VIRT_HOST_EXTN,
.def_scope = SCOPE_SYSTEM,
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
.matches = runs_at_el2,
.enable = cpu_copy_el2regs,
.cpu_enable = cpu_copy_el2regs,
},
{
.desc = "32-bit EL0 Support",
.capability = ARM64_HAS_32BIT_EL0,
.def_scope = SCOPE_SYSTEM,
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
.matches = has_cpuid_feature,
.sys_reg = SYS_ID_AA64PFR0_EL1,
.sign = FTR_UNSIGNED,
@@ -943,31 +976,31 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
{
.desc = "Reduced HYP mapping offset",
.capability = ARM64_HYP_OFFSET_LOW,
.def_scope = SCOPE_SYSTEM,
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
.matches = hyp_offset_low,
},
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
{
.desc = "Kernel page table isolation (KPTI)",
.capability = ARM64_UNMAP_KERNEL_AT_EL0,
.def_scope = SCOPE_SYSTEM,
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
.matches = unmap_kernel_at_el0,
.enable = kpti_install_ng_mappings,
.cpu_enable = kpti_install_ng_mappings,
},
#endif
{},
};
#define HWCAP_CAP(reg, field, s, min_value, type, cap) \
#define HWCAP_CAP(reg, field, s, min_value, cap_type, cap) \
{ \
.desc = #cap, \
.def_scope = SCOPE_SYSTEM, \
.type = ARM64_CPUCAP_SYSTEM_FEATURE, \
.matches = has_cpuid_feature, \
.sys_reg = reg, \
.field_pos = field, \
.sign = s, \
.min_field_value = min_value, \
.hwcap_type = type, \
.hwcap_type = cap_type, \
.hwcap = cap, \
}
@@ -1044,7 +1077,7 @@ static bool cpus_have_elf_hwcap(const struct arm64_cpu_capabilities *cap)
static void __init setup_elf_hwcaps(const struct arm64_cpu_capabilities *hwcaps)
{
for (; hwcaps->matches; hwcaps++)
if (hwcaps->matches(hwcaps, hwcaps->def_scope))
if (hwcaps->matches(hwcaps, cpucap_default_scope(hwcaps)))
cap_set_elf_hwcap(hwcaps);
}
@@ -1067,11 +1100,11 @@ static bool __this_cpu_has_cap(const struct arm64_cpu_capabilities *cap_array,
return false;
}
void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps,
const char *info)
static void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps,
const char *info)
{
for (; caps->matches; caps++) {
if (!caps->matches(caps, caps->def_scope))
if (!caps->matches(caps, cpucap_default_scope(caps)))
continue;
if (!cpus_have_cap(caps->capability) && caps->desc)
@@ -1080,11 +1113,20 @@ void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps,
}
}
static int __enable_cpu_capability(void *arg)
{
const struct arm64_cpu_capabilities *cap = arg;
cap->cpu_enable(cap);
return 0;
}
/*
* Run through the enabled capabilities and enable() it on all active
* CPUs
*/
void __init enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps)
static void __init
enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps)
{
for (; caps->matches; caps++) {
unsigned int num = caps->capability;
@@ -1095,14 +1137,15 @@ void __init enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps)
/* Ensure cpus_have_const_cap(num) works */
static_branch_enable(&cpu_hwcap_keys[num]);
if (caps->enable) {
if (caps->cpu_enable) {
/*
* Use stop_machine() as it schedules the work allowing
* us to modify PSTATE, instead of on_each_cpu() which
* uses an IPI, giving us a PSTATE that disappears when
* we return.
*/
stop_machine(caps->enable, (void *)caps, cpu_online_mask);
stop_machine(__enable_cpu_capability, (void *)caps,
cpu_online_mask);
}
}
}
@@ -1160,11 +1203,44 @@ verify_local_cpu_features(const struct arm64_cpu_capabilities *caps_list)
smp_processor_id(), caps->desc);
cpu_die_early();
}
if (caps->enable)
caps->enable((void *)caps);
if (caps->cpu_enable)
caps->cpu_enable(caps);
}
}
/*
* The CPU Errata work arounds are detected and applied at boot time
* and the related information is freed soon after. If the new CPU requires
* an errata not detected at boot, fail this CPU.
*/
static void verify_local_cpu_errata_workarounds(void)
{
const struct arm64_cpu_capabilities *caps = arm64_errata;
for (; caps->matches; caps++) {
if (cpus_have_cap(caps->capability)) {
if (caps->cpu_enable)
caps->cpu_enable(caps);
} else if (caps->matches(caps, SCOPE_LOCAL_CPU)) {
pr_crit("CPU%d: Requires work around for %s, not detected"
" at boot time\n",
smp_processor_id(),
caps->desc ? : "an erratum");
cpu_die_early();
}
}
}
static void update_cpu_errata_workarounds(void)
{
update_cpu_capabilities(arm64_errata, "enabling workaround for");
}
static void __init enable_errata_workarounds(void)
{
enable_cpu_capabilities(arm64_errata);
}
/*
* Run through the enabled system capabilities and enable() it on this CPU.
* The capabilities were decided based on the available CPUs at the boot time.

1
arch/arm64/kernel/cpuinfo.c
View File

@@ -384,6 +384,7 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info)
info->reg_id_aa64dfr1 = read_cpuid(ID_AA64DFR1_EL1);
info->reg_id_aa64isar0 = read_cpuid(ID_AA64ISAR0_EL1);
info->reg_id_aa64isar1 = read_cpuid(ID_AA64ISAR1_EL1);
info->reg_id_aa64isar2 = read_cpuid(ID_AA64ISAR2_EL1);
info->reg_id_aa64mmfr0 = read_cpuid(ID_AA64MMFR0_EL1);
info->reg_id_aa64mmfr1 = read_cpuid(ID_AA64MMFR1_EL1);
info->reg_id_aa64mmfr2 = read_cpuid(ID_AA64MMFR2_EL1);

187
arch/arm64/kernel/entry.S
View File

@@ -76,26 +76,33 @@
.macro kernel_ventry, el, label, regsize = 64
.align 7
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
alternative_if ARM64_UNMAP_KERNEL_AT_EL0
.Lventry_start\@:
.if \el == 0
/*
* This must be the first instruction of the EL0 vector entries. It is
* skipped by the trampoline vectors, to trigger the cleanup.
*/
b .Lskip_tramp_vectors_cleanup\@
.if \regsize == 64
mrs x30, tpidrro_el0
msr tpidrro_el0, xzr
.else
mov x30, xzr
.endif
.Lskip_tramp_vectors_cleanup\@:
.endif
alternative_else_nop_endif
#endif
sub sp, sp, #S_FRAME_SIZE
b el\()\el\()_\label
.org .Lventry_start\@ + 128 // Did we overflow the ventry slot?
.endm
.macro tramp_alias, dst, sym
.macro tramp_alias, dst, sym, tmp
mov_q \dst, TRAMP_VALIAS
add \dst, \dst, #(\sym - .entry.tramp.text)
adr_l \tmp, \sym
add \dst, \dst, \tmp
adr_l \tmp, .entry.tramp.text
sub \dst, \dst, \tmp
.endm
// This macro corrupts x0-x3. It is the caller's duty
@@ -300,21 +307,25 @@ alternative_else_nop_endif
ldp x24, x25, [sp, #16 * 12]
ldp x26, x27, [sp, #16 * 13]
ldp x28, x29, [sp, #16 * 14]
ldr lr, [sp, #S_LR]
add sp, sp, #S_FRAME_SIZE // restore sp
.if \el == 0
alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0
ldr lr, [sp, #S_LR]
add sp, sp, #S_FRAME_SIZE // restore sp
eret
alternative_else_nop_endif
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
bne 4f
msr far_el1, x30
tramp_alias x30, tramp_exit_native
msr far_el1, x29
tramp_alias x30, tramp_exit_native, x29
br x30
4:
tramp_alias x30, tramp_exit_compat
tramp_alias x30, tramp_exit_compat, x29
br x30
#endif
.else
ldr lr, [sp, #S_LR]
add sp, sp, #S_FRAME_SIZE // restore sp
eret
.endif
.endm
@@ -947,12 +958,7 @@ __ni_sys_trace:
.popsection // .entry.text
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
/*
* Exception vectors trampoline.
*/
.pushsection ".entry.tramp.text", "ax"
// Move from tramp_pg_dir to swapper_pg_dir
.macro tramp_map_kernel, tmp
mrs \tmp, ttbr1_el1
sub \tmp, \tmp, #(SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE)
@@ -982,12 +988,47 @@ __ni_sys_trace:
*/
.endm
.macro tramp_ventry, regsize = 64
.macro tramp_data_page dst
adr_l \dst, .entry.tramp.text
sub \dst, \dst, PAGE_SIZE
.endm
.macro tramp_data_read_var dst, var
#ifdef CONFIG_RANDOMIZE_BASE
tramp_data_page \dst
add \dst, \dst, #:lo12:__entry_tramp_data_\var
ldr \dst, [\dst]
#else
ldr \dst, =\var
#endif
.endm
#define BHB_MITIGATION_NONE 0
#define BHB_MITIGATION_LOOP 1
#define BHB_MITIGATION_FW 2
#define BHB_MITIGATION_INSN 3
.macro tramp_ventry, vector_start, regsize, kpti, bhb
.align 7
1:
.if \regsize == 64
msr tpidrro_el0, x30 // Restored in kernel_ventry
.endif
.if \bhb == BHB_MITIGATION_LOOP
/*
* This sequence must appear before the first indirect branch. i.e. the
* ret out of tramp_ventry. It appears here because x30 is free.
*/
__mitigate_spectre_bhb_loop x30
.endif // \bhb == BHB_MITIGATION_LOOP
.if \bhb == BHB_MITIGATION_INSN
clearbhb
isb
.endif // \bhb == BHB_MITIGATION_INSN
.if \kpti == 1
/*
* Defend against branch aliasing attacks by pushing a dummy
* entry onto the return stack and using a RET instruction to
@@ -1008,34 +1049,71 @@ __ni_sys_trace:
#endif
prfm plil1strm, [x30, #(1b - tramp_vectors)]
msr vbar_el1, x30
add x30, x30, #(1b - tramp_vectors)
isb
.else
ldr x30, =vectors
.endif // \kpti == 1
.if \bhb == BHB_MITIGATION_FW
/*
* The firmware sequence must appear before the first indirect branch.
* i.e. the ret out of tramp_ventry. But it also needs the stack to be
* mapped to save/restore the registers the SMC clobbers.
*/
__mitigate_spectre_bhb_fw
.endif // \bhb == BHB_MITIGATION_FW
add x30, x30, #(1b - \vector_start + 4)
ret
.org 1b + 128 // Did we overflow the ventry slot?
.endm
.macro tramp_exit, regsize = 64
adr x30, tramp_vectors
tramp_data_read_var x30, this_cpu_vector
alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
mrs x29, tpidr_el1
alternative_else
mrs x29, tpidr_el2
alternative_endif
ldr x30, [x30, x29]
msr vbar_el1, x30
tramp_unmap_kernel x30
ldr lr, [sp, #S_LR]
tramp_unmap_kernel x29
.if \regsize == 64
mrs x30, far_el1
mrs x29, far_el1
.endif
add sp, sp, #S_FRAME_SIZE // restore sp
eret
.endm
.align 11
ENTRY(tramp_vectors)
.macro generate_tramp_vector, kpti, bhb
.Lvector_start\@:
.space 0x400
tramp_ventry
tramp_ventry
tramp_ventry
tramp_ventry
.rept 4
tramp_ventry .Lvector_start\@, 64, \kpti, \bhb
.endr
.rept 4
tramp_ventry .Lvector_start\@, 32, \kpti, \bhb
.endr
.endm
tramp_ventry 32
tramp_ventry 32
tramp_ventry 32
tramp_ventry 32
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
/*
* Exception vectors trampoline.
* The order must match __bp_harden_el1_vectors and the
* arm64_bp_harden_el1_vectors enum.
*/
.pushsection ".entry.tramp.text", "ax"
.align 11
ENTRY(tramp_vectors)
#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_LOOP
generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_FW
generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_INSN
#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_NONE
END(tramp_vectors)
ENTRY(tramp_exit_native)
@@ -1053,11 +1131,54 @@ END(tramp_exit_compat)
.align PAGE_SHIFT
.globl __entry_tramp_data_start
__entry_tramp_data_start:
__entry_tramp_data_vectors:
.quad vectors
#ifdef CONFIG_ARM_SDE_INTERFACE
__entry_tramp_data___sdei_asm_trampoline_next_handler:
.quad __sdei_asm_handler
#endif /* CONFIG_ARM_SDE_INTERFACE */
__entry_tramp_data_this_cpu_vector:
.quad this_cpu_vector
.popsection // .rodata
#endif /* CONFIG_RANDOMIZE_BASE */
#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
/*
* Exception vectors for spectre mitigations on entry from EL1 when
* kpti is not in use.
*/
.macro generate_el1_vector, bhb
.Lvector_start\@:
kernel_ventry 1, sync_invalid // Synchronous EL1t
kernel_ventry 1, irq_invalid // IRQ EL1t
kernel_ventry 1, fiq_invalid // FIQ EL1t
kernel_ventry 1, error_invalid // Error EL1t
kernel_ventry 1, sync // Synchronous EL1h
kernel_ventry 1, irq // IRQ EL1h
kernel_ventry 1, fiq_invalid // FIQ EL1h
kernel_ventry 1, error_invalid // Error EL1h
.rept 4
tramp_ventry .Lvector_start\@, 64, 0, \bhb
.endr
.rept 4
tramp_ventry .Lvector_start\@, 32, 0, \bhb
.endr
.endm
/* The order must match tramp_vecs and the arm64_bp_harden_el1_vectors enum. */
.pushsection ".entry.text", "ax"
.align 11
ENTRY(__bp_harden_el1_vectors)
#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
generate_el1_vector bhb=BHB_MITIGATION_LOOP
generate_el1_vector bhb=BHB_MITIGATION_FW
generate_el1_vector bhb=BHB_MITIGATION_INSN
#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
END(__bp_harden_el1_vectors)
.popsection
/*
* Special system call wrappers.
*/

1
arch/arm64/kernel/fpsimd.c
View File

@@ -26,6 +26,7 @@
#include <linux/hardirq.h>
#include <asm/fpsimd.h>
#include <asm/cpufeature.h>
#include <asm/cputype.h>
#define FPEXC_IOF (1 << 0)

29
arch/arm64/kernel/insn.c
View File

@@ -418,6 +418,35 @@ u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
return insn;
}
u32 aarch64_insn_decode_register(enum aarch64_insn_register_type type,
u32 insn)
{
int shift;
switch (type) {
case AARCH64_INSN_REGTYPE_RT:
case AARCH64_INSN_REGTYPE_RD:
shift = 0;
break;
case AARCH64_INSN_REGTYPE_RN:
shift = 5;
break;
case AARCH64_INSN_REGTYPE_RT2:
case AARCH64_INSN_REGTYPE_RA:
shift = 10;
break;
case AARCH64_INSN_REGTYPE_RM:
shift = 16;
break;
default:
pr_err("%s: unknown register type encoding %d\n", __func__,
type);
return 0;
}
return (insn >> shift) & GENMASK(4, 0);
}
static u32 aarch64_insn_encode_register(enum aarch64_insn_register_type type,
u32 insn,
enum aarch64_insn_register reg)

6
arch/arm64/kernel/smp.c
View File

@@ -460,12 +460,6 @@ void __init smp_prepare_boot_cpu(void)
jump_label_init();
cpuinfo_store_boot_cpu();
save_boot_cpu_run_el();
/*
* Run the errata work around checks on the boot CPU, once we have
* initialised the cpu feature infrastructure from
* cpuinfo_store_boot_cpu() above.
*/
update_cpu_errata_workarounds();
}
static u64 __init of_get_cpu_mpidr(struct device_node *dn)

4
arch/arm64/kernel/traps.c
View File

@@ -35,6 +35,7 @@
#include <asm/atomic.h>
#include <asm/barrier.h>
#include <asm/bug.h>
#include <asm/cpufeature.h>
#include <asm/debug-monitors.h>
#include <asm/esr.h>
#include <asm/insn.h>
@@ -478,10 +479,9 @@ asmlinkage void __exception do_undefinstr(struct pt_regs *regs)
force_signal_inject(SIGILL, ILL_ILLOPC, regs, 0);
}
int cpu_enable_cache_maint_trap(void *__unused)
void cpu_enable_cache_maint_trap(const struct arm64_cpu_capabilities *__unused)
{
config_sctlr_el1(SCTLR_EL1_UCI, 0);
return 0;
}
#define __user_cache_maint(insn, address, res) \

2
arch/arm64/kernel/vmlinux.lds.S
View File

@@ -266,7 +266,7 @@ ASSERT(__hibernate_exit_text_end - (__hibernate_exit_text_start & ~(SZ_4K - 1))
<= SZ_4K, "Hibernate exit text too big or misaligned")
#endif
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
ASSERT((__entry_tramp_text_end - __entry_tramp_text_start) == PAGE_SIZE,
ASSERT((__entry_tramp_text_end - __entry_tramp_text_start) <= 3*PAGE_SIZE,
"Entry trampoline text too big")
#endif
/*

4
arch/arm64/kvm/hyp/hyp-entry.S
View File

@@ -136,6 +136,10 @@ el1_hvc_guest:
/* ARM_SMCCC_ARCH_WORKAROUND_2 handling */
eor w1, w1, #(ARM_SMCCC_ARCH_WORKAROUND_1 ^ \
ARM_SMCCC_ARCH_WORKAROUND_2)
cbz w1, wa_epilogue
eor w1, w1, #(ARM_SMCCC_ARCH_WORKAROUND_2 ^ \
ARM_SMCCC_ARCH_WORKAROUND_3)
cbnz w1, el1_trap
#ifdef CONFIG_ARM64_SSBD

9
arch/arm64/kvm/hyp/switch.c
View File

@@ -26,7 +26,7 @@
#include <asm/kvm_emulate.h>
#include <asm/kvm_hyp.h>
#include <asm/uaccess.h>
#include <asm/vectors.h>
extern struct exception_table_entry __start___kvm_ex_table;
extern struct exception_table_entry __stop___kvm_ex_table;
@@ -107,11 +107,14 @@ static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu)
static void __hyp_text __deactivate_traps_vhe(void)
{
extern char vectors[]; /* kernel exception vectors */
const char *host_vectors = vectors;
write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2);
write_sysreg(CPACR_EL1_FPEN, cpacr_el1);
write_sysreg(vectors, vbar_el1);
if (!arm64_kernel_unmapped_at_el0())
host_vectors = __this_cpu_read(this_cpu_vector);
write_sysreg(host_vectors, vbar_el1);
}
static void __hyp_text __deactivate_traps_nvhe(void)

17
arch/arm64/mm/fault.c
View File

@@ -838,7 +838,7 @@ asmlinkage int __exception do_debug_exception(unsigned long addr_if_watchpoint,
NOKPROBE_SYMBOL(do_debug_exception);
#ifdef CONFIG_ARM64_PAN
int cpu_enable_pan(void *__unused)
void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused)
{
/*
* We modify PSTATE. This won't work from irq context as the PSTATE
@@ -848,20 +848,5 @@ int cpu_enable_pan(void *__unused)
config_sctlr_el1(SCTLR_EL1_SPAN, 0);
asm(SET_PSTATE_PAN(1));
return 0;
}
#endif /* CONFIG_ARM64_PAN */
#ifdef CONFIG_ARM64_UAO
/*
* Kernel threads have fs=KERNEL_DS by default, and don't need to call
* set_fs(), devtmpfs in particular relies on this behaviour.
* We need to enable the feature at runtime (instead of adding it to
* PSR_MODE_EL1h) as the feature may not be implemented by the cpu.
*/
int cpu_enable_uao(void *__unused)
{
asm(SET_PSTATE_UAO(1));
return 0;
}
#endif /* CONFIG_ARM64_UAO */

11
arch/arm64/mm/mmu.c
View File

@@ -449,6 +449,7 @@ static void __init map_kernel_segment(pgd_t *pgd, void *va_start, void *va_end,
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
static int __init map_entry_trampoline(void)
{
int i;
extern char __entry_tramp_text_start[];
pgprot_t prot = rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC;
@@ -459,11 +460,15 @@ static int __init map_entry_trampoline(void)
/* Map only the text into the trampoline page table */
memset(tramp_pg_dir, 0, PGD_SIZE);
__create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, PAGE_SIZE,
prot, pgd_pgtable_alloc, 0);
__create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS,
entry_tramp_text_size(), prot, pgd_pgtable_alloc,
0);
/* Map both the text and data into the kernel page table */
__set_fixmap(FIX_ENTRY_TRAMP_TEXT, pa_start, prot);
for (i = 0; i < DIV_ROUND_UP(entry_tramp_text_size(), PAGE_SIZE); i++)
__set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i,
pa_start + i * PAGE_SIZE, prot);
if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) {
extern char __entry_tramp_data_start[];

4
drivers/clocksource/Kconfig
View File

@@ -312,11 +312,15 @@ config ARM_ARCH_TIMER_VCT_ACCESS
help
This option enables support for reading the ARM architected timer's
virtual counter in userspace.
config ARM_ARCH_TIMER_OOL_WORKAROUND
bool
config FSL_ERRATUM_A008585
bool "Workaround for Freescale/NXP Erratum A-008585"
default y
depends on ARM_ARCH_TIMER && ARM64
select ARM_ARCH_TIMER_OOL_WORKAROUND
help
This option enables a workaround for Freescale/NXP Erratum
A-008585 ("ARM generic timer may contain an erroneous

202
drivers/clocksource/arm_arch_timer.c
View File

@@ -96,41 +96,159 @@ early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg);
*/
#ifdef CONFIG_FSL_ERRATUM_A008585
DEFINE_STATIC_KEY_FALSE(arch_timer_read_ool_enabled);
EXPORT_SYMBOL_GPL(arch_timer_read_ool_enabled);
/*
* The number of retries is an arbitrary value well beyond the highest number
* of iterations the loop has been observed to take.
*/
#define __fsl_a008585_read_reg(reg) ({ \
u64 _old, _new; \
int _retries = 200; \
\
do { \
_old = read_sysreg(reg); \
_new = read_sysreg(reg); \
_retries--; \
} while (unlikely(_old != _new) && _retries); \
\
WARN_ON_ONCE(!_retries); \
_new; \
})
static int fsl_a008585_enable = -1;
static int __init early_fsl_a008585_cfg(char *buf)
{
int ret;
bool val;
ret = strtobool(buf, &val);
if (ret)
return ret;
fsl_a008585_enable = val;
return 0;
}
early_param("clocksource.arm_arch_timer.fsl-a008585", early_fsl_a008585_cfg);
u32 __fsl_a008585_read_cntp_tval_el0(void)
static u32 notrace fsl_a008585_read_cntp_tval_el0(void)
{
return __fsl_a008585_read_reg(cntp_tval_el0);
}
u32 __fsl_a008585_read_cntv_tval_el0(void)
static u32 notrace fsl_a008585_read_cntv_tval_el0(void)
{
return __fsl_a008585_read_reg(cntv_tval_el0);
}
u64 __fsl_a008585_read_cntvct_el0(void)
static u64 notrace fsl_a008585_read_cntvct_el0(void)
{
return __fsl_a008585_read_reg(cntvct_el0);
}
EXPORT_SYMBOL(__fsl_a008585_read_cntvct_el0);
#endif /* CONFIG_FSL_ERRATUM_A008585 */
#endif
#ifdef CONFIG_ARM64_ERRATUM_1188873
static u64 notrace arm64_1188873_read_cntvct_el0(void)
{
return read_sysreg(cntvct_el0);
}
#endif
#ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND
const struct arch_timer_erratum_workaround *timer_unstable_counter_workaround = NULL;
EXPORT_SYMBOL_GPL(timer_unstable_counter_workaround);
DEFINE_STATIC_KEY_FALSE(arch_timer_read_ool_enabled);
EXPORT_SYMBOL_GPL(arch_timer_read_ool_enabled);
static const struct arch_timer_erratum_workaround ool_workarounds[] = {
#ifdef CONFIG_FSL_ERRATUM_A008585
{
.match_type = ate_match_dt,
.id = "fsl,erratum-a008585",
.desc = "Freescale erratum a005858",
.read_cntp_tval_el0 = fsl_a008585_read_cntp_tval_el0,
.read_cntv_tval_el0 = fsl_a008585_read_cntv_tval_el0,
.read_cntvct_el0 = fsl_a008585_read_cntvct_el0,
},
#endif
#ifdef CONFIG_ARM64_ERRATUM_1188873
{
.match_type = ate_match_local_cap_id,
.id = (void *)ARM64_WORKAROUND_1188873,
.desc = "ARM erratum 1188873",
.read_cntvct_el0 = arm64_1188873_read_cntvct_el0,
},
#endif
};
typedef bool (*ate_match_fn_t)(const struct arch_timer_erratum_workaround *,
const void *);
static
bool arch_timer_check_dt_erratum(const struct arch_timer_erratum_workaround *wa,
const void *arg)
{
const struct device_node *np = arg;
return of_property_read_bool(np, wa->id);
}
static
bool arch_timer_check_local_cap_erratum(const struct arch_timer_erratum_workaround *wa,
const void *arg)
{
return this_cpu_has_cap((uintptr_t)wa->id);
}
static const struct arch_timer_erratum_workaround *
arch_timer_iterate_errata(enum arch_timer_erratum_match_type type,
ate_match_fn_t match_fn,
void *arg)
{
int i;
for (i = 0; i < ARRAY_SIZE(ool_workarounds); i++) {
if (ool_workarounds[i].match_type != type)
continue;
if (match_fn(&ool_workarounds[i], arg))
return &ool_workarounds[i];
}
return NULL;
}
static
void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa)
{
timer_unstable_counter_workaround = wa;
static_branch_enable(&arch_timer_read_ool_enabled);
}
static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type type,
void *arg)
{
const struct arch_timer_erratum_workaround *wa;
ate_match_fn_t match_fn = NULL;
bool local = false;
switch (type) {
case ate_match_dt:
match_fn = arch_timer_check_dt_erratum;
break;
case ate_match_local_cap_id:
match_fn = arch_timer_check_local_cap_erratum;
local = true;
break;
default:
WARN_ON(1);
return;
}
wa = arch_timer_iterate_errata(type, match_fn, arg);
if (!wa)
return;
if (needs_unstable_timer_counter_workaround()) {
if (wa != timer_unstable_counter_workaround)
pr_warn("Can't enable workaround for %s (clashes with %s\n)",
wa->desc,
timer_unstable_counter_workaround->desc);
return;
}
arch_timer_enable_workaround(wa);
pr_info("Enabling %s workaround for %s\n",
local ? "local" : "global", wa->desc);
}
#else
#define arch_timer_check_ool_workaround(t,a) do { } while(0)
#endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */
static __always_inline
void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,
@@ -281,8 +399,8 @@ static __always_inline void set_next_event(const int access, unsigned long evt,
arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
}
#ifdef CONFIG_FSL_ERRATUM_A008585
static __always_inline void fsl_a008585_set_next_event(const int access,
#ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND
static __always_inline void erratum_set_next_event_generic(const int access,
unsigned long evt, struct clock_event_device *clk)
{
unsigned long ctrl;
@@ -300,20 +418,20 @@ static __always_inline void fsl_a008585_set_next_event(const int access,
arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
}
static int fsl_a008585_set_next_event_virt(unsigned long evt,
static int erratum_set_next_event_virt(unsigned long evt,
struct clock_event_device *clk)
{
fsl_a008585_set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
erratum_set_next_event_generic(ARCH_TIMER_VIRT_ACCESS, evt, clk);
return 0;
}
static int fsl_a008585_set_next_event_phys(unsigned long evt,
static int erratum_set_next_event_phys(unsigned long evt,
struct clock_event_device *clk)
{
fsl_a008585_set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
erratum_set_next_event_generic(ARCH_TIMER_PHYS_ACCESS, evt, clk);
return 0;
}
#endif /* CONFIG_FSL_ERRATUM_A008585 */
#endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */
static int arch_timer_set_next_event_virt(unsigned long evt,
struct clock_event_device *clk)
@@ -343,16 +461,16 @@ static int arch_timer_set_next_event_phys_mem(unsigned long evt,
return 0;
}
static void fsl_a008585_set_sne(struct clock_event_device *clk)
static void erratum_workaround_set_sne(struct clock_event_device *clk)
{
#ifdef CONFIG_FSL_ERRATUM_A008585
#ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND
if (!static_branch_unlikely(&arch_timer_read_ool_enabled))
return;
if (arch_timer_uses_ppi == VIRT_PPI)
clk->set_next_event = fsl_a008585_set_next_event_virt;
clk->set_next_event = erratum_set_next_event_virt;
else
clk->set_next_event = fsl_a008585_set_next_event_phys;
clk->set_next_event = erratum_set_next_event_phys;
#endif
}
@@ -385,7 +503,9 @@ static void __arch_timer_setup(unsigned type,
BUG();
}
fsl_a008585_set_sne(clk);
arch_timer_check_ool_workaround(ate_match_local_cap_id, NULL);
erratum_workaround_set_sne(clk);
} else {
clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
clk->name = "arch_mem_timer";
@@ -622,7 +742,7 @@ static void __init arch_counter_register(unsigned type)
clocksource_counter.archdata.vdso_direct = true;
#ifdef CONFIG_FSL_ERRATUM_A008585
#ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND
/*
* Don't use the vdso fastpath if errata require using
* the out-of-line counter accessor.
@@ -910,14 +1030,8 @@ static int __init arch_timer_of_init(struct device_node *np)
arch_timer_c3stop = !of_property_read_bool(np, "always-on");
#ifdef CONFIG_FSL_ERRATUM_A008585
if (fsl_a008585_enable < 0)
fsl_a008585_enable = of_property_read_bool(np, "fsl,erratum-a008585");
if (fsl_a008585_enable) {
static_branch_enable(&arch_timer_read_ool_enabled);
pr_info("Enabling workaround for FSL erratum A-008585\n");
}
#endif
/* Check for globally applicable workarounds */
arch_timer_check_ool_workaround(ate_match_dt, np);
/*
* If we cannot rely on firmware initializing the timer registers then

7
include/linux/arm-smccc.h
View File

@@ -85,6 +85,13 @@
ARM_SMCCC_SMC_32, \
0, 0x7fff)
#define ARM_SMCCC_ARCH_WORKAROUND_3 \
ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \
ARM_SMCCC_SMC_32, \
0, 0x3fff)
#define SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED 1
#ifndef __ASSEMBLY__
#include <linux/linkage.h>