diff --git a/Documentation/ABI/testing/sysfs-firmware-sgi_uv b/Documentation/ABI/testing/sysfs-firmware-sgi_uv index 66800baab096..637c668cbe45 100644 --- a/Documentation/ABI/testing/sysfs-firmware-sgi_uv +++ b/Documentation/ABI/testing/sysfs-firmware-sgi_uv @@ -1,27 +1,159 @@ What: /sys/firmware/sgi_uv/ -Date: August 2008 -Contact: Russ Anderson +Date: September 2020 +Contact: Justin Ernst Description: The /sys/firmware/sgi_uv directory contains information - about the SGI UV platform. + about the UV platform. - Under that directory are a number of files:: + Under that directory are a number of read-only attributes:: + archtype + hub_type + hubless partition_id coherence_id + uv_type + + The archtype entry contains the UV architecture type that + is used to select arch-dependent addresses and features. + It can be set via the OEM_ID in the ACPI MADT table or by + UVsystab entry both passed from UV BIOS. + + The hub_type entry is used to select the type of hub which is + similar to uv_type but encoded in a binary format. Include + the file uv_hub.h to get the definitions. + + The hubless entry basically is present and set only if there + is no hub. In this case the hub_type entry is not present. The partition_id entry contains the partition id. - SGI UV systems can be partitioned into multiple physical + UV systems can be partitioned into multiple physical machines, which each partition running a unique copy - of the operating system. Each partition will have a unique - partition id. To display the partition id, use the command:: - - cat /sys/firmware/sgi_uv/partition_id + of the operating system. Each partition will have a unique + partition id. The coherence_id entry contains the coherence id. - A partitioned SGI UV system can have one or more coherence - domain. The coherence id indicates which coherence domain - this partition is in. To display the coherence id, use the - command:: + A partitioned UV system can have one or more coherence + domains. The coherence id indicates which coherence domain + this partition is in. - cat /sys/firmware/sgi_uv/coherence_id + The uv_type entry contains the hub revision number. + This value can be used to identify the UV system version:: + "0.*" = Hubless UV ('*' is subtype) + + "3.0" = UV2 + "5.0" = UV3 + "7.0" = UV4 + "7.1" = UV4a + "9.0" = UV5 + + The /sys/firmware/sgi_uv directory also contains two directories:: + + hubs/ + pcibuses/ + + The hubs directory contains a number of hub objects, each representing + a UV Hub visible to the BIOS. Each hub object's name is appended by a + unique ordinal value (ex. /sys/firmware/sgi_uv/hubs/hub_5) + + Each hub object directory contains a number of read-only attributes:: + + cnode + location + name + nasid + shared + this_partition + + The cnode entry contains the cnode number of the corresponding hub. + If a cnode value is not applicable, the value returned will be -1. + + The location entry contains the location string of the corresponding hub. + This value is used to physically identify a hub within a system. + + The name entry contains the name of the corresponding hub. This name can + be two variants:: + + "UVHub x.x" = A 'node' ASIC, connecting a CPU to the interconnect + fabric. The 'x.x' value represents the ASIC revision. + (ex. 'UVHub 5.0') + + "NLxRouter" = A 'router ASIC, only connecting other ASICs to + the interconnect fabric. The 'x' value representing + the fabric technology version. (ex. 'NL8Router') + + The nasid entry contains the nasid number of the corresponding hub. + If a nasid value is not applicable, the value returned will be -1. + + The shared entry contains a boolean value describing whether the + corresponding hub is shared between system partitions. + + The this_partition entry contains a boolean value describing whether + the corresponding hub is local to the current partition. + + Each hub object directory also contains a number of port objects, + each representing a fabric port on the corresponding hub. + A port object's name is appended by a unique ordinal value + (ex. /sys/firmware/sgi_uv/hubs/hub_5/port_3) + + Each port object directory contains a number of read-only attributes:: + + conn_hub + conn_port + + The conn_hub entry contains a value representing the unique + oridinal value of the hub on the other end of the fabric + cable plugged into the port. If the port is disconnected, + the value returned will be -1. + + The conn_port entry contains a value representing the unique + oridinal value of the port on the other end of the fabric cable + plugged into the port. If the port is disconnected, the value + returned will be -1. + + Ex: + A value of '3' is read from: + /sys/firmware/sgi_uv/hubs/hub_5/port_3/conn_hub + + and a value of '6' is read from: + /sys/firmware/sgi_uv/hubs/hub_5/port_3/conn_port + + representing that this port is connected to: + /sys/firmware/sgi_uv/hubs/hub_3/port_6 + + The pcibuses directory contains a number of PCI bus objects. + Each PCI bus object's name is appended by its PCI bus address. + (ex. pcibus_0003:80) + + Each pcibus object has a number of possible read-only attributes:: + + type + location + slot + ppb_addr + iio_stack + + The type entry contains a value describing the type of IO at + the corresponding PCI bus address. Known possible values + across all UV versions are:: + + BASE IO + PCIe IO + PCIe SLOT + NODE IO + Riser + PPB + + The location entry contains the location string of the UV Hub + of the CPU physically connected to the corresponding PCI bus. + + The slot entry contains the physical slot number of the + corresponding PCI bus. This value is used to physically locate + PCI cards within a system. + + The ppb_addr entry contains the PCI address string of the + bridged PCI bus. This entry is only present when the PCI bus + object type is 'PPB'. + + The iio_stack entry contains a value describing the IIO stack + number that the corresponding PCI bus object is connected to. diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index b4b4e54b5eee..4f8a54765d3c 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -3379,6 +3379,8 @@ nosep [BUGS=X86-32] Disables x86 SYSENTER/SYSEXIT support. + nosgx [X86-64,SGX] Disables Intel SGX kernel support. + nosmp [SMP] Tells an SMP kernel to act as a UP kernel, and disable the IO APIC. legacy for "maxcpus=0". diff --git a/Documentation/devicetree/bindings/crypto/intel,keembay-ocs-aes.yaml b/Documentation/devicetree/bindings/crypto/intel,keembay-ocs-aes.yaml new file mode 100644 index 000000000000..ee2c099981b2 --- /dev/null +++ b/Documentation/devicetree/bindings/crypto/intel,keembay-ocs-aes.yaml @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/crypto/intel,keembay-ocs-aes.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Intel Keem Bay OCS AES Device Tree Bindings + +maintainers: + - Daniele Alessandrelli + +description: + The Intel Keem Bay Offload and Crypto Subsystem (OCS) AES engine provides + hardware-accelerated AES/SM4 encryption/decryption. + +properties: + compatible: + const: intel,keembay-ocs-aes + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + - clocks + +additionalProperties: false + +examples: + - | + #include + crypto@30008000 { + compatible = "intel,keembay-ocs-aes"; + reg = <0x30008000 0x1000>; + interrupts = ; + clocks = <&scmi_clk 95>; + }; diff --git a/Documentation/devicetree/bindings/edac/aspeed-sdram-edac.txt b/Documentation/devicetree/bindings/edac/aspeed-sdram-edac.txt index 6a0f3d90d682..8ca9e0a049d8 100644 --- a/Documentation/devicetree/bindings/edac/aspeed-sdram-edac.txt +++ b/Documentation/devicetree/bindings/edac/aspeed-sdram-edac.txt @@ -1,6 +1,6 @@ -Aspeed AST2500 SoC EDAC node +Aspeed BMC SoC EDAC node -The Aspeed AST2500 SoC supports DDR3 and DDR4 memory with and without ECC (error +The Aspeed BMC SoC supports DDR3 and DDR4 memory with and without ECC (error correction check). The memory controller supports SECDED (single bit error correction, double bit @@ -11,7 +11,10 @@ Note, the bootloader must configure ECC mode in the memory controller. Required properties: -- compatible: should be "aspeed,ast2500-sdram-edac" +- compatible: should be one of + - "aspeed,ast2400-sdram-edac" + - "aspeed,ast2500-sdram-edac" + - "aspeed,ast2600-sdram-edac" - reg: sdram controller register set should be <0x1e6e0000 0x174> - interrupts: should be AVIC interrupt #0 diff --git a/Documentation/filesystems/fsverity.rst b/Documentation/filesystems/fsverity.rst index 895e9711ed88..e0204a23e997 100644 --- a/Documentation/filesystems/fsverity.rst +++ b/Documentation/filesystems/fsverity.rst @@ -27,9 +27,9 @@ automatically verified against the file's Merkle tree. Reads of any corrupted data, including mmap reads, will fail. Userspace can use another ioctl to retrieve the root hash (actually -the "file measurement", which is a hash that includes the root hash) -that fs-verity is enforcing for the file. This ioctl executes in -constant time, regardless of the file size. +the "fs-verity file digest", which is a hash that includes the Merkle +tree root hash) that fs-verity is enforcing for the file. This ioctl +executes in constant time, regardless of the file size. fs-verity is essentially a way to hash a file in constant time, subject to the caveat that reads which would violate the hash will @@ -177,9 +177,10 @@ FS_IOC_ENABLE_VERITY can fail with the following errors: FS_IOC_MEASURE_VERITY --------------------- -The FS_IOC_MEASURE_VERITY ioctl retrieves the measurement of a verity -file. The file measurement is a digest that cryptographically -identifies the file contents that are being enforced on reads. +The FS_IOC_MEASURE_VERITY ioctl retrieves the digest of a verity file. +The fs-verity file digest is a cryptographic digest that identifies +the file contents that are being enforced on reads; it is computed via +a Merkle tree and is different from a traditional full-file digest. This ioctl takes in a pointer to a variable-length structure:: @@ -197,7 +198,7 @@ On success, 0 is returned and the kernel fills in the structure as follows: - ``digest_algorithm`` will be the hash algorithm used for the file - measurement. It will match ``fsverity_enable_arg::hash_algorithm``. + digest. It will match ``fsverity_enable_arg::hash_algorithm``. - ``digest_size`` will be the size of the digest in bytes, e.g. 32 for SHA-256. (This can be redundant with ``digest_algorithm``.) - ``digest`` will be the actual bytes of the digest. @@ -257,25 +258,24 @@ non-verity one, with the following exceptions: with EIO (for read()) or SIGBUS (for mmap() reads). - If the sysctl "fs.verity.require_signatures" is set to 1 and the - file's verity measurement is not signed by a key in the fs-verity - keyring, then opening the file will fail. See `Built-in signature - verification`_. + file is not signed by a key in the fs-verity keyring, then opening + the file will fail. See `Built-in signature verification`_. Direct access to the Merkle tree is not supported. Therefore, if a verity file is copied, or is backed up and restored, then it will lose its "verity"-ness. fs-verity is primarily meant for files like executables that are managed by a package manager. -File measurement computation -============================ +File digest computation +======================= This section describes how fs-verity hashes the file contents using a -Merkle tree to produce the "file measurement" which cryptographically -identifies the file contents. This algorithm is the same for all -filesystems that support fs-verity. +Merkle tree to produce the digest which cryptographically identifies +the file contents. This algorithm is the same for all filesystems +that support fs-verity. Userspace only needs to be aware of this algorithm if it needs to -compute the file measurement itself, e.g. in order to sign the file. +compute fs-verity file digests itself, e.g. in order to sign files. .. _fsverity_merkle_tree: @@ -325,26 +325,22 @@ can't a distinguish a large file from a small second file whose data is exactly the top-level hash block of the first file. Ambiguities also arise from the convention of padding to the next block boundary. -To solve this problem, the verity file measurement is actually -computed as a hash of the following structure, which contains the -Merkle tree root hash as well as other fields such as the file size:: +To solve this problem, the fs-verity file digest is actually computed +as a hash of the following structure, which contains the Merkle tree +root hash as well as other fields such as the file size:: struct fsverity_descriptor { __u8 version; /* must be 1 */ __u8 hash_algorithm; /* Merkle tree hash algorithm */ __u8 log_blocksize; /* log2 of size of data and tree blocks */ __u8 salt_size; /* size of salt in bytes; 0 if none */ - __le32 sig_size; /* must be 0 */ + __le32 __reserved_0x04; /* must be 0 */ __le64 data_size; /* size of file the Merkle tree is built over */ __u8 root_hash[64]; /* Merkle tree root hash */ __u8 salt[32]; /* salt prepended to each hashed block */ __u8 __reserved[144]; /* must be 0's */ }; -Note that the ``sig_size`` field must be set to 0 for the purpose of -computing the file measurement, even if a signature was provided (or -will be provided) to `FS_IOC_ENABLE_VERITY`_. - Built-in signature verification =============================== @@ -359,20 +355,20 @@ kernel. Specifically, it adds support for: certificates from being added. 2. `FS_IOC_ENABLE_VERITY`_ accepts a pointer to a PKCS#7 formatted - detached signature in DER format of the file measurement. On - success, this signature is persisted alongside the Merkle tree. + detached signature in DER format of the file's fs-verity digest. + On success, this signature is persisted alongside the Merkle tree. Then, any time the file is opened, the kernel will verify the - file's actual measurement against this signature, using the - certificates in the ".fs-verity" keyring. + file's actual digest against this signature, using the certificates + in the ".fs-verity" keyring. 3. A new sysctl "fs.verity.require_signatures" is made available. When set to 1, the kernel requires that all verity files have a - correctly signed file measurement as described in (2). + correctly signed digest as described in (2). -File measurements must be signed in the following format, which is -similar to the structure used by `FS_IOC_MEASURE_VERITY`_:: +fs-verity file digests must be signed in the following format, which +is similar to the structure used by `FS_IOC_MEASURE_VERITY`_:: - struct fsverity_signed_digest { + struct fsverity_formatted_digest { char magic[8]; /* must be "FSVerity" */ __le16 digest_algorithm; __le16 digest_size; @@ -421,8 +417,8 @@ can only be set by `FS_IOC_ENABLE_VERITY`_, and it cannot be cleared. ext4 also supports encryption, which can be used simultaneously with fs-verity. In this case, the plaintext data is verified rather than -the ciphertext. This is necessary in order to make the file -measurement meaningful, since every file is encrypted differently. +the ciphertext. This is necessary in order to make the fs-verity file +digest meaningful, since every file is encrypted differently. ext4 stores the verity metadata (Merkle tree and fsverity_descriptor) past the end of the file, starting at the first 64K boundary beyond @@ -592,8 +588,8 @@ weren't already directly answered in other parts of this document. :Q: Isn't fs-verity useless because the attacker can just modify the hashes in the Merkle tree, which is stored on-disk? :A: To verify the authenticity of an fs-verity file you must verify - the authenticity of the "file measurement", which is basically the - root hash of the Merkle tree. See `Use cases`_. + the authenticity of the "fs-verity file digest", which + incorporates the root hash of the Merkle tree. See `Use cases`_. :Q: Isn't fs-verity useless because the attacker can just replace a verity file with a non-verity one? diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst index 55a2d9b2ce33..a4c75a28c839 100644 --- a/Documentation/userspace-api/ioctl/ioctl-number.rst +++ b/Documentation/userspace-api/ioctl/ioctl-number.rst @@ -323,6 +323,7 @@ Code Seq# Include File Comments 0xA3 90-9F linux/dtlk.h 0xA4 00-1F uapi/linux/tee.h Generic TEE subsystem +0xA4 00-1F uapi/asm/sgx.h 0xAA 00-3F linux/uapi/linux/userfaultfd.h 0xAB 00-1F linux/nbd.h 0xAC 00-1F linux/raw.h diff --git a/Documentation/x86/index.rst b/Documentation/x86/index.rst index b224d12c880b..e7eb84484ddc 100644 --- a/Documentation/x86/index.rst +++ b/Documentation/x86/index.rst @@ -27,9 +27,10 @@ x86-specific Documentation pti mds microcode - resctrl_ui + resctrl tsx_async_abort usb-legacy-support i386/index x86_64/index sva + sgx diff --git a/Documentation/x86/resctrl_ui.rst b/Documentation/x86/resctrl.rst similarity index 90% rename from Documentation/x86/resctrl_ui.rst rename to Documentation/x86/resctrl.rst index e59b7b93a9b4..71a531061e4e 100644 --- a/Documentation/x86/resctrl_ui.rst +++ b/Documentation/x86/resctrl.rst @@ -1209,3 +1209,96 @@ View the llc occupancy snapshot:: # cat /sys/fs/resctrl/p1/mon_data/mon_L3_00/llc_occupancy 11234000 + +Intel RDT Errata +================ + +Intel MBM Counters May Report System Memory Bandwidth Incorrectly +----------------------------------------------------------------- + +Errata SKX99 for Skylake server and BDF102 for Broadwell server. + +Problem: Intel Memory Bandwidth Monitoring (MBM) counters track metrics +according to the assigned Resource Monitor ID (RMID) for that logical +core. The IA32_QM_CTR register (MSR 0xC8E), used to report these +metrics, may report incorrect system bandwidth for certain RMID values. + +Implication: Due to the errata, system memory bandwidth may not match +what is reported. + +Workaround: MBM total and local readings are corrected according to the +following correction factor table: + ++---------------+---------------+---------------+-----------------+ +|core count |rmid count |rmid threshold |correction factor| ++---------------+---------------+---------------+-----------------+ +|1 |8 |0 |1.000000 | ++---------------+---------------+---------------+-----------------+ +|2 |16 |0 |1.000000 | ++---------------+---------------+---------------+-----------------+ +|3 |24 |15 |0.969650 | ++---------------+---------------+---------------+-----------------+ +|4 |32 |0 |1.000000 | ++---------------+---------------+---------------+-----------------+ +|6 |48 |31 |0.969650 | ++---------------+---------------+---------------+-----------------+ +|7 |56 |47 |1.142857 | ++---------------+---------------+---------------+-----------------+ +|8 |64 |0 |1.000000 | ++---------------+---------------+---------------+-----------------+ +|9 |72 |63 |1.185115 | ++---------------+---------------+---------------+-----------------+ +|10 |80 |63 |1.066553 | ++---------------+---------------+---------------+-----------------+ +|11 |88 |79 |1.454545 | ++---------------+---------------+---------------+-----------------+ +|12 |96 |0 |1.000000 | ++---------------+---------------+---------------+-----------------+ +|13 |104 |95 |1.230769 | ++---------------+---------------+---------------+-----------------+ +|14 |112 |95 |1.142857 | ++---------------+---------------+---------------+-----------------+ +|15 |120 |95 |1.066667 | ++---------------+---------------+---------------+-----------------+ +|16 |128 |0 |1.000000 | ++---------------+---------------+---------------+-----------------+ +|17 |136 |127 |1.254863 | ++---------------+---------------+---------------+-----------------+ +|18 |144 |127 |1.185255 | ++---------------+---------------+---------------+-----------------+ +|19 |152 |0 |1.000000 | ++---------------+---------------+---------------+-----------------+ +|20 |160 |127 |1.066667 | ++---------------+---------------+---------------+-----------------+ +|21 |168 |0 |1.000000 | ++---------------+---------------+---------------+-----------------+ +|22 |176 |159 |1.454334 | ++---------------+---------------+---------------+-----------------+ +|23 |184 |0 |1.000000 | ++---------------+---------------+---------------+-----------------+ +|24 |192 |127 |0.969744 | ++---------------+---------------+---------------+-----------------+ +|25 |200 |191 |1.280246 | ++---------------+---------------+---------------+-----------------+ +|26 |208 |191 |1.230921 | ++---------------+---------------+---------------+-----------------+ +|27 |216 |0 |1.000000 | ++---------------+---------------+---------------+-----------------+ +|28 |224 |191 |1.143118 | ++---------------+---------------+---------------+-----------------+ + +If rmid > rmid threshold, MBM total and local values should be multiplied +by the correction factor. + +See: + +1. Erratum SKX99 in Intel Xeon Processor Scalable Family Specification Update: +http://web.archive.org/web/20200716124958/https://www.intel.com/content/www/us/en/processors/xeon/scalable/xeon-scalable-spec-update.html + +2. Erratum BDF102 in Intel Xeon E5-2600 v4 Processor Product Family Specification Update: +http://web.archive.org/web/20191125200531/https://www.intel.com/content/dam/www/public/us/en/documents/specification-updates/xeon-e5-v4-spec-update.pdf + +3. The errata in Intel Resource Director Technology (Intel RDT) on 2nd Generation Intel Xeon Scalable Processors Reference Manual: +https://software.intel.com/content/www/us/en/develop/articles/intel-resource-director-technology-rdt-reference-manual.html + +for further information. diff --git a/Documentation/x86/sgx.rst b/Documentation/x86/sgx.rst new file mode 100644 index 000000000000..eaee1368b4fd --- /dev/null +++ b/Documentation/x86/sgx.rst @@ -0,0 +1,211 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=============================== +Software Guard eXtensions (SGX) +=============================== + +Overview +======== + +Software Guard eXtensions (SGX) hardware enables for user space applications +to set aside private memory regions of code and data: + +* Privileged (ring-0) ENCLS functions orchestrate the construction of the. + regions. +* Unprivileged (ring-3) ENCLU functions allow an application to enter and + execute inside the regions. + +These memory regions are called enclaves. An enclave can be only entered at a +fixed set of entry points. Each entry point can hold a single hardware thread +at a time. While the enclave is loaded from a regular binary file by using +ENCLS functions, only the threads inside the enclave can access its memory. The +region is denied from outside access by the CPU, and encrypted before it leaves +from LLC. + +The support can be determined by + + ``grep sgx /proc/cpuinfo`` + +SGX must both be supported in the processor and enabled by the BIOS. If SGX +appears to be unsupported on a system which has hardware support, ensure +support is enabled in the BIOS. If a BIOS presents a choice between "Enabled" +and "Software Enabled" modes for SGX, choose "Enabled". + +Enclave Page Cache +================== + +SGX utilizes an *Enclave Page Cache (EPC)* to store pages that are associated +with an enclave. It is contained in a BIOS-reserved region of physical memory. +Unlike pages used for regular memory, pages can only be accessed from outside of +the enclave during enclave construction with special, limited SGX instructions. + +Only a CPU executing inside an enclave can directly access enclave memory. +However, a CPU executing inside an enclave may access normal memory outside the +enclave. + +The kernel manages enclave memory similar to how it treats device memory. + +Enclave Page Types +------------------ + +**SGX Enclave Control Structure (SECS)** + Enclave's address range, attributes and other global data are defined + by this structure. + +**Regular (REG)** + Regular EPC pages contain the code and data of an enclave. + +**Thread Control Structure (TCS)** + Thread Control Structure pages define the entry points to an enclave and + track the execution state of an enclave thread. + +**Version Array (VA)** + Version Array pages contain 512 slots, each of which can contain a version + number for a page evicted from the EPC. + +Enclave Page Cache Map +---------------------- + +The processor tracks EPC pages in a hardware metadata structure called the +*Enclave Page Cache Map (EPCM)*. The EPCM contains an entry for each EPC page +which describes the owning enclave, access rights and page type among the other +things. + +EPCM permissions are separate from the normal page tables. This prevents the +kernel from, for instance, allowing writes to data which an enclave wishes to +remain read-only. EPCM permissions may only impose additional restrictions on +top of normal x86 page permissions. + +For all intents and purposes, the SGX architecture allows the processor to +invalidate all EPCM entries at will. This requires that software be prepared to +handle an EPCM fault at any time. In practice, this can happen on events like +power transitions when the ephemeral key that encrypts enclave memory is lost. + +Application interface +===================== + +Enclave build functions +----------------------- + +In addition to the traditional compiler and linker build process, SGX has a +separate enclave “build” process. Enclaves must be built before they can be +executed (entered). The first step in building an enclave is opening the +**/dev/sgx_enclave** device. Since enclave memory is protected from direct +access, special privileged instructions are Then used to copy data into enclave +pages and establish enclave page permissions. + +.. kernel-doc:: arch/x86/kernel/cpu/sgx/ioctl.c + :functions: sgx_ioc_enclave_create + sgx_ioc_enclave_add_pages + sgx_ioc_enclave_init + sgx_ioc_enclave_provision + +Enclave vDSO +------------ + +Entering an enclave can only be done through SGX-specific EENTER and ERESUME +functions, and is a non-trivial process. Because of the complexity of +transitioning to and from an enclave, enclaves typically utilize a library to +handle the actual transitions. This is roughly analogous to how glibc +implementations are used by most applications to wrap system calls. + +Another crucial characteristic of enclaves is that they can generate exceptions +as part of their normal operation that need to be handled in the enclave or are +unique to SGX. + +Instead of the traditional signal mechanism to handle these exceptions, SGX +can leverage special exception fixup provided by the vDSO. The kernel-provided +vDSO function wraps low-level transitions to/from the enclave like EENTER and +ERESUME. The vDSO function intercepts exceptions that would otherwise generate +a signal and return the fault information directly to its caller. This avoids +the need to juggle signal handlers. + +.. kernel-doc:: arch/x86/include/uapi/asm/sgx.h + :functions: vdso_sgx_enter_enclave_t + +ksgxd +===== + +SGX support includes a kernel thread called *ksgxwapd*. + +EPC sanitization +---------------- + +ksgxd is started when SGX initializes. Enclave memory is typically ready +For use when the processor powers on or resets. However, if SGX has been in +use since the reset, enclave pages may be in an inconsistent state. This might +occur after a crash and kexec() cycle, for instance. At boot, ksgxd +reinitializes all enclave pages so that they can be allocated and re-used. + +The sanitization is done by going through EPC address space and applying the +EREMOVE function to each physical page. Some enclave pages like SECS pages have +hardware dependencies on other pages which prevents EREMOVE from functioning. +Executing two EREMOVE passes removes the dependencies. + +Page reclaimer +-------------- + +Similar to the core kswapd, ksgxd, is responsible for managing the +overcommitment of enclave memory. If the system runs out of enclave memory, +*ksgxwapd* “swaps” enclave memory to normal memory. + +Launch Control +============== + +SGX provides a launch control mechanism. After all enclave pages have been +copied, kernel executes EINIT function, which initializes the enclave. Only after +this the CPU can execute inside the enclave. + +ENIT function takes an RSA-3072 signature of the enclave measurement. The function +checks that the measurement is correct and signature is signed with the key +hashed to the four **IA32_SGXLEPUBKEYHASH{0, 1, 2, 3}** MSRs representing the +SHA256 of a public key. + +Those MSRs can be configured by the BIOS to be either readable or writable. +Linux supports only writable configuration in order to give full control to the +kernel on launch control policy. Before calling EINIT function, the driver sets +the MSRs to match the enclave's signing key. + +Encryption engines +================== + +In order to conceal the enclave data while it is out of the CPU package, the +memory controller has an encryption engine to transparently encrypt and decrypt +enclave memory. + +In CPUs prior to Ice Lake, the Memory Encryption Engine (MEE) is used to +encrypt pages leaving the CPU caches. MEE uses a n-ary Merkle tree with root in +SRAM to maintain integrity of the encrypted data. This provides integrity and +anti-replay protection but does not scale to large memory sizes because the time +required to update the Merkle tree grows logarithmically in relation to the +memory size. + +CPUs starting from Icelake use Total Memory Encryption (TME) in the place of +MEE. TME-based SGX implementations do not have an integrity Merkle tree, which +means integrity and replay-attacks are not mitigated. B, it includes +additional changes to prevent cipher text from being returned and SW memory +aliases from being Created. + +DMA to enclave memory is blocked by range registers on both MEE and TME systems +(SDM section 41.10). + +Usage Models +============ + +Shared Library +-------------- + +Sensitive data and the code that acts on it is partitioned from the application +into a separate library. The library is then linked as a DSO which can be loaded +into an enclave. The application can then make individual function calls into +the enclave through special SGX instructions. A run-time within the enclave is +configured to marshal function parameters into and out of the enclave and to +call the correct library function. + +Application Container +--------------------- + +An application may be loaded into a container enclave which is specially +configured with a library OS and run-time which permits the application to run. +The enclave run-time and library OS work together to execute the application +when a thread enters the enclave. diff --git a/Documentation/x86/topology.rst b/Documentation/x86/topology.rst index e29739904e37..7f58010ea86a 100644 --- a/Documentation/x86/topology.rst +++ b/Documentation/x86/topology.rst @@ -41,6 +41,8 @@ Package Packages contain a number of cores plus shared resources, e.g. DRAM controller, shared caches etc. +Modern systems may also use the term 'Die' for package. + AMD nomenclature for package is 'Node'. Package-related topology information in the kernel: @@ -53,11 +55,18 @@ Package-related topology information in the kernel: The number of dies in a package. This information is retrieved via CPUID. + - cpuinfo_x86.cpu_die_id: + + The physical ID of the die. This information is retrieved via CPUID. + - cpuinfo_x86.phys_proc_id: The physical ID of the package. This information is retrieved via CPUID and deduced from the APIC IDs of the cores in the package. + Modern systems use this value for the socket. There may be multiple + packages within a socket. This value may differ from cpu_die_id. + - cpuinfo_x86.logical_proc_id: The logical ID of the package. As we do not trust BIOSes to enumerate the diff --git a/MAINTAINERS b/MAINTAINERS index 40da849016d5..82944eaadfb5 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2485,7 +2485,7 @@ F: drivers/clk/socfpga/ ARM/SOCFPGA EDAC SUPPORT M: Dinh Nguyen S: Maintained -F: drivers/edac/altera_edac. +F: drivers/edac/altera_edac.[ch] ARM/SPREADTRUM SoC SUPPORT M: Orson Zhai @@ -6370,6 +6370,13 @@ L: linux-edac@vger.kernel.org S: Maintained F: drivers/edac/ie31200_edac.c +EDAC-IGEN6 +M: Tony Luck +R: Qiuxu Zhuo +L: linux-edac@vger.kernel.org +S: Maintained +F: drivers/edac/igen6_edac.c + EDAC-MPC85XX M: Johannes Thumshirn L: linux-edac@vger.kernel.org @@ -6419,7 +6426,7 @@ EDAC-SKYLAKE M: Tony Luck L: linux-edac@vger.kernel.org S: Maintained -F: drivers/edac/skx_*.c +F: drivers/edac/skx_*.[ch] EDAC-TI M: Tero Kristo @@ -8016,7 +8023,7 @@ F: drivers/staging/hikey9xx/ HISILICON TRUE RANDOM NUMBER GENERATOR V2 SUPPORT M: Zaibo Xu S: Maintained -F: drivers/char/hw_random/hisi-trng-v2.c +F: drivers/crypto/hisilicon/trng/trng.c HISILICON V3XX SPI NOR FLASH Controller Driver M: John Garry @@ -8982,13 +8989,23 @@ M: Deepak Saxena S: Maintained F: drivers/char/hw_random/ixp4xx-rng.c -INTEL KEEMBAY DRM DRIVER +INTEL KEEM BAY DRM DRIVER M: Anitha Chrisanthus M: Edmund Dea S: Maintained F: Documentation/devicetree/bindings/display/intel,kmb_display.yaml F: drivers/gpu/drm/kmb/ +INTEL KEEM BAY OCS AES/SM4 CRYPTO DRIVER +M: Daniele Alessandrelli +S: Maintained +F: Documentation/devicetree/bindings/crypto/intel,keembay-ocs-aes.yaml +F: drivers/crypto/keembay/Kconfig +F: drivers/crypto/keembay/Makefile +F: drivers/crypto/keembay/keembay-ocs-aes-core.c +F: drivers/crypto/keembay/ocs-aes.c +F: drivers/crypto/keembay/ocs-aes.h + INTEL MANAGEMENT ENGINE (mei) M: Tomas Winkler L: linux-kernel@vger.kernel.org @@ -9157,6 +9174,19 @@ F: Documentation/x86/intel_txt.rst F: arch/x86/kernel/tboot.c F: include/linux/tboot.h +INTEL SGX +M: Jarkko Sakkinen +L: linux-sgx@vger.kernel.org +S: Supported +Q: https://patchwork.kernel.org/project/intel-sgx/list/ +T: git git://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-sgx.git +F: Documentation/x86/sgx.rst +F: arch/x86/entry/vdso/vsgx.S +F: arch/x86/include/uapi/asm/sgx.h +F: arch/x86/kernel/cpu/sgx/* +F: tools/testing/selftests/sgx/* +K: \bSGX_ + INTERCONNECT API M: Georgi Djakov L: linux-pm@vger.kernel.org @@ -10403,6 +10433,8 @@ L: linux-m68k@lists.linux-m68k.org S: Maintained W: http://www.mac.linux-m68k.org/ F: arch/m68k/mac/ +F: drivers/macintosh/adb-iop.c +F: drivers/macintosh/via-macii.c M68K ON HP9000/300 M: Philip Blundell @@ -15888,13 +15920,14 @@ F: include/linux/sfp.h K: phylink\.h|struct\s+phylink|\.phylink|>phylink_|phylink_(autoneg|clear|connect|create|destroy|disconnect|ethtool|helper|mac|mii|of|set|start|stop|test|validate) SGI GRU DRIVER -M: Dimitri Sivanich +M: Dimitri Sivanich S: Maintained F: drivers/misc/sgi-gru/ SGI XP/XPC/XPNET DRIVER -M: Cliff Whickman M: Robin Holt +M: Steve Wahl +R: Mike Travis S: Maintained F: drivers/misc/sgi-xp/ @@ -18437,6 +18470,12 @@ F: include/uapi/linux/uuid.h F: lib/test_uuid.c F: lib/uuid.c +UV SYSFS DRIVER +M: Justin Ernst +L: platform-driver-x86@vger.kernel.org +S: Maintained +F: drivers/platform/x86/uv_sysfs.c + UVESAFB DRIVER M: Michal Januszewski L: linux-fbdev@vger.kernel.org @@ -19143,6 +19182,7 @@ F: arch/x86/platform X86 PLATFORM UV HPE SUPERDOME FLEX M: Steve Wahl +R: Mike Travis R: Dimitri Sivanich R: Russ Anderson S: Supported diff --git a/arch/arm/boot/dts/aspeed-g6.dtsi b/arch/arm/boot/dts/aspeed-g6.dtsi index b58220a49cbd..74367ee96f20 100644 --- a/arch/arm/boot/dts/aspeed-g6.dtsi +++ b/arch/arm/boot/dts/aspeed-g6.dtsi @@ -69,6 +69,12 @@ always-on; }; + edac: sdram@1e6e0000 { + compatible = "aspeed,ast2600-sdram-edac", "syscon"; + reg = <0x1e6e0000 0x174>; + interrupts = ; + }; + ahb { compatible = "simple-bus"; #address-cells = <1>; diff --git a/arch/arm/crypto/aes-ce-core.S b/arch/arm/crypto/aes-ce-core.S index 4d1707388d94..312428d83eed 100644 --- a/arch/arm/crypto/aes-ce-core.S +++ b/arch/arm/crypto/aes-ce-core.S @@ -386,20 +386,32 @@ ENTRY(ce_aes_ctr_encrypt) .Lctrloop4x: subs r4, r4, #4 bmi .Lctr1x - add r6, r6, #1 + + /* + * NOTE: the sequence below has been carefully tweaked to avoid + * a silicon erratum that exists in Cortex-A57 (#1742098) and + * Cortex-A72 (#1655431) cores, where AESE/AESMC instruction pairs + * may produce an incorrect result if they take their input from a + * register of which a single 32-bit lane has been updated the last + * time it was modified. To work around this, the lanes of registers + * q0-q3 below are not manipulated individually, and the different + * counter values are prepared by successive manipulations of q7. + */ + add ip, r6, #1 vmov q0, q7 + rev ip, ip + add lr, r6, #2 + vmov s31, ip @ set lane 3 of q1 via q7 + add ip, r6, #3 + rev lr, lr vmov q1, q7 - rev ip, r6 - add r6, r6, #1 + vmov s31, lr @ set lane 3 of q2 via q7 + rev ip, ip vmov q2, q7 - vmov s7, ip - rev ip, r6 - add r6, r6, #1 + vmov s31, ip @ set lane 3 of q3 via q7 + add r6, r6, #4 vmov q3, q7 - vmov s11, ip - rev ip, r6 - add r6, r6, #1 - vmov s15, ip + vld1.8 {q4-q5}, [r1]! vld1.8 {q6}, [r1]! vld1.8 {q15}, [r1]! diff --git a/arch/arm/crypto/aes-neonbs-glue.c b/arch/arm/crypto/aes-neonbs-glue.c index bda8bf17631e..f70af1d0514b 100644 --- a/arch/arm/crypto/aes-neonbs-glue.c +++ b/arch/arm/crypto/aes-neonbs-glue.c @@ -19,7 +19,7 @@ MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL v2"); MODULE_ALIAS_CRYPTO("ecb(aes)"); -MODULE_ALIAS_CRYPTO("cbc(aes)"); +MODULE_ALIAS_CRYPTO("cbc(aes)-all"); MODULE_ALIAS_CRYPTO("ctr(aes)"); MODULE_ALIAS_CRYPTO("xts(aes)"); @@ -191,7 +191,8 @@ static int cbc_init(struct crypto_skcipher *tfm) struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); unsigned int reqsize; - ctx->enc_tfm = crypto_alloc_skcipher("cbc(aes)", 0, CRYPTO_ALG_ASYNC); + ctx->enc_tfm = crypto_alloc_skcipher("cbc(aes)", 0, CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(ctx->enc_tfm)) return PTR_ERR(ctx->enc_tfm); @@ -441,7 +442,8 @@ static struct skcipher_alg aes_algs[] = { { .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct aesbs_cbc_ctx), .base.cra_module = THIS_MODULE, - .base.cra_flags = CRYPTO_ALG_INTERNAL, + .base.cra_flags = CRYPTO_ALG_INTERNAL | + CRYPTO_ALG_NEED_FALLBACK, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, diff --git a/arch/arm/crypto/chacha-glue.c b/arch/arm/crypto/chacha-glue.c index 59da6c0b63b6..7b5cf8430c6d 100644 --- a/arch/arm/crypto/chacha-glue.c +++ b/arch/arm/crypto/chacha-glue.c @@ -23,7 +23,7 @@ asmlinkage void chacha_block_xor_neon(const u32 *state, u8 *dst, const u8 *src, int nrounds); asmlinkage void chacha_4block_xor_neon(const u32 *state, u8 *dst, const u8 *src, - int nrounds); + int nrounds, unsigned int nbytes); asmlinkage void hchacha_block_arm(const u32 *state, u32 *out, int nrounds); asmlinkage void hchacha_block_neon(const u32 *state, u32 *out, int nrounds); @@ -42,24 +42,24 @@ static void chacha_doneon(u32 *state, u8 *dst, const u8 *src, { u8 buf[CHACHA_BLOCK_SIZE]; - while (bytes >= CHACHA_BLOCK_SIZE * 4) { - chacha_4block_xor_neon(state, dst, src, nrounds); - bytes -= CHACHA_BLOCK_SIZE * 4; - src += CHACHA_BLOCK_SIZE * 4; - dst += CHACHA_BLOCK_SIZE * 4; - state[12] += 4; - } - while (bytes >= CHACHA_BLOCK_SIZE) { - chacha_block_xor_neon(state, dst, src, nrounds); - bytes -= CHACHA_BLOCK_SIZE; - src += CHACHA_BLOCK_SIZE; - dst += CHACHA_BLOCK_SIZE; - state[12]++; + while (bytes > CHACHA_BLOCK_SIZE) { + unsigned int l = min(bytes, CHACHA_BLOCK_SIZE * 4U); + + chacha_4block_xor_neon(state, dst, src, nrounds, l); + bytes -= l; + src += l; + dst += l; + state[12] += DIV_ROUND_UP(l, CHACHA_BLOCK_SIZE); } if (bytes) { - memcpy(buf, src, bytes); - chacha_block_xor_neon(state, buf, buf, nrounds); - memcpy(dst, buf, bytes); + const u8 *s = src; + u8 *d = dst; + + if (bytes != CHACHA_BLOCK_SIZE) + s = d = memcpy(buf, src, bytes); + chacha_block_xor_neon(state, d, s, nrounds); + if (d != dst) + memcpy(dst, buf, bytes); } } diff --git a/arch/arm/crypto/chacha-neon-core.S b/arch/arm/crypto/chacha-neon-core.S index eb22926d4912..13d12f672656 100644 --- a/arch/arm/crypto/chacha-neon-core.S +++ b/arch/arm/crypto/chacha-neon-core.S @@ -47,6 +47,7 @@ */ #include +#include .text .fpu neon @@ -205,7 +206,7 @@ ENDPROC(hchacha_block_neon) .align 5 ENTRY(chacha_4block_xor_neon) - push {r4-r5} + push {r4, lr} mov r4, sp // preserve the stack pointer sub ip, sp, #0x20 // allocate a 32 byte buffer bic ip, ip, #0x1f // aligned to 32 bytes @@ -229,10 +230,10 @@ ENTRY(chacha_4block_xor_neon) vld1.32 {q0-q1}, [r0] vld1.32 {q2-q3}, [ip] - adr r5, .Lctrinc + adr lr, .Lctrinc vdup.32 q15, d7[1] vdup.32 q14, d7[0] - vld1.32 {q4}, [r5, :128] + vld1.32 {q4}, [lr, :128] vdup.32 q13, d6[1] vdup.32 q12, d6[0] vdup.32 q11, d5[1] @@ -455,7 +456,7 @@ ENTRY(chacha_4block_xor_neon) // Re-interleave the words in the first two rows of each block (x0..7). // Also add the counter values 0-3 to x12[0-3]. - vld1.32 {q8}, [r5, :128] // load counter values 0-3 + vld1.32 {q8}, [lr, :128] // load counter values 0-3 vzip.32 q0, q1 // => (0 1 0 1) (0 1 0 1) vzip.32 q2, q3 // => (2 3 2 3) (2 3 2 3) vzip.32 q4, q5 // => (4 5 4 5) (4 5 4 5) @@ -493,6 +494,8 @@ ENTRY(chacha_4block_xor_neon) // Re-interleave the words in the last two rows of each block (x8..15). vld1.32 {q8-q9}, [sp, :256] + mov sp, r4 // restore original stack pointer + ldr r4, [r4, #8] // load number of bytes vzip.32 q12, q13 // => (12 13 12 13) (12 13 12 13) vzip.32 q14, q15 // => (14 15 14 15) (14 15 14 15) vzip.32 q8, q9 // => (8 9 8 9) (8 9 8 9) @@ -520,41 +523,121 @@ ENTRY(chacha_4block_xor_neon) // XOR the rest of the data with the keystream vld1.8 {q0-q1}, [r2]! + subs r4, r4, #96 veor q0, q0, q8 veor q1, q1, q12 + ble .Lle96 vst1.8 {q0-q1}, [r1]! vld1.8 {q0-q1}, [r2]! + subs r4, r4, #32 veor q0, q0, q2 veor q1, q1, q6 + ble .Lle128 vst1.8 {q0-q1}, [r1]! vld1.8 {q0-q1}, [r2]! + subs r4, r4, #32 veor q0, q0, q10 veor q1, q1, q14 + ble .Lle160 vst1.8 {q0-q1}, [r1]! vld1.8 {q0-q1}, [r2]! + subs r4, r4, #32 veor q0, q0, q4 veor q1, q1, q5 + ble .Lle192 vst1.8 {q0-q1}, [r1]! vld1.8 {q0-q1}, [r2]! + subs r4, r4, #32 veor q0, q0, q9 veor q1, q1, q13 + ble .Lle224 vst1.8 {q0-q1}, [r1]! vld1.8 {q0-q1}, [r2]! + subs r4, r4, #32 veor q0, q0, q3 veor q1, q1, q7 + blt .Llt256 +.Lout: vst1.8 {q0-q1}, [r1]! vld1.8 {q0-q1}, [r2] - mov sp, r4 // restore original stack pointer veor q0, q0, q11 veor q1, q1, q15 vst1.8 {q0-q1}, [r1] - pop {r4-r5} - bx lr + pop {r4, pc} + +.Lle192: + vmov q4, q9 + vmov q5, q13 + +.Lle160: + // nothing to do + +.Lfinalblock: + // Process the final block if processing less than 4 full blocks. + // Entered with 32 bytes of ChaCha cipher stream in q4-q5, and the + // previous 32 byte output block that still needs to be written at + // [r1] in q0-q1. + beq .Lfullblock + +.Lpartialblock: + adr lr, .Lpermute + 32 + add r2, r2, r4 + add lr, lr, r4 + add r4, r4, r1 + + vld1.8 {q2-q3}, [lr] + vld1.8 {q6-q7}, [r2] + + add r4, r4, #32 + + vtbl.8 d4, {q4-q5}, d4 + vtbl.8 d5, {q4-q5}, d5 + vtbl.8 d6, {q4-q5}, d6 + vtbl.8 d7, {q4-q5}, d7 + + veor q6, q6, q2 + veor q7, q7, q3 + + vst1.8 {q6-q7}, [r4] // overlapping stores + vst1.8 {q0-q1}, [r1] + pop {r4, pc} + +.Lfullblock: + vmov q11, q4 + vmov q15, q5 + b .Lout +.Lle96: + vmov q4, q2 + vmov q5, q6 + b .Lfinalblock +.Lle128: + vmov q4, q10 + vmov q5, q14 + b .Lfinalblock +.Lle224: + vmov q4, q3 + vmov q5, q7 + b .Lfinalblock +.Llt256: + vmov q4, q11 + vmov q5, q15 + b .Lpartialblock ENDPROC(chacha_4block_xor_neon) + + .align L1_CACHE_SHIFT +.Lpermute: + .byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + .byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f + .byte 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17 + .byte 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f + .byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + .byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f + .byte 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17 + .byte 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f diff --git a/arch/arm/crypto/sha1-ce-glue.c b/arch/arm/crypto/sha1-ce-glue.c index e79b1fb4b4dc..de9100c67b37 100644 --- a/arch/arm/crypto/sha1-ce-glue.c +++ b/arch/arm/crypto/sha1-ce-glue.c @@ -7,7 +7,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/arm/crypto/sha1.h b/arch/arm/crypto/sha1.h index 758db3e9ff0a..b1b7e21da2c3 100644 --- a/arch/arm/crypto/sha1.h +++ b/arch/arm/crypto/sha1.h @@ -3,7 +3,7 @@ #define ASM_ARM_CRYPTO_SHA1_H #include -#include +#include extern int sha1_update_arm(struct shash_desc *desc, const u8 *data, unsigned int len); diff --git a/arch/arm/crypto/sha1_glue.c b/arch/arm/crypto/sha1_glue.c index 4e954b3f7ecd..6c2b849e459d 100644 --- a/arch/arm/crypto/sha1_glue.c +++ b/arch/arm/crypto/sha1_glue.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/arm/crypto/sha1_neon_glue.c b/arch/arm/crypto/sha1_neon_glue.c index 0071e5e4411a..cfe36ae0f3f5 100644 --- a/arch/arm/crypto/sha1_neon_glue.c +++ b/arch/arm/crypto/sha1_neon_glue.c @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm/crypto/sha2-ce-glue.c b/arch/arm/crypto/sha2-ce-glue.c index 87f0b62386c6..c62ce89dd3e0 100644 --- a/arch/arm/crypto/sha2-ce-glue.c +++ b/arch/arm/crypto/sha2-ce-glue.c @@ -7,7 +7,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/arm/crypto/sha256_glue.c b/arch/arm/crypto/sha256_glue.c index b8a4f79020cf..433ee4ddce6c 100644 --- a/arch/arm/crypto/sha256_glue.c +++ b/arch/arm/crypto/sha256_glue.c @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm/crypto/sha256_neon_glue.c b/arch/arm/crypto/sha256_neon_glue.c index 79820b9e2541..701706262ef3 100644 --- a/arch/arm/crypto/sha256_neon_glue.c +++ b/arch/arm/crypto/sha256_neon_glue.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm/crypto/sha512-glue.c b/arch/arm/crypto/sha512-glue.c index 8775aa42bbbe..0635a65aa488 100644 --- a/arch/arm/crypto/sha512-glue.c +++ b/arch/arm/crypto/sha512-glue.c @@ -6,7 +6,7 @@ */ #include -#include +#include #include #include #include diff --git a/arch/arm/crypto/sha512-neon-glue.c b/arch/arm/crypto/sha512-neon-glue.c index 96cb94403540..c879ad32db51 100644 --- a/arch/arm/crypto/sha512-neon-glue.c +++ b/arch/arm/crypto/sha512-neon-glue.c @@ -7,7 +7,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index 5cfe3cf6f2ac..5e7d86cf5dfa 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -1082,6 +1082,7 @@ CONFIG_CRYPTO_DEV_CCREE=m CONFIG_CRYPTO_DEV_HISI_SEC2=m CONFIG_CRYPTO_DEV_HISI_ZIP=m CONFIG_CRYPTO_DEV_HISI_HPRE=m +CONFIG_CRYPTO_DEV_HISI_TRNG=m CONFIG_CMA_SIZE_MBYTES=32 CONFIG_PRINTK_TIME=y CONFIG_DEBUG_INFO=y diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c index 395bbf64b2ab..34b8a89197be 100644 --- a/arch/arm64/crypto/aes-glue.c +++ b/arch/arm64/crypto/aes-glue.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm64/crypto/chacha-neon-core.S b/arch/arm64/crypto/chacha-neon-core.S index e90386a7db8e..b70ac76f2610 100644 --- a/arch/arm64/crypto/chacha-neon-core.S +++ b/arch/arm64/crypto/chacha-neon-core.S @@ -195,7 +195,6 @@ SYM_FUNC_START(chacha_4block_xor_neon) adr_l x10, .Lpermute and x5, x4, #63 add x10, x10, x5 - add x11, x10, #64 // // This function encrypts four consecutive ChaCha blocks by loading @@ -645,11 +644,11 @@ CPU_BE( rev a15, a15 ) zip2 v31.4s, v14.4s, v15.4s eor a15, a15, w9 - mov x3, #64 + add x3, x2, x4 + sub x3, x3, #128 // start of last block + subs x5, x4, #128 - add x6, x5, x2 - csel x3, x3, xzr, ge - csel x2, x2, x6, ge + csel x2, x2, x3, ge // interleave 64-bit words in state n, n+2 zip1 v0.2d, v16.2d, v18.2d @@ -658,13 +657,10 @@ CPU_BE( rev a15, a15 ) zip1 v8.2d, v17.2d, v19.2d zip2 v12.2d, v17.2d, v19.2d stp a2, a3, [x1, #-56] - ld1 {v16.16b-v19.16b}, [x2], x3 subs x6, x4, #192 - ccmp x3, xzr, #4, lt - add x7, x6, x2 - csel x3, x3, xzr, eq - csel x2, x2, x7, eq + ld1 {v16.16b-v19.16b}, [x2], #64 + csel x2, x2, x3, ge zip1 v1.2d, v20.2d, v22.2d zip2 v5.2d, v20.2d, v22.2d @@ -672,13 +668,10 @@ CPU_BE( rev a15, a15 ) zip1 v9.2d, v21.2d, v23.2d zip2 v13.2d, v21.2d, v23.2d stp a6, a7, [x1, #-40] - ld1 {v20.16b-v23.16b}, [x2], x3 subs x7, x4, #256 - ccmp x3, xzr, #4, lt - add x8, x7, x2 - csel x3, x3, xzr, eq - csel x2, x2, x8, eq + ld1 {v20.16b-v23.16b}, [x2], #64 + csel x2, x2, x3, ge zip1 v2.2d, v24.2d, v26.2d zip2 v6.2d, v24.2d, v26.2d @@ -686,12 +679,10 @@ CPU_BE( rev a15, a15 ) zip1 v10.2d, v25.2d, v27.2d zip2 v14.2d, v25.2d, v27.2d stp a10, a11, [x1, #-24] - ld1 {v24.16b-v27.16b}, [x2], x3 subs x8, x4, #320 - ccmp x3, xzr, #4, lt - add x9, x8, x2 - csel x2, x2, x9, eq + ld1 {v24.16b-v27.16b}, [x2], #64 + csel x2, x2, x3, ge zip1 v3.2d, v28.2d, v30.2d zip2 v7.2d, v28.2d, v30.2d @@ -699,151 +690,105 @@ CPU_BE( rev a15, a15 ) zip1 v11.2d, v29.2d, v31.2d zip2 v15.2d, v29.2d, v31.2d stp a14, a15, [x1, #-8] + + tbnz x5, #63, .Lt128 ld1 {v28.16b-v31.16b}, [x2] // xor with corresponding input, write to output - tbnz x5, #63, 0f eor v16.16b, v16.16b, v0.16b eor v17.16b, v17.16b, v1.16b eor v18.16b, v18.16b, v2.16b eor v19.16b, v19.16b, v3.16b - st1 {v16.16b-v19.16b}, [x1], #64 - cbz x5, .Lout - tbnz x6, #63, 1f + tbnz x6, #63, .Lt192 + eor v20.16b, v20.16b, v4.16b eor v21.16b, v21.16b, v5.16b eor v22.16b, v22.16b, v6.16b eor v23.16b, v23.16b, v7.16b - st1 {v20.16b-v23.16b}, [x1], #64 - cbz x6, .Lout - tbnz x7, #63, 2f + st1 {v16.16b-v19.16b}, [x1], #64 + tbnz x7, #63, .Lt256 + eor v24.16b, v24.16b, v8.16b eor v25.16b, v25.16b, v9.16b eor v26.16b, v26.16b, v10.16b eor v27.16b, v27.16b, v11.16b - st1 {v24.16b-v27.16b}, [x1], #64 - cbz x7, .Lout - tbnz x8, #63, 3f + st1 {v20.16b-v23.16b}, [x1], #64 + tbnz x8, #63, .Lt320 + eor v28.16b, v28.16b, v12.16b eor v29.16b, v29.16b, v13.16b eor v30.16b, v30.16b, v14.16b eor v31.16b, v31.16b, v15.16b + + st1 {v24.16b-v27.16b}, [x1], #64 st1 {v28.16b-v31.16b}, [x1] .Lout: frame_pop ret - // fewer than 128 bytes of in/output -0: ld1 {v8.16b}, [x10] - ld1 {v9.16b}, [x11] - movi v10.16b, #16 - sub x2, x1, #64 - add x1, x1, x5 - ld1 {v16.16b-v19.16b}, [x2] - tbl v4.16b, {v0.16b-v3.16b}, v8.16b - tbx v20.16b, {v16.16b-v19.16b}, v9.16b - add v8.16b, v8.16b, v10.16b - add v9.16b, v9.16b, v10.16b - tbl v5.16b, {v0.16b-v3.16b}, v8.16b - tbx v21.16b, {v16.16b-v19.16b}, v9.16b - add v8.16b, v8.16b, v10.16b - add v9.16b, v9.16b, v10.16b - tbl v6.16b, {v0.16b-v3.16b}, v8.16b - tbx v22.16b, {v16.16b-v19.16b}, v9.16b - add v8.16b, v8.16b, v10.16b - add v9.16b, v9.16b, v10.16b - tbl v7.16b, {v0.16b-v3.16b}, v8.16b - tbx v23.16b, {v16.16b-v19.16b}, v9.16b - - eor v20.16b, v20.16b, v4.16b - eor v21.16b, v21.16b, v5.16b - eor v22.16b, v22.16b, v6.16b - eor v23.16b, v23.16b, v7.16b - st1 {v20.16b-v23.16b}, [x1] - b .Lout - // fewer than 192 bytes of in/output -1: ld1 {v8.16b}, [x10] - ld1 {v9.16b}, [x11] - movi v10.16b, #16 - add x1, x1, x6 - tbl v0.16b, {v4.16b-v7.16b}, v8.16b - tbx v20.16b, {v16.16b-v19.16b}, v9.16b - add v8.16b, v8.16b, v10.16b - add v9.16b, v9.16b, v10.16b - tbl v1.16b, {v4.16b-v7.16b}, v8.16b - tbx v21.16b, {v16.16b-v19.16b}, v9.16b - add v8.16b, v8.16b, v10.16b - add v9.16b, v9.16b, v10.16b - tbl v2.16b, {v4.16b-v7.16b}, v8.16b - tbx v22.16b, {v16.16b-v19.16b}, v9.16b - add v8.16b, v8.16b, v10.16b - add v9.16b, v9.16b, v10.16b - tbl v3.16b, {v4.16b-v7.16b}, v8.16b - tbx v23.16b, {v16.16b-v19.16b}, v9.16b +.Lt192: cbz x5, 1f // exactly 128 bytes? + ld1 {v28.16b-v31.16b}, [x10] + add x5, x5, x1 + tbl v28.16b, {v4.16b-v7.16b}, v28.16b + tbl v29.16b, {v4.16b-v7.16b}, v29.16b + tbl v30.16b, {v4.16b-v7.16b}, v30.16b + tbl v31.16b, {v4.16b-v7.16b}, v31.16b - eor v20.16b, v20.16b, v0.16b - eor v21.16b, v21.16b, v1.16b - eor v22.16b, v22.16b, v2.16b - eor v23.16b, v23.16b, v3.16b - st1 {v20.16b-v23.16b}, [x1] +0: eor v20.16b, v20.16b, v28.16b + eor v21.16b, v21.16b, v29.16b + eor v22.16b, v22.16b, v30.16b + eor v23.16b, v23.16b, v31.16b + st1 {v20.16b-v23.16b}, [x5] // overlapping stores +1: st1 {v16.16b-v19.16b}, [x1] b .Lout + // fewer than 128 bytes of in/output +.Lt128: ld1 {v28.16b-v31.16b}, [x10] + add x5, x5, x1 + sub x1, x1, #64 + tbl v28.16b, {v0.16b-v3.16b}, v28.16b + tbl v29.16b, {v0.16b-v3.16b}, v29.16b + tbl v30.16b, {v0.16b-v3.16b}, v30.16b + tbl v31.16b, {v0.16b-v3.16b}, v31.16b + ld1 {v16.16b-v19.16b}, [x1] // reload first output block + b 0b + // fewer than 256 bytes of in/output -2: ld1 {v4.16b}, [x10] - ld1 {v5.16b}, [x11] - movi v6.16b, #16 - add x1, x1, x7 +.Lt256: cbz x6, 2f // exactly 192 bytes? + ld1 {v4.16b-v7.16b}, [x10] + add x6, x6, x1 tbl v0.16b, {v8.16b-v11.16b}, v4.16b - tbx v24.16b, {v20.16b-v23.16b}, v5.16b - add v4.16b, v4.16b, v6.16b - add v5.16b, v5.16b, v6.16b - tbl v1.16b, {v8.16b-v11.16b}, v4.16b - tbx v25.16b, {v20.16b-v23.16b}, v5.16b - add v4.16b, v4.16b, v6.16b - add v5.16b, v5.16b, v6.16b - tbl v2.16b, {v8.16b-v11.16b}, v4.16b - tbx v26.16b, {v20.16b-v23.16b}, v5.16b - add v4.16b, v4.16b, v6.16b - add v5.16b, v5.16b, v6.16b - tbl v3.16b, {v8.16b-v11.16b}, v4.16b - tbx v27.16b, {v20.16b-v23.16b}, v5.16b - - eor v24.16b, v24.16b, v0.16b - eor v25.16b, v25.16b, v1.16b - eor v26.16b, v26.16b, v2.16b - eor v27.16b, v27.16b, v3.16b - st1 {v24.16b-v27.16b}, [x1] - b .Lout - - // fewer than 320 bytes of in/output -3: ld1 {v4.16b}, [x10] - ld1 {v5.16b}, [x11] - movi v6.16b, #16 - add x1, x1, x8 - tbl v0.16b, {v12.16b-v15.16b}, v4.16b - tbx v28.16b, {v24.16b-v27.16b}, v5.16b - add v4.16b, v4.16b, v6.16b - add v5.16b, v5.16b, v6.16b - tbl v1.16b, {v12.16b-v15.16b}, v4.16b - tbx v29.16b, {v24.16b-v27.16b}, v5.16b - add v4.16b, v4.16b, v6.16b - add v5.16b, v5.16b, v6.16b - tbl v2.16b, {v12.16b-v15.16b}, v4.16b - tbx v30.16b, {v24.16b-v27.16b}, v5.16b - add v4.16b, v4.16b, v6.16b - add v5.16b, v5.16b, v6.16b - tbl v3.16b, {v12.16b-v15.16b}, v4.16b - tbx v31.16b, {v24.16b-v27.16b}, v5.16b + tbl v1.16b, {v8.16b-v11.16b}, v5.16b + tbl v2.16b, {v8.16b-v11.16b}, v6.16b + tbl v3.16b, {v8.16b-v11.16b}, v7.16b eor v28.16b, v28.16b, v0.16b eor v29.16b, v29.16b, v1.16b eor v30.16b, v30.16b, v2.16b eor v31.16b, v31.16b, v3.16b - st1 {v28.16b-v31.16b}, [x1] + st1 {v28.16b-v31.16b}, [x6] // overlapping stores +2: st1 {v20.16b-v23.16b}, [x1] + b .Lout + + // fewer than 320 bytes of in/output +.Lt320: cbz x7, 3f // exactly 256 bytes? + ld1 {v4.16b-v7.16b}, [x10] + add x7, x7, x1 + tbl v0.16b, {v12.16b-v15.16b}, v4.16b + tbl v1.16b, {v12.16b-v15.16b}, v5.16b + tbl v2.16b, {v12.16b-v15.16b}, v6.16b + tbl v3.16b, {v12.16b-v15.16b}, v7.16b + + eor v28.16b, v28.16b, v0.16b + eor v29.16b, v29.16b, v1.16b + eor v30.16b, v30.16b, v2.16b + eor v31.16b, v31.16b, v3.16b + st1 {v28.16b-v31.16b}, [x7] // overlapping stores +3: st1 {v24.16b-v27.16b}, [x1] b .Lout SYM_FUNC_END(chacha_4block_xor_neon) @@ -851,7 +796,7 @@ SYM_FUNC_END(chacha_4block_xor_neon) .align L1_CACHE_SHIFT .Lpermute: .set .Li, 0 - .rept 192 + .rept 128 .byte (.Li - 64) .set .Li, .Li + 1 .endr diff --git a/arch/arm64/crypto/ghash-ce-core.S b/arch/arm64/crypto/ghash-ce-core.S index 6b958dcdf136..7868330dd54e 100644 --- a/arch/arm64/crypto/ghash-ce-core.S +++ b/arch/arm64/crypto/ghash-ce-core.S @@ -544,7 +544,22 @@ CPU_LE( rev w8, w8 ) ext XL.16b, XL.16b, XL.16b, #8 rev64 XL.16b, XL.16b eor XL.16b, XL.16b, KS0.16b + + .if \enc == 1 st1 {XL.16b}, [x10] // store tag + .else + ldp x11, x12, [sp, #40] // load tag pointer and authsize + adr_l x17, .Lpermute_table + ld1 {KS0.16b}, [x11] // load supplied tag + add x17, x17, x12 + ld1 {KS1.16b}, [x17] // load permute vector + + cmeq XL.16b, XL.16b, KS0.16b // compare tags + mvn XL.16b, XL.16b // -1 for fail, 0 for pass + tbl XL.16b, {XL.16b}, KS1.16b // keep authsize bytes only + sminv b0, XL.16b // signed minimum across XL + smov w0, v0.b[0] // return b0 + .endif 4: ldp x29, x30, [sp], #32 ret diff --git a/arch/arm64/crypto/ghash-ce-glue.c b/arch/arm64/crypto/ghash-ce-glue.c index 8536008e3e35..720cd3a58da3 100644 --- a/arch/arm64/crypto/ghash-ce-glue.c +++ b/arch/arm64/crypto/ghash-ce-glue.c @@ -55,10 +55,10 @@ asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src, asmlinkage void pmull_gcm_encrypt(int bytes, u8 dst[], const u8 src[], u64 const h[][2], u64 dg[], u8 ctr[], u32 const rk[], int rounds, u8 tag[]); - -asmlinkage void pmull_gcm_decrypt(int bytes, u8 dst[], const u8 src[], - u64 const h[][2], u64 dg[], u8 ctr[], - u32 const rk[], int rounds, u8 tag[]); +asmlinkage int pmull_gcm_decrypt(int bytes, u8 dst[], const u8 src[], + u64 const h[][2], u64 dg[], u8 ctr[], + u32 const rk[], int rounds, const u8 l[], + const u8 tag[], u64 authsize); static int ghash_init(struct shash_desc *desc) { @@ -168,7 +168,7 @@ static int ghash_final(struct shash_desc *desc, u8 *dst) put_unaligned_be64(ctx->digest[1], dst); put_unaligned_be64(ctx->digest[0], dst + 8); - *ctx = (struct ghash_desc_ctx){}; + memzero_explicit(ctx, sizeof(*ctx)); return 0; } @@ -458,6 +458,7 @@ static int gcm_decrypt(struct aead_request *req) unsigned int authsize = crypto_aead_authsize(aead); int nrounds = num_rounds(&ctx->aes_key); struct skcipher_walk walk; + u8 otag[AES_BLOCK_SIZE]; u8 buf[AES_BLOCK_SIZE]; u8 iv[AES_BLOCK_SIZE]; u64 dg[2] = {}; @@ -474,9 +475,15 @@ static int gcm_decrypt(struct aead_request *req) memcpy(iv, req->iv, GCM_IV_SIZE); put_unaligned_be32(2, iv + GCM_IV_SIZE); + scatterwalk_map_and_copy(otag, req->src, + req->assoclen + req->cryptlen - authsize, + authsize, 0); + err = skcipher_walk_aead_decrypt(&walk, req, false); if (likely(crypto_simd_usable())) { + int ret; + do { const u8 *src = walk.src.virt.addr; u8 *dst = walk.dst.virt.addr; @@ -493,9 +500,10 @@ static int gcm_decrypt(struct aead_request *req) } kernel_neon_begin(); - pmull_gcm_decrypt(nbytes, dst, src, ctx->ghash_key.h, - dg, iv, ctx->aes_key.key_enc, nrounds, - tag); + ret = pmull_gcm_decrypt(nbytes, dst, src, + ctx->ghash_key.h, + dg, iv, ctx->aes_key.key_enc, + nrounds, tag, otag, authsize); kernel_neon_end(); if (unlikely(!nbytes)) @@ -507,6 +515,11 @@ static int gcm_decrypt(struct aead_request *req) err = skcipher_walk_done(&walk, walk.nbytes - nbytes); } while (walk.nbytes); + + if (err) + return err; + if (ret) + return -EBADMSG; } else { while (walk.nbytes >= AES_BLOCK_SIZE) { int blocks = walk.nbytes / AES_BLOCK_SIZE; @@ -548,23 +561,20 @@ static int gcm_decrypt(struct aead_request *req) err = skcipher_walk_done(&walk, 0); } + if (err) + return err; + put_unaligned_be64(dg[1], tag); put_unaligned_be64(dg[0], tag + 8); put_unaligned_be32(1, iv + GCM_IV_SIZE); aes_encrypt(&ctx->aes_key, iv, iv); crypto_xor(tag, iv, AES_BLOCK_SIZE); + + if (crypto_memneq(tag, otag, authsize)) { + memzero_explicit(tag, AES_BLOCK_SIZE); + return -EBADMSG; + } } - - if (err) - return err; - - /* compare calculated auth tag with the stored one */ - scatterwalk_map_and_copy(buf, req->src, - req->assoclen + req->cryptlen - authsize, - authsize, 0); - - if (crypto_memneq(tag, buf, authsize)) - return -EBADMSG; return 0; } diff --git a/arch/arm64/crypto/poly1305-armv8.pl b/arch/arm64/crypto/poly1305-armv8.pl index 6e5576d19af8..cbc980fb02e3 100644 --- a/arch/arm64/crypto/poly1305-armv8.pl +++ b/arch/arm64/crypto/poly1305-armv8.pl @@ -840,7 +840,6 @@ poly1305_blocks_neon: ldp d14,d15,[sp,#64] addp $ACC2,$ACC2,$ACC2 ldr x30,[sp,#8] - .inst 0xd50323bf // autiasp //////////////////////////////////////////////////////////////// // lazy reduction, but without narrowing @@ -882,6 +881,7 @@ poly1305_blocks_neon: str x4,[$ctx,#8] // set is_base2_26 ldr x29,[sp],#80 + .inst 0xd50323bf // autiasp ret .size poly1305_blocks_neon,.-poly1305_blocks_neon diff --git a/arch/arm64/crypto/poly1305-core.S_shipped b/arch/arm64/crypto/poly1305-core.S_shipped index 8d1c4e420ccd..fb2822abf63a 100644 --- a/arch/arm64/crypto/poly1305-core.S_shipped +++ b/arch/arm64/crypto/poly1305-core.S_shipped @@ -779,7 +779,6 @@ poly1305_blocks_neon: ldp d14,d15,[sp,#64] addp v21.2d,v21.2d,v21.2d ldr x30,[sp,#8] - .inst 0xd50323bf // autiasp //////////////////////////////////////////////////////////////// // lazy reduction, but without narrowing @@ -821,6 +820,7 @@ poly1305_blocks_neon: str x4,[x0,#8] // set is_base2_26 ldr x29,[sp],#80 + .inst 0xd50323bf // autiasp ret .size poly1305_blocks_neon,.-poly1305_blocks_neon diff --git a/arch/arm64/crypto/poly1305-glue.c b/arch/arm64/crypto/poly1305-glue.c index f33ada70c4ed..683de671741a 100644 --- a/arch/arm64/crypto/poly1305-glue.c +++ b/arch/arm64/crypto/poly1305-glue.c @@ -177,7 +177,7 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) } poly1305_emit(&dctx->h, dst, dctx->s); - *dctx = (struct poly1305_desc_ctx){}; + memzero_explicit(dctx, sizeof(*dctx)); } EXPORT_SYMBOL(poly1305_final_arch); diff --git a/arch/arm64/crypto/sha1-ce-glue.c b/arch/arm64/crypto/sha1-ce-glue.c index c63b99211db3..c93121bcfdeb 100644 --- a/arch/arm64/crypto/sha1-ce-glue.c +++ b/arch/arm64/crypto/sha1-ce-glue.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm64/crypto/sha2-ce-glue.c b/arch/arm64/crypto/sha2-ce-glue.c index 5e956d7582a5..31ba3da5e61b 100644 --- a/arch/arm64/crypto/sha2-ce-glue.c +++ b/arch/arm64/crypto/sha2-ce-glue.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm64/crypto/sha256-glue.c b/arch/arm64/crypto/sha256-glue.c index 77bc6e72abae..9462f6088b3f 100644 --- a/arch/arm64/crypto/sha256-glue.c +++ b/arch/arm64/crypto/sha256-glue.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm64/crypto/sha3-ce-glue.c b/arch/arm64/crypto/sha3-ce-glue.c index 9a4bbfc45f40..e5a2936f0886 100644 --- a/arch/arm64/crypto/sha3-ce-glue.c +++ b/arch/arm64/crypto/sha3-ce-glue.c @@ -94,7 +94,7 @@ static int sha3_final(struct shash_desc *desc, u8 *out) if (digest_size & 4) put_unaligned_le32(sctx->st[i], (__le32 *)digest); - *sctx = (struct sha3_state){}; + memzero_explicit(sctx, sizeof(*sctx)); return 0; } diff --git a/arch/arm64/crypto/sha512-ce-glue.c b/arch/arm64/crypto/sha512-ce-glue.c index dc890a719f54..faa83f6cf376 100644 --- a/arch/arm64/crypto/sha512-ce-glue.c +++ b/arch/arm64/crypto/sha512-ce-glue.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm64/crypto/sha512-glue.c b/arch/arm64/crypto/sha512-glue.c index 370ccb29602f..2acff1c7df5d 100644 --- a/arch/arm64/crypto/sha512-glue.c +++ b/arch/arm64/crypto/sha512-glue.c @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/m68k/Kconfig.machine b/arch/m68k/Kconfig.machine index 17e8c3a292d7..054ff6392329 100644 --- a/arch/m68k/Kconfig.machine +++ b/arch/m68k/Kconfig.machine @@ -30,11 +30,9 @@ config MAC select HAVE_ARCH_NVRAM_OPS help This option enables support for the Apple Macintosh series of - computers (yes, there is experimental support now, at least for part - of the series). - - Say N unless you're willing to code the remaining necessary support. - ;) + computers. If you plan to use this kernel on a Mac, say Y here and + browse the documentation available at ; + otherwise say N. config APOLLO bool "Apollo support" diff --git a/arch/m68k/amiga/config.c b/arch/m68k/amiga/config.c index bee9f240f35d..3f6f7380d93c 100644 --- a/arch/m68k/amiga/config.c +++ b/arch/m68k/amiga/config.c @@ -383,14 +383,6 @@ void __init config_amiga(void) mach_init_IRQ = amiga_init_IRQ; mach_get_model = amiga_get_model; mach_get_hardware_list = amiga_get_hardware_list; - - /* - * default MAX_DMA=0xffffffff on all machines. If we don't do so, the SCSI - * code will not be able to allocate any mem for transfers, unless we are - * dealing with a Z2 mem only system. /Jes - */ - mach_max_dma_address = 0xffffffff; - mach_reset = amiga_reset; #if IS_ENABLED(CONFIG_INPUT_M68K_BEEP) mach_beep = amiga_mksound; diff --git a/arch/m68k/apollo/config.c b/arch/m68k/apollo/config.c index 762da5d7a415..2ebe5b6d229b 100644 --- a/arch/m68k/apollo/config.c +++ b/arch/m68k/apollo/config.c @@ -150,7 +150,6 @@ void __init config_apollo(void) mach_sched_init=dn_sched_init; /* */ mach_init_IRQ=dn_init_IRQ; - mach_max_dma_address = 0xffffffff; mach_hwclk = dn_dummy_hwclk; /* */ mach_reset = dn_dummy_reset; /* */ #ifdef CONFIG_HEARTBEAT diff --git a/arch/m68k/atari/config.c b/arch/m68k/atari/config.c index 7ec3161e8517..43b4e764a1ac 100644 --- a/arch/m68k/atari/config.c +++ b/arch/m68k/atari/config.c @@ -205,7 +205,6 @@ void __init config_atari(void) mach_get_model = atari_get_model; mach_get_hardware_list = atari_get_hardware_list; mach_reset = atari_reset; - mach_max_dma_address = 0xffffff; #if IS_ENABLED(CONFIG_INPUT_M68K_BEEP) mach_beep = atari_mksound; #endif diff --git a/arch/m68k/bvme6000/config.c b/arch/m68k/bvme6000/config.c index 50f4d01363df..8afaa73ec802 100644 --- a/arch/m68k/bvme6000/config.c +++ b/arch/m68k/bvme6000/config.c @@ -101,7 +101,6 @@ void __init config_bvme6000(void) bvme6000_set_vectors(); #endif - mach_max_dma_address = 0xffffffff; mach_sched_init = bvme6000_sched_init; mach_init_IRQ = bvme6000_init_IRQ; mach_hwclk = bvme6000_hwclk; diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig index f9f4fa595e13..19b40b6bc4b7 100644 --- a/arch/m68k/configs/amiga_defconfig +++ b/arch/m68k/configs/amiga_defconfig @@ -563,6 +563,7 @@ CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECRDSA=m +CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m CONFIG_CRYPTO_CHACHA20POLY1305=m CONFIG_CRYPTO_AEGIS128=m @@ -583,7 +584,6 @@ CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_RMD256=m CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_AES=y @@ -626,6 +626,8 @@ CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m CONFIG_WW_MUTEX_SELFTEST=m CONFIG_EARLY_PRINTK=y +CONFIG_KUNIT=m +CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m @@ -638,7 +640,6 @@ CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m -CONFIG_TEST_BITFIELD=m CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m @@ -653,9 +654,13 @@ CONFIG_TEST_BLACKHOLE_DEV=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m +CONFIG_BITFIELD_KUNIT=m +CONFIG_LINEAR_RANGES_TEST=m +CONFIG_BITS_TEST=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m CONFIG_TEST_MEMCAT_P=m CONFIG_TEST_STACKINIT=m CONFIG_TEST_MEMINIT=m +CONFIG_TEST_FREE_PAGES=m diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig index f4828e86d547..07516abe0489 100644 --- a/arch/m68k/configs/apollo_defconfig +++ b/arch/m68k/configs/apollo_defconfig @@ -519,6 +519,7 @@ CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECRDSA=m +CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m CONFIG_CRYPTO_CHACHA20POLY1305=m CONFIG_CRYPTO_AEGIS128=m @@ -539,7 +540,6 @@ CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_RMD256=m CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_AES=y @@ -582,6 +582,8 @@ CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m CONFIG_WW_MUTEX_SELFTEST=m CONFIG_EARLY_PRINTK=y +CONFIG_KUNIT=m +CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m @@ -594,7 +596,6 @@ CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m -CONFIG_TEST_BITFIELD=m CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m @@ -609,9 +610,13 @@ CONFIG_TEST_BLACKHOLE_DEV=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m +CONFIG_BITFIELD_KUNIT=m +CONFIG_LINEAR_RANGES_TEST=m +CONFIG_BITS_TEST=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m CONFIG_TEST_MEMCAT_P=m CONFIG_TEST_STACKINIT=m CONFIG_TEST_MEMINIT=m +CONFIG_TEST_FREE_PAGES=m diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig index e7911f141de1..cc901c4e9492 100644 --- a/arch/m68k/configs/atari_defconfig +++ b/arch/m68k/configs/atari_defconfig @@ -541,6 +541,7 @@ CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECRDSA=m +CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m CONFIG_CRYPTO_CHACHA20POLY1305=m CONFIG_CRYPTO_AEGIS128=m @@ -561,7 +562,6 @@ CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_RMD256=m CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_AES=y @@ -604,6 +604,8 @@ CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m CONFIG_WW_MUTEX_SELFTEST=m CONFIG_EARLY_PRINTK=y +CONFIG_KUNIT=m +CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m @@ -616,7 +618,6 @@ CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m -CONFIG_TEST_BITFIELD=m CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m @@ -631,9 +632,13 @@ CONFIG_TEST_BLACKHOLE_DEV=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m +CONFIG_BITFIELD_KUNIT=m +CONFIG_LINEAR_RANGES_TEST=m +CONFIG_BITS_TEST=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m CONFIG_TEST_MEMCAT_P=m CONFIG_TEST_STACKINIT=m CONFIG_TEST_MEMINIT=m +CONFIG_TEST_FREE_PAGES=m diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig index d574e438e6db..fc9a94aa7d6b 100644 --- a/arch/m68k/configs/bvme6000_defconfig +++ b/arch/m68k/configs/bvme6000_defconfig @@ -512,6 +512,7 @@ CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECRDSA=m +CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m CONFIG_CRYPTO_CHACHA20POLY1305=m CONFIG_CRYPTO_AEGIS128=m @@ -532,7 +533,6 @@ CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_RMD256=m CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_AES=y @@ -575,6 +575,8 @@ CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m CONFIG_WW_MUTEX_SELFTEST=m CONFIG_EARLY_PRINTK=y +CONFIG_KUNIT=m +CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m @@ -587,7 +589,6 @@ CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m -CONFIG_TEST_BITFIELD=m CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m @@ -602,9 +603,13 @@ CONFIG_TEST_BLACKHOLE_DEV=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m +CONFIG_BITFIELD_KUNIT=m +CONFIG_LINEAR_RANGES_TEST=m +CONFIG_BITS_TEST=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m CONFIG_TEST_MEMCAT_P=m CONFIG_TEST_STACKINIT=m CONFIG_TEST_MEMINIT=m +CONFIG_TEST_FREE_PAGES=m diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig index c7ce206e6138..260f1206c810 100644 --- a/arch/m68k/configs/hp300_defconfig +++ b/arch/m68k/configs/hp300_defconfig @@ -521,6 +521,7 @@ CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECRDSA=m +CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m CONFIG_CRYPTO_CHACHA20POLY1305=m CONFIG_CRYPTO_AEGIS128=m @@ -541,7 +542,6 @@ CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_RMD256=m CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_AES=y @@ -584,6 +584,8 @@ CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m CONFIG_WW_MUTEX_SELFTEST=m CONFIG_EARLY_PRINTK=y +CONFIG_KUNIT=m +CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m @@ -596,7 +598,6 @@ CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m -CONFIG_TEST_BITFIELD=m CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m @@ -611,9 +612,13 @@ CONFIG_TEST_BLACKHOLE_DEV=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m +CONFIG_BITFIELD_KUNIT=m +CONFIG_LINEAR_RANGES_TEST=m +CONFIG_BITS_TEST=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m CONFIG_TEST_MEMCAT_P=m CONFIG_TEST_STACKINIT=m CONFIG_TEST_MEMINIT=m +CONFIG_TEST_FREE_PAGES=m diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig index 3cd76bfaee03..f6d50b3fe8c2 100644 --- a/arch/m68k/configs/mac_defconfig +++ b/arch/m68k/configs/mac_defconfig @@ -544,6 +544,7 @@ CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECRDSA=m +CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m CONFIG_CRYPTO_CHACHA20POLY1305=m CONFIG_CRYPTO_AEGIS128=m @@ -564,7 +565,6 @@ CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_RMD256=m CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_AES=y @@ -607,6 +607,8 @@ CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m CONFIG_WW_MUTEX_SELFTEST=m CONFIG_EARLY_PRINTK=y +CONFIG_KUNIT=m +CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m @@ -619,7 +621,6 @@ CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m -CONFIG_TEST_BITFIELD=m CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m @@ -634,9 +635,13 @@ CONFIG_TEST_BLACKHOLE_DEV=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m +CONFIG_BITFIELD_KUNIT=m +CONFIG_LINEAR_RANGES_TEST=m +CONFIG_BITS_TEST=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m CONFIG_TEST_MEMCAT_P=m CONFIG_TEST_STACKINIT=m CONFIG_TEST_MEMINIT=m +CONFIG_TEST_FREE_PAGES=m diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig index c3d6faa7894f..fbe000ca0003 100644 --- a/arch/m68k/configs/multi_defconfig +++ b/arch/m68k/configs/multi_defconfig @@ -630,6 +630,7 @@ CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECRDSA=m +CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m CONFIG_CRYPTO_CHACHA20POLY1305=m CONFIG_CRYPTO_AEGIS128=m @@ -650,7 +651,6 @@ CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_RMD256=m CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_AES=y @@ -693,6 +693,8 @@ CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m CONFIG_WW_MUTEX_SELFTEST=m CONFIG_EARLY_PRINTK=y +CONFIG_KUNIT=m +CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m @@ -705,7 +707,6 @@ CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m -CONFIG_TEST_BITFIELD=m CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m @@ -720,9 +721,13 @@ CONFIG_TEST_BLACKHOLE_DEV=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m +CONFIG_BITFIELD_KUNIT=m +CONFIG_LINEAR_RANGES_TEST=m +CONFIG_BITS_TEST=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m CONFIG_TEST_MEMCAT_P=m CONFIG_TEST_STACKINIT=m CONFIG_TEST_MEMINIT=m +CONFIG_TEST_FREE_PAGES=m diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig index 5568aa7d9d41..25ca836a5701 100644 --- a/arch/m68k/configs/mvme147_defconfig +++ b/arch/m68k/configs/mvme147_defconfig @@ -511,6 +511,7 @@ CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECRDSA=m +CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m CONFIG_CRYPTO_CHACHA20POLY1305=m CONFIG_CRYPTO_AEGIS128=m @@ -531,7 +532,6 @@ CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_RMD256=m CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_AES=y @@ -574,6 +574,8 @@ CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m CONFIG_WW_MUTEX_SELFTEST=m CONFIG_EARLY_PRINTK=y +CONFIG_KUNIT=m +CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m @@ -586,7 +588,6 @@ CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m -CONFIG_TEST_BITFIELD=m CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m @@ -601,9 +602,13 @@ CONFIG_TEST_BLACKHOLE_DEV=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m +CONFIG_BITFIELD_KUNIT=m +CONFIG_LINEAR_RANGES_TEST=m +CONFIG_BITS_TEST=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m CONFIG_TEST_MEMCAT_P=m CONFIG_TEST_STACKINIT=m CONFIG_TEST_MEMINIT=m +CONFIG_TEST_FREE_PAGES=m diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig index 5b1e72ce53f8..5794e43a2acb 100644 --- a/arch/m68k/configs/mvme16x_defconfig +++ b/arch/m68k/configs/mvme16x_defconfig @@ -512,6 +512,7 @@ CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECRDSA=m +CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m CONFIG_CRYPTO_CHACHA20POLY1305=m CONFIG_CRYPTO_AEGIS128=m @@ -532,7 +533,6 @@ CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_RMD256=m CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_AES=y @@ -575,6 +575,8 @@ CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m CONFIG_WW_MUTEX_SELFTEST=m CONFIG_EARLY_PRINTK=y +CONFIG_KUNIT=m +CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m @@ -587,7 +589,6 @@ CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m -CONFIG_TEST_BITFIELD=m CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m @@ -602,9 +603,13 @@ CONFIG_TEST_BLACKHOLE_DEV=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m +CONFIG_BITFIELD_KUNIT=m +CONFIG_LINEAR_RANGES_TEST=m +CONFIG_BITS_TEST=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m CONFIG_TEST_MEMCAT_P=m CONFIG_TEST_STACKINIT=m CONFIG_TEST_MEMINIT=m +CONFIG_TEST_FREE_PAGES=m diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig index c3a3dcf30fb9..dbfb18938e11 100644 --- a/arch/m68k/configs/q40_defconfig +++ b/arch/m68k/configs/q40_defconfig @@ -530,6 +530,7 @@ CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECRDSA=m +CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m CONFIG_CRYPTO_CHACHA20POLY1305=m CONFIG_CRYPTO_AEGIS128=m @@ -550,7 +551,6 @@ CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_RMD256=m CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_AES=y @@ -593,6 +593,8 @@ CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m CONFIG_WW_MUTEX_SELFTEST=m CONFIG_EARLY_PRINTK=y +CONFIG_KUNIT=m +CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m @@ -605,7 +607,6 @@ CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m -CONFIG_TEST_BITFIELD=m CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m @@ -620,9 +621,13 @@ CONFIG_TEST_BLACKHOLE_DEV=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m +CONFIG_BITFIELD_KUNIT=m +CONFIG_LINEAR_RANGES_TEST=m +CONFIG_BITS_TEST=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m CONFIG_TEST_MEMCAT_P=m CONFIG_TEST_STACKINIT=m CONFIG_TEST_MEMINIT=m +CONFIG_TEST_FREE_PAGES=m diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig index 3c00e52f1bf0..e6afbeee7c4a 100644 --- a/arch/m68k/configs/sun3_defconfig +++ b/arch/m68k/configs/sun3_defconfig @@ -514,6 +514,7 @@ CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECRDSA=m +CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m CONFIG_CRYPTO_CHACHA20POLY1305=m CONFIG_CRYPTO_AEGIS128=m @@ -534,7 +535,6 @@ CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_RMD256=m CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_AES=y @@ -576,6 +576,8 @@ CONFIG_STRING_SELFTEST=m CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m CONFIG_WW_MUTEX_SELFTEST=m +CONFIG_KUNIT=m +CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m @@ -588,7 +590,6 @@ CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m -CONFIG_TEST_BITFIELD=m CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m @@ -603,9 +604,13 @@ CONFIG_TEST_BLACKHOLE_DEV=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m +CONFIG_BITFIELD_KUNIT=m +CONFIG_LINEAR_RANGES_TEST=m +CONFIG_BITS_TEST=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m CONFIG_TEST_MEMCAT_P=m CONFIG_TEST_STACKINIT=m CONFIG_TEST_MEMINIT=m +CONFIG_TEST_FREE_PAGES=m diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig index 241242d73cbd..5340507a9fff 100644 --- a/arch/m68k/configs/sun3x_defconfig +++ b/arch/m68k/configs/sun3x_defconfig @@ -513,6 +513,7 @@ CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECRDSA=m +CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m CONFIG_CRYPTO_CHACHA20POLY1305=m CONFIG_CRYPTO_AEGIS128=m @@ -533,7 +534,6 @@ CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_RMD256=m CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_AES=y @@ -576,6 +576,8 @@ CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m CONFIG_WW_MUTEX_SELFTEST=m CONFIG_EARLY_PRINTK=y +CONFIG_KUNIT=m +CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m @@ -588,7 +590,6 @@ CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m -CONFIG_TEST_BITFIELD=m CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m @@ -603,9 +604,13 @@ CONFIG_TEST_BLACKHOLE_DEV=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m +CONFIG_BITFIELD_KUNIT=m +CONFIG_LINEAR_RANGES_TEST=m +CONFIG_BITS_TEST=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m CONFIG_TEST_MEMCAT_P=m CONFIG_TEST_STACKINIT=m CONFIG_TEST_MEMINIT=m +CONFIG_TEST_FREE_PAGES=m diff --git a/arch/m68k/hp300/config.c b/arch/m68k/hp300/config.c index a161d44fd20b..ce1eb3d3d55d 100644 --- a/arch/m68k/hp300/config.c +++ b/arch/m68k/hp300/config.c @@ -260,7 +260,6 @@ void __init config_hp300(void) #ifdef CONFIG_HEARTBEAT mach_heartbeat = hp300_pulse; #endif - mach_max_dma_address = 0xffffffff; if (hp300_model >= HP_330 && hp300_model <= HP_433S && hp300_model != HP_350) { diff --git a/arch/m68k/include/asm/cmpxchg.h b/arch/m68k/include/asm/cmpxchg.h index 3a3bdcfcd375..a4aa82021d3b 100644 --- a/arch/m68k/include/asm/cmpxchg.h +++ b/arch/m68k/include/asm/cmpxchg.h @@ -76,7 +76,7 @@ static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int siz } #endif -#define xchg(ptr,x) ((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr)))) +#define xchg(ptr,x) ({(__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr)));}) #include @@ -119,11 +119,11 @@ static inline unsigned long __cmpxchg(volatile void *p, unsigned long old, } #define cmpxchg(ptr, o, n) \ - ((__typeof__(*(ptr)))__cmpxchg((ptr), (unsigned long)(o), \ - (unsigned long)(n), sizeof(*(ptr)))) + ({(__typeof__(*(ptr)))__cmpxchg((ptr), (unsigned long)(o), \ + (unsigned long)(n), sizeof(*(ptr)));}) #define cmpxchg_local(ptr, o, n) \ - ((__typeof__(*(ptr)))__cmpxchg((ptr), (unsigned long)(o), \ - (unsigned long)(n), sizeof(*(ptr)))) + ({(__typeof__(*(ptr)))__cmpxchg((ptr), (unsigned long)(o), \ + (unsigned long)(n), sizeof(*(ptr)));}) #define cmpxchg64(ptr, o, n) cmpxchg64_local((ptr), (o), (n)) diff --git a/arch/m68k/include/asm/machdep.h b/arch/m68k/include/asm/machdep.h index 49bd3266b4b1..e564101790ab 100644 --- a/arch/m68k/include/asm/machdep.h +++ b/arch/m68k/include/asm/machdep.h @@ -27,7 +27,6 @@ extern void (*mach_halt)( void ); extern void (*mach_power_off)( void ); extern unsigned long (*mach_hd_init) (unsigned long, unsigned long); extern void (*mach_hd_setup)(char *, int *); -extern long mach_max_dma_address; extern void (*mach_heartbeat) (int); extern void (*mach_l2_flush) (int); extern void (*mach_beep) (unsigned int, unsigned int); diff --git a/arch/m68k/kernel/setup_mm.c b/arch/m68k/kernel/setup_mm.c index ab8aa7be260f..ade96b1ad946 100644 --- a/arch/m68k/kernel/setup_mm.c +++ b/arch/m68k/kernel/setup_mm.c @@ -99,7 +99,6 @@ EXPORT_SYMBOL(mach_set_rtc_pll); void (*mach_reset)( void ); void (*mach_halt)( void ); void (*mach_power_off)( void ); -long mach_max_dma_address = 0x00ffffff; /* default set to the lower 16MB */ #ifdef CONFIG_HEARTBEAT void (*mach_heartbeat) (int); EXPORT_SYMBOL(mach_heartbeat); diff --git a/arch/m68k/kernel/vmlinux-nommu.lds b/arch/m68k/kernel/vmlinux-nommu.lds index 7b975420c3d9..396e126a4258 100644 --- a/arch/m68k/kernel/vmlinux-nommu.lds +++ b/arch/m68k/kernel/vmlinux-nommu.lds @@ -65,7 +65,6 @@ SECTIONS { _edata = .; EXCEPTION_TABLE(16) - NOTES . = ALIGN(PAGE_SIZE); __init_begin = .; @@ -87,7 +86,7 @@ SECTIONS { _end = .; STABS_DEBUG - .comment 0 : { *(.comment) } + ELF_DETAILS /* Sections to be discarded */ DISCARDS diff --git a/arch/m68k/kernel/vmlinux-std.lds b/arch/m68k/kernel/vmlinux-std.lds index 4d33da4e7106..ed1d9eda3190 100644 --- a/arch/m68k/kernel/vmlinux-std.lds +++ b/arch/m68k/kernel/vmlinux-std.lds @@ -49,7 +49,6 @@ SECTIONS *(.m68k_fixup) __stop_fixup = .; } - NOTES .init_end : { /* This ALIGN be in a section so that _end is at the end of the load segment. */ @@ -60,7 +59,7 @@ SECTIONS _end = . ; STABS_DEBUG - .comment 0 : { *(.comment) } + ELF_DETAILS /* Sections to be discarded */ DISCARDS diff --git a/arch/m68k/kernel/vmlinux-sun3.lds b/arch/m68k/kernel/vmlinux-sun3.lds index 87d9f4d08f65..4a52f44f2ef0 100644 --- a/arch/m68k/kernel/vmlinux-sun3.lds +++ b/arch/m68k/kernel/vmlinux-sun3.lds @@ -33,7 +33,6 @@ SECTIONS RW_DATA(16, PAGE_SIZE, THREAD_SIZE) :data /* End of data goes *here* so that freeing init code works properly. */ _edata = .; - NOTES /* will be freed after init */ . = ALIGN(PAGE_SIZE); /* Init code and data */ @@ -53,6 +52,7 @@ __init_begin = .; _end = . ; STABS_DEBUG + ELF_DETAILS /* Sections to be discarded */ DISCARDS diff --git a/arch/m68k/mac/config.c b/arch/m68k/mac/config.c index 0ac53d87493c..e239d2ddccec 100644 --- a/arch/m68k/mac/config.c +++ b/arch/m68k/mac/config.c @@ -55,7 +55,6 @@ struct mac_booter_data mac_bi_data; static unsigned long mac_orig_videoaddr; extern int mac_hwclk(int, struct rtc_time *); -extern void iop_preinit(void); extern void iop_init(void); extern void via_init(void); extern void via_init_clock(irq_handler_t func); @@ -141,7 +140,6 @@ void __init config_mac(void) mach_reset = mac_reset; mach_halt = mac_poweroff; mach_power_off = mac_poweroff; - mach_max_dma_address = 0xffffffff; #if IS_ENABLED(CONFIG_INPUT_M68K_BEEP) mach_beep = mac_mksound; #endif @@ -777,16 +775,12 @@ static struct resource scc_b_rsrcs[] = { struct platform_device scc_a_pdev = { .name = "scc", .id = 0, - .num_resources = ARRAY_SIZE(scc_a_rsrcs), - .resource = scc_a_rsrcs, }; EXPORT_SYMBOL(scc_a_pdev); struct platform_device scc_b_pdev = { .name = "scc", .id = 1, - .num_resources = ARRAY_SIZE(scc_b_rsrcs), - .resource = scc_b_rsrcs, }; EXPORT_SYMBOL(scc_b_pdev); @@ -813,10 +807,15 @@ static void __init mac_identify(void) /* Set up serial port resources for the console initcall. */ - scc_a_rsrcs[0].start = (resource_size_t) mac_bi_data.sccbase + 2; - scc_a_rsrcs[0].end = scc_a_rsrcs[0].start; - scc_b_rsrcs[0].start = (resource_size_t) mac_bi_data.sccbase; - scc_b_rsrcs[0].end = scc_b_rsrcs[0].start; + scc_a_rsrcs[0].start = (resource_size_t)mac_bi_data.sccbase + 2; + scc_a_rsrcs[0].end = scc_a_rsrcs[0].start; + scc_a_pdev.num_resources = ARRAY_SIZE(scc_a_rsrcs); + scc_a_pdev.resource = scc_a_rsrcs; + + scc_b_rsrcs[0].start = (resource_size_t)mac_bi_data.sccbase; + scc_b_rsrcs[0].end = scc_b_rsrcs[0].start; + scc_b_pdev.num_resources = ARRAY_SIZE(scc_b_rsrcs); + scc_b_pdev.resource = scc_b_rsrcs; switch (macintosh_config->scc_type) { case MAC_SCC_PSC: @@ -835,13 +834,6 @@ static void __init mac_identify(void) break; } - /* - * We need to pre-init the IOPs, if any. Otherwise - * the serial console won't work if the user had - * the serial ports set to "Faster" mode in MacOS. - */ - iop_preinit(); - pr_info("Detected Macintosh model: %d\n", model); /* diff --git a/arch/m68k/mac/iop.c b/arch/m68k/mac/iop.c index c669a7644301..de156a027f5b 100644 --- a/arch/m68k/mac/iop.c +++ b/arch/m68k/mac/iop.c @@ -47,6 +47,10 @@ * * TODO: * + * o The SCC IOP has to be placed in bypass mode before the serial console + * gets initialized. iop_init() would be one place to do that. Or the + * bootloader could do that. For now, the Serial Switch control panel + * is needed for that -- contrary to the changelog above. * o Something should be periodically checking iop_alive() to make sure the * IOP hasn't died. * o Some of the IOP manager routines need better error checking and @@ -224,40 +228,6 @@ static struct iop_msg *iop_get_unused_msg(void) return NULL; } -/* - * This is called by the startup code before anything else. Its purpose - * is to find and initialize the IOPs early in the boot sequence, so that - * the serial IOP can be placed into bypass mode _before_ we try to - * initialize the serial console. - */ - -void __init iop_preinit(void) -{ - if (macintosh_config->scc_type == MAC_SCC_IOP) { - if (macintosh_config->ident == MAC_MODEL_IIFX) { - iop_base[IOP_NUM_SCC] = (struct mac_iop *) SCC_IOP_BASE_IIFX; - } else { - iop_base[IOP_NUM_SCC] = (struct mac_iop *) SCC_IOP_BASE_QUADRA; - } - iop_scc_present = 1; - } else { - iop_base[IOP_NUM_SCC] = NULL; - iop_scc_present = 0; - } - if (macintosh_config->adb_type == MAC_ADB_IOP) { - if (macintosh_config->ident == MAC_MODEL_IIFX) { - iop_base[IOP_NUM_ISM] = (struct mac_iop *) ISM_IOP_BASE_IIFX; - } else { - iop_base[IOP_NUM_ISM] = (struct mac_iop *) ISM_IOP_BASE_QUADRA; - } - iop_stop(iop_base[IOP_NUM_ISM]); - iop_ism_present = 1; - } else { - iop_base[IOP_NUM_ISM] = NULL; - iop_ism_present = 0; - } -} - /* * Initialize the IOPs, if present. */ @@ -266,11 +236,23 @@ void __init iop_init(void) { int i; - if (iop_scc_present) { + if (macintosh_config->scc_type == MAC_SCC_IOP) { + if (macintosh_config->ident == MAC_MODEL_IIFX) + iop_base[IOP_NUM_SCC] = (struct mac_iop *)SCC_IOP_BASE_IIFX; + else + iop_base[IOP_NUM_SCC] = (struct mac_iop *)SCC_IOP_BASE_QUADRA; + iop_scc_present = 1; pr_debug("SCC IOP detected at %p\n", iop_base[IOP_NUM_SCC]); } - if (iop_ism_present) { + if (macintosh_config->adb_type == MAC_ADB_IOP) { + if (macintosh_config->ident == MAC_MODEL_IIFX) + iop_base[IOP_NUM_ISM] = (struct mac_iop *)ISM_IOP_BASE_IIFX; + else + iop_base[IOP_NUM_ISM] = (struct mac_iop *)ISM_IOP_BASE_QUADRA; + iop_ism_present = 1; pr_debug("ISM IOP detected at %p\n", iop_base[IOP_NUM_ISM]); + + iop_stop(iop_base[IOP_NUM_ISM]); iop_start(iop_base[IOP_NUM_ISM]); iop_alive(iop_base[IOP_NUM_ISM]); /* clears the alive flag */ } diff --git a/arch/m68k/mac/via.c b/arch/m68k/mac/via.c index ac77d73af19a..9f2b32f22f16 100644 --- a/arch/m68k/mac/via.c +++ b/arch/m68k/mac/via.c @@ -169,8 +169,6 @@ void __init via_init(void) via1[vIER] = 0x7F; via1[vIFR] = 0x7F; - via1[vT1LL] = 0; - via1[vT1LH] = 0; via1[vT1CL] = 0; via1[vT1CH] = 0; via1[vT2CL] = 0; @@ -225,8 +223,6 @@ void __init via_init(void) via2[gIER] = 0x7F; via2[gIFR] = 0x7F | rbv_clear; if (!rbv_present) { - via2[vT1LL] = 0; - via2[vT1LH] = 0; via2[vT1CL] = 0; via2[vT1CH] = 0; via2[vT2CL] = 0; @@ -304,21 +300,6 @@ void via_l2_flush(int writeback) local_irq_restore(flags); } -/* - * Return the status of the L2 cache on a IIci - */ - -int via_get_cache_disable(void) -{ - /* Safeguard against being called accidentally */ - if (!via2) { - printk(KERN_ERR "via_get_cache_disable called on a non-VIA machine!\n"); - return 1; - } - - return (int) via2[gBufB] & VIA2B_vCDis; -} - /* * Initialize VIA2 for Nubus access */ @@ -619,8 +600,6 @@ void __init via_init_clock(irq_handler_t timer_routine) return; } - via1[vT1LL] = VIA_TC_LOW; - via1[vT1LH] = VIA_TC_HIGH; via1[vT1CL] = VIA_TC_LOW; via1[vT1CH] = VIA_TC_HIGH; via1[vACR] |= 0x40; diff --git a/arch/m68k/mvme147/config.c b/arch/m68k/mvme147/config.c index 490700aa2212..ead0e51ecd42 100644 --- a/arch/m68k/mvme147/config.c +++ b/arch/m68k/mvme147/config.c @@ -80,7 +80,6 @@ void __init mvme147_init_IRQ(void) void __init config_mvme147(void) { - mach_max_dma_address = 0x01000000; mach_sched_init = mvme147_sched_init; mach_init_IRQ = mvme147_init_IRQ; mach_hwclk = mvme147_hwclk; diff --git a/arch/m68k/mvme16x/config.c b/arch/m68k/mvme16x/config.c index 5b86d10e0f84..79896c331d55 100644 --- a/arch/m68k/mvme16x/config.c +++ b/arch/m68k/mvme16x/config.c @@ -268,7 +268,6 @@ void __init config_mvme16x(void) char id[40]; uint16_t brdno = be16_to_cpu(p->brdno); - mach_max_dma_address = 0xffffffff; mach_sched_init = mvme16x_sched_init; mach_init_IRQ = mvme16x_init_IRQ; mach_hwclk = mvme16x_hwclk; diff --git a/arch/m68k/q40/config.c b/arch/m68k/q40/config.c index 4627de3c0603..58566262127d 100644 --- a/arch/m68k/q40/config.c +++ b/arch/m68k/q40/config.c @@ -185,11 +185,6 @@ void __init config_q40(void) /* disable a few things that SMSQ might have left enabled */ q40_disable_irqs(); - - /* no DMA at all, but ide-scsi requires it.. make sure - * all physical RAM fits into the boundary - otherwise - * allocator may play costly and useless tricks */ - mach_max_dma_address = 1024*1024*1024; } diff --git a/arch/m68k/sun3x/config.c b/arch/m68k/sun3x/config.c index d806dee71a9c..37121a0f1253 100644 --- a/arch/m68k/sun3x/config.c +++ b/arch/m68k/sun3x/config.c @@ -44,8 +44,6 @@ void __init config_sun3x(void) sun3x_prom_init(); - mach_max_dma_address = 0xffffffff; /* we can DMA anywhere, whee */ - mach_sched_init = sun3x_sched_init; mach_init_IRQ = sun3_init_IRQ; diff --git a/arch/mips/cavium-octeon/crypto/octeon-crypto.h b/arch/mips/cavium-octeon/crypto/octeon-crypto.h index 7315cc307397..cb68f9e284bb 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-crypto.h +++ b/arch/mips/cavium-octeon/crypto/octeon-crypto.h @@ -41,7 +41,7 @@ do { \ */ #define read_octeon_64bit_hash_dword(index) \ ({ \ - u64 __value; \ + __be64 __value; \ \ __asm__ __volatile__ ( \ "dmfc2 %[rt],0x0048+" STR(index) \ diff --git a/arch/mips/cavium-octeon/crypto/octeon-md5.c b/arch/mips/cavium-octeon/crypto/octeon-md5.c index 8c8ea139653e..5ee4ade99b99 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-md5.c +++ b/arch/mips/cavium-octeon/crypto/octeon-md5.c @@ -68,10 +68,11 @@ static int octeon_md5_init(struct shash_desc *desc) { struct md5_state *mctx = shash_desc_ctx(desc); - mctx->hash[0] = cpu_to_le32(MD5_H0); - mctx->hash[1] = cpu_to_le32(MD5_H1); - mctx->hash[2] = cpu_to_le32(MD5_H2); - mctx->hash[3] = cpu_to_le32(MD5_H3); + mctx->hash[0] = MD5_H0; + mctx->hash[1] = MD5_H1; + mctx->hash[2] = MD5_H2; + mctx->hash[3] = MD5_H3; + cpu_to_le32_array(mctx->hash, 4); mctx->byte_count = 0; return 0; @@ -139,8 +140,9 @@ static int octeon_md5_final(struct shash_desc *desc, u8 *out) } memset(p, 0, padding); - mctx->block[14] = cpu_to_le32(mctx->byte_count << 3); - mctx->block[15] = cpu_to_le32(mctx->byte_count >> 29); + mctx->block[14] = mctx->byte_count << 3; + mctx->block[15] = mctx->byte_count >> 29; + cpu_to_le32_array(mctx->block + 14, 2); octeon_md5_transform(mctx->block); octeon_md5_read_hash(mctx); diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha1.c b/arch/mips/cavium-octeon/crypto/octeon-sha1.c index 75e79b47abfe..30f1d75208a5 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha1.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha1.c @@ -14,7 +14,7 @@ */ #include -#include +#include #include #include #include diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha256.c b/arch/mips/cavium-octeon/crypto/octeon-sha256.c index a682ce76716a..36cb92895d72 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha256.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha256.c @@ -15,7 +15,7 @@ */ #include -#include +#include #include #include #include diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha512.c b/arch/mips/cavium-octeon/crypto/octeon-sha512.c index 50722a0cfb53..359f039820d8 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha512.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha512.c @@ -14,7 +14,7 @@ */ #include -#include +#include #include #include #include diff --git a/arch/powerpc/crypto/sha1-spe-glue.c b/arch/powerpc/crypto/sha1-spe-glue.c index cb57be4ada61..b1e577cbf00c 100644 --- a/arch/powerpc/crypto/sha1-spe-glue.c +++ b/arch/powerpc/crypto/sha1-spe-glue.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/powerpc/crypto/sha1.c b/arch/powerpc/crypto/sha1.c index b40dc50a6908..7a55d790cdb1 100644 --- a/arch/powerpc/crypto/sha1.c +++ b/arch/powerpc/crypto/sha1.c @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include void powerpc_sha_transform(u32 *state, const u8 *src); diff --git a/arch/powerpc/crypto/sha256-spe-glue.c b/arch/powerpc/crypto/sha256-spe-glue.c index ceb0b6c980b3..a6e650a97d8f 100644 --- a/arch/powerpc/crypto/sha256-spe-glue.c +++ b/arch/powerpc/crypto/sha256-spe-glue.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include @@ -177,7 +177,7 @@ static int ppc_spe_sha256_final(struct shash_desc *desc, u8 *out) static int ppc_spe_sha224_final(struct shash_desc *desc, u8 *out) { - u32 D[SHA256_DIGEST_SIZE >> 2]; + __be32 D[SHA256_DIGEST_SIZE >> 2]; __be32 *dst = (__be32 *)out; ppc_spe_sha256_final(desc, (u8 *)D); diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 4a2a12be04c9..a60cc523d810 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -53,8 +53,7 @@ config ARCH_SUPPORTS_UPROBES config KASAN_SHADOW_OFFSET hex depends on KASAN - default 0x18000000000000 if KASAN_S390_4_LEVEL_PAGING - default 0x30000000000 + default 0x18000000000000 config S390 def_bool y @@ -191,7 +190,6 @@ config S390 select PCI_DOMAINS if PCI select PCI_MSI if PCI select PCI_MSI_ARCH_FALLBACKS if PCI_MSI - select SET_FS select SPARSE_IRQ select SYSCTL_EXCEPTION_TRACE select THREAD_INFO_IN_TASK @@ -714,7 +712,7 @@ if PCI config PCI_NR_FUNCTIONS int "Maximum number of PCI functions (1-4096)" range 1 4096 - default "128" + default "512" help This allows you to specify the maximum number of PCI functions which this kernel will support. diff --git a/arch/s390/Kconfig.debug b/arch/s390/Kconfig.debug index ab48b694ade8..6bfaceebbbc0 100644 --- a/arch/s390/Kconfig.debug +++ b/arch/s390/Kconfig.debug @@ -5,3 +5,11 @@ config TRACE_IRQFLAGS_SUPPORT config EARLY_PRINTK def_bool y + +config DEBUG_USER_ASCE + bool "Debug User ASCE" + help + Check on exit to user space that address space control + elements are setup correctly. + + If unsure, say N. diff --git a/arch/s390/Makefile b/arch/s390/Makefile index ba94b03c8b2f..8db267d2a543 100644 --- a/arch/s390/Makefile +++ b/arch/s390/Makefile @@ -25,7 +25,7 @@ KBUILD_AFLAGS_DECOMPRESSOR := $(CLANG_FLAGS) -m64 -D__ASSEMBLY__ KBUILD_AFLAGS_DECOMPRESSOR += $(if $(CONFIG_DEBUG_INFO),$(aflags_dwarf)) KBUILD_CFLAGS_DECOMPRESSOR := $(CLANG_FLAGS) -m64 -O2 KBUILD_CFLAGS_DECOMPRESSOR += -DDISABLE_BRANCH_PROFILING -D__NO_FORTIFY -KBUILD_CFLAGS_DECOMPRESSOR += -fno-delete-null-pointer-checks -msoft-float +KBUILD_CFLAGS_DECOMPRESSOR += -fno-delete-null-pointer-checks -msoft-float -mbackchain KBUILD_CFLAGS_DECOMPRESSOR += -fno-asynchronous-unwind-tables KBUILD_CFLAGS_DECOMPRESSOR += -ffreestanding KBUILD_CFLAGS_DECOMPRESSOR += $(call cc-disable-warning, address-of-packed-member) diff --git a/arch/s390/boot/boot.h b/arch/s390/boot/boot.h index 2ea603f70c3b..8b50967f5804 100644 --- a/arch/s390/boot/boot.h +++ b/arch/s390/boot/boot.h @@ -2,20 +2,32 @@ #ifndef BOOT_BOOT_H #define BOOT_BOOT_H +#include + +#define BOOT_STACK_OFFSET 0x8000 + +#ifndef __ASSEMBLY__ + +#include + void startup_kernel(void); -void detect_memory(void); +unsigned long detect_memory(void); +bool is_ipl_block_dump(void); void store_ipl_parmblock(void); void setup_boot_command_line(void); void parse_boot_command_line(void); -void setup_memory_end(void); void verify_facilities(void); void print_missing_facilities(void); void print_pgm_check_info(void); unsigned long get_random_base(unsigned long safe_addr); +void __printf(1, 2) decompressor_printk(const char *fmt, ...); -extern int kaslr_enabled; extern const char kernel_version[]; +extern unsigned long memory_limit; +extern int vmalloc_size_set; +extern int kaslr_enabled; unsigned long read_ipl_report(unsigned long safe_offset); +#endif /* __ASSEMBLY__ */ #endif /* BOOT_BOOT_H */ diff --git a/arch/s390/boot/compressed/.gitignore b/arch/s390/boot/compressed/.gitignore index 765a08f1bd77..01d93832cf4a 100644 --- a/arch/s390/boot/compressed/.gitignore +++ b/arch/s390/boot/compressed/.gitignore @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only vmlinux vmlinux.lds +vmlinux.syms diff --git a/arch/s390/boot/compressed/Makefile b/arch/s390/boot/compressed/Makefile index b235ed95a3d8..de18dab518bb 100644 --- a/arch/s390/boot/compressed/Makefile +++ b/arch/s390/boot/compressed/Makefile @@ -10,21 +10,39 @@ GCOV_PROFILE := n UBSAN_SANITIZE := n KASAN_SANITIZE := n -obj-y := $(if $(CONFIG_KERNEL_UNCOMPRESSED),,decompressor.o) piggy.o info.o +obj-y := $(if $(CONFIG_KERNEL_UNCOMPRESSED),,decompressor.o) info.o +obj-all := $(obj-y) piggy.o syms.o targets := vmlinux.lds vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 targets += vmlinux.bin.xz vmlinux.bin.lzma vmlinux.bin.lzo vmlinux.bin.lz4 -targets += info.bin $(obj-y) +targets += info.bin syms.bin vmlinux.syms $(obj-all) KBUILD_AFLAGS := $(KBUILD_AFLAGS_DECOMPRESSOR) KBUILD_CFLAGS := $(KBUILD_CFLAGS_DECOMPRESSOR) OBJCOPYFLAGS := OBJECTS := $(addprefix $(obj)/,$(obj-y)) +OBJECTS_ALL := $(addprefix $(obj)/,$(obj-all)) -LDFLAGS_vmlinux := --oformat $(LD_BFD) -e startup -T -$(obj)/vmlinux: $(obj)/vmlinux.lds $(objtree)/arch/s390/boot/startup.a $(OBJECTS) FORCE +LDFLAGS_vmlinux := --oformat $(LD_BFD) -e startup --build-id=sha1 -T +$(obj)/vmlinux: $(obj)/vmlinux.lds $(objtree)/arch/s390/boot/startup.a $(OBJECTS_ALL) FORCE $(call if_changed,ld) +LDFLAGS_vmlinux.syms := --oformat $(LD_BFD) -e startup -T +$(obj)/vmlinux.syms: $(obj)/vmlinux.lds $(objtree)/arch/s390/boot/startup.a $(OBJECTS) FORCE + $(call if_changed,ld) + +quiet_cmd_dumpsyms = DUMPSYMS $< +define cmd_dumpsyms + $(NM) -n -S --format=bsd "$<" | $(PERL) -ne '/(\w+)\s+(\w+)\s+[tT]\s+(\w+)/ and printf "%x %x %s\0",hex $$1,hex $$2,$$3' > "$@" +endef + +$(obj)/syms.bin: $(obj)/vmlinux.syms FORCE + $(call if_changed,dumpsyms) + +OBJCOPYFLAGS_syms.o := -I binary -O elf64-s390 -B s390:64-bit --rename-section .data=.decompressor.syms +$(obj)/syms.o: $(obj)/syms.bin FORCE + $(call if_changed,objcopy) + OBJCOPYFLAGS_info.bin := -O binary --only-section=.vmlinux.info --set-section-flags .vmlinux.info=load $(obj)/info.bin: vmlinux FORCE $(call if_changed,objcopy) diff --git a/arch/s390/boot/compressed/decompressor.h b/arch/s390/boot/compressed/decompressor.h index c15eb7114d83..41f0ad97a4db 100644 --- a/arch/s390/boot/compressed/decompressor.h +++ b/arch/s390/boot/compressed/decompressor.h @@ -2,8 +2,10 @@ #ifndef BOOT_COMPRESSED_DECOMPRESSOR_H #define BOOT_COMPRESSED_DECOMPRESSOR_H +#include + #ifdef CONFIG_KERNEL_UNCOMPRESSED -static inline void *decompress_kernel(void) {} +static inline void *decompress_kernel(void) { return NULL; } #else void *decompress_kernel(void); #endif diff --git a/arch/s390/boot/compressed/vmlinux.lds.S b/arch/s390/boot/compressed/vmlinux.lds.S index 9427e2cd0c15..27a09c1c78f6 100644 --- a/arch/s390/boot/compressed/vmlinux.lds.S +++ b/arch/s390/boot/compressed/vmlinux.lds.S @@ -27,6 +27,7 @@ SECTIONS *(.rodata.*) _erodata = . ; } + NOTES .data : { _data = . ; *(.data) @@ -82,6 +83,14 @@ SECTIONS *(.vmlinux.info) } + .decompressor.syms : { + . += 1; /* make sure we have \0 before the first entry */ + . = ALIGN(2); + _decompressor_syms_start = .; + *(.decompressor.syms) + _decompressor_syms_end = .; + } + #ifdef CONFIG_KERNEL_UNCOMPRESSED . = 0x100000; #else diff --git a/arch/s390/boot/head.S b/arch/s390/boot/head.S index 1a2c2b1ed964..dacb7813f982 100644 --- a/arch/s390/boot/head.S +++ b/arch/s390/boot/head.S @@ -28,6 +28,7 @@ #include #include #include +#include "boot.h" #define ARCH_OFFSET 4 @@ -62,8 +63,12 @@ __HEAD .org __LC_RST_NEW_PSW # 0x1a0 .quad 0,iplstart + .org __LC_EXT_NEW_PSW # 0x1b0 + .quad 0x0002000180000000,0x1b0 # disabled wait .org __LC_PGM_NEW_PSW # 0x1d0 .quad 0x0000000180000000,startup_pgm_check_handler + .org __LC_IO_NEW_PSW # 0x1f0 + .quad 0x0002000180000000,0x1f0 # disabled wait .org 0x200 @@ -275,8 +280,8 @@ iplstart: # or linload or SALIPL # .org 0x10000 -ENTRY(startup) - j .Lep_startup_normal +SYM_CODE_START(startup) + j startup_normal .org EP_OFFSET # # This is a list of s390 kernel entry points. At address 0x1000f the number of @@ -290,9 +295,9 @@ ENTRY(startup) # kdump startup-code at 0x10010, running in 64 bit absolute addressing mode # .org 0x10010 -ENTRY(startup_kdump) - j .Lep_startup_kdump -.Lep_startup_normal: + j startup_kdump +SYM_CODE_END(startup) +SYM_CODE_START_LOCAL(startup_normal) mvi __LC_AR_MODE_ID,1 # set esame flag slr %r0,%r0 # set cpuid to zero lhi %r1,2 # mode 2 = esame (dump) @@ -303,6 +308,9 @@ ENTRY(startup_kdump) sam64 # switch to 64 bit addressing mode basr %r13,0 # get base .LPG0: + mvc __LC_EXT_NEW_PSW(16),.Lext_new_psw-.LPG0(%r13) + mvc __LC_PGM_NEW_PSW(16),.Lpgm_new_psw-.LPG0(%r13) + mvc __LC_IO_NEW_PSW(16),.Lio_new_psw-.LPG0(%r13) xc 0x200(256),0x200 # partially clear lowcore xc 0x300(256),0x300 xc 0xe00(256),0xe00 @@ -315,12 +323,18 @@ ENTRY(startup_kdump) l %r15,.Lstack-.LPG0(%r13) brasl %r14,verify_facilities brasl %r14,startup_kernel +SYM_CODE_END(startup_normal) .Lstack: - .long 0x8000 + (1<<(PAGE_SHIFT+BOOT_STACK_ORDER)) - STACK_FRAME_OVERHEAD + .long BOOT_STACK_OFFSET + BOOT_STACK_SIZE - STACK_FRAME_OVERHEAD .align 8 6: .long 0x7fffffff,0xffffffff - +.Lext_new_psw: + .quad 0x0002000180000000,0x1b0 # disabled wait +.Lpgm_new_psw: + .quad 0x0000000180000000,startup_pgm_check_handler +.Lio_new_psw: + .quad 0x0002000180000000,0x1f0 # disabled wait .Lctl: .quad 0x04040000 # cr0: AFP registers & secondary space .quad 0 # cr1: primary space segment table .quad .Lduct # cr2: dispatchable unit control table @@ -359,7 +373,7 @@ ENTRY(startup_kdump) # It simply saves general/control registers and psw in # the save area and does disabled wait with a faulty address. # -ENTRY(startup_pgm_check_handler) +SYM_CODE_START_LOCAL(startup_pgm_check_handler) stmg %r8,%r15,__LC_SAVE_AREA_SYNC la %r8,4095 stctg %c0,%c15,__LC_CREGS_SAVE_AREA-4095(%r8) @@ -378,9 +392,9 @@ ENTRY(startup_pgm_check_handler) la %r8,4095 lmg %r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r8) lpswe __LC_RETURN_PSW # disabled wait +SYM_CODE_END(startup_pgm_check_handler) .Ldump_info_stack: .long 0x5000 + PAGE_SIZE - STACK_FRAME_OVERHEAD -ENDPROC(startup_pgm_check_handler) # # params at 10400 (setup.h) diff --git a/arch/s390/boot/head_kdump.S b/arch/s390/boot/head_kdump.S index 174d6959bf5b..f015469e7db9 100644 --- a/arch/s390/boot/head_kdump.S +++ b/arch/s390/boot/head_kdump.S @@ -19,8 +19,7 @@ # Note: This code has to be position independent # -.align 2 -.Lep_startup_kdump: +SYM_CODE_START_LOCAL(startup_kdump) lhi %r1,2 # mode 2 = esame (dump) sigp %r1,%r0,SIGP_SET_ARCHITECTURE # Switch to esame mode sam64 # Switch to 64 bit addressing @@ -87,14 +86,15 @@ startup_kdump_relocated: basr %r13,0 0: lpswe .Lrestart_psw-0b(%r13) # Start new kernel... +SYM_CODE_END(startup_kdump) .align 8 .Lrestart_psw: .quad 0x0000000080000000,0x0000000000000000 + startup #else -.align 2 -.Lep_startup_kdump: +SYM_CODE_START_LOCAL(startup_kdump) larl %r13,startup_kdump_crash lpswe 0(%r13) +SYM_CODE_END(startup_kdump) .align 8 startup_kdump_crash: .quad 0x0002000080000000,0x0000000000000000 + startup_kdump_crash diff --git a/arch/s390/boot/ipl_parm.c b/arch/s390/boot/ipl_parm.c index f94b91d72620..d372a45fe10e 100644 --- a/arch/s390/boot/ipl_parm.c +++ b/arch/s390/boot/ipl_parm.c @@ -17,10 +17,10 @@ int __bootdata_preserved(ipl_block_valid); unsigned int __bootdata_preserved(zlib_dfltcc_support) = ZLIB_DFLTCC_FULL; unsigned long __bootdata(vmalloc_size) = VMALLOC_DEFAULT_SIZE; -unsigned long __bootdata(memory_end); -int __bootdata(memory_end_set); int __bootdata(noexec_disabled); +unsigned long memory_limit; +int vmalloc_size_set; int kaslr_enabled; static inline int __diag308(unsigned long subcode, void *addr) @@ -57,6 +57,17 @@ void store_ipl_parmblock(void) ipl_block_valid = 1; } +bool is_ipl_block_dump(void) +{ + if (ipl_block.pb0_hdr.pbt == IPL_PBT_FCP && + ipl_block.fcp.opt == IPL_PB0_FCP_OPT_DUMP) + return true; + if (ipl_block.pb0_hdr.pbt == IPL_PBT_NVME && + ipl_block.nvme.opt == IPL_PB0_NVME_OPT_DUMP) + return true; + return false; +} + static size_t scpdata_length(const u8 *buf, size_t count) { while (count) { @@ -237,13 +248,13 @@ void parse_boot_command_line(void) while (*args) { args = next_arg(args, ¶m, &val); - if (!strcmp(param, "mem") && val) { - memory_end = round_down(memparse(val, NULL), PAGE_SIZE); - memory_end_set = 1; - } + if (!strcmp(param, "mem") && val) + memory_limit = round_down(memparse(val, NULL), PAGE_SIZE); - if (!strcmp(param, "vmalloc") && val) + if (!strcmp(param, "vmalloc") && val) { vmalloc_size = round_up(memparse(val, NULL), PAGE_SIZE); + vmalloc_size_set = 1; + } if (!strcmp(param, "dfltcc") && val) { if (!strcmp(val, "off")) @@ -279,27 +290,3 @@ void parse_boot_command_line(void) #endif } } - -static inline bool is_ipl_block_dump(void) -{ - if (ipl_block.pb0_hdr.pbt == IPL_PBT_FCP && - ipl_block.fcp.opt == IPL_PB0_FCP_OPT_DUMP) - return true; - if (ipl_block.pb0_hdr.pbt == IPL_PBT_NVME && - ipl_block.nvme.opt == IPL_PB0_NVME_OPT_DUMP) - return true; - return false; -} - -void setup_memory_end(void) -{ -#ifdef CONFIG_CRASH_DUMP - if (OLDMEM_BASE) { - kaslr_enabled = 0; - } else if (ipl_block_valid && is_ipl_block_dump()) { - kaslr_enabled = 0; - if (!sclp_early_get_hsa_size(&memory_end) && memory_end) - memory_end_set = 1; - } -#endif -} diff --git a/arch/s390/boot/kaslr.c b/arch/s390/boot/kaslr.c index d844a5ef9089..0dd48fbdbaa4 100644 --- a/arch/s390/boot/kaslr.c +++ b/arch/s390/boot/kaslr.c @@ -7,6 +7,7 @@ #include #include #include +#include #include "compressed/decompressor.h" #include "boot.h" @@ -176,8 +177,14 @@ unsigned long get_random_base(unsigned long safe_addr) unsigned long kasan_needs; int i; - if (memory_end_set) - memory_limit = min(memory_limit, memory_end); + memory_limit = min(memory_limit, ident_map_size); + + /* + * Avoid putting kernel in the end of physical memory + * which kasan will use for shadow memory and early pgtable + * mapping allocations. + */ + memory_limit -= kasan_estimate_memory_needs(memory_limit); if (IS_ENABLED(CONFIG_BLK_DEV_INITRD) && INITRD_START && INITRD_SIZE) { if (safe_addr < INITRD_START + INITRD_SIZE) @@ -185,28 +192,6 @@ unsigned long get_random_base(unsigned long safe_addr) } safe_addr = ALIGN(safe_addr, THREAD_SIZE); - if ((IS_ENABLED(CONFIG_KASAN))) { - /* - * Estimate kasan memory requirements, which it will reserve - * at the very end of available physical memory. To estimate - * that, we take into account that kasan would require - * 1/8 of available physical memory (for shadow memory) + - * creating page tables for the whole memory + shadow memory - * region (1 + 1/8). To keep page tables estimates simple take - * the double of combined ptes size. - */ - memory_limit = get_mem_detect_end(); - if (memory_end_set && memory_limit > memory_end) - memory_limit = memory_end; - - /* for shadow memory */ - kasan_needs = memory_limit / 8; - /* for paging structures */ - kasan_needs += (memory_limit + kasan_needs) / PAGE_SIZE / - _PAGE_ENTRIES * _PAGE_TABLE_SIZE * 2; - memory_limit -= kasan_needs; - } - kernel_size = vmlinux.image_size + vmlinux.bss_size; if (safe_addr + kernel_size > memory_limit) return 0; diff --git a/arch/s390/boot/mem_detect.c b/arch/s390/boot/mem_detect.c index 62e7c13ce85c..40168e59abd3 100644 --- a/arch/s390/boot/mem_detect.c +++ b/arch/s390/boot/mem_detect.c @@ -8,7 +8,6 @@ #include "compressed/decompressor.h" #include "boot.h" -unsigned long __bootdata(max_physmem_end); struct mem_detect_info __bootdata(mem_detect); /* up to 256 storage elements, 1020 subincrements each */ @@ -149,27 +148,29 @@ static void search_mem_end(void) add_mem_detect_block(0, (offset + 1) << 20); } -void detect_memory(void) +unsigned long detect_memory(void) { + unsigned long max_physmem_end; + sclp_early_get_memsize(&max_physmem_end); if (!sclp_early_read_storage_info()) { mem_detect.info_source = MEM_DETECT_SCLP_STOR_INFO; - return; + return max_physmem_end; } if (!diag260()) { mem_detect.info_source = MEM_DETECT_DIAG260; - return; + return max_physmem_end; } if (max_physmem_end) { add_mem_detect_block(0, max_physmem_end); mem_detect.info_source = MEM_DETECT_SCLP_READ_INFO; - return; + return max_physmem_end; } search_mem_end(); mem_detect.info_source = MEM_DETECT_BIN_SEARCH; - max_physmem_end = get_mem_detect_end(); + return get_mem_detect_end(); } diff --git a/arch/s390/boot/pgm_check_info.c b/arch/s390/boot/pgm_check_info.c index a3c9862bcede..3a46abed2549 100644 --- a/arch/s390/boot/pgm_check_info.c +++ b/arch/s390/boot/pgm_check_info.c @@ -1,99 +1,181 @@ // SPDX-License-Identifier: GPL-2.0 #include #include +#include +#include +#include #include #include #include +#include +#include #include "boot.h" const char hex_asc[] = "0123456789abcdef"; -#define add_val_as_hex(dst, val) \ - __add_val_as_hex(dst, (const unsigned char *)&val, sizeof(val)) - -static char *__add_val_as_hex(char *dst, const unsigned char *src, size_t count) +static char *as_hex(char *dst, unsigned long val, int pad) { - while (count--) - dst = hex_byte_pack(dst, *src++); - return dst; + char *p, *end = p = dst + max(pad, (int)__fls(val | 1) / 4 + 1); + + for (*p-- = 0; p >= dst; val >>= 4) + *p-- = hex_asc[val & 0x0f]; + return end; } -static char *add_str(char *dst, char *src) +static char *symstart(char *p) { - strcpy(dst, src); - return dst + strlen(dst); + while (*p) + p--; + return p + 1; +} + +extern char _decompressor_syms_start[], _decompressor_syms_end[]; +static noinline char *findsym(unsigned long ip, unsigned short *off, unsigned short *len) +{ + /* symbol entries are in a form "10000 c4 startup\0" */ + char *a = _decompressor_syms_start; + char *b = _decompressor_syms_end; + unsigned long start; + unsigned long size; + char *pivot; + char *endp; + + while (a < b) { + pivot = symstart(a + (b - a) / 2); + start = simple_strtoull(pivot, &endp, 16); + size = simple_strtoull(endp + 1, &endp, 16); + if (ip < start) { + b = pivot; + continue; + } + if (ip > start + size) { + a = pivot + strlen(pivot) + 1; + continue; + } + *off = ip - start; + *len = size; + return endp + 1; + } + return NULL; +} + +static noinline char *strsym(void *ip) +{ + static char buf[64]; + unsigned short off; + unsigned short len; + char *p; + + p = findsym((unsigned long)ip, &off, &len); + if (p) { + strncpy(buf, p, sizeof(buf)); + /* reserve 15 bytes for offset/len in symbol+0x1234/0x1234 */ + p = buf + strnlen(buf, sizeof(buf) - 15); + strcpy(p, "+0x"); + p = as_hex(p + 3, off, 0); + strcpy(p, "/0x"); + as_hex(p + 3, len, 0); + } else { + as_hex(buf, (unsigned long)ip, 16); + } + return buf; +} + +void decompressor_printk(const char *fmt, ...) +{ + char buf[1024] = { 0 }; + char *end = buf + sizeof(buf) - 1; /* make sure buf is 0 terminated */ + unsigned long pad; + char *p = buf; + va_list args; + + va_start(args, fmt); + for (; p < end && *fmt; fmt++) { + if (*fmt != '%') { + *p++ = *fmt; + continue; + } + pad = isdigit(*++fmt) ? simple_strtol(fmt, (char **)&fmt, 10) : 0; + switch (*fmt) { + case 's': + p = buf + strlcat(buf, va_arg(args, char *), sizeof(buf)); + break; + case 'p': + if (*++fmt != 'S') + goto out; + p = buf + strlcat(buf, strsym(va_arg(args, void *)), sizeof(buf)); + break; + case 'l': + if (*++fmt != 'x' || end - p <= max(sizeof(long) * 2, pad)) + goto out; + p = as_hex(p, va_arg(args, unsigned long), pad); + break; + case 'x': + if (end - p <= max(sizeof(int) * 2, pad)) + goto out; + p = as_hex(p, va_arg(args, unsigned int), pad); + break; + default: + goto out; + } + } +out: + va_end(args); + sclp_early_printk(buf); +} + +static noinline void print_stacktrace(void) +{ + struct stack_info boot_stack = { STACK_TYPE_TASK, BOOT_STACK_OFFSET, + BOOT_STACK_OFFSET + BOOT_STACK_SIZE }; + unsigned long sp = S390_lowcore.gpregs_save_area[15]; + bool first = true; + + decompressor_printk("Call Trace:\n"); + while (!(sp & 0x7) && on_stack(&boot_stack, sp, sizeof(struct stack_frame))) { + struct stack_frame *sf = (struct stack_frame *)sp; + + decompressor_printk(first ? "(sp:%016lx [<%016lx>] %pS)\n" : + " sp:%016lx [<%016lx>] %pS\n", + sp, sf->gprs[8], (void *)sf->gprs[8]); + if (sf->back_chain <= sp) + break; + sp = sf->back_chain; + first = false; + } } void print_pgm_check_info(void) { + unsigned long *gpregs = (unsigned long *)S390_lowcore.gpregs_save_area; struct psw_bits *psw = &psw_bits(S390_lowcore.psw_save_area); - unsigned short ilc = S390_lowcore.pgm_ilc >> 1; - char buf[256]; - int row, col; - char *p; - add_str(buf, "Linux version "); - strlcat(buf, kernel_version, sizeof(buf) - 1); - strlcat(buf, "\n", sizeof(buf)); - sclp_early_printk(buf); - - p = add_str(buf, "Kernel fault: interruption code "); - p = add_val_as_hex(buf + strlen(buf), S390_lowcore.pgm_code); - p = add_str(p, " ilc:"); - *p++ = hex_asc_lo(ilc); - add_str(p, "\n"); - sclp_early_printk(buf); - - if (kaslr_enabled) { - p = add_str(buf, "Kernel random base: "); - p = add_val_as_hex(p, __kaslr_offset); - add_str(p, "\n"); - sclp_early_printk(buf); - } - - p = add_str(buf, "PSW : "); - p = add_val_as_hex(p, S390_lowcore.psw_save_area.mask); - p = add_str(p, " "); - p = add_val_as_hex(p, S390_lowcore.psw_save_area.addr); - add_str(p, "\n"); - sclp_early_printk(buf); - - p = add_str(buf, " R:"); - *p++ = hex_asc_lo(psw->per); - p = add_str(p, " T:"); - *p++ = hex_asc_lo(psw->dat); - p = add_str(p, " IO:"); - *p++ = hex_asc_lo(psw->io); - p = add_str(p, " EX:"); - *p++ = hex_asc_lo(psw->ext); - p = add_str(p, " Key:"); - *p++ = hex_asc_lo(psw->key); - p = add_str(p, " M:"); - *p++ = hex_asc_lo(psw->mcheck); - p = add_str(p, " W:"); - *p++ = hex_asc_lo(psw->wait); - p = add_str(p, " P:"); - *p++ = hex_asc_lo(psw->pstate); - p = add_str(p, " AS:"); - *p++ = hex_asc_lo(psw->as); - p = add_str(p, " CC:"); - *p++ = hex_asc_lo(psw->cc); - p = add_str(p, " PM:"); - *p++ = hex_asc_lo(psw->pm); - p = add_str(p, " RI:"); - *p++ = hex_asc_lo(psw->ri); - p = add_str(p, " EA:"); - *p++ = hex_asc_lo(psw->eaba); - add_str(p, "\n"); - sclp_early_printk(buf); - - for (row = 0; row < 4; row++) { - p = add_str(buf, row == 0 ? "GPRS:" : " "); - for (col = 0; col < 4; col++) { - p = add_str(p, " "); - p = add_val_as_hex(p, S390_lowcore.gpregs_save_area[row * 4 + col]); - } - add_str(p, "\n"); - sclp_early_printk(buf); - } + decompressor_printk("Linux version %s\n", kernel_version); + if (!is_prot_virt_guest() && early_command_line[0]) + decompressor_printk("Kernel command line: %s\n", early_command_line); + decompressor_printk("Kernel fault: interruption code %04x ilc:%x\n", + S390_lowcore.pgm_code, S390_lowcore.pgm_ilc >> 1); + if (kaslr_enabled) + decompressor_printk("Kernel random base: %lx\n", __kaslr_offset); + decompressor_printk("PSW : %016lx %016lx (%pS)\n", + S390_lowcore.psw_save_area.mask, + S390_lowcore.psw_save_area.addr, + (void *)S390_lowcore.psw_save_area.addr); + decompressor_printk( + " R:%x T:%x IO:%x EX:%x Key:%x M:%x W:%x P:%x AS:%x CC:%x PM:%x RI:%x EA:%x\n", + psw->per, psw->dat, psw->io, psw->ext, psw->key, psw->mcheck, + psw->wait, psw->pstate, psw->as, psw->cc, psw->pm, psw->ri, + psw->eaba); + decompressor_printk("GPRS: %016lx %016lx %016lx %016lx\n", + gpregs[0], gpregs[1], gpregs[2], gpregs[3]); + decompressor_printk(" %016lx %016lx %016lx %016lx\n", + gpregs[4], gpregs[5], gpregs[6], gpregs[7]); + decompressor_printk(" %016lx %016lx %016lx %016lx\n", + gpregs[8], gpregs[9], gpregs[10], gpregs[11]); + decompressor_printk(" %016lx %016lx %016lx %016lx\n", + gpregs[12], gpregs[13], gpregs[14], gpregs[15]); + print_stacktrace(); + decompressor_printk("Last Breaking-Event-Address:\n"); + decompressor_printk(" [<%016lx>] %pS\n", (unsigned long)S390_lowcore.breaking_event_addr, + (void *)S390_lowcore.breaking_event_addr); } diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index cc96b04cc0ba..05f8eefa3dcf 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -1,7 +1,9 @@ // SPDX-License-Identifier: GPL-2.0 #include #include +#include #include +#include #include #include #include @@ -13,6 +15,7 @@ extern char __boot_data_start[], __boot_data_end[]; extern char __boot_data_preserved_start[], __boot_data_preserved_end[]; unsigned long __bootdata_preserved(__kaslr_offset); +unsigned long __bootdata(ident_map_size); /* * Some code and data needs to stay below 2 GB, even when the kernel would be @@ -58,6 +61,14 @@ void error(char *x) disabled_wait(); } +static void setup_lpp(void) +{ + S390_lowcore.current_pid = 0; + S390_lowcore.lpp = LPP_MAGIC; + if (test_facility(40)) + lpp(&S390_lowcore.lpp); +} + #ifdef CONFIG_KERNEL_UNCOMPRESSED unsigned long mem_safe_offset(void) { @@ -118,6 +129,46 @@ static void handle_relocs(unsigned long offset) } } +/* + * Merge information from several sources into a single ident_map_size value. + * "ident_map_size" represents the upper limit of physical memory we may ever + * reach. It might not be all online memory, but also include standby (offline) + * memory. "ident_map_size" could be lower then actual standby or even online + * memory present, due to limiting factors. We should never go above this limit. + * It is the size of our identity mapping. + * + * Consider the following factors: + * 1. max_physmem_end - end of physical memory online or standby. + * Always <= end of the last online memory block (get_mem_detect_end()). + * 2. CONFIG_MAX_PHYSMEM_BITS - the maximum size of physical memory the + * kernel is able to support. + * 3. "mem=" kernel command line option which limits physical memory usage. + * 4. OLDMEM_BASE which is a kdump memory limit when the kernel is executed as + * crash kernel. + * 5. "hsa" size which is a memory limit when the kernel is executed during + * zfcp/nvme dump. + */ +static void setup_ident_map_size(unsigned long max_physmem_end) +{ + unsigned long hsa_size; + + ident_map_size = max_physmem_end; + if (memory_limit) + ident_map_size = min(ident_map_size, memory_limit); + ident_map_size = min(ident_map_size, 1UL << MAX_PHYSMEM_BITS); + +#ifdef CONFIG_CRASH_DUMP + if (OLDMEM_BASE) { + kaslr_enabled = 0; + ident_map_size = min(ident_map_size, OLDMEM_SIZE); + } else if (ipl_block_valid && is_ipl_block_dump()) { + kaslr_enabled = 0; + if (!sclp_early_get_hsa_size(&hsa_size) && hsa_size) + ident_map_size = min(ident_map_size, hsa_size); + } +#endif +} + /* * This function clears the BSS section of the decompressed Linux kernel and NOT the decompressor's. */ @@ -126,12 +177,27 @@ static void clear_bss_section(void) memset((void *)vmlinux.default_lma + vmlinux.image_size, 0, vmlinux.bss_size); } +/* + * Set vmalloc area size to an 8th of (potential) physical memory + * size, unless size has been set by kernel command line parameter. + */ +static void setup_vmalloc_size(void) +{ + unsigned long size; + + if (vmalloc_size_set) + return; + size = round_up(ident_map_size / 8, _SEGMENT_SIZE); + vmalloc_size = max(size, vmalloc_size); +} + void startup_kernel(void) { unsigned long random_lma; unsigned long safe_addr; void *img; + setup_lpp(); store_ipl_parmblock(); safe_addr = mem_safe_offset(); safe_addr = read_ipl_report(safe_addr); @@ -140,8 +206,8 @@ void startup_kernel(void) sclp_early_read_info(); setup_boot_command_line(); parse_boot_command_line(); - setup_memory_end(); - detect_memory(); + setup_ident_map_size(detect_memory()); + setup_vmalloc_size(); random_lma = __kaslr_offset = 0; if (IS_ENABLED(CONFIG_RANDOMIZE_BASE) && kaslr_enabled) { diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index fe6f529ac82c..c52113a238b1 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -826,6 +826,7 @@ CONFIG_FTRACE_SYSCALLS=y CONFIG_BLK_DEV_IO_TRACE=y CONFIG_BPF_KPROBE_OVERRIDE=y CONFIG_HIST_TRIGGERS=y +CONFIG_DEBUG_USER_ASCE=y CONFIG_NOTIFIER_ERROR_INJECTION=m CONFIG_NETDEV_NOTIFIER_ERROR_INJECT=m CONFIG_FAULT_INJECTION=y diff --git a/arch/s390/crypto/arch_random.c b/arch/s390/crypto/arch_random.c index dd95cdbd22ce..7b947728d57e 100644 --- a/arch/s390/crypto/arch_random.c +++ b/arch/s390/crypto/arch_random.c @@ -2,7 +2,7 @@ /* * s390 arch random implementation. * - * Copyright IBM Corp. 2017, 2018 + * Copyright IBM Corp. 2017, 2020 * Author(s): Harald Freudenberger * * The s390_arch_random_generate() function may be called from random.c @@ -33,6 +33,7 @@ #include #include #include +#include #include DEFINE_STATIC_KEY_FALSE(s390_arch_random_available); @@ -99,6 +100,113 @@ static void arch_rng_refill_buffer(struct work_struct *unused) queue_delayed_work(system_long_wq, &arch_rng_work, delay); } +/* + * Here follows the implementation of s390_arch_get_random_long(). + * + * The random longs to be pulled by arch_get_random_long() are + * prepared in an 4K buffer which is filled from the NIST 800-90 + * compliant s390 drbg. By default the random long buffer is refilled + * 256 times before the drbg itself needs a reseed. The reseed of the + * drbg is done with 32 bytes fetched from the high quality (but slow) + * trng which is assumed to deliver 100% entropy. So the 32 * 8 = 256 + * bits of entropy are spread over 256 * 4KB = 1MB serving 131072 + * arch_get_random_long() invocations before reseeded. + * + * How often the 4K random long buffer is refilled with the drbg + * before the drbg is reseeded can be adjusted. There is a module + * parameter 's390_arch_rnd_long_drbg_reseed' accessible via + * /sys/module/arch_random/parameters/rndlong_drbg_reseed + * or as kernel command line parameter + * arch_random.rndlong_drbg_reseed= + * This parameter tells how often the drbg fills the 4K buffer before + * it is re-seeded by fresh entropy from the trng. + * A value of 16 results in reseeding the drbg at every 16 * 4 KB = 64 + * KB with 32 bytes of fresh entropy pulled from the trng. So a value + * of 16 would result in 256 bits entropy per 64 KB. + * A value of 256 results in 1MB of drbg output before a reseed of the + * drbg is done. So this would spread the 256 bits of entropy among 1MB. + * Setting this parameter to 0 forces the reseed to take place every + * time the 4K buffer is depleted, so the entropy rises to 256 bits + * entropy per 4K or 0.5 bit entropy per arch_get_random_long(). With + * setting this parameter to negative values all this effort is + * disabled, arch_get_random long() returns false and thus indicating + * that the arch_get_random_long() feature is disabled at all. + */ + +static unsigned long rndlong_buf[512]; +static DEFINE_SPINLOCK(rndlong_lock); +static int rndlong_buf_index; + +static int rndlong_drbg_reseed = 256; +module_param_named(rndlong_drbg_reseed, rndlong_drbg_reseed, int, 0600); +MODULE_PARM_DESC(rndlong_drbg_reseed, "s390 arch_get_random_long() drbg reseed"); + +static inline void refill_rndlong_buf(void) +{ + static u8 prng_ws[240]; + static int drbg_counter; + + if (--drbg_counter < 0) { + /* need to re-seed the drbg */ + u8 seed[32]; + + /* fetch seed from trng */ + cpacf_trng(NULL, 0, seed, sizeof(seed)); + /* seed drbg */ + memset(prng_ws, 0, sizeof(prng_ws)); + cpacf_prno(CPACF_PRNO_SHA512_DRNG_SEED, + &prng_ws, NULL, 0, seed, sizeof(seed)); + /* re-init counter for drbg */ + drbg_counter = rndlong_drbg_reseed; + } + + /* fill the arch_get_random_long buffer from drbg */ + cpacf_prno(CPACF_PRNO_SHA512_DRNG_GEN, &prng_ws, + (u8 *) rndlong_buf, sizeof(rndlong_buf), + NULL, 0); +} + +bool s390_arch_get_random_long(unsigned long *v) +{ + bool rc = false; + unsigned long flags; + + /* arch_get_random_long() disabled ? */ + if (rndlong_drbg_reseed < 0) + return false; + + /* try to lock the random long lock */ + if (!spin_trylock_irqsave(&rndlong_lock, flags)) + return false; + + if (--rndlong_buf_index >= 0) { + /* deliver next long value from the buffer */ + *v = rndlong_buf[rndlong_buf_index]; + rc = true; + goto out; + } + + /* buffer is depleted and needs refill */ + if (in_interrupt()) { + /* delay refill in interrupt context to next caller */ + rndlong_buf_index = 0; + goto out; + } + + /* refill random long buffer */ + refill_rndlong_buf(); + rndlong_buf_index = ARRAY_SIZE(rndlong_buf); + + /* and provide one random long */ + *v = rndlong_buf[--rndlong_buf_index]; + rc = true; + +out: + spin_unlock_irqrestore(&rndlong_lock, flags); + return rc; +} +EXPORT_SYMBOL(s390_arch_get_random_long); + static int __init s390_arch_random_init(void) { /* all the needed PRNO subfunctions available ? */ diff --git a/arch/s390/crypto/prng.c b/arch/s390/crypto/prng.c index 5057773f82e9..b2f219ec379c 100644 --- a/arch/s390/crypto/prng.c +++ b/arch/s390/crypto/prng.c @@ -674,20 +674,6 @@ static const struct file_operations prng_tdes_fops = { .llseek = noop_llseek, }; -static struct miscdevice prng_sha512_dev = { - .name = "prandom", - .minor = MISC_DYNAMIC_MINOR, - .mode = 0644, - .fops = &prng_sha512_fops, -}; -static struct miscdevice prng_tdes_dev = { - .name = "prandom", - .minor = MISC_DYNAMIC_MINOR, - .mode = 0644, - .fops = &prng_tdes_fops, -}; - - /* chunksize attribute (ro) */ static ssize_t prng_chunksize_show(struct device *dev, struct device_attribute *attr, @@ -801,18 +787,30 @@ static struct attribute *prng_sha512_dev_attrs[] = { &dev_attr_strength.attr, NULL }; +ATTRIBUTE_GROUPS(prng_sha512_dev); + static struct attribute *prng_tdes_dev_attrs[] = { &dev_attr_chunksize.attr, &dev_attr_byte_counter.attr, &dev_attr_mode.attr, NULL }; +ATTRIBUTE_GROUPS(prng_tdes_dev); -static struct attribute_group prng_sha512_dev_attr_group = { - .attrs = prng_sha512_dev_attrs +static struct miscdevice prng_sha512_dev = { + .name = "prandom", + .minor = MISC_DYNAMIC_MINOR, + .mode = 0644, + .fops = &prng_sha512_fops, + .groups = prng_sha512_dev_groups, }; -static struct attribute_group prng_tdes_dev_attr_group = { - .attrs = prng_tdes_dev_attrs + +static struct miscdevice prng_tdes_dev = { + .name = "prandom", + .minor = MISC_DYNAMIC_MINOR, + .mode = 0644, + .fops = &prng_tdes_fops, + .groups = prng_tdes_dev_groups, }; @@ -867,13 +865,6 @@ static int __init prng_init(void) prng_sha512_deinstantiate(); goto out; } - ret = sysfs_create_group(&prng_sha512_dev.this_device->kobj, - &prng_sha512_dev_attr_group); - if (ret) { - misc_deregister(&prng_sha512_dev); - prng_sha512_deinstantiate(); - goto out; - } } else { @@ -898,14 +889,6 @@ static int __init prng_init(void) prng_tdes_deinstantiate(); goto out; } - ret = sysfs_create_group(&prng_tdes_dev.this_device->kobj, - &prng_tdes_dev_attr_group); - if (ret) { - misc_deregister(&prng_tdes_dev); - prng_tdes_deinstantiate(); - goto out; - } - } out: @@ -916,13 +899,9 @@ out: static void __exit prng_exit(void) { if (prng_mode == PRNG_MODE_SHA512) { - sysfs_remove_group(&prng_sha512_dev.this_device->kobj, - &prng_sha512_dev_attr_group); misc_deregister(&prng_sha512_dev); prng_sha512_deinstantiate(); } else { - sysfs_remove_group(&prng_tdes_dev.this_device->kobj, - &prng_tdes_dev_attr_group); misc_deregister(&prng_tdes_dev); prng_tdes_deinstantiate(); } diff --git a/arch/s390/crypto/sha.h b/arch/s390/crypto/sha.h index ada2f98c27b7..65ea12fc87a1 100644 --- a/arch/s390/crypto/sha.h +++ b/arch/s390/crypto/sha.h @@ -11,7 +11,8 @@ #define _CRYPTO_ARCH_S390_SHA_H #include -#include +#include +#include #include /* must be big enough for the largest SHA variant */ diff --git a/arch/s390/crypto/sha1_s390.c b/arch/s390/crypto/sha1_s390.c index 698b1e6d3c14..a3fabf310a38 100644 --- a/arch/s390/crypto/sha1_s390.c +++ b/arch/s390/crypto/sha1_s390.c @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include #include "sha.h" diff --git a/arch/s390/crypto/sha256_s390.c b/arch/s390/crypto/sha256_s390.c index b52c87e44939..24983f175676 100644 --- a/arch/s390/crypto/sha256_s390.c +++ b/arch/s390/crypto/sha256_s390.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include "sha.h" diff --git a/arch/s390/crypto/sha3_256_s390.c b/arch/s390/crypto/sha3_256_s390.c index 460cbbbaa44a..30ac49b635bf 100644 --- a/arch/s390/crypto/sha3_256_s390.c +++ b/arch/s390/crypto/sha3_256_s390.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include diff --git a/arch/s390/crypto/sha3_512_s390.c b/arch/s390/crypto/sha3_512_s390.c index 72cf460a53e5..e70d50f7620f 100644 --- a/arch/s390/crypto/sha3_512_s390.c +++ b/arch/s390/crypto/sha3_512_s390.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include diff --git a/arch/s390/crypto/sha512_s390.c b/arch/s390/crypto/sha512_s390.c index ad29db085a18..29a6bd404c59 100644 --- a/arch/s390/crypto/sha512_s390.c +++ b/arch/s390/crypto/sha512_s390.c @@ -8,7 +8,7 @@ * Author(s): Jan Glauber (jang@de.ibm.com) */ #include -#include +#include #include #include #include diff --git a/arch/s390/include/asm/archrandom.h b/arch/s390/include/asm/archrandom.h index de61ce562052..5dc712fde3c7 100644 --- a/arch/s390/include/asm/archrandom.h +++ b/arch/s390/include/asm/archrandom.h @@ -2,7 +2,7 @@ /* * Kernel interface for the s390 arch_random_* functions * - * Copyright IBM Corp. 2017 + * Copyright IBM Corp. 2017, 2020 * * Author: Harald Freudenberger * @@ -19,10 +19,13 @@ DECLARE_STATIC_KEY_FALSE(s390_arch_random_available); extern atomic64_t s390_arch_random_counter; +bool s390_arch_get_random_long(unsigned long *v); bool s390_arch_random_generate(u8 *buf, unsigned int nbytes); static inline bool __must_check arch_get_random_long(unsigned long *v) { + if (static_branch_likely(&s390_arch_random_available)) + return s390_arch_get_random_long(v); return false; } diff --git a/arch/s390/include/asm/ccwdev.h b/arch/s390/include/asm/ccwdev.h index c0be5fe1ddba..e3e2ab0acf83 100644 --- a/arch/s390/include/asm/ccwdev.h +++ b/arch/s390/include/asm/ccwdev.h @@ -115,7 +115,7 @@ enum uc_todo { }; /** - * struct ccw driver - device driver for channel attached devices + * struct ccw_driver - device driver for channel attached devices * @ids: ids supported by this driver * @probe: function called on probe * @remove: function called on remove @@ -124,11 +124,6 @@ enum uc_todo { * @notify: notify driver of device state changes * @path_event: notify driver of channel path events * @shutdown: called at device shutdown - * @prepare: prepare for pm state transition - * @complete: undo work done in @prepare - * @freeze: callback for freezing during hibernation snapshotting - * @thaw: undo work done in @freeze - * @restore: callback for restoring after hibernation * @uc_handler: callback for unit check handler * @driver: embedded device driver structure * @int_class: interruption class to use for accounting interrupts @@ -142,11 +137,6 @@ struct ccw_driver { int (*notify) (struct ccw_device *, int); void (*path_event) (struct ccw_device *, int *); void (*shutdown) (struct ccw_device *); - int (*prepare) (struct ccw_device *); - void (*complete) (struct ccw_device *); - int (*freeze)(struct ccw_device *); - int (*thaw) (struct ccw_device *); - int (*restore)(struct ccw_device *); enum uc_todo (*uc_handler) (struct ccw_device *, struct irb *); struct device_driver driver; enum interruption_class int_class; diff --git a/arch/s390/include/asm/cio.h b/arch/s390/include/asm/cio.h index 5c58756d6476..23dceb8d0453 100644 --- a/arch/s390/include/asm/cio.h +++ b/arch/s390/include/asm/cio.h @@ -329,7 +329,7 @@ struct ccw_dev_id { }; /** - * ccw_device_id_is_equal() - compare two ccw_dev_ids + * ccw_dev_id_is_equal() - compare two ccw_dev_ids * @dev_id1: a ccw_dev_id * @dev_id2: another ccw_dev_id * Returns: diff --git a/arch/s390/include/asm/delay.h b/arch/s390/include/asm/delay.h index 898323fd93d2..4a08379cd1eb 100644 --- a/arch/s390/include/asm/delay.h +++ b/arch/s390/include/asm/delay.h @@ -13,6 +13,7 @@ #ifndef _S390_DELAY_H #define _S390_DELAY_H +void udelay_enable(void); void __ndelay(unsigned long long nsecs); void __udelay(unsigned long long usecs); void udelay_simple(unsigned long long usecs); diff --git a/arch/s390/include/asm/ftrace.h b/arch/s390/include/asm/ftrace.h index 68d362f8d6c1..695c61989f97 100644 --- a/arch/s390/include/asm/ftrace.h +++ b/arch/s390/include/asm/ftrace.h @@ -2,16 +2,9 @@ #ifndef _ASM_S390_FTRACE_H #define _ASM_S390_FTRACE_H -#define ARCH_SUPPORTS_FTRACE_OPS 1 - -#if defined(CC_USING_HOTPATCH) || defined(CC_USING_NOP_MCOUNT) -#define MCOUNT_INSN_SIZE 6 -#else -#define MCOUNT_INSN_SIZE 24 -#define MCOUNT_RETURN_FIXUP 18 -#endif - #define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR +#define ARCH_SUPPORTS_FTRACE_OPS 1 +#define MCOUNT_INSN_SIZE 6 #ifndef __ASSEMBLY__ @@ -22,7 +15,6 @@ #define ftrace_return_address(n) __builtin_return_address(n) #endif -void _mcount(void); void ftrace_caller(void); extern char ftrace_graph_caller_end; @@ -30,12 +22,20 @@ extern unsigned long ftrace_plt; struct dyn_arch_ftrace { }; -#define MCOUNT_ADDR ((unsigned long)_mcount) +#define MCOUNT_ADDR 0 #define FTRACE_ADDR ((unsigned long)ftrace_caller) #define KPROBE_ON_FTRACE_NOP 0 #define KPROBE_ON_FTRACE_CALL 1 +struct module; +struct dyn_ftrace; +/* + * Either -mhotpatch or -mnop-mcount is used - no explicit init is required + */ +static inline int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec) { return 0; } +#define ftrace_init_nop ftrace_init_nop + static inline unsigned long ftrace_call_adjust(unsigned long addr) { return addr; @@ -49,28 +49,17 @@ struct ftrace_insn { static inline void ftrace_generate_nop_insn(struct ftrace_insn *insn) { #ifdef CONFIG_FUNCTION_TRACER -#if defined(CC_USING_HOTPATCH) || defined(CC_USING_NOP_MCOUNT) /* brcl 0,0 */ insn->opc = 0xc004; insn->disp = 0; -#else - /* jg .+24 */ - insn->opc = 0xc0f4; - insn->disp = MCOUNT_INSN_SIZE / 2; -#endif #endif } static inline int is_ftrace_nop(struct ftrace_insn *insn) { #ifdef CONFIG_FUNCTION_TRACER -#if defined(CC_USING_HOTPATCH) || defined(CC_USING_NOP_MCOUNT) if (insn->disp == 0) return 1; -#else - if (insn->disp == MCOUNT_INSN_SIZE / 2) - return 1; -#endif #endif return 0; } diff --git a/arch/s390/include/asm/futex.h b/arch/s390/include/asm/futex.h index 26f9144562c9..c22debfcebf1 100644 --- a/arch/s390/include/asm/futex.h +++ b/arch/s390/include/asm/futex.h @@ -26,9 +26,7 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) { int oldval = 0, newval, ret; - mm_segment_t old_fs; - old_fs = enable_sacf_uaccess(); switch (op) { case FUTEX_OP_SET: __futex_atomic_op("lr %2,%5\n", @@ -53,7 +51,6 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, default: ret = -ENOSYS; } - disable_sacf_uaccess(old_fs); if (!ret) *oval = oldval; @@ -64,10 +61,8 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, u32 oldval, u32 newval) { - mm_segment_t old_fs; int ret; - old_fs = enable_sacf_uaccess(); asm volatile( " sacf 256\n" "0: cs %1,%4,0(%5)\n" @@ -77,7 +72,6 @@ static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, : "=d" (ret), "+d" (oldval), "=m" (*uaddr) : "0" (-EFAULT), "d" (newval), "a" (uaddr), "m" (*uaddr) : "cc", "memory"); - disable_sacf_uaccess(old_fs); *uval = oldval; return ret; } diff --git a/arch/s390/include/asm/kasan.h b/arch/s390/include/asm/kasan.h index e9bf486de136..76f351bd6645 100644 --- a/arch/s390/include/asm/kasan.h +++ b/arch/s390/include/asm/kasan.h @@ -2,28 +2,51 @@ #ifndef __ASM_KASAN_H #define __ASM_KASAN_H +#include + #ifdef CONFIG_KASAN #define KASAN_SHADOW_SCALE_SHIFT 3 -#ifdef CONFIG_KASAN_S390_4_LEVEL_PAGING #define KASAN_SHADOW_SIZE \ (_AC(1, UL) << (_REGION1_SHIFT - KASAN_SHADOW_SCALE_SHIFT)) -#else -#define KASAN_SHADOW_SIZE \ - (_AC(1, UL) << (_REGION2_SHIFT - KASAN_SHADOW_SCALE_SHIFT)) -#endif #define KASAN_SHADOW_OFFSET _AC(CONFIG_KASAN_SHADOW_OFFSET, UL) #define KASAN_SHADOW_START KASAN_SHADOW_OFFSET #define KASAN_SHADOW_END (KASAN_SHADOW_START + KASAN_SHADOW_SIZE) extern void kasan_early_init(void); -extern void kasan_copy_shadow(pgd_t *dst); +extern void kasan_copy_shadow_mapping(void); extern void kasan_free_early_identity(void); extern unsigned long kasan_vmax; + +/* + * Estimate kasan memory requirements, which it will reserve + * at the very end of available physical memory. To estimate + * that, we take into account that kasan would require + * 1/8 of available physical memory (for shadow memory) + + * creating page tables for the whole memory + shadow memory + * region (1 + 1/8). To keep page tables estimates simple take + * the double of combined ptes size. + * + * physmem parameter has to be already adjusted if not entire physical memory + * would be used (e.g. due to effect of "mem=" option). + */ +static inline unsigned long kasan_estimate_memory_needs(unsigned long physmem) +{ + unsigned long kasan_needs; + unsigned long pages; + /* for shadow memory */ + kasan_needs = round_up(physmem / 8, PAGE_SIZE); + /* for paging structures */ + pages = DIV_ROUND_UP(physmem + kasan_needs, PAGE_SIZE); + kasan_needs += DIV_ROUND_UP(pages, _PAGE_ENTRIES) * _PAGE_TABLE_SIZE * 2; + + return kasan_needs; +} #else static inline void kasan_early_init(void) { } -static inline void kasan_copy_shadow(pgd_t *dst) { } +static inline void kasan_copy_shadow_mapping(void) { } static inline void kasan_free_early_identity(void) { } +static inline unsigned long kasan_estimate_memory_needs(unsigned long physmem) { return 0; } #endif #endif diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index 612ed3c6d581..69ce9191eaf1 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -116,7 +116,7 @@ struct lowcore { /* Address space pointer. */ __u64 kernel_asce; /* 0x0380 */ __u64 user_asce; /* 0x0388 */ - __u64 vdso_asce; /* 0x0390 */ + __u8 pad_0x0390[0x0398-0x0390]; /* 0x0390 */ /* * The lpp and current_pid fields form a @@ -134,7 +134,7 @@ struct lowcore { __u32 spinlock_index; /* 0x03b0 */ __u32 fpu_flags; /* 0x03b4 */ __u64 percpu_offset; /* 0x03b8 */ - __u64 vdso_per_cpu_data; /* 0x03c0 */ + __u8 pad_0x03c0[0x03c8-0x03c0]; /* 0x03c0 */ __u64 machine_flags; /* 0x03c8 */ __u64 gmap; /* 0x03d0 */ __u8 pad_0x03d8[0x0400-0x03d8]; /* 0x03d8 */ diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h index c9f3d8a52756..5dc49c467319 100644 --- a/arch/s390/include/asm/mmu_context.h +++ b/arch/s390/include/asm/mmu_context.h @@ -71,39 +71,18 @@ static inline int init_new_context(struct task_struct *tsk, #define destroy_context(mm) do { } while (0) -static inline void set_user_asce(struct mm_struct *mm) -{ - S390_lowcore.user_asce = mm->context.asce; - __ctl_load(S390_lowcore.user_asce, 1, 1); - clear_cpu_flag(CIF_ASCE_PRIMARY); -} - -static inline void clear_user_asce(void) -{ - S390_lowcore.user_asce = S390_lowcore.kernel_asce; - __ctl_load(S390_lowcore.kernel_asce, 1, 1); - set_cpu_flag(CIF_ASCE_PRIMARY); -} - -mm_segment_t enable_sacf_uaccess(void); -void disable_sacf_uaccess(mm_segment_t old_fs); - static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, struct task_struct *tsk) { int cpu = smp_processor_id(); - S390_lowcore.user_asce = next->context.asce; + if (next == &init_mm) + S390_lowcore.user_asce = s390_invalid_asce; + else + S390_lowcore.user_asce = next->context.asce; cpumask_set_cpu(cpu, &next->context.cpu_attach_mask); - /* Clear previous user-ASCE from CR1 and CR7 */ - if (!test_cpu_flag(CIF_ASCE_PRIMARY)) { - __ctl_load(S390_lowcore.kernel_asce, 1, 1); - set_cpu_flag(CIF_ASCE_PRIMARY); - } - if (test_cpu_flag(CIF_ASCE_SECONDARY)) { - __ctl_load(S390_lowcore.vdso_asce, 7, 7); - clear_cpu_flag(CIF_ASCE_SECONDARY); - } + /* Clear previous user-ASCE from CR7 */ + __ctl_load(s390_invalid_asce, 7, 7); if (prev != next) cpumask_clear_cpu(cpu, &prev->context.cpu_attach_mask); } @@ -122,7 +101,7 @@ static inline void finish_arch_post_lock_switch(void) __tlb_flush_mm_lazy(mm); preempt_enable(); } - set_fs(current->thread.mm_segment); + __ctl_load(S390_lowcore.user_asce, 7, 7); } #define enter_lazy_tlb(mm,tsk) do { } while (0) @@ -133,7 +112,7 @@ static inline void activate_mm(struct mm_struct *prev, { switch_mm(prev, next, current); cpumask_set_cpu(smp_processor_id(), mm_cpumask(next)); - set_user_asce(next); + __ctl_load(S390_lowcore.user_asce, 7, 7); } #endif /* __S390_MMU_CONTEXT_H */ diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index b5dbae78969b..794746a32806 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -23,6 +23,7 @@ extern pgd_t swapper_pg_dir[]; extern void paging_init(void); +extern unsigned long s390_invalid_asce; enum { PG_DIRECT_MAP_4K = 0, @@ -79,15 +80,15 @@ extern unsigned long zero_page_mask; /* * The vmalloc and module area will always be on the topmost area of the - * kernel mapping. We reserve 128GB (64bit) for vmalloc and modules. - * On 64 bit kernels we have a 2GB area at the top of the vmalloc area where - * modules will reside. That makes sure that inter module branches always - * happen without trampolines and in addition the placement within a 2GB frame - * is branch prediction unit friendly. + * kernel mapping. 512GB are reserved for vmalloc by default. + * At the top of the vmalloc area a 2GB area is reserved where modules + * will reside. That makes sure that inter module branches always + * happen without trampolines and in addition the placement within a + * 2GB frame is branch prediction unit friendly. */ extern unsigned long VMALLOC_START; extern unsigned long VMALLOC_END; -#define VMALLOC_DEFAULT_SIZE ((128UL << 30) - MODULES_LEN) +#define VMALLOC_DEFAULT_SIZE ((512UL << 30) - MODULES_LEN) extern struct page *vmemmap; extern unsigned long vmemmap_size; diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index 962da04234af..6b7269f51f83 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -14,8 +14,6 @@ #include -#define CIF_ASCE_PRIMARY 0 /* primary asce needs fixup / uaccess */ -#define CIF_ASCE_SECONDARY 1 /* secondary asce needs fixup / uaccess */ #define CIF_NOHZ_DELAY 2 /* delay HZ disable for a tick */ #define CIF_FPU 3 /* restore FPU registers */ #define CIF_IGNORE_IRQ 4 /* ignore interrupt (for udelay) */ @@ -23,8 +21,6 @@ #define CIF_MCCK_GUEST 6 /* machine check happening in guest */ #define CIF_DEDICATED_CPU 7 /* this CPU is dedicated */ -#define _CIF_ASCE_PRIMARY BIT(CIF_ASCE_PRIMARY) -#define _CIF_ASCE_SECONDARY BIT(CIF_ASCE_SECONDARY) #define _CIF_NOHZ_DELAY BIT(CIF_NOHZ_DELAY) #define _CIF_FPU BIT(CIF_FPU) #define _CIF_IGNORE_IRQ BIT(CIF_IGNORE_IRQ) @@ -102,8 +98,6 @@ extern void __bpon(void); #define HAVE_ARCH_PICK_MMAP_LAYOUT -typedef unsigned int mm_segment_t; - /* * Thread structure */ @@ -116,7 +110,6 @@ struct thread_struct { unsigned long hardirq_timer; /* task cputime in hardirq context */ unsigned long softirq_timer; /* task cputime in softirq context */ unsigned long sys_call_table; /* system call table address */ - mm_segment_t mm_segment; unsigned long gmap_addr; /* address of last gmap fault. */ unsigned int gmap_write_flag; /* gmap fault write indication */ unsigned int gmap_int_code; /* int code of last gmap fault */ @@ -318,14 +311,10 @@ static __always_inline void __noreturn disabled_wait(void) } /* - * Basic Machine Check/Program Check Handler. + * Basic Program Check Handler. */ - extern void s390_base_pgm_handler(void); -extern void s390_base_ext_handler(void); - extern void (*s390_base_pgm_handler_fn)(void); -extern void (*s390_base_ext_handler_fn)(void); #define ARCH_LOW_ADDRESS_LIMIT 0x7fffffffUL diff --git a/arch/s390/include/asm/ptrace.h b/arch/s390/include/asm/ptrace.h index 16b3e4396312..73ca7f7cac33 100644 --- a/arch/s390/include/asm/ptrace.h +++ b/arch/s390/include/asm/ptrace.h @@ -87,6 +87,7 @@ struct pt_regs unsigned int int_parm; unsigned long int_parm_long; unsigned long flags; + unsigned long cr1; }; /* diff --git a/arch/s390/include/asm/sclp.h b/arch/s390/include/asm/sclp.h index a7bdd128d85b..5763769a39b6 100644 --- a/arch/s390/include/asm/sclp.h +++ b/arch/s390/include/asm/sclp.h @@ -12,7 +12,12 @@ #include #define SCLP_CHP_INFO_MASK_SIZE 32 -#define SCLP_MAX_CORES 256 +#define EARLY_SCCB_SIZE PAGE_SIZE +#define SCLP_MAX_CORES 512 +/* 144 + 16 * SCLP_MAX_CORES + 2 * (SCLP_MAX_CORES - 1) */ +#define EXT_SCCB_READ_SCP (3 * PAGE_SIZE) +/* 24 + 16 * SCLP_MAX_CORES */ +#define EXT_SCCB_READ_CPU (3 * PAGE_SIZE) struct sclp_chp_info { u8 recognized[SCLP_CHP_INFO_MASK_SIZE]; diff --git a/arch/s390/include/asm/setup.h b/arch/s390/include/asm/setup.h index bdb242a1544e..3e388fa208d4 100644 --- a/arch/s390/include/asm/setup.h +++ b/arch/s390/include/asm/setup.h @@ -16,8 +16,6 @@ #define EARLY_SCCB_OFFSET 0x11000 #define HEAD_END 0x12000 -#define EARLY_SCCB_SIZE PAGE_SIZE - /* * Machine features detected in early.c */ @@ -88,10 +86,8 @@ extern unsigned int zlib_dfltcc_support; #define ZLIB_DFLTCC_FULL_DEBUG 4 extern int noexec_disabled; -extern int memory_end_set; -extern unsigned long memory_end; +extern unsigned long ident_map_size; extern unsigned long vmalloc_size; -extern unsigned long max_physmem_end; /* The Write Back bit position in the physaddr is given by the SLPC PCI */ extern unsigned long mio_wb_bit_mask; diff --git a/arch/s390/include/asm/thread_info.h b/arch/s390/include/asm/thread_info.h index 13a04fcf7762..ce788f3e534d 100644 --- a/arch/s390/include/asm/thread_info.h +++ b/arch/s390/include/asm/thread_info.h @@ -18,7 +18,7 @@ #else #define THREAD_SIZE_ORDER 2 #endif -#define BOOT_STACK_ORDER 2 +#define BOOT_STACK_SIZE (PAGE_SIZE << 2) #define THREAD_SIZE (PAGE_SIZE << THREAD_SIZE_ORDER) #ifndef __ASSEMBLY__ diff --git a/arch/s390/include/asm/timex.h b/arch/s390/include/asm/timex.h index 289aaff4d365..c8e244ecdfde 100644 --- a/arch/s390/include/asm/timex.h +++ b/arch/s390/include/asm/timex.h @@ -49,6 +49,13 @@ static inline void set_clock_comparator(__u64 time) asm volatile("sckc %0" : : "Q" (time)); } +static inline void set_tod_programmable_field(u16 val) +{ + register unsigned long reg0 asm("0") = val; + + asm volatile("sckpf" : : "d" (reg0)); +} + void clock_comparator_work(void); void __init time_early_init(void); diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index c868e7ee49b3..c6707885e7c2 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -18,23 +18,7 @@ #include #include -/* - * The fs value determines whether argument validity checking should be - * performed or not. If get_fs() == USER_DS, checking is performed, with - * get_fs() == KERNEL_DS, checking is bypassed. - * - * For historical reasons, these macros are grossly misnamed. - */ - -#define KERNEL_DS (0) -#define KERNEL_DS_SACF (1) -#define USER_DS (2) -#define USER_DS_SACF (3) - -#define get_fs() (current->thread.mm_segment) -#define uaccess_kernel() ((get_fs() & 2) == KERNEL_DS) - -void set_fs(mm_segment_t fs); +void debug_user_asce(void); static inline int __range_ok(unsigned long addr, unsigned long size) { @@ -88,7 +72,7 @@ int __get_user_bad(void) __attribute__((noreturn)); static __always_inline int __put_user_fn(void *x, void __user *ptr, unsigned long size) { - unsigned long spec = 0x010000UL; + unsigned long spec = 0x810000UL; int rc; switch (size) { @@ -121,7 +105,7 @@ static __always_inline int __put_user_fn(void *x, void __user *ptr, unsigned lon static __always_inline int __get_user_fn(void *x, const void __user *ptr, unsigned long size) { - unsigned long spec = 0x01UL; + unsigned long spec = 0x81UL; int rc; switch (size) { diff --git a/arch/s390/include/asm/vdso.h b/arch/s390/include/asm/vdso.h index 29b44a930e71..f65590889054 100644 --- a/arch/s390/include/asm/vdso.h +++ b/arch/s390/include/asm/vdso.h @@ -12,32 +12,9 @@ #ifndef __ASSEMBLY__ -/* - * Note about the vdso_data and vdso_per_cpu_data structures: - * - * NEVER USE THEM IN USERSPACE CODE DIRECTLY. The layout of the - * structure is supposed to be known only to the function in the vdso - * itself and may change without notice. - */ - -struct vdso_per_cpu_data { - /* - * Note: node_id and cpu_nr must be at adjacent memory locations. - * VDSO userspace must read both values with a single instruction. - */ - union { - __u64 getcpu_val; - struct { - __u32 node_id; - __u32 cpu_nr; - }; - }; -}; - extern struct vdso_data *vdso_data; -int vdso_alloc_per_cpu(struct lowcore *lowcore); -void vdso_free_per_cpu(struct lowcore *lowcore); +void vdso_getcpu_init(void); #endif /* __ASSEMBLY__ */ #endif /* __S390_VDSO_H__ */ diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c index 483051e10db3..79724d861dc9 100644 --- a/arch/s390/kernel/asm-offsets.c +++ b/arch/s390/kernel/asm-offsets.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -48,6 +47,7 @@ int main(void) OFFSET(__PT_INT_PARM, pt_regs, int_parm); OFFSET(__PT_INT_PARM_LONG, pt_regs, int_parm_long); OFFSET(__PT_FLAGS, pt_regs, flags); + OFFSET(__PT_CR1, pt_regs, cr1); DEFINE(__PT_SIZE, sizeof(struct pt_regs)); BLANK(); /* stack_frame offsets */ @@ -59,8 +59,6 @@ int main(void) OFFSET(__SF_SIE_REASON, stack_frame, empty1[3]); OFFSET(__SF_SIE_FLAGS, stack_frame, empty1[4]); BLANK(); - OFFSET(__VDSO_GETCPU_VAL, vdso_per_cpu_data, getcpu_val); - BLANK(); /* idle data offsets */ OFFSET(__CLOCK_IDLE_ENTER, s390_idle_data, clock_idle_enter); OFFSET(__CLOCK_IDLE_EXIT, s390_idle_data, clock_idle_exit); @@ -138,12 +136,11 @@ int main(void) OFFSET(__LC_RESTART_FN, lowcore, restart_fn); OFFSET(__LC_RESTART_DATA, lowcore, restart_data); OFFSET(__LC_RESTART_SOURCE, lowcore, restart_source); + OFFSET(__LC_KERNEL_ASCE, lowcore, kernel_asce); OFFSET(__LC_USER_ASCE, lowcore, user_asce); - OFFSET(__LC_VDSO_ASCE, lowcore, vdso_asce); OFFSET(__LC_LPP, lowcore, lpp); OFFSET(__LC_CURRENT_PID, lowcore, current_pid); OFFSET(__LC_PERCPU_OFFSET, lowcore, percpu_offset); - OFFSET(__LC_VDSO_PER_CPU, lowcore, vdso_per_cpu_data); OFFSET(__LC_MACHINE_FLAGS, lowcore, machine_flags); OFFSET(__LC_PREEMPT_COUNT, lowcore, preempt_count); OFFSET(__LC_GMAP, lowcore, gmap); diff --git a/arch/s390/kernel/base.S b/arch/s390/kernel/base.S index b79e0fd571f8..d255c69c1779 100644 --- a/arch/s390/kernel/base.S +++ b/arch/s390/kernel/base.S @@ -11,32 +11,10 @@ #include #include #include -#include GEN_BR_THUNK %r9 GEN_BR_THUNK %r14 -ENTRY(s390_base_ext_handler) - stmg %r0,%r15,__LC_SAVE_AREA_ASYNC - basr %r13,0 -0: aghi %r15,-STACK_FRAME_OVERHEAD - larl %r1,s390_base_ext_handler_fn - lg %r9,0(%r1) - ltgr %r9,%r9 - jz 1f - BASR_EX %r14,%r9 -1: lmg %r0,%r15,__LC_SAVE_AREA_ASYNC - ni __LC_EXT_OLD_PSW+1,0xfd # clear wait state bit - lpswe __LC_EXT_OLD_PSW -ENDPROC(s390_base_ext_handler) - - .section .bss - .align 8 - .globl s390_base_ext_handler_fn -s390_base_ext_handler_fn: - .quad 0 - .previous - ENTRY(s390_base_pgm_handler) stmg %r0,%r15,__LC_SAVE_AREA_SYNC basr %r13,0 diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index 705844f73934..cc89763a4d3c 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -169,12 +169,10 @@ static noinline __init void setup_lowcore_early(void) { psw_t psw; + psw.addr = (unsigned long)s390_base_pgm_handler; psw.mask = PSW_MASK_BASE | PSW_DEFAULT_KEY | PSW_MASK_EA | PSW_MASK_BA; if (IS_ENABLED(CONFIG_KASAN)) psw.mask |= PSW_MASK_DAT; - psw.addr = (unsigned long) s390_base_ext_handler; - S390_lowcore.external_new_psw = psw; - psw.addr = (unsigned long) s390_base_pgm_handler; S390_lowcore.program_new_psw = psw; s390_base_pgm_handler_fn = early_pgm_check_handler; S390_lowcore.preempt_count = INIT_PREEMPT_COUNT; diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 92beb1444644..8bb9ebb71c4b 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -55,7 +55,7 @@ _TIF_WORK = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \ _TIF_UPROBE | _TIF_GUARDED_STORAGE | _TIF_PATCH_PENDING) _TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \ _TIF_SYSCALL_TRACEPOINT) -_CIF_WORK = (_CIF_ASCE_PRIMARY | _CIF_ASCE_SECONDARY | _CIF_FPU) +_CIF_WORK = (_CIF_FPU) _PIF_WORK = (_PIF_PER_TRAP | _PIF_SYSCALL_RESTART) _LPP_OFFSET = __LC_LPP @@ -90,6 +90,12 @@ _LPP_OFFSET = __LC_LPP #endif .endm + .macro DEBUG_USER_ASCE +#ifdef CONFIG_DEBUG_USER_ASCE + brasl %r14,debug_user_asce +#endif + .endm + .macro CHECK_VMAP_STACK savearea,oklabel #ifdef CONFIG_VMAP_STACK lgr %r14,%r15 @@ -110,9 +116,9 @@ _LPP_OFFSET = __LC_LPP #endif .endm - .macro SWITCH_ASYNC savearea,timer + .macro SWITCH_ASYNC savearea,timer,clock tmhh %r8,0x0001 # interrupting from user ? - jnz 2f + jnz 4f #if IS_ENABLED(CONFIG_KVM) lgr %r14,%r9 larl %r13,.Lsie_gmap @@ -125,10 +131,26 @@ _LPP_OFFSET = __LC_LPP #endif 0: larl %r13,.Lpsw_idle_exit cgr %r13,%r9 - jne 1f + jne 3f - mvc __CLOCK_IDLE_EXIT(8,%r2), __LC_INT_CLOCK - mvc __TIMER_IDLE_EXIT(8,%r2), __LC_ASYNC_ENTER_TIMER + larl %r1,smp_cpu_mtid + llgf %r1,0(%r1) + ltgr %r1,%r1 + jz 2f # no SMT, skip mt_cycles calculation + .insn rsy,0xeb0000000017,%r1,5,__SF_EMPTY+80(%r15) + larl %r3,mt_cycles + ag %r3,__LC_PERCPU_OFFSET + la %r4,__SF_EMPTY+16(%r15) +1: lg %r0,0(%r3) + slg %r0,0(%r4) + alg %r0,64(%r4) + stg %r0,0(%r3) + la %r3,8(%r3) + la %r4,8(%r4) + brct %r1,1b + +2: mvc __CLOCK_IDLE_EXIT(8,%r2), \clock + mvc __TIMER_IDLE_EXIT(8,%r2), \timer # account system time going idle ni __LC_CPU_FLAGS+7,255-_CIF_ENABLED_WAIT @@ -146,17 +168,17 @@ _LPP_OFFSET = __LC_LPP mvc __LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2) nihh %r8,0xfcfd # clear wait state and irq bits -1: lg %r14,__LC_ASYNC_STACK # are we already on the target stack? +3: lg %r14,__LC_ASYNC_STACK # are we already on the target stack? slgr %r14,%r15 srag %r14,%r14,STACK_SHIFT - jnz 3f + jnz 5f CHECK_STACK \savearea aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) - j 4f -2: UPDATE_VTIME %r14,%r15,\timer + j 6f +4: UPDATE_VTIME %r14,%r15,\timer BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP -3: lg %r15,__LC_ASYNC_STACK # load async stack -4: la %r11,STACK_FRAME_OVERHEAD(%r15) +5: lg %r15,__LC_ASYNC_STACK # load async stack +6: la %r11,STACK_FRAME_OVERHEAD(%r15) .endm .macro UPDATE_VTIME w1,w2,enter_timer @@ -327,7 +349,7 @@ ENTRY(sie64a) BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST) .Lsie_skip: ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE - lctlg %c1,%c1,__LC_USER_ASCE # load primary asce + lctlg %c1,%c1,__LC_KERNEL_ASCE # load primary asce .Lsie_done: # some program checks are suppressing. C code (e.g. do_protection_exception) # will rewind the PSW by the ILC, which is often 4 bytes in case of SIE. There @@ -380,6 +402,7 @@ ENTRY(system_call) lg %r12,__LC_CURRENT lghi %r14,_PIF_SYSCALL .Lsysc_per: + lctlg %c1,%c1,__LC_KERNEL_ASCE lghi %r13,__TASK_thread lg %r15,__LC_KERNEL_STACK la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs @@ -427,11 +450,9 @@ ENTRY(system_call) jnz .Lsysc_work TSTMSK __TI_flags(%r12),_TIF_WORK jnz .Lsysc_work # check for work - TSTMSK __LC_CPU_FLAGS,(_CIF_WORK-_CIF_FPU) - jnz .Lsysc_work + DEBUG_USER_ASCE + lctlg %c1,%c1,__LC_USER_ASCE BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP -.Lsysc_restore: - DISABLE_INTS TSTMSK __LC_CPU_FLAGS, _CIF_FPU jz .Lsysc_skip_fpu brasl %r14,load_fpu_regs @@ -469,8 +490,6 @@ ENTRY(system_call) jo .Lsysc_sigpending TSTMSK __TI_flags(%r12),_TIF_NOTIFY_RESUME jo .Lsysc_notify_resume - TSTMSK __LC_CPU_FLAGS,(_CIF_ASCE_PRIMARY|_CIF_ASCE_SECONDARY) - jnz .Lsysc_asce j .Lsysc_return # @@ -480,26 +499,6 @@ ENTRY(system_call) larl %r14,.Lsysc_return jg schedule -# -# _CIF_ASCE_PRIMARY and/or _CIF_ASCE_SECONDARY set, load user space asce -# -.Lsysc_asce: - ni __LC_CPU_FLAGS+7,255-_CIF_ASCE_SECONDARY - lctlg %c7,%c7,__LC_VDSO_ASCE # load secondary asce - TSTMSK __LC_CPU_FLAGS,_CIF_ASCE_PRIMARY - jz .Lsysc_return -#ifndef CONFIG_HAVE_MARCH_Z10_FEATURES - tm __LC_STFLE_FAC_LIST+3,0x10 # has MVCOS ? - jnz .Lsysc_set_fs_fixup - ni __LC_CPU_FLAGS+7,255-_CIF_ASCE_PRIMARY - lctlg %c1,%c1,__LC_USER_ASCE # load primary asce - j .Lsysc_return -.Lsysc_set_fs_fixup: -#endif - larl %r14,.Lsysc_return - jg set_fs_fixup - - # # _TIF_SIGPENDING is set, call do_signal # @@ -636,8 +635,11 @@ ENTRY(pgm_check_handler) 0: lg %r12,__LC_CURRENT lghi %r11,0 lmg %r8,%r9,__LC_PGM_OLD_PSW - tmhh %r8,0x0001 # test problem state bit - jnz 3f # -> fault in user space + tmhh %r8,0x0001 # coming from user space? + jno .Lpgm_skip_asce + lctlg %c1,%c1,__LC_KERNEL_ASCE + j 3f +.Lpgm_skip_asce: #if IS_ENABLED(CONFIG_KVM) # cleanup critical section for program checks in sie64a lgr %r14,%r9 @@ -648,7 +650,7 @@ ENTRY(pgm_check_handler) jhe 1f lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE - lctlg %c1,%c1,__LC_USER_ASCE # load primary asce + lctlg %c1,%c1,__LC_KERNEL_ASCE # load primary asce larl %r9,sie_exit # skip forward to sie_exit lghi %r11,_PIF_GUEST_FAULT #endif @@ -709,10 +711,20 @@ ENTRY(pgm_check_handler) .Lpgm_return: LOCKDEP_SYS_EXIT tm __PT_PSW+1(%r11),0x01 # returning to user ? - jno .Lsysc_restore + jno .Lpgm_restore TSTMSK __PT_FLAGS(%r11),_PIF_SYSCALL jo .Lsysc_do_syscall j .Lsysc_tif +.Lpgm_restore: + DISABLE_INTS + TSTMSK __LC_CPU_FLAGS, _CIF_FPU + jz .Lpgm_skip_fpu + brasl %r14,load_fpu_regs +.Lpgm_skip_fpu: + mvc __LC_RETURN_PSW(16),__PT_PSW(%r11) + stpt __LC_EXIT_TIMER + lmg %r0,%r15,__PT_R0(%r11) + b __LC_RETURN_LPSWE # # PER event in supervisor state, must be kprobes @@ -745,7 +757,7 @@ ENTRY(io_int_handler) stmg %r8,%r15,__LC_SAVE_AREA_ASYNC lg %r12,__LC_CURRENT lmg %r8,%r9,__LC_IO_OLD_PSW - SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER + SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER,__LC_INT_CLOCK stmg %r0,%r7,__PT_R0(%r11) # clear user controlled registers to prevent speculative use xgr %r0,%r0 @@ -759,6 +771,10 @@ ENTRY(io_int_handler) xgr %r10,%r10 mvc __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC stmg %r8,%r9,__PT_PSW(%r11) + tm __PT_PSW+1(%r11),0x01 # coming from user space? + jno .Lio_skip_asce + lctlg %c1,%c1,__LC_KERNEL_ASCE +.Lio_skip_asce: mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) TSTMSK __LC_CPU_FLAGS,_CIF_IGNORE_IRQ @@ -790,6 +806,8 @@ ENTRY(io_int_handler) mvc __LC_RETURN_PSW(16),__PT_PSW(%r11) tm __PT_PSW+1(%r11),0x01 # returning to user ? jno .Lio_exit_kernel + DEBUG_USER_ASCE + lctlg %c1,%c1,__LC_USER_ASCE BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP stpt __LC_EXIT_TIMER .Lio_exit_kernel: @@ -855,29 +873,8 @@ ENTRY(io_int_handler) jo .Lio_guarded_storage TSTMSK __LC_CPU_FLAGS,_CIF_FPU jo .Lio_vxrs - TSTMSK __LC_CPU_FLAGS,(_CIF_ASCE_PRIMARY|_CIF_ASCE_SECONDARY) - jnz .Lio_asce j .Lio_return -# -# _CIF_ASCE_PRIMARY and/or CIF_ASCE_SECONDARY set, load user space asce -# -.Lio_asce: - ni __LC_CPU_FLAGS+7,255-_CIF_ASCE_SECONDARY - lctlg %c7,%c7,__LC_VDSO_ASCE # load secondary asce - TSTMSK __LC_CPU_FLAGS,_CIF_ASCE_PRIMARY - jz .Lio_return -#ifndef CONFIG_HAVE_MARCH_Z10_FEATURES - tm __LC_STFLE_FAC_LIST+3,0x10 # has MVCOS ? - jnz .Lio_set_fs_fixup - ni __LC_CPU_FLAGS+7,255-_CIF_ASCE_PRIMARY - lctlg %c1,%c1,__LC_USER_ASCE # load primary asce - j .Lio_return -.Lio_set_fs_fixup: -#endif - larl %r14,.Lio_return - jg set_fs_fixup - # # CIF_FPU is set, restore floating-point controls and floating-point registers. # @@ -945,7 +942,7 @@ ENTRY(ext_int_handler) stmg %r8,%r15,__LC_SAVE_AREA_ASYNC lg %r12,__LC_CURRENT lmg %r8,%r9,__LC_EXT_OLD_PSW - SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER + SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER,__LC_INT_CLOCK stmg %r0,%r7,__PT_R0(%r11) # clear user controlled registers to prevent speculative use xgr %r0,%r0 @@ -959,6 +956,10 @@ ENTRY(ext_int_handler) xgr %r10,%r10 mvc __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC stmg %r8,%r9,__PT_PSW(%r11) + tm __PT_PSW+1(%r11),0x01 # coming from user space? + jno .Lext_skip_asce + lctlg %c1,%c1,__LC_KERNEL_ASCE +.Lext_skip_asce: lghi %r1,__LC_EXT_PARAMS2 mvc __PT_INT_CODE(4,%r11),__LC_EXT_CPU_ADDR mvc __PT_INT_PARM(4,%r11),__LC_EXT_PARAMS @@ -1167,7 +1168,7 @@ ENTRY(mcck_int_handler) TSTMSK __LC_MCCK_CODE,MCCK_CODE_PSW_IA_VALID jno .Lmcck_panic 4: ssm __LC_PGM_NEW_PSW # turn dat on, keep irqs off - SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+64,__LC_MCCK_ENTER_TIMER + SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+64,__LC_MCCK_ENTER_TIMER,__LC_MCCK_CLOCK .Lmcck_skip: lghi %r14,__LC_GPREGS_SAVE_AREA+64 stmg %r0,%r7,__PT_R0(%r11) @@ -1183,6 +1184,9 @@ ENTRY(mcck_int_handler) xgr %r10,%r10 mvc __PT_R8(64,%r11),0(%r14) stmg %r8,%r9,__PT_PSW(%r11) + la %r14,4095 + mvc __PT_CR1(8,%r11),__LC_CREGS_SAVE_AREA-4095+8(%r14) + lctlg %c1,%c1,__LC_KERNEL_ASCE xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) lgr %r2,%r11 # pass pointer to pt_regs @@ -1198,6 +1202,7 @@ ENTRY(mcck_int_handler) brasl %r14,s390_handle_mcck TRACE_IRQS_ON .Lmcck_return: + lctlg %c1,%c1,__PT_CR1(%r11) lmg %r0,%r10,__PT_R0(%r11) mvc __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ? @@ -1274,7 +1279,7 @@ ENDPROC(stack_overflow) 1: BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST) lg %r9,__SF_SIE_CONTROL(%r15) # get control block pointer ni __SIE_PROG0C+3(%r9),0xfe # no longer in SIE - lctlg %c1,%c1,__LC_USER_ASCE # load primary asce + lctlg %c1,%c1,__LC_KERNEL_ASCE larl %r9,sie_exit # skip forward to sie_exit BR_EX %r14,%r11 diff --git a/arch/s390/kernel/entry.h b/arch/s390/kernel/entry.h index d2ca3fe51f8e..a16c33b32ab0 100644 --- a/arch/s390/kernel/entry.h +++ b/arch/s390/kernel/entry.h @@ -83,7 +83,6 @@ long sys_s390_sthyi(unsigned long function_code, void __user *buffer, u64 __user DECLARE_PER_CPU(u64, mt_cycles[8]); void gs_load_bc_cb(struct pt_regs *regs); -void set_fs_fixup(void); unsigned long stack_alloc(void); void stack_free(unsigned long stack); diff --git a/arch/s390/kernel/ftrace.c b/arch/s390/kernel/ftrace.c index b388e87a08bf..ebc1284a618b 100644 --- a/arch/s390/kernel/ftrace.c +++ b/arch/s390/kernel/ftrace.c @@ -22,56 +22,26 @@ #include "entry.h" /* - * The mcount code looks like this: - * stg %r14,8(%r15) # offset 0 - * larl %r1,<&counter> # offset 6 - * brasl %r14,_mcount # offset 12 - * lg %r14,8(%r15) # offset 18 - * Total length is 24 bytes. Only the first instruction will be patched - * by ftrace_make_call / ftrace_make_nop. - * The enabled ftrace code block looks like this: + * To generate function prologue either gcc's hotpatch feature (since gcc 4.8) + * or a combination of -pg -mrecord-mcount -mnop-mcount -mfentry flags + * (since gcc 9 / clang 10) is used. + * In both cases the original and also the disabled function prologue contains + * only a single six byte instruction and looks like this: + * > brcl 0,0 # offset 0 + * To enable ftrace the code gets patched like above and afterwards looks + * like this: * > brasl %r0,ftrace_caller # offset 0 - * larl %r1,<&counter> # offset 6 - * brasl %r14,_mcount # offset 12 - * lg %r14,8(%r15) # offset 18 + * + * The instruction will be patched by ftrace_make_call / ftrace_make_nop. * The ftrace function gets called with a non-standard C function call ABI * where r0 contains the return address. It is also expected that the called * function only clobbers r0 and r1, but restores r2-r15. * For module code we can't directly jump to ftrace caller, but need a * trampoline (ftrace_plt), which clobbers also r1. - * The return point of the ftrace function has offset 24, so execution - * continues behind the mcount block. - * The disabled ftrace code block looks like this: - * > jg .+24 # offset 0 - * larl %r1,<&counter> # offset 6 - * brasl %r14,_mcount # offset 12 - * lg %r14,8(%r15) # offset 18 - * The jg instruction branches to offset 24 to skip as many instructions - * as possible. - * In case we use gcc's hotpatch feature the original and also the disabled - * function prologue contains only a single six byte instruction and looks - * like this: - * > brcl 0,0 # offset 0 - * To enable ftrace the code gets patched like above and afterwards looks - * like this: - * > brasl %r0,ftrace_caller # offset 0 */ unsigned long ftrace_plt; -static inline void ftrace_generate_orig_insn(struct ftrace_insn *insn) -{ -#if defined(CC_USING_HOTPATCH) || defined(CC_USING_NOP_MCOUNT) - /* brcl 0,0 */ - insn->opc = 0xc004; - insn->disp = 0; -#else - /* stg r14,8(r15) */ - insn->opc = 0xe3e0; - insn->disp = 0xf0080024; -#endif -} - int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr) { @@ -85,15 +55,10 @@ int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, if (copy_from_kernel_nofault(&old, (void *) rec->ip, sizeof(old))) return -EFAULT; - if (addr == MCOUNT_ADDR) { - /* Initial code replacement */ - ftrace_generate_orig_insn(&orig); - ftrace_generate_nop_insn(&new); - } else { - /* Replace ftrace call with a nop. */ - ftrace_generate_call_insn(&orig, rec->ip); - ftrace_generate_nop_insn(&new); - } + /* Replace ftrace call with a nop. */ + ftrace_generate_call_insn(&orig, rec->ip); + ftrace_generate_nop_insn(&new); + /* Verify that the to be replaced code matches what we expect. */ if (memcmp(&orig, &old, sizeof(old))) return -EINVAL; diff --git a/arch/s390/kernel/head64.S b/arch/s390/kernel/head64.S index 8b88dbbda7df..0c253886da78 100644 --- a/arch/s390/kernel/head64.S +++ b/arch/s390/kernel/head64.S @@ -18,12 +18,7 @@ __HEAD ENTRY(startup_continue) - tm __LC_STFLE_FAC_LIST+5,0x80 # LPP available ? - jz 0f - xc __LC_LPP+1(7,0),__LC_LPP+1 # clear lpp and current_pid - mvi __LC_LPP,0x80 # and set LPP_MAGIC - .insn s,0xb2800000,__LC_LPP # load program parameter -0: larl %r1,tod_clock_base + larl %r1,tod_clock_base mvc 0(16,%r1),__LC_BOOT_CLOCK larl %r13,.LPG1 # get base # diff --git a/arch/s390/kernel/mcount.S b/arch/s390/kernel/mcount.S index 7458dcfd6464..faf64c2f90f5 100644 --- a/arch/s390/kernel/mcount.S +++ b/arch/s390/kernel/mcount.S @@ -33,11 +33,6 @@ ENDPROC(ftrace_stub) #define TRACED_FUNC_FRAME_SIZE STACK_FRAME_OVERHEAD #endif -ENTRY(_mcount) - BR_EX %r14 -ENDPROC(_mcount) -EXPORT_SYMBOL(_mcount) - ENTRY(ftrace_caller) .globl ftrace_regs_caller .set ftrace_regs_caller,ftrace_caller @@ -46,9 +41,6 @@ ENTRY(ftrace_caller) ipm %r14 # don't put any instructions sllg %r14,%r14,16 # clobbering CC before this point lgr %r1,%r15 -#if !(defined(CC_USING_HOTPATCH) || defined(CC_USING_NOP_MCOUNT)) - aghi %r0,MCOUNT_RETURN_FIXUP -#endif # allocate stack frame for ftrace_caller to contain traced function aghi %r15,-TRACED_FUNC_FRAME_SIZE stg %r1,__SF_BACKCHAIN(%r15) diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index ec801d3bbb37..bc3ca54edfb4 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -94,7 +94,6 @@ int copy_thread(unsigned long clone_flags, unsigned long new_stackp, /* Save access registers to new thread structure. */ save_access_regs(&p->thread.acrs[0]); /* start new process with ar4 pointing to the correct address space */ - p->thread.mm_segment = get_fs(); /* Don't copy debug registers */ memset(&p->thread.per_user, 0, sizeof(p->thread.per_user)); memset(&p->thread.per_event, 0, sizeof(p->thread.per_event)); @@ -208,16 +207,3 @@ unsigned long arch_randomize_brk(struct mm_struct *mm) ret = PAGE_ALIGN(mm->brk + brk_rnd()); return (ret > mm->brk) ? ret : mm->brk; } - -void set_fs_fixup(void) -{ - struct pt_regs *regs = current_pt_regs(); - static bool warned; - - set_fs(USER_DS); - if (warned) - return; - WARN(1, "Unbalanced set_fs - int code: 0x%x\n", regs->int_code); - show_registers(regs); - warned = true; -} diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 4d843e64496f..1f16a03be995 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -49,6 +49,7 @@ #include #include #include +#include #include #include @@ -94,10 +95,8 @@ char elf_platform[ELF_PLATFORM_SIZE]; unsigned long int_hwcap = 0; int __bootdata(noexec_disabled); -int __bootdata(memory_end_set); -unsigned long __bootdata(memory_end); +unsigned long __bootdata(ident_map_size); unsigned long __bootdata(vmalloc_size); -unsigned long __bootdata(max_physmem_end); struct mem_detect_info __bootdata(mem_detect); struct exception_table_entry *__bootdata_preserved(__start_dma_ex_table); @@ -336,6 +335,7 @@ int __init arch_early_irq_init(void) if (!stack) panic("Couldn't allocate async stack"); S390_lowcore.async_stack = stack + STACK_INIT_OFFSET; + udelay_enable(); return 0; } @@ -556,24 +556,25 @@ static void __init setup_resources(void) #endif } -static void __init setup_memory_end(void) +static void __init setup_ident_map_size(void) { unsigned long vmax, tmp; /* Choose kernel address space layout: 3 or 4 levels. */ - tmp = (memory_end ?: max_physmem_end) / PAGE_SIZE; + tmp = ident_map_size / PAGE_SIZE; tmp = tmp * (sizeof(struct page) + PAGE_SIZE); if (tmp + vmalloc_size + MODULES_LEN <= _REGION2_SIZE) vmax = _REGION2_SIZE; /* 3-level kernel page table */ else vmax = _REGION1_SIZE; /* 4-level kernel page table */ - if (is_prot_virt_host()) - adjust_to_uv_max(&vmax); -#ifdef CONFIG_KASAN - vmax = kasan_vmax; -#endif /* module area is at the end of the kernel address space. */ MODULES_END = vmax; + if (is_prot_virt_host()) + adjust_to_uv_max(&MODULES_END); +#ifdef CONFIG_KASAN + vmax = _REGION1_SIZE; + MODULES_END = kasan_vmax; +#endif MODULES_VADDR = MODULES_END - MODULES_LEN; VMALLOC_END = MODULES_VADDR; VMALLOC_START = VMALLOC_END - vmalloc_size; @@ -587,22 +588,22 @@ static void __init setup_memory_end(void) tmp = min(tmp, 1UL << MAX_PHYSMEM_BITS); vmemmap = (struct page *) tmp; - /* Take care that memory_end is set and <= vmemmap */ - memory_end = min(memory_end ?: max_physmem_end, (unsigned long)vmemmap); + /* Take care that ident_map_size <= vmemmap */ + ident_map_size = min(ident_map_size, (unsigned long)vmemmap); #ifdef CONFIG_KASAN - memory_end = min(memory_end, KASAN_SHADOW_START); + ident_map_size = min(ident_map_size, KASAN_SHADOW_START); #endif - vmemmap_size = SECTION_ALIGN_UP(memory_end / PAGE_SIZE) * sizeof(struct page); + vmemmap_size = SECTION_ALIGN_UP(ident_map_size / PAGE_SIZE) * sizeof(struct page); #ifdef CONFIG_KASAN /* move vmemmap above kasan shadow only if stands in a way */ if (KASAN_SHADOW_END > (unsigned long)vmemmap && (unsigned long)vmemmap + vmemmap_size > KASAN_SHADOW_START) vmemmap = max(vmemmap, (struct page *)KASAN_SHADOW_END); #endif - max_pfn = max_low_pfn = PFN_DOWN(memory_end); - memblock_remove(memory_end, ULONG_MAX); + max_pfn = max_low_pfn = PFN_DOWN(ident_map_size); + memblock_remove(ident_map_size, ULONG_MAX); - pr_notice("The maximum memory size is %luMB\n", memory_end >> 20); + pr_notice("The maximum memory size is %luMB\n", ident_map_size >> 20); } #ifdef CONFIG_CRASH_DUMP @@ -632,12 +633,11 @@ static struct notifier_block kdump_mem_nb = { #endif /* - * Make sure that the area behind memory_end is protected + * Make sure that the area above identity mapping is protected */ -static void __init reserve_memory_end(void) +static void __init reserve_above_ident_map(void) { - if (memory_end_set) - memblock_reserve(memory_end, ULONG_MAX); + memblock_reserve(ident_map_size, ULONG_MAX); } /* @@ -674,7 +674,7 @@ static void __init reserve_crashkernel(void) phys_addr_t low, high; int rc; - rc = parse_crashkernel(boot_command_line, memory_end, &crash_size, + rc = parse_crashkernel(boot_command_line, ident_map_size, &crash_size, &crash_base); crash_base = ALIGN(crash_base, KEXEC_CRASH_MEM_ALIGN); @@ -1128,7 +1128,7 @@ void __init setup_arch(char **cmdline_p) setup_control_program_code(); /* Do some memory reservations *before* memory is added to memblock */ - reserve_memory_end(); + reserve_above_ident_map(); reserve_oldmem(); reserve_kernel(); reserve_initrd(); @@ -1143,10 +1143,12 @@ void __init setup_arch(char **cmdline_p) remove_oldmem(); setup_uv(); - setup_memory_end(); + setup_ident_map_size(); setup_memory(); - dma_contiguous_reserve(memory_end); + dma_contiguous_reserve(ident_map_size); vmcp_cma_reserve(); + if (MACHINE_HAS_EDAT2) + hugetlb_cma_reserve(PUD_SHIFT - PAGE_SHIFT); check_initrd(); reserve_crashkernel(); diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 390d97daa2b3..27c763014114 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -47,7 +47,6 @@ #include #include #include -#include #include #include #include @@ -55,6 +54,7 @@ #include #include #include +#include #include "entry.h" enum { @@ -217,14 +217,10 @@ static int pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu) lc->return_mcck_lpswe = gen_lpswe(__LC_RETURN_MCCK_PSW); if (nmi_alloc_per_cpu(lc)) goto out_async; - if (vdso_alloc_per_cpu(lc)) - goto out_mcesa; lowcore_ptr[cpu] = lc; pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, (u32)(unsigned long) lc); return 0; -out_mcesa: - nmi_free_per_cpu(lc); out_async: stack_free(async_stack); out: @@ -245,7 +241,6 @@ static void pcpu_free_lowcore(struct pcpu *pcpu) pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, 0); lowcore_ptr[pcpu - pcpu_devices] = NULL; - vdso_free_per_cpu(pcpu->lowcore); nmi_free_per_cpu(pcpu->lowcore); stack_free(async_stack); if (pcpu == &pcpu_devices[0]) @@ -265,13 +260,13 @@ static void pcpu_prepare_secondary(struct pcpu *pcpu, int cpu) lc->spinlock_index = 0; lc->percpu_offset = __per_cpu_offset[cpu]; lc->kernel_asce = S390_lowcore.kernel_asce; - lc->user_asce = S390_lowcore.kernel_asce; + lc->user_asce = s390_invalid_asce; lc->machine_flags = S390_lowcore.machine_flags; lc->user_timer = lc->system_timer = lc->steal_timer = lc->avg_steal_timer = 0; __ctl_store(lc->cregs_save_area, 0, 15); lc->cregs_save_area[1] = lc->kernel_asce; - lc->cregs_save_area[7] = lc->vdso_asce; + lc->cregs_save_area[7] = lc->user_asce; save_access_regs((unsigned int *) lc->access_regs_save_area); memcpy(lc->stfle_fac_list, S390_lowcore.stfle_fac_list, sizeof(lc->stfle_fac_list)); @@ -859,13 +854,12 @@ static void smp_init_secondary(void) S390_lowcore.last_update_clock = get_tod_clock(); restore_access_regs(S390_lowcore.access_regs_save_area); - set_cpu_flag(CIF_ASCE_PRIMARY); - set_cpu_flag(CIF_ASCE_SECONDARY); cpu_init(); rcu_cpu_starting(cpu); preempt_disable(); init_cpu_timer(); vtime_init(); + vdso_getcpu_init(); pfault_init(); notify_cpu_starting(cpu); if (topology_cpu_dedicated(cpu)) @@ -896,24 +890,12 @@ static void __no_sanitize_address smp_start_secondary(void *cpuvoid) /* Upping and downing of CPUs */ int __cpu_up(unsigned int cpu, struct task_struct *tidle) { - struct pcpu *pcpu; - int base, i, rc; + struct pcpu *pcpu = pcpu_devices + cpu; + int rc; - pcpu = pcpu_devices + cpu; if (pcpu->state != CPU_STATE_CONFIGURED) return -EIO; - base = smp_get_base_cpu(cpu); - for (i = 0; i <= smp_cpu_mtid; i++) { - if (base + i < nr_cpu_ids) - if (cpu_online(base + i)) - break; - } - /* - * If this is the first CPU of the core to get online - * do an initial CPU reset. - */ - if (i > smp_cpu_mtid && - pcpu_sigp_retry(pcpu_devices + base, SIGP_INITIAL_CPU_RESET, 0) != + if (pcpu_sigp_retry(pcpu, SIGP_INITIAL_CPU_RESET, 0) != SIGP_CC_ORDER_CODE_ACCEPTED) return -EIO; diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index 0ac30ee2c633..c59cb44fbb7d 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -927,41 +927,25 @@ static ssize_t online_store(struct device *dev, */ static DEVICE_ATTR_RW(online); -static struct device_attribute *stp_attributes[] = { - &dev_attr_ctn_id, - &dev_attr_ctn_type, - &dev_attr_dst_offset, - &dev_attr_leap_seconds, - &dev_attr_online, - &dev_attr_leap_seconds_scheduled, - &dev_attr_stratum, - &dev_attr_time_offset, - &dev_attr_time_zone_offset, - &dev_attr_timing_mode, - &dev_attr_timing_state, +static struct attribute *stp_dev_attrs[] = { + &dev_attr_ctn_id.attr, + &dev_attr_ctn_type.attr, + &dev_attr_dst_offset.attr, + &dev_attr_leap_seconds.attr, + &dev_attr_online.attr, + &dev_attr_leap_seconds_scheduled.attr, + &dev_attr_stratum.attr, + &dev_attr_time_offset.attr, + &dev_attr_time_zone_offset.attr, + &dev_attr_timing_mode.attr, + &dev_attr_timing_state.attr, NULL }; +ATTRIBUTE_GROUPS(stp_dev); static int __init stp_init_sysfs(void) { - struct device_attribute **attr; - int rc; - - rc = subsys_system_register(&stp_subsys, NULL); - if (rc) - goto out; - for (attr = stp_attributes; *attr; attr++) { - rc = device_create_file(stp_subsys.dev_root, *attr); - if (rc) - goto out_unreg; - } - return 0; -out_unreg: - for (; attr >= stp_attributes; attr--) - device_remove_file(stp_subsys.dev_root, *attr); - bus_unregister(&stp_subsys); -out: - return rc; + return subsys_system_register(&stp_subsys, stp_dev_groups); } device_initcall(stp_init_sysfs); diff --git a/arch/s390/kernel/vdso.c b/arch/s390/kernel/vdso.c index f9da5b149141..aef2edff9959 100644 --- a/arch/s390/kernel/vdso.c +++ b/arch/s390/kernel/vdso.c @@ -29,6 +29,7 @@ #include #include #include +#include extern char vdso64_start, vdso64_end; static void *vdso64_kbase = &vdso64_start; @@ -99,60 +100,10 @@ static union { u8 page[PAGE_SIZE]; } vdso_data_store __page_aligned_data; struct vdso_data *vdso_data = (struct vdso_data *)&vdso_data_store.data; -/* - * Allocate/free per cpu vdso data. - */ -#define SEGMENT_ORDER 2 -int vdso_alloc_per_cpu(struct lowcore *lowcore) +void vdso_getcpu_init(void) { - unsigned long segment_table, page_table, page_frame; - struct vdso_per_cpu_data *vd; - - segment_table = __get_free_pages(GFP_KERNEL, SEGMENT_ORDER); - page_table = get_zeroed_page(GFP_KERNEL); - page_frame = get_zeroed_page(GFP_KERNEL); - if (!segment_table || !page_table || !page_frame) - goto out; - arch_set_page_dat(virt_to_page(segment_table), SEGMENT_ORDER); - arch_set_page_dat(virt_to_page(page_table), 0); - - /* Initialize per-cpu vdso data page */ - vd = (struct vdso_per_cpu_data *) page_frame; - vd->cpu_nr = lowcore->cpu_nr; - vd->node_id = cpu_to_node(vd->cpu_nr); - - /* Set up page table for the vdso address space */ - memset64((u64 *)segment_table, _SEGMENT_ENTRY_EMPTY, _CRST_ENTRIES); - memset64((u64 *)page_table, _PAGE_INVALID, PTRS_PER_PTE); - - *(unsigned long *) segment_table = _SEGMENT_ENTRY + page_table; - *(unsigned long *) page_table = _PAGE_PROTECT + page_frame; - - lowcore->vdso_asce = segment_table + - _ASCE_TABLE_LENGTH + _ASCE_USER_BITS + _ASCE_TYPE_SEGMENT; - lowcore->vdso_per_cpu_data = page_frame; - - return 0; - -out: - free_page(page_frame); - free_page(page_table); - free_pages(segment_table, SEGMENT_ORDER); - return -ENOMEM; -} - -void vdso_free_per_cpu(struct lowcore *lowcore) -{ - unsigned long segment_table, page_table, page_frame; - - segment_table = lowcore->vdso_asce & PAGE_MASK; - page_table = *(unsigned long *) segment_table; - page_frame = *(unsigned long *) page_table; - - free_page(page_frame); - free_page(page_table); - free_pages(segment_table, SEGMENT_ORDER); + set_tod_programmable_field(smp_processor_id()); } /* @@ -225,6 +176,7 @@ static int __init vdso_init(void) { int i; + vdso_getcpu_init(); /* Calculate the size of the 64 bit vDSO */ vdso64_pages = ((&vdso64_end - &vdso64_start + PAGE_SIZE - 1) >> PAGE_SHIFT) + 1; @@ -240,8 +192,6 @@ static int __init vdso_init(void) } vdso64_pagelist[vdso64_pages - 1] = virt_to_page(vdso_data); vdso64_pagelist[vdso64_pages] = NULL; - if (vdso_alloc_per_cpu(&S390_lowcore)) - BUG(); get_page(virt_to_page(vdso_data)); diff --git a/arch/s390/kernel/vdso64/Makefile b/arch/s390/kernel/vdso64/Makefile index 13cc5a3f9abf..a6e0fb6b91d6 100644 --- a/arch/s390/kernel/vdso64/Makefile +++ b/arch/s390/kernel/vdso64/Makefile @@ -6,8 +6,9 @@ ARCH_REL_TYPE_ABS := R_390_COPY|R_390_GLOB_DAT|R_390_JMP_SLOT|R_390_RELATIVE ARCH_REL_TYPE_ABS += R_390_GOT|R_390_PLT include $(srctree)/lib/vdso/Makefile -obj-vdso64 = vdso_user_wrapper.o note.o getcpu.o -obj-cvdso64 = vdso64_generic.o +obj-vdso64 = vdso_user_wrapper.o note.o +obj-cvdso64 = vdso64_generic.o getcpu.o +CFLAGS_REMOVE_getcpu.o = -pg $(CC_FLAGS_FTRACE) $(CC_FLAGS_EXPOLINE) CFLAGS_REMOVE_vdso64_generic.o = -pg $(CC_FLAGS_FTRACE) $(CC_FLAGS_EXPOLINE) # Build rules diff --git a/arch/s390/kernel/vdso64/getcpu.S b/arch/s390/kernel/vdso64/getcpu.S deleted file mode 100644 index 3c04f7328500..000000000000 --- a/arch/s390/kernel/vdso64/getcpu.S +++ /dev/null @@ -1,31 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Userland implementation of getcpu() for 64 bits processes in a - * s390 kernel for use in the vDSO - * - * Copyright IBM Corp. 2016 - * Author(s): Martin Schwidefsky - */ -#include -#include -#include - - .text - .align 4 - .globl __kernel_getcpu - .type __kernel_getcpu,@function -__kernel_getcpu: - CFI_STARTPROC - sacf 256 - lm %r4,%r5,__VDSO_GETCPU_VAL(%r0) - sacf 0 - ltgr %r2,%r2 - jz 2f - st %r5,0(%r2) -2: ltgr %r3,%r3 - jz 3f - st %r4,0(%r3) -3: lghi %r2,0 - br %r14 - CFI_ENDPROC - .size __kernel_getcpu,.-__kernel_getcpu diff --git a/arch/s390/kernel/vdso64/getcpu.c b/arch/s390/kernel/vdso64/getcpu.c new file mode 100644 index 000000000000..5b2bc7494d5b --- /dev/null +++ b/arch/s390/kernel/vdso64/getcpu.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright IBM Corp. 2020 */ + +#include +#include +#include +#include "vdso.h" + +int __s390_vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *unused) +{ + __u16 todval[8]; + + /* CPU number is stored in the programmable field of the TOD clock */ + get_tod_clock_ext((char *)todval); + if (cpu) + *cpu = todval[7]; + /* NUMA node is always zero */ + if (node) + *node = 0; + return 0; +} diff --git a/arch/s390/kernel/vdso64/vdso.h b/arch/s390/kernel/vdso64/vdso.h new file mode 100644 index 000000000000..34c7a2312f9d --- /dev/null +++ b/arch/s390/kernel/vdso64/vdso.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ARCH_S390_KERNEL_VDSO64_VDSO_H +#define __ARCH_S390_KERNEL_VDSO64_VDSO_H + +#include + +struct getcpu_cache; + +int __s390_vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *unused); +int __s390_vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz); +int __s390_vdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts); +int __s390_vdso_clock_getres(clockid_t clock, struct __kernel_timespec *ts); + +#endif /* __ARCH_S390_KERNEL_VDSO64_VDSO_H */ diff --git a/arch/s390/kernel/vdso64/vdso64.lds.S b/arch/s390/kernel/vdso64/vdso64.lds.S index 7ddb116b5e2e..7bde3909290f 100644 --- a/arch/s390/kernel/vdso64/vdso64.lds.S +++ b/arch/s390/kernel/vdso64/vdso64.lds.S @@ -136,7 +136,6 @@ VERSION __kernel_clock_gettime; __kernel_clock_getres; __kernel_getcpu; - local: *; }; } diff --git a/arch/s390/kernel/vdso64/vdso64_generic.c b/arch/s390/kernel/vdso64/vdso64_generic.c index a8cef7e4d137..a9aa75643c08 100644 --- a/arch/s390/kernel/vdso64/vdso64_generic.c +++ b/arch/s390/kernel/vdso64/vdso64_generic.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #include "../../../../lib/vdso/gettimeofday.c" +#include "vdso.h" int __s390_vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) diff --git a/arch/s390/kernel/vdso64/vdso_user_wrapper.S b/arch/s390/kernel/vdso64/vdso_user_wrapper.S index a775d7e52872..f773505c7e63 100644 --- a/arch/s390/kernel/vdso64/vdso_user_wrapper.S +++ b/arch/s390/kernel/vdso64/vdso_user_wrapper.S @@ -36,3 +36,4 @@ __kernel_\func: vdso_func gettimeofday vdso_func clock_getres vdso_func clock_gettime +vdso_func getcpu diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S index 177ccfbda40a..4c0e19145cc6 100644 --- a/arch/s390/kernel/vmlinux.lds.S +++ b/arch/s390/kernel/vmlinux.lds.S @@ -10,7 +10,8 @@ * Put .bss..swapper_pg_dir as the first thing in .bss. This will * make sure it has 16k alignment. */ -#define BSS_FIRST_SECTIONS *(.bss..swapper_pg_dir) +#define BSS_FIRST_SECTIONS *(.bss..swapper_pg_dir) \ + *(.bss..invalid_pg_dir) /* Handle ro_after_init data on our own. */ #define RO_AFTER_INIT_DATA diff --git a/arch/s390/lib/delay.c b/arch/s390/lib/delay.c index 8c0c68e7770e..68d61f2835df 100644 --- a/arch/s390/lib/delay.c +++ b/arch/s390/lib/delay.c @@ -13,11 +13,19 @@ #include #include #include +#include #include #include #include #include +static DEFINE_STATIC_KEY_FALSE(udelay_ready); + +void __init udelay_enable(void) +{ + static_branch_enable(&udelay_ready); +} + void __delay(unsigned long loops) { /* @@ -76,6 +84,11 @@ void __udelay(unsigned long long usecs) { unsigned long flags; + if (!static_branch_likely(&udelay_ready)) { + udelay_simple(usecs); + return; + } + preempt_disable(); local_irq_save(flags); if (in_irq()) { diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c index 0267405ab7c6..e8f642446fed 100644 --- a/arch/s390/lib/uaccess.c +++ b/arch/s390/lib/uaccess.c @@ -16,6 +16,22 @@ #include #include +#ifdef CONFIG_DEBUG_USER_ASCE +void debug_user_asce(void) +{ + unsigned long cr1, cr7; + + __ctl_store(cr1, 1, 1); + __ctl_store(cr7, 7, 7); + if (cr1 == S390_lowcore.kernel_asce && cr7 == S390_lowcore.user_asce) + return; + panic("incorrect ASCE on kernel exit\n" + "cr1: %016lx cr7: %016lx\n" + "kernel: %016llx user: %016llx\n", + cr1, cr7, S390_lowcore.kernel_asce, S390_lowcore.user_asce); +} +#endif /*CONFIG_DEBUG_USER_ASCE */ + #ifndef CONFIG_HAVE_MARCH_Z10_FEATURES static DEFINE_STATIC_KEY_FALSE(have_mvcos); @@ -40,71 +56,10 @@ static inline int copy_with_mvcos(void) } #endif -void set_fs(mm_segment_t fs) -{ - current->thread.mm_segment = fs; - if (fs == USER_DS) { - __ctl_load(S390_lowcore.user_asce, 1, 1); - clear_cpu_flag(CIF_ASCE_PRIMARY); - } else { - __ctl_load(S390_lowcore.kernel_asce, 1, 1); - set_cpu_flag(CIF_ASCE_PRIMARY); - } - if (fs & 1) { - if (fs == USER_DS_SACF) - __ctl_load(S390_lowcore.user_asce, 7, 7); - else - __ctl_load(S390_lowcore.kernel_asce, 7, 7); - set_cpu_flag(CIF_ASCE_SECONDARY); - } -} -EXPORT_SYMBOL(set_fs); - -mm_segment_t enable_sacf_uaccess(void) -{ - mm_segment_t old_fs; - unsigned long asce, cr; - unsigned long flags; - - old_fs = current->thread.mm_segment; - if (old_fs & 1) - return old_fs; - /* protect against a concurrent page table upgrade */ - local_irq_save(flags); - current->thread.mm_segment |= 1; - asce = S390_lowcore.kernel_asce; - if (likely(old_fs == USER_DS)) { - __ctl_store(cr, 1, 1); - if (cr != S390_lowcore.kernel_asce) { - __ctl_load(S390_lowcore.kernel_asce, 1, 1); - set_cpu_flag(CIF_ASCE_PRIMARY); - } - asce = S390_lowcore.user_asce; - } - __ctl_store(cr, 7, 7); - if (cr != asce) { - __ctl_load(asce, 7, 7); - set_cpu_flag(CIF_ASCE_SECONDARY); - } - local_irq_restore(flags); - return old_fs; -} -EXPORT_SYMBOL(enable_sacf_uaccess); - -void disable_sacf_uaccess(mm_segment_t old_fs) -{ - current->thread.mm_segment = old_fs; - if (old_fs == USER_DS && test_facility(27)) { - __ctl_load(S390_lowcore.user_asce, 1, 1); - clear_cpu_flag(CIF_ASCE_PRIMARY); - } -} -EXPORT_SYMBOL(disable_sacf_uaccess); - static inline unsigned long copy_from_user_mvcos(void *x, const void __user *ptr, unsigned long size) { - register unsigned long reg0 asm("0") = 0x01UL; + register unsigned long reg0 asm("0") = 0x81UL; unsigned long tmp1, tmp2; tmp1 = -4096UL; @@ -135,9 +90,7 @@ static inline unsigned long copy_from_user_mvcp(void *x, const void __user *ptr, unsigned long size) { unsigned long tmp1, tmp2; - mm_segment_t old_fs; - old_fs = enable_sacf_uaccess(); tmp1 = -256UL; asm volatile( " sacf 0\n" @@ -164,7 +117,6 @@ static inline unsigned long copy_from_user_mvcp(void *x, const void __user *ptr, EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b) : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2) : : "cc", "memory"); - disable_sacf_uaccess(old_fs); return size; } @@ -179,7 +131,7 @@ EXPORT_SYMBOL(raw_copy_from_user); static inline unsigned long copy_to_user_mvcos(void __user *ptr, const void *x, unsigned long size) { - register unsigned long reg0 asm("0") = 0x010000UL; + register unsigned long reg0 asm("0") = 0x810000UL; unsigned long tmp1, tmp2; tmp1 = -4096UL; @@ -210,9 +162,7 @@ static inline unsigned long copy_to_user_mvcs(void __user *ptr, const void *x, unsigned long size) { unsigned long tmp1, tmp2; - mm_segment_t old_fs; - old_fs = enable_sacf_uaccess(); tmp1 = -256UL; asm volatile( " sacf 0\n" @@ -239,7 +189,6 @@ static inline unsigned long copy_to_user_mvcs(void __user *ptr, const void *x, EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b) : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2) : : "cc", "memory"); - disable_sacf_uaccess(old_fs); return size; } @@ -254,7 +203,7 @@ EXPORT_SYMBOL(raw_copy_to_user); static inline unsigned long copy_in_user_mvcos(void __user *to, const void __user *from, unsigned long size) { - register unsigned long reg0 asm("0") = 0x010001UL; + register unsigned long reg0 asm("0") = 0x810081UL; unsigned long tmp1, tmp2; tmp1 = -4096UL; @@ -277,10 +226,8 @@ static inline unsigned long copy_in_user_mvcos(void __user *to, const void __use static inline unsigned long copy_in_user_mvc(void __user *to, const void __user *from, unsigned long size) { - mm_segment_t old_fs; unsigned long tmp1; - old_fs = enable_sacf_uaccess(); asm volatile( " sacf 256\n" " aghi %0,-1\n" @@ -304,7 +251,6 @@ static inline unsigned long copy_in_user_mvc(void __user *to, const void __user EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b) : "+a" (size), "+a" (to), "+a" (from), "=a" (tmp1) : : "cc", "memory"); - disable_sacf_uaccess(old_fs); return size; } @@ -318,7 +264,7 @@ EXPORT_SYMBOL(raw_copy_in_user); static inline unsigned long clear_user_mvcos(void __user *to, unsigned long size) { - register unsigned long reg0 asm("0") = 0x010000UL; + register unsigned long reg0 asm("0") = 0x810000UL; unsigned long tmp1, tmp2; tmp1 = -4096UL; @@ -346,10 +292,8 @@ static inline unsigned long clear_user_mvcos(void __user *to, unsigned long size static inline unsigned long clear_user_xc(void __user *to, unsigned long size) { - mm_segment_t old_fs; unsigned long tmp1, tmp2; - old_fs = enable_sacf_uaccess(); asm volatile( " sacf 256\n" " aghi %0,-1\n" @@ -378,7 +322,6 @@ static inline unsigned long clear_user_xc(void __user *to, unsigned long size) EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b) : "+a" (size), "+a" (to), "=a" (tmp1), "=a" (tmp2) : : "cc", "memory"); - disable_sacf_uaccess(old_fs); return size; } @@ -414,15 +357,9 @@ static inline unsigned long strnlen_user_srst(const char __user *src, unsigned long __strnlen_user(const char __user *src, unsigned long size) { - mm_segment_t old_fs; - unsigned long len; - if (unlikely(!size)) return 0; - old_fs = enable_sacf_uaccess(); - len = strnlen_user_srst(src, size); - disable_sacf_uaccess(old_fs); - return len; + return strnlen_user_srst(src, size); } EXPORT_SYMBOL(__strnlen_user); diff --git a/arch/s390/mm/dump_pagetables.c b/arch/s390/mm/dump_pagetables.c index 8f9ff7e7187d..e40a30647d99 100644 --- a/arch/s390/mm/dump_pagetables.c +++ b/arch/s390/mm/dump_pagetables.c @@ -255,7 +255,7 @@ static int pt_dump_init(void) */ max_addr = (S390_lowcore.kernel_asce & _REGION_ENTRY_TYPE_MASK) >> 2; max_addr = 1UL << (max_addr * 11 + 31); - address_markers[IDENTITY_AFTER_END_NR].start_address = memory_end; + address_markers[IDENTITY_AFTER_END_NR].start_address = ident_map_size; address_markers[MODULES_NR].start_address = MODULES_VADDR; address_markers[MODULES_END_NR].start_address = MODULES_END; address_markers[VMEMMAP_NR].start_address = (unsigned long) vmemmap; diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 996884dcc9fd..b8210103de14 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -53,7 +53,6 @@ enum fault_type { KERNEL_FAULT, USER_FAULT, - VDSO_FAULT, GMAP_FAULT, }; @@ -77,22 +76,16 @@ static enum fault_type get_fault_type(struct pt_regs *regs) trans_exc_code = regs->int_parm_long & 3; if (likely(trans_exc_code == 0)) { /* primary space exception */ - if (IS_ENABLED(CONFIG_PGSTE) && - test_pt_regs_flag(regs, PIF_GUEST_FAULT)) - return GMAP_FAULT; - if (current->thread.mm_segment == USER_DS) + if (user_mode(regs)) return USER_FAULT; + if (!IS_ENABLED(CONFIG_PGSTE)) + return KERNEL_FAULT; + if (test_pt_regs_flag(regs, PIF_GUEST_FAULT)) + return GMAP_FAULT; return KERNEL_FAULT; } - if (trans_exc_code == 2) { - /* secondary space exception */ - if (current->thread.mm_segment & 1) { - if (current->thread.mm_segment == USER_DS_SACF) - return USER_FAULT; - return KERNEL_FAULT; - } - return VDSO_FAULT; - } + if (trans_exc_code == 2) + return USER_FAULT; if (trans_exc_code == 1) { /* access register mode, not used in the kernel */ return USER_FAULT; @@ -188,10 +181,6 @@ static void dump_fault_info(struct pt_regs *regs) asce = S390_lowcore.user_asce; pr_cont("user "); break; - case VDSO_FAULT: - asce = S390_lowcore.vdso_asce; - pr_cont("vdso "); - break; case GMAP_FAULT: asce = ((struct gmap *) S390_lowcore.gmap)->asce; pr_cont("gmap "); @@ -414,9 +403,6 @@ static inline vm_fault_t do_exception(struct pt_regs *regs, int access) switch (type) { case KERNEL_FAULT: goto out; - case VDSO_FAULT: - fault = VM_FAULT_BADMAP; - goto out; case USER_FAULT: case GMAP_FAULT: if (faulthandler_disabled() || !mm) @@ -834,7 +820,6 @@ void do_secure_storage_access(struct pt_regs *regs) if (rc) BUG(); break; - case VDSO_FAULT: case GMAP_FAULT: default: do_fault_error(regs, VM_READ | VM_WRITE, VM_FAULT_BADMAP); diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index 77767850d0d0..73a163065b95 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -49,6 +49,9 @@ #include pgd_t swapper_pg_dir[PTRS_PER_PGD] __section(".bss..swapper_pg_dir"); +static pgd_t invalid_pg_dir[PTRS_PER_PGD] __section(".bss..invalid_pg_dir"); + +unsigned long s390_invalid_asce; unsigned long empty_zero_page, zero_page_mask; EXPORT_SYMBOL(empty_zero_page); @@ -92,6 +95,9 @@ void __init paging_init(void) unsigned long pgd_type, asce_bits; psw_t psw; + s390_invalid_asce = (unsigned long)invalid_pg_dir; + s390_invalid_asce |= _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH; + crst_table_init((unsigned long *)invalid_pg_dir, _REGION3_ENTRY_EMPTY); init_mm.pgd = swapper_pg_dir; if (VMALLOC_END > _REGION2_SIZE) { asce_bits = _ASCE_TYPE_REGION2 | _ASCE_TABLE_LENGTH; @@ -102,14 +108,14 @@ void __init paging_init(void) } init_mm.context.asce = (__pa(init_mm.pgd) & PAGE_MASK) | asce_bits; S390_lowcore.kernel_asce = init_mm.context.asce; - S390_lowcore.user_asce = S390_lowcore.kernel_asce; + S390_lowcore.user_asce = s390_invalid_asce; crst_table_init((unsigned long *) init_mm.pgd, pgd_type); vmem_map_init(); - kasan_copy_shadow(init_mm.pgd); + kasan_copy_shadow_mapping(); /* enable virtual mapping in kernel mode */ __ctl_load(S390_lowcore.kernel_asce, 1, 1); - __ctl_load(S390_lowcore.kernel_asce, 7, 7); + __ctl_load(S390_lowcore.user_asce, 7, 7); __ctl_load(S390_lowcore.kernel_asce, 13, 13); psw.mask = __extract_psw(); psw_bits(psw).dat = 1; diff --git a/arch/s390/mm/kasan_init.c b/arch/s390/mm/kasan_init.c index 5646b39c728a..db4d303aaaa9 100644 --- a/arch/s390/mm/kasan_init.c +++ b/arch/s390/mm/kasan_init.c @@ -87,7 +87,7 @@ enum populate_mode { POPULATE_ZERO_SHADOW, POPULATE_SHALLOW }; -static void __init kasan_early_vmemmap_populate(unsigned long address, +static void __init kasan_early_pgtable_populate(unsigned long address, unsigned long end, enum populate_mode mode) { @@ -123,8 +123,7 @@ static void __init kasan_early_vmemmap_populate(unsigned long address, pgd_populate(&init_mm, pg_dir, p4_dir); } - if (IS_ENABLED(CONFIG_KASAN_S390_4_LEVEL_PAGING) && - mode == POPULATE_SHALLOW) { + if (mode == POPULATE_SHALLOW) { address = (address + P4D_SIZE) & P4D_MASK; continue; } @@ -143,12 +142,6 @@ static void __init kasan_early_vmemmap_populate(unsigned long address, p4d_populate(&init_mm, p4_dir, pu_dir); } - if (!IS_ENABLED(CONFIG_KASAN_S390_4_LEVEL_PAGING) && - mode == POPULATE_SHALLOW) { - address = (address + PUD_SIZE) & PUD_MASK; - continue; - } - pu_dir = pud_offset(p4_dir, address); if (pud_none(*pu_dir)) { if (mode == POPULATE_ZERO_SHADOW && @@ -281,7 +274,6 @@ void __init kasan_early_init(void) unsigned long shadow_alloc_size; unsigned long vmax_unlimited; unsigned long initrd_end; - unsigned long asce_type; unsigned long memsize; unsigned long pgt_prot = pgprot_val(PAGE_KERNEL_RO); pte_t pte_z; @@ -297,32 +289,26 @@ void __init kasan_early_init(void) memsize = get_mem_detect_end(); if (!memsize) kasan_early_panic("cannot detect physical memory size\n"); - /* respect mem= cmdline parameter */ - if (memory_end_set && memsize > memory_end) - memsize = memory_end; - if (IS_ENABLED(CONFIG_CRASH_DUMP) && OLDMEM_BASE) - memsize = min(memsize, OLDMEM_SIZE); - memsize = min(memsize, KASAN_SHADOW_START); + /* + * Kasan currently supports standby memory but only if it follows + * online memory (default allocation), i.e. no memory holes. + * - memsize represents end of online memory + * - ident_map_size represents online + standby and memory limits + * accounted. + * Kasan maps "memsize" right away. + * [0, memsize] - as identity mapping + * [__sha(0), __sha(memsize)] - shadow memory for identity mapping + * The rest [memsize, ident_map_size] if memsize < ident_map_size + * could be mapped/unmapped dynamically later during memory hotplug. + */ + memsize = min(memsize, ident_map_size); - if (IS_ENABLED(CONFIG_KASAN_S390_4_LEVEL_PAGING)) { - /* 4 level paging */ - BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, P4D_SIZE)); - BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, P4D_SIZE)); - crst_table_init((unsigned long *)early_pg_dir, - _REGION2_ENTRY_EMPTY); - untracked_mem_end = kasan_vmax = vmax_unlimited = _REGION1_SIZE; - if (has_uv_sec_stor_limit()) - kasan_vmax = min(vmax_unlimited, uv_info.max_sec_stor_addr); - asce_type = _ASCE_TYPE_REGION2; - } else { - /* 3 level paging */ - BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, PUD_SIZE)); - BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, PUD_SIZE)); - crst_table_init((unsigned long *)early_pg_dir, - _REGION3_ENTRY_EMPTY); - untracked_mem_end = kasan_vmax = vmax_unlimited = _REGION2_SIZE; - asce_type = _ASCE_TYPE_REGION3; - } + BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, P4D_SIZE)); + BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, P4D_SIZE)); + crst_table_init((unsigned long *)early_pg_dir, _REGION2_ENTRY_EMPTY); + untracked_mem_end = kasan_vmax = vmax_unlimited = _REGION1_SIZE; + if (has_uv_sec_stor_limit()) + kasan_vmax = min(vmax_unlimited, uv_info.max_sec_stor_addr); /* init kasan zero shadow */ crst_table_init((unsigned long *)kasan_early_shadow_p4d, @@ -388,27 +374,25 @@ void __init kasan_early_init(void) * +-----------------+ +- shadow end ---+ */ /* populate kasan shadow (for identity mapping and zero page mapping) */ - kasan_early_vmemmap_populate(__sha(0), __sha(memsize), POPULATE_MAP); + kasan_early_pgtable_populate(__sha(0), __sha(memsize), POPULATE_MAP); if (IS_ENABLED(CONFIG_MODULES)) untracked_mem_end = kasan_vmax - MODULES_LEN; if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) { untracked_mem_end = kasan_vmax - vmalloc_size - MODULES_LEN; /* shallowly populate kasan shadow for vmalloc and modules */ - kasan_early_vmemmap_populate(__sha(untracked_mem_end), - __sha(kasan_vmax), POPULATE_SHALLOW); + kasan_early_pgtable_populate(__sha(untracked_mem_end), __sha(kasan_vmax), + POPULATE_SHALLOW); } /* populate kasan shadow for untracked memory */ - kasan_early_vmemmap_populate(__sha(max_physmem_end), - __sha(untracked_mem_end), + kasan_early_pgtable_populate(__sha(ident_map_size), __sha(untracked_mem_end), POPULATE_ZERO_SHADOW); - kasan_early_vmemmap_populate(__sha(kasan_vmax), - __sha(vmax_unlimited), + kasan_early_pgtable_populate(__sha(kasan_vmax), __sha(vmax_unlimited), POPULATE_ZERO_SHADOW); /* memory allocated for identity mapping structs will be freed later */ pgalloc_freeable = pgalloc_pos; /* populate identity mapping */ - kasan_early_vmemmap_populate(0, memsize, POPULATE_ONE2ONE); - kasan_set_pgd(early_pg_dir, asce_type); + kasan_early_pgtable_populate(0, memsize, POPULATE_ONE2ONE); + kasan_set_pgd(early_pg_dir, _ASCE_TYPE_REGION2); kasan_enable_dat(); /* enable kasan */ init_task.kasan_depth = 0; @@ -416,7 +400,7 @@ void __init kasan_early_init(void) sclp_early_printk("KernelAddressSanitizer initialized\n"); } -void __init kasan_copy_shadow(pgd_t *pg_dir) +void __init kasan_copy_shadow_mapping(void) { /* * At this point we are still running on early pages setup early_pg_dir, @@ -428,24 +412,13 @@ void __init kasan_copy_shadow(pgd_t *pg_dir) pgd_t *pg_dir_dst; p4d_t *p4_dir_src; p4d_t *p4_dir_dst; - pud_t *pu_dir_src; - pud_t *pu_dir_dst; pg_dir_src = pgd_offset_raw(early_pg_dir, KASAN_SHADOW_START); - pg_dir_dst = pgd_offset_raw(pg_dir, KASAN_SHADOW_START); + pg_dir_dst = pgd_offset_raw(init_mm.pgd, KASAN_SHADOW_START); p4_dir_src = p4d_offset(pg_dir_src, KASAN_SHADOW_START); p4_dir_dst = p4d_offset(pg_dir_dst, KASAN_SHADOW_START); - if (!p4d_folded(*p4_dir_src)) { - /* 4 level paging */ - memcpy(p4_dir_dst, p4_dir_src, - (KASAN_SHADOW_SIZE >> P4D_SHIFT) * sizeof(p4d_t)); - return; - } - /* 3 level paging */ - pu_dir_src = pud_offset(p4_dir_src, KASAN_SHADOW_START); - pu_dir_dst = pud_offset(p4_dir_dst, KASAN_SHADOW_START); - memcpy(pu_dir_dst, pu_dir_src, - (KASAN_SHADOW_SIZE >> PUD_SHIFT) * sizeof(pud_t)); + memcpy(p4_dir_dst, p4_dir_src, + (KASAN_SHADOW_SIZE >> P4D_SHIFT) * sizeof(p4d_t)); } void __init kasan_free_early_identity(void) diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c index 11d2c8395e2a..4e87c819ddea 100644 --- a/arch/s390/mm/pgalloc.c +++ b/arch/s390/mm/pgalloc.c @@ -70,19 +70,10 @@ static void __crst_table_upgrade(void *arg) { struct mm_struct *mm = arg; - /* we must change all active ASCEs to avoid the creation of new TLBs */ + /* change all active ASCEs to avoid the creation of new TLBs */ if (current->active_mm == mm) { S390_lowcore.user_asce = mm->context.asce; - if (current->thread.mm_segment == USER_DS) { - __ctl_load(S390_lowcore.user_asce, 1, 1); - /* Mark user-ASCE present in CR1 */ - clear_cpu_flag(CIF_ASCE_PRIMARY); - } - if (current->thread.mm_segment == USER_DS_SACF) { - __ctl_load(S390_lowcore.user_asce, 7, 7); - /* enable_sacf_uaccess does all or nothing */ - WARN_ON(!test_cpu_flag(CIF_ASCE_SECONDARY)); - } + __ctl_load(S390_lowcore.user_asce, 7, 7); } __tlb_flush_local(); } diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index b239f2ba93b0..01f3a5f58e64 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -76,20 +76,20 @@ static void vmem_pte_free(unsigned long *table) /* * The unused vmemmap range, which was not yet memset(PAGE_UNUSED) ranges - * from unused_pmd_start to next PMD_SIZE boundary. + * from unused_sub_pmd_start to next PMD_SIZE boundary. */ -static unsigned long unused_pmd_start; +static unsigned long unused_sub_pmd_start; -static void vmemmap_flush_unused_pmd(void) +static void vmemmap_flush_unused_sub_pmd(void) { - if (!unused_pmd_start) + if (!unused_sub_pmd_start) return; - memset(__va(unused_pmd_start), PAGE_UNUSED, - ALIGN(unused_pmd_start, PMD_SIZE) - unused_pmd_start); - unused_pmd_start = 0; + memset(__va(unused_sub_pmd_start), PAGE_UNUSED, + ALIGN(unused_sub_pmd_start, PMD_SIZE) - unused_sub_pmd_start); + unused_sub_pmd_start = 0; } -static void __vmemmap_use_sub_pmd(unsigned long start, unsigned long end) +static void vmemmap_mark_sub_pmd_used(unsigned long start, unsigned long end) { /* * As we expect to add in the same granularity as we remove, it's @@ -106,24 +106,24 @@ static void vmemmap_use_sub_pmd(unsigned long start, unsigned long end) * We only optimize if the new used range directly follows the * previously unused range (esp., when populating consecutive sections). */ - if (unused_pmd_start == start) { - unused_pmd_start = end; - if (likely(IS_ALIGNED(unused_pmd_start, PMD_SIZE))) - unused_pmd_start = 0; + if (unused_sub_pmd_start == start) { + unused_sub_pmd_start = end; + if (likely(IS_ALIGNED(unused_sub_pmd_start, PMD_SIZE))) + unused_sub_pmd_start = 0; return; } - vmemmap_flush_unused_pmd(); - __vmemmap_use_sub_pmd(start, end); + vmemmap_flush_unused_sub_pmd(); + vmemmap_mark_sub_pmd_used(start, end); } static void vmemmap_use_new_sub_pmd(unsigned long start, unsigned long end) { void *page = __va(ALIGN_DOWN(start, PMD_SIZE)); - vmemmap_flush_unused_pmd(); + vmemmap_flush_unused_sub_pmd(); /* Could be our memmap page is filled with PAGE_UNUSED already ... */ - __vmemmap_use_sub_pmd(start, end); + vmemmap_mark_sub_pmd_used(start, end); /* Mark the unused parts of the new memmap page PAGE_UNUSED. */ if (!IS_ALIGNED(start, PMD_SIZE)) @@ -134,7 +134,7 @@ static void vmemmap_use_new_sub_pmd(unsigned long start, unsigned long end) * unused range in the populated PMD. */ if (!IS_ALIGNED(end, PMD_SIZE)) - unused_pmd_start = end; + unused_sub_pmd_start = end; } /* Returns true if the PMD is completely unused and can be freed. */ @@ -142,7 +142,7 @@ static bool vmemmap_unuse_sub_pmd(unsigned long start, unsigned long end) { void *page = __va(ALIGN_DOWN(start, PMD_SIZE)); - vmemmap_flush_unused_pmd(); + vmemmap_flush_unused_sub_pmd(); memset(__va(start), PAGE_UNUSED, end - start); return !memchr_inv(page, PAGE_UNUSED, PMD_SIZE); } @@ -223,7 +223,7 @@ static int __ref modify_pmd_table(pud_t *pud, unsigned long addr, if (!add) { if (pmd_none(*pmd)) continue; - if (pmd_large(*pmd) && !add) { + if (pmd_large(*pmd)) { if (IS_ALIGNED(addr, PMD_SIZE) && IS_ALIGNED(next, PMD_SIZE)) { if (!direct) diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index 570016ae8bcd..41df8fcfddde 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -851,8 +851,10 @@ static int __init pci_base_init(void) if (!s390_pci_probe) return 0; - if (!test_facility(69) || !test_facility(71)) + if (!test_facility(69) || !test_facility(71)) { + pr_info("PCI is not supported because CPU facilities 69 or 71 are not available\n"); return 0; + } if (test_facility(153) && !s390_pci_no_mio) { static_branch_enable(&have_mio); diff --git a/arch/s390/pci/pci_mmio.c b/arch/s390/pci/pci_mmio.c index 401cf670a243..de3bdbed8881 100644 --- a/arch/s390/pci/pci_mmio.c +++ b/arch/s390/pci/pci_mmio.c @@ -93,12 +93,10 @@ static inline int __memcpy_toio_inuser(void __iomem *dst, { int size, rc = 0; u8 status = 0; - mm_segment_t old_fs; if (!src) return -EINVAL; - old_fs = enable_sacf_uaccess(); while (n > 0) { size = zpci_get_max_write_size((u64 __force) dst, (u64 __force) src, n, @@ -113,39 +111,20 @@ static inline int __memcpy_toio_inuser(void __iomem *dst, dst += size; n -= size; } - disable_sacf_uaccess(old_fs); if (rc) zpci_err_mmio(rc, status, (__force u64) dst); return rc; } -static long get_pfn(unsigned long user_addr, unsigned long access, - unsigned long *pfn) -{ - struct vm_area_struct *vma; - long ret; - - mmap_read_lock(current->mm); - ret = -EINVAL; - vma = find_vma(current->mm, user_addr); - if (!vma) - goto out; - ret = -EACCES; - if (!(vma->vm_flags & access)) - goto out; - ret = follow_pfn(vma, user_addr, pfn); -out: - mmap_read_unlock(current->mm); - return ret; -} - SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr, const void __user *, user_buffer, size_t, length) { u8 local_buf[64]; void __iomem *io_addr; void *buf; - unsigned long pfn; + struct vm_area_struct *vma; + pte_t *ptep; + spinlock_t *ptl; long ret; if (!zpci_is_enabled()) @@ -158,7 +137,7 @@ SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr, * We only support write access to MIO capable devices if we are on * a MIO enabled system. Otherwise we would have to check for every * address if it is a special ZPCI_ADDR and would have to do - * a get_pfn() which we don't need for MIO capable devices. Currently + * a pfn lookup which we don't need for MIO capable devices. Currently * ISM devices are the only devices without MIO support and there is no * known need for accessing these from userspace. */ @@ -176,21 +155,37 @@ SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr, } else buf = local_buf; - ret = get_pfn(mmio_addr, VM_WRITE, &pfn); + ret = -EFAULT; + if (copy_from_user(buf, user_buffer, length)) + goto out_free; + + mmap_read_lock(current->mm); + ret = -EINVAL; + vma = find_vma(current->mm, mmio_addr); + if (!vma) + goto out_unlock_mmap; + if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) + goto out_unlock_mmap; + ret = -EACCES; + if (!(vma->vm_flags & VM_WRITE)) + goto out_unlock_mmap; + + ret = follow_pte_pmd(vma->vm_mm, mmio_addr, NULL, &ptep, NULL, &ptl); if (ret) - goto out; - io_addr = (void __iomem *)((pfn << PAGE_SHIFT) | + goto out_unlock_mmap; + + io_addr = (void __iomem *)((pte_pfn(*ptep) << PAGE_SHIFT) | (mmio_addr & ~PAGE_MASK)); - ret = -EFAULT; if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) - goto out; - - if (copy_from_user(buf, user_buffer, length)) - goto out; + goto out_unlock_pt; ret = zpci_memcpy_toio(io_addr, buf, length); -out: +out_unlock_pt: + pte_unmap_unlock(ptep, ptl); +out_unlock_mmap: + mmap_read_unlock(current->mm); +out_free: if (buf != local_buf) kfree(buf); return ret; @@ -248,9 +243,7 @@ static inline int __memcpy_fromio_inuser(void __user *dst, { int size, rc = 0; u8 status; - mm_segment_t old_fs; - old_fs = enable_sacf_uaccess(); while (n > 0) { size = zpci_get_max_write_size((u64 __force) src, (u64 __force) dst, n, @@ -262,7 +255,6 @@ static inline int __memcpy_fromio_inuser(void __user *dst, dst += size; n -= size; } - disable_sacf_uaccess(old_fs); if (rc) zpci_err_mmio(rc, status, (__force u64) dst); return rc; @@ -274,7 +266,9 @@ SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr, u8 local_buf[64]; void __iomem *io_addr; void *buf; - unsigned long pfn; + struct vm_area_struct *vma; + pte_t *ptep; + spinlock_t *ptl; long ret; if (!zpci_is_enabled()) @@ -287,7 +281,7 @@ SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr, * We only support read access to MIO capable devices if we are on * a MIO enabled system. Otherwise we would have to check for every * address if it is a special ZPCI_ADDR and would have to do - * a get_pfn() which we don't need for MIO capable devices. Currently + * a pfn lookup which we don't need for MIO capable devices. Currently * ISM devices are the only devices without MIO support and there is no * known need for accessing these from userspace. */ @@ -306,22 +300,38 @@ SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr, buf = local_buf; } - ret = get_pfn(mmio_addr, VM_READ, &pfn); + mmap_read_lock(current->mm); + ret = -EINVAL; + vma = find_vma(current->mm, mmio_addr); + if (!vma) + goto out_unlock_mmap; + if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) + goto out_unlock_mmap; + ret = -EACCES; + if (!(vma->vm_flags & VM_WRITE)) + goto out_unlock_mmap; + + ret = follow_pte_pmd(vma->vm_mm, mmio_addr, NULL, &ptep, NULL, &ptl); if (ret) - goto out; - io_addr = (void __iomem *)((pfn << PAGE_SHIFT) | (mmio_addr & ~PAGE_MASK)); + goto out_unlock_mmap; + + io_addr = (void __iomem *)((pte_pfn(*ptep) << PAGE_SHIFT) | + (mmio_addr & ~PAGE_MASK)); if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) { ret = -EFAULT; - goto out; + goto out_unlock_pt; } ret = zpci_memcpy_fromio(buf, io_addr, length); - if (ret) - goto out; - if (copy_to_user(user_buffer, buf, length)) + +out_unlock_pt: + pte_unmap_unlock(ptep, ptl); +out_unlock_mmap: + mmap_read_unlock(current->mm); + + if (!ret && copy_to_user(user_buffer, buf, length)) ret = -EFAULT; -out: if (buf != local_buf) kfree(buf); return ret; diff --git a/arch/s390/purgatory/head.S b/arch/s390/purgatory/head.S index 5a10ce34b95d..3d1c31e0cf3d 100644 --- a/arch/s390/purgatory/head.S +++ b/arch/s390/purgatory/head.S @@ -62,14 +62,15 @@ jh 10b .endm -.macro START_NEXT_KERNEL base +.macro START_NEXT_KERNEL base subcode lg %r4,kernel_entry-\base(%r13) lg %r5,load_psw_mask-\base(%r13) ogr %r4,%r5 stg %r4,0(%r0) xgr %r0,%r0 - diag %r0,%r0,0x308 + lghi %r1,\subcode + diag %r0,%r1,0x308 .endm .text @@ -123,7 +124,7 @@ ENTRY(purgatory_start) je .start_crash_kernel /* start normal kernel */ - START_NEXT_KERNEL .base_crash + START_NEXT_KERNEL .base_crash 0 .return_old_kernel: lmg %r6,%r15,gprregs-.base_crash(%r13) @@ -227,7 +228,7 @@ ENTRY(purgatory_start) MEMCPY %r9,%r10,%r11 /* start crash kernel */ - START_NEXT_KERNEL .base_dst + START_NEXT_KERNEL .base_dst 1 load_psw_mask: diff --git a/arch/s390/purgatory/purgatory.c b/arch/s390/purgatory/purgatory.c index 0a423bcf6746..030efda05dbe 100644 --- a/arch/s390/purgatory/purgatory.c +++ b/arch/s390/purgatory/purgatory.c @@ -9,7 +9,7 @@ #include #include -#include +#include #include int verify_sha256_digest(void) diff --git a/arch/sparc/crypto/crc32c_glue.c b/arch/sparc/crypto/crc32c_glue.c index 4e9323229e71..82efb7f81c28 100644 --- a/arch/sparc/crypto/crc32c_glue.c +++ b/arch/sparc/crypto/crc32c_glue.c @@ -35,7 +35,7 @@ static int crc32c_sparc64_setkey(struct crypto_shash *hash, const u8 *key, if (keylen != sizeof(u32)) return -EINVAL; - *(__le32 *)mctx = le32_to_cpup((__le32 *)key); + *mctx = le32_to_cpup((__le32 *)key); return 0; } diff --git a/arch/sparc/crypto/md5_glue.c b/arch/sparc/crypto/md5_glue.c index 111283fe837e..511db98d590a 100644 --- a/arch/sparc/crypto/md5_glue.c +++ b/arch/sparc/crypto/md5_glue.c @@ -33,10 +33,11 @@ static int md5_sparc64_init(struct shash_desc *desc) { struct md5_state *mctx = shash_desc_ctx(desc); - mctx->hash[0] = cpu_to_le32(MD5_H0); - mctx->hash[1] = cpu_to_le32(MD5_H1); - mctx->hash[2] = cpu_to_le32(MD5_H2); - mctx->hash[3] = cpu_to_le32(MD5_H3); + mctx->hash[0] = MD5_H0; + mctx->hash[1] = MD5_H1; + mctx->hash[2] = MD5_H2; + mctx->hash[3] = MD5_H3; + le32_to_cpu_array(mctx->hash, 4); mctx->byte_count = 0; return 0; diff --git a/arch/sparc/crypto/sha1_glue.c b/arch/sparc/crypto/sha1_glue.c index dc017782be52..86a654cce5ab 100644 --- a/arch/sparc/crypto/sha1_glue.c +++ b/arch/sparc/crypto/sha1_glue.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/sparc/crypto/sha256_glue.c b/arch/sparc/crypto/sha256_glue.c index ca2547df9652..60ec524cf9ca 100644 --- a/arch/sparc/crypto/sha256_glue.c +++ b/arch/sparc/crypto/sha256_glue.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/sparc/crypto/sha512_glue.c b/arch/sparc/crypto/sha512_glue.c index 3b2ca732ff7a..273ce21918c1 100644 --- a/arch/sparc/crypto/sha512_glue.c +++ b/arch/sparc/crypto/sha512_glue.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index fbf26e0f7a6a..52e36adb5112 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1931,6 +1931,23 @@ config X86_INTEL_TSX_MODE_AUTO side channel attacks- equals the tsx=auto command line parameter. endchoice +config X86_SGX + bool "Software Guard eXtensions (SGX)" + depends on X86_64 && CPU_SUP_INTEL + depends on CRYPTO=y + depends on CRYPTO_SHA256=y + select SRCU + select MMU_NOTIFIER + help + Intel(R) Software Guard eXtensions (SGX) is a set of CPU instructions + that can be used by applications to set aside private regions of code + and data, referred to as enclaves. An enclave's private memory can + only be accessed by code running within the enclave. Accesses from + outside the enclave, including other enclaves, are disallowed by + hardware. + + If unsure, say N. + config EFI bool "EFI runtime service support" depends on ACPI diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug index 27b5e2bc6a01..80b57e7f4947 100644 --- a/arch/x86/Kconfig.debug +++ b/arch/x86/Kconfig.debug @@ -62,9 +62,6 @@ config EARLY_PRINTK_USB_XDBC You should normally say N here, unless you want to debug early crashes or need a very simple printk logging facility. -config COPY_MC_TEST - def_bool n - config EFI_PGT_DUMP bool "Dump the EFI pagetable" depends on EFI diff --git a/arch/x86/Makefile b/arch/x86/Makefile index 1bf21746f4ce..7116da3980be 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -24,14 +24,7 @@ endif # How to compile the 16-bit code. Note we always compile for -march=i386; # that way we can complain to the user if the CPU is insufficient. -# -# The -m16 option is supported by GCC >= 4.9 and clang >= 3.5. For -# older versions of GCC, include an *assembly* header to make sure that -# gcc doesn't play any games behind our back. -CODE16GCC_CFLAGS := -m32 -Wa,$(srctree)/arch/x86/boot/code16gcc.h -M16_CFLAGS := $(call cc-option, -m16, $(CODE16GCC_CFLAGS)) - -REALMODE_CFLAGS := $(M16_CFLAGS) -g -Os -DDISABLE_BRANCH_PROFILING \ +REALMODE_CFLAGS := -m16 -g -Os -DDISABLE_BRANCH_PROFILING \ -Wall -Wstrict-prototypes -march=i386 -mregparm=3 \ -fno-strict-aliasing -fomit-frame-pointer -fno-pic \ -mno-mmx -mno-sse diff --git a/arch/x86/boot/code16gcc.h b/arch/x86/boot/code16gcc.h deleted file mode 100644 index e19fd7536307..000000000000 --- a/arch/x86/boot/code16gcc.h +++ /dev/null @@ -1,12 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -# -# code16gcc.h -# -# This file is added to the assembler via -Wa when compiling 16-bit C code. -# This is done this way instead via asm() to make sure gcc does not reorder -# things around us. -# -# gcc 4.9+ has a real -m16 option so we can drop this hack long term. -# - - .code16gcc diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S index 017de6cc87dc..e94874f4bbc1 100644 --- a/arch/x86/boot/compressed/head_64.S +++ b/arch/x86/boot/compressed/head_64.S @@ -241,12 +241,12 @@ SYM_FUNC_START(startup_32) leal rva(startup_64)(%ebp), %eax #ifdef CONFIG_EFI_MIXED movl rva(efi32_boot_args)(%ebp), %edi - cmp $0, %edi + testl %edi, %edi jz 1f leal rva(efi64_stub_entry)(%ebp), %eax movl rva(efi32_boot_args+4)(%ebp), %esi movl rva(efi32_boot_args+8)(%ebp), %edx // saved bootparams pointer - cmpl $0, %edx + testl %edx, %edx jnz 1f /* * efi_pe_entry uses MS calling convention, which requires 32 bytes of @@ -592,7 +592,7 @@ SYM_CODE_START(trampoline_32bit_src) movl %eax, %cr0 /* Check what paging mode we want to be in after the trampoline */ - cmpl $0, %edx + testl %edx, %edx jz 1f /* We want 5-level paging: don't touch CR3 if it already points to 5-level page tables */ @@ -622,7 +622,7 @@ SYM_CODE_START(trampoline_32bit_src) /* Enable PAE and LA57 (if required) paging modes */ movl $X86_CR4_PAE, %eax - cmpl $0, %edx + testl %edx, %edx jz 1f orl $X86_CR4_LA57, %eax 1: diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c index 39b2eded7bc2..f7213d0943b8 100644 --- a/arch/x86/boot/compressed/ident_map_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -168,16 +168,6 @@ void initialize_identity_maps(void *rmode) write_cr3(top_level_pgt); } -/* - * This switches the page tables to the new level4 that has been built - * via calls to add_identity_map() above. If booted via startup_32(), - * this is effectively a no-op. - */ -void finalize_identity_maps(void) -{ - write_cr3(top_level_pgt); -} - static pte_t *split_large_pmd(struct x86_mapping_info *info, pmd_t *pmdp, unsigned long __address) { diff --git a/arch/x86/crypto/aes_glue.c b/arch/x86/crypto/aes_glue.c deleted file mode 100644 index 7b7dc05fa1a4..000000000000 --- a/arch/x86/crypto/aes_glue.c +++ /dev/null @@ -1 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S index 1852b19a73a0..d1436c37008b 100644 --- a/arch/x86/crypto/aesni-intel_asm.S +++ b/arch/x86/crypto/aesni-intel_asm.S @@ -318,7 +318,7 @@ _initial_blocks_\@: # Main loop - Encrypt/Decrypt remaining blocks - cmp $0, %r13 + test %r13, %r13 je _zero_cipher_left_\@ sub $64, %r13 je _four_cipher_left_\@ @@ -437,7 +437,7 @@ _multiple_of_16_bytes_\@: mov PBlockLen(%arg2), %r12 - cmp $0, %r12 + test %r12, %r12 je _partial_done\@ GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6 @@ -474,7 +474,7 @@ _T_8_\@: add $8, %r10 sub $8, %r11 psrldq $8, %xmm0 - cmp $0, %r11 + test %r11, %r11 je _return_T_done_\@ _T_4_\@: movd %xmm0, %eax @@ -482,7 +482,7 @@ _T_4_\@: add $4, %r10 sub $4, %r11 psrldq $4, %xmm0 - cmp $0, %r11 + test %r11, %r11 je _return_T_done_\@ _T_123_\@: movd %xmm0, %eax @@ -619,7 +619,7 @@ _get_AAD_blocks\@: /* read the last <16B of AAD */ _get_AAD_rest\@: - cmp $0, %r11 + test %r11, %r11 je _get_AAD_done\@ READ_PARTIAL_BLOCK %r10, %r11, \TMP1, \TMP7 @@ -640,7 +640,7 @@ _get_AAD_done\@: .macro PARTIAL_BLOCK CYPH_PLAIN_OUT PLAIN_CYPH_IN PLAIN_CYPH_LEN DATA_OFFSET \ AAD_HASH operation mov PBlockLen(%arg2), %r13 - cmp $0, %r13 + test %r13, %r13 je _partial_block_done_\@ # Leave Macro if no partial blocks # Read in input data without over reading cmp $16, \PLAIN_CYPH_LEN @@ -692,7 +692,7 @@ _no_extra_mask_1_\@: pshufb %xmm2, %xmm3 pxor %xmm3, \AAD_HASH - cmp $0, %r10 + test %r10, %r10 jl _partial_incomplete_1_\@ # GHASH computation for the last <16 Byte block @@ -727,7 +727,7 @@ _no_extra_mask_2_\@: pshufb %xmm2, %xmm9 pxor %xmm9, \AAD_HASH - cmp $0, %r10 + test %r10, %r10 jl _partial_incomplete_2_\@ # GHASH computation for the last <16 Byte block @@ -747,7 +747,7 @@ _encode_done_\@: pshufb %xmm2, %xmm9 .endif # output encrypted Bytes - cmp $0, %r10 + test %r10, %r10 jl _partial_fill_\@ mov %r13, %r12 mov $16, %r13 @@ -2720,7 +2720,7 @@ SYM_FUNC_END(aesni_ctr_enc) */ SYM_FUNC_START(aesni_xts_crypt8) FRAME_BEGIN - cmpb $0, %cl + testb %cl, %cl movl $0, %ecx movl $240, %r10d leaq _aesni_enc4, %r11 diff --git a/arch/x86/crypto/aesni-intel_avx-x86_64.S b/arch/x86/crypto/aesni-intel_avx-x86_64.S index 5fee47956f3b..2cf8e94d986a 100644 --- a/arch/x86/crypto/aesni-intel_avx-x86_64.S +++ b/arch/x86/crypto/aesni-intel_avx-x86_64.S @@ -369,7 +369,7 @@ _initial_num_blocks_is_0\@: _initial_blocks_encrypted\@: - cmp $0, %r13 + test %r13, %r13 je _zero_cipher_left\@ sub $128, %r13 @@ -528,7 +528,7 @@ _multiple_of_16_bytes\@: vmovdqu HashKey(arg2), %xmm13 mov PBlockLen(arg2), %r12 - cmp $0, %r12 + test %r12, %r12 je _partial_done\@ #GHASH computation for the last <16 Byte block @@ -573,7 +573,7 @@ _T_8\@: add $8, %r10 sub $8, %r11 vpsrldq $8, %xmm9, %xmm9 - cmp $0, %r11 + test %r11, %r11 je _return_T_done\@ _T_4\@: vmovd %xmm9, %eax @@ -581,7 +581,7 @@ _T_4\@: add $4, %r10 sub $4, %r11 vpsrldq $4, %xmm9, %xmm9 - cmp $0, %r11 + test %r11, %r11 je _return_T_done\@ _T_123\@: vmovd %xmm9, %eax @@ -625,7 +625,7 @@ _get_AAD_blocks\@: cmp $16, %r11 jge _get_AAD_blocks\@ vmovdqu \T8, \T7 - cmp $0, %r11 + test %r11, %r11 je _get_AAD_done\@ vpxor \T7, \T7, \T7 @@ -644,7 +644,7 @@ _get_AAD_rest8\@: vpxor \T1, \T7, \T7 jmp _get_AAD_rest8\@ _get_AAD_rest4\@: - cmp $0, %r11 + test %r11, %r11 jle _get_AAD_rest0\@ mov (%r10), %eax movq %rax, \T1 @@ -749,7 +749,7 @@ _done_read_partial_block_\@: .macro PARTIAL_BLOCK GHASH_MUL CYPH_PLAIN_OUT PLAIN_CYPH_IN PLAIN_CYPH_LEN DATA_OFFSET \ AAD_HASH ENC_DEC mov PBlockLen(arg2), %r13 - cmp $0, %r13 + test %r13, %r13 je _partial_block_done_\@ # Leave Macro if no partial blocks # Read in input data without over reading cmp $16, \PLAIN_CYPH_LEN @@ -801,7 +801,7 @@ _no_extra_mask_1_\@: vpshufb %xmm2, %xmm3, %xmm3 vpxor %xmm3, \AAD_HASH, \AAD_HASH - cmp $0, %r10 + test %r10, %r10 jl _partial_incomplete_1_\@ # GHASH computation for the last <16 Byte block @@ -836,7 +836,7 @@ _no_extra_mask_2_\@: vpshufb %xmm2, %xmm9, %xmm9 vpxor %xmm9, \AAD_HASH, \AAD_HASH - cmp $0, %r10 + test %r10, %r10 jl _partial_incomplete_2_\@ # GHASH computation for the last <16 Byte block @@ -856,7 +856,7 @@ _encode_done_\@: vpshufb %xmm2, %xmm9, %xmm9 .endif # output encrypted Bytes - cmp $0, %r10 + test %r10, %r10 jl _partial_fill_\@ mov %r13, %r12 mov $16, %r13 diff --git a/arch/x86/crypto/poly1305-x86_64-cryptogams.pl b/arch/x86/crypto/poly1305-x86_64-cryptogams.pl index 7d568012cc15..71fae5a09e56 100644 --- a/arch/x86/crypto/poly1305-x86_64-cryptogams.pl +++ b/arch/x86/crypto/poly1305-x86_64-cryptogams.pl @@ -251,7 +251,7 @@ $code.=<<___; mov %rax,8($ctx) mov %rax,16($ctx) - cmp \$0,$inp + test $inp,$inp je .Lno_key ___ $code.=<<___ if (!$kernel); diff --git a/arch/x86/crypto/poly1305_glue.c b/arch/x86/crypto/poly1305_glue.c index c44aba290fbb..646da46e8d10 100644 --- a/arch/x86/crypto/poly1305_glue.c +++ b/arch/x86/crypto/poly1305_glue.c @@ -210,7 +210,7 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) } poly1305_simd_emit(&dctx->h, dst, dctx->s); - *dctx = (struct poly1305_desc_ctx){}; + memzero_explicit(dctx, sizeof(*dctx)); } EXPORT_SYMBOL(poly1305_final_arch); diff --git a/arch/x86/crypto/sha1_ssse3_glue.c b/arch/x86/crypto/sha1_ssse3_glue.c index 18200135603f..44340a1139e0 100644 --- a/arch/x86/crypto/sha1_ssse3_glue.c +++ b/arch/x86/crypto/sha1_ssse3_glue.c @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/x86/crypto/sha256_ssse3_glue.c b/arch/x86/crypto/sha256_ssse3_glue.c index dd06249229e1..3a5f6be7dbba 100644 --- a/arch/x86/crypto/sha256_ssse3_glue.c +++ b/arch/x86/crypto/sha256_ssse3_glue.c @@ -35,7 +35,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/x86/crypto/sha512-avx-asm.S b/arch/x86/crypto/sha512-avx-asm.S index 63470fd6ae32..684d58c8bc4f 100644 --- a/arch/x86/crypto/sha512-avx-asm.S +++ b/arch/x86/crypto/sha512-avx-asm.S @@ -278,7 +278,7 @@ frame_size = frame_GPRSAVE + GPRSAVE_SIZE # "blocks" is the message length in SHA512 blocks ######################################################################## SYM_FUNC_START(sha512_transform_avx) - cmp $0, msglen + test msglen, msglen je nowork # Allocate Stack Space diff --git a/arch/x86/crypto/sha512-ssse3-asm.S b/arch/x86/crypto/sha512-ssse3-asm.S index 7946a1bee85b..50812af0b083 100644 --- a/arch/x86/crypto/sha512-ssse3-asm.S +++ b/arch/x86/crypto/sha512-ssse3-asm.S @@ -280,7 +280,7 @@ frame_size = frame_GPRSAVE + GPRSAVE_SIZE ######################################################################## SYM_FUNC_START(sha512_transform_ssse3) - cmp $0, msglen + test msglen, msglen je nowork # Allocate Stack Space diff --git a/arch/x86/crypto/sha512_ssse3_glue.c b/arch/x86/crypto/sha512_ssse3_glue.c index b0b05c93409e..30e70f4fe2f7 100644 --- a/arch/x86/crypto/sha512_ssse3_glue.c +++ b/arch/x86/crypto/sha512_ssse3_glue.c @@ -34,7 +34,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/x86/entry/vdso/Makefile b/arch/x86/entry/vdso/Makefile index 21243747965d..02e3e42f380b 100644 --- a/arch/x86/entry/vdso/Makefile +++ b/arch/x86/entry/vdso/Makefile @@ -27,9 +27,10 @@ VDSO32-$(CONFIG_IA32_EMULATION) := y vobjs-y := vdso-note.o vclock_gettime.o vgetcpu.o vobjs32-y := vdso32/note.o vdso32/system_call.o vdso32/sigreturn.o vobjs32-y += vdso32/vclock_gettime.o +vobjs-$(CONFIG_X86_SGX) += vsgx.o # files to link into kernel -obj-y += vma.o +obj-y += vma.o extable.o KASAN_SANITIZE_vma.o := y UBSAN_SANITIZE_vma.o := y KCSAN_SANITIZE_vma.o := y @@ -98,6 +99,7 @@ $(vobjs): KBUILD_CFLAGS := $(filter-out $(GCC_PLUGINS_CFLAGS) $(RETPOLINE_CFLAGS CFLAGS_REMOVE_vclock_gettime.o = -pg CFLAGS_REMOVE_vdso32/vclock_gettime.o = -pg CFLAGS_REMOVE_vgetcpu.o = -pg +CFLAGS_REMOVE_vsgx.o = -pg # # X32 processes use x32 vDSO to access 64bit kernel data. @@ -128,8 +130,8 @@ $(obj)/%-x32.o: $(obj)/%.o FORCE targets += vdsox32.lds $(vobjx32s-y) -$(obj)/%.so: OBJCOPYFLAGS := -S -$(obj)/%.so: $(obj)/%.so.dbg FORCE +$(obj)/%.so: OBJCOPYFLAGS := -S --remove-section __ex_table +$(obj)/%.so: $(obj)/%.so.dbg $(call if_changed,objcopy) $(obj)/vdsox32.so.dbg: $(obj)/vdsox32.lds $(vobjx32s) FORCE diff --git a/arch/x86/entry/vdso/extable.c b/arch/x86/entry/vdso/extable.c new file mode 100644 index 000000000000..afcf5b65beef --- /dev/null +++ b/arch/x86/entry/vdso/extable.c @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include + +struct vdso_exception_table_entry { + int insn, fixup; +}; + +bool fixup_vdso_exception(struct pt_regs *regs, int trapnr, + unsigned long error_code, unsigned long fault_addr) +{ + const struct vdso_image *image = current->mm->context.vdso_image; + const struct vdso_exception_table_entry *extable; + unsigned int nr_entries, i; + unsigned long base; + + /* + * Do not attempt to fixup #DB or #BP. It's impossible to identify + * whether or not a #DB/#BP originated from within an SGX enclave and + * SGX enclaves are currently the only use case for vDSO fixup. + */ + if (trapnr == X86_TRAP_DB || trapnr == X86_TRAP_BP) + return false; + + if (!current->mm->context.vdso) + return false; + + base = (unsigned long)current->mm->context.vdso + image->extable_base; + nr_entries = image->extable_len / (sizeof(*extable)); + extable = image->extable; + + for (i = 0; i < nr_entries; i++) { + if (regs->ip == base + extable[i].insn) { + regs->ip = base + extable[i].fixup; + regs->di = trapnr; + regs->si = error_code; + regs->dx = fault_addr; + return true; + } + } + + return false; +} diff --git a/arch/x86/entry/vdso/extable.h b/arch/x86/entry/vdso/extable.h new file mode 100644 index 000000000000..b56f6b012941 --- /dev/null +++ b/arch/x86/entry/vdso/extable.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __VDSO_EXTABLE_H +#define __VDSO_EXTABLE_H + +/* + * Inject exception fixup for vDSO code. Unlike normal exception fixup, + * vDSO uses a dedicated handler the addresses are relative to the overall + * exception table, not each individual entry. + */ +#ifdef __ASSEMBLY__ +#define _ASM_VDSO_EXTABLE_HANDLE(from, to) \ + ASM_VDSO_EXTABLE_HANDLE from to + +.macro ASM_VDSO_EXTABLE_HANDLE from:req to:req + .pushsection __ex_table, "a" + .long (\from) - __ex_table + .long (\to) - __ex_table + .popsection +.endm +#else +#define _ASM_VDSO_EXTABLE_HANDLE(from, to) \ + ".pushsection __ex_table, \"a\"\n" \ + ".long (" #from ") - __ex_table\n" \ + ".long (" #to ") - __ex_table\n" \ + ".popsection\n" +#endif + +#endif /* __VDSO_EXTABLE_H */ diff --git a/arch/x86/entry/vdso/vdso-layout.lds.S b/arch/x86/entry/vdso/vdso-layout.lds.S index 4d152933547d..dc8da7695859 100644 --- a/arch/x86/entry/vdso/vdso-layout.lds.S +++ b/arch/x86/entry/vdso/vdso-layout.lds.S @@ -75,11 +75,18 @@ SECTIONS * stuff that isn't used at runtime in between. */ - .text : { *(.text*) } :text =0x90909090, + .text : { + *(.text*) + *(.fixup) + } :text =0x90909090, + + .altinstructions : { *(.altinstructions) } :text .altinstr_replacement : { *(.altinstr_replacement) } :text + __ex_table : { *(__ex_table) } :text + /DISCARD/ : { *(.discard) *(.discard.*) diff --git a/arch/x86/entry/vdso/vdso.lds.S b/arch/x86/entry/vdso/vdso.lds.S index 36b644e16272..4bf48462fca7 100644 --- a/arch/x86/entry/vdso/vdso.lds.S +++ b/arch/x86/entry/vdso/vdso.lds.S @@ -27,6 +27,7 @@ VERSION { __vdso_time; clock_getres; __vdso_clock_getres; + __vdso_sgx_enter_enclave; local: *; }; } diff --git a/arch/x86/entry/vdso/vdso2c.h b/arch/x86/entry/vdso/vdso2c.h index 6f46e11ce539..1c7cfac7e64a 100644 --- a/arch/x86/entry/vdso/vdso2c.h +++ b/arch/x86/entry/vdso/vdso2c.h @@ -5,6 +5,41 @@ * are built for 32-bit userspace. */ +static void BITSFUNC(copy)(FILE *outfile, const unsigned char *data, size_t len) +{ + size_t i; + + for (i = 0; i < len; i++) { + if (i % 10 == 0) + fprintf(outfile, "\n\t"); + fprintf(outfile, "0x%02X, ", (int)(data)[i]); + } +} + + +/* + * Extract a section from the input data into a standalone blob. Used to + * capture kernel-only data that needs to persist indefinitely, e.g. the + * exception fixup tables, but only in the kernel, i.e. the section can + * be stripped from the final vDSO image. + */ +static void BITSFUNC(extract)(const unsigned char *data, size_t data_len, + FILE *outfile, ELF(Shdr) *sec, const char *name) +{ + unsigned long offset; + size_t len; + + offset = (unsigned long)GET_LE(&sec->sh_offset); + len = (size_t)GET_LE(&sec->sh_size); + + if (offset + len > data_len) + fail("section to extract overruns input data"); + + fprintf(outfile, "static const unsigned char %s[%lu] = {", name, len); + BITSFUNC(copy)(outfile, data + offset, len); + fprintf(outfile, "\n};\n\n"); +} + static void BITSFUNC(go)(void *raw_addr, size_t raw_len, void *stripped_addr, size_t stripped_len, FILE *outfile, const char *image_name) @@ -15,7 +50,7 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len, ELF(Ehdr) *hdr = (ELF(Ehdr) *)raw_addr; unsigned long i, syms_nr; ELF(Shdr) *symtab_hdr = NULL, *strtab_hdr, *secstrings_hdr, - *alt_sec = NULL; + *alt_sec = NULL, *extable_sec = NULL; ELF(Dyn) *dyn = 0, *dyn_end = 0; const char *secstrings; INT_BITS syms[NSYMS] = {}; @@ -77,6 +112,8 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len, if (!strcmp(secstrings + GET_LE(&sh->sh_name), ".altinstructions")) alt_sec = sh; + if (!strcmp(secstrings + GET_LE(&sh->sh_name), "__ex_table")) + extable_sec = sh; } if (!symtab_hdr) @@ -155,6 +192,9 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len, (int)((unsigned char *)stripped_addr)[i]); } fprintf(outfile, "\n};\n\n"); + if (extable_sec) + BITSFUNC(extract)(raw_addr, raw_len, outfile, + extable_sec, "extable"); fprintf(outfile, "const struct vdso_image %s = {\n", image_name); fprintf(outfile, "\t.data = raw_data,\n"); @@ -165,6 +205,14 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len, fprintf(outfile, "\t.alt_len = %lu,\n", (unsigned long)GET_LE(&alt_sec->sh_size)); } + if (extable_sec) { + fprintf(outfile, "\t.extable_base = %lu,\n", + (unsigned long)GET_LE(&extable_sec->sh_offset)); + fprintf(outfile, "\t.extable_len = %lu,\n", + (unsigned long)GET_LE(&extable_sec->sh_size)); + fprintf(outfile, "\t.extable = extable,\n"); + } + for (i = 0; i < NSYMS; i++) { if (required_syms[i].export && syms[i]) fprintf(outfile, "\t.sym_%s = %" PRIi64 ",\n", diff --git a/arch/x86/entry/vdso/vma.c b/arch/x86/entry/vdso/vma.c index 9185cb1d13b9..50e5d3a2e70a 100644 --- a/arch/x86/entry/vdso/vma.c +++ b/arch/x86/entry/vdso/vma.c @@ -413,10 +413,10 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) #ifdef CONFIG_COMPAT int compat_arch_setup_additional_pages(struct linux_binprm *bprm, - int uses_interp) + int uses_interp, bool x32) { #ifdef CONFIG_X86_X32_ABI - if (test_thread_flag(TIF_X32)) { + if (x32) { if (!vdso64_enabled) return 0; return map_vdso_randomized(&vdso_image_x32); diff --git a/arch/x86/entry/vdso/vsgx.S b/arch/x86/entry/vdso/vsgx.S new file mode 100644 index 000000000000..86a0e94f68df --- /dev/null +++ b/arch/x86/entry/vdso/vsgx.S @@ -0,0 +1,151 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include +#include +#include +#include + +#include "extable.h" + +/* Relative to %rbp. */ +#define SGX_ENCLAVE_OFFSET_OF_RUN 16 + +/* The offsets relative to struct sgx_enclave_run. */ +#define SGX_ENCLAVE_RUN_TCS 0 +#define SGX_ENCLAVE_RUN_LEAF 8 +#define SGX_ENCLAVE_RUN_EXCEPTION_VECTOR 12 +#define SGX_ENCLAVE_RUN_EXCEPTION_ERROR_CODE 14 +#define SGX_ENCLAVE_RUN_EXCEPTION_ADDR 16 +#define SGX_ENCLAVE_RUN_USER_HANDLER 24 +#define SGX_ENCLAVE_RUN_USER_DATA 32 /* not used */ +#define SGX_ENCLAVE_RUN_RESERVED_START 40 +#define SGX_ENCLAVE_RUN_RESERVED_END 256 + +.code64 +.section .text, "ax" + +SYM_FUNC_START(__vdso_sgx_enter_enclave) + /* Prolog */ + .cfi_startproc + push %rbp + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset %rbp, 0 + mov %rsp, %rbp + .cfi_def_cfa_register %rbp + push %rbx + .cfi_rel_offset %rbx, -8 + + mov %ecx, %eax +.Lenter_enclave: + /* EENTER <= function <= ERESUME */ + cmp $EENTER, %eax + jb .Linvalid_input + cmp $ERESUME, %eax + ja .Linvalid_input + + mov SGX_ENCLAVE_OFFSET_OF_RUN(%rbp), %rcx + + /* Validate that the reserved area contains only zeros. */ + mov $SGX_ENCLAVE_RUN_RESERVED_START, %rbx +1: + cmpq $0, (%rcx, %rbx) + jne .Linvalid_input + add $8, %rbx + cmpq $SGX_ENCLAVE_RUN_RESERVED_END, %rbx + jne 1b + + /* Load TCS and AEP */ + mov SGX_ENCLAVE_RUN_TCS(%rcx), %rbx + lea .Lasync_exit_pointer(%rip), %rcx + + /* Single ENCLU serving as both EENTER and AEP (ERESUME) */ +.Lasync_exit_pointer: +.Lenclu_eenter_eresume: + enclu + + /* EEXIT jumps here unless the enclave is doing something fancy. */ + mov SGX_ENCLAVE_OFFSET_OF_RUN(%rbp), %rbx + + /* Set exit_reason. */ + movl $EEXIT, SGX_ENCLAVE_RUN_LEAF(%rbx) + + /* Invoke userspace's exit handler if one was provided. */ +.Lhandle_exit: + cmpq $0, SGX_ENCLAVE_RUN_USER_HANDLER(%rbx) + jne .Linvoke_userspace_handler + + /* Success, in the sense that ENCLU was attempted. */ + xor %eax, %eax + +.Lout: + pop %rbx + leave + .cfi_def_cfa %rsp, 8 + ret + + /* The out-of-line code runs with the pre-leave stack frame. */ + .cfi_def_cfa %rbp, 16 + +.Linvalid_input: + mov $(-EINVAL), %eax + jmp .Lout + +.Lhandle_exception: + mov SGX_ENCLAVE_OFFSET_OF_RUN(%rbp), %rbx + + /* Set the exception info. */ + mov %eax, (SGX_ENCLAVE_RUN_LEAF)(%rbx) + mov %di, (SGX_ENCLAVE_RUN_EXCEPTION_VECTOR)(%rbx) + mov %si, (SGX_ENCLAVE_RUN_EXCEPTION_ERROR_CODE)(%rbx) + mov %rdx, (SGX_ENCLAVE_RUN_EXCEPTION_ADDR)(%rbx) + jmp .Lhandle_exit + +.Linvoke_userspace_handler: + /* Pass the untrusted RSP (at exit) to the callback via %rcx. */ + mov %rsp, %rcx + + /* Save struct sgx_enclave_exception %rbx is about to be clobbered. */ + mov %rbx, %rax + + /* Save the untrusted RSP offset in %rbx (non-volatile register). */ + mov %rsp, %rbx + and $0xf, %rbx + + /* + * Align stack per x86_64 ABI. Note, %rsp needs to be 16-byte aligned + * _after_ pushing the parameters on the stack, hence the bonus push. + */ + and $-0x10, %rsp + push %rax + + /* Push struct sgx_enclave_exception as a param to the callback. */ + push %rax + + /* Clear RFLAGS.DF per x86_64 ABI */ + cld + + /* + * Load the callback pointer to %rax and lfence for LVI (load value + * injection) protection before making the call. + */ + mov SGX_ENCLAVE_RUN_USER_HANDLER(%rax), %rax + lfence + call *%rax + + /* Undo the post-exit %rsp adjustment. */ + lea 0x10(%rsp, %rbx), %rsp + + /* + * If the return from callback is zero or negative, return immediately, + * else re-execute ENCLU with the postive return value interpreted as + * the requested ENCLU function. + */ + cmp $0, %eax + jle .Lout + jmp .Lenter_enclave + + .cfi_endproc + +_ASM_VDSO_EXTABLE_HANDLE(.Lenclu_eenter_eresume, .Lhandle_exception) + +SYM_FUNC_END(__vdso_sgx_enter_enclave) diff --git a/arch/x86/entry/vsyscall/vsyscall_64.c b/arch/x86/entry/vsyscall/vsyscall_64.c index 44c33103a955..1b40b9297083 100644 --- a/arch/x86/entry/vsyscall/vsyscall_64.c +++ b/arch/x86/entry/vsyscall/vsyscall_64.c @@ -316,7 +316,7 @@ static struct vm_area_struct gate_vma __ro_after_init = { struct vm_area_struct *get_gate_vma(struct mm_struct *mm) { #ifdef CONFIG_COMPAT - if (!mm || mm->context.ia32_compat) + if (!mm || !(mm->context.flags & MM_CONTEXT_HAS_VSYSCALL)) return NULL; #endif if (vsyscall_mode == NONE) diff --git a/arch/x86/events/amd/core.c b/arch/x86/events/amd/core.c index 39eb276d0277..2c1791c4a518 100644 --- a/arch/x86/events/amd/core.c +++ b/arch/x86/events/amd/core.c @@ -538,7 +538,7 @@ static void amd_pmu_cpu_starting(int cpu) if (!x86_pmu.amd_nb_constraints) return; - nb_id = amd_get_nb_id(cpu); + nb_id = topology_die_id(cpu); WARN_ON_ONCE(nb_id == BAD_APICID); for_each_online_cpu(i) { diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index a88c94d65693..77b963e5e70a 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -2602,7 +2602,7 @@ perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *ent struct stack_frame_ia32 frame; const struct stack_frame_ia32 __user *fp; - if (!test_thread_flag(TIF_IA32)) + if (user_64bit_mode(regs)) return 0; cs_base = get_segment_base(regs->cs); diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 485c5066f8b8..a5c6ff5f1b4c 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -1261,7 +1261,7 @@ static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs) old_to = to; #ifdef CONFIG_X86_64 - is_64bit = kernel_ip(to) || !test_thread_flag(TIF_IA32); + is_64bit = kernel_ip(to) || any_64bit_mode(regs); #endif insn_init(&insn, kaddr, size, is_64bit); insn_get_length(&insn); diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c index 8961653c5dd2..1aadb253d296 100644 --- a/arch/x86/events/intel/lbr.c +++ b/arch/x86/events/intel/lbr.c @@ -1221,7 +1221,7 @@ static int branch_type(unsigned long from, unsigned long to, int abort) * on 64-bit systems running 32-bit apps */ #ifdef CONFIG_X86_64 - is64 = kernel_ip((unsigned long)addr) || !test_thread_flag(TIF_IA32); + is64 = kernel_ip((unsigned long)addr) || any_64bit_mode(current_pt_regs()); #endif insn_init(&insn, addr, bytes_read, is64); insn_get_opcode(&insn); diff --git a/arch/x86/ia32/ia32_signal.c b/arch/x86/ia32/ia32_signal.c index 81cf22398cd1..5e3d9b7fd5fb 100644 --- a/arch/x86/ia32/ia32_signal.c +++ b/arch/x86/ia32/ia32_signal.c @@ -347,7 +347,7 @@ int ia32_setup_rt_frame(int sig, struct ksignal *ksig, */ unsafe_put_user(*((u64 *)&code), (u64 __user *)frame->retcode, Efault); unsafe_put_sigcontext32(&frame->uc.uc_mcontext, fp, regs, set, Efault); - unsafe_put_user(*(__u64 *)set, (__u64 *)&frame->uc.uc_sigmask, Efault); + unsafe_put_user(*(__u64 *)set, (__u64 __user *)&frame->uc.uc_sigmask, Efault); user_access_end(); if (__copy_siginfo_to_user32(&frame->info, &ksig->info)) diff --git a/arch/x86/include/asm/acpi.h b/arch/x86/include/asm/acpi.h index 6d2df1ee427b..65064d9f7fa6 100644 --- a/arch/x86/include/asm/acpi.h +++ b/arch/x86/include/asm/acpi.h @@ -159,6 +159,8 @@ static inline u64 x86_default_get_root_pointer(void) extern int x86_acpi_numa_init(void); #endif /* CONFIG_ACPI_NUMA */ +struct cper_ia_proc_ctx; + #ifdef CONFIG_ACPI_APEI static inline pgprot_t arch_apei_get_mem_attribute(phys_addr_t addr) { @@ -177,6 +179,15 @@ static inline pgprot_t arch_apei_get_mem_attribute(phys_addr_t addr) */ return PAGE_KERNEL_NOENC; } + +int arch_apei_report_x86_error(struct cper_ia_proc_ctx *ctx_info, + u64 lapic_id); +#else +static inline int arch_apei_report_x86_error(struct cper_ia_proc_ctx *ctx_info, + u64 lapic_id) +{ + return -EINVAL; +} #endif #define ACPI_TABLE_UPGRADE_MAX_PHYS (max_low_pfn_mapped << PAGE_SHIFT) diff --git a/arch/x86/include/asm/cacheinfo.h b/arch/x86/include/asm/cacheinfo.h index 86b63c7feab7..86b2e0dcc4bf 100644 --- a/arch/x86/include/asm/cacheinfo.h +++ b/arch/x86/include/asm/cacheinfo.h @@ -2,7 +2,7 @@ #ifndef _ASM_X86_CACHEINFO_H #define _ASM_X86_CACHEINFO_H -void cacheinfo_amd_init_llc_id(struct cpuinfo_x86 *c, int cpu, u8 node_id); -void cacheinfo_hygon_init_llc_id(struct cpuinfo_x86 *c, int cpu, u8 node_id); +void cacheinfo_amd_init_llc_id(struct cpuinfo_x86 *c, int cpu); +void cacheinfo_hygon_init_llc_id(struct cpuinfo_x86 *c, int cpu); #endif /* _ASM_X86_CACHEINFO_H */ diff --git a/arch/x86/include/asm/compat.h b/arch/x86/include/asm/compat.h index 0e327a01f50f..f145e3326c6d 100644 --- a/arch/x86/include/asm/compat.h +++ b/arch/x86/include/asm/compat.h @@ -177,14 +177,13 @@ typedef struct user_regs_struct compat_elf_gregset_t; static inline void __user *arch_compat_alloc_user_space(long len) { - compat_uptr_t sp; + compat_uptr_t sp = task_pt_regs(current)->sp; - if (test_thread_flag(TIF_IA32)) { - sp = task_pt_regs(current)->sp; - } else { - /* -128 for the x32 ABI redzone */ - sp = task_pt_regs(current)->sp - 128; - } + /* + * -128 for the x32 ABI redzone. For IA32, it is not strictly + * necessary, but not harmful. + */ + sp -= 128; return (void __user *)round_down(sp - len, 16); } diff --git a/arch/x86/include/asm/copy_mc_test.h b/arch/x86/include/asm/copy_mc_test.h deleted file mode 100644 index e4991ba96726..000000000000 --- a/arch/x86/include/asm/copy_mc_test.h +++ /dev/null @@ -1,75 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _COPY_MC_TEST_H_ -#define _COPY_MC_TEST_H_ - -#ifndef __ASSEMBLY__ -#ifdef CONFIG_COPY_MC_TEST -extern unsigned long copy_mc_test_src; -extern unsigned long copy_mc_test_dst; - -static inline void copy_mc_inject_src(void *addr) -{ - if (addr) - copy_mc_test_src = (unsigned long) addr; - else - copy_mc_test_src = ~0UL; -} - -static inline void copy_mc_inject_dst(void *addr) -{ - if (addr) - copy_mc_test_dst = (unsigned long) addr; - else - copy_mc_test_dst = ~0UL; -} -#else /* CONFIG_COPY_MC_TEST */ -static inline void copy_mc_inject_src(void *addr) -{ -} - -static inline void copy_mc_inject_dst(void *addr) -{ -} -#endif /* CONFIG_COPY_MC_TEST */ - -#else /* __ASSEMBLY__ */ -#include - -#ifdef CONFIG_COPY_MC_TEST -.macro COPY_MC_TEST_CTL - .pushsection .data - .align 8 - .globl copy_mc_test_src - copy_mc_test_src: - .quad 0 - EXPORT_SYMBOL_GPL(copy_mc_test_src) - .globl copy_mc_test_dst - copy_mc_test_dst: - .quad 0 - EXPORT_SYMBOL_GPL(copy_mc_test_dst) - .popsection -.endm - -.macro COPY_MC_TEST_SRC reg count target - leaq \count(\reg), %r9 - cmp copy_mc_test_src, %r9 - ja \target -.endm - -.macro COPY_MC_TEST_DST reg count target - leaq \count(\reg), %r9 - cmp copy_mc_test_dst, %r9 - ja \target -.endm -#else -.macro COPY_MC_TEST_CTL -.endm - -.macro COPY_MC_TEST_SRC reg count target -.endm - -.macro COPY_MC_TEST_DST reg count target -.endm -#endif /* CONFIG_COPY_MC_TEST */ -#endif /* __ASSEMBLY__ */ -#endif /* _COPY_MC_TEST_H_ */ diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index dad350d42ecf..f5ef2d5b9231 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -241,6 +241,7 @@ /* Intel-defined CPU features, CPUID level 0x00000007:0 (EBX), word 9 */ #define X86_FEATURE_FSGSBASE ( 9*32+ 0) /* RDFSBASE, WRFSBASE, RDGSBASE, WRGSBASE instructions*/ #define X86_FEATURE_TSC_ADJUST ( 9*32+ 1) /* TSC adjustment MSR 0x3B */ +#define X86_FEATURE_SGX ( 9*32+ 2) /* Software Guard Extensions */ #define X86_FEATURE_BMI1 ( 9*32+ 3) /* 1st group bit manipulation extensions */ #define X86_FEATURE_HLE ( 9*32+ 4) /* Hardware Lock Elision */ #define X86_FEATURE_AVX2 ( 9*32+ 5) /* AVX2 instructions */ @@ -356,6 +357,7 @@ #define X86_FEATURE_MOVDIRI (16*32+27) /* MOVDIRI instruction */ #define X86_FEATURE_MOVDIR64B (16*32+28) /* MOVDIR64B instruction */ #define X86_FEATURE_ENQCMD (16*32+29) /* ENQCMD and ENQCMDS instructions */ +#define X86_FEATURE_SGX_LC (16*32+30) /* Software Guard Extensions Launch Control */ /* AMD-defined CPU features, CPUID level 0x80000007 (EBX), word 17 */ #define X86_FEATURE_OVERFLOW_RECOV (17*32+ 0) /* MCA overflow recovery support */ diff --git a/arch/x86/include/asm/disabled-features.h b/arch/x86/include/asm/disabled-features.h index 5861d34f9771..7947cb1782da 100644 --- a/arch/x86/include/asm/disabled-features.h +++ b/arch/x86/include/asm/disabled-features.h @@ -62,6 +62,12 @@ # define DISABLE_ENQCMD (1 << (X86_FEATURE_ENQCMD & 31)) #endif +#ifdef CONFIG_X86_SGX +# define DISABLE_SGX 0 +#else +# define DISABLE_SGX (1 << (X86_FEATURE_SGX & 31)) +#endif + /* * Make sure to add features to the correct mask */ @@ -74,7 +80,7 @@ #define DISABLED_MASK6 0 #define DISABLED_MASK7 (DISABLE_PTI) #define DISABLED_MASK8 0 -#define DISABLED_MASK9 (DISABLE_SMAP) +#define DISABLED_MASK9 (DISABLE_SMAP|DISABLE_SGX) #define DISABLED_MASK10 0 #define DISABLED_MASK11 0 #define DISABLED_MASK12 0 diff --git a/arch/x86/include/asm/elf.h b/arch/x86/include/asm/elf.h index b9a5d488f1a5..44a9b9940535 100644 --- a/arch/x86/include/asm/elf.h +++ b/arch/x86/include/asm/elf.h @@ -186,8 +186,9 @@ static inline void elf_common_init(struct thread_struct *t, #define COMPAT_ELF_PLAT_INIT(regs, load_addr) \ elf_common_init(¤t->thread, regs, __USER_DS) -void compat_start_thread(struct pt_regs *regs, u32 new_ip, u32 new_sp); -#define compat_start_thread compat_start_thread +void compat_start_thread(struct pt_regs *regs, u32 new_ip, u32 new_sp, bool x32); +#define COMPAT_START_THREAD(ex, regs, new_ip, new_sp) \ + compat_start_thread(regs, new_ip, new_sp, ex->e_machine == EM_X86_64) void set_personality_ia32(bool); #define COMPAT_SET_PERSONALITY(ex) \ @@ -361,7 +362,7 @@ do { \ #define AT_SYSINFO 32 #define COMPAT_ARCH_DLINFO \ -if (test_thread_flag(TIF_X32)) \ +if (exec->e_machine == EM_X86_64) \ ARCH_DLINFO_X32; \ else \ ARCH_DLINFO_IA32 @@ -382,8 +383,10 @@ struct linux_binprm; extern int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp); extern int compat_arch_setup_additional_pages(struct linux_binprm *bprm, - int uses_interp); -#define compat_arch_setup_additional_pages compat_arch_setup_additional_pages + int uses_interp, bool x32); +#define COMPAT_ARCH_SETUP_ADDITIONAL_PAGES(bprm, ex, interpreter) \ + compat_arch_setup_additional_pages(bprm, interpreter, \ + (ex->e_machine == EM_X86_64)) /* Do not change the values. See get_align_mask() */ enum align_flags { diff --git a/arch/x86/include/asm/enclu.h b/arch/x86/include/asm/enclu.h new file mode 100644 index 000000000000..b1314e41a744 --- /dev/null +++ b/arch/x86/include/asm/enclu.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_ENCLU_H +#define _ASM_X86_ENCLU_H + +#define EENTER 0x02 +#define ERESUME 0x03 +#define EEXIT 0x04 + +#endif /* _ASM_X86_ENCLU_H */ diff --git a/arch/x86/include/asm/inst.h b/arch/x86/include/asm/inst.h index bd7f02480ca1..438ccd4f3cc4 100644 --- a/arch/x86/include/asm/inst.h +++ b/arch/x86/include/asm/inst.h @@ -143,21 +143,6 @@ .macro MODRM mod opd1 opd2 .byte \mod | (\opd1 & 7) | ((\opd2 & 7) << 3) .endm - -.macro RDPID opd - REG_TYPE rdpid_opd_type \opd - .if rdpid_opd_type == REG_TYPE_R64 - R64_NUM rdpid_opd \opd - .else - R32_NUM rdpid_opd \opd - .endif - .byte 0xf3 - .if rdpid_opd > 7 - PFX_REX rdpid_opd 0 - .endif - .byte 0x0f, 0xc7 - MODRM 0xc0 rdpid_opd 0x7 -.endm #endif #endif diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h index a0f147893a04..56cdeaac76a0 100644 --- a/arch/x86/include/asm/mce.h +++ b/arch/x86/include/asm/mce.h @@ -177,7 +177,8 @@ enum mce_notifier_prios { MCE_PRIO_EXTLOG, MCE_PRIO_UC, MCE_PRIO_EARLY, - MCE_PRIO_CEC + MCE_PRIO_CEC, + MCE_PRIO_HIGHEST = MCE_PRIO_CEC }; struct notifier_block; @@ -198,16 +199,22 @@ static inline void enable_copy_mc_fragile(void) } #endif +struct cper_ia_proc_ctx; + #ifdef CONFIG_X86_MCE int mcheck_init(void); void mcheck_cpu_init(struct cpuinfo_x86 *c); void mcheck_cpu_clear(struct cpuinfo_x86 *c); void mcheck_vendor_init_severity(void); +int apei_smca_report_x86_error(struct cper_ia_proc_ctx *ctx_info, + u64 lapic_id); #else static inline int mcheck_init(void) { return 0; } static inline void mcheck_cpu_init(struct cpuinfo_x86 *c) {} static inline void mcheck_cpu_clear(struct cpuinfo_x86 *c) {} static inline void mcheck_vendor_init_severity(void) {} +static inline int apei_smca_report_x86_error(struct cper_ia_proc_ctx *ctx_info, + u64 lapic_id) { return -EINVAL; } #endif #ifdef CONFIG_X86_ANCIENT_MCE diff --git a/arch/x86/include/asm/mmu.h b/arch/x86/include/asm/mmu.h index 9257667d13c5..5d7494631ea9 100644 --- a/arch/x86/include/asm/mmu.h +++ b/arch/x86/include/asm/mmu.h @@ -6,6 +6,12 @@ #include #include #include +#include + +/* Uprobes on this MM assume 32-bit code */ +#define MM_CONTEXT_UPROBE_IA32 BIT(0) +/* vsyscall page is accessible on this MM */ +#define MM_CONTEXT_HAS_VSYSCALL BIT(1) /* * x86 has arch-specific MMU state beyond what lives in mm_struct. @@ -33,8 +39,7 @@ typedef struct { #endif #ifdef CONFIG_X86_64 - /* True if mm supports a task running in 32 bit compatibility mode. */ - unsigned short ia32_compat; + unsigned short flags; #endif struct mutex lock; diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h index d98016b83755..054a79157323 100644 --- a/arch/x86/include/asm/mmu_context.h +++ b/arch/x86/include/asm/mmu_context.h @@ -177,7 +177,7 @@ static inline void arch_exit_mmap(struct mm_struct *mm) static inline bool is_64bit_mm(struct mm_struct *mm) { return !IS_ENABLED(CONFIG_IA32_EMULATION) || - !(mm->context.ia32_compat == TIF_IA32); + !(mm->context.flags & MM_CONTEXT_UPROBE_IA32); } #else static inline bool is_64bit_mm(struct mm_struct *mm) diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 972a34d93505..6a5085a5f9d8 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -139,6 +139,7 @@ #define MSR_IA32_MCG_CAP 0x00000179 #define MSR_IA32_MCG_STATUS 0x0000017a #define MSR_IA32_MCG_CTL 0x0000017b +#define MSR_ERROR_CONTROL 0x0000017f #define MSR_IA32_MCG_EXT_CTL 0x000004d0 #define MSR_OFFCORE_RSP_0 0x000001a6 @@ -609,6 +610,8 @@ #define FEAT_CTL_LOCKED BIT(0) #define FEAT_CTL_VMX_ENABLED_INSIDE_SMX BIT(1) #define FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX BIT(2) +#define FEAT_CTL_SGX_LC_ENABLED BIT(17) +#define FEAT_CTL_SGX_ENABLED BIT(18) #define FEAT_CTL_LMCE_ENABLED BIT(20) #define MSR_IA32_TSC_ADJUST 0x0000003b @@ -628,6 +631,12 @@ #define MSR_IA32_UCODE_WRITE 0x00000079 #define MSR_IA32_UCODE_REV 0x0000008b +/* Intel SGX Launch Enclave Public Key Hash MSRs */ +#define MSR_IA32_SGXLEPUBKEYHASH0 0x0000008C +#define MSR_IA32_SGXLEPUBKEYHASH1 0x0000008D +#define MSR_IA32_SGXLEPUBKEYHASH2 0x0000008E +#define MSR_IA32_SGXLEPUBKEYHASH3 0x0000008F + #define MSR_IA32_SMM_MONITOR_CTL 0x0000009b #define MSR_IA32_SMBASE 0x0000009e diff --git a/arch/x86/include/asm/page_32_types.h b/arch/x86/include/asm/page_32_types.h index f462895a33e4..faf9cc1c14bb 100644 --- a/arch/x86/include/asm/page_32_types.h +++ b/arch/x86/include/asm/page_32_types.h @@ -53,7 +53,13 @@ #define STACK_TOP_MAX STACK_TOP /* - * Kernel image size is limited to 512 MB (see in arch/x86/kernel/head_32.S) + * In spite of the name, KERNEL_IMAGE_SIZE is a limit on the maximum virtual + * address for the kernel image, rather than the limit on the size itself. On + * 32-bit, this is not a strict limit, but this value is used to limit the + * link-time virtual address range of the kernel, and by KASLR to limit the + * randomized address from which the kernel is executed. A relocatable kernel + * can be loaded somewhat higher than KERNEL_IMAGE_SIZE as long as enough space + * remains for the vmalloc area. */ #define KERNEL_IMAGE_SIZE (512 * 1024 * 1024) diff --git a/arch/x86/include/asm/page_64_types.h b/arch/x86/include/asm/page_64_types.h index 3f49dac03617..645bd1d0ee07 100644 --- a/arch/x86/include/asm/page_64_types.h +++ b/arch/x86/include/asm/page_64_types.h @@ -98,8 +98,10 @@ #define STACK_TOP_MAX TASK_SIZE_MAX /* - * Maximum kernel image size is limited to 1 GiB, due to the fixmap living - * in the next 1 GiB (see level2_kernel_pgt in arch/x86/kernel/head_64.S). + * In spite of the name, KERNEL_IMAGE_SIZE is a limit on the maximum virtual + * address for the kernel image, rather than the limit on the size itself. + * This can be at most 1 GiB, due to the fixmap living in the next 1 GiB (see + * level2_kernel_pgt in arch/x86/kernel/head_64.S). * * On KASLR use 1 GiB by default, leaving 1 GiB for modules once the * page tables are fully set up. diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index d25cc6830e89..f8dce11d2bc1 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -812,17 +812,6 @@ extern void default_banner(void); #endif /* CONFIG_PARAVIRT_XXL */ #endif /* CONFIG_X86_64 */ -#ifdef CONFIG_PARAVIRT_XXL - -#define GET_CR2_INTO_AX \ - PARA_SITE(PARA_PATCH(PV_MMU_read_cr2), \ - ANNOTATE_RETPOLINE_SAFE; \ - call PARA_INDIRECT(pv_ops+PV_MMU_read_cr2); \ - ) - -#endif /* CONFIG_PARAVIRT_XXL */ - - #endif /* __ASSEMBLY__ */ #else /* CONFIG_PARAVIRT */ # define default_banner x86_init_noop diff --git a/arch/x86/include/asm/pgtable_32.h b/arch/x86/include/asm/pgtable_32.h index d7acae4120d5..7c9c968a42ef 100644 --- a/arch/x86/include/asm/pgtable_32.h +++ b/arch/x86/include/asm/pgtable_32.h @@ -57,19 +57,13 @@ do { \ #endif /* - * This is how much memory in addition to the memory covered up to - * and including _end we need mapped initially. - * We need: - * (KERNEL_IMAGE_SIZE/4096) / 1024 pages (worst case, non PAE) - * (KERNEL_IMAGE_SIZE/4096) / 512 + 4 pages (worst case for PAE) + * This is used to calculate the .brk reservation for initial pagetables. + * Enough space is reserved to allocate pagetables sufficient to cover all + * of LOWMEM_PAGES, which is an upper bound on the size of the direct map of + * lowmem. * - * Modulo rounding, each megabyte assigned here requires a kilobyte of - * memory, which is currently unreclaimed. - * - * This should be a multiple of a page. - * - * KERNEL_IMAGE_SIZE should be greater than pa(_end) - * and small than max_low_pfn, otherwise will waste some page table entries + * With PAE paging (PTRS_PER_PMD > 1), we allocate PTRS_PER_PGD == 4 pages for + * the PMD's in addition to the pages required for the last level pagetables. */ #if PTRS_PER_PMD > 1 #define PAGE_TABLE_SIZE(pages) (((pages) / PTRS_PER_PMD) + PTRS_PER_PGD) diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 82a08b585818..c20a52b5534b 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -813,10 +813,8 @@ extern int set_tsc_mode(unsigned int val); DECLARE_PER_CPU(u64, msr_misc_features_shadow); #ifdef CONFIG_CPU_SUP_AMD -extern u16 amd_get_nb_id(int cpu); extern u32 amd_get_nodes_per_socket(void); #else -static inline u16 amd_get_nb_id(int cpu) { return 0; } static inline u32 amd_get_nodes_per_socket(void) { return 0; } #endif diff --git a/arch/x86/include/asm/stacktrace.h b/arch/x86/include/asm/stacktrace.h index 49600643faba..f248eb2ac2d4 100644 --- a/arch/x86/include/asm/stacktrace.h +++ b/arch/x86/include/asm/stacktrace.h @@ -88,9 +88,6 @@ get_stack_pointer(struct task_struct *task, struct pt_regs *regs) return (unsigned long *)task->thread.sp; } -void show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs, - unsigned long *stack, const char *log_lvl); - /* The form of the top of the frame on the stack */ struct stack_frame { struct stack_frame *next_frame; diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h index 44733a4bfc42..a12b9644193b 100644 --- a/arch/x86/include/asm/thread_info.h +++ b/arch/x86/include/asm/thread_info.h @@ -91,7 +91,6 @@ struct thread_info { #define TIF_NEED_FPU_LOAD 14 /* load FPU on return to userspace */ #define TIF_NOCPUID 15 /* CPUID is not accessible in userland */ #define TIF_NOTSC 16 /* TSC is not accessible in userland */ -#define TIF_IA32 17 /* IA32 compatibility process */ #define TIF_SLD 18 /* Restore split lock detection on context switch */ #define TIF_MEMDIE 20 /* is terminating due to OOM killer */ #define TIF_POLLING_NRFLAG 21 /* idle is polling for TIF_NEED_RESCHED */ @@ -101,7 +100,6 @@ struct thread_info { #define TIF_LAZY_MMU_UPDATES 27 /* task is updating the mmu lazily */ #define TIF_SYSCALL_TRACEPOINT 28 /* syscall tracepoint instrumentation */ #define TIF_ADDR32 29 /* 32-bit address space on 64 bits */ -#define TIF_X32 30 /* 32-bit native x86-64 binary */ #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) @@ -120,7 +118,6 @@ struct thread_info { #define _TIF_NEED_FPU_LOAD (1 << TIF_NEED_FPU_LOAD) #define _TIF_NOCPUID (1 << TIF_NOCPUID) #define _TIF_NOTSC (1 << TIF_NOTSC) -#define _TIF_IA32 (1 << TIF_IA32) #define _TIF_SLD (1 << TIF_SLD) #define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG) #define _TIF_IO_BITMAP (1 << TIF_IO_BITMAP) @@ -129,7 +126,6 @@ struct thread_info { #define _TIF_LAZY_MMU_UPDATES (1 << TIF_LAZY_MMU_UPDATES) #define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) #define _TIF_ADDR32 (1 << TIF_ADDR32) -#define _TIF_X32 (1 << TIF_X32) /* flags to check in __switch_to() */ #define _TIF_WORK_CTXSW_BASE \ diff --git a/arch/x86/include/asm/trap_pf.h b/arch/x86/include/asm/trap_pf.h index 305bc1214aef..10b1de500ab1 100644 --- a/arch/x86/include/asm/trap_pf.h +++ b/arch/x86/include/asm/trap_pf.h @@ -11,6 +11,7 @@ * bit 3 == 1: use of reserved bit detected * bit 4 == 1: fault was an instruction fetch * bit 5 == 1: protection keys block access + * bit 15 == 1: SGX MMU page-fault */ enum x86_pf_error_code { X86_PF_PROT = 1 << 0, @@ -19,6 +20,7 @@ enum x86_pf_error_code { X86_PF_RSVD = 1 << 3, X86_PF_INSTR = 1 << 4, X86_PF_PK = 1 << 5, + X86_PF_SGX = 1 << 15, }; #endif /* _ASM_X86_TRAP_PF_H */ diff --git a/arch/x86/include/asm/uv/bios.h b/arch/x86/include/asm/uv/bios.h index 08b3d810dfba..1b6455f881f9 100644 --- a/arch/x86/include/asm/uv/bios.h +++ b/arch/x86/include/asm/uv/bios.h @@ -28,6 +28,20 @@ enum uv_bios_cmd { UV_BIOS_SET_LEGACY_VGA_TARGET }; +#define UV_BIOS_EXTRA 0x10000 +#define UV_BIOS_GET_PCI_TOPOLOGY 0x10001 +#define UV_BIOS_GET_GEOINFO 0x10003 + +#define UV_BIOS_EXTRA_OP_MEM_COPYIN 0x1000 +#define UV_BIOS_EXTRA_OP_MEM_COPYOUT 0x2000 +#define UV_BIOS_EXTRA_OP_MASK 0x0fff +#define UV_BIOS_EXTRA_GET_HEAPSIZE 1 +#define UV_BIOS_EXTRA_INSTALL_HEAP 2 +#define UV_BIOS_EXTRA_MASTER_NASID 3 +#define UV_BIOS_EXTRA_OBJECT_COUNT (10|UV_BIOS_EXTRA_OP_MEM_COPYOUT) +#define UV_BIOS_EXTRA_ENUM_OBJECTS (12|UV_BIOS_EXTRA_OP_MEM_COPYOUT) +#define UV_BIOS_EXTRA_ENUM_PORTS (13|UV_BIOS_EXTRA_OP_MEM_COPYOUT) + /* * Status values returned from a BIOS call. */ @@ -109,6 +123,32 @@ struct uv_systab { } entry[1]; /* additional entries follow */ }; extern struct uv_systab *uv_systab; + +#define UV_BIOS_MAXSTRING 128 +struct uv_bios_hub_info { + unsigned int id; + union { + struct { + unsigned long long this_part:1; + unsigned long long is_shared:1; + unsigned long long is_disabled:1; + } fields; + struct { + unsigned long long flags; + unsigned long long reserved; + } b; + } f; + char name[UV_BIOS_MAXSTRING]; + char location[UV_BIOS_MAXSTRING]; + unsigned int ports; +}; + +struct uv_bios_port_info { + unsigned int port; + unsigned int conn_id; + unsigned int conn_port; +}; + /* (... end of definitions from UV BIOS ...) */ enum { @@ -142,6 +182,15 @@ extern s64 uv_bios_change_memprotect(u64, u64, enum uv_memprotect); extern s64 uv_bios_reserved_page_pa(u64, u64 *, u64 *, u64 *); extern int uv_bios_set_legacy_vga_target(bool decode, int domain, int bus); +extern s64 uv_bios_get_master_nasid(u64 sz, u64 *nasid); +extern s64 uv_bios_get_heapsize(u64 nasid, u64 sz, u64 *heap_sz); +extern s64 uv_bios_install_heap(u64 nasid, u64 sz, u64 *heap); +extern s64 uv_bios_obj_count(u64 nasid, u64 sz, u64 *objcnt); +extern s64 uv_bios_enum_objs(u64 nasid, u64 sz, u64 *objbuf); +extern s64 uv_bios_enum_ports(u64 nasid, u64 obj_id, u64 sz, u64 *portbuf); +extern s64 uv_bios_get_geoinfo(u64 nasid, u64 sz, u64 *geo); +extern s64 uv_bios_get_pci_topology(u64 sz, u64 *buf); + extern int uv_bios_init(void); extern unsigned long get_uv_systab_phys(bool msg); @@ -151,6 +200,8 @@ extern long sn_partition_id; extern long sn_coherency_id; extern long sn_region_size; extern long system_serial_number; +extern ssize_t uv_get_archtype(char *buf, int len); +extern int uv_get_hubless_system(void); extern struct kobject *sgi_uv_kobj; /* /sys/firmware/sgi_uv */ diff --git a/arch/x86/include/asm/uv/uv_geo.h b/arch/x86/include/asm/uv/uv_geo.h new file mode 100644 index 000000000000..f241451035fb --- /dev/null +++ b/arch/x86/include/asm/uv/uv_geo.h @@ -0,0 +1,103 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2020 Hewlett Packard Enterprise Development LP. All rights reserved. + */ + +#ifndef _ASM_UV_GEO_H +#define _ASM_UV_GEO_H + +/* Type declaractions */ + +/* Size of a geoid_s structure (must be before decl. of geoid_u) */ +#define GEOID_SIZE 8 + +/* Fields common to all substructures */ +struct geo_common_s { + unsigned char type; /* What type of h/w is named by this geoid_s */ + unsigned char blade; + unsigned char slot; /* slot is IRU */ + unsigned char upos; + unsigned char rack; +}; + +/* Additional fields for particular types of hardware */ +struct geo_node_s { + struct geo_common_s common; /* No additional fields needed */ +}; + +struct geo_rtr_s { + struct geo_common_s common; /* No additional fields needed */ +}; + +struct geo_iocntl_s { + struct geo_common_s common; /* No additional fields needed */ +}; + +struct geo_pcicard_s { + struct geo_iocntl_s common; + char bus; /* Bus/widget number */ + char slot; /* PCI slot number */ +}; + +/* Subcomponents of a node */ +struct geo_cpu_s { + struct geo_node_s node; + unsigned char socket:4, /* Which CPU on the node */ + thread:4; + unsigned char core; +}; + +struct geo_mem_s { + struct geo_node_s node; + char membus; /* The memory bus on the node */ + char memslot; /* The memory slot on the bus */ +}; + +union geoid_u { + struct geo_common_s common; + struct geo_node_s node; + struct geo_iocntl_s iocntl; + struct geo_pcicard_s pcicard; + struct geo_rtr_s rtr; + struct geo_cpu_s cpu; + struct geo_mem_s mem; + char padsize[GEOID_SIZE]; +}; + +/* Defined constants */ + +#define GEO_MAX_LEN 48 + +#define GEO_TYPE_INVALID 0 +#define GEO_TYPE_MODULE 1 +#define GEO_TYPE_NODE 2 +#define GEO_TYPE_RTR 3 +#define GEO_TYPE_IOCNTL 4 +#define GEO_TYPE_IOCARD 5 +#define GEO_TYPE_CPU 6 +#define GEO_TYPE_MEM 7 +#define GEO_TYPE_MAX (GEO_TYPE_MEM+1) + +static inline int geo_rack(union geoid_u g) +{ + return (g.common.type == GEO_TYPE_INVALID) ? + -1 : g.common.rack; +} + +static inline int geo_slot(union geoid_u g) +{ + return (g.common.type == GEO_TYPE_INVALID) ? + -1 : g.common.upos; +} + +static inline int geo_blade(union geoid_u g) +{ + return (g.common.type == GEO_TYPE_INVALID) ? + -1 : g.common.blade * 2 + g.common.slot; +} + +#endif /* _ASM_UV_GEO_H */ diff --git a/arch/x86/include/asm/vdso.h b/arch/x86/include/asm/vdso.h index bbcdc7b8f963..b5d23470f56b 100644 --- a/arch/x86/include/asm/vdso.h +++ b/arch/x86/include/asm/vdso.h @@ -15,6 +15,8 @@ struct vdso_image { unsigned long size; /* Always a multiple of PAGE_SIZE */ unsigned long alt, alt_len; + unsigned long extable_base, extable_len; + const void *extable; long sym_vvar_start; /* Negative offset to the vvar area */ @@ -45,6 +47,9 @@ extern void __init init_vdso_image(const struct vdso_image *image); extern int map_vdso_once(const struct vdso_image *image, unsigned long addr); +extern bool fixup_vdso_exception(struct pt_regs *regs, int trapnr, + unsigned long error_code, + unsigned long fault_addr); #endif /* __ASSEMBLER__ */ #endif /* _ASM_X86_VDSO_H */ diff --git a/arch/x86/include/uapi/asm/sgx.h b/arch/x86/include/uapi/asm/sgx.h new file mode 100644 index 000000000000..9034f3007c4e --- /dev/null +++ b/arch/x86/include/uapi/asm/sgx.h @@ -0,0 +1,168 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * Copyright(c) 2016-20 Intel Corporation. + */ +#ifndef _UAPI_ASM_X86_SGX_H +#define _UAPI_ASM_X86_SGX_H + +#include +#include + +/** + * enum sgx_page_flags - page control flags + * %SGX_PAGE_MEASURE: Measure the page contents with a sequence of + * ENCLS[EEXTEND] operations. + */ +enum sgx_page_flags { + SGX_PAGE_MEASURE = 0x01, +}; + +#define SGX_MAGIC 0xA4 + +#define SGX_IOC_ENCLAVE_CREATE \ + _IOW(SGX_MAGIC, 0x00, struct sgx_enclave_create) +#define SGX_IOC_ENCLAVE_ADD_PAGES \ + _IOWR(SGX_MAGIC, 0x01, struct sgx_enclave_add_pages) +#define SGX_IOC_ENCLAVE_INIT \ + _IOW(SGX_MAGIC, 0x02, struct sgx_enclave_init) +#define SGX_IOC_ENCLAVE_PROVISION \ + _IOW(SGX_MAGIC, 0x03, struct sgx_enclave_provision) + +/** + * struct sgx_enclave_create - parameter structure for the + * %SGX_IOC_ENCLAVE_CREATE ioctl + * @src: address for the SECS page data + */ +struct sgx_enclave_create { + __u64 src; +}; + +/** + * struct sgx_enclave_add_pages - parameter structure for the + * %SGX_IOC_ENCLAVE_ADD_PAGE ioctl + * @src: start address for the page data + * @offset: starting page offset + * @length: length of the data (multiple of the page size) + * @secinfo: address for the SECINFO data + * @flags: page control flags + * @count: number of bytes added (multiple of the page size) + */ +struct sgx_enclave_add_pages { + __u64 src; + __u64 offset; + __u64 length; + __u64 secinfo; + __u64 flags; + __u64 count; +}; + +/** + * struct sgx_enclave_init - parameter structure for the + * %SGX_IOC_ENCLAVE_INIT ioctl + * @sigstruct: address for the SIGSTRUCT data + */ +struct sgx_enclave_init { + __u64 sigstruct; +}; + +/** + * struct sgx_enclave_provision - parameter structure for the + * %SGX_IOC_ENCLAVE_PROVISION ioctl + * @fd: file handle of /dev/sgx_provision + */ +struct sgx_enclave_provision { + __u64 fd; +}; + +struct sgx_enclave_run; + +/** + * typedef sgx_enclave_user_handler_t - Exit handler function accepted by + * __vdso_sgx_enter_enclave() + * @run: The run instance given by the caller + * + * The register parameters contain the snapshot of their values at enclave + * exit. An invalid ENCLU function number will cause -EINVAL to be returned + * to the caller. + * + * Return: + * - <= 0: The given value is returned back to the caller. + * - > 0: ENCLU function to invoke, either EENTER or ERESUME. + */ +typedef int (*sgx_enclave_user_handler_t)(long rdi, long rsi, long rdx, + long rsp, long r8, long r9, + struct sgx_enclave_run *run); + +/** + * struct sgx_enclave_run - the execution context of __vdso_sgx_enter_enclave() + * @tcs: TCS used to enter the enclave + * @function: The last seen ENCLU function (EENTER, ERESUME or EEXIT) + * @exception_vector: The interrupt vector of the exception + * @exception_error_code: The exception error code pulled out of the stack + * @exception_addr: The address that triggered the exception + * @user_handler: User provided callback run on exception + * @user_data: Data passed to the user handler + * @reserved Reserved for future extensions + * + * If @user_handler is provided, the handler will be invoked on all return paths + * of the normal flow. The user handler may transfer control, e.g. via a + * longjmp() call or a C++ exception, without returning to + * __vdso_sgx_enter_enclave(). + */ +struct sgx_enclave_run { + __u64 tcs; + __u32 function; + __u16 exception_vector; + __u16 exception_error_code; + __u64 exception_addr; + __u64 user_handler; + __u64 user_data; + __u8 reserved[216]; +}; + +/** + * typedef vdso_sgx_enter_enclave_t - Prototype for __vdso_sgx_enter_enclave(), + * a vDSO function to enter an SGX enclave. + * @rdi: Pass-through value for RDI + * @rsi: Pass-through value for RSI + * @rdx: Pass-through value for RDX + * @function: ENCLU function, must be EENTER or ERESUME + * @r8: Pass-through value for R8 + * @r9: Pass-through value for R9 + * @run: struct sgx_enclave_run, must be non-NULL + * + * NOTE: __vdso_sgx_enter_enclave() does not ensure full compliance with the + * x86-64 ABI, e.g. doesn't handle XSAVE state. Except for non-volatile + * general purpose registers, EFLAGS.DF, and RSP alignment, preserving/setting + * state in accordance with the x86-64 ABI is the responsibility of the enclave + * and its runtime, i.e. __vdso_sgx_enter_enclave() cannot be called from C + * code without careful consideration by both the enclave and its runtime. + * + * All general purpose registers except RAX, RBX and RCX are passed as-is to the + * enclave. RAX, RBX and RCX are consumed by EENTER and ERESUME and are loaded + * with @function, asynchronous exit pointer, and @run.tcs respectively. + * + * RBP and the stack are used to anchor __vdso_sgx_enter_enclave() to the + * pre-enclave state, e.g. to retrieve @run.exception and @run.user_handler + * after an enclave exit. All other registers are available for use by the + * enclave and its runtime, e.g. an enclave can push additional data onto the + * stack (and modify RSP) to pass information to the optional user handler (see + * below). + * + * Most exceptions reported on ENCLU, including those that occur within the + * enclave, are fixed up and reported synchronously instead of being delivered + * via a standard signal. Debug Exceptions (#DB) and Breakpoints (#BP) are + * never fixed up and are always delivered via standard signals. On synchrously + * reported exceptions, -EFAULT is returned and details about the exception are + * recorded in @run.exception, the optional sgx_enclave_exception struct. + * + * Return: + * - 0: ENCLU function was successfully executed. + * - -EINVAL: Invalid ENCL number (neither EENTER nor ERESUME). + */ +typedef int (*vdso_sgx_enter_enclave_t)(unsigned long rdi, unsigned long rsi, + unsigned long rdx, unsigned int function, + unsigned long r8, unsigned long r9, + struct sgx_enclave_run *run); + +#endif /* _UAPI_ASM_X86_SGX_H */ diff --git a/arch/x86/kernel/acpi/apei.c b/arch/x86/kernel/acpi/apei.c index c22fb55abcfd..0916f00a992e 100644 --- a/arch/x86/kernel/acpi/apei.c +++ b/arch/x86/kernel/acpi/apei.c @@ -43,3 +43,8 @@ void arch_apei_report_mem_error(int sev, struct cper_sec_mem_err *mem_err) apei_mce_report_mem_error(sev, mem_err); #endif } + +int arch_apei_report_x86_error(struct cper_ia_proc_ctx *ctx_info, u64 lapic_id) +{ + return apei_smca_report_x86_error(ctx_info, lapic_id); +} diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 2400ad62f330..8d778e46725d 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -1374,7 +1374,7 @@ void __ref text_poke_queue(void *addr, const void *opcode, size_t len, const voi * @addr: address to patch * @opcode: opcode of new instruction * @len: length to copy - * @handler: address to jump to when the temporary breakpoint is hit + * @emulate: instruction to be emulated * * Update a single instruction with the vector in the stack, avoiding * dynamically allocated memory. This function should be used when it is diff --git a/arch/x86/kernel/amd_nb.c b/arch/x86/kernel/amd_nb.c index 18f6b7c4bd79..b4396952c9a6 100644 --- a/arch/x86/kernel/amd_nb.c +++ b/arch/x86/kernel/amd_nb.c @@ -384,7 +384,7 @@ struct resource *amd_get_mmconfig_range(struct resource *res) int amd_get_subcaches(int cpu) { - struct pci_dev *link = node_to_amd_nb(amd_get_nb_id(cpu))->link; + struct pci_dev *link = node_to_amd_nb(topology_die_id(cpu))->link; unsigned int mask; if (!amd_nb_has_feature(AMD_NB_L3_PARTITIONING)) @@ -398,7 +398,7 @@ int amd_get_subcaches(int cpu) int amd_set_subcaches(int cpu, unsigned long mask) { static unsigned int reset, ban; - struct amd_northbridge *nb = node_to_amd_nb(amd_get_nb_id(cpu)); + struct amd_northbridge *nb = node_to_amd_nb(topology_die_id(cpu)); unsigned int reg; int cuid; diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c index 235f5cde06fc..4bde125a5bf1 100644 --- a/arch/x86/kernel/apic/x2apic_uv_x.c +++ b/arch/x86/kernel/apic/x2apic_uv_x.c @@ -502,6 +502,18 @@ enum uv_system_type get_uv_system_type(void) return uv_system_type; } +int uv_get_hubless_system(void) +{ + return uv_hubless_system; +} +EXPORT_SYMBOL_GPL(uv_get_hubless_system); + +ssize_t uv_get_archtype(char *buf, int len) +{ + return scnprintf(buf, len, "%s/%s", uv_archtype, oem_table_id); +} +EXPORT_SYMBOL_GPL(uv_get_archtype); + int is_uv_system(void) { return uv_system_type != UV_NONE; @@ -1603,21 +1615,30 @@ static void check_efi_reboot(void) reboot_type = BOOT_ACPI; } -/* Setup user proc fs files */ +/* + * User proc fs file handling now deprecated. + * Recommend using /sys/firmware/sgi_uv/... instead. + */ static int __maybe_unused proc_hubbed_show(struct seq_file *file, void *data) { + pr_notice_once("%s: using deprecated /proc/sgi_uv/hubbed, use /sys/firmware/sgi_uv/hub_type\n", + current->comm); seq_printf(file, "0x%x\n", uv_hubbed_system); return 0; } static int __maybe_unused proc_hubless_show(struct seq_file *file, void *data) { + pr_notice_once("%s: using deprecated /proc/sgi_uv/hubless, use /sys/firmware/sgi_uv/hubless\n", + current->comm); seq_printf(file, "0x%x\n", uv_hubless_system); return 0; } static int __maybe_unused proc_archtype_show(struct seq_file *file, void *data) { + pr_notice_once("%s: using deprecated /proc/sgi_uv/archtype, use /sys/firmware/sgi_uv/archtype\n", + current->comm); seq_printf(file, "%s/%s\n", uv_archtype, oem_table_id); return 0; } diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c index 70b7154f4bdd..60b9f42ce3c1 100644 --- a/arch/x86/kernel/asm-offsets.c +++ b/arch/x86/kernel/asm-offsets.c @@ -66,7 +66,6 @@ static void __used common(void) OFFSET(PV_IRQ_irq_disable, paravirt_patch_template, irq.irq_disable); OFFSET(PV_IRQ_irq_enable, paravirt_patch_template, irq.irq_enable); OFFSET(PV_CPU_iret, paravirt_patch_template, cpu.iret); - OFFSET(PV_MMU_read_cr2, paravirt_patch_template, mmu.read_cr2); #endif #ifdef CONFIG_XEN diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile index 93792b457b81..637b499450d1 100644 --- a/arch/x86/kernel/cpu/Makefile +++ b/arch/x86/kernel/cpu/Makefile @@ -48,6 +48,7 @@ obj-$(CONFIG_X86_MCE) += mce/ obj-$(CONFIG_MTRR) += mtrr/ obj-$(CONFIG_MICROCODE) += microcode/ obj-$(CONFIG_X86_CPU_RESCTRL) += resctrl/ +obj-$(CONFIG_X86_SGX) += sgx/ obj-$(CONFIG_X86_LOCAL_APIC) += perfctr-watchdog.o diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 6062ce586b95..f8ca66f3d861 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -23,7 +23,6 @@ #ifdef CONFIG_X86_64 # include -# include #endif #include "cpu.h" @@ -330,7 +329,6 @@ static void legacy_fixup_core_id(struct cpuinfo_x86 *c) */ static void amd_get_topology(struct cpuinfo_x86 *c) { - u8 node_id; int cpu = smp_processor_id(); /* get information required for multi-node processors */ @@ -340,7 +338,7 @@ static void amd_get_topology(struct cpuinfo_x86 *c) cpuid(0x8000001e, &eax, &ebx, &ecx, &edx); - node_id = ecx & 0xff; + c->cpu_die_id = ecx & 0xff; if (c->x86 == 0x15) c->cu_id = ebx & 0xff; @@ -360,15 +358,15 @@ static void amd_get_topology(struct cpuinfo_x86 *c) if (!err) c->x86_coreid_bits = get_count_order(c->x86_max_cores); - cacheinfo_amd_init_llc_id(c, cpu, node_id); + cacheinfo_amd_init_llc_id(c, cpu); } else if (cpu_has(c, X86_FEATURE_NODEID_MSR)) { u64 value; rdmsrl(MSR_FAM10H_NODE_ID, value); - node_id = value & 7; + c->cpu_die_id = value & 7; - per_cpu(cpu_llc_id, cpu) = node_id; + per_cpu(cpu_llc_id, cpu) = c->cpu_die_id; } else return; @@ -393,7 +391,7 @@ static void amd_detect_cmp(struct cpuinfo_x86 *c) /* Convert the initial APIC ID into the socket ID */ c->phys_proc_id = c->initial_apicid >> bits; /* use socket ID also for last level cache */ - per_cpu(cpu_llc_id, cpu) = c->phys_proc_id; + per_cpu(cpu_llc_id, cpu) = c->cpu_die_id = c->phys_proc_id; } static void amd_detect_ppin(struct cpuinfo_x86 *c) @@ -425,12 +423,6 @@ clear_ppin: clear_cpu_cap(c, X86_FEATURE_AMD_PPIN); } -u16 amd_get_nb_id(int cpu) -{ - return per_cpu(cpu_llc_id, cpu); -} -EXPORT_SYMBOL_GPL(amd_get_nb_id); - u32 amd_get_nodes_per_socket(void) { return nodes_per_socket; @@ -516,26 +508,6 @@ static void early_init_amd_mc(struct cpuinfo_x86 *c) static void bsp_init_amd(struct cpuinfo_x86 *c) { - -#ifdef CONFIG_X86_64 - if (c->x86 >= 0xf) { - unsigned long long tseg; - - /* - * Split up direct mapping around the TSEG SMM area. - * Don't do it for gbpages because there seems very little - * benefit in doing so. - */ - if (!rdmsrl_safe(MSR_K8_TSEG_ADDR, &tseg)) { - unsigned long pfn = tseg >> PAGE_SHIFT; - - pr_debug("tseg: %010llx\n", tseg); - if (pfn_range_is_mapped(pfn, pfn + 1)) - set_memory_4k((unsigned long)__va(tseg), 1); - } - } -#endif - if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) { if (c->x86 > 0x10 || diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index 57074cf3ad7c..3ca9be482a9e 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -580,7 +580,7 @@ static void amd_init_l3_cache(struct _cpuid4_info_regs *this_leaf, int index) if (index < 3) return; - node = amd_get_nb_id(smp_processor_id()); + node = topology_die_id(smp_processor_id()); this_leaf->nb = node_to_amd_nb(node); if (this_leaf->nb && !this_leaf->nb->l3_cache.indices) amd_calc_l3_indices(this_leaf->nb); @@ -646,7 +646,7 @@ static int find_num_cache_leaves(struct cpuinfo_x86 *c) return i; } -void cacheinfo_amd_init_llc_id(struct cpuinfo_x86 *c, int cpu, u8 node_id) +void cacheinfo_amd_init_llc_id(struct cpuinfo_x86 *c, int cpu) { /* * We may have multiple LLCs if L3 caches exist, so check if we @@ -657,7 +657,7 @@ void cacheinfo_amd_init_llc_id(struct cpuinfo_x86 *c, int cpu, u8 node_id) if (c->x86 < 0x17) { /* LLC is at the node level. */ - per_cpu(cpu_llc_id, cpu) = node_id; + per_cpu(cpu_llc_id, cpu) = c->cpu_die_id; } else if (c->x86 == 0x17 && c->x86_model <= 0x1F) { /* * LLC is at the core complex level. @@ -684,7 +684,7 @@ void cacheinfo_amd_init_llc_id(struct cpuinfo_x86 *c, int cpu, u8 node_id) } } -void cacheinfo_hygon_init_llc_id(struct cpuinfo_x86 *c, int cpu, u8 node_id) +void cacheinfo_hygon_init_llc_id(struct cpuinfo_x86 *c, int cpu) { /* * We may have multiple LLCs if L3 caches exist, so check if we diff --git a/arch/x86/kernel/cpu/feat_ctl.c b/arch/x86/kernel/cpu/feat_ctl.c index 29a3bedabd06..3b1b01f2b248 100644 --- a/arch/x86/kernel/cpu/feat_ctl.c +++ b/arch/x86/kernel/cpu/feat_ctl.c @@ -93,16 +93,41 @@ static void init_vmx_capabilities(struct cpuinfo_x86 *c) } #endif /* CONFIG_X86_VMX_FEATURE_NAMES */ +static void clear_sgx_caps(void) +{ + setup_clear_cpu_cap(X86_FEATURE_SGX); + setup_clear_cpu_cap(X86_FEATURE_SGX_LC); +} + +static int __init nosgx(char *str) +{ + clear_sgx_caps(); + + return 0; +} + +early_param("nosgx", nosgx); + void init_ia32_feat_ctl(struct cpuinfo_x86 *c) { bool tboot = tboot_enabled(); + bool enable_sgx; u64 msr; if (rdmsrl_safe(MSR_IA32_FEAT_CTL, &msr)) { clear_cpu_cap(c, X86_FEATURE_VMX); + clear_sgx_caps(); return; } + /* + * Enable SGX if and only if the kernel supports SGX and Launch Control + * is supported, i.e. disable SGX if the LE hash MSRs can't be written. + */ + enable_sgx = cpu_has(c, X86_FEATURE_SGX) && + cpu_has(c, X86_FEATURE_SGX_LC) && + IS_ENABLED(CONFIG_X86_SGX); + if (msr & FEAT_CTL_LOCKED) goto update_caps; @@ -124,13 +149,16 @@ void init_ia32_feat_ctl(struct cpuinfo_x86 *c) msr |= FEAT_CTL_VMX_ENABLED_INSIDE_SMX; } + if (enable_sgx) + msr |= FEAT_CTL_SGX_ENABLED | FEAT_CTL_SGX_LC_ENABLED; + wrmsrl(MSR_IA32_FEAT_CTL, msr); update_caps: set_cpu_cap(c, X86_FEATURE_MSR_IA32_FEAT_CTL); if (!cpu_has(c, X86_FEATURE_VMX)) - return; + goto update_sgx; if ( (tboot && !(msr & FEAT_CTL_VMX_ENABLED_INSIDE_SMX)) || (!tboot && !(msr & FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX))) { @@ -143,4 +171,12 @@ update_caps: init_vmx_capabilities(c); #endif } + +update_sgx: + if (!(msr & FEAT_CTL_SGX_ENABLED) || + !(msr & FEAT_CTL_SGX_LC_ENABLED) || !enable_sgx) { + if (enable_sgx) + pr_err_once("SGX disabled by BIOS\n"); + clear_sgx_caps(); + } } diff --git a/arch/x86/kernel/cpu/hygon.c b/arch/x86/kernel/cpu/hygon.c index ac6c30e5801d..ae59115d18f9 100644 --- a/arch/x86/kernel/cpu/hygon.c +++ b/arch/x86/kernel/cpu/hygon.c @@ -14,9 +14,6 @@ #include #include #include -#ifdef CONFIG_X86_64 -# include -#endif #include "cpu.h" @@ -65,7 +62,6 @@ static void hygon_get_topology_early(struct cpuinfo_x86 *c) */ static void hygon_get_topology(struct cpuinfo_x86 *c) { - u8 node_id; int cpu = smp_processor_id(); /* get information required for multi-node processors */ @@ -75,7 +71,7 @@ static void hygon_get_topology(struct cpuinfo_x86 *c) cpuid(0x8000001e, &eax, &ebx, &ecx, &edx); - node_id = ecx & 0xff; + c->cpu_die_id = ecx & 0xff; c->cpu_core_id = ebx & 0xff; @@ -93,14 +89,14 @@ static void hygon_get_topology(struct cpuinfo_x86 *c) /* Socket ID is ApicId[6] for these processors. */ c->phys_proc_id = c->apicid >> APICID_SOCKET_ID_BIT; - cacheinfo_hygon_init_llc_id(c, cpu, node_id); + cacheinfo_hygon_init_llc_id(c, cpu); } else if (cpu_has(c, X86_FEATURE_NODEID_MSR)) { u64 value; rdmsrl(MSR_FAM10H_NODE_ID, value); - node_id = value & 7; + c->cpu_die_id = value & 7; - per_cpu(cpu_llc_id, cpu) = node_id; + per_cpu(cpu_llc_id, cpu) = c->cpu_die_id; } else return; @@ -123,7 +119,7 @@ static void hygon_detect_cmp(struct cpuinfo_x86 *c) /* Convert the initial APIC ID into the socket ID */ c->phys_proc_id = c->initial_apicid >> bits; /* use socket ID also for last level cache */ - per_cpu(cpu_llc_id, cpu) = c->phys_proc_id; + per_cpu(cpu_llc_id, cpu) = c->cpu_die_id = c->phys_proc_id; } static void srat_detect_node(struct cpuinfo_x86 *c) @@ -204,23 +200,6 @@ static void early_init_hygon_mc(struct cpuinfo_x86 *c) static void bsp_init_hygon(struct cpuinfo_x86 *c) { -#ifdef CONFIG_X86_64 - unsigned long long tseg; - - /* - * Split up direct mapping around the TSEG SMM area. - * Don't do it for gbpages because there seems very little - * benefit in doing so. - */ - if (!rdmsrl_safe(MSR_K8_TSEG_ADDR, &tseg)) { - unsigned long pfn = tseg >> PAGE_SHIFT; - - pr_debug("tseg: %010llx\n", tseg); - if (pfn_range_is_mapped(pfn, pfn + 1)) - set_memory_4k((unsigned long)__va(tseg), 1); - } -#endif - if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) { u64 val; diff --git a/arch/x86/kernel/cpu/mce/amd.c b/arch/x86/kernel/cpu/mce/amd.c index 0c6b02dd744c..e486f96b3cb3 100644 --- a/arch/x86/kernel/cpu/mce/amd.c +++ b/arch/x86/kernel/cpu/mce/amd.c @@ -1341,7 +1341,7 @@ static int threshold_create_bank(struct threshold_bank **bp, unsigned int cpu, return -ENODEV; if (is_shared_bank(bank)) { - nb = node_to_amd_nb(amd_get_nb_id(cpu)); + nb = node_to_amd_nb(topology_die_id(cpu)); /* threshold descriptor already initialized on this node? */ if (nb && nb->bank4) { @@ -1445,7 +1445,7 @@ static void threshold_remove_bank(struct threshold_bank *bank) * The last CPU on this node using the shared bank is going * away, remove that bank now. */ - nb = node_to_amd_nb(amd_get_nb_id(smp_processor_id())); + nb = node_to_amd_nb(topology_die_id(smp_processor_id())); nb->bank4 = NULL; } diff --git a/arch/x86/kernel/cpu/mce/apei.c b/arch/x86/kernel/cpu/mce/apei.c index af8d37962586..b58b85380ddb 100644 --- a/arch/x86/kernel/cpu/mce/apei.c +++ b/arch/x86/kernel/cpu/mce/apei.c @@ -51,6 +51,67 @@ void apei_mce_report_mem_error(int severity, struct cper_sec_mem_err *mem_err) } EXPORT_SYMBOL_GPL(apei_mce_report_mem_error); +int apei_smca_report_x86_error(struct cper_ia_proc_ctx *ctx_info, u64 lapic_id) +{ + const u64 *i_mce = ((const u64 *) (ctx_info + 1)); + unsigned int cpu; + struct mce m; + + if (!boot_cpu_has(X86_FEATURE_SMCA)) + return -EINVAL; + + /* + * The starting address of the register array extracted from BERT must + * match with the first expected register in the register layout of + * SMCA address space. This address corresponds to banks's MCA_STATUS + * register. + * + * Match any MCi_STATUS register by turning off bank numbers. + */ + if ((ctx_info->msr_addr & MSR_AMD64_SMCA_MC0_STATUS) != + MSR_AMD64_SMCA_MC0_STATUS) + return -EINVAL; + + /* + * The register array size must be large enough to include all the + * SMCA registers which need to be extracted. + * + * The number of registers in the register array is determined by + * Register Array Size/8 as defined in UEFI spec v2.8, sec N.2.4.2.2. + * The register layout is fixed and currently the raw data in the + * register array includes 6 SMCA registers which the kernel can + * extract. + */ + if (ctx_info->reg_arr_size < 48) + return -EINVAL; + + mce_setup(&m); + + m.extcpu = -1; + m.socketid = -1; + + for_each_possible_cpu(cpu) { + if (cpu_data(cpu).initial_apicid == lapic_id) { + m.extcpu = cpu; + m.socketid = cpu_data(m.extcpu).phys_proc_id; + break; + } + } + + m.apicid = lapic_id; + m.bank = (ctx_info->msr_addr >> 4) & 0xFF; + m.status = *i_mce; + m.addr = *(i_mce + 1); + m.misc = *(i_mce + 2); + /* Skipping MCA_CONFIG */ + m.ipid = *(i_mce + 4); + m.synd = *(i_mce + 5); + + mce_log(&m); + + return 0; +} + #define CPER_CREATOR_MCE \ GUID_INIT(0x75a574e3, 0x5052, 0x4b29, 0x8a, 0x8e, 0xbe, 0x2c, \ 0x64, 0x90, 0xb8, 0x9d) diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index 32b7099e3511..6af6a3c0698f 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -162,7 +162,8 @@ EXPORT_SYMBOL_GPL(mce_log); void mce_register_decode_chain(struct notifier_block *nb) { - if (WARN_ON(nb->priority > MCE_PRIO_MCELOG && nb->priority < MCE_PRIO_EDAC)) + if (WARN_ON(nb->priority < MCE_PRIO_LOWEST || + nb->priority > MCE_PRIO_HIGHEST)) return; blocking_notifier_chain_register(&x86_mce_decoder_chain, nb); @@ -1265,14 +1266,14 @@ static void kill_me_maybe(struct callback_head *cb) } } -static void queue_task_work(struct mce *m, int kill_it) +static void queue_task_work(struct mce *m, int kill_current_task) { current->mce_addr = m->addr; current->mce_kflags = m->kflags; current->mce_ripv = !!(m->mcgstatus & MCG_STATUS_RIPV); current->mce_whole_page = whole_page(m); - if (kill_it) + if (kill_current_task) current->mce_kill_me.func = kill_me_now; else current->mce_kill_me.func = kill_me_maybe; @@ -1320,10 +1321,10 @@ noinstr void do_machine_check(struct pt_regs *regs) int no_way_out = 0; /* - * If kill_it gets set, there might be a way to recover from this + * If kill_current_task is not set, there might be a way to recover from this * error. */ - int kill_it = 0; + int kill_current_task = 0; /* * MCEs are always local on AMD. Same is determined by MCG_STATUS_LMCES @@ -1350,8 +1351,7 @@ noinstr void do_machine_check(struct pt_regs *regs) * severity is MCE_AR_SEVERITY we have other options. */ if (!(m.mcgstatus & MCG_STATUS_RIPV)) - kill_it = 1; - + kill_current_task = (cfg->tolerant == 3) ? 0 : 1; /* * Check if this MCE is signaled to only this logical processor, * on Intel, Zhaoxin only. @@ -1368,7 +1368,7 @@ noinstr void do_machine_check(struct pt_regs *regs) * to see it will clear it. */ if (lmce) { - if (no_way_out) + if (no_way_out && cfg->tolerant < 3) mce_panic("Fatal local machine check", &m, msg); } else { order = mce_start(&no_way_out); @@ -1387,6 +1387,9 @@ noinstr void do_machine_check(struct pt_regs *regs) if (mce_end(order) < 0) { if (!no_way_out) no_way_out = worst >= MCE_PANIC_SEVERITY; + + if (no_way_out && cfg->tolerant < 3) + mce_panic("Fatal machine check on current CPU", &m, msg); } } else { /* @@ -1403,19 +1406,7 @@ noinstr void do_machine_check(struct pt_regs *regs) } } - /* - * If tolerant is at an insane level we drop requests to kill - * processes and continue even when there is no way out. - */ - if (cfg->tolerant == 3) - kill_it = 0; - else if (no_way_out) - mce_panic("Fatal machine check on current CPU", &m, msg); - - if (worst > 0) - irq_work_queue(&mce_irq_work); - - if (worst != MCE_AR_SEVERITY && !kill_it) + if (worst != MCE_AR_SEVERITY && !kill_current_task) goto out; /* Fault was in user mode and we need to take some action */ @@ -1423,7 +1414,7 @@ noinstr void do_machine_check(struct pt_regs *regs) /* If this triggers there is no way to recover. Die hard. */ BUG_ON(!on_thread_stack() || !user_mode(regs)); - queue_task_work(&m, kill_it); + queue_task_work(&m, kill_current_task); } else { /* @@ -1441,7 +1432,7 @@ noinstr void do_machine_check(struct pt_regs *regs) } if (m.kflags & MCE_IN_KERNEL_COPYIN) - queue_task_work(&m, kill_it); + queue_task_work(&m, kill_current_task); } out: mce_wrmsrl(MSR_IA32_MCG_STATUS, 0); @@ -1583,7 +1574,7 @@ static void __mcheck_cpu_mce_banks_init(void) * __mcheck_cpu_init_clear_banks() does the final bank setup. */ b->ctl = -1ULL; - b->init = 1; + b->init = true; } } @@ -1764,7 +1755,7 @@ static int __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c) */ if (c->x86 == 6 && c->x86_model < 0x1A && this_cpu_read(mce_num_banks) > 0) - mce_banks[0].init = 0; + mce_banks[0].init = false; /* * All newer Intel systems support MCE broadcasting. Enable @@ -1813,11 +1804,9 @@ static int __mcheck_cpu_ancient_init(struct cpuinfo_x86 *c) case X86_VENDOR_INTEL: intel_p5_mcheck_init(c); return 1; - break; case X86_VENDOR_CENTAUR: winchip_mcheck_init(c); return 1; - break; default: return 0; } diff --git a/arch/x86/kernel/cpu/mce/inject.c b/arch/x86/kernel/cpu/mce/inject.c index 3a44346f2276..7b360731fc2d 100644 --- a/arch/x86/kernel/cpu/mce/inject.c +++ b/arch/x86/kernel/cpu/mce/inject.c @@ -522,8 +522,8 @@ static void do_inject(void) if (boot_cpu_has(X86_FEATURE_AMD_DCM) && b == 4 && boot_cpu_data.x86 < 0x17) { - toggle_nb_mca_mst_cpu(amd_get_nb_id(cpu)); - cpu = get_nbc_for_node(amd_get_nb_id(cpu)); + toggle_nb_mca_mst_cpu(topology_die_id(cpu)); + cpu = get_nbc_for_node(topology_die_id(cpu)); } get_online_cpus(); diff --git a/arch/x86/kernel/cpu/mce/intel.c b/arch/x86/kernel/cpu/mce/intel.c index abe9fe0fb851..c2476fe0682e 100644 --- a/arch/x86/kernel/cpu/mce/intel.c +++ b/arch/x86/kernel/cpu/mce/intel.c @@ -509,12 +509,33 @@ static void intel_ppin_init(struct cpuinfo_x86 *c) } } +/* + * Enable additional error logs from the integrated + * memory controller on processors that support this. + */ +static void intel_imc_init(struct cpuinfo_x86 *c) +{ + u64 error_control; + + switch (c->x86_model) { + case INTEL_FAM6_SANDYBRIDGE_X: + case INTEL_FAM6_IVYBRIDGE_X: + case INTEL_FAM6_HASWELL_X: + if (rdmsrl_safe(MSR_ERROR_CONTROL, &error_control)) + return; + error_control |= 2; + wrmsrl_safe(MSR_ERROR_CONTROL, error_control); + break; + } +} + void mce_intel_feature_init(struct cpuinfo_x86 *c) { intel_init_thermal(c); intel_init_cmci(); intel_init_lmce(); intel_ppin_init(c); + intel_imc_init(c); } void mce_intel_feature_clear(struct cpuinfo_x86 *c) diff --git a/arch/x86/kernel/cpu/microcode/amd.c b/arch/x86/kernel/cpu/microcode/amd.c index 3f6b137ef4e6..3d4a48336084 100644 --- a/arch/x86/kernel/cpu/microcode/amd.c +++ b/arch/x86/kernel/cpu/microcode/amd.c @@ -215,7 +215,6 @@ static unsigned int __verify_patch_size(u8 family, u32 sh_psize, size_t buf_size default: WARN(1, "%s: WTF family: 0x%x\n", __func__, family); return 0; - break; } if (sh_psize > min_t(u32, buf_size, max_size)) diff --git a/arch/x86/kernel/cpu/mtrr/mtrr.c b/arch/x86/kernel/cpu/mtrr/mtrr.c index 6a80f36b5d59..08a30c8e9431 100644 --- a/arch/x86/kernel/cpu/mtrr/mtrr.c +++ b/arch/x86/kernel/cpu/mtrr/mtrr.c @@ -813,7 +813,8 @@ void mtrr_ap_init(void) } /** - * Save current fixed-range MTRR state of the first cpu in cpu_online_mask. + * mtrr_save_state - Save current fixed-range MTRR state of the first + * cpu in cpu_online_mask. */ void mtrr_save_state(void) { diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c index e8b5f1cf1ae8..698bb26aeb6e 100644 --- a/arch/x86/kernel/cpu/resctrl/core.c +++ b/arch/x86/kernel/cpu/resctrl/core.c @@ -895,6 +895,10 @@ static __init void __check_quirks_intel(void) set_rdt_options("!cmt,!mbmtotal,!mbmlocal,!l3cat"); else set_rdt_options("!l3cat"); + fallthrough; + case INTEL_FAM6_BROADWELL_X: + intel_rdt_mbm_apply_quirk(); + break; } } diff --git a/arch/x86/kernel/cpu/resctrl/internal.h b/arch/x86/kernel/cpu/resctrl/internal.h index f65d3c0dbc41..ee71c47844cb 100644 --- a/arch/x86/kernel/cpu/resctrl/internal.h +++ b/arch/x86/kernel/cpu/resctrl/internal.h @@ -264,7 +264,7 @@ void __exit rdtgroup_exit(void); struct rftype { char *name; umode_t mode; - struct kernfs_ops *kf_ops; + const struct kernfs_ops *kf_ops; unsigned long flags; unsigned long fflags; @@ -619,6 +619,7 @@ void mon_event_read(struct rmid_read *rr, struct rdt_resource *r, void mbm_setup_overflow_handler(struct rdt_domain *dom, unsigned long delay_ms); void mbm_handle_overflow(struct work_struct *work); +void __init intel_rdt_mbm_apply_quirk(void); bool is_mba_sc(struct rdt_resource *r); void setup_default_ctrlval(struct rdt_resource *r, u32 *dc, u32 *dm); u32 delay_bw_map(unsigned long bw, struct rdt_resource *r); diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c index a98519a3a2e6..7ac31210e452 100644 --- a/arch/x86/kernel/cpu/resctrl/monitor.c +++ b/arch/x86/kernel/cpu/resctrl/monitor.c @@ -64,6 +64,69 @@ unsigned int rdt_mon_features; */ unsigned int resctrl_cqm_threshold; +#define CF(cf) ((unsigned long)(1048576 * (cf) + 0.5)) + +/* + * The correction factor table is documented in Documentation/x86/resctrl.rst. + * If rmid > rmid threshold, MBM total and local values should be multiplied + * by the correction factor. + * + * The original table is modified for better code: + * + * 1. The threshold 0 is changed to rmid count - 1 so don't do correction + * for the case. + * 2. MBM total and local correction table indexed by core counter which is + * equal to (x86_cache_max_rmid + 1) / 8 - 1 and is from 0 up to 27. + * 3. The correction factor is normalized to 2^20 (1048576) so it's faster + * to calculate corrected value by shifting: + * corrected_value = (original_value * correction_factor) >> 20 + */ +static const struct mbm_correction_factor_table { + u32 rmidthreshold; + u64 cf; +} mbm_cf_table[] __initdata = { + {7, CF(1.000000)}, + {15, CF(1.000000)}, + {15, CF(0.969650)}, + {31, CF(1.000000)}, + {31, CF(1.066667)}, + {31, CF(0.969650)}, + {47, CF(1.142857)}, + {63, CF(1.000000)}, + {63, CF(1.185115)}, + {63, CF(1.066553)}, + {79, CF(1.454545)}, + {95, CF(1.000000)}, + {95, CF(1.230769)}, + {95, CF(1.142857)}, + {95, CF(1.066667)}, + {127, CF(1.000000)}, + {127, CF(1.254863)}, + {127, CF(1.185255)}, + {151, CF(1.000000)}, + {127, CF(1.066667)}, + {167, CF(1.000000)}, + {159, CF(1.454334)}, + {183, CF(1.000000)}, + {127, CF(0.969744)}, + {191, CF(1.280246)}, + {191, CF(1.230921)}, + {215, CF(1.000000)}, + {191, CF(1.143118)}, +}; + +static u32 mbm_cf_rmidthreshold __read_mostly = UINT_MAX; +static u64 mbm_cf __read_mostly; + +static inline u64 get_corrected_mbm_count(u32 rmid, unsigned long val) +{ + /* Correct MBM value. */ + if (rmid > mbm_cf_rmidthreshold) + val = (val * mbm_cf) >> 20; + + return val; +} + static inline struct rmid_entry *__rmid_entry(u32 rmid) { struct rmid_entry *entry; @@ -260,7 +323,8 @@ static int __mon_event_count(u32 rmid, struct rmid_read *rr) m->chunks += chunks; m->prev_msr = tval; - rr->val += m->chunks; + rr->val += get_corrected_mbm_count(rmid, m->chunks); + return 0; } @@ -279,7 +343,7 @@ static void mbm_bw_count(u32 rmid, struct rmid_read *rr) return; chunks = mbm_overflow_count(m->prev_bw_msr, tval, rr->r->mbm_width); - cur_bw = (chunks * r->mon_scale) >> 20; + cur_bw = (get_corrected_mbm_count(rmid, chunks) * r->mon_scale) >> 20; if (m->delta_comp) m->delta_bw = abs(cur_bw - m->prev_bw); @@ -642,3 +706,17 @@ int rdt_get_mon_l3_config(struct rdt_resource *r) return 0; } + +void __init intel_rdt_mbm_apply_quirk(void) +{ + int cf_index; + + cf_index = (boot_cpu_data.x86_cache_max_rmid + 1) / 8 - 1; + if (cf_index >= ARRAY_SIZE(mbm_cf_table)) { + pr_info("No MBM correction factor available\n"); + return; + } + + mbm_cf_rmidthreshold = mbm_cf_table[cf_index].rmidthreshold; + mbm_cf = mbm_cf_table[cf_index].cf; +} diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c index f3418428682b..29ffb95b25ff 100644 --- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c +++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c @@ -240,13 +240,13 @@ static ssize_t rdtgroup_file_write(struct kernfs_open_file *of, char *buf, return -EINVAL; } -static struct kernfs_ops rdtgroup_kf_single_ops = { +static const struct kernfs_ops rdtgroup_kf_single_ops = { .atomic_write_len = PAGE_SIZE, .write = rdtgroup_file_write, .seq_show = rdtgroup_seqfile_show, }; -static struct kernfs_ops kf_mondata_ops = { +static const struct kernfs_ops kf_mondata_ops = { .atomic_write_len = PAGE_SIZE, .seq_show = rdtgroup_mondata_show, }; @@ -3023,8 +3023,7 @@ static int rdtgroup_mkdir(struct kernfs_node *parent_kn, const char *name, return -EPERM; } -static int rdtgroup_rmdir_mon(struct kernfs_node *kn, struct rdtgroup *rdtgrp, - cpumask_var_t tmpmask) +static int rdtgroup_rmdir_mon(struct rdtgroup *rdtgrp, cpumask_var_t tmpmask) { struct rdtgroup *prdtgrp = rdtgrp->mon.parent; int cpu; @@ -3056,8 +3055,7 @@ static int rdtgroup_rmdir_mon(struct kernfs_node *kn, struct rdtgroup *rdtgrp, return 0; } -static int rdtgroup_ctrl_remove(struct kernfs_node *kn, - struct rdtgroup *rdtgrp) +static int rdtgroup_ctrl_remove(struct rdtgroup *rdtgrp) { rdtgrp->flags = RDT_DELETED; list_del(&rdtgrp->rdtgroup_list); @@ -3066,8 +3064,7 @@ static int rdtgroup_ctrl_remove(struct kernfs_node *kn, return 0; } -static int rdtgroup_rmdir_ctrl(struct kernfs_node *kn, struct rdtgroup *rdtgrp, - cpumask_var_t tmpmask) +static int rdtgroup_rmdir_ctrl(struct rdtgroup *rdtgrp, cpumask_var_t tmpmask) { int cpu; @@ -3094,7 +3091,7 @@ static int rdtgroup_rmdir_ctrl(struct kernfs_node *kn, struct rdtgroup *rdtgrp, closid_free(rdtgrp->closid); free_rmid(rdtgrp->mon.rmid); - rdtgroup_ctrl_remove(kn, rdtgrp); + rdtgroup_ctrl_remove(rdtgrp); /* * Free all the child monitor group rmids. @@ -3131,13 +3128,13 @@ static int rdtgroup_rmdir(struct kernfs_node *kn) rdtgrp != &rdtgroup_default) { if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP || rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) { - ret = rdtgroup_ctrl_remove(kn, rdtgrp); + ret = rdtgroup_ctrl_remove(rdtgrp); } else { - ret = rdtgroup_rmdir_ctrl(kn, rdtgrp, tmpmask); + ret = rdtgroup_rmdir_ctrl(rdtgrp, tmpmask); } } else if (rdtgrp->type == RDTMON_GROUP && is_mon_groups(parent_kn, kn->name)) { - ret = rdtgroup_rmdir_mon(kn, rdtgrp, tmpmask); + ret = rdtgroup_rmdir_mon(rdtgrp, tmpmask); } else { ret = -EPERM; } diff --git a/arch/x86/kernel/cpu/sgx/Makefile b/arch/x86/kernel/cpu/sgx/Makefile new file mode 100644 index 000000000000..91d3dc784a29 --- /dev/null +++ b/arch/x86/kernel/cpu/sgx/Makefile @@ -0,0 +1,5 @@ +obj-y += \ + driver.o \ + encl.o \ + ioctl.o \ + main.o diff --git a/arch/x86/kernel/cpu/sgx/arch.h b/arch/x86/kernel/cpu/sgx/arch.h new file mode 100644 index 000000000000..dd7602c44c72 --- /dev/null +++ b/arch/x86/kernel/cpu/sgx/arch.h @@ -0,0 +1,338 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/** + * Copyright(c) 2016-20 Intel Corporation. + * + * Contains data structures defined by the SGX architecture. Data structures + * defined by the Linux software stack should not be placed here. + */ +#ifndef _ASM_X86_SGX_ARCH_H +#define _ASM_X86_SGX_ARCH_H + +#include +#include + +/* The SGX specific CPUID function. */ +#define SGX_CPUID 0x12 +/* EPC enumeration. */ +#define SGX_CPUID_EPC 2 +/* An invalid EPC section, i.e. the end marker. */ +#define SGX_CPUID_EPC_INVALID 0x0 +/* A valid EPC section. */ +#define SGX_CPUID_EPC_SECTION 0x1 +/* The bitmask for the EPC section type. */ +#define SGX_CPUID_EPC_MASK GENMASK(3, 0) + +/** + * enum sgx_return_code - The return code type for ENCLS, ENCLU and ENCLV + * %SGX_NOT_TRACKED: Previous ETRACK's shootdown sequence has not + * been completed yet. + * %SGX_INVALID_EINITTOKEN: EINITTOKEN is invalid and enclave signer's + * public key does not match IA32_SGXLEPUBKEYHASH. + * %SGX_UNMASKED_EVENT: An unmasked event, e.g. INTR, was received + */ +enum sgx_return_code { + SGX_NOT_TRACKED = 11, + SGX_INVALID_EINITTOKEN = 16, + SGX_UNMASKED_EVENT = 128, +}; + +/* The modulus size for 3072-bit RSA keys. */ +#define SGX_MODULUS_SIZE 384 + +/** + * enum sgx_miscselect - additional information to an SSA frame + * %SGX_MISC_EXINFO: Report #PF or #GP to the SSA frame. + * + * Save State Area (SSA) is a stack inside the enclave used to store processor + * state when an exception or interrupt occurs. This enum defines additional + * information stored to an SSA frame. + */ +enum sgx_miscselect { + SGX_MISC_EXINFO = BIT(0), +}; + +#define SGX_MISC_RESERVED_MASK GENMASK_ULL(63, 1) + +#define SGX_SSA_GPRS_SIZE 184 +#define SGX_SSA_MISC_EXINFO_SIZE 16 + +/** + * enum sgx_attributes - the attributes field in &struct sgx_secs + * %SGX_ATTR_INIT: Enclave can be entered (is initialized). + * %SGX_ATTR_DEBUG: Allow ENCLS(EDBGRD) and ENCLS(EDBGWR). + * %SGX_ATTR_MODE64BIT: Tell that this a 64-bit enclave. + * %SGX_ATTR_PROVISIONKEY: Allow to use provisioning keys for remote + * attestation. + * %SGX_ATTR_KSS: Allow to use key separation and sharing (KSS). + * %SGX_ATTR_EINITTOKENKEY: Allow to use token signing key that is used to + * sign cryptographic tokens that can be passed to + * EINIT as an authorization to run an enclave. + */ +enum sgx_attribute { + SGX_ATTR_INIT = BIT(0), + SGX_ATTR_DEBUG = BIT(1), + SGX_ATTR_MODE64BIT = BIT(2), + SGX_ATTR_PROVISIONKEY = BIT(4), + SGX_ATTR_EINITTOKENKEY = BIT(5), + SGX_ATTR_KSS = BIT(7), +}; + +#define SGX_ATTR_RESERVED_MASK (BIT_ULL(3) | BIT_ULL(6) | GENMASK_ULL(63, 8)) + +/** + * struct sgx_secs - SGX Enclave Control Structure (SECS) + * @size: size of the address space + * @base: base address of the address space + * @ssa_frame_size: size of an SSA frame + * @miscselect: additional information stored to an SSA frame + * @attributes: attributes for enclave + * @xfrm: XSave-Feature Request Mask (subset of XCR0) + * @mrenclave: SHA256-hash of the enclave contents + * @mrsigner: SHA256-hash of the public key used to sign the SIGSTRUCT + * @config_id: a user-defined value that is used in key derivation + * @isv_prod_id: a user-defined value that is used in key derivation + * @isv_svn: a user-defined value that is used in key derivation + * @config_svn: a user-defined value that is used in key derivation + * + * SGX Enclave Control Structure (SECS) is a special enclave page that is not + * visible in the address space. In fact, this structure defines the address + * range and other global attributes for the enclave and it is the first EPC + * page created for any enclave. It is moved from a temporary buffer to an EPC + * by the means of ENCLS[ECREATE] function. + */ +struct sgx_secs { + u64 size; + u64 base; + u32 ssa_frame_size; + u32 miscselect; + u8 reserved1[24]; + u64 attributes; + u64 xfrm; + u32 mrenclave[8]; + u8 reserved2[32]; + u32 mrsigner[8]; + u8 reserved3[32]; + u32 config_id[16]; + u16 isv_prod_id; + u16 isv_svn; + u16 config_svn; + u8 reserved4[3834]; +} __packed; + +/** + * enum sgx_tcs_flags - execution flags for TCS + * %SGX_TCS_DBGOPTIN: If enabled allows single-stepping and breakpoints + * inside an enclave. It is cleared by EADD but can + * be set later with EDBGWR. + */ +enum sgx_tcs_flags { + SGX_TCS_DBGOPTIN = 0x01, +}; + +#define SGX_TCS_RESERVED_MASK GENMASK_ULL(63, 1) +#define SGX_TCS_RESERVED_SIZE 4024 + +/** + * struct sgx_tcs - Thread Control Structure (TCS) + * @state: used to mark an entered TCS + * @flags: execution flags (cleared by EADD) + * @ssa_offset: SSA stack offset relative to the enclave base + * @ssa_index: the current SSA frame index (cleard by EADD) + * @nr_ssa_frames: the number of frame in the SSA stack + * @entry_offset: entry point offset relative to the enclave base + * @exit_addr: address outside the enclave to exit on an exception or + * interrupt + * @fs_offset: offset relative to the enclave base to become FS + * segment inside the enclave + * @gs_offset: offset relative to the enclave base to become GS + * segment inside the enclave + * @fs_limit: size to become a new FS-limit (only 32-bit enclaves) + * @gs_limit: size to become a new GS-limit (only 32-bit enclaves) + * + * Thread Control Structure (TCS) is an enclave page visible in its address + * space that defines an entry point inside the enclave. A thread enters inside + * an enclave by supplying address of TCS to ENCLU(EENTER). A TCS can be entered + * by only one thread at a time. + */ +struct sgx_tcs { + u64 state; + u64 flags; + u64 ssa_offset; + u32 ssa_index; + u32 nr_ssa_frames; + u64 entry_offset; + u64 exit_addr; + u64 fs_offset; + u64 gs_offset; + u32 fs_limit; + u32 gs_limit; + u8 reserved[SGX_TCS_RESERVED_SIZE]; +} __packed; + +/** + * struct sgx_pageinfo - an enclave page descriptor + * @addr: address of the enclave page + * @contents: pointer to the page contents + * @metadata: pointer either to a SECINFO or PCMD instance + * @secs: address of the SECS page + */ +struct sgx_pageinfo { + u64 addr; + u64 contents; + u64 metadata; + u64 secs; +} __packed __aligned(32); + + +/** + * enum sgx_page_type - bits in the SECINFO flags defining the page type + * %SGX_PAGE_TYPE_SECS: a SECS page + * %SGX_PAGE_TYPE_TCS: a TCS page + * %SGX_PAGE_TYPE_REG: a regular page + * %SGX_PAGE_TYPE_VA: a VA page + * %SGX_PAGE_TYPE_TRIM: a page in trimmed state + */ +enum sgx_page_type { + SGX_PAGE_TYPE_SECS, + SGX_PAGE_TYPE_TCS, + SGX_PAGE_TYPE_REG, + SGX_PAGE_TYPE_VA, + SGX_PAGE_TYPE_TRIM, +}; + +#define SGX_NR_PAGE_TYPES 5 +#define SGX_PAGE_TYPE_MASK GENMASK(7, 0) + +/** + * enum sgx_secinfo_flags - the flags field in &struct sgx_secinfo + * %SGX_SECINFO_R: allow read + * %SGX_SECINFO_W: allow write + * %SGX_SECINFO_X: allow execution + * %SGX_SECINFO_SECS: a SECS page + * %SGX_SECINFO_TCS: a TCS page + * %SGX_SECINFO_REG: a regular page + * %SGX_SECINFO_VA: a VA page + * %SGX_SECINFO_TRIM: a page in trimmed state + */ +enum sgx_secinfo_flags { + SGX_SECINFO_R = BIT(0), + SGX_SECINFO_W = BIT(1), + SGX_SECINFO_X = BIT(2), + SGX_SECINFO_SECS = (SGX_PAGE_TYPE_SECS << 8), + SGX_SECINFO_TCS = (SGX_PAGE_TYPE_TCS << 8), + SGX_SECINFO_REG = (SGX_PAGE_TYPE_REG << 8), + SGX_SECINFO_VA = (SGX_PAGE_TYPE_VA << 8), + SGX_SECINFO_TRIM = (SGX_PAGE_TYPE_TRIM << 8), +}; + +#define SGX_SECINFO_PERMISSION_MASK GENMASK_ULL(2, 0) +#define SGX_SECINFO_PAGE_TYPE_MASK (SGX_PAGE_TYPE_MASK << 8) +#define SGX_SECINFO_RESERVED_MASK ~(SGX_SECINFO_PERMISSION_MASK | \ + SGX_SECINFO_PAGE_TYPE_MASK) + +/** + * struct sgx_secinfo - describes attributes of an EPC page + * @flags: permissions and type + * + * Used together with ENCLS leaves that add or modify an EPC page to an + * enclave to define page permissions and type. + */ +struct sgx_secinfo { + u64 flags; + u8 reserved[56]; +} __packed __aligned(64); + +#define SGX_PCMD_RESERVED_SIZE 40 + +/** + * struct sgx_pcmd - Paging Crypto Metadata (PCMD) + * @enclave_id: enclave identifier + * @mac: MAC over PCMD, page contents and isvsvn + * + * PCMD is stored for every swapped page to the regular memory. When ELDU loads + * the page back it recalculates the MAC by using a isvsvn number stored in a + * VA page. Together these two structures bring integrity and rollback + * protection. + */ +struct sgx_pcmd { + struct sgx_secinfo secinfo; + u64 enclave_id; + u8 reserved[SGX_PCMD_RESERVED_SIZE]; + u8 mac[16]; +} __packed __aligned(128); + +#define SGX_SIGSTRUCT_RESERVED1_SIZE 84 +#define SGX_SIGSTRUCT_RESERVED2_SIZE 20 +#define SGX_SIGSTRUCT_RESERVED3_SIZE 32 +#define SGX_SIGSTRUCT_RESERVED4_SIZE 12 + +/** + * struct sgx_sigstruct_header - defines author of the enclave + * @header1: constant byte string + * @vendor: must be either 0x0000 or 0x8086 + * @date: YYYYMMDD in BCD + * @header2: costant byte string + * @swdefined: software defined value + */ +struct sgx_sigstruct_header { + u64 header1[2]; + u32 vendor; + u32 date; + u64 header2[2]; + u32 swdefined; + u8 reserved1[84]; +} __packed; + +/** + * struct sgx_sigstruct_body - defines contents of the enclave + * @miscselect: additional information stored to an SSA frame + * @misc_mask: required miscselect in SECS + * @attributes: attributes for enclave + * @xfrm: XSave-Feature Request Mask (subset of XCR0) + * @attributes_mask: required attributes in SECS + * @xfrm_mask: required XFRM in SECS + * @mrenclave: SHA256-hash of the enclave contents + * @isvprodid: a user-defined value that is used in key derivation + * @isvsvn: a user-defined value that is used in key derivation + */ +struct sgx_sigstruct_body { + u32 miscselect; + u32 misc_mask; + u8 reserved2[20]; + u64 attributes; + u64 xfrm; + u64 attributes_mask; + u64 xfrm_mask; + u8 mrenclave[32]; + u8 reserved3[32]; + u16 isvprodid; + u16 isvsvn; +} __packed; + +/** + * struct sgx_sigstruct - an enclave signature + * @header: defines author of the enclave + * @modulus: the modulus of the public key + * @exponent: the exponent of the public key + * @signature: the signature calculated over the fields except modulus, + * @body: defines contents of the enclave + * @q1: a value used in RSA signature verification + * @q2: a value used in RSA signature verification + * + * Header and body are the parts that are actual signed. The remaining fields + * define the signature of the enclave. + */ +struct sgx_sigstruct { + struct sgx_sigstruct_header header; + u8 modulus[SGX_MODULUS_SIZE]; + u32 exponent; + u8 signature[SGX_MODULUS_SIZE]; + struct sgx_sigstruct_body body; + u8 reserved4[12]; + u8 q1[SGX_MODULUS_SIZE]; + u8 q2[SGX_MODULUS_SIZE]; +} __packed; + +#define SGX_LAUNCH_TOKEN_SIZE 304 + +#endif /* _ASM_X86_SGX_ARCH_H */ diff --git a/arch/x86/kernel/cpu/sgx/driver.c b/arch/x86/kernel/cpu/sgx/driver.c new file mode 100644 index 000000000000..f2eac41bb4ff --- /dev/null +++ b/arch/x86/kernel/cpu/sgx/driver.c @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2016-20 Intel Corporation. */ + +#include +#include +#include +#include +#include +#include +#include "driver.h" +#include "encl.h" + +u64 sgx_attributes_reserved_mask; +u64 sgx_xfrm_reserved_mask = ~0x3; +u32 sgx_misc_reserved_mask; + +static int sgx_open(struct inode *inode, struct file *file) +{ + struct sgx_encl *encl; + int ret; + + encl = kzalloc(sizeof(*encl), GFP_KERNEL); + if (!encl) + return -ENOMEM; + + kref_init(&encl->refcount); + xa_init(&encl->page_array); + mutex_init(&encl->lock); + INIT_LIST_HEAD(&encl->va_pages); + INIT_LIST_HEAD(&encl->mm_list); + spin_lock_init(&encl->mm_lock); + + ret = init_srcu_struct(&encl->srcu); + if (ret) { + kfree(encl); + return ret; + } + + file->private_data = encl; + + return 0; +} + +static int sgx_release(struct inode *inode, struct file *file) +{ + struct sgx_encl *encl = file->private_data; + struct sgx_encl_mm *encl_mm; + + /* + * Drain the remaining mm_list entries. At this point the list contains + * entries for processes, which have closed the enclave file but have + * not exited yet. The processes, which have exited, are gone from the + * list by sgx_mmu_notifier_release(). + */ + for ( ; ; ) { + spin_lock(&encl->mm_lock); + + if (list_empty(&encl->mm_list)) { + encl_mm = NULL; + } else { + encl_mm = list_first_entry(&encl->mm_list, + struct sgx_encl_mm, list); + list_del_rcu(&encl_mm->list); + } + + spin_unlock(&encl->mm_lock); + + /* The enclave is no longer mapped by any mm. */ + if (!encl_mm) + break; + + synchronize_srcu(&encl->srcu); + mmu_notifier_unregister(&encl_mm->mmu_notifier, encl_mm->mm); + kfree(encl_mm); + } + + kref_put(&encl->refcount, sgx_encl_release); + return 0; +} + +static int sgx_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct sgx_encl *encl = file->private_data; + int ret; + + ret = sgx_encl_may_map(encl, vma->vm_start, vma->vm_end, vma->vm_flags); + if (ret) + return ret; + + ret = sgx_encl_mm_add(encl, vma->vm_mm); + if (ret) + return ret; + + vma->vm_ops = &sgx_vm_ops; + vma->vm_flags |= VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO; + vma->vm_private_data = encl; + + return 0; +} + +static unsigned long sgx_get_unmapped_area(struct file *file, + unsigned long addr, + unsigned long len, + unsigned long pgoff, + unsigned long flags) +{ + if ((flags & MAP_TYPE) == MAP_PRIVATE) + return -EINVAL; + + if (flags & MAP_FIXED) + return addr; + + return current->mm->get_unmapped_area(file, addr, len, pgoff, flags); +} + +#ifdef CONFIG_COMPAT +static long sgx_compat_ioctl(struct file *filep, unsigned int cmd, + unsigned long arg) +{ + return sgx_ioctl(filep, cmd, arg); +} +#endif + +static const struct file_operations sgx_encl_fops = { + .owner = THIS_MODULE, + .open = sgx_open, + .release = sgx_release, + .unlocked_ioctl = sgx_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = sgx_compat_ioctl, +#endif + .mmap = sgx_mmap, + .get_unmapped_area = sgx_get_unmapped_area, +}; + +const struct file_operations sgx_provision_fops = { + .owner = THIS_MODULE, +}; + +static struct miscdevice sgx_dev_enclave = { + .minor = MISC_DYNAMIC_MINOR, + .name = "sgx_enclave", + .nodename = "sgx_enclave", + .fops = &sgx_encl_fops, +}; + +static struct miscdevice sgx_dev_provision = { + .minor = MISC_DYNAMIC_MINOR, + .name = "sgx_provision", + .nodename = "sgx_provision", + .fops = &sgx_provision_fops, +}; + +int __init sgx_drv_init(void) +{ + unsigned int eax, ebx, ecx, edx; + u64 attr_mask; + u64 xfrm_mask; + int ret; + + if (!cpu_feature_enabled(X86_FEATURE_SGX_LC)) + return -ENODEV; + + cpuid_count(SGX_CPUID, 0, &eax, &ebx, &ecx, &edx); + + if (!(eax & 1)) { + pr_err("SGX disabled: SGX1 instruction support not available.\n"); + return -ENODEV; + } + + sgx_misc_reserved_mask = ~ebx | SGX_MISC_RESERVED_MASK; + + cpuid_count(SGX_CPUID, 1, &eax, &ebx, &ecx, &edx); + + attr_mask = (((u64)ebx) << 32) + (u64)eax; + sgx_attributes_reserved_mask = ~attr_mask | SGX_ATTR_RESERVED_MASK; + + if (cpu_feature_enabled(X86_FEATURE_OSXSAVE)) { + xfrm_mask = (((u64)edx) << 32) + (u64)ecx; + sgx_xfrm_reserved_mask = ~xfrm_mask; + } + + ret = misc_register(&sgx_dev_enclave); + if (ret) + return ret; + + ret = misc_register(&sgx_dev_provision); + if (ret) { + misc_deregister(&sgx_dev_enclave); + return ret; + } + + return 0; +} diff --git a/arch/x86/kernel/cpu/sgx/driver.h b/arch/x86/kernel/cpu/sgx/driver.h new file mode 100644 index 000000000000..4eddb4d571ef --- /dev/null +++ b/arch/x86/kernel/cpu/sgx/driver.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ARCH_SGX_DRIVER_H__ +#define __ARCH_SGX_DRIVER_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "sgx.h" + +#define SGX_EINIT_SPIN_COUNT 20 +#define SGX_EINIT_SLEEP_COUNT 50 +#define SGX_EINIT_SLEEP_TIME 20 + +extern u64 sgx_attributes_reserved_mask; +extern u64 sgx_xfrm_reserved_mask; +extern u32 sgx_misc_reserved_mask; + +extern const struct file_operations sgx_provision_fops; + +long sgx_ioctl(struct file *filep, unsigned int cmd, unsigned long arg); + +int sgx_drv_init(void); + +#endif /* __ARCH_X86_SGX_DRIVER_H__ */ diff --git a/arch/x86/kernel/cpu/sgx/encl.c b/arch/x86/kernel/cpu/sgx/encl.c new file mode 100644 index 000000000000..ee50a5010277 --- /dev/null +++ b/arch/x86/kernel/cpu/sgx/encl.c @@ -0,0 +1,740 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2016-20 Intel Corporation. */ + +#include +#include +#include +#include +#include +#include +#include "arch.h" +#include "encl.h" +#include "encls.h" +#include "sgx.h" + +/* + * ELDU: Load an EPC page as unblocked. For more info, see "OS Management of EPC + * Pages" in the SDM. + */ +static int __sgx_encl_eldu(struct sgx_encl_page *encl_page, + struct sgx_epc_page *epc_page, + struct sgx_epc_page *secs_page) +{ + unsigned long va_offset = encl_page->desc & SGX_ENCL_PAGE_VA_OFFSET_MASK; + struct sgx_encl *encl = encl_page->encl; + struct sgx_pageinfo pginfo; + struct sgx_backing b; + pgoff_t page_index; + int ret; + + if (secs_page) + page_index = PFN_DOWN(encl_page->desc - encl_page->encl->base); + else + page_index = PFN_DOWN(encl->size); + + ret = sgx_encl_get_backing(encl, page_index, &b); + if (ret) + return ret; + + pginfo.addr = encl_page->desc & PAGE_MASK; + pginfo.contents = (unsigned long)kmap_atomic(b.contents); + pginfo.metadata = (unsigned long)kmap_atomic(b.pcmd) + + b.pcmd_offset; + + if (secs_page) + pginfo.secs = (u64)sgx_get_epc_virt_addr(secs_page); + else + pginfo.secs = 0; + + ret = __eldu(&pginfo, sgx_get_epc_virt_addr(epc_page), + sgx_get_epc_virt_addr(encl_page->va_page->epc_page) + va_offset); + if (ret) { + if (encls_failed(ret)) + ENCLS_WARN(ret, "ELDU"); + + ret = -EFAULT; + } + + kunmap_atomic((void *)(unsigned long)(pginfo.metadata - b.pcmd_offset)); + kunmap_atomic((void *)(unsigned long)pginfo.contents); + + sgx_encl_put_backing(&b, false); + + return ret; +} + +static struct sgx_epc_page *sgx_encl_eldu(struct sgx_encl_page *encl_page, + struct sgx_epc_page *secs_page) +{ + + unsigned long va_offset = encl_page->desc & SGX_ENCL_PAGE_VA_OFFSET_MASK; + struct sgx_encl *encl = encl_page->encl; + struct sgx_epc_page *epc_page; + int ret; + + epc_page = sgx_alloc_epc_page(encl_page, false); + if (IS_ERR(epc_page)) + return epc_page; + + ret = __sgx_encl_eldu(encl_page, epc_page, secs_page); + if (ret) { + sgx_free_epc_page(epc_page); + return ERR_PTR(ret); + } + + sgx_free_va_slot(encl_page->va_page, va_offset); + list_move(&encl_page->va_page->list, &encl->va_pages); + encl_page->desc &= ~SGX_ENCL_PAGE_VA_OFFSET_MASK; + encl_page->epc_page = epc_page; + + return epc_page; +} + +static struct sgx_encl_page *sgx_encl_load_page(struct sgx_encl *encl, + unsigned long addr, + unsigned long vm_flags) +{ + unsigned long vm_prot_bits = vm_flags & (VM_READ | VM_WRITE | VM_EXEC); + struct sgx_epc_page *epc_page; + struct sgx_encl_page *entry; + + entry = xa_load(&encl->page_array, PFN_DOWN(addr)); + if (!entry) + return ERR_PTR(-EFAULT); + + /* + * Verify that the faulted page has equal or higher build time + * permissions than the VMA permissions (i.e. the subset of {VM_READ, + * VM_WRITE, VM_EXECUTE} in vma->vm_flags). + */ + if ((entry->vm_max_prot_bits & vm_prot_bits) != vm_prot_bits) + return ERR_PTR(-EFAULT); + + /* Entry successfully located. */ + if (entry->epc_page) { + if (entry->desc & SGX_ENCL_PAGE_BEING_RECLAIMED) + return ERR_PTR(-EBUSY); + + return entry; + } + + if (!(encl->secs.epc_page)) { + epc_page = sgx_encl_eldu(&encl->secs, NULL); + if (IS_ERR(epc_page)) + return ERR_CAST(epc_page); + } + + epc_page = sgx_encl_eldu(entry, encl->secs.epc_page); + if (IS_ERR(epc_page)) + return ERR_CAST(epc_page); + + encl->secs_child_cnt++; + sgx_mark_page_reclaimable(entry->epc_page); + + return entry; +} + +static vm_fault_t sgx_vma_fault(struct vm_fault *vmf) +{ + unsigned long addr = (unsigned long)vmf->address; + struct vm_area_struct *vma = vmf->vma; + struct sgx_encl_page *entry; + unsigned long phys_addr; + struct sgx_encl *encl; + unsigned long pfn; + vm_fault_t ret; + + encl = vma->vm_private_data; + + /* + * It's very unlikely but possible that allocating memory for the + * mm_list entry of a forked process failed in sgx_vma_open(). When + * this happens, vm_private_data is set to NULL. + */ + if (unlikely(!encl)) + return VM_FAULT_SIGBUS; + + mutex_lock(&encl->lock); + + entry = sgx_encl_load_page(encl, addr, vma->vm_flags); + if (IS_ERR(entry)) { + mutex_unlock(&encl->lock); + + if (PTR_ERR(entry) == -EBUSY) + return VM_FAULT_NOPAGE; + + return VM_FAULT_SIGBUS; + } + + phys_addr = sgx_get_epc_phys_addr(entry->epc_page); + + /* Check if another thread got here first to insert the PTE. */ + if (!follow_pfn(vma, addr, &pfn)) { + mutex_unlock(&encl->lock); + + return VM_FAULT_NOPAGE; + } + + ret = vmf_insert_pfn(vma, addr, PFN_DOWN(phys_addr)); + if (ret != VM_FAULT_NOPAGE) { + mutex_unlock(&encl->lock); + + return VM_FAULT_SIGBUS; + } + + sgx_encl_test_and_clear_young(vma->vm_mm, entry); + mutex_unlock(&encl->lock); + + return VM_FAULT_NOPAGE; +} + +static void sgx_vma_open(struct vm_area_struct *vma) +{ + struct sgx_encl *encl = vma->vm_private_data; + + /* + * It's possible but unlikely that vm_private_data is NULL. This can + * happen in a grandchild of a process, when sgx_encl_mm_add() had + * failed to allocate memory in this callback. + */ + if (unlikely(!encl)) + return; + + if (sgx_encl_mm_add(encl, vma->vm_mm)) + vma->vm_private_data = NULL; +} + + +/** + * sgx_encl_may_map() - Check if a requested VMA mapping is allowed + * @encl: an enclave pointer + * @start: lower bound of the address range, inclusive + * @end: upper bound of the address range, exclusive + * @vm_flags: VMA flags + * + * Iterate through the enclave pages contained within [@start, @end) to verify + * that the permissions requested by a subset of {VM_READ, VM_WRITE, VM_EXEC} + * do not contain any permissions that are not contained in the build time + * permissions of any of the enclave pages within the given address range. + * + * An enclave creator must declare the strongest permissions that will be + * needed for each enclave page. This ensures that mappings have the identical + * or weaker permissions than the earlier declared permissions. + * + * Return: 0 on success, -EACCES otherwise + */ +int sgx_encl_may_map(struct sgx_encl *encl, unsigned long start, + unsigned long end, unsigned long vm_flags) +{ + unsigned long vm_prot_bits = vm_flags & (VM_READ | VM_WRITE | VM_EXEC); + struct sgx_encl_page *page; + unsigned long count = 0; + int ret = 0; + + XA_STATE(xas, &encl->page_array, PFN_DOWN(start)); + + /* + * Disallow READ_IMPLIES_EXEC tasks as their VMA permissions might + * conflict with the enclave page permissions. + */ + if (current->personality & READ_IMPLIES_EXEC) + return -EACCES; + + mutex_lock(&encl->lock); + xas_lock(&xas); + xas_for_each(&xas, page, PFN_DOWN(end - 1)) { + if (~page->vm_max_prot_bits & vm_prot_bits) { + ret = -EACCES; + break; + } + + /* Reschedule on every XA_CHECK_SCHED iteration. */ + if (!(++count % XA_CHECK_SCHED)) { + xas_pause(&xas); + xas_unlock(&xas); + mutex_unlock(&encl->lock); + + cond_resched(); + + mutex_lock(&encl->lock); + xas_lock(&xas); + } + } + xas_unlock(&xas); + mutex_unlock(&encl->lock); + + return ret; +} + +static int sgx_vma_mprotect(struct vm_area_struct *vma, unsigned long start, + unsigned long end, unsigned long newflags) +{ + return sgx_encl_may_map(vma->vm_private_data, start, end, newflags); +} + +static int sgx_encl_debug_read(struct sgx_encl *encl, struct sgx_encl_page *page, + unsigned long addr, void *data) +{ + unsigned long offset = addr & ~PAGE_MASK; + int ret; + + + ret = __edbgrd(sgx_get_epc_virt_addr(page->epc_page) + offset, data); + if (ret) + return -EIO; + + return 0; +} + +static int sgx_encl_debug_write(struct sgx_encl *encl, struct sgx_encl_page *page, + unsigned long addr, void *data) +{ + unsigned long offset = addr & ~PAGE_MASK; + int ret; + + ret = __edbgwr(sgx_get_epc_virt_addr(page->epc_page) + offset, data); + if (ret) + return -EIO; + + return 0; +} + +/* + * Load an enclave page to EPC if required, and take encl->lock. + */ +static struct sgx_encl_page *sgx_encl_reserve_page(struct sgx_encl *encl, + unsigned long addr, + unsigned long vm_flags) +{ + struct sgx_encl_page *entry; + + for ( ; ; ) { + mutex_lock(&encl->lock); + + entry = sgx_encl_load_page(encl, addr, vm_flags); + if (PTR_ERR(entry) != -EBUSY) + break; + + mutex_unlock(&encl->lock); + } + + if (IS_ERR(entry)) + mutex_unlock(&encl->lock); + + return entry; +} + +static int sgx_vma_access(struct vm_area_struct *vma, unsigned long addr, + void *buf, int len, int write) +{ + struct sgx_encl *encl = vma->vm_private_data; + struct sgx_encl_page *entry = NULL; + char data[sizeof(unsigned long)]; + unsigned long align; + int offset; + int cnt; + int ret = 0; + int i; + + /* + * If process was forked, VMA is still there but vm_private_data is set + * to NULL. + */ + if (!encl) + return -EFAULT; + + if (!test_bit(SGX_ENCL_DEBUG, &encl->flags)) + return -EFAULT; + + for (i = 0; i < len; i += cnt) { + entry = sgx_encl_reserve_page(encl, (addr + i) & PAGE_MASK, + vma->vm_flags); + if (IS_ERR(entry)) { + ret = PTR_ERR(entry); + break; + } + + align = ALIGN_DOWN(addr + i, sizeof(unsigned long)); + offset = (addr + i) & (sizeof(unsigned long) - 1); + cnt = sizeof(unsigned long) - offset; + cnt = min(cnt, len - i); + + ret = sgx_encl_debug_read(encl, entry, align, data); + if (ret) + goto out; + + if (write) { + memcpy(data + offset, buf + i, cnt); + ret = sgx_encl_debug_write(encl, entry, align, data); + if (ret) + goto out; + } else { + memcpy(buf + i, data + offset, cnt); + } + +out: + mutex_unlock(&encl->lock); + + if (ret) + break; + } + + return ret < 0 ? ret : i; +} + +const struct vm_operations_struct sgx_vm_ops = { + .fault = sgx_vma_fault, + .mprotect = sgx_vma_mprotect, + .open = sgx_vma_open, + .access = sgx_vma_access, +}; + +/** + * sgx_encl_release - Destroy an enclave instance + * @kref: address of a kref inside &sgx_encl + * + * Used together with kref_put(). Frees all the resources associated with the + * enclave and the instance itself. + */ +void sgx_encl_release(struct kref *ref) +{ + struct sgx_encl *encl = container_of(ref, struct sgx_encl, refcount); + struct sgx_va_page *va_page; + struct sgx_encl_page *entry; + unsigned long index; + + xa_for_each(&encl->page_array, index, entry) { + if (entry->epc_page) { + /* + * The page and its radix tree entry cannot be freed + * if the page is being held by the reclaimer. + */ + if (sgx_unmark_page_reclaimable(entry->epc_page)) + continue; + + sgx_free_epc_page(entry->epc_page); + encl->secs_child_cnt--; + entry->epc_page = NULL; + } + + kfree(entry); + } + + xa_destroy(&encl->page_array); + + if (!encl->secs_child_cnt && encl->secs.epc_page) { + sgx_free_epc_page(encl->secs.epc_page); + encl->secs.epc_page = NULL; + } + + while (!list_empty(&encl->va_pages)) { + va_page = list_first_entry(&encl->va_pages, struct sgx_va_page, + list); + list_del(&va_page->list); + sgx_free_epc_page(va_page->epc_page); + kfree(va_page); + } + + if (encl->backing) + fput(encl->backing); + + cleanup_srcu_struct(&encl->srcu); + + WARN_ON_ONCE(!list_empty(&encl->mm_list)); + + /* Detect EPC page leak's. */ + WARN_ON_ONCE(encl->secs_child_cnt); + WARN_ON_ONCE(encl->secs.epc_page); + + kfree(encl); +} + +/* + * 'mm' is exiting and no longer needs mmu notifications. + */ +static void sgx_mmu_notifier_release(struct mmu_notifier *mn, + struct mm_struct *mm) +{ + struct sgx_encl_mm *encl_mm = container_of(mn, struct sgx_encl_mm, mmu_notifier); + struct sgx_encl_mm *tmp = NULL; + + /* + * The enclave itself can remove encl_mm. Note, objects can't be moved + * off an RCU protected list, but deletion is ok. + */ + spin_lock(&encl_mm->encl->mm_lock); + list_for_each_entry(tmp, &encl_mm->encl->mm_list, list) { + if (tmp == encl_mm) { + list_del_rcu(&encl_mm->list); + break; + } + } + spin_unlock(&encl_mm->encl->mm_lock); + + if (tmp == encl_mm) { + synchronize_srcu(&encl_mm->encl->srcu); + mmu_notifier_put(mn); + } +} + +static void sgx_mmu_notifier_free(struct mmu_notifier *mn) +{ + struct sgx_encl_mm *encl_mm = container_of(mn, struct sgx_encl_mm, mmu_notifier); + + kfree(encl_mm); +} + +static const struct mmu_notifier_ops sgx_mmu_notifier_ops = { + .release = sgx_mmu_notifier_release, + .free_notifier = sgx_mmu_notifier_free, +}; + +static struct sgx_encl_mm *sgx_encl_find_mm(struct sgx_encl *encl, + struct mm_struct *mm) +{ + struct sgx_encl_mm *encl_mm = NULL; + struct sgx_encl_mm *tmp; + int idx; + + idx = srcu_read_lock(&encl->srcu); + + list_for_each_entry_rcu(tmp, &encl->mm_list, list) { + if (tmp->mm == mm) { + encl_mm = tmp; + break; + } + } + + srcu_read_unlock(&encl->srcu, idx); + + return encl_mm; +} + +int sgx_encl_mm_add(struct sgx_encl *encl, struct mm_struct *mm) +{ + struct sgx_encl_mm *encl_mm; + int ret; + + /* + * Even though a single enclave may be mapped into an mm more than once, + * each 'mm' only appears once on encl->mm_list. This is guaranteed by + * holding the mm's mmap lock for write before an mm can be added or + * remove to an encl->mm_list. + */ + mmap_assert_write_locked(mm); + + /* + * It's possible that an entry already exists in the mm_list, because it + * is removed only on VFS release or process exit. + */ + if (sgx_encl_find_mm(encl, mm)) + return 0; + + encl_mm = kzalloc(sizeof(*encl_mm), GFP_KERNEL); + if (!encl_mm) + return -ENOMEM; + + encl_mm->encl = encl; + encl_mm->mm = mm; + encl_mm->mmu_notifier.ops = &sgx_mmu_notifier_ops; + + ret = __mmu_notifier_register(&encl_mm->mmu_notifier, mm); + if (ret) { + kfree(encl_mm); + return ret; + } + + spin_lock(&encl->mm_lock); + list_add_rcu(&encl_mm->list, &encl->mm_list); + /* Pairs with smp_rmb() in sgx_reclaimer_block(). */ + smp_wmb(); + encl->mm_list_version++; + spin_unlock(&encl->mm_lock); + + return 0; +} + +static struct page *sgx_encl_get_backing_page(struct sgx_encl *encl, + pgoff_t index) +{ + struct inode *inode = encl->backing->f_path.dentry->d_inode; + struct address_space *mapping = inode->i_mapping; + gfp_t gfpmask = mapping_gfp_mask(mapping); + + return shmem_read_mapping_page_gfp(mapping, index, gfpmask); +} + +/** + * sgx_encl_get_backing() - Pin the backing storage + * @encl: an enclave pointer + * @page_index: enclave page index + * @backing: data for accessing backing storage for the page + * + * Pin the backing storage pages for storing the encrypted contents and Paging + * Crypto MetaData (PCMD) of an enclave page. + * + * Return: + * 0 on success, + * -errno otherwise. + */ +int sgx_encl_get_backing(struct sgx_encl *encl, unsigned long page_index, + struct sgx_backing *backing) +{ + pgoff_t pcmd_index = PFN_DOWN(encl->size) + 1 + (page_index >> 5); + struct page *contents; + struct page *pcmd; + + contents = sgx_encl_get_backing_page(encl, page_index); + if (IS_ERR(contents)) + return PTR_ERR(contents); + + pcmd = sgx_encl_get_backing_page(encl, pcmd_index); + if (IS_ERR(pcmd)) { + put_page(contents); + return PTR_ERR(pcmd); + } + + backing->page_index = page_index; + backing->contents = contents; + backing->pcmd = pcmd; + backing->pcmd_offset = + (page_index & (PAGE_SIZE / sizeof(struct sgx_pcmd) - 1)) * + sizeof(struct sgx_pcmd); + + return 0; +} + +/** + * sgx_encl_put_backing() - Unpin the backing storage + * @backing: data for accessing backing storage for the page + * @do_write: mark pages dirty + */ +void sgx_encl_put_backing(struct sgx_backing *backing, bool do_write) +{ + if (do_write) { + set_page_dirty(backing->pcmd); + set_page_dirty(backing->contents); + } + + put_page(backing->pcmd); + put_page(backing->contents); +} + +static int sgx_encl_test_and_clear_young_cb(pte_t *ptep, unsigned long addr, + void *data) +{ + pte_t pte; + int ret; + + ret = pte_young(*ptep); + if (ret) { + pte = pte_mkold(*ptep); + set_pte_at((struct mm_struct *)data, addr, ptep, pte); + } + + return ret; +} + +/** + * sgx_encl_test_and_clear_young() - Test and reset the accessed bit + * @mm: mm_struct that is checked + * @page: enclave page to be tested for recent access + * + * Checks the Access (A) bit from the PTE corresponding to the enclave page and + * clears it. + * + * Return: 1 if the page has been recently accessed and 0 if not. + */ +int sgx_encl_test_and_clear_young(struct mm_struct *mm, + struct sgx_encl_page *page) +{ + unsigned long addr = page->desc & PAGE_MASK; + struct sgx_encl *encl = page->encl; + struct vm_area_struct *vma; + int ret; + + ret = sgx_encl_find(mm, addr, &vma); + if (ret) + return 0; + + if (encl != vma->vm_private_data) + return 0; + + ret = apply_to_page_range(vma->vm_mm, addr, PAGE_SIZE, + sgx_encl_test_and_clear_young_cb, vma->vm_mm); + if (ret < 0) + return 0; + + return ret; +} + +/** + * sgx_alloc_va_page() - Allocate a Version Array (VA) page + * + * Allocate a free EPC page and convert it to a Version Array (VA) page. + * + * Return: + * a VA page, + * -errno otherwise + */ +struct sgx_epc_page *sgx_alloc_va_page(void) +{ + struct sgx_epc_page *epc_page; + int ret; + + epc_page = sgx_alloc_epc_page(NULL, true); + if (IS_ERR(epc_page)) + return ERR_CAST(epc_page); + + ret = __epa(sgx_get_epc_virt_addr(epc_page)); + if (ret) { + WARN_ONCE(1, "EPA returned %d (0x%x)", ret, ret); + sgx_free_epc_page(epc_page); + return ERR_PTR(-EFAULT); + } + + return epc_page; +} + +/** + * sgx_alloc_va_slot - allocate a VA slot + * @va_page: a &struct sgx_va_page instance + * + * Allocates a slot from a &struct sgx_va_page instance. + * + * Return: offset of the slot inside the VA page + */ +unsigned int sgx_alloc_va_slot(struct sgx_va_page *va_page) +{ + int slot = find_first_zero_bit(va_page->slots, SGX_VA_SLOT_COUNT); + + if (slot < SGX_VA_SLOT_COUNT) + set_bit(slot, va_page->slots); + + return slot << 3; +} + +/** + * sgx_free_va_slot - free a VA slot + * @va_page: a &struct sgx_va_page instance + * @offset: offset of the slot inside the VA page + * + * Frees a slot from a &struct sgx_va_page instance. + */ +void sgx_free_va_slot(struct sgx_va_page *va_page, unsigned int offset) +{ + clear_bit(offset >> 3, va_page->slots); +} + +/** + * sgx_va_page_full - is the VA page full? + * @va_page: a &struct sgx_va_page instance + * + * Return: true if all slots have been taken + */ +bool sgx_va_page_full(struct sgx_va_page *va_page) +{ + int slot = find_first_zero_bit(va_page->slots, SGX_VA_SLOT_COUNT); + + return slot == SGX_VA_SLOT_COUNT; +} diff --git a/arch/x86/kernel/cpu/sgx/encl.h b/arch/x86/kernel/cpu/sgx/encl.h new file mode 100644 index 000000000000..d8d30ccbef4c --- /dev/null +++ b/arch/x86/kernel/cpu/sgx/encl.h @@ -0,0 +1,119 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/** + * Copyright(c) 2016-20 Intel Corporation. + * + * Contains the software defined data structures for enclaves. + */ +#ifndef _X86_ENCL_H +#define _X86_ENCL_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "sgx.h" + +/* 'desc' bits holding the offset in the VA (version array) page. */ +#define SGX_ENCL_PAGE_VA_OFFSET_MASK GENMASK_ULL(11, 3) + +/* 'desc' bit marking that the page is being reclaimed. */ +#define SGX_ENCL_PAGE_BEING_RECLAIMED BIT(3) + +struct sgx_encl_page { + unsigned long desc; + unsigned long vm_max_prot_bits; + struct sgx_epc_page *epc_page; + struct sgx_encl *encl; + struct sgx_va_page *va_page; +}; + +enum sgx_encl_flags { + SGX_ENCL_IOCTL = BIT(0), + SGX_ENCL_DEBUG = BIT(1), + SGX_ENCL_CREATED = BIT(2), + SGX_ENCL_INITIALIZED = BIT(3), +}; + +struct sgx_encl_mm { + struct sgx_encl *encl; + struct mm_struct *mm; + struct list_head list; + struct mmu_notifier mmu_notifier; +}; + +struct sgx_encl { + unsigned long base; + unsigned long size; + unsigned long flags; + unsigned int page_cnt; + unsigned int secs_child_cnt; + struct mutex lock; + struct xarray page_array; + struct sgx_encl_page secs; + unsigned long attributes; + unsigned long attributes_mask; + + cpumask_t cpumask; + struct file *backing; + struct kref refcount; + struct list_head va_pages; + unsigned long mm_list_version; + struct list_head mm_list; + spinlock_t mm_lock; + struct srcu_struct srcu; +}; + +#define SGX_VA_SLOT_COUNT 512 + +struct sgx_va_page { + struct sgx_epc_page *epc_page; + DECLARE_BITMAP(slots, SGX_VA_SLOT_COUNT); + struct list_head list; +}; + +struct sgx_backing { + pgoff_t page_index; + struct page *contents; + struct page *pcmd; + unsigned long pcmd_offset; +}; + +extern const struct vm_operations_struct sgx_vm_ops; + +static inline int sgx_encl_find(struct mm_struct *mm, unsigned long addr, + struct vm_area_struct **vma) +{ + struct vm_area_struct *result; + + result = find_vma(mm, addr); + if (!result || result->vm_ops != &sgx_vm_ops || addr < result->vm_start) + return -EINVAL; + + *vma = result; + + return 0; +} + +int sgx_encl_may_map(struct sgx_encl *encl, unsigned long start, + unsigned long end, unsigned long vm_flags); + +void sgx_encl_release(struct kref *ref); +int sgx_encl_mm_add(struct sgx_encl *encl, struct mm_struct *mm); +int sgx_encl_get_backing(struct sgx_encl *encl, unsigned long page_index, + struct sgx_backing *backing); +void sgx_encl_put_backing(struct sgx_backing *backing, bool do_write); +int sgx_encl_test_and_clear_young(struct mm_struct *mm, + struct sgx_encl_page *page); + +struct sgx_epc_page *sgx_alloc_va_page(void); +unsigned int sgx_alloc_va_slot(struct sgx_va_page *va_page); +void sgx_free_va_slot(struct sgx_va_page *va_page, unsigned int offset); +bool sgx_va_page_full(struct sgx_va_page *va_page); + +#endif /* _X86_ENCL_H */ diff --git a/arch/x86/kernel/cpu/sgx/encls.h b/arch/x86/kernel/cpu/sgx/encls.h new file mode 100644 index 000000000000..443188fe7e70 --- /dev/null +++ b/arch/x86/kernel/cpu/sgx/encls.h @@ -0,0 +1,231 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _X86_ENCLS_H +#define _X86_ENCLS_H + +#include +#include +#include +#include +#include +#include +#include +#include "sgx.h" + +enum sgx_encls_function { + ECREATE = 0x00, + EADD = 0x01, + EINIT = 0x02, + EREMOVE = 0x03, + EDGBRD = 0x04, + EDGBWR = 0x05, + EEXTEND = 0x06, + ELDU = 0x08, + EBLOCK = 0x09, + EPA = 0x0A, + EWB = 0x0B, + ETRACK = 0x0C, +}; + +/** + * ENCLS_FAULT_FLAG - flag signifying an ENCLS return code is a trapnr + * + * ENCLS has its own (positive value) error codes and also generates + * ENCLS specific #GP and #PF faults. And the ENCLS values get munged + * with system error codes as everything percolates back up the stack. + * Unfortunately (for us), we need to precisely identify each unique + * error code, e.g. the action taken if EWB fails varies based on the + * type of fault and on the exact SGX error code, i.e. we can't simply + * convert all faults to -EFAULT. + * + * To make all three error types coexist, we set bit 30 to identify an + * ENCLS fault. Bit 31 (technically bits N:31) is used to differentiate + * between positive (faults and SGX error codes) and negative (system + * error codes) values. + */ +#define ENCLS_FAULT_FLAG 0x40000000 + +/* Retrieve the encoded trapnr from the specified return code. */ +#define ENCLS_TRAPNR(r) ((r) & ~ENCLS_FAULT_FLAG) + +/* Issue a WARN() about an ENCLS function. */ +#define ENCLS_WARN(r, name) { \ + do { \ + int _r = (r); \ + WARN_ONCE(_r, "%s returned %d (0x%x)\n", (name), _r, _r); \ + } while (0); \ +} + +/** + * encls_failed() - Check if an ENCLS function failed + * @ret: the return value of an ENCLS function call + * + * Check if an ENCLS function failed. This happens when the function causes a + * fault that is not caused by an EPCM conflict or when the function returns a + * non-zero value. + */ +static inline bool encls_failed(int ret) +{ + if (ret & ENCLS_FAULT_FLAG) + return ENCLS_TRAPNR(ret) != X86_TRAP_PF; + + return !!ret; +} + +/** + * __encls_ret_N - encode an ENCLS function that returns an error code in EAX + * @rax: function number + * @inputs: asm inputs for the function + * + * Emit assembly for an ENCLS function that returns an error code, e.g. EREMOVE. + * And because SGX isn't complex enough as it is, function that return an error + * code also modify flags. + * + * Return: + * 0 on success, + * SGX error code on failure + */ +#define __encls_ret_N(rax, inputs...) \ + ({ \ + int ret; \ + asm volatile( \ + "1: .byte 0x0f, 0x01, 0xcf;\n\t" \ + "2:\n" \ + ".section .fixup,\"ax\"\n" \ + "3: orl $"__stringify(ENCLS_FAULT_FLAG)",%%eax\n" \ + " jmp 2b\n" \ + ".previous\n" \ + _ASM_EXTABLE_FAULT(1b, 3b) \ + : "=a"(ret) \ + : "a"(rax), inputs \ + : "memory", "cc"); \ + ret; \ + }) + +#define __encls_ret_1(rax, rcx) \ + ({ \ + __encls_ret_N(rax, "c"(rcx)); \ + }) + +#define __encls_ret_2(rax, rbx, rcx) \ + ({ \ + __encls_ret_N(rax, "b"(rbx), "c"(rcx)); \ + }) + +#define __encls_ret_3(rax, rbx, rcx, rdx) \ + ({ \ + __encls_ret_N(rax, "b"(rbx), "c"(rcx), "d"(rdx)); \ + }) + +/** + * __encls_N - encode an ENCLS function that doesn't return an error code + * @rax: function number + * @rbx_out: optional output variable + * @inputs: asm inputs for the function + * + * Emit assembly for an ENCLS function that does not return an error code, e.g. + * ECREATE. Leaves without error codes either succeed or fault. @rbx_out is an + * optional parameter for use by EDGBRD, which returns the requested value in + * RBX. + * + * Return: + * 0 on success, + * trapnr with ENCLS_FAULT_FLAG set on fault + */ +#define __encls_N(rax, rbx_out, inputs...) \ + ({ \ + int ret; \ + asm volatile( \ + "1: .byte 0x0f, 0x01, 0xcf;\n\t" \ + " xor %%eax,%%eax;\n" \ + "2:\n" \ + ".section .fixup,\"ax\"\n" \ + "3: orl $"__stringify(ENCLS_FAULT_FLAG)",%%eax\n" \ + " jmp 2b\n" \ + ".previous\n" \ + _ASM_EXTABLE_FAULT(1b, 3b) \ + : "=a"(ret), "=b"(rbx_out) \ + : "a"(rax), inputs \ + : "memory"); \ + ret; \ + }) + +#define __encls_2(rax, rbx, rcx) \ + ({ \ + unsigned long ign_rbx_out; \ + __encls_N(rax, ign_rbx_out, "b"(rbx), "c"(rcx)); \ + }) + +#define __encls_1_1(rax, data, rcx) \ + ({ \ + unsigned long rbx_out; \ + int ret = __encls_N(rax, rbx_out, "c"(rcx)); \ + if (!ret) \ + data = rbx_out; \ + ret; \ + }) + +static inline int __ecreate(struct sgx_pageinfo *pginfo, void *secs) +{ + return __encls_2(ECREATE, pginfo, secs); +} + +static inline int __eextend(void *secs, void *addr) +{ + return __encls_2(EEXTEND, secs, addr); +} + +static inline int __eadd(struct sgx_pageinfo *pginfo, void *addr) +{ + return __encls_2(EADD, pginfo, addr); +} + +static inline int __einit(void *sigstruct, void *token, void *secs) +{ + return __encls_ret_3(EINIT, sigstruct, secs, token); +} + +static inline int __eremove(void *addr) +{ + return __encls_ret_1(EREMOVE, addr); +} + +static inline int __edbgwr(void *addr, unsigned long *data) +{ + return __encls_2(EDGBWR, *data, addr); +} + +static inline int __edbgrd(void *addr, unsigned long *data) +{ + return __encls_1_1(EDGBRD, *data, addr); +} + +static inline int __etrack(void *addr) +{ + return __encls_ret_1(ETRACK, addr); +} + +static inline int __eldu(struct sgx_pageinfo *pginfo, void *addr, + void *va) +{ + return __encls_ret_3(ELDU, pginfo, addr, va); +} + +static inline int __eblock(void *addr) +{ + return __encls_ret_1(EBLOCK, addr); +} + +static inline int __epa(void *addr) +{ + unsigned long rbx = SGX_PAGE_TYPE_VA; + + return __encls_2(EPA, rbx, addr); +} + +static inline int __ewb(struct sgx_pageinfo *pginfo, void *addr, + void *va) +{ + return __encls_ret_3(EWB, pginfo, addr, va); +} + +#endif /* _X86_ENCLS_H */ diff --git a/arch/x86/kernel/cpu/sgx/ioctl.c b/arch/x86/kernel/cpu/sgx/ioctl.c new file mode 100644 index 000000000000..90a5caf76939 --- /dev/null +++ b/arch/x86/kernel/cpu/sgx/ioctl.c @@ -0,0 +1,716 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2016-20 Intel Corporation. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "driver.h" +#include "encl.h" +#include "encls.h" + +static struct sgx_va_page *sgx_encl_grow(struct sgx_encl *encl) +{ + struct sgx_va_page *va_page = NULL; + void *err; + + BUILD_BUG_ON(SGX_VA_SLOT_COUNT != + (SGX_ENCL_PAGE_VA_OFFSET_MASK >> 3) + 1); + + if (!(encl->page_cnt % SGX_VA_SLOT_COUNT)) { + va_page = kzalloc(sizeof(*va_page), GFP_KERNEL); + if (!va_page) + return ERR_PTR(-ENOMEM); + + va_page->epc_page = sgx_alloc_va_page(); + if (IS_ERR(va_page->epc_page)) { + err = ERR_CAST(va_page->epc_page); + kfree(va_page); + return err; + } + + WARN_ON_ONCE(encl->page_cnt % SGX_VA_SLOT_COUNT); + } + encl->page_cnt++; + return va_page; +} + +static void sgx_encl_shrink(struct sgx_encl *encl, struct sgx_va_page *va_page) +{ + encl->page_cnt--; + + if (va_page) { + sgx_free_epc_page(va_page->epc_page); + list_del(&va_page->list); + kfree(va_page); + } +} + +static int sgx_encl_create(struct sgx_encl *encl, struct sgx_secs *secs) +{ + struct sgx_epc_page *secs_epc; + struct sgx_va_page *va_page; + struct sgx_pageinfo pginfo; + struct sgx_secinfo secinfo; + unsigned long encl_size; + struct file *backing; + long ret; + + va_page = sgx_encl_grow(encl); + if (IS_ERR(va_page)) + return PTR_ERR(va_page); + else if (va_page) + list_add(&va_page->list, &encl->va_pages); + /* else the tail page of the VA page list had free slots. */ + + /* The extra page goes to SECS. */ + encl_size = secs->size + PAGE_SIZE; + + backing = shmem_file_setup("SGX backing", encl_size + (encl_size >> 5), + VM_NORESERVE); + if (IS_ERR(backing)) { + ret = PTR_ERR(backing); + goto err_out_shrink; + } + + encl->backing = backing; + + secs_epc = sgx_alloc_epc_page(&encl->secs, true); + if (IS_ERR(secs_epc)) { + ret = PTR_ERR(secs_epc); + goto err_out_backing; + } + + encl->secs.epc_page = secs_epc; + + pginfo.addr = 0; + pginfo.contents = (unsigned long)secs; + pginfo.metadata = (unsigned long)&secinfo; + pginfo.secs = 0; + memset(&secinfo, 0, sizeof(secinfo)); + + ret = __ecreate((void *)&pginfo, sgx_get_epc_virt_addr(secs_epc)); + if (ret) { + ret = -EIO; + goto err_out; + } + + if (secs->attributes & SGX_ATTR_DEBUG) + set_bit(SGX_ENCL_DEBUG, &encl->flags); + + encl->secs.encl = encl; + encl->base = secs->base; + encl->size = secs->size; + encl->attributes = secs->attributes; + encl->attributes_mask = SGX_ATTR_DEBUG | SGX_ATTR_MODE64BIT | SGX_ATTR_KSS; + + /* Set only after completion, as encl->lock has not been taken. */ + set_bit(SGX_ENCL_CREATED, &encl->flags); + + return 0; + +err_out: + sgx_free_epc_page(encl->secs.epc_page); + encl->secs.epc_page = NULL; + +err_out_backing: + fput(encl->backing); + encl->backing = NULL; + +err_out_shrink: + sgx_encl_shrink(encl, va_page); + + return ret; +} + +/** + * sgx_ioc_enclave_create() - handler for %SGX_IOC_ENCLAVE_CREATE + * @encl: An enclave pointer. + * @arg: The ioctl argument. + * + * Allocate kernel data structures for the enclave and invoke ECREATE. + * + * Return: + * - 0: Success. + * - -EIO: ECREATE failed. + * - -errno: POSIX error. + */ +static long sgx_ioc_enclave_create(struct sgx_encl *encl, void __user *arg) +{ + struct sgx_enclave_create create_arg; + void *secs; + int ret; + + if (test_bit(SGX_ENCL_CREATED, &encl->flags)) + return -EINVAL; + + if (copy_from_user(&create_arg, arg, sizeof(create_arg))) + return -EFAULT; + + secs = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!secs) + return -ENOMEM; + + if (copy_from_user(secs, (void __user *)create_arg.src, PAGE_SIZE)) + ret = -EFAULT; + else + ret = sgx_encl_create(encl, secs); + + kfree(secs); + return ret; +} + +static struct sgx_encl_page *sgx_encl_page_alloc(struct sgx_encl *encl, + unsigned long offset, + u64 secinfo_flags) +{ + struct sgx_encl_page *encl_page; + unsigned long prot; + + encl_page = kzalloc(sizeof(*encl_page), GFP_KERNEL); + if (!encl_page) + return ERR_PTR(-ENOMEM); + + encl_page->desc = encl->base + offset; + encl_page->encl = encl; + + prot = _calc_vm_trans(secinfo_flags, SGX_SECINFO_R, PROT_READ) | + _calc_vm_trans(secinfo_flags, SGX_SECINFO_W, PROT_WRITE) | + _calc_vm_trans(secinfo_flags, SGX_SECINFO_X, PROT_EXEC); + + /* + * TCS pages must always RW set for CPU access while the SECINFO + * permissions are *always* zero - the CPU ignores the user provided + * values and silently overwrites them with zero permissions. + */ + if ((secinfo_flags & SGX_SECINFO_PAGE_TYPE_MASK) == SGX_SECINFO_TCS) + prot |= PROT_READ | PROT_WRITE; + + /* Calculate maximum of the VM flags for the page. */ + encl_page->vm_max_prot_bits = calc_vm_prot_bits(prot, 0); + + return encl_page; +} + +static int sgx_validate_secinfo(struct sgx_secinfo *secinfo) +{ + u64 perm = secinfo->flags & SGX_SECINFO_PERMISSION_MASK; + u64 pt = secinfo->flags & SGX_SECINFO_PAGE_TYPE_MASK; + + if (pt != SGX_SECINFO_REG && pt != SGX_SECINFO_TCS) + return -EINVAL; + + if ((perm & SGX_SECINFO_W) && !(perm & SGX_SECINFO_R)) + return -EINVAL; + + /* + * CPU will silently overwrite the permissions as zero, which means + * that we need to validate it ourselves. + */ + if (pt == SGX_SECINFO_TCS && perm) + return -EINVAL; + + if (secinfo->flags & SGX_SECINFO_RESERVED_MASK) + return -EINVAL; + + if (memchr_inv(secinfo->reserved, 0, sizeof(secinfo->reserved))) + return -EINVAL; + + return 0; +} + +static int __sgx_encl_add_page(struct sgx_encl *encl, + struct sgx_encl_page *encl_page, + struct sgx_epc_page *epc_page, + struct sgx_secinfo *secinfo, unsigned long src) +{ + struct sgx_pageinfo pginfo; + struct vm_area_struct *vma; + struct page *src_page; + int ret; + + /* Deny noexec. */ + vma = find_vma(current->mm, src); + if (!vma) + return -EFAULT; + + if (!(vma->vm_flags & VM_MAYEXEC)) + return -EACCES; + + ret = get_user_pages(src, 1, 0, &src_page, NULL); + if (ret < 1) + return -EFAULT; + + pginfo.secs = (unsigned long)sgx_get_epc_virt_addr(encl->secs.epc_page); + pginfo.addr = encl_page->desc & PAGE_MASK; + pginfo.metadata = (unsigned long)secinfo; + pginfo.contents = (unsigned long)kmap_atomic(src_page); + + ret = __eadd(&pginfo, sgx_get_epc_virt_addr(epc_page)); + + kunmap_atomic((void *)pginfo.contents); + put_page(src_page); + + return ret ? -EIO : 0; +} + +/* + * If the caller requires measurement of the page as a proof for the content, + * use EEXTEND to add a measurement for 256 bytes of the page. Repeat this + * operation until the entire page is measured." + */ +static int __sgx_encl_extend(struct sgx_encl *encl, + struct sgx_epc_page *epc_page) +{ + unsigned long offset; + int ret; + + for (offset = 0; offset < PAGE_SIZE; offset += SGX_EEXTEND_BLOCK_SIZE) { + ret = __eextend(sgx_get_epc_virt_addr(encl->secs.epc_page), + sgx_get_epc_virt_addr(epc_page) + offset); + if (ret) { + if (encls_failed(ret)) + ENCLS_WARN(ret, "EEXTEND"); + + return -EIO; + } + } + + return 0; +} + +static int sgx_encl_add_page(struct sgx_encl *encl, unsigned long src, + unsigned long offset, struct sgx_secinfo *secinfo, + unsigned long flags) +{ + struct sgx_encl_page *encl_page; + struct sgx_epc_page *epc_page; + struct sgx_va_page *va_page; + int ret; + + encl_page = sgx_encl_page_alloc(encl, offset, secinfo->flags); + if (IS_ERR(encl_page)) + return PTR_ERR(encl_page); + + epc_page = sgx_alloc_epc_page(encl_page, true); + if (IS_ERR(epc_page)) { + kfree(encl_page); + return PTR_ERR(epc_page); + } + + va_page = sgx_encl_grow(encl); + if (IS_ERR(va_page)) { + ret = PTR_ERR(va_page); + goto err_out_free; + } + + mmap_read_lock(current->mm); + mutex_lock(&encl->lock); + + /* + * Adding to encl->va_pages must be done under encl->lock. Ditto for + * deleting (via sgx_encl_shrink()) in the error path. + */ + if (va_page) + list_add(&va_page->list, &encl->va_pages); + + /* + * Insert prior to EADD in case of OOM. EADD modifies MRENCLAVE, i.e. + * can't be gracefully unwound, while failure on EADD/EXTEND is limited + * to userspace errors (or kernel/hardware bugs). + */ + ret = xa_insert(&encl->page_array, PFN_DOWN(encl_page->desc), + encl_page, GFP_KERNEL); + if (ret) + goto err_out_unlock; + + ret = __sgx_encl_add_page(encl, encl_page, epc_page, secinfo, + src); + if (ret) + goto err_out; + + /* + * Complete the "add" before doing the "extend" so that the "add" + * isn't in a half-baked state in the extremely unlikely scenario + * the enclave will be destroyed in response to EEXTEND failure. + */ + encl_page->encl = encl; + encl_page->epc_page = epc_page; + encl->secs_child_cnt++; + + if (flags & SGX_PAGE_MEASURE) { + ret = __sgx_encl_extend(encl, epc_page); + if (ret) + goto err_out; + } + + sgx_mark_page_reclaimable(encl_page->epc_page); + mutex_unlock(&encl->lock); + mmap_read_unlock(current->mm); + return ret; + +err_out: + xa_erase(&encl->page_array, PFN_DOWN(encl_page->desc)); + +err_out_unlock: + sgx_encl_shrink(encl, va_page); + mutex_unlock(&encl->lock); + mmap_read_unlock(current->mm); + +err_out_free: + sgx_free_epc_page(epc_page); + kfree(encl_page); + + return ret; +} + +/** + * sgx_ioc_enclave_add_pages() - The handler for %SGX_IOC_ENCLAVE_ADD_PAGES + * @encl: an enclave pointer + * @arg: a user pointer to a struct sgx_enclave_add_pages instance + * + * Add one or more pages to an uninitialized enclave, and optionally extend the + * measurement with the contents of the page. The SECINFO and measurement mask + * are applied to all pages. + * + * A SECINFO for a TCS is required to always contain zero permissions because + * CPU silently zeros them. Allowing anything else would cause a mismatch in + * the measurement. + * + * mmap()'s protection bits are capped by the page permissions. For each page + * address, the maximum protection bits are computed with the following + * heuristics: + * + * 1. A regular page: PROT_R, PROT_W and PROT_X match the SECINFO permissions. + * 2. A TCS page: PROT_R | PROT_W. + * + * mmap() is not allowed to surpass the minimum of the maximum protection bits + * within the given address range. + * + * The function deinitializes kernel data structures for enclave and returns + * -EIO in any of the following conditions: + * + * - Enclave Page Cache (EPC), the physical memory holding enclaves, has + * been invalidated. This will cause EADD and EEXTEND to fail. + * - If the source address is corrupted somehow when executing EADD. + * + * Return: + * - 0: Success. + * - -EACCES: The source page is located in a noexec partition. + * - -ENOMEM: Out of EPC pages. + * - -EINTR: The call was interrupted before data was processed. + * - -EIO: Either EADD or EEXTEND failed because invalid source address + * or power cycle. + * - -errno: POSIX error. + */ +static long sgx_ioc_enclave_add_pages(struct sgx_encl *encl, void __user *arg) +{ + struct sgx_enclave_add_pages add_arg; + struct sgx_secinfo secinfo; + unsigned long c; + int ret; + + if (!test_bit(SGX_ENCL_CREATED, &encl->flags) || + test_bit(SGX_ENCL_INITIALIZED, &encl->flags)) + return -EINVAL; + + if (copy_from_user(&add_arg, arg, sizeof(add_arg))) + return -EFAULT; + + if (!IS_ALIGNED(add_arg.offset, PAGE_SIZE) || + !IS_ALIGNED(add_arg.src, PAGE_SIZE)) + return -EINVAL; + + if (!add_arg.length || add_arg.length & (PAGE_SIZE - 1)) + return -EINVAL; + + if (add_arg.offset + add_arg.length - PAGE_SIZE >= encl->size) + return -EINVAL; + + if (copy_from_user(&secinfo, (void __user *)add_arg.secinfo, + sizeof(secinfo))) + return -EFAULT; + + if (sgx_validate_secinfo(&secinfo)) + return -EINVAL; + + for (c = 0 ; c < add_arg.length; c += PAGE_SIZE) { + if (signal_pending(current)) { + if (!c) + ret = -ERESTARTSYS; + + break; + } + + if (need_resched()) + cond_resched(); + + ret = sgx_encl_add_page(encl, add_arg.src + c, add_arg.offset + c, + &secinfo, add_arg.flags); + if (ret) + break; + } + + add_arg.count = c; + + if (copy_to_user(arg, &add_arg, sizeof(add_arg))) + return -EFAULT; + + return ret; +} + +static int __sgx_get_key_hash(struct crypto_shash *tfm, const void *modulus, + void *hash) +{ + SHASH_DESC_ON_STACK(shash, tfm); + + shash->tfm = tfm; + + return crypto_shash_digest(shash, modulus, SGX_MODULUS_SIZE, hash); +} + +static int sgx_get_key_hash(const void *modulus, void *hash) +{ + struct crypto_shash *tfm; + int ret; + + tfm = crypto_alloc_shash("sha256", 0, CRYPTO_ALG_ASYNC); + if (IS_ERR(tfm)) + return PTR_ERR(tfm); + + ret = __sgx_get_key_hash(tfm, modulus, hash); + + crypto_free_shash(tfm); + return ret; +} + +static int sgx_encl_init(struct sgx_encl *encl, struct sgx_sigstruct *sigstruct, + void *token) +{ + u64 mrsigner[4]; + int i, j, k; + void *addr; + int ret; + + /* + * Deny initializing enclaves with attributes (namely provisioning) + * that have not been explicitly allowed. + */ + if (encl->attributes & ~encl->attributes_mask) + return -EACCES; + + /* + * Attributes should not be enforced *only* against what's available on + * platform (done in sgx_encl_create) but checked and enforced against + * the mask for enforcement in sigstruct. For example an enclave could + * opt to sign with AVX bit in xfrm, but still be loadable on a platform + * without it if the sigstruct->body.attributes_mask does not turn that + * bit on. + */ + if (sigstruct->body.attributes & sigstruct->body.attributes_mask & + sgx_attributes_reserved_mask) + return -EINVAL; + + if (sigstruct->body.miscselect & sigstruct->body.misc_mask & + sgx_misc_reserved_mask) + return -EINVAL; + + if (sigstruct->body.xfrm & sigstruct->body.xfrm_mask & + sgx_xfrm_reserved_mask) + return -EINVAL; + + ret = sgx_get_key_hash(sigstruct->modulus, mrsigner); + if (ret) + return ret; + + mutex_lock(&encl->lock); + + /* + * ENCLS[EINIT] is interruptible because it has such a high latency, + * e.g. 50k+ cycles on success. If an IRQ/NMI/SMI becomes pending, + * EINIT may fail with SGX_UNMASKED_EVENT so that the event can be + * serviced. + */ + for (i = 0; i < SGX_EINIT_SLEEP_COUNT; i++) { + for (j = 0; j < SGX_EINIT_SPIN_COUNT; j++) { + addr = sgx_get_epc_virt_addr(encl->secs.epc_page); + + preempt_disable(); + + for (k = 0; k < 4; k++) + wrmsrl(MSR_IA32_SGXLEPUBKEYHASH0 + k, mrsigner[k]); + + ret = __einit(sigstruct, token, addr); + + preempt_enable(); + + if (ret == SGX_UNMASKED_EVENT) + continue; + else + break; + } + + if (ret != SGX_UNMASKED_EVENT) + break; + + msleep_interruptible(SGX_EINIT_SLEEP_TIME); + + if (signal_pending(current)) { + ret = -ERESTARTSYS; + goto err_out; + } + } + + if (ret & ENCLS_FAULT_FLAG) { + if (encls_failed(ret)) + ENCLS_WARN(ret, "EINIT"); + + ret = -EIO; + } else if (ret) { + pr_debug("EINIT returned %d\n", ret); + ret = -EPERM; + } else { + set_bit(SGX_ENCL_INITIALIZED, &encl->flags); + } + +err_out: + mutex_unlock(&encl->lock); + return ret; +} + +/** + * sgx_ioc_enclave_init() - handler for %SGX_IOC_ENCLAVE_INIT + * @encl: an enclave pointer + * @arg: userspace pointer to a struct sgx_enclave_init instance + * + * Flush any outstanding enqueued EADD operations and perform EINIT. The + * Launch Enclave Public Key Hash MSRs are rewritten as necessary to match + * the enclave's MRSIGNER, which is caculated from the provided sigstruct. + * + * Return: + * - 0: Success. + * - -EPERM: Invalid SIGSTRUCT. + * - -EIO: EINIT failed because of a power cycle. + * - -errno: POSIX error. + */ +static long sgx_ioc_enclave_init(struct sgx_encl *encl, void __user *arg) +{ + struct sgx_sigstruct *sigstruct; + struct sgx_enclave_init init_arg; + struct page *initp_page; + void *token; + int ret; + + if (!test_bit(SGX_ENCL_CREATED, &encl->flags) || + test_bit(SGX_ENCL_INITIALIZED, &encl->flags)) + return -EINVAL; + + if (copy_from_user(&init_arg, arg, sizeof(init_arg))) + return -EFAULT; + + initp_page = alloc_page(GFP_KERNEL); + if (!initp_page) + return -ENOMEM; + + sigstruct = kmap(initp_page); + token = (void *)((unsigned long)sigstruct + PAGE_SIZE / 2); + memset(token, 0, SGX_LAUNCH_TOKEN_SIZE); + + if (copy_from_user(sigstruct, (void __user *)init_arg.sigstruct, + sizeof(*sigstruct))) { + ret = -EFAULT; + goto out; + } + + /* + * A legacy field used with Intel signed enclaves. These used to mean + * regular and architectural enclaves. The CPU only accepts these values + * but they do not have any other meaning. + * + * Thus, reject any other values. + */ + if (sigstruct->header.vendor != 0x0000 && + sigstruct->header.vendor != 0x8086) { + ret = -EINVAL; + goto out; + } + + ret = sgx_encl_init(encl, sigstruct, token); + +out: + kunmap(initp_page); + __free_page(initp_page); + return ret; +} + +/** + * sgx_ioc_enclave_provision() - handler for %SGX_IOC_ENCLAVE_PROVISION + * @encl: an enclave pointer + * @arg: userspace pointer to a struct sgx_enclave_provision instance + * + * Allow ATTRIBUTE.PROVISION_KEY for an enclave by providing a file handle to + * /dev/sgx_provision. + * + * Return: + * - 0: Success. + * - -errno: Otherwise. + */ +static long sgx_ioc_enclave_provision(struct sgx_encl *encl, void __user *arg) +{ + struct sgx_enclave_provision params; + struct file *file; + + if (copy_from_user(¶ms, arg, sizeof(params))) + return -EFAULT; + + file = fget(params.fd); + if (!file) + return -EINVAL; + + if (file->f_op != &sgx_provision_fops) { + fput(file); + return -EINVAL; + } + + encl->attributes_mask |= SGX_ATTR_PROVISIONKEY; + + fput(file); + return 0; +} + +long sgx_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) +{ + struct sgx_encl *encl = filep->private_data; + int ret; + + if (test_and_set_bit(SGX_ENCL_IOCTL, &encl->flags)) + return -EBUSY; + + switch (cmd) { + case SGX_IOC_ENCLAVE_CREATE: + ret = sgx_ioc_enclave_create(encl, (void __user *)arg); + break; + case SGX_IOC_ENCLAVE_ADD_PAGES: + ret = sgx_ioc_enclave_add_pages(encl, (void __user *)arg); + break; + case SGX_IOC_ENCLAVE_INIT: + ret = sgx_ioc_enclave_init(encl, (void __user *)arg); + break; + case SGX_IOC_ENCLAVE_PROVISION: + ret = sgx_ioc_enclave_provision(encl, (void __user *)arg); + break; + default: + ret = -ENOIOCTLCMD; + break; + } + + clear_bit(SGX_ENCL_IOCTL, &encl->flags); + return ret; +} diff --git a/arch/x86/kernel/cpu/sgx/main.c b/arch/x86/kernel/cpu/sgx/main.c new file mode 100644 index 000000000000..c519fc5f6948 --- /dev/null +++ b/arch/x86/kernel/cpu/sgx/main.c @@ -0,0 +1,733 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2016-20 Intel Corporation. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "driver.h" +#include "encl.h" +#include "encls.h" + +struct sgx_epc_section sgx_epc_sections[SGX_MAX_EPC_SECTIONS]; +static int sgx_nr_epc_sections; +static struct task_struct *ksgxd_tsk; +static DECLARE_WAIT_QUEUE_HEAD(ksgxd_waitq); + +/* + * These variables are part of the state of the reclaimer, and must be accessed + * with sgx_reclaimer_lock acquired. + */ +static LIST_HEAD(sgx_active_page_list); + +static DEFINE_SPINLOCK(sgx_reclaimer_lock); + +/* + * Reset dirty EPC pages to uninitialized state. Laundry can be left with SECS + * pages whose child pages blocked EREMOVE. + */ +static void sgx_sanitize_section(struct sgx_epc_section *section) +{ + struct sgx_epc_page *page; + LIST_HEAD(dirty); + int ret; + + /* init_laundry_list is thread-local, no need for a lock: */ + while (!list_empty(§ion->init_laundry_list)) { + if (kthread_should_stop()) + return; + + /* needed for access to ->page_list: */ + spin_lock(§ion->lock); + + page = list_first_entry(§ion->init_laundry_list, + struct sgx_epc_page, list); + + ret = __eremove(sgx_get_epc_virt_addr(page)); + if (!ret) + list_move(&page->list, §ion->page_list); + else + list_move_tail(&page->list, &dirty); + + spin_unlock(§ion->lock); + + cond_resched(); + } + + list_splice(&dirty, §ion->init_laundry_list); +} + +static bool sgx_reclaimer_age(struct sgx_epc_page *epc_page) +{ + struct sgx_encl_page *page = epc_page->owner; + struct sgx_encl *encl = page->encl; + struct sgx_encl_mm *encl_mm; + bool ret = true; + int idx; + + idx = srcu_read_lock(&encl->srcu); + + list_for_each_entry_rcu(encl_mm, &encl->mm_list, list) { + if (!mmget_not_zero(encl_mm->mm)) + continue; + + mmap_read_lock(encl_mm->mm); + ret = !sgx_encl_test_and_clear_young(encl_mm->mm, page); + mmap_read_unlock(encl_mm->mm); + + mmput_async(encl_mm->mm); + + if (!ret) + break; + } + + srcu_read_unlock(&encl->srcu, idx); + + if (!ret) + return false; + + return true; +} + +static void sgx_reclaimer_block(struct sgx_epc_page *epc_page) +{ + struct sgx_encl_page *page = epc_page->owner; + unsigned long addr = page->desc & PAGE_MASK; + struct sgx_encl *encl = page->encl; + unsigned long mm_list_version; + struct sgx_encl_mm *encl_mm; + struct vm_area_struct *vma; + int idx, ret; + + do { + mm_list_version = encl->mm_list_version; + + /* Pairs with smp_rmb() in sgx_encl_mm_add(). */ + smp_rmb(); + + idx = srcu_read_lock(&encl->srcu); + + list_for_each_entry_rcu(encl_mm, &encl->mm_list, list) { + if (!mmget_not_zero(encl_mm->mm)) + continue; + + mmap_read_lock(encl_mm->mm); + + ret = sgx_encl_find(encl_mm->mm, addr, &vma); + if (!ret && encl == vma->vm_private_data) + zap_vma_ptes(vma, addr, PAGE_SIZE); + + mmap_read_unlock(encl_mm->mm); + + mmput_async(encl_mm->mm); + } + + srcu_read_unlock(&encl->srcu, idx); + } while (unlikely(encl->mm_list_version != mm_list_version)); + + mutex_lock(&encl->lock); + + ret = __eblock(sgx_get_epc_virt_addr(epc_page)); + if (encls_failed(ret)) + ENCLS_WARN(ret, "EBLOCK"); + + mutex_unlock(&encl->lock); +} + +static int __sgx_encl_ewb(struct sgx_epc_page *epc_page, void *va_slot, + struct sgx_backing *backing) +{ + struct sgx_pageinfo pginfo; + int ret; + + pginfo.addr = 0; + pginfo.secs = 0; + + pginfo.contents = (unsigned long)kmap_atomic(backing->contents); + pginfo.metadata = (unsigned long)kmap_atomic(backing->pcmd) + + backing->pcmd_offset; + + ret = __ewb(&pginfo, sgx_get_epc_virt_addr(epc_page), va_slot); + + kunmap_atomic((void *)(unsigned long)(pginfo.metadata - + backing->pcmd_offset)); + kunmap_atomic((void *)(unsigned long)pginfo.contents); + + return ret; +} + +static void sgx_ipi_cb(void *info) +{ +} + +static const cpumask_t *sgx_encl_ewb_cpumask(struct sgx_encl *encl) +{ + cpumask_t *cpumask = &encl->cpumask; + struct sgx_encl_mm *encl_mm; + int idx; + + /* + * Can race with sgx_encl_mm_add(), but ETRACK has already been + * executed, which means that the CPUs running in the new mm will enter + * into the enclave with a fresh epoch. + */ + cpumask_clear(cpumask); + + idx = srcu_read_lock(&encl->srcu); + + list_for_each_entry_rcu(encl_mm, &encl->mm_list, list) { + if (!mmget_not_zero(encl_mm->mm)) + continue; + + cpumask_or(cpumask, cpumask, mm_cpumask(encl_mm->mm)); + + mmput_async(encl_mm->mm); + } + + srcu_read_unlock(&encl->srcu, idx); + + return cpumask; +} + +/* + * Swap page to the regular memory transformed to the blocked state by using + * EBLOCK, which means that it can no loger be referenced (no new TLB entries). + * + * The first trial just tries to write the page assuming that some other thread + * has reset the count for threads inside the enlave by using ETRACK, and + * previous thread count has been zeroed out. The second trial calls ETRACK + * before EWB. If that fails we kick all the HW threads out, and then do EWB, + * which should be guaranteed the succeed. + */ +static void sgx_encl_ewb(struct sgx_epc_page *epc_page, + struct sgx_backing *backing) +{ + struct sgx_encl_page *encl_page = epc_page->owner; + struct sgx_encl *encl = encl_page->encl; + struct sgx_va_page *va_page; + unsigned int va_offset; + void *va_slot; + int ret; + + encl_page->desc &= ~SGX_ENCL_PAGE_BEING_RECLAIMED; + + va_page = list_first_entry(&encl->va_pages, struct sgx_va_page, + list); + va_offset = sgx_alloc_va_slot(va_page); + va_slot = sgx_get_epc_virt_addr(va_page->epc_page) + va_offset; + if (sgx_va_page_full(va_page)) + list_move_tail(&va_page->list, &encl->va_pages); + + ret = __sgx_encl_ewb(epc_page, va_slot, backing); + if (ret == SGX_NOT_TRACKED) { + ret = __etrack(sgx_get_epc_virt_addr(encl->secs.epc_page)); + if (ret) { + if (encls_failed(ret)) + ENCLS_WARN(ret, "ETRACK"); + } + + ret = __sgx_encl_ewb(epc_page, va_slot, backing); + if (ret == SGX_NOT_TRACKED) { + /* + * Slow path, send IPIs to kick cpus out of the + * enclave. Note, it's imperative that the cpu + * mask is generated *after* ETRACK, else we'll + * miss cpus that entered the enclave between + * generating the mask and incrementing epoch. + */ + on_each_cpu_mask(sgx_encl_ewb_cpumask(encl), + sgx_ipi_cb, NULL, 1); + ret = __sgx_encl_ewb(epc_page, va_slot, backing); + } + } + + if (ret) { + if (encls_failed(ret)) + ENCLS_WARN(ret, "EWB"); + + sgx_free_va_slot(va_page, va_offset); + } else { + encl_page->desc |= va_offset; + encl_page->va_page = va_page; + } +} + +static void sgx_reclaimer_write(struct sgx_epc_page *epc_page, + struct sgx_backing *backing) +{ + struct sgx_encl_page *encl_page = epc_page->owner; + struct sgx_encl *encl = encl_page->encl; + struct sgx_backing secs_backing; + int ret; + + mutex_lock(&encl->lock); + + sgx_encl_ewb(epc_page, backing); + encl_page->epc_page = NULL; + encl->secs_child_cnt--; + + if (!encl->secs_child_cnt && test_bit(SGX_ENCL_INITIALIZED, &encl->flags)) { + ret = sgx_encl_get_backing(encl, PFN_DOWN(encl->size), + &secs_backing); + if (ret) + goto out; + + sgx_encl_ewb(encl->secs.epc_page, &secs_backing); + + sgx_free_epc_page(encl->secs.epc_page); + encl->secs.epc_page = NULL; + + sgx_encl_put_backing(&secs_backing, true); + } + +out: + mutex_unlock(&encl->lock); +} + +/* + * Take a fixed number of pages from the head of the active page pool and + * reclaim them to the enclave's private shmem files. Skip the pages, which have + * been accessed since the last scan. Move those pages to the tail of active + * page pool so that the pages get scanned in LRU like fashion. + * + * Batch process a chunk of pages (at the moment 16) in order to degrade amount + * of IPI's and ETRACK's potentially required. sgx_encl_ewb() does degrade a bit + * among the HW threads with three stage EWB pipeline (EWB, ETRACK + EWB and IPI + * + EWB) but not sufficiently. Reclaiming one page at a time would also be + * problematic as it would increase the lock contention too much, which would + * halt forward progress. + */ +static void sgx_reclaim_pages(void) +{ + struct sgx_epc_page *chunk[SGX_NR_TO_SCAN]; + struct sgx_backing backing[SGX_NR_TO_SCAN]; + struct sgx_epc_section *section; + struct sgx_encl_page *encl_page; + struct sgx_epc_page *epc_page; + pgoff_t page_index; + int cnt = 0; + int ret; + int i; + + spin_lock(&sgx_reclaimer_lock); + for (i = 0; i < SGX_NR_TO_SCAN; i++) { + if (list_empty(&sgx_active_page_list)) + break; + + epc_page = list_first_entry(&sgx_active_page_list, + struct sgx_epc_page, list); + list_del_init(&epc_page->list); + encl_page = epc_page->owner; + + if (kref_get_unless_zero(&encl_page->encl->refcount) != 0) + chunk[cnt++] = epc_page; + else + /* The owner is freeing the page. No need to add the + * page back to the list of reclaimable pages. + */ + epc_page->flags &= ~SGX_EPC_PAGE_RECLAIMER_TRACKED; + } + spin_unlock(&sgx_reclaimer_lock); + + for (i = 0; i < cnt; i++) { + epc_page = chunk[i]; + encl_page = epc_page->owner; + + if (!sgx_reclaimer_age(epc_page)) + goto skip; + + page_index = PFN_DOWN(encl_page->desc - encl_page->encl->base); + ret = sgx_encl_get_backing(encl_page->encl, page_index, &backing[i]); + if (ret) + goto skip; + + mutex_lock(&encl_page->encl->lock); + encl_page->desc |= SGX_ENCL_PAGE_BEING_RECLAIMED; + mutex_unlock(&encl_page->encl->lock); + continue; + +skip: + spin_lock(&sgx_reclaimer_lock); + list_add_tail(&epc_page->list, &sgx_active_page_list); + spin_unlock(&sgx_reclaimer_lock); + + kref_put(&encl_page->encl->refcount, sgx_encl_release); + + chunk[i] = NULL; + } + + for (i = 0; i < cnt; i++) { + epc_page = chunk[i]; + if (epc_page) + sgx_reclaimer_block(epc_page); + } + + for (i = 0; i < cnt; i++) { + epc_page = chunk[i]; + if (!epc_page) + continue; + + encl_page = epc_page->owner; + sgx_reclaimer_write(epc_page, &backing[i]); + sgx_encl_put_backing(&backing[i], true); + + kref_put(&encl_page->encl->refcount, sgx_encl_release); + epc_page->flags &= ~SGX_EPC_PAGE_RECLAIMER_TRACKED; + + section = &sgx_epc_sections[epc_page->section]; + spin_lock(§ion->lock); + list_add_tail(&epc_page->list, §ion->page_list); + section->free_cnt++; + spin_unlock(§ion->lock); + } +} + +static unsigned long sgx_nr_free_pages(void) +{ + unsigned long cnt = 0; + int i; + + for (i = 0; i < sgx_nr_epc_sections; i++) + cnt += sgx_epc_sections[i].free_cnt; + + return cnt; +} + +static bool sgx_should_reclaim(unsigned long watermark) +{ + return sgx_nr_free_pages() < watermark && + !list_empty(&sgx_active_page_list); +} + +static int ksgxd(void *p) +{ + int i; + + set_freezable(); + + /* + * Sanitize pages in order to recover from kexec(). The 2nd pass is + * required for SECS pages, whose child pages blocked EREMOVE. + */ + for (i = 0; i < sgx_nr_epc_sections; i++) + sgx_sanitize_section(&sgx_epc_sections[i]); + + for (i = 0; i < sgx_nr_epc_sections; i++) { + sgx_sanitize_section(&sgx_epc_sections[i]); + + /* Should never happen. */ + if (!list_empty(&sgx_epc_sections[i].init_laundry_list)) + WARN(1, "EPC section %d has unsanitized pages.\n", i); + } + + while (!kthread_should_stop()) { + if (try_to_freeze()) + continue; + + wait_event_freezable(ksgxd_waitq, + kthread_should_stop() || + sgx_should_reclaim(SGX_NR_HIGH_PAGES)); + + if (sgx_should_reclaim(SGX_NR_HIGH_PAGES)) + sgx_reclaim_pages(); + + cond_resched(); + } + + return 0; +} + +static bool __init sgx_page_reclaimer_init(void) +{ + struct task_struct *tsk; + + tsk = kthread_run(ksgxd, NULL, "ksgxd"); + if (IS_ERR(tsk)) + return false; + + ksgxd_tsk = tsk; + + return true; +} + +static struct sgx_epc_page *__sgx_alloc_epc_page_from_section(struct sgx_epc_section *section) +{ + struct sgx_epc_page *page; + + spin_lock(§ion->lock); + + if (list_empty(§ion->page_list)) { + spin_unlock(§ion->lock); + return NULL; + } + + page = list_first_entry(§ion->page_list, struct sgx_epc_page, list); + list_del_init(&page->list); + section->free_cnt--; + + spin_unlock(§ion->lock); + return page; +} + +/** + * __sgx_alloc_epc_page() - Allocate an EPC page + * + * Iterate through EPC sections and borrow a free EPC page to the caller. When a + * page is no longer needed it must be released with sgx_free_epc_page(). + * + * Return: + * an EPC page, + * -errno on error + */ +struct sgx_epc_page *__sgx_alloc_epc_page(void) +{ + struct sgx_epc_section *section; + struct sgx_epc_page *page; + int i; + + for (i = 0; i < sgx_nr_epc_sections; i++) { + section = &sgx_epc_sections[i]; + + page = __sgx_alloc_epc_page_from_section(section); + if (page) + return page; + } + + return ERR_PTR(-ENOMEM); +} + +/** + * sgx_mark_page_reclaimable() - Mark a page as reclaimable + * @page: EPC page + * + * Mark a page as reclaimable and add it to the active page list. Pages + * are automatically removed from the active list when freed. + */ +void sgx_mark_page_reclaimable(struct sgx_epc_page *page) +{ + spin_lock(&sgx_reclaimer_lock); + page->flags |= SGX_EPC_PAGE_RECLAIMER_TRACKED; + list_add_tail(&page->list, &sgx_active_page_list); + spin_unlock(&sgx_reclaimer_lock); +} + +/** + * sgx_unmark_page_reclaimable() - Remove a page from the reclaim list + * @page: EPC page + * + * Clear the reclaimable flag and remove the page from the active page list. + * + * Return: + * 0 on success, + * -EBUSY if the page is in the process of being reclaimed + */ +int sgx_unmark_page_reclaimable(struct sgx_epc_page *page) +{ + spin_lock(&sgx_reclaimer_lock); + if (page->flags & SGX_EPC_PAGE_RECLAIMER_TRACKED) { + /* The page is being reclaimed. */ + if (list_empty(&page->list)) { + spin_unlock(&sgx_reclaimer_lock); + return -EBUSY; + } + + list_del(&page->list); + page->flags &= ~SGX_EPC_PAGE_RECLAIMER_TRACKED; + } + spin_unlock(&sgx_reclaimer_lock); + + return 0; +} + +/** + * sgx_alloc_epc_page() - Allocate an EPC page + * @owner: the owner of the EPC page + * @reclaim: reclaim pages if necessary + * + * Iterate through EPC sections and borrow a free EPC page to the caller. When a + * page is no longer needed it must be released with sgx_free_epc_page(). If + * @reclaim is set to true, directly reclaim pages when we are out of pages. No + * mm's can be locked when @reclaim is set to true. + * + * Finally, wake up ksgxd when the number of pages goes below the watermark + * before returning back to the caller. + * + * Return: + * an EPC page, + * -errno on error + */ +struct sgx_epc_page *sgx_alloc_epc_page(void *owner, bool reclaim) +{ + struct sgx_epc_page *page; + + for ( ; ; ) { + page = __sgx_alloc_epc_page(); + if (!IS_ERR(page)) { + page->owner = owner; + break; + } + + if (list_empty(&sgx_active_page_list)) + return ERR_PTR(-ENOMEM); + + if (!reclaim) { + page = ERR_PTR(-EBUSY); + break; + } + + if (signal_pending(current)) { + page = ERR_PTR(-ERESTARTSYS); + break; + } + + sgx_reclaim_pages(); + cond_resched(); + } + + if (sgx_should_reclaim(SGX_NR_LOW_PAGES)) + wake_up(&ksgxd_waitq); + + return page; +} + +/** + * sgx_free_epc_page() - Free an EPC page + * @page: an EPC page + * + * Call EREMOVE for an EPC page and insert it back to the list of free pages. + */ +void sgx_free_epc_page(struct sgx_epc_page *page) +{ + struct sgx_epc_section *section = &sgx_epc_sections[page->section]; + int ret; + + WARN_ON_ONCE(page->flags & SGX_EPC_PAGE_RECLAIMER_TRACKED); + + ret = __eremove(sgx_get_epc_virt_addr(page)); + if (WARN_ONCE(ret, "EREMOVE returned %d (0x%x)", ret, ret)) + return; + + spin_lock(§ion->lock); + list_add_tail(&page->list, §ion->page_list); + section->free_cnt++; + spin_unlock(§ion->lock); +} + +static bool __init sgx_setup_epc_section(u64 phys_addr, u64 size, + unsigned long index, + struct sgx_epc_section *section) +{ + unsigned long nr_pages = size >> PAGE_SHIFT; + unsigned long i; + + section->virt_addr = memremap(phys_addr, size, MEMREMAP_WB); + if (!section->virt_addr) + return false; + + section->pages = vmalloc(nr_pages * sizeof(struct sgx_epc_page)); + if (!section->pages) { + memunmap(section->virt_addr); + return false; + } + + section->phys_addr = phys_addr; + spin_lock_init(§ion->lock); + INIT_LIST_HEAD(§ion->page_list); + INIT_LIST_HEAD(§ion->init_laundry_list); + + for (i = 0; i < nr_pages; i++) { + section->pages[i].section = index; + section->pages[i].flags = 0; + section->pages[i].owner = NULL; + list_add_tail(§ion->pages[i].list, §ion->init_laundry_list); + } + + section->free_cnt = nr_pages; + return true; +} + +/** + * A section metric is concatenated in a way that @low bits 12-31 define the + * bits 12-31 of the metric and @high bits 0-19 define the bits 32-51 of the + * metric. + */ +static inline u64 __init sgx_calc_section_metric(u64 low, u64 high) +{ + return (low & GENMASK_ULL(31, 12)) + + ((high & GENMASK_ULL(19, 0)) << 32); +} + +static bool __init sgx_page_cache_init(void) +{ + u32 eax, ebx, ecx, edx, type; + u64 pa, size; + int i; + + for (i = 0; i < ARRAY_SIZE(sgx_epc_sections); i++) { + cpuid_count(SGX_CPUID, i + SGX_CPUID_EPC, &eax, &ebx, &ecx, &edx); + + type = eax & SGX_CPUID_EPC_MASK; + if (type == SGX_CPUID_EPC_INVALID) + break; + + if (type != SGX_CPUID_EPC_SECTION) { + pr_err_once("Unknown EPC section type: %u\n", type); + break; + } + + pa = sgx_calc_section_metric(eax, ebx); + size = sgx_calc_section_metric(ecx, edx); + + pr_info("EPC section 0x%llx-0x%llx\n", pa, pa + size - 1); + + if (!sgx_setup_epc_section(pa, size, i, &sgx_epc_sections[i])) { + pr_err("No free memory for an EPC section\n"); + break; + } + + sgx_nr_epc_sections++; + } + + if (!sgx_nr_epc_sections) { + pr_err("There are zero EPC sections.\n"); + return false; + } + + return true; +} + +static void __init sgx_init(void) +{ + int ret; + int i; + + if (!cpu_feature_enabled(X86_FEATURE_SGX)) + return; + + if (!sgx_page_cache_init()) + return; + + if (!sgx_page_reclaimer_init()) + goto err_page_cache; + + ret = sgx_drv_init(); + if (ret) + goto err_kthread; + + return; + +err_kthread: + kthread_stop(ksgxd_tsk); + +err_page_cache: + for (i = 0; i < sgx_nr_epc_sections; i++) { + vfree(sgx_epc_sections[i].pages); + memunmap(sgx_epc_sections[i].virt_addr); + } +} + +device_initcall(sgx_init); diff --git a/arch/x86/kernel/cpu/sgx/sgx.h b/arch/x86/kernel/cpu/sgx/sgx.h new file mode 100644 index 000000000000..5fa42d143feb --- /dev/null +++ b/arch/x86/kernel/cpu/sgx/sgx.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _X86_SGX_H +#define _X86_SGX_H + +#include +#include +#include +#include +#include +#include +#include "arch.h" + +#undef pr_fmt +#define pr_fmt(fmt) "sgx: " fmt + +#define SGX_MAX_EPC_SECTIONS 8 +#define SGX_EEXTEND_BLOCK_SIZE 256 +#define SGX_NR_TO_SCAN 16 +#define SGX_NR_LOW_PAGES 32 +#define SGX_NR_HIGH_PAGES 64 + +/* Pages, which are being tracked by the page reclaimer. */ +#define SGX_EPC_PAGE_RECLAIMER_TRACKED BIT(0) + +struct sgx_epc_page { + unsigned int section; + unsigned int flags; + struct sgx_encl_page *owner; + struct list_head list; +}; + +/* + * The firmware can define multiple chunks of EPC to the different areas of the + * physical memory e.g. for memory areas of the each node. This structure is + * used to store EPC pages for one EPC section and virtual memory area where + * the pages have been mapped. + * + * 'lock' must be held before accessing 'page_list' or 'free_cnt'. + */ +struct sgx_epc_section { + unsigned long phys_addr; + void *virt_addr; + struct sgx_epc_page *pages; + + spinlock_t lock; + struct list_head page_list; + unsigned long free_cnt; + + /* + * Pages which need EREMOVE run on them before they can be + * used. Only safe to be accessed in ksgxd and init code. + * Not protected by locks. + */ + struct list_head init_laundry_list; +}; + +extern struct sgx_epc_section sgx_epc_sections[SGX_MAX_EPC_SECTIONS]; + +static inline unsigned long sgx_get_epc_phys_addr(struct sgx_epc_page *page) +{ + struct sgx_epc_section *section = &sgx_epc_sections[page->section]; + unsigned long index; + + index = ((unsigned long)page - (unsigned long)section->pages) / sizeof(*page); + + return section->phys_addr + index * PAGE_SIZE; +} + +static inline void *sgx_get_epc_virt_addr(struct sgx_epc_page *page) +{ + struct sgx_epc_section *section = &sgx_epc_sections[page->section]; + unsigned long index; + + index = ((unsigned long)page - (unsigned long)section->pages) / sizeof(*page); + + return section->virt_addr + index * PAGE_SIZE; +} + +struct sgx_epc_page *__sgx_alloc_epc_page(void); +void sgx_free_epc_page(struct sgx_epc_page *page); + +void sgx_mark_page_reclaimable(struct sgx_epc_page *page); +int sgx_unmark_page_reclaimable(struct sgx_epc_page *page); +struct sgx_epc_page *sgx_alloc_epc_page(void *owner, bool reclaim); + +#endif /* _X86_SGX_H */ diff --git a/arch/x86/kernel/cpu/topology.c b/arch/x86/kernel/cpu/topology.c index d3a0791bc052..1068002c8532 100644 --- a/arch/x86/kernel/cpu/topology.c +++ b/arch/x86/kernel/cpu/topology.c @@ -96,6 +96,7 @@ int detect_extended_topology(struct cpuinfo_x86 *c) unsigned int ht_mask_width, core_plus_mask_width, die_plus_mask_width; unsigned int core_select_mask, core_level_siblings; unsigned int die_select_mask, die_level_siblings; + bool die_level_present = false; int leaf; leaf = detect_extended_topology_leaf(c); @@ -126,6 +127,7 @@ int detect_extended_topology(struct cpuinfo_x86 *c) die_plus_mask_width = BITS_SHIFT_NEXT_LEVEL(eax); } if (LEAFB_SUBTYPE(ecx) == DIE_TYPE) { + die_level_present = true; die_level_siblings = LEVEL_MAX_SIBLINGS(ebx); die_plus_mask_width = BITS_SHIFT_NEXT_LEVEL(eax); } @@ -139,8 +141,12 @@ int detect_extended_topology(struct cpuinfo_x86 *c) c->cpu_core_id = apic->phys_pkg_id(c->initial_apicid, ht_mask_width) & core_select_mask; - c->cpu_die_id = apic->phys_pkg_id(c->initial_apicid, - core_plus_mask_width) & die_select_mask; + + if (die_level_present) { + c->cpu_die_id = apic->phys_pkg_id(c->initial_apicid, + core_plus_mask_width) & die_select_mask; + } + c->phys_proc_id = apic->phys_pkg_id(c->initial_apicid, die_plus_mask_width); /* diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c index 97aa900386cb..299c20f0a38b 100644 --- a/arch/x86/kernel/dumpstack.c +++ b/arch/x86/kernel/dumpstack.c @@ -183,7 +183,7 @@ static void show_regs_if_on_stack(struct stack_info *info, struct pt_regs *regs, } } -void show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs, +static void show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs, unsigned long *stack, const char *log_lvl) { struct unwind_state state; diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index 05e117137b45..5e9beb77cafd 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -37,7 +37,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index 3c417734790f..04bddaaba8e2 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -26,15 +26,6 @@ #include #include -#ifdef CONFIG_PARAVIRT_XXL -#include -#include -#define GET_CR2_INTO(reg) GET_CR2_INTO_AX ; _ASM_MOV %_ASM_AX, reg -#else -#define INTERRUPT_RETURN iretq -#define GET_CR2_INTO(reg) _ASM_MOV %cr2, reg -#endif - /* * We are not able to switch in one step to the final KERNEL ADDRESS SPACE * because we need identity-mapped pages. @@ -540,21 +531,19 @@ SYM_DATA_END(level3_kernel_pgt) SYM_DATA_START_PAGE_ALIGNED(level2_kernel_pgt) /* - * 512 MB kernel mapping. We spend a full page on this pagetable - * anyway. + * Kernel high mapping. * - * The kernel code+data+bss must not be bigger than that. + * The kernel code+data+bss must be located below KERNEL_IMAGE_SIZE in + * virtual address space, which is 1 GiB if RANDOMIZE_BASE is enabled, + * 512 MiB otherwise. * - * (NOTE: at +512MB starts the module area, see MODULES_VADDR. - * If you want to increase this then increase MODULES_VADDR - * too.) + * (NOTE: after that starts the module area, see MODULES_VADDR.) * - * This table is eventually used by the kernel during normal - * runtime. Care must be taken to clear out undesired bits - * later, like _PAGE_RW or _PAGE_GLOBAL in some cases. + * This table is eventually used by the kernel during normal runtime. + * Care must be taken to clear out undesired bits later, like _PAGE_RW + * or _PAGE_GLOBAL in some cases. */ - PMDS(0, __PAGE_KERNEL_LARGE_EXEC, - KERNEL_IMAGE_SIZE/PMD_SIZE) + PMDS(0, __PAGE_KERNEL_LARGE_EXEC, KERNEL_IMAGE_SIZE/PMD_SIZE) SYM_DATA_END(level2_kernel_pgt) SYM_DATA_START_PAGE_ALIGNED(level2_fixmap_pgt) diff --git a/arch/x86/kernel/msr.c b/arch/x86/kernel/msr.c index c0d409810658..8a67d1fa8dc5 100644 --- a/arch/x86/kernel/msr.c +++ b/arch/x86/kernel/msr.c @@ -99,11 +99,9 @@ static int filter_write(u32 reg) if (!__ratelimit(&fw_rs)) return 0; - if (reg == MSR_IA32_ENERGY_PERF_BIAS) - return 0; - - pr_err("Write to unrecognized MSR 0x%x by %s (pid: %d). Please report to x86@kernel.org.\n", - reg, current->comm, current->pid); + pr_warn("Write to unrecognized MSR 0x%x by %s (pid: %d).\n", + reg, current->comm, current->pid); + pr_warn("See https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git/about for details.\n"); return 0; } diff --git a/arch/x86/kernel/perf_regs.c b/arch/x86/kernel/perf_regs.c index f9e5352b3bef..624703af80a1 100644 --- a/arch/x86/kernel/perf_regs.c +++ b/arch/x86/kernel/perf_regs.c @@ -122,7 +122,7 @@ int perf_reg_validate(u64 mask) u64 perf_reg_abi(struct task_struct *task) { - if (test_tsk_thread_flag(task, TIF_IA32)) + if (!user_64bit_mode(task_pt_regs(task))) return PERF_SAMPLE_REGS_ABI_32; else return PERF_SAMPLE_REGS_ABI_64; diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index df342bedea88..ad582f9ac5a6 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -511,11 +511,10 @@ start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp) EXPORT_SYMBOL_GPL(start_thread); #ifdef CONFIG_COMPAT -void compat_start_thread(struct pt_regs *regs, u32 new_ip, u32 new_sp) +void compat_start_thread(struct pt_regs *regs, u32 new_ip, u32 new_sp, bool x32) { start_thread_common(regs, new_ip, new_sp, - test_thread_flag(TIF_X32) - ? __USER_CS : __USER32_CS, + x32 ? __USER_CS : __USER32_CS, __USER_DS, __USER_DS); } #endif @@ -641,16 +640,12 @@ void set_personality_64bit(void) /* inherit personality from parent */ /* Make sure to be in 64bit mode */ - clear_thread_flag(TIF_IA32); clear_thread_flag(TIF_ADDR32); - clear_thread_flag(TIF_X32); /* Pretend that this comes from a 64bit execve */ task_pt_regs(current)->orig_ax = __NR_execve; current_thread_info()->status &= ~TS_COMPAT; - - /* Ensure the corresponding mm is not marked. */ if (current->mm) - current->mm->context.ia32_compat = 0; + current->mm->context.flags = MM_CONTEXT_HAS_VSYSCALL; /* TBD: overwrites user setup. Should have two bits. But 64bit processes have always behaved this way, @@ -662,10 +657,9 @@ void set_personality_64bit(void) static void __set_personality_x32(void) { #ifdef CONFIG_X86_X32 - clear_thread_flag(TIF_IA32); - set_thread_flag(TIF_X32); if (current->mm) - current->mm->context.ia32_compat = TIF_X32; + current->mm->context.flags = 0; + current->personality &= ~READ_IMPLIES_EXEC; /* * in_32bit_syscall() uses the presence of the x32 syscall bit @@ -683,10 +677,14 @@ static void __set_personality_x32(void) static void __set_personality_ia32(void) { #ifdef CONFIG_IA32_EMULATION - set_thread_flag(TIF_IA32); - clear_thread_flag(TIF_X32); - if (current->mm) - current->mm->context.ia32_compat = TIF_IA32; + if (current->mm) { + /* + * uprobes applied to this MM need to know this and + * cannot use user_64bit_mode() at that time. + */ + current->mm->context.flags = MM_CONTEXT_UPROBE_IA32; + } + current->personality |= force_personality32; /* Prepare the first "return" to user space */ task_pt_regs(current)->orig_ax = __NR_ia32_execve; diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index 84f581c91db4..a23130c86bdd 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -119,11 +119,6 @@ EXPORT_SYMBOL(boot_cpu_data); unsigned int def_to_bigsmp; -/* For MCA, but anyone else can use it if they want */ -unsigned int machine_id; -unsigned int machine_submodel_id; -unsigned int BIOS_revision; - struct apm_info apm_info; EXPORT_SYMBOL(apm_info); diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index e19df6cde35d..7798d862983f 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -60,6 +60,7 @@ #include #include #include +#include #ifdef CONFIG_X86_64 #include @@ -117,6 +118,9 @@ do_trap_no_signal(struct task_struct *tsk, int trapnr, const char *str, tsk->thread.error_code = error_code; tsk->thread.trap_nr = trapnr; die(str, regs, error_code); + } else { + if (fixup_vdso_exception(regs, trapnr, error_code, 0)) + return 0; } /* @@ -550,6 +554,9 @@ DEFINE_IDTENTRY_ERRORCODE(exc_general_protection) tsk->thread.error_code = error_code; tsk->thread.trap_nr = X86_TRAP_GP; + if (fixup_vdso_exception(regs, X86_TRAP_GP, error_code, 0)) + return; + show_signal(tsk, SIGSEGV, "", desc, regs, error_code); force_sig(SIGSEGV); goto exit; @@ -1048,6 +1055,9 @@ static void math_error(struct pt_regs *regs, int trapnr) if (!si_code) goto exit; + if (fixup_vdso_exception(regs, trapnr, 0, 0)) + return; + force_sig_fault(SIGFPE, si_code, (void __user *)uprobe_get_trap_addr(regs)); exit: diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index bf9e0adb5b7e..efd9e9ea17f2 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -454,13 +454,13 @@ SECTIONS ASSERT(SIZEOF(.rela.dyn) == 0, "Unexpected run-time relocations (.rela) detected!") } -#ifdef CONFIG_X86_32 /* * The ASSERT() sink to . is intentional, for binutils 2.14 compatibility: */ . = ASSERT((_end - LOAD_OFFSET <= KERNEL_IMAGE_SIZE), "kernel image bigger than KERNEL_IMAGE_SIZE"); -#else + +#ifdef CONFIG_X86_64 /* * Per-cpu symbols which need to be offset from __per_cpu_load * for the boot processor. @@ -470,18 +470,12 @@ INIT_PER_CPU(gdt_page); INIT_PER_CPU(fixed_percpu_data); INIT_PER_CPU(irq_stack_backing_store); -/* - * Build-time check on the image size: - */ -. = ASSERT((_end - _text <= KERNEL_IMAGE_SIZE), - "kernel image bigger than KERNEL_IMAGE_SIZE"); - #ifdef CONFIG_SMP . = ASSERT((fixed_percpu_data == 0), "fixed_percpu_data is not at start of per-cpu area"); #endif -#endif /* CONFIG_X86_32 */ +#endif /* CONFIG_X86_64 */ #ifdef CONFIG_KEXEC_CORE #include diff --git a/arch/x86/lib/copy_mc.c b/arch/x86/lib/copy_mc.c index c13e8c9ee926..80efd45a7761 100644 --- a/arch/x86/lib/copy_mc.c +++ b/arch/x86/lib/copy_mc.c @@ -10,10 +10,6 @@ #include #ifdef CONFIG_X86_MCE -/* - * See COPY_MC_TEST for self-test of the copy_mc_fragile() - * implementation. - */ static DEFINE_STATIC_KEY_FALSE(copy_mc_fragile_key); void enable_copy_mc_fragile(void) diff --git a/arch/x86/lib/copy_mc_64.S b/arch/x86/lib/copy_mc_64.S index 892d8915f609..e5f77e293034 100644 --- a/arch/x86/lib/copy_mc_64.S +++ b/arch/x86/lib/copy_mc_64.S @@ -2,14 +2,11 @@ /* Copyright(c) 2016-2020 Intel Corporation. All rights reserved. */ #include -#include -#include #include #ifndef CONFIG_UML #ifdef CONFIG_X86_MCE -COPY_MC_TEST_CTL /* * copy_mc_fragile - copy memory with indication if an exception / fault happened @@ -38,8 +35,6 @@ SYM_FUNC_START(copy_mc_fragile) subl %ecx, %edx .L_read_leading_bytes: movb (%rsi), %al - COPY_MC_TEST_SRC %rsi 1 .E_leading_bytes - COPY_MC_TEST_DST %rdi 1 .E_leading_bytes .L_write_leading_bytes: movb %al, (%rdi) incq %rsi @@ -55,8 +50,6 @@ SYM_FUNC_START(copy_mc_fragile) .L_read_words: movq (%rsi), %r8 - COPY_MC_TEST_SRC %rsi 8 .E_read_words - COPY_MC_TEST_DST %rdi 8 .E_write_words .L_write_words: movq %r8, (%rdi) addq $8, %rsi @@ -73,8 +66,6 @@ SYM_FUNC_START(copy_mc_fragile) movl %edx, %ecx .L_read_trailing_bytes: movb (%rsi), %al - COPY_MC_TEST_SRC %rsi 1 .E_trailing_bytes - COPY_MC_TEST_DST %rdi 1 .E_trailing_bytes .L_write_trailing_bytes: movb %al, (%rdi) incq %rsi @@ -88,7 +79,6 @@ SYM_FUNC_START(copy_mc_fragile) .L_done: ret SYM_FUNC_END(copy_mc_fragile) -EXPORT_SYMBOL_GPL(copy_mc_fragile) .section .fixup, "ax" /* diff --git a/arch/x86/lib/usercopy.c b/arch/x86/lib/usercopy.c index 3f435d7fca5e..c3e8a62ca561 100644 --- a/arch/x86/lib/usercopy.c +++ b/arch/x86/lib/usercopy.c @@ -9,9 +9,23 @@ #include -/* - * We rely on the nested NMI work to allow atomic faults from the NMI path; the - * nested NMI paths are careful to preserve CR2. +/** + * copy_from_user_nmi - NMI safe copy from user + * @to: Pointer to the destination buffer + * @from: Pointer to a user space address of the current task + * @n: Number of bytes to copy + * + * Returns: The number of not copied bytes. 0 is success, i.e. all bytes copied + * + * Contrary to other copy_from_user() variants this function can be called + * from NMI context. Despite the name it is not restricted to be called + * from NMI context. It is safe to be called from any other context as + * well. It disables pagefaults across the copy which means a fault will + * abort the copy. + * + * For NMI context invocations this relies on the nested NMI work to allow + * atomic faults from the NMI path; the nested NMI paths are careful to + * preserve CR2. */ unsigned long copy_from_user_nmi(void *to, const void __user *from, unsigned long n) @@ -27,7 +41,7 @@ copy_from_user_nmi(void *to, const void __user *from, unsigned long n) /* * Even though this function is typically called from NMI/IRQ context * disable pagefaults so that its behaviour is consistent even when - * called form other contexts. + * called from other contexts. */ pagefault_disable(); ret = __copy_from_user_inatomic(to, from, n); diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 82bf37a5c9ec..f1f1b5a0956a 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -30,6 +30,7 @@ #include /* exception stack */ #include /* VMALLOC_START, ... */ #include /* kvm_handle_async_pf */ +#include /* fixup_vdso_exception() */ #define CREATE_TRACE_POINTS #include @@ -602,11 +603,9 @@ pgtable_bad(struct pt_regs *regs, unsigned long error_code, oops_end(flags, regs, sig); } -static void set_signal_archinfo(unsigned long address, - unsigned long error_code) +static void sanitize_error_code(unsigned long address, + unsigned long *error_code) { - struct task_struct *tsk = current; - /* * To avoid leaking information about the kernel page * table layout, pretend that user-mode accesses to @@ -617,7 +616,13 @@ static void set_signal_archinfo(unsigned long address, * information and does not appear to cause any problems. */ if (address >= TASK_SIZE_MAX) - error_code |= X86_PF_PROT; + *error_code |= X86_PF_PROT; +} + +static void set_signal_archinfo(unsigned long address, + unsigned long error_code) +{ + struct task_struct *tsk = current; tsk->thread.trap_nr = X86_TRAP_PF; tsk->thread.error_code = error_code | X86_PF_USER; @@ -658,6 +663,8 @@ no_context(struct pt_regs *regs, unsigned long error_code, * faulting through the emulate_vsyscall() logic. */ if (current->thread.sig_on_uaccess_err && signal) { + sanitize_error_code(address, &error_code); + set_signal_archinfo(address, error_code); /* XXX: hwpoison faults will set the wrong code. */ @@ -806,13 +813,10 @@ __bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code, if (is_errata100(regs, address)) return; - /* - * To avoid leaking information about the kernel page table - * layout, pretend that user-mode accesses to kernel addresses - * are always protection faults. - */ - if (address >= TASK_SIZE_MAX) - error_code |= X86_PF_PROT; + sanitize_error_code(address, &error_code); + + if (fixup_vdso_exception(regs, X86_TRAP_PF, error_code, address)) + return; if (likely(show_unhandled_signals)) show_signal_msg(regs, error_code, address, tsk); @@ -931,6 +935,11 @@ do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address, if (is_prefetch(regs, error_code, address)) return; + sanitize_error_code(address, &error_code); + + if (fixup_vdso_exception(regs, X86_TRAP_PF, error_code, address)) + return; + set_signal_archinfo(address, error_code); #ifdef CONFIG_MEMORY_FAILURE @@ -1101,6 +1110,18 @@ access_error(unsigned long error_code, struct vm_area_struct *vma) if (error_code & X86_PF_PK) return 1; + /* + * SGX hardware blocked the access. This usually happens + * when the enclave memory contents have been destroyed, like + * after a suspend/resume cycle. In any case, the kernel can't + * fix the cause of the fault. Handle the fault as an access + * error even in cases where no actual access violation + * occurred. This allows userspace to rebuild the enclave in + * response to the signal. + */ + if (unlikely(error_code & X86_PF_SGX)) + return 1; + /* * Make sure to check the VMA so that we do not perform * faults just to hit a X86_PF_PK as soon as we fill in a diff --git a/arch/x86/mm/ident_map.c b/arch/x86/mm/ident_map.c index fe7a12599d8e..968d7005f4a7 100644 --- a/arch/x86/mm/ident_map.c +++ b/arch/x86/mm/ident_map.c @@ -62,6 +62,7 @@ static int ident_p4d_init(struct x86_mapping_info *info, p4d_t *p4d_page, unsigned long addr, unsigned long end) { unsigned long next; + int result; for (; addr < end; addr = next) { p4d_t *p4d = p4d_page + p4d_index(addr); @@ -73,13 +74,20 @@ static int ident_p4d_init(struct x86_mapping_info *info, p4d_t *p4d_page, if (p4d_present(*p4d)) { pud = pud_offset(p4d, 0); - ident_pud_init(info, pud, addr, next); + result = ident_pud_init(info, pud, addr, next); + if (result) + return result; + continue; } pud = (pud_t *)info->alloc_pgt_page(info->context); if (!pud) return -ENOMEM; - ident_pud_init(info, pud, addr, next); + + result = ident_pud_init(info, pud, addr, next); + if (result) + return result; + set_p4d(p4d, __p4d(__pa(pud) | info->kernpg_flag)); } diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c index c7a47603537f..e26f5c5c6565 100644 --- a/arch/x86/mm/init.c +++ b/arch/x86/mm/init.c @@ -596,7 +596,7 @@ static unsigned long __init get_new_step_size(unsigned long step_size) static void __init memory_map_top_down(unsigned long map_start, unsigned long map_end) { - unsigned long real_end, start, last_start; + unsigned long real_end, last_start; unsigned long step_size; unsigned long addr; unsigned long mapped_ram_size = 0; @@ -609,7 +609,7 @@ static void __init memory_map_top_down(unsigned long map_start, step_size = PMD_SIZE; max_pfn_mapped = 0; /* will get exact value next */ min_pfn_mapped = real_end >> PAGE_SHIFT; - last_start = start = real_end; + last_start = real_end; /* * We start from the top (end of memory) and go to the bottom. @@ -618,6 +618,8 @@ static void __init memory_map_top_down(unsigned long map_start, * for page table. */ while (last_start > map_start) { + unsigned long start; + if (last_start > step_size) { start = round_down(last_start - 1, step_size); if (start < map_start) diff --git a/arch/x86/oprofile/backtrace.c b/arch/x86/oprofile/backtrace.c index a2488b6e27d6..1d8391fcca68 100644 --- a/arch/x86/oprofile/backtrace.c +++ b/arch/x86/oprofile/backtrace.c @@ -49,7 +49,7 @@ x86_backtrace_32(struct pt_regs * const regs, unsigned int depth) struct stack_frame_ia32 *head; /* User process is IA32 */ - if (!current || !test_thread_flag(TIF_IA32)) + if (!current || user_64bit_mode(regs)) return 0; head = (struct stack_frame_ia32 *) regs->bp; diff --git a/arch/x86/pci/i386.c b/arch/x86/pci/i386.c index fa855bbaebaf..f2f4a5d50b27 100644 --- a/arch/x86/pci/i386.c +++ b/arch/x86/pci/i386.c @@ -366,9 +366,9 @@ static int __init pcibios_assign_resources(void) return 0; } -/** - * called in fs_initcall (one below subsys_initcall), - * give a chance for motherboard reserve resources +/* + * This is an fs_initcall (one below subsys_initcall) in order to reserve + * resources properly. */ fs_initcall(pcibios_assign_resources); diff --git a/arch/x86/pci/mmconfig-shared.c b/arch/x86/pci/mmconfig-shared.c index 6fa42e9c4e6f..234998f196d4 100644 --- a/arch/x86/pci/mmconfig-shared.c +++ b/arch/x86/pci/mmconfig-shared.c @@ -425,7 +425,7 @@ static acpi_status find_mboard_resource(acpi_handle handle, u32 lvl, return AE_OK; } -static bool is_acpi_reserved(u64 start, u64 end, unsigned not_used) +static bool is_acpi_reserved(u64 start, u64 end, enum e820_type not_used) { struct resource mcfg_res; @@ -442,7 +442,7 @@ static bool is_acpi_reserved(u64 start, u64 end, unsigned not_used) return mcfg_res.flags; } -typedef bool (*check_reserved_t)(u64 start, u64 end, unsigned type); +typedef bool (*check_reserved_t)(u64 start, u64 end, enum e820_type type); static bool __ref is_mmconf_reserved(check_reserved_t is_reserved, struct pci_mmcfg_region *cfg, diff --git a/arch/x86/platform/uv/Makefile b/arch/x86/platform/uv/Makefile index 224ff0504890..1441dda8edf7 100644 --- a/arch/x86/platform/uv/Makefile +++ b/arch/x86/platform/uv/Makefile @@ -1,2 +1,2 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_X86_UV) += bios_uv.o uv_irq.o uv_sysfs.o uv_time.o uv_nmi.o +obj-$(CONFIG_X86_UV) += bios_uv.o uv_irq.o uv_time.o uv_nmi.o diff --git a/arch/x86/platform/uv/bios_uv.c b/arch/x86/platform/uv/bios_uv.c index 54511eaccf4d..bf31af3d32d6 100644 --- a/arch/x86/platform/uv/bios_uv.c +++ b/arch/x86/platform/uv/bios_uv.c @@ -72,6 +72,7 @@ static s64 uv_bios_call_irqsave(enum uv_bios_cmd which, u64 a1, u64 a2, u64 a3, long sn_partition_id; EXPORT_SYMBOL_GPL(sn_partition_id); long sn_coherency_id; +EXPORT_SYMBOL_GPL(sn_coherency_id); long sn_region_size; EXPORT_SYMBOL_GPL(sn_region_size); long system_serial_number; @@ -171,6 +172,60 @@ int uv_bios_set_legacy_vga_target(bool decode, int domain, int bus) (u64)decode, (u64)domain, (u64)bus, 0, 0); } +extern s64 uv_bios_get_master_nasid(u64 size, u64 *master_nasid) +{ + return uv_bios_call(UV_BIOS_EXTRA, 0, UV_BIOS_EXTRA_MASTER_NASID, 0, + size, (u64)master_nasid); +} +EXPORT_SYMBOL_GPL(uv_bios_get_master_nasid); + +extern s64 uv_bios_get_heapsize(u64 nasid, u64 size, u64 *heap_size) +{ + return uv_bios_call(UV_BIOS_EXTRA, nasid, UV_BIOS_EXTRA_GET_HEAPSIZE, + 0, size, (u64)heap_size); +} +EXPORT_SYMBOL_GPL(uv_bios_get_heapsize); + +extern s64 uv_bios_install_heap(u64 nasid, u64 heap_size, u64 *bios_heap) +{ + return uv_bios_call(UV_BIOS_EXTRA, nasid, UV_BIOS_EXTRA_INSTALL_HEAP, + 0, heap_size, (u64)bios_heap); +} +EXPORT_SYMBOL_GPL(uv_bios_install_heap); + +extern s64 uv_bios_obj_count(u64 nasid, u64 size, u64 *objcnt) +{ + return uv_bios_call(UV_BIOS_EXTRA, nasid, UV_BIOS_EXTRA_OBJECT_COUNT, + 0, size, (u64)objcnt); +} +EXPORT_SYMBOL_GPL(uv_bios_obj_count); + +extern s64 uv_bios_enum_objs(u64 nasid, u64 size, u64 *objbuf) +{ + return uv_bios_call(UV_BIOS_EXTRA, nasid, UV_BIOS_EXTRA_ENUM_OBJECTS, + 0, size, (u64)objbuf); +} +EXPORT_SYMBOL_GPL(uv_bios_enum_objs); + +extern s64 uv_bios_enum_ports(u64 nasid, u64 obj_id, u64 size, u64 *portbuf) +{ + return uv_bios_call(UV_BIOS_EXTRA, nasid, UV_BIOS_EXTRA_ENUM_PORTS, + obj_id, size, (u64)portbuf); +} +EXPORT_SYMBOL_GPL(uv_bios_enum_ports); + +extern s64 uv_bios_get_geoinfo(u64 nasid, u64 size, u64 *buf) +{ + return uv_bios_call(UV_BIOS_GET_GEOINFO, nasid, (u64)buf, size, 0, 0); +} +EXPORT_SYMBOL_GPL(uv_bios_get_geoinfo); + +extern s64 uv_bios_get_pci_topology(u64 size, u64 *buf) +{ + return uv_bios_call(UV_BIOS_GET_PCI_TOPOLOGY, (u64)buf, size, 0, 0, 0); +} +EXPORT_SYMBOL_GPL(uv_bios_get_pci_topology); + unsigned long get_uv_systab_phys(bool msg) { if ((uv_systab_phys == EFI_INVALID_TABLE_ADDR) || diff --git a/arch/x86/platform/uv/uv_sysfs.c b/arch/x86/platform/uv/uv_sysfs.c deleted file mode 100644 index 266773e2fb37..000000000000 --- a/arch/x86/platform/uv/uv_sysfs.c +++ /dev/null @@ -1,63 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * This file supports the /sys/firmware/sgi_uv interfaces for SGI UV. - * - * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved. - * Copyright (c) Russ Anderson - */ - -#include -#include -#include - -struct kobject *sgi_uv_kobj; - -static ssize_t partition_id_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buf) -{ - return snprintf(buf, PAGE_SIZE, "%ld\n", sn_partition_id); -} - -static ssize_t coherence_id_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buf) -{ - return snprintf(buf, PAGE_SIZE, "%ld\n", sn_coherency_id); -} - -static struct kobj_attribute partition_id_attr = - __ATTR(partition_id, S_IRUGO, partition_id_show, NULL); - -static struct kobj_attribute coherence_id_attr = - __ATTR(coherence_id, S_IRUGO, coherence_id_show, NULL); - - -static int __init sgi_uv_sysfs_init(void) -{ - unsigned long ret; - - if (!is_uv_system()) - return -ENODEV; - - if (!sgi_uv_kobj) - sgi_uv_kobj = kobject_create_and_add("sgi_uv", firmware_kobj); - if (!sgi_uv_kobj) { - printk(KERN_WARNING "kobject_create_and_add sgi_uv failed\n"); - return -EINVAL; - } - - ret = sysfs_create_file(sgi_uv_kobj, &partition_id_attr.attr); - if (ret) { - printk(KERN_WARNING "sysfs_create_file partition_id failed\n"); - return ret; - } - - ret = sysfs_create_file(sgi_uv_kobj, &coherence_id_attr.attr); - if (ret) { - printk(KERN_WARNING "sysfs_create_file coherence_id failed\n"); - return ret; - } - - return 0; -} - -device_initcall(sgi_uv_sysfs_init); diff --git a/arch/x86/purgatory/purgatory.c b/arch/x86/purgatory/purgatory.c index 7b37a412f829..f03b64d9cb51 100644 --- a/arch/x86/purgatory/purgatory.c +++ b/arch/x86/purgatory/purgatory.c @@ -9,7 +9,7 @@ */ #include -#include +#include #include #include "../boot/string.h" diff --git a/crypto/Kconfig b/crypto/Kconfig index 094ef56ab7b4..a367fcfeb5d4 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -145,7 +145,7 @@ config CRYPTO_MANAGER_DISABLE_TESTS config CRYPTO_MANAGER_EXTRA_TESTS bool "Enable extra run-time crypto self tests" - depends on DEBUG_KERNEL && !CRYPTO_MANAGER_DISABLE_TESTS + depends on DEBUG_KERNEL && !CRYPTO_MANAGER_DISABLE_TESTS && CRYPTO_MANAGER help Enable extra run-time self tests of registered crypto algorithms, including randomized fuzz tests. @@ -201,7 +201,7 @@ config CRYPTO_AUTHENC config CRYPTO_TEST tristate "Testing module" - depends on m + depends on m || EXPERT select CRYPTO_MANAGER help Quick & dirty crypto test module. diff --git a/crypto/aegis128-core.c b/crypto/aegis128-core.c index 44fb4956f0dd..89dc1c559689 100644 --- a/crypto/aegis128-core.c +++ b/crypto/aegis128-core.c @@ -67,9 +67,11 @@ void crypto_aegis128_encrypt_chunk_simd(struct aegis_state *state, u8 *dst, const u8 *src, unsigned int size); void crypto_aegis128_decrypt_chunk_simd(struct aegis_state *state, u8 *dst, const u8 *src, unsigned int size); -void crypto_aegis128_final_simd(struct aegis_state *state, - union aegis_block *tag_xor, - u64 assoclen, u64 cryptlen); +int crypto_aegis128_final_simd(struct aegis_state *state, + union aegis_block *tag_xor, + unsigned int assoclen, + unsigned int cryptlen, + unsigned int authsize); static void crypto_aegis128_update(struct aegis_state *state) { @@ -84,9 +86,10 @@ static void crypto_aegis128_update(struct aegis_state *state) } static void crypto_aegis128_update_a(struct aegis_state *state, - const union aegis_block *msg) + const union aegis_block *msg, + bool do_simd) { - if (aegis128_do_simd()) { + if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD) && do_simd) { crypto_aegis128_update_simd(state, msg); return; } @@ -95,9 +98,10 @@ static void crypto_aegis128_update_a(struct aegis_state *state, crypto_aegis_block_xor(&state->blocks[0], msg); } -static void crypto_aegis128_update_u(struct aegis_state *state, const void *msg) +static void crypto_aegis128_update_u(struct aegis_state *state, const void *msg, + bool do_simd) { - if (aegis128_do_simd()) { + if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD) && do_simd) { crypto_aegis128_update_simd(state, msg); return; } @@ -126,27 +130,28 @@ static void crypto_aegis128_init(struct aegis_state *state, crypto_aegis_block_xor(&state->blocks[4], &crypto_aegis_const[1]); for (i = 0; i < 5; i++) { - crypto_aegis128_update_a(state, key); - crypto_aegis128_update_a(state, &key_iv); + crypto_aegis128_update_a(state, key, false); + crypto_aegis128_update_a(state, &key_iv, false); } } static void crypto_aegis128_ad(struct aegis_state *state, - const u8 *src, unsigned int size) + const u8 *src, unsigned int size, + bool do_simd) { if (AEGIS_ALIGNED(src)) { const union aegis_block *src_blk = (const union aegis_block *)src; while (size >= AEGIS_BLOCK_SIZE) { - crypto_aegis128_update_a(state, src_blk); + crypto_aegis128_update_a(state, src_blk, do_simd); size -= AEGIS_BLOCK_SIZE; src_blk++; } } else { while (size >= AEGIS_BLOCK_SIZE) { - crypto_aegis128_update_u(state, src); + crypto_aegis128_update_u(state, src, do_simd); size -= AEGIS_BLOCK_SIZE; src += AEGIS_BLOCK_SIZE; @@ -154,6 +159,12 @@ static void crypto_aegis128_ad(struct aegis_state *state, } } +static void crypto_aegis128_wipe_chunk(struct aegis_state *state, u8 *dst, + const u8 *src, unsigned int size) +{ + memzero_explicit(dst, size); +} + static void crypto_aegis128_encrypt_chunk(struct aegis_state *state, u8 *dst, const u8 *src, unsigned int size) { @@ -172,7 +183,7 @@ static void crypto_aegis128_encrypt_chunk(struct aegis_state *state, u8 *dst, crypto_aegis_block_xor(&tmp, &state->blocks[1]); crypto_aegis_block_xor(&tmp, src_blk); - crypto_aegis128_update_a(state, src_blk); + crypto_aegis128_update_a(state, src_blk, false); *dst_blk = tmp; @@ -188,7 +199,7 @@ static void crypto_aegis128_encrypt_chunk(struct aegis_state *state, u8 *dst, crypto_aegis_block_xor(&tmp, &state->blocks[1]); crypto_xor(tmp.bytes, src, AEGIS_BLOCK_SIZE); - crypto_aegis128_update_u(state, src); + crypto_aegis128_update_u(state, src, false); memcpy(dst, tmp.bytes, AEGIS_BLOCK_SIZE); @@ -207,7 +218,7 @@ static void crypto_aegis128_encrypt_chunk(struct aegis_state *state, u8 *dst, crypto_aegis_block_xor(&tmp, &state->blocks[4]); crypto_aegis_block_xor(&tmp, &state->blocks[1]); - crypto_aegis128_update_a(state, &msg); + crypto_aegis128_update_a(state, &msg, false); crypto_aegis_block_xor(&msg, &tmp); @@ -233,7 +244,7 @@ static void crypto_aegis128_decrypt_chunk(struct aegis_state *state, u8 *dst, crypto_aegis_block_xor(&tmp, &state->blocks[1]); crypto_aegis_block_xor(&tmp, src_blk); - crypto_aegis128_update_a(state, &tmp); + crypto_aegis128_update_a(state, &tmp, false); *dst_blk = tmp; @@ -249,7 +260,7 @@ static void crypto_aegis128_decrypt_chunk(struct aegis_state *state, u8 *dst, crypto_aegis_block_xor(&tmp, &state->blocks[1]); crypto_xor(tmp.bytes, src, AEGIS_BLOCK_SIZE); - crypto_aegis128_update_a(state, &tmp); + crypto_aegis128_update_a(state, &tmp, false); memcpy(dst, tmp.bytes, AEGIS_BLOCK_SIZE); @@ -271,7 +282,7 @@ static void crypto_aegis128_decrypt_chunk(struct aegis_state *state, u8 *dst, memset(msg.bytes + size, 0, AEGIS_BLOCK_SIZE - size); - crypto_aegis128_update_a(state, &msg); + crypto_aegis128_update_a(state, &msg, false); memcpy(dst, msg.bytes, size); } @@ -279,7 +290,8 @@ static void crypto_aegis128_decrypt_chunk(struct aegis_state *state, u8 *dst, static void crypto_aegis128_process_ad(struct aegis_state *state, struct scatterlist *sg_src, - unsigned int assoclen) + unsigned int assoclen, + bool do_simd) { struct scatter_walk walk; union aegis_block buf; @@ -296,13 +308,13 @@ static void crypto_aegis128_process_ad(struct aegis_state *state, if (pos > 0) { unsigned int fill = AEGIS_BLOCK_SIZE - pos; memcpy(buf.bytes + pos, src, fill); - crypto_aegis128_update_a(state, &buf); + crypto_aegis128_update_a(state, &buf, do_simd); pos = 0; left -= fill; src += fill; } - crypto_aegis128_ad(state, src, left); + crypto_aegis128_ad(state, src, left, do_simd); src += left & ~(AEGIS_BLOCK_SIZE - 1); left &= AEGIS_BLOCK_SIZE - 1; } @@ -318,13 +330,12 @@ static void crypto_aegis128_process_ad(struct aegis_state *state, if (pos > 0) { memset(buf.bytes + pos, 0, AEGIS_BLOCK_SIZE - pos); - crypto_aegis128_update_a(state, &buf); + crypto_aegis128_update_a(state, &buf, do_simd); } } static __always_inline int crypto_aegis128_process_crypt(struct aegis_state *state, - struct aead_request *req, struct skcipher_walk *walk, void (*crypt)(struct aegis_state *state, u8 *dst, const u8 *src, @@ -361,7 +372,7 @@ static void crypto_aegis128_final(struct aegis_state *state, crypto_aegis_block_xor(&tmp, &state->blocks[3]); for (i = 0; i < 7; i++) - crypto_aegis128_update_a(state, &tmp); + crypto_aegis128_update_a(state, &tmp, false); for (i = 0; i < AEGIS128_STATE_BLOCKS; i++) crypto_aegis_block_xor(tag_xor, &state->blocks[i]); @@ -389,7 +400,7 @@ static int crypto_aegis128_setauthsize(struct crypto_aead *tfm, return 0; } -static int crypto_aegis128_encrypt(struct aead_request *req) +static int crypto_aegis128_encrypt_generic(struct aead_request *req) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); union aegis_block tag = {}; @@ -400,27 +411,18 @@ static int crypto_aegis128_encrypt(struct aead_request *req) struct aegis_state state; skcipher_walk_aead_encrypt(&walk, req, false); - if (aegis128_do_simd()) { - crypto_aegis128_init_simd(&state, &ctx->key, req->iv); - crypto_aegis128_process_ad(&state, req->src, req->assoclen); - crypto_aegis128_process_crypt(&state, req, &walk, - crypto_aegis128_encrypt_chunk_simd); - crypto_aegis128_final_simd(&state, &tag, req->assoclen, - cryptlen); - } else { - crypto_aegis128_init(&state, &ctx->key, req->iv); - crypto_aegis128_process_ad(&state, req->src, req->assoclen); - crypto_aegis128_process_crypt(&state, req, &walk, - crypto_aegis128_encrypt_chunk); - crypto_aegis128_final(&state, &tag, req->assoclen, cryptlen); - } + crypto_aegis128_init(&state, &ctx->key, req->iv); + crypto_aegis128_process_ad(&state, req->src, req->assoclen, false); + crypto_aegis128_process_crypt(&state, &walk, + crypto_aegis128_encrypt_chunk); + crypto_aegis128_final(&state, &tag, req->assoclen, cryptlen); scatterwalk_map_and_copy(tag.bytes, req->dst, req->assoclen + cryptlen, authsize, 1); return 0; } -static int crypto_aegis128_decrypt(struct aead_request *req) +static int crypto_aegis128_decrypt_generic(struct aead_request *req) { static const u8 zeros[AEGIS128_MAX_AUTH_SIZE] = {}; struct crypto_aead *tfm = crypto_aead_reqtfm(req); @@ -435,60 +437,152 @@ static int crypto_aegis128_decrypt(struct aead_request *req) authsize, 0); skcipher_walk_aead_decrypt(&walk, req, false); - if (aegis128_do_simd()) { - crypto_aegis128_init_simd(&state, &ctx->key, req->iv); - crypto_aegis128_process_ad(&state, req->src, req->assoclen); - crypto_aegis128_process_crypt(&state, req, &walk, - crypto_aegis128_decrypt_chunk_simd); - crypto_aegis128_final_simd(&state, &tag, req->assoclen, - cryptlen); - } else { - crypto_aegis128_init(&state, &ctx->key, req->iv); - crypto_aegis128_process_ad(&state, req->src, req->assoclen); - crypto_aegis128_process_crypt(&state, req, &walk, - crypto_aegis128_decrypt_chunk); - crypto_aegis128_final(&state, &tag, req->assoclen, cryptlen); - } + crypto_aegis128_init(&state, &ctx->key, req->iv); + crypto_aegis128_process_ad(&state, req->src, req->assoclen, false); + crypto_aegis128_process_crypt(&state, &walk, + crypto_aegis128_decrypt_chunk); + crypto_aegis128_final(&state, &tag, req->assoclen, cryptlen); - return crypto_memneq(tag.bytes, zeros, authsize) ? -EBADMSG : 0; + if (unlikely(crypto_memneq(tag.bytes, zeros, authsize))) { + /* + * From Chapter 4. 'Security Analysis' of the AEGIS spec [0] + * + * "3. If verification fails, the decrypted plaintext and the + * wrong authentication tag should not be given as output." + * + * [0] https://competitions.cr.yp.to/round3/aegisv11.pdf + */ + skcipher_walk_aead_decrypt(&walk, req, false); + crypto_aegis128_process_crypt(NULL, &walk, + crypto_aegis128_wipe_chunk); + memzero_explicit(&tag, sizeof(tag)); + return -EBADMSG; + } + return 0; } -static struct aead_alg crypto_aegis128_alg = { - .setkey = crypto_aegis128_setkey, - .setauthsize = crypto_aegis128_setauthsize, - .encrypt = crypto_aegis128_encrypt, - .decrypt = crypto_aegis128_decrypt, +static int crypto_aegis128_encrypt_simd(struct aead_request *req) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + union aegis_block tag = {}; + unsigned int authsize = crypto_aead_authsize(tfm); + struct aegis_ctx *ctx = crypto_aead_ctx(tfm); + unsigned int cryptlen = req->cryptlen; + struct skcipher_walk walk; + struct aegis_state state; - .ivsize = AEGIS128_NONCE_SIZE, - .maxauthsize = AEGIS128_MAX_AUTH_SIZE, - .chunksize = AEGIS_BLOCK_SIZE, + if (!aegis128_do_simd()) + return crypto_aegis128_encrypt_generic(req); - .base = { - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct aegis_ctx), - .cra_alignmask = 0, + skcipher_walk_aead_encrypt(&walk, req, false); + crypto_aegis128_init_simd(&state, &ctx->key, req->iv); + crypto_aegis128_process_ad(&state, req->src, req->assoclen, true); + crypto_aegis128_process_crypt(&state, &walk, + crypto_aegis128_encrypt_chunk_simd); + crypto_aegis128_final_simd(&state, &tag, req->assoclen, cryptlen, 0); - .cra_priority = 100, + scatterwalk_map_and_copy(tag.bytes, req->dst, req->assoclen + cryptlen, + authsize, 1); + return 0; +} - .cra_name = "aegis128", - .cra_driver_name = "aegis128-generic", +static int crypto_aegis128_decrypt_simd(struct aead_request *req) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + union aegis_block tag; + unsigned int authsize = crypto_aead_authsize(tfm); + unsigned int cryptlen = req->cryptlen - authsize; + struct aegis_ctx *ctx = crypto_aead_ctx(tfm); + struct skcipher_walk walk; + struct aegis_state state; - .cra_module = THIS_MODULE, + if (!aegis128_do_simd()) + return crypto_aegis128_decrypt_generic(req); + + scatterwalk_map_and_copy(tag.bytes, req->src, req->assoclen + cryptlen, + authsize, 0); + + skcipher_walk_aead_decrypt(&walk, req, false); + crypto_aegis128_init_simd(&state, &ctx->key, req->iv); + crypto_aegis128_process_ad(&state, req->src, req->assoclen, true); + crypto_aegis128_process_crypt(&state, &walk, + crypto_aegis128_decrypt_chunk_simd); + + if (unlikely(crypto_aegis128_final_simd(&state, &tag, req->assoclen, + cryptlen, authsize))) { + skcipher_walk_aead_decrypt(&walk, req, false); + crypto_aegis128_process_crypt(NULL, &walk, + crypto_aegis128_wipe_chunk); + return -EBADMSG; } + return 0; +} + +static struct aead_alg crypto_aegis128_alg_generic = { + .setkey = crypto_aegis128_setkey, + .setauthsize = crypto_aegis128_setauthsize, + .encrypt = crypto_aegis128_encrypt_generic, + .decrypt = crypto_aegis128_decrypt_generic, + + .ivsize = AEGIS128_NONCE_SIZE, + .maxauthsize = AEGIS128_MAX_AUTH_SIZE, + .chunksize = AEGIS_BLOCK_SIZE, + + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct aegis_ctx), + .base.cra_alignmask = 0, + .base.cra_priority = 100, + .base.cra_name = "aegis128", + .base.cra_driver_name = "aegis128-generic", + .base.cra_module = THIS_MODULE, +}; + +static struct aead_alg crypto_aegis128_alg_simd = { + .setkey = crypto_aegis128_setkey, + .setauthsize = crypto_aegis128_setauthsize, + .encrypt = crypto_aegis128_encrypt_simd, + .decrypt = crypto_aegis128_decrypt_simd, + + .ivsize = AEGIS128_NONCE_SIZE, + .maxauthsize = AEGIS128_MAX_AUTH_SIZE, + .chunksize = AEGIS_BLOCK_SIZE, + + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct aegis_ctx), + .base.cra_alignmask = 0, + .base.cra_priority = 200, + .base.cra_name = "aegis128", + .base.cra_driver_name = "aegis128-simd", + .base.cra_module = THIS_MODULE, }; static int __init crypto_aegis128_module_init(void) { - if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD) && - crypto_aegis128_have_simd()) - static_branch_enable(&have_simd); + int ret; - return crypto_register_aead(&crypto_aegis128_alg); + ret = crypto_register_aead(&crypto_aegis128_alg_generic); + if (ret) + return ret; + + if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD) && + crypto_aegis128_have_simd()) { + ret = crypto_register_aead(&crypto_aegis128_alg_simd); + if (ret) { + crypto_unregister_aead(&crypto_aegis128_alg_generic); + return ret; + } + static_branch_enable(&have_simd); + } + return 0; } static void __exit crypto_aegis128_module_exit(void) { - crypto_unregister_aead(&crypto_aegis128_alg); + if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD) && + crypto_aegis128_have_simd()) + crypto_unregister_aead(&crypto_aegis128_alg_simd); + + crypto_unregister_aead(&crypto_aegis128_alg_generic); } subsys_initcall(crypto_aegis128_module_init); @@ -499,3 +593,4 @@ MODULE_AUTHOR("Ondrej Mosnacek "); MODULE_DESCRIPTION("AEGIS-128 AEAD algorithm"); MODULE_ALIAS_CRYPTO("aegis128"); MODULE_ALIAS_CRYPTO("aegis128-generic"); +MODULE_ALIAS_CRYPTO("aegis128-simd"); diff --git a/crypto/aegis128-neon-inner.c b/crypto/aegis128-neon-inner.c index 2a660ac1bc3a..7de485907d81 100644 --- a/crypto/aegis128-neon-inner.c +++ b/crypto/aegis128-neon-inner.c @@ -20,7 +20,6 @@ extern int aegis128_have_aes_insn; void *memcpy(void *dest, const void *src, size_t n); -void *memset(void *s, int c, size_t n); struct aegis128_state { uint8x16_t v[5]; @@ -173,10 +172,57 @@ void crypto_aegis128_update_neon(void *state, const void *msg) aegis128_save_state_neon(st, state); } +#ifdef CONFIG_ARM +/* + * AArch32 does not provide these intrinsics natively because it does not + * implement the underlying instructions. AArch32 only provides 64-bit + * wide vtbl.8/vtbx.8 instruction, so use those instead. + */ +static uint8x16_t vqtbl1q_u8(uint8x16_t a, uint8x16_t b) +{ + union { + uint8x16_t val; + uint8x8x2_t pair; + } __a = { a }; + + return vcombine_u8(vtbl2_u8(__a.pair, vget_low_u8(b)), + vtbl2_u8(__a.pair, vget_high_u8(b))); +} + +static uint8x16_t vqtbx1q_u8(uint8x16_t v, uint8x16_t a, uint8x16_t b) +{ + union { + uint8x16_t val; + uint8x8x2_t pair; + } __a = { a }; + + return vcombine_u8(vtbx2_u8(vget_low_u8(v), __a.pair, vget_low_u8(b)), + vtbx2_u8(vget_high_u8(v), __a.pair, vget_high_u8(b))); +} + +static int8_t vminvq_s8(int8x16_t v) +{ + int8x8_t s = vpmin_s8(vget_low_s8(v), vget_high_s8(v)); + + s = vpmin_s8(s, s); + s = vpmin_s8(s, s); + s = vpmin_s8(s, s); + + return vget_lane_s8(s, 0); +} +#endif + +static const uint8_t permute[] __aligned(64) = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, +}; + void crypto_aegis128_encrypt_chunk_neon(void *state, void *dst, const void *src, unsigned int size) { struct aegis128_state st = aegis128_load_state_neon(state); + const int short_input = size < AEGIS_BLOCK_SIZE; uint8x16_t msg; preload_sbox(); @@ -186,7 +232,8 @@ void crypto_aegis128_encrypt_chunk_neon(void *state, void *dst, const void *src, msg = vld1q_u8(src); st = aegis128_update_neon(st, msg); - vst1q_u8(dst, msg ^ s); + msg ^= s; + vst1q_u8(dst, msg); size -= AEGIS_BLOCK_SIZE; src += AEGIS_BLOCK_SIZE; @@ -195,13 +242,26 @@ void crypto_aegis128_encrypt_chunk_neon(void *state, void *dst, const void *src, if (size > 0) { uint8x16_t s = st.v[1] ^ (st.v[2] & st.v[3]) ^ st.v[4]; - uint8_t buf[AEGIS_BLOCK_SIZE] = {}; + uint8_t buf[AEGIS_BLOCK_SIZE]; + const void *in = src; + void *out = dst; + uint8x16_t m; - memcpy(buf, src, size); - msg = vld1q_u8(buf); - st = aegis128_update_neon(st, msg); - vst1q_u8(buf, msg ^ s); - memcpy(dst, buf, size); + if (__builtin_expect(short_input, 0)) + in = out = memcpy(buf + AEGIS_BLOCK_SIZE - size, src, size); + + m = vqtbl1q_u8(vld1q_u8(in + size - AEGIS_BLOCK_SIZE), + vld1q_u8(permute + 32 - size)); + + st = aegis128_update_neon(st, m); + + vst1q_u8(out + size - AEGIS_BLOCK_SIZE, + vqtbl1q_u8(m ^ s, vld1q_u8(permute + size))); + + if (__builtin_expect(short_input, 0)) + memcpy(dst, out, size); + else + vst1q_u8(out - AEGIS_BLOCK_SIZE, msg); } aegis128_save_state_neon(st, state); @@ -211,6 +271,7 @@ void crypto_aegis128_decrypt_chunk_neon(void *state, void *dst, const void *src, unsigned int size) { struct aegis128_state st = aegis128_load_state_neon(state); + const int short_input = size < AEGIS_BLOCK_SIZE; uint8x16_t msg; preload_sbox(); @@ -228,21 +289,34 @@ void crypto_aegis128_decrypt_chunk_neon(void *state, void *dst, const void *src, if (size > 0) { uint8x16_t s = st.v[1] ^ (st.v[2] & st.v[3]) ^ st.v[4]; uint8_t buf[AEGIS_BLOCK_SIZE]; + const void *in = src; + void *out = dst; + uint8x16_t m; - vst1q_u8(buf, s); - memcpy(buf, src, size); - msg = vld1q_u8(buf) ^ s; - vst1q_u8(buf, msg); - memcpy(dst, buf, size); + if (__builtin_expect(short_input, 0)) + in = out = memcpy(buf + AEGIS_BLOCK_SIZE - size, src, size); - st = aegis128_update_neon(st, msg); + m = s ^ vqtbx1q_u8(s, vld1q_u8(in + size - AEGIS_BLOCK_SIZE), + vld1q_u8(permute + 32 - size)); + + st = aegis128_update_neon(st, m); + + vst1q_u8(out + size - AEGIS_BLOCK_SIZE, + vqtbl1q_u8(m, vld1q_u8(permute + size))); + + if (__builtin_expect(short_input, 0)) + memcpy(dst, out, size); + else + vst1q_u8(out - AEGIS_BLOCK_SIZE, msg); } aegis128_save_state_neon(st, state); } -void crypto_aegis128_final_neon(void *state, void *tag_xor, uint64_t assoclen, - uint64_t cryptlen) +int crypto_aegis128_final_neon(void *state, void *tag_xor, + unsigned int assoclen, + unsigned int cryptlen, + unsigned int authsize) { struct aegis128_state st = aegis128_load_state_neon(state); uint8x16_t v; @@ -250,13 +324,21 @@ void crypto_aegis128_final_neon(void *state, void *tag_xor, uint64_t assoclen, preload_sbox(); - v = st.v[3] ^ (uint8x16_t)vcombine_u64(vmov_n_u64(8 * assoclen), - vmov_n_u64(8 * cryptlen)); + v = st.v[3] ^ (uint8x16_t)vcombine_u64(vmov_n_u64(8ULL * assoclen), + vmov_n_u64(8ULL * cryptlen)); for (i = 0; i < 7; i++) st = aegis128_update_neon(st, v); - v = vld1q_u8(tag_xor); - v ^= st.v[0] ^ st.v[1] ^ st.v[2] ^ st.v[3] ^ st.v[4]; + v = st.v[0] ^ st.v[1] ^ st.v[2] ^ st.v[3] ^ st.v[4]; + + if (authsize > 0) { + v = vqtbl1q_u8(~vceqq_u8(v, vld1q_u8(tag_xor)), + vld1q_u8(permute + authsize)); + + return vminvq_s8((int8x16_t)v); + } + vst1q_u8(tag_xor, v); + return 0; } diff --git a/crypto/aegis128-neon.c b/crypto/aegis128-neon.c index 8271b1fa0fbc..94d591a002a4 100644 --- a/crypto/aegis128-neon.c +++ b/crypto/aegis128-neon.c @@ -14,8 +14,10 @@ void crypto_aegis128_encrypt_chunk_neon(void *state, void *dst, const void *src, unsigned int size); void crypto_aegis128_decrypt_chunk_neon(void *state, void *dst, const void *src, unsigned int size); -void crypto_aegis128_final_neon(void *state, void *tag_xor, uint64_t assoclen, - uint64_t cryptlen); +int crypto_aegis128_final_neon(void *state, void *tag_xor, + unsigned int assoclen, + unsigned int cryptlen, + unsigned int authsize); int aegis128_have_aes_insn __ro_after_init; @@ -60,11 +62,18 @@ void crypto_aegis128_decrypt_chunk_simd(union aegis_block *state, u8 *dst, kernel_neon_end(); } -void crypto_aegis128_final_simd(union aegis_block *state, - union aegis_block *tag_xor, - u64 assoclen, u64 cryptlen) +int crypto_aegis128_final_simd(union aegis_block *state, + union aegis_block *tag_xor, + unsigned int assoclen, + unsigned int cryptlen, + unsigned int authsize) { + int ret; + kernel_neon_begin(); - crypto_aegis128_final_neon(state, tag_xor, assoclen, cryptlen); + ret = crypto_aegis128_final_neon(state, tag_xor, assoclen, cryptlen, + authsize); kernel_neon_end(); + + return ret; } diff --git a/crypto/af_alg.c b/crypto/af_alg.c index d11db80d24cd..9acb9d2c4bcf 100644 --- a/crypto/af_alg.c +++ b/crypto/af_alg.c @@ -147,7 +147,7 @@ static int alg_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) const u32 allowed = CRYPTO_ALG_KERN_DRIVER_ONLY; struct sock *sk = sock->sk; struct alg_sock *ask = alg_sk(sk); - struct sockaddr_alg *sa = (void *)uaddr; + struct sockaddr_alg_new *sa = (void *)uaddr; const struct af_alg_type *type; void *private; int err; @@ -155,7 +155,11 @@ static int alg_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) if (sock->state == SS_CONNECTED) return -EINVAL; - if (addr_len < sizeof(*sa)) + BUILD_BUG_ON(offsetof(struct sockaddr_alg_new, salg_name) != + offsetof(struct sockaddr_alg, salg_name)); + BUILD_BUG_ON(offsetof(struct sockaddr_alg, salg_name) != sizeof(*sa)); + + if (addr_len < sizeof(*sa) + 1) return -EINVAL; /* If caller uses non-allowed flag, return error. */ @@ -163,7 +167,7 @@ static int alg_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) return -EINVAL; sa->salg_type[sizeof(sa->salg_type) - 1] = 0; - sa->salg_name[sizeof(sa->salg_name) + addr_len - sizeof(*sa) - 1] = 0; + sa->salg_name[addr_len - sizeof(*sa) - 1] = 0; type = alg_get_type(sa->salg_type); if (PTR_ERR(type) == -ENOENT) { diff --git a/crypto/asymmetric_keys/asym_tpm.c b/crypto/asymmetric_keys/asym_tpm.c index 378b18b9bc34..511932aa94a6 100644 --- a/crypto/asymmetric_keys/asym_tpm.c +++ b/crypto/asymmetric_keys/asym_tpm.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/crypto/ecdh.c b/crypto/ecdh.c index b0232d6ab4ce..d56b8603dec9 100644 --- a/crypto/ecdh.c +++ b/crypto/ecdh.c @@ -53,12 +53,13 @@ static int ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, return ecc_gen_privkey(ctx->curve_id, ctx->ndigits, ctx->private_key); - if (ecc_is_key_valid(ctx->curve_id, ctx->ndigits, - (const u64 *)params.key, params.key_size) < 0) - return -EINVAL; - memcpy(ctx->private_key, params.key, params.key_size); + if (ecc_is_key_valid(ctx->curve_id, ctx->ndigits, + ctx->private_key, params.key_size) < 0) { + memzero_explicit(ctx->private_key, params.key_size); + return -EINVAL; + } return 0; } diff --git a/crypto/seed.c b/crypto/seed.c index 5e3bef3a617d..27720140820e 100644 --- a/crypto/seed.c +++ b/crypto/seed.c @@ -322,7 +322,7 @@ static const u32 KC[SEED_NUM_KCONSTANTS] = { SS2[byte(t1, 2)] ^ SS3[byte(t1, 3)]; \ t0 += t1; \ X1 ^= t0; \ - X2 ^= t1; + X2 ^= t1 static int seed_set_key(struct crypto_tfm *tfm, const u8 *in_key, unsigned int key_len) diff --git a/crypto/sha1_generic.c b/crypto/sha1_generic.c index 1d43472fecbd..325b57fe28dc 100644 --- a/crypto/sha1_generic.c +++ b/crypto/sha1_generic.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include diff --git a/crypto/sha256_generic.c b/crypto/sha256_generic.c index 88156e3e2a33..3b377197236e 100644 --- a/crypto/sha256_generic.c +++ b/crypto/sha256_generic.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/crypto/sha512_generic.c b/crypto/sha512_generic.c index e34d09dd9971..c72d72ad828e 100644 --- a/crypto/sha512_generic.c +++ b/crypto/sha512_generic.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/crypto/sm2.c b/crypto/sm2.c index 767e160333f6..b21addc3ac06 100644 --- a/crypto/sm2.c +++ b/crypto/sm2.c @@ -119,12 +119,6 @@ static void sm2_ec_ctx_deinit(struct mpi_ec_ctx *ec) memset(ec, 0, sizeof(*ec)); } -static int sm2_ec_ctx_reset(struct mpi_ec_ctx *ec) -{ - sm2_ec_ctx_deinit(ec); - return sm2_ec_ctx_init(ec); -} - /* RESULT must have been initialized and is set on success to the * point given by VALUE. */ @@ -132,55 +126,48 @@ static int sm2_ecc_os2ec(MPI_POINT result, MPI value) { int rc; size_t n; - const unsigned char *buf; - unsigned char *buf_memory; + unsigned char *buf; MPI x, y; - n = (mpi_get_nbits(value)+7)/8; - buf_memory = kmalloc(n, GFP_KERNEL); - rc = mpi_print(GCRYMPI_FMT_USG, buf_memory, n, &n, value); - if (rc) { - kfree(buf_memory); - return rc; - } - buf = buf_memory; + n = MPI_NBYTES(value); + buf = kmalloc(n, GFP_KERNEL); + if (!buf) + return -ENOMEM; - if (n < 1) { - kfree(buf_memory); - return -EINVAL; - } - if (*buf != 4) { - kfree(buf_memory); - return -EINVAL; /* No support for point compression. */ - } - if (((n-1)%2)) { - kfree(buf_memory); - return -EINVAL; - } - n = (n-1)/2; + rc = mpi_print(GCRYMPI_FMT_USG, buf, n, &n, value); + if (rc) + goto err_freebuf; + + rc = -EINVAL; + if (n < 1 || ((n - 1) % 2)) + goto err_freebuf; + /* No support for point compression */ + if (*buf != 0x4) + goto err_freebuf; + + rc = -ENOMEM; + n = (n - 1) / 2; x = mpi_read_raw_data(buf + 1, n); - if (!x) { - kfree(buf_memory); - return -ENOMEM; - } + if (!x) + goto err_freebuf; y = mpi_read_raw_data(buf + 1 + n, n); - kfree(buf_memory); - if (!y) { - mpi_free(x); - return -ENOMEM; - } + if (!y) + goto err_freex; mpi_normalize(x); mpi_normalize(y); - mpi_set(result->x, x); mpi_set(result->y, y); mpi_set_ui(result->z, 1); - mpi_free(x); - mpi_free(y); + rc = 0; - return 0; + mpi_free(y); +err_freex: + mpi_free(x); +err_freebuf: + kfree(buf); + return rc; } struct sm2_signature_ctx { @@ -399,10 +386,6 @@ static int sm2_set_pub_key(struct crypto_akcipher *tfm, MPI a; int rc; - rc = sm2_ec_ctx_reset(ec); - if (rc) - return rc; - ec->Q = mpi_point_new(0); if (!ec->Q) return -ENOMEM; diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index eea0f453cfb6..a647bb298fbc 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -77,8 +77,8 @@ static const char *check[] = { NULL }; -static u32 block_sizes[] = { 16, 64, 256, 1024, 1472, 8192, 0 }; -static u32 aead_sizes[] = { 16, 64, 256, 512, 1024, 2048, 4096, 8192, 0 }; +static const int block_sizes[] = { 16, 64, 256, 1024, 1420, 4096, 0 }; +static const int aead_sizes[] = { 16, 64, 256, 512, 1024, 1420, 4096, 8192, 0 }; #define XBUFSIZE 8 #define MAX_IVLEN 32 @@ -256,10 +256,10 @@ static void test_mb_aead_speed(const char *algo, int enc, int secs, struct test_mb_aead_data *data; struct crypto_aead *tfm; unsigned int i, j, iv_len; + const int *b_size; const char *key; const char *e; void *assoc; - u32 *b_size; char *iv; int ret; @@ -337,15 +337,17 @@ static void test_mb_aead_speed(const char *algo, int enc, int secs, do { b_size = aead_sizes; do { - if (*b_size + authsize > XBUFSIZE * PAGE_SIZE) { + int bs = round_up(*b_size, crypto_aead_blocksize(tfm)); + + if (bs + authsize > XBUFSIZE * PAGE_SIZE) { pr_err("template (%u) too big for buffer (%lu)\n", - authsize + *b_size, + authsize + bs, XBUFSIZE * PAGE_SIZE); goto out; } pr_info("test %u (%d bit key, %d byte blocks): ", i, - *keysize * 8, *b_size); + *keysize * 8, bs); /* Set up tfm global state, i.e. the key */ @@ -380,11 +382,11 @@ static void test_mb_aead_speed(const char *algo, int enc, int secs, memset(assoc, 0xff, aad_size); sg_init_aead(cur->sg, cur->xbuf, - *b_size + (enc ? 0 : authsize), + bs + (enc ? 0 : authsize), assoc, aad_size); sg_init_aead(cur->sgout, cur->xoutbuf, - *b_size + (enc ? authsize : 0), + bs + (enc ? authsize : 0), assoc, aad_size); aead_request_set_ad(cur->req, aad_size); @@ -394,7 +396,7 @@ static void test_mb_aead_speed(const char *algo, int enc, int secs, aead_request_set_crypt(cur->req, cur->sgout, cur->sg, - *b_size, iv); + bs, iv); ret = crypto_aead_encrypt(cur->req); ret = do_one_aead_op(cur->req, ret); @@ -406,18 +408,18 @@ static void test_mb_aead_speed(const char *algo, int enc, int secs, } aead_request_set_crypt(cur->req, cur->sg, - cur->sgout, *b_size + + cur->sgout, bs + (enc ? 0 : authsize), iv); } if (secs) { - ret = test_mb_aead_jiffies(data, enc, *b_size, + ret = test_mb_aead_jiffies(data, enc, bs, secs, num_mb); cond_resched(); } else { - ret = test_mb_aead_cycles(data, enc, *b_size, + ret = test_mb_aead_cycles(data, enc, bs, num_mb); } @@ -534,7 +536,7 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs, char *xbuf[XBUFSIZE]; char *xoutbuf[XBUFSIZE]; char *axbuf[XBUFSIZE]; - unsigned int *b_size; + const int *b_size; unsigned int iv_len; struct crypto_wait wait; @@ -590,12 +592,14 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs, do { b_size = aead_sizes; do { + u32 bs = round_up(*b_size, crypto_aead_blocksize(tfm)); + assoc = axbuf[0]; memset(assoc, 0xff, aad_size); - if ((*keysize + *b_size) > TVMEMSIZE * PAGE_SIZE) { + if ((*keysize + bs) > TVMEMSIZE * PAGE_SIZE) { pr_err("template (%u) too big for tvmem (%lu)\n", - *keysize + *b_size, + *keysize + bs, TVMEMSIZE * PAGE_SIZE); goto out; } @@ -616,7 +620,7 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs, crypto_aead_clear_flags(tfm, ~0); printk(KERN_INFO "test %u (%d bit key, %d byte blocks): ", - i, *keysize * 8, *b_size); + i, *keysize * 8, bs); memset(tvmem[0], 0xff, PAGE_SIZE); @@ -627,11 +631,11 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs, goto out; } - sg_init_aead(sg, xbuf, *b_size + (enc ? 0 : authsize), + sg_init_aead(sg, xbuf, bs + (enc ? 0 : authsize), assoc, aad_size); sg_init_aead(sgout, xoutbuf, - *b_size + (enc ? authsize : 0), assoc, + bs + (enc ? authsize : 0), assoc, aad_size); aead_request_set_ad(req, aad_size); @@ -644,7 +648,7 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs, * reversed (input <-> output) to calculate it */ aead_request_set_crypt(req, sgout, sg, - *b_size, iv); + bs, iv); ret = do_one_aead_op(req, crypto_aead_encrypt(req)); @@ -656,15 +660,15 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs, } aead_request_set_crypt(req, sg, sgout, - *b_size + (enc ? 0 : authsize), + bs + (enc ? 0 : authsize), iv); if (secs) { - ret = test_aead_jiffies(req, enc, *b_size, + ret = test_aead_jiffies(req, enc, bs, secs); cond_resched(); } else { - ret = test_aead_cycles(req, enc, *b_size); + ret = test_aead_cycles(req, enc, bs); } if (ret) { @@ -1253,9 +1257,9 @@ static void test_mb_skcipher_speed(const char *algo, int enc, int secs, struct test_mb_skcipher_data *data; struct crypto_skcipher *tfm; unsigned int i, j, iv_len; + const int *b_size; const char *key; const char *e; - u32 *b_size; char iv[128]; int ret; @@ -1316,14 +1320,16 @@ static void test_mb_skcipher_speed(const char *algo, int enc, int secs, do { b_size = block_sizes; do { - if (*b_size > XBUFSIZE * PAGE_SIZE) { + u32 bs = round_up(*b_size, crypto_skcipher_blocksize(tfm)); + + if (bs > XBUFSIZE * PAGE_SIZE) { pr_err("template (%u) too big for buffer (%lu)\n", *b_size, XBUFSIZE * PAGE_SIZE); goto out; } pr_info("test %u (%d bit key, %d byte blocks): ", i, - *keysize * 8, *b_size); + *keysize * 8, bs); /* Set up tfm global state, i.e. the key */ @@ -1353,7 +1359,7 @@ static void test_mb_skcipher_speed(const char *algo, int enc, int secs, for (j = 0; j < num_mb; ++j) { struct test_mb_skcipher_data *cur = &data[j]; - unsigned int k = *b_size; + unsigned int k = bs; unsigned int pages = DIV_ROUND_UP(k, PAGE_SIZE); unsigned int p = 0; @@ -1377,12 +1383,12 @@ static void test_mb_skcipher_speed(const char *algo, int enc, int secs, if (secs) { ret = test_mb_acipher_jiffies(data, enc, - *b_size, secs, + bs, secs, num_mb); cond_resched(); } else { ret = test_mb_acipher_cycles(data, enc, - *b_size, num_mb); + bs, num_mb); } if (ret) { @@ -1497,8 +1503,8 @@ static void test_skcipher_speed(const char *algo, int enc, unsigned int secs, char iv[128]; struct skcipher_request *req; struct crypto_skcipher *tfm; + const int *b_size; const char *e; - u32 *b_size; if (enc == ENCRYPT) e = "encryption"; @@ -1533,17 +1539,18 @@ static void test_skcipher_speed(const char *algo, int enc, unsigned int secs, b_size = block_sizes; do { + u32 bs = round_up(*b_size, crypto_skcipher_blocksize(tfm)); struct scatterlist sg[TVMEMSIZE]; - if ((*keysize + *b_size) > TVMEMSIZE * PAGE_SIZE) { + if ((*keysize + bs) > TVMEMSIZE * PAGE_SIZE) { pr_err("template (%u) too big for " - "tvmem (%lu)\n", *keysize + *b_size, + "tvmem (%lu)\n", *keysize + bs, TVMEMSIZE * PAGE_SIZE); goto out_free_req; } pr_info("test %u (%d bit key, %d byte blocks): ", i, - *keysize * 8, *b_size); + *keysize * 8, bs); memset(tvmem[0], 0xff, PAGE_SIZE); @@ -1565,7 +1572,7 @@ static void test_skcipher_speed(const char *algo, int enc, unsigned int secs, goto out_free_req; } - k = *keysize + *b_size; + k = *keysize + bs; sg_init_table(sg, DIV_ROUND_UP(k, PAGE_SIZE)); if (k > PAGE_SIZE) { @@ -1582,22 +1589,22 @@ static void test_skcipher_speed(const char *algo, int enc, unsigned int secs, sg_set_buf(sg + j, tvmem[j], k); memset(tvmem[j], 0xff, k); } else { - sg_set_buf(sg, tvmem[0] + *keysize, *b_size); + sg_set_buf(sg, tvmem[0] + *keysize, bs); } iv_len = crypto_skcipher_ivsize(tfm); if (iv_len) memset(&iv, 0xff, iv_len); - skcipher_request_set_crypt(req, sg, sg, *b_size, iv); + skcipher_request_set_crypt(req, sg, sg, bs, iv); if (secs) { ret = test_acipher_jiffies(req, enc, - *b_size, secs); + bs, secs); cond_resched(); } else { ret = test_acipher_cycles(req, enc, - *b_size); + bs); } if (ret) { @@ -3066,7 +3073,7 @@ err_free_tv: */ static void __exit tcrypt_mod_fini(void) { } -subsys_initcall(tcrypt_mod_init); +late_initcall(tcrypt_mod_init); module_exit(tcrypt_mod_fini); module_param(alg, charp, 0); diff --git a/crypto/testmgr.c b/crypto/testmgr.c index a64a639eddfa..321e38eef51b 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -1171,8 +1171,7 @@ static inline const void *sg_data(struct scatterlist *sg) } /* Test one hash test vector in one configuration, using the shash API */ -static int test_shash_vec_cfg(const char *driver, - const struct hash_testvec *vec, +static int test_shash_vec_cfg(const struct hash_testvec *vec, const char *vec_name, const struct testvec_config *cfg, struct shash_desc *desc, @@ -1183,6 +1182,7 @@ static int test_shash_vec_cfg(const char *driver, const unsigned int alignmask = crypto_shash_alignmask(tfm); const unsigned int digestsize = crypto_shash_digestsize(tfm); const unsigned int statesize = crypto_shash_statesize(tfm); + const char *driver = crypto_shash_driver_name(tfm); const struct test_sg_division *divs[XBUFSIZE]; unsigned int i; u8 result[HASH_MAX_DIGESTSIZE + TESTMGR_POISON_LEN]; @@ -1355,8 +1355,7 @@ static int check_nonfinal_ahash_op(const char *op, int err, } /* Test one hash test vector in one configuration, using the ahash API */ -static int test_ahash_vec_cfg(const char *driver, - const struct hash_testvec *vec, +static int test_ahash_vec_cfg(const struct hash_testvec *vec, const char *vec_name, const struct testvec_config *cfg, struct ahash_request *req, @@ -1367,6 +1366,7 @@ static int test_ahash_vec_cfg(const char *driver, const unsigned int alignmask = crypto_ahash_alignmask(tfm); const unsigned int digestsize = crypto_ahash_digestsize(tfm); const unsigned int statesize = crypto_ahash_statesize(tfm); + const char *driver = crypto_ahash_driver_name(tfm); const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags; const struct test_sg_division *divs[XBUFSIZE]; DECLARE_CRYPTO_WAIT(wait); @@ -1521,8 +1521,7 @@ result_ready: driver, cfg); } -static int test_hash_vec_cfg(const char *driver, - const struct hash_testvec *vec, +static int test_hash_vec_cfg(const struct hash_testvec *vec, const char *vec_name, const struct testvec_config *cfg, struct ahash_request *req, @@ -1539,20 +1538,18 @@ static int test_hash_vec_cfg(const char *driver, */ if (desc) { - err = test_shash_vec_cfg(driver, vec, vec_name, cfg, desc, tsgl, + err = test_shash_vec_cfg(vec, vec_name, cfg, desc, tsgl, hashstate); if (err) return err; } - return test_ahash_vec_cfg(driver, vec, vec_name, cfg, req, tsgl, - hashstate); + return test_ahash_vec_cfg(vec, vec_name, cfg, req, tsgl, hashstate); } -static int test_hash_vec(const char *driver, const struct hash_testvec *vec, - unsigned int vec_num, struct ahash_request *req, - struct shash_desc *desc, struct test_sglist *tsgl, - u8 *hashstate) +static int test_hash_vec(const struct hash_testvec *vec, unsigned int vec_num, + struct ahash_request *req, struct shash_desc *desc, + struct test_sglist *tsgl, u8 *hashstate) { char vec_name[16]; unsigned int i; @@ -1561,7 +1558,7 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec, sprintf(vec_name, "%u", vec_num); for (i = 0; i < ARRAY_SIZE(default_hash_testvec_configs); i++) { - err = test_hash_vec_cfg(driver, vec, vec_name, + err = test_hash_vec_cfg(vec, vec_name, &default_hash_testvec_configs[i], req, desc, tsgl, hashstate); if (err) @@ -1576,7 +1573,7 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec, for (i = 0; i < fuzz_iterations; i++) { generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); - err = test_hash_vec_cfg(driver, vec, vec_name, &cfg, + err = test_hash_vec_cfg(vec, vec_name, &cfg, req, desc, tsgl, hashstate); if (err) return err; @@ -1633,8 +1630,7 @@ done: * Test the hash algorithm represented by @req against the corresponding generic * implementation, if one is available. */ -static int test_hash_vs_generic_impl(const char *driver, - const char *generic_driver, +static int test_hash_vs_generic_impl(const char *generic_driver, unsigned int maxkeysize, struct ahash_request *req, struct shash_desc *desc, @@ -1646,6 +1642,7 @@ static int test_hash_vs_generic_impl(const char *driver, const unsigned int blocksize = crypto_ahash_blocksize(tfm); const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN; const char *algname = crypto_hash_alg_common(tfm)->base.cra_name; + const char *driver = crypto_ahash_driver_name(tfm); char _generic_driver[CRYPTO_MAX_ALG_NAME]; struct crypto_shash *generic_tfm = NULL; struct shash_desc *generic_desc = NULL; @@ -1732,7 +1729,7 @@ static int test_hash_vs_generic_impl(const char *driver, vec_name, sizeof(vec_name)); generate_random_testvec_config(cfg, cfgname, sizeof(cfgname)); - err = test_hash_vec_cfg(driver, &vec, vec_name, cfg, + err = test_hash_vec_cfg(&vec, vec_name, cfg, req, desc, tsgl, hashstate); if (err) goto out; @@ -1749,8 +1746,7 @@ out: return err; } #else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ -static int test_hash_vs_generic_impl(const char *driver, - const char *generic_driver, +static int test_hash_vs_generic_impl(const char *generic_driver, unsigned int maxkeysize, struct ahash_request *req, struct shash_desc *desc, @@ -1820,6 +1816,7 @@ static int __alg_test_hash(const struct hash_testvec *vecs, driver, PTR_ERR(atfm)); return PTR_ERR(atfm); } + driver = crypto_ahash_driver_name(atfm); req = ahash_request_alloc(atfm, GFP_KERNEL); if (!req) { @@ -1859,13 +1856,12 @@ static int __alg_test_hash(const struct hash_testvec *vecs, } for (i = 0; i < num_vecs; i++) { - err = test_hash_vec(driver, &vecs[i], i, req, desc, tsgl, - hashstate); + err = test_hash_vec(&vecs[i], i, req, desc, tsgl, hashstate); if (err) goto out; cond_resched(); } - err = test_hash_vs_generic_impl(driver, generic_driver, maxkeysize, req, + err = test_hash_vs_generic_impl(generic_driver, maxkeysize, req, desc, tsgl, hashstate); out: kfree(hashstate); @@ -1923,8 +1919,7 @@ static int alg_test_hash(const struct alg_test_desc *desc, const char *driver, return err; } -static int test_aead_vec_cfg(const char *driver, int enc, - const struct aead_testvec *vec, +static int test_aead_vec_cfg(int enc, const struct aead_testvec *vec, const char *vec_name, const struct testvec_config *cfg, struct aead_request *req, @@ -1934,6 +1929,7 @@ static int test_aead_vec_cfg(const char *driver, int enc, const unsigned int alignmask = crypto_aead_alignmask(tfm); const unsigned int ivsize = crypto_aead_ivsize(tfm); const unsigned int authsize = vec->clen - vec->plen; + const char *driver = crypto_aead_driver_name(tfm); const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags; const char *op = enc ? "encryption" : "decryption"; DECLARE_CRYPTO_WAIT(wait); @@ -2106,9 +2102,8 @@ static int test_aead_vec_cfg(const char *driver, int enc, return 0; } -static int test_aead_vec(const char *driver, int enc, - const struct aead_testvec *vec, unsigned int vec_num, - struct aead_request *req, +static int test_aead_vec(int enc, const struct aead_testvec *vec, + unsigned int vec_num, struct aead_request *req, struct cipher_test_sglists *tsgls) { char vec_name[16]; @@ -2121,7 +2116,7 @@ static int test_aead_vec(const char *driver, int enc, sprintf(vec_name, "%u", vec_num); for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++) { - err = test_aead_vec_cfg(driver, enc, vec, vec_name, + err = test_aead_vec_cfg(enc, vec, vec_name, &default_cipher_testvec_configs[i], req, tsgls); if (err) @@ -2136,7 +2131,7 @@ static int test_aead_vec(const char *driver, int enc, for (i = 0; i < fuzz_iterations; i++) { generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); - err = test_aead_vec_cfg(driver, enc, vec, vec_name, + err = test_aead_vec_cfg(enc, vec, vec_name, &cfg, req, tsgls); if (err) return err; @@ -2152,7 +2147,6 @@ static int test_aead_vec(const char *driver, int enc, struct aead_extra_tests_ctx { struct aead_request *req; struct crypto_aead *tfm; - const char *driver; const struct alg_test_desc *test_desc; struct cipher_test_sglists *tsgls; unsigned int maxdatasize; @@ -2358,7 +2352,7 @@ static int test_aead_inauthentic_inputs(struct aead_extra_tests_ctx *ctx) if (ctx->vec.novrfy) { generate_random_testvec_config(&ctx->cfg, ctx->cfgname, sizeof(ctx->cfgname)); - err = test_aead_vec_cfg(ctx->driver, DECRYPT, &ctx->vec, + err = test_aead_vec_cfg(DECRYPT, &ctx->vec, ctx->vec_name, &ctx->cfg, ctx->req, ctx->tsgls); if (err) @@ -2377,7 +2371,7 @@ static int test_aead_vs_generic_impl(struct aead_extra_tests_ctx *ctx) { struct crypto_aead *tfm = ctx->tfm; const char *algname = crypto_aead_alg(tfm)->base.cra_name; - const char *driver = ctx->driver; + const char *driver = crypto_aead_driver_name(tfm); const char *generic_driver = ctx->test_desc->generic_driver; char _generic_driver[CRYPTO_MAX_ALG_NAME]; struct crypto_aead *generic_tfm = NULL; @@ -2454,14 +2448,14 @@ static int test_aead_vs_generic_impl(struct aead_extra_tests_ctx *ctx) generate_random_testvec_config(&ctx->cfg, ctx->cfgname, sizeof(ctx->cfgname)); if (!ctx->vec.novrfy) { - err = test_aead_vec_cfg(driver, ENCRYPT, &ctx->vec, + err = test_aead_vec_cfg(ENCRYPT, &ctx->vec, ctx->vec_name, &ctx->cfg, ctx->req, ctx->tsgls); if (err) goto out; } if (ctx->vec.crypt_error == 0 || ctx->vec.novrfy) { - err = test_aead_vec_cfg(driver, DECRYPT, &ctx->vec, + err = test_aead_vec_cfg(DECRYPT, &ctx->vec, ctx->vec_name, &ctx->cfg, ctx->req, ctx->tsgls); if (err) @@ -2476,8 +2470,7 @@ out: return err; } -static int test_aead_extra(const char *driver, - const struct alg_test_desc *test_desc, +static int test_aead_extra(const struct alg_test_desc *test_desc, struct aead_request *req, struct cipher_test_sglists *tsgls) { @@ -2493,7 +2486,6 @@ static int test_aead_extra(const char *driver, return -ENOMEM; ctx->req = req; ctx->tfm = crypto_aead_reqtfm(req); - ctx->driver = driver; ctx->test_desc = test_desc; ctx->tsgls = tsgls; ctx->maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN; @@ -2528,8 +2520,7 @@ out: return err; } #else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ -static int test_aead_extra(const char *driver, - const struct alg_test_desc *test_desc, +static int test_aead_extra(const struct alg_test_desc *test_desc, struct aead_request *req, struct cipher_test_sglists *tsgls) { @@ -2537,8 +2528,7 @@ static int test_aead_extra(const char *driver, } #endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ -static int test_aead(const char *driver, int enc, - const struct aead_test_suite *suite, +static int test_aead(int enc, const struct aead_test_suite *suite, struct aead_request *req, struct cipher_test_sglists *tsgls) { @@ -2546,8 +2536,7 @@ static int test_aead(const char *driver, int enc, int err; for (i = 0; i < suite->count; i++) { - err = test_aead_vec(driver, enc, &suite->vecs[i], i, req, - tsgls); + err = test_aead_vec(enc, &suite->vecs[i], i, req, tsgls); if (err) return err; cond_resched(); @@ -2575,6 +2564,7 @@ static int alg_test_aead(const struct alg_test_desc *desc, const char *driver, driver, PTR_ERR(tfm)); return PTR_ERR(tfm); } + driver = crypto_aead_driver_name(tfm); req = aead_request_alloc(tfm, GFP_KERNEL); if (!req) { @@ -2592,15 +2582,15 @@ static int alg_test_aead(const struct alg_test_desc *desc, const char *driver, goto out; } - err = test_aead(driver, ENCRYPT, suite, req, tsgls); + err = test_aead(ENCRYPT, suite, req, tsgls); if (err) goto out; - err = test_aead(driver, DECRYPT, suite, req, tsgls); + err = test_aead(DECRYPT, suite, req, tsgls); if (err) goto out; - err = test_aead_extra(driver, desc, req, tsgls); + err = test_aead_extra(desc, req, tsgls); out: free_cipher_test_sglists(tsgls); aead_request_free(req); @@ -2695,8 +2685,7 @@ out_nobuf: return ret; } -static int test_skcipher_vec_cfg(const char *driver, int enc, - const struct cipher_testvec *vec, +static int test_skcipher_vec_cfg(int enc, const struct cipher_testvec *vec, const char *vec_name, const struct testvec_config *cfg, struct skcipher_request *req, @@ -2705,6 +2694,7 @@ static int test_skcipher_vec_cfg(const char *driver, int enc, struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); const unsigned int alignmask = crypto_skcipher_alignmask(tfm); const unsigned int ivsize = crypto_skcipher_ivsize(tfm); + const char *driver = crypto_skcipher_driver_name(tfm); const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags; const char *op = enc ? "encryption" : "decryption"; DECLARE_CRYPTO_WAIT(wait); @@ -2859,8 +2849,7 @@ static int test_skcipher_vec_cfg(const char *driver, int enc, return 0; } -static int test_skcipher_vec(const char *driver, int enc, - const struct cipher_testvec *vec, +static int test_skcipher_vec(int enc, const struct cipher_testvec *vec, unsigned int vec_num, struct skcipher_request *req, struct cipher_test_sglists *tsgls) @@ -2875,7 +2864,7 @@ static int test_skcipher_vec(const char *driver, int enc, sprintf(vec_name, "%u", vec_num); for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++) { - err = test_skcipher_vec_cfg(driver, enc, vec, vec_name, + err = test_skcipher_vec_cfg(enc, vec, vec_name, &default_cipher_testvec_configs[i], req, tsgls); if (err) @@ -2890,7 +2879,7 @@ static int test_skcipher_vec(const char *driver, int enc, for (i = 0; i < fuzz_iterations; i++) { generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); - err = test_skcipher_vec_cfg(driver, enc, vec, vec_name, + err = test_skcipher_vec_cfg(enc, vec, vec_name, &cfg, req, tsgls); if (err) return err; @@ -2961,8 +2950,7 @@ done: * Test the skcipher algorithm represented by @req against the corresponding * generic implementation, if one is available. */ -static int test_skcipher_vs_generic_impl(const char *driver, - const char *generic_driver, +static int test_skcipher_vs_generic_impl(const char *generic_driver, struct skcipher_request *req, struct cipher_test_sglists *tsgls) { @@ -2972,6 +2960,7 @@ static int test_skcipher_vs_generic_impl(const char *driver, const unsigned int blocksize = crypto_skcipher_blocksize(tfm); const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN; const char *algname = crypto_skcipher_alg(tfm)->base.cra_name; + const char *driver = crypto_skcipher_driver_name(tfm); char _generic_driver[CRYPTO_MAX_ALG_NAME]; struct crypto_skcipher *generic_tfm = NULL; struct skcipher_request *generic_req = NULL; @@ -3077,11 +3066,11 @@ static int test_skcipher_vs_generic_impl(const char *driver, vec_name, sizeof(vec_name)); generate_random_testvec_config(cfg, cfgname, sizeof(cfgname)); - err = test_skcipher_vec_cfg(driver, ENCRYPT, &vec, vec_name, + err = test_skcipher_vec_cfg(ENCRYPT, &vec, vec_name, cfg, req, tsgls); if (err) goto out; - err = test_skcipher_vec_cfg(driver, DECRYPT, &vec, vec_name, + err = test_skcipher_vec_cfg(DECRYPT, &vec, vec_name, cfg, req, tsgls); if (err) goto out; @@ -3099,8 +3088,7 @@ out: return err; } #else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ -static int test_skcipher_vs_generic_impl(const char *driver, - const char *generic_driver, +static int test_skcipher_vs_generic_impl(const char *generic_driver, struct skcipher_request *req, struct cipher_test_sglists *tsgls) { @@ -3108,8 +3096,7 @@ static int test_skcipher_vs_generic_impl(const char *driver, } #endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ -static int test_skcipher(const char *driver, int enc, - const struct cipher_test_suite *suite, +static int test_skcipher(int enc, const struct cipher_test_suite *suite, struct skcipher_request *req, struct cipher_test_sglists *tsgls) { @@ -3117,8 +3104,7 @@ static int test_skcipher(const char *driver, int enc, int err; for (i = 0; i < suite->count; i++) { - err = test_skcipher_vec(driver, enc, &suite->vecs[i], i, req, - tsgls); + err = test_skcipher_vec(enc, &suite->vecs[i], i, req, tsgls); if (err) return err; cond_resched(); @@ -3146,6 +3132,7 @@ static int alg_test_skcipher(const struct alg_test_desc *desc, driver, PTR_ERR(tfm)); return PTR_ERR(tfm); } + driver = crypto_skcipher_driver_name(tfm); req = skcipher_request_alloc(tfm, GFP_KERNEL); if (!req) { @@ -3163,16 +3150,15 @@ static int alg_test_skcipher(const struct alg_test_desc *desc, goto out; } - err = test_skcipher(driver, ENCRYPT, suite, req, tsgls); + err = test_skcipher(ENCRYPT, suite, req, tsgls); if (err) goto out; - err = test_skcipher(driver, DECRYPT, suite, req, tsgls); + err = test_skcipher(DECRYPT, suite, req, tsgls); if (err) goto out; - err = test_skcipher_vs_generic_impl(driver, desc->generic_driver, req, - tsgls); + err = test_skcipher_vs_generic_impl(desc->generic_driver, req, tsgls); out: free_cipher_test_sglists(tsgls); skcipher_request_free(req); @@ -3602,6 +3588,7 @@ static int alg_test_crc32c(const struct alg_test_desc *desc, "%ld\n", driver, PTR_ERR(tfm)); return PTR_ERR(tfm); } + driver = crypto_shash_driver_name(tfm); do { SHASH_DESC_ON_STACK(shash, tfm); @@ -5677,15 +5664,21 @@ int alg_test(const char *driver, const char *alg, u32 type, u32 mask) type, mask); test_done: - if (rc && (fips_enabled || panic_on_fail)) { - fips_fail_notify(); - panic("alg: self-tests for %s (%s) failed in %s mode!\n", - driver, alg, fips_enabled ? "fips" : "panic_on_fail"); + if (rc) { + if (fips_enabled || panic_on_fail) { + fips_fail_notify(); + panic("alg: self-tests for %s (%s) failed in %s mode!\n", + driver, alg, + fips_enabled ? "fips" : "panic_on_fail"); + } + WARN(1, "alg: self-tests for %s (%s) failed (rc=%d)", + driver, alg, rc); + } else { + if (fips_enabled) + pr_info("alg: self-tests for %s (%s) passed\n", + driver, alg); } - if (fips_enabled && !rc) - pr_info("alg: self-tests for %s (%s) passed\n", driver, alg); - return rc; notest: diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig index e92c4d9469d8..17c1df8c909a 100644 --- a/drivers/char/hw_random/Kconfig +++ b/drivers/char/hw_random/Kconfig @@ -348,19 +348,6 @@ config HW_RANDOM_HISI If unsure, say Y. -config HW_RANDOM_HISI_V2 - tristate "HiSilicon True Random Number Generator V2 support" - depends on HW_RANDOM && ARM64 && ACPI - default HW_RANDOM - help - This driver provides kernel-side support for the True Random Number - Generator V2 hardware found on HiSilicon Hi1620 SoC. - - To compile this driver as a module, choose M here: the - module will be called hisi-trng-v2. - - If unsure, say Y. - config HW_RANDOM_ST tristate "ST Microelectronics HW Random Number Generator support" depends on HW_RANDOM && ARCH_STI @@ -508,6 +495,7 @@ config HW_RANDOM_NPCM config HW_RANDOM_KEYSTONE depends on ARCH_KEYSTONE || COMPILE_TEST + depends on HAS_IOMEM && OF default HW_RANDOM tristate "TI Keystone NETCP SA Hardware random number generator" help diff --git a/drivers/char/hw_random/Makefile b/drivers/char/hw_random/Makefile index 5da344509a4d..8933fada74f2 100644 --- a/drivers/char/hw_random/Makefile +++ b/drivers/char/hw_random/Makefile @@ -30,7 +30,6 @@ obj-$(CONFIG_HW_RANDOM_NOMADIK) += nomadik-rng.o obj-$(CONFIG_HW_RANDOM_PSERIES) += pseries-rng.o obj-$(CONFIG_HW_RANDOM_POWERNV) += powernv-rng.o obj-$(CONFIG_HW_RANDOM_HISI) += hisi-rng.o -obj-$(CONFIG_HW_RANDOM_HISI_V2) += hisi-trng-v2.o obj-$(CONFIG_HW_RANDOM_BCM2835) += bcm2835-rng.o obj-$(CONFIG_HW_RANDOM_IPROC_RNG200) += iproc-rng200.o obj-$(CONFIG_HW_RANDOM_ST) += st-rng.o diff --git a/drivers/char/hw_random/hisi-trng-v2.c b/drivers/char/hw_random/hisi-trng-v2.c deleted file mode 100644 index 6a65b8232ce0..000000000000 --- a/drivers/char/hw_random/hisi-trng-v2.c +++ /dev/null @@ -1,99 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* Copyright (c) 2019 HiSilicon Limited. */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define HISI_TRNG_REG 0x00F0 -#define HISI_TRNG_BYTES 4 -#define HISI_TRNG_QUALITY 512 -#define SLEEP_US 10 -#define TIMEOUT_US 10000 - -struct hisi_trng { - void __iomem *base; - struct hwrng rng; -}; - -static int hisi_trng_read(struct hwrng *rng, void *buf, size_t max, bool wait) -{ - struct hisi_trng *trng; - int currsize = 0; - u32 val = 0; - u32 ret; - - trng = container_of(rng, struct hisi_trng, rng); - - do { - ret = readl_poll_timeout(trng->base + HISI_TRNG_REG, val, - val, SLEEP_US, TIMEOUT_US); - if (ret) - return currsize; - - if (max - currsize >= HISI_TRNG_BYTES) { - memcpy(buf + currsize, &val, HISI_TRNG_BYTES); - currsize += HISI_TRNG_BYTES; - if (currsize == max) - return currsize; - continue; - } - - /* copy remaining bytes */ - memcpy(buf + currsize, &val, max - currsize); - currsize = max; - } while (currsize < max); - - return currsize; -} - -static int hisi_trng_probe(struct platform_device *pdev) -{ - struct hisi_trng *trng; - int ret; - - trng = devm_kzalloc(&pdev->dev, sizeof(*trng), GFP_KERNEL); - if (!trng) - return -ENOMEM; - - trng->base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(trng->base)) - return PTR_ERR(trng->base); - - trng->rng.name = pdev->name; - trng->rng.read = hisi_trng_read; - trng->rng.quality = HISI_TRNG_QUALITY; - - ret = devm_hwrng_register(&pdev->dev, &trng->rng); - if (ret) - dev_err(&pdev->dev, "failed to register hwrng!\n"); - - return ret; -} - -static const struct acpi_device_id hisi_trng_acpi_match[] = { - { "HISI02B3", 0 }, - { } -}; -MODULE_DEVICE_TABLE(acpi, hisi_trng_acpi_match); - -static struct platform_driver hisi_trng_driver = { - .probe = hisi_trng_probe, - .driver = { - .name = "hisi-trng-v2", - .acpi_match_table = ACPI_PTR(hisi_trng_acpi_match), - }, -}; - -module_platform_driver(hisi_trng_driver); - -MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Weili Qian "); -MODULE_AUTHOR("Zaibo Xu "); -MODULE_DESCRIPTION("HiSilicon true random number generator V2 driver"); diff --git a/drivers/char/hw_random/imx-rngc.c b/drivers/char/hw_random/imx-rngc.c index 61c844baf26e..b05d676ca814 100644 --- a/drivers/char/hw_random/imx-rngc.c +++ b/drivers/char/hw_random/imx-rngc.c @@ -252,10 +252,8 @@ static int imx_rngc_probe(struct platform_device *pdev) } irq = platform_get_irq(pdev, 0); - if (irq <= 0) { - dev_err(&pdev->dev, "Couldn't get irq %d\n", irq); + if (irq < 0) return irq; - } ret = clk_prepare_enable(rngc->clk); if (ret) diff --git a/drivers/char/hw_random/s390-trng.c b/drivers/char/hw_random/s390-trng.c index 413cacbb08e2..7c673afd7241 100644 --- a/drivers/char/hw_random/s390-trng.c +++ b/drivers/char/hw_random/s390-trng.c @@ -192,14 +192,15 @@ static int trng_hwrng_read(struct hwrng *rng, void *data, size_t max, bool wait) /* * hwrng register struct - * The trng is suppost to have 100% entropy, and thus - * we register with a very high quality value. + * The trng is supposed to have 100% entropy, and thus we register with a very + * high quality value. If we ever have a better driver in the future, we should + * change this value again when we merge this driver. */ static struct hwrng trng_hwrng_dev = { .name = "s390-trng", .data_read = trng_hwrng_data_read, .read = trng_hwrng_read, - .quality = 999, + .quality = 1024, }; diff --git a/drivers/char/random.c b/drivers/char/random.c index 2a41b21623ae..5f3b8ac9d97b 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -336,7 +336,7 @@ #include #include #include -#include +#include #include #include diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 37da0c070a88..bbd51703e738 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -548,6 +548,7 @@ config CRYPTO_DEV_ATMEL_SHA config CRYPTO_DEV_ATMEL_I2C tristate + select BITREVERSE config CRYPTO_DEV_ATMEL_ECC tristate "Support for Microchip / Atmel ECC hw accelerator" @@ -648,7 +649,7 @@ choice default CRYPTO_DEV_QCE_ENABLE_ALL depends on CRYPTO_DEV_QCE help - This option allows to choose whether to build support for all algorihtms + This option allows to choose whether to build support for all algorithms (default), hashes-only, or skciphers-only. The QCE engine does not appear to scale as well as the CPU to handle @@ -900,4 +901,6 @@ config CRYPTO_DEV_SA2UL used for crypto offload. Select this if you want to use hardware acceleration for cryptographic algorithms on these devices. +source "drivers/crypto/keembay/Kconfig" + endif # CRYPTO_HW diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index 53fc115cf459..fff9a70348e1 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -51,3 +51,4 @@ obj-$(CONFIG_CRYPTO_DEV_ARTPEC6) += axis/ obj-$(CONFIG_CRYPTO_DEV_ZYNQMP_AES) += xilinx/ obj-y += hisilicon/ obj-$(CONFIG_CRYPTO_DEV_AMLOGIC_GXL) += amlogic/ +obj-y += keembay/ diff --git a/drivers/crypto/allwinner/Kconfig b/drivers/crypto/allwinner/Kconfig index 0cdfe0e8cc66..180c8a9db819 100644 --- a/drivers/crypto/allwinner/Kconfig +++ b/drivers/crypto/allwinner/Kconfig @@ -43,7 +43,7 @@ config CRYPTO_DEV_SUN8I_CE depends on CRYPTO_DEV_ALLWINNER depends on PM help - Select y here to have support for the crypto Engine availlable on + Select y here to have support for the crypto Engine available on Allwinner SoC H2+, H3, H5, H6, R40 and A64. The Crypto Engine handle AES/3DES ciphers in ECB/CBC mode. diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h index 163962f9e284..5c291e4a6857 100644 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h @@ -25,7 +25,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c index a94bf28f858a..2f09a37306e2 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c @@ -13,7 +13,8 @@ #include #include #include -#include +#include +#include #include #include "sun8i-ce.h" @@ -262,13 +263,13 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) u32 common; u64 byte_count; __le32 *bf; - void *buf; + void *buf = NULL; int j, i, todo; int nbw = 0; u64 fill, min_fill; __be64 *bebits; __le64 *lebits; - void *result; + void *result = NULL; u64 bs; int digestsize; dma_addr_t addr_res, addr_pad; @@ -285,13 +286,17 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) /* the padding could be up to two block. */ buf = kzalloc(bs * 2, GFP_KERNEL | GFP_DMA); - if (!buf) - return -ENOMEM; + if (!buf) { + err = -ENOMEM; + goto theend; + } bf = (__le32 *)buf; result = kzalloc(digestsize, GFP_KERNEL | GFP_DMA); - if (!result) - return -ENOMEM; + if (!result) { + err = -ENOMEM; + goto theend; + } flow = rctx->flow; chan = &ce->chanlist[flow]; @@ -403,11 +408,11 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) dma_unmap_sg(ce->dev, areq->src, nr_sgs, DMA_TO_DEVICE); dma_unmap_single(ce->dev, addr_res, digestsize, DMA_FROM_DEVICE); - kfree(buf); memcpy(areq->result, result, algt->alg.hash.halg.digestsize); - kfree(result); theend: + kfree(buf); + kfree(result); crypto_finalize_hash_request(engine, breq, err); return 0; } diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h index 558027516aed..cec781d5063c 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h @@ -16,7 +16,8 @@ #include #include #include -#include +#include +#include /* CE Registers */ #define CE_TDQ 0x00 diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c index b6ab2054f217..11cbcbc83a7b 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c @@ -13,7 +13,8 @@ #include #include #include -#include +#include +#include #include #include "sun8i-ss.h" diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h index 1a66457f4a20..28188685b910 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h @@ -15,7 +15,8 @@ #include #include #include -#include +#include +#include #define SS_START 1 diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index 7729a637fb02..a3fa849b139a 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include "crypto4xx_reg_def.h" diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 981de43ea5e2..8d1b918a0533 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -30,7 +30,7 @@ #include #include #include -#include +#include #include #include #include @@ -917,7 +917,7 @@ int crypto4xx_build_pd(struct crypto_async_request *req, } pd->pd_ctl.w = PD_CTL_HOST_READY | - ((crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AHASH) | + ((crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AHASH) || (crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AEAD) ? PD_CTL_HASH_FINAL : 0); pd->pd_ctl_len.w = 0x00400000 | (assoclen + datalen); diff --git a/drivers/crypto/atmel-authenc.h b/drivers/crypto/atmel-authenc.h index c6530a1c8c20..45171e89a7d2 100644 --- a/drivers/crypto/atmel-authenc.h +++ b/drivers/crypto/atmel-authenc.h @@ -16,7 +16,8 @@ #include #include -#include +#include +#include #include "atmel-sha-regs.h" struct atmel_aes_dev; diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c index 75ccf41a7cb9..352d80cb5ae9 100644 --- a/drivers/crypto/atmel-sha.c +++ b/drivers/crypto/atmel-sha.c @@ -33,7 +33,8 @@ #include #include #include -#include +#include +#include #include #include #include "atmel-sha-regs.h" @@ -459,7 +460,6 @@ static int atmel_sha_init(struct ahash_request *req) break; default: return -EINVAL; - break; } ctx->bufcnt = 0; diff --git a/drivers/crypto/axis/artpec6_crypto.c b/drivers/crypto/axis/artpec6_crypto.c index 809c3033ca74..9ad188cffd0d 100644 --- a/drivers/crypto/axis/artpec6_crypto.c +++ b/drivers/crypto/axis/artpec6_crypto.c @@ -28,7 +28,8 @@ #include #include #include -#include +#include +#include #include /* Max length of a line in all cache levels for Artpec SoCs. */ diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c index 50d169e61b41..30390a7324b2 100644 --- a/drivers/crypto/bcm/cipher.c +++ b/drivers/crypto/bcm/cipher.c @@ -26,11 +26,12 @@ #include #include #include -#include #include #include #include #include +#include +#include #include #include "util.h" diff --git a/drivers/crypto/bcm/cipher.h b/drivers/crypto/bcm/cipher.h index 035c8389cb3d..0ad5892b445d 100644 --- a/drivers/crypto/bcm/cipher.h +++ b/drivers/crypto/bcm/cipher.h @@ -16,7 +16,8 @@ #include #include #include -#include +#include +#include #include #include "spu.h" diff --git a/drivers/crypto/bcm/spu.h b/drivers/crypto/bcm/spu.h index dd132389bcaa..1c386a2d5506 100644 --- a/drivers/crypto/bcm/spu.h +++ b/drivers/crypto/bcm/spu.h @@ -17,7 +17,8 @@ #include #include -#include +#include +#include enum spu_cipher_alg { CIPHER_ALG_NONE = 0x0, diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index cf5bd7666dfc..8697ae53b063 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -3404,8 +3404,8 @@ static int caam_cra_init(struct crypto_skcipher *tfm) fallback = crypto_alloc_skcipher(tfm_name, 0, CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(fallback)) { - dev_err(ctx->jrdev, "Failed to allocate %s fallback: %ld\n", - tfm_name, PTR_ERR(fallback)); + pr_err("Failed to allocate %s fallback: %ld\n", + tfm_name, PTR_ERR(fallback)); return PTR_ERR(fallback); } diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c index 66f60d78bdc8..189a7438b29c 100644 --- a/drivers/crypto/caam/caamalg_qi.c +++ b/drivers/crypto/caam/caamalg_qi.c @@ -852,7 +852,7 @@ static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx, cpu = smp_processor_id(); drv_ctx = caam_drv_ctx_init(ctx->qidev, &cpu, desc); - if (!IS_ERR_OR_NULL(drv_ctx)) + if (!IS_ERR(drv_ctx)) drv_ctx->op_type = type; ctx->drv_ctx[type] = drv_ctx; @@ -955,7 +955,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, struct caam_drv_ctx *drv_ctx; drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT); - if (IS_ERR_OR_NULL(drv_ctx)) + if (IS_ERR(drv_ctx)) return (struct aead_edesc *)drv_ctx; /* allocate space for base edesc and hw desc commands, link tables */ @@ -1165,7 +1165,7 @@ static inline int aead_crypt(struct aead_request *req, bool encrypt) /* allocate extended descriptor */ edesc = aead_edesc_alloc(req, encrypt); - if (IS_ERR_OR_NULL(edesc)) + if (IS_ERR(edesc)) return PTR_ERR(edesc); /* Create and submit job descriptor */ @@ -1259,7 +1259,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, struct caam_drv_ctx *drv_ctx; drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT); - if (IS_ERR_OR_NULL(drv_ctx)) + if (IS_ERR(drv_ctx)) return (struct skcipher_edesc *)drv_ctx; src_nents = sg_nents_for_len(req->src, req->cryptlen); @@ -2502,8 +2502,8 @@ static int caam_cra_init(struct crypto_skcipher *tfm) fallback = crypto_alloc_skcipher(tfm_name, 0, CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(fallback)) { - dev_err(ctx->jrdev, "Failed to allocate %s fallback: %ld\n", - tfm_name, PTR_ERR(fallback)); + pr_err("Failed to allocate %s fallback: %ld\n", + tfm_name, PTR_ERR(fallback)); return PTR_ERR(fallback); } diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c index 98c1ff1744bb..a780e627838a 100644 --- a/drivers/crypto/caam/caamalg_qi2.c +++ b/drivers/crypto/caam/caamalg_qi2.c @@ -1611,7 +1611,8 @@ static int caam_cra_init_skcipher(struct crypto_skcipher *tfm) fallback = crypto_alloc_skcipher(tfm_name, 0, CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(fallback)) { - dev_err(ctx->dev, "Failed to allocate %s fallback: %ld\n", + dev_err(caam_alg->caam.dev, + "Failed to allocate %s fallback: %ld\n", tfm_name, PTR_ERR(fallback)); return PTR_ERR(fallback); } diff --git a/drivers/crypto/caam/compat.h b/drivers/crypto/caam/compat.h index c3c22a8de4c0..c4f79764172b 100644 --- a/drivers/crypto/caam/compat.h +++ b/drivers/crypto/caam/compat.h @@ -34,7 +34,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h index 9112279a4de0..7d45b21bd55a 100644 --- a/drivers/crypto/caam/intern.h +++ b/drivers/crypto/caam/intern.h @@ -16,6 +16,14 @@ /* Currently comes from Kconfig param as a ^2 (driver-required) */ #define JOBR_DEPTH (1 << CONFIG_CRYPTO_DEV_FSL_CAAM_RINGSIZE) +/* + * Maximum size for crypto-engine software queue based on Job Ring + * size (JOBR_DEPTH) and a THRESHOLD (reserved for the non-crypto-API + * requests that are not passed through crypto-engine) + */ +#define THRESHOLD 15 +#define CRYPTO_ENGINE_MAX_QLEN (JOBR_DEPTH - THRESHOLD) + /* Kconfig params for interrupt coalescing if selected (else zero) */ #ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_INTC #define JOBR_INTC JRCFG_ICEN diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c index 6f669966ba2c..7f2b1101f567 100644 --- a/drivers/crypto/caam/jr.c +++ b/drivers/crypto/caam/jr.c @@ -550,7 +550,9 @@ static int caam_jr_probe(struct platform_device *pdev) } /* Initialize crypto engine */ - jrpriv->engine = crypto_engine_alloc_init(jrdev, false); + jrpriv->engine = crypto_engine_alloc_init_and_set(jrdev, true, NULL, + false, + CRYPTO_ENGINE_MAX_QLEN); if (!jrpriv->engine) { dev_err(jrdev, "Could not init crypto-engine\n"); return -ENOMEM; diff --git a/drivers/crypto/cavium/cpt/cptpf_main.c b/drivers/crypto/cavium/cpt/cptpf_main.c index 781949027451..711b1acdd4e0 100644 --- a/drivers/crypto/cavium/cpt/cptpf_main.c +++ b/drivers/crypto/cavium/cpt/cptpf_main.c @@ -244,7 +244,7 @@ cpt_init_fail: struct ucode_header { u8 version[CPT_UCODE_VERSION_SZ]; - u32 code_length; + __be32 code_length; u32 data_length; u64 sram_address; }; @@ -288,10 +288,10 @@ static int cpt_ucode_load_fw(struct cpt_device *cpt, const u8 *fw, bool is_ae) /* Byte swap 64-bit */ for (j = 0; j < (mcode->code_size / 8); j++) - ((u64 *)mcode->code)[j] = cpu_to_be64(((u64 *)mcode->code)[j]); + ((__be64 *)mcode->code)[j] = cpu_to_be64(((u64 *)mcode->code)[j]); /* MC needs 16-bit swap */ for (j = 0; j < (mcode->code_size / 2); j++) - ((u16 *)mcode->code)[j] = cpu_to_be16(((u16 *)mcode->code)[j]); + ((__be16 *)mcode->code)[j] = cpu_to_be16(((u16 *)mcode->code)[j]); dev_dbg(dev, "mcode->code_size = %u\n", mcode->code_size); dev_dbg(dev, "mcode->is_ae = %u\n", mcode->is_ae); @@ -569,15 +569,9 @@ static int cpt_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto cpt_err_disable_device; } - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48)); + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); if (err) { - dev_err(dev, "Unable to get usable DMA configuration\n"); - goto cpt_err_release_regions; - } - - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48)); - if (err) { - dev_err(dev, "Unable to get 48-bit DMA for consistent allocations\n"); + dev_err(dev, "Unable to get usable 48-bit DMA configuration\n"); goto cpt_err_release_regions; } diff --git a/drivers/crypto/cavium/cpt/cptvf_main.c b/drivers/crypto/cavium/cpt/cptvf_main.c index a15245992cf9..f016448e43bb 100644 --- a/drivers/crypto/cavium/cpt/cptvf_main.c +++ b/drivers/crypto/cavium/cpt/cptvf_main.c @@ -687,15 +687,9 @@ static int cptvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } /* Mark as VF driver */ cptvf->flags |= CPT_FLAG_VF_DRIVER; - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48)); + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); if (err) { - dev_err(dev, "Unable to get usable DMA configuration\n"); - goto cptvf_err_release_regions; - } - - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48)); - if (err) { - dev_err(dev, "Unable to get 48-bit DMA for consistent allocations\n"); + dev_err(dev, "Unable to get usable 48-bit DMA configuration\n"); goto cptvf_err_release_regions; } diff --git a/drivers/crypto/cavium/nitrox/nitrox_aead.c b/drivers/crypto/cavium/nitrox/nitrox_aead.c index 1be2571363fe..c93c4e41d267 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_aead.c +++ b/drivers/crypto/cavium/nitrox/nitrox_aead.c @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include @@ -45,9 +44,9 @@ static int nitrox_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key, /* fill crypto context */ fctx = nctx->u.fctx; - flags.f = be64_to_cpu(fctx->flags.f); + flags.fu = be64_to_cpu(fctx->flags.f); flags.w0.aes_keylen = aes_keylen; - fctx->flags.f = cpu_to_be64(flags.f); + fctx->flags.f = cpu_to_be64(flags.fu); /* copy enc key to context */ memset(&fctx->crypto, 0, sizeof(fctx->crypto)); @@ -63,9 +62,9 @@ static int nitrox_aead_setauthsize(struct crypto_aead *aead, struct flexi_crypto_context *fctx = nctx->u.fctx; union fc_ctx_flags flags; - flags.f = be64_to_cpu(fctx->flags.f); + flags.fu = be64_to_cpu(fctx->flags.f); flags.w0.mac_len = authsize; - fctx->flags.f = cpu_to_be64(flags.f); + fctx->flags.f = cpu_to_be64(flags.fu); aead->authsize = authsize; @@ -319,7 +318,7 @@ static int nitrox_gcm_common_init(struct crypto_aead *aead) flags->w0.iv_source = IV_FROM_DPTR; /* ask microcode to calculate ipad/opad */ flags->w0.auth_input_type = 1; - flags->f = be64_to_cpu(flags->f); + flags->f = cpu_to_be64(flags->fu); return 0; } diff --git a/drivers/crypto/cavium/nitrox/nitrox_debugfs.c b/drivers/crypto/cavium/nitrox/nitrox_debugfs.c index 16f7d0bd1303..741572a01995 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_debugfs.c +++ b/drivers/crypto/cavium/nitrox/nitrox_debugfs.c @@ -3,6 +3,7 @@ #include #include "nitrox_csr.h" +#include "nitrox_debugfs.h" #include "nitrox_dev.h" static int firmware_show(struct seq_file *s, void *v) diff --git a/drivers/crypto/cavium/nitrox/nitrox_hal.c b/drivers/crypto/cavium/nitrox/nitrox_hal.c index 34a2f4f30a7e..13b137410b75 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_hal.c +++ b/drivers/crypto/cavium/nitrox/nitrox_hal.c @@ -3,6 +3,7 @@ #include "nitrox_dev.h" #include "nitrox_csr.h" +#include "nitrox_hal.h" #define PLL_REF_CLK 50 #define MAX_CSR_RETRIES 10 diff --git a/drivers/crypto/cavium/nitrox/nitrox_isr.c b/drivers/crypto/cavium/nitrox/nitrox_isr.c index 3dec570a190a..99b053094f5a 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_isr.c +++ b/drivers/crypto/cavium/nitrox/nitrox_isr.c @@ -7,6 +7,7 @@ #include "nitrox_csr.h" #include "nitrox_common.h" #include "nitrox_hal.h" +#include "nitrox_isr.h" #include "nitrox_mbx.h" /** diff --git a/drivers/crypto/cavium/nitrox/nitrox_isr.h b/drivers/crypto/cavium/nitrox/nitrox_isr.h index 1062c9336c1f..2bb123cd2f1c 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_isr.h +++ b/drivers/crypto/cavium/nitrox/nitrox_isr.h @@ -9,4 +9,13 @@ void nitrox_unregister_interrupts(struct nitrox_device *ndev); int nitrox_sriov_register_interupts(struct nitrox_device *ndev); void nitrox_sriov_unregister_interrupts(struct nitrox_device *ndev); +#ifdef CONFIG_PCI_IOV +int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs); +#else +static inline int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs) +{ + return 0; +} +#endif + #endif /* __NITROX_ISR_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c index 9d14be97e381..facc8e6bc580 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_main.c +++ b/drivers/crypto/cavium/nitrox/nitrox_main.c @@ -49,15 +49,6 @@ static unsigned int qlen = DEFAULT_CMD_QLEN; module_param(qlen, uint, 0644); MODULE_PARM_DESC(qlen, "Command queue length - default 2048"); -#ifdef CONFIG_PCI_IOV -int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs); -#else -int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs) -{ - return 0; -} -#endif - /** * struct ucode - Firmware Header * @id: microcode ID @@ -555,10 +546,8 @@ static void nitrox_remove(struct pci_dev *pdev) nitrox_remove_from_devlist(ndev); -#ifdef CONFIG_PCI_IOV /* disable SR-IOV */ nitrox_sriov_configure(pdev, 0); -#endif nitrox_crypto_unregister(); nitrox_debugfs_exit(ndev); nitrox_pf_sw_cleanup(ndev); @@ -584,9 +573,7 @@ static struct pci_driver nitrox_driver = { .probe = nitrox_probe, .remove = nitrox_remove, .shutdown = nitrox_shutdown, -#ifdef CONFIG_PCI_IOV .sriov_configure = nitrox_sriov_configure, -#endif }; module_pci_driver(nitrox_driver); diff --git a/drivers/crypto/cavium/nitrox/nitrox_mbx.c b/drivers/crypto/cavium/nitrox/nitrox_mbx.c index b51b0449b478..c1af9d4fca6e 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_mbx.c +++ b/drivers/crypto/cavium/nitrox/nitrox_mbx.c @@ -4,6 +4,7 @@ #include "nitrox_csr.h" #include "nitrox_hal.h" #include "nitrox_dev.h" +#include "nitrox_mbx.h" #define RING_TO_VFNO(_x, _y) ((_x) / (_y)) @@ -112,7 +113,7 @@ static void pf2vf_resp_handler(struct work_struct *work) case MBX_MSG_TYPE_ACK: case MBX_MSG_TYPE_NACK: break; - }; + } kfree(pf2vf_resp); } diff --git a/drivers/crypto/cavium/nitrox/nitrox_req.h b/drivers/crypto/cavium/nitrox/nitrox_req.h index 12282c1b14f5..ed174883c8e3 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_req.h +++ b/drivers/crypto/cavium/nitrox/nitrox_req.h @@ -149,6 +149,7 @@ struct auth_keys { union fc_ctx_flags { __be64 f; + u64 fu; struct { #if defined(__BIG_ENDIAN_BITFIELD) u64 cipher_type : 4; @@ -280,6 +281,7 @@ struct nitrox_rfc4106_rctx { * - packet payload bytes */ union pkt_instr_hdr { + __be64 bev; u64 value; struct { #if defined(__BIG_ENDIAN_BITFIELD) @@ -324,6 +326,7 @@ union pkt_instr_hdr { * @ctxp: Context pointer. CTXP<63,2:0> must be zero in all cases. */ union pkt_hdr { + __be64 bev[2]; u64 value[2]; struct { #if defined(__BIG_ENDIAN_BITFIELD) @@ -370,6 +373,7 @@ union pkt_hdr { * sglist components at [RPTR] on the remote host. */ union slc_store_info { + __be64 bev[2]; u64 value[2]; struct { #if defined(__BIG_ENDIAN_BITFIELD) diff --git a/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c index 5826c2c98a50..53ef06792133 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c +++ b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c @@ -3,6 +3,7 @@ #include #include +#include "nitrox_common.h" #include "nitrox_dev.h" #include "nitrox_req.h" #include "nitrox_csr.h" @@ -448,7 +449,7 @@ int nitrox_process_se_request(struct nitrox_device *ndev, sr->instr.ih.s.ssz = sr->out.sgmap_cnt; sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr); sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes; - sr->instr.ih.value = cpu_to_be64(sr->instr.ih.value); + sr->instr.ih.bev = cpu_to_be64(sr->instr.ih.value); /* word 2 */ sr->instr.irh.value[0] = 0; @@ -460,7 +461,7 @@ int nitrox_process_se_request(struct nitrox_device *ndev, sr->instr.irh.s.ctxc = req->ctrl.s.ctxc; sr->instr.irh.s.arg = req->ctrl.s.arg; sr->instr.irh.s.opcode = req->opcode; - sr->instr.irh.value[0] = cpu_to_be64(sr->instr.irh.value[0]); + sr->instr.irh.bev[0] = cpu_to_be64(sr->instr.irh.value[0]); /* word 3 */ sr->instr.irh.s.ctxp = cpu_to_be64(ctx_handle); @@ -468,7 +469,7 @@ int nitrox_process_se_request(struct nitrox_device *ndev, /* word 4 */ sr->instr.slc.value[0] = 0; sr->instr.slc.s.ssz = sr->out.sgmap_cnt; - sr->instr.slc.value[0] = cpu_to_be64(sr->instr.slc.value[0]); + sr->instr.slc.bev[0] = cpu_to_be64(sr->instr.slc.value[0]); /* word 5 */ sr->instr.slc.s.rptr = cpu_to_be64(sr->out.sgcomp_dma); diff --git a/drivers/crypto/cavium/zip/zip_main.c b/drivers/crypto/cavium/zip/zip_main.c index d35216e2f6cd..812b4ac9afd6 100644 --- a/drivers/crypto/cavium/zip/zip_main.c +++ b/drivers/crypto/cavium/zip/zip_main.c @@ -263,15 +263,9 @@ static int zip_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto err_disable_device; } - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48)); + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); if (err) { - dev_err(dev, "Unable to get usable DMA configuration\n"); - goto err_release_regions; - } - - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48)); - if (err) { - dev_err(dev, "Unable to get 48-bit DMA for allocations\n"); + dev_err(dev, "Unable to get usable 48-bit DMA configuration\n"); goto err_release_regions; } diff --git a/drivers/crypto/ccp/ccp-crypto-sha.c b/drivers/crypto/ccp/ccp-crypto-sha.c index 8fbfdb9e8cd3..74fa5360e722 100644 --- a/drivers/crypto/ccp/ccp-crypto-sha.c +++ b/drivers/crypto/ccp/ccp-crypto-sha.c @@ -17,7 +17,8 @@ #include #include #include -#include +#include +#include #include #include diff --git a/drivers/crypto/ccp/ccp-crypto.h b/drivers/crypto/ccp/ccp-crypto.h index aed3d2192d01..e42450d07168 100644 --- a/drivers/crypto/ccp/ccp-crypto.h +++ b/drivers/crypto/ccp/ccp-crypto.h @@ -19,7 +19,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/ccree/cc_cipher.c b/drivers/crypto/ccree/cc_cipher.c index dafa6577a845..cdfee501fbd9 100644 --- a/drivers/crypto/ccree/cc_cipher.c +++ b/drivers/crypto/ccree/cc_cipher.c @@ -97,6 +97,7 @@ static int validate_keys_sizes(struct cc_cipher_ctx *ctx_p, u32 size) case S_DIN_to_SM4: if (size == SM4_KEY_SIZE) return 0; + break; default: break; } @@ -139,9 +140,11 @@ static int validate_data_size(struct cc_cipher_ctx *ctx_p, case DRV_CIPHER_CBC: if (IS_ALIGNED(size, SM4_BLOCK_SIZE)) return 0; + break; default: break; } + break; default: break; } diff --git a/drivers/crypto/ccree/cc_driver.c b/drivers/crypto/ccree/cc_driver.c index 6f519d3e896c..d0e59e942568 100644 --- a/drivers/crypto/ccree/cc_driver.c +++ b/drivers/crypto/ccree/cc_driver.c @@ -100,6 +100,57 @@ static const struct of_device_id arm_ccree_dev_of_match[] = { }; MODULE_DEVICE_TABLE(of, arm_ccree_dev_of_match); +static void init_cc_cache_params(struct cc_drvdata *drvdata) +{ + struct device *dev = drvdata_to_dev(drvdata); + u32 cache_params, ace_const, val, mask; + + /* compute CC_AXIM_CACHE_PARAMS */ + cache_params = cc_ioread(drvdata, CC_REG(AXIM_CACHE_PARAMS)); + dev_dbg(dev, "Cache params previous: 0x%08X\n", cache_params); + + /* non cached or write-back, write allocate */ + val = drvdata->coherent ? 0xb : 0x2; + + mask = CC_GENMASK(CC_AXIM_CACHE_PARAMS_AWCACHE); + cache_params &= ~mask; + cache_params |= FIELD_PREP(mask, val); + + mask = CC_GENMASK(CC_AXIM_CACHE_PARAMS_AWCACHE_LAST); + cache_params &= ~mask; + cache_params |= FIELD_PREP(mask, val); + + mask = CC_GENMASK(CC_AXIM_CACHE_PARAMS_ARCACHE); + cache_params &= ~mask; + cache_params |= FIELD_PREP(mask, val); + + drvdata->cache_params = cache_params; + + dev_dbg(dev, "Cache params current: 0x%08X\n", cache_params); + + if (drvdata->hw_rev <= CC_HW_REV_710) + return; + + /* compute CC_AXIM_ACE_CONST */ + ace_const = cc_ioread(drvdata, CC_REG(AXIM_ACE_CONST)); + dev_dbg(dev, "ACE-const previous: 0x%08X\n", ace_const); + + /* system or outer-sharable */ + val = drvdata->coherent ? 0x2 : 0x3; + + mask = CC_GENMASK(CC_AXIM_ACE_CONST_ARDOMAIN); + ace_const &= ~mask; + ace_const |= FIELD_PREP(mask, val); + + mask = CC_GENMASK(CC_AXIM_ACE_CONST_AWDOMAIN); + ace_const &= ~mask; + ace_const |= FIELD_PREP(mask, val); + + dev_dbg(dev, "ACE-const current: 0x%08X\n", ace_const); + + drvdata->ace_const = ace_const; +} + static u32 cc_read_idr(struct cc_drvdata *drvdata, const u32 *idr_offsets) { int i; @@ -218,9 +269,9 @@ bool cc_wait_for_reset_completion(struct cc_drvdata *drvdata) return false; } -int init_cc_regs(struct cc_drvdata *drvdata, bool is_probe) +int init_cc_regs(struct cc_drvdata *drvdata) { - unsigned int val, cache_params; + unsigned int val; struct device *dev = drvdata_to_dev(drvdata); /* Unmask all AXI interrupt sources AXI_CFG1 register */ @@ -245,19 +296,9 @@ int init_cc_regs(struct cc_drvdata *drvdata, bool is_probe) cc_iowrite(drvdata, CC_REG(HOST_IMR), ~val); - cache_params = (drvdata->coherent ? CC_COHERENT_CACHE_PARAMS : 0x0); - - val = cc_ioread(drvdata, CC_REG(AXIM_CACHE_PARAMS)); - - if (is_probe) - dev_dbg(dev, "Cache params previous: 0x%08X\n", val); - - cc_iowrite(drvdata, CC_REG(AXIM_CACHE_PARAMS), cache_params); - val = cc_ioread(drvdata, CC_REG(AXIM_CACHE_PARAMS)); - - if (is_probe) - dev_dbg(dev, "Cache params current: 0x%08X (expect: 0x%08X)\n", - val, cache_params); + cc_iowrite(drvdata, CC_REG(AXIM_CACHE_PARAMS), drvdata->cache_params); + if (drvdata->hw_rev >= CC_HW_REV_712) + cc_iowrite(drvdata, CC_REG(AXIM_ACE_CONST), drvdata->ace_const); return 0; } @@ -445,7 +486,9 @@ static int init_cc_resources(struct platform_device *plat_dev) } dev_dbg(dev, "Registered to IRQ: %d\n", irq); - rc = init_cc_regs(new_drvdata, true); + init_cc_cache_params(new_drvdata); + + rc = init_cc_regs(new_drvdata); if (rc) { dev_err(dev, "init_cc_regs failed\n"); goto post_pm_err; diff --git a/drivers/crypto/ccree/cc_driver.h b/drivers/crypto/ccree/cc_driver.h index af77b2020350..5f1d4602eb8f 100644 --- a/drivers/crypto/ccree/cc_driver.h +++ b/drivers/crypto/ccree/cc_driver.h @@ -17,7 +17,8 @@ #include #include #include -#include +#include +#include #include #include #include @@ -49,8 +50,6 @@ enum cc_std_body { CC_STD_ALL = 0x3 }; -#define CC_COHERENT_CACHE_PARAMS 0xEEE - #define CC_PINS_FULL 0x0 #define CC_PINS_SLIM 0x9F @@ -155,6 +154,8 @@ struct cc_drvdata { int std_bodies; bool sec_disabled; u32 comp_mask; + u32 cache_params; + u32 ace_const; }; struct cc_crypto_alg { @@ -205,7 +206,7 @@ static inline void dump_byte_array(const char *name, const u8 *the_array, } bool cc_wait_for_reset_completion(struct cc_drvdata *drvdata); -int init_cc_regs(struct cc_drvdata *drvdata, bool is_probe); +int init_cc_regs(struct cc_drvdata *drvdata); void fini_cc_regs(struct cc_drvdata *drvdata); unsigned int cc_get_default_hash_len(struct cc_drvdata *drvdata); diff --git a/drivers/crypto/ccree/cc_pm.c b/drivers/crypto/ccree/cc_pm.c index 3c65bf070c90..d5421b0c6831 100644 --- a/drivers/crypto/ccree/cc_pm.c +++ b/drivers/crypto/ccree/cc_pm.c @@ -45,7 +45,7 @@ static int cc_pm_resume(struct device *dev) } cc_iowrite(drvdata, CC_REG(HOST_POWER_DOWN_EN), POWER_DOWN_DISABLE); - rc = init_cc_regs(drvdata, false); + rc = init_cc_regs(drvdata); if (rc) { dev_err(dev, "init_cc_regs (%x)\n", rc); return rc; diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 13b908ea4873..f5a336634daa 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -53,7 +53,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/hisilicon/Kconfig b/drivers/crypto/hisilicon/Kconfig index 9c3b3ca815e6..843192666dc3 100644 --- a/drivers/crypto/hisilicon/Kconfig +++ b/drivers/crypto/hisilicon/Kconfig @@ -71,3 +71,11 @@ config CRYPTO_DEV_HISI_HPRE help Support for HiSilicon HPRE(High Performance RSA Engine) accelerator, which can accelerate RSA and DH algorithms. + +config CRYPTO_DEV_HISI_TRNG + tristate "Support for HISI TRNG Driver" + depends on ARM64 && ACPI + select HW_RANDOM + select CRYPTO_RNG + help + Support for HiSilicon TRNG Driver. diff --git a/drivers/crypto/hisilicon/Makefile b/drivers/crypto/hisilicon/Makefile index 7f5f74c72baa..1e89269a2e4b 100644 --- a/drivers/crypto/hisilicon/Makefile +++ b/drivers/crypto/hisilicon/Makefile @@ -5,3 +5,4 @@ obj-$(CONFIG_CRYPTO_DEV_HISI_SEC2) += sec2/ obj-$(CONFIG_CRYPTO_DEV_HISI_QM) += hisi_qm.o hisi_qm-objs = qm.o sgl.o obj-$(CONFIG_CRYPTO_DEV_HISI_ZIP) += zip/ +obj-$(CONFIG_CRYPTO_DEV_HISI_TRNG) += trng/ diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index a33394d91bbf..e5c991913f09 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -705,9 +705,7 @@ static int hpre_debugfs_init(struct hisi_qm *qm) qm->debug.sqe_mask_offset = HPRE_SQE_MASK_OFFSET; qm->debug.sqe_mask_len = HPRE_SQE_MASK_LEN; - ret = hisi_qm_debug_init(qm); - if (ret) - goto failed_to_create; + hisi_qm_debug_init(qm); if (qm->pdev->device == HPRE_PCI_DEVICE_ID) { ret = hpre_ctrl_debug_init(qm); diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 530f23116d7c..f21ccae0e8ea 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -473,7 +473,7 @@ static int qm_wait_mb_ready(struct hisi_qm *qm) return readl_relaxed_poll_timeout(qm->io_base + QM_MB_CMD_SEND_BASE, val, !((val >> QM_MB_BUSY_SHIFT) & - 0x1), 10, 1000); + 0x1), POLL_PERIOD, POLL_TIMEOUT); } /* 128 bit should be written to hardware at one time to trigger a mailbox */ @@ -583,7 +583,8 @@ static int qm_dev_mem_reset(struct hisi_qm *qm) writel(0x1, qm->io_base + QM_MEM_START_INIT); return readl_relaxed_poll_timeout(qm->io_base + QM_MEM_INIT_DONE, val, - val & BIT(0), 10, 1000); + val & BIT(0), POLL_PERIOD, + POLL_TIMEOUT); } static u32 qm_get_irq_num_v1(struct hisi_qm *qm) @@ -804,7 +805,8 @@ static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type, int ret; ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, - val & BIT(0), 10, 1000); + val & BIT(0), POLL_PERIOD, + POLL_TIMEOUT); if (ret) return ret; @@ -818,7 +820,8 @@ static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type, writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE); return readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, - val & BIT(0), 10, 1000); + val & BIT(0), POLL_PERIOD, + POLL_TIMEOUT); } /* The config should be conducted after qm_dev_mem_reset() */ @@ -929,7 +932,8 @@ static ssize_t qm_debug_read(struct file *filp, char __user *buf, return -EINVAL; } mutex_unlock(&file->lock); - ret = sprintf(tbuf, "%u\n", val); + + ret = scnprintf(tbuf, QM_DBG_TMP_BUF_LEN, "%u\n", val); return simple_read_from_buffer(buf, count, pos, tbuf, ret); } @@ -1517,7 +1521,7 @@ static const struct file_operations qm_cmd_fops = { .write = qm_cmd_write, }; -static int qm_create_debugfs_file(struct hisi_qm *qm, enum qm_debug_file index) +static void qm_create_debugfs_file(struct hisi_qm *qm, enum qm_debug_file index) { struct dentry *qm_d = qm->debug.qm_d; struct debugfs_file *file = qm->debug.files + index; @@ -1528,8 +1532,6 @@ static int qm_create_debugfs_file(struct hisi_qm *qm, enum qm_debug_file index) file->index = index; mutex_init(&file->lock); file->debug = &qm->debug; - - return 0; } static void qm_hw_error_init_v1(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe) @@ -1733,19 +1735,15 @@ void hisi_qm_release_qp(struct hisi_qp *qp) } EXPORT_SYMBOL_GPL(hisi_qm_release_qp); -static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) +static int qm_sq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) { struct hisi_qm *qm = qp->qm; struct device *dev = &qm->pdev->dev; enum qm_hw_ver ver = qm->ver; struct qm_sqc *sqc; - struct qm_cqc *cqc; dma_addr_t sqc_dma; - dma_addr_t cqc_dma; int ret; - qm_init_qp_status(qp); - sqc = kzalloc(sizeof(struct qm_sqc), GFP_KERNEL); if (!sqc) return -ENOMEM; @@ -1770,8 +1768,18 @@ static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) ret = qm_mb(qm, QM_MB_CMD_SQC, sqc_dma, qp_id, 0); dma_unmap_single(dev, sqc_dma, sizeof(struct qm_sqc), DMA_TO_DEVICE); kfree(sqc); - if (ret) - return ret; + + return ret; +} + +static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) +{ + struct hisi_qm *qm = qp->qm; + struct device *dev = &qm->pdev->dev; + enum qm_hw_ver ver = qm->ver; + struct qm_cqc *cqc; + dma_addr_t cqc_dma; + int ret; cqc = kzalloc(sizeof(struct qm_cqc), GFP_KERNEL); if (!cqc) @@ -1785,11 +1793,12 @@ static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) INIT_QC_COMMON(cqc, qp->cqe_dma, pasid); if (ver == QM_HW_V1) { - cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V1(0, 0, 0, 4)); + cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V1(0, 0, 0, + QM_QC_CQE_SIZE)); cqc->w8 = cpu_to_le16(QM_Q_DEPTH - 1); } else { - cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V2(4)); - cqc->w8 = 0; + cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V2(QM_QC_CQE_SIZE)); + cqc->w8 = 0; /* rand_qc */ } cqc->dw6 = cpu_to_le32(1 << QM_CQ_PHASE_SHIFT | 1 << QM_CQ_FLAG_SHIFT); @@ -1800,6 +1809,19 @@ static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) return ret; } +static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) +{ + int ret; + + qm_init_qp_status(qp); + + ret = qm_sq_ctx_cfg(qp, qp_id, pasid); + if (ret) + return ret; + + return qm_cq_ctx_cfg(qp, qp_id, pasid); +} + static int qm_start_qp_nolock(struct hisi_qp *qp, unsigned long arg) { struct hisi_qm *qm = qp->qm; @@ -1843,6 +1865,9 @@ int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg) EXPORT_SYMBOL_GPL(hisi_qm_start_qp); /** + * qm_drain_qp() - Drain a qp. + * @qp: The qp we want to drain. + * * Determine whether the queue is cleared by judging the tail pointers of * sq and cq. */ @@ -2008,7 +2033,8 @@ static void hisi_qm_cache_wb(struct hisi_qm *qm) writel(0x1, qm->io_base + QM_CACHE_WB_START); if (readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE, - val, val & BIT(0), 10, 1000)) + val, val & BIT(0), POLL_PERIOD, + POLL_TIMEOUT)) dev_err(&qm->pdev->dev, "QM writeback sqc cache fail!\n"); } @@ -2112,7 +2138,7 @@ static void hisi_qm_uacce_stop_queue(struct uacce_queue *q) hisi_qm_stop_qp(q->priv); } -static int qm_set_sqctype(struct uacce_queue *q, u16 type) +static void qm_set_sqctype(struct uacce_queue *q, u16 type) { struct hisi_qm *qm = q->uacce->priv; struct hisi_qp *qp = q->priv; @@ -2120,8 +2146,6 @@ static int qm_set_sqctype(struct uacce_queue *q, u16 type) down_write(&qm->qps_lock); qp->alg_type = type; up_write(&qm->qps_lock); - - return 0; } static long hisi_qm_uacce_ioctl(struct uacce_queue *q, unsigned int cmd, @@ -2418,6 +2442,16 @@ static void hisi_qm_pre_init(struct hisi_qm *qm) qm->is_frozen = false; } +static void hisi_qm_pci_uninit(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + + pci_free_irq_vectors(pdev); + iounmap(qm->io_base); + pci_release_mem_regions(pdev); + pci_disable_device(pdev); +} + /** * hisi_qm_uninit() - Uninitialize qm. * @qm: The qm needed uninit. @@ -2436,9 +2470,6 @@ void hisi_qm_uninit(struct hisi_qm *qm) return; } - uacce_remove(qm->uacce); - qm->uacce = NULL; - hisi_qp_memory_uninit(qm, qm->qp_num); idr_destroy(&qm->qp_idr); @@ -2450,10 +2481,9 @@ void hisi_qm_uninit(struct hisi_qm *qm) } qm_irq_unregister(qm); - pci_free_irq_vectors(pdev); - iounmap(qm->io_base); - pci_release_mem_regions(pdev); - pci_disable_device(pdev); + hisi_qm_pci_uninit(qm); + uacce_remove(qm->uacce); + qm->uacce = NULL; up_write(&qm->qps_lock); } @@ -2486,6 +2516,12 @@ int hisi_qm_get_vft(struct hisi_qm *qm, u32 *base, u32 *number) EXPORT_SYMBOL_GPL(hisi_qm_get_vft); /** + * hisi_qm_set_vft() - Set vft to a qm. + * @qm: The qm we want to set its vft. + * @fun_num: The function number. + * @base: The base number of queue in vft. + * @number: The number of queues in vft. + * * This function is alway called in PF driver, it is used to assign queues * among PF and VFs. * @@ -2519,14 +2555,10 @@ static int qm_eq_ctx_cfg(struct hisi_qm *qm) { struct device *dev = &qm->pdev->dev; struct qm_eqc *eqc; - struct qm_aeqc *aeqc; dma_addr_t eqc_dma; - dma_addr_t aeqc_dma; int ret; - qm_init_eq_aeq_status(qm); - - eqc = kzalloc(sizeof(struct qm_eqc), GFP_KERNEL); + eqc = kzalloc(sizeof(struct qm_eqc), GFP_KERNEL); //todo if (!eqc) return -ENOMEM; eqc_dma = dma_map_single(dev, eqc, sizeof(struct qm_eqc), @@ -2541,11 +2573,20 @@ static int qm_eq_ctx_cfg(struct hisi_qm *qm) if (qm->ver == QM_HW_V1) eqc->dw3 = cpu_to_le32(QM_EQE_AEQE_SIZE); eqc->dw6 = cpu_to_le32((QM_EQ_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT)); + ret = qm_mb(qm, QM_MB_CMD_EQC, eqc_dma, 0, 0); dma_unmap_single(dev, eqc_dma, sizeof(struct qm_eqc), DMA_TO_DEVICE); kfree(eqc); - if (ret) - return ret; + + return ret; +} + +static int qm_aeq_ctx_cfg(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + struct qm_aeqc *aeqc; + dma_addr_t aeqc_dma; + int ret; aeqc = kzalloc(sizeof(struct qm_aeqc), GFP_KERNEL); if (!aeqc) @@ -2568,6 +2609,22 @@ static int qm_eq_ctx_cfg(struct hisi_qm *qm) return ret; } +static int qm_eq_aeq_ctx_cfg(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + int ret; + + qm_init_eq_aeq_status(qm); + + ret = qm_eq_ctx_cfg(qm); + if (ret) { + dev_err(dev, "Set eqc failed!\n"); + return ret; + } + + return qm_aeq_ctx_cfg(qm); +} + static int __hisi_qm_start(struct hisi_qm *qm) { int ret; @@ -2584,7 +2641,7 @@ static int __hisi_qm_start(struct hisi_qm *qm) return ret; } - ret = qm_eq_ctx_cfg(qm); + ret = qm_eq_aeq_ctx_cfg(qm); if (ret) return ret; @@ -2690,7 +2747,11 @@ static int qm_stop_started_qp(struct hisi_qm *qm) return 0; } + /** + * qm_clear_queues() - Clear all queues memory in a qm. + * @qm: The qm in which the queues will be cleared. + * * This function clears all queues memory in a qm. Reset of accelerator can * use this to clear queues. */ @@ -2806,12 +2867,12 @@ DEFINE_DEBUGFS_ATTRIBUTE(qm_atomic64_ops, qm_debugfs_atomic64_get, * * Create qm related debugfs files. */ -int hisi_qm_debug_init(struct hisi_qm *qm) +void hisi_qm_debug_init(struct hisi_qm *qm) { struct qm_dfx *dfx = &qm->debug.dfx; struct dentry *qm_d; void *data; - int i, ret; + int i; qm_d = debugfs_create_dir("qm", qm->debug.debug_root); qm->debug.qm_d = qm_d; @@ -2819,10 +2880,7 @@ int hisi_qm_debug_init(struct hisi_qm *qm) /* only show this in PF */ if (qm->fun_type == QM_HW_PF) for (i = CURRENT_Q; i < DEBUG_FILE_NUM; i++) - if (qm_create_debugfs_file(qm, i)) { - ret = -ENOENT; - goto failed_to_create; - } + qm_create_debugfs_file(qm, i); debugfs_create_file("regs", 0444, qm->debug.qm_d, qm, &qm_regs_fops); @@ -2838,12 +2896,6 @@ int hisi_qm_debug_init(struct hisi_qm *qm) data, &qm_atomic64_ops); } - - return 0; - -failed_to_create: - debugfs_remove_recursive(qm_d); - return ret; } EXPORT_SYMBOL_GPL(hisi_qm_debug_init); @@ -3273,7 +3325,7 @@ pci_ers_result_t hisi_qm_dev_err_detected(struct pci_dev *pdev, } EXPORT_SYMBOL_GPL(hisi_qm_dev_err_detected); -static int qm_get_hw_error_status(struct hisi_qm *qm) +static u32 qm_get_hw_error_status(struct hisi_qm *qm) { return readl(qm->io_base + QM_ABNORMAL_INT_STATUS); } @@ -3572,7 +3624,7 @@ restart_fail: return ret; } -static int qm_get_dev_err_status(struct hisi_qm *qm) +static u32 qm_get_dev_err_status(struct hisi_qm *qm) { return qm->err_ini->get_dev_hw_err_status(qm); } @@ -3992,34 +4044,22 @@ void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list) } EXPORT_SYMBOL_GPL(hisi_qm_alg_unregister); -/** - * hisi_qm_init() - Initialize configures about qm. - * @qm: The qm needing init. - * - * This function init qm, then we can call hisi_qm_start to put qm into work. - */ -int hisi_qm_init(struct hisi_qm *qm) +static int hisi_qm_pci_init(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; struct device *dev = &pdev->dev; unsigned int num_vec; int ret; - hisi_qm_pre_init(qm); - - ret = qm_alloc_uacce(qm); - if (ret < 0) - dev_warn(&pdev->dev, "fail to alloc uacce (%d)\n", ret); - ret = pci_enable_device_mem(pdev); if (ret < 0) { - dev_err(&pdev->dev, "Failed to enable device mem!\n"); - goto err_remove_uacce; + dev_err(dev, "Failed to enable device mem!\n"); + return ret; } ret = pci_request_mem_regions(pdev, qm->dev_name); if (ret < 0) { - dev_err(&pdev->dev, "Failed to request mem regions!\n"); + dev_err(dev, "Failed to request mem regions!\n"); goto err_disable_pcidev; } @@ -4047,9 +4087,42 @@ int hisi_qm_init(struct hisi_qm *qm) goto err_iounmap; } + return 0; + +err_iounmap: + iounmap(qm->io_base); +err_release_mem_regions: + pci_release_mem_regions(pdev); +err_disable_pcidev: + pci_disable_device(pdev); + return ret; +} + +/** + * hisi_qm_init() - Initialize configures about qm. + * @qm: The qm needing init. + * + * This function init qm, then we can call hisi_qm_start to put qm into work. + */ +int hisi_qm_init(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + struct device *dev = &pdev->dev; + int ret; + + hisi_qm_pre_init(qm); + + ret = qm_alloc_uacce(qm); + if (ret < 0) + dev_warn(dev, "fail to alloc uacce (%d)\n", ret); + + ret = hisi_qm_pci_init(qm); + if (ret) + goto err_remove_uacce; + ret = qm_irq_register(qm); if (ret) - goto err_free_irq_vectors; + goto err_pci_uninit; if (qm->fun_type == QM_HW_VF && qm->ver != QM_HW_V1) { /* v2 starts to support get vft by mailbox */ @@ -4072,14 +4145,8 @@ int hisi_qm_init(struct hisi_qm *qm) err_irq_unregister: qm_irq_unregister(qm); -err_free_irq_vectors: - pci_free_irq_vectors(pdev); -err_iounmap: - iounmap(qm->io_base); -err_release_mem_regions: - pci_release_mem_regions(pdev); -err_disable_pcidev: - pci_disable_device(pdev); +err_pci_uninit: + hisi_qm_pci_uninit(qm); err_remove_uacce: uacce_remove(qm->uacce); qm->uacce = NULL; @@ -4087,7 +4154,6 @@ err_remove_uacce: } EXPORT_SYMBOL_GPL(hisi_qm_init); - MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Zhou Wang "); MODULE_DESCRIPTION("HiSilicon Accelerator queue manager driver"); diff --git a/drivers/crypto/hisilicon/qm.h b/drivers/crypto/hisilicon/qm.h index 0420f4ce7197..8624d1288afe 100644 --- a/drivers/crypto/hisilicon/qm.h +++ b/drivers/crypto/hisilicon/qm.h @@ -350,7 +350,7 @@ void hisi_qm_release_qp(struct hisi_qp *qp); int hisi_qp_send(struct hisi_qp *qp, const void *msg); int hisi_qm_get_free_qp_num(struct hisi_qm *qm); int hisi_qm_get_vft(struct hisi_qm *qm, u32 *base, u32 *number); -int hisi_qm_debug_init(struct hisi_qm *qm); +void hisi_qm_debug_init(struct hisi_qm *qm); enum qm_hw_ver hisi_qm_get_hw_version(struct pci_dev *pdev); void hisi_qm_debug_regs_clear(struct hisi_qm *qm); int hisi_qm_sriov_enable(struct pci_dev *pdev, int max_vfs); diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h index 037762b531e2..08491912afd5 100644 --- a/drivers/crypto/hisilicon/sec2/sec.h +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -109,7 +109,6 @@ struct sec_qp_ctx { struct list_head backlog; struct hisi_acc_sgl_pool *c_in_pool; struct hisi_acc_sgl_pool *c_out_pool; - atomic_t pending_reqs; }; enum sec_alg_type { @@ -180,7 +179,6 @@ struct sec_dev { struct sec_debug debug; u32 ctx_q_num; bool iommu_used; - unsigned long status; }; void sec_destroy_qps(struct hisi_qp **qps, int qp_num); diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index bb493423668c..2eaa516b3231 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -7,7 +7,8 @@ #include #include #include -#include +#include +#include #include #include #include @@ -101,6 +102,7 @@ static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx) req->qp_ctx = qp_ctx; qp_ctx->req_list[req_id] = req; + return req_id; } @@ -317,6 +319,7 @@ static int sec_alloc_pbuf_resource(struct device *dev, struct sec_alg_res *res) j * SEC_PBUF_PKG + pbuf_page_offset; } } + return 0; } @@ -345,12 +348,12 @@ static int sec_alg_resource_alloc(struct sec_ctx *ctx, } return 0; + alloc_pbuf_fail: if (ctx->alg_type == SEC_AEAD) sec_free_mac_resource(dev, qp_ctx->res); alloc_fail: sec_free_civ_resource(dev, res); - return ret; } @@ -419,7 +422,6 @@ err_free_c_in_pool: hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); err_destroy_idr: idr_destroy(&qp_ctx->req_idr); - return ret; } @@ -557,9 +559,9 @@ static int sec_skcipher_init(struct crypto_skcipher *tfm) goto err_cipher_init; return 0; + err_cipher_init: sec_ctx_base_uninit(ctx); - return ret; } @@ -740,7 +742,6 @@ static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req, if (unlikely(pbuf_length != copy_size)) dev_err(dev, "copy pbuf data to dst error!\n"); - } static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req, @@ -857,7 +858,7 @@ static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx, struct crypto_authenc_keys *keys) { struct crypto_shash *hash_tfm = ctx->hash_tfm; - int blocksize, ret; + int blocksize, digestsize, ret; if (!keys->authkeylen) { pr_err("hisi_sec2: aead auth key error!\n"); @@ -865,6 +866,7 @@ static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx, } blocksize = crypto_shash_blocksize(hash_tfm); + digestsize = crypto_shash_digestsize(hash_tfm); if (keys->authkeylen > blocksize) { ret = crypto_shash_tfm_digest(hash_tfm, keys->authkey, keys->authkeylen, ctx->a_key); @@ -872,7 +874,7 @@ static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx, pr_err("hisi_sec2: aead auth digest error!\n"); return -EINVAL; } - ctx->a_key_len = blocksize; + ctx->a_key_len = digestsize; } else { memcpy(ctx->a_key, keys->authkey, keys->authkeylen); ctx->a_key_len = keys->authkeylen; @@ -913,9 +915,9 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, } return 0; + bad_key: memzero_explicit(&keys, sizeof(struct crypto_authenc_keys)); - return -EINVAL; } @@ -966,7 +968,6 @@ static int sec_request_transfer(struct sec_ctx *ctx, struct sec_req *req) unmap_req_buf: ctx->req_op->buf_unmap(ctx, req); - return ret; } @@ -1107,7 +1108,6 @@ static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req, atomic64_inc(&ctx->sec->debug.dfx.recv_busy_cnt); } - sk_req->base.complete(&sk_req->base, err); } @@ -1279,7 +1279,6 @@ err_send_req: sec_request_untransfer(ctx, req); err_uninit_req: sec_request_uninit(ctx, req); - return ret; } @@ -1349,7 +1348,6 @@ err_cipher_init: sec_auth_uninit(ctx); err_auth_init: sec_ctx_base_uninit(ctx); - return ret; } @@ -1437,8 +1435,8 @@ static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq) } return 0; } - dev_err(dev, "skcipher algorithm error!\n"); + return -EINVAL; } @@ -1554,7 +1552,6 @@ static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq) if (unlikely(c_alg != SEC_CALG_AES)) { dev_err(SEC_CTX_DEV(ctx), "aead crypto alg error!\n"); return -EINVAL; - } if (sreq->c_req.encrypt) sreq->c_req.c_len = req->cryptlen; diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 548896394c4b..b35c1c2271a3 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -652,20 +652,16 @@ static int sec_debugfs_init(struct hisi_qm *qm) sec_debugfs_root); qm->debug.sqe_mask_offset = SEC_SQE_MASK_OFFSET; qm->debug.sqe_mask_len = SEC_SQE_MASK_LEN; - ret = hisi_qm_debug_init(qm); - if (ret) - goto failed_to_create; + hisi_qm_debug_init(qm); ret = sec_debug_init(qm); if (ret) goto failed_to_create; - return 0; failed_to_create: debugfs_remove_recursive(sec_debugfs_root); - return ret; } @@ -683,13 +679,13 @@ static void sec_log_hw_error(struct hisi_qm *qm, u32 err_sts) while (errs->msg) { if (errs->int_msk & err_sts) { dev_err(dev, "%s [error status=0x%x] found\n", - errs->msg, errs->int_msk); + errs->msg, errs->int_msk); if (SEC_CORE_INT_STATUS_M_ECC & errs->int_msk) { err_val = readl(qm->io_base + SEC_CORE_SRAM_ECC_ERR_INFO); dev_err(dev, "multi ecc sram num=0x%x\n", - SEC_ECC_NUM(err_val)); + SEC_ECC_NUM(err_val)); } } errs++; @@ -724,13 +720,13 @@ static const struct hisi_qm_err_ini sec_err_ini = { .log_dev_hw_err = sec_log_hw_error, .open_axi_master_ooo = sec_open_axi_master_ooo, .err_info = { - .ce = QM_BASE_CE, - .nfe = QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT | - QM_ACC_WB_NOT_READY_TIMEOUT, - .fe = 0, - .ecc_2bits_mask = SEC_CORE_INT_STATUS_M_ECC, - .msi_wr_port = BIT(0), - .acpi_rst = "SRST", + .ce = QM_BASE_CE, + .nfe = QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT | + QM_ACC_WB_NOT_READY_TIMEOUT, + .fe = 0, + .ecc_2bits_mask = SEC_CORE_INT_STATUS_M_ECC, + .msi_wr_port = BIT(0), + .acpi_rst = "SRST", } }; @@ -899,17 +895,13 @@ static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id) err_alg_unregister: hisi_qm_alg_unregister(qm, &sec_devices); - err_qm_stop: sec_debugfs_exit(qm); hisi_qm_stop(qm, QM_NORMAL); - err_probe_uninit: sec_probe_uninit(qm); - err_qm_uninit: sec_qm_uninit(qm); - return ret; } @@ -936,9 +928,9 @@ static void sec_remove(struct pci_dev *pdev) static const struct pci_error_handlers sec_err_handler = { .error_detected = hisi_qm_dev_err_detected, - .slot_reset = hisi_qm_dev_slot_reset, - .reset_prepare = hisi_qm_reset_prepare, - .reset_done = hisi_qm_reset_done, + .slot_reset = hisi_qm_dev_slot_reset, + .reset_prepare = hisi_qm_reset_prepare, + .reset_done = hisi_qm_reset_done, }; static struct pci_driver sec_pci_driver = { diff --git a/drivers/crypto/hisilicon/sgl.c b/drivers/crypto/hisilicon/sgl.c index 725a739800b0..3bff6394acaf 100644 --- a/drivers/crypto/hisilicon/sgl.c +++ b/drivers/crypto/hisilicon/sgl.c @@ -246,8 +246,6 @@ EXPORT_SYMBOL_GPL(hisi_acc_sg_buf_map_to_hw_sgl); * @dev: The device which hw sgl belongs to. * @sgl: Related scatterlist. * @hw_sgl: Virtual address of hw sgl. - * @hw_sgl_dma: DMA address of hw sgl. - * @pool: Pool which hw sgl is allocated in. * * This function unmaps allocated hw sgl. */ diff --git a/drivers/crypto/hisilicon/trng/Makefile b/drivers/crypto/hisilicon/trng/Makefile new file mode 100644 index 000000000000..d909079f351c --- /dev/null +++ b/drivers/crypto/hisilicon/trng/Makefile @@ -0,0 +1,2 @@ +obj-$(CONFIG_CRYPTO_DEV_HISI_TRNG) += hisi-trng-v2.o +hisi-trng-v2-objs = trng.o diff --git a/drivers/crypto/hisilicon/trng/trng.c b/drivers/crypto/hisilicon/trng/trng.c new file mode 100644 index 000000000000..29712685498a --- /dev/null +++ b/drivers/crypto/hisilicon/trng/trng.c @@ -0,0 +1,334 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2019 HiSilicon Limited. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define HISI_TRNG_REG 0x00F0 +#define HISI_TRNG_BYTES 4 +#define HISI_TRNG_QUALITY 512 +#define SLEEP_US 10 +#define TIMEOUT_US 10000 +#define SW_DRBG_NUM_SHIFT 2 +#define SW_DRBG_KEY_BASE 0x082C +#define SW_DRBG_SEED(n) (SW_DRBG_KEY_BASE - ((n) << SW_DRBG_NUM_SHIFT)) +#define SW_DRBG_SEED_REGS_NUM 12 +#define SW_DRBG_SEED_SIZE 48 +#define SW_DRBG_BLOCKS 0x0830 +#define SW_DRBG_INIT 0x0834 +#define SW_DRBG_GEN 0x083c +#define SW_DRBG_STATUS 0x0840 +#define SW_DRBG_BLOCKS_NUM 4095 +#define SW_DRBG_DATA_BASE 0x0850 +#define SW_DRBG_DATA_NUM 4 +#define SW_DRBG_DATA(n) (SW_DRBG_DATA_BASE - ((n) << SW_DRBG_NUM_SHIFT)) +#define SW_DRBG_BYTES 16 +#define SW_DRBG_ENABLE_SHIFT 12 +#define SEED_SHIFT_24 24 +#define SEED_SHIFT_16 16 +#define SEED_SHIFT_8 8 + +struct hisi_trng_list { + struct mutex lock; + struct list_head list; + bool is_init; +}; + +struct hisi_trng { + void __iomem *base; + struct hisi_trng_list *trng_list; + struct list_head list; + struct hwrng rng; + bool is_used; + struct mutex mutex; +}; + +struct hisi_trng_ctx { + struct hisi_trng *trng; +}; + +static atomic_t trng_active_devs; +static struct hisi_trng_list trng_devices; + +static void hisi_trng_set_seed(struct hisi_trng *trng, const u8 *seed) +{ + u32 val, seed_reg, i; + + for (i = 0; i < SW_DRBG_SEED_SIZE; + i += SW_DRBG_SEED_SIZE / SW_DRBG_SEED_REGS_NUM) { + val = seed[i] << SEED_SHIFT_24; + val |= seed[i + 1UL] << SEED_SHIFT_16; + val |= seed[i + 2UL] << SEED_SHIFT_8; + val |= seed[i + 3UL]; + + seed_reg = (i >> SW_DRBG_NUM_SHIFT) % SW_DRBG_SEED_REGS_NUM; + writel(val, trng->base + SW_DRBG_SEED(seed_reg)); + } +} + +static int hisi_trng_seed(struct crypto_rng *tfm, const u8 *seed, + unsigned int slen) +{ + struct hisi_trng_ctx *ctx = crypto_rng_ctx(tfm); + struct hisi_trng *trng = ctx->trng; + u32 val = 0; + int ret = 0; + + if (slen < SW_DRBG_SEED_SIZE) { + pr_err("slen(%u) is not matched with trng(%d)\n", slen, + SW_DRBG_SEED_SIZE); + return -EINVAL; + } + + writel(0x0, trng->base + SW_DRBG_BLOCKS); + hisi_trng_set_seed(trng, seed); + + writel(SW_DRBG_BLOCKS_NUM | (0x1 << SW_DRBG_ENABLE_SHIFT), + trng->base + SW_DRBG_BLOCKS); + writel(0x1, trng->base + SW_DRBG_INIT); + + ret = readl_relaxed_poll_timeout(trng->base + SW_DRBG_STATUS, + val, val & BIT(0), SLEEP_US, TIMEOUT_US); + if (ret) + pr_err("fail to init trng(%d)\n", ret); + + return ret; +} + +static int hisi_trng_generate(struct crypto_rng *tfm, const u8 *src, + unsigned int slen, u8 *dstn, unsigned int dlen) +{ + struct hisi_trng_ctx *ctx = crypto_rng_ctx(tfm); + struct hisi_trng *trng = ctx->trng; + u32 data[SW_DRBG_DATA_NUM]; + u32 currsize = 0; + u32 val = 0; + int ret; + u32 i; + + if (dlen > SW_DRBG_BLOCKS_NUM * SW_DRBG_BYTES || dlen == 0) { + pr_err("dlen(%d) exceeds limit(%d)!\n", dlen, + SW_DRBG_BLOCKS_NUM * SW_DRBG_BYTES); + return -EINVAL; + } + + do { + ret = readl_relaxed_poll_timeout(trng->base + SW_DRBG_STATUS, + val, val & BIT(1), SLEEP_US, TIMEOUT_US); + if (ret) { + pr_err("fail to generate random number(%d)!\n", ret); + break; + } + + for (i = 0; i < SW_DRBG_DATA_NUM; i++) + data[i] = readl(trng->base + SW_DRBG_DATA(i)); + + if (dlen - currsize >= SW_DRBG_BYTES) { + memcpy(dstn + currsize, data, SW_DRBG_BYTES); + currsize += SW_DRBG_BYTES; + } else { + memcpy(dstn + currsize, data, dlen - currsize); + currsize = dlen; + } + + writel(0x1, trng->base + SW_DRBG_GEN); + } while (currsize < dlen); + + return ret; +} + +static int hisi_trng_init(struct crypto_tfm *tfm) +{ + struct hisi_trng_ctx *ctx = crypto_tfm_ctx(tfm); + struct hisi_trng *trng; + int ret = -EBUSY; + + mutex_lock(&trng_devices.lock); + list_for_each_entry(trng, &trng_devices.list, list) { + if (!trng->is_used) { + trng->is_used = true; + ctx->trng = trng; + ret = 0; + break; + } + } + mutex_unlock(&trng_devices.lock); + + return ret; +} + +static void hisi_trng_exit(struct crypto_tfm *tfm) +{ + struct hisi_trng_ctx *ctx = crypto_tfm_ctx(tfm); + + mutex_lock(&trng_devices.lock); + ctx->trng->is_used = false; + mutex_unlock(&trng_devices.lock); +} + +static int hisi_trng_read(struct hwrng *rng, void *buf, size_t max, bool wait) +{ + struct hisi_trng *trng; + int currsize = 0; + u32 val = 0; + u32 ret; + + trng = container_of(rng, struct hisi_trng, rng); + + do { + ret = readl_poll_timeout(trng->base + HISI_TRNG_REG, val, + val, SLEEP_US, TIMEOUT_US); + if (ret) + return currsize; + + if (max - currsize >= HISI_TRNG_BYTES) { + memcpy(buf + currsize, &val, HISI_TRNG_BYTES); + currsize += HISI_TRNG_BYTES; + if (currsize == max) + return currsize; + continue; + } + + /* copy remaining bytes */ + memcpy(buf + currsize, &val, max - currsize); + currsize = max; + } while (currsize < max); + + return currsize; +} + +static struct rng_alg hisi_trng_alg = { + .generate = hisi_trng_generate, + .seed = hisi_trng_seed, + .seedsize = SW_DRBG_SEED_SIZE, + .base = { + .cra_name = "stdrng", + .cra_driver_name = "hisi_stdrng", + .cra_priority = 300, + .cra_ctxsize = sizeof(struct hisi_trng_ctx), + .cra_module = THIS_MODULE, + .cra_init = hisi_trng_init, + .cra_exit = hisi_trng_exit, + }, +}; + +static void hisi_trng_add_to_list(struct hisi_trng *trng) +{ + mutex_lock(&trng_devices.lock); + list_add_tail(&trng->list, &trng_devices.list); + mutex_unlock(&trng_devices.lock); +} + +static int hisi_trng_del_from_list(struct hisi_trng *trng) +{ + int ret = -EBUSY; + + mutex_lock(&trng_devices.lock); + if (!trng->is_used) { + list_del(&trng->list); + ret = 0; + } + mutex_unlock(&trng_devices.lock); + + return ret; +} + +static int hisi_trng_probe(struct platform_device *pdev) +{ + struct hisi_trng *trng; + int ret; + + trng = devm_kzalloc(&pdev->dev, sizeof(*trng), GFP_KERNEL); + if (!trng) + return -ENOMEM; + + platform_set_drvdata(pdev, trng); + + trng->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(trng->base)) + return PTR_ERR(trng->base); + + trng->is_used = false; + if (!trng_devices.is_init) { + INIT_LIST_HEAD(&trng_devices.list); + mutex_init(&trng_devices.lock); + trng_devices.is_init = true; + } + + hisi_trng_add_to_list(trng); + if (atomic_inc_return(&trng_active_devs) == 1) { + ret = crypto_register_rng(&hisi_trng_alg); + if (ret) { + dev_err(&pdev->dev, + "failed to register crypto(%d)\n", ret); + atomic_dec_return(&trng_active_devs); + goto err_remove_from_list; + } + } + + trng->rng.name = pdev->name; + trng->rng.read = hisi_trng_read; + trng->rng.quality = HISI_TRNG_QUALITY; + ret = devm_hwrng_register(&pdev->dev, &trng->rng); + if (ret) { + dev_err(&pdev->dev, "failed to register hwrng: %d!\n", ret); + goto err_crypto_unregister; + } + + return ret; + +err_crypto_unregister: + if (atomic_dec_return(&trng_active_devs) == 0) + crypto_unregister_rng(&hisi_trng_alg); + +err_remove_from_list: + hisi_trng_del_from_list(trng); + return ret; +} + +static int hisi_trng_remove(struct platform_device *pdev) +{ + struct hisi_trng *trng = platform_get_drvdata(pdev); + + /* Wait until the task is finished */ + while (hisi_trng_del_from_list(trng)) + ; + + if (atomic_dec_return(&trng_active_devs) == 0) + crypto_unregister_rng(&hisi_trng_alg); + + return 0; +} + +static const struct acpi_device_id hisi_trng_acpi_match[] = { + { "HISI02B3", 0 }, + { } +}; +MODULE_DEVICE_TABLE(acpi, hisi_trng_acpi_match); + +static struct platform_driver hisi_trng_driver = { + .probe = hisi_trng_probe, + .remove = hisi_trng_remove, + .driver = { + .name = "hisi-trng-v2", + .acpi_match_table = ACPI_PTR(hisi_trng_acpi_match), + }, +}; + +module_platform_driver(hisi_trng_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Weili Qian "); +MODULE_AUTHOR("Zaibo Xu "); +MODULE_DESCRIPTION("HiSilicon true random number generator V2 driver"); diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 4bd2c811abba..4fb5a32bf830 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -590,9 +590,7 @@ static int hisi_zip_debugfs_init(struct hisi_qm *qm) qm->debug.sqe_mask_offset = HZIP_SQE_MASK_OFFSET; qm->debug.sqe_mask_len = HZIP_SQE_MASK_LEN; qm->debug.debug_root = dev_d; - ret = hisi_qm_debug_init(qm); - if (ret) - goto failed_to_create; + hisi_qm_debug_init(qm); if (qm->fun_type == QM_HW_PF) { ret = hisi_zip_ctrl_debug_init(qm); @@ -749,6 +747,8 @@ static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) { + int ret; + qm->pdev = pdev; qm->ver = pdev->revision; qm->algs = "zlib\ngzip"; @@ -774,7 +774,25 @@ static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) qm->qp_num = HZIP_QUEUE_NUM_V1 - HZIP_PF_DEF_Q_NUM; } - return hisi_qm_init(qm); + qm->wq = alloc_workqueue("%s", WQ_HIGHPRI | WQ_MEM_RECLAIM | + WQ_UNBOUND, num_online_cpus(), + pci_name(qm->pdev)); + if (!qm->wq) { + pci_err(qm->pdev, "fail to alloc workqueue\n"); + return -ENOMEM; + } + + ret = hisi_qm_init(qm); + if (ret) + destroy_workqueue(qm->wq); + + return ret; +} + +static void hisi_zip_qm_uninit(struct hisi_qm *qm) +{ + hisi_qm_uninit(qm); + destroy_workqueue(qm->wq); } static int hisi_zip_probe_init(struct hisi_zip *hisi_zip) @@ -856,7 +874,7 @@ err_dev_err_uninit: hisi_qm_dev_err_uninit(qm); err_qm_uninit: - hisi_qm_uninit(qm); + hisi_zip_qm_uninit(qm); return ret; } @@ -874,7 +892,7 @@ static void hisi_zip_remove(struct pci_dev *pdev) hisi_zip_debugfs_exit(qm); hisi_qm_stop(qm, QM_NORMAL); hisi_qm_dev_err_uninit(qm); - hisi_qm_uninit(qm); + hisi_zip_qm_uninit(qm); } static const struct pci_error_handlers hisi_zip_err_handler = { diff --git a/drivers/crypto/img-hash.c b/drivers/crypto/img-hash.c index 91f555ccbb31..e813115d5432 100644 --- a/drivers/crypto/img-hash.c +++ b/drivers/crypto/img-hash.c @@ -19,7 +19,8 @@ #include #include -#include +#include +#include #define CR_RESET 0 #define CR_RESET_SET 1 diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index eb2418450f12..2e1562108a85 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1639,7 +1639,7 @@ static int safexcel_probe_generic(void *pdev, priv->ring[i].rdr_req = devm_kcalloc(dev, EIP197_DEFAULT_RING_SIZE, - sizeof(priv->ring[i].rdr_req), + sizeof(*priv->ring[i].rdr_req), GFP_KERNEL); if (!priv->ring[i].rdr_req) return -ENOMEM; diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 9045f2d7f4c6..ce1e611a163e 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -11,7 +11,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 9bcfb79a030f..d68ef16650d4 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -18,7 +18,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index 56d5ccb5cc00..50fb6d90a2e0 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -8,7 +8,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c index 276012e7c482..8b0f17fc09fb 100644 --- a/drivers/crypto/ixp4xx_crypto.c +++ b/drivers/crypto/ixp4xx_crypto.c @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/crypto/keembay/Kconfig b/drivers/crypto/keembay/Kconfig new file mode 100644 index 000000000000..3c16797b25b9 --- /dev/null +++ b/drivers/crypto/keembay/Kconfig @@ -0,0 +1,39 @@ +config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4 + tristate "Support for Intel Keem Bay OCS AES/SM4 HW acceleration" + depends on OF || COMPILE_TEST + select CRYPTO_SKCIPHER + select CRYPTO_AEAD + select CRYPTO_ENGINE + help + Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) AES and + SM4 cihper hardware acceleration for use with Crypto API. + + Provides HW acceleration for the following transformations: + cbc(aes), ctr(aes), ccm(aes), gcm(aes), cbc(sm4), ctr(sm4), ccm(sm4) + and gcm(sm4). + + Optionally, support for the following transformations can also be + enabled: ecb(aes), cts(cbc(aes)), ecb(sm4) and cts(cbc(sm4)). + +config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB + bool "Support for Intel Keem Bay OCS AES/SM4 ECB HW acceleration" + depends on CRYPTO_DEV_KEEMBAY_OCS_AES_SM4 + help + Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) + AES/SM4 ECB mode hardware acceleration for use with Crypto API. + + Provides OCS version of ecb(aes) and ecb(sm4) + + Intel does not recommend use of ECB mode with AES/SM4. + +config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS + bool "Support for Intel Keem Bay OCS AES/SM4 CTS HW acceleration" + depends on CRYPTO_DEV_KEEMBAY_OCS_AES_SM4 + help + Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) + AES/SM4 CBC with CTS mode hardware acceleration for use with + Crypto API. + + Provides OCS version of cts(cbc(aes)) and cts(cbc(sm4)). + + Intel does not recommend use of CTS mode with AES/SM4. diff --git a/drivers/crypto/keembay/Makefile b/drivers/crypto/keembay/Makefile new file mode 100644 index 000000000000..f21e2c4ab3b3 --- /dev/null +++ b/drivers/crypto/keembay/Makefile @@ -0,0 +1,5 @@ +# +# Makefile for Intel Keem Bay OCS Crypto API Linux drivers +# +obj-$(CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4) += keembay-ocs-aes.o +keembay-ocs-aes-objs := keembay-ocs-aes-core.o ocs-aes.o diff --git a/drivers/crypto/keembay/keembay-ocs-aes-core.c b/drivers/crypto/keembay/keembay-ocs-aes-core.c new file mode 100644 index 000000000000..b6b25d994af3 --- /dev/null +++ b/drivers/crypto/keembay/keembay-ocs-aes-core.c @@ -0,0 +1,1713 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Intel Keem Bay OCS AES Crypto Driver. + * + * Copyright (C) 2018-2020 Intel Corporation + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include "ocs-aes.h" + +#define KMB_OCS_PRIORITY 350 +#define DRV_NAME "keembay-ocs-aes" + +#define OCS_AES_MIN_KEY_SIZE 16 +#define OCS_AES_MAX_KEY_SIZE 32 +#define OCS_AES_KEYSIZE_128 16 +#define OCS_AES_KEYSIZE_192 24 +#define OCS_AES_KEYSIZE_256 32 +#define OCS_SM4_KEY_SIZE 16 + +/** + * struct ocs_aes_tctx - OCS AES Transform context + * @engine_ctx: Engine context. + * @aes_dev: The OCS AES device. + * @key: AES/SM4 key. + * @key_len: The length (in bytes) of @key. + * @cipher: OCS cipher to use (either AES or SM4). + * @sw_cipher: The cipher to use as fallback. + * @use_fallback: Whether or not fallback cipher should be used. + */ +struct ocs_aes_tctx { + struct crypto_engine_ctx engine_ctx; + struct ocs_aes_dev *aes_dev; + u8 key[OCS_AES_KEYSIZE_256]; + unsigned int key_len; + enum ocs_cipher cipher; + union { + struct crypto_sync_skcipher *sk; + struct crypto_aead *aead; + } sw_cipher; + bool use_fallback; +}; + +/** + * struct ocs_aes_rctx - OCS AES Request context. + * @instruction: Instruction to be executed (encrypt / decrypt). + * @mode: Mode to use (ECB, CBC, CTR, CCm, GCM, CTS) + * @src_nents: Number of source SG entries. + * @dst_nents: Number of destination SG entries. + * @src_dma_count: The number of DMA-mapped entries of the source SG. + * @dst_dma_count: The number of DMA-mapped entries of the destination SG. + * @in_place: Whether or not this is an in place request, i.e., + * src_sg == dst_sg. + * @src_dll: OCS DMA linked list for input data. + * @dst_dll: OCS DMA linked list for output data. + * @last_ct_blk: Buffer to hold last cipher text block (only used in CBC + * mode). + * @cts_swap: Whether or not CTS swap must be performed. + * @aad_src_dll: OCS DMA linked list for input AAD data. + * @aad_dst_dll: OCS DMA linked list for output AAD data. + * @in_tag: Buffer to hold input encrypted tag (only used for + * CCM/GCM decrypt). + * @out_tag: Buffer to hold output encrypted / decrypted tag (only + * used for GCM encrypt / decrypt). + */ +struct ocs_aes_rctx { + /* Fields common across all modes. */ + enum ocs_instruction instruction; + enum ocs_mode mode; + int src_nents; + int dst_nents; + int src_dma_count; + int dst_dma_count; + bool in_place; + struct ocs_dll_desc src_dll; + struct ocs_dll_desc dst_dll; + + /* CBC specific */ + u8 last_ct_blk[AES_BLOCK_SIZE]; + + /* CTS specific */ + int cts_swap; + + /* CCM/GCM specific */ + struct ocs_dll_desc aad_src_dll; + struct ocs_dll_desc aad_dst_dll; + u8 in_tag[AES_BLOCK_SIZE]; + + /* GCM specific */ + u8 out_tag[AES_BLOCK_SIZE]; +}; + +/* Driver data. */ +struct ocs_aes_drv { + struct list_head dev_list; + spinlock_t lock; /* Protects dev_list. */ +}; + +static struct ocs_aes_drv ocs_aes = { + .dev_list = LIST_HEAD_INIT(ocs_aes.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(ocs_aes.lock), +}; + +static struct ocs_aes_dev *kmb_ocs_aes_find_dev(struct ocs_aes_tctx *tctx) +{ + struct ocs_aes_dev *aes_dev; + + spin_lock(&ocs_aes.lock); + + if (tctx->aes_dev) { + aes_dev = tctx->aes_dev; + goto exit; + } + + /* Only a single OCS device available */ + aes_dev = list_first_entry(&ocs_aes.dev_list, struct ocs_aes_dev, list); + tctx->aes_dev = aes_dev; + +exit: + spin_unlock(&ocs_aes.lock); + + return aes_dev; +} + +/* + * Ensure key is 128-bit or 256-bit for AES or 128-bit for SM4 and an actual + * key is being passed in. + * + * Return: 0 if key is valid, -EINVAL otherwise. + */ +static int check_key(const u8 *in_key, size_t key_len, enum ocs_cipher cipher) +{ + if (!in_key) + return -EINVAL; + + /* For AES, only 128-byte or 256-byte keys are supported. */ + if (cipher == OCS_AES && (key_len == OCS_AES_KEYSIZE_128 || + key_len == OCS_AES_KEYSIZE_256)) + return 0; + + /* For SM4, only 128-byte keys are supported. */ + if (cipher == OCS_SM4 && key_len == OCS_AES_KEYSIZE_128) + return 0; + + /* Everything else is unsupported. */ + return -EINVAL; +} + +/* Save key into transformation context. */ +static int save_key(struct ocs_aes_tctx *tctx, const u8 *in_key, size_t key_len, + enum ocs_cipher cipher) +{ + int ret; + + ret = check_key(in_key, key_len, cipher); + if (ret) + return ret; + + memcpy(tctx->key, in_key, key_len); + tctx->key_len = key_len; + tctx->cipher = cipher; + + return 0; +} + +/* Set key for symmetric cypher. */ +static int kmb_ocs_sk_set_key(struct crypto_skcipher *tfm, const u8 *in_key, + size_t key_len, enum ocs_cipher cipher) +{ + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + + /* Fallback is used for AES with 192-bit key. */ + tctx->use_fallback = (cipher == OCS_AES && + key_len == OCS_AES_KEYSIZE_192); + + if (!tctx->use_fallback) + return save_key(tctx, in_key, key_len, cipher); + + crypto_sync_skcipher_clear_flags(tctx->sw_cipher.sk, + CRYPTO_TFM_REQ_MASK); + crypto_sync_skcipher_set_flags(tctx->sw_cipher.sk, + tfm->base.crt_flags & + CRYPTO_TFM_REQ_MASK); + + return crypto_sync_skcipher_setkey(tctx->sw_cipher.sk, in_key, key_len); +} + +/* Set key for AEAD cipher. */ +static int kmb_ocs_aead_set_key(struct crypto_aead *tfm, const u8 *in_key, + size_t key_len, enum ocs_cipher cipher) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); + + /* Fallback is used for AES with 192-bit key. */ + tctx->use_fallback = (cipher == OCS_AES && + key_len == OCS_AES_KEYSIZE_192); + + if (!tctx->use_fallback) + return save_key(tctx, in_key, key_len, cipher); + + crypto_aead_clear_flags(tctx->sw_cipher.aead, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(tctx->sw_cipher.aead, + crypto_aead_get_flags(tfm) & CRYPTO_TFM_REQ_MASK); + + return crypto_aead_setkey(tctx->sw_cipher.aead, in_key, key_len); +} + +/* Swap two AES blocks in SG lists. */ +static void sg_swap_blocks(struct scatterlist *sgl, unsigned int nents, + off_t blk1_offset, off_t blk2_offset) +{ + u8 tmp_buf1[AES_BLOCK_SIZE], tmp_buf2[AES_BLOCK_SIZE]; + + /* + * No easy way to copy within sg list, so copy both blocks to temporary + * buffers first. + */ + sg_pcopy_to_buffer(sgl, nents, tmp_buf1, AES_BLOCK_SIZE, blk1_offset); + sg_pcopy_to_buffer(sgl, nents, tmp_buf2, AES_BLOCK_SIZE, blk2_offset); + sg_pcopy_from_buffer(sgl, nents, tmp_buf1, AES_BLOCK_SIZE, blk2_offset); + sg_pcopy_from_buffer(sgl, nents, tmp_buf2, AES_BLOCK_SIZE, blk1_offset); +} + +/* Initialize request context to default values. */ +static void ocs_aes_init_rctx(struct ocs_aes_rctx *rctx) +{ + /* Zero everything. */ + memset(rctx, 0, sizeof(*rctx)); + + /* Set initial value for DMA addresses. */ + rctx->src_dll.dma_addr = DMA_MAPPING_ERROR; + rctx->dst_dll.dma_addr = DMA_MAPPING_ERROR; + rctx->aad_src_dll.dma_addr = DMA_MAPPING_ERROR; + rctx->aad_dst_dll.dma_addr = DMA_MAPPING_ERROR; +} + +static int kmb_ocs_sk_validate_input(struct skcipher_request *req, + enum ocs_mode mode) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + int iv_size = crypto_skcipher_ivsize(tfm); + + switch (mode) { + case OCS_MODE_ECB: + /* Ensure input length is multiple of block size */ + if (req->cryptlen % AES_BLOCK_SIZE != 0) + return -EINVAL; + + return 0; + + case OCS_MODE_CBC: + /* Ensure input length is multiple of block size */ + if (req->cryptlen % AES_BLOCK_SIZE != 0) + return -EINVAL; + + /* Ensure IV is present and block size in length */ + if (!req->iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + /* + * NOTE: Since req->cryptlen == 0 case was already handled in + * kmb_ocs_sk_common(), the above two conditions also guarantee + * that: cryptlen >= iv_size + */ + return 0; + + case OCS_MODE_CTR: + /* Ensure IV is present and block size in length */ + if (!req->iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + return 0; + + case OCS_MODE_CTS: + /* Ensure input length >= block size */ + if (req->cryptlen < AES_BLOCK_SIZE) + return -EINVAL; + + /* Ensure IV is present and block size in length */ + if (!req->iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + return 0; + default: + return -EINVAL; + } +} + +/* + * Called by encrypt() / decrypt() skcipher functions. + * + * Use fallback if needed, otherwise initialize context and enqueue request + * into engine. + */ +static int kmb_ocs_sk_common(struct skcipher_request *req, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + enum ocs_mode mode) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + struct ocs_aes_dev *aes_dev; + int rc; + + if (tctx->use_fallback) { + SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, tctx->sw_cipher.sk); + + skcipher_request_set_sync_tfm(subreq, tctx->sw_cipher.sk); + skcipher_request_set_callback(subreq, req->base.flags, NULL, + NULL); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); + + if (instruction == OCS_ENCRYPT) + rc = crypto_skcipher_encrypt(subreq); + else + rc = crypto_skcipher_decrypt(subreq); + + skcipher_request_zero(subreq); + + return rc; + } + + /* + * If cryptlen == 0, no processing needed for ECB, CBC and CTR. + * + * For CTS continue: kmb_ocs_sk_validate_input() will return -EINVAL. + */ + if (!req->cryptlen && mode != OCS_MODE_CTS) + return 0; + + rc = kmb_ocs_sk_validate_input(req, mode); + if (rc) + return rc; + + aes_dev = kmb_ocs_aes_find_dev(tctx); + if (!aes_dev) + return -ENODEV; + + if (cipher != tctx->cipher) + return -EINVAL; + + ocs_aes_init_rctx(rctx); + rctx->instruction = instruction; + rctx->mode = mode; + + return crypto_transfer_skcipher_request_to_engine(aes_dev->engine, req); +} + +static void cleanup_ocs_dma_linked_list(struct device *dev, + struct ocs_dll_desc *dll) +{ + if (dll->vaddr) + dma_free_coherent(dev, dll->size, dll->vaddr, dll->dma_addr); + dll->vaddr = NULL; + dll->size = 0; + dll->dma_addr = DMA_MAPPING_ERROR; +} + +static void kmb_ocs_sk_dma_cleanup(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + struct device *dev = tctx->aes_dev->dev; + + if (rctx->src_dma_count) { + dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_TO_DEVICE); + rctx->src_dma_count = 0; + } + + if (rctx->dst_dma_count) { + dma_unmap_sg(dev, req->dst, rctx->dst_nents, rctx->in_place ? + DMA_BIDIRECTIONAL : + DMA_FROM_DEVICE); + rctx->dst_dma_count = 0; + } + + /* Clean up OCS DMA linked lists */ + cleanup_ocs_dma_linked_list(dev, &rctx->src_dll); + cleanup_ocs_dma_linked_list(dev, &rctx->dst_dll); +} + +static int kmb_ocs_sk_prepare_inplace(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + int iv_size = crypto_skcipher_ivsize(tfm); + int rc; + + /* + * For CBC decrypt, save last block (iv) to last_ct_blk buffer. + * + * Note: if we are here, we already checked that cryptlen >= iv_size + * and iv_size == AES_BLOCK_SIZE (i.e., the size of last_ct_blk); see + * kmb_ocs_sk_validate_input(). + */ + if (rctx->mode == OCS_MODE_CBC && rctx->instruction == OCS_DECRYPT) + scatterwalk_map_and_copy(rctx->last_ct_blk, req->src, + req->cryptlen - iv_size, iv_size, 0); + + /* For CTS decrypt, swap last two blocks, if needed. */ + if (rctx->cts_swap && rctx->instruction == OCS_DECRYPT) + sg_swap_blocks(req->dst, rctx->dst_nents, + req->cryptlen - AES_BLOCK_SIZE, + req->cryptlen - (2 * AES_BLOCK_SIZE)); + + /* src and dst buffers are the same, use bidirectional DMA mapping. */ + rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst, + rctx->dst_nents, DMA_BIDIRECTIONAL); + if (rctx->dst_dma_count == 0) { + dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n"); + return -ENOMEM; + } + + /* Create DST linked list */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, &rctx->dst_dll, + req->cryptlen, 0); + if (rc) + return rc; + /* + * If descriptor creation was successful, set the src_dll.dma_addr to + * the value of dst_dll.dma_addr, as we do in-place AES operation on + * the src. + */ + rctx->src_dll.dma_addr = rctx->dst_dll.dma_addr; + + return 0; +} + +static int kmb_ocs_sk_prepare_notinplace(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + int rc; + + rctx->src_nents = sg_nents_for_len(req->src, req->cryptlen); + if (rctx->src_nents < 0) + return -EBADMSG; + + /* Map SRC SG. */ + rctx->src_dma_count = dma_map_sg(tctx->aes_dev->dev, req->src, + rctx->src_nents, DMA_TO_DEVICE); + if (rctx->src_dma_count == 0) { + dev_err(tctx->aes_dev->dev, "Failed to map source sg\n"); + return -ENOMEM; + } + + /* Create SRC linked list */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src, + rctx->src_dma_count, &rctx->src_dll, + req->cryptlen, 0); + if (rc) + return rc; + + /* Map DST SG. */ + rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst, + rctx->dst_nents, DMA_FROM_DEVICE); + if (rctx->dst_dma_count == 0) { + dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n"); + return -ENOMEM; + } + + /* Create DST linked list */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, &rctx->dst_dll, + req->cryptlen, 0); + if (rc) + return rc; + + /* If this is not a CTS decrypt operation with swapping, we are done. */ + if (!(rctx->cts_swap && rctx->instruction == OCS_DECRYPT)) + return 0; + + /* + * Otherwise, we have to copy src to dst (as we cannot modify src). + * Use OCS AES bypass mode to copy src to dst via DMA. + * + * NOTE: for anything other than small data sizes this is rather + * inefficient. + */ + rc = ocs_aes_bypass_op(tctx->aes_dev, rctx->dst_dll.dma_addr, + rctx->src_dll.dma_addr, req->cryptlen); + if (rc) + return rc; + + /* + * Now dst == src, so clean up what we did so far and use in_place + * logic. + */ + kmb_ocs_sk_dma_cleanup(req); + rctx->in_place = true; + + return kmb_ocs_sk_prepare_inplace(req); +} + +static int kmb_ocs_sk_run(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + struct ocs_aes_dev *aes_dev = tctx->aes_dev; + int iv_size = crypto_skcipher_ivsize(tfm); + int rc; + + rctx->dst_nents = sg_nents_for_len(req->dst, req->cryptlen); + if (rctx->dst_nents < 0) + return -EBADMSG; + + /* + * If 2 blocks or greater, and multiple of block size swap last two + * blocks to be compatible with other crypto API CTS implementations: + * OCS mode uses CBC-CS2, whereas other crypto API implementations use + * CBC-CS3. + * CBC-CS2 and CBC-CS3 defined by: + * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38a-add.pdf + */ + rctx->cts_swap = (rctx->mode == OCS_MODE_CTS && + req->cryptlen > AES_BLOCK_SIZE && + req->cryptlen % AES_BLOCK_SIZE == 0); + + rctx->in_place = (req->src == req->dst); + + if (rctx->in_place) + rc = kmb_ocs_sk_prepare_inplace(req); + else + rc = kmb_ocs_sk_prepare_notinplace(req); + + if (rc) + goto error; + + rc = ocs_aes_op(aes_dev, rctx->mode, tctx->cipher, rctx->instruction, + rctx->dst_dll.dma_addr, rctx->src_dll.dma_addr, + req->cryptlen, req->iv, iv_size); + if (rc) + goto error; + + /* Clean-up DMA before further processing output. */ + kmb_ocs_sk_dma_cleanup(req); + + /* For CTS Encrypt, swap last 2 blocks, if needed. */ + if (rctx->cts_swap && rctx->instruction == OCS_ENCRYPT) { + sg_swap_blocks(req->dst, rctx->dst_nents, + req->cryptlen - AES_BLOCK_SIZE, + req->cryptlen - (2 * AES_BLOCK_SIZE)); + return 0; + } + + /* For CBC copy IV to req->IV. */ + if (rctx->mode == OCS_MODE_CBC) { + /* CBC encrypt case. */ + if (rctx->instruction == OCS_ENCRYPT) { + scatterwalk_map_and_copy(req->iv, req->dst, + req->cryptlen - iv_size, + iv_size, 0); + return 0; + } + /* CBC decrypt case. */ + if (rctx->in_place) + memcpy(req->iv, rctx->last_ct_blk, iv_size); + else + scatterwalk_map_and_copy(req->iv, req->src, + req->cryptlen - iv_size, + iv_size, 0); + return 0; + } + /* For all other modes there's nothing to do. */ + + return 0; + +error: + kmb_ocs_sk_dma_cleanup(req); + + return rc; +} + +static int kmb_ocs_aead_validate_input(struct aead_request *req, + enum ocs_instruction instruction, + enum ocs_mode mode) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + int tag_size = crypto_aead_authsize(tfm); + int iv_size = crypto_aead_ivsize(tfm); + + /* For decrypt crytplen == len(PT) + len(tag). */ + if (instruction == OCS_DECRYPT && req->cryptlen < tag_size) + return -EINVAL; + + /* IV is mandatory. */ + if (!req->iv) + return -EINVAL; + + switch (mode) { + case OCS_MODE_GCM: + if (iv_size != GCM_AES_IV_SIZE) + return -EINVAL; + + return 0; + + case OCS_MODE_CCM: + /* Ensure IV is present and block size in length */ + if (iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + return 0; + + default: + return -EINVAL; + } +} + +/* + * Called by encrypt() / decrypt() aead functions. + * + * Use fallback if needed, otherwise initialize context and enqueue request + * into engine. + */ +static int kmb_ocs_aead_common(struct aead_request *req, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + enum ocs_mode mode) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct ocs_aes_rctx *rctx = aead_request_ctx(req); + struct ocs_aes_dev *dd; + int rc; + + if (tctx->use_fallback) { + struct aead_request *subreq = aead_request_ctx(req); + + aead_request_set_tfm(subreq, tctx->sw_cipher.aead); + aead_request_set_callback(subreq, req->base.flags, + req->base.complete, req->base.data); + aead_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); + aead_request_set_ad(subreq, req->assoclen); + rc = crypto_aead_setauthsize(tctx->sw_cipher.aead, + crypto_aead_authsize(crypto_aead_reqtfm(req))); + if (rc) + return rc; + + return (instruction == OCS_ENCRYPT) ? + crypto_aead_encrypt(subreq) : + crypto_aead_decrypt(subreq); + } + + rc = kmb_ocs_aead_validate_input(req, instruction, mode); + if (rc) + return rc; + + dd = kmb_ocs_aes_find_dev(tctx); + if (!dd) + return -ENODEV; + + if (cipher != tctx->cipher) + return -EINVAL; + + ocs_aes_init_rctx(rctx); + rctx->instruction = instruction; + rctx->mode = mode; + + return crypto_transfer_aead_request_to_engine(dd->engine, req); +} + +static void kmb_ocs_aead_dma_cleanup(struct aead_request *req) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct ocs_aes_rctx *rctx = aead_request_ctx(req); + struct device *dev = tctx->aes_dev->dev; + + if (rctx->src_dma_count) { + dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_TO_DEVICE); + rctx->src_dma_count = 0; + } + + if (rctx->dst_dma_count) { + dma_unmap_sg(dev, req->dst, rctx->dst_nents, rctx->in_place ? + DMA_BIDIRECTIONAL : + DMA_FROM_DEVICE); + rctx->dst_dma_count = 0; + } + /* Clean up OCS DMA linked lists */ + cleanup_ocs_dma_linked_list(dev, &rctx->src_dll); + cleanup_ocs_dma_linked_list(dev, &rctx->dst_dll); + cleanup_ocs_dma_linked_list(dev, &rctx->aad_src_dll); + cleanup_ocs_dma_linked_list(dev, &rctx->aad_dst_dll); +} + +/** + * kmb_ocs_aead_dma_prepare() - Do DMA mapping for AEAD processing. + * @req: The AEAD request being processed. + * @src_dll_size: Where to store the length of the data mapped into the + * src_dll OCS DMA list. + * + * Do the following: + * - DMA map req->src and req->dst + * - Initialize the following OCS DMA linked lists: rctx->src_dll, + * rctx->dst_dll, rctx->aad_src_dll and rxtc->aad_dst_dll. + * + * Return: 0 on success, negative error code otherwise. + */ +static int kmb_ocs_aead_dma_prepare(struct aead_request *req, u32 *src_dll_size) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + const int tag_size = crypto_aead_authsize(crypto_aead_reqtfm(req)); + struct ocs_aes_rctx *rctx = aead_request_ctx(req); + u32 in_size; /* The length of the data to be mapped by src_dll. */ + u32 out_size; /* The length of the data to be mapped by dst_dll. */ + u32 dst_size; /* The length of the data in dst_sg. */ + int rc; + + /* Get number of entries in input data SG list. */ + rctx->src_nents = sg_nents_for_len(req->src, + req->assoclen + req->cryptlen); + if (rctx->src_nents < 0) + return -EBADMSG; + + if (rctx->instruction == OCS_DECRYPT) { + /* + * For decrypt: + * - src sg list is: AAD|CT|tag + * - dst sg list expects: AAD|PT + * + * in_size == len(CT); out_size == len(PT) + */ + + /* req->cryptlen includes both CT and tag. */ + in_size = req->cryptlen - tag_size; + + /* out_size = PT size == CT size */ + out_size = in_size; + + /* len(dst_sg) == len(AAD) + len(PT) */ + dst_size = req->assoclen + out_size; + + /* + * Copy tag from source SG list to 'in_tag' buffer. + * + * Note: this needs to be done here, before DMA mapping src_sg. + */ + sg_pcopy_to_buffer(req->src, rctx->src_nents, rctx->in_tag, + tag_size, req->assoclen + in_size); + + } else { /* OCS_ENCRYPT */ + /* + * For encrypt: + * src sg list is: AAD|PT + * dst sg list expects: AAD|CT|tag + */ + /* in_size == len(PT) */ + in_size = req->cryptlen; + + /* + * In CCM mode the OCS engine appends the tag to the ciphertext, + * but in GCM mode the tag must be read from the tag registers + * and appended manually below + */ + out_size = (rctx->mode == OCS_MODE_CCM) ? in_size + tag_size : + in_size; + /* len(dst_sg) == len(AAD) + len(CT) + len(tag) */ + dst_size = req->assoclen + in_size + tag_size; + } + *src_dll_size = in_size; + + /* Get number of entries in output data SG list. */ + rctx->dst_nents = sg_nents_for_len(req->dst, dst_size); + if (rctx->dst_nents < 0) + return -EBADMSG; + + rctx->in_place = (req->src == req->dst) ? 1 : 0; + + /* Map destination; use bidirectional mapping for in-place case. */ + rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst, + rctx->dst_nents, + rctx->in_place ? DMA_BIDIRECTIONAL : + DMA_FROM_DEVICE); + if (rctx->dst_dma_count == 0 && rctx->dst_nents != 0) { + dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n"); + return -ENOMEM; + } + + /* Create AAD DST list: maps dst[0:AAD_SIZE-1]. */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, + &rctx->aad_dst_dll, req->assoclen, + 0); + if (rc) + return rc; + + /* Create DST list: maps dst[AAD_SIZE:out_size] */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, &rctx->dst_dll, + out_size, req->assoclen); + if (rc) + return rc; + + if (rctx->in_place) { + /* If this is not CCM encrypt, we are done. */ + if (!(rctx->mode == OCS_MODE_CCM && + rctx->instruction == OCS_ENCRYPT)) { + /* + * SRC and DST are the same, so re-use the same DMA + * addresses (to avoid allocating new DMA lists + * identical to the dst ones). + */ + rctx->src_dll.dma_addr = rctx->dst_dll.dma_addr; + rctx->aad_src_dll.dma_addr = rctx->aad_dst_dll.dma_addr; + + return 0; + } + /* + * For CCM encrypt the input and output linked lists contain + * different amounts of data, so, we need to create different + * SRC and AAD SRC lists, even for the in-place case. + */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, + &rctx->aad_src_dll, + req->assoclen, 0); + if (rc) + return rc; + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, + &rctx->src_dll, in_size, + req->assoclen); + if (rc) + return rc; + + return 0; + } + /* Not in-place case. */ + + /* Map source SG. */ + rctx->src_dma_count = dma_map_sg(tctx->aes_dev->dev, req->src, + rctx->src_nents, DMA_TO_DEVICE); + if (rctx->src_dma_count == 0 && rctx->src_nents != 0) { + dev_err(tctx->aes_dev->dev, "Failed to map source sg\n"); + return -ENOMEM; + } + + /* Create AAD SRC list. */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src, + rctx->src_dma_count, + &rctx->aad_src_dll, + req->assoclen, 0); + if (rc) + return rc; + + /* Create SRC list. */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src, + rctx->src_dma_count, + &rctx->src_dll, in_size, + req->assoclen); + if (rc) + return rc; + + if (req->assoclen == 0) + return 0; + + /* Copy AAD from src sg to dst sg using OCS DMA. */ + rc = ocs_aes_bypass_op(tctx->aes_dev, rctx->aad_dst_dll.dma_addr, + rctx->aad_src_dll.dma_addr, req->cryptlen); + if (rc) + dev_err(tctx->aes_dev->dev, + "Failed to copy source AAD to destination AAD\n"); + + return rc; +} + +static int kmb_ocs_aead_run(struct aead_request *req) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + const int tag_size = crypto_aead_authsize(crypto_aead_reqtfm(req)); + struct ocs_aes_rctx *rctx = aead_request_ctx(req); + u32 in_size; /* The length of the data mapped by src_dll. */ + int rc; + + rc = kmb_ocs_aead_dma_prepare(req, &in_size); + if (rc) + goto exit; + + /* For CCM, we just call the OCS processing and we are done. */ + if (rctx->mode == OCS_MODE_CCM) { + rc = ocs_aes_ccm_op(tctx->aes_dev, tctx->cipher, + rctx->instruction, rctx->dst_dll.dma_addr, + rctx->src_dll.dma_addr, in_size, + req->iv, + rctx->aad_src_dll.dma_addr, req->assoclen, + rctx->in_tag, tag_size); + goto exit; + } + /* GCM case; invoke OCS processing. */ + rc = ocs_aes_gcm_op(tctx->aes_dev, tctx->cipher, + rctx->instruction, + rctx->dst_dll.dma_addr, + rctx->src_dll.dma_addr, in_size, + req->iv, + rctx->aad_src_dll.dma_addr, req->assoclen, + rctx->out_tag, tag_size); + if (rc) + goto exit; + + /* For GCM decrypt, we have to compare in_tag with out_tag. */ + if (rctx->instruction == OCS_DECRYPT) { + rc = memcmp(rctx->in_tag, rctx->out_tag, tag_size) ? + -EBADMSG : 0; + goto exit; + } + + /* For GCM encrypt, we must manually copy out_tag to DST sg. */ + + /* Clean-up must be called before the sg_pcopy_from_buffer() below. */ + kmb_ocs_aead_dma_cleanup(req); + + /* Copy tag to destination sg after AAD and CT. */ + sg_pcopy_from_buffer(req->dst, rctx->dst_nents, rctx->out_tag, + tag_size, req->assoclen + req->cryptlen); + + /* Return directly as DMA cleanup already done. */ + return 0; + +exit: + kmb_ocs_aead_dma_cleanup(req); + + return rc; +} + +static int kmb_ocs_aes_sk_do_one_request(struct crypto_engine *engine, + void *areq) +{ + struct skcipher_request *req = + container_of(areq, struct skcipher_request, base); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + int err; + + if (!tctx->aes_dev) { + err = -ENODEV; + goto exit; + } + + err = ocs_aes_set_key(tctx->aes_dev, tctx->key_len, tctx->key, + tctx->cipher); + if (err) + goto exit; + + err = kmb_ocs_sk_run(req); + +exit: + crypto_finalize_skcipher_request(engine, req, err); + + return 0; +} + +static int kmb_ocs_aes_aead_do_one_request(struct crypto_engine *engine, + void *areq) +{ + struct aead_request *req = container_of(areq, + struct aead_request, base); + struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + int err; + + if (!tctx->aes_dev) + return -ENODEV; + + err = ocs_aes_set_key(tctx->aes_dev, tctx->key_len, tctx->key, + tctx->cipher); + if (err) + goto exit; + + err = kmb_ocs_aead_run(req); + +exit: + crypto_finalize_aead_request(tctx->aes_dev->engine, req, err); + + return 0; +} + +static int kmb_ocs_aes_set_key(struct crypto_skcipher *tfm, const u8 *in_key, + unsigned int key_len) +{ + return kmb_ocs_sk_set_key(tfm, in_key, key_len, OCS_AES); +} + +static int kmb_ocs_aes_aead_set_key(struct crypto_aead *tfm, const u8 *in_key, + unsigned int key_len) +{ + return kmb_ocs_aead_set_key(tfm, in_key, key_len, OCS_AES); +} + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB +static int kmb_ocs_aes_ecb_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_ECB); +} + +static int kmb_ocs_aes_ecb_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_ECB); +} +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ + +static int kmb_ocs_aes_cbc_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CBC); +} + +static int kmb_ocs_aes_cbc_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CBC); +} + +static int kmb_ocs_aes_ctr_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CTR); +} + +static int kmb_ocs_aes_ctr_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CTR); +} + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS +static int kmb_ocs_aes_cts_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CTS); +} + +static int kmb_ocs_aes_cts_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CTS); +} +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ + +static int kmb_ocs_aes_gcm_encrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_GCM); +} + +static int kmb_ocs_aes_gcm_decrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_GCM); +} + +static int kmb_ocs_aes_ccm_encrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CCM); +} + +static int kmb_ocs_aes_ccm_decrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CCM); +} + +static int kmb_ocs_sm4_set_key(struct crypto_skcipher *tfm, const u8 *in_key, + unsigned int key_len) +{ + return kmb_ocs_sk_set_key(tfm, in_key, key_len, OCS_SM4); +} + +static int kmb_ocs_sm4_aead_set_key(struct crypto_aead *tfm, const u8 *in_key, + unsigned int key_len) +{ + return kmb_ocs_aead_set_key(tfm, in_key, key_len, OCS_SM4); +} + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB +static int kmb_ocs_sm4_ecb_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_ECB); +} + +static int kmb_ocs_sm4_ecb_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_ECB); +} +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ + +static int kmb_ocs_sm4_cbc_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CBC); +} + +static int kmb_ocs_sm4_cbc_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CBC); +} + +static int kmb_ocs_sm4_ctr_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CTR); +} + +static int kmb_ocs_sm4_ctr_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CTR); +} + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS +static int kmb_ocs_sm4_cts_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CTS); +} + +static int kmb_ocs_sm4_cts_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CTS); +} +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ + +static int kmb_ocs_sm4_gcm_encrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_GCM); +} + +static int kmb_ocs_sm4_gcm_decrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_GCM); +} + +static int kmb_ocs_sm4_ccm_encrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CCM); +} + +static int kmb_ocs_sm4_ccm_decrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CCM); +} + +static inline int ocs_common_init(struct ocs_aes_tctx *tctx) +{ + tctx->engine_ctx.op.prepare_request = NULL; + tctx->engine_ctx.op.do_one_request = kmb_ocs_aes_sk_do_one_request; + tctx->engine_ctx.op.unprepare_request = NULL; + + return 0; +} + +static int ocs_aes_init_tfm(struct crypto_skcipher *tfm) +{ + const char *alg_name = crypto_tfm_alg_name(&tfm->base); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + struct crypto_sync_skcipher *blk; + + /* set fallback cipher in case it will be needed */ + blk = crypto_alloc_sync_skcipher(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(blk)) + return PTR_ERR(blk); + + tctx->sw_cipher.sk = blk; + + crypto_skcipher_set_reqsize(tfm, sizeof(struct ocs_aes_rctx)); + + return ocs_common_init(tctx); +} + +static int ocs_sm4_init_tfm(struct crypto_skcipher *tfm) +{ + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + + crypto_skcipher_set_reqsize(tfm, sizeof(struct ocs_aes_rctx)); + + return ocs_common_init(tctx); +} + +static inline void clear_key(struct ocs_aes_tctx *tctx) +{ + memzero_explicit(tctx->key, OCS_AES_KEYSIZE_256); + + /* Zero key registers if set */ + if (tctx->aes_dev) + ocs_aes_set_key(tctx->aes_dev, OCS_AES_KEYSIZE_256, + tctx->key, OCS_AES); +} + +static void ocs_exit_tfm(struct crypto_skcipher *tfm) +{ + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + + clear_key(tctx); + + if (tctx->sw_cipher.sk) { + crypto_free_sync_skcipher(tctx->sw_cipher.sk); + tctx->sw_cipher.sk = NULL; + } +} + +static inline int ocs_common_aead_init(struct ocs_aes_tctx *tctx) +{ + tctx->engine_ctx.op.prepare_request = NULL; + tctx->engine_ctx.op.do_one_request = kmb_ocs_aes_aead_do_one_request; + tctx->engine_ctx.op.unprepare_request = NULL; + + return 0; +} + +static int ocs_aes_aead_cra_init(struct crypto_aead *tfm) +{ + const char *alg_name = crypto_tfm_alg_name(&tfm->base); + struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); + struct crypto_aead *blk; + + /* Set fallback cipher in case it will be needed */ + blk = crypto_alloc_aead(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(blk)) + return PTR_ERR(blk); + + tctx->sw_cipher.aead = blk; + + crypto_aead_set_reqsize(tfm, + max(sizeof(struct ocs_aes_rctx), + (sizeof(struct aead_request) + + crypto_aead_reqsize(tctx->sw_cipher.aead)))); + + return ocs_common_aead_init(tctx); +} + +static int kmb_ocs_aead_ccm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + switch (authsize) { + case 4: + case 6: + case 8: + case 10: + case 12: + case 14: + case 16: + return 0; + default: + return -EINVAL; + } +} + +static int kmb_ocs_aead_gcm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + return crypto_gcm_check_authsize(authsize); +} + +static int ocs_sm4_aead_cra_init(struct crypto_aead *tfm) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); + + crypto_aead_set_reqsize(tfm, sizeof(struct ocs_aes_rctx)); + + return ocs_common_aead_init(tctx); +} + +static void ocs_aead_cra_exit(struct crypto_aead *tfm) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); + + clear_key(tctx); + + if (tctx->sw_cipher.aead) { + crypto_free_aead(tctx->sw_cipher.aead); + tctx->sw_cipher.aead = NULL; + } +} + +static struct skcipher_alg algs[] = { +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB + { + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "ecb-aes-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_AES_MIN_KEY_SIZE, + .max_keysize = OCS_AES_MAX_KEY_SIZE, + .setkey = kmb_ocs_aes_set_key, + .encrypt = kmb_ocs_aes_ecb_encrypt, + .decrypt = kmb_ocs_aes_ecb_decrypt, + .init = ocs_aes_init_tfm, + .exit = ocs_exit_tfm, + }, +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ + { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cbc-aes-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_AES_MIN_KEY_SIZE, + .max_keysize = OCS_AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = kmb_ocs_aes_set_key, + .encrypt = kmb_ocs_aes_cbc_encrypt, + .decrypt = kmb_ocs_aes_cbc_decrypt, + .init = ocs_aes_init_tfm, + .exit = ocs_exit_tfm, + }, + { + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "ctr-aes-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_AES_MIN_KEY_SIZE, + .max_keysize = OCS_AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = kmb_ocs_aes_set_key, + .encrypt = kmb_ocs_aes_ctr_encrypt, + .decrypt = kmb_ocs_aes_ctr_decrypt, + .init = ocs_aes_init_tfm, + .exit = ocs_exit_tfm, + }, +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS + { + .base.cra_name = "cts(cbc(aes))", + .base.cra_driver_name = "cts-aes-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_AES_MIN_KEY_SIZE, + .max_keysize = OCS_AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = kmb_ocs_aes_set_key, + .encrypt = kmb_ocs_aes_cts_encrypt, + .decrypt = kmb_ocs_aes_cts_decrypt, + .init = ocs_aes_init_tfm, + .exit = ocs_exit_tfm, + }, +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB + { + .base.cra_name = "ecb(sm4)", + .base.cra_driver_name = "ecb-sm4-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_SM4_KEY_SIZE, + .max_keysize = OCS_SM4_KEY_SIZE, + .setkey = kmb_ocs_sm4_set_key, + .encrypt = kmb_ocs_sm4_ecb_encrypt, + .decrypt = kmb_ocs_sm4_ecb_decrypt, + .init = ocs_sm4_init_tfm, + .exit = ocs_exit_tfm, + }, +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ + { + .base.cra_name = "cbc(sm4)", + .base.cra_driver_name = "cbc-sm4-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_SM4_KEY_SIZE, + .max_keysize = OCS_SM4_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = kmb_ocs_sm4_set_key, + .encrypt = kmb_ocs_sm4_cbc_encrypt, + .decrypt = kmb_ocs_sm4_cbc_decrypt, + .init = ocs_sm4_init_tfm, + .exit = ocs_exit_tfm, + }, + { + .base.cra_name = "ctr(sm4)", + .base.cra_driver_name = "ctr-sm4-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_SM4_KEY_SIZE, + .max_keysize = OCS_SM4_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = kmb_ocs_sm4_set_key, + .encrypt = kmb_ocs_sm4_ctr_encrypt, + .decrypt = kmb_ocs_sm4_ctr_decrypt, + .init = ocs_sm4_init_tfm, + .exit = ocs_exit_tfm, + }, +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS + { + .base.cra_name = "cts(cbc(sm4))", + .base.cra_driver_name = "cts-sm4-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_SM4_KEY_SIZE, + .max_keysize = OCS_SM4_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = kmb_ocs_sm4_set_key, + .encrypt = kmb_ocs_sm4_cts_encrypt, + .decrypt = kmb_ocs_sm4_cts_decrypt, + .init = ocs_sm4_init_tfm, + .exit = ocs_exit_tfm, + } +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ +}; + +static struct aead_alg algs_aead[] = { + { + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "gcm-aes-keembay-ocs", + .cra_priority = KMB_OCS_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct ocs_aes_tctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = ocs_aes_aead_cra_init, + .exit = ocs_aead_cra_exit, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .setauthsize = kmb_ocs_aead_gcm_setauthsize, + .setkey = kmb_ocs_aes_aead_set_key, + .encrypt = kmb_ocs_aes_gcm_encrypt, + .decrypt = kmb_ocs_aes_gcm_decrypt, + }, + { + .base = { + .cra_name = "ccm(aes)", + .cra_driver_name = "ccm-aes-keembay-ocs", + .cra_priority = KMB_OCS_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct ocs_aes_tctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = ocs_aes_aead_cra_init, + .exit = ocs_aead_cra_exit, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .setauthsize = kmb_ocs_aead_ccm_setauthsize, + .setkey = kmb_ocs_aes_aead_set_key, + .encrypt = kmb_ocs_aes_ccm_encrypt, + .decrypt = kmb_ocs_aes_ccm_decrypt, + }, + { + .base = { + .cra_name = "gcm(sm4)", + .cra_driver_name = "gcm-sm4-keembay-ocs", + .cra_priority = KMB_OCS_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct ocs_aes_tctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = ocs_sm4_aead_cra_init, + .exit = ocs_aead_cra_exit, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .setauthsize = kmb_ocs_aead_gcm_setauthsize, + .setkey = kmb_ocs_sm4_aead_set_key, + .encrypt = kmb_ocs_sm4_gcm_encrypt, + .decrypt = kmb_ocs_sm4_gcm_decrypt, + }, + { + .base = { + .cra_name = "ccm(sm4)", + .cra_driver_name = "ccm-sm4-keembay-ocs", + .cra_priority = KMB_OCS_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct ocs_aes_tctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = ocs_sm4_aead_cra_init, + .exit = ocs_aead_cra_exit, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .setauthsize = kmb_ocs_aead_ccm_setauthsize, + .setkey = kmb_ocs_sm4_aead_set_key, + .encrypt = kmb_ocs_sm4_ccm_encrypt, + .decrypt = kmb_ocs_sm4_ccm_decrypt, + } +}; + +static void unregister_aes_algs(struct ocs_aes_dev *aes_dev) +{ + crypto_unregister_aeads(algs_aead, ARRAY_SIZE(algs_aead)); + crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); +} + +static int register_aes_algs(struct ocs_aes_dev *aes_dev) +{ + int ret; + + /* + * If any algorithm fails to register, all preceding algorithms that + * were successfully registered will be automatically unregistered. + */ + ret = crypto_register_aeads(algs_aead, ARRAY_SIZE(algs_aead)); + if (ret) + return ret; + + ret = crypto_register_skciphers(algs, ARRAY_SIZE(algs)); + if (ret) + crypto_unregister_aeads(algs_aead, ARRAY_SIZE(algs)); + + return ret; +} + +/* Device tree driver match. */ +static const struct of_device_id kmb_ocs_aes_of_match[] = { + { + .compatible = "intel,keembay-ocs-aes", + }, + {} +}; + +static int kmb_ocs_aes_remove(struct platform_device *pdev) +{ + struct ocs_aes_dev *aes_dev; + + aes_dev = platform_get_drvdata(pdev); + if (!aes_dev) + return -ENODEV; + + unregister_aes_algs(aes_dev); + + spin_lock(&ocs_aes.lock); + list_del(&aes_dev->list); + spin_unlock(&ocs_aes.lock); + + crypto_engine_exit(aes_dev->engine); + + return 0; +} + +static int kmb_ocs_aes_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct ocs_aes_dev *aes_dev; + struct resource *aes_mem; + int rc; + + aes_dev = devm_kzalloc(dev, sizeof(*aes_dev), GFP_KERNEL); + if (!aes_dev) + return -ENOMEM; + + aes_dev->dev = dev; + + platform_set_drvdata(pdev, aes_dev); + + rc = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); + if (rc) { + dev_err(dev, "Failed to set 32 bit dma mask %d\n", rc); + return rc; + } + + /* Get base register address. */ + aes_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!aes_mem) { + dev_err(dev, "Could not retrieve io mem resource\n"); + return -ENODEV; + } + + aes_dev->base_reg = devm_ioremap_resource(&pdev->dev, aes_mem); + if (IS_ERR(aes_dev->base_reg)) { + dev_err(dev, "Failed to get base address\n"); + return PTR_ERR(aes_dev->base_reg); + } + + /* Get and request IRQ */ + aes_dev->irq = platform_get_irq(pdev, 0); + if (aes_dev->irq < 0) + return aes_dev->irq; + + rc = devm_request_threaded_irq(dev, aes_dev->irq, ocs_aes_irq_handler, + NULL, 0, "keembay-ocs-aes", aes_dev); + if (rc < 0) { + dev_err(dev, "Could not request IRQ\n"); + return rc; + } + + INIT_LIST_HEAD(&aes_dev->list); + spin_lock(&ocs_aes.lock); + list_add_tail(&aes_dev->list, &ocs_aes.dev_list); + spin_unlock(&ocs_aes.lock); + + init_completion(&aes_dev->irq_completion); + + /* Initialize crypto engine */ + aes_dev->engine = crypto_engine_alloc_init(dev, true); + if (!aes_dev->engine) + goto list_del; + + rc = crypto_engine_start(aes_dev->engine); + if (rc) { + dev_err(dev, "Could not start crypto engine\n"); + goto cleanup; + } + + rc = register_aes_algs(aes_dev); + if (rc) { + dev_err(dev, + "Could not register OCS algorithms with Crypto API\n"); + goto cleanup; + } + + return 0; + +cleanup: + crypto_engine_exit(aes_dev->engine); +list_del: + spin_lock(&ocs_aes.lock); + list_del(&aes_dev->list); + spin_unlock(&ocs_aes.lock); + + return rc; +} + +/* The OCS driver is a platform device. */ +static struct platform_driver kmb_ocs_aes_driver = { + .probe = kmb_ocs_aes_probe, + .remove = kmb_ocs_aes_remove, + .driver = { + .name = DRV_NAME, + .of_match_table = kmb_ocs_aes_of_match, + }, +}; + +module_platform_driver(kmb_ocs_aes_driver); + +MODULE_DESCRIPTION("Intel Keem Bay Offload and Crypto Subsystem (OCS) AES/SM4 Driver"); +MODULE_LICENSE("GPL"); + +MODULE_ALIAS_CRYPTO("cbc-aes-keembay-ocs"); +MODULE_ALIAS_CRYPTO("ctr-aes-keembay-ocs"); +MODULE_ALIAS_CRYPTO("gcm-aes-keembay-ocs"); +MODULE_ALIAS_CRYPTO("ccm-aes-keembay-ocs"); + +MODULE_ALIAS_CRYPTO("cbc-sm4-keembay-ocs"); +MODULE_ALIAS_CRYPTO("ctr-sm4-keembay-ocs"); +MODULE_ALIAS_CRYPTO("gcm-sm4-keembay-ocs"); +MODULE_ALIAS_CRYPTO("ccm-sm4-keembay-ocs"); + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB +MODULE_ALIAS_CRYPTO("ecb-aes-keembay-ocs"); +MODULE_ALIAS_CRYPTO("ecb-sm4-keembay-ocs"); +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS +MODULE_ALIAS_CRYPTO("cts-aes-keembay-ocs"); +MODULE_ALIAS_CRYPTO("cts-sm4-keembay-ocs"); +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ diff --git a/drivers/crypto/keembay/ocs-aes.c b/drivers/crypto/keembay/ocs-aes.c new file mode 100644 index 000000000000..cc286adb1c4a --- /dev/null +++ b/drivers/crypto/keembay/ocs-aes.c @@ -0,0 +1,1489 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Intel Keem Bay OCS AES Crypto Driver. + * + * Copyright (C) 2018-2020 Intel Corporation + */ + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include "ocs-aes.h" + +#define AES_COMMAND_OFFSET 0x0000 +#define AES_KEY_0_OFFSET 0x0004 +#define AES_KEY_1_OFFSET 0x0008 +#define AES_KEY_2_OFFSET 0x000C +#define AES_KEY_3_OFFSET 0x0010 +#define AES_KEY_4_OFFSET 0x0014 +#define AES_KEY_5_OFFSET 0x0018 +#define AES_KEY_6_OFFSET 0x001C +#define AES_KEY_7_OFFSET 0x0020 +#define AES_IV_0_OFFSET 0x0024 +#define AES_IV_1_OFFSET 0x0028 +#define AES_IV_2_OFFSET 0x002C +#define AES_IV_3_OFFSET 0x0030 +#define AES_ACTIVE_OFFSET 0x0034 +#define AES_STATUS_OFFSET 0x0038 +#define AES_KEY_SIZE_OFFSET 0x0044 +#define AES_IER_OFFSET 0x0048 +#define AES_ISR_OFFSET 0x005C +#define AES_MULTIPURPOSE1_0_OFFSET 0x0200 +#define AES_MULTIPURPOSE1_1_OFFSET 0x0204 +#define AES_MULTIPURPOSE1_2_OFFSET 0x0208 +#define AES_MULTIPURPOSE1_3_OFFSET 0x020C +#define AES_MULTIPURPOSE2_0_OFFSET 0x0220 +#define AES_MULTIPURPOSE2_1_OFFSET 0x0224 +#define AES_MULTIPURPOSE2_2_OFFSET 0x0228 +#define AES_MULTIPURPOSE2_3_OFFSET 0x022C +#define AES_BYTE_ORDER_CFG_OFFSET 0x02C0 +#define AES_TLEN_OFFSET 0x0300 +#define AES_T_MAC_0_OFFSET 0x0304 +#define AES_T_MAC_1_OFFSET 0x0308 +#define AES_T_MAC_2_OFFSET 0x030C +#define AES_T_MAC_3_OFFSET 0x0310 +#define AES_PLEN_OFFSET 0x0314 +#define AES_A_DMA_SRC_ADDR_OFFSET 0x0400 +#define AES_A_DMA_DST_ADDR_OFFSET 0x0404 +#define AES_A_DMA_SRC_SIZE_OFFSET 0x0408 +#define AES_A_DMA_DST_SIZE_OFFSET 0x040C +#define AES_A_DMA_DMA_MODE_OFFSET 0x0410 +#define AES_A_DMA_NEXT_SRC_DESCR_OFFSET 0x0418 +#define AES_A_DMA_NEXT_DST_DESCR_OFFSET 0x041C +#define AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET 0x0420 +#define AES_A_DMA_LOG_OFFSET 0x0424 +#define AES_A_DMA_STATUS_OFFSET 0x0428 +#define AES_A_DMA_PERF_CNTR_OFFSET 0x042C +#define AES_A_DMA_MSI_ISR_OFFSET 0x0480 +#define AES_A_DMA_MSI_IER_OFFSET 0x0484 +#define AES_A_DMA_MSI_MASK_OFFSET 0x0488 +#define AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET 0x0600 +#define AES_A_DMA_OUTBUFFER_READ_FIFO_OFFSET 0x0700 + +/* + * AES_A_DMA_DMA_MODE register. + * Default: 0x00000000. + * bit[31] ACTIVE + * This bit activates the DMA. When the DMA finishes, it resets + * this bit to zero. + * bit[30:26] Unused by this driver. + * bit[25] SRC_LINK_LIST_EN + * Source link list enable bit. When the linked list is terminated + * this bit is reset by the DMA. + * bit[24] DST_LINK_LIST_EN + * Destination link list enable bit. When the linked list is + * terminated this bit is reset by the DMA. + * bit[23:0] Unused by this driver. + */ +#define AES_A_DMA_DMA_MODE_ACTIVE BIT(31) +#define AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN BIT(25) +#define AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN BIT(24) + +/* + * AES_ACTIVE register + * default 0x00000000 + * bit[31:10] Reserved + * bit[9] LAST_ADATA + * bit[8] LAST_GCX + * bit[7:2] Reserved + * bit[1] TERMINATION + * bit[0] TRIGGER + */ +#define AES_ACTIVE_LAST_ADATA BIT(9) +#define AES_ACTIVE_LAST_CCM_GCM BIT(8) +#define AES_ACTIVE_TERMINATION BIT(1) +#define AES_ACTIVE_TRIGGER BIT(0) + +#define AES_DISABLE_INT 0x00000000 +#define AES_DMA_CPD_ERR_INT BIT(8) +#define AES_DMA_OUTBUF_RD_ERR_INT BIT(7) +#define AES_DMA_OUTBUF_WR_ERR_INT BIT(6) +#define AES_DMA_INBUF_RD_ERR_INT BIT(5) +#define AES_DMA_INBUF_WR_ERR_INT BIT(4) +#define AES_DMA_BAD_COMP_INT BIT(3) +#define AES_DMA_SAI_INT BIT(2) +#define AES_DMA_SRC_DONE_INT BIT(0) +#define AES_COMPLETE_INT BIT(1) + +#define AES_DMA_MSI_MASK_CLEAR BIT(0) + +#define AES_128_BIT_KEY 0x00000000 +#define AES_256_BIT_KEY BIT(0) + +#define AES_DEACTIVATE_PERF_CNTR 0x00000000 +#define AES_ACTIVATE_PERF_CNTR BIT(0) + +#define AES_MAX_TAG_SIZE_U32 4 + +#define OCS_LL_DMA_FLAG_TERMINATE BIT(31) + +/* + * There is an inconsistency in the documentation. This is documented as a + * 11-bit value, but it is actually 10-bits. + */ +#define AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK 0x3FF + +/* + * During CCM decrypt, the OCS block needs to finish processing the ciphertext + * before the tag is written. For 128-bit mode this required delay is 28 OCS + * clock cycles. For 256-bit mode it is 36 OCS clock cycles. + */ +#define CCM_DECRYPT_DELAY_TAG_CLK_COUNT 36UL + +/* + * During CCM decrypt there must be a delay of at least 42 OCS clock cycles + * between setting the TRIGGER bit in AES_ACTIVE and setting the LAST_CCM_GCM + * bit in the same register (as stated in the OCS databook) + */ +#define CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT 42UL + +/* See RFC3610 section 2.2 */ +#define L_PRIME_MIN (1) +#define L_PRIME_MAX (7) +/* + * CCM IV format from RFC 3610 section 2.3 + * + * Octet Number Contents + * ------------ --------- + * 0 Flags + * 1 ... 15-L Nonce N + * 16-L ... 15 Counter i + * + * Flags = L' = L - 1 + */ +#define L_PRIME_IDX 0 +#define COUNTER_START(lprime) (16 - ((lprime) + 1)) +#define COUNTER_LEN(lprime) ((lprime) + 1) + +enum aes_counter_mode { + AES_CTR_M_NO_INC = 0, + AES_CTR_M_32_INC = 1, + AES_CTR_M_64_INC = 2, + AES_CTR_M_128_INC = 3, +}; + +/** + * struct ocs_dma_linked_list - OCS DMA linked list entry. + * @src_addr: Source address of the data. + * @src_len: Length of data to be fetched. + * @next: Next dma_list to fetch. + * @ll_flags: Flags (Freeze @ terminate) for the DMA engine. + */ +struct ocs_dma_linked_list { + u32 src_addr; + u32 src_len; + u32 next; + u32 ll_flags; +} __packed; + +/* + * Set endianness of inputs and outputs + * AES_BYTE_ORDER_CFG + * default 0x00000000 + * bit [10] - KEY_HI_LO_SWAP + * bit [9] - KEY_HI_SWAP_DWORDS_IN_OCTWORD + * bit [8] - KEY_HI_SWAP_BYTES_IN_DWORD + * bit [7] - KEY_LO_SWAP_DWORDS_IN_OCTWORD + * bit [6] - KEY_LO_SWAP_BYTES_IN_DWORD + * bit [5] - IV_SWAP_DWORDS_IN_OCTWORD + * bit [4] - IV_SWAP_BYTES_IN_DWORD + * bit [3] - DOUT_SWAP_DWORDS_IN_OCTWORD + * bit [2] - DOUT_SWAP_BYTES_IN_DWORD + * bit [1] - DOUT_SWAP_DWORDS_IN_OCTWORD + * bit [0] - DOUT_SWAP_BYTES_IN_DWORD + */ +static inline void aes_a_set_endianness(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(0x7FF, aes_dev->base_reg + AES_BYTE_ORDER_CFG_OFFSET); +} + +/* Trigger AES process start. */ +static inline void aes_a_op_trigger(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_ACTIVE_TRIGGER, aes_dev->base_reg + AES_ACTIVE_OFFSET); +} + +/* Indicate last bulk of data. */ +static inline void aes_a_op_termination(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_ACTIVE_TERMINATION, + aes_dev->base_reg + AES_ACTIVE_OFFSET); +} + +/* + * Set LAST_CCM_GCM in AES_ACTIVE register and clear all other bits. + * + * Called when DMA is programmed to fetch the last batch of data. + * - For AES-CCM it is called for the last batch of Payload data and Ciphertext + * data. + * - For AES-GCM, it is called for the last batch of Plaintext data and + * Ciphertext data. + */ +static inline void aes_a_set_last_gcx(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_ACTIVE_LAST_CCM_GCM, + aes_dev->base_reg + AES_ACTIVE_OFFSET); +} + +/* Wait for LAST_CCM_GCM bit to be unset. */ +static inline void aes_a_wait_last_gcx(const struct ocs_aes_dev *aes_dev) +{ + u32 aes_active_reg; + + do { + aes_active_reg = ioread32(aes_dev->base_reg + + AES_ACTIVE_OFFSET); + } while (aes_active_reg & AES_ACTIVE_LAST_CCM_GCM); +} + +/* Wait for 10 bits of input occupancy. */ +static void aes_a_dma_wait_input_buffer_occupancy(const struct ocs_aes_dev *aes_dev) +{ + u32 reg; + + do { + reg = ioread32(aes_dev->base_reg + AES_A_DMA_STATUS_OFFSET); + } while (reg & AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK); +} + + /* + * Set LAST_CCM_GCM and LAST_ADATA bits in AES_ACTIVE register (and clear all + * other bits). + * + * Called when DMA is programmed to fetch the last batch of Associated Data + * (CCM case) or Additional Authenticated Data (GCM case). + */ +static inline void aes_a_set_last_gcx_and_adata(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_ACTIVE_LAST_ADATA | AES_ACTIVE_LAST_CCM_GCM, + aes_dev->base_reg + AES_ACTIVE_OFFSET); +} + +/* Set DMA src and dst transfer size to 0 */ +static inline void aes_a_dma_set_xfer_size_zero(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET); + iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET); +} + +/* Activate DMA for zero-byte transfer case. */ +static inline void aes_a_dma_active(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_A_DMA_DMA_MODE_ACTIVE, + aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); +} + +/* Activate DMA and enable src linked list */ +static inline void aes_a_dma_active_src_ll_en(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_A_DMA_DMA_MODE_ACTIVE | + AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN, + aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); +} + +/* Activate DMA and enable dst linked list */ +static inline void aes_a_dma_active_dst_ll_en(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_A_DMA_DMA_MODE_ACTIVE | + AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN, + aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); +} + +/* Activate DMA and enable src and dst linked lists */ +static inline void aes_a_dma_active_src_dst_ll_en(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_A_DMA_DMA_MODE_ACTIVE | + AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN | + AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN, + aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); +} + +/* Reset PERF_CNTR to 0 and activate it */ +static inline void aes_a_dma_reset_and_activate_perf_cntr(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(0x00000000, aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET); + iowrite32(AES_ACTIVATE_PERF_CNTR, + aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET); +} + +/* Wait until PERF_CNTR is > delay, then deactivate it */ +static inline void aes_a_dma_wait_and_deactivate_perf_cntr(const struct ocs_aes_dev *aes_dev, + int delay) +{ + while (ioread32(aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET) < delay) + ; + iowrite32(AES_DEACTIVATE_PERF_CNTR, + aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET); +} + +/* Disable AES and DMA IRQ. */ +static void aes_irq_disable(struct ocs_aes_dev *aes_dev) +{ + u32 isr_val = 0; + + /* Disable interrupts */ + iowrite32(AES_DISABLE_INT, + aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); + iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET); + + /* Clear any pending interrupt */ + isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); + if (isr_val) + iowrite32(isr_val, + aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); + + isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET); + if (isr_val) + iowrite32(isr_val, + aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET); + + isr_val = ioread32(aes_dev->base_reg + AES_ISR_OFFSET); + if (isr_val) + iowrite32(isr_val, aes_dev->base_reg + AES_ISR_OFFSET); +} + +/* Enable AES or DMA IRQ. IRQ is disabled once fired. */ +static void aes_irq_enable(struct ocs_aes_dev *aes_dev, u8 irq) +{ + if (irq == AES_COMPLETE_INT) { + /* Ensure DMA error interrupts are enabled */ + iowrite32(AES_DMA_CPD_ERR_INT | + AES_DMA_OUTBUF_RD_ERR_INT | + AES_DMA_OUTBUF_WR_ERR_INT | + AES_DMA_INBUF_RD_ERR_INT | + AES_DMA_INBUF_WR_ERR_INT | + AES_DMA_BAD_COMP_INT | + AES_DMA_SAI_INT, + aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); + /* + * AES_IER + * default 0x00000000 + * bits [31:3] - reserved + * bit [2] - EN_SKS_ERR + * bit [1] - EN_AES_COMPLETE + * bit [0] - reserved + */ + iowrite32(AES_COMPLETE_INT, aes_dev->base_reg + AES_IER_OFFSET); + return; + } + if (irq == AES_DMA_SRC_DONE_INT) { + /* Ensure AES interrupts are disabled */ + iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET); + /* + * DMA_MSI_IER + * default 0x00000000 + * bits [31:9] - reserved + * bit [8] - CPD_ERR_INT_EN + * bit [7] - OUTBUF_RD_ERR_INT_EN + * bit [6] - OUTBUF_WR_ERR_INT_EN + * bit [5] - INBUF_RD_ERR_INT_EN + * bit [4] - INBUF_WR_ERR_INT_EN + * bit [3] - BAD_COMP_INT_EN + * bit [2] - SAI_INT_EN + * bit [1] - DST_DONE_INT_EN + * bit [0] - SRC_DONE_INT_EN + */ + iowrite32(AES_DMA_CPD_ERR_INT | + AES_DMA_OUTBUF_RD_ERR_INT | + AES_DMA_OUTBUF_WR_ERR_INT | + AES_DMA_INBUF_RD_ERR_INT | + AES_DMA_INBUF_WR_ERR_INT | + AES_DMA_BAD_COMP_INT | + AES_DMA_SAI_INT | + AES_DMA_SRC_DONE_INT, + aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); + } +} + +/* Enable and wait for IRQ (either from OCS AES engine or DMA) */ +static int ocs_aes_irq_enable_and_wait(struct ocs_aes_dev *aes_dev, u8 irq) +{ + int rc; + + reinit_completion(&aes_dev->irq_completion); + aes_irq_enable(aes_dev, irq); + rc = wait_for_completion_interruptible(&aes_dev->irq_completion); + if (rc) + return rc; + + return aes_dev->dma_err_mask ? -EIO : 0; +} + +/* Configure DMA to OCS, linked list mode */ +static inline void dma_to_ocs_aes_ll(struct ocs_aes_dev *aes_dev, + dma_addr_t dma_list) +{ + iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET); + iowrite32(dma_list, + aes_dev->base_reg + AES_A_DMA_NEXT_SRC_DESCR_OFFSET); +} + +/* Configure DMA from OCS, linked list mode */ +static inline void dma_from_ocs_aes_ll(struct ocs_aes_dev *aes_dev, + dma_addr_t dma_list) +{ + iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET); + iowrite32(dma_list, + aes_dev->base_reg + AES_A_DMA_NEXT_DST_DESCR_OFFSET); +} + +irqreturn_t ocs_aes_irq_handler(int irq, void *dev_id) +{ + struct ocs_aes_dev *aes_dev = dev_id; + u32 aes_dma_isr; + + /* Read DMA ISR status. */ + aes_dma_isr = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); + + /* Disable and clear interrupts. */ + aes_irq_disable(aes_dev); + + /* Save DMA error status. */ + aes_dev->dma_err_mask = aes_dma_isr & + (AES_DMA_CPD_ERR_INT | + AES_DMA_OUTBUF_RD_ERR_INT | + AES_DMA_OUTBUF_WR_ERR_INT | + AES_DMA_INBUF_RD_ERR_INT | + AES_DMA_INBUF_WR_ERR_INT | + AES_DMA_BAD_COMP_INT | + AES_DMA_SAI_INT); + + /* Signal IRQ completion. */ + complete(&aes_dev->irq_completion); + + return IRQ_HANDLED; +} + +/** + * ocs_aes_set_key() - Write key into OCS AES hardware. + * @aes_dev: The OCS AES device to write the key to. + * @key_size: The size of the key (in bytes). + * @key: The key to write. + * @cipher: The cipher the key is for. + * + * For AES @key_size must be either 16 or 32. For SM4 @key_size must be 16. + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_aes_set_key(struct ocs_aes_dev *aes_dev, u32 key_size, const u8 *key, + enum ocs_cipher cipher) +{ + const u32 *key_u32; + u32 val; + int i; + + /* OCS AES supports 128-bit and 256-bit keys only. */ + if (cipher == OCS_AES && !(key_size == 32 || key_size == 16)) { + dev_err(aes_dev->dev, + "%d-bit keys not supported by AES cipher\n", + key_size * 8); + return -EINVAL; + } + /* OCS SM4 supports 128-bit keys only. */ + if (cipher == OCS_SM4 && key_size != 16) { + dev_err(aes_dev->dev, + "%d-bit keys not supported for SM4 cipher\n", + key_size * 8); + return -EINVAL; + } + + if (!key) + return -EINVAL; + + key_u32 = (const u32 *)key; + + /* Write key to AES_KEY[0-7] registers */ + for (i = 0; i < (key_size / sizeof(u32)); i++) { + iowrite32(key_u32[i], + aes_dev->base_reg + AES_KEY_0_OFFSET + + (i * sizeof(u32))); + } + /* + * Write key size + * bits [31:1] - reserved + * bit [0] - AES_KEY_SIZE + * 0 - 128 bit key + * 1 - 256 bit key + */ + val = (key_size == 16) ? AES_128_BIT_KEY : AES_256_BIT_KEY; + iowrite32(val, aes_dev->base_reg + AES_KEY_SIZE_OFFSET); + + return 0; +} + +/* Write AES_COMMAND */ +static inline void set_ocs_aes_command(struct ocs_aes_dev *aes_dev, + enum ocs_cipher cipher, + enum ocs_mode mode, + enum ocs_instruction instruction) +{ + u32 val; + + /* AES_COMMAND + * default 0x000000CC + * bit [14] - CIPHER_SELECT + * 0 - AES + * 1 - SM4 + * bits [11:8] - OCS_AES_MODE + * 0000 - ECB + * 0001 - CBC + * 0010 - CTR + * 0110 - CCM + * 0111 - GCM + * 1001 - CTS + * bits [7:6] - AES_INSTRUCTION + * 00 - ENCRYPT + * 01 - DECRYPT + * 10 - EXPAND + * 11 - BYPASS + * bits [3:2] - CTR_M_BITS + * 00 - No increment + * 01 - Least significant 32 bits are incremented + * 10 - Least significant 64 bits are incremented + * 11 - Full 128 bits are incremented + */ + val = (cipher << 14) | (mode << 8) | (instruction << 6) | + (AES_CTR_M_128_INC << 2); + iowrite32(val, aes_dev->base_reg + AES_COMMAND_OFFSET); +} + +static void ocs_aes_init(struct ocs_aes_dev *aes_dev, + enum ocs_mode mode, + enum ocs_cipher cipher, + enum ocs_instruction instruction) +{ + /* Ensure interrupts are disabled and pending interrupts cleared. */ + aes_irq_disable(aes_dev); + + /* Set endianness recommended by data-sheet. */ + aes_a_set_endianness(aes_dev); + + /* Set AES_COMMAND register. */ + set_ocs_aes_command(aes_dev, cipher, mode, instruction); +} + +/* + * Write the byte length of the last AES/SM4 block of Payload data (without + * zero padding and without the length of the MAC) in register AES_PLEN. + */ +static inline void ocs_aes_write_last_data_blk_len(struct ocs_aes_dev *aes_dev, + u32 size) +{ + u32 val; + + if (size == 0) { + val = 0; + goto exit; + } + + val = size % AES_BLOCK_SIZE; + if (val == 0) + val = AES_BLOCK_SIZE; + +exit: + iowrite32(val, aes_dev->base_reg + AES_PLEN_OFFSET); +} + +/* + * Validate inputs according to mode. + * If OK return 0; else return -EINVAL. + */ +static int ocs_aes_validate_inputs(dma_addr_t src_dma_list, u32 src_size, + const u8 *iv, u32 iv_size, + dma_addr_t aad_dma_list, u32 aad_size, + const u8 *tag, u32 tag_size, + enum ocs_cipher cipher, enum ocs_mode mode, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list) +{ + /* Ensure cipher, mode and instruction are valid. */ + if (!(cipher == OCS_AES || cipher == OCS_SM4)) + return -EINVAL; + + if (mode != OCS_MODE_ECB && mode != OCS_MODE_CBC && + mode != OCS_MODE_CTR && mode != OCS_MODE_CCM && + mode != OCS_MODE_GCM && mode != OCS_MODE_CTS) + return -EINVAL; + + if (instruction != OCS_ENCRYPT && instruction != OCS_DECRYPT && + instruction != OCS_EXPAND && instruction != OCS_BYPASS) + return -EINVAL; + + /* + * When instruction is OCS_BYPASS, OCS simply copies data from source + * to destination using DMA. + * + * AES mode is irrelevant, but both source and destination DMA + * linked-list must be defined. + */ + if (instruction == OCS_BYPASS) { + if (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + return 0; + } + + /* + * For performance reasons switch based on mode to limit unnecessary + * conditionals for each mode + */ + switch (mode) { + case OCS_MODE_ECB: + /* Ensure input length is multiple of block size */ + if (src_size % AES_BLOCK_SIZE != 0) + return -EINVAL; + + /* Ensure source and destination linked lists are created */ + if (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + return 0; + + case OCS_MODE_CBC: + /* Ensure input length is multiple of block size */ + if (src_size % AES_BLOCK_SIZE != 0) + return -EINVAL; + + /* Ensure source and destination linked lists are created */ + if (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* Ensure IV is present and block size in length */ + if (!iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + return 0; + + case OCS_MODE_CTR: + /* Ensure input length of 1 byte or greater */ + if (src_size == 0) + return -EINVAL; + + /* Ensure source and destination linked lists are created */ + if (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* Ensure IV is present and block size in length */ + if (!iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + return 0; + + case OCS_MODE_CTS: + /* Ensure input length >= block size */ + if (src_size < AES_BLOCK_SIZE) + return -EINVAL; + + /* Ensure source and destination linked lists are created */ + if (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* Ensure IV is present and block size in length */ + if (!iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + return 0; + + case OCS_MODE_GCM: + /* Ensure IV is present and GCM_AES_IV_SIZE in length */ + if (!iv || iv_size != GCM_AES_IV_SIZE) + return -EINVAL; + + /* + * If input data present ensure source and destination linked + * lists are created + */ + if (src_size && (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR)) + return -EINVAL; + + /* If aad present ensure aad linked list is created */ + if (aad_size && aad_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* Ensure tag destination is set */ + if (!tag) + return -EINVAL; + + /* Just ensure that tag_size doesn't cause overflows. */ + if (tag_size > (AES_MAX_TAG_SIZE_U32 * sizeof(u32))) + return -EINVAL; + + return 0; + + case OCS_MODE_CCM: + /* Ensure IV is present and block size in length */ + if (!iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + /* 2 <= L <= 8, so 1 <= L' <= 7 */ + if (iv[L_PRIME_IDX] < L_PRIME_MIN || + iv[L_PRIME_IDX] > L_PRIME_MAX) + return -EINVAL; + + /* If aad present ensure aad linked list is created */ + if (aad_size && aad_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* Just ensure that tag_size doesn't cause overflows. */ + if (tag_size > (AES_MAX_TAG_SIZE_U32 * sizeof(u32))) + return -EINVAL; + + if (instruction == OCS_DECRYPT) { + /* + * If input data present ensure source and destination + * linked lists are created + */ + if (src_size && (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR)) + return -EINVAL; + + /* Ensure input tag is present */ + if (!tag) + return -EINVAL; + + return 0; + } + + /* Instruction == OCS_ENCRYPT */ + + /* + * Destination linked list always required (for tag even if no + * input data) + */ + if (dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* If input data present ensure src linked list is created */ + if (src_size && src_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + return 0; + + default: + return -EINVAL; + } +} + +/** + * ocs_aes_op() - Perform AES/SM4 operation. + * @aes_dev: The OCS AES device to use. + * @mode: The mode to use (ECB, CBC, CTR, or CTS). + * @cipher: The cipher to use (AES or SM4). + * @instruction: The instruction to perform (encrypt or decrypt). + * @dst_dma_list: The OCS DMA list mapping output memory. + * @src_dma_list: The OCS DMA list mapping input payload data. + * @src_size: The amount of data mapped by @src_dma_list. + * @iv: The IV vector. + * @iv_size: The size (in bytes) of @iv. + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_aes_op(struct ocs_aes_dev *aes_dev, + enum ocs_mode mode, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + u8 *iv, + u32 iv_size) +{ + u32 *iv32; + int rc; + + rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, iv_size, 0, 0, + NULL, 0, cipher, mode, instruction, + dst_dma_list); + if (rc) + return rc; + /* + * ocs_aes_validate_inputs() is a generic check, now ensure mode is not + * GCM or CCM. + */ + if (mode == OCS_MODE_GCM || mode == OCS_MODE_CCM) + return -EINVAL; + + /* Cast IV to u32 array. */ + iv32 = (u32 *)iv; + + ocs_aes_init(aes_dev, mode, cipher, instruction); + + if (mode == OCS_MODE_CTS) { + /* Write the byte length of the last data block to engine. */ + ocs_aes_write_last_data_blk_len(aes_dev, src_size); + } + + /* ECB is the only mode that doesn't use IV. */ + if (mode != OCS_MODE_ECB) { + iowrite32(iv32[0], aes_dev->base_reg + AES_IV_0_OFFSET); + iowrite32(iv32[1], aes_dev->base_reg + AES_IV_1_OFFSET); + iowrite32(iv32[2], aes_dev->base_reg + AES_IV_2_OFFSET); + iowrite32(iv32[3], aes_dev->base_reg + AES_IV_3_OFFSET); + } + + /* Set AES_ACTIVE.TRIGGER to start the operation. */ + aes_a_op_trigger(aes_dev); + + /* Configure and activate input / output DMA. */ + dma_to_ocs_aes_ll(aes_dev, src_dma_list); + dma_from_ocs_aes_ll(aes_dev, dst_dma_list); + aes_a_dma_active_src_dst_ll_en(aes_dev); + + if (mode == OCS_MODE_CTS) { + /* + * For CTS mode, instruct engine to activate ciphertext + * stealing if last block of data is incomplete. + */ + aes_a_set_last_gcx(aes_dev); + } else { + /* For all other modes, just write the 'termination' bit. */ + aes_a_op_termination(aes_dev); + } + + /* Wait for engine to complete processing. */ + rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); + if (rc) + return rc; + + if (mode == OCS_MODE_CTR) { + /* Read back IV for streaming mode */ + iv32[0] = ioread32(aes_dev->base_reg + AES_IV_0_OFFSET); + iv32[1] = ioread32(aes_dev->base_reg + AES_IV_1_OFFSET); + iv32[2] = ioread32(aes_dev->base_reg + AES_IV_2_OFFSET); + iv32[3] = ioread32(aes_dev->base_reg + AES_IV_3_OFFSET); + } + + return 0; +} + +/* Compute and write J0 to engine registers. */ +static void ocs_aes_gcm_write_j0(const struct ocs_aes_dev *aes_dev, + const u8 *iv) +{ + const u32 *j0 = (u32 *)iv; + + /* + * IV must be 12 bytes; Other sizes not supported as Linux crypto API + * does only expects/allows 12 byte IV for GCM + */ + iowrite32(0x00000001, aes_dev->base_reg + AES_IV_0_OFFSET); + iowrite32(__swab32(j0[2]), aes_dev->base_reg + AES_IV_1_OFFSET); + iowrite32(__swab32(j0[1]), aes_dev->base_reg + AES_IV_2_OFFSET); + iowrite32(__swab32(j0[0]), aes_dev->base_reg + AES_IV_3_OFFSET); +} + +/* Read GCM tag from engine registers. */ +static inline void ocs_aes_gcm_read_tag(struct ocs_aes_dev *aes_dev, + u8 *tag, u32 tag_size) +{ + u32 tag_u32[AES_MAX_TAG_SIZE_U32]; + + /* + * The Authentication Tag T is stored in Little Endian order in the + * registers with the most significant bytes stored from AES_T_MAC[3] + * downward. + */ + tag_u32[0] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_3_OFFSET)); + tag_u32[1] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_2_OFFSET)); + tag_u32[2] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_1_OFFSET)); + tag_u32[3] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_0_OFFSET)); + + memcpy(tag, tag_u32, tag_size); +} + +/** + * ocs_aes_gcm_op() - Perform GCM operation. + * @aes_dev: The OCS AES device to use. + * @cipher: The Cipher to use (AES or SM4). + * @instruction: The instruction to perform (encrypt or decrypt). + * @dst_dma_list: The OCS DMA list mapping output memory. + * @src_dma_list: The OCS DMA list mapping input payload data. + * @src_size: The amount of data mapped by @src_dma_list. + * @iv: The input IV vector. + * @aad_dma_list: The OCS DMA list mapping input AAD data. + * @aad_size: The amount of data mapped by @aad_dma_list. + * @out_tag: Where to store computed tag. + * @tag_size: The size (in bytes) of @out_tag. + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_aes_gcm_op(struct ocs_aes_dev *aes_dev, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + const u8 *iv, + dma_addr_t aad_dma_list, + u32 aad_size, + u8 *out_tag, + u32 tag_size) +{ + u64 bit_len; + u32 val; + int rc; + + rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, + GCM_AES_IV_SIZE, aad_dma_list, + aad_size, out_tag, tag_size, cipher, + OCS_MODE_GCM, instruction, + dst_dma_list); + if (rc) + return rc; + + ocs_aes_init(aes_dev, OCS_MODE_GCM, cipher, instruction); + + /* Compute and write J0 to OCS HW. */ + ocs_aes_gcm_write_j0(aes_dev, iv); + + /* Write out_tag byte length */ + iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET); + + /* Write the byte length of the last plaintext / ciphertext block. */ + ocs_aes_write_last_data_blk_len(aes_dev, src_size); + + /* Write ciphertext bit length */ + bit_len = src_size * 8; + val = bit_len & 0xFFFFFFFF; + iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_0_OFFSET); + val = bit_len >> 32; + iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_1_OFFSET); + + /* Write aad bit length */ + bit_len = aad_size * 8; + val = bit_len & 0xFFFFFFFF; + iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_2_OFFSET); + val = bit_len >> 32; + iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_3_OFFSET); + + /* Set AES_ACTIVE.TRIGGER to start the operation. */ + aes_a_op_trigger(aes_dev); + + /* Process AAD. */ + if (aad_size) { + /* If aad present, configure DMA to feed it to the engine. */ + dma_to_ocs_aes_ll(aes_dev, aad_dma_list); + aes_a_dma_active_src_ll_en(aes_dev); + + /* Instructs engine to pad last block of aad, if needed. */ + aes_a_set_last_gcx_and_adata(aes_dev); + + /* Wait for DMA transfer to complete. */ + rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT); + if (rc) + return rc; + } else { + aes_a_set_last_gcx_and_adata(aes_dev); + } + + /* Wait until adata (if present) has been processed. */ + aes_a_wait_last_gcx(aes_dev); + aes_a_dma_wait_input_buffer_occupancy(aes_dev); + + /* Now process payload. */ + if (src_size) { + /* Configure and activate DMA for both input and output data. */ + dma_to_ocs_aes_ll(aes_dev, src_dma_list); + dma_from_ocs_aes_ll(aes_dev, dst_dma_list); + aes_a_dma_active_src_dst_ll_en(aes_dev); + } else { + aes_a_dma_set_xfer_size_zero(aes_dev); + aes_a_dma_active(aes_dev); + } + + /* Instruct AES/SMA4 engine payload processing is over. */ + aes_a_set_last_gcx(aes_dev); + + /* Wait for OCS AES engine to complete processing. */ + rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); + if (rc) + return rc; + + ocs_aes_gcm_read_tag(aes_dev, out_tag, tag_size); + + return 0; +} + +/* Write encrypted tag to AES/SM4 engine. */ +static void ocs_aes_ccm_write_encrypted_tag(struct ocs_aes_dev *aes_dev, + const u8 *in_tag, u32 tag_size) +{ + int i; + + /* Ensure DMA input buffer is empty */ + aes_a_dma_wait_input_buffer_occupancy(aes_dev); + + /* + * During CCM decrypt, the OCS block needs to finish processing the + * ciphertext before the tag is written. So delay needed after DMA has + * completed writing the ciphertext + */ + aes_a_dma_reset_and_activate_perf_cntr(aes_dev); + aes_a_dma_wait_and_deactivate_perf_cntr(aes_dev, + CCM_DECRYPT_DELAY_TAG_CLK_COUNT); + + /* Write encrypted tag to AES/SM4 engine. */ + for (i = 0; i < tag_size; i++) { + iowrite8(in_tag[i], aes_dev->base_reg + + AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET); + } +} + +/* + * Write B0 CCM block to OCS AES HW. + * + * Note: B0 format is documented in NIST Special Publication 800-38C + * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf + * (see Section A.2.1) + */ +static int ocs_aes_ccm_write_b0(const struct ocs_aes_dev *aes_dev, + const u8 *iv, u32 adata_size, u32 tag_size, + u32 cryptlen) +{ + u8 b0[16]; /* CCM B0 block is 16 bytes long. */ + int i, q; + + /* Initialize B0 to 0. */ + memset(b0, 0, sizeof(b0)); + + /* + * B0[0] is the 'Flags Octet' and has the following structure: + * bit 7: Reserved + * bit 6: Adata flag + * bit 5-3: t value encoded as (t-2)/2 + * bit 2-0: q value encoded as q - 1 + */ + /* If there is AAD data, set the Adata flag. */ + if (adata_size) + b0[0] |= BIT(6); + /* + * t denotes the octet length of T. + * t can only be an element of { 4, 6, 8, 10, 12, 14, 16} and is + * encoded as (t - 2) / 2 + */ + b0[0] |= (((tag_size - 2) / 2) & 0x7) << 3; + /* + * q is the octet length of Q. + * q can only be an element of {2, 3, 4, 5, 6, 7, 8} and is encoded as + * q - 1 == iv[0] + */ + b0[0] |= iv[0] & 0x7; + /* + * Copy the Nonce N from IV to B0; N is located in iv[1]..iv[15 - q] + * and must be copied to b0[1]..b0[15-q]. + * q == iv[0] + 1 + */ + q = iv[0] + 1; + for (i = 1; i <= 15 - q; i++) + b0[i] = iv[i]; + /* + * The rest of B0 must contain Q, i.e., the message length. + * Q is encoded in q octets, in big-endian order, so to write it, we + * start from the end of B0 and we move backward. + */ + i = sizeof(b0) - 1; + while (q) { + b0[i] = cryptlen & 0xff; + cryptlen >>= 8; + i--; + q--; + } + /* + * If cryptlen is not zero at this point, it means that its original + * value was too big. + */ + if (cryptlen) + return -EOVERFLOW; + /* Now write B0 to OCS AES input buffer. */ + for (i = 0; i < sizeof(b0); i++) + iowrite8(b0[i], aes_dev->base_reg + + AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET); + return 0; +} + +/* + * Write adata length to OCS AES HW. + * + * Note: adata len encoding is documented in NIST Special Publication 800-38C + * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf + * (see Section A.2.2) + */ +static void ocs_aes_ccm_write_adata_len(const struct ocs_aes_dev *aes_dev, + u64 adata_len) +{ + u8 enc_a[10]; /* Maximum encoded size: 10 octets. */ + int i, len; + + /* + * adata_len ('a') is encoded as follows: + * If 0 < a < 2^16 - 2^8 ==> 'a' encoded as [a]16, i.e., two octets + * (big endian). + * If 2^16 - 2^8 ≤ a < 2^32 ==> 'a' encoded as 0xff || 0xfe || [a]32, + * i.e., six octets (big endian). + * If 2^32 ≤ a < 2^64 ==> 'a' encoded as 0xff || 0xff || [a]64, + * i.e., ten octets (big endian). + */ + if (adata_len < 65280) { + len = 2; + *(__be16 *)enc_a = cpu_to_be16(adata_len); + } else if (adata_len <= 0xFFFFFFFF) { + len = 6; + *(__be16 *)enc_a = cpu_to_be16(0xfffe); + *(__be32 *)&enc_a[2] = cpu_to_be32(adata_len); + } else { /* adata_len >= 2^32 */ + len = 10; + *(__be16 *)enc_a = cpu_to_be16(0xffff); + *(__be64 *)&enc_a[2] = cpu_to_be64(adata_len); + } + for (i = 0; i < len; i++) + iowrite8(enc_a[i], + aes_dev->base_reg + + AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET); +} + +static int ocs_aes_ccm_do_adata(struct ocs_aes_dev *aes_dev, + dma_addr_t adata_dma_list, u32 adata_size) +{ + int rc; + + if (!adata_size) { + /* Since no aad the LAST_GCX bit can be set now */ + aes_a_set_last_gcx_and_adata(aes_dev); + goto exit; + } + + /* Adata case. */ + + /* + * Form the encoding of the Associated data length and write it + * to the AES/SM4 input buffer. + */ + ocs_aes_ccm_write_adata_len(aes_dev, adata_size); + + /* Configure the AES/SM4 DMA to fetch the Associated Data */ + dma_to_ocs_aes_ll(aes_dev, adata_dma_list); + + /* Activate DMA to fetch Associated data. */ + aes_a_dma_active_src_ll_en(aes_dev); + + /* Set LAST_GCX and LAST_ADATA in AES ACTIVE register. */ + aes_a_set_last_gcx_and_adata(aes_dev); + + /* Wait for DMA transfer to complete. */ + rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT); + if (rc) + return rc; + +exit: + /* Wait until adata (if present) has been processed. */ + aes_a_wait_last_gcx(aes_dev); + aes_a_dma_wait_input_buffer_occupancy(aes_dev); + + return 0; +} + +static int ocs_aes_ccm_encrypt_do_payload(struct ocs_aes_dev *aes_dev, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size) +{ + if (src_size) { + /* + * Configure and activate DMA for both input and output + * data. + */ + dma_to_ocs_aes_ll(aes_dev, src_dma_list); + dma_from_ocs_aes_ll(aes_dev, dst_dma_list); + aes_a_dma_active_src_dst_ll_en(aes_dev); + } else { + /* Configure and activate DMA for output data only. */ + dma_from_ocs_aes_ll(aes_dev, dst_dma_list); + aes_a_dma_active_dst_ll_en(aes_dev); + } + + /* + * Set the LAST GCX bit in AES_ACTIVE Register to instruct + * AES/SM4 engine to pad the last block of data. + */ + aes_a_set_last_gcx(aes_dev); + + /* We are done, wait for IRQ and return. */ + return ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); +} + +static int ocs_aes_ccm_decrypt_do_payload(struct ocs_aes_dev *aes_dev, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size) +{ + if (!src_size) { + /* Let engine process 0-length input. */ + aes_a_dma_set_xfer_size_zero(aes_dev); + aes_a_dma_active(aes_dev); + aes_a_set_last_gcx(aes_dev); + + return 0; + } + + /* + * Configure and activate DMA for both input and output + * data. + */ + dma_to_ocs_aes_ll(aes_dev, src_dma_list); + dma_from_ocs_aes_ll(aes_dev, dst_dma_list); + aes_a_dma_active_src_dst_ll_en(aes_dev); + /* + * Set the LAST GCX bit in AES_ACTIVE Register; this allows the + * AES/SM4 engine to differentiate between encrypted data and + * encrypted MAC. + */ + aes_a_set_last_gcx(aes_dev); + /* + * Enable DMA DONE interrupt; once DMA transfer is over, + * interrupt handler will process the MAC/tag. + */ + return ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT); +} + +/* + * Compare Tag to Yr. + * + * Only used at the end of CCM decrypt. If tag == yr, message authentication + * has succeeded. + */ +static inline int ccm_compare_tag_to_yr(struct ocs_aes_dev *aes_dev, + u8 tag_size_bytes) +{ + u32 tag[AES_MAX_TAG_SIZE_U32]; + u32 yr[AES_MAX_TAG_SIZE_U32]; + u8 i; + + /* Read Tag and Yr from AES registers. */ + for (i = 0; i < AES_MAX_TAG_SIZE_U32; i++) { + tag[i] = ioread32(aes_dev->base_reg + + AES_T_MAC_0_OFFSET + (i * sizeof(u32))); + yr[i] = ioread32(aes_dev->base_reg + + AES_MULTIPURPOSE2_0_OFFSET + + (i * sizeof(u32))); + } + + return memcmp(tag, yr, tag_size_bytes) ? -EBADMSG : 0; +} + +/** + * ocs_aes_ccm_op() - Perform CCM operation. + * @aes_dev: The OCS AES device to use. + * @cipher: The Cipher to use (AES or SM4). + * @instruction: The instruction to perform (encrypt or decrypt). + * @dst_dma_list: The OCS DMA list mapping output memory. + * @src_dma_list: The OCS DMA list mapping input payload data. + * @src_size: The amount of data mapped by @src_dma_list. + * @iv: The input IV vector. + * @adata_dma_list: The OCS DMA list mapping input A-data. + * @adata_size: The amount of data mapped by @adata_dma_list. + * @in_tag: Input tag. + * @tag_size: The size (in bytes) of @in_tag. + * + * Note: for encrypt the tag is appended to the ciphertext (in the memory + * mapped by @dst_dma_list). + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_aes_ccm_op(struct ocs_aes_dev *aes_dev, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + u8 *iv, + dma_addr_t adata_dma_list, + u32 adata_size, + u8 *in_tag, + u32 tag_size) +{ + u32 *iv_32; + u8 lprime; + int rc; + + rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, + AES_BLOCK_SIZE, adata_dma_list, adata_size, + in_tag, tag_size, cipher, OCS_MODE_CCM, + instruction, dst_dma_list); + if (rc) + return rc; + + ocs_aes_init(aes_dev, OCS_MODE_CCM, cipher, instruction); + + /* + * Note: rfc 3610 and NIST 800-38C require counter of zero to encrypt + * auth tag so ensure this is the case + */ + lprime = iv[L_PRIME_IDX]; + memset(&iv[COUNTER_START(lprime)], 0, COUNTER_LEN(lprime)); + + /* + * Nonce is already converted to ctr0 before being passed into this + * function as iv. + */ + iv_32 = (u32 *)iv; + iowrite32(__swab32(iv_32[0]), + aes_dev->base_reg + AES_MULTIPURPOSE1_3_OFFSET); + iowrite32(__swab32(iv_32[1]), + aes_dev->base_reg + AES_MULTIPURPOSE1_2_OFFSET); + iowrite32(__swab32(iv_32[2]), + aes_dev->base_reg + AES_MULTIPURPOSE1_1_OFFSET); + iowrite32(__swab32(iv_32[3]), + aes_dev->base_reg + AES_MULTIPURPOSE1_0_OFFSET); + + /* Write MAC/tag length in register AES_TLEN */ + iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET); + /* + * Write the byte length of the last AES/SM4 block of Payload data + * (without zero padding and without the length of the MAC) in register + * AES_PLEN. + */ + ocs_aes_write_last_data_blk_len(aes_dev, src_size); + + /* Set AES_ACTIVE.TRIGGER to start the operation. */ + aes_a_op_trigger(aes_dev); + + aes_a_dma_reset_and_activate_perf_cntr(aes_dev); + + /* Form block B0 and write it to the AES/SM4 input buffer. */ + rc = ocs_aes_ccm_write_b0(aes_dev, iv, adata_size, tag_size, src_size); + if (rc) + return rc; + /* + * Ensure there has been at least CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT + * clock cycles since TRIGGER bit was set + */ + aes_a_dma_wait_and_deactivate_perf_cntr(aes_dev, + CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT); + + /* Process Adata. */ + ocs_aes_ccm_do_adata(aes_dev, adata_dma_list, adata_size); + + /* For Encrypt case we just process the payload and return. */ + if (instruction == OCS_ENCRYPT) { + return ocs_aes_ccm_encrypt_do_payload(aes_dev, dst_dma_list, + src_dma_list, src_size); + } + /* For Decypt we need to process the payload and then the tag. */ + rc = ocs_aes_ccm_decrypt_do_payload(aes_dev, dst_dma_list, + src_dma_list, src_size); + if (rc) + return rc; + + /* Process MAC/tag directly: feed tag to engine and wait for IRQ. */ + ocs_aes_ccm_write_encrypted_tag(aes_dev, in_tag, tag_size); + rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); + if (rc) + return rc; + + return ccm_compare_tag_to_yr(aes_dev, tag_size); +} + +/** + * ocs_create_linked_list_from_sg() - Create OCS DMA linked list from SG list. + * @aes_dev: The OCS AES device the list will be created for. + * @sg: The SG list OCS DMA linked list will be created from. When + * passed to this function, @sg must have been already mapped + * with dma_map_sg(). + * @sg_dma_count: The number of DMA-mapped entries in @sg. This must be the + * value returned by dma_map_sg() when @sg was mapped. + * @dll_desc: The OCS DMA dma_list to use to store information about the + * created linked list. + * @data_size: The size of the data (from the SG list) to be mapped into the + * OCS DMA linked list. + * @data_offset: The offset (within the SG list) of the data to be mapped. + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_create_linked_list_from_sg(const struct ocs_aes_dev *aes_dev, + struct scatterlist *sg, + int sg_dma_count, + struct ocs_dll_desc *dll_desc, + size_t data_size, size_t data_offset) +{ + struct ocs_dma_linked_list *ll = NULL; + struct scatterlist *sg_tmp; + unsigned int tmp; + int dma_nents; + int i; + + if (!dll_desc || !sg || !aes_dev) + return -EINVAL; + + /* Default values for when no ddl_desc is created. */ + dll_desc->vaddr = NULL; + dll_desc->dma_addr = DMA_MAPPING_ERROR; + dll_desc->size = 0; + + if (data_size == 0) + return 0; + + /* Loop over sg_list until we reach entry at specified offset. */ + while (data_offset >= sg_dma_len(sg)) { + data_offset -= sg_dma_len(sg); + sg_dma_count--; + sg = sg_next(sg); + /* If we reach the end of the list, offset was invalid. */ + if (!sg || sg_dma_count == 0) + return -EINVAL; + } + + /* Compute number of DMA-mapped SG entries to add into OCS DMA list. */ + dma_nents = 0; + tmp = 0; + sg_tmp = sg; + while (tmp < data_offset + data_size) { + /* If we reach the end of the list, data_size was invalid. */ + if (!sg_tmp) + return -EINVAL; + tmp += sg_dma_len(sg_tmp); + dma_nents++; + sg_tmp = sg_next(sg_tmp); + } + if (dma_nents > sg_dma_count) + return -EINVAL; + + /* Allocate the DMA list, one entry for each SG entry. */ + dll_desc->size = sizeof(struct ocs_dma_linked_list) * dma_nents; + dll_desc->vaddr = dma_alloc_coherent(aes_dev->dev, dll_desc->size, + &dll_desc->dma_addr, GFP_KERNEL); + if (!dll_desc->vaddr) + return -ENOMEM; + + /* Populate DMA linked list entries. */ + ll = dll_desc->vaddr; + for (i = 0; i < dma_nents; i++, sg = sg_next(sg)) { + ll[i].src_addr = sg_dma_address(sg) + data_offset; + ll[i].src_len = (sg_dma_len(sg) - data_offset) < data_size ? + (sg_dma_len(sg) - data_offset) : data_size; + data_offset = 0; + data_size -= ll[i].src_len; + /* Current element points to the DMA address of the next one. */ + ll[i].next = dll_desc->dma_addr + (sizeof(*ll) * (i + 1)); + ll[i].ll_flags = 0; + } + /* Terminate last element. */ + ll[i - 1].next = 0; + ll[i - 1].ll_flags = OCS_LL_DMA_FLAG_TERMINATE; + + return 0; +} diff --git a/drivers/crypto/keembay/ocs-aes.h b/drivers/crypto/keembay/ocs-aes.h new file mode 100644 index 000000000000..c035fc48b7ed --- /dev/null +++ b/drivers/crypto/keembay/ocs-aes.h @@ -0,0 +1,129 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Intel Keem Bay OCS AES Crypto Driver. + * + * Copyright (C) 2018-2020 Intel Corporation + */ + +#ifndef _CRYPTO_OCS_AES_H +#define _CRYPTO_OCS_AES_H + +#include + +enum ocs_cipher { + OCS_AES = 0, + OCS_SM4 = 1, +}; + +enum ocs_mode { + OCS_MODE_ECB = 0, + OCS_MODE_CBC = 1, + OCS_MODE_CTR = 2, + OCS_MODE_CCM = 6, + OCS_MODE_GCM = 7, + OCS_MODE_CTS = 9, +}; + +enum ocs_instruction { + OCS_ENCRYPT = 0, + OCS_DECRYPT = 1, + OCS_EXPAND = 2, + OCS_BYPASS = 3, +}; + +/** + * struct ocs_aes_dev - AES device context. + * @list: List head for insertion into device list hold + * by driver. + * @dev: OCS AES device. + * @irq: IRQ number. + * @base_reg: IO base address of OCS AES. + * @irq_copy_completion: Completion to indicate IRQ has been triggered. + * @dma_err_mask: Error reported by OCS DMA interrupts. + * @engine: Crypto engine for the device. + */ +struct ocs_aes_dev { + struct list_head list; + struct device *dev; + int irq; + void __iomem *base_reg; + struct completion irq_completion; + u32 dma_err_mask; + struct crypto_engine *engine; +}; + +/** + * struct ocs_dll_desc - Descriptor of an OCS DMA Linked List. + * @vaddr: Virtual address of the linked list head. + * @dma_addr: DMA address of the linked list head. + * @size: Size (in bytes) of the linked list. + */ +struct ocs_dll_desc { + void *vaddr; + dma_addr_t dma_addr; + size_t size; +}; + +int ocs_aes_set_key(struct ocs_aes_dev *aes_dev, const u32 key_size, + const u8 *key, const enum ocs_cipher cipher); + +int ocs_aes_op(struct ocs_aes_dev *aes_dev, + enum ocs_mode mode, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + u8 *iv, + u32 iv_size); + +/** + * ocs_aes_bypass_op() - Use OCS DMA to copy data. + * @aes_dev: The OCS AES device to use. + * @dst_dma_list: The OCS DMA list mapping the memory where input data + * will be copied to. + * @src_dma_list: The OCS DMA list mapping input data. + * @src_size: The amount of data to copy. + */ +static inline int ocs_aes_bypass_op(struct ocs_aes_dev *aes_dev, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, u32 src_size) +{ + return ocs_aes_op(aes_dev, OCS_MODE_ECB, OCS_AES, OCS_BYPASS, + dst_dma_list, src_dma_list, src_size, NULL, 0); +} + +int ocs_aes_gcm_op(struct ocs_aes_dev *aes_dev, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + const u8 *iv, + dma_addr_t aad_dma_list, + u32 aad_size, + u8 *out_tag, + u32 tag_size); + +int ocs_aes_ccm_op(struct ocs_aes_dev *aes_dev, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + u8 *iv, + dma_addr_t adata_dma_list, + u32 adata_size, + u8 *in_tag, + u32 tag_size); + +int ocs_create_linked_list_from_sg(const struct ocs_aes_dev *aes_dev, + struct scatterlist *sg, + int sg_dma_count, + struct ocs_dll_desc *dll_desc, + size_t data_size, + size_t data_offset); + +irqreturn_t ocs_aes_irq_handler(int irq, void *dev_id); + +#endif diff --git a/drivers/crypto/marvell/cesa/hash.c b/drivers/crypto/marvell/cesa/hash.c index add7ea011c98..8cf9fd518d86 100644 --- a/drivers/crypto/marvell/cesa/hash.c +++ b/drivers/crypto/marvell/cesa/hash.c @@ -11,7 +11,8 @@ #include #include -#include +#include +#include #include #include diff --git a/drivers/crypto/marvell/octeontx/otx_cptpf_main.c b/drivers/crypto/marvell/octeontx/otx_cptpf_main.c index 34bb3063eb70..14a42559f81d 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptpf_main.c +++ b/drivers/crypto/marvell/octeontx/otx_cptpf_main.c @@ -212,15 +212,9 @@ static int otx_cpt_probe(struct pci_dev *pdev, goto err_disable_device; } - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48)); + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); if (err) { - dev_err(dev, "Unable to get usable DMA configuration\n"); - goto err_release_regions; - } - - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48)); - if (err) { - dev_err(dev, "Unable to get 48-bit DMA for consistent allocations\n"); + dev_err(dev, "Unable to get usable 48-bit DMA configuration\n"); goto err_release_regions; } diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c b/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c index 90bb31329d4b..ccbef01888d4 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c +++ b/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c @@ -13,7 +13,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_main.c b/drivers/crypto/marvell/octeontx/otx_cptvf_main.c index 228fe8e47e0e..c076d0b3ad5f 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptvf_main.c +++ b/drivers/crypto/marvell/octeontx/otx_cptvf_main.c @@ -804,15 +804,9 @@ static int otx_cptvf_probe(struct pci_dev *pdev, dev_err(dev, "PCI request regions failed 0x%x\n", err); goto disable_device; } - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48)); + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); if (err) { - dev_err(dev, "Unable to get usable DMA configuration\n"); - goto release_regions; - } - - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48)); - if (err) { - dev_err(dev, "Unable to get 48-bit DMA for consistent allocations\n"); + dev_err(dev, "Unable to get usable 48-bit DMA configuration\n"); goto release_regions; } diff --git a/drivers/crypto/mediatek/mtk-sha.c b/drivers/crypto/mediatek/mtk-sha.c index 3d5d7d68b03b..f55aacdafbef 100644 --- a/drivers/crypto/mediatek/mtk-sha.c +++ b/drivers/crypto/mediatek/mtk-sha.c @@ -10,7 +10,8 @@ */ #include -#include +#include +#include #include "mtk-platform.h" #define SHA_ALIGN_MSK (sizeof(u32) - 1) diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c index 909a7eb748e3..d6a7784d2988 100644 --- a/drivers/crypto/mxs-dcp.c +++ b/drivers/crypto/mxs-dcp.c @@ -17,7 +17,8 @@ #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c index 3642bf83d809..3b0bf6fea491 100644 --- a/drivers/crypto/n2_core.c +++ b/drivers/crypto/n2_core.c @@ -15,7 +15,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/nx/nx-sha256.c b/drivers/crypto/nx/nx-sha256.c index 02fb53453195..90d9a37a57f6 100644 --- a/drivers/crypto/nx/nx-sha256.c +++ b/drivers/crypto/nx/nx-sha256.c @@ -8,7 +8,7 @@ */ #include -#include +#include #include #include #include diff --git a/drivers/crypto/nx/nx-sha512.c b/drivers/crypto/nx/nx-sha512.c index 4c7a3e3eeebf..eb8627a0f317 100644 --- a/drivers/crypto/nx/nx-sha512.c +++ b/drivers/crypto/nx/nx-sha512.c @@ -8,7 +8,7 @@ */ #include -#include +#include #include #include diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c index 40882d6d52c1..0d2dc5be7f19 100644 --- a/drivers/crypto/nx/nx.c +++ b/drivers/crypto/nx/nx.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 4fd14d90cc40..a45bdcf3026d 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -105,6 +105,7 @@ static int omap_aes_hw_init(struct omap_aes_dev *dd) err = pm_runtime_get_sync(dd->dev); if (err < 0) { + pm_runtime_put_noidle(dd->dev); dev_err(dd->dev, "failed to get sync: %d\n", err); return err; } @@ -1137,7 +1138,7 @@ static int omap_aes_probe(struct platform_device *pdev) if (err < 0) { dev_err(dev, "%s: failed to get_sync(%d)\n", __func__, err); - goto err_res; + goto err_pm_disable; } omap_aes_dma_stop(dd); @@ -1246,6 +1247,7 @@ err_engine: omap_aes_dma_cleanup(dd); err_irq: tasklet_kill(&dd->done_task); +err_pm_disable: pm_runtime_disable(dev); err_res: dd = NULL; diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index a3b38d2c92e7..ae0d320d3c60 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -35,7 +35,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/padlock-sha.c b/drivers/crypto/padlock-sha.c index a697a4a3f2d0..6865c7f1fc1a 100644 --- a/drivers/crypto/padlock-sha.c +++ b/drivers/crypto/padlock-sha.c @@ -9,7 +9,8 @@ #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/picoxcell_crypto.c b/drivers/crypto/picoxcell_crypto.c index fb34bf92861d..84f9c16d984c 100644 --- a/drivers/crypto/picoxcell_crypto.c +++ b/drivers/crypto/picoxcell_crypto.c @@ -8,7 +8,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/qat/Kconfig b/drivers/crypto/qat/Kconfig index 2006322345de..beb379b23dc3 100644 --- a/drivers/crypto/qat/Kconfig +++ b/drivers/crypto/qat/Kconfig @@ -46,6 +46,17 @@ config CRYPTO_DEV_QAT_C62X To compile this as a module, choose M here: the module will be called qat_c62x. +config CRYPTO_DEV_QAT_4XXX + tristate "Support for Intel(R) QAT_4XXX" + depends on X86 && PCI + select CRYPTO_DEV_QAT + help + Support for Intel(R) QuickAssist Technology QAT_4xxx + for accelerating crypto and compression workloads. + + To compile this as a module, choose M here: the module + will be called qat_4xxx. + config CRYPTO_DEV_QAT_DH895xCCVF tristate "Support for Intel(R) DH895xCC Virtual Function" depends on X86 && PCI diff --git a/drivers/crypto/qat/Makefile b/drivers/crypto/qat/Makefile index 7dd15e751d02..258c8a626ce0 100644 --- a/drivers/crypto/qat/Makefile +++ b/drivers/crypto/qat/Makefile @@ -3,6 +3,7 @@ obj-$(CONFIG_CRYPTO_DEV_QAT) += qat_common/ obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCC) += qat_dh895xcc/ obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXX) += qat_c3xxx/ obj-$(CONFIG_CRYPTO_DEV_QAT_C62X) += qat_c62x/ +obj-$(CONFIG_CRYPTO_DEV_QAT_4XXX) += qat_4xxx/ obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCCVF) += qat_dh895xccvf/ obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXXVF) += qat_c3xxxvf/ obj-$(CONFIG_CRYPTO_DEV_QAT_C62XVF) += qat_c62xvf/ diff --git a/drivers/crypto/qat/qat_4xxx/Makefile b/drivers/crypto/qat/qat_4xxx/Makefile new file mode 100644 index 000000000000..ff9c8b5897ea --- /dev/null +++ b/drivers/crypto/qat/qat_4xxx/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +ccflags-y := -I $(srctree)/$(src)/../qat_common +obj-$(CONFIG_CRYPTO_DEV_QAT_4XXX) += qat_4xxx.o +qat_4xxx-objs := adf_drv.o adf_4xxx_hw_data.o diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c new file mode 100644 index 000000000000..344bfae45bff --- /dev/null +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2020 Intel Corporation */ +#include +#include +#include +#include +#include "adf_4xxx_hw_data.h" +#include "icp_qat_hw.h" + +struct adf_fw_config { + u32 ae_mask; + char *obj_name; +}; + +static struct adf_fw_config adf_4xxx_fw_config[] = { + {0xF0, ADF_4XXX_SYM_OBJ}, + {0xF, ADF_4XXX_ASYM_OBJ}, + {0x100, ADF_4XXX_ADMIN_OBJ}, +}; + +/* Worker thread to service arbiter mappings */ +static u32 thrd_to_arb_map[] = { + 0x5555555, 0x5555555, 0x5555555, 0x5555555, + 0xAAAAAAA, 0xAAAAAAA, 0xAAAAAAA, 0xAAAAAAA, + 0x0 +}; + +static struct adf_hw_device_class adf_4xxx_class = { + .name = ADF_4XXX_DEVICE_NAME, + .type = DEV_4XXX, + .instances = 0, +}; + +static u32 get_accel_mask(struct adf_hw_device_data *self) +{ + return ADF_4XXX_ACCELERATORS_MASK; +} + +static u32 get_ae_mask(struct adf_hw_device_data *self) +{ + u32 me_disable = self->fuses; + + return ~me_disable & ADF_4XXX_ACCELENGINES_MASK; +} + +static u32 get_num_accels(struct adf_hw_device_data *self) +{ + return ADF_4XXX_MAX_ACCELERATORS; +} + +static u32 get_num_aes(struct adf_hw_device_data *self) +{ + if (!self || !self->ae_mask) + return 0; + + return hweight32(self->ae_mask); +} + +static u32 get_misc_bar_id(struct adf_hw_device_data *self) +{ + return ADF_4XXX_PMISC_BAR; +} + +static u32 get_etr_bar_id(struct adf_hw_device_data *self) +{ + return ADF_4XXX_ETR_BAR; +} + +static u32 get_sram_bar_id(struct adf_hw_device_data *self) +{ + return ADF_4XXX_SRAM_BAR; +} + +/* + * The vector routing table is used to select the MSI-X entry to use for each + * interrupt source. + * The first ADF_4XXX_ETR_MAX_BANKS entries correspond to ring interrupts. + * The final entry corresponds to VF2PF or error interrupts. + * This vector table could be used to configure one MSI-X entry to be shared + * between multiple interrupt sources. + * + * The default routing is set to have a one to one correspondence between the + * interrupt source and the MSI-X entry used. + */ +static void set_msix_default_rttable(struct adf_accel_dev *accel_dev) +{ + void __iomem *csr; + int i; + + csr = (&GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR])->virt_addr; + for (i = 0; i <= ADF_4XXX_ETR_MAX_BANKS; i++) + ADF_CSR_WR(csr, ADF_4XXX_MSIX_RTTABLE_OFFSET(i), i); +} + +static u32 get_accel_cap(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_dev->accel_pci_dev.pci_dev; + u32 fusectl1; + u32 capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | + ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | + ICP_ACCEL_CAPABILITIES_AUTHENTICATION | + ICP_ACCEL_CAPABILITIES_AES_V2; + + /* Read accelerator capabilities mask */ + pci_read_config_dword(pdev, ADF_4XXX_FUSECTL1_OFFSET, &fusectl1); + + if (fusectl1 & ICP_ACCEL_4XXX_MASK_CIPHER_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; + if (fusectl1 & ICP_ACCEL_4XXX_MASK_AUTH_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + if (fusectl1 & ICP_ACCEL_4XXX_MASK_PKE_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; + + return capabilities; +} + +static enum dev_sku_info get_sku(struct adf_hw_device_data *self) +{ + return DEV_SKU_1; +} + +static void adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev, + u32 const **arb_map_config) +{ + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + unsigned long ae_mask = hw_device->ae_mask; + int i; + + for_each_clear_bit(i, &ae_mask, ADF_4XXX_MAX_ACCELENGINES) + thrd_to_arb_map[i] = 0; + + *arb_map_config = thrd_to_arb_map; +} + +static void get_arb_info(struct arb_info *arb_info) +{ + arb_info->arb_cfg = ADF_4XXX_ARB_CONFIG; + arb_info->arb_offset = ADF_4XXX_ARB_OFFSET; + arb_info->wt2sam_offset = ADF_4XXX_ARB_WRK_2_SER_MAP_OFFSET; +} + +static void get_admin_info(struct admin_info *admin_csrs_info) +{ + admin_csrs_info->mailbox_offset = ADF_4XXX_MAILBOX_BASE_OFFSET; + admin_csrs_info->admin_msg_ur = ADF_4XXX_ADMINMSGUR_OFFSET; + admin_csrs_info->admin_msg_lr = ADF_4XXX_ADMINMSGLR_OFFSET; +} + +static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) +{ + struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR]; + void __iomem *csr = misc_bar->virt_addr; + + /* Enable all in errsou3 except VFLR notification on host */ + ADF_CSR_WR(csr, ADF_4XXX_ERRMSK3, ADF_4XXX_VFLNOTIFY); +} + +static void adf_enable_ints(struct adf_accel_dev *accel_dev) +{ + void __iomem *addr; + + addr = (&GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR])->virt_addr; + + /* Enable bundle interrupts */ + ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_RP_X0_MASK_OFFSET, 0); + ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_RP_X1_MASK_OFFSET, 0); + + /* Enable misc interrupts */ + ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_MASK_OFFSET, 0); +} + +static int adf_pf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev) +{ + return 0; +} + +static u32 uof_get_num_objs(void) +{ + return ARRAY_SIZE(adf_4xxx_fw_config); +} + +static char *uof_get_name(u32 obj_num) +{ + return adf_4xxx_fw_config[obj_num].obj_name; +} + +static u32 uof_get_ae_mask(u32 obj_num) +{ + return adf_4xxx_fw_config[obj_num].ae_mask; +} + +void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class = &adf_4xxx_class; + hw_data->instance_id = adf_4xxx_class.instances++; + hw_data->num_banks = ADF_4XXX_ETR_MAX_BANKS; + hw_data->num_rings_per_bank = ADF_4XXX_NUM_RINGS_PER_BANK; + hw_data->num_accel = ADF_4XXX_MAX_ACCELERATORS; + hw_data->num_engines = ADF_4XXX_MAX_ACCELENGINES; + hw_data->num_logical_accel = 1; + hw_data->tx_rx_gap = ADF_4XXX_RX_RINGS_OFFSET; + hw_data->tx_rings_mask = ADF_4XXX_TX_RINGS_MASK; + hw_data->alloc_irq = adf_isr_resource_alloc; + hw_data->free_irq = adf_isr_resource_free; + hw_data->enable_error_correction = adf_enable_error_correction; + hw_data->get_accel_mask = get_accel_mask; + hw_data->get_ae_mask = get_ae_mask; + hw_data->get_num_accels = get_num_accels; + hw_data->get_num_aes = get_num_aes; + hw_data->get_sram_bar_id = get_sram_bar_id; + hw_data->get_etr_bar_id = get_etr_bar_id; + hw_data->get_misc_bar_id = get_misc_bar_id; + hw_data->get_arb_info = get_arb_info; + hw_data->get_admin_info = get_admin_info; + hw_data->get_accel_cap = get_accel_cap; + hw_data->get_sku = get_sku; + hw_data->fw_name = ADF_4XXX_FW; + hw_data->fw_mmp_name = ADF_4XXX_MMP; + hw_data->init_admin_comms = adf_init_admin_comms; + hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->disable_iov = adf_disable_sriov; + hw_data->send_admin_init = adf_send_admin_init; + hw_data->init_arb = adf_init_arb; + hw_data->exit_arb = adf_exit_arb; + hw_data->get_arb_mapping = adf_get_arbiter_mapping; + hw_data->enable_ints = adf_enable_ints; + hw_data->enable_vf2pf_comms = adf_pf_enable_vf2pf_comms; + hw_data->reset_device = adf_reset_flr; + hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; + hw_data->admin_ae_mask = ADF_4XXX_ADMIN_AE_MASK; + hw_data->uof_get_num_objs = uof_get_num_objs; + hw_data->uof_get_name = uof_get_name; + hw_data->uof_get_ae_mask = uof_get_ae_mask; + hw_data->set_msix_rttable = set_msix_default_rttable; + + adf_gen4_init_hw_csr_ops(&hw_data->csr_ops); +} + +void adf_clean_hw_data_4xxx(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class->instances--; +} diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h new file mode 100644 index 000000000000..4fe2a776293c --- /dev/null +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#ifndef ADF_4XXX_HW_DATA_H_ +#define ADF_4XXX_HW_DATA_H_ + +#include + +/* PCIe configuration space */ +#define ADF_4XXX_SRAM_BAR 0 +#define ADF_4XXX_PMISC_BAR 1 +#define ADF_4XXX_ETR_BAR 2 +#define ADF_4XXX_RX_RINGS_OFFSET 1 +#define ADF_4XXX_TX_RINGS_MASK 0x1 +#define ADF_4XXX_MAX_ACCELERATORS 1 +#define ADF_4XXX_MAX_ACCELENGINES 9 +#define ADF_4XXX_BAR_MASK (BIT(0) | BIT(2) | BIT(4)) + +/* Physical function fuses */ +#define ADF_4XXX_FUSECTL0_OFFSET (0x2C8) +#define ADF_4XXX_FUSECTL1_OFFSET (0x2CC) +#define ADF_4XXX_FUSECTL2_OFFSET (0x2D0) +#define ADF_4XXX_FUSECTL3_OFFSET (0x2D4) +#define ADF_4XXX_FUSECTL4_OFFSET (0x2D8) +#define ADF_4XXX_FUSECTL5_OFFSET (0x2DC) + +#define ADF_4XXX_ACCELERATORS_MASK (0x1) +#define ADF_4XXX_ACCELENGINES_MASK (0x1FF) +#define ADF_4XXX_ADMIN_AE_MASK (0x100) + +#define ADF_4XXX_ETR_MAX_BANKS 64 + +/* MSIX interrupt */ +#define ADF_4XXX_SMIAPF_RP_X0_MASK_OFFSET (0x41A040) +#define ADF_4XXX_SMIAPF_RP_X1_MASK_OFFSET (0x41A044) +#define ADF_4XXX_SMIAPF_MASK_OFFSET (0x41A084) +#define ADF_4XXX_MSIX_RTTABLE_OFFSET(i) (0x409000 + ((i) * 0x04)) + +/* Bank and ring configuration */ +#define ADF_4XXX_NUM_RINGS_PER_BANK 2 + +/* Error source registers */ +#define ADF_4XXX_ERRSOU0 (0x41A200) +#define ADF_4XXX_ERRSOU1 (0x41A204) +#define ADF_4XXX_ERRSOU2 (0x41A208) +#define ADF_4XXX_ERRSOU3 (0x41A20C) + +/* Error source mask registers */ +#define ADF_4XXX_ERRMSK0 (0x41A210) +#define ADF_4XXX_ERRMSK1 (0x41A214) +#define ADF_4XXX_ERRMSK2 (0x41A218) +#define ADF_4XXX_ERRMSK3 (0x41A21C) + +#define ADF_4XXX_VFLNOTIFY BIT(7) + +/* Arbiter configuration */ +#define ADF_4XXX_ARB_CONFIG (BIT(31) | BIT(6) | BIT(0)) +#define ADF_4XXX_ARB_OFFSET (0x0) +#define ADF_4XXX_ARB_WRK_2_SER_MAP_OFFSET (0x400) + +/* Admin Interface Reg Offset */ +#define ADF_4XXX_ADMINMSGUR_OFFSET (0x500574) +#define ADF_4XXX_ADMINMSGLR_OFFSET (0x500578) +#define ADF_4XXX_MAILBOX_BASE_OFFSET (0x600970) + +/* Firmware Binaries */ +#define ADF_4XXX_FW "qat_4xxx.bin" +#define ADF_4XXX_MMP "qat_4xxx_mmp.bin" +#define ADF_4XXX_SYM_OBJ "qat_4xxx_sym.bin" +#define ADF_4XXX_ASYM_OBJ "qat_4xxx_asym.bin" +#define ADF_4XXX_ADMIN_OBJ "qat_4xxx_admin.bin" + +/* qat_4xxx fuse bits are different from old GENs, redefine them */ +enum icp_qat_4xxx_slice_mask { + ICP_ACCEL_4XXX_MASK_CIPHER_SLICE = BIT(0), + ICP_ACCEL_4XXX_MASK_AUTH_SLICE = BIT(1), + ICP_ACCEL_4XXX_MASK_PKE_SLICE = BIT(2), + ICP_ACCEL_4XXX_MASK_COMPRESS_SLICE = BIT(3), + ICP_ACCEL_4XXX_MASK_UCS_SLICE = BIT(4), + ICP_ACCEL_4XXX_MASK_EIA3_SLICE = BIT(5), + ICP_ACCEL_4XXX_MASK_SMX_SLICE = BIT(6), +}; + +void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data); +void adf_clean_hw_data_4xxx(struct adf_hw_device_data *hw_data); + +#endif diff --git a/drivers/crypto/qat/qat_4xxx/adf_drv.c b/drivers/crypto/qat/qat_4xxx/adf_drv.c new file mode 100644 index 000000000000..a8805c815d16 --- /dev/null +++ b/drivers/crypto/qat/qat_4xxx/adf_drv.c @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2020 Intel Corporation */ +#include +#include +#include + +#include +#include +#include + +#include "adf_4xxx_hw_data.h" +#include "qat_crypto.h" +#include "adf_transport_access_macros.h" + +static const struct pci_device_id adf_pci_tbl[] = { + { PCI_VDEVICE(INTEL, ADF_4XXX_PCI_DEVICE_ID), }, + { } +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) +{ + if (accel_dev->hw_device) { + adf_clean_hw_data_4xxx(accel_dev->hw_device); + accel_dev->hw_device = NULL; + } + adf_cfg_dev_remove(accel_dev); + debugfs_remove(accel_dev->debugfs_dir); + adf_devmgr_rm_dev(accel_dev, NULL); +} + +static int adf_crypto_dev_config(struct adf_accel_dev *accel_dev) +{ + char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; + int banks = GET_MAX_BANKS(accel_dev); + int cpus = num_online_cpus(); + unsigned long bank, val; + int instances; + int ret; + int i; + + if (adf_hw_dev_has_crypto(accel_dev)) + instances = min(cpus, banks / 2); + else + instances = 0; + + ret = adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC); + if (ret) + goto err; + + ret = adf_cfg_section_add(accel_dev, "Accelerator0"); + if (ret) + goto err; + + for (i = 0; i < instances; i++) { + val = i; + bank = i * 2; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_BANK_NUM, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &bank, ADF_DEC); + if (ret) + goto err; + + bank += 1; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_BANK_NUM, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &bank, ADF_DEC); + if (ret) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY, + i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i); + val = 128; + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 512; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 0; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 0; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 1; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 1; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = ADF_COALESCING_DEF_TIME; + snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i); + ret = adf_cfg_add_key_value_param(accel_dev, "Accelerator0", + key, &val, ADF_DEC); + if (ret) + goto err; + } + + val = i; + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_CY, + &val, ADF_DEC); + if (ret) + goto err; + + set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); + return 0; +err: + dev_err(&GET_DEV(accel_dev), "Failed to start QAT accel dev\n"); + return ret; +} + +static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct adf_accel_dev *accel_dev; + struct adf_accel_pci *accel_pci_dev; + struct adf_hw_device_data *hw_data; + char name[ADF_DEVICE_NAME_LENGTH]; + unsigned int i, bar_nr; + unsigned long bar_mask; + struct adf_bar *bar; + int ret; + + if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) { + /* + * If the accelerator is connected to a node with no memory + * there is no point in using the accelerator since the remote + * memory transaction will be very slow. + */ + dev_err(&pdev->dev, "Invalid NUMA configuration.\n"); + return -EINVAL; + } + + accel_dev = devm_kzalloc(&pdev->dev, sizeof(*accel_dev), GFP_KERNEL); + if (!accel_dev) + return -ENOMEM; + + INIT_LIST_HEAD(&accel_dev->crypto_list); + accel_pci_dev = &accel_dev->accel_pci_dev; + accel_pci_dev->pci_dev = pdev; + + /* + * Add accel device to accel table + * This should be called before adf_cleanup_accel is called + */ + if (adf_devmgr_add_dev(accel_dev, NULL)) { + dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); + return -EFAULT; + } + + accel_dev->owner = THIS_MODULE; + /* Allocate and initialise device hardware meta-data structure */ + hw_data = devm_kzalloc(&pdev->dev, sizeof(*hw_data), GFP_KERNEL); + if (!hw_data) { + ret = -ENOMEM; + goto out_err; + } + + accel_dev->hw_device = hw_data; + adf_init_hw_data_4xxx(accel_dev->hw_device); + + pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); + pci_read_config_dword(pdev, ADF_4XXX_FUSECTL4_OFFSET, &hw_data->fuses); + + /* Get Accelerators and Accelerators Engines masks */ + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); + accel_pci_dev->sku = hw_data->get_sku(hw_data); + /* If the device has no acceleration engines then ignore it */ + if (!hw_data->accel_mask || !hw_data->ae_mask || + (~hw_data->ae_mask & 0x01)) { + dev_err(&pdev->dev, "No acceleration units found.\n"); + ret = -EFAULT; + goto out_err; + } + + /* Create dev top level debugfs entry */ + snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX, + hw_data->dev_class->name, pci_name(pdev)); + + accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); + + /* Create device configuration table */ + ret = adf_cfg_dev_add(accel_dev); + if (ret) + goto out_err; + + /* Enable PCI device */ + ret = pcim_enable_device(pdev); + if (ret) { + dev_err(&pdev->dev, "Can't enable PCI device.\n"); + goto out_err; + } + + /* Set DMA identifier */ + if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { + if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))) { + dev_err(&pdev->dev, "No usable DMA configuration.\n"); + ret = -EFAULT; + goto out_err; + } else { + pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); + } + } else { + pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); + } + + /* Get accelerator capabilities mask */ + hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); + + /* Find and map all the device's BARS */ + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM) & ADF_4XXX_BAR_MASK; + + ret = pcim_iomap_regions_request_all(pdev, bar_mask, pci_name(pdev)); + if (ret) { + dev_err(&pdev->dev, "Failed to map pci regions.\n"); + goto out_err; + } + + i = 0; + for_each_set_bit(bar_nr, &bar_mask, PCI_STD_NUM_BARS) { + bar = &accel_pci_dev->pci_bars[i++]; + bar->virt_addr = pcim_iomap_table(pdev)[bar_nr]; + } + + pci_set_master(pdev); + + if (adf_enable_aer(accel_dev)) { + dev_err(&pdev->dev, "Failed to enable aer.\n"); + ret = -EFAULT; + goto out_err; + } + + if (pci_save_state(pdev)) { + dev_err(&pdev->dev, "Failed to save pci state.\n"); + ret = -ENOMEM; + goto out_err_disable_aer; + } + + ret = adf_crypto_dev_config(accel_dev); + if (ret) + goto out_err_disable_aer; + + ret = adf_dev_init(accel_dev); + if (ret) + goto out_err_dev_shutdown; + + ret = adf_dev_start(accel_dev); + if (ret) + goto out_err_dev_stop; + + return ret; + +out_err_dev_stop: + adf_dev_stop(accel_dev); +out_err_dev_shutdown: + adf_dev_shutdown(accel_dev); +out_err_disable_aer: + adf_disable_aer(accel_dev); +out_err: + adf_cleanup_accel(accel_dev); + return ret; +} + +static void adf_remove(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + if (!accel_dev) { + pr_err("QAT: Driver removal failed\n"); + return; + } + adf_dev_stop(accel_dev); + adf_dev_shutdown(accel_dev); + adf_disable_aer(accel_dev); + adf_cleanup_accel(accel_dev); +} + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = ADF_4XXX_DEVICE_NAME, + .probe = adf_probe, + .remove = adf_remove, + .sriov_configure = adf_sriov_configure, +}; + +module_pci_driver(adf_driver); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Intel"); +MODULE_FIRMWARE(ADF_4XXX_FW); +MODULE_FIRMWARE(ADF_4XXX_MMP); +MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); +MODULE_VERSION(ADF_DRV_VERSION); +MODULE_SOFTDEP("pre: crypto-intel_qat"); diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c index aee494d3da52..eb45f1b1ae3e 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -3,7 +3,9 @@ #include #include #include +#include #include "adf_c3xxx_hw_data.h" +#include "icp_qat_hw.h" /* Worker thread to service arbiter mappings based on dev SKUs */ static const u32 thrd_to_arb_map_6_me_sku[] = { @@ -17,15 +19,33 @@ static struct adf_hw_device_class c3xxx_class = { .instances = 0 }; -static u32 get_accel_mask(u32 fuse) +static u32 get_accel_mask(struct adf_hw_device_data *self) { - return (~fuse) >> ADF_C3XXX_ACCELERATORS_REG_OFFSET & - ADF_C3XXX_ACCELERATORS_MASK; + u32 straps = self->straps; + u32 fuses = self->fuses; + u32 accel; + + accel = ~(fuses | straps) >> ADF_C3XXX_ACCELERATORS_REG_OFFSET; + accel &= ADF_C3XXX_ACCELERATORS_MASK; + + return accel; } -static u32 get_ae_mask(u32 fuse) +static u32 get_ae_mask(struct adf_hw_device_data *self) { - return (~fuse) & ADF_C3XXX_ACCELENGINES_MASK; + u32 straps = self->straps; + u32 fuses = self->fuses; + unsigned long disabled; + u32 ae_disable; + int accel; + + /* If an accel is disabled, then disable the corresponding two AEs */ + disabled = ~get_accel_mask(self) & ADF_C3XXX_ACCELERATORS_MASK; + ae_disable = BIT(1) | BIT(0); + for_each_set_bit(accel, &disabled, ADF_C3XXX_MAX_ACCELERATORS) + straps |= ae_disable << (accel << 1); + + return ~(fuses | straps) & ADF_C3XXX_ACCELENGINES_MASK; } static u32 get_num_accels(struct adf_hw_device_data *self) @@ -109,11 +129,13 @@ static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_device = accel_dev->hw_device; struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_C3XXX_PMISC_BAR]; + unsigned long accel_mask = hw_device->accel_mask; + unsigned long ae_mask = hw_device->ae_mask; void __iomem *csr = misc_bar->virt_addr; unsigned int val, i; /* Enable Accel Engine error detection & correction */ - for (i = 0; i < hw_device->get_num_aes(hw_device); i++) { + for_each_set_bit(i, &ae_mask, GET_MAX_ACCELENGINES(accel_dev)) { val = ADF_CSR_RD(csr, ADF_C3XXX_AE_CTX_ENABLES(i)); val |= ADF_C3XXX_ENABLE_AE_ECC_ERR; ADF_CSR_WR(csr, ADF_C3XXX_AE_CTX_ENABLES(i), val); @@ -123,7 +145,7 @@ static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) } /* Enable shared memory error detection & correction */ - for (i = 0; i < hw_device->get_num_accels(hw_device); i++) { + for_each_set_bit(i, &accel_mask, ADF_C3XXX_MAX_ACCELERATORS) { val = ADF_CSR_RD(csr, ADF_C3XXX_UERRSSMSH(i)); val |= ADF_C3XXX_ERRSSMSH_EN; ADF_CSR_WR(csr, ADF_C3XXX_UERRSSMSH(i), val); @@ -151,11 +173,19 @@ static int adf_pf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev) return 0; } +static void configure_iov_threads(struct adf_accel_dev *accel_dev, bool enable) +{ + adf_gen2_cfg_iov_thds(accel_dev, enable, + ADF_C3XXX_AE2FUNC_MAP_GRP_A_NUM_REGS, + ADF_C3XXX_AE2FUNC_MAP_GRP_B_NUM_REGS); +} + void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) { hw_data->dev_class = &c3xxx_class; hw_data->instance_id = c3xxx_class.instances++; hw_data->num_banks = ADF_C3XXX_ETR_MAX_BANKS; + hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; hw_data->num_accel = ADF_C3XXX_MAX_ACCELERATORS; hw_data->num_logical_accel = 1; hw_data->num_engines = ADF_C3XXX_MAX_ACCELENGINES; @@ -166,6 +196,7 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) hw_data->enable_error_correction = adf_enable_error_correction; hw_data->get_accel_mask = get_accel_mask; hw_data->get_ae_mask = get_ae_mask; + hw_data->get_accel_cap = adf_gen2_get_accel_cap; hw_data->get_num_accels = get_num_accels; hw_data->get_num_aes = get_num_aes; hw_data->get_sram_bar_id = get_sram_bar_id; @@ -173,11 +204,14 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) hw_data->get_misc_bar_id = get_misc_bar_id; hw_data->get_pf2vf_offset = get_pf2vf_offset; hw_data->get_vintmsk_offset = get_vintmsk_offset; + hw_data->get_admin_info = adf_gen2_get_admin_info; + hw_data->get_arb_info = adf_gen2_get_arb_info; hw_data->get_sku = get_sku; hw_data->fw_name = ADF_C3XXX_FW; hw_data->fw_mmp_name = ADF_C3XXX_MMP; hw_data->init_admin_comms = adf_init_admin_comms; hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->configure_iov_threads = configure_iov_threads; hw_data->disable_iov = adf_disable_sriov; hw_data->send_admin_init = adf_send_admin_init; hw_data->init_arb = adf_init_arb; @@ -187,6 +221,7 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) hw_data->enable_vf2pf_comms = adf_pf_enable_vf2pf_comms; hw_data->reset_device = adf_reset_flr; hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; + adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); } void adf_clean_hw_data_c3xxx(struct adf_hw_device_data *hw_data) diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h index 8b5dd2c94ebf..fece8e38025a 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h @@ -18,6 +18,7 @@ #define ADF_C3XXX_SMIAPF1_MASK_OFFSET (0x3A000 + 0x30) #define ADF_C3XXX_SMIA0_MASK 0xFFFF #define ADF_C3XXX_SMIA1_MASK 0x1 +#define ADF_C3XXX_SOFTSTRAP_CSR_OFFSET 0x2EC /* Error detection and correction */ #define ADF_C3XXX_AE_CTX_ENABLES(i) (i * 0x1000 + 0x20818) #define ADF_C3XXX_AE_MISC_CONTROL(i) (i * 0x1000 + 0x20960) @@ -30,6 +31,10 @@ #define ADF_C3XXX_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) #define ADF_C3XXX_VINTMSK_OFFSET(i) (0x3A000 + 0x200 + ((i) * 0x04)) +/* AE to function mapping */ +#define ADF_C3XXX_AE2FUNC_MAP_GRP_A_NUM_REGS 48 +#define ADF_C3XXX_AE2FUNC_MAP_GRP_B_NUM_REGS 6 + /* Firmware Binary */ #define ADF_C3XXX_FW "qat_c3xxx.bin" #define ADF_C3XXX_MMP "qat_c3xxx_mmp.bin" diff --git a/drivers/crypto/qat/qat_c3xxx/adf_drv.c b/drivers/crypto/qat/qat_c3xxx/adf_drv.c index ed0e8e33fe4b..7fb3343ae8b0 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_drv.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_drv.c @@ -126,10 +126,12 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET, &hw_data->fuses); + pci_read_config_dword(pdev, ADF_C3XXX_SOFTSTRAP_CSR_OFFSET, + &hw_data->straps); /* Get Accelerators and Accelerators Engines masks */ - hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); - hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); accel_pci_dev->sku = hw_data->get_sku(hw_data); /* If the device has no acceleration engines then ignore it. */ if (!hw_data->accel_mask || !hw_data->ae_mask || @@ -175,9 +177,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto out_err_disable; } - /* Read accelerator capabilities mask */ - pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET, - &hw_data->accel_capabilities_mask); + /* Get accelerator capabilities mask */ + hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); /* Find and map all the device's BARS */ i = 0; diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c index d2fedbd7113c..15f6b9bdfb22 100644 --- a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "adf_c3xxxvf_hw_data.h" static struct adf_hw_device_class c3xxxiov_class = { @@ -11,12 +12,12 @@ static struct adf_hw_device_class c3xxxiov_class = { .instances = 0 }; -static u32 get_accel_mask(u32 fuse) +static u32 get_accel_mask(struct adf_hw_device_data *self) { return ADF_C3XXXIOV_ACCELERATORS_MASK; } -static u32 get_ae_mask(u32 fuse) +static u32 get_ae_mask(struct adf_hw_device_data *self) { return ADF_C3XXXIOV_ACCELENGINES_MASK; } @@ -69,6 +70,7 @@ void adf_init_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data) { hw_data->dev_class = &c3xxxiov_class; hw_data->num_banks = ADF_C3XXXIOV_ETR_MAX_BANKS; + hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; hw_data->num_accel = ADF_C3XXXIOV_MAX_ACCELERATORS; hw_data->num_logical_accel = 1; hw_data->num_engines = ADF_C3XXXIOV_MAX_ACCELENGINES; @@ -97,6 +99,7 @@ void adf_init_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data) hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; hw_data->dev_class->instances++; adf_devmgr_update_class_index(hw_data); + adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); } void adf_clean_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data) diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c b/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c index 456979b136a2..1d1532e8fb6d 100644 --- a/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c +++ b/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c @@ -119,8 +119,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_c3xxxiov(accel_dev->hw_device); /* Get Accelerators and Accelerators Engines masks */ - hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); - hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); accel_pci_dev->sku = hw_data->get_sku(hw_data); /* Create dev top level debugfs entry */ diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c index 844ad5ed33fc..babdffbcb846 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -3,7 +3,9 @@ #include #include #include +#include #include "adf_c62x_hw_data.h" +#include "icp_qat_hw.h" /* Worker thread to service arbiter mappings based on dev SKUs */ static const u32 thrd_to_arb_map_8_me_sku[] = { @@ -22,15 +24,33 @@ static struct adf_hw_device_class c62x_class = { .instances = 0 }; -static u32 get_accel_mask(u32 fuse) +static u32 get_accel_mask(struct adf_hw_device_data *self) { - return (~fuse) >> ADF_C62X_ACCELERATORS_REG_OFFSET & - ADF_C62X_ACCELERATORS_MASK; + u32 straps = self->straps; + u32 fuses = self->fuses; + u32 accel; + + accel = ~(fuses | straps) >> ADF_C62X_ACCELERATORS_REG_OFFSET; + accel &= ADF_C62X_ACCELERATORS_MASK; + + return accel; } -static u32 get_ae_mask(u32 fuse) +static u32 get_ae_mask(struct adf_hw_device_data *self) { - return (~fuse) & ADF_C62X_ACCELENGINES_MASK; + u32 straps = self->straps; + u32 fuses = self->fuses; + unsigned long disabled; + u32 ae_disable; + int accel; + + /* If an accel is disabled, then disable the corresponding two AEs */ + disabled = ~get_accel_mask(self) & ADF_C62X_ACCELERATORS_MASK; + ae_disable = BIT(1) | BIT(0); + for_each_set_bit(accel, &disabled, ADF_C62X_MAX_ACCELERATORS) + straps |= ae_disable << (accel << 1); + + return ~(fuses | straps) & ADF_C62X_ACCELENGINES_MASK; } static u32 get_num_accels(struct adf_hw_device_data *self) @@ -119,11 +139,13 @@ static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_device = accel_dev->hw_device; struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_C62X_PMISC_BAR]; + unsigned long accel_mask = hw_device->accel_mask; + unsigned long ae_mask = hw_device->ae_mask; void __iomem *csr = misc_bar->virt_addr; unsigned int val, i; /* Enable Accel Engine error detection & correction */ - for (i = 0; i < hw_device->get_num_aes(hw_device); i++) { + for_each_set_bit(i, &ae_mask, GET_MAX_ACCELENGINES(accel_dev)) { val = ADF_CSR_RD(csr, ADF_C62X_AE_CTX_ENABLES(i)); val |= ADF_C62X_ENABLE_AE_ECC_ERR; ADF_CSR_WR(csr, ADF_C62X_AE_CTX_ENABLES(i), val); @@ -133,7 +155,7 @@ static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) } /* Enable shared memory error detection & correction */ - for (i = 0; i < hw_device->get_num_accels(hw_device); i++) { + for_each_set_bit(i, &accel_mask, ADF_C62X_MAX_ACCELERATORS) { val = ADF_CSR_RD(csr, ADF_C62X_UERRSSMSH(i)); val |= ADF_C62X_ERRSSMSH_EN; ADF_CSR_WR(csr, ADF_C62X_UERRSSMSH(i), val); @@ -161,11 +183,19 @@ static int adf_pf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev) return 0; } +static void configure_iov_threads(struct adf_accel_dev *accel_dev, bool enable) +{ + adf_gen2_cfg_iov_thds(accel_dev, enable, + ADF_C62X_AE2FUNC_MAP_GRP_A_NUM_REGS, + ADF_C62X_AE2FUNC_MAP_GRP_B_NUM_REGS); +} + void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) { hw_data->dev_class = &c62x_class; hw_data->instance_id = c62x_class.instances++; hw_data->num_banks = ADF_C62X_ETR_MAX_BANKS; + hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; hw_data->num_accel = ADF_C62X_MAX_ACCELERATORS; hw_data->num_logical_accel = 1; hw_data->num_engines = ADF_C62X_MAX_ACCELENGINES; @@ -176,6 +206,7 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) hw_data->enable_error_correction = adf_enable_error_correction; hw_data->get_accel_mask = get_accel_mask; hw_data->get_ae_mask = get_ae_mask; + hw_data->get_accel_cap = adf_gen2_get_accel_cap; hw_data->get_num_accels = get_num_accels; hw_data->get_num_aes = get_num_aes; hw_data->get_sram_bar_id = get_sram_bar_id; @@ -183,11 +214,14 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) hw_data->get_misc_bar_id = get_misc_bar_id; hw_data->get_pf2vf_offset = get_pf2vf_offset; hw_data->get_vintmsk_offset = get_vintmsk_offset; + hw_data->get_admin_info = adf_gen2_get_admin_info; + hw_data->get_arb_info = adf_gen2_get_arb_info; hw_data->get_sku = get_sku; hw_data->fw_name = ADF_C62X_FW; hw_data->fw_mmp_name = ADF_C62X_MMP; hw_data->init_admin_comms = adf_init_admin_comms; hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->configure_iov_threads = configure_iov_threads; hw_data->disable_iov = adf_disable_sriov; hw_data->send_admin_init = adf_send_admin_init; hw_data->init_arb = adf_init_arb; @@ -197,6 +231,7 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) hw_data->enable_vf2pf_comms = adf_pf_enable_vf2pf_comms; hw_data->reset_device = adf_reset_flr; hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; + adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); } void adf_clean_hw_data_c62x(struct adf_hw_device_data *hw_data) diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h index 88504d2bf30d..53d3cb577f5b 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h @@ -19,6 +19,7 @@ #define ADF_C62X_SMIAPF1_MASK_OFFSET (0x3A000 + 0x30) #define ADF_C62X_SMIA0_MASK 0xFFFF #define ADF_C62X_SMIA1_MASK 0x1 +#define ADF_C62X_SOFTSTRAP_CSR_OFFSET 0x2EC /* Error detection and correction */ #define ADF_C62X_AE_CTX_ENABLES(i) (i * 0x1000 + 0x20818) #define ADF_C62X_AE_MISC_CONTROL(i) (i * 0x1000 + 0x20960) @@ -31,6 +32,10 @@ #define ADF_C62X_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) #define ADF_C62X_VINTMSK_OFFSET(i) (0x3A000 + 0x200 + ((i) * 0x04)) +/* AE to function mapping */ +#define ADF_C62X_AE2FUNC_MAP_GRP_A_NUM_REGS 80 +#define ADF_C62X_AE2FUNC_MAP_GRP_B_NUM_REGS 10 + /* Firmware Binary */ #define ADF_C62X_FW "qat_c62x.bin" #define ADF_C62X_MMP "qat_c62x_mmp.bin" diff --git a/drivers/crypto/qat/qat_c62x/adf_drv.c b/drivers/crypto/qat/qat_c62x/adf_drv.c index d8e7c9c25590..1f5de442e1e6 100644 --- a/drivers/crypto/qat/qat_c62x/adf_drv.c +++ b/drivers/crypto/qat/qat_c62x/adf_drv.c @@ -126,10 +126,12 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET, &hw_data->fuses); + pci_read_config_dword(pdev, ADF_C62X_SOFTSTRAP_CSR_OFFSET, + &hw_data->straps); /* Get Accelerators and Accelerators Engines masks */ - hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); - hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); accel_pci_dev->sku = hw_data->get_sku(hw_data); /* If the device has no acceleration engines then ignore it. */ if (!hw_data->accel_mask || !hw_data->ae_mask || @@ -175,9 +177,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto out_err_disable; } - /* Read accelerator capabilities mask */ - pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET, - &hw_data->accel_capabilities_mask); + /* Get accelerator capabilities mask */ + hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); /* Find and map all the device's BARS */ i = (hw_data->fuses & ADF_DEVICE_FUSECTL_MASK) ? 1 : 0; diff --git a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c index 29fd3f1091ab..d231583428c9 100644 --- a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c +++ b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "adf_c62xvf_hw_data.h" static struct adf_hw_device_class c62xiov_class = { @@ -11,12 +12,12 @@ static struct adf_hw_device_class c62xiov_class = { .instances = 0 }; -static u32 get_accel_mask(u32 fuse) +static u32 get_accel_mask(struct adf_hw_device_data *self) { return ADF_C62XIOV_ACCELERATORS_MASK; } -static u32 get_ae_mask(u32 fuse) +static u32 get_ae_mask(struct adf_hw_device_data *self) { return ADF_C62XIOV_ACCELENGINES_MASK; } @@ -69,6 +70,7 @@ void adf_init_hw_data_c62xiov(struct adf_hw_device_data *hw_data) { hw_data->dev_class = &c62xiov_class; hw_data->num_banks = ADF_C62XIOV_ETR_MAX_BANKS; + hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; hw_data->num_accel = ADF_C62XIOV_MAX_ACCELERATORS; hw_data->num_logical_accel = 1; hw_data->num_engines = ADF_C62XIOV_MAX_ACCELENGINES; @@ -97,6 +99,7 @@ void adf_init_hw_data_c62xiov(struct adf_hw_device_data *hw_data) hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; hw_data->dev_class->instances++; adf_devmgr_update_class_index(hw_data); + adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); } void adf_clean_hw_data_c62xiov(struct adf_hw_device_data *hw_data) diff --git a/drivers/crypto/qat/qat_c62xvf/adf_drv.c b/drivers/crypto/qat/qat_c62xvf/adf_drv.c index b9810f79eb84..04742a6d91ca 100644 --- a/drivers/crypto/qat/qat_c62xvf/adf_drv.c +++ b/drivers/crypto/qat/qat_c62xvf/adf_drv.c @@ -119,8 +119,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_c62xiov(accel_dev->hw_device); /* Get Accelerators and Accelerators Engines masks */ - hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); - hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); accel_pci_dev->sku = hw_data->get_sku(hw_data); /* Create dev top level debugfs entry */ diff --git a/drivers/crypto/qat/qat_common/Makefile b/drivers/crypto/qat/qat_common/Makefile index 47a8e3d8b81a..9c57abdf56b7 100644 --- a/drivers/crypto/qat/qat_common/Makefile +++ b/drivers/crypto/qat/qat_common/Makefile @@ -10,6 +10,8 @@ intel_qat-objs := adf_cfg.o \ adf_transport.o \ adf_admin.o \ adf_hw_arbiter.o \ + adf_gen2_hw_data.o \ + adf_gen4_hw_data.o \ qat_crypto.o \ qat_algs.o \ qat_asym_algs.o \ diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 06952ece53d9..c46a5805b294 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -15,6 +15,9 @@ #define ADF_C62XVF_DEVICE_NAME "c6xxvf" #define ADF_C3XXX_DEVICE_NAME "c3xxx" #define ADF_C3XXXVF_DEVICE_NAME "c3xxxvf" +#define ADF_4XXX_DEVICE_NAME "4xxx" +#define ADF_4XXX_PCI_DEVICE_ID 0x4940 +#define ADF_4XXXIOV_PCI_DEVICE_ID 0x4941 #define ADF_ERRSOU3 (0x3A000 + 0x0C) #define ADF_ERRSOU5 (0x3A000 + 0xD8) #define ADF_DEVICE_FUSECTL_OFFSET 0x40 @@ -97,6 +100,46 @@ struct adf_hw_device_class { u32 instances; } __packed; +struct arb_info { + u32 arb_cfg; + u32 arb_offset; + u32 wt2sam_offset; +}; + +struct admin_info { + u32 admin_msg_ur; + u32 admin_msg_lr; + u32 mailbox_offset; +}; + +struct adf_hw_csr_ops { + u64 (*build_csr_ring_base_addr)(dma_addr_t addr, u32 size); + u32 (*read_csr_ring_head)(void __iomem *csr_base_addr, u32 bank, + u32 ring); + void (*write_csr_ring_head)(void __iomem *csr_base_addr, u32 bank, + u32 ring, u32 value); + u32 (*read_csr_ring_tail)(void __iomem *csr_base_addr, u32 bank, + u32 ring); + void (*write_csr_ring_tail)(void __iomem *csr_base_addr, u32 bank, + u32 ring, u32 value); + u32 (*read_csr_e_stat)(void __iomem *csr_base_addr, u32 bank); + void (*write_csr_ring_config)(void __iomem *csr_base_addr, u32 bank, + u32 ring, u32 value); + void (*write_csr_ring_base)(void __iomem *csr_base_addr, u32 bank, + u32 ring, dma_addr_t addr); + void (*write_csr_int_flag)(void __iomem *csr_base_addr, u32 bank, + u32 value); + void (*write_csr_int_srcsel)(void __iomem *csr_base_addr, u32 bank); + void (*write_csr_int_col_en)(void __iomem *csr_base_addr, u32 bank, + u32 value); + void (*write_csr_int_col_ctl)(void __iomem *csr_base_addr, u32 bank, + u32 value); + void (*write_csr_int_flag_and_col)(void __iomem *csr_base_addr, + u32 bank, u32 value); + void (*write_csr_ring_srv_arb_en)(void __iomem *csr_base_addr, u32 bank, + u32 value); +}; + struct adf_cfg_device_data; struct adf_accel_dev; struct adf_etr_data; @@ -104,8 +147,9 @@ struct adf_etr_ring_data; struct adf_hw_device_data { struct adf_hw_device_class *dev_class; - u32 (*get_accel_mask)(u32 fuse); - u32 (*get_ae_mask)(u32 fuse); + u32 (*get_accel_mask)(struct adf_hw_device_data *self); + u32 (*get_ae_mask)(struct adf_hw_device_data *self); + u32 (*get_accel_cap)(struct adf_accel_dev *accel_dev); u32 (*get_sram_bar_id)(struct adf_hw_device_data *self); u32 (*get_misc_bar_id)(struct adf_hw_device_data *self); u32 (*get_etr_bar_id)(struct adf_hw_device_data *self); @@ -113,6 +157,8 @@ struct adf_hw_device_data { u32 (*get_num_accels)(struct adf_hw_device_data *self); u32 (*get_pf2vf_offset)(u32 i); u32 (*get_vintmsk_offset)(u32 i); + void (*get_arb_info)(struct arb_info *arb_csrs_info); + void (*get_admin_info)(struct admin_info *admin_csrs_info); enum dev_sku_info (*get_sku)(struct adf_hw_device_data *self); int (*alloc_irq)(struct adf_accel_dev *accel_dev); void (*free_irq)(struct adf_accel_dev *accel_dev); @@ -125,19 +171,29 @@ struct adf_hw_device_data { void (*get_arb_mapping)(struct adf_accel_dev *accel_dev, const u32 **cfg); void (*disable_iov)(struct adf_accel_dev *accel_dev); + void (*configure_iov_threads)(struct adf_accel_dev *accel_dev, + bool enable); void (*enable_ints)(struct adf_accel_dev *accel_dev); int (*enable_vf2pf_comms)(struct adf_accel_dev *accel_dev); void (*reset_device)(struct adf_accel_dev *accel_dev); + void (*set_msix_rttable)(struct adf_accel_dev *accel_dev); + char *(*uof_get_name)(u32 obj_num); + u32 (*uof_get_num_objs)(void); + u32 (*uof_get_ae_mask)(u32 obj_num); + struct adf_hw_csr_ops csr_ops; const char *fw_name; const char *fw_mmp_name; u32 fuses; + u32 straps; u32 accel_capabilities_mask; u32 instance_id; u16 accel_mask; - u16 ae_mask; + u32 ae_mask; + u32 admin_ae_mask; u16 tx_rings_mask; u8 tx_rx_gap; u8 num_banks; + u8 num_rings_per_bank; u8 num_accel; u8 num_logical_accel; u8 num_engines; @@ -155,7 +211,10 @@ struct adf_hw_device_data { #define GET_BARS(accel_dev) ((accel_dev)->accel_pci_dev.pci_bars) #define GET_HW_DATA(accel_dev) (accel_dev->hw_device) #define GET_MAX_BANKS(accel_dev) (GET_HW_DATA(accel_dev)->num_banks) +#define GET_NUM_RINGS_PER_BANK(accel_dev) \ + GET_HW_DATA(accel_dev)->num_rings_per_bank #define GET_MAX_ACCELENGINES(accel_dev) (GET_HW_DATA(accel_dev)->num_engines) +#define GET_CSR_OPS(accel_dev) (&(accel_dev)->hw_device->csr_ops) #define accel_to_pci_dev(accel_ptr) accel_ptr->accel_pci_dev.pci_dev struct adf_admin_comms; diff --git a/drivers/crypto/qat/qat_common/adf_accel_engine.c b/drivers/crypto/qat/qat_common/adf_accel_engine.c index c8ad85b882be..ca4eae8cdd0b 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_engine.c +++ b/drivers/crypto/qat/qat_common/adf_accel_engine.c @@ -7,12 +7,55 @@ #include "adf_common_drv.h" #include "icp_qat_uclo.h" +static int adf_ae_fw_load_images(struct adf_accel_dev *accel_dev, void *fw_addr, + u32 fw_size) +{ + struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + struct icp_qat_fw_loader_handle *loader; + char *obj_name; + u32 num_objs; + u32 ae_mask; + int i; + + loader = loader_data->fw_loader; + num_objs = hw_device->uof_get_num_objs(); + + for (i = 0; i < num_objs; i++) { + obj_name = hw_device->uof_get_name(i); + ae_mask = hw_device->uof_get_ae_mask(i); + + if (qat_uclo_set_cfg_ae_mask(loader, ae_mask)) { + dev_err(&GET_DEV(accel_dev), + "Invalid mask for UOF image\n"); + goto out_err; + } + if (qat_uclo_map_obj(loader, fw_addr, fw_size, obj_name)) { + dev_err(&GET_DEV(accel_dev), + "Failed to map UOF firmware\n"); + goto out_err; + } + if (qat_uclo_wr_all_uimage(loader)) { + dev_err(&GET_DEV(accel_dev), + "Failed to load UOF firmware\n"); + goto out_err; + } + qat_uclo_del_obj(loader); + } + + return 0; + +out_err: + adf_ae_fw_release(accel_dev); + return -EFAULT; +} + int adf_ae_fw_load(struct adf_accel_dev *accel_dev) { struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; struct adf_hw_device_data *hw_device = accel_dev->hw_device; - void *uof_addr, *mmp_addr; - u32 uof_size, mmp_size; + void *fw_addr, *mmp_addr; + u32 fw_size, mmp_size; if (!hw_device->fw_name) return 0; @@ -30,15 +73,20 @@ int adf_ae_fw_load(struct adf_accel_dev *accel_dev) goto out_err; } - uof_size = loader_data->uof_fw->size; - uof_addr = (void *)loader_data->uof_fw->data; + fw_size = loader_data->uof_fw->size; + fw_addr = (void *)loader_data->uof_fw->data; mmp_size = loader_data->mmp_fw->size; mmp_addr = (void *)loader_data->mmp_fw->data; + if (qat_uclo_wr_mimage(loader_data->fw_loader, mmp_addr, mmp_size)) { dev_err(&GET_DEV(accel_dev), "Failed to load MMP\n"); goto out_err; } - if (qat_uclo_map_obj(loader_data->fw_loader, uof_addr, uof_size)) { + + if (hw_device->uof_get_num_objs) + return adf_ae_fw_load_images(accel_dev, fw_addr, fw_size); + + if (qat_uclo_map_obj(loader_data->fw_loader, fw_addr, fw_size, NULL)) { dev_err(&GET_DEV(accel_dev), "Failed to map FW\n"); goto out_err; } @@ -61,7 +109,7 @@ void adf_ae_fw_release(struct adf_accel_dev *accel_dev) if (!hw_device->fw_name) return; - qat_uclo_del_uof_obj(loader_data->fw_loader); + qat_uclo_del_obj(loader_data->fw_loader); qat_hal_deinit(loader_data->fw_loader); release_firmware(loader_data->uof_fw); release_firmware(loader_data->mmp_fw); @@ -74,17 +122,12 @@ int adf_ae_start(struct adf_accel_dev *accel_dev) { struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; struct adf_hw_device_data *hw_data = accel_dev->hw_device; - u32 ae_ctr, ae, max_aes = GET_MAX_ACCELENGINES(accel_dev); + u32 ae_ctr; if (!hw_data->fw_name) return 0; - for (ae = 0, ae_ctr = 0; ae < max_aes; ae++) { - if (hw_data->ae_mask & (1 << ae)) { - qat_hal_start(loader_data->fw_loader, ae, 0xFF); - ae_ctr++; - } - } + ae_ctr = qat_hal_start(loader_data->fw_loader); dev_info(&GET_DEV(accel_dev), "qat_dev%d started %d acceleration engines\n", accel_dev->accel_id, ae_ctr); diff --git a/drivers/crypto/qat/qat_common/adf_admin.c b/drivers/crypto/qat/qat_common/adf_admin.c index ec9b390276d6..43680e178242 100644 --- a/drivers/crypto/qat/qat_common/adf_admin.c +++ b/drivers/crypto/qat/qat_common/adf_admin.c @@ -10,11 +10,7 @@ #include "adf_common_drv.h" #include "icp_qat_fw_init_admin.h" -/* Admin Messages Registers */ -#define ADF_DH895XCC_ADMINMSGUR_OFFSET (0x3A000 + 0x574) -#define ADF_DH895XCC_ADMINMSGLR_OFFSET (0x3A000 + 0x578) -#define ADF_DH895XCC_MAILBOX_BASE_OFFSET 0x20970 -#define ADF_DH895XCC_MAILBOX_STRIDE 0x1000 +#define ADF_ADMIN_MAILBOX_STRIDE 0x1000 #define ADF_ADMINMSG_LEN 32 #define ADF_CONST_TABLE_SIZE 1024 #define ADF_ADMIN_POLL_DELAY_US 20 @@ -70,28 +66,28 @@ static const u8 const_tab[1024] __aligned(1024) = { 0xf8, 0x2b, 0xa5, 0x4f, 0xf5, 0x3a, 0x5f, 0x1d, 0x36, 0xf1, 0x51, 0x0e, 0x52, 0x7f, 0xad, 0xe6, 0x82, 0xd1, 0x9b, 0x05, 0x68, 0x8c, 0x2b, 0x3e, 0x6c, 0x1f, 0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd, 0x6b, 0x5b, 0xe0, 0xcd, 0x19, 0x13, -0x7e, 0x21, 0x79, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x7e, 0x21, 0x79, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x01, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x15, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x02, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x14, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x02, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x24, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, +0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x45, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, 0x01, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x01, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x2B, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -118,7 +114,7 @@ static int adf_put_admin_msg_sync(struct adf_accel_dev *accel_dev, u32 ae, struct adf_admin_comms *admin = accel_dev->admin; int offset = ae * ADF_ADMINMSG_LEN * 2; void __iomem *mailbox = admin->mailbox_addr; - int mb_offset = ae * ADF_DH895XCC_MAILBOX_STRIDE; + int mb_offset = ae * ADF_ADMIN_MAILBOX_STRIDE; struct icp_qat_fw_init_admin_req *request = in; mutex_lock(&admin->lock); @@ -167,7 +163,7 @@ static int adf_send_admin(struct adf_accel_dev *accel_dev, return 0; } -static int adf_init_me(struct adf_accel_dev *accel_dev) +static int adf_init_ae(struct adf_accel_dev *accel_dev) { struct icp_qat_fw_init_admin_req req; struct icp_qat_fw_init_admin_resp resp; @@ -176,7 +172,7 @@ static int adf_init_me(struct adf_accel_dev *accel_dev) memset(&req, 0, sizeof(req)); memset(&resp, 0, sizeof(resp)); - req.cmd_id = ICP_QAT_FW_INIT_ME; + req.cmd_id = ICP_QAT_FW_INIT_AE; return adf_send_admin(accel_dev, &req, &resp, ae_mask); } @@ -186,7 +182,7 @@ static int adf_set_fw_constants(struct adf_accel_dev *accel_dev) struct icp_qat_fw_init_admin_req req; struct icp_qat_fw_init_admin_resp resp; struct adf_hw_device_data *hw_device = accel_dev->hw_device; - u32 ae_mask = hw_device->ae_mask; + u32 ae_mask = hw_device->admin_ae_mask ?: hw_device->ae_mask; memset(&req, 0, sizeof(req)); memset(&resp, 0, sizeof(resp)); @@ -210,11 +206,11 @@ int adf_send_admin_init(struct adf_accel_dev *accel_dev) { int ret; - ret = adf_init_me(accel_dev); + ret = adf_set_fw_constants(accel_dev); if (ret) return ret; - return adf_set_fw_constants(accel_dev); + return adf_init_ae(accel_dev); } EXPORT_SYMBOL_GPL(adf_send_admin_init); @@ -225,8 +221,9 @@ int adf_init_admin_comms(struct adf_accel_dev *accel_dev) struct adf_bar *pmisc = &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; void __iomem *csr = pmisc->virt_addr; - void __iomem *mailbox = (void __iomem *)((uintptr_t)csr + - ADF_DH895XCC_MAILBOX_BASE_OFFSET); + struct admin_info admin_csrs_info; + u32 mailbox_offset, adminmsg_u, adminmsg_l; + void __iomem *mailbox; u64 reg_val; admin = kzalloc_node(sizeof(*accel_dev->admin), GFP_KERNEL, @@ -254,9 +251,17 @@ int adf_init_admin_comms(struct adf_accel_dev *accel_dev) } memcpy(admin->virt_tbl_addr, const_tab, sizeof(const_tab)); + hw_data->get_admin_info(&admin_csrs_info); + + mailbox_offset = admin_csrs_info.mailbox_offset; + mailbox = csr + mailbox_offset; + adminmsg_u = admin_csrs_info.admin_msg_ur; + adminmsg_l = admin_csrs_info.admin_msg_lr; + reg_val = (u64)admin->phy_addr; - ADF_CSR_WR(csr, ADF_DH895XCC_ADMINMSGUR_OFFSET, reg_val >> 32); - ADF_CSR_WR(csr, ADF_DH895XCC_ADMINMSGLR_OFFSET, reg_val); + ADF_CSR_WR(csr, adminmsg_u, upper_32_bits(reg_val)); + ADF_CSR_WR(csr, adminmsg_l, lower_32_bits(reg_val)); + mutex_init(&admin->lock); admin->mailbox_addr = mailbox; accel_dev->admin = admin; diff --git a/drivers/crypto/qat/qat_common/adf_cfg.c b/drivers/crypto/qat/qat_common/adf_cfg.c index 22ae32838113..575b6f002303 100644 --- a/drivers/crypto/qat/qat_common/adf_cfg.c +++ b/drivers/crypto/qat/qat_common/adf_cfg.c @@ -196,7 +196,7 @@ static int adf_cfg_key_val_get(struct adf_accel_dev *accel_dev, memcpy(val, keyval->val, ADF_CFG_MAX_VAL_LEN_IN_BYTES); return 0; } - return -1; + return -ENODATA; } /** @@ -243,7 +243,7 @@ int adf_cfg_add_key_value_param(struct adf_accel_dev *accel_dev, } else { dev_err(&GET_DEV(accel_dev), "Unknown type given.\n"); kfree(key_val); - return -1; + return -EINVAL; } key_val->type = type; down_write(&cfg->lock); diff --git a/drivers/crypto/qat/qat_common/adf_cfg_common.h b/drivers/crypto/qat/qat_common/adf_cfg_common.h index 1ef46ccfba47..4fabb70b1f18 100644 --- a/drivers/crypto/qat/qat_common/adf_cfg_common.h +++ b/drivers/crypto/qat/qat_common/adf_cfg_common.h @@ -32,7 +32,8 @@ enum adf_device_type { DEV_C62X, DEV_C62XVF, DEV_C3XXX, - DEV_C3XXXVF + DEV_C3XXXVF, + DEV_4XXX, }; struct adf_dev_status_info { diff --git a/drivers/crypto/qat/qat_common/adf_cfg_strings.h b/drivers/crypto/qat/qat_common/adf_cfg_strings.h index 314790f5b0af..09651e1f937a 100644 --- a/drivers/crypto/qat/qat_common/adf_cfg_strings.h +++ b/drivers/crypto/qat/qat_common/adf_cfg_strings.h @@ -18,7 +18,8 @@ #define ADF_RING_DC_TX "RingTx" #define ADF_RING_DC_RX "RingRx" #define ADF_ETRMGR_BANK "Bank" -#define ADF_RING_BANK_NUM "BankNumber" +#define ADF_RING_SYM_BANK_NUM "BankSymNumber" +#define ADF_RING_ASYM_BANK_NUM "BankAsymNumber" #define ADF_CY "Cy" #define ADF_DC "Dc" #define ADF_ETRMGR_COALESCING_ENABLED "InterruptCoalescingEnabled" diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index f22342f612c1..c61476553728 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -133,8 +133,7 @@ void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev); int qat_hal_init(struct adf_accel_dev *accel_dev); void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle); -void qat_hal_start(struct icp_qat_fw_loader_handle *handle, unsigned char ae, - unsigned int ctx_mask); +int qat_hal_start(struct icp_qat_fw_loader_handle *handle); void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae, unsigned int ctx_mask); void qat_hal_reset(struct icp_qat_fw_loader_handle *handle); @@ -163,28 +162,32 @@ int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle, unsigned char ae, struct icp_qat_uof_batch_init *lm_init_header); int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx_mask, + unsigned char ae, unsigned long ctx_mask, enum icp_qat_uof_regtype reg_type, unsigned short reg_num, unsigned int regdata); int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx_mask, + unsigned char ae, unsigned long ctx_mask, enum icp_qat_uof_regtype reg_type, unsigned short reg_num, unsigned int regdata); int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx_mask, + unsigned char ae, unsigned long ctx_mask, enum icp_qat_uof_regtype reg_type, unsigned short reg_num, unsigned int regdata); int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx_mask, + unsigned char ae, unsigned long ctx_mask, unsigned short reg_num, unsigned int regdata); int qat_hal_wr_lm(struct icp_qat_fw_loader_handle *handle, unsigned char ae, unsigned short lm_addr, unsigned int value); +void qat_hal_set_ae_tindex_mode(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char mode); int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle); -void qat_uclo_del_uof_obj(struct icp_qat_fw_loader_handle *handle); +void qat_uclo_del_obj(struct icp_qat_fw_loader_handle *handle); int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, void *addr_ptr, int mem_size); int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle, - void *addr_ptr, int mem_size); + void *addr_ptr, u32 mem_size, char *obj_name); +int qat_uclo_set_cfg_ae_mask(struct icp_qat_fw_loader_handle *handle, + unsigned int cfg_ae_mask); #if defined(CONFIG_PCI_IOV) int adf_sriov_configure(struct pci_dev *pdev, int numvfs); void adf_disable_sriov(struct adf_accel_dev *accel_dev); diff --git a/drivers/crypto/qat/qat_common/adf_dev_mgr.c b/drivers/crypto/qat/qat_common/adf_dev_mgr.c index 92ec035576df..4c752eed10fe 100644 --- a/drivers/crypto/qat/qat_common/adf_dev_mgr.c +++ b/drivers/crypto/qat/qat_common/adf_dev_mgr.c @@ -151,8 +151,8 @@ int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev, mutex_lock(&table_lock); atomic_set(&accel_dev->ref_count, 0); - /* PF on host or VF on guest */ - if (!accel_dev->is_vf || (accel_dev->is_vf && !pf)) { + /* PF on host or VF on guest - optimized to remove redundant is_vf */ + if (!accel_dev->is_vf || !pf) { struct vf_id_map *map; list_for_each(itr, &accel_table) { @@ -248,7 +248,8 @@ void adf_devmgr_rm_dev(struct adf_accel_dev *accel_dev, struct adf_accel_dev *pf) { mutex_lock(&table_lock); - if (!accel_dev->is_vf || (accel_dev->is_vf && !pf)) { + /* PF on host or VF on guest - optimized to remove redundant is_vf */ + if (!accel_dev->is_vf || !pf) { id_map[accel_dev->accel_id] = 0; num_devices--; } else if (accel_dev->is_vf && pf) { @@ -285,9 +286,9 @@ struct adf_accel_dev *adf_devmgr_get_first(void) /** * adf_devmgr_pci_to_accel_dev() - Get accel_dev associated with the pci_dev. - * @pci_dev: Pointer to pci device. + * @pci_dev: Pointer to PCI device. * - * Function returns acceleration device associated with the given pci device. + * Function returns acceleration device associated with the given PCI device. * To be used by QAT device specific drivers. * * Return: pointer to accel_dev or NULL if not found. diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c new file mode 100644 index 000000000000..1aa17303838d --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2020 Intel Corporation */ +#include "adf_gen2_hw_data.h" +#include "icp_qat_hw.h" +#include + +void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, + int num_a_regs, int num_b_regs) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + void __iomem *pmisc_addr; + struct adf_bar *pmisc; + int pmisc_id, i; + u32 reg; + + pmisc_id = hw_data->get_misc_bar_id(hw_data); + pmisc = &GET_BARS(accel_dev)[pmisc_id]; + pmisc_addr = pmisc->virt_addr; + + /* Set/Unset Valid bit in AE Thread to PCIe Function Mapping Group A */ + for (i = 0; i < num_a_regs; i++) { + reg = READ_CSR_AE2FUNCTION_MAP_A(pmisc_addr, i); + if (enable) + reg |= AE2FUNCTION_MAP_VALID; + else + reg &= ~AE2FUNCTION_MAP_VALID; + WRITE_CSR_AE2FUNCTION_MAP_A(pmisc_addr, i, reg); + } + + /* Set/Unset Valid bit in AE Thread to PCIe Function Mapping Group B */ + for (i = 0; i < num_b_regs; i++) { + reg = READ_CSR_AE2FUNCTION_MAP_B(pmisc_addr, i); + if (enable) + reg |= AE2FUNCTION_MAP_VALID; + else + reg &= ~AE2FUNCTION_MAP_VALID; + WRITE_CSR_AE2FUNCTION_MAP_B(pmisc_addr, i, reg); + } +} +EXPORT_SYMBOL_GPL(adf_gen2_cfg_iov_thds); + +void adf_gen2_get_admin_info(struct admin_info *admin_csrs_info) +{ + admin_csrs_info->mailbox_offset = ADF_MAILBOX_BASE_OFFSET; + admin_csrs_info->admin_msg_ur = ADF_ADMINMSGUR_OFFSET; + admin_csrs_info->admin_msg_lr = ADF_ADMINMSGLR_OFFSET; +} +EXPORT_SYMBOL_GPL(adf_gen2_get_admin_info); + +void adf_gen2_get_arb_info(struct arb_info *arb_info) +{ + arb_info->arb_cfg = ADF_ARB_CONFIG; + arb_info->arb_offset = ADF_ARB_OFFSET; + arb_info->wt2sam_offset = ADF_ARB_WRK_2_SER_MAP_OFFSET; +} +EXPORT_SYMBOL_GPL(adf_gen2_get_arb_info); + +static u64 build_csr_ring_base_addr(dma_addr_t addr, u32 size) +{ + return BUILD_RING_BASE_ADDR(addr, size); +} + +static u32 read_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring) +{ + return READ_CSR_RING_HEAD(csr_base_addr, bank, ring); +} + +static void write_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring, + u32 value) +{ + WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value); +} + +static u32 read_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring) +{ + return READ_CSR_RING_TAIL(csr_base_addr, bank, ring); +} + +static void write_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring, + u32 value) +{ + WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value); +} + +static u32 read_csr_e_stat(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_E_STAT(csr_base_addr, bank); +} + +static void write_csr_ring_config(void __iomem *csr_base_addr, u32 bank, + u32 ring, u32 value) +{ + WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value); +} + +static void write_csr_ring_base(void __iomem *csr_base_addr, u32 bank, u32 ring, + dma_addr_t addr) +{ + WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, addr); +} + +static void write_csr_int_flag(void __iomem *csr_base_addr, u32 bank, u32 value) +{ + WRITE_CSR_INT_FLAG(csr_base_addr, bank, value); +} + +static void write_csr_int_srcsel(void __iomem *csr_base_addr, u32 bank) +{ + WRITE_CSR_INT_SRCSEL(csr_base_addr, bank); +} + +static void write_csr_int_col_en(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value); +} + +static void write_csr_int_col_ctl(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value); +} + +static void write_csr_int_flag_and_col(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value); +} + +static void write_csr_ring_srv_arb_en(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value); +} + +void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops) +{ + csr_ops->build_csr_ring_base_addr = build_csr_ring_base_addr; + csr_ops->read_csr_ring_head = read_csr_ring_head; + csr_ops->write_csr_ring_head = write_csr_ring_head; + csr_ops->read_csr_ring_tail = read_csr_ring_tail; + csr_ops->write_csr_ring_tail = write_csr_ring_tail; + csr_ops->read_csr_e_stat = read_csr_e_stat; + csr_ops->write_csr_ring_config = write_csr_ring_config; + csr_ops->write_csr_ring_base = write_csr_ring_base; + csr_ops->write_csr_int_flag = write_csr_int_flag; + csr_ops->write_csr_int_srcsel = write_csr_int_srcsel; + csr_ops->write_csr_int_col_en = write_csr_int_col_en; + csr_ops->write_csr_int_col_ctl = write_csr_int_col_ctl; + csr_ops->write_csr_int_flag_and_col = write_csr_int_flag_and_col; + csr_ops->write_csr_ring_srv_arb_en = write_csr_ring_srv_arb_en; +} +EXPORT_SYMBOL_GPL(adf_gen2_init_hw_csr_ops); + +u32 adf_gen2_get_accel_cap(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct pci_dev *pdev = accel_dev->accel_pci_dev.pci_dev; + u32 straps = hw_data->straps; + u32 fuses = hw_data->fuses; + u32 legfuses; + u32 capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | + ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | + ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + + /* Read accelerator capabilities mask */ + pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET, &legfuses); + + if (legfuses & ICP_ACCEL_MASK_CIPHER_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; + if (legfuses & ICP_ACCEL_MASK_PKE_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; + if (legfuses & ICP_ACCEL_MASK_AUTH_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + + if ((straps | fuses) & ADF_POWERGATE_PKE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; + + return capabilities; +} +EXPORT_SYMBOL_GPL(adf_gen2_get_accel_cap); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h new file mode 100644 index 000000000000..3816e6500352 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h @@ -0,0 +1,123 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2020 Intel Corporation */ +#ifndef ADF_GEN2_HW_DATA_H_ +#define ADF_GEN2_HW_DATA_H_ + +#include "adf_accel_devices.h" + +/* Transport access */ +#define ADF_BANK_INT_SRC_SEL_MASK_0 0x4444444CUL +#define ADF_BANK_INT_SRC_SEL_MASK_X 0x44444444UL +#define ADF_RING_CSR_RING_CONFIG 0x000 +#define ADF_RING_CSR_RING_LBASE 0x040 +#define ADF_RING_CSR_RING_UBASE 0x080 +#define ADF_RING_CSR_RING_HEAD 0x0C0 +#define ADF_RING_CSR_RING_TAIL 0x100 +#define ADF_RING_CSR_E_STAT 0x14C +#define ADF_RING_CSR_INT_FLAG 0x170 +#define ADF_RING_CSR_INT_SRCSEL 0x174 +#define ADF_RING_CSR_INT_SRCSEL_2 0x178 +#define ADF_RING_CSR_INT_COL_EN 0x17C +#define ADF_RING_CSR_INT_COL_CTL 0x180 +#define ADF_RING_CSR_INT_FLAG_AND_COL 0x184 +#define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000 +#define ADF_RING_BUNDLE_SIZE 0x1000 + +#define BUILD_RING_BASE_ADDR(addr, size) \ + (((addr) >> 6) & (GENMASK_ULL(63, 0) << (size))) +#define READ_CSR_RING_HEAD(csr_base_addr, bank, ring) \ + ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_HEAD + ((ring) << 2)) +#define READ_CSR_RING_TAIL(csr_base_addr, bank, ring) \ + ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_TAIL + ((ring) << 2)) +#define READ_CSR_E_STAT(csr_base_addr, bank) \ + ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_E_STAT) +#define WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_CONFIG + ((ring) << 2), value) +#define WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, value) \ +do { \ + u32 l_base = 0, u_base = 0; \ + l_base = (u32)((value) & 0xFFFFFFFF); \ + u_base = (u32)(((value) & 0xFFFFFFFF00000000ULL) >> 32); \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_LBASE + ((ring) << 2), l_base); \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_UBASE + ((ring) << 2), u_base); \ +} while (0) + +#define WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_HEAD + ((ring) << 2), value) +#define WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_TAIL + ((ring) << 2), value) +#define WRITE_CSR_INT_FLAG(csr_base_addr, bank, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_FLAG, value) +#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank) \ +do { \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_SRCSEL, ADF_BANK_INT_SRC_SEL_MASK_0); \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_SRCSEL_2, ADF_BANK_INT_SRC_SEL_MASK_X); \ +} while (0) +#define WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_COL_EN, value) +#define WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_COL_CTL, \ + ADF_RING_CSR_INT_COL_CTL_ENABLE | (value)) +#define WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_FLAG_AND_COL, value) + +/* AE to function map */ +#define AE2FUNCTION_MAP_A_OFFSET (0x3A400 + 0x190) +#define AE2FUNCTION_MAP_B_OFFSET (0x3A400 + 0x310) +#define AE2FUNCTION_MAP_REG_SIZE 4 +#define AE2FUNCTION_MAP_VALID BIT(7) + +#define READ_CSR_AE2FUNCTION_MAP_A(pmisc_bar_addr, index) \ + ADF_CSR_RD(pmisc_bar_addr, AE2FUNCTION_MAP_A_OFFSET + \ + AE2FUNCTION_MAP_REG_SIZE * (index)) +#define WRITE_CSR_AE2FUNCTION_MAP_A(pmisc_bar_addr, index, value) \ + ADF_CSR_WR(pmisc_bar_addr, AE2FUNCTION_MAP_A_OFFSET + \ + AE2FUNCTION_MAP_REG_SIZE * (index), value) +#define READ_CSR_AE2FUNCTION_MAP_B(pmisc_bar_addr, index) \ + ADF_CSR_RD(pmisc_bar_addr, AE2FUNCTION_MAP_B_OFFSET + \ + AE2FUNCTION_MAP_REG_SIZE * (index)) +#define WRITE_CSR_AE2FUNCTION_MAP_B(pmisc_bar_addr, index, value) \ + ADF_CSR_WR(pmisc_bar_addr, AE2FUNCTION_MAP_B_OFFSET + \ + AE2FUNCTION_MAP_REG_SIZE * (index), value) + +/* Admin Interface Offsets */ +#define ADF_ADMINMSGUR_OFFSET (0x3A000 + 0x574) +#define ADF_ADMINMSGLR_OFFSET (0x3A000 + 0x578) +#define ADF_MAILBOX_BASE_OFFSET 0x20970 + +/* Arbiter configuration */ +#define ADF_ARB_OFFSET 0x30000 +#define ADF_ARB_WRK_2_SER_MAP_OFFSET 0x180 +#define ADF_ARB_CONFIG (BIT(31) | BIT(6) | BIT(0)) +#define ADF_ARB_REG_SLOT 0x1000 +#define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C + +#define WRITE_CSR_RING_SRV_ARB_EN(csr_addr, index, value) \ + ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \ + (ADF_ARB_REG_SLOT * (index)), value) + +/* Power gating */ +#define ADF_POWERGATE_PKE BIT(24) + +void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, + int num_a_regs, int num_b_regs); +void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); +void adf_gen2_get_admin_info(struct admin_info *admin_csrs_info); +void adf_gen2_get_arb_info(struct arb_info *arb_info); +u32 adf_gen2_get_accel_cap(struct adf_accel_dev *accel_dev); + +#endif diff --git a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c new file mode 100644 index 000000000000..b72ff58e0bc7 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2020 Intel Corporation */ +#include "adf_accel_devices.h" +#include "adf_gen4_hw_data.h" + +static u64 build_csr_ring_base_addr(dma_addr_t addr, u32 size) +{ + return BUILD_RING_BASE_ADDR(addr, size); +} + +static u32 read_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring) +{ + return READ_CSR_RING_HEAD(csr_base_addr, bank, ring); +} + +static void write_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring, + u32 value) +{ + WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value); +} + +static u32 read_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring) +{ + return READ_CSR_RING_TAIL(csr_base_addr, bank, ring); +} + +static void write_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring, + u32 value) +{ + WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value); +} + +static u32 read_csr_e_stat(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_E_STAT(csr_base_addr, bank); +} + +static void write_csr_ring_config(void __iomem *csr_base_addr, u32 bank, u32 ring, + u32 value) +{ + WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value); +} + +static void write_csr_ring_base(void __iomem *csr_base_addr, u32 bank, u32 ring, + dma_addr_t addr) +{ + WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, addr); +} + +static void write_csr_int_flag(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_FLAG(csr_base_addr, bank, value); +} + +static void write_csr_int_srcsel(void __iomem *csr_base_addr, u32 bank) +{ + WRITE_CSR_INT_SRCSEL(csr_base_addr, bank); +} + +static void write_csr_int_col_en(void __iomem *csr_base_addr, u32 bank, u32 value) +{ + WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value); +} + +static void write_csr_int_col_ctl(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value); +} + +static void write_csr_int_flag_and_col(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value); +} + +static void write_csr_ring_srv_arb_en(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value); +} + +void adf_gen4_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops) +{ + csr_ops->build_csr_ring_base_addr = build_csr_ring_base_addr; + csr_ops->read_csr_ring_head = read_csr_ring_head; + csr_ops->write_csr_ring_head = write_csr_ring_head; + csr_ops->read_csr_ring_tail = read_csr_ring_tail; + csr_ops->write_csr_ring_tail = write_csr_ring_tail; + csr_ops->read_csr_e_stat = read_csr_e_stat; + csr_ops->write_csr_ring_config = write_csr_ring_config; + csr_ops->write_csr_ring_base = write_csr_ring_base; + csr_ops->write_csr_int_flag = write_csr_int_flag; + csr_ops->write_csr_int_srcsel = write_csr_int_srcsel; + csr_ops->write_csr_int_col_en = write_csr_int_col_en; + csr_ops->write_csr_int_col_ctl = write_csr_int_col_ctl; + csr_ops->write_csr_int_flag_and_col = write_csr_int_flag_and_col; + csr_ops->write_csr_ring_srv_arb_en = write_csr_ring_srv_arb_en; +} +EXPORT_SYMBOL_GPL(adf_gen4_init_hw_csr_ops); diff --git a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h new file mode 100644 index 000000000000..8ab62b2ac311 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2020 Intel Corporation */ +#ifndef ADF_GEN4_HW_CSR_DATA_H_ +#define ADF_GEN4_HW_CSR_DATA_H_ + +#include "adf_accel_devices.h" + +/* Transport access */ +#define ADF_BANK_INT_SRC_SEL_MASK 0x44UL +#define ADF_RING_CSR_RING_CONFIG 0x1000 +#define ADF_RING_CSR_RING_LBASE 0x1040 +#define ADF_RING_CSR_RING_UBASE 0x1080 +#define ADF_RING_CSR_RING_HEAD 0x0C0 +#define ADF_RING_CSR_RING_TAIL 0x100 +#define ADF_RING_CSR_E_STAT 0x14C +#define ADF_RING_CSR_INT_FLAG 0x170 +#define ADF_RING_CSR_INT_SRCSEL 0x174 +#define ADF_RING_CSR_INT_COL_CTL 0x180 +#define ADF_RING_CSR_INT_FLAG_AND_COL 0x184 +#define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000 +#define ADF_RING_CSR_INT_COL_EN 0x17C +#define ADF_RING_CSR_ADDR_OFFSET 0x100000 +#define ADF_RING_BUNDLE_SIZE 0x2000 + +#define BUILD_RING_BASE_ADDR(addr, size) \ + ((((addr) >> 6) & (GENMASK_ULL(63, 0) << (size))) << 6) +#define READ_CSR_RING_HEAD(csr_base_addr, bank, ring) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_HEAD + ((ring) << 2)) +#define READ_CSR_RING_TAIL(csr_base_addr, bank, ring) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_TAIL + ((ring) << 2)) +#define READ_CSR_E_STAT(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_E_STAT) +#define WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_CONFIG + ((ring) << 2), value) +#define WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, value) \ +do { \ + void __iomem *_csr_base_addr = csr_base_addr; \ + u32 _bank = bank; \ + u32 _ring = ring; \ + dma_addr_t _value = value; \ + u32 l_base = 0, u_base = 0; \ + l_base = lower_32_bits(_value); \ + u_base = upper_32_bits(_value); \ + ADF_CSR_WR((_csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (_bank) + \ + ADF_RING_CSR_RING_LBASE + ((_ring) << 2), l_base); \ + ADF_CSR_WR((_csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (_bank) + \ + ADF_RING_CSR_RING_UBASE + ((_ring) << 2), u_base); \ +} while (0) + +#define WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_HEAD + ((ring) << 2), value) +#define WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_TAIL + ((ring) << 2), value) +#define WRITE_CSR_INT_FLAG(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_FLAG, (value)) +#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_SRCSEL, ADF_BANK_INT_SRC_SEL_MASK) +#define WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_COL_EN, (value)) +#define WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_COL_CTL, \ + ADF_RING_CSR_INT_COL_CTL_ENABLE | (value)) +#define WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_FLAG_AND_COL, (value)) + +/* Arbiter configuration */ +#define ADF_RING_CSR_RING_SRV_ARB_EN 0x19C + +#define WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_SRV_ARB_EN, (value)) + +void adf_gen4_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); + +#endif diff --git a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c index d4162783f970..9f5240d9488b 100644 --- a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c +++ b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c @@ -6,48 +6,33 @@ #define ADF_ARB_NUM 4 #define ADF_ARB_REG_SIZE 0x4 -#define ADF_ARB_WTR_SIZE 0x20 -#define ADF_ARB_OFFSET 0x30000 -#define ADF_ARB_REG_SLOT 0x1000 -#define ADF_ARB_WTR_OFFSET 0x010 -#define ADF_ARB_RO_EN_OFFSET 0x090 -#define ADF_ARB_WQCFG_OFFSET 0x100 -#define ADF_ARB_WRK_2_SER_MAP_OFFSET 0x180 -#define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C -#define WRITE_CSR_ARB_RINGSRVARBEN(csr_addr, index, value) \ - ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \ - (ADF_ARB_REG_SLOT * index), value) +#define WRITE_CSR_ARB_SARCONFIG(csr_addr, arb_offset, index, value) \ + ADF_CSR_WR(csr_addr, (arb_offset) + \ + (ADF_ARB_REG_SIZE * (index)), value) -#define WRITE_CSR_ARB_SARCONFIG(csr_addr, index, value) \ - ADF_CSR_WR(csr_addr, ADF_ARB_OFFSET + \ - (ADF_ARB_REG_SIZE * index), value) - -#define WRITE_CSR_ARB_WRK_2_SER_MAP(csr_addr, index, value) \ - ADF_CSR_WR(csr_addr, (ADF_ARB_OFFSET + \ - ADF_ARB_WRK_2_SER_MAP_OFFSET) + \ - (ADF_ARB_REG_SIZE * index), value) - -#define WRITE_CSR_ARB_WQCFG(csr_addr, index, value) \ - ADF_CSR_WR(csr_addr, (ADF_ARB_OFFSET + \ - ADF_ARB_WQCFG_OFFSET) + (ADF_ARB_REG_SIZE * index), value) +#define WRITE_CSR_ARB_WT2SAM(csr_addr, arb_offset, wt_offset, index, value) \ + ADF_CSR_WR(csr_addr, ((arb_offset) + (wt_offset)) + \ + (ADF_ARB_REG_SIZE * (index)), value) int adf_init_arb(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; void __iomem *csr = accel_dev->transport->banks[0].csr_addr; - u32 arb_cfg = 0x1 << 31 | 0x4 << 4 | 0x1; - u32 arb, i; + u32 arb_off, wt_off, arb_cfg; const u32 *thd_2_arb_cfg; + struct arb_info info; + int arb, i; + + hw_data->get_arb_info(&info); + arb_cfg = info.arb_cfg; + arb_off = info.arb_offset; + wt_off = info.wt2sam_offset; /* Service arb configured for 32 bytes responses and * ring flow control check enabled. */ for (arb = 0; arb < ADF_ARB_NUM; arb++) - WRITE_CSR_ARB_SARCONFIG(csr, arb, arb_cfg); - - /* Setup worker queue registers */ - for (i = 0; i < hw_data->num_engines; i++) - WRITE_CSR_ARB_WQCFG(csr, i, i); + WRITE_CSR_ARB_SARCONFIG(csr, arb_off, arb, arb_cfg); /* Map worker threads to service arbiters */ hw_data->get_arb_mapping(accel_dev, &thd_2_arb_cfg); @@ -56,7 +41,7 @@ int adf_init_arb(struct adf_accel_dev *accel_dev) return -EFAULT; for (i = 0; i < hw_data->num_engines; i++) - WRITE_CSR_ARB_WRK_2_SER_MAP(csr, i, *(thd_2_arb_cfg + i)); + WRITE_CSR_ARB_WT2SAM(csr, arb_off, wt_off, i, thd_2_arb_cfg[i]); return 0; } @@ -64,36 +49,59 @@ EXPORT_SYMBOL_GPL(adf_init_arb); void adf_update_ring_arb(struct adf_etr_ring_data *ring) { - WRITE_CSR_ARB_RINGSRVARBEN(ring->bank->csr_addr, - ring->bank->bank_number, - ring->bank->ring_mask & 0xFF); + struct adf_accel_dev *accel_dev = ring->bank->accel_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); + u32 tx_ring_mask = hw_data->tx_rings_mask; + u32 shift = hw_data->tx_rx_gap; + u32 arben, arben_tx, arben_rx; + u32 rx_ring_mask; + + /* + * Enable arbitration on a ring only if the TX half of the ring mask + * matches the RX part. This results in writes to CSR on both TX and + * RX update - only one is necessary, but both are done for + * simplicity. + */ + rx_ring_mask = tx_ring_mask << shift; + arben_tx = (ring->bank->ring_mask & tx_ring_mask) >> 0; + arben_rx = (ring->bank->ring_mask & rx_ring_mask) >> shift; + arben = arben_tx & arben_rx; + + csr_ops->write_csr_ring_srv_arb_en(ring->bank->csr_addr, + ring->bank->bank_number, arben); } void adf_exit_arb(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); + u32 arb_off, wt_off; + struct arb_info info; void __iomem *csr; unsigned int i; + hw_data->get_arb_info(&info); + arb_off = info.arb_offset; + wt_off = info.wt2sam_offset; + if (!accel_dev->transport) return; csr = accel_dev->transport->banks[0].csr_addr; + hw_data->get_arb_info(&info); + /* Reset arbiter configuration */ for (i = 0; i < ADF_ARB_NUM; i++) - WRITE_CSR_ARB_SARCONFIG(csr, i, 0); - - /* Shutdown work queue */ - for (i = 0; i < hw_data->num_engines; i++) - WRITE_CSR_ARB_WQCFG(csr, i, 0); + WRITE_CSR_ARB_SARCONFIG(csr, arb_off, i, 0); /* Unmap worker threads to service arbiters */ for (i = 0; i < hw_data->num_engines; i++) - WRITE_CSR_ARB_WRK_2_SER_MAP(csr, i, 0); + WRITE_CSR_ARB_WT2SAM(csr, arb_off, wt_off, i, 0); /* Disable arbitration on all rings */ for (i = 0; i < GET_MAX_BANKS(accel_dev); i++) - WRITE_CSR_ARB_RINGSRVARBEN(csr, i, 0); + csr_ops->write_csr_ring_srv_arb_en(csr, i, 0); } EXPORT_SYMBOL_GPL(adf_exit_arb); diff --git a/drivers/crypto/qat/qat_common/adf_isr.c b/drivers/crypto/qat/qat_common/adf_isr.c index 36136f7db509..c45853463530 100644 --- a/drivers/crypto/qat/qat_common/adf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_isr.c @@ -21,6 +21,9 @@ static int adf_enable_msix(struct adf_accel_dev *accel_dev) struct adf_hw_device_data *hw_data = accel_dev->hw_device; u32 msix_num_entries = 1; + if (hw_data->set_msix_rttable) + hw_data->set_msix_rttable(accel_dev); + /* If SR-IOV is disabled, add entries for each bank */ if (!accel_dev->pf.vf_info) { int i; @@ -50,8 +53,10 @@ static void adf_disable_msix(struct adf_accel_pci *pci_dev_info) static irqreturn_t adf_msix_isr_bundle(int irq, void *bank_ptr) { struct adf_etr_bank_data *bank = bank_ptr; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); - WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr, bank->bank_number, 0); + csr_ops->write_csr_int_flag_and_col(bank->csr_addr, bank->bank_number, + 0); tasklet_hi_schedule(&bank->resp_handler); return IRQ_HANDLED; } diff --git a/drivers/crypto/qat/qat_common/adf_sriov.c b/drivers/crypto/qat/qat_common/adf_sriov.c index 963b2bea78f2..8c822c2861c2 100644 --- a/drivers/crypto/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/qat/qat_common/adf_sriov.c @@ -10,31 +10,6 @@ static struct workqueue_struct *pf2vf_resp_wq; -#define ME2FUNCTION_MAP_A_OFFSET (0x3A400 + 0x190) -#define ME2FUNCTION_MAP_A_NUM_REGS 96 - -#define ME2FUNCTION_MAP_B_OFFSET (0x3A400 + 0x310) -#define ME2FUNCTION_MAP_B_NUM_REGS 12 - -#define ME2FUNCTION_MAP_REG_SIZE 4 -#define ME2FUNCTION_MAP_VALID BIT(7) - -#define READ_CSR_ME2FUNCTION_MAP_A(pmisc_bar_addr, index) \ - ADF_CSR_RD(pmisc_bar_addr, ME2FUNCTION_MAP_A_OFFSET + \ - ME2FUNCTION_MAP_REG_SIZE * index) - -#define WRITE_CSR_ME2FUNCTION_MAP_A(pmisc_bar_addr, index, value) \ - ADF_CSR_WR(pmisc_bar_addr, ME2FUNCTION_MAP_A_OFFSET + \ - ME2FUNCTION_MAP_REG_SIZE * index, value) - -#define READ_CSR_ME2FUNCTION_MAP_B(pmisc_bar_addr, index) \ - ADF_CSR_RD(pmisc_bar_addr, ME2FUNCTION_MAP_B_OFFSET + \ - ME2FUNCTION_MAP_REG_SIZE * index) - -#define WRITE_CSR_ME2FUNCTION_MAP_B(pmisc_bar_addr, index, value) \ - ADF_CSR_WR(pmisc_bar_addr, ME2FUNCTION_MAP_B_OFFSET + \ - ME2FUNCTION_MAP_REG_SIZE * index, value) - struct adf_pf2vf_resp { struct work_struct pf2vf_resp_work; struct adf_accel_vf_info *vf_info; @@ -68,12 +43,8 @@ static int adf_enable_sriov(struct adf_accel_dev *accel_dev) struct pci_dev *pdev = accel_to_pci_dev(accel_dev); int totalvfs = pci_sriov_get_totalvfs(pdev); struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct adf_bar *pmisc = - &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; - void __iomem *pmisc_addr = pmisc->virt_addr; struct adf_accel_vf_info *vf_info; int i; - u32 reg; for (i = 0, vf_info = accel_dev->pf.vf_info; i < totalvfs; i++, vf_info++) { @@ -90,22 +61,13 @@ static int adf_enable_sriov(struct adf_accel_dev *accel_dev) DEFAULT_RATELIMIT_BURST); } - /* Set Valid bits in ME Thread to PCIe Function Mapping Group A */ - for (i = 0; i < ME2FUNCTION_MAP_A_NUM_REGS; i++) { - reg = READ_CSR_ME2FUNCTION_MAP_A(pmisc_addr, i); - reg |= ME2FUNCTION_MAP_VALID; - WRITE_CSR_ME2FUNCTION_MAP_A(pmisc_addr, i, reg); - } - - /* Set Valid bits in ME Thread to PCIe Function Mapping Group B */ - for (i = 0; i < ME2FUNCTION_MAP_B_NUM_REGS; i++) { - reg = READ_CSR_ME2FUNCTION_MAP_B(pmisc_addr, i); - reg |= ME2FUNCTION_MAP_VALID; - WRITE_CSR_ME2FUNCTION_MAP_B(pmisc_addr, i, reg); - } + /* Set Valid bits in AE Thread to PCIe Function Mapping */ + if (hw_data->configure_iov_threads) + hw_data->configure_iov_threads(accel_dev, true); /* Enable VF to PF interrupts for all VFs */ - adf_enable_vf2pf_interrupts(accel_dev, GENMASK_ULL(totalvfs - 1, 0)); + if (hw_data->get_pf2vf_offset) + adf_enable_vf2pf_interrupts(accel_dev, BIT_ULL(totalvfs) - 1); /* * Due to the hardware design, when SR-IOV and the ring arbiter @@ -127,37 +89,25 @@ static int adf_enable_sriov(struct adf_accel_dev *accel_dev) void adf_disable_sriov(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct adf_bar *pmisc = - &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; - void __iomem *pmisc_addr = pmisc->virt_addr; int totalvfs = pci_sriov_get_totalvfs(accel_to_pci_dev(accel_dev)); struct adf_accel_vf_info *vf; - u32 reg; int i; if (!accel_dev->pf.vf_info) return; - adf_pf2vf_notify_restarting(accel_dev); + if (hw_data->get_pf2vf_offset) + adf_pf2vf_notify_restarting(accel_dev); pci_disable_sriov(accel_to_pci_dev(accel_dev)); /* Disable VF to PF interrupts */ - adf_disable_vf2pf_interrupts(accel_dev, 0xFFFFFFFF); + if (hw_data->get_pf2vf_offset) + adf_disable_vf2pf_interrupts(accel_dev, GENMASK(31, 0)); - /* Clear Valid bits in ME Thread to PCIe Function Mapping Group A */ - for (i = 0; i < ME2FUNCTION_MAP_A_NUM_REGS; i++) { - reg = READ_CSR_ME2FUNCTION_MAP_A(pmisc_addr, i); - reg &= ~ME2FUNCTION_MAP_VALID; - WRITE_CSR_ME2FUNCTION_MAP_A(pmisc_addr, i, reg); - } - - /* Clear Valid bits in ME Thread to PCIe Function Mapping Group B */ - for (i = 0; i < ME2FUNCTION_MAP_B_NUM_REGS; i++) { - reg = READ_CSR_ME2FUNCTION_MAP_B(pmisc_addr, i); - reg &= ~ME2FUNCTION_MAP_VALID; - WRITE_CSR_ME2FUNCTION_MAP_B(pmisc_addr, i, reg); - } + /* Clear Valid bits in AE Thread to PCIe Function Mapping */ + if (hw_data->configure_iov_threads) + hw_data->configure_iov_threads(accel_dev, false); for (i = 0, vf = accel_dev->pf.vf_info; i < totalvfs; i++, vf++) { tasklet_disable(&vf->vf2pf_bh_tasklet); @@ -172,13 +122,13 @@ EXPORT_SYMBOL_GPL(adf_disable_sriov); /** * adf_sriov_configure() - Enable SRIOV for the device - * @pdev: Pointer to pci device. + * @pdev: Pointer to PCI device. * @numvfs: Number of virtual functions (VFs) to enable. * * Note that the @numvfs parameter is ignored and all VFs supported by the * device are enabled due to the design of the hardware. * - * Function enables SRIOV for the pci device. + * Function enables SRIOV for the PCI device. * * Return: number of VFs enabled on success, error code otherwise. */ diff --git a/drivers/crypto/qat/qat_common/adf_transport.c b/drivers/crypto/qat/qat_common/adf_transport.c index 2ad774017200..5a7030acdc33 100644 --- a/drivers/crypto/qat/qat_common/adf_transport.c +++ b/drivers/crypto/qat/qat_common/adf_transport.c @@ -54,24 +54,32 @@ static void adf_unreserve_ring(struct adf_etr_bank_data *bank, u32 ring) static void adf_enable_ring_irq(struct adf_etr_bank_data *bank, u32 ring) { + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); + spin_lock_bh(&bank->lock); bank->irq_mask |= (1 << ring); spin_unlock_bh(&bank->lock); - WRITE_CSR_INT_COL_EN(bank->csr_addr, bank->bank_number, bank->irq_mask); - WRITE_CSR_INT_COL_CTL(bank->csr_addr, bank->bank_number, - bank->irq_coalesc_timer); + csr_ops->write_csr_int_col_en(bank->csr_addr, bank->bank_number, + bank->irq_mask); + csr_ops->write_csr_int_col_ctl(bank->csr_addr, bank->bank_number, + bank->irq_coalesc_timer); } static void adf_disable_ring_irq(struct adf_etr_bank_data *bank, u32 ring) { + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); + spin_lock_bh(&bank->lock); bank->irq_mask &= ~(1 << ring); spin_unlock_bh(&bank->lock); - WRITE_CSR_INT_COL_EN(bank->csr_addr, bank->bank_number, bank->irq_mask); + csr_ops->write_csr_int_col_en(bank->csr_addr, bank->bank_number, + bank->irq_mask); } int adf_send_message(struct adf_etr_ring_data *ring, u32 *msg) { + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); + if (atomic_add_return(1, ring->inflights) > ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size)) { atomic_dec(ring->inflights); @@ -84,14 +92,17 @@ int adf_send_message(struct adf_etr_ring_data *ring, u32 *msg) ring->tail = adf_modulo(ring->tail + ADF_MSG_SIZE_TO_BYTES(ring->msg_size), ADF_RING_SIZE_MODULO(ring->ring_size)); - WRITE_CSR_RING_TAIL(ring->bank->csr_addr, ring->bank->bank_number, - ring->ring_number, ring->tail); + csr_ops->write_csr_ring_tail(ring->bank->csr_addr, + ring->bank->bank_number, ring->ring_number, + ring->tail); spin_unlock_bh(&ring->lock); + return 0; } static int adf_handle_response(struct adf_etr_ring_data *ring) { + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); u32 msg_counter = 0; u32 *msg = (u32 *)((uintptr_t)ring->base_addr + ring->head); @@ -105,30 +116,36 @@ static int adf_handle_response(struct adf_etr_ring_data *ring) msg_counter++; msg = (u32 *)((uintptr_t)ring->base_addr + ring->head); } - if (msg_counter > 0) - WRITE_CSR_RING_HEAD(ring->bank->csr_addr, - ring->bank->bank_number, - ring->ring_number, ring->head); + if (msg_counter > 0) { + csr_ops->write_csr_ring_head(ring->bank->csr_addr, + ring->bank->bank_number, + ring->ring_number, ring->head); + } return 0; } static void adf_configure_tx_ring(struct adf_etr_ring_data *ring) { + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); u32 ring_config = BUILD_RING_CONFIG(ring->ring_size); - WRITE_CSR_RING_CONFIG(ring->bank->csr_addr, ring->bank->bank_number, - ring->ring_number, ring_config); + csr_ops->write_csr_ring_config(ring->bank->csr_addr, + ring->bank->bank_number, + ring->ring_number, ring_config); + } static void adf_configure_rx_ring(struct adf_etr_ring_data *ring) { + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); u32 ring_config = BUILD_RESP_RING_CONFIG(ring->ring_size, ADF_RING_NEAR_WATERMARK_512, ADF_RING_NEAR_WATERMARK_0); - WRITE_CSR_RING_CONFIG(ring->bank->csr_addr, ring->bank->bank_number, - ring->ring_number, ring_config); + csr_ops->write_csr_ring_config(ring->bank->csr_addr, + ring->bank->bank_number, + ring->ring_number, ring_config); } static int adf_init_ring(struct adf_etr_ring_data *ring) @@ -136,6 +153,7 @@ static int adf_init_ring(struct adf_etr_ring_data *ring) struct adf_etr_bank_data *bank = ring->bank; struct adf_accel_dev *accel_dev = bank->accel_dev; struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); u64 ring_base; u32 ring_size_bytes = ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size); @@ -162,9 +180,12 @@ static int adf_init_ring(struct adf_etr_ring_data *ring) else adf_configure_rx_ring(ring); - ring_base = BUILD_RING_BASE_ADDR(ring->dma_addr, ring->ring_size); - WRITE_CSR_RING_BASE(ring->bank->csr_addr, ring->bank->bank_number, - ring->ring_number, ring_base); + ring_base = csr_ops->build_csr_ring_base_addr(ring->dma_addr, + ring->ring_size); + + csr_ops->write_csr_ring_base(ring->bank->csr_addr, + ring->bank->bank_number, ring->ring_number, + ring_base); spin_lock_init(&ring->lock); return 0; } @@ -190,6 +211,7 @@ int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section, struct adf_etr_ring_data **ring_ptr) { struct adf_etr_data *transport_data = accel_dev->transport; + u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(accel_dev); struct adf_etr_bank_data *bank; struct adf_etr_ring_data *ring; char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; @@ -219,7 +241,7 @@ int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section, dev_err(&GET_DEV(accel_dev), "Can't get ring number\n"); return -EFAULT; } - if (ring_num >= ADF_ETR_MAX_RINGS_PER_BANK) { + if (ring_num >= num_rings_per_bank) { dev_err(&GET_DEV(accel_dev), "Invalid ring number\n"); return -EFAULT; } @@ -268,15 +290,17 @@ err: void adf_remove_ring(struct adf_etr_ring_data *ring) { struct adf_etr_bank_data *bank = ring->bank; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); /* Disable interrupts for the given ring */ adf_disable_ring_irq(bank, ring->ring_number); /* Clear PCI config space */ - WRITE_CSR_RING_CONFIG(bank->csr_addr, bank->bank_number, - ring->ring_number, 0); - WRITE_CSR_RING_BASE(bank->csr_addr, bank->bank_number, - ring->ring_number, 0); + + csr_ops->write_csr_ring_config(bank->csr_addr, bank->bank_number, + ring->ring_number, 0); + csr_ops->write_csr_ring_base(bank->csr_addr, bank->bank_number, + ring->ring_number, 0); adf_ring_debugfs_rm(ring); adf_unreserve_ring(bank, ring->ring_number); /* Disable HW arbitration for the given ring */ @@ -286,25 +310,30 @@ void adf_remove_ring(struct adf_etr_ring_data *ring) static void adf_ring_response_handler(struct adf_etr_bank_data *bank) { - u32 empty_rings, i; + struct adf_accel_dev *accel_dev = bank->accel_dev; + u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(accel_dev); + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); + unsigned long empty_rings; + int i; - empty_rings = READ_CSR_E_STAT(bank->csr_addr, bank->bank_number); + empty_rings = csr_ops->read_csr_e_stat(bank->csr_addr, + bank->bank_number); empty_rings = ~empty_rings & bank->irq_mask; - for (i = 0; i < ADF_ETR_MAX_RINGS_PER_BANK; ++i) { - if (empty_rings & (1 << i)) - adf_handle_response(&bank->rings[i]); - } + for_each_set_bit(i, &empty_rings, num_rings_per_bank) + adf_handle_response(&bank->rings[i]); } void adf_response_handler(uintptr_t bank_addr) { struct adf_etr_bank_data *bank = (void *)bank_addr; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); /* Handle all the responses and reenable IRQs */ adf_ring_response_handler(bank); - WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr, bank->bank_number, - bank->irq_mask); + + csr_ops->write_csr_int_flag_and_col(bank->csr_addr, bank->bank_number, + bank->irq_mask); } static inline int adf_get_cfg_int(struct adf_accel_dev *accel_dev, @@ -343,9 +372,14 @@ static int adf_init_bank(struct adf_accel_dev *accel_dev, u32 bank_num, void __iomem *csr_addr) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u8 num_rings_per_bank = hw_data->num_rings_per_bank; + struct adf_hw_csr_ops *csr_ops = &hw_data->csr_ops; + u32 irq_mask = BIT(num_rings_per_bank) - 1; struct adf_etr_ring_data *ring; struct adf_etr_ring_data *tx_ring; u32 i, coalesc_enabled = 0; + unsigned long ring_mask; + int size; memset(bank, 0, sizeof(*bank)); bank->bank_number = bank_num; @@ -353,6 +387,13 @@ static int adf_init_bank(struct adf_accel_dev *accel_dev, bank->accel_dev = accel_dev; spin_lock_init(&bank->lock); + /* Allocate the rings in the bank */ + size = num_rings_per_bank * sizeof(struct adf_etr_ring_data); + bank->rings = kzalloc_node(size, GFP_KERNEL, + dev_to_node(&GET_DEV(accel_dev))); + if (!bank->rings) + return -ENOMEM; + /* Enable IRQ coalescing always. This will allow to use * the optimised flag and coalesc register. * If it is disabled in the config file just use min time value */ @@ -363,9 +404,10 @@ static int adf_init_bank(struct adf_accel_dev *accel_dev, else bank->irq_coalesc_timer = ADF_COALESCING_MIN_TIME; - for (i = 0; i < ADF_ETR_MAX_RINGS_PER_BANK; i++) { - WRITE_CSR_RING_CONFIG(csr_addr, bank_num, i, 0); - WRITE_CSR_RING_BASE(csr_addr, bank_num, i, 0); + for (i = 0; i < num_rings_per_bank; i++) { + csr_ops->write_csr_ring_config(csr_addr, bank_num, i, 0); + csr_ops->write_csr_ring_base(csr_addr, bank_num, i, 0); + ring = &bank->rings[i]; if (hw_data->tx_rings_mask & (1 << i)) { ring->inflights = @@ -390,15 +432,18 @@ static int adf_init_bank(struct adf_accel_dev *accel_dev, goto err; } - WRITE_CSR_INT_FLAG(csr_addr, bank_num, ADF_BANK_INT_FLAG_CLEAR_MASK); - WRITE_CSR_INT_SRCSEL(csr_addr, bank_num); + csr_ops->write_csr_int_flag(csr_addr, bank_num, irq_mask); + csr_ops->write_csr_int_srcsel(csr_addr, bank_num); + return 0; err: - for (i = 0; i < ADF_ETR_MAX_RINGS_PER_BANK; i++) { + ring_mask = hw_data->tx_rings_mask; + for_each_set_bit(i, &ring_mask, num_rings_per_bank) { ring = &bank->rings[i]; - if (hw_data->tx_rings_mask & (1 << i)) - kfree(ring->inflights); + kfree(ring->inflights); + ring->inflights = NULL; } + kfree(bank->rings); return -ENOMEM; } @@ -464,11 +509,12 @@ EXPORT_SYMBOL_GPL(adf_init_etr_data); static void cleanup_bank(struct adf_etr_bank_data *bank) { + struct adf_accel_dev *accel_dev = bank->accel_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u8 num_rings_per_bank = hw_data->num_rings_per_bank; u32 i; - for (i = 0; i < ADF_ETR_MAX_RINGS_PER_BANK; i++) { - struct adf_accel_dev *accel_dev = bank->accel_dev; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; + for (i = 0; i < num_rings_per_bank; i++) { struct adf_etr_ring_data *ring = &bank->rings[i]; if (bank->ring_mask & (1 << i)) @@ -477,6 +523,7 @@ static void cleanup_bank(struct adf_etr_bank_data *bank) if (hw_data->tx_rings_mask & (1 << i)) kfree(ring->inflights); } + kfree(bank->rings); adf_bank_debugfs_rm(bank); memset(bank, 0, sizeof(*bank)); } @@ -507,6 +554,7 @@ void adf_cleanup_etr_data(struct adf_accel_dev *accel_dev) if (etr_data) { adf_cleanup_etr_handles(accel_dev); debugfs_remove(etr_data->debug); + kfree(etr_data->banks->rings); kfree(etr_data->banks); kfree(etr_data); accel_dev->transport = NULL; diff --git a/drivers/crypto/qat/qat_common/adf_transport_access_macros.h b/drivers/crypto/qat/qat_common/adf_transport_access_macros.h index 950d1988556c..3b6b0267bbec 100644 --- a/drivers/crypto/qat/qat_common/adf_transport_access_macros.h +++ b/drivers/crypto/qat/qat_common/adf_transport_access_macros.h @@ -4,23 +4,6 @@ #define ADF_TRANSPORT_ACCESS_MACROS_H #include "adf_accel_devices.h" -#define ADF_BANK_INT_SRC_SEL_MASK_0 0x4444444CUL -#define ADF_BANK_INT_SRC_SEL_MASK_X 0x44444444UL -#define ADF_BANK_INT_FLAG_CLEAR_MASK 0xFFFF -#define ADF_RING_CSR_RING_CONFIG 0x000 -#define ADF_RING_CSR_RING_LBASE 0x040 -#define ADF_RING_CSR_RING_UBASE 0x080 -#define ADF_RING_CSR_RING_HEAD 0x0C0 -#define ADF_RING_CSR_RING_TAIL 0x100 -#define ADF_RING_CSR_E_STAT 0x14C -#define ADF_RING_CSR_INT_FLAG 0x170 -#define ADF_RING_CSR_INT_SRCSEL 0x174 -#define ADF_RING_CSR_INT_SRCSEL_2 0x178 -#define ADF_RING_CSR_INT_COL_EN 0x17C -#define ADF_RING_CSR_INT_COL_CTL 0x180 -#define ADF_RING_CSR_INT_FLAG_AND_COL 0x184 -#define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000 -#define ADF_RING_BUNDLE_SIZE 0x1000 #define ADF_RING_CONFIG_NEAR_FULL_WM 0x0A #define ADF_RING_CONFIG_NEAR_EMPTY_WM 0x05 #define ADF_COALESCING_MIN_TIME 0x1FF @@ -72,54 +55,4 @@ ((watermark_nf << ADF_RING_CONFIG_NEAR_FULL_WM) \ | (watermark_ne << ADF_RING_CONFIG_NEAR_EMPTY_WM) \ | size) -#define BUILD_RING_BASE_ADDR(addr, size) \ - ((addr >> 6) & (0xFFFFFFFFFFFFFFFFULL << size)) -#define READ_CSR_RING_HEAD(csr_base_addr, bank, ring) \ - ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_RING_HEAD + (ring << 2)) -#define READ_CSR_RING_TAIL(csr_base_addr, bank, ring) \ - ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_RING_TAIL + (ring << 2)) -#define READ_CSR_E_STAT(csr_base_addr, bank) \ - ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_E_STAT) -#define WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_RING_CONFIG + (ring << 2), value) -#define WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, value) \ -do { \ - u32 l_base = 0, u_base = 0; \ - l_base = (u32)(value & 0xFFFFFFFF); \ - u_base = (u32)((value & 0xFFFFFFFF00000000ULL) >> 32); \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_RING_LBASE + (ring << 2), l_base); \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_RING_UBASE + (ring << 2), u_base); \ -} while (0) -#define WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_RING_HEAD + (ring << 2), value) -#define WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_RING_TAIL + (ring << 2), value) -#define WRITE_CSR_INT_FLAG(csr_base_addr, bank, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_INT_FLAG, value) -#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank) \ -do { \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_INT_SRCSEL, ADF_BANK_INT_SRC_SEL_MASK_0); \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_INT_SRCSEL_2, ADF_BANK_INT_SRC_SEL_MASK_X); \ -} while (0) -#define WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_INT_COL_EN, value) -#define WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_INT_COL_CTL, \ - ADF_RING_CSR_INT_COL_CTL_ENABLE | value) -#define WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_INT_FLAG_AND_COL, value) #endif diff --git a/drivers/crypto/qat/qat_common/adf_transport_debug.c b/drivers/crypto/qat/qat_common/adf_transport_debug.c index dac25ba47260..1205186ad51e 100644 --- a/drivers/crypto/qat/qat_common/adf_transport_debug.c +++ b/drivers/crypto/qat/qat_common/adf_transport_debug.c @@ -42,16 +42,17 @@ static int adf_ring_show(struct seq_file *sfile, void *v) { struct adf_etr_ring_data *ring = sfile->private; struct adf_etr_bank_data *bank = ring->bank; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); void __iomem *csr = ring->bank->csr_addr; if (v == SEQ_START_TOKEN) { int head, tail, empty; - head = READ_CSR_RING_HEAD(csr, bank->bank_number, - ring->ring_number); - tail = READ_CSR_RING_TAIL(csr, bank->bank_number, - ring->ring_number); - empty = READ_CSR_E_STAT(csr, bank->bank_number); + head = csr_ops->read_csr_ring_head(csr, bank->bank_number, + ring->ring_number); + tail = csr_ops->read_csr_ring_tail(csr, bank->bank_number, + ring->ring_number); + empty = csr_ops->read_csr_e_stat(csr, bank->bank_number); seq_puts(sfile, "------- Ring configuration -------\n"); seq_printf(sfile, "ring name: %s\n", @@ -117,11 +118,14 @@ void adf_ring_debugfs_rm(struct adf_etr_ring_data *ring) static void *adf_bank_start(struct seq_file *sfile, loff_t *pos) { + struct adf_etr_bank_data *bank = sfile->private; + u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(bank->accel_dev); + mutex_lock(&bank_read_lock); if (*pos == 0) return SEQ_START_TOKEN; - if (*pos >= ADF_ETR_MAX_RINGS_PER_BANK) + if (*pos >= num_rings_per_bank) return NULL; return pos; @@ -129,7 +133,10 @@ static void *adf_bank_start(struct seq_file *sfile, loff_t *pos) static void *adf_bank_next(struct seq_file *sfile, void *v, loff_t *pos) { - if (++(*pos) >= ADF_ETR_MAX_RINGS_PER_BANK) + struct adf_etr_bank_data *bank = sfile->private; + u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(bank->accel_dev); + + if (++(*pos) >= num_rings_per_bank) return NULL; return pos; @@ -138,6 +145,7 @@ static void *adf_bank_next(struct seq_file *sfile, void *v, loff_t *pos) static int adf_bank_show(struct seq_file *sfile, void *v) { struct adf_etr_bank_data *bank = sfile->private; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); if (v == SEQ_START_TOKEN) { seq_printf(sfile, "------- Bank %d configuration -------\n", @@ -151,11 +159,11 @@ static int adf_bank_show(struct seq_file *sfile, void *v) if (!(bank->ring_mask & 1 << ring_id)) return 0; - head = READ_CSR_RING_HEAD(csr, bank->bank_number, - ring->ring_number); - tail = READ_CSR_RING_TAIL(csr, bank->bank_number, - ring->ring_number); - empty = READ_CSR_E_STAT(csr, bank->bank_number); + head = csr_ops->read_csr_ring_head(csr, bank->bank_number, + ring->ring_number); + tail = csr_ops->read_csr_ring_tail(csr, bank->bank_number, + ring->ring_number); + empty = csr_ops->read_csr_e_stat(csr, bank->bank_number); seq_printf(sfile, "ring num %02d, head %04x, tail %04x, empty: %d\n", diff --git a/drivers/crypto/qat/qat_common/adf_transport_internal.h b/drivers/crypto/qat/qat_common/adf_transport_internal.h index c7faf4e2d302..501bcf0f1809 100644 --- a/drivers/crypto/qat/qat_common/adf_transport_internal.h +++ b/drivers/crypto/qat/qat_common/adf_transport_internal.h @@ -28,7 +28,7 @@ struct adf_etr_ring_data { }; struct adf_etr_bank_data { - struct adf_etr_ring_data rings[ADF_ETR_MAX_RINGS_PER_BANK]; + struct adf_etr_ring_data *rings; struct tasklet_struct resp_handler; void __iomem *csr_addr; u32 irq_coalesc_timer; diff --git a/drivers/crypto/qat/qat_common/adf_vf_isr.c b/drivers/crypto/qat/qat_common/adf_vf_isr.c index c4a44dc6af3e..38d316a42ba6 100644 --- a/drivers/crypto/qat/qat_common/adf_vf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_vf_isr.c @@ -156,6 +156,7 @@ static irqreturn_t adf_isr(int irq, void *privdata) { struct adf_accel_dev *accel_dev = privdata; struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_hw_csr_ops *csr_ops = &hw_data->csr_ops; struct adf_bar *pmisc = &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; void __iomem *pmisc_bar_addr = pmisc->virt_addr; @@ -180,8 +181,8 @@ static irqreturn_t adf_isr(int irq, void *privdata) struct adf_etr_bank_data *bank = &etr_data->banks[0]; /* Disable Flag and Coalesce Ring Interrupts */ - WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr, bank->bank_number, - 0); + csr_ops->write_csr_int_flag_and_col(bank->csr_addr, + bank->bank_number, 0); tasklet_hi_schedule(&bank->resp_handler); return IRQ_HANDLED; } diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h b/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h index d4d188cd7ed0..f05ad17fbdd6 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h @@ -6,7 +6,7 @@ #include "icp_qat_fw.h" enum icp_qat_fw_init_admin_cmd_id { - ICP_QAT_FW_INIT_ME = 0, + ICP_QAT_FW_INIT_AE = 0, ICP_QAT_FW_TRNG_ENABLE = 1, ICP_QAT_FW_TRNG_DISABLE = 2, ICP_QAT_FW_CONSTANTS_CFG = 3, @@ -39,7 +39,7 @@ struct icp_qat_fw_init_admin_req { }; __u32 resrvd4; -}; +} __packed; struct icp_qat_fw_init_admin_resp { __u8 flags; @@ -92,7 +92,7 @@ struct icp_qat_fw_init_admin_resp { __u64 resrvd8; }; }; -}; +} __packed; #define ICP_QAT_FW_COMN_HEARTBEAT_OK 0 #define ICP_QAT_FW_COMN_HEARTBEAT_BLOCKED 1 diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_la.h b/drivers/crypto/qat/qat_common/icp_qat_fw_la.h index 6757ec09d81f..28fa17f14be4 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_la.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_la.h @@ -33,6 +33,9 @@ struct icp_qat_fw_la_bulk_req { struct icp_qat_fw_comn_req_cd_ctrl cd_ctrl; }; +#define ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE 1 +#define QAT_LA_SLICE_TYPE_BITPOS 14 +#define QAT_LA_SLICE_TYPE_MASK 0x3 #define ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS 1 #define ICP_QAT_FW_LA_GCM_IV_LEN_NOT_12_OCTETS 0 #define QAT_FW_LA_ZUC_3G_PROTO_FLAG_BITPOS 12 @@ -179,6 +182,10 @@ struct icp_qat_fw_la_bulk_req { QAT_FIELD_SET(flags, val, QAT_LA_PARTIAL_BITPOS, \ QAT_LA_PARTIAL_MASK) +#define ICP_QAT_FW_LA_SLICE_TYPE_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_LA_SLICE_TYPE_BITPOS, \ + QAT_LA_SLICE_TYPE_MASK) + struct icp_qat_fw_cipher_req_hdr_cd_pars { union { struct { diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index 3e8e291cd122..b8f3463be6ef 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -15,6 +15,7 @@ struct icp_qat_fw_loader_ae_data { struct icp_qat_fw_loader_hal_handle { struct icp_qat_fw_loader_ae_data aes[ICP_QAT_UCLO_MAX_AE]; unsigned int ae_mask; + unsigned int admin_ae_mask; unsigned int slice_mask; unsigned int revision_id; unsigned int ae_max_num; @@ -22,12 +23,35 @@ struct icp_qat_fw_loader_hal_handle { unsigned int max_ustore; }; +struct icp_qat_fw_loader_chip_info { + bool sram_visible; + bool nn; + bool lm2lm3; + u32 lm_size; + u32 icp_rst_csr; + u32 icp_rst_mask; + u32 glb_clk_enable_csr; + u32 misc_ctl_csr; + u32 wakeup_event_val; + bool fw_auth; + bool css_3k; + bool tgroup_share_ustore; + u32 fcu_ctl_csr; + u32 fcu_sts_csr; + u32 fcu_dram_addr_hi; + u32 fcu_dram_addr_lo; + u32 fcu_loaded_ae_csr; + u8 fcu_loaded_ae_pos; +}; + struct icp_qat_fw_loader_handle { struct icp_qat_fw_loader_hal_handle *hal_handle; + struct icp_qat_fw_loader_chip_info *chip_info; struct pci_dev *pci_dev; void *obj_handle; void *sobj_handle; - bool fw_auth; + void *mobj_handle; + unsigned int cfg_ae_mask; void __iomem *hal_sram_addr_v; void __iomem *hal_cap_g_ctl_csr_addr_v; void __iomem *hal_cap_ae_xfer_csr_addr_v; diff --git a/drivers/crypto/qat/qat_common/icp_qat_hal.h b/drivers/crypto/qat/qat_common/icp_qat_hal.h index c0e9fc0c93dd..20b2ee1fc65a 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hal.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hal.h @@ -5,9 +5,17 @@ #include "icp_qat_fw_loader_handle.h" enum hal_global_csr { - MISC_CONTROL = 0x04, - ICP_RESET = 0x0c, - ICP_GLOBAL_CLK_ENABLE = 0x50 + MISC_CONTROL = 0xA04, + ICP_RESET = 0xA0c, + ICP_GLOBAL_CLK_ENABLE = 0xA50 +}; + +enum { + MISC_CONTROL_C4XXX = 0xAA0, + ICP_RESET_CPP0 = 0x938, + ICP_RESET_CPP1 = 0x93c, + ICP_GLOBAL_CLK_ENABLE_CPP0 = 0x964, + ICP_GLOBAL_CLK_ENABLE_CPP1 = 0x968 }; enum hal_ae_csr { @@ -26,8 +34,14 @@ enum hal_ae_csr { CTX_WAKEUP_EVENTS_INDIRECT = 0x050, LM_ADDR_0_INDIRECT = 0x060, LM_ADDR_1_INDIRECT = 0x068, + LM_ADDR_2_INDIRECT = 0x0cc, + LM_ADDR_3_INDIRECT = 0x0d4, INDIRECT_LM_ADDR_0_BYTE_INDEX = 0x0e0, INDIRECT_LM_ADDR_1_BYTE_INDEX = 0x0e8, + INDIRECT_LM_ADDR_2_BYTE_INDEX = 0x10c, + INDIRECT_LM_ADDR_3_BYTE_INDEX = 0x114, + INDIRECT_T_INDEX = 0x0f8, + INDIRECT_T_INDEX_BYTE_INDEX = 0x0fc, FUTURE_COUNT_SIGNAL_INDIRECT = 0x078, TIMESTAMP_LOW = 0x0c0, TIMESTAMP_HIGH = 0x0c4, @@ -47,6 +61,15 @@ enum fcu_csr { FCU_RAMBASE_ADDR_LO = 0x8d8 }; +enum fcu_csr_4xxx { + FCU_CONTROL_4XXX = 0x1000, + FCU_STATUS_4XXX = 0x1004, + FCU_ME_BROADCAST_MASK_TYPE = 0x1008, + FCU_AE_LOADED_4XXX = 0x1010, + FCU_DRAM_ADDR_LO_4XXX = 0x1014, + FCU_DRAM_ADDR_HI_4XXX = 0x1018, +}; + enum fcu_cmd { FCU_CTRL_CMD_NOOP = 0, FCU_CTRL_CMD_AUTH = 1, @@ -62,12 +85,17 @@ enum fcu_sts { FCU_STS_LOAD_FAIL = 4, FCU_STS_BUSY = 5 }; + +#define ALL_AE_MASK 0xFFFFFFFF #define UA_ECS (0x1 << 31) #define ACS_ABO_BITPOS 31 #define ACS_ACNO 0x7 #define CE_ENABLE_BITPOS 0x8 #define CE_LMADDR_0_GLOBAL_BITPOS 16 #define CE_LMADDR_1_GLOBAL_BITPOS 17 +#define CE_LMADDR_2_GLOBAL_BITPOS 22 +#define CE_LMADDR_3_GLOBAL_BITPOS 23 +#define CE_T_INDEX_GLOBAL_BITPOS 21 #define CE_NN_MODE_BITPOS 20 #define CE_REG_PAR_ERR_BITPOS 25 #define CE_BREAKPOINT_BITPOS 27 @@ -78,7 +106,8 @@ enum fcu_sts { #define XCWE_VOLUNTARY (0x1) #define LCS_STATUS (0x1) #define MMC_SHARE_CS_BITPOS 2 -#define GLOBAL_CSR 0xA00 +#define WAKEUP_EVENT 0x10000 +#define FCU_CTRL_BROADCAST_POS 0x4 #define FCU_CTRL_AE_POS 0x8 #define FCU_AUTH_STS_MASK 0x7 #define FCU_STS_DONE_POS 0x9 @@ -86,27 +115,29 @@ enum fcu_sts { #define FCU_LOADED_AE_POS 0x16 #define FW_AUTH_WAIT_PERIOD 10 #define FW_AUTH_MAX_RETRY 300 - +#define ICP_QAT_AE_OFFSET 0x20000 +#define ICP_QAT_CAP_OFFSET (ICP_QAT_AE_OFFSET + 0x10000) +#define LOCAL_TO_XFER_REG_OFFSET 0x800 +#define ICP_QAT_EP_OFFSET 0x3a000 +#define ICP_QAT_EP_OFFSET_4XXX 0x200000 /* HI MMIO CSRs */ +#define ICP_QAT_AE_OFFSET_4XXX 0x600000 +#define ICP_QAT_CAP_OFFSET_4XXX 0x640000 #define SET_CAP_CSR(handle, csr, val) \ - ADF_CSR_WR(handle->hal_cap_g_ctl_csr_addr_v, csr, val) + ADF_CSR_WR((handle)->hal_cap_g_ctl_csr_addr_v, csr, val) #define GET_CAP_CSR(handle, csr) \ - ADF_CSR_RD(handle->hal_cap_g_ctl_csr_addr_v, csr) -#define SET_GLB_CSR(handle, csr, val) SET_CAP_CSR(handle, csr + GLOBAL_CSR, val) -#define GET_GLB_CSR(handle, csr) GET_CAP_CSR(handle, GLOBAL_CSR + csr) + ADF_CSR_RD((handle)->hal_cap_g_ctl_csr_addr_v, csr) #define AE_CSR(handle, ae) \ - ((char __iomem *)handle->hal_cap_ae_local_csr_addr_v + \ - ((ae & handle->hal_handle->ae_mask) << 12)) -#define AE_CSR_ADDR(handle, ae, csr) (AE_CSR(handle, ae) + (0x3ff & csr)) + ((char __iomem *)(handle)->hal_cap_ae_local_csr_addr_v + ((ae) << 12)) +#define AE_CSR_ADDR(handle, ae, csr) (AE_CSR(handle, ae) + (0x3ff & (csr))) #define SET_AE_CSR(handle, ae, csr, val) \ ADF_CSR_WR(AE_CSR_ADDR(handle, ae, csr), 0, val) #define GET_AE_CSR(handle, ae, csr) ADF_CSR_RD(AE_CSR_ADDR(handle, ae, csr), 0) #define AE_XFER(handle, ae) \ - ((char __iomem *)handle->hal_cap_ae_xfer_csr_addr_v + \ - ((ae & handle->hal_handle->ae_mask) << 12)) + ((char __iomem *)(handle)->hal_cap_ae_xfer_csr_addr_v + ((ae) << 12)) #define AE_XFER_ADDR(handle, ae, reg) (AE_XFER(handle, ae) + \ - ((reg & 0xff) << 2)) + (((reg) & 0xff) << 2)) #define SET_AE_XFER(handle, ae, reg, val) \ ADF_CSR_WR(AE_XFER_ADDR(handle, ae, reg), 0, val) #define SRAM_WRITE(handle, addr, val) \ - ADF_CSR_WR(handle->hal_sram_addr_v, addr, val) + ADF_CSR_WR((handle)->hal_sram_addr_v, addr, val) #endif diff --git a/drivers/crypto/qat/qat_common/icp_qat_hw.h b/drivers/crypto/qat/qat_common/icp_qat_hw.h index c4b6ef1506ab..e39e8a2d51a7 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hw.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hw.h @@ -65,6 +65,36 @@ struct icp_qat_hw_auth_config { __u32 reserved; }; +struct icp_qat_hw_ucs_cipher_config { + __u32 val; + __u32 reserved[3]; +}; + +enum icp_qat_slice_mask { + ICP_ACCEL_MASK_CIPHER_SLICE = BIT(0), + ICP_ACCEL_MASK_AUTH_SLICE = BIT(1), + ICP_ACCEL_MASK_PKE_SLICE = BIT(2), + ICP_ACCEL_MASK_COMPRESS_SLICE = BIT(3), + ICP_ACCEL_MASK_LZS_SLICE = BIT(4), + ICP_ACCEL_MASK_EIA3_SLICE = BIT(5), + ICP_ACCEL_MASK_SHA3_SLICE = BIT(6), +}; + +enum icp_qat_capabilities_mask { + ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC = BIT(0), + ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC = BIT(1), + ICP_ACCEL_CAPABILITIES_CIPHER = BIT(2), + ICP_ACCEL_CAPABILITIES_AUTHENTICATION = BIT(3), + ICP_ACCEL_CAPABILITIES_RESERVED_1 = BIT(4), + ICP_ACCEL_CAPABILITIES_COMPRESSION = BIT(5), + ICP_ACCEL_CAPABILITIES_LZS_COMPRESSION = BIT(6), + ICP_ACCEL_CAPABILITIES_RAND = BIT(7), + ICP_ACCEL_CAPABILITIES_ZUC = BIT(8), + ICP_ACCEL_CAPABILITIES_SHA3 = BIT(9), + /* Bits 10-25 are currently reserved */ + ICP_ACCEL_CAPABILITIES_AES_V2 = BIT(26) +}; + #define QAT_AUTH_MODE_BITPOS 4 #define QAT_AUTH_MODE_MASK 0xF #define QAT_AUTH_ALGO_BITPOS 0 @@ -255,7 +285,15 @@ struct icp_qat_hw_cipher_aes256_f8 { __u8 key[ICP_QAT_HW_AES_256_F8_KEY_SZ]; }; +struct icp_qat_hw_ucs_cipher_aes256_f8 { + struct icp_qat_hw_ucs_cipher_config cipher_config; + __u8 key[ICP_QAT_HW_AES_256_F8_KEY_SZ]; +}; + struct icp_qat_hw_cipher_algo_blk { - struct icp_qat_hw_cipher_aes256_f8 aes; + union { + struct icp_qat_hw_cipher_aes256_f8 aes; + struct icp_qat_hw_ucs_cipher_aes256_f8 ucs_aes; + }; } __aligned(64); #endif diff --git a/drivers/crypto/qat/qat_common/icp_qat_uclo.h b/drivers/crypto/qat/qat_common/icp_qat_uclo.h index 8fe1ec344fa2..4b36869bf460 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_uclo.h +++ b/drivers/crypto/qat/qat_common/icp_qat_uclo.h @@ -6,6 +6,7 @@ #define ICP_QAT_AC_895XCC_DEV_TYPE 0x00400000 #define ICP_QAT_AC_C62X_DEV_TYPE 0x01000000 #define ICP_QAT_AC_C3XXX_DEV_TYPE 0x02000000 +#define ICP_QAT_AC_4XXX_A_DEV_TYPE 0x08000000 #define ICP_QAT_UCLO_MAX_AE 12 #define ICP_QAT_UCLO_MAX_CTX 8 #define ICP_QAT_UCLO_MAX_UIMAGE (ICP_QAT_UCLO_MAX_AE * ICP_QAT_UCLO_MAX_CTX) @@ -13,6 +14,7 @@ #define ICP_QAT_UCLO_MAX_XFER_REG 128 #define ICP_QAT_UCLO_MAX_GPR_REG 128 #define ICP_QAT_UCLO_MAX_LMEM_REG 1024 +#define ICP_QAT_UCLO_MAX_LMEM_REG_2X 1280 #define ICP_QAT_UCLO_AE_ALL_CTX 0xff #define ICP_QAT_UOF_OBJID_LEN 8 #define ICP_QAT_UOF_FID 0xc6c2 @@ -31,26 +33,59 @@ #define ICP_QAT_SUOF_FID 0x53554f46 #define ICP_QAT_SUOF_MAJVER 0x0 #define ICP_QAT_SUOF_MINVER 0x1 +#define ICP_QAT_SUOF_OBJ_NAME_LEN 128 +#define ICP_QAT_MOF_OBJ_ID_LEN 8 +#define ICP_QAT_MOF_OBJ_CHUNKID_LEN 8 +#define ICP_QAT_MOF_FID 0x00666f6d +#define ICP_QAT_MOF_MAJVER 0x0 +#define ICP_QAT_MOF_MINVER 0x1 +#define ICP_QAT_MOF_SYM_OBJS "SYM_OBJS" +#define ICP_QAT_SUOF_OBJS "SUF_OBJS" +#define ICP_QAT_SUOF_IMAG "SUF_IMAG" #define ICP_QAT_SIMG_AE_INIT_SEQ_LEN (50 * sizeof(unsigned long long)) #define ICP_QAT_SIMG_AE_INSTS_LEN (0x4000 * sizeof(unsigned long long)) -#define ICP_QAT_CSS_FWSK_MODULUS_LEN 256 -#define ICP_QAT_CSS_FWSK_EXPONENT_LEN 4 -#define ICP_QAT_CSS_FWSK_PAD_LEN 252 -#define ICP_QAT_CSS_FWSK_PUB_LEN (ICP_QAT_CSS_FWSK_MODULUS_LEN + \ - ICP_QAT_CSS_FWSK_EXPONENT_LEN + \ - ICP_QAT_CSS_FWSK_PAD_LEN) -#define ICP_QAT_CSS_SIGNATURE_LEN 256 + +#define DSS_FWSK_MODULUS_LEN 384 /* RSA3K */ +#define DSS_FWSK_EXPONENT_LEN 4 +#define DSS_FWSK_PADDING_LEN 380 +#define DSS_SIGNATURE_LEN 384 /* RSA3K */ + +#define CSS_FWSK_MODULUS_LEN 256 /* RSA2K */ +#define CSS_FWSK_EXPONENT_LEN 4 +#define CSS_FWSK_PADDING_LEN 252 +#define CSS_SIGNATURE_LEN 256 /* RSA2K */ + +#define ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) ((handle)->chip_info->css_3k ? \ + DSS_FWSK_MODULUS_LEN : \ + CSS_FWSK_MODULUS_LEN) + +#define ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle) ((handle)->chip_info->css_3k ? \ + DSS_FWSK_EXPONENT_LEN : \ + CSS_FWSK_EXPONENT_LEN) + +#define ICP_QAT_CSS_FWSK_PAD_LEN(handle) ((handle)->chip_info->css_3k ? \ + DSS_FWSK_PADDING_LEN : \ + CSS_FWSK_PADDING_LEN) + +#define ICP_QAT_CSS_FWSK_PUB_LEN(handle) (ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + \ + ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle) + \ + ICP_QAT_CSS_FWSK_PAD_LEN(handle)) + +#define ICP_QAT_CSS_SIGNATURE_LEN(handle) ((handle)->chip_info->css_3k ? \ + DSS_SIGNATURE_LEN : \ + CSS_SIGNATURE_LEN) + #define ICP_QAT_CSS_AE_IMG_LEN (sizeof(struct icp_qat_simg_ae_mode) + \ ICP_QAT_SIMG_AE_INIT_SEQ_LEN + \ ICP_QAT_SIMG_AE_INSTS_LEN) -#define ICP_QAT_CSS_AE_SIMG_LEN (sizeof(struct icp_qat_css_hdr) + \ - ICP_QAT_CSS_FWSK_PUB_LEN + \ - ICP_QAT_CSS_SIGNATURE_LEN + \ - ICP_QAT_CSS_AE_IMG_LEN) -#define ICP_QAT_AE_IMG_OFFSET (sizeof(struct icp_qat_css_hdr) + \ - ICP_QAT_CSS_FWSK_MODULUS_LEN + \ - ICP_QAT_CSS_FWSK_EXPONENT_LEN + \ - ICP_QAT_CSS_SIGNATURE_LEN) +#define ICP_QAT_CSS_AE_SIMG_LEN(handle) (sizeof(struct icp_qat_css_hdr) + \ + ICP_QAT_CSS_FWSK_PUB_LEN(handle) + \ + ICP_QAT_CSS_SIGNATURE_LEN(handle) + \ + ICP_QAT_CSS_AE_IMG_LEN) +#define ICP_QAT_AE_IMG_OFFSET(handle) (sizeof(struct icp_qat_css_hdr) + \ + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + \ + ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle) + \ + ICP_QAT_CSS_SIGNATURE_LEN(handle)) #define ICP_QAT_CSS_MAX_IMAGE_LEN 0x40000 #define ICP_QAT_CTX_MODE(ae_mode) ((ae_mode) & 0xf) @@ -60,6 +95,9 @@ #define ICP_QAT_LOC_MEM0_MODE(ae_mode) (((ae_mode) >> 0x8) & 0x1) #define ICP_QAT_LOC_MEM1_MODE(ae_mode) (((ae_mode) >> 0x9) & 0x1) +#define ICP_QAT_LOC_MEM2_MODE(ae_mode) (((ae_mode) >> 0x6) & 0x1) +#define ICP_QAT_LOC_MEM3_MODE(ae_mode) (((ae_mode) >> 0x7) & 0x1) +#define ICP_QAT_LOC_TINDEX_MODE(ae_mode) (((ae_mode) >> 0xe) & 0x1) enum icp_qat_uof_mem_region { ICP_QAT_UOF_SRAM_REGION = 0x0, @@ -89,6 +127,8 @@ enum icp_qat_uof_regtype { ICP_LMEM0 = 27, ICP_LMEM1 = 28, ICP_NEIGH_REL = 31, + ICP_LMEM2 = 61, + ICP_LMEM3 = 62, }; enum icp_qat_css_fwtype { @@ -394,7 +434,7 @@ struct icp_qat_suof_handle { struct icp_qat_fw_auth_desc { unsigned int img_len; - unsigned int reserved; + unsigned int ae_mask; unsigned int css_hdr_high; unsigned int css_hdr_low; unsigned int img_high; @@ -481,4 +521,64 @@ struct icp_qat_suof_objhdr { unsigned int img_length; unsigned int reserved; }; + +struct icp_qat_mof_file_hdr { + unsigned int file_id; + unsigned int checksum; + char min_ver; + char maj_ver; + unsigned short reserved; + unsigned short max_chunks; + unsigned short num_chunks; +}; + +struct icp_qat_mof_chunkhdr { + char chunk_id[ICP_QAT_MOF_OBJ_ID_LEN]; + u64 offset; + u64 size; +}; + +struct icp_qat_mof_str_table { + unsigned int tab_len; + unsigned int strings; +}; + +struct icp_qat_mof_obj_hdr { + unsigned short max_chunks; + unsigned short num_chunks; + unsigned int reserved; +}; + +struct icp_qat_mof_obj_chunkhdr { + char chunk_id[ICP_QAT_MOF_OBJ_CHUNKID_LEN]; + u64 offset; + u64 size; + unsigned int name; + unsigned int reserved; +}; + +struct icp_qat_mof_objhdr { + char *obj_name; + char *obj_buf; + unsigned int obj_size; +}; + +struct icp_qat_mof_table { + unsigned int num_objs; + struct icp_qat_mof_objhdr *obj_hdr; +}; + +struct icp_qat_mof_handle { + unsigned int file_id; + unsigned int checksum; + char min_ver; + char maj_ver; + char *mof_buf; + u32 mof_size; + char *sym_str; + unsigned int sym_size; + char *uobjs_hdr; + char *sobjs_hdr; + struct icp_qat_mof_table obj_table; +}; #endif diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index d552dbcfe0a0..31c7a206a629 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -6,11 +6,13 @@ #include #include #include -#include +#include +#include #include #include #include #include +#include #include #include #include "adf_accel_devices.h" @@ -31,6 +33,15 @@ ICP_QAT_HW_CIPHER_KEY_CONVERT, \ ICP_QAT_HW_CIPHER_DECRYPT) +#define QAT_AES_HW_CONFIG_DEC_NO_CONV(alg, mode) \ + ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \ + ICP_QAT_HW_CIPHER_NO_CONVERT, \ + ICP_QAT_HW_CIPHER_DECRYPT) + +#define HW_CAP_AES_V2(accel_dev) \ + (GET_HW_DATA(accel_dev)->accel_capabilities_mask & \ + ICP_ACCEL_CAPABILITIES_AES_V2) + static DEFINE_MUTEX(algs_lock); static unsigned int active_devs; @@ -89,7 +100,9 @@ struct qat_alg_skcipher_ctx { struct icp_qat_fw_la_bulk_req dec_fw_req; struct qat_crypto_instance *inst; struct crypto_skcipher *ftfm; + struct crypto_cipher *tweak; bool fallback; + int mode; }; static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg) @@ -103,7 +116,7 @@ static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg) return ICP_QAT_HW_SHA512_STATE1_SZ; default: return -EFAULT; - }; + } return -EFAULT; } @@ -214,24 +227,7 @@ static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash, return 0; } -static void qat_alg_init_hdr_iv_updt(struct icp_qat_fw_comn_req_hdr *header) -{ - ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags, - ICP_QAT_FW_CIPH_IV_64BIT_PTR); - ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_UPDATE_STATE); -} - -static void qat_alg_init_hdr_no_iv_updt(struct icp_qat_fw_comn_req_hdr *header) -{ - ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags, - ICP_QAT_FW_CIPH_IV_16BYTE_DATA); - ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_NO_UPDATE_STATE); -} - -static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header, - int aead) +static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header) { header->hdr_flags = ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET); @@ -241,12 +237,12 @@ static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header, QAT_COMN_PTR_TYPE_SGL); ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags, ICP_QAT_FW_LA_PARTIAL_NONE); - if (aead) - qat_alg_init_hdr_no_iv_updt(header); - else - qat_alg_init_hdr_iv_updt(header); + ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags, + ICP_QAT_FW_CIPH_IV_16BYTE_DATA); ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags, ICP_QAT_FW_LA_NO_PROTO); + ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_NO_UPDATE_STATE); } static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm, @@ -281,7 +277,7 @@ static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm, return -EFAULT; /* Request setup */ - qat_alg_init_common_hdr(header, 1); + qat_alg_init_common_hdr(header); header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER_HASH; ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, ICP_QAT_FW_LA_DIGEST_IN_BUFFER); @@ -368,7 +364,7 @@ static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm, return -EFAULT; /* Request setup */ - qat_alg_init_common_hdr(header, 1); + qat_alg_init_common_hdr(header); header->service_cmd_id = ICP_QAT_FW_LA_CMD_HASH_CIPHER; ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, ICP_QAT_FW_LA_DIGEST_IN_BUFFER); @@ -430,12 +426,32 @@ static void qat_alg_skcipher_init_com(struct qat_alg_skcipher_ctx *ctx, struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars; struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr; struct icp_qat_fw_cipher_cd_ctrl_hdr *cd_ctrl = (void *)&req->cd_ctrl; + bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev); + int mode = ctx->mode; - memcpy(cd->aes.key, key, keylen); - qat_alg_init_common_hdr(header, 0); + qat_alg_init_common_hdr(header); header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER; cd_pars->u.s.content_desc_params_sz = sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3; + + if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_XTS_MODE) { + ICP_QAT_FW_LA_SLICE_TYPE_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE); + + /* Store both XTS keys in CD, only the first key is sent + * to the HW, the second key is used for tweak calculation + */ + memcpy(cd->ucs_aes.key, key, keylen); + keylen = keylen / 2; + } else if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_CTR_MODE) { + ICP_QAT_FW_LA_SLICE_TYPE_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE); + keylen = round_up(keylen, 16); + memcpy(cd->ucs_aes.key, key, keylen); + } else { + memcpy(cd->aes.key, key, keylen); + } + /* Cipher CD config setup */ cd_ctrl->cipher_key_sz = keylen >> 3; cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3; @@ -457,6 +473,28 @@ static void qat_alg_skcipher_init_enc(struct qat_alg_skcipher_ctx *ctx, enc_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode); } +static void qat_alg_xts_reverse_key(const u8 *key_forward, unsigned int keylen, + u8 *key_reverse) +{ + struct crypto_aes_ctx aes_expanded; + int nrounds; + u8 *key; + + aes_expandkey(&aes_expanded, key_forward, keylen); + if (keylen == AES_KEYSIZE_128) { + nrounds = 10; + key = (u8 *)aes_expanded.key_enc + (AES_BLOCK_SIZE * nrounds); + memcpy(key_reverse, key, AES_BLOCK_SIZE); + } else { + /* AES_KEYSIZE_256 */ + nrounds = 14; + key = (u8 *)aes_expanded.key_enc + (AES_BLOCK_SIZE * nrounds); + memcpy(key_reverse, key, AES_BLOCK_SIZE); + memcpy(key_reverse + AES_BLOCK_SIZE, key - AES_BLOCK_SIZE, + AES_BLOCK_SIZE); + } +} + static void qat_alg_skcipher_init_dec(struct qat_alg_skcipher_ctx *ctx, int alg, const u8 *key, unsigned int keylen, int mode) @@ -464,16 +502,26 @@ static void qat_alg_skcipher_init_dec(struct qat_alg_skcipher_ctx *ctx, struct icp_qat_hw_cipher_algo_blk *dec_cd = ctx->dec_cd; struct icp_qat_fw_la_bulk_req *req = &ctx->dec_fw_req; struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars; + bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev); qat_alg_skcipher_init_com(ctx, req, dec_cd, key, keylen); cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr; - if (mode != ICP_QAT_HW_CIPHER_CTR_MODE) + if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_XTS_MODE) { + /* Key reversing not supported, set no convert */ + dec_cd->aes.cipher_config.val = + QAT_AES_HW_CONFIG_DEC_NO_CONV(alg, mode); + + /* In-place key reversal */ + qat_alg_xts_reverse_key(dec_cd->ucs_aes.key, keylen / 2, + dec_cd->ucs_aes.key); + } else if (mode != ICP_QAT_HW_CIPHER_CTR_MODE) { dec_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_DEC(alg, mode); - else + } else { dec_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode); + } } static int qat_alg_validate_key(int key_len, int *alg, int mode) @@ -787,6 +835,61 @@ static void qat_aead_alg_callback(struct icp_qat_fw_la_resp *qat_resp, areq->base.complete(&areq->base, res); } +static void qat_alg_update_iv_ctr_mode(struct qat_crypto_request *qat_req) +{ + struct skcipher_request *sreq = qat_req->skcipher_req; + u64 iv_lo_prev; + u64 iv_lo; + u64 iv_hi; + + memcpy(qat_req->iv, sreq->iv, AES_BLOCK_SIZE); + + iv_lo = be64_to_cpu(qat_req->iv_lo); + iv_hi = be64_to_cpu(qat_req->iv_hi); + + iv_lo_prev = iv_lo; + iv_lo += DIV_ROUND_UP(sreq->cryptlen, AES_BLOCK_SIZE); + if (iv_lo < iv_lo_prev) + iv_hi++; + + qat_req->iv_lo = cpu_to_be64(iv_lo); + qat_req->iv_hi = cpu_to_be64(iv_hi); +} + +static void qat_alg_update_iv_cbc_mode(struct qat_crypto_request *qat_req) +{ + struct skcipher_request *sreq = qat_req->skcipher_req; + int offset = sreq->cryptlen - AES_BLOCK_SIZE; + struct scatterlist *sgl; + + if (qat_req->encryption) + sgl = sreq->dst; + else + sgl = sreq->src; + + scatterwalk_map_and_copy(qat_req->iv, sgl, offset, AES_BLOCK_SIZE, 0); +} + +static void qat_alg_update_iv(struct qat_crypto_request *qat_req) +{ + struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx; + struct device *dev = &GET_DEV(ctx->inst->accel_dev); + + switch (ctx->mode) { + case ICP_QAT_HW_CIPHER_CTR_MODE: + qat_alg_update_iv_ctr_mode(qat_req); + break; + case ICP_QAT_HW_CIPHER_CBC_MODE: + qat_alg_update_iv_cbc_mode(qat_req); + break; + case ICP_QAT_HW_CIPHER_XTS_MODE: + break; + default: + dev_warn(dev, "Unsupported IV update for cipher mode %d\n", + ctx->mode); + } +} + static void qat_skcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp, struct qat_crypto_request *qat_req) { @@ -794,16 +897,16 @@ static void qat_skcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp, struct qat_crypto_instance *inst = ctx->inst; struct skcipher_request *sreq = qat_req->skcipher_req; u8 stat_filed = qat_resp->comn_resp.comn_status; - struct device *dev = &GET_DEV(ctx->inst->accel_dev); int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed); qat_alg_free_bufl(inst, qat_req); if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK)) res = -EINVAL; + if (qat_req->encryption) + qat_alg_update_iv(qat_req); + memcpy(sreq->iv, qat_req->iv, AES_BLOCK_SIZE); - dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv, - qat_req->iv_paddr); sreq->base.complete(&sreq->base, res); } @@ -981,6 +1084,8 @@ static int qat_alg_skcipher_setkey(struct crypto_skcipher *tfm, { struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); + ctx->mode = mode; + if (ctx->enc_cd) return qat_alg_skcipher_rekey(ctx, key, keylen, mode); else @@ -1023,8 +1128,33 @@ static int qat_alg_skcipher_xts_setkey(struct crypto_skcipher *tfm, ctx->fallback = false; - return qat_alg_skcipher_setkey(tfm, key, keylen, - ICP_QAT_HW_CIPHER_XTS_MODE); + ret = qat_alg_skcipher_setkey(tfm, key, keylen, + ICP_QAT_HW_CIPHER_XTS_MODE); + if (ret) + return ret; + + if (HW_CAP_AES_V2(ctx->inst->accel_dev)) + ret = crypto_cipher_setkey(ctx->tweak, key + (keylen / 2), + keylen / 2); + + return ret; +} + +static void qat_alg_set_req_iv(struct qat_crypto_request *qat_req) +{ + struct icp_qat_fw_la_cipher_req_params *cipher_param; + struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx; + bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev); + u8 *iv = qat_req->skcipher_req->iv; + + cipher_param = (void *)&qat_req->req.serv_specif_rqpars; + + if (aes_v2_capable && ctx->mode == ICP_QAT_HW_CIPHER_XTS_MODE) + crypto_cipher_encrypt_one(ctx->tweak, + (u8 *)cipher_param->u.cipher_IV_array, + iv); + else + memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE); } static int qat_alg_skcipher_encrypt(struct skcipher_request *req) @@ -1035,23 +1165,14 @@ static int qat_alg_skcipher_encrypt(struct skcipher_request *req) struct qat_crypto_request *qat_req = skcipher_request_ctx(req); struct icp_qat_fw_la_cipher_req_params *cipher_param; struct icp_qat_fw_la_bulk_req *msg; - struct device *dev = &GET_DEV(ctx->inst->accel_dev); int ret, ctr = 0; if (req->cryptlen == 0) return 0; - qat_req->iv = dma_alloc_coherent(dev, AES_BLOCK_SIZE, - &qat_req->iv_paddr, GFP_ATOMIC); - if (!qat_req->iv) - return -ENOMEM; - ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req); - if (unlikely(ret)) { - dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv, - qat_req->iv_paddr); + if (unlikely(ret)) return ret; - } msg = &qat_req->req; *msg = ctx->enc_fw_req; @@ -1061,19 +1182,19 @@ static int qat_alg_skcipher_encrypt(struct skcipher_request *req) qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req; qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; + qat_req->encryption = true; cipher_param = (void *)&qat_req->req.serv_specif_rqpars; cipher_param->cipher_length = req->cryptlen; cipher_param->cipher_offset = 0; - cipher_param->u.s.cipher_IV_ptr = qat_req->iv_paddr; - memcpy(qat_req->iv, req->iv, AES_BLOCK_SIZE); + + qat_alg_set_req_iv(qat_req); + do { ret = adf_send_message(ctx->inst->sym_tx, (u32 *)msg); } while (ret == -EAGAIN && ctr++ < 10); if (ret == -EAGAIN) { qat_alg_free_bufl(ctx->inst, qat_req); - dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv, - qat_req->iv_paddr); return -EBUSY; } return -EINPROGRESS; @@ -1113,23 +1234,14 @@ static int qat_alg_skcipher_decrypt(struct skcipher_request *req) struct qat_crypto_request *qat_req = skcipher_request_ctx(req); struct icp_qat_fw_la_cipher_req_params *cipher_param; struct icp_qat_fw_la_bulk_req *msg; - struct device *dev = &GET_DEV(ctx->inst->accel_dev); int ret, ctr = 0; if (req->cryptlen == 0) return 0; - qat_req->iv = dma_alloc_coherent(dev, AES_BLOCK_SIZE, - &qat_req->iv_paddr, GFP_ATOMIC); - if (!qat_req->iv) - return -ENOMEM; - ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req); - if (unlikely(ret)) { - dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv, - qat_req->iv_paddr); + if (unlikely(ret)) return ret; - } msg = &qat_req->req; *msg = ctx->dec_fw_req; @@ -1139,19 +1251,20 @@ static int qat_alg_skcipher_decrypt(struct skcipher_request *req) qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req; qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; + qat_req->encryption = false; cipher_param = (void *)&qat_req->req.serv_specif_rqpars; cipher_param->cipher_length = req->cryptlen; cipher_param->cipher_offset = 0; - cipher_param->u.s.cipher_IV_ptr = qat_req->iv_paddr; - memcpy(qat_req->iv, req->iv, AES_BLOCK_SIZE); + + qat_alg_set_req_iv(qat_req); + qat_alg_update_iv(qat_req); + do { ret = adf_send_message(ctx->inst->sym_tx, (u32 *)msg); } while (ret == -EAGAIN && ctr++ < 10); if (ret == -EAGAIN) { qat_alg_free_bufl(ctx->inst, qat_req); - dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv, - qat_req->iv_paddr); return -EBUSY; } return -EINPROGRESS; @@ -1253,6 +1366,12 @@ static int qat_alg_skcipher_init_xts_tfm(struct crypto_skcipher *tfm) if (IS_ERR(ctx->ftfm)) return PTR_ERR(ctx->ftfm); + ctx->tweak = crypto_alloc_cipher("aes", 0, 0); + if (IS_ERR(ctx->tweak)) { + crypto_free_skcipher(ctx->ftfm); + return PTR_ERR(ctx->tweak); + } + reqsize = max(sizeof(struct qat_crypto_request), sizeof(struct skcipher_request) + crypto_skcipher_reqsize(ctx->ftfm)); @@ -1295,6 +1414,9 @@ static void qat_alg_skcipher_exit_xts_tfm(struct crypto_skcipher *tfm) if (ctx->ftfm) crypto_free_skcipher(ctx->ftfm); + if (ctx->tweak) + crypto_free_cipher(ctx->tweak); + qat_alg_skcipher_exit_tfm(tfm); } diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c index 846569ec9066..2c863d25327a 100644 --- a/drivers/crypto/qat/qat_common/qat_asym_algs.c +++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c @@ -201,12 +201,7 @@ static unsigned long qat_dh_fn_id(unsigned int len, bool g2) return g2 ? PKE_DH_G2_4096 : PKE_DH_4096; default: return 0; - }; -} - -static inline struct qat_dh_ctx *qat_dh_get_params(struct crypto_kpp *tfm) -{ - return kpp_tfm_ctx(tfm); + } } static int qat_dh_compute_value(struct kpp_request *req) @@ -577,7 +572,7 @@ static unsigned long qat_rsa_enc_fn_id(unsigned int len) return PKE_RSA_EP_4096; default: return 0; - }; + } } #define PKE_RSA_DP1_512 0x1c161b3c @@ -606,7 +601,7 @@ static unsigned long qat_rsa_dec_fn_id(unsigned int len) return PKE_RSA_DP1_4096; default: return 0; - }; + } } #define PKE_RSA_DP2_512 0x1c131b57 @@ -635,7 +630,7 @@ static unsigned long qat_rsa_dec_fn_id_crt(unsigned int len) return PKE_RSA_DP2_4096; default: return 0; - }; + } } static int qat_rsa_enc(struct akcipher_request *req) diff --git a/drivers/crypto/qat/qat_common/qat_crypto.c b/drivers/crypto/qat/qat_common/qat_crypto.c index ab621b7dbd20..ece6776fbd53 100644 --- a/drivers/crypto/qat/qat_common/qat_crypto.c +++ b/drivers/crypto/qat/qat_common/qat_crypto.c @@ -115,165 +115,217 @@ struct qat_crypto_instance *qat_crypto_get_instance_node(int node) */ int qat_crypto_dev_config(struct adf_accel_dev *accel_dev) { - int cpus = num_online_cpus(); - int banks = GET_MAX_BANKS(accel_dev); - int instances = min(cpus, banks); char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; - int i; + int banks = GET_MAX_BANKS(accel_dev); + int cpus = num_online_cpus(); unsigned long val; + int instances; + int ret; + int i; - if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC)) + if (adf_hw_dev_has_crypto(accel_dev)) + instances = min(cpus, banks); + else + instances = 0; + + ret = adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC); + if (ret) goto err; - if (adf_cfg_section_add(accel_dev, "Accelerator0")) + + ret = adf_cfg_section_add(accel_dev, "Accelerator0"); + if (ret) goto err; + for (i = 0; i < instances; i++) { val = i; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_BANK_NUM, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_BANK_NUM, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) goto err; snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) goto err; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i); val = 128; - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) goto err; val = 512; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) goto err; val = 0; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) goto err; val = 2; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) goto err; val = 8; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) goto err; val = 10; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) goto err; val = ADF_COALESCING_DEF_TIME; snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i); - if (adf_cfg_add_key_value_param(accel_dev, "Accelerator0", - key, (void *)&val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, "Accelerator0", + key, &val, ADF_DEC); + if (ret) goto err; } val = i; - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - ADF_NUM_CY, (void *)&val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_CY, + &val, ADF_DEC); + if (ret) goto err; set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); return 0; err: dev_err(&GET_DEV(accel_dev), "Failed to start QAT accel dev\n"); - return -EINVAL; + return ret; } EXPORT_SYMBOL_GPL(qat_crypto_dev_config); static int qat_crypto_create_instances(struct adf_accel_dev *accel_dev) { - int i; - unsigned long bank; unsigned long num_inst, num_msg_sym, num_msg_asym; - int msg_size; - struct qat_crypto_instance *inst; char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; + unsigned long sym_bank, asym_bank; + struct qat_crypto_instance *inst; + int msg_size; + int ret; + int i; INIT_LIST_HEAD(&accel_dev->crypto_list); - if (adf_cfg_get_param_value(accel_dev, SEC, ADF_NUM_CY, val)) - return -EFAULT; + ret = adf_cfg_get_param_value(accel_dev, SEC, ADF_NUM_CY, val); + if (ret) + return ret; - if (kstrtoul(val, 0, &num_inst)) - return -EFAULT; + ret = kstrtoul(val, 0, &num_inst); + if (ret) + return ret; for (i = 0; i < num_inst; i++) { inst = kzalloc_node(sizeof(*inst), GFP_KERNEL, dev_to_node(&GET_DEV(accel_dev))); - if (!inst) + if (!inst) { + ret = -ENOMEM; goto err; + } list_add_tail(&inst->list, &accel_dev->crypto_list); inst->id = i; atomic_set(&inst->refctr, 0); inst->accel_dev = accel_dev; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, i); - if (adf_cfg_get_param_value(accel_dev, SEC, key, val)) + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_BANK_NUM, i); + ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); + if (ret) goto err; - if (kstrtoul(val, 10, &bank)) + ret = kstrtoul(val, 10, &sym_bank); + if (ret) goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_BANK_NUM, i); + ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); + if (ret) + goto err; + + ret = kstrtoul(val, 10, &asym_bank); + if (ret) + goto err; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i); - if (adf_cfg_get_param_value(accel_dev, SEC, key, val)) + ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); + if (ret) goto err; - if (kstrtoul(val, 10, &num_msg_sym)) + ret = kstrtoul(val, 10, &num_msg_sym); + if (ret) goto err; num_msg_sym = num_msg_sym >> 1; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i); - if (adf_cfg_get_param_value(accel_dev, SEC, key, val)) + ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); + if (ret) goto err; - if (kstrtoul(val, 10, &num_msg_asym)) + ret = kstrtoul(val, 10, &num_msg_asym); + if (ret) goto err; num_msg_asym = num_msg_asym >> 1; msg_size = ICP_QAT_FW_REQ_DEFAULT_SZ; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i); - if (adf_create_ring(accel_dev, SEC, bank, num_msg_sym, - msg_size, key, NULL, 0, &inst->sym_tx)) + ret = adf_create_ring(accel_dev, SEC, sym_bank, num_msg_sym, + msg_size, key, NULL, 0, &inst->sym_tx); + if (ret) goto err; msg_size = msg_size >> 1; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); - if (adf_create_ring(accel_dev, SEC, bank, num_msg_asym, - msg_size, key, NULL, 0, &inst->pke_tx)) + ret = adf_create_ring(accel_dev, SEC, asym_bank, num_msg_asym, + msg_size, key, NULL, 0, &inst->pke_tx); + if (ret) goto err; msg_size = ICP_QAT_FW_RESP_DEFAULT_SZ; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i); - if (adf_create_ring(accel_dev, SEC, bank, num_msg_sym, - msg_size, key, qat_alg_callback, 0, - &inst->sym_rx)) + ret = adf_create_ring(accel_dev, SEC, sym_bank, num_msg_sym, + msg_size, key, qat_alg_callback, 0, + &inst->sym_rx); + if (ret) goto err; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); - if (adf_create_ring(accel_dev, SEC, bank, num_msg_asym, - msg_size, key, qat_alg_asym_callback, 0, - &inst->pke_rx)) + ret = adf_create_ring(accel_dev, SEC, asym_bank, num_msg_asym, + msg_size, key, qat_alg_asym_callback, 0, + &inst->pke_rx); + if (ret) goto err; } return 0; err: qat_crypto_free_instances(accel_dev); - return -ENOMEM; + return ret; } static int qat_crypto_init(struct adf_accel_dev *accel_dev) diff --git a/drivers/crypto/qat/qat_common/qat_crypto.h b/drivers/crypto/qat/qat_common/qat_crypto.h index 12682d1e9f5f..b6a4c95ae003 100644 --- a/drivers/crypto/qat/qat_common/qat_crypto.h +++ b/drivers/crypto/qat/qat_common/qat_crypto.h @@ -3,6 +3,7 @@ #ifndef _QAT_CRYPTO_INSTANCE_H_ #define _QAT_CRYPTO_INSTANCE_H_ +#include #include #include #include "adf_accel_devices.h" @@ -44,8 +45,29 @@ struct qat_crypto_request { struct qat_crypto_request_buffs buf; void (*cb)(struct icp_qat_fw_la_resp *resp, struct qat_crypto_request *req); - void *iv; - dma_addr_t iv_paddr; + union { + struct { + __be64 iv_hi; + __be64 iv_lo; + }; + u8 iv[AES_BLOCK_SIZE]; + }; + bool encryption; }; +static inline bool adf_hw_dev_has_crypto(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + u32 mask = ~hw_device->accel_capabilities_mask; + + if (mask & ADF_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC) + return false; + if (mask & ADF_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC) + return false; + if (mask & ADF_ACCEL_CAPABILITIES_AUTHENTICATION) + return false; + + return true; +} + #endif diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 6b9d47682d04..bd3028126cbe 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -33,7 +33,7 @@ ((((const_val) << 12) & 0x0FF00000ull) | \ (((const_val) << 0) & 0x000000FFull)))) -#define AE(handle, ae) handle->hal_handle->aes[ae] +#define AE(handle, ae) ((handle)->hal_handle->aes[ae]) static const u64 inst_4b[] = { 0x0F0400C0000ull, 0x0F4400C0000ull, 0x0F040000300ull, 0x0F440000300ull, @@ -150,15 +150,15 @@ static int qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle, return 0; } -#define CLR_BIT(wrd, bit) (wrd & ~(1 << bit)) -#define SET_BIT(wrd, bit) (wrd | 1 << bit) +#define CLR_BIT(wrd, bit) ((wrd) & ~(1 << (bit))) +#define SET_BIT(wrd, bit) ((wrd) | 1 << (bit)) int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle, unsigned char ae, unsigned char mode) { unsigned int csr, new_csr; - if ((mode != 4) && (mode != 8)) { + if (mode != 4 && mode != 8) { pr_err("QAT: bad ctx mode=%d\n", mode); return -EINVAL; } @@ -210,6 +210,16 @@ int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle, SET_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS) : CLR_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS); break; + case ICP_LMEM2: + new_csr = (mode) ? + SET_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS) : + CLR_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS); + break; + case ICP_LMEM3: + new_csr = (mode) ? + SET_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS) : + CLR_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS); + break; default: pr_err("QAT: lmType = 0x%x\n", lm_type); return -EINVAL; @@ -220,6 +230,20 @@ int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle, return 0; } +void qat_hal_set_ae_tindex_mode(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char mode) +{ + unsigned int csr, new_csr; + + csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); + csr &= IGNORE_W1C_MASK; + new_csr = (mode) ? + SET_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS) : + CLR_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS); + if (new_csr != csr) + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); +} + static unsigned short qat_hal_get_reg_addr(unsigned int type, unsigned short reg_num) { @@ -259,6 +283,12 @@ static unsigned short qat_hal_get_reg_addr(unsigned int type, case ICP_LMEM1: reg_addr = 0x220; break; + case ICP_LMEM2: + reg_addr = 0x2c0; + break; + case ICP_LMEM3: + reg_addr = 0x2e0; + break; case ICP_NO_DEST: reg_addr = 0x300 | (reg_num & 0xff); break; @@ -271,12 +301,13 @@ static unsigned short qat_hal_get_reg_addr(unsigned int type, void qat_hal_reset(struct icp_qat_fw_loader_handle *handle) { - unsigned int ae_reset_csr; + unsigned int reset_mask = handle->chip_info->icp_rst_mask; + unsigned int reset_csr = handle->chip_info->icp_rst_csr; + unsigned int csr_val; - ae_reset_csr = GET_GLB_CSR(handle, ICP_RESET); - ae_reset_csr |= handle->hal_handle->ae_mask << RST_CSR_AE_LSB; - ae_reset_csr |= handle->hal_handle->slice_mask << RST_CSR_QAT_LSB; - SET_GLB_CSR(handle, ICP_RESET, ae_reset_csr); + csr_val = GET_CAP_CSR(handle, reset_csr); + csr_val |= reset_mask; + SET_CAP_CSR(handle, reset_csr, csr_val); } static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle, @@ -346,11 +377,12 @@ static void qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle, static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle) { + unsigned long ae_mask = handle->hal_handle->ae_mask; unsigned int base_cnt, cur_cnt; unsigned char ae; int times = MAX_RETRY_TIMES; - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT); base_cnt &= 0xffff; @@ -384,21 +416,23 @@ int qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle, static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle) { - unsigned int misc_ctl; + unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned int misc_ctl_csr, misc_ctl; unsigned char ae; + misc_ctl_csr = handle->chip_info->misc_ctl_csr; /* stop the timestamp timers */ - misc_ctl = GET_GLB_CSR(handle, MISC_CONTROL); + misc_ctl = GET_CAP_CSR(handle, misc_ctl_csr); if (misc_ctl & MC_TIMESTAMP_ENABLE) - SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl & + SET_CAP_CSR(handle, misc_ctl_csr, misc_ctl & (~MC_TIMESTAMP_ENABLE)); - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_LOW, 0); qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0); } /* start timestamp timers */ - SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl | MC_TIMESTAMP_ENABLE); + SET_CAP_CSR(handle, misc_ctl_csr, misc_ctl | MC_TIMESTAMP_ENABLE); } #define ESRAM_AUTO_TINIT BIT(2) @@ -428,7 +462,7 @@ static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle) qat_hal_wait_cycles(handle, 0, ESRAM_AUTO_INIT_USED_CYCLES, 0); csr_val = ADF_CSR_RD(csr_addr, 0); } while (!(csr_val & ESRAM_AUTO_TINIT_DONE) && times--); - if ((times < 0)) { + if (times < 0) { pr_err("QAT: Fail to init eSram!\n"); return -EFAULT; } @@ -438,33 +472,33 @@ static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle) #define SHRAM_INIT_CYCLES 2060 int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle) { - unsigned int ae_reset_csr; - unsigned char ae; - unsigned int clk_csr; + unsigned int clk_csr = handle->chip_info->glb_clk_enable_csr; + unsigned int reset_mask = handle->chip_info->icp_rst_mask; + unsigned int reset_csr = handle->chip_info->icp_rst_csr; + unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned char ae = 0; unsigned int times = 100; - unsigned int csr; + unsigned int csr_val; /* write to the reset csr */ - ae_reset_csr = GET_GLB_CSR(handle, ICP_RESET); - ae_reset_csr &= ~(handle->hal_handle->ae_mask << RST_CSR_AE_LSB); - ae_reset_csr &= ~(handle->hal_handle->slice_mask << RST_CSR_QAT_LSB); + csr_val = GET_CAP_CSR(handle, reset_csr); + csr_val &= ~reset_mask; do { - SET_GLB_CSR(handle, ICP_RESET, ae_reset_csr); + SET_CAP_CSR(handle, reset_csr, csr_val); if (!(times--)) goto out_err; - csr = GET_GLB_CSR(handle, ICP_RESET); - } while ((handle->hal_handle->ae_mask | - (handle->hal_handle->slice_mask << RST_CSR_QAT_LSB)) & csr); + csr_val = GET_CAP_CSR(handle, reset_csr); + csr_val &= reset_mask; + } while (csr_val); /* enable clock */ - clk_csr = GET_GLB_CSR(handle, ICP_GLOBAL_CLK_ENABLE); - clk_csr |= handle->hal_handle->ae_mask << 0; - clk_csr |= handle->hal_handle->slice_mask << 20; - SET_GLB_CSR(handle, ICP_GLOBAL_CLK_ENABLE, clk_csr); + csr_val = GET_CAP_CSR(handle, clk_csr); + csr_val |= reset_mask; + SET_CAP_CSR(handle, clk_csr, csr_val); if (qat_hal_check_ae_alive(handle)) goto out_err; /* Set undefined power-up/reset states to reasonable default values */ - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, INIT_CTX_ENABLE_VALUE); qat_hal_wr_indr_csr(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX, @@ -570,10 +604,11 @@ static void qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle, static void qat_hal_clear_xfer(struct icp_qat_fw_loader_handle *handle) { + unsigned long ae_mask = handle->hal_handle->ae_mask; unsigned char ae; unsigned short reg; - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { for (reg = 0; reg < ICP_QAT_UCLO_MAX_GPR_REG; reg++) { qat_hal_init_rd_xfer(handle, ae, 0, ICP_SR_RD_ABS, reg, 0); @@ -585,6 +620,7 @@ static void qat_hal_clear_xfer(struct icp_qat_fw_loader_handle *handle) static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) { + unsigned long ae_mask = handle->hal_handle->ae_mask; unsigned char ae; unsigned int ctx_mask = ICP_QAT_UCLO_AE_ALL_CTX; int times = MAX_RETRY_TIMES; @@ -592,13 +628,15 @@ static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) unsigned int savctx = 0; int ret = 0; - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL); csr_val &= ~(1 << MMC_SHARE_CS_BITPOS); qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val); csr_val = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); csr_val &= IGNORE_W1C_MASK; - csr_val |= CE_NN_MODE; + if (handle->chip_info->nn) + csr_val |= CE_NN_MODE; + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val); qat_hal_wr_uwords(handle, ae, 0, ARRAY_SIZE(inst), (u64 *)inst); @@ -613,7 +651,7 @@ static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0); qat_hal_enable_ctx(handle, ae, ctx_mask); } - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { /* wait for AE to finish */ do { ret = qat_hal_wait_cycles(handle, ae, 20, 1); @@ -641,57 +679,143 @@ static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) return 0; } -#define ICP_QAT_AE_OFFSET 0x20000 -#define ICP_QAT_CAP_OFFSET (ICP_QAT_AE_OFFSET + 0x10000) -#define LOCAL_TO_XFER_REG_OFFSET 0x800 -#define ICP_QAT_EP_OFFSET 0x3a000 -int qat_hal_init(struct adf_accel_dev *accel_dev) +static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, + struct adf_accel_dev *accel_dev) { - unsigned char ae; - unsigned int max_en_ae_id = 0; - struct icp_qat_fw_loader_handle *handle; struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; struct adf_hw_device_data *hw_data = accel_dev->hw_device; struct adf_bar *misc_bar = &pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)]; + unsigned int max_en_ae_id = 0; struct adf_bar *sram_bar; + unsigned int csr_val = 0; + unsigned long ae_mask; + unsigned char ae = 0; + int ret = 0; - handle = kzalloc(sizeof(*handle), GFP_KERNEL); - if (!handle) - return -ENOMEM; - - handle->hal_cap_g_ctl_csr_addr_v = - (void __iomem *)((uintptr_t)misc_bar->virt_addr + - ICP_QAT_CAP_OFFSET); - handle->hal_cap_ae_xfer_csr_addr_v = - (void __iomem *)((uintptr_t)misc_bar->virt_addr + - ICP_QAT_AE_OFFSET); - handle->hal_ep_csr_addr_v = - (void __iomem *)((uintptr_t)misc_bar->virt_addr + - ICP_QAT_EP_OFFSET); - handle->hal_cap_ae_local_csr_addr_v = - (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + - LOCAL_TO_XFER_REG_OFFSET); handle->pci_dev = pci_info->pci_dev; - if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_DH895XCC) { + switch (handle->pci_dev->device) { + case ADF_4XXX_PCI_DEVICE_ID: + handle->chip_info->sram_visible = false; + handle->chip_info->nn = false; + handle->chip_info->lm2lm3 = true; + handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG_2X; + handle->chip_info->icp_rst_csr = ICP_RESET_CPP0; + handle->chip_info->icp_rst_mask = 0x100015; + handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE_CPP0; + handle->chip_info->misc_ctl_csr = MISC_CONTROL_C4XXX; + handle->chip_info->wakeup_event_val = 0x80000000; + handle->chip_info->fw_auth = true; + handle->chip_info->css_3k = true; + handle->chip_info->tgroup_share_ustore = true; + handle->chip_info->fcu_ctl_csr = FCU_CONTROL_4XXX; + handle->chip_info->fcu_sts_csr = FCU_STATUS_4XXX; + handle->chip_info->fcu_dram_addr_hi = FCU_DRAM_ADDR_HI_4XXX; + handle->chip_info->fcu_dram_addr_lo = FCU_DRAM_ADDR_LO_4XXX; + handle->chip_info->fcu_loaded_ae_csr = FCU_AE_LOADED_4XXX; + handle->chip_info->fcu_loaded_ae_pos = 0; + + handle->hal_cap_g_ctl_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_CAP_OFFSET_4XXX); + handle->hal_cap_ae_xfer_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_AE_OFFSET_4XXX); + handle->hal_ep_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_EP_OFFSET_4XXX); + handle->hal_cap_ae_local_csr_addr_v = + (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + + LOCAL_TO_XFER_REG_OFFSET); + break; + case PCI_DEVICE_ID_INTEL_QAT_C62X: + case PCI_DEVICE_ID_INTEL_QAT_C3XXX: + handle->chip_info->sram_visible = false; + handle->chip_info->nn = true; + handle->chip_info->lm2lm3 = false; + handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; + handle->chip_info->icp_rst_csr = ICP_RESET; + handle->chip_info->icp_rst_mask = (hw_data->ae_mask << RST_CSR_AE_LSB) | + (hw_data->accel_mask << RST_CSR_QAT_LSB); + handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE; + handle->chip_info->misc_ctl_csr = MISC_CONTROL; + handle->chip_info->wakeup_event_val = WAKEUP_EVENT; + handle->chip_info->fw_auth = true; + handle->chip_info->css_3k = false; + handle->chip_info->tgroup_share_ustore = false; + handle->chip_info->fcu_ctl_csr = FCU_CONTROL; + handle->chip_info->fcu_sts_csr = FCU_STATUS; + handle->chip_info->fcu_dram_addr_hi = FCU_DRAM_ADDR_HI; + handle->chip_info->fcu_dram_addr_lo = FCU_DRAM_ADDR_LO; + handle->chip_info->fcu_loaded_ae_csr = FCU_STATUS; + handle->chip_info->fcu_loaded_ae_pos = FCU_LOADED_AE_POS; + handle->hal_cap_g_ctl_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_CAP_OFFSET); + handle->hal_cap_ae_xfer_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_AE_OFFSET); + handle->hal_ep_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_EP_OFFSET); + handle->hal_cap_ae_local_csr_addr_v = + (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + + LOCAL_TO_XFER_REG_OFFSET); + break; + case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: + handle->chip_info->sram_visible = true; + handle->chip_info->nn = true; + handle->chip_info->lm2lm3 = false; + handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; + handle->chip_info->icp_rst_csr = ICP_RESET; + handle->chip_info->icp_rst_mask = (hw_data->ae_mask << RST_CSR_AE_LSB) | + (hw_data->accel_mask << RST_CSR_QAT_LSB); + handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE; + handle->chip_info->misc_ctl_csr = MISC_CONTROL; + handle->chip_info->wakeup_event_val = WAKEUP_EVENT; + handle->chip_info->fw_auth = false; + handle->chip_info->css_3k = false; + handle->chip_info->tgroup_share_ustore = false; + handle->chip_info->fcu_ctl_csr = 0; + handle->chip_info->fcu_sts_csr = 0; + handle->chip_info->fcu_dram_addr_hi = 0; + handle->chip_info->fcu_dram_addr_lo = 0; + handle->chip_info->fcu_loaded_ae_csr = 0; + handle->chip_info->fcu_loaded_ae_pos = 0; + handle->hal_cap_g_ctl_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_CAP_OFFSET); + handle->hal_cap_ae_xfer_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_AE_OFFSET); + handle->hal_ep_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_EP_OFFSET); + handle->hal_cap_ae_local_csr_addr_v = + (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + + LOCAL_TO_XFER_REG_OFFSET); + break; + default: + ret = -EINVAL; + goto out_err; + } + + if (handle->chip_info->sram_visible) { sram_bar = &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)]; handle->hal_sram_addr_v = sram_bar->virt_addr; } - handle->fw_auth = (handle->pci_dev->device == - PCI_DEVICE_ID_INTEL_QAT_DH895XCC) ? false : true; - handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL); - if (!handle->hal_handle) - goto out_hal_handle; handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid; handle->hal_handle->ae_mask = hw_data->ae_mask; + handle->hal_handle->admin_ae_mask = hw_data->admin_ae_mask; handle->hal_handle->slice_mask = hw_data->accel_mask; + handle->cfg_ae_mask = ALL_AE_MASK; /* create AE objects */ handle->hal_handle->upc_mask = 0x1ffff; handle->hal_handle->max_ustore = 0x4000; - for (ae = 0; ae < ICP_QAT_UCLO_MAX_AE; ae++) { - if (!(hw_data->ae_mask & (1 << ae))) - continue; + + ae_mask = handle->hal_handle->ae_mask; + for_each_set_bit(ae, &ae_mask, ICP_QAT_UCLO_MAX_AE) { handle->hal_handle->aes[ae].free_addr = 0; handle->hal_handle->aes[ae].free_size = handle->hal_handle->max_ustore; @@ -702,70 +826,116 @@ int qat_hal_init(struct adf_accel_dev *accel_dev) max_en_ae_id = ae; } handle->hal_handle->ae_max_num = max_en_ae_id + 1; - /* take all AEs out of reset */ - if (qat_hal_clr_reset(handle)) { - dev_err(&GET_DEV(accel_dev), "qat_hal_clr_reset error\n"); - goto out_err; - } - qat_hal_clear_xfer(handle); - if (!handle->fw_auth) { - if (qat_hal_clear_gpr(handle)) - goto out_err; - } /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */ - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { - unsigned int csr_val = 0; - + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { csr_val = qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE); csr_val |= 0x1; qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val); } +out_err: + return ret; +} + +int qat_hal_init(struct adf_accel_dev *accel_dev) +{ + struct icp_qat_fw_loader_handle *handle; + int ret = 0; + + handle = kzalloc(sizeof(*handle), GFP_KERNEL); + if (!handle) + return -ENOMEM; + + handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL); + if (!handle->hal_handle) { + ret = -ENOMEM; + goto out_hal_handle; + } + + handle->chip_info = kzalloc(sizeof(*handle->chip_info), GFP_KERNEL); + if (!handle->chip_info) { + ret = -ENOMEM; + goto out_chip_info; + } + + ret = qat_hal_chip_init(handle, accel_dev); + if (ret) { + dev_err(&GET_DEV(accel_dev), "qat_hal_chip_init error\n"); + goto out_err; + } + + /* take all AEs out of reset */ + ret = qat_hal_clr_reset(handle); + if (ret) { + dev_err(&GET_DEV(accel_dev), "qat_hal_clr_reset error\n"); + goto out_err; + } + + qat_hal_clear_xfer(handle); + if (!handle->chip_info->fw_auth) { + ret = qat_hal_clear_gpr(handle); + if (ret) + goto out_err; + } + accel_dev->fw_loader->fw_loader = handle; return 0; out_err: + kfree(handle->chip_info); +out_chip_info: kfree(handle->hal_handle); out_hal_handle: kfree(handle); - return -EFAULT; + return ret; } void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle) { if (!handle) return; + kfree(handle->chip_info); kfree(handle->hal_handle); kfree(handle); } -void qat_hal_start(struct icp_qat_fw_loader_handle *handle, unsigned char ae, - unsigned int ctx_mask) +int qat_hal_start(struct icp_qat_fw_loader_handle *handle) { + unsigned long ae_mask = handle->hal_handle->ae_mask; + u32 wakeup_val = handle->chip_info->wakeup_event_val; + u32 fcu_ctl_csr, fcu_sts_csr; + unsigned int fcu_sts; + unsigned char ae; + u32 ae_ctr = 0; int retry = 0; - unsigned int fcu_sts = 0; - if (handle->fw_auth) { - SET_CAP_CSR(handle, FCU_CONTROL, FCU_CTRL_CMD_START); + if (handle->chip_info->fw_auth) { + fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; + fcu_sts_csr = handle->chip_info->fcu_sts_csr; + ae_ctr = hweight32(ae_mask); + SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_START); do { msleep(FW_AUTH_WAIT_PERIOD); - fcu_sts = GET_CAP_CSR(handle, FCU_STATUS); + fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); if (((fcu_sts >> FCU_STS_DONE_POS) & 0x1)) - return; + return ae_ctr; } while (retry++ < FW_AUTH_MAX_RETRY); - pr_err("QAT: start error (AE 0x%x FCU_STS = 0x%x)\n", ae, - fcu_sts); + pr_err("QAT: start error (FCU_STS = 0x%x)\n", fcu_sts); + return 0; } else { - qat_hal_put_wakeup_event(handle, ae, (~ctx_mask) & - ICP_QAT_UCLO_AE_ALL_CTX, 0x10000); - qat_hal_enable_ctx(handle, ae, ctx_mask); + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { + qat_hal_put_wakeup_event(handle, ae, 0, wakeup_val); + qat_hal_enable_ctx(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX); + ae_ctr++; + } + return ae_ctr; } } void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae, unsigned int ctx_mask) { - if (!handle->fw_auth) + if (!handle->chip_info->fw_auth) qat_hal_disable_ctx(handle, ae, ctx_mask); } @@ -832,9 +1002,12 @@ static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle, int code_off, unsigned int max_cycle, unsigned int *endpc) { + unsigned int ind_lm_addr_byte0 = 0, ind_lm_addr_byte1 = 0; + unsigned int ind_lm_addr_byte2 = 0, ind_lm_addr_byte3 = 0; + unsigned int ind_t_index = 0, ind_t_index_byte = 0; + unsigned int ind_lm_addr0 = 0, ind_lm_addr1 = 0; + unsigned int ind_lm_addr2 = 0, ind_lm_addr3 = 0; u64 savuwords[MAX_EXEC_INST]; - unsigned int ind_lm_addr0, ind_lm_addr1; - unsigned int ind_lm_addr_byte0, ind_lm_addr_byte1; unsigned int ind_cnt_sig; unsigned int ind_sig, act_sig; unsigned int csr_val = 0, newcsr_val; @@ -853,6 +1026,20 @@ static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle, INDIRECT_LM_ADDR_0_BYTE_INDEX); ind_lm_addr_byte1 = qat_hal_rd_indr_csr(handle, ae, ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX); + if (handle->chip_info->lm2lm3) { + ind_lm_addr2 = qat_hal_rd_indr_csr(handle, ae, ctx, + LM_ADDR_2_INDIRECT); + ind_lm_addr3 = qat_hal_rd_indr_csr(handle, ae, ctx, + LM_ADDR_3_INDIRECT); + ind_lm_addr_byte2 = qat_hal_rd_indr_csr(handle, ae, ctx, + INDIRECT_LM_ADDR_2_BYTE_INDEX); + ind_lm_addr_byte3 = qat_hal_rd_indr_csr(handle, ae, ctx, + INDIRECT_LM_ADDR_3_BYTE_INDEX); + ind_t_index = qat_hal_rd_indr_csr(handle, ae, ctx, + INDIRECT_T_INDEX); + ind_t_index_byte = qat_hal_rd_indr_csr(handle, ae, ctx, + INDIRECT_T_INDEX_BYTE_INDEX); + } if (inst_num <= MAX_EXEC_INST) qat_hal_get_uwords(handle, ae, 0, inst_num, savuwords); qat_hal_get_wakeup_event(handle, ae, ctx, &wakeup_events); @@ -910,6 +1097,23 @@ static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle, INDIRECT_LM_ADDR_0_BYTE_INDEX, ind_lm_addr_byte0); qat_hal_wr_indr_csr(handle, ae, (1 << ctx), INDIRECT_LM_ADDR_1_BYTE_INDEX, ind_lm_addr_byte1); + if (handle->chip_info->lm2lm3) { + qat_hal_wr_indr_csr(handle, ae, BIT(ctx), LM_ADDR_2_INDIRECT, + ind_lm_addr2); + qat_hal_wr_indr_csr(handle, ae, BIT(ctx), LM_ADDR_3_INDIRECT, + ind_lm_addr3); + qat_hal_wr_indr_csr(handle, ae, BIT(ctx), + INDIRECT_LM_ADDR_2_BYTE_INDEX, + ind_lm_addr_byte2); + qat_hal_wr_indr_csr(handle, ae, BIT(ctx), + INDIRECT_LM_ADDR_3_BYTE_INDEX, + ind_lm_addr_byte3); + qat_hal_wr_indr_csr(handle, ae, BIT(ctx), + INDIRECT_T_INDEX, ind_t_index); + qat_hal_wr_indr_csr(handle, ae, BIT(ctx), + INDIRECT_T_INDEX_BYTE_INDEX, + ind_t_index_byte); + } qat_hal_wr_indr_csr(handle, ae, (1 << ctx), FUTURE_COUNT_SIGNAL_INDIRECT, ind_cnt_sig); qat_hal_wr_indr_csr(handle, ae, (1 << ctx), @@ -1125,7 +1329,7 @@ int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle, plm_init = plm_init->next; } /* exec micro codes */ - if (micro_inst_arry && (micro_inst_num > 0)) { + if (micro_inst_arry && micro_inst_num > 0) { micro_inst_arry[micro_inst_num++] = 0x0E000010000ull; stat = qat_hal_exec_micro_init_lm(handle, ae, 0, &first_exec, micro_inst_arry, @@ -1146,7 +1350,7 @@ static int qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle, unsigned short mask; unsigned short dr_offset = 0x10; - status = ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); + ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); if (CE_INUSE_CONTEXTS & ctx_enables) { if (ctx & 0x1) { pr_err("QAT: bad 4-ctx mode,ctx=0x%x\n", ctx); @@ -1271,7 +1475,7 @@ static int qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle } int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx_mask, + unsigned char ae, unsigned long ctx_mask, enum icp_qat_uof_regtype reg_type, unsigned short reg_num, unsigned int regdata) { @@ -1291,7 +1495,7 @@ int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle, } else { reg = reg_num; type = reg_type; - if (!test_bit(ctx, (unsigned long *)&ctx_mask)) + if (!test_bit(ctx, &ctx_mask)) continue; } stat = qat_hal_wr_rel_reg(handle, ae, ctx, type, reg, regdata); @@ -1305,7 +1509,7 @@ int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle, } int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx_mask, + unsigned char ae, unsigned long ctx_mask, enum icp_qat_uof_regtype reg_type, unsigned short reg_num, unsigned int regdata) { @@ -1325,7 +1529,7 @@ int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle, } else { reg = reg_num; type = reg_type; - if (!test_bit(ctx, (unsigned long *)&ctx_mask)) + if (!test_bit(ctx, &ctx_mask)) continue; } stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, type, reg, @@ -1340,7 +1544,7 @@ int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle, } int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx_mask, + unsigned char ae, unsigned long ctx_mask, enum icp_qat_uof_regtype reg_type, unsigned short reg_num, unsigned int regdata) { @@ -1360,7 +1564,7 @@ int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle, } else { reg = reg_num; type = reg_type; - if (!test_bit(ctx, (unsigned long *)&ctx_mask)) + if (!test_bit(ctx, &ctx_mask)) continue; } stat = qat_hal_put_rel_rd_xfer(handle, ae, ctx, type, reg, @@ -1375,17 +1579,22 @@ int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle, } int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx_mask, + unsigned char ae, unsigned long ctx_mask, unsigned short reg_num, unsigned int regdata) { int stat = 0; unsigned char ctx; + if (!handle->chip_info->nn) { + dev_err(&handle->pci_dev->dev, "QAT: No next neigh in 0x%x\n", + handle->pci_dev->device); + return -EINVAL; + } if (ctx_mask == 0) return -EINVAL; for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { - if (!test_bit(ctx, (unsigned long *)&ctx_mask)) + if (!test_bit(ctx, &ctx_mask)) continue; stat = qat_hal_put_rel_nn(handle, ae, ctx, reg_num, regdata); if (stat) { diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 5d1f28cd6680..1fb5fc852f6b 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -74,7 +74,7 @@ static int qat_uclo_free_ae_data(struct icp_qat_uclo_aedata *ae_data) static char *qat_uclo_get_string(struct icp_qat_uof_strtable *str_table, unsigned int str_offset) { - if ((!str_table->table_len) || (str_offset > str_table->table_len)) + if (!str_table->table_len || str_offset > str_table->table_len) return NULL; return (char *)(((uintptr_t)(str_table->strings)) + str_offset); } @@ -311,7 +311,7 @@ static int qat_uclo_init_lmem_seg(struct icp_qat_fw_loader_handle *handle, unsigned int ae; if (qat_uclo_fetch_initmem_ae(handle, init_mem, - ICP_QAT_UCLO_MAX_LMEM_REG, &ae)) + handle->chip_info->lm_size, &ae)) return -EINVAL; if (qat_uclo_create_batch_init_list(handle, init_mem, ae, &obj_handle->lm_init_tab[ae])) @@ -324,6 +324,7 @@ static int qat_uclo_init_umem_seg(struct icp_qat_fw_loader_handle *handle, { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; unsigned int ae, ustore_size, uaddr, i; + struct icp_qat_uclo_aedata *aed; ustore_size = obj_handle->ustore_phy_size; if (qat_uclo_fetch_initmem_ae(handle, init_mem, ustore_size, &ae)) @@ -333,11 +334,10 @@ static int qat_uclo_init_umem_seg(struct icp_qat_fw_loader_handle *handle, return -EINVAL; /* set the highest ustore address referenced */ uaddr = (init_mem->addr + init_mem->num_in_bytes) >> 0x2; - for (i = 0; i < obj_handle->ae_data[ae].slice_num; i++) { - if (obj_handle->ae_data[ae].ae_slices[i]. - encap_image->uwords_num < uaddr) - obj_handle->ae_data[ae].ae_slices[i]. - encap_image->uwords_num = uaddr; + aed = &obj_handle->ae_data[ae]; + for (i = 0; i < aed->slice_num; i++) { + if (aed->ae_slices[i].encap_image->uwords_num < uaddr) + aed->ae_slices[i].encap_image->uwords_num = uaddr; } return 0; } @@ -373,6 +373,8 @@ static int qat_uclo_init_ustore(struct icp_qat_fw_loader_handle *handle, unsigned int ustore_size; unsigned int patt_pos; struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned long cfg_ae_mask = handle->cfg_ae_mask; u64 *fill_data; uof_image = image->img_ptr; @@ -385,9 +387,13 @@ static int qat_uclo_init_ustore(struct icp_qat_fw_loader_handle *handle, sizeof(u64)); page = image->page; - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { if (!test_bit(ae, (unsigned long *)&uof_image->ae_assigned)) continue; + + if (!test_bit(ae, &cfg_ae_mask)) + continue; + ustore_size = obj_handle->ae_data[ae].eff_ustore_size; patt_pos = page->beg_addr_p + page->micro_words_num; @@ -406,6 +412,7 @@ static int qat_uclo_init_memory(struct icp_qat_fw_loader_handle *handle) int i, ae; struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; struct icp_qat_uof_initmem *initmem = obj_handle->init_mem_tab.init_mem; + unsigned long ae_mask = handle->hal_handle->ae_mask; for (i = 0; i < obj_handle->init_mem_tab.entry_num; i++) { if (initmem->num_in_bytes) { @@ -418,7 +425,8 @@ static int qat_uclo_init_memory(struct icp_qat_fw_loader_handle *handle) (sizeof(struct icp_qat_uof_memvar_attr) * initmem->val_attr_num)); } - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { if (qat_hal_batch_wr_lm(handle, ae, obj_handle->lm_init_tab[ae])) { pr_err("QAT: fail to batch init lmem for AE %d\n", ae); @@ -536,7 +544,7 @@ qat_uclo_check_image_compat(struct icp_qat_uof_encap_obj *encap_uof_obj, (encap_uof_obj->beg_uof + code_page->neigh_reg_tab_offset); if (neigh_reg_tab->entry_num) { - pr_err("QAT: UOF can't contain shared control store feature\n"); + pr_err("QAT: UOF can't contain neighbor register table\n"); return -EINVAL; } if (image->numpages > 1) { @@ -649,11 +657,13 @@ static int qat_uclo_map_ae(struct icp_qat_fw_loader_handle *handle, int max_ae) int i, ae; int mflag = 0; struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned long cfg_ae_mask = handle->cfg_ae_mask; - for (ae = 0; ae < max_ae; ae++) { - if (!test_bit(ae, - (unsigned long *)&handle->hal_handle->ae_mask)) + for_each_set_bit(ae, &ae_mask, max_ae) { + if (!test_bit(ae, &cfg_ae_mask)) continue; + for (i = 0; i < obj_handle->uimage_num; i++) { if (!test_bit(ae, (unsigned long *) &obj_handle->ae_uimage[i].img_ptr->ae_assigned)) @@ -718,6 +728,8 @@ qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle) return ICP_QAT_AC_C62X_DEV_TYPE; case PCI_DEVICE_ID_INTEL_QAT_C3XXX: return ICP_QAT_AC_C3XXX_DEV_TYPE; + case ADF_4XXX_PCI_DEVICE_ID: + return ICP_QAT_AC_4XXX_A_DEV_TYPE; default: pr_err("QAT: unsupported device 0x%x\n", handle->pci_dev->device); @@ -736,8 +748,8 @@ static int qat_uclo_check_uof_compat(struct icp_qat_uclo_objhandle *obj_handle) return -EINVAL; } maj_ver = obj_handle->prod_rev & 0xff; - if ((obj_handle->encap_uof_obj.obj_hdr->max_cpu_ver < maj_ver) || - (obj_handle->encap_uof_obj.obj_hdr->min_cpu_ver > maj_ver)) { + if (obj_handle->encap_uof_obj.obj_hdr->max_cpu_ver < maj_ver || + obj_handle->encap_uof_obj.obj_hdr->min_cpu_ver > maj_ver) { pr_err("QAT: UOF majVer 0x%x out of range\n", maj_ver); return -EINVAL; } @@ -845,6 +857,8 @@ static int qat_uclo_init_reg_sym(struct icp_qat_fw_loader_handle *handle, static int qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle) { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned long ae_mask = handle->hal_handle->ae_mask; + struct icp_qat_uclo_aedata *aed; unsigned int s, ae; if (obj_handle->global_inited) @@ -855,13 +869,13 @@ static int qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle) return -EINVAL; } } - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { - for (s = 0; s < obj_handle->ae_data[ae].slice_num; s++) { - if (!obj_handle->ae_data[ae].ae_slices[s].encap_image) + + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { + aed = &obj_handle->ae_data[ae]; + for (s = 0; s < aed->slice_num; s++) { + if (!aed->ae_slices[s].encap_image) continue; - if (qat_uclo_init_reg_sym(handle, ae, - obj_handle->ae_data[ae]. - ae_slices[s].encap_image)) + if (qat_uclo_init_reg_sym(handle, ae, aed->ae_slices[s].encap_image)) return -EINVAL; } } @@ -869,46 +883,83 @@ static int qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle) return 0; } +static int qat_hal_set_modes(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_uclo_objhandle *obj_handle, + unsigned char ae, + struct icp_qat_uof_image *uof_image) +{ + unsigned char mode; + int ret; + + mode = ICP_QAT_CTX_MODE(uof_image->ae_mode); + ret = qat_hal_set_ae_ctx_mode(handle, ae, mode); + if (ret) { + pr_err("QAT: qat_hal_set_ae_ctx_mode error\n"); + return ret; + } + if (handle->chip_info->nn) { + mode = ICP_QAT_NN_MODE(uof_image->ae_mode); + ret = qat_hal_set_ae_nn_mode(handle, ae, mode); + if (ret) { + pr_err("QAT: qat_hal_set_ae_nn_mode error\n"); + return ret; + } + } + mode = ICP_QAT_LOC_MEM0_MODE(uof_image->ae_mode); + ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM0, mode); + if (ret) { + pr_err("QAT: qat_hal_set_ae_lm_mode LMEM0 error\n"); + return ret; + } + mode = ICP_QAT_LOC_MEM1_MODE(uof_image->ae_mode); + ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM1, mode); + if (ret) { + pr_err("QAT: qat_hal_set_ae_lm_mode LMEM1 error\n"); + return ret; + } + if (handle->chip_info->lm2lm3) { + mode = ICP_QAT_LOC_MEM2_MODE(uof_image->ae_mode); + ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM2, mode); + if (ret) { + pr_err("QAT: qat_hal_set_ae_lm_mode LMEM2 error\n"); + return ret; + } + mode = ICP_QAT_LOC_MEM3_MODE(uof_image->ae_mode); + ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM3, mode); + if (ret) { + pr_err("QAT: qat_hal_set_ae_lm_mode LMEM3 error\n"); + return ret; + } + mode = ICP_QAT_LOC_TINDEX_MODE(uof_image->ae_mode); + qat_hal_set_ae_tindex_mode(handle, ae, mode); + } + return 0; +} + static int qat_uclo_set_ae_mode(struct icp_qat_fw_loader_handle *handle) { - unsigned char ae, nn_mode, s; struct icp_qat_uof_image *uof_image; struct icp_qat_uclo_aedata *ae_data; struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned long cfg_ae_mask = handle->cfg_ae_mask; + unsigned char ae, s; + int error; - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { - if (!test_bit(ae, - (unsigned long *)&handle->hal_handle->ae_mask)) + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { + if (!test_bit(ae, &cfg_ae_mask)) continue; + ae_data = &obj_handle->ae_data[ae]; for (s = 0; s < min_t(unsigned int, ae_data->slice_num, ICP_QAT_UCLO_MAX_CTX); s++) { if (!obj_handle->ae_data[ae].ae_slices[s].encap_image) continue; uof_image = ae_data->ae_slices[s].encap_image->img_ptr; - if (qat_hal_set_ae_ctx_mode(handle, ae, - (char)ICP_QAT_CTX_MODE - (uof_image->ae_mode))) { - pr_err("QAT: qat_hal_set_ae_ctx_mode error\n"); - return -EFAULT; - } - nn_mode = ICP_QAT_NN_MODE(uof_image->ae_mode); - if (qat_hal_set_ae_nn_mode(handle, ae, nn_mode)) { - pr_err("QAT: qat_hal_set_ae_nn_mode error\n"); - return -EFAULT; - } - if (qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM0, - (char)ICP_QAT_LOC_MEM0_MODE - (uof_image->ae_mode))) { - pr_err("QAT: qat_hal_set_ae_lm_mode LMEM0 error\n"); - return -EFAULT; - } - if (qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM1, - (char)ICP_QAT_LOC_MEM1_MODE - (uof_image->ae_mode))) { - pr_err("QAT: qat_hal_set_ae_lm_mode LMEM1 error\n"); - return -EFAULT; - } + error = qat_hal_set_modes(handle, obj_handle, ae, + uof_image); + if (error) + return error; } } return 0; @@ -1003,10 +1054,11 @@ static int qat_uclo_map_suof_file_hdr(struct icp_qat_fw_loader_handle *handle, return 0; } -static void qat_uclo_map_simg(struct icp_qat_suof_handle *suof_handle, +static void qat_uclo_map_simg(struct icp_qat_fw_loader_handle *handle, struct icp_qat_suof_img_hdr *suof_img_hdr, struct icp_qat_suof_chunk_hdr *suof_chunk_hdr) { + struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; struct icp_qat_simg_ae_mode *ae_mode; struct icp_qat_suof_objhdr *suof_objhdr; @@ -1021,10 +1073,10 @@ static void qat_uclo_map_simg(struct icp_qat_suof_handle *suof_handle, suof_img_hdr->css_key = (suof_img_hdr->css_header + sizeof(struct icp_qat_css_hdr)); suof_img_hdr->css_signature = suof_img_hdr->css_key + - ICP_QAT_CSS_FWSK_MODULUS_LEN + - ICP_QAT_CSS_FWSK_EXPONENT_LEN; + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + + ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle); suof_img_hdr->css_simg = suof_img_hdr->css_signature + - ICP_QAT_CSS_SIGNATURE_LEN; + ICP_QAT_CSS_SIGNATURE_LEN(handle); ae_mode = (struct icp_qat_simg_ae_mode *)(suof_img_hdr->css_simg); suof_img_hdr->ae_mask = ae_mode->ae_mask; @@ -1064,8 +1116,8 @@ static int qat_uclo_check_simg_compat(struct icp_qat_fw_loader_handle *handle, return -EINVAL; } maj_ver = prod_rev & 0xff; - if ((maj_ver > img_ae_mode->devmax_ver) || - (maj_ver < img_ae_mode->devmin_ver)) { + if (maj_ver > img_ae_mode->devmax_ver || + maj_ver < img_ae_mode->devmin_ver) { pr_err("QAT: incompatible device majver 0x%x\n", maj_ver); return -EINVAL; } @@ -1108,7 +1160,7 @@ static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, unsigned int i = 0; struct icp_qat_suof_img_hdr img_header; - if (!suof_ptr || (suof_size == 0)) { + if (!suof_ptr || suof_size == 0) { pr_err("QAT: input parameter SUOF pointer/size is NULL\n"); return -EINVAL; } @@ -1130,20 +1182,24 @@ static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, if (!suof_img_hdr) return -ENOMEM; suof_handle->img_table.simg_hdr = suof_img_hdr; - } - for (i = 0; i < suof_handle->img_table.num_simgs; i++) { - qat_uclo_map_simg(handle->sobj_handle, &suof_img_hdr[i], - &suof_chunk_hdr[1 + i]); - ret = qat_uclo_check_simg_compat(handle, - &suof_img_hdr[i]); - if (ret) - return ret; - if ((suof_img_hdr[i].ae_mask & 0x1) != 0) - ae0_img = i; + for (i = 0; i < suof_handle->img_table.num_simgs; i++) { + qat_uclo_map_simg(handle, &suof_img_hdr[i], + &suof_chunk_hdr[1 + i]); + ret = qat_uclo_check_simg_compat(handle, + &suof_img_hdr[i]); + if (ret) + return ret; + suof_img_hdr[i].ae_mask &= handle->cfg_ae_mask; + if ((suof_img_hdr[i].ae_mask & 0x1) != 0) + ae0_img = i; + } + + if (!handle->chip_info->tgroup_share_ustore) { + qat_uclo_tail_img(suof_img_hdr, ae0_img, + suof_handle->img_table.num_simgs); + } } - qat_uclo_tail_img(suof_img_hdr, ae0_img, - suof_handle->img_table.num_simgs); return 0; } @@ -1153,18 +1209,26 @@ static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, static int qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle, struct icp_qat_fw_auth_desc *desc) { - unsigned int fcu_sts, retry = 0; + u32 fcu_sts, retry = 0; + u32 fcu_ctl_csr, fcu_sts_csr; + u32 fcu_dram_hi_csr, fcu_dram_lo_csr; u64 bus_addr; bus_addr = ADD_ADDR(desc->css_hdr_high, desc->css_hdr_low) - sizeof(struct icp_qat_auth_chunk); - SET_CAP_CSR(handle, FCU_DRAM_ADDR_HI, (bus_addr >> BITS_IN_DWORD)); - SET_CAP_CSR(handle, FCU_DRAM_ADDR_LO, bus_addr); - SET_CAP_CSR(handle, FCU_CONTROL, FCU_CTRL_CMD_AUTH); + + fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; + fcu_sts_csr = handle->chip_info->fcu_sts_csr; + fcu_dram_hi_csr = handle->chip_info->fcu_dram_addr_hi; + fcu_dram_lo_csr = handle->chip_info->fcu_dram_addr_lo; + + SET_CAP_CSR(handle, fcu_dram_hi_csr, (bus_addr >> BITS_IN_DWORD)); + SET_CAP_CSR(handle, fcu_dram_lo_csr, bus_addr); + SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_AUTH); do { msleep(FW_AUTH_WAIT_PERIOD); - fcu_sts = GET_CAP_CSR(handle, FCU_STATUS); + fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_FAIL) goto auth_fail; if (((fcu_sts >> FCU_STS_AUTHFWLD_POS) & 0x1)) @@ -1177,6 +1241,83 @@ auth_fail: return -EINVAL; } +static bool qat_uclo_is_broadcast(struct icp_qat_fw_loader_handle *handle, + int imgid) +{ + struct icp_qat_suof_handle *sobj_handle; + + if (!handle->chip_info->tgroup_share_ustore) + return false; + + sobj_handle = (struct icp_qat_suof_handle *)handle->sobj_handle; + if (handle->hal_handle->admin_ae_mask & + sobj_handle->img_table.simg_hdr[imgid].ae_mask) + return false; + + return true; +} + +static int qat_uclo_broadcast_load_fw(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_fw_auth_desc *desc) +{ + unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned long desc_ae_mask = desc->ae_mask; + u32 fcu_sts, ae_broadcast_mask = 0; + u32 fcu_loaded_csr, ae_loaded; + u32 fcu_sts_csr, fcu_ctl_csr; + unsigned int ae, retry = 0; + + if (handle->chip_info->tgroup_share_ustore) { + fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; + fcu_sts_csr = handle->chip_info->fcu_sts_csr; + fcu_loaded_csr = handle->chip_info->fcu_loaded_ae_csr; + } else { + pr_err("Chip 0x%x doesn't support broadcast load\n", + handle->pci_dev->device); + return -EINVAL; + } + + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { + if (qat_hal_check_ae_active(handle, (unsigned char)ae)) { + pr_err("QAT: Broadcast load failed. AE is not enabled or active.\n"); + return -EINVAL; + } + + if (test_bit(ae, &desc_ae_mask)) + ae_broadcast_mask |= 1 << ae; + } + + if (ae_broadcast_mask) { + SET_CAP_CSR(handle, FCU_ME_BROADCAST_MASK_TYPE, + ae_broadcast_mask); + + SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_LOAD); + + do { + msleep(FW_AUTH_WAIT_PERIOD); + fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); + fcu_sts &= FCU_AUTH_STS_MASK; + + if (fcu_sts == FCU_STS_LOAD_FAIL) { + pr_err("Broadcast load failed: 0x%x)\n", fcu_sts); + return -EINVAL; + } else if (fcu_sts == FCU_STS_LOAD_DONE) { + ae_loaded = GET_CAP_CSR(handle, fcu_loaded_csr); + ae_loaded >>= handle->chip_info->fcu_loaded_ae_pos; + + if ((ae_loaded & ae_broadcast_mask) == ae_broadcast_mask) + break; + } + } while (retry++ < FW_AUTH_MAX_RETRY); + + if (retry > FW_AUTH_MAX_RETRY) { + pr_err("QAT: broadcast load failed timeout %d\n", retry); + return -EINVAL; + } + } + return 0; +} + static int qat_uclo_simg_alloc(struct icp_qat_fw_loader_handle *handle, struct icp_firml_dram_desc *dram_desc, unsigned int size) @@ -1197,11 +1338,15 @@ static int qat_uclo_simg_alloc(struct icp_qat_fw_loader_handle *handle, static void qat_uclo_simg_free(struct icp_qat_fw_loader_handle *handle, struct icp_firml_dram_desc *dram_desc) { - dma_free_coherent(&handle->pci_dev->dev, - (size_t)(dram_desc->dram_size), - (dram_desc->dram_base_addr_v), - dram_desc->dram_bus_addr); - memset(dram_desc, 0, sizeof(*dram_desc)); + if (handle && dram_desc && dram_desc->dram_base_addr_v) { + dma_free_coherent(&handle->pci_dev->dev, + (size_t)(dram_desc->dram_size), + dram_desc->dram_base_addr_v, + dram_desc->dram_bus_addr); + } + + if (dram_desc) + memset(dram_desc, 0, sizeof(*dram_desc)); } static void qat_uclo_ummap_auth_fw(struct icp_qat_fw_loader_handle *handle, @@ -1209,12 +1354,14 @@ static void qat_uclo_ummap_auth_fw(struct icp_qat_fw_loader_handle *handle, { struct icp_firml_dram_desc dram_desc; - dram_desc.dram_base_addr_v = *desc; - dram_desc.dram_bus_addr = ((struct icp_qat_auth_chunk *) - (*desc))->chunk_bus_addr; - dram_desc.dram_size = ((struct icp_qat_auth_chunk *) - (*desc))->chunk_size; - qat_uclo_simg_free(handle, &dram_desc); + if (*desc) { + dram_desc.dram_base_addr_v = *desc; + dram_desc.dram_bus_addr = ((struct icp_qat_auth_chunk *) + (*desc))->chunk_bus_addr; + dram_desc.dram_size = ((struct icp_qat_auth_chunk *) + (*desc))->chunk_size; + qat_uclo_simg_free(handle, &dram_desc); + } } static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, @@ -1226,15 +1373,16 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, struct icp_qat_auth_chunk *auth_chunk; u64 virt_addr, bus_addr, virt_base; unsigned int length, simg_offset = sizeof(*auth_chunk); + struct icp_qat_simg_ae_mode *simg_ae_mode; struct icp_firml_dram_desc img_desc; - if (size > (ICP_QAT_AE_IMG_OFFSET + ICP_QAT_CSS_MAX_IMAGE_LEN)) { + if (size > (ICP_QAT_AE_IMG_OFFSET(handle) + ICP_QAT_CSS_MAX_IMAGE_LEN)) { pr_err("QAT: error, input image size overflow %d\n", size); return -EINVAL; } length = (css_hdr->fw_type == CSS_AE_FIRMWARE) ? - ICP_QAT_CSS_AE_SIMG_LEN + simg_offset : - size + ICP_QAT_CSS_FWSK_PAD_LEN + simg_offset; + ICP_QAT_CSS_AE_SIMG_LEN(handle) + simg_offset : + size + ICP_QAT_CSS_FWSK_PAD_LEN(handle) + simg_offset; if (qat_uclo_simg_alloc(handle, &img_desc, length)) { pr_err("QAT: error, allocate continuous dram fail\n"); return -ENOMEM; @@ -1261,42 +1409,42 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, memcpy((void *)(uintptr_t)virt_addr, (void *)(image + sizeof(*css_hdr)), - ICP_QAT_CSS_FWSK_MODULUS_LEN); + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)); /* padding */ - memset((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN), - 0, ICP_QAT_CSS_FWSK_PAD_LEN); + memset((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)), + 0, ICP_QAT_CSS_FWSK_PAD_LEN(handle)); /* exponent */ - memcpy((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN + - ICP_QAT_CSS_FWSK_PAD_LEN), + memcpy((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + + ICP_QAT_CSS_FWSK_PAD_LEN(handle)), (void *)(image + sizeof(*css_hdr) + - ICP_QAT_CSS_FWSK_MODULUS_LEN), + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)), sizeof(unsigned int)); /* signature */ bus_addr = ADD_ADDR(auth_desc->fwsk_pub_high, auth_desc->fwsk_pub_low) + - ICP_QAT_CSS_FWSK_PUB_LEN; - virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN; + ICP_QAT_CSS_FWSK_PUB_LEN(handle); + virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN(handle); auth_desc->signature_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); auth_desc->signature_low = (unsigned int)bus_addr; memcpy((void *)(uintptr_t)virt_addr, (void *)(image + sizeof(*css_hdr) + - ICP_QAT_CSS_FWSK_MODULUS_LEN + - ICP_QAT_CSS_FWSK_EXPONENT_LEN), - ICP_QAT_CSS_SIGNATURE_LEN); + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + + ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle)), + ICP_QAT_CSS_SIGNATURE_LEN(handle)); bus_addr = ADD_ADDR(auth_desc->signature_high, auth_desc->signature_low) + - ICP_QAT_CSS_SIGNATURE_LEN; - virt_addr += ICP_QAT_CSS_SIGNATURE_LEN; + ICP_QAT_CSS_SIGNATURE_LEN(handle); + virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle); auth_desc->img_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); auth_desc->img_low = (unsigned int)bus_addr; - auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET; + auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET(handle); memcpy((void *)(uintptr_t)virt_addr, - (void *)(image + ICP_QAT_AE_IMG_OFFSET), + (void *)(image + ICP_QAT_AE_IMG_OFFSET(handle)), auth_desc->img_len); virt_addr = virt_base; /* AE firmware */ @@ -1315,6 +1463,11 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, auth_desc->img_ae_insts_high = (unsigned int) (bus_addr >> BITS_IN_DWORD); auth_desc->img_ae_insts_low = (unsigned int)bus_addr; + virt_addr += sizeof(struct icp_qat_css_hdr); + virt_addr += ICP_QAT_CSS_FWSK_PUB_LEN(handle); + virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle); + simg_ae_mode = (struct icp_qat_simg_ae_mode *)(uintptr_t)virt_addr; + auth_desc->ae_mask = simg_ae_mode->ae_mask & handle->cfg_ae_mask; } else { auth_desc->img_ae_insts_high = auth_desc->img_high; auth_desc->img_ae_insts_low = auth_desc->img_low; @@ -1326,35 +1479,40 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, struct icp_qat_fw_auth_desc *desc) { + unsigned long ae_mask = handle->hal_handle->ae_mask; + u32 fcu_sts_csr, fcu_ctl_csr; + u32 loaded_aes, loaded_csr; unsigned int i; - unsigned int fcu_sts; - struct icp_qat_simg_ae_mode *virt_addr; - unsigned int fcu_loaded_ae_pos = FCU_LOADED_AE_POS; + u32 fcu_sts; - virt_addr = (void *)((uintptr_t)desc + - sizeof(struct icp_qat_auth_chunk) + - sizeof(struct icp_qat_css_hdr) + - ICP_QAT_CSS_FWSK_PUB_LEN + - ICP_QAT_CSS_SIGNATURE_LEN); - for (i = 0; i < handle->hal_handle->ae_max_num; i++) { + fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; + fcu_sts_csr = handle->chip_info->fcu_sts_csr; + loaded_csr = handle->chip_info->fcu_loaded_ae_csr; + + for_each_set_bit(i, &ae_mask, handle->hal_handle->ae_max_num) { int retry = 0; - if (!((virt_addr->ae_mask >> i) & 0x1)) + if (!((desc->ae_mask >> i) & 0x1)) continue; if (qat_hal_check_ae_active(handle, i)) { pr_err("QAT: AE %d is active\n", i); return -EINVAL; } - SET_CAP_CSR(handle, FCU_CONTROL, - (FCU_CTRL_CMD_LOAD | (i << FCU_CTRL_AE_POS))); + SET_CAP_CSR(handle, fcu_ctl_csr, + (FCU_CTRL_CMD_LOAD | + (1 << FCU_CTRL_BROADCAST_POS) | + (i << FCU_CTRL_AE_POS))); do { msleep(FW_AUTH_WAIT_PERIOD); - fcu_sts = GET_CAP_CSR(handle, FCU_STATUS); - if (((fcu_sts & FCU_AUTH_STS_MASK) == - FCU_STS_LOAD_DONE) && - ((fcu_sts >> fcu_loaded_ae_pos) & (1 << i))) - break; + fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); + if ((fcu_sts & FCU_AUTH_STS_MASK) == + FCU_STS_LOAD_DONE) { + loaded_aes = GET_CAP_CSR(handle, loaded_csr); + loaded_aes >>= handle->chip_info->fcu_loaded_ae_pos; + if (loaded_aes & (1 << i)) + break; + } } while (retry++ < FW_AUTH_MAX_RETRY); if (retry > FW_AUTH_MAX_RETRY) { pr_err("QAT: firmware load failed timeout %x\n", retry); @@ -1387,14 +1545,16 @@ int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, struct icp_qat_fw_auth_desc *desc = NULL; int status = 0; - if (handle->fw_auth) { + if (handle->chip_info->fw_auth) { if (!qat_uclo_map_auth_fw(handle, addr_ptr, mem_size, &desc)) status = qat_uclo_auth_fw(handle, desc); qat_uclo_ummap_auth_fw(handle, &desc); } else { - if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_C3XXX) { - pr_err("QAT: C3XXX doesn't support unsigned MMP\n"); - return -EINVAL; + if (!handle->chip_info->sram_visible) { + dev_dbg(&handle->pci_dev->dev, + "QAT MMP fw not loaded for device 0x%x", + handle->pci_dev->device); + return status; } qat_uclo_wr_sram_by_words(handle, 0, addr_ptr, mem_size); } @@ -1437,25 +1597,281 @@ out_objbuf_err: return -ENOMEM; } -int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle, - void *addr_ptr, int mem_size) +static int qat_uclo_map_mof_file_hdr(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_mof_file_hdr *mof_ptr, + u32 mof_size) { + struct icp_qat_mof_handle *mobj_handle = handle->mobj_handle; + unsigned int min_ver_offset; + unsigned int checksum; + + mobj_handle->file_id = ICP_QAT_MOF_FID; + mobj_handle->mof_buf = (char *)mof_ptr; + mobj_handle->mof_size = mof_size; + + min_ver_offset = mof_size - offsetof(struct icp_qat_mof_file_hdr, + min_ver); + checksum = qat_uclo_calc_str_checksum(&mof_ptr->min_ver, + min_ver_offset); + if (checksum != mof_ptr->checksum) { + pr_err("QAT: incorrect MOF checksum\n"); + return -EINVAL; + } + + mobj_handle->checksum = mof_ptr->checksum; + mobj_handle->min_ver = mof_ptr->min_ver; + mobj_handle->maj_ver = mof_ptr->maj_ver; + return 0; +} + +static void qat_uclo_del_mof(struct icp_qat_fw_loader_handle *handle) +{ + struct icp_qat_mof_handle *mobj_handle = handle->mobj_handle; + + kfree(mobj_handle->obj_table.obj_hdr); + mobj_handle->obj_table.obj_hdr = NULL; + kfree(handle->mobj_handle); + handle->mobj_handle = NULL; +} + +static int qat_uclo_seek_obj_inside_mof(struct icp_qat_mof_handle *mobj_handle, + char *obj_name, char **obj_ptr, + unsigned int *obj_size) +{ + struct icp_qat_mof_objhdr *obj_hdr = mobj_handle->obj_table.obj_hdr; + unsigned int i; + + for (i = 0; i < mobj_handle->obj_table.num_objs; i++) { + if (!strncmp(obj_hdr[i].obj_name, obj_name, + ICP_QAT_SUOF_OBJ_NAME_LEN)) { + *obj_ptr = obj_hdr[i].obj_buf; + *obj_size = obj_hdr[i].obj_size; + return 0; + } + } + + pr_err("QAT: object %s is not found inside MOF\n", obj_name); + return -EINVAL; +} + +static int qat_uclo_map_obj_from_mof(struct icp_qat_mof_handle *mobj_handle, + struct icp_qat_mof_objhdr *mobj_hdr, + struct icp_qat_mof_obj_chunkhdr *obj_chunkhdr) +{ + u8 *obj; + + if (!strncmp(obj_chunkhdr->chunk_id, ICP_QAT_UOF_IMAG, + ICP_QAT_MOF_OBJ_CHUNKID_LEN)) { + obj = mobj_handle->uobjs_hdr + obj_chunkhdr->offset; + } else if (!strncmp(obj_chunkhdr->chunk_id, ICP_QAT_SUOF_IMAG, + ICP_QAT_MOF_OBJ_CHUNKID_LEN)) { + obj = mobj_handle->sobjs_hdr + obj_chunkhdr->offset; + } else { + pr_err("QAT: unsupported chunk id\n"); + return -EINVAL; + } + mobj_hdr->obj_buf = obj; + mobj_hdr->obj_size = (unsigned int)obj_chunkhdr->size; + mobj_hdr->obj_name = obj_chunkhdr->name + mobj_handle->sym_str; + return 0; +} + +static int qat_uclo_map_objs_from_mof(struct icp_qat_mof_handle *mobj_handle) +{ + struct icp_qat_mof_obj_chunkhdr *uobj_chunkhdr; + struct icp_qat_mof_obj_chunkhdr *sobj_chunkhdr; + struct icp_qat_mof_obj_hdr *uobj_hdr; + struct icp_qat_mof_obj_hdr *sobj_hdr; + struct icp_qat_mof_objhdr *mobj_hdr; + unsigned int uobj_chunk_num = 0; + unsigned int sobj_chunk_num = 0; + unsigned int *valid_chunk; + int ret, i; + + uobj_hdr = (struct icp_qat_mof_obj_hdr *)mobj_handle->uobjs_hdr; + sobj_hdr = (struct icp_qat_mof_obj_hdr *)mobj_handle->sobjs_hdr; + if (uobj_hdr) + uobj_chunk_num = uobj_hdr->num_chunks; + if (sobj_hdr) + sobj_chunk_num = sobj_hdr->num_chunks; + + mobj_hdr = kzalloc((uobj_chunk_num + sobj_chunk_num) * + sizeof(*mobj_hdr), GFP_KERNEL); + if (!mobj_hdr) + return -ENOMEM; + + mobj_handle->obj_table.obj_hdr = mobj_hdr; + valid_chunk = &mobj_handle->obj_table.num_objs; + uobj_chunkhdr = (struct icp_qat_mof_obj_chunkhdr *) + ((uintptr_t)uobj_hdr + sizeof(*uobj_hdr)); + sobj_chunkhdr = (struct icp_qat_mof_obj_chunkhdr *) + ((uintptr_t)sobj_hdr + sizeof(*sobj_hdr)); + + /* map uof objects */ + for (i = 0; i < uobj_chunk_num; i++) { + ret = qat_uclo_map_obj_from_mof(mobj_handle, + &mobj_hdr[*valid_chunk], + &uobj_chunkhdr[i]); + if (ret) + return ret; + (*valid_chunk)++; + } + + /* map suof objects */ + for (i = 0; i < sobj_chunk_num; i++) { + ret = qat_uclo_map_obj_from_mof(mobj_handle, + &mobj_hdr[*valid_chunk], + &sobj_chunkhdr[i]); + if (ret) + return ret; + (*valid_chunk)++; + } + + if ((uobj_chunk_num + sobj_chunk_num) != *valid_chunk) { + pr_err("QAT: inconsistent UOF/SUOF chunk amount\n"); + return -EINVAL; + } + return 0; +} + +static void qat_uclo_map_mof_symobjs(struct icp_qat_mof_handle *mobj_handle, + struct icp_qat_mof_chunkhdr *mof_chunkhdr) +{ + char **sym_str = (char **)&mobj_handle->sym_str; + unsigned int *sym_size = &mobj_handle->sym_size; + struct icp_qat_mof_str_table *str_table_obj; + + *sym_size = *(unsigned int *)(uintptr_t) + (mof_chunkhdr->offset + mobj_handle->mof_buf); + *sym_str = (char *)(uintptr_t) + (mobj_handle->mof_buf + mof_chunkhdr->offset + + sizeof(str_table_obj->tab_len)); +} + +static void qat_uclo_map_mof_chunk(struct icp_qat_mof_handle *mobj_handle, + struct icp_qat_mof_chunkhdr *mof_chunkhdr) +{ + char *chunk_id = mof_chunkhdr->chunk_id; + + if (!strncmp(chunk_id, ICP_QAT_MOF_SYM_OBJS, ICP_QAT_MOF_OBJ_ID_LEN)) + qat_uclo_map_mof_symobjs(mobj_handle, mof_chunkhdr); + else if (!strncmp(chunk_id, ICP_QAT_UOF_OBJS, ICP_QAT_MOF_OBJ_ID_LEN)) + mobj_handle->uobjs_hdr = mobj_handle->mof_buf + + mof_chunkhdr->offset; + else if (!strncmp(chunk_id, ICP_QAT_SUOF_OBJS, ICP_QAT_MOF_OBJ_ID_LEN)) + mobj_handle->sobjs_hdr = mobj_handle->mof_buf + + mof_chunkhdr->offset; +} + +static int qat_uclo_check_mof_format(struct icp_qat_mof_file_hdr *mof_hdr) +{ + int maj = mof_hdr->maj_ver & 0xff; + int min = mof_hdr->min_ver & 0xff; + + if (mof_hdr->file_id != ICP_QAT_MOF_FID) { + pr_err("QAT: invalid header 0x%x\n", mof_hdr->file_id); + return -EINVAL; + } + + if (mof_hdr->num_chunks <= 0x1) { + pr_err("QAT: MOF chunk amount is incorrect\n"); + return -EINVAL; + } + if (maj != ICP_QAT_MOF_MAJVER || min != ICP_QAT_MOF_MINVER) { + pr_err("QAT: bad MOF version, major 0x%x, minor 0x%x\n", + maj, min); + return -EINVAL; + } + return 0; +} + +static int qat_uclo_map_mof_obj(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_mof_file_hdr *mof_ptr, + u32 mof_size, char *obj_name, char **obj_ptr, + unsigned int *obj_size) +{ + struct icp_qat_mof_chunkhdr *mof_chunkhdr; + unsigned int file_id = mof_ptr->file_id; + struct icp_qat_mof_handle *mobj_handle; + unsigned short chunks_num; + unsigned int i; + int ret; + + if (file_id == ICP_QAT_UOF_FID || file_id == ICP_QAT_SUOF_FID) { + if (obj_ptr) + *obj_ptr = (char *)mof_ptr; + if (obj_size) + *obj_size = mof_size; + return 0; + } + if (qat_uclo_check_mof_format(mof_ptr)) + return -EINVAL; + + mobj_handle = kzalloc(sizeof(*mobj_handle), GFP_KERNEL); + if (!mobj_handle) + return -ENOMEM; + + handle->mobj_handle = mobj_handle; + ret = qat_uclo_map_mof_file_hdr(handle, mof_ptr, mof_size); + if (ret) + return ret; + + mof_chunkhdr = (void *)mof_ptr + sizeof(*mof_ptr); + chunks_num = mof_ptr->num_chunks; + + /* Parse MOF file chunks */ + for (i = 0; i < chunks_num; i++) + qat_uclo_map_mof_chunk(mobj_handle, &mof_chunkhdr[i]); + + /* All sym_objs uobjs and sobjs should be available */ + if (!mobj_handle->sym_str || + (!mobj_handle->uobjs_hdr && !mobj_handle->sobjs_hdr)) + return -EINVAL; + + ret = qat_uclo_map_objs_from_mof(mobj_handle); + if (ret) + return ret; + + /* Seek specified uof object in MOF */ + return qat_uclo_seek_obj_inside_mof(mobj_handle, obj_name, + obj_ptr, obj_size); +} + +int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle, + void *addr_ptr, u32 mem_size, char *obj_name) +{ + char *obj_addr; + u32 obj_size; + int ret; + BUILD_BUG_ON(ICP_QAT_UCLO_MAX_AE >= (sizeof(handle->hal_handle->ae_mask) * 8)); if (!handle || !addr_ptr || mem_size < 24) return -EINVAL; - return (handle->fw_auth) ? - qat_uclo_map_suof_obj(handle, addr_ptr, mem_size) : - qat_uclo_map_uof_obj(handle, addr_ptr, mem_size); + if (obj_name) { + ret = qat_uclo_map_mof_obj(handle, addr_ptr, mem_size, obj_name, + &obj_addr, &obj_size); + if (ret) + return ret; + } else { + obj_addr = addr_ptr; + obj_size = mem_size; + } + + return (handle->chip_info->fw_auth) ? + qat_uclo_map_suof_obj(handle, obj_addr, obj_size) : + qat_uclo_map_uof_obj(handle, obj_addr, obj_size); } -void qat_uclo_del_uof_obj(struct icp_qat_fw_loader_handle *handle) +void qat_uclo_del_obj(struct icp_qat_fw_loader_handle *handle) { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; unsigned int a; + if (handle->mobj_handle) + qat_uclo_del_mof(handle); if (handle->sobj_handle) qat_uclo_del_suof(handle); if (!obj_handle) @@ -1479,23 +1895,24 @@ static void qat_uclo_fill_uwords(struct icp_qat_uclo_objhandle *obj_handle, u64 *uword, unsigned int addr_p, unsigned int raddr, u64 fill) { + unsigned int i, addr; u64 uwrd = 0; - unsigned int i; if (!encap_page) { *uword = fill; return; } + addr = (encap_page->page_region) ? raddr : addr_p; for (i = 0; i < encap_page->uwblock_num; i++) { - if (raddr >= encap_page->uwblock[i].start_addr && - raddr <= encap_page->uwblock[i].start_addr + + if (addr >= encap_page->uwblock[i].start_addr && + addr <= encap_page->uwblock[i].start_addr + encap_page->uwblock[i].words_num - 1) { - raddr -= encap_page->uwblock[i].start_addr; - raddr *= obj_handle->uword_in_bytes; + addr -= encap_page->uwblock[i].start_addr; + addr *= obj_handle->uword_in_bytes; memcpy(&uwrd, (void *)(((uintptr_t) - encap_page->uwblock[i].micro_words) + raddr), + encap_page->uwblock[i].micro_words) + addr), obj_handle->uword_in_bytes); - uwrd = uwrd & 0xbffffffffffull; + uwrd = uwrd & GENMASK_ULL(43, 0); } } *uword = uwrd; @@ -1546,6 +1963,10 @@ static void qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle, struct icp_qat_uof_image *image) { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned long cfg_ae_mask = handle->cfg_ae_mask; + unsigned long ae_assigned = image->ae_assigned; + struct icp_qat_uclo_aedata *aed; unsigned int ctx_mask, s; struct icp_qat_uclo_page *page; unsigned char ae; @@ -1557,25 +1978,30 @@ static void qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle, ctx_mask = 0x55; /* load the default page and set assigned CTX PC * to the entrypoint address */ - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { - if (!test_bit(ae, (unsigned long *)&image->ae_assigned)) + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { + if (!test_bit(ae, &cfg_ae_mask)) continue; + + if (!test_bit(ae, &ae_assigned)) + continue; + + aed = &obj_handle->ae_data[ae]; /* find the slice to which this image is assigned */ - for (s = 0; s < obj_handle->ae_data[ae].slice_num; s++) { - if (image->ctx_assigned & obj_handle->ae_data[ae]. - ae_slices[s].ctx_mask_assigned) + for (s = 0; s < aed->slice_num; s++) { + if (image->ctx_assigned & + aed->ae_slices[s].ctx_mask_assigned) break; } - if (s >= obj_handle->ae_data[ae].slice_num) + if (s >= aed->slice_num) continue; - page = obj_handle->ae_data[ae].ae_slices[s].page; + page = aed->ae_slices[s].page; if (!page->encap_page->def_page) continue; qat_uclo_wr_uimage_raw_page(handle, page->encap_page, ae); - page = obj_handle->ae_data[ae].ae_slices[s].page; + page = aed->ae_slices[s].page; for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) - obj_handle->ae_data[ae].ae_slices[s].cur_page[ctx] = + aed->ae_slices[s].cur_page[ctx] = (ctx_mask & (1 << ctx)) ? page : NULL; qat_hal_set_live_ctx(handle, (unsigned char)ae, image->ctx_assigned); @@ -1595,13 +2021,18 @@ static int qat_uclo_wr_suof_img(struct icp_qat_fw_loader_handle *handle) if (qat_uclo_map_auth_fw(handle, (char *)simg_hdr[i].simg_buf, (unsigned int) - (simg_hdr[i].simg_len), + simg_hdr[i].simg_len, &desc)) goto wr_err; if (qat_uclo_auth_fw(handle, desc)) goto wr_err; - if (qat_uclo_load_fw(handle, desc)) - goto wr_err; + if (qat_uclo_is_broadcast(handle, i)) { + if (qat_uclo_broadcast_load_fw(handle, desc)) + goto wr_err; + } else { + if (qat_uclo_load_fw(handle, desc)) + goto wr_err; + } qat_uclo_ummap_auth_fw(handle, &desc); } return 0; @@ -1630,6 +2061,16 @@ static int qat_uclo_wr_uof_img(struct icp_qat_fw_loader_handle *handle) int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle) { - return (handle->fw_auth) ? qat_uclo_wr_suof_img(handle) : + return (handle->chip_info->fw_auth) ? qat_uclo_wr_suof_img(handle) : qat_uclo_wr_uof_img(handle); } + +int qat_uclo_set_cfg_ae_mask(struct icp_qat_fw_loader_handle *handle, + unsigned int cfg_ae_mask) +{ + if (!cfg_ae_mask) + return -EINVAL; + + handle->cfg_ae_mask = cfg_ae_mask; + return 0; +} diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index b975c263446d..1e83d9397b11 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -3,7 +3,9 @@ #include #include #include +#include #include "adf_dh895xcc_hw_data.h" +#include "icp_qat_hw.h" /* Worker thread to service arbiter mappings based on dev SKUs */ static const u32 thrd_to_arb_map_sku4[] = { @@ -24,15 +26,19 @@ static struct adf_hw_device_class dh895xcc_class = { .instances = 0 }; -static u32 get_accel_mask(u32 fuse) +static u32 get_accel_mask(struct adf_hw_device_data *self) { - return (~fuse) >> ADF_DH895XCC_ACCELERATORS_REG_OFFSET & - ADF_DH895XCC_ACCELERATORS_MASK; + u32 fuses = self->fuses; + + return ~fuses >> ADF_DH895XCC_ACCELERATORS_REG_OFFSET & + ADF_DH895XCC_ACCELERATORS_MASK; } -static u32 get_ae_mask(u32 fuse) +static u32 get_ae_mask(struct adf_hw_device_data *self) { - return (~fuse) & ADF_DH895XCC_ACCELENGINES_MASK; + u32 fuses = self->fuses; + + return ~fuses & ADF_DH895XCC_ACCELENGINES_MASK; } static u32 get_num_accels(struct adf_hw_device_data *self) @@ -78,6 +84,29 @@ static u32 get_sram_bar_id(struct adf_hw_device_data *self) return ADF_DH895XCC_SRAM_BAR; } +static u32 get_accel_cap(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_dev->accel_pci_dev.pci_dev; + u32 capabilities; + u32 legfuses; + + capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | + ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | + ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + + /* Read accelerator capabilities mask */ + pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET, &legfuses); + + if (legfuses & ICP_ACCEL_MASK_CIPHER_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; + if (legfuses & ICP_ACCEL_MASK_PKE_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; + if (legfuses & ICP_ACCEL_MASK_AUTH_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + + return capabilities; +} + static enum dev_sku_info get_sku(struct adf_hw_device_data *self) { int sku = (self->fuses & ADF_DH895XCC_FUSECTL_SKU_MASK) @@ -131,11 +160,13 @@ static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_device = accel_dev->hw_device; struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_DH895XCC_PMISC_BAR]; + unsigned long accel_mask = hw_device->accel_mask; + unsigned long ae_mask = hw_device->ae_mask; void __iomem *csr = misc_bar->virt_addr; unsigned int val, i; /* Enable Accel Engine error detection & correction */ - for (i = 0; i < hw_device->get_num_aes(hw_device); i++) { + for_each_set_bit(i, &ae_mask, GET_MAX_ACCELENGINES(accel_dev)) { val = ADF_CSR_RD(csr, ADF_DH895XCC_AE_CTX_ENABLES(i)); val |= ADF_DH895XCC_ENABLE_AE_ECC_ERR; ADF_CSR_WR(csr, ADF_DH895XCC_AE_CTX_ENABLES(i), val); @@ -145,7 +176,7 @@ static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) } /* Enable shared memory error detection & correction */ - for (i = 0; i < hw_device->get_num_accels(hw_device); i++) { + for_each_set_bit(i, &accel_mask, ADF_DH895XCC_MAX_ACCELERATORS) { val = ADF_CSR_RD(csr, ADF_DH895XCC_UERRSSMSH(i)); val |= ADF_DH895XCC_ERRSSMSH_EN; ADF_CSR_WR(csr, ADF_DH895XCC_UERRSSMSH(i), val); @@ -164,7 +195,7 @@ static void adf_enable_ints(struct adf_accel_dev *accel_dev) /* Enable bundle and misc interrupts */ ADF_CSR_WR(addr, ADF_DH895XCC_SMIAPF0_MASK_OFFSET, accel_dev->pf.vf_info ? 0 : - GENMASK_ULL(GET_MAX_BANKS(accel_dev) - 1, 0)); + BIT_ULL(GET_MAX_BANKS(accel_dev)) - 1); ADF_CSR_WR(addr, ADF_DH895XCC_SMIAPF1_MASK_OFFSET, ADF_DH895XCC_SMIA1_MASK); } @@ -174,11 +205,19 @@ static int adf_pf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev) return 0; } +static void configure_iov_threads(struct adf_accel_dev *accel_dev, bool enable) +{ + adf_gen2_cfg_iov_thds(accel_dev, enable, + ADF_DH895XCC_AE2FUNC_MAP_GRP_A_NUM_REGS, + ADF_DH895XCC_AE2FUNC_MAP_GRP_B_NUM_REGS); +} + void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) { hw_data->dev_class = &dh895xcc_class; hw_data->instance_id = dh895xcc_class.instances++; hw_data->num_banks = ADF_DH895XCC_ETR_MAX_BANKS; + hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; hw_data->num_accel = ADF_DH895XCC_MAX_ACCELERATORS; hw_data->num_logical_accel = 1; hw_data->num_engines = ADF_DH895XCC_MAX_ACCELENGINES; @@ -189,18 +228,22 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->enable_error_correction = adf_enable_error_correction; hw_data->get_accel_mask = get_accel_mask; hw_data->get_ae_mask = get_ae_mask; + hw_data->get_accel_cap = get_accel_cap; hw_data->get_num_accels = get_num_accels; hw_data->get_num_aes = get_num_aes; hw_data->get_etr_bar_id = get_etr_bar_id; hw_data->get_misc_bar_id = get_misc_bar_id; hw_data->get_pf2vf_offset = get_pf2vf_offset; hw_data->get_vintmsk_offset = get_vintmsk_offset; + hw_data->get_admin_info = adf_gen2_get_admin_info; + hw_data->get_arb_info = adf_gen2_get_arb_info; hw_data->get_sram_bar_id = get_sram_bar_id; hw_data->get_sku = get_sku; hw_data->fw_name = ADF_DH895XCC_FW; hw_data->fw_mmp_name = ADF_DH895XCC_MMP; hw_data->init_admin_comms = adf_init_admin_comms; hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->configure_iov_threads = configure_iov_threads; hw_data->disable_iov = adf_disable_sriov; hw_data->send_admin_init = adf_send_admin_init; hw_data->init_arb = adf_init_arb; @@ -210,6 +253,7 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->enable_vf2pf_comms = adf_pf_enable_vf2pf_comms; hw_data->reset_device = adf_reset_sbr; hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; + adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); } void adf_clean_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h index 082a04466dca..4d613923d155 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h @@ -36,6 +36,11 @@ #define ADF_DH895XCC_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) #define ADF_DH895XCC_VINTMSK_OFFSET(i) (0x3A000 + 0x200 + ((i) * 0x04)) + +/* AE to function mapping */ +#define ADF_DH895XCC_AE2FUNC_MAP_GRP_A_NUM_REGS 96 +#define ADF_DH895XCC_AE2FUNC_MAP_GRP_B_NUM_REGS 12 + /* FW names */ #define ADF_DH895XCC_FW "qat_895xcc.bin" #define ADF_DH895XCC_MMP "qat_895xcc_mmp.bin" diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c index ecb4f6f20e22..a9ec4357144c 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c @@ -128,8 +128,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) &hw_data->fuses); /* Get Accelerators and Accelerators Engines masks */ - hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); - hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); accel_pci_dev->sku = hw_data->get_sku(hw_data); /* If the device has no acceleration engines then ignore it. */ if (!hw_data->accel_mask || !hw_data->ae_mask || @@ -177,9 +177,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto out_err_disable; } - /* Read accelerator capabilities mask */ - pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET, - &hw_data->accel_capabilities_mask); + /* Get accelerator capabilities mask */ + hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); /* Find and map all the device's BARS */ i = 0; diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c index 5246f0524ca3..f14fb82ed6df 100644 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "adf_dh895xccvf_hw_data.h" static struct adf_hw_device_class dh895xcciov_class = { @@ -11,12 +12,12 @@ static struct adf_hw_device_class dh895xcciov_class = { .instances = 0 }; -static u32 get_accel_mask(u32 fuse) +static u32 get_accel_mask(struct adf_hw_device_data *self) { return ADF_DH895XCCIOV_ACCELERATORS_MASK; } -static u32 get_ae_mask(u32 fuse) +static u32 get_ae_mask(struct adf_hw_device_data *self) { return ADF_DH895XCCIOV_ACCELENGINES_MASK; } @@ -69,6 +70,7 @@ void adf_init_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data) { hw_data->dev_class = &dh895xcciov_class; hw_data->num_banks = ADF_DH895XCCIOV_ETR_MAX_BANKS; + hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; hw_data->num_accel = ADF_DH895XCCIOV_MAX_ACCELERATORS; hw_data->num_logical_accel = 1; hw_data->num_engines = ADF_DH895XCCIOV_MAX_ACCELENGINES; @@ -97,6 +99,7 @@ void adf_init_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data) hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; hw_data->dev_class->instances++; adf_devmgr_update_class_index(hw_data); + adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); } void adf_clean_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data) diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c b/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c index 404cf9df6922..c972554a755e 100644 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c @@ -119,8 +119,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_dh895xcciov(accel_dev->hw_device); /* Get Accelerators and Accelerators Engines masks */ - hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); - hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); accel_pci_dev->sku = hw_data->get_sku(hw_data); /* Create dev top level debugfs entry */ diff --git a/drivers/crypto/qce/common.c b/drivers/crypto/qce/common.c index 5006e74c40cd..a73db2a5637f 100644 --- a/drivers/crypto/qce/common.c +++ b/drivers/crypto/qce/common.c @@ -7,7 +7,8 @@ #include #include #include -#include +#include +#include #include "cipher.h" #include "common.h" diff --git a/drivers/crypto/qce/core.c b/drivers/crypto/qce/core.c index ea616b7259ae..80b75085c265 100644 --- a/drivers/crypto/qce/core.c +++ b/drivers/crypto/qce/core.c @@ -13,7 +13,6 @@ #include #include #include -#include #include "core.h" #include "cipher.h" @@ -160,7 +159,21 @@ static int qce_check_version(struct qce_device *qce) return -ENODEV; qce->burst_size = QCE_BAM_BURST_SIZE; - qce->pipe_pair_id = 1; + + /* + * Rx and tx pipes are treated as a pair inside CE. + * Pipe pair number depends on the actual BAM dma pipe + * that is used for transfers. The BAM dma pipes are passed + * from the device tree and used to derive the pipe pair + * id in the CE driver as follows. + * BAM dma pipes(rx, tx) CE pipe pair id + * 0,1 0 + * 2,3 1 + * 4,5 2 + * 6,7 3 + * ... + */ + qce->pipe_pair_id = qce->dma.rxchan->chan_id >> 1; dev_dbg(qce->dev, "Crypto device found, version %d.%d.%d\n", major, minor, step); @@ -261,6 +274,7 @@ static int qce_crypto_remove(struct platform_device *pdev) static const struct of_device_id qce_crypto_of_match[] = { { .compatible = "qcom,crypto-v5.1", }, + { .compatible = "qcom,crypto-v5.4", }, {} }; MODULE_DEVICE_TABLE(of, qce_crypto_of_match); diff --git a/drivers/crypto/qce/sha.c b/drivers/crypto/qce/sha.c index 87be96a0b0bb..61c418c12345 100644 --- a/drivers/crypto/qce/sha.c +++ b/drivers/crypto/qce/sha.c @@ -48,7 +48,7 @@ static void qce_ahash_done(void *data) dma_unmap_sg(qce->dev, &rctx->result_sg, 1, DMA_FROM_DEVICE); memcpy(rctx->digest, result->auth_iv, digestsize); - if (req->result) + if (req->result && rctx->last_blk) memcpy(req->result, result->auth_iv, digestsize); rctx->byte_count[0] = cpu_to_be32(result->auth_byte_count[0]); diff --git a/drivers/crypto/qce/sha.h b/drivers/crypto/qce/sha.h index d63526e3804d..a22695361f16 100644 --- a/drivers/crypto/qce/sha.h +++ b/drivers/crypto/qce/sha.h @@ -7,7 +7,8 @@ #define _SHA_H_ #include -#include +#include +#include #include "common.h" #include "core.h" diff --git a/drivers/crypto/rockchip/rk3288_crypto.h b/drivers/crypto/rockchip/rk3288_crypto.h index 3db595570c9c..97278c2574ff 100644 --- a/drivers/crypto/rockchip/rk3288_crypto.h +++ b/drivers/crypto/rockchip/rk3288_crypto.h @@ -12,7 +12,8 @@ #include #include -#include +#include +#include #define _SBF(v, f) ((v) << (f)) diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c index 88a6c853ffd7..682c8a450a57 100644 --- a/drivers/crypto/s5p-sss.c +++ b/drivers/crypto/s5p-sss.c @@ -30,7 +30,8 @@ #include #include -#include +#include +#include #include #define _SBF(s, v) ((v) << (s)) diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c index eda93fab95fe..f300b0a5958a 100644 --- a/drivers/crypto/sa2ul.c +++ b/drivers/crypto/sa2ul.c @@ -9,8 +9,10 @@ * Tero Kristo */ #include +#include #include #include +#include #include #include #include @@ -23,7 +25,8 @@ #include #include #include -#include +#include +#include #include "sa2ul.h" @@ -361,42 +364,45 @@ static void sa_swiz_128(u8 *in, u16 len) } /* Prepare the ipad and opad from key as per SHA algorithm step 1*/ -static void prepare_kiopad(u8 *k_ipad, u8 *k_opad, const u8 *key, u16 key_sz) +static void prepare_kipad(u8 *k_ipad, const u8 *key, u16 key_sz) { int i; - for (i = 0; i < key_sz; i++) { + for (i = 0; i < key_sz; i++) k_ipad[i] = key[i] ^ 0x36; - k_opad[i] = key[i] ^ 0x5c; - } /* Instead of XOR with 0 */ - for (; i < SHA1_BLOCK_SIZE; i++) { + for (; i < SHA1_BLOCK_SIZE; i++) k_ipad[i] = 0x36; - k_opad[i] = 0x5c; - } } -static void sa_export_shash(struct shash_desc *hash, int block_size, +static void prepare_kopad(u8 *k_opad, const u8 *key, u16 key_sz) +{ + int i; + + for (i = 0; i < key_sz; i++) + k_opad[i] = key[i] ^ 0x5c; + + /* Instead of XOR with 0 */ + for (; i < SHA1_BLOCK_SIZE; i++) + k_opad[i] = 0x5c; +} + +static void sa_export_shash(void *state, struct shash_desc *hash, int digest_size, __be32 *out) { - union { - struct sha1_state sha1; - struct sha256_state sha256; - struct sha512_state sha512; - } sha; - void *state; + struct sha1_state *sha1; + struct sha256_state *sha256; u32 *result; - int i; switch (digest_size) { case SHA1_DIGEST_SIZE: - state = &sha.sha1; - result = sha.sha1.state; + sha1 = state; + result = sha1->state; break; case SHA256_DIGEST_SIZE: - state = &sha.sha256; - result = sha.sha256.state; + sha256 = state; + result = sha256->state; break; default: dev_err(sa_k3_dev, "%s: bad digest_size=%d\n", __func__, @@ -406,8 +412,7 @@ static void sa_export_shash(struct shash_desc *hash, int block_size, crypto_shash_export(hash, state); - for (i = 0; i < digest_size >> 2; i++) - out[i] = cpu_to_be32(result[i]); + cpu_to_be32_array(out, result, digest_size / 4); } static void sa_prepare_iopads(struct algo_data *data, const u8 *key, @@ -416,24 +421,28 @@ static void sa_prepare_iopads(struct algo_data *data, const u8 *key, SHASH_DESC_ON_STACK(shash, data->ctx->shash); int block_size = crypto_shash_blocksize(data->ctx->shash); int digest_size = crypto_shash_digestsize(data->ctx->shash); - u8 k_ipad[SHA1_BLOCK_SIZE]; - u8 k_opad[SHA1_BLOCK_SIZE]; + union { + struct sha1_state sha1; + struct sha256_state sha256; + u8 k_pad[SHA1_BLOCK_SIZE]; + } sha; shash->tfm = data->ctx->shash; - prepare_kiopad(k_ipad, k_opad, key, key_sz); - - memzero_explicit(ipad, block_size); - memzero_explicit(opad, block_size); + prepare_kipad(sha.k_pad, key, key_sz); crypto_shash_init(shash); - crypto_shash_update(shash, k_ipad, block_size); - sa_export_shash(shash, block_size, digest_size, ipad); + crypto_shash_update(shash, sha.k_pad, block_size); + sa_export_shash(&sha, shash, digest_size, ipad); + + prepare_kopad(sha.k_pad, key, key_sz); crypto_shash_init(shash); - crypto_shash_update(shash, k_opad, block_size); + crypto_shash_update(shash, sha.k_pad, block_size); - sa_export_shash(shash, block_size, digest_size, opad); + sa_export_shash(&sha, shash, digest_size, opad); + + memzero_explicit(&sha, sizeof(sha)); } /* Derive the inverse key used in AES-CBC decryption operation */ @@ -506,7 +515,8 @@ static int sa_set_sc_enc(struct algo_data *ad, const u8 *key, u16 key_sz, static void sa_set_sc_auth(struct algo_data *ad, const u8 *key, u16 key_sz, u8 *sc_buf) { - __be32 ipad[64], opad[64]; + __be32 *ipad = (void *)(sc_buf + 32); + __be32 *opad = (void *)(sc_buf + 64); /* Set Authentication mode selector to hash processing */ sc_buf[0] = SA_HASH_PROCESSING; @@ -515,14 +525,9 @@ static void sa_set_sc_auth(struct algo_data *ad, const u8 *key, u16 key_sz, sc_buf[1] |= ad->auth_ctrl; /* Copy the keys or ipad/opad */ - if (ad->keyed_mac) { + if (ad->keyed_mac) ad->prep_iopad(ad, key, key_sz, ipad, opad); - - /* Copy ipad to AuthKey */ - memcpy(&sc_buf[32], ipad, ad->hash_size); - /* Copy opad to Aux-1 */ - memcpy(&sc_buf[64], opad, ad->hash_size); - } else { + else { /* basic hash */ sc_buf[1] |= SA_BASIC_HASH; } @@ -819,7 +824,7 @@ static void sa_cipher_cra_exit(struct crypto_skcipher *tfm) sa_free_ctx_info(&ctx->enc, data); sa_free_ctx_info(&ctx->dec, data); - crypto_free_sync_skcipher(ctx->fallback.skcipher); + crypto_free_skcipher(ctx->fallback.skcipher); } static int sa_cipher_cra_init(struct crypto_skcipher *tfm) @@ -827,6 +832,7 @@ static int sa_cipher_cra_init(struct crypto_skcipher *tfm) struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); const char *name = crypto_tfm_alg_name(&tfm->base); + struct crypto_skcipher *child; int ret; memzero_explicit(ctx, sizeof(*ctx)); @@ -841,14 +847,17 @@ static int sa_cipher_cra_init(struct crypto_skcipher *tfm) return ret; } - ctx->fallback.skcipher = - crypto_alloc_sync_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); + child = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(ctx->fallback.skcipher)) { + if (IS_ERR(child)) { dev_err(sa_k3_dev, "Error allocating fallback algo %s\n", name); - return PTR_ERR(ctx->fallback.skcipher); + return PTR_ERR(child); } + ctx->fallback.skcipher = child; + crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(child) + + sizeof(struct skcipher_request)); + dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, ctx->dec.sc_id, &ctx->dec.sc_phys); @@ -859,6 +868,7 @@ static int sa_cipher_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen, struct algo_data *ad) { struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + struct crypto_skcipher *child = ctx->fallback.skcipher; int cmdl_len; struct sa_cmdl_cfg cfg; int ret; @@ -874,12 +884,10 @@ static int sa_cipher_setkey(struct crypto_skcipher *tfm, const u8 *key, cfg.enc_eng_id = ad->enc_eng.eng_id; cfg.iv_size = crypto_skcipher_ivsize(tfm); - crypto_sync_skcipher_clear_flags(ctx->fallback.skcipher, + crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(child, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); - crypto_sync_skcipher_set_flags(ctx->fallback.skcipher, - tfm->base.crt_flags & - CRYPTO_TFM_REQ_MASK); - ret = crypto_sync_skcipher_setkey(ctx->fallback.skcipher, key, keylen); + ret = crypto_skcipher_setkey(child, key, keylen); if (ret) return ret; @@ -1270,7 +1278,6 @@ static int sa_cipher_run(struct skcipher_request *req, u8 *iv, int enc) crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); struct crypto_alg *alg = req->base.tfm->__crt_alg; struct sa_req sa_req = { 0 }; - int ret; if (!req->cryptlen) return 0; @@ -1282,20 +1289,18 @@ static int sa_cipher_run(struct skcipher_request *req, u8 *iv, int enc) if (req->cryptlen > SA_MAX_DATA_SZ || (req->cryptlen >= SA_UNSAFE_DATA_SZ_MIN && req->cryptlen <= SA_UNSAFE_DATA_SZ_MAX)) { - SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback.skcipher); + struct skcipher_request *subreq = skcipher_request_ctx(req); - skcipher_request_set_sync_tfm(subreq, ctx->fallback.skcipher); + skcipher_request_set_tfm(subreq, ctx->fallback.skcipher); skcipher_request_set_callback(subreq, req->base.flags, - NULL, NULL); + req->base.complete, + req->base.data); skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, req->iv); if (enc) - ret = crypto_skcipher_encrypt(subreq); + return crypto_skcipher_encrypt(subreq); else - ret = crypto_skcipher_decrypt(subreq); - - skcipher_request_zero(subreq); - return ret; + return crypto_skcipher_decrypt(subreq); } sa_req.size = req->cryptlen; diff --git a/drivers/crypto/sa2ul.h b/drivers/crypto/sa2ul.h index 7f7e3fe60d11..f597ddecde34 100644 --- a/drivers/crypto/sa2ul.h +++ b/drivers/crypto/sa2ul.h @@ -12,10 +12,9 @@ #ifndef _K3_SA2UL_ #define _K3_SA2UL_ -#include -#include -#include #include +#include +#include #define SA_ENGINE_ENABLE_CONTROL 0x1000 @@ -311,7 +310,7 @@ struct sa_tfm_ctx { struct crypto_shash *shash; /* for fallback */ union { - struct crypto_sync_skcipher *skcipher; + struct crypto_skcipher *skcipher; struct crypto_ahash *ahash; struct crypto_aead *aead; } fallback; diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c index d60679c79822..8b5be29cb4dc 100644 --- a/drivers/crypto/sahara.c +++ b/drivers/crypto/sahara.c @@ -15,7 +15,8 @@ #include #include #include -#include +#include +#include #include #include diff --git a/drivers/crypto/stm32/stm32-hash.c b/drivers/crypto/stm32/stm32-hash.c index e3e25278a970..7ac0573ef663 100644 --- a/drivers/crypto/stm32/stm32-hash.c +++ b/drivers/crypto/stm32/stm32-hash.c @@ -25,7 +25,8 @@ #include #include #include -#include +#include +#include #include #define HASH_CR 0x00 diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 66773892f665..4fd85f31630a 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -31,7 +31,8 @@ #include #include #include -#include +#include +#include #include #include #include @@ -460,7 +461,7 @@ DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE) /* * locate current (offending) descriptor */ -static u32 current_desc_hdr(struct device *dev, int ch) +static __be32 current_desc_hdr(struct device *dev, int ch) { struct talitos_private *priv = dev_get_drvdata(dev); int tail, iter; @@ -478,7 +479,7 @@ static u32 current_desc_hdr(struct device *dev, int ch) iter = tail; while (priv->chan[ch].fifo[iter].dma_desc != cur_desc && - priv->chan[ch].fifo[iter].desc->next_desc != cur_desc) { + priv->chan[ch].fifo[iter].desc->next_desc != cpu_to_be32(cur_desc)) { iter = (iter + 1) & (priv->fifo_len - 1); if (iter == tail) { dev_err(dev, "couldn't locate current descriptor\n"); @@ -486,7 +487,7 @@ static u32 current_desc_hdr(struct device *dev, int ch) } } - if (priv->chan[ch].fifo[iter].desc->next_desc == cur_desc) { + if (priv->chan[ch].fifo[iter].desc->next_desc == cpu_to_be32(cur_desc)) { struct talitos_edesc *edesc; edesc = container_of(priv->chan[ch].fifo[iter].desc, @@ -501,13 +502,13 @@ static u32 current_desc_hdr(struct device *dev, int ch) /* * user diagnostics; report root cause of error based on execution unit status */ -static void report_eu_error(struct device *dev, int ch, u32 desc_hdr) +static void report_eu_error(struct device *dev, int ch, __be32 desc_hdr) { struct talitos_private *priv = dev_get_drvdata(dev); int i; if (!desc_hdr) - desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF); + desc_hdr = cpu_to_be32(in_be32(priv->chan[ch].reg + TALITOS_DESCBUF)); switch (desc_hdr & DESC_HDR_SEL0_MASK) { case DESC_HDR_SEL0_AFEU: diff --git a/drivers/crypto/ux500/hash/hash_core.c b/drivers/crypto/ux500/hash/hash_core.c index 3d407eebb2ba..da284b0ea1b2 100644 --- a/drivers/crypto/ux500/hash/hash_core.c +++ b/drivers/crypto/ux500/hash/hash_core.c @@ -31,7 +31,8 @@ #include #include -#include +#include +#include #include #include diff --git a/drivers/edac/Kconfig b/drivers/edac/Kconfig index 7a47680d6f07..81c42664f21b 100644 --- a/drivers/edac/Kconfig +++ b/drivers/edac/Kconfig @@ -269,6 +269,15 @@ config EDAC_PND2 first used on the Apollo Lake platform and Denverton micro-server but may appear on others in the future. +config EDAC_IGEN6 + tristate "Intel client SoC Integrated MC" + depends on PCI && X86_64 && PCI_MMCONFIG && ARCH_HAVE_NMI_SAFE_CMPXCHG + help + Support for error detection and correction on the Intel + client SoC Integrated Memory Controller using In-Band ECC IP. + This In-Band ECC is first used on the Elkhart Lake SoC but + may appear on others in the future. + config EDAC_MPC85XX bool "Freescale MPC83xx / MPC85xx" depends on FSL_SOC && EDAC=y @@ -283,13 +292,6 @@ config EDAC_LAYERSCAPE Support for error detection and correction on Freescale memory controllers on Layerscape SoCs. -config EDAC_MV64X60 - tristate "Marvell MV64x60" - depends on MV64X60 - help - Support for error detection and correction on the Marvell - MV64360 and MV64460 chipsets. - config EDAC_PASEMI tristate "PA Semi PWRficient" depends on PPC_PASEMI && PCI @@ -515,10 +517,10 @@ config EDAC_QCOM health, you should probably say 'Y' here. config EDAC_ASPEED - tristate "Aspeed AST 2500 SoC" - depends on MACH_ASPEED_G5 + tristate "Aspeed AST BMC SoC" + depends on ARCH_ASPEED help - Support for error detection and correction on the Aspeed AST 2500 SoC. + Support for error detection and correction on the Aspeed AST BMC SoC. First, ECC must be configured in the bootloader. Then, this driver will expose error counters via the EDAC kernel framework. diff --git a/drivers/edac/Makefile b/drivers/edac/Makefile index 3a849168780d..464d3d8d850a 100644 --- a/drivers/edac/Makefile +++ b/drivers/edac/Makefile @@ -32,6 +32,7 @@ obj-$(CONFIG_EDAC_I7300) += i7300_edac.o obj-$(CONFIG_EDAC_I7CORE) += i7core_edac.o obj-$(CONFIG_EDAC_SBRIDGE) += sb_edac.o obj-$(CONFIG_EDAC_PND2) += pnd2_edac.o +obj-$(CONFIG_EDAC_IGEN6) += igen6_edac.o obj-$(CONFIG_EDAC_E7XXX) += e7xxx_edac.o obj-$(CONFIG_EDAC_E752X) += e752x_edac.o obj-$(CONFIG_EDAC_I82443BXGX) += i82443bxgx_edac.o @@ -64,7 +65,6 @@ obj-$(CONFIG_EDAC_SKX) += skx_edac.o i10nm_edac-y := skx_common.o i10nm_base.o obj-$(CONFIG_EDAC_I10NM) += i10nm_edac.o -obj-$(CONFIG_EDAC_MV64X60) += mv64x60_edac.o obj-$(CONFIG_EDAC_CELL) += cell_edac.o obj-$(CONFIG_EDAC_PPC4XX) += ppc4xx_edac.o obj-$(CONFIG_EDAC_AMD8111) += amd8111_edac.o diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 1362274d840b..f7087ddddb90 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -18,6 +18,9 @@ static struct amd64_family_type *fam_type; /* Per-node stuff */ static struct ecc_settings **ecc_stngs; +/* Device for the PCI component */ +static struct device *pci_ctl_dev; + /* * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching- @@ -1133,7 +1136,7 @@ static int k8_early_channel_count(struct amd64_pvt *pvt) /* On F10h and later ErrAddr is MC4_ADDR[47:1] */ static u64 get_error_address(struct amd64_pvt *pvt, struct mce *m) { - u16 mce_nid = amd_get_nb_id(m->extcpu); + u16 mce_nid = topology_die_id(m->extcpu); struct mem_ctl_info *mci; u8 start_bit = 1; u8 end_bit = 47; @@ -2461,14 +2464,11 @@ static int map_err_sym_to_channel(int err_sym, int sym_size) case 0x20: case 0x21: return 0; - break; case 0x22: case 0x23: return 1; - break; default: return err_sym >> 4; - break; } /* x8 symbols */ else @@ -2478,17 +2478,12 @@ static int map_err_sym_to_channel(int err_sym, int sym_size) WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n", err_sym); return -1; - break; - case 0x11: return 0; - break; case 0x12: return 1; - break; default: return err_sym >> 3; - break; } return -1; } @@ -2683,6 +2678,9 @@ reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 pci_id1, u16 pci_id2) return -ENODEV; } + if (!pci_ctl_dev) + pci_ctl_dev = &pvt->F0->dev; + edac_dbg(1, "F0: %s\n", pci_name(pvt->F0)); edac_dbg(1, "F3: %s\n", pci_name(pvt->F3)); edac_dbg(1, "F6: %s\n", pci_name(pvt->F6)); @@ -2707,6 +2705,9 @@ reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 pci_id1, u16 pci_id2) return -ENODEV; } + if (!pci_ctl_dev) + pci_ctl_dev = &pvt->F2->dev; + edac_dbg(1, "F1: %s\n", pci_name(pvt->F1)); edac_dbg(1, "F2: %s\n", pci_name(pvt->F2)); edac_dbg(1, "F3: %s\n", pci_name(pvt->F3)); @@ -3046,7 +3047,7 @@ static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, u16 nid) int cpu; for_each_online_cpu(cpu) - if (amd_get_nb_id(cpu) == nid) + if (topology_die_id(cpu) == nid) cpumask_set_cpu(cpu, mask); } @@ -3623,21 +3624,10 @@ static void remove_one_instance(unsigned int nid) static void setup_pci_device(void) { - struct mem_ctl_info *mci; - struct amd64_pvt *pvt; - if (pci_ctl) return; - mci = edac_mc_find(0); - if (!mci) - return; - - pvt = mci->pvt_info; - if (pvt->umc) - pci_ctl = edac_pci_create_generic_ctl(&pvt->F0->dev, EDAC_MOD_STR); - else - pci_ctl = edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR); + pci_ctl = edac_pci_create_generic_ctl(pci_ctl_dev, EDAC_MOD_STR); if (!pci_ctl) { pr_warn("%s(): Unable to create PCI control\n", __func__); pr_warn("%s(): PCI error report via EDAC not set\n", __func__); @@ -3716,6 +3706,8 @@ static int __init amd64_edac_init(void) return 0; err_pci: + pci_ctl_dev = NULL; + msrs_free(msrs); msrs = NULL; @@ -3745,6 +3737,8 @@ static void __exit amd64_edac_exit(void) kfree(ecc_stngs); ecc_stngs = NULL; + pci_ctl_dev = NULL; + msrs_free(msrs); msrs = NULL; } diff --git a/drivers/edac/amd76x_edac.c b/drivers/edac/amd76x_edac.c index 9c6e326b4c14..2a49f68a7cf9 100644 --- a/drivers/edac/amd76x_edac.c +++ b/drivers/edac/amd76x_edac.c @@ -179,7 +179,6 @@ static int amd76x_process_error_info(struct mem_ctl_info *mci, static void amd76x_check(struct mem_ctl_info *mci) { struct amd76x_error_info info; - edac_dbg(3, "\n"); amd76x_get_error_info(mci, &info); amd76x_process_error_info(mci, &info, 1); } diff --git a/drivers/edac/aspeed_edac.c b/drivers/edac/aspeed_edac.c index fde809efc520..a46da56d6d54 100644 --- a/drivers/edac/aspeed_edac.c +++ b/drivers/edac/aspeed_edac.c @@ -239,7 +239,7 @@ static int init_csrows(struct mem_ctl_info *mci) int rc; /* retrieve info about physical memory from device tree */ - np = of_find_node_by_path("/memory"); + np = of_find_node_by_name(NULL, "memory"); if (!np) { dev_err(mci->pdev, "dt: missing /memory node\n"); return -ENODEV; @@ -375,10 +375,13 @@ static int aspeed_remove(struct platform_device *pdev) static const struct of_device_id aspeed_of_match[] = { + { .compatible = "aspeed,ast2400-sdram-edac" }, { .compatible = "aspeed,ast2500-sdram-edac" }, + { .compatible = "aspeed,ast2600-sdram-edac" }, {}, }; +MODULE_DEVICE_TABLE(of, aspeed_of_match); static struct platform_driver aspeed_driver = { .driver = { @@ -392,5 +395,5 @@ module_platform_driver(aspeed_driver); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Stefan Schaeckeler "); -MODULE_DESCRIPTION("Aspeed AST2500 EDAC driver"); +MODULE_DESCRIPTION("Aspeed BMC SoC EDAC driver"); MODULE_VERSION("1.0"); diff --git a/drivers/edac/e752x_edac.c b/drivers/edac/e752x_edac.c index 313d08018166..ac7c9b42d4c7 100644 --- a/drivers/edac/e752x_edac.c +++ b/drivers/edac/e752x_edac.c @@ -980,7 +980,6 @@ static void e752x_check(struct mem_ctl_info *mci) { struct e752x_error_info info; - edac_dbg(3, "\n"); e752x_get_error_info(mci, &info); e752x_process_error_info(mci, &info, 1); } diff --git a/drivers/edac/e7xxx_edac.c b/drivers/edac/e7xxx_edac.c index 75d7ce62b3be..497e710fca3d 100644 --- a/drivers/edac/e7xxx_edac.c +++ b/drivers/edac/e7xxx_edac.c @@ -333,7 +333,6 @@ static void e7xxx_check(struct mem_ctl_info *mci) { struct e7xxx_error_info info; - edac_dbg(3, "\n"); e7xxx_get_error_info(mci, &info); e7xxx_process_error_info(mci, &info, 1); } diff --git a/drivers/edac/edac_device.h b/drivers/edac/edac_device.h index c4c0e0bdce14..fc2d2c218064 100644 --- a/drivers/edac/edac_device.h +++ b/drivers/edac/edac_device.h @@ -258,7 +258,7 @@ extern struct edac_device_ctl_info *edac_device_alloc_ctl_info( extern void edac_device_free_ctl_info(struct edac_device_ctl_info *ctl_info); /** - * edac_device_add_device: Insert the 'edac_dev' structure into the + * edac_device_add_device - Insert the 'edac_dev' structure into the * edac_device global list and create sysfs entries associated with * edac_device structure. * @@ -271,9 +271,8 @@ extern void edac_device_free_ctl_info(struct edac_device_ctl_info *ctl_info); extern int edac_device_add_device(struct edac_device_ctl_info *edac_dev); /** - * edac_device_del_device: - * Remove sysfs entries for specified edac_device structure and - * then remove edac_device structure from global list + * edac_device_del_device - Remove sysfs entries for specified edac_device + * structure and then remove edac_device structure from global list * * @dev: * Pointer to struct &device representing the edac device @@ -286,7 +285,7 @@ extern int edac_device_add_device(struct edac_device_ctl_info *edac_dev); extern struct edac_device_ctl_info *edac_device_del_device(struct device *dev); /** - * Log correctable errors. + * edac_device_handle_ce_count - Log correctable errors. * * @edac_dev: pointer to struct &edac_device_ctl_info * @inst_nr: number of the instance where the CE error happened @@ -299,7 +298,7 @@ void edac_device_handle_ce_count(struct edac_device_ctl_info *edac_dev, const char *msg); /** - * Log uncorrectable errors. + * edac_device_handle_ue_count - Log uncorrectable errors. * * @edac_dev: pointer to struct &edac_device_ctl_info * @inst_nr: number of the instance where the CE error happened diff --git a/drivers/edac/edac_mc.c b/drivers/edac/edac_mc.c index 01ff71f7b645..f6d462d0be2d 100644 --- a/drivers/edac/edac_mc.c +++ b/drivers/edac/edac_mc.c @@ -158,10 +158,14 @@ const char * const edac_mem_types[] = { [MEM_DDR3] = "Unbuffered-DDR3", [MEM_RDDR3] = "Registered-DDR3", [MEM_LRDDR3] = "Load-Reduced-DDR3-RAM", + [MEM_LPDDR3] = "Low-Power-DDR3-RAM", [MEM_DDR4] = "Unbuffered-DDR4", [MEM_RDDR4] = "Registered-DDR4", + [MEM_LPDDR4] = "Low-Power-DDR4-RAM", [MEM_LRDDR4] = "Load-Reduced-DDR4-RAM", + [MEM_DDR5] = "Unbuffered-DDR5", [MEM_NVDIMM] = "Non-volatile-RAM", + [MEM_WIO2] = "Wide-IO-2", }; EXPORT_SYMBOL_GPL(edac_mem_types); diff --git a/drivers/edac/i10nm_base.c b/drivers/edac/i10nm_base.c index c8d11da85bec..238a4ad1e526 100644 --- a/drivers/edac/i10nm_base.c +++ b/drivers/edac/i10nm_base.c @@ -6,27 +6,32 @@ */ #include +#include #include #include #include #include "edac_module.h" #include "skx_common.h" -#define I10NM_REVISION "v0.0.3" +#define I10NM_REVISION "v0.0.4" #define EDAC_MOD_STR "i10nm_edac" /* Debug macros */ #define i10nm_printk(level, fmt, arg...) \ edac_printk(level, "i10nm", fmt, ##arg) -#define I10NM_GET_SCK_BAR(d, reg) \ +#define I10NM_GET_SCK_BAR(d, reg) \ pci_read_config_dword((d)->uracu, 0xd0, &(reg)) #define I10NM_GET_IMC_BAR(d, i, reg) \ pci_read_config_dword((d)->uracu, 0xd8 + (i) * 4, &(reg)) #define I10NM_GET_DIMMMTR(m, i, j) \ - (*(u32 *)((m)->mbase + 0x2080c + (i) * 0x4000 + (j) * 4)) + readl((m)->mbase + 0x2080c + (i) * (m)->chan_mmio_sz + (j) * 4) #define I10NM_GET_MCDDRTCFG(m, i, j) \ - (*(u32 *)((m)->mbase + 0x20970 + (i) * 0x4000 + (j) * 4)) + readl((m)->mbase + 0x20970 + (i) * (m)->chan_mmio_sz + (j) * 4) +#define I10NM_GET_MCMTR(m, i) \ + readl((m)->mbase + 0x20ef8 + (i) * (m)->chan_mmio_sz) +#define I10NM_GET_AMAP(m, i) \ + readl((m)->mbase + 0x20814 + (i) * (m)->chan_mmio_sz) #define I10NM_GET_SCK_MMIO_BASE(reg) (GET_BITFIELD(reg, 0, 28) << 23) #define I10NM_GET_IMC_MMIO_OFFSET(reg) (GET_BITFIELD(reg, 0, 10) << 12) @@ -126,12 +131,22 @@ static struct res_config i10nm_cfg0 = { .type = I10NM, .decs_did = 0x3452, .busno_cfg_offset = 0xcc, + .ddr_chan_mmio_sz = 0x4000, }; static struct res_config i10nm_cfg1 = { .type = I10NM, .decs_did = 0x3452, .busno_cfg_offset = 0xd0, + .ddr_chan_mmio_sz = 0x4000, +}; + +static struct res_config spr_cfg = { + .type = SPR, + .decs_did = 0x3252, + .busno_cfg_offset = 0xd0, + .ddr_chan_mmio_sz = 0x8000, + .support_ddr5 = true, }; static const struct x86_cpu_id i10nm_cpuids[] = { @@ -140,6 +155,7 @@ static const struct x86_cpu_id i10nm_cpuids[] = { X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(ICELAKE_X, X86_STEPPINGS(0x0, 0x3), &i10nm_cfg0), X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(ICELAKE_X, X86_STEPPINGS(0x4, 0xf), &i10nm_cfg1), X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(ICELAKE_D, X86_STEPPINGS(0x0, 0xf), &i10nm_cfg1), + X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SAPPHIRERAPIDS_X, X86_STEPPINGS(0x0, 0xf), &spr_cfg), {} }; MODULE_DEVICE_TABLE(x86cpu, i10nm_cpuids); @@ -148,18 +164,19 @@ static bool i10nm_check_ecc(struct skx_imc *imc, int chan) { u32 mcmtr; - mcmtr = *(u32 *)(imc->mbase + 0x20ef8 + chan * 0x4000); + mcmtr = I10NM_GET_MCMTR(imc, chan); edac_dbg(1, "ch%d mcmtr reg %x\n", chan, mcmtr); return !!GET_BITFIELD(mcmtr, 2, 2); } -static int i10nm_get_dimm_config(struct mem_ctl_info *mci) +static int i10nm_get_dimm_config(struct mem_ctl_info *mci, + struct res_config *cfg) { struct skx_pvt *pvt = mci->pvt_info; struct skx_imc *imc = pvt->imc; + u32 mtr, amap, mcddrtcfg; struct dimm_info *dimm; - u32 mtr, mcddrtcfg; int i, j, ndimms; for (i = 0; i < I10NM_NUM_CHANNELS; i++) { @@ -167,6 +184,7 @@ static int i10nm_get_dimm_config(struct mem_ctl_info *mci) continue; ndimms = 0; + amap = I10NM_GET_AMAP(imc, i); for (j = 0; j < I10NM_NUM_DIMMS; j++) { dimm = edac_get_dimm(mci, i, j, 0); mtr = I10NM_GET_DIMMMTR(imc, i, j); @@ -175,8 +193,8 @@ static int i10nm_get_dimm_config(struct mem_ctl_info *mci) mtr, mcddrtcfg, imc->mc, i, j); if (IS_DIMM_PRESENT(mtr)) - ndimms += skx_get_dimm_info(mtr, 0, 0, dimm, - imc, i, j); + ndimms += skx_get_dimm_info(mtr, 0, amap, dimm, + imc, i, j, cfg); else if (IS_NVDIMM_PRESENT(mcddrtcfg, j)) ndimms += skx_get_nvdimm_info(dimm, imc, i, j, EDAC_MOD_STR); @@ -300,10 +318,11 @@ static int __init i10nm_init(void) d->imc[i].lmc = i; d->imc[i].src_id = src_id; d->imc[i].node_id = node_id; + d->imc[i].chan_mmio_sz = cfg->ddr_chan_mmio_sz; rc = skx_register_mci(&d->imc[i], d->imc[i].mdev, "Intel_10nm Socket", EDAC_MOD_STR, - i10nm_get_dimm_config); + i10nm_get_dimm_config, cfg); if (rc < 0) goto fail; } diff --git a/drivers/edac/i3000_edac.c b/drivers/edac/i3000_edac.c index 5c1eea96230c..9065bc4386ff 100644 --- a/drivers/edac/i3000_edac.c +++ b/drivers/edac/i3000_edac.c @@ -273,7 +273,6 @@ static void i3000_check(struct mem_ctl_info *mci) { struct i3000_error_info info; - edac_dbg(1, "MC%d\n", mci->mc_idx); i3000_get_error_info(mci, &info); i3000_process_error_info(mci, &info, 1); } diff --git a/drivers/edac/i3200_edac.c b/drivers/edac/i3200_edac.c index a8988db6d423..afccdebf5ac1 100644 --- a/drivers/edac/i3200_edac.c +++ b/drivers/edac/i3200_edac.c @@ -253,7 +253,6 @@ static void i3200_check(struct mem_ctl_info *mci) { struct i3200_error_info info; - edac_dbg(1, "MC%d\n", mci->mc_idx); i3200_get_and_clear_error_info(mci, &info); i3200_process_error_info(mci, &info); } diff --git a/drivers/edac/i5000_edac.c b/drivers/edac/i5000_edac.c index 1a6f69c859ab..ba46057d4220 100644 --- a/drivers/edac/i5000_edac.c +++ b/drivers/edac/i5000_edac.c @@ -765,7 +765,7 @@ static void i5000_clear_error(struct mem_ctl_info *mci) static void i5000_check_error(struct mem_ctl_info *mci) { struct i5000_error_info info; - edac_dbg(4, "MC%d\n", mci->mc_idx); + i5000_get_error_info(mci, &info); i5000_process_error_info(mci, &info, 1); } diff --git a/drivers/edac/i5400_edac.c b/drivers/edac/i5400_edac.c index 92d63eb533ae..f76624ee82ef 100644 --- a/drivers/edac/i5400_edac.c +++ b/drivers/edac/i5400_edac.c @@ -686,7 +686,7 @@ static void i5400_clear_error(struct mem_ctl_info *mci) static void i5400_check_error(struct mem_ctl_info *mci) { struct i5400_error_info info; - edac_dbg(4, "MC%d\n", mci->mc_idx); + i5400_get_error_info(mci, &info); i5400_process_error_info(mci, &info); } diff --git a/drivers/edac/i82443bxgx_edac.c b/drivers/edac/i82443bxgx_edac.c index a2ca929e2168..933dcf3cfdff 100644 --- a/drivers/edac/i82443bxgx_edac.c +++ b/drivers/edac/i82443bxgx_edac.c @@ -176,7 +176,6 @@ static void i82443bxgx_edacmc_check(struct mem_ctl_info *mci) { struct i82443bxgx_edacmc_error_info info; - edac_dbg(1, "MC%d\n", mci->mc_idx); i82443bxgx_edacmc_get_error_info(mci, &info); i82443bxgx_edacmc_process_error_info(mci, &info, 1); } diff --git a/drivers/edac/i82860_edac.c b/drivers/edac/i82860_edac.c index 3e3a80ffb322..fbec90d00f1e 100644 --- a/drivers/edac/i82860_edac.c +++ b/drivers/edac/i82860_edac.c @@ -135,7 +135,6 @@ static void i82860_check(struct mem_ctl_info *mci) { struct i82860_error_info info; - edac_dbg(1, "MC%d\n", mci->mc_idx); i82860_get_error_info(mci, &info); i82860_process_error_info(mci, &info, 1); } diff --git a/drivers/edac/i82875p_edac.c b/drivers/edac/i82875p_edac.c index ceac925af38c..553880b9fc12 100644 --- a/drivers/edac/i82875p_edac.c +++ b/drivers/edac/i82875p_edac.c @@ -262,7 +262,6 @@ static void i82875p_check(struct mem_ctl_info *mci) { struct i82875p_error_info info; - edac_dbg(1, "MC%d\n", mci->mc_idx); i82875p_get_error_info(mci, &info); i82875p_process_error_info(mci, &info, 1); } diff --git a/drivers/edac/i82975x_edac.c b/drivers/edac/i82975x_edac.c index 6be99e0d850d..d99f005832cf 100644 --- a/drivers/edac/i82975x_edac.c +++ b/drivers/edac/i82975x_edac.c @@ -330,7 +330,6 @@ static void i82975x_check(struct mem_ctl_info *mci) { struct i82975x_error_info info; - edac_dbg(1, "MC%d\n", mci->mc_idx); i82975x_get_error_info(mci, &info); i82975x_process_error_info(mci, &info, 1); } diff --git a/drivers/edac/ie31200_edac.c b/drivers/edac/ie31200_edac.c index c47963240b65..9a9ff5ad611a 100644 --- a/drivers/edac/ie31200_edac.c +++ b/drivers/edac/ie31200_edac.c @@ -333,7 +333,6 @@ static void ie31200_check(struct mem_ctl_info *mci) { struct ie31200_error_info info; - edac_dbg(1, "MC%d\n", mci->mc_idx); ie31200_get_and_clear_error_info(mci, &info); ie31200_process_error_info(mci, &info); } diff --git a/drivers/edac/igen6_edac.c b/drivers/edac/igen6_edac.c new file mode 100644 index 000000000000..6be9986fc6bd --- /dev/null +++ b/drivers/edac/igen6_edac.c @@ -0,0 +1,977 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Driver for Intel client SoC with integrated memory controller using IBECC + * + * Copyright (C) 2020 Intel Corporation + * + * The In-Band ECC (IBECC) IP provides ECC protection to all or specific + * regions of the physical memory space. It's used for memory controllers + * that don't support the out-of-band ECC which often needs an additional + * storage device to each channel for storing ECC data. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "edac_mc.h" +#include "edac_module.h" + +#define IGEN6_REVISION "v2.4" + +#define EDAC_MOD_STR "igen6_edac" +#define IGEN6_NMI_NAME "igen6_ibecc" + +/* Debug macros */ +#define igen6_printk(level, fmt, arg...) \ + edac_printk(level, "igen6", fmt, ##arg) + +#define igen6_mc_printk(mci, level, fmt, arg...) \ + edac_mc_chipset_printk(mci, level, "igen6", fmt, ##arg) + +#define GET_BITFIELD(v, lo, hi) (((v) & GENMASK_ULL(hi, lo)) >> (lo)) + +#define NUM_IMC 1 /* Max memory controllers */ +#define NUM_CHANNELS 2 /* Max channels */ +#define NUM_DIMMS 2 /* Max DIMMs per channel */ + +#define _4GB BIT_ULL(32) + +/* Size of physical memory */ +#define TOM_OFFSET 0xa0 +/* Top of low usable DRAM */ +#define TOLUD_OFFSET 0xbc +/* Capability register C */ +#define CAPID_C_OFFSET 0xec +#define CAPID_C_IBECC BIT(15) + +/* Error Status */ +#define ERRSTS_OFFSET 0xc8 +#define ERRSTS_CE BIT_ULL(6) +#define ERRSTS_UE BIT_ULL(7) + +/* Error Command */ +#define ERRCMD_OFFSET 0xca +#define ERRCMD_CE BIT_ULL(6) +#define ERRCMD_UE BIT_ULL(7) + +/* IBECC MMIO base address */ +#define IBECC_BASE (res_cfg->ibecc_base) +#define IBECC_ACTIVATE_OFFSET IBECC_BASE +#define IBECC_ACTIVATE_EN BIT(0) + +/* IBECC error log */ +#define ECC_ERROR_LOG_OFFSET (IBECC_BASE + 0x170) +#define ECC_ERROR_LOG_CE BIT_ULL(62) +#define ECC_ERROR_LOG_UE BIT_ULL(63) +#define ECC_ERROR_LOG_ADDR_SHIFT 5 +#define ECC_ERROR_LOG_ADDR(v) GET_BITFIELD(v, 5, 38) +#define ECC_ERROR_LOG_SYND(v) GET_BITFIELD(v, 46, 61) + +/* Host MMIO base address */ +#define MCHBAR_OFFSET 0x48 +#define MCHBAR_EN BIT_ULL(0) +#define MCHBAR_BASE(v) (GET_BITFIELD(v, 16, 38) << 16) +#define MCHBAR_SIZE 0x10000 + +/* Parameters for the channel decode stage */ +#define MAD_INTER_CHANNEL_OFFSET 0x5000 +#define MAD_INTER_CHANNEL_DDR_TYPE(v) GET_BITFIELD(v, 0, 2) +#define MAD_INTER_CHANNEL_ECHM(v) GET_BITFIELD(v, 3, 3) +#define MAD_INTER_CHANNEL_CH_L_MAP(v) GET_BITFIELD(v, 4, 4) +#define MAD_INTER_CHANNEL_CH_S_SIZE(v) ((u64)GET_BITFIELD(v, 12, 19) << 29) + +/* Parameters for DRAM decode stage */ +#define MAD_INTRA_CH0_OFFSET 0x5004 +#define MAD_INTRA_CH_DIMM_L_MAP(v) GET_BITFIELD(v, 0, 0) + +/* DIMM characteristics */ +#define MAD_DIMM_CH0_OFFSET 0x500c +#define MAD_DIMM_CH_DIMM_L_SIZE(v) ((u64)GET_BITFIELD(v, 0, 6) << 29) +#define MAD_DIMM_CH_DLW(v) GET_BITFIELD(v, 7, 8) +#define MAD_DIMM_CH_DIMM_S_SIZE(v) ((u64)GET_BITFIELD(v, 16, 22) << 29) +#define MAD_DIMM_CH_DSW(v) GET_BITFIELD(v, 24, 25) + +/* Hash for channel selection */ +#define CHANNEL_HASH_OFFSET 0X5024 +/* Hash for enhanced channel selection */ +#define CHANNEL_EHASH_OFFSET 0X5028 +#define CHANNEL_HASH_MASK(v) (GET_BITFIELD(v, 6, 19) << 6) +#define CHANNEL_HASH_LSB_MASK_BIT(v) GET_BITFIELD(v, 24, 26) +#define CHANNEL_HASH_MODE(v) GET_BITFIELD(v, 28, 28) + +static struct res_config { + int num_imc; + u32 ibecc_base; + bool (*ibecc_available)(struct pci_dev *pdev); + /* Convert error address logged in IBECC to system physical address */ + u64 (*err_addr_to_sys_addr)(u64 eaddr); + /* Convert error address logged in IBECC to integrated memory controller address */ + u64 (*err_addr_to_imc_addr)(u64 eaddr); +} *res_cfg; + +struct igen6_imc { + int mc; + struct mem_ctl_info *mci; + struct pci_dev *pdev; + struct device dev; + void __iomem *window; + u64 ch_s_size; + int ch_l_map; + u64 dimm_s_size[NUM_CHANNELS]; + u64 dimm_l_size[NUM_CHANNELS]; + int dimm_l_map[NUM_CHANNELS]; +}; + +static struct igen6_pvt { + struct igen6_imc imc[NUM_IMC]; +} *igen6_pvt; + +/* The top of low usable DRAM */ +static u32 igen6_tolud; +/* The size of physical memory */ +static u64 igen6_tom; + +struct decoded_addr { + int mc; + u64 imc_addr; + u64 sys_addr; + int channel_idx; + u64 channel_addr; + int sub_channel_idx; + u64 sub_channel_addr; +}; + +struct ecclog_node { + struct llist_node llnode; + int mc; + u64 ecclog; +}; + +/* + * In the NMI handler, the driver uses the lock-less memory allocator + * to allocate memory to store the IBECC error logs and links the logs + * to the lock-less list. Delay printk() and the work of error reporting + * to EDAC core in a worker. + */ +#define ECCLOG_POOL_SIZE PAGE_SIZE +static LLIST_HEAD(ecclog_llist); +static struct gen_pool *ecclog_pool; +static char ecclog_buf[ECCLOG_POOL_SIZE]; +static struct irq_work ecclog_irq_work; +static struct work_struct ecclog_work; + +/* Compute die IDs for Elkhart Lake with IBECC */ +#define DID_EHL_SKU5 0x4514 +#define DID_EHL_SKU6 0x4528 +#define DID_EHL_SKU7 0x452a +#define DID_EHL_SKU8 0x4516 +#define DID_EHL_SKU9 0x452c +#define DID_EHL_SKU10 0x452e +#define DID_EHL_SKU11 0x4532 +#define DID_EHL_SKU12 0x4518 +#define DID_EHL_SKU13 0x451a +#define DID_EHL_SKU14 0x4534 +#define DID_EHL_SKU15 0x4536 + +static bool ehl_ibecc_available(struct pci_dev *pdev) +{ + u32 v; + + if (pci_read_config_dword(pdev, CAPID_C_OFFSET, &v)) + return false; + + return !!(CAPID_C_IBECC & v); +} + +static u64 ehl_err_addr_to_sys_addr(u64 eaddr) +{ + return eaddr; +} + +static u64 ehl_err_addr_to_imc_addr(u64 eaddr) +{ + if (eaddr < igen6_tolud) + return eaddr; + + if (igen6_tom <= _4GB) + return eaddr + igen6_tolud - _4GB; + + if (eaddr < _4GB) + return eaddr + igen6_tolud - igen6_tom; + + return eaddr; +} + +static struct res_config ehl_cfg = { + .num_imc = 1, + .ibecc_base = 0xdc00, + .ibecc_available = ehl_ibecc_available, + .err_addr_to_sys_addr = ehl_err_addr_to_sys_addr, + .err_addr_to_imc_addr = ehl_err_addr_to_imc_addr, +}; + +static const struct pci_device_id igen6_pci_tbl[] = { + { PCI_VDEVICE(INTEL, DID_EHL_SKU5), (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU6), (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU7), (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU8), (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU9), (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU10), (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU11), (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU12), (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU13), (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU14), (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU15), (kernel_ulong_t)&ehl_cfg }, + { }, +}; +MODULE_DEVICE_TABLE(pci, igen6_pci_tbl); + +static enum dev_type get_width(int dimm_l, u32 mad_dimm) +{ + u32 w = dimm_l ? MAD_DIMM_CH_DLW(mad_dimm) : + MAD_DIMM_CH_DSW(mad_dimm); + + switch (w) { + case 0: + return DEV_X8; + case 1: + return DEV_X16; + case 2: + return DEV_X32; + default: + return DEV_UNKNOWN; + } +} + +static enum mem_type get_memory_type(u32 mad_inter) +{ + u32 t = MAD_INTER_CHANNEL_DDR_TYPE(mad_inter); + + switch (t) { + case 0: + return MEM_DDR4; + case 1: + return MEM_DDR3; + case 2: + return MEM_LPDDR3; + case 3: + return MEM_LPDDR4; + case 4: + return MEM_WIO2; + default: + return MEM_UNKNOWN; + } +} + +static int decode_chan_idx(u64 addr, u64 mask, int intlv_bit) +{ + u64 hash_addr = addr & mask, hash = 0; + u64 intlv = (addr >> intlv_bit) & 1; + int i; + + for (i = 6; i < 20; i++) + hash ^= (hash_addr >> i) & 1; + + return (int)hash ^ intlv; +} + +static u64 decode_channel_addr(u64 addr, int intlv_bit) +{ + u64 channel_addr; + + /* Remove the interleave bit and shift upper part down to fill gap */ + channel_addr = GET_BITFIELD(addr, intlv_bit + 1, 63) << intlv_bit; + channel_addr |= GET_BITFIELD(addr, 0, intlv_bit - 1); + + return channel_addr; +} + +static void decode_addr(u64 addr, u32 hash, u64 s_size, int l_map, + int *idx, u64 *sub_addr) +{ + int intlv_bit = CHANNEL_HASH_LSB_MASK_BIT(hash) + 6; + + if (addr > 2 * s_size) { + *sub_addr = addr - s_size; + *idx = l_map; + return; + } + + if (CHANNEL_HASH_MODE(hash)) { + *sub_addr = decode_channel_addr(addr, intlv_bit); + *idx = decode_chan_idx(addr, CHANNEL_HASH_MASK(hash), intlv_bit); + } else { + *sub_addr = decode_channel_addr(addr, 6); + *idx = GET_BITFIELD(addr, 6, 6); + } +} + +static int igen6_decode(struct decoded_addr *res) +{ + struct igen6_imc *imc = &igen6_pvt->imc[res->mc]; + u64 addr = res->imc_addr, sub_addr, s_size; + int idx, l_map; + u32 hash; + + if (addr >= igen6_tom) { + edac_dbg(0, "Address 0x%llx out of range\n", addr); + return -EINVAL; + } + + /* Decode channel */ + hash = readl(imc->window + CHANNEL_HASH_OFFSET); + s_size = imc->ch_s_size; + l_map = imc->ch_l_map; + decode_addr(addr, hash, s_size, l_map, &idx, &sub_addr); + res->channel_idx = idx; + res->channel_addr = sub_addr; + + /* Decode sub-channel/DIMM */ + hash = readl(imc->window + CHANNEL_EHASH_OFFSET); + s_size = imc->dimm_s_size[idx]; + l_map = imc->dimm_l_map[idx]; + decode_addr(res->channel_addr, hash, s_size, l_map, &idx, &sub_addr); + res->sub_channel_idx = idx; + res->sub_channel_addr = sub_addr; + + return 0; +} + +static void igen6_output_error(struct decoded_addr *res, + struct mem_ctl_info *mci, u64 ecclog) +{ + enum hw_event_mc_err_type type = ecclog & ECC_ERROR_LOG_UE ? + HW_EVENT_ERR_UNCORRECTED : + HW_EVENT_ERR_CORRECTED; + + edac_mc_handle_error(type, mci, 1, + res->sys_addr >> PAGE_SHIFT, + res->sys_addr & ~PAGE_MASK, + ECC_ERROR_LOG_SYND(ecclog), + res->channel_idx, res->sub_channel_idx, + -1, "", ""); +} + +static struct gen_pool *ecclog_gen_pool_create(void) +{ + struct gen_pool *pool; + + pool = gen_pool_create(ilog2(sizeof(struct ecclog_node)), -1); + if (!pool) + return NULL; + + if (gen_pool_add(pool, (unsigned long)ecclog_buf, ECCLOG_POOL_SIZE, -1)) { + gen_pool_destroy(pool); + return NULL; + } + + return pool; +} + +static int ecclog_gen_pool_add(int mc, u64 ecclog) +{ + struct ecclog_node *node; + + node = (void *)gen_pool_alloc(ecclog_pool, sizeof(*node)); + if (!node) + return -ENOMEM; + + node->mc = mc; + node->ecclog = ecclog; + llist_add(&node->llnode, &ecclog_llist); + + return 0; +} + +/* + * Either the memory-mapped I/O status register ECC_ERROR_LOG or the PCI + * configuration space status register ERRSTS can indicate whether a + * correctable error or an uncorrectable error occurred. We only use the + * ECC_ERROR_LOG register to check error type, but need to clear both + * registers to enable future error events. + */ +static u64 ecclog_read_and_clear(struct igen6_imc *imc) +{ + u64 ecclog = readq(imc->window + ECC_ERROR_LOG_OFFSET); + + if (ecclog & (ECC_ERROR_LOG_CE | ECC_ERROR_LOG_UE)) { + /* Clear CE/UE bits by writing 1s */ + writeq(ecclog, imc->window + ECC_ERROR_LOG_OFFSET); + return ecclog; + } + + return 0; +} + +static void errsts_clear(struct igen6_imc *imc) +{ + u16 errsts; + + if (pci_read_config_word(imc->pdev, ERRSTS_OFFSET, &errsts)) { + igen6_printk(KERN_ERR, "Failed to read ERRSTS\n"); + return; + } + + /* Clear CE/UE bits by writing 1s */ + if (errsts & (ERRSTS_CE | ERRSTS_UE)) + pci_write_config_word(imc->pdev, ERRSTS_OFFSET, errsts); +} + +static int errcmd_enable_error_reporting(bool enable) +{ + struct igen6_imc *imc = &igen6_pvt->imc[0]; + u16 errcmd; + int rc; + + rc = pci_read_config_word(imc->pdev, ERRCMD_OFFSET, &errcmd); + if (rc) + return rc; + + if (enable) + errcmd |= ERRCMD_CE | ERRSTS_UE; + else + errcmd &= ~(ERRCMD_CE | ERRSTS_UE); + + rc = pci_write_config_word(imc->pdev, ERRCMD_OFFSET, errcmd); + if (rc) + return rc; + + return 0; +} + +static int ecclog_handler(void) +{ + struct igen6_imc *imc; + int i, n = 0; + u64 ecclog; + + for (i = 0; i < res_cfg->num_imc; i++) { + imc = &igen6_pvt->imc[i]; + + /* errsts_clear() isn't NMI-safe. Delay it in the IRQ context */ + + ecclog = ecclog_read_and_clear(imc); + if (!ecclog) + continue; + + if (!ecclog_gen_pool_add(i, ecclog)) + irq_work_queue(&ecclog_irq_work); + + n++; + } + + return n; +} + +static void ecclog_work_cb(struct work_struct *work) +{ + struct ecclog_node *node, *tmp; + struct mem_ctl_info *mci; + struct llist_node *head; + struct decoded_addr res; + u64 eaddr; + + head = llist_del_all(&ecclog_llist); + if (!head) + return; + + llist_for_each_entry_safe(node, tmp, head, llnode) { + memset(&res, 0, sizeof(res)); + eaddr = ECC_ERROR_LOG_ADDR(node->ecclog) << + ECC_ERROR_LOG_ADDR_SHIFT; + res.mc = node->mc; + res.sys_addr = res_cfg->err_addr_to_sys_addr(eaddr); + res.imc_addr = res_cfg->err_addr_to_imc_addr(eaddr); + + mci = igen6_pvt->imc[res.mc].mci; + + edac_dbg(2, "MC %d, ecclog = 0x%llx\n", node->mc, node->ecclog); + igen6_mc_printk(mci, KERN_DEBUG, "HANDLING IBECC MEMORY ERROR\n"); + igen6_mc_printk(mci, KERN_DEBUG, "ADDR 0x%llx ", res.sys_addr); + + if (!igen6_decode(&res)) + igen6_output_error(&res, mci, node->ecclog); + + gen_pool_free(ecclog_pool, (unsigned long)node, sizeof(*node)); + } +} + +static void ecclog_irq_work_cb(struct irq_work *irq_work) +{ + int i; + + for (i = 0; i < res_cfg->num_imc; i++) + errsts_clear(&igen6_pvt->imc[i]); + + if (!llist_empty(&ecclog_llist)) + schedule_work(&ecclog_work); +} + +static int ecclog_nmi_handler(unsigned int cmd, struct pt_regs *regs) +{ + unsigned char reason; + + if (!ecclog_handler()) + return NMI_DONE; + + /* + * Both In-Band ECC correctable error and uncorrectable error are + * reported by SERR# NMI. The NMI generic code (see pci_serr_error()) + * doesn't clear the bit NMI_REASON_CLEAR_SERR (in port 0x61) to + * re-enable the SERR# NMI after NMI handling. So clear this bit here + * to re-enable SERR# NMI for receiving future In-Band ECC errors. + */ + reason = x86_platform.get_nmi_reason() & NMI_REASON_CLEAR_MASK; + reason |= NMI_REASON_CLEAR_SERR; + outb(reason, NMI_REASON_PORT); + reason &= ~NMI_REASON_CLEAR_SERR; + outb(reason, NMI_REASON_PORT); + + return NMI_HANDLED; +} + +static bool igen6_check_ecc(struct igen6_imc *imc) +{ + u32 activate = readl(imc->window + IBECC_ACTIVATE_OFFSET); + + return !!(activate & IBECC_ACTIVATE_EN); +} + +static int igen6_get_dimm_config(struct mem_ctl_info *mci) +{ + struct igen6_imc *imc = mci->pvt_info; + u32 mad_inter, mad_intra, mad_dimm; + int i, j, ndimms, mc = imc->mc; + struct dimm_info *dimm; + enum mem_type mtype; + enum dev_type dtype; + u64 dsize; + bool ecc; + + edac_dbg(2, "\n"); + + mad_inter = readl(imc->window + MAD_INTER_CHANNEL_OFFSET); + mtype = get_memory_type(mad_inter); + ecc = igen6_check_ecc(imc); + imc->ch_s_size = MAD_INTER_CHANNEL_CH_S_SIZE(mad_inter); + imc->ch_l_map = MAD_INTER_CHANNEL_CH_L_MAP(mad_inter); + + for (i = 0; i < NUM_CHANNELS; i++) { + mad_intra = readl(imc->window + MAD_INTRA_CH0_OFFSET + i * 4); + mad_dimm = readl(imc->window + MAD_DIMM_CH0_OFFSET + i * 4); + + imc->dimm_l_size[i] = MAD_DIMM_CH_DIMM_L_SIZE(mad_dimm); + imc->dimm_s_size[i] = MAD_DIMM_CH_DIMM_S_SIZE(mad_dimm); + imc->dimm_l_map[i] = MAD_INTRA_CH_DIMM_L_MAP(mad_intra); + ndimms = 0; + + for (j = 0; j < NUM_DIMMS; j++) { + dimm = edac_get_dimm(mci, i, j, 0); + + if (j ^ imc->dimm_l_map[i]) { + dtype = get_width(0, mad_dimm); + dsize = imc->dimm_s_size[i]; + } else { + dtype = get_width(1, mad_dimm); + dsize = imc->dimm_l_size[i]; + } + + if (!dsize) + continue; + + dimm->grain = 64; + dimm->mtype = mtype; + dimm->dtype = dtype; + dimm->nr_pages = MiB_TO_PAGES(dsize >> 20); + dimm->edac_mode = EDAC_SECDED; + snprintf(dimm->label, sizeof(dimm->label), + "MC#%d_Chan#%d_DIMM#%d", mc, i, j); + edac_dbg(0, "MC %d, Channel %d, DIMM %d, Size %llu MiB (%u pages)\n", + mc, i, j, dsize >> 20, dimm->nr_pages); + + ndimms++; + } + + if (ndimms && !ecc) { + igen6_printk(KERN_ERR, "MC%d In-Band ECC is disabled\n", mc); + return -ENODEV; + } + } + + return 0; +} + +#ifdef CONFIG_EDAC_DEBUG +/* Top of upper usable DRAM */ +static u64 igen6_touud; +#define TOUUD_OFFSET 0xa8 + +static void igen6_reg_dump(struct igen6_imc *imc) +{ + int i; + + edac_dbg(2, "CHANNEL_HASH : 0x%x\n", + readl(imc->window + CHANNEL_HASH_OFFSET)); + edac_dbg(2, "CHANNEL_EHASH : 0x%x\n", + readl(imc->window + CHANNEL_EHASH_OFFSET)); + edac_dbg(2, "MAD_INTER_CHANNEL: 0x%x\n", + readl(imc->window + MAD_INTER_CHANNEL_OFFSET)); + edac_dbg(2, "ECC_ERROR_LOG : 0x%llx\n", + readq(imc->window + ECC_ERROR_LOG_OFFSET)); + + for (i = 0; i < NUM_CHANNELS; i++) { + edac_dbg(2, "MAD_INTRA_CH%d : 0x%x\n", i, + readl(imc->window + MAD_INTRA_CH0_OFFSET + i * 4)); + edac_dbg(2, "MAD_DIMM_CH%d : 0x%x\n", i, + readl(imc->window + MAD_DIMM_CH0_OFFSET + i * 4)); + } + edac_dbg(2, "TOLUD : 0x%x", igen6_tolud); + edac_dbg(2, "TOUUD : 0x%llx", igen6_touud); + edac_dbg(2, "TOM : 0x%llx", igen6_tom); +} + +static struct dentry *igen6_test; + +static int debugfs_u64_set(void *data, u64 val) +{ + u64 ecclog; + + if ((val >= igen6_tolud && val < _4GB) || val >= igen6_touud) { + edac_dbg(0, "Address 0x%llx out of range\n", val); + return 0; + } + + pr_warn_once("Fake error to 0x%llx injected via debugfs\n", val); + + val >>= ECC_ERROR_LOG_ADDR_SHIFT; + ecclog = (val << ECC_ERROR_LOG_ADDR_SHIFT) | ECC_ERROR_LOG_CE; + + if (!ecclog_gen_pool_add(0, ecclog)) + irq_work_queue(&ecclog_irq_work); + + return 0; +} +DEFINE_SIMPLE_ATTRIBUTE(fops_u64_wo, NULL, debugfs_u64_set, "%llu\n"); + +static void igen6_debug_setup(void) +{ + igen6_test = edac_debugfs_create_dir("igen6_test"); + if (!igen6_test) + return; + + if (!edac_debugfs_create_file("addr", 0200, igen6_test, + NULL, &fops_u64_wo)) { + debugfs_remove(igen6_test); + igen6_test = NULL; + } +} + +static void igen6_debug_teardown(void) +{ + debugfs_remove_recursive(igen6_test); +} +#else +static void igen6_reg_dump(struct igen6_imc *imc) {} +static void igen6_debug_setup(void) {} +static void igen6_debug_teardown(void) {} +#endif + +static int igen6_pci_setup(struct pci_dev *pdev, u64 *mchbar) +{ + union { + u64 v; + struct { + u32 v_lo; + u32 v_hi; + }; + } u; + + edac_dbg(2, "\n"); + + if (!res_cfg->ibecc_available(pdev)) { + edac_dbg(2, "No In-Band ECC IP\n"); + goto fail; + } + + if (pci_read_config_dword(pdev, TOLUD_OFFSET, &igen6_tolud)) { + igen6_printk(KERN_ERR, "Failed to read TOLUD\n"); + goto fail; + } + + igen6_tolud &= GENMASK(31, 20); + + if (pci_read_config_dword(pdev, TOM_OFFSET, &u.v_lo)) { + igen6_printk(KERN_ERR, "Failed to read lower TOM\n"); + goto fail; + } + + if (pci_read_config_dword(pdev, TOM_OFFSET + 4, &u.v_hi)) { + igen6_printk(KERN_ERR, "Failed to read upper TOM\n"); + goto fail; + } + + igen6_tom = u.v & GENMASK_ULL(38, 20); + + if (pci_read_config_dword(pdev, MCHBAR_OFFSET, &u.v_lo)) { + igen6_printk(KERN_ERR, "Failed to read lower MCHBAR\n"); + goto fail; + } + + if (pci_read_config_dword(pdev, MCHBAR_OFFSET + 4, &u.v_hi)) { + igen6_printk(KERN_ERR, "Failed to read upper MCHBAR\n"); + goto fail; + } + + if (!(u.v & MCHBAR_EN)) { + igen6_printk(KERN_ERR, "MCHBAR is disabled\n"); + goto fail; + } + + *mchbar = MCHBAR_BASE(u.v); + +#ifdef CONFIG_EDAC_DEBUG + if (pci_read_config_dword(pdev, TOUUD_OFFSET, &u.v_lo)) + edac_dbg(2, "Failed to read lower TOUUD\n"); + else if (pci_read_config_dword(pdev, TOUUD_OFFSET + 4, &u.v_hi)) + edac_dbg(2, "Failed to read upper TOUUD\n"); + else + igen6_touud = u.v & GENMASK_ULL(38, 20); +#endif + + return 0; +fail: + return -ENODEV; +} + +static int igen6_register_mci(int mc, u64 mchbar, struct pci_dev *pdev) +{ + struct edac_mc_layer layers[2]; + struct mem_ctl_info *mci; + struct igen6_imc *imc; + void __iomem *window; + int rc; + + edac_dbg(2, "\n"); + + mchbar += mc * MCHBAR_SIZE; + window = ioremap(mchbar, MCHBAR_SIZE); + if (!window) { + igen6_printk(KERN_ERR, "Failed to ioremap 0x%llx\n", mchbar); + return -ENODEV; + } + + layers[0].type = EDAC_MC_LAYER_CHANNEL; + layers[0].size = NUM_CHANNELS; + layers[0].is_virt_csrow = false; + layers[1].type = EDAC_MC_LAYER_SLOT; + layers[1].size = NUM_DIMMS; + layers[1].is_virt_csrow = true; + + mci = edac_mc_alloc(mc, ARRAY_SIZE(layers), layers, 0); + if (!mci) { + rc = -ENOMEM; + goto fail; + } + + mci->ctl_name = kasprintf(GFP_KERNEL, "Intel_client_SoC MC#%d", mc); + if (!mci->ctl_name) { + rc = -ENOMEM; + goto fail2; + } + + mci->mtype_cap = MEM_FLAG_LPDDR4 | MEM_FLAG_DDR4; + mci->edac_ctl_cap = EDAC_FLAG_SECDED; + mci->edac_cap = EDAC_FLAG_SECDED; + mci->mod_name = EDAC_MOD_STR; + mci->dev_name = pci_name(pdev); + mci->pvt_info = &igen6_pvt->imc[mc]; + + imc = mci->pvt_info; + device_initialize(&imc->dev); + /* + * EDAC core uses mci->pdev(pointer of structure device) as + * memory controller ID. The client SoCs attach one or more + * memory controllers to single pci_dev (single pci_dev->dev + * can be for multiple memory controllers). + * + * To make mci->pdev unique, assign pci_dev->dev to mci->pdev + * for the first memory controller and assign a unique imc->dev + * to mci->pdev for each non-first memory controller. + */ + mci->pdev = mc ? &imc->dev : &pdev->dev; + imc->mc = mc; + imc->pdev = pdev; + imc->window = window; + + igen6_reg_dump(imc); + + rc = igen6_get_dimm_config(mci); + if (rc) + goto fail3; + + rc = edac_mc_add_mc(mci); + if (rc) { + igen6_printk(KERN_ERR, "Failed to register mci#%d\n", mc); + goto fail3; + } + + imc->mci = mci; + return 0; +fail3: + kfree(mci->ctl_name); +fail2: + edac_mc_free(mci); +fail: + iounmap(window); + return rc; +} + +static void igen6_unregister_mcis(void) +{ + struct mem_ctl_info *mci; + struct igen6_imc *imc; + int i; + + edac_dbg(2, "\n"); + + for (i = 0; i < res_cfg->num_imc; i++) { + imc = &igen6_pvt->imc[i]; + mci = imc->mci; + if (!mci) + continue; + + edac_mc_del_mc(mci->pdev); + kfree(mci->ctl_name); + edac_mc_free(mci); + iounmap(imc->window); + } +} + +static int igen6_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + u64 mchbar; + int i, rc; + + edac_dbg(2, "\n"); + + igen6_pvt = kzalloc(sizeof(*igen6_pvt), GFP_KERNEL); + if (!igen6_pvt) + return -ENOMEM; + + res_cfg = (struct res_config *)ent->driver_data; + + rc = igen6_pci_setup(pdev, &mchbar); + if (rc) + goto fail; + + for (i = 0; i < res_cfg->num_imc; i++) { + rc = igen6_register_mci(i, mchbar, pdev); + if (rc) + goto fail2; + } + + ecclog_pool = ecclog_gen_pool_create(); + if (!ecclog_pool) { + rc = -ENOMEM; + goto fail2; + } + + INIT_WORK(&ecclog_work, ecclog_work_cb); + init_irq_work(&ecclog_irq_work, ecclog_irq_work_cb); + + /* Check if any pending errors before registering the NMI handler */ + ecclog_handler(); + + rc = register_nmi_handler(NMI_SERR, ecclog_nmi_handler, + 0, IGEN6_NMI_NAME); + if (rc) { + igen6_printk(KERN_ERR, "Failed to register NMI handler\n"); + goto fail3; + } + + /* Enable error reporting */ + rc = errcmd_enable_error_reporting(true); + if (rc) { + igen6_printk(KERN_ERR, "Failed to enable error reporting\n"); + goto fail4; + } + + igen6_debug_setup(); + return 0; +fail4: + unregister_nmi_handler(NMI_SERR, IGEN6_NMI_NAME); +fail3: + gen_pool_destroy(ecclog_pool); +fail2: + igen6_unregister_mcis(); +fail: + kfree(igen6_pvt); + return rc; +} + +static void igen6_remove(struct pci_dev *pdev) +{ + edac_dbg(2, "\n"); + + igen6_debug_teardown(); + errcmd_enable_error_reporting(false); + unregister_nmi_handler(NMI_SERR, IGEN6_NMI_NAME); + irq_work_sync(&ecclog_irq_work); + flush_work(&ecclog_work); + gen_pool_destroy(ecclog_pool); + igen6_unregister_mcis(); + kfree(igen6_pvt); +} + +static struct pci_driver igen6_driver = { + .name = EDAC_MOD_STR, + .probe = igen6_probe, + .remove = igen6_remove, + .id_table = igen6_pci_tbl, +}; + +static int __init igen6_init(void) +{ + const char *owner; + int rc; + + edac_dbg(2, "\n"); + + owner = edac_get_owner(); + if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR))) + return -ENODEV; + + edac_op_state = EDAC_OPSTATE_NMI; + + rc = pci_register_driver(&igen6_driver); + if (rc) + return rc; + + igen6_printk(KERN_INFO, "%s\n", IGEN6_REVISION); + + return 0; +} + +static void __exit igen6_exit(void) +{ + edac_dbg(2, "\n"); + + pci_unregister_driver(&igen6_driver); +} + +module_init(igen6_init); +module_exit(igen6_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Qiuxu Zhuo"); +MODULE_DESCRIPTION("MC Driver for Intel client SoC using In-Band ECC"); diff --git a/drivers/edac/mce_amd.c b/drivers/edac/mce_amd.c index 7f28edb070bd..5dd905a3f30c 100644 --- a/drivers/edac/mce_amd.c +++ b/drivers/edac/mce_amd.c @@ -869,7 +869,7 @@ static void decode_mc3_mce(struct mce *m) static void decode_mc4_mce(struct mce *m) { unsigned int fam = x86_family(m->cpuid); - int node_id = amd_get_nb_id(m->extcpu); + int node_id = topology_die_id(m->extcpu); u16 ec = EC(m->status); u8 xec = XEC(m->status, 0x1f); u8 offset = 0; @@ -1003,7 +1003,7 @@ static void decode_smca_error(struct mce *m) pr_cont(", %s.\n", smca_mce_descs[bank_type].descs[xec]); if (bank_type == SMCA_UMC && xec == 0 && decode_dram_ecc) - decode_dram_ecc(cpu_to_node(m->extcpu), m); + decode_dram_ecc(topology_die_id(m->extcpu), m); } static inline void amd_decode_err_code(u16 ec) diff --git a/drivers/edac/mv64x60_edac.c b/drivers/edac/mv64x60_edac.c deleted file mode 100644 index 3c68bb525d5d..000000000000 --- a/drivers/edac/mv64x60_edac.c +++ /dev/null @@ -1,883 +0,0 @@ -/* - * Marvell MV64x60 Memory Controller kernel module for PPC platforms - * - * Author: Dave Jiang - * - * 2006-2007 (c) MontaVista Software, Inc. This file is licensed under - * the terms of the GNU General Public License version 2. This program - * is licensed "as is" without any warranty of any kind, whether express - * or implied. - * - */ - -#include -#include -#include -#include -#include -#include - -#include "edac_module.h" -#include "mv64x60_edac.h" - -static const char *mv64x60_ctl_name = "MV64x60"; -static int edac_dev_idx; -static int edac_pci_idx; -static int edac_mc_idx; - -/*********************** PCI err device **********************************/ -#ifdef CONFIG_PCI -static void mv64x60_pci_check(struct edac_pci_ctl_info *pci) -{ - struct mv64x60_pci_pdata *pdata = pci->pvt_info; - u32 cause; - - cause = readl(pdata->pci_vbase + MV64X60_PCI_ERROR_CAUSE); - if (!cause) - return; - - printk(KERN_ERR "Error in PCI %d Interface\n", pdata->pci_hose); - printk(KERN_ERR "Cause register: 0x%08x\n", cause); - printk(KERN_ERR "Address Low: 0x%08x\n", - readl(pdata->pci_vbase + MV64X60_PCI_ERROR_ADDR_LO)); - printk(KERN_ERR "Address High: 0x%08x\n", - readl(pdata->pci_vbase + MV64X60_PCI_ERROR_ADDR_HI)); - printk(KERN_ERR "Attribute: 0x%08x\n", - readl(pdata->pci_vbase + MV64X60_PCI_ERROR_ATTR)); - printk(KERN_ERR "Command: 0x%08x\n", - readl(pdata->pci_vbase + MV64X60_PCI_ERROR_CMD)); - writel(~cause, pdata->pci_vbase + MV64X60_PCI_ERROR_CAUSE); - - if (cause & MV64X60_PCI_PE_MASK) - edac_pci_handle_pe(pci, pci->ctl_name); - - if (!(cause & MV64X60_PCI_PE_MASK)) - edac_pci_handle_npe(pci, pci->ctl_name); -} - -static irqreturn_t mv64x60_pci_isr(int irq, void *dev_id) -{ - struct edac_pci_ctl_info *pci = dev_id; - struct mv64x60_pci_pdata *pdata = pci->pvt_info; - u32 val; - - val = readl(pdata->pci_vbase + MV64X60_PCI_ERROR_CAUSE); - if (!val) - return IRQ_NONE; - - mv64x60_pci_check(pci); - - return IRQ_HANDLED; -} - -/* - * Bit 0 of MV64x60_PCIx_ERR_MASK does not exist on the 64360 and because of - * errata FEr-#11 and FEr-##16 for the 64460, it should be 0 on that chip as - * well. IOW, don't set bit 0. - */ - -/* Erratum FEr PCI-#16: clear bit 0 of PCI SERRn Mask reg. */ -static int __init mv64x60_pci_fixup(struct platform_device *pdev) -{ - struct resource *r; - void __iomem *pci_serr; - - r = platform_get_resource(pdev, IORESOURCE_MEM, 1); - if (!r) { - printk(KERN_ERR "%s: Unable to get resource for " - "PCI err regs\n", __func__); - return -ENOENT; - } - - pci_serr = ioremap(r->start, resource_size(r)); - if (!pci_serr) - return -ENOMEM; - - writel(readl(pci_serr) & ~0x1, pci_serr); - iounmap(pci_serr); - - return 0; -} - -static int mv64x60_pci_err_probe(struct platform_device *pdev) -{ - struct edac_pci_ctl_info *pci; - struct mv64x60_pci_pdata *pdata; - struct resource *r; - int res = 0; - - if (!devres_open_group(&pdev->dev, mv64x60_pci_err_probe, GFP_KERNEL)) - return -ENOMEM; - - pci = edac_pci_alloc_ctl_info(sizeof(*pdata), "mv64x60_pci_err"); - if (!pci) - return -ENOMEM; - - pdata = pci->pvt_info; - - pdata->pci_hose = pdev->id; - pdata->name = "mv64x60_pci_err"; - platform_set_drvdata(pdev, pci); - pci->dev = &pdev->dev; - pci->dev_name = dev_name(&pdev->dev); - pci->mod_name = EDAC_MOD_STR; - pci->ctl_name = pdata->name; - - if (edac_op_state == EDAC_OPSTATE_POLL) - pci->edac_check = mv64x60_pci_check; - - pdata->edac_idx = edac_pci_idx++; - - r = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!r) { - printk(KERN_ERR "%s: Unable to get resource for " - "PCI err regs\n", __func__); - res = -ENOENT; - goto err; - } - - if (!devm_request_mem_region(&pdev->dev, - r->start, - resource_size(r), - pdata->name)) { - printk(KERN_ERR "%s: Error while requesting mem region\n", - __func__); - res = -EBUSY; - goto err; - } - - pdata->pci_vbase = devm_ioremap(&pdev->dev, - r->start, - resource_size(r)); - if (!pdata->pci_vbase) { - printk(KERN_ERR "%s: Unable to setup PCI err regs\n", __func__); - res = -ENOMEM; - goto err; - } - - res = mv64x60_pci_fixup(pdev); - if (res < 0) { - printk(KERN_ERR "%s: PCI fixup failed\n", __func__); - goto err; - } - - writel(0, pdata->pci_vbase + MV64X60_PCI_ERROR_CAUSE); - writel(0, pdata->pci_vbase + MV64X60_PCI_ERROR_MASK); - writel(MV64X60_PCIx_ERR_MASK_VAL, - pdata->pci_vbase + MV64X60_PCI_ERROR_MASK); - - if (edac_pci_add_device(pci, pdata->edac_idx) > 0) { - edac_dbg(3, "failed edac_pci_add_device()\n"); - goto err; - } - - if (edac_op_state == EDAC_OPSTATE_INT) { - pdata->irq = platform_get_irq(pdev, 0); - res = devm_request_irq(&pdev->dev, - pdata->irq, - mv64x60_pci_isr, - 0, - "[EDAC] PCI err", - pci); - if (res < 0) { - printk(KERN_ERR "%s: Unable to request irq %d for " - "MV64x60 PCI ERR\n", __func__, pdata->irq); - res = -ENODEV; - goto err2; - } - printk(KERN_INFO EDAC_MOD_STR " acquired irq %d for PCI Err\n", - pdata->irq); - } - - devres_remove_group(&pdev->dev, mv64x60_pci_err_probe); - - /* get this far and it's successful */ - edac_dbg(3, "success\n"); - - return 0; - -err2: - edac_pci_del_device(&pdev->dev); -err: - edac_pci_free_ctl_info(pci); - devres_release_group(&pdev->dev, mv64x60_pci_err_probe); - return res; -} - -static int mv64x60_pci_err_remove(struct platform_device *pdev) -{ - struct edac_pci_ctl_info *pci = platform_get_drvdata(pdev); - - edac_dbg(0, "\n"); - - edac_pci_del_device(&pdev->dev); - - edac_pci_free_ctl_info(pci); - - return 0; -} - -static struct platform_driver mv64x60_pci_err_driver = { - .probe = mv64x60_pci_err_probe, - .remove = mv64x60_pci_err_remove, - .driver = { - .name = "mv64x60_pci_err", - } -}; - -#endif /* CONFIG_PCI */ - -/*********************** SRAM err device **********************************/ -static void mv64x60_sram_check(struct edac_device_ctl_info *edac_dev) -{ - struct mv64x60_sram_pdata *pdata = edac_dev->pvt_info; - u32 cause; - - cause = readl(pdata->sram_vbase + MV64X60_SRAM_ERR_CAUSE); - if (!cause) - return; - - printk(KERN_ERR "Error in internal SRAM\n"); - printk(KERN_ERR "Cause register: 0x%08x\n", cause); - printk(KERN_ERR "Address Low: 0x%08x\n", - readl(pdata->sram_vbase + MV64X60_SRAM_ERR_ADDR_LO)); - printk(KERN_ERR "Address High: 0x%08x\n", - readl(pdata->sram_vbase + MV64X60_SRAM_ERR_ADDR_HI)); - printk(KERN_ERR "Data Low: 0x%08x\n", - readl(pdata->sram_vbase + MV64X60_SRAM_ERR_DATA_LO)); - printk(KERN_ERR "Data High: 0x%08x\n", - readl(pdata->sram_vbase + MV64X60_SRAM_ERR_DATA_HI)); - printk(KERN_ERR "Parity: 0x%08x\n", - readl(pdata->sram_vbase + MV64X60_SRAM_ERR_PARITY)); - writel(0, pdata->sram_vbase + MV64X60_SRAM_ERR_CAUSE); - - edac_device_handle_ue(edac_dev, 0, 0, edac_dev->ctl_name); -} - -static irqreturn_t mv64x60_sram_isr(int irq, void *dev_id) -{ - struct edac_device_ctl_info *edac_dev = dev_id; - struct mv64x60_sram_pdata *pdata = edac_dev->pvt_info; - u32 cause; - - cause = readl(pdata->sram_vbase + MV64X60_SRAM_ERR_CAUSE); - if (!cause) - return IRQ_NONE; - - mv64x60_sram_check(edac_dev); - - return IRQ_HANDLED; -} - -static int mv64x60_sram_err_probe(struct platform_device *pdev) -{ - struct edac_device_ctl_info *edac_dev; - struct mv64x60_sram_pdata *pdata; - struct resource *r; - int res = 0; - - if (!devres_open_group(&pdev->dev, mv64x60_sram_err_probe, GFP_KERNEL)) - return -ENOMEM; - - edac_dev = edac_device_alloc_ctl_info(sizeof(*pdata), - "sram", 1, NULL, 0, 0, NULL, 0, - edac_dev_idx); - if (!edac_dev) { - devres_release_group(&pdev->dev, mv64x60_sram_err_probe); - return -ENOMEM; - } - - pdata = edac_dev->pvt_info; - pdata->name = "mv64x60_sram_err"; - edac_dev->dev = &pdev->dev; - platform_set_drvdata(pdev, edac_dev); - edac_dev->dev_name = dev_name(&pdev->dev); - - r = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!r) { - printk(KERN_ERR "%s: Unable to get resource for " - "SRAM err regs\n", __func__); - res = -ENOENT; - goto err; - } - - if (!devm_request_mem_region(&pdev->dev, - r->start, - resource_size(r), - pdata->name)) { - printk(KERN_ERR "%s: Error while request mem region\n", - __func__); - res = -EBUSY; - goto err; - } - - pdata->sram_vbase = devm_ioremap(&pdev->dev, - r->start, - resource_size(r)); - if (!pdata->sram_vbase) { - printk(KERN_ERR "%s: Unable to setup SRAM err regs\n", - __func__); - res = -ENOMEM; - goto err; - } - - /* setup SRAM err registers */ - writel(0, pdata->sram_vbase + MV64X60_SRAM_ERR_CAUSE); - - edac_dev->mod_name = EDAC_MOD_STR; - edac_dev->ctl_name = pdata->name; - - if (edac_op_state == EDAC_OPSTATE_POLL) - edac_dev->edac_check = mv64x60_sram_check; - - pdata->edac_idx = edac_dev_idx++; - - if (edac_device_add_device(edac_dev) > 0) { - edac_dbg(3, "failed edac_device_add_device()\n"); - goto err; - } - - if (edac_op_state == EDAC_OPSTATE_INT) { - pdata->irq = platform_get_irq(pdev, 0); - res = devm_request_irq(&pdev->dev, - pdata->irq, - mv64x60_sram_isr, - 0, - "[EDAC] SRAM err", - edac_dev); - if (res < 0) { - printk(KERN_ERR - "%s: Unable to request irq %d for " - "MV64x60 SRAM ERR\n", __func__, pdata->irq); - res = -ENODEV; - goto err2; - } - - printk(KERN_INFO EDAC_MOD_STR " acquired irq %d for SRAM Err\n", - pdata->irq); - } - - devres_remove_group(&pdev->dev, mv64x60_sram_err_probe); - - /* get this far and it's successful */ - edac_dbg(3, "success\n"); - - return 0; - -err2: - edac_device_del_device(&pdev->dev); -err: - devres_release_group(&pdev->dev, mv64x60_sram_err_probe); - edac_device_free_ctl_info(edac_dev); - return res; -} - -static int mv64x60_sram_err_remove(struct platform_device *pdev) -{ - struct edac_device_ctl_info *edac_dev = platform_get_drvdata(pdev); - - edac_dbg(0, "\n"); - - edac_device_del_device(&pdev->dev); - edac_device_free_ctl_info(edac_dev); - - return 0; -} - -static struct platform_driver mv64x60_sram_err_driver = { - .probe = mv64x60_sram_err_probe, - .remove = mv64x60_sram_err_remove, - .driver = { - .name = "mv64x60_sram_err", - } -}; - -/*********************** CPU err device **********************************/ -static void mv64x60_cpu_check(struct edac_device_ctl_info *edac_dev) -{ - struct mv64x60_cpu_pdata *pdata = edac_dev->pvt_info; - u32 cause; - - cause = readl(pdata->cpu_vbase[1] + MV64x60_CPU_ERR_CAUSE) & - MV64x60_CPU_CAUSE_MASK; - if (!cause) - return; - - printk(KERN_ERR "Error on CPU interface\n"); - printk(KERN_ERR "Cause register: 0x%08x\n", cause); - printk(KERN_ERR "Address Low: 0x%08x\n", - readl(pdata->cpu_vbase[0] + MV64x60_CPU_ERR_ADDR_LO)); - printk(KERN_ERR "Address High: 0x%08x\n", - readl(pdata->cpu_vbase[0] + MV64x60_CPU_ERR_ADDR_HI)); - printk(KERN_ERR "Data Low: 0x%08x\n", - readl(pdata->cpu_vbase[1] + MV64x60_CPU_ERR_DATA_LO)); - printk(KERN_ERR "Data High: 0x%08x\n", - readl(pdata->cpu_vbase[1] + MV64x60_CPU_ERR_DATA_HI)); - printk(KERN_ERR "Parity: 0x%08x\n", - readl(pdata->cpu_vbase[1] + MV64x60_CPU_ERR_PARITY)); - writel(0, pdata->cpu_vbase[1] + MV64x60_CPU_ERR_CAUSE); - - edac_device_handle_ue(edac_dev, 0, 0, edac_dev->ctl_name); -} - -static irqreturn_t mv64x60_cpu_isr(int irq, void *dev_id) -{ - struct edac_device_ctl_info *edac_dev = dev_id; - struct mv64x60_cpu_pdata *pdata = edac_dev->pvt_info; - u32 cause; - - cause = readl(pdata->cpu_vbase[1] + MV64x60_CPU_ERR_CAUSE) & - MV64x60_CPU_CAUSE_MASK; - if (!cause) - return IRQ_NONE; - - mv64x60_cpu_check(edac_dev); - - return IRQ_HANDLED; -} - -static int mv64x60_cpu_err_probe(struct platform_device *pdev) -{ - struct edac_device_ctl_info *edac_dev; - struct resource *r; - struct mv64x60_cpu_pdata *pdata; - int res = 0; - - if (!devres_open_group(&pdev->dev, mv64x60_cpu_err_probe, GFP_KERNEL)) - return -ENOMEM; - - edac_dev = edac_device_alloc_ctl_info(sizeof(*pdata), - "cpu", 1, NULL, 0, 0, NULL, 0, - edac_dev_idx); - if (!edac_dev) { - devres_release_group(&pdev->dev, mv64x60_cpu_err_probe); - return -ENOMEM; - } - - pdata = edac_dev->pvt_info; - pdata->name = "mv64x60_cpu_err"; - edac_dev->dev = &pdev->dev; - platform_set_drvdata(pdev, edac_dev); - edac_dev->dev_name = dev_name(&pdev->dev); - - r = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!r) { - printk(KERN_ERR "%s: Unable to get resource for " - "CPU err regs\n", __func__); - res = -ENOENT; - goto err; - } - - if (!devm_request_mem_region(&pdev->dev, - r->start, - resource_size(r), - pdata->name)) { - printk(KERN_ERR "%s: Error while requesting mem region\n", - __func__); - res = -EBUSY; - goto err; - } - - pdata->cpu_vbase[0] = devm_ioremap(&pdev->dev, - r->start, - resource_size(r)); - if (!pdata->cpu_vbase[0]) { - printk(KERN_ERR "%s: Unable to setup CPU err regs\n", __func__); - res = -ENOMEM; - goto err; - } - - r = platform_get_resource(pdev, IORESOURCE_MEM, 1); - if (!r) { - printk(KERN_ERR "%s: Unable to get resource for " - "CPU err regs\n", __func__); - res = -ENOENT; - goto err; - } - - if (!devm_request_mem_region(&pdev->dev, - r->start, - resource_size(r), - pdata->name)) { - printk(KERN_ERR "%s: Error while requesting mem region\n", - __func__); - res = -EBUSY; - goto err; - } - - pdata->cpu_vbase[1] = devm_ioremap(&pdev->dev, - r->start, - resource_size(r)); - if (!pdata->cpu_vbase[1]) { - printk(KERN_ERR "%s: Unable to setup CPU err regs\n", __func__); - res = -ENOMEM; - goto err; - } - - /* setup CPU err registers */ - writel(0, pdata->cpu_vbase[1] + MV64x60_CPU_ERR_CAUSE); - writel(0, pdata->cpu_vbase[1] + MV64x60_CPU_ERR_MASK); - writel(0x000000ff, pdata->cpu_vbase[1] + MV64x60_CPU_ERR_MASK); - - edac_dev->mod_name = EDAC_MOD_STR; - edac_dev->ctl_name = pdata->name; - if (edac_op_state == EDAC_OPSTATE_POLL) - edac_dev->edac_check = mv64x60_cpu_check; - - pdata->edac_idx = edac_dev_idx++; - - if (edac_device_add_device(edac_dev) > 0) { - edac_dbg(3, "failed edac_device_add_device()\n"); - goto err; - } - - if (edac_op_state == EDAC_OPSTATE_INT) { - pdata->irq = platform_get_irq(pdev, 0); - res = devm_request_irq(&pdev->dev, - pdata->irq, - mv64x60_cpu_isr, - 0, - "[EDAC] CPU err", - edac_dev); - if (res < 0) { - printk(KERN_ERR - "%s: Unable to request irq %d for MV64x60 " - "CPU ERR\n", __func__, pdata->irq); - res = -ENODEV; - goto err2; - } - - printk(KERN_INFO EDAC_MOD_STR - " acquired irq %d for CPU Err\n", pdata->irq); - } - - devres_remove_group(&pdev->dev, mv64x60_cpu_err_probe); - - /* get this far and it's successful */ - edac_dbg(3, "success\n"); - - return 0; - -err2: - edac_device_del_device(&pdev->dev); -err: - devres_release_group(&pdev->dev, mv64x60_cpu_err_probe); - edac_device_free_ctl_info(edac_dev); - return res; -} - -static int mv64x60_cpu_err_remove(struct platform_device *pdev) -{ - struct edac_device_ctl_info *edac_dev = platform_get_drvdata(pdev); - - edac_dbg(0, "\n"); - - edac_device_del_device(&pdev->dev); - edac_device_free_ctl_info(edac_dev); - return 0; -} - -static struct platform_driver mv64x60_cpu_err_driver = { - .probe = mv64x60_cpu_err_probe, - .remove = mv64x60_cpu_err_remove, - .driver = { - .name = "mv64x60_cpu_err", - } -}; - -/*********************** DRAM err device **********************************/ - -static void mv64x60_mc_check(struct mem_ctl_info *mci) -{ - struct mv64x60_mc_pdata *pdata = mci->pvt_info; - u32 reg; - u32 err_addr; - u32 sdram_ecc; - u32 comp_ecc; - u32 syndrome; - - reg = readl(pdata->mc_vbase + MV64X60_SDRAM_ERR_ADDR); - if (!reg) - return; - - err_addr = reg & ~0x3; - sdram_ecc = readl(pdata->mc_vbase + MV64X60_SDRAM_ERR_ECC_RCVD); - comp_ecc = readl(pdata->mc_vbase + MV64X60_SDRAM_ERR_ECC_CALC); - syndrome = sdram_ecc ^ comp_ecc; - - /* first bit clear in ECC Err Reg, 1 bit error, correctable by HW */ - if (!(reg & 0x1)) - edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, 1, - err_addr >> PAGE_SHIFT, - err_addr & PAGE_MASK, syndrome, - 0, 0, -1, - mci->ctl_name, ""); - else /* 2 bit error, UE */ - edac_mc_handle_error(HW_EVENT_ERR_UNCORRECTED, mci, 1, - err_addr >> PAGE_SHIFT, - err_addr & PAGE_MASK, 0, - 0, 0, -1, - mci->ctl_name, ""); - - /* clear the error */ - writel(0, pdata->mc_vbase + MV64X60_SDRAM_ERR_ADDR); -} - -static irqreturn_t mv64x60_mc_isr(int irq, void *dev_id) -{ - struct mem_ctl_info *mci = dev_id; - struct mv64x60_mc_pdata *pdata = mci->pvt_info; - u32 reg; - - reg = readl(pdata->mc_vbase + MV64X60_SDRAM_ERR_ADDR); - if (!reg) - return IRQ_NONE; - - /* writing 0's to the ECC err addr in check function clears irq */ - mv64x60_mc_check(mci); - - return IRQ_HANDLED; -} - -static void get_total_mem(struct mv64x60_mc_pdata *pdata) -{ - struct device_node *np = NULL; - const unsigned int *reg; - - np = of_find_node_by_type(NULL, "memory"); - if (!np) - return; - - reg = of_get_property(np, "reg", NULL); - - pdata->total_mem = reg[1]; -} - -static void mv64x60_init_csrows(struct mem_ctl_info *mci, - struct mv64x60_mc_pdata *pdata) -{ - struct csrow_info *csrow; - struct dimm_info *dimm; - - u32 devtype; - u32 ctl; - - get_total_mem(pdata); - - ctl = readl(pdata->mc_vbase + MV64X60_SDRAM_CONFIG); - - csrow = mci->csrows[0]; - dimm = csrow->channels[0]->dimm; - - dimm->nr_pages = pdata->total_mem >> PAGE_SHIFT; - dimm->grain = 8; - - dimm->mtype = (ctl & MV64X60_SDRAM_REGISTERED) ? MEM_RDDR : MEM_DDR; - - devtype = (ctl >> 20) & 0x3; - switch (devtype) { - case 0x0: - dimm->dtype = DEV_X32; - break; - case 0x2: /* could be X8 too, but no way to tell */ - dimm->dtype = DEV_X16; - break; - case 0x3: - dimm->dtype = DEV_X4; - break; - default: - dimm->dtype = DEV_UNKNOWN; - break; - } - - dimm->edac_mode = EDAC_SECDED; -} - -static int mv64x60_mc_err_probe(struct platform_device *pdev) -{ - struct mem_ctl_info *mci; - struct edac_mc_layer layers[2]; - struct mv64x60_mc_pdata *pdata; - struct resource *r; - u32 ctl; - int res = 0; - - if (!devres_open_group(&pdev->dev, mv64x60_mc_err_probe, GFP_KERNEL)) - return -ENOMEM; - - layers[0].type = EDAC_MC_LAYER_CHIP_SELECT; - layers[0].size = 1; - layers[0].is_virt_csrow = true; - layers[1].type = EDAC_MC_LAYER_CHANNEL; - layers[1].size = 1; - layers[1].is_virt_csrow = false; - mci = edac_mc_alloc(edac_mc_idx, ARRAY_SIZE(layers), layers, - sizeof(struct mv64x60_mc_pdata)); - if (!mci) { - printk(KERN_ERR "%s: No memory for CPU err\n", __func__); - devres_release_group(&pdev->dev, mv64x60_mc_err_probe); - return -ENOMEM; - } - - pdata = mci->pvt_info; - mci->pdev = &pdev->dev; - platform_set_drvdata(pdev, mci); - pdata->name = "mv64x60_mc_err"; - mci->dev_name = dev_name(&pdev->dev); - pdata->edac_idx = edac_mc_idx++; - - r = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!r) { - printk(KERN_ERR "%s: Unable to get resource for " - "MC err regs\n", __func__); - res = -ENOENT; - goto err; - } - - if (!devm_request_mem_region(&pdev->dev, - r->start, - resource_size(r), - pdata->name)) { - printk(KERN_ERR "%s: Error while requesting mem region\n", - __func__); - res = -EBUSY; - goto err; - } - - pdata->mc_vbase = devm_ioremap(&pdev->dev, - r->start, - resource_size(r)); - if (!pdata->mc_vbase) { - printk(KERN_ERR "%s: Unable to setup MC err regs\n", __func__); - res = -ENOMEM; - goto err; - } - - ctl = readl(pdata->mc_vbase + MV64X60_SDRAM_CONFIG); - if (!(ctl & MV64X60_SDRAM_ECC)) { - /* Non-ECC RAM? */ - printk(KERN_WARNING "%s: No ECC DIMMs discovered\n", __func__); - res = -ENODEV; - goto err; - } - - edac_dbg(3, "init mci\n"); - mci->mtype_cap = MEM_FLAG_RDDR | MEM_FLAG_DDR; - mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED; - mci->edac_cap = EDAC_FLAG_SECDED; - mci->mod_name = EDAC_MOD_STR; - mci->ctl_name = mv64x60_ctl_name; - - if (edac_op_state == EDAC_OPSTATE_POLL) - mci->edac_check = mv64x60_mc_check; - - mci->ctl_page_to_phys = NULL; - - mci->scrub_mode = SCRUB_SW_SRC; - - mv64x60_init_csrows(mci, pdata); - - /* setup MC registers */ - writel(0, pdata->mc_vbase + MV64X60_SDRAM_ERR_ADDR); - ctl = readl(pdata->mc_vbase + MV64X60_SDRAM_ERR_ECC_CNTL); - ctl = (ctl & 0xff00ffff) | 0x10000; - writel(ctl, pdata->mc_vbase + MV64X60_SDRAM_ERR_ECC_CNTL); - - res = edac_mc_add_mc(mci); - if (res) { - edac_dbg(3, "failed edac_mc_add_mc()\n"); - goto err; - } - - if (edac_op_state == EDAC_OPSTATE_INT) { - /* acquire interrupt that reports errors */ - pdata->irq = platform_get_irq(pdev, 0); - res = devm_request_irq(&pdev->dev, - pdata->irq, - mv64x60_mc_isr, - 0, - "[EDAC] MC err", - mci); - if (res < 0) { - printk(KERN_ERR "%s: Unable to request irq %d for " - "MV64x60 DRAM ERR\n", __func__, pdata->irq); - res = -ENODEV; - goto err2; - } - - printk(KERN_INFO EDAC_MOD_STR " acquired irq %d for MC Err\n", - pdata->irq); - } - - /* get this far and it's successful */ - edac_dbg(3, "success\n"); - - return 0; - -err2: - edac_mc_del_mc(&pdev->dev); -err: - devres_release_group(&pdev->dev, mv64x60_mc_err_probe); - edac_mc_free(mci); - return res; -} - -static int mv64x60_mc_err_remove(struct platform_device *pdev) -{ - struct mem_ctl_info *mci = platform_get_drvdata(pdev); - - edac_dbg(0, "\n"); - - edac_mc_del_mc(&pdev->dev); - edac_mc_free(mci); - return 0; -} - -static struct platform_driver mv64x60_mc_err_driver = { - .probe = mv64x60_mc_err_probe, - .remove = mv64x60_mc_err_remove, - .driver = { - .name = "mv64x60_mc_err", - } -}; - -static struct platform_driver * const drivers[] = { - &mv64x60_mc_err_driver, - &mv64x60_cpu_err_driver, - &mv64x60_sram_err_driver, -#ifdef CONFIG_PCI - &mv64x60_pci_err_driver, -#endif -}; - -static int __init mv64x60_edac_init(void) -{ - - printk(KERN_INFO "Marvell MV64x60 EDAC driver " MV64x60_REVISION "\n"); - printk(KERN_INFO "\t(C) 2006-2007 MontaVista Software\n"); - - /* make sure error reporting method is sane */ - switch (edac_op_state) { - case EDAC_OPSTATE_POLL: - case EDAC_OPSTATE_INT: - break; - default: - edac_op_state = EDAC_OPSTATE_INT; - break; - } - - return platform_register_drivers(drivers, ARRAY_SIZE(drivers)); -} -module_init(mv64x60_edac_init); - -static void __exit mv64x60_edac_exit(void) -{ - platform_unregister_drivers(drivers, ARRAY_SIZE(drivers)); -} -module_exit(mv64x60_edac_exit); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Montavista Software, Inc."); -module_param(edac_op_state, int, 0444); -MODULE_PARM_DESC(edac_op_state, - "EDAC Error Reporting state: 0=Poll, 2=Interrupt"); diff --git a/drivers/edac/mv64x60_edac.h b/drivers/edac/mv64x60_edac.h deleted file mode 100644 index c7f209c92a1a..000000000000 --- a/drivers/edac/mv64x60_edac.h +++ /dev/null @@ -1,114 +0,0 @@ -/* - * EDAC defs for Marvell MV64x60 bridge chip - * - * Author: Dave Jiang - * - * 2007 (c) MontaVista Software, Inc. This file is licensed under - * the terms of the GNU General Public License version 2. This program - * is licensed "as is" without any warranty of any kind, whether express - * or implied. - * - */ -#ifndef _MV64X60_EDAC_H_ -#define _MV64X60_EDAC_H_ - -#define MV64x60_REVISION " Ver: 2.0.0" -#define EDAC_MOD_STR "MV64x60_edac" - -#define mv64x60_printk(level, fmt, arg...) \ - edac_printk(level, "MV64x60", fmt, ##arg) - -#define mv64x60_mc_printk(mci, level, fmt, arg...) \ - edac_mc_chipset_printk(mci, level, "MV64x60", fmt, ##arg) - -/* CPU Error Report Registers */ -#define MV64x60_CPU_ERR_ADDR_LO 0x00 /* 0x0070 */ -#define MV64x60_CPU_ERR_ADDR_HI 0x08 /* 0x0078 */ -#define MV64x60_CPU_ERR_DATA_LO 0x00 /* 0x0128 */ -#define MV64x60_CPU_ERR_DATA_HI 0x08 /* 0x0130 */ -#define MV64x60_CPU_ERR_PARITY 0x10 /* 0x0138 */ -#define MV64x60_CPU_ERR_CAUSE 0x18 /* 0x0140 */ -#define MV64x60_CPU_ERR_MASK 0x20 /* 0x0148 */ - -#define MV64x60_CPU_CAUSE_MASK 0x07ffffff - -/* SRAM Error Report Registers */ -#define MV64X60_SRAM_ERR_CAUSE 0x08 /* 0x0388 */ -#define MV64X60_SRAM_ERR_ADDR_LO 0x10 /* 0x0390 */ -#define MV64X60_SRAM_ERR_ADDR_HI 0x78 /* 0x03f8 */ -#define MV64X60_SRAM_ERR_DATA_LO 0x18 /* 0x0398 */ -#define MV64X60_SRAM_ERR_DATA_HI 0x20 /* 0x03a0 */ -#define MV64X60_SRAM_ERR_PARITY 0x28 /* 0x03a8 */ - -/* SDRAM Controller Registers */ -#define MV64X60_SDRAM_CONFIG 0x00 /* 0x1400 */ -#define MV64X60_SDRAM_ERR_DATA_HI 0x40 /* 0x1440 */ -#define MV64X60_SDRAM_ERR_DATA_LO 0x44 /* 0x1444 */ -#define MV64X60_SDRAM_ERR_ECC_RCVD 0x48 /* 0x1448 */ -#define MV64X60_SDRAM_ERR_ECC_CALC 0x4c /* 0x144c */ -#define MV64X60_SDRAM_ERR_ADDR 0x50 /* 0x1450 */ -#define MV64X60_SDRAM_ERR_ECC_CNTL 0x54 /* 0x1454 */ -#define MV64X60_SDRAM_ERR_ECC_ERR_CNT 0x58 /* 0x1458 */ - -#define MV64X60_SDRAM_REGISTERED 0x20000 -#define MV64X60_SDRAM_ECC 0x40000 - -#ifdef CONFIG_PCI -/* - * Bit 0 of MV64x60_PCIx_ERR_MASK does not exist on the 64360 and because of - * errata FEr-#11 and FEr-##16 for the 64460, it should be 0 on that chip as - * well. IOW, don't set bit 0. - */ -#define MV64X60_PCIx_ERR_MASK_VAL 0x00a50c24 - -/* Register offsets from PCIx error address low register */ -#define MV64X60_PCI_ERROR_ADDR_LO 0x00 -#define MV64X60_PCI_ERROR_ADDR_HI 0x04 -#define MV64X60_PCI_ERROR_ATTR 0x08 -#define MV64X60_PCI_ERROR_CMD 0x10 -#define MV64X60_PCI_ERROR_CAUSE 0x18 -#define MV64X60_PCI_ERROR_MASK 0x1c - -#define MV64X60_PCI_ERR_SWrPerr 0x0002 -#define MV64X60_PCI_ERR_SRdPerr 0x0004 -#define MV64X60_PCI_ERR_MWrPerr 0x0020 -#define MV64X60_PCI_ERR_MRdPerr 0x0040 - -#define MV64X60_PCI_PE_MASK (MV64X60_PCI_ERR_SWrPerr | \ - MV64X60_PCI_ERR_SRdPerr | \ - MV64X60_PCI_ERR_MWrPerr | \ - MV64X60_PCI_ERR_MRdPerr) - -struct mv64x60_pci_pdata { - int pci_hose; - void __iomem *pci_vbase; - char *name; - int irq; - int edac_idx; -}; - -#endif /* CONFIG_PCI */ - -struct mv64x60_mc_pdata { - void __iomem *mc_vbase; - int total_mem; - char *name; - int irq; - int edac_idx; -}; - -struct mv64x60_cpu_pdata { - void __iomem *cpu_vbase[2]; - char *name; - int irq; - int edac_idx; -}; - -struct mv64x60_sram_pdata { - void __iomem *sram_vbase; - char *name; - int irq; - int edac_idx; -}; - -#endif diff --git a/drivers/edac/r82600_edac.c b/drivers/edac/r82600_edac.c index 851e53e122aa..d0aef83dca2a 100644 --- a/drivers/edac/r82600_edac.c +++ b/drivers/edac/r82600_edac.c @@ -204,7 +204,6 @@ static void r82600_check(struct mem_ctl_info *mci) { struct r82600_error_info info; - edac_dbg(1, "MC%d\n", mci->mc_idx); r82600_get_error_info(mci, &info); r82600_process_error_info(mci, &info, 1); } diff --git a/drivers/edac/skx_base.c b/drivers/edac/skx_base.c index 2c7db95df326..6a4f0b27c654 100644 --- a/drivers/edac/skx_base.c +++ b/drivers/edac/skx_base.c @@ -174,7 +174,7 @@ static bool skx_check_ecc(u32 mcmtr) return !!GET_BITFIELD(mcmtr, 2, 2); } -static int skx_get_dimm_config(struct mem_ctl_info *mci) +static int skx_get_dimm_config(struct mem_ctl_info *mci, struct res_config *cfg) { struct skx_pvt *pvt = mci->pvt_info; u32 mtr, mcmtr, amap, mcddrtcfg; @@ -195,7 +195,7 @@ static int skx_get_dimm_config(struct mem_ctl_info *mci) pci_read_config_dword(imc->chan[i].cdev, 0x80 + 4 * j, &mtr); if (IS_DIMM_PRESENT(mtr)) { - ndimms += skx_get_dimm_info(mtr, mcmtr, amap, dimm, imc, i, j); + ndimms += skx_get_dimm_info(mtr, mcmtr, amap, dimm, imc, i, j, cfg); } else if (IS_NVDIMM_PRESENT(mcddrtcfg, j)) { ndimms += skx_get_nvdimm_info(dimm, imc, i, j, EDAC_MOD_STR); @@ -702,7 +702,7 @@ static int __init skx_init(void) d->imc[i].node_id = node_id; rc = skx_register_mci(&d->imc[i], d->imc[i].chan[0].cdev, "Skylake Socket", EDAC_MOD_STR, - skx_get_dimm_config); + skx_get_dimm_config, cfg); if (rc < 0) goto fail; } diff --git a/drivers/edac/skx_common.c b/drivers/edac/skx_common.c index 2b4ce8e5ac2f..81c3e2ec6f56 100644 --- a/drivers/edac/skx_common.c +++ b/drivers/edac/skx_common.c @@ -304,15 +304,25 @@ static int skx_get_dimm_attr(u32 reg, int lobit, int hibit, int add, #define numcol(reg) skx_get_dimm_attr(reg, 0, 1, 10, 0, 2, "cols") int skx_get_dimm_info(u32 mtr, u32 mcmtr, u32 amap, struct dimm_info *dimm, - struct skx_imc *imc, int chan, int dimmno) + struct skx_imc *imc, int chan, int dimmno, + struct res_config *cfg) { - int banks = 16, ranks, rows, cols, npages; + int banks, ranks, rows, cols, npages; + enum mem_type mtype; u64 size; ranks = numrank(mtr); rows = numrow(mtr); cols = numcol(mtr); + if (cfg->support_ddr5 && (amap & 0x8)) { + banks = 32; + mtype = MEM_DDR5; + } else { + banks = 16; + mtype = MEM_DDR4; + } + /* * Compute size in 8-byte (2^3) words, then shift to MiB (2^20) */ @@ -332,7 +342,7 @@ int skx_get_dimm_info(u32 mtr, u32 mcmtr, u32 amap, struct dimm_info *dimm, dimm->nr_pages = npages; dimm->grain = 32; dimm->dtype = get_width(mtr); - dimm->mtype = MEM_DDR4; + dimm->mtype = mtype; dimm->edac_mode = EDAC_SECDED; /* likely better than this */ snprintf(dimm->label, sizeof(dimm->label), "CPU_SrcID#%u_MC#%u_Chan#%u_DIMM#%u", imc->src_id, imc->lmc, chan, dimmno); @@ -390,7 +400,8 @@ unknown_size: int skx_register_mci(struct skx_imc *imc, struct pci_dev *pdev, const char *ctl_name, const char *mod_str, - get_dimm_config_f get_dimm_config) + get_dimm_config_f get_dimm_config, + struct res_config *cfg) { struct mem_ctl_info *mci; struct edac_mc_layer layers[2]; @@ -425,13 +436,15 @@ int skx_register_mci(struct skx_imc *imc, struct pci_dev *pdev, } mci->mtype_cap = MEM_FLAG_DDR4 | MEM_FLAG_NVDIMM; + if (cfg->support_ddr5) + mci->mtype_cap |= MEM_FLAG_DDR5; mci->edac_ctl_cap = EDAC_FLAG_NONE; mci->edac_cap = EDAC_FLAG_NONE; mci->mod_name = mod_str; mci->dev_name = pci_name(pdev); mci->ctl_page_to_phys = NULL; - rc = get_dimm_config(mci); + rc = get_dimm_config(mci, cfg); if (rc < 0) goto fail; diff --git a/drivers/edac/skx_common.h b/drivers/edac/skx_common.h index 78f8c1de0b71..bf56bebff138 100644 --- a/drivers/edac/skx_common.h +++ b/drivers/edac/skx_common.h @@ -59,6 +59,7 @@ struct skx_dev { struct mem_ctl_info *mci; struct pci_dev *mdev; /* for i10nm CPU */ void __iomem *mbase; /* for i10nm CPU */ + int chan_mmio_sz; /* for i10nm CPU */ u8 mc; /* system wide mc# */ u8 lmc; /* socket relative mc# */ u8 src_id, node_id; @@ -82,7 +83,8 @@ struct skx_pvt { enum type { SKX, - I10NM + I10NM, + SPR }; enum { @@ -118,9 +120,13 @@ struct res_config { unsigned int decs_did; /* Default bus number configuration register offset */ int busno_cfg_offset; + /* Per DDR channel memory-mapped I/O size */ + int ddr_chan_mmio_sz; + bool support_ddr5; }; -typedef int (*get_dimm_config_f)(struct mem_ctl_info *mci); +typedef int (*get_dimm_config_f)(struct mem_ctl_info *mci, + struct res_config *cfg); typedef bool (*skx_decode_f)(struct decoded_addr *res); typedef void (*skx_show_retry_log_f)(struct decoded_addr *res, char *msg, int len); @@ -136,14 +142,16 @@ int skx_get_all_bus_mappings(struct res_config *cfg, struct list_head **list); int skx_get_hi_lo(unsigned int did, int off[], u64 *tolm, u64 *tohm); int skx_get_dimm_info(u32 mtr, u32 mcmtr, u32 amap, struct dimm_info *dimm, - struct skx_imc *imc, int chan, int dimmno); + struct skx_imc *imc, int chan, int dimmno, + struct res_config *cfg); int skx_get_nvdimm_info(struct dimm_info *dimm, struct skx_imc *imc, int chan, int dimmno, const char *mod_str); int skx_register_mci(struct skx_imc *imc, struct pci_dev *pdev, const char *ctl_name, const char *mod_str, - get_dimm_config_f get_dimm_config); + get_dimm_config_f get_dimm_config, + struct res_config *cfg); int skx_mce_check_error(struct notifier_block *nb, unsigned long val, void *data); diff --git a/drivers/edac/synopsys_edac.c b/drivers/edac/synopsys_edac.c index 12211dc040e8..7e7146b22c16 100644 --- a/drivers/edac/synopsys_edac.c +++ b/drivers/edac/synopsys_edac.c @@ -1344,7 +1344,8 @@ static int mc_probe(struct platform_device *pdev) #ifdef CONFIG_EDAC_DEBUG if (priv->p_data->quirks & DDR_ECC_DATA_POISON_SUPPORT) { - if (edac_create_sysfs_attributes(mci)) { + rc = edac_create_sysfs_attributes(mci); + if (rc) { edac_printk(KERN_ERR, EDAC_MC, "Failed to create sysfs entries\n"); goto free_edac_mc; diff --git a/drivers/edac/x38_edac.c b/drivers/edac/x38_edac.c index a65e2f78a402..49ab5721aab2 100644 --- a/drivers/edac/x38_edac.c +++ b/drivers/edac/x38_edac.c @@ -238,7 +238,6 @@ static void x38_check(struct mem_ctl_info *mci) { struct x38_error_info info; - edac_dbg(1, "MC%d\n", mci->mc_idx); x38_get_and_clear_error_info(mci, &info); x38_process_error_info(mci, &info); } diff --git a/drivers/firmware/efi/cper-x86.c b/drivers/firmware/efi/cper-x86.c index 2531de49f56c..438ed9eff6d0 100644 --- a/drivers/firmware/efi/cper-x86.c +++ b/drivers/firmware/efi/cper-x86.c @@ -2,6 +2,7 @@ // Copyright (C) 2018, Advanced Micro Devices, Inc. #include +#include /* * We don't need a "CPER_IA" prefix since these are all locally defined. @@ -347,9 +348,13 @@ void cper_print_proc_ia(const char *pfx, const struct cper_sec_proc_ia *proc) ctx_info->mm_reg_addr); } - printk("%sRegister Array:\n", newpfx); - print_hex_dump(newpfx, "", DUMP_PREFIX_OFFSET, 16, groupsize, - (ctx_info + 1), ctx_info->reg_arr_size, 0); + if (ctx_info->reg_ctx_type != CTX_TYPE_MSR || + arch_apei_report_x86_error(ctx_info, proc->lapic_id)) { + printk("%sRegister Array:\n", newpfx); + print_hex_dump(newpfx, "", DUMP_PREFIX_OFFSET, 16, + groupsize, (ctx_info + 1), + ctx_info->reg_arr_size, 0); + } ctx_info = (struct cper_ia_proc_ctx *)((long)ctx_info + size); } diff --git a/drivers/firmware/efi/embedded-firmware.c b/drivers/firmware/efi/embedded-firmware.c index 21ae0c48232a..f5be8e22305b 100644 --- a/drivers/firmware/efi/embedded-firmware.c +++ b/drivers/firmware/efi/embedded-firmware.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include /* Exported for use by lib/test_firmware.c only */ LIST_HEAD(efi_embedded_fw_list); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index b848f9e97613..4d8f19ab1014 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -551,25 +551,12 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict, struct ttm_resource *old_mem = &bo->mem; int r; - if ((old_mem->mem_type == TTM_PL_SYSTEM && - new_mem->mem_type == TTM_PL_VRAM) || - (old_mem->mem_type == TTM_PL_VRAM && - new_mem->mem_type == TTM_PL_SYSTEM)) { - hop->fpfn = 0; - hop->lpfn = 0; - hop->mem_type = TTM_PL_TT; - hop->flags = 0; - return -EMULTIHOP; - } - if (new_mem->mem_type == TTM_PL_TT) { r = amdgpu_ttm_backend_bind(bo->bdev, bo->ttm, new_mem); if (r) return r; } - amdgpu_bo_move_notify(bo, evict, new_mem); - /* Can't move a pinned BO */ abo = ttm_to_amdgpu_bo(bo); if (WARN_ON_ONCE(abo->tbo.pin_count > 0)) @@ -579,24 +566,23 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict, if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { ttm_bo_move_null(bo, new_mem); - return 0; + goto out; } if (old_mem->mem_type == TTM_PL_SYSTEM && new_mem->mem_type == TTM_PL_TT) { ttm_bo_move_null(bo, new_mem); - return 0; + goto out; } - if (old_mem->mem_type == TTM_PL_TT && new_mem->mem_type == TTM_PL_SYSTEM) { r = ttm_bo_wait_ctx(bo, ctx); if (r) - goto fail; + return r; amdgpu_ttm_backend_unbind(bo->bdev, bo->ttm); ttm_resource_free(bo, &bo->mem); ttm_bo_assign_mem(bo, new_mem); - return 0; + goto out; } if (old_mem->mem_type == AMDGPU_PL_GDS || @@ -607,27 +593,37 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict, new_mem->mem_type == AMDGPU_PL_OA) { /* Nothing to save here */ ttm_bo_move_null(bo, new_mem); - return 0; + goto out; } - if (!adev->mman.buffer_funcs_enabled) { + if (adev->mman.buffer_funcs_enabled) { + if (((old_mem->mem_type == TTM_PL_SYSTEM && + new_mem->mem_type == TTM_PL_VRAM) || + (old_mem->mem_type == TTM_PL_VRAM && + new_mem->mem_type == TTM_PL_SYSTEM))) { + hop->fpfn = 0; + hop->lpfn = 0; + hop->mem_type = TTM_PL_TT; + hop->flags = 0; + return -EMULTIHOP; + } + + r = amdgpu_move_blit(bo, evict, new_mem, old_mem); + } else { r = -ENODEV; - goto memcpy; } - r = amdgpu_move_blit(bo, evict, new_mem, old_mem); if (r) { -memcpy: /* Check that all memory is CPU accessible */ if (!amdgpu_mem_visible(adev, old_mem) || !amdgpu_mem_visible(adev, new_mem)) { pr_err("Move buffer fallback to memcpy unavailable\n"); - goto fail; + return r; } r = ttm_bo_move_memcpy(bo, ctx, new_mem); if (r) - goto fail; + return r; } if (bo->type == ttm_bo_type_device && @@ -639,14 +635,11 @@ memcpy: abo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; } +out: /* update statistics */ atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &adev->num_bytes_moved); + amdgpu_bo_move_notify(bo, evict, new_mem); return 0; -fail: - swap(*new_mem, bo->mem); - amdgpu_bo_move_notify(bo, false, new_mem); - swap(*new_mem, bo->mem); - return r; } /* diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c index 28b300ed200e..d59ef6e92a40 100644 --- a/drivers/gpu/drm/radeon/radeon_ttm.c +++ b/drivers/gpu/drm/radeon/radeon_ttm.c @@ -217,27 +217,15 @@ static int radeon_bo_move(struct ttm_buffer_object *bo, bool evict, struct ttm_resource *old_mem = &bo->mem; int r; - if ((old_mem->mem_type == TTM_PL_SYSTEM && - new_mem->mem_type == TTM_PL_VRAM) || - (old_mem->mem_type == TTM_PL_VRAM && - new_mem->mem_type == TTM_PL_SYSTEM)) { - hop->fpfn = 0; - hop->lpfn = 0; - hop->mem_type = TTM_PL_TT; - hop->flags = 0; - return -EMULTIHOP; - } - if (new_mem->mem_type == TTM_PL_TT) { r = radeon_ttm_tt_bind(bo->bdev, bo->ttm, new_mem); if (r) return r; } - radeon_bo_move_notify(bo, evict, new_mem); r = ttm_bo_wait_ctx(bo, ctx); if (r) - goto fail; + return r; /* Can't move a pinned BO */ rbo = container_of(bo, struct radeon_bo, tbo); @@ -247,12 +235,12 @@ static int radeon_bo_move(struct ttm_buffer_object *bo, bool evict, rdev = radeon_get_rdev(bo->bdev); if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { ttm_bo_move_null(bo, new_mem); - return 0; + goto out; } if (old_mem->mem_type == TTM_PL_SYSTEM && new_mem->mem_type == TTM_PL_TT) { ttm_bo_move_null(bo, new_mem); - return 0; + goto out; } if (old_mem->mem_type == TTM_PL_TT && @@ -260,31 +248,37 @@ static int radeon_bo_move(struct ttm_buffer_object *bo, bool evict, radeon_ttm_tt_unbind(bo->bdev, bo->ttm); ttm_resource_free(bo, &bo->mem); ttm_bo_assign_mem(bo, new_mem); - return 0; + goto out; } - if (!rdev->ring[radeon_copy_ring_index(rdev)].ready || - rdev->asic->copy.copy == NULL) { - /* use memcpy */ - goto memcpy; - } - - r = radeon_move_blit(bo, evict, new_mem, old_mem); - if (r) { -memcpy: - r = ttm_bo_move_memcpy(bo, ctx, new_mem); - if (r) { - goto fail; + if (rdev->ring[radeon_copy_ring_index(rdev)].ready && + rdev->asic->copy.copy != NULL) { + if ((old_mem->mem_type == TTM_PL_SYSTEM && + new_mem->mem_type == TTM_PL_VRAM) || + (old_mem->mem_type == TTM_PL_VRAM && + new_mem->mem_type == TTM_PL_SYSTEM)) { + hop->fpfn = 0; + hop->lpfn = 0; + hop->mem_type = TTM_PL_TT; + hop->flags = 0; + return -EMULTIHOP; } + + r = radeon_move_blit(bo, evict, new_mem, old_mem); + } else { + r = -ENODEV; } + if (r) { + r = ttm_bo_move_memcpy(bo, ctx, new_mem); + if (r) + return r; + } + +out: /* update statistics */ atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &rdev->num_bytes_moved); + radeon_bo_move_notify(bo, evict, new_mem); return 0; -fail: - swap(*new_mem, bo->mem); - radeon_bo_move_notify(bo, false, new_mem); - swap(*new_mem, bo->mem); - return r; } static int radeon_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_resource *mem) diff --git a/drivers/macintosh/adb-iop.c b/drivers/macintosh/adb-iop.c index f3d1a460fbce..0ee327249150 100644 --- a/drivers/macintosh/adb-iop.c +++ b/drivers/macintosh/adb-iop.c @@ -25,6 +25,7 @@ static struct adb_request *current_req; static struct adb_request *last_req; static unsigned int autopoll_devs; +static u8 autopoll_addr; static enum adb_iop_state { idle, @@ -41,6 +42,11 @@ static int adb_iop_autopoll(int); static void adb_iop_poll(void); static int adb_iop_reset_bus(void); +/* ADB command byte structure */ +#define ADDR_MASK 0xF0 +#define OP_MASK 0x0C +#define TALK 0x0C + struct adb_driver adb_iop_driver = { .name = "ISM IOP", .probe = adb_iop_probe, @@ -78,10 +84,7 @@ static void adb_iop_complete(struct iop_msg *msg) local_irq_save(flags); - if (current_req->reply_expected) - adb_iop_state = awaiting_reply; - else - adb_iop_done(); + adb_iop_state = awaiting_reply; local_irq_restore(flags); } @@ -89,38 +92,52 @@ static void adb_iop_complete(struct iop_msg *msg) /* * Listen for ADB messages from the IOP. * - * This will be called when unsolicited messages (usually replies to TALK - * commands or autopoll packets) are received. + * This will be called when unsolicited IOP messages are received. + * These IOP messages can carry ADB autopoll responses and also occur + * after explicit ADB commands. */ static void adb_iop_listen(struct iop_msg *msg) { struct adb_iopmsg *amsg = (struct adb_iopmsg *)msg->message; + u8 addr = (amsg->cmd & ADDR_MASK) >> 4; + u8 op = amsg->cmd & OP_MASK; unsigned long flags; bool req_done = false; local_irq_save(flags); - /* Handle a timeout. Timeout packets seem to occur even after - * we've gotten a valid reply to a TALK, presumably because of - * autopolling. + /* Responses to Talk commands may be unsolicited as they are + * produced when the IOP polls devices. They are mostly timeouts. */ + if (op == TALK && ((1 << addr) & autopoll_devs)) + autopoll_addr = addr; - if (amsg->flags & ADB_IOP_EXPLICIT) { + switch (amsg->flags & (ADB_IOP_EXPLICIT | + ADB_IOP_AUTOPOLL | + ADB_IOP_TIMEOUT)) { + case ADB_IOP_EXPLICIT: + case ADB_IOP_EXPLICIT | ADB_IOP_TIMEOUT: if (adb_iop_state == awaiting_reply) { struct adb_request *req = current_req; - req->reply_len = amsg->count + 1; - memcpy(req->reply, &amsg->cmd, req->reply_len); + if (req->reply_expected) { + req->reply_len = amsg->count + 1; + memcpy(req->reply, &amsg->cmd, req->reply_len); + } req_done = true; } - } else if (!(amsg->flags & ADB_IOP_TIMEOUT)) { - adb_input(&amsg->cmd, amsg->count + 1, - amsg->flags & ADB_IOP_AUTOPOLL); + break; + case ADB_IOP_AUTOPOLL: + if (((1 << addr) & autopoll_devs) && + amsg->cmd == ADB_READREG(addr, 0)) + adb_input(&amsg->cmd, amsg->count + 1, 1); + break; } - - msg->reply[0] = autopoll_devs ? ADB_IOP_AUTOPOLL : 0; + msg->reply[0] = autopoll_addr ? ADB_IOP_AUTOPOLL : 0; + msg->reply[1] = 0; + msg->reply[2] = autopoll_addr ? ADB_READREG(autopoll_addr, 0) : 0; iop_complete_message(msg); if (req_done) @@ -233,6 +250,9 @@ static void adb_iop_set_ap_complete(struct iop_msg *msg) struct adb_iopmsg *amsg = (struct adb_iopmsg *)msg->message; autopoll_devs = (amsg->data[1] << 8) | amsg->data[0]; + if (autopoll_devs & (1 << autopoll_addr)) + return; + autopoll_addr = autopoll_devs ? (ffs(autopoll_devs) - 1) : 0; } static int adb_iop_autopoll(int devs) diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c b/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c index 072299b14b8d..47d9268a7e3c 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c +++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c @@ -51,7 +51,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h index 2d3dfdd2a716..65617752c630 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h +++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h @@ -9,7 +9,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/nfc/s3fwrn5/firmware.c b/drivers/nfc/s3fwrn5/firmware.c index ec930ee2c847..5d5ad8307211 100644 --- a/drivers/nfc/s3fwrn5/firmware.c +++ b/drivers/nfc/s3fwrn5/firmware.c @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include "s3fwrn5.h" #include "firmware.h" diff --git a/drivers/platform/x86/Kconfig b/drivers/platform/x86/Kconfig index 0d91d136bc3b..ba34153571b8 100644 --- a/drivers/platform/x86/Kconfig +++ b/drivers/platform/x86/Kconfig @@ -78,6 +78,17 @@ config HUAWEI_WMI To compile this driver as a module, choose M here: the module will be called huawei-wmi. +config UV_SYSFS + tristate "Sysfs structure for UV systems" + depends on X86_UV + depends on SYSFS + help + This driver supports a sysfs tree describing information about + UV systems at /sys/firmware/sgi_uv/. + + To compile this driver as a module, choose M here: the module will + be called uv_sysfs. + config INTEL_WMI_SBL_FW_UPDATE tristate "Intel WMI Slim Bootloader firmware update signaling driver" depends on ACPI_WMI diff --git a/drivers/platform/x86/Makefile b/drivers/platform/x86/Makefile index 5f823f7eff45..a34875d833dd 100644 --- a/drivers/platform/x86/Makefile +++ b/drivers/platform/x86/Makefile @@ -62,6 +62,9 @@ obj-$(CONFIG_HP_WIRELESS) += hp-wireless.o obj-$(CONFIG_HP_WMI) += hp-wmi.o obj-$(CONFIG_TC1100_WMI) += tc1100-wmi.o +# Hewlett Packard Enterprise +obj-$(CONFIG_UV_SYSFS) += uv_sysfs.o + # IBM Thinkpad and Lenovo obj-$(CONFIG_IBM_RTL) += ibm_rtl.o obj-$(CONFIG_IDEAPAD_LAPTOP) += ideapad-laptop.o diff --git a/drivers/platform/x86/uv_sysfs.c b/drivers/platform/x86/uv_sysfs.c new file mode 100644 index 000000000000..7badcfa3f384 --- /dev/null +++ b/drivers/platform/x86/uv_sysfs.c @@ -0,0 +1,929 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * This file supports the /sys/firmware/sgi_uv topology tree on HPE UV. + * + * Copyright (c) 2020 Hewlett Packard Enterprise. All Rights Reserved. + * Copyright (c) Justin Ernst + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define INVALID_CNODE -1 + +struct kobject *sgi_uv_kobj; +static struct kset *uv_pcibus_kset; +static struct kset *uv_hubs_kset; +static struct uv_bios_hub_info *hub_buf; +static struct uv_bios_port_info **port_buf; +static struct uv_hub **uv_hubs; +static struct uv_pci_top_obj **uv_pci_objs; +static int num_pci_lines; +static int num_cnodes; +static int *prev_obj_to_cnode; +static int uv_bios_obj_cnt; +static signed short uv_master_nasid = -1; +static void *uv_biosheap; + +static const char *uv_type_string(void) +{ + if (is_uv5_hub()) + return "9.0"; + else if (is_uv4a_hub()) + return "7.1"; + else if (is_uv4_hub()) + return "7.0"; + else if (is_uv3_hub()) + return "5.0"; + else if (is_uv2_hub()) + return "3.0"; + else if (uv_get_hubless_system()) + return "0.1"; + else + return "unknown"; +} + +static int ordinal_to_nasid(int ordinal) +{ + if (ordinal < num_cnodes && ordinal >= 0) + return UV_PNODE_TO_NASID(uv_blade_to_pnode(ordinal)); + else + return -1; +} + +static union geoid_u cnode_to_geoid(int cnode) +{ + union geoid_u geoid; + + uv_bios_get_geoinfo(ordinal_to_nasid(cnode), (u64)sizeof(union geoid_u), (u64 *)&geoid); + return geoid; +} + +static int location_to_bpos(char *location, int *rack, int *slot, int *blade) +{ + char type, r, b, h; + int idb, idh; + + if (sscanf(location, "%c%03d%c%02d%c%2d%c%d", + &r, rack, &type, slot, &b, &idb, &h, &idh) != 8) + return -1; + *blade = idb * 2 + idh; + + return 0; +} + +static int cache_obj_to_cnode(struct uv_bios_hub_info *obj) +{ + int cnode; + union geoid_u geoid; + int obj_rack, obj_slot, obj_blade; + int rack, slot, blade; + + if (!obj->f.fields.this_part && !obj->f.fields.is_shared) + return 0; + + if (location_to_bpos(obj->location, &obj_rack, &obj_slot, &obj_blade)) + return -1; + + for (cnode = 0; cnode < num_cnodes; cnode++) { + geoid = cnode_to_geoid(cnode); + rack = geo_rack(geoid); + slot = geo_slot(geoid); + blade = geo_blade(geoid); + if (obj_rack == rack && obj_slot == slot && obj_blade == blade) + prev_obj_to_cnode[obj->id] = cnode; + } + + return 0; +} + +static int get_obj_to_cnode(int obj_id) +{ + return prev_obj_to_cnode[obj_id]; +} + +struct uv_hub { + struct kobject kobj; + struct uv_bios_hub_info *hub_info; + struct uv_port **ports; +}; + +#define to_uv_hub(kobj_ptr) container_of(kobj_ptr, struct uv_hub, kobj) + +static ssize_t hub_name_show(struct uv_bios_hub_info *hub_info, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%s\n", hub_info->name); +} + +static ssize_t hub_location_show(struct uv_bios_hub_info *hub_info, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%s\n", hub_info->location); +} + +static ssize_t hub_partition_show(struct uv_bios_hub_info *hub_info, char *buf) +{ + return sprintf(buf, "%d\n", hub_info->f.fields.this_part); +} + +static ssize_t hub_shared_show(struct uv_bios_hub_info *hub_info, char *buf) +{ + return sprintf(buf, "%d\n", hub_info->f.fields.is_shared); +} +static ssize_t hub_nasid_show(struct uv_bios_hub_info *hub_info, char *buf) +{ + int cnode = get_obj_to_cnode(hub_info->id); + + return sprintf(buf, "%d\n", ordinal_to_nasid(cnode)); +} +static ssize_t hub_cnode_show(struct uv_bios_hub_info *hub_info, char *buf) +{ + return sprintf(buf, "%d\n", get_obj_to_cnode(hub_info->id)); +} + +struct hub_sysfs_entry { + struct attribute attr; + ssize_t (*show)(struct uv_bios_hub_info *hub_info, char *buf); + ssize_t (*store)(struct uv_bios_hub_info *hub_info, const char *buf, size_t sz); +}; + +static struct hub_sysfs_entry name_attribute = + __ATTR(name, 0444, hub_name_show, NULL); +static struct hub_sysfs_entry location_attribute = + __ATTR(location, 0444, hub_location_show, NULL); +static struct hub_sysfs_entry partition_attribute = + __ATTR(this_partition, 0444, hub_partition_show, NULL); +static struct hub_sysfs_entry shared_attribute = + __ATTR(shared, 0444, hub_shared_show, NULL); +static struct hub_sysfs_entry nasid_attribute = + __ATTR(nasid, 0444, hub_nasid_show, NULL); +static struct hub_sysfs_entry cnode_attribute = + __ATTR(cnode, 0444, hub_cnode_show, NULL); + +static struct attribute *uv_hub_attrs[] = { + &name_attribute.attr, + &location_attribute.attr, + &partition_attribute.attr, + &shared_attribute.attr, + &nasid_attribute.attr, + &cnode_attribute.attr, + NULL, +}; + +static void hub_release(struct kobject *kobj) +{ + struct uv_hub *hub = to_uv_hub(kobj); + + kfree(hub); +} + +static ssize_t hub_type_show(struct kobject *kobj, struct attribute *attr, + char *buf) +{ + struct uv_hub *hub = to_uv_hub(kobj); + struct uv_bios_hub_info *bios_hub_info = hub->hub_info; + struct hub_sysfs_entry *entry; + + entry = container_of(attr, struct hub_sysfs_entry, attr); + + if (!entry->show) + return -EIO; + + return entry->show(bios_hub_info, buf); +} + +static const struct sysfs_ops hub_sysfs_ops = { + .show = hub_type_show, +}; + +static struct kobj_type hub_attr_type = { + .release = hub_release, + .sysfs_ops = &hub_sysfs_ops, + .default_attrs = uv_hub_attrs, +}; + +static int uv_hubs_init(void) +{ + s64 biosr; + u64 sz; + int i, ret; + + prev_obj_to_cnode = kmalloc_array(uv_bios_obj_cnt, sizeof(*prev_obj_to_cnode), + GFP_KERNEL); + if (!prev_obj_to_cnode) + return -ENOMEM; + + for (i = 0; i < uv_bios_obj_cnt; i++) + prev_obj_to_cnode[i] = INVALID_CNODE; + + uv_hubs_kset = kset_create_and_add("hubs", NULL, sgi_uv_kobj); + if (!uv_hubs_kset) { + ret = -ENOMEM; + goto err_hubs_kset; + } + sz = uv_bios_obj_cnt * sizeof(*hub_buf); + hub_buf = kzalloc(sz, GFP_KERNEL); + if (!hub_buf) { + ret = -ENOMEM; + goto err_hub_buf; + } + + biosr = uv_bios_enum_objs((u64)uv_master_nasid, sz, (u64 *)hub_buf); + if (biosr) { + ret = -EINVAL; + goto err_enum_objs; + } + + uv_hubs = kcalloc(uv_bios_obj_cnt, sizeof(*uv_hubs), GFP_KERNEL); + if (!uv_hubs) { + ret = -ENOMEM; + goto err_enum_objs; + } + + for (i = 0; i < uv_bios_obj_cnt; i++) { + uv_hubs[i] = kzalloc(sizeof(*uv_hubs[i]), GFP_KERNEL); + if (!uv_hubs[i]) { + i--; + ret = -ENOMEM; + goto err_hubs; + } + + uv_hubs[i]->hub_info = &hub_buf[i]; + cache_obj_to_cnode(uv_hubs[i]->hub_info); + + uv_hubs[i]->kobj.kset = uv_hubs_kset; + + ret = kobject_init_and_add(&uv_hubs[i]->kobj, &hub_attr_type, + NULL, "hub_%u", hub_buf[i].id); + if (ret) + goto err_hubs; + kobject_uevent(&uv_hubs[i]->kobj, KOBJ_ADD); + } + return 0; + +err_hubs: + for (; i >= 0; i--) + kobject_put(&uv_hubs[i]->kobj); + kfree(uv_hubs); +err_enum_objs: + kfree(hub_buf); +err_hub_buf: + kset_unregister(uv_hubs_kset); +err_hubs_kset: + kfree(prev_obj_to_cnode); + return ret; + +} + +static void uv_hubs_exit(void) +{ + int i; + + for (i = 0; i < uv_bios_obj_cnt; i++) + kobject_put(&uv_hubs[i]->kobj); + + kfree(uv_hubs); + kfree(hub_buf); + kset_unregister(uv_hubs_kset); + kfree(prev_obj_to_cnode); +} + +struct uv_port { + struct kobject kobj; + struct uv_bios_port_info *port_info; +}; + +#define to_uv_port(kobj_ptr) container_of(kobj_ptr, struct uv_port, kobj) + +static ssize_t uv_port_conn_hub_show(struct uv_bios_port_info *port, char *buf) +{ + return sprintf(buf, "%d\n", port->conn_id); +} + +static ssize_t uv_port_conn_port_show(struct uv_bios_port_info *port, char *buf) +{ + return sprintf(buf, "%d\n", port->conn_port); +} + +struct uv_port_sysfs_entry { + struct attribute attr; + ssize_t (*show)(struct uv_bios_port_info *port_info, char *buf); + ssize_t (*store)(struct uv_bios_port_info *port_info, const char *buf, size_t size); +}; + +static struct uv_port_sysfs_entry uv_port_conn_hub_attribute = + __ATTR(conn_hub, 0444, uv_port_conn_hub_show, NULL); +static struct uv_port_sysfs_entry uv_port_conn_port_attribute = + __ATTR(conn_port, 0444, uv_port_conn_port_show, NULL); + +static struct attribute *uv_port_attrs[] = { + &uv_port_conn_hub_attribute.attr, + &uv_port_conn_port_attribute.attr, + NULL, +}; + +static void uv_port_release(struct kobject *kobj) +{ + struct uv_port *port = to_uv_port(kobj); + + kfree(port); +} + +static ssize_t uv_port_type_show(struct kobject *kobj, struct attribute *attr, + char *buf) +{ + struct uv_port *port = to_uv_port(kobj); + struct uv_bios_port_info *port_info = port->port_info; + struct uv_port_sysfs_entry *entry; + + entry = container_of(attr, struct uv_port_sysfs_entry, attr); + + if (!entry->show) + return -EIO; + + return entry->show(port_info, buf); +} + +static const struct sysfs_ops uv_port_sysfs_ops = { + .show = uv_port_type_show, +}; + +static struct kobj_type uv_port_attr_type = { + .release = uv_port_release, + .sysfs_ops = &uv_port_sysfs_ops, + .default_attrs = uv_port_attrs, +}; + +static int uv_ports_init(void) +{ + s64 biosr; + int j = 0, k = 0, ret, sz; + + port_buf = kcalloc(uv_bios_obj_cnt, sizeof(*port_buf), GFP_KERNEL); + if (!port_buf) + return -ENOMEM; + + for (j = 0; j < uv_bios_obj_cnt; j++) { + sz = hub_buf[j].ports * sizeof(*port_buf[j]); + port_buf[j] = kzalloc(sz, GFP_KERNEL); + if (!port_buf[j]) { + ret = -ENOMEM; + j--; + goto err_port_info; + } + biosr = uv_bios_enum_ports((u64)uv_master_nasid, (u64)hub_buf[j].id, sz, + (u64 *)port_buf[j]); + if (biosr) { + ret = -EINVAL; + goto err_port_info; + } + } + for (j = 0; j < uv_bios_obj_cnt; j++) { + uv_hubs[j]->ports = kcalloc(hub_buf[j].ports, + sizeof(*uv_hubs[j]->ports), GFP_KERNEL); + if (!uv_hubs[j]->ports) { + ret = -ENOMEM; + j--; + goto err_ports; + } + } + for (j = 0; j < uv_bios_obj_cnt; j++) { + for (k = 0; k < hub_buf[j].ports; k++) { + uv_hubs[j]->ports[k] = kzalloc(sizeof(*uv_hubs[j]->ports[k]), GFP_KERNEL); + if (!uv_hubs[j]->ports[k]) { + ret = -ENOMEM; + k--; + goto err_kobj_ports; + } + uv_hubs[j]->ports[k]->port_info = &port_buf[j][k]; + ret = kobject_init_and_add(&uv_hubs[j]->ports[k]->kobj, &uv_port_attr_type, + &uv_hubs[j]->kobj, "port_%d", port_buf[j][k].port); + if (ret) + goto err_kobj_ports; + kobject_uevent(&uv_hubs[j]->ports[k]->kobj, KOBJ_ADD); + } + } + return 0; + +err_kobj_ports: + for (; j >= 0; j--) { + for (; k >= 0; k--) + kobject_put(&uv_hubs[j]->ports[k]->kobj); + if (j > 0) + k = hub_buf[j-1].ports - 1; + } + j = uv_bios_obj_cnt - 1; +err_ports: + for (; j >= 0; j--) + kfree(uv_hubs[j]->ports); + j = uv_bios_obj_cnt - 1; +err_port_info: + for (; j >= 0; j--) + kfree(port_buf[j]); + kfree(port_buf); + return ret; +} + +static void uv_ports_exit(void) +{ + int j, k; + + for (j = 0; j < uv_bios_obj_cnt; j++) { + for (k = hub_buf[j].ports - 1; k >= 0; k--) + kobject_put(&uv_hubs[j]->ports[k]->kobj); + } + for (j = 0; j < uv_bios_obj_cnt; j++) { + kfree(uv_hubs[j]->ports); + kfree(port_buf[j]); + } + kfree(port_buf); +} + +struct uv_pci_top_obj { + struct kobject kobj; + char *type; + char *location; + int iio_stack; + char *ppb_addr; + int slot; +}; + +#define to_uv_pci_top_obj(kobj_ptr) container_of(kobj_ptr, struct uv_pci_top_obj, kobj) + +static ssize_t uv_pci_type_show(struct uv_pci_top_obj *top_obj, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%s\n", top_obj->type); +} + +static ssize_t uv_pci_location_show(struct uv_pci_top_obj *top_obj, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%s\n", top_obj->location); +} + +static ssize_t uv_pci_iio_stack_show(struct uv_pci_top_obj *top_obj, char *buf) +{ + return sprintf(buf, "%d\n", top_obj->iio_stack); +} + +static ssize_t uv_pci_ppb_addr_show(struct uv_pci_top_obj *top_obj, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%s\n", top_obj->ppb_addr); +} + +static ssize_t uv_pci_slot_show(struct uv_pci_top_obj *top_obj, char *buf) +{ + return sprintf(buf, "%d\n", top_obj->slot); +} + +struct uv_pci_top_sysfs_entry { + struct attribute attr; + ssize_t (*show)(struct uv_pci_top_obj *top_obj, char *buf); + ssize_t (*store)(struct uv_pci_top_obj *top_obj, const char *buf, size_t size); +}; + +static struct uv_pci_top_sysfs_entry uv_pci_type_attribute = + __ATTR(type, 0444, uv_pci_type_show, NULL); +static struct uv_pci_top_sysfs_entry uv_pci_location_attribute = + __ATTR(location, 0444, uv_pci_location_show, NULL); +static struct uv_pci_top_sysfs_entry uv_pci_iio_stack_attribute = + __ATTR(iio_stack, 0444, uv_pci_iio_stack_show, NULL); +static struct uv_pci_top_sysfs_entry uv_pci_ppb_addr_attribute = + __ATTR(ppb_addr, 0444, uv_pci_ppb_addr_show, NULL); +static struct uv_pci_top_sysfs_entry uv_pci_slot_attribute = + __ATTR(slot, 0444, uv_pci_slot_show, NULL); + +static void uv_pci_top_release(struct kobject *kobj) +{ + struct uv_pci_top_obj *top_obj = to_uv_pci_top_obj(kobj); + + kfree(top_obj->type); + kfree(top_obj->location); + kfree(top_obj->ppb_addr); + kfree(top_obj); +} + +static ssize_t pci_top_type_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct uv_pci_top_obj *top_obj = to_uv_pci_top_obj(kobj); + struct uv_pci_top_sysfs_entry *entry; + + entry = container_of(attr, struct uv_pci_top_sysfs_entry, attr); + + if (!entry->show) + return -EIO; + + return entry->show(top_obj, buf); +} + +static const struct sysfs_ops uv_pci_top_sysfs_ops = { + .show = pci_top_type_show, +}; + +static struct kobj_type uv_pci_top_attr_type = { + .release = uv_pci_top_release, + .sysfs_ops = &uv_pci_top_sysfs_ops, +}; + +static int init_pci_top_obj(struct uv_pci_top_obj *top_obj, char *line) +{ + char *start; + char type[11], location[14], ppb_addr[15]; + int str_cnt, ret; + unsigned int tmp_match[2]; + + // Minimum line length + if (strlen(line) < 36) + return -EINVAL; + + //Line must match format "pcibus %4x:%2x" to be valid + str_cnt = sscanf(line, "pcibus %4x:%2x", &tmp_match[0], &tmp_match[1]); + if (str_cnt < 2) + return -EINVAL; + + /* Connect pcibus to segment:bus number with '_' + * to concatenate name tokens. + * pcibus 0000:00 ... -> pcibus_0000:00 ... + */ + line[6] = '_'; + + /* Null terminate after the concatencated name tokens + * to produce kobj name string. + */ + line[14] = '\0'; + + // Use start to index after name tokens string for remainder of line info. + start = &line[15]; + + top_obj->iio_stack = -1; + top_obj->slot = -1; + + /* r001i01b00h0 BASE IO (IIO Stack 0) + * r001i01b00h1 PCIe IO (IIO Stack 1) + * r001i01b03h1 PCIe SLOT + * r001i01b00h0 NODE IO + * r001i01b00h0 Riser + * (IIO Stack #) may not be present. + */ + if (start[0] == 'r') { + str_cnt = sscanf(start, "%13s %10[^(] %*s %*s %d)", + location, type, &top_obj->iio_stack); + if (str_cnt < 2) + return -EINVAL; + top_obj->type = kstrdup(type, GFP_KERNEL); + if (!top_obj->type) + return -ENOMEM; + top_obj->location = kstrdup(location, GFP_KERNEL); + if (!top_obj->location) { + kfree(top_obj->type); + return -ENOMEM; + } + } + /* PPB at 0000:80:00.00 (slot 3) + * (slot #) may not be present. + */ + else if (start[0] == 'P') { + str_cnt = sscanf(start, "%10s %*s %14s %*s %d)", + type, ppb_addr, &top_obj->slot); + if (str_cnt < 2) + return -EINVAL; + top_obj->type = kstrdup(type, GFP_KERNEL); + if (!top_obj->type) + return -ENOMEM; + top_obj->ppb_addr = kstrdup(ppb_addr, GFP_KERNEL); + if (!top_obj->ppb_addr) { + kfree(top_obj->type); + return -ENOMEM; + } + } else + return -EINVAL; + + top_obj->kobj.kset = uv_pcibus_kset; + + ret = kobject_init_and_add(&top_obj->kobj, &uv_pci_top_attr_type, NULL, "%s", line); + if (ret) + goto err_add_sysfs; + + if (top_obj->type) { + ret = sysfs_create_file(&top_obj->kobj, &uv_pci_type_attribute.attr); + if (ret) + goto err_add_sysfs; + } + if (top_obj->location) { + ret = sysfs_create_file(&top_obj->kobj, &uv_pci_location_attribute.attr); + if (ret) + goto err_add_sysfs; + } + if (top_obj->iio_stack >= 0) { + ret = sysfs_create_file(&top_obj->kobj, &uv_pci_iio_stack_attribute.attr); + if (ret) + goto err_add_sysfs; + } + if (top_obj->ppb_addr) { + ret = sysfs_create_file(&top_obj->kobj, &uv_pci_ppb_addr_attribute.attr); + if (ret) + goto err_add_sysfs; + } + if (top_obj->slot >= 0) { + ret = sysfs_create_file(&top_obj->kobj, &uv_pci_slot_attribute.attr); + if (ret) + goto err_add_sysfs; + } + + kobject_uevent(&top_obj->kobj, KOBJ_ADD); + return 0; + +err_add_sysfs: + kobject_put(&top_obj->kobj); + return ret; +} + +static int pci_topology_init(void) +{ + char *pci_top_str, *start, *found, *count; + size_t sz; + s64 biosr; + int l = 0, k = 0; + int len, ret; + + uv_pcibus_kset = kset_create_and_add("pcibuses", NULL, sgi_uv_kobj); + if (!uv_pcibus_kset) + return -ENOMEM; + + for (sz = PAGE_SIZE; sz < 16 * PAGE_SIZE; sz += PAGE_SIZE) { + pci_top_str = kmalloc(sz, GFP_KERNEL); + if (!pci_top_str) { + ret = -ENOMEM; + goto err_pci_top_str; + } + biosr = uv_bios_get_pci_topology((u64)sz, (u64 *)pci_top_str); + if (biosr == BIOS_STATUS_SUCCESS) { + len = strnlen(pci_top_str, sz); + for (count = pci_top_str; count < pci_top_str + len; count++) { + if (*count == '\n') + l++; + } + num_pci_lines = l; + + uv_pci_objs = kcalloc(num_pci_lines, + sizeof(*uv_pci_objs), GFP_KERNEL); + if (!uv_pci_objs) { + kfree(pci_top_str); + ret = -ENOMEM; + goto err_pci_top_str; + } + start = pci_top_str; + while ((found = strsep(&start, "\n")) != NULL) { + uv_pci_objs[k] = kzalloc(sizeof(*uv_pci_objs[k]), GFP_KERNEL); + if (!uv_pci_objs[k]) { + ret = -ENOMEM; + goto err_pci_obj; + } + ret = init_pci_top_obj(uv_pci_objs[k], found); + if (ret) + goto err_pci_obj; + k++; + if (k == num_pci_lines) + break; + } + } + kfree(pci_top_str); + if (biosr == BIOS_STATUS_SUCCESS || biosr == BIOS_STATUS_UNIMPLEMENTED) + break; + } + + return 0; +err_pci_obj: + k--; + for (; k >= 0; k--) + kobject_put(&uv_pci_objs[k]->kobj); + kfree(uv_pci_objs); + kfree(pci_top_str); +err_pci_top_str: + kset_unregister(uv_pcibus_kset); + return ret; +} + +static void pci_topology_exit(void) +{ + int k; + + for (k = 0; k < num_pci_lines; k++) + kobject_put(&uv_pci_objs[k]->kobj); + kset_unregister(uv_pcibus_kset); + kfree(uv_pci_objs); +} + +static ssize_t partition_id_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%ld\n", sn_partition_id); +} + +static ssize_t coherence_id_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%ld\n", sn_coherency_id); +} + +static ssize_t uv_type_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%s\n", uv_type_string()); +} + +static ssize_t uv_archtype_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return uv_get_archtype(buf, PAGE_SIZE); +} + +static ssize_t uv_hub_type_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "0x%x\n", uv_hub_type()); +} + +static ssize_t uv_hubless_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "0x%x\n", uv_get_hubless_system()); +} + +static struct kobj_attribute partition_id_attr = + __ATTR(partition_id, 0444, partition_id_show, NULL); +static struct kobj_attribute coherence_id_attr = + __ATTR(coherence_id, 0444, coherence_id_show, NULL); +static struct kobj_attribute uv_type_attr = + __ATTR(uv_type, 0444, uv_type_show, NULL); +static struct kobj_attribute uv_archtype_attr = + __ATTR(archtype, 0444, uv_archtype_show, NULL); +static struct kobj_attribute uv_hub_type_attr = + __ATTR(hub_type, 0444, uv_hub_type_show, NULL); +static struct kobj_attribute uv_hubless_attr = + __ATTR(hubless, 0444, uv_hubless_show, NULL); + +static struct attribute *base_attrs[] = { + &partition_id_attr.attr, + &coherence_id_attr.attr, + &uv_type_attr.attr, + &uv_archtype_attr.attr, + &uv_hub_type_attr.attr, + NULL, +}; + +static struct attribute_group base_attr_group = { + .attrs = base_attrs +}; + +static int initial_bios_setup(void) +{ + u64 v; + s64 biosr; + + biosr = uv_bios_get_master_nasid((u64)sizeof(uv_master_nasid), (u64 *)&uv_master_nasid); + if (biosr) + return -EINVAL; + + biosr = uv_bios_get_heapsize((u64)uv_master_nasid, (u64)sizeof(u64), &v); + if (biosr) + return -EINVAL; + + uv_biosheap = vmalloc(v); + if (!uv_biosheap) + return -ENOMEM; + + biosr = uv_bios_install_heap((u64)uv_master_nasid, v, (u64 *)uv_biosheap); + if (biosr) { + vfree(uv_biosheap); + return -EINVAL; + } + + biosr = uv_bios_obj_count((u64)uv_master_nasid, sizeof(u64), &v); + if (biosr) { + vfree(uv_biosheap); + return -EINVAL; + } + uv_bios_obj_cnt = (int)v; + + return 0; +} + +static struct attribute *hubless_base_attrs[] = { + &partition_id_attr.attr, + &uv_type_attr.attr, + &uv_archtype_attr.attr, + &uv_hubless_attr.attr, + NULL, +}; + +static struct attribute_group hubless_base_attr_group = { + .attrs = hubless_base_attrs +}; + + +static int __init uv_sysfs_hubless_init(void) +{ + int ret; + + ret = sysfs_create_group(sgi_uv_kobj, &hubless_base_attr_group); + if (ret) { + pr_warn("sysfs_create_group hubless_base_attr_group failed\n"); + kobject_put(sgi_uv_kobj); + } + return ret; +} + +static int __init uv_sysfs_init(void) +{ + int ret = 0; + + if (!is_uv_system() && !uv_get_hubless_system()) + return -ENODEV; + + num_cnodes = uv_num_possible_blades(); + + if (!sgi_uv_kobj) + sgi_uv_kobj = kobject_create_and_add("sgi_uv", firmware_kobj); + if (!sgi_uv_kobj) { + pr_warn("kobject_create_and_add sgi_uv failed\n"); + return -EINVAL; + } + + if (uv_get_hubless_system()) + return uv_sysfs_hubless_init(); + + ret = sysfs_create_group(sgi_uv_kobj, &base_attr_group); + if (ret) { + pr_warn("sysfs_create_group base_attr_group failed\n"); + goto err_create_group; + } + + ret = initial_bios_setup(); + if (ret) + goto err_bios_setup; + + ret = uv_hubs_init(); + if (ret) + goto err_hubs_init; + + ret = uv_ports_init(); + if (ret) + goto err_ports_init; + + ret = pci_topology_init(); + if (ret) + goto err_pci_init; + + return 0; + +err_pci_init: + uv_ports_exit(); +err_ports_init: + uv_hubs_exit(); +err_hubs_init: + vfree(uv_biosheap); +err_bios_setup: + sysfs_remove_group(sgi_uv_kobj, &base_attr_group); +err_create_group: + kobject_put(sgi_uv_kobj); + return ret; +} + +static void __exit uv_sysfs_hubless_exit(void) +{ + sysfs_remove_group(sgi_uv_kobj, &hubless_base_attr_group); + kobject_put(sgi_uv_kobj); +} + +static void __exit uv_sysfs_exit(void) +{ + if (!is_uv_system()) { + if (uv_get_hubless_system()) + uv_sysfs_hubless_exit(); + return; + } + + pci_topology_exit(); + uv_ports_exit(); + uv_hubs_exit(); + vfree(uv_biosheap); + sysfs_remove_group(sgi_uv_kobj, &base_attr_group); + kobject_put(sgi_uv_kobj); +} + +#ifndef MODULE +device_initcall(uv_sysfs_init); +#else +module_init(uv_sysfs_init); +#endif +module_exit(uv_sysfs_exit); + +MODULE_AUTHOR("Hewlett Packard Enterprise"); +MODULE_LICENSE("GPL"); diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c index 217a7b84abdf..fd568248fd26 100644 --- a/drivers/s390/block/dasd.c +++ b/drivers/s390/block/dasd.c @@ -75,7 +75,6 @@ static int dasd_flush_block_queue(struct dasd_block *); static void dasd_device_tasklet(unsigned long); static void dasd_block_tasklet(unsigned long); static void do_kick_device(struct work_struct *); -static void do_restore_device(struct work_struct *); static void do_reload_device(struct work_struct *); static void do_requeue_requests(struct work_struct *); static void dasd_return_cqr_cb(struct dasd_ccw_req *, void *); @@ -138,7 +137,6 @@ struct dasd_device *dasd_alloc_device(void) INIT_LIST_HEAD(&device->ccw_queue); timer_setup(&device->timer, dasd_device_timeout, 0); INIT_WORK(&device->kick_work, do_kick_device); - INIT_WORK(&device->restore_device, do_restore_device); INIT_WORK(&device->reload_device, do_reload_device); INIT_WORK(&device->requeue_requests, do_requeue_requests); device->state = DASD_STATE_NEW; @@ -620,26 +618,6 @@ void dasd_reload_device(struct dasd_device *device) } EXPORT_SYMBOL(dasd_reload_device); -/* - * dasd_restore_device will schedule a call do do_restore_device to the kernel - * event daemon. - */ -static void do_restore_device(struct work_struct *work) -{ - struct dasd_device *device = container_of(work, struct dasd_device, - restore_device); - device->cdev->drv->restore(device->cdev); - dasd_put_device(device); -} - -void dasd_restore_device(struct dasd_device *device) -{ - dasd_get_device(device); - /* queue call to dasd_restore_device to the kernel event daemon. */ - if (!schedule_work(&device->restore_device)) - dasd_put_device(device); -} - /* * Set the target state for a device and starts the state change. */ @@ -1514,7 +1492,6 @@ int dasd_start_IO(struct dasd_ccw_req *cqr) "start_IO: -EIO device gone, retry"); break; case -EINVAL: - /* most likely caused in power management context */ DBF_DEV_EVENT(DBF_WARNING, device, "%s", "start_IO: -EINVAL device currently " "not accessible"); @@ -2048,7 +2025,7 @@ static void __dasd_device_check_expire(struct dasd_device *device) static int __dasd_device_is_unusable(struct dasd_device *device, struct dasd_ccw_req *cqr) { - int mask = ~(DASD_STOPPED_DC_WAIT | DASD_UNRESUMED_PM | DASD_STOPPED_NOSPC); + int mask = ~(DASD_STOPPED_DC_WAIT | DASD_STOPPED_NOSPC); if (test_bit(DASD_FLAG_OFFLINE, &device->flags) && !test_bit(DASD_FLAG_SAFE_OFFLINE_RUNNING, &device->flags)) { @@ -2112,8 +2089,7 @@ static void __dasd_device_check_path_events(struct dasd_device *device) if (!dasd_path_get_tbvpm(device)) return; - if (device->stopped & - ~(DASD_STOPPED_DC_WAIT | DASD_UNRESUMED_PM)) + if (device->stopped & ~(DASD_STOPPED_DC_WAIT)) return; rc = device->discipline->verify_path(device, dasd_path_get_tbvpm(device)); @@ -3794,11 +3770,6 @@ int dasd_generic_path_operational(struct dasd_device *device) "operational\n"); DBF_DEV_EVENT(DBF_WARNING, device, "%s", "path operational"); dasd_device_remove_stop_bits(device, DASD_STOPPED_DC_WAIT); - if (device->stopped & DASD_UNRESUMED_PM) { - dasd_device_remove_stop_bits(device, DASD_UNRESUMED_PM); - dasd_restore_device(device); - return 1; - } dasd_schedule_device_bh(device); if (device->block) { dasd_schedule_block_bh(device->block); @@ -4058,66 +4029,6 @@ void dasd_schedule_requeue(struct dasd_device *device) } EXPORT_SYMBOL(dasd_schedule_requeue); -int dasd_generic_pm_freeze(struct ccw_device *cdev) -{ - struct dasd_device *device = dasd_device_from_cdev(cdev); - - if (IS_ERR(device)) - return PTR_ERR(device); - - /* mark device as suspended */ - set_bit(DASD_FLAG_SUSPENDED, &device->flags); - - if (device->discipline->freeze) - device->discipline->freeze(device); - - /* disallow new I/O */ - dasd_device_set_stop_bits(device, DASD_STOPPED_PM); - - return dasd_generic_requeue_all_requests(device); -} -EXPORT_SYMBOL_GPL(dasd_generic_pm_freeze); - -int dasd_generic_restore_device(struct ccw_device *cdev) -{ - struct dasd_device *device = dasd_device_from_cdev(cdev); - int rc = 0; - - if (IS_ERR(device)) - return PTR_ERR(device); - - /* allow new IO again */ - dasd_device_remove_stop_bits(device, - (DASD_STOPPED_PM | DASD_UNRESUMED_PM)); - - dasd_schedule_device_bh(device); - - /* - * call discipline restore function - * if device is stopped do nothing e.g. for disconnected devices - */ - if (device->discipline->restore && !(device->stopped)) - rc = device->discipline->restore(device); - if (rc || device->stopped) - /* - * if the resume failed for the DASD we put it in - * an UNRESUMED stop state - */ - device->stopped |= DASD_UNRESUMED_PM; - - if (device->block) { - dasd_schedule_block_bh(device->block); - if (device->block->request_queue) - blk_mq_run_hw_queues(device->block->request_queue, - true); - } - - clear_bit(DASD_FLAG_SUSPENDED, &device->flags); - dasd_put_device(device); - return 0; -} -EXPORT_SYMBOL_GPL(dasd_generic_restore_device); - static struct dasd_ccw_req *dasd_generic_build_rdc(struct dasd_device *device, int rdc_buffer_size, int magic) diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index ad44d22e8859..758ee4153ac1 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -5716,95 +5716,6 @@ static void dasd_eckd_dump_sense(struct dasd_device *device, } } -static int dasd_eckd_pm_freeze(struct dasd_device *device) -{ - /* - * the device should be disconnected from our LCU structure - * on restore we will reconnect it and reread LCU specific - * information like PAV support that might have changed - */ - dasd_alias_remove_device(device); - dasd_alias_disconnect_device_from_lcu(device); - - return 0; -} - -static int dasd_eckd_restore_device(struct dasd_device *device) -{ - struct dasd_eckd_private *private = device->private; - struct dasd_eckd_characteristics temp_rdc_data; - int rc; - struct dasd_uid temp_uid; - unsigned long flags; - unsigned long cqr_flags = 0; - - /* Read Configuration Data */ - rc = dasd_eckd_read_conf(device); - if (rc) { - DBF_EVENT_DEVID(DBF_WARNING, device->cdev, - "Read configuration data failed, rc=%d", rc); - goto out_err; - } - - dasd_eckd_get_uid(device, &temp_uid); - /* Generate device unique id */ - rc = dasd_eckd_generate_uid(device); - spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags); - if (memcmp(&private->uid, &temp_uid, sizeof(struct dasd_uid)) != 0) - dev_err(&device->cdev->dev, "The UID of the DASD has " - "changed\n"); - spin_unlock_irqrestore(get_ccwdev_lock(device->cdev), flags); - if (rc) - goto out_err; - - /* register lcu with alias handling, enable PAV if this is a new lcu */ - rc = dasd_alias_make_device_known_to_lcu(device); - if (rc) - goto out_err; - - set_bit(DASD_CQR_FLAGS_FAILFAST, &cqr_flags); - dasd_eckd_validate_server(device, cqr_flags); - - /* RE-Read Configuration Data */ - rc = dasd_eckd_read_conf(device); - if (rc) { - DBF_EVENT_DEVID(DBF_WARNING, device->cdev, - "Read configuration data failed, rc=%d", rc); - goto out_err2; - } - - /* Read Feature Codes */ - dasd_eckd_read_features(device); - - /* Read Volume Information */ - dasd_eckd_read_vol_info(device); - - /* Read Extent Pool Information */ - dasd_eckd_read_ext_pool_info(device); - - /* Read Device Characteristics */ - rc = dasd_generic_read_dev_chars(device, DASD_ECKD_MAGIC, - &temp_rdc_data, 64); - if (rc) { - DBF_EVENT_DEVID(DBF_WARNING, device->cdev, - "Read device characteristic failed, rc=%d", rc); - goto out_err2; - } - spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags); - memcpy(&private->rdc_data, &temp_rdc_data, sizeof(temp_rdc_data)); - spin_unlock_irqrestore(get_ccwdev_lock(device->cdev), flags); - - /* add device to alias management */ - dasd_alias_add_device(device); - - return 0; - -out_err2: - dasd_alias_disconnect_device_from_lcu(device); -out_err: - return -1; -} - static int dasd_eckd_reload_device(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; @@ -6668,9 +6579,6 @@ static struct ccw_driver dasd_eckd_driver = { .notify = dasd_generic_notify, .path_event = dasd_generic_path_event, .shutdown = dasd_generic_shutdown, - .freeze = dasd_generic_pm_freeze, - .thaw = dasd_generic_restore_device, - .restore = dasd_generic_restore_device, .uc_handler = dasd_generic_uc_handler, .int_class = IRQIO_DAS, }; @@ -6702,8 +6610,6 @@ static struct dasd_discipline dasd_eckd_discipline = { .dump_sense_dbf = dasd_eckd_dump_sense_dbf, .fill_info = dasd_eckd_fill_info, .ioctl = dasd_eckd_ioctl, - .freeze = dasd_eckd_pm_freeze, - .restore = dasd_eckd_restore_device, .reload = dasd_eckd_reload_device, .get_uid = dasd_eckd_get_uid, .kick_validate = dasd_eckd_kick_validate_server, diff --git a/drivers/s390/block/dasd_fba.c b/drivers/s390/block/dasd_fba.c index 1a44e321b54e..c027344ee225 100644 --- a/drivers/s390/block/dasd_fba.c +++ b/drivers/s390/block/dasd_fba.c @@ -79,9 +79,6 @@ static struct ccw_driver dasd_fba_driver = { .set_online = dasd_fba_set_online, .notify = dasd_generic_notify, .path_event = dasd_generic_path_event, - .freeze = dasd_generic_pm_freeze, - .thaw = dasd_generic_restore_device, - .restore = dasd_generic_restore_device, .int_class = IRQIO_DAS, }; diff --git a/drivers/s390/block/dasd_int.h b/drivers/s390/block/dasd_int.h index fa552f9f1666..7a34161ea5c6 100644 --- a/drivers/s390/block/dasd_int.h +++ b/drivers/s390/block/dasd_int.h @@ -355,10 +355,6 @@ struct dasd_discipline { int (*fill_info) (struct dasd_device *, struct dasd_information2_t *); int (*ioctl) (struct dasd_block *, unsigned int, void __user *); - /* suspend/resume functions */ - int (*freeze) (struct dasd_device *); - int (*restore) (struct dasd_device *); - /* reload device after state change */ int (*reload) (struct dasd_device *); @@ -520,7 +516,6 @@ struct dasd_device { atomic_t tasklet_scheduled; struct tasklet_struct tasklet; struct work_struct kick_work; - struct work_struct restore_device; struct work_struct reload_device; struct work_struct kick_validate; struct work_struct suc_work; @@ -592,8 +587,6 @@ struct dasd_queue { #define DASD_STOPPED_PENDING 4 /* long busy */ #define DASD_STOPPED_DC_WAIT 8 /* disconnected, wait */ #define DASD_STOPPED_SU 16 /* summary unit check handling */ -#define DASD_STOPPED_PM 32 /* pm state transition */ -#define DASD_UNRESUMED_PM 64 /* pm resume failed state */ #define DASD_STOPPED_NOSPC 128 /* no space left */ /* per device flags */ @@ -753,7 +746,6 @@ enum blk_eh_timer_return dasd_times_out(struct request *req, bool reserved); void dasd_enable_device(struct dasd_device *); void dasd_set_target_state(struct dasd_device *, int); void dasd_kick_device(struct dasd_device *); -void dasd_restore_device(struct dasd_device *); void dasd_reload_device(struct dasd_device *); void dasd_schedule_requeue(struct dasd_device *); @@ -785,8 +777,6 @@ int dasd_generic_path_operational(struct dasd_device *); void dasd_generic_shutdown(struct ccw_device *); void dasd_generic_handle_state_change(struct dasd_device *); -int dasd_generic_pm_freeze(struct ccw_device *); -int dasd_generic_restore_device(struct ccw_device *); enum uc_todo dasd_generic_uc_handler(struct ccw_device *, struct irb *); void dasd_generic_path_event(struct ccw_device *, int *); int dasd_generic_verify_path(struct dasd_device *, __u8); diff --git a/drivers/s390/char/con3215.c b/drivers/s390/char/con3215.c index d8acabbb1ed3..1354c42d95aa 100644 --- a/drivers/s390/char/con3215.c +++ b/drivers/s390/char/con3215.c @@ -289,16 +289,14 @@ static void raw3215_timeout(struct timer_list *t) spin_lock_irqsave(get_ccwdev_lock(raw->cdev), flags); raw->flags &= ~RAW3215_TIMER_RUNS; - if (!tty_port_suspended(&raw->port)) { - raw3215_mk_write_req(raw); - raw3215_start_io(raw); - if ((raw->queued_read || raw->queued_write) && - !(raw->flags & RAW3215_WORKING) && - !(raw->flags & RAW3215_TIMER_RUNS)) { - raw->timer.expires = RAW3215_TIMEOUT + jiffies; - add_timer(&raw->timer); - raw->flags |= RAW3215_TIMER_RUNS; - } + raw3215_mk_write_req(raw); + raw3215_start_io(raw); + if ((raw->queued_read || raw->queued_write) && + !(raw->flags & RAW3215_WORKING) && + !(raw->flags & RAW3215_TIMER_RUNS)) { + raw->timer.expires = RAW3215_TIMEOUT + jiffies; + add_timer(&raw->timer); + raw->flags |= RAW3215_TIMER_RUNS; } spin_unlock_irqrestore(get_ccwdev_lock(raw->cdev), flags); } @@ -311,7 +309,7 @@ static void raw3215_timeout(struct timer_list *t) */ static inline void raw3215_try_io(struct raw3215_info *raw) { - if (!tty_port_initialized(&raw->port) || tty_port_suspended(&raw->port)) + if (!tty_port_initialized(&raw->port)) return; if (raw->queued_read != NULL) raw3215_start_io(raw); @@ -463,26 +461,6 @@ put_tty: tty_kref_put(tty); } -/* - * Drop the oldest line from the output buffer. - */ -static void raw3215_drop_line(struct raw3215_info *raw) -{ - int ix; - char ch; - - BUG_ON(raw->written != 0); - ix = (raw->head - raw->count) & (RAW3215_BUFFER_SIZE - 1); - while (raw->count > 0) { - ch = raw->buffer[ix]; - ix = (ix + 1) & (RAW3215_BUFFER_SIZE - 1); - raw->count--; - if (ch == 0x15) - break; - } - raw->head = ix; -} - /* * Wait until length bytes are available int the output buffer. * Has to be called with the s390irq lock held. Can be called @@ -491,13 +469,6 @@ static void raw3215_drop_line(struct raw3215_info *raw) static void raw3215_make_room(struct raw3215_info *raw, unsigned int length) { while (RAW3215_BUFFER_SIZE - raw->count < length) { - /* While console is frozen for suspend we have no other - * choice but to drop message from the buffer to make - * room for even more messages. */ - if (tty_port_suspended(&raw->port)) { - raw3215_drop_line(raw); - continue; - } /* there might be a request pending */ raw->flags |= RAW3215_FLUSHING; raw3215_mk_write_req(raw); @@ -763,36 +734,6 @@ static int raw3215_set_offline (struct ccw_device *cdev) return 0; } -static int raw3215_pm_stop(struct ccw_device *cdev) -{ - struct raw3215_info *raw; - unsigned long flags; - - /* Empty the output buffer, then prevent new I/O. */ - raw = dev_get_drvdata(&cdev->dev); - spin_lock_irqsave(get_ccwdev_lock(raw->cdev), flags); - raw3215_make_room(raw, RAW3215_BUFFER_SIZE); - tty_port_set_suspended(&raw->port, 1); - spin_unlock_irqrestore(get_ccwdev_lock(raw->cdev), flags); - return 0; -} - -static int raw3215_pm_start(struct ccw_device *cdev) -{ - struct raw3215_info *raw; - unsigned long flags; - - /* Allow I/O again and flush output buffer. */ - raw = dev_get_drvdata(&cdev->dev); - spin_lock_irqsave(get_ccwdev_lock(raw->cdev), flags); - tty_port_set_suspended(&raw->port, 0); - raw->flags |= RAW3215_FLUSHING; - raw3215_try_io(raw); - raw->flags &= ~RAW3215_FLUSHING; - spin_unlock_irqrestore(get_ccwdev_lock(raw->cdev), flags); - return 0; -} - static struct ccw_device_id raw3215_id[] = { { CCW_DEVICE(0x3215, 0) }, { /* end of list */ }, @@ -808,9 +749,6 @@ static struct ccw_driver raw3215_ccw_driver = { .remove = &raw3215_remove, .set_online = &raw3215_set_online, .set_offline = &raw3215_set_offline, - .freeze = &raw3215_pm_stop, - .thaw = &raw3215_pm_start, - .restore = &raw3215_pm_start, .int_class = IRQIO_C15, }; @@ -858,11 +796,6 @@ static void con3215_flush(void) unsigned long flags; raw = raw3215[0]; /* console 3215 is the first one */ - if (tty_port_suspended(&raw->port)) - /* The console is still frozen for suspend. */ - if (ccw_device_force_console(raw->cdev)) - /* Forcing didn't work, no panic message .. */ - return; spin_lock_irqsave(get_ccwdev_lock(raw->cdev), flags); raw3215_make_room(raw, RAW3215_BUFFER_SIZE); spin_unlock_irqrestore(get_ccwdev_lock(raw->cdev), flags); diff --git a/drivers/s390/char/con3270.c b/drivers/s390/char/con3270.c index e17364e13d2f..e21962c0fd94 100644 --- a/drivers/s390/char/con3270.c +++ b/drivers/s390/char/con3270.c @@ -544,7 +544,6 @@ con3270_flush(void) cp = condev; if (!cp->view.dev) return; - raw3270_pm_unfreeze(&cp->view); raw3270_activate_view(&cp->view); spin_lock_irqsave(&cp->view.lock, flags); con3270_wait_write(cp); diff --git a/drivers/s390/char/raw3270.c b/drivers/s390/char/raw3270.c index 63a41b168761..646ec796bb83 100644 --- a/drivers/s390/char/raw3270.c +++ b/drivers/s390/char/raw3270.c @@ -67,7 +67,6 @@ struct raw3270 { #define RAW3270_FLAGS_14BITADDR 0 /* 14-bit buffer addresses */ #define RAW3270_FLAGS_BUSY 1 /* Device busy, leave it alone */ #define RAW3270_FLAGS_CONSOLE 2 /* Device is the console. */ -#define RAW3270_FLAGS_FROZEN 3 /* set if 3270 is frozen for suspend */ /* Semaphore to protect global data of raw3270 (devices, views, etc). */ static DEFINE_MUTEX(raw3270_mutex); @@ -260,8 +259,7 @@ raw3270_view_active(struct raw3270_view *view) { struct raw3270 *rp = view->dev; - return rp && rp->view == view && - !test_bit(RAW3270_FLAGS_FROZEN, &rp->flags); + return rp && rp->view == view; } int @@ -273,8 +271,7 @@ raw3270_start(struct raw3270_view *view, struct raw3270_request *rq) spin_lock_irqsave(get_ccwdev_lock(view->dev->cdev), flags); rp = view->dev; - if (!rp || rp->view != view || - test_bit(RAW3270_FLAGS_FROZEN, &rp->flags)) + if (!rp || rp->view != view) rc = -EACCES; else if (!raw3270_state_ready(rp)) rc = -EBUSY; @@ -291,8 +288,7 @@ raw3270_start_locked(struct raw3270_view *view, struct raw3270_request *rq) int rc; rp = view->dev; - if (!rp || rp->view != view || - test_bit(RAW3270_FLAGS_FROZEN, &rp->flags)) + if (!rp || rp->view != view) rc = -EACCES; else if (!raw3270_state_ready(rp)) rc = -EBUSY; @@ -629,8 +625,7 @@ raw3270_reset(struct raw3270_view *view) int rc; rp = view->dev; - if (!rp || rp->view != view || - test_bit(RAW3270_FLAGS_FROZEN, &rp->flags)) + if (!rp || rp->view != view) rc = -EACCES; else if (!raw3270_state_ready(rp)) rc = -EBUSY; @@ -854,8 +849,6 @@ raw3270_activate_view(struct raw3270_view *view) rc = 0; else if (!raw3270_state_ready(rp)) rc = -EBUSY; - else if (test_bit(RAW3270_FLAGS_FROZEN, &rp->flags)) - rc = -EACCES; else { oldview = NULL; if (rp->view && rp->view->fn->deactivate) { @@ -903,8 +896,7 @@ raw3270_deactivate_view(struct raw3270_view *view) list_del_init(&view->list); list_add_tail(&view->list, &rp->view_list); /* Try to activate another view. */ - if (raw3270_state_ready(rp) && - !test_bit(RAW3270_FLAGS_FROZEN, &rp->flags)) { + if (raw3270_state_ready(rp)) { list_for_each_entry(view, &rp->view_list, list) { rp->view = view; if (view->fn->activate(view) == 0) @@ -999,8 +991,7 @@ raw3270_del_view(struct raw3270_view *view) rp->view = NULL; } list_del_init(&view->list); - if (!rp->view && raw3270_state_ready(rp) && - !test_bit(RAW3270_FLAGS_FROZEN, &rp->flags)) { + if (!rp->view && raw3270_state_ready(rp)) { /* Try to activate another view. */ list_for_each_entry(nv, &rp->view_list, list) { if (nv->fn->activate(nv) == 0) { @@ -1215,60 +1206,6 @@ raw3270_set_offline (struct ccw_device *cdev) return 0; } -static int raw3270_pm_stop(struct ccw_device *cdev) -{ - struct raw3270 *rp; - struct raw3270_view *view; - unsigned long flags; - - rp = dev_get_drvdata(&cdev->dev); - if (!rp) - return 0; - spin_lock_irqsave(get_ccwdev_lock(rp->cdev), flags); - if (rp->view && rp->view->fn->deactivate) - rp->view->fn->deactivate(rp->view); - if (!test_bit(RAW3270_FLAGS_CONSOLE, &rp->flags)) { - /* - * Release tty and fullscreen for all non-console - * devices. - */ - list_for_each_entry(view, &rp->view_list, list) { - if (view->fn->release) - view->fn->release(view); - } - } - set_bit(RAW3270_FLAGS_FROZEN, &rp->flags); - spin_unlock_irqrestore(get_ccwdev_lock(rp->cdev), flags); - return 0; -} - -static int raw3270_pm_start(struct ccw_device *cdev) -{ - struct raw3270 *rp; - unsigned long flags; - - rp = dev_get_drvdata(&cdev->dev); - if (!rp) - return 0; - spin_lock_irqsave(get_ccwdev_lock(rp->cdev), flags); - clear_bit(RAW3270_FLAGS_FROZEN, &rp->flags); - if (rp->view && rp->view->fn->activate) - rp->view->fn->activate(rp->view); - spin_unlock_irqrestore(get_ccwdev_lock(rp->cdev), flags); - return 0; -} - -void raw3270_pm_unfreeze(struct raw3270_view *view) -{ -#ifdef CONFIG_TN3270_CONSOLE - struct raw3270 *rp; - - rp = view->dev; - if (rp && test_bit(RAW3270_FLAGS_FROZEN, &rp->flags)) - ccw_device_force_console(rp->cdev); -#endif -} - static struct ccw_device_id raw3270_id[] = { { CCW_DEVICE(0x3270, 0) }, { CCW_DEVICE(0x3271, 0) }, @@ -1294,9 +1231,6 @@ static struct ccw_driver raw3270_ccw_driver = { .remove = &raw3270_remove, .set_online = &raw3270_set_online, .set_offline = &raw3270_set_offline, - .freeze = &raw3270_pm_stop, - .thaw = &raw3270_pm_start, - .restore = &raw3270_pm_start, .int_class = IRQIO_C70, }; diff --git a/drivers/s390/char/raw3270.h b/drivers/s390/char/raw3270.h index 8d979e0ee605..c6645167cd2b 100644 --- a/drivers/s390/char/raw3270.h +++ b/drivers/s390/char/raw3270.h @@ -199,7 +199,6 @@ struct raw3270_notifier { int raw3270_register_notifier(struct raw3270_notifier *); void raw3270_unregister_notifier(struct raw3270_notifier *); -void raw3270_pm_unfreeze(struct raw3270_view *); /* * Little memory allocator for string objects. diff --git a/drivers/s390/char/sclp.h b/drivers/s390/char/sclp.h index 69d9cde9ff5a..6de919944a39 100644 --- a/drivers/s390/char/sclp.h +++ b/drivers/s390/char/sclp.h @@ -156,7 +156,11 @@ struct read_cpu_info_sccb { u16 offset_configured; u16 nr_standby; u16 offset_standby; - u8 reserved[4096 - 16]; + /* + * Without ext sccb, struct size is PAGE_SIZE. + * With ext sccb, struct size is EXT_SCCB_READ_CPU. + */ + u8 reserved[]; } __attribute__((packed, aligned(PAGE_SIZE))); struct read_info_sccb { @@ -199,7 +203,7 @@ struct read_info_sccb { u8 byte_134; /* 134 */ u8 cpudirq; /* 135 */ u16 cbl; /* 136-137 */ - u8 _pad_138[4096 - 138]; /* 138-4095 */ + u8 _pad_138[EXT_SCCB_READ_SCP - 138]; } __packed __aligned(PAGE_SIZE); struct read_storage_sccb { @@ -328,7 +332,7 @@ unsigned int sclp_early_con_check_vt220(struct init_sccb *sccb); int sclp_early_set_event_mask(struct init_sccb *sccb, sccb_mask_t receive_mask, sccb_mask_t send_mask); -int sclp_early_get_info(struct read_info_sccb *info); +struct read_info_sccb * __init sclp_early_get_info(void); /* useful inlines */ diff --git a/drivers/s390/char/sclp_cmd.c b/drivers/s390/char/sclp_cmd.c index f6e97f0830f6..d41bc144c183 100644 --- a/drivers/s390/char/sclp_cmd.c +++ b/drivers/s390/char/sclp_cmd.c @@ -27,6 +27,7 @@ #include #include #include +#include #include "sclp.h" @@ -87,14 +88,17 @@ out: int _sclp_get_core_info(struct sclp_core_info *info) { int rc; + int length = test_facility(140) ? EXT_SCCB_READ_CPU : PAGE_SIZE; struct read_cpu_info_sccb *sccb; if (!SCLP_HAS_CPU_INFO) return -EOPNOTSUPP; - sccb = (void *) get_zeroed_page(GFP_KERNEL | GFP_DMA); + + sccb = (void *)__get_free_pages(GFP_KERNEL | GFP_DMA | __GFP_ZERO, get_order(length)); if (!sccb) return -ENOMEM; - sccb->header.length = sizeof(*sccb); + sccb->header.length = length; + sccb->header.control_mask[2] = 0x80; rc = sclp_sync_request_timeout(SCLP_CMDW_READ_CPU_INFO, sccb, SCLP_QUEUE_INTERVAL); if (rc) @@ -107,7 +111,7 @@ int _sclp_get_core_info(struct sclp_core_info *info) } sclp_fill_core_info(info, sccb); out: - free_page((unsigned long) sccb); + free_pages((unsigned long) sccb, get_order(length)); return rc; } @@ -397,10 +401,10 @@ static void __init add_memory_merged(u16 rn) goto skip_add; if (start + size > VMEM_MAX_PHYS) size = VMEM_MAX_PHYS - start; - if (memory_end_set && (start >= memory_end)) + if (start >= ident_map_size) goto skip_add; - if (memory_end_set && (start + size > memory_end)) - size = memory_end - start; + if (start + size > ident_map_size) + size = ident_map_size - start; block_size = memory_block_size_bytes(); align_to_block_size(&start, &size, block_size); if (!size) diff --git a/drivers/s390/char/sclp_early.c b/drivers/s390/char/sclp_early.c index cc5e84b80c69..2f3515fa242a 100644 --- a/drivers/s390/char/sclp_early.c +++ b/drivers/s390/char/sclp_early.c @@ -9,9 +9,12 @@ #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt #include +#include #include #include #include +#include +#include #include "sclp_sdias.h" #include "sclp.h" @@ -20,12 +23,14 @@ static struct sclp_ipl_info sclp_ipl_info; struct sclp_info sclp; EXPORT_SYMBOL(sclp); -static void __init sclp_early_facilities_detect(struct read_info_sccb *sccb) +static void __init sclp_early_facilities_detect(void) { struct sclp_core_entry *cpue; + struct read_info_sccb *sccb; u16 boot_cpu_address, cpu; - if (sclp_early_get_info(sccb)) + sccb = sclp_early_get_info(); + if (!sccb) return; sclp.facilities = sccb->facilities; @@ -107,29 +112,34 @@ void __init sclp_early_get_ipl_info(struct sclp_ipl_info *info) *info = sclp_ipl_info; } -static struct sclp_core_info sclp_early_core_info __initdata; -static int sclp_early_core_info_valid __initdata; - -static void __init sclp_early_init_core_info(struct read_cpu_info_sccb *sccb) -{ - if (!SCLP_HAS_CPU_INFO) - return; - memset(sccb, 0, sizeof(*sccb)); - sccb->header.length = sizeof(*sccb); - if (sclp_early_cmd(SCLP_CMDW_READ_CPU_INFO, sccb)) - return; - if (sccb->header.response_code != 0x0010) - return; - sclp_fill_core_info(&sclp_early_core_info, sccb); - sclp_early_core_info_valid = 1; -} - int __init sclp_early_get_core_info(struct sclp_core_info *info) { - if (!sclp_early_core_info_valid) - return -EIO; - *info = sclp_early_core_info; - return 0; + struct read_cpu_info_sccb *sccb; + int length = test_facility(140) ? EXT_SCCB_READ_CPU : PAGE_SIZE; + int rc = 0; + + if (!SCLP_HAS_CPU_INFO) + return -EOPNOTSUPP; + + sccb = memblock_alloc_low(length, PAGE_SIZE); + if (!sccb) + return -ENOMEM; + + memset(sccb, 0, length); + sccb->header.length = length; + sccb->header.control_mask[2] = 0x80; + if (sclp_early_cmd(SCLP_CMDW_READ_CPU_INFO, sccb)) { + rc = -EIO; + goto out; + } + if (sccb->header.response_code != 0x0010) { + rc = -EIO; + goto out; + } + sclp_fill_core_info(info, sccb); +out: + memblock_free_early((unsigned long)sccb, length); + return rc; } static void __init sclp_early_console_detect(struct init_sccb *sccb) @@ -148,8 +158,7 @@ void __init sclp_early_detect(void) { void *sccb = sclp_early_sccb; - sclp_early_facilities_detect(sccb); - sclp_early_init_core_info(sccb); + sclp_early_facilities_detect(); /* * Turn off SCLP event notifications. Also save remote masks in the diff --git a/drivers/s390/char/sclp_early_core.c b/drivers/s390/char/sclp_early_core.c index a960afa974bf..ec9f8ad5341c 100644 --- a/drivers/s390/char/sclp_early_core.c +++ b/drivers/s390/char/sclp_early_core.c @@ -11,6 +11,7 @@ #include #include #include +#include #include "sclp.h" #include "sclp_rw.h" @@ -237,13 +238,14 @@ void sclp_early_printk(const char *str) int __init sclp_early_read_info(void) { int i; + int length = test_facility(140) ? EXT_SCCB_READ_SCP : PAGE_SIZE; struct read_info_sccb *sccb = &sclp_info_sccb; sclp_cmdw_t commands[] = {SCLP_CMDW_READ_SCP_INFO_FORCED, SCLP_CMDW_READ_SCP_INFO}; for (i = 0; i < ARRAY_SIZE(commands); i++) { - memset(sccb, 0, sizeof(*sccb)); - sccb->header.length = sizeof(*sccb); + memset(sccb, 0, length); + sccb->header.length = length; sccb->header.function_code = 0x80; sccb->header.control_mask[2] = 0x80; if (sclp_early_cmd(commands[i], sccb)) @@ -258,13 +260,12 @@ int __init sclp_early_read_info(void) return -EIO; } -int __init sclp_early_get_info(struct read_info_sccb *info) +struct read_info_sccb * __init sclp_early_get_info(void) { if (!sclp_info_sccb_valid) - return -EIO; + return NULL; - *info = sclp_info_sccb; - return 0; + return &sclp_info_sccb; } int __init sclp_early_get_memsize(unsigned long *mem) diff --git a/drivers/s390/char/tape.h b/drivers/s390/char/tape.h index e2c60475dfa8..4e5d5efa978f 100644 --- a/drivers/s390/char/tape.h +++ b/drivers/s390/char/tape.h @@ -264,7 +264,6 @@ extern void tape_state_set(struct tape_device *, enum tape_state); extern int tape_generic_online(struct tape_device *, struct tape_discipline *); extern int tape_generic_offline(struct ccw_device *); -extern int tape_generic_pm_suspend(struct ccw_device *); /* Externals from tape_devmap.c */ extern int tape_generic_probe(struct ccw_device *); diff --git a/drivers/s390/char/tape_34xx.c b/drivers/s390/char/tape_34xx.c index 6d73ee3f827a..7ada994d4592 100644 --- a/drivers/s390/char/tape_34xx.c +++ b/drivers/s390/char/tape_34xx.c @@ -1191,7 +1191,6 @@ static struct ccw_driver tape_34xx_driver = { .remove = tape_generic_remove, .set_online = tape_34xx_online, .set_offline = tape_generic_offline, - .freeze = tape_generic_pm_suspend, .int_class = IRQIO_TAP, }; diff --git a/drivers/s390/char/tape_3590.c b/drivers/s390/char/tape_3590.c index 4554cdf4d6bd..ecf8c5006a0e 100644 --- a/drivers/s390/char/tape_3590.c +++ b/drivers/s390/char/tape_3590.c @@ -1651,7 +1651,6 @@ static struct ccw_driver tape_3590_driver = { .remove = tape_generic_remove, .set_offline = tape_generic_offline, .set_online = tape_3590_online, - .freeze = tape_generic_pm_suspend, .int_class = IRQIO_TAP, }; diff --git a/drivers/s390/char/tape_core.c b/drivers/s390/char/tape_core.c index 380e6a67719c..a6d2a4792185 100644 --- a/drivers/s390/char/tape_core.c +++ b/drivers/s390/char/tape_core.c @@ -427,55 +427,6 @@ tape_cleanup_device(struct tape_device *device) tape_med_state_set(device, MS_UNKNOWN); } -/* - * Suspend device. - * - * Called by the common I/O layer if the drive should be suspended on user - * request. We refuse to suspend if the device is loaded or in use for the - * following reason: - * While the Linux guest is suspended, it might be logged off which causes - * devices to be detached. Tape devices are automatically rewound and unloaded - * during DETACH processing (unless the tape device was attached with the - * NOASSIGN or MULTIUSER option). After rewind/unload, there is no way to - * resume the original state of the tape device, since we would need to - * manually re-load the cartridge which was active at suspend time. - */ -int tape_generic_pm_suspend(struct ccw_device *cdev) -{ - struct tape_device *device; - - device = dev_get_drvdata(&cdev->dev); - if (!device) { - return -ENODEV; - } - - DBF_LH(3, "(%08x): tape_generic_pm_suspend(%p)\n", - device->cdev_id, device); - - if (device->medium_state != MS_UNLOADED) { - pr_err("A cartridge is loaded in tape device %s, " - "refusing to suspend\n", dev_name(&cdev->dev)); - return -EBUSY; - } - - spin_lock_irq(get_ccwdev_lock(device->cdev)); - switch (device->tape_state) { - case TS_INIT: - case TS_NOT_OPER: - case TS_UNUSED: - spin_unlock_irq(get_ccwdev_lock(device->cdev)); - break; - default: - pr_err("Tape device %s is busy, refusing to " - "suspend\n", dev_name(&cdev->dev)); - spin_unlock_irq(get_ccwdev_lock(device->cdev)); - return -EBUSY; - } - - DBF_LH(3, "(%08x): Drive suspended.\n", device->cdev_id); - return 0; -} - /* * Set device offline. * @@ -1360,7 +1311,6 @@ EXPORT_SYMBOL(tape_generic_remove); EXPORT_SYMBOL(tape_generic_probe); EXPORT_SYMBOL(tape_generic_online); EXPORT_SYMBOL(tape_generic_offline); -EXPORT_SYMBOL(tape_generic_pm_suspend); EXPORT_SYMBOL(tape_put_device); EXPORT_SYMBOL(tape_get_device); EXPORT_SYMBOL(tape_state_verbose); diff --git a/drivers/s390/char/vmur.c b/drivers/s390/char/vmur.c index cbde65ab2170..1bbf27b98cf6 100644 --- a/drivers/s390/char/vmur.c +++ b/drivers/s390/char/vmur.c @@ -62,7 +62,6 @@ static int ur_probe(struct ccw_device *cdev); static void ur_remove(struct ccw_device *cdev); static int ur_set_online(struct ccw_device *cdev); static int ur_set_offline(struct ccw_device *cdev); -static int ur_pm_suspend(struct ccw_device *cdev); static struct ccw_driver ur_driver = { .driver = { @@ -74,7 +73,6 @@ static struct ccw_driver ur_driver = { .remove = ur_remove, .set_online = ur_set_online, .set_offline = ur_set_offline, - .freeze = ur_pm_suspend, .int_class = IRQIO_VMR, }; @@ -164,28 +162,6 @@ static void urdev_put(struct urdev *urd) urdev_free(urd); } -/* - * State and contents of ur devices can be changed by class D users issuing - * CP commands such as PURGE or TRANSFER, while the Linux guest is suspended. - * Also the Linux guest might be logged off, which causes all active spool - * files to be closed. - * So we cannot guarantee that spool files are still the same when the Linux - * guest is resumed. In order to avoid unpredictable results at resume time - * we simply refuse to suspend if a ur device node is open. - */ -static int ur_pm_suspend(struct ccw_device *cdev) -{ - struct urdev *urd = dev_get_drvdata(&cdev->dev); - - TRACE("ur_pm_suspend: cdev=%p\n", cdev); - if (urd->open_flag) { - pr_err("Unit record device %s is busy, %s refusing to " - "suspend.\n", dev_name(&cdev->dev), ur_banner); - return -EBUSY; - } - return 0; -} - /* * Low-level functions to do I/O to a ur device. * alloc_chan_prog diff --git a/drivers/s390/cio/chsc_sch.c b/drivers/s390/cio/chsc_sch.c index 8f080d3fd380..c42405c620b5 100644 --- a/drivers/s390/cio/chsc_sch.c +++ b/drivers/s390/cio/chsc_sch.c @@ -120,31 +120,6 @@ static void chsc_subchannel_shutdown(struct subchannel *sch) cio_disable_subchannel(sch); } -static int chsc_subchannel_prepare(struct subchannel *sch) -{ - int cc; - struct schib schib; - /* - * Don't allow suspend while the subchannel is not idle - * since we don't have a way to clear the subchannel and - * cannot disable it with a request running. - */ - cc = stsch(sch->schid, &schib); - if (!cc && scsw_stctl(&schib.scsw)) - return -EAGAIN; - return 0; -} - -static int chsc_subchannel_freeze(struct subchannel *sch) -{ - return cio_disable_subchannel(sch); -} - -static int chsc_subchannel_restore(struct subchannel *sch) -{ - return cio_enable_subchannel(sch, (u32)(unsigned long)sch); -} - static struct css_device_id chsc_subchannel_ids[] = { { .match_flags = 0x1, .type =SUBCHANNEL_TYPE_CHSC, }, { /* end of list */ }, @@ -161,10 +136,6 @@ static struct css_driver chsc_subchannel_driver = { .probe = chsc_subchannel_probe, .remove = chsc_subchannel_remove, .shutdown = chsc_subchannel_shutdown, - .prepare = chsc_subchannel_prepare, - .freeze = chsc_subchannel_freeze, - .thaw = chsc_subchannel_restore, - .restore = chsc_subchannel_restore, }; static int __init chsc_init_dbfs(void) diff --git a/drivers/s390/cio/cmf.c b/drivers/s390/cio/cmf.c index 72dd2471ec1e..b7b590646d58 100644 --- a/drivers/s390/cio/cmf.c +++ b/drivers/s390/cio/cmf.c @@ -1109,11 +1109,6 @@ static ssize_t cmb_enable_store(struct device *dev, } DEVICE_ATTR_RW(cmb_enable); -int ccw_set_cmf(struct ccw_device *cdev, int enable) -{ - return cmbops->set(cdev, enable ? 2 : 0); -} - /** * enable_cmf() - switch on the channel measurement for a specific device * @cdev: The ccw device to be enabled diff --git a/drivers/s390/cio/css.c b/drivers/s390/cio/css.c index cca1a7c4bb33..94c6470de635 100644 --- a/drivers/s390/cio/css.c +++ b/drivers/s390/cio/css.c @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -1044,59 +1043,6 @@ static struct notifier_block css_reboot_notifier = { .notifier_call = css_reboot_event, }; -/* - * Since the css devices are neither on a bus nor have a class - * nor have a special device type, we cannot stop/restart channel - * path measurements via the normal suspend/resume callbacks, but have - * to use notifiers. - */ -static int css_power_event(struct notifier_block *this, unsigned long event, - void *ptr) -{ - struct channel_subsystem *css; - int ret; - - switch (event) { - case PM_HIBERNATION_PREPARE: - case PM_SUSPEND_PREPARE: - ret = NOTIFY_DONE; - for_each_css(css) { - mutex_lock(&css->mutex); - if (!css->cm_enabled) { - mutex_unlock(&css->mutex); - continue; - } - ret = __chsc_do_secm(css, 0); - ret = notifier_from_errno(ret); - mutex_unlock(&css->mutex); - } - break; - case PM_POST_HIBERNATION: - case PM_POST_SUSPEND: - ret = NOTIFY_DONE; - for_each_css(css) { - mutex_lock(&css->mutex); - if (!css->cm_enabled) { - mutex_unlock(&css->mutex); - continue; - } - ret = __chsc_do_secm(css, 1); - ret = notifier_from_errno(ret); - mutex_unlock(&css->mutex); - } - /* search for subchannels, which appeared during hibernation */ - css_schedule_reprobe(); - break; - default: - ret = NOTIFY_DONE; - } - return ret; - -} -static struct notifier_block css_power_notifier = { - .notifier_call = css_power_event, -}; - #define CIO_DMA_GFP (GFP_KERNEL | __GFP_ZERO) static struct gen_pool *cio_dma_pool; @@ -1242,12 +1188,9 @@ static int __init css_bus_init(void) ret = register_reboot_notifier(&css_reboot_notifier); if (ret) goto out_unregister; - ret = register_pm_notifier(&css_power_notifier); - if (ret) - goto out_unregister_rn; ret = cio_dma_pool_init(); if (ret) - goto out_unregister_pmn; + goto out_unregister_rn; airq_init(); css_init_done = 1; @@ -1255,8 +1198,6 @@ static int __init css_bus_init(void) isc_register(IO_SCH_ISC); return 0; -out_unregister_pmn: - unregister_pm_notifier(&css_power_notifier); out_unregister_rn: unregister_reboot_notifier(&css_reboot_notifier); out_unregister: @@ -1456,74 +1397,6 @@ static int css_uevent(struct device *dev, struct kobj_uevent_env *env) return ret; } -static int css_pm_prepare(struct device *dev) -{ - struct subchannel *sch = to_subchannel(dev); - struct css_driver *drv; - - if (mutex_is_locked(&sch->reg_mutex)) - return -EAGAIN; - if (!sch->dev.driver) - return 0; - drv = to_cssdriver(sch->dev.driver); - /* Notify drivers that they may not register children. */ - return drv->prepare ? drv->prepare(sch) : 0; -} - -static void css_pm_complete(struct device *dev) -{ - struct subchannel *sch = to_subchannel(dev); - struct css_driver *drv; - - if (!sch->dev.driver) - return; - drv = to_cssdriver(sch->dev.driver); - if (drv->complete) - drv->complete(sch); -} - -static int css_pm_freeze(struct device *dev) -{ - struct subchannel *sch = to_subchannel(dev); - struct css_driver *drv; - - if (!sch->dev.driver) - return 0; - drv = to_cssdriver(sch->dev.driver); - return drv->freeze ? drv->freeze(sch) : 0; -} - -static int css_pm_thaw(struct device *dev) -{ - struct subchannel *sch = to_subchannel(dev); - struct css_driver *drv; - - if (!sch->dev.driver) - return 0; - drv = to_cssdriver(sch->dev.driver); - return drv->thaw ? drv->thaw(sch) : 0; -} - -static int css_pm_restore(struct device *dev) -{ - struct subchannel *sch = to_subchannel(dev); - struct css_driver *drv; - - css_update_ssd_info(sch); - if (!sch->dev.driver) - return 0; - drv = to_cssdriver(sch->dev.driver); - return drv->restore ? drv->restore(sch) : 0; -} - -static const struct dev_pm_ops css_pm_ops = { - .prepare = css_pm_prepare, - .complete = css_pm_complete, - .freeze = css_pm_freeze, - .thaw = css_pm_thaw, - .restore = css_pm_restore, -}; - static struct bus_type css_bus_type = { .name = "css", .match = css_bus_match, @@ -1531,7 +1404,6 @@ static struct bus_type css_bus_type = { .remove = css_remove, .shutdown = css_shutdown, .uevent = css_uevent, - .pm = &css_pm_ops, }; /** diff --git a/drivers/s390/cio/css.h b/drivers/s390/cio/css.h index 3f322ea0f498..2eddfc47f687 100644 --- a/drivers/s390/cio/css.h +++ b/drivers/s390/cio/css.h @@ -72,11 +72,6 @@ struct chp_link; * @probe: function called on probe * @remove: function called on remove * @shutdown: called at device shutdown - * @prepare: prepare for pm state transition - * @complete: undo work done in @prepare - * @freeze: callback for freezing during hibernation snapshotting - * @thaw: undo work done in @freeze - * @restore: callback for restoring after hibernation * @settle: wait for asynchronous work to finish */ struct css_driver { @@ -88,11 +83,6 @@ struct css_driver { int (*probe)(struct subchannel *); int (*remove)(struct subchannel *); void (*shutdown)(struct subchannel *); - int (*prepare) (struct subchannel *); - void (*complete) (struct subchannel *); - int (*freeze)(struct subchannel *); - int (*thaw) (struct subchannel *); - int (*restore)(struct subchannel *); int (*settle)(void); }; diff --git a/drivers/s390/cio/device.c b/drivers/s390/cio/device.c index b29fe8d50baf..e0005a4fc978 100644 --- a/drivers/s390/cio/device.c +++ b/drivers/s390/cio/device.c @@ -149,19 +149,6 @@ static struct css_device_id io_subchannel_ids[] = { { /* end of list */ }, }; -static int io_subchannel_prepare(struct subchannel *sch) -{ - struct ccw_device *cdev; - /* - * Don't allow suspend while a ccw device registration - * is still outstanding. - */ - cdev = sch_get_cdev(sch); - if (cdev && !device_is_registered(&cdev->dev)) - return -EAGAIN; - return 0; -} - static int io_subchannel_settle(void) { int ret; @@ -186,7 +173,6 @@ static struct css_driver io_subchannel_driver = { .probe = io_subchannel_probe, .remove = io_subchannel_remove, .shutdown = io_subchannel_shutdown, - .prepare = io_subchannel_prepare, .settle = io_subchannel_settle, }; @@ -1422,7 +1408,7 @@ static enum io_sch_action sch_get_action(struct subchannel *sch) } if (device_is_disconnected(cdev)) return IO_SCH_REPROBE; - if (cdev->online && !cdev->private->flags.resuming) + if (cdev->online) return IO_SCH_VERIFY; if (cdev->private->state == DEV_STATE_NOT_OPER) return IO_SCH_UNREG_ATTACH; @@ -1514,11 +1500,6 @@ static int io_subchannel_sch_event(struct subchannel *sch, int process) break; case IO_SCH_UNREG_ATTACH: spin_lock_irqsave(sch->lock, flags); - if (cdev->private->flags.resuming) { - /* Device will be handled later. */ - rc = 0; - goto out_unlock; - } sch_set_cdev(sch, NULL); spin_unlock_irqrestore(sch->lock, flags); /* Unregister ccw device. */ @@ -1531,7 +1512,7 @@ static int io_subchannel_sch_event(struct subchannel *sch, int process) switch (action) { case IO_SCH_ORPH_UNREG: case IO_SCH_UNREG: - if (!cdev || !cdev->private->flags.resuming) + if (!cdev) css_sch_device_unregister(sch); break; case IO_SCH_ORPH_ATTACH: @@ -1664,10 +1645,10 @@ void __init ccw_device_destroy_console(struct ccw_device *cdev) struct io_subchannel_private *io_priv = to_io_private(sch); set_io_private(sch, NULL); - put_device(&sch->dev); - put_device(&cdev->dev); dma_free_coherent(&sch->dev, sizeof(*io_priv->dma_area), io_priv->dma_area, io_priv->dma_area_dma); + put_device(&sch->dev); + put_device(&cdev->dev); kfree(io_priv); } @@ -1690,14 +1671,6 @@ void ccw_device_wait_idle(struct ccw_device *cdev) udelay_simple(100); } } - -static int ccw_device_pm_restore(struct device *dev); - -int ccw_device_force_console(struct ccw_device *cdev) -{ - return ccw_device_pm_restore(&cdev->dev); -} -EXPORT_SYMBOL_GPL(ccw_device_force_console); #endif /** @@ -1798,235 +1771,6 @@ static void ccw_device_shutdown(struct device *dev) __disable_cmf(cdev); } -static int ccw_device_pm_prepare(struct device *dev) -{ - struct ccw_device *cdev = to_ccwdev(dev); - - if (work_pending(&cdev->private->todo_work)) - return -EAGAIN; - /* Fail while device is being set online/offline. */ - if (atomic_read(&cdev->private->onoff)) - return -EAGAIN; - - if (cdev->online && cdev->drv && cdev->drv->prepare) - return cdev->drv->prepare(cdev); - - return 0; -} - -static void ccw_device_pm_complete(struct device *dev) -{ - struct ccw_device *cdev = to_ccwdev(dev); - - if (cdev->online && cdev->drv && cdev->drv->complete) - cdev->drv->complete(cdev); -} - -static int ccw_device_pm_freeze(struct device *dev) -{ - struct ccw_device *cdev = to_ccwdev(dev); - struct subchannel *sch = to_subchannel(cdev->dev.parent); - int ret, cm_enabled; - - /* Fail suspend while device is in transistional state. */ - if (!dev_fsm_final_state(cdev)) - return -EAGAIN; - if (!cdev->online) - return 0; - if (cdev->drv && cdev->drv->freeze) { - ret = cdev->drv->freeze(cdev); - if (ret) - return ret; - } - - spin_lock_irq(sch->lock); - cm_enabled = cdev->private->cmb != NULL; - spin_unlock_irq(sch->lock); - if (cm_enabled) { - /* Don't have the css write on memory. */ - ret = ccw_set_cmf(cdev, 0); - if (ret) - return ret; - } - /* From here on, disallow device driver I/O. */ - spin_lock_irq(sch->lock); - ret = cio_disable_subchannel(sch); - spin_unlock_irq(sch->lock); - - return ret; -} - -static int ccw_device_pm_thaw(struct device *dev) -{ - struct ccw_device *cdev = to_ccwdev(dev); - struct subchannel *sch = to_subchannel(cdev->dev.parent); - int ret, cm_enabled; - - if (!cdev->online) - return 0; - - spin_lock_irq(sch->lock); - /* Allow device driver I/O again. */ - ret = cio_enable_subchannel(sch, (u32)(addr_t)sch); - cm_enabled = cdev->private->cmb != NULL; - spin_unlock_irq(sch->lock); - if (ret) - return ret; - - if (cm_enabled) { - ret = ccw_set_cmf(cdev, 1); - if (ret) - return ret; - } - - if (cdev->drv && cdev->drv->thaw) - ret = cdev->drv->thaw(cdev); - - return ret; -} - -static void __ccw_device_pm_restore(struct ccw_device *cdev) -{ - struct subchannel *sch = to_subchannel(cdev->dev.parent); - - spin_lock_irq(sch->lock); - if (cio_is_console(sch->schid)) { - cio_enable_subchannel(sch, (u32)(addr_t)sch); - goto out_unlock; - } - /* - * While we were sleeping, devices may have gone or become - * available again. Kick re-detection. - */ - cdev->private->flags.resuming = 1; - cdev->private->path_new_mask = LPM_ANYPATH; - css_sched_sch_todo(sch, SCH_TODO_EVAL); - spin_unlock_irq(sch->lock); - css_wait_for_slow_path(); - - /* cdev may have been moved to a different subchannel. */ - sch = to_subchannel(cdev->dev.parent); - spin_lock_irq(sch->lock); - if (cdev->private->state != DEV_STATE_ONLINE && - cdev->private->state != DEV_STATE_OFFLINE) - goto out_unlock; - - ccw_device_recognition(cdev); - spin_unlock_irq(sch->lock); - wait_event(cdev->private->wait_q, dev_fsm_final_state(cdev) || - cdev->private->state == DEV_STATE_DISCONNECTED); - spin_lock_irq(sch->lock); - -out_unlock: - cdev->private->flags.resuming = 0; - spin_unlock_irq(sch->lock); -} - -static int resume_handle_boxed(struct ccw_device *cdev) -{ - cdev->private->state = DEV_STATE_BOXED; - if (ccw_device_notify(cdev, CIO_BOXED) == NOTIFY_OK) - return 0; - ccw_device_sched_todo(cdev, CDEV_TODO_UNREG); - return -ENODEV; -} - -static int resume_handle_disc(struct ccw_device *cdev) -{ - cdev->private->state = DEV_STATE_DISCONNECTED; - if (ccw_device_notify(cdev, CIO_GONE) == NOTIFY_OK) - return 0; - ccw_device_sched_todo(cdev, CDEV_TODO_UNREG); - return -ENODEV; -} - -static int ccw_device_pm_restore(struct device *dev) -{ - struct ccw_device *cdev = to_ccwdev(dev); - struct subchannel *sch; - int ret = 0; - - __ccw_device_pm_restore(cdev); - sch = to_subchannel(cdev->dev.parent); - spin_lock_irq(sch->lock); - if (cio_is_console(sch->schid)) - goto out_restore; - - /* check recognition results */ - switch (cdev->private->state) { - case DEV_STATE_OFFLINE: - case DEV_STATE_ONLINE: - cdev->private->flags.donotify = 0; - break; - case DEV_STATE_BOXED: - ret = resume_handle_boxed(cdev); - if (ret) - goto out_unlock; - goto out_restore; - default: - ret = resume_handle_disc(cdev); - if (ret) - goto out_unlock; - goto out_restore; - } - /* check if the device type has changed */ - if (!ccw_device_test_sense_data(cdev)) { - ccw_device_update_sense_data(cdev); - ccw_device_sched_todo(cdev, CDEV_TODO_REBIND); - ret = -ENODEV; - goto out_unlock; - } - if (!cdev->online) - goto out_unlock; - - if (ccw_device_online(cdev)) { - ret = resume_handle_disc(cdev); - if (ret) - goto out_unlock; - goto out_restore; - } - spin_unlock_irq(sch->lock); - wait_event(cdev->private->wait_q, dev_fsm_final_state(cdev)); - spin_lock_irq(sch->lock); - - if (ccw_device_notify(cdev, CIO_OPER) == NOTIFY_BAD) { - ccw_device_sched_todo(cdev, CDEV_TODO_UNREG); - ret = -ENODEV; - goto out_unlock; - } - - /* reenable cmf, if needed */ - if (cdev->private->cmb) { - spin_unlock_irq(sch->lock); - ret = ccw_set_cmf(cdev, 1); - spin_lock_irq(sch->lock); - if (ret) { - CIO_MSG_EVENT(2, "resume: cdev 0.%x.%04x: cmf failed " - "(rc=%d)\n", cdev->private->dev_id.ssid, - cdev->private->dev_id.devno, ret); - ret = 0; - } - } - -out_restore: - spin_unlock_irq(sch->lock); - if (cdev->online && cdev->drv && cdev->drv->restore) - ret = cdev->drv->restore(cdev); - return ret; - -out_unlock: - spin_unlock_irq(sch->lock); - return ret; -} - -static const struct dev_pm_ops ccw_pm_ops = { - .prepare = ccw_device_pm_prepare, - .complete = ccw_device_pm_complete, - .freeze = ccw_device_pm_freeze, - .thaw = ccw_device_pm_thaw, - .restore = ccw_device_pm_restore, -}; - static struct bus_type ccw_bus_type = { .name = "ccw", .match = ccw_bus_match, @@ -2034,7 +1778,6 @@ static struct bus_type ccw_bus_type = { .probe = ccw_device_probe, .remove = ccw_device_remove, .shutdown = ccw_device_shutdown, - .pm = &ccw_pm_ops, }; /** diff --git a/drivers/s390/cio/device.h b/drivers/s390/cio/device.h index 853b6a8ca095..24b2fce69590 100644 --- a/drivers/s390/cio/device.h +++ b/drivers/s390/cio/device.h @@ -143,6 +143,5 @@ void retry_set_schib(struct ccw_device *cdev); void cmf_retry_copy_block(struct ccw_device *); int cmf_reenable(struct ccw_device *); void cmf_reactivate(void); -int ccw_set_cmf(struct ccw_device *cdev, int enable); extern struct device_attribute dev_attr_cmb_enable; #endif diff --git a/drivers/s390/cio/device_fsm.c b/drivers/s390/cio/device_fsm.c index 8fc267324ebb..6420b197bb05 100644 --- a/drivers/s390/cio/device_fsm.c +++ b/drivers/s390/cio/device_fsm.c @@ -224,12 +224,6 @@ ccw_device_recog_done(struct ccw_device *cdev, int state) wake_up(&cdev->private->wait_q); return; } - if (cdev->private->flags.resuming) { - cdev->private->state = state; - cdev->private->flags.recog_done = 1; - wake_up(&cdev->private->wait_q); - return; - } switch (state) { case DEV_STATE_NOT_OPER: break; diff --git a/drivers/s390/cio/eadm_sch.c b/drivers/s390/cio/eadm_sch.c index 53468ae64b99..c8964e0a23e7 100644 --- a/drivers/s390/cio/eadm_sch.c +++ b/drivers/s390/cio/eadm_sch.c @@ -306,16 +306,6 @@ static void eadm_subchannel_shutdown(struct subchannel *sch) eadm_quiesce(sch); } -static int eadm_subchannel_freeze(struct subchannel *sch) -{ - return cio_disable_subchannel(sch); -} - -static int eadm_subchannel_restore(struct subchannel *sch) -{ - return cio_enable_subchannel(sch, (u32)(unsigned long)sch); -} - /** * eadm_subchannel_sch_event - process subchannel event * @sch: subchannel @@ -369,9 +359,6 @@ static struct css_driver eadm_subchannel_driver = { .remove = eadm_subchannel_remove, .shutdown = eadm_subchannel_shutdown, .sch_event = eadm_subchannel_sch_event, - .freeze = eadm_subchannel_freeze, - .thaw = eadm_subchannel_restore, - .restore = eadm_subchannel_restore, }; static int __init eadm_sch_init(void) diff --git a/drivers/s390/cio/io_sch.h b/drivers/s390/cio/io_sch.h index c03b4a19974e..85a11c1836e5 100644 --- a/drivers/s390/cio/io_sch.h +++ b/drivers/s390/cio/io_sch.h @@ -160,7 +160,6 @@ struct ccw_device_private { unsigned int donotify:1; /* call notify function */ unsigned int recog_done:1; /* dev. recog. complete */ unsigned int fake_irb:2; /* deliver faked irb */ - unsigned int resuming:1; /* recognition while resume */ unsigned int pgroup:1; /* pathgroup is set up */ unsigned int mpath:1; /* multipathing is set up */ unsigned int pgid_unknown:1;/* unknown pgid state */ diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index ef738b42a092..2758d05a802d 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -1,11 +1,12 @@ // SPDX-License-Identifier: GPL-2.0+ /* - * Copyright IBM Corp. 2006, 2012 + * Copyright IBM Corp. 2006, 2020 * Author(s): Cornelia Huck * Martin Schwidefsky * Ralph Wuerthner * Felix Beck * Holger Dengler + * Harald Freudenberger * * Adjunct processor bus. */ @@ -73,6 +74,12 @@ EXPORT_SYMBOL(ap_perms); DEFINE_MUTEX(ap_perms_mutex); EXPORT_SYMBOL(ap_perms_mutex); +/* # of bus scans since init */ +static atomic64_t ap_scan_bus_count; + +/* completion for initial APQN bindings complete */ +static DECLARE_COMPLETION(ap_init_apqn_bindings_complete); + static struct ap_config_info *ap_qci_info; /* @@ -577,23 +584,125 @@ static int ap_bus_match(struct device *dev, struct device_driver *drv) */ static int ap_uevent(struct device *dev, struct kobj_uevent_env *env) { + int rc; struct ap_device *ap_dev = to_ap_dev(dev); - int retval = 0; - if (!ap_dev) - return -ENODEV; + /* Uevents from ap bus core don't need extensions to the env */ + if (dev == ap_root_device) + return 0; /* Set up DEV_TYPE environment variable. */ - retval = add_uevent_var(env, "DEV_TYPE=%04X", ap_dev->device_type); - if (retval) - return retval; + rc = add_uevent_var(env, "DEV_TYPE=%04X", ap_dev->device_type); + if (rc) + return rc; /* Add MODALIAS= */ - retval = add_uevent_var(env, "MODALIAS=ap:t%02X", ap_dev->device_type); + rc = add_uevent_var(env, "MODALIAS=ap:t%02X", ap_dev->device_type); + if (rc) + return rc; - return retval; + return 0; } +static void ap_send_init_scan_done_uevent(void) +{ + char *envp[] = { "INITSCAN=done", NULL }; + + kobject_uevent_env(&ap_root_device->kobj, KOBJ_CHANGE, envp); +} + +static void ap_send_bindings_complete_uevent(void) +{ + char *envp[] = { "BINDINGS=complete", NULL }; + + kobject_uevent_env(&ap_root_device->kobj, KOBJ_CHANGE, envp); +} + +/* + * calc # of bound APQNs + */ + +struct __ap_calc_ctrs { + unsigned int apqns; + unsigned int bound; +}; + +static int __ap_calc_helper(struct device *dev, void *arg) +{ + struct __ap_calc_ctrs *pctrs = (struct __ap_calc_ctrs *) arg; + + if (is_queue_dev(dev)) { + pctrs->apqns++; + if ((to_ap_dev(dev))->drv) + pctrs->bound++; + } + + return 0; +} + +static void ap_calc_bound_apqns(unsigned int *apqns, unsigned int *bound) +{ + struct __ap_calc_ctrs ctrs; + + memset(&ctrs, 0, sizeof(ctrs)); + bus_for_each_dev(&ap_bus_type, NULL, (void *) &ctrs, __ap_calc_helper); + + *apqns = ctrs.apqns; + *bound = ctrs.bound; +} + +/* + * After initial ap bus scan do check if all existing APQNs are + * bound to device drivers. + */ +static void ap_check_bindings_complete(void) +{ + unsigned int apqns, bound; + + if (atomic64_read(&ap_scan_bus_count) >= 1) { + ap_calc_bound_apqns(&apqns, &bound); + if (bound == apqns) { + if (!completion_done(&ap_init_apqn_bindings_complete)) { + complete_all(&ap_init_apqn_bindings_complete); + AP_DBF(DBF_INFO, "%s complete\n", __func__); + } + ap_send_bindings_complete_uevent(); + } + } +} + +/* + * Interface to wait for the AP bus to have done one initial ap bus + * scan and all detected APQNs have been bound to device drivers. + * If these both conditions are not fulfilled, this function blocks + * on a condition with wait_for_completion_interruptible_timeout(). + * If these both conditions are fulfilled (before the timeout hits) + * the return value is 0. If the timeout (in jiffies) hits instead + * -ETIME is returned. On failures negative return values are + * returned to the caller. + */ +int ap_wait_init_apqn_bindings_complete(unsigned long timeout) +{ + long l; + + if (completion_done(&ap_init_apqn_bindings_complete)) + return 0; + + if (timeout) + l = wait_for_completion_interruptible_timeout( + &ap_init_apqn_bindings_complete, timeout); + else + l = wait_for_completion_interruptible( + &ap_init_apqn_bindings_complete); + if (l < 0) + return l == -ERESTARTSYS ? -EINTR : l; + else if (l == 0 && timeout) + return -ETIME; + + return 0; +} +EXPORT_SYMBOL(ap_wait_init_apqn_bindings_complete); + static int __ap_queue_devices_with_id_unregister(struct device *dev, void *data) { if (is_queue_dev(dev) && @@ -602,12 +711,6 @@ static int __ap_queue_devices_with_id_unregister(struct device *dev, void *data) return 0; } -static struct bus_type ap_bus_type = { - .name = "ap", - .match = &ap_bus_match, - .uevent = &ap_uevent, -}; - static int __ap_revise_reserved(struct device *dev, void *dummy) { int rc, card, queue, devres, drvres; @@ -719,7 +822,8 @@ static int ap_device_probe(struct device *dev) hash_del(&to_ap_queue(dev)->hnode); spin_unlock_bh(&ap_queues_lock); ap_dev->drv = NULL; - } + } else + ap_check_bindings_complete(); out: if (rc) @@ -749,6 +853,7 @@ static int ap_device_remove(struct device *dev) if (is_queue_dev(dev)) hash_del(&to_ap_queue(dev)->hnode); spin_unlock_bh(&ap_queues_lock); + ap_dev->drv = NULL; put_device(dev); @@ -1166,21 +1271,55 @@ static ssize_t aqmask_store(struct bus_type *bus, const char *buf, static BUS_ATTR_RW(aqmask); -static struct bus_attribute *const ap_bus_attrs[] = { - &bus_attr_ap_domain, - &bus_attr_ap_control_domain_mask, - &bus_attr_ap_usage_domain_mask, - &bus_attr_ap_adapter_mask, - &bus_attr_config_time, - &bus_attr_poll_thread, - &bus_attr_ap_interrupts, - &bus_attr_poll_timeout, - &bus_attr_ap_max_domain_id, - &bus_attr_ap_max_adapter_id, - &bus_attr_apmask, - &bus_attr_aqmask, +static ssize_t scans_show(struct bus_type *bus, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%llu\n", + atomic64_read(&ap_scan_bus_count)); +} + +static BUS_ATTR_RO(scans); + +static ssize_t bindings_show(struct bus_type *bus, char *buf) +{ + int rc; + unsigned int apqns, n; + + ap_calc_bound_apqns(&apqns, &n); + if (atomic64_read(&ap_scan_bus_count) >= 1 && n == apqns) + rc = scnprintf(buf, PAGE_SIZE, "%u/%u (complete)\n", n, apqns); + else + rc = scnprintf(buf, PAGE_SIZE, "%u/%u\n", n, apqns); + + return rc; +} + +static BUS_ATTR_RO(bindings); + +static struct attribute *ap_bus_attrs[] = { + &bus_attr_ap_domain.attr, + &bus_attr_ap_control_domain_mask.attr, + &bus_attr_ap_usage_domain_mask.attr, + &bus_attr_ap_adapter_mask.attr, + &bus_attr_config_time.attr, + &bus_attr_poll_thread.attr, + &bus_attr_ap_interrupts.attr, + &bus_attr_poll_timeout.attr, + &bus_attr_ap_max_domain_id.attr, + &bus_attr_ap_max_adapter_id.attr, + &bus_attr_apmask.attr, + &bus_attr_aqmask.attr, + &bus_attr_scans.attr, + &bus_attr_bindings.attr, NULL, }; +ATTRIBUTE_GROUPS(ap_bus); + +static struct bus_type ap_bus_type = { + .name = "ap", + .bus_groups = ap_bus_groups, + .match = &ap_bus_match, + .uevent = &ap_uevent, +}; /** * ap_select_domain(): Select an AP domain if possible and we haven't @@ -1608,6 +1747,12 @@ static void ap_scan_bus(struct work_struct *unused) ap_domain_index); } + if (atomic64_inc_return(&ap_scan_bus_count) == 1) { + AP_DBF(DBF_DEBUG, "%s init scan complete\n", __func__); + ap_send_init_scan_done_uevent(); + ap_check_bindings_complete(); + } + mod_timer(&ap_config_timer, jiffies + ap_config_time * HZ); } @@ -1655,7 +1800,7 @@ static void __init ap_perms_init(void) */ static int __init ap_module_init(void) { - int rc, i; + int rc; rc = ap_debug_init(); if (rc) @@ -1694,17 +1839,13 @@ static int __init ap_module_init(void) rc = bus_register(&ap_bus_type); if (rc) goto out; - for (i = 0; ap_bus_attrs[i]; i++) { - rc = bus_create_file(&ap_bus_type, ap_bus_attrs[i]); - if (rc) - goto out_bus; - } /* Create /sys/devices/ap. */ ap_root_device = root_device_register("ap"); rc = PTR_ERR_OR_ZERO(ap_root_device); if (rc) goto out_bus; + ap_root_device->bus = &ap_bus_type; /* Setup the AP bus rescan timer. */ timer_setup(&ap_config_timer, ap_config_timeout, 0); @@ -1733,8 +1874,6 @@ out_work: hrtimer_cancel(&ap_poll_timer); root_device_unregister(ap_root_device); out_bus: - while (i--) - bus_remove_file(&ap_bus_type, ap_bus_attrs[i]); bus_unregister(&ap_bus_type); out: if (ap_using_interrupts()) diff --git a/drivers/s390/crypto/ap_bus.h b/drivers/s390/crypto/ap_bus.h index 5029b80132aa..472efd3a755c 100644 --- a/drivers/s390/crypto/ap_bus.h +++ b/drivers/s390/crypto/ap_bus.h @@ -350,4 +350,16 @@ int ap_parse_mask_str(const char *str, unsigned long *bitmap, int bits, struct mutex *lock); +/* + * Interface to wait for the AP bus to have done one initial ap bus + * scan and all detected APQNs have been bound to device drivers. + * If these both conditions are not fulfilled, this function blocks + * on a condition with wait_for_completion_killable_timeout(). + * If these both conditions are fulfilled (before the timeout hits) + * the return value is 0. If the timeout (in jiffies) hits instead + * -ETIME is returned. On failures negative return values are + * returned to the caller. + */ +int ap_wait_init_apqn_bindings_complete(unsigned long timeout); + #endif /* _AP_BUS_H_ */ diff --git a/drivers/s390/crypto/pkey_api.c b/drivers/s390/crypto/pkey_api.c index dd84995049b9..cf23ce1b1146 100644 --- a/drivers/s390/crypto/pkey_api.c +++ b/drivers/s390/crypto/pkey_api.c @@ -150,6 +150,8 @@ static int pkey_skey2pkey(const u8 *key, struct pkey_protkey *pkey) u16 cardnr, domain; struct keytoken_header *hdr = (struct keytoken_header *)key; + zcrypt_wait_api_operational(); + /* * The cca_xxx2protkey call may fail when a card has been * addressed where the master key was changed after last fetch @@ -197,6 +199,8 @@ static int pkey_clr2ep11key(const u8 *clrkey, size_t clrkeylen, u16 card, dom; u32 nr_apqns, *apqns = NULL; + zcrypt_wait_api_operational(); + /* build a list of apqns suitable for ep11 keys with cpacf support */ rc = ep11_findcard2(&apqns, &nr_apqns, 0xFFFF, 0xFFFF, ZCRYPT_CEX7, EP11_API_V, NULL); @@ -230,6 +234,8 @@ static int pkey_ep11key2pkey(const u8 *key, struct pkey_protkey *pkey) u32 nr_apqns, *apqns = NULL; struct ep11keyblob *kb = (struct ep11keyblob *) key; + zcrypt_wait_api_operational(); + /* build a list of apqns suitable for this key */ rc = ep11_findcard2(&apqns, &nr_apqns, 0xFFFF, 0xFFFF, ZCRYPT_CEX7, EP11_API_V, kb->wkvp); @@ -436,6 +442,7 @@ static int pkey_nonccatok2pkey(const u8 *key, u32 keylen, if (rc == 0) break; /* PCKMO failed, so try the CCA secure key way */ + zcrypt_wait_api_operational(); rc = cca_clr2seckey(0xFFFF, 0xFFFF, t->keytype, ckey.clrkey, tmpbuf); if (rc == 0) @@ -625,6 +632,8 @@ static int pkey_clr2seckey2(const struct pkey_apqn *apqns, size_t nr_apqns, return -EINVAL; } + zcrypt_wait_api_operational(); + /* simple try all apqns from the list */ for (i = 0, rc = -ENODEV; i < nr_apqns; i++) { card = apqns[i].card; @@ -801,6 +810,8 @@ static int pkey_keyblob2pkey2(const struct pkey_apqn *apqns, size_t nr_apqns, return -EINVAL; } + zcrypt_wait_api_operational(); + /* simple try all apqns from the list */ for (i = 0, rc = -ENODEV; i < nr_apqns; i++) { card = apqns[i].card; @@ -838,6 +849,8 @@ static int pkey_apqns4key(const u8 *key, size_t keylen, u32 flags, if (keylen < sizeof(struct keytoken_header) || flags == 0) return -EINVAL; + zcrypt_wait_api_operational(); + if (hdr->type == TOKTYPE_NON_CCA && (hdr->version == TOKVER_EP11_AES_WITH_HEADER || hdr->version == TOKVER_EP11_ECC_WITH_HEADER) @@ -941,6 +954,8 @@ static int pkey_apqns4keytype(enum pkey_key_type ktype, int rc; u32 _nr_apqns, *_apqns = NULL; + zcrypt_wait_api_operational(); + if (ktype == PKEY_TYPE_CCA_DATA || ktype == PKEY_TYPE_CCA_CIPHER) { u64 cur_mkvp = 0, old_mkvp = 0; int minhwtype = ZCRYPT_CEX3C; diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c index f60f9fb25214..10206e4498d0 100644 --- a/drivers/s390/crypto/zcrypt_api.c +++ b/drivers/s390/crypto/zcrypt_api.c @@ -1992,6 +1992,72 @@ void zcrypt_rng_device_remove(void) mutex_unlock(&zcrypt_rng_mutex); } +/* + * Wait until the zcrypt api is operational. + * The AP bus scan and the binding of ap devices to device drivers is + * an asynchronous job. This function waits until these initial jobs + * are done and so the zcrypt api should be ready to serve crypto + * requests - if there are resources available. The function uses an + * internal timeout of 60s. The very first caller will either wait for + * ap bus bindings complete or the timeout happens. This state will be + * remembered for further callers which will only be blocked until a + * decision is made (timeout or bindings complete). + * On timeout -ETIME is returned, on success the return value is 0. + */ +int zcrypt_wait_api_operational(void) +{ + static DEFINE_MUTEX(zcrypt_wait_api_lock); + static int zcrypt_wait_api_state; + int rc; + + rc = mutex_lock_interruptible(&zcrypt_wait_api_lock); + if (rc) + return rc; + + switch (zcrypt_wait_api_state) { + case 0: + /* initial state, invoke wait for the ap bus complete */ + rc = ap_wait_init_apqn_bindings_complete( + msecs_to_jiffies(60 * 1000)); + switch (rc) { + case 0: + /* ap bus bindings are complete */ + zcrypt_wait_api_state = 1; + break; + case -EINTR: + /* interrupted, go back to caller */ + break; + case -ETIME: + /* timeout */ + ZCRYPT_DBF(DBF_WARN, + "%s ap_wait_init_apqn_bindings_complete() returned with ETIME\n", + __func__); + zcrypt_wait_api_state = -ETIME; + break; + default: + /* other failure */ + ZCRYPT_DBF(DBF_DEBUG, + "%s ap_wait_init_apqn_bindings_complete() failure rc=%d\n", + __func__, rc); + break; + } + break; + case 1: + /* a previous caller already found ap bus bindings complete */ + rc = 0; + break; + default: + /* a previous caller had timeout or other failure */ + rc = zcrypt_wait_api_state; + break; + } + + mutex_unlock(&zcrypt_wait_api_lock); + + return rc; +} +EXPORT_SYMBOL(zcrypt_wait_api_operational); + int __init zcrypt_debug_init(void) { zcrypt_dbf_info = debug_register("zcrypt", 1, 1, diff --git a/drivers/s390/crypto/zcrypt_api.h b/drivers/s390/crypto/zcrypt_api.h index 51c0b8bdef50..16219efb2f61 100644 --- a/drivers/s390/crypto/zcrypt_api.h +++ b/drivers/s390/crypto/zcrypt_api.h @@ -162,6 +162,8 @@ void zcrypt_device_status_mask_ext(struct zcrypt_device_status_ext *devstatus); int zcrypt_device_status_ext(int card, int queue, struct zcrypt_device_status_ext *devstatus); +int zcrypt_wait_api_operational(void); + static inline unsigned long z_copy_from_user(bool userspace, void *to, const void __user *from, diff --git a/drivers/s390/scsi/zfcp_ccw.c b/drivers/s390/scsi/zfcp_ccw.c index d9fd0a41da64..bdf2cc1ea713 100644 --- a/drivers/s390/scsi/zfcp_ccw.c +++ b/drivers/s390/scsi/zfcp_ccw.c @@ -193,30 +193,6 @@ static int zfcp_ccw_set_online(struct ccw_device *cdev) return 0; } -/** - * zfcp_ccw_offline_sync - shut down adapter and wait for it to finish - * @cdev: pointer to belonging ccw device - * @set: Status flags to set. - * @tag: s390dbf trace record tag - * - * This function gets called by the common i/o layer and sets an adapter - * into state offline. - */ -static int zfcp_ccw_offline_sync(struct ccw_device *cdev, int set, char *tag) -{ - struct zfcp_adapter *adapter = zfcp_ccw_adapter_by_cdev(cdev); - - if (!adapter) - return 0; - - zfcp_erp_set_adapter_status(adapter, set); - zfcp_erp_adapter_shutdown(adapter, 0, tag); - zfcp_erp_wait(adapter); - - zfcp_ccw_adapter_put(adapter); - return 0; -} - /** * zfcp_ccw_set_offline - set_offline function of zfcp driver * @cdev: pointer to belonging ccw device @@ -226,7 +202,17 @@ static int zfcp_ccw_offline_sync(struct ccw_device *cdev, int set, char *tag) */ static int zfcp_ccw_set_offline(struct ccw_device *cdev) { - return zfcp_ccw_offline_sync(cdev, 0, "ccsoff1"); + struct zfcp_adapter *adapter = zfcp_ccw_adapter_by_cdev(cdev); + + if (!adapter) + return 0; + + zfcp_erp_set_adapter_status(adapter, 0); + zfcp_erp_adapter_shutdown(adapter, 0, "ccsoff1"); + zfcp_erp_wait(adapter); + + zfcp_ccw_adapter_put(adapter); + return 0; } /** @@ -246,11 +232,6 @@ static int zfcp_ccw_notify(struct ccw_device *cdev, int event) switch (event) { case CIO_GONE: - if (atomic_read(&adapter->status) & - ZFCP_STATUS_ADAPTER_SUSPENDED) { /* notification ignore */ - zfcp_dbf_hba_basic("ccnigo1", adapter); - break; - } dev_warn(&cdev->dev, "The FCP device has been detached\n"); zfcp_erp_adapter_shutdown(adapter, 0, "ccnoti1"); break; @@ -260,11 +241,6 @@ static int zfcp_ccw_notify(struct ccw_device *cdev, int event) zfcp_erp_adapter_shutdown(adapter, 0, "ccnoti2"); break; case CIO_OPER: - if (atomic_read(&adapter->status) & - ZFCP_STATUS_ADAPTER_SUSPENDED) { /* notification ignore */ - zfcp_dbf_hba_basic("ccniop1", adapter); - break; - } dev_info(&cdev->dev, "The FCP device is operational again\n"); zfcp_erp_set_adapter_status(adapter, ZFCP_STATUS_COMMON_RUNNING); @@ -300,28 +276,6 @@ static void zfcp_ccw_shutdown(struct ccw_device *cdev) zfcp_ccw_adapter_put(adapter); } -static int zfcp_ccw_suspend(struct ccw_device *cdev) -{ - zfcp_ccw_offline_sync(cdev, ZFCP_STATUS_ADAPTER_SUSPENDED, "ccsusp1"); - return 0; -} - -static int zfcp_ccw_thaw(struct ccw_device *cdev) -{ - /* trace records for thaw and final shutdown during suspend - can only be found in system dump until the end of suspend - but not after resume because it's based on the memory image - right after the very first suspend (freeze) callback */ - zfcp_ccw_activate(cdev, 0, "ccthaw1"); - return 0; -} - -static int zfcp_ccw_resume(struct ccw_device *cdev) -{ - zfcp_ccw_activate(cdev, ZFCP_STATUS_ADAPTER_SUSPENDED, "ccresu1"); - return 0; -} - struct ccw_driver zfcp_ccw_driver = { .driver = { .owner = THIS_MODULE, @@ -334,7 +288,4 @@ struct ccw_driver zfcp_ccw_driver = { .set_offline = zfcp_ccw_set_offline, .notify = zfcp_ccw_notify, .shutdown = zfcp_ccw_shutdown, - .freeze = zfcp_ccw_suspend, - .thaw = zfcp_ccw_thaw, - .restore = zfcp_ccw_resume, }; diff --git a/drivers/s390/scsi/zfcp_dbf.c b/drivers/s390/scsi/zfcp_dbf.c index 673e42defb91..ca473b368905 100644 --- a/drivers/s390/scsi/zfcp_dbf.c +++ b/drivers/s390/scsi/zfcp_dbf.c @@ -263,31 +263,6 @@ void zfcp_dbf_hba_def_err(struct zfcp_adapter *adapter, u64 req_id, u16 scount, spin_unlock_irqrestore(&dbf->pay_lock, flags); } -/** - * zfcp_dbf_hba_basic - trace event for basic adapter events - * @tag: identifier for event - * @adapter: pointer to struct zfcp_adapter - */ -void zfcp_dbf_hba_basic(char *tag, struct zfcp_adapter *adapter) -{ - struct zfcp_dbf *dbf = adapter->dbf; - struct zfcp_dbf_hba *rec = &dbf->hba_buf; - static int const level = 1; - unsigned long flags; - - if (unlikely(!debug_level_enabled(dbf->hba, level))) - return; - - spin_lock_irqsave(&dbf->hba_lock, flags); - memset(rec, 0, sizeof(*rec)); - - memcpy(rec->tag, tag, ZFCP_DBF_TAG_LEN); - rec->id = ZFCP_DBF_HBA_BASIC; - - debug_event(dbf->hba, level, rec, sizeof(*rec)); - spin_unlock_irqrestore(&dbf->hba_lock, flags); -} - static void zfcp_dbf_set_common(struct zfcp_dbf_rec *rec, struct zfcp_adapter *adapter, struct zfcp_port *port, diff --git a/drivers/s390/scsi/zfcp_def.h b/drivers/s390/scsi/zfcp_def.h index da8a5ceb615c..5069b555c6c1 100644 --- a/drivers/s390/scsi/zfcp_def.h +++ b/drivers/s390/scsi/zfcp_def.h @@ -70,7 +70,6 @@ #define ZFCP_STATUS_ADAPTER_SIOSL_ISSUED 0x00000004 #define ZFCP_STATUS_ADAPTER_XCONFIG_OK 0x00000008 #define ZFCP_STATUS_ADAPTER_HOST_CON_INIT 0x00000010 -#define ZFCP_STATUS_ADAPTER_SUSPENDED 0x00000040 #define ZFCP_STATUS_ADAPTER_ERP_PENDING 0x00000100 #define ZFCP_STATUS_ADAPTER_LINK_UNPLUGGED 0x00000200 #define ZFCP_STATUS_ADAPTER_DATA_DIV_ENABLED 0x00000400 diff --git a/drivers/s390/scsi/zfcp_ext.h b/drivers/s390/scsi/zfcp_ext.h index 3ef5d74331c3..fdac6350c579 100644 --- a/drivers/s390/scsi/zfcp_ext.h +++ b/drivers/s390/scsi/zfcp_ext.h @@ -49,7 +49,6 @@ extern void zfcp_dbf_hba_fsf_fces(char *tag, const struct zfcp_fsf_req *req, u32 fc_security_new); extern void zfcp_dbf_hba_bit_err(char *, struct zfcp_fsf_req *); extern void zfcp_dbf_hba_def_err(struct zfcp_adapter *, u64, u16, void **); -extern void zfcp_dbf_hba_basic(char *, struct zfcp_adapter *); extern void zfcp_dbf_san_req(char *, struct zfcp_fsf_req *, u32); extern void zfcp_dbf_san_res(char *, struct zfcp_fsf_req *); extern void zfcp_dbf_san_in_els(char *, struct zfcp_fsf_req *); diff --git a/drivers/tee/tee_core.c b/drivers/tee/tee_core.c index 6ade4a5c4840..480d294a23ab 100644 --- a/drivers/tee/tee_core.c +++ b/drivers/tee/tee_core.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include "tee_private.h" #define TEE_NUM_DEVICES 32 diff --git a/drivers/tty/serial/pmac_zilog.c b/drivers/tty/serial/pmac_zilog.c index 063484b22523..d6aef8a1f0a4 100644 --- a/drivers/tty/serial/pmac_zilog.c +++ b/drivers/tty/serial/pmac_zilog.c @@ -1693,22 +1693,26 @@ static int __init pmz_probe(void) #else +/* On PCI PowerMacs, pmz_probe() does an explicit search of the OpenFirmware + * tree to obtain the device_nodes needed to start the console before the + * macio driver. On Macs without OpenFirmware, global platform_devices take + * the place of those device_nodes. + */ extern struct platform_device scc_a_pdev, scc_b_pdev; static int __init pmz_init_port(struct uart_pmac_port *uap) { - struct resource *r_ports; - int irq; + struct resource *r_ports, *r_irq; r_ports = platform_get_resource(uap->pdev, IORESOURCE_MEM, 0); - irq = platform_get_irq(uap->pdev, 0); - if (!r_ports || irq <= 0) + r_irq = platform_get_resource(uap->pdev, IORESOURCE_IRQ, 0); + if (!r_ports || !r_irq) return -ENODEV; uap->port.mapbase = r_ports->start; uap->port.membase = (unsigned char __iomem *) r_ports->start; uap->port.iotype = UPIO_MEM; - uap->port.irq = irq; + uap->port.irq = r_irq->start; uap->port.uartclk = ZS_CLOCK; uap->port.fifosize = 1; uap->port.ops = &pmz_pops; diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index fa50e8936f5f..ac0b5fc30ea6 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -1246,7 +1246,7 @@ out_free_interp: set_binfmt(&elf_format); #ifdef ARCH_HAS_SETUP_ADDITIONAL_PAGES - retval = arch_setup_additional_pages(bprm, !!interpreter); + retval = ARCH_SETUP_ADDITIONAL_PAGES(bprm, elf_ex, !!interpreter); if (retval < 0) goto out; #endif /* ARCH_HAS_SETUP_ADDITIONAL_PAGES */ @@ -1307,7 +1307,7 @@ out_free_interp: #endif finalize_exec(bprm); - start_thread(regs, elf_entry, bprm->p); + START_THREAD(elf_ex, regs, elf_entry, bprm->p); retval = 0; out: return retval; diff --git a/fs/compat_binfmt_elf.c b/fs/compat_binfmt_elf.c index 2d24c765cbd7..2c557229696a 100644 --- a/fs/compat_binfmt_elf.c +++ b/fs/compat_binfmt_elf.c @@ -106,15 +106,25 @@ #endif #ifdef compat_start_thread -#undef start_thread -#define start_thread compat_start_thread +#define COMPAT_START_THREAD(ex, regs, new_ip, new_sp) \ + compat_start_thread(regs, new_ip, new_sp) #endif -#ifdef compat_arch_setup_additional_pages +#ifdef COMPAT_START_THREAD +#undef START_THREAD +#define START_THREAD COMPAT_START_THREAD +#endif + +#ifdef compat_arch_setup_additional_pages +#define COMPAT_ARCH_SETUP_ADDITIONAL_PAGES(bprm, ex, interpreter) \ + compat_arch_setup_additional_pages(bprm, interpreter) +#endif + +#ifdef COMPAT_ARCH_SETUP_ADDITIONAL_PAGES #undef ARCH_HAS_SETUP_ADDITIONAL_PAGES #define ARCH_HAS_SETUP_ADDITIONAL_PAGES 1 -#undef arch_setup_additional_pages -#define arch_setup_additional_pages compat_arch_setup_additional_pages +#undef ARCH_SETUP_ADDITIONAL_PAGES +#define ARCH_SETUP_ADDITIONAL_PAGES COMPAT_ARCH_SETUP_ADDITIONAL_PAGES #endif #ifdef compat_elf_read_implies_exec diff --git a/fs/crypto/fname.c b/fs/crypto/fname.c index f3cd592f5171..6ca7d16593ff 100644 --- a/fs/crypto/fname.c +++ b/fs/crypto/fname.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include "fscrypt_private.h" diff --git a/fs/crypto/hkdf.c b/fs/crypto/hkdf.c index 0cba7928446d..e0ec21055505 100644 --- a/fs/crypto/hkdf.c +++ b/fs/crypto/hkdf.c @@ -10,7 +10,7 @@ */ #include -#include +#include #include "fscrypt_private.h" diff --git a/fs/ubifs/auth.c b/fs/ubifs/auth.c index b93b3cd10bfd..0886d835f597 100644 --- a/fs/ubifs/auth.c +++ b/fs/ubifs/auth.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include diff --git a/fs/verity/enable.c b/fs/verity/enable.c index 5ab3bbec8108..f7e997a01ad0 100644 --- a/fs/verity/enable.c +++ b/fs/verity/enable.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * fs/verity/enable.c: ioctl to enable verity on a file + * Ioctl to enable verity on a file * * Copyright 2019 Google LLC */ @@ -398,9 +398,9 @@ int fsverity_ioctl_enable(struct file *filp, const void __user *uarg) * Some pages of the file may have been evicted from pagecache after * being used in the Merkle tree construction, then read into pagecache * again by another process reading from the file concurrently. Since - * these pages didn't undergo verification against the file measurement - * which fs-verity now claims to be enforcing, we have to wipe the - * pagecache to ensure that all future reads are verified. + * these pages didn't undergo verification against the file digest which + * fs-verity now claims to be enforcing, we have to wipe the pagecache + * to ensure that all future reads are verified. */ filemap_write_and_wait(inode->i_mapping); invalidate_inode_pages2(inode->i_mapping); diff --git a/fs/verity/fsverity_private.h b/fs/verity/fsverity_private.h index e96d99d5145e..6413d28664d6 100644 --- a/fs/verity/fsverity_private.h +++ b/fs/verity/fsverity_private.h @@ -14,7 +14,7 @@ #define pr_fmt(fmt) "fs-verity: " fmt -#include +#include #include #include @@ -67,52 +67,22 @@ struct merkle_tree_params { * When a verity file is first opened, an instance of this struct is allocated * and stored in ->i_verity_info; it remains until the inode is evicted. It * caches information about the Merkle tree that's needed to efficiently verify - * data read from the file. It also caches the file measurement. The Merkle - * tree pages themselves are not cached here, but the filesystem may cache them. + * data read from the file. It also caches the file digest. The Merkle tree + * pages themselves are not cached here, but the filesystem may cache them. */ struct fsverity_info { struct merkle_tree_params tree_params; u8 root_hash[FS_VERITY_MAX_DIGEST_SIZE]; - u8 measurement[FS_VERITY_MAX_DIGEST_SIZE]; + u8 file_digest[FS_VERITY_MAX_DIGEST_SIZE]; const struct inode *inode; }; -/* - * Merkle tree properties. The file measurement is the hash of this structure - * excluding the signature and with the sig_size field set to 0. - */ -struct fsverity_descriptor { - __u8 version; /* must be 1 */ - __u8 hash_algorithm; /* Merkle tree hash algorithm */ - __u8 log_blocksize; /* log2 of size of data and tree blocks */ - __u8 salt_size; /* size of salt in bytes; 0 if none */ - __le32 sig_size; /* size of signature in bytes; 0 if none */ - __le64 data_size; /* size of file the Merkle tree is built over */ - __u8 root_hash[64]; /* Merkle tree root hash */ - __u8 salt[32]; /* salt prepended to each hashed block */ - __u8 __reserved[144]; /* must be 0's */ - __u8 signature[]; /* optional PKCS#7 signature */ -}; - /* Arbitrary limit to bound the kmalloc() size. Can be changed. */ #define FS_VERITY_MAX_DESCRIPTOR_SIZE 16384 #define FS_VERITY_MAX_SIGNATURE_SIZE (FS_VERITY_MAX_DESCRIPTOR_SIZE - \ sizeof(struct fsverity_descriptor)) -/* - * Format in which verity file measurements are signed. This is the same as - * 'struct fsverity_digest', except here some magic bytes are prepended to - * provide some context about what is being signed in case the same key is used - * for non-fsverity purposes, and here the fields have fixed endianness. - */ -struct fsverity_signed_digest { - char magic[8]; /* must be "FSVerity" */ - __le16 digest_algorithm; - __le16 digest_size; - __u8 digest[]; -}; - /* hash_algs.c */ extern struct fsverity_hash_alg fsverity_hash_algs[]; diff --git a/fs/verity/hash_algs.c b/fs/verity/hash_algs.c index c37e186ebeb6..71d0fccb6d4c 100644 --- a/fs/verity/hash_algs.c +++ b/fs/verity/hash_algs.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * fs/verity/hash_algs.c: fs-verity hash algorithms + * fs-verity hash algorithms * * Copyright 2019 Google LLC */ diff --git a/fs/verity/init.c b/fs/verity/init.c index 94c104e00861..c98b7016f446 100644 --- a/fs/verity/init.c +++ b/fs/verity/init.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * fs/verity/init.c: fs-verity module initialization and logging + * fs-verity module initialization and logging * * Copyright 2019 Google LLC */ diff --git a/fs/verity/measure.c b/fs/verity/measure.c index df409a5682ed..f0d7b30c62db 100644 --- a/fs/verity/measure.c +++ b/fs/verity/measure.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * fs/verity/measure.c: ioctl to get a verity file's measurement + * Ioctl to get a verity file's digest * * Copyright 2019 Google LLC */ @@ -10,12 +10,12 @@ #include /** - * fsverity_ioctl_measure() - get a verity file's measurement - * @filp: file to get measurement of + * fsverity_ioctl_measure() - get a verity file's digest + * @filp: file to get digest of * @_uarg: user pointer to fsverity_digest * - * Retrieve the file measurement that the kernel is enforcing for reads from a - * verity file. See the "FS_IOC_MEASURE_VERITY" section of + * Retrieve the file digest that the kernel is enforcing for reads from a verity + * file. See the "FS_IOC_MEASURE_VERITY" section of * Documentation/filesystems/fsverity.rst for the documentation. * * Return: 0 on success, -errno on failure @@ -51,7 +51,7 @@ int fsverity_ioctl_measure(struct file *filp, void __user *_uarg) if (copy_to_user(uarg, &arg, sizeof(arg))) return -EFAULT; - if (copy_to_user(uarg->digest, vi->measurement, hash_alg->digest_size)) + if (copy_to_user(uarg->digest, vi->file_digest, hash_alg->digest_size)) return -EFAULT; return 0; diff --git a/fs/verity/open.c b/fs/verity/open.c index bfe0280c14e4..228d0eca3e2e 100644 --- a/fs/verity/open.c +++ b/fs/verity/open.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * fs/verity/open.c: opening fs-verity files + * Opening fs-verity files * * Copyright 2019 Google LLC */ @@ -124,18 +124,18 @@ out_err: } /* - * Compute the file measurement by hashing the fsverity_descriptor excluding the + * Compute the file digest by hashing the fsverity_descriptor excluding the * signature and with the sig_size field set to 0. */ -static int compute_file_measurement(struct fsverity_hash_alg *hash_alg, - struct fsverity_descriptor *desc, - u8 *measurement) +static int compute_file_digest(struct fsverity_hash_alg *hash_alg, + struct fsverity_descriptor *desc, + u8 *file_digest) { __le32 sig_size = desc->sig_size; int err; desc->sig_size = 0; - err = fsverity_hash_buffer(hash_alg, desc, sizeof(*desc), measurement); + err = fsverity_hash_buffer(hash_alg, desc, sizeof(*desc), file_digest); desc->sig_size = sig_size; return err; @@ -199,15 +199,15 @@ struct fsverity_info *fsverity_create_info(const struct inode *inode, memcpy(vi->root_hash, desc->root_hash, vi->tree_params.digest_size); - err = compute_file_measurement(vi->tree_params.hash_alg, desc, - vi->measurement); + err = compute_file_digest(vi->tree_params.hash_alg, desc, + vi->file_digest); if (err) { - fsverity_err(inode, "Error %d computing file measurement", err); + fsverity_err(inode, "Error %d computing file digest", err); goto out; } - pr_debug("Computed file measurement: %s:%*phN\n", + pr_debug("Computed file digest: %s:%*phN\n", vi->tree_params.hash_alg->name, - vi->tree_params.digest_size, vi->measurement); + vi->tree_params.digest_size, vi->file_digest); err = fsverity_verify_signature(vi, desc, desc_size); out: @@ -354,7 +354,7 @@ int __init fsverity_init_info_cache(void) { fsverity_info_cachep = KMEM_CACHE_USERCOPY(fsverity_info, SLAB_RECLAIM_ACCOUNT, - measurement); + file_digest); if (!fsverity_info_cachep) return -ENOMEM; return 0; diff --git a/fs/verity/signature.c b/fs/verity/signature.c index b14ed96387ec..012468eda2a7 100644 --- a/fs/verity/signature.c +++ b/fs/verity/signature.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * fs/verity/signature.c: verification of builtin signatures + * Verification of builtin signatures * * Copyright 2019 Google LLC */ @@ -32,8 +32,8 @@ static struct key *fsverity_keyring; * @desc: the file's fsverity_descriptor * @desc_size: size of @desc * - * If the file's fs-verity descriptor includes a signature of the file - * measurement, verify it against the certificates in the fs-verity keyring. + * If the file's fs-verity descriptor includes a signature of the file digest, + * verify it against the certificates in the fs-verity keyring. * * Return: 0 on success (signature valid or not required); -errno on failure */ @@ -44,7 +44,7 @@ int fsverity_verify_signature(const struct fsverity_info *vi, const struct inode *inode = vi->inode; const struct fsverity_hash_alg *hash_alg = vi->tree_params.hash_alg; const u32 sig_size = le32_to_cpu(desc->sig_size); - struct fsverity_signed_digest *d; + struct fsverity_formatted_digest *d; int err; if (sig_size == 0) { @@ -67,7 +67,7 @@ int fsverity_verify_signature(const struct fsverity_info *vi, memcpy(d->magic, "FSVerity", 8); d->digest_algorithm = cpu_to_le16(hash_alg - fsverity_hash_algs); d->digest_size = cpu_to_le16(hash_alg->digest_size); - memcpy(d->digest, vi->measurement, hash_alg->digest_size); + memcpy(d->digest, vi->file_digest, hash_alg->digest_size); err = verify_pkcs7_signature(d, sizeof(*d) + hash_alg->digest_size, desc->signature, sig_size, @@ -90,8 +90,8 @@ int fsverity_verify_signature(const struct fsverity_info *vi, return err; } - pr_debug("Valid signature for file measurement %s:%*phN\n", - hash_alg->name, hash_alg->digest_size, vi->measurement); + pr_debug("Valid signature for file digest %s:%*phN\n", + hash_alg->name, hash_alg->digest_size, vi->file_digest); return 0; } diff --git a/fs/verity/verify.c b/fs/verity/verify.c index a8b68c6f663d..0adb970f4e73 100644 --- a/fs/verity/verify.c +++ b/fs/verity/verify.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * fs/verity/verify.c: data verification functions, i.e. hooks for ->readpages() + * Data verification functions, i.e. hooks for ->readpages() * * Copyright 2019 Google LLC */ diff --git a/include/crypto/aead.h b/include/crypto/aead.h index c32a6f5664e9..fcc12c593ef8 100644 --- a/include/crypto/aead.h +++ b/include/crypto/aead.h @@ -191,6 +191,11 @@ static inline void crypto_free_aead(struct crypto_aead *tfm) crypto_destroy_tfm(tfm, crypto_aead_tfm(tfm)); } +static inline const char *crypto_aead_driver_name(struct crypto_aead *tfm) +{ + return crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)); +} + static inline struct aead_alg *crypto_aead_alg(struct crypto_aead *tfm) { return container_of(crypto_aead_tfm(tfm)->__crt_alg, diff --git a/include/crypto/curve25519.h b/include/crypto/curve25519.h index 4e6dc840b159..ece6a9b5fafc 100644 --- a/include/crypto/curve25519.h +++ b/include/crypto/curve25519.h @@ -28,6 +28,8 @@ void curve25519_arch(u8 out[CURVE25519_KEY_SIZE], void curve25519_base_arch(u8 pub[CURVE25519_KEY_SIZE], const u8 secret[CURVE25519_KEY_SIZE]); +bool curve25519_selftest(void); + static inline bool __must_check curve25519(u8 mypublic[CURVE25519_KEY_SIZE], const u8 secret[CURVE25519_KEY_SIZE], diff --git a/include/crypto/hash_info.h b/include/crypto/hash_info.h index eb9d2e368969..dd4f06785049 100644 --- a/include/crypto/hash_info.h +++ b/include/crypto/hash_info.h @@ -8,7 +8,8 @@ #ifndef _CRYPTO_HASH_INFO_H #define _CRYPTO_HASH_INFO_H -#include +#include +#include #include #include diff --git a/include/crypto/internal/blake2s.h b/include/crypto/internal/blake2s.h index 74ff77032e52..6e376ae6b6b5 100644 --- a/include/crypto/internal/blake2s.h +++ b/include/crypto/internal/blake2s.h @@ -16,6 +16,8 @@ void blake2s_compress_generic(struct blake2s_state *state,const u8 *block, void blake2s_compress_arch(struct blake2s_state *state,const u8 *block, size_t nblocks, const u32 inc); +bool blake2s_selftest(void); + static inline void blake2s_set_lastblock(struct blake2s_state *state) { state->f[0] = -1; diff --git a/include/crypto/sha1.h b/include/crypto/sha1.h new file mode 100644 index 000000000000..044ecea60ac8 --- /dev/null +++ b/include/crypto/sha1.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Common values for SHA-1 algorithms + */ + +#ifndef _CRYPTO_SHA1_H +#define _CRYPTO_SHA1_H + +#include + +#define SHA1_DIGEST_SIZE 20 +#define SHA1_BLOCK_SIZE 64 + +#define SHA1_H0 0x67452301UL +#define SHA1_H1 0xefcdab89UL +#define SHA1_H2 0x98badcfeUL +#define SHA1_H3 0x10325476UL +#define SHA1_H4 0xc3d2e1f0UL + +extern const u8 sha1_zero_message_hash[SHA1_DIGEST_SIZE]; + +struct sha1_state { + u32 state[SHA1_DIGEST_SIZE / 4]; + u64 count; + u8 buffer[SHA1_BLOCK_SIZE]; +}; + +struct shash_desc; + +extern int crypto_sha1_update(struct shash_desc *desc, const u8 *data, + unsigned int len); + +extern int crypto_sha1_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *hash); + +/* + * An implementation of SHA-1's compression function. Don't use in new code! + * You shouldn't be using SHA-1, and even if you *have* to use SHA-1, this isn't + * the correct way to hash something with SHA-1 (use crypto_shash instead). + */ +#define SHA1_DIGEST_WORDS (SHA1_DIGEST_SIZE / 4) +#define SHA1_WORKSPACE_WORDS 16 +void sha1_init(__u32 *buf); +void sha1_transform(__u32 *digest, const char *data, __u32 *W); + +#endif /* _CRYPTO_SHA1_H */ diff --git a/include/crypto/sha1_base.h b/include/crypto/sha1_base.h index 20fd1f7468af..2e0e7c3827d1 100644 --- a/include/crypto/sha1_base.h +++ b/include/crypto/sha1_base.h @@ -9,9 +9,10 @@ #define _CRYPTO_SHA1_BASE_H #include -#include +#include #include #include +#include #include @@ -101,7 +102,7 @@ static inline int sha1_base_finish(struct shash_desc *desc, u8 *out) for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(__be32); i++) put_unaligned_be32(sctx->state[i], digest++); - *sctx = (struct sha1_state){}; + memzero_explicit(sctx, sizeof(*sctx)); return 0; } diff --git a/include/crypto/sha.h b/include/crypto/sha2.h similarity index 77% rename from include/crypto/sha.h rename to include/crypto/sha2.h index 4ff3da816630..2838f529f31e 100644 --- a/include/crypto/sha.h +++ b/include/crypto/sha2.h @@ -1,16 +1,13 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * Common values for SHA algorithms + * Common values for SHA-2 algorithms */ -#ifndef _CRYPTO_SHA_H -#define _CRYPTO_SHA_H +#ifndef _CRYPTO_SHA2_H +#define _CRYPTO_SHA2_H #include -#define SHA1_DIGEST_SIZE 20 -#define SHA1_BLOCK_SIZE 64 - #define SHA224_DIGEST_SIZE 28 #define SHA224_BLOCK_SIZE 64 @@ -23,12 +20,6 @@ #define SHA512_DIGEST_SIZE 64 #define SHA512_BLOCK_SIZE 128 -#define SHA1_H0 0x67452301UL -#define SHA1_H1 0xefcdab89UL -#define SHA1_H2 0x98badcfeUL -#define SHA1_H3 0x10325476UL -#define SHA1_H4 0xc3d2e1f0UL - #define SHA224_H0 0xc1059ed8UL #define SHA224_H1 0x367cd507UL #define SHA224_H2 0x3070dd17UL @@ -65,8 +56,6 @@ #define SHA512_H6 0x1f83d9abfb41bd6bULL #define SHA512_H7 0x5be0cd19137e2179ULL -extern const u8 sha1_zero_message_hash[SHA1_DIGEST_SIZE]; - extern const u8 sha224_zero_message_hash[SHA224_DIGEST_SIZE]; extern const u8 sha256_zero_message_hash[SHA256_DIGEST_SIZE]; @@ -75,12 +64,6 @@ extern const u8 sha384_zero_message_hash[SHA384_DIGEST_SIZE]; extern const u8 sha512_zero_message_hash[SHA512_DIGEST_SIZE]; -struct sha1_state { - u32 state[SHA1_DIGEST_SIZE / 4]; - u64 count; - u8 buffer[SHA1_BLOCK_SIZE]; -}; - struct sha256_state { u32 state[SHA256_DIGEST_SIZE / 4]; u64 count; @@ -95,12 +78,6 @@ struct sha512_state { struct shash_desc; -extern int crypto_sha1_update(struct shash_desc *desc, const u8 *data, - unsigned int len); - -extern int crypto_sha1_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash); - extern int crypto_sha256_update(struct shash_desc *desc, const u8 *data, unsigned int len); @@ -113,16 +90,6 @@ extern int crypto_sha512_update(struct shash_desc *desc, const u8 *data, extern int crypto_sha512_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *hash); -/* - * An implementation of SHA-1's compression function. Don't use in new code! - * You shouldn't be using SHA-1, and even if you *have* to use SHA-1, this isn't - * the correct way to hash something with SHA-1 (use crypto_shash instead). - */ -#define SHA1_DIGEST_WORDS (SHA1_DIGEST_SIZE / 4) -#define SHA1_WORKSPACE_WORDS 16 -void sha1_init(__u32 *buf); -void sha1_transform(__u32 *digest, const char *data, __u32 *W); - /* * Stand-alone implementation of the SHA256 algorithm. It is designed to * have as little dependencies as possible so it can be used in the @@ -164,4 +131,4 @@ static inline void sha224_init(struct sha256_state *sctx) void sha224_update(struct sha256_state *sctx, const u8 *data, unsigned int len); void sha224_final(struct sha256_state *sctx, u8 *out); -#endif +#endif /* _CRYPTO_SHA2_H */ diff --git a/include/crypto/sha256_base.h b/include/crypto/sha256_base.h index 6ded110783ae..76173c613058 100644 --- a/include/crypto/sha256_base.h +++ b/include/crypto/sha256_base.h @@ -9,9 +9,10 @@ #define _CRYPTO_SHA256_BASE_H #include -#include +#include #include #include +#include #include @@ -105,7 +106,7 @@ static inline int sha256_base_finish(struct shash_desc *desc, u8 *out) for (i = 0; digest_size > 0; i++, digest_size -= sizeof(__be32)) put_unaligned_be32(sctx->state[i], digest++); - *sctx = (struct sha256_state){}; + memzero_explicit(sctx, sizeof(*sctx)); return 0; } diff --git a/include/crypto/sha512_base.h b/include/crypto/sha512_base.h index fb19c77494dc..b370b3340b16 100644 --- a/include/crypto/sha512_base.h +++ b/include/crypto/sha512_base.h @@ -9,9 +9,10 @@ #define _CRYPTO_SHA512_BASE_H #include -#include +#include #include #include +#include #include @@ -126,7 +127,7 @@ static inline int sha512_base_finish(struct shash_desc *desc, u8 *out) for (i = 0; digest_size > 0; i++, digest_size -= sizeof(__be64)) put_unaligned_be64(sctx->state[i], digest++); - *sctx = (struct sha512_state){}; + memzero_explicit(sctx, sizeof(*sctx)); return 0; } diff --git a/include/crypto/sm3_base.h b/include/crypto/sm3_base.h index 1cbf9aa1fe52..2f3a32ab97bb 100644 --- a/include/crypto/sm3_base.h +++ b/include/crypto/sm3_base.h @@ -13,6 +13,7 @@ #include #include #include +#include #include typedef void (sm3_block_fn)(struct sm3_state *sst, u8 const *src, int blocks); @@ -104,7 +105,7 @@ static inline int sm3_base_finish(struct shash_desc *desc, u8 *out) for (i = 0; i < SM3_DIGEST_SIZE / sizeof(__be32); i++) put_unaligned_be32(sctx->state[i], digest++); - *sctx = (struct sm3_state){}; + memzero_explicit(sctx, sizeof(*sctx)); return 0; } diff --git a/include/linux/ccp.h b/include/linux/ccp.h index a5dfbaf2470d..868924dec5a1 100644 --- a/include/linux/ccp.h +++ b/include/linux/ccp.h @@ -15,7 +15,8 @@ #include #include #include -#include +#include +#include struct ccp_device; struct ccp_cmd; diff --git a/include/linux/edac.h b/include/linux/edac.h index 15e8f3d8a895..76d3562d3006 100644 --- a/include/linux/edac.h +++ b/include/linux/edac.h @@ -175,11 +175,15 @@ static inline char *mc_event_error_type(const unsigned int err_type) * @MEM_RDDR3: Registered DDR3 RAM * This is a variant of the DDR3 memories. * @MEM_LRDDR3: Load-Reduced DDR3 memory. + * @MEM_LPDDR3: Low-Power DDR3 memory. * @MEM_DDR4: Unbuffered DDR4 RAM * @MEM_RDDR4: Registered DDR4 RAM * This is a variant of the DDR4 memories. * @MEM_LRDDR4: Load-Reduced DDR4 memory. + * @MEM_LPDDR4: Low-Power DDR4 memory. + * @MEM_DDR5: Unbuffered DDR5 RAM * @MEM_NVDIMM: Non-volatile RAM + * @MEM_WIO2: Wide I/O 2. */ enum mem_type { MEM_EMPTY = 0, @@ -200,10 +204,14 @@ enum mem_type { MEM_DDR3, MEM_RDDR3, MEM_LRDDR3, + MEM_LPDDR3, MEM_DDR4, MEM_RDDR4, MEM_LRDDR4, + MEM_LPDDR4, + MEM_DDR5, MEM_NVDIMM, + MEM_WIO2, }; #define MEM_FLAG_EMPTY BIT(MEM_EMPTY) @@ -223,13 +231,17 @@ enum mem_type { #define MEM_FLAG_XDR BIT(MEM_XDR) #define MEM_FLAG_DDR3 BIT(MEM_DDR3) #define MEM_FLAG_RDDR3 BIT(MEM_RDDR3) +#define MEM_FLAG_LPDDR3 BIT(MEM_LPDDR3) #define MEM_FLAG_DDR4 BIT(MEM_DDR4) #define MEM_FLAG_RDDR4 BIT(MEM_RDDR4) #define MEM_FLAG_LRDDR4 BIT(MEM_LRDDR4) +#define MEM_FLAG_LPDDR4 BIT(MEM_LPDDR4) +#define MEM_FLAG_DDR5 BIT(MEM_DDR5) #define MEM_FLAG_NVDIMM BIT(MEM_NVDIMM) +#define MEM_FLAG_WIO2 BIT(MEM_WIO2) /** - * enum edac-type - Error Detection and Correction capabilities and mode + * enum edac_type - Error Detection and Correction capabilities and mode * @EDAC_UNKNOWN: Unknown if ECC is available * @EDAC_NONE: Doesn't support ECC * @EDAC_RESERVED: Reserved ECC type @@ -309,7 +321,7 @@ enum scrub_type { #define OP_OFFLINE 0x300 /** - * enum edac_mc_layer - memory controller hierarchy layer + * enum edac_mc_layer_type - memory controller hierarchy layer * * @EDAC_MC_LAYER_BRANCH: memory layer is named "branch" * @EDAC_MC_LAYER_CHANNEL: memory layer is named "channel" diff --git a/include/linux/elf.h b/include/linux/elf.h index 5d5b0321da0b..c9a46c4e183b 100644 --- a/include/linux/elf.h +++ b/include/linux/elf.h @@ -22,6 +22,16 @@ SET_PERSONALITY(ex) #endif +#ifndef START_THREAD +#define START_THREAD(elf_ex, regs, elf_entry, start_stack) \ + start_thread(regs, elf_entry, start_stack) +#endif + +#if defined(ARCH_HAS_SETUP_ADDITIONAL_PAGES) && !defined(ARCH_SETUP_ADDITIONAL_PAGES) +#define ARCH_SETUP_ADDITIONAL_PAGES(bprm, ex, interpreter) \ + arch_setup_additional_pages(bprm, interpreter) +#endif + #define ELF32_GNU_PROPERTY_ALIGN 4 #define ELF64_GNU_PROPERTY_ALIGN 8 diff --git a/include/linux/filter.h b/include/linux/filter.h index 1b62397bd124..29c27656165b 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include diff --git a/include/linux/mm.h b/include/linux/mm.h index b75193c58078..2788b2078993 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -559,6 +559,13 @@ struct vm_operations_struct { void (*close)(struct vm_area_struct * area); int (*split)(struct vm_area_struct * area, unsigned long addr); int (*mremap)(struct vm_area_struct * area); + /* + * Called by mprotect() to make driver-specific permission + * checks before mprotect() is finalised. The VMA must not + * be modified. Returns 0 if eprotect() can proceed. + */ + int (*mprotect)(struct vm_area_struct *vma, unsigned long start, + unsigned long end, unsigned long newflags); vm_fault_t (*fault)(struct vm_fault *vmf); vm_fault_t (*huge_fault)(struct vm_fault *vmf, enum page_entry_size pe_size); diff --git a/include/linux/purgatory.h b/include/linux/purgatory.h index b950e961cfa8..d7dc1559427f 100644 --- a/include/linux/purgatory.h +++ b/include/linux/purgatory.h @@ -3,7 +3,7 @@ #define _LINUX_PURGATORY_H #include -#include +#include #include struct kexec_sha_region { diff --git a/include/uapi/linux/fsverity.h b/include/uapi/linux/fsverity.h index da0daf6c193b..33f44156f8ea 100644 --- a/include/uapi/linux/fsverity.h +++ b/include/uapi/linux/fsverity.h @@ -34,6 +34,55 @@ struct fsverity_digest { __u8 digest[]; }; +/* + * Struct containing a file's Merkle tree properties. The fs-verity file digest + * is the hash of this struct. A userspace program needs this struct only if it + * needs to compute fs-verity file digests itself, e.g. in order to sign files. + * It isn't needed just to enable fs-verity on a file. + * + * Note: when computing the file digest, 'sig_size' and 'signature' must be left + * zero and empty, respectively. These fields are present only because some + * filesystems reuse this struct as part of their on-disk format. + */ +struct fsverity_descriptor { + __u8 version; /* must be 1 */ + __u8 hash_algorithm; /* Merkle tree hash algorithm */ + __u8 log_blocksize; /* log2 of size of data and tree blocks */ + __u8 salt_size; /* size of salt in bytes; 0 if none */ +#ifdef __KERNEL__ + __le32 sig_size; +#else + __le32 __reserved_0x04; /* must be 0 */ +#endif + __le64 data_size; /* size of file the Merkle tree is built over */ + __u8 root_hash[64]; /* Merkle tree root hash */ + __u8 salt[32]; /* salt prepended to each hashed block */ + __u8 __reserved[144]; /* must be 0's */ +#ifdef __KERNEL__ + __u8 signature[]; +#endif +}; + +/* + * Format in which fs-verity file digests are signed in built-in signatures. + * This is the same as 'struct fsverity_digest', except here some magic bytes + * are prepended to provide some context about what is being signed in case the + * same key is used for non-fsverity purposes, and here the fields have fixed + * endianness. + * + * This struct is specific to the built-in signature verification support, which + * is optional. fs-verity users may also verify signatures in userspace, in + * which case userspace is responsible for deciding on what bytes are signed. + * This struct may still be used, but it doesn't have to be. For example, + * userspace could instead use a string like "sha256:$digest_as_hex_string". + */ +struct fsverity_formatted_digest { + char magic[8]; /* must be "FSVerity" */ + __le16 digest_algorithm; + __le16 digest_size; + __u8 digest[]; +}; + #define FS_IOC_ENABLE_VERITY _IOW('f', 133, struct fsverity_enable_arg) #define FS_IOC_MEASURE_VERITY _IOWR('f', 134, struct fsverity_digest) diff --git a/include/uapi/linux/if_alg.h b/include/uapi/linux/if_alg.h index 60b7c2efd921..dc52a11ba6d1 100644 --- a/include/uapi/linux/if_alg.h +++ b/include/uapi/linux/if_alg.h @@ -24,6 +24,22 @@ struct sockaddr_alg { __u8 salg_name[64]; }; +/* + * Linux v4.12 and later removed the 64-byte limit on salg_name[]; it's now an + * arbitrary-length field. We had to keep the original struct above for source + * compatibility with existing userspace programs, though. Use the new struct + * below if support for very long algorithm names is needed. To do this, + * allocate 'sizeof(struct sockaddr_alg_new) + strlen(algname) + 1' bytes, and + * copy algname (including the null terminator) into salg_name. + */ +struct sockaddr_alg_new { + __u16 salg_family; + __u8 salg_type[14]; + __u32 salg_feat; + __u32 salg_mask; + __u8 salg_name[]; +}; + struct af_alg_iv { __u32 ivlen; __u8 iv[0]; diff --git a/init/Kconfig b/init/Kconfig index 188dde6ed879..dc9412ab55a1 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -114,7 +114,7 @@ config INIT_ENV_ARG_LIMIT config COMPILE_TEST bool "Compile also drivers which will not load" - depends on !UML + depends on !UML && !S390 default n help Some drivers can be compiled on a different platform than they are diff --git a/kernel/crash_core.c b/kernel/crash_core.c index 106e4500fd53..4fcfe0b70c4e 100644 --- a/kernel/crash_core.c +++ b/kernel/crash_core.c @@ -11,7 +11,7 @@ #include #include -#include +#include /* vmcoreinfo stuff */ unsigned char *vmcoreinfo_data; diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index 8798a8183974..4f8efc278aa7 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -42,7 +42,6 @@ #include #include -#include #include "kexec_internal.h" DEFINE_MUTEX(kexec_mutex); diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index e21f6b9234f7..b02086d70492 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/lib/Kconfig.kasan b/lib/Kconfig.kasan index 542a9c18398e..8fb097057fec 100644 --- a/lib/Kconfig.kasan +++ b/lib/Kconfig.kasan @@ -136,15 +136,6 @@ config KASAN_STACK default 1 if KASAN_STACK_ENABLE || CC_IS_GCC default 0 -config KASAN_S390_4_LEVEL_PAGING - bool "KASan: use 4-level paging" - depends on S390 - help - Compiling the kernel with KASan disables automatic 3-level vs - 4-level paging selection. 3-level paging is used by default (up - to 3TB of RAM with KASan enabled). This options allows to force - 4-level paging instead. - config KASAN_SW_TAGS_IDENTIFY bool "Enable memory corruption identification" depends on KASAN_SW_TAGS diff --git a/lib/crypto/blake2s-selftest.c b/lib/crypto/blake2s-selftest.c index 79ef404a990d..5d9ea53be973 100644 --- a/lib/crypto/blake2s-selftest.c +++ b/lib/crypto/blake2s-selftest.c @@ -3,7 +3,7 @@ * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. */ -#include +#include #include /* diff --git a/lib/crypto/blake2s.c b/lib/crypto/blake2s.c index 41025a30c524..6a4b6b78d630 100644 --- a/lib/crypto/blake2s.c +++ b/lib/crypto/blake2s.c @@ -17,8 +17,6 @@ #include #include -bool blake2s_selftest(void); - void blake2s_update(struct blake2s_state *state, const u8 *in, size_t inlen) { const size_t fill = BLAKE2S_BLOCK_SIZE - state->buflen; diff --git a/lib/crypto/curve25519.c b/lib/crypto/curve25519.c index 288a62cd29b2..fb29739e8c29 100644 --- a/lib/crypto/curve25519.c +++ b/lib/crypto/curve25519.c @@ -13,8 +13,6 @@ #include #include -bool curve25519_selftest(void); - static int __init mod_init(void) { if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) && diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index 2321f6cb322f..72a4b0b1df28 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -15,9 +15,28 @@ #include #include #include -#include +#include #include +static const u32 SHA256_K[] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, +}; + static inline u32 Ch(u32 x, u32 y, u32 z) { return z ^ (x & (y ^ z)); @@ -43,173 +62,68 @@ static inline void BLEND_OP(int I, u32 *W) W[I] = s1(W[I-2]) + W[I-7] + s0(W[I-15]) + W[I-16]; } -static void sha256_transform(u32 *state, const u8 *input) +#define SHA256_ROUND(i, a, b, c, d, e, f, g, h) do { \ + u32 t1, t2; \ + t1 = h + e1(e) + Ch(e, f, g) + SHA256_K[i] + W[i]; \ + t2 = e0(a) + Maj(a, b, c); \ + d += t1; \ + h = t1 + t2; \ +} while (0) + +static void sha256_transform(u32 *state, const u8 *input, u32 *W) { - u32 a, b, c, d, e, f, g, h, t1, t2; - u32 W[64]; + u32 a, b, c, d, e, f, g, h; int i; /* load the input */ - for (i = 0; i < 16; i++) - LOAD_OP(i, W, input); + for (i = 0; i < 16; i += 8) { + LOAD_OP(i + 0, W, input); + LOAD_OP(i + 1, W, input); + LOAD_OP(i + 2, W, input); + LOAD_OP(i + 3, W, input); + LOAD_OP(i + 4, W, input); + LOAD_OP(i + 5, W, input); + LOAD_OP(i + 6, W, input); + LOAD_OP(i + 7, W, input); + } /* now blend */ - for (i = 16; i < 64; i++) - BLEND_OP(i, W); + for (i = 16; i < 64; i += 8) { + BLEND_OP(i + 0, W); + BLEND_OP(i + 1, W); + BLEND_OP(i + 2, W); + BLEND_OP(i + 3, W); + BLEND_OP(i + 4, W); + BLEND_OP(i + 5, W); + BLEND_OP(i + 6, W); + BLEND_OP(i + 7, W); + } /* load the state into our registers */ a = state[0]; b = state[1]; c = state[2]; d = state[3]; e = state[4]; f = state[5]; g = state[6]; h = state[7]; /* now iterate */ - t1 = h + e1(e) + Ch(e, f, g) + 0x428a2f98 + W[0]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; - t1 = g + e1(d) + Ch(d, e, f) + 0x71374491 + W[1]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; - t1 = f + e1(c) + Ch(c, d, e) + 0xb5c0fbcf + W[2]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; - t1 = e + e1(b) + Ch(b, c, d) + 0xe9b5dba5 + W[3]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; - t1 = d + e1(a) + Ch(a, b, c) + 0x3956c25b + W[4]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; - t1 = c + e1(h) + Ch(h, a, b) + 0x59f111f1 + W[5]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; - t1 = b + e1(g) + Ch(g, h, a) + 0x923f82a4 + W[6]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; - t1 = a + e1(f) + Ch(f, g, h) + 0xab1c5ed5 + W[7]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; - - t1 = h + e1(e) + Ch(e, f, g) + 0xd807aa98 + W[8]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; - t1 = g + e1(d) + Ch(d, e, f) + 0x12835b01 + W[9]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; - t1 = f + e1(c) + Ch(c, d, e) + 0x243185be + W[10]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; - t1 = e + e1(b) + Ch(b, c, d) + 0x550c7dc3 + W[11]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; - t1 = d + e1(a) + Ch(a, b, c) + 0x72be5d74 + W[12]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; - t1 = c + e1(h) + Ch(h, a, b) + 0x80deb1fe + W[13]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; - t1 = b + e1(g) + Ch(g, h, a) + 0x9bdc06a7 + W[14]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; - t1 = a + e1(f) + Ch(f, g, h) + 0xc19bf174 + W[15]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; - - t1 = h + e1(e) + Ch(e, f, g) + 0xe49b69c1 + W[16]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; - t1 = g + e1(d) + Ch(d, e, f) + 0xefbe4786 + W[17]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; - t1 = f + e1(c) + Ch(c, d, e) + 0x0fc19dc6 + W[18]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; - t1 = e + e1(b) + Ch(b, c, d) + 0x240ca1cc + W[19]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; - t1 = d + e1(a) + Ch(a, b, c) + 0x2de92c6f + W[20]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; - t1 = c + e1(h) + Ch(h, a, b) + 0x4a7484aa + W[21]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; - t1 = b + e1(g) + Ch(g, h, a) + 0x5cb0a9dc + W[22]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; - t1 = a + e1(f) + Ch(f, g, h) + 0x76f988da + W[23]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; - - t1 = h + e1(e) + Ch(e, f, g) + 0x983e5152 + W[24]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; - t1 = g + e1(d) + Ch(d, e, f) + 0xa831c66d + W[25]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; - t1 = f + e1(c) + Ch(c, d, e) + 0xb00327c8 + W[26]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; - t1 = e + e1(b) + Ch(b, c, d) + 0xbf597fc7 + W[27]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; - t1 = d + e1(a) + Ch(a, b, c) + 0xc6e00bf3 + W[28]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; - t1 = c + e1(h) + Ch(h, a, b) + 0xd5a79147 + W[29]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; - t1 = b + e1(g) + Ch(g, h, a) + 0x06ca6351 + W[30]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; - t1 = a + e1(f) + Ch(f, g, h) + 0x14292967 + W[31]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; - - t1 = h + e1(e) + Ch(e, f, g) + 0x27b70a85 + W[32]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; - t1 = g + e1(d) + Ch(d, e, f) + 0x2e1b2138 + W[33]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; - t1 = f + e1(c) + Ch(c, d, e) + 0x4d2c6dfc + W[34]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; - t1 = e + e1(b) + Ch(b, c, d) + 0x53380d13 + W[35]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; - t1 = d + e1(a) + Ch(a, b, c) + 0x650a7354 + W[36]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; - t1 = c + e1(h) + Ch(h, a, b) + 0x766a0abb + W[37]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; - t1 = b + e1(g) + Ch(g, h, a) + 0x81c2c92e + W[38]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; - t1 = a + e1(f) + Ch(f, g, h) + 0x92722c85 + W[39]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; - - t1 = h + e1(e) + Ch(e, f, g) + 0xa2bfe8a1 + W[40]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; - t1 = g + e1(d) + Ch(d, e, f) + 0xa81a664b + W[41]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; - t1 = f + e1(c) + Ch(c, d, e) + 0xc24b8b70 + W[42]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; - t1 = e + e1(b) + Ch(b, c, d) + 0xc76c51a3 + W[43]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; - t1 = d + e1(a) + Ch(a, b, c) + 0xd192e819 + W[44]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; - t1 = c + e1(h) + Ch(h, a, b) + 0xd6990624 + W[45]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; - t1 = b + e1(g) + Ch(g, h, a) + 0xf40e3585 + W[46]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; - t1 = a + e1(f) + Ch(f, g, h) + 0x106aa070 + W[47]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; - - t1 = h + e1(e) + Ch(e, f, g) + 0x19a4c116 + W[48]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; - t1 = g + e1(d) + Ch(d, e, f) + 0x1e376c08 + W[49]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; - t1 = f + e1(c) + Ch(c, d, e) + 0x2748774c + W[50]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; - t1 = e + e1(b) + Ch(b, c, d) + 0x34b0bcb5 + W[51]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; - t1 = d + e1(a) + Ch(a, b, c) + 0x391c0cb3 + W[52]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; - t1 = c + e1(h) + Ch(h, a, b) + 0x4ed8aa4a + W[53]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; - t1 = b + e1(g) + Ch(g, h, a) + 0x5b9cca4f + W[54]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; - t1 = a + e1(f) + Ch(f, g, h) + 0x682e6ff3 + W[55]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; - - t1 = h + e1(e) + Ch(e, f, g) + 0x748f82ee + W[56]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; - t1 = g + e1(d) + Ch(d, e, f) + 0x78a5636f + W[57]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; - t1 = f + e1(c) + Ch(c, d, e) + 0x84c87814 + W[58]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; - t1 = e + e1(b) + Ch(b, c, d) + 0x8cc70208 + W[59]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; - t1 = d + e1(a) + Ch(a, b, c) + 0x90befffa + W[60]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; - t1 = c + e1(h) + Ch(h, a, b) + 0xa4506ceb + W[61]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; - t1 = b + e1(g) + Ch(g, h, a) + 0xbef9a3f7 + W[62]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; - t1 = a + e1(f) + Ch(f, g, h) + 0xc67178f2 + W[63]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; + for (i = 0; i < 64; i += 8) { + SHA256_ROUND(i + 0, a, b, c, d, e, f, g, h); + SHA256_ROUND(i + 1, h, a, b, c, d, e, f, g); + SHA256_ROUND(i + 2, g, h, a, b, c, d, e, f); + SHA256_ROUND(i + 3, f, g, h, a, b, c, d, e); + SHA256_ROUND(i + 4, e, f, g, h, a, b, c, d); + SHA256_ROUND(i + 5, d, e, f, g, h, a, b, c); + SHA256_ROUND(i + 6, c, d, e, f, g, h, a, b); + SHA256_ROUND(i + 7, b, c, d, e, f, g, h, a); + } state[0] += a; state[1] += b; state[2] += c; state[3] += d; state[4] += e; state[5] += f; state[6] += g; state[7] += h; - - /* clear any sensitive info... */ - a = b = c = d = e = f = g = h = t1 = t2 = 0; - memzero_explicit(W, 64 * sizeof(u32)); } void sha256_update(struct sha256_state *sctx, const u8 *data, unsigned int len) { unsigned int partial, done; const u8 *src; + u32 W[64]; partial = sctx->count & 0x3f; sctx->count += len; @@ -224,11 +138,13 @@ void sha256_update(struct sha256_state *sctx, const u8 *data, unsigned int len) } do { - sha256_transform(sctx->state, src); + sha256_transform(sctx->state, src, W); done += 64; src = data + done; } while (done + 63 < len); + memzero_explicit(W, sizeof(W)); + partial = 0; } memcpy(sctx->buf + partial, src, len - done); @@ -265,7 +181,7 @@ static void __sha256_final(struct sha256_state *sctx, u8 *out, int digest_words) put_unaligned_be32(sctx->state[i], &dst[i]); /* Zeroize sensitive information. */ - memset(sctx, 0, sizeof(*sctx)); + memzero_explicit(sctx, sizeof(*sctx)); } void sha256_final(struct sha256_state *sctx, u8 *out) diff --git a/lib/digsig.c b/lib/digsig.c index e0627c3e53b2..04b5e55ed95f 100644 --- a/lib/digsig.c +++ b/lib/digsig.c @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/lib/mpi/ec.c b/lib/mpi/ec.c index c21470122dfc..40f5908e57a4 100644 --- a/lib/mpi/ec.c +++ b/lib/mpi/ec.c @@ -1252,7 +1252,6 @@ void mpi_ec_mul_point(MPI_POINT result, MPI_POINT q1, q2, prd, sum; unsigned long sw; mpi_size_t rsize; - int scalar_copied = 0; /* Compute scalar point multiplication with Montgomery Ladder. * Note that we don't use Y-coordinate in the points at all. @@ -1314,8 +1313,6 @@ void mpi_ec_mul_point(MPI_POINT result, point_free(&p2); point_free(&p1_); point_free(&p2_); - if (scalar_copied) - mpi_free(scalar); return; } diff --git a/lib/sha1.c b/lib/sha1.c index 49257a915bb6..9bd1935a1472 100644 --- a/lib/sha1.c +++ b/lib/sha1.c @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include /* diff --git a/mm/mprotect.c b/mm/mprotect.c index a2a62d62a8b5..d8c39370017d 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -616,9 +616,16 @@ static int do_mprotect_pkey(unsigned long start, size_t len, tmp = vma->vm_end; if (tmp > end) tmp = end; + + if (vma->vm_ops && vma->vm_ops->mprotect) + error = vma->vm_ops->mprotect(vma, nstart, tmp, newflags); + if (error) + goto out; + error = mprotect_fixup(vma, &prev, nstart, tmp, newflags); if (error) goto out; + nstart = tmp; if (nstart < prev->vm_end) diff --git a/net/ipv6/seg6_hmac.c b/net/ipv6/seg6_hmac.c index 85dddfe3a2c6..687d95dce085 100644 --- a/net/ipv6/seg6_hmac.c +++ b/net/ipv6/seg6_hmac.c @@ -35,7 +35,6 @@ #include #include -#include #include #include #include diff --git a/net/mptcp/crypto.c b/net/mptcp/crypto.c index 05d398d3fde4..b472dc149856 100644 --- a/net/mptcp/crypto.c +++ b/net/mptcp/crypto.c @@ -21,7 +21,7 @@ */ #include -#include +#include #include #include "protocol.h" diff --git a/net/mptcp/options.c b/net/mptcp/options.c index a044dd43411d..90cd52df99a6 100644 --- a/net/mptcp/options.c +++ b/net/mptcp/options.c @@ -7,7 +7,7 @@ #define pr_fmt(fmt) "MPTCP: " fmt #include -#include +#include #include #include #include "protocol.h" diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c index 953906e40742..c21a852a6ffa 100644 --- a/net/mptcp/subflow.c +++ b/net/mptcp/subflow.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/scripts/recordmcount.pl b/scripts/recordmcount.pl index 3f77a5d695c1..56c801502b9a 100755 --- a/scripts/recordmcount.pl +++ b/scripts/recordmcount.pl @@ -254,9 +254,6 @@ if ($arch eq "x86_64") { if ($cc =~ /-DCC_USING_HOTPATCH/) { $mcount_regex = "^\\s*([0-9a-fA-F]+):\\s*c0 04 00 00 00 00\\s*brcl\\s*0,[0-9a-f]+ <([^\+]*)>\$"; $mcount_adjust = 0; - } else { - $mcount_regex = "^\\s*([0-9a-fA-F]+):\\s*R_390_(PC|PLT)32DBL\\s+_mcount\\+0x2\$"; - $mcount_adjust = -14; } $alignment = 8; $type = ".quad"; diff --git a/security/integrity/integrity.h b/security/integrity/integrity.h index 413c803c5208..547425c20e11 100644 --- a/security/integrity/integrity.h +++ b/security/integrity/integrity.h @@ -14,7 +14,7 @@ #include #include -#include +#include #include #include diff --git a/security/keys/encrypted-keys/encrypted.c b/security/keys/encrypted-keys/encrypted.c index 192e531c146f..87432b35d771 100644 --- a/security/keys/encrypted-keys/encrypted.c +++ b/security/keys/encrypted-keys/encrypted.c @@ -29,7 +29,7 @@ #include #include #include -#include +#include #include #include "encrypted.h" diff --git a/security/keys/trusted-keys/trusted_tpm1.c b/security/keys/trusted-keys/trusted_tpm1.c index b9fe02e5f84f..74d82093cbaa 100644 --- a/security/keys/trusted-keys/trusted_tpm1.c +++ b/security/keys/trusted-keys/trusted_tpm1.c @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/sound/soc/codecs/cros_ec_codec.c b/sound/soc/codecs/cros_ec_codec.c index 28f039adfa13..58894bf47514 100644 --- a/sound/soc/codecs/cros_ec_codec.c +++ b/sound/soc/codecs/cros_ec_codec.c @@ -8,7 +8,7 @@ * EC for audio function. */ -#include +#include #include #include #include diff --git a/tools/power/cpupower/lib/cpupower.c b/tools/power/cpupower/lib/cpupower.c index 3656e697537e..3f7d0c0c5067 100644 --- a/tools/power/cpupower/lib/cpupower.c +++ b/tools/power/cpupower/lib/cpupower.c @@ -16,8 +16,8 @@ unsigned int cpupower_read_sysfs(const char *path, char *buf, size_t buflen) { - int fd; ssize_t numread; + int fd; fd = open(path, O_RDONLY); if (fd == -1) @@ -35,6 +35,27 @@ unsigned int cpupower_read_sysfs(const char *path, char *buf, size_t buflen) return (unsigned int) numread; } +unsigned int cpupower_write_sysfs(const char *path, char *buf, size_t buflen) +{ + ssize_t numwritten; + int fd; + + fd = open(path, O_WRONLY); + if (fd == -1) + return 0; + + numwritten = write(fd, buf, buflen - 1); + if (numwritten < 1) { + perror(path); + close(fd); + return -1; + } + + close(fd); + + return (unsigned int) numwritten; +} + /* * Detect whether a CPU is online * diff --git a/tools/power/cpupower/lib/cpupower_intern.h b/tools/power/cpupower/lib/cpupower_intern.h index 4887c76d23f8..ac1112b956ec 100644 --- a/tools/power/cpupower/lib/cpupower_intern.h +++ b/tools/power/cpupower/lib/cpupower_intern.h @@ -1,6 +1,11 @@ /* SPDX-License-Identifier: GPL-2.0 */ #define PATH_TO_CPU "/sys/devices/system/cpu/" + +#ifndef MAX_LINE_LEN #define MAX_LINE_LEN 4096 +#endif + #define SYSFS_PATH_MAX 255 unsigned int cpupower_read_sysfs(const char *path, char *buf, size_t buflen); +unsigned int cpupower_write_sysfs(const char *path, char *buf, size_t buflen); diff --git a/tools/power/cpupower/utils/cpupower-info.c b/tools/power/cpupower/utils/cpupower-info.c index 0ba61a2c4d81..06345b543786 100644 --- a/tools/power/cpupower/utils/cpupower-info.c +++ b/tools/power/cpupower/utils/cpupower-info.c @@ -101,7 +101,7 @@ int cmd_info(int argc, char **argv) } if (params.perf_bias) { - ret = msr_intel_get_perf_bias(cpu); + ret = cpupower_intel_get_perf_bias(cpu); if (ret < 0) { fprintf(stderr, _("Could not read perf-bias value[%d]\n"), ret); diff --git a/tools/power/cpupower/utils/cpupower-set.c b/tools/power/cpupower/utils/cpupower-set.c index 052044d7e012..180d5ba877e6 100644 --- a/tools/power/cpupower/utils/cpupower-set.c +++ b/tools/power/cpupower/utils/cpupower-set.c @@ -95,7 +95,7 @@ int cmd_set(int argc, char **argv) } if (params.perf_bias) { - ret = msr_intel_set_perf_bias(cpu, perf_bias); + ret = cpupower_intel_set_perf_bias(cpu, perf_bias); if (ret) { fprintf(stderr, _("Error setting perf-bias " "value on CPU %d\n"), cpu); diff --git a/tools/power/cpupower/utils/helpers/helpers.h b/tools/power/cpupower/utils/helpers/helpers.h index c258eeccd05f..37dac161f3fe 100644 --- a/tools/power/cpupower/utils/helpers/helpers.h +++ b/tools/power/cpupower/utils/helpers/helpers.h @@ -105,8 +105,8 @@ extern struct cpupower_cpu_info cpupower_cpu_info; extern int read_msr(int cpu, unsigned int idx, unsigned long long *val); extern int write_msr(int cpu, unsigned int idx, unsigned long long val); -extern int msr_intel_set_perf_bias(unsigned int cpu, unsigned int val); -extern int msr_intel_get_perf_bias(unsigned int cpu); +extern int cpupower_intel_set_perf_bias(unsigned int cpu, unsigned int val); +extern int cpupower_intel_get_perf_bias(unsigned int cpu); extern unsigned long long msr_intel_get_turbo_ratio(unsigned int cpu); /* Read/Write msr ****************************/ @@ -150,9 +150,9 @@ static inline int read_msr(int cpu, unsigned int idx, unsigned long long *val) { return -1; }; static inline int write_msr(int cpu, unsigned int idx, unsigned long long val) { return -1; }; -static inline int msr_intel_set_perf_bias(unsigned int cpu, unsigned int val) +static inline int cpupower_intel_set_perf_bias(unsigned int cpu, unsigned int val) { return -1; }; -static inline int msr_intel_get_perf_bias(unsigned int cpu) +static inline int cpupower_intel_get_perf_bias(unsigned int cpu) { return -1; }; static inline unsigned long long msr_intel_get_turbo_ratio(unsigned int cpu) { return 0; }; diff --git a/tools/power/cpupower/utils/helpers/misc.c b/tools/power/cpupower/utils/helpers/misc.c index f406adc40bad..e8f8f643a627 100644 --- a/tools/power/cpupower/utils/helpers/misc.c +++ b/tools/power/cpupower/utils/helpers/misc.c @@ -1,7 +1,15 @@ // SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include + #if defined(__i386__) || defined(__x86_64__) #include "helpers/helpers.h" +#include "helpers/sysfs.h" + +#include "cpupower_intern.h" #define MSR_AMD_HWCR 0xc0010015 @@ -40,4 +48,44 @@ int cpufreq_has_boost_support(unsigned int cpu, int *support, int *active, *support = *active = 1; return 0; } + +int cpupower_intel_get_perf_bias(unsigned int cpu) +{ + char linebuf[MAX_LINE_LEN]; + char path[SYSFS_PATH_MAX]; + unsigned long val; + char *endp; + + if (!(cpupower_cpu_info.caps & CPUPOWER_CAP_PERF_BIAS)) + return -1; + + snprintf(path, sizeof(path), PATH_TO_CPU "cpu%u/power/energy_perf_bias", cpu); + + if (cpupower_read_sysfs(path, linebuf, MAX_LINE_LEN) == 0) + return -1; + + val = strtol(linebuf, &endp, 0); + if (endp == linebuf || errno == ERANGE) + return -1; + + return val; +} + +int cpupower_intel_set_perf_bias(unsigned int cpu, unsigned int val) +{ + char path[SYSFS_PATH_MAX]; + char linebuf[3] = {}; + + if (!(cpupower_cpu_info.caps & CPUPOWER_CAP_PERF_BIAS)) + return -1; + + snprintf(path, sizeof(path), PATH_TO_CPU "cpu%u/power/energy_perf_bias", cpu); + snprintf(linebuf, sizeof(linebuf), "%d", val); + + if (cpupower_write_sysfs(path, linebuf, 3) <= 0) + return -1; + + return 0; +} + #endif /* #if defined(__i386__) || defined(__x86_64__) */ diff --git a/tools/power/cpupower/utils/helpers/msr.c b/tools/power/cpupower/utils/helpers/msr.c index ab9950748838..8b0b6be74bb8 100644 --- a/tools/power/cpupower/utils/helpers/msr.c +++ b/tools/power/cpupower/utils/helpers/msr.c @@ -11,7 +11,6 @@ /* Intel specific MSRs */ #define MSR_IA32_PERF_STATUS 0x198 #define MSR_IA32_MISC_ENABLES 0x1a0 -#define MSR_IA32_ENERGY_PERF_BIAS 0x1b0 #define MSR_NEHALEM_TURBO_RATIO_LIMIT 0x1ad /* @@ -73,33 +72,6 @@ int write_msr(int cpu, unsigned int idx, unsigned long long val) return -1; } -int msr_intel_get_perf_bias(unsigned int cpu) -{ - unsigned long long val; - int ret; - - if (!(cpupower_cpu_info.caps & CPUPOWER_CAP_PERF_BIAS)) - return -1; - - ret = read_msr(cpu, MSR_IA32_ENERGY_PERF_BIAS, &val); - if (ret) - return ret; - return val; -} - -int msr_intel_set_perf_bias(unsigned int cpu, unsigned int val) -{ - int ret; - - if (!(cpupower_cpu_info.caps & CPUPOWER_CAP_PERF_BIAS)) - return -1; - - ret = write_msr(cpu, MSR_IA32_ENERGY_PERF_BIAS, val); - if (ret) - return ret; - return 0; -} - unsigned long long msr_intel_get_turbo_ratio(unsigned int cpu) { unsigned long long val; diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c index f3a1746f7f45..389ea5209a83 100644 --- a/tools/power/x86/turbostat/turbostat.c +++ b/tools/power/x86/turbostat/turbostat.c @@ -1831,6 +1831,25 @@ int get_mp(int cpu, struct msr_counter *mp, unsigned long long *counterp) return 0; } +int get_epb(int cpu) +{ + char path[128 + PATH_BYTES]; + int ret, epb = -1; + FILE *fp; + + sprintf(path, "/sys/devices/system/cpu/cpu%d/power/energy_perf_bias", cpu); + + fp = fopen_or_die(path, "r"); + + ret = fscanf(fp, "%d", &epb); + if (ret != 1) + err(1, "%s(%s)", __func__, path); + + fclose(fp); + + return epb; +} + void get_apic_id(struct thread_data *t) { unsigned int eax, ebx, ecx, edx; @@ -3917,9 +3936,8 @@ dump_sysfs_pstate_config(void) */ int print_epb(struct thread_data *t, struct core_data *c, struct pkg_data *p) { - unsigned long long msr; char *epb_string; - int cpu; + int cpu, epb; if (!has_epb) return 0; @@ -3935,10 +3953,11 @@ int print_epb(struct thread_data *t, struct core_data *c, struct pkg_data *p) return -1; } - if (get_msr(cpu, MSR_IA32_ENERGY_PERF_BIAS, &msr)) + epb = get_epb(cpu); + if (epb < 0) return 0; - switch (msr & 0xF) { + switch (epb) { case ENERGY_PERF_BIAS_PERFORMANCE: epb_string = "performance"; break; @@ -3952,7 +3971,7 @@ int print_epb(struct thread_data *t, struct core_data *c, struct pkg_data *p) epb_string = "custom"; break; } - fprintf(outf, "cpu%d: MSR_IA32_ENERGY_PERF_BIAS: 0x%08llx (%s)\n", cpu, msr, epb_string); + fprintf(outf, "cpu%d: EPB: %d (%s)\n", cpu, epb, epb_string); return 0; } diff --git a/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c b/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c index ff6c6661f075..5fd9e594079c 100644 --- a/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c +++ b/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c @@ -91,6 +91,9 @@ unsigned int has_hwp_request_pkg; /* IA32_HWP_REQUEST_PKG */ unsigned int bdx_highest_ratio; +#define PATH_TO_CPU "/sys/devices/system/cpu/" +#define SYSFS_PATH_MAX 255 + /* * maintain compatibility with original implementation, but don't document it: */ @@ -721,6 +724,48 @@ int put_msr(int cpu, int offset, unsigned long long new_msr) return 0; } +static unsigned int read_sysfs(const char *path, char *buf, size_t buflen) +{ + ssize_t numread; + int fd; + + fd = open(path, O_RDONLY); + if (fd == -1) + return 0; + + numread = read(fd, buf, buflen - 1); + if (numread < 1) { + close(fd); + return 0; + } + + buf[numread] = '\0'; + close(fd); + + return (unsigned int) numread; +} + +static unsigned int write_sysfs(const char *path, char *buf, size_t buflen) +{ + ssize_t numwritten; + int fd; + + fd = open(path, O_WRONLY); + if (fd == -1) + return 0; + + numwritten = write(fd, buf, buflen - 1); + if (numwritten < 1) { + perror("write failed\n"); + close(fd); + return -1; + } + + close(fd); + + return (unsigned int) numwritten; +} + void print_hwp_cap(int cpu, struct msr_hwp_cap *cap, char *str) { if (cpu != -1) @@ -798,17 +843,61 @@ void write_hwp_request(int cpu, struct msr_hwp_request *hwp_req, unsigned int ms put_msr(cpu, msr_offset, msr); } +static int get_epb(int cpu) +{ + char path[SYSFS_PATH_MAX]; + char linebuf[3]; + char *endp; + long val; + + if (!has_epb) + return -1; + + snprintf(path, sizeof(path), PATH_TO_CPU "cpu%u/power/energy_perf_bias", cpu); + + if (!read_sysfs(path, linebuf, 3)) + return -1; + + val = strtol(linebuf, &endp, 0); + if (endp == linebuf || errno == ERANGE) + return -1; + + return (int)val; +} + +static int set_epb(int cpu, int val) +{ + char path[SYSFS_PATH_MAX]; + char linebuf[3]; + char *endp; + int ret; + + if (!has_epb) + return -1; + + snprintf(path, sizeof(path), PATH_TO_CPU "cpu%u/power/energy_perf_bias", cpu); + snprintf(linebuf, sizeof(linebuf), "%d", val); + + ret = write_sysfs(path, linebuf, 3); + if (ret <= 0) + return -1; + + val = strtol(linebuf, &endp, 0); + if (endp == linebuf || errno == ERANGE) + return -1; + + return (int)val; +} + int print_cpu_msrs(int cpu) { - unsigned long long msr; struct msr_hwp_request req; struct msr_hwp_cap cap; + int epb; - if (has_epb) { - get_msr(cpu, MSR_IA32_ENERGY_PERF_BIAS, &msr); - - printf("cpu%d: EPB %u\n", cpu, (unsigned int) msr); - } + epb = get_epb(cpu); + if (epb >= 0) + printf("cpu%d: EPB %u\n", cpu, (unsigned int) epb); if (!has_hwp) return 0; @@ -1091,15 +1180,15 @@ int enable_hwp_on_cpu(int cpu) int update_cpu_msrs(int cpu) { unsigned long long msr; - + int epb; if (update_epb) { - get_msr(cpu, MSR_IA32_ENERGY_PERF_BIAS, &msr); - put_msr(cpu, MSR_IA32_ENERGY_PERF_BIAS, new_epb); + epb = get_epb(cpu); + set_epb(cpu, new_epb); if (verbose) printf("cpu%d: ENERGY_PERF_BIAS old: %d new: %d\n", - cpu, (unsigned int) msr, (unsigned int) new_epb); + cpu, epb, (unsigned int) new_epb); } if (update_turbo) { diff --git a/tools/testing/nvdimm/test/nfit.c b/tools/testing/nvdimm/test/nfit.c index 2ac0fff6dad8..9b185bf82da8 100644 --- a/tools/testing/nvdimm/test/nfit.c +++ b/tools/testing/nvdimm/test/nfit.c @@ -23,7 +23,6 @@ #include "nfit_test.h" #include "../watermark.h" -#include #include /* @@ -3284,107 +3283,6 @@ static struct platform_driver nfit_test_driver = { .id_table = nfit_test_id, }; -static char copy_mc_buf[PAGE_SIZE] __attribute__((__aligned__(PAGE_SIZE))); - -enum INJECT { - INJECT_NONE, - INJECT_SRC, - INJECT_DST, -}; - -static void copy_mc_test_init(char *dst, char *src, size_t size) -{ - size_t i; - - memset(dst, 0xff, size); - for (i = 0; i < size; i++) - src[i] = (char) i; -} - -static bool copy_mc_test_validate(unsigned char *dst, unsigned char *src, - size_t size, unsigned long rem) -{ - size_t i; - - for (i = 0; i < size - rem; i++) - if (dst[i] != (unsigned char) i) { - pr_info_once("%s:%d: offset: %zd got: %#x expect: %#x\n", - __func__, __LINE__, i, dst[i], - (unsigned char) i); - return false; - } - for (i = size - rem; i < size; i++) - if (dst[i] != 0xffU) { - pr_info_once("%s:%d: offset: %zd got: %#x expect: 0xff\n", - __func__, __LINE__, i, dst[i]); - return false; - } - return true; -} - -void copy_mc_test(void) -{ - char *inject_desc[] = { "none", "source", "destination" }; - enum INJECT inj; - - if (IS_ENABLED(CONFIG_COPY_MC_TEST)) { - pr_info("%s: run...\n", __func__); - } else { - pr_info("%s: disabled, skip.\n", __func__); - return; - } - - for (inj = INJECT_NONE; inj <= INJECT_DST; inj++) { - int i; - - pr_info("%s: inject: %s\n", __func__, inject_desc[inj]); - for (i = 0; i < 512; i++) { - unsigned long expect, rem; - void *src, *dst; - bool valid; - - switch (inj) { - case INJECT_NONE: - copy_mc_inject_src(NULL); - copy_mc_inject_dst(NULL); - dst = ©_mc_buf[2048]; - src = ©_mc_buf[1024 - i]; - expect = 0; - break; - case INJECT_SRC: - copy_mc_inject_src(©_mc_buf[1024]); - copy_mc_inject_dst(NULL); - dst = ©_mc_buf[2048]; - src = ©_mc_buf[1024 - i]; - expect = 512 - i; - break; - case INJECT_DST: - copy_mc_inject_src(NULL); - copy_mc_inject_dst(©_mc_buf[2048]); - dst = ©_mc_buf[2048 - i]; - src = ©_mc_buf[1024]; - expect = 512 - i; - break; - } - - copy_mc_test_init(dst, src, 512); - rem = copy_mc_fragile(dst, src, 512); - valid = copy_mc_test_validate(dst, src, 512, expect); - if (rem == expect && valid) - continue; - pr_info("%s: copy(%#lx, %#lx, %d) off: %d rem: %ld %s expect: %ld\n", - __func__, - ((unsigned long) dst) & ~PAGE_MASK, - ((unsigned long ) src) & ~PAGE_MASK, - 512, i, rem, valid ? "valid" : "bad", - expect); - } - } - - copy_mc_inject_src(NULL); - copy_mc_inject_dst(NULL); -} - static __init int nfit_test_init(void) { int rc, i; @@ -3393,7 +3291,6 @@ static __init int nfit_test_init(void) libnvdimm_test(); acpi_nfit_test(); device_dax_test(); - copy_mc_test(); dax_pmem_test(); dax_pmem_core_test(); #ifdef CONFIG_DEV_DAX_PMEM_COMPAT diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index d9c283503159..2e20e30a6faa 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -50,6 +50,7 @@ TARGETS += openat2 TARGETS += rseq TARGETS += rtc TARGETS += seccomp +TARGETS += sgx TARGETS += sigaltstack TARGETS += size TARGETS += sparc64 diff --git a/tools/testing/selftests/sgx/.gitignore b/tools/testing/selftests/sgx/.gitignore new file mode 100644 index 000000000000..fbaf0bda9a92 --- /dev/null +++ b/tools/testing/selftests/sgx/.gitignore @@ -0,0 +1,2 @@ +test_sgx +test_encl.elf diff --git a/tools/testing/selftests/sgx/Makefile b/tools/testing/selftests/sgx/Makefile new file mode 100644 index 000000000000..7f12d55b97f8 --- /dev/null +++ b/tools/testing/selftests/sgx/Makefile @@ -0,0 +1,57 @@ +top_srcdir = ../../../.. + +include ../lib.mk + +.PHONY: all clean + +CAN_BUILD_X86_64 := $(shell ../x86/check_cc.sh $(CC) \ + ../x86/trivial_64bit_program.c) + +ifndef OBJCOPY +OBJCOPY := $(CROSS_COMPILE)objcopy +endif + +INCLUDES := -I$(top_srcdir)/tools/include +HOST_CFLAGS := -Wall -Werror -g $(INCLUDES) -fPIC -z noexecstack +ENCL_CFLAGS := -Wall -Werror -static -nostdlib -nostartfiles -fPIC \ + -fno-stack-protector -mrdrnd $(INCLUDES) + +TEST_CUSTOM_PROGS := $(OUTPUT)/test_sgx + +ifeq ($(CAN_BUILD_X86_64), 1) +all: $(TEST_CUSTOM_PROGS) $(OUTPUT)/test_encl.elf +endif + +$(OUTPUT)/test_sgx: $(OUTPUT)/main.o \ + $(OUTPUT)/load.o \ + $(OUTPUT)/sigstruct.o \ + $(OUTPUT)/call.o \ + $(OUTPUT)/sign_key.o + $(CC) $(HOST_CFLAGS) -o $@ $^ -lcrypto + +$(OUTPUT)/main.o: main.c + $(CC) $(HOST_CFLAGS) -c $< -o $@ + +$(OUTPUT)/load.o: load.c + $(CC) $(HOST_CFLAGS) -c $< -o $@ + +$(OUTPUT)/sigstruct.o: sigstruct.c + $(CC) $(HOST_CFLAGS) -c $< -o $@ + +$(OUTPUT)/call.o: call.S + $(CC) $(HOST_CFLAGS) -c $< -o $@ + +$(OUTPUT)/sign_key.o: sign_key.S + $(CC) $(HOST_CFLAGS) -c $< -o $@ + +$(OUTPUT)/test_encl.elf: test_encl.lds test_encl.c test_encl_bootstrap.S + $(CC) $(ENCL_CFLAGS) -T $^ -o $@ + +EXTRA_CLEAN := \ + $(OUTPUT)/test_encl.elf \ + $(OUTPUT)/load.o \ + $(OUTPUT)/call.o \ + $(OUTPUT)/main.o \ + $(OUTPUT)/sigstruct.o \ + $(OUTPUT)/test_sgx \ + $(OUTPUT)/test_sgx.o \ diff --git a/tools/testing/selftests/sgx/call.S b/tools/testing/selftests/sgx/call.S new file mode 100644 index 000000000000..4ecadc7490f4 --- /dev/null +++ b/tools/testing/selftests/sgx/call.S @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/** +* Copyright(c) 2016-20 Intel Corporation. +*/ + + .text + + .global sgx_call_vdso +sgx_call_vdso: + .cfi_startproc + push %r15 + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset %r15, 0 + push %r14 + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset %r14, 0 + push %r13 + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset %r13, 0 + push %r12 + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset %r12, 0 + push %rbx + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset %rbx, 0 + push $0 + .cfi_adjust_cfa_offset 8 + push 0x38(%rsp) + .cfi_adjust_cfa_offset 8 + call *eenter(%rip) + add $0x10, %rsp + .cfi_adjust_cfa_offset -0x10 + pop %rbx + .cfi_adjust_cfa_offset -8 + pop %r12 + .cfi_adjust_cfa_offset -8 + pop %r13 + .cfi_adjust_cfa_offset -8 + pop %r14 + .cfi_adjust_cfa_offset -8 + pop %r15 + .cfi_adjust_cfa_offset -8 + ret + .cfi_endproc diff --git a/tools/testing/selftests/sgx/defines.h b/tools/testing/selftests/sgx/defines.h new file mode 100644 index 000000000000..592c1ccf4576 --- /dev/null +++ b/tools/testing/selftests/sgx/defines.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright(c) 2016-20 Intel Corporation. + */ + +#ifndef DEFINES_H +#define DEFINES_H + +#include + +#define PAGE_SIZE 4096 +#define PAGE_MASK (~(PAGE_SIZE - 1)) + +#define __aligned(x) __attribute__((__aligned__(x))) +#define __packed __attribute__((packed)) + +#include "../../../../arch/x86/kernel/cpu/sgx/arch.h" +#include "../../../../arch/x86/include/asm/enclu.h" +#include "../../../../arch/x86/include/uapi/asm/sgx.h" + +#endif /* DEFINES_H */ diff --git a/tools/testing/selftests/sgx/load.c b/tools/testing/selftests/sgx/load.c new file mode 100644 index 000000000000..9d43b75aaa55 --- /dev/null +++ b/tools/testing/selftests/sgx/load.c @@ -0,0 +1,277 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2016-20 Intel Corporation. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "defines.h" +#include "main.h" + +void encl_delete(struct encl *encl) +{ + if (encl->encl_base) + munmap((void *)encl->encl_base, encl->encl_size); + + if (encl->bin) + munmap(encl->bin, encl->bin_size); + + if (encl->fd) + close(encl->fd); + + if (encl->segment_tbl) + free(encl->segment_tbl); + + memset(encl, 0, sizeof(*encl)); +} + +static bool encl_map_bin(const char *path, struct encl *encl) +{ + struct stat sb; + void *bin; + int ret; + int fd; + + fd = open(path, O_RDONLY); + if (fd == -1) { + perror("open()"); + return false; + } + + ret = stat(path, &sb); + if (ret) { + perror("stat()"); + goto err; + } + + bin = mmap(NULL, sb.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + if (bin == MAP_FAILED) { + perror("mmap()"); + goto err; + } + + encl->bin = bin; + encl->bin_size = sb.st_size; + + close(fd); + return true; + +err: + close(fd); + return false; +} + +static bool encl_ioc_create(struct encl *encl) +{ + struct sgx_secs *secs = &encl->secs; + struct sgx_enclave_create ioc; + int rc; + + assert(encl->encl_base != 0); + + memset(secs, 0, sizeof(*secs)); + secs->ssa_frame_size = 1; + secs->attributes = SGX_ATTR_MODE64BIT; + secs->xfrm = 3; + secs->base = encl->encl_base; + secs->size = encl->encl_size; + + ioc.src = (unsigned long)secs; + rc = ioctl(encl->fd, SGX_IOC_ENCLAVE_CREATE, &ioc); + if (rc) { + fprintf(stderr, "SGX_IOC_ENCLAVE_CREATE failed: errno=%d\n", + errno); + munmap((void *)secs->base, encl->encl_size); + return false; + } + + return true; +} + +static bool encl_ioc_add_pages(struct encl *encl, struct encl_segment *seg) +{ + struct sgx_enclave_add_pages ioc; + struct sgx_secinfo secinfo; + int rc; + + memset(&secinfo, 0, sizeof(secinfo)); + secinfo.flags = seg->flags; + + ioc.src = (uint64_t)encl->src + seg->offset; + ioc.offset = seg->offset; + ioc.length = seg->size; + ioc.secinfo = (unsigned long)&secinfo; + ioc.flags = SGX_PAGE_MEASURE; + + rc = ioctl(encl->fd, SGX_IOC_ENCLAVE_ADD_PAGES, &ioc); + if (rc < 0) { + fprintf(stderr, "SGX_IOC_ENCLAVE_ADD_PAGES failed: errno=%d.\n", + errno); + return false; + } + + return true; +} + +bool encl_load(const char *path, struct encl *encl) +{ + Elf64_Phdr *phdr_tbl; + off_t src_offset; + Elf64_Ehdr *ehdr; + int i, j; + int ret; + + memset(encl, 0, sizeof(*encl)); + + ret = open("/dev/sgx_enclave", O_RDWR); + if (ret < 0) { + fprintf(stderr, "Unable to open /dev/sgx_enclave\n"); + goto err; + } + + encl->fd = ret; + + if (!encl_map_bin(path, encl)) + goto err; + + ehdr = encl->bin; + phdr_tbl = encl->bin + ehdr->e_phoff; + + for (i = 0; i < ehdr->e_phnum; i++) { + Elf64_Phdr *phdr = &phdr_tbl[i]; + + if (phdr->p_type == PT_LOAD) + encl->nr_segments++; + } + + encl->segment_tbl = calloc(encl->nr_segments, + sizeof(struct encl_segment)); + if (!encl->segment_tbl) + goto err; + + for (i = 0, j = 0; i < ehdr->e_phnum; i++) { + Elf64_Phdr *phdr = &phdr_tbl[i]; + unsigned int flags = phdr->p_flags; + struct encl_segment *seg; + + if (phdr->p_type != PT_LOAD) + continue; + + seg = &encl->segment_tbl[j]; + + if (!!(flags & ~(PF_R | PF_W | PF_X))) { + fprintf(stderr, + "%d has invalid segment flags 0x%02x.\n", i, + phdr->p_flags); + goto err; + } + + if (j == 0 && flags != (PF_R | PF_W)) { + fprintf(stderr, + "TCS has invalid segment flags 0x%02x.\n", + phdr->p_flags); + goto err; + } + + if (j == 0) { + src_offset = phdr->p_offset & PAGE_MASK; + + seg->prot = PROT_READ | PROT_WRITE; + seg->flags = SGX_PAGE_TYPE_TCS << 8; + } else { + seg->prot = (phdr->p_flags & PF_R) ? PROT_READ : 0; + seg->prot |= (phdr->p_flags & PF_W) ? PROT_WRITE : 0; + seg->prot |= (phdr->p_flags & PF_X) ? PROT_EXEC : 0; + seg->flags = (SGX_PAGE_TYPE_REG << 8) | seg->prot; + } + + seg->offset = (phdr->p_offset & PAGE_MASK) - src_offset; + seg->size = (phdr->p_filesz + PAGE_SIZE - 1) & PAGE_MASK; + + printf("0x%016lx 0x%016lx 0x%02x\n", seg->offset, seg->size, + seg->prot); + + j++; + } + + assert(j == encl->nr_segments); + + encl->src = encl->bin + src_offset; + encl->src_size = encl->segment_tbl[j - 1].offset + + encl->segment_tbl[j - 1].size; + + for (encl->encl_size = 4096; encl->encl_size < encl->src_size; ) + encl->encl_size <<= 1; + + return true; + +err: + encl_delete(encl); + return false; +} + +static bool encl_map_area(struct encl *encl) +{ + size_t encl_size = encl->encl_size; + void *area; + + area = mmap(NULL, encl_size * 2, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (area == MAP_FAILED) { + perror("mmap"); + return false; + } + + encl->encl_base = ((uint64_t)area + encl_size - 1) & ~(encl_size - 1); + + munmap(area, encl->encl_base - (uint64_t)area); + munmap((void *)(encl->encl_base + encl_size), + (uint64_t)area + encl_size - encl->encl_base); + + return true; +} + +bool encl_build(struct encl *encl) +{ + struct sgx_enclave_init ioc; + int ret; + int i; + + if (!encl_map_area(encl)) + return false; + + if (!encl_ioc_create(encl)) + return false; + + /* + * Pages must be added before mapping VMAs because their permissions + * cap the VMA permissions. + */ + for (i = 0; i < encl->nr_segments; i++) { + struct encl_segment *seg = &encl->segment_tbl[i]; + + if (!encl_ioc_add_pages(encl, seg)) + return false; + } + + ioc.sigstruct = (uint64_t)&encl->sigstruct; + ret = ioctl(encl->fd, SGX_IOC_ENCLAVE_INIT, &ioc); + if (ret) { + fprintf(stderr, "SGX_IOC_ENCLAVE_INIT failed: errno=%d\n", + errno); + return false; + } + + return true; +} diff --git a/tools/testing/selftests/sgx/main.c b/tools/testing/selftests/sgx/main.c new file mode 100644 index 000000000000..724cec700926 --- /dev/null +++ b/tools/testing/selftests/sgx/main.c @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2016-20 Intel Corporation. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "defines.h" +#include "main.h" +#include "../kselftest.h" + +static const uint64_t MAGIC = 0x1122334455667788ULL; +vdso_sgx_enter_enclave_t eenter; + +struct vdso_symtab { + Elf64_Sym *elf_symtab; + const char *elf_symstrtab; + Elf64_Word *elf_hashtab; +}; + +static void *vdso_get_base_addr(char *envp[]) +{ + Elf64_auxv_t *auxv; + int i; + + for (i = 0; envp[i]; i++) + ; + + auxv = (Elf64_auxv_t *)&envp[i + 1]; + + for (i = 0; auxv[i].a_type != AT_NULL; i++) { + if (auxv[i].a_type == AT_SYSINFO_EHDR) + return (void *)auxv[i].a_un.a_val; + } + + return NULL; +} + +static Elf64_Dyn *vdso_get_dyntab(void *addr) +{ + Elf64_Ehdr *ehdr = addr; + Elf64_Phdr *phdrtab = addr + ehdr->e_phoff; + int i; + + for (i = 0; i < ehdr->e_phnum; i++) + if (phdrtab[i].p_type == PT_DYNAMIC) + return addr + phdrtab[i].p_offset; + + return NULL; +} + +static void *vdso_get_dyn(void *addr, Elf64_Dyn *dyntab, Elf64_Sxword tag) +{ + int i; + + for (i = 0; dyntab[i].d_tag != DT_NULL; i++) + if (dyntab[i].d_tag == tag) + return addr + dyntab[i].d_un.d_ptr; + + return NULL; +} + +static bool vdso_get_symtab(void *addr, struct vdso_symtab *symtab) +{ + Elf64_Dyn *dyntab = vdso_get_dyntab(addr); + + symtab->elf_symtab = vdso_get_dyn(addr, dyntab, DT_SYMTAB); + if (!symtab->elf_symtab) + return false; + + symtab->elf_symstrtab = vdso_get_dyn(addr, dyntab, DT_STRTAB); + if (!symtab->elf_symstrtab) + return false; + + symtab->elf_hashtab = vdso_get_dyn(addr, dyntab, DT_HASH); + if (!symtab->elf_hashtab) + return false; + + return true; +} + +static unsigned long elf_sym_hash(const char *name) +{ + unsigned long h = 0, high; + + while (*name) { + h = (h << 4) + *name++; + high = h & 0xf0000000; + + if (high) + h ^= high >> 24; + + h &= ~high; + } + + return h; +} + +static Elf64_Sym *vdso_symtab_get(struct vdso_symtab *symtab, const char *name) +{ + Elf64_Word bucketnum = symtab->elf_hashtab[0]; + Elf64_Word *buckettab = &symtab->elf_hashtab[2]; + Elf64_Word *chaintab = &symtab->elf_hashtab[2 + bucketnum]; + Elf64_Sym *sym; + Elf64_Word i; + + for (i = buckettab[elf_sym_hash(name) % bucketnum]; i != STN_UNDEF; + i = chaintab[i]) { + sym = &symtab->elf_symtab[i]; + if (!strcmp(name, &symtab->elf_symstrtab[sym->st_name])) + return sym; + } + + return NULL; +} + +bool report_results(struct sgx_enclave_run *run, int ret, uint64_t result, + const char *test) +{ + bool valid = true; + + if (ret) { + printf("FAIL: %s() returned: %d\n", test, ret); + valid = false; + } + + if (run->function != EEXIT) { + printf("FAIL: %s() function, expected: %u, got: %u\n", test, EEXIT, + run->function); + valid = false; + } + + if (result != MAGIC) { + printf("FAIL: %s(), expected: 0x%lx, got: 0x%lx\n", test, MAGIC, + result); + valid = false; + } + + if (run->user_data) { + printf("FAIL: %s() user data, expected: 0x0, got: 0x%llx\n", + test, run->user_data); + valid = false; + } + + return valid; +} + +static int user_handler(long rdi, long rsi, long rdx, long ursp, long r8, long r9, + struct sgx_enclave_run *run) +{ + run->user_data = 0; + return 0; +} + +int main(int argc, char *argv[], char *envp[]) +{ + struct sgx_enclave_run run; + struct vdso_symtab symtab; + Elf64_Sym *eenter_sym; + uint64_t result = 0; + struct encl encl; + unsigned int i; + void *addr; + int ret; + + memset(&run, 0, sizeof(run)); + + if (!encl_load("test_encl.elf", &encl)) { + encl_delete(&encl); + ksft_exit_skip("cannot load enclaves\n"); + } + + if (!encl_measure(&encl)) + goto err; + + if (!encl_build(&encl)) + goto err; + + /* + * An enclave consumer only must do this. + */ + for (i = 0; i < encl.nr_segments; i++) { + struct encl_segment *seg = &encl.segment_tbl[i]; + + addr = mmap((void *)encl.encl_base + seg->offset, seg->size, + seg->prot, MAP_SHARED | MAP_FIXED, encl.fd, 0); + if (addr == MAP_FAILED) { + fprintf(stderr, "mmap() failed, errno=%d.\n", errno); + exit(KSFT_FAIL); + } + } + + memset(&run, 0, sizeof(run)); + run.tcs = encl.encl_base; + + addr = vdso_get_base_addr(envp); + if (!addr) + goto err; + + if (!vdso_get_symtab(addr, &symtab)) + goto err; + + eenter_sym = vdso_symtab_get(&symtab, "__vdso_sgx_enter_enclave"); + if (!eenter_sym) + goto err; + + eenter = addr + eenter_sym->st_value; + + ret = sgx_call_vdso((void *)&MAGIC, &result, 0, EENTER, NULL, NULL, &run); + if (!report_results(&run, ret, result, "sgx_call_vdso")) + goto err; + + + /* Invoke the vDSO directly. */ + result = 0; + ret = eenter((unsigned long)&MAGIC, (unsigned long)&result, 0, EENTER, + 0, 0, &run); + if (!report_results(&run, ret, result, "eenter")) + goto err; + + /* And with an exit handler. */ + run.user_handler = (__u64)user_handler; + run.user_data = 0xdeadbeef; + ret = eenter((unsigned long)&MAGIC, (unsigned long)&result, 0, EENTER, + 0, 0, &run); + if (!report_results(&run, ret, result, "user_handler")) + goto err; + + printf("SUCCESS\n"); + encl_delete(&encl); + exit(KSFT_PASS); + +err: + encl_delete(&encl); + exit(KSFT_FAIL); +} diff --git a/tools/testing/selftests/sgx/main.h b/tools/testing/selftests/sgx/main.h new file mode 100644 index 000000000000..67211a708f04 --- /dev/null +++ b/tools/testing/selftests/sgx/main.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright(c) 2016-20 Intel Corporation. + */ + +#ifndef MAIN_H +#define MAIN_H + +struct encl_segment { + off_t offset; + size_t size; + unsigned int prot; + unsigned int flags; +}; + +struct encl { + int fd; + void *bin; + off_t bin_size; + void *src; + size_t src_size; + size_t encl_size; + off_t encl_base; + unsigned int nr_segments; + struct encl_segment *segment_tbl; + struct sgx_secs secs; + struct sgx_sigstruct sigstruct; +}; + +extern unsigned char sign_key[]; +extern unsigned char sign_key_end[]; + +void encl_delete(struct encl *ctx); +bool encl_load(const char *path, struct encl *encl); +bool encl_measure(struct encl *encl); +bool encl_build(struct encl *encl); + +int sgx_call_vdso(void *rdi, void *rsi, long rdx, u32 function, void *r8, void *r9, + struct sgx_enclave_run *run); + +#endif /* MAIN_H */ diff --git a/tools/testing/selftests/sgx/sign_key.S b/tools/testing/selftests/sgx/sign_key.S new file mode 100644 index 000000000000..e4fbe948444a --- /dev/null +++ b/tools/testing/selftests/sgx/sign_key.S @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/** +* Copyright(c) 2016-20 Intel Corporation. +*/ + + .section ".rodata", "a" + +sign_key: + .globl sign_key + .incbin "sign_key.pem" +sign_key_end: + .globl sign_key_end diff --git a/tools/testing/selftests/sgx/sign_key.pem b/tools/testing/selftests/sgx/sign_key.pem new file mode 100644 index 000000000000..d76f21f19187 --- /dev/null +++ b/tools/testing/selftests/sgx/sign_key.pem @@ -0,0 +1,39 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIG4wIBAAKCAYEApalGbq7Q+usM91CPtksu3D+b0Prc8gAFL6grM3mg85A5Bx8V +cfMXPgtrw8EYFwQxDAvzZWwl+9VfOX0ECrFRBkOHcOiG0SnADN8+FLj1UiNUQwbp +S6OzhNWuRcSbGraSOyUlVlV0yMQSvewyzGklOaXBe30AJqzIBc8QfdSxKuP8rs0Z +ga6k/Bl73osrYKByILJTUUeZqjLERsE6GebsdzbWgKn8qVqng4ZS4yMNg6LeRlH3 ++9CIPgg4jwpSLHcp7dq2qTIB9a0tGe9ayp+5FbucpB6U7ePold0EeRN6RlJGDF9k +L93v8P5ykz5G5gYZ2g0K1X2sHIWV4huxPgv5PXgdyQYbK+6olqj0d5rjYuwX57Ul +k6SroPS1U6UbdCjG5txM+BNGU0VpD0ZhrIRw0leQdnNcCO9sTJuInZrgYacSVJ7u +mtB+uCt+uzUesc+l+xPRYA+9e14lLkZp7AAmo9FvL816XDI09deehJ3i/LmHKCRN +tuqC5TprRjFwUr6dAgEDAoIBgG5w2Z8fNfycs0+LCnmHdJLVEotR6KFVWMpwHMz7 +wKJgJgS/Y6FMuilc8oKAuroCy11dTO5IGVKOP3uorVx2NgQtBPXwWeDGgAiU1A3Q +o4wXjYIEm4fCd63jyYPYZ2ckYXzDbjmOTdstYdPyzIhGGNEZK6eoqsRzMAPfYFPj +IMdCqHSIu6vJw1K7p+myHOsVoWshjODaZnF3LYSA0WaZ8vokjwBxUxuRxQJZjJds +s60XPtmL+qfgWtQFewoG4XL6GuD8FcXccynRRtzrLtFNPIl9BQfWfjBBhTC1/Te1 +0Z6XbZvpdUTD9OfLB7SbR2OUFNpKQgriO0iYVdbW3cr7uu38Zwp4W1TX73DPjoi6 +KNooP6SGWd4mRJW2+dUmSYS4QNG8eVVZswKcploEIXlAKRsOe4kzJJ1iETugIe85 +uX8nd1WYEp65xwoRUg8hqng0MeyveVbXqNKuJG6tzNDt9kgFYo+hmC/oouAW2Dtc +T9jdRAwKJXqA2Eg6OkgXCEv+kwKBwQDYaQiFMlFhsmLlqI+EzCUh7c941/cL7m6U +7j98+8ngl0HgCEcrc10iJVCKakQW3YbPzAx3XkKTaGjWazvvrFarXIGlOud64B8a +iWyQ7VdlnmZnNEdk+C83tI91OQeaTKqRLDGzKh29Ry/jL8Pcbazt+kDgxa0H7qJp +roADUanLQuNkYubpbhFBh3xpa2EExaVq6rF7nIVsD8W9TrbmPKA4LgH7z0iy544D +kVCNYsTjYDdUWP+WiSor8kCnnpjnN9sCgcEAw/eNezUD1UDf6OYFC9+5JZJFn4Tg +mZMyN93JKIb199ffwnjtHUSjcyiWeesXucpzwtGbTcwQnDisSW4oneYKLSEBlBaq +scqiUugyGZZOthFSCbdXYXMViK2vHrKlkse7GxVlROKcEhM/pRBrmjaGO8eWR+D4 +FO2wCXzVs3KgV6j779frw0vC54oHOxc9+Lu1rSHp4i+600koyvL/zF6U/5tZXIvN +YW2yoiQJnjCmVA1pwbwV6KAUTPDTMnBK+YjnAoHBAJBGBa4hi5Z27JkbCliIGMFJ +NPs6pLKe9GNJf6in2+sPgUAFhMeiPhbDiwbxgrnpBIqICE+ULGJFmzmc0p/IOceT +ARjR76dAFLxbnbXzj5kURETNhO36yiUjCk4mBRGIcbYddndxaSjaH+zKgpLzyJ6m +1esuc1qfFvEfAAI2cTIsl5hB70ZJYNZaUvDyQK3ZGPHxy6e9rkgKg9OJz0QoatAe +q/002yHvtAJg4F5B2JeVejg7VQ8GHB1MKxppu0TP5wKBwQCCpQj8zgKOKz/wmViy +lSYZDC5qWJW7t3bP6TDFr06lOpUsUJ4TgxeiGw778g/RMaKB4RIz3WBoJcgw9BsT +7rFza1ZiucchMcGMmswRDt8kC4wGejpA92Owc8oUdxkMhSdnY5jYlxK2t3/DYEe8 +JFl9L7mFQKVjSSAGUzkiTGrlG1Kf5UfXh9dFBq98uilQfSPIwUaWynyM23CHTKqI +Pw3/vOY9sojrnncWwrEUIG7is5vWfWPwargzSzd29YdRBe8CgcEAuRVewK/YeNOX +B7ZG6gKKsfsvrGtY7FPETzLZAHjoVXYNea4LVZ2kn4hBXXlvw/4HD+YqcTt4wmif +5JQlDvjNobUiKJZpzy7hklVhF7wZFl4pCF7Yh43q9iQ7gKTaeUG7MiaK+G8Zz8aY +HW9rsiihbdZkccMvnPfO9334XMxl3HtBRzLstjUlbLB7Sdh+7tZ3JQidCOFNs5pE +XyWwnASPu4tKfDahH1UUTp1uJcq/6716CSWg080avYxFcn75qqsb +-----END RSA PRIVATE KEY----- diff --git a/tools/testing/selftests/sgx/sigstruct.c b/tools/testing/selftests/sgx/sigstruct.c new file mode 100644 index 000000000000..dee7a3d6c5a5 --- /dev/null +++ b/tools/testing/selftests/sgx/sigstruct.c @@ -0,0 +1,381 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2016-20 Intel Corporation. */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "defines.h" +#include "main.h" + +struct q1q2_ctx { + BN_CTX *bn_ctx; + BIGNUM *m; + BIGNUM *s; + BIGNUM *q1; + BIGNUM *qr; + BIGNUM *q2; +}; + +static void free_q1q2_ctx(struct q1q2_ctx *ctx) +{ + BN_CTX_free(ctx->bn_ctx); + BN_free(ctx->m); + BN_free(ctx->s); + BN_free(ctx->q1); + BN_free(ctx->qr); + BN_free(ctx->q2); +} + +static bool alloc_q1q2_ctx(const uint8_t *s, const uint8_t *m, + struct q1q2_ctx *ctx) +{ + ctx->bn_ctx = BN_CTX_new(); + ctx->s = BN_bin2bn(s, SGX_MODULUS_SIZE, NULL); + ctx->m = BN_bin2bn(m, SGX_MODULUS_SIZE, NULL); + ctx->q1 = BN_new(); + ctx->qr = BN_new(); + ctx->q2 = BN_new(); + + if (!ctx->bn_ctx || !ctx->s || !ctx->m || !ctx->q1 || !ctx->qr || + !ctx->q2) { + free_q1q2_ctx(ctx); + return false; + } + + return true; +} + +static bool calc_q1q2(const uint8_t *s, const uint8_t *m, uint8_t *q1, + uint8_t *q2) +{ + struct q1q2_ctx ctx; + + if (!alloc_q1q2_ctx(s, m, &ctx)) { + fprintf(stderr, "Not enough memory for Q1Q2 calculation\n"); + return false; + } + + if (!BN_mul(ctx.q1, ctx.s, ctx.s, ctx.bn_ctx)) + goto out; + + if (!BN_div(ctx.q1, ctx.qr, ctx.q1, ctx.m, ctx.bn_ctx)) + goto out; + + if (BN_num_bytes(ctx.q1) > SGX_MODULUS_SIZE) { + fprintf(stderr, "Too large Q1 %d bytes\n", + BN_num_bytes(ctx.q1)); + goto out; + } + + if (!BN_mul(ctx.q2, ctx.s, ctx.qr, ctx.bn_ctx)) + goto out; + + if (!BN_div(ctx.q2, NULL, ctx.q2, ctx.m, ctx.bn_ctx)) + goto out; + + if (BN_num_bytes(ctx.q2) > SGX_MODULUS_SIZE) { + fprintf(stderr, "Too large Q2 %d bytes\n", + BN_num_bytes(ctx.q2)); + goto out; + } + + BN_bn2bin(ctx.q1, q1); + BN_bn2bin(ctx.q2, q2); + + free_q1q2_ctx(&ctx); + return true; +out: + free_q1q2_ctx(&ctx); + return false; +} + +struct sgx_sigstruct_payload { + struct sgx_sigstruct_header header; + struct sgx_sigstruct_body body; +}; + +static bool check_crypto_errors(void) +{ + int err; + bool had_errors = false; + const char *filename; + int line; + char str[256]; + + for ( ; ; ) { + if (ERR_peek_error() == 0) + break; + + had_errors = true; + err = ERR_get_error_line(&filename, &line); + ERR_error_string_n(err, str, sizeof(str)); + fprintf(stderr, "crypto: %s: %s:%d\n", str, filename, line); + } + + return had_errors; +} + +static inline const BIGNUM *get_modulus(RSA *key) +{ + const BIGNUM *n; + + RSA_get0_key(key, &n, NULL, NULL); + return n; +} + +static RSA *gen_sign_key(void) +{ + unsigned long sign_key_length; + BIO *bio; + RSA *key; + + sign_key_length = (unsigned long)&sign_key_end - + (unsigned long)&sign_key; + + bio = BIO_new_mem_buf(&sign_key, sign_key_length); + if (!bio) + return NULL; + + key = PEM_read_bio_RSAPrivateKey(bio, NULL, NULL, NULL); + BIO_free(bio); + + return key; +} + +static void reverse_bytes(void *data, int length) +{ + int i = 0; + int j = length - 1; + uint8_t temp; + uint8_t *ptr = data; + + while (i < j) { + temp = ptr[i]; + ptr[i] = ptr[j]; + ptr[j] = temp; + i++; + j--; + } +} + +enum mrtags { + MRECREATE = 0x0045544145524345, + MREADD = 0x0000000044444145, + MREEXTEND = 0x00444E4554584545, +}; + +static bool mrenclave_update(EVP_MD_CTX *ctx, const void *data) +{ + if (!EVP_DigestUpdate(ctx, data, 64)) { + fprintf(stderr, "digest update failed\n"); + return false; + } + + return true; +} + +static bool mrenclave_commit(EVP_MD_CTX *ctx, uint8_t *mrenclave) +{ + unsigned int size; + + if (!EVP_DigestFinal_ex(ctx, (unsigned char *)mrenclave, &size)) { + fprintf(stderr, "digest commit failed\n"); + return false; + } + + if (size != 32) { + fprintf(stderr, "invalid digest size = %u\n", size); + return false; + } + + return true; +} + +struct mrecreate { + uint64_t tag; + uint32_t ssaframesize; + uint64_t size; + uint8_t reserved[44]; +} __attribute__((__packed__)); + + +static bool mrenclave_ecreate(EVP_MD_CTX *ctx, uint64_t blob_size) +{ + struct mrecreate mrecreate; + uint64_t encl_size; + + for (encl_size = 0x1000; encl_size < blob_size; ) + encl_size <<= 1; + + memset(&mrecreate, 0, sizeof(mrecreate)); + mrecreate.tag = MRECREATE; + mrecreate.ssaframesize = 1; + mrecreate.size = encl_size; + + if (!EVP_DigestInit_ex(ctx, EVP_sha256(), NULL)) + return false; + + return mrenclave_update(ctx, &mrecreate); +} + +struct mreadd { + uint64_t tag; + uint64_t offset; + uint64_t flags; /* SECINFO flags */ + uint8_t reserved[40]; +} __attribute__((__packed__)); + +static bool mrenclave_eadd(EVP_MD_CTX *ctx, uint64_t offset, uint64_t flags) +{ + struct mreadd mreadd; + + memset(&mreadd, 0, sizeof(mreadd)); + mreadd.tag = MREADD; + mreadd.offset = offset; + mreadd.flags = flags; + + return mrenclave_update(ctx, &mreadd); +} + +struct mreextend { + uint64_t tag; + uint64_t offset; + uint8_t reserved[48]; +} __attribute__((__packed__)); + +static bool mrenclave_eextend(EVP_MD_CTX *ctx, uint64_t offset, + const uint8_t *data) +{ + struct mreextend mreextend; + int i; + + for (i = 0; i < 0x1000; i += 0x100) { + memset(&mreextend, 0, sizeof(mreextend)); + mreextend.tag = MREEXTEND; + mreextend.offset = offset + i; + + if (!mrenclave_update(ctx, &mreextend)) + return false; + + if (!mrenclave_update(ctx, &data[i + 0x00])) + return false; + + if (!mrenclave_update(ctx, &data[i + 0x40])) + return false; + + if (!mrenclave_update(ctx, &data[i + 0x80])) + return false; + + if (!mrenclave_update(ctx, &data[i + 0xC0])) + return false; + } + + return true; +} + +static bool mrenclave_segment(EVP_MD_CTX *ctx, struct encl *encl, + struct encl_segment *seg) +{ + uint64_t end = seg->offset + seg->size; + uint64_t offset; + + for (offset = seg->offset; offset < end; offset += PAGE_SIZE) { + if (!mrenclave_eadd(ctx, offset, seg->flags)) + return false; + + if (!mrenclave_eextend(ctx, offset, encl->src + offset)) + return false; + } + + return true; +} + +bool encl_measure(struct encl *encl) +{ + uint64_t header1[2] = {0x000000E100000006, 0x0000000000010000}; + uint64_t header2[2] = {0x0000006000000101, 0x0000000100000060}; + struct sgx_sigstruct *sigstruct = &encl->sigstruct; + struct sgx_sigstruct_payload payload; + uint8_t digest[SHA256_DIGEST_LENGTH]; + unsigned int siglen; + RSA *key = NULL; + EVP_MD_CTX *ctx; + int i; + + memset(sigstruct, 0, sizeof(*sigstruct)); + + sigstruct->header.header1[0] = header1[0]; + sigstruct->header.header1[1] = header1[1]; + sigstruct->header.header2[0] = header2[0]; + sigstruct->header.header2[1] = header2[1]; + sigstruct->exponent = 3; + sigstruct->body.attributes = SGX_ATTR_MODE64BIT; + sigstruct->body.xfrm = 3; + + /* sanity check */ + if (check_crypto_errors()) + goto err; + + key = gen_sign_key(); + if (!key) { + ERR_print_errors_fp(stdout); + goto err; + } + + BN_bn2bin(get_modulus(key), sigstruct->modulus); + + ctx = EVP_MD_CTX_create(); + if (!ctx) + goto err; + + if (!mrenclave_ecreate(ctx, encl->src_size)) + goto err; + + for (i = 0; i < encl->nr_segments; i++) { + struct encl_segment *seg = &encl->segment_tbl[i]; + + if (!mrenclave_segment(ctx, encl, seg)) + goto err; + } + + if (!mrenclave_commit(ctx, sigstruct->body.mrenclave)) + goto err; + + memcpy(&payload.header, &sigstruct->header, sizeof(sigstruct->header)); + memcpy(&payload.body, &sigstruct->body, sizeof(sigstruct->body)); + + SHA256((unsigned char *)&payload, sizeof(payload), digest); + + if (!RSA_sign(NID_sha256, digest, SHA256_DIGEST_LENGTH, + sigstruct->signature, &siglen, key)) + goto err; + + if (!calc_q1q2(sigstruct->signature, sigstruct->modulus, sigstruct->q1, + sigstruct->q2)) + goto err; + + /* BE -> LE */ + reverse_bytes(sigstruct->signature, SGX_MODULUS_SIZE); + reverse_bytes(sigstruct->modulus, SGX_MODULUS_SIZE); + reverse_bytes(sigstruct->q1, SGX_MODULUS_SIZE); + reverse_bytes(sigstruct->q2, SGX_MODULUS_SIZE); + + EVP_MD_CTX_destroy(ctx); + RSA_free(key); + return true; + +err: + EVP_MD_CTX_destroy(ctx); + RSA_free(key); + return false; +} diff --git a/tools/testing/selftests/sgx/test_encl.c b/tools/testing/selftests/sgx/test_encl.c new file mode 100644 index 000000000000..cf25b5dc1e03 --- /dev/null +++ b/tools/testing/selftests/sgx/test_encl.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2016-20 Intel Corporation. */ + +#include +#include "defines.h" + +static void *memcpy(void *dest, const void *src, size_t n) +{ + size_t i; + + for (i = 0; i < n; i++) + ((char *)dest)[i] = ((char *)src)[i]; + + return dest; +} + +void encl_body(void *rdi, void *rsi) +{ + memcpy(rsi, rdi, 8); +} diff --git a/tools/testing/selftests/sgx/test_encl.lds b/tools/testing/selftests/sgx/test_encl.lds new file mode 100644 index 000000000000..0fbbda7e665e --- /dev/null +++ b/tools/testing/selftests/sgx/test_encl.lds @@ -0,0 +1,40 @@ +OUTPUT_FORMAT(elf64-x86-64) + +PHDRS +{ + tcs PT_LOAD; + text PT_LOAD; + data PT_LOAD; +} + +SECTIONS +{ + . = 0; + .tcs : { + *(.tcs*) + } : tcs + + . = ALIGN(4096); + .text : { + *(.text*) + *(.rodata*) + } : text + + . = ALIGN(4096); + .data : { + *(.data*) + } : data + + /DISCARD/ : { + *(.comment*) + *(.note*) + *(.debug*) + *(.eh_frame*) + } +} + +ASSERT(!DEFINED(.altinstructions), "ALTERNATIVES are not supported in enclaves") +ASSERT(!DEFINED(.altinstr_replacement), "ALTERNATIVES are not supported in enclaves") +ASSERT(!DEFINED(.discard.retpoline_safe), "RETPOLINE ALTERNATIVES are not supported in enclaves") +ASSERT(!DEFINED(.discard.nospec), "RETPOLINE ALTERNATIVES are not supported in enclaves") +ASSERT(!DEFINED(.got.plt), "Libcalls are not supported in enclaves") diff --git a/tools/testing/selftests/sgx/test_encl_bootstrap.S b/tools/testing/selftests/sgx/test_encl_bootstrap.S new file mode 100644 index 000000000000..5d5680d4ea39 --- /dev/null +++ b/tools/testing/selftests/sgx/test_encl_bootstrap.S @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright(c) 2016-20 Intel Corporation. + */ + + .macro ENCLU + .byte 0x0f, 0x01, 0xd7 + .endm + + .section ".tcs", "aw" + .balign 4096 + + .fill 1, 8, 0 # STATE (set by CPU) + .fill 1, 8, 0 # FLAGS + .quad encl_ssa # OSSA + .fill 1, 4, 0 # CSSA (set by CPU) + .fill 1, 4, 1 # NSSA + .quad encl_entry # OENTRY + .fill 1, 8, 0 # AEP (set by EENTER and ERESUME) + .fill 1, 8, 0 # OFSBASE + .fill 1, 8, 0 # OGSBASE + .fill 1, 4, 0xFFFFFFFF # FSLIMIT + .fill 1, 4, 0xFFFFFFFF # GSLIMIT + .fill 4024, 1, 0 # Reserved + + # Identical to the previous TCS. + .fill 1, 8, 0 # STATE (set by CPU) + .fill 1, 8, 0 # FLAGS + .quad encl_ssa # OSSA + .fill 1, 4, 0 # CSSA (set by CPU) + .fill 1, 4, 1 # NSSA + .quad encl_entry # OENTRY + .fill 1, 8, 0 # AEP (set by EENTER and ERESUME) + .fill 1, 8, 0 # OFSBASE + .fill 1, 8, 0 # OGSBASE + .fill 1, 4, 0xFFFFFFFF # FSLIMIT + .fill 1, 4, 0xFFFFFFFF # GSLIMIT + .fill 4024, 1, 0 # Reserved + + .text + +encl_entry: + # RBX contains the base address for TCS, which is also the first address + # inside the enclave. By adding the value of le_stack_end to it, we get + # the absolute address for the stack. + lea (encl_stack)(%rbx), %rax + xchg %rsp, %rax + push %rax + + push %rcx # push the address after EENTER + push %rbx # push the enclave base address + + call encl_body + + pop %rbx # pop the enclave base address + + /* Clear volatile GPRs, except RAX (EEXIT function). */ + xor %rcx, %rcx + xor %rdx, %rdx + xor %rdi, %rdi + xor %rsi, %rsi + xor %r8, %r8 + xor %r9, %r9 + xor %r10, %r10 + xor %r11, %r11 + + # Reset status flags. + add %rdx, %rdx # OF = SF = AF = CF = 0; ZF = PF = 1 + + # Prepare EEXIT target by popping the address of the instruction after + # EENTER to RBX. + pop %rbx + + # Restore the caller stack. + pop %rax + mov %rax, %rsp + + # EEXIT + mov $4, %rax + enclu + + .section ".data", "aw" + +encl_ssa: + .space 4096 + + .balign 4096 + .space 8192 +encl_stack: diff --git a/tools/testing/selftests/x86/fsgsbase.c b/tools/testing/selftests/x86/fsgsbase.c index 7161cfc2e60b..8c780cce941d 100644 --- a/tools/testing/selftests/x86/fsgsbase.c +++ b/tools/testing/selftests/x86/fsgsbase.c @@ -392,8 +392,8 @@ static void set_gs_and_switch_to(unsigned long local, local = read_base(GS); /* - * Signal delivery seems to mess up weird selectors. Put it - * back. + * Signal delivery is quite likely to change a selector + * of 1, 2, or 3 back to 0 due to IRET being defective. */ asm volatile ("mov %0, %%gs" : : "rm" (force_sel)); } else { @@ -411,6 +411,14 @@ static void set_gs_and_switch_to(unsigned long local, if (base == local && sel_pre_sched == sel_post_sched) { printf("[OK]\tGS/BASE remained 0x%hx/0x%lx\n", sel_pre_sched, local); + } else if (base == local && sel_pre_sched >= 1 && sel_pre_sched <= 3 && + sel_post_sched == 0) { + /* + * IRET is misdesigned and will squash selectors 1, 2, or 3 + * to zero. Don't fail the test just because this happened. + */ + printf("[OK]\tGS/BASE changed from 0x%hx/0x%lx to 0x%hx/0x%lx because IRET is defective\n", + sel_pre_sched, local, sel_post_sched, base); } else { nerrs++; printf("[FAIL]\tGS/BASE changed from 0x%hx/0x%lx to 0x%hx/0x%lx\n", diff --git a/tools/testing/selftests/x86/raw_syscall_helper_32.S b/tools/testing/selftests/x86/raw_syscall_helper_32.S index 94410fa2b5ed..a10d36afdca0 100644 --- a/tools/testing/selftests/x86/raw_syscall_helper_32.S +++ b/tools/testing/selftests/x86/raw_syscall_helper_32.S @@ -45,3 +45,5 @@ int80_and_ret: .type int80_and_ret, @function .size int80_and_ret, .-int80_and_ret + +.section .note.GNU-stack,"",%progbits diff --git a/tools/testing/selftests/x86/thunks.S b/tools/testing/selftests/x86/thunks.S index 1bb5d62c16a4..a2d47d8344d4 100644 --- a/tools/testing/selftests/x86/thunks.S +++ b/tools/testing/selftests/x86/thunks.S @@ -57,3 +57,5 @@ call32_from_64: ret .size call32_from_64, .-call32_from_64 + +.section .note.GNU-stack,"",%progbits