diff --git a/BUILD.bazel b/BUILD.bazel index 8a7f131ba6e7..9885450c59b1 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,31 +1,23 @@ +# SPDX-License-Identifier: GPL-2.0 # Copyright (C) 2021 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. + +load("//build/kernel/kleaf:common_kernels.bzl", "define_common_kernels") + package( default_visibility = [ "//visibility:public", ], ) -load("//build/kernel/kleaf:common_kernels.bzl", "define_common_kernels") - _aarch64_additional_kmi_symbol_lists = [ # keep sorted "android/abi_gki_aarch64_core", "android/abi_gki_aarch64_db845c", + "android/abi_gki_aarch64_exynos", "android/abi_gki_aarch64_fips140", "android/abi_gki_aarch64_generic", "android/abi_gki_aarch64_hikey960", + "android/abi_gki_aarch64_rockchip", "android/abi_gki_aarch64_type_visibility", "android/abi_gki_aarch64_virtual_device", ] diff --git a/Documentation/ABI/testing/sysfs-bus-usb b/Documentation/ABI/testing/sysfs-bus-usb index bf2c1968525f..c2d32240831a 100644 --- a/Documentation/ABI/testing/sysfs-bus-usb +++ b/Documentation/ABI/testing/sysfs-bus-usb @@ -255,6 +255,17 @@ Description: is permitted, "u2" if only u2 is permitted, "u1_u2" if both u1 and u2 are permitted. +What: /sys/bus/usb/devices/...//port/early_stop +Date: Sep 2022 +Contact: Ray Chi +Description: + Some USB hosts have some watchdog mechanisms so that the device + may enter ramdump if it takes a long time during port initialization. + This attribute allows each port just has two attempts so that the + port initialization will be failed quickly. In addition, if a port + which is marked with early_stop has failed to initialize, it will ignore + all future connections until this attribute is clear. + What: /sys/bus/usb/devices/.../power/usb2_lpm_l1_timeout Date: May 2013 Contact: Mathias Nyman diff --git a/Documentation/ABI/testing/sysfs-driver-ufs b/Documentation/ABI/testing/sysfs-driver-ufs index f276b38287f8..ea6c54a3a256 100644 --- a/Documentation/ABI/testing/sysfs-driver-ufs +++ b/Documentation/ABI/testing/sysfs-driver-ufs @@ -916,21 +916,24 @@ Date: September 2014 Contact: Subhash Jadavani Description: This entry could be used to set or show the UFS device runtime power management level. The current driver - implementation supports 6 levels with next target states: + implementation supports 7 levels with next target states: == ==================================================== - 0 an UFS device will stay active, an UIC link will + 0 UFS device will stay active, UIC link will stay active - 1 an UFS device will stay active, an UIC link will + 1 UFS device will stay active, UIC link will hibernate - 2 an UFS device will moved to sleep, an UIC link will + 2 UFS device will be moved to sleep, UIC link will stay active - 3 an UFS device will moved to sleep, an UIC link will + 3 UFS device will be moved to sleep, UIC link will hibernate - 4 an UFS device will be powered off, an UIC link will + 4 UFS device will be powered off, UIC link will hibernate - 5 an UFS device will be powered off, an UIC link will + 5 UFS device will be powered off, UIC link will be powered off + 6 UFS device will be moved to deep sleep, UIC link + will be powered off. Note, deep sleep might not be + supported in which case this value will not be accepted == ==================================================== What: /sys/bus/platform/drivers/ufshcd/*/rpm_target_dev_state @@ -954,21 +957,24 @@ Date: September 2014 Contact: Subhash Jadavani Description: This entry could be used to set or show the UFS device system power management level. The current driver - implementation supports 6 levels with next target states: + implementation supports 7 levels with next target states: == ==================================================== - 0 an UFS device will stay active, an UIC link will + 0 UFS device will stay active, UIC link will stay active - 1 an UFS device will stay active, an UIC link will + 1 UFS device will stay active, UIC link will hibernate - 2 an UFS device will moved to sleep, an UIC link will + 2 UFS device will be moved to sleep, UIC link will stay active - 3 an UFS device will moved to sleep, an UIC link will + 3 UFS device will be moved to sleep, UIC link will hibernate - 4 an UFS device will be powered off, an UIC link will + 4 UFS device will be powered off, UIC link will hibernate - 5 an UFS device will be powered off, an UIC link will + 5 UFS device will be powered off, UIC link will be powered off + 6 UFS device will be moved to deep sleep, UIC link + will be powered off. Note, deep sleep might not be + supported in which case this value will not be accepted == ==================================================== What: /sys/bus/platform/drivers/ufshcd/*/spm_target_dev_state diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index ea2984de84db..8d21895db7bd 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -3412,8 +3412,7 @@ difficult since unequal pointers can no longer be compared. However, if this command-line option is specified, then all normal pointers will have their true - value printed. Pointers printed via %pK may still be - hashed. This option should only be specified when + value printed. This option should only be specified when debugging the kernel. Please do not use on production kernels. diff --git a/Documentation/admin-guide/mm/multigen_lru.rst b/Documentation/admin-guide/mm/multigen_lru.rst index 3d9a6ef84229..33e068830497 100644 --- a/Documentation/admin-guide/mm/multigen_lru.rst +++ b/Documentation/admin-guide/mm/multigen_lru.rst @@ -24,12 +24,13 @@ following subsections. Kill switch ----------- -``enable`` accepts different values to enable or disable the following -components. Its default value depends on ``CONFIG_LRU_GEN_ENABLED``. -All the components should be enabled unless some of them have -unforeseen side effects. Writing to ``enable`` has no effect when a -component is not supported by the hardware, and valid values will be -accepted even when the main switch is off. +``enabled`` accepts different values to enable or disable the +following components. Its default value depends on +``CONFIG_LRU_GEN_ENABLED``. All the components should be enabled +unless some of them have unforeseen side effects. Writing to +``enabled`` has no effect when a component is not supported by the +hardware, and valid values will be accepted even when the main switch +is off. ====== =============================================================== Values Components @@ -39,7 +40,9 @@ Values Components batches, when MMU sets it (e.g., on x86). This behavior can theoretically worsen lock contention (mmap_lock). If it is disabled, the multi-gen LRU will suffer a minor performance - degradation. + degradation for workloads that contiguously map hot pages, + whose accessed bits can be otherwise cleared by fewer larger + batches. 0x0004 Clearing the accessed bit in non-leaf page table entries as well, when MMU sets it (e.g., on x86). This behavior was not verified on x86 varieties other than Intel and AMD. If it is @@ -90,18 +93,19 @@ evicted generations in this file. Working set estimation ---------------------- -Working set estimation measures how much memory an application -requires in a given time interval, and it is usually done with little -impact on the performance of the application. E.g., data centers want -to optimize job scheduling (bin packing) to improve memory -utilizations. When a new job comes in, the job scheduler needs to find -out whether each server it manages can allocate a certain amount of -memory for this new job before it can pick a candidate. To do so, this -job scheduler needs to estimate the working sets of the existing jobs. +Working set estimation measures how much memory an application needs +in a given time interval, and it is usually done with little impact on +the performance of the application. E.g., data centers want to +optimize job scheduling (bin packing) to improve memory utilizations. +When a new job comes in, the job scheduler needs to find out whether +each server it manages can allocate a certain amount of memory for +this new job before it can pick a candidate. To do so, the job +scheduler needs to estimate the working sets of the existing jobs. When it is read, ``lru_gen`` returns a histogram of numbers of pages accessed over different time intervals for each memcg and node. -``MAX_NR_GENS`` decides the number of bins for each histogram. +``MAX_NR_GENS`` decides the number of bins for each histogram. The +histograms are noncumulative. :: memcg memcg_id memcg_path @@ -110,43 +114,49 @@ accessed over different time intervals for each memcg and node. ... max_gen_nr age_in_ms nr_anon_pages nr_file_pages -Each generation contains an estimated number of pages that have been -accessed within ``age_in_ms`` non-cumulatively. E.g., ``min_gen_nr`` -contains the coldest pages and ``max_gen_nr`` contains the hottest -pages, since ``age_in_ms`` of the former is the largest and that of -the latter is the smallest. +Each bin contains an estimated number of pages that have been accessed +within ``age_in_ms``. E.g., ``min_gen_nr`` contains the coldest pages +and ``max_gen_nr`` contains the hottest pages, since ``age_in_ms`` of +the former is the largest and that of the latter is the smallest. -Users can write ``+ memcg_id node_id max_gen_nr -[can_swap[full_scan]]`` to ``lru_gen`` to create a new generation -``max_gen_nr+1``. ``can_swap`` defaults to the swap setting and, if it -is set to ``1``, it forces the scan of anon pages when swap is off. -``full_scan`` defaults to ``1`` and, if it is set to ``0``, it reduces -the overhead as well as the coverage when scanning page tables. +Users can write the following command to ``lru_gen`` to create a new +generation ``max_gen_nr+1``: -A typical use case is that a job scheduler writes to ``lru_gen`` at a + ``+ memcg_id node_id max_gen_nr [can_swap [force_scan]]`` + +``can_swap`` defaults to the swap setting and, if it is set to ``1``, +it forces the scan of anon pages when swap is off, and vice versa. +``force_scan`` defaults to ``1`` and, if it is set to ``0``, it +employs heuristics to reduce the overhead, which is likely to reduce +the coverage as well. + +A typical use case is that a job scheduler runs this command at a certain time interval to create new generations, and it ranks the -servers it manages based on the sizes of their cold memory defined by +servers it manages based on the sizes of their cold pages defined by this time interval. Proactive reclaim ----------------- -Proactive reclaim induces memory reclaim when there is no memory -pressure and usually targets cold memory only. E.g., when a new job -comes in, the job scheduler wants to proactively reclaim memory on the -server it has selected to improve the chance of successfully landing +Proactive reclaim induces page reclaim when there is no memory +pressure. It usually targets cold pages only. E.g., when a new job +comes in, the job scheduler wants to proactively reclaim cold pages on +the server it selected, to improve the chance of successfully landing this new job. -Users can write ``- memcg_id node_id min_gen_nr [swappiness -[nr_to_reclaim]]`` to ``lru_gen`` to evict generations less than or -equal to ``min_gen_nr``. Note that ``min_gen_nr`` should be less than -``max_gen_nr-1`` as ``max_gen_nr`` and ``max_gen_nr-1`` are not fully -aged and therefore cannot be evicted. ``swappiness`` overrides the -default value in ``/proc/sys/vm/swappiness``. ``nr_to_reclaim`` limits -the number of pages to evict. +Users can write the following command to ``lru_gen`` to evict +generations less than or equal to ``min_gen_nr``. -A typical use case is that a job scheduler writes to ``lru_gen`` -before it tries to land a new job on a server, and if it fails to -materialize the cold memory without impacting the existing jobs on -this server, it retries on the next server according to the ranking -result obtained from the working set estimation step described -earlier. + ``- memcg_id node_id min_gen_nr [swappiness [nr_to_reclaim]]`` + +``min_gen_nr`` should be less than ``max_gen_nr-1``, since +``max_gen_nr`` and ``max_gen_nr-1`` are not fully aged (equivalent to +the active list) and therefore cannot be evicted. ``swappiness`` +overrides the default value in ``/proc/sys/vm/swappiness``. +``nr_to_reclaim`` limits the number of pages to evict. + +A typical use case is that a job scheduler runs this command before it +tries to land a new job on a server. If it fails to materialize enough +cold pages because of the overestimation, it retries on the next +server according to the ranking result obtained from the working set +estimation step. This less forceful approach limits the impacts on the +existing jobs. diff --git a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.yaml b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.yaml index 56c78317f9b7..57b9d2344b8a 100644 --- a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.yaml +++ b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.yaml @@ -82,10 +82,9 @@ allOf: - mediatek,mt2701-smi-common then: properties: - clock: - items: - minItems: 3 - maxItems: 3 + clocks: + minItems: 3 + maxItems: 3 clock-names: items: - const: apb @@ -101,10 +100,9 @@ allOf: then: properties: - clock: - items: - minItems: 4 - maxItems: 4 + clocks: + minItems: 4 + maxItems: 4 clock-names: items: - const: apb @@ -114,10 +112,9 @@ allOf: else: # for gen2 HW that don't have gals properties: - clock: - items: - minItems: 2 - maxItems: 2 + clocks: + minItems: 2 + maxItems: 2 clock-names: items: - const: apb diff --git a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.yaml b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.yaml index 06b623b34f48..2335a52f6f5a 100644 --- a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.yaml +++ b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.yaml @@ -77,11 +77,11 @@ allOf: then: properties: - clock: - items: - minItems: 3 - maxItems: 3 + clocks: + minItems: 2 + maxItems: 3 clock-names: + minItems: 2 items: - const: apb - const: smi @@ -89,10 +89,9 @@ allOf: else: properties: - clock: - items: - minItems: 2 - maxItems: 2 + clocks: + minItems: 2 + maxItems: 2 clock-names: items: - const: apb @@ -106,7 +105,6 @@ allOf: - mediatek,mt2701-smi-larb - mediatek,mt2712-smi-larb - mediatek,mt6779-smi-larb - - mediatek,mt8167-smi-larb then: required: diff --git a/Documentation/vm/multigen_lru.rst b/Documentation/vm/multigen_lru.rst index 8f6498002576..d7062c6a8946 100644 --- a/Documentation/vm/multigen_lru.rst +++ b/Documentation/vm/multigen_lru.rst @@ -22,15 +22,15 @@ The design objectives are: The representation of access recency is at the core of all LRU implementations. In the multi-gen LRU, each generation represents a group of pages with similar access recency. Generations establish a -common frame of reference and therefore help make better choices, -e.g., between different memcgs on a computer or different computers in -a data center (for job scheduling). +(time-based) common frame of reference and therefore help make better +choices, e.g., between different memcgs on a computer or different +computers in a data center (for job scheduling). Exploiting spatial locality improves efficiency when gathering the accessed bit. A rmap walk targets a single page and does not try to profit from discovering a young PTE. A page table walk can sweep all the young PTEs in an address space, but the address space can be too -large to make a profit. The key is to optimize both methods and use +sparse to make a profit. The key is to optimize both methods and use them in combination. Fast paths reduce code complexity and runtime overhead. Unmapped pages @@ -38,7 +38,7 @@ do not require TLB flushes; clean pages do not require writeback. These facts are only helpful when other conditions, e.g., access recency, are similar. With generations as a common frame of reference, additional factors stand out. But obvious choices might not be good -choices; thus self-correction is required. +choices; thus self-correction is necessary. The benefits of simple self-correcting heuristics are self-evident. Again, with generations as a common frame of reference, this becomes @@ -64,7 +64,7 @@ The protection of the former channel is by design stronger because: 3. The penalty of underprotecting the former channel is higher because applications usually do not prepare themselves for major page faults like they do for blocked I/O. E.g., GUI applications - commonly use dedicated I/O threads to avoid blocking the rendering + commonly use dedicated I/O threads to avoid blocking rendering threads. There are also two access patterns: @@ -88,19 +88,19 @@ pages can be evicted regardless of swap constraints. These three variables are monotonically increasing. Generation numbers are truncated into ``order_base_2(MAX_NR_GENS+1)`` -bits in order to fit into the gen counter in ``page->flags``. Each +bits in order to fit into the gen counter in ``folio->flags``. Each truncated generation number is an index to ``lrugen->lists[]``. The sliding window technique is used to track at least ``MIN_NR_GENS`` and at most ``MAX_NR_GENS`` generations. The gen counter stores a value within ``[1, MAX_NR_GENS]`` while a page is on one of ``lrugen->lists[]``; otherwise it stores zero. -Each generation is divided into multiple tiers. Tiers represent -different ranges of numbers of accesses through file descriptors. A -page accessed ``N`` times through file descriptors is in tier -``order_base_2(N)``. In contrast to moving across generations, which -requires the LRU lock, moving across tiers only requires operations on -``page->flags`` and therefore has a negligible cost. A feedback loop +Each generation is divided into multiple tiers. A page accessed ``N`` +times through file descriptors is in tier ``order_base_2(N)``. Unlike +generations, tiers do not have dedicated ``lrugen->lists[]``. In +contrast to moving across generations, which requires the LRU lock, +moving across tiers only involves atomic operations on +``folio->flags`` and therefore has a negligible cost. A feedback loop modeled after the PID controller monitors refaults over all the tiers from anon and file types and decides which tiers from which types to evict or protect. @@ -118,12 +118,11 @@ demotion of cold pages happens consequently when it increments ``max_seq``. The aging uses page table walks and rmap walks to find young PTEs. For the former, it iterates ``lruvec_memcg()->mm_list`` and calls ``walk_page_range()`` with each ``mm_struct`` on this list -to scan PTEs. On finding a young PTE, it clears the accessed bit and -updates the gen counter of the page mapped by this PTE to -``(max_seq%MAX_NR_GENS)+1``. After each iteration of this list, it -increments ``max_seq``. For the latter, when the eviction walks the -rmap and finds a young PTE, the aging scans the adjacent PTEs and -follows the same steps just described. +to scan PTEs, and after each iteration, it increments ``max_seq``. For +the latter, when the eviction walks the rmap and finds a young PTE, +the aging scans the adjacent PTEs. For both, on finding a young PTE, +the aging clears the accessed bit and updates the gen counter of the +page mapped by this PTE to ``(max_seq%MAX_NR_GENS)+1``. Eviction -------- @@ -134,12 +133,12 @@ evict from, it first compares ``min_seq[]`` to select the older type. If both types are equally old, it selects the one whose first tier has a lower refault percentage. The first tier contains single-use unmapped clean pages, which are the best bet. The eviction sorts a -page according to the gen counter if the aging has found this page -accessed through page tables and updated the gen counter. It also +page according to its gen counter if the aging has found this page +accessed through page tables and updated its gen counter. It also moves a page to the next generation, i.e., ``min_seq+1``, if this page was accessed multiple times through file descriptors and the feedback loop has detected outlying refaults from the tier this page is in. To -do this, the feedback loop uses the first tier as the baseline, for +this end, the feedback loop uses the first tier as the baseline, for the reason stated earlier. Summary @@ -147,14 +146,14 @@ Summary The multi-gen LRU can be disassembled into the following parts: * Generations -* Page table walks * Rmap walks +* Page table walks * Bloom filters -* The PID controller +* PID controller -The aging and the eviction is a producer-consumer model; specifically, -the latter drives the former by the sliding window over generations. -Within the aging, rmap walks drive page table walks by inserting hot -densely populated page tables to the Bloom filters. Within the -eviction, the PID controller uses refaults as the feedback to select -types to evict and tiers to protect. +The aging and the eviction form a producer-consumer model; +specifically, the latter drives the former by the sliding window over +generations. Within the aging, rmap walks drive page table walks by +inserting hot densely populated page tables to the Bloom filters. +Within the eviction, the PID controller uses refaults as the feedback +to select types to evict and tiers to protect. diff --git a/android/abi_gki_aarch64.xml b/android/abi_gki_aarch64.xml index aae00a689754..f2c916b26239 100644 --- a/android/abi_gki_aarch64.xml +++ b/android/abi_gki_aarch64.xml @@ -39,6 +39,7 @@ + @@ -87,16 +88,20 @@ + + + + @@ -115,6 +120,7 @@ + @@ -149,6 +155,7 @@ + @@ -209,8 +216,10 @@ + + @@ -219,24 +228,38 @@ + + + + + + + + + + + + + + @@ -245,19 +268,26 @@ + + + + + + + @@ -273,7 +303,10 @@ + + + @@ -281,6 +314,10 @@ + + + + @@ -304,14 +341,26 @@ + + + + + + + + + + + + @@ -327,7 +376,11 @@ + + + + @@ -336,14 +389,18 @@ + + + + @@ -374,11 +431,13 @@ + + @@ -414,6 +473,7 @@ + @@ -425,6 +485,7 @@ + @@ -568,6 +629,7 @@ + @@ -596,6 +658,9 @@ + + + @@ -614,6 +679,7 @@ + @@ -629,6 +695,10 @@ + + + + @@ -637,6 +707,7 @@ + @@ -652,6 +723,7 @@ + @@ -665,9 +737,12 @@ + + + @@ -702,10 +777,14 @@ + + + + @@ -715,6 +794,11 @@ + + + + + @@ -723,6 +807,7 @@ + @@ -740,7 +825,10 @@ + + + @@ -837,7 +925,11 @@ + + + + @@ -847,6 +939,7 @@ + @@ -910,14 +1003,18 @@ + + + + @@ -935,9 +1032,15 @@ + + + + + + @@ -978,6 +1081,7 @@ + @@ -986,6 +1090,10 @@ + + + + @@ -1003,6 +1111,8 @@ + + @@ -1024,6 +1134,7 @@ + @@ -1036,6 +1147,7 @@ + @@ -1082,6 +1194,7 @@ + @@ -1111,6 +1224,7 @@ + @@ -1139,6 +1253,7 @@ + @@ -1179,6 +1294,7 @@ + @@ -1187,6 +1303,7 @@ + @@ -1602,7 +1719,11 @@ + + + + @@ -1618,6 +1739,7 @@ + @@ -1646,6 +1768,7 @@ + @@ -1711,8 +1834,12 @@ + + + + @@ -1751,9 +1878,12 @@ + + + @@ -1857,7 +1987,20 @@ + + + + + + + + + + + + + @@ -1892,6 +2035,7 @@ + @@ -1976,6 +2120,7 @@ + @@ -1987,7 +2132,10 @@ + + + @@ -2007,6 +2155,8 @@ + + @@ -2097,6 +2247,8 @@ + + @@ -2104,6 +2256,8 @@ + + @@ -2111,6 +2265,7 @@ + @@ -2277,6 +2432,7 @@ + @@ -2297,6 +2453,7 @@ + @@ -2367,6 +2524,7 @@ + @@ -2498,6 +2656,8 @@ + + @@ -2515,6 +2675,7 @@ + @@ -2550,6 +2711,12 @@ + + + + + + @@ -2558,24 +2725,30 @@ + + + + + + @@ -2590,8 +2763,10 @@ + + @@ -2600,6 +2775,8 @@ + + @@ -2642,6 +2819,7 @@ + @@ -2650,9 +2828,11 @@ + + @@ -2713,7 +2893,10 @@ + + + @@ -2728,15 +2911,19 @@ + + + + @@ -2748,12 +2935,15 @@ + + + @@ -2765,6 +2955,7 @@ + @@ -2790,6 +2981,7 @@ + @@ -2822,6 +3014,8 @@ + + @@ -2899,9 +3093,11 @@ + + @@ -2914,6 +3110,7 @@ + @@ -2921,6 +3118,7 @@ + @@ -2973,6 +3171,7 @@ + @@ -3000,6 +3199,7 @@ + @@ -3022,17 +3222,21 @@ + + + + @@ -3043,6 +3247,7 @@ + @@ -3055,6 +3260,7 @@ + @@ -3067,20 +3273,26 @@ + + + + + + @@ -3094,17 +3306,22 @@ + + + + + @@ -3167,6 +3384,8 @@ + + @@ -3210,6 +3429,7 @@ + @@ -3218,6 +3438,7 @@ + @@ -3227,6 +3448,7 @@ + @@ -3298,9 +3520,17 @@ + + + + + + + + @@ -3490,21 +3720,32 @@ + + + + + + + + + + + @@ -3516,9 +3757,18 @@ + + + + + + + + + @@ -3541,6 +3791,7 @@ + @@ -3630,6 +3881,7 @@ + @@ -3648,6 +3900,7 @@ + @@ -3661,6 +3914,7 @@ + @@ -3734,8 +3988,10 @@ + + @@ -3744,24 +4000,38 @@ + + + + + + + + + + + + + + @@ -3770,19 +4040,26 @@ + + + + + + + @@ -3798,7 +4075,10 @@ + + + @@ -3806,6 +4086,10 @@ + + + + @@ -3829,14 +4113,26 @@ + + + + + + + + + + + + @@ -3852,7 +4148,11 @@ + + + + @@ -3861,8 +4161,14 @@ + + + + + + @@ -3885,6 +4191,7 @@ + @@ -3896,6 +4203,7 @@ + @@ -3914,6 +4222,7 @@ + @@ -3930,6 +4239,7 @@ + @@ -3945,6 +4255,7 @@ + @@ -3952,9 +4263,11 @@ + + @@ -3966,19 +4279,32 @@ + + + + + + + + + + + + + - + @@ -4082,6 +4408,7 @@ + @@ -4129,16 +4456,16 @@ - + - + - + - + @@ -4233,6 +4560,14 @@ + + + + + + + + @@ -4258,6 +4593,11 @@ + + + + + @@ -4370,7 +4710,20 @@ - + + + + + + + + + + + + + + @@ -4447,7 +4800,7 @@ - + @@ -4482,7 +4835,7 @@ - + @@ -4702,6 +5055,7 @@ + @@ -4750,6 +5104,7 @@ + @@ -4771,7 +5126,65 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -4809,6 +5222,7 @@ + @@ -4872,6 +5286,14 @@ + + + + + + + + @@ -4905,7 +5327,7 @@ - + @@ -4937,11 +5359,60 @@ - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -5024,6 +5495,10 @@ + + + + @@ -5075,6 +5550,14 @@ + + + + + + + + @@ -5191,11 +5674,15 @@ + + + + @@ -5352,15 +5839,26 @@ - + + + + + + + + + + + + @@ -5411,6 +5909,17 @@ + + + + + + + + + + + @@ -5418,7 +5927,14 @@ - + + + + + + + + @@ -5482,6 +5998,8 @@ + + @@ -5658,6 +6176,7 @@ + @@ -5779,6 +6298,7 @@ + @@ -6033,6 +6553,14 @@ + + + + + + + + @@ -6168,6 +6696,7 @@ + @@ -6334,10 +6863,42 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -6389,7 +6950,7 @@ - + @@ -6397,20 +6958,6 @@ - - - - - - - - - - - - - - @@ -6425,7 +6972,21 @@ - + + + + + + + + + + + + + + + @@ -6461,6 +7022,17 @@ + + + + + + + + + + + @@ -6930,7 +7502,7 @@ - + @@ -6939,6 +7511,7 @@ + @@ -7035,6 +7608,7 @@ + @@ -7075,7 +7649,7 @@ - + @@ -7136,7 +7710,9 @@ + + @@ -7160,6 +7736,7 @@ + @@ -7191,6 +7768,7 @@ + @@ -7230,6 +7808,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -7310,6 +7923,13 @@ + + + + + + + @@ -7328,6 +7948,18 @@ + + + + + + + + + + + + @@ -7366,6 +7998,12 @@ + + + + + + @@ -7603,7 +8241,7 @@ - + @@ -7612,7 +8250,17 @@ - + + + + + + + + + + + @@ -7640,7 +8288,7 @@ - + @@ -7689,6 +8337,10 @@ + + + + @@ -7771,7 +8423,7 @@ - + @@ -7830,6 +8482,10 @@ + + + + @@ -7932,6 +8588,7 @@ + @@ -8116,6 +8773,7 @@ + @@ -8282,6 +8940,7 @@ + @@ -8376,6 +9035,14 @@ + + + + + + + + @@ -8454,6 +9121,7 @@ + @@ -8764,7 +9432,7 @@ - + @@ -8822,6 +9490,7 @@ + @@ -9047,6 +9716,11 @@ + + + + + @@ -9148,7 +9822,62 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -9178,7 +9907,7 @@ - + @@ -9238,7 +9967,7 @@ - + @@ -9260,6 +9989,23 @@ + + + + + + + + + + + + + + + + + @@ -9314,6 +10060,11 @@ + + + + + @@ -9399,7 +10150,7 @@ - + @@ -9661,12 +10412,13 @@ - + + @@ -9886,7 +10638,7 @@ - + @@ -9898,7 +10650,7 @@ - + @@ -9925,10 +10677,10 @@ - + - + @@ -10622,6 +11374,12 @@ + + + + + + @@ -10759,7 +11517,7 @@ - + @@ -10788,10 +11546,18 @@ + + + + + + + + - + @@ -10982,6 +11748,7 @@ + @@ -11057,6 +11824,7 @@ + @@ -11267,10 +12035,14 @@ + + + + - + @@ -11398,7 +12170,7 @@ - + @@ -11446,6 +12218,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -11502,87 +12315,87 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -11609,7 +12422,7 @@ - + @@ -11671,7 +12484,7 @@ - + @@ -11864,7 +12677,7 @@ - + @@ -11883,6 +12696,7 @@ + @@ -11966,6 +12780,18 @@ + + + + + + + + + + + + @@ -12195,6 +13021,7 @@ + @@ -12242,6 +13069,14 @@ + + + + + + + + @@ -12272,6 +13107,7 @@ + @@ -12636,6 +13472,7 @@ + @@ -13094,9 +13931,18 @@ + + + + + + + + + - + @@ -13382,6 +14228,7 @@ + @@ -13393,6 +14240,8 @@ + + @@ -13424,6 +14273,11 @@ + + + + + @@ -13457,7 +14311,7 @@ - + @@ -13571,7 +14425,56 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -13615,6 +14518,15 @@ + + + + + + + + + @@ -13645,6 +14557,7 @@ + @@ -13823,6 +14736,7 @@ + @@ -13855,7 +14769,65 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -14039,6 +15011,7 @@ + @@ -14139,6 +15112,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -14148,7 +15147,7 @@ - + @@ -14530,6 +15529,10 @@ + + + + @@ -14578,7 +15581,7 @@ - + @@ -14756,26 +15759,7 @@ - - - - - - - - - - - - - - - - - - - - + @@ -14830,7 +15814,7 @@ - + @@ -14939,6 +15923,7 @@ + @@ -15008,6 +15993,7 @@ + @@ -15046,6 +16032,7 @@ + @@ -15162,6 +16149,14 @@ + + + + + + + + @@ -15174,6 +16169,14 @@ + + + + + + + + @@ -15268,6 +16271,12 @@ + + + + + + @@ -15409,6 +16418,20 @@ + + + + + + + + + + + + + + @@ -15488,7 +16511,7 @@ - + @@ -15874,6 +16897,53 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -15894,7 +16964,7 @@ - + @@ -15911,7 +16981,17 @@ - + + + + + + + + + + + @@ -16085,7 +17165,16 @@ + + + + + + + + + @@ -16306,17 +17395,100 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + @@ -16385,7 +17557,7 @@ - + @@ -16405,6 +17577,23 @@ + + + + + + + + + + + + + + + + + @@ -16479,7 +17668,7 @@ - + @@ -16778,6 +17967,7 @@ + @@ -16792,10 +17982,15 @@ + + + + + @@ -16855,6 +18050,9 @@ + + + @@ -17186,6 +18384,7 @@ + @@ -17325,7 +18524,33 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -17510,10 +18735,7 @@ - - - - + @@ -17573,6 +18795,13 @@ + + + + + + + @@ -17588,6 +18817,12 @@ + + + + + + @@ -17757,7 +18992,7 @@ - + @@ -17801,6 +19036,12 @@ + + + + + + @@ -17859,6 +19100,7 @@ + @@ -17934,6 +19176,14 @@ + + + + + + + + @@ -18083,7 +19333,7 @@ - + @@ -18193,6 +19443,7 @@ + @@ -18201,6 +19452,7 @@ + @@ -18233,6 +19485,12 @@ + + + + + + @@ -18267,6 +19525,17 @@ + + + + + + + + + + + @@ -18319,6 +19588,7 @@ + @@ -18374,7 +19644,20 @@ - + + + + + + + + + + + + + + @@ -18398,6 +19681,7 @@ + @@ -18472,6 +19756,7 @@ + @@ -18498,222 +19783,222 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -19043,12 +20328,12 @@ - + - + - + @@ -19071,8 +20356,8 @@ - - + + @@ -19124,7 +20409,7 @@ - + @@ -19257,6 +20542,7 @@ + @@ -19409,7 +20695,7 @@ - + @@ -19422,6 +20708,10 @@ + + + + @@ -19510,7 +20800,7 @@ - + @@ -19522,7 +20812,7 @@ - + @@ -19623,6 +20913,7 @@ + @@ -19699,8 +20990,20 @@ + + + + + + + + + + + + @@ -19723,7 +21026,7 @@ - + @@ -19857,6 +21160,7 @@ + @@ -19901,6 +21205,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -19962,6 +21296,11 @@ + + + + + @@ -20172,7 +21511,7 @@ - + @@ -20288,11 +21627,12 @@ + - + @@ -20361,6 +21701,10 @@ + + + + @@ -20735,6 +22079,7 @@ + @@ -20796,7 +22141,7 @@ - + @@ -20887,7 +22232,7 @@ - + @@ -20945,7 +22290,7 @@ - + @@ -20992,6 +22337,14 @@ + + + + + + + + @@ -21024,6 +22377,20 @@ + + + + + + + + + + + + + + @@ -21074,6 +22441,9 @@ + + + @@ -21241,22 +22611,30 @@ - + + + + + + + + + - + - + @@ -21329,6 +22707,7 @@ + @@ -21357,6 +22736,7 @@ + @@ -21366,12 +22746,62 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -21386,6 +22816,7 @@ + @@ -21873,7 +23304,7 @@ - + @@ -21939,6 +23370,10 @@ + + + + @@ -21953,7 +23388,7 @@ - + @@ -22007,6 +23442,12 @@ + + + + + + @@ -22026,33 +23467,33 @@ - + - + - + - + - + - + - + - + - + - + @@ -22098,7 +23539,7 @@ - + @@ -22227,7 +23668,7 @@ - + @@ -22275,12 +23716,12 @@ - + - + - + @@ -22401,7 +23842,7 @@ - + @@ -22609,6 +24050,7 @@ + @@ -22701,6 +24143,11 @@ + + + + + @@ -22747,18 +24194,18 @@ - + - + - + - + - + @@ -22772,9 +24219,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + @@ -22811,13 +24282,62 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -23116,6 +24636,7 @@ + @@ -23151,7 +24672,7 @@ - + @@ -23187,12 +24708,14 @@ + + @@ -23204,6 +24727,7 @@ + @@ -23384,6 +24908,7 @@ + @@ -23633,7 +25158,7 @@ - + @@ -23834,6 +25359,26 @@ + + + + + + + + + + + + + + + + + + + + @@ -24240,6 +25785,13 @@ + + + + + + + @@ -24254,6 +25806,13 @@ + + + + + + + @@ -24675,6 +26234,7 @@ + @@ -24758,8 +26318,43 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -24786,6 +26381,9 @@ + + + @@ -24971,6 +26569,7 @@ + @@ -25115,6 +26714,50 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -25478,7 +27121,23 @@ - + + + + + + + + + + + + + + + + + @@ -25588,7 +27247,7 @@ - + @@ -25875,7 +27534,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + @@ -25967,7 +27651,127 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -26052,6 +27856,7 @@ + @@ -26069,7 +27874,7 @@ - + @@ -26097,6 +27902,7 @@ + @@ -26470,6 +28276,14 @@ + + + + + + + + @@ -26668,6 +28482,17 @@ + + + + + + + + + + + @@ -26847,6 +28672,11 @@ + + + + + @@ -27046,6 +28876,12 @@ + + + + + + @@ -27165,7 +29001,7 @@ - + @@ -27181,6 +29017,7 @@ + @@ -27216,7 +29053,7 @@ - + @@ -27231,6 +29068,23 @@ + + + + + + + + + + + + + + + + + @@ -27291,6 +29145,9 @@ + + + @@ -27372,6 +29229,7 @@ + @@ -27419,6 +29277,7 @@ + @@ -27433,7 +29292,7 @@ - + @@ -27479,7 +29338,7 @@ - + @@ -27555,6 +29414,7 @@ + @@ -27590,7 +29450,7 @@ - + @@ -27613,7 +29473,20 @@ - + + + + + + + + + + + + + + @@ -27636,6 +29509,7 @@ + @@ -27770,10 +29644,20 @@ + + + + + + + + + + @@ -27790,7 +29674,7 @@ - + @@ -27798,6 +29682,23 @@ + + + + + + + + + + + + + + + + + @@ -27917,13 +29818,18 @@ - - - + + + + + + + + @@ -27990,6 +29896,7 @@ + @@ -28352,11 +30259,19 @@ + + + + + + + + - + - + @@ -28396,11 +30311,12 @@ - + + @@ -28610,6 +30526,7 @@ + @@ -28688,6 +30605,7 @@ + @@ -28735,6 +30653,12 @@ + + + + + + @@ -28769,9 +30693,13 @@ - + + + + + @@ -28909,68 +30837,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -29010,6 +30877,14 @@ + + + + + + + + @@ -29100,7 +30975,7 @@ - + @@ -29171,6 +31046,7 @@ + @@ -29257,7 +31133,7 @@ - + @@ -29401,6 +31277,7 @@ + @@ -29422,6 +31299,7 @@ + @@ -29506,6 +31384,7 @@ + @@ -29615,6 +31494,7 @@ + @@ -29981,6 +31861,83 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -30021,7 +31978,7 @@ - + @@ -30387,6 +32344,20 @@ + + + + + + + + + + + + + + @@ -30439,7 +32410,7 @@ - + @@ -30570,7 +32541,7 @@ - + @@ -31057,6 +33028,20 @@ + + + + + + + + + + + + + + @@ -31218,6 +33203,7 @@ + @@ -31380,6 +33366,12 @@ + + + + + + @@ -31560,6 +33552,12 @@ + + + + + + @@ -32109,6 +34107,7 @@ + @@ -32321,7 +34320,7 @@ - + @@ -32662,7 +34661,7 @@ - + @@ -32673,7 +34672,14 @@ - + + + + + + + + @@ -32787,6 +34793,13 @@ + + + + + + + @@ -32806,9 +34819,48 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -32830,6 +34882,7 @@ + @@ -32855,6 +34908,7 @@ + @@ -32906,6 +34960,7 @@ + @@ -33015,6 +35070,7 @@ + @@ -33060,7 +35116,7 @@ - + @@ -33235,7 +35291,7 @@ - + @@ -33247,6 +35303,7 @@ + @@ -33273,6 +35330,65 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -33307,99 +35423,99 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -33449,7 +35565,7 @@ - + @@ -33558,6 +35674,9 @@ + + + @@ -33824,6 +35943,20 @@ + + + + + + + + + + + + + + @@ -33909,6 +36042,11 @@ + + + + + @@ -33932,6 +36070,9 @@ + + + @@ -34553,6 +36694,7 @@ + @@ -34567,7 +36709,7 @@ - + @@ -34633,7 +36775,7 @@ - + @@ -34644,9 +36786,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + @@ -34707,6 +36873,7 @@ + @@ -34842,7 +37009,7 @@ - + @@ -34864,6 +37031,14 @@ + + + + + + + + @@ -35090,13 +37265,13 @@ - + - + - + @@ -35120,7 +37295,7 @@ - + @@ -35138,13 +37313,13 @@ - + - + @@ -35153,7 +37328,7 @@ - + @@ -35162,7 +37337,7 @@ - + @@ -35534,6 +37709,14 @@ + + + + + + + + @@ -35805,7 +37988,7 @@ - + @@ -36084,13 +38267,13 @@ - + - + @@ -36108,7 +38291,7 @@ - + @@ -36337,6 +38520,7 @@ + @@ -36683,6 +38867,76 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -37006,7 +39260,7 @@ - + @@ -37048,6 +39302,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -37091,7 +39383,29 @@ - + + + + + + + + + + + + + + + + + + + + + + + @@ -37122,7 +39436,49 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -37219,7 +39575,29 @@ - + + + + + + + + + + + + + + + + + + + + + + + @@ -37358,6 +39736,9 @@ + + + @@ -37400,7 +39781,7 @@ - + @@ -37409,7 +39790,7 @@ - + @@ -38162,7 +40543,7 @@ - + @@ -38335,6 +40716,11 @@ + + + + + @@ -38426,6 +40812,7 @@ + @@ -38734,7 +41121,7 @@ - + @@ -38974,7 +41361,18 @@ - + + + + + + + + + + + + @@ -39260,6 +41658,7 @@ + @@ -39402,6 +41801,12 @@ + + + + + + @@ -39409,7 +41814,24 @@ - + + + + + + + + + + + + + + + + + + @@ -39440,7 +41862,7 @@ - + @@ -39793,6 +42215,17 @@ + + + + + + + + + + + @@ -40002,6 +42435,9 @@ + + + @@ -40052,9 +42488,15 @@ - + + + + + + + @@ -40133,6 +42575,14 @@ + + + + + + + + @@ -40335,12 +42785,12 @@ - + - + - + @@ -40351,6 +42801,7 @@ + @@ -40422,6 +42873,53 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -40510,6 +43008,12 @@ + + + + + + @@ -41215,7 +43719,7 @@ - + @@ -41283,9 +43787,116 @@ + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -41337,6 +43948,7 @@ + @@ -41419,8 +44031,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -41744,7 +44399,7 @@ - + @@ -41820,6 +44475,7 @@ + @@ -42222,24 +44878,24 @@ - + - + - + - + - + - + - + @@ -42338,7 +44994,7 @@ - + @@ -42492,7 +45148,35 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -42678,18 +45362,18 @@ - + - + - + - + - + @@ -42772,6 +45456,11 @@ + + + + + @@ -43000,7 +45689,7 @@ - + @@ -43017,7 +45706,14 @@ - + + + + + + + + @@ -43543,6 +46239,7 @@ + @@ -43619,10 +46316,12 @@ + + @@ -44302,6 +47001,9 @@ + + + @@ -44335,6 +47037,23 @@ + + + + + + + + + + + + + + + + + @@ -44440,6 +47159,7 @@ + @@ -44513,9 +47233,74 @@ + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -44736,6 +47521,7 @@ + @@ -44776,6 +47562,12 @@ + + + + + + @@ -44790,6 +47582,7 @@ + @@ -44885,6 +47678,7 @@ + @@ -45134,7 +47928,7 @@ - + @@ -45283,7 +48077,7 @@ - + @@ -45314,6 +48108,14 @@ + + + + + + + + @@ -45528,6 +48330,7 @@ + @@ -46194,6 +48997,7 @@ + @@ -46306,7 +49110,7 @@ - + @@ -46588,6 +49392,23 @@ + + + + + + + + + + + + + + + + + @@ -46640,6 +49461,7 @@ + @@ -46776,9 +49598,30 @@ + + + + + + + + + + + + + + + + + + + + + @@ -46882,13 +49725,13 @@ - + - + - + @@ -46897,7 +49740,7 @@ - + @@ -46911,7 +49754,7 @@ - + @@ -46932,14 +49775,6 @@ - - - - - - - - @@ -46980,7 +49815,7 @@ - + @@ -47001,12 +49836,10 @@ + + + - - - - - @@ -47225,6 +50058,9 @@ + + + @@ -47279,6 +50115,26 @@ + + + + + + + + + + + + + + + + + + + + @@ -47392,28 +50248,13 @@ - - - - - - - - - - - - - - - - - + + @@ -47457,6 +50298,7 @@ + @@ -47573,6 +50415,7 @@ + @@ -47726,7 +50569,35 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -47908,6 +50779,7 @@ + @@ -48178,7 +51050,7 @@ - + @@ -48291,7 +51163,59 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -48453,6 +51377,15 @@ + + + + + + + + + @@ -48470,11 +51403,36 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + @@ -48564,6 +51522,12 @@ + + + + + + @@ -49015,10 +51979,14 @@ + + + + @@ -49090,7 +52058,7 @@ - + @@ -49189,7 +52157,92 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -49560,6 +52613,20 @@ + + + + + + + + + + + + + + @@ -49687,6 +52754,11 @@ + + + + + @@ -49704,6 +52776,10 @@ + + + + @@ -49874,10 +52950,8 @@ - - - - + + @@ -49905,7 +52979,7 @@ - + @@ -49983,6 +53057,7 @@ + @@ -50014,6 +53089,7 @@ + @@ -50043,6 +53119,7 @@ + @@ -50189,7 +53266,29 @@ - + + + + + + + + + + + + + + + + + + + + + + + @@ -50401,6 +53500,14 @@ + + + + + + + + @@ -50924,7 +54031,7 @@ - + @@ -51393,6 +54500,29 @@ + + + + + + + + + + + + + + + + + + + + + + + @@ -51428,6 +54558,7 @@ + @@ -51639,6 +54770,11 @@ + + + + + @@ -51789,6 +54925,7 @@ + @@ -51808,7 +54945,7 @@ - + @@ -51967,6 +55104,10 @@ + + + + @@ -52008,144 +55149,144 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - + + @@ -52725,6 +55866,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -52741,6 +55908,11 @@ + + + + + @@ -53035,6 +56207,7 @@ + @@ -53191,9 +56364,10 @@ + - + @@ -53262,6 +56436,7 @@ + @@ -53433,7 +56608,7 @@ - + @@ -53647,6 +56822,7 @@ + @@ -53736,6 +56912,7 @@ + @@ -53790,6 +56967,9 @@ + + + @@ -53810,6 +56990,174 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -53824,6 +57172,14 @@ + + + + + + + + @@ -53922,7 +57278,38 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -53948,7 +57335,7 @@ - + @@ -53981,10 +57368,10 @@ - + - + @@ -54009,6 +57396,7 @@ + @@ -54177,7 +57565,7 @@ - + @@ -54323,10 +57711,50 @@ + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -54446,6 +57874,20 @@ + + + + + + + + + + + + + + @@ -54507,11 +57949,34 @@ - + + + + + + + + + + + + + + + + + + + + + + + + @@ -54679,7 +58144,7 @@ - + @@ -54770,7 +58235,7 @@ - + @@ -55019,6 +58484,17 @@ + + + + + + + + + + + @@ -55051,6 +58527,12 @@ + + + + + + @@ -55274,10 +58756,15 @@ - + + + + + + @@ -55303,7 +58790,7 @@ - + @@ -55358,6 +58845,7 @@ + @@ -55550,6 +59038,12 @@ + + + + + + @@ -55870,6 +59364,7 @@ + @@ -56138,7 +59633,20 @@ - + + + + + + + + + + + + + + @@ -56154,6 +59662,7 @@ + @@ -56207,7 +59716,7 @@ - + @@ -56226,6 +59735,7 @@ + @@ -56537,6 +60047,9 @@ + + + @@ -56709,7 +60222,7 @@ - + @@ -56771,13 +60284,13 @@ - + - + @@ -56929,61 +60442,61 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -57135,19 +60648,20 @@ - + - + - + - + + @@ -57253,6 +60767,7 @@ + @@ -57263,6 +60778,10 @@ + + + + @@ -57272,7 +60791,20 @@ - + + + + + + + + + + + + + + @@ -57288,51 +60820,51 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -57876,6 +61408,11 @@ + + + + + @@ -58339,10 +61876,10 @@ - + - + @@ -58720,7 +62257,7 @@ - + @@ -58870,6 +62407,11 @@ + + + + + @@ -58968,6 +62510,12 @@ + + + + + + @@ -59013,7 +62561,7 @@ - + @@ -59036,6 +62584,7 @@ + @@ -59143,6 +62692,7 @@ + @@ -59330,6 +62880,7 @@ + @@ -59397,7 +62948,48 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -59599,6 +63191,29 @@ + + + + + + + + + + + + + + + + + + + + + + + @@ -59705,7 +63320,7 @@ - + @@ -59768,6 +63383,7 @@ + @@ -59784,6 +63400,12 @@ + + + + + + @@ -59840,6 +63462,7 @@ + @@ -59960,8 +63583,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -60006,6 +63656,7 @@ + @@ -60069,7 +63720,7 @@ - + @@ -60146,6 +63797,7 @@ + @@ -60178,6 +63830,20 @@ + + + + + + + + + + + + + + @@ -60239,6 +63905,7 @@ + @@ -60337,6 +64004,15 @@ + + + + + + + + + @@ -60384,12 +64060,12 @@ - + - + - + @@ -60499,7 +64175,7 @@ - + @@ -60571,6 +64247,15 @@ + + + + + + + + + @@ -60581,7 +64266,7 @@ - + @@ -60683,6 +64368,29 @@ + + + + + + + + + + + + + + + + + + + + + + + @@ -60741,6 +64449,29 @@ + + + + + + + + + + + + + + + + + + + + + + + @@ -60758,7 +64489,41 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -60883,7 +64648,7 @@ - + @@ -60893,14 +64658,29 @@ + + + + + + + + + + + + + + + - + @@ -61085,7 +64865,7 @@ - + @@ -61152,13 +64932,13 @@ - + - + @@ -61170,7 +64950,7 @@ - + @@ -61184,6 +64964,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -61429,7 +65238,7 @@ - + @@ -61450,6 +65259,7 @@ + @@ -61548,7 +65358,27 @@ - + + + + + + + + + + + + + + + + + + + + + @@ -61577,6 +65407,14 @@ + + + + + + + + @@ -61916,10 +65754,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + @@ -61995,6 +65858,7 @@ + @@ -62048,7 +65912,7 @@ - + @@ -62059,6 +65923,7 @@ + @@ -62105,7 +65970,7 @@ - + @@ -62173,8 +66038,17 @@ + + + + + + + + + @@ -62270,6 +66144,14 @@ + + + + + + + + @@ -62891,6 +66773,8 @@ + + @@ -62904,6 +66788,26 @@ + + + + + + + + + + + + + + + + + + + + @@ -63091,36 +66995,36 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -63135,7 +67039,7 @@ - + @@ -63207,7 +67111,7 @@ - + @@ -63242,6 +67146,11 @@ + + + + + @@ -63251,7 +67160,7 @@ - + @@ -63651,6 +67560,11 @@ + + + + + @@ -63915,6 +67829,9 @@ + + + @@ -64042,6 +67959,7 @@ + @@ -64093,7 +68011,7 @@ - + @@ -64265,6 +68183,7 @@ + @@ -64274,6 +68193,7 @@ + @@ -64315,6 +68235,7 @@ + @@ -64345,7 +68266,7 @@ - + @@ -64363,7 +68284,7 @@ - + @@ -64390,7 +68311,7 @@ - + @@ -64632,22 +68553,26 @@ + + + + - + - + - + @@ -64700,12 +68625,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -64793,6 +68752,20 @@ + + + + + + + + + + + + + + @@ -65060,7 +69033,7 @@ - + @@ -65207,6 +69180,10 @@ + + + + @@ -65262,6 +69239,12 @@ + + + + + + @@ -65282,10 +69265,11 @@ + - - + + @@ -65305,6 +69289,7 @@ + @@ -65584,7 +69569,7 @@ - + @@ -65695,6 +69680,7 @@ + @@ -65705,6 +69691,14 @@ + + + + + + + + @@ -65753,6 +69747,7 @@ + @@ -65780,7 +69775,7 @@ - + @@ -65803,6 +69798,56 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -65914,25 +69959,25 @@ - + - + - + - + - + - + - + @@ -65955,7 +70000,7 @@ - + @@ -65970,6 +70015,7 @@ + @@ -65982,6 +70028,7 @@ + @@ -66085,6 +70132,7 @@ + @@ -66145,7 +70193,7 @@ - + @@ -66219,14 +70267,45 @@ - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -66251,7 +70330,7 @@ - + @@ -66337,10 +70416,13 @@ - + + + + @@ -66398,6 +70480,7 @@ + @@ -66426,6 +70509,9 @@ + + + @@ -66708,6 +70794,20 @@ + + + + + + + + + + + + + + @@ -66823,7 +70923,7 @@ - + @@ -66864,7 +70964,7 @@ - + @@ -66944,6 +71044,11 @@ + + + + + @@ -67010,7 +71115,7 @@ - + @@ -67040,18 +71145,18 @@ - + - + - + - + - + @@ -67109,6 +71214,9 @@ + + + @@ -67527,6 +71635,13 @@ + + + + + + + @@ -67725,7 +71840,7 @@ - + @@ -67759,6 +71874,131 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -67861,6 +72101,7 @@ + @@ -67873,7 +72114,7 @@ - + @@ -68029,7 +72270,7 @@ - + @@ -68215,7 +72456,7 @@ - + @@ -68224,7 +72465,32 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -68286,170 +72552,15 @@ - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -68457,7 +72568,7 @@ - + @@ -68465,7 +72576,7 @@ - + @@ -68473,9 +72584,9 @@ - + - + @@ -68484,7 +72595,7 @@ - + @@ -68492,7 +72603,7 @@ - + @@ -68500,23 +72611,23 @@ - + - + - + - + - + @@ -68524,7 +72635,7 @@ - + @@ -68532,7 +72643,7 @@ - + @@ -68540,210 +72651,182 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + - + - + - + - + - + - + - + - + - - - - + - + - + - + - + - + - + - - - - - - - + - + - + - + - + - + - - - - + - + - + + + + - + - + - + + + + - + + + + + + + + + @@ -68751,7 +72834,7 @@ - + @@ -68759,7 +72842,7 @@ - + @@ -68767,7 +72850,7 @@ - + @@ -68784,7 +72867,949 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -68801,64 +73826,15 @@ - + - + - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -68869,7 +73845,7 @@ - + @@ -68880,12 +73856,12 @@ - + - + - + @@ -68893,13 +73869,24 @@ - + - + - + + + + + + + + + + + + @@ -68907,31 +73894,94 @@ - + - + - + - + - + - + - + - + - + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -68942,7 +73992,7 @@ - + @@ -68956,7 +74006,7 @@ - + @@ -68964,23 +74014,31 @@ - + - + - + - + - + - + - + + + + + + + + + @@ -69006,12 +74064,12 @@ - + - + @@ -69019,23 +74077,23 @@ - + - + - + - + - + - + - + @@ -69043,7 +74101,7 @@ - + @@ -69069,7 +74127,7 @@ - + @@ -69080,134 +74138,18 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - + - + @@ -69215,7 +74157,7 @@ - + @@ -69223,23 +74165,31 @@ - + - + - + - + - + - + - + + + + + + + + + @@ -69250,603 +74200,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -69854,12 +74208,12 @@ - + - + - + @@ -69867,15 +74221,15 @@ - + - + - + @@ -69895,7 +74249,7 @@ - + @@ -69903,18 +74257,29 @@ - + - + - + - + - + + + + + + + + + + + + @@ -69922,7 +74287,23 @@ - + + + + + + + + + + + + + + + + + @@ -69930,7 +74311,7 @@ - + @@ -69938,7 +74319,7 @@ - + @@ -69952,7 +74333,15 @@ - + + + + + + + + + @@ -69972,7 +74361,7 @@ - + @@ -69981,26 +74370,26 @@ - + - + - + - + - + - + @@ -70008,15 +74397,15 @@ - + - + - + @@ -70024,7 +74413,7 @@ - + @@ -70035,7 +74424,7 @@ - + @@ -70043,7 +74432,7 @@ - + @@ -70054,7 +74443,7 @@ - + @@ -70065,7 +74454,7 @@ - + @@ -70079,7 +74468,7 @@ - + @@ -70087,7 +74476,7 @@ - + @@ -70104,7 +74493,7 @@ - + @@ -70112,15 +74501,15 @@ - + - + - + @@ -70128,7 +74517,7 @@ - + @@ -70154,12 +74543,12 @@ - + - + @@ -70200,7 +74589,18 @@ - + + + + + + + + + + + + @@ -70208,39 +74608,223 @@ - + - + - + - + - + - + - + - + - + + + + + + + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -70248,141 +74832,73 @@ - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -70393,38 +74909,18 @@ - + - + - + - + - - - - - - - - - - - - - - - - - - - - - + @@ -70433,23 +74929,43 @@ - + - + - + - + - + + + + + + + + + + + + + - + + + + + + + + + @@ -70457,180 +74973,109 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -70638,7 +75083,7 @@ - + @@ -70649,7 +75094,7 @@ - + @@ -70657,7 +75102,7 @@ - + @@ -70667,11 +75112,17 @@ + + + + + + @@ -70701,7 +75152,7 @@ - + @@ -70739,18 +75190,11 @@ + - - - - - - - - @@ -70779,6 +75223,8 @@ + + @@ -71127,9 +75573,16 @@ + + + + + + + @@ -71152,6 +75605,7 @@ + @@ -71248,19 +75702,19 @@ - + - + - + - + @@ -71395,7 +75849,7 @@ - + @@ -71413,16 +75867,16 @@ - + - + - + @@ -71449,31 +75903,31 @@ - + - + - + - + - + - + - + - + - + @@ -71707,9 +76161,11 @@ + + @@ -71730,6 +76186,7 @@ + @@ -71759,11 +76216,13 @@ + + @@ -71879,6 +76338,7 @@ + @@ -72605,6 +77065,7 @@ + @@ -72910,8 +77371,14 @@ + + + + + + @@ -73456,7 +77923,7 @@ - + @@ -73495,6 +77962,7 @@ + @@ -73502,6 +77970,18 @@ + + + + + + + + + + + + @@ -73631,10 +78111,26 @@ - + + + + + + + + + + + + + + + + + + - @@ -73859,7 +78355,7 @@ - + @@ -73958,6 +78454,14 @@ + + + + + + + + @@ -73971,9 +78475,10 @@ - + + @@ -74129,6 +78634,9 @@ + + + @@ -74252,7 +78760,7 @@ - + @@ -74273,7 +78781,7 @@ - + @@ -74349,7 +78857,7 @@ - + @@ -74418,8 +78926,8 @@ - - + + @@ -74461,6 +78969,7 @@ + @@ -74976,6 +79485,7 @@ + @@ -75025,7 +79535,7 @@ - + @@ -75157,6 +79667,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -75172,7 +79708,7 @@ - + @@ -75183,7 +79719,7 @@ - + @@ -75205,13 +79741,72 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -75536,6 +80131,7 @@ + @@ -75551,6 +80147,23 @@ + + + + + + + + + + + + + + + + + @@ -75805,7 +80418,7 @@ - + @@ -75874,6 +80487,65 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -75950,6 +80622,7 @@ + @@ -76100,6 +80773,7 @@ + @@ -76112,6 +80786,14 @@ + + + + + + + + @@ -76382,6 +81064,29 @@ + + + + + + + + + + + + + + + + + + + + + + + @@ -76490,6 +81195,14 @@ + + + + + + + + @@ -76594,6 +81307,7 @@ + @@ -76638,11 +81352,17 @@ + + + + + + - + @@ -76652,6 +81372,17 @@ + + + + + + + + + + + @@ -76680,6 +81411,7 @@ + @@ -76778,6 +81510,14 @@ + + + + + + + + @@ -77518,7 +82258,7 @@ - + @@ -77636,6 +82376,7 @@ + @@ -78269,11 +83010,19 @@ - + + + + + + + + + @@ -78736,6 +83485,11 @@ + + + + + @@ -78918,6 +83672,7 @@ + @@ -78955,117 +83710,117 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -79154,7 +83909,14 @@ - + + + + + + + + @@ -79230,6 +83992,14 @@ + + + + + + + + @@ -79263,6 +84033,7 @@ + @@ -79387,6 +84158,13 @@ + + + + + + + @@ -79405,6 +84183,7 @@ + @@ -79438,6 +84217,26 @@ + + + + + + + + + + + + + + + + + + + + @@ -79473,6 +84272,13 @@ + + + + + + + @@ -79505,6 +84311,12 @@ + + + + + + @@ -79715,7 +84527,7 @@ - + @@ -79776,10 +84588,25 @@ + + + + + + + + + + + + + + + @@ -79839,7 +84666,7 @@ - + @@ -79960,10 +84787,10 @@ - + - + @@ -80163,7 +84990,38 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -80332,7 +85190,7 @@ - + @@ -80667,7 +85525,7 @@ - + @@ -80685,6 +85543,7 @@ + @@ -80743,6 +85602,7 @@ + @@ -80769,7 +85629,7 @@ - + @@ -80786,6 +85646,17 @@ + + + + + + + + + + + @@ -80840,16 +85711,31 @@ + + + + + + + + + + + + + + + @@ -81136,6 +86022,7 @@ + @@ -81145,6 +86032,14 @@ + + + + + + + + @@ -81957,7 +86852,53 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -82288,6 +87229,7 @@ + @@ -82452,6 +87394,7 @@ + @@ -82557,6 +87500,7 @@ + @@ -82724,7 +87668,23 @@ - + + + + + + + + + + + + + + + + + @@ -82882,6 +87842,13 @@ + + + + + + + @@ -83021,6 +87988,11 @@ + + + + + @@ -83280,7 +88252,7 @@ - + @@ -83319,6 +88291,7 @@ + @@ -83370,7 +88343,7 @@ - + @@ -83402,6 +88375,7 @@ + @@ -83696,12 +88670,12 @@ - + - + - + @@ -83757,7 +88731,9 @@ + + @@ -83771,7 +88747,7 @@ - + @@ -83807,7 +88783,7 @@ - + @@ -83984,6 +88960,17 @@ + + + + + + + + + + + @@ -84286,7 +89273,21 @@ - + + + + + + + + + + + + + + + @@ -84551,6 +89552,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -84579,13 +89609,13 @@ - + - + @@ -84685,7 +89715,7 @@ - + @@ -84762,7 +89792,7 @@ - + @@ -84797,7 +89827,7 @@ - + @@ -84811,7 +89841,7 @@ - + @@ -84933,13 +89963,13 @@ - + - + - + @@ -84987,7 +90017,7 @@ - + @@ -85093,7 +90123,20 @@ + + + + + + + + + + + + + @@ -85105,9 +90148,18 @@ + + + + + + + + + @@ -85179,6 +90231,11 @@ + + + + + @@ -85210,7 +90267,7 @@ - + @@ -85242,7 +90299,12 @@ + + + + + @@ -85290,22 +90352,22 @@ - + - + - + - + - + - + @@ -85323,6 +90385,12 @@ + + + + + + @@ -85382,6 +90450,7 @@ + @@ -85419,7 +90488,7 @@ - + @@ -85452,6 +90521,17 @@ + + + + + + + + + + + @@ -85524,6 +90604,7 @@ + @@ -85865,7 +90946,7 @@ - + @@ -85957,6 +91038,7 @@ + @@ -86041,6 +91123,12 @@ + + + + + + @@ -86091,7 +91179,7 @@ - + @@ -86148,13 +91236,14 @@ + - + @@ -86226,7 +91315,7 @@ - + @@ -86331,7 +91420,7 @@ - + @@ -86543,6 +91632,7 @@ + @@ -86561,6 +91651,9 @@ + + + @@ -86653,8 +91746,22 @@ + + + + + + + + + + + + + + - + @@ -86694,6 +91801,23 @@ + + + + + + + + + + + + + + + + + @@ -86797,6 +91921,7 @@ + @@ -86940,7 +92065,7 @@ - + @@ -86998,6 +92123,7 @@ + @@ -87123,7 +92249,7 @@ - + @@ -87696,15 +92822,16 @@ - + + - + - + - + @@ -88112,7 +93239,7 @@ - + @@ -88121,7 +93248,7 @@ - + @@ -88310,6 +93437,11 @@ + + + + + @@ -88464,12 +93596,12 @@ - + - + - + @@ -88690,6 +93822,7 @@ + @@ -89124,7 +94257,7 @@ - + @@ -89201,7 +94334,7 @@ - + @@ -89237,6 +94370,7 @@ + @@ -89308,7 +94442,17 @@ - + + + + + + + + + + + @@ -89373,10 +94517,10 @@ - + - + @@ -89657,6 +94801,7 @@ + @@ -89664,7 +94809,7 @@ - + @@ -90219,10 +95364,10 @@ - + - + @@ -90234,7 +95379,7 @@ - + @@ -90392,6 +95537,7 @@ + @@ -90470,6 +95616,11 @@ + + + + + @@ -90582,6 +95733,7 @@ + @@ -90682,7 +95834,47 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -90713,6 +95905,7 @@ + @@ -90727,6 +95920,23 @@ + + + + + + + + + + + + + + + + + @@ -90777,6 +95987,7 @@ + @@ -90792,7 +96003,86 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -90950,7 +96240,7 @@ - + @@ -91051,7 +96341,7 @@ - + @@ -91269,7 +96559,44 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -91295,6 +96622,14 @@ + + + + + + + + @@ -91317,6 +96652,7 @@ + @@ -91356,6 +96692,29 @@ + + + + + + + + + + + + + + + + + + + + + + + @@ -91433,7 +96792,7 @@ - + @@ -91571,6 +96930,17 @@ + + + + + + + + + + + @@ -91642,21 +97012,21 @@ - + - + - + - + - + - + @@ -91747,6 +97117,7 @@ + @@ -91933,7 +97304,7 @@ - + @@ -92591,6 +97962,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -92618,6 +98018,11 @@ + + + + + @@ -92906,7 +98311,7 @@ - + @@ -92930,6 +98335,13 @@ + + + + + + + @@ -93073,7 +98485,7 @@ - + @@ -93137,6 +98549,7 @@ + @@ -93312,6 +98725,14 @@ + + + + + + + + @@ -93350,6 +98771,7 @@ + @@ -93522,6 +98944,7 @@ + @@ -93671,6 +99094,9 @@ + + + @@ -93761,6 +99187,10 @@ + + + + @@ -94272,7 +99702,7 @@ - + @@ -94383,16 +99813,16 @@ - + - + - + @@ -94760,7 +100190,7 @@ - + @@ -94906,6 +100336,7 @@ + @@ -95069,7 +100500,85 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -95083,866 +100592,15 @@ - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -95950,7 +100608,23 @@ - + + + + + + + + + + + + + + + + + @@ -95958,7 +100632,7 @@ - + @@ -95969,7 +100643,15 @@ - + + + + + + + + + @@ -95977,7 +100659,7 @@ - + @@ -95994,7 +100676,7 @@ - + @@ -96002,7 +100684,7 @@ - + @@ -96010,7 +100692,7 @@ - + @@ -96024,7 +100706,7 @@ - + @@ -96038,14 +100720,14 @@ - + - + - + @@ -96054,10 +100736,10 @@ - + - + @@ -96074,7 +100756,7 @@ - + @@ -96085,7 +100767,7 @@ - + @@ -96096,13 +100778,13 @@ - + - + @@ -96110,7 +100792,7 @@ - + @@ -96121,7 +100803,7 @@ - + @@ -96132,7 +100814,7 @@ - + @@ -96140,7 +100822,7 @@ - + @@ -96316,179 +100998,131 @@ - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + - + - + + + + + + + + + + + - + - + - + - - + + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - - - - - - - - - - - - - - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -96508,7 +101142,7 @@ - + @@ -96516,13 +101150,13 @@ - + - + @@ -96539,7 +101173,7 @@ - + @@ -96553,21 +101187,53 @@ - + - - - - + - + + + + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -96575,7 +101241,7 @@ - + @@ -96589,7 +101255,7 @@ - + @@ -96600,7 +101266,7 @@ - + @@ -96617,15 +101283,15 @@ - + - + - + @@ -96633,7 +101299,7 @@ - + @@ -96644,7 +101310,7 @@ - + @@ -96655,7 +101321,7 @@ - + @@ -96663,7 +101329,36 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -96680,100 +101375,719 @@ - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + + + + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + - + - - + + - - - - - - - - - - - - - - + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -96784,7 +102098,7 @@ - + @@ -96792,7 +102106,7 @@ - + @@ -96812,7 +102126,7 @@ - + @@ -96823,7 +102137,7 @@ - + @@ -96834,12 +102148,12 @@ - + - + @@ -96847,7 +102161,56 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -96855,7 +102218,7 @@ - + @@ -96863,7 +102226,7 @@ - + @@ -96871,12 +102234,12 @@ - + - + @@ -96884,7 +102247,18 @@ - + + + + + + + + + + + + @@ -96892,7 +102266,7 @@ - + @@ -96900,7 +102274,7 @@ - + @@ -96908,7 +102282,7 @@ - + @@ -96916,7 +102290,7 @@ - + @@ -96924,7 +102298,7 @@ - + @@ -96965,7 +102339,7 @@ - + @@ -96973,13 +102347,13 @@ - + - + @@ -97005,7 +102379,7 @@ - + @@ -97070,7 +102444,7 @@ - + @@ -97081,7 +102455,7 @@ - + @@ -97098,7 +102472,7 @@ - + @@ -97142,9 +102516,9 @@ - + - + @@ -97153,7 +102527,7 @@ - + @@ -97164,7 +102538,7 @@ - + @@ -97184,7 +102558,7 @@ - + @@ -97192,7 +102566,7 @@ - + @@ -97209,7 +102583,7 @@ - + @@ -97238,12 +102612,12 @@ - + - + @@ -97252,10 +102626,10 @@ - + - + @@ -97269,17 +102643,17 @@ - + - + - + @@ -97287,7 +102661,7 @@ - + @@ -97298,7 +102672,7 @@ - + @@ -97306,48 +102680,355 @@ - + - - - - - - - - - - - - - + - + - - - - - - - - - - - - - - - - - - - - + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -97355,7 +103036,7 @@ - + @@ -97363,7 +103044,7 @@ - + @@ -97377,7 +103058,7 @@ - + @@ -97397,7 +103078,7 @@ - + @@ -97405,7 +103086,7 @@ - + @@ -97413,63 +103094,12 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + @@ -97478,12 +103108,12 @@ - + - + @@ -97491,12 +103121,12 @@ - + - + @@ -97504,20 +103134,12 @@ - + - - - - - - - - - + @@ -97534,7 +103156,7 @@ - + @@ -97542,38 +103164,9 @@ - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -97588,12 +103181,12 @@ - + - + @@ -97619,7 +103212,7 @@ - + @@ -97630,7 +103223,7 @@ - + @@ -97650,7 +103243,58 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -97673,7 +103317,7 @@ - + @@ -97708,7 +103352,7 @@ - + @@ -97719,7 +103363,7 @@ - + @@ -97733,12 +103377,12 @@ - + - + @@ -97749,7 +103393,7 @@ - + @@ -97763,7 +103407,7 @@ - + @@ -97774,7 +103418,7 @@ - + @@ -97788,7 +103432,7 @@ - + @@ -97802,20 +103446,20 @@ - + - + - + - + @@ -97829,7 +103473,7 @@ - + @@ -97837,7 +103481,7 @@ - + @@ -97848,7 +103492,7 @@ - + @@ -97865,7 +103509,7 @@ - + @@ -97885,7 +103529,7 @@ - + @@ -97902,7 +103546,7 @@ - + @@ -97913,7 +103557,7 @@ - + @@ -97921,7 +103565,7 @@ - + @@ -97929,7 +103573,7 @@ - + @@ -97937,7 +103581,7 @@ - + @@ -97951,7 +103595,7 @@ - + @@ -97959,7 +103603,7 @@ - + @@ -97970,7 +103614,7 @@ - + @@ -97987,7 +103631,7 @@ - + @@ -98013,45 +103657,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -98358,7 +103964,7 @@ - + @@ -98425,7 +104031,7 @@ - + @@ -98444,6 +104050,11 @@ + + + + + @@ -98506,6 +104117,7 @@ + @@ -98619,7 +104231,7 @@ - + @@ -98733,6 +104345,7 @@ + @@ -98790,7 +104403,7 @@ - + @@ -98818,6 +104431,7 @@ + @@ -99096,6 +104710,26 @@ + + + + + + + + + + + + + + + + + + + + @@ -99150,6 +104784,7 @@ + @@ -99212,6 +104847,70 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -99229,6 +104928,7 @@ + @@ -99639,6 +105339,71 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -99875,39 +105640,39 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -99918,6 +105683,7 @@ + @@ -99954,6 +105720,7 @@ + @@ -100068,6 +105835,17 @@ + + + + + + + + + + + @@ -100105,7 +105883,7 @@ - + @@ -100313,10 +106091,33 @@ - + + + + + + + + + + + + + + + + + + + + + + + - + + @@ -100574,7 +106375,7 @@ - + @@ -100662,6 +106463,29 @@ + + + + + + + + + + + + + + + + + + + + + + + @@ -100710,6 +106534,7 @@ + @@ -100815,6 +106640,20 @@ + + + + + + + + + + + + + + @@ -100955,6 +106794,7 @@ + @@ -101277,7 +107117,50 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -101312,7 +107195,7 @@ - + @@ -101404,6 +107287,7 @@ + @@ -101583,27 +107467,27 @@ - + - + - + - + - + - + - + - + @@ -101745,7 +107629,7 @@ - + @@ -101847,6 +107731,11 @@ + + + + + @@ -101925,6 +107814,20 @@ + + + + + + + + + + + + + + @@ -101949,6 +107852,7 @@ + @@ -101974,6 +107878,7 @@ + @@ -102008,10 +107913,10 @@ - + - + @@ -102433,7 +108338,7 @@ - + @@ -102685,7 +108590,7 @@ - + @@ -102736,7 +108641,7 @@ - + @@ -102767,7 +108672,7 @@ - + @@ -102845,6 +108750,7 @@ + @@ -102998,6 +108904,7 @@ + @@ -103050,7 +108957,9 @@ + + @@ -103207,7 +109116,7 @@ - + @@ -103322,6 +109231,7 @@ + @@ -103370,6 +109280,7 @@ + @@ -103551,7 +109462,7 @@ - + @@ -103564,6 +109475,14 @@ + + + + + + + + @@ -103762,12 +109681,16 @@ + + + + @@ -104129,6 +110052,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -104190,7 +110139,7 @@ - + @@ -104555,7 +110504,7 @@ - + @@ -104613,6 +110562,7 @@ + @@ -104717,9 +110667,6 @@ - - - @@ -104769,6 +110716,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -104822,7 +110795,183 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -104933,6 +111082,104 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -105032,7 +111279,7 @@ - + @@ -105490,7 +111737,7 @@ - + @@ -105714,7 +111961,19 @@ + + + + + + + + + + + + @@ -106005,7 +112264,7 @@ - + @@ -106040,6 +112299,9 @@ + + + @@ -106057,7 +112319,38 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -106084,6 +112377,7 @@ + @@ -106104,6 +112398,20 @@ + + + + + + + + + + + + + + @@ -106242,7 +112550,7 @@ - + @@ -106314,7 +112622,7 @@ - + @@ -106335,7 +112643,7 @@ - + @@ -106347,7 +112655,7 @@ - + @@ -106387,21 +112695,21 @@ - + - + - + - + - + - + @@ -106462,13 +112770,13 @@ - + - + @@ -106729,7 +113037,7 @@ - + @@ -106806,7 +113114,7 @@ - + @@ -106961,36 +113269,36 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -107099,6 +113407,7 @@ + @@ -107121,6 +113430,17 @@ + + + + + + + + + + + @@ -107201,8 +113521,8 @@ - - + + @@ -107414,6 +113734,11 @@ + + + + + @@ -107635,7 +113960,6 @@ - @@ -107698,6 +114022,11 @@ + + + + + @@ -107709,6 +114038,14 @@ + + + + + + + + @@ -107726,6 +114063,12 @@ + + + + + + @@ -107766,6 +114109,11 @@ + + + + + @@ -107880,6 +114228,13 @@ + + + + + + + @@ -108077,6 +114432,12 @@ + + + + + + @@ -108119,17 +114480,17 @@ - - - + + + - - + + @@ -108270,8 +114631,8 @@ - - + + @@ -108290,10 +114651,10 @@ - - - - + + + + @@ -108379,10 +114740,10 @@ - - - - + + + + @@ -108392,6 +114753,13 @@ + + + + + + + @@ -108399,22 +114767,28 @@ - - - - - - - - - - - + + + + + + + + + + + - - - + + + + + + + + + @@ -108423,12 +114797,12 @@ - - - - - - + + + + + + @@ -108438,17 +114812,17 @@ - - - - - + + + + + - - - - + + + + @@ -108467,6 +114841,11 @@ + + + + + @@ -108496,11 +114875,20 @@ - - - - - + + + + + + + + + + + + + + @@ -108511,6 +114899,20 @@ + + + + + + + + + + + + + + @@ -108529,9 +114931,9 @@ - - - + + + @@ -108546,10 +114948,25 @@ - - - - + + + + + + + + + + + + + + + + + + + @@ -108557,6 +114974,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -108587,6 +115044,18 @@ + + + + + + + + + + + + @@ -108602,21 +115071,21 @@ - - - - - - - + + + + + + + - - - - - - + + + + + + @@ -108630,39 +115099,51 @@ - - - + + + - - - - - - - - + - - - + + + - - - - - - + + + + + - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -108704,10 +115185,35 @@ - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -108737,6 +115243,12 @@ + + + + + + @@ -108746,10 +115258,10 @@ - - - - + + + + @@ -108787,9 +115299,9 @@ - - - + + + @@ -108822,13 +115334,19 @@ - + + + + + + + + + + - - - - + @@ -108837,6 +115355,17 @@ + + + + + + + + + + + @@ -108852,11 +115381,11 @@ - - - - - + + + + + @@ -108864,11 +115393,11 @@ - - - - - + + + + + @@ -108885,6 +115414,27 @@ + + + + + + + + + + + + + + + + + + + + + @@ -109034,6 +115584,27 @@ + + + + + + + + + + + + + + + + + + + + + @@ -109065,6 +115636,11 @@ + + + + + @@ -109086,6 +115662,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -109173,6 +115789,16 @@ + + + + + + + + + + @@ -109180,6 +115806,17 @@ + + + + + + + + + + + @@ -109188,80 +115825,110 @@ - + + - - + + + - + - - + + + - + + + + - + - + + + + + + + + + + + - - + + - - - - - + + + + + + + - + + + + + + - + - + - + + + + - + - + + + + + @@ -109285,14 +115952,26 @@ + + + + + + + + + + + + @@ -109308,7 +115987,11 @@ + + + + @@ -109350,6 +116033,11 @@ + + + + + @@ -109371,6 +116059,12 @@ + + + + + + @@ -109401,6 +116095,14 @@ + + + + + + + + @@ -109541,6 +116243,10 @@ + + + + @@ -109565,37 +116271,43 @@ + + + + + + - - - + + + - - + + - - - + + + - - - + + + - - - + + + @@ -109608,16 +116320,16 @@ - - - - + + + + - - - - + + + + @@ -109687,38 +116399,38 @@ - - - - - - - + + + + + + + - - - + + + - - - - - - + + + + + + - - - - + + + + - - + + @@ -109751,15 +116463,22 @@ + + + + + + + - - + + - - - - + + + + @@ -109781,15 +116500,15 @@ - - - - + + + + - - - + + + @@ -109797,15 +116516,15 @@ - - - - - + + + + + - - + + @@ -109813,6 +116532,10 @@ + + + + @@ -109843,10 +116566,10 @@ - - - - + + + + @@ -109860,9 +116583,9 @@ - - - + + + @@ -109881,58 +116604,58 @@ - - - - + + + + - - - - - + + + + + - - - - + + + + - - + + - - - + + + - - - + + + - - - - - + + + + + - - + + - - - - + + + + - - + + @@ -109940,32 +116663,32 @@ - - - - - - - + + + + + + + - - - - + + + + - - - - - + + + + + - - - - + + + + @@ -109999,14 +116722,14 @@ - - - - - - - - + + + + + + + + @@ -110024,12 +116747,12 @@ - - + + - - + + @@ -110042,10 +116765,10 @@ - - - - + + + + @@ -110072,13 +116795,13 @@ - - - + + + - - + + @@ -110107,8 +116830,8 @@ - - + + @@ -110117,8 +116840,8 @@ - - + + @@ -110163,8 +116886,8 @@ - - + + @@ -110186,10 +116909,10 @@ - - - - + + + + @@ -110225,9 +116948,9 @@ - - - + + + @@ -110339,8 +117062,8 @@ - - + + @@ -110358,43 +117081,43 @@ - - - - + + + + - - - - + + + + - - - + + + - - - - + + + + - - - + + + - - - + + + - - - - + + + + @@ -110412,9 +117135,9 @@ - - - + + + @@ -110462,67 +117185,67 @@ - - - - + + + + - - - - - + + + + + - - - - - - + + + + + + - - - - - - - + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - - + + + + + + + + + + @@ -110538,26 +117261,26 @@ - - + + - - + + - - + + - - + + - - + + @@ -110574,23 +117297,23 @@ - - - + + + - - - - - + + + + + - - - - - + + + + + @@ -110600,8 +117323,8 @@ - - + + @@ -110614,8 +117337,8 @@ - - + + @@ -110664,8 +117387,8 @@ - - + + @@ -110681,19 +117404,19 @@ - - - + + + - - - + + + - - - + + + @@ -110765,10 +117488,26 @@ - - - - + + + + + + + + + + + + + + + + + + + + @@ -110799,13 +117538,13 @@ - - - - - - - + + + + + + + @@ -110899,9 +117638,9 @@ - - - + + + @@ -110912,6 +117651,11 @@ + + + + + @@ -110997,63 +117741,94 @@ - - + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - + + + + - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + + + + + + - - + + + - - + + + + + + + + - - - + + + + + - - + + @@ -111089,11 +117864,25 @@ - - - + + + + + + + + + + + + + + + + + @@ -111114,8 +117903,8 @@ - - + + @@ -111130,27 +117919,29 @@ - - - + + + + + - - - + + + - - + + - - - + + + @@ -111159,6 +117950,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -111187,6 +118005,20 @@ + + + + + + + + + + + + + + @@ -111197,28 +118029,28 @@ - - - + + + - - + + - - + + - - - - - + + + + + @@ -111233,30 +118065,30 @@ - - - - - + + + + + - - - + + + - - + + - - + + - - - - + + + + @@ -111269,8 +118101,8 @@ - - + + @@ -111301,10 +118133,10 @@ - - - - + + + + @@ -111331,14 +118163,14 @@ - - + + - - - - + + + + @@ -111347,8 +118179,8 @@ - - + + @@ -111359,7 +118191,14 @@ + + + + + + + @@ -111367,17 +118206,23 @@ - - + + - - + + + + + + + + @@ -111412,8 +118257,8 @@ - - + + @@ -111424,18 +118269,38 @@ - - + + - - - + + + + + + + + + + + + + + + + + + + + + + + - - - + + + @@ -111447,8 +118312,8 @@ - - + + @@ -111464,6 +118329,11 @@ + + + + + @@ -111494,12 +118364,12 @@ - - + + - - + + @@ -111510,23 +118380,23 @@ - - + + - - - + + + - - - - + + + + @@ -111537,6 +118407,15 @@ + + + + + + + + + @@ -111544,6 +118423,10 @@ + + + + @@ -111557,45 +118440,45 @@ - - - + + + - - - + + + - - - + + + - - - - + + + + - - - - - + + + + + - - - - - + + + + + - + - + @@ -111629,8 +118512,8 @@ - - + + @@ -111647,10 +118530,10 @@ - - - - + + + + @@ -111699,32 +118582,32 @@ - - - - - - + + + + + + - - - - - - + + + + + + - - - + + + - + @@ -111767,10 +118650,10 @@ - - - - + + + + @@ -111779,7 +118662,7 @@ - + @@ -111800,8 +118683,8 @@ - - + + @@ -111826,22 +118709,22 @@ - - + + - - - + + + - - - + + + @@ -111874,11 +118757,11 @@ - - - - - + + + + + @@ -111886,44 +118769,44 @@ - - - - - + + + + + - - - - + + + + - - - - - + + + + + - - - - + + + + - - + + - - + + - - - - + + + + @@ -111940,8 +118823,8 @@ - - + + @@ -111965,14 +118848,14 @@ - - - + + + - - - + + + @@ -111992,10 +118875,10 @@ - - - - + + + + @@ -112006,8 +118889,8 @@ - - + + @@ -112023,26 +118906,48 @@ - - + + - - + + - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -112071,32 +118976,32 @@ - - - - - + + + + + - - - + + + - - - - - - + + + + + + - - - - - - + + + + + + @@ -112106,6 +119011,12 @@ + + + + + + @@ -112120,11 +119031,11 @@ - - - - - + + + + + @@ -112148,18 +119059,18 @@ - - - + + + - - + + - - - + + + @@ -112182,12 +119093,12 @@ - - + + - - + + @@ -112198,8 +119109,8 @@ - - + + @@ -112229,29 +119140,29 @@ - - + + - - - - + + + + - - + + - - - + + + - - - + + + @@ -112264,9 +119175,9 @@ - - - + + + @@ -112278,9 +119189,9 @@ - - - + + + @@ -112288,9 +119199,9 @@ - - - + + + @@ -112298,10 +119209,10 @@ - - - - + + + + @@ -112327,15 +119238,15 @@ - - - - + + + + - - - + + + @@ -112355,8 +119266,8 @@ - - + + @@ -112372,8 +119283,8 @@ - - + + @@ -112381,8 +119292,8 @@ - - + + @@ -112393,9 +119304,9 @@ - - - + + + @@ -112412,22 +119323,31 @@ - - + + - - + + + + + + + + + + + @@ -112442,6 +119362,11 @@ + + + + + @@ -112464,6 +119389,10 @@ + + + + @@ -112505,9 +119434,9 @@ - - - + + + @@ -112515,17 +119444,17 @@ - - - - + + + + - - - - + + + + @@ -112533,14 +119462,14 @@ - - - + + + - - - + + + @@ -112549,16 +119478,23 @@ - - - + + + - - - - - + + + + + + + + + + + + @@ -112569,6 +119505,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + @@ -112596,16 +119556,16 @@ - - + + - - + + - - + + @@ -112618,8 +119578,8 @@ - - + + @@ -112641,12 +119601,12 @@ - - - - - - + + + + + + @@ -112655,40 +119615,40 @@ - - - + + + - - - - - - - + + + + + + + - - + + - - - + + + - - - - + + + + - - - - + + + + @@ -112719,19 +119679,19 @@ - - - + + + - - + + - - - - + + + + @@ -112743,9 +119703,9 @@ - - - + + + @@ -112768,11 +119728,18 @@ - - - - - + + + + + + + + + + + + @@ -112788,14 +119755,14 @@ - - - + + + - - - + + + @@ -112803,32 +119770,58 @@ - - - + + + - - - + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - - - + + + + @@ -112851,10 +119844,10 @@ - - - - + + + + @@ -112862,15 +119855,15 @@ - - - - + + + + - - - + + + @@ -112878,9 +119871,9 @@ - - - + + + @@ -112895,9 +119888,9 @@ - - - + + + @@ -112907,6 +119900,16 @@ + + + + + + + + + + @@ -113034,6 +120037,11 @@ + + + + + @@ -113079,35 +120087,42 @@ - - - - + + + + - - - + + + - - - - + + + + - - - - - + + + + + - - - - + + + + + + + + + + + @@ -113146,14 +120161,14 @@ - - - - - - - - + + + + + + + + @@ -113161,15 +120176,15 @@ - - - + + + - - - - + + + + @@ -113184,8 +120199,8 @@ - - + + @@ -113225,20 +120240,20 @@ - - - + + + - - - + + + - - - - + + + + @@ -113262,15 +120277,15 @@ - - - - + + + + - - - + + + @@ -113340,16 +120355,16 @@ - - - + + + - - - - - + + + + + @@ -113377,6 +120392,11 @@ + + + + + @@ -113388,39 +120408,39 @@ - - - + + + - - - - - + + + + + - - + + - - - - - + + + + + - - + + - - + + @@ -113519,9 +120539,9 @@ - - - + + + @@ -113536,74 +120556,79 @@ - - + + - - - + + + - - + + - - - + + + + + + + + - - - - + + + + - - + + - - - - + + + + - - + + - - - + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - + + + + - - + + @@ -113620,12 +120645,12 @@ - - - - - - + + + + + + @@ -113637,23 +120662,23 @@ - - - + + + - - + + - - - - + + + + @@ -113705,6 +120730,10 @@ + + + + @@ -113796,31 +120825,31 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - - - - + + + + + @@ -113934,10 +120963,15 @@ - - + + + + + + + @@ -113951,12 +120985,12 @@ - - + + - - + + @@ -113968,25 +121002,29 @@ - - + + + + + + - - + + - - - + + + @@ -113996,57 +121034,57 @@ - - + + - - - + + + - - + + - - - + + + - - - - + + + + - - - + + + - - - + + + - - - + + + - - + + - - - + + + - - - + + + @@ -114059,14 +121097,14 @@ - - - + + + - - - + + + @@ -114092,13 +121130,13 @@ - - - - - - - + + + + + + + @@ -114280,69 +121318,69 @@ - - + + - - - + + + - - + + - - - - - + + + + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - + + - - + + - - - - - + + + + + - - - - + + + + @@ -114350,8 +121388,8 @@ - - + + @@ -114364,13 +121402,13 @@ - - - + + + - - + + @@ -114378,19 +121416,19 @@ - - - - - + + + + + - - - - - - + + + + + + @@ -114398,16 +121436,16 @@ - - + + - - + + - - + + @@ -114415,13 +121453,13 @@ - - + + - - - + + + @@ -114444,19 +121482,19 @@ - - - - - - - + + + + + + + - - - + + + @@ -114469,8 +121507,8 @@ - - + + @@ -114488,12 +121526,12 @@ - - + + - - + + @@ -114506,19 +121544,19 @@ - - - - - + + + + + - - + + @@ -114526,24 +121564,24 @@ - - - - + + + + - - - + + + - - + + - - + + @@ -114669,12 +121707,12 @@ - - - - - - + + + + + + @@ -114719,10 +121757,10 @@ - - - - + + + + @@ -114821,10 +121859,10 @@ - - - - + + + + @@ -114869,9 +121907,9 @@ - - - + + + @@ -114992,19 +122030,19 @@ - - - + + + - - - + + + - - - + + + @@ -115049,8 +122087,8 @@ - - + + @@ -115087,8 +122125,8 @@ - - + + @@ -115103,32 +122141,32 @@ - - + + - - - + + + - - - + + + - - + + - - - + + + - - - + + + @@ -115148,12 +122186,12 @@ - - + + - - + + @@ -115162,62 +122200,62 @@ - - - + + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - - + + + - - + + - - + + - - + + - - - - + + + + @@ -115241,17 +122279,17 @@ - - + + - - + + - - - + + + @@ -115261,20 +122299,20 @@ - - + + - - + + - - + + @@ -115331,9 +122369,9 @@ - - - + + + @@ -115341,52 +122379,52 @@ - - + + - - - - - + + + + + - - - - - - - + + + + + + + - - - - + + + + - - - - + + + + - - - - - - + + + + + + - - - - - - + + + + + + @@ -115402,9 +122440,9 @@ - - - + + + @@ -115419,29 +122457,29 @@ - - - - - + + + + + - - - - - + + + + + - - - - + + + + - - - + + + @@ -115454,8 +122492,8 @@ - - + + @@ -115514,20 +122552,20 @@ - - + + - - - - - - - - - - + + + + + + + + + + @@ -115555,24 +122593,24 @@ - - - - + + + + - - + + - - - - + + + + @@ -115625,24 +122663,24 @@ - - + + - - + + - - - + + + - - + + @@ -115650,14 +122688,14 @@ - - - + + + - - - + + + @@ -115665,21 +122703,21 @@ - - - + + + - - + + - - + + @@ -115694,28 +122732,28 @@ - - - - + + + + - - + + - - + + - - - + + + - - - + + + @@ -115725,15 +122763,15 @@ - - - + + + - - - - + + + + @@ -115755,27 +122793,27 @@ - - - - + + + + - - - - - + + + + + - - - + + + - - + + @@ -115784,9 +122822,9 @@ - - - + + + @@ -115795,30 +122833,30 @@ - - - + + + - - + + - - - + + + - - - - + + + + - - - - + + + + @@ -115826,22 +122864,22 @@ - - + + - - - + + + - - + + - - - + + + @@ -115851,8 +122889,8 @@ - - + + @@ -115863,8 +122901,8 @@ - - + + @@ -115892,36 +122930,36 @@ - - - + + + - - + + - - - + + + - - - + + + - - + + - - - + + + - - + + @@ -115957,8 +122995,8 @@ - - + + @@ -115969,12 +123007,12 @@ - - + + - - + + @@ -115995,39 +123033,39 @@ - - - + + + - - - - - - - + + + + + + + - - - - - - + + + + + + - - + + - - - + + + - - + + @@ -116054,21 +123092,21 @@ - - - - - + + + + + - - - - + + + + - - + + @@ -116085,8 +123123,8 @@ - - + + @@ -116109,31 +123147,51 @@ - - + + - - + + - - - - - - - + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + - - - - + + + + @@ -116146,8 +123204,8 @@ - - + + @@ -116157,22 +123215,22 @@ - - + + - + - - - - + + + + - - - + + + @@ -116185,7 +123243,7 @@ - + @@ -116197,8 +123255,12 @@ - - + + + + + + @@ -116212,33 +123274,33 @@ - - + + - - - - - - + + + + + + - - - - - + + + + + - - - - - - - - + + + + + + + + @@ -116248,8 +123310,8 @@ - - + + @@ -116326,59 +123388,63 @@ - - - + + + - - - + + + - - - + + + - - - + + + + + + + - - - + + + - - - - + + + + - - + + - - - - - + + + + + - - - - - - + + + + + + @@ -116406,9 +123472,9 @@ - - - + + + @@ -116417,13 +123483,13 @@ - - + + - - - + + + @@ -116513,13 +123579,13 @@ - - - + + + - - + + @@ -116563,10 +123629,10 @@ - + - + @@ -116581,22 +123647,22 @@ - - - + + + - - - + + + - - + + - - + + @@ -116625,39 +123691,44 @@ - - - - + + + + - - - - - - - - + + + + + + + + - - + + - - + + - + - - + + + + + + + @@ -116666,30 +123737,42 @@ - - - - + + - - + + + + + + + + + + + + + + + + - - + + - - - - + + + + - - - - + + + + @@ -116703,28 +123786,28 @@ - - - - - + + + + + - - + + - - - - - + + + + + - - - - + + + + @@ -116746,17 +123829,17 @@ - - + + - - + + - - + + @@ -116765,12 +123848,12 @@ - - + + - - + + @@ -116807,9 +123890,9 @@ - - - + + + @@ -116844,23 +123927,30 @@ - - - + + + - - + + - - - + + + - - - + + + + + + + + + + @@ -116868,37 +123958,50 @@ - - - - + + + + - - - - - - - - + + + + + - - - - + + + + + + + + + + + + + + + + + + + + - - - - - + + + + + - - + + @@ -116909,12 +124012,12 @@ - - + + - - + + @@ -116943,14 +124046,14 @@ - - + + - - - - + + + + @@ -116971,36 +124074,36 @@ - - - - - + + + + + - - + + - - - + + + - - - - - - + + + + + + - - - - - - + + + + + + @@ -117011,33 +124114,33 @@ - - + + - - - + + + - - - - + + + + - - - + + + - - + + - - - + + + @@ -117222,15 +124325,15 @@ - - - - + + + + - - - + + + @@ -117263,11 +124366,11 @@ - - - - - + + + + + @@ -117343,9 +124446,9 @@ - - - + + + @@ -117405,27 +124508,89 @@ - - + + - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + @@ -117444,9 +124609,9 @@ - - - + + + @@ -117455,9 +124620,9 @@ - - - + + + @@ -117465,19 +124630,19 @@ - - - - - - + + + + + + - - - + + + @@ -117487,8 +124652,8 @@ - - + + @@ -117515,8 +124680,8 @@ - - + + @@ -117549,20 +124714,20 @@ - - + + - - + + - - + + @@ -117589,16 +124754,21 @@ - - + + - - + + - - + + + + + + + @@ -117606,26 +124776,26 @@ - - - + + + - - - - + + + + - - - - + + + + - - - + + + @@ -117634,11 +124804,11 @@ - + - - + + @@ -117705,9 +124875,9 @@ - - - + + + @@ -117841,24 +125011,24 @@ - - + + - - - - - - + + + + + + - - - - - - + + + + + + @@ -117866,12 +125036,12 @@ - - + + - - + + @@ -117886,8 +125056,8 @@ - - + + @@ -117896,12 +125066,12 @@ - - + + - - + + @@ -117910,10 +125080,10 @@ - - - - + + + + @@ -117931,8 +125101,8 @@ - - + + @@ -117956,13 +125126,13 @@ - - - - - - - + + + + + + + @@ -117971,32 +125141,32 @@ - - - + + + - - + + - - - - - - + + + + + + - - - - - - - - - + + + + + + + + + @@ -118023,9 +125193,9 @@ - - - + + + @@ -118037,15 +125207,21 @@ - - - - + + + + + + + + + + - - - + + + @@ -118053,17 +125229,17 @@ - - - - + + + + - - - - - + + + + + @@ -118082,22 +125258,36 @@ - - + + - - + + - - + + - - + + + + + + + + + + + + + + + + @@ -118110,14 +125300,15 @@ - - - + + + + - - + + @@ -118137,7 +125328,7 @@ - + @@ -118176,30 +125367,37 @@ - - + + - - + + - - + + + + + + + + + - - - - + + + + @@ -118219,21 +125417,21 @@ - - - - + + + + - - - + + + - - - - + + + + @@ -118244,73 +125442,73 @@ - - - - - - - - + + + + + + + + - - + + - - - + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - + + + - - + + - - + + - - - + + + - - - - - + + + + + - - + + @@ -118323,31 +125521,31 @@ - - - - + + + + - - - - + + + + - - - - + + + + - - - + + + @@ -118355,34 +125553,34 @@ - - - - + + + + - - - + + + - - - - + + + + - - - - + + + + - - - - - + + + + + @@ -118392,10 +125590,10 @@ - - - - + + + + @@ -118404,16 +125602,16 @@ - - - - + + + + - - - - + + + + @@ -118423,37 +125621,37 @@ - - - - + + + + - - - - - + + + + + - - - - - + + + + + - - - - + + + + - - - - - + + + + + @@ -118461,9 +125659,9 @@ - - - + + + @@ -118474,11 +125672,11 @@ - - - - - + + + + + @@ -118521,11 +125719,11 @@ - + - - + + @@ -118545,10 +125743,10 @@ - + - + @@ -118561,30 +125759,30 @@ - - + + - - + + - - + + - - - - + + + + - - + + @@ -118592,37 +125790,37 @@ - - - - + + + + - - - - + + + + - - + + - - - - + + + + - - - - + + + + - - - + + + @@ -118636,10 +125834,10 @@ - + - + @@ -118650,11 +125848,11 @@ - - - - - + + + + + @@ -118670,14 +125868,27 @@ + + + + + + + + + + + + + - - - + + + @@ -118688,39 +125899,60 @@ - - + + - - - + + + - - - + + + - - - - + + + + + + + + + + + + + + + + + - - - + + + - - - + + + - - - + + + + + + + + + + + @@ -118773,10 +126005,10 @@ - - - - + + + + @@ -118906,18 +126138,18 @@ - - + + - - + + - - - - + + + + @@ -118958,8 +126190,8 @@ - - + + @@ -119012,8 +126244,8 @@ - - + + @@ -119025,9 +126257,9 @@ - - - + + + @@ -119038,23 +126270,23 @@ - - - - - - + + + + + + - - + + - - - - - + + + + + @@ -119111,8 +126343,8 @@ - - + + @@ -119127,9 +126359,9 @@ - - - + + + @@ -119171,8 +126403,8 @@ - - + + @@ -119195,45 +126427,46 @@ - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - + + + + @@ -119241,8 +126474,8 @@ - - + + @@ -119250,13 +126483,13 @@ - - - + + + - - + + @@ -119267,9 +126500,9 @@ - - - + + + @@ -119279,9 +126512,9 @@ - - - + + + @@ -119308,105 +126541,105 @@ - - - - + + + + - - + + - - + + - - + + - - - - + + + + - - - + + + - - - + + + - - + + - - + + - - + + - - + + - - - - - + + + + + - - + + - - + + - - + + - - + + - - - + + + - - - + + + - - - + + + - - + + - - + + @@ -119417,48 +126650,48 @@ - - - + + + - - + + - - - - - + + + + + - - + + - - + + - - - - + + + + - - - - + + + + @@ -119473,11 +126706,11 @@ - - - - - + + + + + @@ -119494,16 +126727,16 @@ - - - - + + + + - - - - + + + + @@ -119541,8 +126774,12 @@ - - + + + + + + @@ -119567,124 +126804,128 @@ - - - - - - - - - + + + + + + + + + - - - + + + - - + + - - - + + + - - - - - + + + + + - - - - - + + + + + - - - - + + + + - - - + + + - - - - + + + + - - - - + + + + - - + + - - - + + + - - - + + + - - - + + + + + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - - + + + + - - + + @@ -119693,12 +126934,12 @@ - - + + - - + + @@ -119706,26 +126947,26 @@ - - - - + + + + - - - + + + - - - - - + + + + + @@ -119738,8 +126979,8 @@ - - + + @@ -119754,14 +126995,14 @@ - - - - + + + + - - + + @@ -119772,44 +127013,44 @@ - - - - + + + + - - - + + + - - + + - - - + + + - - - + + + - - - + + + - - - - + + + + @@ -119827,9 +127068,9 @@ - - - + + + @@ -119837,9 +127078,9 @@ - - - + + + @@ -119858,28 +127099,28 @@ - - - + + + - - - + + + - - + + - - + + - - - - + + + + @@ -119894,9 +127135,9 @@ - - - + + + @@ -119907,33 +127148,33 @@ - - - - + + + + - - + + - - + + - - - + + + - - - + + + - - - + + + @@ -119941,95 +127182,100 @@ - - - + + + - - + + - - - + + + - - - + + + - - - - + + + + - - + + - - - + + + - - - + + + - - + + - - + + - - - + + + + + + + + - - - - + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - + + @@ -120037,65 +127283,65 @@ - - + + - - - - - + + + + + - - - + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - - - + + + + + + - - - - - + + + + + - - - - + + + + @@ -120113,12 +127359,12 @@ - - - - - - + + + + + + @@ -120129,12 +127375,12 @@ - - - - - - + + + + + + @@ -120142,18 +127388,18 @@ - - - - + + + + - - + + @@ -120170,9 +127416,9 @@ - - - + + + @@ -120208,12 +127454,12 @@ - - + + - - + + @@ -120242,6 +127488,7 @@ + @@ -120269,17 +127516,17 @@ - - - - - - + + + + + + - - - + + + @@ -120287,8 +127534,8 @@ - - + + @@ -120303,12 +127550,12 @@ - - + + - - + + @@ -120342,19 +127589,19 @@ - - - + + + - - - + + + - - - + + + @@ -120369,8 +127616,8 @@ - - + + @@ -120379,9 +127626,9 @@ - - - + + + @@ -120407,14 +127654,14 @@ - - - + + + - - - + + + @@ -120458,18 +127705,18 @@ - - - + + + - - - + + + @@ -120503,8 +127750,8 @@ - - + + @@ -120569,10 +127816,10 @@ - - - - + + + + @@ -120611,20 +127858,20 @@ - - - - - - + + + + + + - - + + - - + + @@ -120632,21 +127879,21 @@ - - - - - + + + + + - - - - + + + + - - + + @@ -120673,21 +127920,21 @@ - - - - + + + + - - - - - - - - - + + + + + + + + + @@ -120703,6 +127950,17 @@ + + + + + + + + + + + @@ -120725,9 +127983,9 @@ - - - + + + @@ -120735,12 +127993,12 @@ - - - - - - + + + + + + @@ -120795,6 +128053,10 @@ + + + + @@ -120873,8 +128135,8 @@ - - + + @@ -120894,29 +128156,29 @@ - - - + + + - - + + - - + + - - + + - - + + - - + + @@ -120928,14 +128190,14 @@ - - - + + + - - - + + + @@ -120943,28 +128205,54 @@ - - - - + + + + - - - - + + + + - - + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -120992,16 +128280,20 @@ - - + + - - + + + + + + - - + + @@ -121021,9 +128313,13 @@ - - - + + + + + + + @@ -121034,28 +128330,42 @@ - - - - + + + + - - - - + + + + - - + + + + + + - - + + + + + + + + + + + + @@ -121075,8 +128385,13 @@ - - + + + + + + + @@ -121089,88 +128404,102 @@ - - - - + + + + - - + + - - - + + + - + - + - + - - - - + + + + - - - - + + + + - - - - - - - - - + + + + + + + + + - - + + - - + + - - - - - + + + + + + + + + + - - - - - - + + + + + + - - - - - - + + + + + + + + + + + + + + + @@ -121197,9 +128526,9 @@ - - - + + + @@ -121209,8 +128538,8 @@ - - + + @@ -121218,13 +128547,25 @@ - - - + + + + + + + + + + + + + + + - - + + @@ -121277,23 +128618,23 @@ - - - - - + + + + + - - + + - - + + @@ -121301,8 +128642,8 @@ - - + + @@ -121360,10 +128701,10 @@ - - - - + + + + @@ -121371,23 +128712,23 @@ - - - - + + + + - - - - - + + + + + - - - - + + + + @@ -121401,19 +128742,19 @@ - - - + + + - - - - + + + + - - + + @@ -121431,31 +128772,36 @@ - - - + + + - - - + + + + + + + + - - + + - - - + + + - - + + - - + + @@ -121473,18 +128819,26 @@ + + + + - - + + - - - + + + + + + + @@ -121538,8 +128892,8 @@ - - + + @@ -121554,16 +128908,16 @@ - - + + - - + + - - + + @@ -121574,8 +128928,8 @@ - - + + @@ -121594,12 +128948,12 @@ - - + + - - + + @@ -121615,8 +128969,8 @@ - - + + @@ -121719,10 +129073,10 @@ - - - - + + + + @@ -121731,30 +129085,30 @@ - - - - - - - - + + + + + + + + - - - - + + + + - - - + + + - - - + + + @@ -121767,34 +129121,47 @@ - - + + - - + + + + + + + + + + + + + + + - - + + @@ -121836,10 +129203,10 @@ - - - - + + + + @@ -121857,6 +129224,11 @@ + + + + + @@ -121867,6 +129239,11 @@ + + + + + @@ -121882,6 +129259,11 @@ + + + + + @@ -121897,40 +129279,45 @@ - - + + - - + + + - - - - - - + + + + + + + + + + - - - - + + + + - - - + + + - - - + + + @@ -121963,28 +129350,36 @@ - - - - - - - + + + + + + + + + + + - - + + - - + + + + + + @@ -121998,17 +129393,22 @@ + + + + + - - - - - - + + + + + + - - + + @@ -122020,45 +129420,48 @@ - - + + - - + + - - - + + + - - - - + + + + - - - + + + - - - - + + + + + + + - + @@ -122165,17 +129568,22 @@ - - + + - - + + - - - + + + + + + + + @@ -122206,12 +129614,12 @@ - - + + - - + + @@ -122229,10 +129637,10 @@ - - - - + + + + @@ -122243,89 +129651,100 @@ - + - - + + + - - - + + + - - - - + + + + - - - - + + + + - - + + - - + + - - + + - - - - + + + + - - + + - - + + - - + + - - + + - - + + - + - - + + - - + + - - + + - - - - + + + + + + + + + + + + + + @@ -122335,8 +129754,8 @@ - - + + @@ -122347,8 +129766,8 @@ - - + + @@ -122400,10 +129819,10 @@ - - - - + + + + @@ -122428,11 +129847,11 @@ - - - - - + + + + + @@ -122504,16 +129923,16 @@ - - + + - - + + - - + + @@ -122554,17 +129973,17 @@ - - + + - - + + - - - + + + @@ -122587,10 +130006,10 @@ - - - - + + + + @@ -122604,31 +130023,31 @@ - - - - + + + + - - - + + + - - - + + + - - - + + + - - - + + + @@ -122638,26 +130057,26 @@ - - - + + + - - - + + + - - - - + + + + - - - - + + + + @@ -122695,9 +130114,9 @@ - - - + + + @@ -122706,9 +130125,9 @@ - - - + + + @@ -122716,32 +130135,42 @@ - + - - - - + + + - - - - - - - - - - - + + + + - - - - + + + + + + + + + + + + + + + + + + + + + + @@ -122762,52 +130191,57 @@ - - - - - + + + + + - - - - - + + + + + - - - - + + + + - - - + + + - - + + - - - - - + + + + + - - + + - - + + + + + + + @@ -122846,12 +130280,16 @@ - - - - - - + + + + + + + + + + @@ -122880,12 +130318,12 @@ - - - - - - + + + + + + @@ -122899,16 +130337,16 @@ - - - - + + + + - - - - + + + + @@ -122925,10 +130363,10 @@ - - - - + + + + @@ -122938,42 +130376,42 @@ - - - + + + - - - - - - + + + + + + - - + + - - - - - - - + + + + + + + - - - - - + + + + + - - - + + + @@ -122983,38 +130421,38 @@ - - - + + + - - - - - + + + + + - - - - - + + + + + - - - - - + + + + + - - + + - - + + @@ -123022,50 +130460,50 @@ - - - - + + + + - - - - + + + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - - + + + - - - + + + @@ -123075,38 +130513,38 @@ - - - + + + - - - - - + + + + + - - - - - + + + + + - - - + + + - - - + + + - - - + + + @@ -123131,15 +130569,15 @@ - - - + + + - - - - + + + + @@ -123152,11 +130590,18 @@ - - - - - + + + + + + + + + + + + @@ -123166,25 +130611,25 @@ - - + + - - + + - - - - - - - + + + + + + + - - + + @@ -123195,9 +130640,9 @@ - - - + + + @@ -123249,9 +130694,9 @@ - - - + + + @@ -123280,22 +130725,22 @@ - - - - - + + + + + - - - - - + + + + + - - + + @@ -123304,6 +130749,13 @@ + + + + + + + @@ -123367,8 +130819,8 @@ - - + + @@ -123380,9 +130832,9 @@ - - - + + + @@ -123392,11 +130844,11 @@ - - - - - + + + + + @@ -123406,8 +130858,8 @@ - - + + @@ -123415,21 +130867,25 @@ - - + + - - - - + + + + - - - + + + + + + + @@ -123447,22 +130903,27 @@ - - + + - - - - - - + + + + + + + + + + + - - - - + + + + @@ -123482,6 +130943,11 @@ + + + + + @@ -123489,9 +130955,9 @@ - - - + + + @@ -123503,6 +130969,10 @@ + + + + @@ -123528,38 +130998,45 @@ - - - + + + - - - - + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - + + + + + + + + + + + + @@ -123588,9 +131065,9 @@ - - - + + + @@ -123616,10 +131093,10 @@ - - - - + + + + @@ -123638,6 +131115,11 @@ + + + + + @@ -123701,10 +131183,15 @@ - - - - + + + + + + + + + @@ -123722,19 +131209,34 @@ - - + + + + + + + + + + + + + + + + + @@ -123750,6 +131252,11 @@ + + + + + @@ -123770,6 +131277,11 @@ + + + + + @@ -123845,6 +131357,10 @@ + + + + @@ -123874,6 +131390,11 @@ + + + + + @@ -123896,9 +131417,28 @@ - - - + + + + + + + + + + + + + + + + + + + + + + @@ -123954,9 +131494,9 @@ - - - + + + @@ -123977,12 +131517,12 @@ - - - - - - + + + + + + @@ -123995,44 +131535,44 @@ - - - + + + - - - - + + + + - - + + - - + + - - - - - - + + + + + + - - + + - - - + + + @@ -124048,6 +131588,7 @@ + @@ -124092,9 +131633,9 @@ - - - + + + @@ -124106,9 +131647,9 @@ - - - + + + @@ -124174,9 +131715,9 @@ - - - + + + @@ -124200,9 +131741,9 @@ - - - + + + @@ -124210,20 +131751,33 @@ - - + + - - + + - - - - + + + + + + + + + + + + + + + + + @@ -124239,9 +131793,9 @@ - - - + + + @@ -124259,8 +131813,8 @@ - - + + @@ -124295,10 +131849,10 @@ - - - - + + + + @@ -124307,21 +131861,21 @@ - - - - + + + + - - - + + + - - - - + + + + @@ -124329,15 +131883,15 @@ - - - - + + + + - - - + + + @@ -124350,12 +131904,12 @@ - - + + - - + + @@ -124367,36 +131921,36 @@ - - + + - - + + - + - + - - + + - - + + - - - + + + @@ -124414,10 +131968,15 @@ - - - - + + + + + + + + + @@ -124430,27 +131989,27 @@ - - - + + + - - - - + + + + - - - + + + - - - - + + + + @@ -124460,21 +132019,26 @@ + + + + + - - - - + + + + - - - + + + @@ -124482,9 +132046,9 @@ - - - + + + @@ -124493,9 +132057,9 @@ - - - + + + @@ -124503,9 +132067,13 @@ - + + + + + @@ -124513,8 +132081,8 @@ - - + + @@ -124546,29 +132114,29 @@ - - + + - - - + + + - - + + - - + + - - + + @@ -124581,27 +132149,27 @@ - - + + - - + + - - - - - + + + + + - - + + - - + + @@ -124610,8 +132178,8 @@ - - + + @@ -124633,19 +132201,19 @@ - - - - - - - - - + + + + + + + + + - - + + @@ -124657,9 +132225,9 @@ - - - + + + @@ -124678,14 +132246,14 @@ - - - - + + + + - - + + @@ -124709,21 +132277,21 @@ - + - - + + - - - - + + + + - - + + @@ -124736,8 +132304,8 @@ - - + + @@ -124747,24 +132315,24 @@ - - - - - + + + + + - - - - + + + + - - - - - + + + + + @@ -124774,32 +132342,32 @@ - - - - + + + + - - - + + + - - - + + + - - - + + + - - - - + + + + @@ -124808,8 +132376,8 @@ - - + + @@ -124835,8 +132403,8 @@ - - + + @@ -124850,13 +132418,13 @@ - - - + + + - - + + @@ -124869,6 +132437,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -124882,6 +132479,14 @@ + + + + + + + + @@ -124892,11 +132497,11 @@ - - - - - + + + + + @@ -125025,9 +132630,9 @@ - - - + + + @@ -125108,15 +132713,15 @@ - - + + - - - - - + + + + + @@ -125137,8 +132742,8 @@ - - + + @@ -125149,21 +132754,21 @@ - - + + - - + + - - + + - - - + + + @@ -125174,8 +132779,8 @@ - - + + @@ -125198,16 +132803,16 @@ - - + + - - + + - - + + @@ -125226,12 +132831,12 @@ - - + + - - + + @@ -125260,9 +132865,9 @@ - - - + + + @@ -125280,9 +132885,9 @@ - - - + + + @@ -125293,12 +132898,12 @@ - - + + - - + + @@ -125317,16 +132922,16 @@ - - - - - - - - - - + + + + + + + + + + @@ -125346,32 +132951,31 @@ - - - + + - - - - + + + + - - - + + + - - - + + + @@ -125383,9 +132987,9 @@ - - - + + + @@ -125404,15 +133008,15 @@ - - - + + + - - - - + + + + @@ -125430,8 +133034,8 @@ - - + + @@ -125442,38 +133046,38 @@ - - + + - - + + - - + + - - - + + + - - + + - - - + + + - - + + @@ -125543,14 +133147,14 @@ - - - + + + - - - + + + @@ -125587,18 +133191,18 @@ - - + + - - + + - - - - + + + + @@ -125609,44 +133213,44 @@ - - + + - - + + - - - + + + - - - + + + - - + + - - + + - - - - + + + + @@ -125657,13 +133261,13 @@ - - - + + + - - + + @@ -125674,9 +133278,9 @@ - - - + + + @@ -125688,10 +133292,10 @@ - - - - + + + + @@ -125727,8 +133331,8 @@ - - + + @@ -125736,9 +133340,9 @@ - - - + + + @@ -125750,9 +133354,9 @@ - - - + + + @@ -125760,8 +133364,8 @@ - - + + @@ -125774,14 +133378,14 @@ - - - - - - - - + + + + + + + + @@ -125794,13 +133398,13 @@ - - - + + + - - + + @@ -125815,6 +133419,10 @@ + + + + @@ -125845,36 +133453,80 @@ - - - + + + + + + + + - - + + + + + + + + + + + + + + + + + + - - + + - - + + + + + + + - - + + - - - + + + + + + + + + + + + + + + + + + + + + @@ -125894,6 +133546,12 @@ + + + + + + @@ -125952,24 +133610,82 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + @@ -125983,10 +133699,10 @@ - - - - + + + + @@ -126058,9 +133774,9 @@ - - - + + + @@ -126076,49 +133792,55 @@ - - - + + + - - - - + + + + - - - - + + + + - - - + + + + - - + + + + + + + - - - + + + - - - + + + - - - + + + @@ -126149,12 +133871,12 @@ - - + + - - + + @@ -126174,8 +133896,8 @@ - - + + @@ -126207,8 +133929,8 @@ - - + + @@ -126244,8 +133966,8 @@ - - + + @@ -126312,9 +134034,9 @@ - - - + + + @@ -126385,25 +134107,25 @@ - - - - - - - + + + + + + + - - + + - - + + @@ -126441,20 +134163,20 @@ - - + + - - + + - - + + @@ -126492,16 +134214,16 @@ - - - - + + + + - - - - + + + + @@ -126524,52 +134246,56 @@ - - + + - - + + + + + + - - - - - + + + + + - - - + + + - - - - + + + + - - - - + + + + - - - - - + + + + + - - - + + + @@ -126619,11 +134345,11 @@ - - - - - + + + + + @@ -126633,31 +134359,37 @@ - - - + + + - - + + - - + + - - - - + + + + - - + + + + + + + + - - + + @@ -126665,8 +134397,8 @@ - - + + @@ -126683,13 +134415,13 @@ - - + + - - - + + + @@ -126703,9 +134435,13 @@ - - - + + + + + + + @@ -126724,17 +134460,17 @@ - - - + + + - - + + @@ -126829,34 +134565,34 @@ - - - + + + - - - - - + + + + + - - - - - + + + + + - - - - + + + + - - - + + + @@ -126872,17 +134608,17 @@ - - + + - - - - + + + + @@ -126893,26 +134629,26 @@ - - - - + + + + - - - - + + + + - - + + - - - - + + + + @@ -126951,15 +134687,15 @@ - - - - + + + + - - - + + + @@ -126985,9 +134721,9 @@ - - - + + + @@ -127011,6 +134747,7 @@ + diff --git a/android/abi_gki_aarch64_exynos b/android/abi_gki_aarch64_exynos new file mode 100644 index 000000000000..03e6c4c8e24e --- /dev/null +++ b/android/abi_gki_aarch64_exynos @@ -0,0 +1,1842 @@ +[abi_symbol_list] +# commonly used symbols + add_timer + add_timer_on + adjust_managed_page_count + alloc_chrdev_region + alloc_netdev_mqs + __alloc_pages_nodemask + __alloc_percpu + __alloc_skb + alloc_workqueue + __arch_copy_from_user + __arch_copy_to_user + arm64_const_caps_ready + arm64_use_ng_mappings + __arm_smccc_smc + atomic_notifier_call_chain + atomic_notifier_chain_register + atomic_notifier_chain_unregister + bcmp + bitmap_parse + bitmap_parselist + bitmap_print_to_pagebuf + blocking_notifier_call_chain + blocking_notifier_chain_register + blocking_notifier_chain_unregister + bpf_trace_run1 + bpf_trace_run2 + bpf_trace_run3 + bpf_trace_run4 + bpf_trace_run5 + bpf_trace_run6 + bpf_trace_run7 + cancel_delayed_work + cancel_delayed_work_sync + cancel_work_sync + cdev_add + cdev_del + cdev_init + __cfi_slowpath + __check_object_size + __class_create + class_destroy + class_unregister + clk_disable + clk_enable + clk_get + clk_get_rate + __clk_is_enabled + clk_prepare + clk_put + clk_set_parent + clk_set_rate + clk_unprepare + cma_alloc + cma_release + compat_alloc_user_space + complete + complete_all + completion_done + config_ep_by_speed + config_group_init_type_name + console_suspend_enabled + console_unlock + __const_udelay + __cpu_active_mask + cpu_bit_bitmap + cpufreq_cpu_get + cpufreq_cpu_put + cpufreq_quick_get + cpufreq_register_notifier + cpufreq_unregister_notifier + __cpuhp_remove_state + __cpuhp_setup_state + cpu_hwcap_keys + cpu_hwcaps + cpumask_next + cpumask_next_and + cpu_number + __cpu_online_mask + __cpu_possible_mask + cpu_scale + cpu_subsys + crc32_le + crypto_destroy_tfm + crypto_register_alg + crypto_register_scomp + crypto_unregister_alg + crypto_unregister_scomp + _ctype + debugfs_create_bool + debugfs_create_dir + debugfs_create_file + debugfs_create_u32 + debugfs_remove + default_llseek + delayed_work_timer_fn + del_timer + del_timer_sync + destroy_workqueue + dev_driver_string + _dev_emerg + _dev_err + device_create + device_create_bin_file + device_create_file + device_destroy + device_for_each_child + device_initialize + device_init_wakeup + device_property_present + device_property_read_u32_array + device_remove_file + _dev_info + __dev_kfree_skb_any + devm_add_action + devm_clk_get + devm_free_irq + devm_ioremap + devm_ioremap_resource + devm_iounmap + devm_kasprintf + devm_kfree + devm_kmalloc + devm_kmemdup + __devm_of_phy_provider_register + devm_phy_create + devm_phy_get + devm_pinctrl_get + devm_pinctrl_put + devm_platform_ioremap_resource + devm_regulator_get + __devm_request_region + devm_request_threaded_irq + __devm_reset_control_get + _dev_notice + dev_pm_opp_add + dev_pm_opp_find_freq_ceil + dev_pm_opp_get_voltage + dev_pm_opp_put + dev_set_name + _dev_warn + disable_irq + disable_irq_nosync + dma_alloc_attrs + dma_buf_attach + dma_buf_begin_cpu_access + dma_buf_detach + dma_buf_end_cpu_access + dma_buf_export + dma_buf_get + dma_buf_map_attachment + dma_buf_mmap + dma_buf_put + dma_buf_unmap_attachment + dma_buf_vmap + dma_buf_vunmap + dma_fence_add_callback + dma_fence_context_alloc + dma_fence_default_wait + dma_fence_get_status + dma_fence_init + dma_fence_release + dma_fence_remove_callback + dma_fence_signal + dma_free_attrs + dma_heap_buffer_alloc + dma_heap_find + dma_heap_put + dmam_alloc_attrs + dma_map_page_attrs + dma_map_sg_attrs + dma_release_channel + dma_request_chan + dma_set_coherent_mask + dma_set_mask + dma_sync_sg_for_cpu + dma_sync_sg_for_device + dma_sync_single_for_cpu + dma_sync_single_for_device + dma_unmap_page_attrs + dma_unmap_sg_attrs + down + down_interruptible + down_read + down_write + driver_unregister + dump_stack + enable_irq + event_triggers_call + failure_tracking + fd_install + find_last_bit + find_next_bit + find_next_zero_bit + find_vpid + finish_wait + flush_dcache_page + flush_work + flush_workqueue + fput + free_irq + free_netdev + __free_pages + free_pages + free_percpu + freq_qos_update_request + generic_file_llseek + gen_pool_add_owner + gen_pool_alloc_algo_owner + gen_pool_free_owner + get_cpu_device + get_device + __get_free_pages + get_thermal_instance + get_unused_fd_flags + gic_nonsecure_priorities + gpiod_direction_input + gpiod_direction_output_raw + gpiod_get_raw_value + gpiod_set_raw_value + gpiod_to_irq + gpio_free + gpio_request + gpio_request_one + gpio_to_desc + gserial_alloc_line + gserial_connect + gserial_disconnect + handle_level_irq + handle_nested_irq + hex_dump_to_buffer + hrtimer_active + hrtimer_cancel + hrtimer_forward + hrtimer_init + hrtimer_start_range_ns + i2c_add_numbered_adapter + i2c_del_adapter + i2c_del_driver + i2c_new_dummy_device + i2c_register_driver + i2c_smbus_read_byte_data + i2c_smbus_read_i2c_block_data + i2c_smbus_write_byte_data + i2c_smbus_write_i2c_block_data + i2c_transfer + i2c_transfer_buffer_flags + i2c_unregister_device + idr_alloc + idr_remove + __iio_device_register + iio_device_unregister + __init_rwsem + __init_swait_queue_head + init_task + init_timer_key + init_wait_entry + __init_waitqueue_head + input_allocate_device + input_event + input_free_device + input_register_device + input_set_capability + input_unregister_device + iomem_resource + iommu_get_domain_for_dev + iommu_map_sg + iommu_register_device_fault_handler + iommu_unmap + iommu_unregister_device_fault_handler + __ioremap + iounmap + __irq_alloc_descs + irq_get_irq_data + irq_modify_status + irq_of_parse_and_map + irq_set_affinity_hint + irq_set_chip_and_handler_name + irq_set_chip_data + irq_set_irq_wake + irq_to_desc + is_console_locked + is_vmalloc_addr + jiffies + jiffies_to_msecs + jiffies_to_usecs + kasan_flag_enabled + kasprintf + kernel_kobj + __kfifo_alloc + __kfifo_free + __kfifo_in + __kfifo_to_user + kfree + kfree_skb + kimage_voffset + __kmalloc + kmalloc_caches + kmalloc_order_trace + kmem_cache_alloc + kmem_cache_alloc_trace + kmem_cache_create + kmem_cache_destroy + kmem_cache_free + kmemdup + kobject_create_and_add + kobject_init_and_add + kobject_put + kobject_uevent_env + krealloc + kstrdup + kstrndup + kstrtobool + kstrtobool_from_user + kstrtoint + kstrtoint_from_user + kstrtoll + kstrtou8 + kstrtouint + kstrtouint_from_user + kstrtoull + kthread_bind + kthread_create_on_node + kthread_delayed_work_timer_fn + kthread_flush_worker + __kthread_init_worker + kthread_queue_work + kthread_should_stop + kthread_stop + kthread_worker_fn + ktime_get + ktime_get_mono_fast_ns + ktime_get_raw_ts64 + ktime_get_real_ts64 + ktime_get_ts64 + ktime_get_with_offset + kvfree + kvfree_call_rcu + kvmalloc_node + __list_add_valid + __list_del_entry_valid + __log_post_read_mmio + __log_read_mmio + __log_write_mmio + loops_per_jiffy + lzo1x_decompress_safe + memcpy + __memcpy_fromio + __memcpy_toio + memdup_user + memset + __memset_io + memstart_addr + mfd_add_devices + mfd_remove_devices + misc_deregister + misc_register + mod_delayed_work_on + mod_timer + module_layout + module_put + __msecs_to_jiffies + msleep + msleep_interruptible + __mutex_init + mutex_is_locked + mutex_lock + mutex_lock_interruptible + mutex_trylock + mutex_unlock + napi_complete_done + napi_gro_receive + __napi_schedule + napi_schedule_prep + __netdev_alloc_skb + netif_napi_add + netif_receive_skb + netif_tx_wake_queue + nla_put + no_llseek + nonseekable_open + noop_llseek + nr_cpu_ids + ns_to_kernel_old_timeval + ns_to_timespec64 + __num_online_cpus + of_address_to_resource + of_alias_get_id + of_count_phandle_with_args + of_device_get_match_data + of_device_is_available + of_device_is_compatible + of_find_compatible_node + of_find_device_by_node + of_find_matching_node_and_match + of_find_node_by_name + of_find_node_opts_by_path + of_find_property + of_get_child_by_name + of_get_cpu_node + of_get_named_gpio_flags + of_get_next_available_child + of_get_next_child + of_get_property + of_iomap + of_machine_is_compatible + of_match_device + of_match_node + of_n_addr_cells + of_n_size_cells + of_parse_phandle + of_phandle_iterator_init + of_phandle_iterator_next + of_platform_populate + of_property_count_elems_of_size + of_property_match_string + of_property_read_string + of_property_read_string_helper + of_property_read_u32_index + of_property_read_variable_u32_array + of_prop_next_string + of_prop_next_u32 + of_reserved_mem_device_init_by_idx + of_reserved_mem_device_release + of_reserved_mem_lookup + of_root + __page_pinner_put_page + panic + panic_notifier_list + param_array_ops + param_ops_bool + param_ops_charp + param_ops_int + param_ops_string + param_ops_uint + param_ops_ulong + PDE_DATA + __per_cpu_offset + perf_trace_buf_alloc + perf_trace_run_bpf_submit + pfn_valid + phy_power_off + phy_power_on + pinctrl_lookup_state + pinctrl_select_state + pin_user_pages + platform_device_unregister + __platform_driver_register + platform_driver_unregister + platform_get_irq + platform_get_irq_byname + platform_get_resource + platform_get_resource_byname + __pm_relax + pm_relax + __pm_runtime_disable + pm_runtime_enable + pm_runtime_forbid + pm_runtime_force_resume + pm_runtime_force_suspend + __pm_runtime_idle + __pm_runtime_resume + pm_runtime_set_autosuspend_delay + __pm_runtime_set_status + __pm_runtime_suspend + __pm_runtime_use_autosuspend + __pm_stay_awake + pm_stay_awake + pm_wakeup_ws_event + power_supply_changed + power_supply_get_by_name + power_supply_get_drvdata + power_supply_get_property + power_supply_register + power_supply_set_property + power_supply_unregister + preempt_schedule + preempt_schedule_notrace + prepare_to_wait_event + print_hex_dump + printk + proc_create + proc_create_data + proc_mkdir + proc_set_user + put_device + __put_page + __put_task_struct + put_unused_fd + queue_delayed_work_on + queue_work_on + ___ratelimit + raw_notifier_call_chain + raw_notifier_chain_register + _raw_read_lock + _raw_read_unlock + _raw_spin_lock + _raw_spin_lock_bh + _raw_spin_lock_irq + _raw_spin_lock_irqsave + _raw_spin_trylock + _raw_spin_unlock + _raw_spin_unlock_bh + _raw_spin_unlock_irq + _raw_spin_unlock_irqrestore + _raw_write_lock_irqsave + _raw_write_unlock_irqrestore + rb_insert_color + __rcu_read_lock + __rcu_read_unlock + refcount_warn_saturate + regcache_cache_only + __register_chrdev + register_chrdev_region + register_pm_notifier + register_reboot_notifier + register_restart_handler + register_shrinker + register_syscore_ops + regmap_read + regmap_update_bits_base + regmap_write + regulator_disable + regulator_enable + regulator_get + regulator_get_optional + regulator_is_enabled + regulator_put + regulator_set_voltage + release_firmware + __release_region + remap_pfn_range + remove_cpu + remove_proc_entry + request_firmware + __request_region + request_threaded_irq + reset_control_assert + reset_control_deassert + return_address + rtc_class_close + rtc_class_open + rtc_read_time + sched_clock + sched_setscheduler_nocheck + schedule + schedule_timeout + scnprintf + seq_lseek + seq_printf + seq_puts + seq_read + seq_release + set_cpus_allowed_ptr + sg_alloc_table + sg_free_table + sg_init_table + sg_miter_next + sg_miter_start + sg_miter_stop + sg_next + __sg_page_iter_next + __sg_page_iter_start + simple_attr_open + simple_attr_read + simple_attr_release + simple_attr_write + simple_open + simple_read_from_buffer + simple_write_to_buffer + single_open + single_release + skb_copy_expand + skb_dequeue + skb_pull + skb_push + skb_put + skb_queue_head + skb_queue_purge + skb_queue_tail + skb_trim + smp_call_function + smp_call_function_single + snd_pcm_format_width + snd_soc_dapm_get_enum_double + snd_soc_dapm_get_volsw + snd_soc_dapm_ignore_suspend + snd_soc_dapm_info_pin_switch + snd_soc_dapm_put_enum_double + snd_soc_dapm_put_volsw + snd_soc_dapm_sync + snd_soc_get_enum_double + snd_soc_get_volsw + snd_soc_info_enum_double + snd_soc_info_volsw + snd_soc_put_enum_double + snd_soc_put_volsw + snd_soc_register_component + snd_soc_unregister_component + snprintf + sort + sprintf + sscanf + __stack_chk_fail + static_key_slow_dec + static_key_slow_inc + stpcpy + strcasecmp + strcat + strchr + strcmp + strcpy + strlcat + strlcpy + strlen + strncasecmp + strncmp + strncpy + strnlen + strrchr + strscpy + strsep + strstr + __sw_hweight32 + __sw_hweight64 + sync_file_create + sync_file_get_fence + synchronize_rcu + syscon_regmap_lookup_by_phandle + sysfs_add_file_to_group + sysfs_create_bin_file + sysfs_create_file_ns + sysfs_create_group + sysfs_create_groups + sysfs_create_link + sysfs_emit + sysfs_notify + sysfs_remove_file_ns + sysfs_remove_group + sysfs_streq + system_wq + sys_tz + __tasklet_hi_schedule + tasklet_init + tasklet_kill + __tasklet_schedule + thermal_of_cooling_device_register + thermal_zone_get_temp + thermal_zone_get_zone_by_name + time64_to_tm + _totalram_pages + trace_event_buffer_commit + trace_event_buffer_reserve + trace_event_ignore_this_pid + trace_event_raw_init + trace_event_reg + trace_handle_return + __traceiter_android_vh_cpu_idle_enter + __traceiter_android_vh_cpu_idle_exit + __traceiter_device_pm_callback_end + __traceiter_device_pm_callback_start + __traceiter_pelt_cfs_tp + __traceiter_rwmmio_post_read + __traceiter_rwmmio_read + __traceiter_rwmmio_write + __traceiter_suspend_resume + __tracepoint_android_vh_cpu_idle_enter + __tracepoint_android_vh_cpu_idle_exit + __tracepoint_device_pm_callback_end + __tracepoint_device_pm_callback_start + __tracepoint_pelt_cfs_tp + tracepoint_probe_register + __tracepoint_rwmmio_post_read + __tracepoint_rwmmio_read + __tracepoint_rwmmio_write + __tracepoint_suspend_resume + trace_print_array_seq + trace_raw_output_prep + trace_seq_printf + try_module_get + __udelay + unpin_user_page + __unregister_chrdev + unregister_chrdev_region + unregister_pm_notifier + unregister_shrinker + up + up_read + up_write + usb_add_function + usb_copy_descriptors + usb_ep_autoconfig + usb_function_register + usb_function_unregister + usb_hub_find_child + usb_interface_id + usb_put_function_instance + usb_register_notify + usb_string_id + usb_unregister_notify + __usecs_to_jiffies + usleep_range + v4l2_ctrl_handler_free + v4l2_ctrl_handler_init_class + v4l2_ctrl_handler_setup + v4l2_ctrl_new_custom + v4l2_ctrl_new_std + v4l2_ctrl_new_std_menu + v4l2_device_register + v4l2_device_register_subdev + v4l2_device_set_name + v4l2_device_unregister + v4l2_device_unregister_subdev + v4l2_fh_add + v4l2_fh_del + v4l2_fh_exit + v4l2_fh_init + v4l2_i2c_subdev_init + v4l2_m2m_buf_queue + v4l2_m2m_buf_remove + v4l2_m2m_ctx_init + v4l2_m2m_ctx_release + v4l2_m2m_dqbuf + v4l2_m2m_get_curr_priv + v4l2_m2m_get_vq + v4l2_m2m_init + v4l2_m2m_job_finish + v4l2_m2m_mmap + v4l2_m2m_next_buf + v4l2_m2m_poll + v4l2_m2m_qbuf + v4l2_m2m_release + v4l2_m2m_reqbufs + v4l2_m2m_streamoff + v4l2_m2m_streamon + v4l2_subdev_call_wrappers + v4l2_subdev_init + v4l_bound_align_image + vabits_actual + vb2_buffer_done + vb2_dma_sg_memops + vb2_dqbuf + vb2_mmap + vb2_plane_cookie + vb2_plane_vaddr + vb2_poll + vb2_qbuf + vb2_querybuf + vb2_queue_init + vb2_queue_release + vb2_reqbufs + vb2_streamoff + vb2_streamon + vfree + video_devdata + video_device_alloc + video_device_release + video_ioctl2 + __video_register_device + video_unregister_device + vmalloc + vmalloc_to_pfn + vmap + vscnprintf + vsnprintf + vunmap + vzalloc + wait_for_completion + wait_for_completion_interruptible + wait_for_completion_interruptible_timeout + wait_for_completion_timeout + __wake_up + wake_up_process + wakeup_source_add + wakeup_source_register + wakeup_source_remove + wakeup_source_unregister + __warn_printk + +# required by clk_exynos.ko + __clk_get_hw + __clk_get_name + clk_get_parent + clk_hw_get_flags + clk_hw_get_name + clk_hw_get_parent + clk_register + clk_register_clkdev + clk_register_composite + clk_register_fixed_factor + clk_register_fixed_rate + clk_register_gate + clk_round_rate + of_clk_add_provider + of_clk_src_onecell_get + +# required by cmupmucal.ko + debugfs_create_x32 + +# required by cpif.ko + dev_get_by_name + init_dummy_netdev + init_net + register_netdev + si_meminfo + skb_dequeue_tail + unregister_netdev + +# required by cqhci.ko + devm_blk_ksm_init + dmam_free_coherent + mmc_cqe_request_done + mmc_hw_reset + +# required by dpu.ko + backlight_device_unregister + console_trylock + devm_backlight_device_register + dma_fence_wait_timeout + __do_once_done + __do_once_start + phy_init + regulator_get_exclusive + __v4l2_device_register_subdev_nodes + +# required by dss.ko + android_debug_symbol + dump_backtrace + input_close_device + input_open_device + input_register_handle + input_register_handler + input_unregister_handle + kstat + nr_irqs + register_console + register_die_notifier + rtc_time64_to_tm + touch_softlockup_watchdog + __traceiter_android_vh_ipi_stop + __traceiter_hrtimer_expire_entry + __traceiter_hrtimer_expire_exit + __traceiter_irq_handler_entry + __traceiter_irq_handler_exit + __traceiter_sched_switch + __traceiter_workqueue_execute_end + __traceiter_workqueue_execute_start + __tracepoint_android_vh_ipi_stop + __tracepoint_hrtimer_expire_entry + __tracepoint_hrtimer_expire_exit + __tracepoint_irq_handler_entry + __tracepoint_irq_handler_exit + __tracepoint_sched_switch + __tracepoint_workqueue_execute_end + __tracepoint_workqueue_execute_start + +# required by dw_mmc-exynos-fmp.ko + blk_ksm_init_passthrough + +# required by dw_mmc-exynos.ko + mmc_wait_for_req + sg_init_one + +# required by dw_mmc-srpmb.ko + blkdev_get_by_path + blk_execute_rq + blk_get_request + blk_put_request + capable + +# required by dw_mmc.ko + debugfs_create_x64 + device_property_read_string_array + mmc_add_host + mmc_alloc_host + mmc_can_gpio_cd + mmc_detect_change + mmc_free_host + mmc_gpio_get_cd + mmc_gpio_get_ro + mmc_of_parse + mmc_regulator_get_supply + mmc_regulator_set_ocr + mmc_regulator_set_vqmmc + mmc_remove_host + mmc_request_done + sdio_signal_irq + +# required by dwc3-exynos-usb.ko + platform_device_add + platform_device_add_properties + platform_device_add_resources + platform_device_alloc + platform_device_del + platform_device_put + pm_runtime_allow + __traceiter_dwc3_ep_queue + __traceiter_dwc3_readl + __traceiter_dwc3_writel + __tracepoint_dwc3_ep_queue + __tracepoint_dwc3_readl + __tracepoint_dwc3_writel + usb_gadget_set_state + usb_otg_state_string + +# required by ems.ko + activate_task + android_rvh_probe_register + available_idle_cpu + bpf_trace_run10 + bpf_trace_run11 + bpf_trace_run8 + bpf_trace_run9 + cgroup_taskset_first + cgroup_taskset_next + check_preempt_curr + cpufreq_add_update_util_hook + cpufreq_disable_fast_switch + cpufreq_driver_fast_switch + __cpufreq_driver_target + cpufreq_enable_fast_switch + cpufreq_frequency_table_get_index + cpufreq_register_governor + cpufreq_remove_update_util_hook + cpufreq_this_cpu_can_update + cpu_topology + deactivate_task + dev_pm_qos_add_notifier + dev_pm_qos_add_request + dev_pm_qos_remove_request + dev_pm_qos_update_request + get_pid_task + housekeeping_cpumask + irq_work_queue + irq_work_sync + kthread_cancel_work_sync + match_string + memory_read_from_buffer + raw_notifier_chain_unregister + _raw_write_trylock + reciprocal_value + resched_curr + runqueues + sched_feat_keys + sched_feat_names + sched_uclamp_used + set_next_entity + set_task_cpu + static_key_disable + stop_one_cpu_nowait + sysctl_sched_features + system_32bit_el0_cpumask + task_rq_lock + __traceiter_android_rvh_can_migrate_task + __traceiter_android_rvh_check_preempt_wakeup + __traceiter_android_rvh_cpu_cgroup_can_attach + __traceiter_android_rvh_dequeue_task + __traceiter_android_rvh_do_sched_yield + __traceiter_android_rvh_enqueue_task + __traceiter_android_rvh_find_busiest_queue + __traceiter_android_rvh_find_lowest_rq + __traceiter_android_rvh_find_new_ilb + __traceiter_android_rvh_post_init_entity_util_avg + __traceiter_android_rvh_replace_next_task_fair + __traceiter_android_rvh_sched_fork_init + __traceiter_android_rvh_sched_newidle_balance + __traceiter_android_rvh_sched_nohz_balancer_kick + __traceiter_android_rvh_sched_rebalance_domains + __traceiter_android_rvh_schedule + __traceiter_android_rvh_select_fallback_rq + __traceiter_android_rvh_select_task_rq_fair + __traceiter_android_rvh_select_task_rq_rt + __traceiter_android_rvh_set_task_cpu + __traceiter_android_rvh_uclamp_rq_util_with + __traceiter_android_rvh_update_misfit_status + __traceiter_android_rvh_wake_up_new_task + __traceiter_android_vh_arch_set_freq_scale + __traceiter_android_vh_binder_restore_priority + __traceiter_android_vh_binder_set_priority + __traceiter_android_vh_binder_wakeup_ilocked + __traceiter_android_vh_scheduler_tick + __traceiter_binder_transaction_received + __traceiter_pelt_dl_tp + __traceiter_pelt_irq_tp + __traceiter_pelt_rt_tp + __traceiter_pelt_se_tp + __traceiter_sched_overutilized_tp + __tracepoint_android_rvh_can_migrate_task + __tracepoint_android_rvh_check_preempt_wakeup + __tracepoint_android_rvh_cpu_cgroup_can_attach + __tracepoint_android_rvh_dequeue_task + __tracepoint_android_rvh_do_sched_yield + __tracepoint_android_rvh_enqueue_task + __tracepoint_android_rvh_find_busiest_queue + __tracepoint_android_rvh_find_lowest_rq + __tracepoint_android_rvh_find_new_ilb + __tracepoint_android_rvh_post_init_entity_util_avg + __tracepoint_android_rvh_replace_next_task_fair + __tracepoint_android_rvh_sched_fork_init + __tracepoint_android_rvh_sched_newidle_balance + __tracepoint_android_rvh_sched_nohz_balancer_kick + __tracepoint_android_rvh_sched_rebalance_domains + __tracepoint_android_rvh_schedule + __tracepoint_android_rvh_select_fallback_rq + __tracepoint_android_rvh_select_task_rq_fair + __tracepoint_android_rvh_select_task_rq_rt + __tracepoint_android_rvh_set_task_cpu + __tracepoint_android_rvh_uclamp_rq_util_with + __tracepoint_android_rvh_update_misfit_status + __tracepoint_android_rvh_wake_up_new_task + __tracepoint_android_vh_arch_set_freq_scale + __tracepoint_android_vh_binder_restore_priority + __tracepoint_android_vh_binder_set_priority + __tracepoint_android_vh_binder_wakeup_ilocked + __tracepoint_android_vh_scheduler_tick + __tracepoint_binder_transaction_received + __tracepoint_pelt_dl_tp + __tracepoint_pelt_irq_tp + __tracepoint_pelt_rt_tp + __tracepoint_pelt_se_tp + __tracepoint_sched_overutilized_tp + uclamp_eff_value + update_rq_clock + wake_up_if_idle + +# required by exynos-acme.ko + cpufreq_freq_transition_begin + cpufreq_freq_transition_end + cpufreq_generic_attr + cpufreq_register_driver + cpuhp_tasks_frozen + dev_pm_opp_of_register_em + refresh_frequency_limits + topology_set_thermal_pressure + +# required by exynos-bcm_dbg.ko + caches_clean_inval_pou + hrtimer_try_to_cancel + kick_all_cpus_sync + __platform_driver_probe + +# required by exynos-bts.ko + devm_kstrdup + +# required by exynos-chipid_v2.ko + soc_device_register + subsys_system_register + +# required by exynos-coresight.ko + __cpuhp_setup_state_cpuslocked + cpu_pm_register_notifier + +# required by exynos-cpuhp.ko + add_cpu + +# required by exynos-cpupm.ko + cpuidle_get_cpu_driver + of_find_node_by_type + smp_call_function_many + __traceiter_ipi_entry + __traceiter_ipi_exit + __traceiter_ipi_raise + __tracepoint_ipi_entry + __tracepoint_ipi_exit + __tracepoint_ipi_raise + +# required by exynos-debug-test.ko + _dev_crit + +# required by exynos-ecc-handler.ko + cpu_hotplug_disable + cpu_hotplug_enable + +# required by exynos-itmon.ko + int_pow + +# required by exynos-pd.ko + of_genpd_add_provider_simple + pm_genpd_add_subdomain + pm_genpd_init + +# required by exynos-reboot.ko + pm_power_off + +# required by exynos-ssld.ko + bus_register + bus_unregister + device_register + device_unregister + driver_register + tracepoint_probe_unregister + +# required by exynos3830_sound.ko + dapm_regulator_event + devm_snd_soc_register_card + snd_soc_component_set_pll + snd_soc_component_set_sysclk + snd_soc_dai_set_sysclk + snd_soc_dai_set_tdm_slot + snd_soc_dapm_get_pin_switch + snd_soc_dapm_new_control + snd_soc_dapm_put_pin_switch + snd_soc_get_pcm_runtime + snd_soc_of_get_dai_link_codecs + snd_soc_of_get_dai_name + snd_soc_of_parse_audio_routing + snd_soc_of_parse_daifmt + snd_soc_of_put_dai_link_codecs + snd_soc_params_to_bclk + snd_soc_pm_ops + +# required by exynos_adc.ko + devm_iio_device_alloc + +# required by exynos_devfreq.ko + devfreq_add_device + devfreq_add_governor + devfreq_recommended_opp + devfreq_register_opp_notifier + devfreq_remove_device + devfreq_unregister_opp_notifier + dev_pm_opp_disable + dev_pm_opp_get_freq + strpbrk + unregister_reboot_notifier + update_devfreq + +# required by exynos_mct.ko + clockevents_config_and_register + disable_percpu_irq + enable_percpu_irq + free_percpu_irq + of_clk_get_by_name + of_irq_parse_one + __request_percpu_irq + +# required by exynos_mfc.ko + dma_buf_begin_cpu_access_partial + dma_buf_end_cpu_access_partial + +# required by exynos_pm_qos.ko + _raw_read_lock_irqsave + _raw_read_unlock_irqrestore + +# required by exynos_thermal.ko + dev_pm_opp_find_freq_exact + dev_pm_opp_get_opp_count + get_cpu_idle_time + ida_alloc_range + ida_free + kthread_cancel_delayed_work_sync + kthread_flush_work + kthread_mod_delayed_work + of_thermal_get_ntrips + of_thermal_is_trip_valid + thermal_zone_device_disable + thermal_zone_device_enable + thermal_zone_device_is_enabled + thermal_zone_device_update + thermal_zone_of_sensor_register + thermal_zone_of_sensor_unregister + +# required by exynos_tty.ko + console_stop + dma_get_slave_caps + do_SAK + handle_sysrq + sysrq_mask + tty_flip_buffer_push + tty_insert_flip_string_fixed_flag + tty_kref_put + tty_port_tty_get + uart_add_one_port + uart_console_write + uart_get_baud_rate + uart_parse_options + uart_register_driver + uart_remove_one_port + uart_resume_port + uart_set_options + uart_suspend_port + uart_try_toggle_sysrq + uart_unregister_driver + uart_update_timeout + uart_write_wakeup + +# required by exyswd-rng.ko + devm_hwrng_register + platform_device_register + +# required by fb.ko + __arch_copy_in_user + console_lock + fb_mode_option + ignore_console_lock_warning + oops_in_progress + proc_create_seq_private + __request_module + simple_strtol + simple_strtoul + vm_get_page_prot + vm_iomap_memory + +# required by fimc-is.ko + cpuidle_pause_and_lock + cpuidle_resume_and_unlock + device_get_dma_attr + devm_phy_put + get_random_u32 + __get_vm_area_caller + llist_add_batch + map_kernel_range + names_cachep + param_set_uint + phy_configure + regulator_force_disable + regulator_get_voltage + regulator_set_mode + spi_bus_type + __spi_register_driver + spi_setup + spi_sync + vb2_prepare_buf + vsprintf + +# required by fm_s610.ko + of_find_i2c_device_by_node + of_property_read_variable_u16_array + v4l2_ctrl_poll + v4l2_ctrl_subscribe_event + v4l2_event_unsubscribe + v4l2_fh_is_singular + v4l2_fh_open + v4l2_fh_release + video_device_release_empty + +# required by freq-qos-tracer.ko + freq_qos_add_request + freq_qos_remove_request + +# required by gdc.ko + v4l2_m2m_querybuf + +# required by gnssif.ko + skb_clone + +# required by gpu_cooling.ko + thermal_cooling_device_unregister + +# required by hardlockup-watchdog.ko + cpus_read_lock + cpus_read_unlock + smp_call_on_cpu + +# required by i2c-dev.ko + bus_register_notifier + bus_unregister_notifier + cdev_device_add + cdev_device_del + i2c_adapter_type + i2c_bus_type + i2c_for_each_dev + i2c_get_adapter + i2c_put_adapter + i2c_smbus_xfer + i2c_verify_client + +# required by i2c-s3c2410.ko + __ndelay + +# required by is-actuator-dw9839.ko + kfree_sensitive + +# required by leds-s2mu106.ko + devm_led_classdev_register_ext + +# required by lzo-rle.ko + lzorle1x_1_compress + +# required by lzo.ko + lzo1x_1_compress + +# required by mali_kbase.ko + anon_inode_getfd + __arch_clear_user + __bitmap_weight + cache_line_size + clear_page + debugfs_attr_read + debugfs_attr_write + dev_pm_opp_of_add_table + dev_pm_opp_of_remove_table + dev_pm_opp_put_regulators + dev_pm_opp_set_regulators + downgrade_write + down_trylock + find_get_pid + find_vma + __get_task_comm + get_user_pages + get_user_pages_fast + ktime_get_raw + __mmdrop + of_clk_get + of_dma_is_coherent + of_property_read_variable_u8_array + param_ops_byte + pid_task + pin_user_pages_remote + put_pid + rb_erase + rb_first + rb_next + rb_prev + rb_replace_node + register_oom_notifier + remap_vmalloc_range + seq_open + seq_putc + seq_write + set_page_dirty_lock + strcspn + synchronize_irq + __traceiter_gpu_mem_total + trace_output_call + __tracepoint_gpu_mem_total + trace_print_flags_seq + trace_print_symbols_seq + unmap_mapping_range + unregister_oom_notifier + vmalloc_user + vmf_insert_pfn_prot + +# required by mcDrvModule.ko + crypto_alloc_shash + crypto_shash_final + crypto_shash_update + d_path + freezing_slow_path + get_task_exe_file + get_task_mm + get_zeroed_page + kstrtol_from_user + mmput + __refrigerator + release_pages + system_freezing_cnt + vmalloc_to_page + wait_for_completion_killable + +# required by mx_client_test.ko + cdev_alloc + +# required by nanohub.ko + arch_timer_read_counter + autoremove_wake_function + clocks_calc_mult_shift + do_wait_intr + iio_device_alloc + iio_device_free + rtc_set_time + rtc_tm_to_time64 + sched_setscheduler + sysfs_remove_link + __wake_up_locked_key + __wake_up_sync + +# required by pinctrl-samsung-core.ko + bitmap_to_arr32 + devm_gpiochip_add_data_with_key + devm_pinctrl_register + generic_handle_irq + gpiochip_generic_free + gpiochip_generic_request + gpiochip_get_data + gpiochip_lock_as_irq + gpiochip_unlock_as_irq + handle_edge_irq + irq_create_mapping_affinity + __irq_domain_add + irq_domain_remove + irq_domain_xlate_twocell + irq_find_mapping + irq_set_chained_handler_and_data + of_node_name_eq + pinctrl_add_gpio_range + pinctrl_dev_get_drvdata + pinctrl_force_sleep + pinctrl_remove_gpio_range + pin_get_name + +# required by pl330.ko + amba_driver_register + amba_driver_unregister + cpu_all_bits + dev_err_probe + dma_async_device_register + dma_async_device_unregister + dma_async_tx_descriptor_init + dmaengine_unmap_put + dma_get_slave_channel + dma_map_resource + dma_unmap_resource + of_dma_controller_free + of_dma_controller_register + pm_runtime_irq_safe + +# required by pwm-samsung.ko + of_pwm_xlate_with_flags + pwmchip_add + pwmchip_remove + pwm_get_chip_data + pwm_set_chip_data + +# required by rtc-s2mpu12.ko + devm_rtc_device_register + rtc_update_irq + rtc_valid_tm + +# required by s2m_chg_manager.ko + alarm_cancel + alarm_init + alarm_start_relative + +# required by s2m_muic_module.ko + printk_timed_ratelimit + +# required by s2m_pdic_module.ko + typec_get_drvdata + typec_register_partner + typec_register_port + typec_set_data_role + typec_set_pwr_opmode + typec_set_pwr_role + typec_unregister_partner + typec_unregister_port + +# required by s2mpu12-regulator.ko + of_get_regulator_init_data + rdev_get_drvdata + rdev_get_id + regulator_list_voltage_linear + regulator_map_voltage_linear + regulator_register + regulator_unregister + +# required by s2mu106_mfd.ko + i2c_smbus_read_word_data + i2c_smbus_write_word_data + +# required by s3c2410_wdt.ko + system_state + watchdog_init_timeout + watchdog_register_device + watchdog_set_restart_priority + watchdog_unregister_device + +# required by samsung-iommu-group.ko + iommu_group_alloc + iommu_group_set_name + +# required by samsung-secure-iova.ko + gen_pool_avail + gen_pool_create + gen_pool_destroy + gen_pool_first_fit_align + gen_pool_size + +# required by samsung_dma_heap.ko + anon_inode_getfile + deferred_free + devm_gen_pool_create + dma_buf_get_flags + dmabuf_page_pool_alloc + dmabuf_page_pool_create + dmabuf_page_pool_destroy + dmabuf_page_pool_free + dma_heap_add + dma_heap_get_dev + dma_heap_get_drvdata + dma_heap_get_name + is_dma_buf_file + iterate_fd + __traceiter_android_vh_show_mem + __tracepoint_android_vh_show_mem + +# required by samsung_iommu.ko + bus_set_iommu + device_link_add + device_link_del + devres_add + devres_alloc_node + iommu_alloc_resv_region + iommu_device_register + iommu_device_sysfs_add + iommu_device_sysfs_remove + iommu_device_unlink + iommu_device_unregister + iommu_fwspec_add_ids + iommu_fwspec_free + iommu_get_dma_cookie + iommu_group_for_each_dev + iommu_group_get + iommu_group_get_iommudata + iommu_group_set_iommudata + iommu_put_dma_cookie + iommu_report_device_fault + of_find_node_with_property + of_get_dma_window + platform_bus_type + split_page + +# required by scaler.ko + dma_heap_buffer_free + flush_delayed_work + of_parse_phandle_with_fixed_args + sg_nents_for_len + v4l2_m2m_try_schedule + work_busy + +# required by scsc_bt.ko + param_ops_ullong + +# required by scsc_log_collection.ko + list_sort + +# required by scsc_logring.ko + dev_vprintk_emit + vprintk_emit + +# required by scsc_mx.ko + genlmsg_multicast_allns + genlmsg_put + genl_register_family + genl_unregister_family + kobject_get + kobject_uevent + +# required by scsc_platform_mif.ko + irq_get_irqchip_state + +# required by scsc_wlan.ko + __cfg80211_alloc_event_skb + __cfg80211_alloc_reply_skb + cfg80211_chandef_create + cfg80211_chandef_valid + cfg80211_ch_switch_notify + cfg80211_classify8021d + cfg80211_connect_done + cfg80211_conn_failed + cfg80211_del_sta_sinfo + cfg80211_disconnected + cfg80211_external_auth_request + cfg80211_find_elem_match + cfg80211_find_vendor_elem + cfg80211_get_bss + cfg80211_inform_bss_frame_data + cfg80211_mgmt_tx_status + cfg80211_michael_mic_failure + cfg80211_new_sta + cfg80211_put_bss + cfg80211_ready_on_channel + cfg80211_ref_bss + cfg80211_remain_on_channel_expired + cfg80211_roamed + cfg80211_rx_mgmt_khz + cfg80211_scan_done + cfg80211_sched_scan_results + cfg80211_sched_scan_stopped + __cfg80211_send_event_skb + cfg80211_vendor_cmd_reply + consume_skb + csum_partial + csum_tcpudp_nofold + dev_alloc_name + dev_queue_xmit + ether_setup + eth_type_trans + ieee80211_channel_to_freq_khz + ieee80211_freq_khz_to_channel + ieee80211_get_channel_khz + ip_send_check + __kfifo_out + kfree_skb_list + __local_bh_enable_ip + napi_disable + netif_carrier_off + netif_carrier_on + __netif_napi_del + netif_rx_ni + netif_tx_stop_all_queues + net_ratelimit + nla_memcpy + nla_put_nohdr + __printk_ratelimit + _raw_read_lock_bh + _raw_read_unlock_bh + _raw_write_lock_bh + _raw_write_unlock_bh + register_inet6addr_notifier + register_inetaddr_notifier + register_netdevice + rps_needed + rtnl_is_locked + rtnl_lock + rtnl_unlock + skb_copy + skb_copy_bits + skb_realloc_headroom + synchronize_net + unregister_inet6addr_notifier + unregister_inetaddr_notifier + unregister_netdevice_queue + wiphy_apply_custom_regulatory + wiphy_free + wiphy_new_nm + wiphy_register + wiphy_unregister + +# required by smfc.ko + v4l2_ctrl_log_status + v4l2_m2m_fop_mmap + v4l2_m2m_fop_poll + v4l2_m2m_ioctl_dqbuf + v4l2_m2m_ioctl_querybuf + v4l2_m2m_ioctl_reqbufs + v4l2_m2m_ioctl_streamoff + v4l2_m2m_ioctl_streamon + vb2_ops_wait_finish + vb2_ops_wait_prepare + vb2_wait_for_all_buffers + +# required by snd-soc-aud3004x-5p.ko + devm_input_allocate_device + __devm_regmap_init + iio_channel_get_all + iio_channel_release + iio_read_channel_raw + of_parse_phandle_with_args + regcache_cache_bypass + wakeup_source_create + wakeup_source_destroy + +# required by snd-soc-aud3004x-dump.ko + class_create_file_ns + strspn + +# required by snd-soc-s3c-dma.ko + devm_snd_dmaengine_pcm_register + snd_dmaengine_pcm_prepare_slave_config + +# required by snd-soc-samsung-abox.ko + dapm_pinctrl_event + debugfs_create_symlink + dev_get_regmap + device_show_bool + device_show_int + device_store_bool + device_store_int + devm_krealloc + __devm_regmap_init_mmio_clk + devm_snd_soc_register_component + dma_buf_fd + dma_mmap_attrs + firmware_request_nowarn + iommu_iova_to_phys + iommu_map + kfree_const + kstrtoull_from_user + kvasprintf + nsecs_to_jiffies + pinctrl_get + pinctrl_put + platform_device_register_full + __platform_register_drivers + pm_runtime_barrier + pm_runtime_get_if_active + pm_runtime_no_callbacks + pm_wakeup_dev_event + proc_remove + proc_set_size + proc_symlink + regcache_mark_dirty + regcache_sync + regmap_multi_reg_write + request_firmware_direct + request_firmware_nowait + snd_device_free + snd_dma_alloc_pages + snd_dma_free_pages + snd_hwdep_new + snd_pcm_format_physical_width + snd_pcm_lib_free_pages + snd_pcm_lib_malloc_pages + snd_pcm_lib_preallocate_free_for_all + snd_pcm_lib_preallocate_pages + snd_pcm_period_elapsed + snd_pcm_rate_range_to_bits + snd_soc_add_component_controls + snd_soc_bytes_info_ext + snd_soc_bytes_tlv_callback + snd_soc_component_async_complete + snd_soc_component_init_regmap + snd_soc_component_read + snd_soc_component_update_bits + snd_soc_component_update_bits_async + snd_soc_component_write + snd_soc_dapm_add_routes + snd_soc_dapm_disable_pin + snd_soc_dapm_enable_pin + snd_soc_dapm_get_pin_status + snd_soc_dapm_new_controls + snd_soc_dapm_weak_routes + snd_soc_find_dai + snd_soc_get_xr_sx + snd_soc_info_volsw_sx + snd_soc_info_xr_sx + snd_soc_lookup_component + snd_soc_new_compress + snd_soc_put_xr_sx + snd_soc_register_card + snd_soc_set_runtime_hwparams + snd_soc_tplg_component_load + snd_soc_tplg_component_remove + snd_soc_tplg_widget_bind_event + snd_soc_unregister_card + strim + system_unbound_wq + +# required by spi-s3c64xx.ko + __spi_alloc_controller + spi_controller_resume + spi_controller_suspend + spi_finalize_current_message + spi_register_controller + spi_unregister_controller + +# required by synaptics_rmi.ko + input_mt_init_slots + input_mt_report_slot_state + input_set_abs_params + strncat + sysfs_remove_bin_file + +# required by usb_f_dm.ko + usb_ep_alloc_request + usb_ep_free_request + usb_ep_queue + +# required by usb_notifier.ko + power_supply_put + +# required by usb_notify_layer.ko + complete_and_exit + usb_speed_string + +# required by xhci-exynos.ko + device_set_wakeup_enable + device_wakeup_enable + devm_clk_get_optional + devm_usb_get_phy_by_phandle + of_fwnode_ops + of_usb_host_tpl_support + pci_bus_type + usb_add_hcd + __usb_create_hcd + usb_disabled + usb_enable_autosuspend + usb_hcd_is_primary_hcd + usb_put_hcd + usb_remove_hcd + xhci_bus_resume + xhci_bus_suspend + xhci_gen_setup + xhci_init_driver + xhci_resume + xhci_run + xhci_suspend + xhci_vendor_get_ops + +# required by xperf.ko + cpufreq_quick_get_max + kthread_bind_mask + +# required by zram.ko + __alloc_disk_node + bdget_disk + bdput + bio_endio + blk_alloc_queue + blk_cleanup_queue + blk_queue_flag_clear + blk_queue_flag_set + blk_queue_io_min + blk_queue_io_opt + blk_queue_logical_block_size + blk_queue_max_discard_sectors + blk_queue_max_write_zeroes_sectors + blk_queue_physical_block_size + __class_register + __cpuhp_state_add_instance + __cpuhp_state_remove_instance + crypto_alloc_base + crypto_comp_compress + crypto_comp_decompress + crypto_has_alg + del_gendisk + device_add_disk + disk_end_io_acct + disk_start_io_acct + fsync_bdev + idr_destroy + idr_find + idr_for_each + kstrtou16 + memparse + memset64 + page_endio + put_disk + register_blkdev + revalidate_disk_size + __sysfs_match_string + unregister_blkdev + +# required by zsmalloc.ko + alloc_anon_inode + __ClearPageMovable + contig_page_data + dec_zone_page_state + inc_zone_page_state + init_pseudo + iput + kern_mount + kern_unmount + kill_anon_super + __lock_page + page_mapping + _raw_write_lock + _raw_write_unlock + __SetPageMovable + unlock_page diff --git a/android/abi_gki_aarch64_fips140 b/android/abi_gki_aarch64_fips140 index 361966d13262..541e351809d0 100644 --- a/android/abi_gki_aarch64_fips140 +++ b/android/abi_gki_aarch64_fips140 @@ -111,6 +111,7 @@ _raw_spin_lock _raw_spin_unlock refcount_warn_saturate + rng_is_initialized scatterwalk_ffwd scatterwalk_map_and_copy sg_init_one diff --git a/android/abi_gki_aarch64_generic b/android/abi_gki_aarch64_generic index a7069c587ab7..2d377591d6fb 100644 --- a/android/abi_gki_aarch64_generic +++ b/android/abi_gki_aarch64_generic @@ -189,6 +189,7 @@ cfg80211_vendor_cmd_reply __cfi_slowpath __check_object_size + check_preempt_curr __class_create class_destroy class_interface_unregister @@ -993,8 +994,12 @@ hid_allocate_device hid_debug hid_destroy_device + hid_hw_start hid_input_report + hid_open_report hid_parse_report + __hid_register_driver + hid_unregister_driver hrtimer_active hrtimer_cancel hrtimer_forward @@ -1800,9 +1805,11 @@ __request_percpu_irq __request_region request_threaded_irq + resched_curr resume_cpus return_address revalidate_disk_size + reweight_task rfkill_alloc rfkill_destroy rfkill_init_sw_state @@ -2219,12 +2226,15 @@ __traceiter_android_rvh_post_init_entity_util_avg __traceiter_android_rvh_preempt_disable __traceiter_android_rvh_preempt_enable + __traceiter_android_rvh_prepare_prio_fork __traceiter_android_rvh_remove_entity_load_avg __traceiter_android_rvh_sched_fork __traceiter_android_rvh_select_task_rq_fair __traceiter_android_rvh_select_task_rq_rt __traceiter_android_rvh_set_iowait + __traceiter_android_rvh_setscheduler __traceiter_android_rvh_set_task_cpu + __traceiter_android_rvh_set_user_nice __traceiter_android_rvh_typec_tcpci_chk_contaminant __traceiter_android_rvh_typec_tcpci_get_vbus __traceiter_android_rvh_uclamp_eff_get @@ -2237,11 +2247,15 @@ __traceiter_android_rvh_update_rt_rq_load_avg __traceiter_android_rvh_util_est_update __traceiter_android_vh_arch_set_freq_scale + __traceiter_android_vh_bh_lru_install + __traceiter_android_vh_binder_restore_priority + __traceiter_android_vh_binder_set_priority __traceiter_android_vh_cma_alloc_adjust __traceiter_android_vh_cma_alloc_finish __traceiter_android_vh_cma_alloc_start __traceiter_android_vh_cpu_idle_enter __traceiter_android_vh_cpu_idle_exit + __traceiter_android_vh_do_madvise_blk_plug __traceiter_android_vh_dump_throttled_rt_tasks __traceiter_android_vh_dup_task_struct __traceiter_android_vh_early_resume_begin @@ -2256,13 +2270,19 @@ __traceiter_android_vh_mm_compaction_end __traceiter_android_vh_of_i2c_get_board_info __traceiter_android_vh_pagecache_get_page + __traceiter_android_vh_pagevec_drain __traceiter_android_vh_pin_user_pages + __traceiter_android_vh_rebuild_root_domains_bypass + __traceiter_android_vh_reclaim_pages_plug __traceiter_android_vh_resume_end __traceiter_android_vh_rmqueue __traceiter_android_vh_sched_setaffinity_early __traceiter_android_vh_scheduler_tick __traceiter_android_vh_setscheduler_uclamp __traceiter_android_vh_show_max_freq + __traceiter_android_vh_shrink_inactive_list_blk_plug + __traceiter_android_vh_shrink_lruvec_blk_plug + __traceiter_android_vh_skip_lru_disable __traceiter_android_vh_snd_compr_use_pause_in_drain __traceiter_android_vh_sound_usb_support_cpu_suspend __traceiter_android_vh_sysrq_crash @@ -2286,6 +2306,9 @@ __traceiter_android_vh_ufs_update_sysfs __traceiter_android_vh_usb_dev_resume __traceiter_android_vh_usb_dev_suspend + __traceiter_android_vh_zap_pte_range_tlb_end + __traceiter_android_vh_zap_pte_range_tlb_force_flush + __traceiter_android_vh_zap_pte_range_tlb_start __traceiter_clock_set_rate __traceiter_cpu_frequency __traceiter_device_pm_callback_end @@ -2293,6 +2316,8 @@ __traceiter_dwc3_readl __traceiter_dwc3_writel __traceiter_gpu_mem_total + __traceiter_irq_handler_entry + __traceiter_irq_handler_exit __traceiter_mm_vmscan_direct_reclaim_begin __traceiter_mm_vmscan_direct_reclaim_end __traceiter_pelt_cfs_tp @@ -2308,6 +2333,8 @@ __traceiter_sched_switch __traceiter_sched_util_est_cfs_tp __traceiter_sched_util_est_se_tp + __traceiter_softirq_entry + __traceiter_softirq_exit __traceiter_suspend_resume trace_output_call __tracepoint_android_rvh_arm64_serror_panic @@ -2335,12 +2362,15 @@ __tracepoint_android_rvh_post_init_entity_util_avg __tracepoint_android_rvh_preempt_disable __tracepoint_android_rvh_preempt_enable + __tracepoint_android_rvh_prepare_prio_fork __tracepoint_android_rvh_remove_entity_load_avg __tracepoint_android_rvh_sched_fork __tracepoint_android_rvh_select_task_rq_fair __tracepoint_android_rvh_select_task_rq_rt __tracepoint_android_rvh_set_iowait + __tracepoint_android_rvh_setscheduler __tracepoint_android_rvh_set_task_cpu + __tracepoint_android_rvh_set_user_nice __tracepoint_android_rvh_typec_tcpci_chk_contaminant __tracepoint_android_rvh_typec_tcpci_get_vbus __tracepoint_android_rvh_uclamp_eff_get @@ -2353,11 +2383,15 @@ __tracepoint_android_rvh_update_rt_rq_load_avg __tracepoint_android_rvh_util_est_update __tracepoint_android_vh_arch_set_freq_scale + __tracepoint_android_vh_bh_lru_install + __tracepoint_android_vh_binder_restore_priority + __tracepoint_android_vh_binder_set_priority __tracepoint_android_vh_cma_alloc_adjust __tracepoint_android_vh_cma_alloc_finish __tracepoint_android_vh_cma_alloc_start __tracepoint_android_vh_cpu_idle_enter __tracepoint_android_vh_cpu_idle_exit + __tracepoint_android_vh_do_madvise_blk_plug __tracepoint_android_vh_dump_throttled_rt_tasks __tracepoint_android_vh_dup_task_struct __tracepoint_android_vh_early_resume_begin @@ -2372,13 +2406,19 @@ __tracepoint_android_vh_mm_compaction_end __tracepoint_android_vh_of_i2c_get_board_info __tracepoint_android_vh_pagecache_get_page + __tracepoint_android_vh_pagevec_drain __tracepoint_android_vh_pin_user_pages + __tracepoint_android_vh_rebuild_root_domains_bypass + __tracepoint_android_vh_reclaim_pages_plug __tracepoint_android_vh_resume_end __tracepoint_android_vh_rmqueue __tracepoint_android_vh_sched_setaffinity_early __tracepoint_android_vh_scheduler_tick __tracepoint_android_vh_setscheduler_uclamp __tracepoint_android_vh_show_max_freq + __tracepoint_android_vh_shrink_inactive_list_blk_plug + __tracepoint_android_vh_shrink_lruvec_blk_plug + __tracepoint_android_vh_skip_lru_disable __tracepoint_android_vh_snd_compr_use_pause_in_drain __tracepoint_android_vh_sound_usb_support_cpu_suspend __tracepoint_android_vh_sysrq_crash @@ -2402,6 +2442,9 @@ __tracepoint_android_vh_ufs_update_sysfs __tracepoint_android_vh_usb_dev_resume __tracepoint_android_vh_usb_dev_suspend + __tracepoint_android_vh_zap_pte_range_tlb_end + __tracepoint_android_vh_zap_pte_range_tlb_force_flush + __tracepoint_android_vh_zap_pte_range_tlb_start __tracepoint_clock_set_rate __tracepoint_cpu_frequency __tracepoint_device_pm_callback_end @@ -2409,6 +2452,8 @@ __tracepoint_dwc3_readl __tracepoint_dwc3_writel __tracepoint_gpu_mem_total + __tracepoint_irq_handler_entry + __tracepoint_irq_handler_exit __tracepoint_mm_vmscan_direct_reclaim_begin __tracepoint_mm_vmscan_direct_reclaim_end __tracepoint_pelt_cfs_tp @@ -2426,6 +2471,8 @@ __tracepoint_sched_switch __tracepoint_sched_util_est_cfs_tp __tracepoint_sched_util_est_se_tp + __tracepoint_softirq_entry + __tracepoint_softirq_exit __tracepoint_suspend_resume trace_print_array_seq trace_print_bitmask_seq diff --git a/android/abi_gki_aarch64_rockchip b/android/abi_gki_aarch64_rockchip new file mode 100644 index 000000000000..8074877bc467 --- /dev/null +++ b/android/abi_gki_aarch64_rockchip @@ -0,0 +1,536 @@ +[abi_symbol_list] +# commonly used symbols + alloc_workqueue + arm64_const_caps_ready + atomic_notifier_chain_register + atomic_notifier_chain_unregister + cancel_delayed_work_sync + __cfi_slowpath + clk_bulk_disable + clk_bulk_enable + clk_bulk_prepare + clk_bulk_unprepare + clk_disable + clk_enable + clk_gate_ops + clk_get + __clk_get_name + clk_get_rate + clk_notifier_register + clk_prepare + clk_put + clk_register + clk_set_rate + clk_unprepare + __const_udelay + __cpufreq_driver_target + cpufreq_generic_suspend + cpufreq_register_governor + cpufreq_register_notifier + cpufreq_unregister_governor + cpu_hwcap_keys + cpu_latency_qos_add_request + cpu_latency_qos_update_request + cpumask_next + cpu_number + __cpu_online_mask + __cpu_possible_mask + cpus_read_lock + cpus_read_unlock + delayed_work_timer_fn + _dev_err + devfreq_recommended_opp + dev_fwnode + device_create_file + device_init_wakeup + device_property_present + device_property_read_u32_array + device_property_read_u8_array + _dev_info + devm_clk_bulk_get_all + devm_clk_get + devm_clk_hw_register + devm_clk_register + devm_gpiod_get_optional + devm_ioremap_resource + devm_kfree + devm_kmalloc + devm_nvmem_register + devm_platform_ioremap_resource + devm_power_supply_register + devm_regmap_field_alloc + __devm_regmap_init_i2c + devm_regulator_get + devm_regulator_get_optional + devm_regulator_register + devm_request_threaded_irq + devm_reset_control_array_get + dev_pm_opp_find_freq_ceil + dev_pm_opp_get_opp_count + dev_pm_opp_get_opp_table + dev_pm_opp_get_voltage + dev_pm_opp_of_get_sharing_cpus + dev_pm_opp_put + dev_pm_opp_put_opp_table + dev_pm_opp_put_regulators + dev_pm_opp_register_set_opp_helper + dev_pm_opp_set_rate + dev_pm_opp_set_regulators + devres_add + devres_alloc_node + devres_free + _dev_warn + down_read + down_write + freq_qos_add_request + freq_qos_remove_request + freq_qos_update_request + get_cpu_device + gpiod_direction_output + gpiod_get_value + i2c_del_driver + i2c_register_driver + i2c_smbus_read_i2c_block_data + i2c_smbus_write_i2c_block_data + init_timer_key + iounmap + irq_set_irq_wake + irq_to_desc + kfree + __kmalloc + kmalloc_caches + kmem_cache_alloc_trace + ktime_get + __list_add_valid + __list_del_entry_valid + __log_post_read_mmio + __log_read_mmio + __log_write_mmio + memcpy + memset + module_layout + __msecs_to_jiffies + msleep + __mutex_init + mutex_lock + mutex_unlock + nr_cpu_ids + nvmem_cell_put + nvmem_cell_read + of_clk_add_provider + of_clk_del_provider + of_clk_get + of_clk_get_by_name + of_clk_get_parent_count + of_clk_src_simple_get + of_count_phandle_with_args + of_device_get_match_data + of_device_is_available + of_device_is_compatible + of_find_matching_node_and_match + of_find_node_by_name + of_find_property + of_get_child_by_name + of_get_next_available_child + of_get_next_child + of_iomap + of_machine_is_compatible + of_match_device + of_match_node + of_nvmem_cell_get + of_parse_phandle + of_property_count_elems_of_size + of_property_read_string + of_property_read_string_helper + of_property_read_u32_index + of_property_read_variable_u32_array + panic_notifier_list + param_ops_int + __per_cpu_offset + __platform_driver_probe + __platform_driver_register + platform_driver_unregister + platform_get_irq + platform_get_resource + pm_clk_create + pm_clk_destroy + power_supply_am_i_supplied + power_supply_changed + power_supply_get_battery_info + power_supply_get_by_phandle + power_supply_get_drvdata + power_supply_get_property + power_supply_put_battery_info + power_supply_register + power_supply_reg_notifier + power_supply_unregister + print_hex_dump + printk + __put_task_struct + queue_delayed_work_on + queue_work_on + rdev_get_drvdata + register_pm_notifier + regmap_bulk_read + regmap_field_read + regmap_field_update_bits_base + regmap_raw_read + regmap_raw_write + regmap_read + regmap_update_bits_base + regmap_write + regulator_disable + regulator_enable + regulator_get_voltage + regulator_put + regulator_set_voltage + reset_control_assert + reset_control_deassert + snprintf + sprintf + sscanf + __stack_chk_fail + strcmp + syscon_node_to_regmap + syscon_regmap_lookup_by_phandle + sysfs_create_file_ns + system_freezable_wq + system_wq + thermal_zone_get_zone_by_name + __traceiter_rwmmio_post_read + __traceiter_rwmmio_read + __traceiter_rwmmio_write + __tracepoint_rwmmio_post_read + __tracepoint_rwmmio_read + __tracepoint_rwmmio_write + __udelay + update_devfreq + up_read + up_write + usleep_range + +# required by bq25700_charger.ko + extcon_get_edev_by_phandle + extcon_get_state + extcon_register_notifier + irq_set_irq_type + of_find_compatible_node + power_supply_get_by_name + +# required by bq25890_charger.ko + devm_gpiod_get + devm_usb_get_phy + gpiod_to_irq + system_power_efficient_wq + +# required by clk-link.ko + pm_clk_add + pm_clk_resume + pm_clk_suspend + __pm_runtime_disable + pm_runtime_enable + +# required by clk-pwm.ko + devm_pwm_get + of_clk_add_hw_provider + of_clk_hw_simple_get + pwm_apply_state + +# required by clk-rockchip.ko + clk_divider_ops + clk_divider_ro_ops + clk_fixed_factor_ops + clk_fractional_divider_ops + __clk_get_hw + clk_get_parent + clk_hw_get_flags + clk_hw_get_name + clk_hw_get_parent + clk_hw_get_rate + clk_hw_register_composite + clk_hw_round_rate + __clk_mux_determine_rate + clk_mux_ops + clk_mux_ro_ops + clk_notifier_unregister + clk_register_divider_table + clk_register_fixed_factor + clk_register_gate + clk_register_mux_table + clk_unregister + gcd + kmemdup + match_string + of_clk_src_onecell_get + rational_best_approximation + _raw_spin_lock_irqsave + _raw_spin_unlock_irqrestore + register_restart_handler + reset_controller_register + __warn_printk + +# required by clk-scmi.ko + clk_hw_set_rate_range + devm_of_clk_add_hw_provider + of_clk_hw_onecell_get + scmi_driver_register + scmi_driver_unregister + +# required by cpufreq-dt.ko + cpufreq_enable_boost_support + cpufreq_freq_attr_scaling_available_freqs + cpufreq_freq_attr_scaling_boost_freqs + cpufreq_generic_frequency_table_verify + cpufreq_generic_get + cpufreq_register_driver + cpufreq_unregister_driver + dev_pm_opp_free_cpufreq_table + dev_pm_opp_get_max_transition_latency + dev_pm_opp_get_sharing_cpus + dev_pm_opp_get_suspend_opp_freq + dev_pm_opp_init_cpufreq_table + dev_pm_opp_of_cpumask_add_table + dev_pm_opp_of_cpumask_remove_table + dev_pm_opp_of_register_em + dev_pm_opp_set_sharing_cpus + policy_has_boost_freq + +# required by cpufreq_ondemand.ko + cpufreq_cpu_get_raw + cpufreq_dbs_governor_exit + cpufreq_dbs_governor_init + cpufreq_dbs_governor_limits + cpufreq_dbs_governor_start + cpufreq_dbs_governor_stop + cpufreq_table_index_unsorted + dbs_update + get_cpu_idle_time_us + gov_update_cpu_data + preempt_schedule + store_sampling_rate + +# required by cw2015_battery.ko + dev_err_probe + +# required by cw2017_battery.ko + bcmp + +# required by cw221x_battery.ko + cancel_delayed_work + destroy_workqueue + +# required by industrialio-triggered-buffer.ko + iio_alloc_pollfunc + iio_dealloc_pollfunc + iio_device_attach_buffer + +# required by io-domain.ko + _dev_crit + regulator_register_notifier + regulator_unregister_notifier + +# required by kfifo_buf.ko + iio_buffer_init + iio_buffer_put + __kfifo_alloc + __kfifo_free + __kfifo_in + __kfifo_to_user + mutex_lock_interruptible + +# required by nvmem-rockchip-otp.ko + devm_clk_bulk_get + param_ops_uint + +# required by pm_domains.ko + clk_bulk_put + of_genpd_add_provider_onecell + panic + param_ops_bool + pm_clk_add_clk + pm_genpd_add_subdomain + pm_genpd_init + pm_genpd_remove + strrchr + +# required by rockchip-cpufreq.ko + cpufreq_unregister_notifier + dev_pm_opp_put_prop_name + dev_pm_opp_set_supported_hw + of_property_match_string + platform_device_register_full + +# required by rockchip-cpuinfo.ko + nvmem_cell_get + +# required by rockchip-dfi.ko + devm_devfreq_event_add_edev + gic_nonsecure_priorities + +# required by rockchip.ko + __genphy_config_aneg + genphy_resume + genphy_soft_reset + genphy_suspend + mdiobus_read + mdiobus_write + phy_drivers_register + phy_drivers_unregister + +# required by rockchip_bus.ko + cpu_topology + +# required by rockchip_debug.ko + nr_irqs + +# required by rockchip_dmc.ko + cpufreq_cpu_get + cpufreq_cpu_put + cpufreq_quick_get + devfreq_add_governor + devfreq_event_disable_edev + devfreq_event_enable_edev + devfreq_event_get_edev_by_phandle + devfreq_event_get_edev_count + devfreq_event_get_event + devfreq_monitor_resume + devfreq_monitor_start + devfreq_monitor_stop + devfreq_monitor_suspend + devfreq_resume_device + devfreq_suspend_device + devfreq_update_interval + devm_devfreq_add_device + devm_devfreq_register_opp_notifier + _dev_notice + dev_pm_opp_unregister_set_opp_helper + disable_irq + enable_irq + finish_wait + init_wait_entry + __init_waitqueue_head + input_close_device + input_open_device + input_register_handle + input_register_handler + input_unregister_handle + kstrtouint + __memset_io + of_devfreq_cooling_register_power + platform_get_irq_byname + prepare_to_wait_event + ___ratelimit + schedule_timeout + __wake_up + +# required by rockchip_dmc_common.ko + down_write_trylock + +# required by rockchip_opp_select.ko + dev_pm_opp_disable + dev_pm_opp_of_add_table + dev_pm_opp_set_prop_name + of_find_node_opts_by_path + regulator_get_linear_step + regulator_get_optional + +# required by rockchip_pm_config.ko + of_get_named_gpio_flags + +# required by rockchip_pvtm.ko + debugfs_create_dir + debugfs_create_file + debugfs_remove + seq_lseek + seq_printf + seq_puts + seq_read + single_open + single_release + +# required by rockchip_pwm_remotectl.ko + devm_input_allocate_device + input_event + input_register_device + input_set_capability + irq_set_affinity_hint + jiffies + jiffies_to_msecs + mod_timer + of_get_property + __pm_relax + pm_wakeup_ws_event + __tasklet_hi_schedule + tasklet_init + wakeup_source_add + wakeup_source_remove + +# required by rockchip_saradc.ko + complete + devm_add_action + devm_iio_device_alloc + __devm_iio_device_register + __devm_reset_control_get + find_next_bit + iio_get_time_ns + iio_push_to_buffers + iio_trigger_notify_done + __init_swait_queue_head + wait_for_completion_timeout + +# required by rockchip_sip.ko + arm64_use_ng_mappings + __arm_smccc_smc + __ioremap + memstart_addr + pfn_valid + sched_clock + vmap + +# required by rockchip_system_monitor.ko + add_cpu + bitmap_parselist + blocking_notifier_call_chain + blocking_notifier_chain_register + blocking_notifier_chain_unregister + dev_pm_opp_find_freq_floor + dev_pm_qos_add_request + dev_pm_qos_remove_request + dev_pm_qos_update_request + kobject_create_and_add + kstrdup + kstrtoull + mod_delayed_work_on + register_reboot_notifier + regulator_get + remove_cpu + strchr + strsep + strstr + thermal_zone_get_temp + +# required by rockchip_thermal.ko + devm_pinctrl_get + devm_thermal_zone_of_sensor_register + pinctrl_lookup_state + pinctrl_select_state + thermal_zone_device_disable + thermal_zone_device_enable + thermal_zone_device_update + +# required by rtc-hym8563.ko + _bcd2bin + _bin2bcd + devm_rtc_device_register + i2c_smbus_read_byte_data + i2c_smbus_write_byte_data + rtc_valid_tm + +# required by sgm41542_charger.ko + regulator_unregister + strncpy + +# required by timer-rockchip.ko + clockevents_config_and_register + irq_of_parse_and_map + request_threaded_irq diff --git a/arch/Kconfig b/arch/Kconfig index d39f92e9cf51..303098484b36 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -1182,7 +1182,6 @@ config ARCH_SPLIT_ARG64 config ARCH_HAS_NONLEAF_PMD_YOUNG bool - depends on PGTABLE_LEVELS > 2 help Architectures that select this option are capable of setting the accessed bit in non-leaf PMD entries when using them as part of linear diff --git a/arch/arm64/Kconfig.platforms b/arch/arm64/Kconfig.platforms index e6a2f21ba3f8..6e6198fbf17b 100644 --- a/arch/arm64/Kconfig.platforms +++ b/arch/arm64/Kconfig.platforms @@ -152,6 +152,7 @@ config ARCH_MEDIATEK config ARCH_MESON bool "Amlogic Platforms" + select COMMON_CLK help This enables support for the arm64 based Amlogic SoCs such as the s905, S905X/D, S912, A113X/D or S905X/D2 diff --git a/arch/arm64/configs/gki_defconfig b/arch/arm64/configs/gki_defconfig index 97039ed7a44d..ce7f112af579 100644 --- a/arch/arm64/configs/gki_defconfig +++ b/arch/arm64/configs/gki_defconfig @@ -513,6 +513,7 @@ CONFIG_USB_CONFIGFS_F_AUDIO_SRC=y CONFIG_USB_CONFIGFS_F_UAC2=y CONFIG_USB_CONFIGFS_F_MIDI=y CONFIG_USB_CONFIGFS_F_HID=y +CONFIG_USB_CONFIGFS_F_UVC=y CONFIG_TYPEC=y CONFIG_TYPEC_TCPM=y CONFIG_TYPEC_TCPCI=y diff --git a/arch/arm64/configs/rockchip_gki.fragment b/arch/arm64/configs/rockchip_gki.fragment new file mode 100644 index 000000000000..6253108101a8 --- /dev/null +++ b/arch/arm64/configs/rockchip_gki.fragment @@ -0,0 +1,342 @@ +CONFIG_AP6XXX=m +CONFIG_ARCH_ROCKCHIP=y +CONFIG_ARM_ROCKCHIP_BUS_DEVFREQ=m +CONFIG_ARM_ROCKCHIP_CPUFREQ=m +CONFIG_ARM_ROCKCHIP_DMC_DEVFREQ=m +CONFIG_BACKLIGHT_PWM=m +CONFIG_BATTERY_CW2015=m +CONFIG_BATTERY_CW2017=m +CONFIG_BATTERY_CW221X=m +CONFIG_BATTERY_RK817=m +CONFIG_BATTERY_RK818=m +CONFIG_BLK_DEV_NVME=m +CONFIG_BMA2XX_ACC=m +CONFIG_CHARGER_BQ25700=m +CONFIG_CHARGER_BQ25890=m +CONFIG_CHARGER_RK817=m +CONFIG_CHARGER_RK818=m +CONFIG_CHARGER_SC89890=m +CONFIG_CHARGER_SGM41542=m +CONFIG_CHR_DEV_SCH=m +CONFIG_CHR_DEV_SG=m +CONFIG_COMMON_CLK_PWM=m +CONFIG_COMMON_CLK_RK808=m +CONFIG_COMMON_CLK_ROCKCHIP=m +CONFIG_COMMON_CLK_SCMI=m +CONFIG_COMPASS_AK8963=m +CONFIG_COMPASS_AK8975=m +CONFIG_COMPASS_DEVICE=m +CONFIG_CPUFREQ_DT=m +CONFIG_CPU_FREQ_GOV_ONDEMAND=m +CONFIG_CPU_FREQ_GOV_USERSPACE=m +CONFIG_CPU_RK3588=y +CONFIG_CRYPTO_AES_ARM64_CE_CCM=m +CONFIG_CRYPTO_DEV_ROCKCHIP=m +CONFIG_CRYPTO_DEV_ROCKCHIP_DEV=m +CONFIG_CRYPTO_GHASH_ARM64_CE=m +CONFIG_CRYPTO_SHA1_ARM64_CE=m +CONFIG_CRYPTO_TWOFISH=m +CONFIG_DEVFREQ_EVENT_ROCKCHIP_NOCP=m +CONFIG_DMABUF_HEAPS_CMA=m +CONFIG_DMABUF_HEAPS_SYSTEM=m +CONFIG_DRAGONRISE_FF=y +CONFIG_DRM_DISPLAY_CONNECTOR=m +CONFIG_DRM_DW_HDMI_CEC=m +CONFIG_DRM_DW_HDMI_I2S_AUDIO=m +CONFIG_DRM_MAXIM_MAX96745=m +CONFIG_DRM_MAXIM_MAX96752F=m +CONFIG_DRM_MAXIM_MAX96755F=m +CONFIG_DRM_PANEL_SIMPLE=m +CONFIG_DRM_RK1000_TVE=m +CONFIG_DRM_RK630_TVE=m +CONFIG_DRM_ROCKCHIP=m +CONFIG_DRM_ROCKCHIP_RK628=m +CONFIG_DRM_ROHM_BU18XL82=m +CONFIG_DRM_SII902X=m +CONFIG_DTC_SYMBOLS=y +CONFIG_DW_WATCHDOG=m +CONFIG_GPIO_ROCKCHIP=m +CONFIG_GREENASIA_FF=y +CONFIG_GSENSOR_DEVICE=m +CONFIG_GS_DA223=m +CONFIG_GS_KXTJ9=m +CONFIG_GS_LIS3DH=m +CONFIG_GS_LSM303D=m +CONFIG_GS_MC3230=m +CONFIG_GS_MMA7660=m +CONFIG_GS_MMA8452=m +CONFIG_GS_MXC6655XA=m +CONFIG_GS_SC7660=m +CONFIG_GS_SC7A20=m +CONFIG_GS_SC7A30=m +CONFIG_GYROSCOPE_DEVICE=m +CONFIG_GYRO_EWTSA=m +CONFIG_GYRO_L3G20D=m +CONFIG_GYRO_L3G4200D=m +CONFIG_GYRO_LSM330=m +CONFIG_GYRO_MPU6500=m +CONFIG_GYRO_MPU6880=m +CONFIG_HALL_DEVICE=m +CONFIG_HID_A4TECH=m +CONFIG_HID_ACRUX=m +CONFIG_HID_ACRUX_FF=y +CONFIG_HID_ALPS=m +CONFIG_HID_APPLEIR=m +CONFIG_HID_AUREAL=m +CONFIG_HID_BELKIN=m +CONFIG_HID_CHERRY=m +CONFIG_HID_CHICONY=m +CONFIG_HID_CYPRESS=m +CONFIG_HID_DRAGONRISE=m +CONFIG_HID_EMS_FF=m +CONFIG_HID_EZKEY=m +CONFIG_HID_GREENASIA=m +CONFIG_HID_GYRATION=m +CONFIG_HID_HOLTEK=m +CONFIG_HID_ICADE=m +CONFIG_HID_KENSINGTON=m +CONFIG_HID_KEYTOUCH=m +CONFIG_HID_KYE=m +CONFIG_HID_LCPOWER=m +CONFIG_HID_LENOVO=m +CONFIG_HID_MONTEREY=m +CONFIG_HID_NTRIG=m +CONFIG_HID_ORTEK=m +CONFIG_HID_PANTHERLORD=m +CONFIG_HID_PETALYNX=m +CONFIG_HID_PRIMAX=m +CONFIG_HID_SAITEK=m +CONFIG_HID_SAMSUNG=m +CONFIG_HID_SMARTJOYPLUS=m +CONFIG_HID_SPEEDLINK=m +CONFIG_HID_STEELSERIES=m +CONFIG_HID_SUNPLUS=m +CONFIG_HID_THINGM=m +CONFIG_HID_THRUSTMASTER=m +CONFIG_HID_TIVO=m +CONFIG_HID_TOPSEED=m +CONFIG_HID_TWINHAN=m +CONFIG_HID_WALTOP=m +CONFIG_HID_ZEROPLUS=m +CONFIG_HID_ZYDACRON=m +CONFIG_HS_MH248=m +CONFIG_HW_RANDOM_ROCKCHIP=m +CONFIG_I2C_CHARDEV=m +CONFIG_I2C_GPIO=m +CONFIG_I2C_HID=m +CONFIG_I2C_RK3X=m +CONFIG_IEP=m +CONFIG_IIO_BUFFER_CB=m +CONFIG_INPUT_RK805_PWRKEY=m +CONFIG_KEYBOARD_ADC=m +CONFIG_LEDS_GPIO=m +CONFIG_LEDS_RGB13H=m +CONFIG_LEDS_TRIGGER_BACKLIGHT=m +CONFIG_LEDS_TRIGGER_DEFAULT_ON=m +CONFIG_LEDS_TRIGGER_HEARTBEAT=m +CONFIG_LIGHT_DEVICE=m +CONFIG_LSM330_ACC=m +CONFIG_LS_CM3217=m +CONFIG_LS_CM3218=m +CONFIG_LS_STK3410=m +CONFIG_LS_UCS14620=m +CONFIG_MALI_BIFROST=m +CONFIG_MALI_BIFROST_DEBUG=y +CONFIG_MALI_BIFROST_EXPERT=y +CONFIG_MALI_CSF_SUPPORT=y +CONFIG_MALI_PLATFORM_NAME="rk" +CONFIG_MALI_PWRSOFT_765=y +CONFIG_MFD_RK628=m +CONFIG_MFD_RK630_I2C=m +CONFIG_MFD_RK806_SPI=m +CONFIG_MFD_RK808=m +CONFIG_MMC_DW=m +CONFIG_MMC_DW_ROCKCHIP=m +CONFIG_MMC_SDHCI_OF_ARASAN=m +CONFIG_MMC_SDHCI_OF_DWCMSHC=m +CONFIG_MPU6500_ACC=m +CONFIG_MPU6880_ACC=m +CONFIG_OPTEE=m +CONFIG_PANTHERLORD_FF=y +CONFIG_PCIEASPM_EXT=m +CONFIG_PCIE_DW_ROCKCHIP=m +CONFIG_PCIE_ROCKCHIP_HOST=m +CONFIG_PHY_ROCKCHIP_CSI2_DPHY=m +CONFIG_PHY_ROCKCHIP_DP=m +CONFIG_PHY_ROCKCHIP_EMMC=m +CONFIG_PHY_ROCKCHIP_INNO_DSIDPHY=m +CONFIG_PHY_ROCKCHIP_INNO_HDMI=m +CONFIG_PHY_ROCKCHIP_INNO_USB2=m +CONFIG_PHY_ROCKCHIP_INNO_USB3=m +CONFIG_PHY_ROCKCHIP_NANENG_COMBO_PHY=m +CONFIG_PHY_ROCKCHIP_NANENG_EDP=m +CONFIG_PHY_ROCKCHIP_PCIE=m +CONFIG_PHY_ROCKCHIP_SAMSUNG_DCPHY=m +CONFIG_PHY_ROCKCHIP_SAMSUNG_HDPTX=m +CONFIG_PHY_ROCKCHIP_SAMSUNG_HDPTX_HDMI=m +CONFIG_PHY_ROCKCHIP_SNPS_PCIE3=m +CONFIG_PHY_ROCKCHIP_TYPEC=m +CONFIG_PHY_ROCKCHIP_USB=m +CONFIG_PHY_ROCKCHIP_USBDP=m +CONFIG_PINCTRL_RK805=m +CONFIG_PINCTRL_RK806=m +CONFIG_PINCTRL_ROCKCHIP=m +CONFIG_PL330_DMA=m +CONFIG_PROXIMITY_DEVICE=m +CONFIG_PS_STK3410=m +CONFIG_PS_UCS14620=m +CONFIG_PWM_ROCKCHIP=m +CONFIG_REGULATOR_ACT8865=m +CONFIG_REGULATOR_FAN53555=m +CONFIG_REGULATOR_GPIO=m +CONFIG_REGULATOR_LP8752=m +CONFIG_REGULATOR_MP8865=m +CONFIG_REGULATOR_PWM=m +CONFIG_REGULATOR_RK806=m +CONFIG_REGULATOR_RK808=m +CONFIG_REGULATOR_RK860X=m +CONFIG_REGULATOR_TPS65132=m +CONFIG_REGULATOR_WL2868C=m +CONFIG_REGULATOR_XZ3216=m +CONFIG_RFKILL_RK=m +CONFIG_RK_CONSOLE_THREAD=y +CONFIG_RK_HEADSET=m +CONFIG_ROCKCHIP_ANALOGIX_DP=y +CONFIG_ROCKCHIP_CDN_DP=y +CONFIG_ROCKCHIP_CPUINFO=m +CONFIG_ROCKCHIP_DEBUG=m +CONFIG_ROCKCHIP_DW_DP=y +CONFIG_ROCKCHIP_DW_HDCP2=m +CONFIG_ROCKCHIP_DW_HDMI=y +CONFIG_ROCKCHIP_DW_MIPI_DSI=y +CONFIG_ROCKCHIP_EFUSE=m +CONFIG_ROCKCHIP_GRF=m +CONFIG_ROCKCHIP_INNO_HDMI=y +CONFIG_ROCKCHIP_IODOMAIN=m +CONFIG_ROCKCHIP_IOMMU=m +CONFIG_ROCKCHIP_IPA=m +CONFIG_ROCKCHIP_LVDS=y +CONFIG_ROCKCHIP_MPP_AV1DEC=y +CONFIG_ROCKCHIP_MPP_IEP2=y +CONFIG_ROCKCHIP_MPP_JPGDEC=y +CONFIG_ROCKCHIP_MPP_RKVDEC=y +CONFIG_ROCKCHIP_MPP_RKVDEC2=y +CONFIG_ROCKCHIP_MPP_RKVENC=y +CONFIG_ROCKCHIP_MPP_RKVENC2=y +CONFIG_ROCKCHIP_MPP_SERVICE=m +CONFIG_ROCKCHIP_MPP_VDPU1=y +CONFIG_ROCKCHIP_MPP_VDPU2=y +CONFIG_ROCKCHIP_MPP_VEPU1=y +CONFIG_ROCKCHIP_MPP_VEPU2=y +CONFIG_ROCKCHIP_MULTI_RGA=m +CONFIG_ROCKCHIP_OPP=m +CONFIG_ROCKCHIP_OTP=m +CONFIG_ROCKCHIP_PHY=m +CONFIG_ROCKCHIP_PM_DOMAINS=m +CONFIG_ROCKCHIP_PVTM=m +CONFIG_ROCKCHIP_REMOTECTL=m +CONFIG_ROCKCHIP_REMOTECTL_PWM=m +CONFIG_ROCKCHIP_RGB=y +CONFIG_ROCKCHIP_RKNPU=m +CONFIG_ROCKCHIP_SARADC=m +CONFIG_ROCKCHIP_SIP=m +CONFIG_ROCKCHIP_SUSPEND_MODE=m +CONFIG_ROCKCHIP_SYSTEM_MONITOR=m +CONFIG_ROCKCHIP_THERMAL=m +CONFIG_ROCKCHIP_TIMER=m +CONFIG_ROCKCHIP_VENDOR_STORAGE=m +CONFIG_ROCKCHIP_VENDOR_STORAGE_UPDATE_LOADER=y +CONFIG_RTC_DRV_HYM8563=m +CONFIG_RTC_DRV_RK808=m +CONFIG_SENSOR_DEVICE=m +CONFIG_SERIAL_8250_DW=m +CONFIG_SMARTJOYPLUS_FF=y +CONFIG_SND_SIMPLE_CARD=m +CONFIG_SND_SOC_BT_SCO=m +CONFIG_SND_SOC_CX2072X=m +CONFIG_SND_SOC_DUMMY_CODEC=m +CONFIG_SND_SOC_ES7202=m +CONFIG_SND_SOC_ES7210=m +CONFIG_SND_SOC_ES7243E=m +CONFIG_SND_SOC_ES8311=m +CONFIG_SND_SOC_ES8316=m +CONFIG_SND_SOC_ES8323=m +CONFIG_SND_SOC_ES8326=m +CONFIG_SND_SOC_ES8396=m +CONFIG_SND_SOC_RK3328=m +CONFIG_SND_SOC_RK817=m +CONFIG_SND_SOC_RK_CODEC_DIGITAL=m +CONFIG_SND_SOC_ROCKCHIP=m +CONFIG_SND_SOC_ROCKCHIP_HDMI=m +CONFIG_SND_SOC_ROCKCHIP_I2S=m +CONFIG_SND_SOC_ROCKCHIP_I2S_TDM=m +CONFIG_SND_SOC_ROCKCHIP_MULTICODECS=m +CONFIG_SND_SOC_ROCKCHIP_PDM=m +CONFIG_SND_SOC_ROCKCHIP_SPDIF=m +CONFIG_SND_SOC_ROCKCHIP_SPDIFRX=m +CONFIG_SND_SOC_RT5640=m +CONFIG_SND_SOC_SPDIF=m +CONFIG_SPI_ROCKCHIP=m +CONFIG_SPI_SPIDEV=m +CONFIG_SW_SYNC=m +CONFIG_SYSCON_REBOOT_MODE=m +CONFIG_TEE=m +CONFIG_TEST_POWER=m +CONFIG_TOUCHSCREEN_ELAN5515=m +CONFIG_TOUCHSCREEN_GSL3673=m +CONFIG_TOUCHSCREEN_GSLX680_PAD=m +CONFIG_TYPEC_DP_ALTMODE=m +CONFIG_TYPEC_FUSB302=m +CONFIG_TYPEC_HUSB311=m +CONFIG_UCS12CM0=m +CONFIG_USB_DWC2=m +CONFIG_USB_NET_CDC_MBIM=m +CONFIG_USB_NET_DM9601=m +CONFIG_USB_NET_GL620A=m +CONFIG_USB_NET_KALMIA=m +CONFIG_USB_NET_MCS7830=m +CONFIG_USB_NET_PLUSB=m +CONFIG_USB_NET_SMSC75XX=m +CONFIG_USB_NET_SMSC95XX=m +CONFIG_USB_OHCI_HCD=m +# CONFIG_USB_OHCI_HCD_PCI is not set +CONFIG_USB_OHCI_HCD_PLATFORM=m +CONFIG_USB_PRINTER=m +CONFIG_USB_SERIAL=m +CONFIG_USB_SERIAL_GENERIC=y +CONFIG_USB_TRANCEVIBRATOR=m +CONFIG_VIDEO_AW36518=m +CONFIG_VIDEO_AW8601=m +CONFIG_VIDEO_CN3927V=m +CONFIG_VIDEO_DW9714=m +CONFIG_VIDEO_FP5510=m +CONFIG_VIDEO_GC2145=m +CONFIG_VIDEO_GC2385=m +CONFIG_VIDEO_GC4C33=m +CONFIG_VIDEO_GC8034=m +CONFIG_VIDEO_IMX415=m +CONFIG_VIDEO_LT6911UXC=m +CONFIG_VIDEO_LT7911D=m +CONFIG_VIDEO_NVP6188=m +CONFIG_VIDEO_OV02B10=m +CONFIG_VIDEO_OV13850=m +CONFIG_VIDEO_OV13855=m +CONFIG_VIDEO_OV50C40=m +CONFIG_VIDEO_OV5695=m +CONFIG_VIDEO_OV8858=m +CONFIG_VIDEO_RK628_BT1120=m +CONFIG_VIDEO_RK628_CSI=m +CONFIG_VIDEO_RK_IRCUT=m +CONFIG_VIDEO_ROCKCHIP_CIF=m +CONFIG_VIDEO_ROCKCHIP_ISP=m +CONFIG_VIDEO_ROCKCHIP_ISPP=m +CONFIG_VIDEO_S5K3L6XX=m +CONFIG_VIDEO_S5KJN1=m +CONFIG_VIDEO_SGM3784=m +CONFIG_VIDEO_THCV244=m +CONFIG_VL6180=m +CONFIG_WIFI_BUILD_MODULE=y +CONFIG_WL_ROCKCHIP=m +CONFIG_ZRAM=m +CONFIG_ZSMALLOC=m +# CONFIG_USB_DUMMY_HCD is not set diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h index 1bc198a5faad..2c916076832d 100644 --- a/arch/arm64/include/asm/el2_setup.h +++ b/arch/arm64/include/asm/el2_setup.h @@ -107,7 +107,7 @@ msr_s SYS_ICC_SRE_EL2, x0 isb // Make sure SRE is now set mrs_s x0, SYS_ICC_SRE_EL2 // Read SRE back, - tbz x0, #0, 1f // and check that it sticks + tbz x0, #0, .Lskip_gicv3_\@ // and check that it sticks msr_s SYS_ICH_HCR_EL2, xzr // Reset ICC_HCR_EL2 to defaults .Lskip_gicv3_\@: .endm diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index ef1cf41ea394..edca12fd852e 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -75,6 +75,7 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___vgic_v3_save_vmcr_aprs, __KVM_HOST_SMCCC_FUNC___vgic_v3_restore_vmcr_aprs, __KVM_HOST_SMCCC_FUNC___pkvm_init_shadow, + __KVM_HOST_SMCCC_FUNC___pkvm_init_shadow_vcpu, __KVM_HOST_SMCCC_FUNC___pkvm_teardown_shadow, __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_load, __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_put, diff --git a/arch/arm64/include/asm/mte-kasan.h b/arch/arm64/include/asm/mte-kasan.h index e4704a403237..a857bcacf0fe 100644 --- a/arch/arm64/include/asm/mte-kasan.h +++ b/arch/arm64/include/asm/mte-kasan.h @@ -5,6 +5,7 @@ #ifndef __ASM_MTE_KASAN_H #define __ASM_MTE_KASAN_H +#include #include #ifndef __ASSEMBLY__ diff --git a/arch/arm64/include/asm/spectre.h b/arch/arm64/include/asm/spectre.h index c04d01dd4457..71060c1c23fe 100644 --- a/arch/arm64/include/asm/spectre.h +++ b/arch/arm64/include/asm/spectre.h @@ -67,7 +67,8 @@ struct bp_hardening_data { DECLARE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data); -static inline void arm64_apply_bp_hardening(void) +/* Called during entry so must be __always_inline */ +static __always_inline void arm64_apply_bp_hardening(void) { struct bp_hardening_data *d; diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c index 3c35f9376d3d..40be3a7c2c53 100644 --- a/arch/arm64/kernel/proton-pack.c +++ b/arch/arm64/kernel/proton-pack.c @@ -233,17 +233,20 @@ static void install_bp_hardening_cb(bp_hardening_cb_t fn) __this_cpu_write(bp_hardening_data.slot, HYP_VECTOR_SPECTRE_DIRECT); } -static void call_smc_arch_workaround_1(void) +/* Called during entry so must be noinstr */ +static noinstr void call_smc_arch_workaround_1(void) { arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL); } -static void call_hvc_arch_workaround_1(void) +/* Called during entry so must be noinstr */ +static noinstr void call_hvc_arch_workaround_1(void) { arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL); } -static void qcom_link_stack_sanitisation(void) +/* Called during entry so must be noinstr */ +static noinstr void qcom_link_stack_sanitisation(void) { u64 tmp; diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 8e22c300752c..366999f8698a 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -2129,6 +2129,17 @@ static int pkvm_drop_host_privileges(void) * once the host stage 2 is installed. */ static_branch_enable(&kvm_protected_mode_initialized); + + /* + * Fixup the boot mode so that we don't take spurious round + * trips via EL2 on cpu_resume. Flush to the PoC for a good + * measure, so that it can be observed by a CPU coming out of + * suspend with the MMU off. + */ + __boot_cpu_mode[0] = __boot_cpu_mode[1] = BOOT_CPU_MODE_EL1; + dcache_clean_poc((unsigned long)__boot_cpu_mode, + (unsigned long)(__boot_cpu_mode + 2)); + on_each_cpu(_kvm_host_prot_finalize, &ret, 1); return ret; } diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c index dbb7097396f1..5ccf487936d2 100644 --- a/arch/arm64/kvm/handle_exit.c +++ b/arch/arm64/kvm/handle_exit.c @@ -241,6 +241,14 @@ int handle_exit(struct kvm_vcpu *vcpu, int exception_index) { struct kvm_run *run = vcpu->run; + if (ARM_SERROR_PENDING(exception_index)) { + /* + * The SError is handled by handle_exit_early(). If the guest + * survives it will re-execute the original instruction. + */ + return 1; + } + exception_index = ARM_EXCEPTION_CODE(exception_index); switch (exception_index) { diff --git a/arch/arm64/kvm/hyp/exception.c b/arch/arm64/kvm/hyp/exception.c index 14a80b0e2f91..ceb6808c2d80 100644 --- a/arch/arm64/kvm/hyp/exception.c +++ b/arch/arm64/kvm/hyp/exception.c @@ -13,6 +13,7 @@ #include #include #include +#include #if !defined (__KVM_NVHE_HYPERVISOR__) && !defined (__KVM_VHE_HYPERVISOR__) #error Hypervisor code only! @@ -165,7 +166,8 @@ static void enter_exception64(struct kvm_vcpu *vcpu, unsigned long target_mode, *vcpu_pc(vcpu) = vbar + offset; old = *vcpu_cpsr(vcpu); - new = get_except64_cpsr(old, kvm_has_mte(vcpu->kvm), sctlr, target_mode); + new = get_except64_cpsr(old, kvm_has_mte(kern_hyp_va(vcpu->kvm)), sctlr, + target_mode); *vcpu_cpsr(vcpu) = new; __vcpu_write_spsr(vcpu, old); } diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index c3463dda749c..fcf2c0e8f600 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -420,7 +420,8 @@ static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code) if (ARM_EXCEPTION_CODE(*exit_code) != ARM_EXCEPTION_IRQ) vcpu->arch.fault.esr_el2 = read_sysreg_el2(SYS_ESR); - if (ARM_SERROR_PENDING(*exit_code)) { + if (ARM_SERROR_PENDING(*exit_code) && + ARM_EXCEPTION_CODE(*exit_code) != ARM_EXCEPTION_IRQ) { u8 esr_ec = kvm_vcpu_trap_get_class(vcpu); /* diff --git a/arch/arm64/kvm/hyp/include/nvhe/iommu.h b/arch/arm64/kvm/hyp/include/nvhe/iommu.h index 69bce1f01717..f9a75f49c499 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/iommu.h +++ b/arch/arm64/kvm/hyp/include/nvhe/iommu.h @@ -48,6 +48,12 @@ struct pkvm_iommu_ops { void (*host_stage2_idmap_apply)(struct pkvm_iommu *dev, phys_addr_t start, phys_addr_t end); + /* + * Callback to finish a host stage-2 mapping change at device level. + * Called after 'host_stage2_idmap_apply' with host lock held. + */ + void (*host_stage2_idmap_complete)(struct pkvm_iommu *dev); + /* Power management callbacks. Called with host lock held. */ int (*suspend)(struct pkvm_iommu *dev); int (*resume)(struct pkvm_iommu *dev); diff --git a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h index 80ecc831fc31..de7f607dde58 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h @@ -42,14 +42,20 @@ struct kvm_shadow_vm { /* The total size of the donated shadow area. */ size_t shadow_area_size; + /* + * The number of vcpus initialized and ready to run in the shadow vm. + * Modifying this is protected by shadow_lock. + */ + unsigned int nr_vcpus; + struct kvm_arch arch; struct kvm_pgtable pgt; struct kvm_pgtable_mm_ops mm_ops; struct hyp_pool pool; hyp_spinlock_t lock; - /* Array of the shadow state per vcpu. */ - struct shadow_vcpu_state shadow_vcpus[0]; + /* Array of the shadow state pointers per vcpu. */ + struct shadow_vcpu_state *shadow_vcpus[0]; }; static inline bool vcpu_is_protected(struct kvm_vcpu *vcpu) @@ -65,6 +71,9 @@ extern phys_addr_t pvmfw_size; void hyp_shadow_table_init(void *tbl); int __pkvm_init_shadow(struct kvm *kvm, void *shadow_va, size_t size, void *pgd); +int __pkvm_init_shadow_vcpu(unsigned int shadow_handle, + struct kvm_vcpu *host_vcpu, + void *shadow_vcpu_hva); int __pkvm_teardown_shadow(int shadow_handle); struct kvm_vcpu *get_shadow_vcpu(int shadow_handle, unsigned int vcpu_idx); void put_shadow_vcpu(struct kvm_vcpu *vcpu); diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 14f9da9f87b9..9901e102f330 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -1006,6 +1006,17 @@ static void handle___pkvm_init_shadow(struct kvm_cpu_context *host_ctxt) shadow_size, pgd); } +static void handle___pkvm_init_shadow_vcpu(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(unsigned int, shadow_handle, host_ctxt, 1); + DECLARE_REG(struct kvm_vcpu *, host_vcpu, host_ctxt, 2); + DECLARE_REG(void *, shadow_vcpu_hva, host_ctxt, 3); + + cpu_reg(host_ctxt, 1) = __pkvm_init_shadow_vcpu(shadow_handle, + host_vcpu, + shadow_vcpu_hva); +} + static void handle___pkvm_teardown_shadow(struct kvm_cpu_context *host_ctxt) { DECLARE_REG(int, shadow_handle, host_ctxt, 1); @@ -1079,6 +1090,7 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__vgic_v3_save_vmcr_aprs), HANDLE_FUNC(__vgic_v3_restore_vmcr_aprs), HANDLE_FUNC(__pkvm_init_shadow), + HANDLE_FUNC(__pkvm_init_shadow_vcpu), HANDLE_FUNC(__pkvm_teardown_shadow), HANDLE_FUNC(__pkvm_vcpu_load), HANDLE_FUNC(__pkvm_vcpu_put), diff --git a/arch/arm64/kvm/hyp/nvhe/iommu.c b/arch/arm64/kvm/hyp/nvhe/iommu.c index 3fe47e542088..9327dd78af80 100644 --- a/arch/arm64/kvm/hyp/nvhe/iommu.c +++ b/arch/arm64/kvm/hyp/nvhe/iommu.c @@ -546,4 +546,9 @@ void pkvm_iommu_host_stage2_idmap(phys_addr_t start, phys_addr_t end, if (dev->powered && dev->ops->host_stage2_idmap_apply) dev->ops->host_stage2_idmap_apply(dev, start, end); } + + list_for_each_entry(dev, &iommu_list, list) { + if (dev->powered && dev->ops->host_stage2_idmap_complete) + dev->ops->host_stage2_idmap_complete(dev); + } } diff --git a/arch/arm64/kvm/hyp/nvhe/iommu/s2mpu.c b/arch/arm64/kvm/hyp/nvhe/iommu/s2mpu.c index ff5d7d1044e5..8da0e43bab32 100644 --- a/arch/arm64/kvm/hyp/nvhe/iommu/s2mpu.c +++ b/arch/arm64/kvm/hyp/nvhe/iommu/s2mpu.c @@ -24,6 +24,10 @@ #define PA_MAX ((phys_addr_t)SZ_1G * NR_GIGABYTES) +#define SYNC_MAX_RETRIES 5 +#define SYNC_TIMEOUT 5 +#define SYNC_TIMEOUT_MULTIPLIER 3 + #define CTX_CFG_ENTRY(ctxid, nr_ctx, vid) \ (CONTEXT_CFG_VALID_VID_CTX_VID(ctxid, vid) \ | (((ctxid) < (nr_ctx)) ? CONTEXT_CFG_VALID_VID_CTX_VALID(ctxid) : 0)) @@ -158,11 +162,20 @@ static void __set_control_regs(struct pkvm_iommu *dev) writel_relaxed(ctrl0, dev->va + REG_NS_CTRL0); } -/* Poll the given SFR until its value has all bits of a given mask set. */ -static void __wait_until(void __iomem *addr, u32 mask) +/* + * Poll the given SFR until its value has all bits of a given mask set. + * Returns true if successful, false if not successful after a given number of + * attempts. + */ +static bool __wait_until(void __iomem *addr, u32 mask, size_t max_attempts) { - while ((readl_relaxed(addr) & mask) != mask) - continue; + size_t i; + + for (i = 0; i < max_attempts; i++) { + if ((readl_relaxed(addr) & mask) == mask) + return true; + } + return false; } /* Poll the given SFR as long as its value has all bits of a given mask set. */ @@ -172,17 +185,54 @@ static void __wait_while(void __iomem *addr, u32 mask) continue; } -static void __wait_for_invalidation_complete(struct pkvm_iommu *dev) +static void __sync_cmd_start(struct pkvm_iommu *sync) { - struct pkvm_iommu *sync; + writel_relaxed(SYNC_CMD_SYNC, sync->va + REG_NS_SYNC_CMD); +} + +static void __invalidation_barrier_slow(struct pkvm_iommu *sync) +{ + size_t i, timeout; /* * Wait for transactions to drain if SysMMU_SYNCs were registered. * Assumes that they are in the same power domain as the S2MPU. + * + * The algorithm will try initiating the SYNC if the SYNC_COMP_COMPLETE + * bit has not been set after a given number of attempts, increasing the + * timeout exponentially each time. If this cycle fails a given number + * of times, the algorithm will give up completely to avoid deadlock. + */ + timeout = SYNC_TIMEOUT; + for (i = 0; i < SYNC_MAX_RETRIES; i++) { + __sync_cmd_start(sync); + if (__wait_until(sync->va + REG_NS_SYNC_COMP, SYNC_COMP_COMPLETE, timeout)) + break; + timeout *= SYNC_TIMEOUT_MULTIPLIER; + } +} + +/* Initiate invalidation barrier. */ +static void __invalidation_barrier_init(struct pkvm_iommu *dev) +{ + struct pkvm_iommu *sync; + + for_each_child(sync, dev) + __sync_cmd_start(sync); +} + +/* Wait for invalidation to complete. */ +static void __invalidation_barrier_complete(struct pkvm_iommu *dev) +{ + struct pkvm_iommu *sync; + + /* + * Check if the SYNC_COMP_COMPLETE bit has been set for individual + * devices. If not, fall back to non-parallel invalidation. */ for_each_child(sync, dev) { - writel_relaxed(SYNC_CMD_SYNC, sync->va + REG_NS_SYNC_CMD); - __wait_until(sync->va + REG_NS_SYNC_COMP, SYNC_COMP_COMPLETE); + if (!(readl_relaxed(sync->va + REG_NS_SYNC_COMP) & SYNC_COMP_COMPLETE)) + __invalidation_barrier_slow(sync); } /* Must not access SFRs while S2MPU is busy invalidating (v9 only). */ @@ -195,11 +245,12 @@ static void __wait_for_invalidation_complete(struct pkvm_iommu *dev) static void __all_invalidation(struct pkvm_iommu *dev) { writel_relaxed(INVALIDATION_INVALIDATE, dev->va + REG_NS_ALL_INVALIDATION); - __wait_for_invalidation_complete(dev); + __invalidation_barrier_init(dev); + __invalidation_barrier_complete(dev); } -static void __range_invalidation(struct pkvm_iommu *dev, phys_addr_t first_byte, - phys_addr_t last_byte) +static void __range_invalidation_init(struct pkvm_iommu *dev, phys_addr_t first_byte, + phys_addr_t last_byte) { u32 start_ppn = first_byte >> RANGE_INVALIDATION_PPN_SHIFT; u32 end_ppn = last_byte >> RANGE_INVALIDATION_PPN_SHIFT; @@ -207,7 +258,7 @@ static void __range_invalidation(struct pkvm_iommu *dev, phys_addr_t first_byte, writel_relaxed(start_ppn, dev->va + REG_NS_RANGE_INVALIDATION_START_PPN); writel_relaxed(end_ppn, dev->va + REG_NS_RANGE_INVALIDATION_END_PPN); writel_relaxed(INVALIDATION_INVALIDATE, dev->va + REG_NS_RANGE_INVALIDATION); - __wait_for_invalidation_complete(dev); + __invalidation_barrier_init(dev); } static void __set_l1entry_attr_with_prot(struct pkvm_iommu *dev, unsigned int gb, @@ -341,7 +392,13 @@ static void __mpt_idmap_apply(struct pkvm_iommu *dev, struct mpt *mpt, __set_l1entry_attr_with_fmpt(dev, gb, vid, fmpt); } } - __range_invalidation(dev, first_byte, last_byte); + /* Initiate invalidation, completed in __mdt_idmap_complete. */ + __range_invalidation_init(dev, first_byte, last_byte); +} + +static void __mpt_idmap_complete(struct pkvm_iommu *dev, struct mpt *mpt) +{ + __invalidation_barrier_complete(dev); } static void s2mpu_host_stage2_idmap_prepare(phys_addr_t start, phys_addr_t end, @@ -362,6 +419,11 @@ static void s2mpu_host_stage2_idmap_apply(struct pkvm_iommu *dev, __mpt_idmap_apply(dev, &host_mpt, start, end - 1); } +static void s2mpu_host_stage2_idmap_complete(struct pkvm_iommu *dev) +{ + __mpt_idmap_complete(dev, &host_mpt); +} + static int s2mpu_resume(struct pkvm_iommu *dev) { /* @@ -550,6 +612,7 @@ const struct pkvm_iommu_ops pkvm_s2mpu_ops = (struct pkvm_iommu_ops){ .suspend = s2mpu_suspend, .host_stage2_idmap_prepare = s2mpu_host_stage2_idmap_prepare, .host_stage2_idmap_apply = s2mpu_host_stage2_idmap_apply, + .host_stage2_idmap_complete = s2mpu_host_stage2_idmap_complete, .host_dabt_handler = s2mpu_host_dabt_handler, .data_size = sizeof(struct s2mpu_drv_data), }; diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index 6ca172ac3445..5d2ce6e14405 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -929,6 +929,9 @@ static int host_complete_share(u64 addr, const struct pkvm_mem_transition *tx, { u64 size = tx->nr_pages * PAGE_SIZE; + if (tx->initiator.id == PKVM_ID_GUEST) + psci_mem_protect_dec(); + return __host_set_page_state_range(addr, size, PKVM_PAGE_SHARED_BORROWED); } @@ -937,6 +940,9 @@ static int host_complete_unshare(u64 addr, const struct pkvm_mem_transition *tx) u64 size = tx->nr_pages * PAGE_SIZE; pkvm_id owner_id = initiator_owner_id(tx); + if (tx->initiator.id == PKVM_ID_GUEST) + psci_mem_protect_inc(); + return host_stage2_set_owner_locked(addr, size, owner_id); } @@ -1915,7 +1921,14 @@ static int hyp_zero_page(phys_addr_t phys) if (!addr) return -EINVAL; memset(addr, 0, PAGE_SIZE); - __clean_dcache_guest_page(addr, PAGE_SIZE); + /* + * Prefer kvm_flush_dcache_to_poc() over __clean_dcache_guest_page() + * here as the latter may elide the CMO under the assumption that FWB + * will be enabled on CPUs that support it. This is incorrect for the + * host stage-2 and would otherwise lead to a malicious host potentially + * being able to read the content of newly reclaimed guest pages. + */ + kvm_flush_dcache_to_poc(addr, PAGE_SIZE); return hyp_fixmap_unmap(); } diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 50717a46e735..cc2612937e7e 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -276,9 +276,9 @@ struct kvm_vcpu *get_shadow_vcpu(int shadow_handle, unsigned int vcpu_idx) hyp_spin_lock(&shadow_lock); vm = find_shadow_by_handle(shadow_handle); - if (!vm || vm->created_vcpus <= vcpu_idx) + if (!vm || vm->nr_vcpus <= vcpu_idx) goto unlock; - vcpu = &vm->shadow_vcpus[vcpu_idx].vcpu; + vcpu = &vm->shadow_vcpus[vcpu_idx]->vcpu; /* Ensure vcpu isn't loaded on more than one cpu simultaneously. */ if (unlikely(vcpu->arch.pkvm.loaded_on_cpu)) { @@ -370,48 +370,28 @@ static int copy_features(struct kvm_vcpu *shadow_vcpu, struct kvm_vcpu *host_vcp return 0; } -static void unpin_host_vcpus(struct shadow_vcpu_state *shadow_vcpus, int nr_vcpus) +static void unpin_host_vcpu(struct shadow_vcpu_state *shadow_vcpu) { - int i; + struct kvm_vcpu *host_vcpu = shadow_vcpu->vcpu.arch.pkvm.host_vcpu; + size_t sve_state_size; + void *sve_state = shadow_vcpu->vcpu.arch.sve_state; - for (i = 0; i < nr_vcpus; i++) { - struct kvm_vcpu *host_vcpu = shadow_vcpus[i].vcpu.arch.pkvm.host_vcpu; - struct kvm_vcpu *shadow_vcpu = &shadow_vcpus[i].vcpu; - size_t sve_state_size; - void *sve_state; + hyp_unpin_shared_mem(host_vcpu, host_vcpu + 1); - hyp_unpin_shared_mem(host_vcpu, host_vcpu + 1); + if (!sve_state) + return; - if (!test_bit(KVM_ARM_VCPU_SVE, shadow_vcpu->arch.features)) - continue; - - sve_state = shadow_vcpu->arch.sve_state; - sve_state = kern_hyp_va(sve_state); - sve_state_size = vcpu_sve_state_size(shadow_vcpu); - hyp_unpin_shared_mem(sve_state, sve_state + sve_state_size); - } + sve_state = kern_hyp_va(sve_state); + sve_state_size = vcpu_sve_state_size(&shadow_vcpu->vcpu); + hyp_unpin_shared_mem(sve_state, sve_state + sve_state_size); } -static int set_host_vcpus(struct shadow_vcpu_state *shadow_vcpus, int nr_vcpus, - struct kvm_vcpu **vcpu_array, size_t vcpu_array_size) +static void unpin_host_vcpus(struct shadow_vcpu_state *shadow_vcpus[], int nr_vcpus) { int i; - if (vcpu_array_size < sizeof(*vcpu_array) * nr_vcpus) - return -EINVAL; - - for (i = 0; i < nr_vcpus; i++) { - struct kvm_vcpu *host_vcpu = kern_hyp_va(vcpu_array[i]); - - if (hyp_pin_shared_mem(host_vcpu, host_vcpu + 1)) { - unpin_host_vcpus(shadow_vcpus, i); - return -EBUSY; - } - - shadow_vcpus[i].vcpu.arch.pkvm.host_vcpu = host_vcpu; - } - - return 0; + for (i = 0; i < nr_vcpus; i++) + unpin_host_vcpu(shadow_vcpus[i]); } static int init_ptrauth(struct kvm_vcpu *shadow_vcpu) @@ -423,86 +403,106 @@ static int init_ptrauth(struct kvm_vcpu *shadow_vcpu) return ret; } -static int init_shadow_structs(struct kvm *kvm, struct kvm_shadow_vm *vm, - struct kvm_vcpu **vcpu_array, int nr_vcpus) +static void init_shadow_vm(struct kvm *kvm, struct kvm_shadow_vm *vm, + int nr_vcpus) { - int i; - int ret; - vm->host_kvm = kvm; vm->created_vcpus = nr_vcpus; vm->arch.pkvm.pvmfw_load_addr = kvm->arch.pkvm.pvmfw_load_addr; vm->arch.pkvm.enabled = READ_ONCE(kvm->arch.pkvm.enabled); +} - for (i = 0; i < nr_vcpus; i++) { - struct shadow_vcpu_state *shadow_state = &vm->shadow_vcpus[i]; - struct kvm_vcpu *shadow_vcpu = &shadow_state->vcpu; - struct kvm_vcpu *host_vcpu = shadow_vcpu->arch.pkvm.host_vcpu; +static int init_shadow_vcpu(struct shadow_vcpu_state *shadow_state, + struct kvm_vcpu *host_vcpu, + struct kvm_shadow_vm *vm, int vcpu_idx) +{ + struct kvm_vcpu *shadow_vcpu = &shadow_state->vcpu; + int ret; - shadow_vcpu->kvm = kvm; - shadow_vcpu->vcpu_id = host_vcpu->vcpu_id; - shadow_vcpu->vcpu_idx = i; + host_vcpu = kern_hyp_va(host_vcpu); + if (hyp_pin_shared_mem(host_vcpu, host_vcpu + 1)) + return -EBUSY; - ret = copy_features(shadow_vcpu, host_vcpu); - if (ret) - return ret; + if (host_vcpu->vcpu_idx != vcpu_idx) { + ret = -EINVAL; + goto done; + } - ret = init_ptrauth(shadow_vcpu); - if (ret) - return ret; + shadow_vcpu->arch.pkvm.host_vcpu = host_vcpu; + shadow_vcpu->kvm = vm->host_kvm; + shadow_vcpu->vcpu_id = host_vcpu->vcpu_id; + shadow_vcpu->vcpu_idx = vcpu_idx; - if (test_bit(KVM_ARM_VCPU_SVE, shadow_vcpu->arch.features)) { - size_t sve_state_size; - void *sve_state; + ret = copy_features(shadow_vcpu, host_vcpu); + if (ret) + goto done; - shadow_vcpu->arch.sve_state = READ_ONCE(host_vcpu->arch.sve_state); - shadow_vcpu->arch.sve_max_vl = READ_ONCE(host_vcpu->arch.sve_max_vl); + ret = init_ptrauth(shadow_vcpu); + if (ret) + goto done; - sve_state = kern_hyp_va(shadow_vcpu->arch.sve_state); - sve_state_size = vcpu_sve_state_size(shadow_vcpu); + if (test_bit(KVM_ARM_VCPU_SVE, shadow_vcpu->arch.features)) { + size_t sve_state_size; + void *sve_state; - if (!shadow_vcpu->arch.sve_state || !sve_state_size || - hyp_pin_shared_mem(sve_state, - sve_state + sve_state_size)) { - clear_bit(KVM_ARM_VCPU_SVE, - shadow_vcpu->arch.features); - shadow_vcpu->arch.sve_state = NULL; - shadow_vcpu->arch.sve_max_vl = 0; - return -EINVAL; - } - } + shadow_vcpu->arch.sve_state = + READ_ONCE(host_vcpu->arch.sve_state); + shadow_vcpu->arch.sve_max_vl = + READ_ONCE(host_vcpu->arch.sve_max_vl); - if (vm->arch.pkvm.enabled) - pkvm_vcpu_init_traps(shadow_vcpu); - kvm_reset_pvm_sys_regs(shadow_vcpu); + sve_state = kern_hyp_va(shadow_vcpu->arch.sve_state); + sve_state_size = vcpu_sve_state_size(shadow_vcpu); - vm->vcpus[i] = shadow_vcpu; - shadow_state->vm = vm; - - shadow_vcpu->arch.hw_mmu = &vm->arch.mmu; - shadow_vcpu->arch.pkvm.shadow_vm = vm; - shadow_vcpu->arch.power_off = true; - - if (test_bit(KVM_ARM_VCPU_POWER_OFF, shadow_vcpu->arch.features)) { - shadow_vcpu->arch.pkvm.power_state = PSCI_0_2_AFFINITY_LEVEL_OFF; - } else if (pvm_has_pvmfw(vm)) { - if (vm->pvmfw_entry_vcpu) - return -EINVAL; - - vm->pvmfw_entry_vcpu = shadow_vcpu; - shadow_vcpu->arch.reset_state.reset = true; - shadow_vcpu->arch.pkvm.power_state = PSCI_0_2_AFFINITY_LEVEL_ON_PENDING; - } else { - struct vcpu_reset_state *reset_state = &shadow_vcpu->arch.reset_state; - - reset_state->pc = *vcpu_pc(host_vcpu); - reset_state->r0 = vcpu_get_reg(host_vcpu, 0); - reset_state->reset = true; - shadow_vcpu->arch.pkvm.power_state = PSCI_0_2_AFFINITY_LEVEL_ON_PENDING; + if (!shadow_vcpu->arch.sve_state || !sve_state_size || + hyp_pin_shared_mem(sve_state, sve_state + sve_state_size)) { + clear_bit(KVM_ARM_VCPU_SVE, shadow_vcpu->arch.features); + shadow_vcpu->arch.sve_state = NULL; + shadow_vcpu->arch.sve_max_vl = 0; + ret = -EINVAL; + goto done; } } - return 0; + if (vm->arch.pkvm.enabled) + pkvm_vcpu_init_traps(shadow_vcpu); + kvm_reset_pvm_sys_regs(shadow_vcpu); + + vm->vcpus[vcpu_idx] = shadow_vcpu; + shadow_state->vm = vm; + + shadow_vcpu->arch.hw_mmu = &vm->arch.mmu; + shadow_vcpu->arch.pkvm.shadow_vm = vm; + shadow_vcpu->arch.power_off = true; + + if (test_bit(KVM_ARM_VCPU_POWER_OFF, shadow_vcpu->arch.features)) { + shadow_vcpu->arch.pkvm.power_state = + PSCI_0_2_AFFINITY_LEVEL_OFF; + } else if (pvm_has_pvmfw(vm)) { + if (vm->pvmfw_entry_vcpu) { + ret = -EINVAL; + goto done; + } + + vm->pvmfw_entry_vcpu = shadow_vcpu; + shadow_vcpu->arch.reset_state.reset = true; + shadow_vcpu->arch.pkvm.power_state = + PSCI_0_2_AFFINITY_LEVEL_ON_PENDING; + } else { + struct vcpu_reset_state *reset_state = + &shadow_vcpu->arch.reset_state; + + reset_state->pc = *vcpu_pc(host_vcpu); + reset_state->r0 = vcpu_get_reg(host_vcpu, 0); + reset_state->reset = true; + shadow_vcpu->arch.pkvm.power_state = + PSCI_0_2_AFFINITY_LEVEL_ON_PENDING; + } + +done: + if (ret) + unpin_host_vcpu(shadow_state); + + return ret; } static bool __exists_shadow(struct kvm *host_kvm) @@ -591,7 +591,7 @@ static size_t pkvm_get_shadow_size(int num_vcpus) { /* Shadow space for the vm struct and all of its vcpu states. */ return sizeof(struct kvm_shadow_vm) + - sizeof(struct shadow_vcpu_state) * num_vcpus; + sizeof(struct shadow_vcpu_state *) * num_vcpus; } /* @@ -613,14 +613,14 @@ static int check_shadow_size(int nr_vcpus, size_t shadow_size) return 0; } -static void drain_shadow_vcpus(struct shadow_vcpu_state *shadow_vcpus, +static void drain_shadow_vcpus(struct shadow_vcpu_state *shadow_vcpus[], unsigned int nr_vcpus, struct kvm_hyp_memcache *mc) { int i; for (i = 0; i < nr_vcpus; i++) { - struct kvm_vcpu *shadow_vcpu = &shadow_vcpus[i].vcpu; + struct kvm_vcpu *shadow_vcpu = &shadow_vcpus[i]->vcpu; struct kvm_hyp_memcache *vcpu_mc = &shadow_vcpu->arch.pkvm_memcache; void *addr; @@ -645,8 +645,6 @@ static void drain_shadow_vcpus(struct shadow_vcpu_state *shadow_vcpus, * Must be a multiple of the page size. * pgd: The host va of the area being donated for the stage-2 PGD for the VM. * Must be page aligned. Its size is implied by the VM's VTCR. - * Note: An array to the host KVM VCPUs (host VA) is passed via the pgd, as to - * not to be dependent on how the VCPU's are layed out in struct kvm. * * Return a unique handle to the protected VM on success, * negative error code on failure. @@ -698,19 +696,13 @@ int __pkvm_init_shadow(struct kvm *kvm, if (ret) goto err_remove_mappings; - ret = set_host_vcpus(vm->shadow_vcpus, nr_vcpus, pgd, pgd_size); - if (ret) - goto err_remove_pgd; - - ret = init_shadow_structs(kvm, vm, pgd, nr_vcpus); - if (ret < 0) - goto err_unpin_host_vcpus; + init_shadow_vm(kvm, vm, nr_vcpus); /* Add the entry to the shadow table. */ hyp_spin_lock(&shadow_lock); ret = insert_shadow_table(kvm, vm, shadow_size); if (ret < 0) - goto err_unlock_unpin_host_vcpus; + goto err_unlock; ret = kvm_guest_prepare_stage2(vm, pgd); if (ret) @@ -721,34 +713,106 @@ int __pkvm_init_shadow(struct kvm *kvm, err_remove_shadow_table: remove_shadow_table(vm->shadow_handle); -err_unlock_unpin_host_vcpus: +err_unlock: hyp_spin_unlock(&shadow_lock); -err_unpin_host_vcpus: - unpin_host_vcpus(vm->shadow_vcpus, nr_vcpus); -err_remove_pgd: WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(pgd), nr_pgd_pages)); - err_remove_mappings: /* Clear the donated shadow memory on failure to avoid data leaks. */ memset(vm, 0, shadow_size); WARN_ON(__pkvm_hyp_donate_host(hyp_phys_to_pfn(shadow_pa), shadow_size >> PAGE_SHIFT)); - err: hyp_unpin_shared_mem(kvm, kvm + 1); return ret; } +/* + * Initialize the protected vcpu state shadow copy in host-donated memory. + * + * shadow_handle: The handle for the protected vm. + * host_vcpu: A pointer to the corresponding host vcpu (host va). + * shadow_vcpu_hva: The host va of the area being donated for the vcpu state. + * Must be page aligned. The size of the area must be equal to + * the paged-aligned size of kvm_shadow_vcpu_state. + * + * Return 0 on success, negative error code on failure. + */ +int __pkvm_init_shadow_vcpu(unsigned int shadow_handle, + struct kvm_vcpu *host_vcpu, + void *shadow_vcpu_hva) +{ + struct kvm_shadow_vm *vm; + struct shadow_vcpu_state *shadow_state = kern_hyp_va(shadow_vcpu_hva); + size_t vcpu_state_sz = sizeof(*shadow_state); + u64 nr_pages = PAGE_ALIGN(vcpu_state_sz) >> PAGE_SHIFT; + unsigned int idx; + int ret; + + if (!PAGE_ALIGNED(shadow_vcpu_hva)) + return -EINVAL; + + ret = __pkvm_host_donate_hyp(hyp_virt_to_pfn(shadow_state), + nr_pages); + if (ret) + return ret; + + memset(shadow_state, 0, vcpu_state_sz); + + hyp_spin_lock(&shadow_lock); + + vm = find_shadow_by_handle(shadow_handle); + if (!vm) { + ret = -ENOENT; + goto unlock; + } + + idx = vm->nr_vcpus; + if (idx >= vm->created_vcpus) { + ret = -EINVAL; + goto unlock; + } + + ret = init_shadow_vcpu(shadow_state, host_vcpu, vm, idx); + if (ret) + goto unlock; + + vm->shadow_vcpus[idx] = shadow_state; + vm->nr_vcpus++; +unlock: + hyp_spin_unlock(&shadow_lock); + + if (ret) { + memset(shadow_state, 0, vcpu_state_sz); + WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(shadow_state), + nr_pages)); + } + + return ret; +} + +static void teardown_donated_memory(struct kvm_hyp_memcache *mc, void *addr, + size_t size) +{ + u64 pfn = hyp_phys_to_pfn(__hyp_pa(addr)); + u64 nr_pages = size >> PAGE_SHIFT; + void *start; + + memset(addr, 0, size); + kvm_flush_dcache_to_poc(addr, size); + + for (start = addr; start < addr + size; start += PAGE_SIZE) + push_hyp_memcache(mc, start, hyp_virt_to_phys); + + WARN_ON(__pkvm_hyp_donate_host(pfn, nr_pages)); +} + int __pkvm_teardown_shadow(int shadow_handle) { struct kvm_hyp_memcache *mc; struct kvm_shadow_vm *vm; struct kvm *host_kvm; - size_t shadow_size; + unsigned int nr_vcpus; int err; - u64 pfn; - u64 nr_pages; - void *addr; int i; /* Lookup then remove entry from the shadow table. */ @@ -764,6 +828,9 @@ int __pkvm_teardown_shadow(int shadow_handle) goto err_unlock; } + host_kvm = vm->host_kvm; + nr_vcpus = vm->nr_vcpus; + /* * Clear the tracking for last_loaded_vcpu for all cpus for this vm in * case the same addresses for those vcpus are reused for future vms. @@ -783,22 +850,17 @@ int __pkvm_teardown_shadow(int shadow_handle) hyp_spin_unlock(&shadow_lock); /* Reclaim guest pages, and page-table pages */ - mc = &vm->host_kvm->arch.pkvm.teardown_mc; + mc = &host_kvm->arch.pkvm.teardown_mc; reclaim_guest_pages(vm, mc); - drain_shadow_vcpus(vm->shadow_vcpus, vm->created_vcpus, mc); - unpin_host_vcpus(vm->shadow_vcpus, vm->created_vcpus); + drain_shadow_vcpus(vm->shadow_vcpus, nr_vcpus, mc); + unpin_host_vcpus(vm->shadow_vcpus, nr_vcpus); + + for (i = 0; i < nr_vcpus; i++) + teardown_donated_memory(mc, vm->shadow_vcpus[i], + PAGE_ALIGN(sizeof(vm->shadow_vcpus[i]))); + teardown_donated_memory(mc, vm, vm->shadow_area_size); - /* Push the metadata pages to the teardown memcache */ - shadow_size = vm->shadow_area_size; - host_kvm = vm->host_kvm; - memset(vm, 0, shadow_size); - for (addr = vm; addr < ((void *)vm + shadow_size); addr += PAGE_SIZE) - push_hyp_memcache(mc, addr, hyp_virt_to_phys); hyp_unpin_shared_mem(host_kvm, host_kvm + 1); - - pfn = hyp_phys_to_pfn(__hyp_pa(vm)); - nr_pages = shadow_size >> PAGE_SHIFT; - WARN_ON(__pkvm_hyp_donate_host(pfn, nr_pages)); return 0; err_unlock: @@ -919,7 +981,7 @@ struct kvm_vcpu *pvm_mpidr_to_vcpu(struct kvm_shadow_vm *vm, unsigned long mpidr mpidr &= MPIDR_HWID_BITMASK; - for (i = 0; i < vm->created_vcpus; i++) { + for (i = 0; i < READ_ONCE(vm->nr_vcpus); i++) { vcpu = vm->vcpus[i]; if (mpidr == kvm_vcpu_get_mpidr_aff(vcpu)) @@ -1036,7 +1098,7 @@ static bool pvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu) * then if at least one is PENDING_ON then return PENDING_ON. * Otherwise, return OFF. */ - for (i = 0; i < vm->created_vcpus; i++) { + for (i = 0; i < READ_ONCE(vm->nr_vcpus); i++) { tmp = vm->vcpus[i]; mpidr = kvm_vcpu_get_mpidr_aff(tmp); @@ -1246,7 +1308,7 @@ out_guest_err: static bool pkvm_install_ioguard_page(struct kvm_vcpu *vcpu, u64 *exit_code) { - u32 retval = SMCCC_RET_SUCCESS; + u64 retval = SMCCC_RET_SUCCESS; u64 ipa = smccc_get_arg1(vcpu); int ret; @@ -1338,6 +1400,8 @@ bool kvm_handle_pvm_hvc64(struct kvm_vcpu *vcpu, u64 *exit_code) return pkvm_install_ioguard_page(vcpu, exit_code); case ARM_SMCCC_VENDOR_HYP_KVM_MMIO_GUARD_UNMAP_FUNC_ID: if (__pkvm_remove_ioguard_page(vcpu, vcpu_get_reg(vcpu, 1))) + val[0] = SMCCC_RET_INVALID_PARAMETER; + else val[0] = SMCCC_RET_SUCCESS; break; case ARM_SMCCC_VENDOR_HYP_KVM_MMIO_GUARD_INFO_FUNC_ID: diff --git a/arch/arm64/kvm/hyp/nvhe/psci-relay.c b/arch/arm64/kvm/hyp/nvhe/psci-relay.c index 85f7cad956d5..81aa189d4ca6 100644 --- a/arch/arm64/kvm/hyp/nvhe/psci-relay.c +++ b/arch/arm64/kvm/hyp/nvhe/psci-relay.c @@ -235,7 +235,7 @@ static u64 psci_mem_protect(s64 offset) return cnt; if (!cnt || !new) - psci_call(PSCI_1_1_FN64_MEM_PROTECT, offset < 0 ? 0 : 1, 0, 0); + psci_call(PSCI_1_1_FN_MEM_PROTECT, offset < 0 ? 0 : 1, 0, 0); cnt = new; return cnt; @@ -307,10 +307,8 @@ static unsigned long psci_1_0_handler(u64 func_id, struct kvm_cpu_context *host_ case PSCI_1_1_FN64_SYSTEM_RESET2: pkvm_clear_pvmfw_pages(); hyp_spin_lock(&mem_protect_lock); - if (psci_mem_protect_active()) { - return psci_0_2_handler(PSCI_0_2_FN_SYSTEM_RESET, - host_ctxt); - } + if (psci_mem_protect_active()) + cpu_reg(host_ctxt, 0) = PSCI_0_2_FN_SYSTEM_RESET; fallthrough; case PSCI_1_0_FN_PSCI_FEATURES: case PSCI_1_0_FN_SET_SUSPEND_MODE: diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c index 283bac5c2959..1f069b6439c8 100644 --- a/arch/arm64/kvm/hyp/nvhe/switch.c +++ b/arch/arm64/kvm/hyp/nvhe/switch.c @@ -120,7 +120,7 @@ static void __hyp_vgic_restore_state(struct kvm_vcpu *vcpu) } } -/** +/* * Disable host events, enable guest events */ static bool __pmu_switch_to_guest(struct kvm_cpu_context *host_ctxt) @@ -140,7 +140,7 @@ static bool __pmu_switch_to_guest(struct kvm_cpu_context *host_ctxt) return (pmu->events_host || pmu->events_guest); } -/** +/* * Disable guest events, enable host events */ static void __pmu_switch_to_host(struct kvm_cpu_context *host_ctxt) @@ -158,7 +158,7 @@ static void __pmu_switch_to_host(struct kvm_cpu_context *host_ctxt) write_sysreg(pmu->events_host, pmcntenset_el0); } -/** +/* * Handler for protected VM MSR, MRS or System instruction execution in AArch64. * * Returns true if the hypervisor has handled the exit, and control should go @@ -175,7 +175,7 @@ static bool kvm_handle_pvm_sys64(struct kvm_vcpu *vcpu, u64 *exit_code) kvm_handle_pvm_sysreg(vcpu, exit_code)); } -/** +/* * Handler for protected floating-point and Advanced SIMD accesses. * * Returns true if the hypervisor has handled the exit, and control should go diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index f7f6b08dd371..e335c9b2c00a 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -120,8 +120,8 @@ void __init kvm_hyp_reserve(void) */ static int __create_el2_shadow(struct kvm *kvm) { - struct kvm_vcpu *vcpu, **vcpu_array; - size_t pgd_sz, shadow_sz; + struct kvm_vcpu *vcpu; + size_t pgd_sz, shadow_sz, vcpu_state_sz; void *pgd, *shadow_addr; unsigned long idx; int shadow_handle; @@ -140,21 +140,15 @@ static int __create_el2_shadow(struct kvm *kvm) if (!pgd) return -ENOMEM; - /* Allocate memory to donate to hyp for the kvm and vcpu state. */ + /* Allocate memory to donate to hyp for the kvm and vcpu state pointers. */ shadow_sz = PAGE_ALIGN(KVM_SHADOW_VM_SIZE + - SHADOW_VCPU_STATE_SIZE * kvm->created_vcpus); + sizeof(void *) * kvm->created_vcpus); shadow_addr = alloc_pages_exact(shadow_sz, GFP_KERNEL_ACCOUNT); if (!shadow_addr) { ret = -ENOMEM; goto free_pgd; } - /* Stash the vcpu pointers into the PGD */ - BUILD_BUG_ON(KVM_MAX_VCPUS > (PAGE_SIZE / sizeof(u64))); - vcpu_array = pgd; - kvm_for_each_vcpu(idx, vcpu, kvm) - vcpu_array[idx] = vcpu; - /* Donate the shadow memory to hyp and let hyp initialize it. */ ret = kvm_call_hyp_nvhe(__pkvm_init_shadow, kvm, shadow_addr, shadow_sz, pgd); @@ -166,8 +160,35 @@ static int __create_el2_shadow(struct kvm *kvm) /* Store the shadow handle given by hyp for future call reference. */ kvm->arch.pkvm.shadow_handle = shadow_handle; + /* Donate memory for the vcpu state at hyp and initialize it. */ + vcpu_state_sz = PAGE_ALIGN(SHADOW_VCPU_STATE_SIZE); + kvm_for_each_vcpu (idx, vcpu, kvm) { + void *vcpu_state; + + /* Indexing of the vcpus to be sequential starting at 0. */ + if (WARN_ON(vcpu->vcpu_idx != idx)) { + ret = -EINVAL; + goto destroy_vm; + } + vcpu_state = alloc_pages_exact(vcpu_state_sz, GFP_KERNEL_ACCOUNT); + if (!vcpu_state) { + ret = -ENOMEM; + goto destroy_vm; + } + + ret = kvm_call_hyp_nvhe(__pkvm_init_shadow_vcpu, shadow_handle, + vcpu, vcpu_state); + if (ret) { + free_pages_exact(vcpu_state, vcpu_state_sz); + goto destroy_vm; + } + } + return 0; +destroy_vm: + kvm_shadow_destroy(kvm); + return ret; free_shadow: free_pages_exact(shadow_addr, shadow_sz); free_pgd: diff --git a/arch/arm64/kvm/psci.c b/arch/arm64/kvm/psci.c index 371ebbc6b529..2208bac03611 100644 --- a/arch/arm64/kvm/psci.c +++ b/arch/arm64/kvm/psci.c @@ -188,15 +188,11 @@ static void kvm_psci_system_reset2(struct kvm_vcpu *vcpu) static unsigned long kvm_psci_check_allowed_function(struct kvm_vcpu *vcpu, u32 fn) { - switch(fn) { - case PSCI_0_2_FN64_CPU_SUSPEND: - case PSCI_0_2_FN64_CPU_ON: - case PSCI_0_2_FN64_AFFINITY_INFO: - /* Disallow these functions for 32bit guests */ - if (vcpu_mode_is_32bit(vcpu)) - return PSCI_RET_NOT_SUPPORTED; - break; - } + /* + * Prevent 32 bit guests from calling 64 bit PSCI functions. + */ + if ((fn & PSCI_0_2_64BIT) && vcpu_mode_is_32bit(vcpu)) + return PSCI_RET_NOT_SUPPORTED; return 0; } @@ -208,10 +204,6 @@ static int kvm_psci_0_2_call(struct kvm_vcpu *vcpu) unsigned long val; int ret = 1; - val = kvm_psci_check_allowed_function(vcpu, psci_fn); - if (val) - goto out; - switch (psci_fn) { case PSCI_0_2_FN_PSCI_VERSION: /* @@ -279,7 +271,6 @@ static int kvm_psci_0_2_call(struct kvm_vcpu *vcpu) break; } -out: smccc_set_retval(vcpu, val, 0, 0, 0); return ret; } @@ -399,6 +390,15 @@ static int kvm_psci_0_1_call(struct kvm_vcpu *vcpu) */ int kvm_psci_call(struct kvm_vcpu *vcpu) { + u32 psci_fn = smccc_get_function(vcpu); + unsigned long val; + + val = kvm_psci_check_allowed_function(vcpu, psci_fn); + if (val) { + smccc_set_retval(vcpu, val, 0, 0, 0); + return 1; + } + switch (kvm_psci_version(vcpu, vcpu->kvm)) { case KVM_ARM_PSCI_1_1: return kvm_psci_1_x_call(vcpu, 1); diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 95b5caad48e9..978feb4c35d0 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -1048,8 +1048,7 @@ static u64 read_id_reg(const struct kvm_vcpu *vcpu, val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_CSV2), (u64)vcpu->kvm->arch.pfr0_csv2); val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_CSV3); val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_CSV3), (u64)vcpu->kvm->arch.pfr0_csv3); - if (irqchip_in_kernel(vcpu->kvm) && - vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) { + if (kvm_vgic_global_state.type == VGIC_V3) { val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_GIC); val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_GIC), 1); } diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index b7d3f944520e..22aa1b0f8b61 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -76,7 +76,7 @@ config X86 select ARCH_HAS_PMEM_API if X86_64 select ARCH_HAS_PTE_DEVMAP if X86_64 select ARCH_HAS_PTE_SPECIAL - select ARCH_HAS_NONLEAF_PMD_YOUNG + select ARCH_HAS_NONLEAF_PMD_YOUNG if PGTABLE_LEVELS > 2 select ARCH_HAS_UACCESS_FLUSHCACHE if X86_64 select ARCH_HAS_COPY_MC if X86_64 select ARCH_HAS_SET_MEMORY diff --git a/arch/x86/configs/gki_defconfig b/arch/x86/configs/gki_defconfig index f240feaf7672..0c7b7bbd811a 100644 --- a/arch/x86/configs/gki_defconfig +++ b/arch/x86/configs/gki_defconfig @@ -462,6 +462,7 @@ CONFIG_USB_CONFIGFS_F_ACC=y CONFIG_USB_CONFIGFS_F_AUDIO_SRC=y CONFIG_USB_CONFIGFS_F_MIDI=y CONFIG_USB_CONFIGFS_F_HID=y +CONFIG_USB_CONFIGFS_F_UVC=y CONFIG_TYPEC=y CONFIG_TYPEC_TCPM=y CONFIG_TYPEC_TCPCI=y diff --git a/block/mq-deadline.c b/block/mq-deadline.c index 6556aa2182a5..2d488228a73e 100644 --- a/block/mq-deadline.c +++ b/block/mq-deadline.c @@ -737,6 +737,7 @@ static void dd_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, if (at_head) { list_add(&rq->queuelist, &per_prio->dispatch); + rq->fifo_time = jiffies; } else { deadline_add_rq_rb(per_prio, rq); diff --git a/build.config.gki.aarch64 b/build.config.gki.aarch64 index dde70d69dd33..a571f19915a1 100644 --- a/build.config.gki.aarch64 +++ b/build.config.gki.aarch64 @@ -13,11 +13,13 @@ KMI_SYMBOL_LIST=android/abi_gki_aarch64 ADDITIONAL_KMI_SYMBOL_LISTS=" android/abi_gki_aarch64_type_visibility android/abi_gki_aarch64_core +android/abi_gki_aarch64_exynos android/abi_gki_aarch64_fips140 android/abi_gki_aarch64_generic android/abi_gki_aarch64_virtual_device android/abi_gki_aarch64_db845c android/abi_gki_aarch64_hikey960 +android/abi_gki_aarch64_rockchip " FILES="${FILES} diff --git a/build.config.rockchip b/build.config.rockchip new file mode 100644 index 000000000000..297c4c95a14b --- /dev/null +++ b/build.config.rockchip @@ -0,0 +1,8 @@ +. ${ROOT_DIR}/${KERNEL_DIR}/build.config.gki.aarch64 + + +DEFCONFIG=rockchip_aarch64_gki_defconfig +KMI_SYMBOL_LIST=android/abi_gki_aarch64_rockchip +PRE_DEFCONFIG_CMDS="KCONFIG_CONFIG=${ROOT_DIR}/common/arch/arm64/configs/${DEFCONFIG} ${ROOT_DIR}/common/scripts/kconfig/merge_config.sh -m -r ${ROOT_DIR}/common/arch/arm64/configs/gki_defconfig ${ROOT_DIR}/common/arch/arm64/configs/rockchip_gki.fragment" +POST_DEFCONFIG_CMDS="rm ${ROOT_DIR}/common/arch/arm64/configs/${DEFCONFIG}" + diff --git a/crypto/Makefile b/crypto/Makefile index 0f3f4813ba8c..2785e5fab9e7 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -211,7 +211,7 @@ UBSAN_SANITIZE_jitterentropy-fips.o = n # module that is in scope for FIPS 140-2 certification # crypto-fips-objs := drbg.o ecb.o cbc.o ctr.o cts.o gcm.o xts.o hmac.o cmac.o \ - memneq.o gf128mul.o aes_generic.o lib-crypto-aes.o \ + gf128mul.o aes_generic.o lib-crypto-aes.o \ jitterentropy.o jitterentropy-kcapi.o \ sha1_generic.o sha256_generic.o sha512_generic.o \ lib-sha1.o lib-crypto-sha256.o diff --git a/crypto/jitterentropy.c b/crypto/jitterentropy.c index 37c4c308339e..423c55d0e165 100644 --- a/crypto/jitterentropy.c +++ b/crypto/jitterentropy.c @@ -117,6 +117,22 @@ struct rand_data { #define JENT_EHEALTH 9 /* Health test failed during initialization */ #define JENT_ERCT 10 /* RCT failed during initialization */ +/* + * The output n bits can receive more than n bits of min entropy, of course, + * but the fixed output of the conditioning function can only asymptotically + * approach the output size bits of min entropy, not attain that bound. Random + * maps will tend to have output collisions, which reduces the creditable + * output entropy (that is what SP 800-90B Section 3.1.5.1.2 attempts to bound). + * + * The value "64" is justified in Appendix A.4 of the current 90C draft, + * and aligns with NIST's in "epsilon" definition in this document, which is + * that a string can be considered "full entropy" if you can bound the min + * entropy in each bit of output to at least 1-epsilon, where epsilon is + * required to be <= 2^(-32). + */ +#define JENT_ENTROPY_SAFETY_FACTOR 64 + +#include #include "jitterentropy.h" /*************************************************************************** @@ -546,7 +562,10 @@ static int jent_measure_jitter(struct rand_data *ec) */ static void jent_gen_entropy(struct rand_data *ec) { - unsigned int k = 0; + unsigned int k = 0, safety_factor = 0; + + if (fips_enabled) + safety_factor = JENT_ENTROPY_SAFETY_FACTOR; /* priming of the ->prev_time value */ jent_measure_jitter(ec); @@ -560,7 +579,7 @@ static void jent_gen_entropy(struct rand_data *ec) * We multiply the loop value with ->osr to obtain the * oversampling rate requested by the caller */ - if (++k >= (DATA_SIZE_BITS * ec->osr)) + if (++k >= ((DATA_SIZE_BITS + safety_factor) * ec->osr)) break; } } diff --git a/drivers/android/binder.c b/drivers/android/binder.c index 07330027e6c5..f0d8989a2130 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -1813,15 +1813,21 @@ static void binder_cleanup_transaction(struct binder_transaction *t, /** * binder_get_object() - gets object and checks for valid metadata * @proc: binder_proc owning the buffer + * @u: sender's user pointer to base of buffer * @buffer: binder_buffer that we're parsing. * @offset: offset in the @buffer at which to validate an object. * @object: struct binder_object to read into * - * Return: If there's a valid metadata object at @offset in @buffer, the + * Copy the binder object at the given offset into @object. If @u is + * provided then the copy is from the sender's buffer. If not, then + * it is copied from the target's @buffer. + * + * Return: If there's a valid metadata object at @offset, the * size of that object. Otherwise, it returns zero. The object * is read into the struct binder_object pointed to by @object. */ static size_t binder_get_object(struct binder_proc *proc, + const void __user *u, struct binder_buffer *buffer, unsigned long offset, struct binder_object *object) @@ -1831,10 +1837,16 @@ static size_t binder_get_object(struct binder_proc *proc, size_t object_size = 0; read_size = min_t(size_t, sizeof(*object), buffer->data_size - offset); - if (offset > buffer->data_size || read_size < sizeof(*hdr) || - binder_alloc_copy_from_buffer(&proc->alloc, object, buffer, - offset, read_size)) + if (offset > buffer->data_size || read_size < sizeof(*hdr)) return 0; + if (u) { + if (copy_from_user(object, u + offset, read_size)) + return 0; + } else { + if (binder_alloc_copy_from_buffer(&proc->alloc, object, buffer, + offset, read_size)) + return 0; + } /* Ok, now see if we read a complete object. */ hdr = &object->hdr; @@ -1907,7 +1919,7 @@ static struct binder_buffer_object *binder_validate_ptr( b, buffer_offset, sizeof(object_offset))) return NULL; - object_size = binder_get_object(proc, b, object_offset, object); + object_size = binder_get_object(proc, NULL, b, object_offset, object); if (!object_size || object->hdr.type != BINDER_TYPE_PTR) return NULL; if (object_offsetp) @@ -1972,7 +1984,8 @@ static bool binder_validate_fixup(struct binder_proc *proc, unsigned long buffer_offset; struct binder_object last_object; struct binder_buffer_object *last_bbo; - size_t object_size = binder_get_object(proc, b, last_obj_offset, + size_t object_size = binder_get_object(proc, NULL, b, + last_obj_offset, &last_object); if (object_size != sizeof(*last_bbo)) return false; @@ -2087,7 +2100,7 @@ static void binder_transaction_buffer_release(struct binder_proc *proc, if (!binder_alloc_copy_from_buffer(&proc->alloc, &object_offset, buffer, buffer_offset, sizeof(object_offset))) - object_size = binder_get_object(proc, buffer, + object_size = binder_get_object(proc, NULL, buffer, object_offset, &object); if (object_size == 0) { pr_err("transaction release %d bad object at offset %lld, size %zd\n", @@ -2425,16 +2438,266 @@ err_fd_not_accepted: return ret; } -static int binder_translate_fd_array(struct binder_fd_array_object *fda, +/** + * struct binder_ptr_fixup - data to be fixed-up in target buffer + * @offset offset in target buffer to fixup + * @skip_size bytes to skip in copy (fixup will be written later) + * @fixup_data data to write at fixup offset + * @node list node + * + * This is used for the pointer fixup list (pf) which is created and consumed + * during binder_transaction() and is only accessed locally. No + * locking is necessary. + * + * The list is ordered by @offset. + */ +struct binder_ptr_fixup { + binder_size_t offset; + size_t skip_size; + binder_uintptr_t fixup_data; + struct list_head node; +}; + +/** + * struct binder_sg_copy - scatter-gather data to be copied + * @offset offset in target buffer + * @sender_uaddr user address in source buffer + * @length bytes to copy + * @node list node + * + * This is used for the sg copy list (sgc) which is created and consumed + * during binder_transaction() and is only accessed locally. No + * locking is necessary. + * + * The list is ordered by @offset. + */ +struct binder_sg_copy { + binder_size_t offset; + const void __user *sender_uaddr; + size_t length; + struct list_head node; +}; + +/** + * binder_do_deferred_txn_copies() - copy and fixup scatter-gather data + * @alloc: binder_alloc associated with @buffer + * @buffer: binder buffer in target process + * @sgc_head: list_head of scatter-gather copy list + * @pf_head: list_head of pointer fixup list + * + * Processes all elements of @sgc_head, applying fixups from @pf_head + * and copying the scatter-gather data from the source process' user + * buffer to the target's buffer. It is expected that the list creation + * and processing all occurs during binder_transaction() so these lists + * are only accessed in local context. + * + * Return: 0=success, else -errno + */ +static int binder_do_deferred_txn_copies(struct binder_alloc *alloc, + struct binder_buffer *buffer, + struct list_head *sgc_head, + struct list_head *pf_head) +{ + int ret = 0; + struct binder_sg_copy *sgc, *tmpsgc; + struct binder_ptr_fixup *tmppf; + struct binder_ptr_fixup *pf = + list_first_entry_or_null(pf_head, struct binder_ptr_fixup, + node); + + list_for_each_entry_safe(sgc, tmpsgc, sgc_head, node) { + size_t bytes_copied = 0; + + while (bytes_copied < sgc->length) { + size_t copy_size; + size_t bytes_left = sgc->length - bytes_copied; + size_t offset = sgc->offset + bytes_copied; + + /* + * We copy up to the fixup (pointed to by pf) + */ + copy_size = pf ? min(bytes_left, (size_t)pf->offset - offset) + : bytes_left; + if (!ret && copy_size) + ret = binder_alloc_copy_user_to_buffer( + alloc, buffer, + offset, + sgc->sender_uaddr + bytes_copied, + copy_size); + bytes_copied += copy_size; + if (copy_size != bytes_left) { + BUG_ON(!pf); + /* we stopped at a fixup offset */ + if (pf->skip_size) { + /* + * we are just skipping. This is for + * BINDER_TYPE_FDA where the translated + * fds will be fixed up when we get + * to target context. + */ + bytes_copied += pf->skip_size; + } else { + /* apply the fixup indicated by pf */ + if (!ret) + ret = binder_alloc_copy_to_buffer( + alloc, buffer, + pf->offset, + &pf->fixup_data, + sizeof(pf->fixup_data)); + bytes_copied += sizeof(pf->fixup_data); + } + list_del(&pf->node); + kfree(pf); + pf = list_first_entry_or_null(pf_head, + struct binder_ptr_fixup, node); + } + } + list_del(&sgc->node); + kfree(sgc); + } + list_for_each_entry_safe(pf, tmppf, pf_head, node) { + BUG_ON(pf->skip_size == 0); + list_del(&pf->node); + kfree(pf); + } + BUG_ON(!list_empty(sgc_head)); + + return ret > 0 ? -EINVAL : ret; +} + +/** + * binder_cleanup_deferred_txn_lists() - free specified lists + * @sgc_head: list_head of scatter-gather copy list + * @pf_head: list_head of pointer fixup list + * + * Called to clean up @sgc_head and @pf_head if there is an + * error. + */ +static void binder_cleanup_deferred_txn_lists(struct list_head *sgc_head, + struct list_head *pf_head) +{ + struct binder_sg_copy *sgc, *tmpsgc; + struct binder_ptr_fixup *pf, *tmppf; + + list_for_each_entry_safe(sgc, tmpsgc, sgc_head, node) { + list_del(&sgc->node); + kfree(sgc); + } + list_for_each_entry_safe(pf, tmppf, pf_head, node) { + list_del(&pf->node); + kfree(pf); + } +} + +/** + * binder_defer_copy() - queue a scatter-gather buffer for copy + * @sgc_head: list_head of scatter-gather copy list + * @offset: binder buffer offset in target process + * @sender_uaddr: user address in source process + * @length: bytes to copy + * + * Specify a scatter-gather block to be copied. The actual copy must + * be deferred until all the needed fixups are identified and queued. + * Then the copy and fixups are done together so un-translated values + * from the source are never visible in the target buffer. + * + * We are guaranteed that repeated calls to this function will have + * monotonically increasing @offset values so the list will naturally + * be ordered. + * + * Return: 0=success, else -errno + */ +static int binder_defer_copy(struct list_head *sgc_head, binder_size_t offset, + const void __user *sender_uaddr, size_t length) +{ + struct binder_sg_copy *bc = kzalloc(sizeof(*bc), GFP_KERNEL); + + if (!bc) + return -ENOMEM; + + bc->offset = offset; + bc->sender_uaddr = sender_uaddr; + bc->length = length; + INIT_LIST_HEAD(&bc->node); + + /* + * We are guaranteed that the deferred copies are in-order + * so just add to the tail. + */ + list_add_tail(&bc->node, sgc_head); + + return 0; +} + +/** + * binder_add_fixup() - queue a fixup to be applied to sg copy + * @pf_head: list_head of binder ptr fixup list + * @offset: binder buffer offset in target process + * @fixup: bytes to be copied for fixup + * @skip_size: bytes to skip when copying (fixup will be applied later) + * + * Add the specified fixup to a list ordered by @offset. When copying + * the scatter-gather buffers, the fixup will be copied instead of + * data from the source buffer. For BINDER_TYPE_FDA fixups, the fixup + * will be applied later (in target process context), so we just skip + * the bytes specified by @skip_size. If @skip_size is 0, we copy the + * value in @fixup. + * + * This function is called *mostly* in @offset order, but there are + * exceptions. Since out-of-order inserts are relatively uncommon, + * we insert the new element by searching backward from the tail of + * the list. + * + * Return: 0=success, else -errno + */ +static int binder_add_fixup(struct list_head *pf_head, binder_size_t offset, + binder_uintptr_t fixup, size_t skip_size) +{ + struct binder_ptr_fixup *pf = kzalloc(sizeof(*pf), GFP_KERNEL); + struct binder_ptr_fixup *tmppf; + + if (!pf) + return -ENOMEM; + + pf->offset = offset; + pf->fixup_data = fixup; + pf->skip_size = skip_size; + INIT_LIST_HEAD(&pf->node); + + /* Fixups are *mostly* added in-order, but there are some + * exceptions. Look backwards through list for insertion point. + */ + list_for_each_entry_reverse(tmppf, pf_head, node) { + if (tmppf->offset < pf->offset) { + list_add(&pf->node, &tmppf->node); + return 0; + } + } + /* + * if we get here, then the new offset is the lowest so + * insert at the head + */ + list_add(&pf->node, pf_head); + return 0; +} + +static int binder_translate_fd_array(struct list_head *pf_head, + struct binder_fd_array_object *fda, + const void __user *sender_ubuffer, struct binder_buffer_object *parent, + struct binder_buffer_object *sender_uparent, struct binder_transaction *t, struct binder_thread *thread, struct binder_transaction *in_reply_to) { binder_size_t fdi, fd_buf_size; binder_size_t fda_offset; + const void __user *sender_ufda_base; struct binder_proc *proc = thread->proc; - struct binder_proc *target_proc = t->to_proc; + int ret; + + if (fda->num_fds == 0) + return 0; fd_buf_size = sizeof(u32) * fda->num_fds; if (fda->num_fds >= SIZE_MAX / sizeof(u32)) { @@ -2458,19 +2721,25 @@ static int binder_translate_fd_array(struct binder_fd_array_object *fda, */ fda_offset = (parent->buffer - (uintptr_t)t->buffer->user_data) + fda->parent_offset; - if (!IS_ALIGNED((unsigned long)fda_offset, sizeof(u32))) { + sender_ufda_base = (void __user *)(uintptr_t)sender_uparent->buffer + + fda->parent_offset; + + if (!IS_ALIGNED((unsigned long)fda_offset, sizeof(u32)) || + !IS_ALIGNED((unsigned long)sender_ufda_base, sizeof(u32))) { binder_user_error("%d:%d parent offset not aligned correctly.\n", proc->pid, thread->pid); return -EINVAL; } + ret = binder_add_fixup(pf_head, fda_offset, 0, fda->num_fds * sizeof(u32)); + if (ret) + return ret; + for (fdi = 0; fdi < fda->num_fds; fdi++) { u32 fd; - int ret; binder_size_t offset = fda_offset + fdi * sizeof(fd); + binder_size_t sender_uoffset = fdi * sizeof(fd); - ret = binder_alloc_copy_from_buffer(&target_proc->alloc, - &fd, t->buffer, - offset, sizeof(fd)); + ret = copy_from_user(&fd, sender_ufda_base + sender_uoffset, sizeof(fd)); if (!ret) ret = binder_translate_fd(fd, offset, t, thread, in_reply_to); @@ -2480,7 +2749,8 @@ static int binder_translate_fd_array(struct binder_fd_array_object *fda, return 0; } -static int binder_fixup_parent(struct binder_transaction *t, +static int binder_fixup_parent(struct list_head *pf_head, + struct binder_transaction *t, struct binder_thread *thread, struct binder_buffer_object *bp, binder_size_t off_start_offset, @@ -2526,14 +2796,7 @@ static int binder_fixup_parent(struct binder_transaction *t, } buffer_offset = bp->parent_offset + (uintptr_t)parent->buffer - (uintptr_t)b->user_data; - if (binder_alloc_copy_to_buffer(&target_proc->alloc, b, buffer_offset, - &bp->buffer, sizeof(bp->buffer))) { - binder_user_error("%d:%d got transaction with invalid parent offset\n", - proc->pid, thread->pid); - return -EINVAL; - } - - return 0; + return binder_add_fixup(pf_head, buffer_offset, bp->buffer, 0); } /** @@ -2744,6 +3007,7 @@ static void binder_transaction(struct binder_proc *proc, binder_size_t off_start_offset, off_end_offset; binder_size_t off_min; binder_size_t sg_buf_offset, sg_buf_end_offset; + binder_size_t user_offset = 0; struct binder_proc *target_proc = NULL; struct binder_thread *target_thread = NULL; struct binder_node *target_node = NULL; @@ -2759,6 +3023,12 @@ static void binder_transaction(struct binder_proc *proc, char *secctx = NULL; u32 secctx_sz = 0; bool is_nested = false; + struct list_head sgc_head; + struct list_head pf_head; + const void __user *user_buffer = (const void __user *) + (uintptr_t)tr->data.ptr.buffer; + INIT_LIST_HEAD(&sgc_head); + INIT_LIST_HEAD(&pf_head); e = binder_transaction_log_add(&binder_transaction_log); e->debug_id = t_debug_id; @@ -3101,19 +3371,6 @@ static void binder_transaction(struct binder_proc *proc, t->buffer->clear_on_free = !!(t->flags & TF_CLEAR_BUF); trace_binder_transaction_alloc_buf(t->buffer); - if (binder_alloc_copy_user_to_buffer( - &target_proc->alloc, - t->buffer, 0, - (const void __user *) - (uintptr_t)tr->data.ptr.buffer, - tr->data_size)) { - binder_user_error("%d:%d got transaction with invalid data ptr\n", - proc->pid, thread->pid); - return_error = BR_FAILED_REPLY; - return_error_param = -EFAULT; - return_error_line = __LINE__; - goto err_copy_data_failed; - } if (binder_alloc_copy_user_to_buffer( &target_proc->alloc, t->buffer, @@ -3158,6 +3415,7 @@ static void binder_transaction(struct binder_proc *proc, size_t object_size; struct binder_object object; binder_size_t object_offset; + binder_size_t copy_size; if (binder_alloc_copy_from_buffer(&target_proc->alloc, &object_offset, @@ -3169,8 +3427,27 @@ static void binder_transaction(struct binder_proc *proc, return_error_line = __LINE__; goto err_bad_offset; } - object_size = binder_get_object(target_proc, t->buffer, - object_offset, &object); + + /* + * Copy the source user buffer up to the next object + * that will be processed. + */ + copy_size = object_offset - user_offset; + if (copy_size && (user_offset > object_offset || + binder_alloc_copy_user_to_buffer( + &target_proc->alloc, + t->buffer, user_offset, + user_buffer + user_offset, + copy_size))) { + binder_user_error("%d:%d got transaction with invalid data ptr\n", + proc->pid, thread->pid); + return_error = BR_FAILED_REPLY; + return_error_param = -EFAULT; + return_error_line = __LINE__; + goto err_copy_data_failed; + } + object_size = binder_get_object(target_proc, user_buffer, + t->buffer, object_offset, &object); if (object_size == 0 || object_offset < off_min) { binder_user_error("%d:%d got transaction with invalid offset (%lld, min %lld max %lld) or object.\n", proc->pid, thread->pid, @@ -3182,6 +3459,11 @@ static void binder_transaction(struct binder_proc *proc, return_error_line = __LINE__; goto err_bad_offset; } + /* + * Set offset to the next buffer fragment to be + * copied + */ + user_offset = object_offset + object_size; hdr = &object.hdr; off_min = object_offset + object_size; @@ -3244,6 +3526,8 @@ static void binder_transaction(struct binder_proc *proc, case BINDER_TYPE_FDA: { struct binder_object ptr_object; binder_size_t parent_offset; + struct binder_object user_object; + size_t user_parent_size; struct binder_fd_array_object *fda = to_binder_fd_array_object(hdr); size_t num_valid = (buffer_offset - off_start_offset) / @@ -3275,11 +3559,35 @@ static void binder_transaction(struct binder_proc *proc, return_error_line = __LINE__; goto err_bad_parent; } - ret = binder_translate_fd_array(fda, parent, t, thread, - in_reply_to); - if (ret < 0) { + /* + * We need to read the user version of the parent + * object to get the original user offset + */ + user_parent_size = + binder_get_object(proc, user_buffer, t->buffer, + parent_offset, &user_object); + if (user_parent_size != sizeof(user_object.bbo)) { + binder_user_error("%d:%d invalid ptr object size: %zd vs %zd\n", + proc->pid, thread->pid, + user_parent_size, + sizeof(user_object.bbo)); return_error = BR_FAILED_REPLY; - return_error_param = ret; + return_error_param = -EINVAL; + return_error_line = __LINE__; + goto err_bad_parent; + } + ret = binder_translate_fd_array(&pf_head, fda, + user_buffer, parent, + &user_object.bbo, t, + thread, in_reply_to); + if (!ret) + ret = binder_alloc_copy_to_buffer(&target_proc->alloc, + t->buffer, + object_offset, + fda, sizeof(*fda)); + if (ret) { + return_error = BR_FAILED_REPLY; + return_error_param = ret > 0 ? -EINVAL : ret; return_error_line = __LINE__; goto err_translate_failed; } @@ -3301,19 +3609,14 @@ static void binder_transaction(struct binder_proc *proc, return_error_line = __LINE__; goto err_bad_offset; } - if (binder_alloc_copy_user_to_buffer( - &target_proc->alloc, - t->buffer, - sg_buf_offset, - (const void __user *) - (uintptr_t)bp->buffer, - bp->length)) { - binder_user_error("%d:%d got transaction with invalid offsets ptr\n", - proc->pid, thread->pid); - return_error_param = -EFAULT; + ret = binder_defer_copy(&sgc_head, sg_buf_offset, + (const void __user *)(uintptr_t)bp->buffer, + bp->length); + if (ret) { return_error = BR_FAILED_REPLY; + return_error_param = ret; return_error_line = __LINE__; - goto err_copy_data_failed; + goto err_translate_failed; } /* Fixup buffer pointer to target proc address space */ bp->buffer = (uintptr_t) @@ -3322,7 +3625,8 @@ static void binder_transaction(struct binder_proc *proc, num_valid = (buffer_offset - off_start_offset) / sizeof(binder_size_t); - ret = binder_fixup_parent(t, thread, bp, + ret = binder_fixup_parent(&pf_head, t, + thread, bp, off_start_offset, num_valid, last_fixup_obj_off, @@ -3349,6 +3653,30 @@ static void binder_transaction(struct binder_proc *proc, goto err_bad_object_type; } } + /* Done processing objects, copy the rest of the buffer */ + if (binder_alloc_copy_user_to_buffer( + &target_proc->alloc, + t->buffer, user_offset, + user_buffer + user_offset, + tr->data_size - user_offset)) { + binder_user_error("%d:%d got transaction with invalid data ptr\n", + proc->pid, thread->pid); + return_error = BR_FAILED_REPLY; + return_error_param = -EFAULT; + return_error_line = __LINE__; + goto err_copy_data_failed; + } + + ret = binder_do_deferred_txn_copies(&target_proc->alloc, t->buffer, + &sgc_head, &pf_head); + if (ret) { + binder_user_error("%d:%d got transaction with invalid offsets ptr\n", + proc->pid, thread->pid); + return_error = BR_FAILED_REPLY; + return_error_param = ret; + return_error_line = __LINE__; + goto err_copy_data_failed; + } if (t->buffer->oneway_spam_suspect) tcomplete->type = BINDER_WORK_TRANSACTION_ONEWAY_SPAM_SUSPECT; else @@ -3430,6 +3758,7 @@ err_bad_object_type: err_bad_offset: err_bad_parent: err_copy_data_failed: + binder_cleanup_deferred_txn_lists(&sgc_head, &pf_head); binder_free_txn_fixups(t); trace_binder_transaction_failed_buffer_release(t->buffer); binder_transaction_buffer_release(target_proc, NULL, t->buffer, diff --git a/drivers/android/binder_alloc.c b/drivers/android/binder_alloc.c index d30267e08536..447342a878ff 100644 --- a/drivers/android/binder_alloc.c +++ b/drivers/android/binder_alloc.c @@ -213,7 +213,7 @@ static int binder_update_page_range(struct binder_alloc *alloc, int allocate, mm = alloc->vma_vm_mm; if (mm) { - mmap_read_lock(mm); + mmap_write_lock(mm); vma = alloc->vma; } @@ -271,7 +271,7 @@ static int binder_update_page_range(struct binder_alloc *alloc, int allocate, trace_binder_alloc_page_end(alloc, index); } if (mm) { - mmap_read_unlock(mm); + mmap_write_unlock(mm); mmput(mm); } return 0; @@ -304,7 +304,7 @@ err_page_ptr_cleared: } err_no_vma: if (mm) { - mmap_read_unlock(mm); + mmap_write_unlock(mm); mmput(mm); } return vma ? -ENOMEM : -ESRCH; diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 73efd5503d2f..ecad931b264c 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -61,6 +61,7 @@ #include #include #include +#include #ifdef __GENKSYMS__ #include #endif @@ -181,7 +182,10 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_irqs_disable); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_irqs_enable); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_task_cpu); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_fork); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_wake_up_new_task); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_schedule); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_build_perf_domains); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_update_misfit_status); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_cpu_cgroup_attach); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_cpu_cgroup_can_attach); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_cpu_cgroup_online); @@ -225,6 +229,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_calc_alloc_flags); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_compaction_begin); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_compaction_end); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rmqueue); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_pagevec_drain); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_pagecache_get_page); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_filemap_fault_get_page); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_filemap_fault_cache_page); @@ -253,6 +258,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_create_worker); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_check_preempt_tick); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_check_preempt_wakeup_ignore); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_replace_next_task_fair); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_do_sched_yield); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_wait_for_work); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_sync_txn_recvd); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_build_sched_domains); @@ -281,6 +287,15 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_get_from_fragment_pool); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_exclude_reserved_zone); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_include_reserved_zone); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_alloc_pages_slowpath); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_do_madvise_blk_plug); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_shrink_inactive_list_blk_plug); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_shrink_lruvec_blk_plug); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_reclaim_pages_plug); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_zap_pte_range_tlb_start); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_zap_pte_range_tlb_force_flush); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_zap_pte_range_tlb_end); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_bh_lru_install); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_skip_lru_disable); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cma_alloc_adjust); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_show_mem); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_print_slabinfo_header); diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c index b1974d3e54a6..bc695f880a22 100644 --- a/drivers/base/power/domain.c +++ b/drivers/base/power/domain.c @@ -2052,9 +2052,9 @@ static int genpd_remove(struct generic_pm_domain *genpd) kfree(link); } - genpd_debug_remove(genpd); list_del(&genpd->gpd_list_node); genpd_unlock(genpd); + genpd_debug_remove(genpd); cancel_work_sync(&genpd->power_off_work); if (genpd_is_cpu_domain(genpd)) free_cpumask_var(genpd->cpus); diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c index f3487fb94a54..771cdce7b4ac 100644 --- a/drivers/clk/clk.c +++ b/drivers/clk/clk.c @@ -3809,8 +3809,9 @@ struct clk *clk_hw_create_clk(struct device *dev, struct clk_hw *hw, struct clk *clk_hw_get_clk(struct clk_hw *hw, const char *con_id) { struct device *dev = hw->core->dev; + const char *name = dev ? dev_name(dev) : NULL; - return clk_hw_create_clk(dev, hw, dev_name(dev), con_id); + return clk_hw_create_clk(dev, hw, name, con_id); } EXPORT_SYMBOL(clk_hw_get_clk); diff --git a/drivers/clk/versatile/Kconfig b/drivers/clk/versatile/Kconfig index 792315d893db..481de5657d85 100644 --- a/drivers/clk/versatile/Kconfig +++ b/drivers/clk/versatile/Kconfig @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only menu "Clock driver for ARM Reference designs" + depends on HAS_IOMEM config ICST bool "Clock driver for ARM Reference designs ICST" diff --git a/drivers/dma-buf/Kconfig b/drivers/dma-buf/Kconfig index 8634c09036ac..3280ca972502 100644 --- a/drivers/dma-buf/Kconfig +++ b/drivers/dma-buf/Kconfig @@ -67,7 +67,7 @@ menuconfig DMABUF_HEAPS menuconfig DMABUF_SYSFS_STATS bool "DMA-BUF sysfs statistics" - select DMA_SHARED_BUFFER + depends on DMA_SHARED_BUFFER help Choose this option to enable DMA-BUF sysfs statistics in location /sys/kernel/dmabuf/buffers. diff --git a/drivers/dma-buf/heaps/cma_heap.c b/drivers/dma-buf/heaps/cma_heap.c index fd564aa70ee9..511d678ecbfc 100644 --- a/drivers/dma-buf/heaps/cma_heap.c +++ b/drivers/dma-buf/heaps/cma_heap.c @@ -126,10 +126,11 @@ static int cma_heap_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, struct cma_heap_buffer *buffer = dmabuf->priv; struct dma_heap_attachment *a; + mutex_lock(&buffer->lock); + if (buffer->vmap_cnt) invalidate_kernel_vmap_range(buffer->vaddr, buffer->len); - mutex_lock(&buffer->lock); list_for_each_entry(a, &buffer->attachments, list) { if (!a->mapped) continue; @@ -146,10 +147,11 @@ static int cma_heap_dma_buf_end_cpu_access(struct dma_buf *dmabuf, struct cma_heap_buffer *buffer = dmabuf->priv; struct dma_heap_attachment *a; + mutex_lock(&buffer->lock); + if (buffer->vmap_cnt) flush_kernel_vmap_range(buffer->vaddr, buffer->len); - mutex_lock(&buffer->lock); list_for_each_entry(a, &buffer->attachments, list) { if (!a->mapped) continue; diff --git a/drivers/firmware/arm_ffa/driver.c b/drivers/firmware/arm_ffa/driver.c index fc6d3b9bd472..bed57b9147fe 100644 --- a/drivers/firmware/arm_ffa/driver.c +++ b/drivers/firmware/arm_ffa/driver.c @@ -501,7 +501,7 @@ static int ffa_partition_info_get(const char *uuid_str, return -ENODEV; } - count = ffa_partition_probe(&uuid_null, &pbuf); + count = ffa_partition_probe(&uuid, &pbuf); if (count <= 0) return -ENOENT; @@ -607,8 +607,6 @@ static void ffa_setup_partitions(void) __func__, tpbuf->id); continue; } - - ffa_dev_set_drvdata(ffa_dev, drv_info); } kfree(pbuf); } diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index e864288a6e0e..46f641481ac9 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -458,7 +458,7 @@ static void mtk_iommu_domain_free(struct iommu_domain *domain) static int mtk_iommu_attach_device(struct iommu_domain *domain, struct device *dev) { - struct mtk_iommu_data *data = dev_iommu_priv_get(dev); + struct mtk_iommu_data *data = dev_iommu_priv_get(dev), *frstdata; struct mtk_iommu_domain *dom = to_mtk_domain(domain); struct device *m4udev = data->dev; int ret, domid; @@ -468,20 +468,24 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, return domid; if (!dom->data) { - if (mtk_iommu_domain_finalise(dom, data, domid)) + /* Data is in the frstdata in sharing pgtable case. */ + frstdata = mtk_iommu_get_m4u_data(); + + if (mtk_iommu_domain_finalise(dom, frstdata, domid)) return -ENODEV; dom->data = data; } + mutex_lock(&data->mutex); if (!data->m4u_dom) { /* Initialize the M4U HW */ ret = pm_runtime_resume_and_get(m4udev); if (ret < 0) - return ret; + goto err_unlock; ret = mtk_iommu_hw_init(data); if (ret) { pm_runtime_put(m4udev); - return ret; + goto err_unlock; } data->m4u_dom = dom; writel(dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK, @@ -489,9 +493,14 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, pm_runtime_put(m4udev); } + mutex_unlock(&data->mutex); mtk_iommu_config(data, dev, true, domid); return 0; + +err_unlock: + mutex_unlock(&data->mutex); + return ret; } static void mtk_iommu_detach_device(struct iommu_domain *domain, @@ -603,6 +612,7 @@ static struct iommu_group *mtk_iommu_device_group(struct device *dev) if (domid < 0) return ERR_PTR(domid); + mutex_lock(&data->mutex); group = data->m4u_group[domid]; if (!group) { group = iommu_group_alloc(); @@ -611,6 +621,7 @@ static struct iommu_group *mtk_iommu_device_group(struct device *dev) } else { iommu_group_ref_get(group); } + mutex_unlock(&data->mutex); return group; } @@ -884,6 +895,7 @@ static int mtk_iommu_probe(struct platform_device *pdev) } platform_set_drvdata(pdev, data); + mutex_init(&data->mutex); ret = iommu_device_sysfs_add(&data->iommu, dev, NULL, "mtk-iommu.%pa", &ioaddr); @@ -934,7 +946,6 @@ static int mtk_iommu_remove(struct platform_device *pdev) list_del(&data->list); - clk_disable_unprepare(data->bclk); device_link_remove(data->smicomm_dev, &pdev->dev); pm_runtime_disable(&pdev->dev); devm_free_irq(&pdev->dev, data->irq, data); diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h index f81fa8862ed0..f413546ac6e5 100644 --- a/drivers/iommu/mtk_iommu.h +++ b/drivers/iommu/mtk_iommu.h @@ -80,6 +80,8 @@ struct mtk_iommu_data { struct dma_iommu_mapping *mapping; /* For mtk_iommu_v1.c */ + struct mutex mutex; /* Protect m4u_group/m4u_dom above */ + struct list_head list; struct mtk_smi_larb_iommu larb_imu[MTK_LARB_NR_MAX]; }; diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index e9a4fb326d24..2c93c53fc688 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -3024,18 +3024,12 @@ static int __init allocate_lpi_tables(void) return 0; } -static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set) +static u64 read_vpend_dirty_clear(void __iomem *vlpi_base) { u32 count = 1000000; /* 1s! */ bool clean; u64 val; - val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER); - val &= ~GICR_VPENDBASER_Valid; - val &= ~clr; - val |= set; - gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); - do { val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER); clean = !(val & GICR_VPENDBASER_Dirty); @@ -3046,10 +3040,26 @@ static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set) } } while (!clean && count); - if (unlikely(val & GICR_VPENDBASER_Dirty)) { + if (unlikely(!clean)) pr_err_ratelimited("ITS virtual pending table not cleaning\n"); + + return val; +} + +static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set) +{ + u64 val; + + /* Make sure we wait until the RD is done with the initial scan */ + val = read_vpend_dirty_clear(vlpi_base); + val &= ~GICR_VPENDBASER_Valid; + val &= ~clr; + val |= set; + gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); + + val = read_vpend_dirty_clear(vlpi_base); + if (unlikely(val & GICR_VPENDBASER_Dirty)) val |= GICR_VPENDBASER_PendingLast; - } return val; } diff --git a/drivers/md/dm-user.c b/drivers/md/dm-user.c index e5a85202d8a0..a8d771855e79 100644 --- a/drivers/md/dm-user.c +++ b/drivers/md/dm-user.c @@ -188,7 +188,6 @@ static void message_kill(struct message *m, mempool_t *pool) { m->bio->bi_status = BLK_STS_IOERR; bio_endio(m->bio); - bio_put(m->bio); mempool_free(m, pool); } @@ -989,7 +988,6 @@ finish_bio: */ WARN_ON(bio_size(c->cur_from_user->bio) != 0); bio_endio(c->cur_from_user->bio); - bio_put(c->cur_from_user->bio); /* * We don't actually need to take the target lock here, as all @@ -1227,7 +1225,6 @@ static int user_map(struct dm_target *ti, struct bio *bio) return DM_MAPIO_REQUEUE; } - bio_get(bio); entry->msg.type = bio_type_to_user_type(bio); entry->msg.flags = bio_flags_to_user_flags(bio); entry->msg.sector = bio->bi_iter.bi_sector; diff --git a/drivers/regulator/scmi-regulator.c b/drivers/regulator/scmi-regulator.c index 14d846bbf0bd..c48f7e134465 100644 --- a/drivers/regulator/scmi-regulator.c +++ b/drivers/regulator/scmi-regulator.c @@ -350,7 +350,7 @@ static int scmi_regulator_probe(struct scmi_device *sdev) if (ret == -ENOMEM) return ret; } - + of_node_put(np); /* * Register a regulator for each valid regulator-DT-entry that we * can successfully reach via SCMI and has a valid associated voltage diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c index 6f3a1f1f7ec1..2a22f4ee1f6a 100644 --- a/drivers/scsi/scsi_scan.c +++ b/drivers/scsi/scsi_scan.c @@ -97,7 +97,7 @@ MODULE_PARM_DESC(max_luns, #define SCSI_SCAN_TYPE_DEFAULT "sync" #endif -char scsi_scan_type[7] = SCSI_SCAN_TYPE_DEFAULT; +static char scsi_scan_type[7] = SCSI_SCAN_TYPE_DEFAULT; module_param_string(scan, scsi_scan_type, sizeof(scsi_scan_type), S_IRUGO|S_IWUSR); @@ -122,7 +122,7 @@ struct async_scan_data { struct completion prev_finished; }; -/** +/* * scsi_enable_async_suspend - Enable async suspend and resume */ void scsi_enable_async_suspend(struct device *dev) diff --git a/drivers/staging/android/ashmem.c b/drivers/staging/android/ashmem.c index 05e47ce9657d..448ac60ccb7b 100644 --- a/drivers/staging/android/ashmem.c +++ b/drivers/staging/android/ashmem.c @@ -816,6 +816,7 @@ out_unlock: static long ashmem_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { struct ashmem_area *asma = file->private_data; + unsigned long ino; long ret = -ENOTTY; switch (cmd) { @@ -859,6 +860,23 @@ static long ashmem_ioctl(struct file *file, unsigned int cmd, unsigned long arg) ashmem_shrink_scan(&ashmem_shrinker, &sc); } break; + case ASHMEM_GET_FILE_ID: + /* Lock around our check to avoid racing with ashmem_mmap(). */ + mutex_lock(&ashmem_mutex); + if (!asma || !asma->file) { + mutex_unlock(&ashmem_mutex); + ret = -EINVAL; + break; + } + ino = file_inode(asma->file)->i_ino; + mutex_unlock(&ashmem_mutex); + + if (copy_to_user((void __user *)arg, &ino, sizeof(ino))) { + ret = -EFAULT; + break; + } + ret = 0; + break; } return ret; diff --git a/drivers/staging/android/uapi/ashmem.h b/drivers/staging/android/uapi/ashmem.h index 5442e0019dcd..1eb88ce192a8 100644 --- a/drivers/staging/android/uapi/ashmem.h +++ b/drivers/staging/android/uapi/ashmem.h @@ -41,5 +41,6 @@ struct ashmem_pin { #define ASHMEM_UNPIN _IOW(__ASHMEMIOC, 8, struct ashmem_pin) #define ASHMEM_GET_PIN_STATUS _IO(__ASHMEMIOC, 9) #define ASHMEM_PURGE_ALL_CACHES _IO(__ASHMEMIOC, 10) +#define ASHMEM_GET_FILE_ID _IOR(__ASHMEMIOC, 11, unsigned long) #endif /* _UAPI_LINUX_ASHMEM_H */ diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c index 18ee3914b468..b53aff809ecc 100644 --- a/drivers/usb/core/hub.c +++ b/drivers/usb/core/hub.c @@ -3016,6 +3016,48 @@ done: return status; } +/* + * hub_port_stop_enumerate - stop USB enumeration or ignore port events + * @hub: target hub + * @port1: port num of the port + * @retries: port retries number of hub_port_init() + * + * Return: + * true: ignore port actions/events or give up connection attempts. + * false: keep original behavior. + * + * This function will be based on retries to check whether the port which is + * marked with early_stop attribute would stop enumeration or ignore events. + * + * Note: + * This function didn't change anything if early_stop is not set, and it will + * prevent all connection attempts when early_stop is set and the attempts of + * the port are more than 1. + */ +static bool hub_port_stop_enumerate(struct usb_hub *hub, int port1, int retries) +{ + struct usb_port *port_dev = hub->ports[port1 - 1]; + + if (port_dev->early_stop) { + if (port_dev->ignore_event) + return true; + + /* + * We want unsuccessful attempts to fail quickly. + * Since some devices may need one failure during + * port initialization, we allow two tries but no + * more. + */ + if (retries < 2) + return false; + + port_dev->ignore_event = 1; + } else + port_dev->ignore_event = 0; + + return port_dev->ignore_event; +} + /* Check if a port is power on */ static int port_is_power_on(struct usb_hub *hub, unsigned portstatus) { @@ -4729,6 +4771,11 @@ hub_port_init(struct usb_hub *hub, struct usb_device *udev, int port1, do_new_scheme = use_new_scheme(udev, retry_counter, port_dev); for (retries = 0; retries < GET_DESCRIPTOR_TRIES; (++retries, msleep(100))) { + if (hub_port_stop_enumerate(hub, port1, retries)) { + retval = -ENODEV; + break; + } + if (do_new_scheme) { struct usb_device_descriptor *buf; int r = 0; @@ -5173,6 +5220,11 @@ static void hub_port_connect(struct usb_hub *hub, int port1, u16 portstatus, status = 0; for (i = 0; i < PORT_INIT_TRIES; i++) { + if (hub_port_stop_enumerate(hub, port1, i)) { + status = -ENODEV; + break; + } + usb_lock_port(port_dev); mutex_lock(hcd->address0_mutex); retry_locked = true; @@ -5542,6 +5594,10 @@ static void port_event(struct usb_hub *hub, int port1) if (!pm_runtime_active(&port_dev->dev)) return; + /* skip port actions if ignore_event and early_stop are true */ + if (port_dev->ignore_event && port_dev->early_stop) + return; + if (hub_handle_remote_wakeup(hub, port1, portstatus, portchange)) connect_change = 1; @@ -5854,6 +5910,10 @@ static int usb_reset_and_verify_device(struct usb_device *udev) mutex_lock(hcd->address0_mutex); for (i = 0; i < PORT_INIT_TRIES; ++i) { + if (hub_port_stop_enumerate(parent_hub, port1, i)) { + ret = -ENODEV; + break; + } /* ep0 maxpacket size may change; let the HCD know about it. * Other endpoints will be handled by re-enumeration. */ diff --git a/drivers/usb/core/hub.h b/drivers/usb/core/hub.h index 22ea1f4f2d66..fdb5d8c21d91 100644 --- a/drivers/usb/core/hub.h +++ b/drivers/usb/core/hub.h @@ -89,6 +89,8 @@ struct usb_hub { * @is_superspeed cache super-speed status * @usb3_lpm_u1_permit: whether USB3 U1 LPM is permitted. * @usb3_lpm_u2_permit: whether USB3 U2 LPM is permitted. + * @early_stop: whether port initialization will be stopped earlier. + * @ignore_event: whether events of the port are ignored. */ struct usb_port { struct usb_device *child; @@ -102,6 +104,8 @@ struct usb_port { u32 over_current_count; u8 portnum; u32 quirks; + unsigned int early_stop:1; + unsigned int ignore_event:1; unsigned int is_superspeed:1; unsigned int usb3_lpm_u1_permit:1; unsigned int usb3_lpm_u2_permit:1; diff --git a/drivers/usb/core/port.c b/drivers/usb/core/port.c index 235a7c645503..54270f9394ef 100644 --- a/drivers/usb/core/port.c +++ b/drivers/usb/core/port.c @@ -16,6 +16,32 @@ static int usb_port_block_power_off; static const struct attribute_group *port_dev_group[]; +static ssize_t early_stop_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct usb_port *port_dev = to_usb_port(dev); + + return sysfs_emit(buf, "%s\n", port_dev->early_stop ? "yes" : "no"); +} + +static ssize_t early_stop_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct usb_port *port_dev = to_usb_port(dev); + bool value; + + if (kstrtobool(buf, &value)) + return -EINVAL; + + if (value) + port_dev->early_stop = 1; + else + port_dev->early_stop = 0; + + return count; +} +static DEVICE_ATTR_RW(early_stop); + static ssize_t location_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -152,6 +178,7 @@ static struct attribute *port_dev_attrs[] = { &dev_attr_location.attr, &dev_attr_quirks.attr, &dev_attr_over_current_count.attr, + &dev_attr_early_stop.attr, NULL, }; diff --git a/drivers/usb/typec/tcpm/tcpci.c b/drivers/usb/typec/tcpm/tcpci.c index fb9c14d0d584..17cea37c73fd 100644 --- a/drivers/usb/typec/tcpm/tcpci.c +++ b/drivers/usb/typec/tcpm/tcpci.c @@ -85,9 +85,25 @@ static int tcpci_write16(struct tcpci *tcpci, unsigned int reg, u16 val) static int tcpci_set_cc(struct tcpc_dev *tcpc, enum typec_cc_status cc) { struct tcpci *tcpci = tcpc_to_tcpci(tcpc); + bool vconn_pres; + enum typec_cc_polarity polarity = TYPEC_POLARITY_CC1; unsigned int reg; int ret; + ret = regmap_read(tcpci->regmap, TCPC_POWER_STATUS, ®); + if (ret < 0) + return ret; + + vconn_pres = !!(reg & TCPC_POWER_STATUS_VCONN_PRES); + if (vconn_pres) { + ret = regmap_read(tcpci->regmap, TCPC_TCPC_CTRL, ®); + if (ret < 0) + return ret; + + if (reg & TCPC_TCPC_CTRL_ORIENTATION) + polarity = TYPEC_POLARITY_CC2; + } + switch (cc) { case TYPEC_CC_RA: reg = (TCPC_ROLE_CTRL_CC_RA << TCPC_ROLE_CTRL_CC1_SHIFT) | @@ -122,6 +138,16 @@ static int tcpci_set_cc(struct tcpc_dev *tcpc, enum typec_cc_status cc) break; } + if (vconn_pres) { + if (polarity == TYPEC_POLARITY_CC2) { + reg &= ~(TCPC_ROLE_CTRL_CC1_MASK << TCPC_ROLE_CTRL_CC1_SHIFT); + reg |= (TCPC_ROLE_CTRL_CC_OPEN << TCPC_ROLE_CTRL_CC1_SHIFT); + } else { + reg &= ~(TCPC_ROLE_CTRL_CC2_MASK << TCPC_ROLE_CTRL_CC2_SHIFT); + reg |= (TCPC_ROLE_CTRL_CC_OPEN << TCPC_ROLE_CTRL_CC2_SHIFT); + } + } + ret = regmap_write(tcpci->regmap, TCPC_ROLE_CTRL, reg); if (ret < 0) return ret; diff --git a/drivers/usb/typec/tcpm/tcpci.h b/drivers/usb/typec/tcpm/tcpci.h index 2be7a77d400e..b2edd45f13c6 100644 --- a/drivers/usb/typec/tcpm/tcpci.h +++ b/drivers/usb/typec/tcpm/tcpci.h @@ -98,6 +98,7 @@ #define TCPC_POWER_STATUS_SOURCING_VBUS BIT(4) #define TCPC_POWER_STATUS_VBUS_DET BIT(3) #define TCPC_POWER_STATUS_VBUS_PRES BIT(2) +#define TCPC_POWER_STATUS_VCONN_PRES BIT(1) #define TCPC_POWER_STATUS_SINKING_VBUS BIT(0) #define TCPC_FAULT_STATUS 0x1f diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index fed8153ea7f0..05e2647481fc 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -5384,6 +5384,10 @@ static void _tcpm_pd_vbus_vsafe0v(struct tcpm_port *port) case PR_SWAP_SNK_SRC_SOURCE_ON: /* Do nothing, vsafe0v is expected during transition */ break; + case SNK_ATTACH_WAIT: + case SNK_DEBOUNCED: + /*Do nothing, still waiting for VSAFE5V for connect */ + break; default: if (port->pwr_role == TYPEC_SINK && port->auto_vbus_discharge_enabled) tcpm_set_state(port, SNK_UNATTACHED, 0); diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c index 441bc057896f..97b58c51dcc2 100644 --- a/drivers/virtio/virtio.c +++ b/drivers/virtio/virtio.c @@ -452,8 +452,9 @@ int virtio_device_restore(struct virtio_device *dev) goto err; } - /* Finally, tell the device we're all set */ - virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK); + /* If restore didn't do it, mark device DRIVER_OK ourselves. */ + if (!(dev->config->get_status(dev) & VIRTIO_CONFIG_S_DRIVER_OK)) + virtio_device_ready(dev); virtio_config_enable(dev); diff --git a/fs/buffer.c b/fs/buffer.c index 13dd0f71f762..029e806a85bd 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -51,6 +51,8 @@ #include "internal.h" +#include + static int fsync_buffers_list(spinlock_t *lock, struct list_head *list); static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh, enum rw_hint hint, struct writeback_control *wbc); @@ -1262,6 +1264,7 @@ static void bh_lru_install(struct buffer_head *bh) struct buffer_head *evictee = bh; struct bh_lru *b; int i; + bool skip = false; check_irqs_on(); /* @@ -1273,6 +1276,10 @@ static void bh_lru_install(struct buffer_head *bh) if (lru_cache_disabled()) return; + trace_android_vh_bh_lru_install(bh->b_page, &skip); + if (skip) + return; + bh_lru_lock(); b = this_cpu_ptr(&bh_lrus); diff --git a/fs/erofs/sysfs.c b/fs/erofs/sysfs.c index 33e15fa63c82..dd122dd1c38b 100644 --- a/fs/erofs/sysfs.c +++ b/fs/erofs/sysfs.c @@ -204,9 +204,11 @@ void erofs_unregister_sysfs(struct super_block *sb) { struct erofs_sb_info *sbi = EROFS_SB(sb); - kobject_del(&sbi->s_kobj); - kobject_put(&sbi->s_kobj); - wait_for_completion(&sbi->s_kobj_unregister); + if (sbi->s_kobj.state_in_sysfs) { + kobject_del(&sbi->s_kobj); + kobject_put(&sbi->s_kobj); + wait_for_completion(&sbi->s_kobj_unregister); + } } int __init erofs_init_sysfs(void) diff --git a/fs/exec.c b/fs/exec.c index 7ba60c0f10a0..bc503539257d 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -1029,10 +1029,10 @@ static int exec_mmap(struct mm_struct *mm) activate_mm(active_mm, mm); if (IS_ENABLED(CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM)) local_irq_enable(); - lru_gen_use_mm(mm); tsk->mm->vmacache_seqnum = 0; vmacache_flush(tsk); task_unlock(tsk); + lru_gen_use_mm(mm); if (old_mm) { mmap_read_unlock(old_mm); BUG_ON(active_mm != old_mm); diff --git a/fs/ext4/verity.c b/fs/ext4/verity.c index 00e3cbde472e..35be8e7ec2a0 100644 --- a/fs/ext4/verity.c +++ b/fs/ext4/verity.c @@ -370,13 +370,14 @@ static struct page *ext4_read_merkle_tree_page(struct inode *inode, pgoff_t index, unsigned long num_ra_pages) { - DEFINE_READAHEAD(ractl, NULL, inode->i_mapping, index); struct page *page; index += ext4_verity_metadata_pos(inode) >> PAGE_SHIFT; page = find_get_page_flags(inode->i_mapping, index, FGP_ACCESSED); if (!page || !PageUptodate(page)) { + DEFINE_READAHEAD(ractl, NULL, inode->i_mapping, index); + if (page) put_page(page); else if (num_ra_pages > 1) diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c index df73ec82d4ff..e621560635af 100644 --- a/fs/f2fs/checkpoint.c +++ b/fs/f2fs/checkpoint.c @@ -26,12 +26,16 @@ static struct kmem_cache *ino_entry_slab; struct kmem_cache *f2fs_inode_entry_slab; -void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io) +void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io, + unsigned char reason) { f2fs_build_fault_attr(sbi, 0, 0); set_ckpt_flags(sbi, CP_ERROR_FLAG); - if (!end_io) + if (!end_io) { f2fs_flush_merged_writes(sbi); + + f2fs_handle_stop(sbi, reason); + } } /* @@ -122,7 +126,7 @@ retry: if (PTR_ERR(page) == -EIO && ++count <= DEFAULT_RETRY_IO_COUNT) goto retry; - f2fs_stop_checkpoint(sbi, false); + f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_META_PAGE); } return page; } diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index b60962461f96..c71c3e70833c 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -336,7 +336,8 @@ static void f2fs_write_end_io(struct bio *bio) mempool_free(page, sbi->write_io_dummy); if (unlikely(bio->bi_status)) - f2fs_stop_checkpoint(sbi, true); + f2fs_stop_checkpoint(sbi, true, + STOP_CP_REASON_WRITE_FAIL); continue; } @@ -352,7 +353,8 @@ static void f2fs_write_end_io(struct bio *bio) if (unlikely(bio->bi_status)) { mapping_set_error(page->mapping, -EIO); if (type == F2FS_WB_CP_DATA) - f2fs_stop_checkpoint(sbi, true); + f2fs_stop_checkpoint(sbi, true, + STOP_CP_REASON_WRITE_FAIL); } f2fs_bug_on(sbi, page->mapping == NODE_MAPPING(sbi) && diff --git a/fs/f2fs/dir.c b/fs/f2fs/dir.c index 4316fbfc5034..1bc7e0387842 100644 --- a/fs/f2fs/dir.c +++ b/fs/f2fs/dir.c @@ -82,7 +82,8 @@ int f2fs_init_casefolded_name(const struct inode *dir, #ifdef CONFIG_UNICODE struct super_block *sb = dir->i_sb; - if (IS_CASEFOLDED(dir)) { + if (IS_CASEFOLDED(dir) && + !is_dot_dotdot(fname->usr_fname->name, fname->usr_fname->len)) { fname->cf_name.name = f2fs_kmem_cache_alloc(f2fs_cf_name_slab, GFP_NOFS, false, F2FS_SB(sb)); if (!fname->cf_name.name) diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 7ff0b057d305..d9e038d42548 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -512,11 +512,11 @@ struct f2fs_filename { #ifdef CONFIG_UNICODE /* * For casefolded directories: the casefolded name, but it's left NULL - * if the original name is not valid Unicode, if the directory is both - * casefolded and encrypted and its encryption key is unavailable, or if - * the filesystem is doing an internal operation where usr_fname is also - * NULL. In all these cases we fall back to treating the name as an - * opaque byte sequence. + * if the original name is not valid Unicode, if the original name is + * "." or "..", if the directory is both casefolded and encrypted and + * its encryption key is unavailable, or if the filesystem is doing an + * internal operation where usr_fname is also NULL. In all these cases + * we fall back to treating the name as an opaque byte sequence. */ struct fscrypt_str cf_name; #endif @@ -3527,6 +3527,7 @@ int f2fs_enable_quota_files(struct f2fs_sb_info *sbi, bool rdonly); int f2fs_quota_sync(struct super_block *sb, int type); loff_t max_file_blocks(struct inode *inode); void f2fs_quota_off_umount(struct super_block *sb); +void f2fs_handle_stop(struct f2fs_sb_info *sbi, unsigned char reason); int f2fs_commit_super(struct f2fs_sb_info *sbi, bool recover); int f2fs_sync_fs(struct super_block *sb, int sync); int f2fs_sanity_check_ckpt(struct f2fs_sb_info *sbi); @@ -3691,7 +3692,8 @@ static inline bool f2fs_need_rand_seg(struct f2fs_sb_info *sbi) /* * checkpoint.c */ -void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io); +void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io, + unsigned char reason); void f2fs_flush_ckpt_thread(struct f2fs_sb_info *sbi); struct page *f2fs_grab_meta_page(struct f2fs_sb_info *sbi, pgoff_t index); struct page *f2fs_get_meta_page(struct f2fs_sb_info *sbi, pgoff_t index); diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index d3e40a3c0918..af753352f565 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -2278,7 +2278,8 @@ static int f2fs_ioc_shutdown(struct file *filp, unsigned long arg) if (ret) { if (ret == -EROFS) { ret = 0; - f2fs_stop_checkpoint(sbi, false); + f2fs_stop_checkpoint(sbi, false, + STOP_CP_REASON_SHUTDOWN); set_sbi_flag(sbi, SBI_IS_SHUTDOWN); trace_f2fs_shutdown(sbi, in, ret); } @@ -2291,7 +2292,7 @@ static int f2fs_ioc_shutdown(struct file *filp, unsigned long arg) ret = freeze_bdev(sb->s_bdev); if (ret) goto out; - f2fs_stop_checkpoint(sbi, false); + f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_SHUTDOWN); set_sbi_flag(sbi, SBI_IS_SHUTDOWN); thaw_bdev(sb->s_bdev); break; @@ -2300,16 +2301,16 @@ static int f2fs_ioc_shutdown(struct file *filp, unsigned long arg) ret = f2fs_sync_fs(sb, 1); if (ret) goto out; - f2fs_stop_checkpoint(sbi, false); + f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_SHUTDOWN); set_sbi_flag(sbi, SBI_IS_SHUTDOWN); break; case F2FS_GOING_DOWN_NOSYNC: - f2fs_stop_checkpoint(sbi, false); + f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_SHUTDOWN); set_sbi_flag(sbi, SBI_IS_SHUTDOWN); break; case F2FS_GOING_DOWN_METAFLUSH: f2fs_sync_meta_pages(sbi, META, LONG_MAX, FS_META_IO); - f2fs_stop_checkpoint(sbi, false); + f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_SHUTDOWN); set_sbi_flag(sbi, SBI_IS_SHUTDOWN); break; case F2FS_GOING_DOWN_NEED_FSCK: diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c index 15fefb5089fa..8577589fe543 100644 --- a/fs/f2fs/gc.c +++ b/fs/f2fs/gc.c @@ -70,7 +70,8 @@ static int gc_thread_func(void *data) if (time_to_inject(sbi, FAULT_CHECKPOINT)) { f2fs_show_injection_info(sbi, FAULT_CHECKPOINT); - f2fs_stop_checkpoint(sbi, false); + f2fs_stop_checkpoint(sbi, false, + STOP_CP_REASON_FAULT_INJECT); } if (!sb_start_write_trylock(sbi->sb)) { @@ -1660,7 +1661,8 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, f2fs_err(sbi, "Inconsistent segment (%u) type [%d, %d] in SSA and SIT", segno, type, GET_SUM_TYPE((&sum->footer))); set_sbi_flag(sbi, SBI_NEED_FSCK); - f2fs_stop_checkpoint(sbi, false); + f2fs_stop_checkpoint(sbi, false, + STOP_CP_REASON_CORRUPTED_SUMMARY); goto skip; } diff --git a/fs/f2fs/hash.c b/fs/f2fs/hash.c index e3beac546c63..2788ceeaf5c2 100644 --- a/fs/f2fs/hash.c +++ b/fs/f2fs/hash.c @@ -91,7 +91,7 @@ static u32 TEA_hash_name(const u8 *p, size_t len) /* * Compute @fname->hash. For all directories, @fname->disk_name must be set. * For casefolded directories, @fname->usr_fname must be set, and also - * @fname->cf_name if the filename is valid Unicode. + * @fname->cf_name if the filename is valid Unicode and is not "." or "..". */ void f2fs_hash_filename(const struct inode *dir, struct f2fs_filename *fname) { @@ -110,10 +110,11 @@ void f2fs_hash_filename(const struct inode *dir, struct f2fs_filename *fname) /* * If the casefolded name is provided, hash it instead of the * on-disk name. If the casefolded name is *not* provided, that - * should only be because the name wasn't valid Unicode, so fall - * back to treating the name as an opaque byte sequence. Note - * that to handle encrypted directories, the fallback must use - * usr_fname (plaintext) rather than disk_name (ciphertext). + * should only be because the name wasn't valid Unicode or was + * "." or "..", so fall back to treating the name as an opaque + * byte sequence. Note that to handle encrypted directories, + * the fallback must use usr_fname (plaintext) rather than + * disk_name (ciphertext). */ WARN_ON_ONCE(!fname->usr_fname->name); if (fname->cf_name.name) { diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c index 2849eb9d18dd..829d767a77bf 100644 --- a/fs/f2fs/inode.c +++ b/fs/f2fs/inode.c @@ -699,7 +699,8 @@ retry: cond_resched(); goto retry; } else if (err != -ENOENT) { - f2fs_stop_checkpoint(sbi, false); + f2fs_stop_checkpoint(sbi, false, + STOP_CP_REASON_UPDATE_INODE); } return; } diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c index 9d6e9ff468d1..acb4ea1aeabb 100644 --- a/fs/f2fs/namei.c +++ b/fs/f2fs/namei.c @@ -628,6 +628,8 @@ static int f2fs_unlink(struct inode *dir, struct dentry *dentry) goto fail; } f2fs_delete_entry(de, page, dir, inode); + f2fs_unlock_op(sbi); + #ifdef CONFIG_UNICODE /* VFS negative dentries are incompatible with Encoding and * Case-insensitiveness. Eventually we'll want avoid @@ -638,8 +640,6 @@ static int f2fs_unlink(struct inode *dir, struct dentry *dentry) if (IS_CASEFOLDED(dir)) d_invalidate(dentry); #endif - f2fs_unlock_op(sbi); - if (IS_DIRSYNC(dir)) f2fs_sync_fs(sbi->sb, 1); fail: diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index 4cbdce62b892..7f637b7c34cd 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -1362,8 +1362,7 @@ static int read_node_page(struct page *page, int op_flags) return err; /* NEW_ADDR can be seen, after cp_error drops some dirty node pages */ - if (unlikely(ni.blk_addr == NULL_ADDR || ni.blk_addr == NEW_ADDR) || - is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN)) { + if (unlikely(ni.blk_addr == NULL_ADDR || ni.blk_addr == NEW_ADDR)) { ClearPageUptodate(page); return -ENOENT; } diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index fe08b5c18315..f21b1cb13326 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -501,7 +501,7 @@ void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need) { if (time_to_inject(sbi, FAULT_CHECKPOINT)) { f2fs_show_injection_info(sbi, FAULT_CHECKPOINT); - f2fs_stop_checkpoint(sbi, false); + f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_FAULT_INJECT); } /* balance_fs_bg is able to be pending */ @@ -812,7 +812,8 @@ int f2fs_flush_device_cache(struct f2fs_sb_info *sbi) } while (ret && --count); if (ret) { - f2fs_stop_checkpoint(sbi, false); + f2fs_stop_checkpoint(sbi, false, + STOP_CP_REASON_FLUSH_FAIL); break; } diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index ca1160d7d5e5..0d20b52ae247 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -3863,6 +3863,26 @@ int f2fs_commit_super(struct f2fs_sb_info *sbi, bool recover) return err; } +void f2fs_handle_stop(struct f2fs_sb_info *sbi, unsigned char reason) +{ + struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); + int err; + + f2fs_bug_on(sbi, reason >= MAX_STOP_REASON); + + f2fs_down_write(&sbi->sb_lock); + + if (raw_super->s_stop_reason[reason] < ((1 << BITS_PER_BYTE) - 1)) + raw_super->s_stop_reason[reason]++; + + err = f2fs_commit_super(sbi, false); + if (err) + f2fs_err(sbi, "f2fs_commit_super fails to record reason:%u err:%d", + reason, err); + + f2fs_up_write(&sbi->sb_lock); +} + static int f2fs_scan_devices(struct f2fs_sb_info *sbi) { struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); diff --git a/fs/f2fs/verity.c b/fs/f2fs/verity.c index 5cbc46324787..8f03cbe0b5fe 100644 --- a/fs/f2fs/verity.c +++ b/fs/f2fs/verity.c @@ -261,13 +261,14 @@ static struct page *f2fs_read_merkle_tree_page(struct inode *inode, pgoff_t index, unsigned long num_ra_pages) { - DEFINE_READAHEAD(ractl, NULL, inode->i_mapping, index); struct page *page; index += f2fs_verity_metadata_pos(inode) >> PAGE_SHIFT; page = find_get_page_flags(inode->i_mapping, index, FGP_ACCESSED); if (!page || !PageUptodate(page)) { + DEFINE_READAHEAD(ractl, NULL, inode->i_mapping, index); + if (page) put_page(page); else if (num_ra_pages > 1) diff --git a/fs/proc/fd.c b/fs/proc/fd.c index 6157913911f8..6b634c0a9b6e 100644 --- a/fs/proc/fd.c +++ b/fs/proc/fd.c @@ -71,7 +71,7 @@ out: return 0; } -static int seq_fdinfo_open(struct inode *inode, struct file *file) +static int proc_fdinfo_access_allowed(struct inode *inode) { bool allowed = false; struct task_struct *task = get_proc_task(inode); @@ -85,6 +85,16 @@ static int seq_fdinfo_open(struct inode *inode, struct file *file) if (!allowed) return -EACCES; + return 0; +} + +static int seq_fdinfo_open(struct inode *inode, struct file *file) +{ + int ret = proc_fdinfo_access_allowed(inode); + + if (ret) + return ret; + return single_open(file, seq_show, inode); } @@ -365,12 +375,23 @@ static int proc_readfdinfo(struct file *file, struct dir_context *ctx) proc_fdinfo_instantiate); } +static int proc_open_fdinfo(struct inode *inode, struct file *file) +{ + int ret = proc_fdinfo_access_allowed(inode); + + if (ret) + return ret; + + return 0; +} + const struct inode_operations proc_fdinfo_inode_operations = { .lookup = proc_lookupfdinfo, .setattr = proc_setattr, }; const struct file_operations proc_fdinfo_operations = { + .open = proc_open_fdinfo, .read = generic_read_dir, .iterate_shared = proc_readfdinfo, .llseek = generic_file_llseek, diff --git a/include/linux/f2fs_fs.h b/include/linux/f2fs_fs.h index d445150c5350..5dd1e52b8997 100644 --- a/include/linux/f2fs_fs.h +++ b/include/linux/f2fs_fs.h @@ -73,6 +73,20 @@ struct f2fs_device { __le32 total_segments; } __packed; +/* reason of stop_checkpoint */ +enum stop_cp_reason { + STOP_CP_REASON_SHUTDOWN, + STOP_CP_REASON_FAULT_INJECT, + STOP_CP_REASON_META_PAGE, + STOP_CP_REASON_WRITE_FAIL, + STOP_CP_REASON_CORRUPTED_SUMMARY, + STOP_CP_REASON_UPDATE_INODE, + STOP_CP_REASON_FLUSH_FAIL, + STOP_CP_REASON_MAX, +}; + +#define MAX_STOP_REASON 32 + struct f2fs_super_block { __le32 magic; /* Magic Number */ __le16 major_ver; /* Major Version */ @@ -116,7 +130,8 @@ struct f2fs_super_block { __u8 hot_ext_count; /* # of hot file extension */ __le16 s_encoding; /* Filename charset encoding */ __le16 s_encoding_flags; /* Filename charset encoding flags */ - __u8 reserved[306]; /* valid reserved region */ + __u8 s_stop_reason[MAX_STOP_REASON]; /* stop checkpoint reason */ + __u8 reserved[274]; /* valid reserved region */ __le32 crc; /* checksum of superblock */ } __packed; diff --git a/include/linux/freezer.h b/include/linux/freezer.h index f753c307b8b3..181aac4b5187 100644 --- a/include/linux/freezer.h +++ b/include/linux/freezer.h @@ -8,6 +8,9 @@ #include #include #include +#if defined(CONFIG_ARM64) && !defined(__GENKSYMS__) +#include +#endif #ifdef CONFIG_FREEZER extern atomic_t system_freezing_cnt; /* nr of freezing conds in effect */ @@ -108,10 +111,15 @@ static inline bool cgroup_freezing(struct task_struct *task) * The caller shouldn't do anything which isn't allowed for a frozen task * until freezer_cont() is called. Usually, freezer[_do_not]_count() pair * wrap a scheduling operation and nothing much else. + * + * The write to current->flags uses release semantics to prevent a concurrent + * freezer_should_skip() from observing this write before a write to on_rq + * during a prior call to activate_task(), which may cause it to return true + * before deactivate_task() is called. */ static inline void freezer_do_not_count(void) { - current->flags |= PF_FREEZER_SKIP; + smp_store_release(¤t->flags, current->flags | PF_FREEZER_SKIP); } /** @@ -161,7 +169,19 @@ static inline bool freezer_should_skip(struct task_struct *p) * clearing %PF_FREEZER_SKIP. */ smp_mb(); +#ifdef CONFIG_ARM64 + return (p->flags & PF_FREEZER_SKIP) && + (!p->on_rq || task_cpu_possible_mask(p) == cpu_possible_mask); +#else + /* + * On non-aarch64, avoid depending on task_cpu_possible_mask(), which is + * defined in , because including that header from + * here exposes a tricky bug in the tracepoint headers on x86, and that + * macro would end up being defined equal to cpu_possible_mask on other + * architectures anyway. + */ return p->flags & PF_FREEZER_SKIP; +#endif } /* diff --git a/include/linux/mm.h b/include/linux/mm.h index 6c40625f26d0..0755c09c441e 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1761,6 +1761,12 @@ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, #ifdef CONFIG_SPECULATIVE_PAGE_FAULT static inline void vm_write_begin(struct vm_area_struct *vma) { + /* + * Isolated vma might be freed without exclusive mmap_lock but + * speculative page fault handler still needs to know it was changed. + */ + if (!RB_EMPTY_NODE(&vma->vm_rb)) + mmap_assert_write_locked(vma->vm_mm); /* * The reads never spins and preemption * disablement is not required. diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index af0790da3b9f..e1f93b2d9ddb 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -27,11 +27,12 @@ static inline int page_is_file_lru(struct page *page) static __always_inline void __update_lru_size(struct lruvec *lruvec, enum lru_list lru, enum zone_type zid, - int nr_pages) + long nr_pages) { struct pglist_data *pgdat = lruvec_pgdat(lruvec); lockdep_assert_held(&pgdat->lru_lock); + WARN_ON_ONCE(nr_pages != (int)nr_pages); __mod_lruvec_state(lruvec, NR_LRU_BASE + lru, nr_pages); __mod_zone_page_state(&pgdat->node_zones[zid], @@ -91,18 +92,21 @@ static __always_inline enum lru_list page_lru(struct page *page) #ifdef CONFIG_LRU_GEN +#ifdef CONFIG_LRU_GEN_ENABLED static inline bool lru_gen_enabled(void) { -#ifdef CONFIG_LRU_GEN_ENABLED DECLARE_STATIC_KEY_TRUE(lru_gen_caps[NR_LRU_GEN_CAPS]); return static_branch_likely(&lru_gen_caps[LRU_GEN_CORE]); +} #else +static inline bool lru_gen_enabled(void) +{ DECLARE_STATIC_KEY_FALSE(lru_gen_caps[NR_LRU_GEN_CAPS]); return static_branch_unlikely(&lru_gen_caps[LRU_GEN_CORE]); -#endif } +#endif static inline bool lru_gen_in_fault(void) { @@ -121,17 +125,38 @@ static inline int lru_hist_from_seq(unsigned long seq) static inline int lru_tier_from_refs(int refs) { - VM_BUG_ON(refs > BIT(LRU_REFS_WIDTH)); + VM_WARN_ON_ONCE(refs > BIT(LRU_REFS_WIDTH)); - /* see the comment on MAX_NR_TIERS */ + /* see the comment in page_lru_refs() */ return order_base_2(refs + 1); } +static inline int page_lru_refs(struct page *page) +{ + unsigned long flags = READ_ONCE(page->flags); + bool workingset = flags & BIT(PG_workingset); + + /* + * Return the number of accesses beyond PG_referenced, i.e., N-1 if the + * total number of accesses is N>1, since N=0,1 both map to the first + * tier. lru_tier_from_refs() will account for this off-by-one. Also see + * the comment on MAX_NR_TIERS. + */ + return ((flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF) + workingset; +} + +static inline int page_lru_gen(struct page *page) +{ + unsigned long flags = READ_ONCE(page->flags); + + return ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; +} + static inline bool lru_gen_is_active(struct lruvec *lruvec, int gen) { unsigned long max_seq = lruvec->lrugen.max_seq; - VM_BUG_ON(gen >= MAX_NR_GENS); + VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); /* see the comment on MIN_NR_GENS */ return gen == lru_gen_from_seq(max_seq) || gen == lru_gen_from_seq(max_seq - 1); @@ -146,9 +171,9 @@ static inline void lru_gen_update_size(struct lruvec *lruvec, struct page *page, enum lru_list lru = type * LRU_INACTIVE_FILE; struct lru_gen_struct *lrugen = &lruvec->lrugen; - VM_BUG_ON(old_gen != -1 && old_gen >= MAX_NR_GENS); - VM_BUG_ON(new_gen != -1 && new_gen >= MAX_NR_GENS); - VM_BUG_ON(old_gen == -1 && new_gen == -1); + VM_WARN_ON_ONCE(old_gen != -1 && old_gen >= MAX_NR_GENS); + VM_WARN_ON_ONCE(new_gen != -1 && new_gen >= MAX_NR_GENS); + VM_WARN_ON_ONCE(old_gen == -1 && new_gen == -1); if (old_gen >= 0) WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone], @@ -180,17 +205,20 @@ static inline void lru_gen_update_size(struct lruvec *lruvec, struct page *page, } /* demotion requires isolation, e.g., lru_deactivate_fn() */ - VM_BUG_ON(lru_gen_is_active(lruvec, old_gen) && !lru_gen_is_active(lruvec, new_gen)); + VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) && !lru_gen_is_active(lruvec, new_gen)); } static inline bool lru_gen_add_page(struct lruvec *lruvec, struct page *page, bool reclaiming) { - int gen; - unsigned long old_flags, new_flags; + unsigned long seq; + unsigned long flags; + int gen = page_lru_gen(page); int type = page_is_file_lru(page); int zone = page_zonenum(page); struct lru_gen_struct *lrugen = &lruvec->lrugen; + VM_WARN_ON_ONCE_PAGE(gen != -1, page); + if (PageUnevictable(page) || !lrugen->enabled) return false; /* @@ -203,21 +231,18 @@ static inline bool lru_gen_add_page(struct lruvec *lruvec, struct page *page, bo * 3. Everything else (clean, cold) is added to the oldest generation. */ if (PageActive(page)) - gen = lru_gen_from_seq(lrugen->max_seq); + seq = lrugen->max_seq; else if ((type == LRU_GEN_ANON && !PageSwapCache(page)) || - (PageReclaim(page) && (PageDirty(page) || PageWriteback(page)))) - gen = lru_gen_from_seq(lrugen->min_seq[type] + 1); + (PageReclaim(page) && + (PageDirty(page) || PageWriteback(page)))) + seq = lrugen->min_seq[type] + 1; else - gen = lru_gen_from_seq(lrugen->min_seq[type]); + seq = lrugen->min_seq[type]; - do { - new_flags = old_flags = READ_ONCE(page->flags); - VM_BUG_ON_PAGE(new_flags & LRU_GEN_MASK, page); - - /* see the comment on MIN_NR_GENS */ - new_flags &= ~(LRU_GEN_MASK | BIT(PG_active)); - new_flags |= (gen + 1UL) << LRU_GEN_PGOFF; - } while (cmpxchg(&page->flags, old_flags, new_flags) != old_flags); + gen = lru_gen_from_seq(seq); + flags = (gen + 1UL) << LRU_GEN_PGOFF; + /* see the comment on MIN_NR_GENS about PG_active */ + set_mask_bits(&page->flags, LRU_GEN_MASK | BIT(PG_active), flags); lru_gen_update_size(lruvec, page, -1, gen); /* for rotate_reclaimable_page() */ @@ -231,28 +256,19 @@ static inline bool lru_gen_add_page(struct lruvec *lruvec, struct page *page, bo static inline bool lru_gen_del_page(struct lruvec *lruvec, struct page *page, bool reclaiming) { - int gen; - unsigned long old_flags, new_flags; + unsigned long flags; + int gen = page_lru_gen(page); - do { - new_flags = old_flags = READ_ONCE(page->flags); - if (!(new_flags & LRU_GEN_MASK)) - return false; + if (gen < 0) + return false; - VM_BUG_ON_PAGE(PageActive(page), page); - VM_BUG_ON_PAGE(PageUnevictable(page), page); + VM_WARN_ON_ONCE_PAGE(PageActive(page), page); + VM_WARN_ON_ONCE_PAGE(PageUnevictable(page), page); - gen = ((new_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; - - new_flags &= ~LRU_GEN_MASK; - if (!(new_flags & BIT(PG_referenced))) - new_flags &= ~(LRU_REFS_MASK | LRU_REFS_FLAGS); - /* for shrink_page_list() */ - if (reclaiming) - new_flags &= ~(BIT(PG_referenced) | BIT(PG_reclaim)); - else if (lru_gen_is_active(lruvec, gen)) - new_flags |= BIT(PG_active); - } while (cmpxchg(&page->flags, old_flags, new_flags) != old_flags); + /* for migrate_page_states() */ + flags = !reclaiming && lru_gen_is_active(lruvec, gen) ? BIT(PG_active) : 0; + flags = set_mask_bits(&page->flags, LRU_GEN_MASK, flags); + gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; lru_gen_update_size(lruvec, page, gen, -1); list_del(&page->lru); @@ -260,7 +276,7 @@ static inline bool lru_gen_del_page(struct lruvec *lruvec, struct page *page, bo return true; } -#else +#else /* !CONFIG_LRU_GEN */ static inline bool lru_gen_enabled(void) { diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 988f4e6a4e39..80126b5c2568 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -617,7 +617,6 @@ struct mm_struct { #ifdef CONFIG_IOMMU_SUPPORT u32 pasid; #endif - #ifdef CONFIG_LRU_GEN struct { /* this mm_struct is on lru_gen_mm_list */ @@ -680,19 +679,20 @@ void lru_gen_migrate_mm(struct mm_struct *mm); static inline void lru_gen_init_mm(struct mm_struct *mm) { INIT_LIST_HEAD(&mm->lru_gen.list); + nodes_clear(mm->lru_gen.nodes); #ifdef CONFIG_MEMCG mm->lru_gen.memcg = NULL; #endif - nodes_clear(mm->lru_gen.nodes); } static inline void lru_gen_use_mm(struct mm_struct *mm) { - /* unlikely but not a bug when racing with lru_gen_migrate_mm() */ - VM_WARN_ON(list_empty(&mm->lru_gen.list)); - - if (!(current->flags & PF_KTHREAD) && !nodes_full(mm->lru_gen.nodes)) - nodes_setall(mm->lru_gen.nodes); + /* + * When the bitmap is set, page reclaim knows this mm_struct has been + * used since the last time it cleared the bitmap. So it might be worth + * walking the page tables of this mm_struct to clear the accessed bit. + */ + nodes_setall(mm->lru_gen.nodes); } #else /* !CONFIG_LRU_GEN */ diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 6977a57b177b..f820fff9f81d 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -287,9 +287,9 @@ enum lruvec_flags { * Evictable pages are divided into multiple generations. The youngest and the * oldest generation numbers, max_seq and min_seq, are monotonically increasing. * They form a sliding window of a variable size [MIN_NR_GENS, MAX_NR_GENS]. An - * offset within MAX_NR_GENS, gen, indexes the LRU list of the corresponding - * generation. The gen counter in page->flags stores gen+1 while a page is on - * one of lrugen->lists[]. Otherwise it stores 0. + * offset within MAX_NR_GENS, i.e., gen, indexes the LRU list of the + * corresponding generation. The gen counter in page->flags stores gen+1 while + * a page is on one of lrugen->lists[]. Otherwise it stores 0. * * A page is added to the youngest generation on faulting. The aging needs to * check the accessed bit at least twice before handing this page over to the @@ -297,38 +297,42 @@ enum lruvec_flags { * fault; the second check makes sure this page hasn't been used since then. * This process, AKA second chance, requires a minimum of two generations, * hence MIN_NR_GENS. And to maintain ABI compatibility with the active/inactive - * LRU, these two generations are considered active; the rest of generations, if - * they exist, are considered inactive. See lru_gen_is_active(). PG_active is - * always cleared while a page is on one of lrugen->lists[] so that the aging - * needs not to worry about it. And it's set again when a page considered active - * is isolated for non-reclaiming purposes, e.g., migration. See - * lru_gen_add_page() and lru_gen_del_page(). + * LRU, e.g., /proc/vmstat, these two generations are considered active; the + * rest of generations, if they exist, are considered inactive. See + * lru_gen_is_active(). * - * MAX_NR_GENS is set to 4 so that the multi-gen LRU can support twice of the - * categories of the active/inactive LRU when keeping track of accesses through - * page tables. It requires order_base_2(MAX_NR_GENS+1) bits in page->flags. + * PG_active is always cleared while a page is on one of lrugen->lists[] so that + * the aging needs not to worry about it. And it's set again when a page + * considered active is isolated for non-reclaiming purposes, e.g., migration. + * See lru_gen_add_page() and lru_gen_del_page(). + * + * MAX_NR_GENS is set to 4 so that the multi-gen LRU can support twice the + * number of categories of the active/inactive LRU when keeping track of + * accesses through page tables. This requires order_base_2(MAX_NR_GENS+1) bits + * in page->flags. */ #define MIN_NR_GENS 2U #define MAX_NR_GENS 4U /* - * Each generation is divided into multiple tiers. Tiers represent different - * ranges of numbers of accesses through file descriptors. A page accessed N - * times through file descriptors is in tier order_base_2(N). A page in the - * first tier (N=0,1) is marked by PG_referenced unless it was faulted in - * though page tables or read ahead. A page in any other tier (N>1) is marked - * by PG_referenced and PG_workingset. + * Each generation is divided into multiple tiers. A page accessed N times + * through file descriptors is in tier order_base_2(N). A page in the first tier + * (N=0,1) is marked by PG_referenced unless it was faulted in through page + * tables or read ahead. A page in any other tier (N>1) is marked by + * PG_referenced and PG_workingset. This implies a minimum of two tiers is + * supported without using additional bits in page->flags. * * In contrast to moving across generations which requires the LRU lock, moving - * across tiers only requires operations on page->flags and therefore has a - * negligible cost in the buffered access path. In the eviction path, + * across tiers only involves atomic operations on page->flags and therefore + * has a negligible cost in the buffered access path. In the eviction path, * comparisons of refaulted/(evicted+protected) from the first tier and the * rest infer whether pages accessed multiple times through file descriptors * are statistically hot and thus worth protecting. * - * MAX_NR_TIERS is set to 4 so that the multi-gen LRU can support twice of the - * categories of the active/inactive LRU when keeping track of accesses through - * file descriptors. It requires MAX_NR_TIERS-2 additional bits in page->flags. + * MAX_NR_TIERS is set to 4 so that the multi-gen LRU can support twice the + * number of categories of the active/inactive LRU when keeping track of + * accesses through file descriptors. This uses MAX_NR_TIERS-2 spare bits in + * page->flags. */ #define MAX_NR_TIERS 4U @@ -339,7 +343,6 @@ struct page_vma_mapped_walk; #define LRU_GEN_MASK ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF) #define LRU_REFS_MASK ((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF) -#define LRU_REFS_FLAGS (BIT(PG_referenced) | BIT(PG_workingset)) #ifdef CONFIG_LRU_GEN @@ -356,7 +359,7 @@ enum { }; #define MIN_LRU_BATCH BITS_PER_LONG -#define MAX_LRU_BATCH (MIN_LRU_BATCH * 128) +#define MAX_LRU_BATCH (MIN_LRU_BATCH * 64) /* whether to keep historical stats from evicted generations */ #ifdef CONFIG_LRU_GEN_STATS @@ -374,6 +377,9 @@ enum { * Normally anon and file min_seq are in sync. But if swapping is constrained, * e.g., out of swap space, file min_seq is allowed to advance and leave anon * min_seq behind. + * + * The number of pages in each generation is eventually consistent and therefore + * can be transiently negative when reset_batch_size() is pending. */ struct lru_gen_struct { /* the aging increments the youngest generation number */ @@ -382,9 +388,9 @@ struct lru_gen_struct { unsigned long min_seq[ANON_AND_FILE]; /* the birth time of each generation in jiffies */ unsigned long timestamps[MAX_NR_GENS]; - /* the multi-gen LRU lists */ + /* the multi-gen LRU lists, lazily sorted on eviction */ struct list_head lists[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; - /* the sizes of the above lists */ + /* the multi-gen LRU sizes, eventually consistent */ unsigned long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; /* the exponential moving average of refaulted */ unsigned long avg_refaulted[ANON_AND_FILE][MAX_NR_TIERS]; @@ -400,27 +406,24 @@ struct lru_gen_struct { }; enum { - MM_PTE_TOTAL, /* total leaf entries */ - MM_PTE_OLD, /* old leaf entries */ - MM_PTE_YOUNG, /* young leaf entries */ - MM_PMD_TOTAL, /* total non-leaf entries */ - MM_PMD_FOUND, /* non-leaf entries found in Bloom filters */ - MM_PMD_ADDED, /* non-leaf entries added to Bloom filters */ + MM_LEAF_TOTAL, /* total leaf entries */ + MM_LEAF_OLD, /* old leaf entries */ + MM_LEAF_YOUNG, /* young leaf entries */ + MM_NONLEAF_TOTAL, /* total non-leaf entries */ + MM_NONLEAF_FOUND, /* non-leaf entries found in Bloom filters */ + MM_NONLEAF_ADDED, /* non-leaf entries added to Bloom filters */ NR_MM_STATS }; -/* mnemonic codes for the mm stats above */ -#define MM_STAT_CODES "toydfa" - /* double-buffering Bloom filters */ #define NR_BLOOM_FILTERS 2 struct lru_gen_mm_state { /* set to max_seq after each iteration */ unsigned long seq; - /* where the current iteration starts (inclusive) */ + /* where the current iteration continues (inclusive) */ struct list_head *head; - /* where the last iteration ends (exclusive) */ + /* where the last iteration ended (exclusive) */ struct list_head *tail; /* to wait for the last page table walker to finish */ struct wait_queue_head wait; @@ -439,7 +442,7 @@ struct lru_gen_mm_walk { unsigned long max_seq; /* the next address within an mm to scan */ unsigned long next_addr; - /* to batch page table entries */ + /* Unused -- for ABI compatibility */ unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)]; /* to batch promoted pages */ int nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; diff --git a/include/linux/module.h b/include/linux/module.h index c5c3ce08f646..4cd6d889d5ba 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -293,8 +293,7 @@ extern typeof(name) __mod_##type##__##name##_device_table \ * files require multiple MODULE_FIRMWARE() specifiers */ #define MODULE_FIRMWARE(_firmware) MODULE_INFO(firmware, _firmware) -#define _MODULE_IMPORT_NS(ns) MODULE_INFO(import_ns, #ns) -#define MODULE_IMPORT_NS(ns) _MODULE_IMPORT_NS(ns) +#define MODULE_IMPORT_NS(ns) MODULE_INFO(import_ns, __stringify(ns)) struct notifier_block; diff --git a/include/linux/mtd/mtd.h b/include/linux/mtd/mtd.h index fc41fecfe44f..157357ec1441 100644 --- a/include/linux/mtd/mtd.h +++ b/include/linux/mtd/mtd.h @@ -388,8 +388,10 @@ struct mtd_info { /* List of partitions attached to this MTD device */ struct list_head partitions; - struct mtd_part part; - struct mtd_master master; + union { + struct mtd_part part; + struct mtd_master master; + }; }; static inline struct mtd_info *mtd_get_master(struct mtd_info *mtd) diff --git a/include/linux/page-flags-layout.h b/include/linux/page-flags-layout.h index c1946cdb845f..7d79818dc065 100644 --- a/include/linux/page-flags-layout.h +++ b/include/linux/page-flags-layout.h @@ -55,7 +55,7 @@ #define SECTIONS_WIDTH 0 #endif -#if ZONES_WIDTH + LRU_GEN_WIDTH + LRU_REFS_WIDTH + SECTIONS_WIDTH + NODES_SHIFT \ +#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_SHIFT \ <= BITS_PER_LONG - NR_PAGEFLAGS #define NODES_WIDTH NODES_SHIFT #elif defined(CONFIG_SPARSEMEM_VMEMMAP) @@ -90,7 +90,7 @@ #define LAST_CPUPID_SHIFT 0 #endif -#if ZONES_WIDTH + LRU_GEN_WIDTH + LRU_REFS_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \ +#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \ KASAN_TAG_WIDTH + LAST_CPUPID_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS #define LAST_CPUPID_WIDTH LAST_CPUPID_SHIFT #else @@ -101,10 +101,15 @@ #define LAST_CPUPID_NOT_IN_PAGE_FLAGS #endif -#if ZONES_WIDTH + LRU_GEN_WIDTH + LRU_REFS_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \ +#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \ KASAN_TAG_WIDTH + LAST_CPUPID_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS #error "Not enough bits in page flags" #endif +/* see the comment on MAX_NR_TIERS */ +#define LRU_REFS_WIDTH min(__LRU_REFS_WIDTH, BITS_PER_LONG - NR_PAGEFLAGS - \ + ZONES_WIDTH - LRU_GEN_WIDTH - SECTIONS_WIDTH - \ + NODES_WIDTH - KASAN_TAG_WIDTH - LAST_CPUPID_WIDTH) + #endif #endif /* _LINUX_PAGE_FLAGS_LAYOUT */ diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h index eeacb4a16fe3..df9505faed27 100644 --- a/include/linux/sched/task.h +++ b/include/linux/sched/task.h @@ -55,8 +55,8 @@ extern asmlinkage void schedule_tail(struct task_struct *prev); extern void init_idle(struct task_struct *idle, int cpu); extern int sched_fork(unsigned long clone_flags, struct task_struct *p); -extern void sched_post_fork(struct task_struct *p, - struct kernel_clone_args *kargs); +extern void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); +extern void sched_post_fork(struct task_struct *p); extern void sched_dead(struct task_struct *p); void __noreturn do_task_dead(void); diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 365c416e0f53..bb1e05763c59 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1436,6 +1436,11 @@ static inline unsigned int skb_end_offset(const struct sk_buff *skb) { return skb->end; } + +static inline void skb_set_end_offset(struct sk_buff *skb, unsigned int offset) +{ + skb->end = offset; +} #else static inline unsigned char *skb_end_pointer(const struct sk_buff *skb) { @@ -1446,6 +1451,11 @@ static inline unsigned int skb_end_offset(const struct sk_buff *skb) { return skb->end - skb->head; } + +static inline void skb_set_end_offset(struct sk_buff *skb, unsigned int offset) +{ + skb->end = skb->head + offset; +} #endif /* Internal */ @@ -1665,19 +1675,19 @@ static inline int skb_unclone(struct sk_buff *skb, gfp_t pri) return 0; } -/* This variant of skb_unclone() makes sure skb->truesize is not changed */ +/* This variant of skb_unclone() makes sure skb->truesize + * and skb_end_offset() are not changed, whenever a new skb->head is needed. + * + * Indeed there is no guarantee that ksize(kmalloc(X)) == ksize(kmalloc(X)) + * when various debugging features are in place. + */ +int __skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri); static inline int skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri) { might_sleep_if(gfpflags_allow_blocking(pri)); - if (skb_cloned(skb)) { - unsigned int save = skb->truesize; - int res; - - res = pskb_expand_head(skb, 0, 0, pri); - skb->truesize = save; - return res; - } + if (skb_cloned(skb)) + return __skb_unclone_keeptruesize(skb, pri); return 0; } diff --git a/include/linux/swap.h b/include/linux/swap.h index dfdf026dc54d..592e6eca9090 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -355,7 +355,6 @@ extern void lru_add_drain_cpu(int cpu); extern void lru_add_drain_cpu_zone(struct zone *zone); extern void lru_add_drain_all(void); extern void rotate_reclaimable_page(struct page *page); -extern void activate_page(struct page *page); extern void deactivate_file_page(struct page *page); extern void deactivate_page(struct page *page); extern void mark_page_lazyfree(struct page *page); diff --git a/include/trace/hooks/buffer.h b/include/trace/hooks/buffer.h new file mode 100644 index 000000000000..50e8f71dab9f --- /dev/null +++ b/include/trace/hooks/buffer.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM buffer + +#define TRACE_INCLUDE_PATH trace/hooks + +#if !defined(_TRACE_HOOK_BUFFER_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_HOOK_BUFFER_H + +#include +#include + +DECLARE_HOOK(android_vh_bh_lru_install, + TP_PROTO(struct page *page, bool *flush), + TP_ARGS(page, flush)); + +/* macro versions of hooks are no longer required */ + +#endif /* _TRACE_HOOK_BUFFER_H */ + +/* This part must be outside protection */ +#include diff --git a/include/trace/hooks/mm.h b/include/trace/hooks/mm.h index 263b867d453c..d9be2be468de 100644 --- a/include/trace/hooks/mm.h +++ b/include/trace/hooks/mm.h @@ -61,6 +61,9 @@ DECLARE_HOOK(android_vh_rmqueue, unsigned int alloc_flags, int migratetype), TP_ARGS(preferred_zone, zone, order, gfp_flags, alloc_flags, migratetype)); +DECLARE_HOOK(android_vh_pagevec_drain, + TP_PROTO(struct page *page, bool *ret), + TP_ARGS(page, ret)); DECLARE_HOOK(android_vh_pagecache_get_page, TP_PROTO(struct address_space *mapping, pgoff_t index, int fgp_flags, gfp_t gfp_mask, struct page *page), @@ -97,6 +100,30 @@ DECLARE_HOOK(android_vh_alloc_pages_slowpath, DECLARE_HOOK(android_vh_cma_alloc_adjust, TP_PROTO(struct zone *zone, bool *is_cma_alloc), TP_ARGS(zone, is_cma_alloc)); +DECLARE_HOOK(android_vh_do_madvise_blk_plug, + TP_PROTO(int behavior, bool *do_plug), + TP_ARGS(behavior, do_plug)); +DECLARE_HOOK(android_vh_shrink_inactive_list_blk_plug, + TP_PROTO(bool *do_plug), + TP_ARGS(do_plug)); +DECLARE_HOOK(android_vh_shrink_lruvec_blk_plug, + TP_PROTO(bool *do_plug), + TP_ARGS(do_plug)); +DECLARE_HOOK(android_vh_reclaim_pages_plug, + TP_PROTO(bool *do_plug), + TP_ARGS(do_plug)); +DECLARE_HOOK(android_vh_zap_pte_range_tlb_start, + TP_PROTO(void *ret), + TP_ARGS(ret)); +DECLARE_HOOK(android_vh_zap_pte_range_tlb_force_flush, + TP_PROTO(struct page *page, bool *flush), + TP_ARGS(page, flush)); +DECLARE_HOOK(android_vh_zap_pte_range_tlb_end, + TP_PROTO(void *ret), + TP_ARGS(ret)); +DECLARE_HOOK(android_vh_skip_lru_disable, + TP_PROTO(bool *skip), + TP_ARGS(skip)); DECLARE_HOOK(android_vh_print_slabinfo_header, TP_PROTO(struct seq_file *m), TP_ARGS(m)); diff --git a/include/trace/hooks/sched.h b/include/trace/hooks/sched.h index 4233384f6ff8..112bb8a4479d 100644 --- a/include/trace/hooks/sched.h +++ b/include/trace/hooks/sched.h @@ -149,10 +149,22 @@ DECLARE_RESTRICTED_HOOK(android_rvh_sched_fork, TP_PROTO(struct task_struct *p), TP_ARGS(p), 1); +DECLARE_RESTRICTED_HOOK(android_rvh_wake_up_new_task, + TP_PROTO(struct task_struct *p), + TP_ARGS(p), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_schedule, + TP_PROTO(struct task_struct *prev, struct task_struct *next, struct rq *rq), + TP_ARGS(prev, next, rq), 1); + DECLARE_RESTRICTED_HOOK(android_rvh_build_perf_domains, TP_PROTO(bool *eas_check), TP_ARGS(eas_check), 1); +DECLARE_RESTRICTED_HOOK(android_rvh_update_misfit_status, + TP_PROTO(struct task_struct *p, struct rq *rq, bool *need_update), + TP_ARGS(p, rq, need_update), 1); + DECLARE_RESTRICTED_HOOK(android_rvh_cpu_cgroup_attach, TP_PROTO(struct cgroup_taskset *tset), TP_ARGS(tset), 1); @@ -291,6 +303,10 @@ DECLARE_RESTRICTED_HOOK(android_rvh_cpumask_any_and_distribute, const struct cpumask *new_mask, int *dest_cpu), TP_ARGS(p, cpu_valid_mask, new_mask, dest_cpu), 1); +DECLARE_RESTRICTED_HOOK(android_rvh_do_sched_yield, + TP_PROTO(struct rq *rq), + TP_ARGS(rq), 1); + DECLARE_HOOK(android_vh_free_task, TP_PROTO(struct task_struct *p), TP_ARGS(p)); diff --git a/include/uapi/linux/psci.h b/include/uapi/linux/psci.h index b1484a5a3c36..1529f3154d5d 100644 --- a/include/uapi/linux/psci.h +++ b/include/uapi/linux/psci.h @@ -51,12 +51,11 @@ #define PSCI_1_0_FN_SYSTEM_SUSPEND PSCI_0_2_FN(14) #define PSCI_1_0_FN_SET_SUSPEND_MODE PSCI_0_2_FN(15) #define PSCI_1_1_FN_SYSTEM_RESET2 PSCI_0_2_FN(18) +#define PSCI_1_1_FN_MEM_PROTECT PSCI_0_2_FN(19) #define PSCI_1_0_FN64_SYSTEM_SUSPEND PSCI_0_2_FN64(14) #define PSCI_1_1_FN64_SYSTEM_RESET2 PSCI_0_2_FN64(18) -#define PSCI_1_1_FN64_MEM_PROTECT PSCI_0_2_FN64(19) - /* PSCI v0.2 power state encoding for CPU_SUSPEND function */ #define PSCI_0_2_POWER_STATE_ID_MASK 0xffff #define PSCI_0_2_POWER_STATE_ID_SHIFT 0 diff --git a/init/do_mounts_initrd.c b/init/do_mounts_initrd.c index 533d81ed74d4..cd4a81114f75 100644 --- a/init/do_mounts_initrd.c +++ b/init/do_mounts_initrd.c @@ -83,7 +83,7 @@ static void __init handle_initrd(void) * In case that a resume from disk is carried out by linuxrc or one of * its children, we need to tell the freezer not to wait for us. */ - current->flags |= PF_FREEZER_SKIP; + freezer_do_not_count(); info = call_usermodehelper_setup("/linuxrc", argv, envp_init, GFP_KERNEL, init_linuxrc, NULL, NULL); diff --git a/kernel/audit.h b/kernel/audit.h index 1918019e6aaf..3b9c0945225a 100644 --- a/kernel/audit.h +++ b/kernel/audit.h @@ -191,10 +191,6 @@ struct audit_context { struct { char *name; } module; - struct { - struct audit_ntp_data ntp_data; - struct timespec64 tk_injoffset; - } time; }; int fds[2]; struct audit_proctitle proctitle; diff --git a/kernel/auditsc.c b/kernel/auditsc.c index 07e2788bbbf1..638f424859ed 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -1214,53 +1214,6 @@ static void audit_log_fcaps(struct audit_buffer *ab, struct audit_names *name) from_kuid(&init_user_ns, name->fcap.rootid)); } -static void audit_log_time(struct audit_context *context, struct audit_buffer **ab) -{ - const struct audit_ntp_data *ntp = &context->time.ntp_data; - const struct timespec64 *tk = &context->time.tk_injoffset; - static const char * const ntp_name[] = { - "offset", - "freq", - "status", - "tai", - "tick", - "adjust", - }; - int type; - - if (context->type == AUDIT_TIME_ADJNTPVAL) { - for (type = 0; type < AUDIT_NTP_NVALS; type++) { - if (ntp->vals[type].newval != ntp->vals[type].oldval) { - if (!*ab) { - *ab = audit_log_start(context, - GFP_KERNEL, - AUDIT_TIME_ADJNTPVAL); - if (!*ab) - return; - } - audit_log_format(*ab, "op=%s old=%lli new=%lli", - ntp_name[type], - ntp->vals[type].oldval, - ntp->vals[type].newval); - audit_log_end(*ab); - *ab = NULL; - } - } - } - if (tk->tv_sec != 0 || tk->tv_nsec != 0) { - if (!*ab) { - *ab = audit_log_start(context, GFP_KERNEL, - AUDIT_TIME_INJOFFSET); - if (!*ab) - return; - } - audit_log_format(*ab, "sec=%lli nsec=%li", - (long long)tk->tv_sec, tk->tv_nsec); - audit_log_end(*ab); - *ab = NULL; - } -} - static void show_special(struct audit_context *context, int *call_panic) { struct audit_buffer *ab; @@ -1366,11 +1319,6 @@ static void show_special(struct audit_context *context, int *call_panic) audit_log_format(ab, "(null)"); break; - case AUDIT_TIME_ADJNTPVAL: - case AUDIT_TIME_INJOFFSET: - /* this call deviates from the rest, eating the buffer */ - audit_log_time(context, &ab); - break; } audit_log_end(ab); } @@ -2612,26 +2560,31 @@ void __audit_fanotify(unsigned int response) void __audit_tk_injoffset(struct timespec64 offset) { - struct audit_context *context = audit_context(); + audit_log(audit_context(), GFP_KERNEL, AUDIT_TIME_INJOFFSET, + "sec=%lli nsec=%li", + (long long)offset.tv_sec, offset.tv_nsec); +} - /* only set type if not already set by NTP */ - if (!context->type) - context->type = AUDIT_TIME_INJOFFSET; - memcpy(&context->time.tk_injoffset, &offset, sizeof(offset)); +static void audit_log_ntp_val(const struct audit_ntp_data *ad, + const char *op, enum audit_ntp_type type) +{ + const struct audit_ntp_val *val = &ad->vals[type]; + + if (val->newval == val->oldval) + return; + + audit_log(audit_context(), GFP_KERNEL, AUDIT_TIME_ADJNTPVAL, + "op=%s old=%lli new=%lli", op, val->oldval, val->newval); } void __audit_ntp_log(const struct audit_ntp_data *ad) { - struct audit_context *context = audit_context(); - int type; - - for (type = 0; type < AUDIT_NTP_NVALS; type++) - if (ad->vals[type].newval != ad->vals[type].oldval) { - /* unconditionally set type, overwriting TK */ - context->type = AUDIT_TIME_ADJNTPVAL; - memcpy(&context->time.ntp_data, ad, sizeof(*ad)); - break; - } + audit_log_ntp_val(ad, "offset", AUDIT_NTP_OFFSET); + audit_log_ntp_val(ad, "freq", AUDIT_NTP_FREQ); + audit_log_ntp_val(ad, "status", AUDIT_NTP_STATUS); + audit_log_ntp_val(ad, "tai", AUDIT_NTP_TAI); + audit_log_ntp_val(ad, "tick", AUDIT_NTP_TICK); + audit_log_ntp_val(ad, "adjust", AUDIT_NTP_ADJUST); } void __audit_log_nfcfg(const char *name, u8 af, unsigned int nentries, diff --git a/kernel/bounds.c b/kernel/bounds.c index 10dd9e6b03e5..b529182e8b04 100644 --- a/kernel/bounds.c +++ b/kernel/bounds.c @@ -24,10 +24,10 @@ int main(void) DEFINE(SPINLOCK_SIZE, sizeof(spinlock_t)); #ifdef CONFIG_LRU_GEN DEFINE(LRU_GEN_WIDTH, order_base_2(MAX_NR_GENS + 1)); - DEFINE(LRU_REFS_WIDTH, MAX_NR_TIERS - 2); + DEFINE(__LRU_REFS_WIDTH, MAX_NR_TIERS - 2); #else DEFINE(LRU_GEN_WIDTH, 0); - DEFINE(LRU_REFS_WIDTH, 0); + DEFINE(__LRU_REFS_WIDTH, 0); #endif /* End of constants */ diff --git a/kernel/fork.c b/kernel/fork.c index a0d95fd031d9..26ee6f8c2617 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -2261,6 +2261,17 @@ static __latent_entropy struct task_struct *copy_process( if (retval) goto bad_fork_put_pidfd; + /* + * Now that the cgroups are pinned, re-clone the parent cgroup and put + * the new task on the correct runqueue. All this *before* the task + * becomes visible. + * + * This isn't part of ->can_fork() because while the re-cloning is + * cgroup specific, it unconditionally needs to place the task on a + * runqueue. + */ + sched_cgroup_fork(p, args); + /* * From this point on we must avoid any synchronous user-space * communication until we take the tasklist-lock. In particular, we do @@ -2368,7 +2379,7 @@ static __latent_entropy struct task_struct *copy_process( fd_install(pidfd, pidfile); proc_fork_connector(p); - sched_post_fork(p, args); + sched_post_fork(p); cgroup_post_fork(p, args); perf_event_fork(p); diff --git a/kernel/power/main.c b/kernel/power/main.c index d6140ed15d0b..f985c3ccbf2f 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -23,7 +23,7 @@ void lock_system_sleep(void) { - current->flags |= PF_FREEZER_SKIP; + freezer_do_not_count(); mutex_lock(&system_transition_mutex); } EXPORT_SYMBOL_GPL(lock_system_sleep); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index fb2a0ba31331..0f1c19a38cb3 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3512,6 +3512,7 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) init_entity_runnable_average(&p->se); trace_android_rvh_finish_prio_fork(p); + #ifdef CONFIG_SCHED_INFO if (likely(sched_info_on())) memset(&p->sched_info, 0, sizeof(p->sched_info)); @@ -3527,18 +3528,23 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) return 0; } -void sched_post_fork(struct task_struct *p, struct kernel_clone_args *kargs) +void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) { unsigned long flags; -#ifdef CONFIG_CGROUP_SCHED - struct task_group *tg; -#endif + /* + * Because we're not yet on the pid-hash, p->pi_lock isn't strictly + * required yet, but lockdep gets upset if rules are violated. + */ raw_spin_lock_irqsave(&p->pi_lock, flags); #ifdef CONFIG_CGROUP_SCHED - tg = container_of(kargs->cset->subsys[cpu_cgrp_id], - struct task_group, css); - p->sched_task_group = autogroup_task_group(p, tg); + if (1) { + struct task_group *tg; + tg = container_of(kargs->cset->subsys[cpu_cgrp_id], + struct task_group, css); + tg = autogroup_task_group(p, tg); + p->sched_task_group = tg; + } #endif rseq_migrate(p); /* @@ -3549,7 +3555,10 @@ void sched_post_fork(struct task_struct *p, struct kernel_clone_args *kargs) if (p->sched_class->task_fork) p->sched_class->task_fork(p); raw_spin_unlock_irqrestore(&p->pi_lock, flags); +} +void sched_post_fork(struct task_struct *p) +{ uclamp_post_fork(p); } @@ -3581,6 +3590,8 @@ void wake_up_new_task(struct task_struct *p) struct rq_flags rf; struct rq *rq; + trace_android_rvh_wake_up_new_task(p); + raw_spin_lock_irqsave(&p->pi_lock, rf.flags); p->state = TASK_RUNNING; #ifdef CONFIG_SMP @@ -4623,23 +4634,6 @@ restart: BUG(); } -static bool __task_can_run(struct task_struct *prev) -{ - if (__fatal_signal_pending(prev)) - return true; - - if (!frozen_or_skipped(prev)) - return true; - - /* - * We can't safely go back on the runqueue if we're an asymmetric - * task skipping the freezer. Doing so can lead to migration failures - * later on if there aren't any suitable CPUs left around for us to - * move to. - */ - return task_cpu_possible_mask(prev) == cpu_possible_mask; -} - /* * __schedule() is the main scheduler function. * @@ -4733,7 +4727,7 @@ static void __sched notrace __schedule(bool preempt) */ prev_state = prev->state; if (!preempt && prev_state) { - if (signal_pending_state(prev_state, prev) && __task_can_run(prev)) { + if (signal_pending_state(prev_state, prev)) { prev->state = TASK_RUNNING; } else { prev->sched_contributes_to_load = @@ -4769,6 +4763,7 @@ static void __sched notrace __schedule(bool preempt) clear_tsk_need_resched(prev); clear_preempt_need_resched(); + trace_android_rvh_schedule(prev, next, rq); if (likely(prev != next)) { rq->nr_switches++; /* @@ -6367,6 +6362,8 @@ static void do_sched_yield(void) schedstat_inc(rq->yld_count); current->sched_class->yield_task(rq); + trace_android_rvh_do_sched_yield(rq); + preempt_disable(); rq_unlock_irq(rq, &rf); sched_preempt_enable_no_resched(); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index c3478864e88b..def3ccb1f0e8 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -3137,6 +3137,7 @@ void reweight_task(struct task_struct *p, int prio) reweight_entity(cfs_rq, se, weight); load->inv_weight = sched_prio_to_wmult[prio]; } +EXPORT_SYMBOL_GPL(reweight_task); #ifdef CONFIG_FAIR_GROUP_SCHED #ifdef CONFIG_SMP @@ -4111,6 +4112,7 @@ static inline void update_misfit_status(struct task_struct *p, struct rq *rq) { bool need_update = true; + trace_android_rvh_update_misfit_status(p, rq, &need_update); if (!static_branch_unlikely(&sched_asym_cpucapacity) || !need_update) return; diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c index f46ac0f39777..e14917cd2a1d 100644 --- a/kernel/sched/psi.c +++ b/kernel/sched/psi.c @@ -1020,7 +1020,7 @@ void psi_cgroup_free(struct cgroup *cgroup) */ void cgroup_move_task(struct task_struct *task, struct css_set *to) { - unsigned int task_flags = 0; + unsigned int task_flags; struct rq_flags rf; struct rq *rq; @@ -1035,15 +1035,31 @@ void cgroup_move_task(struct task_struct *task, struct css_set *to) rq = task_rq_lock(task, &rf); - if (task_on_rq_queued(task)) { - task_flags = TSK_RUNNING; - if (task_current(rq, task)) - task_flags |= TSK_ONCPU; - } else if (task->in_iowait) - task_flags = TSK_IOWAIT; - - if (task->in_memstall) - task_flags |= TSK_MEMSTALL; + /* + * We may race with schedule() dropping the rq lock between + * deactivating prev and switching to next. Because the psi + * updates from the deactivation are deferred to the switch + * callback to save cgroup tree updates, the task's scheduling + * state here is not coherent with its psi state: + * + * schedule() cgroup_move_task() + * rq_lock() + * deactivate_task() + * p->on_rq = 0 + * psi_dequeue() // defers TSK_RUNNING & TSK_IOWAIT updates + * pick_next_task() + * rq_unlock() + * rq_lock() + * psi_task_change() // old cgroup + * task->cgroups = to + * psi_task_change() // new cgroup + * rq_unlock() + * rq_lock() + * psi_sched_switch() // does deferred updates in new cgroup + * + * Don't rely on the scheduling state. Use psi_flags instead. + */ + task_flags = task->psi_flags; if (task_flags) psi_task_change(task, task_flags, 0); diff --git a/lib/test_kasan.c b/lib/test_kasan.c index 4de52eb5b93b..7e7d229bd638 100644 --- a/lib/test_kasan.c +++ b/lib/test_kasan.c @@ -879,11 +879,14 @@ static void kmem_cache_invalid_free(struct kunit *test) kmem_cache_destroy(cache); } +static void empty_cache_ctor(void *object) { } + static void kmem_cache_double_destroy(struct kunit *test) { struct kmem_cache *cache; - cache = kmem_cache_create("test_cache", 200, 0, 0, NULL); + /* Provide a constructor to prevent cache merging. */ + cache = kmem_cache_create("test_cache", 200, 0, 0, empty_cache_ctor); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache); kmem_cache_destroy(cache); KUNIT_EXPECT_KASAN_FAIL(test, kmem_cache_destroy(cache)); diff --git a/lib/vsprintf.c b/lib/vsprintf.c index 93293eb04df9..5a1bf311b913 100644 --- a/lib/vsprintf.c +++ b/lib/vsprintf.c @@ -53,6 +53,10 @@ #include #include "kstrtox.h" +/* Disable pointer hashing if requested */ +bool no_hash_pointers __ro_after_init; +EXPORT_SYMBOL_GPL(no_hash_pointers); + static unsigned long long simple_strntoull(const char *startp, size_t max_chars, char **endp, unsigned int base) { @@ -849,6 +853,19 @@ static char *ptr_to_id(char *buf, char *end, const void *ptr, return pointer_string(buf, end, (const void *)hashval, spec); } +static char *default_pointer(char *buf, char *end, const void *ptr, + struct printf_spec spec) +{ + /* + * default is to _not_ leak addresses, so hash before printing, + * unless no_hash_pointers is specified on the command line. + */ + if (unlikely(no_hash_pointers)) + return pointer_string(buf, end, ptr, spec); + + return ptr_to_id(buf, end, ptr, spec); +} + int kptr_restrict __read_mostly; static noinline_for_stack @@ -858,7 +875,7 @@ char *restricted_pointer(char *buf, char *end, const void *ptr, switch (kptr_restrict) { case 0: /* Handle as %p, hash and do _not_ leak addresses. */ - return ptr_to_id(buf, end, ptr, spec); + return default_pointer(buf, end, ptr, spec); case 1: { const struct cred *cred; @@ -2118,10 +2135,6 @@ char *fwnode_string(char *buf, char *end, struct fwnode_handle *fwnode, return widen_string(buf, buf - buf_start, end, spec); } -/* Disable pointer hashing if requested */ -bool no_hash_pointers __ro_after_init; -EXPORT_SYMBOL_GPL(no_hash_pointers); - static int __init no_hash_pointers_enable(char *str) { no_hash_pointers = true; @@ -2339,7 +2352,7 @@ char *pointer(const char *fmt, char *buf, char *end, void *ptr, case 'e': /* %pe with a non-ERR_PTR gets treated as plain %p */ if (!IS_ERR(ptr)) - break; + return default_pointer(buf, end, ptr, spec); return err_ptr(buf, end, ptr, spec); case 'u': case 'k': @@ -2349,16 +2362,9 @@ char *pointer(const char *fmt, char *buf, char *end, void *ptr, default: return error_string(buf, end, "(einval)", spec); } + default: + return default_pointer(buf, end, ptr, spec); } - - /* - * default is to _not_ leak addresses, so hash before printing, - * unless no_hash_pointers is specified on the command line. - */ - if (unlikely(no_hash_pointers)) - return pointer_string(buf, end, ptr, spec); - else - return ptr_to_id(buf, end, ptr, spec); } /* diff --git a/mm/Kconfig b/mm/Kconfig index 27cdf23dd46e..60481d53b946 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -843,7 +843,7 @@ config SPECULATIVE_PAGE_FAULT bool "Speculative page faults" default y depends on ARCH_SUPPORTS_SPECULATIVE_PAGE_FAULT - depends on MMU && SMP + depends on MMU && SMP && !NUMA help Try to handle user space page faults without holding the mmap_sem. @@ -919,8 +919,8 @@ config ANON_VMA_NAME config LRU_GEN bool "Multi-Gen LRU" depends on MMU - # the following options can use up the spare bits in page flags - depends on !MAXSMP && (64BIT || !SPARSEMEM || SPARSEMEM_VMEMMAP) + # make sure page->flags has enough spare bits + depends on 64BIT || !SPARSEMEM || SPARSEMEM_VMEMMAP help A high performance LRU implementation to overcommit memory. See Documentation/admin-guide/mm/multigen_lru.rst for details. diff --git a/mm/cma.c b/mm/cma.c index 126a6d3bdc9b..6d347864693a 100644 --- a/mm/cma.c +++ b/mm/cma.c @@ -44,6 +44,7 @@ struct cma cma_areas[MAX_CMA_AREAS]; unsigned cma_area_count; +static DEFINE_MUTEX(cma_mutex); phys_addr_t cma_get_base(const struct cma *cma) { @@ -508,7 +509,9 @@ struct page *cma_alloc(struct cma *cma, size_t count, unsigned int align, mutex_unlock(&cma->lock); pfn = cma->base_pfn + (bitmap_no << cma->order_per_bit); + mutex_lock(&cma_mutex); ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA, gfp_mask, &info); + mutex_unlock(&cma_mutex); cma_info.nr_migrated += info.nr_migrated; cma_info.nr_reclaimed += info.nr_reclaimed; cma_info.nr_mapped += info.nr_mapped; diff --git a/mm/filemap.c b/mm/filemap.c index 632734e9f8f1..9a42edafb5fb 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2706,7 +2706,9 @@ static struct file *do_async_mmap_readahead(struct vm_fault *vmf, * it in the page cache, and handles the special cases reasonably without * having a lot of duplicated code. * - * vma->vm_mm->mmap_lock must be held on entry. + * If FAULT_FLAG_SPECULATIVE is set, this function runs with elevated vma + * refcount and with mmap lock not held. + * Otherwise, vma->vm_mm->mmap_lock must be held on entry. * * If our return value has VM_FAULT_RETRY set, it's because the mmap_lock * may be dropped before doing I/O or by lock_page_maybe_drop_mmap(). @@ -2732,6 +2734,47 @@ vm_fault_t filemap_fault(struct vm_fault *vmf) vm_fault_t ret = 0; bool retry = false; + if (vmf->flags & FAULT_FLAG_SPECULATIVE) { + page = find_get_page(mapping, offset); + if (unlikely(!page) || unlikely(PageReadahead(page))) + return VM_FAULT_RETRY; + + if (!trylock_page(page)) + return VM_FAULT_RETRY; + + if (unlikely(compound_head(page)->mapping != mapping)) + goto page_unlock; + VM_BUG_ON_PAGE(page_to_pgoff(page) != offset, page); + if (unlikely(!PageUptodate(page))) + goto page_unlock; + + max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); + if (unlikely(offset >= max_off)) + goto page_unlock; + + /* + * Update readahead mmap_miss statistic. + * + * Note that we are not sure if finish_fault() will + * manage to complete the transaction. If it fails, + * we'll come back to filemap_fault() non-speculative + * case which will update mmap_miss a second time. + * This is not ideal, we would prefer to guarantee the + * update will happen exactly once. + */ + if (!(vmf->vma->vm_flags & VM_RAND_READ) && ra->ra_pages) { + unsigned int mmap_miss = READ_ONCE(ra->mmap_miss); + if (mmap_miss) + WRITE_ONCE(ra->mmap_miss, --mmap_miss); + } + + vmf->page = page; + return VM_FAULT_LOCKED; +page_unlock: + unlock_page(page); + return VM_FAULT_RETRY; + } + max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); if (unlikely(offset >= max_off)) return VM_FAULT_SIGBUS; @@ -2874,11 +2917,6 @@ static bool filemap_map_pmd(struct vm_fault *vmf, struct page *page) } if (pmd_none(*vmf->pmd)) { - if (vmf->flags & FAULT_FLAG_SPECULATIVE) { - unlock_page(page); - put_page(page); - return true; - } vmf->ptl = pmd_lock(mm, vmf->pmd); if (likely(pmd_none(*vmf->pmd))) { mm_inc_nr_ptes(mm); @@ -2976,20 +3014,16 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, XA_STATE(xas, &mapping->i_pages, start_pgoff); struct page *head, *page; unsigned int mmap_miss = READ_ONCE(file->f_ra.mmap_miss); - vm_fault_t ret = 0; + vm_fault_t ret = (vmf->flags & FAULT_FLAG_SPECULATIVE) ? + VM_FAULT_RETRY : 0; rcu_read_lock(); head = first_map_page(mapping, &xas, end_pgoff); if (!head) goto out; - if (filemap_map_pmd(vmf, head)) { - if (pmd_none(*vmf->pmd) && - vmf->flags & FAULT_FLAG_SPECULATIVE) { - ret = VM_FAULT_RETRY; - goto out; - } - + if (!(vmf->flags & FAULT_FLAG_SPECULATIVE) && + filemap_map_pmd(vmf, head)) { ret = VM_FAULT_NOPAGE; goto out; } @@ -2998,7 +3032,6 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, if (!pte_map_lock_addr(vmf, addr)) { unlock_page(head); put_page(head); - ret = VM_FAULT_RETRY; goto out; } diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 4b5581acee40..d466484fe912 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -5780,6 +5780,7 @@ void hugetlb_unshare_all_pmds(struct vm_area_struct *vma) if (start >= end) return; + flush_cache_range(vma, start, end); /* * No need to call adjust_range_if_pmd_sharing_possible(), because * we have already done the PUD_SIZE alignment. diff --git a/mm/internal.h b/mm/internal.h index 5a3f3725d306..6aa4e8cab454 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -35,6 +35,7 @@ void page_writeback_init(void); vm_fault_t do_swap_page(struct vm_fault *vmf); +void activate_page(struct page *page); #ifdef CONFIG_SPECULATIVE_PAGE_FAULT extern struct vm_area_struct *get_vma(struct mm_struct *mm, diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 9b2065984369..a110eb77cae8 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1474,6 +1474,7 @@ void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr) if (!pmd) goto drop_hpage; + vm_write_begin(vma); start_pte = pte_offset_map_lock(mm, pmd, haddr, &ptl); /* step 1: check all mapped PTEs are to the right huge page */ @@ -1523,6 +1524,7 @@ void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr) ptl = pmd_lock(vma->vm_mm, pmd); _pmd = pmdp_collapse_flush(vma, haddr, pmd); spin_unlock(ptl); + vm_write_end(vma); mm_dec_nr_ptes(mm); pte_free(mm, pmd_pgtable(_pmd)); @@ -1533,6 +1535,7 @@ drop_hpage: abort: pte_unmap_unlock(start_pte, ptl); + vm_write_end(vma); goto drop_hpage; } @@ -1604,10 +1607,13 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) */ if (mmap_write_trylock(mm)) { if (!khugepaged_test_exit(mm)) { - spinlock_t *ptl = pmd_lock(mm, pmd); + spinlock_t *ptl; + vm_write_begin(vma); + ptl = pmd_lock(mm, pmd); /* assume page table is clear */ _pmd = pmdp_collapse_flush(vma, addr, pmd); spin_unlock(ptl); + vm_write_end(vma); mm_dec_nr_ptes(mm); pte_free(mm, pmd_pgtable(_pmd)); } diff --git a/mm/madvise.c b/mm/madvise.c index 4cd63118efda..d14a4e6497ae 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -31,6 +31,7 @@ #include #include #include +#include #include @@ -500,11 +501,9 @@ static void madvise_cold_page_range(struct mmu_gather *tlb, .tlb = tlb, }; - vm_write_begin(vma); tlb_start_vma(tlb, vma); walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, &walk_private); tlb_end_vma(tlb, vma); - vm_write_end(vma); } static long madvise_cold(struct vm_area_struct *vma, @@ -535,11 +534,9 @@ static void madvise_pageout_page_range(struct mmu_gather *tlb, .tlb = tlb, }; - vm_write_begin(vma); tlb_start_vma(tlb, vma); walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, &walk_private); tlb_end_vma(tlb, vma); - vm_write_end(vma); } static inline bool can_do_pageout(struct vm_area_struct *vma) @@ -742,12 +739,10 @@ static int madvise_free_single_vma(struct vm_area_struct *vma, update_hiwater_rss(mm); mmu_notifier_invalidate_range_start(&range); - vm_write_begin(vma); tlb_start_vma(&tlb, vma); walk_page_range(vma->vm_mm, range.start, range.end, &madvise_free_walk_ops, &tlb); tlb_end_vma(&tlb, vma); - vm_write_end(vma); mmu_notifier_invalidate_range_end(&range); tlb_finish_mmu(&tlb, range.start, range.end); @@ -1269,6 +1264,7 @@ int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int beh int write; size_t len; struct blk_plug plug; + bool do_plug = true; start = untagged_addr(start); @@ -1303,10 +1299,13 @@ int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int beh mmap_read_lock(mm); } - blk_start_plug(&plug); + trace_android_vh_do_madvise_blk_plug(behavior, &do_plug); + if (do_plug) + blk_start_plug(&plug); error = madvise_walk_vmas(mm, start, end, behavior, madvise_vma_behavior); - blk_finish_plug(&plug); + if (do_plug) + blk_finish_plug(&plug); if (write) mmap_write_unlock(mm); else diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 8853786e2b1e..d661df256357 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5330,8 +5330,8 @@ static struct mem_cgroup *mem_cgroup_alloc(void) memcg->deferred_split_queue.split_queue_len = 0; #endif idr_replace(&mem_cgroup_idr, memcg, memcg->id.id); - trace_android_vh_mem_cgroup_alloc(memcg); lru_gen_init_memcg(memcg); + trace_android_vh_mem_cgroup_alloc(memcg); return memcg; fail: mem_cgroup_id_remove(memcg); @@ -6229,9 +6229,10 @@ static void mem_cgroup_move_task(void) #ifdef CONFIG_LRU_GEN static void mem_cgroup_attach(struct cgroup_taskset *tset) { + struct task_struct *task; struct cgroup_subsys_state *css; - struct task_struct *task = NULL; + /* find the first leader if there is any */ cgroup_taskset_for_each_leader(task, css, tset) break; @@ -6239,7 +6240,7 @@ static void mem_cgroup_attach(struct cgroup_taskset *tset) return; task_lock(task); - if (task->mm && task->mm->owner == task) + if (task->mm && READ_ONCE(task->mm->owner) == task) lru_gen_migrate_mm(task->mm); task_unlock(task); } diff --git a/mm/memory.c b/mm/memory.c index 376731fa8766..45a15e26507c 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1241,15 +1241,18 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, pte_t *start_pte; pte_t *pte; swp_entry_t entry; + int v_ret = 0; tlb_change_page_size(tlb, PAGE_SIZE); again: + trace_android_vh_zap_pte_range_tlb_start(&v_ret); init_rss_vec(rss); start_pte = pte_offset_map_lock(mm, pmd, addr, &ptl); pte = start_pte; flush_tlb_batched_pending(mm); arch_enter_lazy_mmu_mode(); do { + bool flush = false; pte_t ptent = *pte; if (pte_none(ptent)) continue; @@ -1290,8 +1293,9 @@ again: page_remove_rmap(page, false); if (unlikely(page_mapcount(page) < 0)) print_bad_pte(vma, addr, ptent, page); + trace_android_vh_zap_pte_range_tlb_force_flush(page, &flush); if (unlikely(__tlb_remove_page(tlb, page)) || - lru_cache_disabled()) { + lru_cache_disabled() || flush) { force_flush = 1; addr += PAGE_SIZE; break; @@ -1359,6 +1363,7 @@ again: tlb_flush_mmu(tlb); } + trace_android_vh_zap_pte_range_tlb_end(&v_ret); if (addr != end) { cond_resched(); goto again; @@ -1470,7 +1475,6 @@ void unmap_page_range(struct mmu_gather *tlb, unsigned long next; BUG_ON(addr >= end); - vm_write_begin(vma); tlb_start_vma(tlb, vma); pgd = pgd_offset(vma->vm_mm, addr); do { @@ -1480,7 +1484,6 @@ void unmap_page_range(struct mmu_gather *tlb, next = zap_p4d_range(tlb, vma, pgd, addr, next, details); } while (pgd++, addr = next, addr != end); tlb_end_vma(tlb, vma); - vm_write_end(vma); } @@ -3361,6 +3364,8 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf) if (userfaultfd_pte_wp(vma, *vmf->pte)) { pte_unmap_unlock(vmf->pte, vmf->ptl); + if (vmf->flags & FAULT_FLAG_SPECULATIVE) + return VM_FAULT_RETRY; return handle_userfault(vmf, VM_UFFD_WP); } @@ -3573,6 +3578,11 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) vm_fault_t ret; void *shadow = NULL; + if (vmf->flags & FAULT_FLAG_SPECULATIVE) { + pte_unmap(vmf->pte); + return VM_FAULT_RETRY; + } + ret = pte_unmap_same(vmf); if (ret) { /* @@ -3831,6 +3841,10 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) if (vmf->vma_flags & VM_SHARED) return VM_FAULT_SIGBUS; + /* Do not check unstable pmd, if it's changed will retry later */ + if (vmf->flags & FAULT_FLAG_SPECULATIVE) + goto skip_pmd_checks; + /* * Use pte_alloc() instead of pte_alloc_map(). We can't run * pte_offset_map() on pmds where a huge pmd might be created @@ -3848,6 +3862,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) if (unlikely(pmd_trans_unstable(vmf->pmd))) return 0; +skip_pmd_checks: /* Use the zero-page for reads */ if (!(vmf->flags & FAULT_FLAG_WRITE) && !mm_forbids_zeropage(vma->vm_mm)) { @@ -3955,6 +3970,10 @@ static vm_fault_t __do_fault(struct vm_fault *vmf) struct vm_area_struct *vma = vmf->vma; vm_fault_t ret; + /* Do not check unstable pmd, if it's changed will retry later */ + if (vmf->flags & FAULT_FLAG_SPECULATIVE) + goto skip_pmd_checks; + /* * Preallocate pte before we take page_lock because this might lead to * deadlocks for memcg reclaim which waits for pages under writeback: @@ -3977,6 +3996,7 @@ static vm_fault_t __do_fault(struct vm_fault *vmf) smp_wmb(); /* See comment in __pte_alloc() */ } +skip_pmd_checks: ret = vma->vm_ops->fault(vmf); if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY | VM_FAULT_DONE_COW))) @@ -4152,7 +4172,11 @@ vm_fault_t finish_fault(struct vm_fault *vmf) return ret; } - if (pmd_none(*vmf->pmd) && !(vmf->flags & FAULT_FLAG_SPECULATIVE)) { + /* Do not check unstable pmd, if it's changed will retry later */ + if (vmf->flags & FAULT_FLAG_SPECULATIVE) + goto skip_pmd_checks; + + if (pmd_none(*vmf->pmd)) { if (PageTransCompound(page)) { ret = do_set_pmd(vmf, page); if (ret != VM_FAULT_FALLBACK) @@ -4176,6 +4200,7 @@ vm_fault_t finish_fault(struct vm_fault *vmf) if (pmd_devmap_trans_unstable(vmf->pmd)) return 0; +skip_pmd_checks: if (!pte_map_lock(vmf)) return VM_FAULT_RETRY; @@ -4275,7 +4300,8 @@ static vm_fault_t do_fault_around(struct vm_fault *vmf) end_pgoff = min3(end_pgoff, vma_pages(vmf->vma) + vmf->vma->vm_pgoff - 1, start_pgoff + nr_pages - 1); - if (pmd_none(*vmf->pmd)) { + if (!(vmf->flags & FAULT_FLAG_SPECULATIVE) && + pmd_none(*vmf->pmd)) { vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm); if (!vmf->prealloc_pte) return VM_FAULT_OOM; @@ -4633,16 +4659,11 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf) pte_t entry; vm_fault_t ret = 0; + /* Do not check unstable pmd, if it's changed will retry later */ + if (vmf->flags & FAULT_FLAG_SPECULATIVE) + goto skip_pmd_checks; + if (unlikely(pmd_none(*vmf->pmd))) { - /* - * In the case of the speculative page fault handler we abort - * the speculative path immediately as the pmd is probably - * in the way to be converted in a huge one. We will try - * again holding the mmap_sem (which implies that the collapse - * operation is done). - */ - if (vmf->flags & FAULT_FLAG_SPECULATIVE) - return VM_FAULT_RETRY; /* * Leave __pte_alloc() until later: because vm_ops->fault may * want to allocate huge page, and if we expose page table @@ -4650,7 +4671,7 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf) * concurrent faults and from rmap lookups. */ vmf->pte = NULL; - } else if (!(vmf->flags & FAULT_FLAG_SPECULATIVE)) { + } else { /* * If a huge pmd materialized under us just retry later. Use * pmd_trans_unstable() via pmd_devmap_trans_unstable() instead @@ -4692,6 +4713,7 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf) } } +skip_pmd_checks: if (!vmf->pte) { if (vma_is_anonymous(vmf->vma)) return do_anonymous_page(vmf); @@ -4976,11 +4998,13 @@ static vm_fault_t ___handle_speculative_fault(struct mm_struct *mm, vmf.vma_flags = READ_ONCE(vmf.vma->vm_flags); vmf.vma_page_prot = READ_ONCE(vmf.vma->vm_page_prot); +#ifdef CONFIG_USERFAULTFD /* Can't call userland page fault handler in the speculative path */ - if (unlikely(vmf.vma_flags & VM_UFFD_MISSING)) { + if (unlikely(vmf.vma_flags & __VM_UFFD_FLAGS)) { trace_spf_vma_notsup(_RET_IP_, vmf.vma, address); return VM_FAULT_RETRY; } +#endif if (vmf.vma_flags & VM_GROWSDOWN || vmf.vma_flags & VM_GROWSUP) { /* @@ -5019,11 +5043,10 @@ static vm_fault_t ___handle_speculative_fault(struct mm_struct *mm, pol = __get_vma_policy(vmf.vma, address); if (!pol) pol = get_task_policy(current); - if (!pol) - if (pol && pol->mode == MPOL_INTERLEAVE) { - trace_spf_vma_notsup(_RET_IP_, vmf.vma, address); - return VM_FAULT_RETRY; - } + if (pol && pol->mode == MPOL_INTERLEAVE) { + trace_spf_vma_notsup(_RET_IP_, vmf.vma, address); + return VM_FAULT_RETRY; + } #endif /* diff --git a/mm/mempolicy.c b/mm/mempolicy.c index d76a6958805c..c9fe24bf18b3 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -657,11 +657,9 @@ unsigned long change_prot_numa(struct vm_area_struct *vma, { int nr_updated; - vm_write_begin(vma); nr_updated = change_protection(vma, addr, end, PAGE_NONE, MM_CP_PROT_NUMA); if (nr_updated) count_vm_numa_events(NUMA_PTE_UPDATES, nr_updated); - vm_write_end(vma); return nr_updated; } diff --git a/mm/mmap.c b/mm/mmap.c index 590068b396b5..84c4c7affe6d 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -3344,6 +3344,7 @@ void exit_mmap(struct mm_struct *mm) vma = remove_vma(vma); cond_resched(); } + mm->mmap = NULL; mmap_write_unlock(mm); vm_unacct_memory(nr_accounted); } diff --git a/mm/mremap.c b/mm/mremap.c index 99b843a91207..6abbaf1eea97 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -210,7 +210,11 @@ static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd, drop_rmap_locks(vma); } -#ifdef CONFIG_HAVE_MOVE_PMD +/* + * Speculative page fault handlers will not detect page table changes done + * without ptl locking. + */ +#if defined(CONFIG_HAVE_MOVE_PMD) && !defined(CONFIG_SPECULATIVE_PAGE_FAULT) static bool move_normal_pmd(struct vm_area_struct *vma, unsigned long old_addr, unsigned long new_addr, pmd_t *old_pmd, pmd_t *new_pmd) { @@ -277,7 +281,11 @@ static inline bool move_normal_pmd(struct vm_area_struct *vma, } #endif -#ifdef CONFIG_HAVE_MOVE_PUD +/* + * Speculative page fault handlers will not detect page table changes done + * without ptl locking. + */ +#if defined(CONFIG_HAVE_MOVE_PUD) && !defined(CONFIG_SPECULATIVE_PAGE_FAULT) static bool move_normal_pud(struct vm_area_struct *vma, unsigned long old_addr, unsigned long new_addr, pud_t *old_pud, pud_t *new_pud) { diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 666d26f72e32..08c53fcda50f 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -8882,6 +8882,8 @@ static int __alloc_contig_migrate_range(struct compact_control *cc, unsigned int tries = 0; unsigned int max_tries = 5; int ret = 0; + bool skip = false; + struct page *page; struct migration_target_control mtc = { .nid = zone_to_nid(cc->zone), @@ -8891,7 +8893,9 @@ static int __alloc_contig_migrate_range(struct compact_control *cc, if (cc->alloc_contig && cc->mode == MIGRATE_ASYNC) max_tries = 1; - lru_cache_disable(); + trace_android_vh_skip_lru_disable(&skip); + if (!skip) + lru_cache_disable(); while (pfn < end || !list_empty(&cc->migratepages)) { if (fatal_signal_pending(current)) { @@ -8926,7 +8930,8 @@ static int __alloc_contig_migrate_range(struct compact_control *cc, info->nr_migrated += cc->nr_migratepages; } - lru_cache_enable(); + if (!skip) + lru_cache_enable(); if (ret < 0) { if (ret == -EBUSY) { alloc_contig_dump_pages(&cc->migratepages); diff --git a/mm/slab.c b/mm/slab.c index eced754636ad..731b868a65c8 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -3428,6 +3428,7 @@ static __always_inline void __cache_free(struct kmem_cache *cachep, void *objp, if (is_kfence_address(objp)) { kmemleak_free_recursive(objp, cachep->flags); + memcg_slab_free_hook(cachep, &objp, 1); __kfence_free(objp); return; } diff --git a/mm/swap.c b/mm/swap.c index 2f69e44ca737..a5394d6ab88f 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -43,6 +43,9 @@ #define CREATE_TRACE_POINTS #include +#undef CREATE_TRACE_POINTS +#include + /* How many pages do we try to swap or page in/out together? */ int page_cluster; @@ -267,6 +270,7 @@ static bool pagevec_add_and_need_flush(struct pagevec *pvec, struct page *page) lru_cache_disabled()) ret = true; + trace_android_vh_pagevec_drain(page, &ret); return ret; } @@ -423,33 +427,31 @@ static void __lru_cache_activate_page(struct page *page) #ifdef CONFIG_LRU_GEN static void page_inc_refs(struct page *page) { - unsigned long refs; - unsigned long old_flags, new_flags; + unsigned long new_flags, old_flags; if (PageUnevictable(page)) return; + if (!PageReferenced(page)) { + SetPageReferenced(page); + return; + } + + if (!PageWorkingset(page)) { + SetPageWorkingset(page); + return; + } + /* see the comment on MAX_NR_TIERS */ do { - new_flags = old_flags = READ_ONCE(page->flags); + old_flags = READ_ONCE(page->flags); + new_flags = old_flags & LRU_REFS_MASK; + if (new_flags == LRU_REFS_MASK) + break; - if (!(new_flags & BIT(PG_referenced))) { - new_flags |= BIT(PG_referenced); - continue; - } - - if (!(new_flags & BIT(PG_workingset))) { - new_flags |= BIT(PG_workingset); - continue; - } - - refs = new_flags & LRU_REFS_MASK; - refs = min(refs + BIT(LRU_REFS_PGOFF), LRU_REFS_MASK); - - new_flags &= ~LRU_REFS_MASK; - new_flags |= refs; - } while (new_flags != old_flags && - cmpxchg(&page->flags, old_flags, new_flags) != old_flags); + new_flags += BIT(LRU_REFS_PGOFF); + new_flags |= old_flags & ~LRU_REFS_MASK; + } while (cmpxchg(&page->flags, old_flags, new_flags) != old_flags); } #else static void page_inc_refs(struct page *page) diff --git a/mm/vmscan.c b/mm/vmscan.c index d09d63861544..bb80ecfa2871 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -70,6 +70,9 @@ #undef CREATE_TRACE_POINTS #include +#undef CREATE_TRACE_POINTS +#include + EXPORT_TRACEPOINT_SYMBOL_GPL(mm_vmscan_direct_reclaim_begin); EXPORT_TRACEPOINT_SYMBOL_GPL(mm_vmscan_direct_reclaim_end); @@ -134,10 +137,9 @@ struct scan_control { unsigned int file_is_tiny:1; #ifdef CONFIG_LRU_GEN - /* help make better choices when multiple memcgs are available */ + /* help kswapd make better choices among multiple memcgs */ unsigned int memcgs_need_aging:1; - unsigned int memcgs_need_swapping:1; - unsigned int memcgs_avoid_swapping:1; + unsigned long last_reclaimed; #endif /* Allocation order */ @@ -2005,6 +2007,8 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, enum vm_event_item item; struct pglist_data *pgdat = lruvec_pgdat(lruvec); bool stalled = false; + struct blk_plug plug; + bool do_plug = false; while (unlikely(too_many_isolated(pgdat, file, sc))) { if (stalled) @@ -2038,6 +2042,9 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, if (nr_taken == 0) return 0; + trace_android_vh_shrink_inactive_list_blk_plug(&do_plug); + if (do_plug) + blk_start_plug(&plug); nr_reclaimed = shrink_page_list(&page_list, pgdat, sc, &stat, false); spin_lock_irq(&pgdat->lru_lock); @@ -2054,6 +2061,9 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, spin_unlock_irq(&pgdat->lru_lock); + if (do_plug) + blk_finish_plug(&plug); + mem_cgroup_uncharge_list(&page_list); free_unref_page_list(&page_list); @@ -2188,6 +2198,8 @@ unsigned long reclaim_pages(struct list_head *page_list) LIST_HEAD(node_page_list); struct reclaim_stat dummy_stat; struct page *page; + struct blk_plug plug; + bool do_plug = false; struct scan_control sc = { .gfp_mask = GFP_KERNEL, .priority = DEF_PRIORITY, @@ -2196,6 +2208,10 @@ unsigned long reclaim_pages(struct list_head *page_list) .may_swap = 1, }; + trace_android_vh_reclaim_pages_plug(&do_plug); + if (do_plug) + blk_start_plug(&plug); + while (!list_empty(page_list)) { page = lru_to_page(page_list); if (nid == NUMA_NO_NODE) { @@ -2231,6 +2247,8 @@ unsigned long reclaim_pages(struct list_head *page_list) putback_lru_page(page); } } + if (do_plug) + blk_finish_plug(&plug); return nr_reclaimed; } @@ -2622,14 +2640,18 @@ out: #ifdef CONFIG_LRU_GEN_ENABLED DEFINE_STATIC_KEY_ARRAY_TRUE(lru_gen_caps, NR_LRU_GEN_CAPS); +#define get_cap(cap) static_branch_likely(&lru_gen_caps[cap]) #else DEFINE_STATIC_KEY_ARRAY_FALSE(lru_gen_caps, NR_LRU_GEN_CAPS); +#define get_cap(cap) static_branch_unlikely(&lru_gen_caps[cap]) #endif /****************************************************************************** * shorthand helpers ******************************************************************************/ +#define LRU_REFS_FLAGS (BIT(PG_referenced) | BIT(PG_workingset)) + #define DEFINE_MAX_SEQ(lruvec) \ unsigned long max_seq = READ_ONCE((lruvec)->lrugen.max_seq) @@ -2644,33 +2666,6 @@ DEFINE_STATIC_KEY_ARRAY_FALSE(lru_gen_caps, NR_LRU_GEN_CAPS); for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \ for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++) -static int page_lru_gen(struct page *page) -{ - unsigned long flags = READ_ONCE(page->flags); - - return ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; -} - -static int page_lru_tier(struct page *page) -{ - int refs; - unsigned long flags = READ_ONCE(page->flags); - - refs = (flags & LRU_REFS_FLAGS) == LRU_REFS_FLAGS ? - ((flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF) + 1 : 0; - - return lru_tier_from_refs(refs); -} - -static bool get_cap(int cap) -{ -#ifdef CONFIG_LRU_GEN_ENABLED - return static_branch_likely(&lru_gen_caps[cap]); -#else - return static_branch_unlikely(&lru_gen_caps[cap]); -#endif -} - static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid) { struct pglist_data *pgdat = NODE_DATA(nid); @@ -2686,7 +2681,7 @@ static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid) return lruvec; } #endif - VM_BUG_ON(!mem_cgroup_disabled()); + VM_WARN_ON_ONCE(!mem_cgroup_disabled()); return pgdat ? &pgdat->__lruvec : NULL; } @@ -2729,7 +2724,7 @@ static struct lru_gen_mm_list *get_mm_list(struct mem_cgroup *memcg) if (memcg) return &memcg->mm_list; #endif - VM_BUG_ON(!mem_cgroup_disabled()); + VM_WARN_ON_ONCE(!mem_cgroup_disabled()); return &mm_list; } @@ -2740,9 +2735,9 @@ void lru_gen_add_mm(struct mm_struct *mm) struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm); struct lru_gen_mm_list *mm_list = get_mm_list(memcg); - VM_BUG_ON_MM(!list_empty(&mm->lru_gen.list), mm); + VM_WARN_ON_ONCE(!list_empty(&mm->lru_gen.list)); #ifdef CONFIG_MEMCG - VM_BUG_ON_MM(mm->lru_gen.memcg, mm); + VM_WARN_ON_ONCE(mm->lru_gen.memcg); mm->lru_gen.memcg = memcg; #endif spin_lock(&mm_list->lock); @@ -2753,6 +2748,7 @@ void lru_gen_add_mm(struct mm_struct *mm) if (!lruvec) continue; + /* the first addition since the last iteration */ if (lruvec->mm_state.tail == &mm_list->fifo) lruvec->mm_state.tail = &mm->lru_gen.list; } @@ -2784,13 +2780,16 @@ void lru_gen_del_mm(struct mm_struct *mm) if (!lruvec) continue; + /* where the last iteration ended (exclusive) */ if (lruvec->mm_state.tail == &mm->lru_gen.list) lruvec->mm_state.tail = lruvec->mm_state.tail->next; + /* where the current iteration continues (inclusive) */ if (lruvec->mm_state.head != &mm->lru_gen.list) continue; lruvec->mm_state.head = lruvec->mm_state.head->next; + /* the deletion ends the current iteration */ if (lruvec->mm_state.head == &mm_list->fifo) WRITE_ONCE(lruvec->mm_state.seq, lruvec->mm_state.seq + 1); } @@ -2809,21 +2808,23 @@ void lru_gen_del_mm(struct mm_struct *mm) void lru_gen_migrate_mm(struct mm_struct *mm) { struct mem_cgroup *memcg; + struct task_struct *task = rcu_dereference_protected(mm->owner, true); - lockdep_assert_held(&mm->owner->alloc_lock); + VM_WARN_ON_ONCE(task->mm != mm); + lockdep_assert_held(&task->alloc_lock); /* for mm_update_next_owner() */ if (mem_cgroup_disabled()) return; rcu_read_lock(); - memcg = mem_cgroup_from_task(mm->owner); + memcg = mem_cgroup_from_task(task); rcu_read_unlock(); if (memcg == mm->lru_gen.memcg) return; - VM_BUG_ON_MM(!mm->lru_gen.memcg, mm); - VM_BUG_ON_MM(list_empty(&mm->lru_gen.list), mm); + VM_WARN_ON_ONCE(!mm->lru_gen.memcg); + VM_WARN_ON_ONCE(list_empty(&mm->lru_gen.list)); lru_gen_del_mm(mm); lru_gen_add_mm(mm); @@ -2874,15 +2875,14 @@ static void reset_bloom_filter(struct lruvec *lruvec, unsigned long seq) unsigned long *filter; int gen = filter_gen_from_seq(seq); - lockdep_assert_held(&get_mm_list(lruvec_memcg(lruvec))->lock); - filter = lruvec->mm_state.filters[gen]; if (filter) { bitmap_clear(filter, 0, BIT(BLOOM_FILTER_SHIFT)); return; } - filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT), GFP_ATOMIC); + filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT), + __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); WRITE_ONCE(lruvec->mm_state.filters[gen], filter); } @@ -2949,12 +2949,12 @@ static bool should_skip_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk) int type; unsigned long size = 0; struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); + int key = pgdat->node_id; - if (!walk->full_scan && cpumask_empty(mm_cpumask(mm)) && - !node_isset(pgdat->node_id, mm->lru_gen.nodes)) + if (!walk->full_scan && !node_isset(key, mm->lru_gen.nodes)) return true; - node_clear(pgdat->node_id, mm->lru_gen.nodes); + node_clear(key, mm->lru_gen.nodes); for (type = !walk->can_swap; type < ANON_AND_FILE; type++) { size += type ? get_mm_counter(mm, MM_FILEPAGES) : @@ -2965,9 +2965,6 @@ static bool should_skip_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk) if (size < MIN_LRU_BATCH) return true; - if (mm_is_oom_victim(mm)) - return true; - return !mmget_not_zero(mm); } @@ -2984,7 +2981,7 @@ static bool iterate_mm_list(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, /* * There are four interesting cases for this page table walker: * 1. It tries to start a new iteration of mm_list with a stale max_seq; - * there is nothing to be done. + * there is nothing left to do. * 2. It's the first of the current generation, and it needs to reset * the Bloom filter for the next generation. * 3. It reaches the end of mm_list, and it needs to increment @@ -2992,16 +2989,11 @@ static bool iterate_mm_list(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, * 4. It's the last of the current generation, and it needs to reset the * mm stats counters for the next generation. */ - if (*iter) - mmput_async(*iter); - else if (walk->max_seq <= READ_ONCE(mm_state->seq)) - return false; - spin_lock(&mm_list->lock); - VM_BUG_ON(mm_state->seq + 1 < walk->max_seq); - VM_BUG_ON(*iter && mm_state->seq > walk->max_seq); - VM_BUG_ON(*iter && !mm_state->nr_walkers); + VM_WARN_ON_ONCE(mm_state->seq + 1 < walk->max_seq); + VM_WARN_ON_ONCE(*iter && mm_state->seq > walk->max_seq); + VM_WARN_ON_ONCE(*iter && !mm_state->nr_walkers); if (walk->max_seq <= mm_state->seq) { if (!*iter) @@ -3010,7 +3002,7 @@ static bool iterate_mm_list(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, } if (!mm_state->nr_walkers) { - VM_BUG_ON(mm_state->head && mm_state->head != &mm_list->fifo); + VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo); mm_state->head = mm_list->fifo.next; first = true; @@ -3021,7 +3013,7 @@ static bool iterate_mm_list(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, mm_state->head = mm_state->head->next; - /* full scan for those added after the last iteration */ + /* force scan for those added after the last iteration */ if (!mm_state->tail || mm_state->tail == &mm->lru_gen.list) { mm_state->tail = mm_state->head; walk->full_scan = true; @@ -3042,14 +3034,17 @@ done: if (mm_state->nr_walkers) last = false; - if (mm && first) - reset_bloom_filter(lruvec, walk->max_seq + 1); - if (*iter || last) reset_mm_stats(lruvec, walk, last); spin_unlock(&mm_list->lock); + if (mm && first) + reset_bloom_filter(lruvec, walk->max_seq + 1); + + if (*iter) + mmput_async(*iter); + *iter = mm; return last; @@ -3062,15 +3057,12 @@ static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long max_seq) struct lru_gen_mm_list *mm_list = get_mm_list(memcg); struct lru_gen_mm_state *mm_state = &lruvec->mm_state; - if (max_seq <= READ_ONCE(mm_state->seq)) - return false; - spin_lock(&mm_list->lock); - VM_BUG_ON(mm_state->seq + 1 < max_seq); + VM_WARN_ON_ONCE(mm_state->seq + 1 < max_seq); if (max_seq > mm_state->seq && !mm_state->nr_walkers) { - VM_BUG_ON(mm_state->head && mm_state->head != &mm_list->fifo); + VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo); WRITE_ONCE(mm_state->seq, mm_state->seq + 1); reset_mm_stats(lruvec, NULL, true); @@ -3098,7 +3090,7 @@ static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long max_seq) * (PV) is either any tier of the other type or any other tier of the same * type. * - * The error is the difference between the SP and the PV; the correction is + * The error is the difference between the SP and the PV; the correction is to * turn off protection when SP>PV or turn on protection when SP= MAX_NR_GENS); - VM_BUG_ON(!rcu_read_lock_held()); + VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); + VM_WARN_ON_ONCE(!rcu_read_lock_held()); do { - new_flags = old_flags = READ_ONCE(page->flags); + old_flags = READ_ONCE(page->flags); - /* for shrink_page_list() */ - if (!(new_flags & LRU_GEN_MASK)) { - new_flags |= BIT(PG_referenced); + /* lru_gen_del_page() has isolated this page? */ + if (!(old_flags & LRU_GEN_MASK)) { + /* for shrink_page_list() */ + new_flags = old_flags | BIT(PG_referenced); continue; } - new_flags &= ~LRU_GEN_MASK; + new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS); new_flags |= (gen + 1UL) << LRU_GEN_PGOFF; - new_flags &= ~(LRU_REFS_MASK | LRU_REFS_FLAGS); - } while (new_flags != old_flags && - cmpxchg(&page->flags, old_flags, new_flags) != old_flags); + } while (cmpxchg(&page->flags, old_flags, new_flags) != old_flags); return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; } +/* protect pages accessed multiple times through file descriptors */ static int page_inc_gen(struct lruvec *lruvec, struct page *page, bool reclaiming) { - unsigned long old_flags, new_flags; int type = page_is_file_lru(page); struct lru_gen_struct *lrugen = &lruvec->lrugen; int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); + unsigned long new_flags, old_flags; do { - new_flags = old_flags = READ_ONCE(page->flags); - VM_BUG_ON_PAGE(!(new_flags & LRU_GEN_MASK), page); + old_flags = READ_ONCE(page->flags); - new_gen = ((new_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; + VM_WARN_ON_ONCE_PAGE(!(old_flags & LRU_GEN_MASK), page); + + new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; /* page_update_gen() has promoted this page? */ if (new_gen >= 0 && new_gen != old_gen) return new_gen; new_gen = (old_gen + 1) % MAX_NR_GENS; - new_flags &= ~LRU_GEN_MASK; + new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS); new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF; - new_flags &= ~(LRU_REFS_MASK | LRU_REFS_FLAGS); /* for end_page_writeback() */ if (reclaiming) new_flags |= BIT(PG_reclaim); @@ -3242,8 +3235,8 @@ static void update_batch_size(struct lru_gen_mm_walk *walk, struct page *page, int zone = page_zonenum(page); int delta = thp_nr_pages(page); - VM_BUG_ON(old_gen >= MAX_NR_GENS); - VM_BUG_ON(new_gen >= MAX_NR_GENS); + VM_WARN_ON_ONCE(old_gen >= MAX_NR_GENS); + VM_WARN_ON_ONCE(new_gen >= MAX_NR_GENS); walk->batched++; @@ -3275,19 +3268,26 @@ static void reset_batch_size(struct lruvec *lruvec, struct lru_gen_mm_walk *walk } } -static int should_skip_vma(unsigned long start, unsigned long end, struct mm_walk *walk) +static int should_skip_vma(unsigned long start, unsigned long end, struct mm_walk *args) { struct address_space *mapping; - struct vm_area_struct *vma = walk->vma; - struct lru_gen_mm_walk *priv = walk->private; + struct vm_area_struct *vma = args->vma; + struct lru_gen_mm_walk *walk = args->private; - if (!vma_is_accessible(vma) || is_vm_hugetlb_page(vma) || - (vma->vm_flags & (VM_LOCKED | VM_SPECIAL | VM_SEQ_READ | VM_RAND_READ)) || - vma == get_gate_vma(vma->vm_mm)) + if (!vma_is_accessible(vma)) + return true; + + if (is_vm_hugetlb_page(vma)) + return true; + + if (vma->vm_flags & (VM_LOCKED | VM_SPECIAL | VM_SEQ_READ | VM_RAND_READ)) + return true; + + if (vma == get_gate_vma(vma->vm_mm)) return true; if (vma_is_anonymous(vma)) - return !priv->can_swap; + return !walk->can_swap; if (WARN_ON_ONCE(!vma->vm_file || !vma->vm_file->f_mapping)) return true; @@ -3296,8 +3296,11 @@ static int should_skip_vma(unsigned long start, unsigned long end, struct mm_wal if (mapping_unevictable(mapping)) return true; - /* check readpage to exclude special mappings like dax, etc. */ - return shmem_mapping(mapping) ? !priv->can_swap : !mapping->a_ops->readpage; + if (shmem_mapping(mapping)) + return !walk->can_swap; + + /* to exclude special mappings like dax, etc. */ + return !mapping->a_ops->readpage; } /* @@ -3305,33 +3308,31 @@ static int should_skip_vma(unsigned long start, unsigned long end, struct mm_wal * returning back to the PGD table for each of such VMAs, finish an entire PMD * table to reduce zigzags and improve cache performance. */ -static bool get_next_vma(struct mm_walk *walk, unsigned long mask, unsigned long size, - unsigned long *start, unsigned long *end) +static bool get_next_vma(unsigned long mask, unsigned long size, struct mm_walk *args, + unsigned long *vm_start, unsigned long *vm_end) { - unsigned long next = round_up(*end, size); + unsigned long start = round_up(*vm_end, size); + unsigned long end = (start | ~mask) + 1; - VM_BUG_ON(mask & size); - VM_BUG_ON(*start >= *end); - VM_BUG_ON((next & mask) != (*start & mask)); + VM_WARN_ON_ONCE(mask & size); + VM_WARN_ON_ONCE((start & mask) != (*vm_start & mask)); - while (walk->vma) { - if (next >= walk->vma->vm_end) { - walk->vma = walk->vma->vm_next; + while (args->vma) { + if (start >= args->vma->vm_end) { + args->vma = args->vma->vm_next; continue; } - if ((next & mask) != (walk->vma->vm_start & mask)) + if (end && end <= args->vma->vm_start) return false; - if (should_skip_vma(walk->vma->vm_start, walk->vma->vm_end, walk)) { - walk->vma = walk->vma->vm_next; + if (should_skip_vma(args->vma->vm_start, args->vma->vm_end, args)) { + args->vma = args->vma->vm_next; continue; } - *start = max(next, walk->vma->vm_start); - next = (next | ~mask) + 1; - /* rounded-up boundaries can wrap to 0 */ - *end = next && next < walk->vma->vm_end ? next : walk->vma->vm_end; + *vm_start = max(start, args->vma->vm_start); + *vm_end = min(end - 1, args->vma->vm_end - 1) + 1; return true; } @@ -3339,6 +3340,67 @@ static bool get_next_vma(struct mm_walk *walk, unsigned long mask, unsigned long return false; } +static unsigned long get_pte_pfn(pte_t pte, struct vm_area_struct *vma, unsigned long addr) +{ + unsigned long pfn = pte_pfn(pte); + + VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); + + if (!pte_present(pte) || is_zero_pfn(pfn)) + return -1; + + if (WARN_ON_ONCE(pte_devmap(pte) || pte_special(pte))) + return -1; + + if (WARN_ON_ONCE(!pfn_valid(pfn))) + return -1; + + return pfn; +} + +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) +static unsigned long get_pmd_pfn(pmd_t pmd, struct vm_area_struct *vma, unsigned long addr) +{ + unsigned long pfn = pmd_pfn(pmd); + + VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); + + if (!pmd_present(pmd) || is_huge_zero_pmd(pmd)) + return -1; + + if (WARN_ON_ONCE(pmd_devmap(pmd))) + return -1; + + if (WARN_ON_ONCE(!pfn_valid(pfn))) + return -1; + + return pfn; +} +#endif + +static struct page *get_pfn_page(unsigned long pfn, struct mem_cgroup *memcg, + struct pglist_data *pgdat, bool can_swap) +{ + struct page *page; + + /* try to avoid unnecessary memory loads */ + if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) + return NULL; + + page = compound_head(pfn_to_page(pfn)); + if (page_to_nid(page) != pgdat->node_id) + return NULL; + + if (page_memcg_rcu(page) != memcg) + return NULL; + + /* file VMAs can contain anon pages from COW */ + if (!page_is_file_lru(page) && !can_swap) + return NULL; + + return page; +} + static bool suitable_to_scan(int total, int young) { int n = clamp_t(int, cache_line_size() / sizeof(pte_t), 2, 8); @@ -3348,7 +3410,7 @@ static bool suitable_to_scan(int total, int young) } static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, - struct mm_walk *walk) + struct mm_walk *args) { int i; pte_t *pte; @@ -3356,14 +3418,14 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, unsigned long addr; int total = 0; int young = 0; - struct lru_gen_mm_walk *priv = walk->private; - struct mem_cgroup *memcg = lruvec_memcg(priv->lruvec); - struct pglist_data *pgdat = lruvec_pgdat(priv->lruvec); - int old_gen, new_gen = lru_gen_from_seq(priv->max_seq); + struct lru_gen_mm_walk *walk = args->private; + struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); + struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); + int old_gen, new_gen = lru_gen_from_seq(walk->max_seq); - VM_BUG_ON(pmd_leaf(*pmd)); + VM_WARN_ON_ONCE(pmd_leaf(*pmd)); - ptl = pte_lockptr(walk->mm, pmd); + ptl = pte_lockptr(args->mm, pmd); if (!spin_trylock(ptl)) return false; @@ -3372,52 +3434,42 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, pte = pte_offset_map(pmd, start & PMD_MASK); restart: for (i = pte_index(start), addr = start; addr != end; i++, addr += PAGE_SIZE) { + unsigned long pfn; struct page *page; - unsigned long pfn = pte_pfn(pte[i]); - - VM_BUG_ON(addr < walk->vma->vm_start || addr >= walk->vma->vm_end); total++; - priv->mm_stats[MM_PTE_TOTAL]++; + walk->mm_stats[MM_LEAF_TOTAL]++; - if (!pte_present(pte[i]) || is_zero_pfn(pfn)) - continue; - - if (WARN_ON_ONCE(pte_devmap(pte[i]) || pte_special(pte[i]))) + pfn = get_pte_pfn(pte[i], args->vma, addr); + if (pfn == -1) continue; if (!pte_young(pte[i])) { - priv->mm_stats[MM_PTE_OLD]++; + walk->mm_stats[MM_LEAF_OLD]++; continue; } - VM_BUG_ON(!pfn_valid(pfn)); - if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) + page = get_pfn_page(pfn, memcg, pgdat, walk->can_swap); + if (!page) continue; - page = compound_head(pfn_to_page(pfn)); - if (page_to_nid(page) != pgdat->node_id) - continue; - - if (page_memcg_rcu(page) != memcg) - continue; - - if (!ptep_test_and_clear_young(walk->vma, addr, pte + i)) - continue; + if (!ptep_test_and_clear_young(args->vma, addr, pte + i)) + VM_WARN_ON_ONCE(true); young++; - priv->mm_stats[MM_PTE_YOUNG]++; + walk->mm_stats[MM_LEAF_YOUNG]++; if (pte_dirty(pte[i]) && !PageDirty(page) && - !(PageAnon(page) && PageSwapBacked(page) && !PageSwapCache(page))) + !(PageAnon(page) && PageSwapBacked(page) && + !PageSwapCache(page))) set_page_dirty(page); old_gen = page_update_gen(page, new_gen); if (old_gen >= 0 && old_gen != new_gen) - update_batch_size(priv, page, old_gen, new_gen); + update_batch_size(walk, page, old_gen, new_gen); } - if (i < PTRS_PER_PTE && get_next_vma(walk, PMD_MASK, PAGE_SIZE, &start, &end)) + if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end)) goto restart; pte_unmap(pte); @@ -3430,17 +3482,17 @@ restart: #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma, - struct mm_walk *walk, unsigned long *start) + struct mm_walk *args, unsigned long *bitmap, unsigned long *start) { int i; pmd_t *pmd; spinlock_t *ptl; - struct lru_gen_mm_walk *priv = walk->private; - struct mem_cgroup *memcg = lruvec_memcg(priv->lruvec); - struct pglist_data *pgdat = lruvec_pgdat(priv->lruvec); - int old_gen, new_gen = lru_gen_from_seq(priv->max_seq); + struct lru_gen_mm_walk *walk = args->private; + struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); + struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); + int old_gen, new_gen = lru_gen_from_seq(walk->max_seq); - VM_BUG_ON(pud_leaf(*pud)); + VM_WARN_ON_ONCE(pud_leaf(*pud)); /* try to batch at most 1+MIN_LRU_BATCH+1 entries */ if (*start == -1) { @@ -3450,29 +3502,25 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area i = next == -1 ? 0 : pmd_index(next) - pmd_index(*start); if (i && i <= MIN_LRU_BATCH) { - __set_bit(i - 1, priv->bitmap); + __set_bit(i - 1, bitmap); return; } pmd = pmd_offset(pud, *start); - ptl = pmd_lockptr(walk->mm, pmd); + ptl = pmd_lockptr(args->mm, pmd); if (!spin_trylock(ptl)) goto done; arch_enter_lazy_mmu_mode(); do { + unsigned long pfn; struct page *page; - unsigned long pfn = pmd_pfn(pmd[i]); unsigned long addr = i ? (*start & PMD_MASK) + i * PMD_SIZE : *start; - VM_BUG_ON(addr < vma->vm_start || addr >= vma->vm_end); - - if (!pmd_present(pmd[i]) || is_huge_zero_pmd(pmd[i])) - goto next; - - if (WARN_ON_ONCE(pmd_devmap(pmd[i]))) + pfn = get_pmd_pfn(pmd[i], vma, addr); + if (pfn == -1) goto next; if (!pmd_trans_huge(pmd[i])) { @@ -3482,50 +3530,42 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area goto next; } - VM_BUG_ON(!pfn_valid(pfn)); - if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) - goto next; - - page = pfn_to_page(pfn); - VM_BUG_ON_PAGE(PageTail(page), page); - if (page_to_nid(page) != pgdat->node_id) - goto next; - - if (page_memcg_rcu(page) != memcg) + page = get_pfn_page(pfn, memcg, pgdat, walk->can_swap); + if (!page) goto next; if (!pmdp_test_and_clear_young(vma, addr, pmd + i)) goto next; - priv->mm_stats[MM_PTE_YOUNG]++; + walk->mm_stats[MM_LEAF_YOUNG]++; if (pmd_dirty(pmd[i]) && !PageDirty(page) && - !(PageAnon(page) && PageSwapBacked(page) && !PageSwapCache(page))) + !(PageAnon(page) && PageSwapBacked(page) && + !PageSwapCache(page))) set_page_dirty(page); old_gen = page_update_gen(page, new_gen); if (old_gen >= 0 && old_gen != new_gen) - update_batch_size(priv, page, old_gen, new_gen); + update_batch_size(walk, page, old_gen, new_gen); next: - i = i > MIN_LRU_BATCH ? 0 : - find_next_bit(priv->bitmap, MIN_LRU_BATCH, i) + 1; + i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1; } while (i <= MIN_LRU_BATCH); arch_leave_lazy_mmu_mode(); spin_unlock(ptl); done: *start = -1; - bitmap_zero(priv->bitmap, MIN_LRU_BATCH); + bitmap_zero(bitmap, MIN_LRU_BATCH); } #else static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma, - struct mm_walk *walk, unsigned long *start) + struct mm_walk *args, unsigned long *bitmap, unsigned long *start) { } #endif static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, - struct mm_walk *walk) + struct mm_walk *args) { int i; pmd_t *pmd; @@ -3533,9 +3573,10 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, unsigned long addr; struct vm_area_struct *vma; unsigned long pos = -1; - struct lru_gen_mm_walk *priv = walk->private; + struct lru_gen_mm_walk *walk = args->private; + unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {}; - VM_BUG_ON(pud_leaf(*pud)); + VM_WARN_ON_ONCE(pud_leaf(*pud)); /* * Finish an entire PMD in two passes: the first only reaches to PTE @@ -3545,7 +3586,7 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, pmd = pmd_offset(pud, start & PUD_MASK); restart: /* walk_pte_range() may call get_next_vma() */ - vma = walk->vma; + vma = args->vma; for (i = pmd_index(start), addr = start; addr != end; i++, addr = next) { pmd_t val = pmd_read_atomic(pmd + i); @@ -3554,73 +3595,71 @@ restart: next = pmd_addr_end(addr, end); - if (!pmd_present(val)) { - priv->mm_stats[MM_PTE_TOTAL]++; + if (!pmd_present(val) || is_huge_zero_pmd(val)) { + walk->mm_stats[MM_LEAF_TOTAL]++; continue; } #ifdef CONFIG_TRANSPARENT_HUGEPAGE if (pmd_trans_huge(val)) { unsigned long pfn = pmd_pfn(val); - struct pglist_data *pgdat = lruvec_pgdat(priv->lruvec); + struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); - priv->mm_stats[MM_PTE_TOTAL]++; - - if (is_huge_zero_pmd(val)) - continue; + walk->mm_stats[MM_LEAF_TOTAL]++; if (!pmd_young(val)) { - priv->mm_stats[MM_PTE_OLD]++; + walk->mm_stats[MM_LEAF_OLD]++; continue; } + /* try to avoid unnecessary memory loads */ if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) continue; - walk_pmd_range_locked(pud, addr, vma, walk, &pos); + walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos); continue; } #endif - priv->mm_stats[MM_PMD_TOTAL]++; + walk->mm_stats[MM_NONLEAF_TOTAL]++; #ifdef CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG if (get_cap(LRU_GEN_NONLEAF_YOUNG)) { if (!pmd_young(val)) continue; - walk_pmd_range_locked(pud, addr, vma, walk, &pos); + walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos); } #endif - if (!priv->full_scan && !test_bloom_filter(priv->lruvec, priv->max_seq, pmd + i)) + if (!walk->full_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i)) continue; - priv->mm_stats[MM_PMD_FOUND]++; + walk->mm_stats[MM_NONLEAF_FOUND]++; - if (!walk_pte_range(&val, addr, next, walk)) + if (!walk_pte_range(&val, addr, next, args)) continue; - priv->mm_stats[MM_PMD_ADDED]++; + walk->mm_stats[MM_NONLEAF_ADDED]++; /* carry over to the next generation */ - update_bloom_filter(priv->lruvec, priv->max_seq + 1, pmd + i); + update_bloom_filter(walk->lruvec, walk->max_seq + 1, pmd + i); } - walk_pmd_range_locked(pud, -1, vma, walk, &pos); + walk_pmd_range_locked(pud, -1, vma, args, bitmap, &pos); - if (i < PTRS_PER_PMD && get_next_vma(walk, PUD_MASK, PMD_SIZE, &start, &end)) + if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end)) goto restart; } static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, - struct mm_walk *walk) + struct mm_walk *args) { int i; pud_t *pud; unsigned long addr; unsigned long next; - struct lru_gen_mm_walk *priv = walk->private; + struct lru_gen_mm_walk *walk = args->private; - VM_BUG_ON(p4d_leaf(*p4d)); + VM_WARN_ON_ONCE(p4d_leaf(*p4d)); pud = pud_offset(p4d, start & P4D_MASK); restart: @@ -3632,21 +3671,27 @@ restart: if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val))) continue; - walk_pmd_range(&val, addr, next, walk); + walk_pmd_range(&val, addr, next, args); - if (priv->batched >= MAX_LRU_BATCH) { + /* a racy check to curtail the waiting time */ + if (wq_has_sleeper(&walk->lruvec->mm_state.wait)) + return 1; + + if (need_resched() || walk->batched >= MAX_LRU_BATCH) { end = (addr | ~PUD_MASK) + 1; goto done; } } - if (i < PTRS_PER_PUD && get_next_vma(walk, P4D_MASK, PUD_SIZE, &start, &end)) + if (i < PTRS_PER_PUD && get_next_vma(P4D_MASK, PUD_SIZE, args, &start, &end)) goto restart; end = round_up(end, P4D_SIZE); done: - /* rounded-up boundaries can wrap to 0 */ - priv->next_addr = end && walk->vma ? max(end, walk->vma->vm_start) : 0; + if (!end || !args->vma) + return 1; + + walk->next_addr = max(end, args->vma->vm_start); return -EAGAIN; } @@ -3673,55 +3718,89 @@ static void walk_mm(struct lruvec *lruvec, struct mm_struct *mm, struct lru_gen_ /* the caller might be holding the lock for write */ if (mmap_read_trylock(mm)) { - unsigned long start = walk->next_addr; - unsigned long end = mm->highest_vm_end; - - err = walk_page_range(mm, start, end, &mm_walk_ops, walk); + err = walk_page_range(mm, walk->next_addr, ULONG_MAX, &mm_walk_ops, walk); mmap_read_unlock(mm); - - if (walk->batched) { - spin_lock_irq(&pgdat->lru_lock); - reset_batch_size(lruvec, walk); - spin_unlock_irq(&pgdat->lru_lock); - } } mem_cgroup_unlock_pages(); + if (walk->batched) { + spin_lock_irq(&pgdat->lru_lock); + reset_batch_size(lruvec, walk); + spin_unlock_irq(&pgdat->lru_lock); + } + cond_resched(); - } while (err == -EAGAIN && walk->next_addr && !mm_is_oom_victim(mm)); + } while (err == -EAGAIN); } -static struct lru_gen_mm_walk *alloc_mm_walk(void) +static struct lru_gen_mm_walk *set_mm_walk(struct pglist_data *pgdat) { - if (current->reclaim_state && current->reclaim_state->mm_walk) - return current->reclaim_state->mm_walk; + struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; - return kzalloc(sizeof(struct lru_gen_mm_walk), - __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); + if (pgdat && current_is_kswapd()) { + VM_WARN_ON_ONCE(walk); + + walk = &pgdat->mm_walk; + } else if (!pgdat && !walk) { + VM_WARN_ON_ONCE(current_is_kswapd()); + + walk = kzalloc(sizeof(*walk), __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); + } + + current->reclaim_state->mm_walk = walk; + + return walk; } -static void free_mm_walk(struct lru_gen_mm_walk *walk) +static void clear_mm_walk(void) { - if (!current->reclaim_state || !current->reclaim_state->mm_walk) + struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; + + VM_WARN_ON_ONCE(walk && memchr_inv(walk->nr_pages, 0, sizeof(walk->nr_pages))); + VM_WARN_ON_ONCE(walk && memchr_inv(walk->mm_stats, 0, sizeof(walk->mm_stats))); + + current->reclaim_state->mm_walk = NULL; + + if (!current_is_kswapd()) kfree(walk); } -static void inc_min_seq(struct lruvec *lruvec) +static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap) { - int type; + int zone; + int remaining = MAX_LRU_BATCH; struct lru_gen_struct *lrugen = &lruvec->lrugen; + int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); - VM_BUG_ON(!seq_is_valid(lruvec)); + if (type == LRU_GEN_ANON && !can_swap) + goto done; - for (type = 0; type < ANON_AND_FILE; type++) { - if (get_nr_gens(lruvec, type) != MAX_NR_GENS) - continue; + /* prevent cold/hot inversion if full_scan is true */ + for (zone = 0; zone < MAX_NR_ZONES; zone++) { + struct list_head *head = &lrugen->lists[old_gen][type][zone]; - reset_ctrl_pos(lruvec, type, true); - WRITE_ONCE(lrugen->min_seq[type], lrugen->min_seq[type] + 1); + while (!list_empty(head)) { + struct page *page = lru_to_page(head); + + VM_WARN_ON_ONCE_PAGE(PageUnevictable(page), page); + VM_WARN_ON_ONCE_PAGE(PageActive(page), page); + VM_WARN_ON_ONCE_PAGE(page_is_file_lru(page) != type, page); + VM_WARN_ON_ONCE_PAGE(page_zonenum(page) != zone, page); + + new_gen = page_inc_gen(lruvec, page, false); + list_move_tail(&page->lru, &lrugen->lists[new_gen][type][zone]); + + if (!--remaining) + return false; + } } +done: + reset_ctrl_pos(lruvec, type, true); + WRITE_ONCE(lrugen->min_seq[type], lrugen->min_seq[type] + 1); + + return true; } static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap) @@ -3731,8 +3810,9 @@ static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap) struct lru_gen_struct *lrugen = &lruvec->lrugen; DEFINE_MIN_SEQ(lruvec); - VM_BUG_ON(!seq_is_valid(lruvec)); + VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); + /* find the oldest populated generation */ for (type = !can_swap; type < ANON_AND_FILE; type++) { while (min_seq[type] + MIN_NR_GENS <= lrugen->max_seq) { gen = lru_gen_from_seq(min_seq[type]); @@ -3766,7 +3846,7 @@ next: return success; } -static void inc_max_seq(struct lruvec *lruvec) +static void inc_max_seq(struct lruvec *lruvec, bool can_swap, bool full_scan) { int prev, next; int type, zone; @@ -3775,16 +3855,26 @@ static void inc_max_seq(struct lruvec *lruvec) spin_lock_irq(&pgdat->lru_lock); - VM_BUG_ON(!seq_is_valid(lruvec)); + VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); - inc_min_seq(lruvec); + for (type = ANON_AND_FILE - 1; type >= 0; type--) { + if (get_nr_gens(lruvec, type) != MAX_NR_GENS) + continue; + + VM_WARN_ON_ONCE(!full_scan && (type == LRU_GEN_FILE || can_swap)); + + while (!inc_min_seq(lruvec, type, can_swap)) { + spin_unlock_irq(&pgdat->lru_lock); + cond_resched(); + spin_lock_irq(&pgdat->lru_lock); + } + } /* * Update the active/inactive LRU sizes for compatibility. Both sides of * the current max_seq need to be covered, since max_seq+1 can overlap * with min_seq[LRU_GEN_ANON] if swapping is constrained. And if they do - * overlap, cold/hot inversion happens. This can be solved by moving - * pages from min_seq to min_seq+1 but is omitted for simplicity. + * overlap, cold/hot inversion happens. */ prev = lru_gen_from_seq(lrugen->max_seq - 1); next = lru_gen_from_seq(lrugen->max_seq + 1); @@ -3798,8 +3888,6 @@ static void inc_max_seq(struct lruvec *lruvec) if (!delta) continue; - WARN_ON_ONCE(delta != (int)delta); - __update_lru_size(lruvec, lru, zone, delta); __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, -delta); } @@ -3823,7 +3911,13 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, struct mm_struct *mm = NULL; struct lru_gen_struct *lrugen = &lruvec->lrugen; - VM_BUG_ON(max_seq > READ_ONCE(lrugen->max_seq)); + VM_WARN_ON_ONCE(max_seq > READ_ONCE(lrugen->max_seq)); + + /* see the comment in iterate_mm_list() */ + if (max_seq <= READ_ONCE(lruvec->mm_state.seq)) { + success = false; + goto done; + } /* * If the hardware doesn't automatically set the accessed bit, fallback @@ -3831,12 +3925,12 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, * handful of PTEs. Spreading the work out over a period of time usually * is less efficient, but it avoids bursty page faults. */ - if (!full_scan && (!arch_has_hw_pte_young() || !get_cap(LRU_GEN_MM_WALK))) { + if (!full_scan && !(arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK))) { success = iterate_mm_list_nowalk(lruvec, max_seq); goto done; } - walk = alloc_mm_walk(); + walk = set_mm_walk(NULL); if (!walk) { success = iterate_mm_list_nowalk(lruvec, max_seq); goto done; @@ -3854,119 +3948,114 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, cond_resched(); } while (mm); - - free_mm_walk(walk); done: if (!success) { - if (!current_is_kswapd() && !sc->priority) + if (sc->priority <= DEF_PRIORITY - 2) wait_event_killable(lruvec->mm_state.wait, max_seq < READ_ONCE(lrugen->max_seq)); return max_seq < READ_ONCE(lrugen->max_seq); } - VM_BUG_ON(max_seq != READ_ONCE(lrugen->max_seq)); + VM_WARN_ON_ONCE(max_seq != READ_ONCE(lrugen->max_seq)); - inc_max_seq(lruvec); + inc_max_seq(lruvec, can_swap, full_scan); /* either this sees any waiters or they will see updated max_seq */ if (wq_has_sleeper(&lruvec->mm_state.wait)) wake_up_all(&lruvec->mm_state.wait); - wakeup_flusher_threads(WB_REASON_VMSCAN); - return true; } -static long get_nr_evictable(struct lruvec *lruvec, unsigned long max_seq, - unsigned long *min_seq, bool can_swap, bool *need_aging) +static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, unsigned long *min_seq, + struct scan_control *sc, bool can_swap, unsigned long *nr_to_scan) { int gen, type, zone; - long old = 0; - long young = 0; - long total = 0; + unsigned long old = 0; + unsigned long young = 0; + unsigned long total = 0; struct lru_gen_struct *lrugen = &lruvec->lrugen; + struct mem_cgroup *memcg = lruvec_memcg(lruvec); for (type = !can_swap; type < ANON_AND_FILE; type++) { unsigned long seq; for (seq = min_seq[type]; seq <= max_seq; seq++) { - long size = 0; + unsigned long size = 0; gen = lru_gen_from_seq(seq); for (zone = 0; zone < MAX_NR_ZONES; zone++) - size += READ_ONCE(lrugen->nr_pages[gen][type][zone]); + size += max_t(long, READ_ONCE(lrugen->nr_pages[gen][type][zone]), + 0); total += size; if (seq == max_seq) young += size; - if (seq + MIN_NR_GENS == max_seq) + else if (seq + MIN_NR_GENS == max_seq) old += size; } } - /* - * The aging and the eviction is a typical producer-consumer model. The - * aging tries to be lazy to reduce the unnecessary overhead. On the - * other hand, the eviction stalls when the number of generations - * reaches MIN_NR_GENS. So ideally, there should be MIN_NR_GENS+1 - * generations, hence the first two if's. - * - * In addition, it's ideal to spread pages out evenly, meaning - * 1/(MIN_NR_GENS+1) of the total number of pages for each generation. A - * reasonable range for this average portion would [1/MIN_NR_GENS, - * 1/(MIN_NR_GENS+2)]. From the consumer's POV, the eviction only cares - * about the lower bound of cold pages, i.e., 1/(MIN_NR_GENS+2), whereas - * from the producer's POV, the aging only cares about the upper bound - * of hot pages, i.e., 1/MIN_NR_GENS. - */ - if (min_seq[LRU_GEN_FILE] + MIN_NR_GENS > max_seq) - *need_aging = true; - else if (min_seq[LRU_GEN_FILE] + MIN_NR_GENS < max_seq) - *need_aging = false; - else if (young * MIN_NR_GENS > total) - *need_aging = true; - else if (old * (MIN_NR_GENS + 2) < total) - *need_aging = true; - else - *need_aging = false; + /* try to scrape all its memory if this memcg was deleted */ + *nr_to_scan = mem_cgroup_online(memcg) ? (total >> sc->priority) : total; - return total > 0 ? total : 0; + /* + * The aging tries to be lazy to reduce the overhead, while the eviction + * stalls when the number of generations reaches MIN_NR_GENS. Hence, the + * ideal number of generations is MIN_NR_GENS+1. + */ + if (min_seq[!can_swap] + MIN_NR_GENS > max_seq) + return true; + if (min_seq[!can_swap] + MIN_NR_GENS < max_seq) + return false; + + /* + * It's also ideal to spread pages out evenly, i.e., 1/(MIN_NR_GENS+1) + * of the total number of pages for each generation. A reasonable range + * for this average portion is [1/MIN_NR_GENS, 1/(MIN_NR_GENS+2)]. The + * aging cares about the upper bound of hot pages, while the eviction + * cares about the lower bound of cold pages. + */ + if (young * MIN_NR_GENS > total) + return true; + if (old * (MIN_NR_GENS + 2) < total) + return true; + + return false; } -static bool age_lruvec(struct lruvec *lruvec, struct scan_control *sc, - unsigned long min_ttl) +static bool age_lruvec(struct lruvec *lruvec, struct scan_control *sc, unsigned long min_ttl) { bool need_aging; - long nr_to_scan; + unsigned long nr_to_scan; int swappiness = get_swappiness(lruvec, sc); struct mem_cgroup *memcg = lruvec_memcg(lruvec); DEFINE_MAX_SEQ(lruvec); DEFINE_MIN_SEQ(lruvec); + VM_WARN_ON_ONCE(sc->memcg_low_reclaim); + + mem_cgroup_calculate_protection(NULL, memcg); + + if (mem_cgroup_below_min(memcg)) + return false; + + need_aging = should_run_aging(lruvec, max_seq, min_seq, sc, swappiness, &nr_to_scan); + if (min_ttl) { int gen = lru_gen_from_seq(min_seq[LRU_GEN_FILE]); unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); if (time_is_after_jiffies(birth + min_ttl)) return false; + + /* the size is likely too small to be helpful */ + if (!nr_to_scan && sc->priority != DEF_PRIORITY) + return false; } - mem_cgroup_calculate_protection(NULL, memcg); - - if (mem_cgroup_below_min(memcg)) - return false; - - nr_to_scan = get_nr_evictable(lruvec, max_seq, min_seq, swappiness, &need_aging); - if (!nr_to_scan) - return false; - - nr_to_scan >>= sc->priority; - - if (!mem_cgroup_online(memcg)) - nr_to_scan++; - - if (nr_to_scan && need_aging && (!mem_cgroup_below_low(memcg) || sc->memcg_low_reclaim)) + if (need_aging) try_to_inc_max_seq(lruvec, max_seq, sc, swappiness, false); return true; @@ -3981,25 +4070,22 @@ static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) bool success = false; unsigned long min_ttl = READ_ONCE(lru_gen_min_ttl); - VM_BUG_ON(!current_is_kswapd()); + VM_WARN_ON_ONCE(!current_is_kswapd()); + + sc->last_reclaimed = sc->nr_reclaimed; /* - * To reduce the chance of going into the aging path or swapping, which - * can be costly, optimistically skip them unless their corresponding - * flags were cleared in the eviction path. This improves the overall - * performance when multiple memcgs are available. + * To reduce the chance of going into the aging path, which can be + * costly, optimistically skip it if the flag below was cleared in the + * eviction path. This improves the overall performance when multiple + * memcgs are available. */ if (!sc->memcgs_need_aging) { sc->memcgs_need_aging = true; - sc->memcgs_avoid_swapping = !sc->memcgs_need_swapping; - sc->memcgs_need_swapping = true; return; } - sc->memcgs_need_swapping = true; - sc->memcgs_avoid_swapping = true; - - current->reclaim_state->mm_walk = &pgdat->mm_walk; + set_mm_walk(pgdat); memcg = mem_cgroup_iter(NULL, NULL, NULL); do { @@ -4011,17 +4097,20 @@ static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) cond_resched(); } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); - current->reclaim_state->mm_walk = NULL; + clear_mm_walk(); + + /* check the order to exclude compaction-induced reclaim */ + if (success || !min_ttl || sc->order) + return; /* * The main goal is to OOM kill if every generation from all memcgs is - * younger than min_ttl. However, another theoretical possibility is all - * memcgs are either below min or empty. + * younger than min_ttl. However, another possibility is all memcgs are + * either below min or empty. */ - if (!success && mutex_trylock(&oom_lock)) { + if (mutex_trylock(&oom_lock)) { struct oom_control oc = { .gfp_mask = sc->gfp_mask, - .order = sc->order, }; out_of_memory(&oc); @@ -4032,10 +4121,10 @@ static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) /* * This function exploits spatial locality when shrink_page_list() walks the - * rmap. It scans the adjacent PTEs of a young PTE and promotes hot pages. - * If the scan was done cacheline efficiently, it adds the PMD entry pointing - * to the PTE table to the Bloom filter. This process is a feedback loop from - * the eviction to the aging. + * rmap. It scans the adjacent PTEs of a young PTE and promotes hot pages. If + * the scan was done cacheline efficiently, it adds the PMD entry pointing to + * the PTE table to the Bloom filter. This forms a feedback loop between the + * eviction and the aging. */ void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) { @@ -4044,24 +4133,27 @@ void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) unsigned long start; unsigned long end; unsigned long addr; - struct page *page; struct lru_gen_mm_walk *walk; int young = 0; unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {}; - struct mem_cgroup *memcg = page_memcg(pvmw->page); - struct pglist_data *pgdat = page_pgdat(pvmw->page); + struct page *page = pvmw->page; + struct mem_cgroup *memcg = page_memcg(page); + struct pglist_data *pgdat = page_pgdat(page); struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); DEFINE_MAX_SEQ(lruvec); int old_gen, new_gen = lru_gen_from_seq(max_seq); lockdep_assert_held(pvmw->ptl); - VM_BUG_ON_PAGE(PageLRU(pvmw->page), pvmw->page); + VM_WARN_ON_ONCE_PAGE(PageLRU(page), page); if (spin_is_contended(pvmw->ptl)) return; + /* avoid taking the LRU lock under the PTL when possible */ + walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL; + start = max(pvmw->address & PMD_MASK, pvmw->vma->vm_start); - end = pmd_addr_end(pvmw->address, pvmw->vma->vm_end); + end = min(pvmw->address | ~PMD_MASK, pvmw->vma->vm_end - 1) + 1; if (end - start > MIN_LRU_BATCH * PAGE_SIZE) { if (pvmw->address - start < MIN_LRU_BATCH * PAGE_SIZE / 2) @@ -4080,37 +4172,27 @@ void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) arch_enter_lazy_mmu_mode(); for (i = 0, addr = start; addr != end; i++, addr += PAGE_SIZE) { - unsigned long pfn = pte_pfn(pte[i]); + unsigned long pfn; - VM_BUG_ON(addr < pvmw->vma->vm_start || addr >= pvmw->vma->vm_end); - - if (!pte_present(pte[i]) || is_zero_pfn(pfn)) - continue; - - if (WARN_ON_ONCE(pte_devmap(pte[i]) || pte_special(pte[i]))) + pfn = get_pte_pfn(pte[i], pvmw->vma, addr); + if (pfn == -1) continue; if (!pte_young(pte[i])) continue; - VM_BUG_ON(!pfn_valid(pfn)); - if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) - continue; - - page = compound_head(pfn_to_page(pfn)); - if (page_to_nid(page) != pgdat->node_id) - continue; - - if (page_memcg_rcu(page) != memcg) + page = get_pfn_page(pfn, memcg, pgdat, !walk || walk->can_swap); + if (!page) continue; if (!ptep_test_and_clear_young(pvmw->vma, addr, pte + i)) - continue; + VM_WARN_ON_ONCE(true); young++; if (pte_dirty(pte[i]) && !PageDirty(page) && - !(PageAnon(page) && PageSwapBacked(page) && !PageSwapCache(page))) + !(PageAnon(page) && PageSwapBacked(page) && + !PageSwapCache(page))) set_page_dirty(page); old_gen = page_lru_gen(page); @@ -4127,11 +4209,11 @@ void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) if (suitable_to_scan(i, young)) update_bloom_filter(lruvec, max_seq, pvmw->pmd); - walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL; - if (!walk && bitmap_weight(bitmap, MIN_LRU_BATCH) < PAGEVEC_SIZE) { - for_each_set_bit(i, bitmap, MIN_LRU_BATCH) - activate_page(pte_page(pte[i])); + for_each_set_bit(i, bitmap, MIN_LRU_BATCH) { + page = pte_page(pte[i]); + activate_page(page); + } return; } @@ -4175,34 +4257,39 @@ static bool sort_page(struct lruvec *lruvec, struct page *page, int tier_idx) int gen = page_lru_gen(page); int type = page_is_file_lru(page); int zone = page_zonenum(page); - int tier = page_lru_tier(page); int delta = thp_nr_pages(page); + int refs = page_lru_refs(page); + int tier = lru_tier_from_refs(refs); struct lru_gen_struct *lrugen = &lruvec->lrugen; - VM_BUG_ON_PAGE(gen >= MAX_NR_GENS, page); + VM_WARN_ON_ONCE_PAGE(gen >= MAX_NR_GENS, page); + /* unevictable */ if (!page_evictable(page)) { success = lru_gen_del_page(lruvec, page, true); - VM_BUG_ON_PAGE(!success, page); + VM_WARN_ON_ONCE_PAGE(!success, page); SetPageUnevictable(page); add_page_to_lru_list(page, lruvec); __count_vm_events(UNEVICTABLE_PGCULLED, delta); return true; } + /* dirty lazyfree */ if (type == LRU_GEN_FILE && PageAnon(page) && PageDirty(page)) { success = lru_gen_del_page(lruvec, page, true); - VM_BUG_ON_PAGE(!success, page); + VM_WARN_ON_ONCE_PAGE(!success, page); SetPageSwapBacked(page); add_page_to_lru_list_tail(page, lruvec); return true; } + /* promoted */ if (gen != lru_gen_from_seq(lrugen->min_seq[type])) { list_move(&page->lru, &lrugen->lists[gen][type][zone]); return true; } + /* protected */ if (tier > tier_idx) { int hist = lru_hist_from_seq(lrugen->min_seq[type]); @@ -4215,6 +4302,7 @@ static bool sort_page(struct lruvec *lruvec, struct page *page, int tier_idx) return true; } + /* waiting for writeback */ if (PageLocked(page) || PageWriteback(page) || (type == LRU_GEN_FILE && PageDirty(page))) { gen = page_inc_gen(lruvec, page, true); @@ -4229,20 +4317,32 @@ static bool isolate_page(struct lruvec *lruvec, struct page *page, struct scan_c { bool success; + /* unmapping inhibited */ if (!sc->may_unmap && page_mapped(page)) return false; + /* swapping inhibited */ if (!(sc->may_writepage && (sc->gfp_mask & __GFP_IO)) && - (PageDirty(page) || (PageAnon(page) && !PageSwapCache(page)))) + (PageDirty(page) || + (PageAnon(page) && !PageSwapCache(page)))) return false; + /* raced with release_pages() */ if (!get_page_unless_zero(page)) return false; ClearPageLRU(page); + /* see the comment on MAX_NR_TIERS */ + if (!PageReferenced(page)) + set_mask_bits(&page->flags, LRU_REFS_MASK | LRU_REFS_FLAGS, 0); + + /* for shrink_page_list() */ + ClearPageReclaim(page); + ClearPageReferenced(page); + success = lru_gen_del_page(lruvec, page, true); - VM_BUG_ON_PAGE(!success, page); + VM_WARN_ON_ONCE_PAGE(!success, page); return true; } @@ -4259,7 +4359,7 @@ static int scan_pages(struct lruvec *lruvec, struct scan_control *sc, struct lru_gen_struct *lrugen = &lruvec->lrugen; struct mem_cgroup *memcg = lruvec_memcg(lruvec); - VM_BUG_ON(!list_empty(list)); + VM_WARN_ON_ONCE(!list_empty(list)); if (get_nr_gens(lruvec, type) == MIN_NR_GENS) return 0; @@ -4275,13 +4375,10 @@ static int scan_pages(struct lruvec *lruvec, struct scan_control *sc, struct page *page = lru_to_page(head); int delta = thp_nr_pages(page); - VM_BUG_ON_PAGE(PageTail(page), page); - VM_BUG_ON_PAGE(PageUnevictable(page), page); - VM_BUG_ON_PAGE(PageActive(page), page); - VM_BUG_ON_PAGE(page_is_file_lru(page) != type, page); - VM_BUG_ON_PAGE(page_zonenum(page) != zone, page); - - prefetchw_prev_lru_page(page, head, flags); + VM_WARN_ON_ONCE_PAGE(PageUnevictable(page), page); + VM_WARN_ON_ONCE_PAGE(PageActive(page), page); + VM_WARN_ON_ONCE_PAGE(page_is_file_lru(page) != type, page); + VM_WARN_ON_ONCE_PAGE(page_zonenum(page) != zone, page); scanned += delta; @@ -4319,8 +4416,8 @@ static int scan_pages(struct lruvec *lruvec, struct scan_control *sc, /* * There might not be eligible pages due to reclaim_idx, may_unmap and - * may_writepage. Check the remaining to prevent livelock if there is no - * progress. + * may_writepage. Check the remaining to prevent livelock if it's not + * making progress. */ return isolated || !remaining ? scanned : 0; } @@ -4382,8 +4479,6 @@ static int isolate_pages(struct lruvec *lruvec, struct scan_control *sc, int swa int tier = -1; DEFINE_MIN_SEQ(lruvec); - VM_BUG_ON(!seq_is_valid(lruvec)); - /* * Try to make the obvious choice first. When anon and file are both * available from the same generation, interpret swappiness 1 as file @@ -4418,7 +4513,7 @@ static int isolate_pages(struct lruvec *lruvec, struct scan_control *sc, int swa } static int evict_pages(struct lruvec *lruvec, struct scan_control *sc, int swappiness, - bool *swapped) + bool *need_swapping) { int type; int scanned; @@ -4435,10 +4530,9 @@ static int evict_pages(struct lruvec *lruvec, struct scan_control *sc, int swapp scanned = isolate_pages(lruvec, sc, swappiness, &type, &list); - if (try_to_inc_min_seq(lruvec, swappiness)) - scanned++; + scanned += try_to_inc_min_seq(lruvec, swappiness); - if (get_nr_gens(lruvec, LRU_GEN_FILE) == MIN_NR_GENS) + if (get_nr_gens(lruvec, !swappiness) == MIN_NR_GENS) scanned = 0; spin_unlock_irq(&pgdat->lru_lock); @@ -4448,15 +4542,14 @@ static int evict_pages(struct lruvec *lruvec, struct scan_control *sc, int swapp reclaimed = shrink_page_list(&list, pgdat, sc, &stat, false); - /* - * To avoid livelock, don't add rejected pages back to the same lists - * they were isolated from. See lru_gen_add_page(). - */ list_for_each_entry(page, &list, lru) { - ClearPageReferenced(page); - ClearPageWorkingset(page); + /* restore LRU_REFS_FLAGS cleared by isolate_page() */ + if (PageWorkingset(page)) + SetPageReferenced(page); - if (PageReclaim(page) && (PageDirty(page) || PageWriteback(page))) + /* don't add rejected pages to the oldest generation */ + if (PageReclaim(page) && + (PageDirty(page) || PageWriteback(page))) ClearPageActive(page); else SetPageActive(page); @@ -4466,7 +4559,7 @@ static int evict_pages(struct lruvec *lruvec, struct scan_control *sc, int swapp move_pages_to_lru(lruvec, &list); - walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL; + walk = current->reclaim_state->mm_walk; if (walk && walk->batched) reset_batch_size(lruvec, walk); @@ -4483,16 +4576,21 @@ static int evict_pages(struct lruvec *lruvec, struct scan_control *sc, int swapp sc->nr_reclaimed += reclaimed; - if (type == LRU_GEN_ANON && swapped) - *swapped = true; + if (need_swapping && type == LRU_GEN_ANON) + *need_swapping = true; return scanned; } -static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, bool can_swap) +/* + * For future optimizations: + * 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg + * reclaim. + */ +static unsigned long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, + bool can_swap, bool *need_aging) { - bool need_aging; - long nr_to_scan; + unsigned long nr_to_scan; struct mem_cgroup *memcg = lruvec_memcg(lruvec); DEFINE_MAX_SEQ(lruvec); DEFINE_MIN_SEQ(lruvec); @@ -4501,59 +4599,96 @@ static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, bool (mem_cgroup_below_low(memcg) && !sc->memcg_low_reclaim)) return 0; - nr_to_scan = get_nr_evictable(lruvec, max_seq, min_seq, can_swap, &need_aging); - if (!nr_to_scan) - return 0; - - /* reset the priority if the target has been met */ - nr_to_scan >>= sc->nr_reclaimed < sc->nr_to_reclaim ? sc->priority : DEF_PRIORITY; - - if (!mem_cgroup_online(memcg)) - nr_to_scan++; - - if (!nr_to_scan) - return 0; - - if (!need_aging) { - sc->memcgs_need_aging = false; + *need_aging = should_run_aging(lruvec, max_seq, min_seq, sc, can_swap, &nr_to_scan); + if (!*need_aging) return nr_to_scan; - } + + /* skip the aging path at the default priority */ + if (sc->priority == DEF_PRIORITY) + goto done; /* leave the work to lru_gen_age_node() */ if (current_is_kswapd()) return 0; - /* try other memcgs before going to the aging path */ - if (!cgroup_reclaim(sc) && !sc->force_deactivate) { - sc->skipped_deactivate = true; - return 0; - } - if (try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, false)) return nr_to_scan; +done: + return min_seq[!can_swap] + MIN_NR_GENS <= max_seq ? nr_to_scan : 0; +} - return min_seq[LRU_GEN_FILE] + MIN_NR_GENS <= max_seq ? nr_to_scan : 0; +static bool should_abort_scan(struct lruvec *lruvec, unsigned long seq, + struct scan_control *sc, bool need_swapping) +{ + int i; + DEFINE_MAX_SEQ(lruvec); + + if (!current_is_kswapd()) { + /* age each memcg at most once to ensure fairness */ + if (max_seq - seq > 1) + return true; + + /* over-swapping can increase allocation latency */ + if (sc->nr_reclaimed >= sc->nr_to_reclaim && need_swapping) + return true; + + /* give this thread a chance to exit and free its memory */ + if (fatal_signal_pending(current)) { + sc->nr_reclaimed += MIN_LRU_BATCH; + return true; + } + + if (cgroup_reclaim(sc)) + return false; + } else if (sc->nr_reclaimed - sc->last_reclaimed < sc->nr_to_reclaim) + return false; + + /* keep scanning at low priorities to ensure fairness */ + if (sc->priority > DEF_PRIORITY - 2) + return false; + + /* + * A minimum amount of work was done under global memory pressure. For + * kswapd, it may be overshooting. For direct reclaim, the allocation + * may succeed if all suitable zones are somewhat safe. In either case, + * it's better to stop now, and restart later if necessary. + */ + for (i = 0; i <= sc->reclaim_idx; i++) { + unsigned long wmark; + struct zone *zone = lruvec_pgdat(lruvec)->node_zones + i; + + if (!managed_zone(zone)) + continue; + + wmark = current_is_kswapd() ? high_wmark_pages(zone) : low_wmark_pages(zone); + if (wmark > zone_page_state(zone, NR_FREE_PAGES)) + return false; + } + + sc->nr_reclaimed += MIN_LRU_BATCH; + + return true; } static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) { struct blk_plug plug; - long scanned = 0; - bool swapped = false; + bool need_aging = false; + bool need_swapping = false; + unsigned long scanned = 0; unsigned long reclaimed = sc->nr_reclaimed; - struct pglist_data *pgdat = lruvec_pgdat(lruvec); + DEFINE_MAX_SEQ(lruvec); lru_add_drain(); blk_start_plug(&plug); - if (current_is_kswapd()) - current->reclaim_state->mm_walk = &pgdat->mm_walk; + set_mm_walk(lruvec_pgdat(lruvec)); while (true) { int delta; int swappiness; - long nr_to_scan; + unsigned long nr_to_scan; if (sc->may_swap) swappiness = get_swappiness(lruvec, sc); @@ -4562,29 +4697,29 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc else swappiness = 0; - nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness); + nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness, &need_aging); if (!nr_to_scan) - break; + goto done; - delta = evict_pages(lruvec, sc, swappiness, &swapped); + delta = evict_pages(lruvec, sc, swappiness, &need_swapping); if (!delta) - break; - - if (sc->memcgs_avoid_swapping && swappiness < 200 && swapped) - break; + goto done; scanned += delta; - if (scanned >= nr_to_scan) { - if (!swapped && sc->nr_reclaimed - reclaimed >= MIN_LRU_BATCH) - sc->memcgs_need_swapping = false; + if (scanned >= nr_to_scan) + break; + + if (should_abort_scan(lruvec, max_seq, sc, need_swapping)) break; - } cond_resched(); } - if (current_is_kswapd()) - current->reclaim_state->mm_walk = NULL; + /* see the comment in lru_gen_age_node() */ + if (sc->nr_reclaimed - reclaimed >= MIN_LRU_BATCH && !need_aging) + sc->memcgs_need_aging = false; +done: + clear_mm_walk(); blk_finish_plug(&plug); } @@ -4610,9 +4745,6 @@ static bool __maybe_unused state_is_valid(struct lruvec *lruvec) for_each_gen_type_zone(gen, type, zone) { if (!list_empty(&lrugen->lists[gen][type][zone])) return false; - - /* unlikely but not a bug when reset_batch_size() is pending */ - VM_WARN_ON(lrugen->nr_pages[gen][type][zone]); } } @@ -4633,17 +4765,14 @@ static bool fill_evictable(struct lruvec *lruvec) bool success; struct page *page = lru_to_page(head); - VM_BUG_ON_PAGE(PageTail(page), page); - VM_BUG_ON_PAGE(PageUnevictable(page), page); - VM_BUG_ON_PAGE(PageActive(page) != active, page); - VM_BUG_ON_PAGE(page_is_file_lru(page) != type, page); - VM_BUG_ON_PAGE(page_lru_gen(page) < MAX_NR_GENS, page); - - prefetchw_prev_lru_page(page, head, flags); + VM_WARN_ON_ONCE_PAGE(PageUnevictable(page), page); + VM_WARN_ON_ONCE_PAGE(PageActive(page) != active, page); + VM_WARN_ON_ONCE_PAGE(page_is_file_lru(page) != type, page); + VM_WARN_ON_ONCE_PAGE(page_lru_gen(page) != -1, page); del_page_from_lru_list(page, lruvec); success = lru_gen_add_page(lruvec, page, false); - VM_BUG_ON(!success); + VM_WARN_ON_ONCE(!success); if (!--remaining) return false; @@ -4665,16 +4794,13 @@ static bool drain_evictable(struct lruvec *lruvec) bool success; struct page *page = lru_to_page(head); - VM_BUG_ON_PAGE(PageTail(page), page); - VM_BUG_ON_PAGE(PageUnevictable(page), page); - VM_BUG_ON_PAGE(PageActive(page), page); - VM_BUG_ON_PAGE(page_is_file_lru(page) != type, page); - VM_BUG_ON_PAGE(page_zonenum(page) != zone, page); - - prefetchw_prev_lru_page(page, head, flags); + VM_WARN_ON_ONCE_PAGE(PageUnevictable(page), page); + VM_WARN_ON_ONCE_PAGE(PageActive(page), page); + VM_WARN_ON_ONCE_PAGE(page_is_file_lru(page) != type, page); + VM_WARN_ON_ONCE_PAGE(page_zonenum(page) != zone, page); success = lru_gen_del_page(lruvec, page, false); - VM_BUG_ON(!success); + VM_WARN_ON_ONCE(!success); add_page_to_lru_list(page, lruvec); if (!--remaining) @@ -4685,7 +4811,7 @@ static bool drain_evictable(struct lruvec *lruvec) return true; } -static void lru_gen_change_state(bool enable) +static void lru_gen_change_state(bool enabled) { static DEFINE_MUTEX(state_mutex); @@ -4696,10 +4822,10 @@ static void lru_gen_change_state(bool enable) get_online_mems(); mutex_lock(&state_mutex); - if (enable == lru_gen_enabled()) + if (enabled == lru_gen_enabled()) goto unlock; - if (enable) + if (enabled) static_branch_enable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); else static_branch_disable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); @@ -4715,19 +4841,14 @@ static void lru_gen_change_state(bool enable) if (!lruvec) continue; - if (!pgdat) { - lruvec->lrugen.enabled = enable; - continue; - } - spin_lock_irq(&pgdat->lru_lock); - VM_BUG_ON(!seq_is_valid(lruvec)); - VM_BUG_ON(!state_is_valid(lruvec)); + VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); + VM_WARN_ON_ONCE(!state_is_valid(lruvec)); - lruvec->lrugen.enabled = enable; + lruvec->lrugen.enabled = enabled; - while (!(enable ? fill_evictable(lruvec) : drain_evictable(lruvec))) { + while (!(enabled ? fill_evictable(lruvec) : drain_evictable(lruvec))) { spin_unlock_irq(&pgdat->lru_lock); cond_resched(); spin_lock_irq(&pgdat->lru_lock); @@ -4754,6 +4875,7 @@ static ssize_t show_min_ttl(struct kobject *kobj, struct kobj_attribute *attr, c return sprintf(buf, "%u\n", jiffies_to_msecs(READ_ONCE(lru_gen_min_ttl))); } +/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ static ssize_t store_min_ttl(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t len) { @@ -4771,7 +4893,7 @@ static struct kobj_attribute lru_gen_min_ttl_attr = __ATTR( min_ttl_ms, 0644, show_min_ttl, store_min_ttl ); -static ssize_t show_enable(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +static ssize_t show_enabled(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { unsigned int caps = 0; @@ -4787,8 +4909,9 @@ static ssize_t show_enable(struct kobject *kobj, struct kobj_attribute *attr, ch return snprintf(buf, PAGE_SIZE, "0x%04x\n", caps); } -static ssize_t store_enable(struct kobject *kobj, struct kobj_attribute *attr, - const char *buf, size_t len) +/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ +static ssize_t store_enabled(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t len) { int i; unsigned int caps; @@ -4801,11 +4924,11 @@ static ssize_t store_enable(struct kobject *kobj, struct kobj_attribute *attr, return -EINVAL; for (i = 0; i < NR_LRU_GEN_CAPS; i++) { - bool enable = caps & BIT(i); + bool enabled = caps & BIT(i); if (i == LRU_GEN_CORE) - lru_gen_change_state(enable); - else if (enable) + lru_gen_change_state(enabled); + else if (enabled) static_branch_enable(&lru_gen_caps[i]); else static_branch_disable(&lru_gen_caps[i]); @@ -4815,7 +4938,7 @@ static ssize_t store_enable(struct kobject *kobj, struct kobj_attribute *attr, } static struct kobj_attribute lru_gen_enabled_attr = __ATTR( - enabled, 0644, show_enable, store_enable + enabled, 0644, show_enabled, store_enabled ); static struct attribute *lru_gen_attrs[] = { @@ -4895,40 +5018,46 @@ static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec, for (tier = 0; tier < MAX_NR_TIERS; tier++) { seq_printf(m, " %10d", tier); for (type = 0; type < ANON_AND_FILE; type++) { + const char *s = " "; unsigned long n[3] = {}; if (seq == max_seq) { + s = "RT "; n[0] = READ_ONCE(lrugen->avg_refaulted[type][tier]); n[1] = READ_ONCE(lrugen->avg_total[type][tier]); - - seq_printf(m, " %10luR %10luT %10lu ", n[0], n[1], n[2]); } else if (seq == min_seq[type] || NR_HIST_GENS > 1) { + s = "rep"; n[0] = atomic_long_read(&lrugen->refaulted[hist][type][tier]); n[1] = atomic_long_read(&lrugen->evicted[hist][type][tier]); if (tier) n[2] = READ_ONCE(lrugen->protected[hist][type][tier - 1]); + } - seq_printf(m, " %10lur %10lue %10lup", n[0], n[1], n[2]); - } else - seq_puts(m, " 0 0 0 "); + for (i = 0; i < 3; i++) + seq_printf(m, " %10lu%c", n[i], s[i]); } seq_putc(m, '\n'); } seq_puts(m, " "); for (i = 0; i < NR_MM_STATS; i++) { - if (seq == max_seq && NR_HIST_GENS == 1) - seq_printf(m, " %10lu%c", READ_ONCE(lruvec->mm_state.stats[hist][i]), - toupper(MM_STAT_CODES[i])); - else if (seq != max_seq && NR_HIST_GENS > 1) - seq_printf(m, " %10lu%c", READ_ONCE(lruvec->mm_state.stats[hist][i]), - MM_STAT_CODES[i]); - else - seq_puts(m, " 0 "); + const char *s = " "; + unsigned long n = 0; + + if (seq == max_seq && NR_HIST_GENS == 1) { + s = "LOYNFA"; + n = READ_ONCE(lruvec->mm_state.stats[hist][i]); + } else if (seq != max_seq && NR_HIST_GENS > 1) { + s = "loynfa"; + n = READ_ONCE(lruvec->mm_state.stats[hist][i]); + } + + seq_printf(m, " %10lu%c", n, s[i]); } seq_putc(m, '\n'); } +/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ static int lru_gen_seq_show(struct seq_file *m, void *v) { unsigned long seq; @@ -4967,13 +5096,14 @@ static int lru_gen_seq_show(struct seq_file *m, void *v) seq_printf(m, " %10lu %10u", seq, jiffies_to_msecs(jiffies - birth)); for (type = 0; type < ANON_AND_FILE; type++) { - long size = 0; + unsigned long size = 0; char mark = full && seq < min_seq[type] ? 'x' : ' '; for (zone = 0; zone < MAX_NR_ZONES; zone++) - size += READ_ONCE(lrugen->nr_pages[gen][type][zone]); + size += max_t(long, READ_ONCE(lrugen->nr_pages[gen][type][zone]), + 0); - seq_printf(m, " %10lu%c", max(size, 0L), mark); + seq_printf(m, " %10lu%c", size, mark); } seq_putc(m, '\n'); @@ -4996,18 +5126,25 @@ static int run_aging(struct lruvec *lruvec, unsigned long seq, struct scan_contr bool can_swap, bool full_scan) { DEFINE_MAX_SEQ(lruvec); + DEFINE_MIN_SEQ(lruvec); - if (seq == max_seq) - try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, full_scan); + if (seq < max_seq) + return 0; - return seq > max_seq ? -EINVAL : 0; + if (seq > max_seq) + return -EINVAL; + + if (!full_scan && min_seq[!can_swap] + MAX_NR_GENS - 1 <= max_seq) + return -ERANGE; + + try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, full_scan); + + return 0; } static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, int swappiness, unsigned long nr_to_reclaim) { - struct blk_plug plug; - int err = -EINTR; DEFINE_MAX_SEQ(lruvec); if (seq + MIN_NR_GENS > max_seq) @@ -5015,23 +5152,22 @@ static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_co sc->nr_reclaimed = 0; - blk_start_plug(&plug); - while (!signal_pending(current)) { DEFINE_MIN_SEQ(lruvec); - if (seq < min_seq[!swappiness] || sc->nr_reclaimed >= nr_to_reclaim || - !evict_pages(lruvec, sc, swappiness, NULL)) { - err = 0; - break; - } + if (seq < min_seq[!swappiness]) + return 0; + + if (sc->nr_reclaimed >= nr_to_reclaim) + return 0; + + if (!evict_pages(lruvec, sc, swappiness, NULL)) + return 0; cond_resched(); } - blk_finish_plug(&plug); - - return err; + return -EINTR; } static int run_cmd(char cmd, int memcg_id, int nid, unsigned long seq, @@ -5041,6 +5177,9 @@ static int run_cmd(char cmd, int memcg_id, int nid, unsigned long seq, int err = -EINVAL; struct mem_cgroup *memcg = NULL; + if (nid < 0 || nid >= MAX_NUMNODES || !node_state(nid, N_MEMORY)) + return -EINVAL; + if (!mem_cgroup_disabled()) { rcu_read_lock(); memcg = mem_cgroup_from_id(memcg_id); @@ -5051,12 +5190,10 @@ static int run_cmd(char cmd, int memcg_id, int nid, unsigned long seq, rcu_read_unlock(); if (!memcg) - goto done; + return -EINVAL; } - if (memcg_id != mem_cgroup_id(memcg)) - goto done; - if (nid < 0 || nid >= MAX_NUMNODES || !node_state(nid, N_MEMORY)) + if (memcg_id != mem_cgroup_id(memcg)) goto done; lruvec = get_lruvec(memcg, nid); @@ -5080,13 +5217,15 @@ done: return err; } +/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ static ssize_t lru_gen_seq_write(struct file *file, const char __user *src, size_t len, loff_t *pos) { void *buf; char *cur, *next; unsigned int flags; - int err = 0; + struct blk_plug plug; + int err = -EINVAL; struct scan_control sc = { .may_writepage = true, .may_unmap = true, @@ -5104,17 +5243,16 @@ static ssize_t lru_gen_seq_write(struct file *file, const char __user *src, return -EFAULT; } - next = buf; - next[len] = '\0'; - - sc.reclaim_state.mm_walk = alloc_mm_walk(); - if (!sc.reclaim_state.mm_walk) { - kvfree(buf); - return -ENOMEM; - } - set_task_reclaim_state(current, &sc.reclaim_state); flags = memalloc_noreclaim_save(); + blk_start_plug(&plug); + if (!set_mm_walk(NULL)) { + err = -ENOMEM; + goto done; + } + + next = buf; + next[len] = '\0'; while ((cur = strsep(&next, ",;\n"))) { int n; @@ -5141,11 +5279,12 @@ static ssize_t lru_gen_seq_write(struct file *file, const char __user *src, if (err) break; } - +done: + clear_mm_walk(); + blk_finish_plug(&plug); memalloc_noreclaim_restore(flags); set_task_reclaim_state(current, NULL); - free_mm_walk(sc.reclaim_state.mm_walk); kvfree(buf); return err ? : len; @@ -5209,8 +5348,8 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg) for_each_node(nid) { struct lruvec *lruvec = get_lruvec(memcg, nid); - VM_BUG_ON(memchr_inv(lruvec->lrugen.nr_pages, 0, - sizeof(lruvec->lrugen.nr_pages))); + VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0, + sizeof(lruvec->lrugen.nr_pages))); for (i = 0; i < NR_BLOOM_FILTERS; i++) { bitmap_free(lruvec->mm_state.filters[i]); @@ -5224,7 +5363,6 @@ static int __init init_lru_gen(void) { BUILD_BUG_ON(MIN_NR_GENS + 1 >= MAX_NR_GENS); BUILD_BUG_ON(BIT(LRU_GEN_WIDTH) <= MAX_NR_GENS); - BUILD_BUG_ON(sizeof(MM_STAT_CODES) != NR_MM_STATS + 1); if (sysfs_create_group(mm_kobj, &lru_gen_attr_group)) pr_err("lru_gen: failed to create sysfs group\n"); @@ -5236,7 +5374,7 @@ static int __init init_lru_gen(void) }; late_initcall(init_lru_gen); -#else +#else /* !CONFIG_LRU_GEN */ static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) { @@ -5257,6 +5395,7 @@ static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) unsigned long nr_reclaimed = 0; unsigned long nr_to_reclaim = sc->nr_to_reclaim; struct blk_plug plug; + bool do_plug = true; bool scan_adjusted; if (lru_gen_enabled()) { @@ -5283,7 +5422,9 @@ static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) scan_adjusted = (!cgroup_reclaim(sc) && !current_is_kswapd() && sc->priority == DEF_PRIORITY); - blk_start_plug(&plug); + trace_android_vh_shrink_lruvec_blk_plug(&do_plug); + if (do_plug) + blk_start_plug(&plug); while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] || nr[LRU_INACTIVE_FILE]) { unsigned long nr_anon, nr_file, percentage; @@ -5355,7 +5496,8 @@ static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) scan_adjusted = true; } - blk_finish_plug(&plug); + if (do_plug) + blk_finish_plug(&plug); sc->nr_reclaimed += nr_reclaimed; /* @@ -6102,8 +6244,7 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, EXPORT_SYMBOL_GPL(try_to_free_mem_cgroup_pages); #endif -static void age_active_anon(struct pglist_data *pgdat, - struct scan_control *sc) +static void kswapd_age_node(struct pglist_data *pgdat, struct scan_control *sc) { struct mem_cgroup *memcg; struct lruvec *lruvec; @@ -6396,12 +6537,11 @@ restart: sc.may_swap = !nr_boost_reclaim; /* - * Do some background aging of the anon list, to give - * pages a chance to be referenced before reclaiming. All - * pages are rotated regardless of classzone as this is - * about consistent aging. + * Do some background aging, to give pages a chance to be + * referenced before reclaiming. All pages are rotated + * regardless of classzone as this is about consistent aging. */ - age_active_anon(pgdat, &sc); + kswapd_age_node(pgdat, &sc); /* * If we're getting trouble reclaiming, start doing writepage diff --git a/mm/workingset.c b/mm/workingset.c index 25eaab881192..beacfc21a5e8 100644 --- a/mm/workingset.c +++ b/mm/workingset.c @@ -215,40 +215,31 @@ static void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat, #ifdef CONFIG_LRU_GEN -static int page_lru_refs(struct page *page) -{ - unsigned long flags = READ_ONCE(page->flags); - - BUILD_BUG_ON(LRU_GEN_WIDTH + LRU_REFS_WIDTH > BITS_PER_LONG - EVICTION_SHIFT); - - /* see the comment on MAX_NR_TIERS */ - return flags & BIT(PG_workingset) ? (flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF : 0; -} - static void *lru_gen_eviction(struct page *page) { - int hist, tier; + int hist; unsigned long token; unsigned long min_seq; struct lruvec *lruvec; struct lru_gen_struct *lrugen; int type = page_is_file_lru(page); - int refs = page_lru_refs(page); int delta = thp_nr_pages(page); - bool workingset = PageWorkingset(page); + int refs = page_lru_refs(page); + int tier = lru_tier_from_refs(refs); struct mem_cgroup *memcg = page_memcg(page); struct pglist_data *pgdat = page_pgdat(page); + BUILD_BUG_ON(LRU_GEN_WIDTH + LRU_REFS_WIDTH > BITS_PER_LONG - EVICTION_SHIFT); + lruvec = mem_cgroup_lruvec(memcg, pgdat); lrugen = &lruvec->lrugen; min_seq = READ_ONCE(lrugen->min_seq[type]); - token = (min_seq << LRU_REFS_WIDTH) | refs; + token = (min_seq << LRU_REFS_WIDTH) | max(refs - 1, 0); hist = lru_hist_from_seq(min_seq); - tier = lru_tier_from_refs(refs + workingset); atomic_long_add(delta, &lrugen->evicted[hist][type][tier]); - return pack_shadow(mem_cgroup_id(memcg), pgdat, token, workingset); + return pack_shadow(mem_cgroup_id(memcg), pgdat, token, refs); } static void lru_gen_refault(struct page *page, void *shadow) @@ -267,27 +258,27 @@ static void lru_gen_refault(struct page *page, void *shadow) unpack_shadow(shadow, &memcg_id, &pgdat, &token, &workingset); - refs = token & (BIT(LRU_REFS_WIDTH) - 1); - if (refs && !workingset) - return; - - if (page_pgdat(page) != pgdat) + if (pgdat != page_pgdat(page)) return; rcu_read_lock(); + memcg = page_memcg_rcu(page); - if (mem_cgroup_id(memcg) != memcg_id) + if (memcg_id != mem_cgroup_id(memcg)) goto unlock; - token >>= LRU_REFS_WIDTH; lruvec = mem_cgroup_lruvec(memcg, pgdat); lrugen = &lruvec->lrugen; + min_seq = READ_ONCE(lrugen->min_seq[type]); - if (token != (min_seq & (EVICTION_MASK >> LRU_REFS_WIDTH))) + if ((token >> LRU_REFS_WIDTH) != (min_seq & (EVICTION_MASK >> LRU_REFS_WIDTH))) goto unlock; hist = lru_hist_from_seq(min_seq); - tier = lru_tier_from_refs(refs + workingset); + /* see the comment in page_lru_refs() */ + refs = (token & (BIT(LRU_REFS_WIDTH) - 1)) + workingset; + tier = lru_tier_from_refs(refs); + atomic_long_add(delta, &lrugen->refaulted[hist][type][tier]); mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + type, delta); @@ -295,10 +286,10 @@ static void lru_gen_refault(struct page *page, void *shadow) * Count the following two cases as stalls: * 1. For pages accessed through page tables, hotter pages pushed out * hot pages which refaulted immediately. - * 2. For pages accessed through file descriptors, numbers of accesses - * might have been beyond the limit. + * 2. For pages accessed multiple times through file descriptors, + * numbers of accesses might have been out of the range. */ - if (lru_gen_in_fault() || refs + workingset == BIT(LRU_REFS_WIDTH)) { + if (lru_gen_in_fault() || refs == BIT(LRU_REFS_WIDTH)) { SetPageWorkingset(page); mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + type, delta); } @@ -306,7 +297,7 @@ unlock: rcu_read_unlock(); } -#else +#else /* !CONFIG_LRU_GEN */ static void *lru_gen_eviction(struct page *page) { @@ -362,14 +353,14 @@ void *workingset_eviction(struct page *page, struct mem_cgroup *target_memcg) struct lruvec *lruvec; int memcgid; + if (lru_gen_enabled()) + return lru_gen_eviction(page); + /* Page is fully exclusive and pins page->mem_cgroup */ VM_BUG_ON_PAGE(PageLRU(page), page); VM_BUG_ON_PAGE(page_count(page), page); VM_BUG_ON_PAGE(!PageLocked(page), page); - if (lru_gen_enabled()) - return lru_gen_eviction(page); - lruvec = mem_cgroup_lruvec(target_memcg, pgdat); workingset_age_nonresident(lruvec, thp_nr_pages(page)); /* XXX: target_memcg can be NULL, go through lruvec */ diff --git a/net/core/skbuff.c b/net/core/skbuff.c index f00b266aa6eb..61a9a858aef2 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1684,11 +1684,10 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, skb->head = data; skb->head_frag = 0; skb->data += off; + + skb_set_end_offset(skb, size); #ifdef NET_SKBUFF_DATA_USES_OFFSET - skb->end = size; off = nhead; -#else - skb->end = skb->head + size; #endif skb->tail += off; skb_headers_offset_update(skb, nhead); @@ -1736,6 +1735,38 @@ struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom) } EXPORT_SYMBOL(skb_realloc_headroom); +int __skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri) +{ + unsigned int saved_end_offset, saved_truesize; + struct skb_shared_info *shinfo; + int res; + + saved_end_offset = skb_end_offset(skb); + saved_truesize = skb->truesize; + + res = pskb_expand_head(skb, 0, 0, pri); + if (res) + return res; + + skb->truesize = saved_truesize; + + if (likely(skb_end_offset(skb) == saved_end_offset)) + return 0; + + shinfo = skb_shinfo(skb); + + /* We are about to change back skb->end, + * we need to move skb_shinfo() to its new location. + */ + memmove(skb->head + saved_end_offset, + shinfo, + offsetof(struct skb_shared_info, frags[shinfo->nr_frags])); + + skb_set_end_offset(skb, saved_end_offset); + + return 0; +} + /** * skb_copy_expand - copy and expand sk_buff * @skb: buffer to copy @@ -5982,11 +6013,7 @@ static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off, skb->head = data; skb->data = data; skb->head_frag = 0; -#ifdef NET_SKBUFF_DATA_USES_OFFSET - skb->end = size; -#else - skb->end = skb->head + size; -#endif + skb_set_end_offset(skb, size); skb_set_tail_pointer(skb, skb_headlen(skb)); skb_headers_offset_update(skb, 0); skb->cloned = 0; @@ -6124,11 +6151,7 @@ static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off, skb->head = data; skb->head_frag = 0; skb->data = data; -#ifdef NET_SKBUFF_DATA_USES_OFFSET - skb->end = size; -#else - skb->end = skb->head + size; -#endif + skb_set_end_offset(skb, size); skb_reset_tail_pointer(skb); skb_headers_offset_update(skb, 0); skb->cloned = 0; diff --git a/scripts/adjust_autoksyms.sh b/scripts/adjust_autoksyms.sh index 2b366d945ccb..d8f6f9c63043 100755 --- a/scripts/adjust_autoksyms.sh +++ b/scripts/adjust_autoksyms.sh @@ -34,9 +34,6 @@ case "$KBUILD_VERBOSE" in ;; esac -# We need access to CONFIG_ symbols -. include/config/auto.conf - # Generate a new symbol list file $CONFIG_SHELL $srctree/scripts/gen_autoksyms.sh "$new_ksyms_file" diff --git a/scripts/dummy-tools/gcc b/scripts/dummy-tools/gcc index 485427f40dba..4fcae6445e42 100755 --- a/scripts/dummy-tools/gcc +++ b/scripts/dummy-tools/gcc @@ -57,9 +57,9 @@ if arg_contain --version "$@"; then fi if arg_contain -E "$@"; then - # For scripts/gcc-version.sh; This emulates GCC 20.0.0 + # For scripts/cc-version.sh; This emulates GCC 20.0.0 if arg_contain - "$@"; then - sed 's/^__GNUC__$/20/; s/^__GNUC_MINOR__$/0/; s/^__GNUC_PATCHLEVEL__$/0/' + sed -n '/^GCC/{s/__GNUC__/20/; s/__GNUC_MINOR__/0/; s/__GNUC_PATCHLEVEL__/0/; p;}' exit 0 else echo "no input files" >&2 diff --git a/tools/build/Build.include b/tools/build/Build.include index 585486e40995..2cf3b1bde86e 100644 --- a/tools/build/Build.include +++ b/tools/build/Build.include @@ -100,3 +100,27 @@ cxx_flags = -Wp,-MD,$(depfile) -Wp,-MT,$@ $(CXXFLAGS) -D"BUILD_STR(s)=\#s" $(CXX ## HOSTCC C flags host_c_flags = -Wp,-MD,$(depfile) -Wp,-MT,$@ $(KBUILD_HOSTCFLAGS) -D"BUILD_STR(s)=\#s" $(HOSTCFLAGS_$(basetarget).o) $(HOSTCFLAGS_$(obj)) + +# output directory for tests below +TMPOUT = .tmp_$$$$ + +# try-run +# Usage: option = $(call try-run, $(CC)...-o "$$TMP",option-ok,otherwise) +# Exit code chooses option. "$$TMP" serves as a temporary file and is +# automatically cleaned up. +try-run = $(shell set -e; \ + TMP=$(TMPOUT)/tmp; \ + mkdir -p $(TMPOUT); \ + trap "rm -rf $(TMPOUT)" EXIT; \ + if ($(1)) >/dev/null 2>&1; \ + then echo "$(2)"; \ + else echo "$(3)"; \ + fi) + +# cc-option +# Usage: cflags-y += $(call cc-option,-march=winchip-c6,-march=i586) +cc-option = $(call try-run, \ + $(CC) -Werror $(1) -c -x c /dev/null -o "$$TMP",$(1),$(2)) + +# delete partially updated (i.e. corrupted) files on error +.DELETE_ON_ERROR: diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index d9c283503159..5074c8b82367 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -9,6 +9,7 @@ TARGETS += clone3 TARGETS += core TARGETS += cpufreq TARGETS += cpu-hotplug +TARGETS += damon TARGETS += drivers/dma-buf TARGETS += efivarfs TARGETS += exec diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 1d9155533360..ace079e0b210 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -include ../../../../scripts/Kbuild.include +include ../../../build/Build.include include ../../../scripts/Makefile.arch CXX ?= $(CROSS_COMPILE)g++ diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index 8470601b89ae..0623a77e6723 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only -include ../../../../scripts/Kbuild.include +include ../../../build/Build.include all: diff --git a/tools/testing/selftests/powerpc/pmu/ebb/Makefile b/tools/testing/selftests/powerpc/pmu/ebb/Makefile index af3df79d8163..c5ecb4634094 100644 --- a/tools/testing/selftests/powerpc/pmu/ebb/Makefile +++ b/tools/testing/selftests/powerpc/pmu/ebb/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -include ../../../../../../scripts/Kbuild.include +include ../../../../../build/Build.include noarg: $(MAKE) -C ../../ diff --git a/tools/thermal/tmon/Makefile b/tools/thermal/tmon/Makefile index 25d7f8f37cfd..610334f86f63 100644 --- a/tools/thermal/tmon/Makefile +++ b/tools/thermal/tmon/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 # We need this for the "cc-option" macro. -include ../../../scripts/Kbuild.include +include ../../build/Build.include VERSION = 1.0