From 8672a5ee4dcef93936359d429402d0835c93e723 Mon Sep 17 00:00:00 2001 From: John Scheible Date: Tue, 16 Apr 2024 21:17:12 -0700 Subject: [PATCH 01/86] ANDROID: Update the ABI symbol list Adding the following symbols: - devm_pm_runtime_enable Bug: 335356311 Change-Id: Iecd45183cead8807974bb2a065c48aab86e47e89 Signed-off-by: John Scheible --- android/abi_gki_aarch64_pixel | 1 + 1 file changed, 1 insertion(+) diff --git a/android/abi_gki_aarch64_pixel b/android/abi_gki_aarch64_pixel index fe4e815bf551..05104f550161 100644 --- a/android/abi_gki_aarch64_pixel +++ b/android/abi_gki_aarch64_pixel @@ -476,6 +476,7 @@ devm_pinctrl_register_and_init devm_platform_ioremap_resource devm_platform_ioremap_resource_byname + devm_pm_runtime_enable devm_power_supply_register devm_pwm_get devm_regmap_add_irq_chip From 141ebdcb283b1248438238afbf1d27aef600b0eb Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Wed, 20 Sep 2023 14:30:30 +0800 Subject: [PATCH 02/86] UPSTREAM: usb:typec:tcpm:support double Rp to Vbus cable as sink The USB Type-C Cable and Connector Specification defines the wire connections for the USB Type-C to USB 2.0 Standard-A cable assembly (Release 2.2, Chapter 3.5.2). The Notes says that Pin A5 (CC) of the USB Type-C plug shall be connected to Vbus through a resister Rp. However, there is a large amount of such double Rp connected to Vbus non-standard cables which produced by UGREEN circulating on the market, and it can affects the normal operations of the state machine easily, especially to CC1 and CC2 be pulled up at the same time. In fact, we can regard those cables as sink to avoid abnormal state. Message as follow: [ 58.900212] VBUS on [ 59.265433] CC1: 0 -> 3, CC2: 0 -> 3 [state TOGGLING, polarity 0, connected] [ 62.623308] CC1: 3 -> 0, CC2: 3 -> 0 [state TOGGLING, polarity 0, disconnected] [ 62.625006] VBUS off [ 62.625012] VBUS VSAFE0V Bug: 335057705 Change-Id: I415db22b0012ace9535039bc4c8e5ec113482e33 Signed-off-by: Michael Wu Reviewed-by: Guenter Roeck Reviewed-by: Heikki Krogerus Link: https://lore.kernel.org/r/20230920063030.66312-1-michael@allwinnertech.com Signed-off-by: Greg Kroah-Hartman Signed-off-by: Frank Wang (cherry picked from commit dbc1defec1aa7d8d80da3ea9e3ddafbcfca8f822) --- drivers/usb/typec/tcpm/tcpm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index 9aa216db0f10..b6235a78615a 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -530,9 +530,9 @@ static const char * const pd_rev[] = { ((cc) == TYPEC_CC_RP_DEF || (cc) == TYPEC_CC_RP_1_5 || \ (cc) == TYPEC_CC_RP_3_0) +/* As long as cc is pulled up, we can consider it as sink. */ #define tcpm_port_is_sink(port) \ - ((tcpm_cc_is_sink((port)->cc1) && !tcpm_cc_is_sink((port)->cc2)) || \ - (tcpm_cc_is_sink((port)->cc2) && !tcpm_cc_is_sink((port)->cc1))) + (tcpm_cc_is_sink((port)->cc1) || tcpm_cc_is_sink((port)->cc2)) #define tcpm_cc_is_source(cc) ((cc) == TYPEC_CC_RD) #define tcpm_cc_is_audio(cc) ((cc) == TYPEC_CC_RA) From 24edb63b854ab73fa1bf86b4dde98cdb648cba35 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Mon, 12 Jun 2023 13:33:12 -0700 Subject: [PATCH 03/86] Reapply "ANDROID: block: Add support for filesystem requests and small segments" This reverts commit I764adf995cae6b485d4d98e410c78128a88647e0. Bug: 333812722 Bug: 308663717 Bug: 319125789 Change-Id: Ie7f03b9d1ab67a33ccbc4311ba26cd746e1aaa22 Signed-off-by: Bart Van Assche [jyescas@google.com: Call blk_segments() in blk_mq_submit_bio() for the case when request is defined or it is null] Signed-off-by: Juan Yescas --- block/blk-merge.c | 8 ++++++-- block/blk-mq.c | 4 ++++ block/blk.h | 11 +++++------ 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/block/blk-merge.c b/block/blk-merge.c index cc7f6a4a255c..0f62c68b455c 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -285,7 +285,8 @@ static struct bio *bio_split_rw(struct bio *bio, struct queue_limits *lim, if (nsegs < lim->max_segments && bytes + bv.bv_len <= max_bytes && bv.bv_offset + bv.bv_len <= PAGE_SIZE) { - nsegs++; + /* single-page bvec optimization */ + nsegs += blk_segments(lim, bv.bv_len); bytes += bv.bv_len; } else { if (bvec_split_segs(lim, &bv, &nsegs, &bytes, @@ -533,7 +534,10 @@ static int __blk_bios_map_sg(struct request_queue *q, struct bio *bio, __blk_segment_map_sg_merge(q, &bvec, &bvprv, sg)) goto next_bvec; - if (bvec.bv_offset + bvec.bv_len <= PAGE_SIZE) + if (bvec.bv_offset + bvec.bv_len <= PAGE_SIZE && + (!blk_queue_sub_page_limits(&q->limits) || + bvec.bv_len <= q->limits.max_segment_size)) + /* single-segment bvec optimization */ nsegs += __blk_bvec_map_sg(bvec, sglist, sg); else nsegs += blk_bvec_map_sg(q, &bvec, sglist, sg); diff --git a/block/blk-mq.c b/block/blk-mq.c index 0e56e3e73019..276b0a6eaec5 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -2956,6 +2956,8 @@ void blk_mq_submit_bio(struct bio *bio) bio = __bio_split_to_limits(bio, &q->limits, &nr_segs); if (!bio) return; + } else if (bio->bi_vcnt == 1) { + nr_segs = blk_segments(&q->limits, bio->bi_io_vec[0].bv_len); } if (!bio_integrity_prep(bio)) return; @@ -2971,6 +2973,8 @@ void blk_mq_submit_bio(struct bio *bio) bio = __bio_split_to_limits(bio, &q->limits, &nr_segs); if (!bio) goto fail; + } else if (bio->bi_vcnt == 1) { + nr_segs = blk_segments(&q->limits, bio->bi_io_vec[0].bv_len); } if (!bio_integrity_prep(bio)) goto fail; diff --git a/block/blk.h b/block/blk.h index f6e59cd7cfa1..82143bf38064 100644 --- a/block/blk.h +++ b/block/blk.h @@ -337,13 +337,12 @@ static inline bool bio_may_exceed_limits(struct bio *bio, } /* - * All drivers must accept single-segments bios that are <= PAGE_SIZE. - * This is a quick and dirty check that relies on the fact that - * bi_io_vec[0] is always valid if a bio has data. The check might - * lead to occasional false negatives when bios are cloned, but compared - * to the performance impact of cloned bios themselves the loop below - * doesn't matter anyway. + * Check whether bio splitting should be performed. This check may + * trigger the bio splitting code even if splitting is not necessary. */ + if (blk_queue_sub_page_limits(lim) && bio->bi_io_vec && + bio->bi_io_vec->bv_len > lim->max_segment_size) + return true; return lim->chunk_sectors || bio->bi_vcnt != 1 || bio->bi_io_vec->bv_len + bio->bi_io_vec->bv_offset > PAGE_SIZE; } From 4bfe25d0b6c370f026d66d3f2fd8d4d78b624594 Mon Sep 17 00:00:00 2001 From: Ken Huang Date: Wed, 17 Apr 2024 20:28:56 +0800 Subject: [PATCH 04/86] ANDROID: Update the ABI symbol list Adding the following symbols: - devm_drm_bridge_add Bug: 333511135 Change-Id: I4f815c00515c1f2660032e584778edf1c2c41da4 Signed-off-by: Ken Huang --- android/abi_gki_aarch64.stg | 16 ++++++++++++++++ android/abi_gki_aarch64_pixel | 1 + 2 files changed, 17 insertions(+) diff --git a/android/abi_gki_aarch64.stg b/android/abi_gki_aarch64.stg index 5aa89d5c6901..c42f8772d3c7 100644 --- a/android/abi_gki_aarch64.stg +++ b/android/abi_gki_aarch64.stg @@ -327921,6 +327921,12 @@ function { parameter_id: 0xc9082b19 parameter_id: 0x38d23361 } +function { + id: 0x9d8f7eed + return_type_id: 0x6720d32f + parameter_id: 0x0258f96e + parameter_id: 0x2668e644 +} function { id: 0x9d8f9d80 return_type_id: 0x6720d32f @@ -358433,6 +358439,15 @@ elf_symbol { type_id: 0x10c35fd7 full_name: "devm_device_remove_group" } +elf_symbol { + id: 0x4e304be8 + name: "devm_drm_bridge_add" + is_defined: true + symbol_type: FUNCTION + crc: 0x681dc9e1 + type_id: 0x9d8f7eed + full_name: "devm_drm_bridge_add" +} elf_symbol { id: 0x6bf4c5d2 name: "devm_drm_of_get_bridge" @@ -408700,6 +408715,7 @@ interface { symbol_id: 0x8ed64fa4 symbol_id: 0x0764373d symbol_id: 0x72706e55 + symbol_id: 0x4e304be8 symbol_id: 0x6bf4c5d2 symbol_id: 0x4a6566b1 symbol_id: 0x0e6117e6 diff --git a/android/abi_gki_aarch64_pixel b/android/abi_gki_aarch64_pixel index 05104f550161..d986d4cb211e 100644 --- a/android/abi_gki_aarch64_pixel +++ b/android/abi_gki_aarch64_pixel @@ -427,6 +427,7 @@ devm_device_add_group devm_device_add_groups devm_device_remove_group + devm_drm_bridge_add __devm_drm_dev_alloc devm_drm_of_get_bridge devm_drm_panel_bridge_add_typed From a5ce14670ae4ca1ea2fbfb36fbe57a54ff0ef32c Mon Sep 17 00:00:00 2001 From: AngeloGioacchino Del Regno Date: Mon, 6 Mar 2023 15:05:26 +0100 Subject: [PATCH 05/86] BACKPORT: clk: mediatek: Add MODULE_LICENSE() where missing In order to successfully build clock drivers as modules it is required to declare a module license: add it where missing. While at it, also change the MODULE_LICENSE text from "GPL v2" to "GPL" (which means the same) on clk-mt7981-eth.c. Bug: 335112842 (cherry picked from commit a451da86cf6d10e94372d20622ec41aac9ec00b5 https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/ master) Signed-off-by: AngeloGioacchino Del Regno Reviewed-by: Chen-Yu Tsai Tested-by: Miles Chen Tested-by: Chen-Yu Tsai # MT8183, MT8192, MT8195 Chromebooks Link: https://lore.kernel.org/r/20230306140543.1813621-38-angelogioacchino.delregno@collabora.com Signed-off-by: Stephen Boyd Change-Id: Id78b0caa4520350049ae6162481c381eb3309897 --- drivers/clk/mediatek/clk-mt2701-aud.c | 4 ++-- drivers/clk/mediatek/clk-mt2701-bdp.c | 4 ++-- drivers/clk/mediatek/clk-mt2701-eth.c | 4 ++-- drivers/clk/mediatek/clk-mt2701-g3d.c | 4 ++-- drivers/clk/mediatek/clk-mt2701-hif.c | 4 ++-- drivers/clk/mediatek/clk-mt2701-img.c | 4 ++-- drivers/clk/mediatek/clk-mt2701-mm.c | 4 ++-- drivers/clk/mediatek/clk-mt2701-vdec.c | 4 ++-- drivers/clk/mediatek/clk-mt2701.c | 1 + drivers/clk/mediatek/clk-mt2712-bdp.c | 4 ++-- drivers/clk/mediatek/clk-mt2712-img.c | 4 ++-- drivers/clk/mediatek/clk-mt2712-jpgdec.c | 4 ++-- drivers/clk/mediatek/clk-mt2712-mfg.c | 4 ++-- drivers/clk/mediatek/clk-mt2712-mm.c | 4 ++-- drivers/clk/mediatek/clk-mt2712-vdec.c | 4 ++-- drivers/clk/mediatek/clk-mt2712-venc.c | 4 ++-- drivers/clk/mediatek/clk-mt2712.c | 9 ++------- drivers/clk/mediatek/clk-mt6765-audio.c | 4 ++-- drivers/clk/mediatek/clk-mt6765-cam.c | 4 ++-- drivers/clk/mediatek/clk-mt6765-img.c | 4 ++-- drivers/clk/mediatek/clk-mt6765-mipi0a.c | 4 ++-- drivers/clk/mediatek/clk-mt6765-mm.c | 4 ++-- drivers/clk/mediatek/clk-mt6765-vcodec.c | 4 ++-- drivers/clk/mediatek/clk-mt6765.c | 1 + drivers/clk/mediatek/clk-mt6797-img.c | 4 ++-- drivers/clk/mediatek/clk-mt6797-mm.c | 4 ++-- drivers/clk/mediatek/clk-mt6797-vdec.c | 4 ++-- drivers/clk/mediatek/clk-mt6797-venc.c | 4 ++-- drivers/clk/mediatek/clk-mt6797.c | 1 + drivers/clk/mediatek/clk-mt7622-aud.c | 4 ++-- drivers/clk/mediatek/clk-mt7622-eth.c | 4 ++-- drivers/clk/mediatek/clk-mt7622-hif.c | 4 ++-- drivers/clk/mediatek/clk-mt7629-eth.c | 1 + drivers/clk/mediatek/clk-mt7629-hif.c | 4 ++-- drivers/clk/mediatek/clk-mt7629.c | 1 + drivers/clk/mediatek/clk-mt7986-apmixed.c | 1 + drivers/clk/mediatek/clk-mt7986-topckgen.c | 3 ++- drivers/clk/mediatek/clk-mt8183-audio.c | 4 ++-- drivers/clk/mediatek/clk-mt8183-cam.c | 4 ++-- drivers/clk/mediatek/clk-mt8183-img.c | 4 ++-- drivers/clk/mediatek/clk-mt8183-ipu0.c | 4 ++-- drivers/clk/mediatek/clk-mt8183-ipu1.c | 4 ++-- drivers/clk/mediatek/clk-mt8183-ipu_adl.c | 4 ++-- drivers/clk/mediatek/clk-mt8183-ipu_conn.c | 4 ++-- drivers/clk/mediatek/clk-mt8183-mfgcfg.c | 4 ++-- drivers/clk/mediatek/clk-mt8183-mm.c | 4 ++-- drivers/clk/mediatek/clk-mt8183-vdec.c | 4 ++-- drivers/clk/mediatek/clk-mt8183-venc.c | 4 ++-- drivers/clk/mediatek/clk-mt8183.c | 9 ++------- drivers/clk/mediatek/clk-mt8186-apmixedsys.c | 3 ++- drivers/clk/mediatek/clk-mt8186-cam.c | 3 ++- drivers/clk/mediatek/clk-mt8186-img.c | 3 ++- drivers/clk/mediatek/clk-mt8186-imp_iic_wrap.c | 3 ++- drivers/clk/mediatek/clk-mt8186-infra_ao.c | 3 ++- drivers/clk/mediatek/clk-mt8186-ipe.c | 3 ++- drivers/clk/mediatek/clk-mt8186-mcu.c | 3 ++- drivers/clk/mediatek/clk-mt8186-mdp.c | 3 ++- drivers/clk/mediatek/clk-mt8186-mfg.c | 3 ++- drivers/clk/mediatek/clk-mt8186-mm.c | 3 ++- drivers/clk/mediatek/clk-mt8186-topckgen.c | 3 ++- drivers/clk/mediatek/clk-mt8186-vdec.c | 3 ++- drivers/clk/mediatek/clk-mt8186-venc.c | 3 ++- drivers/clk/mediatek/clk-mt8186-wpe.c | 3 ++- drivers/clk/mediatek/clk-mt8192-aud.c | 4 ++-- drivers/clk/mediatek/clk-mt8192-cam.c | 4 ++-- drivers/clk/mediatek/clk-mt8192-img.c | 4 ++-- drivers/clk/mediatek/clk-mt8192-imp_iic_wrap.c | 4 ++-- drivers/clk/mediatek/clk-mt8192-ipe.c | 4 ++-- drivers/clk/mediatek/clk-mt8192-mdp.c | 4 ++-- drivers/clk/mediatek/clk-mt8192-mfg.c | 4 ++-- drivers/clk/mediatek/clk-mt8192-mm.c | 4 ++-- drivers/clk/mediatek/clk-mt8192-msdc.c | 4 ++-- drivers/clk/mediatek/clk-mt8192-scp_adsp.c | 4 ++-- drivers/clk/mediatek/clk-mt8192-vdec.c | 4 ++-- drivers/clk/mediatek/clk-mt8192-venc.c | 4 ++-- drivers/clk/mediatek/clk-mt8192.c | 1 + drivers/clk/mediatek/clk-mt8195-apmixedsys.c | 3 ++- drivers/clk/mediatek/clk-mt8195-apusys_pll.c | 3 ++- drivers/clk/mediatek/clk-mt8195-cam.c | 3 ++- drivers/clk/mediatek/clk-mt8195-ccu.c | 3 ++- drivers/clk/mediatek/clk-mt8195-img.c | 3 ++- drivers/clk/mediatek/clk-mt8195-imp_iic_wrap.c | 3 ++- drivers/clk/mediatek/clk-mt8195-infra_ao.c | 3 ++- drivers/clk/mediatek/clk-mt8195-ipe.c | 3 ++- drivers/clk/mediatek/clk-mt8195-mfg.c | 3 ++- drivers/clk/mediatek/clk-mt8195-peri_ao.c | 3 ++- drivers/clk/mediatek/clk-mt8195-scp_adsp.c | 3 ++- drivers/clk/mediatek/clk-mt8195-topckgen.c | 3 ++- drivers/clk/mediatek/clk-mt8195-vdec.c | 3 ++- drivers/clk/mediatek/clk-mt8195-vdo0.c | 3 ++- drivers/clk/mediatek/clk-mt8195-vdo1.c | 3 ++- drivers/clk/mediatek/clk-mt8195-venc.c | 3 ++- drivers/clk/mediatek/clk-mt8195-vpp0.c | 3 ++- drivers/clk/mediatek/clk-mt8195-vpp1.c | 3 ++- drivers/clk/mediatek/clk-mt8195-wpe.c | 3 ++- 95 files changed, 183 insertions(+), 152 deletions(-) diff --git a/drivers/clk/mediatek/clk-mt2701-aud.c b/drivers/clk/mediatek/clk-mt2701-aud.c index 4287bd3f545e..7e35a4f95da4 100644 --- a/drivers/clk/mediatek/clk-mt2701-aud.c +++ b/drivers/clk/mediatek/clk-mt2701-aud.c @@ -158,5 +158,5 @@ static struct platform_driver clk_mt2701_aud_drv = { .of_match_table = of_match_clk_mt2701_aud, }, }; - -builtin_platform_driver(clk_mt2701_aud_drv); +module_platform_driver(clk_mt2701_aud_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt2701-bdp.c b/drivers/clk/mediatek/clk-mt2701-bdp.c index b0f057207945..e7b43329d305 100644 --- a/drivers/clk/mediatek/clk-mt2701-bdp.c +++ b/drivers/clk/mediatek/clk-mt2701-bdp.c @@ -104,5 +104,5 @@ static struct platform_driver clk_mt2701_bdp_drv = { .of_match_table = of_match_clk_mt2701_bdp, }, }; - -builtin_platform_driver(clk_mt2701_bdp_drv); +module_platform_driver(clk_mt2701_bdp_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt2701-eth.c b/drivers/clk/mediatek/clk-mt2701-eth.c index 601358748750..9e7bbdf8e0a2 100644 --- a/drivers/clk/mediatek/clk-mt2701-eth.c +++ b/drivers/clk/mediatek/clk-mt2701-eth.c @@ -72,5 +72,5 @@ static struct platform_driver clk_mt2701_eth_drv = { .of_match_table = of_match_clk_mt2701_eth, }, }; - -builtin_platform_driver(clk_mt2701_eth_drv); +module_platform_driver(clk_mt2701_eth_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt2701-g3d.c b/drivers/clk/mediatek/clk-mt2701-g3d.c index 8d1fc8e3336e..ca7b3b1cce61 100644 --- a/drivers/clk/mediatek/clk-mt2701-g3d.c +++ b/drivers/clk/mediatek/clk-mt2701-g3d.c @@ -93,5 +93,5 @@ static struct platform_driver clk_mt2701_g3d_drv = { .of_match_table = of_match_clk_mt2701_g3d, }, }; - -builtin_platform_driver(clk_mt2701_g3d_drv); +module_platform_driver(clk_mt2701_g3d_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt2701-hif.c b/drivers/clk/mediatek/clk-mt2701-hif.c index edeeb033a235..806465373a0f 100644 --- a/drivers/clk/mediatek/clk-mt2701-hif.c +++ b/drivers/clk/mediatek/clk-mt2701-hif.c @@ -71,5 +71,5 @@ static struct platform_driver clk_mt2701_hif_drv = { .of_match_table = of_match_clk_mt2701_hif, }, }; - -builtin_platform_driver(clk_mt2701_hif_drv); +module_platform_driver(clk_mt2701_hif_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt2701-img.c b/drivers/clk/mediatek/clk-mt2701-img.c index eb172473f075..35a5fd938a6e 100644 --- a/drivers/clk/mediatek/clk-mt2701-img.c +++ b/drivers/clk/mediatek/clk-mt2701-img.c @@ -52,5 +52,5 @@ static struct platform_driver clk_mt2701_img_drv = { .of_match_table = of_match_clk_mt2701_img, }, }; - -builtin_platform_driver(clk_mt2701_img_drv); +module_platform_driver(clk_mt2701_img_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt2701-mm.c b/drivers/clk/mediatek/clk-mt2701-mm.c index eb069f3bc9a2..1b0f380e15ce 100644 --- a/drivers/clk/mediatek/clk-mt2701-mm.c +++ b/drivers/clk/mediatek/clk-mt2701-mm.c @@ -94,5 +94,5 @@ static struct platform_driver clk_mt2701_mm_drv = { .name = "clk-mt2701-mm", }, }; - -builtin_platform_driver(clk_mt2701_mm_drv); +module_platform_driver(clk_mt2701_mm_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt2701-vdec.c b/drivers/clk/mediatek/clk-mt2701-vdec.c index 0f07c5d731df..dca4335a52a7 100644 --- a/drivers/clk/mediatek/clk-mt2701-vdec.c +++ b/drivers/clk/mediatek/clk-mt2701-vdec.c @@ -57,5 +57,5 @@ static struct platform_driver clk_mt2701_vdec_drv = { .of_match_table = of_match_clk_mt2701_vdec, }, }; - -builtin_platform_driver(clk_mt2701_vdec_drv); +module_platform_driver(clk_mt2701_vdec_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt2701.c b/drivers/clk/mediatek/clk-mt2701.c index 00d2e81bdd43..5193b8b30f5a 100644 --- a/drivers/clk/mediatek/clk-mt2701.c +++ b/drivers/clk/mediatek/clk-mt2701.c @@ -1038,3 +1038,4 @@ static int __init clk_mt2701_init(void) } arch_initcall(clk_mt2701_init); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt2712-bdp.c b/drivers/clk/mediatek/clk-mt2712-bdp.c index 5e668651dd90..d4979f2fb1d3 100644 --- a/drivers/clk/mediatek/clk-mt2712-bdp.c +++ b/drivers/clk/mediatek/clk-mt2712-bdp.c @@ -74,5 +74,5 @@ static struct platform_driver clk_mt2712_bdp_drv = { .of_match_table = of_match_clk_mt2712_bdp, }, }; - -builtin_platform_driver(clk_mt2712_bdp_drv); +module_platform_driver(clk_mt2712_bdp_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt2712-img.c b/drivers/clk/mediatek/clk-mt2712-img.c index 3ffa51384e6b..cd2d2af6bd56 100644 --- a/drivers/clk/mediatek/clk-mt2712-img.c +++ b/drivers/clk/mediatek/clk-mt2712-img.c @@ -52,5 +52,5 @@ static struct platform_driver clk_mt2712_img_drv = { .of_match_table = of_match_clk_mt2712_img, }, }; - -builtin_platform_driver(clk_mt2712_img_drv); +module_platform_driver(clk_mt2712_img_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt2712-jpgdec.c b/drivers/clk/mediatek/clk-mt2712-jpgdec.c index 8c768d5ce24d..815be759e7c9 100644 --- a/drivers/clk/mediatek/clk-mt2712-jpgdec.c +++ b/drivers/clk/mediatek/clk-mt2712-jpgdec.c @@ -48,5 +48,5 @@ static struct platform_driver clk_mt2712_jpgdec_drv = { .of_match_table = of_match_clk_mt2712_jpgdec, }, }; - -builtin_platform_driver(clk_mt2712_jpgdec_drv); +module_platform_driver(clk_mt2712_jpgdec_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt2712-mfg.c b/drivers/clk/mediatek/clk-mt2712-mfg.c index 8949315c2dd2..9f958fadca86 100644 --- a/drivers/clk/mediatek/clk-mt2712-mfg.c +++ b/drivers/clk/mediatek/clk-mt2712-mfg.c @@ -47,5 +47,5 @@ static struct platform_driver clk_mt2712_mfg_drv = { .of_match_table = of_match_clk_mt2712_mfg, }, }; - -builtin_platform_driver(clk_mt2712_mfg_drv); +module_platform_driver(clk_mt2712_mfg_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt2712-mm.c b/drivers/clk/mediatek/clk-mt2712-mm.c index ad6daa8f28a8..8e2cd30fab6a 100644 --- a/drivers/clk/mediatek/clk-mt2712-mm.c +++ b/drivers/clk/mediatek/clk-mt2712-mm.c @@ -135,5 +135,5 @@ static struct platform_driver clk_mt2712_mm_drv = { .name = "clk-mt2712-mm", }, }; - -builtin_platform_driver(clk_mt2712_mm_drv); +module_platform_driver(clk_mt2712_mm_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt2712-vdec.c b/drivers/clk/mediatek/clk-mt2712-vdec.c index 572290dd43c8..1cb150fa7ab4 100644 --- a/drivers/clk/mediatek/clk-mt2712-vdec.c +++ b/drivers/clk/mediatek/clk-mt2712-vdec.c @@ -60,5 +60,5 @@ static struct platform_driver clk_mt2712_vdec_drv = { .of_match_table = of_match_clk_mt2712_vdec, }, }; - -builtin_platform_driver(clk_mt2712_vdec_drv); +module_platform_driver(clk_mt2712_vdec_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt2712-venc.c b/drivers/clk/mediatek/clk-mt2712-venc.c index 9588eb03016e..4073fa6c040f 100644 --- a/drivers/clk/mediatek/clk-mt2712-venc.c +++ b/drivers/clk/mediatek/clk-mt2712-venc.c @@ -49,5 +49,5 @@ static struct platform_driver clk_mt2712_venc_drv = { .of_match_table = of_match_clk_mt2712_venc, }, }; - -builtin_platform_driver(clk_mt2712_venc_drv); +module_platform_driver(clk_mt2712_venc_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt2712.c b/drivers/clk/mediatek/clk-mt2712.c index d6c2cc183b1a..5f7b148c2cca 100644 --- a/drivers/clk/mediatek/clk-mt2712.c +++ b/drivers/clk/mediatek/clk-mt2712.c @@ -1453,10 +1453,5 @@ static struct platform_driver clk_mt2712_drv = { .of_match_table = of_match_clk_mt2712, }, }; - -static int __init clk_mt2712_init(void) -{ - return platform_driver_register(&clk_mt2712_drv); -} - -arch_initcall(clk_mt2712_init); +module_platform_driver(clk_mt2712_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt6765-audio.c b/drivers/clk/mediatek/clk-mt6765-audio.c index 5682e0302eee..c1fecf22f8ad 100644 --- a/drivers/clk/mediatek/clk-mt6765-audio.c +++ b/drivers/clk/mediatek/clk-mt6765-audio.c @@ -74,5 +74,5 @@ static struct platform_driver clk_mt6765_audio_drv = { .of_match_table = of_match_clk_mt6765_audio, }, }; - -builtin_platform_driver(clk_mt6765_audio_drv); +module_platform_driver(clk_mt6765_audio_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt6765-cam.c b/drivers/clk/mediatek/clk-mt6765-cam.c index 6e7d192c19cb..15737c8f6b69 100644 --- a/drivers/clk/mediatek/clk-mt6765-cam.c +++ b/drivers/clk/mediatek/clk-mt6765-cam.c @@ -55,5 +55,5 @@ static struct platform_driver clk_mt6765_cam_drv = { .of_match_table = of_match_clk_mt6765_cam, }, }; - -builtin_platform_driver(clk_mt6765_cam_drv); +module_platform_driver(clk_mt6765_cam_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt6765-img.c b/drivers/clk/mediatek/clk-mt6765-img.c index cfbc907988af..7e421f4d01b8 100644 --- a/drivers/clk/mediatek/clk-mt6765-img.c +++ b/drivers/clk/mediatek/clk-mt6765-img.c @@ -51,5 +51,5 @@ static struct platform_driver clk_mt6765_img_drv = { .of_match_table = of_match_clk_mt6765_img, }, }; - -builtin_platform_driver(clk_mt6765_img_drv); +module_platform_driver(clk_mt6765_img_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt6765-mipi0a.c b/drivers/clk/mediatek/clk-mt6765-mipi0a.c index f2b9dc808480..17186a308e94 100644 --- a/drivers/clk/mediatek/clk-mt6765-mipi0a.c +++ b/drivers/clk/mediatek/clk-mt6765-mipi0a.c @@ -48,5 +48,5 @@ static struct platform_driver clk_mt6765_mipi0a_drv = { .of_match_table = of_match_clk_mt6765_mipi0a, }, }; - -builtin_platform_driver(clk_mt6765_mipi0a_drv); +module_platform_driver(clk_mt6765_mipi0a_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt6765-mm.c b/drivers/clk/mediatek/clk-mt6765-mm.c index a4570c9dbefa..af2213db17cb 100644 --- a/drivers/clk/mediatek/clk-mt6765-mm.c +++ b/drivers/clk/mediatek/clk-mt6765-mm.c @@ -77,5 +77,5 @@ static struct platform_driver clk_mt6765_mm_drv = { .of_match_table = of_match_clk_mt6765_mm, }, }; - -builtin_platform_driver(clk_mt6765_mm_drv); +module_platform_driver(clk_mt6765_mm_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt6765-vcodec.c b/drivers/clk/mediatek/clk-mt6765-vcodec.c index 75d72b9b4032..48014cc71d9f 100644 --- a/drivers/clk/mediatek/clk-mt6765-vcodec.c +++ b/drivers/clk/mediatek/clk-mt6765-vcodec.c @@ -50,5 +50,5 @@ static struct platform_driver clk_mt6765_vcodec_drv = { .of_match_table = of_match_clk_mt6765_vcodec, }, }; - -builtin_platform_driver(clk_mt6765_vcodec_drv); +module_platform_driver(clk_mt6765_vcodec_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt6765.c b/drivers/clk/mediatek/clk-mt6765.c index 2c6a52ff5564..25a36966aab5 100644 --- a/drivers/clk/mediatek/clk-mt6765.c +++ b/drivers/clk/mediatek/clk-mt6765.c @@ -878,3 +878,4 @@ static int __init clk_mt6765_init(void) } arch_initcall(clk_mt6765_init); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt6797-img.c b/drivers/clk/mediatek/clk-mt6797-img.c index 06441393478f..22779829370f 100644 --- a/drivers/clk/mediatek/clk-mt6797-img.c +++ b/drivers/clk/mediatek/clk-mt6797-img.c @@ -48,5 +48,5 @@ static struct platform_driver clk_mt6797_img_drv = { .of_match_table = of_match_clk_mt6797_img, }, }; - -builtin_platform_driver(clk_mt6797_img_drv); +module_platform_driver(clk_mt6797_img_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt6797-mm.c b/drivers/clk/mediatek/clk-mt6797-mm.c index 99a63f46642f..ec436770f176 100644 --- a/drivers/clk/mediatek/clk-mt6797-mm.c +++ b/drivers/clk/mediatek/clk-mt6797-mm.c @@ -107,5 +107,5 @@ static struct platform_driver clk_mt6797_mm_drv = { .name = "clk-mt6797-mm", }, }; - -builtin_platform_driver(clk_mt6797_mm_drv); +module_platform_driver(clk_mt6797_mm_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt6797-vdec.c b/drivers/clk/mediatek/clk-mt6797-vdec.c index 8622ddd87a5b..c5a82d4a958b 100644 --- a/drivers/clk/mediatek/clk-mt6797-vdec.c +++ b/drivers/clk/mediatek/clk-mt6797-vdec.c @@ -59,5 +59,5 @@ static struct platform_driver clk_mt6797_vdec_drv = { .of_match_table = of_match_clk_mt6797_vdec, }, }; - -builtin_platform_driver(clk_mt6797_vdec_drv); +module_platform_driver(clk_mt6797_vdec_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt6797-venc.c b/drivers/clk/mediatek/clk-mt6797-venc.c index 928d611a476e..afdd288d313a 100644 --- a/drivers/clk/mediatek/clk-mt6797-venc.c +++ b/drivers/clk/mediatek/clk-mt6797-venc.c @@ -50,5 +50,5 @@ static struct platform_driver clk_mt6797_venc_drv = { .of_match_table = of_match_clk_mt6797_venc, }, }; - -builtin_platform_driver(clk_mt6797_venc_drv); +module_platform_driver(clk_mt6797_venc_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt6797.c b/drivers/clk/mediatek/clk-mt6797.c index b362e99c8f53..fb67a662d415 100644 --- a/drivers/clk/mediatek/clk-mt6797.c +++ b/drivers/clk/mediatek/clk-mt6797.c @@ -708,3 +708,4 @@ static int __init clk_mt6797_init(void) } arch_initcall(clk_mt6797_init); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt7622-aud.c b/drivers/clk/mediatek/clk-mt7622-aud.c index b17731fa1144..626885e29fe3 100644 --- a/drivers/clk/mediatek/clk-mt7622-aud.c +++ b/drivers/clk/mediatek/clk-mt7622-aud.c @@ -172,5 +172,5 @@ static struct platform_driver clk_mt7622_aud_drv = { .of_match_table = of_match_clk_mt7622_aud, }, }; - -builtin_platform_driver(clk_mt7622_aud_drv); +module_platform_driver(clk_mt7622_aud_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt7622-eth.c b/drivers/clk/mediatek/clk-mt7622-eth.c index a60190e83418..2416a30ba0e4 100644 --- a/drivers/clk/mediatek/clk-mt7622-eth.c +++ b/drivers/clk/mediatek/clk-mt7622-eth.c @@ -140,5 +140,5 @@ static struct platform_driver clk_mt7622_eth_drv = { .of_match_table = of_match_clk_mt7622_eth, }, }; - -builtin_platform_driver(clk_mt7622_eth_drv); +module_platform_driver(clk_mt7622_eth_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt7622-hif.c b/drivers/clk/mediatek/clk-mt7622-hif.c index 55baa6d06a20..fe39027b4da4 100644 --- a/drivers/clk/mediatek/clk-mt7622-hif.c +++ b/drivers/clk/mediatek/clk-mt7622-hif.c @@ -153,5 +153,5 @@ static struct platform_driver clk_mt7622_hif_drv = { .of_match_table = of_match_clk_mt7622_hif, }, }; - -builtin_platform_driver(clk_mt7622_hif_drv); +module_platform_driver(clk_mt7622_hif_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt7629-eth.c b/drivers/clk/mediatek/clk-mt7629-eth.c index e1d2635c72c1..8f72bc86bcfc 100644 --- a/drivers/clk/mediatek/clk-mt7629-eth.c +++ b/drivers/clk/mediatek/clk-mt7629-eth.c @@ -157,3 +157,4 @@ static struct platform_driver clk_mt7629_eth_drv = { }; builtin_platform_driver(clk_mt7629_eth_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt7629-hif.c b/drivers/clk/mediatek/clk-mt7629-hif.c index 3628811a2f57..cfa5bcbd33d5 100644 --- a/drivers/clk/mediatek/clk-mt7629-hif.c +++ b/drivers/clk/mediatek/clk-mt7629-hif.c @@ -148,5 +148,5 @@ static struct platform_driver clk_mt7629_hif_drv = { .of_match_table = of_match_clk_mt7629_hif, }, }; - -builtin_platform_driver(clk_mt7629_hif_drv); +module_platform_driver(clk_mt7629_hif_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt7629.c b/drivers/clk/mediatek/clk-mt7629.c index 01ee45fcd7e3..39bdb4fd5cd8 100644 --- a/drivers/clk/mediatek/clk-mt7629.c +++ b/drivers/clk/mediatek/clk-mt7629.c @@ -697,3 +697,4 @@ static int clk_mt7629_init(void) } arch_initcall(clk_mt7629_init); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt7986-apmixed.c b/drivers/clk/mediatek/clk-mt7986-apmixed.c index 62080ee4dbe3..b25f79a0a5c7 100644 --- a/drivers/clk/mediatek/clk-mt7986-apmixed.c +++ b/drivers/clk/mediatek/clk-mt7986-apmixed.c @@ -100,3 +100,4 @@ static struct platform_driver clk_mt7986_apmixed_drv = { }, }; builtin_platform_driver(clk_mt7986_apmixed_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt7986-topckgen.c b/drivers/clk/mediatek/clk-mt7986-topckgen.c index de5121cf2877..0743e917c9fe 100644 --- a/drivers/clk/mediatek/clk-mt7986-topckgen.c +++ b/drivers/clk/mediatek/clk-mt7986-topckgen.c @@ -339,4 +339,5 @@ static struct platform_driver clk_mt7986_topckgen_drv = { .of_match_table = of_match_clk_mt7986_topckgen, }, }; -builtin_platform_driver(clk_mt7986_topckgen_drv); +module_platform_driver(clk_mt7986_topckgen_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8183-audio.c b/drivers/clk/mediatek/clk-mt8183-audio.c index b2d7746eddbe..1a3858816fd0 100644 --- a/drivers/clk/mediatek/clk-mt8183-audio.c +++ b/drivers/clk/mediatek/clk-mt8183-audio.c @@ -101,5 +101,5 @@ static struct platform_driver clk_mt8183_audio_drv = { .of_match_table = of_match_clk_mt8183_audio, }, }; - -builtin_platform_driver(clk_mt8183_audio_drv); +module_platform_driver(clk_mt8183_audio_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8183-cam.c b/drivers/clk/mediatek/clk-mt8183-cam.c index 6907b1a6a824..7ecbfc50e8fe 100644 --- a/drivers/clk/mediatek/clk-mt8183-cam.c +++ b/drivers/clk/mediatek/clk-mt8183-cam.c @@ -56,5 +56,5 @@ static struct platform_driver clk_mt8183_cam_drv = { .of_match_table = of_match_clk_mt8183_cam, }, }; - -builtin_platform_driver(clk_mt8183_cam_drv); +module_platform_driver(clk_mt8183_cam_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8183-img.c b/drivers/clk/mediatek/clk-mt8183-img.c index 8d884425d79f..e75d6e6205f9 100644 --- a/drivers/clk/mediatek/clk-mt8183-img.c +++ b/drivers/clk/mediatek/clk-mt8183-img.c @@ -56,5 +56,5 @@ static struct platform_driver clk_mt8183_img_drv = { .of_match_table = of_match_clk_mt8183_img, }, }; - -builtin_platform_driver(clk_mt8183_img_drv); +module_platform_driver(clk_mt8183_img_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8183-ipu0.c b/drivers/clk/mediatek/clk-mt8183-ipu0.c index 953a8a33d048..d1dd76e7eafd 100644 --- a/drivers/clk/mediatek/clk-mt8183-ipu0.c +++ b/drivers/clk/mediatek/clk-mt8183-ipu0.c @@ -49,5 +49,5 @@ static struct platform_driver clk_mt8183_ipu_core0_drv = { .of_match_table = of_match_clk_mt8183_ipu_core0, }, }; - -builtin_platform_driver(clk_mt8183_ipu_core0_drv); +module_platform_driver(clk_mt8183_ipu_core0_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8183-ipu1.c b/drivers/clk/mediatek/clk-mt8183-ipu1.c index 221d12265974..0b4906cca119 100644 --- a/drivers/clk/mediatek/clk-mt8183-ipu1.c +++ b/drivers/clk/mediatek/clk-mt8183-ipu1.c @@ -49,5 +49,5 @@ static struct platform_driver clk_mt8183_ipu_core1_drv = { .of_match_table = of_match_clk_mt8183_ipu_core1, }, }; - -builtin_platform_driver(clk_mt8183_ipu_core1_drv); +module_platform_driver(clk_mt8183_ipu_core1_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8183-ipu_adl.c b/drivers/clk/mediatek/clk-mt8183-ipu_adl.c index 8c4fd96df821..32a82ec965ef 100644 --- a/drivers/clk/mediatek/clk-mt8183-ipu_adl.c +++ b/drivers/clk/mediatek/clk-mt8183-ipu_adl.c @@ -47,5 +47,5 @@ static struct platform_driver clk_mt8183_ipu_adl_drv = { .of_match_table = of_match_clk_mt8183_ipu_adl, }, }; - -builtin_platform_driver(clk_mt8183_ipu_adl_drv); +module_platform_driver(clk_mt8183_ipu_adl_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8183-ipu_conn.c b/drivers/clk/mediatek/clk-mt8183-ipu_conn.c index 14a4c3ff82a1..1e2ebc0eab8b 100644 --- a/drivers/clk/mediatek/clk-mt8183-ipu_conn.c +++ b/drivers/clk/mediatek/clk-mt8183-ipu_conn.c @@ -116,5 +116,5 @@ static struct platform_driver clk_mt8183_ipu_conn_drv = { .of_match_table = of_match_clk_mt8183_ipu_conn, }, }; - -builtin_platform_driver(clk_mt8183_ipu_conn_drv); +module_platform_driver(clk_mt8183_ipu_conn_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8183-mfgcfg.c b/drivers/clk/mediatek/clk-mt8183-mfgcfg.c index 730c9ae5ea12..20cdbd74817f 100644 --- a/drivers/clk/mediatek/clk-mt8183-mfgcfg.c +++ b/drivers/clk/mediatek/clk-mt8183-mfgcfg.c @@ -48,5 +48,5 @@ static struct platform_driver clk_mt8183_mfg_drv = { .of_match_table = of_match_clk_mt8183_mfg, }, }; - -builtin_platform_driver(clk_mt8183_mfg_drv); +module_platform_driver(clk_mt8183_mfg_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8183-mm.c b/drivers/clk/mediatek/clk-mt8183-mm.c index 11ecc6fb0065..2573566a9e90 100644 --- a/drivers/clk/mediatek/clk-mt8183-mm.c +++ b/drivers/clk/mediatek/clk-mt8183-mm.c @@ -102,5 +102,5 @@ static struct platform_driver clk_mt8183_mm_drv = { .name = "clk-mt8183-mm", }, }; - -builtin_platform_driver(clk_mt8183_mm_drv); +module_platform_driver(clk_mt8183_mm_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8183-vdec.c b/drivers/clk/mediatek/clk-mt8183-vdec.c index c294e50b96b7..0182745e2a18 100644 --- a/drivers/clk/mediatek/clk-mt8183-vdec.c +++ b/drivers/clk/mediatek/clk-mt8183-vdec.c @@ -60,5 +60,5 @@ static struct platform_driver clk_mt8183_vdec_drv = { .of_match_table = of_match_clk_mt8183_vdec, }, }; - -builtin_platform_driver(clk_mt8183_vdec_drv); +module_platform_driver(clk_mt8183_vdec_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8183-venc.c b/drivers/clk/mediatek/clk-mt8183-venc.c index 0051c5d92fc5..c840e6ebe375 100644 --- a/drivers/clk/mediatek/clk-mt8183-venc.c +++ b/drivers/clk/mediatek/clk-mt8183-venc.c @@ -52,5 +52,5 @@ static struct platform_driver clk_mt8183_venc_drv = { .of_match_table = of_match_clk_mt8183_venc, }, }; - -builtin_platform_driver(clk_mt8183_venc_drv); +module_platform_driver(clk_mt8183_venc_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8183.c b/drivers/clk/mediatek/clk-mt8183.c index 1860a35a723a..d03001e86df2 100644 --- a/drivers/clk/mediatek/clk-mt8183.c +++ b/drivers/clk/mediatek/clk-mt8183.c @@ -1360,10 +1360,5 @@ static struct platform_driver clk_mt8183_drv = { .of_match_table = of_match_clk_mt8183, }, }; - -static int __init clk_mt8183_init(void) -{ - return platform_driver_register(&clk_mt8183_drv); -} - -arch_initcall(clk_mt8183_init); +module_platform_driver(clk_mt8183_drv) +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8186-apmixedsys.c b/drivers/clk/mediatek/clk-mt8186-apmixedsys.c index e692a2a67ce1..28490569b7b4 100644 --- a/drivers/clk/mediatek/clk-mt8186-apmixedsys.c +++ b/drivers/clk/mediatek/clk-mt8186-apmixedsys.c @@ -130,4 +130,5 @@ static struct platform_driver clk_mt8186_apmixed_drv = { .of_match_table = of_match_clk_mt8186_apmixed, }, }; -builtin_platform_driver(clk_mt8186_apmixed_drv); +module_platform_driver(clk_mt8186_apmixed_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8186-cam.c b/drivers/clk/mediatek/clk-mt8186-cam.c index 9ec345a2ce66..90d91d045909 100644 --- a/drivers/clk/mediatek/clk-mt8186-cam.c +++ b/drivers/clk/mediatek/clk-mt8186-cam.c @@ -87,4 +87,5 @@ static struct platform_driver clk_mt8186_cam_drv = { .of_match_table = of_match_clk_mt8186_cam, }, }; -builtin_platform_driver(clk_mt8186_cam_drv); +module_platform_driver(clk_mt8186_cam_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8186-img.c b/drivers/clk/mediatek/clk-mt8186-img.c index 08a625475aee..11314dba4e41 100644 --- a/drivers/clk/mediatek/clk-mt8186-img.c +++ b/drivers/clk/mediatek/clk-mt8186-img.c @@ -65,4 +65,5 @@ static struct platform_driver clk_mt8186_img_drv = { .of_match_table = of_match_clk_mt8186_img, }, }; -builtin_platform_driver(clk_mt8186_img_drv); +module_platform_driver(clk_mt8186_img_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8186-imp_iic_wrap.c b/drivers/clk/mediatek/clk-mt8186-imp_iic_wrap.c index 47f2e480a05e..f8d1ee294f09 100644 --- a/drivers/clk/mediatek/clk-mt8186-imp_iic_wrap.c +++ b/drivers/clk/mediatek/clk-mt8186-imp_iic_wrap.c @@ -64,4 +64,5 @@ static struct platform_driver clk_mt8186_imp_iic_wrap_drv = { .of_match_table = of_match_clk_mt8186_imp_iic_wrap, }, }; -builtin_platform_driver(clk_mt8186_imp_iic_wrap_drv); +module_platform_driver(clk_mt8186_imp_iic_wrap_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8186-infra_ao.c b/drivers/clk/mediatek/clk-mt8186-infra_ao.c index df2a6bd1aefa..766c8e6bcf8d 100644 --- a/drivers/clk/mediatek/clk-mt8186-infra_ao.c +++ b/drivers/clk/mediatek/clk-mt8186-infra_ao.c @@ -236,4 +236,5 @@ static struct platform_driver clk_mt8186_infra_ao_drv = { .of_match_table = of_match_clk_mt8186_infra_ao, }, }; -builtin_platform_driver(clk_mt8186_infra_ao_drv); +module_platform_driver(clk_mt8186_infra_ao_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8186-ipe.c b/drivers/clk/mediatek/clk-mt8186-ipe.c index 8fca148effa6..0876440839df 100644 --- a/drivers/clk/mediatek/clk-mt8186-ipe.c +++ b/drivers/clk/mediatek/clk-mt8186-ipe.c @@ -52,4 +52,5 @@ static struct platform_driver clk_mt8186_ipe_drv = { .of_match_table = of_match_clk_mt8186_ipe, }, }; -builtin_platform_driver(clk_mt8186_ipe_drv); +module_platform_driver(clk_mt8186_ipe_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8186-mcu.c b/drivers/clk/mediatek/clk-mt8186-mcu.c index dfc305c1fc5d..512b3fd32997 100644 --- a/drivers/clk/mediatek/clk-mt8186-mcu.c +++ b/drivers/clk/mediatek/clk-mt8186-mcu.c @@ -105,4 +105,5 @@ static struct platform_driver clk_mt8186_mcu_drv = { .of_match_table = of_match_clk_mt8186_mcu, }, }; -builtin_platform_driver(clk_mt8186_mcu_drv); +module_platform_driver(clk_mt8186_mcu_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8186-mdp.c b/drivers/clk/mediatek/clk-mt8186-mdp.c index 05174088ef20..194f20f16f8c 100644 --- a/drivers/clk/mediatek/clk-mt8186-mdp.c +++ b/drivers/clk/mediatek/clk-mt8186-mdp.c @@ -77,4 +77,5 @@ static struct platform_driver clk_mt8186_mdp_drv = { .of_match_table = of_match_clk_mt8186_mdp, }, }; -builtin_platform_driver(clk_mt8186_mdp_drv); +module_platform_driver(clk_mt8186_mdp_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8186-mfg.c b/drivers/clk/mediatek/clk-mt8186-mfg.c index f1f92216f894..762e17a15b20 100644 --- a/drivers/clk/mediatek/clk-mt8186-mfg.c +++ b/drivers/clk/mediatek/clk-mt8186-mfg.c @@ -45,4 +45,5 @@ static struct platform_driver clk_mt8186_mfg_drv = { .of_match_table = of_match_clk_mt8186_mfg, }, }; -builtin_platform_driver(clk_mt8186_mfg_drv); +module_platform_driver(clk_mt8186_mfg_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8186-mm.c b/drivers/clk/mediatek/clk-mt8186-mm.c index 1d33be407947..1aefe960967c 100644 --- a/drivers/clk/mediatek/clk-mt8186-mm.c +++ b/drivers/clk/mediatek/clk-mt8186-mm.c @@ -108,4 +108,5 @@ static struct platform_driver clk_mt8186_mm_drv = { .name = "clk-mt8186-mm", }, }; -builtin_platform_driver(clk_mt8186_mm_drv); +module_platform_driver(clk_mt8186_mm_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8186-topckgen.c b/drivers/clk/mediatek/clk-mt8186-topckgen.c index d7f2c4663c85..26caa3d6cc8a 100644 --- a/drivers/clk/mediatek/clk-mt8186-topckgen.c +++ b/drivers/clk/mediatek/clk-mt8186-topckgen.c @@ -777,4 +777,5 @@ static struct platform_driver clk_mt8186_topck_drv = { .of_match_table = of_match_clk_mt8186_topck, }, }; -builtin_platform_driver(clk_mt8186_topck_drv); +module_platform_driver(clk_mt8186_topck_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8186-vdec.c b/drivers/clk/mediatek/clk-mt8186-vdec.c index 5ad7e1ae0bac..877ca9d8ff65 100644 --- a/drivers/clk/mediatek/clk-mt8186-vdec.c +++ b/drivers/clk/mediatek/clk-mt8186-vdec.c @@ -85,4 +85,5 @@ static struct platform_driver clk_mt8186_vdec_drv = { .of_match_table = of_match_clk_mt8186_vdec, }, }; -builtin_platform_driver(clk_mt8186_vdec_drv); +module_platform_driver(clk_mt8186_vdec_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8186-venc.c b/drivers/clk/mediatek/clk-mt8186-venc.c index f5519f794c45..a89e82285d4c 100644 --- a/drivers/clk/mediatek/clk-mt8186-venc.c +++ b/drivers/clk/mediatek/clk-mt8186-venc.c @@ -48,4 +48,5 @@ static struct platform_driver clk_mt8186_venc_drv = { .of_match_table = of_match_clk_mt8186_venc, }, }; -builtin_platform_driver(clk_mt8186_venc_drv); +module_platform_driver(clk_mt8186_venc_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8186-wpe.c b/drivers/clk/mediatek/clk-mt8186-wpe.c index 8db3e9178a1e..fba55f08129c 100644 --- a/drivers/clk/mediatek/clk-mt8186-wpe.c +++ b/drivers/clk/mediatek/clk-mt8186-wpe.c @@ -48,4 +48,5 @@ static struct platform_driver clk_mt8186_wpe_drv = { .of_match_table = of_match_clk_mt8186_wpe, }, }; -builtin_platform_driver(clk_mt8186_wpe_drv); +module_platform_driver(clk_mt8186_wpe_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8192-aud.c b/drivers/clk/mediatek/clk-mt8192-aud.c index 8c989bffd8c7..4ca1f6fd5fce 100644 --- a/drivers/clk/mediatek/clk-mt8192-aud.c +++ b/drivers/clk/mediatek/clk-mt8192-aud.c @@ -114,5 +114,5 @@ static struct platform_driver clk_mt8192_aud_drv = { .of_match_table = of_match_clk_mt8192_aud, }, }; - -builtin_platform_driver(clk_mt8192_aud_drv); +module_platform_driver(clk_mt8192_aud_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8192-cam.c b/drivers/clk/mediatek/clk-mt8192-cam.c index 90b57d46eef7..321693aff33f 100644 --- a/drivers/clk/mediatek/clk-mt8192-cam.c +++ b/drivers/clk/mediatek/clk-mt8192-cam.c @@ -104,5 +104,5 @@ static struct platform_driver clk_mt8192_cam_drv = { .of_match_table = of_match_clk_mt8192_cam, }, }; - -builtin_platform_driver(clk_mt8192_cam_drv); +module_platform_driver(clk_mt8192_cam_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8192-img.c b/drivers/clk/mediatek/clk-mt8192-img.c index da82d65a7650..b2c148ead7f8 100644 --- a/drivers/clk/mediatek/clk-mt8192-img.c +++ b/drivers/clk/mediatek/clk-mt8192-img.c @@ -67,5 +67,5 @@ static struct platform_driver clk_mt8192_img_drv = { .of_match_table = of_match_clk_mt8192_img, }, }; - -builtin_platform_driver(clk_mt8192_img_drv); +module_platform_driver(clk_mt8192_img_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8192-imp_iic_wrap.c b/drivers/clk/mediatek/clk-mt8192-imp_iic_wrap.c index ff8e20bb44bb..fd731eb6d62e 100644 --- a/drivers/clk/mediatek/clk-mt8192-imp_iic_wrap.c +++ b/drivers/clk/mediatek/clk-mt8192-imp_iic_wrap.c @@ -116,5 +116,5 @@ static struct platform_driver clk_mt8192_imp_iic_wrap_drv = { .of_match_table = of_match_clk_mt8192_imp_iic_wrap, }, }; - -builtin_platform_driver(clk_mt8192_imp_iic_wrap_drv); +module_platform_driver(clk_mt8192_imp_iic_wrap_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8192-ipe.c b/drivers/clk/mediatek/clk-mt8192-ipe.c index 0225abe4170a..3fd37977ea16 100644 --- a/drivers/clk/mediatek/clk-mt8192-ipe.c +++ b/drivers/clk/mediatek/clk-mt8192-ipe.c @@ -54,5 +54,5 @@ static struct platform_driver clk_mt8192_ipe_drv = { .of_match_table = of_match_clk_mt8192_ipe, }, }; - -builtin_platform_driver(clk_mt8192_ipe_drv); +module_platform_driver(clk_mt8192_ipe_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8192-mdp.c b/drivers/clk/mediatek/clk-mt8192-mdp.c index 4675788d7816..817e8bf4543b 100644 --- a/drivers/clk/mediatek/clk-mt8192-mdp.c +++ b/drivers/clk/mediatek/clk-mt8192-mdp.c @@ -79,5 +79,5 @@ static struct platform_driver clk_mt8192_mdp_drv = { .of_match_table = of_match_clk_mt8192_mdp, }, }; - -builtin_platform_driver(clk_mt8192_mdp_drv); +module_platform_driver(clk_mt8192_mdp_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8192-mfg.c b/drivers/clk/mediatek/clk-mt8192-mfg.c index ec5b44ffa458..7f77e2d8b765 100644 --- a/drivers/clk/mediatek/clk-mt8192-mfg.c +++ b/drivers/clk/mediatek/clk-mt8192-mfg.c @@ -49,5 +49,5 @@ static struct platform_driver clk_mt8192_mfg_drv = { .of_match_table = of_match_clk_mt8192_mfg, }, }; - -builtin_platform_driver(clk_mt8192_mfg_drv); +module_platform_driver(clk_mt8192_mfg_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8192-mm.c b/drivers/clk/mediatek/clk-mt8192-mm.c index 1be3ff4d407d..2c557049cf9a 100644 --- a/drivers/clk/mediatek/clk-mt8192-mm.c +++ b/drivers/clk/mediatek/clk-mt8192-mm.c @@ -104,5 +104,5 @@ static struct platform_driver clk_mt8192_mm_drv = { .name = "clk-mt8192-mm", }, }; - -builtin_platform_driver(clk_mt8192_mm_drv); +module_platform_driver(clk_mt8192_mm_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8192-msdc.c b/drivers/clk/mediatek/clk-mt8192-msdc.c index a72e1b73fce8..cde8c88673e4 100644 --- a/drivers/clk/mediatek/clk-mt8192-msdc.c +++ b/drivers/clk/mediatek/clk-mt8192-msdc.c @@ -61,5 +61,5 @@ static struct platform_driver clk_mt8192_msdc_drv = { .of_match_table = of_match_clk_mt8192_msdc, }, }; - -builtin_platform_driver(clk_mt8192_msdc_drv); +module_platform_driver(clk_mt8192_msdc_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8192-scp_adsp.c b/drivers/clk/mediatek/clk-mt8192-scp_adsp.c index 18a8679108b8..df086b95d447 100644 --- a/drivers/clk/mediatek/clk-mt8192-scp_adsp.c +++ b/drivers/clk/mediatek/clk-mt8192-scp_adsp.c @@ -47,5 +47,5 @@ static struct platform_driver clk_mt8192_scp_adsp_drv = { .of_match_table = of_match_clk_mt8192_scp_adsp, }, }; - -builtin_platform_driver(clk_mt8192_scp_adsp_drv); +module_platform_driver(clk_mt8192_scp_adsp_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8192-vdec.c b/drivers/clk/mediatek/clk-mt8192-vdec.c index e149962dbbf9..2ea8b6d4d405 100644 --- a/drivers/clk/mediatek/clk-mt8192-vdec.c +++ b/drivers/clk/mediatek/clk-mt8192-vdec.c @@ -91,5 +91,5 @@ static struct platform_driver clk_mt8192_vdec_drv = { .of_match_table = of_match_clk_mt8192_vdec, }, }; - -builtin_platform_driver(clk_mt8192_vdec_drv); +module_platform_driver(clk_mt8192_vdec_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8192-venc.c b/drivers/clk/mediatek/clk-mt8192-venc.c index 80b8bb170996..d73c12bc2ae9 100644 --- a/drivers/clk/mediatek/clk-mt8192-venc.c +++ b/drivers/clk/mediatek/clk-mt8192-venc.c @@ -50,5 +50,5 @@ static struct platform_driver clk_mt8192_venc_drv = { .of_match_table = of_match_clk_mt8192_venc, }, }; - -builtin_platform_driver(clk_mt8192_venc_drv); +module_platform_driver(clk_mt8192_venc_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8192.c b/drivers/clk/mediatek/clk-mt8192.c index d0f226931070..43c10eeac317 100644 --- a/drivers/clk/mediatek/clk-mt8192.c +++ b/drivers/clk/mediatek/clk-mt8192.c @@ -1251,3 +1251,4 @@ static int __init clk_mt8192_init(void) } arch_initcall(clk_mt8192_init); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8195-apmixedsys.c b/drivers/clk/mediatek/clk-mt8195-apmixedsys.c index 0dfed6ec4d15..0206d5b264d1 100644 --- a/drivers/clk/mediatek/clk-mt8195-apmixedsys.c +++ b/drivers/clk/mediatek/clk-mt8195-apmixedsys.c @@ -166,4 +166,5 @@ static struct platform_driver clk_mt8195_apmixed_drv = { .of_match_table = of_match_clk_mt8195_apmixed, }, }; -builtin_platform_driver(clk_mt8195_apmixed_drv); +module_platform_driver(clk_mt8195_apmixed_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8195-apusys_pll.c b/drivers/clk/mediatek/clk-mt8195-apusys_pll.c index 0b52f6a009c4..b167b8d31bcc 100644 --- a/drivers/clk/mediatek/clk-mt8195-apusys_pll.c +++ b/drivers/clk/mediatek/clk-mt8195-apusys_pll.c @@ -110,4 +110,5 @@ static struct platform_driver clk_mt8195_apusys_pll_drv = { .of_match_table = of_match_clk_mt8195_apusys_pll, }, }; -builtin_platform_driver(clk_mt8195_apusys_pll_drv); +module_platform_driver(clk_mt8195_apusys_pll_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8195-cam.c b/drivers/clk/mediatek/clk-mt8195-cam.c index e4d00fe6e757..c515650214bd 100644 --- a/drivers/clk/mediatek/clk-mt8195-cam.c +++ b/drivers/clk/mediatek/clk-mt8195-cam.c @@ -140,4 +140,5 @@ static struct platform_driver clk_mt8195_cam_drv = { .of_match_table = of_match_clk_mt8195_cam, }, }; -builtin_platform_driver(clk_mt8195_cam_drv); +module_platform_driver(clk_mt8195_cam_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8195-ccu.c b/drivers/clk/mediatek/clk-mt8195-ccu.c index 4e326b6301ba..a86f848a785d 100644 --- a/drivers/clk/mediatek/clk-mt8195-ccu.c +++ b/drivers/clk/mediatek/clk-mt8195-ccu.c @@ -48,4 +48,5 @@ static struct platform_driver clk_mt8195_ccu_drv = { .of_match_table = of_match_clk_mt8195_ccu, }, }; -builtin_platform_driver(clk_mt8195_ccu_drv); +module_platform_driver(clk_mt8195_ccu_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8195-img.c b/drivers/clk/mediatek/clk-mt8195-img.c index 12f5c436d075..214baa2bc974 100644 --- a/drivers/clk/mediatek/clk-mt8195-img.c +++ b/drivers/clk/mediatek/clk-mt8195-img.c @@ -94,4 +94,5 @@ static struct platform_driver clk_mt8195_img_drv = { .of_match_table = of_match_clk_mt8195_img, }, }; -builtin_platform_driver(clk_mt8195_img_drv); +module_platform_driver(clk_mt8195_img_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8195-imp_iic_wrap.c b/drivers/clk/mediatek/clk-mt8195-imp_iic_wrap.c index fbc809d05072..191f482186ae 100644 --- a/drivers/clk/mediatek/clk-mt8195-imp_iic_wrap.c +++ b/drivers/clk/mediatek/clk-mt8195-imp_iic_wrap.c @@ -64,4 +64,5 @@ static struct platform_driver clk_mt8195_imp_iic_wrap_drv = { .of_match_table = of_match_clk_mt8195_imp_iic_wrap, }, }; -builtin_platform_driver(clk_mt8195_imp_iic_wrap_drv); +module_platform_driver(clk_mt8195_imp_iic_wrap_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8195-infra_ao.c b/drivers/clk/mediatek/clk-mt8195-infra_ao.c index fcd410461d3b..6ad72a320d5e 100644 --- a/drivers/clk/mediatek/clk-mt8195-infra_ao.c +++ b/drivers/clk/mediatek/clk-mt8195-infra_ao.c @@ -238,4 +238,5 @@ static struct platform_driver clk_mt8195_infra_ao_drv = { .of_match_table = of_match_clk_mt8195_infra_ao, }, }; -builtin_platform_driver(clk_mt8195_infra_ao_drv); +module_platform_driver(clk_mt8195_infra_ao_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8195-ipe.c b/drivers/clk/mediatek/clk-mt8195-ipe.c index b0d745cf7752..b547a99b4ad9 100644 --- a/drivers/clk/mediatek/clk-mt8195-ipe.c +++ b/drivers/clk/mediatek/clk-mt8195-ipe.c @@ -49,4 +49,5 @@ static struct platform_driver clk_mt8195_ipe_drv = { .of_match_table = of_match_clk_mt8195_ipe, }, }; -builtin_platform_driver(clk_mt8195_ipe_drv); +module_platform_driver(clk_mt8195_ipe_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8195-mfg.c b/drivers/clk/mediatek/clk-mt8195-mfg.c index c94cb71bd9b9..75cf643ab885 100644 --- a/drivers/clk/mediatek/clk-mt8195-mfg.c +++ b/drivers/clk/mediatek/clk-mt8195-mfg.c @@ -47,4 +47,5 @@ static struct platform_driver clk_mt8195_mfg_drv = { .of_match_table = of_match_clk_mt8195_mfg, }, }; -builtin_platform_driver(clk_mt8195_mfg_drv); +module_platform_driver(clk_mt8195_mfg_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8195-peri_ao.c b/drivers/clk/mediatek/clk-mt8195-peri_ao.c index 2f6b3bb657db..7d65a478182d 100644 --- a/drivers/clk/mediatek/clk-mt8195-peri_ao.c +++ b/drivers/clk/mediatek/clk-mt8195-peri_ao.c @@ -60,4 +60,5 @@ static struct platform_driver clk_mt8195_peri_ao_drv = { .of_match_table = of_match_clk_mt8195_peri_ao, }, }; -builtin_platform_driver(clk_mt8195_peri_ao_drv); +module_platform_driver(clk_mt8195_peri_ao_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8195-scp_adsp.c b/drivers/clk/mediatek/clk-mt8195-scp_adsp.c index e16c383f631b..24e09339ed7d 100644 --- a/drivers/clk/mediatek/clk-mt8195-scp_adsp.c +++ b/drivers/clk/mediatek/clk-mt8195-scp_adsp.c @@ -45,4 +45,5 @@ static struct platform_driver clk_mt8195_scp_adsp_drv = { .of_match_table = of_match_clk_mt8195_scp_adsp, }, }; -builtin_platform_driver(clk_mt8195_scp_adsp_drv); +module_platform_driver(clk_mt8195_scp_adsp_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8195-topckgen.c b/drivers/clk/mediatek/clk-mt8195-topckgen.c index 1e016329c1d2..726bdcb3622d 100644 --- a/drivers/clk/mediatek/clk-mt8195-topckgen.c +++ b/drivers/clk/mediatek/clk-mt8195-topckgen.c @@ -1337,4 +1337,5 @@ static struct platform_driver clk_mt8195_topck_drv = { .of_match_table = of_match_clk_mt8195_topck, }, }; -builtin_platform_driver(clk_mt8195_topck_drv); +module_platform_driver(clk_mt8195_topck_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8195-vdec.c b/drivers/clk/mediatek/clk-mt8195-vdec.c index a1446b666385..f9e2bd402627 100644 --- a/drivers/clk/mediatek/clk-mt8195-vdec.c +++ b/drivers/clk/mediatek/clk-mt8195-vdec.c @@ -102,4 +102,5 @@ static struct platform_driver clk_mt8195_vdec_drv = { .of_match_table = of_match_clk_mt8195_vdec, }, }; -builtin_platform_driver(clk_mt8195_vdec_drv); +module_platform_driver(clk_mt8195_vdec_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8195-vdo0.c b/drivers/clk/mediatek/clk-mt8195-vdo0.c index 07b46bfd5040..e6003a07df4a 100644 --- a/drivers/clk/mediatek/clk-mt8195-vdo0.c +++ b/drivers/clk/mediatek/clk-mt8195-vdo0.c @@ -143,4 +143,5 @@ static struct platform_driver clk_mt8195_vdo0_drv = { .name = "clk-mt8195-vdo0", }, }; -builtin_platform_driver(clk_mt8195_vdo0_drv); +module_platform_driver(clk_mt8195_vdo0_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8195-vdo1.c b/drivers/clk/mediatek/clk-mt8195-vdo1.c index 835335b9d87b..ed564b2255e4 100644 --- a/drivers/clk/mediatek/clk-mt8195-vdo1.c +++ b/drivers/clk/mediatek/clk-mt8195-vdo1.c @@ -170,4 +170,5 @@ static struct platform_driver clk_mt8195_vdo1_drv = { .name = "clk-mt8195-vdo1", }, }; -builtin_platform_driver(clk_mt8195_vdo1_drv); +module_platform_driver(clk_mt8195_vdo1_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8195-venc.c b/drivers/clk/mediatek/clk-mt8195-venc.c index 622f57804f96..d050d87bb3f2 100644 --- a/drivers/clk/mediatek/clk-mt8195-venc.c +++ b/drivers/clk/mediatek/clk-mt8195-venc.c @@ -67,4 +67,5 @@ static struct platform_driver clk_mt8195_venc_drv = { .of_match_table = of_match_clk_mt8195_venc, }, }; -builtin_platform_driver(clk_mt8195_venc_drv); +module_platform_driver(clk_mt8195_venc_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8195-vpp0.c b/drivers/clk/mediatek/clk-mt8195-vpp0.c index bf2939c3a023..67c2509ae39e 100644 --- a/drivers/clk/mediatek/clk-mt8195-vpp0.c +++ b/drivers/clk/mediatek/clk-mt8195-vpp0.c @@ -108,4 +108,5 @@ static struct platform_driver clk_mt8195_vpp0_drv = { .of_match_table = of_match_clk_mt8195_vpp0, }, }; -builtin_platform_driver(clk_mt8195_vpp0_drv); +module_platform_driver(clk_mt8195_vpp0_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8195-vpp1.c b/drivers/clk/mediatek/clk-mt8195-vpp1.c index ffd52c762890..7e997a0794d8 100644 --- a/drivers/clk/mediatek/clk-mt8195-vpp1.c +++ b/drivers/clk/mediatek/clk-mt8195-vpp1.c @@ -106,4 +106,5 @@ static struct platform_driver clk_mt8195_vpp1_drv = { .of_match_table = of_match_clk_mt8195_vpp1, }, }; -builtin_platform_driver(clk_mt8195_vpp1_drv); +module_platform_driver(clk_mt8195_vpp1_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8195-wpe.c b/drivers/clk/mediatek/clk-mt8195-wpe.c index b483fab10e18..52cebe297abf 100644 --- a/drivers/clk/mediatek/clk-mt8195-wpe.c +++ b/drivers/clk/mediatek/clk-mt8195-wpe.c @@ -141,4 +141,5 @@ static struct platform_driver clk_mt8195_wpe_drv = { .of_match_table = of_match_clk_mt8195_wpe, }, }; -builtin_platform_driver(clk_mt8195_wpe_drv); +module_platform_driver(clk_mt8195_wpe_drv); +MODULE_LICENSE("GPL"); From 34e8dc4ed04d344180c2e11376418c63692411c9 Mon Sep 17 00:00:00 2001 From: AngeloGioacchino Del Regno Date: Mon, 6 Mar 2023 15:05:35 +0100 Subject: [PATCH 06/86] BACKPORT: clk: mediatek: Split configuration options for MT8186 clock drivers When building clock drivers for MT8186, some may want to build in only some of them to, for example, get CPUFreq up faster, and some may want to leave out some clock drivers entirely as a machine may not need the Warp Engine or the camera ISP (hence, their clock drivers). Split the various clock drivers in their own configuration options, keeping MT8186 configuration options consistent with other MediaTek SoCs. While at it, also allow building the remaining clock drivers as modules by switching COMMON_CLK_MT8186 to tristate. Bug: 335112842 (cherry picked from commit 5baf38e06a570a2a4ed471a996aff6d6ba69cceb https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/ master) Signed-off-by: AngeloGioacchino Del Regno Reviewed-by: Chen-Yu Tsai Link: https://lore.kernel.org/r/20230306140543.1813621-47-angelogioacchino.delregno@collabora.com Signed-off-by: Stephen Boyd Change-Id: Id9dad0e1d56fcae3bb9302b7db63be31afc91d5d --- drivers/clk/mediatek/Kconfig | 79 ++++++++++++++++++++++++++++++++++- drivers/clk/mediatek/Makefile | 18 +++++--- 2 files changed, 91 insertions(+), 6 deletions(-) diff --git a/drivers/clk/mediatek/Kconfig b/drivers/clk/mediatek/Kconfig index 843cea0c7a44..32bad023da98 100644 --- a/drivers/clk/mediatek/Kconfig +++ b/drivers/clk/mediatek/Kconfig @@ -550,13 +550,90 @@ config COMMON_CLK_MT8183_VENCSYS This driver supports MediaTek MT8183 vencsys clocks. config COMMON_CLK_MT8186 - bool "Clock driver for MediaTek MT8186" + tristate "Clock driver for MediaTek MT8186" depends on ARM64 || COMPILE_TEST select COMMON_CLK_MEDIATEK default ARCH_MEDIATEK help This driver supports MediaTek MT8186 clocks. +config COMMON_CLK_MT8186_CAMSYS + tristate "Clock driver for MediaTek MT8186 camsys" + depends on COMMON_CLK_MT8186 + default COMMON_CLK_MT8186 + help + This driver supports MediaTek MT8186 camsys and camsys_raw clocks. + +config COMMON_CLK_MT8186_IMGSYS + tristate "Clock driver for MediaTek MT8186 imgsys" + depends on COMMON_CLK_MT8186 + default COMMON_CLK_MT8186 + help + This driver supports MediaTek MT8186 imgsys and imgsys2 clocks. + +config COMMON_CLK_MT8186_IPESYS + tristate "Clock driver for MediaTek MT8186 ipesys" + depends on COMMON_CLK_MT8186_IMGSYS + default COMMON_CLK_MT8186_IMGSYS + help + This driver supports MediaTek MT8186 ipesys clocks. + +config COMMON_CLK_MT8186_WPESYS + tristate "Clock driver for MediaTek MT8186 wpesys" + depends on COMMON_CLK_MT8186_IMGSYS + default COMMON_CLK_MT8186_IMGSYS + help + This driver supports MediaTek MT8186 Warp Engine clocks. + +config COMMON_CLK_MT8186_IMP_IIC_WRAP + tristate "Clock driver for MediaTek MT8186 imp_iic_wrap" + depends on COMMON_CLK_MT8186 + default COMMON_CLK_MT8186 + help + This driver supports MediaTek MT8186 imp_iic_wrap clocks. + +config COMMON_CLK_MT8186_MCUSYS + tristate "Clock driver for MediaTek MT8186 mcusys" + depends on COMMON_CLK_MT8186 + default COMMON_CLK_MT8186 + help + This driver supports MediaTek MT8186 mcusys clocks. + +config COMMON_CLK_MT8186_MDPSYS + tristate "Clock driver for MediaTek MT8186 mdpsys" + depends on COMMON_CLK_MT8186 + default COMMON_CLK_MT8186 + help + This driver supports MediaTek MT8186 mdpsys clocks. + +config COMMON_CLK_MT8186_MFGCFG + tristate "Clock driver for MediaTek MT8186 mfgcfg" + depends on COMMON_CLK_MT8186 + default COMMON_CLK_MT8186 + help + This driver supports MediaTek MT8186 mfgcfg clocks. + +config COMMON_CLK_MT8186_MMSYS + tristate "Clock driver for MediaTek MT8186 mmsys" + depends on COMMON_CLK_MT8186 + default COMMON_CLK_MT8186 + help + This driver supports MediaTek MT8186 mmsys clocks. + +config COMMON_CLK_MT8186_VDECSYS + tristate "Clock driver for MediaTek MT8186 vdecsys" + depends on COMMON_CLK_MT8186 + default COMMON_CLK_MT8186 + help + This driver supports MediaTek MT8186 vdecsys and vdecsys_soc clocks. + +config COMMON_CLK_MT8186_VENCSYS + tristate "Clock driver for MediaTek MT8186 vencsys" + depends on COMMON_CLK_MT8186 + default COMMON_CLK_MT8186 + help + This driver supports MediaTek MT8186 vencsys clocks. + config COMMON_CLK_MT8192 bool "Clock driver for MediaTek MT8192" depends on ARM64 || COMPILE_TEST diff --git a/drivers/clk/mediatek/Makefile b/drivers/clk/mediatek/Makefile index ea3b73240303..a40e9e669516 100644 --- a/drivers/clk/mediatek/Makefile +++ b/drivers/clk/mediatek/Makefile @@ -77,11 +77,19 @@ obj-$(CONFIG_COMMON_CLK_MT8183_MFGCFG) += clk-mt8183-mfgcfg.o obj-$(CONFIG_COMMON_CLK_MT8183_MMSYS) += clk-mt8183-mm.o obj-$(CONFIG_COMMON_CLK_MT8183_VDECSYS) += clk-mt8183-vdec.o obj-$(CONFIG_COMMON_CLK_MT8183_VENCSYS) += clk-mt8183-venc.o -obj-$(CONFIG_COMMON_CLK_MT8186) += clk-mt8186-mcu.o clk-mt8186-topckgen.o clk-mt8186-infra_ao.o \ - clk-mt8186-apmixedsys.o clk-mt8186-imp_iic_wrap.o \ - clk-mt8186-mfg.o clk-mt8186-mm.o clk-mt8186-wpe.o \ - clk-mt8186-img.o clk-mt8186-vdec.o clk-mt8186-venc.o \ - clk-mt8186-cam.o clk-mt8186-mdp.o clk-mt8186-ipe.o +obj-$(CONFIG_COMMON_CLK_MT8186) += clk-mt8186-apmixedsys.o clk-mt8186-topckgen.o \ + clk-mt8186-infra_ao.o +obj-$(CONFIG_COMMON_CLK_MT8186_CAMSYS) += clk-mt8186-cam.o +obj-$(CONFIG_COMMON_CLK_MT8186_IMGSYS) += clk-mt8186-img.o +obj-$(CONFIG_COMMON_CLK_MT8186_IMP_IIC_WRAP) += clk-mt8186-imp_iic_wrap.o +obj-$(CONFIG_COMMON_CLK_MT8186_IPESYS) += clk-mt8186-ipe.o +obj-$(CONFIG_COMMON_CLK_MT8186_MCUSYS) += clk-mt8186-mcu.o +obj-$(CONFIG_COMMON_CLK_MT8186_MDPSYS) += clk-mt8186-mdp.o +obj-$(CONFIG_COMMON_CLK_MT8186_MFGCFG) += clk-mt8186-mfg.o +obj-$(CONFIG_COMMON_CLK_MT8186_MMSYS) += clk-mt8186-mm.o +obj-$(CONFIG_COMMON_CLK_MT8186_VDECSYS) += clk-mt8186-vdec.o +obj-$(CONFIG_COMMON_CLK_MT8186_VENCSYS) += clk-mt8186-venc.o +obj-$(CONFIG_COMMON_CLK_MT8186_WPESYS) += clk-mt8186-wpe.o obj-$(CONFIG_COMMON_CLK_MT8192) += clk-mt8192.o obj-$(CONFIG_COMMON_CLK_MT8192_AUDSYS) += clk-mt8192-aud.o obj-$(CONFIG_COMMON_CLK_MT8192_CAMSYS) += clk-mt8192-cam.o From 8943be7d1bf3a057b624757161b48c8aa385b72a Mon Sep 17 00:00:00 2001 From: Yongqiang Niu Date: Fri, 18 Nov 2022 14:30:18 +0800 Subject: [PATCH 07/86] BACKPORT: mtk-mmsys: Change mtk-mmsys & mtk-mutex to modules Change mtk-mmsys & mtk-mutex to modules for gki Bug: 335112842 (cherry picked from commit a7596e62dac7318456c1aa9af5bfccf0f8e6ad7e https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/ master) Signed-off-by: Yongqiang Niu Reviewed-by: AngeloGioacchino Del Regno Link: https://lore.kernel.org/r/20221118063018.13520-1-yongqiang.niu@mediatek.com Signed-off-by: Matthias Brugger Change-Id: I5a0789d26b188ca576a92df34af5940f539f1ac3 --- drivers/soc/mediatek/Kconfig | 2 +- drivers/soc/mediatek/mtk-mmsys.c | 18 +++++++++++++++++- drivers/soc/mediatek/mtk-mutex.c | 17 ++++++++++++++++- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/drivers/soc/mediatek/Kconfig b/drivers/soc/mediatek/Kconfig index 40d0cc600cae..20c77152c282 100644 --- a/drivers/soc/mediatek/Kconfig +++ b/drivers/soc/mediatek/Kconfig @@ -68,7 +68,7 @@ config MTK_SCPSYS_PM_DOMAINS tasks in the system. config MTK_MMSYS - bool "MediaTek MMSYS Support" + tristate "MediaTek MMSYS Support" default ARCH_MEDIATEK depends on HAS_IOMEM help diff --git a/drivers/soc/mediatek/mtk-mmsys.c b/drivers/soc/mediatek/mtk-mmsys.c index d2c7a87aab87..d517c84605b8 100644 --- a/drivers/soc/mediatek/mtk-mmsys.c +++ b/drivers/soc/mediatek/mtk-mmsys.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -426,4 +427,19 @@ static struct platform_driver mtk_mmsys_drv = { .probe = mtk_mmsys_probe, }; -builtin_platform_driver(mtk_mmsys_drv); +static int __init mtk_mmsys_init(void) +{ + return platform_driver_register(&mtk_mmsys_drv); +} + +static void __exit mtk_mmsys_exit(void) +{ + platform_driver_unregister(&mtk_mmsys_drv); +} + +module_init(mtk_mmsys_init); +module_exit(mtk_mmsys_exit); + +MODULE_AUTHOR("Yongqiang Niu "); +MODULE_DESCRIPTION("MediaTek SoC MMSYS driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/soc/mediatek/mtk-mutex.c b/drivers/soc/mediatek/mtk-mutex.c index c1a33d52038e..84cd1a858f06 100644 --- a/drivers/soc/mediatek/mtk-mutex.c +++ b/drivers/soc/mediatek/mtk-mutex.c @@ -874,4 +874,19 @@ static struct platform_driver mtk_mutex_driver = { }, }; -builtin_platform_driver(mtk_mutex_driver); +static int __init mtk_mutex_init(void) +{ + return platform_driver_register(&mtk_mutex_driver); +} + +static void __exit mtk_mutex_exit(void) +{ + platform_driver_unregister(&mtk_mutex_driver); +} + +module_init(mtk_mutex_init); +module_exit(mtk_mutex_exit); + +MODULE_AUTHOR("Yongqiang Niu "); +MODULE_DESCRIPTION("MediaTek SoC MUTEX driver"); +MODULE_LICENSE("GPL"); From 692e3553d2e90f8605312e75deecb939bb1659ab Mon Sep 17 00:00:00 2001 From: Andre Ding Date: Wed, 17 Apr 2024 20:23:20 +0530 Subject: [PATCH 08/86] ANDROID: abi_gki_aarch64_qcom: Update symbol list Symbols updated to QCOM abi symbol list for Marvell Phy and Mdio Bus Mux: ethnl_cable_test_amplitude ethnl_cable_test_pulse ethnl_cable_test_step genphy_check_and_restart_aneg genphy_read_status_fixed of_mdio_find_bus phy_config_aneg phy_gbit_fibre_features Bug: 335414016 Change-Id: I1fa732935cb1a33bcde782fdfe38f5b3fcfae4cb Signed-off-by: Andre Ding --- android/abi_gki_aarch64.stg | 106 +++++++++++++++++++++++++++++++++++ android/abi_gki_aarch64_qcom | 8 +++ 2 files changed, 114 insertions(+) diff --git a/android/abi_gki_aarch64.stg b/android/abi_gki_aarch64.stg index c42f8772d3c7..26ceb203bdab 100644 --- a/android/abi_gki_aarch64.stg +++ b/android/abi_gki_aarch64.stg @@ -310597,6 +310597,13 @@ function { parameter_id: 0x6720d32f parameter_id: 0x6720d32f } +function { + id: 0x91725507 + return_type_id: 0x6720d32f + parameter_id: 0x3176a085 + parameter_id: 0x295c7202 + parameter_id: 0x29bf06ba +} function { id: 0x91726dda return_type_id: 0x6720d32f @@ -312896,6 +312903,14 @@ function { parameter_id: 0x3c07fb40 parameter_id: 0x11cfee5a } +function { + id: 0x92c80d88 + return_type_id: 0x6720d32f + parameter_id: 0x3176a085 + parameter_id: 0xc9082b19 + parameter_id: 0xc9082b19 + parameter_id: 0xc9082b19 +} function { id: 0x92cb9949 return_type_id: 0xf435685e @@ -313923,6 +313938,12 @@ function { return_type_id: 0x6720d32f parameter_id: 0x3863a91c } +function { + id: 0x93987df1 + return_type_id: 0x6720d32f + parameter_id: 0x3176a085 + parameter_id: 0x914dbfdc +} function { id: 0x9398dfdf return_type_id: 0x6720d32f @@ -331726,6 +331747,11 @@ function { parameter_id: 0xac0d3a85 parameter_id: 0x92233392 } +function { + id: 0xa395744b + return_type_id: 0x35d510c3 + parameter_id: 0x347303b4 +} function { id: 0xa3adbe6c return_type_id: 0x25b73daa @@ -365857,6 +365883,15 @@ elf_symbol { type_id: 0x1c31d966 full_name: "ether_setup" } +elf_symbol { + id: 0xc2ec4486 + name: "ethnl_cable_test_amplitude" + is_defined: true + symbol_type: FUNCTION + crc: 0xcaa7ffcf + type_id: 0x91725507 + full_name: "ethnl_cable_test_amplitude" +} elf_symbol { id: 0x592619b0 name: "ethnl_cable_test_fault_length" @@ -365866,6 +365901,15 @@ elf_symbol { type_id: 0x914a78cc full_name: "ethnl_cable_test_fault_length" } +elf_symbol { + id: 0xd3d73b52 + name: "ethnl_cable_test_pulse" + is_defined: true + symbol_type: FUNCTION + crc: 0xeed4092e + type_id: 0x93987df1 + full_name: "ethnl_cable_test_pulse" +} elf_symbol { id: 0xc00e9f7c name: "ethnl_cable_test_result" @@ -365875,6 +365919,15 @@ elf_symbol { type_id: 0x91726dda full_name: "ethnl_cable_test_result" } +elf_symbol { + id: 0x4d66a0fd + name: "ethnl_cable_test_step" + is_defined: true + symbol_type: FUNCTION + crc: 0xc62cb5bc + type_id: 0x92c80d88 + full_name: "ethnl_cable_test_step" +} elf_symbol { id: 0xfe219f57 name: "ethtool_convert_legacy_u32_to_link_mode" @@ -367873,6 +367926,15 @@ elf_symbol { type_id: 0x91dd4b0e full_name: "genphy_c45_read_status" } +elf_symbol { + id: 0x07b1add0 + name: "genphy_check_and_restart_aneg" + is_defined: true + symbol_type: FUNCTION + crc: 0x23d24b71 + type_id: 0x9068b671 + full_name: "genphy_check_and_restart_aneg" +} elf_symbol { id: 0x618539bc name: "genphy_handle_interrupt_no_ack" @@ -367918,6 +367980,15 @@ elf_symbol { type_id: 0x91dd4b0e full_name: "genphy_read_status" } +elf_symbol { + id: 0x97ef6102 + name: "genphy_read_status_fixed" + is_defined: true + symbol_type: FUNCTION + crc: 0x6761e7a9 + type_id: 0x91dd4b0e + full_name: "genphy_read_status_fixed" +} elf_symbol { id: 0xa94a7a92 name: "genphy_restart_aneg" @@ -380617,6 +380688,15 @@ elf_symbol { type_id: 0xd0b47295 full_name: "of_match_node" } +elf_symbol { + id: 0x62a72fbe + name: "of_mdio_find_bus" + is_defined: true + symbol_type: FUNCTION + crc: 0x5ab1b075 + type_id: 0xa395744b + full_name: "of_mdio_find_bus" +} elf_symbol { id: 0x23d1db24 name: "of_modalias_node" @@ -383005,6 +383085,15 @@ elf_symbol { type_id: 0x9f7fc723 full_name: "phy_calibrate" } +elf_symbol { + id: 0x4219bd3e + name: "phy_config_aneg" + is_defined: true + symbol_type: FUNCTION + crc: 0xd4984810 + type_id: 0x91dd4b0e + full_name: "phy_config_aneg" +} elf_symbol { id: 0xc278f4fa name: "phy_configure" @@ -383194,6 +383283,15 @@ elf_symbol { type_id: 0xbeab1c63 full_name: "phy_find_first" } +elf_symbol { + id: 0xd23d219e + name: "phy_gbit_fibre_features" + is_defined: true + symbol_type: OBJECT + crc: 0x9de31b2a + type_id: 0x607419c2 + full_name: "phy_gbit_fibre_features" +} elf_symbol { id: 0x0dfe21ad name: "phy_get" @@ -409537,8 +409635,11 @@ interface { symbol_id: 0x9c46e3cf symbol_id: 0xd4d3c19b symbol_id: 0x3276ffad + symbol_id: 0xc2ec4486 symbol_id: 0x592619b0 + symbol_id: 0xd3d73b52 symbol_id: 0xc00e9f7c + symbol_id: 0x4d66a0fd symbol_id: 0xfe219f57 symbol_id: 0x6e69a2b9 symbol_id: 0xd3082d07 @@ -409761,11 +409862,13 @@ interface { symbol_id: 0x607c937c symbol_id: 0xd99708c7 symbol_id: 0xa3a0e5a9 + symbol_id: 0x07b1add0 symbol_id: 0x618539bc symbol_id: 0xe00705b3 symbol_id: 0xcaf802e6 symbol_id: 0xbaba47ed symbol_id: 0x67f44b83 + symbol_id: 0x97ef6102 symbol_id: 0xa94a7a92 symbol_id: 0x855de755 symbol_id: 0xb3b7ca78 @@ -411177,6 +411280,7 @@ interface { symbol_id: 0x318c8fe5 symbol_id: 0x16c4cdb3 symbol_id: 0x3965b143 + symbol_id: 0x62a72fbe symbol_id: 0x23d1db24 symbol_id: 0x8ac60fa1 symbol_id: 0xc9f43d6f @@ -411442,6 +411546,7 @@ interface { symbol_id: 0x5fa10488 symbol_id: 0x9db95d0c symbol_id: 0x81e2991f + symbol_id: 0x4219bd3e symbol_id: 0xc278f4fa symbol_id: 0x9c15bd7a symbol_id: 0x866cf90c @@ -411463,6 +411568,7 @@ interface { symbol_id: 0xcd9e585e symbol_id: 0x69eeb51b symbol_id: 0x6cf83bd0 + symbol_id: 0xd23d219e symbol_id: 0x0dfe21ad symbol_id: 0x7de90fab symbol_id: 0x30e87de6 diff --git a/android/abi_gki_aarch64_qcom b/android/abi_gki_aarch64_qcom index da4958245f91..0fafde6e16e1 100644 --- a/android/abi_gki_aarch64_qcom +++ b/android/abi_gki_aarch64_qcom @@ -1094,8 +1094,11 @@ ether_setup eth_header_parse eth_mac_addr + ethnl_cable_test_amplitude ethnl_cable_test_fault_length + ethnl_cable_test_pulse ethnl_cable_test_result + ethnl_cable_test_step eth_platform_get_mac_address eth_prepare_mac_addr_change ethtool_convert_legacy_u32_to_link_mode @@ -1212,10 +1215,12 @@ genphy_c45_check_and_restart_aneg genphy_c45_pma_setup_forced genphy_c45_read_status + genphy_check_and_restart_aneg __genphy_config_aneg genphy_read_abilities genphy_read_mmd_unsupported genphy_read_status + genphy_read_status_fixed genphy_restart_aneg genphy_resume genphy_soft_reset @@ -2197,6 +2202,7 @@ of_match_device of_match_node __of_mdiobus_register + of_mdio_find_bus of_modalias_node of_n_addr_cells of_node_name_eq @@ -2337,6 +2343,7 @@ perf_trace_run_bpf_submit phy_attached_info phy_calibrate + phy_config_aneg phy_device_free phy_device_register phy_do_ioctl_running @@ -2346,6 +2353,7 @@ phy_ethtool_get_wol phy_ethtool_set_wol phy_exit + phy_gbit_fibre_features phy_init phy_init_eee phy_init_hw From dcb09569bbff14d203a6b2ffa40da7b724665e31 Mon Sep 17 00:00:00 2001 From: Youngmin Nam Date: Thu, 18 Apr 2024 14:01:15 +0900 Subject: [PATCH 09/86] ANDROID: ABI: Update symbol list for Exynos SoC There are no new symbols to be added to GKI symbol list. We simply update our symbol list. Bug: 335537438 Change-Id: Iae0594ff776853df2b38eb3215a5378a03995c40 Signed-off-by: Youngmin Nam --- android/abi_gki_aarch64_exynos | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/android/abi_gki_aarch64_exynos b/android/abi_gki_aarch64_exynos index 9a3f3ef4681c..de25e02ad009 100644 --- a/android/abi_gki_aarch64_exynos +++ b/android/abi_gki_aarch64_exynos @@ -562,7 +562,6 @@ netif_receive_skb netif_rx netif_tx_wake_queue - netlink_broadcast nla_memcpy nla_put nla_put_64bit @@ -1070,6 +1069,7 @@ kfree_sensitive ktime_get_coarse_with_offset netdev_err + netlink_broadcast netlink_register_notifier netlink_unicast netlink_unregister_notifier @@ -1193,6 +1193,7 @@ platform_device_del platform_device_put pm_runtime_allow + usb_ep_set_halt usb_gadget_set_state usb_otg_state_string usb_role_switch_set_role @@ -2040,6 +2041,7 @@ irq_get_irqchip_state of_property_read_variable_u16_array pci_enable_wake + pci_load_and_free_saved_state pcim_enable_device pcim_iomap_regions pcim_iomap_table @@ -2057,6 +2059,7 @@ __cpuhp_state_remove_instance dev_addr_mod dev_alloc_name + dev_change_flags __dev_queue_xmit dev_set_threaded down_trylock From d224d17a1464d57fd5c2bacb15312139b3df6fa2 Mon Sep 17 00:00:00 2001 From: Nikhil V Date: Mon, 22 Jan 2024 18:45:25 +0530 Subject: [PATCH 10/86] BACKPORT: PM: hibernate: Rename lzo* to make it generic Renaming lzo* to generic names, except for lzo_xxx() APIs. This is used in the next patch where we move to crypto based APIs for compression. There are no functional changes introduced by this approach. Bug: 335581841 Signed-off-by: Nikhil V Signed-off-by: Rafael J. Wysocki (cherry picked from commit 89a807625f9701154167bf6bf136adfa1be4d849) Change-Id: I8e2032658132965bed4c7b24b7409ae7a1bfa6cd [quic_nprakash: Resolved minor conflicts in kernel/power/swap.c] Signed-off-by: Nikhil V --- kernel/power/swap.c | 120 ++++++++++++++++++++++---------------------- 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/kernel/power/swap.c b/kernel/power/swap.c index 678727c4dead..c0e9587078b6 100644 --- a/kernel/power/swap.c +++ b/kernel/power/swap.c @@ -520,23 +520,23 @@ static int swap_writer_finish(struct swap_map_handle *handle, } /* We need to remember how much compressed data we need to read. */ -#define LZO_HEADER sizeof(size_t) +#define CMP_HEADER sizeof(size_t) /* Number of pages/bytes we'll compress at one time. */ -#define LZO_UNC_PAGES 32 -#define LZO_UNC_SIZE (LZO_UNC_PAGES * PAGE_SIZE) +#define UNC_PAGES 32 +#define UNC_SIZE (UNC_PAGES * PAGE_SIZE) -/* Number of pages/bytes we need for compressed data (worst case). */ -#define LZO_CMP_PAGES DIV_ROUND_UP(lzo1x_worst_compress(LZO_UNC_SIZE) + \ - LZO_HEADER, PAGE_SIZE) -#define LZO_CMP_SIZE (LZO_CMP_PAGES * PAGE_SIZE) +/* Number of pages we need for compressed data (worst case). */ +#define CMP_PAGES DIV_ROUND_UP(lzo1x_worst_compress(UNC_SIZE) + \ + CMP_HEADER, PAGE_SIZE) +#define CMP_SIZE (CMP_PAGES * PAGE_SIZE) /* Maximum number of threads for compression/decompression. */ -#define LZO_THREADS 3 +#define CMP_THREADS 3 /* Minimum/maximum number of pages for read buffering. */ -#define LZO_MIN_RD_PAGES 1024 -#define LZO_MAX_RD_PAGES 8192 +#define CMP_MIN_RD_PAGES 1024 +#define CMP_MAX_RD_PAGES 8192 /** @@ -600,8 +600,8 @@ struct crc_data { wait_queue_head_t go; /* start crc update */ wait_queue_head_t done; /* crc update done */ u32 *crc32; /* points to handle's crc32 */ - size_t *unc_len[LZO_THREADS]; /* uncompressed lengths */ - unsigned char *unc[LZO_THREADS]; /* uncompressed data */ + size_t *unc_len[CMP_THREADS]; /* uncompressed lengths */ + unsigned char *unc[CMP_THREADS]; /* uncompressed data */ }; /** @@ -632,7 +632,7 @@ static int crc32_threadfn(void *data) return 0; } /** - * Structure used for LZO data compression. + * Structure used for data compression. */ struct cmp_data { struct task_struct *thr; /* thread */ @@ -643,15 +643,15 @@ struct cmp_data { wait_queue_head_t done; /* compression done */ size_t unc_len; /* uncompressed length */ size_t cmp_len; /* compressed length */ - unsigned char unc[LZO_UNC_SIZE]; /* uncompressed buffer */ - unsigned char cmp[LZO_CMP_SIZE]; /* compressed buffer */ + unsigned char unc[UNC_SIZE]; /* uncompressed buffer */ + unsigned char cmp[CMP_SIZE]; /* compressed buffer */ unsigned char wrk[LZO1X_1_MEM_COMPRESS]; /* compression workspace */ }; /** * Compression function that runs in its own thread. */ -static int lzo_compress_threadfn(void *data) +static int compress_threadfn(void *data) { struct cmp_data *d = data; @@ -668,7 +668,7 @@ static int lzo_compress_threadfn(void *data) atomic_set(&d->ready, 0); d->ret = lzo1x_1_compress(d->unc, d->unc_len, - d->cmp + LZO_HEADER, &d->cmp_len, + d->cmp + CMP_HEADER, &d->cmp_len, d->wrk); atomic_set(&d->stop, 1); wake_up(&d->done); @@ -677,14 +677,14 @@ static int lzo_compress_threadfn(void *data) } /** - * save_image_lzo - Save the suspend image data compressed with LZO. + * save_compressed_image - Save the suspend image data after compression. * @handle: Swap map handle to use for saving the image. * @snapshot: Image to read data from. * @nr_to_write: Number of pages to save. */ -static int save_image_lzo(struct swap_map_handle *handle, - struct snapshot_handle *snapshot, - unsigned int nr_to_write) +static int save_compressed_image(struct swap_map_handle *handle, + struct snapshot_handle *snapshot, + unsigned int nr_to_write) { unsigned int m; int ret = 0; @@ -706,18 +706,18 @@ static int save_image_lzo(struct swap_map_handle *handle, * footprint. */ nr_threads = num_online_cpus() - 1; - nr_threads = clamp_val(nr_threads, 1, LZO_THREADS); + nr_threads = clamp_val(nr_threads, 1, CMP_THREADS); page = (void *)__get_free_page(GFP_NOIO | __GFP_HIGH); if (!page) { - pr_err("Failed to allocate LZO page\n"); + pr_err("Failed to allocate compression page\n"); ret = -ENOMEM; goto out_clean; } data = vzalloc(array_size(nr_threads, sizeof(*data))); if (!data) { - pr_err("Failed to allocate LZO data\n"); + pr_err("Failed to allocate compression data\n"); ret = -ENOMEM; goto out_clean; } @@ -736,7 +736,7 @@ static int save_image_lzo(struct swap_map_handle *handle, init_waitqueue_head(&data[thr].go); init_waitqueue_head(&data[thr].done); - data[thr].thr = kthread_run(lzo_compress_threadfn, + data[thr].thr = kthread_run(compress_threadfn, &data[thr], "image_compress/%u", thr); if (IS_ERR(data[thr].thr)) { @@ -784,7 +784,7 @@ static int save_image_lzo(struct swap_map_handle *handle, start = ktime_get(); for (;;) { for (thr = 0; thr < nr_threads; thr++) { - for (off = 0; off < LZO_UNC_SIZE; off += PAGE_SIZE) { + for (off = 0; off < UNC_SIZE; off += PAGE_SIZE) { ret = snapshot_read_next(snapshot); if (ret < 0) goto out_finish; @@ -824,14 +824,14 @@ static int save_image_lzo(struct swap_map_handle *handle, ret = data[thr].ret; if (ret < 0) { - pr_err("LZO compression failed\n"); + pr_err("compression failed\n"); goto out_finish; } if (unlikely(!data[thr].cmp_len || data[thr].cmp_len > lzo1x_worst_compress(data[thr].unc_len))) { - pr_err("Invalid LZO compressed length\n"); + pr_err("Invalid compressed length\n"); ret = -1; goto out_finish; } @@ -847,7 +847,7 @@ static int save_image_lzo(struct swap_map_handle *handle, * read it. */ for (off = 0; - off < LZO_HEADER + data[thr].cmp_len; + off < CMP_HEADER + data[thr].cmp_len; off += PAGE_SIZE) { memcpy(page, data[thr].cmp + off, PAGE_SIZE); @@ -950,7 +950,7 @@ int swsusp_write(unsigned int flags) if (!error) { error = (flags & SF_NOCOMPRESS_MODE) ? save_image(&handle, &snapshot, pages - 1) : - save_image_lzo(&handle, &snapshot, pages - 1); + save_compressed_image(&handle, &snapshot, pages - 1); } out_finish: error = swap_writer_finish(&handle, flags, error); @@ -1117,7 +1117,7 @@ static int load_image(struct swap_map_handle *handle, } /** - * Structure used for LZO data decompression. + * Structure used for data decompression. */ struct dec_data { struct task_struct *thr; /* thread */ @@ -1128,14 +1128,14 @@ struct dec_data { wait_queue_head_t done; /* decompression done */ size_t unc_len; /* uncompressed length */ size_t cmp_len; /* compressed length */ - unsigned char unc[LZO_UNC_SIZE]; /* uncompressed buffer */ - unsigned char cmp[LZO_CMP_SIZE]; /* compressed buffer */ + unsigned char unc[UNC_SIZE]; /* uncompressed buffer */ + unsigned char cmp[CMP_SIZE]; /* compressed buffer */ }; /** * Decompression function that runs in its own thread. */ -static int lzo_decompress_threadfn(void *data) +static int decompress_threadfn(void *data) { struct dec_data *d = data; @@ -1151,9 +1151,9 @@ static int lzo_decompress_threadfn(void *data) } atomic_set(&d->ready, 0); - d->unc_len = LZO_UNC_SIZE; - d->ret = lzo1x_decompress_safe(d->cmp + LZO_HEADER, d->cmp_len, - d->unc, &d->unc_len); + d->unc_len = UNC_SIZE; + d->ret = lzo1x_decompress_safe(d->cmp + CMP_HEADER, d->cmp_len, + d->unc, &d->unc_len); if (clean_pages_on_decompress) flush_icache_range((unsigned long)d->unc, (unsigned long)d->unc + d->unc_len); @@ -1165,14 +1165,14 @@ static int lzo_decompress_threadfn(void *data) } /** - * load_image_lzo - Load compressed image data and decompress them with LZO. + * load_compressed_image - Load compressed image data and decompress it. * @handle: Swap map handle to use for loading data. * @snapshot: Image to copy uncompressed data into. * @nr_to_read: Number of pages to load. */ -static int load_image_lzo(struct swap_map_handle *handle, - struct snapshot_handle *snapshot, - unsigned int nr_to_read) +static int load_compressed_image(struct swap_map_handle *handle, + struct snapshot_handle *snapshot, + unsigned int nr_to_read) { unsigned int m; int ret = 0; @@ -1197,18 +1197,18 @@ static int load_image_lzo(struct swap_map_handle *handle, * footprint. */ nr_threads = num_online_cpus() - 1; - nr_threads = clamp_val(nr_threads, 1, LZO_THREADS); + nr_threads = clamp_val(nr_threads, 1, CMP_THREADS); - page = vmalloc(array_size(LZO_MAX_RD_PAGES, sizeof(*page))); + page = vmalloc(array_size(CMP_MAX_RD_PAGES, sizeof(*page))); if (!page) { - pr_err("Failed to allocate LZO page\n"); + pr_err("Failed to allocate compression page\n"); ret = -ENOMEM; goto out_clean; } data = vzalloc(array_size(nr_threads, sizeof(*data))); if (!data) { - pr_err("Failed to allocate LZO data\n"); + pr_err("Failed to allocate compression data\n"); ret = -ENOMEM; goto out_clean; } @@ -1229,7 +1229,7 @@ static int load_image_lzo(struct swap_map_handle *handle, init_waitqueue_head(&data[thr].go); init_waitqueue_head(&data[thr].done); - data[thr].thr = kthread_run(lzo_decompress_threadfn, + data[thr].thr = kthread_run(decompress_threadfn, &data[thr], "image_decompress/%u", thr); if (IS_ERR(data[thr].thr)) { @@ -1270,18 +1270,18 @@ static int load_image_lzo(struct swap_map_handle *handle, */ if (low_free_pages() > snapshot_get_image_size()) read_pages = (low_free_pages() - snapshot_get_image_size()) / 2; - read_pages = clamp_val(read_pages, LZO_MIN_RD_PAGES, LZO_MAX_RD_PAGES); + read_pages = clamp_val(read_pages, CMP_MIN_RD_PAGES, CMP_MAX_RD_PAGES); for (i = 0; i < read_pages; i++) { - page[i] = (void *)__get_free_page(i < LZO_CMP_PAGES ? + page[i] = (void *)__get_free_page(i < CMP_PAGES ? GFP_NOIO | __GFP_HIGH : GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY); if (!page[i]) { - if (i < LZO_CMP_PAGES) { + if (i < CMP_PAGES) { ring_size = i; - pr_err("Failed to allocate LZO pages\n"); + pr_err("Failed to allocate compression pages\n"); ret = -ENOMEM; goto out_clean; } else { @@ -1352,13 +1352,13 @@ static int load_image_lzo(struct swap_map_handle *handle, data[thr].cmp_len = *(size_t *)page[pg]; if (unlikely(!data[thr].cmp_len || data[thr].cmp_len > - lzo1x_worst_compress(LZO_UNC_SIZE))) { - pr_err("Invalid LZO compressed length\n"); + lzo1x_worst_compress(UNC_SIZE))) { + pr_err("Invalid compressed length\n"); ret = -1; goto out_finish; } - need = DIV_ROUND_UP(data[thr].cmp_len + LZO_HEADER, + need = DIV_ROUND_UP(data[thr].cmp_len + CMP_HEADER, PAGE_SIZE); if (need > have) { if (eof > 1) { @@ -1369,7 +1369,7 @@ static int load_image_lzo(struct swap_map_handle *handle, } for (off = 0; - off < LZO_HEADER + data[thr].cmp_len; + off < CMP_HEADER + data[thr].cmp_len; off += PAGE_SIZE) { memcpy(data[thr].cmp + off, page[pg], PAGE_SIZE); @@ -1386,7 +1386,7 @@ static int load_image_lzo(struct swap_map_handle *handle, /* * Wait for more data while we are decompressing. */ - if (have < LZO_CMP_PAGES && asked) { + if (have < CMP_PAGES && asked) { ret = hib_wait_io(&hb); if (ret) goto out_finish; @@ -1404,14 +1404,14 @@ static int load_image_lzo(struct swap_map_handle *handle, ret = data[thr].ret; if (ret < 0) { - pr_err("LZO decompression failed\n"); + pr_err("decompression failed\n"); goto out_finish; } if (unlikely(!data[thr].unc_len || - data[thr].unc_len > LZO_UNC_SIZE || - data[thr].unc_len & (PAGE_SIZE - 1))) { - pr_err("Invalid LZO uncompressed length\n"); + data[thr].unc_len > UNC_SIZE || + data[thr].unc_len & (PAGE_SIZE - 1))) { + pr_err("Invalid uncompressed length\n"); ret = -1; goto out_finish; } @@ -1508,7 +1508,7 @@ int swsusp_read(unsigned int *flags_p) if (!error) { error = (*flags_p & SF_NOCOMPRESS_MODE) ? load_image(&handle, &snapshot, header->pages - 1) : - load_image_lzo(&handle, &snapshot, header->pages - 1); + load_compressed_image(&handle, &snapshot, header->pages - 1); } swap_reader_finish(&handle); end: From 990d3701d0aa4109b7f1d9ad34066a0c0116edda Mon Sep 17 00:00:00 2001 From: Nikhil V Date: Mon, 22 Jan 2024 18:45:26 +0530 Subject: [PATCH 11/86] BACKPORT: PM: hibernate: Move to crypto APIs for LZO compression Currently for hibernation, LZO is the only compression algorithm available and uses the existing LZO library calls. However, there is no flexibility to switch to other algorithms which provides better results. The main idea is that different compression algorithms have different characteristics and hibernation may benefit when it uses alternate algorithms. By moving to crypto based APIs, it lays a foundation to use other compression algorithms for hibernation. There are no functional changes introduced by this approach. Bug: 335581841 Signed-off-by: Nikhil V Signed-off-by: Rafael J. Wysocki (cherry picked from commit a06c6f5d3cc90b3b070d7b99979d57238db77a86) Change-Id: I8d15262f9823219d291b84eab28b2ec44474dad4 [quic_nprakash: Resolved minor conflicts in kernel/power/(power.h,swap.c)] Signed-off-by: Nikhil V --- kernel/power/Kconfig | 21 +++++++- kernel/power/hibernate.c | 33 +++++++++++++ kernel/power/power.h | 5 ++ kernel/power/swap.c | 101 +++++++++++++++++++++++++++++---------- 4 files changed, 132 insertions(+), 28 deletions(-) diff --git a/kernel/power/Kconfig b/kernel/power/Kconfig index 60a1d3051cc7..1a202bb1de78 100644 --- a/kernel/power/Kconfig +++ b/kernel/power/Kconfig @@ -39,9 +39,9 @@ config HIBERNATION bool "Hibernation (aka 'suspend to disk')" depends on SWAP && ARCH_HIBERNATION_POSSIBLE select HIBERNATE_CALLBACKS - select LZO_COMPRESS - select LZO_DECOMPRESS select CRC32 + select CRYPTO + select CRYPTO_LZO help Enable the suspend to disk (STD) functionality, which is usually called "hibernation" in user interfaces. STD checkpoints the @@ -92,6 +92,23 @@ config HIBERNATION_SNAPSHOT_DEV If in doubt, say Y. +choice + prompt "Default compressor" + default HIBERNATION_COMP_LZO + depends on HIBERNATION + +config HIBERNATION_COMP_LZO + bool "lzo" + depends on CRYPTO_LZO + +endchoice + +config HIBERNATION_DEF_COMP + string + default "lzo" if HIBERNATION_COMP_LZO + help + Default compressor to be used for hibernation. + config PM_STD_PARTITION string "Default resume partition" depends on HIBERNATION diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c index 30d1274f03f6..5ba0fc139ac3 100644 --- a/kernel/power/hibernate.c +++ b/kernel/power/hibernate.c @@ -46,6 +46,15 @@ dev_t swsusp_resume_device; sector_t swsusp_resume_block; __visible int in_suspend __nosavedata; +static const char *default_compressor = CONFIG_HIBERNATION_DEF_COMP; + +/* + * Compression/decompression algorithm to be used while saving/loading + * image to/from disk. This would later be used in 'kernel/power/swap.c' + * to allocate comp streams. + */ +char hib_comp_algo[CRYPTO_MAX_ALG_NAME]; + enum { HIBERNATION_INVALID, HIBERNATION_PLATFORM, @@ -729,6 +738,17 @@ int hibernate(void) return -EPERM; } + /* + * Query for the compression algorithm support if compression is enabled. + */ + if (!nocompress) { + strscpy(hib_comp_algo, default_compressor, sizeof(hib_comp_algo)); + if (crypto_has_comp(hib_comp_algo, 0, 0) != 1) { + pr_err("%s compression is not available\n", hib_comp_algo); + return -EOPNOTSUPP; + } + } + sleep_flags = lock_system_sleep(); /* The snapshot device should not be opened while we're running */ if (!hibernate_acquire()) { @@ -996,6 +1016,19 @@ static int software_resume(void) if (error) goto Unlock; + /* + * Check if the hibernation image is compressed. If so, query for + * the algorithm support. + */ + if (!(swsusp_header_flags & SF_NOCOMPRESS_MODE)) { + strscpy(hib_comp_algo, default_compressor, sizeof(hib_comp_algo)); + if (crypto_has_comp(hib_comp_algo, 0, 0) != 1) { + pr_err("%s compression is not available\n", hib_comp_algo); + error = -EOPNOTSUPP; + goto Unlock; + } + } + /* The snapshot device should not be opened while we're running */ if (!hibernate_acquire()) { error = -EBUSY; diff --git a/kernel/power/power.h b/kernel/power/power.h index b83c8d5e188d..bae801d4bb9b 100644 --- a/kernel/power/power.h +++ b/kernel/power/power.h @@ -6,6 +6,7 @@ #include #include #include +#include struct swsusp_info { struct new_utsname uts; @@ -60,6 +61,10 @@ asmlinkage int swsusp_save(void); /* kernel/power/hibernate.c */ extern bool freezer_test_done; extern bool snapshot_test; +extern char hib_comp_algo[CRYPTO_MAX_ALG_NAME]; + +/* kernel/power/swap.c */ +extern unsigned int swsusp_header_flags; extern int hibernation_snapshot(int platform_mode); extern int hibernation_restore(int platform_mode); diff --git a/kernel/power/swap.c b/kernel/power/swap.c index c0e9587078b6..017a55966411 100644 --- a/kernel/power/swap.c +++ b/kernel/power/swap.c @@ -23,7 +23,6 @@ #include #include #include -#include #include #include #include @@ -340,6 +339,13 @@ static int mark_swapfiles(struct swap_map_handle *handle, unsigned int flags) return error; } +/* + * Hold the swsusp_header flag. This is used in software_resume() in + * 'kernel/power/hibernate' to check if the image is compressed and query + * for the compression algorithm support(if so). + */ +unsigned int swsusp_header_flags; + /** * swsusp_swap_check - check if the resume device is a swap device * and get its index (if so) @@ -519,6 +525,12 @@ static int swap_writer_finish(struct swap_map_handle *handle, return error; } +/* + * Bytes we need for compressed data in worst case. We assume(limitation) + * this is the worst of all the compression algorithms. + */ +#define bytes_worst_compress(x) ((x) + ((x) / 16) + 64 + 3 + 2) + /* We need to remember how much compressed data we need to read. */ #define CMP_HEADER sizeof(size_t) @@ -527,7 +539,7 @@ static int swap_writer_finish(struct swap_map_handle *handle, #define UNC_SIZE (UNC_PAGES * PAGE_SIZE) /* Number of pages we need for compressed data (worst case). */ -#define CMP_PAGES DIV_ROUND_UP(lzo1x_worst_compress(UNC_SIZE) + \ +#define CMP_PAGES DIV_ROUND_UP(bytes_worst_compress(UNC_SIZE) + \ CMP_HEADER, PAGE_SIZE) #define CMP_SIZE (CMP_PAGES * PAGE_SIZE) @@ -538,7 +550,6 @@ static int swap_writer_finish(struct swap_map_handle *handle, #define CMP_MIN_RD_PAGES 1024 #define CMP_MAX_RD_PAGES 8192 - /** * save_image - save the suspend image data */ @@ -636,6 +647,7 @@ static int crc32_threadfn(void *data) */ struct cmp_data { struct task_struct *thr; /* thread */ + struct crypto_comp *cc; /* crypto compressor stream */ atomic_t ready; /* ready to start flag */ atomic_t stop; /* ready to stop flag */ int ret; /* return code */ @@ -645,15 +657,18 @@ struct cmp_data { size_t cmp_len; /* compressed length */ unsigned char unc[UNC_SIZE]; /* uncompressed buffer */ unsigned char cmp[CMP_SIZE]; /* compressed buffer */ - unsigned char wrk[LZO1X_1_MEM_COMPRESS]; /* compression workspace */ }; +/* Indicates the image size after compression */ +static atomic_t compressed_size = ATOMIC_INIT(0); + /** * Compression function that runs in its own thread. */ static int compress_threadfn(void *data) { struct cmp_data *d = data; + unsigned int cmp_len = 0; while (1) { wait_event(d->go, atomic_read(&d->ready) || @@ -667,9 +682,13 @@ static int compress_threadfn(void *data) } atomic_set(&d->ready, 0); - d->ret = lzo1x_1_compress(d->unc, d->unc_len, - d->cmp + CMP_HEADER, &d->cmp_len, - d->wrk); + cmp_len = CMP_SIZE - CMP_HEADER; + d->ret = crypto_comp_compress(d->cc, d->unc, d->unc_len, + d->cmp + CMP_HEADER, + &cmp_len); + d->cmp_len = cmp_len; + + atomic_set(&compressed_size, atomic_read(&compressed_size) + d->cmp_len); atomic_set(&d->stop, 1); wake_up(&d->done); } @@ -701,6 +720,8 @@ static int save_compressed_image(struct swap_map_handle *handle, hib_init_batch(&hb); + atomic_set(&compressed_size, 0); + /* * We'll limit the number of threads for compression to limit memory * footprint. @@ -710,14 +731,14 @@ static int save_compressed_image(struct swap_map_handle *handle, page = (void *)__get_free_page(GFP_NOIO | __GFP_HIGH); if (!page) { - pr_err("Failed to allocate compression page\n"); + pr_err("Failed to allocate %s page\n", hib_comp_algo); ret = -ENOMEM; goto out_clean; } data = vzalloc(array_size(nr_threads, sizeof(*data))); if (!data) { - pr_err("Failed to allocate compression data\n"); + pr_err("Failed to allocate %s data\n", hib_comp_algo); ret = -ENOMEM; goto out_clean; } @@ -736,6 +757,13 @@ static int save_compressed_image(struct swap_map_handle *handle, init_waitqueue_head(&data[thr].go); init_waitqueue_head(&data[thr].done); + data[thr].cc = crypto_alloc_comp(hib_comp_algo, 0, 0); + if (IS_ERR_OR_NULL(data[thr].cc)) { + pr_err("Could not allocate comp stream %ld\n", PTR_ERR(data[thr].cc)); + ret = -EFAULT; + goto out_clean; + } + data[thr].thr = kthread_run(compress_threadfn, &data[thr], "image_compress/%u", thr); @@ -774,7 +802,7 @@ static int save_compressed_image(struct swap_map_handle *handle, */ handle->reqd_free_pages = reqd_free_pages(); - pr_info("Using %u thread(s) for compression\n", nr_threads); + pr_info("Using %u thread(s) for %s compression\n", nr_threads, hib_comp_algo); pr_info("Compressing and saving image data (%u pages)...\n", nr_to_write); m = nr_to_write / 10; @@ -824,14 +852,14 @@ static int save_compressed_image(struct swap_map_handle *handle, ret = data[thr].ret; if (ret < 0) { - pr_err("compression failed\n"); + pr_err("%s compression failed\n", hib_comp_algo); goto out_finish; } if (unlikely(!data[thr].cmp_len || data[thr].cmp_len > - lzo1x_worst_compress(data[thr].unc_len))) { - pr_err("Invalid compressed length\n"); + bytes_worst_compress(data[thr].unc_len))) { + pr_err("Invalid %s compressed length\n", hib_comp_algo); ret = -1; goto out_finish; } @@ -869,6 +897,9 @@ out_finish: if (!ret) pr_info("Image saving done\n"); swsusp_show_speed(start, stop, nr_to_write, "Wrote"); + pr_info("Image size after compression: %d kbytes\n", + (atomic_read(&compressed_size) / 1024)); + out_clean: hib_finish_batch(&hb); if (crc) { @@ -877,9 +908,12 @@ out_clean: kfree(crc); } if (data) { - for (thr = 0; thr < nr_threads; thr++) + for (thr = 0; thr < nr_threads; thr++) { if (data[thr].thr) kthread_stop(data[thr].thr); + if (data[thr].cc) + crypto_free_comp(data[thr].cc); + } vfree(data); } if (page) free_page((unsigned long)page); @@ -1121,6 +1155,7 @@ static int load_image(struct swap_map_handle *handle, */ struct dec_data { struct task_struct *thr; /* thread */ + struct crypto_comp *cc; /* crypto compressor stream */ atomic_t ready; /* ready to start flag */ atomic_t stop; /* ready to stop flag */ int ret; /* return code */ @@ -1138,6 +1173,7 @@ struct dec_data { static int decompress_threadfn(void *data) { struct dec_data *d = data; + unsigned int unc_len = 0; while (1) { wait_event(d->go, atomic_read(&d->ready) || @@ -1151,9 +1187,11 @@ static int decompress_threadfn(void *data) } atomic_set(&d->ready, 0); - d->unc_len = UNC_SIZE; - d->ret = lzo1x_decompress_safe(d->cmp + CMP_HEADER, d->cmp_len, - d->unc, &d->unc_len); + unc_len = UNC_SIZE; + d->ret = crypto_comp_decompress(d->cc, d->cmp + CMP_HEADER, d->cmp_len, + d->unc, &unc_len); + d->unc_len = unc_len; + if (clean_pages_on_decompress) flush_icache_range((unsigned long)d->unc, (unsigned long)d->unc + d->unc_len); @@ -1201,14 +1239,14 @@ static int load_compressed_image(struct swap_map_handle *handle, page = vmalloc(array_size(CMP_MAX_RD_PAGES, sizeof(*page))); if (!page) { - pr_err("Failed to allocate compression page\n"); + pr_err("Failed to allocate %s page\n", hib_comp_algo); ret = -ENOMEM; goto out_clean; } data = vzalloc(array_size(nr_threads, sizeof(*data))); if (!data) { - pr_err("Failed to allocate compression data\n"); + pr_err("Failed to allocate %s data\n", hib_comp_algo); ret = -ENOMEM; goto out_clean; } @@ -1229,6 +1267,13 @@ static int load_compressed_image(struct swap_map_handle *handle, init_waitqueue_head(&data[thr].go); init_waitqueue_head(&data[thr].done); + data[thr].cc = crypto_alloc_comp(hib_comp_algo, 0, 0); + if (IS_ERR_OR_NULL(data[thr].cc)) { + pr_err("Could not allocate comp stream %ld\n", PTR_ERR(data[thr].cc)); + ret = -EFAULT; + goto out_clean; + } + data[thr].thr = kthread_run(decompress_threadfn, &data[thr], "image_decompress/%u", thr); @@ -1281,7 +1326,7 @@ static int load_compressed_image(struct swap_map_handle *handle, if (!page[i]) { if (i < CMP_PAGES) { ring_size = i; - pr_err("Failed to allocate compression pages\n"); + pr_err("Failed to allocate %s pages\n", hib_comp_algo); ret = -ENOMEM; goto out_clean; } else { @@ -1291,7 +1336,7 @@ static int load_compressed_image(struct swap_map_handle *handle, } want = ring_size = i; - pr_info("Using %u thread(s) for decompression\n", nr_threads); + pr_info("Using %u thread(s) for %s decompression\n", nr_threads, hib_comp_algo); pr_info("Loading and decompressing image data (%u pages)...\n", nr_to_read); m = nr_to_read / 10; @@ -1352,8 +1397,8 @@ static int load_compressed_image(struct swap_map_handle *handle, data[thr].cmp_len = *(size_t *)page[pg]; if (unlikely(!data[thr].cmp_len || data[thr].cmp_len > - lzo1x_worst_compress(UNC_SIZE))) { - pr_err("Invalid compressed length\n"); + bytes_worst_compress(UNC_SIZE))) { + pr_err("Invalid %s compressed length\n", hib_comp_algo); ret = -1; goto out_finish; } @@ -1404,14 +1449,14 @@ static int load_compressed_image(struct swap_map_handle *handle, ret = data[thr].ret; if (ret < 0) { - pr_err("decompression failed\n"); + pr_err("%s decompression failed\n", hib_comp_algo); goto out_finish; } if (unlikely(!data[thr].unc_len || data[thr].unc_len > UNC_SIZE || data[thr].unc_len & (PAGE_SIZE - 1))) { - pr_err("Invalid uncompressed length\n"); + pr_err("Invalid %s uncompressed length\n", hib_comp_algo); ret = -1; goto out_finish; } @@ -1472,9 +1517,12 @@ out_clean: kfree(crc); } if (data) { - for (thr = 0; thr < nr_threads; thr++) + for (thr = 0; thr < nr_threads; thr++) { if (data[thr].thr) kthread_stop(data[thr].thr); + if (data[thr].cc) + crypto_free_comp(data[thr].cc); + } vfree(data); } vfree(page); @@ -1545,6 +1593,7 @@ int swsusp_check(void) if (!memcmp(HIBERNATE_SIG, swsusp_header->sig, 10)) { memcpy(swsusp_header->sig, swsusp_header->orig_sig, 10); + swsusp_header_flags = swsusp_header->flags; /* Reset swap signature now */ error = hib_submit_io(REQ_OP_WRITE | REQ_SYNC, swsusp_resume_block, From 76c7e9747bcc22c4681c7f426105e5b53d1ca575 Mon Sep 17 00:00:00 2001 From: Nikhil V Date: Mon, 22 Jan 2024 18:45:27 +0530 Subject: [PATCH 12/86] UPSTREAM: PM: hibernate: Add support for LZ4 compression for hibernation Extend the support for LZ4 compression to be used with hibernation. The main idea is that different compression algorithms have different characteristics and hibernation may benefit when it uses any of these algorithms: a default algorithm, having higher compression rate but is slower(compression/decompression) and a secondary algorithm, that is faster(compression/decompression) but has lower compression rate. LZ4 algorithm has better decompression speeds over LZO. This reduces the hibernation image restore time. As per test results: LZO LZ4 Size before Compression(bytes) 682696704 682393600 Size after Compression(bytes) 146502402 155993547 Decompression Rate 335.02 MB/s 501.05 MB/s Restore time 4.4s 3.8s LZO is the default compression algorithm used for hibernation. Enable CONFIG_HIBERNATION_COMP_LZ4 to set the default compressor as LZ4. Bug: 335581841 Signed-off-by: Nikhil V Signed-off-by: Rafael J. Wysocki (cherry picked from commit 8bc29736357e7f9a6bd0d16b57b5612197e1924b) Change-Id: I640d834bb626e9a139e41740d4bef7548d5c6401 Signed-off-by: Nikhil V --- kernel/power/Kconfig | 5 +++++ kernel/power/hibernate.c | 25 ++++++++++++++++++++++--- kernel/power/power.h | 14 ++++++++++++++ 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/kernel/power/Kconfig b/kernel/power/Kconfig index 1a202bb1de78..884acc3c1c60 100644 --- a/kernel/power/Kconfig +++ b/kernel/power/Kconfig @@ -101,11 +101,16 @@ config HIBERNATION_COMP_LZO bool "lzo" depends on CRYPTO_LZO +config HIBERNATION_COMP_LZ4 + bool "lz4" + depends on CRYPTO_LZ4 + endchoice config HIBERNATION_DEF_COMP string default "lzo" if HIBERNATION_COMP_LZO + default "lz4" if HIBERNATION_COMP_LZ4 help Default compressor to be used for hibernation. diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c index 5ba0fc139ac3..a981f7175da5 100644 --- a/kernel/power/hibernate.c +++ b/kernel/power/hibernate.c @@ -725,6 +725,9 @@ static int load_image_and_restore(void) return error; } +#define COMPRESSION_ALGO_LZO "lzo" +#define COMPRESSION_ALGO_LZ4 "lz4" + /** * hibernate - Carry out system hibernation, including saving the image. */ @@ -786,11 +789,24 @@ int hibernate(void) if (hibernation_mode == HIBERNATION_PLATFORM) flags |= SF_PLATFORM_MODE; - if (nocompress) + if (nocompress) { flags |= SF_NOCOMPRESS_MODE; - else + } else { flags |= SF_CRC32_MODE; + /* + * By default, LZO compression is enabled. Use SF_COMPRESSION_ALG_LZ4 + * to override this behaviour and use LZ4. + * + * Refer kernel/power/power.h for more details + */ + + if (!strcmp(hib_comp_algo, COMPRESSION_ALGO_LZ4)) + flags |= SF_COMPRESSION_ALG_LZ4; + else + flags |= SF_COMPRESSION_ALG_LZO; + } + pm_pr_dbg("Writing hibernation image.\n"); error = swsusp_write(flags); swsusp_free(); @@ -1021,7 +1037,10 @@ static int software_resume(void) * the algorithm support. */ if (!(swsusp_header_flags & SF_NOCOMPRESS_MODE)) { - strscpy(hib_comp_algo, default_compressor, sizeof(hib_comp_algo)); + if (swsusp_header_flags & SF_COMPRESSION_ALG_LZ4) + strscpy(hib_comp_algo, COMPRESSION_ALGO_LZ4, sizeof(hib_comp_algo)); + else + strscpy(hib_comp_algo, default_compressor, sizeof(hib_comp_algo)); if (crypto_has_comp(hib_comp_algo, 0, 0) != 1) { pr_err("%s compression is not available\n", hib_comp_algo); error = -EOPNOTSUPP; diff --git a/kernel/power/power.h b/kernel/power/power.h index bae801d4bb9b..f8491615b85d 100644 --- a/kernel/power/power.h +++ b/kernel/power/power.h @@ -173,11 +173,25 @@ extern int swsusp_swap_in_use(void); * Flags that can be passed from the hibernatig hernel to the "boot" kernel in * the image header. */ +#define SF_COMPRESSION_ALG_LZO 0 /* dummy, details given below */ #define SF_PLATFORM_MODE 1 #define SF_NOCOMPRESS_MODE 2 #define SF_CRC32_MODE 4 #define SF_HW_SIG 8 +/* + * Bit to indicate the compression algorithm to be used(for LZ4). The same + * could be checked while saving/loading image to/from disk to use the + * corresponding algorithms. + * + * By default, LZO compression is enabled if SF_CRC32_MODE is set. Use + * SF_COMPRESSION_ALG_LZ4 to override this behaviour and use LZ4. + * + * SF_CRC32_MODE, SF_COMPRESSION_ALG_LZO(dummy) -> Compression, LZO + * SF_CRC32_MODE, SF_COMPRESSION_ALG_LZ4 -> Compression, LZ4 + */ +#define SF_COMPRESSION_ALG_LZ4 16 + /* kernel/power/hibernate.c */ extern int swsusp_check(void); extern void swsusp_free(void); From 54c2418b76d3ceeda53cde278f74ba14f0046bab Mon Sep 17 00:00:00 2001 From: Nikhil V Date: Wed, 14 Feb 2024 13:09:32 +0530 Subject: [PATCH 13/86] UPSTREAM: PM: hibernate: Support to select compression algorithm Currently the default compression algorithm is selected based on compile time options. Introduce a module parameter "hibernate.compressor" to override this behaviour. Different compression algorithms have different characteristics and hibernation may benefit when it uses any of these algorithms, especially when a secondary algorithm(LZ4) offers better decompression speeds over a default algorithm(LZO), which in turn reduces hibernation image restore time. Users can override the default algorithm in two ways: 1) Passing "hibernate.compressor" as kernel command line parameter. Usage: LZO: hibernate.compressor=lzo LZ4: hibernate.compressor=lz4 2) Specifying the algorithm at runtime. Usage: LZO: echo lzo > /sys/module/hibernate/parameters/compressor LZ4: echo lz4 > /sys/module/hibernate/parameters/compressor Currently LZO and LZ4 are the supported algorithms. LZO is the default compression algorithm used with hibernation. Bug: 335581841 Signed-off-by: Nikhil V Signed-off-by: Rafael J. Wysocki (cherry picked from commit 3fec6e5961b77af6a952b77f5c2ea26f7513b216) Change-Id: I3c787939c8d37dfeb2c164de69078d303411f938 Signed-off-by: Nikhil V --- .../admin-guide/kernel-parameters.txt | 11 ++++ kernel/power/hibernate.c | 57 ++++++++++++++++++- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 18bc322f2724..9bdb54a1df5d 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -1697,6 +1697,17 @@ (that will set all pages holding image data during restoration read-only). + hibernate.compressor= [HIBERNATION] Compression algorithm to be + used with hibernation. + Format: { lzo | lz4 } + Default: lzo + + lzo: Select LZO compression algorithm to + compress/decompress hibernation image. + + lz4: Select LZ4 compression algorithm to + compress/decompress hibernation image. + highmem=nn[KMG] [KNL,BOOT] forces the highmem zone to have an exact size of . This works even on boxes that have no highmem otherwise. This also works to reduce highmem diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c index a981f7175da5..678b12d5789f 100644 --- a/kernel/power/hibernate.c +++ b/kernel/power/hibernate.c @@ -46,7 +46,7 @@ dev_t swsusp_resume_device; sector_t swsusp_resume_block; __visible int in_suspend __nosavedata; -static const char *default_compressor = CONFIG_HIBERNATION_DEF_COMP; +static char hibernate_compressor[CRYPTO_MAX_ALG_NAME] = CONFIG_HIBERNATION_DEF_COMP; /* * Compression/decompression algorithm to be used while saving/loading @@ -745,7 +745,7 @@ int hibernate(void) * Query for the compression algorithm support if compression is enabled. */ if (!nocompress) { - strscpy(hib_comp_algo, default_compressor, sizeof(hib_comp_algo)); + strscpy(hib_comp_algo, hibernate_compressor, sizeof(hib_comp_algo)); if (crypto_has_comp(hib_comp_algo, 0, 0) != 1) { pr_err("%s compression is not available\n", hib_comp_algo); return -EOPNOTSUPP; @@ -1040,7 +1040,7 @@ static int software_resume(void) if (swsusp_header_flags & SF_COMPRESSION_ALG_LZ4) strscpy(hib_comp_algo, COMPRESSION_ALGO_LZ4, sizeof(hib_comp_algo)); else - strscpy(hib_comp_algo, default_compressor, sizeof(hib_comp_algo)); + strscpy(hib_comp_algo, COMPRESSION_ALGO_LZO, sizeof(hib_comp_algo)); if (crypto_has_comp(hib_comp_algo, 0, 0) != 1) { pr_err("%s compression is not available\n", hib_comp_algo); error = -EOPNOTSUPP; @@ -1410,6 +1410,57 @@ static int __init nohibernate_setup(char *str) return 1; } +static const char * const comp_alg_enabled[] = { +#if IS_ENABLED(CONFIG_CRYPTO_LZO) + COMPRESSION_ALGO_LZO, +#endif +#if IS_ENABLED(CONFIG_CRYPTO_LZ4) + COMPRESSION_ALGO_LZ4, +#endif +}; + +static int hibernate_compressor_param_set(const char *compressor, + const struct kernel_param *kp) +{ + unsigned int sleep_flags; + int index, ret; + + sleep_flags = lock_system_sleep(); + + index = sysfs_match_string(comp_alg_enabled, compressor); + if (index >= 0) { + ret = param_set_copystring(comp_alg_enabled[index], kp); + if (!ret) + strscpy(hib_comp_algo, comp_alg_enabled[index], + sizeof(hib_comp_algo)); + } else { + ret = index; + } + + unlock_system_sleep(sleep_flags); + + if (ret) + pr_debug("Cannot set specified compressor %s\n", + compressor); + + return ret; +} + +static const struct kernel_param_ops hibernate_compressor_param_ops = { + .set = hibernate_compressor_param_set, + .get = param_get_string, +}; + +static struct kparam_string hibernate_compressor_param_string = { + .maxlen = sizeof(hibernate_compressor), + .string = hibernate_compressor, +}; + +module_param_cb(compressor, &hibernate_compressor_param_ops, + &hibernate_compressor_param_string, 0644); +MODULE_PARM_DESC(compressor, + "Compression algorithm to be used with hibernation"); + __setup("noresume", noresume_setup); __setup("resume_offset=", resume_offset_setup); __setup("resume=", resume_setup); From e7e89326007ede26044492caf55bcc1e9a9d7d8e Mon Sep 17 00:00:00 2001 From: Nikhil V Date: Tue, 19 Sep 2023 16:00:51 +0530 Subject: [PATCH 14/86] ANDROID: gki_defconfig: Sync gki_defconfig After applying commit 990d3701d0aa4 ("BACKPORT: PM: hibernate: Move to crypto APIs for LZO compression"), CRPTO_LZO is selected by default. Sync the gki_defconfig accordingly. This doesn't add any functional change. Bug: 335581841 Change-Id: Iafa7211245f6d66a96f8f0030e2574c7a220d3a4 Signed-off-by: Nikhil V --- arch/arm64/configs/gki_defconfig | 1 - arch/x86/configs/gki_defconfig | 1 - 2 files changed, 2 deletions(-) diff --git a/arch/arm64/configs/gki_defconfig b/arch/arm64/configs/gki_defconfig index 5d6490f1bc35..869d9da6819e 100644 --- a/arch/arm64/configs/gki_defconfig +++ b/arch/arm64/configs/gki_defconfig @@ -692,7 +692,6 @@ CONFIG_CRYPTO_BLAKE2B=y CONFIG_CRYPTO_CMAC=y CONFIG_CRYPTO_MD5=y CONFIG_CRYPTO_XCBC=y -CONFIG_CRYPTO_LZO=y CONFIG_CRYPTO_LZ4=y CONFIG_CRYPTO_ZSTD=y CONFIG_CRYPTO_ANSI_CPRNG=y diff --git a/arch/x86/configs/gki_defconfig b/arch/x86/configs/gki_defconfig index 8330afe50d28..23efcc88be4b 100644 --- a/arch/x86/configs/gki_defconfig +++ b/arch/x86/configs/gki_defconfig @@ -635,7 +635,6 @@ CONFIG_CRYPTO_BLAKE2B=y CONFIG_CRYPTO_CMAC=y CONFIG_CRYPTO_MD5=y CONFIG_CRYPTO_XCBC=y -CONFIG_CRYPTO_LZO=y CONFIG_CRYPTO_LZ4=y CONFIG_CRYPTO_ZSTD=y CONFIG_CRYPTO_ANSI_CPRNG=y From 8f08ea0d599e65d0a7eb230ad37caa3f37c31a70 Mon Sep 17 00:00:00 2001 From: Nikhil V Date: Mon, 11 Sep 2023 11:31:38 +0530 Subject: [PATCH 15/86] ANDROID: vendor_hooks: Add hooks to support hibernation In case of hibernation with compression enabled, 'n' number of pages will be compressed to 'x' number of pages before being written to the disk. Keep a note of these compressed block counts so that bootloader can directly read 'x' pages and pass it on to the decompressor. An array will be maintained which will hold the count of these compressed blocks and later on written to the the disk as part of the hibernation image save process. The vendor hook '__tracepoint_android_vh_hibernated_do_mem_alloc' does the required memory allocations, for example, the array which is dynamically allocated based on the snapshot image size so as to hold the compressed block counts etc. This memory is later freed as part of PM_POST_HIBERNATION notifier call. The vendor hook '__tracepoint_android_vh_hibernate_save_cmp_len' saves the compressed block counts to the array which is later written to the disk. Bug: 335581841 Change-Id: I574b641e2d9f4cd503c7768a66a7be3142c2686b Signed-off-by: Nikhil V --- drivers/android/vendor_hooks.c | 2 ++ include/trace/hooks/bl_hib.h | 9 +++++++++ kernel/power/swap.c | 14 +++++++++++++- 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 9322df8ce828..f742029f9086 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -406,3 +406,5 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_delayacct_wpcopy_end); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_usb_dev_suspend); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_usb_dev_resume); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_sound_usb_support_cpu_suspend); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_hibernated_do_mem_alloc); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_hibernate_save_cmp_len); diff --git a/include/trace/hooks/bl_hib.h b/include/trace/hooks/bl_hib.h index 7a5dd310a6aa..f334e2b61320 100644 --- a/include/trace/hooks/bl_hib.h +++ b/include/trace/hooks/bl_hib.h @@ -39,6 +39,15 @@ DECLARE_HOOK(android_vh_post_image_save, TP_PROTO(unsigned short root_swap), TP_ARGS(root_swap)); +DECLARE_HOOK(android_vh_hibernated_do_mem_alloc, + TP_PROTO(unsigned long nr_pages, unsigned int swsusp_header_flags, + int *ret), + TP_ARGS(nr_pages, swsusp_header_flags, ret)); + +DECLARE_HOOK(android_vh_hibernate_save_cmp_len, + TP_PROTO(size_t cmp_len), + TP_ARGS(cmp_len)); + #endif /* _TRACE_HOOK_BL_HIB_H */ /* This part must be outside protection */ #include diff --git a/kernel/power/swap.c b/kernel/power/swap.c index 017a55966411..f4f7f8d03cd4 100644 --- a/kernel/power/swap.c +++ b/kernel/power/swap.c @@ -883,6 +883,7 @@ static int save_compressed_image(struct swap_map_handle *handle, if (ret) goto out_finish; } + trace_android_vh_hibernate_save_cmp_len(data[thr].cmp_len + CMP_HEADER); } wait_event(crc->done, atomic_read(&crc->stop)); @@ -955,9 +956,20 @@ int swsusp_write(unsigned int flags) struct snapshot_handle snapshot; struct swsusp_info *header; unsigned long pages; - int error; + int error = 0; pages = snapshot_get_image_size(); + + /* + * The memory allocated by this vendor hook is later freed as part of + * PM_POST_HIBERNATION notifier call. + */ + trace_android_vh_hibernated_do_mem_alloc(pages, flags, &error); + if (error < 0) { + pr_err("Failed to allocate required memory\n"); + return error; + } + error = get_swap_writer(&handle); if (error) { pr_err("Cannot get swap writer\n"); From 3e99ae28ea8fc07e22535581e23dc3370845cdfb Mon Sep 17 00:00:00 2001 From: Nikhil V Date: Fri, 19 Apr 2024 11:39:30 +0530 Subject: [PATCH 16/86] ANDROID: abi_gki_aarch64_qcom: Update symbol list Add android_vh_hibernate_save_cmp_len, android_vh_hibernated_do_mem_alloc symbols to support compression with hibernation. Symbols added: __traceiter_android_vh_hibernate_save_cmp_len __traceiter_android_vh_hibernated_do_mem_alloc __tracepoint_android_vh_hibernate_save_cmp_len __tracepoint_android_vh_hibernated_do_mem_alloc Bug: 335581841 Change-Id: I99e704bd54f220bac180c5bbfec48da44359f27e Signed-off-by: Nikhil V --- android/abi_gki_aarch64.stg | 54 ++++++++++++++++++++++++++++++++++++ android/abi_gki_aarch64_qcom | 4 +++ 2 files changed, 58 insertions(+) diff --git a/android/abi_gki_aarch64.stg b/android/abi_gki_aarch64.stg index 26ceb203bdab..4ed0b449d63e 100644 --- a/android/abi_gki_aarch64.stg +++ b/android/abi_gki_aarch64.stg @@ -317934,6 +317934,12 @@ function { return_type_id: 0x6720d32f parameter_id: 0x17fa285b } +function { + id: 0x987f6fc2 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0xf435685e +} function { id: 0x987f99ef return_type_id: 0x01f1fcec @@ -322827,6 +322833,14 @@ function { parameter_id: 0x6720d32f parameter_id: 0x1c898f28 } +function { + id: 0x9b7343c8 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x33756485 + parameter_id: 0x4585663f + parameter_id: 0x13580d6c +} function { id: 0x9b735e47 return_type_id: 0x6720d32f @@ -342302,6 +342316,24 @@ elf_symbol { type_id: 0x9b1477ea full_name: "__traceiter_android_vh_gic_v3_suspend" } +elf_symbol { + id: 0x089c5048 + name: "__traceiter_android_vh_hibernate_save_cmp_len" + is_defined: true + symbol_type: FUNCTION + crc: 0x32d19ddb + type_id: 0x987f6fc2 + full_name: "__traceiter_android_vh_hibernate_save_cmp_len" +} +elf_symbol { + id: 0x1ba2e410 + name: "__traceiter_android_vh_hibernated_do_mem_alloc" + is_defined: true + symbol_type: FUNCTION + crc: 0x69a16a12 + type_id: 0x9b7343c8 + full_name: "__traceiter_android_vh_hibernated_do_mem_alloc" +} elf_symbol { id: 0xdb6662e8 name: "__traceiter_android_vh_ignore_dmabuf_vmap_bounds" @@ -346532,6 +346564,24 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_gic_v3_suspend" } +elf_symbol { + id: 0xc388a58e + name: "__tracepoint_android_vh_hibernate_save_cmp_len" + is_defined: true + symbol_type: OBJECT + crc: 0xa114dd34 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_hibernate_save_cmp_len" +} +elf_symbol { + id: 0xc62a97aa + name: "__tracepoint_android_vh_hibernated_do_mem_alloc" + is_defined: true + symbol_type: OBJECT + crc: 0xcf28c832 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_hibernated_do_mem_alloc" +} elf_symbol { id: 0xf68497fa name: "__tracepoint_android_vh_ignore_dmabuf_vmap_bounds" @@ -407016,6 +407066,8 @@ interface { symbol_id: 0x1df03151 symbol_id: 0xdbe4202a symbol_id: 0x88586647 + symbol_id: 0x089c5048 + symbol_id: 0x1ba2e410 symbol_id: 0xdb6662e8 symbol_id: 0x64c2fe7a symbol_id: 0x4b67a5dc @@ -407486,6 +407538,8 @@ interface { symbol_id: 0x8b7beb3b symbol_id: 0x60a121b4 symbol_id: 0xf18e312d + symbol_id: 0xc388a58e + symbol_id: 0xc62a97aa symbol_id: 0xf68497fa symbol_id: 0xb6603d58 symbol_id: 0xbbfc3b7e diff --git a/android/abi_gki_aarch64_qcom b/android/abi_gki_aarch64_qcom index 0fafde6e16e1..d3aa2d29239d 100644 --- a/android/abi_gki_aarch64_qcom +++ b/android/abi_gki_aarch64_qcom @@ -3438,6 +3438,8 @@ __traceiter_android_vh_ftrace_oops_exit __traceiter_android_vh_ftrace_size_check __traceiter_android_vh_gic_v3_suspend + __traceiter_android_vh_hibernate_save_cmp_len + __traceiter_android_vh_hibernated_do_mem_alloc __traceiter_android_vh_ignore_dmabuf_vmap_bounds __traceiter_android_vh_init_aes_encrypt __traceiter_android_vh_ipi_stop @@ -3587,6 +3589,8 @@ __tracepoint_android_vh_ftrace_oops_exit __tracepoint_android_vh_ftrace_size_check __tracepoint_android_vh_gic_v3_suspend + __tracepoint_android_vh_hibernate_save_cmp_len + __tracepoint_android_vh_hibernated_do_mem_alloc __tracepoint_android_vh_ignore_dmabuf_vmap_bounds __tracepoint_android_vh_init_aes_encrypt __tracepoint_android_vh_ipi_stop From 503add1843888822f114019f2cf0e84ec936ae4f Mon Sep 17 00:00:00 2001 From: Nikhil V Date: Mon, 11 Sep 2023 11:34:34 +0530 Subject: [PATCH 17/86] ANDROID: PM: hibernate: Encryption support with compression Currently only the uncompressed hibernation snapshot image is encrypted before being written to the swap partition. Extend the encryption support for compression enabled scenarios as well. Bug: 335581841 Change-Id: Ida781b727f56b664a67e2887a4db3d6b355dafdb Signed-off-by: Nikhil V --- kernel/power/swap.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kernel/power/swap.c b/kernel/power/swap.c index f4f7f8d03cd4..1e7f41c09a7e 100644 --- a/kernel/power/swap.c +++ b/kernel/power/swap.c @@ -596,7 +596,6 @@ static int save_image(struct swap_map_handle *handle, if (!ret) pr_info("Image saving done\n"); swsusp_show_speed(start, stop, nr_to_write, "Wrote"); - trace_android_vh_post_image_save(root_swap); return ret; } @@ -879,6 +878,7 @@ static int save_compressed_image(struct swap_map_handle *handle, off += PAGE_SIZE) { memcpy(page, data[thr].cmp + off, PAGE_SIZE); + trace_android_vh_encrypt_page(page); ret = swap_write_page(handle, page, &hb); if (ret) goto out_finish; @@ -997,6 +997,9 @@ int swsusp_write(unsigned int flags) error = (flags & SF_NOCOMPRESS_MODE) ? save_image(&handle, &snapshot, pages - 1) : save_compressed_image(&handle, &snapshot, pages - 1); + + if (!error) + trace_android_vh_post_image_save(root_swap); } out_finish: error = swap_writer_finish(&handle, flags, error); From 25db7c13d8af7f648b578b7ae253d0f929709915 Mon Sep 17 00:00:00 2001 From: Andrea Arcangeli Date: Wed, 6 Dec 2023 02:36:55 -0800 Subject: [PATCH 18/86] UPSTREAM: mm/rmap: support move to different root anon_vma in folio_move_anon_rmap() Patch series "userfaultfd move option", v6. This patch series introduces UFFDIO_MOVE feature to userfaultfd, which has long been implemented and maintained by Andrea in his local tree [1], but was not upstreamed due to lack of use cases where this approach would be better than allocating a new page and copying the contents. Previous upstraming attempts could be found at [6] and [7]. UFFDIO_COPY performs ~20% better than UFFDIO_MOVE when the application needs pages to be allocated [2]. However, with UFFDIO_MOVE, if pages are available (in userspace) for recycling, as is usually the case in heap compaction algorithms, then we can avoid the page allocation and memcpy (done by UFFDIO_COPY). Also, since the pages are recycled in the userspace, we avoid the need to release (via madvise) the pages back to the kernel [3]. We see over 40% reduction (on a Google pixel 6 device) in the compacting thread's completion time by using UFFDIO_MOVE vs. UFFDIO_COPY. This was measured using a benchmark that emulates a heap compaction implementation using userfaultfd (to allow concurrent accesses by application threads). More details of the usecase are explained in [3]. Furthermore, UFFDIO_MOVE enables moving swapped-out pages without touching them within the same vma. Today, it can only be done by mremap, however it forces splitting the vma. TODOs for follow-up improvements: - cross-mm support. Known differences from single-mm and missing pieces: - memcg recharging (might need to isolate pages in the process) - mm counters - cross-mm deposit table moves - cross-mm test - document the address space where src and dest reside in struct uffdio_move - TLB flush batching. Will require extensive changes to PTL locking in move_pages_pte(). OTOH that might let us reuse parts of mremap code. This patch (of 5): For now, folio_move_anon_rmap() was only used to move a folio to a different anon_vma after fork(), whereby the root anon_vma stayed unchanged. For that, it was sufficient to hold the folio lock when calling folio_move_anon_rmap(). However, we want to make use of folio_move_anon_rmap() to move folios between VMAs that have a different root anon_vma. As folio_referenced() performs an RMAP walk without holding the folio lock but only holding the anon_vma in read mode, holding the folio lock is insufficient. When moving to an anon_vma with a different root anon_vma, we'll have to hold both, the folio lock and the anon_vma lock in write mode. Consequently, whenever we succeeded in folio_lock_anon_vma_read() to read-lock the anon_vma, we have to re-check if the mapping was changed in the meantime. If that was the case, we have to retry. Note that folio_move_anon_rmap() must only be called if the anon page is exclusive to a process, and must not be called on KSM folios. This is a preparation for UFFDIO_MOVE, which will hold the folio lock, the anon_vma lock in write mode, and the mmap_lock in read mode. Link: https://lkml.kernel.org/r/20231206103702.3873743-1-surenb@google.com Link: https://lkml.kernel.org/r/20231206103702.3873743-2-surenb@google.com Signed-off-by: Andrea Arcangeli Signed-off-by: Suren Baghdasaryan Acked-by: Peter Xu Cc: Al Viro Cc: Axel Rasmussen Cc: Brian Geffon Cc: Christian Brauner Cc: David Hildenbrand Cc: Hugh Dickins Cc: Jann Horn Cc: Kalesh Singh Cc: kernel-team@android.com Cc: Liam R. Howlett Cc: Lokesh Gidra Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Mike Rapoport (IBM) Cc: Nicolas Geoffray Cc: Ryan Roberts Cc: Shuah Khan Cc: ZhangPeng Signed-off-by: Andrew Morton (cherry picked from commit 880a99b60d467eefd96322e27b0a8c0b805dfa43) Bug: 274911254 Change-Id: Iad9619c0273e050af26356f66ae9fc88b56d68bd Signed-off-by: Suren Baghdasaryan Signed-off-by: Lokesh Gidra --- mm/rmap.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/mm/rmap.c b/mm/rmap.c index 707de1aa3ceb..1c9007cd4ca0 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -543,6 +543,7 @@ struct anon_vma *folio_lock_anon_vma_read(struct folio *folio, struct anon_vma *root_anon_vma; unsigned long anon_mapping; +retry: rcu_read_lock(); anon_mapping = (unsigned long)READ_ONCE(folio->mapping); if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON) @@ -553,6 +554,17 @@ struct anon_vma *folio_lock_anon_vma_read(struct folio *folio, anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON); root_anon_vma = READ_ONCE(anon_vma->root); if (down_read_trylock(&root_anon_vma->rwsem)) { + /* + * folio_move_anon_rmap() might have changed the anon_vma as we + * might not hold the folio lock here. + */ + if (unlikely((unsigned long)READ_ONCE(folio->mapping) != + anon_mapping)) { + up_read(&root_anon_vma->rwsem); + rcu_read_unlock(); + goto retry; + } + /* * If the folio is still mapped, then this anon_vma is still * its anon_vma, and holding the mutex ensures that it will @@ -587,6 +599,18 @@ struct anon_vma *folio_lock_anon_vma_read(struct folio *folio, rcu_read_unlock(); anon_vma_lock_read(anon_vma); + /* + * folio_move_anon_rmap() might have changed the anon_vma as we might + * not hold the folio lock here. + */ + if (unlikely((unsigned long)READ_ONCE(folio->mapping) != + anon_mapping)) { + anon_vma_unlock_read(anon_vma); + put_anon_vma(anon_vma); + anon_vma = NULL; + goto retry; + } + if (atomic_dec_and_test(&anon_vma->refcount)) { /* * Oops, we held the last refcount, release the lock From 5025ad140e977654afb4896e2882bfc7fd6e48a5 Mon Sep 17 00:00:00 2001 From: Andrea Arcangeli Date: Wed, 6 Dec 2023 02:36:56 -0800 Subject: [PATCH 19/86] BACKPORT: userfaultfd: UFFDIO_MOVE uABI Implement the uABI of UFFDIO_MOVE ioctl. UFFDIO_COPY performs ~20% better than UFFDIO_MOVE when the application needs pages to be allocated [1]. However, with UFFDIO_MOVE, if pages are available (in userspace) for recycling, as is usually the case in heap compaction algorithms, then we can avoid the page allocation and memcpy (done by UFFDIO_COPY). Also, since the pages are recycled in the userspace, we avoid the need to release (via madvise) the pages back to the kernel [2]. We see over 40% reduction (on a Google pixel 6 device) in the compacting thread's completion time by using UFFDIO_MOVE vs. UFFDIO_COPY. This was measured using a benchmark that emulates a heap compaction implementation using userfaultfd (to allow concurrent accesses by application threads). More details of the usecase are explained in [2]. Furthermore, UFFDIO_MOVE enables moving swapped-out pages without touching them within the same vma. Today, it can only be done by mremap, however it forces splitting the vma. [1] https://lore.kernel.org/all/1425575884-2574-1-git-send-email-aarcange@redhat.com/ [2] https://lore.kernel.org/linux-mm/CA+EESO4uO84SSnBhArH4HvLNhaUQ5nZKNKXqxRCyjniNVjp0Aw@mail.gmail.com/ Update for the ioctl_userfaultfd(2) manpage: UFFDIO_MOVE (Since Linux xxx) Move a continuous memory chunk into the userfault registered range and optionally wake up the blocked thread. The source and destination addresses and the number of bytes to move are specified by the src, dst, and len fields of the uffdio_move structure pointed to by argp: struct uffdio_move { __u64 dst; /* Destination of move */ __u64 src; /* Source of move */ __u64 len; /* Number of bytes to move */ __u64 mode; /* Flags controlling behavior of move */ __s64 move; /* Number of bytes moved, or negated error */ }; The following value may be bitwise ORed in mode to change the behavior of the UFFDIO_MOVE operation: UFFDIO_MOVE_MODE_DONTWAKE Do not wake up the thread that waits for page-fault resolution UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES Allow holes in the source virtual range that is being moved. When not specified, the holes will result in ENOENT error. When specified, the holes will be accounted as successfully moved memory. This is mostly useful to move hugepage aligned virtual regions without knowing if there are transparent hugepages in the regions or not, but preventing the risk of having to split the hugepage during the operation. The move field is used by the kernel to return the number of bytes that was actually moved, or an error (a negated errno- style value). If the value returned in move doesn't match the value that was specified in len, the operation fails with the error EAGAIN. The move field is output-only; it is not read by the UFFDIO_MOVE operation. The operation may fail for various reasons. Usually, remapping of pages that are not exclusive to the given process fail; once KSM might deduplicate pages or fork() COW-shares pages during fork() with child processes, they are no longer exclusive. Further, the kernel might only perform lightweight checks for detecting whether the pages are exclusive, and return -EBUSY in case that check fails. To make the operation more likely to succeed, KSM should be disabled, fork() should be avoided or MADV_DONTFORK should be configured for the source VMA before fork(). This ioctl(2) operation returns 0 on success. In this case, the entire area was moved. On error, -1 is returned and errno is set to indicate the error. Possible errors include: EAGAIN The number of bytes moved (i.e., the value returned in the move field) does not equal the value that was specified in the len field. EINVAL Either dst or len was not a multiple of the system page size, or the range specified by src and len or dst and len was invalid. EINVAL An invalid bit was specified in the mode field. ENOENT The source virtual memory range has unmapped holes and UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES is not set. EEXIST The destination virtual memory range is fully or partially mapped. EBUSY The pages in the source virtual memory range are either pinned or not exclusive to the process. The kernel might only perform lightweight checks for detecting whether the pages are exclusive. To make the operation more likely to succeed, KSM should be disabled, fork() should be avoided or MADV_DONTFORK should be configured for the source virtual memory area before fork(). ENOMEM Allocating memory needed for the operation failed. ESRCH The target process has exited at the time of a UFFDIO_MOVE operation. Link: https://lkml.kernel.org/r/20231206103702.3873743-3-surenb@google.com Signed-off-by: Andrea Arcangeli Signed-off-by: Suren Baghdasaryan Cc: Al Viro Cc: Axel Rasmussen Cc: Brian Geffon Cc: Christian Brauner Cc: David Hildenbrand Cc: Hugh Dickins Cc: Jann Horn Cc: Kalesh Singh Cc: Liam R. Howlett Cc: Lokesh Gidra Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Mike Rapoport (IBM) Cc: Nicolas Geoffray Cc: Peter Xu Cc: Ryan Roberts Cc: Shuah Khan Cc: ZhangPeng Signed-off-by: Andrew Morton (cherry picked from commit adef440691bab824e39c1b17382322d195e1fab0) Conflicts: mm/huge_memory.c mm/userfaultfd.c 1. Add vma parameter in mmu_notifier_range_init() calls. 2. Replace folio_move_anon_rmap() with page_move_anon_rmap(). 3. Remove vma parameter in pmd_mkwrite() calls. 4. Replace pte_offset_map_nolock() with pte_offset_map()+pte_lockptr() combo. 5. Remove VM_SHADOW_STACK in vma_move_compatible(). 6. Replace pmdp_get_lockless() with pmd_read_atomic(). Bug: 274911254 Change-Id: I1116f15a395f1a8bac176906f7f9c2411e59dc54 Signed-off-by: Suren Baghdasaryan Signed-off-by: Lokesh Gidra --- Documentation/admin-guide/mm/userfaultfd.rst | 3 + fs/userfaultfd.c | 72 +++ include/linux/rmap.h | 5 + include/linux/userfaultfd_k.h | 11 + include/uapi/linux/userfaultfd.h | 29 +- mm/huge_memory.c | 122 ++++ mm/khugepaged.c | 3 + mm/rmap.c | 6 + mm/userfaultfd.c | 617 +++++++++++++++++++ 9 files changed, 867 insertions(+), 1 deletion(-) diff --git a/Documentation/admin-guide/mm/userfaultfd.rst b/Documentation/admin-guide/mm/userfaultfd.rst index 83f31919ebb3..3c6f33cf8960 100644 --- a/Documentation/admin-guide/mm/userfaultfd.rst +++ b/Documentation/admin-guide/mm/userfaultfd.rst @@ -115,6 +115,9 @@ events, except page fault notifications, may be generated: areas. ``UFFD_FEATURE_MINOR_SHMEM`` is the analogous feature indicating support for shmem virtual memory areas. +- ``UFFD_FEATURE_MOVE`` indicates that the kernel supports moving an + existing page contents from userspace. + The userland application should set the feature flags it intends to use when invoking the ``UFFDIO_API`` ioctl, to request that those features be enabled if supported. diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index bec3e2341421..e7ba93d22874 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -1944,6 +1944,75 @@ static inline unsigned int uffd_ctx_features(__u64 user_features) return (unsigned int)user_features | UFFD_FEATURE_INITIALIZED; } +static int userfaultfd_move(struct userfaultfd_ctx *ctx, + unsigned long arg) +{ + __s64 ret; + struct uffdio_move uffdio_move; + struct uffdio_move __user *user_uffdio_move; + struct userfaultfd_wake_range range; + struct mm_struct *mm = ctx->mm; + + user_uffdio_move = (struct uffdio_move __user *) arg; + + if (atomic_read(&ctx->mmap_changing)) + return -EAGAIN; + + if (copy_from_user(&uffdio_move, user_uffdio_move, + /* don't copy "move" last field */ + sizeof(uffdio_move)-sizeof(__s64))) + return -EFAULT; + + /* Do not allow cross-mm moves. */ + if (mm != current->mm) + return -EINVAL; + + ret = validate_range(mm, uffdio_move.dst, uffdio_move.len); + if (ret) + return ret; + + ret = validate_range(mm, uffdio_move.src, uffdio_move.len); + if (ret) + return ret; + + if (uffdio_move.mode & ~(UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES| + UFFDIO_MOVE_MODE_DONTWAKE)) + return -EINVAL; + + if (mmget_not_zero(mm)) { + mmap_read_lock(mm); + + /* Re-check after taking mmap_lock */ + if (likely(!atomic_read(&ctx->mmap_changing))) + ret = move_pages(ctx, mm, uffdio_move.dst, uffdio_move.src, + uffdio_move.len, uffdio_move.mode); + else + ret = -EINVAL; + + mmap_read_unlock(mm); + mmput(mm); + } else { + return -ESRCH; + } + + if (unlikely(put_user(ret, &user_uffdio_move->move))) + return -EFAULT; + if (ret < 0) + goto out; + + /* len == 0 would wake all */ + VM_WARN_ON(!ret); + range.len = ret; + if (!(uffdio_move.mode & UFFDIO_MOVE_MODE_DONTWAKE)) { + range.start = uffdio_move.dst; + wake_userfault(ctx, &range); + } + ret = range.len == uffdio_move.len ? 0 : -EAGAIN; + +out: + return ret; +} + /* * userland asks for a certain API version and we return which bits * and ioctl commands are implemented in this kernel for such API @@ -2029,6 +2098,9 @@ static long userfaultfd_ioctl(struct file *file, unsigned cmd, case UFFDIO_ZEROPAGE: ret = userfaultfd_zeropage(ctx, arg); break; + case UFFDIO_MOVE: + ret = userfaultfd_move(ctx, arg); + break; case UFFDIO_WRITEPROTECT: ret = userfaultfd_writeprotect(ctx, arg); break; diff --git a/include/linux/rmap.h b/include/linux/rmap.h index 2bdba700bc3e..83796abbb3cb 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -121,6 +121,11 @@ static inline void anon_vma_lock_write(struct anon_vma *anon_vma) down_write(&anon_vma->root->rwsem); } +static inline int anon_vma_trylock_write(struct anon_vma *anon_vma) +{ + return down_write_trylock(&anon_vma->root->rwsem); +} + static inline void anon_vma_unlock_write(struct anon_vma *anon_vma) { up_write(&anon_vma->root->rwsem); diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index 8c32d18648fc..29c0f4fd1511 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -76,6 +76,17 @@ extern int mwriteprotect_range(struct mm_struct *dst_mm, extern void uffd_wp_range(struct mm_struct *dst_mm, struct vm_area_struct *vma, unsigned long start, unsigned long len, bool enable_wp); +/* move_pages */ +void double_pt_lock(spinlock_t *ptl1, spinlock_t *ptl2); +void double_pt_unlock(spinlock_t *ptl1, spinlock_t *ptl2); +ssize_t move_pages(struct userfaultfd_ctx *ctx, struct mm_struct *mm, + unsigned long dst_start, unsigned long src_start, + unsigned long len, __u64 flags); +int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pmd_t dst_pmdval, + struct vm_area_struct *dst_vma, + struct vm_area_struct *src_vma, + unsigned long dst_addr, unsigned long src_addr); + /* mm helpers */ static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, struct vm_userfaultfd_ctx vm_ctx) diff --git a/include/uapi/linux/userfaultfd.h b/include/uapi/linux/userfaultfd.h index 005e5e306266..2be5c8899f1f 100644 --- a/include/uapi/linux/userfaultfd.h +++ b/include/uapi/linux/userfaultfd.h @@ -38,7 +38,8 @@ UFFD_FEATURE_MINOR_HUGETLBFS | \ UFFD_FEATURE_MINOR_SHMEM | \ UFFD_FEATURE_EXACT_ADDRESS | \ - UFFD_FEATURE_WP_HUGETLBFS_SHMEM) + UFFD_FEATURE_WP_HUGETLBFS_SHMEM | \ + UFFD_FEATURE_MOVE) #define UFFD_API_IOCTLS \ ((__u64)1 << _UFFDIO_REGISTER | \ (__u64)1 << _UFFDIO_UNREGISTER | \ @@ -47,6 +48,7 @@ ((__u64)1 << _UFFDIO_WAKE | \ (__u64)1 << _UFFDIO_COPY | \ (__u64)1 << _UFFDIO_ZEROPAGE | \ + (__u64)1 << _UFFDIO_MOVE | \ (__u64)1 << _UFFDIO_WRITEPROTECT | \ (__u64)1 << _UFFDIO_CONTINUE) #define UFFD_API_RANGE_IOCTLS_BASIC \ @@ -68,6 +70,7 @@ #define _UFFDIO_WAKE (0x02) #define _UFFDIO_COPY (0x03) #define _UFFDIO_ZEROPAGE (0x04) +#define _UFFDIO_MOVE (0x05) #define _UFFDIO_WRITEPROTECT (0x06) #define _UFFDIO_CONTINUE (0x07) #define _UFFDIO_API (0x3F) @@ -86,6 +89,8 @@ struct uffdio_copy) #define UFFDIO_ZEROPAGE _IOWR(UFFDIO, _UFFDIO_ZEROPAGE, \ struct uffdio_zeropage) +#define UFFDIO_MOVE _IOWR(UFFDIO, _UFFDIO_MOVE, \ + struct uffdio_move) #define UFFDIO_WRITEPROTECT _IOWR(UFFDIO, _UFFDIO_WRITEPROTECT, \ struct uffdio_writeprotect) #define UFFDIO_CONTINUE _IOWR(UFFDIO, _UFFDIO_CONTINUE, \ @@ -203,6 +208,9 @@ struct uffdio_api { * * UFFD_FEATURE_WP_HUGETLBFS_SHMEM indicates that userfaultfd * write-protection mode is supported on both shmem and hugetlbfs. + * + * UFFD_FEATURE_MOVE indicates that the kernel supports moving an + * existing page contents from userspace. */ #define UFFD_FEATURE_PAGEFAULT_FLAG_WP (1<<0) #define UFFD_FEATURE_EVENT_FORK (1<<1) @@ -217,6 +225,7 @@ struct uffdio_api { #define UFFD_FEATURE_MINOR_SHMEM (1<<10) #define UFFD_FEATURE_EXACT_ADDRESS (1<<11) #define UFFD_FEATURE_WP_HUGETLBFS_SHMEM (1<<12) +#define UFFD_FEATURE_MOVE (1<<16) __u64 features; __u64 ioctls; @@ -306,6 +315,24 @@ struct uffdio_continue { __s64 mapped; }; +struct uffdio_move { + __u64 dst; + __u64 src; + __u64 len; + /* + * Especially if used to atomically remove memory from the + * address space the wake on the dst range is not needed. + */ +#define UFFDIO_MOVE_MODE_DONTWAKE ((__u64)1<<0) +#define UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES ((__u64)1<<1) + __u64 mode; + /* + * "move" is written by the ioctl and must be at the end: the + * copy_from_user will not read the last 8 bytes. + */ + __s64 move; +}; + /* * Flags for the userfaultfd(2) system call itself. */ diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 59577946735b..7a159e142c73 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1913,6 +1913,128 @@ unlock: return ret; } +#ifdef CONFIG_USERFAULTFD +/* + * The PT lock for src_pmd and the mmap_lock for reading are held by + * the caller, but it must return after releasing the page_table_lock. + * Just move the page from src_pmd to dst_pmd if possible. + * Return zero if succeeded in moving the page, -EAGAIN if it needs to be + * repeated by the caller, or other errors in case of failure. + */ +int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pmd_t dst_pmdval, + struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, + unsigned long dst_addr, unsigned long src_addr) +{ + pmd_t _dst_pmd, src_pmdval; + struct page *src_page; + struct folio *src_folio; + struct anon_vma *src_anon_vma; + spinlock_t *src_ptl, *dst_ptl; + pgtable_t src_pgtable; + struct mmu_notifier_range range; + int err = 0; + + src_pmdval = *src_pmd; + src_ptl = pmd_lockptr(mm, src_pmd); + + lockdep_assert_held(src_ptl); + mmap_assert_locked(mm); + + /* Sanity checks before the operation */ + if (WARN_ON_ONCE(!pmd_none(dst_pmdval)) || WARN_ON_ONCE(src_addr & ~HPAGE_PMD_MASK) || + WARN_ON_ONCE(dst_addr & ~HPAGE_PMD_MASK)) { + spin_unlock(src_ptl); + return -EINVAL; + } + + if (!pmd_trans_huge(src_pmdval)) { + spin_unlock(src_ptl); + if (is_pmd_migration_entry(src_pmdval)) { + pmd_migration_entry_wait(mm, &src_pmdval); + return -EAGAIN; + } + return -ENOENT; + } + + src_page = pmd_page(src_pmdval); + if (unlikely(!PageAnonExclusive(src_page))) { + spin_unlock(src_ptl); + return -EBUSY; + } + + src_folio = page_folio(src_page); + folio_get(src_folio); + spin_unlock(src_ptl); + + flush_cache_range(src_vma, src_addr, src_addr + HPAGE_PMD_SIZE); + mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, src_vma, mm, src_addr, + src_addr + HPAGE_PMD_SIZE); + mmu_notifier_invalidate_range_start(&range); + + folio_lock(src_folio); + + /* + * split_huge_page walks the anon_vma chain without the page + * lock. Serialize against it with the anon_vma lock, the page + * lock is not enough. + */ + src_anon_vma = folio_get_anon_vma(src_folio); + if (!src_anon_vma) { + err = -EAGAIN; + goto unlock_folio; + } + anon_vma_lock_write(src_anon_vma); + + dst_ptl = pmd_lockptr(mm, dst_pmd); + double_pt_lock(src_ptl, dst_ptl); + if (unlikely(!pmd_same(*src_pmd, src_pmdval) || + !pmd_same(*dst_pmd, dst_pmdval))) { + err = -EAGAIN; + goto unlock_ptls; + } + if (folio_maybe_dma_pinned(src_folio) || + !PageAnonExclusive(&src_folio->page)) { + err = -EBUSY; + goto unlock_ptls; + } + + if (WARN_ON_ONCE(!folio_test_head(src_folio)) || + WARN_ON_ONCE(!folio_test_anon(src_folio))) { + err = -EBUSY; + goto unlock_ptls; + } + + page_move_anon_rmap(&src_folio->page, dst_vma); + WRITE_ONCE(src_folio->index, linear_page_index(dst_vma, dst_addr)); + + src_pmdval = pmdp_huge_clear_flush(src_vma, src_addr, src_pmd); + /* Folio got pinned from under us. Put it back and fail the move. */ + if (folio_maybe_dma_pinned(src_folio)) { + set_pmd_at(mm, src_addr, src_pmd, src_pmdval); + err = -EBUSY; + goto unlock_ptls; + } + + _dst_pmd = mk_huge_pmd(&src_folio->page, dst_vma->vm_page_prot); + /* Follow mremap() behavior and treat the entry dirty after the move */ + _dst_pmd = pmd_mkwrite(pmd_mkdirty(_dst_pmd)); + set_pmd_at(mm, dst_addr, dst_pmd, _dst_pmd); + + src_pgtable = pgtable_trans_huge_withdraw(mm, src_pmd); + pgtable_trans_huge_deposit(mm, dst_pmd, src_pgtable); +unlock_ptls: + double_pt_unlock(src_ptl, dst_ptl); + anon_vma_unlock_write(src_anon_vma); + put_anon_vma(src_anon_vma); +unlock_folio: + /* unblock rmap walks */ + folio_unlock(src_folio); + mmu_notifier_invalidate_range_end(&range); + folio_put(src_folio); + return err; +} +#endif /* CONFIG_USERFAULTFD */ + /* * Returns page table lock pointer if a given pmd maps a thp, NULL otherwise. * diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 01829b3c6ee5..a796760be1c3 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1032,6 +1032,9 @@ static int collapse_huge_page(struct mm_struct *mm, unsigned long address, * Prevent all access to pagetables with the exception of * gup_fast later handled by the ptep_clear_flush and the VM * handled by the anon_vma lock + PG_lock. + * + * UFFDIO_MOVE is prevented to race as well thanks to the + * mmap_lock. */ mmap_write_lock(mm); result = hugepage_vma_revalidate(mm, address, true, &vma, cc); diff --git a/mm/rmap.c b/mm/rmap.c index 1c9007cd4ca0..e1f7f7f86a8c 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -491,6 +491,12 @@ void __init anon_vma_init(void) * page_remove_rmap() that the anon_vma pointer from page->mapping is valid * if there is a mapcount, we can dereference the anon_vma after observing * those. + * + * NOTE: the caller should normally hold folio lock when calling this. If + * not, the caller needs to double check the anon_vma didn't change after + * taking the anon_vma lock for either read or write (UFFDIO_MOVE can modify it + * concurrently without folio lock protection). See folio_lock_anon_vma_read() + * which has already covered that, and comment above remap_pages(). */ struct anon_vma *folio_get_anon_vma(struct folio *folio) { diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 650ab6cfd5f4..9063874993d3 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -791,3 +791,620 @@ out_unlock: mmap_read_unlock(dst_mm); return err; } + + +void double_pt_lock(spinlock_t *ptl1, + spinlock_t *ptl2) + __acquires(ptl1) + __acquires(ptl2) +{ + spinlock_t *ptl_tmp; + + if (ptl1 > ptl2) { + /* exchange ptl1 and ptl2 */ + ptl_tmp = ptl1; + ptl1 = ptl2; + ptl2 = ptl_tmp; + } + /* lock in virtual address order to avoid lock inversion */ + spin_lock(ptl1); + if (ptl1 != ptl2) + spin_lock_nested(ptl2, SINGLE_DEPTH_NESTING); + else + __acquire(ptl2); +} + +void double_pt_unlock(spinlock_t *ptl1, + spinlock_t *ptl2) + __releases(ptl1) + __releases(ptl2) +{ + spin_unlock(ptl1); + if (ptl1 != ptl2) + spin_unlock(ptl2); + else + __release(ptl2); +} + + +static int move_present_pte(struct mm_struct *mm, + struct vm_area_struct *dst_vma, + struct vm_area_struct *src_vma, + unsigned long dst_addr, unsigned long src_addr, + pte_t *dst_pte, pte_t *src_pte, + pte_t orig_dst_pte, pte_t orig_src_pte, + spinlock_t *dst_ptl, spinlock_t *src_ptl, + struct folio *src_folio) +{ + int err = 0; + + double_pt_lock(dst_ptl, src_ptl); + + if (!pte_same(*src_pte, orig_src_pte) || + !pte_same(*dst_pte, orig_dst_pte)) { + err = -EAGAIN; + goto out; + } + if (folio_test_large(src_folio) || + folio_maybe_dma_pinned(src_folio) || + !PageAnonExclusive(&src_folio->page)) { + err = -EBUSY; + goto out; + } + + page_move_anon_rmap(&src_folio->page, dst_vma); + WRITE_ONCE(src_folio->index, linear_page_index(dst_vma, dst_addr)); + + orig_src_pte = ptep_clear_flush(src_vma, src_addr, src_pte); + /* Folio got pinned from under us. Put it back and fail the move. */ + if (folio_maybe_dma_pinned(src_folio)) { + set_pte_at(mm, src_addr, src_pte, orig_src_pte); + err = -EBUSY; + goto out; + } + + orig_dst_pte = mk_pte(&src_folio->page, dst_vma->vm_page_prot); + /* Follow mremap() behavior and treat the entry dirty after the move */ + orig_dst_pte = pte_mkwrite(pte_mkdirty(orig_dst_pte)); + + set_pte_at(mm, dst_addr, dst_pte, orig_dst_pte); +out: + double_pt_unlock(dst_ptl, src_ptl); + return err; +} + +static int move_swap_pte(struct mm_struct *mm, + unsigned long dst_addr, unsigned long src_addr, + pte_t *dst_pte, pte_t *src_pte, + pte_t orig_dst_pte, pte_t orig_src_pte, + spinlock_t *dst_ptl, spinlock_t *src_ptl) +{ + if (!pte_swp_exclusive(orig_src_pte)) + return -EBUSY; + + double_pt_lock(dst_ptl, src_ptl); + + if (!pte_same(*src_pte, orig_src_pte) || + !pte_same(*dst_pte, orig_dst_pte)) { + double_pt_unlock(dst_ptl, src_ptl); + return -EAGAIN; + } + + orig_src_pte = ptep_get_and_clear(mm, src_addr, src_pte); + set_pte_at(mm, dst_addr, dst_pte, orig_src_pte); + double_pt_unlock(dst_ptl, src_ptl); + + return 0; +} + +/* + * The mmap_lock for reading is held by the caller. Just move the page + * from src_pmd to dst_pmd if possible, and return true if succeeded + * in moving the page. + */ +static int move_pages_pte(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, + struct vm_area_struct *dst_vma, + struct vm_area_struct *src_vma, + unsigned long dst_addr, unsigned long src_addr, + __u64 mode) +{ + swp_entry_t entry; + pte_t orig_src_pte, orig_dst_pte; + pte_t src_folio_pte; + spinlock_t *src_ptl, *dst_ptl; + pte_t *src_pte = NULL; + pte_t *dst_pte = NULL; + + struct folio *src_folio = NULL; + struct anon_vma *src_anon_vma = NULL; + struct mmu_notifier_range range; + int err = 0; + + flush_cache_range(src_vma, src_addr, src_addr + PAGE_SIZE); + mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, src_vma, mm, + src_addr, src_addr + PAGE_SIZE); + mmu_notifier_invalidate_range_start(&range); +retry: + dst_pte = pte_offset_map(dst_pmd, dst_addr); + dst_ptl = pte_lockptr(mm, dst_pmd); + + + /* Retry if a huge pmd materialized from under us */ + if (unlikely(!dst_pte)) { + err = -EAGAIN; + goto out; + } + + src_pte = pte_offset_map(src_pmd, src_addr); + src_ptl = pte_lockptr(mm, src_pmd); + + /* + * We held the mmap_lock for reading so MADV_DONTNEED + * can zap transparent huge pages under us, or the + * transparent huge page fault can establish new + * transparent huge pages under us. + */ + if (unlikely(!src_pte)) { + err = -EAGAIN; + goto out; + } + + /* Sanity checks before the operation */ + if (WARN_ON_ONCE(pmd_none(*dst_pmd)) || WARN_ON_ONCE(pmd_none(*src_pmd)) || + WARN_ON_ONCE(pmd_trans_huge(*dst_pmd)) || WARN_ON_ONCE(pmd_trans_huge(*src_pmd))) { + err = -EINVAL; + goto out; + } + + spin_lock(dst_ptl); + orig_dst_pte = *dst_pte; + spin_unlock(dst_ptl); + if (!pte_none(orig_dst_pte)) { + err = -EEXIST; + goto out; + } + + spin_lock(src_ptl); + orig_src_pte = *src_pte; + spin_unlock(src_ptl); + if (pte_none(orig_src_pte)) { + if (!(mode & UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES)) + err = -ENOENT; + else /* nothing to do to move a hole */ + err = 0; + goto out; + } + + /* If PTE changed after we locked the folio them start over */ + if (src_folio && unlikely(!pte_same(src_folio_pte, orig_src_pte))) { + err = -EAGAIN; + goto out; + } + + if (pte_present(orig_src_pte)) { + /* + * Pin and lock both source folio and anon_vma. Since we are in + * RCU read section, we can't block, so on contention have to + * unmap the ptes, obtain the lock and retry. + */ + if (!src_folio) { + struct folio *folio; + + /* + * Pin the page while holding the lock to be sure the + * page isn't freed under us + */ + spin_lock(src_ptl); + if (!pte_same(orig_src_pte, *src_pte)) { + spin_unlock(src_ptl); + err = -EAGAIN; + goto out; + } + + folio = vm_normal_folio(src_vma, src_addr, orig_src_pte); + if (!folio || !PageAnonExclusive(&folio->page)) { + spin_unlock(src_ptl); + err = -EBUSY; + goto out; + } + + folio_get(folio); + src_folio = folio; + src_folio_pte = orig_src_pte; + spin_unlock(src_ptl); + + if (!folio_trylock(src_folio)) { + pte_unmap(&orig_src_pte); + pte_unmap(&orig_dst_pte); + src_pte = dst_pte = NULL; + /* now we can block and wait */ + folio_lock(src_folio); + goto retry; + } + + if (WARN_ON_ONCE(!folio_test_anon(src_folio))) { + err = -EBUSY; + goto out; + } + } + + /* at this point we have src_folio locked */ + if (folio_test_large(src_folio)) { + err = split_folio(src_folio); + if (err) + goto out; + } + + if (!src_anon_vma) { + /* + * folio_referenced walks the anon_vma chain + * without the folio lock. Serialize against it with + * the anon_vma lock, the folio lock is not enough. + */ + src_anon_vma = folio_get_anon_vma(src_folio); + if (!src_anon_vma) { + /* page was unmapped from under us */ + err = -EAGAIN; + goto out; + } + if (!anon_vma_trylock_write(src_anon_vma)) { + pte_unmap(&orig_src_pte); + pte_unmap(&orig_dst_pte); + src_pte = dst_pte = NULL; + /* now we can block and wait */ + anon_vma_lock_write(src_anon_vma); + goto retry; + } + } + + err = move_present_pte(mm, dst_vma, src_vma, + dst_addr, src_addr, dst_pte, src_pte, + orig_dst_pte, orig_src_pte, + dst_ptl, src_ptl, src_folio); + } else { + entry = pte_to_swp_entry(orig_src_pte); + if (non_swap_entry(entry)) { + if (is_migration_entry(entry)) { + pte_unmap(&orig_src_pte); + pte_unmap(&orig_dst_pte); + src_pte = dst_pte = NULL; + migration_entry_wait(mm, src_pmd, src_addr); + err = -EAGAIN; + } else + err = -EFAULT; + goto out; + } + + err = move_swap_pte(mm, dst_addr, src_addr, + dst_pte, src_pte, + orig_dst_pte, orig_src_pte, + dst_ptl, src_ptl); + } + +out: + if (src_anon_vma) { + anon_vma_unlock_write(src_anon_vma); + put_anon_vma(src_anon_vma); + } + if (src_folio) { + folio_unlock(src_folio); + folio_put(src_folio); + } + if (dst_pte) + pte_unmap(dst_pte); + if (src_pte) + pte_unmap(src_pte); + mmu_notifier_invalidate_range_end(&range); + + return err; +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static inline bool move_splits_huge_pmd(unsigned long dst_addr, + unsigned long src_addr, + unsigned long src_end) +{ + return (src_addr & ~HPAGE_PMD_MASK) || (dst_addr & ~HPAGE_PMD_MASK) || + src_end - src_addr < HPAGE_PMD_SIZE; +} +#else +static inline bool move_splits_huge_pmd(unsigned long dst_addr, + unsigned long src_addr, + unsigned long src_end) +{ + /* This is unreachable anyway, just to avoid warnings when HPAGE_PMD_SIZE==0 */ + return false; +} +#endif + +static inline bool vma_move_compatible(struct vm_area_struct *vma) +{ + return !(vma->vm_flags & (VM_PFNMAP | VM_IO | VM_HUGETLB | + VM_MIXEDMAP)); +} + +static int validate_move_areas(struct userfaultfd_ctx *ctx, + struct vm_area_struct *src_vma, + struct vm_area_struct *dst_vma) +{ + /* Only allow moving if both have the same access and protection */ + if ((src_vma->vm_flags & VM_ACCESS_FLAGS) != (dst_vma->vm_flags & VM_ACCESS_FLAGS) || + pgprot_val(src_vma->vm_page_prot) != pgprot_val(dst_vma->vm_page_prot)) + return -EINVAL; + + /* Only allow moving if both are mlocked or both aren't */ + if ((src_vma->vm_flags & VM_LOCKED) != (dst_vma->vm_flags & VM_LOCKED)) + return -EINVAL; + + /* + * For now, we keep it simple and only move between writable VMAs. + * Access flags are equal, therefore cheching only the source is enough. + */ + if (!(src_vma->vm_flags & VM_WRITE)) + return -EINVAL; + + /* Check if vma flags indicate content which can be moved */ + if (!vma_move_compatible(src_vma) || !vma_move_compatible(dst_vma)) + return -EINVAL; + + /* Ensure dst_vma is registered in uffd we are operating on */ + if (!dst_vma->vm_userfaultfd_ctx.ctx || + dst_vma->vm_userfaultfd_ctx.ctx != ctx) + return -EINVAL; + + /* Only allow moving across anonymous vmas */ + if (!vma_is_anonymous(src_vma) || !vma_is_anonymous(dst_vma)) + return -EINVAL; + + /* + * Ensure the dst_vma has a anon_vma or this page + * would get a NULL anon_vma when moved in the + * dst_vma. + */ + if (unlikely(anon_vma_prepare(dst_vma))) + return -ENOMEM; + + return 0; +} + +/** + * move_pages - move arbitrary anonymous pages of an existing vma + * @ctx: pointer to the userfaultfd context + * @mm: the address space to move pages + * @dst_start: start of the destination virtual memory range + * @src_start: start of the source virtual memory range + * @len: length of the virtual memory range + * @mode: flags from uffdio_move.mode + * + * Must be called with mmap_lock held for read. + * + * move_pages() remaps arbitrary anonymous pages atomically in zero + * copy. It only works on non shared anonymous pages because those can + * be relocated without generating non linear anon_vmas in the rmap + * code. + * + * It provides a zero copy mechanism to handle userspace page faults. + * The source vma pages should have mapcount == 1, which can be + * enforced by using madvise(MADV_DONTFORK) on src vma. + * + * The thread receiving the page during the userland page fault + * will receive the faulting page in the source vma through the network, + * storage or any other I/O device (MADV_DONTFORK in the source vma + * avoids move_pages() to fail with -EBUSY if the process forks before + * move_pages() is called), then it will call move_pages() to map the + * page in the faulting address in the destination vma. + * + * This userfaultfd command works purely via pagetables, so it's the + * most efficient way to move physical non shared anonymous pages + * across different virtual addresses. Unlike mremap()/mmap()/munmap() + * it does not create any new vmas. The mapping in the destination + * address is atomic. + * + * It only works if the vma protection bits are identical from the + * source and destination vma. + * + * It can remap non shared anonymous pages within the same vma too. + * + * If the source virtual memory range has any unmapped holes, or if + * the destination virtual memory range is not a whole unmapped hole, + * move_pages() will fail respectively with -ENOENT or -EEXIST. This + * provides a very strict behavior to avoid any chance of memory + * corruption going unnoticed if there are userland race conditions. + * Only one thread should resolve the userland page fault at any given + * time for any given faulting address. This means that if two threads + * try to both call move_pages() on the same destination address at the + * same time, the second thread will get an explicit error from this + * command. + * + * The command retval will return "len" is successful. The command + * however can be interrupted by fatal signals or errors. If + * interrupted it will return the number of bytes successfully + * remapped before the interruption if any, or the negative error if + * none. It will never return zero. Either it will return an error or + * an amount of bytes successfully moved. If the retval reports a + * "short" remap, the move_pages() command should be repeated by + * userland with src+retval, dst+reval, len-retval if it wants to know + * about the error that interrupted it. + * + * The UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES flag can be specified to + * prevent -ENOENT errors to materialize if there are holes in the + * source virtual range that is being remapped. The holes will be + * accounted as successfully remapped in the retval of the + * command. This is mostly useful to remap hugepage naturally aligned + * virtual regions without knowing if there are transparent hugepage + * in the regions or not, but preventing the risk of having to split + * the hugepmd during the remap. + * + * If there's any rmap walk that is taking the anon_vma locks without + * first obtaining the folio lock (the only current instance is + * folio_referenced), they will have to verify if the folio->mapping + * has changed after taking the anon_vma lock. If it changed they + * should release the lock and retry obtaining a new anon_vma, because + * it means the anon_vma was changed by move_pages() before the lock + * could be obtained. This is the only additional complexity added to + * the rmap code to provide this anonymous page remapping functionality. + */ +ssize_t move_pages(struct userfaultfd_ctx *ctx, struct mm_struct *mm, + unsigned long dst_start, unsigned long src_start, + unsigned long len, __u64 mode) +{ + struct vm_area_struct *src_vma, *dst_vma; + unsigned long src_addr, dst_addr; + pmd_t *src_pmd, *dst_pmd; + long err = -EINVAL; + ssize_t moved = 0; + + /* Sanitize the command parameters. */ + if (WARN_ON_ONCE(src_start & ~PAGE_MASK) || + WARN_ON_ONCE(dst_start & ~PAGE_MASK) || + WARN_ON_ONCE(len & ~PAGE_MASK)) + goto out; + + /* Does the address range wrap, or is the span zero-sized? */ + if (WARN_ON_ONCE(src_start + len <= src_start) || + WARN_ON_ONCE(dst_start + len <= dst_start)) + goto out; + + /* + * Make sure the vma is not shared, that the src and dst remap + * ranges are both valid and fully within a single existing + * vma. + */ + src_vma = find_vma(mm, src_start); + if (!src_vma || (src_vma->vm_flags & VM_SHARED)) + goto out; + if (src_start < src_vma->vm_start || + src_start + len > src_vma->vm_end) + goto out; + + dst_vma = find_vma(mm, dst_start); + if (!dst_vma || (dst_vma->vm_flags & VM_SHARED)) + goto out; + if (dst_start < dst_vma->vm_start || + dst_start + len > dst_vma->vm_end) + goto out; + + err = validate_move_areas(ctx, src_vma, dst_vma); + if (err) + goto out; + + for (src_addr = src_start, dst_addr = dst_start; + src_addr < src_start + len;) { + spinlock_t *ptl; + pmd_t dst_pmdval; + unsigned long step_size; + + /* + * Below works because anonymous area would not have a + * transparent huge PUD. If file-backed support is added, + * that case would need to be handled here. + */ + src_pmd = mm_find_pmd(mm, src_addr); + if (unlikely(!src_pmd)) { + if (!(mode & UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES)) { + err = -ENOENT; + break; + } + src_pmd = mm_alloc_pmd(mm, src_addr); + if (unlikely(!src_pmd)) { + err = -ENOMEM; + break; + } + } + dst_pmd = mm_alloc_pmd(mm, dst_addr); + if (unlikely(!dst_pmd)) { + err = -ENOMEM; + break; + } + + dst_pmdval = pmd_read_atomic(dst_pmd); + /* + * If the dst_pmd is mapped as THP don't override it and just + * be strict. If dst_pmd changes into TPH after this check, the + * move_pages_huge_pmd() will detect the change and retry + * while move_pages_pte() will detect the change and fail. + */ + if (unlikely(pmd_trans_huge(dst_pmdval))) { + err = -EEXIST; + break; + } + + ptl = pmd_trans_huge_lock(src_pmd, src_vma); + if (ptl) { + if (pmd_devmap(*src_pmd)) { + spin_unlock(ptl); + err = -ENOENT; + break; + } + + /* Check if we can move the pmd without splitting it. */ + if (move_splits_huge_pmd(dst_addr, src_addr, src_start + len) || + !pmd_none(dst_pmdval)) { + struct folio *folio = pfn_folio(pmd_pfn(*src_pmd)); + + if (!folio || !PageAnonExclusive(&folio->page)) { + spin_unlock(ptl); + err = -EBUSY; + break; + } + + spin_unlock(ptl); + split_huge_pmd(src_vma, src_pmd, src_addr); + /* The folio will be split by move_pages_pte() */ + continue; + } + + err = move_pages_huge_pmd(mm, dst_pmd, src_pmd, + dst_pmdval, dst_vma, src_vma, + dst_addr, src_addr); + step_size = HPAGE_PMD_SIZE; + } else { + if (pmd_none(*src_pmd)) { + if (!(mode & UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES)) { + err = -ENOENT; + break; + } + if (unlikely(__pte_alloc(mm, src_pmd))) { + err = -ENOMEM; + break; + } + } + + if (unlikely(pte_alloc(mm, dst_pmd))) { + err = -ENOMEM; + break; + } + + err = move_pages_pte(mm, dst_pmd, src_pmd, + dst_vma, src_vma, + dst_addr, src_addr, mode); + step_size = PAGE_SIZE; + } + + cond_resched(); + + if (fatal_signal_pending(current)) { + /* Do not override an error */ + if (!err || err == -EAGAIN) + err = -EINTR; + break; + } + + if (err) { + if (err == -EAGAIN) + continue; + break; + } + + /* Proceed to the next page */ + dst_addr += step_size; + src_addr += step_size; + moved += step_size; + } + +out: + VM_WARN_ON(moved < 0); + VM_WARN_ON(err > 0); + VM_WARN_ON(!moved && !err); + return moved ? moved : err; +} From 60c5a0e02390868b67f3ddfe63285a132de5db5d Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Tue, 2 Jan 2024 15:32:56 -0800 Subject: [PATCH 20/86] UPSTREAM: userfaultfd: fix move_pages_pte() splitting folio under RCU read lock While testing the split PMD path with lockdep enabled I've got an "Invalid wait context" error caused by split_huge_page_to_list() trying to lock anon_vma->rwsem while inside RCU read section. The issues is due to move_pages_pte() calling split_folio() under RCU read lock. Fix this by unmapping the PTEs and exiting RCU read section before splitting the folio and then retrying. The same retry pattern is used when locking the folio or anon_vma in this function. After splitting the large folio we unlock and release it because after the split the old folio might not be the one that contains the src_addr. Link: https://lkml.kernel.org/r/20240102233256.1077959-1-surenb@google.com Fixes: adef440691ba ("userfaultfd: UFFDIO_MOVE uABI") Signed-off-by: Suren Baghdasaryan Reviewed-by: Peter Xu Cc: Al Viro Cc: Andrea Arcangeli Cc: Axel Rasmussen Cc: Brian Geffon Cc: Christian Brauner Cc: David Hildenbrand Cc: Hugh Dickins Cc: Jann Horn Cc: Kalesh Singh Cc: Liam R. Howlett Cc: Lokesh Gidra Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Mike Rapoport (IBM) Cc: Nicolas Geoffray Cc: Peter Xu Cc: Ryan Roberts Cc: Shuah Khan Cc: ZhangPeng Signed-off-by: Andrew Morton (cherry picked from commit 982ae058b2f08f576e4f3d4055f8916ba789f3d4) Bug: 274911254 Change-Id: I382c6631d821b0ed26d9b15afa78a417dafaeb2e Signed-off-by: Suren Baghdasaryan Signed-off-by: Lokesh Gidra --- mm/userfaultfd.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 9063874993d3..bb19455d36e2 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -1030,9 +1030,18 @@ retry: /* at this point we have src_folio locked */ if (folio_test_large(src_folio)) { + /* split_folio() can block */ + pte_unmap(&orig_src_pte); + pte_unmap(&orig_dst_pte); + src_pte = dst_pte = NULL; err = split_folio(src_folio); if (err) goto out; + /* have to reacquire the folio after it got split */ + folio_unlock(src_folio); + folio_put(src_folio); + src_folio = NULL; + goto retry; } if (!src_anon_vma) { From e275c2b743d0e943df95f181118f924cb3f97bb7 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Thu, 11 Jan 2024 17:39:35 -0800 Subject: [PATCH 21/86] UPSTREAM: userfaultfd: avoid huge_zero_page in UFFDIO_MOVE While testing UFFDIO_MOVE ioctl, syzbot triggered VM_BUG_ON_PAGE caused by a call to PageAnonExclusive() with a huge_zero_page as a parameter. UFFDIO_MOVE does not yet handle zeropages and returns EBUSY when one is encountered. Add an early huge_zero_page check in the PMD move path to avoid this situation. Link: https://lkml.kernel.org/r/20240112013935.1474648-1-surenb@google.com Fixes: adef440691ba ("userfaultfd: UFFDIO_MOVE uABI") Reported-by: syzbot+705209281e36404998f6@syzkaller.appspotmail.com Signed-off-by: Suren Baghdasaryan Acked-by: David Hildenbrand Cc: Andrea Arcangeli Cc: Peter Xu Cc: Stephen Rothwell Signed-off-by: Andrew Morton (cherry picked from commit 5d4747a6cc8e78ce74742d557fc9b7697fcacc95) Bug: 274911254 Change-Id: I7096b02b3a5b101e049608703ee77179d469a434 Signed-off-by: Lokesh Gidra --- mm/userfaultfd.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index bb19455d36e2..6e10b63990a9 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -1345,6 +1345,12 @@ ssize_t move_pages(struct userfaultfd_ctx *ctx, struct mm_struct *mm, err = -ENOENT; break; } + /* Avoid moving zeropages for now */ + if (is_huge_zero_pmd(*src_pmd)) { + spin_unlock(ptl); + err = -EBUSY; + break; + } /* Check if we can move the pmd without splitting it. */ if (move_splits_huge_pmd(dst_addr, src_addr, src_start + len) || From 6ecd08eaf4c438723f74ac7bedbb96356efffd66 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Wed, 31 Jan 2024 09:56:18 -0800 Subject: [PATCH 22/86] BACKPORT: userfaultfd: handle zeropage moves by UFFDIO_MOVE Current implementation of UFFDIO_MOVE fails to move zeropages and returns EBUSY when it encounters one. We can handle them by mapping a zeropage at the destination and clearing the mapping at the source. This is done both for ordinary and for huge zeropages. Link: https://lkml.kernel.org/r/20240131175618.2417291-1-surenb@google.com Signed-off-by: Suren Baghdasaryan Reported-by: kernel test robot Reported-by: Dan Carpenter Closes: https://lore.kernel.org/r/202401300107.U8iMAkTl-lkp@intel.com/ Cc: Alexander Viro Cc: Andrea Arcangeli Cc: Axel Rasmussen Cc: Brian Geffon Cc: Christian Brauner Cc: David Hildenbrand Cc: Hugh Dickins Cc: Jann Horn Cc: Kalesh Singh Cc: Liam R. Howlett Cc: Lokesh Gidra Cc: Matthew Wilcox Cc: Michal Hocko Cc: Mike Rapoport (IBM) Cc: Nicolas Geoffray Cc: Peter Xu Cc: Ryan Roberts Cc: Shuah Khan Cc: ZhangPeng Signed-off-by: Andrew Morton (cherry picked from commit eb1521dad8f391d3f3b88f589db27a288b55b8ed) Conflicts: mm/huge_memory.c 1. Replace folio_move_anon_rmap() with page_move_anon_rmap(). 2. Remove vma parameter in pmd_mkwrite() calls. Bug: 274911254 Change-Id: I271aa365bb3930e7e480d5749b44863eeca74dda Signed-off-by: Lokesh Gidra --- mm/huge_memory.c | 111 +++++++++++++++++++++++++++-------------------- mm/userfaultfd.c | 44 ++++++++++++++++--- 2 files changed, 101 insertions(+), 54 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 7a159e142c73..243b1390e9bd 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1957,13 +1957,18 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm } src_page = pmd_page(src_pmdval); - if (unlikely(!PageAnonExclusive(src_page))) { - spin_unlock(src_ptl); - return -EBUSY; - } - src_folio = page_folio(src_page); - folio_get(src_folio); + if (!is_huge_zero_pmd(src_pmdval)) { + if (unlikely(!PageAnonExclusive(src_page))) { + spin_unlock(src_ptl); + return -EBUSY; + } + + src_folio = page_folio(src_page); + folio_get(src_folio); + } else + src_folio = NULL; + spin_unlock(src_ptl); flush_cache_range(src_vma, src_addr, src_addr + HPAGE_PMD_SIZE); @@ -1971,19 +1976,22 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm src_addr + HPAGE_PMD_SIZE); mmu_notifier_invalidate_range_start(&range); - folio_lock(src_folio); + if (src_folio) { + folio_lock(src_folio); - /* - * split_huge_page walks the anon_vma chain without the page - * lock. Serialize against it with the anon_vma lock, the page - * lock is not enough. - */ - src_anon_vma = folio_get_anon_vma(src_folio); - if (!src_anon_vma) { - err = -EAGAIN; - goto unlock_folio; - } - anon_vma_lock_write(src_anon_vma); + /* + * split_huge_page walks the anon_vma chain without the page + * lock. Serialize against it with the anon_vma lock, the page + * lock is not enough. + */ + src_anon_vma = folio_get_anon_vma(src_folio); + if (!src_anon_vma) { + err = -EAGAIN; + goto unlock_folio; + } + anon_vma_lock_write(src_anon_vma); + } else + src_anon_vma = NULL; dst_ptl = pmd_lockptr(mm, dst_pmd); double_pt_lock(src_ptl, dst_ptl); @@ -1992,45 +2000,54 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm err = -EAGAIN; goto unlock_ptls; } - if (folio_maybe_dma_pinned(src_folio) || - !PageAnonExclusive(&src_folio->page)) { - err = -EBUSY; - goto unlock_ptls; + if (src_folio) { + if (folio_maybe_dma_pinned(src_folio) || + !PageAnonExclusive(&src_folio->page)) { + err = -EBUSY; + goto unlock_ptls; + } + + if (WARN_ON_ONCE(!folio_test_head(src_folio)) || + WARN_ON_ONCE(!folio_test_anon(src_folio))) { + err = -EBUSY; + goto unlock_ptls; + } + + page_move_anon_rmap(&src_folio->page, dst_vma); + WRITE_ONCE(src_folio->index, linear_page_index(dst_vma, dst_addr)); + + src_pmdval = pmdp_huge_clear_flush(src_vma, src_addr, src_pmd); + /* Folio got pinned from under us. Put it back and fail the move. */ + if (folio_maybe_dma_pinned(src_folio)) { + set_pmd_at(mm, src_addr, src_pmd, src_pmdval); + err = -EBUSY; + goto unlock_ptls; + } + + _dst_pmd = mk_huge_pmd(&src_folio->page, dst_vma->vm_page_prot); + /* Follow mremap() behavior and treat the entry dirty after the move */ + _dst_pmd = pmd_mkwrite(pmd_mkdirty(_dst_pmd)); + } else { + src_pmdval = pmdp_huge_clear_flush(src_vma, src_addr, src_pmd); + _dst_pmd = mk_huge_pmd(src_page, dst_vma->vm_page_prot); } - - if (WARN_ON_ONCE(!folio_test_head(src_folio)) || - WARN_ON_ONCE(!folio_test_anon(src_folio))) { - err = -EBUSY; - goto unlock_ptls; - } - - page_move_anon_rmap(&src_folio->page, dst_vma); - WRITE_ONCE(src_folio->index, linear_page_index(dst_vma, dst_addr)); - - src_pmdval = pmdp_huge_clear_flush(src_vma, src_addr, src_pmd); - /* Folio got pinned from under us. Put it back and fail the move. */ - if (folio_maybe_dma_pinned(src_folio)) { - set_pmd_at(mm, src_addr, src_pmd, src_pmdval); - err = -EBUSY; - goto unlock_ptls; - } - - _dst_pmd = mk_huge_pmd(&src_folio->page, dst_vma->vm_page_prot); - /* Follow mremap() behavior and treat the entry dirty after the move */ - _dst_pmd = pmd_mkwrite(pmd_mkdirty(_dst_pmd)); set_pmd_at(mm, dst_addr, dst_pmd, _dst_pmd); src_pgtable = pgtable_trans_huge_withdraw(mm, src_pmd); pgtable_trans_huge_deposit(mm, dst_pmd, src_pgtable); unlock_ptls: double_pt_unlock(src_ptl, dst_ptl); - anon_vma_unlock_write(src_anon_vma); - put_anon_vma(src_anon_vma); + if (src_anon_vma) { + anon_vma_unlock_write(src_anon_vma); + put_anon_vma(src_anon_vma); + } unlock_folio: /* unblock rmap walks */ - folio_unlock(src_folio); + if (src_folio) + folio_unlock(src_folio); mmu_notifier_invalidate_range_end(&range); - folio_put(src_folio); + if (src_folio) + folio_put(src_folio); return err; } #endif /* CONFIG_USERFAULTFD */ diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 6e10b63990a9..1f8e977e830e 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -897,6 +897,33 @@ static int move_swap_pte(struct mm_struct *mm, return 0; } +static int move_zeropage_pte(struct mm_struct *mm, + struct vm_area_struct *dst_vma, + struct vm_area_struct *src_vma, + unsigned long dst_addr, unsigned long src_addr, + pte_t *dst_pte, pte_t *src_pte, + pte_t orig_dst_pte, pte_t orig_src_pte, + spinlock_t *dst_ptl, spinlock_t *src_ptl) +{ + pte_t zero_pte; + + double_pt_lock(dst_ptl, src_ptl); + if (!pte_same(ptep_get(src_pte), orig_src_pte) || + !pte_same(ptep_get(dst_pte), orig_dst_pte)) { + double_pt_unlock(dst_ptl, src_ptl); + return -EAGAIN; + } + + zero_pte = pte_mkspecial(pfn_pte(my_zero_pfn(dst_addr), + dst_vma->vm_page_prot)); + ptep_clear_flush(src_vma, src_addr, src_pte); + set_pte_at(mm, dst_addr, dst_pte, zero_pte); + double_pt_unlock(dst_ptl, src_ptl); + + return 0; +} + + /* * The mmap_lock for reading is held by the caller. Just move the page * from src_pmd to dst_pmd if possible, and return true if succeeded @@ -982,6 +1009,14 @@ retry: } if (pte_present(orig_src_pte)) { + if (is_zero_pfn(pte_pfn(orig_src_pte))) { + err = move_zeropage_pte(mm, dst_vma, src_vma, + dst_addr, src_addr, dst_pte, src_pte, + orig_dst_pte, orig_src_pte, + dst_ptl, src_ptl); + goto out; + } + /* * Pin and lock both source folio and anon_vma. Since we are in * RCU read section, we can't block, so on contention have to @@ -1345,19 +1380,14 @@ ssize_t move_pages(struct userfaultfd_ctx *ctx, struct mm_struct *mm, err = -ENOENT; break; } - /* Avoid moving zeropages for now */ - if (is_huge_zero_pmd(*src_pmd)) { - spin_unlock(ptl); - err = -EBUSY; - break; - } /* Check if we can move the pmd without splitting it. */ if (move_splits_huge_pmd(dst_addr, src_addr, src_start + len) || !pmd_none(dst_pmdval)) { struct folio *folio = pfn_folio(pmd_pfn(*src_pmd)); - if (!folio || !PageAnonExclusive(&folio->page)) { + if (!folio || (!is_huge_zero_page(&folio->page) && + !PageAnonExclusive(&folio->page))) { spin_unlock(ptl); err = -EBUSY; break; From bfb4b24b642e2776dbfa0720fc57239386f9234e Mon Sep 17 00:00:00 2001 From: Qi Zheng Date: Thu, 22 Feb 2024 16:08:15 +0800 Subject: [PATCH 23/86] BACKPORT: mm: userfaultfd: fix unexpected change to src_folio when UFFDIO_MOVE fails After ptep_clear_flush(), if we find that src_folio is pinned we will fail UFFDIO_MOVE and put src_folio back to src_pte entry, but the change to src_folio->{mapping,index} is not restored in this process. This is not what we expected, so fix it. This can cause the rmap for that page to be invalid, possibly resulting in memory corruption. At least swapout+migration would no longer work, because we might fail to locate the mappings of that folio. Link: https://lkml.kernel.org/r/20240222080815.46291-1-zhengqi.arch@bytedance.com Fixes: adef440691ba ("userfaultfd: UFFDIO_MOVE uABI") Signed-off-by: Qi Zheng Reviewed-by: David Hildenbrand Reviewed-by: Suren Baghdasaryan Cc: Andrea Arcangeli Cc: Signed-off-by: Andrew Morton (cherry picked from commit d7a08838ab74652f2b53fee9763f0178278c3a4b) Conflicts: mm/userfaultfd.c 1. Replace folio_move_anon_rmap() with page_move_anon_rmap(). Bug: 274911254 Change-Id: Ie4bf5785244271ab233c6230ed71460fd571bd1a Signed-off-by: Lokesh Gidra --- mm/userfaultfd.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 1f8e977e830e..9149a95282e0 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -852,9 +852,6 @@ static int move_present_pte(struct mm_struct *mm, goto out; } - page_move_anon_rmap(&src_folio->page, dst_vma); - WRITE_ONCE(src_folio->index, linear_page_index(dst_vma, dst_addr)); - orig_src_pte = ptep_clear_flush(src_vma, src_addr, src_pte); /* Folio got pinned from under us. Put it back and fail the move. */ if (folio_maybe_dma_pinned(src_folio)) { @@ -863,6 +860,9 @@ static int move_present_pte(struct mm_struct *mm, goto out; } + page_move_anon_rmap(&src_folio->page, dst_vma); + WRITE_ONCE(src_folio->index, linear_page_index(dst_vma, dst_addr)); + orig_dst_pte = mk_pte(&src_folio->page, dst_vma->vm_page_prot); /* Follow mremap() behavior and treat the entry dirty after the move */ orig_dst_pte = pte_mkwrite(pte_mkdirty(orig_dst_pte)); From 4f658d77230e4d34734632f41af8267f93547dbb Mon Sep 17 00:00:00 2001 From: Lokesh Gidra Date: Thu, 4 Apr 2024 10:17:26 -0700 Subject: [PATCH 24/86] BACKPORT: userfaultfd: change src_folio after ensuring it's unpinned in UFFDIO_MOVE Commit d7a08838ab74 ("mm: userfaultfd: fix unexpected change to src_folio when UFFDIO_MOVE fails") moved the src_folio->{mapping, index} changing to after clearing the page-table and ensuring that it's not pinned. This avoids failure of swapout+migration and possibly memory corruption. However, the commit missed fixing it in the huge-page case. Link: https://lkml.kernel.org/r/20240404171726.2302435-1-lokeshgidra@google.com Fixes: adef440691ba ("userfaultfd: UFFDIO_MOVE uABI") Signed-off-by: Lokesh Gidra Acked-by: David Hildenbrand Cc: Andrea Arcangeli Cc: Kalesh Singh Cc: Lokesh Gidra Cc: Nicolas Geoffray Cc: Peter Xu Cc: Qi Zheng Cc: Matthew Wilcox Cc: Signed-off-by: Andrew Morton (cherry picked from commit c0205eaf3af9f5db14d4b5ee4abacf4a583c3c50) Conflicts: mm/huge_memory.c 1. Replace folio_move_anon_rmap() with page_move_anon_rmap(). Bug: 274911254 Change-Id: I15a07ea22de7ae38ed20320a73c995c7c48ef42b Signed-off-by: Lokesh Gidra --- mm/huge_memory.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 243b1390e9bd..1bc744166e9b 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2013,9 +2013,6 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm goto unlock_ptls; } - page_move_anon_rmap(&src_folio->page, dst_vma); - WRITE_ONCE(src_folio->index, linear_page_index(dst_vma, dst_addr)); - src_pmdval = pmdp_huge_clear_flush(src_vma, src_addr, src_pmd); /* Folio got pinned from under us. Put it back and fail the move. */ if (folio_maybe_dma_pinned(src_folio)) { @@ -2024,6 +2021,9 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm goto unlock_ptls; } + page_move_anon_rmap(&src_folio->page, dst_vma); + WRITE_ONCE(src_folio->index, linear_page_index(dst_vma, dst_addr)); + _dst_pmd = mk_huge_pmd(&src_folio->page, dst_vma->vm_page_prot); /* Follow mremap() behavior and treat the entry dirty after the move */ _dst_pmd = pmd_mkwrite(pmd_mkdirty(_dst_pmd)); From abd6748ba6aa8240a4b9072fdcbd5701ac542afe Mon Sep 17 00:00:00 2001 From: Lokesh Gidra Date: Wed, 17 Jan 2024 14:39:21 -0800 Subject: [PATCH 25/86] UPSTREAM: userfaultfd: fix return error if mmap_changing is non-zero in MOVE ioctl To be consistent with other uffd ioctl's returning EAGAIN when mmap_changing is detected, we should change UFFDIO_MOVE to do the same. Link: https://lkml.kernel.org/r/20240117223922.1445327-1-lokeshgidra@google.com Signed-off-by: Lokesh Gidra Acked-by: Suren Baghdasaryan Acked-by: Mike Rapoport (IBM) Cc: Andrea Arcangeli Cc: Axel Rasmussen Cc: Brian Geffon Cc: David Hildenbrand Cc: Jann Horn Cc: Kalesh Singh Cc: Matthew Wilcox (Oracle) Cc: Nicolas Geoffray Cc: Peter Xu Signed-off-by: Andrew Morton (cherry picked from commit 6ca03f1bb5a7427a66df62c954b3500a4255cdb9) Bug: 274911254 Change-Id: I3499e3987bf72d3d7a307165e0f9d1ed6d2b0611 Signed-off-by: Lokesh Gidra --- fs/userfaultfd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index e7ba93d22874..70c23e87943e 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -1987,7 +1987,7 @@ static int userfaultfd_move(struct userfaultfd_ctx *ctx, ret = move_pages(ctx, mm, uffdio_move.dst, uffdio_move.src, uffdio_move.len, uffdio_move.mode); else - ret = -EINVAL; + ret = -EAGAIN; mmap_read_unlock(mm); mmput(mm); From ee72d5a7d95e79ccde2109905370a584f7465a33 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Wed, 6 Dec 2023 02:36:57 -0800 Subject: [PATCH 26/86] BACKPORT: selftests/mm: call uffd_test_ctx_clear at the end of the test uffd_test_ctx_clear() is being called from uffd_test_ctx_init() to unmap areas used in the previous test run. This approach is problematic because while unmapping areas uffd_test_ctx_clear() uses page_size and nr_pages which might differ from one test run to another. Fix this by calling uffd_test_ctx_clear() after each test is done. Link: https://lkml.kernel.org/r/20231206103702.3873743-4-surenb@google.com Signed-off-by: Suren Baghdasaryan Reviewed-by: Peter Xu Reviewed-by: Axel Rasmussen Cc: Al Viro Cc: Andrea Arcangeli Cc: Brian Geffon Cc: Christian Brauner Cc: David Hildenbrand Cc: Hugh Dickins Cc: Jann Horn Cc: Kalesh Singh Cc: Liam R. Howlett Cc: Lokesh Gidra Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Mike Rapoport (IBM) Cc: Nicolas Geoffray Cc: Ryan Roberts Cc: Shuah Khan Cc: ZhangPeng Signed-off-by: Andrew Morton (cherry picked from commit 1c8d39fa7b63dcbb77af7b0325fdc519c35fe618) Conflicts: tools/testing/selftests/mm/uffd-common.c tools/testing/selftests/mm/uffd-common.h tools/testing/selftests/mm/uffd-stress.c tools/testing/selftests/mm/uffd-unit-tests.c tools/testing/selftests/vm/userfaultfd.c 1. Userfaultfd selftest was split into separate files and as a consequence the code moved from selftests/vm/userfaultfd.c to selftests/mm/uffd_* files. Bug: 274911254 Change-Id: Ic224c3965a645342dc0f41e743d3c072b7bb852e Signed-off-by: Lokesh Gidra --- tools/testing/selftests/vm/userfaultfd.c | 36 ++++++++++++++++++------ 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/tools/testing/selftests/vm/userfaultfd.c b/tools/testing/selftests/vm/userfaultfd.c index 297f250c1d95..b01dc5e4beb4 100644 --- a/tools/testing/selftests/vm/userfaultfd.c +++ b/tools/testing/selftests/vm/userfaultfd.c @@ -517,8 +517,6 @@ static void uffd_test_ctx_init(uint64_t features) { unsigned long nr, cpu; - uffd_test_ctx_clear(); - uffd_test_ops->allocate_area((void **)&area_src, true); uffd_test_ops->allocate_area((void **)&area_dst, false); @@ -1195,6 +1193,7 @@ static int userfaultfd_zeropage_test(void) if (my_bcmp(area_dst, zeropage, page_size)) err("zeropage is not zero"); + uffd_test_ctx_clear(); printf("done.\n"); return 0; } @@ -1207,6 +1206,7 @@ static int userfaultfd_events_test(void) pid_t pid; char c; struct uffd_stats stats = { 0 }; + int ret; printf("testing events (fork, remap, remove): "); fflush(stdout); @@ -1243,12 +1243,16 @@ static int userfaultfd_events_test(void) err("faulting process failed"); if (write(pipefd[1], &c, sizeof(c)) != sizeof(c)) err("pipe write"); + ret = 1; if (pthread_join(uffd_mon, NULL)) - return 1; + goto out; uffd_stats_report(&stats, 1); - return stats.missing_faults != nr_pages; + ret = stats.missing_faults != nr_pages; +out: + uffd_test_ctx_clear(); + return ret; } static int userfaultfd_sig_test(void) @@ -1260,6 +1264,7 @@ static int userfaultfd_sig_test(void) pid_t pid; char c; struct uffd_stats stats = { 0 }; + int ret; printf("testing signal delivery: "); fflush(stdout); @@ -1300,14 +1305,18 @@ static int userfaultfd_sig_test(void) err("faulting process failed"); if (write(pipefd[1], &c, sizeof(c)) != sizeof(c)) err("pipe write"); + ret = 1; if (pthread_join(uffd_mon, (void **)&userfaults)) - return 1; + goto out; printf("done.\n"); if (userfaults) err("Signal test failed, userfaults: %ld", userfaults); - return userfaults != 0; + ret = userfaults != 0; +out: + uffd_test_ctx_clear(); + return ret; } void check_memory_contents(char *p) @@ -1336,6 +1345,7 @@ static int userfaultfd_minor_test(void) pthread_t uffd_mon; char c; struct uffd_stats stats = { 0 }; + int ret; if (!test_uffdio_minor) return 0; @@ -1376,8 +1386,9 @@ static int userfaultfd_minor_test(void) if (write(pipefd[1], &c, sizeof(c)) != sizeof(c)) err("pipe write"); + ret = 1; if (pthread_join(uffd_mon, NULL)) - return 1; + goto out; uffd_stats_report(&stats, 1); @@ -1398,7 +1409,10 @@ static int userfaultfd_minor_test(void) printf(" done.\n"); } - return stats.missing_faults != 0 || stats.minor_faults != nr_pages; + ret = stats.missing_faults != 0 || stats.minor_faults != nr_pages; +out: + uffd_test_ctx_clear(); + return ret; } #define BIT_ULL(nr) (1ULL << (nr)) @@ -1532,6 +1546,7 @@ static void userfaultfd_pagemap_test(unsigned int test_pgsize) pagemap_check_wp(value, false); close(pagemap_fd); + uffd_test_ctx_clear(); printf("done\n"); } @@ -1621,8 +1636,10 @@ static int userfaultfd_stress(void) uffd_stats_reset(uffd_stats, nr_cpus); /* bounce pass */ - if (stress(uffd_stats)) + if (stress(uffd_stats)) { + uffd_test_ctx_clear(); return 1; + } /* Clear all the write protections if there is any */ if (test_uffdio_wp) @@ -1654,6 +1671,7 @@ static int userfaultfd_stress(void) uffd_stats_report(uffd_stats, nr_cpus); } + uffd_test_ctx_clear(); if (test_type == TEST_ANON) { /* From a5d504c067b3c0a25c57db65fa9c330e12385a4a Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Wed, 6 Dec 2023 02:36:58 -0800 Subject: [PATCH 27/86] BACKPORT: selftests/mm: add uffd_test_case_ops to allow test case-specific operations Currently each test can specify unique operations using uffd_test_ops, however these operations are per-memory type and not per-test. Add uffd_test_case_ops which each test case can customize for its own needs regardless of the memory type being used. Pre- and post-allocation operations are added, some of which will be used in the next patch to implement test-specific operations like madvise after memory is allocated but before it is accessed. Link: https://lkml.kernel.org/r/20231206103702.3873743-5-surenb@google.com Signed-off-by: Suren Baghdasaryan Cc: Al Viro Cc: Andrea Arcangeli Cc: Axel Rasmussen Cc: Brian Geffon Cc: Christian Brauner Cc: David Hildenbrand Cc: Hugh Dickins Cc: Jann Horn Cc: Kalesh Singh Cc: Liam R. Howlett Cc: Lokesh Gidra Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Mike Rapoport (IBM) Cc: Nicolas Geoffray Cc: Peter Xu Cc: Ryan Roberts Cc: Shuah Khan Cc: ZhangPeng Signed-off-by: Andrew Morton (cherry picked from commit e8a422408ba9760e2640ca57e4b79c3dd7f48bd2) Conflicts: tools/testing/selftests/mm/uffd-common.c tools/testing/selftests/mm/uffd-common.h tools/testing/selftests/mm/uffd-unit-tests.c tools/testing/selftests/vm/userfaultfd.c 1. Userfaultfd selftest was split into separate uffd-* files and moved to selftests/mm. 2. In 6.1 there is no mechanism to run individual unit-tests. All unit-tests are run after stress test. Consequently, the tests are not abstracted using 'uffd_test_case_t'. Therefore, added 'uffd_test_case_ops' as a parameter to uffd_test_ctx_init(). Bug: 274911254 Change-Id: I6480abf1709ca717d9baad5047bf675852f10726 Signed-off-by: Lokesh Gidra --- tools/testing/selftests/vm/userfaultfd.c | 26 +++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/tools/testing/selftests/vm/userfaultfd.c b/tools/testing/selftests/vm/userfaultfd.c index b01dc5e4beb4..f416413a219b 100644 --- a/tools/testing/selftests/vm/userfaultfd.c +++ b/tools/testing/selftests/vm/userfaultfd.c @@ -365,6 +365,11 @@ static void shmem_check_pmd_mapping(void *p, int expect_nr_hpages) expect_nr_hpages); } +struct uffd_test_case_ops { + void (*pre_alloc)(); + void (*post_alloc)(); +}; + struct uffd_test_ops { void (*allocate_area)(void **alloc_area, bool is_src); void (*release_pages)(char *rel_area); @@ -513,13 +518,20 @@ static void uffd_test_ctx_clear(void) munmap_area((void **)&area_remap); } -static void uffd_test_ctx_init(uint64_t features) +static void uffd_test_ctx_init(uint64_t features, + struct uffd_test_case_ops *uffd_test_case_ops) { unsigned long nr, cpu; + if (uffd_test_case_ops && uffd_test_case_ops->pre_alloc) + uffd_test_case_ops->pre_alloc(); + uffd_test_ops->allocate_area((void **)&area_src, true); uffd_test_ops->allocate_area((void **)&area_dst, false); + if (uffd_test_case_ops && uffd_test_case_ops->post_alloc) + uffd_test_case_ops->post_alloc(); + userfaultfd_open(&features); count_verify = malloc(nr_pages * sizeof(unsigned long long)); @@ -1176,7 +1188,7 @@ static int userfaultfd_zeropage_test(void) printf("testing UFFDIO_ZEROPAGE: "); fflush(stdout); - uffd_test_ctx_init(0); + uffd_test_ctx_init(0, NULL); uffdio_register.range.start = (unsigned long) area_dst; uffdio_register.range.len = nr_pages * page_size; @@ -1213,7 +1225,7 @@ static int userfaultfd_events_test(void) features = UFFD_FEATURE_EVENT_FORK | UFFD_FEATURE_EVENT_REMAP | UFFD_FEATURE_EVENT_REMOVE; - uffd_test_ctx_init(features); + uffd_test_ctx_init(features, NULL); fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK); @@ -1270,7 +1282,7 @@ static int userfaultfd_sig_test(void) fflush(stdout); features = UFFD_FEATURE_EVENT_FORK|UFFD_FEATURE_SIGBUS; - uffd_test_ctx_init(features); + uffd_test_ctx_init(features, NULL); fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK); @@ -1353,7 +1365,7 @@ static int userfaultfd_minor_test(void) printf("testing minor faults: "); fflush(stdout); - uffd_test_ctx_init(uffd_minor_feature()); + uffd_test_ctx_init(uffd_minor_feature(), NULL); uffdio_register.range.start = (unsigned long)area_dst_alias; uffdio_register.range.len = nr_pages * page_size; @@ -1492,7 +1504,7 @@ static void userfaultfd_pagemap_test(unsigned int test_pgsize) /* Flush so it doesn't flush twice in parent/child later */ fflush(stdout); - uffd_test_ctx_init(0); + uffd_test_ctx_init(0, NULL); if (test_pgsize > page_size) { /* This is a thp test */ @@ -1557,7 +1569,7 @@ static int userfaultfd_stress(void) struct uffdio_register uffdio_register; struct uffd_stats uffd_stats[nr_cpus]; - uffd_test_ctx_init(0); + uffd_test_ctx_init(0, NULL); if (posix_memalign(&area, page_size, page_size)) err("out of memory"); From f15269151532fdeb5ea43be483a3e97531399f47 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Wed, 6 Dec 2023 02:36:59 -0800 Subject: [PATCH 28/86] BACKPORT: selftests/mm: add UFFDIO_MOVE ioctl test Add tests for new UFFDIO_MOVE ioctl which uses uffd to move source into destination buffer while checking the contents of both after the move. After the operation the content of the destination buffer should match the original source buffer's content while the source buffer should be zeroed. Separate tests are designed for PMD aligned and unaligned cases because they utilize different code paths in the kernel. Link: https://lkml.kernel.org/r/20231206103702.3873743-6-surenb@google.com Signed-off-by: Suren Baghdasaryan Cc: Al Viro Cc: Andrea Arcangeli Cc: Axel Rasmussen Cc: Brian Geffon Cc: Christian Brauner Cc: David Hildenbrand Cc: Hugh Dickins Cc: Jann Horn Cc: Kalesh Singh Cc: Liam R. Howlett Cc: Lokesh Gidra Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Mike Rapoport (IBM) Cc: Nicolas Geoffray Cc: Peter Xu Cc: Ryan Roberts Cc: Shuah Khan Cc: ZhangPeng Signed-off-by: Andrew Morton (cherry picked from commit a2bf6a9ca80532b75f8f8b6a1cd75ef7e5150576) Conflicts: tools/testing/selftests/mm/uffd-common.c tools/testing/selftests/mm/uffd-common.h tools/testing/selftests/mm/uffd-unit-tests.c tools/testing/selftests/vm/userfaultfd.c 1. Removed errmsg parameter from prevent_hugepages() and post_hugepages() 2. Removed uffd_test_args parameter from uffd_move_* functions 3. Added uffd_test_case_ops parameter in uffd_move_test_common() 4. Added userfaultfd_move_test() for all 'move' tests, called from userfaultfd_stress() 5. Added 'test_uffdio_move' global bool variable, which is set to true only when testing anon mappings 6. Added call to uffd_test_ctx_init() and uffd_test_ctx_clear() in uffd_move_test_common() 7. Replaced uffd_args with uffd_stats 8. Converted return type of uffd_move_test() and uffd_move_pmd_test() to `int` 9. Added uffd_test_page_fault_handler as uffd_args doesn't exist. uffd_poll_thread() checks if it is NULL then calls uffd_handle_page_fault(). 10. Replaced uffd_register() (isn't defined on 6.1) with UFFDIO_REGISTER ioctl call 11. Added printf() calls to log when the test is starting and finishing. 12. Change return type of uffd_move_test_common() to int. Bug: 274911254 Change-Id: I1c68445d9c64533aab0ba27c2e010347d0807981 Signed-off-by: Lokesh Gidra --- tools/testing/selftests/vm/userfaultfd.c | 232 ++++++++++++++++++++++- 1 file changed, 230 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/vm/userfaultfd.c b/tools/testing/selftests/vm/userfaultfd.c index f416413a219b..0e200472d82c 100644 --- a/tools/testing/selftests/vm/userfaultfd.c +++ b/tools/testing/selftests/vm/userfaultfd.c @@ -65,6 +65,9 @@ #ifdef __NR_userfaultfd +#define ALIGN_UP(x, align_to) \ + ((__typeof__(x))((((unsigned long)(x)) + ((align_to)-1)) & ~((align_to)-1))) + static unsigned long nr_cpus, nr_pages, nr_pages_per_cpu, page_size, hpage_size; #define BOUNCE_RANDOM (1<<0) @@ -93,6 +96,8 @@ static volatile bool test_uffdio_zeropage_eexist = true; static bool test_uffdio_wp = true; /* Whether to test uffd minor faults */ static bool test_uffdio_minor = false; +/* Whether to test uffd move ioctl */ +static bool test_uffdio_move = false; static bool map_shared; static int shm_fd; @@ -113,6 +118,8 @@ struct uffd_stats { unsigned long minor_faults; }; +static void (*uffd_test_page_fault_handler)(struct uffd_msg *msg, struct uffd_stats *args); + /* pthread_mutex_t starts at page offset 0 */ #define area_mutex(___area, ___nr) \ ((pthread_mutex_t *) ((___area) + (___nr)*page_size)) @@ -731,6 +738,30 @@ static int copy_page(int ufd, unsigned long offset) return __copy_page(ufd, offset, false); } +static int move_page(int ufd, unsigned long offset, unsigned long len) +{ + struct uffdio_move uffdio_move; + + if (offset + len > nr_pages * page_size) + err("unexpected offset %lu and length %lu\n", offset, len); + uffdio_move.dst = (unsigned long) area_dst + offset; + uffdio_move.src = (unsigned long) area_src + offset; + uffdio_move.len = len; + uffdio_move.mode = UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES; + uffdio_move.move = 0; + if (ioctl(ufd, UFFDIO_MOVE, &uffdio_move)) { + /* real retval in uffdio_move.move */ + if (uffdio_move.move != -EEXIST) + err("UFFDIO_MOVE error: %"PRId64, + (int64_t)uffdio_move.move); + wake_range(ufd, uffdio_move.dst, len); + } else if (uffdio_move.move != len) { + err("UFFDIO_MOVE error: %"PRId64, (int64_t)uffdio_move.move); + } else + return 1; + return 0; +} + static int uffd_read_msg(int ufd, struct uffd_msg *msg) { int ret = read(uffd, msg, sizeof(*msg)); @@ -852,7 +883,10 @@ static void *uffd_poll_thread(void *arg) err("unexpected msg event %u\n", msg.event); break; case UFFD_EVENT_PAGEFAULT: - uffd_handle_page_fault(&msg, stats); + if (uffd_test_page_fault_handler) + uffd_test_page_fault_handler(&msg, stats); + else + uffd_handle_page_fault(&msg, stats); break; case UFFD_EVENT_FORK: close(uffd); @@ -1210,6 +1244,198 @@ static int userfaultfd_zeropage_test(void) return 0; } +static void prevent_hugepages() +{ + /* This should be done before source area is populated */ + if (madvise(area_src, nr_pages * page_size, MADV_NOHUGEPAGE)) { + /* Ignore only if CONFIG_TRANSPARENT_HUGEPAGE=n */ + if (errno != EINVAL) { + err("madvise(MADV_NOHUGEPAGE) failed"); + } + } +} + +static void request_hugepages() +{ + /* This should be done before source area is populated */ + if (madvise(area_src, nr_pages * page_size, MADV_HUGEPAGE)) { + if (errno == EINVAL) + err("CONFIG_TRANSPARENT_HUGEPAGE is not set"); + else + err("madvise(MADV_HUGEPAGE) failed"); + } +} + +struct uffd_test_case_ops uffd_move_test_case_ops = { + .post_alloc = prevent_hugepages, +}; + +struct uffd_test_case_ops uffd_move_test_pmd_case_ops = { + .post_alloc = request_hugepages, +}; + +static void +uffd_move_handle_fault_common(struct uffd_msg *msg, struct uffd_stats *args, + unsigned long len) +{ + unsigned long offset; + + if (msg->event != UFFD_EVENT_PAGEFAULT) + err("unexpected msg event %u", msg->event); + + if (msg->arg.pagefault.flags & + (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR | UFFD_PAGEFAULT_FLAG_WRITE)) + err("unexpected fault type %llu", msg->arg.pagefault.flags); + + offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst; + offset &= ~(len-1); + + if (move_page(uffd, offset, len)) + args->missing_faults++; +} + +static void uffd_move_handle_fault(struct uffd_msg *msg, + struct uffd_stats *args) +{ + uffd_move_handle_fault_common(msg, args, page_size); +} + +static void uffd_move_pmd_handle_fault(struct uffd_msg *msg, + struct uffd_stats *args) +{ + uffd_move_handle_fault_common(msg, args, read_pmd_pagesize()); +} + +static int +uffd_move_test_common(struct uffd_test_case_ops *uffd_test_case_ops, + unsigned long chunk_size, + void (*handle_fault)(struct uffd_msg *msg, struct uffd_stats *args)) +{ + struct uffdio_register uffdio_register; + unsigned long nr; + pthread_t uffd_mon; + char c; + unsigned long long count; + struct uffd_stats args = { 0 }; + char *orig_area_src, *orig_area_dst; + unsigned long step_size, step_count; + unsigned long src_offs = 0; + unsigned long dst_offs = 0; + + uffd_test_ctx_init(UFFD_FEATURE_MOVE, uffd_test_case_ops); + + /* Prevent source pages from being mapped more than once */ + if (madvise(area_src, nr_pages * page_size, MADV_DONTFORK)) + err("madvise(MADV_DONTFORK) failure"); + + uffdio_register.range.start = (unsigned long) area_dst; + uffdio_register.range.len = nr_pages * page_size; + uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING; + if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register)) + err("register failure"); + + uffd_test_page_fault_handler = handle_fault; + if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args)) + err("uffd_poll_thread create"); + + step_size = chunk_size / page_size; + step_count = nr_pages / step_size; + + if (chunk_size > page_size) { + char *aligned_src = ALIGN_UP(area_src, chunk_size); + char *aligned_dst = ALIGN_UP(area_dst, chunk_size); + + if (aligned_src != area_src || aligned_dst != area_dst) { + src_offs = (aligned_src - area_src) / page_size; + dst_offs = (aligned_dst - area_dst) / page_size; + step_count--; + } + orig_area_src = area_src; + orig_area_dst = area_dst; + area_src = aligned_src; + area_dst = aligned_dst; + } + + /* + * Read each of the pages back using the UFFD-registered mapping. We + * expect that the first time we touch a page, it will result in a missing + * fault. uffd_poll_thread will resolve the fault by moving source + * page to destination. + */ + for (nr = 0; nr < step_count * step_size; nr += step_size) { + unsigned long i; + + /* Check area_src content */ + for (i = 0; i < step_size; i++) { + count = *area_count(area_src, nr + i); + if (count != count_verify[src_offs + nr + i]) + err("nr %lu source memory invalid %llu %llu\n", + nr + i, count, count_verify[src_offs + nr + i]); + } + + /* Faulting into area_dst should move the page or the huge page */ + for (i = 0; i < step_size; i++) { + count = *area_count(area_dst, nr + i); + if (count != count_verify[dst_offs + nr + i]) + err("nr %lu memory corruption %llu %llu\n", + nr, count, count_verify[dst_offs + nr + i]); + } + + /* Re-check area_src content which should be empty */ + for (i = 0; i < step_size; i++) { + count = *area_count(area_src, nr + i); + if (count != 0) + err("nr %lu move failed %llu %llu\n", + nr, count, count_verify[src_offs + nr + i]); + } + } + if (step_size > page_size) { + area_src = orig_area_src; + area_dst = orig_area_dst; + } + + if (write(pipefd[1], &c, sizeof(c)) != sizeof(c)) + err("pipe write"); + if (pthread_join(uffd_mon, NULL)) + err("join() failed"); + + uffd_test_page_fault_handler = NULL; + uffd_test_ctx_clear(); + + return args.missing_faults != step_count || args.minor_faults != 0; +} + +static int uffd_move_test(void) +{ + printf("move "); + return uffd_move_test_common(&uffd_move_test_case_ops, page_size, + uffd_move_handle_fault); +} + +static int uffd_move_pmd_test(void) +{ + printf("move-pmd "); + return uffd_move_test_common(&uffd_move_test_pmd_case_ops, + read_pmd_pagesize(), + uffd_move_pmd_handle_fault); +} + +static int userfaultfd_move_test(void) +{ + int ret; + + if (!test_uffdio_move) + return 0; + + printf("testing UFFDIO_MOVE: "); + fflush(stdout); + + ret = uffd_move_test() || uffd_move_pmd_test(); + + printf("done.\n"); + return ret; +} + static int userfaultfd_events_test(void) { struct uffdio_register uffdio_register; @@ -1701,7 +1927,8 @@ static int userfaultfd_stress(void) } return userfaultfd_zeropage_test() || userfaultfd_sig_test() - || userfaultfd_events_test() || userfaultfd_minor_test(); + || userfaultfd_events_test() || userfaultfd_minor_test() + || userfaultfd_move_test(); } /* @@ -1733,6 +1960,7 @@ static void set_test_type(const char *type) if (!strcmp(type, "anon")) { test_type = TEST_ANON; uffd_test_ops = &anon_uffd_test_ops; + test_uffdio_move = true; } else if (!strcmp(type, "hugetlb")) { test_type = TEST_HUGETLB; uffd_test_ops = &hugetlb_uffd_test_ops; From 51eab7ecc4e9ef7628b098ed69ad63aa4c4b6e1b Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Fri, 29 Dec 2023 18:56:36 -0800 Subject: [PATCH 29/86] BACKPORT: selftests/mm: add separate UFFDIO_MOVE test for PMD splitting Add a test for UFFDIO_MOVE ioctl operating on a hugepage which has to be split because destination is marked with MADV_NOHUGEPAGE. With this we cover all 3 cases: normal page move, hugepage move, hugepage splitting before move. Link: https://lkml.kernel.org/r/20231230025636.2477429-1-surenb@google.com Signed-off-by: Suren Baghdasaryan Cc: Al Viro Cc: Andrea Arcangeli Cc: Axel Rasmussen Cc: Brian Geffon Cc: Christian Brauner Cc: David Hildenbrand Cc: Hugh Dickins Cc: Jann Horn Cc: Kalesh Singh Cc: Liam R. Howlett Cc: Lokesh Gidra Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Mike Rapoport (IBM) Cc: Nicolas Geoffray Cc: Peter Xu Cc: Ryan Roberts Cc: Shuah Khan Cc: ZhangPeng Signed-off-by: Andrew Morton (cherry picked from commit a5b7620bab81f16e8bbb04f4aea94c4c7feb0d77) Conflicts: tools/testing/selftests/mm/uffd-unit-tests.c tools/testing/selftests/vm/userfaultfd.c 1. Add request_src_hugepages() to enable THP on src 2. Add madvise() to enable THP on dst in request_hugepages() 3. Add request_split_hugepages() to enable THP on src and disable on dst 4. Change return type of uffd_move_pmd_split_test() to int Bug: 274911254 Change-Id: I21147a5b7f3e8bbe2befa8bff536e62826e9f6e3 Signed-off-by: Lokesh Gidra --- tools/testing/selftests/vm/userfaultfd.c | 31 ++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/vm/userfaultfd.c b/tools/testing/selftests/vm/userfaultfd.c index 0e200472d82c..4f3f0f7fe55e 100644 --- a/tools/testing/selftests/vm/userfaultfd.c +++ b/tools/testing/selftests/vm/userfaultfd.c @@ -1255,7 +1255,7 @@ static void prevent_hugepages() } } -static void request_hugepages() +static void request_src_hugepages() { /* This should be done before source area is populated */ if (madvise(area_src, nr_pages * page_size, MADV_HUGEPAGE)) { @@ -1266,6 +1266,20 @@ static void request_hugepages() } } +static void request_hugepages() +{ + request_src_hugepages(); + if (madvise(area_dst, nr_pages * page_size, MADV_HUGEPAGE)) + err("madvise(MADV_HUGEPAGE) failure"); +} + +static void request_split_hugepages() +{ + request_src_hugepages(); + if (madvise(area_dst, nr_pages * page_size, MADV_NOHUGEPAGE)) + err("madvise(MADV_NOHUGEPAGE) failure"); +} + struct uffd_test_case_ops uffd_move_test_case_ops = { .post_alloc = prevent_hugepages, }; @@ -1274,6 +1288,10 @@ struct uffd_test_case_ops uffd_move_test_pmd_case_ops = { .post_alloc = request_hugepages, }; +struct uffd_test_case_ops uffd_move_test_pmd_split_case_ops = { + .post_alloc = request_split_hugepages, +}; + static void uffd_move_handle_fault_common(struct uffd_msg *msg, struct uffd_stats *args, unsigned long len) @@ -1420,6 +1438,14 @@ static int uffd_move_pmd_test(void) uffd_move_pmd_handle_fault); } +static int uffd_move_pmd_split_test(void) +{ + printf("move-pmd-split "); + return uffd_move_test_common(&uffd_move_test_pmd_split_case_ops, + read_pmd_pagesize(), + uffd_move_pmd_handle_fault); +} + static int userfaultfd_move_test(void) { int ret; @@ -1430,7 +1456,8 @@ static int userfaultfd_move_test(void) printf("testing UFFDIO_MOVE: "); fflush(stdout); - ret = uffd_move_test() || uffd_move_pmd_test(); + ret = uffd_move_test() || uffd_move_pmd_test() + || uffd_move_pmd_split_test(); printf("done.\n"); return ret; From ac96edb501b6313955813ec4b4ae475867cf3751 Mon Sep 17 00:00:00 2001 From: Lokesh Gidra Date: Wed, 17 Jan 2024 14:37:29 -0800 Subject: [PATCH 30/86] BACKPORT: userfaultfd: fix mmap_changing checking in mfill_atomic_hugetlb In mfill_atomic_hugetlb(), mmap_changing isn't being checked again if we drop mmap_lock and reacquire it. When the lock is not held, mmap_changing could have been incremented. This is also inconsistent with the behavior in mfill_atomic(). Link: https://lkml.kernel.org/r/20240117223729.1444522-1-lokeshgidra@google.com Fixes: df2cc96e77011 ("userfaultfd: prevent non-cooperative events vs mcopy_atomic races") Signed-off-by: Lokesh Gidra Cc: Andrea Arcangeli Cc: Mike Rapoport Cc: Axel Rasmussen Cc: Brian Geffon Cc: David Hildenbrand Cc: Jann Horn Cc: Kalesh Singh Cc: Matthew Wilcox (Oracle) Cc: Nicolas Geoffray Cc: Peter Xu Cc: Suren Baghdasaryan Cc: Signed-off-by: Andrew Morton (cherry picked from commit 67695f18d55924b2013534ef3bdc363bc9e14605) Conflicts: mm/userfaultfd.c 1. Update mfill_atomic_hugetlb() parameters to pass 'wp_copy' and 'mode' instead of 'flags'. Bug: 320478828 Change-Id: I11ef09b2b8e477c32cc731205fd48b25bcbd020f Signed-off-by: Lokesh Gidra --- mm/userfaultfd.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 9149a95282e0..af165216fbbd 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -328,7 +328,8 @@ static __always_inline ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm, unsigned long src_start, unsigned long len, enum mcopy_atomic_mode mode, - bool wp_copy) + bool wp_copy, + atomic_t *mmap_changing) { int vm_shared = dst_vma->vm_flags & VM_SHARED; ssize_t err; @@ -445,6 +446,15 @@ retry: goto out; } mmap_read_lock(dst_mm); + /* + * If memory mappings are changing because of non-cooperative + * operation (e.g. mremap) running in parallel, bail out and + * request the user to retry later + */ + if (mmap_changing && atomic_read(mmap_changing)) { + err = -EAGAIN; + break; + } dst_vma = NULL; goto retry; @@ -481,7 +491,8 @@ extern ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm, unsigned long src_start, unsigned long len, enum mcopy_atomic_mode mode, - bool wp_copy); + bool wp_copy, + atomic_t *mmap_changing); #endif /* CONFIG_HUGETLB_PAGE */ static __always_inline ssize_t mfill_atomic_pte(struct mm_struct *dst_mm, @@ -602,7 +613,7 @@ retry: if (is_vm_hugetlb_page(dst_vma)) return __mcopy_atomic_hugetlb(dst_mm, dst_vma, dst_start, src_start, len, mcopy_mode, - wp_copy); + wp_copy, mmap_changing); if (!vma_is_anonymous(dst_vma) && !vma_is_shmem(dst_vma)) goto out_unlock; From 6b5ee039a1efad553335d81af6f5083b0c9c6ad7 Mon Sep 17 00:00:00 2001 From: Lokesh Gidra Date: Thu, 15 Feb 2024 10:27:53 -0800 Subject: [PATCH 31/86] BACKPORT: userfaultfd: move userfaultfd_ctx struct to header file Patch series "per-vma locks in userfaultfd", v7. Performing userfaultfd operations (like copy/move etc.) in critical section of mmap_lock (read-mode) causes significant contention on the lock when operations requiring the lock in write-mode are taking place concurrently. We can use per-vma locks instead to significantly reduce the contention issue. Android runtime's Garbage Collector uses userfaultfd for concurrent compaction. mmap-lock contention during compaction potentially causes jittery experience for the user. During one such reproducible scenario, we observed the following improvements with this patch-set: - Wall clock time of compaction phase came down from ~3s to <500ms - Uninterruptible sleep time (across all threads in the process) was ~10ms (none in mmap_lock) during compaction, instead of >20s This patch (of 4): Move the struct to userfaultfd_k.h to be accessible from mm/userfaultfd.c. There are no other changes in the struct. This is required to prepare for using per-vma locks in userfaultfd operations. Link: https://lkml.kernel.org/r/20240215182756.3448972-1-lokeshgidra@google.com Link: https://lkml.kernel.org/r/20240215182756.3448972-2-lokeshgidra@google.com Signed-off-by: Lokesh Gidra Reviewed-by: Mike Rapoport (IBM) Reviewed-by: Liam R. Howlett Cc: Andrea Arcangeli Cc: Axel Rasmussen Cc: Brian Geffon Cc: David Hildenbrand Cc: Jann Horn Cc: Kalesh Singh Cc: Matthew Wilcox (Oracle) Cc: Nicolas Geoffray Cc: Peter Xu Cc: Ryan Roberts Cc: Suren Baghdasaryan Cc: Tim Murray Signed-off-by: Andrew Morton (cherry picked from commit f91e6b41dd11daffb138e3afdb4804aefc3d4e1b) Conflicts: include/linux/userfaultfd_k.h 1. Retain 'sysctl_unprivileged_userfaultfd' global variable. Bug: 320478828 Change-Id: Iebaae028d5e793dd50342b141c1d46b79026834a Signed-off-by: Lokesh Gidra --- fs/userfaultfd.c | 39 ----------------------------------- include/linux/userfaultfd_k.h | 39 +++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 70c23e87943e..e2fc9491048d 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -36,45 +36,6 @@ int sysctl_unprivileged_userfaultfd __read_mostly; static struct kmem_cache *userfaultfd_ctx_cachep __read_mostly; -/* - * Start with fault_pending_wqh and fault_wqh so they're more likely - * to be in the same cacheline. - * - * Locking order: - * fd_wqh.lock - * fault_pending_wqh.lock - * fault_wqh.lock - * event_wqh.lock - * - * To avoid deadlocks, IRQs must be disabled when taking any of the above locks, - * since fd_wqh.lock is taken by aio_poll() while it's holding a lock that's - * also taken in IRQ context. - */ -struct userfaultfd_ctx { - /* waitqueue head for the pending (i.e. not read) userfaults */ - wait_queue_head_t fault_pending_wqh; - /* waitqueue head for the userfaults */ - wait_queue_head_t fault_wqh; - /* waitqueue head for the pseudo fd to wakeup poll/read */ - wait_queue_head_t fd_wqh; - /* waitqueue head for events */ - wait_queue_head_t event_wqh; - /* a refile sequence protected by fault_pending_wqh lock */ - seqcount_spinlock_t refile_seq; - /* pseudo fd refcounting */ - refcount_t refcount; - /* userfaultfd syscall flags */ - unsigned int flags; - /* features requested from the userspace */ - unsigned int features; - /* released */ - bool released; - /* memory mappings are changing because of non-cooperative event */ - atomic_t mmap_changing; - /* mm with one ore more vmas attached to this userfaultfd_ctx */ - struct mm_struct *mm; -}; - struct userfaultfd_fork_ctx { struct userfaultfd_ctx *orig; struct userfaultfd_ctx *new; diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index 29c0f4fd1511..7bbd671e912f 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -36,6 +36,45 @@ #define UFFD_SHARED_FCNTL_FLAGS (O_CLOEXEC | O_NONBLOCK) #define UFFD_FLAGS_SET (EFD_SHARED_FCNTL_FLAGS) +/* + * Start with fault_pending_wqh and fault_wqh so they're more likely + * to be in the same cacheline. + * + * Locking order: + * fd_wqh.lock + * fault_pending_wqh.lock + * fault_wqh.lock + * event_wqh.lock + * + * To avoid deadlocks, IRQs must be disabled when taking any of the above locks, + * since fd_wqh.lock is taken by aio_poll() while it's holding a lock that's + * also taken in IRQ context. + */ +struct userfaultfd_ctx { + /* waitqueue head for the pending (i.e. not read) userfaults */ + wait_queue_head_t fault_pending_wqh; + /* waitqueue head for the userfaults */ + wait_queue_head_t fault_wqh; + /* waitqueue head for the pseudo fd to wakeup poll/read */ + wait_queue_head_t fd_wqh; + /* waitqueue head for events */ + wait_queue_head_t event_wqh; + /* a refile sequence protected by fault_pending_wqh lock */ + seqcount_spinlock_t refile_seq; + /* pseudo fd refcounting */ + refcount_t refcount; + /* userfaultfd syscall flags */ + unsigned int flags; + /* features requested from the userspace */ + unsigned int features; + /* released */ + bool released; + /* memory mappings are changing because of non-cooperative event */ + atomic_t mmap_changing; + /* mm with one ore more vmas attached to this userfaultfd_ctx */ + struct mm_struct *mm; +}; + extern int sysctl_unprivileged_userfaultfd; extern vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason); From a5b6040d5cb800d56a2d2c5d106e1213838eed4e Mon Sep 17 00:00:00 2001 From: Lokesh Gidra Date: Thu, 15 Feb 2024 10:27:54 -0800 Subject: [PATCH 32/86] BACKPORT: userfaultfd: protect mmap_changing with rw_sem in userfaulfd_ctx Increments and loads to mmap_changing are always in mmap_lock critical section. This ensures that if userspace requests event notification for non-cooperative operations (e.g. mremap), userfaultfd operations don't occur concurrently. This can be achieved by using a separate read-write semaphore in userfaultfd_ctx such that increments are done in write-mode and loads in read-mode, thereby eliminating the dependency on mmap_lock for this purpose. This is a preparatory step before we replace mmap_lock usage with per-vma locks in fill/move ioctls. Link: https://lkml.kernel.org/r/20240215182756.3448972-3-lokeshgidra@google.com Signed-off-by: Lokesh Gidra Reviewed-by: Mike Rapoport (IBM) Reviewed-by: Liam R. Howlett Cc: Andrea Arcangeli Cc: Axel Rasmussen Cc: Brian Geffon Cc: David Hildenbrand Cc: Jann Horn Cc: Kalesh Singh Cc: Matthew Wilcox (Oracle) Cc: Nicolas Geoffray Cc: Peter Xu Cc: Ryan Roberts Cc: Suren Baghdasaryan Cc: Tim Murray Signed-off-by: Andrew Morton (cherry picked from commit 5e4c24a57b0c126686534b5b159a406c5dd02400) Conflicts: fs/userfaultfd.c include/linux/userfaultfd_k.h mm/userfaultfd.c 1. Functions passing control from fs/userfaultfd.c to mm/userfaultfd.c were renamed after 6.1. a. Replace mfill_atomic_copy() with mcopy_atomic() b. Replace mfill_atomic_zeropage() with mfill_zeropage() c. Replace mfill_atomic_continue() with mcopy_continue() d. Replace mfill_atomic() with __mcopy_atomic() e. Replace mfill_atomic_hugetlb() with __mcopy_atomic_hugetlb() 2. uffd flags were unified into a single parameter after 6.1. Replace 'flags' with 'mcopy_mode' and 'mode'. 3. Fetch dst_mm from dst_vma in __mcopy_atomic_hugetlb(). Bug: 320478828 Change-Id: I77615c36a0c891801c9eb9de3609df4e7f125c39 Signed-off-by: Lokesh Gidra --- fs/userfaultfd.c | 35 +++++++++++-------- include/linux/userfaultfd_k.h | 27 ++++++++------- mm/userfaultfd.c | 64 +++++++++++++++++++---------------- 3 files changed, 71 insertions(+), 55 deletions(-) diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index e2fc9491048d..0767c8e4a059 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -650,12 +650,15 @@ int dup_userfaultfd(struct vm_area_struct *vma, struct list_head *fcs) ctx->flags = octx->flags; ctx->features = octx->features; ctx->released = false; + init_rwsem(&ctx->map_changing_lock); atomic_set(&ctx->mmap_changing, 0); ctx->mm = vma->vm_mm; mmgrab(ctx->mm); userfaultfd_ctx_get(octx); + down_write(&octx->map_changing_lock); atomic_inc(&octx->mmap_changing); + up_write(&octx->map_changing_lock); fctx->orig = octx; fctx->new = ctx; list_add_tail(&fctx->list, fcs); @@ -702,7 +705,9 @@ void mremap_userfaultfd_prep(struct vm_area_struct *vma, if (ctx->features & UFFD_FEATURE_EVENT_REMAP) { vm_ctx->ctx = ctx; userfaultfd_ctx_get(ctx); + down_write(&ctx->map_changing_lock); atomic_inc(&ctx->mmap_changing); + up_write(&ctx->map_changing_lock); } else { /* Drop uffd context if remap feature not enabled */ vma_start_write(vma); @@ -748,7 +753,9 @@ bool userfaultfd_remove(struct vm_area_struct *vma, return true; userfaultfd_ctx_get(ctx); + down_write(&ctx->map_changing_lock); atomic_inc(&ctx->mmap_changing); + up_write(&ctx->map_changing_lock); mmap_read_unlock(mm); msg_init(&ewq.msg); @@ -790,7 +797,9 @@ int userfaultfd_unmap_prep(struct vm_area_struct *vma, unsigned long start, return -ENOMEM; userfaultfd_ctx_get(ctx); + down_write(&ctx->map_changing_lock); atomic_inc(&ctx->mmap_changing); + up_write(&ctx->map_changing_lock); unmap_ctx->ctx = ctx; unmap_ctx->start = start; unmap_ctx->end = end; @@ -1708,9 +1717,8 @@ static int userfaultfd_copy(struct userfaultfd_ctx *ctx, if (uffdio_copy.mode & ~(UFFDIO_COPY_MODE_DONTWAKE|UFFDIO_COPY_MODE_WP)) goto out; if (mmget_not_zero(ctx->mm)) { - ret = mcopy_atomic(ctx->mm, uffdio_copy.dst, uffdio_copy.src, - uffdio_copy.len, &ctx->mmap_changing, - uffdio_copy.mode); + ret = mcopy_atomic(ctx, uffdio_copy.dst, uffdio_copy.src, + uffdio_copy.len, uffdio_copy.mode); mmput(ctx->mm); } else { return -ESRCH; @@ -1760,9 +1768,8 @@ static int userfaultfd_zeropage(struct userfaultfd_ctx *ctx, goto out; if (mmget_not_zero(ctx->mm)) { - ret = mfill_zeropage(ctx->mm, uffdio_zeropage.range.start, - uffdio_zeropage.range.len, - &ctx->mmap_changing); + ret = mfill_zeropage(ctx, uffdio_zeropage.range.start, + uffdio_zeropage.range.len); mmput(ctx->mm); } else { return -ESRCH; @@ -1817,9 +1824,8 @@ static int userfaultfd_writeprotect(struct userfaultfd_ctx *ctx, return -EINVAL; if (mmget_not_zero(ctx->mm)) { - ret = mwriteprotect_range(ctx->mm, uffdio_wp.range.start, - uffdio_wp.range.len, mode_wp, - &ctx->mmap_changing); + ret = mwriteprotect_range(ctx, uffdio_wp.range.start, + uffdio_wp.range.len, mode_wp); mmput(ctx->mm); } else { return -ESRCH; @@ -1870,9 +1876,8 @@ static int userfaultfd_continue(struct userfaultfd_ctx *ctx, unsigned long arg) goto out; if (mmget_not_zero(ctx->mm)) { - ret = mcopy_continue(ctx->mm, uffdio_continue.range.start, - uffdio_continue.range.len, - &ctx->mmap_changing); + ret = mcopy_continue(ctx, uffdio_continue.range.start, + uffdio_continue.range.len); mmput(ctx->mm); } else { return -ESRCH; @@ -1943,13 +1948,14 @@ static int userfaultfd_move(struct userfaultfd_ctx *ctx, if (mmget_not_zero(mm)) { mmap_read_lock(mm); - /* Re-check after taking mmap_lock */ + /* Re-check after taking map_changing_lock */ + down_read(&ctx->map_changing_lock); if (likely(!atomic_read(&ctx->mmap_changing))) ret = move_pages(ctx, mm, uffdio_move.dst, uffdio_move.src, uffdio_move.len, uffdio_move.mode); else ret = -EAGAIN; - + up_read(&ctx->map_changing_lock); mmap_read_unlock(mm); mmput(mm); } else { @@ -2146,6 +2152,7 @@ static int new_userfaultfd(int flags) ctx->flags = flags; ctx->features = 0; ctx->released = false; + init_rwsem(&ctx->map_changing_lock); atomic_set(&ctx->mmap_changing, 0); ctx->mm = current->mm; /* prevent the mm struct to be freed */ diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index 7bbd671e912f..0fb4e4028eba 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -69,6 +69,13 @@ struct userfaultfd_ctx { unsigned int features; /* released */ bool released; + /* + * Prevents userfaultfd operations (fill/move/wp) from happening while + * some non-cooperative event(s) is taking place. Increments are done + * in write-mode. Whereas, userfaultfd operations, which includes + * reading mmap_changing, is done under read-mode. + */ + struct rw_semaphore map_changing_lock; /* memory mappings are changing because of non-cooperative event */ atomic_t mmap_changing; /* mm with one ore more vmas attached to this userfaultfd_ctx */ @@ -100,18 +107,14 @@ extern int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd, unsigned long dst_addr, struct page *page, bool newly_allocated, bool wp_copy); -extern ssize_t mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start, - unsigned long src_start, unsigned long len, - atomic_t *mmap_changing, __u64 mode); -extern ssize_t mfill_zeropage(struct mm_struct *dst_mm, - unsigned long dst_start, - unsigned long len, - atomic_t *mmap_changing); -extern ssize_t mcopy_continue(struct mm_struct *dst_mm, unsigned long dst_start, - unsigned long len, atomic_t *mmap_changing); -extern int mwriteprotect_range(struct mm_struct *dst_mm, - unsigned long start, unsigned long len, - bool enable_wp, atomic_t *mmap_changing); +extern ssize_t mcopy_atomic(struct userfaultfd_ctx *ctx, unsigned long dst_start, + unsigned long src_start, unsigned long len, __u64 mode); +extern ssize_t mfill_zeropage(struct userfaultfd_ctx *ctx, + unsigned long dst_start, unsigned long len); +extern ssize_t mcopy_continue(struct userfaultfd_ctx *ctx, unsigned long dst_start, + unsigned long len); +extern int mwriteprotect_range(struct userfaultfd_ctx *ctx, unsigned long start, + unsigned long len, bool enable_wp); extern void uffd_wp_range(struct mm_struct *dst_mm, struct vm_area_struct *vma, unsigned long start, unsigned long len, bool enable_wp); diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index af165216fbbd..b37bc54eb29c 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -322,15 +322,16 @@ static pmd_t *mm_alloc_pmd(struct mm_struct *mm, unsigned long address) * __mcopy_atomic processing for HUGETLB vmas. Note that this routine is * called with mmap_lock held, it will release mmap_lock before returning. */ -static __always_inline ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm, +static __always_inline ssize_t __mcopy_atomic_hugetlb( + struct userfaultfd_ctx *ctx, struct vm_area_struct *dst_vma, unsigned long dst_start, unsigned long src_start, unsigned long len, enum mcopy_atomic_mode mode, - bool wp_copy, - atomic_t *mmap_changing) + bool wp_copy) { + struct mm_struct *dst_mm = dst_vma->vm_mm; int vm_shared = dst_vma->vm_flags & VM_SHARED; ssize_t err; pte_t *dst_pte; @@ -349,6 +350,7 @@ static __always_inline ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm, * feature is not supported. */ if (mode == MCOPY_ATOMIC_ZEROPAGE) { + up_read(&ctx->map_changing_lock); mmap_read_unlock(dst_mm); return -EINVAL; } @@ -434,6 +436,7 @@ retry: cond_resched(); if (unlikely(err == -ENOENT)) { + up_read(&ctx->map_changing_lock); mmap_read_unlock(dst_mm); BUG_ON(!page); @@ -446,12 +449,13 @@ retry: goto out; } mmap_read_lock(dst_mm); + down_read(&ctx->map_changing_lock); /* * If memory mappings are changing because of non-cooperative * operation (e.g. mremap) running in parallel, bail out and * request the user to retry later */ - if (mmap_changing && atomic_read(mmap_changing)) { + if (atomic_read(&ctx->mmap_changing)) { err = -EAGAIN; break; } @@ -474,6 +478,7 @@ retry: } out_unlock: + up_read(&ctx->map_changing_lock); mmap_read_unlock(dst_mm); out: if (page) @@ -485,14 +490,13 @@ out: } #else /* !CONFIG_HUGETLB_PAGE */ /* fail at build time if gcc attempts to use this */ -extern ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm, +extern ssize_t __mcopy_atomic_hugetlb(struct userfaultfd_ctx *ctx, struct vm_area_struct *dst_vma, unsigned long dst_start, unsigned long src_start, unsigned long len, enum mcopy_atomic_mode mode, - bool wp_copy, - atomic_t *mmap_changing); + bool wp_copy); #endif /* CONFIG_HUGETLB_PAGE */ static __always_inline ssize_t mfill_atomic_pte(struct mm_struct *dst_mm, @@ -539,14 +543,14 @@ static __always_inline ssize_t mfill_atomic_pte(struct mm_struct *dst_mm, return err; } -static __always_inline ssize_t __mcopy_atomic(struct mm_struct *dst_mm, +static __always_inline ssize_t __mcopy_atomic(struct userfaultfd_ctx *ctx, unsigned long dst_start, unsigned long src_start, unsigned long len, enum mcopy_atomic_mode mcopy_mode, - atomic_t *mmap_changing, __u64 mode) { + struct mm_struct *dst_mm = ctx->mm; struct vm_area_struct *dst_vma; ssize_t err; pmd_t *dst_pmd; @@ -577,8 +581,9 @@ retry: * operation (e.g. mremap) running in parallel, bail out and * request the user to retry later */ + down_read(&ctx->map_changing_lock); err = -EAGAIN; - if (mmap_changing && atomic_read(mmap_changing)) + if (atomic_read(&ctx->mmap_changing)) goto out_unlock; /* @@ -611,9 +616,9 @@ retry: * If this is a HUGETLB vma, pass off to appropriate routine */ if (is_vm_hugetlb_page(dst_vma)) - return __mcopy_atomic_hugetlb(dst_mm, dst_vma, dst_start, + return __mcopy_atomic_hugetlb(ctx, dst_vma, dst_start, src_start, len, mcopy_mode, - wp_copy, mmap_changing); + wp_copy); if (!vma_is_anonymous(dst_vma) && !vma_is_shmem(dst_vma)) goto out_unlock; @@ -671,6 +676,7 @@ retry: if (unlikely(err == -ENOENT)) { void *page_kaddr; + up_read(&ctx->map_changing_lock); mmap_read_unlock(dst_mm); BUG_ON(!page); @@ -701,6 +707,7 @@ retry: } out_unlock: + up_read(&ctx->map_changing_lock); mmap_read_unlock(dst_mm); out: if (page) @@ -711,26 +718,23 @@ out: return copied ? copied : err; } -ssize_t mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start, - unsigned long src_start, unsigned long len, - atomic_t *mmap_changing, __u64 mode) +ssize_t mcopy_atomic(struct userfaultfd_ctx *ctx, unsigned long dst_start, + unsigned long src_start, unsigned long len, __u64 mode) { - return __mcopy_atomic(dst_mm, dst_start, src_start, len, - MCOPY_ATOMIC_NORMAL, mmap_changing, mode); + return __mcopy_atomic(ctx, dst_start, src_start, len, + MCOPY_ATOMIC_NORMAL, mode); } -ssize_t mfill_zeropage(struct mm_struct *dst_mm, unsigned long start, - unsigned long len, atomic_t *mmap_changing) +ssize_t mfill_zeropage(struct userfaultfd_ctx *ctx, unsigned long start, + unsigned long len) { - return __mcopy_atomic(dst_mm, start, 0, len, MCOPY_ATOMIC_ZEROPAGE, - mmap_changing, 0); + return __mcopy_atomic(ctx, start, 0, len, MCOPY_ATOMIC_ZEROPAGE, 0); } -ssize_t mcopy_continue(struct mm_struct *dst_mm, unsigned long start, - unsigned long len, atomic_t *mmap_changing) +ssize_t mcopy_continue(struct userfaultfd_ctx *ctx, unsigned long start, + unsigned long len) { - return __mcopy_atomic(dst_mm, start, 0, len, MCOPY_ATOMIC_CONTINUE, - mmap_changing, 0); + return __mcopy_atomic(ctx, start, 0, len, MCOPY_ATOMIC_CONTINUE, 0); } void uffd_wp_range(struct mm_struct *dst_mm, struct vm_area_struct *dst_vma, @@ -750,10 +754,10 @@ void uffd_wp_range(struct mm_struct *dst_mm, struct vm_area_struct *dst_vma, tlb_finish_mmu(&tlb); } -int mwriteprotect_range(struct mm_struct *dst_mm, unsigned long start, - unsigned long len, bool enable_wp, - atomic_t *mmap_changing) +int mwriteprotect_range(struct userfaultfd_ctx *ctx, unsigned long start, + unsigned long len, bool enable_wp) { + struct mm_struct *dst_mm = ctx->mm; struct vm_area_struct *dst_vma; unsigned long page_mask; int err; @@ -774,8 +778,9 @@ int mwriteprotect_range(struct mm_struct *dst_mm, unsigned long start, * operation (e.g. mremap) running in parallel, bail out and * request the user to retry later */ + down_read(&ctx->map_changing_lock); err = -EAGAIN; - if (mmap_changing && atomic_read(mmap_changing)) + if (atomic_read(&ctx->mmap_changing)) goto out_unlock; err = -ENOENT; @@ -799,6 +804,7 @@ int mwriteprotect_range(struct mm_struct *dst_mm, unsigned long start, err = 0; out_unlock: + up_read(&ctx->map_changing_lock); mmap_read_unlock(dst_mm); return err; } From daf0b0fc4a07a13fd8016c09d8425e25a2f59027 Mon Sep 17 00:00:00 2001 From: Lokesh Gidra Date: Thu, 15 Feb 2024 10:27:55 -0800 Subject: [PATCH 33/86] BACKPORT: mm: add vma_assert_locked() for !CONFIG_PER_VMA_LOCK vma_assert_locked() is needed to replace mmap_assert_locked() once we start using per-vma locks in userfaultfd operations. In !CONFIG_PER_VMA_LOCK case when mm is locked, it implies that the given VMA is locked. Link: https://lkml.kernel.org/r/20240215182756.3448972-4-lokeshgidra@google.com Signed-off-by: Lokesh Gidra Reviewed-by: Suren Baghdasaryan Cc: Andrea Arcangeli Cc: Axel Rasmussen Cc: Brian Geffon Cc: David Hildenbrand Cc: Jann Horn Cc: Kalesh Singh Cc: Liam R. Howlett Cc: Matthew Wilcox (Oracle) Cc: Mike Rapoport (IBM) Cc: Nicolas Geoffray Cc: Peter Xu Cc: Ryan Roberts Cc: Tim Murray Signed-off-by: Andrew Morton (cherry picked from commit 32af81af2f6f4c23b1b4ff68410e91da660af102) Conflicts: include/linux/mm.h 1. lock_vma_under_rcu() definition in !CONFIG_PER_VMA_LOCK case doesn't exist in 6.1. Resolved cherry-pick conflict due to that. Bug: 320478828 Change-Id: I76d414cd08c3d696d3886921a7e27cf94fd17b76 Signed-off-by: Lokesh Gidra --- include/linux/mm.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/linux/mm.h b/include/linux/mm.h index 4082f86f8f9f..a17bd3a6413f 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -780,6 +780,11 @@ static inline void vma_assert_write_locked(struct vm_area_struct *vma) static inline void vma_mark_detached(struct vm_area_struct *vma, bool detached) {} +static inline void vma_assert_locked(struct vm_area_struct *vma) +{ + mmap_assert_locked(vma->vm_mm); +} + static inline void release_fault_lock(struct vm_fault *vmf) { mmap_read_unlock(vmf->vma->vm_mm); From ce2896c0c6a5777721bdc703b1b89b1760e79854 Mon Sep 17 00:00:00 2001 From: Lokesh Gidra Date: Thu, 15 Feb 2024 10:27:56 -0800 Subject: [PATCH 34/86] BACKPORT: userfaultfd: use per-vma locks in userfaultfd operations All userfaultfd operations, except write-protect, opportunistically use per-vma locks to lock vmas. On failure, attempt again inside mmap_lock critical section. Write-protect operation requires mmap_lock as it iterates over multiple vmas. Link: https://lkml.kernel.org/r/20240215182756.3448972-5-lokeshgidra@google.com Signed-off-by: Lokesh Gidra Reviewed-by: Liam R. Howlett Cc: Andrea Arcangeli Cc: Axel Rasmussen Cc: Brian Geffon Cc: David Hildenbrand Cc: Jann Horn Cc: Kalesh Singh Cc: Matthew Wilcox (Oracle) Cc: Mike Rapoport (IBM) Cc: Nicolas Geoffray Cc: Peter Xu Cc: Ryan Roberts Cc: Suren Baghdasaryan Cc: Tim Murray Signed-off-by: Andrew Morton (cherry picked from commit 867a43a34ff8a38772212045262b2c9b77807ea3) Conflicts: mm/userfaultfd.c 1. Resolve conflict in validate_dst_vma() due to absence of range_in_vma(). 2. Use 'page' instead of 'folio' for BUG_ON on copy_from_user() failure in COPY ioctl. 3. Resolve conflict around mfill_file_over_size(). 4. Resolve conflict in comment for __mcopy_atomic_hugetlb() due to function name change. 5. Resolve conflict due to use of 'flags' instead of 'mode' in __mcopy_atomic_hugetlb(). 6. Use find_vma() and validate_dst_vma() in mwriteprotect_range() instead of find_dst_vma(). Bug: 320478828 Change-Id: I6d5b7101218cb1b11329108c3f31f12bb1caebc6 Signed-off-by: Lokesh Gidra --- fs/userfaultfd.c | 13 +- include/linux/userfaultfd_k.h | 5 +- mm/huge_memory.c | 5 +- mm/userfaultfd.c | 392 ++++++++++++++++++++++++++-------- 4 files changed, 304 insertions(+), 111 deletions(-) diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 0767c8e4a059..53c35356ed1d 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -1946,17 +1946,8 @@ static int userfaultfd_move(struct userfaultfd_ctx *ctx, return -EINVAL; if (mmget_not_zero(mm)) { - mmap_read_lock(mm); - - /* Re-check after taking map_changing_lock */ - down_read(&ctx->map_changing_lock); - if (likely(!atomic_read(&ctx->mmap_changing))) - ret = move_pages(ctx, mm, uffdio_move.dst, uffdio_move.src, - uffdio_move.len, uffdio_move.mode); - else - ret = -EAGAIN; - up_read(&ctx->map_changing_lock); - mmap_read_unlock(mm); + ret = move_pages(ctx, uffdio_move.dst, uffdio_move.src, + uffdio_move.len, uffdio_move.mode); mmput(mm); } else { return -ESRCH; diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index 0fb4e4028eba..ef7056fb9411 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -121,9 +121,8 @@ extern void uffd_wp_range(struct mm_struct *dst_mm, struct vm_area_struct *vma, /* move_pages */ void double_pt_lock(spinlock_t *ptl1, spinlock_t *ptl2); void double_pt_unlock(spinlock_t *ptl1, spinlock_t *ptl2); -ssize_t move_pages(struct userfaultfd_ctx *ctx, struct mm_struct *mm, - unsigned long dst_start, unsigned long src_start, - unsigned long len, __u64 flags); +ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start, + unsigned long src_start, unsigned long len, __u64 flags); int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pmd_t dst_pmdval, struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 1bc744166e9b..17f7cfcf007c 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1915,7 +1915,7 @@ unlock: #ifdef CONFIG_USERFAULTFD /* - * The PT lock for src_pmd and the mmap_lock for reading are held by + * The PT lock for src_pmd and dst_vma/src_vma (for reading) are locked by * the caller, but it must return after releasing the page_table_lock. * Just move the page from src_pmd to dst_pmd if possible. * Return zero if succeeded in moving the page, -EAGAIN if it needs to be @@ -1938,7 +1938,8 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm src_ptl = pmd_lockptr(mm, src_pmd); lockdep_assert_held(src_ptl); - mmap_assert_locked(mm); + vma_assert_locked(src_vma); + vma_assert_locked(dst_vma); /* Sanity checks before the operation */ if (WARN_ON_ONCE(!pmd_none(dst_pmdval)) || WARN_ON_ONCE(src_addr & ~HPAGE_PMD_MASK) || diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index b37bc54eb29c..99b0d56beb5d 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -20,23 +20,11 @@ #include "internal.h" static __always_inline -struct vm_area_struct *find_dst_vma(struct mm_struct *dst_mm, - unsigned long dst_start, - unsigned long len) +bool validate_dst_vma(struct vm_area_struct *dst_vma, unsigned long dst_end) { - /* - * Make sure that the dst range is both valid and fully within a - * single existing vma. - */ - struct vm_area_struct *dst_vma; - - dst_vma = find_vma(dst_mm, dst_start); - if (!dst_vma) - return NULL; - - if (dst_start < dst_vma->vm_start || - dst_start + len > dst_vma->vm_end) - return NULL; + /* Make sure that the dst range is fully within dst_vma. */ + if (dst_end > dst_vma->vm_end) + return false; /* * Check the vma is registered in uffd, this is required to @@ -44,11 +32,122 @@ struct vm_area_struct *find_dst_vma(struct mm_struct *dst_mm, * time. */ if (!dst_vma->vm_userfaultfd_ctx.ctx) - return NULL; + return false; + return true; +} + +static __always_inline +struct vm_area_struct *find_vma_and_prepare_anon(struct mm_struct *mm, + unsigned long addr) +{ + struct vm_area_struct *vma; + + mmap_assert_locked(mm); + vma = vma_lookup(mm, addr); + if (!vma) + vma = ERR_PTR(-ENOENT); + else if (!(vma->vm_flags & VM_SHARED) && + unlikely(anon_vma_prepare(vma))) + vma = ERR_PTR(-ENOMEM); + + return vma; +} + +#ifdef CONFIG_PER_VMA_LOCK +/* + * lock_vma() - Lookup and lock vma corresponding to @address. + * @mm: mm to search vma in. + * @address: address that the vma should contain. + * + * Should be called without holding mmap_lock. vma should be unlocked after use + * with unlock_vma(). + * + * Return: A locked vma containing @address, -ENOENT if no vma is found, or + * -ENOMEM if anon_vma couldn't be allocated. + */ +static struct vm_area_struct *lock_vma(struct mm_struct *mm, + unsigned long address) +{ + struct vm_area_struct *vma; + + vma = lock_vma_under_rcu(mm, address); + if (vma) { + /* + * lock_vma_under_rcu() only checks anon_vma for private + * anonymous mappings. But we need to ensure it is assigned in + * private file-backed vmas as well. + */ + if (!(vma->vm_flags & VM_SHARED) && unlikely(!vma->anon_vma)) + vma_end_read(vma); + else + return vma; + } + + mmap_read_lock(mm); + vma = find_vma_and_prepare_anon(mm, address); + if (!IS_ERR(vma)) { + /* + * We cannot use vma_start_read() as it may fail due to + * false locked (see comment in vma_start_read()). We + * can avoid that by directly locking vm_lock under + * mmap_lock, which guarantees that nobody can lock the + * vma for write (vma_start_write()) under us. + */ + down_read(&vma->vm_lock->lock); + } + + mmap_read_unlock(mm); + return vma; +} + +static struct vm_area_struct *uffd_mfill_lock(struct mm_struct *dst_mm, + unsigned long dst_start, + unsigned long len) +{ + struct vm_area_struct *dst_vma; + + dst_vma = lock_vma(dst_mm, dst_start); + if (IS_ERR(dst_vma) || validate_dst_vma(dst_vma, dst_start + len)) + return dst_vma; + + vma_end_read(dst_vma); + return ERR_PTR(-ENOENT); +} + +static void uffd_mfill_unlock(struct vm_area_struct *vma) +{ + vma_end_read(vma); +} + +#else + +static struct vm_area_struct *uffd_mfill_lock(struct mm_struct *dst_mm, + unsigned long dst_start, + unsigned long len) +{ + struct vm_area_struct *dst_vma; + + mmap_read_lock(dst_mm); + dst_vma = find_vma_and_prepare_anon(dst_mm, dst_start); + if (IS_ERR(dst_vma)) + goto out_unlock; + + if (validate_dst_vma(dst_vma, dst_start + len)) + return dst_vma; + + dst_vma = ERR_PTR(-ENOENT); +out_unlock: + mmap_read_unlock(dst_mm); return dst_vma; } +static void uffd_mfill_unlock(struct vm_area_struct *vma) +{ + mmap_read_unlock(vma->vm_mm); +} +#endif + /* * Install PTEs, to map dst_addr (within dst_vma) to page. * @@ -320,7 +419,8 @@ static pmd_t *mm_alloc_pmd(struct mm_struct *mm, unsigned long address) #ifdef CONFIG_HUGETLB_PAGE /* * __mcopy_atomic processing for HUGETLB vmas. Note that this routine is - * called with mmap_lock held, it will release mmap_lock before returning. + * called with either vma-lock or mmap_lock held, it will release the lock + * before returning. */ static __always_inline ssize_t __mcopy_atomic_hugetlb( struct userfaultfd_ctx *ctx, @@ -332,7 +432,6 @@ static __always_inline ssize_t __mcopy_atomic_hugetlb( bool wp_copy) { struct mm_struct *dst_mm = dst_vma->vm_mm; - int vm_shared = dst_vma->vm_flags & VM_SHARED; ssize_t err; pte_t *dst_pte; unsigned long src_addr, dst_addr; @@ -351,7 +450,7 @@ static __always_inline ssize_t __mcopy_atomic_hugetlb( */ if (mode == MCOPY_ATOMIC_ZEROPAGE) { up_read(&ctx->map_changing_lock); - mmap_read_unlock(dst_mm); + uffd_mfill_unlock(dst_vma); return -EINVAL; } @@ -374,24 +473,28 @@ retry: * retry, dst_vma will be set to NULL and we must lookup again. */ if (!dst_vma) { + dst_vma = uffd_mfill_lock(dst_mm, dst_start, len); + if (IS_ERR(dst_vma)) { + err = PTR_ERR(dst_vma); + goto out; + } + err = -ENOENT; - dst_vma = find_dst_vma(dst_mm, dst_start, len); - if (!dst_vma || !is_vm_hugetlb_page(dst_vma)) - goto out_unlock; + if (!is_vm_hugetlb_page(dst_vma)) + goto out_unlock_vma; err = -EINVAL; if (vma_hpagesize != vma_kernel_pagesize(dst_vma)) - goto out_unlock; + goto out_unlock_vma; - vm_shared = dst_vma->vm_flags & VM_SHARED; - } - - /* - * If not shared, ensure the dst_vma has a anon_vma. - */ - err = -ENOMEM; - if (!vm_shared) { - if (unlikely(anon_vma_prepare(dst_vma))) + /* + * If memory mappings are changing because of non-cooperative + * operation (e.g. mremap) running in parallel, bail out and + * request the user to retry later + */ + down_read(&ctx->map_changing_lock); + err = -EAGAIN; + if (atomic_read(&ctx->mmap_changing)) goto out_unlock; } @@ -437,7 +540,7 @@ retry: if (unlikely(err == -ENOENT)) { up_read(&ctx->map_changing_lock); - mmap_read_unlock(dst_mm); + uffd_mfill_unlock(dst_vma); BUG_ON(!page); err = copy_huge_page_from_user(page, @@ -448,17 +551,6 @@ retry: err = -EFAULT; goto out; } - mmap_read_lock(dst_mm); - down_read(&ctx->map_changing_lock); - /* - * If memory mappings are changing because of non-cooperative - * operation (e.g. mremap) running in parallel, bail out and - * request the user to retry later - */ - if (atomic_read(&ctx->mmap_changing)) { - err = -EAGAIN; - break; - } dst_vma = NULL; goto retry; @@ -479,7 +571,8 @@ retry: out_unlock: up_read(&ctx->map_changing_lock); - mmap_read_unlock(dst_mm); +out_unlock_vma: + uffd_mfill_unlock(dst_vma); out: if (page) put_page(page); @@ -574,7 +667,15 @@ static __always_inline ssize_t __mcopy_atomic(struct userfaultfd_ctx *ctx, copied = 0; page = NULL; retry: - mmap_read_lock(dst_mm); + /* + * Make sure the vma is not shared, that the dst range is + * both valid and fully within a single existing vma. + */ + dst_vma = uffd_mfill_lock(dst_mm, dst_start, len); + if (IS_ERR(dst_vma)) { + err = PTR_ERR(dst_vma); + goto out; + } /* * If memory mappings are changing because of non-cooperative @@ -586,15 +687,6 @@ retry: if (atomic_read(&ctx->mmap_changing)) goto out_unlock; - /* - * Make sure the vma is not shared, that the dst range is - * both valid and fully within a single existing vma. - */ - err = -ENOENT; - dst_vma = find_dst_vma(dst_mm, dst_start, len); - if (!dst_vma) - goto out_unlock; - err = -EINVAL; /* * shmem_zero_setup is invoked in mmap for MAP_ANONYMOUS|MAP_SHARED but @@ -625,16 +717,6 @@ retry: if (!vma_is_shmem(dst_vma) && mcopy_mode == MCOPY_ATOMIC_CONTINUE) goto out_unlock; - /* - * Ensure the dst_vma has a anon_vma or this page - * would get a NULL anon_vma when moved in the - * dst_vma. - */ - err = -ENOMEM; - if (!(dst_vma->vm_flags & VM_SHARED) && - unlikely(anon_vma_prepare(dst_vma))) - goto out_unlock; - while (src_addr < src_start + len) { pmd_t dst_pmdval; @@ -677,7 +759,7 @@ retry: void *page_kaddr; up_read(&ctx->map_changing_lock); - mmap_read_unlock(dst_mm); + uffd_mfill_unlock(dst_vma); BUG_ON(!page); page_kaddr = kmap_local_page(page); @@ -708,7 +790,7 @@ retry: out_unlock: up_read(&ctx->map_changing_lock); - mmap_read_unlock(dst_mm); + uffd_mfill_unlock(dst_vma); out: if (page) put_page(page); @@ -784,10 +866,12 @@ int mwriteprotect_range(struct userfaultfd_ctx *ctx, unsigned long start, goto out_unlock; err = -ENOENT; - dst_vma = find_dst_vma(dst_mm, start, len); + dst_vma = find_vma(dst_mm, start); if (!dst_vma) goto out_unlock; + if (start < dst_vma->vm_start || !validate_dst_vma(dst_vma, start + len)) + goto out_unlock; if (!userfaultfd_wp(dst_vma)) goto out_unlock; if (!vma_can_userfault(dst_vma, dst_vma->vm_flags)) @@ -1217,27 +1301,136 @@ static int validate_move_areas(struct userfaultfd_ctx *ctx, if (!vma_is_anonymous(src_vma) || !vma_is_anonymous(dst_vma)) return -EINVAL; - /* - * Ensure the dst_vma has a anon_vma or this page - * would get a NULL anon_vma when moved in the - * dst_vma. - */ - if (unlikely(anon_vma_prepare(dst_vma))) - return -ENOMEM; - return 0; } +static __always_inline +int find_vmas_mm_locked(struct mm_struct *mm, + unsigned long dst_start, + unsigned long src_start, + struct vm_area_struct **dst_vmap, + struct vm_area_struct **src_vmap) +{ + struct vm_area_struct *vma; + + mmap_assert_locked(mm); + vma = find_vma_and_prepare_anon(mm, dst_start); + if (IS_ERR(vma)) + return PTR_ERR(vma); + + *dst_vmap = vma; + /* Skip finding src_vma if src_start is in dst_vma */ + if (src_start >= vma->vm_start && src_start < vma->vm_end) + goto out_success; + + vma = vma_lookup(mm, src_start); + if (!vma) + return -ENOENT; +out_success: + *src_vmap = vma; + return 0; +} + +#ifdef CONFIG_PER_VMA_LOCK +static int uffd_move_lock(struct mm_struct *mm, + unsigned long dst_start, + unsigned long src_start, + struct vm_area_struct **dst_vmap, + struct vm_area_struct **src_vmap) +{ + struct vm_area_struct *vma; + int err; + + vma = lock_vma(mm, dst_start); + if (IS_ERR(vma)) + return PTR_ERR(vma); + + *dst_vmap = vma; + /* + * Skip finding src_vma if src_start is in dst_vma. This also ensures + * that we don't lock the same vma twice. + */ + if (src_start >= vma->vm_start && src_start < vma->vm_end) { + *src_vmap = vma; + return 0; + } + + /* + * Using lock_vma() to get src_vma can lead to following deadlock: + * + * Thread1 Thread2 + * ------- ------- + * vma_start_read(dst_vma) + * mmap_write_lock(mm) + * vma_start_write(src_vma) + * vma_start_read(src_vma) + * mmap_read_lock(mm) + * vma_start_write(dst_vma) + */ + *src_vmap = lock_vma_under_rcu(mm, src_start); + if (likely(*src_vmap)) + return 0; + + /* Undo any locking and retry in mmap_lock critical section */ + vma_end_read(*dst_vmap); + + mmap_read_lock(mm); + err = find_vmas_mm_locked(mm, dst_start, src_start, dst_vmap, src_vmap); + if (!err) { + /* + * See comment in lock_vma() as to why not using + * vma_start_read() here. + */ + down_read(&(*dst_vmap)->vm_lock->lock); + if (*dst_vmap != *src_vmap) + down_read(&(*src_vmap)->vm_lock->lock); + } + mmap_read_unlock(mm); + return err; +} + +static void uffd_move_unlock(struct vm_area_struct *dst_vma, + struct vm_area_struct *src_vma) +{ + vma_end_read(src_vma); + if (src_vma != dst_vma) + vma_end_read(dst_vma); +} + +#else + +static int uffd_move_lock(struct mm_struct *mm, + unsigned long dst_start, + unsigned long src_start, + struct vm_area_struct **dst_vmap, + struct vm_area_struct **src_vmap) +{ + int err; + + mmap_read_lock(mm); + err = find_vmas_mm_locked(mm, dst_start, src_start, dst_vmap, src_vmap); + if (err) + mmap_read_unlock(mm); + return err; +} + +static void uffd_move_unlock(struct vm_area_struct *dst_vma, + struct vm_area_struct *src_vma) +{ + mmap_assert_locked(src_vma->vm_mm); + mmap_read_unlock(dst_vma->vm_mm); +} +#endif + /** * move_pages - move arbitrary anonymous pages of an existing vma * @ctx: pointer to the userfaultfd context - * @mm: the address space to move pages * @dst_start: start of the destination virtual memory range * @src_start: start of the source virtual memory range * @len: length of the virtual memory range * @mode: flags from uffdio_move.mode * - * Must be called with mmap_lock held for read. + * It will either use the mmap_lock in read mode or per-vma locks * * move_pages() remaps arbitrary anonymous pages atomically in zero * copy. It only works on non shared anonymous pages because those can @@ -1305,10 +1498,10 @@ static int validate_move_areas(struct userfaultfd_ctx *ctx, * could be obtained. This is the only additional complexity added to * the rmap code to provide this anonymous page remapping functionality. */ -ssize_t move_pages(struct userfaultfd_ctx *ctx, struct mm_struct *mm, - unsigned long dst_start, unsigned long src_start, - unsigned long len, __u64 mode) +ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start, + unsigned long src_start, unsigned long len, __u64 mode) { + struct mm_struct *mm = ctx->mm; struct vm_area_struct *src_vma, *dst_vma; unsigned long src_addr, dst_addr; pmd_t *src_pmd, *dst_pmd; @@ -1326,28 +1519,34 @@ ssize_t move_pages(struct userfaultfd_ctx *ctx, struct mm_struct *mm, WARN_ON_ONCE(dst_start + len <= dst_start)) goto out; + err = uffd_move_lock(mm, dst_start, src_start, &dst_vma, &src_vma); + if (err) + goto out; + + /* Re-check after taking map_changing_lock */ + err = -EAGAIN; + down_read(&ctx->map_changing_lock); + if (likely(atomic_read(&ctx->mmap_changing))) + goto out_unlock; /* * Make sure the vma is not shared, that the src and dst remap * ranges are both valid and fully within a single existing * vma. */ - src_vma = find_vma(mm, src_start); - if (!src_vma || (src_vma->vm_flags & VM_SHARED)) - goto out; - if (src_start < src_vma->vm_start || - src_start + len > src_vma->vm_end) - goto out; + err = -EINVAL; + if (src_vma->vm_flags & VM_SHARED) + goto out_unlock; + if (src_start + len > src_vma->vm_end) + goto out_unlock; - dst_vma = find_vma(mm, dst_start); - if (!dst_vma || (dst_vma->vm_flags & VM_SHARED)) - goto out; - if (dst_start < dst_vma->vm_start || - dst_start + len > dst_vma->vm_end) - goto out; + if (dst_vma->vm_flags & VM_SHARED) + goto out_unlock; + if (dst_start + len > dst_vma->vm_end) + goto out_unlock; err = validate_move_areas(ctx, src_vma, dst_vma); if (err) - goto out; + goto out_unlock; for (src_addr = src_start, dst_addr = dst_start; src_addr < src_start + len;) { @@ -1464,6 +1663,9 @@ ssize_t move_pages(struct userfaultfd_ctx *ctx, struct mm_struct *mm, moved += step_size; } +out_unlock: + up_read(&ctx->map_changing_lock); + uffd_move_unlock(dst_vma, src_vma); out: VM_WARN_ON(moved < 0); VM_WARN_ON(err > 0); From 8dd482be44dcf8530b75f5572707363a101ff1ea Mon Sep 17 00:00:00 2001 From: Lokesh Gidra Date: Thu, 21 Mar 2024 16:58:18 -0700 Subject: [PATCH 35/86] UPSTREAM: userfaultfd: fix deadlock warning when locking src and dst VMAs Use down_read_nested() to avoid the warning. Link: https://lkml.kernel.org/r/20240321235818.125118-1-lokeshgidra@google.com Fixes: 867a43a34ff8 ("userfaultfd: use per-vma locks in userfaultfd operations") Reported-by: syzbot+49056626fe41e01f2ba7@syzkaller.appspotmail.com Signed-off-by: Lokesh Gidra Cc: Andrea Arcangeli Cc: Axel Rasmussen Cc: Brian Geffon Cc: David Hildenbrand Cc: Hillf Danton Cc: Jann Horn [Bug #2] Cc: Kalesh Singh Cc: Lokesh Gidra Cc: Mike Rapoport (IBM) Cc: Nicolas Geoffray Cc: Peter Xu Cc: Suren Baghdasaryan Signed-off-by: Andrew Morton (cherry picked from commit 30af24facf0aed12dec23bdf6eac6a907f88306a) Bug: 320478828 Change-Id: I56d7e33878d6248bba28e1e4204e2b9005d87e4d Signed-off-by: Lokesh Gidra --- mm/userfaultfd.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 99b0d56beb5d..dffa3fe756a6 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -1383,7 +1383,8 @@ static int uffd_move_lock(struct mm_struct *mm, */ down_read(&(*dst_vmap)->vm_lock->lock); if (*dst_vmap != *src_vmap) - down_read(&(*src_vmap)->vm_lock->lock); + down_read_nested(&(*src_vmap)->vm_lock->lock, + SINGLE_DEPTH_NESTING); } mmap_read_unlock(mm); return err; From 3dfddcb9c2779eb7c91d7f0f1108574193970eec Mon Sep 17 00:00:00 2001 From: Lokesh Gidra Date: Fri, 5 Apr 2024 16:22:11 -0700 Subject: [PATCH 36/86] ANDROID: GKI: fix ABI breakage in struct userfaultfd_ctx The following two commits move 'userfaultfd_ctx' struct from fs/userfaultfd.c to header file and then add a rw_semaphore to it. The ABI is broken by the change. However, given that the type should be private and not accessed by vendor modules, use some GENKSYMS #define magic to preserve the CRC. Also update the .stg file for offset adjustment within 'userfaultfd_ctx'. 5e4c24a57b0c ("userfaultfd: protect mmap_changing with rw_sem in userfaulfd_ctx") f91e6b41dd11 ("userfaultfd: move userfaultfd_ctx struct to header file") Bug: 320478828 Change-Id: I5f97ff34dd8c88fe3d18c4dc902452488ba28cbd Signed-off-by: Lokesh Gidra --- android/abi_gki_aarch64.stg | 22 +++++++++++++++------- include/linux/userfaultfd_k.h | 9 +++++++++ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/android/abi_gki_aarch64.stg b/android/abi_gki_aarch64.stg index 4ed0b449d63e..f3588814586a 100644 --- a/android/abi_gki_aarch64.stg +++ b/android/abi_gki_aarch64.stg @@ -122166,6 +122166,12 @@ member { type_id: 0x1d46f863 offset: 128 } +member { + id: 0xa03c8629 + name: "map_changing_lock" + type_id: 0x28b9ec9a + offset: 960 +} member { id: 0x7031173c name: "map_check_btf" @@ -128778,10 +128784,10 @@ member { offset: 128 } member { - id: 0x3025dd18 + id: 0x3025df0c name: "mm" type_id: 0x1b36c7a2 - offset: 960 + offset: 1536 } member { id: 0xf2c3ca3b @@ -128982,10 +128988,10 @@ member { offset: 576 } member { - id: 0xdd180b6b + id: 0xdd180ba9 name: "mmap_changing" type_id: 0x74d29cf1 - offset: 928 + offset: 1472 } member { id: 0xc392230a @@ -267504,7 +267510,7 @@ struct_union { kind: STRUCT name: "userfaultfd_ctx" definition { - bytesize: 128 + bytesize: 208 member_id: 0x3162bad8 member_id: 0x05e73814 member_id: 0xc409a485 @@ -267514,8 +267520,10 @@ struct_union { member_id: 0x2d2d0282 member_id: 0xc55df36f member_id: 0x79d263fe - member_id: 0xdd180b6b - member_id: 0x3025dd18 + member_id: 0xa03c8629 + member_id: 0xdd180ba9 + member_id: 0x3025df0c + member_id: 0x2d081c8a } } struct_union { diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index ef7056fb9411..7d881c5df6ad 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -50,6 +50,12 @@ * since fd_wqh.lock is taken by aio_poll() while it's holding a lock that's * also taken in IRQ context. */ + +/* + * ANDROID: CRC fix for commit f91e6b41dd11 ("userfaultfd: move userfaultfd_ctx + * struct to header file") + */ +#ifndef __GENKSYMS__ struct userfaultfd_ctx { /* waitqueue head for the pending (i.e. not read) userfaults */ wait_queue_head_t fault_pending_wqh; @@ -80,7 +86,10 @@ struct userfaultfd_ctx { atomic_t mmap_changing; /* mm with one ore more vmas attached to this userfaultfd_ctx */ struct mm_struct *mm; + + ANDROID_KABI_RESERVE(1); }; +#endif extern int sysctl_unprivileged_userfaultfd; From 94c88f80ffddff00f0af448c02dfd8a3f3cdd692 Mon Sep 17 00:00:00 2001 From: Kuniyuki Iwashima Date: Tue, 23 Jan 2024 09:08:53 -0800 Subject: [PATCH 37/86] UPSTREAM: af_unix: Do not use atomic ops for unix_sk(sk)->inflight. [ Upstream commit 97af84a6bba2ab2b9c704c08e67de3b5ea551bb2 ] When touching unix_sk(sk)->inflight, we are always under spin_lock(&unix_gc_lock). Let's convert unix_sk(sk)->inflight to the normal unsigned long. Bug: 336226035 Signed-off-by: Kuniyuki Iwashima Reviewed-by: Simon Horman Link: https://lore.kernel.org/r/20240123170856.41348-3-kuniyu@amazon.com Signed-off-by: Jakub Kicinski Stable-dep-of: 47d8ac011fe1 ("af_unix: Fix garbage collector racing against connect()") Signed-off-by: Sasha Levin (cherry picked from commit 301fdbaa0bba4653570f07789909939f977a7620) Signed-off-by: Lee Jones Change-Id: I0d965d5f2a863d798c06de9f21d0467f256b538e --- include/net/af_unix.h | 2 +- net/unix/af_unix.c | 4 ++-- net/unix/garbage.c | 17 ++++++++--------- net/unix/scm.c | 8 +++++--- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/include/net/af_unix.h b/include/net/af_unix.h index 480fa579787e..af4646fd12b0 100644 --- a/include/net/af_unix.h +++ b/include/net/af_unix.h @@ -60,7 +60,7 @@ struct unix_sock { struct mutex iolock, bindlock; struct sock *peer; struct list_head link; - atomic_long_t inflight; + unsigned long inflight; spinlock_t lock; unsigned long gc_flags; #define UNIX_GC_CANDIDATE 0 diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index 6dbeb8007333..6757d11caa8c 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -970,11 +970,11 @@ static struct sock *unix_create1(struct net *net, struct socket *sock, int kern, sk->sk_write_space = unix_write_space; sk->sk_max_ack_backlog = net->unx.sysctl_max_dgram_qlen; sk->sk_destruct = unix_sock_destructor; - u = unix_sk(sk); + u = unix_sk(sk); + u->inflight = 0; u->path.dentry = NULL; u->path.mnt = NULL; spin_lock_init(&u->lock); - atomic_long_set(&u->inflight, 0); INIT_LIST_HEAD(&u->link); mutex_init(&u->iolock); /* single task reading lock */ mutex_init(&u->bindlock); /* single task binding lock */ diff --git a/net/unix/garbage.c b/net/unix/garbage.c index dc2763540393..312474c23e9a 100644 --- a/net/unix/garbage.c +++ b/net/unix/garbage.c @@ -166,17 +166,18 @@ static void scan_children(struct sock *x, void (*func)(struct unix_sock *), static void dec_inflight(struct unix_sock *usk) { - atomic_long_dec(&usk->inflight); + usk->inflight--; } static void inc_inflight(struct unix_sock *usk) { - atomic_long_inc(&usk->inflight); + usk->inflight++; } static void inc_inflight_move_tail(struct unix_sock *u) { - atomic_long_inc(&u->inflight); + u->inflight++; + /* If this still might be part of a cycle, move it to the end * of the list, so that it's checked even if it was already * passed over @@ -237,14 +238,12 @@ void unix_gc(void) */ list_for_each_entry_safe(u, next, &gc_inflight_list, link) { long total_refs; - long inflight_refs; total_refs = file_count(u->sk.sk_socket->file); - inflight_refs = atomic_long_read(&u->inflight); - BUG_ON(inflight_refs < 1); - BUG_ON(total_refs < inflight_refs); - if (total_refs == inflight_refs) { + BUG_ON(!u->inflight); + BUG_ON(total_refs < u->inflight); + if (total_refs == u->inflight) { list_move_tail(&u->link, &gc_candidates); __set_bit(UNIX_GC_CANDIDATE, &u->gc_flags); __set_bit(UNIX_GC_MAYBE_CYCLE, &u->gc_flags); @@ -271,7 +270,7 @@ void unix_gc(void) /* Move cursor to after the current position. */ list_move(&cursor, &u->link); - if (atomic_long_read(&u->inflight) > 0) { + if (u->inflight) { list_move_tail(&u->link, ¬_cycle_list); __clear_bit(UNIX_GC_MAYBE_CYCLE, &u->gc_flags); scan_children(&u->sk, inc_inflight_move_tail, NULL); diff --git a/net/unix/scm.c b/net/unix/scm.c index e8e2a00bb0f5..bdcda4e41f10 100644 --- a/net/unix/scm.c +++ b/net/unix/scm.c @@ -54,12 +54,13 @@ void unix_inflight(struct user_struct *user, struct file *fp) if (s) { struct unix_sock *u = unix_sk(s); - if (atomic_long_inc_return(&u->inflight) == 1) { + if (!u->inflight) { BUG_ON(!list_empty(&u->link)); list_add_tail(&u->link, &gc_inflight_list); } else { BUG_ON(list_empty(&u->link)); } + u->inflight++; /* Paired with READ_ONCE() in wait_for_unix_gc() */ WRITE_ONCE(unix_tot_inflight, unix_tot_inflight + 1); } @@ -76,10 +77,11 @@ void unix_notinflight(struct user_struct *user, struct file *fp) if (s) { struct unix_sock *u = unix_sk(s); - BUG_ON(!atomic_long_read(&u->inflight)); + BUG_ON(!u->inflight); BUG_ON(list_empty(&u->link)); - if (atomic_long_dec_and_test(&u->inflight)) + u->inflight--; + if (!u->inflight) list_del_init(&u->link); /* Paired with READ_ONCE() in wait_for_unix_gc() */ WRITE_ONCE(unix_tot_inflight, unix_tot_inflight - 1); From d0c6724b0f6fa713e7fcfc0eb0173625519ba570 Mon Sep 17 00:00:00 2001 From: Michal Luczaj Date: Tue, 9 Apr 2024 22:09:39 +0200 Subject: [PATCH 38/86] UPSTREAM: af_unix: Fix garbage collector racing against connect() [ Upstream commit 47d8ac011fe1c9251070e1bd64cb10b48193ec51 ] Garbage collector does not take into account the risk of embryo getting enqueued during the garbage collection. If such embryo has a peer that carries SCM_RIGHTS, two consecutive passes of scan_children() may see a different set of children. Leading to an incorrectly elevated inflight count, and then a dangling pointer within the gc_inflight_list. sockets are AF_UNIX/SOCK_STREAM S is an unconnected socket L is a listening in-flight socket bound to addr, not in fdtable V's fd will be passed via sendmsg(), gets inflight count bumped connect(S, addr) sendmsg(S, [V]); close(V) __unix_gc() ---------------- ------------------------- ----------- NS = unix_create1() skb1 = sock_wmalloc(NS) L = unix_find_other(addr) unix_state_lock(L) unix_peer(S) = NS // V count=1 inflight=0 NS = unix_peer(S) skb2 = sock_alloc() skb_queue_tail(NS, skb2[V]) // V became in-flight // V count=2 inflight=1 close(V) // V count=1 inflight=1 // GC candidate condition met for u in gc_inflight_list: if (total_refs == inflight_refs) add u to gc_candidates // gc_candidates={L, V} for u in gc_candidates: scan_children(u, dec_inflight) // embryo (skb1) was not // reachable from L yet, so V's // inflight remains unchanged __skb_queue_tail(L, skb1) unix_state_unlock(L) for u in gc_candidates: if (u.inflight) scan_children(u, inc_inflight_move_tail) // V count=1 inflight=2 (!) If there is a GC-candidate listening socket, lock/unlock its state. This makes GC wait until the end of any ongoing connect() to that socket. After flipping the lock, a possibly SCM-laden embryo is already enqueued. And if there is another embryo coming, it can not possibly carry SCM_RIGHTS. At this point, unix_inflight() can not happen because unix_gc_lock is already taken. Inflight graph remains unaffected. Bug: 336226035 Fixes: 1fd05ba5a2f2 ("[AF_UNIX]: Rewrite garbage collector, fixes race.") Signed-off-by: Michal Luczaj Reviewed-by: Kuniyuki Iwashima Link: https://lore.kernel.org/r/20240409201047.1032217-1-mhal@rbox.co Signed-off-by: Paolo Abeni Signed-off-by: Sasha Levin (cherry picked from commit 507cc232ffe53a352847893f8177d276c3b532a9) Signed-off-by: Lee Jones Change-Id: If321f78b8b3220f5a1caea4b5e9450f1235b0770 --- net/unix/garbage.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/net/unix/garbage.c b/net/unix/garbage.c index 312474c23e9a..67b2c3bfa113 100644 --- a/net/unix/garbage.c +++ b/net/unix/garbage.c @@ -235,11 +235,22 @@ void unix_gc(void) * receive queues. Other, non candidate sockets _can_ be * added to queue, so we must make sure only to touch * candidates. + * + * Embryos, though never candidates themselves, affect which + * candidates are reachable by the garbage collector. Before + * being added to a listener's queue, an embryo may already + * receive data carrying SCM_RIGHTS, potentially making the + * passed socket a candidate that is not yet reachable by the + * collector. It becomes reachable once the embryo is + * enqueued. Therefore, we must ensure that no SCM-laden + * embryo appears in a (candidate) listener's queue between + * consecutive scan_children() calls. */ list_for_each_entry_safe(u, next, &gc_inflight_list, link) { + struct sock *sk = &u->sk; long total_refs; - total_refs = file_count(u->sk.sk_socket->file); + total_refs = file_count(sk->sk_socket->file); BUG_ON(!u->inflight); BUG_ON(total_refs < u->inflight); @@ -247,6 +258,11 @@ void unix_gc(void) list_move_tail(&u->link, &gc_candidates); __set_bit(UNIX_GC_CANDIDATE, &u->gc_flags); __set_bit(UNIX_GC_MAYBE_CYCLE, &u->gc_flags); + + if (sk->sk_state == TCP_LISTEN) { + unix_state_lock(sk); + unix_state_unlock(sk); + } } } From 260bfad693a3c5e8859e26363cc535a077c18a84 Mon Sep 17 00:00:00 2001 From: xieliujie Date: Thu, 28 Mar 2024 18:36:46 +0800 Subject: [PATCH 39/86] ANDROID: vendor_hook: Add hooks to support reader optimistic spin in rwsem Since upstream commit 617f3ef95177 ("locking/rwsem: Remove reader optimistic spinning"), vendors have seen increased contention and blocking on rwsems. There are attempts to actively fix this upstream: https://lore.kernel.org/lkml/20240406081126.8030-1-bongkyu7.kim@samsung.com/ But in the meantime, provide vendorhooks so that vendors can implement their own optimistic spin routine. In doing so, vendors see improvements in cold launch times on important apps. Bug: 331742151 Change-Id: I7466413de9ee1293e86f73880931235d7a9142ac Signed-off-by: xieliujie [jstultz: Rewrote commit message] Signed-off-by: John Stultz --- drivers/android/vendor_hooks.c | 2 ++ include/trace/hooks/rwsem.h | 6 ++++++ kernel/locking/osq_lock.c | 2 ++ kernel/locking/rwsem.c | 14 ++++++++++++-- 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index f742029f9086..7a600198d24b 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -111,6 +111,8 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_opt_spin_start); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_opt_spin_finish); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_can_spin_on_owner); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_sched_show_task); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_direct_rsteal); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_optimistic_rspin); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cpu_idle_enter); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cpu_idle_exit); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mpam_set); diff --git a/include/trace/hooks/rwsem.h b/include/trace/hooks/rwsem.h index a2fb006cd457..4ed664556e8c 100644 --- a/include/trace/hooks/rwsem.h +++ b/include/trace/hooks/rwsem.h @@ -28,6 +28,12 @@ DECLARE_HOOK(android_vh_alter_rwsem_list_add, DECLARE_HOOK(android_vh_rwsem_wake_finish, TP_PROTO(struct rw_semaphore *sem), TP_ARGS(sem)); +DECLARE_HOOK(android_vh_rwsem_direct_rsteal, + TP_PROTO(struct rw_semaphore *sem, bool *steal), + TP_ARGS(sem, steal)); +DECLARE_HOOK(android_vh_rwsem_optimistic_rspin, + TP_PROTO(struct rw_semaphore *sem, long *adjustment, bool *rspin), + TP_ARGS(sem, adjustment, rspin)); #endif /* _TRACE_HOOK_RWSEM_H */ /* This part must be outside protection */ #include diff --git a/kernel/locking/osq_lock.c b/kernel/locking/osq_lock.c index d5610ad52b92..b752ec5cc6b0 100644 --- a/kernel/locking/osq_lock.c +++ b/kernel/locking/osq_lock.c @@ -203,6 +203,7 @@ bool osq_lock(struct optimistic_spin_queue *lock) return false; } +EXPORT_SYMBOL_GPL(osq_lock); void osq_unlock(struct optimistic_spin_queue *lock) { @@ -230,3 +231,4 @@ void osq_unlock(struct optimistic_spin_queue *lock) if (next) WRITE_ONCE(next->locked, 1); } +EXPORT_SYMBOL_GPL(osq_unlock); diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c index aa85792d352e..a36788e0aa16 100644 --- a/kernel/locking/rwsem.c +++ b/kernel/locking/rwsem.c @@ -1025,6 +1025,8 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat struct rwsem_waiter waiter; DEFINE_WAKE_Q(wake_q); bool already_on_list = false; + bool steal = true; + bool rspin = false; /* * To prevent a constant stream of readers from starving a sleeping @@ -1038,7 +1040,8 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat /* * Reader optimistic lock stealing. */ - if (!(count & (RWSEM_WRITER_LOCKED | RWSEM_FLAG_HANDOFF))) { + trace_android_vh_rwsem_direct_rsteal(sem, &steal); + if (steal && !(count & (RWSEM_WRITER_LOCKED | RWSEM_FLAG_HANDOFF))) { rwsem_set_reader_owned(sem); lockevent_inc(rwsem_rlock_steal); @@ -1046,7 +1049,8 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat * Wake up other readers in the wait queue if it is * the first reader. */ - if ((rcnt == 1) && (count & RWSEM_FLAG_WAITERS)) { +wake_readers: + if ((rcnt == 1 || rspin) && (count & RWSEM_FLAG_WAITERS)) { raw_spin_lock_irq(&sem->wait_lock); if (!list_empty(&sem->wait_list)) rwsem_mark_wake(sem, RWSEM_WAKE_READ_OWNED, @@ -1057,6 +1061,12 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat trace_android_vh_record_rwsem_lock_starttime(current, jiffies); return sem; } + /* + * Reader optimistic spinning and stealing. + */ + trace_android_vh_rwsem_optimistic_rspin(sem, &adjustment, &rspin); + if (rspin) + goto wake_readers; queue: waiter.task = current; From c61278bb705a1891885b578a71656264da61be5b Mon Sep 17 00:00:00 2001 From: xieliujie Date: Wed, 24 Apr 2024 10:47:53 +0800 Subject: [PATCH 40/86] ANDROID: GKI: Update symbols to symbol list Update symbols for vendor hooks of reader optimistic spin. 4 function symbol(s) added 'int __traceiter_android_vh_rwsem_direct_rsteal(void*, struct rw_semaphore*, bool*)' 'int __traceiter_android_vh_rwsem_optimistic_rspin(void*, struct rw_semaphore*, long*, bool*)' 'bool osq_lock(struct optimistic_spin_queue*)' 'void osq_unlock(struct optimistic_spin_queue*)' 2 variable symbol(s) added 'struct tracepoint __tracepoint_android_vh_rwsem_direct_rsteal' 'struct tracepoint __tracepoint_android_vh_rwsem_optimistic_rspin' Bug: 331742151 Change-Id: I6603ec88f84a9a8adb30b802ba2fdd9b0dc8a016 Signed-off-by: xieliujie --- android/abi_gki_aarch64.stg | 83 +++++++++++++++++++++++++++++++++++ android/abi_gki_aarch64_oplus | 6 +++ 2 files changed, 89 insertions(+) diff --git a/android/abi_gki_aarch64.stg b/android/abi_gki_aarch64.stg index f3588814586a..c321dfae30ca 100644 --- a/android/abi_gki_aarch64.stg +++ b/android/abi_gki_aarch64.stg @@ -11368,6 +11368,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x54a91269 } +pointer_reference { + id: 0x1fbeff3e + kind: POINTER + pointee_type_id: 0x54bb1a65 +} pointer_reference { id: 0x1fc1dca8 kind: POINTER @@ -297074,6 +297079,11 @@ function { return_type_id: 0x48b5725f parameter_id: 0x1fba7d3d } +function { + id: 0x1777ee5c + return_type_id: 0x48b5725f + parameter_id: 0x1fbeff3e +} function { id: 0x177cb160 return_type_id: 0x48b5725f @@ -323449,6 +323459,14 @@ function { parameter_id: 0x18150d9f parameter_id: 0x06b96242 } +function { + id: 0x9ba060b9 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x00be4281 + parameter_id: 0x3593bec8 + parameter_id: 0x11cfee5a +} function { id: 0x9ba0658b return_type_id: 0x6720d32f @@ -336049,6 +336067,11 @@ function { parameter_id: 0x18019702 parameter_id: 0x6720d32f } +function { + id: 0xf94be4cb + return_type_id: 0x6d7f5ff6 + parameter_id: 0x1fbeff3e +} function { id: 0xf9525760 return_type_id: 0x6d7f5ff6 @@ -343008,6 +343031,15 @@ elf_symbol { type_id: 0x9ba93092 full_name: "__traceiter_android_vh_rwsem_can_spin_on_owner" } +elf_symbol { + id: 0xaa42999f + name: "__traceiter_android_vh_rwsem_direct_rsteal" + is_defined: true + symbol_type: FUNCTION + crc: 0xf066f9e5 + type_id: 0x9ba93092 + full_name: "__traceiter_android_vh_rwsem_direct_rsteal" +} elf_symbol { id: 0xb1847a6f name: "__traceiter_android_vh_rwsem_init" @@ -343035,6 +343067,15 @@ elf_symbol { type_id: 0x9ba966ad full_name: "__traceiter_android_vh_rwsem_opt_spin_start" } +elf_symbol { + id: 0x66c7c5f3 + name: "__traceiter_android_vh_rwsem_optimistic_rspin" + is_defined: true + symbol_type: FUNCTION + crc: 0xd12e2760 + type_id: 0x9ba060b9 + full_name: "__traceiter_android_vh_rwsem_optimistic_rspin" +} elf_symbol { id: 0xbffefc2b name: "__traceiter_android_vh_rwsem_read_wait_finish" @@ -347256,6 +347297,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_rwsem_can_spin_on_owner" } +elf_symbol { + id: 0x403cf0dd + name: "__tracepoint_android_vh_rwsem_direct_rsteal" + is_defined: true + symbol_type: OBJECT + crc: 0xd7439dcb + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_rwsem_direct_rsteal" +} elf_symbol { id: 0x84628825 name: "__tracepoint_android_vh_rwsem_init" @@ -347283,6 +347333,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_rwsem_opt_spin_start" } +elf_symbol { + id: 0x8866a3b9 + name: "__tracepoint_android_vh_rwsem_optimistic_rspin" + is_defined: true + symbol_type: OBJECT + crc: 0x89c242e2 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_rwsem_optimistic_rspin" +} elf_symbol { id: 0xe7ef7059 name: "__tracepoint_android_vh_rwsem_read_wait_finish" @@ -381277,6 +381336,24 @@ elf_symbol { type_id: 0x0bc7866e full_name: "orderly_poweroff" } +elf_symbol { + id: 0x33abdf76 + name: "osq_lock" + is_defined: true + symbol_type: FUNCTION + crc: 0x1f8df714 + type_id: 0xf94be4cb + full_name: "osq_lock" +} +elf_symbol { + id: 0x2ffaca4e + name: "osq_unlock" + is_defined: true + symbol_type: FUNCTION + crc: 0xa1d5320d + type_id: 0x1777ee5c + full_name: "osq_unlock" +} elf_symbol { id: 0x57ecab02 name: "out_of_line_wait_on_bit" @@ -407150,9 +407227,11 @@ interface { symbol_id: 0xfb1b8d64 symbol_id: 0xc56d7179 symbol_id: 0x5858f827 + symbol_id: 0xaa42999f symbol_id: 0xb1847a6f symbol_id: 0x958d8cdb symbol_id: 0xfde8086a + symbol_id: 0x66c7c5f3 symbol_id: 0xbffefc2b symbol_id: 0x3d83999a symbol_id: 0xf7bca936 @@ -407622,9 +407701,11 @@ interface { symbol_id: 0xf01f02ea symbol_id: 0xeaebbadf symbol_id: 0xe471b8d5 + symbol_id: 0x403cf0dd symbol_id: 0x84628825 symbol_id: 0x8d0ce77d symbol_id: 0x50ab483c + symbol_id: 0x8866a3b9 symbol_id: 0xe7ef7059 symbol_id: 0xf2d006c8 symbol_id: 0x85a885d0 @@ -411401,6 +411482,8 @@ interface { symbol_id: 0xa50d59b7 symbol_id: 0x224cb6b2 symbol_id: 0x67bce5b1 + symbol_id: 0x33abdf76 + symbol_id: 0x2ffaca4e symbol_id: 0x57ecab02 symbol_id: 0x27d0c7b6 symbol_id: 0x3d8c84e9 diff --git a/android/abi_gki_aarch64_oplus b/android/abi_gki_aarch64_oplus index 2205fef8484e..9818f1f154b7 100644 --- a/android/abi_gki_aarch64_oplus +++ b/android/abi_gki_aarch64_oplus @@ -63,6 +63,8 @@ nf_unregister_net_hooks nr_running of_css + osq_lock + osq_unlock __page_file_index __page_mapcount pm_get_active_wakeup_sources @@ -169,6 +171,8 @@ __traceiter_android_vh_rtmutex_waiter_prio __traceiter_android_vh_rt_mutex_steal __traceiter_android_vh_rwsem_can_spin_on_owner + __traceiter_android_vh_rwsem_direct_rsteal + __traceiter_android_vh_rwsem_optimistic_rspin __traceiter_android_vh_rwsem_opt_spin_finish __traceiter_android_vh_rwsem_opt_spin_start __traceiter_android_vh_rwsem_wake_finish @@ -278,6 +282,8 @@ __tracepoint_android_vh_rtmutex_waiter_prio __tracepoint_android_vh_rt_mutex_steal __tracepoint_android_vh_rwsem_can_spin_on_owner + __tracepoint_android_vh_rwsem_direct_rsteal + __tracepoint_android_vh_rwsem_optimistic_rspin __tracepoint_android_vh_rwsem_opt_spin_finish __tracepoint_android_vh_rwsem_opt_spin_start __tracepoint_android_vh_rwsem_wake_finish From fa87a072a7fccf51d1c23869af2cb6b423e7b38a Mon Sep 17 00:00:00 2001 From: Norihiko Hama Date: Fri, 19 Apr 2024 10:01:37 +0900 Subject: [PATCH 41/86] ANDROID: GKI: Update honda symbol list for led-trigger Add some missing symbols required for led-trigger 2 function symbol(s) added 'u32* led_get_default_pattern(struct led_classdev*, unsigned int*)' 'void led_set_brightness(struct led_classdev*, unsigned int)' Bug: 333795249 Change-Id: I9935592d63175a2328c2b8a95556fd3ee6898fdd Signed-off-by: Norihiko Hama --- android/abi_gki_aarch64.stg | 26 ++++++++++++++++++++++++++ android/abi_gki_aarch64_honda | 2 ++ 2 files changed, 28 insertions(+) diff --git a/android/abi_gki_aarch64.stg b/android/abi_gki_aarch64.stg index c321dfae30ca..6bc0a1ebd43b 100644 --- a/android/abi_gki_aarch64.stg +++ b/android/abi_gki_aarch64.stg @@ -307272,6 +307272,12 @@ function { parameter_id: 0x6720d32f parameter_id: 0x1bf16028 } +function { + id: 0x6f6bc9a5 + return_type_id: 0x38d23361 + parameter_id: 0x32caaf24 + parameter_id: 0x1bf16028 +} function { id: 0x6f9cf068 return_type_id: 0x1d44326e @@ -375985,6 +375991,15 @@ elf_symbol { type_id: 0x1c2afa5a full_name: "led_classdev_unregister" } +elf_symbol { + id: 0x61740d72 + name: "led_get_default_pattern" + is_defined: true + symbol_type: FUNCTION + crc: 0x34241cd4 + type_id: 0x6f6bc9a5 + full_name: "led_get_default_pattern" +} elf_symbol { id: 0x62cd5012 name: "led_get_flash_fault" @@ -376012,6 +376027,15 @@ elf_symbol { type_id: 0x927aedd4 full_name: "led_mc_calc_color_components" } +elf_symbol { + id: 0xa5be7046 + name: "led_set_brightness" + is_defined: true + symbol_type: FUNCTION + crc: 0x0cc5de59 + type_id: 0x1d3cefc2 + full_name: "led_set_brightness" +} elf_symbol { id: 0xbea2c272 name: "led_set_brightness_nosleep" @@ -410887,9 +410911,11 @@ interface { symbol_id: 0xa58e6f5d symbol_id: 0xfe24eb71 symbol_id: 0x27d77d55 + symbol_id: 0x61740d72 symbol_id: 0x62cd5012 symbol_id: 0xda95494e symbol_id: 0x76f00da5 + symbol_id: 0xa5be7046 symbol_id: 0xbea2c272 symbol_id: 0x2e74d698 symbol_id: 0x7bbb8d5f diff --git a/android/abi_gki_aarch64_honda b/android/abi_gki_aarch64_honda index 4153b8ae27e9..3eef62046e28 100644 --- a/android/abi_gki_aarch64_honda +++ b/android/abi_gki_aarch64_honda @@ -11,6 +11,8 @@ # commonly required by ledtrig-*.ko modules led_blink_set_oneshot + led_get_default_pattern + led_set_brightness led_set_brightness_nosleep led_stop_software_blink From d9604db041c41a84672490c88fb9f16d08daf173 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Fri, 12 Apr 2024 15:26:59 +0100 Subject: [PATCH 42/86] FROMGIT: coresight: etm4x: Do not hardcode IOMEM access for register restore When we restore the register state for ETM4x, while coming back from CPU idle, we hardcode IOMEM access. This is wrong and could blow up for an ETM with system instructions access (and for ETE). Fixes: f5bd523690d2 ("coresight: etm4x: Convert all register accesses") Reported-by: Yabin Cui Reviewed-by: Mike Leach Signed-off-by: Suzuki K Poulose Tested-by: Yabin Cui Link: https://lore.kernel.org/r/20240412142702.2882478-2-suzuki.poulose@arm.com Bug: 335234033 (cherry picked from commit 1e7ba33fa591de1cf60afffcabb45600b3607025 https://git.kernel.org/pub/scm/linux/kernel/git/coresight/linux.git next) Change-Id: Id2ea066374933de51a90f1fca8304338b741845d Signed-off-by: Yabin Cui --- drivers/hwtracing/coresight/coresight-etm4x-core.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index fda48a0afc1a..6d80fb59b35f 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -1726,8 +1726,10 @@ static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata) { int i; struct etmv4_save_state *state = drvdata->save_state; - struct csdev_access tmp_csa = CSDEV_ACCESS_IOMEM(drvdata->base); - struct csdev_access *csa = &tmp_csa; + struct csdev_access *csa = &drvdata->csdev->access; + + if (WARN_ON(!drvdata->csdev)) + return; etm4_cs_unlock(drvdata, csa); etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET); From f401cce7d9c187d0437db645ccf5465591e7bb12 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Fri, 12 Apr 2024 15:27:00 +0100 Subject: [PATCH 43/86] FROMGIT: coresight: etm4x: Do not save/restore Data trace control registers ETM4x doesn't support Data trace on A class CPUs. As such do not access the Data trace control registers during CPU idle. This could cause problems for ETE. While at it, remove all references to the Data trace control registers. Fixes: f188b5e76aae ("coresight: etm4x: Save/restore state across CPU low power states") Reported-by: Yabin Cui Reviewed-by: Mike Leach Signed-off-by: Suzuki K Poulose Tested-by: Yabin Cui Link: https://lore.kernel.org/r/20240412142702.2882478-3-suzuki.poulose@arm.com Bug: 335234033 (cherry picked from commit 5eb3a0c2c52368cb9902e9a6ea04888e093c487d https://git.kernel.org/pub/scm/linux/kernel/git/coresight/linux.git next) Change-Id: I06977d86aa2d876d166db0fac8fbccf48fd07229 Signed-off-by: Yabin Cui --- .../coresight/coresight-etm4x-core.c | 6 ---- drivers/hwtracing/coresight/coresight-etm4x.h | 28 ------------------- 2 files changed, 34 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 6d80fb59b35f..587506a83e91 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -1622,9 +1622,6 @@ static int __etm4_cpu_save(struct etmv4_drvdata *drvdata) state->trcvissctlr = etm4x_read32(csa, TRCVISSCTLR); if (drvdata->nr_pe_cmp) state->trcvipcssctlr = etm4x_read32(csa, TRCVIPCSSCTLR); - state->trcvdctlr = etm4x_read32(csa, TRCVDCTLR); - state->trcvdsacctlr = etm4x_read32(csa, TRCVDSACCTLR); - state->trcvdarcctlr = etm4x_read32(csa, TRCVDARCCTLR); for (i = 0; i < drvdata->nrseqstate - 1; i++) state->trcseqevr[i] = etm4x_read32(csa, TRCSEQEVRn(i)); @@ -1755,9 +1752,6 @@ static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata) etm4x_relaxed_write32(csa, state->trcvissctlr, TRCVISSCTLR); if (drvdata->nr_pe_cmp) etm4x_relaxed_write32(csa, state->trcvipcssctlr, TRCVIPCSSCTLR); - etm4x_relaxed_write32(csa, state->trcvdctlr, TRCVDCTLR); - etm4x_relaxed_write32(csa, state->trcvdsacctlr, TRCVDSACCTLR); - etm4x_relaxed_write32(csa, state->trcvdarcctlr, TRCVDARCCTLR); for (i = 0; i < drvdata->nrseqstate - 1; i++) etm4x_relaxed_write32(csa, state->trcseqevr[i], TRCSEQEVRn(i)); diff --git a/drivers/hwtracing/coresight/coresight-etm4x.h b/drivers/hwtracing/coresight/coresight-etm4x.h index d8e4d902b01a..ff2a643fb54d 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x.h +++ b/drivers/hwtracing/coresight/coresight-etm4x.h @@ -43,9 +43,6 @@ #define TRCVIIECTLR 0x084 #define TRCVISSCTLR 0x088 #define TRCVIPCSSCTLR 0x08C -#define TRCVDCTLR 0x0A0 -#define TRCVDSACCTLR 0x0A4 -#define TRCVDARCCTLR 0x0A8 /* Derived resources registers */ #define TRCSEQEVRn(n) (0x100 + (n * 4)) /* n = 0-2 */ #define TRCSEQRSTEVR 0x118 @@ -90,9 +87,6 @@ /* Address Comparator registers n = 0-15 */ #define TRCACVRn(n) (0x400 + (n * 8)) #define TRCACATRn(n) (0x480 + (n * 8)) -/* Data Value Comparator Value registers, n = 0-7 */ -#define TRCDVCVRn(n) (0x500 + (n * 16)) -#define TRCDVCMRn(n) (0x580 + (n * 16)) /* ContextID/Virtual ContextID comparators, n = 0-7 */ #define TRCCIDCVRn(n) (0x600 + (n * 8)) #define TRCVMIDCVRn(n) (0x640 + (n * 8)) @@ -272,9 +266,6 @@ /* List of registers accessible via System instructions */ #define ETM4x_ONLY_SYSREG_LIST(op, val) \ CASE_##op((val), TRCPROCSELR) \ - CASE_##op((val), TRCVDCTLR) \ - CASE_##op((val), TRCVDSACCTLR) \ - CASE_##op((val), TRCVDARCCTLR) \ CASE_##op((val), TRCOSLAR) #define ETM_COMMON_SYSREG_LIST(op, val) \ @@ -422,22 +413,6 @@ CASE_##op((val), TRCACATRn(13)) \ CASE_##op((val), TRCACATRn(14)) \ CASE_##op((val), TRCACATRn(15)) \ - CASE_##op((val), TRCDVCVRn(0)) \ - CASE_##op((val), TRCDVCVRn(1)) \ - CASE_##op((val), TRCDVCVRn(2)) \ - CASE_##op((val), TRCDVCVRn(3)) \ - CASE_##op((val), TRCDVCVRn(4)) \ - CASE_##op((val), TRCDVCVRn(5)) \ - CASE_##op((val), TRCDVCVRn(6)) \ - CASE_##op((val), TRCDVCVRn(7)) \ - CASE_##op((val), TRCDVCMRn(0)) \ - CASE_##op((val), TRCDVCMRn(1)) \ - CASE_##op((val), TRCDVCMRn(2)) \ - CASE_##op((val), TRCDVCMRn(3)) \ - CASE_##op((val), TRCDVCMRn(4)) \ - CASE_##op((val), TRCDVCMRn(5)) \ - CASE_##op((val), TRCDVCMRn(6)) \ - CASE_##op((val), TRCDVCMRn(7)) \ CASE_##op((val), TRCCIDCVRn(0)) \ CASE_##op((val), TRCCIDCVRn(1)) \ CASE_##op((val), TRCCIDCVRn(2)) \ @@ -905,9 +880,6 @@ struct etmv4_save_state { u32 trcviiectlr; u32 trcvissctlr; u32 trcvipcssctlr; - u32 trcvdctlr; - u32 trcvdsacctlr; - u32 trcvdarcctlr; u32 trcseqevr[ETM_MAX_SEQ_STATES]; u32 trcseqrstevr; From 7ff054397ae0ac51d487c8d95a22f0d43507bfa8 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Fri, 12 Apr 2024 15:27:01 +0100 Subject: [PATCH 44/86] FROMGIT: coresight: etm4x: Safe access for TRCQCLTR ETM4x implements TRCQCLTR only when the Q elements are supported and the Q element filtering is supported (TRCIDR0.QFILT). Access to the register otherwise could be fatal. Fix this by tracking the availability, like the others. Fixes: f188b5e76aae ("coresight: etm4x: Save/restore state across CPU low power states") Reported-by: Yabin Cui Reviewed-by: Mike Leach Signed-off-by: Suzuki K Poulose Tested-by: Yabin Cui Link: https://lore.kernel.org/r/20240412142702.2882478-4-suzuki.poulose@arm.com Bug: 335234033 (cherry picked from commit 46bf8d7cd8530eca607379033b9bc4ac5590a0cd https://git.kernel.org/pub/scm/linux/kernel/git/coresight/linux.git next) Change-Id: Id848fa14ba8003149f76b5ca54562593f6164150 Signed-off-by: Yabin Cui --- drivers/hwtracing/coresight/coresight-etm4x-core.c | 8 ++++++-- drivers/hwtracing/coresight/coresight-etm4x.h | 3 +++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 587506a83e91..802aa0fad59e 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -1120,6 +1120,8 @@ static void etm4_init_arch_data(void *info) drvdata->nr_event = FIELD_GET(TRCIDR0_NUMEVENT_MASK, etmidr0); /* QSUPP, bits[16:15] Q element support field */ drvdata->q_support = FIELD_GET(TRCIDR0_QSUPP_MASK, etmidr0); + if (drvdata->q_support) + drvdata->q_filt = !!(etmidr0 & TRCIDR0_QFILT); /* TSSIZE, bits[28:24] Global timestamp size field */ drvdata->ts_size = FIELD_GET(TRCIDR0_TSSIZE_MASK, etmidr0); @@ -1615,7 +1617,8 @@ static int __etm4_cpu_save(struct etmv4_drvdata *drvdata) state->trcccctlr = etm4x_read32(csa, TRCCCCTLR); state->trcbbctlr = etm4x_read32(csa, TRCBBCTLR); state->trctraceidr = etm4x_read32(csa, TRCTRACEIDR); - state->trcqctlr = etm4x_read32(csa, TRCQCTLR); + if (drvdata->q_filt) + state->trcqctlr = etm4x_read32(csa, TRCQCTLR); state->trcvictlr = etm4x_read32(csa, TRCVICTLR); state->trcviiectlr = etm4x_read32(csa, TRCVIIECTLR); @@ -1745,7 +1748,8 @@ static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata) etm4x_relaxed_write32(csa, state->trcccctlr, TRCCCCTLR); etm4x_relaxed_write32(csa, state->trcbbctlr, TRCBBCTLR); etm4x_relaxed_write32(csa, state->trctraceidr, TRCTRACEIDR); - etm4x_relaxed_write32(csa, state->trcqctlr, TRCQCTLR); + if (drvdata->q_filt) + etm4x_relaxed_write32(csa, state->trcqctlr, TRCQCTLR); etm4x_relaxed_write32(csa, state->trcvictlr, TRCVICTLR); etm4x_relaxed_write32(csa, state->trcviiectlr, TRCVIIECTLR); diff --git a/drivers/hwtracing/coresight/coresight-etm4x.h b/drivers/hwtracing/coresight/coresight-etm4x.h index ff2a643fb54d..31754173091b 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x.h +++ b/drivers/hwtracing/coresight/coresight-etm4x.h @@ -135,6 +135,7 @@ #define TRCIDR0_TRCCCI BIT(7) #define TRCIDR0_RETSTACK BIT(9) #define TRCIDR0_NUMEVENT_MASK GENMASK(11, 10) +#define TRCIDR0_QFILT BIT(14) #define TRCIDR0_QSUPP_MASK GENMASK(16, 15) #define TRCIDR0_TSSIZE_MASK GENMASK(28, 24) @@ -951,6 +952,7 @@ struct etmv4_save_state { * @os_unlock: True if access to management registers is allowed. * @instrp0: Tracing of load and store instructions * as P0 elements is supported. + * @q_filt: Q element filtering support, if Q elements are supported. * @trcbb: Indicates if the trace unit supports branch broadcast tracing. * @trccond: If the trace unit supports conditional * instruction tracing. @@ -1013,6 +1015,7 @@ struct etmv4_drvdata { bool boot_enable; bool os_unlock; bool instrp0; + bool q_filt; bool trcbb; bool trccond; bool retstack; From 08cc4037cf3245eb07b7cbd2d5c90796f6ab3dfd Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Fri, 12 Apr 2024 15:27:02 +0100 Subject: [PATCH 45/86] FROMGIT: coresight: etm4x: Fix access to resource selector registers Resource selector pair 0 is always implemented and reserved. We must not touch it, even during save/restore for CPU Idle. Rest of the driver is well behaved. Fix the offending ones. Reported-by: Yabin Cui Fixes: f188b5e76aae ("coresight: etm4x: Save/restore state across CPU low power states") Signed-off-by: Suzuki K Poulose Tested-by: Yabin Cui Reviewed-by: Mike Leach Link: https://lore.kernel.org/r/20240412142702.2882478-5-suzuki.poulose@arm.com Bug: 335234033 (cherry picked from commit d6fc00d0f640d6010b51054aa8b0fd191177dbc9 https://git.kernel.org/pub/scm/linux/kernel/git/coresight/linux.git next) Change-Id: I5f3385cb269969a299402fa258b30ab43e95805f Signed-off-by: Yabin Cui --- drivers/hwtracing/coresight/coresight-etm4x-core.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 802aa0fad59e..fe0f168da7c4 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -1641,7 +1641,8 @@ static int __etm4_cpu_save(struct etmv4_drvdata *drvdata) state->trccntvr[i] = etm4x_read32(csa, TRCCNTVRn(i)); } - for (i = 0; i < drvdata->nr_resource * 2; i++) + /* Resource selector pair 0 is reserved */ + for (i = 2; i < drvdata->nr_resource * 2; i++) state->trcrsctlr[i] = etm4x_read32(csa, TRCRSCTLRn(i)); for (i = 0; i < drvdata->nr_ss_cmp; i++) { @@ -1772,7 +1773,8 @@ static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata) etm4x_relaxed_write32(csa, state->trccntvr[i], TRCCNTVRn(i)); } - for (i = 0; i < drvdata->nr_resource * 2; i++) + /* Resource selector pair 0 is reserved */ + for (i = 2; i < drvdata->nr_resource * 2; i++) etm4x_relaxed_write32(csa, state->trcrsctlr[i], TRCRSCTLRn(i)); for (i = 0; i < drvdata->nr_ss_cmp; i++) { From 4744b3a4ed7241fa7b282171d838caa0c82f9b46 Mon Sep 17 00:00:00 2001 From: Vincent Palomares Date: Mon, 13 Jun 2022 18:23:23 -0700 Subject: [PATCH 46/86] ANDROID: GKI: Expose device async to userspace Setting CONFIG_PM_ADVANCED_DEBUG=y to expose device async fields to userspace, allowing to fine-tune the suspend/resume path. Bug: 235135485 Change-Id: I75060e88ce0c1e199aa8740f446a2c0f8167f3d7 Signed-off-by: Vincent Palomares --- arch/arm64/configs/gki_defconfig | 2 ++ arch/x86/configs/gki_defconfig | 2 ++ 2 files changed, 4 insertions(+) diff --git a/arch/arm64/configs/gki_defconfig b/arch/arm64/configs/gki_defconfig index 869d9da6819e..502f707b59e6 100644 --- a/arch/arm64/configs/gki_defconfig +++ b/arch/arm64/configs/gki_defconfig @@ -68,6 +68,8 @@ CONFIG_HIBERNATION=y CONFIG_PM_WAKELOCKS=y CONFIG_PM_WAKELOCKS_LIMIT=0 # CONFIG_PM_WAKELOCKS_GC is not set +CONFIG_PM_DEBUG=y +CONFIG_PM_ADVANCED_DEBUG=y CONFIG_ENERGY_MODEL=y CONFIG_CPU_IDLE=y CONFIG_CPU_IDLE_GOV_MENU=y diff --git a/arch/x86/configs/gki_defconfig b/arch/x86/configs/gki_defconfig index 23efcc88be4b..debf9895c937 100644 --- a/arch/x86/configs/gki_defconfig +++ b/arch/x86/configs/gki_defconfig @@ -67,6 +67,8 @@ CONFIG_HIBERNATION=y CONFIG_PM_WAKELOCKS=y CONFIG_PM_WAKELOCKS_LIMIT=0 # CONFIG_PM_WAKELOCKS_GC is not set +CONFIG_PM_DEBUG=y +CONFIG_PM_ADVANCED_DEBUG=y # CONFIG_ACPI_AC is not set # CONFIG_ACPI_BATTERY is not set # CONFIG_ACPI_FAN is not set From 6364d594125d5489b4f160c055505ec08c68c4eb Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 4 Apr 2024 09:51:05 +0100 Subject: [PATCH 47/86] ANDROID: KVM: arm64: wait_for_initramfs for pKVM module loading procfs Of course, the initramfs needs to be ready before procfs can be mounted... in the initramfs. While at it, only mount if a pKVM module must be loaded and only print a warning in case of failure. Bug: 278749606 Bug: 301483379 Bug: 331152809 Change-Id: Ie56bd26d4575f69cb1f06ba6317a098649f6da44 Reported-by: Mankyum Kim Signed-off-by: Vincent Donnefort (cherry picked from commit 7d5843b59548672c23c977b4666c3779d31695fb) --- arch/arm64/kvm/pkvm.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 10b899483d43..08ce8d77fe3d 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -4,6 +4,8 @@ * Author: Quentin Perret */ +#include +#include #include #include #include @@ -627,6 +629,7 @@ static int __init __pkvm_request_early_module(char *module_name, "PATH=/sbin:/usr/sbin:/bin:/usr/bin", NULL }; + static bool proc; char **argv; int idx = 0; @@ -658,6 +661,15 @@ static int __init __pkvm_request_early_module(char *module_name, /* Even with CONFIG_STATIC_USERMODEHELPER we really want this path */ info->path = modprobe_path; + if (!proc) { + wait_for_initramfs(); + if (init_mount("proc", "/proc", "proc", + MS_SILENT | MS_NOEXEC | MS_NOSUID, NULL)) + pr_warn("Couldn't mount /proc, pKVM module parameters will be ignored\n"); + + proc = true; + } + return call_usermodehelper_exec(info, UMH_WAIT_PROC | UMH_KILLABLE); err: kfree(argv); @@ -686,11 +698,7 @@ int __init pkvm_load_early_modules(void) { char *token, *buf = early_pkvm_modules; char *module_path = CONFIG_PKVM_MODULE_PATH; - int err = init_mount("proc", "/proc", "proc", - MS_SILENT | MS_NOEXEC | MS_NOSUID, NULL); - - if (err) - return err; + int err; while (true) { token = strsep(&buf, ","); From 14f07c1db0daa8dffab64ae08342ebbaf5828367 Mon Sep 17 00:00:00 2001 From: xieliujie Date: Fri, 26 Apr 2024 10:02:17 +0800 Subject: [PATCH 48/86] ANDROID: vendor_hooks: add two hooks for lazy preemption add some changes to achieve the lazy preemption feature in our baseline. - android_vh_read_lazy_flag - android_vh_set_tsk_need_resched_lazy Bug: 336982374 Change-Id: I09f1110a2a11da4dbf0d4d0cca3500d1a6ee6a74 Signed-off-by: xieliujie --- arch/arm64/kernel/signal.c | 7 ++++++- drivers/android/vendor_hooks.c | 2 ++ include/trace/hooks/dtask.h | 8 ++++++++ kernel/sched/core.c | 6 +++++- 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index 82f4572c8ddf..9fc32366ac7e 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -20,6 +20,8 @@ #include #include +#include + #include #include #include @@ -1107,8 +1109,11 @@ static void do_signal(struct pt_regs *regs) void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags) { + int thread_lazy_flag = 0; + do { - if (thread_flags & _TIF_NEED_RESCHED) { + trace_android_vh_read_lazy_flag(&thread_lazy_flag, &thread_flags); + if ((thread_flags & _TIF_NEED_RESCHED) || thread_lazy_flag) { /* Unmask Debug and SError for the next task */ local_daif_restore(DAIF_PROCCTX_NOIRQ); diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 7a600198d24b..675646d924e7 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -410,3 +410,5 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_usb_dev_resume); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_sound_usb_support_cpu_suspend); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_hibernated_do_mem_alloc); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_hibernate_save_cmp_len); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_read_lazy_flag); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_tsk_need_resched_lazy); diff --git a/include/trace/hooks/dtask.h b/include/trace/hooks/dtask.h index b51147089b2d..ecf3f4b03ab7 100644 --- a/include/trace/hooks/dtask.h +++ b/include/trace/hooks/dtask.h @@ -124,6 +124,14 @@ DECLARE_HOOK(android_vh_freeze_whether_wake, TP_PROTO(struct task_struct *t, bool *wake), TP_ARGS(t, wake)); +DECLARE_HOOK(android_vh_read_lazy_flag, + TP_PROTO(int *thread_lazy_flag, unsigned long *thread_flags), + TP_ARGS(thread_lazy_flag, thread_flags)); + +DECLARE_HOOK(android_vh_set_tsk_need_resched_lazy, + TP_PROTO(struct task_struct *p, struct rq *rq, int *need_lazy), + TP_ARGS(p, rq, need_lazy)); + #endif /* _TRACE_HOOK_DTASK_H */ /* This part must be outside protection */ diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 922c52aeeeb7..20080b633588 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -1043,13 +1043,17 @@ void wake_up_q(struct wake_q_head *head) void resched_curr(struct rq *rq) { struct task_struct *curr = rq->curr; - int cpu; + int cpu, need_lazy = 0; lockdep_assert_rq_held(rq); if (test_tsk_need_resched(curr)) return; + trace_android_vh_set_tsk_need_resched_lazy(curr, rq, &need_lazy); + if (need_lazy) + return; + cpu = cpu_of(rq); if (cpu == smp_processor_id()) { From 1d37bc9913cc0fab656e4f14274e20587fbf2396 Mon Sep 17 00:00:00 2001 From: xieliujie Date: Sun, 28 Apr 2024 10:56:14 +0800 Subject: [PATCH 49/86] ANDROID: vendor_hooks: add symbols for lazy preemption add some symnols to achieve the lazy preemption feature in our baseline. - __traceiter_android_vh_read_lazy_flag - __traceiter_android_vh_set_tsk_need_resched_lazy - __tracepoint_android_vh_read_lazy_flag - __tracepoint_android_vh_set_tsk_need_resched_lazy Bug: 336982374 Change-Id: I7807617575da9365edd2e8fccd01a22913aaffc1 Signed-off-by: xieliujie --- android/abi_gki_aarch64.stg | 47 +++++++++++++++++++++++++++++++++++ android/abi_gki_aarch64_oplus | 4 +++ 2 files changed, 51 insertions(+) diff --git a/android/abi_gki_aarch64.stg b/android/abi_gki_aarch64.stg index 6bc0a1ebd43b..1c01560d1a91 100644 --- a/android/abi_gki_aarch64.stg +++ b/android/abi_gki_aarch64.stg @@ -325161,6 +325161,13 @@ function { parameter_id: 0x18bd6530 parameter_id: 0x13580d6c } +function { + id: 0x9be3490e + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x13580d6c + parameter_id: 0x064d6086 +} function { id: 0x9be40c05 return_type_id: 0x6720d32f @@ -342875,6 +342882,15 @@ elf_symbol { type_id: 0x9b3343fb full_name: "__traceiter_android_vh_ra_tuning_max_page" } +elf_symbol { + id: 0x96662dde + name: "__traceiter_android_vh_read_lazy_flag" + is_defined: true + symbol_type: FUNCTION + crc: 0x0b137c49 + type_id: 0x9be3490e + full_name: "__traceiter_android_vh_read_lazy_flag" +} elf_symbol { id: 0xb35da0ec name: "__traceiter_android_vh_read_pages" @@ -343226,6 +343242,15 @@ elf_symbol { type_id: 0x9ba62ef4 full_name: "__traceiter_android_vh_sd_update_bus_speed_mode" } +elf_symbol { + id: 0x874fcdb2 + name: "__traceiter_android_vh_set_tsk_need_resched_lazy" + is_defined: true + symbol_type: FUNCTION + crc: 0x178a43de + type_id: 0x9bd01103 + full_name: "__traceiter_android_vh_set_tsk_need_resched_lazy" +} elf_symbol { id: 0xaa3f6a65 name: "__traceiter_android_vh_set_wake_flags" @@ -347141,6 +347166,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_ra_tuning_max_page" } +elf_symbol { + id: 0x2d9a331c + name: "__tracepoint_android_vh_read_lazy_flag" + is_defined: true + symbol_type: OBJECT + crc: 0xfac8f7c6 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_read_lazy_flag" +} elf_symbol { id: 0x9fc2933e name: "__tracepoint_android_vh_read_pages" @@ -347492,6 +347526,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_sd_update_bus_speed_mode" } +elf_symbol { + id: 0xb6de2d88 + name: "__tracepoint_android_vh_set_tsk_need_resched_lazy" + is_defined: true + symbol_type: OBJECT + crc: 0xfd867ee2 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_set_tsk_need_resched_lazy" +} elf_symbol { id: 0xd7ceb15f name: "__tracepoint_android_vh_set_wake_flags" @@ -407233,6 +407276,7 @@ interface { symbol_id: 0xf2c39651 symbol_id: 0x93303c51 symbol_id: 0x3a545b61 + symbol_id: 0x96662dde symbol_id: 0xb35da0ec symbol_id: 0x7d069e91 symbol_id: 0x0fa39b81 @@ -407272,6 +407316,7 @@ interface { symbol_id: 0xedb34a44 symbol_id: 0xb841c45e symbol_id: 0xe4ec133d + symbol_id: 0x874fcdb2 symbol_id: 0xaa3f6a65 symbol_id: 0xa2fe718f symbol_id: 0x98a943de @@ -407707,6 +407752,7 @@ interface { symbol_id: 0x0e92ee53 symbol_id: 0xb0c197a3 symbol_id: 0x811d5fab + symbol_id: 0x2d9a331c symbol_id: 0x9fc2933e symbol_id: 0x761f292f symbol_id: 0xef7ad117 @@ -407746,6 +407792,7 @@ interface { symbol_id: 0xaaf6a826 symbol_id: 0xff2bccb8 symbol_id: 0xcfab6e83 + symbol_id: 0xb6de2d88 symbol_id: 0xd7ceb15f symbol_id: 0x923147c1 symbol_id: 0x5cfdecb8 diff --git a/android/abi_gki_aarch64_oplus b/android/abi_gki_aarch64_oplus index 9818f1f154b7..610e9fcf22f7 100644 --- a/android/abi_gki_aarch64_oplus +++ b/android/abi_gki_aarch64_oplus @@ -168,6 +168,7 @@ __traceiter_android_vh_cleanup_old_buffers_bypass __traceiter_android_vh_dm_bufio_shrink_scan_bypass __traceiter_android_vh_mutex_unlock_slowpath + __traceiter_android_vh_read_lazy_flag __traceiter_android_vh_rtmutex_waiter_prio __traceiter_android_vh_rt_mutex_steal __traceiter_android_vh_rwsem_can_spin_on_owner @@ -182,6 +183,7 @@ __traceiter_android_vh_sched_stat_runtime_rt __traceiter_android_vh_shrink_node_memcgs __traceiter_android_vh_sync_txn_recvd + __traceiter_android_vh_set_tsk_need_resched_lazy __traceiter_android_vh_task_blocks_on_rtmutex __traceiter_block_bio_queue __traceiter_block_getrq @@ -277,6 +279,7 @@ __tracepoint_android_vh_record_mutex_lock_starttime __tracepoint_android_vh_record_pcpu_rwsem_starttime __tracepoint_android_vh_percpu_rwsem_wq_add + __tracepoint_android_vh_read_lazy_flag __tracepoint_android_vh_record_rtmutex_lock_starttime __tracepoint_android_vh_record_rwsem_lock_starttime __tracepoint_android_vh_rtmutex_waiter_prio @@ -293,6 +296,7 @@ __tracepoint_android_vh_sched_stat_runtime_rt __tracepoint_android_vh_shrink_node_memcgs __tracepoint_android_vh_sync_txn_recvd + __tracepoint_android_vh_set_tsk_need_resched_lazy __tracepoint_android_vh_task_blocks_on_rtmutex __tracepoint_android_vh_test_clear_look_around_ref __tracepoint_android_vh_tune_swappiness From 6657c436ed73d9e7974a2e77a5fe572e63bf353f Mon Sep 17 00:00:00 2001 From: Badhri Jagan Sridharan Date: Sat, 27 Apr 2024 20:28:12 +0000 Subject: [PATCH 50/86] FROMGIT: usb: typec: tcpm: Check for port partner validity before consuming it typec_register_partner() does not guarantee partner registration to always succeed. In the event of failure, port->partner is set to the error value or NULL. Given that port->partner validity is not checked, this results in the following crash: Unable to handle kernel NULL pointer dereference at virtual address xx pc : run_state_machine+0x1bc8/0x1c08 lr : run_state_machine+0x1b90/0x1c08 .. Call trace: run_state_machine+0x1bc8/0x1c08 tcpm_state_machine_work+0x94/0xe4 kthread_worker_fn+0x118/0x328 kthread+0x1d0/0x23c ret_from_fork+0x10/0x20 To prevent the crash, check for port->partner validity before derefencing it in all the call sites. Cc: stable@vger.kernel.org Fixes: c97cd0b4b54e ("usb: typec: tcpm: set initial svdm version based on pd revision") Signed-off-by: Badhri Jagan Sridharan Reviewed-by: Heikki Krogerus Reviewed-by: Dmitry Baryshkov Link: https://lore.kernel.org/r/20240427202812.3435268-1-badhri@google.com Signed-off-by: Greg Kroah-Hartman Bug: 321849121 (cherry picked from commit ae11f04b452b5205536e1c02d31f8045eba249dd https://kernel.googlesource.com/pub/scm/linux/kernel/git/gregkh/usb usb-linus) Change-Id: I01510c86e147b3011afc5d475fc1dc38d2636a60 Signed-off-by: Zheng Pan --- drivers/usb/typec/tcpm/tcpm.c | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index b6235a78615a..02af2f25fd01 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -1505,7 +1505,8 @@ static void svdm_consume_identity(struct tcpm_port *port, const u32 *p, int cnt) port->partner_ident.cert_stat = p[VDO_INDEX_CSTAT]; port->partner_ident.product = product; - typec_partner_set_identity(port->partner); + if (port->partner) + typec_partner_set_identity(port->partner); tcpm_log(port, "Identity: %04x:%04x.%04x", PD_IDH_VID(vdo), @@ -1593,6 +1594,9 @@ static void tcpm_register_partner_altmodes(struct tcpm_port *port) struct typec_altmode *altmode; int i; + if (!port->partner) + return; + for (i = 0; i < modep->altmodes; i++) { altmode = typec_partner_register_altmode(port->partner, &modep->altmode_desc[i]); @@ -3591,7 +3595,10 @@ static int tcpm_init_vconn(struct tcpm_port *port) static void tcpm_typec_connect(struct tcpm_port *port) { + struct typec_partner *partner; + if (!port->connected) { + port->connected = true; /* Make sure we don't report stale identity information */ memset(&port->partner_ident, 0, sizeof(port->partner_ident)); port->partner_desc.usb_pd = port->pd_capable; @@ -3601,9 +3608,13 @@ static void tcpm_typec_connect(struct tcpm_port *port) port->partner_desc.accessory = TYPEC_ACCESSORY_AUDIO; else port->partner_desc.accessory = TYPEC_ACCESSORY_NONE; - port->partner = typec_register_partner(port->typec_port, - &port->partner_desc); - port->connected = true; + partner = typec_register_partner(port->typec_port, &port->partner_desc); + if (IS_ERR(partner)) { + dev_err(port->dev, "Failed to register partner (%ld)\n", PTR_ERR(partner)); + return; + } + + port->partner = partner; typec_partner_set_usb_power_delivery(port->partner, port->partner_pd); } } @@ -3672,10 +3683,12 @@ out_disable_mux: static void tcpm_typec_disconnect(struct tcpm_port *port) { - if (port->connected) { - typec_partner_set_usb_power_delivery(port->partner, NULL); - typec_unregister_partner(port->partner); - port->partner = NULL; + if (port->partner) { + if (port->connected) { + typec_partner_set_usb_power_delivery(port->partner, NULL); + typec_unregister_partner(port->partner); + port->partner = NULL; + } port->connected = false; } } @@ -3891,6 +3904,9 @@ static enum typec_cc_status tcpm_pwr_opmode_to_rp(enum typec_pwr_opmode opmode) static void tcpm_set_initial_svdm_version(struct tcpm_port *port) { + if (!port->partner) + return; + switch (port->negotiated_rev) { case PD_REV30: break; From 13f322e958dfa27d9cfc75431329433552fb2da6 Mon Sep 17 00:00:00 2001 From: Zheng Pan Date: Wed, 1 May 2024 15:54:56 -0700 Subject: [PATCH 51/86] Revert "FROMGIT: usb: typec: tcpm: Check for port partner validity before consuming it" This reverts commit 6657c436ed73d9e7974a2e77a5fe572e63bf353f. Revert reason: patch has mistake and need resubmit Change-Id: Ic39b13cfe9b38d7bbbad2a99fa8e3eed44e1374b Signed-off-by: Zheng Pan --- drivers/usb/typec/tcpm/tcpm.c | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index 02af2f25fd01..b6235a78615a 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -1505,8 +1505,7 @@ static void svdm_consume_identity(struct tcpm_port *port, const u32 *p, int cnt) port->partner_ident.cert_stat = p[VDO_INDEX_CSTAT]; port->partner_ident.product = product; - if (port->partner) - typec_partner_set_identity(port->partner); + typec_partner_set_identity(port->partner); tcpm_log(port, "Identity: %04x:%04x.%04x", PD_IDH_VID(vdo), @@ -1594,9 +1593,6 @@ static void tcpm_register_partner_altmodes(struct tcpm_port *port) struct typec_altmode *altmode; int i; - if (!port->partner) - return; - for (i = 0; i < modep->altmodes; i++) { altmode = typec_partner_register_altmode(port->partner, &modep->altmode_desc[i]); @@ -3595,10 +3591,7 @@ static int tcpm_init_vconn(struct tcpm_port *port) static void tcpm_typec_connect(struct tcpm_port *port) { - struct typec_partner *partner; - if (!port->connected) { - port->connected = true; /* Make sure we don't report stale identity information */ memset(&port->partner_ident, 0, sizeof(port->partner_ident)); port->partner_desc.usb_pd = port->pd_capable; @@ -3608,13 +3601,9 @@ static void tcpm_typec_connect(struct tcpm_port *port) port->partner_desc.accessory = TYPEC_ACCESSORY_AUDIO; else port->partner_desc.accessory = TYPEC_ACCESSORY_NONE; - partner = typec_register_partner(port->typec_port, &port->partner_desc); - if (IS_ERR(partner)) { - dev_err(port->dev, "Failed to register partner (%ld)\n", PTR_ERR(partner)); - return; - } - - port->partner = partner; + port->partner = typec_register_partner(port->typec_port, + &port->partner_desc); + port->connected = true; typec_partner_set_usb_power_delivery(port->partner, port->partner_pd); } } @@ -3683,12 +3672,10 @@ out_disable_mux: static void tcpm_typec_disconnect(struct tcpm_port *port) { - if (port->partner) { - if (port->connected) { - typec_partner_set_usb_power_delivery(port->partner, NULL); - typec_unregister_partner(port->partner); - port->partner = NULL; - } + if (port->connected) { + typec_partner_set_usb_power_delivery(port->partner, NULL); + typec_unregister_partner(port->partner); + port->partner = NULL; port->connected = false; } } @@ -3904,9 +3891,6 @@ static enum typec_cc_status tcpm_pwr_opmode_to_rp(enum typec_pwr_opmode opmode) static void tcpm_set_initial_svdm_version(struct tcpm_port *port) { - if (!port->partner) - return; - switch (port->negotiated_rev) { case PD_REV30: break; From ea3c70fb959a921b517903f2ae7273a06c8d1aec Mon Sep 17 00:00:00 2001 From: Badhri Jagan Sridharan Date: Sat, 27 Apr 2024 20:28:12 +0000 Subject: [PATCH 52/86] FROMGIT: usb: typec: tcpm: Check for port partner validity before consuming it typec_register_partner() does not guarantee partner registration to always succeed. In the event of failure, port->partner is set to the error value or NULL. Given that port->partner validity is not checked, this results in the following crash: Unable to handle kernel NULL pointer dereference at virtual address xx pc : run_state_machine+0x1bc8/0x1c08 lr : run_state_machine+0x1b90/0x1c08 .. Call trace: run_state_machine+0x1bc8/0x1c08 tcpm_state_machine_work+0x94/0xe4 kthread_worker_fn+0x118/0x328 kthread+0x1d0/0x23c ret_from_fork+0x10/0x20 To prevent the crash, check for port->partner validity before derefencing it in all the call sites. Cc: stable@vger.kernel.org Fixes: c97cd0b4b54e ("usb: typec: tcpm: set initial svdm version based on pd revision") Signed-off-by: Badhri Jagan Sridharan Reviewed-by: Heikki Krogerus Reviewed-by: Dmitry Baryshkov Link: https://lore.kernel.org/r/20240427202812.3435268-1-badhri@google.com Signed-off-by: Greg Kroah-Hartman Bug: 321849121 (cherry picked from commit ae11f04b452b5205536e1c02d31f8045eba249dd https: //kernel.googlesource.com/pub/scm/linux/kernel/git/gregkh/usb usb-linus) Change-Id: I8e6d61816bd5ef22bc781e5d433e68ae078aac2d Signed-off-by: Zheng Pan --- drivers/usb/typec/tcpm/tcpm.c | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index b6235a78615a..3d2c05a60d85 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -1505,7 +1505,8 @@ static void svdm_consume_identity(struct tcpm_port *port, const u32 *p, int cnt) port->partner_ident.cert_stat = p[VDO_INDEX_CSTAT]; port->partner_ident.product = product; - typec_partner_set_identity(port->partner); + if (port->partner) + typec_partner_set_identity(port->partner); tcpm_log(port, "Identity: %04x:%04x.%04x", PD_IDH_VID(vdo), @@ -1593,6 +1594,9 @@ static void tcpm_register_partner_altmodes(struct tcpm_port *port) struct typec_altmode *altmode; int i; + if (!port->partner) + return; + for (i = 0; i < modep->altmodes; i++) { altmode = typec_partner_register_altmode(port->partner, &modep->altmode_desc[i]); @@ -3591,7 +3595,10 @@ static int tcpm_init_vconn(struct tcpm_port *port) static void tcpm_typec_connect(struct tcpm_port *port) { + struct typec_partner *partner; + if (!port->connected) { + port->connected = true; /* Make sure we don't report stale identity information */ memset(&port->partner_ident, 0, sizeof(port->partner_ident)); port->partner_desc.usb_pd = port->pd_capable; @@ -3601,9 +3608,13 @@ static void tcpm_typec_connect(struct tcpm_port *port) port->partner_desc.accessory = TYPEC_ACCESSORY_AUDIO; else port->partner_desc.accessory = TYPEC_ACCESSORY_NONE; - port->partner = typec_register_partner(port->typec_port, - &port->partner_desc); - port->connected = true; + partner = typec_register_partner(port->typec_port, &port->partner_desc); + if (IS_ERR(partner)) { + dev_err(port->dev, "Failed to register partner (%ld)\n", PTR_ERR(partner)); + return; + } + + port->partner = partner; typec_partner_set_usb_power_delivery(port->partner, port->partner_pd); } } @@ -3673,9 +3684,11 @@ out_disable_mux: static void tcpm_typec_disconnect(struct tcpm_port *port) { if (port->connected) { - typec_partner_set_usb_power_delivery(port->partner, NULL); - typec_unregister_partner(port->partner); - port->partner = NULL; + if (port->partner) { + typec_partner_set_usb_power_delivery(port->partner, NULL); + typec_unregister_partner(port->partner); + port->partner = NULL; + } port->connected = false; } } @@ -3891,6 +3904,9 @@ static enum typec_cc_status tcpm_pwr_opmode_to_rp(enum typec_pwr_opmode opmode) static void tcpm_set_initial_svdm_version(struct tcpm_port *port) { + if (!port->partner) + return; + switch (port->negotiated_rev) { case PD_REV30: break; From e79c1d45908063d1d7bc247e0e9bde6d9283a09e Mon Sep 17 00:00:00 2001 From: Kalesh Singh Date: Fri, 5 Apr 2024 11:00:40 -0700 Subject: [PATCH 53/86] ANDROID: 16K: Introduce /sys/kernel/mm/pgsize_miration/enabled Migrating from 4kB to 16kB page-size in Android requires first making the platform page-agnostic, which involves increasing Android-ELFs' max-page-size (p_align) from 4kB to 16kB. Increasing the ELF max-page-size was found to cause compatibility issues in apps that use obfuscation or depend on the ELF segments being mapped based on 4kB-alignment. Working around these compatibility issues involves both kernel and userspace (dynamic linker) changes. Introduce a knob for userspace (dynamic linker) to determine whether the kernel supports the mitigations needed for page-size migration compatibility. The knob also allows for userspace to turn on or off these mitigations by writing 1 or 0 to /sys/kernel/mm/pgsize_miration/enabled: echo 1 > /sys/kernel/mm//pgsize_miration/enabled # Enable echo 0 > /sys/kernel/mm//pgsize_miration/enabled # Disable Bug: 330117029 Bug: 327600007 Bug: 330767927 Bug: 328266487 Bug: 329803029 Change-Id: I9ac1d15d397b8226b27827ecffa30502da91e10e Signed-off-by: Kalesh Singh --- mm/Makefile | 3 +- mm/pgsize_migration.c | 105 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 mm/pgsize_migration.c diff --git a/mm/Makefile b/mm/Makefile index 6878edc7c7d0..9e6600740b80 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -54,7 +54,8 @@ obj-y := filemap.o mempool.o oom_kill.o fadvise.o \ mm_init.o percpu.o slab_common.o \ compaction.o \ interval_tree.o list_lru.o workingset.o \ - debug.o gup.o mmap_lock.o $(mmu-y) + debug.o gup.o mmap_lock.o pgsize_migration.o \ + $(mmu-y) # Give 'page_alloc' its own module-parameter namespace page-alloc-y := page_alloc.o diff --git a/mm/pgsize_migration.c b/mm/pgsize_migration.c new file mode 100644 index 000000000000..e840cda99e22 --- /dev/null +++ b/mm/pgsize_migration.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Page Size Migration + * + * This file contains the core logic of mitigations to ensure + * app compatibility during the transition from 4kB to 16kB + * page size in Android. + * + * Copyright (c) 2024, Google LLC. + * Author: Kalesh Singh + */ + +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_64BIT +#if PAGE_SIZE == SZ_4K +DEFINE_STATIC_KEY_TRUE(pgsize_migration_enabled); + +#define is_pgsize_migration_enabled() (static_branch_likely(&pgsize_migration_enabled)) +#else /* PAGE_SIZE != SZ_4K */ +DEFINE_STATIC_KEY_FALSE(pgsize_migration_enabled); + +#define is_pgsize_migration_enabled() (static_branch_unlikely(&pgsize_migration_enabled)) +#endif /* PAGE_SIZE == SZ_4K */ + +static ssize_t show_pgsize_migration_enabled(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + if (is_pgsize_migration_enabled()) + return sprintf(buf, "%d\n", 1); + else + return sprintf(buf, "%d\n", 0); +} + +static ssize_t store_pgsize_migration_enabled(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t n) +{ + unsigned long val; + + /* Migration is only applicable to 4kB kernels */ + if (PAGE_SIZE != SZ_4K) + return n; + + if (kstrtoul(buf, 10, &val)) + return -EINVAL; + + if (val > 1) + return -EINVAL; + + if (val == 1) + static_branch_enable(&pgsize_migration_enabled); + else if (val == 0) + static_branch_disable(&pgsize_migration_enabled); + + return n; +} + +static struct kobj_attribute pgsize_migration_enabled_attr = __ATTR( + enabled, + 0644, + show_pgsize_migration_enabled, + store_pgsize_migration_enabled +); + +static struct attribute *pgsize_migration_attrs[] = { + &pgsize_migration_enabled_attr.attr, + NULL +}; + +static struct attribute_group pgsize_migration_attr_group = { + .name = "pgsize_migration", + .attrs = pgsize_migration_attrs, +}; + +/** + * What: /sys/kernel/mm/pgsize_migration/enabled + * Date: April 2024 + * KernelVersion: v5.4+ (GKI kernels) + * Contact: Kalesh Singh + * Description: /sys/kernel/mm/pgsize_migration/enabled + * allows for userspace to turn on or off page size + * migration mitigations necessary for app compatibility + * during Android's transition from 4kB to 16kB page size. + * Such mitigations include preserving /proc//[s]maps + * output as if there was no segment extension by the + * dynamic loader; and preventing fault around in the padding + * sections of ELF LOAD segment mappings. + * Users: Bionic's dynamic linker + */ +static int __init init_pgsize_migration(void) +{ + if (sysfs_create_group(mm_kobj, &pgsize_migration_attr_group)) + pr_err("pgsize_migration: failed to create sysfs group\n"); + + return 0; +}; +late_initcall(init_pgsize_migration); +#endif /* CONFIG_64BIT */ From 6b9e4046753a61c8a5814419b3424778f34c0a9d Mon Sep 17 00:00:00 2001 From: Kalesh Singh Date: Thu, 4 Apr 2024 22:21:32 -0700 Subject: [PATCH 54/86] ANDROID: 16K: Introduce ELF padding representation for VMAs The dynamic linker may extend ELF LOAD segment mappings to be contiguous in memory when loading a 16kB compatible ELF on a 4kB page-size system. This is done to reduce the use of unreclaimable VMA slab memory for the otherwise necessary "gap" VMAs. The extended portion of the mapping (VMA) can be viewed as "padding", meaning that the mapping in that range corresponds to an area of the file that does not contain contents of the respective segments (maybe zero's depending on how the ELF is built). For some compatibility mitigations, the region of a VMA corresponding to these padding sections need to be known. In order to represent such regions without adding addtional overhead or breaking ABI, some upper bits of vm_flags are used. Add the VMA padding pages representation and the necessary APIs to manipulate it. Bug: 330117029 Bug: 327600007 Bug: 330767927 Bug: 328266487 Bug: 329803029 Change-Id: Ieb9fa98e30ec9b0bec62256624f14e3ed6062a75 Signed-off-by: Kalesh Singh --- include/linux/pgsize_migration.h | 64 ++++++++++++++++++++++++++++++++ mm/pgsize_migration.c | 22 ++++++++++- 2 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 include/linux/pgsize_migration.h diff --git a/include/linux/pgsize_migration.h b/include/linux/pgsize_migration.h new file mode 100644 index 000000000000..60f719d44107 --- /dev/null +++ b/include/linux/pgsize_migration.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_PAGE_SIZE_MIGRATION_H +#define _LINUX_PAGE_SIZE_MIGRATION_H + +/* + * Page Size Migration + * + * Copyright (c) 2024, Google LLC. + * Author: Kalesh Singh + * + * This file contains the APIs for mitigations to ensure + * app compatibility during the transition from 4kB to 16kB + * page size in Android. + */ + +#include +#include + +/* + * vm_flags representation of VMA padding pages. + * + * This allows the kernel to identify the portion of an ELF LOAD segment VMA + * that is padding. + * + * 4 high bits of vm_flags [63,60] are used to represent ELF segment padding + * up to 60kB, which is sufficient for ELFs of both 16kB and 64kB segment + * alignment (p_align). + * + * The representation is illustrated below. + * + * 63 62 61 60 + * _________ _________ _________ _________ + * | Bit 3 | Bit 2 | Bit 1 | Bit 0 | + * | of 4kB | of 4kB | of 4kB | of 4kB | + * | chunks | chunks | chunks | chunks | + * |_________|_________|_________|_________| + */ + +#define VM_PAD_WIDTH 4 +#define VM_PAD_SHIFT (BITS_PER_LONG - VM_PAD_WIDTH) +#define VM_TOTAL_PAD_PAGES ((1ULL << VM_PAD_WIDTH) - 1) + +#if PAGE_SIZE == SZ_4K && defined(CONFIG_64BIT) +extern void vma_set_pad_pages(struct vm_area_struct *vma, + unsigned long nr_pages); + +extern unsigned long vma_pad_pages(struct vm_area_struct *vma); +#else /* PAGE_SIZE != SZ_4K || !defined(CONFIG_64BIT) */ +static inline void vma_set_pad_pages(struct vm_area_struct *vma, + unsigned long nr_pages) +{ +} + +static inline unsigned long vma_pad_pages(struct vm_area_struct *vma) +{ + return 0; +} +#endif /* PAGE_SIZE == SZ_4K && defined(CONFIG_64BIT) */ + +static inline unsigned long vma_data_pages(struct vm_area_struct *vma) +{ + return vma_pages(vma) - vma_pad_pages(vma); +} +#endif /* _LINUX_PAGE_SIZE_MIGRATION_H */ diff --git a/mm/pgsize_migration.c b/mm/pgsize_migration.c index e840cda99e22..54b29a8269bf 100644 --- a/mm/pgsize_migration.c +++ b/mm/pgsize_migration.c @@ -10,11 +10,12 @@ * Author: Kalesh Singh */ +#include + #include #include #include #include -#include #include #ifdef CONFIG_64BIT @@ -102,4 +103,23 @@ static int __init init_pgsize_migration(void) return 0; }; late_initcall(init_pgsize_migration); + +#if PAGE_SIZE == SZ_4K +void vma_set_pad_pages(struct vm_area_struct *vma, + unsigned long nr_pages) +{ + if (!is_pgsize_migration_enabled()) + return; + + vm_flags_set(vma, nr_pages << VM_PAD_SHIFT); +} + +unsigned long vma_pad_pages(struct vm_area_struct *vma) +{ + if (!is_pgsize_migration_enabled()) + return 0; + + return vma->vm_flags >> VM_PAD_SHIFT; +} +#endif /* PAGE_SIZE == SZ_4K */ #endif /* CONFIG_64BIT */ From 6815ef31953426c6fea60b033481938baf860658 Mon Sep 17 00:00:00 2001 From: Kalesh Singh Date: Thu, 4 Apr 2024 22:21:32 -0700 Subject: [PATCH 55/86] ANDROID: 16K: Use MADV_DONTNEED to save VMA padding pages. When performing LOAD segment extension, the dynamic linker knows what portion of the VMA is padding. In order for the kernel to implement mitigations that ensure app compatibility, the extent of the padding must be made available to the kernel. To achieve this, reuse MADV_DONTNEED on single VMAs to hint the padding range to the kernel. This information is then stored in vm_flag bits. This allows userspace (dynamic linker) to set the padding pages on the VMA without a need for new out-of-tree UAPI. Bug: 330117029 Bug: 327600007 Bug: 330767927 Bug: 328266487 Bug: 329803029 Change-Id: I3421de32ab38ad3cb0fbce73ecbd8f7314287cde Signed-off-by: Kalesh Singh --- include/linux/pgsize_migration.h | 8 +++++ mm/madvise.c | 3 ++ mm/pgsize_migration.c | 56 ++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+) diff --git a/include/linux/pgsize_migration.h b/include/linux/pgsize_migration.h index 60f719d44107..fd1e74ea4283 100644 --- a/include/linux/pgsize_migration.h +++ b/include/linux/pgsize_migration.h @@ -45,6 +45,9 @@ extern void vma_set_pad_pages(struct vm_area_struct *vma, unsigned long nr_pages); extern unsigned long vma_pad_pages(struct vm_area_struct *vma); + +extern void madvise_vma_pad_pages(struct vm_area_struct *vma, + unsigned long start, unsigned long end); #else /* PAGE_SIZE != SZ_4K || !defined(CONFIG_64BIT) */ static inline void vma_set_pad_pages(struct vm_area_struct *vma, unsigned long nr_pages) @@ -55,6 +58,11 @@ static inline unsigned long vma_pad_pages(struct vm_area_struct *vma) { return 0; } + +static inline void madvise_vma_pad_pages(struct vm_area_struct *vma, + unsigned long start, unsigned long end) +{ +} #endif /* PAGE_SIZE == SZ_4K && defined(CONFIG_64BIT) */ static inline unsigned long vma_data_pages(struct vm_area_struct *vma) diff --git a/mm/madvise.c b/mm/madvise.c index b2a9177d8993..fa716030ba1d 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -824,6 +825,8 @@ static int madvise_free_single_vma(struct vm_area_struct *vma, static long madvise_dontneed_single_vma(struct vm_area_struct *vma, unsigned long start, unsigned long end) { + madvise_vma_pad_pages(vma, start, end); + zap_page_range_single(vma, start, end - start, NULL); return 0; } diff --git a/mm/pgsize_migration.c b/mm/pgsize_migration.c index 54b29a8269bf..ce77c7af86de 100644 --- a/mm/pgsize_migration.c +++ b/mm/pgsize_migration.c @@ -121,5 +121,61 @@ unsigned long vma_pad_pages(struct vm_area_struct *vma) return vma->vm_flags >> VM_PAD_SHIFT; } + +static __always_inline bool str_has_suffix(const char *str, const char *suffix) +{ + size_t str_len = strlen(str); + size_t suffix_len = strlen(suffix); + + if (str_len < suffix_len) + return false; + + return !strncmp(str + str_len - suffix_len, suffix, suffix_len); +} + +/* + * Saves the number of padding pages for an ELF segment mapping + * in vm_flags. + * + * The number of padding pages is deduced from the madvise DONTNEED range [start, end) + * if the following conditions are met: + * 1) The range is enclosed by a single VMA + * 2) The range ends at the end address of the VMA + * 3) The range starts at an address greater than the start address of the VMA + * 4) The number of the pages in the range does not exceed VM_TOTAL_PAD_PAGES. + * 5) The VMA is a regular file backed VMA (filemap_fault) + * 6) The file backing the VMA is a shared library (*.so) + */ +void madvise_vma_pad_pages(struct vm_area_struct *vma, + unsigned long start, unsigned long end) +{ + unsigned long nr_pad_pages; + + if (!is_pgsize_migration_enabled()) + return; + + /* Only handle this for file backed VMAs */ + if (!vma->vm_file || !vma->vm_ops || vma->vm_ops->fault != filemap_fault) + return; + + + /* Limit this to only shared libraries (*.so) */ + if (!str_has_suffix(vma->vm_file->f_path.dentry->d_name.name, ".so")) + return; + + /* + * If the madvise range is it at the end of the file save the number of + * pages in vm_flags (only need 4 bits are needed for 16kB aligned ELFs). + */ + if (start <= vma->vm_start || end != vma->vm_end) + return; + + nr_pad_pages = (end - start) >> PAGE_SHIFT; + + if (!nr_pad_pages || nr_pad_pages > VM_TOTAL_PAD_PAGES) + return; + + vma_set_pad_pages(vma, nr_pad_pages); +} #endif /* PAGE_SIZE == SZ_4K */ #endif /* CONFIG_64BIT */ From 1537dbe21b5d5d3dc2cb6594a781c6f2017e1400 Mon Sep 17 00:00:00 2001 From: Kalesh Singh Date: Thu, 4 Apr 2024 22:37:48 -0700 Subject: [PATCH 56/86] ANDROID: 16K: Exclude ELF padding for fault around range Userspace apps often analyze memory consumption by the use of mm rss_stat counters -- via the kmem/rss_stat trace event or from /proc//statm. rss_stat counters are only updated when the PTEs are updated. What this means is that pages can be present in the page cache from readahead but not visible to userspace (not attributed to the app) as there is no corresponding VMA (PTEs) for the respective page cache pages. A side effect of the loader now extending ELF LOAD segments to be contiguously mapped in the virtual address space, means that the VMA is extended to cover the padding pages. When filesystems, such as f2fs and ext4, that implement vm_ops->map_pages() attempt to perform a do_fault_around() the extent of the fault around is restricted by the area of the enclosing VMA. Since the loader extends LOAD segment VMAs to be contiguously mapped, the extent of the fault around is also increased. The result of which, is that the PTEs corresponding to the padding pages are updated and reflected in the rss_stat counters. It is not common that userspace application developers be aware of this nuance in the kernel's memory accounting. To avoid apparent regressions in memory usage to userspace, restrict the fault around range to only valid data pages (i.e. exclude the padding pages at the end of the VMA). Bug: 330117029 Bug: 327600007 Bug: 330767927 Bug: 328266487 Bug: 329803029 Change-Id: I2c7a39ec1b040be2b9fb47801f95042f5dbf869d Signed-off-by: Kalesh Singh --- mm/memory.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/memory.c b/mm/memory.c index f1049f6cdca6..66cbb29cd444 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -59,6 +59,7 @@ #include #include #include +#include #include #include #include @@ -4615,7 +4616,7 @@ static vm_fault_t do_fault_around(struct vm_fault *vmf) end_pgoff = start_pgoff - ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) + PTRS_PER_PTE - 1; - end_pgoff = min3(end_pgoff, vma_pages(vmf->vma) + vmf->vma->vm_pgoff - 1, + end_pgoff = min3(end_pgoff, vma_data_pages(vmf->vma) + vmf->vma->vm_pgoff - 1, start_pgoff + nr_pages - 1); if (pmd_none(*vmf->pmd)) { From 2ca5e076c97a56a68a51be610c9451f39a194046 Mon Sep 17 00:00:00 2001 From: Kalesh Singh Date: Thu, 4 Apr 2024 23:02:30 -0700 Subject: [PATCH 57/86] ANDROID: 16K: Separate padding from ELF LOAD segment mappings In has been found that some in-field apps depend on the output of /proc/*/maps to determine the address ranges of other operations. With the extension of LOAD segments VMAs to be contiguous in memory, the apps may perform operations on an area that is not backed by the underlying file, which results in a SIGBUS. Other apps have crashed with yet unindentified reasons. To avoid breaking in-field apps, maintain the output of /proc/*/[s]maps with PROT_NONE VMAs for the padding pages of LOAD segments instead of showing the segment extensions. NOTE: This does not allocate actual backing VMAs for the shown PROT_NONE mappings. This approach maintains 2 possible assumptions that userspace (apps) could be depending on: 1) That LOAD segment mappings are "contiguous" (not speparated by unrelated mappings) in memory. 2) That no virtual address space is available between mappings of consecutive LOAD segments for the same ELF. For example the output of /proc/*/[s]maps before and after this change is shown below. Segments maintain PROT_NONE gaps ("[page size compat]") for app compatiblity but these are not backed by actual slab VMA memory. Maps Before: 7fb03604d000-7fb036051000 r--p 00000000 fe:09 21935719 /system/lib64/libnetd_client.so 7fb036051000-7fb036055000 r-xp 00004000 fe:09 21935719 /system/lib64/libnetd_client.so 7fb036055000-7fb036059000 r--p 00008000 fe:09 21935719 /system/lib64/libnetd_client.so 7fb036059000-7fb03605a000 rw-p 0000c000 fe:09 21935719 /system/lib64/libnetd_client.so Maps After: 7fc707390000-7fc707393000 r--p 00000000 fe:09 21935719 /system/lib64/libnetd_client.so 7fc707393000-7fc707394000 ---p 00000000 00:00 0 [page size compat] 7fc707394000-7fc707398000 r-xp 00004000 fe:09 21935719 /system/lib64/libnetd_client.so 7fc707398000-7fc707399000 r--p 00008000 fe:09 21935719 /system/lib64/libnetd_client.so 7fc707399000-7fc70739c000 ---p 00000000 00:00 0 [page size compat] 7fc70739c000-7fc70739d000 rw-p 0000c000 fe:09 21935719 /system/lib64/libnetd_client.so Smaps Before: 7fb03604d000-7fb036051000 r--p 00000000 fe:09 21935719 /system/lib64/libnetd_client.so Size: 16 kB KernelPageSize: 4 kB MMUPageSize: 4 kB Rss: 16 kB Pss: 0 kB Pss_Dirty: 0 kB Shared_Clean: 16 kB Shared_Dirty: 0 kB Private_Clean: 0 kB Private_Dirty: 0 kB Referenced: 16 kB Anonymous: 0 kB LazyFree: 0 kB AnonHugePages: 0 kB ShmemPmdMapped: 0 kB FilePmdMapped: 0 kB Shared_Hugetlb: 0 kB Private_Hugetlb: 0 kB Swap: 0 kB SwapPss: 0 kB Locked: 0 kB THPeligible: 0 VmFlags: rd mr mw me 7fb036051000-7fb036055000 r-xp 00004000 fe:09 21935719 /system/lib64/libnetd_client.so Size: 16 kB KernelPageSize: 4 kB MMUPageSize: 4 kB Rss: 16 kB Pss: 0 kB Pss_Dirty: 0 kB Shared_Clean: 16 kB Shared_Dirty: 0 kB Private_Clean: 0 kB Private_Dirty: 0 kB Referenced: 16 kB Anonymous: 0 kB LazyFree: 0 kB AnonHugePages: 0 kB ShmemPmdMapped: 0 kB FilePmdMapped: 0 kB Shared_Hugetlb: 0 kB Private_Hugetlb: 0 kB Swap: 0 kB SwapPss: 0 kB Locked: 0 kB THPeligible: 0 VmFlags: rd ex mr mw me 7fb036055000-7fb036059000 r--p 00008000 fe:09 21935719 /system/lib64/libnetd_client.so Size: 16 kB KernelPageSize: 4 kB MMUPageSize: 4 kB Rss: 4 kB Pss: 4 kB Pss_Dirty: 4 kB Shared_Clean: 0 kB Shared_Dirty: 0 kB Private_Clean: 0 kB Private_Dirty: 4 kB Referenced: 4 kB Anonymous: 4 kB LazyFree: 0 kB AnonHugePages: 0 kB ShmemPmdMapped: 0 kB FilePmdMapped: 0 kB Shared_Hugetlb: 0 kB Private_Hugetlb: 0 kB Swap: 0 kB SwapPss: 0 kB Locked: 0 kB THPeligible: 0 VmFlags: rd mr mw me ac 7fb036059000-7fb03605a000 rw-p 0000c000 fe:09 21935719 /system/lib64/libnetd_client.so Size: 4 kB KernelPageSize: 4 kB MMUPageSize: 4 kB Rss: 4 kB Pss: 4 kB Pss_Dirty: 4 kB Shared_Clean: 0 kB Shared_Dirty: 0 kB Private_Clean: 0 kB Private_Dirty: 4 kB Referenced: 4 kB Anonymous: 4 kB LazyFree: 0 kB AnonHugePages: 0 kB ShmemPmdMapped: 0 kB FilePmdMapped: 0 kB Shared_Hugetlb: 0 kB Private_Hugetlb: 0 kB Swap: 0 kB SwapPss: 0 kB Locked: 0 kB THPeligible: 0 VmFlags: rd wr mr mw me ac Smaps After: 7fc707390000-7fc707393000 r--p 00000000 fe:09 21935719 /system/lib64/libnetd_client.so Size: 12 kB KernelPageSize: 4 kB MMUPageSize: 4 kB Rss: 12 kB Pss: 0 kB Shared_Clean: 12 kB Shared_Dirty: 0 kB Private_Clean: 0 kB Private_Dirty: 0 kB Referenced: 12 kB Anonymous: 0 kB LazyFree: 0 kB AnonHugePages: 0 kB ShmemPmdMapped: 0 kB FilePmdMapped: 0 kB Shared_Hugetlb: 0 kB Private_Hugetlb: 0 kB Swap: 0 kB SwapPss: 0 kB Locked: 0 kB THPeligible: 0 VmFlags: rd mr mw me ?? 7fc707393000-7fc707394000 ---p 00000000 00:00 0 [page size compat] Size: 4 kB KernelPageSize: 4 kB MMUPageSize: 4 kB Rss: 0 kB Pss: 0 kB Shared_Clean: 0 kB Shared_Dirty: 0 kB Private_Clean: 0 kB Private_Dirty: 0 kB Referenced: 0 kB Anonymous: 0 kB LazyFree: 0 kB AnonHugePages: 0 kB ShmemPmdMapped: 0 kB FilePmdMapped: 0 kB Shared_Hugetlb: 0 kB Private_Hugetlb: 0 kB Swap: 0 kB SwapPss: 0 kB Locked: 0 kB THPeligible: 0 VmFlags: mr mw me 7fc707394000-7fc707398000 r-xp 00004000 fe:09 21935719 /system/lib64/libnetd_client.so Size: 16 kB KernelPageSize: 4 kB MMUPageSize: 4 kB Rss: 16 kB Pss: 0 kB Shared_Clean: 16 kB Shared_Dirty: 0 kB Private_Clean: 0 kB Private_Dirty: 0 kB Referenced: 16 kB Anonymous: 0 kB LazyFree: 0 kB AnonHugePages: 0 kB ShmemPmdMapped: 0 kB FilePmdMapped: 0 kB Shared_Hugetlb: 0 kB Private_Hugetlb: 0 kB Swap: 0 kB SwapPss: 0 kB Locked: 0 kB THPeligible: 0 VmFlags: rd ex mr mw me 7fc707398000-7fc707399000 r--p 00008000 fe:09 21935719 /system/lib64/libnetd_client.so Size: 4 kB KernelPageSize: 4 kB MMUPageSize: 4 kB Rss: 4 kB Pss: 4 kB Shared_Clean: 0 kB Shared_Dirty: 0 kB Private_Clean: 0 kB Private_Dirty: 4 kB Referenced: 4 kB Anonymous: 4 kB LazyFree: 0 kB AnonHugePages: 0 kB ShmemPmdMapped: 0 kB FilePmdMapped: 0 kB Shared_Hugetlb: 0 kB Private_Hugetlb: 0 kB Swap: 0 kB SwapPss: 0 kB Locked: 0 kB THPeligible: 0 VmFlags: rd mr mw me ac ?? ?? 7fc707399000-7fc70739c000 ---p 00000000 00:00 0 [page size compat] Size: 12 kB KernelPageSize: 4 kB MMUPageSize: 4 kB Rss: 0 kB Pss: 0 kB Shared_Clean: 0 kB Shared_Dirty: 0 kB Private_Clean: 0 kB Private_Dirty: 0 kB Referenced: 0 kB Anonymous: 0 kB LazyFree: 0 kB AnonHugePages: 0 kB ShmemPmdMapped: 0 kB FilePmdMapped: 0 kB Shared_Hugetlb: 0 kB Private_Hugetlb: 0 kB Swap: 0 kB SwapPss: 0 kB Locked: 0 kB THPeligible: 0 VmFlags: mr mw me ac 7fc70739c000-7fc70739d000 rw-p 0000c000 fe:09 21935719 /system/lib64/libnetd_client.so Size: 4 kB KernelPageSize: 4 kB MMUPageSize: 4 kB Rss: 4 kB Pss: 4 kB Shared_Clean: 0 kB Shared_Dirty: 0 kB Private_Clean: 0 kB Private_Dirty: 4 kB Referenced: 4 kB Anonymous: 4 kB LazyFree: 0 kB AnonHugePages: 0 kB ShmemPmdMapped: 0 kB FilePmdMapped: 0 kB Shared_Hugetlb: 0 kB Private_Hugetlb: 0 kB Swap: 0 kB SwapPss: 0 kB Locked: 0 kB THPeligible: 0 VmFlags: rd wr mr mw me ac Bug: 330117029 Bug: 327600007 Bug: 330767927 Bug: 328266487 Bug: 329803029 Change-Id: I12bf2c106fafc74a500d79155b81dde5db42661e Signed-off-by: Kalesh Singh --- fs/proc/task_mmu.c | 14 ++++- include/linux/pgsize_migration.h | 29 ++++++++++ mm/pgsize_migration.c | 92 ++++++++++++++++++++++++++++++++ 3 files changed, 133 insertions(+), 2 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index fe244a271620..c906bd13f51e 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -348,7 +349,13 @@ done: static int show_map(struct seq_file *m, void *v) { - show_map_vma(m, v); + struct vm_area_struct *pad_vma = get_pad_vma(v); + struct vm_area_struct *vma = get_data_vma(v); + + show_map_vma(m, vma); + + show_map_pad_vma(vma, pad_vma, m, show_map_vma); + return 0; } @@ -858,7 +865,8 @@ static void __show_smap(struct seq_file *m, const struct mem_size_stats *mss, static int show_smap(struct seq_file *m, void *v) { - struct vm_area_struct *vma = v; + struct vm_area_struct *pad_vma = get_pad_vma(v); + struct vm_area_struct *vma = get_data_vma(v); struct mem_size_stats mss; memset(&mss, 0, sizeof(mss)); @@ -881,6 +889,8 @@ static int show_smap(struct seq_file *m, void *v) seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma)); show_smap_vma_flags(m, vma); + show_map_pad_vma(vma, pad_vma, m, (show_pad_vma_fn)((void*)show_smap)); + return 0; } diff --git a/include/linux/pgsize_migration.h b/include/linux/pgsize_migration.h index fd1e74ea4283..7ab0f288bcf9 100644 --- a/include/linux/pgsize_migration.h +++ b/include/linux/pgsize_migration.h @@ -14,6 +14,7 @@ */ #include +#include #include /* @@ -39,6 +40,10 @@ #define VM_PAD_WIDTH 4 #define VM_PAD_SHIFT (BITS_PER_LONG - VM_PAD_WIDTH) #define VM_TOTAL_PAD_PAGES ((1ULL << VM_PAD_WIDTH) - 1) +#define VM_PAD_MASK (VM_TOTAL_PAD_PAGES << VM_PAD_SHIFT) +#define VMA_PAD_START(vma) (vma->vm_end - (vma_pad_pages(vma) << PAGE_SHIFT)) + +typedef void (*show_pad_vma_fn)(struct seq_file *m, struct vm_area_struct *vma); #if PAGE_SIZE == SZ_4K && defined(CONFIG_64BIT) extern void vma_set_pad_pages(struct vm_area_struct *vma, @@ -48,6 +53,14 @@ extern unsigned long vma_pad_pages(struct vm_area_struct *vma); extern void madvise_vma_pad_pages(struct vm_area_struct *vma, unsigned long start, unsigned long end); + +extern struct vm_area_struct *get_pad_vma(struct vm_area_struct *vma); + +extern struct vm_area_struct *get_data_vma(struct vm_area_struct *vma); + +extern void show_map_pad_vma(struct vm_area_struct *vma, + struct vm_area_struct *pad, + struct seq_file *m, show_pad_vma_fn func); #else /* PAGE_SIZE != SZ_4K || !defined(CONFIG_64BIT) */ static inline void vma_set_pad_pages(struct vm_area_struct *vma, unsigned long nr_pages) @@ -63,6 +76,22 @@ static inline void madvise_vma_pad_pages(struct vm_area_struct *vma, unsigned long start, unsigned long end) { } + +static inline struct vm_area_struct *get_pad_vma(struct vm_area_struct *vma) +{ + return NULL; +} + +static inline struct vm_area_struct *get_data_vma(struct vm_area_struct *vma) +{ + return vma; +} + +static inline void show_map_pad_vma(struct vm_area_struct *vma, + struct vm_area_struct *pad, + struct seq_file *m, show_pad_vma_fn func) +{ +} #endif /* PAGE_SIZE == SZ_4K && defined(CONFIG_64BIT) */ static inline unsigned long vma_data_pages(struct vm_area_struct *vma) diff --git a/mm/pgsize_migration.c b/mm/pgsize_migration.c index ce77c7af86de..53d4f41b9e8e 100644 --- a/mm/pgsize_migration.c +++ b/mm/pgsize_migration.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #ifdef CONFIG_64BIT @@ -177,5 +178,96 @@ void madvise_vma_pad_pages(struct vm_area_struct *vma, vma_set_pad_pages(vma, nr_pad_pages); } + +static const char *pad_vma_name(struct vm_area_struct *vma) +{ + return "[page size compat]"; +} + +static const struct vm_operations_struct pad_vma_ops = { + .name = pad_vma_name, +}; + +/* + * Returns a new VMA representing the padding in @vma, if no padding + * in @vma returns NULL. + */ +struct vm_area_struct *get_pad_vma(struct vm_area_struct *vma) +{ + struct vm_area_struct *pad; + + if (!is_pgsize_migration_enabled() || !(vma->vm_flags & VM_PAD_MASK)) + return NULL; + + pad = kzalloc(sizeof(struct vm_area_struct), GFP_KERNEL); + + memcpy(pad, vma, sizeof(struct vm_area_struct)); + + /* Remove file */ + pad->vm_file = NULL; + + /* Add vm_ops->name */ + pad->vm_ops = &pad_vma_ops; + + /* Adjust the start to begin at the start of the padding section */ + pad->vm_start = VMA_PAD_START(pad); + + /* Make the pad vma PROT_NONE */ + vm_flags_clear(pad, VM_READ|VM_WRITE|VM_EXEC); + + /* Remove padding bits */ + vm_flags_clear(pad, VM_PAD_MASK); + + return pad; +} + +/* + * Returns a new VMA exclusing the padding from @vma; if no padding in + * @vma returns @vma. + */ +struct vm_area_struct *get_data_vma(struct vm_area_struct *vma) +{ + struct vm_area_struct *data; + + if (!is_pgsize_migration_enabled() || !(vma->vm_flags & VM_PAD_MASK)) + return vma; + + data = kzalloc(sizeof(struct vm_area_struct), GFP_KERNEL); + + memcpy(data, vma, sizeof(struct vm_area_struct)); + + /* Adjust the end to the start of the padding section */ + data->vm_end = VMA_PAD_START(data); + + return data; +} + +/* + * Calls the show_pad_vma_fn on the @pad VMA, and frees the copies of @vma + * and @pad. + */ +void show_map_pad_vma(struct vm_area_struct *vma, struct vm_area_struct *pad, + struct seq_file *m, show_pad_vma_fn func) +{ + if (!pad) + return; + + /* + * This cannot happen. If @pad vma was allocated the corresponding + * @vma should have the VM_PAD_MASK bit(s) set. + */ + BUG_ON(!(vma->vm_flags & VM_PAD_MASK)); + + /* + * This cannot happen. @pad is a section of the original VMA. + * Therefore @vma cannot be null if @pad is not null. + */ + BUG_ON(!vma); + + func(m, pad); + + kfree(pad); + kfree(vma); +} #endif /* PAGE_SIZE == SZ_4K */ #endif /* CONFIG_64BIT */ From 1657717c12d251fd9ede68563ff0f332b73fa8fd Mon Sep 17 00:00:00 2001 From: Kalesh Singh Date: Fri, 19 Apr 2024 14:41:35 -0700 Subject: [PATCH 58/86] ANDROID: 16K: Only madvise padding from dynamic linker context Only preform padding advise from the execution context on bionic's dynamic linker. This ensures that madvise() doesn't have unwanted side effects. Also rearrange the order of fail checks in madvise_vma_pad_pages() in order of ascending cost. Bug: 330117029 Bug: 327600007 Bug: 330767927 Bug: 328266487 Bug: 329803029 Change-Id: I3e05b8780c6eda78007f86b613f8c11dd18ac28f Signed-off-by: Kalesh Singh --- mm/pgsize_migration.c | 75 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 65 insertions(+), 10 deletions(-) diff --git a/mm/pgsize_migration.c b/mm/pgsize_migration.c index 53d4f41b9e8e..94b2646595b3 100644 --- a/mm/pgsize_migration.c +++ b/mm/pgsize_migration.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -134,6 +135,56 @@ static __always_inline bool str_has_suffix(const char *str, const char *suffix) return !strncmp(str + str_len - suffix_len, suffix, suffix_len); } +/* + * The dynamic linker, or interpreter, operates within the process context + * of the binary that necessitated dynamic linking. + * + * Consequently, process context identifiers; like PID, comm, ...; cannot + * be used to differentiate whether the execution context belongs to the + * dynamic linker or not. + * + * linker_ctx() deduces whether execution is currently in the dynamic linker's + * context by correlating the current userspace instruction pointer with the + * VMAs of the current task. + * + * Returns true if in linker context, otherwise false. + * + * Caller must hold mmap lock in read mode. + */ +static inline bool linker_ctx(void) +{ + struct pt_regs *regs = task_pt_regs(current); + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma; + struct file *file; + + if (!regs) + return false; + + vma = find_vma(mm, instruction_pointer(regs)); + + /* Current execution context, the VMA must be present */ + BUG_ON(!vma); + + file = vma->vm_file; + if (!file) + return false; + + if ((vma->vm_flags & VM_EXEC)) { + char buf[64]; + const int bufsize = sizeof(buf); + char *path; + + memset(buf, 0, bufsize); + path = d_path(&file->f_path, buf, bufsize); + + if (!strcmp(path, "/system/bin/linker64")) + return true; + } + + return false; +} + /* * Saves the number of padding pages for an ELF segment mapping * in vm_flags. @@ -146,6 +197,7 @@ static __always_inline bool str_has_suffix(const char *str, const char *suffix) * 4) The number of the pages in the range does not exceed VM_TOTAL_PAD_PAGES. * 5) The VMA is a regular file backed VMA (filemap_fault) * 6) The file backing the VMA is a shared library (*.so) + * 7) The madvise was requested by bionic's dynamic linker. */ void madvise_vma_pad_pages(struct vm_area_struct *vma, unsigned long start, unsigned long end) @@ -155,18 +207,9 @@ void madvise_vma_pad_pages(struct vm_area_struct *vma, if (!is_pgsize_migration_enabled()) return; - /* Only handle this for file backed VMAs */ - if (!vma->vm_file || !vma->vm_ops || vma->vm_ops->fault != filemap_fault) - return; - - - /* Limit this to only shared libraries (*.so) */ - if (!str_has_suffix(vma->vm_file->f_path.dentry->d_name.name, ".so")) - return; - /* * If the madvise range is it at the end of the file save the number of - * pages in vm_flags (only need 4 bits are needed for 16kB aligned ELFs). + * pages in vm_flags (only need 4 bits are needed for up to 64kB aligned ELFs). */ if (start <= vma->vm_start || end != vma->vm_end) return; @@ -176,6 +219,18 @@ void madvise_vma_pad_pages(struct vm_area_struct *vma, if (!nr_pad_pages || nr_pad_pages > VM_TOTAL_PAD_PAGES) return; + /* Only handle this for file backed VMAs */ + if (!vma->vm_file || !vma->vm_ops || vma->vm_ops->fault != filemap_fault) + return; + + /* Limit this to only shared libraries (*.so) */ + if (!str_has_suffix(vma->vm_file->f_path.dentry->d_name.name, ".so")) + return; + + /* Only bionic's dynamic linker needs to hint padding pages. */ + if (!linker_ctx()) + return; + vma_set_pad_pages(vma, nr_pad_pages); } From b86b5cb22de6ed589f12e1ceb20498d6114b6356 Mon Sep 17 00:00:00 2001 From: Kalesh Singh Date: Thu, 25 Apr 2024 09:59:08 -0700 Subject: [PATCH 59/86] ANDROID: 16K: madvise_vma_pad_pages: Remove filemap_fault check Some file systems like F2FS use a custom filemap_fault ops. Remove this check, as checking vm_file is sufficient. Bug: 330117029 Bug: 327600007 Bug: 330767927 Bug: 328266487 Bug: 329803029 Change-Id: Id6a584d934f06650c0a95afd1823669fc77ba2c2 Signed-off-by: Kalesh Singh --- mm/pgsize_migration.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/pgsize_migration.c b/mm/pgsize_migration.c index 94b2646595b3..3151722a0329 100644 --- a/mm/pgsize_migration.c +++ b/mm/pgsize_migration.c @@ -195,7 +195,7 @@ static inline bool linker_ctx(void) * 2) The range ends at the end address of the VMA * 3) The range starts at an address greater than the start address of the VMA * 4) The number of the pages in the range does not exceed VM_TOTAL_PAD_PAGES. - * 5) The VMA is a regular file backed VMA (filemap_fault) + * 5) The VMA is a file backed VMA. * 6) The file backing the VMA is a shared library (*.so) * 7) The madvise was requested by bionic's dynamic linker. */ @@ -220,7 +220,7 @@ void madvise_vma_pad_pages(struct vm_area_struct *vma, return; /* Only handle this for file backed VMAs */ - if (!vma->vm_file || !vma->vm_ops || vma->vm_ops->fault != filemap_fault) + if (!vma->vm_file) return; /* Limit this to only shared libraries (*.so) */ From 72a9c0a2054ebac45352b7261a4175f5693733b7 Mon Sep 17 00:00:00 2001 From: Kalesh Singh Date: Mon, 22 Apr 2024 14:24:59 -0700 Subject: [PATCH 60/86] ANDROID: 16K: Handle pad VMA splits and merges In some cases a VMA with padding representation may be split, and therefore the padding flags must be updated accordingly. There are 3 cases to handle: Given: | DDDDPPPP | where: - D represents 1 page of data; - P represents 1 page of padding; - | represents the boundaries (start/end) of the VMA 1) Split exactly at the padding boundary | DDDDPPPP | --> | DDDD | PPPP | - Remove padding flags from the first VMA. - The second VMA is all padding 2) Split within the padding area | DDDDPPPP | --> | DDDDPP | PP | - Subtract the length of the second VMA from the first VMA's padding. - The second VMA is all padding, adjust its padding length (flags) 3) Split within the data area | DDDDPPPP | --> | DD | DDPPPP | - Remove padding flags from the first VMA. - The second VMA is has the same padding as from before the split. To simplify the semantics merging of padding VMAs is not allowed. If a split produces a VMA that is entirely padding, show_[s]maps() only outputs the padding VMA entry (as the data entry is of length 0). Bug: 330117029 Bug: 327600007 Bug: 330767927 Bug: 328266487 Bug: 329803029 Change-Id: Ie2628ced5512e2c7f8af25fabae1f38730c8bb1a Signed-off-by: Kalesh Singh --- fs/proc/task_mmu.c | 7 +++- include/linux/pgsize_migration.h | 34 +++++++++++++++++ mm/mlock.c | 3 +- mm/mmap.c | 7 +++- mm/mprotect.c | 3 +- mm/pgsize_migration.c | 65 ++++++++++++++++++++++++++++++++ 6 files changed, 115 insertions(+), 4 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index c906bd13f51e..1416772416cb 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -352,7 +352,8 @@ static int show_map(struct seq_file *m, void *v) struct vm_area_struct *pad_vma = get_pad_vma(v); struct vm_area_struct *vma = get_data_vma(v); - show_map_vma(m, vma); + if (vma_pages(vma)) + show_map_vma(m, vma); show_map_pad_vma(vma, pad_vma, m, show_map_vma); @@ -871,6 +872,9 @@ static int show_smap(struct seq_file *m, void *v) memset(&mss, 0, sizeof(mss)); + if (!vma_pages(vma)) + goto show_pad; + smap_gather_stats(vma, &mss, 0); show_map_vma(m, vma); @@ -889,6 +893,7 @@ static int show_smap(struct seq_file *m, void *v) seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma)); show_smap_vma_flags(m, vma); +show_pad: show_map_pad_vma(vma, pad_vma, m, (show_pad_vma_fn)((void*)show_smap)); return 0; diff --git a/include/linux/pgsize_migration.h b/include/linux/pgsize_migration.h index 7ab0f288bcf9..5c47ec28ea7d 100644 --- a/include/linux/pgsize_migration.h +++ b/include/linux/pgsize_migration.h @@ -61,6 +61,9 @@ extern struct vm_area_struct *get_data_vma(struct vm_area_struct *vma); extern void show_map_pad_vma(struct vm_area_struct *vma, struct vm_area_struct *pad, struct seq_file *m, show_pad_vma_fn func); + +extern void split_pad_vma(struct vm_area_struct *vma, struct vm_area_struct *new, + unsigned long addr, int new_below); #else /* PAGE_SIZE != SZ_4K || !defined(CONFIG_64BIT) */ static inline void vma_set_pad_pages(struct vm_area_struct *vma, unsigned long nr_pages) @@ -92,10 +95,41 @@ static inline void show_map_pad_vma(struct vm_area_struct *vma, struct seq_file *m, show_pad_vma_fn func) { } + +static inline void split_pad_vma(struct vm_area_struct *vma, struct vm_area_struct *new, + unsigned long addr, int new_below) +{ +} #endif /* PAGE_SIZE == SZ_4K && defined(CONFIG_64BIT) */ static inline unsigned long vma_data_pages(struct vm_area_struct *vma) { return vma_pages(vma) - vma_pad_pages(vma); } + +/* + * Sets the correct padding bits / flags for a VMA split. + */ +static inline unsigned long vma_pad_fixup_flags(struct vm_area_struct *vma, + unsigned long newflags) +{ + if (newflags & VM_PAD_MASK) + return (newflags & ~VM_PAD_MASK) | (vma->vm_flags & VM_PAD_MASK); + else + return newflags; +} + +/* + * Merging of padding VMAs is uncommon, as padding is only allowed + * from the linker context. + * + * To simplify the semantics, adjacent VMAs with padding are not + * allowed to merge. + */ +static inline bool is_mergable_pad_vma(struct vm_area_struct *vma, + unsigned long vm_flags) +{ + /* Padding VMAs cannot be merged with other padding or real VMAs */ + return !((vma->vm_flags | vm_flags) & VM_PAD_MASK); +} #endif /* _LINUX_PAGE_SIZE_MIGRATION_H */ diff --git a/mm/mlock.c b/mm/mlock.c index 580f1d39f454..4f029298b09e 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -458,7 +459,7 @@ success: if ((newflags & VM_LOCKED) && (oldflags & VM_LOCKED)) { /* No work to do, and mlocking twice would be wrong */ vma_start_write(vma); - vm_flags_reset(vma, newflags); + vm_flags_reset(vma, vma_pad_fixup_flags(vma, newflags)); } else { mlock_vma_pages_range(vma, start, end, newflags); } diff --git a/mm/mmap.c b/mm/mmap.c index 62324b21d721..2686e9e93915 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -940,6 +941,8 @@ static inline int is_mergeable_vma(struct vm_area_struct *vma, return 0; if (!anon_vma_name_eq(anon_vma_name(vma), anon_name)) return 0; + if (!is_mergable_pad_vma(vma, vm_flags)) + return 0; return 1; } @@ -2455,8 +2458,10 @@ int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new); /* Success. */ - if (!err) + if (!err) { + split_pad_vma(vma, new, addr, new_below); return 0; + } /* Avoid vm accounting in close() operation */ new->vm_start = new->vm_end; diff --git a/mm/mprotect.c b/mm/mprotect.c index 45a87f0ce625..044c17ffaecf 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -632,7 +633,7 @@ success: * held in write mode. */ vma_start_write(vma); - vm_flags_reset(vma, newflags); + vm_flags_reset(vma, vma_pad_fixup_flags(vma, newflags)); /* * We want to check manually if we can change individual PTEs writable * if we can't do that automatically for all PTEs in a mapping. For diff --git a/mm/pgsize_migration.c b/mm/pgsize_migration.c index 3151722a0329..8b31cdb54383 100644 --- a/mm/pgsize_migration.c +++ b/mm/pgsize_migration.c @@ -113,6 +113,7 @@ void vma_set_pad_pages(struct vm_area_struct *vma, if (!is_pgsize_migration_enabled()) return; + vm_flags_clear(vma, VM_PAD_MASK); vm_flags_set(vma, nr_pages << VM_PAD_SHIFT); } @@ -324,5 +325,69 @@ void show_map_pad_vma(struct vm_area_struct *vma, struct vm_area_struct *pad, kfree(pad); kfree(vma); } + +/* + * When splitting a padding VMA there are a couple of cases to handle. + * + * Given: + * + * | DDDDPPPP | + * + * where: + * - D represents 1 page of data; + * - P represents 1 page of padding; + * - | represents the boundaries (start/end) of the VMA + * + * + * 1) Split exactly at the padding boundary + * + * | DDDDPPPP | --> | DDDD | PPPP | + * + * - Remove padding flags from the first VMA. + * - The second VMA is all padding + * + * 2) Split within the padding area + * + * | DDDDPPPP | --> | DDDDPP | PP | + * + * - Subtract the length of the second VMA from the first VMA's padding. + * - The second VMA is all padding, adjust its padding length (flags) + * + * 3) Split within the data area + * + * | DDDDPPPP | --> | DD | DDPPPP | + * + * - Remove padding flags from the first VMA. + * - The second VMA is has the same padding as from before the split. + */ +void split_pad_vma(struct vm_area_struct *vma, struct vm_area_struct *new, + unsigned long addr, int new_below) +{ + unsigned long nr_pad_pages = vma_pad_pages(vma); + unsigned long nr_vma2_pages; + struct vm_area_struct *first; + struct vm_area_struct *second; + + if (!nr_pad_pages) + return; + + if (new_below) { + first = new; + second = vma; + } else { + first = vma; + second = new; + } + + nr_vma2_pages = vma_pages(second); + + if (nr_vma2_pages >= nr_pad_pages) { /* Case 1 & 3*/ + vm_flags_clear(first, VM_PAD_MASK); + vma_set_pad_pages(second, nr_pad_pages); + } else { /* Case 2 */ + vma_set_pad_pages(first, nr_pad_pages - nr_vma2_pages); + vma_set_pad_pages(second, nr_vma2_pages); + } +} #endif /* PAGE_SIZE == SZ_4K */ #endif /* CONFIG_64BIT */ From 1794308d463f6fc05c5fef7192f55882ba6252ce Mon Sep 17 00:00:00 2001 From: Kalesh Singh Date: Tue, 30 Apr 2024 13:42:47 -0700 Subject: [PATCH 61/86] ANDROID: 16K: Fix show maps CFI failure If the kernel is built CONFIG_CFI_CLANG=y, reading smaps may cause a panic. This is due to a failed CFI check; which is triggered becuase the signature of the function pointer for printing smaps padding VMAs does not match exactly with that for show_smap(). Fix this by casting the function pointer to the expected type based on whether printing maps or smaps padding. Bug: 330117029 Bug: 327600007 Bug: 330767927 Bug: 328266487 Bug: 329803029 Change-Id: I65564a547dacbc4131f8557344c8c96e51f90cd5 Signed-off-by: Kalesh Singh --- fs/proc/task_mmu.c | 4 ++-- include/linux/pgsize_migration.h | 6 ++---- mm/pgsize_migration.c | 10 ++++++++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 1416772416cb..9f09aa1127e0 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -355,7 +355,7 @@ static int show_map(struct seq_file *m, void *v) if (vma_pages(vma)) show_map_vma(m, vma); - show_map_pad_vma(vma, pad_vma, m, show_map_vma); + show_map_pad_vma(vma, pad_vma, m, show_map_vma, false); return 0; } @@ -894,7 +894,7 @@ static int show_smap(struct seq_file *m, void *v) show_smap_vma_flags(m, vma); show_pad: - show_map_pad_vma(vma, pad_vma, m, (show_pad_vma_fn)((void*)show_smap)); + show_map_pad_vma(vma, pad_vma, m, show_smap, true); return 0; } diff --git a/include/linux/pgsize_migration.h b/include/linux/pgsize_migration.h index 5c47ec28ea7d..fbfb1b9b9196 100644 --- a/include/linux/pgsize_migration.h +++ b/include/linux/pgsize_migration.h @@ -43,8 +43,6 @@ #define VM_PAD_MASK (VM_TOTAL_PAD_PAGES << VM_PAD_SHIFT) #define VMA_PAD_START(vma) (vma->vm_end - (vma_pad_pages(vma) << PAGE_SHIFT)) -typedef void (*show_pad_vma_fn)(struct seq_file *m, struct vm_area_struct *vma); - #if PAGE_SIZE == SZ_4K && defined(CONFIG_64BIT) extern void vma_set_pad_pages(struct vm_area_struct *vma, unsigned long nr_pages); @@ -60,7 +58,7 @@ extern struct vm_area_struct *get_data_vma(struct vm_area_struct *vma); extern void show_map_pad_vma(struct vm_area_struct *vma, struct vm_area_struct *pad, - struct seq_file *m, show_pad_vma_fn func); + struct seq_file *m, void *func, bool smaps); extern void split_pad_vma(struct vm_area_struct *vma, struct vm_area_struct *new, unsigned long addr, int new_below); @@ -92,7 +90,7 @@ static inline struct vm_area_struct *get_data_vma(struct vm_area_struct *vma) static inline void show_map_pad_vma(struct vm_area_struct *vma, struct vm_area_struct *pad, - struct seq_file *m, show_pad_vma_fn func) + struct seq_file *m, void *func, bool smaps) { } diff --git a/mm/pgsize_migration.c b/mm/pgsize_migration.c index 8b31cdb54383..a972b9ba921c 100644 --- a/mm/pgsize_migration.c +++ b/mm/pgsize_migration.c @@ -20,6 +20,9 @@ #include #include +typedef void (*show_pad_maps_fn) (struct seq_file *m, struct vm_area_struct *vma); +typedef int (*show_pad_smaps_fn) (struct seq_file *m, void *v); + #ifdef CONFIG_64BIT #if PAGE_SIZE == SZ_4K DEFINE_STATIC_KEY_TRUE(pgsize_migration_enabled); @@ -303,7 +306,7 @@ struct vm_area_struct *get_data_vma(struct vm_area_struct *vma) * and @pad. */ void show_map_pad_vma(struct vm_area_struct *vma, struct vm_area_struct *pad, - struct seq_file *m, show_pad_vma_fn func) + struct seq_file *m, void *func, bool smaps) { if (!pad) return; @@ -320,7 +323,10 @@ void show_map_pad_vma(struct vm_area_struct *vma, struct vm_area_struct *pad, */ BUG_ON(!vma); - func(m, pad); + if (smaps) + ((show_pad_smaps_fn)func)(m, pad); + else + ((show_pad_maps_fn)func)(m, pad); kfree(pad); kfree(vma); From dc6facfe02f89d24550471718c5427d36ec878c4 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Tue, 6 Feb 2024 17:18:21 -0800 Subject: [PATCH 62/86] UPSTREAM: net: tls: handle backlogging of crypto requests commit 8590541473188741055d27b955db0777569438e3 upstream. Since we're setting the CRYPTO_TFM_REQ_MAY_BACKLOG flag on our requests to the crypto API, crypto_aead_{encrypt,decrypt} can return -EBUSY instead of -EINPROGRESS in valid situations. For example, when the cryptd queue for AESNI is full (easy to trigger with an artificially low cryptd.cryptd_max_cpu_qlen), requests will be enqueued to the backlog but still processed. In that case, the async callback will also be called twice: first with err == -EINPROGRESS, which it seems we can just ignore, then with err == 0. Compared to Sabrina's original patch this version uses the new tls_*crypt_async_wait() helpers and converts the EBUSY to EINPROGRESS to avoid having to modify all the error handling paths. The handling is identical. Bug: 326215202 Fixes: a54667f6728c ("tls: Add support for encryption using async offload accelerator") Fixes: 94524d8fc965 ("net/tls: Add support for async decryption of tls records") Co-developed-by: Sabrina Dubroca Signed-off-by: Sabrina Dubroca Link: https://lore.kernel.org/netdev/9681d1febfec295449a62300938ed2ae66983f28.1694018970.git.sd@queasysnail.net/ Signed-off-by: Jakub Kicinski Reviewed-by: Simon Horman Signed-off-by: David S. Miller [Srish: v2: fixed hunk failures fixed merge-conflict in stable branch linux-6.1.y, needs to go on top of https://lore.kernel.org/stable/20240307155930.913525-1-lee@kernel.org/] Signed-off-by: Srish Srinivasan Signed-off-by: Greg Kroah-Hartman (cherry picked from commit cd1bbca03f3c1d845ce274c0d0a66de8e5929f72) Signed-off-by: Lee Jones Change-Id: I6aedd018e89a9aa2ace6633e02308336ed19fe13 --- net/tls/tls_sw.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c index 6767e19be79b..6b65e1e955a6 100644 --- a/net/tls/tls_sw.c +++ b/net/tls/tls_sw.c @@ -195,6 +195,17 @@ static void tls_decrypt_done(crypto_completion_data_t *data, int err) struct sock *sk; int aead_size; + /* If requests get too backlogged crypto API returns -EBUSY and calls + * ->complete(-EINPROGRESS) immediately followed by ->complete(0) + * to make waiting for backlog to flush with crypto_wait_req() easier. + * First wait converts -EBUSY -> -EINPROGRESS, and the second one + * -EINPROGRESS -> 0. + * We have a single struct crypto_async_request per direction, this + * scheme doesn't help us, so just ignore the first ->complete(). + */ + if (err == -EINPROGRESS) + return; + aead_size = sizeof(*aead_req) + crypto_aead_reqsize(aead); aead_size = ALIGN(aead_size, __alignof__(*dctx)); dctx = (void *)((u8 *)aead_req + aead_size); @@ -268,6 +279,10 @@ static int tls_do_decryption(struct sock *sk, } ret = crypto_aead_decrypt(aead_req); + if (ret == -EBUSY) { + ret = tls_decrypt_async_wait(ctx); + ret = ret ?: -EINPROGRESS; + } if (ret == -EINPROGRESS) { if (darg->async) return 0; @@ -449,6 +464,9 @@ static void tls_encrypt_done(crypto_completion_data_t *data, int err) struct tls_rec *rec; struct sock *sk; + if (err == -EINPROGRESS) /* see the comment in tls_decrypt_done() */ + return; + rec = container_of(aead_req, struct tls_rec, aead_req); msg_en = &rec->msg_encrypted; @@ -554,6 +572,10 @@ static int tls_do_encryption(struct sock *sk, atomic_inc(&ctx->encrypt_pending); rc = crypto_aead_encrypt(aead_req); + if (rc == -EBUSY) { + rc = tls_encrypt_async_wait(ctx); + rc = rc ?: -EINPROGRESS; + } if (!rc || rc != -EINPROGRESS) { atomic_dec(&ctx->encrypt_pending); sge->offset -= prot->prepend_size; From d5e04556d43461e0fd639618f19467656ac974ae Mon Sep 17 00:00:00 2001 From: Florian Westphal Date: Wed, 10 Apr 2024 21:05:13 +0200 Subject: [PATCH 63/86] UPSTREAM: netfilter: nft_set_pipapo: do not free live element [ Upstream commit 3cfc9ec039af60dbd8965ae085b2c2ccdcfbe1cc ] Pablo reports a crash with large batches of elements with a back-to-back add/remove pattern. Quoting Pablo: add_elem("00000000") timeout 100 ms ... add_elem("0000000X") timeout 100 ms del_elem("0000000X") <---------------- delete one that was just added ... add_elem("00005000") timeout 100 ms 1) nft_pipapo_remove() removes element 0000000X Then, KASAN shows a splat. Looking at the remove function there is a chance that we will drop a rule that maps to a non-deactivated element. Removal happens in two steps, first we do a lookup for key k and return the to-be-removed element and mark it as inactive in the next generation. Then, in a second step, the element gets removed from the set/map. The _remove function does not work correctly if we have more than one element that share the same key. This can happen if we insert an element into a set when the set already holds an element with same key, but the element mapping to the existing key has timed out or is not active in the next generation. In such case its possible that removal will unmap the wrong element. If this happens, we will leak the non-deactivated element, it becomes unreachable. The element that got deactivated (and will be freed later) will remain reachable in the set data structure, this can result in a crash when such an element is retrieved during lookup (stale pointer). Add a check that the fully matching key does in fact map to the element that we have marked as inactive in the deactivation step. If not, we need to continue searching. Add a bug/warn trap at the end of the function as well, the remove function must not ever be called with an invisible/unreachable/non-existent element. v2: avoid uneeded temporary variable (Stefano) Bug: 336735501 Fixes: 3c4287f62044 ("nf_tables: Add set type for arbitrary concatenation of ranges") Reported-by: Pablo Neira Ayuso Reviewed-by: Stefano Brivio Signed-off-by: Florian Westphal Signed-off-by: Pablo Neira Ayuso Signed-off-by: Sasha Levin (cherry picked from commit ebf7c9746f073035ee26209e38c3a1170f7b349a) Signed-off-by: Lee Jones Change-Id: Ic9a48ac9ac0f9960fea9e066d9a0a9fb93f7b633 --- net/netfilter/nft_set_pipapo.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/net/netfilter/nft_set_pipapo.c b/net/netfilter/nft_set_pipapo.c index 050672ccfa7e..381c1871be27 100644 --- a/net/netfilter/nft_set_pipapo.c +++ b/net/netfilter/nft_set_pipapo.c @@ -1975,6 +1975,8 @@ static void nft_pipapo_remove(const struct net *net, const struct nft_set *set, rules_fx = rules_f0; nft_pipapo_for_each_field(f, i, m) { + bool last = i == m->field_count - 1; + if (!pipapo_match_field(f, start, rules_fx, match_start, match_end)) break; @@ -1987,16 +1989,18 @@ static void nft_pipapo_remove(const struct net *net, const struct nft_set *set, match_start += NFT_PIPAPO_GROUPS_PADDED_SIZE(f); match_end += NFT_PIPAPO_GROUPS_PADDED_SIZE(f); - } - if (i == m->field_count) { - priv->dirty = true; - pipapo_drop(m, rulemap); - return; + if (last && f->mt[rulemap[i].to].e == e) { + priv->dirty = true; + pipapo_drop(m, rulemap); + return; + } } first_rule += rules_f0; } + + WARN_ON_ONCE(1); /* elem_priv not found */ } /** From fae94bc4e7aed899b755dbe7d23d563dd5a68340 Mon Sep 17 00:00:00 2001 From: fengqi Date: Mon, 15 Apr 2024 18:04:44 +0800 Subject: [PATCH 64/86] ANDROID: GKI: update symbol list file for xiaomi INFO: 68 function symbol(s) added 'int __break_lease(struct inode*, unsigned int, unsigned int)' 'void __d_lookup_unhash_wake(struct dentry*)' 'int __folio_lock_killable(struct folio*)' 'int add_swap_extent(struct swap_info_struct*, unsigned long, unsigned long, sector_t)' 'int asn1_ber_decoder(const struct asn1_decoder*, void*, const unsigned char*, size_t)' 'size_t copy_page_from_iter(struct page*, size_t, size_t, struct iov_iter*)' 'size_t copy_page_to_iter(struct page*, size_t, size_t, struct iov_iter*)' 'struct dentry* d_alloc_parallel(struct dentry*, const struct qstr*, wait_queue_head_t*)' 'void d_drop(struct dentry*)' 'struct dentry* d_hash_and_lookup(struct dentry*, struct qstr*)' 'void d_invalidate(struct dentry*)' 'void d_move(struct dentry*, struct dentry*)' 'void deactivate_super(struct super_block*)' 'int errseq_check(errseq_t*, errseq_t)' 'int file_modified(struct file*)' 'vm_fault_t filemap_map_pages(struct vm_fault*, unsigned long, unsigned long)' 'int finish_no_open(struct file*, struct dentry*)' 'int finish_open(struct file*, struct dentry*, int(*)(struct inode*, struct file*))' 'bool folio_clear_dirty_for_io(struct folio*)' 'void folio_wait_private_2(struct folio*)' 'int fs_param_is_bool(struct p_log*, const struct fs_parameter_spec*, struct fs_parameter*, struct fs_parse_result*)' 'int fs_param_is_u64(struct p_log*, const struct fs_parameter_spec*, struct fs_parameter*, struct fs_parse_result*)' 'void generate_random_uuid(unsigned char*)' 'ssize_t generic_copy_file_range(struct file*, loff_t, struct file*, loff_t, size_t, unsigned int)' 'ssize_t generic_file_write_iter(struct kiocb*, struct iov_iter*)' 'int generic_key_instantiate(struct key*, struct key_preparsed_payload*)' 'int generic_setlease(struct file*, long, struct file_lock**, void**)' 'int invalidate_inode_pages2(struct address_space*)' 'void iov_iter_bvec(struct iov_iter*, unsigned int, const struct bio_vec*, unsigned long, size_t)' 'void iov_iter_discard(struct iov_iter*, unsigned int, size_t)' 'ssize_t iov_iter_get_pages_alloc2(struct iov_iter*, struct page***, size_t, size_t*)' 'int iov_iter_npages(const struct iov_iter*, int)' 'void key_invalidate(struct key*)' 'void key_revoke(struct key*)' 'int key_validate(const struct key*)' 'void lock_two_nondirectories(struct inode*, struct inode*)' 'int locks_lock_inode_wait(struct inode*, struct file_lock*)' 'enum OID look_up_OID(const void*, size_t)' 'struct dentry* lookup_positive_unlocked(const char*, struct dentry*, int)' 'int out_of_line_wait_on_bit_lock(void*, int, wait_bit_action_f*, unsigned int)' 'const struct cred* override_creds(const struct cred*)' 'int posix_lock_file(struct file*, struct file_lock*, struct file_lock*)' 'void posix_test_lock(struct file*, struct file_lock*)' 'struct cred* prepare_kernel_cred(struct task_struct*)' 'int register_key_type(struct key_type*)' 'struct key* request_key_tag(struct key_type*, const char*, struct key_tag*, const char*)' 'void revert_creds(const struct cred*)' 'int security_sb_eat_lsm_opts(char*, void**)' 'void seq_escape_mem(struct seq_file*, const char*, size_t, unsigned int, const char*)' 'int set_anon_super(struct super_block*, void*)' 'struct super_block* sget(struct file_system_type*, int(*)(struct super_block*, void*), int(*)(struct super_block*, void*), int, void*)' 'int sock_sendmsg(struct socket*, struct msghdr*)' 'int sprint_oid(const void*, size_t, char*, size_t)' 'int super_setup_bdi(struct super_block*)' 'void tcp_sock_set_cork(struct sock*, bool)' 'void tcp_sock_set_nodelay(struct sock*)' 'void truncate_inode_pages_range(struct address_space*, loff_t, loff_t)' 'void truncate_pagecache_range(struct inode*, loff_t, loff_t)' 'void unlock_two_nondirectories(struct inode*, struct inode*)' 'void unregister_key_type(struct key_type*)' 'void user_describe(const struct key*, struct seq_file*)' 'void user_destroy(struct key*)' 'long user_read(const struct key*, char*, size_t)' 'void user_revoke(struct key*)' 'int utf8s_to_utf16s(const u8*, int, enum utf16_endian, wchar_t*, int)' 'int vfs_parse_fs_string(struct fs_context*, const char*, const char*, size_t)' 'loff_t vfs_setpos(struct file*, loff_t, loff_t)' 'void wait_on_page_writeback(struct page*)' 1 variable symbol(s) added 'struct key_type key_type_logon' Bug: 334801999 Change-Id: I89f2029bb73155ffd0dd4236de32a7c74d1a8f48 Signed-off-by: fengqi --- android/abi_gki_aarch64.stg | 1033 ++++++++++++++++++++++++++++++++ android/abi_gki_aarch64_xiaomi | 84 +++ 2 files changed, 1117 insertions(+) diff --git a/android/abi_gki_aarch64.stg b/android/abi_gki_aarch64.stg index 1c01560d1a91..6be4e52b1610 100644 --- a/android/abi_gki_aarch64.stg +++ b/android/abi_gki_aarch64.stg @@ -2313,6 +2313,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x0b2665f4 } +pointer_reference { + id: 0x085c0227 + kind: POINTER + pointee_type_id: 0x0b30ee00 +} pointer_reference { id: 0x08670159 kind: POINTER @@ -15598,6 +15603,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x9865c1f0 } +pointer_reference { + id: 0x2c8a0c1e + kind: POINTER + pointee_type_id: 0x9868d6e6 +} pointer_reference { id: 0x2c8a36ad kind: POINTER @@ -21778,6 +21788,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x94570fc4 } +pointer_reference { + id: 0x2f865a8b + kind: POINTER + pointee_type_id: 0x94598cb1 +} pointer_reference { id: 0x2f892792 kind: POINTER @@ -24418,6 +24433,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0xfb4fcd23 } +pointer_reference { + id: 0x3448bbf6 + kind: POINTER + pointee_type_id: 0xfb620945 +} pointer_reference { id: 0x344a8f68 kind: POINTER @@ -26843,6 +26863,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0xce8060c4 } +pointer_reference { + id: 0x39306b3c + kind: POINTER + pointee_type_id: 0xce814a6c +} pointer_reference { id: 0x3930a655 kind: POINTER @@ -29978,6 +30003,11 @@ typedef { name: "arch_spinlock_t" referred_type_id: 0xeea291c0 } +typedef { + id: 0x9dcf939f + name: "asn1_action_t" + referred_type_id: 0x2c8a0c1e +} typedef { id: 0x66ba4c2f name: "assert_format_t" @@ -32833,6 +32863,11 @@ qualified { qualifier: CONST qualified_type_id: 0x4a463598 } +qualified { + id: 0xce814a6c + qualifier: CONST + qualified_type_id: 0x4a429f38 +} qualified { id: 0xce827fcb qualifier: CONST @@ -35263,6 +35298,11 @@ qualified { qualifier: CONST qualified_type_id: 0x9d788207 } +qualified { + id: 0xfb620945 + qualifier: CONST + qualified_type_id: 0x9dcf939f +} qualified { id: 0xfb6adb3d qualifier: CONST @@ -43749,6 +43789,12 @@ member { type_id: 0x057af395 offset: 64 } +member { + id: 0x0d6a6841 + name: "actions" + type_id: 0x3448bbf6 + offset: 128 +} member { id: 0xdc3d085b name: "activate" @@ -121594,6 +121640,17 @@ member { name: "machine" type_id: 0x3e10b518 } +member { + id: 0xcddcf0f0 + name: "machine" + type_id: 0x384c5795 +} +member { + id: 0x76de8d67 + name: "machlen" + type_id: 0xf435685e + offset: 64 +} member { id: 0xd88139c8 name: "macsec_info" @@ -219130,6 +219187,17 @@ struct_union { member_id: 0x5161b04d } } +struct_union { + id: 0x4a429f38 + kind: STRUCT + name: "asn1_decoder" + definition { + bytesize: 24 + member_id: 0xcddcf0f0 + member_id: 0x76de8d67 + member_id: 0x0d6a6841 + } +} struct_union { id: 0xaf70b18b kind: STRUCT @@ -290763,6 +290831,11 @@ function { parameter_id: 0xb3e7bac9 parameter_id: 0xa9403390 } +function { + id: 0x01831291 + return_type_id: 0x30650ba6 + parameter_id: 0x30650ba6 +} function { id: 0x019aa1b7 return_type_id: 0x48b5725f @@ -293028,6 +293101,15 @@ function { parameter_id: 0x00b7947f parameter_id: 0x5d8155a5 } +function { + id: 0x11c57e21 + return_type_id: 0x48b5725f + parameter_id: 0x0665e6b6 + parameter_id: 0x3e10b518 + parameter_id: 0xf435685e + parameter_id: 0x4585663f + parameter_id: 0x3e10b518 +} function { id: 0x11c6c94b return_type_id: 0x48b5725f @@ -294878,6 +294960,12 @@ function { parameter_id: 0x13b93798 parameter_id: 0x0980a662 } +function { + id: 0x145114a4 + return_type_id: 0x48b5725f + parameter_id: 0x120540d1 + parameter_id: 0x120540d1 +} function { id: 0x14532743 return_type_id: 0xd5cc9c9a @@ -295466,6 +295554,11 @@ function { parameter_id: 0x12e24ee1 parameter_id: 0x4585663f } +function { + id: 0x1539d76a + return_type_id: 0x48b5725f + parameter_id: 0x16861be6 +} function { id: 0x1539f399 return_type_id: 0x48b5725f @@ -295652,6 +295745,13 @@ function { parameter_id: 0x0483e6f8 parameter_id: 0x6720d32f } +function { + id: 0x156f8dbb + return_type_id: 0x48b5725f + parameter_id: 0x1582ab06 + parameter_id: 0x27a7c613 + parameter_id: 0x27a7c613 +} function { id: 0x15705d39 return_type_id: 0x48b5725f @@ -296030,6 +296130,12 @@ function { parameter_id: 0x184052fc parameter_id: 0x27a7c613 } +function { + id: 0x161723fb + return_type_id: 0x48b5725f + parameter_id: 0x1b8590a8 + parameter_id: 0x1b8590a8 +} function { id: 0x16188b98 return_type_id: 0x48b5725f @@ -296114,6 +296220,12 @@ function { return_type_id: 0x48b5725f parameter_id: 0x1b55a8f2 } +function { + id: 0x1652567d + return_type_id: 0x48b5725f + parameter_id: 0x18ea6ae3 + parameter_id: 0x3c2755a3 +} function { id: 0x165365f0 return_type_id: 0x48b5725f @@ -296326,6 +296438,12 @@ function { parameter_id: 0xa9183061 parameter_id: 0x6720d32f } +function { + id: 0x167ca077 + return_type_id: 0x48b5725f + parameter_id: 0x1d44326e + parameter_id: 0x6d7f5ff6 +} function { id: 0x167dac02 return_type_id: 0xf23e8927 @@ -297376,6 +297494,11 @@ function { parameter_id: 0x1b36c7a2 parameter_id: 0x6d7f5ff6 } +function { + id: 0x17e44aa0 + return_type_id: 0x48b5725f + parameter_id: 0x1df06cce +} function { id: 0x17e4a48e return_type_id: 0x48b5725f @@ -298531,6 +298654,15 @@ function { parameter_id: 0x33756485 parameter_id: 0xf435685e } +function { + id: 0x1a5df559 + return_type_id: 0x48b5725f + parameter_id: 0x2f73da3b + parameter_id: 0x4585663f + parameter_id: 0x3edc1393 + parameter_id: 0x33756485 + parameter_id: 0xf435685e +} function { id: 0x1a5fed1b return_type_id: 0x48b5725f @@ -298555,6 +298687,13 @@ function { return_type_id: 0x48b5725f parameter_id: 0x2bdd20a6 } +function { + id: 0x1a6fbfdf + return_type_id: 0x48b5725f + parameter_id: 0x2f73da3b + parameter_id: 0x4585663f + parameter_id: 0xf435685e +} function { id: 0x1a70a9d4 return_type_id: 0x48b5725f @@ -299895,6 +300034,11 @@ function { parameter_id: 0x32a623d7 parameter_id: 0x139e8957 } +function { + id: 0x1c81137a + return_type_id: 0x48b5725f + parameter_id: 0x30650ba6 +} function { id: 0x1c822746 return_type_id: 0x48b5725f @@ -302201,6 +302345,14 @@ function { parameter_id: 0x3e10b518 parameter_id: 0x1b44744f } +function { + id: 0x1f31b428 + return_type_id: 0xd5cc9c9a + parameter_id: 0x2f73da3b + parameter_id: 0x085c0227 + parameter_id: 0xf435685e + parameter_id: 0x379d63b0 +} function { id: 0x1f380b01 return_type_id: 0x48b5725f @@ -303446,6 +303598,13 @@ function { parameter_id: 0x2e18f543 parameter_id: 0x1bf16028 } +function { + id: 0x2d748ffb + return_type_id: 0x27a7c613 + parameter_id: 0x18ea6ae3 + parameter_id: 0x27a7c613 + parameter_id: 0x27a7c613 +} function { id: 0x2d7bcaf0 return_type_id: 0x27a7c613 @@ -304845,6 +305004,11 @@ function { return_type_id: 0xc9082b19 parameter_id: 0x0c0dfa25 } +function { + id: 0x472c0143 + return_type_id: 0x3d551c03 + parameter_id: 0x1d19a9d5 +} function { id: 0x47570b9f return_type_id: 0x2131312a @@ -308067,6 +308231,13 @@ function { parameter_id: 0x6720d32f parameter_id: 0x0258f96e } +function { + id: 0x83a40349 + return_type_id: 0x120540d1 + parameter_id: 0x3e10b518 + parameter_id: 0x120540d1 + parameter_id: 0x6720d32f +} function { id: 0x83b24f50 return_type_id: 0x120540d1 @@ -308426,6 +308597,14 @@ function { parameter_id: 0x92233392 parameter_id: 0x49f541d1 } +function { + id: 0x8659e72e + return_type_id: 0x01a688ae + parameter_id: 0x16861be6 + parameter_id: 0x3e10b518 + parameter_id: 0x22bb3fbf + parameter_id: 0x3e10b518 +} function { id: 0x86c30f0f return_type_id: 0x6720d32f @@ -308497,6 +308676,13 @@ function { parameter_id: 0x120540d1 parameter_id: 0x3e10b518 } +function { + id: 0x880a3e61 + return_type_id: 0x120540d1 + parameter_id: 0x120540d1 + parameter_id: 0x3e001c39 + parameter_id: 0x0a747547 +} function { id: 0x8817e7ae return_type_id: 0x120540d1 @@ -308530,6 +308716,12 @@ function { parameter_id: 0x120540d1 parameter_id: 0x3e10b518 } +function { + id: 0x88e3a605 + return_type_id: 0x120540d1 + parameter_id: 0x120540d1 + parameter_id: 0x04c1427f +} function { id: 0x88f0a30c return_type_id: 0x120540d1 @@ -309053,6 +309245,14 @@ function { parameter_id: 0x23c30672 parameter_id: 0x18bd6530 } +function { + id: 0x901503d8 + return_type_id: 0x6720d32f + parameter_id: 0x391f15ea + parameter_id: 0xf435685e + parameter_id: 0x0483e6f8 + parameter_id: 0xf435685e +} function { id: 0x90166130 return_type_id: 0x6720d32f @@ -310235,6 +310435,12 @@ function { return_type_id: 0x6720d32f parameter_id: 0x32941b2b } +function { + id: 0x91274ee9 + return_type_id: 0x6720d32f + parameter_id: 0x34ecba28 + parameter_id: 0x6720d32f +} function { id: 0x91296bda return_type_id: 0x6720d32f @@ -312215,6 +312421,11 @@ function { parameter_id: 0x3f0ff1b1 parameter_id: 0x054f691a } +function { + id: 0x9256988c + return_type_id: 0x6720d32f + parameter_id: 0x3f59ee8d +} function { id: 0x9257838d return_type_id: 0xf435685e @@ -314050,6 +314261,14 @@ function { parameter_id: 0x13580d6c parameter_id: 0xc582424e } +function { + id: 0x93a3cfb4 + return_type_id: 0x6720d32f + parameter_id: 0x39306b3c + parameter_id: 0x18bd6530 + parameter_id: 0x384c5795 + parameter_id: 0xf435685e +} function { id: 0x93a3ea47 return_type_id: 0x3e10b518 @@ -314271,6 +314490,15 @@ function { parameter_id: 0x33756485 parameter_id: 0x33756485 } +function { + id: 0x93e80c53 + return_type_id: 0x6720d32f + parameter_id: 0x3f0185ef + parameter_id: 0x6720d32f + parameter_id: 0xd0cce131 + parameter_id: 0x09eee7e5 + parameter_id: 0x6720d32f +} function { id: 0x93e8301d return_type_id: 0x6720d32f @@ -314730,6 +314958,12 @@ function { parameter_id: 0x34434827 parameter_id: 0xcddc4fff } +function { + id: 0x94598cb1 + return_type_id: 0x6720d32f + parameter_id: 0x26ee682a + parameter_id: 0x18bd6530 +} function { id: 0x945a66f1 return_type_id: 0x6720d32f @@ -315487,6 +315721,14 @@ function { parameter_id: 0x18bd6530 parameter_id: 0x4585663f } +function { + id: 0x95727bc1 + return_type_id: 0xf435685e + parameter_id: 0x06835e9c + parameter_id: 0xf435685e + parameter_id: 0xf435685e + parameter_id: 0x2f73da3b +} function { id: 0x9573e3f3 return_type_id: 0x6720d32f @@ -315816,6 +316058,14 @@ function { parameter_id: 0x6720d32f parameter_id: 0x3a8e2ac6 } +function { + id: 0x95bf15d7 + return_type_id: 0x6720d32f + parameter_id: 0x23f6f3f6 + parameter_id: 0x33756485 + parameter_id: 0x33756485 + parameter_id: 0xd0b3a203 +} function { id: 0x95c0ec46 return_type_id: 0x6720d32f @@ -316346,6 +316596,14 @@ function { parameter_id: 0x03c33b38 parameter_id: 0x19671b46 } +function { + id: 0x96a3069e + return_type_id: 0x6720d32f + parameter_id: 0x2f5fce77 + parameter_id: 0x3e10b518 + parameter_id: 0x3e10b518 + parameter_id: 0xf435685e +} function { id: 0x96a3f292 return_type_id: 0x3e10b518 @@ -317466,6 +317724,11 @@ function { parameter_id: 0x33756485 parameter_id: 0x33756485 } +function { + id: 0x982165d6 + return_type_id: 0x6720d32f + parameter_id: 0x16861be6 +} function { id: 0x982246a7 return_type_id: 0x6720d32f @@ -317785,6 +318048,15 @@ function { return_type_id: 0x6720d32f parameter_id: 0x17a37c1c } +function { + id: 0x9868d6e6 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0xf435685e + parameter_id: 0x5d8155a5 + parameter_id: 0x391f15ea + parameter_id: 0xf435685e +} function { id: 0x986a2160 return_type_id: 0x6720d32f @@ -318774,6 +319046,12 @@ function { parameter_id: 0x3806a5e7 parameter_id: 0x4585663f } +function { + id: 0x992b8783 + return_type_id: 0x6720d32f + parameter_id: 0x106029c6 + parameter_id: 0x2cdbb77a +} function { id: 0x992e9a67 return_type_id: 0x6720d32f @@ -320467,6 +320745,13 @@ function { parameter_id: 0x2cdbb77a parameter_id: 0x10e15e7e } +function { + id: 0x9a66f3c4 + return_type_id: 0x6720d32f + parameter_id: 0x1b8590a8 + parameter_id: 0x4585663f + parameter_id: 0x4585663f +} function { id: 0x9a68883d return_type_id: 0x2668e644 @@ -321996,6 +322281,13 @@ function { parameter_id: 0x1dc3d428 parameter_id: 0x6d7f5ff6 } +function { + id: 0x9b45ed14 + return_type_id: 0x6720d32f + parameter_id: 0x18ea6ae3 + parameter_id: 0x3c2755a3 + parameter_id: 0x3c2755a3 +} function { id: 0x9b45fcbe return_type_id: 0x6720d32f @@ -323301,6 +323593,12 @@ function { return_type_id: 0x6720d32f parameter_id: 0x1847a854 } +function { + id: 0x9b911a53 + return_type_id: 0x6720d32f + parameter_id: 0x1b8590a8 + parameter_id: 0x3c2755a3 +} function { id: 0x9b91c1d8 return_type_id: 0x6720d32f @@ -325319,6 +325617,12 @@ function { parameter_id: 0x11b57133 parameter_id: 0x6d7f5ff6 } +function { + id: 0x9bf26c94 + return_type_id: 0x6720d32f + parameter_id: 0x18ea6ae3 + parameter_id: 0x120540d1 +} function { id: 0x9bf2ff2c return_type_id: 0x6720d32f @@ -325406,6 +325710,13 @@ function { parameter_id: 0x18bd6530 parameter_id: 0x15a30023 } +function { + id: 0x9bf978b8 + return_type_id: 0x6720d32f + parameter_id: 0x18ea6ae3 + parameter_id: 0x120540d1 + parameter_id: 0x2c50b20c +} function { id: 0x9bf97de0 return_type_id: 0x6720d32f @@ -326325,6 +326636,12 @@ function { parameter_id: 0x0d107c2b parameter_id: 0x6720d32f } +function { + id: 0x9c92e712 + return_type_id: 0x6720d32f + parameter_id: 0x0483e6f8 + parameter_id: 0x0cbf60eb +} function { id: 0x9c9a309d return_type_id: 0x6720d32f @@ -326671,6 +326988,12 @@ function { parameter_id: 0x0ca27481 parameter_id: 0xc9082b19 } +function { + id: 0x9cd412ab + return_type_id: 0x09f65231 + parameter_id: 0x391f15ea + parameter_id: 0xf435685e +} function { id: 0x9cd67c20 return_type_id: 0x6720d32f @@ -329171,6 +329494,12 @@ function { parameter_id: 0x29ef8105 parameter_id: 0x07dcdbe1 } +function { + id: 0x9e1c79d2 + return_type_id: 0x6720d32f + parameter_id: 0x0f12d1e7 + parameter_id: 0x160ba102 +} function { id: 0x9e1dd697 return_type_id: 0x6720d32f @@ -335885,6 +336214,11 @@ function { parameter_id: 0x5d8155a5 parameter_id: 0x6720d32f } +function { + id: 0xf6f86f1f + return_type_id: 0x6d7f5ff6 + parameter_id: 0x2170d06d +} function { id: 0xf6fffafa return_type_id: 0x6d7f5ff6 @@ -335896,6 +336230,15 @@ function { return_type_id: 0x3e6239e1 parameter_id: 0x269c8e40 } +function { + id: 0xf719dbb9 + return_type_id: 0x26ee682a + parameter_id: 0x32ec08f2 + parameter_id: 0x2f865a8b + parameter_id: 0x2f865a8b + parameter_id: 0x6720d32f + parameter_id: 0x18bd6530 +} function { id: 0xf71fc10e return_type_id: 0x6d7f5ff6 @@ -337202,6 +337545,15 @@ elf_symbol { type_id: 0x10c07815 full_name: "__breadahead" } +elf_symbol { + id: 0x23bb1952 + name: "__break_lease" + is_defined: true + symbol_type: FUNCTION + crc: 0x453c1451 + type_id: 0x9a66f3c4 + full_name: "__break_lease" +} elf_symbol { id: 0x2f449b9e name: "__brelse" @@ -337481,6 +337833,15 @@ elf_symbol { type_id: 0x105b48b9 full_name: "__crypto_xor" } +elf_symbol { + id: 0xc1737e0f + name: "__d_lookup_unhash_wake" + is_defined: true + symbol_type: FUNCTION + crc: 0xe37fbf73 + type_id: 0x141901a7 + full_name: "__d_lookup_unhash_wake" +} elf_symbol { id: 0x087c7a06 name: "__dev_change_net_namespace" @@ -338165,6 +338526,15 @@ elf_symbol { type_id: 0x18c46588 full_name: "__folio_lock" } +elf_symbol { + id: 0x69ff7fd9 + name: "__folio_lock_killable" + is_defined: true + symbol_type: FUNCTION + crc: 0xf1e1ecf4 + type_id: 0x95dcd734 + full_name: "__folio_lock_killable" +} elf_symbol { id: 0xebf4b11f name: "__folio_put" @@ -349578,6 +349948,15 @@ elf_symbol { type_id: 0x8625f433 full_name: "add_memory" } +elf_symbol { + id: 0x7951d828 + name: "add_swap_extent" + is_defined: true + symbol_type: FUNCTION + crc: 0x2a731aba + type_id: 0x95bf15d7 + full_name: "add_swap_extent" +} elf_symbol { id: 0xa5dcf915 name: "add_taint" @@ -350134,6 +350513,15 @@ elf_symbol { type_id: 0x77670b3b full_name: "arp_tbl" } +elf_symbol { + id: 0xac417394 + name: "asn1_ber_decoder" + is_defined: true + symbol_type: FUNCTION + crc: 0x13ce87e8 + type_id: 0x93a3cfb4 + full_name: "asn1_ber_decoder" +} elf_symbol { id: 0xd9184490 name: "async_schedule_node" @@ -354412,6 +354800,15 @@ elf_symbol { symbol_type: FUNCTION crc: 0x4d0d163d } +elf_symbol { + id: 0xdd2942d2 + name: "copy_page_from_iter" + is_defined: true + symbol_type: FUNCTION + crc: 0x2ab5859e + type_id: 0x95727bc1 + full_name: "copy_page_from_iter" +} elf_symbol { id: 0x9f1f7cee name: "copy_page_from_iter_atomic" @@ -354421,6 +354818,15 @@ elf_symbol { type_id: 0x97b4bbf8 full_name: "copy_page_from_iter_atomic" } +elf_symbol { + id: 0x5e964278 + name: "copy_page_to_iter" + is_defined: true + symbol_type: FUNCTION + crc: 0x6d647988 + type_id: 0x95727bc1 + full_name: "copy_page_to_iter" +} elf_symbol { id: 0xd89255c2 name: "cpu_all_bits" @@ -356138,6 +356544,24 @@ elf_symbol { type_id: 0x8808e1d8 full_name: "d_alloc_name" } +elf_symbol { + id: 0x4b8dc0ec + name: "d_alloc_parallel" + is_defined: true + symbol_type: FUNCTION + crc: 0xc0b750e0 + type_id: 0x880a3e61 + full_name: "d_alloc_parallel" +} +elf_symbol { + id: 0xfeea461b + name: "d_drop" + is_defined: true + symbol_type: FUNCTION + crc: 0xfc09b04e + type_id: 0x141901a7 + full_name: "d_drop" +} elf_symbol { id: 0x9534fec7 name: "d_find_alias" @@ -356147,6 +356571,15 @@ elf_symbol { type_id: 0x8a909712 full_name: "d_find_alias" } +elf_symbol { + id: 0x66e2e005 + name: "d_hash_and_lookup" + is_defined: true + symbol_type: FUNCTION + crc: 0x2bf908e6 + type_id: 0x88e3a605 + full_name: "d_hash_and_lookup" +} elf_symbol { id: 0x04f5a6e5 name: "d_instantiate" @@ -356156,6 +356589,15 @@ elf_symbol { type_id: 0x147717e5 full_name: "d_instantiate" } +elf_symbol { + id: 0x53946e0e + name: "d_invalidate" + is_defined: true + symbol_type: FUNCTION + crc: 0x0d4bcdcf + type_id: 0x141901a7 + full_name: "d_invalidate" +} elf_symbol { id: 0x52864a7e name: "d_make_root" @@ -356165,6 +356607,15 @@ elf_symbol { type_id: 0x8a909712 full_name: "d_make_root" } +elf_symbol { + id: 0x03fd2deb + name: "d_move" + is_defined: true + symbol_type: FUNCTION + crc: 0xbdb87322 + type_id: 0x145114a4 + full_name: "d_move" +} elf_symbol { id: 0x8d5ab7b4 name: "d_obtain_alias" @@ -356255,6 +356706,15 @@ elf_symbol { type_id: 0x1923cb99 full_name: "deactivate_locked_super" } +elf_symbol { + id: 0xd6386fd8 + name: "deactivate_super" + is_defined: true + symbol_type: FUNCTION + crc: 0xa0853c0c + type_id: 0x1923cb99 + full_name: "deactivate_super" +} elf_symbol { id: 0x36a5b3df name: "deactivate_task" @@ -365941,6 +366401,15 @@ elf_symbol { type_id: 0x53bd7367 full_name: "errno_to_blk_status" } +elf_symbol { + id: 0x3fdfcd47 + name: "errseq_check" + is_defined: true + symbol_type: FUNCTION + crc: 0x769f6e64 + type_id: 0x9e1c79d2 + full_name: "errseq_check" +} elf_symbol { id: 0xe506b877 name: "errseq_set" @@ -366418,6 +366887,15 @@ elf_symbol { type_id: 0x9bba7997 full_name: "file_check_and_advance_wb_err" } +elf_symbol { + id: 0x08df0c2d + name: "file_modified" + is_defined: true + symbol_type: FUNCTION + crc: 0xef52b15f + type_id: 0x9bba7997 + full_name: "file_modified" +} elf_symbol { id: 0xe7aacfe5 name: "file_path" @@ -366535,6 +367013,15 @@ elf_symbol { type_id: 0xcbaec654 full_name: "filemap_get_folios" } +elf_symbol { + id: 0x70cbb212 + name: "filemap_map_pages" + is_defined: true + symbol_type: FUNCTION + crc: 0xa7ecd728 + type_id: 0xce25f646 + full_name: "filemap_map_pages" +} elf_symbol { id: 0xc7c68a05 name: "filemap_write_and_wait_range" @@ -366643,6 +367130,24 @@ elf_symbol { type_id: 0x318306ba full_name: "find_vpid" } +elf_symbol { + id: 0xdc38ff2a + name: "finish_no_open" + is_defined: true + symbol_type: FUNCTION + crc: 0x57a019b3 + type_id: 0x9bf26c94 + full_name: "finish_no_open" +} +elf_symbol { + id: 0x557a4fc7 + name: "finish_open" + is_defined: true + symbol_type: FUNCTION + crc: 0xf036e691 + type_id: 0x9bf978b8 + full_name: "finish_open" +} elf_symbol { id: 0xf882020f name: "finish_wait" @@ -366814,6 +367319,15 @@ elf_symbol { type_id: 0x18c46588 full_name: "folio_add_lru" } +elf_symbol { + id: 0xab55569c + name: "folio_clear_dirty_for_io" + is_defined: true + symbol_type: FUNCTION + crc: 0x017bb38c + type_id: 0xf6f86f1f + full_name: "folio_clear_dirty_for_io" +} elf_symbol { id: 0x159a69a3 name: "folio_mapping" @@ -366841,6 +367355,15 @@ elf_symbol { type_id: 0x1958e6c4 full_name: "folio_wait_bit" } +elf_symbol { + id: 0xb917d8a9 + name: "folio_wait_private_2" + is_defined: true + symbol_type: FUNCTION + crc: 0x96cda8a4 + type_id: 0x18c46588 + full_name: "folio_wait_private_2" +} elf_symbol { id: 0x07508208 name: "folio_write_one" @@ -367138,6 +367661,15 @@ elf_symbol { type_id: 0x23f09c34 full_name: "fs_kobj" } +elf_symbol { + id: 0x55ce3162 + name: "fs_param_is_bool" + is_defined: true + symbol_type: FUNCTION + crc: 0xe07fe2d5 + type_id: 0x96b04c18 + full_name: "fs_param_is_bool" +} elf_symbol { id: 0x6c9f28bc name: "fs_param_is_string" @@ -367156,6 +367688,15 @@ elf_symbol { type_id: 0x96b04c18 full_name: "fs_param_is_u32" } +elf_symbol { + id: 0x4bab97c7 + name: "fs_param_is_u64" + is_defined: true + symbol_type: FUNCTION + crc: 0x2de74831 + type_id: 0x96b04c18 + full_name: "fs_param_is_u64" +} elf_symbol { id: 0xbc2aa09b name: "fscrypt_file_open" @@ -367633,6 +368174,15 @@ elf_symbol { type_id: 0x044f79a4 full_name: "gen_pool_virt_to_phys" } +elf_symbol { + id: 0xb7f431e8 + name: "generate_random_uuid" + is_defined: true + symbol_type: FUNCTION + crc: 0xa681fe88 + type_id: 0x17e44aa0 + full_name: "generate_random_uuid" +} elf_symbol { id: 0x0c22ac35 name: "generic_block_bmap" @@ -367642,6 +368192,15 @@ elf_symbol { type_id: 0xaf2275d4 full_name: "generic_block_bmap" } +elf_symbol { + id: 0x1b6a5b31 + name: "generic_copy_file_range" + is_defined: true + symbol_type: FUNCTION + crc: 0x1f38659f + type_id: 0x12537c3f + full_name: "generic_copy_file_range" +} elf_symbol { id: 0x30828743 name: "generic_device_group" @@ -367741,6 +368300,15 @@ elf_symbol { type_id: 0x12c79320 full_name: "generic_file_splice_read" } +elf_symbol { + id: 0xc7f4ee02 + name: "generic_file_write_iter" + is_defined: true + symbol_type: FUNCTION + crc: 0xf92b9d1b + type_id: 0x16637235 + full_name: "generic_file_write_iter" +} elf_symbol { id: 0x3812bb22 name: "generic_fillattr" @@ -367777,6 +368345,15 @@ elf_symbol { type_id: 0x8ce1baa0 full_name: "generic_handle_irq_safe" } +elf_symbol { + id: 0xf85d118d + name: "generic_key_instantiate" + is_defined: true + symbol_type: FUNCTION + crc: 0x87517341 + type_id: 0x9ddefbf5 + full_name: "generic_key_instantiate" +} elf_symbol { id: 0x95a840b8 name: "generic_mii_ioctl" @@ -367813,6 +368390,15 @@ elf_symbol { type_id: 0x12e4741f full_name: "generic_read_dir" } +elf_symbol { + id: 0x242af10d + name: "generic_setlease" + is_defined: true + symbol_type: FUNCTION + crc: 0x97921cea + type_id: 0x984b1567 + full_name: "generic_setlease" +} elf_symbol { id: 0x798f83da name: "generic_shutdown_super" @@ -372658,6 +373244,15 @@ elf_symbol { type_id: 0x167935b9 full_name: "invalidate_inode_buffers" } +elf_symbol { + id: 0x431bb67c + name: "invalidate_inode_pages2" + is_defined: true + symbol_type: FUNCTION + crc: 0x24c45d61 + type_id: 0x98e049ee + full_name: "invalidate_inode_pages2" +} elf_symbol { id: 0x38472bf4 name: "invalidate_inode_pages2_range" @@ -373144,6 +373739,24 @@ elf_symbol { type_id: 0x3c97e3d9 full_name: "iov_iter_alignment" } +elf_symbol { + id: 0xe15835cc + name: "iov_iter_bvec" + is_defined: true + symbol_type: FUNCTION + crc: 0x92e4a4e4 + type_id: 0x1a5df559 + full_name: "iov_iter_bvec" +} +elf_symbol { + id: 0x067b9161 + name: "iov_iter_discard" + is_defined: true + symbol_type: FUNCTION + crc: 0x183c762d + type_id: 0x1a6fbfdf + full_name: "iov_iter_discard" +} elf_symbol { id: 0x3d6623cb name: "iov_iter_get_pages2" @@ -373153,6 +373766,15 @@ elf_symbol { type_id: 0x1f3dc21f full_name: "iov_iter_get_pages2" } +elf_symbol { + id: 0x530f6f25 + name: "iov_iter_get_pages_alloc2" + is_defined: true + symbol_type: FUNCTION + crc: 0xf80a9ab0 + type_id: 0x1f31b428 + full_name: "iov_iter_get_pages_alloc2" +} elf_symbol { id: 0x71726a83 name: "iov_iter_init" @@ -373171,6 +373793,15 @@ elf_symbol { type_id: 0x1a5c3241 full_name: "iov_iter_kvec" } +elf_symbol { + id: 0x9978b03b + name: "iov_iter_npages" + is_defined: true + symbol_type: FUNCTION + crc: 0x3d8e20f5 + type_id: 0x91274ee9 + full_name: "iov_iter_npages" +} elf_symbol { id: 0x48a6183b name: "iov_iter_revert" @@ -374495,6 +375126,15 @@ elf_symbol { type_id: 0xf3944fac full_name: "key_create_or_update" } +elf_symbol { + id: 0x86d6e923 + name: "key_invalidate" + is_defined: true + symbol_type: FUNCTION + crc: 0x615cff5b + type_id: 0x10f1f3b8 + full_name: "key_invalidate" +} elf_symbol { id: 0xc2d6ae96 name: "key_put" @@ -374504,6 +375144,33 @@ elf_symbol { type_id: 0x10f1f3b8 full_name: "key_put" } +elf_symbol { + id: 0xfeabff2c + name: "key_revoke" + is_defined: true + symbol_type: FUNCTION + crc: 0xeae3efbe + type_id: 0x10f1f3b8 + full_name: "key_revoke" +} +elf_symbol { + id: 0x94513ffd + name: "key_type_logon" + is_defined: true + symbol_type: OBJECT + crc: 0x29f78983 + type_id: 0x70588905 + full_name: "key_type_logon" +} +elf_symbol { + id: 0xf23047dc + name: "key_validate" + is_defined: true + symbol_type: FUNCTION + crc: 0xf842e072 + type_id: 0x9256988c + full_name: "key_validate" +} elf_symbol { id: 0x6dbdf8e6 name: "keyring_alloc" @@ -376295,6 +376962,15 @@ elf_symbol { type_id: 0x1655de44 full_name: "lock_sock_nested" } +elf_symbol { + id: 0x0a889b64 + name: "lock_two_nondirectories" + is_defined: true + symbol_type: FUNCTION + crc: 0x90214118 + type_id: 0x161723fb + full_name: "lock_two_nondirectories" +} elf_symbol { id: 0x42eaca18 name: "lockref_get" @@ -376304,6 +376980,15 @@ elf_symbol { type_id: 0x18daa769 full_name: "lockref_get" } +elf_symbol { + id: 0x8425ece7 + name: "locks_lock_inode_wait" + is_defined: true + symbol_type: FUNCTION + crc: 0x09bde97a + type_id: 0x9b911a53 + full_name: "locks_lock_inode_wait" +} elf_symbol { id: 0x408fc274 name: "log_abnormal_wakeup_reason" @@ -376367,6 +377052,15 @@ elf_symbol { type_id: 0x11cf2aed full_name: "logfc" } +elf_symbol { + id: 0xcda3f19d + name: "look_up_OID" + is_defined: true + symbol_type: FUNCTION + crc: 0xeaf0a57c + type_id: 0x9cd412ab + full_name: "look_up_OID" +} elf_symbol { id: 0xad10cf0f name: "lookup_bdev" @@ -376376,6 +377070,15 @@ elf_symbol { type_id: 0x92c581e2 full_name: "lookup_bdev" } +elf_symbol { + id: 0x3c2ed3a0 + name: "lookup_positive_unlocked" + is_defined: true + symbol_type: FUNCTION + crc: 0xa91d8fcb + type_id: 0x83a40349 + full_name: "lookup_positive_unlocked" +} elf_symbol { id: 0x493ce9fc name: "loops_per_jiffy" @@ -381430,6 +382133,15 @@ elf_symbol { type_id: 0x9a3ef313 full_name: "out_of_line_wait_on_bit" } +elf_symbol { + id: 0xc31e86d2 + name: "out_of_line_wait_on_bit_lock" + is_defined: true + symbol_type: FUNCTION + crc: 0xb3687850 + type_id: 0x9a3ef313 + full_name: "out_of_line_wait_on_bit_lock" +} elf_symbol { id: 0x27d0c7b6 name: "out_of_line_wait_on_bit_timeout" @@ -381457,6 +382169,15 @@ elf_symbol { type_id: 0x6720d32f full_name: "overflowuid" } +elf_symbol { + id: 0xd1ef7302 + name: "override_creds" + is_defined: true + symbol_type: FUNCTION + crc: 0xecfbf6ac + type_id: 0x01831291 + full_name: "override_creds" +} elf_symbol { id: 0x23a386a4 name: "page_cache_async_ra" @@ -385321,6 +386042,24 @@ elf_symbol { type_id: 0x9e008a7e full_name: "posix_acl_chmod" } +elf_symbol { + id: 0x48573a01 + name: "posix_lock_file" + is_defined: true + symbol_type: FUNCTION + crc: 0x7b5cd306 + type_id: 0x9b45ed14 + full_name: "posix_lock_file" +} +elf_symbol { + id: 0x4d2108fd + name: "posix_test_lock" + is_defined: true + symbol_type: FUNCTION + crc: 0xfcc16f6a + type_id: 0x1652567d + full_name: "posix_test_lock" +} elf_symbol { id: 0x7bf7b7b1 name: "post_page_relinquish_tlb_inv" @@ -385690,6 +386429,15 @@ elf_symbol { type_id: 0x10985193 full_name: "preempt_schedule_notrace" } +elf_symbol { + id: 0x80c0972e + name: "prepare_kernel_cred" + is_defined: true + symbol_type: FUNCTION + crc: 0x3ad48e57 + type_id: 0x472c0143 + full_name: "prepare_kernel_cred" +} elf_symbol { id: 0x49317a7a name: "prepare_to_wait" @@ -387247,6 +387995,15 @@ elf_symbol { type_id: 0x16e6f635 full_name: "register_kernel_break_hook" } +elf_symbol { + id: 0x5d077441 + name: "register_key_type" + is_defined: true + symbol_type: FUNCTION + crc: 0xf1ed30b4 + type_id: 0x982165d6 + full_name: "register_key_type" +} elf_symbol { id: 0xba3fffd9 name: "register_kprobe" @@ -388363,6 +389120,15 @@ elf_symbol { type_id: 0x9ef35806 full_name: "request_firmware_nowait" } +elf_symbol { + id: 0x156ef85f + name: "request_key_tag" + is_defined: true + symbol_type: FUNCTION + crc: 0xb77f05e5 + type_id: 0x8659e72e + full_name: "request_key_tag" +} elf_symbol { id: 0x0d3c3e14 name: "request_threaded_irq" @@ -388507,6 +389273,15 @@ elf_symbol { type_id: 0x47f7e87e full_name: "return_address" } +elf_symbol { + id: 0x7f13df64 + name: "revert_creds" + is_defined: true + symbol_type: FUNCTION + crc: 0xf6b5de4e + type_id: 0x1c81137a + full_name: "revert_creds" +} elf_symbol { id: 0xd8c7c137 name: "reweight_task" @@ -390775,6 +391550,15 @@ elf_symbol { type_id: 0x9b034295 full_name: "security_inode_init_security" } +elf_symbol { + id: 0xb166d440 + name: "security_sb_eat_lsm_opts" + is_defined: true + symbol_type: FUNCTION + crc: 0xb48d4d22 + type_id: 0x9c92e712 + full_name: "security_sb_eat_lsm_opts" +} elf_symbol { id: 0x24b9d7ed name: "security_sk_classify_flow" @@ -390838,6 +391622,15 @@ elf_symbol { type_id: 0x9ccb56bc full_name: "seq_buf_printf" } +elf_symbol { + id: 0x854e793d + name: "seq_escape_mem" + is_defined: true + symbol_type: FUNCTION + crc: 0x756be456 + type_id: 0x11c57e21 + full_name: "seq_escape_mem" +} elf_symbol { id: 0xb2b0c8fa name: "seq_hex_dump" @@ -391252,6 +392045,15 @@ elf_symbol { type_id: 0x10256efa full_name: "serio_unregister_port" } +elf_symbol { + id: 0xa2d400c7 + name: "set_anon_super" + is_defined: true + symbol_type: FUNCTION + crc: 0x08e45b2e + type_id: 0x94598cb1 + full_name: "set_anon_super" +} elf_symbol { id: 0x334adb18 name: "set_bh_page" @@ -391612,6 +392414,15 @@ elf_symbol { type_id: 0x925feb11 full_name: "sg_pcopy_to_buffer" } +elf_symbol { + id: 0xe5950caf + name: "sget" + is_defined: true + symbol_type: FUNCTION + crc: 0x49069f09 + type_id: 0xf719dbb9 + full_name: "sget" +} elf_symbol { id: 0x12402fa7 name: "sget_fc" @@ -394862,6 +395673,15 @@ elf_symbol { type_id: 0x11cb8bd5 full_name: "sock_rfree" } +elf_symbol { + id: 0xcc5c7948 + name: "sock_sendmsg" + is_defined: true + symbol_type: FUNCTION + crc: 0x9e96b686 + type_id: 0x992b8783 + full_name: "sock_sendmsg" +} elf_symbol { id: 0x7b771682 name: "sock_setsockopt" @@ -395303,6 +396123,15 @@ elf_symbol { type_id: 0x9a1cd590 full_name: "spmi_register_zero_write" } +elf_symbol { + id: 0x6da39b17 + name: "sprint_oid" + is_defined: true + symbol_type: FUNCTION + crc: 0xfc201b66 + type_id: 0x901503d8 + full_name: "sprint_oid" +} elf_symbol { id: 0x863d7345 name: "sprint_symbol" @@ -395873,6 +396702,15 @@ elf_symbol { type_id: 0x9d6a72fd full_name: "subsys_system_register" } +elf_symbol { + id: 0xdf289104 + name: "super_setup_bdi" + is_defined: true + symbol_type: FUNCTION + crc: 0xf37b0103 + type_id: 0x943b7925 + full_name: "super_setup_bdi" +} elf_symbol { id: 0x44e002a2 name: "suspend_set_ops" @@ -396881,6 +397719,24 @@ elf_symbol { type_id: 0x4022c156 full_name: "tcp_slow_start" } +elf_symbol { + id: 0xb56d0844 + name: "tcp_sock_set_cork" + is_defined: true + symbol_type: FUNCTION + crc: 0x05f3393e + type_id: 0x167ca077 + full_name: "tcp_sock_set_cork" +} +elf_symbol { + id: 0x51ff5f80 + name: "tcp_sock_set_nodelay" + is_defined: true + symbol_type: FUNCTION + crc: 0xc079048b + type_id: 0x17c95d08 + full_name: "tcp_sock_set_nodelay" +} elf_symbol { id: 0xefab10bf name: "tcp_unregister_congestion_control" @@ -397736,6 +398592,15 @@ elf_symbol { type_id: 0x15f8fb52 full_name: "truncate_inode_pages_final" } +elf_symbol { + id: 0xeea45f7f + name: "truncate_inode_pages_range" + is_defined: true + symbol_type: FUNCTION + crc: 0xe4035799 + type_id: 0x156f8dbb + full_name: "truncate_inode_pages_range" +} elf_symbol { id: 0x7a43283c name: "truncate_pagecache" @@ -397745,6 +398610,15 @@ elf_symbol { type_id: 0x16e7aaa1 full_name: "truncate_pagecache" } +elf_symbol { + id: 0x4eb5a934 + name: "truncate_pagecache_range" + is_defined: true + symbol_type: FUNCTION + crc: 0x99b15f06 + type_id: 0x16ee4350 + full_name: "truncate_pagecache_range" +} elf_symbol { id: 0x44cd0751 name: "truncate_setsize" @@ -399896,6 +400770,15 @@ elf_symbol { type_id: 0x11388634 full_name: "unlock_page" } +elf_symbol { + id: 0x3cace439 + name: "unlock_two_nondirectories" + is_defined: true + symbol_type: FUNCTION + crc: 0xac2299ff + type_id: 0x161723fb + full_name: "unlock_two_nondirectories" +} elf_symbol { id: 0xe41b8f90 name: "unmap_mapping_range" @@ -400013,6 +400896,15 @@ elf_symbol { type_id: 0x9b6d4a33 full_name: "unregister_inetaddr_notifier" } +elf_symbol { + id: 0x31dfbba5 + name: "unregister_key_type" + is_defined: true + symbol_type: FUNCTION + crc: 0x819cd856 + type_id: 0x1539d76a + full_name: "unregister_key_type" +} elf_symbol { id: 0x4bd3e4fd name: "unregister_kprobe" @@ -402317,6 +403209,42 @@ elf_symbol { type_id: 0x9e5fde3c full_name: "usbnet_write_cmd_nopm" } +elf_symbol { + id: 0x37ac0936 + name: "user_describe" + is_defined: true + symbol_type: FUNCTION + crc: 0x3bf7597d + type_id: 0x1f57bdaa + full_name: "user_describe" +} +elf_symbol { + id: 0xa3900874 + name: "user_destroy" + is_defined: true + symbol_type: FUNCTION + crc: 0xa10287fa + type_id: 0x10f1f3b8 + full_name: "user_destroy" +} +elf_symbol { + id: 0x4cfd694f + name: "user_read" + is_defined: true + symbol_type: FUNCTION + crc: 0x6d8d52c4 + type_id: 0xa8d942fb + full_name: "user_read" +} +elf_symbol { + id: 0x22e80038 + name: "user_revoke" + is_defined: true + symbol_type: FUNCTION + crc: 0x5f2e03d3 + type_id: 0x10f1f3b8 + full_name: "user_revoke" +} elf_symbol { id: 0xb5437ef4 name: "usleep_range_state" @@ -402353,6 +403281,15 @@ elf_symbol { type_id: 0x93de5e2a full_name: "utf8_to_utf32" } +elf_symbol { + id: 0xa040d5a9 + name: "utf8s_to_utf16s" + is_defined: true + symbol_type: FUNCTION + crc: 0x97adb487 + type_id: 0x93e80c53 + full_name: "utf8s_to_utf16s" +} elf_symbol { id: 0xb0c1eaf9 name: "uuid_gen" @@ -403964,6 +404901,24 @@ elf_symbol { type_id: 0x2d64ae3e full_name: "vfs_llseek" } +elf_symbol { + id: 0x5abb2c12 + name: "vfs_parse_fs_string" + is_defined: true + symbol_type: FUNCTION + crc: 0x0eeb7380 + type_id: 0x96a3069e + full_name: "vfs_parse_fs_string" +} +elf_symbol { + id: 0x2ad025fd + name: "vfs_setpos" + is_defined: true + symbol_type: FUNCTION + crc: 0xd372ed5c + type_id: 0x2d748ffb + full_name: "vfs_setpos" +} elf_symbol { id: 0xf33dadbd name: "vhost_add_used_and_signal" @@ -405260,6 +406215,15 @@ elf_symbol { type_id: 0x10985193 full_name: "wait_for_device_probe" } +elf_symbol { + id: 0x50838b40 + name: "wait_on_page_writeback" + is_defined: true + symbol_type: FUNCTION + crc: 0x372dabd9 + type_id: 0x11388634 + full_name: "wait_on_page_writeback" +} elf_symbol { id: 0xa4027847 name: "wait_woken" @@ -406645,6 +407609,7 @@ interface { symbol_id: 0x2d1c1f2d symbol_id: 0x9bed4545 symbol_id: 0xefdcf4c1 + symbol_id: 0x23bb1952 symbol_id: 0x2f449b9e symbol_id: 0x2057d4fc symbol_id: 0x4598418e @@ -406676,6 +407641,7 @@ interface { symbol_id: 0x42e03bd1 symbol_id: 0x857a0e42 symbol_id: 0xe889d441 + symbol_id: 0xc1737e0f symbol_id: 0x087c7a06 symbol_id: 0x8f70cdad symbol_id: 0x2d91cc56 @@ -406752,6 +407718,7 @@ interface { symbol_id: 0x6a92cb2d symbol_id: 0x6fc34fdb symbol_id: 0x47a334c4 + symbol_id: 0x69ff7fd9 symbol_id: 0xebf4b11f symbol_id: 0x5b1ea047 symbol_id: 0x07f88ce8 @@ -408020,6 +408987,7 @@ interface { symbol_id: 0xd20e014a symbol_id: 0xf09e1ca4 symbol_id: 0xcef4ad72 + symbol_id: 0x7951d828 symbol_id: 0xa5dcf915 symbol_id: 0xf2768ed7 symbol_id: 0xfbeae533 @@ -408082,6 +409050,7 @@ interface { symbol_id: 0xb61718ed symbol_id: 0x4f8a812a symbol_id: 0x1eee53db + symbol_id: 0xac417394 symbol_id: 0xd9184490 symbol_id: 0xe6ba30b7 symbol_id: 0xa09651b4 @@ -408558,7 +409527,9 @@ interface { symbol_id: 0x610edc84 symbol_id: 0xd71898b4 symbol_id: 0xc1167624 + symbol_id: 0xdd2942d2 symbol_id: 0x9f1f7cee + symbol_id: 0x5e964278 symbol_id: 0xd89255c2 symbol_id: 0x962b6a68 symbol_id: 0xb8669175 @@ -408750,9 +409721,14 @@ interface { symbol_id: 0x171eca8a symbol_id: 0x82537a82 symbol_id: 0x79187969 + symbol_id: 0x4b8dc0ec + symbol_id: 0xfeea461b symbol_id: 0x9534fec7 + symbol_id: 0x66e2e005 symbol_id: 0x04f5a6e5 + symbol_id: 0x53946e0e symbol_id: 0x52864a7e + symbol_id: 0x03fd2deb symbol_id: 0x8d5ab7b4 symbol_id: 0xccc25fc1 symbol_id: 0xb96af991 @@ -408763,6 +409739,7 @@ interface { symbol_id: 0xf94e1278 symbol_id: 0xf1e399f5 symbol_id: 0x02f83230 + symbol_id: 0xd6386fd8 symbol_id: 0x36a5b3df symbol_id: 0x7225f00d symbol_id: 0xe2150034 @@ -409837,6 +410814,7 @@ interface { symbol_id: 0x45c0e0ad symbol_id: 0xf7fd2915 symbol_id: 0x7b9ffce9 + symbol_id: 0x3fdfcd47 symbol_id: 0xe506b877 symbol_id: 0xda116c83 symbol_id: 0xa279fd1d @@ -409890,6 +410868,7 @@ interface { symbol_id: 0xab45f2a8 symbol_id: 0x41eed79f symbol_id: 0xcdc0fffd + symbol_id: 0x08df0c2d symbol_id: 0xe7aacfe5 symbol_id: 0x9e942fbe symbol_id: 0x99b0f07b @@ -409903,6 +410882,7 @@ interface { symbol_id: 0x82b19820 symbol_id: 0x7c937fc2 symbol_id: 0xa6f98003 + symbol_id: 0x70cbb212 symbol_id: 0xc7c68a05 symbol_id: 0xb9dc79d0 symbol_id: 0x8bbdb127 @@ -409915,6 +410895,8 @@ interface { symbol_id: 0x69beea41 symbol_id: 0xd2a94cce symbol_id: 0x40cf5b57 + symbol_id: 0xdc38ff2a + symbol_id: 0x557a4fc7 symbol_id: 0xf882020f symbol_id: 0x2e62a121 symbol_id: 0x24954a6b @@ -409934,9 +410916,11 @@ interface { symbol_id: 0xa9d940b8 symbol_id: 0x3c7c2553 symbol_id: 0x06c58be7 + symbol_id: 0xab55569c symbol_id: 0x159a69a3 symbol_id: 0x39840ab2 symbol_id: 0x3dca9a13 + symbol_id: 0xb917d8a9 symbol_id: 0x07508208 symbol_id: 0xcb14b5cc symbol_id: 0x1721c938 @@ -409970,8 +410954,10 @@ interface { symbol_id: 0x4c985dbf symbol_id: 0xec31d371 symbol_id: 0x9a2423a0 + symbol_id: 0x55ce3162 symbol_id: 0x6c9f28bc symbol_id: 0x35a180e7 + symbol_id: 0x4bab97c7 symbol_id: 0xbc2aa09b symbol_id: 0x196ad62a symbol_id: 0x804c5038 @@ -410025,7 +411011,9 @@ interface { symbol_id: 0x62469fff symbol_id: 0xc5fee33e symbol_id: 0x7419d447 + symbol_id: 0xb7f431e8 symbol_id: 0x0c22ac35 + symbol_id: 0x1b6a5b31 symbol_id: 0x30828743 symbol_id: 0xe21d8d05 symbol_id: 0xc99c6e1c @@ -410037,14 +411025,17 @@ interface { symbol_id: 0x54756210 symbol_id: 0x5ec3daa8 symbol_id: 0x5530f759 + symbol_id: 0xc7f4ee02 symbol_id: 0x3812bb22 symbol_id: 0x2f7205ee symbol_id: 0x7d577f86 symbol_id: 0xd5bc36ee + symbol_id: 0xf85d118d symbol_id: 0x95a840b8 symbol_id: 0x7f639ef1 symbol_id: 0x6bd69c06 symbol_id: 0x4c4073c3 + symbol_id: 0x242af10d symbol_id: 0x798f83da symbol_id: 0x542785af symbol_id: 0xb45dfa4f @@ -410583,6 +411574,7 @@ interface { symbol_id: 0xe7fc8dce symbol_id: 0x22167fe7 symbol_id: 0xae43d3d4 + symbol_id: 0x431bb67c symbol_id: 0x38472bf4 symbol_id: 0xc3e2d4ae symbol_id: 0x6b8c216a @@ -410637,9 +411629,13 @@ interface { symbol_id: 0xfb789e26 symbol_id: 0x382b3272 symbol_id: 0x29ecb8e9 + symbol_id: 0xe15835cc + symbol_id: 0x067b9161 symbol_id: 0x3d6623cb + symbol_id: 0x530f6f25 symbol_id: 0x71726a83 symbol_id: 0xffb394c4 + symbol_id: 0x9978b03b symbol_id: 0x48a6183b symbol_id: 0x1c2c7614 symbol_id: 0x38ac7923 @@ -410787,7 +411783,11 @@ interface { symbol_id: 0x1185f06e symbol_id: 0x1ac8e26f symbol_id: 0x1feedde3 + symbol_id: 0x86d6e923 symbol_id: 0xc2d6ae96 + symbol_id: 0xfeabff2c + symbol_id: 0x94513ffd + symbol_id: 0xf23047dc symbol_id: 0x6dbdf8e6 symbol_id: 0x6bb359de symbol_id: 0xe097a7bd @@ -410987,7 +411987,9 @@ interface { symbol_id: 0x6b29ddfb symbol_id: 0xc05c2da4 symbol_id: 0xaaab26df + symbol_id: 0x0a889b64 symbol_id: 0x42eaca18 + symbol_id: 0x8425ece7 symbol_id: 0x408fc274 symbol_id: 0x8fab82b5 symbol_id: 0xe1482206 @@ -410995,7 +411997,9 @@ interface { symbol_id: 0xc41c43d2 symbol_id: 0x02bffe2a symbol_id: 0x06e891b4 + symbol_id: 0xcda3f19d symbol_id: 0xad10cf0f + symbol_id: 0x3c2ed3a0 symbol_id: 0x493ce9fc symbol_id: 0x1440b3fe symbol_id: 0xf625170a @@ -411558,9 +412562,11 @@ interface { symbol_id: 0x33abdf76 symbol_id: 0x2ffaca4e symbol_id: 0x57ecab02 + symbol_id: 0xc31e86d2 symbol_id: 0x27d0c7b6 symbol_id: 0x3d8c84e9 symbol_id: 0x28b33064 + symbol_id: 0xd1ef7302 symbol_id: 0x23a386a4 symbol_id: 0xdea2b008 symbol_id: 0xdf2c83f5 @@ -411990,6 +412996,8 @@ interface { symbol_id: 0x25c6452e symbol_id: 0x324d8228 symbol_id: 0x3e0dea4e + symbol_id: 0x48573a01 + symbol_id: 0x4d2108fd symbol_id: 0x7bf7b7b1 symbol_id: 0xd612789c symbol_id: 0x561323f1 @@ -412031,6 +413039,7 @@ interface { symbol_id: 0x48fb8c7e symbol_id: 0x180b11d3 symbol_id: 0xd46a0ddb + symbol_id: 0x80c0972e symbol_id: 0x49317a7a symbol_id: 0xd780999c symbol_id: 0xe8fde451 @@ -412204,6 +413213,7 @@ interface { symbol_id: 0x2c5e821c symbol_id: 0xa36bcd31 symbol_id: 0x6c1eddbc + symbol_id: 0x5d077441 symbol_id: 0xba3fffd9 symbol_id: 0x20a31fba symbol_id: 0xd467aa67 @@ -412328,6 +413338,7 @@ interface { symbol_id: 0xb261904e symbol_id: 0x8732c1c9 symbol_id: 0x677926cf + symbol_id: 0x156ef85f symbol_id: 0x0d3c3e14 symbol_id: 0xe5340075 symbol_id: 0x6aab6dda @@ -412344,6 +413355,7 @@ interface { symbol_id: 0xd41c441b symbol_id: 0x8607d899 symbol_id: 0x1d945826 + symbol_id: 0x7f13df64 symbol_id: 0xd8c7c137 symbol_id: 0xa3f52060 symbol_id: 0x001f1a24 @@ -412596,6 +413608,7 @@ interface { symbol_id: 0xb3b8cbe2 symbol_id: 0xec171799 symbol_id: 0xb7265b03 + symbol_id: 0xb166d440 symbol_id: 0x24b9d7ed symbol_id: 0x25471057 symbol_id: 0xb77dc501 @@ -412603,6 +413616,7 @@ interface { symbol_id: 0x569844be symbol_id: 0x17b3f8eb symbol_id: 0x4814ecd5 + symbol_id: 0x854e793d symbol_id: 0xb2b0c8fa symbol_id: 0xfb710d3e symbol_id: 0x0af60b8b @@ -412649,6 +413663,7 @@ interface { symbol_id: 0x9a2a9bda symbol_id: 0x54732419 symbol_id: 0x7df87144 + symbol_id: 0xa2d400c7 symbol_id: 0x334adb18 symbol_id: 0xdfe1cb53 symbol_id: 0x2c6a64fa @@ -412689,6 +413704,7 @@ interface { symbol_id: 0x55c5bdf3 symbol_id: 0x7c74f0ec symbol_id: 0x8525915d + symbol_id: 0xe5950caf symbol_id: 0x12402fa7 symbol_id: 0x0b4df89e symbol_id: 0x7996527c @@ -413050,6 +414066,7 @@ interface { symbol_id: 0xd326bb7c symbol_id: 0xe138dce0 symbol_id: 0xeffe0f16 + symbol_id: 0xcc5c7948 symbol_id: 0x7b771682 symbol_id: 0x39362be4 symbol_id: 0xeab36c96 @@ -413099,6 +414116,7 @@ interface { symbol_id: 0x0d5e8c9c symbol_id: 0xc08173b3 symbol_id: 0x581a0bb3 + symbol_id: 0x6da39b17 symbol_id: 0x863d7345 symbol_id: 0xad8ec240 symbol_id: 0x2c7eedb1 @@ -413163,6 +414181,7 @@ interface { symbol_id: 0x5577d5a8 symbol_id: 0x0e1b0b82 symbol_id: 0xfb52ace2 + symbol_id: 0xdf289104 symbol_id: 0x44e002a2 symbol_id: 0xd5e0987f symbol_id: 0x1fe11014 @@ -413275,6 +414294,8 @@ interface { symbol_id: 0x97edc1c5 symbol_id: 0xdb8e8104 symbol_id: 0xe6a46ba8 + symbol_id: 0xb56d0844 + symbol_id: 0x51ff5f80 symbol_id: 0xefab10bf symbol_id: 0x857e5dc9 symbol_id: 0x85459353 @@ -413370,7 +414391,9 @@ interface { symbol_id: 0xd75a472d symbol_id: 0x3f07269b symbol_id: 0x3c7c6ce9 + symbol_id: 0xeea45f7f symbol_id: 0x7a43283c + symbol_id: 0x4eb5a934 symbol_id: 0x44cd0751 symbol_id: 0x3d986fb5 symbol_id: 0x2f855953 @@ -413610,6 +414633,7 @@ interface { symbol_id: 0x9f5197f0 symbol_id: 0x5ad67dea symbol_id: 0x11fd8c82 + symbol_id: 0x3cace439 symbol_id: 0xe41b8f90 symbol_id: 0x60ffe144 symbol_id: 0xffd70606 @@ -413623,6 +414647,7 @@ interface { symbol_id: 0x6e1b5152 symbol_id: 0x4144b792 symbol_id: 0x4615e3af + symbol_id: 0x31dfbba5 symbol_id: 0x4bd3e4fd symbol_id: 0x67d5fb19 symbol_id: 0xd3f1f6f2 @@ -413879,10 +414904,15 @@ interface { symbol_id: 0xb011e423 symbol_id: 0x27363bb6 symbol_id: 0x10932377 + symbol_id: 0x37ac0936 + symbol_id: 0xa3900874 + symbol_id: 0x4cfd694f + symbol_id: 0x22e80038 symbol_id: 0xb5437ef4 symbol_id: 0x1473720a symbol_id: 0xa459c02b symbol_id: 0x4b0dcba7 + symbol_id: 0xa040d5a9 symbol_id: 0xb0c1eaf9 symbol_id: 0xe7b3f166 symbol_id: 0xb21b47da @@ -414062,6 +415092,8 @@ interface { symbol_id: 0xa7ab6134 symbol_id: 0x1499f6a1 symbol_id: 0xf89b49b7 + symbol_id: 0x5abb2c12 + symbol_id: 0x2ad025fd symbol_id: 0xf33dadbd symbol_id: 0x743ba358 symbol_id: 0x9f33c2c6 @@ -414206,6 +415238,7 @@ interface { symbol_id: 0x718f5c64 symbol_id: 0x60488ecf symbol_id: 0x9d727719 + symbol_id: 0x50838b40 symbol_id: 0xa4027847 symbol_id: 0x738279bf symbol_id: 0x7540b5a8 diff --git a/android/abi_gki_aarch64_xiaomi b/android/abi_gki_aarch64_xiaomi index 63104d8e367e..02f24786a457 100644 --- a/android/abi_gki_aarch64_xiaomi +++ b/android/abi_gki_aarch64_xiaomi @@ -371,3 +371,87 @@ pci_sriov_set_totalvfs pci_num_vf +#required by cifs.ko + add_swap_extent + asn1_ber_decoder + copy_page_from_iter + copy_page_to_iter + clear_nlink + deactivate_super + dentry_path_raw + d_drop + d_move + d_hash_and_lookup + d_alloc_parallel + d_invalidate + errseq_check + file_modified + fs_param_is_bool + fs_param_is_u32 + fs_param_is_u64 + fs_param_is_string + finish_no_open + finish_open + filemap_map_pages + find_get_pages_range_tag + folio_wait_private_2 + folio_clear_dirty_for_io + generate_random_uuid + generic_permission + generic_file_write_iter + generic_setlease + generic_copy_file_range + generic_writepages + generic_key_instantiate + iov_iter_discard + iov_iter_bvec + iov_iter_get_pages_alloc2 + iget_failed + invalidate_inode_pages2 + iov_iter_npages + kfree_link + key_type_logon + key_invalidate + key_revoke + key_validate + lock_two_nondirectories + lookup_positive_unlocked + locks_lock_inode_wait + look_up_OID + overflowgid + out_of_line_wait_on_bit_lock + override_creds + posix_lock_file + posix_test_lock + prepare_kernel_cred + request_key_tag + revert_creds + register_key_type + sget + set_anon_super + super_setup_bdi + seq_escape_mem + setattr_copy + sock_sendmsg + security_sb_eat_lsm_opts + sprint_oid + truncate_inode_pages_range + tcp_sock_set_nodelay + tcp_sock_set_cork + truncate_pagecache_range + unlock_two_nondirectories + utf8s_to_utf16s + utf8_to_utf32 + unregister_key_type + user_describe + user_revoke + user_destroy + user_read + vfs_setpos + vfs_parse_fs_string + wait_on_page_writeback + __generic_file_write_iter + __folio_lock_killable + __break_lease + __d_lookup_unhash_wake + __fs_parse From cf685d2b02734245c3e179f3541f67ae3feee796 Mon Sep 17 00:00:00 2001 From: Priyansh Jain Date: Thu, 2 May 2024 15:30:42 +0530 Subject: [PATCH 65/86] ANDROID: GKI: Update QCOM symbol list and ABI STG Update QCOM symbol list in android/abi_gki_aarch64_qcom with following symbols. 4 function symbol(s) added 'struct powercap_control_type* powercap_register_control_type(struct powercap_control_type*, const char*, const struct powercap_control_type_ops*)' 'struct powercap_zone* powercap_register_zone(struct powercap_zone*, struct powercap_control_type*, const char*, struct powercap_zone*, const struct powercap_zone_ops*, int, const struct powercap_zone_constraint_ops*)' 'int powercap_unregister_control_type(struct powercap_control_type*)' 'int powercap_unregister_zone(struct powercap_control_type*, struct powercap_zone*)' Bug: 338523446 Change-Id: Ie1305e6231532c824bd11bcc1e0abecdb55109aa Signed-off-by: Priyansh Jain Signed-off-by: Will McVicker --- android/abi_gki_aarch64.stg | 551 ++++++++++++++++++++++++++++++++++- android/abi_gki_aarch64_qcom | 4 + 2 files changed, 554 insertions(+), 1 deletion(-) diff --git a/android/abi_gki_aarch64.stg b/android/abi_gki_aarch64.stg index 6be4e52b1610..10ca10afffa0 100644 --- a/android/abi_gki_aarch64.stg +++ b/android/abi_gki_aarch64.stg @@ -8678,6 +8678,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x64da9c42 } +pointer_reference { + id: 0x13b72e31 + kind: POINTER + pointee_type_id: 0x649c5e5b +} pointer_reference { id: 0x13b93798 kind: POINTER @@ -14358,6 +14363,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x9ab9a458 } +pointer_reference { + id: 0x2c3e6eed + kind: POINTER + pointee_type_id: 0x9ab95d2b +} pointer_reference { id: 0x2c3f2dba kind: POINTER @@ -15808,6 +15818,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x98c71afc } +pointer_reference { + id: 0x2ca5fa72 + kind: POINTER + pointee_type_id: 0x98d70f54 +} pointer_reference { id: 0x2ca6a571 kind: POINTER @@ -20088,6 +20103,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x902e10a9 } +pointer_reference { + id: 0x2e9c2274 + kind: POINTER + pointee_type_id: 0x90306f4f +} pointer_reference { id: 0x2e9e4338 kind: POINTER @@ -20673,6 +20693,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x91c118ea } +pointer_reference { + id: 0x2ee092c5 + kind: POINTER + pointee_type_id: 0x91c2ad89 +} pointer_reference { id: 0x2ee126d9 kind: POINTER @@ -20903,6 +20928,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x9185daf6 } +pointer_reference { + id: 0x2ef15d2b + kind: POINTER + pointee_type_id: 0x91859230 +} pointer_reference { id: 0x2ef22e9f kind: POINTER @@ -21203,6 +21233,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x9636894a } +pointer_reference { + id: 0x2f1dff77 + kind: POINTER + pointee_type_id: 0x96371b41 +} pointer_reference { id: 0x2f1f19f7 kind: POINTER @@ -21233,6 +21268,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x96c7ba5c } +pointer_reference { + id: 0x2f22286c + kind: POINTER + pointee_type_id: 0x96c8472d +} pointer_reference { id: 0x2f2278cb kind: POINTER @@ -21343,6 +21383,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x968ac51c } +pointer_reference { + id: 0x2f33e782 + kind: POINTER + pointee_type_id: 0x968f7894 +} pointer_reference { id: 0x2f36a854 kind: POINTER @@ -21508,6 +21553,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x9710bae0 } +pointer_reference { + id: 0x2f5626de + kind: POINTER + pointee_type_id: 0x97187de5 +} pointer_reference { id: 0x2f575961 kind: POINTER @@ -21543,11 +21593,21 @@ pointer_reference { kind: POINTER pointee_type_id: 0x9735a176 } +pointer_reference { + id: 0x2f5de562 + kind: POINTER + pointee_type_id: 0x97377314 +} pointer_reference { id: 0x2f5e345a kind: POINTER pointee_type_id: 0x973837f7 } +pointer_reference { + id: 0x2f5e98dd + kind: POINTER + pointee_type_id: 0x973a85eb +} pointer_reference { id: 0x2f5ea9dc kind: POINTER @@ -22548,6 +22608,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0xea7f94f9 } +pointer_reference { + id: 0x3015c47d + kind: POINTER + pointee_type_id: 0xea17f769 +} pointer_reference { id: 0x3017223b kind: POINTER @@ -23148,6 +23213,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0xefa4cae6 } +pointer_reference { + id: 0x317fb103 + kind: POINTER + pointee_type_id: 0xefbe2291 +} pointer_reference { id: 0x3181aee5 kind: POINTER @@ -25173,6 +25243,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0xfc8f4f95 } +pointer_reference { + id: 0x35b4b910 + kind: POINTER + pointee_type_id: 0xfc9202dd +} pointer_reference { id: 0x35bad1e5 kind: POINTER @@ -27988,6 +28063,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0xc538e39d } +pointer_reference { + id: 0x3be1e049 + kind: POINTER + pointee_type_id: 0xc5c767b8 +} pointer_reference { id: 0x3be4ec15 kind: POINTER @@ -32333,6 +32413,11 @@ qualified { qualifier: CONST qualified_type_id: 0x6682a341 } +qualified { + id: 0xc5c767b8 + qualifier: CONST + qualified_type_id: 0x675a2868 +} qualified { id: 0xc5d356cb qualifier: CONST @@ -34733,6 +34818,11 @@ qualified { qualifier: CONST qualified_type_id: 0xcd7704bf } +qualified { + id: 0xefbe2291 + qualifier: CONST + qualified_type_id: 0xcebf3ccf +} qualified { id: 0xefcb4746 qualifier: CONST @@ -35358,6 +35448,11 @@ qualified { qualifier: CONST qualified_type_id: 0x81bb7781 } +qualified { + id: 0xfc9202dd + qualifier: CONST + qualified_type_id: 0x820fbdfd +} qualified { id: 0xfcaba10a qualifier: CONST @@ -46043,6 +46138,18 @@ member { name: "allocated" type_id: 0x6d7f5ff6 } +member { + id: 0x41611b26 + name: "allocated" + type_id: 0x6d7f5ff6 + offset: 8000 +} +member { + id: 0x41611ed7 + name: "allocated" + type_id: 0x6d7f5ff6 + offset: 8512 +} member { id: 0x41b3ffd4 name: "allocated_buffers" @@ -66140,6 +66247,12 @@ member { type_id: 0x5d8155a5 offset: 2880 } +member { + id: 0x9b6df7c9 + name: "const_id_cnt" + type_id: 0x6720d32f + offset: 7552 +} member { id: 0x9e45699e name: "const_type" @@ -66203,6 +66316,12 @@ member { type_id: 0x1bab96d2 offset: 384 } +member { + id: 0xfd7a5c77 + name: "constraints" + type_id: 0x13b72e31 + offset: 8576 +} member { id: 0xfdfa65af name: "constraints" @@ -66508,6 +66627,12 @@ member { name: "control_load" type_id: 0x2c808592 } +member { + id: 0xf0f0bb7d + name: "control_type_inst" + type_id: 0x18bd6530 + offset: 128 +} member { id: 0x28722417 name: "control_unload" @@ -75535,6 +75660,12 @@ member { type_id: 0x11cf9350 offset: 704 } +member { + id: 0x6d170429 + name: "dev_attr_groups" + type_id: 0x607a4116 + offset: 8384 +} member { id: 0x81f55009 name: "dev_base_head" @@ -76077,6 +76208,12 @@ member { type_id: 0x6d7f5ff6 offset: 7488 } +member { + id: 0x327f9c2d + name: "dev_zone_attr_group" + type_id: 0xed8451a2 + offset: 8064 +} member { id: 0x3a503292 name: "devaddr" @@ -96765,6 +96902,24 @@ member { type_id: 0x2eda6351 offset: 768 } +member { + id: 0x1e0abfd9 + name: "get_enable" + type_id: 0x2ee092c5 + offset: 64 +} +member { + id: 0x1e0b7810 + name: "get_enable" + type_id: 0x2f22286c + offset: 384 +} +member { + id: 0xc5eb7d5e + name: "get_energy_uj" + type_id: 0x2f1dff77 + offset: 64 +} member { id: 0x522a2d74 name: "get_error_flags" @@ -97095,11 +97250,34 @@ member { type_id: 0x3ab13613 offset: 704 } +member { + id: 0x870b1d4a + name: "get_max_energy_range_uj" + type_id: 0x2f1dff77 +} +member { + id: 0x48da06d8 + name: "get_max_power_range_uw" + type_id: 0x2f1dff77 + offset: 192 +} +member { + id: 0x690ef040 + name: "get_max_power_uw" + type_id: 0x2f5626de + offset: 256 +} member { id: 0x9a73d86b name: "get_max_state" type_id: 0x2dbae528 } +member { + id: 0x39de1472 + name: "get_max_time_window_us" + type_id: 0x2f5626de + offset: 384 +} member { id: 0x98fbeba0 name: "get_max_timeout_count" @@ -97154,6 +97332,18 @@ member { type_id: 0x3ab13613 offset: 768 } +member { + id: 0x0a0601b8 + name: "get_min_power_uw" + type_id: 0x2f5626de + offset: 320 +} +member { + id: 0xbbc23eea + name: "get_min_time_window_us" + type_id: 0x2f5626de + offset: 448 +} member { id: 0x9c2b1ad1 name: "get_mode" @@ -97243,6 +97433,12 @@ member { type_id: 0x2cd5f6d3 offset: 192 } +member { + id: 0x1a6adf19 + name: "get_name" + type_id: 0x2ca5fa72 + offset: 512 +} member { id: 0x1a6b98f3 name: "get_name" @@ -97481,12 +97677,24 @@ member { name: "get_port" type_id: 0x0ddd9122 } +member { + id: 0x4eae2f0e + name: "get_power_limit_uw" + type_id: 0x2f5626de + offset: 64 +} member { id: 0xa9079cd6 name: "get_power_status" type_id: 0x2c80812c offset: 256 } +member { + id: 0xf4ddc253 + name: "get_power_uw" + type_id: 0x2f1dff77 + offset: 256 +} member { id: 0x5a0e21ce name: "get_priv" @@ -97846,6 +98054,12 @@ member { type_id: 0x2cbf0dea offset: 576 } +member { + id: 0xcec3c09c + name: "get_time_window_us" + type_id: 0x2f5626de + offset: 192 +} member { id: 0x4b2d8406 name: "get_timeleft" @@ -105868,11 +106082,23 @@ member { type_id: 0xb3e7bac9 offset: 8 } +member { + id: 0xb00531dc + name: "idr" + type_id: 0x04b89667 + offset: 7296 +} member { id: 0xb0053b5f name: "idr" type_id: 0x04b89667 } +member { + id: 0xb0053df5 + name: "idr" + type_id: 0x04b89667 + offset: 7616 +} member { id: 0xb0053fbe name: "idr" @@ -132541,6 +132767,12 @@ member { type_id: 0x0483e6f8 offset: 1280 } +member { + id: 0x0ddfeb5a + name: "name" + type_id: 0x0483e6f8 + offset: 64 +} member { id: 0x0ddfef10 name: "name" @@ -136503,6 +136735,12 @@ member { type_id: 0xd3c80119 offset: 1024 } +member { + id: 0x0fa6c1c0 + name: "node" + type_id: 0xd3c80119 + offset: 8064 +} member { id: 0x0fa6c3e5 name: "node" @@ -138503,6 +138741,12 @@ member { type_id: 0x6720d32f offset: 51840 } +member { + id: 0x429469eb + name: "nr_zones" + type_id: 0x6720d32f + offset: 7488 +} member { id: 0x42b6c2d1 name: "nr_zones" @@ -142897,6 +143141,12 @@ member { name: "ops" type_id: 0x3b991d29 } +member { + id: 0xafb0e881 + name: "ops" + type_id: 0x3be1e049 + offset: 128 +} member { id: 0xafb0ea3a name: "ops" @@ -143297,6 +143547,12 @@ member { type_id: 0x3162e876 offset: 8448 } +member { + id: 0xafba7798 + name: "ops" + type_id: 0x317fb103 + offset: 7552 +} member { id: 0xafbac999 name: "ops" @@ -143432,6 +143688,12 @@ member { type_id: 0x356e4b27 offset: 192 } +member { + id: 0xafbebcd3 + name: "ops" + type_id: 0x35b4b910 + offset: 192 +} member { id: 0xafbed153 name: "ops" @@ -146663,6 +146925,12 @@ member { type_id: 0x05f260a2 offset: 256 } +member { + id: 0x8c6379be + name: "parent_idr" + type_id: 0x0bbe1c3e + offset: 7808 +} member { id: 0x018c7f62 name: "parent_idx" @@ -153012,6 +153280,12 @@ member { type_id: 0x6720d32f offset: 576 } +member { + id: 0x1d43d5c8 + name: "power_zone" + type_id: 0x2c3e6eed + offset: 64 +} member { id: 0x9865a105 name: "poweroff" @@ -155175,6 +155449,12 @@ member { id: 0xf21ad1f2 name: "private_data" type_id: 0x18bd6530 + offset: 7872 +} +member { + id: 0xf21ad1f3 + name: "private_data" + type_id: 0x18bd6530 offset: 3392 } member { @@ -164131,6 +164411,12 @@ member { type_id: 0x2d30dbe4 offset: 4416 } +member { + id: 0xaeb60a04 + name: "release" + type_id: 0x2ef15d2b + offset: 128 +} member { id: 0xaeb63cec name: "release" @@ -164149,6 +164435,12 @@ member { type_id: 0x2f764d18 offset: 960 } +member { + id: 0xaeb7c60b + name: "release" + type_id: 0x2f33e782 + offset: 448 +} member { id: 0xe8e1b292 name: "release_agent_path" @@ -166977,6 +167269,12 @@ member { type_id: 0xfa03c3cb offset: 3776 } +member { + id: 0xf3a83d0c + name: "reset_energy_uj" + type_id: 0x2f33e782 + offset: 128 +} member { id: 0x158b8e39 name: "reset_frame_context" @@ -176288,6 +176586,17 @@ member { type_id: 0x2ff974a7 offset: 704 } +member { + id: 0x917a718a + name: "set_enable" + type_id: 0x2f5e98dd + offset: 320 +} +member { + id: 0x917bb4f7 + name: "set_enable" + type_id: 0x2e9c2274 +} member { id: 0xc51a7ec1 name: "set_ent" @@ -176833,6 +177142,11 @@ member { type_id: 0x2de9ae36 offset: 2432 } +member { + id: 0x7baafc27 + name: "set_power_limit_uw" + type_id: 0x2f5de562 +} member { id: 0x7eba804e name: "set_pretimeout" @@ -177217,6 +177531,12 @@ member { type_id: 0xc9082b19 offset: 224 } +member { + id: 0x462303e7 + name: "set_time_window_us" + type_id: 0x2f5de562 + offset: 128 +} member { id: 0x05571fa2 name: "set_timeout" @@ -210914,6 +211234,18 @@ member { name: "zone" type_id: 0x0a63398f } +member { + id: 0xb67e31dd + name: "zone_attr_count" + type_id: 0x6720d32f + offset: 8000 +} +member { + id: 0x6612b97a + name: "zone_dev_attrs" + type_id: 0x0ae4cc78 + offset: 7936 +} member { id: 0x86c953a0 name: "zone_device_data" @@ -251018,6 +251350,99 @@ struct_union { member_id: 0x7f88b19c } } +struct_union { + id: 0xea17f769 + kind: STRUCT + name: "powercap_control_type" + definition { + bytesize: 1024 + member_id: 0xce1ac149 + member_id: 0xb00531dc + member_id: 0x429469eb + member_id: 0xafba7798 + member_id: 0x2d4b3aaa + member_id: 0x41611b26 + member_id: 0x0fa6c1c0 + } +} +struct_union { + id: 0xcebf3ccf + kind: STRUCT + name: "powercap_control_type_ops" + definition { + bytesize: 24 + member_id: 0x917bb4f7 + member_id: 0x1e0abfd9 + member_id: 0xaeb60a04 + } +} +struct_union { + id: 0x9ab95d2b + kind: STRUCT + name: "powercap_zone" + definition { + bytesize: 1080 + member_id: 0xcc480c42 + member_id: 0x0ddfeb5a + member_id: 0xf0f0bb7d + member_id: 0xafbebcd3 + member_id: 0xce1ac1e2 + member_id: 0x9b6df7c9 + member_id: 0xb0053df5 + member_id: 0x8c6379be + member_id: 0xf21ad1f2 + member_id: 0x6612b97a + member_id: 0xb67e31dd + member_id: 0x327f9c2d + member_id: 0x6d170429 + member_id: 0x41611ed7 + member_id: 0xfd7a5c77 + } +} +struct_union { + id: 0x649c5e5b + kind: STRUCT + name: "powercap_zone_constraint" + definition { + bytesize: 24 + member_id: 0xcc480c42 + member_id: 0x1d43d5c8 + member_id: 0xafb0e881 + } +} +struct_union { + id: 0x675a2868 + kind: STRUCT + name: "powercap_zone_constraint_ops" + definition { + bytesize: 72 + member_id: 0x7baafc27 + member_id: 0x4eae2f0e + member_id: 0x462303e7 + member_id: 0xcec3c09c + member_id: 0x690ef040 + member_id: 0x0a0601b8 + member_id: 0x39de1472 + member_id: 0xbbc23eea + member_id: 0x1a6adf19 + } +} +struct_union { + id: 0x820fbdfd + kind: STRUCT + name: "powercap_zone_ops" + definition { + bytesize: 64 + member_id: 0x870b1d4a + member_id: 0xc5eb7d5e + member_id: 0xf3a83d0c + member_id: 0x48da06d8 + member_id: 0xf4ddc253 + member_id: 0x917a718a + member_id: 0x1e0b7810 + member_id: 0xaeb7c60b + } +} struct_union { id: 0x8d548cbf kind: STRUCT @@ -262804,7 +263229,7 @@ struct_union { member_id: 0xb3bb33a4 member_id: 0x18f3f23f member_id: 0xc75c5a51 - member_id: 0xf21ad1f2 + member_id: 0xf21ad1f3 } } struct_union { @@ -295233,6 +295658,17 @@ function { return_type_id: 0x48b5725f parameter_id: 0x10f3dd9c } +function { + id: 0x14b01c1c + return_type_id: 0x2c3e6eed + parameter_id: 0x2c3e6eed + parameter_id: 0x3015c47d + parameter_id: 0x3e10b518 + parameter_id: 0x2c3e6eed + parameter_id: 0x35b4b910 + parameter_id: 0x6720d32f + parameter_id: 0x3be1e049 +} function { id: 0x14b1b37d return_type_id: 0x48b5725f @@ -296886,6 +297322,13 @@ function { parameter_id: 0x4585663f parameter_id: 0x06835e9c } +function { + id: 0x16fd178e + return_type_id: 0x3015c47d + parameter_id: 0x3015c47d + parameter_id: 0x3e10b518 + parameter_id: 0x317fb103 +} function { id: 0x16fd805c return_type_id: 0x48b5725f @@ -309351,6 +309794,12 @@ function { parameter_id: 0x30cfc1c2 parameter_id: 0x6720d32f } +function { + id: 0x90306f4f + return_type_id: 0x6720d32f + parameter_id: 0x3015c47d + parameter_id: 0x6d7f5ff6 +} function { id: 0x903246db return_type_id: 0x6720d32f @@ -310501,6 +310950,12 @@ function { parameter_id: 0x3f0185ef parameter_id: 0x4585663f } +function { + id: 0x91356b8b + return_type_id: 0x6720d32f + parameter_id: 0x3015c47d + parameter_id: 0x2c3e6eed +} function { id: 0x913618b5 return_type_id: 0x6720d32f @@ -310949,6 +311404,11 @@ function { parameter_id: 0x2ad2338c parameter_id: 0x07dcdbe1 } +function { + id: 0x91859230 + return_type_id: 0x6720d32f + parameter_id: 0x3015c47d +} function { id: 0x9185daf6 return_type_id: 0x6720d32f @@ -311369,6 +311829,12 @@ function { parameter_id: 0x38d23361 parameter_id: 0x00c72527 } +function { + id: 0x91c2ad89 + return_type_id: 0x6720d32f + parameter_id: 0x3015c47d + parameter_id: 0x11cfee5a +} function { id: 0x91c2e0c4 return_type_id: 0x6720d32f @@ -316372,6 +316838,12 @@ function { return_type_id: 0x6720d32f parameter_id: 0x2ed870a7 } +function { + id: 0x96371b41 + return_type_id: 0x6720d32f + parameter_id: 0x2c3e6eed + parameter_id: 0x2e18f543 +} function { id: 0x963decf6 return_type_id: 0x6720d32f @@ -316548,6 +317020,11 @@ function { parameter_id: 0x2f5fce77 parameter_id: 0x37b51a1e } +function { + id: 0x968f7894 + return_type_id: 0x6720d32f + parameter_id: 0x2c3e6eed +} function { id: 0x968fd49c return_type_id: 0x6720d32f @@ -316770,6 +317247,12 @@ function { parameter_id: 0x384c5795 parameter_id: 0x6720d32f } +function { + id: 0x96c8472d + return_type_id: 0x6720d32f + parameter_id: 0x2c3e6eed + parameter_id: 0x11cfee5a +} function { id: 0x96c905b2 return_type_id: 0x6720d32f @@ -317014,6 +317497,13 @@ function { parameter_id: 0x2b8f13b3 parameter_id: 0x1ce748dd } +function { + id: 0x97187de5 + return_type_id: 0x6720d32f + parameter_id: 0x2c3e6eed + parameter_id: 0x6720d32f + parameter_id: 0x2e18f543 +} function { id: 0x9719ad86 return_type_id: 0x6720d32f @@ -317089,6 +317579,13 @@ function { parameter_id: 0x0b27dc43 parameter_id: 0x3e10b518 } +function { + id: 0x97377314 + return_type_id: 0x6720d32f + parameter_id: 0x2c3e6eed + parameter_id: 0x6720d32f + parameter_id: 0x92233392 +} function { id: 0x973837f7 return_type_id: 0x6720d32f @@ -317102,6 +317599,12 @@ function { parameter_id: 0x2b8f13b3 parameter_id: 0x16598b41 } +function { + id: 0x973a85eb + return_type_id: 0x6720d32f + parameter_id: 0x2c3e6eed + parameter_id: 0x6d7f5ff6 +} function { id: 0x973d5c81 return_type_id: 0x6720d32f @@ -318673,6 +319176,12 @@ function { parameter_id: 0x295c7202 parameter_id: 0x295c7202 } +function { + id: 0x98d70f54 + return_type_id: 0x3e10b518 + parameter_id: 0x2c3e6eed + parameter_id: 0x6720d32f +} function { id: 0x98da7358 return_type_id: 0x6720d32f @@ -386258,6 +386767,42 @@ elf_symbol { type_id: 0x169d12f4 full_name: "power_supply_unregister" } +elf_symbol { + id: 0x5103fa37 + name: "powercap_register_control_type" + is_defined: true + symbol_type: FUNCTION + crc: 0x74c29aa3 + type_id: 0x16fd178e + full_name: "powercap_register_control_type" +} +elf_symbol { + id: 0x5469c8f3 + name: "powercap_register_zone" + is_defined: true + symbol_type: FUNCTION + crc: 0x72a0a390 + type_id: 0x14b01c1c + full_name: "powercap_register_zone" +} +elf_symbol { + id: 0x5f2a259f + name: "powercap_unregister_control_type" + is_defined: true + symbol_type: FUNCTION + crc: 0xd30deb7a + type_id: 0x91859230 + full_name: "powercap_unregister_control_type" +} +elf_symbol { + id: 0x116aa230 + name: "powercap_unregister_zone" + is_defined: true + symbol_type: FUNCTION + crc: 0xe0264b73 + type_id: 0x91356b8b + full_name: "powercap_unregister_zone" +} elf_symbol { id: 0x1a646dda name: "ppp_channel_index" @@ -413020,6 +413565,10 @@ interface { symbol_id: 0x8900d981 symbol_id: 0x9d05e78b symbol_id: 0x8a65993b + symbol_id: 0x5103fa37 + symbol_id: 0x5469c8f3 + symbol_id: 0x5f2a259f + symbol_id: 0x116aa230 symbol_id: 0x1a646dda symbol_id: 0x52651755 symbol_id: 0x3987d121 diff --git a/android/abi_gki_aarch64_qcom b/android/abi_gki_aarch64_qcom index d3aa2d29239d..aef840a271ba 100644 --- a/android/abi_gki_aarch64_qcom +++ b/android/abi_gki_aarch64_qcom @@ -2489,6 +2489,10 @@ power_supply_set_property power_supply_unregister power_supply_unreg_notifier + powercap_register_control_type + powercap_register_zone + powercap_unregister_control_type + powercap_unregister_zone ppp_channel_index ppp_dev_name ppp_input From 4d55129aea65ac23a34b9053f3f1ee4d92883032 Mon Sep 17 00:00:00 2001 From: Joe Fradley Date: Tue, 22 Nov 2022 14:54:49 -0800 Subject: [PATCH 66/86] UPSTREAM: crypto: x86/curve25519 - disable gcov curve25519-x86_64.c fails to build when CONFIG_GCOV_KERNEL is enabled. The error is "inline assembly requires more registers than available" thrown from the `fsqr()` function. Therefore, excluding this file from GCOV profiling until this issue is resolved. Thereby allowing CONFIG_GCOV_PROFILE_ALL to be enabled for x86. Change-Id: Icbb1ed0e1b99f1cb0a21ea7a2920b3edce70c38d Signed-off-by: Joe Fradley Signed-off-by: Herbert Xu (cherry picked from commit c390c452ebeb44cb979b7374d3acc3859415e86c) --- arch/x86/crypto/Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index 3b1d701a4f6c..3e7a329235bd 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -107,3 +107,6 @@ quiet_cmd_perlasm = PERLASM $@ cmd_perlasm = $(PERL) $< > $@ $(obj)/%.S: $(src)/%.pl FORCE $(call if_changed,perlasm) + +# Disable GCOV in odd or sensitive code +GCOV_PROFILE_curve25519-x86_64.o := n From 93188d7732900c73116ff8e481d7e6c9fee6c2c7 Mon Sep 17 00:00:00 2001 From: RD Babiera Date: Thu, 2 May 2024 18:58:12 +0000 Subject: [PATCH 67/86] BACKPORT: usb: typec: tcpm: enforce ready state when queueing alt mode vdm Before sending Enter Mode for an Alt Mode, there is a gap between Discover Modes and the Alt Mode driver queueing the Enter Mode VDM for the port partner to send a message to the port. If this message results in unregistering Alt Modes such as in a DR_SWAP, then the following deadlock can occur with respect to the DisplayPort Alt Mode driver: 1. The DR_SWAP state holds port->lock. Unregistering the Alt Mode driver results in a cancel_work_sync() that waits for the current dp_altmode_work to finish. 2. dp_altmode_work makes a call to tcpm_altmode_enter. The deadlock occurs because tcpm_queue_vdm_unlock attempts to hold port->lock. Before attempting to grab the lock, ensure that the port is in a state vdm_run_state_machine can run in. Alt Mode unregistration will not occur in these states. Fixes: 03eafcfb60c0 ("usb: typec: tcpm: Add tcpm_queue_vdm_unlocked() helper") Cc: stable@vger.kernel.org Signed-off-by: RD Babiera Reviewed-by: Heikki Krogerus Link: https://lore.kernel.org/r/20240423202356.3372314-2-rdbabiera@google.com Signed-off-by: Greg Kroah-Hartman Bug: 333787869 (cherry picked from commit cdc9946ea6377e8e214b135ccc308c5e514ba25f) [rd: removed SRC_VDM_IDENTITY_REQUEST check, state not defined in branch] Change-Id: I8018d1fdc294885ae609b6e45e9bf6ab190897b9 Signed-off-by: RD Babiera --- drivers/usb/typec/tcpm/tcpm.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index 3d2c05a60d85..cc55d03ef2f3 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -1489,6 +1489,9 @@ static void tcpm_queue_vdm(struct tcpm_port *port, const u32 header, static void tcpm_queue_vdm_unlocked(struct tcpm_port *port, const u32 header, const u32 *data, int cnt) { + if (port->state != SRC_READY && port->state != SNK_READY) + return; + mutex_lock(&port->lock); tcpm_queue_vdm(port, header, data, cnt); mutex_unlock(&port->lock); From 47dfe41d57ca67b3fbe52a119f5b663e7269f55d Mon Sep 17 00:00:00 2001 From: RD Babiera Date: Thu, 2 May 2024 18:50:51 +0000 Subject: [PATCH 68/86] UPSTREAM: usb: typec: tcpm: clear pd_event queue in PORT_RESET When a Fast Role Swap control message attempt results in a transition to ERROR_RECOVERY, the TCPC can still queue a TCPM_SOURCING_VBUS event. If the event is queued but processed after the tcpm_reset_port() call in the PORT_RESET state, then the following occurs: 1. tcpm_reset_port() calls tcpm_init_vbus() to reset the vbus sourcing and sinking state 2. tcpm_pd_event_handler() turns VBUS on before the port is in the default state. 3. The port resolves as a sink. In the SNK_DISCOVERY state, tcpm_set_charge() cannot set vbus to charge. Clear pd events within PORT_RESET to get rid of non-applicable events. Fixes: b17dd57118fe ("staging: typec: tcpm: Improve role swap with non PD capable partners") Cc: stable@vger.kernel.org Signed-off-by: RD Babiera Reviewed-by: Heikki Krogerus Link: https://lore.kernel.org/r/20240423202715.3375827-2-rdbabiera@google.com Signed-off-by: Greg Kroah-Hartman Bug: 311127232 (cherry picked from commit bf20c69cf3cf9c6445c4925dd9a8a6ca1b78bfdf) Change-Id: I9b27d040d0acdeb2af74fd3fe90d246b864b5141 Signed-off-by: RD Babiera --- drivers/usb/typec/tcpm/tcpm.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index cc55d03ef2f3..34543d2c1f0b 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -4928,6 +4928,7 @@ static void run_state_machine(struct tcpm_port *port) break; case PORT_RESET: tcpm_reset_port(port); + port->pd_events = 0; tcpm_set_cc(port, TYPEC_CC_OPEN); tcpm_set_state(port, PORT_RESET_WAIT_OFF, PD_T_ERROR_RECOVERY); From 039d2a958c13dace23cc21ba85f98988fbdc8525 Mon Sep 17 00:00:00 2001 From: Matias Ezequiel Vara Larsen Date: Wed, 25 Oct 2023 11:49:19 +0200 Subject: [PATCH 69/86] UPSTREAM: ALSA: virtio: use ack callback This commit uses the ack() callback to determine when a buffer has been updated, then exposes it to guest. The current mechanism splits a dma buffer into descriptors that are exposed to the device. This dma buffer is shared with the user application. When the device consumes a buffer, the driver moves the request from the used ring to available ring. The driver exposes the buffer to the device without knowing if the content has been updated from the user. The section 2.8.21.1 of the virtio spec states that: "The device MAY access the descriptor chains the driver created and the memory they refer to immediately". If the device picks up buffers from the available ring just after it is notified, it happens that the content may be old. When the ack() callback is invoked, the driver exposes only the buffers that have already been updated, i.e., enqueued in the available ring. Thus, the device always picks up a buffer that is updated. For capturing, the driver starts by exposing all the available buffers to device. After device updates the content of a buffer, it enqueues it in the used ring. It is only after the ack() for capturing is issued that the driver re-enqueues the buffer in the available ring. Co-developed-by: Anton Yakovlev Signed-off-by: Anton Yakovlev Signed-off-by: Matias Ezequiel Vara Larsen Link: https://lore.kernel.org/r/ZTjkn1YAFz67yfqx@fedora Signed-off-by: Takashi Iwai Bug: 331600622 Change-Id: If1269f1777432256eea0e0275b678d0d649753bb (cherry picked from commit fe981e67568c41de6caae25d70b5f203b94452cc) Signed-off-by: Brian Daniels --- sound/virtio/virtio_pcm.c | 6 +- sound/virtio/virtio_pcm.h | 9 ++- sound/virtio/virtio_pcm_msg.c | 73 ++++++++++++-------- sound/virtio/virtio_pcm_ops.c | 125 +++++++++++++++++++++++++++------- 4 files changed, 155 insertions(+), 58 deletions(-) diff --git a/sound/virtio/virtio_pcm.c b/sound/virtio/virtio_pcm.c index c10d91fff2fb..967e4c45be9b 100644 --- a/sound/virtio/virtio_pcm.c +++ b/sound/virtio/virtio_pcm.c @@ -109,7 +109,9 @@ static int virtsnd_pcm_build_hw(struct virtio_pcm_substream *vss, SNDRV_PCM_INFO_BATCH | SNDRV_PCM_INFO_BLOCK_TRANSFER | SNDRV_PCM_INFO_INTERLEAVED | - SNDRV_PCM_INFO_PAUSE; + SNDRV_PCM_INFO_PAUSE | + SNDRV_PCM_INFO_NO_REWINDS | + SNDRV_PCM_INFO_SYNC_APPLPTR; if (!info->channels_min || info->channels_min > info->channels_max) { dev_err(&vdev->dev, @@ -471,7 +473,7 @@ int virtsnd_pcm_build_devs(struct virtio_snd *snd) for (kss = ks->substream; kss; kss = kss->next) vs->substreams[kss->number]->substream = kss; - snd_pcm_set_ops(vpcm->pcm, i, &virtsnd_pcm_ops); + snd_pcm_set_ops(vpcm->pcm, i, &virtsnd_pcm_ops[i]); } snd_pcm_set_managed_buffer_all(vpcm->pcm, diff --git a/sound/virtio/virtio_pcm.h b/sound/virtio/virtio_pcm.h index 062eb8e8f2cf..5dd1b43b9493 100644 --- a/sound/virtio/virtio_pcm.h +++ b/sound/virtio/virtio_pcm.h @@ -9,6 +9,7 @@ #include #include #include +#include struct virtio_pcm; struct virtio_pcm_msg; @@ -21,6 +22,7 @@ struct virtio_pcm_msg; * @direction: Stream data flow direction (SNDRV_PCM_STREAM_XXX). * @features: Stream VirtIO feature bit map (1 << VIRTIO_SND_PCM_F_XXX). * @substream: Kernel ALSA substream. + * @pcm_indirect: Kernel indirect pcm structure. * @hw: Kernel ALSA substream hardware descriptor. * @elapsed_period: Kernel work to handle the elapsed period state. * @lock: Spinlock that protects fields shared by interrupt handlers and @@ -46,6 +48,7 @@ struct virtio_pcm_substream { u32 direction; u32 features; struct snd_pcm_substream *substream; + struct snd_pcm_indirect pcm_indirect; struct snd_pcm_hardware hw; struct work_struct elapsed_period; spinlock_t lock; @@ -57,7 +60,6 @@ struct virtio_pcm_substream { bool suspended; struct virtio_pcm_msg **msgs; unsigned int nmsgs; - int msg_last_enqueued; unsigned int msg_count; wait_queue_head_t msg_empty; }; @@ -90,7 +92,7 @@ struct virtio_pcm { struct virtio_pcm_stream streams[SNDRV_PCM_STREAM_LAST + 1]; }; -extern const struct snd_pcm_ops virtsnd_pcm_ops; +extern const struct snd_pcm_ops virtsnd_pcm_ops[]; int virtsnd_pcm_validate(struct virtio_device *vdev); @@ -117,7 +119,8 @@ int virtsnd_pcm_msg_alloc(struct virtio_pcm_substream *vss, void virtsnd_pcm_msg_free(struct virtio_pcm_substream *vss); -int virtsnd_pcm_msg_send(struct virtio_pcm_substream *vss); +int virtsnd_pcm_msg_send(struct virtio_pcm_substream *vss, unsigned long offset, + unsigned long bytes); unsigned int virtsnd_pcm_msg_pending_num(struct virtio_pcm_substream *vss); diff --git a/sound/virtio/virtio_pcm_msg.c b/sound/virtio/virtio_pcm_msg.c index aca2dc1989ba..542446c4c7ba 100644 --- a/sound/virtio/virtio_pcm_msg.c +++ b/sound/virtio/virtio_pcm_msg.c @@ -155,7 +155,6 @@ int virtsnd_pcm_msg_alloc(struct virtio_pcm_substream *vss, sizeof(msg->xfer)); sg_init_one(&msg->sgs[PCM_MSG_SG_STATUS], &msg->status, sizeof(msg->status)); - msg->length = period_bytes; virtsnd_pcm_sg_from(&msg->sgs[PCM_MSG_SG_DATA], sg_num, data, period_bytes); @@ -186,61 +185,75 @@ void virtsnd_pcm_msg_free(struct virtio_pcm_substream *vss) /** * virtsnd_pcm_msg_send() - Send asynchronous I/O messages. * @vss: VirtIO PCM substream. + * @offset: starting position that has been updated + * @bytes: number of bytes that has been updated * * All messages are organized in an ordered circular list. Each time the * function is called, all currently non-enqueued messages are added to the - * virtqueue. For this, the function keeps track of two values: - * - * msg_last_enqueued = index of the last enqueued message, - * msg_count = # of pending messages in the virtqueue. + * virtqueue. For this, the function uses offset and bytes to calculate the + * messages that need to be added. * * Context: Any context. Expects the tx/rx queue and the VirtIO substream * spinlocks to be held by caller. * Return: 0 on success, -errno on failure. */ -int virtsnd_pcm_msg_send(struct virtio_pcm_substream *vss) +int virtsnd_pcm_msg_send(struct virtio_pcm_substream *vss, unsigned long offset, + unsigned long bytes) { - struct snd_pcm_runtime *runtime = vss->substream->runtime; struct virtio_snd *snd = vss->snd; struct virtio_device *vdev = snd->vdev; struct virtqueue *vqueue = virtsnd_pcm_queue(vss)->vqueue; - int i; - int n; + unsigned long period_bytes = snd_pcm_lib_period_bytes(vss->substream); + unsigned long start, end, i; + unsigned int msg_count = vss->msg_count; bool notify = false; + int rc; - i = (vss->msg_last_enqueued + 1) % runtime->periods; - n = runtime->periods - vss->msg_count; + start = offset / period_bytes; + end = (offset + bytes - 1) / period_bytes; - for (; n; --n, i = (i + 1) % runtime->periods) { + for (i = start; i <= end; i++) { struct virtio_pcm_msg *msg = vss->msgs[i]; struct scatterlist *psgs[] = { &msg->sgs[PCM_MSG_SG_XFER], &msg->sgs[PCM_MSG_SG_DATA], &msg->sgs[PCM_MSG_SG_STATUS] }; - int rc; + unsigned long n; - msg->xfer.stream_id = cpu_to_le32(vss->sid); - memset(&msg->status, 0, sizeof(msg->status)); + n = period_bytes - (offset % period_bytes); + if (n > bytes) + n = bytes; - if (vss->direction == SNDRV_PCM_STREAM_PLAYBACK) - rc = virtqueue_add_sgs(vqueue, psgs, 2, 1, msg, - GFP_ATOMIC); - else - rc = virtqueue_add_sgs(vqueue, psgs, 1, 2, msg, - GFP_ATOMIC); + msg->length += n; + if (msg->length == period_bytes) { + msg->xfer.stream_id = cpu_to_le32(vss->sid); + memset(&msg->status, 0, sizeof(msg->status)); - if (rc) { - dev_err(&vdev->dev, - "SID %u: failed to send I/O message\n", - vss->sid); - return rc; + if (vss->direction == SNDRV_PCM_STREAM_PLAYBACK) + rc = virtqueue_add_sgs(vqueue, psgs, 2, 1, msg, + GFP_ATOMIC); + else + rc = virtqueue_add_sgs(vqueue, psgs, 1, 2, msg, + GFP_ATOMIC); + + if (rc) { + dev_err(&vdev->dev, + "SID %u: failed to send I/O message\n", + vss->sid); + return rc; + } + + vss->msg_count++; } - vss->msg_last_enqueued = i; - vss->msg_count++; + offset = 0; + bytes -= n; } + if (msg_count == vss->msg_count) + return 0; + if (!(vss->features & (1U << VIRTIO_SND_PCM_F_MSG_POLLING))) notify = virtqueue_kick_prepare(vqueue); @@ -309,6 +322,8 @@ static void virtsnd_pcm_msg_complete(struct virtio_pcm_msg *msg, if (vss->hw_ptr >= vss->buffer_bytes) vss->hw_ptr -= vss->buffer_bytes; + msg->length = 0; + vss->xfer_xrun = false; vss->msg_count--; @@ -320,8 +335,6 @@ static void virtsnd_pcm_msg_complete(struct virtio_pcm_msg *msg, le32_to_cpu(msg->status.latency_bytes)); schedule_work(&vss->elapsed_period); - - virtsnd_pcm_msg_send(vss); } else if (!vss->msg_count) { wake_up_all(&vss->msg_empty); } diff --git a/sound/virtio/virtio_pcm_ops.c b/sound/virtio/virtio_pcm_ops.c index f8bfb87624be..ad12aae52fc3 100644 --- a/sound/virtio/virtio_pcm_ops.c +++ b/sound/virtio/virtio_pcm_ops.c @@ -282,7 +282,6 @@ static int virtsnd_pcm_prepare(struct snd_pcm_substream *substream) vss->buffer_bytes = snd_pcm_lib_buffer_bytes(substream); vss->hw_ptr = 0; - vss->msg_last_enqueued = -1; } else { struct snd_pcm_runtime *runtime = substream->runtime; unsigned int buffer_bytes = snd_pcm_lib_buffer_bytes(substream); @@ -300,6 +299,11 @@ static int virtsnd_pcm_prepare(struct snd_pcm_substream *substream) vss->suspended = false; vss->msg_count = 0; + memset(&vss->pcm_indirect, 0, sizeof(vss->pcm_indirect)); + vss->pcm_indirect.sw_buffer_size = + vss->pcm_indirect.hw_buffer_size = + snd_pcm_lib_buffer_bytes(substream); + msg = virtsnd_pcm_ctl_msg_alloc(vss, VIRTIO_SND_R_PCM_PREPARE, GFP_KERNEL); if (!msg) @@ -324,7 +328,7 @@ static int virtsnd_pcm_trigger(struct snd_pcm_substream *substream, int command) struct virtio_snd_queue *queue; struct virtio_snd_msg *msg; unsigned long flags; - int rc; + int rc = 0; switch (command) { case SNDRV_PCM_TRIGGER_START: @@ -333,7 +337,8 @@ static int virtsnd_pcm_trigger(struct snd_pcm_substream *substream, int command) spin_lock_irqsave(&queue->lock, flags); spin_lock(&vss->lock); - rc = virtsnd_pcm_msg_send(vss); + if (vss->direction == SNDRV_PCM_STREAM_CAPTURE) + rc = virtsnd_pcm_msg_send(vss, 0, vss->buffer_bytes); if (!rc) vss->xfer_enabled = true; spin_unlock(&vss->lock); @@ -428,37 +433,111 @@ static int virtsnd_pcm_sync_stop(struct snd_pcm_substream *substream) } /** - * virtsnd_pcm_pointer() - Get the current hardware position for the PCM - * substream. + * virtsnd_pcm_pb_pointer() - Get the current hardware position for the PCM + * substream for playback. * @substream: Kernel ALSA substream. * - * Context: Any context. Takes and releases the VirtIO substream spinlock. + * Context: Any context. * Return: Hardware position in frames inside [0 ... buffer_size) range. */ static snd_pcm_uframes_t -virtsnd_pcm_pointer(struct snd_pcm_substream *substream) +virtsnd_pcm_pb_pointer(struct snd_pcm_substream *substream) { struct virtio_pcm_substream *vss = snd_pcm_substream_chip(substream); - snd_pcm_uframes_t hw_ptr = SNDRV_PCM_POS_XRUN; + + return snd_pcm_indirect_playback_pointer(substream, + &vss->pcm_indirect, + vss->hw_ptr); +} + +/** + * virtsnd_pcm_cp_pointer() - Get the current hardware position for the PCM + * substream for capture. + * @substream: Kernel ALSA substream. + * + * Context: Any context. + * Return: Hardware position in frames inside [0 ... buffer_size) range. + */ +static snd_pcm_uframes_t +virtsnd_pcm_cp_pointer(struct snd_pcm_substream *substream) +{ + struct virtio_pcm_substream *vss = snd_pcm_substream_chip(substream); + + return snd_pcm_indirect_capture_pointer(substream, + &vss->pcm_indirect, + vss->hw_ptr); +} + +static void virtsnd_pcm_trans_copy(struct snd_pcm_substream *substream, + struct snd_pcm_indirect *rec, size_t bytes) +{ + struct virtio_pcm_substream *vss = snd_pcm_substream_chip(substream); + + virtsnd_pcm_msg_send(vss, rec->sw_data, bytes); +} + +static int virtsnd_pcm_pb_ack(struct snd_pcm_substream *substream) +{ + struct virtio_pcm_substream *vss = snd_pcm_substream_chip(substream); + struct virtio_snd_queue *queue = virtsnd_pcm_queue(vss); unsigned long flags; + int rc; - spin_lock_irqsave(&vss->lock, flags); - if (!vss->xfer_xrun) - hw_ptr = bytes_to_frames(substream->runtime, vss->hw_ptr); - spin_unlock_irqrestore(&vss->lock, flags); + spin_lock_irqsave(&queue->lock, flags); + spin_lock(&vss->lock); - return hw_ptr; + rc = snd_pcm_indirect_playback_transfer(substream, &vss->pcm_indirect, + virtsnd_pcm_trans_copy); + + spin_unlock(&vss->lock); + spin_unlock_irqrestore(&queue->lock, flags); + + return rc; +} + +static int virtsnd_pcm_cp_ack(struct snd_pcm_substream *substream) +{ + struct virtio_pcm_substream *vss = snd_pcm_substream_chip(substream); + struct virtio_snd_queue *queue = virtsnd_pcm_queue(vss); + unsigned long flags; + int rc; + + spin_lock_irqsave(&queue->lock, flags); + spin_lock(&vss->lock); + + rc = snd_pcm_indirect_capture_transfer(substream, &vss->pcm_indirect, + virtsnd_pcm_trans_copy); + + spin_unlock(&vss->lock); + spin_unlock_irqrestore(&queue->lock, flags); + + return rc; } /* PCM substream operators map. */ -const struct snd_pcm_ops virtsnd_pcm_ops = { - .open = virtsnd_pcm_open, - .close = virtsnd_pcm_close, - .ioctl = snd_pcm_lib_ioctl, - .hw_params = virtsnd_pcm_hw_params, - .hw_free = virtsnd_pcm_hw_free, - .prepare = virtsnd_pcm_prepare, - .trigger = virtsnd_pcm_trigger, - .sync_stop = virtsnd_pcm_sync_stop, - .pointer = virtsnd_pcm_pointer, +const struct snd_pcm_ops virtsnd_pcm_ops[] = { + { + .open = virtsnd_pcm_open, + .close = virtsnd_pcm_close, + .ioctl = snd_pcm_lib_ioctl, + .hw_params = virtsnd_pcm_hw_params, + .hw_free = virtsnd_pcm_hw_free, + .prepare = virtsnd_pcm_prepare, + .trigger = virtsnd_pcm_trigger, + .sync_stop = virtsnd_pcm_sync_stop, + .pointer = virtsnd_pcm_pb_pointer, + .ack = virtsnd_pcm_pb_ack, + }, + { + .open = virtsnd_pcm_open, + .close = virtsnd_pcm_close, + .ioctl = snd_pcm_lib_ioctl, + .hw_params = virtsnd_pcm_hw_params, + .hw_free = virtsnd_pcm_hw_free, + .prepare = virtsnd_pcm_prepare, + .trigger = virtsnd_pcm_trigger, + .sync_stop = virtsnd_pcm_sync_stop, + .pointer = virtsnd_pcm_cp_pointer, + .ack = virtsnd_pcm_cp_ack, + }, }; From 0d080e01a24c65b2eba12f0784900c39df2cdd67 Mon Sep 17 00:00:00 2001 From: Qais Yousef Date: Mon, 13 May 2024 18:32:23 +0000 Subject: [PATCH 70/86] ANDROID: Export sysctl_sched_wakeup_granularity to enable modifying it The value is only available in debugfs. Export it to allow out of tree modules to modify it. Bug: 337139487 Signed-off-by: Qais Yousef Change-Id: I4b3a5aeeee0539b88756aacc97df6181f68d9d0d --- kernel/sched/fair.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index a1456fd7ced6..3eeb4e7db055 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -129,6 +129,7 @@ unsigned int sysctl_sched_child_runs_first __read_mostly; * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds) */ unsigned int sysctl_sched_wakeup_granularity = 1000000UL; +EXPORT_SYMBOL_GPL(sysctl_sched_wakeup_granularity); static unsigned int normalized_sysctl_sched_wakeup_granularity = 1000000UL; const_debug unsigned int sysctl_sched_migration_cost = 500000UL; From 117a94122698a771f8ee6c4912afe33cf29bd01f Mon Sep 17 00:00:00 2001 From: Qais Yousef Date: Mon, 13 May 2024 18:40:44 +0000 Subject: [PATCH 71/86] ANDROID: Update the ABI symbol list Adding the following symbols: - drm_hdmi_avi_infoframe_from_display_mode - hdmi_avi_infoframe_pack - sysctl_sched_wakeup_granularity Bug: 337139487 Change-Id: I08ff1f4406e14fd42fee255d3eb49ccbd9c9b338 Signed-off-by: Qais Yousef --- android/abi_gki_aarch64.stg | 10 ++++++++++ android/abi_gki_aarch64_pixel | 3 +++ 2 files changed, 13 insertions(+) diff --git a/android/abi_gki_aarch64.stg b/android/abi_gki_aarch64.stg index 10ca10afffa0..7f8fba36fd85 100644 --- a/android/abi_gki_aarch64.stg +++ b/android/abi_gki_aarch64.stg @@ -397535,6 +397535,15 @@ elf_symbol { type_id: 0x4585663f full_name: "sysctl_sched_min_granularity" } +elf_symbol { + id: 0xc5679836 + name: "sysctl_sched_wakeup_granularity" + is_defined: true + symbol_type: OBJECT + crc: 0x110347cc + type_id: 0x4585663f + full_name: "sysctl_sched_wakeup_granularity" +} elf_symbol { id: 0x18d0dd21 name: "sysctl_vals" @@ -414762,6 +414771,7 @@ interface { symbol_id: 0xe6ea21b1 symbol_id: 0x87812861 symbol_id: 0x34555a8a + symbol_id: 0xc5679836 symbol_id: 0x18d0dd21 symbol_id: 0x92705587 symbol_id: 0xdbe66171 diff --git a/android/abi_gki_aarch64_pixel b/android/abi_gki_aarch64_pixel index d986d4cb211e..8e9744fc23c9 100644 --- a/android/abi_gki_aarch64_pixel +++ b/android/abi_gki_aarch64_pixel @@ -763,6 +763,7 @@ drm_gem_vm_open drm_get_edid drm_get_format_info + drm_hdmi_avi_infoframe_from_display_mode drm_helper_mode_fill_fb_struct drm_helper_probe_single_connector_modes drm_ioctl @@ -1034,6 +1035,7 @@ handle_sysrq hashlen_string have_governor_per_policy + hdmi_avi_infoframe_pack hex2bin hex_dump_to_buffer hex_to_bin @@ -2237,6 +2239,7 @@ sysctl_sched_idle_min_granularity sysctl_sched_latency sysctl_sched_min_granularity + sysctl_sched_wakeup_granularity sysfs_add_file_to_group sysfs_add_link_to_group sysfs_create_file_ns From 274e3e96961da98fd615b0bac6b08c8cd06ad615 Mon Sep 17 00:00:00 2001 From: Robin Hsu Date: Mon, 13 May 2024 09:45:16 +0000 Subject: [PATCH 72/86] ANDROID: export one function for mm metrics export function for sysfs node formating Bug: 299190787 Change-Id: I71e6a0815efa8df99d036bf457b8a0081999f3de Signed-off-by: Robin Hsu --- android/abi_gki_aarch64.stg | 18 ++++++++++++++++++ android/abi_gki_aarch64_pixel | 1 + fs/seq_file.c | 1 + 3 files changed, 20 insertions(+) diff --git a/android/abi_gki_aarch64.stg b/android/abi_gki_aarch64.stg index 7f8fba36fd85..abd99877e657 100644 --- a/android/abi_gki_aarch64.stg +++ b/android/abi_gki_aarch64.stg @@ -293560,6 +293560,14 @@ function { parameter_id: 0x3e10b518 parameter_id: 0xd5df6730 } +function { + id: 0x11cd0b26 + return_type_id: 0x48b5725f + parameter_id: 0x0665e6b6 + parameter_id: 0x3e10b518 + parameter_id: 0xd5df6730 + parameter_id: 0x4585663f +} function { id: 0x11cf2aed return_type_id: 0x48b5725f @@ -392275,6 +392283,15 @@ elf_symbol { type_id: 0x11cc1d33 full_name: "seq_put_decimal_ull" } +elf_symbol { + id: 0xaa52ec65 + name: "seq_put_decimal_ull_width" + is_defined: true + symbol_type: FUNCTION + crc: 0x20672e2a + type_id: 0x11cd0b26 + full_name: "seq_put_decimal_ull_width" +} elf_symbol { id: 0x25bebf3b name: "seq_putc" @@ -414186,6 +414203,7 @@ interface { symbol_id: 0xc5442265 symbol_id: 0xe76b6b25 symbol_id: 0x05b70a90 + symbol_id: 0xaa52ec65 symbol_id: 0x25bebf3b symbol_id: 0x59b4ca07 symbol_id: 0xba8007cd diff --git a/android/abi_gki_aarch64_pixel b/android/abi_gki_aarch64_pixel index 8e9744fc23c9..d38c701bef44 100644 --- a/android/abi_gki_aarch64_pixel +++ b/android/abi_gki_aarch64_pixel @@ -1988,6 +1988,7 @@ seq_putc seq_put_decimal_ll seq_put_decimal_ull + seq_put_decimal_ull_width seq_puts seq_read seq_release diff --git a/fs/seq_file.c b/fs/seq_file.c index f5fdaf3b1572..35dd5f911c87 100644 --- a/fs/seq_file.c +++ b/fs/seq_file.c @@ -726,6 +726,7 @@ void seq_put_decimal_ull_width(struct seq_file *m, const char *delimiter, overflow: seq_set_overflow(m); } +EXPORT_SYMBOL(seq_put_decimal_ull_width); void seq_put_decimal_ull(struct seq_file *m, const char *delimiter, unsigned long long num) From 6a3d68af9ce6665e7cffa3f668b77cdda4e10fc7 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Tue, 2 Apr 2024 14:36:57 -0700 Subject: [PATCH 73/86] ANDROID: Zap kernel/sched/android.h stubs Long ago with android-mainline, kernel/sched/android.h was added to include stub functions for out-of-tree logic that we wanted to remove from android-mainline without breaking vendor drivers that relied on these stubs. Unfortunately, when the android release branches (android14-6.1 and android15-6.6) forked from android-mainline, the stubs were kept. In some cases, the functionality dropped was re-added to the release branches, but using new function names. Drivers that continued to use the old names, would then end up using the stub impelementations, likely without noticing. Lets try to clean this up by removing the stub header. This should not affect the GKI KABI, and pre-existing modules should continue to work, but when modules are re-build, they may run into build issues and will need tweaks to remove the old stub functions. Bug: 332588830 Change-Id: I516913f2e4c36e6edbffb3c12b633a158ba04287 Signed-off-by: John Stultz --- kernel/sched/android.h | 27 --------------------------- kernel/sched/sched.h | 1 - 2 files changed, 28 deletions(-) delete mode 100644 kernel/sched/android.h diff --git a/kernel/sched/android.h b/kernel/sched/android.h deleted file mode 100644 index 5fe6bb772368..000000000000 --- a/kernel/sched/android.h +++ /dev/null @@ -1,27 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Android scheduler hooks and modifications - * - * Put all of the android-specific scheduler hooks and changes - * in this .h file to make merges and modifications easier. It's also - * simpler to notice what is, and is not, an upstream change this way over time. - */ - - -/* - * task_may_not_preempt - check whether a task may not be preemptible soon - */ -static inline bool task_may_not_preempt(struct task_struct *task, int cpu) -{ - return false; -} - -static inline bool uclamp_boosted(struct task_struct *p) -{ - return false; -} - -static inline bool uclamp_latency_sensitive(struct task_struct *p) -{ - return false; -} diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index f3ee288cc680..8ccf52d0bf2b 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -70,7 +70,6 @@ #include #include #include -#include "android.h" #include #include From 23f2a9f5f13426263f557ff67ce8f4e3f7965d12 Mon Sep 17 00:00:00 2001 From: hulianqin Date: Sun, 12 May 2024 20:25:02 +0800 Subject: [PATCH 74/86] ANDROID: usb: Optimize the problem of slow transfer rate in USB accessory mode The data transfer rate using Google Restore in USB3.2 mode is slower, only about 140MB/s at 5Gbps. The bMaxBurst is not set, and num_fifos in dwc3_gadget_resize_tx_fifosis 1, which results in only 131btye of dwc3 ram space being allocated to ep. Modify bMaxBurst to 6. The 5Gbps rate increases from 140MB/s to 350MB/s. The 10Gbps rate is increased from 220MB/s to 500MB/s. Bug: 340049583 Change-Id: I5710af32c72d0b57afaecc00c4f0909af4b9a299 Signed-off-by: Lianqin Hu Signed-off-by: Lianqin Hu --- drivers/usb/gadget/function/f_accessory.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/usb/gadget/function/f_accessory.c b/drivers/usb/gadget/function/f_accessory.c index ce7814c22057..9a4aec8dbaf0 100644 --- a/drivers/usb/gadget/function/f_accessory.c +++ b/drivers/usb/gadget/function/f_accessory.c @@ -171,7 +171,7 @@ static struct usb_ss_ep_comp_descriptor acc_superspeedplus_comp_desc = { .bDescriptorType = USB_DT_SS_ENDPOINT_COMP, /* the following 2 values can be tweaked if necessary */ - /* .bMaxBurst = 0, */ + .bMaxBurst = 6, /* .bmAttributes = 0, */ }; @@ -196,7 +196,7 @@ static struct usb_ss_ep_comp_descriptor acc_superspeed_comp_desc = { .bDescriptorType = USB_DT_SS_ENDPOINT_COMP, /* the following 2 values can be tweaked if necessary */ - /* .bMaxBurst = 0, */ + .bMaxBurst = 6, /* .bmAttributes = 0, */ }; From c7fcb9bf9aaee38025c43516f8f655acc23bc108 Mon Sep 17 00:00:00 2001 From: Dezhi Huang Date: Wed, 15 May 2024 14:39:06 +0800 Subject: [PATCH 75/86] ANDROID: add vendor hook in do_read_fault to tune fault_around_bytes with this vendor_hook, oem can dynamically adjust fault_around_bytes to balance memory usage and performance Bug: 340749845 Change-Id: I429f4302caf44a769696ccec84e9cc13ea8892ea Signed-off-by: Dezhi Huang --- drivers/android/vendor_hooks.c | 1 + include/trace/hooks/mm.h | 3 +++ mm/memory.c | 3 +++ 3 files changed, 7 insertions(+) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 675646d924e7..ff9a0a78977a 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -275,6 +275,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_readahead_gfp_mask); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_alter_mutex_list_add); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mutex_unlock_slowpath); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_wake_finish); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_tune_fault_around_bytes); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_adjust_alloc_flags); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_adjust_kvmalloc_flags); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_looper_state_registered); diff --git a/include/trace/hooks/mm.h b/include/trace/hooks/mm.h index 28dcf14b9d72..73a544af12b7 100644 --- a/include/trace/hooks/mm.h +++ b/include/trace/hooks/mm.h @@ -208,6 +208,9 @@ DECLARE_HOOK(android_vh_slab_folio_alloced, DECLARE_HOOK(android_vh_kmalloc_large_alloced, TP_PROTO(struct page *page, unsigned int order, gfp_t flags), TP_ARGS(page, order, flags)); +DECLARE_HOOK(android_vh_tune_fault_around_bytes, + TP_PROTO(unsigned long *fault_around_bytes), + TP_ARGS(fault_around_bytes)); #endif /* _TRACE_HOOK_MM_H */ /* This part must be outside protection */ diff --git a/mm/memory.c b/mm/memory.c index 66cbb29cd444..ffbf374a787d 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -92,6 +92,7 @@ #include "pgalloc-track.h" #include "internal.h" #include "swap.h" +#include #if defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) && !defined(CONFIG_COMPILE_TEST) #warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid. @@ -4645,6 +4646,8 @@ static vm_fault_t do_read_fault(struct vm_fault *vmf) { vm_fault_t ret = 0; + trace_android_vh_tune_fault_around_bytes(&fault_around_bytes); + /* * Let's call ->map_pages() first and use ->fault() as fallback * if page by the offset is not ready to be mapped (cold cache or From f9ca61c8d8bb8a2fa0ec90ac2a51874b0e130794 Mon Sep 17 00:00:00 2001 From: Dezhi Huang Date: Wed, 15 May 2024 14:40:45 +0800 Subject: [PATCH 76/86] ANDROID: ABI: Update honor symbol list 1 function symbol(s) added 'int __traceiter_android_vh_tune_fault_around_bytes(unsigned long *)' 1 variable symbol(s) added 'struct tracepoint __tracepoint_android_vh_tune_fault_around_bytes' with this vendor_hook, oem can dynamically adjust fault_around_bytes to balance memory usage and performance Bug: 340749845 Change-Id: I0ec6c5cc0b021dd7e7854dbe13f13c3f6b488b64 Signed-off-by: Dezhi Huang --- android/abi_gki_aarch64.stg | 20 ++++++++++++++++++++ android/abi_gki_aarch64_honor | 4 +++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/android/abi_gki_aarch64.stg b/android/abi_gki_aarch64.stg index abd99877e657..4a8f15b75f4d 100644 --- a/android/abi_gki_aarch64.stg +++ b/android/abi_gki_aarch64.stg @@ -344426,6 +344426,15 @@ elf_symbol { type_id: 0x9b2837bd full_name: "__traceiter_android_vh_try_to_unmap_one" } +elf_symbol { + id: 0x60f9a030 + name: "__traceiter_android_vh_tune_fault_around_bytes" + is_defined: true + symbol_type: FUNCTION + crc: 0xefba8dab + type_id: 0x9bb68fe1 + full_name: "__traceiter_android_vh_tune_fault_around_bytes" +} elf_symbol { id: 0xc0482ac3 name: "__traceiter_android_vh_tune_mmap_readaround" @@ -348710,6 +348719,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_try_to_unmap_one" } +elf_symbol { + id: 0xe94eddba + name: "__tracepoint_android_vh_tune_fault_around_bytes" + is_defined: true + symbol_type: OBJECT + crc: 0xb2b0382b + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_tune_fault_around_bytes" +} elf_symbol { id: 0x0f6579e9 name: "__tracepoint_android_vh_tune_mmap_readaround" @@ -408887,6 +408905,7 @@ interface { symbol_id: 0x2bc25325 symbol_id: 0x0119fc41 symbol_id: 0xd9f43028 + symbol_id: 0x60f9a030 symbol_id: 0xc0482ac3 symbol_id: 0x39155e73 symbol_id: 0x3947ecb4 @@ -409363,6 +409382,7 @@ interface { symbol_id: 0xd9d2bcff symbol_id: 0x09ba106b symbol_id: 0xf9580976 + symbol_id: 0xe94eddba symbol_id: 0x0f6579e9 symbol_id: 0x49b955bd symbol_id: 0xfe6b6962 diff --git a/android/abi_gki_aarch64_honor b/android/abi_gki_aarch64_honor index 29d60a8effd0..ffc84d30e5eb 100644 --- a/android/abi_gki_aarch64_honor +++ b/android/abi_gki_aarch64_honor @@ -7,6 +7,7 @@ __tracepoint_android_vh_file_is_tiny_bypass __tracepoint_android_vh_modify_scan_control __tracepoint_android_vh_should_continue_reclaim + __tracepoint_android_vh_tune_fault_around_bytes __traceiter_android_vh_get_page_wmark __traceiter_android_vh_page_add_new_anon_rmap __traceiter_android_vh_do_shrink_slab @@ -14,4 +15,5 @@ __traceiter_android_vh_mutex_init __traceiter_android_vh_file_is_tiny_bypass __traceiter_android_vh_modify_scan_control - __traceiter_android_vh_should_continue_reclaim \ No newline at end of file + __traceiter_android_vh_should_continue_reclaim + __traceiter_android_vh_tune_fault_around_bytes From 1d241d978d94256071bc3bd1a6537cea2959d712 Mon Sep 17 00:00:00 2001 From: Prashanth K Date: Thu, 2 May 2024 10:11:03 +0530 Subject: [PATCH 77/86] FROMGIT: usb: dwc3: Wait unconditionally after issuing EndXfer command Currently all controller IP/revisions except DWC3_usb3 >= 310a wait 1ms unconditionally for ENDXFER completion when IOC is not set. This is because DWC_usb3 controller revisions >= 3.10a supports GUCTL2[14: Rst_actbitlater] bit which allows polling CMDACT bit to know whether ENDXFER command is completed. Consider a case where an IN request was queued, and parallelly soft_disconnect was called (due to ffs_epfile_release). This eventually calls stop_active_transfer with IOC cleared, hence send_gadget_ep_cmd() skips waiting for CMDACT cleared during EndXfer. For DWC3 controllers with revisions >= 310a, we don't forcefully wait for 1ms either, and we proceed by unmapping the requests. If ENDXFER didn't complete by this time, it leads to SMMU faults since the controller would still be accessing those requests. Fix this by ensuring ENDXFER completion by adding 1ms delay in __dwc3_stop_active_transfer() unconditionally. Cc: stable@vger.kernel.org Fixes: b353eb6dc285 ("usb: dwc3: gadget: Skip waiting for CMDACT cleared during endxfer") Signed-off-by: Prashanth K Acked-by: Thinh Nguyen Link: https://lore.kernel.org/r/20240502044103.1066350-1-quic_prashk@quicinc.com Signed-off-by: Greg Kroah-Hartman Bug: 340415457 (cherry picked from commit 1d26ba0944d398f88aaf997bda3544646cf21945 https: //git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git/ usb-next) Signed-off-by: Prashanth K Change-Id: I0d3792d620ecd380f2a0a80ae5137c25d1adac4d --- drivers/usb/dwc3/gadget.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index ec6ce18ab9d4..23a95fd04e80 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -1708,7 +1708,6 @@ static int __dwc3_gadget_get_frame(struct dwc3 *dwc) */ static int __dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt) { - struct dwc3 *dwc = dep->dwc; struct dwc3_gadget_ep_cmd_params params; u32 cmd; int ret; @@ -1733,8 +1732,7 @@ static int __dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool int dep->resource_index = 0; if (!interrupt) { - if (!DWC3_IP_IS(DWC3) || DWC3_VER_IS_PRIOR(DWC3, 310A)) - mdelay(1); + mdelay(1); dep->flags &= ~DWC3_EP_TRANSFER_STARTED; } else if (!ret) { dep->flags |= DWC3_EP_END_TRANSFER_PENDING; From 6316af1012e7e52b40aa18e4277525c3c4974f63 Mon Sep 17 00:00:00 2001 From: Minchan Kim Date: Tue, 7 May 2024 20:52:33 +0000 Subject: [PATCH 78/86] ANDROID: add vendor hooks and expoert reclaim_pages to reclaim memory Add vendor hooks to reclaim MADV_PAGEOUT memory for asynchrnous device. It also exports reclaim_pages to reclaim memory. Bug: 326662423 Change-Id: Ic2516c64a9dbd53173a3bfb19b6cd21636916c27 Signed-off-by: Minchan Kim --- drivers/android/vendor_hooks.c | 3 +++ include/linux/swap.h | 1 + include/trace/hooks/mm.h | 9 +++++++++ mm/madvise.c | 12 ++++++++++-- mm/vmscan.c | 29 +++++++++++++++++++++-------- 5 files changed, 44 insertions(+), 10 deletions(-) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index ff9a0a78977a..e8af7b774588 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -331,6 +331,9 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_alloc_pages_failure_bypass); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_madvise_pageout_swap_entry); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_madvise_swapin_walk_pmd_entry); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_process_madvise_end); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_madvise_pageout_begin); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_madvise_pageout_end); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_reclaim_folio_list); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_smaps_pte_entry); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_show_smap); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_ctl_dirty_rate); diff --git a/include/linux/swap.h b/include/linux/swap.h index 6d17e7bb330c..671853df78ac 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -439,6 +439,7 @@ extern int vm_swappiness; long remove_mapping(struct address_space *mapping, struct folio *folio); extern unsigned long reclaim_pages(struct list_head *page_list); +extern unsigned long __reclaim_pages(struct list_head *page_list, void *private); #ifdef CONFIG_NUMA extern int node_reclaim_mode; extern int sysctl_min_unmapped_ratio; diff --git a/include/trace/hooks/mm.h b/include/trace/hooks/mm.h index 73a544af12b7..81ce1268c4fd 100644 --- a/include/trace/hooks/mm.h +++ b/include/trace/hooks/mm.h @@ -99,6 +99,15 @@ DECLARE_HOOK(android_vh_madvise_swapin_walk_pmd_entry, DECLARE_HOOK(android_vh_process_madvise_end, TP_PROTO(int behavior, ssize_t *ret), TP_ARGS(behavior, ret)); +DECLARE_RESTRICTED_HOOK(android_rvh_madvise_pageout_begin, + TP_PROTO(void **private), + TP_ARGS(private), 1); +DECLARE_RESTRICTED_HOOK(android_rvh_madvise_pageout_end, + TP_PROTO(void *private, struct list_head *folio_list), + TP_ARGS(private, folio_list), 1); +DECLARE_RESTRICTED_HOOK(android_rvh_reclaim_folio_list, + TP_PROTO(struct list_head *folio_list, void *private), + TP_ARGS(folio_list, private), 1); DECLARE_HOOK(android_vh_smaps_pte_entry, TP_PROTO(swp_entry_t entry, unsigned long *writeback, unsigned long *same, unsigned long *huge), diff --git a/mm/madvise.c b/mm/madvise.c index fa716030ba1d..d084cea48ffe 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -42,6 +42,7 @@ struct madvise_walk_private { struct mmu_gather *tlb; bool pageout; + void *private; }; /* @@ -426,7 +427,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, huge_unlock: spin_unlock(ptl); if (pageout) - reclaim_pages(&page_list); + __reclaim_pages(&page_list, private->private); return 0; } @@ -534,7 +535,7 @@ regular_page: arch_leave_lazy_mmu_mode(); pte_unmap_unlock(orig_pte, ptl); if (pageout) - reclaim_pages(&page_list); + __reclaim_pages(&page_list, private->private); cond_resched(); return 0; @@ -591,10 +592,17 @@ static void madvise_pageout_page_range(struct mmu_gather *tlb, .pageout = true, .tlb = tlb, }; + LIST_HEAD(folio_list); + + trace_android_rvh_madvise_pageout_begin(&walk_private.private); tlb_start_vma(tlb, vma); walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, &walk_private); tlb_end_vma(tlb, vma); + + trace_android_rvh_madvise_pageout_end(walk_private.private, &folio_list); + if (!list_empty(&folio_list)) + reclaim_pages(&folio_list); } static long madvise_pageout(struct vm_area_struct *vma, diff --git a/mm/vmscan.c b/mm/vmscan.c index 29ae2be21936..6061aa4fb1b6 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -73,6 +73,9 @@ #undef CREATE_TRACE_POINTS #include +#undef CREATE_TRACE_POINTS +#include + EXPORT_TRACEPOINT_SYMBOL_GPL(mm_vmscan_direct_reclaim_begin); EXPORT_TRACEPOINT_SYMBOL_GPL(mm_vmscan_direct_reclaim_end); EXPORT_TRACEPOINT_SYMBOL_GPL(mm_vmscan_kswapd_wake); @@ -2717,7 +2720,8 @@ static void shrink_active_list(unsigned long nr_to_scan, } static unsigned int reclaim_folio_list(struct list_head *folio_list, - struct pglist_data *pgdat) + struct pglist_data *pgdat, + void *private) { struct reclaim_stat dummy_stat; unsigned int nr_reclaimed; @@ -2731,16 +2735,20 @@ static unsigned int reclaim_folio_list(struct list_head *folio_list, }; nr_reclaimed = shrink_folio_list(folio_list, pgdat, &sc, &dummy_stat, false); - while (!list_empty(folio_list)) { - folio = lru_to_folio(folio_list); - list_del(&folio->lru); - folio_putback_lru(folio); + if (private) { + trace_android_rvh_reclaim_folio_list(folio_list, private); + } else { + while (!list_empty(folio_list)) { + folio = lru_to_folio(folio_list); + list_del(&folio->lru); + folio_putback_lru(folio); + } } return nr_reclaimed; } -unsigned long reclaim_pages(struct list_head *folio_list) +unsigned long __reclaim_pages(struct list_head *folio_list, void *private) { int nid; unsigned int nr_reclaimed = 0; @@ -2762,17 +2770,22 @@ unsigned long reclaim_pages(struct list_head *folio_list) continue; } - nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid)); + nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid), private); nid = folio_nid(lru_to_folio(folio_list)); } while (!list_empty(folio_list)); - nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid)); + nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid), private); memalloc_noreclaim_restore(noreclaim_flag); return nr_reclaimed; } +unsigned long reclaim_pages(struct list_head *folio_list) +{ + return __reclaim_pages(folio_list, NULL); +} + static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan, struct lruvec *lruvec, struct scan_control *sc) { From a9dca663a7f2d061ef1fae5e3db0e5b4bd516868 Mon Sep 17 00:00:00 2001 From: Minchan Kim Date: Wed, 15 May 2024 23:30:54 +0000 Subject: [PATCH 79/86] ANDROID: Update the ABI symbol list 3 function symbol(s) added 'int __traceiter_android_rvh_madvise_pageout_end(void*, void*, struct list_head*)' 'int __traceiter_android_rvh_madvise_pageout_begin(void*, void**)' 'int __traceiter_android_rvh_reclaim_folio_list(void*, struct list_head*, void*)' 3 variable symbol(s) added 'struct tracepoint __tracepoint_android_rvh_madvise_pageout_end' 'struct tracepoint __tracepoint_android_rvh_madvise_pageout_begin' 'struct tracepoint __tracepoint_android_rvh_reclaim_folio_list' Bug: 326662423 Change-Id: Ie5805e48060f9e91b87c739ad8a01977b38c2f08 Signed-off-by: Minchan Kim --- android/abi_gki_aarch64.stg | 80 +++++++++++++++++++++++++++++++++++ android/abi_gki_aarch64_pixel | 7 +++ 2 files changed, 87 insertions(+) diff --git a/android/abi_gki_aarch64.stg b/android/abi_gki_aarch64.stg index 4a8f15b75f4d..8759582438c9 100644 --- a/android/abi_gki_aarch64.stg +++ b/android/abi_gki_aarch64.stg @@ -323042,6 +323042,13 @@ function { parameter_id: 0x160238dd parameter_id: 0xd41e888f } +function { + id: 0x9b501ddd + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x3e6239e1 + parameter_id: 0x18bd6530 +} function { id: 0x9b506e5c return_type_id: 0x6720d32f @@ -324248,6 +324255,12 @@ function { parameter_id: 0x29c4a9fa parameter_id: 0x052fac00 } +function { + id: 0x9b9d47e0 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x0cbf60eb +} function { id: 0x9b9d9a99 return_type_id: 0x6720d32f @@ -325053,6 +325066,13 @@ function { parameter_id: 0x2e0f9112 parameter_id: 0x6d7f5ff6 } +function { + id: 0x9bc2d779 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x18bd6530 + parameter_id: 0x3e6239e1 +} function { id: 0x9bc302e5 return_type_id: 0x6720d32f @@ -341483,6 +341503,24 @@ elf_symbol { type_id: 0x9bc253cf full_name: "__traceiter_android_rvh_is_cpu_allowed" } +elf_symbol { + id: 0xc6985675 + name: "__traceiter_android_rvh_madvise_pageout_begin" + is_defined: true + symbol_type: FUNCTION + crc: 0x84bd70b4 + type_id: 0x9b9d47e0 + full_name: "__traceiter_android_rvh_madvise_pageout_begin" +} +elf_symbol { + id: 0x15c6d0dd + name: "__traceiter_android_rvh_madvise_pageout_end" + is_defined: true + symbol_type: FUNCTION + crc: 0x007f7315 + type_id: 0x9bc2d779 + full_name: "__traceiter_android_rvh_madvise_pageout_end" +} elf_symbol { id: 0xed2646da name: "__traceiter_android_rvh_meminfo_proc_show" @@ -341618,6 +341656,15 @@ elf_symbol { type_id: 0x9a374ad4 full_name: "__traceiter_android_rvh_psci_tos_resident_on" } +elf_symbol { + id: 0xeb5708b8 + name: "__traceiter_android_rvh_reclaim_folio_list" + is_defined: true + symbol_type: FUNCTION + crc: 0xee799035 + type_id: 0x9b501ddd + full_name: "__traceiter_android_rvh_reclaim_folio_list" +} elf_symbol { id: 0x13f466b7 name: "__traceiter_android_rvh_refrigerator" @@ -345776,6 +345823,24 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_rvh_is_cpu_allowed" } +elf_symbol { + id: 0xf35a6597 + name: "__tracepoint_android_rvh_madvise_pageout_begin" + is_defined: true + symbol_type: OBJECT + crc: 0xcb97142e + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_rvh_madvise_pageout_begin" +} +elf_symbol { + id: 0xa667d2d3 + name: "__tracepoint_android_rvh_madvise_pageout_end" + is_defined: true + symbol_type: OBJECT + crc: 0xcec8d47b + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_rvh_madvise_pageout_end" +} elf_symbol { id: 0x2252c1f0 name: "__tracepoint_android_rvh_meminfo_proc_show" @@ -345911,6 +345976,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_rvh_psci_tos_resident_on" } +elf_symbol { + id: 0xeda23c8e + name: "__tracepoint_android_rvh_reclaim_folio_list" + is_defined: true + symbol_type: OBJECT + crc: 0x9986a700 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_rvh_reclaim_folio_list" +} elf_symbol { id: 0x3b6248c1 name: "__tracepoint_android_rvh_refrigerator" @@ -408578,6 +408652,8 @@ interface { symbol_id: 0xfc83b254 symbol_id: 0x7e99bc71 symbol_id: 0x7c212080 + symbol_id: 0xc6985675 + symbol_id: 0x15c6d0dd symbol_id: 0xed2646da symbol_id: 0x1ea5d323 symbol_id: 0xc1a482d8 @@ -408593,6 +408669,7 @@ interface { symbol_id: 0xeccbc3c1 symbol_id: 0xbd2d27dc symbol_id: 0xe41b0c78 + symbol_id: 0xeb5708b8 symbol_id: 0x13f466b7 symbol_id: 0xc191b2f6 symbol_id: 0xe3e24295 @@ -409055,6 +409132,8 @@ interface { symbol_id: 0x4fd98142 symbol_id: 0x695c4baf symbol_id: 0x99d57c12 + symbol_id: 0xf35a6597 + symbol_id: 0xa667d2d3 symbol_id: 0x2252c1f0 symbol_id: 0x3cb4db49 symbol_id: 0x3a6f3fb2 @@ -409070,6 +409149,7 @@ interface { symbol_id: 0x50605d97 symbol_id: 0xa5e991f6 symbol_id: 0x261c997e + symbol_id: 0xeda23c8e symbol_id: 0x3b6248c1 symbol_id: 0x5ac6bcbc symbol_id: 0x18bac297 diff --git a/android/abi_gki_aarch64_pixel b/android/abi_gki_aarch64_pixel index d38c701bef44..8d58030fc3f8 100644 --- a/android/abi_gki_aarch64_pixel +++ b/android/abi_gki_aarch64_pixel @@ -2338,11 +2338,14 @@ __traceiter_android_rvh_iommu_limit_align_shift __traceiter_android_rvh_irqs_disable __traceiter_android_rvh_irqs_enable + __traceiter_android_rvh_madvise_pageout_begin + __traceiter_android_rvh_madvise_pageout_end __traceiter_android_rvh_meminfo_proc_show __traceiter_android_rvh_post_init_entity_util_avg __traceiter_android_rvh_preempt_disable __traceiter_android_rvh_preempt_enable __traceiter_android_rvh_prepare_prio_fork + __traceiter_android_rvh_reclaim_folio_list __traceiter_android_rvh_remove_entity_load_avg __traceiter_android_rvh_rtmutex_prepare_setprio __traceiter_android_rvh_sched_newidle_balance @@ -2462,11 +2465,14 @@ __tracepoint_android_rvh_iommu_limit_align_shift __tracepoint_android_rvh_irqs_disable __tracepoint_android_rvh_irqs_enable + __tracepoint_android_rvh_madvise_pageout_begin + __tracepoint_android_rvh_madvise_pageout_end __tracepoint_android_rvh_meminfo_proc_show __tracepoint_android_rvh_post_init_entity_util_avg __tracepoint_android_rvh_preempt_disable __tracepoint_android_rvh_preempt_enable __tracepoint_android_rvh_prepare_prio_fork + __tracepoint_android_rvh_reclaim_folio_list __tracepoint_android_rvh_remove_entity_load_avg __tracepoint_android_rvh_rtmutex_prepare_setprio __tracepoint_android_rvh_sched_newidle_balance @@ -2795,6 +2801,7 @@ wait_for_completion_interruptible_timeout wait_for_completion_killable wait_for_completion_timeout + wait_on_page_writeback wait_woken __wake_up __wake_up_locked From 51c421385e7a5a006c5ba80bfdfaaf60edb5a2c6 Mon Sep 17 00:00:00 2001 From: Srinivasarao Pathipati Date: Fri, 17 May 2024 13:43:02 +0530 Subject: [PATCH 80/86] ANDROID: abi_gki_aarch64_qcom: Update symbol list The commit 893597cbabfb ("r8152: add USB device driver for config selection"), uses usb register/degister symbols. Add them to the symbol list. usb_deregister_device_driver usb_register_device_driver Bug: 341211984 Change-Id: Ifcf37d5630891c5963123af09843b00395f9eff6 Signed-off-by: Srinivasarao Pathipati --- android/abi_gki_aarch64.stg | 148 +++++++++++++++++++++++++++++++++++ android/abi_gki_aarch64_qcom | 2 + 2 files changed, 150 insertions(+) diff --git a/android/abi_gki_aarch64.stg b/android/abi_gki_aarch64.stg index 8759582438c9..ea783905526b 100644 --- a/android/abi_gki_aarch64.stg +++ b/android/abi_gki_aarch64.stg @@ -6048,6 +6048,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x13c36ed4 } +pointer_reference { + id: 0x0e61e3bf + kind: POINTER + pointee_type_id: 0x13c76860 +} pointer_reference { id: 0x0e627592 kind: POINTER @@ -8943,6 +8948,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x7ebccaf7 } +pointer_reference { + id: 0x1549bcfb + kind: POINTER + pointee_type_id: 0x7f661570 +} pointer_reference { id: 0x154aa8e2 kind: POINTER @@ -17018,6 +17028,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x9edc3d1c } +pointer_reference { + id: 0x2d27cf10 + kind: POINTER + pointee_type_id: 0x9edfdadc +} pointer_reference { id: 0x2d28e8bc kind: POINTER @@ -18618,6 +18633,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0x9d1fcda3 } +pointer_reference { + id: 0x2dd7d891 + kind: POINTER + pointee_type_id: 0x9d1f84da +} pointer_reference { id: 0x2dd8ba7a kind: POINTER @@ -25368,6 +25388,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0xfdf9d6ed } +pointer_reference { + id: 0x35eee11a + kind: POINTER + pointee_type_id: 0xfdfb62f7 +} pointer_reference { id: 0x35ef2eea kind: POINTER @@ -75778,6 +75803,12 @@ member { type_id: 0x06433852 offset: 192 } +member { + id: 0xf9546acb + name: "dev_groups" + type_id: 0x06433852 + offset: 384 +} member { id: 0xf9546bf2 name: "dev_groups" @@ -78204,6 +78235,12 @@ member { type_id: 0x0e996219 offset: 512 } +member { + id: 0x8da5d9e5 + name: "disconnect" + type_id: 0x0e61e3bf + offset: 192 +} member { id: 0x8da69b4b name: "disconnect" @@ -80806,6 +80843,12 @@ member { type_id: 0x3c6b8341 offset: 896 } +member { + id: 0x51e20c8a + name: "drvwrap" + type_id: 0x3c6b8341 + offset: 448 +} member { id: 0xb5b01443 name: "dsa" @@ -96408,6 +96451,13 @@ member { type_id: 0x6d7f5ff6 offset: 3104 } +member { + id: 0x1aea9f33 + name: "generic_subclass" + type_id: 0x4585663f + offset: 1985 + bitsize: 1 +} member { id: 0x4a8a7a23 name: "genfs" @@ -105757,6 +105807,12 @@ member { type_id: 0x39947122 offset: 1792 } +member { + id: 0xc4fbc2aa + name: "id_table" + type_id: 0x38040a6c + offset: 1920 +} member { id: 0xc4fbc795 name: "id_table" @@ -123768,6 +123824,12 @@ member { type_id: 0x358a1c52 offset: 320 } +member { + id: 0xe616e079 + name: "match" + type_id: 0x35eee11a + offset: 64 +} member { id: 0xe6179934 name: "match" @@ -155843,6 +155905,12 @@ member { type_id: 0x2d196253 offset: 384 } +member { + id: 0xd77b2d3a + name: "probe" + type_id: 0x2d27cf10 + offset: 128 +} member { id: 0xd77b67e3 name: "probe" @@ -167986,6 +168054,12 @@ member { type_id: 0x2de9e33f offset: 1280 } +member { + id: 0xcab23073 + name: "resume" + type_id: 0x2dd7d891 + offset: 320 +} member { id: 0xcab23d11 name: "resume" @@ -188058,6 +188132,13 @@ member { type_id: 0x295c7202 offset: 40 } +member { + id: 0x26cce749 + name: "supports_autosuspend" + type_id: 0x4585663f + offset: 1984 + bitsize: 1 +} member { id: 0x26cce85e name: "supports_autosuspend" @@ -188217,6 +188298,12 @@ member { type_id: 0x2c1e7e89 offset: 384 } +member { + id: 0xf397014a + name: "suspend" + type_id: 0x2dd7d891 + offset: 256 +} member { id: 0xf39706bf name: "suspend" @@ -266842,6 +266929,25 @@ struct_union { member_id: 0xd8aa6512 } } +struct_union { + id: 0x7f661570 + kind: STRUCT + name: "usb_device_driver" + definition { + bytesize: 256 + member_id: 0x0de57ce8 + member_id: 0xe616e079 + member_id: 0xd77b2d3a + member_id: 0x8da5d9e5 + member_id: 0xf397014a + member_id: 0xcab23073 + member_id: 0xf9546acb + member_id: 0x51e20c8a + member_id: 0xc4fbc2aa + member_id: 0x26cce749 + member_id: 0x1aea9f33 + } +} struct_union { id: 0x59048a3d kind: STRUCT @@ -296426,6 +296532,11 @@ function { parameter_id: 0x14fe2ad8 parameter_id: 0x18bd6530 } +function { + id: 0x15ca3ead + return_type_id: 0x48b5725f + parameter_id: 0x1549bcfb +} function { id: 0x15cafbab return_type_id: 0x48b5725f @@ -319353,6 +319464,12 @@ function { return_type_id: 0x6720d32f parameter_id: 0x15f20052 } +function { + id: 0x98fe1360 + return_type_id: 0x6720d32f + parameter_id: 0x1549bcfb + parameter_id: 0x0b27dc43 +} function { id: 0x98ff2dd7 return_type_id: 0x6720d32f @@ -327988,6 +328105,12 @@ function { parameter_id: 0x03ce74cd parameter_id: 0x1bb6a471 } +function { + id: 0x9d1f84da + return_type_id: 0x6720d32f + parameter_id: 0x0d7ce7cc + parameter_id: 0xf017819f +} function { id: 0x9d1fcda3 return_type_id: 0x6720d32f @@ -337391,6 +337514,11 @@ function { parameter_id: 0x0ca27481 parameter_id: 0x1d44326e } +function { + id: 0xfdfb62f7 + return_type_id: 0x6d7f5ff6 + parameter_id: 0x0d7ce7cc +} function { id: 0xfe11706f return_type_id: 0x4a500dc8 @@ -402261,6 +402389,15 @@ elf_symbol { type_id: 0x18159e1c full_name: "usb_deregister_dev" } +elf_symbol { + id: 0x3be9765e + name: "usb_deregister_device_driver" + is_defined: true + symbol_type: FUNCTION + crc: 0xdadf653f + type_id: 0x15ca3ead + full_name: "usb_deregister_device_driver" +} elf_symbol { id: 0xd75ad246 name: "usb_device_match_id" @@ -403143,6 +403280,15 @@ elf_symbol { type_id: 0x950d2ca0 full_name: "usb_register_dev" } +elf_symbol { + id: 0xcbba001f + name: "usb_register_device_driver" + is_defined: true + symbol_type: FUNCTION + crc: 0xb4a30ad8 + type_id: 0x98fe1360 + full_name: "usb_register_device_driver" +} elf_symbol { id: 0x944deaea name: "usb_register_driver" @@ -415402,6 +415548,7 @@ interface { symbol_id: 0x109adee2 symbol_id: 0x784d1cb4 symbol_id: 0x201afca9 + symbol_id: 0x3be9765e symbol_id: 0xd75ad246 symbol_id: 0xdefa9d60 symbol_id: 0x905187d4 @@ -415500,6 +415647,7 @@ interface { symbol_id: 0xa834737e symbol_id: 0x869e15d7 symbol_id: 0xa60cf3b7 + symbol_id: 0xcbba001f symbol_id: 0x944deaea symbol_id: 0x413dc89c symbol_id: 0xbc035d91 diff --git a/android/abi_gki_aarch64_qcom b/android/abi_gki_aarch64_qcom index aef840a271ba..ba3c1dfb35ff 100644 --- a/android/abi_gki_aarch64_qcom +++ b/android/abi_gki_aarch64_qcom @@ -3816,6 +3816,7 @@ usb_debug_root usb_decode_ctrl usb_deregister + usb_deregister_device_driver usb_device_match_id usb_disabled usb_driver_claim_interface @@ -3851,6 +3852,7 @@ usb_put_dev usb_put_function_instance usb_put_intf + usb_register_device_driver usb_register_driver usb_register_notify usb_remove_phy From 29a00abe4379dab2a34e918f6ce5f991e683ade5 Mon Sep 17 00:00:00 2001 From: liangjlee Date: Fri, 10 May 2024 01:39:10 +0800 Subject: [PATCH 81/86] ANDROID: mm: Add restricted vendor hook in do_read_fault() This patch add a restricted vendor hook in do_read_fault() for tracking which file and offsets are faulted. Bug: 336736235 Change-Id: I425690e58550c4ac44912daa10b5eac0728bfb4e Signed-off-by: liangjlee --- drivers/android/vendor_hooks.c | 1 + include/trace/hooks/mm.h | 4 ++++ mm/memory.c | 2 ++ 3 files changed, 7 insertions(+) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index e8af7b774588..a45022e569e2 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -416,3 +416,4 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_hibernated_do_mem_alloc); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_hibernate_save_cmp_len); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_read_lazy_flag); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_tsk_need_resched_lazy); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_do_read_fault); diff --git a/include/trace/hooks/mm.h b/include/trace/hooks/mm.h index 81ce1268c4fd..e13159239923 100644 --- a/include/trace/hooks/mm.h +++ b/include/trace/hooks/mm.h @@ -25,6 +25,10 @@ DECLARE_RESTRICTED_HOOK(android_rvh_set_gfp_zone_flags, DECLARE_RESTRICTED_HOOK(android_rvh_set_readahead_gfp_mask, TP_PROTO(unsigned int *flags), /* gfp_t *flags */ TP_ARGS(flags), 1); +DECLARE_RESTRICTED_HOOK(android_rvh_do_read_fault, + TP_PROTO(struct file *file, pgoff_t pgoff, + unsigned long *fault_around_bytes), + TP_ARGS(file, pgoff, fault_around_bytes), 1); DECLARE_HOOK(android_vh_dm_bufio_shrink_scan_bypass, TP_PROTO(unsigned long dm_bufio_current_allocated, bool *bypass), TP_ARGS(dm_bufio_current_allocated, bypass)); diff --git a/mm/memory.c b/mm/memory.c index ffbf374a787d..68634e96c175 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4647,6 +4647,8 @@ static vm_fault_t do_read_fault(struct vm_fault *vmf) vm_fault_t ret = 0; trace_android_vh_tune_fault_around_bytes(&fault_around_bytes); + trace_android_rvh_do_read_fault(vmf->vma->vm_file, vmf->pgoff, + &fault_around_bytes); /* * Let's call ->map_pages() first and use ->fault() as fallback From b1e11ffd90765407448616ad42a2aa8d7dd3e4fb Mon Sep 17 00:00:00 2001 From: liangjlee Date: Sat, 18 May 2024 04:35:59 +0800 Subject: [PATCH 82/86] ANDROID: Update the ABI symbol list Adding the following symbols: - __traceiter_android_rvh_do_read_fault - __tracepoint_android_rvh_do_read_fault Bug: 336873696 Change-Id: I7ff2b064942826dcadc949595c9d7df917123986 Signed-off-by: liangjlee --- android/abi_gki_aarch64.stg | 28 ++++++++++++++++++++++++++++ android/abi_gki_aarch64_pixel | 2 ++ 2 files changed, 30 insertions(+) diff --git a/android/abi_gki_aarch64.stg b/android/abi_gki_aarch64.stg index ea783905526b..9388238c1629 100644 --- a/android/abi_gki_aarch64.stg +++ b/android/abi_gki_aarch64.stg @@ -325127,6 +325127,14 @@ function { parameter_id: 0xaad76fc1 parameter_id: 0x3e10b518 } +function { + id: 0x9bc0d7a4 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0x18ea6ae3 + parameter_id: 0x33756485 + parameter_id: 0x064d6086 +} function { id: 0x9bc0fd25 return_type_id: 0x6720d32f @@ -341397,6 +341405,15 @@ elf_symbol { type_id: 0x9ba7ef87 full_name: "__traceiter_android_rvh_do_el1_fpac" } +elf_symbol { + id: 0x1151c5b7 + name: "__traceiter_android_rvh_do_read_fault" + is_defined: true + symbol_type: FUNCTION + crc: 0x221751d1 + type_id: 0x9bc0d7a4 + full_name: "__traceiter_android_rvh_do_read_fault" +} elf_symbol { id: 0x81449840 name: "__traceiter_android_rvh_do_sched_yield" @@ -345717,6 +345734,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_rvh_do_el1_fpac" } +elf_symbol { + id: 0xf51681a1 + name: "__tracepoint_android_rvh_do_read_fault" + is_defined: true + symbol_type: OBJECT + crc: 0x7c957a3e + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_rvh_do_read_fault" +} elf_symbol { id: 0x411e7a56 name: "__tracepoint_android_rvh_do_sched_yield" @@ -408772,6 +408798,7 @@ interface { symbol_id: 0xa3b41867 symbol_id: 0x40034e11 symbol_id: 0x6e62463e + symbol_id: 0x1151c5b7 symbol_id: 0x81449840 symbol_id: 0x611114e0 symbol_id: 0x76eb11ee @@ -409252,6 +409279,7 @@ interface { symbol_id: 0x11f903b1 symbol_id: 0x25e320a3 symbol_id: 0xcd81b300 + symbol_id: 0xf51681a1 symbol_id: 0x411e7a56 symbol_id: 0xcfcbb846 symbol_id: 0x1d7748c4 diff --git a/android/abi_gki_aarch64_pixel b/android/abi_gki_aarch64_pixel index 8d58030fc3f8..0d1b0c47954a 100644 --- a/android/abi_gki_aarch64_pixel +++ b/android/abi_gki_aarch64_pixel @@ -2330,6 +2330,7 @@ __traceiter_android_rvh_dequeue_task __traceiter_android_rvh_dequeue_task_fair __traceiter_android_rvh_detach_entity_load_avg + __traceiter_android_rvh_do_read_fault __traceiter_android_rvh_enqueue_task __traceiter_android_rvh_enqueue_task_fair __traceiter_android_rvh_find_lowest_rq @@ -2457,6 +2458,7 @@ __tracepoint_android_rvh_dequeue_task __tracepoint_android_rvh_dequeue_task_fair __tracepoint_android_rvh_detach_entity_load_avg + __tracepoint_android_rvh_do_read_fault __tracepoint_android_rvh_enqueue_task __tracepoint_android_rvh_enqueue_task_fair __tracepoint_android_rvh_find_lowest_rq From 4ed706c20a76f36b919147910a4df9142f6ab0f9 Mon Sep 17 00:00:00 2001 From: Qais Yousef Date: Mon, 1 Apr 2024 03:04:00 +0100 Subject: [PATCH 83/86] FROMLIST: sched/pi: Reweight fair_policy() tasks when inheriting prio For fair tasks inheriting the priority (nice) without reweighting is a NOP as the task's share won't change. This is visible when running with PTHREAD_PRIO_INHERIT where fair tasks with low priority values are susceptible to starvation leading to PI like impact on lock contention. The logic in rt_mutex will reset these low priority fair tasks into nice 0, but without the additional reweight operation to actually update the weights, it doesn't have the desired impact of boosting them to allow them to run sooner/longer to release the lock. Apply the reweight for fair_policy() tasks to achieve the desired boost for those low nice values tasks. Note that boost here means resetting their nice to 0; as this is what the current logic does for fair tasks. We need to re-instate ordering fair tasks by their priority order on the waiter tree to ensure we inherit the top_waiter properly. Handling of idle_policy() requires more code refactoring and is not handled yet. idle_policy() are treated specially and only run when the CPU is idle and get a hardcoded low weight value. Changing weights won't be enough without a promotion first to SCHED_OTHER. Tested with a test program that creates three threads. 1. main thread that spawns high prio and low prio task and busy loops 2. low priority thread that holds a pthread_mutex() with PTHREAD_PRIO_INHERIT protocol. Runs at nice +10. Busy loops after holding the lock. 3. high priority thread that holds a pthread_mutex() with PTHREADPTHREAD_PRIO_INHERIT, but made to start after the low priority thread. Runs at nice 0. Should remain blocked by the low priority thread. All tasks are pinned to CPU0. Without the patch I can see the low priority thread running only for ~10% of the time which is what expected without it being boosted. With the patch the low priority thread runs for ~50% which is what expected if it gets boosted to nice 0. I modified the test program logic afterwards to ensure that after releasing the lock the low priority thread goes back to running for 10% of the time, and it does. Bug: 263876335 Link: https://lore.kernel.org/lkml/20240514160711.hpdg64grdwc43ux7@airbuntu/ Reported-by: Yabin Cui Signed-off-by: Qais Yousef [Fix trivial conflict with vendor hook] Signed-off-by: Qais Yousef Change-Id: Ia954ee528495b5cf5c3a2157c68b4a757cef1f83 (cherry picked from commit 23ac35ed8fc6220e4e498a21d22a9dbe67e7da9b) Signed-off-by: Qais Yousef --- kernel/locking/rtmutex.c | 6 +----- kernel/sched/core.c | 4 +++- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c index 8207fdade4a8..958eba094274 100644 --- a/kernel/locking/rtmutex.c +++ b/kernel/locking/rtmutex.c @@ -326,17 +326,13 @@ static __always_inline bool unlock_rt_mutex_safe(struct rt_mutex_base *lock, static __always_inline int __waiter_prio(struct task_struct *task) { - int prio = task->prio; int waiter_prio = 0; trace_android_vh_rtmutex_waiter_prio(task, &waiter_prio); if (waiter_prio > 0) return waiter_prio; - if (!rt_prio(prio)) - return DEFAULT_PRIO; - - return prio; + return task->prio; } static __always_inline void diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 20080b633588..1e5dc0964c7b 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -7196,8 +7196,10 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) } else { if (dl_prio(oldprio)) p->dl.pi_se = &p->dl; - if (rt_prio(oldprio)) + else if (rt_prio(oldprio)) p->rt.timeout = 0; + else if (!task_has_idle_policy(p)) + reweight_task(p, prio - MAX_RT_PRIO); } __setscheduler_prio(p, prio); From bd552fcbbd0fb7ecd8c9c1eed047ea1b2319a6d0 Mon Sep 17 00:00:00 2001 From: Kever Yang Date: Fri, 17 May 2024 17:15:54 +0800 Subject: [PATCH 84/86] ANDROID: GKI: Update rockchip symbols to add iova APIs INFO: 2 function symbol(s) added 'struct iova* alloc_iova(struct iova_domain*, unsigned long, unsigned long, bool)' 'void free_iova(struct iova_domain*, unsigned long)' Bug: 300024866 Change-Id: Iccdadf2b516343411871f1df0f46299af9b51c97 Signed-off-by: Kever Yang --- android/abi_gki_aarch64.stg | 34 ++++++++++++++++++++++++++++++++ android/abi_gki_aarch64_rockchip | 6 ++++-- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/android/abi_gki_aarch64.stg b/android/abi_gki_aarch64.stg index 9388238c1629..07031ed590d9 100644 --- a/android/abi_gki_aarch64.stg +++ b/android/abi_gki_aarch64.stg @@ -303391,6 +303391,12 @@ function { parameter_id: 0x3ea2e6c1 parameter_id: 0x2124c9f1 } +function { + id: 0x1fb4bf8e + return_type_id: 0x48b5725f + parameter_id: 0x3f84ee3c + parameter_id: 0x33756485 +} function { id: 0x1fb5d3a2 return_type_id: 0x48b5725f @@ -305600,6 +305606,14 @@ function { return_type_id: 0x18bd6530 parameter_id: 0x4585663f } +function { + id: 0x4812e6a9 + return_type_id: 0x35304fb1 + parameter_id: 0x3f84ee3c + parameter_id: 0x33756485 + parameter_id: 0x33756485 + parameter_id: 0x6d7f5ff6 +} function { id: 0x48135354 return_type_id: 0x35304fb1 @@ -351008,6 +351022,15 @@ elf_symbol { type_id: 0xca4b711f full_name: "alloc_io_pgtable_ops" } +elf_symbol { + id: 0x82a5f160 + name: "alloc_iova" + is_defined: true + symbol_type: FUNCTION + crc: 0xfd3617ee + type_id: 0x4812e6a9 + full_name: "alloc_iova" +} elf_symbol { id: 0x9cff6ebf name: "alloc_iova_fast" @@ -368262,6 +368285,15 @@ elf_symbol { type_id: 0x165a4073 full_name: "free_io_pgtable_ops" } +elf_symbol { + id: 0x25649749 + name: "free_iova" + is_defined: true + symbol_type: FUNCTION + crc: 0x42e921d3 + type_id: 0x1fb4bf8e + full_name: "free_iova" +} elf_symbol { id: 0x998ad938 name: "free_iova_fast" @@ -409865,6 +409897,7 @@ interface { symbol_id: 0xc8c5af1c symbol_id: 0xdeecf7a4 symbol_id: 0xeba70df6 + symbol_id: 0x82a5f160 symbol_id: 0x9cff6ebf symbol_id: 0xcf6df527 symbol_id: 0x00374dfe @@ -411781,6 +411814,7 @@ interface { symbol_id: 0x0513219e symbol_id: 0x1b00ff86 symbol_id: 0x9b815cdc + symbol_id: 0x25649749 symbol_id: 0x998ad938 symbol_id: 0x27e49ce0 symbol_id: 0xa0e729a5 diff --git a/android/abi_gki_aarch64_rockchip b/android/abi_gki_aarch64_rockchip index 87a61c986242..1679dba4127d 100644 --- a/android/abi_gki_aarch64_rockchip +++ b/android/abi_gki_aarch64_rockchip @@ -267,6 +267,8 @@ devm_usb_get_phy _dev_notice dev_open + dev_pm_domain_attach_by_name + dev_pm_domain_detach dev_pm_opp_clear_config dev_pm_opp_disable dev_pm_opp_find_freq_ceil @@ -2334,12 +2336,12 @@ reserve_iova # required by rknpu.ko - dev_pm_domain_attach_by_name - dev_pm_domain_detach + alloc_iova drm_gem_handle_delete drm_gem_mmap_obj drm_gem_prime_export drm_gem_prime_import_dev + free_iova of_dma_configure_id set_user_nice vmf_insert_mixed From 34a15d350726c5770943d8288bf0c5e2d8859e94 Mon Sep 17 00:00:00 2001 From: Krishna Kurapati Date: Mon, 5 Feb 2024 13:16:50 +0530 Subject: [PATCH 85/86] UPSTREAM: usb: gadget: ncm: Avoid dropping datagrams of properly parsed NTBs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It is observed sometimes when tethering is used over NCM with Windows 11 as host, at some instances, the gadget_giveback has one byte appended at the end of a proper NTB. When the NTB is parsed, unwrap call looks for any leftover bytes in SKB provided by u_ether and if there are any pending bytes, it treats them as a separate NTB and parses it. But in case the second NTB (as per unwrap call) is faulty/corrupt, all the datagrams that were parsed properly in the first NTB and saved in rx_list are dropped. Adding a few custom traces showed the following: [002] d..1 7828.532866: dwc3_gadget_giveback: ep1out: req 000000003868811a length 1025/16384 zsI ==> 0 [002] d..1 7828.532867: ncm_unwrap_ntb: K: ncm_unwrap_ntb toprocess: 1025 [002] d..1 7828.532867: ncm_unwrap_ntb: K: ncm_unwrap_ntb nth: 1751999342 [002] d..1 7828.532868: ncm_unwrap_ntb: K: ncm_unwrap_ntb seq: 0xce67 [002] d..1 7828.532868: ncm_unwrap_ntb: K: ncm_unwrap_ntb blk_len: 0x400 [002] d..1 7828.532868: ncm_unwrap_ntb: K: ncm_unwrap_ntb ndp_len: 0x10 [002] d..1 7828.532869: ncm_unwrap_ntb: K: Parsed NTB with 1 frames In this case, the giveback is of 1025 bytes and block length is 1024. The rest 1 byte (which is 0x00) won't be parsed resulting in drop of all datagrams in rx_list. Same is case with packets of size 2048: [002] d..1 7828.557948: dwc3_gadget_giveback: ep1out: req 0000000011dfd96e length 2049/16384 zsI ==> 0 [002] d..1 7828.557949: ncm_unwrap_ntb: K: ncm_unwrap_ntb nth: 1751999342 [002] d..1 7828.557950: ncm_unwrap_ntb: K: ncm_unwrap_ntb blk_len: 0x800 Lecroy shows one byte coming in extra confirming that the byte is coming in from PC: Transfer 2959 - Bytes Transferred(1025) Timestamp((18.524 843 590) - Transaction 8391 - Data(1025 bytes) Timestamp(18.524 843 590) --- Packet 4063861 Data(1024 bytes) Duration(2.117us) Idle(14.700ns) Timestamp(18.524 843 590) --- Packet 4063863 Data(1 byte) Duration(66.160ns) Time(282.000ns) Timestamp(18.524 845 722) According to Windows driver, no ZLP is needed if wBlockLength is non-zero, because the non-zero wBlockLength has already told the function side the size of transfer to be expected. However, there are in-market NCM devices that rely on ZLP as long as the wBlockLength is multiple of wMaxPacketSize. To deal with such devices, it pads an extra 0 at end so the transfer is no longer multiple of wMaxPacketSize. Cc: Fixes: 9f6ce4240a2b ("usb: gadget: f_ncm.c added") Signed-off-by: Krishna Kurapati Reviewed-by: Maciej Żenczykowski Link: https://lore.kernel.org/r/20240205074650.200304-1-quic_kriskura@quicinc.com Signed-off-by: Greg Kroah-Hartman (cherry picked from commit 76c51146820c5dac629f21deafab0a7039bc3ccd https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git master) Bug: 320608613 Change-Id: Iee598bcbede12582235fca38a0c9f50f3b7375c5 Signed-off-by: Krishna Kurapati (cherry picked from commit c344c3ebe3fead1ed0c12bd686be083748011342) --- drivers/usb/gadget/function/f_ncm.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/usb/gadget/function/f_ncm.c b/drivers/usb/gadget/function/f_ncm.c index bbb6ff6b11aa..5e78fcc63e4d 100644 --- a/drivers/usb/gadget/function/f_ncm.c +++ b/drivers/usb/gadget/function/f_ncm.c @@ -1340,7 +1340,15 @@ parse_ntb: "Parsed NTB with %d frames\n", dgram_counter); to_process -= block_len; - if (to_process != 0) { + + /* + * Windows NCM driver avoids USB ZLPs by adding a 1-byte + * zero pad as needed. + */ + if (to_process == 1 && + (*(unsigned char *)(ntb_ptr + block_len) == 0x00)) { + to_process--; + } else if (to_process > 0) { ntb_ptr = (unsigned char *)(ntb_ptr + block_len); goto parse_ntb; } From 0a5aada71cc9872687dce7afde9824de85455509 Mon Sep 17 00:00:00 2001 From: Seiya Wang Date: Tue, 21 May 2024 11:29:52 +0800 Subject: [PATCH 86/86] ANDROID: GKI: Update symbol list for mtk 3 function symbol(s) added 'int dev_pm_opp_register_notifier(struct device*, struct notifier_block*)' 'int dev_pm_opp_unregister_notifier(struct device*, struct notifier_block*)' 'int snd_soc_suspend(struct device*)' Bug: 341821144 Change-Id: Iafcfaede99a35e10d9162e0298a7e3feb43cec73 Signed-off-by: Seiya Wang --- android/abi_gki_aarch64.stg | 30 ++++++++++++++++++++++++++++++ android/abi_gki_aarch64_mtk | 3 +++ 2 files changed, 33 insertions(+) diff --git a/android/abi_gki_aarch64.stg b/android/abi_gki_aarch64.stg index 07031ed590d9..2072d7f55eaa 100644 --- a/android/abi_gki_aarch64.stg +++ b/android/abi_gki_aarch64.stg @@ -358509,6 +358509,15 @@ elf_symbol { type_id: 0x196c5dcc full_name: "dev_pm_opp_put_opp_table" } +elf_symbol { + id: 0x1312b8f1 + name: "dev_pm_opp_register_notifier" + is_defined: true + symbol_type: FUNCTION + crc: 0xe13e98dc + type_id: 0x9d7807e5 + full_name: "dev_pm_opp_register_notifier" +} elf_symbol { id: 0xe6f275cc name: "dev_pm_opp_remove" @@ -358563,6 +358572,15 @@ elf_symbol { type_id: 0x9dd2e674 full_name: "dev_pm_opp_set_sharing_cpus" } +elf_symbol { + id: 0x0e308173 + name: "dev_pm_opp_unregister_notifier" + is_defined: true + symbol_type: FUNCTION + crc: 0x792ecfff + type_id: 0x9d7807e5 + full_name: "dev_pm_opp_unregister_notifier" +} elf_symbol { id: 0x72797495 name: "dev_pm_qos_add_notifier" @@ -396045,6 +396063,15 @@ elf_symbol { type_id: 0x98508ee0 full_name: "snd_soc_set_runtime_hwparams" } +elf_symbol { + id: 0xcea952e4 + name: "snd_soc_suspend" + is_defined: true + symbol_type: FUNCTION + crc: 0x828d799f + type_id: 0x9d16dd74 + full_name: "snd_soc_suspend" +} elf_symbol { id: 0x93803c84 name: "snd_soc_tplg_component_load" @@ -410730,12 +410757,14 @@ interface { symbol_id: 0x706732e6 symbol_id: 0x52c8a093 symbol_id: 0x4624f3d3 + symbol_id: 0x1312b8f1 symbol_id: 0xe6f275cc symbol_id: 0x90259097 symbol_id: 0x4b0b944c symbol_id: 0x9b2f909d symbol_id: 0xc927ca5d symbol_id: 0xe3ab91c3 + symbol_id: 0x0e308173 symbol_id: 0x72797495 symbol_id: 0x8b918cf2 symbol_id: 0x0343af6f @@ -414898,6 +414927,7 @@ interface { symbol_id: 0x2673800e symbol_id: 0x04af389d symbol_id: 0x685b01fa + symbol_id: 0xcea952e4 symbol_id: 0x93803c84 symbol_id: 0x7c7f5167 symbol_id: 0xe5cef1d3 diff --git a/android/abi_gki_aarch64_mtk b/android/abi_gki_aarch64_mtk index a6f94dd75484..b836bc579e1a 100644 --- a/android/abi_gki_aarch64_mtk +++ b/android/abi_gki_aarch64_mtk @@ -589,8 +589,10 @@ dev_pm_opp_of_remove_table dev_pm_opp_put dev_pm_opp_put_opp_table + dev_pm_opp_register_notifier dev_pm_opp_remove_all_dynamic dev_pm_opp_set_config + dev_pm_opp_unregister_notifier dev_pm_qos_add_notifier dev_pm_qos_remove_notifier dev_pm_qos_update_request @@ -2517,6 +2519,7 @@ snd_soc_resume snd_soc_rtdcom_lookup snd_soc_set_runtime_hwparams + snd_soc_suspend snd_soc_unregister_component snd_vendor_set_ops snprintf