diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index a7e3f39d11a2..a7309321f02c 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -6705,6 +6705,9 @@ usb-storage.delay_use= [UMS] The delay in seconds before a new device is scanned for Logical Units (default 1). + Optionally the delay in milliseconds if the value has + suffix with "ms". + Example: delay_use=2567ms usb-storage.quirks= [UMS] A list of quirks entries to supplement or diff --git a/android/abi_gki_aarch64.stg b/android/abi_gki_aarch64.stg index b50c2f469354..f149d8523964 100644 --- a/android/abi_gki_aarch64.stg +++ b/android/abi_gki_aarch64.stg @@ -24638,6 +24638,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0xe5b47ef3 } +pointer_reference { + id: 0x340055b2 + kind: POINTER + pointee_type_id: 0xfa41b054 +} pointer_reference { id: 0x34016e82 kind: POINTER @@ -26753,6 +26758,11 @@ pointer_reference { kind: POINTER pointee_type_id: 0xca69055f } +pointer_reference { + id: 0x380af085 + kind: POINTER + pointee_type_id: 0xca6b248b +} pointer_reference { id: 0x380c33d1 kind: POINTER @@ -35938,6 +35948,11 @@ qualified { qualifier: CONST qualified_type_id: 0x98e32fbd } +qualified { + id: 0xfa41b054 + qualifier: CONST + qualified_type_id: 0x994177db +} qualified { id: 0xfa455c97 qualifier: CONST @@ -41974,6 +41989,29 @@ member { type_id: 0x92233392 offset: 1024 } +member { + id: 0x9583c91c + name: "HB0" + type_id: 0x295c7202 +} +member { + id: 0xd444664c + name: "HB1" + type_id: 0x295c7202 + offset: 8 +} +member { + id: 0x1603fa52 + name: "HB2" + type_id: 0x295c7202 + offset: 16 +} +member { + id: 0x56c4b67f + name: "HB3" + type_id: 0x295c7202 + offset: 24 +} member { id: 0xc31a401f name: "InOctetsDecrypted" @@ -73029,6 +73067,12 @@ member { type_id: 0x295c7202 offset: 112 } +member { + id: 0xd75b166e + name: "db" + type_id: 0x5e9b9471 + offset: 32 +} member { id: 0xf540eeb1 name: "db_off" @@ -176291,6 +176335,11 @@ member { type_id: 0xc9082b19 offset: 10176 } +member { + id: 0x0fed75b2 + name: "sdp_header" + type_id: 0x90206a5a +} member { id: 0x682990f8 name: "sdp_max" @@ -230244,6 +230293,28 @@ struct_union { member_id: 0x24d1edc6 } } +struct_union { + id: 0xca6b248b + kind: STRUCT + name: "dp_sdp" + definition { + bytesize: 36 + member_id: 0x0fed75b2 + member_id: 0xd75b166e + } +} +struct_union { + id: 0x90206a5a + kind: STRUCT + name: "dp_sdp_header" + definition { + bytesize: 4 + member_id: 0x9583c91c + member_id: 0xd444664c + member_id: 0x1603fa52 + member_id: 0x56c4b67f + } +} struct_union { id: 0x33fed362 kind: STRUCT @@ -301796,6 +301867,13 @@ function { parameter_id: 0x25653b02 parameter_id: 0x35d17e4b } +function { + id: 0x191b8833 + return_type_id: 0xd5cc9c9a + parameter_id: 0x340055b2 + parameter_id: 0x380af085 + parameter_id: 0x295c7202 +} function { id: 0x191d871c return_type_id: 0x48b5725f @@ -312975,6 +313053,12 @@ function { return_type_id: 0x3e10b518 parameter_id: 0x6720d32f } +function { + id: 0x8c02526b + return_type_id: 0x6720d32f + parameter_id: 0x4585663f + parameter_id: 0x38fa32ef +} function { id: 0x8c19f874 return_type_id: 0x6720d32f @@ -322767,6 +322851,17 @@ function { parameter_id: 0x4585663f parameter_id: 0x33756485 } +function { + id: 0x98789491 + return_type_id: 0x6720d32f + parameter_id: 0x18bd6530 + parameter_id: 0xf1a6dfed + parameter_id: 0x4585663f + parameter_id: 0x33756485 + parameter_id: 0x6720d32f + parameter_id: 0x33756485 + parameter_id: 0x11cfee5a +} function { id: 0x98792c3d return_type_id: 0x3e10b518 @@ -335502,6 +335597,13 @@ function { parameter_id: 0x33b77109 parameter_id: 0x3283ded6 } +function { + id: 0x9eb38da8 + return_type_id: 0x6720d32f + parameter_id: 0x0ab9fa4c + parameter_id: 0x6720d32f + parameter_id: 0x064d6086 +} function { id: 0x9eb3dea3 return_type_id: 0x6720d32f @@ -337132,6 +337234,12 @@ function { parameter_id: 0x0d30b9c3 parameter_id: 0x15a30023 } +function { + id: 0x9fe48d4f + return_type_id: 0x6720d32f + parameter_id: 0x0aa6efc8 + parameter_id: 0x33756485 +} function { id: 0x9fe6297b return_type_id: 0x6720d32f @@ -344305,6 +344413,15 @@ elf_symbol { type_id: 0x9e215925 full_name: "__hwspin_lock_timeout" } +elf_symbol { + id: 0xac885058 + name: "__hwspin_trylock" + is_defined: true + symbol_type: FUNCTION + crc: 0xe4d4ab7e + type_id: 0x9eb38da8 + full_name: "__hwspin_trylock" +} elf_symbol { id: 0x09a111a4 name: "__hwspin_unlock" @@ -347213,6 +347330,15 @@ elf_symbol { type_id: 0x9bdbdcc4 full_name: "__traceiter_android_rvh_setscheduler" } +elf_symbol { + id: 0x1228e7e9 + name: "__traceiter_android_rvh_setscheduler_prio" + is_defined: true + symbol_type: FUNCTION + crc: 0x116cab3c + type_id: 0x9bdbdcc4 + full_name: "__traceiter_android_rvh_setscheduler_prio" +} elf_symbol { id: 0x73c83ef4 name: "__traceiter_android_rvh_shmem_get_folio" @@ -348401,6 +348527,15 @@ elf_symbol { type_id: 0x9b2a7922 full_name: "__traceiter_android_vh_do_wp_page" } +elf_symbol { + id: 0xf80eb64b + name: "__traceiter_android_vh_drain_all_pages_bypass" + is_defined: true + symbol_type: FUNCTION + crc: 0xe9b2635d + type_id: 0x98789491 + full_name: "__traceiter_android_vh_drain_all_pages_bypass" +} elf_symbol { id: 0x42312ccc name: "__traceiter_android_vh_dump_throttled_rt_tasks" @@ -349364,6 +349499,15 @@ elf_symbol { type_id: 0x9b26096d full_name: "__traceiter_android_vh_page_should_be_protected" } +elf_symbol { + id: 0x1fc96009 + name: "__traceiter_android_vh_pageset_update" + is_defined: true + symbol_type: FUNCTION + crc: 0xf5c57c7a + type_id: 0x9bb71cb9 + full_name: "__traceiter_android_vh_pageset_update" +} elf_symbol { id: 0x13b0736e name: "__traceiter_android_vh_percpu_rwsem_down_read" @@ -352316,6 +352460,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_rvh_setscheduler" } +elf_symbol { + id: 0x8a4070f7 + name: "__tracepoint_android_rvh_setscheduler_prio" + is_defined: true + symbol_type: OBJECT + crc: 0xa79bc306 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_rvh_setscheduler_prio" +} elf_symbol { id: 0x00b7ed82 name: "__tracepoint_android_rvh_shmem_get_folio" @@ -353504,6 +353657,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_do_wp_page" } +elf_symbol { + id: 0x8405c9a1 + name: "__tracepoint_android_vh_drain_all_pages_bypass" + is_defined: true + symbol_type: OBJECT + crc: 0x786ed430 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_drain_all_pages_bypass" +} elf_symbol { id: 0x988719fa name: "__tracepoint_android_vh_dump_throttled_rt_tasks" @@ -354467,6 +354629,15 @@ elf_symbol { type_id: 0x18ccbd2c full_name: "__tracepoint_android_vh_page_should_be_protected" } +elf_symbol { + id: 0x94cb1cab + name: "__tracepoint_android_vh_pageset_update" + is_defined: true + symbol_type: OBJECT + crc: 0x644520c9 + type_id: 0x18ccbd2c + full_name: "__tracepoint_android_vh_pageset_update" +} elf_symbol { id: 0xa4c454d8 name: "__tracepoint_android_vh_percpu_rwsem_down_read" @@ -378871,6 +379042,15 @@ elf_symbol { type_id: 0x1f9e005b full_name: "hdmi_audio_infoframe_pack" } +elf_symbol { + id: 0x6392ceb7 + name: "hdmi_audio_infoframe_pack_for_dp" + is_defined: true + symbol_type: FUNCTION + crc: 0x7e0b255f + type_id: 0x191b8833 + full_name: "hdmi_audio_infoframe_pack_for_dp" +} elf_symbol { id: 0x0e1d2fa4 name: "hdmi_avi_infoframe_check" @@ -383092,6 +383272,15 @@ elf_symbol { type_id: 0x8d53ba62 full_name: "irq_get_irqchip_state" } +elf_symbol { + id: 0xe9124d83 + name: "irq_get_percpu_devid_partition" + is_defined: true + symbol_type: FUNCTION + crc: 0x6cfe3b6f + type_id: 0x8c02526b + full_name: "irq_get_percpu_devid_partition" +} elf_symbol { id: 0x28fabc56 name: "irq_modify_status" @@ -393100,6 +393289,15 @@ elf_symbol { type_id: 0x107966af full_name: "perf_aux_output_flag" } +elf_symbol { + id: 0x4e76f69f + name: "perf_aux_output_skip" + is_defined: true + symbol_type: FUNCTION + crc: 0x5a8de386 + type_id: 0x9fe48d4f + full_name: "perf_aux_output_skip" +} elf_symbol { id: 0x84bf9f80 name: "perf_event_addr_filters_sync" @@ -408440,6 +408638,15 @@ elf_symbol { type_id: 0x95a406a6 full_name: "thermal_zone_unbind_cooling_device" } +elf_symbol { + id: 0xc20a8a50 + name: "this_cpu_has_cap" + is_defined: true + symbol_type: FUNCTION + crc: 0x46e67a71 + type_id: 0xefc5028b + full_name: "this_cpu_has_cap" +} elf_symbol { id: 0x68e98442 name: "thread_group_cputime_adjusted" @@ -411986,6 +412193,15 @@ elf_symbol { type_id: 0xf38427c4 full_name: "usb_check_bulk_endpoints" } +elf_symbol { + id: 0xcf4d7b06 + name: "usb_check_int_endpoints" + is_defined: true + symbol_type: FUNCTION + crc: 0x18bb04b2 + type_id: 0xf38427c4 + full_name: "usb_check_int_endpoints" +} elf_symbol { id: 0x23a5ab99 name: "usb_choose_configuration" @@ -418640,6 +418856,7 @@ interface { symbol_id: 0x58de7795 symbol_id: 0x7874d435 symbol_id: 0xc1db2428 + symbol_id: 0xac885058 symbol_id: 0x09a111a4 symbol_id: 0x9ff710d8 symbol_id: 0xee9e2392 @@ -418963,6 +419180,7 @@ interface { symbol_id: 0x9b0cc890 symbol_id: 0x559e0725 symbol_id: 0xa01b20ce + symbol_id: 0x1228e7e9 symbol_id: 0x73c83ef4 symbol_id: 0x46515de8 symbol_id: 0x955e6fc1 @@ -419095,6 +419313,7 @@ interface { symbol_id: 0x54bc5972 symbol_id: 0x9dbd7b92 symbol_id: 0x2576f1c7 + symbol_id: 0xf80eb64b symbol_id: 0x42312ccc symbol_id: 0xf432d1c9 symbol_id: 0x02c8f91b @@ -419202,6 +419421,7 @@ interface { symbol_id: 0xacaadcc9 symbol_id: 0x3246acbb symbol_id: 0xb4d5ffdc + symbol_id: 0x1fc96009 symbol_id: 0x13b0736e symbol_id: 0xc72f2012 symbol_id: 0xd14f3adb @@ -419530,6 +419750,7 @@ interface { symbol_id: 0x42fff08e symbol_id: 0x74f29f73 symbol_id: 0xe48123a4 + symbol_id: 0x8a4070f7 symbol_id: 0x00b7ed82 symbol_id: 0xe8cacf26 symbol_id: 0xad588d93 @@ -419662,6 +419883,7 @@ interface { symbol_id: 0xeb9f1c78 symbol_id: 0xe2d7542c symbol_id: 0x15374b6d + symbol_id: 0x8405c9a1 symbol_id: 0x988719fa symbol_id: 0x732a182b symbol_id: 0xe5deb919 @@ -419769,6 +419991,7 @@ interface { symbol_id: 0x20d2ceb3 symbol_id: 0x4a5e6e41 symbol_id: 0x352038ba + symbol_id: 0x94cb1cab symbol_id: 0xa4c454d8 symbol_id: 0x7d42b7c8 symbol_id: 0x3d63616d @@ -422479,6 +422702,7 @@ interface { symbol_id: 0x3163ad8e symbol_id: 0x78d85567 symbol_id: 0xcd9a2048 + symbol_id: 0x6392ceb7 symbol_id: 0x0e1d2fa4 symbol_id: 0x306e3b3d symbol_id: 0x684435da @@ -422948,6 +423172,7 @@ interface { symbol_id: 0x2ed6bfeb symbol_id: 0xa9c80d6c symbol_id: 0xb1d265b1 + symbol_id: 0xe9124d83 symbol_id: 0x28fabc56 symbol_id: 0x68e07680 symbol_id: 0xcd991820 @@ -424060,6 +424285,7 @@ interface { symbol_id: 0xf9e83d36 symbol_id: 0x80bebca0 symbol_id: 0xe207c73b + symbol_id: 0x4e76f69f symbol_id: 0x84bf9f80 symbol_id: 0xea4938d9 symbol_id: 0xdc02a166 @@ -425765,6 +425991,7 @@ interface { symbol_id: 0x66782435 symbol_id: 0x793a755b symbol_id: 0x61ab1273 + symbol_id: 0xc20a8a50 symbol_id: 0x68e98442 symbol_id: 0x1cf36c3c symbol_id: 0x9fc8421c @@ -426159,6 +426386,7 @@ interface { symbol_id: 0x3d66dcb8 symbol_id: 0x1f68a496 symbol_id: 0x12289dad + symbol_id: 0xcf4d7b06 symbol_id: 0x23a5ab99 symbol_id: 0x2a589f64 symbol_id: 0x7da41bc7 diff --git a/android/abi_gki_aarch64_bcmstb b/android/abi_gki_aarch64_bcmstb index e41670aeff26..446862098288 100644 --- a/android/abi_gki_aarch64_bcmstb +++ b/android/abi_gki_aarch64_bcmstb @@ -1604,6 +1604,7 @@ sdhci_set_clock sdhci_set_uhs_signaling sdhci_setup_host + __hwspin_trylock # required by slcan.ko hex_asc_upper diff --git a/android/abi_gki_aarch64_mtk b/android/abi_gki_aarch64_mtk index 26727ae17c50..238637837c11 100644 --- a/android/abi_gki_aarch64_mtk +++ b/android/abi_gki_aarch64_mtk @@ -1155,6 +1155,7 @@ handle_sysrq have_governor_per_policy hci_cmd_sync_status + hdmi_audio_infoframe_pack_for_dp hex2bin hex_asc hex_asc_upper @@ -3229,6 +3230,7 @@ usb_autopm_put_interface usb_autopm_put_interface_async usb_check_bulk_endpoints + usb_check_int_endpoints usb_clear_halt usb_composite_probe usb_composite_unregister diff --git a/android/abi_gki_aarch64_pixel b/android/abi_gki_aarch64_pixel index 8c48602bccf2..4678f40d85ef 100644 --- a/android/abi_gki_aarch64_pixel +++ b/android/abi_gki_aarch64_pixel @@ -1198,6 +1198,7 @@ __ipv6_addr_type __irq_alloc_descs __irq_apply_affinity_hint + irq_check_status_bit irq_create_mapping_affinity irq_create_of_mapping __irq_domain_add @@ -1208,6 +1209,7 @@ irq_domain_xlate_twocell irq_force_affinity irq_get_irq_data + irq_get_percpu_devid_partition irq_modify_status irq_of_parse_and_map __irq_resolve_mapping @@ -1219,6 +1221,7 @@ irq_set_irq_wake irq_to_desc irq_work_queue + irq_work_run irq_work_sync is_vmalloc_addr jiffies @@ -1604,6 +1607,7 @@ param_ops_long param_ops_string param_ops_uint + param_ops_ullong param_ops_ulong param_set_copystring param_set_int @@ -1651,6 +1655,7 @@ perf_aux_output_begin perf_aux_output_end perf_aux_output_flag + perf_aux_output_skip perf_event_addr_filters_sync perf_event_create_kernel_counter perf_event_disable @@ -2337,6 +2342,7 @@ thermal_zone_device_update thermal_zone_get_temp thermal_zone_get_zone_by_name + this_cpu_has_cap thread_group_cputime_adjusted tick_nohz_get_idle_calls_cpu time64_to_tm @@ -2387,6 +2393,7 @@ __traceiter_android_rvh_set_cpus_allowed_by_task __traceiter_android_rvh_set_iowait __traceiter_android_rvh_setscheduler + __traceiter_android_rvh_setscheduler_prio __traceiter_android_rvh_set_task_cpu __traceiter_android_rvh_set_user_nice __traceiter_android_rvh_set_user_nice_locked @@ -2412,6 +2419,7 @@ __traceiter_android_vh_binder_proc_transaction_finish __traceiter_android_vh_binder_restore_priority __traceiter_android_vh_binder_set_priority + __traceiter_android_vh_calculate_totalreserve_pages __traceiter_android_vh_cpu_idle_enter __traceiter_android_vh_cpu_idle_exit __traceiter_android_vh_dump_throttled_rt_tasks @@ -2455,7 +2463,6 @@ __traceiter_android_vh_usb_dev_resume __traceiter_android_vh_use_amu_fie __traceiter_android_vh_vmscan_kswapd_done - __traceiter_android_vh_calculate_totalreserve_pages __traceiter_clock_set_rate __traceiter_cma_alloc_finish __traceiter_cma_alloc_start @@ -2528,6 +2535,7 @@ __tracepoint_android_rvh_set_cpus_allowed_by_task __tracepoint_android_rvh_set_iowait __tracepoint_android_rvh_setscheduler + __tracepoint_android_rvh_setscheduler_prio __tracepoint_android_rvh_set_task_cpu __tracepoint_android_rvh_set_user_nice __tracepoint_android_rvh_set_user_nice_locked @@ -2553,6 +2561,7 @@ __tracepoint_android_vh_binder_proc_transaction_finish __tracepoint_android_vh_binder_restore_priority __tracepoint_android_vh_binder_set_priority + __tracepoint_android_vh_calculate_totalreserve_pages __tracepoint_android_vh_cpu_idle_enter __tracepoint_android_vh_cpu_idle_exit __tracepoint_android_vh_dump_throttled_rt_tasks @@ -2596,7 +2605,6 @@ __tracepoint_android_vh_usb_dev_resume __tracepoint_android_vh_use_amu_fie __tracepoint_android_vh_vmscan_kswapd_done - __tracepoint_android_vh_calculate_totalreserve_pages __tracepoint_clock_set_rate __tracepoint_cma_alloc_finish __tracepoint_cma_alloc_start @@ -2732,7 +2740,9 @@ usb_function_register usb_function_unregister usb_gadget_activate + usb_gadget_connect usb_gadget_deactivate + usb_gadget_disconnect usb_gadget_set_state usb_gstrings_attach usb_hcd_is_primary_hcd diff --git a/android/abi_gki_aarch64_xiaomi b/android/abi_gki_aarch64_xiaomi index da618470dc5f..2ebc76e2121d 100644 --- a/android/abi_gki_aarch64_xiaomi +++ b/android/abi_gki_aarch64_xiaomi @@ -524,3 +524,8 @@ __tracepoint_android_vh_filemap_map_pages_range #required by rtase.ko proc_get_parent_data netdev_stats_to_stats64 + +__traceiter_android_vh_drain_all_pages_bypass +__tracepoint_android_vh_drain_all_pages_bypass +__traceiter_android_vh_pageset_update +__tracepoint_android_vh_pageset_update diff --git a/android/abi_gki_protected_exports_aarch64 b/android/abi_gki_protected_exports_aarch64 index c9f44faef737..c2cc936669e5 100644 --- a/android/abi_gki_protected_exports_aarch64 +++ b/android/abi_gki_protected_exports_aarch64 @@ -351,4 +351,4 @@ wwan_port_txoff wwan_port_txon wwan_register_ops wwan_remove_port -wwan_unregister_ops +wwan_unregister_ops \ No newline at end of file diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index 930b0e6c9462..3544dfcc67a1 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -56,6 +56,7 @@ extern void fpsimd_signal_preserve_current_state(void); extern void fpsimd_preserve_current_state(void); extern void fpsimd_restore_current_state(void); extern void fpsimd_update_current_state(struct user_fpsimd_state const *state); +extern void fpsimd_kvm_prepare(void); extern void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *state, void *sve_state, unsigned int sve_vl, diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 1ad2a045f8e7..3fc67e4b1b60 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -438,6 +438,8 @@ struct kvm_vcpu_arch { /* Values of trap registers for the guest. */ u64 hcr_el2; u64 mdcr_el2; + + /* DO NOT USE: Removed upstream. Kept to not break the KMI. */ u64 cptr_el2; /* Values of trap registers for the host before guest entry. */ @@ -488,7 +490,8 @@ struct kvm_vcpu_arch { struct kvm_guest_debug_arch vcpu_debug_state; struct kvm_guest_debug_arch external_debug_state; - struct user_fpsimd_state *host_fpsimd_state; /* hyp VA */ + /* DO NOT USE: Removed upstream. Kept to not break the KMI. */ + struct user_fpsimd_state *host_fpsimd_state; struct { /* {Break,watch}point registers */ @@ -692,10 +695,6 @@ struct kvm_vcpu_arch { /* pKVM host vcpu state is dirty, needs resync */ #define PKVM_HOST_STATE_DIRTY __vcpu_single_flag(iflags, BIT(7)) -/* SVE enabled for host EL0 */ -#define HOST_SVE_ENABLED __vcpu_single_flag(sflags, BIT(0)) -/* SME enabled for EL0 */ -#define HOST_SME_ENABLED __vcpu_single_flag(sflags, BIT(1)) /* Physical CPU not in supported_cpus */ #define ON_UNSUPPORTED_CPU __vcpu_single_flag(sflags, BIT(2)) /* WFIT instruction trapped */ diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h index 861049a6416f..b0ed038ff4e9 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h @@ -121,6 +121,9 @@ void __noreturn __host_enter(struct kvm_cpu_context *host_ctxt); #ifdef __KVM_NVHE_HYPERVISOR__ struct user_fpsimd_state *get_host_fpsimd_state(struct kvm_vcpu *vcpu); struct kvm_host_sve_state *get_host_sve_state(struct kvm_vcpu *vcpu); +#else +#define get_host_fpsimd_state(vcpu) NULL +#define get_host_sve_state(vcpu) NULL #endif extern u64 kvm_nvhe_sym(id_aa64pfr0_el1_sys_val); diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 3d36e5aa3767..d37fdbde77a6 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -1361,7 +1361,6 @@ static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu, } vcpu_reset_hcr(vcpu); - vcpu->arch.cptr_el2 = CPTR_EL2_DEFAULT; /* * Handle the "start in power-off" case. diff --git a/arch/arm64/kvm/fpsimd.c b/arch/arm64/kvm/fpsimd.c index 453c6a541d87..ad716eadc4c8 100644 --- a/arch/arm64/kvm/fpsimd.c +++ b/arch/arm64/kvm/fpsimd.c @@ -37,8 +37,6 @@ int kvm_arch_vcpu_run_map_fp(struct kvm_vcpu *vcpu) if (ret) return ret; - vcpu->arch.host_fpsimd_state = kern_hyp_va(fpsimd); - return 0; } @@ -52,36 +50,20 @@ int kvm_arch_vcpu_run_map_fp(struct kvm_vcpu *vcpu) void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu) { BUG_ON(!current->mm); - BUG_ON(test_thread_flag(TIF_SVE)); if (!system_supports_fpsimd()) return; - vcpu->arch.fp_state = FP_STATE_HOST_OWNED; - - vcpu_clear_flag(vcpu, HOST_SVE_ENABLED); - if (read_sysreg(cpacr_el1) & CPACR_EL1_ZEN_EL0EN) - vcpu_set_flag(vcpu, HOST_SVE_ENABLED); - /* - * We don't currently support SME guests but if we leave - * things in streaming mode then when the guest starts running - * FPSIMD or SVE code it may generate SME traps so as a - * special case if we are in streaming mode we force the host - * state to be saved now and exit streaming mode so that we - * don't have to handle any SME traps for valid guest - * operations. Do this for ZA as well for now for simplicity. + * Ensure that any host FPSIMD/SVE/SME state is saved and unbound such + * that the host kernel is responsible for restoring this state upon + * return to userspace, and the hyp code doesn't need to save anything. + * + * When the host may use SME, fpsimd_save_and_flush_cpu_state() ensures + * that PSTATE.{SM,ZA} == {0,0}. */ - if (system_supports_sme()) { - vcpu_clear_flag(vcpu, HOST_SME_ENABLED); - if (read_sysreg(cpacr_el1) & CPACR_EL1_SMEN_EL0EN) - vcpu_set_flag(vcpu, HOST_SME_ENABLED); - - if (read_sysreg_s(SYS_SVCR) & (SVCR_SM_MASK | SVCR_ZA_MASK)) { - vcpu->arch.fp_state = FP_STATE_FREE; - fpsimd_save_and_flush_cpu_state(); - } - } + fpsimd_save_and_flush_cpu_state(); + vcpu->arch.fp_state = FP_STATE_FREE; } /* @@ -135,56 +117,18 @@ void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu) local_irq_save(flags); - /* - * If we have VHE then the Hyp code will reset CPACR_EL1 to - * CPACR_EL1_DEFAULT and we need to reenable SME. - */ - if (has_vhe() && system_supports_sme()) { - /* Also restore EL0 state seen on entry */ - if (vcpu_get_flag(vcpu, HOST_SME_ENABLED)) - sysreg_clear_set(CPACR_EL1, 0, - CPACR_EL1_SMEN_EL0EN | - CPACR_EL1_SMEN_EL1EN); - else - sysreg_clear_set(CPACR_EL1, - CPACR_EL1_SMEN_EL0EN, - CPACR_EL1_SMEN_EL1EN); - } - if (vcpu->arch.fp_state == FP_STATE_GUEST_OWNED) { - if (vcpu_has_sve(vcpu)) { - __vcpu_sys_reg(vcpu, ZCR_EL1) = read_sysreg_el1(SYS_ZCR); - - /* - * Restore the VL that was saved when bound to the CPU, - * which is the maximum VL for the guest. Because - * the layout of the data when saving the sve state - * depends on the VL, we need to use a consistent VL. - * Note that this means that at guest exit ZCR_EL1 is - * not necessarily the same as on guest entry. - * - * Flushing the cpu state sets the TIF_FOREIGN_FPSTATE - * bit for the context, which lets the kernel restore - * the sve state, including ZCR_EL1 later. - */ - if (!has_vhe()) - sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, - SYS_ZCR_EL1); - } - - fpsimd_save_and_flush_cpu_state(); - } else if (has_vhe() && system_supports_sve()) { /* - * The FPSIMD/SVE state in the CPU has not been touched, and we - * have SVE (and VHE): CPACR_EL1 (alias CPTR_EL2) has been - * reset to CPACR_EL1_DEFAULT by the Hyp code, disabling SVE - * for EL0. To avoid spurious traps, restore the trap state - * seen by kvm_arch_vcpu_load_fp(): + * Flush (save and invalidate) the fpsimd/sve state so that if + * the host tries to use fpsimd/sve, it's not using stale data + * from the guest. + * + * Flushing the state sets the TIF_FOREIGN_FPSTATE bit for the + * context unconditionally, in both nVHE and VHE. This allows + * the kernel to restore the fpsimd/sve state, including ZCR_EL1 + * when needed. */ - if (vcpu_get_flag(vcpu, HOST_SVE_ENABLED)) - sysreg_clear_set(CPACR_EL1, 0, CPACR_EL1_ZEN_EL0EN); - else - sysreg_clear_set(CPACR_EL1, CPACR_EL1_ZEN_EL0EN, 0); + fpsimd_save_and_flush_cpu_state(); } update_thread_flag(TIF_SVE, 0); diff --git a/arch/arm64/kvm/hyp/entry.S b/arch/arm64/kvm/hyp/entry.S index 435346ea1504..d8c94c45cb2f 100644 --- a/arch/arm64/kvm/hyp/entry.S +++ b/arch/arm64/kvm/hyp/entry.S @@ -44,6 +44,11 @@ alternative_if ARM64_HAS_RAS_EXTN alternative_else_nop_endif mrs x1, isr_el1 cbz x1, 1f + + // Ensure that __guest_enter() always provides a context + // synchronization event so that callers don't need ISBs for anything + // that would usually be synchonized by the ERET. + isb mov x0, #ARM_EXCEPTION_IRQ ret diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index 89f7a56dac73..33f6af14ba3b 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -37,6 +38,12 @@ struct kvm_exception_table_entry { extern struct kvm_exception_table_entry __start___kvm_ex_table; extern struct kvm_exception_table_entry __stop___kvm_ex_table; +/* Check whether the FP regs are owned by the guest */ +static inline bool guest_owns_fp_regs(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.fp_state == FP_STATE_GUEST_OWNED; +} + /* Save the 32-bit only FPSIMD system register state */ static inline void __fpsimd_save_fpexc32(struct kvm_vcpu *vcpu) { @@ -161,9 +168,107 @@ static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu) write_sysreg_el1(__vcpu_sys_reg(vcpu, ZCR_EL1), SYS_ZCR); } -static void kvm_hyp_handle_fpsimd_host(struct kvm_vcpu *vcpu); +static inline void fpsimd_lazy_switch_to_guest(struct kvm_vcpu *vcpu) +{ + u64 zcr_el1, zcr_el2; -static void __deactivate_fpsimd_traps(struct kvm_vcpu *vcpu); + if (!guest_owns_fp_regs(vcpu)) + return; + + if (vcpu_has_sve(vcpu)) { + zcr_el2 = vcpu_sve_max_vq(vcpu) - 1; + + write_sysreg_el2(zcr_el2, SYS_ZCR); + + zcr_el1 = __vcpu_sys_reg(vcpu, ZCR_EL1); + write_sysreg_el1(zcr_el1, SYS_ZCR); + } +} + +static inline void fpsimd_lazy_switch_to_host(struct kvm_vcpu *vcpu) +{ + u64 zcr_el1, zcr_el2; + + if (!guest_owns_fp_regs(vcpu)) + return; + + /* + * When the guest owns the FP regs, we know that guest+hyp traps for + * any FPSIMD/SVE/SME features exposed to the guest have been disabled + * by either fpsimd_lazy_switch_to_guest() or kvm_hyp_handle_fpsimd() + * prior to __guest_entry(). As __guest_entry() guarantees a context + * synchronization event, we don't need an ISB here to avoid taking + * traps for anything that was exposed to the guest. + */ + if (vcpu_has_sve(vcpu)) { + zcr_el1 = read_sysreg_el1(SYS_ZCR); + __vcpu_sys_reg(vcpu, ZCR_EL1) = zcr_el1; + + /* + * The guest's state is always saved using the guest's max VL. + * Ensure that the host has the guest's max VL active such that + * the host can save the guest's state lazily, but don't + * artificially restrict the host to the guest's max VL. + */ + if (has_vhe()) { + zcr_el2 = vcpu_sve_max_vq(vcpu) - 1; + write_sysreg_el2(zcr_el2, SYS_ZCR); + } else { + zcr_el2 = sve_vq_from_vl(kvm_host_sve_max_vl) - 1; + write_sysreg_el2(zcr_el2, SYS_ZCR); + + zcr_el1 = vcpu_sve_max_vq(vcpu) - 1; + write_sysreg_el1(zcr_el1, SYS_ZCR); + } + } +} + +static void kvm_hyp_handle_fpsimd_host(struct kvm_vcpu *vcpu) +{ + /* + * Non-protected kvm relies on the host restoring its sve state. + * Protected kvm restores the host's sve state as not to reveal that + * fpsimd was used by a guest nor leak upper sve bits. + */ + if (system_supports_sve()) { + struct kvm_host_sve_state *sve_state = get_host_sve_state(vcpu); + + sve_state->zcr_el1 = read_sysreg_el1(SYS_ZCR); + sve_cond_update_zcr_vq(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, + SYS_ZCR_EL2); + __sve_save_state(sve_state->sve_regs + + sve_ffr_offset(kvm_host_sve_max_vl), + &sve_state->fpsr); + + /* Still trap SVE since it's handled by hyp in pKVM. */ + if (!vcpu_has_sve(vcpu)) + sysreg_clear_set(cptr_el2, 0, CPTR_EL2_TZ); + } else { + __fpsimd_save_state(get_host_fpsimd_state(vcpu)); + } +} + +static void __deactivate_fpsimd_traps(struct kvm_vcpu *vcpu) +{ + if (has_vhe()) { + u64 reg = CPACR_EL1_FPEN_EL0EN | CPACR_EL1_FPEN_EL1EN; + + if (vcpu_has_sve(vcpu)) + reg |= CPACR_EL1_ZEN_EL0EN | CPACR_EL1_ZEN_EL1EN; + + sysreg_clear_set(cpacr_el1, 0, reg); + + } else { + u64 reg = CPTR_EL2_TFP; + + if (vcpu_has_sve(vcpu) || + (is_protected_kvm_enabled() && system_supports_sve())) { + reg |= CPTR_EL2_TZ; + } + + sysreg_clear_set(cptr_el2, reg, 0); + } +} /* * We trap the first access to the FP/SIMD to save the host context and @@ -171,7 +276,7 @@ static void __deactivate_fpsimd_traps(struct kvm_vcpu *vcpu); * If FP/SIMD is not implemented, handle the trap and inject an undefined * instruction exception to the guest. Similarly for trapped SVE accesses. */ -static bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code) +static inline bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code) { bool sve_guest; u8 esr_ec; @@ -193,7 +298,7 @@ static bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code) isb(); /* Write out the host state if it's in the registers */ - if (vcpu->arch.fp_state == FP_STATE_HOST_OWNED) + if (is_protected_kvm_enabled() && vcpu->arch.fp_state == FP_STATE_HOST_OWNED) kvm_hyp_handle_fpsimd_host(vcpu); /* Restore the guest state */ @@ -320,7 +425,7 @@ static bool kvm_hyp_handle_ptrauth(struct kvm_vcpu *vcpu, u64 *exit_code) return true; } -static bool kvm_hyp_handle_sysreg(struct kvm_vcpu *vcpu, u64 *exit_code) +static inline bool kvm_hyp_handle_sysreg(struct kvm_vcpu *vcpu, u64 *exit_code) { if (cpus_have_final_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM) && handle_tx2_tvm(vcpu)) @@ -336,7 +441,7 @@ static bool kvm_hyp_handle_sysreg(struct kvm_vcpu *vcpu, u64 *exit_code) return false; } -static bool kvm_hyp_handle_cp15_32(struct kvm_vcpu *vcpu, u64 *exit_code) +static inline bool kvm_hyp_handle_cp15_32(struct kvm_vcpu *vcpu, u64 *exit_code) { if (static_branch_unlikely(&vgic_v3_cpuif_trap) && __vgic_v3_perform_cpuif_access(vcpu) == 1) @@ -345,19 +450,18 @@ static bool kvm_hyp_handle_cp15_32(struct kvm_vcpu *vcpu, u64 *exit_code) return false; } -static bool kvm_hyp_handle_memory_fault(struct kvm_vcpu *vcpu, u64 *exit_code) +static inline bool kvm_hyp_handle_memory_fault(struct kvm_vcpu *vcpu, + u64 *exit_code) { if (!__populate_fault_info(vcpu)) return true; return false; } -static bool kvm_hyp_handle_iabt_low(struct kvm_vcpu *vcpu, u64 *exit_code) - __alias(kvm_hyp_handle_memory_fault); -static bool kvm_hyp_handle_watchpt_low(struct kvm_vcpu *vcpu, u64 *exit_code) - __alias(kvm_hyp_handle_memory_fault); +#define kvm_hyp_handle_iabt_low kvm_hyp_handle_memory_fault +#define kvm_hyp_handle_watchpt_low kvm_hyp_handle_memory_fault -static bool kvm_hyp_handle_dabt_low(struct kvm_vcpu *vcpu, u64 *exit_code) +static inline bool kvm_hyp_handle_dabt_low(struct kvm_vcpu *vcpu, u64 *exit_code) { if (kvm_hyp_handle_memory_fault(vcpu, exit_code)) return true; @@ -387,23 +491,16 @@ static bool kvm_hyp_handle_dabt_low(struct kvm_vcpu *vcpu, u64 *exit_code) typedef bool (*exit_handler_fn)(struct kvm_vcpu *, u64 *); -static const exit_handler_fn *kvm_get_exit_handler_array(struct kvm_vcpu *vcpu); - -static void early_exit_filter(struct kvm_vcpu *vcpu, u64 *exit_code); - /* * Allow the hypervisor to handle the exit with an exit handler if it has one. * * Returns true if the hypervisor handled the exit, and control should go back * to the guest, or false if it hasn't. */ -static inline bool kvm_hyp_handle_exit(struct kvm_vcpu *vcpu, u64 *exit_code) +static inline bool kvm_hyp_handle_exit(struct kvm_vcpu *vcpu, u64 *exit_code, + const exit_handler_fn *handlers) { - const exit_handler_fn *handlers = kvm_get_exit_handler_array(vcpu); - exit_handler_fn fn; - - fn = handlers[kvm_vcpu_trap_get_class(vcpu)]; - + exit_handler_fn fn = handlers[kvm_vcpu_trap_get_class(vcpu)]; if (fn) return fn(vcpu, exit_code); @@ -433,20 +530,9 @@ static inline void synchronize_vcpu_pstate(struct kvm_vcpu *vcpu, u64 *exit_code * the guest, false when we should restore the host state and return to the * main run loop. */ -static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code) +static inline bool __fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code, + const exit_handler_fn *handlers) { - /* - * Save PSTATE early so that we can evaluate the vcpu mode - * early on. - */ - synchronize_vcpu_pstate(vcpu, exit_code); - - /* - * Check whether we want to repaint the state one way or - * another. - */ - early_exit_filter(vcpu, exit_code); - if (ARM_EXCEPTION_CODE(*exit_code) != ARM_EXCEPTION_IRQ) vcpu->arch.fault.esr_el2 = read_sysreg_el2(SYS_ESR); @@ -476,7 +562,7 @@ static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code) goto exit; /* Check if there's an exit handler and allow it to handle the exit. */ - if (kvm_hyp_handle_exit(vcpu, exit_code)) + if (kvm_hyp_handle_exit(vcpu, exit_code, handlers)) goto guest; exit: /* Return to the host kernel and handle the exit */ diff --git a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h index 943cf7fc7124..00526b8863e8 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h @@ -149,12 +149,6 @@ static inline bool pkvm_ipa_range_has_pvmfw(struct pkvm_hyp_vm *vm, return ipa_end > pkvm->pvmfw_load_addr && ipa_start < pvmfw_load_end; } -static inline void pkvm_set_max_sve_vq(void) -{ - sve_cond_update_zcr_vq(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, - SYS_ZCR_EL2); -} - int pkvm_load_pvmfw_pages(struct pkvm_hyp_vm *vm, u64 ipa, phys_addr_t phys, u64 size); void pkvm_poison_pvmfw_pages(void); diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 931152dc3aa8..1b5fdbfa6de8 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -34,6 +35,8 @@ DEFINE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params); void __kvm_hyp_host_forward_smc(struct kvm_cpu_context *host_ctxt); +static void fpsimd_host_restore(struct kvm_vcpu *vcpu); + static bool (*default_host_smc_handler)(struct kvm_cpu_context *host_ctxt); static bool (*default_trap_handler)(struct kvm_cpu_context *host_ctxt); @@ -580,6 +583,8 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) hyp_entry_exit_handler_fn ec_handler; u8 esr_ec; + hyp_vcpu->vcpu.arch.fp_state = FP_STATE_HOST_OWNED; + /* * If we deal with a non-protected guest and the state is potentially * dirty (from a host perspective), copy the state back into the hyp @@ -666,49 +671,48 @@ static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, u32 exit_reason) else host_vcpu->arch.iflags = hyp_vcpu->vcpu.arch.iflags; + if (hyp_vcpu->vcpu.arch.fp_state != FP_STATE_HOST_OWNED) + fpsimd_host_restore(&hyp_vcpu->vcpu); + hyp_vcpu->exit_code = exit_reason; } -static void __hyp_sve_save_guest(struct pkvm_hyp_vcpu *hyp_vcpu) +static void __hyp_sve_save_guest(struct kvm_vcpu *vcpu) { - struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; - __sve_save_state(vcpu_sve_pffr(vcpu), &vcpu->arch.ctxt.fp_regs.fpsr); __vcpu_sys_reg(vcpu, ZCR_EL1) = read_sysreg_el1(SYS_ZCR); sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL1); } -static void fpsimd_host_restore(void) +static void fpsimd_host_restore(struct kvm_vcpu *vcpu) { - sysreg_clear_set(cptr_el2, CPTR_EL2_TZ | CPTR_EL2_TFP, 0); - isb(); - - if (unlikely(is_protected_kvm_enabled())) { - struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); - struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; - - if (vcpu_has_sve(vcpu)) - __hyp_sve_save_guest(hyp_vcpu); - else - __fpsimd_save_state(&hyp_vcpu->vcpu.arch.ctxt.fp_regs); - - if (system_supports_sve()) { - struct kvm_host_sve_state *sve_state = get_host_sve_state(vcpu); - - write_sysreg_el1(sve_state->zcr_el1, SYS_ZCR); - pkvm_set_max_sve_vq(); - __sve_restore_state(sve_state->sve_regs + - sve_ffr_offset(kvm_host_sve_max_vl), - &sve_state->fpsr); - } else { - __fpsimd_restore_state(get_host_fpsimd_state(vcpu)); - } - - hyp_vcpu->vcpu.arch.fp_state = FP_STATE_HOST_OWNED; - } + u64 reg = CPTR_EL2_TFP; if (system_supports_sve()) - sve_cond_update_zcr_vq(ZCR_ELx_LEN_MASK, SYS_ZCR_EL2); + reg |= CPTR_EL2_TZ; + + sysreg_clear_set(cptr_el2, reg, 0); + isb(); + + if (vcpu_has_sve(vcpu)) + __hyp_sve_save_guest(vcpu); + else + __fpsimd_save_state(&vcpu->arch.ctxt.fp_regs); + + if (system_supports_sve()) { + struct kvm_host_sve_state *sve_state = get_host_sve_state(vcpu); + + write_sysreg_el1(sve_state->zcr_el1, SYS_ZCR); + sve_cond_update_zcr_vq(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, + SYS_ZCR_EL2); + __sve_restore_state(sve_state->sve_regs + + sve_ffr_offset(kvm_host_sve_max_vl), + &sve_state->fpsr); + } else { + __fpsimd_restore_state(get_host_fpsimd_state(vcpu)); + } + + vcpu->arch.fp_state = FP_STATE_HOST_OWNED; } static void handle___pkvm_vcpu_load(struct kvm_cpu_context *host_ctxt) @@ -739,8 +743,6 @@ static void handle___pkvm_vcpu_load(struct kvm_cpu_context *host_ctxt) *last_ran = hyp_vcpu->vcpu.vcpu_id; } - hyp_vcpu->vcpu.arch.fp_state = FP_STATE_HOST_OWNED; - if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { /* Propagate WFx trapping flags, trap ptrauth */ hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_TWE | HCR_TWI | @@ -760,9 +762,6 @@ static void handle___pkvm_vcpu_put(struct kvm_cpu_context *host_ctxt) if (hyp_vcpu) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; - if (hyp_vcpu->vcpu.arch.fp_state == FP_STATE_GUEST_OWNED) - fpsimd_host_restore(); - if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu) && !vcpu_get_flag(host_vcpu, PKVM_HOST_STATE_DIRTY)) { __sync_hyp_vcpu(hyp_vcpu); @@ -783,9 +782,6 @@ static void handle___pkvm_vcpu_sync_state(struct kvm_cpu_context *host_ctxt) if (!hyp_vcpu || pkvm_hyp_vcpu_is_protected(hyp_vcpu)) return; - if (hyp_vcpu->vcpu.arch.fp_state == FP_STATE_GUEST_OWNED) - fpsimd_host_restore(); - __sync_hyp_vcpu(hyp_vcpu); } @@ -848,26 +844,13 @@ static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt) goto out; flush_hyp_vcpu(hyp_vcpu); - ret = __kvm_vcpu_run(&hyp_vcpu->vcpu); - sync_hyp_vcpu(hyp_vcpu, ret); - - if (hyp_vcpu->vcpu.arch.fp_state == FP_STATE_GUEST_OWNED) { - /* - * The guest has used the FP, trap all accesses - * from the host (both FP and SVE). - */ - u64 reg = CPTR_EL2_TFP; - - if (system_supports_sve()) - reg |= CPTR_EL2_TZ; - - sysreg_clear_set(cptr_el2, 0, reg); - } } else { /* The host is fully trusted, run its vCPU directly. */ + fpsimd_lazy_switch_to_guest(kern_hyp_va(host_vcpu)); ret = __kvm_vcpu_run(host_vcpu); + fpsimd_lazy_switch_to_host(kern_hyp_va(host_vcpu)); } out: cpu_reg(host_ctxt, 1) = ret; @@ -1382,13 +1365,8 @@ inval: static void handle_host_smc(struct kvm_cpu_context *host_ctxt) { DECLARE_REG(u64, func_id, host_ctxt, 0); - struct pkvm_hyp_vcpu *hyp_vcpu; bool handled; - hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); - if (hyp_vcpu && hyp_vcpu->vcpu.arch.fp_state == FP_STATE_GUEST_OWNED) - fpsimd_host_restore(); - handled = kvm_host_psci_handler(host_ctxt); if (!handled) handled = kvm_host_ffa_handler(host_ctxt); @@ -1420,10 +1398,6 @@ void handle_trap(struct kvm_cpu_context *host_ctxt) case ESR_ELx_EC_SMC64: handle_host_smc(host_ctxt); break; - case ESR_ELx_EC_FP_ASIMD: - case ESR_ELx_EC_SVE: - fpsimd_host_restore(); - break; case ESR_ELx_EC_IABT_LOW: case ESR_ELx_EC_DABT_LOW: handle_host_mem_abort(host_ctxt); diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index b3acb60a2d6c..ba09c7de6901 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -54,9 +54,6 @@ static void *__get_host_fpsimd_bytes(void) struct user_fpsimd_state *get_host_fpsimd_state(struct kvm_vcpu *vcpu) { - if (likely(!is_protected_kvm_enabled())) - return vcpu->arch.host_fpsimd_state; - WARN_ON(system_supports_sve()); return __get_host_fpsimd_bytes(); } @@ -76,7 +73,6 @@ static void pvm_init_traps_aa64pfr0(struct kvm_vcpu *vcpu) const u64 feature_ids = pvm_read_id_reg(vcpu, SYS_ID_AA64PFR0_EL1); u64 hcr_set = HCR_RW; u64 hcr_clear = 0; - u64 cptr_set = 0; /* Protected KVM does not support AArch32 guests. */ BUILD_BUG_ON(FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL0), @@ -103,16 +99,10 @@ static void pvm_init_traps_aa64pfr0(struct kvm_vcpu *vcpu) /* Trap AMU */ if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_AMU), feature_ids)) { hcr_clear |= HCR_AMVOFFEN; - cptr_set |= CPTR_EL2_TAM; } - /* Trap SVE */ - if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_SVE), feature_ids)) - cptr_set |= CPTR_EL2_TZ; - vcpu->arch.hcr_el2 |= hcr_set; vcpu->arch.hcr_el2 &= ~hcr_clear; - vcpu->arch.cptr_el2 |= cptr_set; } /* @@ -142,7 +132,6 @@ static void pvm_init_traps_aa64dfr0(struct kvm_vcpu *vcpu) const u64 feature_ids = pvm_read_id_reg(vcpu, SYS_ID_AA64DFR0_EL1); u64 mdcr_set = 0; u64 mdcr_clear = 0; - u64 cptr_set = 0; /* Trap/constrain PMU */ if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMUVer), feature_ids)) { @@ -169,13 +158,8 @@ static void pvm_init_traps_aa64dfr0(struct kvm_vcpu *vcpu) if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_TraceFilt), feature_ids)) mdcr_set |= MDCR_EL2_TTRF; - /* Trap Trace */ - if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_TraceVer), feature_ids)) - cptr_set |= CPTR_EL2_TTA; - vcpu->arch.mdcr_el2 |= mdcr_set; vcpu->arch.mdcr_el2 &= ~mdcr_clear; - vcpu->arch.cptr_el2 |= cptr_set; } /* @@ -240,7 +224,6 @@ static void pvm_init_trap_regs(struct kvm_vcpu *vcpu) */ static void pkvm_vcpu_init_traps(struct pkvm_hyp_vcpu *hyp_vcpu) { - hyp_vcpu->vcpu.arch.cptr_el2 = CPTR_EL2_DEFAULT; hyp_vcpu->vcpu.arch.mdcr_el2 = 0; if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c index 2363f862abc9..a496258d42e6 100644 --- a/arch/arm64/kvm/hyp/nvhe/switch.c +++ b/arch/arm64/kvm/hyp/nvhe/switch.c @@ -36,23 +36,54 @@ DEFINE_PER_CPU(unsigned long, kvm_hyp_vector); extern void kvm_nvhe_prepare_backtrace(unsigned long fp, unsigned long pc); -static void __activate_traps(struct kvm_vcpu *vcpu) +static void __activate_cptr_traps(struct kvm_vcpu *vcpu) { - u64 val; + u64 val = CPTR_EL2_TAM; /* Same bit irrespective of E2H */ - ___activate_traps(vcpu); - __activate_traps_common(vcpu); - - val = vcpu->arch.cptr_el2; - val |= CPTR_EL2_TTA | CPTR_EL2_TAM; - if (vcpu->arch.fp_state != FP_STATE_GUEST_OWNED) { - val |= CPTR_EL2_TFP | CPTR_EL2_TZ; + if (!guest_owns_fp_regs(vcpu)) __activate_traps_fpsimd32(vcpu); - } - if (cpus_have_final_cap(ARM64_SME)) + + /* !hVHE case upstream */ + if (1) { + val |= CPTR_EL2_TTA | CPTR_NVHE_EL2_RES1; + + /* + * Always trap SME since it's not supported in KVM. + * TSM is RES1 if SME isn't implemented. + */ val |= CPTR_EL2_TSM; - write_sysreg(val, cptr_el2); + if (!vcpu_has_sve(vcpu) || !guest_owns_fp_regs(vcpu)) + val |= CPTR_EL2_TZ; + + if (!guest_owns_fp_regs(vcpu)) + val |= CPTR_EL2_TFP; + + write_sysreg(val, cptr_el2); + } +} + +static void __deactivate_cptr_traps(struct kvm_vcpu *vcpu) +{ + /* !hVHE case upstream */ + if (1) { + u64 val = CPTR_NVHE_EL2_RES1; + + if (!cpus_have_final_cap(ARM64_SVE)) + val |= CPTR_EL2_TZ; + if (!cpus_have_final_cap(ARM64_SME)) + val |= CPTR_EL2_TSM; + + write_sysreg(val, cptr_el2); + } +} + +static void __activate_traps(struct kvm_vcpu *vcpu) +{ + ___activate_traps(vcpu); + __activate_traps_common(vcpu); + __activate_cptr_traps(vcpu); + write_sysreg(__this_cpu_read(kvm_hyp_vector), vbar_el2); if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) { @@ -73,7 +104,6 @@ static void __activate_traps(struct kvm_vcpu *vcpu) static void __deactivate_traps(struct kvm_vcpu *vcpu) { extern char __kvm_hyp_host_vector[]; - u64 cptr; ___deactivate_traps(vcpu); @@ -98,28 +128,10 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu) write_sysreg(this_cpu_ptr(&kvm_init_params)->hcr_el2, hcr_el2); - cptr = CPTR_EL2_DEFAULT; - if (vcpu_has_sve(vcpu) && (vcpu->arch.fp_state == FP_STATE_GUEST_OWNED)) - cptr |= CPTR_EL2_TZ; - if (cpus_have_final_cap(ARM64_SME)) - cptr &= ~CPTR_EL2_TSM; - - write_sysreg(cptr, cptr_el2); + __deactivate_cptr_traps(vcpu); write_sysreg(__kvm_hyp_host_vector, vbar_el2); } -static void __deactivate_fpsimd_traps(struct kvm_vcpu *vcpu) -{ - u64 reg = CPTR_EL2_TFP; - - if (vcpu_has_sve(vcpu) || - (is_protected_kvm_enabled() && system_supports_sve())) { - reg |= CPTR_EL2_TZ; - } - - sysreg_clear_set(cptr_el2, reg, 0); -} - /* Save VGICv3 state on non-VHE systems */ static void __hyp_vgic_save_state(struct kvm_vcpu *vcpu) { @@ -190,30 +202,6 @@ static bool kvm_handle_pvm_sys64(struct kvm_vcpu *vcpu, u64 *exit_code) kvm_handle_pvm_sysreg(vcpu, exit_code)); } -static void kvm_hyp_handle_fpsimd_host(struct kvm_vcpu *vcpu) -{ - /* - * Non-protected kvm relies on the host restoring its sve state. - * Protected kvm restores the host's sve state as not to reveal that - * fpsimd was used by a guest nor leak upper sve bits. - */ - if (unlikely(is_protected_kvm_enabled() && system_supports_sve())) { - struct kvm_host_sve_state *sve_state = get_host_sve_state(vcpu); - - sve_state->zcr_el1 = read_sysreg_el1(SYS_ZCR); - pkvm_set_max_sve_vq(); - __sve_save_state(sve_state->sve_regs + - sve_ffr_offset(kvm_host_sve_max_vl), - &sve_state->fpsr); - - /* Still trap SVE since it's handled by hyp in pKVM. */ - if (!vcpu_has_sve(vcpu)) - sysreg_clear_set(cptr_el2, 0, CPTR_EL2_TZ); - } else { - __fpsimd_save_state(get_host_fpsimd_state(vcpu)); - } -} - static const exit_handler_fn hyp_exit_handlers[] = { [0 ... ESR_ELx_EC_MAX] = NULL, [ESR_ELx_EC_CP15_32] = kvm_hyp_handle_cp15_32, @@ -247,20 +235,23 @@ static const exit_handler_fn *kvm_get_exit_handler_array(struct kvm_vcpu *vcpu) return hyp_exit_handlers; } -/* - * Some guests (e.g., protected VMs) are not be allowed to run in AArch32. - * The ARMv8 architecture does not give the hypervisor a mechanism to prevent a - * guest from dropping to AArch32 EL0 if implemented by the CPU. If the - * hypervisor spots a guest in such a state ensure it is handled, and don't - * trust the host to spot or fix it. The check below is based on the one in - * kvm_arch_vcpu_ioctl_run(). - * - * Returns false if the guest ran in AArch32 when it shouldn't have, and - * thus should exit to the host, or true if a the guest run loop can continue. - */ -static void early_exit_filter(struct kvm_vcpu *vcpu, u64 *exit_code) +static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code) { - if (unlikely(vcpu_is_protected(vcpu) && vcpu_mode_is_32bit(vcpu))) { + const exit_handler_fn *handlers = kvm_get_exit_handler_array(vcpu); + struct kvm *kvm = kern_hyp_va(vcpu->kvm); + + synchronize_vcpu_pstate(vcpu, exit_code); + + /* + * Some guests (e.g., protected VMs) are not be allowed to run in + * AArch32. The ARMv8 architecture does not give the hypervisor a + * mechanism to prevent a guest from dropping to AArch32 EL0 if + * implemented by the CPU. If the hypervisor spots a guest in such a + * state ensure it is handled, and don't trust the host to spot or fix + * it. The check below is based on the one in + * kvm_arch_vcpu_ioctl_run(). + */ + if (kvm_vm_is_protected(kvm) && vcpu_mode_is_32bit(vcpu)) { /* * As we have caught the guest red-handed, decide that it isn't * fit for purpose anymore by making the vcpu invalid. The VMM @@ -272,6 +263,8 @@ static void early_exit_filter(struct kvm_vcpu *vcpu, u64 *exit_code) *exit_code &= BIT(ARM_EXIT_WITH_SERROR_BIT); *exit_code |= ARM_EXCEPTION_IL; } + + return __fixup_guest_exit(vcpu, exit_code, handlers); } /* Switch to the guest for legacy non-VHE systems */ diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c index 91211f86fec2..179152bb9e42 100644 --- a/arch/arm64/kvm/hyp/vhe/switch.c +++ b/arch/arm64/kvm/hyp/vhe/switch.c @@ -55,7 +55,7 @@ static void __activate_traps(struct kvm_vcpu *vcpu) val |= CPTR_EL2_TAM; - if (vcpu->arch.fp_state == FP_STATE_GUEST_OWNED) { + if (guest_owns_fp_regs(vcpu)) { if (vcpu_has_sve(vcpu)) val |= CPACR_EL1_ZEN_EL0EN | CPACR_EL1_ZEN_EL1EN; } else { @@ -102,21 +102,6 @@ void deactivate_traps_vhe_put(struct kvm_vcpu *vcpu) __deactivate_traps_common(vcpu); } -static void __deactivate_fpsimd_traps(struct kvm_vcpu *vcpu) -{ - u64 reg = CPACR_EL1_FPEN_EL0EN | CPACR_EL1_FPEN_EL1EN; - - if (vcpu_has_sve(vcpu)) - reg |= CPACR_EL1_ZEN_EL0EN | CPACR_EL1_ZEN_EL1EN; - - sysreg_clear_set(cpacr_el1, 0, reg); -} - -static void kvm_hyp_handle_fpsimd_host(struct kvm_vcpu *vcpu) -{ - __fpsimd_save_state(vcpu->arch.host_fpsimd_state); -} - static const exit_handler_fn hyp_exit_handlers[] = { [0 ... ESR_ELx_EC_MAX] = NULL, [ESR_ELx_EC_CP15_32] = kvm_hyp_handle_cp15_32, @@ -129,13 +114,11 @@ static const exit_handler_fn hyp_exit_handlers[] = { [ESR_ELx_EC_PAC] = kvm_hyp_handle_ptrauth, }; -static const exit_handler_fn *kvm_get_exit_handler_array(struct kvm_vcpu *vcpu) +static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code) { - return hyp_exit_handlers; -} + synchronize_vcpu_pstate(vcpu, exit_code); -static void early_exit_filter(struct kvm_vcpu *vcpu, u64 *exit_code) -{ + return __fixup_guest_exit(vcpu, exit_code, hyp_exit_handlers); } /* Switch to the guest for VHE systems running in EL2 */ @@ -151,6 +134,8 @@ static int __kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu) sysreg_save_host_state_vhe(host_ctxt); + fpsimd_lazy_switch_to_guest(vcpu); + /* * ARM erratum 1165522 requires us to configure both stage 1 and * stage 2 translation for the guest context before we clear @@ -181,6 +166,8 @@ static int __kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu) __deactivate_traps(vcpu); + fpsimd_lazy_switch_to_host(vcpu); + sysreg_restore_host_state_vhe(host_ctxt); if (vcpu->arch.fp_state == FP_STATE_GUEST_OWNED) diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c index 4b80f4e2b438..324b3338ab6b 100644 --- a/arch/arm64/kvm/reset.c +++ b/arch/arm64/kvm/reset.c @@ -40,6 +40,7 @@ int kvm_arm_init_sve(void) if (system_supports_sve()) { kvm_sve_max_vl = sve_max_virtualisable_vl(); kvm_host_sve_max_vl = sve_max_vl(); + kvm_nvhe_sym(kvm_host_sve_max_vl) = kvm_host_sve_max_vl; /* * The get_sve_reg()/set_sve_reg() ioctl interface will need diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index 059d00c17d26..0c976b3dd2b4 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -482,14 +482,23 @@ int kvm_vgic_map_resources(struct kvm *kvm) if (ret) goto out; - dist->ready = true; dist_base = dist->vgic_dist_base; mutex_unlock(&kvm->arch.config_lock); ret = vgic_register_dist_iodev(kvm, dist_base, type); - if (ret) + if (ret) { kvm_err("Unable to register VGIC dist MMIO regions\n"); + goto out_slots; + } + /* + * kvm_io_bus_register_dev() guarantees all readers see the new MMIO + * registration before returning through synchronize_srcu(), which also + * implies a full memory barrier. As such, marking the distributor as + * 'ready' here is guaranteed to be ordered after all vCPUs having seen + * a completely configured distributor. + */ + dist->ready = true; goto out_slots; out: mutex_unlock(&kvm->arch.config_lock); diff --git a/arch/x86/events/intel/pt.c b/arch/x86/events/intel/pt.c index 7ee8dc80a359..5ce05f559c64 100644 --- a/arch/x86/events/intel/pt.c +++ b/arch/x86/events/intel/pt.c @@ -1793,6 +1793,8 @@ static __init int pt_init(void) if (!intel_pt_validate_hw_cap(PT_CAP_topa_multiple_entries)) pt_pmu.pmu.capabilities = PERF_PMU_CAP_AUX_NO_SG; + else + pt_pmu.pmu.capabilities = PERF_PMU_CAP_AUX_PREFER_LARGE; pt_pmu.pmu.capabilities |= PERF_PMU_CAP_EXCLUSIVE | PERF_PMU_CAP_ITRACE; pt_pmu.pmu.attr_groups = pt_attr_groups; diff --git a/drivers/android/binder.c b/drivers/android/binder.c index 7811ced3ac50..877a7e357c8b 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -842,8 +842,8 @@ static void binder_transaction_priority(struct binder_thread *thread, desired.sched_policy = SCHED_NORMAL; } - if (node_prio.prio < t->priority.prio || - (node_prio.prio == t->priority.prio && + if (node_prio.prio < desired.prio || + (node_prio.prio == desired.prio && node_prio.sched_policy == SCHED_FIFO)) { /* * In case the minimum priority on the node is diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 77dd1eb1b1f3..84184cc4a83e 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -481,3 +481,5 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_filemap_map_pages_range); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_vprintk_store); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_folio_referenced_check_bypass); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_calculate_totalreserve_pages); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_drain_all_pages_bypass); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_pageset_update); diff --git a/drivers/base/power/common.c b/drivers/base/power/common.c index 299496e1381d..d853c75dda43 100644 --- a/drivers/base/power/common.c +++ b/drivers/base/power/common.c @@ -195,6 +195,7 @@ int dev_pm_domain_attach_list(struct device *dev, struct device *pd_dev = NULL; int ret, i, num_pds = 0; bool by_id = true; + size_t size; u32 pd_flags = data ? data->pd_flags : 0; u32 link_flags = pd_flags & PD_FLAG_NO_DEV_LINK ? 0 : DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME; @@ -217,19 +218,17 @@ int dev_pm_domain_attach_list(struct device *dev, if (num_pds <= 0) return 0; - pds = devm_kzalloc(dev, sizeof(*pds), GFP_KERNEL); + pds = kzalloc(sizeof(*pds), GFP_KERNEL); if (!pds) return -ENOMEM; - pds->pd_devs = devm_kcalloc(dev, num_pds, sizeof(*pds->pd_devs), - GFP_KERNEL); - if (!pds->pd_devs) - return -ENOMEM; - - pds->pd_links = devm_kcalloc(dev, num_pds, sizeof(*pds->pd_links), - GFP_KERNEL); - if (!pds->pd_links) - return -ENOMEM; + size = sizeof(*pds->pd_devs) + sizeof(*pds->pd_links); + pds->pd_devs = kcalloc(num_pds, size, GFP_KERNEL); + if (!pds->pd_devs) { + ret = -ENOMEM; + goto free_pds; + } + pds->pd_links = (void *)(pds->pd_devs + num_pds); if (link_flags && pd_flags & PD_FLAG_DEV_LINK_ON) link_flags |= DL_FLAG_RPM_ACTIVE; @@ -272,6 +271,9 @@ err_attach: device_link_del(pds->pd_links[i]); dev_pm_domain_detach(pds->pd_devs[i], true); } + kfree(pds->pd_devs); +free_pds: + kfree(pds); return ret; } EXPORT_SYMBOL_GPL(dev_pm_domain_attach_list); @@ -318,6 +320,9 @@ void dev_pm_domain_detach_list(struct dev_pm_domain_list *list) device_link_del(list->pd_links[i]); dev_pm_domain_detach(list->pd_devs[i], true); } + + kfree(list->pd_devs); + kfree(list); } EXPORT_SYMBOL_GPL(dev_pm_domain_detach_list); diff --git a/drivers/tty/serial/qcom_geni_serial.c b/drivers/tty/serial/qcom_geni_serial.c index ae82136f0aeb..dda017e40cb8 100644 --- a/drivers/tty/serial/qcom_geni_serial.c +++ b/drivers/tty/serial/qcom_geni_serial.c @@ -764,17 +764,27 @@ static void qcom_geni_serial_start_rx_fifo(struct uart_port *uport) static void qcom_geni_serial_stop_rx_dma(struct uart_port *uport) { struct qcom_geni_serial_port *port = to_dev_port(uport); + bool done; if (!qcom_geni_serial_secondary_active(uport)) return; geni_se_cancel_s_cmd(&port->se); - qcom_geni_serial_poll_bit(uport, SE_GENI_S_IRQ_STATUS, - S_CMD_CANCEL_EN, true); - - if (qcom_geni_serial_secondary_active(uport)) + done = qcom_geni_serial_poll_bit(uport, SE_DMA_RX_IRQ_STAT, + RX_EOT, true); + if (done) { + writel(RX_EOT | RX_DMA_DONE, + uport->membase + SE_DMA_RX_IRQ_CLR); + } else { qcom_geni_serial_abort_rx(uport); + writel(1, uport->membase + SE_DMA_RX_FSM_RST); + qcom_geni_serial_poll_bit(uport, SE_DMA_RX_IRQ_STAT, + RX_RESET_DONE, true); + writel(RX_RESET_DONE | RX_DMA_DONE, + uport->membase + SE_DMA_RX_IRQ_CLR); + } + if (port->rx_dma_addr) { geni_se_rx_dma_unprep(&port->se, port->rx_dma_addr, DMA_RX_BUF_SIZE); @@ -1118,7 +1128,6 @@ static int qcom_geni_serial_port_setup(struct uart_port *uport) false, true, true); geni_se_init(&port->se, UART_RX_WM, port->rx_fifo_depth - 2); geni_se_select_mode(&port->se, port->dev_data->mode); - qcom_geni_serial_start_rx(uport); port->setup = true; return 0; @@ -1134,6 +1143,11 @@ static int qcom_geni_serial_startup(struct uart_port *uport) if (ret) return ret; } + + uart_port_lock_irq(uport); + qcom_geni_serial_start_rx(uport); + uart_port_unlock_irq(uport); + enable_irq(uport->irq); return 0; @@ -1218,7 +1232,6 @@ static void qcom_geni_serial_set_termios(struct uart_port *uport, u32 ver, sampling_rate; unsigned int avg_bw_core; - qcom_geni_serial_stop_rx(uport); /* baud rate */ baud = uart_get_baud_rate(uport, termios, old, 300, 4000000); port->baud = baud; @@ -1235,7 +1248,7 @@ static void qcom_geni_serial_set_termios(struct uart_port *uport, dev_err(port->se.dev, "Couldn't find suitable clock rate for %u\n", baud * sampling_rate); - goto out_restart_rx; + return; } dev_dbg(port->se.dev, "desired_rate = %u, clk_rate = %lu, clk_div = %u\n", @@ -1314,8 +1327,6 @@ static void qcom_geni_serial_set_termios(struct uart_port *uport, writel(stop_bit_len, uport->membase + SE_UART_TX_STOP_BIT_LEN); writel(ser_clk_cfg, uport->membase + GENI_SER_M_CLK_CFG); writel(ser_clk_cfg, uport->membase + GENI_SER_S_CLK_CFG); -out_restart_rx: - qcom_geni_serial_start_rx(uport); } #ifdef CONFIG_SERIAL_QCOM_GENI_CONSOLE @@ -1734,38 +1745,6 @@ static int qcom_geni_serial_sys_resume(struct device *dev) return ret; } -static int qcom_geni_serial_sys_hib_resume(struct device *dev) -{ - int ret = 0; - struct uart_port *uport; - struct qcom_geni_private_data *private_data; - struct qcom_geni_serial_port *port = dev_get_drvdata(dev); - - uport = &port->uport; - private_data = uport->private_data; - - if (uart_console(uport)) { - geni_icc_set_tag(&port->se, QCOM_ICC_TAG_ALWAYS); - geni_icc_set_bw(&port->se); - ret = uart_resume_port(private_data->drv, uport); - /* - * For hibernation usecase clients for - * console UART won't call port setup during restore, - * hence call port setup for console uart. - */ - qcom_geni_serial_port_setup(uport); - } else { - /* - * Peripheral register settings are lost during hibernation. - * Update setup flag such that port setup happens again - * during next session. Clients of HS-UART will close and - * open the port during hibernation. - */ - port->setup = false; - } - return ret; -} - static const struct qcom_geni_device_data qcom_geni_console_data = { .console = true, .mode = GENI_SE_FIFO, @@ -1777,12 +1756,8 @@ static const struct qcom_geni_device_data qcom_geni_uart_data = { }; static const struct dev_pm_ops qcom_geni_serial_pm_ops = { - .suspend = pm_sleep_ptr(qcom_geni_serial_sys_suspend), - .resume = pm_sleep_ptr(qcom_geni_serial_sys_resume), - .freeze = pm_sleep_ptr(qcom_geni_serial_sys_suspend), - .poweroff = pm_sleep_ptr(qcom_geni_serial_sys_suspend), - .restore = pm_sleep_ptr(qcom_geni_serial_sys_hib_resume), - .thaw = pm_sleep_ptr(qcom_geni_serial_sys_hib_resume), + SYSTEM_SLEEP_PM_OPS(qcom_geni_serial_sys_suspend, + qcom_geni_serial_sys_resume) }; static const struct of_device_id qcom_geni_serial_match_table[] = { diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c index b757fe1e5d96..1ec8522fc93d 100644 --- a/drivers/usb/dwc3/core.c +++ b/drivers/usb/dwc3/core.c @@ -1480,7 +1480,7 @@ static int dwc3_core_get_phy(struct dwc3 *dwc) struct phy *temp_phy = NULL; char phy_name[9]; int ret; - int i; + u8 i; if (node) { dwc->usb2_phy = devm_usb_get_phy_by_phandle(dev, "usb-phy", 0); @@ -1510,7 +1510,7 @@ static int dwc3_core_get_phy(struct dwc3 *dwc) if (vdwc->num_usb2_ports == 1) snprintf(phy_name, sizeof(phy_name), "usb2-phy"); else - snprintf(phy_name, sizeof(phy_name), "usb2-%d", i); + snprintf(phy_name, sizeof(phy_name), "usb2-%u", i); temp_phy = devm_phy_get(dev, phy_name); if (IS_ERR(temp_phy)) { @@ -1532,7 +1532,7 @@ static int dwc3_core_get_phy(struct dwc3 *dwc) if (vdwc->num_usb3_ports == 1) snprintf(phy_name, sizeof(phy_name), "usb3-phy"); else - snprintf(phy_name, sizeof(phy_name), "usb3-%d", i); + snprintf(phy_name, sizeof(phy_name), "usb3-%u", i); temp_phy = devm_phy_get(dev, phy_name); if (IS_ERR(temp_phy)) { diff --git a/drivers/usb/gadget/function/f_midi.c b/drivers/usb/gadget/function/f_midi.c index 9c8d56a496c9..5f022db4fa71 100644 --- a/drivers/usb/gadget/function/f_midi.c +++ b/drivers/usb/gadget/function/f_midi.c @@ -906,6 +906,15 @@ static int f_midi_bind(struct usb_configuration *c, struct usb_function *f) status = -ENODEV; + /* + * Reset wMaxPacketSize with maximum packet size of FS bulk transfer before + * endpoint claim. This ensures that the wMaxPacketSize does not exceed the + * limit during bind retries where configured dwc3 TX/RX FIFO's maxpacket + * size of 512 bytes for IN/OUT endpoints in support HS speed only. + */ + bulk_in_desc.wMaxPacketSize = cpu_to_le16(64); + bulk_out_desc.wMaxPacketSize = cpu_to_le16(64); + /* allocate instance-specific endpoints */ midi->in_ep = usb_ep_autoconfig(cdev->gadget, &bulk_in_desc); if (!midi->in_ep) diff --git a/drivers/usb/storage/usb.c b/drivers/usb/storage/usb.c index ed7c6ad96a74..dc4e3983f7c5 100644 --- a/drivers/usb/storage/usb.c +++ b/drivers/usb/storage/usb.c @@ -67,9 +67,102 @@ MODULE_AUTHOR("Matthew Dharm "); MODULE_DESCRIPTION("USB Mass Storage driver for Linux"); MODULE_LICENSE("GPL"); -static unsigned int delay_use = 1; -module_param(delay_use, uint, S_IRUGO | S_IWUSR); -MODULE_PARM_DESC(delay_use, "seconds to delay before using a new device"); +static unsigned int delay_use = 1 * MSEC_PER_SEC; + +/** + * parse_delay_str - parse an unsigned decimal integer delay + * @str: String to parse. + * @ndecimals: Number of decimal to scale up. + * @suffix: Suffix string to parse. + * @val: Where to store the parsed value. + * + * Parse an unsigned decimal value in @str, optionally end with @suffix. + * Stores the parsed value in @val just as it is if @str ends with @suffix. + * Otherwise store the value scale up by 10^(@ndecimal). + * + * Returns 0 on success, a negative error code otherwise. + */ +static int parse_delay_str(const char *str, int ndecimals, const char *suffix, + unsigned int *val) +{ + int n, n2, l; + char buf[16]; + + l = strlen(suffix); + n = strlen(str); + if (n > 0 && str[n - 1] == '\n') + --n; + if (n >= l && !strncmp(&str[n - l], suffix, l)) { + n -= l; + n2 = 0; + } else + n2 = ndecimals; + + if (n + n2 > sizeof(buf) - 1) + return -EINVAL; + + memcpy(buf, str, n); + while (n2-- > 0) + buf[n++] = '0'; + buf[n] = 0; + + return kstrtouint(buf, 10, val); +} + +/** + * format_delay_ms - format an integer value into a delay string + * @val: The integer value to format, scaled by 10^(@ndecimals). + * @ndecimals: Number of decimal to scale down. + * @suffix: Suffix string to format. + * @str: Where to store the formatted string. + * @size: The size of buffer for @str. + * + * Format an integer value in @val scale down by 10^(@ndecimals) without @suffix + * if @val is divisible by 10^(@ndecimals). + * Otherwise format a value in @val just as it is with @suffix + * + * Returns the number of characters written into @str. + */ +static int format_delay_ms(unsigned int val, int ndecimals, const char *suffix, + char *str, int size) +{ + u64 delay_ms = val; + unsigned int rem = do_div(delay_ms, int_pow(10, ndecimals)); + int ret; + + if (rem) + ret = scnprintf(str, size, "%u%s\n", val, suffix); + else + ret = scnprintf(str, size, "%u\n", (unsigned int)delay_ms); + return ret; +} + +static int delay_use_set(const char *s, const struct kernel_param *kp) +{ + unsigned int delay_ms; + int ret; + + ret = parse_delay_str(skip_spaces(s), 3, "ms", &delay_ms); + if (ret < 0) + return ret; + + *((unsigned int *)kp->arg) = delay_ms; + return 0; +} + +static int delay_use_get(char *s, const struct kernel_param *kp) +{ + unsigned int delay_ms = *((unsigned int *)kp->arg); + + return format_delay_ms(delay_ms, 3, "ms", s, PAGE_SIZE); +} + +static const struct kernel_param_ops delay_use_ops = { + .set = delay_use_set, + .get = delay_use_get, +}; +module_param_cb(delay_use, &delay_use_ops, &delay_use, 0644); +MODULE_PARM_DESC(delay_use, "time to delay before using a new device"); static char quirks[128]; module_param_string(quirks, quirks, sizeof(quirks), S_IRUGO | S_IWUSR); @@ -1066,7 +1159,7 @@ int usb_stor_probe2(struct us_data *us) if (delay_use > 0) dev_dbg(dev, "waiting for device to settle before scanning\n"); queue_delayed_work(system_freezable_wq, &us->scan_dwork, - delay_use * HZ); + msecs_to_jiffies(delay_use)); return 0; /* We come here if there are any problems */ diff --git a/fs/proc/base.c b/fs/proc/base.c index 77b3b1efe43d..bf14f94df793 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -65,6 +65,7 @@ #include #include #include +#include #include #include #include @@ -2476,7 +2477,7 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx) } p->start = vma->vm_start; - p->end = vma->vm_end; + p->end = VMA_PAD_START(vma); p->mode = vma->vm_file->f_mode; } mmap_read_unlock(mm); diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index f36e6e018b26..e8baf3e8a86a 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -692,6 +692,34 @@ void dup_userfaultfd_complete(struct list_head *fcs) } } +void dup_userfaultfd_fail(struct list_head *fcs) +{ + struct userfaultfd_fork_ctx *fctx, *n; + + /* + * An error has occurred on fork, we will tear memory down, but have + * allocated memory for fctx's and raised reference counts for both the + * original and child contexts (and on the mm for each as a result). + * + * These would ordinarily be taken care of by a user handling the event, + * but we are no longer doing so, so manually clean up here. + * + * mm tear down will take care of cleaning up VMA contexts. + */ + list_for_each_entry_safe(fctx, n, fcs, list) { + struct userfaultfd_ctx *octx = fctx->orig; + struct userfaultfd_ctx *ctx = fctx->new; + + atomic_dec(&octx->mmap_changing); + VM_BUG_ON(atomic_read(&octx->mmap_changing) < 0); + userfaultfd_ctx_put(octx); + userfaultfd_ctx_put(ctx); + + list_del(&fctx->list); + kfree(fctx); + } +} + void mremap_userfaultfd_prep(struct vm_area_struct *vma, struct vm_userfaultfd_ctx *vm_ctx) { diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 92d866352f35..6ef9152c8348 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -286,6 +286,7 @@ struct perf_event; #define PERF_PMU_CAP_NO_EXCLUDE 0x0080 #define PERF_PMU_CAP_AUX_OUTPUT 0x0100 #define PERF_PMU_CAP_EXTENDED_HW_TYPE 0x0200 +#define PERF_PMU_CAP_AUX_PREFER_LARGE 0x0400 struct perf_output_handle; diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index 7d881c5df6ad..587f718ccc90 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -225,6 +225,7 @@ static inline bool vma_can_userfault(struct vm_area_struct *vma, extern int dup_userfaultfd(struct vm_area_struct *, struct list_head *); extern void dup_userfaultfd_complete(struct list_head *); +void dup_userfaultfd_fail(struct list_head *); extern void mremap_userfaultfd_prep(struct vm_area_struct *, struct vm_userfaultfd_ctx *); @@ -299,6 +300,10 @@ static inline void dup_userfaultfd_complete(struct list_head *l) { } +static inline void dup_userfaultfd_fail(struct list_head *l) +{ +} + static inline void mremap_userfaultfd_prep(struct vm_area_struct *vma, struct vm_userfaultfd_ctx *ctx) { diff --git a/include/trace/hooks/mm.h b/include/trace/hooks/mm.h index fdbdd8080370..0e0403625dbf 100644 --- a/include/trace/hooks/mm.h +++ b/include/trace/hooks/mm.h @@ -315,6 +315,14 @@ DECLARE_HOOK(android_vh_filemap_map_pages_range, DECLARE_HOOK(android_vh_calculate_totalreserve_pages, TP_PROTO(bool *skip), TP_ARGS(skip)); +DECLARE_HOOK(android_vh_drain_all_pages_bypass, + TP_PROTO(gfp_t gfp_mask, unsigned int order, unsigned long alloc_flags, + int migratetype, unsigned long did_some_progress, + bool *bypass), + TP_ARGS(gfp_mask, order, alloc_flags, migratetype, did_some_progress, bypass)); +DECLARE_HOOK(android_vh_pageset_update, + TP_PROTO(unsigned long *high, unsigned long *batch), + TP_ARGS(high, batch)); #endif /* _TRACE_HOOK_MM_H */ /* This part must be outside protection */ diff --git a/include/trace/hooks/sched.h b/include/trace/hooks/sched.h index ce26b58b9e3c..af49cae410e6 100644 --- a/include/trace/hooks/sched.h +++ b/include/trace/hooks/sched.h @@ -84,6 +84,10 @@ DECLARE_RESTRICTED_HOOK(android_rvh_setscheduler, TP_PROTO(struct task_struct *p), TP_ARGS(p), 1); +DECLARE_RESTRICTED_HOOK(android_rvh_setscheduler_prio, + TP_PROTO(struct task_struct *p), + TP_ARGS(p), 1); + struct sched_group; DECLARE_RESTRICTED_HOOK(android_rvh_find_busiest_group, TP_PROTO(struct sched_group *busiest, struct rq *dst_rq, int *out_balance), diff --git a/include/uapi/drm/msm_drm.h b/include/uapi/drm/msm_drm.h index 3c7b097c4e3d..16f2eb4b65ee 100644 --- a/include/uapi/drm/msm_drm.h +++ b/include/uapi/drm/msm_drm.h @@ -180,7 +180,11 @@ struct drm_msm_gem_cpu_fini { */ struct drm_msm_gem_submit_reloc { __u32 submit_offset; /* in, offset from submit_bo */ +#ifdef __cplusplus + __u32 _or; /* in, value OR'd with result */ +#else __u32 or; /* in, value OR'd with result */ +#endif __s32 shift; /* in, amount of left shift (can be negative) */ __u32 reloc_idx; /* in, index of reloc_bo buffer */ __u64 reloc_offset; /* in, offset from start of reloc_bo */ diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c index 644dfed04926..b9c010a0e0fe 100644 --- a/kernel/events/ring_buffer.c +++ b/kernel/events/ring_buffer.c @@ -672,15 +672,23 @@ int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event, { bool overwrite = !(flags & RING_BUFFER_WRITABLE); int node = (event->cpu == -1) ? -1 : cpu_to_node(event->cpu); - int ret = -ENOMEM, max_order; + bool use_contiguous_pages = event->pmu->capabilities & ( + PERF_PMU_CAP_AUX_NO_SG | PERF_PMU_CAP_AUX_PREFER_LARGE); + /* + * Initialize max_order to 0 for page allocation. This allocates single + * pages to minimize memory fragmentation. This is overridden if the + * PMU needs or prefers contiguous pages (use_contiguous_pages = true). + */ + int max_order = 0; + int ret = -ENOMEM; if (!has_aux(event)) return -EOPNOTSUPP; if (!overwrite) { /* - * Watermark defaults to half the buffer, and so does the - * max_order, to aid PMU drivers in double buffering. + * Watermark defaults to half the buffer, to aid PMU drivers + * in double buffering. */ if (!watermark) watermark = min_t(unsigned long, @@ -688,16 +696,19 @@ int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event, (unsigned long)nr_pages << (PAGE_SHIFT - 1)); /* - * Use aux_watermark as the basis for chunking to - * help PMU drivers honor the watermark. + * If using contiguous pages, use aux_watermark as the basis + * for chunking to help PMU drivers honor the watermark. */ - max_order = get_order(watermark); + if (use_contiguous_pages) + max_order = get_order(watermark); } else { /* - * We need to start with the max_order that fits in nr_pages, - * not the other way around, hence ilog2() and not get_order. + * If using contiguous pages, we need to start with the + * max_order that fits in nr_pages, not the other way around, + * hence ilog2() and not get_order. */ - max_order = ilog2(nr_pages); + if (use_contiguous_pages) + max_order = ilog2(nr_pages); watermark = 0; } diff --git a/kernel/fork.c b/kernel/fork.c index 2258ff82172e..91357988f282 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -662,11 +662,8 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, LIST_HEAD(uf); MA_STATE(mas, &mm->mm_mt, 0, 0); - uprobe_start_dup_mmap(); - if (mmap_write_lock_killable(oldmm)) { - retval = -EINTR; - goto fail_uprobe_end; - } + if (mmap_write_lock_killable(oldmm)) + return -EINTR; flush_cache_dup_mm(oldmm); uprobe_dup_mmap(oldmm, mm); /* @@ -806,9 +803,10 @@ out: mmap_write_unlock(mm); flush_tlb_mm(oldmm); mmap_write_unlock(oldmm); - dup_userfaultfd_complete(&uf); -fail_uprobe_end: - uprobe_end_dup_mmap(); + if (!retval) + dup_userfaultfd_complete(&uf); + else + dup_userfaultfd_fail(&uf); return retval; fail_nomem_anon_vma_fork: @@ -1639,9 +1637,11 @@ static struct mm_struct *dup_mm(struct task_struct *tsk, if (!mm_init(mm, tsk, mm->user_ns)) goto fail_nomem; + uprobe_start_dup_mmap(); err = dup_mmap(mm, oldmm); if (err) goto free_pt; + uprobe_end_dup_mmap(); mm->hiwater_rss = get_mm_rss(mm); mm->hiwater_vm = mm->total_vm; @@ -1656,6 +1656,8 @@ free_pt: mm->binfmt = NULL; mm_init_owner(mm, NULL); mmput(mm); + if (err) + uprobe_end_dup_mmap(); fail_nomem: return NULL; diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 64a231649cce..2174dd3ffb2a 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -7085,6 +7085,7 @@ static void __setscheduler_prio(struct task_struct *p, int prio) p->sched_class = &fair_sched_class; p->prio = prio; + trace_android_rvh_setscheduler_prio(p); } #ifdef CONFIG_RT_MUTEXES diff --git a/kernel/sched/vendor_hooks.c b/kernel/sched/vendor_hooks.c index 46a67d9b6344..89a4a4174cde 100644 --- a/kernel/sched/vendor_hooks.c +++ b/kernel/sched/vendor_hooks.c @@ -27,6 +27,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_rtmutex_prepare_setprio); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_user_nice); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_user_nice_locked); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_setscheduler); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_setscheduler_prio); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_find_busiest_group); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_dump_throttled_rt_tasks); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_jiffies_update); diff --git a/mm/memfd-ashmem-shim.c b/mm/memfd-ashmem-shim.c index e09d95a8b274..258498cca9bb 100644 --- a/mm/memfd-ashmem-shim.c +++ b/mm/memfd-ashmem-shim.c @@ -16,58 +16,31 @@ #include "memfd-ashmem-shim.h" #include "memfd-ashmem-shim-internal.h" -/* file_path() returns the path of the file including the root, hence the additional "/". */ -#define MEMFD_PATH_PREFIX "/memfd:" -#define MEMFD_PATH_PREFIX_LEN (sizeof(MEMFD_PATH_PREFIX) - 1) +/* memfd file names all start with memfd: */ +#define MEMFD_PREFIX "memfd:" +#define MEMFD_PREFIX_LEN (sizeof(MEMFD_PREFIX) - 1) -/* All memfd files are unlinked, and are therefore suffixed with the " (deleted)" string. */ -#define UNLINKED_FILE_SUFFIX " (deleted)" -#define UNLINKED_FILE_SUFFIX_LEN (sizeof(UNLINKED_FILE_SUFFIX) - 1) - -/* - * 1 character for the start of the path (/), NAME_MAX for the maximum length of a full memfd file - * name, UNLINKED_FILE_SUFFIX_LEN for the " (deleted)" suffix, and 1 for the NUL terminating - * character. - */ -#define MAX_FILE_PATH_SIZE (1 + NAME_MAX + UNLINKED_FILE_SUFFIX_LEN + 1) - -static char *get_memfd_file_name(struct file *file, char *buf, size_t size) +static const char *get_memfd_name(struct file *file) { - char *name_end; - char *path = file_path(file, buf, size); + /* This pointer is always valid, so no need to check if it's NULL. */ + const char *file_name = file->f_path.dentry->d_name.name; - if (IS_ERR(path)) - return path; + if (file_name != strstr(file_name, MEMFD_PREFIX)) + return NULL; - /* Only handle memfds; we cannot make assumptions about other file names. */ - name_end = strstr(path, UNLINKED_FILE_SUFFIX); - if ((strstr(path, MEMFD_PATH_PREFIX) != path) || !name_end) - return ERR_PTR(-EINVAL); - - /* - * Since file_path() returns the full path of the file, including the root, the format will - * be: - * - * "/memfd:testbuf (deleted)" - * - * But the ASHMEM_GET_NAME ioctl only returns the name of the buffer without any prefixes - * or suffixes. So, terminate the string at the start of the " (deleted)" suffix so that - * strlen() can be used on it from the start of the name. - */ - *name_end = '\0'; - - /* return a pointer to the start of the name */ - return &path[MEMFD_PATH_PREFIX_LEN]; + return file_name; } static long get_name(struct file *file, void __user *name) { - char buf[MAX_FILE_PATH_SIZE]; - char *file_name = get_memfd_file_name(file, buf, sizeof(buf)); + const char *file_name = get_memfd_name(file); size_t len; - if (IS_ERR(file_name)) - return PTR_ERR(file_name); + if (!file_name) + return -EINVAL; + + /* Strip MEMFD_PREFIX to retain compatibility with ashmem driver. */ + file_name = &file_name[MEMFD_PREFIX_LEN]; /* * The expectation is that the user provided buffer is ASHMEM_NAME_LEN in size, which is diff --git a/mm/page_alloc.c b/mm/page_alloc.c index ffc6abc47849..a1605834867e 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -5026,6 +5026,7 @@ __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, struct page *page = NULL; unsigned long pflags; bool drained = false; + bool skip_pcp_drain = false; trace_android_vh_mm_alloc_pages_direct_reclaim_enter(order); psi_memstall_enter(&pflags); @@ -5043,7 +5044,10 @@ retry: */ if (!page && !drained) { unreserve_highatomic_pageblock(ac, false); - drain_all_pages(NULL); + trace_android_vh_drain_all_pages_bypass(gfp_mask, order, + alloc_flags, ac->migratetype, *did_some_progress, &skip_pcp_drain); + if (!skip_pcp_drain) + drain_all_pages(NULL); drained = true; ++retry_times; goto retry; @@ -7475,6 +7479,7 @@ static int zone_highsize(struct zone *zone, int batch, int cpu_online) static void pageset_update(struct per_cpu_pages *pcp, unsigned long high, unsigned long batch) { + trace_android_vh_pageset_update(&high, &batch); WRITE_ONCE(pcp->batch, batch); WRITE_ONCE(pcp->high, high); } diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index b45edacc7436..468747538b41 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -966,8 +966,13 @@ static int move_present_pte(struct mm_struct *mm, WRITE_ONCE(src_folio->index, linear_page_index(dst_vma, dst_addr)); orig_dst_pte = mk_pte(&src_folio->page, dst_vma->vm_page_prot); - /* Follow mremap() behavior and treat the entry dirty after the move */ - orig_dst_pte = pte_mkwrite(pte_mkdirty(orig_dst_pte)); + /* Set soft dirty bit so userspace can notice the pte was moved */ +#ifdef CONFIG_MEM_SOFT_DIRTY + orig_dst_pte = pte_mksoft_dirty(orig_dst_pte); +#endif + if (pte_dirty(orig_src_pte)) + orig_dst_pte = pte_mkdirty(orig_dst_pte); + orig_dst_pte = pte_mkwrite(orig_dst_pte); set_pte_at(mm, dst_addr, dst_pte, orig_dst_pte); out: @@ -1001,6 +1006,9 @@ static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma, } orig_src_pte = ptep_get_and_clear(mm, src_addr, src_pte); +#ifdef CONFIG_MEM_SOFT_DIRTY + orig_src_pte = pte_swp_mksoft_dirty(orig_src_pte); +#endif set_pte_at(mm, dst_addr, dst_pte, orig_src_pte); double_pt_unlock(dst_ptl, src_ptl); diff --git a/mm/vmscan.c b/mm/vmscan.c index 71fcab2182fc..8f6b7ebddd7f 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -5654,8 +5654,8 @@ static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control * blk_finish_plug(&plug); done: - /* kswapd should never fail */ - pgdat->kswapd_failures = 0; + if (sc->nr_reclaimed > reclaimed) + pgdat->kswapd_failures = 0; } /******************************************************************************