diff --git a/.clang-format b/.clang-format index 15d4eaabc6b5..fa959436bcfd 100644 --- a/.clang-format +++ b/.clang-format @@ -216,7 +216,6 @@ ForEachMacros: - 'for_each_migratetype_order' - 'for_each_msi_entry' - 'for_each_msi_entry_safe' - - 'for_each_msi_vector' - 'for_each_net' - 'for_each_net_continue_reverse' - 'for_each_netdev' diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 4a61b47a9314..8a809e8dfc6c 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -612,8 +612,8 @@ clocksource.max_cswd_read_retries= [KNL] Number of clocksource_watchdog() retries due to external delays before the clock will be marked - unstable. Defaults to three retries, that is, - four attempts to read the clock under test. + unstable. Defaults to two retries, that is, + three attempts to read the clock under test. clocksource.verify_n_cpus= [KNL] Limit the number of CPUs checked for clocksources diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,apmixedsys.txt b/Documentation/devicetree/bindings/arm/mediatek/mediatek,apmixedsys.txt index ea827e8763de..3fa755866528 100644 --- a/Documentation/devicetree/bindings/arm/mediatek/mediatek,apmixedsys.txt +++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,apmixedsys.txt @@ -14,6 +14,7 @@ Required Properties: - "mediatek,mt7622-apmixedsys" - "mediatek,mt7623-apmixedsys", "mediatek,mt2701-apmixedsys" - "mediatek,mt7629-apmixedsys" + - "mediatek,mt7986-apmixedsys" - "mediatek,mt8135-apmixedsys" - "mediatek,mt8167-apmixedsys", "syscon" - "mediatek,mt8173-apmixedsys" diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,ethsys.txt b/Documentation/devicetree/bindings/arm/mediatek/mediatek,ethsys.txt index 6b7e8067e7aa..0502db73686b 100644 --- a/Documentation/devicetree/bindings/arm/mediatek/mediatek,ethsys.txt +++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,ethsys.txt @@ -10,6 +10,7 @@ Required Properties: - "mediatek,mt7622-ethsys", "syscon" - "mediatek,mt7623-ethsys", "mediatek,mt2701-ethsys", "syscon" - "mediatek,mt7629-ethsys", "syscon" + - "mediatek,mt7986-ethsys", "syscon" - #clock-cells: Must be 1 - #reset-cells: Must be 1 diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,infracfg.txt b/Documentation/devicetree/bindings/arm/mediatek/mediatek,infracfg.txt index eb3523c7a7be..f66bd720571d 100644 --- a/Documentation/devicetree/bindings/arm/mediatek/mediatek,infracfg.txt +++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,infracfg.txt @@ -15,6 +15,7 @@ Required Properties: - "mediatek,mt7622-infracfg", "syscon" - "mediatek,mt7623-infracfg", "mediatek,mt2701-infracfg", "syscon" - "mediatek,mt7629-infracfg", "syscon" + - "mediatek,mt7986-infracfg", "syscon" - "mediatek,mt8135-infracfg", "syscon" - "mediatek,mt8167-infracfg", "syscon" - "mediatek,mt8173-infracfg", "syscon" diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,sgmiisys.txt b/Documentation/devicetree/bindings/arm/mediatek/mediatek,sgmiisys.txt index 30cb645c0e54..29ca7a10b315 100644 --- a/Documentation/devicetree/bindings/arm/mediatek/mediatek,sgmiisys.txt +++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,sgmiisys.txt @@ -8,6 +8,8 @@ Required Properties: - compatible: Should be: - "mediatek,mt7622-sgmiisys", "syscon" - "mediatek,mt7629-sgmiisys", "syscon" + - "mediatek,mt7986-sgmiisys_0", "syscon" + - "mediatek,mt7986-sgmiisys_1", "syscon" - #clock-cells: Must be 1 The SGMIISYS controller uses the common clk binding from diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,topckgen.txt b/Documentation/devicetree/bindings/arm/mediatek/mediatek,topckgen.txt index 5ce7578cf274..b82422bb717f 100644 --- a/Documentation/devicetree/bindings/arm/mediatek/mediatek,topckgen.txt +++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,topckgen.txt @@ -14,6 +14,7 @@ Required Properties: - "mediatek,mt7622-topckgen" - "mediatek,mt7623-topckgen", "mediatek,mt2701-topckgen" - "mediatek,mt7629-topckgen" + - "mediatek,mt7986-topckgen", "syscon" - "mediatek,mt8135-topckgen" - "mediatek,mt8167-topckgen", "syscon" - "mediatek,mt8173-topckgen" diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-ccu.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-ccu.yaml index c4b7243ddcf2..15ed64d35261 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-ccu.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-ccu.yaml @@ -34,6 +34,8 @@ properties: - allwinner,sun8i-v3-ccu - allwinner,sun8i-v3s-ccu - allwinner,sun9i-a80-ccu + - allwinner,sun20i-d1-ccu + - allwinner,sun20i-d1-r-ccu - allwinner,sun50i-a64-ccu - allwinner,sun50i-a64-r-ccu - allwinner,sun50i-a100-ccu @@ -79,6 +81,7 @@ if: enum: - allwinner,sun8i-a83t-r-ccu - allwinner,sun8i-h3-r-ccu + - allwinner,sun20i-d1-r-ccu - allwinner,sun50i-a64-r-ccu - allwinner,sun50i-a100-r-ccu - allwinner,sun50i-h6-r-ccu @@ -99,6 +102,7 @@ else: properties: compatible: enum: + - allwinner,sun20i-d1-ccu - allwinner,sun50i-a100-ccu - allwinner,sun50i-h6-ccu - allwinner,sun50i-h616-ccu diff --git a/Documentation/devicetree/bindings/clock/microchip,lan966x-gck.yaml b/Documentation/devicetree/bindings/clock/microchip,lan966x-gck.yaml new file mode 100644 index 000000000000..df2bec188706 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/microchip,lan966x-gck.yaml @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/microchip,lan966x-gck.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Microchip LAN966X Generic Clock Controller + +maintainers: + - Kavyasree Kotagiri + +description: | + The LAN966X Generic clock controller contains 3 PLLs - cpu_clk, + ddr_clk and sys_clk. This clock controller generates and supplies + clock to various peripherals within the SoC. + +properties: + compatible: + const: microchip,lan966x-gck + + reg: + minItems: 1 + items: + - description: Generic clock registers + - description: Optional gate clock registers + + clocks: + items: + - description: CPU clock source + - description: DDR clock source + - description: System clock source + + clock-names: + items: + - const: cpu + - const: ddr + - const: sys + + '#clock-cells': + const: 1 + +required: + - compatible + - reg + - clocks + - clock-names + - '#clock-cells' + +additionalProperties: false + +examples: + - | + clks: clock-controller@e00c00a8 { + compatible = "microchip,lan966x-gck"; + #clock-cells = <1>; + clocks = <&cpu_clk>, <&ddr_clk>, <&sys_clk>; + clock-names = "cpu", "ddr", "sys"; + reg = <0xe00c00a8 0x38>; + }; +... diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8976.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8976.yaml new file mode 100644 index 000000000000..f3430b159caa --- /dev/null +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8976.yaml @@ -0,0 +1,97 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/qcom,gcc-msm8976.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Global Clock & Reset Controller Binding for MSM8976 + +maintainers: + - Stephen Boyd + - Taniya Das + +description: | + Qualcomm global clock control module which supports the clocks, resets and + power domains on MSM8976. + + See also: + - dt-bindings/clock/qcom,gcc-msm8976.h + +properties: + compatible: + enum: + - qcom,gcc-msm8976 + - qcom,gcc-msm8976-v1.1 + + clocks: + items: + - description: XO source + - description: Always-on XO source + - description: Pixel clock from DSI PHY0 + - description: Byte clock from DSI PHY0 + - description: Pixel clock from DSI PHY1 + - description: Byte clock from DSI PHY1 + + clock-names: + items: + - const: xo + - const: xo_a + - const: dsi0pll + - const: dsi0pllbyte + - const: dsi1pll + - const: dsi1pllbyte + + vdd_gfx-supply: + description: + Phandle to voltage regulator providing power to the GX domain. + + '#clock-cells': + const: 1 + + '#reset-cells': + const: 1 + + '#power-domain-cells': + const: 1 + + reg: + maxItems: 1 + +required: + - compatible + - reg + - clocks + - clock-names + - vdd_gfx-supply + - '#clock-cells' + - '#reset-cells' + - '#power-domain-cells' + +additionalProperties: false + +examples: + - | + clock-controller@1800000 { + compatible = "qcom,gcc-msm8976"; + #clock-cells = <1>; + #reset-cells = <1>; + #power-domain-cells = <1>; + reg = <0x1800000 0x80000>; + + clocks = <&xo_board>, + <&xo_board>, + <&dsi0_phy 1>, + <&dsi0_phy 0>, + <&dsi1_phy 1>, + <&dsi1_phy 0>; + + clock-names = "xo", + "xo_a", + "dsi0pll", + "dsi0pllbyte", + "dsi1pll", + "dsi1pllbyte"; + + vdd_gfx-supply = <&pm8004_s5>; + }; +... diff --git a/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml b/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml index 72212970e6f5..8406dde17937 100644 --- a/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml @@ -22,10 +22,12 @@ properties: - qcom,sc8180x-rpmh-clk - qcom,sdm845-rpmh-clk - qcom,sdx55-rpmh-clk + - qcom,sdx65-rpmh-clk - qcom,sm6350-rpmh-clk - qcom,sm8150-rpmh-clk - qcom,sm8250-rpmh-clk - qcom,sm8350-rpmh-clk + - qcom,sm8450-rpmh-clk clocks: maxItems: 1 diff --git a/Documentation/devicetree/bindings/clock/renesas,cpg-mssr.yaml b/Documentation/devicetree/bindings/clock/renesas,cpg-mssr.yaml index 9b414fbde6d7..e0b86214f0f5 100644 --- a/Documentation/devicetree/bindings/clock/renesas,cpg-mssr.yaml +++ b/Documentation/devicetree/bindings/clock/renesas,cpg-mssr.yaml @@ -48,6 +48,7 @@ properties: - renesas,r8a77990-cpg-mssr # R-Car E3 - renesas,r8a77995-cpg-mssr # R-Car D3 - renesas,r8a779a0-cpg-mssr # R-Car V3U + - renesas,r8a779f0-cpg-mssr # R-Car S4-8 reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/clock/samsung,exynos7885-clock.yaml b/Documentation/devicetree/bindings/clock/samsung,exynos7885-clock.yaml new file mode 100644 index 000000000000..7e5a9cac2fd2 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/samsung,exynos7885-clock.yaml @@ -0,0 +1,166 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/samsung,exynos7885-clock.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Samsung Exynos7885 SoC clock controller + +maintainers: + - Dávid Virág + - Chanwoo Choi + - Krzysztof Kozlowski + - Sylwester Nawrocki + - Tomasz Figa + +description: | + Exynos7885 clock controller is comprised of several CMU units, generating + clocks for different domains. Those CMU units are modeled as separate device + tree nodes, and might depend on each other. The root clock in that root tree + is an external clock: OSCCLK (26 MHz). This external clock must be defined + as a fixed-rate clock in dts. + + CMU_TOP is a top-level CMU, where all base clocks are prepared using PLLs and + dividers; all other leaf clocks (other CMUs) are usually derived from CMU_TOP. + + Each clock is assigned an identifier and client nodes can use this identifier + to specify the clock which they consume. All clocks available for usage + in clock consumer nodes are defined as preprocessor macros in + 'dt-bindings/clock/exynos7885.h' header. + +properties: + compatible: + enum: + - samsung,exynos7885-cmu-top + - samsung,exynos7885-cmu-core + - samsung,exynos7885-cmu-peri + + clocks: + minItems: 1 + maxItems: 10 + + clock-names: + minItems: 1 + maxItems: 10 + + "#clock-cells": + const: 1 + + reg: + maxItems: 1 + +allOf: + - if: + properties: + compatible: + contains: + const: samsung,exynos7885-cmu-top + + then: + properties: + clocks: + items: + - description: External reference clock (26 MHz) + + clock-names: + items: + - const: oscclk + + - if: + properties: + compatible: + contains: + const: samsung,exynos7885-cmu-core + + then: + properties: + clocks: + items: + - description: External reference clock (26 MHz) + - description: CMU_CORE bus clock (from CMU_TOP) + - description: CCI clock (from CMU_TOP) + - description: G3D clock (from CMU_TOP) + + clock-names: + items: + - const: oscclk + - const: dout_core_bus + - const: dout_core_cci + - const: dout_core_g3d + + - if: + properties: + compatible: + contains: + const: samsung,exynos7885-cmu-peri + + then: + properties: + clocks: + items: + - description: External reference clock (26 MHz) + - description: CMU_PERI bus clock (from CMU_TOP) + - description: SPI0 clock (from CMU_TOP) + - description: SPI1 clock (from CMU_TOP) + - description: UART0 clock (from CMU_TOP) + - description: UART1 clock (from CMU_TOP) + - description: UART2 clock (from CMU_TOP) + - description: USI0 clock (from CMU_TOP) + - description: USI1 clock (from CMU_TOP) + - description: USI2 clock (from CMU_TOP) + + clock-names: + items: + - const: oscclk + - const: dout_peri_bus + - const: dout_peri_spi0 + - const: dout_peri_spi1 + - const: dout_peri_uart0 + - const: dout_peri_uart1 + - const: dout_peri_uart2 + - const: dout_peri_usi0 + - const: dout_peri_usi1 + - const: dout_peri_usi2 + +required: + - compatible + - "#clock-cells" + - clocks + - clock-names + - reg + +additionalProperties: false + +examples: + # Clock controller node for CMU_PERI + - | + #include + + cmu_peri: clock-controller@10010000 { + compatible = "samsung,exynos7885-cmu-peri"; + reg = <0x10010000 0x8000>; + #clock-cells = <1>; + + clocks = <&oscclk>, + <&cmu_top CLK_DOUT_PERI_BUS>, + <&cmu_top CLK_DOUT_PERI_SPI0>, + <&cmu_top CLK_DOUT_PERI_SPI1>, + <&cmu_top CLK_DOUT_PERI_UART0>, + <&cmu_top CLK_DOUT_PERI_UART1>, + <&cmu_top CLK_DOUT_PERI_UART2>, + <&cmu_top CLK_DOUT_PERI_USI0>, + <&cmu_top CLK_DOUT_PERI_USI1>, + <&cmu_top CLK_DOUT_PERI_USI2>; + clock-names = "oscclk", + "dout_peri_bus", + "dout_peri_spi0", + "dout_peri_spi1", + "dout_peri_uart0", + "dout_peri_uart1", + "dout_peri_uart2", + "dout_peri_usi0", + "dout_peri_usi1", + "dout_peri_usi2"; + }; + +... diff --git a/Documentation/devicetree/bindings/clock/samsung,exynos850-clock.yaml b/Documentation/devicetree/bindings/clock/samsung,exynos850-clock.yaml index 7f8c91a29b91..80ba60838f2b 100644 --- a/Documentation/devicetree/bindings/clock/samsung,exynos850-clock.yaml +++ b/Documentation/devicetree/bindings/clock/samsung,exynos850-clock.yaml @@ -32,6 +32,8 @@ properties: compatible: enum: - samsung,exynos850-cmu-top + - samsung,exynos850-cmu-apm + - samsung,exynos850-cmu-cmgp - samsung,exynos850-cmu-core - samsung,exynos850-cmu-dpu - samsung,exynos850-cmu-hsi @@ -68,6 +70,42 @@ allOf: items: - const: oscclk + - if: + properties: + compatible: + contains: + const: samsung,exynos850-cmu-apm + + then: + properties: + clocks: + items: + - description: External reference clock (26 MHz) + - description: CMU_APM bus clock (from CMU_TOP) + + clock-names: + items: + - const: oscclk + - const: dout_clkcmu_apm_bus + + - if: + properties: + compatible: + contains: + const: samsung,exynos850-cmu-cmgp + + then: + properties: + clocks: + items: + - description: External reference clock (26 MHz) + - description: CMU_CMGP bus clock (from CMU_APM) + + clock-names: + items: + - const: oscclk + - const: gout_clkcmu_cmgp_bus + - if: properties: compatible: diff --git a/Documentation/devicetree/bindings/clock/toshiba,tmpv770x-pipllct.yaml b/Documentation/devicetree/bindings/clock/toshiba,tmpv770x-pipllct.yaml new file mode 100644 index 000000000000..7b7300ce96d6 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/toshiba,tmpv770x-pipllct.yaml @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/toshiba,tmpv770x-pipllct.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Toshiba Visconti5 TMPV770X PLL Controller Device Tree Bindings + +maintainers: + - Nobuhiro Iwamatsu + +description: + Toshia Visconti5 PLL controller which supports the PLLs on TMPV770X. + +properties: + compatible: + const: toshiba,tmpv7708-pipllct + + reg: + maxItems: 1 + + '#clock-cells': + const: 1 + + clocks: + description: External reference clock (OSC2) + maxItems: 1 + +required: + - compatible + - reg + - "#clock-cells" + - clocks + +additionalProperties: false + +examples: + - | + + osc2_clk: osc2-clk { + compatible = "fixed-clock"; + clock-frequency = <20000000>; + #clock-cells = <0>; + }; + + soc { + #address-cells = <2>; + #size-cells = <2>; + + pipllct: clock-controller@24220000 { + compatible = "toshiba,tmpv7708-pipllct"; + reg = <0 0x24220000 0 0x820>; + #clock-cells = <1>; + clocks = <&osc2_clk>; + }; + }; +... diff --git a/Documentation/devicetree/bindings/clock/toshiba,tmpv770x-pismu.yaml b/Documentation/devicetree/bindings/clock/toshiba,tmpv770x-pismu.yaml new file mode 100644 index 000000000000..ed79f16fe6bc --- /dev/null +++ b/Documentation/devicetree/bindings/clock/toshiba,tmpv770x-pismu.yaml @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/toshiba,tmpv770x-pismu.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Toshiba Visconti5 TMPV770x SMU controller Device Tree Bindings + +maintainers: + - Nobuhiro Iwamatsu + +description: + Toshia Visconti5 SMU (System Management Unit) which supports the clock + and resets on TMPV770x. + +properties: + compatible: + items: + - const: toshiba,tmpv7708-pismu + - const: syscon + + reg: + maxItems: 1 + + '#clock-cells': + const: 1 + + '#reset-cells': + const: 1 + +required: + - compatible + - reg + - "#clock-cells" + - "#reset-cells" + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + pismu: syscon@24200000 { + compatible = "toshiba,tmpv7708-pismu", "syscon"; + reg = <0 0x24200000 0 0x2140>; + #clock-cells = <1>; + #reset-cells = <1>; + }; + }; +... diff --git a/Documentation/devicetree/bindings/leds/leds-mt6360.yaml b/Documentation/devicetree/bindings/leds/leds-mt6360.yaml new file mode 100644 index 000000000000..b2fe6eb89389 --- /dev/null +++ b/Documentation/devicetree/bindings/leds/leds-mt6360.yaml @@ -0,0 +1,159 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/leds/leds-mt6360.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: LED driver for MT6360 PMIC from MediaTek Integrated. + +maintainers: + - Gene Chen + +description: | + This module is part of the MT6360 MFD device. + see Documentation/devicetree/bindings/mfd/mt6360.yaml + Add MT6360 LED driver include 2-channel Flash LED with torch/strobe mode, + and 4-channel RGB LED support Register/Flash/Breath Mode + +properties: + compatible: + const: mediatek,mt6360-led + + "#address-cells": + const: 1 + + "#size-cells": + const: 0 + +patternProperties: + "^(multi-)?led@[0-5]$": + type: object + $ref: common.yaml# + description: + Properties for a single LED. + + properties: + reg: + description: Index of the LED. + enum: + - 0 # LED output ISINK1 + - 1 # LED output ISINK2 + - 2 # LED output ISINK3 + - 3 # LED output ISINKML + - 4 # LED output FLASH1 + - 5 # LED output FLASH2 + +unevaluatedProperties: false + +required: + - compatible + - "#address-cells" + - "#size-cells" + +additionalProperties: false + +examples: + - | + #include + led-controller { + compatible = "mediatek,mt6360-led"; + #address-cells = <1>; + #size-cells = <0>; + + multi-led@0 { + reg = <0>; + function = LED_FUNCTION_INDICATOR; + color = ; + led-max-microamp = <24000>; + #address-cells = <1>; + #size-cells = <0>; + led@0 { + reg = <0>; + color = ; + }; + led@1 { + reg = <1>; + color = ; + }; + led@2 { + reg = <2>; + color = ; + }; + }; + led@3 { + reg = <3>; + function = LED_FUNCTION_INDICATOR; + color = ; + led-max-microamp = <150000>; + }; + led@4 { + reg = <4>; + function = LED_FUNCTION_FLASH; + color = ; + function-enumerator = <1>; + led-max-microamp = <200000>; + flash-max-microamp = <500000>; + flash-max-timeout-us = <1024000>; + }; + led@5 { + reg = <5>; + function = LED_FUNCTION_FLASH; + color = ; + function-enumerator = <2>; + led-max-microamp = <200000>; + flash-max-microamp = <500000>; + flash-max-timeout-us = <1024000>; + }; + }; + + - | + + led-controller { + compatible = "mediatek,mt6360-led"; + #address-cells = <1>; + #size-cells = <0>; + + led@0 { + reg = <0>; + function = LED_FUNCTION_INDICATOR; + color = ; + led-max-microamp = <24000>; + }; + led@1 { + reg = <1>; + function = LED_FUNCTION_INDICATOR; + color = ; + led-max-microamp = <24000>; + }; + led@2 { + reg = <2>; + function = LED_FUNCTION_INDICATOR; + color = ; + led-max-microamp = <24000>; + }; + led@3 { + reg = <3>; + function = LED_FUNCTION_INDICATOR; + color = ; + led-max-microamp = <150000>; + }; + led@4 { + reg = <4>; + function = LED_FUNCTION_FLASH; + color = ; + function-enumerator = <1>; + led-max-microamp = <200000>; + flash-max-microamp = <500000>; + flash-max-timeout-us = <1024000>; + }; + led@5 { + reg = <5>; + function = LED_FUNCTION_FLASH; + color = ; + function-enumerator = <2>; + led-max-microamp = <200000>; + flash-max-microamp = <500000>; + flash-max-timeout-us = <1024000>; + }; + }; +... diff --git a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml index c4255f42e801..5c5c328b3134 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml @@ -27,14 +27,20 @@ properties: for example for the display controller, the system management controller and the NVMe coprocessor. items: - - const: apple,t8103-asc-mailbox + - enum: + - apple,t8103-asc-mailbox + - apple,t6000-asc-mailbox + - const: apple,asc-mailbox-v4 - description: M3 mailboxes are an older variant with a slightly different MMIO interface still found on the M1. It is used for the Thunderbolt co-processors. items: - - const: apple,t8103-m3-mailbox + - enum: + - apple,t8103-m3-mailbox + - apple,t6000-m3-mailbox + - const: apple,m3-mailbox-v2 reg: maxItems: 1 @@ -71,7 +77,7 @@ additionalProperties: false examples: - | mailbox@77408000 { - compatible = "apple,t8103-asc-mailbox"; + compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; reg = <0x77408000 0x4000>; interrupts = <1 583 4>, <1 584 4>, <1 585 4>, <1 586 4>; interrupt-names = "send-empty", "send-not-empty", diff --git a/Documentation/devicetree/bindings/timer/cdns,ttc.yaml b/Documentation/devicetree/bindings/timer/cdns,ttc.yaml index 8615353f69b4..c3386076a98c 100644 --- a/Documentation/devicetree/bindings/timer/cdns,ttc.yaml +++ b/Documentation/devicetree/bindings/timer/cdns,ttc.yaml @@ -25,6 +25,9 @@ properties: clocks: maxItems: 1 + power-domains: + maxItems: 1 + timer-width: $ref: "/schemas/types.yaml#/definitions/uint32" description: | diff --git a/Documentation/devicetree/bindings/timer/mstar,msc313e-timer.yaml b/Documentation/devicetree/bindings/timer/mstar,msc313e-timer.yaml new file mode 100644 index 000000000000..03d5dba5d5b3 --- /dev/null +++ b/Documentation/devicetree/bindings/timer/mstar,msc313e-timer.yaml @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/timer/mstar,msc313e-timer.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Mstar MSC313e Timer Device Tree Bindings + +maintainers: + - Daniel Palmer + - Romain Perier + +properties: + compatible: + enum: + - mstar,msc313e-timer + - sstar,ssd20xd-timer + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + - clocks + +additionalProperties: false + +examples: + - | + #include + + timer@6040 { + compatible = "mstar,msc313e-timer"; + reg = <0x6040 0x40>; + clocks = <&xtal_div2>; + interrupts-extended = <&intc_fiq GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>; + }; +... diff --git a/Documentation/devicetree/bindings/timer/nxp,tpm-timer.yaml b/Documentation/devicetree/bindings/timer/nxp,tpm-timer.yaml index edd9585f6726..f69773a8e4b9 100644 --- a/Documentation/devicetree/bindings/timer/nxp,tpm-timer.yaml +++ b/Documentation/devicetree/bindings/timer/nxp,tpm-timer.yaml @@ -19,7 +19,11 @@ description: | properties: compatible: - const: fsl,imx7ulp-tpm + oneOf: + - const: fsl,imx7ulp-tpm + - items: + - const: fsl,imx8ulp-tpm + - const: fsl,imx7ulp-tpm reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/timer/renesas,ostm.yaml b/Documentation/devicetree/bindings/timer/renesas,ostm.yaml index 600d47ab7d58..7fa7f977b44c 100644 --- a/Documentation/devicetree/bindings/timer/renesas,ostm.yaml +++ b/Documentation/devicetree/bindings/timer/renesas,ostm.yaml @@ -21,9 +21,10 @@ properties: compatible: items: - enum: - - renesas,r7s72100-ostm # RZ/A1H - - renesas,r7s9210-ostm # RZ/A2M - - const: renesas,ostm # Generic + - renesas,r7s72100-ostm # RZ/A1H + - renesas,r7s9210-ostm # RZ/A2M + - renesas,r9a07g044-ostm # RZ/G2{L,LC} + - const: renesas,ostm # Generic reg: maxItems: 1 @@ -37,6 +38,9 @@ properties: power-domains: maxItems: 1 + resets: + maxItems: 1 + required: - compatible - reg @@ -44,6 +48,16 @@ required: - clocks - power-domains +if: + properties: + compatible: + contains: + enum: + - renesas,r9a07g044-ostm +then: + required: + - resets + additionalProperties: false examples: diff --git a/Documentation/devicetree/bindings/timer/rockchip,rk-timer.yaml b/Documentation/devicetree/bindings/timer/rockchip,rk-timer.yaml index e26ecb5893ae..5d157d87dad5 100644 --- a/Documentation/devicetree/bindings/timer/rockchip,rk-timer.yaml +++ b/Documentation/devicetree/bindings/timer/rockchip,rk-timer.yaml @@ -18,7 +18,6 @@ properties: - enum: - rockchip,rv1108-timer - rockchip,rk3036-timer - - rockchip,rk3066-timer - rockchip,rk3188-timer - rockchip,rk3228-timer - rockchip,rk3229-timer diff --git a/Documentation/driver-api/pci/pci.rst b/Documentation/driver-api/pci/pci.rst index ca85e5e78b2c..4843cfad4f60 100644 --- a/Documentation/driver-api/pci/pci.rst +++ b/Documentation/driver-api/pci/pci.rst @@ -13,7 +13,7 @@ PCI Support Library .. kernel-doc:: drivers/pci/search.c :export: -.. kernel-doc:: drivers/pci/msi.c +.. kernel-doc:: drivers/pci/msi/msi.c :export: .. kernel-doc:: drivers/pci/bus.c diff --git a/MAINTAINERS b/MAINTAINERS index e4d18cf54b36..6485b9dd30e2 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2304,6 +2304,7 @@ F: Documentation/devicetree/bindings/gpio/mstar,msc313-gpio.yaml F: arch/arm/boot/dts/mstar-* F: arch/arm/mach-mstar/ F: drivers/clk/mstar/ +F: drivers/clocksource/timer-msc313e.c F: drivers/gpio/gpio-msc313.c F: drivers/rtc/rtc-msc313.c F: drivers/watchdog/msc313e_wdt.c @@ -2807,12 +2808,15 @@ L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) S: Supported T: git git://git.kernel.org/pub/scm/linux/kernel/git/iwamatsu/linux-visconti.git F: Documentation/devicetree/bindings/arm/toshiba.yaml +F: Documentation/devicetree/bindings/clock/toshiba,tmpv770x-pipllct.yaml +F: Documentation/devicetree/bindings/clock/toshiba,tmpv770x-pismu.yaml F: Documentation/devicetree/bindings/net/toshiba,visconti-dwmac.yaml F: Documentation/devicetree/bindings/gpio/toshiba,gpio-visconti.yaml F: Documentation/devicetree/bindings/pci/toshiba,visconti-pcie.yaml F: Documentation/devicetree/bindings/pinctrl/toshiba,visconti-pinctrl.yaml F: Documentation/devicetree/bindings/watchdog/toshiba,visconti-wdt.yaml F: arch/arm64/boot/dts/toshiba/ +F: drivers/clk/visconti/ F: drivers/net/ethernet/stmicro/stmmac/dwmac-visconti.c F: drivers/gpio/gpio-visconti.c F: drivers/pci/controller/dwc/pcie-visconti.c @@ -15875,6 +15879,15 @@ F: Documentation/admin-guide/media/qcom_camss.rst F: Documentation/devicetree/bindings/media/*camss* F: drivers/media/platform/qcom/camss/ +QUALCOMM CLOCK DRIVERS +M: Bjorn Andersson +L: linux-arm-msm@vger.kernel.org +S: Supported +T: git git://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux.git +F: Documentation/devicetree/bindings/clock/qcom,* +F: drivers/clk/qcom/ +F: include/dt-bindings/clock/qcom,* + QUALCOMM CORE POWER REDUCTION (CPR) AVS DRIVER M: Niklas Cassel L: linux-pm@vger.kernel.org diff --git a/arch/arm/boot/dts/omap3-n900.dts b/arch/arm/boot/dts/omap3-n900.dts index 32335d4ce478..d40c3d2c4914 100644 --- a/arch/arm/boot/dts/omap3-n900.dts +++ b/arch/arm/boot/dts/omap3-n900.dts @@ -8,6 +8,7 @@ #include "omap34xx.dtsi" #include +#include /* * Default secure signed bootloader (Nokia X-Loader) does not enable L3 firewall @@ -630,63 +631,92 @@ }; lp5523: lp5523@32 { + #address-cells = <1>; + #size-cells = <0>; compatible = "national,lp5523"; reg = <0x32>; clock-mode = /bits/ 8 <0>; /* LP55XX_CLOCK_AUTO */ - enable-gpio = <&gpio2 9 GPIO_ACTIVE_HIGH>; /* 41 */ + enable-gpios = <&gpio2 9 GPIO_ACTIVE_HIGH>; /* 41 */ - chan0 { + led@0 { + reg = <0>; chan-name = "lp5523:kb1"; led-cur = /bits/ 8 <50>; max-cur = /bits/ 8 <100>; + color = ; + function = LED_FUNCTION_KBD_BACKLIGHT; }; - chan1 { + led@1 { + reg = <1>; chan-name = "lp5523:kb2"; led-cur = /bits/ 8 <50>; max-cur = /bits/ 8 <100>; + color = ; + function = LED_FUNCTION_KBD_BACKLIGHT; }; - chan2 { + led@2 { + reg = <2>; chan-name = "lp5523:kb3"; led-cur = /bits/ 8 <50>; max-cur = /bits/ 8 <100>; + color = ; + function = LED_FUNCTION_KBD_BACKLIGHT; }; - chan3 { + led@3 { + reg = <3>; chan-name = "lp5523:kb4"; led-cur = /bits/ 8 <50>; max-cur = /bits/ 8 <100>; + color = ; + function = LED_FUNCTION_KBD_BACKLIGHT; }; - chan4 { + led@4 { + reg = <4>; chan-name = "lp5523:b"; led-cur = /bits/ 8 <50>; max-cur = /bits/ 8 <100>; + color = ; + function = LED_FUNCTION_STATUS; }; - chan5 { + led@5 { + reg = <5>; chan-name = "lp5523:g"; led-cur = /bits/ 8 <50>; max-cur = /bits/ 8 <100>; + color = ; + function = LED_FUNCTION_STATUS; }; - chan6 { + led@6 { + reg = <6>; chan-name = "lp5523:r"; led-cur = /bits/ 8 <50>; max-cur = /bits/ 8 <100>; + color = ; + function = LED_FUNCTION_STATUS; }; - chan7 { + led@7 { + reg = <7>; chan-name = "lp5523:kb5"; led-cur = /bits/ 8 <50>; max-cur = /bits/ 8 <100>; + color = ; + function = LED_FUNCTION_KBD_BACKLIGHT; }; - chan8 { + led@8 { + reg = <8>; chan-name = "lp5523:kb6"; led-cur = /bits/ 8 <50>; max-cur = /bits/ 8 <100>; + color = ; + function = LED_FUNCTION_KBD_BACKLIGHT; }; }; diff --git a/arch/mips/pci/msi-octeon.c b/arch/mips/pci/msi-octeon.c index 288b58b00dc8..c2860ebbd863 100644 --- a/arch/mips/pci/msi-octeon.c +++ b/arch/mips/pci/msi-octeon.c @@ -68,6 +68,9 @@ int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc) u64 search_mask; int index; + if (desc->pci.msi_attrib.is_msix) + return -EINVAL; + /* * Read the MSI config to figure out how many IRQs this device * wants. Most devices only want 1, which will give @@ -182,35 +185,6 @@ msi_irq_allocated: return 0; } -int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) -{ - struct msi_desc *entry; - int ret; - - /* - * MSI-X is not supported. - */ - if (type == PCI_CAP_ID_MSIX) - return -EINVAL; - - /* - * If an architecture wants to support multiple MSI, it needs to - * override arch_setup_msi_irqs() - */ - if (type == PCI_CAP_ID_MSI && nvec > 1) - return 1; - - for_each_pci_msi_entry(entry, dev) { - ret = arch_setup_msi_irq(dev, entry); - if (ret < 0) - return ret; - if (ret > 0) - return -ENOSPC; - } - - return 0; -} - /** * Called when a device no longer needs its MSI interrupts. All * MSI interrupts for the device are freed. diff --git a/arch/powerpc/boot/dts/bluestone.dts b/arch/powerpc/boot/dts/bluestone.dts index aa1ae94cd776..6971595319c1 100644 --- a/arch/powerpc/boot/dts/bluestone.dts +++ b/arch/powerpc/boot/dts/bluestone.dts @@ -366,30 +366,5 @@ 0x0 0x0 0x0 0x3 &UIC3 0xe 0x4 /* swizzled int C */ 0x0 0x0 0x0 0x4 &UIC3 0xf 0x4 /* swizzled int D */>; }; - - MSI: ppc4xx-msi@C10000000 { - compatible = "amcc,ppc4xx-msi", "ppc4xx-msi"; - reg = < 0xC 0x10000000 0x100 - 0xC 0x10000000 0x100>; - sdr-base = <0x36C>; - msi-data = <0x00004440>; - msi-mask = <0x0000ffe0>; - interrupts =<0 1 2 3 4 5 6 7>; - interrupt-parent = <&MSI>; - #interrupt-cells = <1>; - #address-cells = <0>; - #size-cells = <0>; - msi-available-ranges = <0x0 0x100>; - interrupt-map = < - 0 &UIC3 0x18 1 - 1 &UIC3 0x19 1 - 2 &UIC3 0x1A 1 - 3 &UIC3 0x1B 1 - 4 &UIC3 0x1C 1 - 5 &UIC3 0x1D 1 - 6 &UIC3 0x1E 1 - 7 &UIC3 0x1F 1 - >; - }; }; }; diff --git a/arch/powerpc/boot/dts/canyonlands.dts b/arch/powerpc/boot/dts/canyonlands.dts index c5fbb08e0a6e..5db1bff6b23d 100644 --- a/arch/powerpc/boot/dts/canyonlands.dts +++ b/arch/powerpc/boot/dts/canyonlands.dts @@ -544,23 +544,5 @@ 0x0 0x0 0x0 0x3 &UIC3 0x12 0x4 /* swizzled int C */ 0x0 0x0 0x0 0x4 &UIC3 0x13 0x4 /* swizzled int D */>; }; - - MSI: ppc4xx-msi@C10000000 { - compatible = "amcc,ppc4xx-msi", "ppc4xx-msi"; - reg = < 0xC 0x10000000 0x100>; - sdr-base = <0x36C>; - msi-data = <0x00000000>; - msi-mask = <0x44440000>; - interrupt-count = <3>; - interrupts = <0 1 2 3>; - interrupt-parent = <&UIC3>; - #interrupt-cells = <1>; - #address-cells = <0>; - #size-cells = <0>; - interrupt-map = <0 &UIC3 0x18 1 - 1 &UIC3 0x19 1 - 2 &UIC3 0x1A 1 - 3 &UIC3 0x1B 1>; - }; }; }; diff --git a/arch/powerpc/boot/dts/katmai.dts b/arch/powerpc/boot/dts/katmai.dts index a8f353229fb7..4262b2bbd6de 100644 --- a/arch/powerpc/boot/dts/katmai.dts +++ b/arch/powerpc/boot/dts/katmai.dts @@ -442,24 +442,6 @@ 0x0 0x0 0x0 0x4 &UIC3 0xb 0x4 /* swizzled int D */>; }; - MSI: ppc4xx-msi@400300000 { - compatible = "amcc,ppc4xx-msi", "ppc4xx-msi"; - reg = < 0x4 0x00300000 0x100>; - sdr-base = <0x3B0>; - msi-data = <0x00000000>; - msi-mask = <0x44440000>; - interrupt-count = <3>; - interrupts =<0 1 2 3>; - interrupt-parent = <&UIC0>; - #interrupt-cells = <1>; - #address-cells = <0>; - #size-cells = <0>; - interrupt-map = <0 &UIC0 0xC 1 - 1 &UIC0 0x0D 1 - 2 &UIC0 0x0E 1 - 3 &UIC0 0x0F 1>; - }; - I2O: i2o@400100000 { compatible = "ibm,i2o-440spe"; reg = <0x00000004 0x00100000 0x100>; diff --git a/arch/powerpc/boot/dts/kilauea.dts b/arch/powerpc/boot/dts/kilauea.dts index a709fb47a180..c07a7525a72c 100644 --- a/arch/powerpc/boot/dts/kilauea.dts +++ b/arch/powerpc/boot/dts/kilauea.dts @@ -403,33 +403,5 @@ 0x0 0x0 0x0 0x3 &UIC2 0xd 0x4 /* swizzled int C */ 0x0 0x0 0x0 0x4 &UIC2 0xe 0x4 /* swizzled int D */>; }; - - MSI: ppc4xx-msi@C10000000 { - compatible = "amcc,ppc4xx-msi", "ppc4xx-msi"; - reg = <0xEF620000 0x100>; - sdr-base = <0x4B0>; - msi-data = <0x00000000>; - msi-mask = <0x44440000>; - interrupt-count = <12>; - interrupts = <0 1 2 3 4 5 6 7 8 9 0xA 0xB 0xC 0xD>; - interrupt-parent = <&UIC2>; - #interrupt-cells = <1>; - #address-cells = <0>; - #size-cells = <0>; - interrupt-map = <0 &UIC2 0x10 1 - 1 &UIC2 0x11 1 - 2 &UIC2 0x12 1 - 2 &UIC2 0x13 1 - 2 &UIC2 0x14 1 - 2 &UIC2 0x15 1 - 2 &UIC2 0x16 1 - 2 &UIC2 0x17 1 - 2 &UIC2 0x18 1 - 2 &UIC2 0x19 1 - 2 &UIC2 0x1A 1 - 2 &UIC2 0x1B 1 - 2 &UIC2 0x1C 1 - 3 &UIC2 0x1D 1>; - }; }; }; diff --git a/arch/powerpc/boot/dts/redwood.dts b/arch/powerpc/boot/dts/redwood.dts index f38035a1f4a1..3c849e23e5f3 100644 --- a/arch/powerpc/boot/dts/redwood.dts +++ b/arch/powerpc/boot/dts/redwood.dts @@ -358,25 +358,6 @@ 0x0 0x0 0x0 0x4 &UIC3 0xb 0x4 /* swizzled int D */>; }; - MSI: ppc4xx-msi@400300000 { - compatible = "amcc,ppc4xx-msi", "ppc4xx-msi"; - reg = < 0x4 0x00300000 0x100 - 0x4 0x00300000 0x100>; - sdr-base = <0x3B0>; - msi-data = <0x00000000>; - msi-mask = <0x44440000>; - interrupt-count = <3>; - interrupts =<0 1 2 3>; - interrupt-parent = <&UIC0>; - #interrupt-cells = <1>; - #address-cells = <0>; - #size-cells = <0>; - interrupt-map = <0 &UIC0 0xC 1 - 1 &UIC0 0x0D 1 - 2 &UIC0 0x0E 1 - 3 &UIC0 0x0F 1>; - }; - }; diff --git a/arch/powerpc/platforms/40x/Kconfig b/arch/powerpc/platforms/40x/Kconfig index e3e5217c9822..614ea6dc994c 100644 --- a/arch/powerpc/platforms/40x/Kconfig +++ b/arch/powerpc/platforms/40x/Kconfig @@ -23,7 +23,6 @@ config KILAUEA select PPC4xx_PCI_EXPRESS select FORCE_PCI select PCI_MSI - select PPC4xx_MSI help This option enables support for the AMCC PPC405EX evaluation board. diff --git a/arch/powerpc/platforms/44x/Kconfig b/arch/powerpc/platforms/44x/Kconfig index 83975ef50975..25b80cd558f8 100644 --- a/arch/powerpc/platforms/44x/Kconfig +++ b/arch/powerpc/platforms/44x/Kconfig @@ -23,7 +23,6 @@ config BLUESTONE select APM821xx select FORCE_PCI select PCI_MSI - select PPC4xx_MSI select PPC4xx_PCI_EXPRESS select IBM_EMAC_RGMII if IBM_EMAC help @@ -73,7 +72,6 @@ config KATMAI select FORCE_PCI select PPC4xx_PCI_EXPRESS select PCI_MSI - select PPC4xx_MSI help This option enables support for the AMCC PPC440SPe evaluation board. @@ -115,7 +113,6 @@ config CANYONLANDS select FORCE_PCI select PPC4xx_PCI_EXPRESS select PCI_MSI - select PPC4xx_MSI select IBM_EMAC_RGMII if IBM_EMAC select IBM_EMAC_ZMII if IBM_EMAC help @@ -141,7 +138,6 @@ config REDWOOD select FORCE_PCI select PPC4xx_PCI_EXPRESS select PCI_MSI - select PPC4xx_MSI help This option enables support for the AMCC PPC460SX Redwood board. diff --git a/arch/powerpc/platforms/4xx/Makefile b/arch/powerpc/platforms/4xx/Makefile index d009d2e0b9e8..2071a0abe09b 100644 --- a/arch/powerpc/platforms/4xx/Makefile +++ b/arch/powerpc/platforms/4xx/Makefile @@ -3,6 +3,5 @@ obj-y += uic.o machine_check.o obj-$(CONFIG_4xx_SOC) += soc.o obj-$(CONFIG_PCI) += pci.o obj-$(CONFIG_PPC4xx_HSTA_MSI) += hsta_msi.o -obj-$(CONFIG_PPC4xx_MSI) += msi.o obj-$(CONFIG_PPC4xx_CPM) += cpm.o obj-$(CONFIG_PPC4xx_GPIO) += gpio.o diff --git a/arch/powerpc/platforms/4xx/hsta_msi.c b/arch/powerpc/platforms/4xx/hsta_msi.c index c950fed43b32..fee430eadcc6 100644 --- a/arch/powerpc/platforms/4xx/hsta_msi.c +++ b/arch/powerpc/platforms/4xx/hsta_msi.c @@ -47,7 +47,7 @@ static int hsta_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) return -EINVAL; } - for_each_pci_msi_entry(entry, dev) { + msi_for_each_desc(entry, &dev->dev, MSI_DESC_NOTASSOCIATED) { irq = msi_bitmap_alloc_hwirqs(&ppc4xx_hsta_msi.bmp, 1); if (irq < 0) { pr_debug("%s: Failed to allocate msi interrupt\n", @@ -105,10 +105,7 @@ static void hsta_teardown_msi_irqs(struct pci_dev *dev) struct msi_desc *entry; int irq; - for_each_pci_msi_entry(entry, dev) { - if (!entry->irq) - continue; - + msi_for_each_desc(entry, &dev->dev, MSI_DESC_ASSOCIATED) { irq = hsta_find_hwirq_offset(entry->irq); /* entry->irq should always be in irq_map */ diff --git a/arch/powerpc/platforms/4xx/msi.c b/arch/powerpc/platforms/4xx/msi.c deleted file mode 100644 index 1051564b94f2..000000000000 --- a/arch/powerpc/platforms/4xx/msi.c +++ /dev/null @@ -1,281 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Adding PCI-E MSI support for PPC4XX SoCs. - * - * Copyright (c) 2010, Applied Micro Circuits Corporation - * Authors: Tirumala R Marri - * Feng Kan - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define PEIH_TERMADH 0x00 -#define PEIH_TERMADL 0x08 -#define PEIH_MSIED 0x10 -#define PEIH_MSIMK 0x18 -#define PEIH_MSIASS 0x20 -#define PEIH_FLUSH0 0x30 -#define PEIH_FLUSH1 0x38 -#define PEIH_CNTRST 0x48 - -static int msi_irqs; - -struct ppc4xx_msi { - u32 msi_addr_lo; - u32 msi_addr_hi; - void __iomem *msi_regs; - int *msi_virqs; - struct msi_bitmap bitmap; - struct device_node *msi_dev; -}; - -static struct ppc4xx_msi ppc4xx_msi; - -static int ppc4xx_msi_init_allocator(struct platform_device *dev, - struct ppc4xx_msi *msi_data) -{ - int err; - - err = msi_bitmap_alloc(&msi_data->bitmap, msi_irqs, - dev->dev.of_node); - if (err) - return err; - - err = msi_bitmap_reserve_dt_hwirqs(&msi_data->bitmap); - if (err < 0) { - msi_bitmap_free(&msi_data->bitmap); - return err; - } - - return 0; -} - -static int ppc4xx_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) -{ - int int_no = -ENOMEM; - unsigned int virq; - struct msi_msg msg; - struct msi_desc *entry; - struct ppc4xx_msi *msi_data = &ppc4xx_msi; - - dev_dbg(&dev->dev, "PCIE-MSI:%s called. vec %x type %d\n", - __func__, nvec, type); - if (type == PCI_CAP_ID_MSIX) - pr_debug("ppc4xx msi: MSI-X untested, trying anyway.\n"); - - msi_data->msi_virqs = kmalloc_array(msi_irqs, sizeof(int), GFP_KERNEL); - if (!msi_data->msi_virqs) - return -ENOMEM; - - for_each_pci_msi_entry(entry, dev) { - int_no = msi_bitmap_alloc_hwirqs(&msi_data->bitmap, 1); - if (int_no >= 0) - break; - if (int_no < 0) { - pr_debug("%s: fail allocating msi interrupt\n", - __func__); - } - virq = irq_of_parse_and_map(msi_data->msi_dev, int_no); - if (!virq) { - dev_err(&dev->dev, "%s: fail mapping irq\n", __func__); - msi_bitmap_free_hwirqs(&msi_data->bitmap, int_no, 1); - return -ENOSPC; - } - dev_dbg(&dev->dev, "%s: virq = %d\n", __func__, virq); - - /* Setup msi address space */ - msg.address_hi = msi_data->msi_addr_hi; - msg.address_lo = msi_data->msi_addr_lo; - - irq_set_msi_desc(virq, entry); - msg.data = int_no; - pci_write_msi_msg(virq, &msg); - } - return 0; -} - -void ppc4xx_teardown_msi_irqs(struct pci_dev *dev) -{ - struct msi_desc *entry; - struct ppc4xx_msi *msi_data = &ppc4xx_msi; - irq_hw_number_t hwirq; - - dev_dbg(&dev->dev, "PCIE-MSI: tearing down msi irqs\n"); - - for_each_pci_msi_entry(entry, dev) { - if (!entry->irq) - continue; - hwirq = virq_to_hw(entry->irq); - irq_set_msi_desc(entry->irq, NULL); - irq_dispose_mapping(entry->irq); - msi_bitmap_free_hwirqs(&msi_data->bitmap, hwirq, 1); - } -} - -static int ppc4xx_setup_pcieh_hw(struct platform_device *dev, - struct resource res, struct ppc4xx_msi *msi) -{ - const u32 *msi_data; - const u32 *msi_mask; - const u32 *sdr_addr; - dma_addr_t msi_phys; - void *msi_virt; - int err; - - sdr_addr = of_get_property(dev->dev.of_node, "sdr-base", NULL); - if (!sdr_addr) - return -EINVAL; - - msi_data = of_get_property(dev->dev.of_node, "msi-data", NULL); - if (!msi_data) - return -EINVAL; - - msi_mask = of_get_property(dev->dev.of_node, "msi-mask", NULL); - if (!msi_mask) - return -EINVAL; - - msi->msi_dev = of_find_node_by_name(NULL, "ppc4xx-msi"); - if (!msi->msi_dev) - return -ENODEV; - - msi->msi_regs = of_iomap(msi->msi_dev, 0); - if (!msi->msi_regs) { - dev_err(&dev->dev, "of_iomap failed\n"); - err = -ENOMEM; - goto node_put; - } - dev_dbg(&dev->dev, "PCIE-MSI: msi register mapped 0x%x 0x%x\n", - (u32) (msi->msi_regs + PEIH_TERMADH), (u32) (msi->msi_regs)); - - msi_virt = dma_alloc_coherent(&dev->dev, 64, &msi_phys, GFP_KERNEL); - if (!msi_virt) { - err = -ENOMEM; - goto iounmap; - } - msi->msi_addr_hi = upper_32_bits(msi_phys); - msi->msi_addr_lo = lower_32_bits(msi_phys & 0xffffffff); - dev_dbg(&dev->dev, "PCIE-MSI: msi address high 0x%x, low 0x%x\n", - msi->msi_addr_hi, msi->msi_addr_lo); - - mtdcri(SDR0, *sdr_addr, upper_32_bits(res.start)); /*HIGH addr */ - mtdcri(SDR0, *sdr_addr + 1, lower_32_bits(res.start)); /* Low addr */ - - /* Progam the Interrupt handler Termination addr registers */ - out_be32(msi->msi_regs + PEIH_TERMADH, msi->msi_addr_hi); - out_be32(msi->msi_regs + PEIH_TERMADL, msi->msi_addr_lo); - - /* Program MSI Expected data and Mask bits */ - out_be32(msi->msi_regs + PEIH_MSIED, *msi_data); - out_be32(msi->msi_regs + PEIH_MSIMK, *msi_mask); - - dma_free_coherent(&dev->dev, 64, msi_virt, msi_phys); - - return 0; - -iounmap: - iounmap(msi->msi_regs); -node_put: - of_node_put(msi->msi_dev); - return err; -} - -static int ppc4xx_of_msi_remove(struct platform_device *dev) -{ - struct ppc4xx_msi *msi = dev->dev.platform_data; - int i; - int virq; - - for (i = 0; i < msi_irqs; i++) { - virq = msi->msi_virqs[i]; - if (virq) - irq_dispose_mapping(virq); - } - - if (msi->bitmap.bitmap) - msi_bitmap_free(&msi->bitmap); - iounmap(msi->msi_regs); - of_node_put(msi->msi_dev); - - return 0; -} - -static int ppc4xx_msi_probe(struct platform_device *dev) -{ - struct ppc4xx_msi *msi; - struct resource res; - int err = 0; - struct pci_controller *phb; - - dev_dbg(&dev->dev, "PCIE-MSI: Setting up MSI support...\n"); - - msi = devm_kzalloc(&dev->dev, sizeof(*msi), GFP_KERNEL); - if (!msi) - return -ENOMEM; - dev->dev.platform_data = msi; - - /* Get MSI ranges */ - err = of_address_to_resource(dev->dev.of_node, 0, &res); - if (err) { - dev_err(&dev->dev, "%pOF resource error!\n", dev->dev.of_node); - return err; - } - - msi_irqs = of_irq_count(dev->dev.of_node); - if (!msi_irqs) - return -ENODEV; - - err = ppc4xx_setup_pcieh_hw(dev, res, msi); - if (err) - return err; - - err = ppc4xx_msi_init_allocator(dev, msi); - if (err) { - dev_err(&dev->dev, "Error allocating MSI bitmap\n"); - goto error_out; - } - ppc4xx_msi = *msi; - - list_for_each_entry(phb, &hose_list, list_node) { - phb->controller_ops.setup_msi_irqs = ppc4xx_setup_msi_irqs; - phb->controller_ops.teardown_msi_irqs = ppc4xx_teardown_msi_irqs; - } - return 0; - -error_out: - ppc4xx_of_msi_remove(dev); - return err; -} -static const struct of_device_id ppc4xx_msi_ids[] = { - { - .compatible = "amcc,ppc4xx-msi", - }, - {} -}; -static struct platform_driver ppc4xx_msi_driver = { - .probe = ppc4xx_msi_probe, - .remove = ppc4xx_of_msi_remove, - .driver = { - .name = "ppc4xx-msi", - .of_match_table = ppc4xx_msi_ids, - }, - -}; - -static __init int ppc4xx_msi_init(void) -{ - return platform_driver_register(&ppc4xx_msi_driver); -} - -subsys_initcall(ppc4xx_msi_init); diff --git a/arch/powerpc/platforms/cell/axon_msi.c b/arch/powerpc/platforms/cell/axon_msi.c index 82335e364c44..354a58c1e6f2 100644 --- a/arch/powerpc/platforms/cell/axon_msi.c +++ b/arch/powerpc/platforms/cell/axon_msi.c @@ -199,7 +199,6 @@ out_error: static int setup_msi_msg_address(struct pci_dev *dev, struct msi_msg *msg) { struct device_node *dn; - struct msi_desc *entry; int len; const u32 *prop; @@ -209,10 +208,8 @@ static int setup_msi_msg_address(struct pci_dev *dev, struct msi_msg *msg) return -ENODEV; } - entry = first_pci_msi_entry(dev); - for (; dn; dn = of_get_next_parent(dn)) { - if (entry->msi_attrib.is_64) { + if (!dev->no_64bit_msi) { prop = of_get_property(dn, "msi-address-64", &len); if (prop) break; @@ -265,7 +262,7 @@ static int axon_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) if (rc) return rc; - for_each_pci_msi_entry(entry, dev) { + msi_for_each_desc(entry, &dev->dev, MSI_DESC_NOTASSOCIATED) { virq = irq_create_direct_mapping(msic->irq_domain); if (!virq) { dev_warn(&dev->dev, @@ -288,10 +285,7 @@ static void axon_msi_teardown_msi_irqs(struct pci_dev *dev) dev_dbg(&dev->dev, "axon_msi: tearing down msi irqs\n"); - for_each_pci_msi_entry(entry, dev) { - if (!entry->irq) - continue; - + msi_for_each_desc(entry, &dev->dev, MSI_DESC_ASSOCIATED) { irq_set_msi_desc(entry->irq, NULL); irq_dispose_mapping(entry->irq); } diff --git a/arch/powerpc/platforms/pasemi/msi.c b/arch/powerpc/platforms/pasemi/msi.c index d38944a1e258..e8af72282682 100644 --- a/arch/powerpc/platforms/pasemi/msi.c +++ b/arch/powerpc/platforms/pasemi/msi.c @@ -62,17 +62,12 @@ static void pasemi_msi_teardown_msi_irqs(struct pci_dev *pdev) pr_debug("pasemi_msi_teardown_msi_irqs, pdev %p\n", pdev); - for_each_pci_msi_entry(entry, pdev) { - if (!entry->irq) - continue; - + msi_for_each_desc(entry, &pdev->dev, MSI_DESC_ASSOCIATED) { hwirq = virq_to_hw(entry->irq); irq_set_msi_desc(entry->irq, NULL); irq_dispose_mapping(entry->irq); msi_bitmap_free_hwirqs(&msi_mpic->msi_bitmap, hwirq, ALLOC_CHUNK); } - - return; } static int pasemi_msi_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) @@ -90,7 +85,7 @@ static int pasemi_msi_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) msg.address_hi = 0; msg.address_lo = PASEMI_MSI_ADDR; - for_each_pci_msi_entry(entry, pdev) { + msi_for_each_desc(entry, &pdev->dev, MSI_DESC_NOTASSOCIATED) { /* Allocate 16 interrupts for now, since that's the grouping for * affinity. This can be changed later if it turns out 32 is too * few MSIs for someone, but restrictions will apply to how the diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c index 004cd6a96c8a..8913c86009d9 100644 --- a/arch/powerpc/platforms/powernv/pci-ioda.c +++ b/arch/powerpc/platforms/powernv/pci-ioda.c @@ -2154,10 +2154,10 @@ static void pnv_msi_compose_msg(struct irq_data *d, struct msi_msg *msg) int rc; rc = __pnv_pci_ioda_msi_setup(phb, pdev, d->hwirq, - entry->msi_attrib.is_64, msg); + entry->pci.msi_attrib.is_64, msg); if (rc) dev_err(&pdev->dev, "Failed to setup %s-bit MSI #%ld : %d\n", - entry->msi_attrib.is_64 ? "64" : "32", d->hwirq, rc); + entry->pci.msi_attrib.is_64 ? "64" : "32", d->hwirq, rc); } /* diff --git a/arch/powerpc/platforms/pseries/msi.c b/arch/powerpc/platforms/pseries/msi.c index 8627362f613e..fb2919fd6bc0 100644 --- a/arch/powerpc/platforms/pseries/msi.c +++ b/arch/powerpc/platforms/pseries/msi.c @@ -321,27 +321,6 @@ out: return request; } -static int check_msix_entries(struct pci_dev *pdev) -{ - struct msi_desc *entry; - int expected; - - /* There's no way for us to express to firmware that we want - * a discontiguous, or non-zero based, range of MSI-X entries. - * So we must reject such requests. */ - - expected = 0; - for_each_pci_msi_entry(entry, pdev) { - if (entry->msi_attrib.entry_nr != expected) { - pr_debug("rtas_msi: bad MSI-X entries.\n"); - return -EINVAL; - } - expected++; - } - - return 0; -} - static void rtas_hack_32bit_msi_gen2(struct pci_dev *pdev) { u32 addr_hi, addr_lo; @@ -380,9 +359,6 @@ static int rtas_prepare_msi_irqs(struct pci_dev *pdev, int nvec_in, int type, if (quota && quota < nvec) return quota; - if (type == PCI_CAP_ID_MSIX && check_msix_entries(pdev)) - return -EINVAL; - /* * Firmware currently refuse any non power of two allocation * so we round up if the quota will allow it. @@ -448,8 +424,7 @@ static int pseries_msi_ops_prepare(struct irq_domain *domain, struct device *dev int nvec, msi_alloc_info_t *arg) { struct pci_dev *pdev = to_pci_dev(dev); - struct msi_desc *desc = first_pci_msi_entry(pdev); - int type = desc->msi_attrib.is_msix ? PCI_CAP_ID_MSIX : PCI_CAP_ID_MSI; + int type = pdev->msix_enabled ? PCI_CAP_ID_MSIX : PCI_CAP_ID_MSI; return rtas_prepare_msi_irqs(pdev, nvec, type, arg); } @@ -530,9 +505,16 @@ static struct irq_chip pseries_pci_msi_irq_chip = { .irq_write_msi_msg = pseries_msi_write_msg, }; + +/* + * Set MSI_FLAG_MSIX_CONTIGUOUS as there is no way to express to + * firmware to request a discontiguous or non-zero based range of + * MSI-X entries. Core code will reject such setup attempts. + */ static struct msi_domain_info pseries_msi_domain_info = { .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | - MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX), + MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX | + MSI_FLAG_MSIX_CONTIGUOUS), .ops = &pseries_pci_msi_domain_ops, .chip = &pseries_pci_msi_irq_chip, }; @@ -580,7 +562,7 @@ static int pseries_irq_domain_alloc(struct irq_domain *domain, unsigned int virq int hwirq; int i, ret; - hwirq = rtas_query_irq_number(pci_get_pdn(pdev), desc->msi_attrib.entry_nr); + hwirq = rtas_query_irq_number(pci_get_pdn(pdev), desc->msi_index); if (hwirq < 0) { dev_err(&pdev->dev, "Failed to query HW IRQ: %d\n", hwirq); return hwirq; diff --git a/arch/powerpc/sysdev/Kconfig b/arch/powerpc/sysdev/Kconfig index 9ebcc1337560..5aa92ff3622d 100644 --- a/arch/powerpc/sysdev/Kconfig +++ b/arch/powerpc/sysdev/Kconfig @@ -12,17 +12,11 @@ config PPC4xx_HSTA_MSI depends on PCI_MSI depends on PCI && 4xx -config PPC4xx_MSI - bool - depends on PCI_MSI - depends on PCI && 4xx - config PPC_MSI_BITMAP bool depends on PCI_MSI default y if MPIC default y if FSL_PCI - default y if PPC4xx_MSI default y if PPC_POWERNV source "arch/powerpc/sysdev/xics/Kconfig" diff --git a/arch/powerpc/sysdev/fsl_msi.c b/arch/powerpc/sysdev/fsl_msi.c index e6b06c3f8197..b3475ae9f236 100644 --- a/arch/powerpc/sysdev/fsl_msi.c +++ b/arch/powerpc/sysdev/fsl_msi.c @@ -125,17 +125,13 @@ static void fsl_teardown_msi_irqs(struct pci_dev *pdev) struct fsl_msi *msi_data; irq_hw_number_t hwirq; - for_each_pci_msi_entry(entry, pdev) { - if (!entry->irq) - continue; + msi_for_each_desc(entry, &pdev->dev, MSI_DESC_ASSOCIATED) { hwirq = virq_to_hw(entry->irq); msi_data = irq_get_chip_data(entry->irq); irq_set_msi_desc(entry->irq, NULL); irq_dispose_mapping(entry->irq); msi_bitmap_free_hwirqs(&msi_data->bitmap, hwirq, 1); } - - return; } static void fsl_compose_msi_msg(struct pci_dev *pdev, int hwirq, @@ -215,7 +211,7 @@ static int fsl_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) } } - for_each_pci_msi_entry(entry, pdev) { + msi_for_each_desc(entry, &pdev->dev, MSI_DESC_NOTASSOCIATED) { /* * Loop over all the MSI devices until we find one that has an * available interrupt. diff --git a/arch/powerpc/sysdev/mpic_u3msi.c b/arch/powerpc/sysdev/mpic_u3msi.c index 3861023d378a..fa53b6d85ef9 100644 --- a/arch/powerpc/sysdev/mpic_u3msi.c +++ b/arch/powerpc/sysdev/mpic_u3msi.c @@ -104,17 +104,12 @@ static void u3msi_teardown_msi_irqs(struct pci_dev *pdev) struct msi_desc *entry; irq_hw_number_t hwirq; - for_each_pci_msi_entry(entry, pdev) { - if (!entry->irq) - continue; - + msi_for_each_desc(entry, &pdev->dev, MSI_DESC_ASSOCIATED) { hwirq = virq_to_hw(entry->irq); irq_set_msi_desc(entry->irq, NULL); irq_dispose_mapping(entry->irq); msi_bitmap_free_hwirqs(&msi_mpic->msi_bitmap, hwirq, 1); } - - return; } static int u3msi_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) @@ -136,7 +131,7 @@ static int u3msi_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) return -ENXIO; } - for_each_pci_msi_entry(entry, pdev) { + msi_for_each_desc(entry, &pdev->dev, MSI_DESC_NOTASSOCIATED) { hwirq = msi_bitmap_alloc_hwirqs(&msi_mpic->msi_bitmap, 1); if (hwirq < 0) { pr_debug("u3msi: failed allocating hwirq\n"); diff --git a/arch/s390/pci/pci_irq.c b/arch/s390/pci/pci_irq.c index aefd306ea71e..2b6062c486f5 100644 --- a/arch/s390/pci/pci_irq.c +++ b/arch/s390/pci/pci_irq.c @@ -303,7 +303,7 @@ int arch_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) /* Request MSI interrupts */ hwirq = bit; - for_each_pci_msi_entry(msi, pdev) { + msi_for_each_desc(msi, &pdev->dev, MSI_DESC_NOTASSOCIATED) { rc = -EIO; if (hwirq - bit >= msi_vecs) break; @@ -362,9 +362,7 @@ void arch_teardown_msi_irqs(struct pci_dev *pdev) return; /* Release MSI interrupts */ - for_each_pci_msi_entry(msi, pdev) { - if (!msi->irq) - continue; + msi_for_each_desc(msi, &pdev->dev, MSI_DESC_ASSOCIATED) { irq_set_msi_desc(msi->irq, NULL); irq_free_desc(msi->irq); msi->msg.address_lo = 0; @@ -387,13 +385,13 @@ void arch_teardown_msi_irqs(struct pci_dev *pdev) airq_iv_free(zpci_ibv[0], zdev->msi_first_bit, zdev->msi_nr_irqs); } -void arch_restore_msi_irqs(struct pci_dev *pdev) +bool arch_restore_msi_irqs(struct pci_dev *pdev) { struct zpci_dev *zdev = to_zpci(pdev); if (!zdev->irqs_registered) zpci_set_irq(zdev); - default_restore_msi_irqs(pdev); + return true; } static struct airq_struct zpci_airq = { diff --git a/arch/sparc/kernel/pci_msi.c b/arch/sparc/kernel/pci_msi.c index fb5899cbfa51..9ed11985768e 100644 --- a/arch/sparc/kernel/pci_msi.c +++ b/arch/sparc/kernel/pci_msi.c @@ -146,13 +146,13 @@ static int sparc64_setup_msi_irq(unsigned int *irq_p, msiqid = pick_msiq(pbm); err = ops->msi_setup(pbm, msiqid, msi, - (entry->msi_attrib.is_64 ? 1 : 0)); + (entry->pci.msi_attrib.is_64 ? 1 : 0)); if (err) goto out_msi_free; pbm->msi_irq_table[msi - pbm->msi_first] = *irq_p; - if (entry->msi_attrib.is_64) { + if (entry->pci.msi_attrib.is_64) { msg.address_hi = pbm->msi64_start >> 32; msg.address_lo = pbm->msi64_start & 0xffffffff; } else { diff --git a/arch/x86/hyperv/irqdomain.c b/arch/x86/hyperv/irqdomain.c index 514fc64e23d5..7e0f6bedc248 100644 --- a/arch/x86/hyperv/irqdomain.c +++ b/arch/x86/hyperv/irqdomain.c @@ -253,64 +253,43 @@ static int hv_unmap_msi_interrupt(struct pci_dev *dev, struct hv_interrupt_entry return hv_unmap_interrupt(hv_build_pci_dev_id(dev).as_uint64, old_entry); } -static void hv_teardown_msi_irq_common(struct pci_dev *dev, struct msi_desc *msidesc, int irq) +static void hv_teardown_msi_irq(struct pci_dev *dev, struct irq_data *irqd) { - u64 status; struct hv_interrupt_entry old_entry; - struct irq_desc *desc; - struct irq_data *data; struct msi_msg msg; + u64 status; - desc = irq_to_desc(irq); - if (!desc) { - pr_debug("%s: no irq desc\n", __func__); - return; - } - - data = &desc->irq_data; - if (!data) { - pr_debug("%s: no irq data\n", __func__); - return; - } - - if (!data->chip_data) { + if (!irqd->chip_data) { pr_debug("%s: no chip data\n!", __func__); return; } - old_entry = *(struct hv_interrupt_entry *)data->chip_data; + old_entry = *(struct hv_interrupt_entry *)irqd->chip_data; entry_to_msi_msg(&old_entry, &msg); - kfree(data->chip_data); - data->chip_data = NULL; + kfree(irqd->chip_data); + irqd->chip_data = NULL; status = hv_unmap_msi_interrupt(dev, &old_entry); - if (status != HV_STATUS_SUCCESS) { + if (status != HV_STATUS_SUCCESS) pr_err("%s: hypercall failed, status %lld\n", __func__, status); - return; - } } -static void hv_msi_domain_free_irqs(struct irq_domain *domain, struct device *dev) +static void hv_msi_free_irq(struct irq_domain *domain, + struct msi_domain_info *info, unsigned int virq) { - int i; - struct msi_desc *entry; - struct pci_dev *pdev; + struct irq_data *irqd = irq_get_irq_data(virq); + struct msi_desc *desc; - if (WARN_ON_ONCE(!dev_is_pci(dev))) + if (!irqd) return; - pdev = to_pci_dev(dev); + desc = irq_data_get_msi_desc(irqd); + if (!desc || !desc->irq || WARN_ON_ONCE(!dev_is_pci(desc->dev))) + return; - for_each_pci_msi_entry(entry, pdev) { - if (entry->irq) { - for (i = 0; i < entry->nvec_used; i++) { - hv_teardown_msi_irq_common(pdev, entry, entry->irq + i); - irq_domain_free_irqs(entry->irq + i, 1); - } - } - } + hv_teardown_msi_irq(to_pci_dev(desc->dev), irqd); } /* @@ -329,7 +308,7 @@ static struct irq_chip hv_pci_msi_controller = { }; static struct msi_domain_ops pci_msi_domain_ops = { - .domain_free_irqs = hv_msi_domain_free_irqs, + .msi_free = hv_msi_free_irq, .msi_prepare = pci_msi_prepare, }; diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h index 5c69f7eb5d47..22b7412c08f6 100644 --- a/arch/x86/include/asm/x86_init.h +++ b/arch/x86/include/asm/x86_init.h @@ -289,12 +289,6 @@ struct x86_platform_ops { struct x86_hyper_runtime hyper; }; -struct pci_dev; - -struct x86_msi_ops { - void (*restore_msi_irqs)(struct pci_dev *dev); -}; - struct x86_apic_ops { unsigned int (*io_apic_read) (unsigned int apic, unsigned int reg); void (*restore)(void); diff --git a/arch/x86/include/asm/xen/hypervisor.h b/arch/x86/include/asm/xen/hypervisor.h index 5adab895127e..1bf2ad34188a 100644 --- a/arch/x86/include/asm/xen/hypervisor.h +++ b/arch/x86/include/asm/xen/hypervisor.h @@ -57,6 +57,14 @@ static inline bool __init xen_x2apic_para_available(void) } #endif +struct pci_dev; + +#ifdef CONFIG_XEN_PV_DOM0 +bool xen_initdom_restore_msi(struct pci_dev *dev); +#else +static inline bool xen_initdom_restore_msi(struct pci_dev *dev) { return true; } +#endif + #ifdef CONFIG_HOTPLUG_CPU void xen_arch_register_cpu(int num); void xen_arch_unregister_cpu(int num); diff --git a/arch/x86/kernel/apic/msi.c b/arch/x86/kernel/apic/msi.c index dbacb9ec8843..7517eb05bdc1 100644 --- a/arch/x86/kernel/apic/msi.c +++ b/arch/x86/kernel/apic/msi.c @@ -19,6 +19,7 @@ #include #include #include +#include struct irq_domain *x86_pci_msi_default_domain __ro_after_init; @@ -159,11 +160,8 @@ static struct irq_chip pci_msi_controller = { int pci_msi_prepare(struct irq_domain *domain, struct device *dev, int nvec, msi_alloc_info_t *arg) { - struct pci_dev *pdev = to_pci_dev(dev); - struct msi_desc *desc = first_pci_msi_entry(pdev); - init_irq_alloc_info(arg, NULL); - if (desc->msi_attrib.is_msix) { + if (to_pci_dev(dev)->msix_enabled) { arg->type = X86_IRQ_ALLOC_TYPE_PCI_MSIX; } else { arg->type = X86_IRQ_ALLOC_TYPE_PCI_MSI; @@ -345,3 +343,8 @@ void dmar_free_hwirq(int irq) irq_domain_free_irqs(irq, 1); } #endif + +bool arch_restore_msi_irqs(struct pci_dev *dev) +{ + return xen_initdom_restore_msi(dev); +} diff --git a/arch/x86/kernel/x86_init.c b/arch/x86/kernel/x86_init.c index 8b395821cb8d..7d20c1d34a3c 100644 --- a/arch/x86/kernel/x86_init.c +++ b/arch/x86/kernel/x86_init.c @@ -145,18 +145,6 @@ struct x86_platform_ops x86_platform __ro_after_init = { EXPORT_SYMBOL_GPL(x86_platform); -#if defined(CONFIG_PCI_MSI) -struct x86_msi_ops x86_msi __ro_after_init = { - .restore_msi_irqs = default_restore_msi_irqs, -}; - -/* MSI arch specific hooks */ -void arch_restore_msi_irqs(struct pci_dev *dev) -{ - x86_msi.restore_msi_irqs(dev); -} -#endif - struct x86_apic_ops x86_apic_ops __ro_after_init = { .io_apic_read = native_io_apic_read, .restore = native_restore_boot_irq_mode, diff --git a/arch/x86/pci/xen.c b/arch/x86/pci/xen.c index 12da00558631..9bb1e2941179 100644 --- a/arch/x86/pci/xen.c +++ b/arch/x86/pci/xen.c @@ -184,7 +184,7 @@ static int xen_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) if (ret) goto error; i = 0; - for_each_pci_msi_entry(msidesc, dev) { + msi_for_each_desc(msidesc, &dev->dev, MSI_DESC_NOTASSOCIATED) { irq = xen_bind_pirq_msi_to_irq(dev, msidesc, v[i], (type == PCI_CAP_ID_MSI) ? nvec : 1, (type == PCI_CAP_ID_MSIX) ? @@ -235,7 +235,7 @@ static int xen_hvm_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) if (type == PCI_CAP_ID_MSI && nvec > 1) return 1; - for_each_pci_msi_entry(msidesc, dev) { + msi_for_each_desc(msidesc, &dev->dev, MSI_DESC_NOTASSOCIATED) { pirq = xen_allocate_pirq_msi(dev, msidesc); if (pirq < 0) { irq = -ENODEV; @@ -270,7 +270,7 @@ static int xen_initdom_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) int ret = 0; struct msi_desc *msidesc; - for_each_pci_msi_entry(msidesc, dev) { + msi_for_each_desc(msidesc, &dev->dev, MSI_DESC_NOTASSOCIATED) { struct physdev_map_pirq map_irq; domid_t domid; @@ -306,7 +306,7 @@ static int xen_initdom_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) return -EINVAL; map_irq.table_base = pci_resource_start(dev, bir); - map_irq.entry_nr = msidesc->msi_attrib.entry_nr; + map_irq.entry_nr = msidesc->msi_index; } ret = -EINVAL; @@ -351,10 +351,13 @@ out: return ret; } -static void xen_initdom_restore_msi_irqs(struct pci_dev *dev) +bool xen_initdom_restore_msi(struct pci_dev *dev) { int ret = 0; + if (!xen_initial_domain()) + return true; + if (pci_seg_supported) { struct physdev_pci_device restore_ext; @@ -375,10 +378,10 @@ static void xen_initdom_restore_msi_irqs(struct pci_dev *dev) ret = HYPERVISOR_physdev_op(PHYSDEVOP_restore_msi, &restore); WARN(ret && ret != -ENOSYS, "restore_msi -> %d\n", ret); } + return false; } #else /* CONFIG_XEN_PV_DOM0 */ #define xen_initdom_setup_msi_irqs NULL -#define xen_initdom_restore_msi_irqs NULL #endif /* !CONFIG_XEN_PV_DOM0 */ static void xen_teardown_msi_irqs(struct pci_dev *dev) @@ -386,19 +389,15 @@ static void xen_teardown_msi_irqs(struct pci_dev *dev) struct msi_desc *msidesc; int i; - for_each_pci_msi_entry(msidesc, dev) { - if (msidesc->irq) { - for (i = 0; i < msidesc->nvec_used; i++) - xen_destroy_irq(msidesc->irq + i); - } + msi_for_each_desc(msidesc, &dev->dev, MSI_DESC_ASSOCIATED) { + for (i = 0; i < msidesc->nvec_used; i++) + xen_destroy_irq(msidesc->irq + i); } } static void xen_pv_teardown_msi_irqs(struct pci_dev *dev) { - struct msi_desc *msidesc = first_pci_msi_entry(dev); - - if (msidesc->msi_attrib.is_msix) + if (dev->msix_enabled) xen_pci_frontend_disable_msix(dev); else xen_pci_frontend_disable_msi(dev); @@ -414,10 +413,7 @@ static int xen_msi_domain_alloc_irqs(struct irq_domain *domain, if (WARN_ON_ONCE(!dev_is_pci(dev))) return -EINVAL; - if (first_msi_entry(dev)->msi_attrib.is_msix) - type = PCI_CAP_ID_MSIX; - else - type = PCI_CAP_ID_MSI; + type = to_pci_dev(dev)->msix_enabled ? PCI_CAP_ID_MSIX : PCI_CAP_ID_MSI; return xen_msi_ops.setup_msi_irqs(to_pci_dev(dev), nvec, type); } @@ -466,12 +462,10 @@ static __init struct irq_domain *xen_create_pci_msi_domain(void) static __init void xen_setup_pci_msi(void) { if (xen_pv_domain()) { - if (xen_initial_domain()) { + if (xen_initial_domain()) xen_msi_ops.setup_msi_irqs = xen_initdom_setup_msi_irqs; - x86_msi.restore_msi_irqs = xen_initdom_restore_msi_irqs; - } else { + else xen_msi_ops.setup_msi_irqs = xen_setup_msi_irqs; - } xen_msi_ops.teardown_msi_irqs = xen_pv_teardown_msi_irqs; pci_msi_ignore_mask = 1; } else if (xen_hvm_domain()) { diff --git a/drivers/acpi/acpi_apd.c b/drivers/acpi/acpi_apd.c index 6e02448d15d9..2b958b426b03 100644 --- a/drivers/acpi/acpi_apd.c +++ b/drivers/acpi/acpi_apd.c @@ -87,8 +87,15 @@ static int fch_misc_setup(struct apd_private_data *pdata) if (ret < 0) return -ENOENT; - if (!acpi_dev_get_property(adev, "is-rv", ACPI_TYPE_INTEGER, &obj)) - clk_data->is_rv = obj->integer.value; + if (!acpi_dev_get_property(adev, "clk-name", ACPI_TYPE_STRING, &obj)) { + clk_data->name = devm_kzalloc(&adev->dev, obj->string.length, + GFP_KERNEL); + + strcpy(clk_data->name, obj->string.pointer); + } else { + /* Set default name to mclk if entry missing in firmware */ + clk_data->name = "mclk"; + } list_for_each_entry(rentry, &resource_list, node) { clk_data->base = devm_ioremap(&adev->dev, rentry->res->start, diff --git a/drivers/base/core.c b/drivers/base/core.c index 824be4ac2438..e38bd8bb7cf8 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -2872,10 +2872,6 @@ void device_initialize(struct device *dev) INIT_LIST_HEAD(&dev->devres_head); device_pm_init(dev); set_dev_node(dev, NUMA_NO_NODE); -#ifdef CONFIG_GENERIC_MSI_IRQ - raw_spin_lock_init(&dev->msi_lock); - INIT_LIST_HEAD(&dev->msi_list); -#endif INIT_LIST_HEAD(&dev->links.consumers); INIT_LIST_HEAD(&dev->links.suppliers); INIT_LIST_HEAD(&dev->links.defer_sync); diff --git a/drivers/base/platform-msi.c b/drivers/base/platform-msi.c index 3d6c8f9caf43..296ea673d661 100644 --- a/drivers/base/platform-msi.c +++ b/drivers/base/platform-msi.c @@ -23,7 +23,6 @@ struct platform_msi_priv_data { struct device *dev; void *host_data; - const struct attribute_group **msi_irq_groups; msi_alloc_info_t arg; irq_write_msi_msg_t write_msg; int devid; @@ -39,11 +38,9 @@ static DEFINE_IDA(platform_msi_devid_ida); */ static irq_hw_number_t platform_msi_calc_hwirq(struct msi_desc *desc) { - u32 devid; + u32 devid = desc->dev->msi.data->platform_data->devid; - devid = desc->platform.msi_priv_data->devid; - - return (devid << (32 - DEV_ID_SHIFT)) | desc->platform.msi_index; + return (devid << (32 - DEV_ID_SHIFT)) | desc->msi_index; } static void platform_msi_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc) @@ -86,11 +83,8 @@ static void platform_msi_update_dom_ops(struct msi_domain_info *info) static void platform_msi_write_msg(struct irq_data *data, struct msi_msg *msg) { struct msi_desc *desc = irq_data_get_msi_desc(data); - struct platform_msi_priv_data *priv_data; - priv_data = desc->platform.msi_priv_data; - - priv_data->write_msg(desc, msg); + desc->dev->msi.data->platform_data->write_msg(desc, msg); } static void platform_msi_update_chip_ops(struct msi_domain_info *info) @@ -113,62 +107,6 @@ static void platform_msi_update_chip_ops(struct msi_domain_info *info) info->flags &= ~MSI_FLAG_LEVEL_CAPABLE; } -static void platform_msi_free_descs(struct device *dev, int base, int nvec) -{ - struct msi_desc *desc, *tmp; - - list_for_each_entry_safe(desc, tmp, dev_to_msi_list(dev), list) { - if (desc->platform.msi_index >= base && - desc->platform.msi_index < (base + nvec)) { - list_del(&desc->list); - free_msi_entry(desc); - } - } -} - -static int platform_msi_alloc_descs_with_irq(struct device *dev, int virq, - int nvec, - struct platform_msi_priv_data *data) - -{ - struct msi_desc *desc; - int i, base = 0; - - if (!list_empty(dev_to_msi_list(dev))) { - desc = list_last_entry(dev_to_msi_list(dev), - struct msi_desc, list); - base = desc->platform.msi_index + 1; - } - - for (i = 0; i < nvec; i++) { - desc = alloc_msi_entry(dev, 1, NULL); - if (!desc) - break; - - desc->platform.msi_priv_data = data; - desc->platform.msi_index = base + i; - desc->irq = virq ? virq + i : 0; - - list_add_tail(&desc->list, dev_to_msi_list(dev)); - } - - if (i != nvec) { - /* Clean up the mess */ - platform_msi_free_descs(dev, base, nvec); - - return -ENOMEM; - } - - return 0; -} - -static int platform_msi_alloc_descs(struct device *dev, int nvec, - struct platform_msi_priv_data *data) - -{ - return platform_msi_alloc_descs_with_irq(dev, 0, nvec, data); -} - /** * platform_msi_create_irq_domain - Create a platform MSI interrupt domain * @fwnode: Optional fwnode of the interrupt controller @@ -191,6 +129,8 @@ struct irq_domain *platform_msi_create_irq_domain(struct fwnode_handle *fwnode, platform_msi_update_dom_ops(info); if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS) platform_msi_update_chip_ops(info); + info->flags |= MSI_FLAG_DEV_SYSFS | MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS | + MSI_FLAG_FREE_MSI_DESCS; domain = msi_create_irq_domain(fwnode, info, parent); if (domain) @@ -199,49 +139,57 @@ struct irq_domain *platform_msi_create_irq_domain(struct fwnode_handle *fwnode, return domain; } -static struct platform_msi_priv_data * -platform_msi_alloc_priv_data(struct device *dev, unsigned int nvec, - irq_write_msi_msg_t write_msi_msg) +static int platform_msi_alloc_priv_data(struct device *dev, unsigned int nvec, + irq_write_msi_msg_t write_msi_msg) { struct platform_msi_priv_data *datap; + int err; + /* * Limit the number of interrupts to 2048 per device. Should we * need to bump this up, DEV_ID_SHIFT should be adjusted * accordingly (which would impact the max number of MSI * capable devices). */ - if (!dev->msi_domain || !write_msi_msg || !nvec || nvec > MAX_DEV_MSIS) - return ERR_PTR(-EINVAL); + if (!dev->msi.domain || !write_msi_msg || !nvec || nvec > MAX_DEV_MSIS) + return -EINVAL; - if (dev->msi_domain->bus_token != DOMAIN_BUS_PLATFORM_MSI) { + if (dev->msi.domain->bus_token != DOMAIN_BUS_PLATFORM_MSI) { dev_err(dev, "Incompatible msi_domain, giving up\n"); - return ERR_PTR(-EINVAL); + return -EINVAL; } - /* Already had a helping of MSI? Greed... */ - if (!list_empty(dev_to_msi_list(dev))) - return ERR_PTR(-EBUSY); + err = msi_setup_device_data(dev); + if (err) + return err; + + /* Already initialized? */ + if (dev->msi.data->platform_data) + return -EBUSY; datap = kzalloc(sizeof(*datap), GFP_KERNEL); if (!datap) - return ERR_PTR(-ENOMEM); + return -ENOMEM; datap->devid = ida_simple_get(&platform_msi_devid_ida, 0, 1 << DEV_ID_SHIFT, GFP_KERNEL); if (datap->devid < 0) { - int err = datap->devid; + err = datap->devid; kfree(datap); - return ERR_PTR(err); + return err; } datap->write_msg = write_msi_msg; datap->dev = dev; - - return datap; + dev->msi.data->platform_data = datap; + return 0; } -static void platform_msi_free_priv_data(struct platform_msi_priv_data *data) +static void platform_msi_free_priv_data(struct device *dev) { + struct platform_msi_priv_data *data = dev->msi.data->platform_data; + + dev->msi.data->platform_data = NULL; ida_simple_remove(&platform_msi_devid_ida, data->devid); kfree(data); } @@ -258,35 +206,15 @@ static void platform_msi_free_priv_data(struct platform_msi_priv_data *data) int platform_msi_domain_alloc_irqs(struct device *dev, unsigned int nvec, irq_write_msi_msg_t write_msi_msg) { - struct platform_msi_priv_data *priv_data; int err; - priv_data = platform_msi_alloc_priv_data(dev, nvec, write_msi_msg); - if (IS_ERR(priv_data)) - return PTR_ERR(priv_data); - - err = platform_msi_alloc_descs(dev, nvec, priv_data); + err = platform_msi_alloc_priv_data(dev, nvec, write_msi_msg); if (err) - goto out_free_priv_data; + return err; - err = msi_domain_alloc_irqs(dev->msi_domain, dev, nvec); + err = msi_domain_alloc_irqs(dev->msi.domain, dev, nvec); if (err) - goto out_free_desc; - - priv_data->msi_irq_groups = msi_populate_sysfs(dev); - if (IS_ERR(priv_data->msi_irq_groups)) { - err = PTR_ERR(priv_data->msi_irq_groups); - goto out_free_irqs; - } - - return 0; - -out_free_irqs: - msi_domain_free_irqs(dev->msi_domain, dev); -out_free_desc: - platform_msi_free_descs(dev, 0, nvec); -out_free_priv_data: - platform_msi_free_priv_data(priv_data); + platform_msi_free_priv_data(dev); return err; } @@ -298,16 +226,8 @@ EXPORT_SYMBOL_GPL(platform_msi_domain_alloc_irqs); */ void platform_msi_domain_free_irqs(struct device *dev) { - if (!list_empty(dev_to_msi_list(dev))) { - struct msi_desc *desc; - - desc = first_msi_entry(dev); - msi_destroy_sysfs(dev, desc->platform.msi_priv_data->msi_irq_groups); - platform_msi_free_priv_data(desc->platform.msi_priv_data); - } - - msi_domain_free_irqs(dev->msi_domain, dev); - platform_msi_free_descs(dev, 0, MAX_DEV_MSIS); + msi_domain_free_irqs(dev->msi.domain, dev); + platform_msi_free_priv_data(dev); } EXPORT_SYMBOL_GPL(platform_msi_domain_free_irqs); @@ -316,17 +236,20 @@ EXPORT_SYMBOL_GPL(platform_msi_domain_free_irqs); * a platform-msi domain * @domain: The platform-msi domain * - * Returns the private data provided when calling - * platform_msi_create_device_domain. + * Return: The private data provided when calling + * platform_msi_create_device_domain(). */ void *platform_msi_get_host_data(struct irq_domain *domain) { struct platform_msi_priv_data *data = domain->host_data; + return data->host_data; } +static struct lock_class_key platform_device_msi_lock_class; + /** - * __platform_msi_create_device_domain - Create a platform-msi domain + * __platform_msi_create_device_domain - Create a platform-msi device domain * * @dev: The device generating the MSIs * @nvec: The number of MSIs that need to be allocated @@ -335,7 +258,11 @@ void *platform_msi_get_host_data(struct irq_domain *domain) * @ops: The hierarchy domain operations to use * @host_data: Private data associated to this domain * - * Returns an irqdomain for @nvec interrupts + * Return: An irqdomain for @nvec interrupts on success, NULL in case of error. + * + * This is for interrupt domains which stack on a platform-msi domain + * created by platform_msi_create_irq_domain(). @dev->msi.domain points to + * that platform-msi domain which is the parent for the new domain. */ struct irq_domain * __platform_msi_create_device_domain(struct device *dev, @@ -349,12 +276,20 @@ __platform_msi_create_device_domain(struct device *dev, struct irq_domain *domain; int err; - data = platform_msi_alloc_priv_data(dev, nvec, write_msi_msg); - if (IS_ERR(data)) + err = platform_msi_alloc_priv_data(dev, nvec, write_msi_msg); + if (err) return NULL; + /* + * Use a separate lock class for the MSI descriptor mutex on + * platform MSI device domains because the descriptor mutex nests + * into the domain mutex. See alloc/free below. + */ + lockdep_set_class(&dev->msi.data->mutex, &platform_device_msi_lock_class); + + data = dev->msi.data->platform_data; data->host_data = host_data; - domain = irq_domain_create_hierarchy(dev->msi_domain, 0, + domain = irq_domain_create_hierarchy(dev->msi.domain, 0, is_tree ? 0 : nvec, dev->fwnode, ops, data); if (!domain) @@ -370,61 +305,46 @@ __platform_msi_create_device_domain(struct device *dev, free_domain: irq_domain_remove(domain); free_priv: - platform_msi_free_priv_data(data); + platform_msi_free_priv_data(dev); return NULL; } /** - * platform_msi_domain_free - Free interrupts associated with a platform-msi - * domain + * platform_msi_device_domain_free - Free interrupts associated with a platform-msi + * device domain * - * @domain: The platform-msi domain + * @domain: The platform-msi device domain * @virq: The base irq from which to perform the free operation - * @nvec: How many interrupts to free from @virq + * @nr_irqs: How many interrupts to free from @virq */ -void platform_msi_domain_free(struct irq_domain *domain, unsigned int virq, - unsigned int nvec) +void platform_msi_device_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) { struct platform_msi_priv_data *data = domain->host_data; - struct msi_desc *desc, *tmp; - for_each_msi_entry_safe(desc, tmp, data->dev) { - if (WARN_ON(!desc->irq || desc->nvec_used != 1)) - return; - if (!(desc->irq >= virq && desc->irq < (virq + nvec))) - continue; - irq_domain_free_irqs_common(domain, desc->irq, 1); - list_del(&desc->list); - free_msi_entry(desc); - } + msi_lock_descs(data->dev); + irq_domain_free_irqs_common(domain, virq, nr_irqs); + msi_free_msi_descs_range(data->dev, MSI_DESC_ALL, virq, virq + nr_irqs - 1); + msi_unlock_descs(data->dev); } /** - * platform_msi_domain_alloc - Allocate interrupts associated with - * a platform-msi domain + * platform_msi_device_domain_alloc - Allocate interrupts associated with + * a platform-msi device domain * - * @domain: The platform-msi domain + * @domain: The platform-msi device domain * @virq: The base irq from which to perform the allocate operation - * @nr_irqs: How many interrupts to free from @virq + * @nr_irqs: How many interrupts to allocate from @virq * * Return 0 on success, or an error code on failure. Must be called * with irq_domain_mutex held (which can only be done as part of a * top-level interrupt allocation). */ -int platform_msi_domain_alloc(struct irq_domain *domain, unsigned int virq, - unsigned int nr_irqs) +int platform_msi_device_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) { struct platform_msi_priv_data *data = domain->host_data; - int err; + struct device *dev = data->dev; - err = platform_msi_alloc_descs_with_irq(data->dev, virq, nr_irqs, data); - if (err) - return err; - - err = msi_domain_populate_irqs(domain->parent, data->dev, - virq, nr_irqs, &data->arg); - if (err) - platform_msi_domain_free(domain, virq, nr_irqs); - - return err; + return msi_domain_populate_irqs(domain->parent, dev, virq, nr_irqs, &data->arg); } diff --git a/drivers/block/rnbd/rnbd-clt.c b/drivers/block/rnbd/rnbd-clt.c index 67a8edbaa1fd..8f8443ee6fe4 100644 --- a/drivers/block/rnbd/rnbd-clt.c +++ b/drivers/block/rnbd/rnbd-clt.c @@ -433,7 +433,7 @@ static void msg_conf(void *priv, int errno) schedule_work(&iu->work); } -static int send_usr_msg(struct rtrs_clt *rtrs, int dir, +static int send_usr_msg(struct rtrs_clt_sess *rtrs, int dir, struct rnbd_iu *iu, struct kvec *vec, size_t len, struct scatterlist *sg, unsigned int sg_len, void (*conf)(struct work_struct *work), @@ -1010,7 +1010,7 @@ static int rnbd_client_xfer_request(struct rnbd_clt_dev *dev, struct request *rq, struct rnbd_iu *iu) { - struct rtrs_clt *rtrs = dev->sess->rtrs; + struct rtrs_clt_sess *rtrs = dev->sess->rtrs; struct rtrs_permit *permit = iu->permit; struct rnbd_msg_io msg; struct rtrs_clt_req_ops req_ops; diff --git a/drivers/block/rnbd/rnbd-clt.h b/drivers/block/rnbd/rnbd-clt.h index 9ef8c4f306f2..0c2cae7f39b9 100644 --- a/drivers/block/rnbd/rnbd-clt.h +++ b/drivers/block/rnbd/rnbd-clt.h @@ -75,7 +75,7 @@ struct rnbd_cpu_qlist { struct rnbd_clt_session { struct list_head list; - struct rtrs_clt *rtrs; + struct rtrs_clt_sess *rtrs; wait_queue_head_t rtrs_waitq; bool rtrs_ready; struct rnbd_cpu_qlist __percpu diff --git a/drivers/block/rnbd/rnbd-srv.c b/drivers/block/rnbd/rnbd-srv.c index aafecfe97055..1ee808fc600c 100644 --- a/drivers/block/rnbd/rnbd-srv.c +++ b/drivers/block/rnbd/rnbd-srv.c @@ -263,15 +263,15 @@ out: kfree(srv_sess); } -static int create_sess(struct rtrs_srv *rtrs) +static int create_sess(struct rtrs_srv_sess *rtrs) { struct rnbd_srv_session *srv_sess; - char sessname[NAME_MAX]; + char pathname[NAME_MAX]; int err; - err = rtrs_srv_get_sess_name(rtrs, sessname, sizeof(sessname)); + err = rtrs_srv_get_path_name(rtrs, pathname, sizeof(pathname)); if (err) { - pr_err("rtrs_srv_get_sess_name(%s): %d\n", sessname, err); + pr_err("rtrs_srv_get_path_name(%s): %d\n", pathname, err); return err; } @@ -284,8 +284,8 @@ static int create_sess(struct rtrs_srv *rtrs) offsetof(struct rnbd_dev_blk_io, bio), BIOSET_NEED_BVECS); if (err) { - pr_err("Allocating srv_session for session %s failed\n", - sessname); + pr_err("Allocating srv_session for path %s failed\n", + pathname); kfree(srv_sess); return err; } @@ -298,14 +298,14 @@ static int create_sess(struct rtrs_srv *rtrs) mutex_unlock(&sess_lock); srv_sess->rtrs = rtrs; - strscpy(srv_sess->sessname, sessname, sizeof(srv_sess->sessname)); + strscpy(srv_sess->sessname, pathname, sizeof(srv_sess->sessname)); rtrs_srv_set_sess_priv(rtrs, srv_sess); return 0; } -static int rnbd_srv_link_ev(struct rtrs_srv *rtrs, +static int rnbd_srv_link_ev(struct rtrs_srv_sess *rtrs, enum rtrs_srv_link_ev ev, void *priv) { struct rnbd_srv_session *srv_sess = priv; diff --git a/drivers/block/rnbd/rnbd-srv.h b/drivers/block/rnbd/rnbd-srv.h index 98ddc31eb408..e5604bce123a 100644 --- a/drivers/block/rnbd/rnbd-srv.h +++ b/drivers/block/rnbd/rnbd-srv.h @@ -20,7 +20,7 @@ struct rnbd_srv_session { /* Entry inside global sess_list */ struct list_head list; - struct rtrs_srv *rtrs; + struct rtrs_srv_sess *rtrs; char sessname[NAME_MAX]; int queue_depth; struct bio_set sess_bio_set; diff --git a/drivers/bus/fsl-mc/dprc-driver.c b/drivers/bus/fsl-mc/dprc-driver.c index 315e830b6ecd..5e70f9775a0e 100644 --- a/drivers/bus/fsl-mc/dprc-driver.c +++ b/drivers/bus/fsl-mc/dprc-driver.c @@ -400,7 +400,7 @@ static irqreturn_t dprc_irq0_handler_thread(int irq_num, void *arg) struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev); struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_dev); struct fsl_mc_io *mc_io = mc_dev->mc_io; - struct msi_desc *msi_desc = mc_dev->irqs[0]->msi_desc; + int irq = mc_dev->irqs[0]->virq; dev_dbg(dev, "DPRC IRQ %d triggered on CPU %u\n", irq_num, smp_processor_id()); @@ -409,7 +409,7 @@ static irqreturn_t dprc_irq0_handler_thread(int irq_num, void *arg) return IRQ_HANDLED; mutex_lock(&mc_bus->scan_mutex); - if (!msi_desc || msi_desc->irq != (u32)irq_num) + if (irq != (u32)irq_num) goto out; status = 0; @@ -521,7 +521,7 @@ static int register_dprc_irq_handler(struct fsl_mc_device *mc_dev) * function that programs the MSI physically in the device */ error = devm_request_threaded_irq(&mc_dev->dev, - irq->msi_desc->irq, + irq->virq, dprc_irq0_handler, dprc_irq0_handler_thread, IRQF_NO_SUSPEND | IRQF_ONESHOT, @@ -771,7 +771,7 @@ static void dprc_teardown_irq(struct fsl_mc_device *mc_dev) (void)disable_dprc_irq(mc_dev); - devm_free_irq(&mc_dev->dev, irq->msi_desc->irq, &mc_dev->dev); + devm_free_irq(&mc_dev->dev, irq->virq, &mc_dev->dev); fsl_mc_free_irqs(mc_dev); } diff --git a/drivers/bus/fsl-mc/fsl-mc-allocator.c b/drivers/bus/fsl-mc/fsl-mc-allocator.c index 6c513556911e..dced427ca8ba 100644 --- a/drivers/bus/fsl-mc/fsl-mc-allocator.c +++ b/drivers/bus/fsl-mc/fsl-mc-allocator.c @@ -350,7 +350,6 @@ int fsl_mc_populate_irq_pool(struct fsl_mc_device *mc_bus_dev, unsigned int irq_count) { unsigned int i; - struct msi_desc *msi_desc; struct fsl_mc_device_irq *irq_resources; struct fsl_mc_device_irq *mc_dev_irq; int error; @@ -388,16 +387,12 @@ int fsl_mc_populate_irq_pool(struct fsl_mc_device *mc_bus_dev, mc_dev_irq->resource.type = res_pool->type; mc_dev_irq->resource.data = mc_dev_irq; mc_dev_irq->resource.parent_pool = res_pool; + mc_dev_irq->virq = msi_get_virq(&mc_bus_dev->dev, i); + mc_dev_irq->resource.id = mc_dev_irq->virq; INIT_LIST_HEAD(&mc_dev_irq->resource.node); list_add_tail(&mc_dev_irq->resource.node, &res_pool->free_list); } - for_each_msi_entry(msi_desc, &mc_bus_dev->dev) { - mc_dev_irq = &irq_resources[msi_desc->fsl_mc.msi_index]; - mc_dev_irq->msi_desc = msi_desc; - mc_dev_irq->resource.id = msi_desc->irq; - } - res_pool->max_count = irq_count; res_pool->free_count = irq_count; mc_bus->irq_resources = irq_resources; diff --git a/drivers/bus/fsl-mc/fsl-mc-msi.c b/drivers/bus/fsl-mc/fsl-mc-msi.c index cf974870ba55..5e0e4393ce4d 100644 --- a/drivers/bus/fsl-mc/fsl-mc-msi.c +++ b/drivers/bus/fsl-mc/fsl-mc-msi.c @@ -29,7 +29,7 @@ static irq_hw_number_t fsl_mc_domain_calc_hwirq(struct fsl_mc_device *dev, * Make the base hwirq value for ICID*10000 so it is readable * as a decimal value in /proc/interrupts. */ - return (irq_hw_number_t)(desc->fsl_mc.msi_index + (dev->icid * 10000)); + return (irq_hw_number_t)(desc->msi_index + (dev->icid * 10000)); } static void fsl_mc_msi_set_desc(msi_alloc_info_t *arg, @@ -58,11 +58,11 @@ static void fsl_mc_msi_update_dom_ops(struct msi_domain_info *info) } static void __fsl_mc_msi_write_msg(struct fsl_mc_device *mc_bus_dev, - struct fsl_mc_device_irq *mc_dev_irq) + struct fsl_mc_device_irq *mc_dev_irq, + struct msi_desc *msi_desc) { int error; struct fsl_mc_device *owner_mc_dev = mc_dev_irq->mc_dev; - struct msi_desc *msi_desc = mc_dev_irq->msi_desc; struct dprc_irq_cfg irq_cfg; /* @@ -122,14 +122,14 @@ static void fsl_mc_msi_write_msg(struct irq_data *irq_data, struct fsl_mc_device *mc_bus_dev = to_fsl_mc_device(msi_desc->dev); struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_bus_dev); struct fsl_mc_device_irq *mc_dev_irq = - &mc_bus->irq_resources[msi_desc->fsl_mc.msi_index]; + &mc_bus->irq_resources[msi_desc->msi_index]; msi_desc->msg = *msg; /* * Program the MSI (paddr, value) pair in the device: */ - __fsl_mc_msi_write_msg(mc_bus_dev, mc_dev_irq); + __fsl_mc_msi_write_msg(mc_bus_dev, mc_dev_irq, msi_desc); } static void fsl_mc_msi_update_chip_ops(struct msi_domain_info *info) @@ -170,6 +170,7 @@ struct irq_domain *fsl_mc_msi_create_irq_domain(struct fwnode_handle *fwnode, fsl_mc_msi_update_dom_ops(info); if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS) fsl_mc_msi_update_chip_ops(info); + info->flags |= MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS | MSI_FLAG_FREE_MSI_DESCS; domain = msi_create_irq_domain(fwnode, info, parent); if (domain) @@ -210,61 +211,21 @@ struct irq_domain *fsl_mc_find_msi_domain(struct device *dev) return msi_domain; } -static void fsl_mc_msi_free_descs(struct device *dev) -{ - struct msi_desc *desc, *tmp; - - list_for_each_entry_safe(desc, tmp, dev_to_msi_list(dev), list) { - list_del(&desc->list); - free_msi_entry(desc); - } -} - -static int fsl_mc_msi_alloc_descs(struct device *dev, unsigned int irq_count) - -{ - unsigned int i; - int error; - struct msi_desc *msi_desc; - - for (i = 0; i < irq_count; i++) { - msi_desc = alloc_msi_entry(dev, 1, NULL); - if (!msi_desc) { - dev_err(dev, "Failed to allocate msi entry\n"); - error = -ENOMEM; - goto cleanup_msi_descs; - } - - msi_desc->fsl_mc.msi_index = i; - INIT_LIST_HEAD(&msi_desc->list); - list_add_tail(&msi_desc->list, dev_to_msi_list(dev)); - } - - return 0; - -cleanup_msi_descs: - fsl_mc_msi_free_descs(dev); - return error; -} - -int fsl_mc_msi_domain_alloc_irqs(struct device *dev, - unsigned int irq_count) +int fsl_mc_msi_domain_alloc_irqs(struct device *dev, unsigned int irq_count) { struct irq_domain *msi_domain; int error; - if (!list_empty(dev_to_msi_list(dev))) + msi_domain = dev_get_msi_domain(dev); + if (!msi_domain) return -EINVAL; - error = fsl_mc_msi_alloc_descs(dev, irq_count); - if (error < 0) + error = msi_setup_device_data(dev); + if (error) return error; - msi_domain = dev_get_msi_domain(dev); - if (!msi_domain) { - error = -EINVAL; - goto cleanup_msi_descs; - } + if (msi_first_desc(dev, MSI_DESC_ALL)) + return -EINVAL; /* * NOTE: Calling this function will trigger the invocation of the @@ -272,15 +233,8 @@ int fsl_mc_msi_domain_alloc_irqs(struct device *dev, */ error = msi_domain_alloc_irqs(msi_domain, dev, irq_count); - if (error) { + if (error) dev_err(dev, "Failed to allocate IRQs\n"); - goto cleanup_msi_descs; - } - - return 0; - -cleanup_msi_descs: - fsl_mc_msi_free_descs(dev); return error; } @@ -293,9 +247,4 @@ void fsl_mc_msi_domain_free_irqs(struct device *dev) return; msi_domain_free_irqs(msi_domain, dev); - - if (list_empty(dev_to_msi_list(dev))) - return; - - fsl_mc_msi_free_descs(dev); } diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig index c91931c94888..ad4256d54361 100644 --- a/drivers/clk/Kconfig +++ b/drivers/clk/Kconfig @@ -169,6 +169,14 @@ config COMMON_CLK_CDCE706 help This driver supports TI CDCE706 programmable 3-PLL clock synthesizer. +config COMMON_CLK_TPS68470 + tristate "Clock Driver for TI TPS68470 PMIC" + depends on I2C + depends on INTEL_SKL_INT3472 || COMPILE_TEST + select REGMAP_I2C + help + This driver supports the clocks provided by the TPS68470 PMIC. + config COMMON_CLK_CDCE925 tristate "Clock driver for TI CDCE913/925/937/949 devices" depends on I2C @@ -221,6 +229,13 @@ config COMMON_CLK_GEMINI This driver supports the SoC clocks on the Cortina Systems Gemini platform, also known as SL3516 or CS3516. +config COMMON_CLK_LAN966X + bool "Generic Clock Controller driver for LAN966X SoC" + help + This driver provides support for Generic Clock Controller(GCK) on + LAN966X SoC. GCK generates and supplies clock to various peripherals + within the SoC. + config COMMON_CLK_ASPEED bool "Clock driver for Aspeed BMC SoCs" depends on ARCH_ASPEED || COMPILE_TEST @@ -339,16 +354,6 @@ config COMMON_CLK_STM32MP157 help Support for stm32mp157 SoC family clocks -config COMMON_CLK_STM32MP157_SCMI - bool "stm32mp157 Clock driver with Trusted Firmware" - depends on COMMON_CLK_STM32MP157 - select COMMON_CLK_SCMI - select ARM_SCMI_PROTOCOL - default y - help - Support for stm32mp157 SoC family clocks with Trusted Firmware using - SCMI protocol. - config COMMON_CLK_STM32F def_bool COMMON_CLK && (MACH_STM32F429 || MACH_STM32F469 || MACH_STM32F746) help @@ -418,6 +423,7 @@ source "drivers/clk/sunxi-ng/Kconfig" source "drivers/clk/tegra/Kconfig" source "drivers/clk/ti/Kconfig" source "drivers/clk/uniphier/Kconfig" +source "drivers/clk/visconti/Kconfig" source "drivers/clk/x86/Kconfig" source "drivers/clk/xilinx/Kconfig" source "drivers/clk/zynqmp/Kconfig" diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile index a9bb2478fbdd..16e588630472 100644 --- a/drivers/clk/Makefile +++ b/drivers/clk/Makefile @@ -37,6 +37,7 @@ obj-$(CONFIG_ARCH_HIGHBANK) += clk-highbank.o obj-$(CONFIG_CLK_HSDK) += clk-hsdk-pll.o obj-$(CONFIG_COMMON_CLK_K210) += clk-k210.o obj-$(CONFIG_LMK04832) += clk-lmk04832.o +obj-$(CONFIG_COMMON_CLK_LAN966X) += clk-lan966x.o obj-$(CONFIG_COMMON_CLK_LOCHNAGAR) += clk-lochnagar.o obj-$(CONFIG_COMMON_CLK_MAX77686) += clk-max77686.o obj-$(CONFIG_COMMON_CLK_MAX9485) += clk-max9485.o @@ -63,6 +64,7 @@ obj-$(CONFIG_COMMON_CLK_SI570) += clk-si570.o obj-$(CONFIG_COMMON_CLK_STM32F) += clk-stm32f4.o obj-$(CONFIG_COMMON_CLK_STM32H7) += clk-stm32h7.o obj-$(CONFIG_COMMON_CLK_STM32MP157) += clk-stm32mp1.o +obj-$(CONFIG_COMMON_CLK_TPS68470) += clk-tps68470.o obj-$(CONFIG_CLK_TWL6040) += clk-twl6040.o obj-$(CONFIG_ARCH_VT8500) += clk-vt8500.o obj-$(CONFIG_COMMON_CLK_VC5) += clk-versaclock5.o @@ -111,12 +113,13 @@ obj-y += sprd/ obj-$(CONFIG_ARCH_STI) += st/ obj-$(CONFIG_SOC_STARFIVE) += starfive/ obj-$(CONFIG_ARCH_SUNXI) += sunxi/ -obj-$(CONFIG_SUNXI_CCU) += sunxi-ng/ +obj-y += sunxi-ng/ obj-$(CONFIG_ARCH_TEGRA) += tegra/ obj-y += ti/ obj-$(CONFIG_CLK_UNIPHIER) += uniphier/ obj-$(CONFIG_ARCH_U8500) += ux500/ obj-y += versatile/ +obj-$(CONFIG_COMMON_CLK_VISCONTI) += visconti/ ifeq ($(CONFIG_COMMON_CLK), y) obj-$(CONFIG_X86) += x86/ endif diff --git a/drivers/clk/clk-bm1880.c b/drivers/clk/clk-bm1880.c index e6d6599d310a..fad78a22218e 100644 --- a/drivers/clk/clk-bm1880.c +++ b/drivers/clk/clk-bm1880.c @@ -522,14 +522,6 @@ static struct clk_hw *bm1880_clk_register_pll(struct bm1880_pll_hw_clock *pll_cl return hw; } -static void bm1880_clk_unregister_pll(struct clk_hw *hw) -{ - struct bm1880_pll_hw_clock *pll_hw = to_bm1880_pll_clk(hw); - - clk_hw_unregister(hw); - kfree(pll_hw); -} - static int bm1880_clk_register_plls(struct bm1880_pll_hw_clock *clks, int num_clks, struct bm1880_clock_data *data) @@ -555,7 +547,7 @@ static int bm1880_clk_register_plls(struct bm1880_pll_hw_clock *clks, err_clk: while (i--) - bm1880_clk_unregister_pll(data->hw_data.hws[clks[i].pll.id]); + clk_hw_unregister(data->hw_data.hws[clks[i].pll.id]); return PTR_ERR(hw); } @@ -695,14 +687,6 @@ static struct clk_hw *bm1880_clk_register_div(struct bm1880_div_hw_clock *div_cl return hw; } -static void bm1880_clk_unregister_div(struct clk_hw *hw) -{ - struct bm1880_div_hw_clock *div_hw = to_bm1880_div_clk(hw); - - clk_hw_unregister(hw); - kfree(div_hw); -} - static int bm1880_clk_register_divs(struct bm1880_div_hw_clock *clks, int num_clks, struct bm1880_clock_data *data) @@ -729,7 +713,7 @@ static int bm1880_clk_register_divs(struct bm1880_div_hw_clock *clks, err_clk: while (i--) - bm1880_clk_unregister_div(data->hw_data.hws[clks[i].div.id]); + clk_hw_unregister(data->hw_data.hws[clks[i].div.id]); return PTR_ERR(hw); } diff --git a/drivers/clk/clk-gate.c b/drivers/clk/clk-gate.c index 070dc47e95a1..64283807600b 100644 --- a/drivers/clk/clk-gate.c +++ b/drivers/clk/clk-gate.c @@ -7,6 +7,7 @@ */ #include +#include #include #include #include @@ -222,3 +223,37 @@ void clk_hw_unregister_gate(struct clk_hw *hw) kfree(gate); } EXPORT_SYMBOL_GPL(clk_hw_unregister_gate); + +static void devm_clk_hw_release_gate(struct device *dev, void *res) +{ + clk_hw_unregister_gate(*(struct clk_hw **)res); +} + +struct clk_hw *__devm_clk_hw_register_gate(struct device *dev, + struct device_node *np, const char *name, + const char *parent_name, const struct clk_hw *parent_hw, + const struct clk_parent_data *parent_data, + unsigned long flags, + void __iomem *reg, u8 bit_idx, + u8 clk_gate_flags, spinlock_t *lock) +{ + struct clk_hw **ptr, *hw; + + ptr = devres_alloc(devm_clk_hw_release_gate, sizeof(*ptr), GFP_KERNEL); + if (!ptr) + return ERR_PTR(-ENOMEM); + + hw = __clk_hw_register_gate(dev, np, name, parent_name, parent_hw, + parent_data, flags, reg, bit_idx, + clk_gate_flags, lock); + + if (!IS_ERR(hw)) { + *ptr = hw; + devres_add(dev, ptr); + } else { + devres_free(ptr); + } + + return hw; +} +EXPORT_SYMBOL_GPL(__devm_clk_hw_register_gate); diff --git a/drivers/clk/clk-gemini.c b/drivers/clk/clk-gemini.c index b51069e794ff..a23fa6d47ef1 100644 --- a/drivers/clk/clk-gemini.c +++ b/drivers/clk/clk-gemini.c @@ -50,7 +50,7 @@ static DEFINE_SPINLOCK(gemini_clk_lock); #define PCI_DLL_TAP_SEL_MASK 0x1f /** - * struct gemini_data_data - Gemini gated clocks + * struct gemini_gate_data - Gemini gated clocks * @bit_idx: the bit used to gate this clock in the clock register * @name: the clock name * @parent_name: the name of the parent clock diff --git a/drivers/clk/clk-lan966x.c b/drivers/clk/clk-lan966x.c new file mode 100644 index 000000000000..d1535ac13e89 --- /dev/null +++ b/drivers/clk/clk-lan966x.c @@ -0,0 +1,293 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Microchip LAN966x SoC Clock driver. + * + * Copyright (C) 2021 Microchip Technology, Inc. and its subsidiaries + * + * Author: Kavyasree Kotagiri + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#define GCK_ENA BIT(0) +#define GCK_SRC_SEL GENMASK(9, 8) +#define GCK_PRESCALER GENMASK(23, 16) + +#define DIV_MAX 255 + +static const char *clk_names[N_CLOCKS] = { + "qspi0", "qspi1", "qspi2", "sdmmc0", + "pi", "mcan0", "mcan1", "flexcom0", + "flexcom1", "flexcom2", "flexcom3", + "flexcom4", "timer1", "usb_refclk", +}; + +struct lan966x_gck { + struct clk_hw hw; + void __iomem *reg; +}; +#define to_lan966x_gck(hw) container_of(hw, struct lan966x_gck, hw) + +static const struct clk_parent_data lan966x_gck_pdata[] = { + { .fw_name = "cpu", }, + { .fw_name = "ddr", }, + { .fw_name = "sys", }, +}; + +static struct clk_init_data init = { + .parent_data = lan966x_gck_pdata, + .num_parents = ARRAY_SIZE(lan966x_gck_pdata), +}; + +struct clk_gate_soc_desc { + const char *name; + int bit_idx; +}; + +static const struct clk_gate_soc_desc clk_gate_desc[] = { + { "uhphs", 11 }, + { "udphs", 10 }, + { "mcramc", 9 }, + { "hmatrix", 8 }, + { } +}; + +static DEFINE_SPINLOCK(clk_gate_lock); +static void __iomem *base; + +static int lan966x_gck_enable(struct clk_hw *hw) +{ + struct lan966x_gck *gck = to_lan966x_gck(hw); + u32 val = readl(gck->reg); + + val |= GCK_ENA; + writel(val, gck->reg); + + return 0; +} + +static void lan966x_gck_disable(struct clk_hw *hw) +{ + struct lan966x_gck *gck = to_lan966x_gck(hw); + u32 val = readl(gck->reg); + + val &= ~GCK_ENA; + writel(val, gck->reg); +} + +static int lan966x_gck_set_rate(struct clk_hw *hw, + unsigned long rate, + unsigned long parent_rate) +{ + struct lan966x_gck *gck = to_lan966x_gck(hw); + u32 div, val = readl(gck->reg); + + if (rate == 0 || parent_rate == 0) + return -EINVAL; + + /* Set Prescalar */ + div = parent_rate / rate; + val &= ~GCK_PRESCALER; + val |= FIELD_PREP(GCK_PRESCALER, (div - 1)); + writel(val, gck->reg); + + return 0; +} + +static long lan966x_gck_round_rate(struct clk_hw *hw, unsigned long rate, + unsigned long *parent_rate) +{ + unsigned int div; + + if (rate == 0 || *parent_rate == 0) + return -EINVAL; + + if (rate >= *parent_rate) + return *parent_rate; + + div = DIV_ROUND_CLOSEST(*parent_rate, rate); + + return *parent_rate / div; +} + +static unsigned long lan966x_gck_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct lan966x_gck *gck = to_lan966x_gck(hw); + u32 div, val = readl(gck->reg); + + div = FIELD_GET(GCK_PRESCALER, val); + + return parent_rate / (div + 1); +} + +static int lan966x_gck_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + struct clk_hw *parent; + int i; + + for (i = 0; i < clk_hw_get_num_parents(hw); ++i) { + parent = clk_hw_get_parent_by_index(hw, i); + if (!parent) + continue; + + /* Allowed prescaler divider range is 0-255 */ + if (clk_hw_get_rate(parent) / req->rate <= DIV_MAX) { + req->best_parent_hw = parent; + req->best_parent_rate = clk_hw_get_rate(parent); + + return 0; + } + } + + return -EINVAL; +} + +static u8 lan966x_gck_get_parent(struct clk_hw *hw) +{ + struct lan966x_gck *gck = to_lan966x_gck(hw); + u32 val = readl(gck->reg); + + return FIELD_GET(GCK_SRC_SEL, val); +} + +static int lan966x_gck_set_parent(struct clk_hw *hw, u8 index) +{ + struct lan966x_gck *gck = to_lan966x_gck(hw); + u32 val = readl(gck->reg); + + val &= ~GCK_SRC_SEL; + val |= FIELD_PREP(GCK_SRC_SEL, index); + writel(val, gck->reg); + + return 0; +} + +static const struct clk_ops lan966x_gck_ops = { + .enable = lan966x_gck_enable, + .disable = lan966x_gck_disable, + .set_rate = lan966x_gck_set_rate, + .round_rate = lan966x_gck_round_rate, + .recalc_rate = lan966x_gck_recalc_rate, + .determine_rate = lan966x_gck_determine_rate, + .set_parent = lan966x_gck_set_parent, + .get_parent = lan966x_gck_get_parent, +}; + +static struct clk_hw *lan966x_gck_clk_register(struct device *dev, int i) +{ + struct lan966x_gck *priv; + int ret; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return ERR_PTR(-ENOMEM); + + priv->reg = base + (i * 4); + priv->hw.init = &init; + ret = devm_clk_hw_register(dev, &priv->hw); + if (ret) + return ERR_PTR(ret); + + return &priv->hw; +}; + +static int lan966x_gate_clk_register(struct device *dev, + struct clk_hw_onecell_data *hw_data, + void __iomem *gate_base) +{ + int i; + + for (i = GCK_GATE_UHPHS; i < N_CLOCKS; ++i) { + int idx = i - GCK_GATE_UHPHS; + + hw_data->hws[i] = + devm_clk_hw_register_gate(dev, clk_gate_desc[idx].name, + "lan966x", 0, base, + clk_gate_desc[idx].bit_idx, + 0, &clk_gate_lock); + + if (IS_ERR(hw_data->hws[i])) + return dev_err_probe(dev, PTR_ERR(hw_data->hws[i]), + "failed to register %s clock\n", + clk_gate_desc[idx].name); + } + + return 0; +} + +static int lan966x_clk_probe(struct platform_device *pdev) +{ + struct clk_hw_onecell_data *hw_data; + struct device *dev = &pdev->dev; + void __iomem *gate_base; + struct resource *res; + int i, ret; + + hw_data = devm_kzalloc(dev, struct_size(hw_data, hws, N_CLOCKS), + GFP_KERNEL); + if (!hw_data) + return -ENOMEM; + + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); + + init.ops = &lan966x_gck_ops; + + hw_data->num = GCK_GATE_UHPHS; + + for (i = 0; i < GCK_GATE_UHPHS; i++) { + init.name = clk_names[i]; + hw_data->hws[i] = lan966x_gck_clk_register(dev, i); + if (IS_ERR(hw_data->hws[i])) { + dev_err(dev, "failed to register %s clock\n", + init.name); + return PTR_ERR(hw_data->hws[i]); + } + } + + res = platform_get_resource(pdev, IORESOURCE_MEM, 1); + if (res) { + gate_base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(gate_base)) + return PTR_ERR(gate_base); + + hw_data->num = N_CLOCKS; + + ret = lan966x_gate_clk_register(dev, hw_data, gate_base); + if (ret) + return ret; + } + + return devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, hw_data); +} + +static const struct of_device_id lan966x_clk_dt_ids[] = { + { .compatible = "microchip,lan966x-gck", }, + { } +}; +MODULE_DEVICE_TABLE(of, lan966x_clk_dt_ids); + +static struct platform_driver lan966x_clk_driver = { + .probe = lan966x_clk_probe, + .driver = { + .name = "lan966x-clk", + .of_match_table = lan966x_clk_dt_ids, + }, +}; +builtin_platform_driver(lan966x_clk_driver); + +MODULE_AUTHOR("Kavyasree Kotagiri "); +MODULE_DESCRIPTION("LAN966X clock driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/clk-stm32f4.c b/drivers/clk/clk-stm32f4.c index af46176ad053..473dfe632cc5 100644 --- a/drivers/clk/clk-stm32f4.c +++ b/drivers/clk/clk-stm32f4.c @@ -129,7 +129,6 @@ static const struct stm32f4_gate_data stm32f429_gates[] __initconst = { { STM32F4_RCC_APB2ENR, 20, "spi5", "apb2_div" }, { STM32F4_RCC_APB2ENR, 21, "spi6", "apb2_div" }, { STM32F4_RCC_APB2ENR, 22, "sai1", "apb2_div" }, - { STM32F4_RCC_APB2ENR, 26, "ltdc", "apb2_div" }, }; static const struct stm32f4_gate_data stm32f469_gates[] __initconst = { @@ -211,7 +210,6 @@ static const struct stm32f4_gate_data stm32f469_gates[] __initconst = { { STM32F4_RCC_APB2ENR, 20, "spi5", "apb2_div" }, { STM32F4_RCC_APB2ENR, 21, "spi6", "apb2_div" }, { STM32F4_RCC_APB2ENR, 22, "sai1", "apb2_div" }, - { STM32F4_RCC_APB2ENR, 26, "ltdc", "apb2_div" }, }; static const struct stm32f4_gate_data stm32f746_gates[] __initconst = { @@ -286,7 +284,6 @@ static const struct stm32f4_gate_data stm32f746_gates[] __initconst = { { STM32F4_RCC_APB2ENR, 21, "spi6", "apb2_div" }, { STM32F4_RCC_APB2ENR, 22, "sai1", "apb2_div" }, { STM32F4_RCC_APB2ENR, 23, "sai2", "apb2_div" }, - { STM32F4_RCC_APB2ENR, 26, "ltdc", "apb2_div" }, }; static const struct stm32f4_gate_data stm32f769_gates[] __initconst = { @@ -364,7 +361,6 @@ static const struct stm32f4_gate_data stm32f769_gates[] __initconst = { { STM32F4_RCC_APB2ENR, 21, "spi6", "apb2_div" }, { STM32F4_RCC_APB2ENR, 22, "sai1", "apb2_div" }, { STM32F4_RCC_APB2ENR, 23, "sai2", "apb2_div" }, - { STM32F4_RCC_APB2ENR, 26, "ltdc", "apb2_div" }, { STM32F4_RCC_APB2ENR, 30, "mdio", "apb2_div" }, }; diff --git a/drivers/clk/clk-stm32mp1.c b/drivers/clk/clk-stm32mp1.c index 4bd1fe7d8af4..863274aa50e3 100644 --- a/drivers/clk/clk-stm32mp1.c +++ b/drivers/clk/clk-stm32mp1.c @@ -2253,8 +2253,6 @@ static int stm32_rcc_reset_init(struct device *dev, void __iomem *base, const struct stm32_rcc_match_data *data = match->data; struct stm32_reset_data *reset_data = NULL; - data = match->data; - reset_data = kzalloc(sizeof(*reset_data), GFP_KERNEL); if (!reset_data) return -ENOMEM; diff --git a/drivers/clk/clk-tps68470.c b/drivers/clk/clk-tps68470.c new file mode 100644 index 000000000000..e5fbefd6ac2d --- /dev/null +++ b/drivers/clk/clk-tps68470.c @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Clock driver for TPS68470 PMIC + * + * Copyright (c) 2021 Red Hat Inc. + * Copyright (C) 2018 Intel Corporation + * + * Authors: + * Hans de Goede + * Zaikuo Wang + * Tianshu Qiu + * Jian Xu Zheng + * Yuning Pu + * Antti Laakso + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define TPS68470_CLK_NAME "tps68470-clk" + +#define to_tps68470_clkdata(clkd) \ + container_of(clkd, struct tps68470_clkdata, clkout_hw) + +static struct tps68470_clkout_freqs { + unsigned long freq; + unsigned int xtaldiv; + unsigned int plldiv; + unsigned int postdiv; + unsigned int buckdiv; + unsigned int boostdiv; +} clk_freqs[] = { +/* + * The PLL is used to multiply the crystal oscillator + * frequency range of 3 MHz to 27 MHz by a programmable + * factor of F = (M/N)*(1/P) such that the output + * available at the HCLK_A or HCLK_B pins are in the range + * of 4 MHz to 64 MHz in increments of 0.1 MHz. + * + * hclk_# = osc_in * (((plldiv*2)+320) / (xtaldiv+30)) * (1 / 2^postdiv) + * + * PLL_REF_CLK should be as close as possible to 100kHz + * PLL_REF_CLK = input clk / XTALDIV[7:0] + 30) + * + * PLL_VCO_CLK = (PLL_REF_CLK * (plldiv*2 + 320)) + * + * BOOST should be as close as possible to 2Mhz + * BOOST = PLL_VCO_CLK / (BOOSTDIV[4:0] + 16) * + * + * BUCK should be as close as possible to 5.2Mhz + * BUCK = PLL_VCO_CLK / (BUCKDIV[3:0] + 5) + * + * osc_in xtaldiv plldiv postdiv hclk_# + * 20Mhz 170 32 1 19.2Mhz + * 20Mhz 170 40 1 20Mhz + * 20Mhz 170 80 1 24Mhz + */ + { 19200000, 170, 32, 1, 2, 3 }, + { 20000000, 170, 40, 1, 3, 4 }, + { 24000000, 170, 80, 1, 4, 8 }, +}; + +struct tps68470_clkdata { + struct clk_hw clkout_hw; + struct regmap *regmap; + unsigned long rate; +}; + +static int tps68470_clk_is_prepared(struct clk_hw *hw) +{ + struct tps68470_clkdata *clkdata = to_tps68470_clkdata(hw); + int val; + + if (regmap_read(clkdata->regmap, TPS68470_REG_PLLCTL, &val)) + return 0; + + return val & TPS68470_PLL_EN_MASK; +} + +static int tps68470_clk_prepare(struct clk_hw *hw) +{ + struct tps68470_clkdata *clkdata = to_tps68470_clkdata(hw); + + regmap_write(clkdata->regmap, TPS68470_REG_CLKCFG1, + (TPS68470_PLL_OUTPUT_ENABLE << TPS68470_OUTPUT_A_SHIFT) | + (TPS68470_PLL_OUTPUT_ENABLE << TPS68470_OUTPUT_B_SHIFT)); + + regmap_update_bits(clkdata->regmap, TPS68470_REG_PLLCTL, + TPS68470_PLL_EN_MASK, TPS68470_PLL_EN_MASK); + + /* + * The PLLCTL reg lock bit is set by the PMIC after approx. 4ms and + * does not indicate a true lock, so just wait 4 ms. + */ + usleep_range(4000, 5000); + + return 0; +} + +static void tps68470_clk_unprepare(struct clk_hw *hw) +{ + struct tps68470_clkdata *clkdata = to_tps68470_clkdata(hw); + + /* Disable clock first ... */ + regmap_update_bits(clkdata->regmap, TPS68470_REG_PLLCTL, TPS68470_PLL_EN_MASK, 0); + + /* ... and then tri-state the clock outputs. */ + regmap_write(clkdata->regmap, TPS68470_REG_CLKCFG1, 0); +} + +static unsigned long tps68470_clk_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) +{ + struct tps68470_clkdata *clkdata = to_tps68470_clkdata(hw); + + return clkdata->rate; +} + +/* + * This returns the index of the clk_freqs[] cfg with the closest rate for + * use in tps68470_clk_round_rate(). tps68470_clk_set_rate() checks that + * the rate of the returned cfg is an exact match. + */ +static unsigned int tps68470_clk_cfg_lookup(unsigned long rate) +{ + long diff, best_diff = LONG_MAX; + unsigned int i, best_idx = 0; + + for (i = 0; i < ARRAY_SIZE(clk_freqs); i++) { + diff = clk_freqs[i].freq - rate; + if (diff == 0) + return i; + + diff = abs(diff); + if (diff < best_diff) { + best_diff = diff; + best_idx = i; + } + } + + return best_idx; +} + +static long tps68470_clk_round_rate(struct clk_hw *hw, unsigned long rate, + unsigned long *parent_rate) +{ + unsigned int idx = tps68470_clk_cfg_lookup(rate); + + return clk_freqs[idx].freq; +} + +static int tps68470_clk_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct tps68470_clkdata *clkdata = to_tps68470_clkdata(hw); + unsigned int idx = tps68470_clk_cfg_lookup(rate); + + if (rate != clk_freqs[idx].freq) + return -EINVAL; + + regmap_write(clkdata->regmap, TPS68470_REG_BOOSTDIV, clk_freqs[idx].boostdiv); + regmap_write(clkdata->regmap, TPS68470_REG_BUCKDIV, clk_freqs[idx].buckdiv); + regmap_write(clkdata->regmap, TPS68470_REG_PLLSWR, TPS68470_PLLSWR_DEFAULT); + regmap_write(clkdata->regmap, TPS68470_REG_XTALDIV, clk_freqs[idx].xtaldiv); + regmap_write(clkdata->regmap, TPS68470_REG_PLLDIV, clk_freqs[idx].plldiv); + regmap_write(clkdata->regmap, TPS68470_REG_POSTDIV, clk_freqs[idx].postdiv); + regmap_write(clkdata->regmap, TPS68470_REG_POSTDIV2, clk_freqs[idx].postdiv); + regmap_write(clkdata->regmap, TPS68470_REG_CLKCFG2, TPS68470_CLKCFG2_DRV_STR_2MA); + + regmap_write(clkdata->regmap, TPS68470_REG_PLLCTL, + TPS68470_OSC_EXT_CAP_DEFAULT << TPS68470_OSC_EXT_CAP_SHIFT | + TPS68470_CLK_SRC_XTAL << TPS68470_CLK_SRC_SHIFT); + + clkdata->rate = rate; + + return 0; +} + +static const struct clk_ops tps68470_clk_ops = { + .is_prepared = tps68470_clk_is_prepared, + .prepare = tps68470_clk_prepare, + .unprepare = tps68470_clk_unprepare, + .recalc_rate = tps68470_clk_recalc_rate, + .round_rate = tps68470_clk_round_rate, + .set_rate = tps68470_clk_set_rate, +}; + +static int tps68470_clk_probe(struct platform_device *pdev) +{ + struct tps68470_clk_platform_data *pdata = pdev->dev.platform_data; + struct clk_init_data tps68470_clk_initdata = { + .name = TPS68470_CLK_NAME, + .ops = &tps68470_clk_ops, + /* Changing the dividers when the PLL is on is not allowed */ + .flags = CLK_SET_RATE_GATE, + }; + struct tps68470_clkdata *tps68470_clkdata; + int ret; + + tps68470_clkdata = devm_kzalloc(&pdev->dev, sizeof(*tps68470_clkdata), + GFP_KERNEL); + if (!tps68470_clkdata) + return -ENOMEM; + + tps68470_clkdata->regmap = dev_get_drvdata(pdev->dev.parent); + tps68470_clkdata->clkout_hw.init = &tps68470_clk_initdata; + + /* Set initial rate */ + tps68470_clk_set_rate(&tps68470_clkdata->clkout_hw, clk_freqs[0].freq, 0); + + ret = devm_clk_hw_register(&pdev->dev, &tps68470_clkdata->clkout_hw); + if (ret) + return ret; + + ret = devm_clk_hw_register_clkdev(&pdev->dev, &tps68470_clkdata->clkout_hw, + TPS68470_CLK_NAME, NULL); + if (ret) + return ret; + + if (pdata) { + ret = devm_clk_hw_register_clkdev(&pdev->dev, + &tps68470_clkdata->clkout_hw, + pdata->consumer_con_id, + pdata->consumer_dev_name); + } + + return ret; +} + +static struct platform_driver tps68470_clk_driver = { + .driver = { + .name = TPS68470_CLK_NAME, + }, + .probe = tps68470_clk_probe, +}; + +/* + * The ACPI tps68470 probe-ordering depends on the clk/gpio/regulator drivers + * registering before the drivers for the camera-sensors which use them bind. + * subsys_initcall() ensures this when the drivers are builtin. + */ +static int __init tps68470_clk_init(void) +{ + return platform_driver_register(&tps68470_clk_driver); +} +subsys_initcall(tps68470_clk_init); + +static void __exit tps68470_clk_exit(void) +{ + platform_driver_unregister(&tps68470_clk_driver); +} +module_exit(tps68470_clk_exit); + +MODULE_ALIAS("platform:tps68470-clk"); +MODULE_DESCRIPTION("clock driver for TPS68470 pmic"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c index f44cb6d4be01..dc9de5a68e97 100644 --- a/drivers/clk/clk.c +++ b/drivers/clk/clk.c @@ -426,19 +426,20 @@ static void clk_core_fill_parent_index(struct clk_core *core, u8 index) if (entry->hw) { parent = entry->hw->core; - /* - * We have a direct reference but it isn't registered yet? - * Orphan it and let clk_reparent() update the orphan status - * when the parent is registered. - */ - if (!parent) - parent = ERR_PTR(-EPROBE_DEFER); } else { parent = clk_core_get(core, index); if (PTR_ERR(parent) == -ENOENT && entry->name) parent = clk_core_lookup(entry->name); } + /* + * We have a direct reference but it isn't registered yet? + * Orphan it and let clk_reparent() update the orphan status + * when the parent is registered. + */ + if (!parent) + parent = ERR_PTR(-EPROBE_DEFER); + /* Only cache it if it's not an error */ if (!IS_ERR(parent)) entry->core = parent; @@ -3044,7 +3045,9 @@ static void clk_summary_show_subtree(struct seq_file *s, struct clk_core *c, { struct clk_core *child; + clk_pm_runtime_get(c); clk_summary_show_one(s, c, level); + clk_pm_runtime_put(c); hlist_for_each_entry(child, &c->children, child_node) clk_summary_show_subtree(s, child, level + 1); @@ -3296,6 +3299,42 @@ static int current_parent_show(struct seq_file *s, void *data) } DEFINE_SHOW_ATTRIBUTE(current_parent); +#ifdef CLOCK_ALLOW_WRITE_DEBUGFS +static ssize_t current_parent_write(struct file *file, const char __user *ubuf, + size_t count, loff_t *ppos) +{ + struct seq_file *s = file->private_data; + struct clk_core *core = s->private; + struct clk_core *parent; + u8 idx; + int err; + + err = kstrtou8_from_user(ubuf, count, 0, &idx); + if (err < 0) + return err; + + parent = clk_core_get_parent_by_index(core, idx); + if (!parent) + return -ENOENT; + + clk_prepare_lock(); + err = clk_core_set_parent_nolock(core, parent); + clk_prepare_unlock(); + if (err) + return err; + + return count; +} + +static const struct file_operations current_parent_rw_fops = { + .open = current_parent_open, + .write = current_parent_write, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; +#endif + static int clk_duty_cycle_show(struct seq_file *s, void *data) { struct clk_core *core = s->private; @@ -3361,8 +3400,12 @@ static void clk_debug_create_one(struct clk_core *core, struct dentry *pdentry) #ifdef CLOCK_ALLOW_WRITE_DEBUGFS debugfs_create_file("clk_prepare_enable", 0644, root, core, &clk_prepare_enable_fops); -#endif + if (core->num_parents > 1) + debugfs_create_file("clk_parent", 0644, root, core, + ¤t_parent_rw_fops); + else +#endif if (core->num_parents > 0) debugfs_create_file("clk_parent", 0444, root, core, ¤t_parent_fops); @@ -3422,6 +3465,24 @@ static int __init clk_debug_init(void) { struct clk_core *core; +#ifdef CLOCK_ALLOW_WRITE_DEBUGFS + pr_warn("\n"); + pr_warn("********************************************************************\n"); + pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n"); + pr_warn("** **\n"); + pr_warn("** WRITEABLE clk DebugFS SUPPORT HAS BEEN ENABLED IN THIS KERNEL **\n"); + pr_warn("** **\n"); + pr_warn("** This means that this kernel is built to expose clk operations **\n"); + pr_warn("** such as parent or rate setting, enabling, disabling, etc. **\n"); + pr_warn("** to userspace, which may compromise security on your system. **\n"); + pr_warn("** **\n"); + pr_warn("** If you see this message and you are not debugging the **\n"); + pr_warn("** kernel, report this immediately to your vendor! **\n"); + pr_warn("** **\n"); + pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n"); + pr_warn("********************************************************************\n"); +#endif + rootdir = debugfs_create_dir("clk", NULL); debugfs_create_file("clk_summary", 0444, rootdir, &all_lists, @@ -3493,9 +3554,6 @@ static int __clk_core_init(struct clk_core *core) unsigned long rate; int phase; - if (!core) - return -EINVAL; - clk_prepare_lock(); /* diff --git a/drivers/clk/imx/clk-imx8mn.c b/drivers/clk/imx/clk-imx8mn.c index c55577604e16..021355a24708 100644 --- a/drivers/clk/imx/clk-imx8mn.c +++ b/drivers/clk/imx/clk-imx8mn.c @@ -277,9 +277,9 @@ static const char * const imx8mn_pdm_sels[] = {"osc_24m", "sys_pll2_100m", "audi static const char * const imx8mn_dram_core_sels[] = {"dram_pll_out", "dram_alt_root", }; -static const char * const imx8mn_clko1_sels[] = {"osc_24m", "sys_pll1_800m", "osc_27m", - "sys_pll1_200m", "audio_pll2_out", "vpu_pll", - "sys_pll1_80m", }; +static const char * const imx8mn_clko1_sels[] = {"osc_24m", "sys_pll1_800m", "dummy", + "sys_pll1_200m", "audio_pll2_out", "sys_pll2_500m", + "dummy", "sys_pll1_80m", }; static const char * const imx8mn_clko2_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll1_400m", "sys_pll2_166m", "sys_pll3_out", "audio_pll1_out", "video_pll1_out", "osc_32k", }; diff --git a/drivers/clk/imx/clk-imx8mp.c b/drivers/clk/imx/clk-imx8mp.c index 12837304545d..c990ad37882b 100644 --- a/drivers/clk/imx/clk-imx8mp.c +++ b/drivers/clk/imx/clk-imx8mp.c @@ -700,7 +700,7 @@ static int imx8mp_clocks_probe(struct platform_device *pdev) hws[IMX8MP_CLK_HDMI_ROOT] = imx_clk_hw_gate4("hdmi_root_clk", "hdmi_axi", ccm_base + 0x45f0, 0); hws[IMX8MP_CLK_TSENSOR_ROOT] = imx_clk_hw_gate4("tsensor_root_clk", "ipg_root", ccm_base + 0x4620, 0); hws[IMX8MP_CLK_VPU_ROOT] = imx_clk_hw_gate4("vpu_root_clk", "vpu_bus", ccm_base + 0x4630, 0); - hws[IMX8MP_CLK_AUDIO_ROOT] = imx_clk_hw_gate4("audio_root_clk", "ipg_root", ccm_base + 0x4650, 0); + hws[IMX8MP_CLK_AUDIO_ROOT] = imx_clk_hw_gate4("audio_root_clk", "audio_ahb", ccm_base + 0x4650, 0); hws[IMX8MP_CLK_ARM] = imx_clk_hw_cpu("arm", "arm_a53_core", hws[IMX8MP_CLK_A53_CORE]->clk, diff --git a/drivers/clk/imx/clk-imx8ulp.c b/drivers/clk/imx/clk-imx8ulp.c index 6699437e17b8..8eb1af2d6429 100644 --- a/drivers/clk/imx/clk-imx8ulp.c +++ b/drivers/clk/imx/clk-imx8ulp.c @@ -559,6 +559,7 @@ static struct platform_driver imx8ulp_clk_driver = { .probe = imx8ulp_clk_probe, .driver = { .name = KBUILD_MODNAME, + .suppress_bind_attrs = true, .of_match_table = imx8ulp_clk_dt_ids, }, }; diff --git a/drivers/clk/imx/clk-pllv1.c b/drivers/clk/imx/clk-pllv1.c index 36ffb0525735..93ee81b28fc7 100644 --- a/drivers/clk/imx/clk-pllv1.c +++ b/drivers/clk/imx/clk-pllv1.c @@ -8,20 +8,19 @@ #include "clk.h" -/** - * pll v1 - * - * @clk_hw clock source - * @parent the parent clock name - * @base base address of pll registers - * - * PLL clock version 1, found on i.MX1/21/25/27/31/35 - */ - #define MFN_BITS (10) #define MFN_SIGN (BIT(MFN_BITS - 1)) #define MFN_MASK (MFN_SIGN - 1) +/** + * struct clk_pllv1 - IMX PLLv1 clock descriptor + * + * @hw: clock source + * @base: base address of pll registers + * @type: type of IMX_PLLV1 + * + * PLL clock version 1, found on i.MX1/21/25/27/31/35 + */ struct clk_pllv1 { struct clk_hw hw; void __iomem *base; diff --git a/drivers/clk/imx/clk-pllv3.c b/drivers/clk/imx/clk-pllv3.c index 20ee9611ba6e..eea32f87c60a 100644 --- a/drivers/clk/imx/clk-pllv3.c +++ b/drivers/clk/imx/clk-pllv3.c @@ -247,7 +247,7 @@ static long clk_pllv3_av_round_rate(struct clk_hw *hw, unsigned long rate, div = rate / parent_rate; temp64 = (u64) (rate - div * parent_rate); temp64 *= mfd; - do_div(temp64, parent_rate); + temp64 = div64_ul(temp64, parent_rate); mfn = temp64; temp64 = (u64)parent_rate; @@ -277,7 +277,7 @@ static int clk_pllv3_av_set_rate(struct clk_hw *hw, unsigned long rate, div = rate / parent_rate; temp64 = (u64) (rate - div * parent_rate); temp64 *= mfd; - do_div(temp64, parent_rate); + temp64 = div64_ul(temp64, parent_rate); mfn = temp64; val = readl_relaxed(pll->base); @@ -334,7 +334,7 @@ static struct clk_pllv3_vf610_mf clk_pllv3_vf610_rate_to_mf( /* rate = parent_rate * (mfi + mfn/mfd) */ temp64 = rate - parent_rate * mf.mfi; temp64 *= mf.mfd; - do_div(temp64, parent_rate); + temp64 = div64_ul(temp64, parent_rate); mf.mfn = temp64; } diff --git a/drivers/clk/ingenic/jz4760-cgu.c b/drivers/clk/ingenic/jz4760-cgu.c index 080d492ac95c..8fdd383560fb 100644 --- a/drivers/clk/ingenic/jz4760-cgu.c +++ b/drivers/clk/ingenic/jz4760-cgu.c @@ -313,6 +313,16 @@ static const struct ingenic_cgu_clk_info jz4760_cgu_clocks[] = { .parents = { JZ4760_CLK_H2CLK, }, .gate = { CGU_REG_CLKGR0, 21 }, }, + [JZ4760_CLK_MDMA] = { + "mdma", CGU_CLK_GATE, + .parents = { JZ4760_CLK_HCLK, }, + .gate = { CGU_REG_CLKGR0, 25 }, + }, + [JZ4760_CLK_BDMA] = { + "bdma", CGU_CLK_GATE, + .parents = { JZ4760_CLK_HCLK, }, + .gate = { CGU_REG_CLKGR1, 0 }, + }, [JZ4760_CLK_I2C0] = { "i2c0", CGU_CLK_GATE, .parents = { JZ4760_CLK_EXT, }, diff --git a/drivers/clk/ingenic/jz4770-cgu.c b/drivers/clk/ingenic/jz4770-cgu.c index 8c6c1208f462..7ef91257630e 100644 --- a/drivers/clk/ingenic/jz4770-cgu.c +++ b/drivers/clk/ingenic/jz4770-cgu.c @@ -329,6 +329,11 @@ static const struct ingenic_cgu_clk_info jz4770_cgu_clocks[] = { .parents = { JZ4770_CLK_H2CLK, }, .gate = { CGU_REG_CLKGR0, 21 }, }, + [JZ4770_CLK_BDMA] = { + "bdma", CGU_CLK_GATE, + .parents = { JZ4770_CLK_H2CLK, }, + .gate = { CGU_REG_CLKGR1, 0 }, + }, [JZ4770_CLK_I2C0] = { "i2c0", CGU_CLK_GATE, .parents = { JZ4770_CLK_EXT, }, diff --git a/drivers/clk/mediatek/Kconfig b/drivers/clk/mediatek/Kconfig index 3ce6fb04d8ff..01ef02c54725 100644 --- a/drivers/clk/mediatek/Kconfig +++ b/drivers/clk/mediatek/Kconfig @@ -344,6 +344,23 @@ config COMMON_CLK_MT7629_HIFSYS This driver supports MediaTek MT7629 HIFSYS clocks providing to PCI-E and USB. +config COMMON_CLK_MT7986 + bool "Clock driver for MediaTek MT7986" + depends on ARCH_MEDIATEK || COMPILE_TEST + select COMMON_CLK_MEDIATEK + default ARCH_MEDIATEK + help + This driver supports MediaTek MT7986 basic clocks and clocks + required for various peripherals found on MediaTek. + +config COMMON_CLK_MT7986_ETHSYS + bool "Clock driver for MediaTek MT7986 ETHSYS" + depends on COMMON_CLK_MT7986 + default COMMON_CLK_MT7986 + help + This driver adds support for clocks for Ethernet and SGMII + required on MediaTek MT7986 SoC. + config COMMON_CLK_MT8135 bool "Clock driver for MediaTek MT8135" depends on (ARCH_MEDIATEK && ARM) || COMPILE_TEST diff --git a/drivers/clk/mediatek/Makefile b/drivers/clk/mediatek/Makefile index dc96038a0155..7b0c2646ce4a 100644 --- a/drivers/clk/mediatek/Makefile +++ b/drivers/clk/mediatek/Makefile @@ -46,6 +46,10 @@ obj-$(CONFIG_COMMON_CLK_MT7622_AUDSYS) += clk-mt7622-aud.o obj-$(CONFIG_COMMON_CLK_MT7629) += clk-mt7629.o obj-$(CONFIG_COMMON_CLK_MT7629_ETHSYS) += clk-mt7629-eth.o obj-$(CONFIG_COMMON_CLK_MT7629_HIFSYS) += clk-mt7629-hif.o +obj-$(CONFIG_COMMON_CLK_MT7986) += clk-mt7986-apmixed.o +obj-$(CONFIG_COMMON_CLK_MT7986) += clk-mt7986-topckgen.o +obj-$(CONFIG_COMMON_CLK_MT7986) += clk-mt7986-infracfg.o +obj-$(CONFIG_COMMON_CLK_MT7986_ETHSYS) += clk-mt7986-eth.o obj-$(CONFIG_COMMON_CLK_MT8135) += clk-mt8135.o obj-$(CONFIG_COMMON_CLK_MT8167) += clk-mt8167.o obj-$(CONFIG_COMMON_CLK_MT8167_AUDSYS) += clk-mt8167-aud.o diff --git a/drivers/clk/mediatek/clk-gate.c b/drivers/clk/mediatek/clk-gate.c index b02d2f74dd0d..5d88b428565b 100644 --- a/drivers/clk/mediatek/clk-gate.c +++ b/drivers/clk/mediatek/clk-gate.c @@ -16,28 +16,24 @@ #include "clk-mtk.h" #include "clk-gate.h" -static int mtk_cg_bit_is_cleared(struct clk_hw *hw) +static u32 mtk_get_clockgating(struct clk_hw *hw) { struct mtk_clk_gate *cg = to_mtk_clk_gate(hw); u32 val; regmap_read(cg->regmap, cg->sta_ofs, &val); - val &= BIT(cg->bit); + return val & BIT(cg->bit); +} - return val == 0; +static int mtk_cg_bit_is_cleared(struct clk_hw *hw) +{ + return mtk_get_clockgating(hw) == 0; } static int mtk_cg_bit_is_set(struct clk_hw *hw) { - struct mtk_clk_gate *cg = to_mtk_clk_gate(hw); - u32 val; - - regmap_read(cg->regmap, cg->sta_ofs, &val); - - val &= BIT(cg->bit); - - return val != 0; + return mtk_get_clockgating(hw) != 0; } static void mtk_cg_set_bit(struct clk_hw *hw) @@ -57,17 +53,15 @@ static void mtk_cg_clr_bit(struct clk_hw *hw) static void mtk_cg_set_bit_no_setclr(struct clk_hw *hw) { struct mtk_clk_gate *cg = to_mtk_clk_gate(hw); - u32 cgbit = BIT(cg->bit); - regmap_update_bits(cg->regmap, cg->sta_ofs, cgbit, cgbit); + regmap_set_bits(cg->regmap, cg->sta_ofs, BIT(cg->bit)); } static void mtk_cg_clr_bit_no_setclr(struct clk_hw *hw) { struct mtk_clk_gate *cg = to_mtk_clk_gate(hw); - u32 cgbit = BIT(cg->bit); - regmap_update_bits(cg->regmap, cg->sta_ofs, cgbit, 0); + regmap_clear_bits(cg->regmap, cg->sta_ofs, BIT(cg->bit)); } static int mtk_cg_enable(struct clk_hw *hw) diff --git a/drivers/clk/mediatek/clk-mt7986-apmixed.c b/drivers/clk/mediatek/clk-mt7986-apmixed.c new file mode 100644 index 000000000000..76c8ebdeae96 --- /dev/null +++ b/drivers/clk/mediatek/clk-mt7986-apmixed.c @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-1.0 +/* + * Copyright (c) 2021 MediaTek Inc. + * Author: Sam Shih + * Author: Wenzhen Yu + */ + +#include +#include +#include +#include +#include +#include "clk-mtk.h" +#include "clk-gate.h" +#include "clk-mux.h" + +#include +#include + +#define MT7986_PLL_FMAX (2500UL * MHZ) +#define CON0_MT7986_RST_BAR BIT(27) + +#define PLL_xtal(_id, _name, _reg, _pwr_reg, _en_mask, _flags, _pcwbits, \ + _pd_reg, _pd_shift, _tuner_reg, _pcw_reg, _pcw_shift, \ + _div_table, _parent_name) \ + { \ + .id = _id, .name = _name, .reg = _reg, .pwr_reg = _pwr_reg, \ + .en_mask = _en_mask, .flags = _flags, \ + .rst_bar_mask = CON0_MT7986_RST_BAR, .fmax = MT7986_PLL_FMAX, \ + .pcwbits = _pcwbits, .pd_reg = _pd_reg, .pd_shift = _pd_shift, \ + .tuner_reg = _tuner_reg, .pcw_reg = _pcw_reg, \ + .pcw_shift = _pcw_shift, .div_table = _div_table, \ + .parent_name = _parent_name, \ + } + +#define PLL(_id, _name, _reg, _pwr_reg, _en_mask, _flags, _pcwbits, _pd_reg, \ + _pd_shift, _tuner_reg, _pcw_reg, _pcw_shift) \ + PLL_xtal(_id, _name, _reg, _pwr_reg, _en_mask, _flags, _pcwbits, \ + _pd_reg, _pd_shift, _tuner_reg, _pcw_reg, _pcw_shift, NULL, \ + "clkxtal") + +static const struct mtk_pll_data plls[] = { + PLL(CLK_APMIXED_ARMPLL, "armpll", 0x0200, 0x020C, 0x00000001, 0, 32, + 0x0200, 4, 0, 0x0204, 0), + PLL(CLK_APMIXED_NET2PLL, "net2pll", 0x0210, 0x021C, 0x00000001, 0, 32, + 0x0210, 4, 0, 0x0214, 0), + PLL(CLK_APMIXED_MMPLL, "mmpll", 0x0220, 0x022C, 0x00000001, 0, 32, + 0x0220, 4, 0, 0x0224, 0), + PLL(CLK_APMIXED_SGMPLL, "sgmpll", 0x0230, 0x023c, 0x00000001, 0, 32, + 0x0230, 4, 0, 0x0234, 0), + PLL(CLK_APMIXED_WEDMCUPLL, "wedmcupll", 0x0240, 0x024c, 0x00000001, 0, + 32, 0x0240, 4, 0, 0x0244, 0), + PLL(CLK_APMIXED_NET1PLL, "net1pll", 0x0250, 0x025c, 0x00000001, 0, 32, + 0x0250, 4, 0, 0x0254, 0), + PLL(CLK_APMIXED_MPLL, "mpll", 0x0260, 0x0270, 0x00000001, 0, 32, 0x0260, + 4, 0, 0x0264, 0), + PLL(CLK_APMIXED_APLL2, "apll2", 0x0278, 0x0288, 0x00000001, 0, 32, + 0x0278, 4, 0, 0x027c, 0), +}; + +static const struct of_device_id of_match_clk_mt7986_apmixed[] = { + { .compatible = "mediatek,mt7986-apmixedsys", }, + {} +}; + +static int clk_mt7986_apmixed_probe(struct platform_device *pdev) +{ + struct clk_onecell_data *clk_data; + struct device_node *node = pdev->dev.of_node; + int r; + + clk_data = mtk_alloc_clk_data(ARRAY_SIZE(plls)); + if (!clk_data) + return -ENOMEM; + + mtk_clk_register_plls(node, plls, ARRAY_SIZE(plls), clk_data); + + clk_prepare_enable(clk_data->clks[CLK_APMIXED_ARMPLL]); + + r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data); + if (r) { + pr_err("%s(): could not register clock provider: %d\n", + __func__, r); + goto free_apmixed_data; + } + return r; + +free_apmixed_data: + mtk_free_clk_data(clk_data); + return r; +} + +static struct platform_driver clk_mt7986_apmixed_drv = { + .probe = clk_mt7986_apmixed_probe, + .driver = { + .name = "clk-mt7986-apmixed", + .of_match_table = of_match_clk_mt7986_apmixed, + }, +}; +builtin_platform_driver(clk_mt7986_apmixed_drv); diff --git a/drivers/clk/mediatek/clk-mt7986-eth.c b/drivers/clk/mediatek/clk-mt7986-eth.c new file mode 100644 index 000000000000..495d023ccad7 --- /dev/null +++ b/drivers/clk/mediatek/clk-mt7986-eth.c @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2021 MediaTek Inc. + * Author: Sam Shih + * Author: Wenzhen Yu + */ + +#include +#include +#include +#include +#include + +#include "clk-mtk.h" +#include "clk-gate.h" + +#include + +static const struct mtk_gate_regs sgmii0_cg_regs = { + .set_ofs = 0xe4, + .clr_ofs = 0xe4, + .sta_ofs = 0xe4, +}; + +#define GATE_SGMII0(_id, _name, _parent, _shift) \ + { \ + .id = _id, .name = _name, .parent_name = _parent, \ + .regs = &sgmii0_cg_regs, .shift = _shift, \ + .ops = &mtk_clk_gate_ops_no_setclr_inv, \ + } + +static const struct mtk_gate sgmii0_clks[] __initconst = { + GATE_SGMII0(CLK_SGMII0_TX250M_EN, "sgmii0_tx250m_en", "top_xtal", 2), + GATE_SGMII0(CLK_SGMII0_RX250M_EN, "sgmii0_rx250m_en", "top_xtal", 3), + GATE_SGMII0(CLK_SGMII0_CDR_REF, "sgmii0_cdr_ref", "top_xtal", 4), + GATE_SGMII0(CLK_SGMII0_CDR_FB, "sgmii0_cdr_fb", "top_xtal", 5), +}; + +static const struct mtk_gate_regs sgmii1_cg_regs = { + .set_ofs = 0xe4, + .clr_ofs = 0xe4, + .sta_ofs = 0xe4, +}; + +#define GATE_SGMII1(_id, _name, _parent, _shift) \ + { \ + .id = _id, .name = _name, .parent_name = _parent, \ + .regs = &sgmii1_cg_regs, .shift = _shift, \ + .ops = &mtk_clk_gate_ops_no_setclr_inv, \ + } + +static const struct mtk_gate sgmii1_clks[] __initconst = { + GATE_SGMII1(CLK_SGMII1_TX250M_EN, "sgmii1_tx250m_en", "top_xtal", 2), + GATE_SGMII1(CLK_SGMII1_RX250M_EN, "sgmii1_rx250m_en", "top_xtal", 3), + GATE_SGMII1(CLK_SGMII1_CDR_REF, "sgmii1_cdr_ref", "top_xtal", 4), + GATE_SGMII1(CLK_SGMII1_CDR_FB, "sgmii1_cdr_fb", "top_xtal", 5), +}; + +static const struct mtk_gate_regs eth_cg_regs = { + .set_ofs = 0x30, + .clr_ofs = 0x30, + .sta_ofs = 0x30, +}; + +#define GATE_ETH(_id, _name, _parent, _shift) \ + { \ + .id = _id, .name = _name, .parent_name = _parent, \ + .regs = ð_cg_regs, .shift = _shift, \ + .ops = &mtk_clk_gate_ops_no_setclr_inv, \ + } + +static const struct mtk_gate eth_clks[] __initconst = { + GATE_ETH(CLK_ETH_FE_EN, "eth_fe_en", "netsys_2x_sel", 6), + GATE_ETH(CLK_ETH_GP2_EN, "eth_gp2_en", "sgm_325m_sel", 7), + GATE_ETH(CLK_ETH_GP1_EN, "eth_gp1_en", "sgm_325m_sel", 8), + GATE_ETH(CLK_ETH_WOCPU1_EN, "eth_wocpu1_en", "netsys_mcu_sel", 14), + GATE_ETH(CLK_ETH_WOCPU0_EN, "eth_wocpu0_en", "netsys_mcu_sel", 15), +}; + +static void __init mtk_sgmiisys_0_init(struct device_node *node) +{ + struct clk_onecell_data *clk_data; + int r; + + clk_data = mtk_alloc_clk_data(ARRAY_SIZE(sgmii0_clks)); + + mtk_clk_register_gates(node, sgmii0_clks, ARRAY_SIZE(sgmii0_clks), + clk_data); + + r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data); + if (r) + pr_err("%s(): could not register clock provider: %d\n", + __func__, r); +} +CLK_OF_DECLARE(mtk_sgmiisys_0, "mediatek,mt7986-sgmiisys_0", + mtk_sgmiisys_0_init); + +static void __init mtk_sgmiisys_1_init(struct device_node *node) +{ + struct clk_onecell_data *clk_data; + int r; + + clk_data = mtk_alloc_clk_data(ARRAY_SIZE(sgmii1_clks)); + + mtk_clk_register_gates(node, sgmii1_clks, ARRAY_SIZE(sgmii1_clks), + clk_data); + + r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data); + + if (r) + pr_err("%s(): could not register clock provider: %d\n", + __func__, r); +} +CLK_OF_DECLARE(mtk_sgmiisys_1, "mediatek,mt7986-sgmiisys_1", + mtk_sgmiisys_1_init); + +static void __init mtk_ethsys_init(struct device_node *node) +{ + struct clk_onecell_data *clk_data; + int r; + + clk_data = mtk_alloc_clk_data(ARRAY_SIZE(eth_clks)); + + mtk_clk_register_gates(node, eth_clks, ARRAY_SIZE(eth_clks), clk_data); + + r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data); + + if (r) + pr_err("%s(): could not register clock provider: %d\n", + __func__, r); +} +CLK_OF_DECLARE(mtk_ethsys, "mediatek,mt7986-ethsys_ck", mtk_ethsys_init); diff --git a/drivers/clk/mediatek/clk-mt7986-infracfg.c b/drivers/clk/mediatek/clk-mt7986-infracfg.c new file mode 100644 index 000000000000..3be168c34fc0 --- /dev/null +++ b/drivers/clk/mediatek/clk-mt7986-infracfg.c @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: GPL-1.0 +/* + * Copyright (c) 2021 MediaTek Inc. + * Author: Sam Shih + * Author: Wenzhen Yu + */ + +#include +#include +#include +#include +#include +#include "clk-mtk.h" +#include "clk-gate.h" +#include "clk-mux.h" + +#include +#include + +static DEFINE_SPINLOCK(mt7986_clk_lock); + +static const struct mtk_fixed_factor infra_divs[] = { + FACTOR(CLK_INFRA_SYSAXI_D2, "infra_sysaxi_d2", "sysaxi_sel", 1, 2), +}; + +static const char *const infra_uart_parent[] __initconst = { "csw_f26m_sel", + "uart_sel" }; + +static const char *const infra_spi_parents[] __initconst = { "i2c_sel", + "spi_sel" }; + +static const char *const infra_pwm_bsel_parents[] __initconst = { + "top_rtc_32p7k", "csw_f26m_sel", "infra_sysaxi_d2", "pwm_sel" +}; + +static const char *const infra_pcie_parents[] __initconst = { + "top_rtc_32p7k", "csw_f26m_sel", "top_xtal", "pextp_tl_ck_sel" +}; + +static const struct mtk_mux infra_muxes[] = { + /* MODULE_CLK_SEL_0 */ + MUX_GATE_CLR_SET_UPD(CLK_INFRA_UART0_SEL, "infra_uart0_sel", + infra_uart_parent, 0x0018, 0x0010, 0x0014, 0, 1, + -1, -1, -1), + MUX_GATE_CLR_SET_UPD(CLK_INFRA_UART1_SEL, "infra_uart1_sel", + infra_uart_parent, 0x0018, 0x0010, 0x0014, 1, 1, + -1, -1, -1), + MUX_GATE_CLR_SET_UPD(CLK_INFRA_UART2_SEL, "infra_uart2_sel", + infra_uart_parent, 0x0018, 0x0010, 0x0014, 2, 1, + -1, -1, -1), + MUX_GATE_CLR_SET_UPD(CLK_INFRA_SPI0_SEL, "infra_spi0_sel", + infra_spi_parents, 0x0018, 0x0010, 0x0014, 4, 1, + -1, -1, -1), + MUX_GATE_CLR_SET_UPD(CLK_INFRA_SPI1_SEL, "infra_spi1_sel", + infra_spi_parents, 0x0018, 0x0010, 0x0014, 5, 1, + -1, -1, -1), + MUX_GATE_CLR_SET_UPD(CLK_INFRA_PWM1_SEL, "infra_pwm1_sel", + infra_pwm_bsel_parents, 0x0018, 0x0010, 0x0014, 9, + 2, -1, -1, -1), + MUX_GATE_CLR_SET_UPD(CLK_INFRA_PWM2_SEL, "infra_pwm2_sel", + infra_pwm_bsel_parents, 0x0018, 0x0010, 0x0014, 11, + 2, -1, -1, -1), + MUX_GATE_CLR_SET_UPD(CLK_INFRA_PWM_BSEL, "infra_pwm_bsel", + infra_pwm_bsel_parents, 0x0018, 0x0010, 0x0014, 13, + 2, -1, -1, -1), + /* MODULE_CLK_SEL_1 */ + MUX_GATE_CLR_SET_UPD(CLK_INFRA_PCIE_SEL, "infra_pcie_sel", + infra_pcie_parents, 0x0028, 0x0020, 0x0024, 0, 2, + -1, -1, -1), +}; + +static const struct mtk_gate_regs infra0_cg_regs = { + .set_ofs = 0x40, + .clr_ofs = 0x44, + .sta_ofs = 0x48, +}; + +static const struct mtk_gate_regs infra1_cg_regs = { + .set_ofs = 0x50, + .clr_ofs = 0x54, + .sta_ofs = 0x58, +}; + +static const struct mtk_gate_regs infra2_cg_regs = { + .set_ofs = 0x60, + .clr_ofs = 0x64, + .sta_ofs = 0x68, +}; + +#define GATE_INFRA0(_id, _name, _parent, _shift) \ + { \ + .id = _id, .name = _name, .parent_name = _parent, \ + .regs = &infra0_cg_regs, .shift = _shift, \ + .ops = &mtk_clk_gate_ops_setclr, \ + } + +#define GATE_INFRA1(_id, _name, _parent, _shift) \ + { \ + .id = _id, .name = _name, .parent_name = _parent, \ + .regs = &infra1_cg_regs, .shift = _shift, \ + .ops = &mtk_clk_gate_ops_setclr, \ + } + +#define GATE_INFRA2(_id, _name, _parent, _shift) \ + { \ + .id = _id, .name = _name, .parent_name = _parent, \ + .regs = &infra2_cg_regs, .shift = _shift, \ + .ops = &mtk_clk_gate_ops_setclr, \ + } + +static const struct mtk_gate infra_clks[] = { + /* INFRA0 */ + GATE_INFRA0(CLK_INFRA_GPT_STA, "infra_gpt_sta", "infra_sysaxi_d2", 0), + GATE_INFRA0(CLK_INFRA_PWM_HCK, "infra_pwm_hck", "infra_sysaxi_d2", 1), + GATE_INFRA0(CLK_INFRA_PWM_STA, "infra_pwm_sta", "infra_pwm_bsel", 2), + GATE_INFRA0(CLK_INFRA_PWM1_CK, "infra_pwm1", "infra_pwm1_sel", 3), + GATE_INFRA0(CLK_INFRA_PWM2_CK, "infra_pwm2", "infra_pwm2_sel", 4), + GATE_INFRA0(CLK_INFRA_CQ_DMA_CK, "infra_cq_dma", "sysaxi_sel", 6), + GATE_INFRA0(CLK_INFRA_EIP97_CK, "infra_eip97", "eip_b_sel", 7), + GATE_INFRA0(CLK_INFRA_AUD_BUS_CK, "infra_aud_bus", "sysaxi_sel", 8), + GATE_INFRA0(CLK_INFRA_AUD_26M_CK, "infra_aud_26m", "csw_f26m_sel", 9), + GATE_INFRA0(CLK_INFRA_AUD_L_CK, "infra_aud_l", "aud_l_sel", 10), + GATE_INFRA0(CLK_INFRA_AUD_AUD_CK, "infra_aud_aud", "a1sys_sel", 11), + GATE_INFRA0(CLK_INFRA_AUD_EG2_CK, "infra_aud_eg2", "a_tuner_sel", 13), + GATE_INFRA0(CLK_INFRA_DRAMC_26M_CK, "infra_dramc_26m", "csw_f26m_sel", + 14), + GATE_INFRA0(CLK_INFRA_DBG_CK, "infra_dbg", "infra_sysaxi_d2", 15), + GATE_INFRA0(CLK_INFRA_AP_DMA_CK, "infra_ap_dma", "infra_sysaxi_d2", 16), + GATE_INFRA0(CLK_INFRA_SEJ_CK, "infra_sej", "infra_sysaxi_d2", 24), + GATE_INFRA0(CLK_INFRA_SEJ_13M_CK, "infra_sej_13m", "csw_f26m_sel", 25), + GATE_INFRA0(CLK_INFRA_TRNG_CK, "infra_trng", "sysaxi_sel", 26), + /* INFRA1 */ + GATE_INFRA1(CLK_INFRA_THERM_CK, "infra_therm", "csw_f26m_sel", 0), + GATE_INFRA1(CLK_INFRA_I2C0_CK, "infra_i2c0", "i2c_sel", 1), + GATE_INFRA1(CLK_INFRA_UART0_CK, "infra_uart0", "infra_uart0_sel", 2), + GATE_INFRA1(CLK_INFRA_UART1_CK, "infra_uart1", "infra_uart1_sel", 3), + GATE_INFRA1(CLK_INFRA_UART2_CK, "infra_uart2", "infra_uart2_sel", 4), + GATE_INFRA1(CLK_INFRA_NFI1_CK, "infra_nfi1", "nfi1x_sel", 8), + GATE_INFRA1(CLK_INFRA_SPINFI1_CK, "infra_spinfi1", "spinfi_sel", 9), + GATE_INFRA1(CLK_INFRA_NFI_HCK_CK, "infra_nfi_hck", "infra_sysaxi_d2", + 10), + GATE_INFRA1(CLK_INFRA_SPI0_CK, "infra_spi0", "infra_spi0_sel", 11), + GATE_INFRA1(CLK_INFRA_SPI1_CK, "infra_spi1", "infra_spi1_sel", 12), + GATE_INFRA1(CLK_INFRA_SPI0_HCK_CK, "infra_spi0_hck", "infra_sysaxi_d2", + 13), + GATE_INFRA1(CLK_INFRA_SPI1_HCK_CK, "infra_spi1_hck", "infra_sysaxi_d2", + 14), + GATE_INFRA1(CLK_INFRA_FRTC_CK, "infra_frtc", "top_rtc_32k", 15), + GATE_INFRA1(CLK_INFRA_MSDC_CK, "infra_msdc", "emmc_416m_sel", 16), + GATE_INFRA1(CLK_INFRA_MSDC_HCK_CK, "infra_msdc_hck", "emmc_250m_sel", + 17), + GATE_INFRA1(CLK_INFRA_MSDC_133M_CK, "infra_msdc_133m", "sysaxi_sel", + 18), + GATE_INFRA1(CLK_INFRA_MSDC_66M_CK, "infra_msdc_66m", "infra_sysaxi_d2", + 19), + GATE_INFRA1(CLK_INFRA_ADC_26M_CK, "infra_adc_26m", "csw_f26m_sel", 20), + GATE_INFRA1(CLK_INFRA_ADC_FRC_CK, "infra_adc_frc", "csw_f26m_sel", 21), + GATE_INFRA1(CLK_INFRA_FBIST2FPC_CK, "infra_fbist2fpc", "nfi1x_sel", 23), + /* INFRA2 */ + GATE_INFRA2(CLK_INFRA_IUSB_133_CK, "infra_iusb_133", "sysaxi_sel", 0), + GATE_INFRA2(CLK_INFRA_IUSB_66M_CK, "infra_iusb_66m", "infra_sysaxi_d2", + 1), + GATE_INFRA2(CLK_INFRA_IUSB_SYS_CK, "infra_iusb_sys", "u2u3_sys_sel", 2), + GATE_INFRA2(CLK_INFRA_IUSB_CK, "infra_iusb", "u2u3_sel", 3), + GATE_INFRA2(CLK_INFRA_IPCIE_CK, "infra_ipcie", "pextp_tl_ck_sel", 12), + GATE_INFRA2(CLK_INFRA_IPCIE_PIPE_CK, "infra_ipcie_pipe", "top_xtal", + 13), + GATE_INFRA2(CLK_INFRA_IPCIER_CK, "infra_ipcier", "csw_f26m_sel", 14), + GATE_INFRA2(CLK_INFRA_IPCIEB_CK, "infra_ipcieb", "sysaxi_sel", 15), +}; + +static int clk_mt7986_infracfg_probe(struct platform_device *pdev) +{ + struct clk_onecell_data *clk_data; + struct device_node *node = pdev->dev.of_node; + int r; + void __iomem *base; + int nr = ARRAY_SIZE(infra_divs) + ARRAY_SIZE(infra_muxes) + + ARRAY_SIZE(infra_clks); + + base = of_iomap(node, 0); + if (!base) { + pr_err("%s(): ioremap failed\n", __func__); + return -ENOMEM; + } + + clk_data = mtk_alloc_clk_data(nr); + + if (!clk_data) + return -ENOMEM; + + mtk_clk_register_factors(infra_divs, ARRAY_SIZE(infra_divs), clk_data); + mtk_clk_register_muxes(infra_muxes, ARRAY_SIZE(infra_muxes), node, + &mt7986_clk_lock, clk_data); + mtk_clk_register_gates(node, infra_clks, ARRAY_SIZE(infra_clks), + clk_data); + + r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data); + if (r) { + pr_err("%s(): could not register clock provider: %d\n", + __func__, r); + goto free_infracfg_data; + } + return r; + +free_infracfg_data: + mtk_free_clk_data(clk_data); + return r; + +} + +static const struct of_device_id of_match_clk_mt7986_infracfg[] = { + { .compatible = "mediatek,mt7986-infracfg", }, + {} +}; + +static struct platform_driver clk_mt7986_infracfg_drv = { + .probe = clk_mt7986_infracfg_probe, + .driver = { + .name = "clk-mt7986-infracfg", + .of_match_table = of_match_clk_mt7986_infracfg, + }, +}; +builtin_platform_driver(clk_mt7986_infracfg_drv); diff --git a/drivers/clk/mediatek/clk-mt7986-topckgen.c b/drivers/clk/mediatek/clk-mt7986-topckgen.c new file mode 100644 index 000000000000..8550e2be7773 --- /dev/null +++ b/drivers/clk/mediatek/clk-mt7986-topckgen.c @@ -0,0 +1,342 @@ +// SPDX-License-Identifier: GPL-1.0 +/* + * Copyright (c) 2021 MediaTek Inc. + * Author: Sam Shih + * Author: Wenzhen Yu + */ + +#include +#include +#include +#include +#include +#include "clk-mtk.h" +#include "clk-gate.h" +#include "clk-mux.h" + +#include +#include + +static DEFINE_SPINLOCK(mt7986_clk_lock); + +static const struct mtk_fixed_clk top_fixed_clks[] = { + FIXED_CLK(CLK_TOP_XTAL, "top_xtal", "clkxtal", 40000000), + FIXED_CLK(CLK_TOP_JTAG, "top_jtag", "clkxtal", 50000000), +}; + +static const struct mtk_fixed_factor top_divs[] = { + /* XTAL */ + FACTOR(CLK_TOP_XTAL_D2, "top_xtal_d2", "top_xtal", 1, 2), + FACTOR(CLK_TOP_RTC_32K, "top_rtc_32k", "top_xtal", 1, 1250), + FACTOR(CLK_TOP_RTC_32P7K, "top_rtc_32p7k", "top_xtal", 1, 1220), + /* MPLL */ + FACTOR(CLK_TOP_MPLL_D2, "top_mpll_d2", "mpll", 1, 2), + FACTOR(CLK_TOP_MPLL_D4, "top_mpll_d4", "mpll", 1, 4), + FACTOR(CLK_TOP_MPLL_D8, "top_mpll_d8", "mpll", 1, 8), + FACTOR(CLK_TOP_MPLL_D8_D2, "top_mpll_d8_d2", "mpll", 1, 16), + FACTOR(CLK_TOP_MPLL_D3_D2, "top_mpll_d3_d2", "mpll", 1, 6), + /* MMPLL */ + FACTOR(CLK_TOP_MMPLL_D2, "top_mmpll_d2", "mmpll", 1, 2), + FACTOR(CLK_TOP_MMPLL_D4, "top_mmpll_d4", "mmpll", 1, 4), + FACTOR(CLK_TOP_MMPLL_D8, "top_mmpll_d8", "mmpll", 1, 8), + FACTOR(CLK_TOP_MMPLL_D8_D2, "top_mmpll_d8_d2", "mmpll", 1, 16), + FACTOR(CLK_TOP_MMPLL_D3_D8, "top_mmpll_d3_d8", "mmpll", 1, 24), + FACTOR(CLK_TOP_MMPLL_U2PHY, "top_mmpll_u2phy", "mmpll", 1, 30), + /* APLL2 */ + FACTOR(CLK_TOP_APLL2_D4, "top_apll2_d4", "apll2", 1, 4), + /* NET1PLL */ + FACTOR(CLK_TOP_NET1PLL_D4, "top_net1pll_d4", "net1pll", 1, 4), + FACTOR(CLK_TOP_NET1PLL_D5, "top_net1pll_d5", "net1pll", 1, 5), + FACTOR(CLK_TOP_NET1PLL_D5_D2, "top_net1pll_d5_d2", "net1pll", 1, 10), + FACTOR(CLK_TOP_NET1PLL_D5_D4, "top_net1pll_d5_d4", "net1pll", 1, 20), + FACTOR(CLK_TOP_NET1PLL_D8_D2, "top_net1pll_d8_d2", "net1pll", 1, 16), + FACTOR(CLK_TOP_NET1PLL_D8_D4, "top_net1pll_d8_d4", "net1pll", 1, 32), + /* NET2PLL */ + FACTOR(CLK_TOP_NET2PLL_D4, "top_net2pll_d4", "net2pll", 1, 4), + FACTOR(CLK_TOP_NET2PLL_D4_D2, "top_net2pll_d4_d2", "net2pll", 1, 8), + FACTOR(CLK_TOP_NET2PLL_D3_D2, "top_net2pll_d3_d2", "net2pll", 1, 2), + /* WEDMCUPLL */ + FACTOR(CLK_TOP_WEDMCUPLL_D5_D2, "top_wedmcupll_d5_d2", "wedmcupll", 1, + 10), +}; + +static const char *const nfi1x_parents[] __initconst = { "top_xtal", + "top_mmpll_d8", + "top_net1pll_d8_d2", + "top_net2pll_d3_d2", + "top_mpll_d4", + "top_mmpll_d8_d2", + "top_wedmcupll_d5_d2", + "top_mpll_d8" }; + +static const char *const spinfi_parents[] __initconst = { + "top_xtal_d2", "top_xtal", "top_net1pll_d5_d4", + "top_mpll_d4", "top_mmpll_d8_d2", "top_wedmcupll_d5_d2", + "top_mmpll_d3_d8", "top_mpll_d8" +}; + +static const char *const spi_parents[] __initconst = { + "top_xtal", "top_mpll_d2", "top_mmpll_d8", + "top_net1pll_d8_d2", "top_net2pll_d3_d2", "top_net1pll_d5_d4", + "top_mpll_d4", "top_wedmcupll_d5_d2" +}; + +static const char *const uart_parents[] __initconst = { "top_xtal", + "top_mpll_d8", + "top_mpll_d8_d2" }; + +static const char *const pwm_parents[] __initconst = { + "top_xtal", "top_net1pll_d8_d2", "top_net1pll_d5_d4", "top_mpll_d4" +}; + +static const char *const i2c_parents[] __initconst = { + "top_xtal", "top_net1pll_d5_d4", "top_mpll_d4", "top_net1pll_d8_d4" +}; + +static const char *const pextp_tl_ck_parents[] __initconst = { + "top_xtal", "top_net1pll_d5_d4", "top_net2pll_d4_d2", "top_rtc_32k" +}; + +static const char *const emmc_250m_parents[] __initconst = { + "top_xtal", "top_net1pll_d5_d2" +}; + +static const char *const emmc_416m_parents[] __initconst = { "top_xtal", + "mpll" }; + +static const char *const f_26m_adc_parents[] __initconst = { "top_xtal", + "top_mpll_d8_d2" }; + +static const char *const dramc_md32_parents[] __initconst = { "top_xtal", + "top_mpll_d2" }; + +static const char *const sysaxi_parents[] __initconst = { "top_xtal", + "top_net1pll_d8_d2", + "top_net2pll_d4" }; + +static const char *const sysapb_parents[] __initconst = { "top_xtal", + "top_mpll_d3_d2", + "top_net2pll_d4_d2" }; + +static const char *const arm_db_main_parents[] __initconst = { + "top_xtal", "top_net2pll_d3_d2" +}; + +static const char *const arm_db_jtsel_parents[] __initconst = { "top_jtag", + "top_xtal" }; + +static const char *const netsys_parents[] __initconst = { "top_xtal", + "top_mmpll_d4" }; + +static const char *const netsys_500m_parents[] __initconst = { + "top_xtal", "top_net1pll_d5" +}; + +static const char *const netsys_mcu_parents[] __initconst = { + "top_xtal", "wedmcupll", "top_mmpll_d2", "top_net1pll_d4", + "top_net1pll_d5" +}; + +static const char *const netsys_2x_parents[] __initconst = { + "top_xtal", "net2pll", "wedmcupll", "top_mmpll_d2" +}; + +static const char *const sgm_325m_parents[] __initconst = { "top_xtal", + "sgmpll" }; + +static const char *const sgm_reg_parents[] __initconst = { + "top_xtal", "top_net1pll_d8_d4" +}; + +static const char *const a1sys_parents[] __initconst = { "top_xtal", + "top_apll2_d4" }; + +static const char *const conn_mcusys_parents[] __initconst = { "top_xtal", + "top_mmpll_d2" }; + +static const char *const eip_b_parents[] __initconst = { "top_xtal", + "net2pll" }; + +static const char *const aud_l_parents[] __initconst = { "top_xtal", "apll2", + "top_mpll_d8_d2" }; + +static const char *const a_tuner_parents[] __initconst = { "top_xtal", + "top_apll2_d4", + "top_mpll_d8_d2" }; + +static const char *const u2u3_sys_parents[] __initconst = { + "top_xtal", "top_net1pll_d5_d4" +}; + +static const char *const da_u2_refsel_parents[] __initconst = { + "top_xtal", "top_mmpll_u2phy" +}; + +static const struct mtk_mux top_muxes[] = { + /* CLK_CFG_0 */ + MUX_GATE_CLR_SET_UPD(CLK_TOP_NFI1X_SEL, "nfi1x_sel", nfi1x_parents, + 0x000, 0x004, 0x008, 0, 3, 7, 0x1C0, 0), + MUX_GATE_CLR_SET_UPD(CLK_TOP_SPINFI_SEL, "spinfi_sel", spinfi_parents, + 0x000, 0x004, 0x008, 8, 3, 15, 0x1C0, 1), + MUX_GATE_CLR_SET_UPD(CLK_TOP_SPI_SEL, "spi_sel", spi_parents, 0x000, + 0x004, 0x008, 16, 3, 23, 0x1C0, 2), + MUX_GATE_CLR_SET_UPD(CLK_TOP_SPIM_MST_SEL, "spim_mst_sel", spi_parents, + 0x000, 0x004, 0x008, 24, 3, 31, 0x1C0, 3), + /* CLK_CFG_1 */ + MUX_GATE_CLR_SET_UPD(CLK_TOP_UART_SEL, "uart_sel", uart_parents, 0x010, + 0x014, 0x018, 0, 2, 7, 0x1C0, 4), + MUX_GATE_CLR_SET_UPD(CLK_TOP_PWM_SEL, "pwm_sel", pwm_parents, 0x010, + 0x014, 0x018, 8, 2, 15, 0x1C0, 5), + MUX_GATE_CLR_SET_UPD(CLK_TOP_I2C_SEL, "i2c_sel", i2c_parents, 0x010, + 0x014, 0x018, 16, 2, 23, 0x1C0, 6), + MUX_GATE_CLR_SET_UPD(CLK_TOP_PEXTP_TL_SEL, "pextp_tl_ck_sel", + pextp_tl_ck_parents, 0x010, 0x014, 0x018, 24, 2, + 31, 0x1C0, 7), + /* CLK_CFG_2 */ + MUX_GATE_CLR_SET_UPD(CLK_TOP_EMMC_250M_SEL, "emmc_250m_sel", + emmc_250m_parents, 0x020, 0x024, 0x028, 0, 1, 7, + 0x1C0, 8), + MUX_GATE_CLR_SET_UPD(CLK_TOP_EMMC_416M_SEL, "emmc_416m_sel", + emmc_416m_parents, 0x020, 0x024, 0x028, 8, 1, 15, + 0x1C0, 9), + MUX_GATE_CLR_SET_UPD(CLK_TOP_F_26M_ADC_SEL, "f_26m_adc_sel", + f_26m_adc_parents, 0x020, 0x024, 0x028, 16, 1, 23, + 0x1C0, 10), + MUX_GATE_CLR_SET_UPD(CLK_TOP_DRAMC_SEL, "dramc_sel", f_26m_adc_parents, + 0x020, 0x024, 0x028, 24, 1, 31, 0x1C0, 11), + /* CLK_CFG_3 */ + MUX_GATE_CLR_SET_UPD(CLK_TOP_DRAMC_MD32_SEL, "dramc_md32_sel", + dramc_md32_parents, 0x030, 0x034, 0x038, 0, 1, 7, + 0x1C0, 12), + MUX_GATE_CLR_SET_UPD(CLK_TOP_SYSAXI_SEL, "sysaxi_sel", sysaxi_parents, + 0x030, 0x034, 0x038, 8, 2, 15, 0x1C0, 13), + MUX_GATE_CLR_SET_UPD(CLK_TOP_SYSAPB_SEL, "sysapb_sel", sysapb_parents, + 0x030, 0x034, 0x038, 16, 2, 23, 0x1C0, 14), + MUX_GATE_CLR_SET_UPD(CLK_TOP_ARM_DB_MAIN_SEL, "arm_db_main_sel", + arm_db_main_parents, 0x030, 0x034, 0x038, 24, 1, + 31, 0x1C0, 15), + /* CLK_CFG_4 */ + MUX_GATE_CLR_SET_UPD(CLK_TOP_ARM_DB_JTSEL, "arm_db_jtsel", + arm_db_jtsel_parents, 0x040, 0x044, 0x048, 0, 1, 7, + 0x1C0, 16), + MUX_GATE_CLR_SET_UPD(CLK_TOP_NETSYS_SEL, "netsys_sel", netsys_parents, + 0x040, 0x044, 0x048, 8, 1, 15, 0x1C0, 17), + MUX_GATE_CLR_SET_UPD(CLK_TOP_NETSYS_500M_SEL, "netsys_500m_sel", + netsys_500m_parents, 0x040, 0x044, 0x048, 16, 1, + 23, 0x1C0, 18), + MUX_GATE_CLR_SET_UPD(CLK_TOP_NETSYS_MCU_SEL, "netsys_mcu_sel", + netsys_mcu_parents, 0x040, 0x044, 0x048, 24, 3, 31, + 0x1C0, 19), + /* CLK_CFG_5 */ + MUX_GATE_CLR_SET_UPD(CLK_TOP_NETSYS_2X_SEL, "netsys_2x_sel", + netsys_2x_parents, 0x050, 0x054, 0x058, 0, 2, 7, + 0x1C0, 20), + MUX_GATE_CLR_SET_UPD(CLK_TOP_SGM_325M_SEL, "sgm_325m_sel", + sgm_325m_parents, 0x050, 0x054, 0x058, 8, 1, 15, + 0x1C0, 21), + MUX_GATE_CLR_SET_UPD(CLK_TOP_SGM_REG_SEL, "sgm_reg_sel", + sgm_reg_parents, 0x050, 0x054, 0x058, 16, 1, 23, + 0x1C0, 22), + MUX_GATE_CLR_SET_UPD(CLK_TOP_A1SYS_SEL, "a1sys_sel", a1sys_parents, + 0x050, 0x054, 0x058, 24, 1, 31, 0x1C0, 23), + /* CLK_CFG_6 */ + MUX_GATE_CLR_SET_UPD(CLK_TOP_CONN_MCUSYS_SEL, "conn_mcusys_sel", + conn_mcusys_parents, 0x060, 0x064, 0x068, 0, 1, 7, + 0x1C0, 24), + MUX_GATE_CLR_SET_UPD(CLK_TOP_EIP_B_SEL, "eip_b_sel", eip_b_parents, + 0x060, 0x064, 0x068, 8, 1, 15, 0x1C0, 25), + MUX_GATE_CLR_SET_UPD(CLK_TOP_PCIE_PHY_SEL, "pcie_phy_sel", + f_26m_adc_parents, 0x060, 0x064, 0x068, 16, 1, 23, + 0x1C0, 26), + MUX_GATE_CLR_SET_UPD(CLK_TOP_USB3_PHY_SEL, "usb3_phy_sel", + f_26m_adc_parents, 0x060, 0x064, 0x068, 24, 1, 31, + 0x1C0, 27), + /* CLK_CFG_7 */ + MUX_GATE_CLR_SET_UPD(CLK_TOP_F26M_SEL, "csw_f26m_sel", + f_26m_adc_parents, 0x070, 0x074, 0x078, 0, 1, 7, + 0x1C0, 28), + MUX_GATE_CLR_SET_UPD(CLK_TOP_AUD_L_SEL, "aud_l_sel", aud_l_parents, + 0x070, 0x074, 0x078, 8, 2, 15, 0x1C0, 29), + MUX_GATE_CLR_SET_UPD(CLK_TOP_A_TUNER_SEL, "a_tuner_sel", + a_tuner_parents, 0x070, 0x074, 0x078, 16, 2, 23, + 0x1C0, 30), + MUX_GATE_CLR_SET_UPD(CLK_TOP_U2U3_SEL, "u2u3_sel", f_26m_adc_parents, + 0x070, 0x074, 0x078, 24, 1, 31, 0x1C4, 0), + /* CLK_CFG_8 */ + MUX_GATE_CLR_SET_UPD(CLK_TOP_U2U3_SYS_SEL, "u2u3_sys_sel", + u2u3_sys_parents, 0x080, 0x084, 0x088, 0, 1, 7, + 0x1C4, 1), + MUX_GATE_CLR_SET_UPD(CLK_TOP_U2U3_XHCI_SEL, "u2u3_xhci_sel", + u2u3_sys_parents, 0x080, 0x084, 0x088, 8, 1, 15, + 0x1C4, 2), + MUX_GATE_CLR_SET_UPD(CLK_TOP_DA_U2_REFSEL, "da_u2_refsel", + da_u2_refsel_parents, 0x080, 0x084, 0x088, 16, 1, + 23, 0x1C4, 3), + MUX_GATE_CLR_SET_UPD(CLK_TOP_DA_U2_CK_1P_SEL, "da_u2_ck_1p_sel", + da_u2_refsel_parents, 0x080, 0x084, 0x088, 24, 1, + 31, 0x1C4, 4), + /* CLK_CFG_9 */ + MUX_GATE_CLR_SET_UPD(CLK_TOP_AP2CNN_HOST_SEL, "ap2cnn_host_sel", + sgm_reg_parents, 0x090, 0x094, 0x098, 0, 1, 7, + 0x1C4, 5), +}; + +static int clk_mt7986_topckgen_probe(struct platform_device *pdev) +{ + struct clk_onecell_data *clk_data; + struct device_node *node = pdev->dev.of_node; + int r; + void __iomem *base; + int nr = ARRAY_SIZE(top_fixed_clks) + ARRAY_SIZE(top_divs) + + ARRAY_SIZE(top_muxes); + + base = of_iomap(node, 0); + if (!base) { + pr_err("%s(): ioremap failed\n", __func__); + return -ENOMEM; + } + + clk_data = mtk_alloc_clk_data(nr); + if (!clk_data) + return -ENOMEM; + + mtk_clk_register_fixed_clks(top_fixed_clks, ARRAY_SIZE(top_fixed_clks), + clk_data); + mtk_clk_register_factors(top_divs, ARRAY_SIZE(top_divs), clk_data); + mtk_clk_register_muxes(top_muxes, ARRAY_SIZE(top_muxes), node, + &mt7986_clk_lock, clk_data); + + clk_prepare_enable(clk_data->clks[CLK_TOP_SYSAXI_SEL]); + clk_prepare_enable(clk_data->clks[CLK_TOP_SYSAPB_SEL]); + clk_prepare_enable(clk_data->clks[CLK_TOP_DRAMC_SEL]); + clk_prepare_enable(clk_data->clks[CLK_TOP_DRAMC_MD32_SEL]); + clk_prepare_enable(clk_data->clks[CLK_TOP_F26M_SEL]); + clk_prepare_enable(clk_data->clks[CLK_TOP_SGM_REG_SEL]); + + r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data); + + if (r) { + pr_err("%s(): could not register clock provider: %d\n", + __func__, r); + goto free_topckgen_data; + } + return r; + +free_topckgen_data: + mtk_free_clk_data(clk_data); + return r; +} + +static const struct of_device_id of_match_clk_mt7986_topckgen[] = { + { .compatible = "mediatek,mt7986-topckgen", }, + {} +}; + +static struct platform_driver clk_mt7986_topckgen_drv = { + .probe = clk_mt7986_topckgen_probe, + .driver = { + .name = "clk-mt7986-topckgen", + .of_match_table = of_match_clk_mt7986_topckgen, + }, +}; +builtin_platform_driver(clk_mt7986_topckgen_drv); diff --git a/drivers/clk/meson/gxbb.c b/drivers/clk/meson/gxbb.c index d6eed760327d..608e0e8ca49a 100644 --- a/drivers/clk/meson/gxbb.c +++ b/drivers/clk/meson/gxbb.c @@ -713,6 +713,35 @@ static struct clk_regmap gxbb_mpll_prediv = { }; static struct clk_regmap gxbb_mpll0_div = { + .data = &(struct meson_clk_mpll_data){ + .sdm = { + .reg_off = HHI_MPLL_CNTL7, + .shift = 0, + .width = 14, + }, + .sdm_en = { + .reg_off = HHI_MPLL_CNTL, + .shift = 25, + .width = 1, + }, + .n2 = { + .reg_off = HHI_MPLL_CNTL7, + .shift = 16, + .width = 9, + }, + .lock = &meson_clk_lock, + }, + .hw.init = &(struct clk_init_data){ + .name = "mpll0_div", + .ops = &meson_clk_mpll_ops, + .parent_hws = (const struct clk_hw *[]) { + &gxbb_mpll_prediv.hw + }, + .num_parents = 1, + }, +}; + +static struct clk_regmap gxl_mpll0_div = { .data = &(struct meson_clk_mpll_data){ .sdm = { .reg_off = HHI_MPLL_CNTL7, @@ -749,7 +778,16 @@ static struct clk_regmap gxbb_mpll0 = { .hw.init = &(struct clk_init_data){ .name = "mpll0", .ops = &clk_regmap_gate_ops, - .parent_hws = (const struct clk_hw *[]) { &gxbb_mpll0_div.hw }, + .parent_data = &(const struct clk_parent_data) { + /* + * Note: + * GXL and GXBB have different SDM_EN registers. We + * fallback to the global naming string mechanism so + * mpll0_div picks up the appropriate one. + */ + .name = "mpll0_div", + .index = -1, + }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, }, @@ -3044,7 +3082,7 @@ static struct clk_hw_onecell_data gxl_hw_onecell_data = { [CLKID_VAPB_1] = &gxbb_vapb_1.hw, [CLKID_VAPB_SEL] = &gxbb_vapb_sel.hw, [CLKID_VAPB] = &gxbb_vapb.hw, - [CLKID_MPLL0_DIV] = &gxbb_mpll0_div.hw, + [CLKID_MPLL0_DIV] = &gxl_mpll0_div.hw, [CLKID_MPLL1_DIV] = &gxbb_mpll1_div.hw, [CLKID_MPLL2_DIV] = &gxbb_mpll2_div.hw, [CLKID_MPLL_PREDIV] = &gxbb_mpll_prediv.hw, @@ -3439,7 +3477,7 @@ static struct clk_regmap *const gxl_clk_regmaps[] = { &gxbb_mpll0, &gxbb_mpll1, &gxbb_mpll2, - &gxbb_mpll0_div, + &gxl_mpll0_div, &gxbb_mpll1_div, &gxbb_mpll2_div, &gxbb_cts_amclk_div, diff --git a/drivers/clk/qcom/Kconfig b/drivers/clk/qcom/Kconfig index 74efc82127e1..42c874194d1a 100644 --- a/drivers/clk/qcom/Kconfig +++ b/drivers/clk/qcom/Kconfig @@ -265,6 +265,14 @@ config MSM_MMCC_8974 Say Y if you want to support multimedia devices such as display, graphics, video encode/decode, camera, etc. +config MSM_GCC_8976 + tristate "MSM8956/76 Global Clock Controller" + select QCOM_GDSC + help + Support for the global clock controller on msm8956/76 devices. + Say Y if you want to use peripheral devices such as UART, SPI, + i2c, USB, SD/eMMC, SATA, PCIe, etc. + config MSM_MMCC_8994 tristate "MSM8994 Multimedia Clock Controller" select MSM_GCC_8994 @@ -564,6 +572,14 @@ config SM_CAMCC_8250 Support for the camera clock controller on SM8250 devices. Say Y if you want to support camera devices and camera functionality. +config SDX_GCC_65 + tristate "SDX65 Global Clock Controller" + select QCOM_GDSC + help + Support for the global clock controller on SDX65 devices. + Say Y if you want to use peripheral devices such as UART, + SPI, I2C, USB, SD/UFS, PCIe etc. + config SM_DISPCC_8250 tristate "SM8150 and SM8250 Display Clock Controller" depends on SM_GCC_8150 || SM_GCC_8250 @@ -618,6 +634,14 @@ config SM_GCC_8350 Say Y if you want to use peripheral devices such as UART, SPI, I2C, USB, SD/UFS, PCIe etc. +config SM_GCC_8450 + tristate "SM8450 Global Clock Controller" + select QCOM_GDSC + help + Support for the global clock controller on SM8450 devices. + Say Y if you want to use peripheral devices such as UART, + SPI, I2C, USB, SD/UFS, PCIe etc. + config SM_GPUCC_8150 tristate "SM8150 Graphics Clock Controller" select SM_GCC_8150 diff --git a/drivers/clk/qcom/Makefile b/drivers/clk/qcom/Makefile index 1718c34d3551..0d98ca9be67f 100644 --- a/drivers/clk/qcom/Makefile +++ b/drivers/clk/qcom/Makefile @@ -36,6 +36,7 @@ obj-$(CONFIG_MSM_GCC_8939) += gcc-msm8939.o obj-$(CONFIG_MSM_GCC_8953) += gcc-msm8953.o obj-$(CONFIG_MSM_GCC_8960) += gcc-msm8960.o obj-$(CONFIG_MSM_GCC_8974) += gcc-msm8974.o +obj-$(CONFIG_MSM_GCC_8976) += gcc-msm8976.o obj-$(CONFIG_MSM_GCC_8994) += gcc-msm8994.o obj-$(CONFIG_MSM_GCC_8996) += gcc-msm8996.o obj-$(CONFIG_MSM_LCC_8960) += lcc-msm8960.o @@ -83,6 +84,7 @@ obj-$(CONFIG_SDM_LPASSCC_845) += lpasscc-sdm845.o obj-$(CONFIG_SDM_VIDEOCC_845) += videocc-sdm845.o obj-$(CONFIG_SDX_GCC_55) += gcc-sdx55.o obj-$(CONFIG_SM_CAMCC_8250) += camcc-sm8250.o +obj-$(CONFIG_SDX_GCC_65) += gcc-sdx65.o obj-$(CONFIG_SM_DISPCC_8250) += dispcc-sm8250.o obj-$(CONFIG_SM_GCC_6115) += gcc-sm6115.o obj-$(CONFIG_SM_GCC_6125) += gcc-sm6125.o @@ -90,6 +92,7 @@ obj-$(CONFIG_SM_GCC_6350) += gcc-sm6350.o obj-$(CONFIG_SM_GCC_8150) += gcc-sm8150.o obj-$(CONFIG_SM_GCC_8250) += gcc-sm8250.o obj-$(CONFIG_SM_GCC_8350) += gcc-sm8350.o +obj-$(CONFIG_SM_GCC_8450) += gcc-sm8450.o obj-$(CONFIG_SM_GPUCC_8150) += gpucc-sm8150.o obj-$(CONFIG_SM_GPUCC_8250) += gpucc-sm8250.o obj-$(CONFIG_SM_VIDEOCC_8150) += videocc-sm8150.o diff --git a/drivers/clk/qcom/clk-alpha-pll.c b/drivers/clk/qcom/clk-alpha-pll.c index 8f65b9bdafce..4406cf609aae 100644 --- a/drivers/clk/qcom/clk-alpha-pll.c +++ b/drivers/clk/qcom/clk-alpha-pll.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* * Copyright (c) 2015, 2018, The Linux Foundation. All rights reserved. + * Copyright (c) 2021, Qualcomm Innovation Center, Inc. All rights reserved. */ #include @@ -139,6 +140,20 @@ const u8 clk_alpha_pll_regs[][PLL_OFF_MAX_REGS] = { [PLL_OFF_OPMODE] = 0x28, [PLL_OFF_STATUS] = 0x38, }, + [CLK_ALPHA_PLL_TYPE_LUCID_EVO] = { + [PLL_OFF_OPMODE] = 0x04, + [PLL_OFF_STATUS] = 0x0c, + [PLL_OFF_L_VAL] = 0x10, + [PLL_OFF_ALPHA_VAL] = 0x14, + [PLL_OFF_USER_CTL] = 0x18, + [PLL_OFF_USER_CTL_U] = 0x1c, + [PLL_OFF_CONFIG_CTL] = 0x20, + [PLL_OFF_CONFIG_CTL_U] = 0x24, + [PLL_OFF_CONFIG_CTL_U1] = 0x28, + [PLL_OFF_TEST_CTL] = 0x2c, + [PLL_OFF_TEST_CTL_U] = 0x30, + [PLL_OFF_TEST_CTL_U1] = 0x34, + }, }; EXPORT_SYMBOL_GPL(clk_alpha_pll_regs); @@ -175,6 +190,10 @@ EXPORT_SYMBOL_GPL(clk_alpha_pll_regs); #define LUCID_5LPE_PLL_LATCH_INPUT BIT(14) #define LUCID_5LPE_ENABLE_VOTE_RUN BIT(21) +/* LUCID EVO PLL specific settings and offsets */ +#define LUCID_EVO_ENABLE_VOTE_RUN BIT(25) +#define LUCID_EVO_PLL_L_VAL_MASK GENMASK(15, 0) + /* ZONDA PLL specific */ #define ZONDA_PLL_OUT_MASK 0xf #define ZONDA_STAY_IN_CFA BIT(16) @@ -204,7 +223,7 @@ static int wait_for_pll(struct clk_alpha_pll *pll, u32 mask, bool inverse, if (ret) return ret; - for (count = 100; count > 0; count--) { + for (count = 200; count > 0; count--) { ret = regmap_read(pll->clkr.regmap, PLL_MODE(pll), &val); if (ret) return ret; @@ -1750,24 +1769,32 @@ static int alpha_pll_lucid_5lpe_set_rate(struct clk_hw *hw, unsigned long rate, LUCID_5LPE_ALPHA_PLL_ACK_LATCH); } -static int clk_lucid_5lpe_pll_postdiv_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) +static int __clk_lucid_pll_postdiv_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate, + unsigned long enable_vote_run) { struct clk_alpha_pll_postdiv *pll = to_clk_alpha_pll_postdiv(hw); - int i, val = 0, div, ret; + struct regmap *regmap = pll->clkr.regmap; + int i, val, div, ret; u32 mask; /* * If the PLL is in FSM mode, then treat set_rate callback as a * no-operation. */ - ret = regmap_read(pll->clkr.regmap, PLL_USER_CTL(pll), &val); + ret = regmap_read(regmap, PLL_USER_CTL(pll), &val); if (ret) return ret; - if (val & LUCID_5LPE_ENABLE_VOTE_RUN) + if (val & enable_vote_run) return 0; + if (!pll->post_div_table) { + pr_err("Missing the post_div_table for the %s PLL\n", + clk_hw_get_name(&pll->clkr.hw)); + return -EINVAL; + } + div = DIV_ROUND_UP_ULL((u64)parent_rate, rate); for (i = 0; i < pll->num_post_div; i++) { if (pll->post_div_table[i].div == div) { @@ -1781,6 +1808,12 @@ static int clk_lucid_5lpe_pll_postdiv_set_rate(struct clk_hw *hw, unsigned long mask, val << pll->post_div_shift); } +static int clk_lucid_5lpe_pll_postdiv_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + return __clk_lucid_pll_postdiv_set_rate(hw, rate, parent_rate, LUCID_5LPE_ENABLE_VOTE_RUN); +} + const struct clk_ops clk_alpha_pll_lucid_5lpe_ops = { .prepare = alpha_pll_lucid_5lpe_prepare, .enable = alpha_pll_lucid_5lpe_enable, @@ -1960,3 +1993,124 @@ const struct clk_ops clk_alpha_pll_zonda_ops = { .set_rate = clk_zonda_pll_set_rate, }; EXPORT_SYMBOL(clk_alpha_pll_zonda_ops); + +static int alpha_pll_lucid_evo_enable(struct clk_hw *hw) +{ + struct clk_alpha_pll *pll = to_clk_alpha_pll(hw); + struct regmap *regmap = pll->clkr.regmap; + u32 val; + int ret; + + ret = regmap_read(regmap, PLL_USER_CTL(pll), &val); + if (ret) + return ret; + + /* If in FSM mode, just vote for it */ + if (val & LUCID_EVO_ENABLE_VOTE_RUN) { + ret = clk_enable_regmap(hw); + if (ret) + return ret; + return wait_for_pll_enable_lock(pll); + } + + /* Check if PLL is already enabled */ + ret = trion_pll_is_enabled(pll, regmap); + if (ret < 0) { + return ret; + } else if (ret) { + pr_warn("%s PLL is already enabled\n", clk_hw_get_name(&pll->clkr.hw)); + return 0; + } + + ret = regmap_update_bits(regmap, PLL_MODE(pll), PLL_RESET_N, PLL_RESET_N); + if (ret) + return ret; + + /* Set operation mode to RUN */ + regmap_write(regmap, PLL_OPMODE(pll), PLL_RUN); + + ret = wait_for_pll_enable_lock(pll); + if (ret) + return ret; + + /* Enable the PLL outputs */ + ret = regmap_update_bits(regmap, PLL_USER_CTL(pll), PLL_OUT_MASK, PLL_OUT_MASK); + if (ret) + return ret; + + /* Enable the global PLL outputs */ + ret = regmap_update_bits(regmap, PLL_MODE(pll), PLL_OUTCTRL, PLL_OUTCTRL); + if (ret) + return ret; + + /* Ensure that the write above goes through before returning. */ + mb(); + return ret; +} + +static void alpha_pll_lucid_evo_disable(struct clk_hw *hw) +{ + struct clk_alpha_pll *pll = to_clk_alpha_pll(hw); + struct regmap *regmap = pll->clkr.regmap; + u32 val; + int ret; + + ret = regmap_read(regmap, PLL_USER_CTL(pll), &val); + if (ret) + return; + + /* If in FSM mode, just unvote it */ + if (val & LUCID_EVO_ENABLE_VOTE_RUN) { + clk_disable_regmap(hw); + return; + } + + /* Disable the global PLL output */ + ret = regmap_update_bits(regmap, PLL_MODE(pll), PLL_OUTCTRL, 0); + if (ret) + return; + + /* Disable the PLL outputs */ + ret = regmap_update_bits(regmap, PLL_USER_CTL(pll), PLL_OUT_MASK, 0); + if (ret) + return; + + /* Place the PLL mode in STANDBY */ + regmap_write(regmap, PLL_OPMODE(pll), PLL_STANDBY); +} + +static unsigned long alpha_pll_lucid_evo_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct clk_alpha_pll *pll = to_clk_alpha_pll(hw); + struct regmap *regmap = pll->clkr.regmap; + u32 l, frac; + + regmap_read(regmap, PLL_L_VAL(pll), &l); + l &= LUCID_EVO_PLL_L_VAL_MASK; + regmap_read(regmap, PLL_ALPHA_VAL(pll), &frac); + + return alpha_pll_calc_rate(parent_rate, l, frac, pll_alpha_width(pll)); +} + +static int clk_lucid_evo_pll_postdiv_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + return __clk_lucid_pll_postdiv_set_rate(hw, rate, parent_rate, LUCID_EVO_ENABLE_VOTE_RUN); +} + +const struct clk_ops clk_alpha_pll_fixed_lucid_evo_ops = { + .enable = alpha_pll_lucid_evo_enable, + .disable = alpha_pll_lucid_evo_disable, + .is_enabled = clk_trion_pll_is_enabled, + .recalc_rate = alpha_pll_lucid_evo_recalc_rate, + .round_rate = clk_alpha_pll_round_rate, +}; +EXPORT_SYMBOL_GPL(clk_alpha_pll_fixed_lucid_evo_ops); + +const struct clk_ops clk_alpha_pll_postdiv_lucid_evo_ops = { + .recalc_rate = clk_alpha_pll_postdiv_fabia_recalc_rate, + .round_rate = clk_alpha_pll_postdiv_fabia_round_rate, + .set_rate = clk_lucid_evo_pll_postdiv_set_rate, +}; +EXPORT_SYMBOL_GPL(clk_alpha_pll_postdiv_lucid_evo_ops); diff --git a/drivers/clk/qcom/clk-alpha-pll.h b/drivers/clk/qcom/clk-alpha-pll.h index 55e4fa47912f..6e9907deaf30 100644 --- a/drivers/clk/qcom/clk-alpha-pll.h +++ b/drivers/clk/qcom/clk-alpha-pll.h @@ -17,6 +17,7 @@ enum { CLK_ALPHA_PLL_TYPE_LUCID = CLK_ALPHA_PLL_TYPE_TRION, CLK_ALPHA_PLL_TYPE_AGERA, CLK_ALPHA_PLL_TYPE_ZONDA, + CLK_ALPHA_PLL_TYPE_LUCID_EVO, CLK_ALPHA_PLL_TYPE_MAX, }; @@ -151,6 +152,8 @@ extern const struct clk_ops clk_alpha_pll_postdiv_lucid_5lpe_ops; extern const struct clk_ops clk_alpha_pll_zonda_ops; #define clk_alpha_pll_postdiv_zonda_ops clk_alpha_pll_postdiv_fabia_ops +extern const struct clk_ops clk_alpha_pll_fixed_lucid_evo_ops; +extern const struct clk_ops clk_alpha_pll_postdiv_lucid_evo_ops; void clk_alpha_pll_configure(struct clk_alpha_pll *pll, struct regmap *regmap, const struct alpha_pll_config *config); diff --git a/drivers/clk/qcom/clk-rpmh.c b/drivers/clk/qcom/clk-rpmh.c index 441d7a20e6f3..74e57c84f60a 100644 --- a/drivers/clk/qcom/clk-rpmh.c +++ b/drivers/clk/qcom/clk-rpmh.c @@ -515,6 +515,32 @@ static const struct clk_rpmh_desc clk_rpmh_sm8350 = { /* Resource name must match resource id present in cmd-db */ DEFINE_CLK_RPMH_ARC(sc7280, bi_tcxo, bi_tcxo_ao, "xo.lvl", 0x3, 4); +DEFINE_CLK_RPMH_VRM(sm8450, ln_bb_clk1, ln_bb_clk1_ao, "lnbclka1", 4); +DEFINE_CLK_RPMH_VRM(sm8450, ln_bb_clk2, ln_bb_clk2_ao, "lnbclka2", 4); + +static struct clk_hw *sm8450_rpmh_clocks[] = { + [RPMH_CXO_CLK] = &sc7280_bi_tcxo.hw, + [RPMH_CXO_CLK_A] = &sc7280_bi_tcxo_ao.hw, + [RPMH_LN_BB_CLK1] = &sm8450_ln_bb_clk1.hw, + [RPMH_LN_BB_CLK1_A] = &sm8450_ln_bb_clk1_ao.hw, + [RPMH_LN_BB_CLK2] = &sm8450_ln_bb_clk2.hw, + [RPMH_LN_BB_CLK2_A] = &sm8450_ln_bb_clk2_ao.hw, + [RPMH_RF_CLK1] = &sdm845_rf_clk1.hw, + [RPMH_RF_CLK1_A] = &sdm845_rf_clk1_ao.hw, + [RPMH_RF_CLK2] = &sdm845_rf_clk2.hw, + [RPMH_RF_CLK2_A] = &sdm845_rf_clk2_ao.hw, + [RPMH_RF_CLK3] = &sdm845_rf_clk3.hw, + [RPMH_RF_CLK3_A] = &sdm845_rf_clk3_ao.hw, + [RPMH_RF_CLK4] = &sm8350_rf_clk4.hw, + [RPMH_RF_CLK4_A] = &sm8350_rf_clk4_ao.hw, + [RPMH_IPA_CLK] = &sdm845_ipa.hw, +}; + +static const struct clk_rpmh_desc clk_rpmh_sm8450 = { + .clks = sm8450_rpmh_clocks, + .num_clks = ARRAY_SIZE(sm8450_rpmh_clocks), +}; + static struct clk_hw *sc7280_rpmh_clocks[] = { [RPMH_CXO_CLK] = &sc7280_bi_tcxo.hw, [RPMH_CXO_CLK_A] = &sc7280_bi_tcxo_ao.hw, @@ -556,6 +582,30 @@ static const struct clk_rpmh_desc clk_rpmh_sm6350 = { .num_clks = ARRAY_SIZE(sm6350_rpmh_clocks), }; +DEFINE_CLK_RPMH_VRM(sdx65, ln_bb_clk1, ln_bb_clk1_ao, "lnbclka1", 4); + +static struct clk_hw *sdx65_rpmh_clocks[] = { + [RPMH_CXO_CLK] = &sc7280_bi_tcxo.hw, + [RPMH_CXO_CLK_A] = &sc7280_bi_tcxo_ao.hw, + [RPMH_LN_BB_CLK1] = &sdx65_ln_bb_clk1.hw, + [RPMH_LN_BB_CLK1_A] = &sdx65_ln_bb_clk1_ao.hw, + [RPMH_RF_CLK1] = &sdm845_rf_clk1.hw, + [RPMH_RF_CLK1_A] = &sdm845_rf_clk1_ao.hw, + [RPMH_RF_CLK2] = &sdm845_rf_clk2.hw, + [RPMH_RF_CLK2_A] = &sdm845_rf_clk2_ao.hw, + [RPMH_RF_CLK3] = &sdm845_rf_clk3.hw, + [RPMH_RF_CLK3_A] = &sdm845_rf_clk3_ao.hw, + [RPMH_RF_CLK4] = &sm8350_rf_clk4.hw, + [RPMH_RF_CLK4_A] = &sm8350_rf_clk4_ao.hw, + [RPMH_IPA_CLK] = &sdm845_ipa.hw, + [RPMH_QPIC_CLK] = &sdx55_qpic_clk.hw, +}; + +static const struct clk_rpmh_desc clk_rpmh_sdx65 = { + .clks = sdx65_rpmh_clocks, + .num_clks = ARRAY_SIZE(sdx65_rpmh_clocks), +}; + static struct clk_hw *of_clk_rpmh_hw_get(struct of_phandle_args *clkspec, void *data) { @@ -643,10 +693,12 @@ static const struct of_device_id clk_rpmh_match_table[] = { { .compatible = "qcom,sc8180x-rpmh-clk", .data = &clk_rpmh_sc8180x}, { .compatible = "qcom,sdm845-rpmh-clk", .data = &clk_rpmh_sdm845}, { .compatible = "qcom,sdx55-rpmh-clk", .data = &clk_rpmh_sdx55}, + { .compatible = "qcom,sdx65-rpmh-clk", .data = &clk_rpmh_sdx65}, { .compatible = "qcom,sm6350-rpmh-clk", .data = &clk_rpmh_sm6350}, { .compatible = "qcom,sm8150-rpmh-clk", .data = &clk_rpmh_sm8150}, { .compatible = "qcom,sm8250-rpmh-clk", .data = &clk_rpmh_sm8250}, { .compatible = "qcom,sm8350-rpmh-clk", .data = &clk_rpmh_sm8350}, + { .compatible = "qcom,sm8450-rpmh-clk", .data = &clk_rpmh_sm8450}, { .compatible = "qcom,sc7280-rpmh-clk", .data = &clk_rpmh_sc7280}, { } }; diff --git a/drivers/clk/qcom/clk-smd-rpm.c b/drivers/clk/qcom/clk-smd-rpm.c index 5776d85a1e5c..ea28e45ca371 100644 --- a/drivers/clk/qcom/clk-smd-rpm.c +++ b/drivers/clk/qcom/clk-smd-rpm.c @@ -17,7 +17,6 @@ #include #include -#include #define QCOM_RPM_KEY_SOFTWARE_ENABLE 0x6e657773 #define QCOM_RPM_KEY_PIN_CTRL_CLK_BUFFER_ENABLE_KEY 0x62636370 @@ -151,12 +150,6 @@ struct clk_smd_rpm_req { __le32 value; }; -struct rpm_cc { - struct qcom_rpm *rpm; - struct clk_smd_rpm **clks; - size_t num_clks; -}; - struct rpm_smd_clk_desc { struct clk_smd_rpm **clks; size_t num_clks; @@ -196,10 +189,6 @@ static int clk_smd_rpm_set_rate_active(struct clk_smd_rpm *r, .value = cpu_to_le32(DIV_ROUND_UP(rate, 1000)), /* to kHz */ }; - /* Buffered clock needs a binary value */ - if (r->rpm_res_type == QCOM_SMD_RPM_CLK_BUF_A) - req.value = cpu_to_le32(!!req.value); - return qcom_rpm_smd_write(r->rpm, QCOM_SMD_RPM_ACTIVE_STATE, r->rpm_res_type, r->rpm_clk_id, &req, sizeof(req)); @@ -214,10 +203,6 @@ static int clk_smd_rpm_set_rate_sleep(struct clk_smd_rpm *r, .value = cpu_to_le32(DIV_ROUND_UP(rate, 1000)), /* to kHz */ }; - /* Buffered clock needs a binary value */ - if (r->rpm_res_type == QCOM_SMD_RPM_CLK_BUF_A) - req.value = cpu_to_le32(!!req.value); - return qcom_rpm_smd_write(r->rpm, QCOM_SMD_RPM_SLEEP_STATE, r->rpm_res_type, r->rpm_clk_id, &req, sizeof(req)); @@ -1159,20 +1144,19 @@ MODULE_DEVICE_TABLE(of, rpm_smd_clk_match_table); static struct clk_hw *qcom_smdrpm_clk_hw_get(struct of_phandle_args *clkspec, void *data) { - struct rpm_cc *rcc = data; + const struct rpm_smd_clk_desc *desc = data; unsigned int idx = clkspec->args[0]; - if (idx >= rcc->num_clks) { + if (idx >= desc->num_clks) { pr_err("%s: invalid index %u\n", __func__, idx); return ERR_PTR(-EINVAL); } - return rcc->clks[idx] ? &rcc->clks[idx]->hw : ERR_PTR(-ENOENT); + return desc->clks[idx] ? &desc->clks[idx]->hw : ERR_PTR(-ENOENT); } static int rpm_smd_clk_probe(struct platform_device *pdev) { - struct rpm_cc *rcc; int ret; size_t num_clks, i; struct qcom_smd_rpm *rpm; @@ -1192,13 +1176,6 @@ static int rpm_smd_clk_probe(struct platform_device *pdev) rpm_smd_clks = desc->clks; num_clks = desc->num_clks; - rcc = devm_kzalloc(&pdev->dev, sizeof(*rcc), GFP_KERNEL); - if (!rcc) - return -ENOMEM; - - rcc->clks = rpm_smd_clks; - rcc->num_clks = num_clks; - for (i = 0; i < num_clks; i++) { if (!rpm_smd_clks[i]) continue; @@ -1224,7 +1201,7 @@ static int rpm_smd_clk_probe(struct platform_device *pdev) } ret = devm_of_clk_add_hw_provider(&pdev->dev, qcom_smdrpm_clk_hw_get, - rcc); + (void *)desc); if (ret) goto err; diff --git a/drivers/clk/qcom/gcc-msm8976.c b/drivers/clk/qcom/gcc-msm8976.c new file mode 100644 index 000000000000..a8b15814933e --- /dev/null +++ b/drivers/clk/qcom/gcc-msm8976.c @@ -0,0 +1,4155 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Qualcomm Global Clock Controller driver for MSM8956/76 + * + * Copyright (c) 2016-2021, AngeloGioacchino Del Regno + * + * + * Driver cleanup and modernization + * Copyright (c) 2021, Konrad Dybcio + * Marijn Suijten + * + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "clk-pll.h" +#include "clk-branch.h" +#include "clk-rcg.h" +#include "common.h" +#include "gdsc.h" +#include "reset.h" + +enum { + P_GPLL0_OUT_MAIN, + P_GPLL0_AUX, + P_GPLL0_OUT, + P_GPLL0_OUT_M, + P_GPLL0_OUT_MDP, + P_GPLL2_AUX, + P_GPLL2_OUT, + P_GPLL4_OUT_MAIN, + P_GPLL4_AUX, + P_GPLL4_OUT, + P_GPLL4_GFX3D, + P_GPLL6_OUT_MAIN, + P_GPLL6_AUX, + P_GPLL6_OUT, + P_GPLL6_GFX3D, + P_DSI0PLL, + P_DSI1PLL, + P_DSI0PLL_BYTE, + P_DSI1PLL_BYTE, + P_XO_A, + P_XO, +}; + +static struct clk_pll gpll0 = { + .l_reg = 0x21004, + .m_reg = 0x21008, + .n_reg = 0x2100c, + .config_reg = 0x21014, + .mode_reg = 0x21000, + .status_reg = 0x2101c, + .status_bit = 17, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gpll0", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_pll_ops, + }, +}; + +static struct clk_regmap gpll0_vote = { + .enable_reg = 0x45000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gpll0_vote", + .parent_hws = (const struct clk_hw *[]) { + &gpll0.clkr.hw, + }, + .num_parents = 1, + /* This clock is required for other ones to function. */ + .flags = CLK_IS_CRITICAL, + .ops = &clk_pll_vote_ops, + }, +}; + +static struct clk_pll gpll2 = { + .l_reg = 0x4a004, + .m_reg = 0x4a008, + .n_reg = 0x4a00c, + .config_reg = 0x4a014, + .mode_reg = 0x4a000, + .status_reg = 0x4a01c, + .status_bit = 17, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gpll2", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_pll_ops, + }, +}; + +static struct clk_regmap gpll2_vote = { + .enable_reg = 0x45000, + .enable_mask = BIT(2), + .hw.init = &(struct clk_init_data){ + .name = "gpll2_vote", + .parent_hws = (const struct clk_hw *[]) { + &gpll2.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_pll_vote_ops, + }, +}; + +static const struct pll_freq_tbl gpll3_freq_tbl[] = { + { 1100000000, 57, 7, 24, 0 }, + { } +}; + +static struct clk_pll gpll3 = { + .l_reg = 0x22004, + .m_reg = 0x22008, + .n_reg = 0x2200c, + .config_reg = 0x22010, + .mode_reg = 0x22000, + .status_reg = 0x22024, + .status_bit = 17, + .freq_tbl = gpll3_freq_tbl, + .clkr.hw.init = &(struct clk_init_data) { + .name = "gpll3", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_pll_ops, + }, +}; + +static struct clk_regmap gpll3_vote = { + .enable_reg = 0x45000, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gpll3_vote", + .parent_hws = (const struct clk_hw *[]) { + &gpll3.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_pll_vote_ops, + }, +}; + +/* GPLL3 at 1100MHz, main output enabled. */ +static const struct pll_config gpll3_config = { + .l = 57, + .m = 7, + .n = 24, + .vco_val = 0x0, + .vco_mask = 0x3 << 20, + .pre_div_val = 0x0, + .pre_div_mask = 0x7 << 12, + .post_div_val = 0x0, + .post_div_mask = 0x3 << 8, + .mn_ena_mask = BIT(24), + .main_output_mask = BIT(0), + .aux_output_mask = BIT(1), +}; + +static struct clk_pll gpll4 = { + .l_reg = 0x24004, + .m_reg = 0x24008, + .n_reg = 0x2400c, + .config_reg = 0x24018, + .mode_reg = 0x24000, + .status_reg = 0x24024, + .status_bit = 17, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gpll4", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_pll_ops, + }, +}; + +static struct clk_regmap gpll4_vote = { + .enable_reg = 0x45000, + .enable_mask = BIT(5), + .hw.init = &(struct clk_init_data){ + .name = "gpll4_vote", + .parent_hws = (const struct clk_hw *[]) { + &gpll4.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_pll_vote_ops, + }, +}; + +static struct clk_pll gpll6 = { + .mode_reg = 0x37000, + .l_reg = 0x37004, + .m_reg = 0x37008, + .n_reg = 0x3700c, + .config_reg = 0x37014, + .status_reg = 0x3701c, + .status_bit = 17, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gpll6", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_pll_ops, + }, +}; + +static struct clk_regmap gpll6_vote = { + .enable_reg = 0x45000, + .enable_mask = BIT(7), + .hw.init = &(struct clk_init_data){ + .name = "gpll6_vote", + .parent_hws = (const struct clk_hw *[]) { + &gpll6.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_pll_vote_ops, + }, +}; + +static const struct parent_map gcc_parent_map_1[] = { + { P_XO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL4_OUT, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_1[] = { + { .fw_name = "xo" }, + { .hw = &gpll0_vote.hw }, + { .hw = &gpll4_vote.hw }, +}; + +static const struct parent_map gcc_parent_map_v1_1[] = { + { P_XO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL2_OUT, 4 }, +}; + +static const struct clk_parent_data gcc_parent_data_v1_1[] = { + { .fw_name = "xo" }, + { .hw = &gpll0_vote.hw }, + { .hw = &gpll2_vote.hw }, +}; + +static const struct parent_map gcc_parent_map_2[] = { + { P_XO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL2_AUX, 3 }, + { P_GPLL4_OUT, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_2[] = { + { .fw_name = "xo" }, + { .hw = &gpll0_vote.hw }, + { .hw = &gpll2_vote.hw }, + { .hw = &gpll4_vote.hw }, +}; + +static const struct parent_map gcc_parent_map_3[] = { + { P_XO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL2_AUX, 3 }, + { P_GPLL6_AUX, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_3[] = { + { .fw_name = "xo" }, + { .hw = &gpll0_vote.hw }, + { .hw = &gpll2_vote.hw }, + { .hw = &gpll6_vote.hw }, +}; + +static const struct parent_map gcc_parent_map_4[] = { + { P_XO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, +}; + +static const struct parent_map gcc_parent_map_4_fs[] = { + { P_XO, 0 }, + { P_GPLL0_OUT, 2 }, +}; + +static const struct parent_map gcc_parent_map_5[] = { + { P_XO, 0 }, + { P_GPLL4_OUT, 2 }, + { P_GPLL6_OUT_MAIN, 1 }, +}; + +static const struct clk_parent_data gcc_parent_data_5[] = { + { .fw_name = "xo" }, + { .hw = &gpll4_vote.hw }, + { .hw = &gpll6_vote.hw }, +}; + +static const struct parent_map gcc_parent_map_6[] = { + { P_XO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL4_OUT_MAIN, 5 }, +}; + +static const struct clk_parent_data gcc_parent_data_6[] = { + { .fw_name = "xo" }, + { .hw = &gpll0_vote.hw }, + { .hw = &gpll4_vote.hw }, +}; + +static const struct parent_map gcc_parent_map_7_mdp[] = { + { P_XO, 0 }, + { P_GPLL6_OUT, 3 }, + { P_GPLL0_OUT_MDP, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_7_mdp[] = { + { .fw_name = "xo" }, + { .hw = &gpll6_vote.hw }, + { .hw = &gpll0_vote.hw }, +}; + +static const struct parent_map gcc_parent_map_7[] = { + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL6_OUT, 3 }, +}; + +static const struct clk_parent_data gcc_parent_data_7[] = { + { .hw = &gpll0_vote.hw }, + { .hw = &gpll6_vote.hw }, +}; + +static const struct parent_map gcc_parent_map_8[] = { + { P_XO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, +}; + +static const struct clk_parent_data gcc_parent_data_4_8[] = { + { .fw_name = "xo" }, + { .hw = &gpll0_vote.hw }, +}; + +static const struct parent_map gcc_parent_map_8_a[] = { + { P_XO_A, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, +}; + +static const struct clk_parent_data gcc_parent_data_8_a[] = { + { .fw_name = "xo_a" }, + { .hw = &gpll0_vote.hw }, +}; + +static const struct parent_map gcc_parent_map_8_gp[] = { + { P_GPLL0_OUT_MAIN, 1 }, +}; + +static const struct clk_parent_data gcc_parent_data_8_gp[] = { + { .hw = &gpll0_vote.hw }, +}; + +static const struct parent_map gcc_parent_map_9[] = { + { P_XO, 0 }, + { P_GPLL6_OUT_MAIN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_9[] = { + { .fw_name = "xo" }, + { .hw = &gpll6_vote.hw }, +}; + +static const struct parent_map gcc_parent_map_10[] = { + { P_XO, 0 }, +}; + +static const struct clk_parent_data gcc_parent_data_10[] = { + { .fw_name = "xo" }, +}; + +static const struct parent_map gcc_parent_map_sdcc_ice[] = { + { P_XO, 0 }, + { P_GPLL0_OUT_M, 3 }, +}; + +static const struct parent_map gcc_parent_map_cci[] = { + { P_XO, 0 }, + { P_GPLL0_AUX, 2 }, +}; + +static const struct parent_map gcc_parent_map_cpp[] = { + { P_XO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL4_AUX, 3 }, +}; + +static const struct parent_map gcc_parent_map_mdss_pix0[] = { + { P_XO, 0 }, + { P_DSI0PLL, 1 }, +}; + +static const struct clk_parent_data gcc_parent_data_mdss_pix0[] = { + { .fw_name = "xo" }, + { .fw_name = "dsi0pll" }, +}; + +static const struct parent_map gcc_parent_map_mdss_pix1[] = { + { P_XO, 0 }, + { P_DSI0PLL, 3 }, + { P_DSI1PLL, 1 }, +}; + +static const struct clk_parent_data gcc_parent_data_mdss_pix1[] = { + { .fw_name = "xo" }, + { .fw_name = "dsi0pll" }, + { .fw_name = "dsi1pll" }, +}; + +static const struct parent_map gcc_parent_map_mdss_byte0[] = { + { P_XO, 0 }, + { P_DSI0PLL_BYTE, 1 }, +}; + +static const struct clk_parent_data gcc_parent_data_mdss_byte0[] = { + { .fw_name = "xo" }, + { .fw_name = "dsi0pllbyte" }, +}; + +static const struct parent_map gcc_parent_map_mdss_byte1[] = { + { P_XO, 0 }, + { P_DSI0PLL_BYTE, 3 }, + { P_DSI1PLL_BYTE, 1 }, +}; + +static const struct clk_parent_data gcc_parent_data_mdss_byte1[] = { + { .fw_name = "xo" }, + { .fw_name = "dsi0pllbyte" }, + { .fw_name = "dsi1pllbyte" }, +}; + +static const struct parent_map gcc_parent_map_gfx3d[] = { + { P_XO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL4_GFX3D, 5 }, + { P_GPLL6_GFX3D, 3 }, +}; + +static const struct clk_parent_data gcc_parent_data_gfx3d[] = { + { .fw_name = "xo" }, + { .hw = &gpll0_vote.hw }, + { .hw = &gpll4_vote.hw }, + { .hw = &gpll6_vote.hw }, +}; + +static const struct freq_tbl ftbl_aps_0_clk_src[] = { + F(19200000, P_XO, 1, 0, 0), + F(300000000, P_GPLL4_OUT, 4, 0, 0), + F(540000000, P_GPLL6_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 aps_0_clk_src = { + .cmd_rcgr = 0x78008, + .hid_width = 5, + .parent_map = gcc_parent_map_5, + .freq_tbl = ftbl_aps_0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "aps_0_clk_src", + .parent_data = gcc_parent_data_5, + .num_parents = ARRAY_SIZE(gcc_parent_data_5), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_aps_1_clk_src[] = { + F(19200000, P_XO, 1, 0, 0), + F(300000000, P_GPLL4_OUT, 4, 0, 0), + F(540000000, P_GPLL6_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 aps_1_clk_src = { + .cmd_rcgr = 0x79008, + .hid_width = 5, + .parent_map = gcc_parent_map_5, + .freq_tbl = ftbl_aps_1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "aps_1_clk_src", + .parent_data = gcc_parent_data_5, + .num_parents = ARRAY_SIZE(gcc_parent_data_5), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_apss_ahb_clk_src[] = { + F(19200000, P_XO_A, 1, 0, 0), + F(50000000, P_GPLL0_OUT_MAIN, 16, 0, 0), + F(88890000, P_GPLL0_OUT_MAIN, 9, 0, 0), + F(100000000, P_GPLL0_OUT_MAIN, 8, 0, 0), + { } +}; + +static struct clk_rcg2 apss_ahb_clk_src = { + .cmd_rcgr = 0x46000, + .hid_width = 5, + .parent_map = gcc_parent_map_8_a, + .freq_tbl = ftbl_apss_ahb_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "apss_ahb_clk_src", + .parent_data = gcc_parent_data_8_a, + .num_parents = ARRAY_SIZE(gcc_parent_data_8_a), + .ops = &clk_rcg2_ops, + /* + * This clock allows the CPUs to communicate with + * the rest of the SoC. Without it, the brain will + * operate without the rest of the body. + */ + .flags = CLK_IS_CRITICAL, + }, +}; + +static const struct freq_tbl ftbl_blsp_i2c_apps_clk_src[] = { + F(19200000, P_XO, 1, 0, 0), + F(50000000, P_GPLL0_OUT_MAIN, 16, 0, 0), + { } +}; + +static struct clk_rcg2 blsp1_qup1_i2c_apps_clk_src = { + .cmd_rcgr = 0x200c, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_blsp_i2c_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup1_i2c_apps_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_blsp_spi_apps_clk_src[] = { + F(960000, P_XO, 10, 1, 2), + F(4800000, P_XO, 4, 0, 0), + F(9600000, P_XO, 2, 0, 0), + F(16000000, P_GPLL0_OUT_MAIN, 10, 1, 5), + F(19200000, P_XO, 1, 0, 0), + F(25000000, P_GPLL0_OUT_MAIN, 16, 1, 2), + F(50000000, P_GPLL0_OUT_MAIN, 16, 0, 0), + { } +}; + +static struct clk_rcg2 blsp1_qup1_spi_apps_clk_src = { + .cmd_rcgr = 0x2024, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_blsp_spi_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup1_spi_apps_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_qup2_i2c_apps_clk_src = { + .cmd_rcgr = 0x3000, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_blsp_i2c_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup2_i2c_apps_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_qup2_spi_apps_clk_src = { + .cmd_rcgr = 0x3014, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_blsp_spi_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup2_spi_apps_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_qup3_i2c_apps_clk_src = { + .cmd_rcgr = 0x4000, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_blsp_i2c_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup3_i2c_apps_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_qup3_spi_apps_clk_src = { + .cmd_rcgr = 0x4024, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_blsp_spi_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup3_spi_apps_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_qup4_i2c_apps_clk_src = { + .cmd_rcgr = 0x5000, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_blsp_i2c_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup4_i2c_apps_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_qup4_spi_apps_clk_src = { + .cmd_rcgr = 0x5024, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_blsp_spi_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup4_spi_apps_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_blsp_uart_apps_clk_src[] = { + F(3686400, P_GPLL0_OUT_MAIN, 1, 72, 15625), + F(7372800, P_GPLL0_OUT_MAIN, 1, 144, 15625), + F(14745600, P_GPLL0_OUT_MAIN, 1, 288, 15625), + F(16000000, P_GPLL0_OUT_MAIN, 10, 1, 5), + F(19200000, P_XO, 1, 0, 0), + F(24000000, P_GPLL0_OUT_MAIN, 1, 3, 100), + F(25000000, P_GPLL0_OUT_MAIN, 16, 1, 2), + F(32000000, P_GPLL0_OUT_MAIN, 1, 1, 25), + F(40000000, P_GPLL0_OUT_MAIN, 1, 1, 20), + F(46400000, P_GPLL0_OUT_MAIN, 1, 29, 500), + F(48000000, P_GPLL0_OUT_MAIN, 1, 3, 50), + F(51200000, P_GPLL0_OUT_MAIN, 1, 8, 125), + F(56000000, P_GPLL0_OUT_MAIN, 1, 7, 100), + F(58982400, P_GPLL0_OUT_MAIN, 1, 1152, 15625), + F(60000000, P_GPLL0_OUT_MAIN, 1, 3, 40), + F(64000000, P_GPLL0_OUT_MAIN, 1, 2, 25), + { } +}; + +static struct clk_rcg2 blsp1_uart1_apps_clk_src = { + .cmd_rcgr = 0x2044, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_blsp_uart_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_uart1_apps_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_uart2_apps_clk_src = { + .cmd_rcgr = 0x3034, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_blsp_uart_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_uart2_apps_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp2_qup1_i2c_apps_clk_src = { + .cmd_rcgr = 0xc00c, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_blsp_i2c_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp2_qup1_i2c_apps_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp2_qup1_spi_apps_clk_src = { + .cmd_rcgr = 0xc024, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_blsp_spi_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp2_qup1_spi_apps_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp2_qup2_i2c_apps_clk_src = { + .cmd_rcgr = 0xd000, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_blsp_i2c_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp2_qup2_i2c_apps_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp2_qup2_spi_apps_clk_src = { + .cmd_rcgr = 0xd014, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_blsp_spi_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp2_qup2_spi_apps_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp2_qup3_i2c_apps_clk_src = { + .cmd_rcgr = 0xf000, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_blsp_i2c_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp2_qup3_i2c_apps_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp2_qup3_spi_apps_clk_src = { + .cmd_rcgr = 0xf024, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_blsp_spi_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp2_qup3_spi_apps_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp2_qup4_i2c_apps_clk_src = { + .cmd_rcgr = 0x18000, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_blsp_i2c_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp2_qup4_i2c_apps_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp2_qup4_spi_apps_clk_src = { + .cmd_rcgr = 0x18024, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_blsp_spi_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp2_qup4_spi_apps_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp2_uart1_apps_clk_src = { + .cmd_rcgr = 0xc044, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_blsp_uart_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp2_uart1_apps_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp2_uart2_apps_clk_src = { + .cmd_rcgr = 0xd034, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_blsp_uart_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp2_uart2_apps_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_cci_clk_src[] = { + F(19200000, P_XO, 1, 0, 0), + F(37500000, P_GPLL0_AUX, 1, 3, 64), + { } +}; + +static struct clk_rcg2 cci_clk_src = { + .cmd_rcgr = 0x51000, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_cci, + .freq_tbl = ftbl_cci_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "cci_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_cpp_clk_src[] = { + F(160000000, P_GPLL0_OUT_MAIN, 5, 0, 0), + F(240000000, P_GPLL4_AUX, 5, 0, 0), + F(320000000, P_GPLL0_OUT_MAIN, 2.5, 0, 0), + F(400000000, P_GPLL0_OUT_MAIN, 2, 0, 0), + F(480000000, P_GPLL4_AUX, 2.5, 0, 0), + { } +}; + +static struct clk_rcg2 cpp_clk_src = { + .cmd_rcgr = 0x58018, + .hid_width = 5, + .parent_map = gcc_parent_map_cpp, + .freq_tbl = ftbl_cpp_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "cpp_clk_src", + .parent_data = gcc_parent_data_6, + .num_parents = ARRAY_SIZE(gcc_parent_data_6), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_csi0_clk_src[] = { + F(100000000, P_GPLL0_OUT_MAIN, 8, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 4, 0, 0), + F(266670000, P_GPLL0_OUT_MAIN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 csi0_clk_src = { + .cmd_rcgr = 0x4e020, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_csi0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi0_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_csi1_clk_src[] = { + F(100000000, P_GPLL0_OUT_MAIN, 8, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 4, 0, 0), + F(266670000, P_GPLL0_OUT_MAIN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 csi1_clk_src = { + .cmd_rcgr = 0x4f020, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_csi1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi1_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_csi2_clk_src[] = { + F(100000000, P_GPLL0_OUT_MAIN, 8, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 4, 0, 0), + F(266670000, P_GPLL0_OUT_MAIN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 csi2_clk_src = { + .cmd_rcgr = 0x3c020, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_csi2_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi2_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_camss_gp0_clk_src[] = { + F(100000000, P_GPLL0_OUT_MAIN, 8, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 4, 0, 0), + F(266670000, P_GPLL0_OUT_MAIN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 camss_gp0_clk_src = { + .cmd_rcgr = 0x54000, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_8_gp, + .freq_tbl = ftbl_camss_gp0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "camss_gp0_clk_src", + .parent_data = gcc_parent_data_8_gp, + .num_parents = ARRAY_SIZE(gcc_parent_data_8_gp), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_camss_gp1_clk_src[] = { + F(100000000, P_GPLL0_OUT_MAIN, 8, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 4, 0, 0), + F(266670000, P_GPLL0_OUT_MAIN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 camss_gp1_clk_src = { + .cmd_rcgr = 0x55000, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_8_gp, + .freq_tbl = ftbl_camss_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "camss_gp1_clk_src", + .parent_data = gcc_parent_data_8_gp, + .num_parents = ARRAY_SIZE(gcc_parent_data_8_gp), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_jpeg0_clk_src[] = { + F(133330000, P_GPLL0_OUT_MAIN, 6, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 4, 0, 0), + F(266666667, P_GPLL0_OUT_MAIN, 3, 0, 0), + F(320000000, P_GPLL0_OUT_MAIN, 2.5, 0, 0), + { } +}; + +static struct clk_rcg2 jpeg0_clk_src = { + .cmd_rcgr = 0x57000, + .hid_width = 5, + .parent_map = gcc_parent_map_6, + .freq_tbl = ftbl_jpeg0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "jpeg0_clk_src", + .parent_data = gcc_parent_data_6, + .num_parents = ARRAY_SIZE(gcc_parent_data_6), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_mclk_clk_src[] = { + F(8000000, P_GPLL0_OUT_MAIN, 1, 1, 100), + F(24000000, P_GPLL6_OUT, 1, 1, 45), + F(66670000, P_GPLL0_OUT_MAIN, 12, 0, 0), + { } +}; + +static struct clk_rcg2 mclk0_clk_src = { + .cmd_rcgr = 0x52000, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_7, + .freq_tbl = ftbl_mclk_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mclk0_clk_src", + .parent_data = gcc_parent_data_7, + .num_parents = ARRAY_SIZE(gcc_parent_data_7), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 mclk1_clk_src = { + .cmd_rcgr = 0x53000, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_7, + .freq_tbl = ftbl_mclk_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mclk1_clk_src", + .parent_data = gcc_parent_data_7, + .num_parents = ARRAY_SIZE(gcc_parent_data_7), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 mclk2_clk_src = { + .cmd_rcgr = 0x5c000, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_7, + .freq_tbl = ftbl_mclk_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mclk2_clk_src", + .parent_data = gcc_parent_data_7, + .num_parents = ARRAY_SIZE(gcc_parent_data_7), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_csi0phytimer_clk_src[] = { + F(100000000, P_GPLL0_OUT_MAIN, 8, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 4, 0, 0), + F(266670000, P_GPLL0_OUT_MAIN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 csi0phytimer_clk_src = { + .cmd_rcgr = 0x4e000, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_csi0phytimer_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi0phytimer_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_csi1phytimer_clk_src[] = { + F(100000000, P_GPLL0_OUT_MAIN, 8, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 4, 0, 0), + F(266670000, P_GPLL0_OUT_MAIN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 csi1phytimer_clk_src = { + .cmd_rcgr = 0x4f000, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_csi1phytimer_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi1phytimer_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_camss_top_ahb_clk_src[] = { + F(40000000, P_GPLL0_OUT_MAIN, 10, 1, 2), + F(80000000, P_GPLL0_OUT_MAIN, 10, 0, 0), + { } +}; + +static struct clk_rcg2 camss_top_ahb_clk_src = { + .cmd_rcgr = 0x5a000, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_camss_top_ahb_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "camss_top_ahb_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_vfe0_clk_src[] = { + F(50000000, P_GPLL0_OUT_MAIN, 16, 0, 0), + F(80000000, P_GPLL0_OUT_MAIN, 10, 0, 0), + F(100000000, P_GPLL0_OUT_MAIN, 8, 0, 0), + F(133333333, P_GPLL0_OUT_MAIN, 6, 0, 0), + F(160000000, P_GPLL0_OUT_MAIN, 5, 0, 0), + F(177777778, P_GPLL0_OUT_MAIN, 4.5, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 4, 0, 0), + F(266666667, P_GPLL0_OUT_MAIN, 3, 0, 0), + F(300000000, P_GPLL4_OUT, 4, 0, 0), + F(320000000, P_GPLL0_OUT_MAIN, 2.5, 0, 0), + F(466000000, P_GPLL2_AUX, 2, 0, 0), + { } +}; + +static struct clk_rcg2 vfe0_clk_src = { + .cmd_rcgr = 0x58000, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_vfe0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "vfe0_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = ARRAY_SIZE(gcc_parent_data_2), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_vfe1_clk_src[] = { + F(50000000, P_GPLL0_OUT_MAIN, 16, 0, 0), + F(80000000, P_GPLL0_OUT_MAIN, 10, 0, 0), + F(100000000, P_GPLL0_OUT_MAIN, 8, 0, 0), + F(133333333, P_GPLL0_OUT_MAIN, 6, 0, 0), + F(160000000, P_GPLL0_OUT_MAIN, 5, 0, 0), + F(177777778, P_GPLL0_OUT_MAIN, 4.5, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 4, 0, 0), + F(266666667, P_GPLL0_OUT_MAIN, 3, 0, 0), + F(300000000, P_GPLL4_OUT, 4, 0, 0), + F(320000000, P_GPLL0_OUT_MAIN, 2.5, 0, 0), + F(466000000, P_GPLL2_AUX, 2, 0, 0), + { } +}; + +static struct clk_rcg2 vfe1_clk_src = { + .cmd_rcgr = 0x58054, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_vfe1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "vfe1_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = ARRAY_SIZE(gcc_parent_data_2), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_crypto_clk_src[] = { + F(50000000, P_GPLL0_OUT_MAIN, 16, 0, 0), + F(80000000, P_GPLL0_OUT_MAIN, 10, 0, 0), + F(100000000, P_GPLL0_OUT_MAIN, 8, 0, 0), + F(160000000, P_GPLL0_OUT_MAIN, 5, 0, 0), + { } +}; + +static struct clk_rcg2 crypto_clk_src = { + .cmd_rcgr = 0x16004, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_crypto_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "crypto_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gp1_clk_src[] = { + F(19200000, P_XO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gp1_clk_src = { + .cmd_rcgr = 0x8004, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_8_gp, + .freq_tbl = ftbl_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gp1_clk_src", + .parent_hws = (const struct clk_hw *[]) { + &gpll0_vote.hw, + }, + .num_parents = 1, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gp2_clk_src[] = { + F(19200000, P_XO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gp2_clk_src = { + .cmd_rcgr = 0x9004, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_8_gp, + .freq_tbl = ftbl_gp2_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gp2_clk_src", + .parent_hws = (const struct clk_hw *[]) { + &gpll0_vote.hw, + }, + .num_parents = 1, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gp3_clk_src[] = { + F(19200000, P_XO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gp3_clk_src = { + .cmd_rcgr = 0xa004, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_8_gp, + .freq_tbl = ftbl_gp3_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gp3_clk_src", + .parent_hws = (const struct clk_hw *[]) { + &gpll0_vote.hw, + }, + .num_parents = 1, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 byte0_clk_src = { + .cmd_rcgr = 0x4d044, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_mdss_byte0, + .clkr.hw.init = &(struct clk_init_data){ + .name = "byte0_clk_src", + .parent_data = gcc_parent_data_mdss_byte0, + .num_parents = ARRAY_SIZE(gcc_parent_data_mdss_byte0), + .ops = &clk_byte2_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_rcg2 byte1_clk_src = { + .cmd_rcgr = 0x4d0b0, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_mdss_byte1, + .clkr.hw.init = &(struct clk_init_data){ + .name = "byte1_clk_src", + .parent_data = gcc_parent_data_mdss_byte1, + .num_parents = ARRAY_SIZE(gcc_parent_data_mdss_byte1), + .ops = &clk_byte2_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static const struct freq_tbl ftbl_esc0_1_clk_src[] = { + F(19200000, P_XO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 esc0_clk_src = { + .cmd_rcgr = 0x4d05c, + .hid_width = 5, + .freq_tbl = ftbl_esc0_1_clk_src, + .parent_map = gcc_parent_map_mdss_byte0, + .clkr.hw.init = &(struct clk_init_data){ + .name = "esc0_clk_src", + .parent_data = gcc_parent_data_mdss_byte0, + .num_parents = ARRAY_SIZE(gcc_parent_data_mdss_byte0), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 esc1_clk_src = { + .cmd_rcgr = 0x4d0a8, + .hid_width = 5, + .freq_tbl = ftbl_esc0_1_clk_src, + .parent_map = gcc_parent_map_mdss_byte1, + .clkr.hw.init = &(struct clk_init_data){ + .name = "esc1_clk_src", + .parent_data = gcc_parent_data_mdss_byte1, + .num_parents = ARRAY_SIZE(gcc_parent_data_mdss_byte1), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_mdp_clk_src[] = { + F(50000000, P_GPLL0_OUT_MDP, 16, 0, 0), + F(80000000, P_GPLL0_OUT_MDP, 10, 0, 0), + F(100000000, P_GPLL0_OUT_MDP, 8, 0, 0), + F(145454545, P_GPLL0_OUT_MDP, 5.5, 0, 0), + F(160000000, P_GPLL0_OUT_MDP, 5, 0, 0), + F(177777778, P_GPLL0_OUT_MDP, 4.5, 0, 0), + F(200000000, P_GPLL0_OUT_MDP, 4, 0, 0), + F(270000000, P_GPLL6_OUT, 4, 0, 0), + F(320000000, P_GPLL0_OUT_MDP, 2.5, 0, 0), + F(360000000, P_GPLL6_OUT, 3, 0, 0), + { } +}; + +static struct clk_rcg2 mdp_clk_src = { + .cmd_rcgr = 0x4d014, + .hid_width = 5, + .parent_map = gcc_parent_map_7_mdp, + .freq_tbl = ftbl_mdp_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mdp_clk_src", + .parent_data = gcc_parent_data_7_mdp, + .num_parents = ARRAY_SIZE(gcc_parent_data_7_mdp), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 pclk0_clk_src = { + .cmd_rcgr = 0x4d000, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_mdss_pix0, + .clkr.hw.init = &(struct clk_init_data){ + .name = "pclk0_clk_src", + .parent_data = gcc_parent_data_mdss_pix0, + .num_parents = ARRAY_SIZE(gcc_parent_data_mdss_pix0), + .ops = &clk_pixel_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_rcg2 pclk1_clk_src = { + .cmd_rcgr = 0x4d0b8, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_mdss_pix1, + .clkr.hw.init = &(struct clk_init_data){ + .name = "pclk1_clk_src", + .parent_data = gcc_parent_data_mdss_pix1, + .num_parents = ARRAY_SIZE(gcc_parent_data_mdss_pix1), + .ops = &clk_pixel_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static const struct freq_tbl ftbl_vsync_clk_src[] = { + F(19200000, P_XO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 vsync_clk_src = { + .cmd_rcgr = 0x4d02c, + .hid_width = 5, + .parent_map = gcc_parent_map_10, + .freq_tbl = ftbl_vsync_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "vsync_clk_src", + .parent_data = gcc_parent_data_10, + .num_parents = ARRAY_SIZE(gcc_parent_data_10), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gfx3d_clk_src[] = { + F(19200000, P_XO, 1, 0, 0), + F(50000000, P_GPLL0_OUT_MAIN, 16, 0, 0), + F(80000000, P_GPLL0_OUT_MAIN, 10, 0, 0), + F(100000000, P_GPLL0_OUT_MAIN, 8, 0, 0), + F(133333333, P_GPLL0_OUT_MAIN, 6, 0, 0), + F(160000000, P_GPLL0_OUT_MAIN, 5, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 4, 0, 0), + F(228571429, P_GPLL0_OUT_MAIN, 3.5, 0, 0), + F(240000000, P_GPLL6_GFX3D, 4.5, 0, 0), + F(266666667, P_GPLL0_OUT_MAIN, 3, 0, 0), + F(300000000, P_GPLL4_GFX3D, 4, 0, 0), + F(360000000, P_GPLL6_GFX3D, 3, 0, 0), + F(400000000, P_GPLL0_OUT_MAIN, 2, 0, 0), + F(432000000, P_GPLL6_GFX3D, 2.5, 0, 0), + F(480000000, P_GPLL4_GFX3D, 2.5, 0, 0), + F(540000000, P_GPLL6_GFX3D, 2, 0, 0), + F(600000000, P_GPLL4_GFX3D, 2, 0, 0), + { } +}; + +static const struct clk_init_data gfx3d_clk_params = { + .name = "gfx3d_clk_src", + .parent_data = gcc_parent_data_gfx3d, + .num_parents = ARRAY_SIZE(gcc_parent_data_gfx3d), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gfx3d_clk_src = { + .cmd_rcgr = 0x59000, + .hid_width = 5, + .parent_map = gcc_parent_map_gfx3d, + .freq_tbl = ftbl_gfx3d_clk_src, + .clkr.hw.init = &gfx3d_clk_params, +}; + +static const struct freq_tbl ftbl_pdm2_clk_src[] = { + F(64000000, P_GPLL0_OUT_MAIN, 12.5, 0, 0), + { } +}; + +static struct clk_rcg2 pdm2_clk_src = { + .cmd_rcgr = 0x44010, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_pdm2_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "pdm2_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_rbcpr_gfx_clk_src[] = { + F(19200000, P_XO, 1, 0, 0), + F(50000000, P_GPLL0_OUT_MAIN, 16, 0, 0), + { } +}; + +static struct clk_rcg2 rbcpr_gfx_clk_src = { + .cmd_rcgr = 0x3a00c, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_rbcpr_gfx_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "rbcpr_gfx_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_sdcc1_apps_clk_src[] = { + F(144000, P_XO, 16, 3, 25), + F(400000, P_XO, 12, 1, 4), + F(20000000, P_GPLL0_OUT_MAIN, 10, 1, 4), + F(25000000, P_GPLL0_OUT_MAIN, 16, 1, 2), + F(50000000, P_GPLL0_OUT_MAIN, 16, 0, 0), + F(100000000, P_GPLL0_OUT_MAIN, 8, 0, 0), + F(177777778, P_GPLL0_OUT_MAIN, 4.5, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 4, 0, 0), + F(342850000, P_GPLL4_OUT, 3.5, 0, 0), + F(400000000, P_GPLL4_OUT, 3, 0, 0), + { } +}; + +static const struct freq_tbl ftbl_sdcc1_8976_v1_1_apps_clk_src[] = { + F(144000, P_XO, 16, 3, 25), + F(400000, P_XO, 12, 1, 4), + F(20000000, P_GPLL0_OUT_MAIN, 10, 1, 4), + F(25000000, P_GPLL0_OUT_MAIN, 16, 1, 2), + F(50000000, P_GPLL0_OUT_MAIN, 16, 0, 0), + F(100000000, P_GPLL0_OUT_MAIN, 8, 0, 0), + F(177777778, P_GPLL0_OUT_MAIN, 4.5, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 4, 0, 0), + F(186400000, P_GPLL2_OUT, 5, 0, 0), + F(372800000, P_GPLL2_OUT, 2.5, 0, 0), + { } +}; + +static const struct clk_init_data sdcc1_apps_clk_src_8976v1_1_init = { + .name = "sdcc1_apps_clk_src", + .parent_data = gcc_parent_data_v1_1, + .num_parents = ARRAY_SIZE(gcc_parent_data_v1_1), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 sdcc1_apps_clk_src = { + .cmd_rcgr = 0x42004, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_sdcc1_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "sdcc1_apps_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = ARRAY_SIZE(gcc_parent_data_1), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_sdcc1_ice_core_clk_src[] = { + F(100000000, P_GPLL0_OUT_M, 8, 0, 0), + F(200000000, P_GPLL0_OUT_M, 4, 0, 0), + { } +}; + +static struct clk_rcg2 sdcc1_ice_core_clk_src = { + .cmd_rcgr = 0x5d000, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_sdcc_ice, + .freq_tbl = ftbl_sdcc1_ice_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "sdcc1_ice_core_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_sdcc2_4_apps_clk_src[] = { + F(144000, P_XO, 16, 3, 25), + F(400000, P_XO, 12, 1, 4), + F(20000000, P_GPLL0_OUT_MAIN, 10, 1, 4), + F(25000000, P_GPLL0_OUT_MAIN, 16, 1, 2), + F(40000000, P_GPLL0_OUT_MAIN, 10, 1, 2), + F(50000000, P_GPLL0_OUT_MAIN, 16, 0, 0), + F(80000000, P_GPLL0_OUT_MAIN, 10, 0, 0), + F(100000000, P_GPLL0_OUT_MAIN, 8, 0, 0), + F(177777778, P_GPLL0_OUT_MAIN, 4.5, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 4, 0, 0), + { } +}; + +static struct clk_rcg2 sdcc2_apps_clk_src = { + .cmd_rcgr = 0x43004, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_4, + .freq_tbl = ftbl_sdcc2_4_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "sdcc2_apps_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 sdcc3_apps_clk_src = { + .cmd_rcgr = 0x39004, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_4, + .freq_tbl = ftbl_sdcc2_4_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "sdcc3_apps_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_usb_fs_ic_clk_src[] = { + F(60000000, P_GPLL6_OUT_MAIN, 6, 1, 3), + { } +}; + +static struct clk_rcg2 usb_fs_ic_clk_src = { + .cmd_rcgr = 0x3f034, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_9, + .freq_tbl = ftbl_usb_fs_ic_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "usb_fs_ic_clk_src", + .parent_data = gcc_parent_data_9, + .num_parents = ARRAY_SIZE(gcc_parent_data_9), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_usb_fs_system_clk_src[] = { + F(64000000, P_GPLL0_OUT, 12.5, 0, 0), + { } +}; + +static struct clk_rcg2 usb_fs_system_clk_src = { + .cmd_rcgr = 0x3f010, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_4_fs, + .freq_tbl = ftbl_usb_fs_system_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "usb_fs_system_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_usb_hs_system_clk_src[] = { + F(57140000, P_GPLL0_OUT_MAIN, 14, 0, 0), + F(100000000, P_GPLL0_OUT_MAIN, 8, 0, 0), + F(133333333, P_GPLL0_OUT_MAIN, 6, 0, 0), + F(177780000, P_GPLL0_OUT_MAIN, 4.5, 0, 0), + { } +}; + +static struct clk_rcg2 usb_hs_system_clk_src = { + .cmd_rcgr = 0x41010, + .hid_width = 5, + .parent_map = gcc_parent_map_4, + .freq_tbl = ftbl_usb_hs_system_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "usb_hs_system_clk_src", + .parent_data = gcc_parent_data_4_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_4_8), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_vcodec0_clk_src[] = { + F(72727200, P_GPLL0_OUT_MAIN, 11, 0, 0), + F(80000000, P_GPLL0_OUT_MAIN, 10, 0, 0), + F(100000000, P_GPLL0_OUT_MAIN, 8, 0, 0), + F(133333333, P_GPLL0_OUT_MAIN, 6, 0, 0), + F(228570000, P_GPLL0_OUT_MAIN, 3.5, 0, 0), + F(310667000, P_GPLL2_AUX, 3, 0, 0), + F(360000000, P_GPLL6_AUX, 3, 0, 0), + F(400000000, P_GPLL0_OUT_MAIN, 2, 0, 0), + F(466000000, P_GPLL2_AUX, 2, 0, 0), + { } +}; + +static struct clk_rcg2 vcodec0_clk_src = { + .cmd_rcgr = 0x4c000, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_3, + .freq_tbl = ftbl_vcodec0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "vcodec0_clk_src", + .parent_data = gcc_parent_data_3, + .num_parents = ARRAY_SIZE(gcc_parent_data_3), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_branch gcc_aps_0_clk = { + .halt_reg = 0x78004, + .clkr = { + .enable_reg = 0x78004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_aps_0_clk", + .parent_hws = (const struct clk_hw *[]) { + &aps_0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aps_1_clk = { + .halt_reg = 0x79004, + .clkr = { + .enable_reg = 0x79004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_aps_1_clk", + .parent_hws = (const struct clk_hw *[]) { + &aps_1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup1_i2c_apps_clk = { + .halt_reg = 0x2008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp1_qup1_i2c_apps_clk", + .parent_hws = (const struct clk_hw *[]) { + &blsp1_qup1_i2c_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup1_spi_apps_clk = { + .halt_reg = 0x2004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp1_qup1_spi_apps_clk", + .parent_hws = (const struct clk_hw *[]) { + &blsp1_qup1_spi_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup2_i2c_apps_clk = { + .halt_reg = 0x3010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp1_qup2_i2c_apps_clk", + .parent_hws = (const struct clk_hw *[]) { + &blsp1_qup2_i2c_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup2_spi_apps_clk = { + .halt_reg = 0x300c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x300c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp1_qup2_spi_apps_clk", + .parent_hws = (const struct clk_hw *[]) { + &blsp1_qup2_spi_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup3_i2c_apps_clk = { + .halt_reg = 0x4020, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp1_qup3_i2c_apps_clk", + .parent_hws = (const struct clk_hw *[]) { + &blsp1_qup3_i2c_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup3_spi_apps_clk = { + .halt_reg = 0x401c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x401c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp1_qup3_spi_apps_clk", + .parent_hws = (const struct clk_hw *[]) { + &blsp1_qup3_spi_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup4_i2c_apps_clk = { + .halt_reg = 0x5020, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x5020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp1_qup4_i2c_apps_clk", + .parent_hws = (const struct clk_hw *[]) { + &blsp1_qup4_i2c_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup4_spi_apps_clk = { + .halt_reg = 0x501c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x501c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp1_qup4_spi_apps_clk", + .parent_hws = (const struct clk_hw *[]) { + &blsp1_qup4_spi_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_uart1_apps_clk = { + .halt_reg = 0x203c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x203c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp1_uart1_apps_clk", + .parent_hws = (const struct clk_hw *[]) { + &blsp1_uart1_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_uart2_apps_clk = { + .halt_reg = 0x302c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x302c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp1_uart2_apps_clk", + .parent_hws = (const struct clk_hw *[]) { + &blsp1_uart2_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_qup1_i2c_apps_clk = { + .halt_reg = 0xc008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xc008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp2_qup1_i2c_apps_clk", + .parent_hws = (const struct clk_hw *[]) { + &blsp2_qup1_i2c_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_qup1_spi_apps_clk = { + .halt_reg = 0xc004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xc004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp2_qup1_spi_apps_clk", + .parent_hws = (const struct clk_hw *[]) { + &blsp2_qup1_spi_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_qup2_i2c_apps_clk = { + .halt_reg = 0xd010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xd010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp2_qup2_i2c_apps_clk", + .parent_hws = (const struct clk_hw *[]) { + &blsp2_qup2_i2c_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_qup2_spi_apps_clk = { + .halt_reg = 0xd00c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xd00c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp2_qup2_spi_apps_clk", + .parent_hws = (const struct clk_hw *[]) { + &blsp2_qup2_spi_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_qup3_i2c_apps_clk = { + .halt_reg = 0xf020, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp2_qup3_i2c_apps_clk", + .parent_hws = (const struct clk_hw *[]) { + &blsp2_qup3_i2c_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_qup3_spi_apps_clk = { + .halt_reg = 0xf01c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf01c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp2_qup3_spi_apps_clk", + .parent_hws = (const struct clk_hw *[]) { + &blsp2_qup3_spi_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_qup4_i2c_apps_clk = { + .halt_reg = 0x18020, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x18020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp2_qup4_i2c_apps_clk", + .parent_hws = (const struct clk_hw *[]) { + &blsp2_qup4_i2c_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_qup4_spi_apps_clk = { + .halt_reg = 0x1801c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1801c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp2_qup4_spi_apps_clk", + .parent_hws = (const struct clk_hw *[]) { + &blsp2_qup4_spi_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_uart1_apps_clk = { + .halt_reg = 0xc03c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xc03c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp2_uart1_apps_clk", + .parent_hws = (const struct clk_hw *[]) { + &blsp2_uart1_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_uart2_apps_clk = { + .halt_reg = 0xd02c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xd02c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp2_uart2_apps_clk", + .parent_hws = (const struct clk_hw *[]) { + &blsp2_uart2_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_cci_ahb_clk = { + .halt_reg = 0x5101c, + .clkr = { + .enable_reg = 0x5101c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_cci_ahb_clk", + .parent_hws = (const struct clk_hw *[]) { + &camss_top_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_cci_clk = { + .halt_reg = 0x51018, + .clkr = { + .enable_reg = 0x51018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_cci_clk", + .parent_hws = (const struct clk_hw *[]) { + &cci_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_cpp_ahb_clk = { + .halt_reg = 0x58040, + .clkr = { + .enable_reg = 0x58040, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_cpp_ahb_clk", + .parent_hws = (const struct clk_hw *[]) { + &camss_top_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_cpp_axi_clk = { + .halt_reg = 0x58064, + .clkr = { + .enable_reg = 0x58064, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_cpp_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_cpp_clk = { + .halt_reg = 0x5803c, + .clkr = { + .enable_reg = 0x5803c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_cpp_clk", + .parent_hws = (const struct clk_hw *[]) { + &cpp_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_csi0_ahb_clk = { + .halt_reg = 0x4e040, + .clkr = { + .enable_reg = 0x4e040, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi0_ahb_clk", + .parent_hws = (const struct clk_hw *[]) { + &camss_top_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_csi0_clk = { + .halt_reg = 0x4e03c, + .clkr = { + .enable_reg = 0x4e03c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi0_clk", + .parent_hws = (const struct clk_hw *[]) { + &csi0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_csi0phy_clk = { + .halt_reg = 0x4e048, + .clkr = { + .enable_reg = 0x4e048, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi0phy_clk", + .parent_hws = (const struct clk_hw *[]) { + &csi0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_csi0pix_clk = { + .halt_reg = 0x4e058, + .clkr = { + .enable_reg = 0x4e058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi0pix_clk", + .parent_hws = (const struct clk_hw *[]) { + &csi0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_csi0rdi_clk = { + .halt_reg = 0x4e050, + .clkr = { + .enable_reg = 0x4e050, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi0rdi_clk", + .parent_hws = (const struct clk_hw *[]) { + &csi0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_csi1_ahb_clk = { + .halt_reg = 0x4f040, + .clkr = { + .enable_reg = 0x4f040, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi1_ahb_clk", + .parent_hws = (const struct clk_hw *[]) { + &camss_top_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_csi1_clk = { + .halt_reg = 0x4f03c, + .clkr = { + .enable_reg = 0x4f03c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi1_clk", + .parent_hws = (const struct clk_hw *[]) { + &csi1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_csi1phy_clk = { + .halt_reg = 0x4f048, + .clkr = { + .enable_reg = 0x4f048, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi1phy_clk", + .parent_hws = (const struct clk_hw *[]) { + &csi1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_csi1pix_clk = { + .halt_reg = 0x4f058, + .clkr = { + .enable_reg = 0x4f058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi1pix_clk", + .parent_hws = (const struct clk_hw *[]) { + &csi1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_csi1rdi_clk = { + .halt_reg = 0x4f050, + .clkr = { + .enable_reg = 0x4f050, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi1rdi_clk", + .parent_hws = (const struct clk_hw *[]) { + &csi1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_csi2_ahb_clk = { + .halt_reg = 0x3c040, + .clkr = { + .enable_reg = 0x3c040, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi2_ahb_clk", + .parent_hws = (const struct clk_hw *[]) { + &camss_top_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_csi2_clk = { + .halt_reg = 0x3c03c, + .clkr = { + .enable_reg = 0x3c03c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi2_clk", + .parent_hws = (const struct clk_hw *[]) { + &csi2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_csi2phy_clk = { + .halt_reg = 0x3c048, + .clkr = { + .enable_reg = 0x3c048, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi2phy_clk", + .parent_hws = (const struct clk_hw *[]) { + &csi2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_csi2pix_clk = { + .halt_reg = 0x3c058, + .clkr = { + .enable_reg = 0x3c058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi2pix_clk", + .parent_hws = (const struct clk_hw *[]) { + &csi2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_csi2rdi_clk = { + .halt_reg = 0x3c050, + .clkr = { + .enable_reg = 0x3c050, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi2rdi_clk", + .parent_hws = (const struct clk_hw *[]) { + &csi2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_csi_vfe0_clk = { + .halt_reg = 0x58050, + .clkr = { + .enable_reg = 0x58050, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi_vfe0_clk", + .parent_hws = (const struct clk_hw *[]) { + &vfe0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_csi_vfe1_clk = { + .halt_reg = 0x58074, + .clkr = { + .enable_reg = 0x58074, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi_vfe1_clk", + .parent_hws = (const struct clk_hw *[]) { + &vfe1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_gp0_clk = { + .halt_reg = 0x54018, + .clkr = { + .enable_reg = 0x54018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_gp0_clk", + .parent_hws = (const struct clk_hw *[]) { + &camss_gp0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_gp1_clk = { + .halt_reg = 0x55018, + .clkr = { + .enable_reg = 0x55018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_gp1_clk", + .parent_hws = (const struct clk_hw *[]) { + &camss_gp1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_ispif_ahb_clk = { + .halt_reg = 0x50004, + .clkr = { + .enable_reg = 0x50004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_ispif_ahb_clk", + .parent_hws = (const struct clk_hw *[]) { + &camss_top_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_jpeg0_clk = { + .halt_reg = 0x57020, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x57020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_jpeg0_clk", + .parent_hws = (const struct clk_hw *[]) { + &jpeg0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_jpeg_ahb_clk = { + .halt_reg = 0x57024, + .clkr = { + .enable_reg = 0x57024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_jpeg_ahb_clk", + .parent_hws = (const struct clk_hw *[]) { + &camss_top_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_jpeg_axi_clk = { + .halt_reg = 0x57028, + .clkr = { + .enable_reg = 0x57028, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_jpeg_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_mclk0_clk = { + .halt_reg = 0x52018, + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_mclk0_clk", + .parent_hws = (const struct clk_hw *[]) { + &mclk0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_mclk1_clk = { + .halt_reg = 0x53018, + .clkr = { + .enable_reg = 0x53018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_mclk1_clk", + .parent_hws = (const struct clk_hw *[]) { + &mclk1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_mclk2_clk = { + .halt_reg = 0x5c018, + .clkr = { + .enable_reg = 0x5c018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_mclk2_clk", + .parent_hws = (const struct clk_hw *[]) { + &mclk2_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_micro_ahb_clk = { + .halt_reg = 0x5600c, + .clkr = { + .enable_reg = 0x5600c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_micro_ahb_clk", + .parent_hws = (const struct clk_hw *[]) { + &camss_top_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_csi0phytimer_clk = { + .halt_reg = 0x4e01c, + .clkr = { + .enable_reg = 0x4e01c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi0phytimer_clk", + .parent_hws = (const struct clk_hw *[]) { + &csi0phytimer_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_csi1phytimer_clk = { + .halt_reg = 0x4f01c, + .clkr = { + .enable_reg = 0x4f01c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi1phytimer_clk", + .parent_hws = (const struct clk_hw *[]) { + &csi1phytimer_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_ahb_clk = { + .halt_reg = 0x56004, + .clkr = { + .enable_reg = 0x56004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_top_ahb_clk = { + .halt_reg = 0x5a014, + .clkr = { + .enable_reg = 0x5a014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_top_ahb_clk", + .parent_hws = (const struct clk_hw *[]) { + &camss_top_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_vfe0_clk = { + .halt_reg = 0x58038, + .clkr = { + .enable_reg = 0x58038, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_vfe0_clk", + .parent_hws = (const struct clk_hw *[]) { + &vfe0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_vfe_ahb_clk = { + .halt_reg = 0x58044, + .clkr = { + .enable_reg = 0x58044, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_vfe_ahb_clk", + .parent_hws = (const struct clk_hw *[]) { + &camss_top_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_vfe_axi_clk = { + .halt_reg = 0x58048, + .clkr = { + .enable_reg = 0x58048, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_vfe_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_vfe1_ahb_clk = { + .halt_reg = 0x58060, + .clkr = { + .enable_reg = 0x58060, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_vfe1_ahb_clk", + .parent_hws = (const struct clk_hw *[]) { + &camss_top_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_vfe1_axi_clk = { + .halt_reg = 0x58068, + .clkr = { + .enable_reg = 0x58068, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_vfe1_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_vfe1_clk = { + .halt_reg = 0x5805c, + .clkr = { + .enable_reg = 0x5805c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_vfe1_clk", + .parent_hws = (const struct clk_hw *[]) { + &vfe1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_dcc_clk = { + .halt_reg = 0x77004, + .clkr = { + .enable_reg = 0x77004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_dcc_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_oxili_gmem_clk = { + .halt_reg = 0x59024, + .clkr = { + .enable_reg = 0x59024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_oxili_gmem_clk", + .parent_hws = (const struct clk_hw *[]) { + &gfx3d_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp1_clk = { + .halt_reg = 0x8000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_gp1_clk", + .parent_hws = (const struct clk_hw *[]) { + &gp1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp2_clk = { + .halt_reg = 0x9000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x9000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_gp2_clk", + .parent_hws = (const struct clk_hw *[]) { + &gp2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp3_clk = { + .halt_reg = 0xa000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xa000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_gp3_clk", + .parent_hws = (const struct clk_hw *[]) { + &gp3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_mdss_ahb_clk = { + .halt_reg = 0x4d07c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4d07c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_mdss_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_mdss_axi_clk = { + .halt_reg = 0x4d080, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4d080, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_mdss_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_mdss_byte0_clk = { + .halt_reg = 0x4d094, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4d094, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_mdss_byte0_clk", + .parent_hws = (const struct clk_hw *[]) { + &byte0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_mdss_byte1_clk = { + .halt_reg = 0x4d0a0, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4d0a0, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_mdss_byte1_clk", + .parent_hws = (const struct clk_hw *[]) { + &byte1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_mdss_esc0_clk = { + .halt_reg = 0x4d098, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4d098, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_mdss_esc0_clk", + .parent_hws = (const struct clk_hw *[]) { + &esc0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_mdss_esc1_clk = { + .halt_reg = 0x4d09c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4d09c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_mdss_esc1_clk", + .parent_hws = (const struct clk_hw *[]) { + &esc1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_mdss_mdp_clk = { + .halt_reg = 0x4d088, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4d088, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_mdss_mdp_clk", + .parent_hws = (const struct clk_hw *[]) { + &mdp_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_mdss_pclk0_clk = { + .halt_reg = 0x4d084, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4d084, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_mdss_pclk0_clk", + .parent_hws = (const struct clk_hw *[]) { + &pclk0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_mdss_pclk1_clk = { + .halt_reg = 0x4d0a4, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4d0a4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_mdss_pclk1_clk", + .parent_hws = (const struct clk_hw *[]) { + &pclk1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_mdss_vsync_clk = { + .halt_reg = 0x4d090, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4d090, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_mdss_vsync_clk", + .parent_hws = (const struct clk_hw *[]) { + &vsync_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_mss_cfg_ahb_clk = { + .halt_reg = 0x49000, + .clkr = { + .enable_reg = 0x49000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_mss_cfg_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_mss_q6_bimc_axi_clk = { + .halt_reg = 0x49004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x49004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_mss_q6_bimc_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_bimc_gfx_clk = { + .halt_reg = 0x59048, + .clkr = { + .enable_reg = 0x59048, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_bimc_gfx_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_oxili_ahb_clk = { + .halt_reg = 0x59028, + .clkr = { + .enable_reg = 0x59028, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_oxili_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_oxili_aon_clk = { + .halt_reg = 0x59044, + .clkr = { + .enable_reg = 0x59044, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_oxili_aon_clk", + .parent_hws = (const struct clk_hw *[]) { + &gfx3d_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_oxili_gfx3d_clk = { + .halt_reg = 0x59020, + .clkr = { + .enable_reg = 0x59020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_oxili_gfx3d_clk", + .parent_hws = (const struct clk_hw *[]) { + &gfx3d_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_oxili_timer_clk = { + .halt_reg = 0x59040, + .clkr = { + .enable_reg = 0x59040, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_oxili_timer_clk", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm2_clk = { + .halt_reg = 0x4400c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4400c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_pdm2_clk", + .parent_hws = (const struct clk_hw *[]) { + &pdm2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm_ahb_clk = { + .halt_reg = 0x44004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x44004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_pdm_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_rbcpr_gfx_ahb_clk = { + .halt_reg = 0x3a008, + .clkr = { + .enable_reg = 0x3a008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_rbcpr_gfx_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_rbcpr_gfx_clk = { + .halt_reg = 0x3a004, + .clkr = { + .enable_reg = 0x3a004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_rbcpr_gfx_clk", + .parent_hws = (const struct clk_hw *[]) { + &rbcpr_gfx_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc1_ahb_clk = { + .halt_reg = 0x4201c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4201c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_sdcc1_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc1_apps_clk = { + .halt_reg = 0x42018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x42018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_sdcc1_apps_clk", + .parent_hws = (const struct clk_hw *[]) { + &sdcc1_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc1_ice_core_clk = { + .halt_reg = 0x5d014, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x5d014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_sdcc1_ice_core_clk", + .parent_hws = (const struct clk_hw *[]) { + &sdcc1_ice_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc2_ahb_clk = { + .halt_reg = 0x4301c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4301c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_sdcc2_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc2_apps_clk = { + .halt_reg = 0x43018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x43018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_sdcc2_apps_clk", + .parent_hws = (const struct clk_hw *[]) { + &sdcc2_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc3_ahb_clk = { + .halt_reg = 0x3901c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3901c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_sdcc3_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc3_apps_clk = { + .halt_reg = 0x39018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x39018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_sdcc3_apps_clk", + .parent_hws = (const struct clk_hw *[]) { + &sdcc3_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb2a_phy_sleep_clk = { + .halt_reg = 0x4102c, + .clkr = { + .enable_reg = 0x4102c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_usb2a_phy_sleep_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb_hs_phy_cfg_ahb_clk = { + .halt_reg = 0x41030, + .clkr = { + .enable_reg = 0x41030, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_usb_hs_phy_cfg_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb_fs_ahb_clk = { + .halt_reg = 0x3f008, + .clkr = { + .enable_reg = 0x3f008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_usb_fs_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb_fs_ic_clk = { + .halt_reg = 0x3f030, + .clkr = { + .enable_reg = 0x3f030, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_usb_fs_ic_clk", + .parent_hws = (const struct clk_hw *[]) { + &usb_fs_ic_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb_fs_system_clk = { + .halt_reg = 0x3f004, + .clkr = { + .enable_reg = 0x3f004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_usb_fs_system_clk", + .parent_hws = (const struct clk_hw *[]) { + &usb_fs_system_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb_hs_ahb_clk = { + .halt_reg = 0x41008, + .clkr = { + .enable_reg = 0x41008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_usb_hs_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb_hs_system_clk = { + .halt_reg = 0x41004, + .clkr = { + .enable_reg = 0x41004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_usb_hs_system_clk", + .parent_hws = (const struct clk_hw *[]) { + &usb_hs_system_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_venus0_ahb_clk = { + .halt_reg = 0x4c020, + .clkr = { + .enable_reg = 0x4c020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_venus0_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_venus0_axi_clk = { + .halt_reg = 0x4c024, + .clkr = { + .enable_reg = 0x4c024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_venus0_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_venus0_core0_vcodec0_clk = { + .halt_reg = 0x4c02c, + .clkr = { + .enable_reg = 0x4c02c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_venus0_core0_vcodec0_clk", + .parent_hws = (const struct clk_hw *[]) { + &vcodec0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_venus0_core1_vcodec0_clk = { + .halt_reg = 0x4c034, + .clkr = { + .enable_reg = 0x4c034, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_venus0_core1_vcodec0_clk", + .parent_hws = (const struct clk_hw *[]) { + &vcodec0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_venus0_vcodec0_clk = { + .halt_reg = 0x4c01c, + .clkr = { + .enable_reg = 0x4c01c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_venus0_vcodec0_clk", + .parent_hws = (const struct clk_hw *[]) { + &vcodec0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* Vote clocks */ +static struct clk_branch gcc_apss_ahb_clk = { + .halt_reg = 0x4601c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x45004, + .enable_mask = BIT(14), + .hw.init = &(struct clk_init_data){ + .name = "gcc_apss_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_apss_axi_clk = { + .halt_reg = 0x46020, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x45004, + .enable_mask = BIT(13), + .hw.init = &(struct clk_init_data){ + .name = "gcc_apss_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_ahb_clk = { + .halt_reg = 0x1008, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x45004, + .enable_mask = BIT(10), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_ahb_clk = { + .halt_reg = 0xb008, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x45004, + .enable_mask = BIT(20), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp2_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_prng_ahb_clk = { + .halt_reg = 0x13004, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x45004, + .enable_mask = BIT(8), + .hw.init = &(struct clk_init_data){ + .name = "gcc_prng_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_boot_rom_ahb_clk = { + .halt_reg = 0x1300c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x45004, + .enable_mask = BIT(7), + .hw.init = &(struct clk_init_data){ + .name = "gcc_boot_rom_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_crypto_ahb_clk = { + .halt_reg = 0x16024, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x45004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_crypto_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_crypto_axi_clk = { + .halt_reg = 0x16020, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x45004, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_crypto_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_crypto_clk = { + .halt_reg = 0x1601c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x45004, + .enable_mask = BIT(2), + .hw.init = &(struct clk_init_data){ + .name = "gcc_crypto_clk", + .parent_hws = (const struct clk_hw *[]) { + &crypto_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cpp_tbu_clk = { + .halt_reg = 0x12040, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x4500c, + .enable_mask = BIT(14), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cpp_tbu_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gfx_1_tbu_clk = { + .halt_reg = 0x12098, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x4500c, + .enable_mask = BIT(19), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gfx_1_tbu_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gfx_tbu_clk = { + .halt_reg = 0x12010, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x4500c, + .enable_mask = BIT(3), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gfx_tbu_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gfx_tcu_clk = { + .halt_reg = 0x12020, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x4500c, + .enable_mask = BIT(2), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gfx_tcu_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_apss_tcu_clk = { + .halt_reg = 0x12018, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x4500c, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_apss_tcu_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gtcu_ahb_clk = { + .halt_reg = 0x12044, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x4500c, + .enable_mask = BIT(13), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gtcu_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_jpeg_tbu_clk = { + .halt_reg = 0x12034, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x4500c, + .enable_mask = BIT(10), + .hw.init = &(struct clk_init_data){ + .name = "gcc_jpeg_tbu_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_mdp_rt_tbu_clk = { + .halt_reg = 0x1204c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x4500c, + .enable_mask = BIT(15), + .hw.init = &(struct clk_init_data){ + .name = "gcc_mdp_rt_tbu_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_mdp_tbu_clk = { + .halt_reg = 0x1201c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x4500c, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gcc_mdp_tbu_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_smmu_cfg_clk = { + .halt_reg = 0x12038, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x4500c, + .enable_mask = BIT(12), + .hw.init = &(struct clk_init_data){ + .name = "gcc_smmu_cfg_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_venus_1_tbu_clk = { + .halt_reg = 0x1209c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x4500c, + .enable_mask = BIT(20), + .hw.init = &(struct clk_init_data){ + .name = "gcc_venus_1_tbu_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_venus_tbu_clk = { + .halt_reg = 0x12014, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x4500c, + .enable_mask = BIT(5), + .hw.init = &(struct clk_init_data){ + .name = "gcc_venus_tbu_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_vfe1_tbu_clk = { + .halt_reg = 0x12090, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x4500c, + .enable_mask = BIT(17), + .hw.init = &(struct clk_init_data){ + .name = "gcc_vfe1_tbu_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_vfe_tbu_clk = { + .halt_reg = 0x1203c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x4500c, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gcc_vfe_tbu_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct gdsc venus_gdsc = { + .gdscr = 0x4c018, + .cxcs = (unsigned int []){ 0x4c024, 0x4c01c }, + .cxc_count = 2, + .pd = { + .name = "venus_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc venus_core0_gdsc = { + .gdscr = 0x4c028, + .cxcs = (unsigned int []){ 0x4c02c }, + .cxc_count = 1, + .pd = { + .name = "venus_core0_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc venus_core1_gdsc = { + .gdscr = 0x4c030, + .pd = { + .name = "venus_core1_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc mdss_gdsc = { + .gdscr = 0x4d078, + .cxcs = (unsigned int []){ 0x4d080, 0x4d088 }, + .cxc_count = 2, + .pd = { + .name = "mdss_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc jpeg_gdsc = { + .gdscr = 0x5701c, + .cxcs = (unsigned int []){ 0x57020, 0x57028 }, + .cxc_count = 2, + .pd = { + .name = "jpeg_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc vfe0_gdsc = { + .gdscr = 0x58034, + .cxcs = (unsigned int []){ 0x58038, 0x58048, 0x5600c, 0x58050 }, + .cxc_count = 4, + .pd = { + .name = "vfe0_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc vfe1_gdsc = { + .gdscr = 0x5806c, + .cxcs = (unsigned int []){ 0x5805c, 0x58068, 0x5600c, 0x58074 }, + .cxc_count = 4, + .pd = { + .name = "vfe1_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc cpp_gdsc = { + .gdscr = 0x58078, + .cxcs = (unsigned int []){ 0x5803c, 0x58064 }, + .cxc_count = 2, + .pd = { + .name = "cpp_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc oxili_cx_gdsc = { + .gdscr = 0x5904c, + .cxcs = (unsigned int []){ 0x59020 }, + .cxc_count = 1, + .pd = { + .name = "oxili_cx_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc oxili_gx_gdsc = { + .gdscr = 0x5901c, + .clamp_io_ctrl = 0x5b00c, + .cxcs = (unsigned int []){ 0x59000, 0x59024 }, + .cxc_count = 2, + .pd = { + .name = "oxili_gx_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .supply = "vdd_gfx", + .flags = CLAMP_IO, +}; + +static struct clk_regmap *gcc_msm8976_clocks[] = { + [GPLL0] = &gpll0.clkr, + [GPLL2] = &gpll2.clkr, + [GPLL3] = &gpll3.clkr, + [GPLL4] = &gpll4.clkr, + [GPLL6] = &gpll6.clkr, + [GPLL0_CLK_SRC] = &gpll0_vote, + [GPLL2_CLK_SRC] = &gpll2_vote, + [GPLL3_CLK_SRC] = &gpll3_vote, + [GPLL4_CLK_SRC] = &gpll4_vote, + [GPLL6_CLK_SRC] = &gpll6_vote, + [GCC_BLSP1_QUP1_SPI_APPS_CLK] = &gcc_blsp1_qup1_spi_apps_clk.clkr, + [GCC_BLSP1_QUP1_I2C_APPS_CLK] = &gcc_blsp1_qup1_i2c_apps_clk.clkr, + [GCC_BLSP1_QUP2_I2C_APPS_CLK] = &gcc_blsp1_qup2_i2c_apps_clk.clkr, + [GCC_BLSP1_QUP2_SPI_APPS_CLK] = &gcc_blsp1_qup2_spi_apps_clk.clkr, + [GCC_BLSP1_QUP3_I2C_APPS_CLK] = &gcc_blsp1_qup3_i2c_apps_clk.clkr, + [GCC_BLSP1_QUP3_SPI_APPS_CLK] = &gcc_blsp1_qup3_spi_apps_clk.clkr, + [GCC_BLSP1_QUP4_I2C_APPS_CLK] = &gcc_blsp1_qup4_i2c_apps_clk.clkr, + [GCC_BLSP1_QUP4_SPI_APPS_CLK] = &gcc_blsp1_qup4_spi_apps_clk.clkr, + [GCC_BLSP1_UART1_APPS_CLK] = &gcc_blsp1_uart1_apps_clk.clkr, + [GCC_BLSP1_UART2_APPS_CLK] = &gcc_blsp1_uart2_apps_clk.clkr, + [GCC_BLSP2_QUP1_I2C_APPS_CLK] = &gcc_blsp2_qup1_i2c_apps_clk.clkr, + [GCC_BLSP2_QUP1_SPI_APPS_CLK] = &gcc_blsp2_qup1_spi_apps_clk.clkr, + [GCC_BLSP2_QUP2_I2C_APPS_CLK] = &gcc_blsp2_qup2_i2c_apps_clk.clkr, + [GCC_BLSP2_QUP2_SPI_APPS_CLK] = &gcc_blsp2_qup2_spi_apps_clk.clkr, + [GCC_BLSP2_QUP3_I2C_APPS_CLK] = &gcc_blsp2_qup3_i2c_apps_clk.clkr, + [GCC_BLSP2_QUP3_SPI_APPS_CLK] = &gcc_blsp2_qup3_spi_apps_clk.clkr, + [GCC_BLSP2_QUP4_I2C_APPS_CLK] = &gcc_blsp2_qup4_i2c_apps_clk.clkr, + [GCC_BLSP2_QUP4_SPI_APPS_CLK] = &gcc_blsp2_qup4_spi_apps_clk.clkr, + [GCC_BLSP2_UART1_APPS_CLK] = &gcc_blsp2_uart1_apps_clk.clkr, + [GCC_BLSP2_UART2_APPS_CLK] = &gcc_blsp2_uart2_apps_clk.clkr, + [GCC_CAMSS_CCI_AHB_CLK] = &gcc_camss_cci_ahb_clk.clkr, + [GCC_CAMSS_CCI_CLK] = &gcc_camss_cci_clk.clkr, + [GCC_CAMSS_CPP_AHB_CLK] = &gcc_camss_cpp_ahb_clk.clkr, + [GCC_CAMSS_CPP_AXI_CLK] = &gcc_camss_cpp_axi_clk.clkr, + [GCC_CAMSS_CPP_CLK] = &gcc_camss_cpp_clk.clkr, + [GCC_CAMSS_CSI0_AHB_CLK] = &gcc_camss_csi0_ahb_clk.clkr, + [GCC_CAMSS_CSI0_CLK] = &gcc_camss_csi0_clk.clkr, + [GCC_CAMSS_CSI0PHY_CLK] = &gcc_camss_csi0phy_clk.clkr, + [GCC_CAMSS_CSI0PIX_CLK] = &gcc_camss_csi0pix_clk.clkr, + [GCC_CAMSS_CSI0RDI_CLK] = &gcc_camss_csi0rdi_clk.clkr, + [GCC_CAMSS_CSI1_AHB_CLK] = &gcc_camss_csi1_ahb_clk.clkr, + [GCC_CAMSS_CSI1_CLK] = &gcc_camss_csi1_clk.clkr, + [GCC_CAMSS_CSI1PHY_CLK] = &gcc_camss_csi1phy_clk.clkr, + [GCC_CAMSS_CSI1PIX_CLK] = &gcc_camss_csi1pix_clk.clkr, + [GCC_CAMSS_CSI1RDI_CLK] = &gcc_camss_csi1rdi_clk.clkr, + [GCC_CAMSS_CSI2_AHB_CLK] = &gcc_camss_csi2_ahb_clk.clkr, + [GCC_CAMSS_CSI2_CLK] = &gcc_camss_csi2_clk.clkr, + [GCC_CAMSS_CSI2PHY_CLK] = &gcc_camss_csi2phy_clk.clkr, + [GCC_CAMSS_CSI2PIX_CLK] = &gcc_camss_csi2pix_clk.clkr, + [GCC_CAMSS_CSI2RDI_CLK] = &gcc_camss_csi2rdi_clk.clkr, + [GCC_CAMSS_CSI_VFE0_CLK] = &gcc_camss_csi_vfe0_clk.clkr, + [GCC_CAMSS_CSI_VFE1_CLK] = &gcc_camss_csi_vfe1_clk.clkr, + [GCC_CAMSS_GP0_CLK] = &gcc_camss_gp0_clk.clkr, + [GCC_CAMSS_GP1_CLK] = &gcc_camss_gp1_clk.clkr, + [GCC_CAMSS_ISPIF_AHB_CLK] = &gcc_camss_ispif_ahb_clk.clkr, + [GCC_CAMSS_JPEG0_CLK] = &gcc_camss_jpeg0_clk.clkr, + [GCC_CAMSS_JPEG_AHB_CLK] = &gcc_camss_jpeg_ahb_clk.clkr, + [GCC_CAMSS_JPEG_AXI_CLK] = &gcc_camss_jpeg_axi_clk.clkr, + [GCC_CAMSS_MCLK0_CLK] = &gcc_camss_mclk0_clk.clkr, + [GCC_CAMSS_MCLK1_CLK] = &gcc_camss_mclk1_clk.clkr, + [GCC_CAMSS_MCLK2_CLK] = &gcc_camss_mclk2_clk.clkr, + [GCC_CAMSS_MICRO_AHB_CLK] = &gcc_camss_micro_ahb_clk.clkr, + [GCC_CAMSS_CSI0PHYTIMER_CLK] = &gcc_camss_csi0phytimer_clk.clkr, + [GCC_CAMSS_CSI1PHYTIMER_CLK] = &gcc_camss_csi1phytimer_clk.clkr, + [GCC_CAMSS_AHB_CLK] = &gcc_camss_ahb_clk.clkr, + [GCC_CAMSS_TOP_AHB_CLK] = &gcc_camss_top_ahb_clk.clkr, + [GCC_CAMSS_VFE0_CLK] = &gcc_camss_vfe0_clk.clkr, + [GCC_CAMSS_VFE_AHB_CLK] = &gcc_camss_vfe_ahb_clk.clkr, + [GCC_CAMSS_VFE_AXI_CLK] = &gcc_camss_vfe_axi_clk.clkr, + [GCC_CAMSS_VFE1_AHB_CLK] = &gcc_camss_vfe1_ahb_clk.clkr, + [GCC_CAMSS_VFE1_AXI_CLK] = &gcc_camss_vfe1_axi_clk.clkr, + [GCC_CAMSS_VFE1_CLK] = &gcc_camss_vfe1_clk.clkr, + [GCC_DCC_CLK] = &gcc_dcc_clk.clkr, + [GCC_GP1_CLK] = &gcc_gp1_clk.clkr, + [GCC_GP2_CLK] = &gcc_gp2_clk.clkr, + [GCC_GP3_CLK] = &gcc_gp3_clk.clkr, + [GCC_MDSS_AHB_CLK] = &gcc_mdss_ahb_clk.clkr, + [GCC_MDSS_AXI_CLK] = &gcc_mdss_axi_clk.clkr, + [GCC_MDSS_ESC0_CLK] = &gcc_mdss_esc0_clk.clkr, + [GCC_MDSS_ESC1_CLK] = &gcc_mdss_esc1_clk.clkr, + [GCC_MDSS_MDP_CLK] = &gcc_mdss_mdp_clk.clkr, + [GCC_MDSS_VSYNC_CLK] = &gcc_mdss_vsync_clk.clkr, + [GCC_MSS_CFG_AHB_CLK] = &gcc_mss_cfg_ahb_clk.clkr, + [GCC_MSS_Q6_BIMC_AXI_CLK] = &gcc_mss_q6_bimc_axi_clk.clkr, + [GCC_PDM2_CLK] = &gcc_pdm2_clk.clkr, + [GCC_PRNG_AHB_CLK] = &gcc_prng_ahb_clk.clkr, + [GCC_PDM_AHB_CLK] = &gcc_pdm_ahb_clk.clkr, + [GCC_RBCPR_GFX_AHB_CLK] = &gcc_rbcpr_gfx_ahb_clk.clkr, + [GCC_RBCPR_GFX_CLK] = &gcc_rbcpr_gfx_clk.clkr, + [GCC_SDCC1_AHB_CLK] = &gcc_sdcc1_ahb_clk.clkr, + [GCC_SDCC1_APPS_CLK] = &gcc_sdcc1_apps_clk.clkr, + [GCC_SDCC1_ICE_CORE_CLK] = &gcc_sdcc1_ice_core_clk.clkr, + [GCC_SDCC2_AHB_CLK] = &gcc_sdcc2_ahb_clk.clkr, + [GCC_SDCC2_APPS_CLK] = &gcc_sdcc2_apps_clk.clkr, + [GCC_SDCC3_AHB_CLK] = &gcc_sdcc3_ahb_clk.clkr, + [GCC_SDCC3_APPS_CLK] = &gcc_sdcc3_apps_clk.clkr, + [GCC_USB2A_PHY_SLEEP_CLK] = &gcc_usb2a_phy_sleep_clk.clkr, + [GCC_USB_HS_PHY_CFG_AHB_CLK] = &gcc_usb_hs_phy_cfg_ahb_clk.clkr, + [GCC_USB_FS_AHB_CLK] = &gcc_usb_fs_ahb_clk.clkr, + [GCC_USB_FS_IC_CLK] = &gcc_usb_fs_ic_clk.clkr, + [GCC_USB_FS_SYSTEM_CLK] = &gcc_usb_fs_system_clk.clkr, + [GCC_USB_HS_AHB_CLK] = &gcc_usb_hs_ahb_clk.clkr, + [GCC_USB_HS_SYSTEM_CLK] = &gcc_usb_hs_system_clk.clkr, + [GCC_VENUS0_AHB_CLK] = &gcc_venus0_ahb_clk.clkr, + [GCC_VENUS0_AXI_CLK] = &gcc_venus0_axi_clk.clkr, + [GCC_VENUS0_CORE0_VCODEC0_CLK] = &gcc_venus0_core0_vcodec0_clk.clkr, + [GCC_VENUS0_CORE1_VCODEC0_CLK] = &gcc_venus0_core1_vcodec0_clk.clkr, + [GCC_VENUS0_VCODEC0_CLK] = &gcc_venus0_vcodec0_clk.clkr, + [GCC_APSS_AHB_CLK] = &gcc_apss_ahb_clk.clkr, + [GCC_APSS_AXI_CLK] = &gcc_apss_axi_clk.clkr, + [GCC_BLSP1_AHB_CLK] = &gcc_blsp1_ahb_clk.clkr, + [GCC_BLSP2_AHB_CLK] = &gcc_blsp2_ahb_clk.clkr, + [GCC_BOOT_ROM_AHB_CLK] = &gcc_boot_rom_ahb_clk.clkr, + [GCC_CRYPTO_AHB_CLK] = &gcc_crypto_ahb_clk.clkr, + [GCC_CRYPTO_AXI_CLK] = &gcc_crypto_axi_clk.clkr, + [GCC_CRYPTO_CLK] = &gcc_crypto_clk.clkr, + [GCC_CPP_TBU_CLK] = &gcc_cpp_tbu_clk.clkr, + [GCC_APSS_TCU_CLK] = &gcc_apss_tcu_clk.clkr, + [GCC_JPEG_TBU_CLK] = &gcc_jpeg_tbu_clk.clkr, + [GCC_MDP_RT_TBU_CLK] = &gcc_mdp_rt_tbu_clk.clkr, + [GCC_MDP_TBU_CLK] = &gcc_mdp_tbu_clk.clkr, + [GCC_SMMU_CFG_CLK] = &gcc_smmu_cfg_clk.clkr, + [GCC_VENUS_1_TBU_CLK] = &gcc_venus_1_tbu_clk.clkr, + [GCC_VENUS_TBU_CLK] = &gcc_venus_tbu_clk.clkr, + [GCC_VFE1_TBU_CLK] = &gcc_vfe1_tbu_clk.clkr, + [GCC_VFE_TBU_CLK] = &gcc_vfe_tbu_clk.clkr, + [GCC_APS_0_CLK] = &gcc_aps_0_clk.clkr, + [GCC_APS_1_CLK] = &gcc_aps_1_clk.clkr, + [APS_0_CLK_SRC] = &aps_0_clk_src.clkr, + [APS_1_CLK_SRC] = &aps_1_clk_src.clkr, + [APSS_AHB_CLK_SRC] = &apss_ahb_clk_src.clkr, + [BLSP1_QUP1_I2C_APPS_CLK_SRC] = &blsp1_qup1_i2c_apps_clk_src.clkr, + [BLSP1_QUP1_SPI_APPS_CLK_SRC] = &blsp1_qup1_spi_apps_clk_src.clkr, + [BLSP1_QUP2_I2C_APPS_CLK_SRC] = &blsp1_qup2_i2c_apps_clk_src.clkr, + [BLSP1_QUP2_SPI_APPS_CLK_SRC] = &blsp1_qup2_spi_apps_clk_src.clkr, + [BLSP1_QUP3_I2C_APPS_CLK_SRC] = &blsp1_qup3_i2c_apps_clk_src.clkr, + [BLSP1_QUP3_SPI_APPS_CLK_SRC] = &blsp1_qup3_spi_apps_clk_src.clkr, + [BLSP1_QUP4_I2C_APPS_CLK_SRC] = &blsp1_qup4_i2c_apps_clk_src.clkr, + [BLSP1_QUP4_SPI_APPS_CLK_SRC] = &blsp1_qup4_spi_apps_clk_src.clkr, + [BLSP1_UART1_APPS_CLK_SRC] = &blsp1_uart1_apps_clk_src.clkr, + [BLSP1_UART2_APPS_CLK_SRC] = &blsp1_uart2_apps_clk_src.clkr, + [BLSP2_QUP1_I2C_APPS_CLK_SRC] = &blsp2_qup1_i2c_apps_clk_src.clkr, + [BLSP2_QUP1_SPI_APPS_CLK_SRC] = &blsp2_qup1_spi_apps_clk_src.clkr, + [BLSP2_QUP2_I2C_APPS_CLK_SRC] = &blsp2_qup2_i2c_apps_clk_src.clkr, + [BLSP2_QUP2_SPI_APPS_CLK_SRC] = &blsp2_qup2_spi_apps_clk_src.clkr, + [BLSP2_QUP3_I2C_APPS_CLK_SRC] = &blsp2_qup3_i2c_apps_clk_src.clkr, + [BLSP2_QUP3_SPI_APPS_CLK_SRC] = &blsp2_qup3_spi_apps_clk_src.clkr, + [BLSP2_QUP4_I2C_APPS_CLK_SRC] = &blsp2_qup4_i2c_apps_clk_src.clkr, + [BLSP2_QUP4_SPI_APPS_CLK_SRC] = &blsp2_qup4_spi_apps_clk_src.clkr, + [BLSP2_UART1_APPS_CLK_SRC] = &blsp2_uart1_apps_clk_src.clkr, + [BLSP2_UART2_APPS_CLK_SRC] = &blsp2_uart2_apps_clk_src.clkr, + [CCI_CLK_SRC] = &cci_clk_src.clkr, + [CPP_CLK_SRC] = &cpp_clk_src.clkr, + [CSI0_CLK_SRC] = &csi0_clk_src.clkr, + [CSI1_CLK_SRC] = &csi1_clk_src.clkr, + [CSI2_CLK_SRC] = &csi2_clk_src.clkr, + [CAMSS_GP0_CLK_SRC] = &camss_gp0_clk_src.clkr, + [CAMSS_GP1_CLK_SRC] = &camss_gp1_clk_src.clkr, + [JPEG0_CLK_SRC] = &jpeg0_clk_src.clkr, + [MCLK0_CLK_SRC] = &mclk0_clk_src.clkr, + [MCLK1_CLK_SRC] = &mclk1_clk_src.clkr, + [MCLK2_CLK_SRC] = &mclk2_clk_src.clkr, + [CSI0PHYTIMER_CLK_SRC] = &csi0phytimer_clk_src.clkr, + [CSI1PHYTIMER_CLK_SRC] = &csi1phytimer_clk_src.clkr, + [CAMSS_TOP_AHB_CLK_SRC] = &camss_top_ahb_clk_src.clkr, + [VFE0_CLK_SRC] = &vfe0_clk_src.clkr, + [VFE1_CLK_SRC] = &vfe1_clk_src.clkr, + [CRYPTO_CLK_SRC] = &crypto_clk_src.clkr, + [GP1_CLK_SRC] = &gp1_clk_src.clkr, + [GP2_CLK_SRC] = &gp2_clk_src.clkr, + [GP3_CLK_SRC] = &gp3_clk_src.clkr, + [ESC0_CLK_SRC] = &esc0_clk_src.clkr, + [ESC1_CLK_SRC] = &esc1_clk_src.clkr, + [MDP_CLK_SRC] = &mdp_clk_src.clkr, + [VSYNC_CLK_SRC] = &vsync_clk_src.clkr, + [PDM2_CLK_SRC] = &pdm2_clk_src.clkr, + [RBCPR_GFX_CLK_SRC] = &rbcpr_gfx_clk_src.clkr, + [SDCC1_APPS_CLK_SRC] = &sdcc1_apps_clk_src.clkr, + [SDCC1_ICE_CORE_CLK_SRC] = &sdcc1_ice_core_clk_src.clkr, + [SDCC2_APPS_CLK_SRC] = &sdcc2_apps_clk_src.clkr, + [SDCC3_APPS_CLK_SRC] = &sdcc3_apps_clk_src.clkr, + [USB_FS_IC_CLK_SRC] = &usb_fs_ic_clk_src.clkr, + [USB_FS_SYSTEM_CLK_SRC] = &usb_fs_system_clk_src.clkr, + [USB_HS_SYSTEM_CLK_SRC] = &usb_hs_system_clk_src.clkr, + [VCODEC0_CLK_SRC] = &vcodec0_clk_src.clkr, + [GCC_MDSS_BYTE0_CLK_SRC] = &byte0_clk_src.clkr, + [GCC_MDSS_BYTE1_CLK_SRC] = &byte1_clk_src.clkr, + [GCC_MDSS_BYTE0_CLK] = &gcc_mdss_byte0_clk.clkr, + [GCC_MDSS_BYTE1_CLK] = &gcc_mdss_byte1_clk.clkr, + [GCC_MDSS_PCLK0_CLK_SRC] = &pclk0_clk_src.clkr, + [GCC_MDSS_PCLK1_CLK_SRC] = &pclk1_clk_src.clkr, + [GCC_MDSS_PCLK0_CLK] = &gcc_mdss_pclk0_clk.clkr, + [GCC_MDSS_PCLK1_CLK] = &gcc_mdss_pclk1_clk.clkr, + [GCC_GFX3D_CLK_SRC] = &gfx3d_clk_src.clkr, + [GCC_GFX3D_OXILI_CLK] = &gcc_oxili_gfx3d_clk.clkr, + [GCC_GFX3D_BIMC_CLK] = &gcc_bimc_gfx_clk.clkr, + [GCC_GFX3D_OXILI_AHB_CLK] = &gcc_oxili_ahb_clk.clkr, + [GCC_GFX3D_OXILI_AON_CLK] = &gcc_oxili_aon_clk.clkr, + [GCC_GFX3D_OXILI_GMEM_CLK] = &gcc_oxili_gmem_clk.clkr, + [GCC_GFX3D_OXILI_TIMER_CLK] = &gcc_oxili_timer_clk.clkr, + [GCC_GFX3D_TBU0_CLK] = &gcc_gfx_tbu_clk.clkr, + [GCC_GFX3D_TBU1_CLK] = &gcc_gfx_1_tbu_clk.clkr, + [GCC_GFX3D_TCU_CLK] = &gcc_gfx_tcu_clk.clkr, + [GCC_GFX3D_GTCU_AHB_CLK] = &gcc_gtcu_ahb_clk.clkr, +}; + +static const struct qcom_reset_map gcc_msm8976_resets[] = { + [RST_CAMSS_MICRO_BCR] = { 0x56008 }, + [RST_USB_HS_BCR] = { 0x41000 }, + [RST_QUSB2_PHY_BCR] = { 0x4103c }, + [RST_USB2_HS_PHY_ONLY_BCR] = { 0x41034 }, + [RST_USB_HS_PHY_CFG_AHB_BCR] = { 0x41038 }, + [RST_USB_FS_BCR] = { 0x3f000 }, + [RST_CAMSS_CSI1PIX_BCR] = { 0x4f054 }, + [RST_CAMSS_CSI_VFE1_BCR] = { 0x58070 }, + [RST_CAMSS_VFE1_BCR] = { 0x5807c }, + [RST_CAMSS_CPP_BCR] = { 0x58080 }, +}; + +static struct gdsc *gcc_msm8976_gdscs[] = { + [VENUS_GDSC] = &venus_gdsc, + [VENUS_CORE0_GDSC] = &venus_core0_gdsc, + [VENUS_CORE1_GDSC] = &venus_core1_gdsc, + [MDSS_GDSC] = &mdss_gdsc, + [JPEG_GDSC] = &jpeg_gdsc, + [VFE0_GDSC] = &vfe0_gdsc, + [VFE1_GDSC] = &vfe1_gdsc, + [CPP_GDSC] = &cpp_gdsc, + [OXILI_GX_GDSC] = &oxili_gx_gdsc, + [OXILI_CX_GDSC] = &oxili_cx_gdsc, +}; + +static const struct regmap_config gcc_msm8976_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x7fffc, + .fast_io = true, +}; + +static const struct qcom_cc_desc gcc_msm8976_desc = { + .config = &gcc_msm8976_regmap_config, + .clks = gcc_msm8976_clocks, + .num_clks = ARRAY_SIZE(gcc_msm8976_clocks), + .resets = gcc_msm8976_resets, + .num_resets = ARRAY_SIZE(gcc_msm8976_resets), + .gdscs = gcc_msm8976_gdscs, + .num_gdscs = ARRAY_SIZE(gcc_msm8976_gdscs), +}; + +static const struct of_device_id gcc_msm8976_match_table[] = { + { .compatible = "qcom,gcc-msm8976" }, /* Also valid for 8x56 */ + { .compatible = "qcom,gcc-msm8976-v1.1" }, + { } +}; +MODULE_DEVICE_TABLE(of, gcc_msm8976_match_table); + +static int gcc_msm8976_probe(struct platform_device *pdev) +{ + struct regmap *regmap; + int ret; + + if (of_device_is_compatible(pdev->dev.of_node, "qcom,gcc-msm8976-v1.1")) { + sdcc1_apps_clk_src.parent_map = gcc_parent_map_v1_1; + sdcc1_apps_clk_src.freq_tbl = ftbl_sdcc1_8976_v1_1_apps_clk_src; + sdcc1_apps_clk_src.clkr.hw.init = &sdcc1_apps_clk_src_8976v1_1_init; + } + + regmap = qcom_cc_map(pdev, &gcc_msm8976_desc); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + /* Set Sleep and Wakeup cycles to 0 for GMEM clock */ + ret = regmap_update_bits(regmap, gcc_oxili_gmem_clk.clkr.enable_reg, 0xff0, 0); + if (ret) + return ret; + + clk_pll_configure_sr_hpm_lp(&gpll3, regmap, &gpll3_config, true); + + /* Enable AUX2 clock for APSS */ + ret = regmap_update_bits(regmap, 0x60000, BIT(2), BIT(2)); + if (ret) + return ret; + + /* Set Sleep cycles to 0 for OXILI clock */ + ret = regmap_update_bits(regmap, gcc_oxili_gfx3d_clk.clkr.enable_reg, 0xf0, 0); + if (ret) + return ret; + + return qcom_cc_really_probe(pdev, &gcc_msm8976_desc, regmap); +} + +static struct platform_driver gcc_msm8976_driver = { + .probe = gcc_msm8976_probe, + .driver = { + .name = "qcom,gcc-msm8976", + .of_match_table = gcc_msm8976_match_table, + }, +}; + +static int __init gcc_msm8976_init(void) +{ + return platform_driver_register(&gcc_msm8976_driver); +} +core_initcall(gcc_msm8976_init); + +static void __exit gcc_msm8976_exit(void) +{ + platform_driver_unregister(&gcc_msm8976_driver); +} +module_exit(gcc_msm8976_exit); + +MODULE_AUTHOR("AngeloGioacchino Del Regno "); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/qcom/gcc-msm8994.c b/drivers/clk/qcom/gcc-msm8994.c index 702a9bdc0559..71aa630fa4bd 100644 --- a/drivers/clk/qcom/gcc-msm8994.c +++ b/drivers/clk/qcom/gcc-msm8994.c @@ -2,6 +2,7 @@ /* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved. */ +#include #include #include #include diff --git a/drivers/clk/qcom/gcc-sc7280.c b/drivers/clk/qcom/gcc-sc7280.c index 8fb6bd69f240..423627d49719 100644 --- a/drivers/clk/qcom/gcc-sc7280.c +++ b/drivers/clk/qcom/gcc-sc7280.c @@ -2917,7 +2917,7 @@ static struct clk_branch gcc_cfg_noc_lpass_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_cfg_noc_lpass_clk", - .ops = &clk_branch2_ops, + .ops = &clk_branch2_aon_ops, }, }, }; diff --git a/drivers/clk/qcom/gcc-sdx65.c b/drivers/clk/qcom/gcc-sdx65.c new file mode 100644 index 000000000000..748ac15b5ed8 --- /dev/null +++ b/drivers/clk/qcom/gcc-sdx65.c @@ -0,0 +1,1611 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021, Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "clk-alpha-pll.h" +#include "clk-branch.h" +#include "clk-rcg.h" +#include "clk-regmap.h" +#include "clk-regmap-divider.h" +#include "clk-regmap-mux.h" +#include "common.h" +#include "gdsc.h" +#include "reset.h" + +enum { + P_BI_TCXO, + P_GPLL0_OUT_EVEN, + P_GPLL0_OUT_MAIN, + P_PCIE_PIPE_CLK, + P_SLEEP_CLK, + P_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK, +}; + +static struct clk_alpha_pll gpll0 = { + .offset = 0x0, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_EVO], + .clkr = { + .enable_reg = 0x6d000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gpll0", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_lucid_evo_ops, + }, + }, +}; + +static const struct clk_div_table post_div_table_gpll0_out_even[] = { + { 0x1, 2 }, + { } +}; + +static struct clk_alpha_pll_postdiv gpll0_out_even = { + .offset = 0x0, + .post_div_shift = 10, + .post_div_table = post_div_table_gpll0_out_even, + .num_post_div = ARRAY_SIZE(post_div_table_gpll0_out_even), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_EVO], + .clkr.hw.init = &(struct clk_init_data){ + .name = "gpll0_out_even", + .parent_hws = (const struct clk_hw *[]){ &gpll0.clkr.hw }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_lucid_evo_ops, + }, +}; + +static const struct parent_map gcc_parent_map_0[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_0[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll0_out_even.clkr.hw }, +}; + +static const struct clk_parent_data gcc_parent_data_0_ao[] = { + { .fw_name = "bi_tcxo_ao" }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_2[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_SLEEP_CLK, 5 }, + { P_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_2[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &gpll0.clkr.hw }, + { .fw_name = "sleep_clk" }, + { .hw = &gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_3[] = { + { P_BI_TCXO, 0 }, + { P_SLEEP_CLK, 5 }, +}; + +static const struct clk_parent_data gcc_parent_data_3[] = { + { .fw_name = "bi_tcxo" }, + { .fw_name = "sleep_clk" }, +}; + +static const struct parent_map gcc_parent_map_4[] = { + { P_BI_TCXO, 2 }, +}; + +static const struct parent_map gcc_parent_map_5[] = { + { P_PCIE_PIPE_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_5[] = { + { .fw_name = "pcie_pipe_clk"}, + { .fw_name = "bi_tcxo"}, +}; + +static const struct parent_map gcc_parent_map_6[] = { + { P_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_6[] = { + { .fw_name = "usb3_phy_wrapper_gcc_usb30_pipe_clk"}, + { .fw_name = "bi_tcxo"}, +}; + +static struct clk_regmap_mux gcc_pcie_aux_clk_src = { + .reg = 0x43060, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_4, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_aux_clk_src", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_pcie_pipe_clk_src = { + .reg = 0x43044, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_5, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_pipe_clk_src", + .parent_data = gcc_parent_data_5, + .num_parents = 2, + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_usb3_phy_pipe_clk_src = { + .reg = 0x1706c, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_6, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_phy_pipe_clk_src", + .parent_data = gcc_parent_data_6, + .num_parents = 2, + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static const struct freq_tbl ftbl_gcc_blsp1_qup1_i2c_apps_clk_src[] = { + F(9600000, P_BI_TCXO, 2, 0, 0), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(50000000, P_GPLL0_OUT_MAIN, 12, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_blsp1_qup1_i2c_apps_clk_src = { + .cmd_rcgr = 0x1c024, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_blsp1_qup1_i2c_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup1_i2c_apps_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_blsp1_qup1_spi_apps_clk_src[] = { + F(960000, P_BI_TCXO, 10, 1, 2), + F(4800000, P_BI_TCXO, 4, 0, 0), + F(9600000, P_BI_TCXO, 2, 0, 0), + F(15000000, P_GPLL0_OUT_EVEN, 5, 1, 4), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(24000000, P_GPLL0_OUT_MAIN, 12.5, 1, 2), + F(25000000, P_GPLL0_OUT_MAIN, 12, 1, 2), + F(50000000, P_GPLL0_OUT_MAIN, 12, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_blsp1_qup1_spi_apps_clk_src = { + .cmd_rcgr = 0x1c00c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_blsp1_qup1_spi_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup1_spi_apps_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_blsp1_qup2_i2c_apps_clk_src = { + .cmd_rcgr = 0x1e024, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_blsp1_qup1_i2c_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup2_i2c_apps_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_blsp1_qup2_spi_apps_clk_src = { + .cmd_rcgr = 0x1e00c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_blsp1_qup1_spi_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup2_spi_apps_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_blsp1_qup3_i2c_apps_clk_src = { + .cmd_rcgr = 0x20024, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_blsp1_qup1_i2c_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup3_i2c_apps_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_blsp1_qup3_spi_apps_clk_src = { + .cmd_rcgr = 0x2000c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_blsp1_qup1_spi_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup3_spi_apps_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_blsp1_qup4_i2c_apps_clk_src = { + .cmd_rcgr = 0x22024, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_blsp1_qup1_i2c_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup4_i2c_apps_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_blsp1_qup4_spi_apps_clk_src = { + .cmd_rcgr = 0x2200c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_blsp1_qup1_spi_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup4_spi_apps_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_blsp1_uart1_apps_clk_src[] = { + F(3686400, P_GPLL0_OUT_EVEN, 1, 192, 15625), + F(7372800, P_GPLL0_OUT_EVEN, 1, 384, 15625), + F(9600000, P_BI_TCXO, 2, 0, 0), + F(14745600, P_GPLL0_OUT_EVEN, 1, 768, 15625), + F(16000000, P_GPLL0_OUT_EVEN, 1, 4, 75), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(19354839, P_GPLL0_OUT_MAIN, 15.5, 1, 2), + F(20000000, P_GPLL0_OUT_MAIN, 15, 1, 2), + F(20689655, P_GPLL0_OUT_MAIN, 14.5, 1, 2), + F(21428571, P_GPLL0_OUT_MAIN, 14, 1, 2), + F(22222222, P_GPLL0_OUT_MAIN, 13.5, 1, 2), + F(23076923, P_GPLL0_OUT_MAIN, 13, 1, 2), + F(24000000, P_GPLL0_OUT_MAIN, 5, 1, 5), + F(25000000, P_GPLL0_OUT_MAIN, 12, 1, 2), + F(26086957, P_GPLL0_OUT_MAIN, 11.5, 1, 2), + F(27272727, P_GPLL0_OUT_MAIN, 11, 1, 2), + F(28571429, P_GPLL0_OUT_MAIN, 10.5, 1, 2), + F(32000000, P_GPLL0_OUT_MAIN, 1, 4, 75), + F(40000000, P_GPLL0_OUT_MAIN, 15, 0, 0), + F(46400000, P_GPLL0_OUT_MAIN, 1, 29, 375), + F(48000000, P_GPLL0_OUT_MAIN, 12.5, 0, 0), + F(51200000, P_GPLL0_OUT_MAIN, 1, 32, 375), + F(56000000, P_GPLL0_OUT_MAIN, 1, 7, 75), + F(58982400, P_GPLL0_OUT_MAIN, 1, 1536, 15625), + F(60000000, P_GPLL0_OUT_MAIN, 10, 0, 0), + F(63157895, P_GPLL0_OUT_MAIN, 9.5, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_blsp1_uart1_apps_clk_src = { + .cmd_rcgr = 0x1d00c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_blsp1_uart1_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_uart1_apps_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_blsp1_uart2_apps_clk_src = { + .cmd_rcgr = 0x1f00c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_blsp1_uart1_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_uart2_apps_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_blsp1_uart3_apps_clk_src = { + .cmd_rcgr = 0x2100c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_blsp1_uart1_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_uart3_apps_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_blsp1_uart4_apps_clk_src = { + .cmd_rcgr = 0x2300c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_blsp1_uart1_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_uart4_apps_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_cpuss_ahb_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(50000000, P_GPLL0_OUT_EVEN, 6, 0, 0), + F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0), + F(133333333, P_GPLL0_OUT_MAIN, 4.5, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_cpuss_ahb_clk_src = { + .cmd_rcgr = 0x3000c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_cpuss_ahb_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_cpuss_ahb_clk_src", + .parent_data = gcc_parent_data_0_ao, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_gp1_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(50000000, P_GPLL0_OUT_EVEN, 6, 0, 0), + F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_gp1_clk_src = { + .cmd_rcgr = 0x37004, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp1_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_gp2_clk_src = { + .cmd_rcgr = 0x38004, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp2_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_gp3_clk_src = { + .cmd_rcgr = 0x39004, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp3_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pcie_aux_phy_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_pcie_aux_phy_clk_src = { + .cmd_rcgr = 0x43048, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_3, + .freq_tbl = ftbl_gcc_pcie_aux_phy_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_aux_phy_clk_src", + .parent_data = gcc_parent_data_3, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pcie_rchng_phy_clk_src[] = { + F(100000000, P_GPLL0_OUT_EVEN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_pcie_rchng_phy_clk_src = { + .cmd_rcgr = 0x43064, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_pcie_rchng_phy_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_rchng_phy_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pdm2_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(60000000, P_GPLL0_OUT_MAIN, 10, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_pdm2_clk_src = { + .cmd_rcgr = 0x24010, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pdm2_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pdm2_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_sdcc1_apps_clk_src[] = { + F(400000, P_BI_TCXO, 12, 1, 4), + F(25000000, P_GPLL0_OUT_EVEN, 12, 0, 0), + F(50000000, P_GPLL0_OUT_EVEN, 6, 0, 0), + F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sdcc1_apps_clk_src = { + .cmd_rcgr = 0x1a010, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_sdcc1_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_apps_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb30_master_clk_src[] = { + F(200000000, P_GPLL0_OUT_EVEN, 1.5, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_usb30_master_clk_src = { + .cmd_rcgr = 0x17030, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_usb30_master_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_master_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb30_mock_utmi_clk_src = { + .cmd_rcgr = 0x17048, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pcie_aux_phy_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_mock_utmi_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb3_phy_aux_clk_src[] = { + F(1000000, P_BI_TCXO, 1, 5, 96), + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_usb3_phy_aux_clk_src = { + .cmd_rcgr = 0x17070, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_3, + .freq_tbl = ftbl_gcc_usb3_phy_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_phy_aux_clk_src", + .parent_data = gcc_parent_data_3, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_regmap_div gcc_cpuss_ahb_postdiv_clk_src = { + .reg = 0x30024, + .shift = 0, + .width = 4, + .clkr.hw.init = &(struct clk_init_data) { + .name = "gcc_cpuss_ahb_postdiv_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_cpuss_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +static struct clk_regmap_div gcc_usb30_mock_utmi_postdiv_clk_src = { + .reg = 0x17060, + .shift = 0, + .width = 4, + .clkr.hw.init = &(struct clk_init_data) { + .name = "gcc_usb30_mock_utmi_postdiv_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_mock_utmi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +static struct clk_branch gcc_ahb_pcie_link_clk = { + .halt_reg = 0x2e004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2e004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ahb_pcie_link_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_ahb_clk = { + .halt_reg = 0x1b004, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x6d008, + .enable_mask = BIT(14), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup1_i2c_apps_clk = { + .halt_reg = 0x1c008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1c008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup1_i2c_apps_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_blsp1_qup1_i2c_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup1_spi_apps_clk = { + .halt_reg = 0x1c004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1c004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup1_spi_apps_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_blsp1_qup1_spi_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup2_i2c_apps_clk = { + .halt_reg = 0x1e008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1e008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup2_i2c_apps_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_blsp1_qup2_i2c_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup2_spi_apps_clk = { + .halt_reg = 0x1e004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1e004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup2_spi_apps_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_blsp1_qup2_spi_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup3_i2c_apps_clk = { + .halt_reg = 0x20008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x20008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup3_i2c_apps_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_blsp1_qup3_i2c_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup3_spi_apps_clk = { + .halt_reg = 0x20004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x20004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup3_spi_apps_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_blsp1_qup3_spi_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup4_i2c_apps_clk = { + .halt_reg = 0x22008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x22008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup4_i2c_apps_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_blsp1_qup4_i2c_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup4_spi_apps_clk = { + .halt_reg = 0x22004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x22004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup4_spi_apps_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_blsp1_qup4_spi_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_sleep_clk = { + .halt_reg = 0x1b00c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x6d008, + .enable_mask = BIT(15), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_sleep_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_uart1_apps_clk = { + .halt_reg = 0x1d004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1d004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_uart1_apps_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_blsp1_uart1_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_uart2_apps_clk = { + .halt_reg = 0x1f004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1f004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_uart2_apps_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_blsp1_uart2_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_uart3_apps_clk = { + .halt_reg = 0x21004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x21004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_uart3_apps_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_blsp1_uart3_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_uart4_apps_clk = { + .halt_reg = 0x23004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x23004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_uart4_apps_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_blsp1_uart4_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_boot_rom_ahb_clk = { + .halt_reg = 0x27004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x27004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x6d008, + .enable_mask = BIT(10), + .hw.init = &(struct clk_init_data){ + .name = "gcc_boot_rom_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp1_clk = { + .halt_reg = 0x37000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x37000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gp1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp2_clk = { + .halt_reg = 0x38000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x38000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gp2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp3_clk = { + .halt_reg = 0x39000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x39000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp3_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gp3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_clkref_en = { + .halt_reg = 0x88004, + /* + * The clock controller does not handle the status bit for + * the clocks with gdscs(powerdomains) in hw controlled mode + * and hence avoid checking for the status bit of those clocks + * by setting the BRANCH_HALT_DELAY flag + */ + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x88004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_clkref_en", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_aux_clk = { + .halt_reg = 0x43034, + /* + * The clock controller does not handle the status bit for + * the clocks with gdscs(powerdomains) in hw controlled mode + * and hence avoid checking for the status bit of those clocks + * by setting the BRANCH_HALT_DELAY flag + */ + .halt_check = BRANCH_HALT_DELAY, + .hwcg_reg = 0x43034, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x6d010, + .enable_mask = BIT(3), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_cfg_ahb_clk = { + .halt_reg = 0x4302c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x4302c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x6d010, + .enable_mask = BIT(2), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_cfg_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_mstr_axi_clk = { + .halt_reg = 0x43024, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x43024, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x6d010, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_mstr_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_pipe_clk = { + .halt_reg = 0x4303c, + /* + * The clock controller does not handle the status bit for + * the clocks with gdscs(powerdomains) in hw controlled mode + * and hence avoid checking for the status bit of those clocks + * by setting the BRANCH_HALT_DELAY flag + */ + .halt_check = BRANCH_HALT_DELAY, + .hwcg_reg = 0x4303c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x6d010, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_pipe_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_rchng_phy_clk = { + .halt_reg = 0x43030, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x43030, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x6d010, + .enable_mask = BIT(7), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_rchng_phy_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_rchng_phy_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_sleep_clk = { + .halt_reg = 0x43038, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x43038, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x6d010, + .enable_mask = BIT(6), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_sleep_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_aux_phy_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_slv_axi_clk = { + .halt_reg = 0x4301c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x4301c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x6d010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_slv_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_slv_q2a_axi_clk = { + .halt_reg = 0x43018, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x43018, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x6d010, + .enable_mask = BIT(5), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_slv_q2a_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm2_clk = { + .halt_reg = 0x2400c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2400c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pdm2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm_ahb_clk = { + .halt_reg = 0x24004, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x24004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x24004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm_xo4_clk = { + .halt_reg = 0x24008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x24008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm_xo4_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_rx1_usb2_clkref_en = { + .halt_reg = 0x88008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x88008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_rx1_usb2_clkref_en", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc1_ahb_clk = { + .halt_reg = 0x1a00c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1a00c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc1_apps_clk = { + .halt_reg = 0x1a004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1a004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_apps_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_sdcc1_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_master_clk = { + .halt_reg = 0x17018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x17018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_master_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_mock_utmi_clk = { + .halt_reg = 0x1702c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1702c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_mock_utmi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = + &gcc_usb30_mock_utmi_postdiv_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_mstr_axi_clk = { + .halt_reg = 0x17020, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x17020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_mstr_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_sleep_clk = { + .halt_reg = 0x17028, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x17028, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sleep_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_slv_ahb_clk = { + .halt_reg = 0x17024, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x17024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_slv_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_phy_aux_clk = { + .halt_reg = 0x17064, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x17064, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_phy_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct gdsc usb30_gdsc = { + .gdscr = 0x17004, + .pd = { + .name = "usb30_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc pcie_gdsc = { + .gdscr = 0x43004, + .pd = { + .name = "pcie_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct clk_branch gcc_usb3_phy_pipe_clk = { + .halt_reg = 0x17068, + /* + * The clock controller does not handle the status bit for + * the clocks with gdscs(powerdomains) in hw controlled mode + * and hence avoid checking for the status bit of those clocks + * by setting the BRANCH_HALT_DELAY flag + */ + .halt_check = BRANCH_HALT_DELAY, + .hwcg_reg = 0x17068, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x17068, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_phy_pipe_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_phy_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_prim_clkref_en = { + .halt_reg = 0x88000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x88000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_clkref_en", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb_phy_cfg_ahb2phy_clk = { + .halt_reg = 0x19008, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x19008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x19008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb_phy_cfg_ahb2phy_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_xo_div4_clk = { + .halt_reg = 0x2e010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2e010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_xo_div4_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_xo_pcie_link_clk = { + .halt_reg = 0x2e008, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x2e008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x2e008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_xo_pcie_link_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_regmap *gcc_sdx65_clocks[] = { + [GCC_AHB_PCIE_LINK_CLK] = &gcc_ahb_pcie_link_clk.clkr, + [GCC_BLSP1_AHB_CLK] = &gcc_blsp1_ahb_clk.clkr, + [GCC_BLSP1_QUP1_I2C_APPS_CLK] = &gcc_blsp1_qup1_i2c_apps_clk.clkr, + [GCC_BLSP1_QUP1_I2C_APPS_CLK_SRC] = &gcc_blsp1_qup1_i2c_apps_clk_src.clkr, + [GCC_BLSP1_QUP1_SPI_APPS_CLK] = &gcc_blsp1_qup1_spi_apps_clk.clkr, + [GCC_BLSP1_QUP1_SPI_APPS_CLK_SRC] = &gcc_blsp1_qup1_spi_apps_clk_src.clkr, + [GCC_BLSP1_QUP2_I2C_APPS_CLK] = &gcc_blsp1_qup2_i2c_apps_clk.clkr, + [GCC_BLSP1_QUP2_I2C_APPS_CLK_SRC] = &gcc_blsp1_qup2_i2c_apps_clk_src.clkr, + [GCC_BLSP1_QUP2_SPI_APPS_CLK] = &gcc_blsp1_qup2_spi_apps_clk.clkr, + [GCC_BLSP1_QUP2_SPI_APPS_CLK_SRC] = &gcc_blsp1_qup2_spi_apps_clk_src.clkr, + [GCC_BLSP1_QUP3_I2C_APPS_CLK] = &gcc_blsp1_qup3_i2c_apps_clk.clkr, + [GCC_BLSP1_QUP3_I2C_APPS_CLK_SRC] = &gcc_blsp1_qup3_i2c_apps_clk_src.clkr, + [GCC_BLSP1_QUP3_SPI_APPS_CLK] = &gcc_blsp1_qup3_spi_apps_clk.clkr, + [GCC_BLSP1_QUP3_SPI_APPS_CLK_SRC] = &gcc_blsp1_qup3_spi_apps_clk_src.clkr, + [GCC_BLSP1_QUP4_I2C_APPS_CLK] = &gcc_blsp1_qup4_i2c_apps_clk.clkr, + [GCC_BLSP1_QUP4_I2C_APPS_CLK_SRC] = &gcc_blsp1_qup4_i2c_apps_clk_src.clkr, + [GCC_BLSP1_QUP4_SPI_APPS_CLK] = &gcc_blsp1_qup4_spi_apps_clk.clkr, + [GCC_BLSP1_QUP4_SPI_APPS_CLK_SRC] = &gcc_blsp1_qup4_spi_apps_clk_src.clkr, + [GCC_BLSP1_SLEEP_CLK] = &gcc_blsp1_sleep_clk.clkr, + [GCC_BLSP1_UART1_APPS_CLK] = &gcc_blsp1_uart1_apps_clk.clkr, + [GCC_BLSP1_UART1_APPS_CLK_SRC] = &gcc_blsp1_uart1_apps_clk_src.clkr, + [GCC_BLSP1_UART2_APPS_CLK] = &gcc_blsp1_uart2_apps_clk.clkr, + [GCC_BLSP1_UART2_APPS_CLK_SRC] = &gcc_blsp1_uart2_apps_clk_src.clkr, + [GCC_BLSP1_UART3_APPS_CLK] = &gcc_blsp1_uart3_apps_clk.clkr, + [GCC_BLSP1_UART3_APPS_CLK_SRC] = &gcc_blsp1_uart3_apps_clk_src.clkr, + [GCC_BLSP1_UART4_APPS_CLK] = &gcc_blsp1_uart4_apps_clk.clkr, + [GCC_BLSP1_UART4_APPS_CLK_SRC] = &gcc_blsp1_uart4_apps_clk_src.clkr, + [GCC_BOOT_ROM_AHB_CLK] = &gcc_boot_rom_ahb_clk.clkr, + [GCC_CPUSS_AHB_CLK_SRC] = &gcc_cpuss_ahb_clk_src.clkr, + [GCC_CPUSS_AHB_POSTDIV_CLK_SRC] = &gcc_cpuss_ahb_postdiv_clk_src.clkr, + [GCC_GP1_CLK] = &gcc_gp1_clk.clkr, + [GCC_GP1_CLK_SRC] = &gcc_gp1_clk_src.clkr, + [GCC_GP2_CLK] = &gcc_gp2_clk.clkr, + [GCC_GP2_CLK_SRC] = &gcc_gp2_clk_src.clkr, + [GCC_GP3_CLK] = &gcc_gp3_clk.clkr, + [GCC_GP3_CLK_SRC] = &gcc_gp3_clk_src.clkr, + [GCC_PCIE_0_CLKREF_EN] = &gcc_pcie_0_clkref_en.clkr, + [GCC_PCIE_AUX_CLK] = &gcc_pcie_aux_clk.clkr, + [GCC_PCIE_AUX_CLK_SRC] = &gcc_pcie_aux_clk_src.clkr, + [GCC_PCIE_AUX_PHY_CLK_SRC] = &gcc_pcie_aux_phy_clk_src.clkr, + [GCC_PCIE_CFG_AHB_CLK] = &gcc_pcie_cfg_ahb_clk.clkr, + [GCC_PCIE_MSTR_AXI_CLK] = &gcc_pcie_mstr_axi_clk.clkr, + [GCC_PCIE_PIPE_CLK] = &gcc_pcie_pipe_clk.clkr, + [GCC_PCIE_PIPE_CLK_SRC] = &gcc_pcie_pipe_clk_src.clkr, + [GCC_PCIE_RCHNG_PHY_CLK] = &gcc_pcie_rchng_phy_clk.clkr, + [GCC_PCIE_RCHNG_PHY_CLK_SRC] = &gcc_pcie_rchng_phy_clk_src.clkr, + [GCC_PCIE_SLEEP_CLK] = &gcc_pcie_sleep_clk.clkr, + [GCC_PCIE_SLV_AXI_CLK] = &gcc_pcie_slv_axi_clk.clkr, + [GCC_PCIE_SLV_Q2A_AXI_CLK] = &gcc_pcie_slv_q2a_axi_clk.clkr, + [GCC_PDM2_CLK] = &gcc_pdm2_clk.clkr, + [GCC_PDM2_CLK_SRC] = &gcc_pdm2_clk_src.clkr, + [GCC_PDM_AHB_CLK] = &gcc_pdm_ahb_clk.clkr, + [GCC_PDM_XO4_CLK] = &gcc_pdm_xo4_clk.clkr, + [GCC_RX1_USB2_CLKREF_EN] = &gcc_rx1_usb2_clkref_en.clkr, + [GCC_SDCC1_AHB_CLK] = &gcc_sdcc1_ahb_clk.clkr, + [GCC_SDCC1_APPS_CLK] = &gcc_sdcc1_apps_clk.clkr, + [GCC_SDCC1_APPS_CLK_SRC] = &gcc_sdcc1_apps_clk_src.clkr, + [GCC_USB30_MASTER_CLK] = &gcc_usb30_master_clk.clkr, + [GCC_USB30_MASTER_CLK_SRC] = &gcc_usb30_master_clk_src.clkr, + [GCC_USB30_MOCK_UTMI_CLK] = &gcc_usb30_mock_utmi_clk.clkr, + [GCC_USB30_MOCK_UTMI_CLK_SRC] = &gcc_usb30_mock_utmi_clk_src.clkr, + [GCC_USB30_MOCK_UTMI_POSTDIV_CLK_SRC] = &gcc_usb30_mock_utmi_postdiv_clk_src.clkr, + [GCC_USB30_MSTR_AXI_CLK] = &gcc_usb30_mstr_axi_clk.clkr, + [GCC_USB30_SLEEP_CLK] = &gcc_usb30_sleep_clk.clkr, + [GCC_USB30_SLV_AHB_CLK] = &gcc_usb30_slv_ahb_clk.clkr, + [GCC_USB3_PHY_AUX_CLK] = &gcc_usb3_phy_aux_clk.clkr, + [GCC_USB3_PHY_AUX_CLK_SRC] = &gcc_usb3_phy_aux_clk_src.clkr, + [GCC_USB3_PHY_PIPE_CLK] = &gcc_usb3_phy_pipe_clk.clkr, + [GCC_USB3_PHY_PIPE_CLK_SRC] = &gcc_usb3_phy_pipe_clk_src.clkr, + [GCC_USB3_PRIM_CLKREF_EN] = &gcc_usb3_prim_clkref_en.clkr, + [GCC_USB_PHY_CFG_AHB2PHY_CLK] = &gcc_usb_phy_cfg_ahb2phy_clk.clkr, + [GCC_XO_DIV4_CLK] = &gcc_xo_div4_clk.clkr, + [GCC_XO_PCIE_LINK_CLK] = &gcc_xo_pcie_link_clk.clkr, + [GPLL0] = &gpll0.clkr, + [GPLL0_OUT_EVEN] = &gpll0_out_even.clkr, +}; + +static const struct qcom_reset_map gcc_sdx65_resets[] = { + [GCC_BLSP1_QUP1_BCR] = { 0x1c000 }, + [GCC_BLSP1_QUP2_BCR] = { 0x1e000 }, + [GCC_BLSP1_QUP3_BCR] = { 0x20000 }, + [GCC_BLSP1_QUP4_BCR] = { 0x22000 }, + [GCC_BLSP1_UART1_BCR] = { 0x1d000 }, + [GCC_BLSP1_UART2_BCR] = { 0x1f000 }, + [GCC_BLSP1_UART3_BCR] = { 0x21000 }, + [GCC_BLSP1_UART4_BCR] = { 0x23000 }, + [GCC_PCIE_BCR] = { 0x43000 }, + [GCC_PCIE_LINK_DOWN_BCR] = { 0x77000 }, + [GCC_PCIE_NOCSR_COM_PHY_BCR] = { 0x78008 }, + [GCC_PCIE_PHY_BCR] = { 0x44000 }, + [GCC_PCIE_PHY_CFG_AHB_BCR] = { 0x78000 }, + [GCC_PCIE_PHY_COM_BCR] = { 0x78004 }, + [GCC_PCIE_PHY_NOCSR_COM_PHY_BCR] = { 0x7800c }, + [GCC_PDM_BCR] = { 0x24000 }, + [GCC_QUSB2PHY_BCR] = { 0x19000 }, + [GCC_SDCC1_BCR] = { 0x1a000 }, + [GCC_TCSR_PCIE_BCR] = { 0x57000 }, + [GCC_USB30_BCR] = { 0x17000 }, + [GCC_USB3_PHY_BCR] = { 0x18000 }, + [GCC_USB3PHY_PHY_BCR] = { 0x18004 }, + [GCC_USB_PHY_CFG_AHB2PHY_BCR] = { 0x19004 }, +}; + +static struct gdsc *gcc_sdx65_gdscs[] = { + [USB30_GDSC] = &usb30_gdsc, + [PCIE_GDSC] = &pcie_gdsc, +}; + +static const struct regmap_config gcc_sdx65_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x1f101c, + .fast_io = true, +}; + +static const struct qcom_cc_desc gcc_sdx65_desc = { + .config = &gcc_sdx65_regmap_config, + .clks = gcc_sdx65_clocks, + .num_clks = ARRAY_SIZE(gcc_sdx65_clocks), + .resets = gcc_sdx65_resets, + .num_resets = ARRAY_SIZE(gcc_sdx65_resets), + .gdscs = gcc_sdx65_gdscs, + .num_gdscs = ARRAY_SIZE(gcc_sdx65_gdscs), +}; + +static const struct of_device_id gcc_sdx65_match_table[] = { + { .compatible = "qcom,gcc-sdx65" }, + { } +}; +MODULE_DEVICE_TABLE(of, gcc_sdx65_match_table); + +static int gcc_sdx65_probe(struct platform_device *pdev) +{ + struct regmap *regmap; + + regmap = qcom_cc_map(pdev, &gcc_sdx65_desc); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + /* + * Keep the clocks always-ON as they are critical to the functioning + * of the system: + * GCC_SYS_NOC_CPUSS_AHB_CLK, GCC_CPUSS_AHB_CLK, GCC_CPUSS_GNOC_CLK + */ + regmap_update_bits(regmap, 0x6d008, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x6d008, BIT(21), BIT(21)); + regmap_update_bits(regmap, 0x6d008, BIT(22), BIT(22)); + + return qcom_cc_really_probe(pdev, &gcc_sdx65_desc, regmap); +} + +static struct platform_driver gcc_sdx65_driver = { + .probe = gcc_sdx65_probe, + .driver = { + .name = "gcc-sdx65", + .of_match_table = gcc_sdx65_match_table, + }, +}; + +static int __init gcc_sdx65_init(void) +{ + return platform_driver_register(&gcc_sdx65_driver); +} +subsys_initcall(gcc_sdx65_init); + +static void __exit gcc_sdx65_exit(void) +{ + platform_driver_unregister(&gcc_sdx65_driver); +} +module_exit(gcc_sdx65_exit); + +MODULE_DESCRIPTION("QTI GCC SDX65 Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/qcom/gcc-sm6350.c b/drivers/clk/qcom/gcc-sm6350.c index 3236706771b1..a4f7fba70393 100644 --- a/drivers/clk/qcom/gcc-sm6350.c +++ b/drivers/clk/qcom/gcc-sm6350.c @@ -4,6 +4,7 @@ * Copyright (c) 2021, Konrad Dybcio */ +#include #include #include #include diff --git a/drivers/clk/qcom/gcc-sm8350.c b/drivers/clk/qcom/gcc-sm8350.c index 6d0a9e2d5104..c3731f96c8e6 100644 --- a/drivers/clk/qcom/gcc-sm8350.c +++ b/drivers/clk/qcom/gcc-sm8350.c @@ -4,6 +4,7 @@ * Copyright (c) 2020-2021, Linaro Limited */ +#include #include #include #include diff --git a/drivers/clk/qcom/gcc-sm8450.c b/drivers/clk/qcom/gcc-sm8450.c new file mode 100644 index 000000000000..593a195467ff --- /dev/null +++ b/drivers/clk/qcom/gcc-sm8450.c @@ -0,0 +1,3304 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2021, Linaro Limited + */ + +#include +#include +#include +#include + +#include + +#include "clk-alpha-pll.h" +#include "clk-branch.h" +#include "clk-rcg.h" +#include "clk-regmap.h" +#include "clk-regmap-divider.h" +#include "clk-regmap-mux.h" +#include "gdsc.h" +#include "reset.h" + +enum { + P_BI_TCXO, + P_GCC_GPLL0_OUT_EVEN, + P_GCC_GPLL0_OUT_MAIN, + P_GCC_GPLL4_OUT_MAIN, + P_GCC_GPLL9_OUT_MAIN, + P_PCIE_0_PIPE_CLK, + P_PCIE_1_PHY_AUX_CLK, + P_PCIE_1_PIPE_CLK, + P_SLEEP_CLK, + P_UFS_PHY_RX_SYMBOL_0_CLK, + P_UFS_PHY_RX_SYMBOL_1_CLK, + P_UFS_PHY_TX_SYMBOL_0_CLK, + P_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK, +}; + +static struct clk_alpha_pll gcc_gpll0 = { + .offset = 0x0, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_EVO], + .clkr = { + .enable_reg = 0x62018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpll0", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_lucid_evo_ops, + }, + }, +}; + +static const struct clk_div_table post_div_table_gcc_gpll0_out_even[] = { + { 0x1, 2 }, + { } +}; + +static struct clk_alpha_pll_postdiv gcc_gpll0_out_even = { + .offset = 0x0, + .post_div_shift = 10, + .post_div_table = post_div_table_gcc_gpll0_out_even, + .num_post_div = ARRAY_SIZE(post_div_table_gcc_gpll0_out_even), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_EVO], + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gpll0_out_even", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gpll0.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_lucid_evo_ops, + }, +}; + +static struct clk_alpha_pll gcc_gpll4 = { + .offset = 0x4000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_EVO], + .clkr = { + .enable_reg = 0x62018, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpll4", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_lucid_evo_ops, + }, + }, +}; + +static struct clk_alpha_pll gcc_gpll9 = { + .offset = 0x9000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_EVO], + .clkr = { + .enable_reg = 0x62018, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpll9", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_lucid_evo_ops, + }, + }, +}; + +static const struct parent_map gcc_parent_map_0[] = { + { P_BI_TCXO, 0 }, + { P_GCC_GPLL0_OUT_MAIN, 1 }, + { P_GCC_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_0[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &gcc_gpll0.clkr.hw }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_1[] = { + { P_BI_TCXO, 0 }, + { P_GCC_GPLL0_OUT_MAIN, 1 }, + { P_SLEEP_CLK, 5 }, + { P_GCC_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_1[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &gcc_gpll0.clkr.hw }, + { .fw_name = "sleep_clk" }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_2[] = { + { P_BI_TCXO, 0 }, + { P_SLEEP_CLK, 5 }, +}; + +static const struct clk_parent_data gcc_parent_data_2[] = { + { .fw_name = "bi_tcxo" }, + { .fw_name = "sleep_clk" }, +}; + +static const struct parent_map gcc_parent_map_3[] = { + { P_BI_TCXO, 0 }, +}; + +static const struct clk_parent_data gcc_parent_data_3[] = { + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_4[] = { + { P_PCIE_0_PIPE_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_4[] = { + { .fw_name = "pcie_0_pipe_clk", }, + { .fw_name = "bi_tcxo", }, +}; + +static const struct parent_map gcc_parent_map_5[] = { + { P_PCIE_1_PHY_AUX_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_5[] = { + { .fw_name = "pcie_1_phy_aux_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_6[] = { + { P_PCIE_1_PIPE_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_6[] = { + { .fw_name = "pcie_1_pipe_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_7[] = { + { P_BI_TCXO, 0 }, + { P_GCC_GPLL0_OUT_MAIN, 1 }, + { P_GCC_GPLL9_OUT_MAIN, 2 }, + { P_GCC_GPLL4_OUT_MAIN, 5 }, + { P_GCC_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_7[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &gcc_gpll0.clkr.hw }, + { .hw = &gcc_gpll9.clkr.hw }, + { .hw = &gcc_gpll4.clkr.hw }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_8[] = { + { P_UFS_PHY_RX_SYMBOL_0_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_8[] = { + { .fw_name = "ufs_phy_rx_symbol_0_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_9[] = { + { P_UFS_PHY_RX_SYMBOL_1_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_9[] = { + { .fw_name = "ufs_phy_rx_symbol_1_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_10[] = { + { P_UFS_PHY_TX_SYMBOL_0_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_10[] = { + { .fw_name = "ufs_phy_tx_symbol_0_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_11[] = { + { P_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_11[] = { + { .fw_name = "usb3_phy_wrapper_gcc_usb30_pipe_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static struct clk_regmap_mux gcc_pcie_0_pipe_clk_src = { + .reg = 0x7b060, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_4, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_pipe_clk_src", + .parent_data = gcc_parent_data_4, + .num_parents = ARRAY_SIZE(gcc_parent_data_4), + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_pcie_1_phy_aux_clk_src = { + .reg = 0x9d080, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_5, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_phy_aux_clk_src", + .parent_data = gcc_parent_data_5, + .num_parents = ARRAY_SIZE(gcc_parent_data_5), + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_pcie_1_pipe_clk_src = { + .reg = 0x9d064, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_6, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_pipe_clk_src", + .parent_data = gcc_parent_data_6, + .num_parents = ARRAY_SIZE(gcc_parent_data_6), + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_ufs_phy_rx_symbol_0_clk_src = { + .reg = 0x87060, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_8, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_rx_symbol_0_clk_src", + .parent_data = gcc_parent_data_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_8), + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_ufs_phy_rx_symbol_1_clk_src = { + .reg = 0x870d0, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_9, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_rx_symbol_1_clk_src", + .parent_data = gcc_parent_data_9, + .num_parents = ARRAY_SIZE(gcc_parent_data_9), + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_ufs_phy_tx_symbol_0_clk_src = { + .reg = 0x87050, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_10, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_tx_symbol_0_clk_src", + .parent_data = gcc_parent_data_10, + .num_parents = ARRAY_SIZE(gcc_parent_data_10), + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_usb3_prim_phy_pipe_clk_src = { + .reg = 0x49068, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_11, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_pipe_clk_src", + .parent_data = gcc_parent_data_11, + .num_parents = ARRAY_SIZE(gcc_parent_data_11), + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static const struct freq_tbl ftbl_gcc_gp1_clk_src[] = { + F(50000000, P_GCC_GPLL0_OUT_EVEN, 6, 0, 0), + F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0), + F(200000000, P_GCC_GPLL0_OUT_MAIN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_gp1_clk_src = { + .cmd_rcgr = 0x74004, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp1_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = ARRAY_SIZE(gcc_parent_data_1), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_gp2_clk_src = { + .cmd_rcgr = 0x75004, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp2_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = ARRAY_SIZE(gcc_parent_data_1), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_gp3_clk_src = { + .cmd_rcgr = 0x76004, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp3_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = ARRAY_SIZE(gcc_parent_data_1), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pcie_0_aux_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_pcie_0_aux_clk_src = { + .cmd_rcgr = 0x7b064, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_aux_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = ARRAY_SIZE(gcc_parent_data_2), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pcie_0_phy_rchng_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_pcie_0_phy_rchng_clk_src = { + .cmd_rcgr = 0x7b048, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pcie_0_phy_rchng_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_phy_rchng_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_pcie_1_aux_clk_src = { + .cmd_rcgr = 0x9d068, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_aux_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = ARRAY_SIZE(gcc_parent_data_2), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_pcie_1_phy_rchng_clk_src = { + .cmd_rcgr = 0x9d04c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pcie_0_phy_rchng_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_phy_rchng_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pdm2_clk_src[] = { + F(60000000, P_GCC_GPLL0_OUT_MAIN, 10, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_pdm2_clk_src = { + .cmd_rcgr = 0x43010, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pdm2_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pdm2_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_qupv3_wrap0_s0_clk_src[] = { + F(7372800, P_GCC_GPLL0_OUT_EVEN, 1, 384, 15625), + F(14745600, P_GCC_GPLL0_OUT_EVEN, 1, 768, 15625), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(29491200, P_GCC_GPLL0_OUT_EVEN, 1, 1536, 15625), + F(32000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 75), + F(48000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 25), + F(64000000, P_GCC_GPLL0_OUT_EVEN, 1, 16, 75), + F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0), + F(80000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 15), + F(96000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 25), + F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0), + { } +}; + +static struct clk_init_data gcc_qupv3_wrap0_s0_clk_src_init = { + .name = "gcc_qupv3_wrap0_s0_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s0_clk_src = { + .cmd_rcgr = 0x27014, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s0_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s1_clk_src_init = { + .name = "gcc_qupv3_wrap0_s1_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s1_clk_src = { + .cmd_rcgr = 0x27148, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s1_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s2_clk_src_init = { + .name = "gcc_qupv3_wrap0_s2_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s2_clk_src = { + .cmd_rcgr = 0x2727c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s2_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s3_clk_src_init = { + .name = "gcc_qupv3_wrap0_s3_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s3_clk_src = { + .cmd_rcgr = 0x273b0, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s3_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s4_clk_src_init = { + .name = "gcc_qupv3_wrap0_s4_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s4_clk_src = { + .cmd_rcgr = 0x274e4, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s4_clk_src_init, +}; + +static const struct freq_tbl ftbl_gcc_qupv3_wrap0_s5_clk_src[] = { + F(7372800, P_GCC_GPLL0_OUT_EVEN, 1, 384, 15625), + F(14745600, P_GCC_GPLL0_OUT_EVEN, 1, 768, 15625), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(29491200, P_GCC_GPLL0_OUT_EVEN, 1, 1536, 15625), + F(32000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 75), + F(37500000, P_GCC_GPLL0_OUT_EVEN, 8, 0, 0), + F(48000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 25), + F(50000000, P_GCC_GPLL0_OUT_MAIN, 12, 0, 0), + { } +}; + +static struct clk_init_data gcc_qupv3_wrap0_s5_clk_src_init = { + .name = "gcc_qupv3_wrap0_s5_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s5_clk_src = { + .cmd_rcgr = 0x27618, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s5_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s5_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s6_clk_src_init = { + .name = "gcc_qupv3_wrap0_s6_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s6_clk_src = { + .cmd_rcgr = 0x2774c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s6_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s7_clk_src_init = { + .name = "gcc_qupv3_wrap0_s7_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s7_clk_src = { + .cmd_rcgr = 0x27880, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s7_clk_src_init, +}; + +static const struct freq_tbl ftbl_gcc_qupv3_wrap1_s0_clk_src[] = { + F(7372800, P_GCC_GPLL0_OUT_EVEN, 1, 384, 15625), + F(14745600, P_GCC_GPLL0_OUT_EVEN, 1, 768, 15625), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(29491200, P_GCC_GPLL0_OUT_EVEN, 1, 1536, 15625), + F(32000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 75), + F(48000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 25), + F(64000000, P_GCC_GPLL0_OUT_EVEN, 1, 16, 75), + F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0), + F(80000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 15), + F(96000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 25), + F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0), + F(102400000, P_GCC_GPLL0_OUT_EVEN, 1, 128, 375), + F(112000000, P_GCC_GPLL0_OUT_EVEN, 1, 28, 75), + F(117964800, P_GCC_GPLL0_OUT_EVEN, 1, 6144, 15625), + F(120000000, P_GCC_GPLL0_OUT_MAIN, 5, 0, 0), + { } +}; + +static struct clk_init_data gcc_qupv3_wrap1_s0_clk_src_init = { + .name = "gcc_qupv3_wrap1_s0_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s0_clk_src = { + .cmd_rcgr = 0x28014, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s0_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s1_clk_src_init = { + .name = "gcc_qupv3_wrap1_s1_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s1_clk_src = { + .cmd_rcgr = 0x28148, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s1_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s2_clk_src_init = { + .name = "gcc_qupv3_wrap1_s2_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s2_clk_src = { + .cmd_rcgr = 0x2827c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s2_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s3_clk_src_init = { + .name = "gcc_qupv3_wrap1_s3_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s3_clk_src = { + .cmd_rcgr = 0x283b0, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s3_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s4_clk_src_init = { + .name = "gcc_qupv3_wrap1_s4_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s4_clk_src = { + .cmd_rcgr = 0x284e4, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s4_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s5_clk_src_init = { + .name = "gcc_qupv3_wrap1_s5_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s5_clk_src = { + .cmd_rcgr = 0x28618, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s5_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s6_clk_src_init = { + .name = "gcc_qupv3_wrap1_s6_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s6_clk_src = { + .cmd_rcgr = 0x2874c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s6_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap2_s0_clk_src_init = { + .name = "gcc_qupv3_wrap2_s0_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s0_clk_src = { + .cmd_rcgr = 0x2e014, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap2_s0_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap2_s1_clk_src_init = { + .name = "gcc_qupv3_wrap2_s1_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s1_clk_src = { + .cmd_rcgr = 0x2e148, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap2_s1_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap2_s2_clk_src_init = { + .name = "gcc_qupv3_wrap2_s2_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s2_clk_src = { + .cmd_rcgr = 0x2e27c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap2_s2_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap2_s3_clk_src_init = { + .name = "gcc_qupv3_wrap2_s3_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s3_clk_src = { + .cmd_rcgr = 0x2e3b0, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap2_s3_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap2_s4_clk_src_init = { + .name = "gcc_qupv3_wrap2_s4_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s4_clk_src = { + .cmd_rcgr = 0x2e4e4, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap2_s4_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap2_s5_clk_src_init = { + .name = "gcc_qupv3_wrap2_s5_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s5_clk_src = { + .cmd_rcgr = 0x2e618, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap2_s5_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap2_s6_clk_src_init = { + .name = "gcc_qupv3_wrap2_s6_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s6_clk_src = { + .cmd_rcgr = 0x2e74c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap2_s6_clk_src_init, +}; + +static const struct freq_tbl ftbl_gcc_sdcc2_apps_clk_src[] = { + F(400000, P_BI_TCXO, 12, 1, 4), + F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0), + F(50000000, P_GCC_GPLL0_OUT_EVEN, 6, 0, 0), + F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0), + F(202000000, P_GCC_GPLL9_OUT_MAIN, 4, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { + .cmd_rcgr = 0x24014, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_7, + .freq_tbl = ftbl_gcc_sdcc2_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc2_apps_clk_src", + .parent_data = gcc_parent_data_7, + .num_parents = ARRAY_SIZE(gcc_parent_data_7), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_sdcc4_apps_clk_src[] = { + F(400000, P_BI_TCXO, 12, 1, 4), + F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0), + F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { + .cmd_rcgr = 0x26014, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_sdcc4_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc4_apps_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_phy_axi_clk_src[] = { + F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0), + F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0), + F(150000000, P_GCC_GPLL0_OUT_MAIN, 4, 0, 0), + F(300000000, P_GCC_GPLL0_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_phy_axi_clk_src = { + .cmd_rcgr = 0x8702c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_phy_axi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_axi_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_phy_ice_core_clk_src[] = { + F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0), + F(150000000, P_GCC_GPLL0_OUT_MAIN, 4, 0, 0), + F(300000000, P_GCC_GPLL0_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_phy_ice_core_clk_src = { + .cmd_rcgr = 0x87074, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_phy_ice_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ice_core_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_phy_phy_aux_clk_src[] = { + F(9600000, P_BI_TCXO, 2, 0, 0), + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_phy_phy_aux_clk_src = { + .cmd_rcgr = 0x870a8, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_3, + .freq_tbl = ftbl_gcc_ufs_phy_phy_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_phy_aux_clk_src", + .parent_data = gcc_parent_data_3, + .num_parents = ARRAY_SIZE(gcc_parent_data_3), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_ufs_phy_unipro_core_clk_src = { + .cmd_rcgr = 0x8708c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_phy_ice_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_unipro_core_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb30_prim_master_clk_src[] = { + F(66666667, P_GCC_GPLL0_OUT_EVEN, 4.5, 0, 0), + F(133333333, P_GCC_GPLL0_OUT_MAIN, 4.5, 0, 0), + F(200000000, P_GCC_GPLL0_OUT_MAIN, 3, 0, 0), + F(240000000, P_GCC_GPLL0_OUT_MAIN, 2.5, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_usb30_prim_master_clk_src = { + .cmd_rcgr = 0x49028, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_usb30_prim_master_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_master_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb30_prim_mock_utmi_clk_src = { + .cmd_rcgr = 0x49040, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_mock_utmi_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb3_prim_phy_aux_clk_src = { + .cmd_rcgr = 0x4906c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_aux_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = ARRAY_SIZE(gcc_parent_data_2), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_regmap_div gcc_usb30_prim_mock_utmi_postdiv_clk_src = { + .reg = 0x49058, + .shift = 0, + .width = 4, + .clkr.hw.init = &(struct clk_init_data) { + .name = "gcc_usb30_prim_mock_utmi_postdiv_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_prim_mock_utmi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +static struct clk_branch gcc_aggre_noc_pcie_0_axi_clk = { + .halt_reg = 0x7b08c, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x7b08c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x62000, + .enable_mask = BIT(12), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_noc_pcie_0_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_noc_pcie_1_axi_clk = { + .halt_reg = 0x9d098, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x9d098, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x62000, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_noc_pcie_1_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_ufs_phy_axi_clk = { + .halt_reg = 0x870d4, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x870d4, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x870d4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_ufs_phy_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_ufs_phy_axi_hw_ctl_clk = { + .halt_reg = 0x870d4, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x870d4, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x870d4, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_ufs_phy_axi_hw_ctl_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_usb3_prim_axi_clk = { + .halt_reg = 0x49088, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x49088, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x49088, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_usb3_prim_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_prim_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_boot_rom_ahb_clk = { + .halt_reg = 0x48004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x48004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x62000, + .enable_mask = BIT(10), + .hw.init = &(struct clk_init_data){ + .name = "gcc_boot_rom_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camera_hf_axi_clk = { + .halt_reg = 0x36010, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x36010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x36010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camera_hf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camera_sf_axi_clk = { + .halt_reg = 0x36018, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x36018, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x36018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camera_sf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cfg_noc_pcie_anoc_ahb_clk = { + .halt_reg = 0x20030, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x20030, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x62000, + .enable_mask = BIT(20), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cfg_noc_pcie_anoc_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cfg_noc_usb3_prim_axi_clk = { + .halt_reg = 0x49084, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x49084, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x49084, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cfg_noc_usb3_prim_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_prim_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ddrss_gpu_axi_clk = { + .halt_reg = 0x81154, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x81154, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x81154, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ddrss_gpu_axi_clk", + .ops = &clk_branch2_aon_ops, + }, + }, +}; + +static struct clk_branch gcc_ddrss_pcie_sf_tbu_clk = { + .halt_reg = 0x9d094, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x9d094, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x62000, + .enable_mask = BIT(19), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ddrss_pcie_sf_tbu_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_disp_hf_axi_clk = { + .halt_reg = 0x3700c, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x3700c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x3700c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_disp_hf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_disp_sf_axi_clk = { + .halt_reg = 0x37014, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x37014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x37014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_disp_sf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_eusb3_0_clkref_en = { + .halt_reg = 0x9c00c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x9c00c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_eusb3_0_clkref_en", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp1_clk = { + .halt_reg = 0x74000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x74000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gp1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp2_clk = { + .halt_reg = 0x75000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x75000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gp2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp3_clk = { + .halt_reg = 0x76000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x76000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp3_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gp3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_gpll0_clk_src = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x62000, + .enable_mask = BIT(15), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_gpll0_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gpll0.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_gpll0_div_clk_src = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x62000, + .enable_mask = BIT(16), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_gpll0_div_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gpll0_out_even.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_memnoc_gfx_clk = { + .halt_reg = 0x81010, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x81010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x81010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_memnoc_gfx_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_snoc_dvm_gfx_clk = { + .halt_reg = 0x81018, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x81018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_snoc_dvm_gfx_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_aux_clk = { + .halt_reg = 0x7b034, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(3), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_0_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_cfg_ahb_clk = { + .halt_reg = 0x7b030, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7b030, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(2), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_cfg_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_clkref_en = { + .halt_reg = 0x9c004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x9c004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_clkref_en", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_mstr_axi_clk = { + .halt_reg = 0x7b028, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_mstr_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_phy_rchng_clk = { + .halt_reg = 0x7b044, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62000, + .enable_mask = BIT(22), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_phy_rchng_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_0_phy_rchng_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_pipe_clk = { + .halt_reg = 0x7b03c, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_pipe_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_0_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_slv_axi_clk = { + .halt_reg = 0x7b020, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7b020, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_slv_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_slv_q2a_axi_clk = { + .halt_reg = 0x7b01c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(5), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_slv_q2a_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_aux_clk = { + .halt_reg = 0x9d030, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62000, + .enable_mask = BIT(29), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_1_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_cfg_ahb_clk = { + .halt_reg = 0x9d02c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x9d02c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x62000, + .enable_mask = BIT(28), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_cfg_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_clkref_en = { + .halt_reg = 0x9c008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x9c008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_clkref_en", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_mstr_axi_clk = { + .halt_reg = 0x9d024, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x62000, + .enable_mask = BIT(27), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_mstr_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_phy_aux_clk = { + .halt_reg = 0x9d038, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62000, + .enable_mask = BIT(24), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_phy_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_1_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_phy_rchng_clk = { + .halt_reg = 0x9d048, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62000, + .enable_mask = BIT(23), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_phy_rchng_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_1_phy_rchng_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_pipe_clk = { + .halt_reg = 0x9d040, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x62000, + .enable_mask = BIT(30), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_pipe_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_1_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_slv_axi_clk = { + .halt_reg = 0x9d01c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x9d01c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x62000, + .enable_mask = BIT(26), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_slv_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_slv_q2a_axi_clk = { + .halt_reg = 0x9d018, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62000, + .enable_mask = BIT(25), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_slv_q2a_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm2_clk = { + .halt_reg = 0x4300c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4300c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pdm2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm_ahb_clk = { + .halt_reg = 0x43004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x43004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x43004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm_xo4_clk = { + .halt_reg = 0x43008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x43008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm_xo4_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_camera_nrt_ahb_clk = { + .halt_reg = 0x36008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x36008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x36008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_camera_nrt_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_camera_rt_ahb_clk = { + .halt_reg = 0x3600c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x3600c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x3600c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_camera_rt_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_disp_ahb_clk = { + .halt_reg = 0x37008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x37008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x37008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_disp_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_gpu_ahb_clk = { + .halt_reg = 0x81008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x81008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x81008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_gpu_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_pcie_ahb_clk = { + .halt_reg = 0x7b018, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7b018, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7b018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_pcie_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_video_cv_cpu_ahb_clk = { + .halt_reg = 0x42014, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x42014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x42014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_video_cv_cpu_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_video_cvp_ahb_clk = { + .halt_reg = 0x42008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x42008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x42008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_video_cvp_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_video_v_cpu_ahb_clk = { + .halt_reg = 0x42010, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x42010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x42010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_video_v_cpu_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_video_vcodec_ahb_clk = { + .halt_reg = 0x4200c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x4200c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x4200c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_video_vcodec_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_core_2x_clk = { + .halt_reg = 0x3300c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_core_2x_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_core_clk = { + .halt_reg = 0x33000, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(8), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s0_clk = { + .halt_reg = 0x2700c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(10), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s1_clk = { + .halt_reg = 0x27140, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s2_clk = { + .halt_reg = 0x27274, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(12), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s3_clk = { + .halt_reg = 0x273a8, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(13), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s3_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s4_clk = { + .halt_reg = 0x274dc, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(14), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s4_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s4_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s5_clk = { + .halt_reg = 0x27610, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(15), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s5_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s5_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s6_clk = { + .halt_reg = 0x27744, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(16), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s6_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s6_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s7_clk = { + .halt_reg = 0x27878, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(17), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s7_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s7_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_core_2x_clk = { + .halt_reg = 0x3314c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(18), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_core_2x_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_core_clk = { + .halt_reg = 0x33140, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(19), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s0_clk = { + .halt_reg = 0x2800c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(22), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s1_clk = { + .halt_reg = 0x28140, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(23), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s2_clk = { + .halt_reg = 0x28274, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(24), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s3_clk = { + .halt_reg = 0x283a8, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(25), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s3_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s4_clk = { + .halt_reg = 0x284dc, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(26), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s4_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s4_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s5_clk = { + .halt_reg = 0x28610, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(27), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s5_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s5_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s6_clk = { + .halt_reg = 0x28744, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(28), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s6_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s6_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_core_2x_clk = { + .halt_reg = 0x3328c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62010, + .enable_mask = BIT(3), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_core_2x_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_core_clk = { + .halt_reg = 0x33280, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s0_clk = { + .halt_reg = 0x2e00c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62010, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap2_s0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s1_clk = { + .halt_reg = 0x2e140, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62010, + .enable_mask = BIT(5), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap2_s1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s2_clk = { + .halt_reg = 0x2e274, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62010, + .enable_mask = BIT(6), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap2_s2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s3_clk = { + .halt_reg = 0x2e3a8, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62010, + .enable_mask = BIT(7), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s3_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap2_s3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s4_clk = { + .halt_reg = 0x2e4dc, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62010, + .enable_mask = BIT(8), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s4_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap2_s4_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s5_clk = { + .halt_reg = 0x2e610, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62010, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s5_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap2_s5_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s6_clk = { + .halt_reg = 0x2e744, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x62010, + .enable_mask = BIT(10), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s6_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap2_s6_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_0_m_ahb_clk = { + .halt_reg = 0x27004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x27004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(6), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_0_m_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_0_s_ahb_clk = { + .halt_reg = 0x27008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x27008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(7), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_0_s_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_1_m_ahb_clk = { + .halt_reg = 0x28004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x28004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(20), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_1_m_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_1_s_ahb_clk = { + .halt_reg = 0x28008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x28008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x62008, + .enable_mask = BIT(21), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_1_s_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_2_m_ahb_clk = { + .halt_reg = 0x2e004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x2e004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x62010, + .enable_mask = BIT(2), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_2_m_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_2_s_ahb_clk = { + .halt_reg = 0x2e008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x2e008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x62010, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_2_s_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc2_ahb_clk = { + .halt_reg = 0x2400c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2400c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc2_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc2_apps_clk = { + .halt_reg = 0x24004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x24004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc2_apps_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_sdcc2_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc2_at_clk = { + .halt_reg = 0x24010, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x24010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x24010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc2_at_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc4_ahb_clk = { + .halt_reg = 0x2600c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2600c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc4_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc4_apps_clk = { + .halt_reg = 0x26004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x26004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc4_apps_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_sdcc4_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc4_at_clk = { + .halt_reg = 0x26010, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x26010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x26010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc4_at_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_0_clkref_en = { + .halt_reg = 0x9c000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x9c000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_0_clkref_en", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_ahb_clk = { + .halt_reg = 0x87020, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x87020, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x87020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_axi_clk = { + .halt_reg = 0x87018, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x87018, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x87018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_axi_hw_ctl_clk = { + .halt_reg = 0x87018, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x87018, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x87018, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_axi_hw_ctl_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_ice_core_clk = { + .halt_reg = 0x8706c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x8706c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x8706c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ice_core_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_ice_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_ice_core_hw_ctl_clk = { + .halt_reg = 0x8706c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x8706c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x8706c, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ice_core_hw_ctl_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_ice_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_phy_aux_clk = { + .halt_reg = 0x870a4, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x870a4, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x870a4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_phy_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_phy_aux_hw_ctl_clk = { + .halt_reg = 0x870a4, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x870a4, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x870a4, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_phy_aux_hw_ctl_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_rx_symbol_0_clk = { + .halt_reg = 0x87028, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x87028, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_rx_symbol_0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_rx_symbol_0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_rx_symbol_1_clk = { + .halt_reg = 0x870c0, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x870c0, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_rx_symbol_1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_rx_symbol_1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_tx_symbol_0_clk = { + .halt_reg = 0x87024, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x87024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_tx_symbol_0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_tx_symbol_0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_unipro_core_clk = { + .halt_reg = 0x87064, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x87064, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x87064, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_unipro_core_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_unipro_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_unipro_core_hw_ctl_clk = { + .halt_reg = 0x87064, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x87064, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x87064, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_unipro_core_hw_ctl_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_unipro_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_master_clk = { + .halt_reg = 0x49018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x49018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_master_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_prim_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_mock_utmi_clk = { + .halt_reg = 0x49024, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x49024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_mock_utmi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_prim_mock_utmi_postdiv_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_sleep_clk = { + .halt_reg = 0x49020, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x49020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_sleep_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_0_clkref_en = { + .halt_reg = 0x9c010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x9c010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_0_clkref_en", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_prim_phy_aux_clk = { + .halt_reg = 0x4905c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4905c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_prim_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_prim_phy_com_aux_clk = { + .halt_reg = 0x49060, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x49060, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_com_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_prim_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_prim_phy_pipe_clk = { + .halt_reg = 0x49064, + .halt_check = BRANCH_HALT_DELAY, + .hwcg_reg = 0x49064, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x49064, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_pipe_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_prim_phy_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_video_axi0_clk = { + .halt_reg = 0x42018, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x42018, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x42018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_video_axi0_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_video_axi1_clk = { + .halt_reg = 0x42020, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x42020, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x42020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_video_axi1_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct gdsc pcie_0_gdsc = { + .gdscr = 0x7b004, + .pd = { + .name = "pcie_0_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc pcie_1_gdsc = { + .gdscr = 0x9d004, + .pd = { + .name = "pcie_1_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc ufs_phy_gdsc = { + .gdscr = 0x87004, + .pd = { + .name = "ufs_phy_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc usb30_prim_gdsc = { + .gdscr = 0x49004, + .pd = { + .name = "usb30_prim_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct clk_regmap *gcc_sm8450_clocks[] = { + [GCC_AGGRE_NOC_PCIE_0_AXI_CLK] = &gcc_aggre_noc_pcie_0_axi_clk.clkr, + [GCC_AGGRE_NOC_PCIE_1_AXI_CLK] = &gcc_aggre_noc_pcie_1_axi_clk.clkr, + [GCC_AGGRE_UFS_PHY_AXI_CLK] = &gcc_aggre_ufs_phy_axi_clk.clkr, + [GCC_AGGRE_UFS_PHY_AXI_HW_CTL_CLK] = &gcc_aggre_ufs_phy_axi_hw_ctl_clk.clkr, + [GCC_AGGRE_USB3_PRIM_AXI_CLK] = &gcc_aggre_usb3_prim_axi_clk.clkr, + [GCC_BOOT_ROM_AHB_CLK] = &gcc_boot_rom_ahb_clk.clkr, + [GCC_CAMERA_HF_AXI_CLK] = &gcc_camera_hf_axi_clk.clkr, + [GCC_CAMERA_SF_AXI_CLK] = &gcc_camera_sf_axi_clk.clkr, + [GCC_CFG_NOC_PCIE_ANOC_AHB_CLK] = &gcc_cfg_noc_pcie_anoc_ahb_clk.clkr, + [GCC_CFG_NOC_USB3_PRIM_AXI_CLK] = &gcc_cfg_noc_usb3_prim_axi_clk.clkr, + [GCC_DDRSS_GPU_AXI_CLK] = &gcc_ddrss_gpu_axi_clk.clkr, + [GCC_DDRSS_PCIE_SF_TBU_CLK] = &gcc_ddrss_pcie_sf_tbu_clk.clkr, + [GCC_DISP_HF_AXI_CLK] = &gcc_disp_hf_axi_clk.clkr, + [GCC_DISP_SF_AXI_CLK] = &gcc_disp_sf_axi_clk.clkr, + [GCC_EUSB3_0_CLKREF_EN] = &gcc_eusb3_0_clkref_en.clkr, + [GCC_GP1_CLK] = &gcc_gp1_clk.clkr, + [GCC_GP1_CLK_SRC] = &gcc_gp1_clk_src.clkr, + [GCC_GP2_CLK] = &gcc_gp2_clk.clkr, + [GCC_GP2_CLK_SRC] = &gcc_gp2_clk_src.clkr, + [GCC_GP3_CLK] = &gcc_gp3_clk.clkr, + [GCC_GP3_CLK_SRC] = &gcc_gp3_clk_src.clkr, + [GCC_GPLL0] = &gcc_gpll0.clkr, + [GCC_GPLL0_OUT_EVEN] = &gcc_gpll0_out_even.clkr, + [GCC_GPLL4] = &gcc_gpll4.clkr, + [GCC_GPLL9] = &gcc_gpll9.clkr, + [GCC_GPU_GPLL0_CLK_SRC] = &gcc_gpu_gpll0_clk_src.clkr, + [GCC_GPU_GPLL0_DIV_CLK_SRC] = &gcc_gpu_gpll0_div_clk_src.clkr, + [GCC_GPU_MEMNOC_GFX_CLK] = &gcc_gpu_memnoc_gfx_clk.clkr, + [GCC_GPU_SNOC_DVM_GFX_CLK] = &gcc_gpu_snoc_dvm_gfx_clk.clkr, + [GCC_PCIE_0_AUX_CLK] = &gcc_pcie_0_aux_clk.clkr, + [GCC_PCIE_0_AUX_CLK_SRC] = &gcc_pcie_0_aux_clk_src.clkr, + [GCC_PCIE_0_CFG_AHB_CLK] = &gcc_pcie_0_cfg_ahb_clk.clkr, + [GCC_PCIE_0_CLKREF_EN] = &gcc_pcie_0_clkref_en.clkr, + [GCC_PCIE_0_MSTR_AXI_CLK] = &gcc_pcie_0_mstr_axi_clk.clkr, + [GCC_PCIE_0_PHY_RCHNG_CLK] = &gcc_pcie_0_phy_rchng_clk.clkr, + [GCC_PCIE_0_PHY_RCHNG_CLK_SRC] = &gcc_pcie_0_phy_rchng_clk_src.clkr, + [GCC_PCIE_0_PIPE_CLK] = &gcc_pcie_0_pipe_clk.clkr, + [GCC_PCIE_0_PIPE_CLK_SRC] = &gcc_pcie_0_pipe_clk_src.clkr, + [GCC_PCIE_0_SLV_AXI_CLK] = &gcc_pcie_0_slv_axi_clk.clkr, + [GCC_PCIE_0_SLV_Q2A_AXI_CLK] = &gcc_pcie_0_slv_q2a_axi_clk.clkr, + [GCC_PCIE_1_AUX_CLK] = &gcc_pcie_1_aux_clk.clkr, + [GCC_PCIE_1_AUX_CLK_SRC] = &gcc_pcie_1_aux_clk_src.clkr, + [GCC_PCIE_1_CFG_AHB_CLK] = &gcc_pcie_1_cfg_ahb_clk.clkr, + [GCC_PCIE_1_CLKREF_EN] = &gcc_pcie_1_clkref_en.clkr, + [GCC_PCIE_1_MSTR_AXI_CLK] = &gcc_pcie_1_mstr_axi_clk.clkr, + [GCC_PCIE_1_PHY_AUX_CLK] = &gcc_pcie_1_phy_aux_clk.clkr, + [GCC_PCIE_1_PHY_AUX_CLK_SRC] = &gcc_pcie_1_phy_aux_clk_src.clkr, + [GCC_PCIE_1_PHY_RCHNG_CLK] = &gcc_pcie_1_phy_rchng_clk.clkr, + [GCC_PCIE_1_PHY_RCHNG_CLK_SRC] = &gcc_pcie_1_phy_rchng_clk_src.clkr, + [GCC_PCIE_1_PIPE_CLK] = &gcc_pcie_1_pipe_clk.clkr, + [GCC_PCIE_1_PIPE_CLK_SRC] = &gcc_pcie_1_pipe_clk_src.clkr, + [GCC_PCIE_1_SLV_AXI_CLK] = &gcc_pcie_1_slv_axi_clk.clkr, + [GCC_PCIE_1_SLV_Q2A_AXI_CLK] = &gcc_pcie_1_slv_q2a_axi_clk.clkr, + [GCC_PDM2_CLK] = &gcc_pdm2_clk.clkr, + [GCC_PDM2_CLK_SRC] = &gcc_pdm2_clk_src.clkr, + [GCC_PDM_AHB_CLK] = &gcc_pdm_ahb_clk.clkr, + [GCC_PDM_XO4_CLK] = &gcc_pdm_xo4_clk.clkr, + [GCC_QMIP_CAMERA_NRT_AHB_CLK] = &gcc_qmip_camera_nrt_ahb_clk.clkr, + [GCC_QMIP_CAMERA_RT_AHB_CLK] = &gcc_qmip_camera_rt_ahb_clk.clkr, + [GCC_QMIP_DISP_AHB_CLK] = &gcc_qmip_disp_ahb_clk.clkr, + [GCC_QMIP_GPU_AHB_CLK] = &gcc_qmip_gpu_ahb_clk.clkr, + [GCC_QMIP_PCIE_AHB_CLK] = &gcc_qmip_pcie_ahb_clk.clkr, + [GCC_QMIP_VIDEO_CV_CPU_AHB_CLK] = &gcc_qmip_video_cv_cpu_ahb_clk.clkr, + [GCC_QMIP_VIDEO_CVP_AHB_CLK] = &gcc_qmip_video_cvp_ahb_clk.clkr, + [GCC_QMIP_VIDEO_V_CPU_AHB_CLK] = &gcc_qmip_video_v_cpu_ahb_clk.clkr, + [GCC_QMIP_VIDEO_VCODEC_AHB_CLK] = &gcc_qmip_video_vcodec_ahb_clk.clkr, + [GCC_QUPV3_WRAP0_CORE_2X_CLK] = &gcc_qupv3_wrap0_core_2x_clk.clkr, + [GCC_QUPV3_WRAP0_CORE_CLK] = &gcc_qupv3_wrap0_core_clk.clkr, + [GCC_QUPV3_WRAP0_S0_CLK] = &gcc_qupv3_wrap0_s0_clk.clkr, + [GCC_QUPV3_WRAP0_S0_CLK_SRC] = &gcc_qupv3_wrap0_s0_clk_src.clkr, + [GCC_QUPV3_WRAP0_S1_CLK] = &gcc_qupv3_wrap0_s1_clk.clkr, + [GCC_QUPV3_WRAP0_S1_CLK_SRC] = &gcc_qupv3_wrap0_s1_clk_src.clkr, + [GCC_QUPV3_WRAP0_S2_CLK] = &gcc_qupv3_wrap0_s2_clk.clkr, + [GCC_QUPV3_WRAP0_S2_CLK_SRC] = &gcc_qupv3_wrap0_s2_clk_src.clkr, + [GCC_QUPV3_WRAP0_S3_CLK] = &gcc_qupv3_wrap0_s3_clk.clkr, + [GCC_QUPV3_WRAP0_S3_CLK_SRC] = &gcc_qupv3_wrap0_s3_clk_src.clkr, + [GCC_QUPV3_WRAP0_S4_CLK] = &gcc_qupv3_wrap0_s4_clk.clkr, + [GCC_QUPV3_WRAP0_S4_CLK_SRC] = &gcc_qupv3_wrap0_s4_clk_src.clkr, + [GCC_QUPV3_WRAP0_S5_CLK] = &gcc_qupv3_wrap0_s5_clk.clkr, + [GCC_QUPV3_WRAP0_S5_CLK_SRC] = &gcc_qupv3_wrap0_s5_clk_src.clkr, + [GCC_QUPV3_WRAP0_S6_CLK] = &gcc_qupv3_wrap0_s6_clk.clkr, + [GCC_QUPV3_WRAP0_S6_CLK_SRC] = &gcc_qupv3_wrap0_s6_clk_src.clkr, + [GCC_QUPV3_WRAP0_S7_CLK] = &gcc_qupv3_wrap0_s7_clk.clkr, + [GCC_QUPV3_WRAP0_S7_CLK_SRC] = &gcc_qupv3_wrap0_s7_clk_src.clkr, + [GCC_QUPV3_WRAP1_CORE_2X_CLK] = &gcc_qupv3_wrap1_core_2x_clk.clkr, + [GCC_QUPV3_WRAP1_CORE_CLK] = &gcc_qupv3_wrap1_core_clk.clkr, + [GCC_QUPV3_WRAP1_S0_CLK] = &gcc_qupv3_wrap1_s0_clk.clkr, + [GCC_QUPV3_WRAP1_S0_CLK_SRC] = &gcc_qupv3_wrap1_s0_clk_src.clkr, + [GCC_QUPV3_WRAP1_S1_CLK] = &gcc_qupv3_wrap1_s1_clk.clkr, + [GCC_QUPV3_WRAP1_S1_CLK_SRC] = &gcc_qupv3_wrap1_s1_clk_src.clkr, + [GCC_QUPV3_WRAP1_S2_CLK] = &gcc_qupv3_wrap1_s2_clk.clkr, + [GCC_QUPV3_WRAP1_S2_CLK_SRC] = &gcc_qupv3_wrap1_s2_clk_src.clkr, + [GCC_QUPV3_WRAP1_S3_CLK] = &gcc_qupv3_wrap1_s3_clk.clkr, + [GCC_QUPV3_WRAP1_S3_CLK_SRC] = &gcc_qupv3_wrap1_s3_clk_src.clkr, + [GCC_QUPV3_WRAP1_S4_CLK] = &gcc_qupv3_wrap1_s4_clk.clkr, + [GCC_QUPV3_WRAP1_S4_CLK_SRC] = &gcc_qupv3_wrap1_s4_clk_src.clkr, + [GCC_QUPV3_WRAP1_S5_CLK] = &gcc_qupv3_wrap1_s5_clk.clkr, + [GCC_QUPV3_WRAP1_S5_CLK_SRC] = &gcc_qupv3_wrap1_s5_clk_src.clkr, + [GCC_QUPV3_WRAP1_S6_CLK] = &gcc_qupv3_wrap1_s6_clk.clkr, + [GCC_QUPV3_WRAP1_S6_CLK_SRC] = &gcc_qupv3_wrap1_s6_clk_src.clkr, + [GCC_QUPV3_WRAP2_CORE_2X_CLK] = &gcc_qupv3_wrap2_core_2x_clk.clkr, + [GCC_QUPV3_WRAP2_CORE_CLK] = &gcc_qupv3_wrap2_core_clk.clkr, + [GCC_QUPV3_WRAP2_S0_CLK] = &gcc_qupv3_wrap2_s0_clk.clkr, + [GCC_QUPV3_WRAP2_S0_CLK_SRC] = &gcc_qupv3_wrap2_s0_clk_src.clkr, + [GCC_QUPV3_WRAP2_S1_CLK] = &gcc_qupv3_wrap2_s1_clk.clkr, + [GCC_QUPV3_WRAP2_S1_CLK_SRC] = &gcc_qupv3_wrap2_s1_clk_src.clkr, + [GCC_QUPV3_WRAP2_S2_CLK] = &gcc_qupv3_wrap2_s2_clk.clkr, + [GCC_QUPV3_WRAP2_S2_CLK_SRC] = &gcc_qupv3_wrap2_s2_clk_src.clkr, + [GCC_QUPV3_WRAP2_S3_CLK] = &gcc_qupv3_wrap2_s3_clk.clkr, + [GCC_QUPV3_WRAP2_S3_CLK_SRC] = &gcc_qupv3_wrap2_s3_clk_src.clkr, + [GCC_QUPV3_WRAP2_S4_CLK] = &gcc_qupv3_wrap2_s4_clk.clkr, + [GCC_QUPV3_WRAP2_S4_CLK_SRC] = &gcc_qupv3_wrap2_s4_clk_src.clkr, + [GCC_QUPV3_WRAP2_S5_CLK] = &gcc_qupv3_wrap2_s5_clk.clkr, + [GCC_QUPV3_WRAP2_S5_CLK_SRC] = &gcc_qupv3_wrap2_s5_clk_src.clkr, + [GCC_QUPV3_WRAP2_S6_CLK] = &gcc_qupv3_wrap2_s6_clk.clkr, + [GCC_QUPV3_WRAP2_S6_CLK_SRC] = &gcc_qupv3_wrap2_s6_clk_src.clkr, + [GCC_QUPV3_WRAP_0_M_AHB_CLK] = &gcc_qupv3_wrap_0_m_ahb_clk.clkr, + [GCC_QUPV3_WRAP_0_S_AHB_CLK] = &gcc_qupv3_wrap_0_s_ahb_clk.clkr, + [GCC_QUPV3_WRAP_1_M_AHB_CLK] = &gcc_qupv3_wrap_1_m_ahb_clk.clkr, + [GCC_QUPV3_WRAP_1_S_AHB_CLK] = &gcc_qupv3_wrap_1_s_ahb_clk.clkr, + [GCC_QUPV3_WRAP_2_M_AHB_CLK] = &gcc_qupv3_wrap_2_m_ahb_clk.clkr, + [GCC_QUPV3_WRAP_2_S_AHB_CLK] = &gcc_qupv3_wrap_2_s_ahb_clk.clkr, + [GCC_SDCC2_AHB_CLK] = &gcc_sdcc2_ahb_clk.clkr, + [GCC_SDCC2_APPS_CLK] = &gcc_sdcc2_apps_clk.clkr, + [GCC_SDCC2_APPS_CLK_SRC] = &gcc_sdcc2_apps_clk_src.clkr, + [GCC_SDCC2_AT_CLK] = &gcc_sdcc2_at_clk.clkr, + [GCC_SDCC4_AHB_CLK] = &gcc_sdcc4_ahb_clk.clkr, + [GCC_SDCC4_APPS_CLK] = &gcc_sdcc4_apps_clk.clkr, + [GCC_SDCC4_APPS_CLK_SRC] = &gcc_sdcc4_apps_clk_src.clkr, + [GCC_SDCC4_AT_CLK] = &gcc_sdcc4_at_clk.clkr, + [GCC_UFS_0_CLKREF_EN] = &gcc_ufs_0_clkref_en.clkr, + [GCC_UFS_PHY_AHB_CLK] = &gcc_ufs_phy_ahb_clk.clkr, + [GCC_UFS_PHY_AXI_CLK] = &gcc_ufs_phy_axi_clk.clkr, + [GCC_UFS_PHY_AXI_CLK_SRC] = &gcc_ufs_phy_axi_clk_src.clkr, + [GCC_UFS_PHY_AXI_HW_CTL_CLK] = &gcc_ufs_phy_axi_hw_ctl_clk.clkr, + [GCC_UFS_PHY_ICE_CORE_CLK] = &gcc_ufs_phy_ice_core_clk.clkr, + [GCC_UFS_PHY_ICE_CORE_CLK_SRC] = &gcc_ufs_phy_ice_core_clk_src.clkr, + [GCC_UFS_PHY_ICE_CORE_HW_CTL_CLK] = &gcc_ufs_phy_ice_core_hw_ctl_clk.clkr, + [GCC_UFS_PHY_PHY_AUX_CLK] = &gcc_ufs_phy_phy_aux_clk.clkr, + [GCC_UFS_PHY_PHY_AUX_CLK_SRC] = &gcc_ufs_phy_phy_aux_clk_src.clkr, + [GCC_UFS_PHY_PHY_AUX_HW_CTL_CLK] = &gcc_ufs_phy_phy_aux_hw_ctl_clk.clkr, + [GCC_UFS_PHY_RX_SYMBOL_0_CLK] = &gcc_ufs_phy_rx_symbol_0_clk.clkr, + [GCC_UFS_PHY_RX_SYMBOL_0_CLK_SRC] = &gcc_ufs_phy_rx_symbol_0_clk_src.clkr, + [GCC_UFS_PHY_RX_SYMBOL_1_CLK] = &gcc_ufs_phy_rx_symbol_1_clk.clkr, + [GCC_UFS_PHY_RX_SYMBOL_1_CLK_SRC] = &gcc_ufs_phy_rx_symbol_1_clk_src.clkr, + [GCC_UFS_PHY_TX_SYMBOL_0_CLK] = &gcc_ufs_phy_tx_symbol_0_clk.clkr, + [GCC_UFS_PHY_TX_SYMBOL_0_CLK_SRC] = &gcc_ufs_phy_tx_symbol_0_clk_src.clkr, + [GCC_UFS_PHY_UNIPRO_CORE_CLK] = &gcc_ufs_phy_unipro_core_clk.clkr, + [GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC] = &gcc_ufs_phy_unipro_core_clk_src.clkr, + [GCC_UFS_PHY_UNIPRO_CORE_HW_CTL_CLK] = &gcc_ufs_phy_unipro_core_hw_ctl_clk.clkr, + [GCC_USB30_PRIM_MASTER_CLK] = &gcc_usb30_prim_master_clk.clkr, + [GCC_USB30_PRIM_MASTER_CLK_SRC] = &gcc_usb30_prim_master_clk_src.clkr, + [GCC_USB30_PRIM_MOCK_UTMI_CLK] = &gcc_usb30_prim_mock_utmi_clk.clkr, + [GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC] = &gcc_usb30_prim_mock_utmi_clk_src.clkr, + [GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC] = &gcc_usb30_prim_mock_utmi_postdiv_clk_src.clkr, + [GCC_USB30_PRIM_SLEEP_CLK] = &gcc_usb30_prim_sleep_clk.clkr, + [GCC_USB3_0_CLKREF_EN] = &gcc_usb3_0_clkref_en.clkr, + [GCC_USB3_PRIM_PHY_AUX_CLK] = &gcc_usb3_prim_phy_aux_clk.clkr, + [GCC_USB3_PRIM_PHY_AUX_CLK_SRC] = &gcc_usb3_prim_phy_aux_clk_src.clkr, + [GCC_USB3_PRIM_PHY_COM_AUX_CLK] = &gcc_usb3_prim_phy_com_aux_clk.clkr, + [GCC_USB3_PRIM_PHY_PIPE_CLK] = &gcc_usb3_prim_phy_pipe_clk.clkr, + [GCC_USB3_PRIM_PHY_PIPE_CLK_SRC] = &gcc_usb3_prim_phy_pipe_clk_src.clkr, + [GCC_VIDEO_AXI0_CLK] = &gcc_video_axi0_clk.clkr, + [GCC_VIDEO_AXI1_CLK] = &gcc_video_axi1_clk.clkr, +}; + +static const struct qcom_reset_map gcc_sm8450_resets[] = { + [GCC_CAMERA_BCR] = { 0x36000 }, + [GCC_DISPLAY_BCR] = { 0x37000 }, + [GCC_GPU_BCR] = { 0x81000 }, + [GCC_PCIE_0_BCR] = { 0x7b000 }, + [GCC_PCIE_0_LINK_DOWN_BCR] = { 0x7c014 }, + [GCC_PCIE_0_NOCSR_COM_PHY_BCR] = { 0x7c020 }, + [GCC_PCIE_0_PHY_BCR] = { 0x7c01c }, + [GCC_PCIE_0_PHY_NOCSR_COM_PHY_BCR] = { 0x7c028 }, + [GCC_PCIE_1_BCR] = { 0x9d000 }, + [GCC_PCIE_1_LINK_DOWN_BCR] = { 0x9e014 }, + [GCC_PCIE_1_NOCSR_COM_PHY_BCR] = { 0x9e020 }, + [GCC_PCIE_1_PHY_BCR] = { 0x9e01c }, + [GCC_PCIE_1_PHY_NOCSR_COM_PHY_BCR] = { 0x9e000 }, + [GCC_PCIE_PHY_BCR] = { 0x7f000 }, + [GCC_PCIE_PHY_CFG_AHB_BCR] = { 0x7f00c }, + [GCC_PCIE_PHY_COM_BCR] = { 0x7f010 }, + [GCC_PDM_BCR] = { 0x43000 }, + [GCC_QUPV3_WRAPPER_0_BCR] = { 0x27000 }, + [GCC_QUPV3_WRAPPER_1_BCR] = { 0x28000 }, + [GCC_QUPV3_WRAPPER_2_BCR] = { 0x2e000 }, + [GCC_QUSB2PHY_PRIM_BCR] = { 0x22000 }, + [GCC_QUSB2PHY_SEC_BCR] = { 0x22004 }, + [GCC_SDCC2_BCR] = { 0x24000 }, + [GCC_SDCC4_BCR] = { 0x26000 }, + [GCC_UFS_PHY_BCR] = { 0x87000 }, + [GCC_USB30_PRIM_BCR] = { 0x49000 }, + [GCC_USB3_DP_PHY_PRIM_BCR] = { 0x60008 }, + [GCC_USB3_DP_PHY_SEC_BCR] = { 0x60014 }, + [GCC_USB3_PHY_PRIM_BCR] = { 0x60000 }, + [GCC_USB3_PHY_SEC_BCR] = { 0x6000c }, + [GCC_USB3PHY_PHY_PRIM_BCR] = { 0x60004 }, + [GCC_USB3PHY_PHY_SEC_BCR] = { 0x60010 }, + [GCC_USB_PHY_CFG_AHB2PHY_BCR] = { 0x7a000 }, + [GCC_VIDEO_AXI0_CLK_ARES] = { 0x42018, 2 }, + [GCC_VIDEO_AXI1_CLK_ARES] = { 0x42020, 2 }, + [GCC_VIDEO_BCR] = { 0x42000 }, +}; + +static const struct clk_rcg_dfs_data gcc_dfs_clocks[] = { + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s0_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s1_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s2_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s3_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s4_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s5_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s6_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s7_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s0_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s1_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s2_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s3_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s4_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s5_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s6_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap2_s0_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap2_s1_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap2_s2_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap2_s3_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap2_s4_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap2_s5_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap2_s6_clk_src), +}; + +static struct gdsc *gcc_sm8450_gdscs[] = { + [PCIE_0_GDSC] = &pcie_0_gdsc, + [PCIE_1_GDSC] = &pcie_1_gdsc, + [UFS_PHY_GDSC] = &ufs_phy_gdsc, + [USB30_PRIM_GDSC] = &usb30_prim_gdsc, +}; + +static const struct regmap_config gcc_sm8450_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x1f1030, + .fast_io = true, +}; + +static const struct qcom_cc_desc gcc_sm8450_desc = { + .config = &gcc_sm8450_regmap_config, + .clks = gcc_sm8450_clocks, + .num_clks = ARRAY_SIZE(gcc_sm8450_clocks), + .resets = gcc_sm8450_resets, + .num_resets = ARRAY_SIZE(gcc_sm8450_resets), + .gdscs = gcc_sm8450_gdscs, + .num_gdscs = ARRAY_SIZE(gcc_sm8450_gdscs), +}; + +static const struct of_device_id gcc_sm8450_match_table[] = { + { .compatible = "qcom,gcc-sm8450" }, + { } +}; +MODULE_DEVICE_TABLE(of, gcc_sm8450_match_table); + +static int gcc_sm8450_probe(struct platform_device *pdev) +{ + struct regmap *regmap; + int ret; + + regmap = qcom_cc_map(pdev, &gcc_sm8450_desc); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + ret = qcom_cc_register_rcg_dfs(regmap, gcc_dfs_clocks, + ARRAY_SIZE(gcc_dfs_clocks)); + if (ret) + return ret; + + /* FORCE_MEM_CORE_ON for ufs phy ice core clocks */ + regmap_update_bits(regmap, gcc_ufs_phy_ice_core_clk.halt_reg, BIT(14), BIT(14)); + + /* + * Keep the critical clock always-On + * gcc_camera_ahb_clk, gcc_camera_xo_clk, gcc_disp_ahb_clk, + * gcc_disp_xo_clk, gcc_gpu_cfg_ahb_clk, gcc_video_ahb_clk, + * gcc_video_xo_clk + */ + regmap_update_bits(regmap, 0x36004, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x36020, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x37004, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x3701c, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x81004, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x42004, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x42028, BIT(0), BIT(0)); + + return qcom_cc_really_probe(pdev, &gcc_sm8450_desc, regmap); +} + +static struct platform_driver gcc_sm8450_driver = { + .probe = gcc_sm8450_probe, + .driver = { + .name = "gcc-sm8450", + .of_match_table = gcc_sm8450_match_table, + }, +}; + +static int __init gcc_sm8450_init(void) +{ + return platform_driver_register(&gcc_sm8450_driver); +} +subsys_initcall(gcc_sm8450_init); + +static void __exit gcc_sm8450_exit(void) +{ + platform_driver_unregister(&gcc_sm8450_driver); +} +module_exit(gcc_sm8450_exit); + +MODULE_DESCRIPTION("QTI GCC SM8450 Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/qcom/lpasscc-sc7280.c b/drivers/clk/qcom/lpasscc-sc7280.c index 89f1ad6631da..b39ee1c9647b 100644 --- a/drivers/clk/qcom/lpasscc-sc7280.c +++ b/drivers/clk/qcom/lpasscc-sc7280.c @@ -3,6 +3,7 @@ * Copyright (c) 2021, The Linux Foundation. All rights reserved. */ +#include #include #include #include diff --git a/drivers/clk/qcom/lpasscc-sdm845.c b/drivers/clk/qcom/lpasscc-sdm845.c index 56d3e9928892..7040da952728 100644 --- a/drivers/clk/qcom/lpasscc-sdm845.c +++ b/drivers/clk/qcom/lpasscc-sdm845.c @@ -3,6 +3,7 @@ * Copyright (c) 2018, The Linux Foundation. All rights reserved. */ +#include #include #include #include diff --git a/drivers/clk/qcom/mmcc-apq8084.c b/drivers/clk/qcom/mmcc-apq8084.c index fbfcf0006739..e9f971359155 100644 --- a/drivers/clk/qcom/mmcc-apq8084.c +++ b/drivers/clk/qcom/mmcc-apq8084.c @@ -3,6 +3,7 @@ * Copyright (c) 2014-2015, The Linux Foundation. All rights reserved. */ +#include #include #include #include diff --git a/drivers/clk/qcom/q6sstop-qcs404.c b/drivers/clk/qcom/q6sstop-qcs404.c index 507386bee07d..780074e05841 100644 --- a/drivers/clk/qcom/q6sstop-qcs404.c +++ b/drivers/clk/qcom/q6sstop-qcs404.c @@ -4,6 +4,7 @@ */ #include +#include #include #include #include diff --git a/drivers/clk/qcom/turingcc-qcs404.c b/drivers/clk/qcom/turingcc-qcs404.c index 4543bda793f4..43184459228f 100644 --- a/drivers/clk/qcom/turingcc-qcs404.c +++ b/drivers/clk/qcom/turingcc-qcs404.c @@ -4,6 +4,7 @@ */ #include +#include #include #include #include diff --git a/drivers/clk/renesas/Kconfig b/drivers/clk/renesas/Kconfig index 6d0280751bb1..be6e6ae7448c 100644 --- a/drivers/clk/renesas/Kconfig +++ b/drivers/clk/renesas/Kconfig @@ -31,6 +31,7 @@ config CLK_RENESAS select CLK_R8A77990 if ARCH_R8A77990 select CLK_R8A77995 if ARCH_R8A77995 select CLK_R8A779A0 if ARCH_R8A779A0 + select CLK_R8A779F0 if ARCH_R8A779F0 select CLK_R9A06G032 if ARCH_R9A06G032 select CLK_R9A07G044 if ARCH_R9A07G044 select CLK_SH73A0 if ARCH_SH73A0 @@ -149,8 +150,11 @@ config CLK_R8A77995 config CLK_R8A779A0 bool "R-Car V3U clock support" if COMPILE_TEST - select CLK_RCAR_CPG_LIB - select CLK_RENESAS_CPG_MSSR + select CLK_RCAR_GEN4_CPG + +config CLK_R8A779F0 + bool "R-Car S4-8 clock support" if COMPILE_TEST + select CLK_RCAR_GEN4_CPG config CLK_R9A06G032 bool "RZ/N1D clock support" if COMPILE_TEST @@ -178,6 +182,11 @@ config CLK_RCAR_GEN3_CPG select CLK_RCAR_CPG_LIB select CLK_RENESAS_CPG_MSSR +config CLK_RCAR_GEN4_CPG + bool "R-Car Gen4 clock support" if COMPILE_TEST + select CLK_RCAR_CPG_LIB + select CLK_RENESAS_CPG_MSSR + config CLK_RCAR_USB2_CLOCK_SEL bool "Renesas R-Car USB2 clock selector support" depends on ARCH_RENESAS || COMPILE_TEST diff --git a/drivers/clk/renesas/Makefile b/drivers/clk/renesas/Makefile index 7d018700d08b..8b34db1a328c 100644 --- a/drivers/clk/renesas/Makefile +++ b/drivers/clk/renesas/Makefile @@ -28,6 +28,7 @@ obj-$(CONFIG_CLK_R8A77980) += r8a77980-cpg-mssr.o obj-$(CONFIG_CLK_R8A77990) += r8a77990-cpg-mssr.o obj-$(CONFIG_CLK_R8A77995) += r8a77995-cpg-mssr.o obj-$(CONFIG_CLK_R8A779A0) += r8a779a0-cpg-mssr.o +obj-$(CONFIG_CLK_R8A779F0) += r8a779f0-cpg-mssr.o obj-$(CONFIG_CLK_R9A06G032) += r9a06g032-clocks.o obj-$(CONFIG_CLK_R9A07G044) += r9a07g044-cpg.o obj-$(CONFIG_CLK_SH73A0) += clk-sh73a0.o @@ -36,6 +37,7 @@ obj-$(CONFIG_CLK_SH73A0) += clk-sh73a0.o obj-$(CONFIG_CLK_RCAR_CPG_LIB) += rcar-cpg-lib.o obj-$(CONFIG_CLK_RCAR_GEN2_CPG) += rcar-gen2-cpg.o obj-$(CONFIG_CLK_RCAR_GEN3_CPG) += rcar-gen3-cpg.o +obj-$(CONFIG_CLK_RCAR_GEN4_CPG) += rcar-gen4-cpg.o obj-$(CONFIG_CLK_RCAR_USB2_CLOCK_SEL) += rcar-usb2-clock-sel.o obj-$(CONFIG_CLK_RZG2L) += rzg2l-cpg.o diff --git a/drivers/clk/renesas/r8a774a1-cpg-mssr.c b/drivers/clk/renesas/r8a774a1-cpg-mssr.c index 39b185d8e957..95dd56b64d64 100644 --- a/drivers/clk/renesas/r8a774a1-cpg-mssr.c +++ b/drivers/clk/renesas/r8a774a1-cpg-mssr.c @@ -100,10 +100,14 @@ static const struct cpg_core_clk r8a774a1_core_clks[] __initconst = { DEF_FIXED("s3d2", R8A774A1_CLK_S3D2, CLK_S3, 2, 1), DEF_FIXED("s3d4", R8A774A1_CLK_S3D4, CLK_S3, 4, 1), - DEF_GEN3_SD("sd0", R8A774A1_CLK_SD0, CLK_SDSRC, 0x074), - DEF_GEN3_SD("sd1", R8A774A1_CLK_SD1, CLK_SDSRC, 0x078), - DEF_GEN3_SD("sd2", R8A774A1_CLK_SD2, CLK_SDSRC, 0x268), - DEF_GEN3_SD("sd3", R8A774A1_CLK_SD3, CLK_SDSRC, 0x26c), + DEF_GEN3_SDH("sd0h", R8A774A1_CLK_SD0H, CLK_SDSRC, 0x074), + DEF_GEN3_SDH("sd1h", R8A774A1_CLK_SD1H, CLK_SDSRC, 0x078), + DEF_GEN3_SDH("sd2h", R8A774A1_CLK_SD2H, CLK_SDSRC, 0x268), + DEF_GEN3_SDH("sd3h", R8A774A1_CLK_SD3H, CLK_SDSRC, 0x26c), + DEF_GEN3_SD("sd0", R8A774A1_CLK_SD0, R8A774A1_CLK_SD0H, 0x074), + DEF_GEN3_SD("sd1", R8A774A1_CLK_SD1, R8A774A1_CLK_SD1H, 0x078), + DEF_GEN3_SD("sd2", R8A774A1_CLK_SD2, R8A774A1_CLK_SD2H, 0x268), + DEF_GEN3_SD("sd3", R8A774A1_CLK_SD3, R8A774A1_CLK_SD3H, 0x26c), DEF_FIXED("cl", R8A774A1_CLK_CL, CLK_PLL1_DIV2, 48, 1), DEF_FIXED("cp", R8A774A1_CLK_CP, CLK_EXTAL, 2, 1), diff --git a/drivers/clk/renesas/r8a774b1-cpg-mssr.c b/drivers/clk/renesas/r8a774b1-cpg-mssr.c index af602d83c8ce..56061b9b8437 100644 --- a/drivers/clk/renesas/r8a774b1-cpg-mssr.c +++ b/drivers/clk/renesas/r8a774b1-cpg-mssr.c @@ -97,10 +97,14 @@ static const struct cpg_core_clk r8a774b1_core_clks[] __initconst = { DEF_FIXED("s3d2", R8A774B1_CLK_S3D2, CLK_S3, 2, 1), DEF_FIXED("s3d4", R8A774B1_CLK_S3D4, CLK_S3, 4, 1), - DEF_GEN3_SD("sd0", R8A774B1_CLK_SD0, CLK_SDSRC, 0x074), - DEF_GEN3_SD("sd1", R8A774B1_CLK_SD1, CLK_SDSRC, 0x078), - DEF_GEN3_SD("sd2", R8A774B1_CLK_SD2, CLK_SDSRC, 0x268), - DEF_GEN3_SD("sd3", R8A774B1_CLK_SD3, CLK_SDSRC, 0x26c), + DEF_GEN3_SDH("sd0h", R8A774B1_CLK_SD0H, CLK_SDSRC, 0x074), + DEF_GEN3_SDH("sd1h", R8A774B1_CLK_SD1H, CLK_SDSRC, 0x078), + DEF_GEN3_SDH("sd2h", R8A774B1_CLK_SD2H, CLK_SDSRC, 0x268), + DEF_GEN3_SDH("sd3h", R8A774B1_CLK_SD3H, CLK_SDSRC, 0x26c), + DEF_GEN3_SD("sd0", R8A774B1_CLK_SD0, R8A774B1_CLK_SD0H, 0x074), + DEF_GEN3_SD("sd1", R8A774B1_CLK_SD1, R8A774B1_CLK_SD1H, 0x078), + DEF_GEN3_SD("sd2", R8A774B1_CLK_SD2, R8A774B1_CLK_SD2H, 0x268), + DEF_GEN3_SD("sd3", R8A774B1_CLK_SD3, R8A774B1_CLK_SD3H, 0x26c), DEF_FIXED("cl", R8A774B1_CLK_CL, CLK_PLL1_DIV2, 48, 1), DEF_FIXED("cp", R8A774B1_CLK_CP, CLK_EXTAL, 2, 1), diff --git a/drivers/clk/renesas/r8a774c0-cpg-mssr.c b/drivers/clk/renesas/r8a774c0-cpg-mssr.c index 5b938eb2df25..b5eb5dc45d62 100644 --- a/drivers/clk/renesas/r8a774c0-cpg-mssr.c +++ b/drivers/clk/renesas/r8a774c0-cpg-mssr.c @@ -108,9 +108,12 @@ static const struct cpg_core_clk r8a774c0_core_clks[] __initconst = { DEF_FIXED("s3d2", R8A774C0_CLK_S3D2, CLK_S3, 2, 1), DEF_FIXED("s3d4", R8A774C0_CLK_S3D4, CLK_S3, 4, 1), - DEF_GEN3_SD("sd0", R8A774C0_CLK_SD0, CLK_SDSRC, 0x0074), - DEF_GEN3_SD("sd1", R8A774C0_CLK_SD1, CLK_SDSRC, 0x0078), - DEF_GEN3_SD("sd3", R8A774C0_CLK_SD3, CLK_SDSRC, 0x026c), + DEF_GEN3_SDH("sd0h", R8A774C0_CLK_SD0H, CLK_SDSRC, 0x0074), + DEF_GEN3_SDH("sd1h", R8A774C0_CLK_SD1H, CLK_SDSRC, 0x0078), + DEF_GEN3_SDH("sd3h", R8A774C0_CLK_SD3H, CLK_SDSRC, 0x026c), + DEF_GEN3_SD("sd0", R8A774C0_CLK_SD0, R8A774C0_CLK_SD0H, 0x0074), + DEF_GEN3_SD("sd1", R8A774C0_CLK_SD1, R8A774C0_CLK_SD1H, 0x0078), + DEF_GEN3_SD("sd3", R8A774C0_CLK_SD3, R8A774C0_CLK_SD3H, 0x026c), DEF_FIXED("cl", R8A774C0_CLK_CL, CLK_PLL1, 48, 1), DEF_FIXED("cp", R8A774C0_CLK_CP, CLK_EXTAL, 2, 1), diff --git a/drivers/clk/renesas/r8a774e1-cpg-mssr.c b/drivers/clk/renesas/r8a774e1-cpg-mssr.c index 40c71466df37..2950f0db90ae 100644 --- a/drivers/clk/renesas/r8a774e1-cpg-mssr.c +++ b/drivers/clk/renesas/r8a774e1-cpg-mssr.c @@ -100,10 +100,14 @@ static const struct cpg_core_clk r8a774e1_core_clks[] __initconst = { DEF_FIXED("s3d2", R8A774E1_CLK_S3D2, CLK_S3, 2, 1), DEF_FIXED("s3d4", R8A774E1_CLK_S3D4, CLK_S3, 4, 1), - DEF_GEN3_SD("sd0", R8A774E1_CLK_SD0, CLK_SDSRC, 0x074), - DEF_GEN3_SD("sd1", R8A774E1_CLK_SD1, CLK_SDSRC, 0x078), - DEF_GEN3_SD("sd2", R8A774E1_CLK_SD2, CLK_SDSRC, 0x268), - DEF_GEN3_SD("sd3", R8A774E1_CLK_SD3, CLK_SDSRC, 0x26c), + DEF_GEN3_SDH("sd0h", R8A774E1_CLK_SD0H, CLK_SDSRC, 0x074), + DEF_GEN3_SDH("sd1h", R8A774E1_CLK_SD1H, CLK_SDSRC, 0x078), + DEF_GEN3_SDH("sd2h", R8A774E1_CLK_SD2H, CLK_SDSRC, 0x268), + DEF_GEN3_SDH("sd3h", R8A774E1_CLK_SD3H, CLK_SDSRC, 0x26c), + DEF_GEN3_SD("sd0", R8A774E1_CLK_SD0, R8A774E1_CLK_SD0H, 0x074), + DEF_GEN3_SD("sd1", R8A774E1_CLK_SD1, R8A774E1_CLK_SD1H, 0x078), + DEF_GEN3_SD("sd2", R8A774E1_CLK_SD2, R8A774E1_CLK_SD2H, 0x268), + DEF_GEN3_SD("sd3", R8A774E1_CLK_SD3, R8A774E1_CLK_SD3H, 0x26c), DEF_FIXED("cl", R8A774E1_CLK_CL, CLK_PLL1_DIV2, 48, 1), DEF_FIXED("cr", R8A774E1_CLK_CR, CLK_PLL1_DIV4, 2, 1), diff --git a/drivers/clk/renesas/r8a7795-cpg-mssr.c b/drivers/clk/renesas/r8a7795-cpg-mssr.c index d6b1d0148bfd..991a44315d71 100644 --- a/drivers/clk/renesas/r8a7795-cpg-mssr.c +++ b/drivers/clk/renesas/r8a7795-cpg-mssr.c @@ -104,10 +104,14 @@ static struct cpg_core_clk r8a7795_core_clks[] __initdata = { DEF_FIXED("s3d2", R8A7795_CLK_S3D2, CLK_S3, 2, 1), DEF_FIXED("s3d4", R8A7795_CLK_S3D4, CLK_S3, 4, 1), - DEF_GEN3_SD("sd0", R8A7795_CLK_SD0, CLK_SDSRC, 0x074), - DEF_GEN3_SD("sd1", R8A7795_CLK_SD1, CLK_SDSRC, 0x078), - DEF_GEN3_SD("sd2", R8A7795_CLK_SD2, CLK_SDSRC, 0x268), - DEF_GEN3_SD("sd3", R8A7795_CLK_SD3, CLK_SDSRC, 0x26c), + DEF_GEN3_SDH("sd0h", R8A7795_CLK_SD0H, CLK_SDSRC, 0x074), + DEF_GEN3_SDH("sd1h", R8A7795_CLK_SD1H, CLK_SDSRC, 0x078), + DEF_GEN3_SDH("sd2h", R8A7795_CLK_SD2H, CLK_SDSRC, 0x268), + DEF_GEN3_SDH("sd3h", R8A7795_CLK_SD3H, CLK_SDSRC, 0x26c), + DEF_GEN3_SD("sd0", R8A7795_CLK_SD0, R8A7795_CLK_SD0H, 0x074), + DEF_GEN3_SD("sd1", R8A7795_CLK_SD1, R8A7795_CLK_SD1H, 0x078), + DEF_GEN3_SD("sd2", R8A7795_CLK_SD2, R8A7795_CLK_SD2H, 0x268), + DEF_GEN3_SD("sd3", R8A7795_CLK_SD3, R8A7795_CLK_SD3H, 0x26c), DEF_FIXED("cl", R8A7795_CLK_CL, CLK_PLL1_DIV2, 48, 1), DEF_FIXED("cr", R8A7795_CLK_CR, CLK_PLL1_DIV4, 2, 1), diff --git a/drivers/clk/renesas/r8a7796-cpg-mssr.c b/drivers/clk/renesas/r8a7796-cpg-mssr.c index 9c22977e42c2..7950313611ef 100644 --- a/drivers/clk/renesas/r8a7796-cpg-mssr.c +++ b/drivers/clk/renesas/r8a7796-cpg-mssr.c @@ -106,10 +106,14 @@ static const struct cpg_core_clk r8a7796_core_clks[] __initconst = { DEF_FIXED("s3d2", R8A7796_CLK_S3D2, CLK_S3, 2, 1), DEF_FIXED("s3d4", R8A7796_CLK_S3D4, CLK_S3, 4, 1), - DEF_GEN3_SD("sd0", R8A7796_CLK_SD0, CLK_SDSRC, 0x074), - DEF_GEN3_SD("sd1", R8A7796_CLK_SD1, CLK_SDSRC, 0x078), - DEF_GEN3_SD("sd2", R8A7796_CLK_SD2, CLK_SDSRC, 0x268), - DEF_GEN3_SD("sd3", R8A7796_CLK_SD3, CLK_SDSRC, 0x26c), + DEF_GEN3_SDH("sd0h", R8A7796_CLK_SD0H, CLK_SDSRC, 0x074), + DEF_GEN3_SDH("sd1h", R8A7796_CLK_SD1H, CLK_SDSRC, 0x078), + DEF_GEN3_SDH("sd2h", R8A7796_CLK_SD2H, CLK_SDSRC, 0x268), + DEF_GEN3_SDH("sd3h", R8A7796_CLK_SD3H, CLK_SDSRC, 0x26c), + DEF_GEN3_SD("sd0", R8A7796_CLK_SD0, R8A7796_CLK_SD0H, 0x074), + DEF_GEN3_SD("sd1", R8A7796_CLK_SD1, R8A7796_CLK_SD1H, 0x078), + DEF_GEN3_SD("sd2", R8A7796_CLK_SD2, R8A7796_CLK_SD2H, 0x268), + DEF_GEN3_SD("sd3", R8A7796_CLK_SD3, R8A7796_CLK_SD3H, 0x26c), DEF_FIXED("cl", R8A7796_CLK_CL, CLK_PLL1_DIV2, 48, 1), DEF_FIXED("cr", R8A7796_CLK_CR, CLK_PLL1_DIV4, 2, 1), diff --git a/drivers/clk/renesas/r8a77965-cpg-mssr.c b/drivers/clk/renesas/r8a77965-cpg-mssr.c index 7eee45a31b2a..d687c29efa3c 100644 --- a/drivers/clk/renesas/r8a77965-cpg-mssr.c +++ b/drivers/clk/renesas/r8a77965-cpg-mssr.c @@ -101,10 +101,14 @@ static const struct cpg_core_clk r8a77965_core_clks[] __initconst = { DEF_FIXED("s3d2", R8A77965_CLK_S3D2, CLK_S3, 2, 1), DEF_FIXED("s3d4", R8A77965_CLK_S3D4, CLK_S3, 4, 1), - DEF_GEN3_SD("sd0", R8A77965_CLK_SD0, CLK_SDSRC, 0x074), - DEF_GEN3_SD("sd1", R8A77965_CLK_SD1, CLK_SDSRC, 0x078), - DEF_GEN3_SD("sd2", R8A77965_CLK_SD2, CLK_SDSRC, 0x268), - DEF_GEN3_SD("sd3", R8A77965_CLK_SD3, CLK_SDSRC, 0x26c), + DEF_GEN3_SDH("sd0h", R8A77965_CLK_SD0H, CLK_SDSRC, 0x074), + DEF_GEN3_SDH("sd1h", R8A77965_CLK_SD1H, CLK_SDSRC, 0x078), + DEF_GEN3_SDH("sd2h", R8A77965_CLK_SD2H, CLK_SDSRC, 0x268), + DEF_GEN3_SDH("sd3h", R8A77965_CLK_SD3H, CLK_SDSRC, 0x26c), + DEF_GEN3_SD("sd0", R8A77965_CLK_SD0, R8A77965_CLK_SD0H, 0x074), + DEF_GEN3_SD("sd1", R8A77965_CLK_SD1, R8A77965_CLK_SD1H, 0x078), + DEF_GEN3_SD("sd2", R8A77965_CLK_SD2, R8A77965_CLK_SD2H, 0x268), + DEF_GEN3_SD("sd3", R8A77965_CLK_SD3, R8A77965_CLK_SD3H, 0x26c), DEF_FIXED("cl", R8A77965_CLK_CL, CLK_PLL1_DIV2, 48, 1), DEF_FIXED("cr", R8A77965_CLK_CR, CLK_PLL1_DIV4, 2, 1), diff --git a/drivers/clk/renesas/r8a77980-cpg-mssr.c b/drivers/clk/renesas/r8a77980-cpg-mssr.c index 9fe372286c1e..f3cd64de4dc6 100644 --- a/drivers/clk/renesas/r8a77980-cpg-mssr.c +++ b/drivers/clk/renesas/r8a77980-cpg-mssr.c @@ -96,7 +96,8 @@ static const struct cpg_core_clk r8a77980_core_clks[] __initconst = { DEF_FIXED("s3d2", R8A77980_CLK_S3D2, CLK_S3, 2, 1), DEF_FIXED("s3d4", R8A77980_CLK_S3D4, CLK_S3, 4, 1), - DEF_GEN3_SD("sd0", R8A77980_CLK_SD0, CLK_SDSRC, 0x0074), + DEF_GEN3_SDH("sd0h", R8A77980_CLK_SD0H, CLK_SDSRC, 0x0074), + DEF_GEN3_SD("sd0", R8A77980_CLK_SD0, R8A77980_CLK_SD0H, 0x0074), DEF_FIXED("cl", R8A77980_CLK_CL, CLK_PLL1_DIV2, 48, 1), DEF_FIXED("cp", R8A77980_CLK_CP, CLK_EXTAL, 2, 1), diff --git a/drivers/clk/renesas/r8a77990-cpg-mssr.c b/drivers/clk/renesas/r8a77990-cpg-mssr.c index a582f2ec3294..faf60f7adc8d 100644 --- a/drivers/clk/renesas/r8a77990-cpg-mssr.c +++ b/drivers/clk/renesas/r8a77990-cpg-mssr.c @@ -100,9 +100,12 @@ static const struct cpg_core_clk r8a77990_core_clks[] __initconst = { DEF_FIXED("s3d2", R8A77990_CLK_S3D2, CLK_S3, 2, 1), DEF_FIXED("s3d4", R8A77990_CLK_S3D4, CLK_S3, 4, 1), - DEF_GEN3_SD("sd0", R8A77990_CLK_SD0, CLK_SDSRC, 0x0074), - DEF_GEN3_SD("sd1", R8A77990_CLK_SD1, CLK_SDSRC, 0x0078), - DEF_GEN3_SD("sd3", R8A77990_CLK_SD3, CLK_SDSRC, 0x026c), + DEF_GEN3_SDH("sd0h", R8A77990_CLK_SD0H, CLK_SDSRC, 0x0074), + DEF_GEN3_SDH("sd1h", R8A77990_CLK_SD1H, CLK_SDSRC, 0x0078), + DEF_GEN3_SDH("sd3h", R8A77990_CLK_SD3H, CLK_SDSRC, 0x026c), + DEF_GEN3_SD("sd0", R8A77990_CLK_SD0, R8A77990_CLK_SD0H, 0x0074), + DEF_GEN3_SD("sd1", R8A77990_CLK_SD1, R8A77990_CLK_SD1H, 0x0078), + DEF_GEN3_SD("sd3", R8A77990_CLK_SD3, R8A77990_CLK_SD3H, 0x026c), DEF_FIXED("cl", R8A77990_CLK_CL, CLK_PLL1, 48, 1), DEF_FIXED("cr", R8A77990_CLK_CR, CLK_PLL1D2, 2, 1), diff --git a/drivers/clk/renesas/r8a77995-cpg-mssr.c b/drivers/clk/renesas/r8a77995-cpg-mssr.c index 81c0bc1e78af..7713cfd99c1d 100644 --- a/drivers/clk/renesas/r8a77995-cpg-mssr.c +++ b/drivers/clk/renesas/r8a77995-cpg-mssr.c @@ -103,7 +103,8 @@ static const struct cpg_core_clk r8a77995_core_clks[] __initconst = { DEF_GEN3_PE("s3d2c", R8A77995_CLK_S3D2C, CLK_S3, 2, CLK_PE, 2), DEF_GEN3_PE("s3d4c", R8A77995_CLK_S3D4C, CLK_S3, 4, CLK_PE, 4), - DEF_GEN3_SD("sd0", R8A77995_CLK_SD0, CLK_SDSRC, 0x268), + DEF_GEN3_SDH("sd0h", R8A77995_CLK_SD0H, CLK_SDSRC, 0x268), + DEF_GEN3_SD("sd0", R8A77995_CLK_SD0, R8A77995_CLK_SD0H, 0x268), DEF_DIV6P1("canfd", R8A77995_CLK_CANFD, CLK_PLL0D3, 0x244), DEF_DIV6P1("mso", R8A77995_CLK_MSO, CLK_PLL1D2, 0x014), diff --git a/drivers/clk/renesas/r8a779a0-cpg-mssr.c b/drivers/clk/renesas/r8a779a0-cpg-mssr.c index fbd7454f2beb..1c09d4ebe90f 100644 --- a/drivers/clk/renesas/r8a779a0-cpg-mssr.c +++ b/drivers/clk/renesas/r8a779a0-cpg-mssr.c @@ -10,46 +10,19 @@ * Copyright (C) 2015 Renesas Electronics Corp. */ -#include #include #include #include #include #include #include -#include #include -#include -#include #include #include -#include "rcar-cpg-lib.h" #include "renesas-cpg-mssr.h" - -enum rcar_r8a779a0_clk_types { - CLK_TYPE_R8A779A0_MAIN = CLK_TYPE_CUSTOM, - CLK_TYPE_R8A779A0_PLL1, - CLK_TYPE_R8A779A0_PLL2X_3X, /* PLL[23][01] */ - CLK_TYPE_R8A779A0_PLL5, - CLK_TYPE_R8A779A0_Z, - CLK_TYPE_R8A779A0_SD, - CLK_TYPE_R8A779A0_MDSEL, /* Select parent/divider using mode pin */ - CLK_TYPE_R8A779A0_OSC, /* OSC EXTAL predivider and fixed divider */ - CLK_TYPE_R8A779A0_RPCSRC, - CLK_TYPE_R8A779A0_RPC, - CLK_TYPE_R8A779A0_RPCD2, -}; - -struct rcar_r8a779a0_cpg_pll_config { - u8 extal_div; - u8 pll1_mult; - u8 pll1_div; - u8 pll5_mult; - u8 pll5_div; - u8 osc_prediv; -}; +#include "rcar-gen4-cpg.h" enum clk_ids { /* Core Clock Outputs exported to DT */ @@ -85,33 +58,18 @@ enum clk_ids { }; #define DEF_PLL(_name, _id, _offset) \ - DEF_BASE(_name, _id, CLK_TYPE_R8A779A0_PLL2X_3X, CLK_MAIN, \ + DEF_BASE(_name, _id, CLK_TYPE_GEN4_PLL2X_3X, CLK_MAIN, \ .offset = _offset) -#define DEF_Z(_name, _id, _parent, _div, _offset) \ - DEF_BASE(_name, _id, CLK_TYPE_R8A779A0_Z, _parent, .div = _div, \ - .offset = _offset) - -#define DEF_SD(_name, _id, _parent, _offset) \ - DEF_BASE(_name, _id, CLK_TYPE_R8A779A0_SD, _parent, .offset = _offset) - -#define DEF_MDSEL(_name, _id, _md, _parent0, _div0, _parent1, _div1) \ - DEF_BASE(_name, _id, CLK_TYPE_R8A779A0_MDSEL, \ - (_parent0) << 16 | (_parent1), \ - .div = (_div0) << 16 | (_div1), .offset = _md) - -#define DEF_OSC(_name, _id, _parent, _div) \ - DEF_BASE(_name, _id, CLK_TYPE_R8A779A0_OSC, _parent, .div = _div) - static const struct cpg_core_clk r8a779a0_core_clks[] __initconst = { /* External Clock Inputs */ DEF_INPUT("extal", CLK_EXTAL), DEF_INPUT("extalr", CLK_EXTALR), /* Internal Core Clocks */ - DEF_BASE(".main", CLK_MAIN, CLK_TYPE_R8A779A0_MAIN, CLK_EXTAL), - DEF_BASE(".pll1", CLK_PLL1, CLK_TYPE_R8A779A0_PLL1, CLK_MAIN), - DEF_BASE(".pll5", CLK_PLL5, CLK_TYPE_R8A779A0_PLL5, CLK_MAIN), + DEF_BASE(".main", CLK_MAIN, CLK_TYPE_GEN4_MAIN, CLK_EXTAL), + DEF_BASE(".pll1", CLK_PLL1, CLK_TYPE_GEN4_PLL1, CLK_MAIN), + DEF_BASE(".pll5", CLK_PLL5, CLK_TYPE_GEN4_PLL5, CLK_MAIN), DEF_PLL(".pll20", CLK_PLL20, 0x0834), DEF_PLL(".pll21", CLK_PLL21, 0x0838), DEF_PLL(".pll30", CLK_PLL30, 0x083c), @@ -128,14 +86,14 @@ static const struct cpg_core_clk r8a779a0_core_clks[] __initconst = { DEF_FIXED(".s3", CLK_S3, CLK_PLL1_DIV2, 4, 1), DEF_FIXED(".sdsrc", CLK_SDSRC, CLK_PLL5_DIV4, 1, 1), DEF_RATE(".oco", CLK_OCO, 32768), - DEF_BASE(".rpcsrc", CLK_RPCSRC, CLK_TYPE_R8A779A0_RPCSRC, CLK_PLL5), - DEF_BASE("rpc", R8A779A0_CLK_RPC, CLK_TYPE_R8A779A0_RPC, CLK_RPCSRC), - DEF_BASE("rpcd2", R8A779A0_CLK_RPCD2, CLK_TYPE_R8A779A0_RPCD2, + DEF_BASE(".rpcsrc", CLK_RPCSRC, CLK_TYPE_GEN4_RPCSRC, CLK_PLL5), + DEF_BASE("rpc", R8A779A0_CLK_RPC, CLK_TYPE_GEN4_RPC, CLK_RPCSRC), + DEF_BASE("rpcd2", R8A779A0_CLK_RPCD2, CLK_TYPE_GEN4_RPCD2, R8A779A0_CLK_RPC), /* Core Clock Outputs */ - DEF_Z("z0", R8A779A0_CLK_Z0, CLK_PLL20, 2, 0), - DEF_Z("z1", R8A779A0_CLK_Z1, CLK_PLL21, 2, 8), + DEF_GEN4_Z("z0", R8A779A0_CLK_Z0, CLK_TYPE_GEN4_Z, CLK_PLL20, 2, 0), + DEF_GEN4_Z("z1", R8A779A0_CLK_Z1, CLK_TYPE_GEN4_Z, CLK_PLL21, 2, 8), DEF_FIXED("zx", R8A779A0_CLK_ZX, CLK_PLL20_DIV2, 2, 1), DEF_FIXED("s1d1", R8A779A0_CLK_S1D1, CLK_S1, 1, 1), DEF_FIXED("s1d2", R8A779A0_CLK_S1D2, CLK_S1, 2, 1), @@ -159,15 +117,16 @@ static const struct cpg_core_clk r8a779a0_core_clks[] __initconst = { DEF_FIXED("cp", R8A779A0_CLK_CP, CLK_EXTAL, 2, 1), DEF_FIXED("cl16mck", R8A779A0_CLK_CL16MCK, CLK_PLL1_DIV2, 64, 1), - DEF_SD("sd0", R8A779A0_CLK_SD0, CLK_SDSRC, 0x870), + DEF_GEN4_SDH("sdh0", R8A779A0_CLK_SD0H, CLK_SDSRC, 0x870), + DEF_GEN4_SD("sd0", R8A779A0_CLK_SD0, R8A779A0_CLK_SD0H, 0x870), DEF_DIV6P1("mso", R8A779A0_CLK_MSO, CLK_PLL5_DIV4, 0x87c), DEF_DIV6P1("canfd", R8A779A0_CLK_CANFD, CLK_PLL5_DIV4, 0x878), DEF_DIV6P1("csi0", R8A779A0_CLK_CSI0, CLK_PLL5_DIV4, 0x880), DEF_DIV6P1("dsi", R8A779A0_CLK_DSI, CLK_PLL5_DIV4, 0x884), - DEF_OSC("osc", R8A779A0_CLK_OSC, CLK_EXTAL, 8), - DEF_MDSEL("r", R8A779A0_CLK_R, 29, CLK_EXTALR, 1, CLK_OCO, 1), + DEF_GEN4_OSC("osc", R8A779A0_CLK_OSC, CLK_EXTAL, 8), + DEF_GEN4_MDSEL("r", R8A779A0_CLK_R, 29, CLK_EXTALR, 1, CLK_OCO, 1), }; static const struct mssr_mod_clk r8a779a0_mod_clks[] __initconst = { @@ -271,256 +230,6 @@ static const struct mssr_mod_clk r8a779a0_mod_clks[] __initconst = { DEF_MOD("vspx3", 1031, R8A779A0_CLK_S1D1), }; -static const struct rcar_r8a779a0_cpg_pll_config *cpg_pll_config __initdata; -static unsigned int cpg_clk_extalr __initdata; -static u32 cpg_mode __initdata; - -/* - * Z0 Clock & Z1 Clock - */ -#define CPG_FRQCRB 0x00000804 -#define CPG_FRQCRB_KICK BIT(31) -#define CPG_FRQCRC 0x00000808 - -struct cpg_z_clk { - struct clk_hw hw; - void __iomem *reg; - void __iomem *kick_reg; - unsigned long max_rate; /* Maximum rate for normal mode */ - unsigned int fixed_div; - u32 mask; -}; - -#define to_z_clk(_hw) container_of(_hw, struct cpg_z_clk, hw) - -static unsigned long cpg_z_clk_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - struct cpg_z_clk *zclk = to_z_clk(hw); - unsigned int mult; - u32 val; - - val = readl(zclk->reg) & zclk->mask; - mult = 32 - (val >> __ffs(zclk->mask)); - - return DIV_ROUND_CLOSEST_ULL((u64)parent_rate * mult, - 32 * zclk->fixed_div); -} - -static int cpg_z_clk_determine_rate(struct clk_hw *hw, - struct clk_rate_request *req) -{ - struct cpg_z_clk *zclk = to_z_clk(hw); - unsigned int min_mult, max_mult, mult; - unsigned long rate, prate; - - rate = min(req->rate, req->max_rate); - if (rate <= zclk->max_rate) { - /* Set parent rate to initial value for normal modes */ - prate = zclk->max_rate; - } else { - /* Set increased parent rate for boost modes */ - prate = rate; - } - req->best_parent_rate = clk_hw_round_rate(clk_hw_get_parent(hw), - prate * zclk->fixed_div); - - prate = req->best_parent_rate / zclk->fixed_div; - min_mult = max(div64_ul(req->min_rate * 32ULL, prate), 1ULL); - max_mult = min(div64_ul(req->max_rate * 32ULL, prate), 32ULL); - if (max_mult < min_mult) - return -EINVAL; - - mult = DIV_ROUND_CLOSEST_ULL(rate * 32ULL, prate); - mult = clamp(mult, min_mult, max_mult); - - req->rate = DIV_ROUND_CLOSEST_ULL((u64)prate * mult, 32); - return 0; -} - -static int cpg_z_clk_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) -{ - struct cpg_z_clk *zclk = to_z_clk(hw); - unsigned int mult; - unsigned int i; - - mult = DIV64_U64_ROUND_CLOSEST(rate * 32ULL * zclk->fixed_div, - parent_rate); - mult = clamp(mult, 1U, 32U); - - if (readl(zclk->kick_reg) & CPG_FRQCRB_KICK) - return -EBUSY; - - cpg_reg_modify(zclk->reg, zclk->mask, (32 - mult) << __ffs(zclk->mask)); - - /* - * Set KICK bit in FRQCRB to update hardware setting and wait for - * clock change completion. - */ - cpg_reg_modify(zclk->kick_reg, 0, CPG_FRQCRB_KICK); - - /* - * Note: There is no HW information about the worst case latency. - * - * Using experimental measurements, it seems that no more than - * ~10 iterations are needed, independently of the CPU rate. - * Since this value might be dependent on external xtal rate, pll1 - * rate or even the other emulation clocks rate, use 1000 as a - * "super" safe value. - */ - for (i = 1000; i; i--) { - if (!(readl(zclk->kick_reg) & CPG_FRQCRB_KICK)) - return 0; - - cpu_relax(); - } - - return -ETIMEDOUT; -} - -static const struct clk_ops cpg_z_clk_ops = { - .recalc_rate = cpg_z_clk_recalc_rate, - .determine_rate = cpg_z_clk_determine_rate, - .set_rate = cpg_z_clk_set_rate, -}; - -static struct clk * __init cpg_z_clk_register(const char *name, - const char *parent_name, - void __iomem *reg, - unsigned int div, - unsigned int offset) -{ - struct clk_init_data init = {}; - struct cpg_z_clk *zclk; - struct clk *clk; - - zclk = kzalloc(sizeof(*zclk), GFP_KERNEL); - if (!zclk) - return ERR_PTR(-ENOMEM); - - init.name = name; - init.ops = &cpg_z_clk_ops; - init.flags = CLK_SET_RATE_PARENT; - init.parent_names = &parent_name; - init.num_parents = 1; - - zclk->reg = reg + CPG_FRQCRC; - zclk->kick_reg = reg + CPG_FRQCRB; - zclk->hw.init = &init; - zclk->mask = GENMASK(offset + 4, offset); - zclk->fixed_div = div; /* PLLVCO x 1/div x SYS-CPU divider */ - - clk = clk_register(NULL, &zclk->hw); - if (IS_ERR(clk)) { - kfree(zclk); - return clk; - } - - zclk->max_rate = clk_hw_get_rate(clk_hw_get_parent(&zclk->hw)) / - zclk->fixed_div; - return clk; -} - -/* - * RPC Clocks - */ -#define CPG_RPCCKCR 0x874 - -static const struct clk_div_table cpg_rpcsrc_div_table[] = { - { 0, 4 }, { 1, 6 }, { 2, 5 }, { 3, 6 }, { 0, 0 }, -}; - -static struct clk * __init rcar_r8a779a0_cpg_clk_register(struct device *dev, - const struct cpg_core_clk *core, const struct cpg_mssr_info *info, - struct clk **clks, void __iomem *base, - struct raw_notifier_head *notifiers) -{ - const struct clk *parent; - unsigned int mult = 1; - unsigned int div = 1; - u32 value; - - parent = clks[core->parent & 0xffff]; /* some types use high bits */ - if (IS_ERR(parent)) - return ERR_CAST(parent); - - switch (core->type) { - case CLK_TYPE_R8A779A0_MAIN: - div = cpg_pll_config->extal_div; - break; - - case CLK_TYPE_R8A779A0_PLL1: - mult = cpg_pll_config->pll1_mult; - div = cpg_pll_config->pll1_div; - break; - - case CLK_TYPE_R8A779A0_PLL2X_3X: - value = readl(base + core->offset); - mult = (((value >> 24) & 0x7f) + 1) * 2; - break; - - case CLK_TYPE_R8A779A0_PLL5: - mult = cpg_pll_config->pll5_mult; - div = cpg_pll_config->pll5_div; - break; - - case CLK_TYPE_R8A779A0_Z: - return cpg_z_clk_register(core->name, __clk_get_name(parent), - base, core->div, core->offset); - - case CLK_TYPE_R8A779A0_SD: - return cpg_sd_clk_register(core->name, base, core->offset, - __clk_get_name(parent), notifiers, - false); - break; - - case CLK_TYPE_R8A779A0_MDSEL: - /* - * Clock selectable between two parents and two fixed dividers - * using a mode pin - */ - if (cpg_mode & BIT(core->offset)) { - div = core->div & 0xffff; - } else { - parent = clks[core->parent >> 16]; - if (IS_ERR(parent)) - return ERR_CAST(parent); - div = core->div >> 16; - } - mult = 1; - break; - - case CLK_TYPE_R8A779A0_OSC: - /* - * Clock combining OSC EXTAL predivider and a fixed divider - */ - div = cpg_pll_config->osc_prediv * core->div; - break; - - case CLK_TYPE_R8A779A0_RPCSRC: - return clk_register_divider_table(NULL, core->name, - __clk_get_name(parent), 0, - base + CPG_RPCCKCR, 3, 2, 0, - cpg_rpcsrc_div_table, - &cpg_lock); - - case CLK_TYPE_R8A779A0_RPC: - return cpg_rpc_clk_register(core->name, base + CPG_RPCCKCR, - __clk_get_name(parent), notifiers); - - case CLK_TYPE_R8A779A0_RPCD2: - return cpg_rpcd2_clk_register(core->name, base + CPG_RPCCKCR, - __clk_get_name(parent)); - - default: - return ERR_PTR(-EINVAL); - } - - return clk_register_fixed_factor(NULL, core->name, - __clk_get_name(parent), 0, mult, div); -} - static const unsigned int r8a779a0_crit_mod_clks[] __initconst = { MOD_CLK_ID(907), /* RWDT */ }; @@ -539,17 +248,19 @@ static const unsigned int r8a779a0_crit_mod_clks[] __initconst = { */ #define CPG_PLL_CONFIG_INDEX(md) ((((md) & BIT(14)) >> 13) | \ (((md) & BIT(13)) >> 13)) - -static const struct rcar_r8a779a0_cpg_pll_config cpg_pll_configs[4] = { - /* EXTAL div PLL1 mult/div PLL5 mult/div OSC prediv */ - { 1, 128, 1, 192, 1, 16, }, - { 1, 106, 1, 160, 1, 19, }, - { 0, 0, 0, 0, 0, 0, }, - { 2, 128, 1, 192, 1, 32, }, +static const struct rcar_gen4_cpg_pll_config cpg_pll_configs[4] = { + /* EXTAL div PLL1 mult/div PLL2 mult/div PLL3 mult/div PLL5 mult/div PLL6 mult/div OSC prediv */ + { 1, 128, 1, 0, 0, 0, 0, 192, 1, 0, 0, 16, }, + { 1, 106, 1, 0, 0, 0, 0, 160, 1, 0, 0, 19, }, + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }, + { 2, 128, 1, 0, 0, 0, 0, 192, 1, 0, 0, 32, }, }; + static int __init r8a779a0_cpg_mssr_init(struct device *dev) { + const struct rcar_gen4_cpg_pll_config *cpg_pll_config; + u32 cpg_mode; int error; error = rcar_rst_read_mode_pins(&cpg_mode); @@ -557,10 +268,8 @@ static int __init r8a779a0_cpg_mssr_init(struct device *dev) return error; cpg_pll_config = &cpg_pll_configs[CPG_PLL_CONFIG_INDEX(cpg_mode)]; - cpg_clk_extalr = CLK_EXTALR; - spin_lock_init(&cpg_lock); - return 0; + return rcar_gen4_cpg_init(cpg_pll_config, CLK_EXTALR, cpg_mode); } const struct cpg_mssr_info r8a779a0_cpg_mssr_info __initconst = { @@ -581,7 +290,7 @@ const struct cpg_mssr_info r8a779a0_cpg_mssr_info __initconst = { /* Callbacks */ .init = r8a779a0_cpg_mssr_init, - .cpg_clk_register = rcar_r8a779a0_cpg_clk_register, + .cpg_clk_register = rcar_gen4_cpg_clk_register, - .reg_layout = CLK_REG_LAYOUT_RCAR_V3U, + .reg_layout = CLK_REG_LAYOUT_RCAR_GEN4, }; diff --git a/drivers/clk/renesas/r8a779f0-cpg-mssr.c b/drivers/clk/renesas/r8a779f0-cpg-mssr.c new file mode 100644 index 000000000000..e6ec02c2c2a8 --- /dev/null +++ b/drivers/clk/renesas/r8a779f0-cpg-mssr.c @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * r8a779f0 Clock Pulse Generator / Module Standby and Software Reset + * + * Copyright (C) 2021 Renesas Electronics Corp. + * + * Based on r8a779a0-cpg-mssr.c + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "renesas-cpg-mssr.h" +#include "rcar-gen4-cpg.h" + +enum clk_ids { + /* Core Clock Outputs exported to DT */ + LAST_DT_CORE_CLK = R8A779F0_CLK_R, + + /* External Input Clocks */ + CLK_EXTAL, + CLK_EXTALR, + + /* Internal Core Clocks */ + CLK_MAIN, + CLK_PLL1, + CLK_PLL2, + CLK_PLL3, + CLK_PLL5, + CLK_PLL6, + CLK_PLL1_DIV2, + CLK_PLL2_DIV2, + CLK_PLL3_DIV2, + CLK_PLL5_DIV2, + CLK_PLL5_DIV4, + CLK_PLL6_DIV2, + CLK_S0, + CLK_SDSRC, + CLK_RPCSRC, + CLK_OCO, + + /* Module Clocks */ + MOD_CLK_BASE +}; + +static const struct cpg_core_clk r8a779f0_core_clks[] __initconst = { + /* External Clock Inputs */ + DEF_INPUT("extal", CLK_EXTAL), + DEF_INPUT("extalr", CLK_EXTALR), + + /* Internal Core Clocks */ + DEF_BASE(".main", CLK_MAIN, CLK_TYPE_GEN4_MAIN, CLK_EXTAL), + DEF_BASE(".pll1", CLK_PLL1, CLK_TYPE_GEN4_PLL1, CLK_MAIN), + DEF_BASE(".pll2", CLK_PLL2, CLK_TYPE_GEN4_PLL2, CLK_MAIN), + DEF_BASE(".pll3", CLK_PLL3, CLK_TYPE_GEN4_PLL3, CLK_MAIN), + DEF_BASE(".pll5", CLK_PLL5, CLK_TYPE_GEN4_PLL5, CLK_MAIN), + DEF_BASE(".pll6", CLK_PLL6, CLK_TYPE_GEN4_PLL6, CLK_MAIN), + + DEF_FIXED(".pll1_div2", CLK_PLL1_DIV2, CLK_PLL1, 2, 1), + DEF_FIXED(".pll2_div2", CLK_PLL2_DIV2, CLK_PLL2, 2, 1), + DEF_FIXED(".pll3_div2", CLK_PLL3_DIV2, CLK_PLL3, 2, 1), + DEF_FIXED(".pll5_div2", CLK_PLL5_DIV2, CLK_PLL5, 2, 1), + DEF_FIXED(".pll5_div4", CLK_PLL5_DIV4, CLK_PLL5_DIV2, 2, 1), + DEF_FIXED(".pll6_div2", CLK_PLL6_DIV2, CLK_PLL6, 2, 1), + DEF_FIXED(".s0", CLK_S0, CLK_PLL1_DIV2, 2, 1), + DEF_BASE(".sdsrc", CLK_SDSRC, CLK_TYPE_GEN4_SDSRC, CLK_PLL5), + DEF_RATE(".oco", CLK_OCO, 32768), + + DEF_BASE(".rpcsrc", CLK_RPCSRC, CLK_TYPE_GEN4_RPCSRC, CLK_PLL5), + DEF_BASE(".rpc", R8A779F0_CLK_RPC, CLK_TYPE_GEN4_RPC, CLK_RPCSRC), + DEF_BASE("rpcd2", R8A779F0_CLK_RPCD2, CLK_TYPE_GEN4_RPCD2, R8A779F0_CLK_RPC), + + /* Core Clock Outputs */ + DEF_FIXED("s0d2", R8A779F0_CLK_S0D2, CLK_S0, 2, 1), + DEF_FIXED("s0d3", R8A779F0_CLK_S0D3, CLK_S0, 3, 1), + DEF_FIXED("s0d4", R8A779F0_CLK_S0D4, CLK_S0, 4, 1), + DEF_FIXED("cl16m", R8A779F0_CLK_CL16M, CLK_S0, 48, 1), + DEF_FIXED("s0d2_mm", R8A779F0_CLK_S0D2_MM, CLK_S0, 2, 1), + DEF_FIXED("s0d3_mm", R8A779F0_CLK_S0D3_MM, CLK_S0, 3, 1), + DEF_FIXED("s0d4_mm", R8A779F0_CLK_S0D4_MM, CLK_S0, 4, 1), + DEF_FIXED("cl16m_mm", R8A779F0_CLK_CL16M_MM, CLK_S0, 48, 1), + DEF_FIXED("s0d2_rt", R8A779F0_CLK_S0D2_RT, CLK_S0, 2, 1), + DEF_FIXED("s0d3_rt", R8A779F0_CLK_S0D3_RT, CLK_S0, 3, 1), + DEF_FIXED("s0d4_rt", R8A779F0_CLK_S0D4_RT, CLK_S0, 4, 1), + DEF_FIXED("s0d6_rt", R8A779F0_CLK_S0D6_RT, CLK_S0, 6, 1), + DEF_FIXED("cl16m_rt", R8A779F0_CLK_CL16M_RT, CLK_S0, 48, 1), + DEF_FIXED("s0d3_per", R8A779F0_CLK_S0D3_PER, CLK_S0, 3, 1), + DEF_FIXED("s0d6_per", R8A779F0_CLK_S0D6_PER, CLK_S0, 6, 1), + DEF_FIXED("s0d12_per", R8A779F0_CLK_S0D12_PER, CLK_S0, 12, 1), + DEF_FIXED("s0d24_per", R8A779F0_CLK_S0D24_PER, CLK_S0, 24, 1), + DEF_FIXED("cl16m_per", R8A779F0_CLK_CL16M_PER, CLK_S0, 48, 1), + DEF_FIXED("s0d2_hsc", R8A779F0_CLK_S0D2_HSC, CLK_S0, 2, 1), + DEF_FIXED("s0d3_hsc", R8A779F0_CLK_S0D3_HSC, CLK_S0, 3, 1), + DEF_FIXED("s0d4_hsc", R8A779F0_CLK_S0D4_HSC, CLK_S0, 4, 1), + DEF_FIXED("s0d6_hsc", R8A779F0_CLK_S0D6_HSC, CLK_S0, 6, 1), + DEF_FIXED("s0d12_hsc", R8A779F0_CLK_S0D12_HSC, CLK_S0, 12, 1), + DEF_FIXED("cl16m_hsc", R8A779F0_CLK_CL16M_HSC, CLK_S0, 48, 1), + DEF_FIXED("s0d2_cc", R8A779F0_CLK_S0D2_CC, CLK_S0, 2, 1), + DEF_FIXED("rsw2", R8A779F0_CLK_RSW2, CLK_PLL5, 2, 1), + DEF_FIXED("cbfusa", R8A779F0_CLK_CBFUSA, CLK_EXTAL, 2, 1), + DEF_FIXED("cpex", R8A779F0_CLK_CPEX, CLK_EXTAL, 2, 1), + + DEF_GEN4_SD("sd0", R8A779F0_CLK_SD0, CLK_SDSRC, 0x870), + DEF_DIV6P1("mso", R8A779F0_CLK_MSO, CLK_PLL5_DIV4, 0x87c), + + DEF_GEN4_OSC("osc", R8A779F0_CLK_OSC, CLK_EXTAL, 8), + DEF_GEN4_MDSEL("r", R8A779F0_CLK_R, 29, CLK_EXTALR, 1, CLK_OCO, 1), +}; + +static const struct mssr_mod_clk r8a779f0_mod_clks[] __initconst = { + DEF_MOD("scif0", 702, R8A779F0_CLK_S0D12_PER), + DEF_MOD("scif1", 703, R8A779F0_CLK_S0D12_PER), + DEF_MOD("scif3", 704, R8A779F0_CLK_S0D12_PER), + DEF_MOD("scif4", 705, R8A779F0_CLK_S0D12_PER), +}; + +/* + * CPG Clock Data + */ +/* + * MD EXTAL PLL1 PLL2 PLL3 PLL5 PLL6 OSC + * 14 13 (MHz) + * ---------------------------------------------------------------- + * 0 0 16 / 1 x200 x150 x200 x200 x134 /15 + * 0 1 20 / 1 x160 x120 x160 x160 x106 /19 + * 1 0 Prohibited setting + * 1 1 40 / 2 x160 x120 x160 x160 x106 /38 + */ +#define CPG_PLL_CONFIG_INDEX(md) ((((md) & BIT(14)) >> 13) | \ + (((md) & BIT(13)) >> 13)) + +static const struct rcar_gen4_cpg_pll_config cpg_pll_configs[4] = { + /* EXTAL div PLL1 mult/div PLL2 mult/div PLL3 mult/div PLL5 mult/div PLL6 mult/div OSC prediv */ + { 1, 200, 1, 150, 1, 200, 1, 200, 1, 134, 1, 15, }, + { 1, 160, 1, 120, 1, 160, 1, 160, 1, 106, 1, 19, }, + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }, + { 2, 160, 1, 120, 1, 160, 1, 160, 1, 106, 1, 38, }, +}; + +static int __init r8a779f0_cpg_mssr_init(struct device *dev) +{ + const struct rcar_gen4_cpg_pll_config *cpg_pll_config; + u32 cpg_mode; + int error; + + error = rcar_rst_read_mode_pins(&cpg_mode); + if (error) + return error; + + cpg_pll_config = &cpg_pll_configs[CPG_PLL_CONFIG_INDEX(cpg_mode)]; + if (!cpg_pll_config->extal_div) { + dev_err(dev, "Prohibited setting (cpg_mode=0x%x)\n", cpg_mode); + return -EINVAL; + } + + return rcar_gen4_cpg_init(cpg_pll_config, CLK_EXTALR, cpg_mode); +} + +const struct cpg_mssr_info r8a779f0_cpg_mssr_info __initconst = { + /* Core Clocks */ + .core_clks = r8a779f0_core_clks, + .num_core_clks = ARRAY_SIZE(r8a779f0_core_clks), + .last_dt_core_clk = LAST_DT_CORE_CLK, + .num_total_core_clks = MOD_CLK_BASE, + + /* Module Clocks */ + .mod_clks = r8a779f0_mod_clks, + .num_mod_clks = ARRAY_SIZE(r8a779f0_mod_clks), + .num_hw_mod_clks = 28 * 32, + + /* Callbacks */ + .init = r8a779f0_cpg_mssr_init, + .cpg_clk_register = rcar_gen4_cpg_clk_register, + + .reg_layout = CLK_REG_LAYOUT_RCAR_GEN4, +}; diff --git a/drivers/clk/renesas/r9a07g044-cpg.c b/drivers/clk/renesas/r9a07g044-cpg.c index 47c16265fca9..79042bf46fe8 100644 --- a/drivers/clk/renesas/r9a07g044-cpg.c +++ b/drivers/clk/renesas/r9a07g044-cpg.c @@ -26,15 +26,15 @@ enum clk_ids { CLK_PLL1, CLK_PLL2, CLK_PLL2_DIV2, - CLK_PLL2_DIV16, - CLK_PLL2_DIV20, + CLK_PLL2_DIV2_8, + CLK_PLL2_DIV2_10, CLK_PLL3, CLK_PLL3_400, CLK_PLL3_533, CLK_PLL3_DIV2, + CLK_PLL3_DIV2_2, CLK_PLL3_DIV2_4, CLK_PLL3_DIV2_4_2, - CLK_PLL3_DIV4, CLK_SEL_PLL3_3, CLK_DIV_PLL3_C, CLK_PLL4, @@ -50,12 +50,21 @@ enum clk_ids { CLK_PLL2_SDHI_266, CLK_SD0_DIV4, CLK_SD1_DIV4, + CLK_SEL_GPU2, /* Module Clocks */ MOD_CLK_BASE, }; /* Divider tables */ +static const struct clk_div_table dtable_1_8[] = { + {0, 1}, + {1, 2}, + {2, 4}, + {3, 8}, + {0, 0}, +}; + static const struct clk_div_table dtable_1_32[] = { {0, 1}, {1, 2}, @@ -69,6 +78,7 @@ static const struct clk_div_table dtable_1_32[] = { static const char * const sel_pll3_3[] = { ".pll3_533", ".pll3_400" }; static const char * const sel_pll6_2[] = { ".pll6_250", ".pll5_250" }; static const char * const sel_shdi[] = { ".clk_533", ".clk_400", ".clk_266" }; +static const char * const sel_gpu2[] = { ".pll6", ".pll3_div2_2" }; static const struct cpg_core_clk r9a07g044_core_clks[] __initconst = { /* External Clock Inputs */ @@ -94,13 +104,13 @@ static const struct cpg_core_clk r9a07g044_core_clks[] __initconst = { DEF_FIXED(".clk_400", CLK_PLL2_SDHI_400, CLK_PLL2_800, 1, 2), DEF_FIXED(".clk_266", CLK_PLL2_SDHI_266, CLK_PLL2_SDHI_533, 1, 2), - DEF_FIXED(".pll2_div16", CLK_PLL2_DIV16, CLK_PLL2, 1, 16), - DEF_FIXED(".pll2_div20", CLK_PLL2_DIV20, CLK_PLL2, 1, 20), + DEF_FIXED(".pll2_div2_8", CLK_PLL2_DIV2_8, CLK_PLL2_DIV2, 1, 8), + DEF_FIXED(".pll2_div2_10", CLK_PLL2_DIV2_10, CLK_PLL2_DIV2, 1, 10), DEF_FIXED(".pll3_div2", CLK_PLL3_DIV2, CLK_PLL3, 1, 2), + DEF_FIXED(".pll3_div2_2", CLK_PLL3_DIV2_2, CLK_PLL3_DIV2, 1, 2), DEF_FIXED(".pll3_div2_4", CLK_PLL3_DIV2_4, CLK_PLL3_DIV2, 1, 4), DEF_FIXED(".pll3_div2_4_2", CLK_PLL3_DIV2_4_2, CLK_PLL3_DIV2_4, 1, 2), - DEF_FIXED(".pll3_div4", CLK_PLL3_DIV4, CLK_PLL3, 1, 4), DEF_MUX(".sel_pll3_3", CLK_SEL_PLL3_3, SEL_PLL3_3, sel_pll3_3, ARRAY_SIZE(sel_pll3_3), 0, CLK_MUX_READ_ONLY), DEF_DIV("divpl3c", CLK_DIV_PLL3_C, CLK_SEL_PLL3_3, @@ -108,13 +118,16 @@ static const struct cpg_core_clk r9a07g044_core_clks[] __initconst = { DEF_FIXED(".pll5_250", CLK_PLL5_250, CLK_PLL5_FOUT3, 1, 2), DEF_FIXED(".pll6_250", CLK_PLL6_250, CLK_PLL6, 1, 2), + DEF_MUX(".sel_gpu2", CLK_SEL_GPU2, SEL_GPU2, + sel_gpu2, ARRAY_SIZE(sel_gpu2), 0, CLK_MUX_READ_ONLY), /* Core output clk */ - DEF_FIXED("I", R9A07G044_CLK_I, CLK_PLL1, 1, 1), - DEF_DIV("P0", R9A07G044_CLK_P0, CLK_PLL2_DIV16, DIVPL2A, + DEF_DIV("I", R9A07G044_CLK_I, CLK_PLL1, DIVPL1A, dtable_1_8, + CLK_DIVIDER_HIWORD_MASK), + DEF_DIV("P0", R9A07G044_CLK_P0, CLK_PLL2_DIV2_8, DIVPL2A, dtable_1_32, CLK_DIVIDER_HIWORD_MASK), DEF_FIXED("P0_DIV2", R9A07G044_CLK_P0_DIV2, R9A07G044_CLK_P0, 1, 2), - DEF_FIXED("TSU", R9A07G044_CLK_TSU, CLK_PLL2_DIV20, 1, 1), + DEF_FIXED("TSU", R9A07G044_CLK_TSU, CLK_PLL2_DIV2_10, 1, 1), DEF_DIV("P1", R9A07G044_CLK_P1, CLK_PLL3_DIV2_4, DIVPL3B, dtable_1_32, CLK_DIVIDER_HIWORD_MASK), DEF_FIXED("P1_DIV2", CLK_P1_DIV2, R9A07G044_CLK_P1, 1, 2), @@ -132,6 +145,8 @@ static const struct cpg_core_clk r9a07g044_core_clks[] __initconst = { sel_shdi, ARRAY_SIZE(sel_shdi)), DEF_FIXED("SD0_DIV4", CLK_SD0_DIV4, R9A07G044_CLK_SD0, 1, 4), DEF_FIXED("SD1_DIV4", CLK_SD1_DIV4, R9A07G044_CLK_SD1, 1, 4), + DEF_DIV("G", R9A07G044_CLK_G, CLK_SEL_GPU2, DIVGPU, dtable_1_8, + CLK_DIVIDER_HIWORD_MASK), }; static struct rzg2l_mod_clk r9a07g044_mod_clks[] = { @@ -145,6 +160,24 @@ static struct rzg2l_mod_clk r9a07g044_mod_clks[] = { 0x52c, 0), DEF_MOD("dmac_pclk", R9A07G044_DMAC_PCLK, CLK_P1_DIV2, 0x52c, 1), + DEF_MOD("ostm0_pclk", R9A07G044_OSTM0_PCLK, R9A07G044_CLK_P0, + 0x534, 0), + DEF_MOD("ostm1_clk", R9A07G044_OSTM1_PCLK, R9A07G044_CLK_P0, + 0x534, 1), + DEF_MOD("ostm2_pclk", R9A07G044_OSTM2_PCLK, R9A07G044_CLK_P0, + 0x534, 2), + DEF_MOD("wdt0_pclk", R9A07G044_WDT0_PCLK, R9A07G044_CLK_P0, + 0x548, 0), + DEF_MOD("wdt0_clk", R9A07G044_WDT0_CLK, R9A07G044_OSCCLK, + 0x548, 1), + DEF_MOD("wdt1_pclk", R9A07G044_WDT1_PCLK, R9A07G044_CLK_P0, + 0x548, 2), + DEF_MOD("wdt1_clk", R9A07G044_WDT1_CLK, R9A07G044_OSCCLK, + 0x548, 3), + DEF_MOD("wdt2_pclk", R9A07G044_WDT2_PCLK, R9A07G044_CLK_P0, + 0x548, 4), + DEF_MOD("wdt2_clk", R9A07G044_WDT2_CLK, R9A07G044_OSCCLK, + 0x548, 5), DEF_MOD("spi_clk2", R9A07G044_SPI_CLK2, R9A07G044_CLK_SPI1, 0x550, 0), DEF_MOD("spi_clk", R9A07G044_SPI_CLK, R9A07G044_CLK_SPI0, @@ -165,6 +198,12 @@ static struct rzg2l_mod_clk r9a07g044_mod_clks[] = { 0x554, 6), DEF_MOD("sdhi1_aclk", R9A07G044_SDHI1_ACLK, R9A07G044_CLK_P1, 0x554, 7), + DEF_MOD("gpu_clk", R9A07G044_GPU_CLK, R9A07G044_CLK_G, + 0x558, 0), + DEF_MOD("gpu_axi_clk", R9A07G044_GPU_AXI_CLK, R9A07G044_CLK_P1, + 0x558, 1), + DEF_MOD("gpu_ace_clk", R9A07G044_GPU_ACE_CLK, R9A07G044_CLK_P1, + 0x558, 2), DEF_MOD("ssi0_pclk", R9A07G044_SSI0_PCLK2, R9A07G044_CLK_P0, 0x570, 0), DEF_MOD("ssi0_sfr", R9A07G044_SSI0_PCLK_SFR, R9A07G044_CLK_P0, @@ -217,6 +256,14 @@ static struct rzg2l_mod_clk r9a07g044_mod_clks[] = { 0x584, 4), DEF_MOD("sci0", R9A07G044_SCI0_CLKP, R9A07G044_CLK_P0, 0x588, 0), + DEF_MOD("sci1", R9A07G044_SCI1_CLKP, R9A07G044_CLK_P0, + 0x588, 1), + DEF_MOD("rspi0", R9A07G044_RSPI0_CLKB, R9A07G044_CLK_P0, + 0x590, 0), + DEF_MOD("rspi1", R9A07G044_RSPI1_CLKB, R9A07G044_CLK_P0, + 0x590, 1), + DEF_MOD("rspi2", R9A07G044_RSPI2_CLKB, R9A07G044_CLK_P0, + 0x590, 2), DEF_MOD("canfd", R9A07G044_CANFD_PCLK, R9A07G044_CLK_P0, 0x594, 0), DEF_MOD("gpio", R9A07G044_GPIO_HCLK, R9A07G044_OSCCLK, @@ -225,6 +272,8 @@ static struct rzg2l_mod_clk r9a07g044_mod_clks[] = { 0x5a8, 0), DEF_MOD("adc_pclk", R9A07G044_ADC_PCLK, R9A07G044_CLK_P0, 0x5a8, 1), + DEF_MOD("tsu_pclk", R9A07G044_TSU_PCLK, R9A07G044_CLK_TSU, + 0x5ac, 0), }; static struct rzg2l_reset r9a07g044_resets[] = { @@ -233,9 +282,18 @@ static struct rzg2l_reset r9a07g044_resets[] = { DEF_RST(R9A07G044_IA55_RESETN, 0x818, 0), DEF_RST(R9A07G044_DMAC_ARESETN, 0x82c, 0), DEF_RST(R9A07G044_DMAC_RST_ASYNC, 0x82c, 1), + DEF_RST(R9A07G044_OSTM0_PRESETZ, 0x834, 0), + DEF_RST(R9A07G044_OSTM1_PRESETZ, 0x834, 1), + DEF_RST(R9A07G044_OSTM2_PRESETZ, 0x834, 2), + DEF_RST(R9A07G044_WDT0_PRESETN, 0x848, 0), + DEF_RST(R9A07G044_WDT1_PRESETN, 0x848, 1), + DEF_RST(R9A07G044_WDT2_PRESETN, 0x848, 2), DEF_RST(R9A07G044_SPI_RST, 0x850, 0), DEF_RST(R9A07G044_SDHI0_IXRST, 0x854, 0), DEF_RST(R9A07G044_SDHI1_IXRST, 0x854, 1), + DEF_RST(R9A07G044_GPU_RESETN, 0x858, 0), + DEF_RST(R9A07G044_GPU_AXI_RESETN, 0x858, 1), + DEF_RST(R9A07G044_GPU_ACE_RESETN, 0x858, 2), DEF_RST(R9A07G044_SSI0_RST_M2_REG, 0x870, 0), DEF_RST(R9A07G044_SSI1_RST_M2_REG, 0x870, 1), DEF_RST(R9A07G044_SSI2_RST_M2_REG, 0x870, 2), @@ -256,6 +314,10 @@ static struct rzg2l_reset r9a07g044_resets[] = { DEF_RST(R9A07G044_SCIF3_RST_SYSTEM_N, 0x884, 3), DEF_RST(R9A07G044_SCIF4_RST_SYSTEM_N, 0x884, 4), DEF_RST(R9A07G044_SCI0_RST, 0x888, 0), + DEF_RST(R9A07G044_SCI1_RST, 0x888, 1), + DEF_RST(R9A07G044_RSPI0_RST, 0x890, 0), + DEF_RST(R9A07G044_RSPI1_RST, 0x890, 1), + DEF_RST(R9A07G044_RSPI2_RST, 0x890, 2), DEF_RST(R9A07G044_CANFD_RSTP_N, 0x894, 0), DEF_RST(R9A07G044_CANFD_RSTC_N, 0x894, 1), DEF_RST(R9A07G044_GPIO_RSTN, 0x898, 0), @@ -263,6 +325,7 @@ static struct rzg2l_reset r9a07g044_resets[] = { DEF_RST(R9A07G044_GPIO_SPARE_RESETN, 0x898, 2), DEF_RST(R9A07G044_ADC_PRESETN, 0x8a8, 0), DEF_RST(R9A07G044_ADC_ADRST_N, 0x8a8, 1), + DEF_RST(R9A07G044_TSU_PRESETN, 0x8ac, 0), }; static const unsigned int r9a07g044_crit_mod_clks[] __initconst = { diff --git a/drivers/clk/renesas/rcar-cpg-lib.c b/drivers/clk/renesas/rcar-cpg-lib.c index e93f0011eb07..e2e0447de190 100644 --- a/drivers/clk/renesas/rcar-cpg-lib.c +++ b/drivers/clk/renesas/rcar-cpg-lib.c @@ -65,206 +65,49 @@ void cpg_simple_notifier_register(struct raw_notifier_head *notifiers, /* * SDn Clock */ -#define CPG_SD_STP_HCK BIT(9) -#define CPG_SD_STP_CK BIT(8) -#define CPG_SD_STP_MASK (CPG_SD_STP_HCK | CPG_SD_STP_CK) -#define CPG_SD_FC_MASK (0x7 << 2 | 0x3 << 0) +#define SDnSRCFC_SHIFT 2 +#define STPnHCK BIT(9 - SDnSRCFC_SHIFT) -#define CPG_SD_DIV_TABLE_DATA(stp_hck, sd_srcfc, sd_fc, sd_div) \ -{ \ - .val = ((stp_hck) ? CPG_SD_STP_HCK : 0) | \ - ((sd_srcfc) << 2) | \ - ((sd_fc) << 0), \ - .div = (sd_div), \ -} - -struct sd_div_table { - u32 val; - unsigned int div; +static const struct clk_div_table cpg_sdh_div_table[] = { + { 0, 1 }, { 1, 2 }, { STPnHCK | 2, 4 }, { STPnHCK | 3, 8 }, + { STPnHCK | 4, 16 }, { 0, 0 }, }; -struct sd_clock { - struct clk_hw hw; - const struct sd_div_table *div_table; - struct cpg_simple_notifier csn; - unsigned int div_num; - unsigned int cur_div_idx; -}; - -/* SDn divider - * sd_srcfc sd_fc div - * stp_hck (div) (div) = sd_srcfc x sd_fc - *--------------------------------------------------------- - * 0 0 (1) 1 (4) 4 : SDR104 / HS200 / HS400 (8 TAP) - * 0 1 (2) 1 (4) 8 : SDR50 - * 1 2 (4) 1 (4) 16 : HS / SDR25 - * 1 3 (8) 1 (4) 32 : NS / SDR12 - * 1 4 (16) 1 (4) 64 - * 0 0 (1) 0 (2) 2 - * 0 1 (2) 0 (2) 4 : SDR104 / HS200 / HS400 (4 TAP) - * 1 2 (4) 0 (2) 8 - * 1 3 (8) 0 (2) 16 - * 1 4 (16) 0 (2) 32 - * - * NOTE: There is a quirk option to ignore the first row of the dividers - * table when searching for suitable settings. This is because HS400 on - * early ES versions of H3 and M3-W requires a specific setting to work. - */ -static const struct sd_div_table cpg_sd_div_table[] = { -/* CPG_SD_DIV_TABLE_DATA(stp_hck, sd_srcfc, sd_fc, sd_div) */ - CPG_SD_DIV_TABLE_DATA(0, 0, 1, 4), - CPG_SD_DIV_TABLE_DATA(0, 1, 1, 8), - CPG_SD_DIV_TABLE_DATA(1, 2, 1, 16), - CPG_SD_DIV_TABLE_DATA(1, 3, 1, 32), - CPG_SD_DIV_TABLE_DATA(1, 4, 1, 64), - CPG_SD_DIV_TABLE_DATA(0, 0, 0, 2), - CPG_SD_DIV_TABLE_DATA(0, 1, 0, 4), - CPG_SD_DIV_TABLE_DATA(1, 2, 0, 8), - CPG_SD_DIV_TABLE_DATA(1, 3, 0, 16), - CPG_SD_DIV_TABLE_DATA(1, 4, 0, 32), -}; - -#define to_sd_clock(_hw) container_of(_hw, struct sd_clock, hw) - -static int cpg_sd_clock_enable(struct clk_hw *hw) +struct clk * __init cpg_sdh_clk_register(const char *name, + void __iomem *sdnckcr, const char *parent_name, + struct raw_notifier_head *notifiers) { - struct sd_clock *clock = to_sd_clock(hw); + struct cpg_simple_notifier *csn; + struct clk *clk; - cpg_reg_modify(clock->csn.reg, CPG_SD_STP_MASK, - clock->div_table[clock->cur_div_idx].val & - CPG_SD_STP_MASK); + csn = kzalloc(sizeof(*csn), GFP_KERNEL); + if (!csn) + return ERR_PTR(-ENOMEM); - return 0; -} + csn->reg = sdnckcr; -static void cpg_sd_clock_disable(struct clk_hw *hw) -{ - struct sd_clock *clock = to_sd_clock(hw); - - cpg_reg_modify(clock->csn.reg, 0, CPG_SD_STP_MASK); -} - -static int cpg_sd_clock_is_enabled(struct clk_hw *hw) -{ - struct sd_clock *clock = to_sd_clock(hw); - - return !(readl(clock->csn.reg) & CPG_SD_STP_MASK); -} - -static unsigned long cpg_sd_clock_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - struct sd_clock *clock = to_sd_clock(hw); - - return DIV_ROUND_CLOSEST(parent_rate, - clock->div_table[clock->cur_div_idx].div); -} - -static int cpg_sd_clock_determine_rate(struct clk_hw *hw, - struct clk_rate_request *req) -{ - unsigned long best_rate = ULONG_MAX, diff_min = ULONG_MAX; - struct sd_clock *clock = to_sd_clock(hw); - unsigned long calc_rate, diff; - unsigned int i; - - for (i = 0; i < clock->div_num; i++) { - calc_rate = DIV_ROUND_CLOSEST(req->best_parent_rate, - clock->div_table[i].div); - if (calc_rate < req->min_rate || calc_rate > req->max_rate) - continue; - - diff = calc_rate > req->rate ? calc_rate - req->rate - : req->rate - calc_rate; - if (diff < diff_min) { - best_rate = calc_rate; - diff_min = diff; - } + clk = clk_register_divider_table(NULL, name, parent_name, 0, sdnckcr, + SDnSRCFC_SHIFT, 8, 0, cpg_sdh_div_table, + &cpg_lock); + if (IS_ERR(clk)) { + kfree(csn); + return clk; } - if (best_rate == ULONG_MAX) - return -EINVAL; - - req->rate = best_rate; - return 0; + cpg_simple_notifier_register(notifiers, csn); + return clk; } -static int cpg_sd_clock_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) -{ - struct sd_clock *clock = to_sd_clock(hw); - unsigned int i; - - for (i = 0; i < clock->div_num; i++) - if (rate == DIV_ROUND_CLOSEST(parent_rate, - clock->div_table[i].div)) - break; - - if (i >= clock->div_num) - return -EINVAL; - - clock->cur_div_idx = i; - - cpg_reg_modify(clock->csn.reg, CPG_SD_STP_MASK | CPG_SD_FC_MASK, - clock->div_table[i].val & - (CPG_SD_STP_MASK | CPG_SD_FC_MASK)); - - return 0; -} - -static const struct clk_ops cpg_sd_clock_ops = { - .enable = cpg_sd_clock_enable, - .disable = cpg_sd_clock_disable, - .is_enabled = cpg_sd_clock_is_enabled, - .recalc_rate = cpg_sd_clock_recalc_rate, - .determine_rate = cpg_sd_clock_determine_rate, - .set_rate = cpg_sd_clock_set_rate, +static const struct clk_div_table cpg_sd_div_table[] = { + { 0, 2 }, { 1, 4 }, { 0, 0 }, }; struct clk * __init cpg_sd_clk_register(const char *name, - void __iomem *base, unsigned int offset, const char *parent_name, - struct raw_notifier_head *notifiers, bool skip_first) + void __iomem *sdnckcr, const char *parent_name) { - struct clk_init_data init = {}; - struct sd_clock *clock; - struct clk *clk; - u32 val; - - clock = kzalloc(sizeof(*clock), GFP_KERNEL); - if (!clock) - return ERR_PTR(-ENOMEM); - - init.name = name; - init.ops = &cpg_sd_clock_ops; - init.flags = CLK_SET_RATE_PARENT; - init.parent_names = &parent_name; - init.num_parents = 1; - - clock->csn.reg = base + offset; - clock->hw.init = &init; - clock->div_table = cpg_sd_div_table; - clock->div_num = ARRAY_SIZE(cpg_sd_div_table); - - if (skip_first) { - clock->div_table++; - clock->div_num--; - } - - val = readl(clock->csn.reg) & ~CPG_SD_FC_MASK; - val |= CPG_SD_STP_MASK | (clock->div_table[0].val & CPG_SD_FC_MASK); - writel(val, clock->csn.reg); - - clk = clk_register(NULL, &clock->hw); - if (IS_ERR(clk)) - goto free_clock; - - cpg_simple_notifier_register(notifiers, &clock->csn); - return clk; - -free_clock: - kfree(clock); - return clk; + return clk_register_divider_table(NULL, name, parent_name, 0, sdnckcr, + 0, 2, 0, cpg_sd_div_table, &cpg_lock); } struct rpc_clock { diff --git a/drivers/clk/renesas/rcar-cpg-lib.h b/drivers/clk/renesas/rcar-cpg-lib.h index 35c0217c2f8b..94627df1c94c 100644 --- a/drivers/clk/renesas/rcar-cpg-lib.h +++ b/drivers/clk/renesas/rcar-cpg-lib.h @@ -26,9 +26,12 @@ void cpg_simple_notifier_register(struct raw_notifier_head *notifiers, void cpg_reg_modify(void __iomem *reg, u32 clear, u32 set); +struct clk * __init cpg_sdh_clk_register(const char *name, + void __iomem *sdnckcr, const char *parent_name, + struct raw_notifier_head *notifiers); + struct clk * __init cpg_sd_clk_register(const char *name, - void __iomem *base, unsigned int offset, const char *parent_name, - struct raw_notifier_head *notifiers, bool skip_first); + void __iomem *sdnckcr, const char *parent_name); struct clk * __init cpg_rpc_clk_register(const char *name, void __iomem *rpcckcr, const char *parent_name, diff --git a/drivers/clk/renesas/rcar-gen3-cpg.c b/drivers/clk/renesas/rcar-gen3-cpg.c index 741f6e74bbcf..e668f23c75e7 100644 --- a/drivers/clk/renesas/rcar-gen3-cpg.c +++ b/drivers/clk/renesas/rcar-gen3-cpg.c @@ -312,29 +312,20 @@ static u32 cpg_quirks __initdata; #define PLL_ERRATA BIT(0) /* Missing PLL0/2/4 post-divider */ #define RCKCR_CKSEL BIT(1) /* Manual RCLK parent selection */ -#define SD_SKIP_FIRST BIT(2) /* Skip first clock in SD table */ static const struct soc_device_attribute cpg_quirks_match[] __initconst = { { .soc_id = "r8a7795", .revision = "ES1.0", - .data = (void *)(PLL_ERRATA | RCKCR_CKSEL | SD_SKIP_FIRST), + .data = (void *)(PLL_ERRATA | RCKCR_CKSEL), }, { .soc_id = "r8a7795", .revision = "ES1.*", - .data = (void *)(RCKCR_CKSEL | SD_SKIP_FIRST), - }, - { - .soc_id = "r8a7795", .revision = "ES2.0", - .data = (void *)SD_SKIP_FIRST, + .data = (void *)(RCKCR_CKSEL), }, { .soc_id = "r8a7796", .revision = "ES1.0", - .data = (void *)(RCKCR_CKSEL | SD_SKIP_FIRST), - }, - { - .soc_id = "r8a7796", .revision = "ES1.1", - .data = (void *)SD_SKIP_FIRST, + .data = (void *)(RCKCR_CKSEL), }, { /* sentinel */ } }; @@ -401,10 +392,13 @@ struct clk * __init rcar_gen3_cpg_clk_register(struct device *dev, mult *= 2; break; + case CLK_TYPE_GEN3_SDH: + return cpg_sdh_clk_register(core->name, base + core->offset, + __clk_get_name(parent), notifiers); + case CLK_TYPE_GEN3_SD: - return cpg_sd_clk_register(core->name, base, core->offset, - __clk_get_name(parent), notifiers, - cpg_quirks & SD_SKIP_FIRST); + return cpg_sd_clk_register(core->name, base + core->offset, + __clk_get_name(parent)); case CLK_TYPE_GEN3_R: if (cpg_quirks & RCKCR_CKSEL) { diff --git a/drivers/clk/renesas/rcar-gen3-cpg.h b/drivers/clk/renesas/rcar-gen3-cpg.h index 3d949c4a3244..2bc0afadf604 100644 --- a/drivers/clk/renesas/rcar-gen3-cpg.h +++ b/drivers/clk/renesas/rcar-gen3-cpg.h @@ -17,6 +17,7 @@ enum rcar_gen3_clk_types { CLK_TYPE_GEN3_PLL2, CLK_TYPE_GEN3_PLL3, CLK_TYPE_GEN3_PLL4, + CLK_TYPE_GEN3_SDH, CLK_TYPE_GEN3_SD, CLK_TYPE_GEN3_R, CLK_TYPE_GEN3_MDSEL, /* Select parent/divider using mode pin */ @@ -32,6 +33,9 @@ enum rcar_gen3_clk_types { CLK_TYPE_GEN3_SOC_BASE, }; +#define DEF_GEN3_SDH(_name, _id, _parent, _offset) \ + DEF_BASE(_name, _id, CLK_TYPE_GEN3_SDH, _parent, .offset = _offset) + #define DEF_GEN3_SD(_name, _id, _parent, _offset) \ DEF_BASE(_name, _id, CLK_TYPE_GEN3_SD, _parent, .offset = _offset) diff --git a/drivers/clk/renesas/rcar-gen4-cpg.c b/drivers/clk/renesas/rcar-gen4-cpg.c new file mode 100644 index 000000000000..54ebf4b3c128 --- /dev/null +++ b/drivers/clk/renesas/rcar-gen4-cpg.c @@ -0,0 +1,305 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * R-Car Gen4 Clock Pulse Generator + * + * Copyright (C) 2021 Renesas Electronics Corp. + * + * Based on rcar-gen3-cpg.c + * + * Copyright (C) 2015-2018 Glider bvba + * Copyright (C) 2019 Renesas Electronics Corp. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "renesas-cpg-mssr.h" +#include "rcar-gen4-cpg.h" +#include "rcar-cpg-lib.h" + +static const struct rcar_gen4_cpg_pll_config *cpg_pll_config __initconst; +static unsigned int cpg_clk_extalr __initdata; +static u32 cpg_mode __initdata; + +/* + * Z0 Clock & Z1 Clock + */ +#define CPG_FRQCRB 0x00000804 +#define CPG_FRQCRB_KICK BIT(31) +#define CPG_FRQCRC 0x00000808 + +struct cpg_z_clk { + struct clk_hw hw; + void __iomem *reg; + void __iomem *kick_reg; + unsigned long max_rate; /* Maximum rate for normal mode */ + unsigned int fixed_div; + u32 mask; +}; + +#define to_z_clk(_hw) container_of(_hw, struct cpg_z_clk, hw) + +static unsigned long cpg_z_clk_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct cpg_z_clk *zclk = to_z_clk(hw); + unsigned int mult; + u32 val; + + val = readl(zclk->reg) & zclk->mask; + mult = 32 - (val >> __ffs(zclk->mask)); + + return DIV_ROUND_CLOSEST_ULL((u64)parent_rate * mult, + 32 * zclk->fixed_div); +} + +static int cpg_z_clk_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + struct cpg_z_clk *zclk = to_z_clk(hw); + unsigned int min_mult, max_mult, mult; + unsigned long rate, prate; + + rate = min(req->rate, req->max_rate); + if (rate <= zclk->max_rate) { + /* Set parent rate to initial value for normal modes */ + prate = zclk->max_rate; + } else { + /* Set increased parent rate for boost modes */ + prate = rate; + } + req->best_parent_rate = clk_hw_round_rate(clk_hw_get_parent(hw), + prate * zclk->fixed_div); + + prate = req->best_parent_rate / zclk->fixed_div; + min_mult = max(div64_ul(req->min_rate * 32ULL, prate), 1ULL); + max_mult = min(div64_ul(req->max_rate * 32ULL, prate), 32ULL); + if (max_mult < min_mult) + return -EINVAL; + + mult = DIV_ROUND_CLOSEST_ULL(rate * 32ULL, prate); + mult = clamp(mult, min_mult, max_mult); + + req->rate = DIV_ROUND_CLOSEST_ULL((u64)prate * mult, 32); + return 0; +} + +static int cpg_z_clk_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct cpg_z_clk *zclk = to_z_clk(hw); + unsigned int mult; + unsigned int i; + + mult = DIV64_U64_ROUND_CLOSEST(rate * 32ULL * zclk->fixed_div, + parent_rate); + mult = clamp(mult, 1U, 32U); + + if (readl(zclk->kick_reg) & CPG_FRQCRB_KICK) + return -EBUSY; + + cpg_reg_modify(zclk->reg, zclk->mask, (32 - mult) << __ffs(zclk->mask)); + + /* + * Set KICK bit in FRQCRB to update hardware setting and wait for + * clock change completion. + */ + cpg_reg_modify(zclk->kick_reg, 0, CPG_FRQCRB_KICK); + + /* + * Note: There is no HW information about the worst case latency. + * + * Using experimental measurements, it seems that no more than + * ~10 iterations are needed, independently of the CPU rate. + * Since this value might be dependent on external xtal rate, pll1 + * rate or even the other emulation clocks rate, use 1000 as a + * "super" safe value. + */ + for (i = 1000; i; i--) { + if (!(readl(zclk->kick_reg) & CPG_FRQCRB_KICK)) + return 0; + + cpu_relax(); + } + + return -ETIMEDOUT; +} + +static const struct clk_ops cpg_z_clk_ops = { + .recalc_rate = cpg_z_clk_recalc_rate, + .determine_rate = cpg_z_clk_determine_rate, + .set_rate = cpg_z_clk_set_rate, +}; + +static struct clk * __init cpg_z_clk_register(const char *name, + const char *parent_name, + void __iomem *reg, + unsigned int div, + unsigned int offset) +{ + struct clk_init_data init = {}; + struct cpg_z_clk *zclk; + struct clk *clk; + + zclk = kzalloc(sizeof(*zclk), GFP_KERNEL); + if (!zclk) + return ERR_PTR(-ENOMEM); + + init.name = name; + init.ops = &cpg_z_clk_ops; + init.flags = CLK_SET_RATE_PARENT; + init.parent_names = &parent_name; + init.num_parents = 1; + + zclk->reg = reg + CPG_FRQCRC; + zclk->kick_reg = reg + CPG_FRQCRB; + zclk->hw.init = &init; + zclk->mask = GENMASK(offset + 4, offset); + zclk->fixed_div = div; /* PLLVCO x 1/div x SYS-CPU divider */ + + clk = clk_register(NULL, &zclk->hw); + if (IS_ERR(clk)) { + kfree(zclk); + return clk; + } + + zclk->max_rate = clk_hw_get_rate(clk_hw_get_parent(&zclk->hw)) / + zclk->fixed_div; + return clk; +} + +/* + * RPC Clocks + */ +static const struct clk_div_table cpg_rpcsrc_div_table[] = { + { 0, 4 }, { 1, 6 }, { 2, 5 }, { 3, 6 }, { 0, 0 }, +}; + +struct clk * __init rcar_gen4_cpg_clk_register(struct device *dev, + const struct cpg_core_clk *core, const struct cpg_mssr_info *info, + struct clk **clks, void __iomem *base, + struct raw_notifier_head *notifiers) +{ + const struct clk *parent; + unsigned int mult = 1; + unsigned int div = 1; + u32 value; + + parent = clks[core->parent & 0xffff]; /* some types use high bits */ + if (IS_ERR(parent)) + return ERR_CAST(parent); + + switch (core->type) { + case CLK_TYPE_GEN4_MAIN: + div = cpg_pll_config->extal_div; + break; + + case CLK_TYPE_GEN4_PLL1: + mult = cpg_pll_config->pll1_mult; + div = cpg_pll_config->pll1_div; + break; + + case CLK_TYPE_GEN4_PLL2: + mult = cpg_pll_config->pll2_mult; + div = cpg_pll_config->pll2_div; + break; + + case CLK_TYPE_GEN4_PLL3: + mult = cpg_pll_config->pll3_mult; + div = cpg_pll_config->pll3_div; + break; + + case CLK_TYPE_GEN4_PLL5: + mult = cpg_pll_config->pll5_mult; + div = cpg_pll_config->pll5_div; + break; + + case CLK_TYPE_GEN4_PLL6: + mult = cpg_pll_config->pll6_mult; + div = cpg_pll_config->pll6_div; + break; + + case CLK_TYPE_GEN4_PLL2X_3X: + value = readl(base + core->offset); + mult = (((value >> 24) & 0x7f) + 1) * 2; + break; + + case CLK_TYPE_GEN4_Z: + return cpg_z_clk_register(core->name, __clk_get_name(parent), + base, core->div, core->offset); + + case CLK_TYPE_GEN4_SDSRC: + div = ((readl(base + SD0CKCR1) >> 29) & 0x03) + 4; + break; + + case CLK_TYPE_GEN4_SDH: + return cpg_sdh_clk_register(core->name, base + core->offset, + __clk_get_name(parent), notifiers); + + case CLK_TYPE_GEN4_SD: + return cpg_sd_clk_register(core->name, base + core->offset, + __clk_get_name(parent)); + + case CLK_TYPE_GEN4_MDSEL: + /* + * Clock selectable between two parents and two fixed dividers + * using a mode pin + */ + if (cpg_mode & BIT(core->offset)) { + div = core->div & 0xffff; + } else { + parent = clks[core->parent >> 16]; + if (IS_ERR(parent)) + return ERR_CAST(parent); + div = core->div >> 16; + } + mult = 1; + break; + + case CLK_TYPE_GEN4_OSC: + /* + * Clock combining OSC EXTAL predivider and a fixed divider + */ + div = cpg_pll_config->osc_prediv * core->div; + break; + + case CLK_TYPE_GEN4_RPCSRC: + return clk_register_divider_table(NULL, core->name, + __clk_get_name(parent), 0, + base + CPG_RPCCKCR, 3, 2, 0, + cpg_rpcsrc_div_table, + &cpg_lock); + + case CLK_TYPE_GEN4_RPC: + return cpg_rpc_clk_register(core->name, base + CPG_RPCCKCR, + __clk_get_name(parent), notifiers); + + case CLK_TYPE_GEN4_RPCD2: + return cpg_rpcd2_clk_register(core->name, base + CPG_RPCCKCR, + __clk_get_name(parent)); + + default: + return ERR_PTR(-EINVAL); + } + + return clk_register_fixed_factor(NULL, core->name, + __clk_get_name(parent), 0, mult, div); +} + +int __init rcar_gen4_cpg_init(const struct rcar_gen4_cpg_pll_config *config, + unsigned int clk_extalr, u32 mode) +{ + cpg_pll_config = config; + cpg_clk_extalr = clk_extalr; + cpg_mode = mode; + + spin_lock_init(&cpg_lock); + + return 0; +} diff --git a/drivers/clk/renesas/rcar-gen4-cpg.h b/drivers/clk/renesas/rcar-gen4-cpg.h new file mode 100644 index 000000000000..afc8c024d538 --- /dev/null +++ b/drivers/clk/renesas/rcar-gen4-cpg.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * R-Car Gen4 Clock Pulse Generator + * + * Copyright (C) 2021 Renesas Electronics Corp. + * + */ + +#ifndef __CLK_RENESAS_RCAR_GEN4_CPG_H__ +#define __CLK_RENESAS_RCAR_GEN4_CPG_H__ + +enum rcar_gen4_clk_types { + CLK_TYPE_GEN4_MAIN = CLK_TYPE_CUSTOM, + CLK_TYPE_GEN4_PLL1, + CLK_TYPE_GEN4_PLL2, + CLK_TYPE_GEN4_PLL2X_3X, /* r8a779a0 only */ + CLK_TYPE_GEN4_PLL3, + CLK_TYPE_GEN4_PLL5, + CLK_TYPE_GEN4_PLL6, + CLK_TYPE_GEN4_SDSRC, + CLK_TYPE_GEN4_SDH, + CLK_TYPE_GEN4_SD, + CLK_TYPE_GEN4_MDSEL, /* Select parent/divider using mode pin */ + CLK_TYPE_GEN4_Z, + CLK_TYPE_GEN4_OSC, /* OSC EXTAL predivider and fixed divider */ + CLK_TYPE_GEN4_RPCSRC, + CLK_TYPE_GEN4_RPC, + CLK_TYPE_GEN4_RPCD2, + + /* SoC specific definitions start here */ + CLK_TYPE_GEN4_SOC_BASE, +}; + +#define DEF_GEN4_SDH(_name, _id, _parent, _offset) \ + DEF_BASE(_name, _id, CLK_TYPE_GEN4_SDH, _parent, .offset = _offset) + +#define DEF_GEN4_SD(_name, _id, _parent, _offset) \ + DEF_BASE(_name, _id, CLK_TYPE_GEN4_SD, _parent, .offset = _offset) + +#define DEF_GEN4_MDSEL(_name, _id, _md, _parent0, _div0, _parent1, _div1) \ + DEF_BASE(_name, _id, CLK_TYPE_GEN4_MDSEL, \ + (_parent0) << 16 | (_parent1), \ + .div = (_div0) << 16 | (_div1), .offset = _md) + +#define DEF_GEN4_OSC(_name, _id, _parent, _div) \ + DEF_BASE(_name, _id, CLK_TYPE_GEN4_OSC, _parent, .div = _div) + +#define DEF_GEN4_Z(_name, _id, _type, _parent, _div, _offset) \ + DEF_BASE(_name, _id, _type, _parent, .div = _div, .offset = _offset) + +struct rcar_gen4_cpg_pll_config { + u8 extal_div; + u8 pll1_mult; + u8 pll1_div; + u8 pll2_mult; + u8 pll2_div; + u8 pll3_mult; + u8 pll3_div; + u8 pll5_mult; + u8 pll5_div; + u8 pll6_mult; + u8 pll6_div; + u8 osc_prediv; +}; + +#define CPG_RPCCKCR 0x874 +#define SD0CKCR1 0x8a4 + +struct clk *rcar_gen4_cpg_clk_register(struct device *dev, + const struct cpg_core_clk *core, const struct cpg_mssr_info *info, + struct clk **clks, void __iomem *base, + struct raw_notifier_head *notifiers); +int rcar_gen4_cpg_init(const struct rcar_gen4_cpg_pll_config *config, + unsigned int clk_extalr, u32 mode); + +#endif diff --git a/drivers/clk/renesas/renesas-cpg-mssr.c b/drivers/clk/renesas/renesas-cpg-mssr.c index 21f762aa2131..5d2c3edbaa14 100644 --- a/drivers/clk/renesas/renesas-cpg-mssr.c +++ b/drivers/clk/renesas/renesas-cpg-mssr.c @@ -57,9 +57,11 @@ static const u16 mstpsr[] = { 0x9A0, 0x9A4, 0x9A8, 0x9AC, }; -static const u16 mstpsr_for_v3u[] = { +static const u16 mstpsr_for_gen4[] = { 0x2E00, 0x2E04, 0x2E08, 0x2E0C, 0x2E10, 0x2E14, 0x2E18, 0x2E1C, - 0x2E20, 0x2E24, 0x2E28, 0x2E2C, 0x2E30, 0x2E34, 0x2E38, + 0x2E20, 0x2E24, 0x2E28, 0x2E2C, 0x2E30, 0x2E34, 0x2E38, 0x2E3C, + 0x2E40, 0x2E44, 0x2E48, 0x2E4C, 0x2E50, 0x2E54, 0x2E58, 0x2E5C, + 0x2E60, 0x2E64, 0x2E68, 0x2E6C, }; /* @@ -71,9 +73,11 @@ static const u16 smstpcr[] = { 0x990, 0x994, 0x998, 0x99C, }; -static const u16 mstpcr_for_v3u[] = { +static const u16 mstpcr_for_gen4[] = { 0x2D00, 0x2D04, 0x2D08, 0x2D0C, 0x2D10, 0x2D14, 0x2D18, 0x2D1C, - 0x2D20, 0x2D24, 0x2D28, 0x2D2C, 0x2D30, 0x2D34, 0x2D38, + 0x2D20, 0x2D24, 0x2D28, 0x2D2C, 0x2D30, 0x2D34, 0x2D38, 0x2D3C, + 0x2D40, 0x2D44, 0x2D48, 0x2D4C, 0x2D50, 0x2D54, 0x2D58, 0x2D5C, + 0x2D60, 0x2D64, 0x2D68, 0x2D6C, }; /* @@ -95,9 +99,11 @@ static const u16 srcr[] = { 0x920, 0x924, 0x928, 0x92C, }; -static const u16 srcr_for_v3u[] = { +static const u16 srcr_for_gen4[] = { 0x2C00, 0x2C04, 0x2C08, 0x2C0C, 0x2C10, 0x2C14, 0x2C18, 0x2C1C, - 0x2C20, 0x2C24, 0x2C28, 0x2C2C, 0x2C30, 0x2C34, 0x2C38, + 0x2C20, 0x2C24, 0x2C28, 0x2C2C, 0x2C30, 0x2C34, 0x2C38, 0x2C3C, + 0x2C40, 0x2C44, 0x2C48, 0x2C4C, 0x2C50, 0x2C54, 0x2C58, 0x2C5C, + 0x2C60, 0x2C64, 0x2C68, 0x2C6C, }; /* @@ -109,9 +115,11 @@ static const u16 srstclr[] = { 0x960, 0x964, 0x968, 0x96C, }; -static const u16 srstclr_for_v3u[] = { +static const u16 srstclr_for_gen4[] = { 0x2C80, 0x2C84, 0x2C88, 0x2C8C, 0x2C90, 0x2C94, 0x2C98, 0x2C9C, - 0x2CA0, 0x2CA4, 0x2CA8, 0x2CAC, 0x2CB0, 0x2CB4, 0x2CB8, + 0x2CA0, 0x2CA4, 0x2CA8, 0x2CAC, 0x2CB0, 0x2CB4, 0x2CB8, 0x2CBC, + 0x2CC0, 0x2CC4, 0x2CC8, 0x2CCC, 0x2CD0, 0x2CD4, 0x2CD8, 0x2CDC, + 0x2CE0, 0x2CE4, 0x2CE8, 0x2CEC, }; /** @@ -158,7 +166,7 @@ struct cpg_mssr_priv { struct { u32 mask; u32 val; - } smstpcr_saved[ARRAY_SIZE(mstpsr_for_v3u)]; + } smstpcr_saved[ARRAY_SIZE(mstpsr_for_gen4)]; struct clk *clks[]; }; @@ -552,6 +560,11 @@ void cpg_mssr_detach_dev(struct generic_pm_domain *unused, struct device *dev) pm_clk_destroy(dev); } +static void cpg_mssr_genpd_remove(void *data) +{ + pm_genpd_remove(data); +} + static int __init cpg_mssr_add_clk_domain(struct device *dev, const unsigned int *core_pm_clks, unsigned int num_core_pm_clks) @@ -560,6 +573,7 @@ static int __init cpg_mssr_add_clk_domain(struct device *dev, struct generic_pm_domain *genpd; struct cpg_mssr_clk_domain *pd; size_t pm_size = num_core_pm_clks * sizeof(core_pm_clks[0]); + int ret; pd = devm_kzalloc(dev, sizeof(*pd) + pm_size, GFP_KERNEL); if (!pd) @@ -574,11 +588,17 @@ static int __init cpg_mssr_add_clk_domain(struct device *dev, GENPD_FLAG_ACTIVE_WAKEUP; genpd->attach_dev = cpg_mssr_attach_dev; genpd->detach_dev = cpg_mssr_detach_dev; - pm_genpd_init(genpd, &pm_domain_always_on_gov, false); + ret = pm_genpd_init(genpd, &pm_domain_always_on_gov, false); + if (ret) + return ret; + + ret = devm_add_action_or_reset(dev, cpg_mssr_genpd_remove, genpd); + if (ret) + return ret; + cpg_mssr_clk_domain = pd; - of_genpd_add_provider_simple(np, genpd); - return 0; + return of_genpd_add_provider_simple(np, genpd); } #ifdef CONFIG_RESET_CONTROLLER @@ -827,6 +847,12 @@ static const struct of_device_id cpg_mssr_match[] = { .compatible = "renesas,r8a779a0-cpg-mssr", .data = &r8a779a0_cpg_mssr_info, }, +#endif +#ifdef CONFIG_CLK_R8A779F0 + { + .compatible = "renesas,r8a779f0-cpg-mssr", + .data = &r8a779f0_cpg_mssr_info, + }, #endif { /* sentinel */ } }; @@ -970,11 +996,11 @@ static int __init cpg_mssr_common_init(struct device *dev, priv->reset_clear_regs = srstclr; } else if (priv->reg_layout == CLK_REG_LAYOUT_RZ_A) { priv->control_regs = stbcr; - } else if (priv->reg_layout == CLK_REG_LAYOUT_RCAR_V3U) { - priv->status_regs = mstpsr_for_v3u; - priv->control_regs = mstpcr_for_v3u; - priv->reset_regs = srcr_for_v3u; - priv->reset_clear_regs = srstclr_for_v3u; + } else if (priv->reg_layout == CLK_REG_LAYOUT_RCAR_GEN4) { + priv->status_regs = mstpsr_for_gen4; + priv->control_regs = mstpcr_for_gen4; + priv->reset_regs = srcr_for_gen4; + priv->reset_clear_regs = srstclr_for_gen4; } else { error = -EINVAL; goto out_err; diff --git a/drivers/clk/renesas/renesas-cpg-mssr.h b/drivers/clk/renesas/renesas-cpg-mssr.h index 6b2a0ade482e..16810dd4e6ac 100644 --- a/drivers/clk/renesas/renesas-cpg-mssr.h +++ b/drivers/clk/renesas/renesas-cpg-mssr.h @@ -88,7 +88,7 @@ struct device_node; enum clk_reg_layout { CLK_REG_LAYOUT_RCAR_GEN2_AND_GEN3 = 0, CLK_REG_LAYOUT_RZ_A, - CLK_REG_LAYOUT_RCAR_V3U, + CLK_REG_LAYOUT_RCAR_GEN4, }; /** @@ -178,6 +178,7 @@ extern const struct cpg_mssr_info r8a77980_cpg_mssr_info; extern const struct cpg_mssr_info r8a77990_cpg_mssr_info; extern const struct cpg_mssr_info r8a77995_cpg_mssr_info; extern const struct cpg_mssr_info r8a779a0_cpg_mssr_info; +extern const struct cpg_mssr_info r8a779f0_cpg_mssr_info; void __init cpg_mssr_early_init(struct device_node *np, const struct cpg_mssr_info *info); diff --git a/drivers/clk/renesas/rzg2l-cpg.c b/drivers/clk/renesas/rzg2l-cpg.c index 4021f6cabda4..edd0abe34a37 100644 --- a/drivers/clk/renesas/rzg2l-cpg.c +++ b/drivers/clk/renesas/rzg2l-cpg.c @@ -74,6 +74,7 @@ struct sd_hw_data { * @clks: Array containing all Core and Module Clocks * @num_core_clks: Number of Core Clocks in clks[] * @num_mod_clks: Number of Module Clocks in clks[] + * @num_resets: Number of Module Resets in info->resets[] * @last_dt_core_clk: ID of the last Core Clock exported to DT * @notifiers: Notifier chain to save/restore clock state for system resume * @info: Pointer to platform data @@ -850,10 +851,16 @@ static void rzg2l_cpg_detach_dev(struct generic_pm_domain *unused, struct device pm_clk_destroy(dev); } +static void rzg2l_cpg_genpd_remove(void *data) +{ + pm_genpd_remove(data); +} + static int __init rzg2l_cpg_add_clk_domain(struct device *dev) { struct device_node *np = dev->of_node; struct generic_pm_domain *genpd; + int ret; genpd = devm_kzalloc(dev, sizeof(*genpd), GFP_KERNEL); if (!genpd) @@ -864,10 +871,15 @@ static int __init rzg2l_cpg_add_clk_domain(struct device *dev) GENPD_FLAG_ACTIVE_WAKEUP; genpd->attach_dev = rzg2l_cpg_attach_dev; genpd->detach_dev = rzg2l_cpg_detach_dev; - pm_genpd_init(genpd, &pm_domain_always_on_gov, false); + ret = pm_genpd_init(genpd, &pm_domain_always_on_gov, false); + if (ret) + return ret; - of_genpd_add_provider_simple(np, genpd); - return 0; + ret = devm_add_action_or_reset(dev, rzg2l_cpg_genpd_remove, genpd); + if (ret) + return ret; + + return of_genpd_add_provider_simple(np, genpd); } static int __init rzg2l_cpg_probe(struct platform_device *pdev) diff --git a/drivers/clk/renesas/rzg2l-cpg.h b/drivers/clk/renesas/rzg2l-cpg.h index 7fb6b4030f72..5729d102034b 100644 --- a/drivers/clk/renesas/rzg2l-cpg.h +++ b/drivers/clk/renesas/rzg2l-cpg.h @@ -9,11 +9,14 @@ #ifndef __RENESAS_RZG2L_CPG_H__ #define __RENESAS_RZG2L_CPG_H__ +#define CPG_PL1_DDIV (0x200) #define CPG_PL2_DDIV (0x204) #define CPG_PL3A_DDIV (0x208) +#define CPG_PL6_DDIV (0x210) #define CPG_PL2SDHI_DSEL (0x218) #define CPG_CLKSTATUS (0x280) #define CPG_PL3_SSEL (0x408) +#define CPG_PL6_SSEL (0x414) #define CPG_PL6_ETH_SSEL (0x418) #define CPG_CLKSTATUS_SELSDHI0_STS BIT(28) @@ -29,16 +32,19 @@ #define DDIV_PACK(offset, bitpos, size) \ (((offset) << 20) | ((bitpos) << 12) | ((size) << 8)) +#define DIVPL1A DDIV_PACK(CPG_PL1_DDIV, 0, 2) #define DIVPL2A DDIV_PACK(CPG_PL2_DDIV, 0, 3) #define DIVPL3A DDIV_PACK(CPG_PL3A_DDIV, 0, 3) #define DIVPL3B DDIV_PACK(CPG_PL3A_DDIV, 4, 3) #define DIVPL3C DDIV_PACK(CPG_PL3A_DDIV, 8, 3) +#define DIVGPU DDIV_PACK(CPG_PL6_DDIV, 0, 2) #define SEL_PLL_PACK(offset, bitpos, size) \ (((offset) << 20) | ((bitpos) << 12) | ((size) << 8)) #define SEL_PLL3_3 SEL_PLL_PACK(CPG_PL3_SSEL, 8, 1) #define SEL_PLL6_2 SEL_PLL_PACK(CPG_PL6_ETH_SSEL, 0, 1) +#define SEL_GPU2 SEL_PLL_PACK(CPG_PL6_SSEL, 12, 1) #define SEL_SDHI0 DDIV_PACK(CPG_PL2SDHI_DSEL, 0, 2) #define SEL_SDHI1 DDIV_PACK(CPG_PL2SDHI_DSEL, 4, 2) @@ -168,6 +174,9 @@ struct rzg2l_reset { * @num_mod_clks: Number of entries in mod_clks[] * @num_hw_mod_clks: Number of Module Clocks supported by the hardware * + * @resets: Array of Module Reset definitions + * @num_resets: Number of entries in resets[] + * * @crit_mod_clks: Array with Module Clock IDs of critical clocks that * should not be disabled without a knowledgeable driver * @num_crit_mod_clks: Number of entries in crit_mod_clks[] diff --git a/drivers/clk/samsung/Makefile b/drivers/clk/samsung/Makefile index c46cf11e4d0b..0df74916a895 100644 --- a/drivers/clk/samsung/Makefile +++ b/drivers/clk/samsung/Makefile @@ -16,7 +16,9 @@ obj-$(CONFIG_EXYNOS_5420_COMMON_CLK) += clk-exynos5-subcmu.o obj-$(CONFIG_EXYNOS_ARM64_COMMON_CLK) += clk-exynos5433.o obj-$(CONFIG_EXYNOS_AUDSS_CLK_CON) += clk-exynos-audss.o obj-$(CONFIG_EXYNOS_CLKOUT) += clk-exynos-clkout.o +obj-$(CONFIG_EXYNOS_ARM64_COMMON_CLK) += clk-exynos-arm64.o obj-$(CONFIG_EXYNOS_ARM64_COMMON_CLK) += clk-exynos7.o +obj-$(CONFIG_EXYNOS_ARM64_COMMON_CLK) += clk-exynos7885.o obj-$(CONFIG_EXYNOS_ARM64_COMMON_CLK) += clk-exynos850.o obj-$(CONFIG_S3C2410_COMMON_CLK)+= clk-s3c2410.o obj-$(CONFIG_S3C2410_COMMON_DCLK)+= clk-s3c2410-dclk.o diff --git a/drivers/clk/samsung/clk-cpu.c b/drivers/clk/samsung/clk-cpu.c index 7f20d9aedaa9..3e62ade120c5 100644 --- a/drivers/clk/samsung/clk-cpu.c +++ b/drivers/clk/samsung/clk-cpu.c @@ -400,7 +400,7 @@ static int exynos5433_cpuclk_notifier_cb(struct notifier_block *nb, } /* helper function to register a CPU clock */ -int __init exynos_register_cpu_clock(struct samsung_clk_provider *ctx, +static int __init exynos_register_cpu_clock(struct samsung_clk_provider *ctx, unsigned int lookup_id, const char *name, const struct clk_hw *parent, const struct clk_hw *alt_parent, unsigned long offset, const struct exynos_cpuclk_cfg_data *cfg, diff --git a/drivers/clk/samsung/clk-cpu.h b/drivers/clk/samsung/clk-cpu.h index af74686db9ef..fc9f67a3b22e 100644 --- a/drivers/clk/samsung/clk-cpu.h +++ b/drivers/clk/samsung/clk-cpu.h @@ -62,11 +62,4 @@ struct exynos_cpuclk { #define CLK_CPU_HAS_E5433_REGS_LAYOUT (1 << 2) }; -int __init exynos_register_cpu_clock(struct samsung_clk_provider *ctx, - unsigned int lookup_id, const char *name, - const struct clk_hw *parent, const struct clk_hw *alt_parent, - unsigned long offset, - const struct exynos_cpuclk_cfg_data *cfg, - unsigned long num_cfgs, unsigned long flags); - #endif /* __SAMSUNG_CLK_CPU_H */ diff --git a/drivers/clk/samsung/clk-exynos-arm64.c b/drivers/clk/samsung/clk-exynos-arm64.c new file mode 100644 index 000000000000..b921b9a1134a --- /dev/null +++ b/drivers/clk/samsung/clk-exynos-arm64.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2021 Linaro Ltd. + * Copyright (C) 2021 Dávid Virág + * Author: Sam Protsenko + * Author: Dávid Virág + * + * This file contains shared functions used by some arm64 Exynos SoCs, + * such as Exynos7885 or Exynos850 to register and init CMUs. + */ +#include +#include + +#include "clk-exynos-arm64.h" + +/* Gate register bits */ +#define GATE_MANUAL BIT(20) +#define GATE_ENABLE_HWACG BIT(28) + +/* Gate register offsets range */ +#define GATE_OFF_START 0x2000 +#define GATE_OFF_END 0x2fff + +/** + * exynos_arm64_init_clocks - Set clocks initial configuration + * @np: CMU device tree node with "reg" property (CMU addr) + * @reg_offs: Register offsets array for clocks to init + * @reg_offs_len: Number of register offsets in reg_offs array + * + * Set manual control mode for all gate clocks. + */ +static void __init exynos_arm64_init_clocks(struct device_node *np, + const unsigned long *reg_offs, size_t reg_offs_len) +{ + void __iomem *reg_base; + size_t i; + + reg_base = of_iomap(np, 0); + if (!reg_base) + panic("%s: failed to map registers\n", __func__); + + for (i = 0; i < reg_offs_len; ++i) { + void __iomem *reg = reg_base + reg_offs[i]; + u32 val; + + /* Modify only gate clock registers */ + if (reg_offs[i] < GATE_OFF_START || reg_offs[i] > GATE_OFF_END) + continue; + + val = readl(reg); + val |= GATE_MANUAL; + val &= ~GATE_ENABLE_HWACG; + writel(val, reg); + } + + iounmap(reg_base); +} + +/** + * exynos_arm64_register_cmu - Register specified Exynos CMU domain + * @dev: Device object; may be NULL if this function is not being + * called from platform driver probe function + * @np: CMU device tree node + * @cmu: CMU data + * + * Register specified CMU domain, which includes next steps: + * + * 1. Enable parent clock of @cmu CMU + * 2. Set initial registers configuration for @cmu CMU clocks + * 3. Register @cmu CMU clocks using Samsung clock framework API + */ +void __init exynos_arm64_register_cmu(struct device *dev, + struct device_node *np, const struct samsung_cmu_info *cmu) +{ + /* Keep CMU parent clock running (needed for CMU registers access) */ + if (cmu->clk_name) { + struct clk *parent_clk; + + if (dev) + parent_clk = clk_get(dev, cmu->clk_name); + else + parent_clk = of_clk_get_by_name(np, cmu->clk_name); + + if (IS_ERR(parent_clk)) { + pr_err("%s: could not find bus clock %s; err = %ld\n", + __func__, cmu->clk_name, PTR_ERR(parent_clk)); + } else { + clk_prepare_enable(parent_clk); + } + } + + exynos_arm64_init_clocks(np, cmu->clk_regs, cmu->nr_clk_regs); + samsung_cmu_register_one(np, cmu); +} diff --git a/drivers/clk/samsung/clk-exynos-arm64.h b/drivers/clk/samsung/clk-exynos-arm64.h new file mode 100644 index 000000000000..0dd174693935 --- /dev/null +++ b/drivers/clk/samsung/clk-exynos-arm64.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2021 Linaro Ltd. + * Copyright (C) 2021 Dávid Virág + * Author: Sam Protsenko + * Author: Dávid Virág + * + * This file contains shared functions used by some arm64 Exynos SoCs, + * such as Exynos7885 or Exynos850 to register and init CMUs. + */ + +#ifndef __CLK_EXYNOS_ARM64_H +#define __CLK_EXYNOS_ARM64_H + +#include "clk.h" + +void exynos_arm64_register_cmu(struct device *dev, + struct device_node *np, const struct samsung_cmu_info *cmu); + +#endif /* __CLK_EXYNOS_ARM64_H */ diff --git a/drivers/clk/samsung/clk-exynos3250.c b/drivers/clk/samsung/clk-exynos3250.c index 17df7f9755aa..6cc65ccf867c 100644 --- a/drivers/clk/samsung/clk-exynos3250.c +++ b/drivers/clk/samsung/clk-exynos3250.c @@ -748,6 +748,31 @@ static const struct samsung_pll_clock exynos3250_plls[] __initconst = { UPLL_LOCK, UPLL_CON0, exynos3250_pll_rates), }; +#define E3250_CPU_DIV0(apll, pclk_dbg, atb, corem) \ + (((apll) << 24) | ((pclk_dbg) << 20) | ((atb) << 16) | \ + ((corem) << 4)) +#define E3250_CPU_DIV1(hpm, copy) \ + (((hpm) << 4) | ((copy) << 0)) + +static const struct exynos_cpuclk_cfg_data e3250_armclk_d[] __initconst = { + { 1000000, E3250_CPU_DIV0(1, 7, 4, 1), E3250_CPU_DIV1(7, 7), }, + { 900000, E3250_CPU_DIV0(1, 7, 3, 1), E3250_CPU_DIV1(7, 7), }, + { 800000, E3250_CPU_DIV0(1, 7, 3, 1), E3250_CPU_DIV1(7, 7), }, + { 700000, E3250_CPU_DIV0(1, 7, 3, 1), E3250_CPU_DIV1(7, 7), }, + { 600000, E3250_CPU_DIV0(1, 7, 3, 1), E3250_CPU_DIV1(7, 7), }, + { 500000, E3250_CPU_DIV0(1, 7, 3, 1), E3250_CPU_DIV1(7, 7), }, + { 400000, E3250_CPU_DIV0(1, 7, 3, 1), E3250_CPU_DIV1(7, 7), }, + { 300000, E3250_CPU_DIV0(1, 5, 3, 1), E3250_CPU_DIV1(7, 7), }, + { 200000, E3250_CPU_DIV0(1, 3, 3, 1), E3250_CPU_DIV1(7, 7), }, + { 100000, E3250_CPU_DIV0(1, 1, 1, 1), E3250_CPU_DIV1(7, 7), }, + { 0 }, +}; + +static const struct samsung_cpu_clock exynos3250_cpu_clks[] __initconst = { + CPU_CLK(CLK_ARM_CLK, "armclk", CLK_MOUT_APLL, CLK_MOUT_MPLL_USER_C, + CLK_CPU_HAS_DIV1, 0x14200, e3250_armclk_d), +}; + static void __init exynos3_core_down_clock(void __iomem *reg_base) { unsigned int tmp; @@ -780,46 +805,21 @@ static const struct samsung_cmu_info cmu_info __initconst = { .nr_gate_clks = ARRAY_SIZE(gate_clks), .fixed_factor_clks = fixed_factor_clks, .nr_fixed_factor_clks = ARRAY_SIZE(fixed_factor_clks), + .cpu_clks = exynos3250_cpu_clks, + .nr_cpu_clks = ARRAY_SIZE(exynos3250_cpu_clks), .nr_clk_ids = CLK_NR_CLKS, .clk_regs = exynos3250_cmu_clk_regs, .nr_clk_regs = ARRAY_SIZE(exynos3250_cmu_clk_regs), }; -#define E3250_CPU_DIV0(apll, pclk_dbg, atb, corem) \ - (((apll) << 24) | ((pclk_dbg) << 20) | ((atb) << 16) | \ - ((corem) << 4)) -#define E3250_CPU_DIV1(hpm, copy) \ - (((hpm) << 4) | ((copy) << 0)) - -static const struct exynos_cpuclk_cfg_data e3250_armclk_d[] __initconst = { - { 1000000, E3250_CPU_DIV0(1, 7, 4, 1), E3250_CPU_DIV1(7, 7), }, - { 900000, E3250_CPU_DIV0(1, 7, 3, 1), E3250_CPU_DIV1(7, 7), }, - { 800000, E3250_CPU_DIV0(1, 7, 3, 1), E3250_CPU_DIV1(7, 7), }, - { 700000, E3250_CPU_DIV0(1, 7, 3, 1), E3250_CPU_DIV1(7, 7), }, - { 600000, E3250_CPU_DIV0(1, 7, 3, 1), E3250_CPU_DIV1(7, 7), }, - { 500000, E3250_CPU_DIV0(1, 7, 3, 1), E3250_CPU_DIV1(7, 7), }, - { 400000, E3250_CPU_DIV0(1, 7, 3, 1), E3250_CPU_DIV1(7, 7), }, - { 300000, E3250_CPU_DIV0(1, 5, 3, 1), E3250_CPU_DIV1(7, 7), }, - { 200000, E3250_CPU_DIV0(1, 3, 3, 1), E3250_CPU_DIV1(7, 7), }, - { 100000, E3250_CPU_DIV0(1, 1, 1, 1), E3250_CPU_DIV1(7, 7), }, - { 0 }, -}; - static void __init exynos3250_cmu_init(struct device_node *np) { struct samsung_clk_provider *ctx; - struct clk_hw **hws; ctx = samsung_cmu_register_one(np, &cmu_info); if (!ctx) return; - hws = ctx->clk_data.hws; - exynos_register_cpu_clock(ctx, CLK_ARM_CLK, "armclk", - hws[CLK_MOUT_APLL], hws[CLK_MOUT_MPLL_USER_C], - 0x14200, e3250_armclk_d, ARRAY_SIZE(e3250_armclk_d), - CLK_CPU_HAS_DIV1); - exynos3_core_down_clock(ctx->reg_base); } CLK_OF_DECLARE(exynos3250_cmu, "samsung,exynos3250-cmu", exynos3250_cmu_init); diff --git a/drivers/clk/samsung/clk-exynos4.c b/drivers/clk/samsung/clk-exynos4.c index bf13e29a655c..22009cb53428 100644 --- a/drivers/clk/samsung/clk-exynos4.c +++ b/drivers/clk/samsung/clk-exynos4.c @@ -437,7 +437,7 @@ static const struct samsung_mux_clock exynos4_mux_clks[] __initconst = { /* list of mux clocks supported in exynos4210 soc */ static const struct samsung_mux_clock exynos4210_mux_early[] __initconst = { - MUX(0, "mout_vpllsrc", mout_vpllsrc_p, SRC_TOP1, 0, 1), + MUX(CLK_MOUT_VPLLSRC, "mout_vpllsrc", mout_vpllsrc_p, SRC_TOP1, 0, 1), }; static const struct samsung_mux_clock exynos4210_mux_clks[] __initconst = { @@ -603,7 +603,7 @@ static const struct samsung_div_clock exynos4_div_clks[] __initconst = { DIV(0, "div_periph", "div_core2", DIV_CPU0, 12, 3), DIV(0, "div_atb", "mout_core", DIV_CPU0, 16, 3), DIV(0, "div_pclk_dbg", "div_atb", DIV_CPU0, 20, 3), - DIV(0, "div_core2", "div_core", DIV_CPU0, 28, 3), + DIV(CLK_DIV_CORE2, "div_core2", "div_core", DIV_CPU0, 28, 3), DIV(0, "div_copy", "mout_hpm", DIV_CPU1, 0, 3), DIV(0, "div_hpm", "div_copy", DIV_CPU1, 4, 3), DIV(0, "div_clkout_cpu", "mout_clkout_cpu", CLKOUT_CMU_CPU, 8, 6), @@ -1228,6 +1228,16 @@ static const struct exynos_cpuclk_cfg_data e4412_armclk_d[] __initconst = { { 0 }, }; +static const struct samsung_cpu_clock exynos4210_cpu_clks[] __initconst = { + CPU_CLK(CLK_ARM_CLK, "armclk", CLK_MOUT_APLL, CLK_SCLK_MPLL, + CLK_CPU_NEEDS_DEBUG_ALT_DIV | CLK_CPU_HAS_DIV1, 0x14200, e4210_armclk_d), +}; + +static const struct samsung_cpu_clock exynos4412_cpu_clks[] __initconst = { + CPU_CLK(CLK_ARM_CLK, "armclk", CLK_MOUT_APLL, CLK_MOUT_MPLL_USER_C, + CLK_CPU_NEEDS_DEBUG_ALT_DIV | CLK_CPU_HAS_DIV1, 0x14200, e4412_armclk_d), +}; + /* register exynos4 clocks */ static void __init exynos4_clk_init(struct device_node *np, enum exynos4_soc soc) @@ -1254,21 +1264,21 @@ static void __init exynos4_clk_init(struct device_node *np, samsung_clk_register_mux(ctx, exynos4210_mux_early, ARRAY_SIZE(exynos4210_mux_early)); - if (_get_rate("fin_pll") == 24000000) { + if (clk_hw_get_rate(hws[CLK_FIN_PLL]) == 24000000) { exynos4210_plls[apll].rate_table = exynos4210_apll_rates; exynos4210_plls[epll].rate_table = exynos4210_epll_rates; } - if (_get_rate("mout_vpllsrc") == 24000000) + if (clk_hw_get_rate(hws[CLK_MOUT_VPLLSRC]) == 24000000) exynos4210_plls[vpll].rate_table = exynos4210_vpll_rates; samsung_clk_register_pll(ctx, exynos4210_plls, ARRAY_SIZE(exynos4210_plls), reg_base); } else { - if (_get_rate("fin_pll") == 24000000) { + if (clk_hw_get_rate(hws[CLK_FIN_PLL]) == 24000000) { exynos4x12_plls[apll].rate_table = exynos4x12_apll_rates; exynos4x12_plls[epll].rate_table = @@ -1304,10 +1314,8 @@ static void __init exynos4_clk_init(struct device_node *np, samsung_clk_register_fixed_factor(ctx, exynos4210_fixed_factor_clks, ARRAY_SIZE(exynos4210_fixed_factor_clks)); - exynos_register_cpu_clock(ctx, CLK_ARM_CLK, "armclk", - hws[CLK_MOUT_APLL], hws[CLK_SCLK_MPLL], 0x14200, - e4210_armclk_d, ARRAY_SIZE(e4210_armclk_d), - CLK_CPU_NEEDS_DEBUG_ALT_DIV | CLK_CPU_HAS_DIV1); + samsung_clk_register_cpu(ctx, exynos4210_cpu_clks, + ARRAY_SIZE(exynos4210_cpu_clks)); } else { samsung_clk_register_mux(ctx, exynos4x12_mux_clks, ARRAY_SIZE(exynos4x12_mux_clks)); @@ -1318,11 +1326,8 @@ static void __init exynos4_clk_init(struct device_node *np, samsung_clk_register_fixed_factor(ctx, exynos4x12_fixed_factor_clks, ARRAY_SIZE(exynos4x12_fixed_factor_clks)); - - exynos_register_cpu_clock(ctx, CLK_ARM_CLK, "armclk", - hws[CLK_MOUT_APLL], hws[CLK_MOUT_MPLL_USER_C], 0x14200, - e4412_armclk_d, ARRAY_SIZE(e4412_armclk_d), - CLK_CPU_NEEDS_DEBUG_ALT_DIV | CLK_CPU_HAS_DIV1); + samsung_clk_register_cpu(ctx, exynos4412_cpu_clks, + ARRAY_SIZE(exynos4412_cpu_clks)); } if (soc == EXYNOS4X12) @@ -1344,9 +1349,11 @@ static void __init exynos4_clk_init(struct device_node *np, pr_info("%s clocks: sclk_apll = %ld, sclk_mpll = %ld\n" "\tsclk_epll = %ld, sclk_vpll = %ld, arm_clk = %ld\n", exynos4_soc == EXYNOS4210 ? "Exynos4210" : "Exynos4x12", - _get_rate("sclk_apll"), _get_rate("sclk_mpll"), - _get_rate("sclk_epll"), _get_rate("sclk_vpll"), - _get_rate("div_core2")); + clk_hw_get_rate(hws[CLK_SCLK_APLL]), + clk_hw_get_rate(hws[CLK_SCLK_MPLL]), + clk_hw_get_rate(hws[CLK_SCLK_EPLL]), + clk_hw_get_rate(hws[CLK_SCLK_VPLL]), + clk_hw_get_rate(hws[CLK_DIV_CORE2])); } diff --git a/drivers/clk/samsung/clk-exynos5250.c b/drivers/clk/samsung/clk-exynos5250.c index 06588fab408a..113df773ee44 100644 --- a/drivers/clk/samsung/clk-exynos5250.c +++ b/drivers/clk/samsung/clk-exynos5250.c @@ -239,7 +239,7 @@ static const struct samsung_fixed_factor_clock exynos5250_fixed_factor_clks[] __ }; static const struct samsung_mux_clock exynos5250_pll_pmux_clks[] __initconst = { - MUX(0, "mout_vpllsrc", mout_vpllsrc_p, SRC_TOP2, 0, 1), + MUX(CLK_MOUT_VPLLSRC, "mout_vpllsrc", mout_vpllsrc_p, SRC_TOP2, 0, 1), }; static const struct samsung_mux_clock exynos5250_mux_clks[] __initconst = { @@ -351,7 +351,7 @@ static const struct samsung_div_clock exynos5250_div_clks[] __initconst = { */ DIV(0, "div_arm", "mout_cpu", DIV_CPU0, 0, 3), DIV(0, "div_apll", "mout_apll", DIV_CPU0, 24, 3), - DIV(0, "div_arm2", "div_arm", DIV_CPU0, 28, 3), + DIV(CLK_DIV_ARM2, "div_arm2", "div_arm", DIV_CPU0, 28, 3), /* * CMU_TOP @@ -772,6 +772,11 @@ static const struct exynos_cpuclk_cfg_data exynos5250_armclk_d[] __initconst = { { 0 }, }; +static const struct samsung_cpu_clock exynos5250_cpu_clks[] __initconst = { + CPU_CLK(CLK_ARM_CLK, "armclk", CLK_MOUT_APLL, CLK_MOUT_MPLL, CLK_CPU_HAS_DIV1, 0x200, + exynos5250_armclk_d), +}; + static const struct of_device_id ext_clk_match[] __initconst = { { .compatible = "samsung,clock-xxti", .data = (void *)0, }, { }, @@ -801,12 +806,12 @@ static void __init exynos5250_clk_init(struct device_node *np) samsung_clk_register_mux(ctx, exynos5250_pll_pmux_clks, ARRAY_SIZE(exynos5250_pll_pmux_clks)); - if (_get_rate("fin_pll") == 24 * MHZ) { + if (clk_hw_get_rate(hws[CLK_FIN_PLL]) == 24 * MHZ) { exynos5250_plls[epll].rate_table = epll_24mhz_tbl; exynos5250_plls[apll].rate_table = apll_24mhz_tbl; } - if (_get_rate("mout_vpllsrc") == 24 * MHZ) + if (clk_hw_get_rate(hws[CLK_MOUT_VPLLSRC]) == 24 * MHZ) exynos5250_plls[vpll].rate_table = vpll_24mhz_tbl; samsung_clk_register_pll(ctx, exynos5250_plls, @@ -822,10 +827,8 @@ static void __init exynos5250_clk_init(struct device_node *np) ARRAY_SIZE(exynos5250_div_clks)); samsung_clk_register_gate(ctx, exynos5250_gate_clks, ARRAY_SIZE(exynos5250_gate_clks)); - exynos_register_cpu_clock(ctx, CLK_ARM_CLK, "armclk", - hws[CLK_MOUT_APLL], hws[CLK_MOUT_MPLL], 0x200, - exynos5250_armclk_d, ARRAY_SIZE(exynos5250_armclk_d), - CLK_CPU_HAS_DIV1); + samsung_clk_register_cpu(ctx, exynos5250_cpu_clks, + ARRAY_SIZE(exynos5250_cpu_clks)); /* * Enable arm clock down (in idle) and set arm divider @@ -855,6 +858,6 @@ static void __init exynos5250_clk_init(struct device_node *np) samsung_clk_of_add_provider(np, ctx); pr_info("Exynos5250: clock setup completed, armclk=%ld\n", - _get_rate("div_arm2")); + clk_hw_get_rate(hws[CLK_DIV_ARM2])); } CLK_OF_DECLARE_DRIVER(exynos5250_clk, "samsung,exynos5250-clock", exynos5250_clk_init); diff --git a/drivers/clk/samsung/clk-exynos5420.c b/drivers/clk/samsung/clk-exynos5420.c index 3ccd4eabd2a6..caad74dee297 100644 --- a/drivers/clk/samsung/clk-exynos5420.c +++ b/drivers/clk/samsung/clk-exynos5420.c @@ -1551,6 +1551,20 @@ static const struct exynos_cpuclk_cfg_data exynos5420_kfcclk_d[] __initconst = { { 0 }, }; +static const struct samsung_cpu_clock exynos5420_cpu_clks[] __initconst = { + CPU_CLK(CLK_ARM_CLK, "armclk", CLK_MOUT_APLL, CLK_MOUT_MSPLL_CPU, 0, 0x200, + exynos5420_eglclk_d), + CPU_CLK(CLK_KFC_CLK, "kfcclk", CLK_MOUT_KPLL, CLK_MOUT_MSPLL_KFC, 0, 0x28200, + exynos5420_kfcclk_d), +}; + +static const struct samsung_cpu_clock exynos5800_cpu_clks[] __initconst = { + CPU_CLK(CLK_ARM_CLK, "armclk", CLK_MOUT_APLL, CLK_MOUT_MSPLL_CPU, 0, 0x200, + exynos5800_eglclk_d), + CPU_CLK(CLK_KFC_CLK, "kfcclk", CLK_MOUT_KPLL, CLK_MOUT_MSPLL_KFC, 0, 0x28200, + exynos5420_kfcclk_d), +}; + static const struct of_device_id ext_clk_match[] __initconst = { { .compatible = "samsung,exynos5420-oscclk", .data = (void *)0, }, { }, @@ -1580,7 +1594,7 @@ static void __init exynos5x_clk_init(struct device_node *np, ARRAY_SIZE(exynos5x_fixed_rate_ext_clks), ext_clk_match); - if (_get_rate("fin_pll") == 24 * MHZ) { + if (clk_hw_get_rate(hws[CLK_FIN_PLL]) == 24 * MHZ) { exynos5x_plls[apll].rate_table = exynos5420_pll2550x_24mhz_tbl; exynos5x_plls[epll].rate_table = exynos5420_epll_24mhz_tbl; exynos5x_plls[kpll].rate_table = exynos5420_pll2550x_24mhz_tbl; @@ -1625,17 +1639,12 @@ static void __init exynos5x_clk_init(struct device_node *np, } if (soc == EXYNOS5420) { - exynos_register_cpu_clock(ctx, CLK_ARM_CLK, "armclk", - hws[CLK_MOUT_APLL], hws[CLK_MOUT_MSPLL_CPU], 0x200, - exynos5420_eglclk_d, ARRAY_SIZE(exynos5420_eglclk_d), 0); + samsung_clk_register_cpu(ctx, exynos5420_cpu_clks, + ARRAY_SIZE(exynos5420_cpu_clks)); } else { - exynos_register_cpu_clock(ctx, CLK_ARM_CLK, "armclk", - hws[CLK_MOUT_APLL], hws[CLK_MOUT_MSPLL_CPU], 0x200, - exynos5800_eglclk_d, ARRAY_SIZE(exynos5800_eglclk_d), 0); + samsung_clk_register_cpu(ctx, exynos5800_cpu_clks, + ARRAY_SIZE(exynos5800_cpu_clks)); } - exynos_register_cpu_clock(ctx, CLK_KFC_CLK, "kfcclk", - hws[CLK_MOUT_KPLL], hws[CLK_MOUT_MSPLL_KFC], 0x28200, - exynos5420_kfcclk_d, ARRAY_SIZE(exynos5420_kfcclk_d), 0); samsung_clk_extended_sleep_init(reg_base, exynos5x_clk_regs, ARRAY_SIZE(exynos5x_clk_regs), diff --git a/drivers/clk/samsung/clk-exynos7885.c b/drivers/clk/samsung/clk-exynos7885.c new file mode 100644 index 000000000000..a7b106302706 --- /dev/null +++ b/drivers/clk/samsung/clk-exynos7885.c @@ -0,0 +1,597 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2021 Dávid Virág + * Author: Dávid Virág + * + * Common Clock Framework support for Exynos7885 SoC. + */ + +#include +#include +#include +#include +#include + +#include + +#include "clk.h" +#include "clk-exynos-arm64.h" + +/* ---- CMU_TOP ------------------------------------------------------------- */ + +/* Register Offset definitions for CMU_TOP (0x12060000) */ +#define PLL_LOCKTIME_PLL_SHARED0 0x0000 +#define PLL_LOCKTIME_PLL_SHARED1 0x0004 +#define PLL_CON0_PLL_SHARED0 0x0100 +#define PLL_CON0_PLL_SHARED1 0x0120 +#define CLK_CON_MUX_MUX_CLKCMU_CORE_BUS 0x1014 +#define CLK_CON_MUX_MUX_CLKCMU_CORE_CCI 0x1018 +#define CLK_CON_MUX_MUX_CLKCMU_CORE_G3D 0x101c +#define CLK_CON_MUX_MUX_CLKCMU_PERI_BUS 0x1058 +#define CLK_CON_MUX_MUX_CLKCMU_PERI_SPI0 0x105c +#define CLK_CON_MUX_MUX_CLKCMU_PERI_SPI1 0x1060 +#define CLK_CON_MUX_MUX_CLKCMU_PERI_UART0 0x1064 +#define CLK_CON_MUX_MUX_CLKCMU_PERI_UART1 0x1068 +#define CLK_CON_MUX_MUX_CLKCMU_PERI_UART2 0x106c +#define CLK_CON_MUX_MUX_CLKCMU_PERI_USI0 0x1070 +#define CLK_CON_MUX_MUX_CLKCMU_PERI_USI1 0x1074 +#define CLK_CON_MUX_MUX_CLKCMU_PERI_USI2 0x1078 +#define CLK_CON_DIV_CLKCMU_CORE_BUS 0x181c +#define CLK_CON_DIV_CLKCMU_CORE_CCI 0x1820 +#define CLK_CON_DIV_CLKCMU_CORE_G3D 0x1824 +#define CLK_CON_DIV_CLKCMU_PERI_BUS 0x1874 +#define CLK_CON_DIV_CLKCMU_PERI_SPI0 0x1878 +#define CLK_CON_DIV_CLKCMU_PERI_SPI1 0x187c +#define CLK_CON_DIV_CLKCMU_PERI_UART0 0x1880 +#define CLK_CON_DIV_CLKCMU_PERI_UART1 0x1884 +#define CLK_CON_DIV_CLKCMU_PERI_UART2 0x1888 +#define CLK_CON_DIV_CLKCMU_PERI_USI0 0x188c +#define CLK_CON_DIV_CLKCMU_PERI_USI1 0x1890 +#define CLK_CON_DIV_CLKCMU_PERI_USI2 0x1894 +#define CLK_CON_DIV_PLL_SHARED0_DIV2 0x189c +#define CLK_CON_DIV_PLL_SHARED0_DIV3 0x18a0 +#define CLK_CON_DIV_PLL_SHARED0_DIV4 0x18a4 +#define CLK_CON_DIV_PLL_SHARED0_DIV5 0x18a8 +#define CLK_CON_DIV_PLL_SHARED1_DIV2 0x18ac +#define CLK_CON_DIV_PLL_SHARED1_DIV3 0x18b0 +#define CLK_CON_DIV_PLL_SHARED1_DIV4 0x18b4 +#define CLK_CON_GAT_GATE_CLKCMUC_PERI_UART1 0x2004 +#define CLK_CON_GAT_GATE_CLKCMU_CORE_BUS 0x201c +#define CLK_CON_GAT_GATE_CLKCMU_CORE_CCI 0x2020 +#define CLK_CON_GAT_GATE_CLKCMU_CORE_G3D 0x2024 +#define CLK_CON_GAT_GATE_CLKCMU_PERI_BUS 0x207c +#define CLK_CON_GAT_GATE_CLKCMU_PERI_SPI0 0x2080 +#define CLK_CON_GAT_GATE_CLKCMU_PERI_SPI1 0x2084 +#define CLK_CON_GAT_GATE_CLKCMU_PERI_UART0 0x2088 +#define CLK_CON_GAT_GATE_CLKCMU_PERI_UART2 0x208c +#define CLK_CON_GAT_GATE_CLKCMU_PERI_USI0 0x2090 +#define CLK_CON_GAT_GATE_CLKCMU_PERI_USI1 0x2094 +#define CLK_CON_GAT_GATE_CLKCMU_PERI_USI2 0x2098 + +static const unsigned long top_clk_regs[] __initconst = { + PLL_LOCKTIME_PLL_SHARED0, + PLL_LOCKTIME_PLL_SHARED1, + PLL_CON0_PLL_SHARED0, + PLL_CON0_PLL_SHARED1, + CLK_CON_MUX_MUX_CLKCMU_CORE_BUS, + CLK_CON_MUX_MUX_CLKCMU_CORE_CCI, + CLK_CON_MUX_MUX_CLKCMU_CORE_G3D, + CLK_CON_MUX_MUX_CLKCMU_PERI_BUS, + CLK_CON_MUX_MUX_CLKCMU_PERI_SPI0, + CLK_CON_MUX_MUX_CLKCMU_PERI_SPI1, + CLK_CON_MUX_MUX_CLKCMU_PERI_UART0, + CLK_CON_MUX_MUX_CLKCMU_PERI_UART1, + CLK_CON_MUX_MUX_CLKCMU_PERI_UART2, + CLK_CON_MUX_MUX_CLKCMU_PERI_USI0, + CLK_CON_MUX_MUX_CLKCMU_PERI_USI1, + CLK_CON_MUX_MUX_CLKCMU_PERI_USI2, + CLK_CON_DIV_CLKCMU_CORE_BUS, + CLK_CON_DIV_CLKCMU_CORE_CCI, + CLK_CON_DIV_CLKCMU_CORE_G3D, + CLK_CON_DIV_CLKCMU_PERI_BUS, + CLK_CON_DIV_CLKCMU_PERI_SPI0, + CLK_CON_DIV_CLKCMU_PERI_SPI1, + CLK_CON_DIV_CLKCMU_PERI_UART0, + CLK_CON_DIV_CLKCMU_PERI_UART1, + CLK_CON_DIV_CLKCMU_PERI_UART2, + CLK_CON_DIV_CLKCMU_PERI_USI0, + CLK_CON_DIV_CLKCMU_PERI_USI1, + CLK_CON_DIV_CLKCMU_PERI_USI2, + CLK_CON_DIV_PLL_SHARED0_DIV2, + CLK_CON_DIV_PLL_SHARED0_DIV3, + CLK_CON_DIV_PLL_SHARED0_DIV4, + CLK_CON_DIV_PLL_SHARED0_DIV5, + CLK_CON_DIV_PLL_SHARED1_DIV2, + CLK_CON_DIV_PLL_SHARED1_DIV3, + CLK_CON_DIV_PLL_SHARED1_DIV4, + CLK_CON_GAT_GATE_CLKCMUC_PERI_UART1, + CLK_CON_GAT_GATE_CLKCMU_CORE_BUS, + CLK_CON_GAT_GATE_CLKCMU_CORE_CCI, + CLK_CON_GAT_GATE_CLKCMU_CORE_G3D, + CLK_CON_GAT_GATE_CLKCMU_PERI_BUS, + CLK_CON_GAT_GATE_CLKCMU_PERI_SPI0, + CLK_CON_GAT_GATE_CLKCMU_PERI_SPI1, + CLK_CON_GAT_GATE_CLKCMU_PERI_UART0, + CLK_CON_GAT_GATE_CLKCMU_PERI_UART2, + CLK_CON_GAT_GATE_CLKCMU_PERI_USI0, + CLK_CON_GAT_GATE_CLKCMU_PERI_USI1, + CLK_CON_GAT_GATE_CLKCMU_PERI_USI2, +}; + +static const struct samsung_pll_clock top_pll_clks[] __initconst = { + PLL(pll_1417x, CLK_FOUT_SHARED0_PLL, "fout_shared0_pll", "oscclk", + PLL_LOCKTIME_PLL_SHARED0, PLL_CON0_PLL_SHARED0, + NULL), + PLL(pll_1417x, CLK_FOUT_SHARED1_PLL, "fout_shared1_pll", "oscclk", + PLL_LOCKTIME_PLL_SHARED1, PLL_CON0_PLL_SHARED1, + NULL), +}; + +/* List of parent clocks for Muxes in CMU_TOP: for CMU_CORE */ +PNAME(mout_core_bus_p) = { "dout_shared0_div2", "dout_shared1_div2", + "dout_shared0_div3", "dout_shared0_div3" }; +PNAME(mout_core_cci_p) = { "dout_shared0_div2", "dout_shared1_div2", + "dout_shared0_div3", "dout_shared0_div3" }; +PNAME(mout_core_g3d_p) = { "dout_shared0_div2", "dout_shared1_div2", + "dout_shared0_div3", "dout_shared0_div3" }; + +/* List of parent clocks for Muxes in CMU_TOP: for CMU_PERI */ +PNAME(mout_peri_bus_p) = { "dout_shared0_div4", "dout_shared1_div4" }; +PNAME(mout_peri_spi0_p) = { "oscclk", "dout_shared0_div4" }; +PNAME(mout_peri_spi1_p) = { "oscclk", "dout_shared0_div4" }; +PNAME(mout_peri_uart0_p) = { "oscclk", "dout_shared0_div4" }; +PNAME(mout_peri_uart1_p) = { "oscclk", "dout_shared0_div4" }; +PNAME(mout_peri_uart2_p) = { "oscclk", "dout_shared0_div4" }; +PNAME(mout_peri_usi0_p) = { "oscclk", "dout_shared0_div4" }; +PNAME(mout_peri_usi1_p) = { "oscclk", "dout_shared0_div4" }; +PNAME(mout_peri_usi2_p) = { "oscclk", "dout_shared0_div4" }; + +static const struct samsung_mux_clock top_mux_clks[] __initconst = { + /* CORE */ + MUX(CLK_MOUT_CORE_BUS, "mout_core_bus", mout_core_bus_p, + CLK_CON_MUX_MUX_CLKCMU_CORE_BUS, 0, 2), + MUX(CLK_MOUT_CORE_CCI, "mout_core_cci", mout_core_cci_p, + CLK_CON_MUX_MUX_CLKCMU_CORE_CCI, 0, 2), + MUX(CLK_MOUT_CORE_G3D, "mout_core_g3d", mout_core_g3d_p, + CLK_CON_MUX_MUX_CLKCMU_CORE_G3D, 0, 2), + + /* PERI */ + MUX(CLK_MOUT_PERI_BUS, "mout_peri_bus", mout_peri_bus_p, + CLK_CON_MUX_MUX_CLKCMU_PERI_BUS, 0, 1), + MUX(CLK_MOUT_PERI_SPI0, "mout_peri_spi0", mout_peri_spi0_p, + CLK_CON_MUX_MUX_CLKCMU_PERI_SPI0, 0, 1), + MUX(CLK_MOUT_PERI_SPI1, "mout_peri_spi1", mout_peri_spi1_p, + CLK_CON_MUX_MUX_CLKCMU_PERI_SPI1, 0, 1), + MUX(CLK_MOUT_PERI_UART0, "mout_peri_uart0", mout_peri_uart0_p, + CLK_CON_MUX_MUX_CLKCMU_PERI_UART0, 0, 1), + MUX(CLK_MOUT_PERI_UART1, "mout_peri_uart1", mout_peri_uart1_p, + CLK_CON_MUX_MUX_CLKCMU_PERI_UART1, 0, 1), + MUX(CLK_MOUT_PERI_UART2, "mout_peri_uart2", mout_peri_uart2_p, + CLK_CON_MUX_MUX_CLKCMU_PERI_UART2, 0, 1), + MUX(CLK_MOUT_PERI_USI0, "mout_peri_usi0", mout_peri_usi0_p, + CLK_CON_MUX_MUX_CLKCMU_PERI_USI0, 0, 1), + MUX(CLK_MOUT_PERI_USI1, "mout_peri_usi1", mout_peri_usi1_p, + CLK_CON_MUX_MUX_CLKCMU_PERI_USI1, 0, 1), + MUX(CLK_MOUT_PERI_USI2, "mout_peri_usi2", mout_peri_usi2_p, + CLK_CON_MUX_MUX_CLKCMU_PERI_USI2, 0, 1), +}; + +static const struct samsung_div_clock top_div_clks[] __initconst = { + /* TOP */ + DIV(CLK_DOUT_SHARED0_DIV2, "dout_shared0_div2", "fout_shared0_pll", + CLK_CON_DIV_PLL_SHARED0_DIV2, 0, 1), + DIV(CLK_DOUT_SHARED0_DIV3, "dout_shared0_div3", "fout_shared0_pll", + CLK_CON_DIV_PLL_SHARED0_DIV3, 0, 2), + DIV(CLK_DOUT_SHARED0_DIV4, "dout_shared0_div4", "fout_shared0_pll", + CLK_CON_DIV_PLL_SHARED0_DIV4, 0, 1), + DIV(CLK_DOUT_SHARED0_DIV5, "dout_shared0_div5", "fout_shared0_pll", + CLK_CON_DIV_PLL_SHARED0_DIV5, 0, 3), + DIV(CLK_DOUT_SHARED1_DIV2, "dout_shared1_div2", "fout_shared1_pll", + CLK_CON_DIV_PLL_SHARED1_DIV2, 0, 1), + DIV(CLK_DOUT_SHARED1_DIV3, "dout_shared1_div3", "fout_shared1_pll", + CLK_CON_DIV_PLL_SHARED1_DIV3, 0, 2), + DIV(CLK_DOUT_SHARED1_DIV4, "dout_shared1_div4", "fout_shared1_pll", + CLK_CON_DIV_PLL_SHARED1_DIV4, 0, 1), + + /* CORE */ + DIV(CLK_DOUT_CORE_BUS, "dout_core_bus", "gout_core_bus", + CLK_CON_DIV_CLKCMU_CORE_BUS, 0, 3), + DIV(CLK_DOUT_CORE_CCI, "dout_core_cci", "gout_core_cci", + CLK_CON_DIV_CLKCMU_CORE_CCI, 0, 3), + DIV(CLK_DOUT_CORE_G3D, "dout_core_g3d", "gout_core_g3d", + CLK_CON_DIV_CLKCMU_CORE_G3D, 0, 3), + + /* PERI */ + DIV(CLK_DOUT_PERI_BUS, "dout_peri_bus", "gout_peri_bus", + CLK_CON_DIV_CLKCMU_PERI_BUS, 0, 4), + DIV(CLK_DOUT_PERI_SPI0, "dout_peri_spi0", "gout_peri_spi0", + CLK_CON_DIV_CLKCMU_PERI_SPI0, 0, 6), + DIV(CLK_DOUT_PERI_SPI1, "dout_peri_spi1", "gout_peri_spi1", + CLK_CON_DIV_CLKCMU_PERI_SPI1, 0, 6), + DIV(CLK_DOUT_PERI_UART0, "dout_peri_uart0", "gout_peri_uart0", + CLK_CON_DIV_CLKCMU_PERI_UART0, 0, 4), + DIV(CLK_DOUT_PERI_UART1, "dout_peri_uart1", "gout_peri_uart1", + CLK_CON_DIV_CLKCMU_PERI_UART1, 0, 4), + DIV(CLK_DOUT_PERI_UART2, "dout_peri_uart2", "gout_peri_uart2", + CLK_CON_DIV_CLKCMU_PERI_UART2, 0, 4), + DIV(CLK_DOUT_PERI_USI0, "dout_peri_usi0", "gout_peri_usi0", + CLK_CON_DIV_CLKCMU_PERI_USI0, 0, 4), + DIV(CLK_DOUT_PERI_USI1, "dout_peri_usi1", "gout_peri_usi1", + CLK_CON_DIV_CLKCMU_PERI_USI1, 0, 4), + DIV(CLK_DOUT_PERI_USI2, "dout_peri_usi2", "gout_peri_usi2", + CLK_CON_DIV_CLKCMU_PERI_USI2, 0, 4), +}; + +static const struct samsung_gate_clock top_gate_clks[] __initconst = { + /* CORE */ + GATE(CLK_GOUT_CORE_BUS, "gout_core_bus", "mout_core_bus", + CLK_CON_GAT_GATE_CLKCMU_CORE_BUS, 21, 0, 0), + GATE(CLK_GOUT_CORE_CCI, "gout_core_cci", "mout_core_cci", + CLK_CON_GAT_GATE_CLKCMU_CORE_CCI, 21, 0, 0), + GATE(CLK_GOUT_CORE_G3D, "gout_core_g3d", "mout_core_g3d", + CLK_CON_GAT_GATE_CLKCMU_CORE_G3D, 21, 0, 0), + + /* PERI */ + GATE(CLK_GOUT_PERI_BUS, "gout_peri_bus", "mout_peri_bus", + CLK_CON_GAT_GATE_CLKCMU_PERI_BUS, 21, 0, 0), + GATE(CLK_GOUT_PERI_SPI0, "gout_peri_spi0", "mout_peri_spi0", + CLK_CON_GAT_GATE_CLKCMU_PERI_SPI0, 21, 0, 0), + GATE(CLK_GOUT_PERI_SPI1, "gout_peri_spi1", "mout_peri_spi1", + CLK_CON_GAT_GATE_CLKCMU_PERI_SPI1, 21, 0, 0), + GATE(CLK_GOUT_PERI_UART0, "gout_peri_uart0", "mout_peri_uart0", + CLK_CON_GAT_GATE_CLKCMU_PERI_UART0, 21, 0, 0), + GATE(CLK_GOUT_PERI_UART1, "gout_peri_uart1", "mout_peri_uart1", + CLK_CON_GAT_GATE_CLKCMUC_PERI_UART1, 21, 0, 0), + GATE(CLK_GOUT_PERI_UART2, "gout_peri_uart2", "mout_peri_uart2", + CLK_CON_GAT_GATE_CLKCMU_PERI_UART2, 21, 0, 0), + GATE(CLK_GOUT_PERI_USI0, "gout_peri_usi0", "mout_peri_usi0", + CLK_CON_GAT_GATE_CLKCMU_PERI_USI0, 21, 0, 0), + GATE(CLK_GOUT_PERI_USI1, "gout_peri_usi1", "mout_peri_usi1", + CLK_CON_GAT_GATE_CLKCMU_PERI_USI1, 21, 0, 0), + GATE(CLK_GOUT_PERI_USI2, "gout_peri_usi2", "mout_peri_usi2", + CLK_CON_GAT_GATE_CLKCMU_PERI_USI2, 21, 0, 0), +}; + +static const struct samsung_cmu_info top_cmu_info __initconst = { + .pll_clks = top_pll_clks, + .nr_pll_clks = ARRAY_SIZE(top_pll_clks), + .mux_clks = top_mux_clks, + .nr_mux_clks = ARRAY_SIZE(top_mux_clks), + .div_clks = top_div_clks, + .nr_div_clks = ARRAY_SIZE(top_div_clks), + .gate_clks = top_gate_clks, + .nr_gate_clks = ARRAY_SIZE(top_gate_clks), + .nr_clk_ids = TOP_NR_CLK, + .clk_regs = top_clk_regs, + .nr_clk_regs = ARRAY_SIZE(top_clk_regs), +}; + +static void __init exynos7885_cmu_top_init(struct device_node *np) +{ + exynos_arm64_register_cmu(NULL, np, &top_cmu_info); +} + +/* Register CMU_TOP early, as it's a dependency for other early domains */ +CLK_OF_DECLARE(exynos7885_cmu_top, "samsung,exynos7885-cmu-top", + exynos7885_cmu_top_init); + +/* ---- CMU_PERI ------------------------------------------------------------ */ + +/* Register Offset definitions for CMU_PERI (0x10010000) */ +#define PLL_CON0_MUX_CLKCMU_PERI_BUS_USER 0x0100 +#define PLL_CON0_MUX_CLKCMU_PERI_SPI0_USER 0x0120 +#define PLL_CON0_MUX_CLKCMU_PERI_SPI1_USER 0x0140 +#define PLL_CON0_MUX_CLKCMU_PERI_UART0_USER 0x0160 +#define PLL_CON0_MUX_CLKCMU_PERI_UART1_USER 0x0180 +#define PLL_CON0_MUX_CLKCMU_PERI_UART2_USER 0x01a0 +#define PLL_CON0_MUX_CLKCMU_PERI_USI0_USER 0x01c0 +#define PLL_CON0_MUX_CLKCMU_PERI_USI1_USER 0x01e0 +#define PLL_CON0_MUX_CLKCMU_PERI_USI2_USER 0x0200 +#define CLK_CON_GAT_GOUT_PERI_GPIO_TOP_PCLK 0x2024 +#define CLK_CON_GAT_GOUT_PERI_HSI2C_0_PCLK 0x2028 +#define CLK_CON_GAT_GOUT_PERI_HSI2C_1_PCLK 0x202c +#define CLK_CON_GAT_GOUT_PERI_HSI2C_2_PCLK 0x2030 +#define CLK_CON_GAT_GOUT_PERI_HSI2C_3_PCLK 0x2034 +#define CLK_CON_GAT_GOUT_PERI_I2C_0_PCLK 0x2038 +#define CLK_CON_GAT_GOUT_PERI_I2C_1_PCLK 0x203c +#define CLK_CON_GAT_GOUT_PERI_I2C_2_PCLK 0x2040 +#define CLK_CON_GAT_GOUT_PERI_I2C_3_PCLK 0x2044 +#define CLK_CON_GAT_GOUT_PERI_I2C_4_PCLK 0x2048 +#define CLK_CON_GAT_GOUT_PERI_I2C_5_PCLK 0x204c +#define CLK_CON_GAT_GOUT_PERI_I2C_6_PCLK 0x2050 +#define CLK_CON_GAT_GOUT_PERI_I2C_7_PCLK 0x2054 +#define CLK_CON_GAT_GOUT_PERI_PWM_MOTOR_PCLK 0x2058 +#define CLK_CON_GAT_GOUT_PERI_SPI_0_PCLK 0x205c +#define CLK_CON_GAT_GOUT_PERI_SPI_0_EXT_CLK 0x2060 +#define CLK_CON_GAT_GOUT_PERI_SPI_1_PCLK 0x2064 +#define CLK_CON_GAT_GOUT_PERI_SPI_1_EXT_CLK 0x2068 +#define CLK_CON_GAT_GOUT_PERI_UART_0_EXT_UCLK 0x206c +#define CLK_CON_GAT_GOUT_PERI_UART_0_PCLK 0x2070 +#define CLK_CON_GAT_GOUT_PERI_UART_1_EXT_UCLK 0x2074 +#define CLK_CON_GAT_GOUT_PERI_UART_1_PCLK 0x2078 +#define CLK_CON_GAT_GOUT_PERI_UART_2_EXT_UCLK 0x207c +#define CLK_CON_GAT_GOUT_PERI_UART_2_PCLK 0x2080 +#define CLK_CON_GAT_GOUT_PERI_USI0_PCLK 0x2084 +#define CLK_CON_GAT_GOUT_PERI_USI0_SCLK 0x2088 +#define CLK_CON_GAT_GOUT_PERI_USI1_PCLK 0x208c +#define CLK_CON_GAT_GOUT_PERI_USI1_SCLK 0x2090 +#define CLK_CON_GAT_GOUT_PERI_USI2_PCLK 0x2094 +#define CLK_CON_GAT_GOUT_PERI_USI2_SCLK 0x2098 +#define CLK_CON_GAT_GOUT_PERI_MCT_PCLK 0x20a0 +#define CLK_CON_GAT_GOUT_PERI_SYSREG_PERI_PCLK 0x20b0 +#define CLK_CON_GAT_GOUT_PERI_WDT_CLUSTER0_PCLK 0x20b4 +#define CLK_CON_GAT_GOUT_PERI_WDT_CLUSTER1_PCLK 0x20b8 + +static const unsigned long peri_clk_regs[] __initconst = { + PLL_CON0_MUX_CLKCMU_PERI_BUS_USER, + PLL_CON0_MUX_CLKCMU_PERI_SPI0_USER, + PLL_CON0_MUX_CLKCMU_PERI_SPI1_USER, + PLL_CON0_MUX_CLKCMU_PERI_UART0_USER, + PLL_CON0_MUX_CLKCMU_PERI_UART1_USER, + PLL_CON0_MUX_CLKCMU_PERI_UART2_USER, + PLL_CON0_MUX_CLKCMU_PERI_USI0_USER, + PLL_CON0_MUX_CLKCMU_PERI_USI1_USER, + PLL_CON0_MUX_CLKCMU_PERI_USI2_USER, + CLK_CON_GAT_GOUT_PERI_GPIO_TOP_PCLK, + CLK_CON_GAT_GOUT_PERI_HSI2C_0_PCLK, + CLK_CON_GAT_GOUT_PERI_HSI2C_1_PCLK, + CLK_CON_GAT_GOUT_PERI_HSI2C_2_PCLK, + CLK_CON_GAT_GOUT_PERI_HSI2C_3_PCLK, + CLK_CON_GAT_GOUT_PERI_I2C_0_PCLK, + CLK_CON_GAT_GOUT_PERI_I2C_1_PCLK, + CLK_CON_GAT_GOUT_PERI_I2C_2_PCLK, + CLK_CON_GAT_GOUT_PERI_I2C_3_PCLK, + CLK_CON_GAT_GOUT_PERI_I2C_4_PCLK, + CLK_CON_GAT_GOUT_PERI_I2C_5_PCLK, + CLK_CON_GAT_GOUT_PERI_I2C_6_PCLK, + CLK_CON_GAT_GOUT_PERI_I2C_7_PCLK, + CLK_CON_GAT_GOUT_PERI_PWM_MOTOR_PCLK, + CLK_CON_GAT_GOUT_PERI_SPI_0_PCLK, + CLK_CON_GAT_GOUT_PERI_SPI_0_EXT_CLK, + CLK_CON_GAT_GOUT_PERI_SPI_1_PCLK, + CLK_CON_GAT_GOUT_PERI_SPI_1_EXT_CLK, + CLK_CON_GAT_GOUT_PERI_UART_0_EXT_UCLK, + CLK_CON_GAT_GOUT_PERI_UART_0_PCLK, + CLK_CON_GAT_GOUT_PERI_UART_1_EXT_UCLK, + CLK_CON_GAT_GOUT_PERI_UART_1_PCLK, + CLK_CON_GAT_GOUT_PERI_UART_2_EXT_UCLK, + CLK_CON_GAT_GOUT_PERI_UART_2_PCLK, + CLK_CON_GAT_GOUT_PERI_USI0_PCLK, + CLK_CON_GAT_GOUT_PERI_USI0_SCLK, + CLK_CON_GAT_GOUT_PERI_USI1_PCLK, + CLK_CON_GAT_GOUT_PERI_USI1_SCLK, + CLK_CON_GAT_GOUT_PERI_USI2_PCLK, + CLK_CON_GAT_GOUT_PERI_USI2_SCLK, + CLK_CON_GAT_GOUT_PERI_MCT_PCLK, + CLK_CON_GAT_GOUT_PERI_SYSREG_PERI_PCLK, + CLK_CON_GAT_GOUT_PERI_WDT_CLUSTER0_PCLK, + CLK_CON_GAT_GOUT_PERI_WDT_CLUSTER1_PCLK, +}; + +/* List of parent clocks for Muxes in CMU_PERI */ +PNAME(mout_peri_bus_user_p) = { "oscclk", "dout_peri_bus" }; +PNAME(mout_peri_spi0_user_p) = { "oscclk", "dout_peri_spi0" }; +PNAME(mout_peri_spi1_user_p) = { "oscclk", "dout_peri_spi1" }; +PNAME(mout_peri_uart0_user_p) = { "oscclk", "dout_peri_uart0" }; +PNAME(mout_peri_uart1_user_p) = { "oscclk", "dout_peri_uart1" }; +PNAME(mout_peri_uart2_user_p) = { "oscclk", "dout_peri_uart2" }; +PNAME(mout_peri_usi0_user_p) = { "oscclk", "dout_peri_usi0" }; +PNAME(mout_peri_usi1_user_p) = { "oscclk", "dout_peri_usi1" }; +PNAME(mout_peri_usi2_user_p) = { "oscclk", "dout_peri_usi2" }; + +static const struct samsung_mux_clock peri_mux_clks[] __initconst = { + MUX(CLK_MOUT_PERI_BUS_USER, "mout_peri_bus_user", mout_peri_bus_user_p, + PLL_CON0_MUX_CLKCMU_PERI_BUS_USER, 4, 1), + MUX(CLK_MOUT_PERI_SPI0_USER, "mout_peri_spi0_user", mout_peri_spi0_user_p, + PLL_CON0_MUX_CLKCMU_PERI_SPI0_USER, 4, 1), + MUX(CLK_MOUT_PERI_SPI1_USER, "mout_peri_spi1_user", mout_peri_spi1_user_p, + PLL_CON0_MUX_CLKCMU_PERI_SPI1_USER, 4, 1), + MUX(CLK_MOUT_PERI_UART0_USER, "mout_peri_uart0_user", + mout_peri_uart0_user_p, PLL_CON0_MUX_CLKCMU_PERI_UART0_USER, 4, 1), + MUX(CLK_MOUT_PERI_UART1_USER, "mout_peri_uart1_user", + mout_peri_uart1_user_p, PLL_CON0_MUX_CLKCMU_PERI_UART1_USER, 4, 1), + MUX(CLK_MOUT_PERI_UART2_USER, "mout_peri_uart2_user", + mout_peri_uart2_user_p, PLL_CON0_MUX_CLKCMU_PERI_UART2_USER, 4, 1), + MUX(CLK_MOUT_PERI_USI0_USER, "mout_peri_usi0_user", + mout_peri_usi0_user_p, PLL_CON0_MUX_CLKCMU_PERI_USI0_USER, 4, 1), + MUX(CLK_MOUT_PERI_USI1_USER, "mout_peri_usi1_user", + mout_peri_usi1_user_p, PLL_CON0_MUX_CLKCMU_PERI_USI1_USER, 4, 1), + MUX(CLK_MOUT_PERI_USI2_USER, "mout_peri_usi2_user", + mout_peri_usi2_user_p, PLL_CON0_MUX_CLKCMU_PERI_USI2_USER, 4, 1), +}; + +static const struct samsung_gate_clock peri_gate_clks[] __initconst = { + /* TODO: Should be enabled in GPIO driver (or made CLK_IS_CRITICAL) */ + GATE(CLK_GOUT_GPIO_TOP_PCLK, "gout_gpio_top_pclk", + "mout_peri_bus_user", + CLK_CON_GAT_GOUT_PERI_GPIO_TOP_PCLK, 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_HSI2C0_PCLK, "gout_hsi2c0_pclk", "mout_peri_bus_user", + CLK_CON_GAT_GOUT_PERI_HSI2C_0_PCLK, 21, 0, 0), + GATE(CLK_GOUT_HSI2C1_PCLK, "gout_hsi2c1_pclk", "mout_peri_bus_user", + CLK_CON_GAT_GOUT_PERI_HSI2C_1_PCLK, 21, 0, 0), + GATE(CLK_GOUT_HSI2C2_PCLK, "gout_hsi2c2_pclk", "mout_peri_bus_user", + CLK_CON_GAT_GOUT_PERI_HSI2C_2_PCLK, 21, 0, 0), + GATE(CLK_GOUT_HSI2C3_PCLK, "gout_hsi2c3_pclk", "mout_peri_bus_user", + CLK_CON_GAT_GOUT_PERI_HSI2C_3_PCLK, 21, 0, 0), + GATE(CLK_GOUT_I2C0_PCLK, "gout_i2c0_pclk", "mout_peri_bus_user", + CLK_CON_GAT_GOUT_PERI_I2C_0_PCLK, 21, 0, 0), + GATE(CLK_GOUT_I2C1_PCLK, "gout_i2c1_pclk", "mout_peri_bus_user", + CLK_CON_GAT_GOUT_PERI_I2C_1_PCLK, 21, 0, 0), + GATE(CLK_GOUT_I2C2_PCLK, "gout_i2c2_pclk", "mout_peri_bus_user", + CLK_CON_GAT_GOUT_PERI_I2C_2_PCLK, 21, 0, 0), + GATE(CLK_GOUT_I2C3_PCLK, "gout_i2c3_pclk", "mout_peri_bus_user", + CLK_CON_GAT_GOUT_PERI_I2C_3_PCLK, 21, 0, 0), + GATE(CLK_GOUT_I2C4_PCLK, "gout_i2c4_pclk", "mout_peri_bus_user", + CLK_CON_GAT_GOUT_PERI_I2C_4_PCLK, 21, 0, 0), + GATE(CLK_GOUT_I2C5_PCLK, "gout_i2c5_pclk", "mout_peri_bus_user", + CLK_CON_GAT_GOUT_PERI_I2C_5_PCLK, 21, 0, 0), + GATE(CLK_GOUT_I2C6_PCLK, "gout_i2c6_pclk", "mout_peri_bus_user", + CLK_CON_GAT_GOUT_PERI_I2C_6_PCLK, 21, 0, 0), + GATE(CLK_GOUT_I2C7_PCLK, "gout_i2c7_pclk", "mout_peri_bus_user", + CLK_CON_GAT_GOUT_PERI_I2C_7_PCLK, 21, 0, 0), + GATE(CLK_GOUT_PWM_MOTOR_PCLK, "gout_pwm_motor_pclk", + "mout_peri_bus_user", + CLK_CON_GAT_GOUT_PERI_PWM_MOTOR_PCLK, 21, 0, 0), + GATE(CLK_GOUT_SPI0_PCLK, "gout_spi0_pclk", "mout_peri_bus_user", + CLK_CON_GAT_GOUT_PERI_SPI_0_PCLK, 21, 0, 0), + GATE(CLK_GOUT_SPI0_EXT_CLK, "gout_spi0_ipclk", "mout_peri_spi0_user", + CLK_CON_GAT_GOUT_PERI_SPI_0_EXT_CLK, 21, 0, 0), + GATE(CLK_GOUT_SPI1_PCLK, "gout_spi1_pclk", "mout_peri_bus_user", + CLK_CON_GAT_GOUT_PERI_SPI_1_PCLK, 21, 0, 0), + GATE(CLK_GOUT_SPI1_EXT_CLK, "gout_spi1_ipclk", "mout_peri_spi1_user", + CLK_CON_GAT_GOUT_PERI_SPI_1_EXT_CLK, 21, 0, 0), + GATE(CLK_GOUT_UART0_EXT_UCLK, "gout_uart0_ext_uclk", "mout_peri_uart0_user", + CLK_CON_GAT_GOUT_PERI_UART_0_EXT_UCLK, 21, 0, 0), + GATE(CLK_GOUT_UART0_PCLK, "gout_uart0_pclk", "mout_peri_bus_user", + CLK_CON_GAT_GOUT_PERI_UART_0_PCLK, 21, 0, 0), + GATE(CLK_GOUT_UART1_EXT_UCLK, "gout_uart1_ext_uclk", "mout_peri_uart1_user", + CLK_CON_GAT_GOUT_PERI_UART_1_EXT_UCLK, 21, 0, 0), + GATE(CLK_GOUT_UART1_PCLK, "gout_uart1_pclk", "mout_peri_bus_user", + CLK_CON_GAT_GOUT_PERI_UART_1_PCLK, 21, 0, 0), + GATE(CLK_GOUT_UART2_EXT_UCLK, "gout_uart2_ext_uclk", "mout_peri_uart2_user", + CLK_CON_GAT_GOUT_PERI_UART_2_EXT_UCLK, 21, 0, 0), + GATE(CLK_GOUT_UART2_PCLK, "gout_uart2_pclk", "mout_peri_bus_user", + CLK_CON_GAT_GOUT_PERI_UART_2_PCLK, 21, 0, 0), + GATE(CLK_GOUT_USI0_PCLK, "gout_usi0_pclk", "mout_peri_bus_user", + CLK_CON_GAT_GOUT_PERI_USI0_PCLK, 21, 0, 0), + GATE(CLK_GOUT_USI0_SCLK, "gout_usi0_sclk", "mout_peri_usi0_user", + CLK_CON_GAT_GOUT_PERI_USI0_SCLK, 21, 0, 0), + GATE(CLK_GOUT_USI1_PCLK, "gout_usi1_pclk", "mout_peri_bus_user", + CLK_CON_GAT_GOUT_PERI_USI1_PCLK, 21, 0, 0), + GATE(CLK_GOUT_USI1_SCLK, "gout_usi1_sclk", "mout_peri_usi1_user", + CLK_CON_GAT_GOUT_PERI_USI1_SCLK, 21, 0, 0), + GATE(CLK_GOUT_USI2_PCLK, "gout_usi2_pclk", "mout_peri_bus_user", + CLK_CON_GAT_GOUT_PERI_USI2_PCLK, 21, 0, 0), + GATE(CLK_GOUT_USI2_SCLK, "gout_usi2_sclk", "mout_peri_usi2_user", + CLK_CON_GAT_GOUT_PERI_USI2_SCLK, 21, 0, 0), + GATE(CLK_GOUT_MCT_PCLK, "gout_mct_pclk", "mout_peri_bus_user", + CLK_CON_GAT_GOUT_PERI_MCT_PCLK, 21, 0, 0), + GATE(CLK_GOUT_SYSREG_PERI_PCLK, "gout_sysreg_peri_pclk", + "mout_peri_bus_user", + CLK_CON_GAT_GOUT_PERI_SYSREG_PERI_PCLK, 21, 0, 0), + GATE(CLK_GOUT_WDT0_PCLK, "gout_wdt0_pclk", "mout_peri_bus_user", + CLK_CON_GAT_GOUT_PERI_WDT_CLUSTER0_PCLK, 21, 0, 0), + GATE(CLK_GOUT_WDT1_PCLK, "gout_wdt1_pclk", "mout_peri_bus_user", + CLK_CON_GAT_GOUT_PERI_WDT_CLUSTER1_PCLK, 21, 0, 0), +}; + +static const struct samsung_cmu_info peri_cmu_info __initconst = { + .mux_clks = peri_mux_clks, + .nr_mux_clks = ARRAY_SIZE(peri_mux_clks), + .gate_clks = peri_gate_clks, + .nr_gate_clks = ARRAY_SIZE(peri_gate_clks), + .nr_clk_ids = PERI_NR_CLK, + .clk_regs = peri_clk_regs, + .nr_clk_regs = ARRAY_SIZE(peri_clk_regs), + .clk_name = "dout_peri_bus", +}; + +static void __init exynos7885_cmu_peri_init(struct device_node *np) +{ + exynos_arm64_register_cmu(NULL, np, &peri_cmu_info); +} + +/* Register CMU_PERI early, as it's needed for MCT timer */ +CLK_OF_DECLARE(exynos7885_cmu_peri, "samsung,exynos7885-cmu-peri", + exynos7885_cmu_peri_init); + +/* ---- CMU_CORE ------------------------------------------------------------ */ + +/* Register Offset definitions for CMU_CORE (0x12000000) */ +#define PLL_CON0_MUX_CLKCMU_CORE_BUS_USER 0x0100 +#define PLL_CON0_MUX_CLKCMU_CORE_CCI_USER 0x0120 +#define PLL_CON0_MUX_CLKCMU_CORE_G3D_USER 0x0140 +#define CLK_CON_MUX_MUX_CLK_CORE_GIC 0x1000 +#define CLK_CON_DIV_DIV_CLK_CORE_BUSP 0x1800 +#define CLK_CON_GAT_GOUT_CORE_CCI_550_ACLK 0x2054 +#define CLK_CON_GAT_GOUT_CORE_GIC400_CLK 0x2058 + +static const unsigned long core_clk_regs[] __initconst = { + PLL_CON0_MUX_CLKCMU_CORE_BUS_USER, + PLL_CON0_MUX_CLKCMU_CORE_CCI_USER, + PLL_CON0_MUX_CLKCMU_CORE_G3D_USER, + CLK_CON_MUX_MUX_CLK_CORE_GIC, + CLK_CON_DIV_DIV_CLK_CORE_BUSP, + CLK_CON_GAT_GOUT_CORE_CCI_550_ACLK, + CLK_CON_GAT_GOUT_CORE_GIC400_CLK, +}; + +/* List of parent clocks for Muxes in CMU_CORE */ +PNAME(mout_core_bus_user_p) = { "oscclk", "dout_core_bus" }; +PNAME(mout_core_cci_user_p) = { "oscclk", "dout_core_cci" }; +PNAME(mout_core_g3d_user_p) = { "oscclk", "dout_core_g3d" }; +PNAME(mout_core_gic_p) = { "dout_core_busp", "oscclk" }; + +static const struct samsung_mux_clock core_mux_clks[] __initconst = { + MUX(CLK_MOUT_CORE_BUS_USER, "mout_core_bus_user", mout_core_bus_user_p, + PLL_CON0_MUX_CLKCMU_CORE_BUS_USER, 4, 1), + MUX(CLK_MOUT_CORE_CCI_USER, "mout_core_cci_user", mout_core_cci_user_p, + PLL_CON0_MUX_CLKCMU_CORE_CCI_USER, 4, 1), + MUX(CLK_MOUT_CORE_G3D_USER, "mout_core_g3d_user", mout_core_g3d_user_p, + PLL_CON0_MUX_CLKCMU_CORE_G3D_USER, 4, 1), + MUX(CLK_MOUT_CORE_GIC, "mout_core_gic", mout_core_gic_p, + CLK_CON_MUX_MUX_CLK_CORE_GIC, 0, 1), +}; + +static const struct samsung_div_clock core_div_clks[] __initconst = { + DIV(CLK_DOUT_CORE_BUSP, "dout_core_busp", "mout_core_bus_user", + CLK_CON_DIV_DIV_CLK_CORE_BUSP, 0, 2), +}; + +static const struct samsung_gate_clock core_gate_clks[] __initconst = { + /* CCI (interconnect) clock must be always running */ + GATE(CLK_GOUT_CCI_ACLK, "gout_cci_aclk", "mout_core_cci_user", + CLK_CON_GAT_GOUT_CORE_CCI_550_ACLK, 21, CLK_IS_CRITICAL, 0), + /* GIC (interrupt controller) clock must be always running */ + GATE(CLK_GOUT_GIC400_CLK, "gout_gic400_clk", "mout_core_gic", + CLK_CON_GAT_GOUT_CORE_GIC400_CLK, 21, CLK_IS_CRITICAL, 0), +}; + +static const struct samsung_cmu_info core_cmu_info __initconst = { + .mux_clks = core_mux_clks, + .nr_mux_clks = ARRAY_SIZE(core_mux_clks), + .div_clks = core_div_clks, + .nr_div_clks = ARRAY_SIZE(core_div_clks), + .gate_clks = core_gate_clks, + .nr_gate_clks = ARRAY_SIZE(core_gate_clks), + .nr_clk_ids = CORE_NR_CLK, + .clk_regs = core_clk_regs, + .nr_clk_regs = ARRAY_SIZE(core_clk_regs), + .clk_name = "dout_core_bus", +}; + +/* ---- platform_driver ----------------------------------------------------- */ + +static int __init exynos7885_cmu_probe(struct platform_device *pdev) +{ + const struct samsung_cmu_info *info; + struct device *dev = &pdev->dev; + + info = of_device_get_match_data(dev); + exynos_arm64_register_cmu(dev, dev->of_node, info); + + return 0; +} + +static const struct of_device_id exynos7885_cmu_of_match[] = { + { + .compatible = "samsung,exynos7885-cmu-core", + .data = &core_cmu_info, + }, { + }, +}; + +static struct platform_driver exynos7885_cmu_driver __refdata = { + .driver = { + .name = "exynos7885-cmu", + .of_match_table = exynos7885_cmu_of_match, + .suppress_bind_attrs = true, + }, + .probe = exynos7885_cmu_probe, +}; + +static int __init exynos7885_cmu_init(void) +{ + return platform_driver_register(&exynos7885_cmu_driver); +} +core_initcall(exynos7885_cmu_init); diff --git a/drivers/clk/samsung/clk-exynos850.c b/drivers/clk/samsung/clk-exynos850.c index 2294989e244c..cd9725f1dbf7 100644 --- a/drivers/clk/samsung/clk-exynos850.c +++ b/drivers/clk/samsung/clk-exynos850.c @@ -9,56 +9,13 @@ #include #include #include -#include #include #include #include #include "clk.h" - -/* Gate register bits */ -#define GATE_MANUAL BIT(20) -#define GATE_ENABLE_HWACG BIT(28) - -/* Gate register offsets range */ -#define GATE_OFF_START 0x2000 -#define GATE_OFF_END 0x2fff - -/** - * exynos850_init_clocks - Set clocks initial configuration - * @np: CMU device tree node with "reg" property (CMU addr) - * @reg_offs: Register offsets array for clocks to init - * @reg_offs_len: Number of register offsets in reg_offs array - * - * Set manual control mode for all gate clocks. - */ -static void __init exynos850_init_clocks(struct device_node *np, - const unsigned long *reg_offs, size_t reg_offs_len) -{ - void __iomem *reg_base; - size_t i; - - reg_base = of_iomap(np, 0); - if (!reg_base) - panic("%s: failed to map registers\n", __func__); - - for (i = 0; i < reg_offs_len; ++i) { - void __iomem *reg = reg_base + reg_offs[i]; - u32 val; - - /* Modify only gate clock registers */ - if (reg_offs[i] < GATE_OFF_START || reg_offs[i] > GATE_OFF_END) - continue; - - val = readl(reg); - val |= GATE_MANUAL; - val &= ~GATE_ENABLE_HWACG; - writel(val, reg); - } - - iounmap(reg_base); -} +#include "clk-exynos-arm64.h" /* ---- CMU_TOP ------------------------------------------------------------- */ @@ -72,6 +29,7 @@ static void __init exynos850_init_clocks(struct device_node *np, #define PLL_CON3_PLL_SHARED0 0x014c #define PLL_CON0_PLL_SHARED1 0x0180 #define PLL_CON3_PLL_SHARED1 0x018c +#define CLK_CON_MUX_MUX_CLKCMU_APM_BUS 0x1000 #define CLK_CON_MUX_MUX_CLKCMU_CORE_BUS 0x1014 #define CLK_CON_MUX_MUX_CLKCMU_CORE_CCI 0x1018 #define CLK_CON_MUX_MUX_CLKCMU_CORE_MMC_EMBD 0x101c @@ -83,6 +41,7 @@ static void __init exynos850_init_clocks(struct device_node *np, #define CLK_CON_MUX_MUX_CLKCMU_PERI_BUS 0x1070 #define CLK_CON_MUX_MUX_CLKCMU_PERI_IP 0x1074 #define CLK_CON_MUX_MUX_CLKCMU_PERI_UART 0x1078 +#define CLK_CON_DIV_CLKCMU_APM_BUS 0x180c #define CLK_CON_DIV_CLKCMU_CORE_BUS 0x1820 #define CLK_CON_DIV_CLKCMU_CORE_CCI 0x1824 #define CLK_CON_DIV_CLKCMU_CORE_MMC_EMBD 0x1828 @@ -100,6 +59,7 @@ static void __init exynos850_init_clocks(struct device_node *np, #define CLK_CON_DIV_PLL_SHARED1_DIV2 0x1898 #define CLK_CON_DIV_PLL_SHARED1_DIV3 0x189c #define CLK_CON_DIV_PLL_SHARED1_DIV4 0x18a0 +#define CLK_CON_GAT_GATE_CLKCMU_APM_BUS 0x2008 #define CLK_CON_GAT_GATE_CLKCMU_CORE_BUS 0x201c #define CLK_CON_GAT_GATE_CLKCMU_CORE_CCI 0x2020 #define CLK_CON_GAT_GATE_CLKCMU_CORE_MMC_EMBD 0x2024 @@ -122,6 +82,7 @@ static const unsigned long top_clk_regs[] __initconst = { PLL_CON3_PLL_SHARED0, PLL_CON0_PLL_SHARED1, PLL_CON3_PLL_SHARED1, + CLK_CON_MUX_MUX_CLKCMU_APM_BUS, CLK_CON_MUX_MUX_CLKCMU_CORE_BUS, CLK_CON_MUX_MUX_CLKCMU_CORE_CCI, CLK_CON_MUX_MUX_CLKCMU_CORE_MMC_EMBD, @@ -133,6 +94,7 @@ static const unsigned long top_clk_regs[] __initconst = { CLK_CON_MUX_MUX_CLKCMU_PERI_BUS, CLK_CON_MUX_MUX_CLKCMU_PERI_IP, CLK_CON_MUX_MUX_CLKCMU_PERI_UART, + CLK_CON_DIV_CLKCMU_APM_BUS, CLK_CON_DIV_CLKCMU_CORE_BUS, CLK_CON_DIV_CLKCMU_CORE_CCI, CLK_CON_DIV_CLKCMU_CORE_MMC_EMBD, @@ -150,6 +112,7 @@ static const unsigned long top_clk_regs[] __initconst = { CLK_CON_DIV_PLL_SHARED1_DIV2, CLK_CON_DIV_PLL_SHARED1_DIV3, CLK_CON_DIV_PLL_SHARED1_DIV4, + CLK_CON_GAT_GATE_CLKCMU_APM_BUS, CLK_CON_GAT_GATE_CLKCMU_CORE_BUS, CLK_CON_GAT_GATE_CLKCMU_CORE_CCI, CLK_CON_GAT_GATE_CLKCMU_CORE_MMC_EMBD, @@ -183,6 +146,8 @@ static const struct samsung_pll_clock top_pll_clks[] __initconst = { PNAME(mout_shared0_pll_p) = { "oscclk", "fout_shared0_pll" }; PNAME(mout_shared1_pll_p) = { "oscclk", "fout_shared1_pll" }; PNAME(mout_mmc_pll_p) = { "oscclk", "fout_mmc_pll" }; +/* List of parent clocks for Muxes in CMU_TOP: for CMU_APM */ +PNAME(mout_clkcmu_apm_bus_p) = { "dout_shared0_div4", "pll_shared1_div4" }; /* List of parent clocks for Muxes in CMU_TOP: for CMU_CORE */ PNAME(mout_core_bus_p) = { "dout_shared1_div2", "dout_shared0_div3", "dout_shared1_div3", "dout_shared0_div4" }; @@ -222,6 +187,10 @@ static const struct samsung_mux_clock top_mux_clks[] __initconst = { MUX(CLK_MOUT_MMC_PLL, "mout_mmc_pll", mout_mmc_pll_p, PLL_CON0_PLL_MMC, 4, 1), + /* APM */ + MUX(CLK_MOUT_CLKCMU_APM_BUS, "mout_clkcmu_apm_bus", + mout_clkcmu_apm_bus_p, CLK_CON_MUX_MUX_CLKCMU_APM_BUS, 0, 1), + /* CORE */ MUX(CLK_MOUT_CORE_BUS, "mout_core_bus", mout_core_bus_p, CLK_CON_MUX_MUX_CLKCMU_CORE_BUS, 0, 2), @@ -268,6 +237,10 @@ static const struct samsung_div_clock top_div_clks[] __initconst = { DIV(CLK_DOUT_SHARED1_DIV4, "dout_shared1_div4", "dout_shared1_div2", CLK_CON_DIV_PLL_SHARED1_DIV4, 0, 1), + /* APM */ + DIV(CLK_DOUT_CLKCMU_APM_BUS, "dout_clkcmu_apm_bus", + "gout_clkcmu_apm_bus", CLK_CON_DIV_CLKCMU_APM_BUS, 0, 3), + /* CORE */ DIV(CLK_DOUT_CORE_BUS, "dout_core_bus", "gout_core_bus", CLK_CON_DIV_CLKCMU_CORE_BUS, 0, 4), @@ -310,6 +283,10 @@ static const struct samsung_gate_clock top_gate_clks[] __initconst = { GATE(CLK_GOUT_CORE_SSS, "gout_core_sss", "mout_core_sss", CLK_CON_GAT_GATE_CLKCMU_CORE_SSS, 21, 0, 0), + /* APM */ + GATE(CLK_GOUT_CLKCMU_APM_BUS, "gout_clkcmu_apm_bus", + "mout_clkcmu_apm_bus", CLK_CON_GAT_GATE_CLKCMU_APM_BUS, 21, 0, 0), + /* DPU */ GATE(CLK_GOUT_DPU, "gout_dpu", "mout_dpu", CLK_CON_GAT_GATE_CLKCMU_DPU, 21, 0, 0), @@ -347,13 +324,248 @@ static const struct samsung_cmu_info top_cmu_info __initconst = { static void __init exynos850_cmu_top_init(struct device_node *np) { - exynos850_init_clocks(np, top_clk_regs, ARRAY_SIZE(top_clk_regs)); - samsung_cmu_register_one(np, &top_cmu_info); + exynos_arm64_register_cmu(NULL, np, &top_cmu_info); } +/* Register CMU_TOP early, as it's a dependency for other early domains */ CLK_OF_DECLARE(exynos850_cmu_top, "samsung,exynos850-cmu-top", exynos850_cmu_top_init); +/* ---- CMU_APM ------------------------------------------------------------- */ + +/* Register Offset definitions for CMU_APM (0x11800000) */ +#define PLL_CON0_MUX_CLKCMU_APM_BUS_USER 0x0600 +#define PLL_CON0_MUX_CLK_RCO_APM_I3C_USER 0x0610 +#define PLL_CON0_MUX_CLK_RCO_APM_USER 0x0620 +#define PLL_CON0_MUX_DLL_USER 0x0630 +#define CLK_CON_MUX_MUX_CLKCMU_CHUB_BUS 0x1000 +#define CLK_CON_MUX_MUX_CLK_APM_BUS 0x1004 +#define CLK_CON_MUX_MUX_CLK_APM_I3C 0x1008 +#define CLK_CON_DIV_CLKCMU_CHUB_BUS 0x1800 +#define CLK_CON_DIV_DIV_CLK_APM_BUS 0x1804 +#define CLK_CON_DIV_DIV_CLK_APM_I3C 0x1808 +#define CLK_CON_GAT_CLKCMU_CMGP_BUS 0x2000 +#define CLK_CON_GAT_GATE_CLKCMU_CHUB_BUS 0x2014 +#define CLK_CON_GAT_GOUT_APM_APBIF_GPIO_ALIVE_PCLK 0x2018 +#define CLK_CON_GAT_GOUT_APM_APBIF_PMU_ALIVE_PCLK 0x2020 +#define CLK_CON_GAT_GOUT_APM_APBIF_RTC_PCLK 0x2024 +#define CLK_CON_GAT_GOUT_APM_APBIF_TOP_RTC_PCLK 0x2028 +#define CLK_CON_GAT_GOUT_APM_I3C_APM_PMIC_I_PCLK 0x2034 +#define CLK_CON_GAT_GOUT_APM_I3C_APM_PMIC_I_SCLK 0x2038 +#define CLK_CON_GAT_GOUT_APM_SPEEDY_APM_PCLK 0x20bc +#define CLK_CON_GAT_GOUT_APM_SYSREG_APM_PCLK 0x20c0 + +static const unsigned long apm_clk_regs[] __initconst = { + PLL_CON0_MUX_CLKCMU_APM_BUS_USER, + PLL_CON0_MUX_CLK_RCO_APM_I3C_USER, + PLL_CON0_MUX_CLK_RCO_APM_USER, + PLL_CON0_MUX_DLL_USER, + CLK_CON_MUX_MUX_CLKCMU_CHUB_BUS, + CLK_CON_MUX_MUX_CLK_APM_BUS, + CLK_CON_MUX_MUX_CLK_APM_I3C, + CLK_CON_DIV_CLKCMU_CHUB_BUS, + CLK_CON_DIV_DIV_CLK_APM_BUS, + CLK_CON_DIV_DIV_CLK_APM_I3C, + CLK_CON_GAT_CLKCMU_CMGP_BUS, + CLK_CON_GAT_GATE_CLKCMU_CHUB_BUS, + CLK_CON_GAT_GOUT_APM_APBIF_GPIO_ALIVE_PCLK, + CLK_CON_GAT_GOUT_APM_APBIF_PMU_ALIVE_PCLK, + CLK_CON_GAT_GOUT_APM_APBIF_RTC_PCLK, + CLK_CON_GAT_GOUT_APM_APBIF_TOP_RTC_PCLK, + CLK_CON_GAT_GOUT_APM_I3C_APM_PMIC_I_PCLK, + CLK_CON_GAT_GOUT_APM_I3C_APM_PMIC_I_SCLK, + CLK_CON_GAT_GOUT_APM_SPEEDY_APM_PCLK, + CLK_CON_GAT_GOUT_APM_SYSREG_APM_PCLK, +}; + +/* List of parent clocks for Muxes in CMU_APM */ +PNAME(mout_apm_bus_user_p) = { "oscclk_rco_apm", "dout_clkcmu_apm_bus" }; +PNAME(mout_rco_apm_i3c_user_p) = { "oscclk_rco_apm", "clk_rco_i3c_pmic" }; +PNAME(mout_rco_apm_user_p) = { "oscclk_rco_apm", "clk_rco_apm__alv" }; +PNAME(mout_dll_user_p) = { "oscclk_rco_apm", "clk_dll_dco" }; +PNAME(mout_clkcmu_chub_bus_p) = { "mout_apm_bus_user", "mout_dll_user" }; +PNAME(mout_apm_bus_p) = { "mout_rco_apm_user", "mout_apm_bus_user", + "mout_dll_user", "oscclk_rco_apm" }; +PNAME(mout_apm_i3c_p) = { "dout_apm_i3c", "mout_rco_apm_i3c_user" }; + +static const struct samsung_fixed_rate_clock apm_fixed_clks[] __initconst = { + FRATE(CLK_RCO_I3C_PMIC, "clk_rco_i3c_pmic", NULL, 0, 491520000), + FRATE(OSCCLK_RCO_APM, "oscclk_rco_apm", NULL, 0, 24576000), + FRATE(CLK_RCO_APM__ALV, "clk_rco_apm__alv", NULL, 0, 49152000), + FRATE(CLK_DLL_DCO, "clk_dll_dco", NULL, 0, 360000000), +}; + +static const struct samsung_mux_clock apm_mux_clks[] __initconst = { + MUX(CLK_MOUT_APM_BUS_USER, "mout_apm_bus_user", mout_apm_bus_user_p, + PLL_CON0_MUX_CLKCMU_APM_BUS_USER, 4, 1), + MUX(CLK_MOUT_RCO_APM_I3C_USER, "mout_rco_apm_i3c_user", + mout_rco_apm_i3c_user_p, PLL_CON0_MUX_CLK_RCO_APM_I3C_USER, 4, 1), + MUX(CLK_MOUT_RCO_APM_USER, "mout_rco_apm_user", mout_rco_apm_user_p, + PLL_CON0_MUX_CLK_RCO_APM_USER, 4, 1), + MUX(CLK_MOUT_DLL_USER, "mout_dll_user", mout_dll_user_p, + PLL_CON0_MUX_DLL_USER, 4, 1), + MUX(CLK_MOUT_CLKCMU_CHUB_BUS, "mout_clkcmu_chub_bus", + mout_clkcmu_chub_bus_p, CLK_CON_MUX_MUX_CLKCMU_CHUB_BUS, 0, 1), + MUX(CLK_MOUT_APM_BUS, "mout_apm_bus", mout_apm_bus_p, + CLK_CON_MUX_MUX_CLK_APM_BUS, 0, 2), + MUX(CLK_MOUT_APM_I3C, "mout_apm_i3c", mout_apm_i3c_p, + CLK_CON_MUX_MUX_CLK_APM_I3C, 0, 1), +}; + +static const struct samsung_div_clock apm_div_clks[] __initconst = { + DIV(CLK_DOUT_CLKCMU_CHUB_BUS, "dout_clkcmu_chub_bus", + "gout_clkcmu_chub_bus", + CLK_CON_DIV_CLKCMU_CHUB_BUS, 0, 3), + DIV(CLK_DOUT_APM_BUS, "dout_apm_bus", "mout_apm_bus", + CLK_CON_DIV_DIV_CLK_APM_BUS, 0, 3), + DIV(CLK_DOUT_APM_I3C, "dout_apm_i3c", "mout_apm_bus", + CLK_CON_DIV_DIV_CLK_APM_I3C, 0, 3), +}; + +static const struct samsung_gate_clock apm_gate_clks[] __initconst = { + GATE(CLK_GOUT_CLKCMU_CMGP_BUS, "gout_clkcmu_cmgp_bus", "dout_apm_bus", + CLK_CON_GAT_CLKCMU_CMGP_BUS, 21, 0, 0), + GATE(CLK_GOUT_CLKCMU_CHUB_BUS, "gout_clkcmu_chub_bus", + "mout_clkcmu_chub_bus", + CLK_CON_GAT_GATE_CLKCMU_CHUB_BUS, 21, 0, 0), + GATE(CLK_GOUT_RTC_PCLK, "gout_rtc_pclk", "dout_apm_bus", + CLK_CON_GAT_GOUT_APM_APBIF_RTC_PCLK, 21, 0, 0), + GATE(CLK_GOUT_TOP_RTC_PCLK, "gout_top_rtc_pclk", "dout_apm_bus", + CLK_CON_GAT_GOUT_APM_APBIF_TOP_RTC_PCLK, 21, 0, 0), + GATE(CLK_GOUT_I3C_PCLK, "gout_i3c_pclk", "dout_apm_bus", + CLK_CON_GAT_GOUT_APM_I3C_APM_PMIC_I_PCLK, 21, 0, 0), + GATE(CLK_GOUT_I3C_SCLK, "gout_i3c_sclk", "mout_apm_i3c", + CLK_CON_GAT_GOUT_APM_I3C_APM_PMIC_I_SCLK, 21, 0, 0), + GATE(CLK_GOUT_SPEEDY_PCLK, "gout_speedy_pclk", "dout_apm_bus", + CLK_CON_GAT_GOUT_APM_SPEEDY_APM_PCLK, 21, 0, 0), + /* TODO: Should be enabled in GPIO driver (or made CLK_IS_CRITICAL) */ + GATE(CLK_GOUT_GPIO_ALIVE_PCLK, "gout_gpio_alive_pclk", "dout_apm_bus", + CLK_CON_GAT_GOUT_APM_APBIF_GPIO_ALIVE_PCLK, 21, CLK_IGNORE_UNUSED, + 0), + GATE(CLK_GOUT_PMU_ALIVE_PCLK, "gout_pmu_alive_pclk", "dout_apm_bus", + CLK_CON_GAT_GOUT_APM_APBIF_PMU_ALIVE_PCLK, 21, 0, 0), + GATE(CLK_GOUT_SYSREG_APM_PCLK, "gout_sysreg_apm_pclk", "dout_apm_bus", + CLK_CON_GAT_GOUT_APM_SYSREG_APM_PCLK, 21, 0, 0), +}; + +static const struct samsung_cmu_info apm_cmu_info __initconst = { + .mux_clks = apm_mux_clks, + .nr_mux_clks = ARRAY_SIZE(apm_mux_clks), + .div_clks = apm_div_clks, + .nr_div_clks = ARRAY_SIZE(apm_div_clks), + .gate_clks = apm_gate_clks, + .nr_gate_clks = ARRAY_SIZE(apm_gate_clks), + .fixed_clks = apm_fixed_clks, + .nr_fixed_clks = ARRAY_SIZE(apm_fixed_clks), + .nr_clk_ids = APM_NR_CLK, + .clk_regs = apm_clk_regs, + .nr_clk_regs = ARRAY_SIZE(apm_clk_regs), + .clk_name = "dout_clkcmu_apm_bus", +}; + +/* ---- CMU_CMGP ------------------------------------------------------------ */ + +/* Register Offset definitions for CMU_CMGP (0x11c00000) */ +#define CLK_CON_MUX_CLK_CMGP_ADC 0x1000 +#define CLK_CON_MUX_MUX_CLK_CMGP_USI_CMGP0 0x1004 +#define CLK_CON_MUX_MUX_CLK_CMGP_USI_CMGP1 0x1008 +#define CLK_CON_DIV_DIV_CLK_CMGP_ADC 0x1800 +#define CLK_CON_DIV_DIV_CLK_CMGP_USI_CMGP0 0x1804 +#define CLK_CON_DIV_DIV_CLK_CMGP_USI_CMGP1 0x1808 +#define CLK_CON_GAT_GOUT_CMGP_ADC_PCLK_S0 0x200c +#define CLK_CON_GAT_GOUT_CMGP_ADC_PCLK_S1 0x2010 +#define CLK_CON_GAT_GOUT_CMGP_GPIO_PCLK 0x2018 +#define CLK_CON_GAT_GOUT_CMGP_SYSREG_CMGP_PCLK 0x2040 +#define CLK_CON_GAT_GOUT_CMGP_USI_CMGP0_IPCLK 0x2044 +#define CLK_CON_GAT_GOUT_CMGP_USI_CMGP0_PCLK 0x2048 +#define CLK_CON_GAT_GOUT_CMGP_USI_CMGP1_IPCLK 0x204c +#define CLK_CON_GAT_GOUT_CMGP_USI_CMGP1_PCLK 0x2050 + +static const unsigned long cmgp_clk_regs[] __initconst = { + CLK_CON_MUX_CLK_CMGP_ADC, + CLK_CON_MUX_MUX_CLK_CMGP_USI_CMGP0, + CLK_CON_MUX_MUX_CLK_CMGP_USI_CMGP1, + CLK_CON_DIV_DIV_CLK_CMGP_ADC, + CLK_CON_DIV_DIV_CLK_CMGP_USI_CMGP0, + CLK_CON_DIV_DIV_CLK_CMGP_USI_CMGP1, + CLK_CON_GAT_GOUT_CMGP_ADC_PCLK_S0, + CLK_CON_GAT_GOUT_CMGP_ADC_PCLK_S1, + CLK_CON_GAT_GOUT_CMGP_GPIO_PCLK, + CLK_CON_GAT_GOUT_CMGP_SYSREG_CMGP_PCLK, + CLK_CON_GAT_GOUT_CMGP_USI_CMGP0_IPCLK, + CLK_CON_GAT_GOUT_CMGP_USI_CMGP0_PCLK, + CLK_CON_GAT_GOUT_CMGP_USI_CMGP1_IPCLK, + CLK_CON_GAT_GOUT_CMGP_USI_CMGP1_PCLK, +}; + +/* List of parent clocks for Muxes in CMU_CMGP */ +PNAME(mout_cmgp_usi0_p) = { "clk_rco_cmgp", "gout_clkcmu_cmgp_bus" }; +PNAME(mout_cmgp_usi1_p) = { "clk_rco_cmgp", "gout_clkcmu_cmgp_bus" }; +PNAME(mout_cmgp_adc_p) = { "oscclk", "dout_cmgp_adc" }; + +static const struct samsung_fixed_rate_clock cmgp_fixed_clks[] __initconst = { + FRATE(CLK_RCO_CMGP, "clk_rco_cmgp", NULL, 0, 49152000), +}; + +static const struct samsung_mux_clock cmgp_mux_clks[] __initconst = { + MUX(CLK_MOUT_CMGP_ADC, "mout_cmgp_adc", mout_cmgp_adc_p, + CLK_CON_MUX_CLK_CMGP_ADC, 0, 1), + MUX(CLK_MOUT_CMGP_USI0, "mout_cmgp_usi0", mout_cmgp_usi0_p, + CLK_CON_MUX_MUX_CLK_CMGP_USI_CMGP0, 0, 1), + MUX(CLK_MOUT_CMGP_USI1, "mout_cmgp_usi1", mout_cmgp_usi1_p, + CLK_CON_MUX_MUX_CLK_CMGP_USI_CMGP1, 0, 1), +}; + +static const struct samsung_div_clock cmgp_div_clks[] __initconst = { + DIV(CLK_DOUT_CMGP_ADC, "dout_cmgp_adc", "gout_clkcmu_cmgp_bus", + CLK_CON_DIV_DIV_CLK_CMGP_ADC, 0, 4), + DIV(CLK_DOUT_CMGP_USI0, "dout_cmgp_usi0", "mout_cmgp_usi0", + CLK_CON_DIV_DIV_CLK_CMGP_USI_CMGP0, 0, 5), + DIV(CLK_DOUT_CMGP_USI1, "dout_cmgp_usi1", "mout_cmgp_usi1", + CLK_CON_DIV_DIV_CLK_CMGP_USI_CMGP1, 0, 5), +}; + +static const struct samsung_gate_clock cmgp_gate_clks[] __initconst = { + GATE(CLK_GOUT_CMGP_ADC_S0_PCLK, "gout_adc_s0_pclk", + "gout_clkcmu_cmgp_bus", + CLK_CON_GAT_GOUT_CMGP_ADC_PCLK_S0, 21, 0, 0), + GATE(CLK_GOUT_CMGP_ADC_S1_PCLK, "gout_adc_s1_pclk", + "gout_clkcmu_cmgp_bus", + CLK_CON_GAT_GOUT_CMGP_ADC_PCLK_S1, 21, 0, 0), + /* TODO: Should be enabled in GPIO driver (or made CLK_IS_CRITICAL) */ + GATE(CLK_GOUT_CMGP_GPIO_PCLK, "gout_gpio_cmgp_pclk", + "gout_clkcmu_cmgp_bus", + CLK_CON_GAT_GOUT_CMGP_GPIO_PCLK, 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_CMGP_USI0_IPCLK, "gout_cmgp_usi0_ipclk", "dout_cmgp_usi0", + CLK_CON_GAT_GOUT_CMGP_USI_CMGP0_IPCLK, 21, 0, 0), + GATE(CLK_GOUT_CMGP_USI0_PCLK, "gout_cmgp_usi0_pclk", + "gout_clkcmu_cmgp_bus", + CLK_CON_GAT_GOUT_CMGP_USI_CMGP0_PCLK, 21, 0, 0), + GATE(CLK_GOUT_CMGP_USI1_IPCLK, "gout_cmgp_usi1_ipclk", "dout_cmgp_usi1", + CLK_CON_GAT_GOUT_CMGP_USI_CMGP1_IPCLK, 21, 0, 0), + GATE(CLK_GOUT_CMGP_USI1_PCLK, "gout_cmgp_usi1_pclk", + "gout_clkcmu_cmgp_bus", + CLK_CON_GAT_GOUT_CMGP_USI_CMGP1_PCLK, 21, 0, 0), + GATE(CLK_GOUT_SYSREG_CMGP_PCLK, "gout_sysreg_cmgp_pclk", + "gout_clkcmu_cmgp_bus", + CLK_CON_GAT_GOUT_CMGP_SYSREG_CMGP_PCLK, 21, 0, 0), +}; + +static const struct samsung_cmu_info cmgp_cmu_info __initconst = { + .mux_clks = cmgp_mux_clks, + .nr_mux_clks = ARRAY_SIZE(cmgp_mux_clks), + .div_clks = cmgp_div_clks, + .nr_div_clks = ARRAY_SIZE(cmgp_div_clks), + .gate_clks = cmgp_gate_clks, + .nr_gate_clks = ARRAY_SIZE(cmgp_gate_clks), + .fixed_clks = cmgp_fixed_clks, + .nr_fixed_clks = ARRAY_SIZE(cmgp_fixed_clks), + .nr_clk_ids = CMGP_NR_CLK, + .clk_regs = cmgp_clk_regs, + .nr_clk_regs = ARRAY_SIZE(cmgp_clk_regs), + .clk_name = "gout_clkcmu_cmgp_bus", +}; + /* ---- CMU_HSI ------------------------------------------------------------- */ /* Register Offset definitions for CMU_HSI (0x13400000) */ @@ -413,8 +625,9 @@ static const struct samsung_gate_clock hsi_gate_clks[] __initconst = { CLK_CON_GAT_HSI_USB20DRD_TOP_I_REF_CLK_50, 21, 0, 0), GATE(CLK_GOUT_USB_PHY_REF_CLK, "gout_usb_phy_ref", "oscclk", CLK_CON_GAT_HSI_USB20DRD_TOP_I_PHY_REFCLK_26, 21, 0, 0), + /* TODO: Should be enabled in GPIO driver (or made CLK_IS_CRITICAL) */ GATE(CLK_GOUT_GPIO_HSI_PCLK, "gout_gpio_hsi_pclk", "mout_hsi_bus_user", - CLK_CON_GAT_GOUT_HSI_GPIO_HSI_PCLK, 21, 0, 0), + CLK_CON_GAT_GOUT_HSI_GPIO_HSI_PCLK, 21, CLK_IGNORE_UNUSED, 0), GATE(CLK_GOUT_MMC_CARD_ACLK, "gout_mmc_card_aclk", "mout_hsi_bus_user", CLK_CON_GAT_GOUT_HSI_MMC_CARD_I_ACLK, 21, 0, 0), GATE(CLK_GOUT_MMC_CARD_SDCLKIN, "gout_mmc_card_sdclkin", @@ -597,9 +810,10 @@ static const struct samsung_gate_clock peri_gate_clks[] __initconst = { CLK_CON_GAT_GOUT_PERI_WDT_0_PCLK, 21, 0, 0), GATE(CLK_GOUT_WDT1_PCLK, "gout_wdt1_pclk", "mout_peri_bus_user", CLK_CON_GAT_GOUT_PERI_WDT_1_PCLK, 21, 0, 0), + /* TODO: Should be enabled in GPIO driver (or made CLK_IS_CRITICAL) */ GATE(CLK_GOUT_GPIO_PERI_PCLK, "gout_gpio_peri_pclk", "mout_peri_bus_user", - CLK_CON_GAT_GOUT_PERI_GPIO_PERI_PCLK, 21, 0, 0), + CLK_CON_GAT_GOUT_PERI_GPIO_PERI_PCLK, 21, CLK_IGNORE_UNUSED, 0), }; static const struct samsung_cmu_info peri_cmu_info __initconst = { @@ -615,6 +829,15 @@ static const struct samsung_cmu_info peri_cmu_info __initconst = { .clk_name = "dout_peri_bus", }; +static void __init exynos850_cmu_peri_init(struct device_node *np) +{ + exynos_arm64_register_cmu(NULL, np, &peri_cmu_info); +} + +/* Register CMU_PERI early, as it's needed for MCT timer */ +CLK_OF_DECLARE(exynos850_cmu_peri, "samsung,exynos850-cmu-peri", + exynos850_cmu_peri_init); + /* ---- CMU_CORE ------------------------------------------------------------ */ /* Register Offset definitions for CMU_CORE (0x12000000) */ @@ -626,10 +849,12 @@ static const struct samsung_cmu_info peri_cmu_info __initconst = { #define CLK_CON_DIV_DIV_CLK_CORE_BUSP 0x1800 #define CLK_CON_GAT_GOUT_CORE_CCI_550_ACLK 0x2038 #define CLK_CON_GAT_GOUT_CORE_GIC_CLK 0x2040 +#define CLK_CON_GAT_GOUT_CORE_GPIO_CORE_PCLK 0x2044 #define CLK_CON_GAT_GOUT_CORE_MMC_EMBD_I_ACLK 0x20e8 #define CLK_CON_GAT_GOUT_CORE_MMC_EMBD_SDCLKIN 0x20ec #define CLK_CON_GAT_GOUT_CORE_SSS_I_ACLK 0x2128 #define CLK_CON_GAT_GOUT_CORE_SSS_I_PCLK 0x212c +#define CLK_CON_GAT_GOUT_CORE_SYSREG_CORE_PCLK 0x2130 static const unsigned long core_clk_regs[] __initconst = { PLL_CON0_MUX_CLKCMU_CORE_BUS_USER, @@ -640,10 +865,12 @@ static const unsigned long core_clk_regs[] __initconst = { CLK_CON_DIV_DIV_CLK_CORE_BUSP, CLK_CON_GAT_GOUT_CORE_CCI_550_ACLK, CLK_CON_GAT_GOUT_CORE_GIC_CLK, + CLK_CON_GAT_GOUT_CORE_GPIO_CORE_PCLK, CLK_CON_GAT_GOUT_CORE_MMC_EMBD_I_ACLK, CLK_CON_GAT_GOUT_CORE_MMC_EMBD_SDCLKIN, CLK_CON_GAT_GOUT_CORE_SSS_I_ACLK, CLK_CON_GAT_GOUT_CORE_SSS_I_PCLK, + CLK_CON_GAT_GOUT_CORE_SYSREG_CORE_PCLK, }; /* List of parent clocks for Muxes in CMU_CORE */ @@ -673,10 +900,12 @@ static const struct samsung_div_clock core_div_clks[] __initconst = { }; static const struct samsung_gate_clock core_gate_clks[] __initconst = { + /* CCI (interconnect) clock must be always running */ GATE(CLK_GOUT_CCI_ACLK, "gout_cci_aclk", "mout_core_cci_user", - CLK_CON_GAT_GOUT_CORE_CCI_550_ACLK, 21, 0, 0), + CLK_CON_GAT_GOUT_CORE_CCI_550_ACLK, 21, CLK_IS_CRITICAL, 0), + /* GIC (interrupt controller) clock must be always running */ GATE(CLK_GOUT_GIC_CLK, "gout_gic_clk", "mout_core_gic", - CLK_CON_GAT_GOUT_CORE_GIC_CLK, 21, 0, 0), + CLK_CON_GAT_GOUT_CORE_GIC_CLK, 21, CLK_IS_CRITICAL, 0), GATE(CLK_GOUT_MMC_EMBD_ACLK, "gout_mmc_embd_aclk", "dout_core_busp", CLK_CON_GAT_GOUT_CORE_MMC_EMBD_I_ACLK, 21, 0, 0), GATE(CLK_GOUT_MMC_EMBD_SDCLKIN, "gout_mmc_embd_sdclkin", @@ -686,6 +915,12 @@ static const struct samsung_gate_clock core_gate_clks[] __initconst = { CLK_CON_GAT_GOUT_CORE_SSS_I_ACLK, 21, 0, 0), GATE(CLK_GOUT_SSS_PCLK, "gout_sss_pclk", "dout_core_busp", CLK_CON_GAT_GOUT_CORE_SSS_I_PCLK, 21, 0, 0), + /* TODO: Should be enabled in GPIO driver (or made CLK_IS_CRITICAL) */ + GATE(CLK_GOUT_GPIO_CORE_PCLK, "gout_gpio_core_pclk", "dout_core_busp", + CLK_CON_GAT_GOUT_CORE_GPIO_CORE_PCLK, 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_SYSREG_CORE_PCLK, "gout_sysreg_core_pclk", + "dout_core_busp", + CLK_CON_GAT_GOUT_CORE_SYSREG_CORE_PCLK, 21, 0, 0), }; static const struct samsung_cmu_info core_cmu_info __initconst = { @@ -742,8 +977,10 @@ static const struct samsung_div_clock dpu_div_clks[] __initconst = { }; static const struct samsung_gate_clock dpu_gate_clks[] __initconst = { + /* TODO: Should be enabled in DSIM driver */ GATE(CLK_GOUT_DPU_CMU_DPU_PCLK, "gout_dpu_cmu_dpu_pclk", - "dout_dpu_busp", CLK_CON_GAT_CLK_DPU_CMU_DPU_PCLK, 21, 0, 0), + "dout_dpu_busp", + CLK_CON_GAT_CLK_DPU_CMU_DPU_PCLK, 21, CLK_IGNORE_UNUSED, 0), GATE(CLK_GOUT_DPU_DECON0_ACLK, "gout_dpu_decon0_aclk", "mout_dpu_user", CLK_CON_GAT_GOUT_DPU_ACLK_DECON0, 21, 0, 0), GATE(CLK_GOUT_DPU_DMA_ACLK, "gout_dpu_dma_aclk", "mout_dpu_user", @@ -779,36 +1016,23 @@ static int __init exynos850_cmu_probe(struct platform_device *pdev) { const struct samsung_cmu_info *info; struct device *dev = &pdev->dev; - struct device_node *np = dev->of_node; info = of_device_get_match_data(dev); - exynos850_init_clocks(np, info->clk_regs, info->nr_clk_regs); - samsung_cmu_register_one(np, info); - - /* Keep bus clock running, so it's possible to access CMU registers */ - if (info->clk_name) { - struct clk *bus_clk; - - bus_clk = clk_get(dev, info->clk_name); - if (IS_ERR(bus_clk)) { - pr_err("%s: could not find bus clock %s; err = %ld\n", - __func__, info->clk_name, PTR_ERR(bus_clk)); - } else { - clk_prepare_enable(bus_clk); - } - } + exynos_arm64_register_cmu(dev, dev->of_node, info); return 0; } -/* CMUs which belong to Power Domains and need runtime PM to be implemented */ static const struct of_device_id exynos850_cmu_of_match[] = { { + .compatible = "samsung,exynos850-cmu-apm", + .data = &apm_cmu_info, + }, { + .compatible = "samsung,exynos850-cmu-cmgp", + .data = &cmgp_cmu_info, + }, { .compatible = "samsung,exynos850-cmu-hsi", .data = &hsi_cmu_info, - }, { - .compatible = "samsung,exynos850-cmu-peri", - .data = &peri_cmu_info, }, { .compatible = "samsung,exynos850-cmu-core", .data = &core_cmu_info, diff --git a/drivers/clk/samsung/clk-pll.c b/drivers/clk/samsung/clk-pll.c index 83d1b03647db..70cdc87f714e 100644 --- a/drivers/clk/samsung/clk-pll.c +++ b/drivers/clk/samsung/clk-pll.c @@ -1476,6 +1476,7 @@ static void __init _samsung_clk_register_pll(struct samsung_clk_provider *ctx, else init.ops = &samsung_pll35xx_clk_ops; break; + case pll_1417x: case pll_0822x: pll->enable_offs = PLL0822X_ENABLE_SHIFT; pll->lock_offs = PLL0822X_LOCK_STAT_SHIFT; diff --git a/drivers/clk/samsung/clk-pll.h b/drivers/clk/samsung/clk-pll.h index a739f2b7ae80..c83a20195f6d 100644 --- a/drivers/clk/samsung/clk-pll.h +++ b/drivers/clk/samsung/clk-pll.h @@ -32,6 +32,7 @@ enum samsung_pll_type { pll_2550xx, pll_2650x, pll_2650xx, + pll_1417x, pll_1450x, pll_1451x, pll_1452x, diff --git a/drivers/clk/samsung/clk-s3c2410.c b/drivers/clk/samsung/clk-s3c2410.c index 5831d0606077..3d152a46169b 100644 --- a/drivers/clk/samsung/clk-s3c2410.c +++ b/drivers/clk/samsung/clk-s3c2410.c @@ -323,6 +323,7 @@ void __init s3c2410_common_clk_init(struct device_node *np, unsigned long xti_f, void __iomem *base) { struct samsung_clk_provider *ctx; + struct clk_hw **hws; reg_base = base; if (np) { @@ -332,13 +333,14 @@ void __init s3c2410_common_clk_init(struct device_node *np, unsigned long xti_f, } ctx = samsung_clk_init(np, reg_base, NR_CLKS); + hws = ctx->clk_data.hws; /* Register external clocks only in non-dt cases */ if (!np) s3c2410_common_clk_register_fixed_ext(ctx, xti_f); if (current_soc == S3C2410) { - if (_get_rate("xti") == 12 * MHZ) { + if (clk_hw_get_rate(hws[XTI]) == 12 * MHZ) { s3c2410_plls[mpll].rate_table = pll_s3c2410_12mhz_tbl; s3c2410_plls[upll].rate_table = pll_s3c2410_12mhz_tbl; } @@ -348,7 +350,7 @@ void __init s3c2410_common_clk_init(struct device_node *np, unsigned long xti_f, ARRAY_SIZE(s3c2410_plls), reg_base); } else { /* S3C2440, S3C2442 */ - if (_get_rate("xti") == 12 * MHZ) { + if (clk_hw_get_rate(hws[XTI]) == 12 * MHZ) { /* * plls follow different calculation schemes, with the * upll following the same scheme as the s3c2410 plls diff --git a/drivers/clk/samsung/clk-s3c64xx.c b/drivers/clk/samsung/clk-s3c64xx.c index 56f95b63f71f..d6b432a26d63 100644 --- a/drivers/clk/samsung/clk-s3c64xx.c +++ b/drivers/clk/samsung/clk-s3c64xx.c @@ -394,6 +394,7 @@ void __init s3c64xx_clk_init(struct device_node *np, unsigned long xtal_f, void __iomem *base) { struct samsung_clk_provider *ctx; + struct clk_hw **hws; reg_base = base; is_s3c6400 = s3c6400; @@ -405,6 +406,7 @@ void __init s3c64xx_clk_init(struct device_node *np, unsigned long xtal_f, } ctx = samsung_clk_init(np, reg_base, NR_CLKS); + hws = ctx->clk_data.hws; /* Register external clocks. */ if (!np) @@ -459,8 +461,10 @@ void __init s3c64xx_clk_init(struct device_node *np, unsigned long xtal_f, pr_info("%s clocks: apll = %lu, mpll = %lu\n" "\tepll = %lu, arm_clk = %lu\n", is_s3c6400 ? "S3C6400" : "S3C6410", - _get_rate("fout_apll"), _get_rate("fout_mpll"), - _get_rate("fout_epll"), _get_rate("armclk")); + clk_hw_get_rate(hws[MOUT_APLL]), + clk_hw_get_rate(hws[MOUT_MPLL]), + clk_hw_get_rate(hws[MOUT_EPLL]), + clk_hw_get_rate(hws[ARMCLK])); } static void __init s3c6400_clk_init(struct device_node *np) diff --git a/drivers/clk/samsung/clk-s5pv210.c b/drivers/clk/samsung/clk-s5pv210.c index e7b68ffe36de..4425186bdcab 100644 --- a/drivers/clk/samsung/clk-s5pv210.c +++ b/drivers/clk/samsung/clk-s5pv210.c @@ -741,8 +741,10 @@ static void __init __s5pv210_clk_init(struct device_node *np, bool is_s5p6442) { struct samsung_clk_provider *ctx; + struct clk_hw **hws; ctx = samsung_clk_init(np, reg_base, NR_CLKS); + hws = ctx->clk_data.hws; samsung_clk_register_mux(ctx, early_mux_clks, ARRAY_SIZE(early_mux_clks)); @@ -789,8 +791,10 @@ static void __init __s5pv210_clk_init(struct device_node *np, pr_info("%s clocks: mout_apll = %ld, mout_mpll = %ld\n" "\tmout_epll = %ld, mout_vpll = %ld\n", is_s5p6442 ? "S5P6442" : "S5PV210", - _get_rate("mout_apll"), _get_rate("mout_mpll"), - _get_rate("mout_epll"), _get_rate("mout_vpll")); + clk_hw_get_rate(hws[MOUT_APLL]), + clk_hw_get_rate(hws[MOUT_MPLL]), + clk_hw_get_rate(hws[MOUT_EPLL]), + clk_hw_get_rate(hws[MOUT_VPLL])); } static void __init s5pv210_clk_dt_init(struct device_node *np) diff --git a/drivers/clk/samsung/clk.c b/drivers/clk/samsung/clk.c index 336243c6f120..bca4731b14ea 100644 --- a/drivers/clk/samsung/clk.c +++ b/drivers/clk/samsung/clk.c @@ -268,20 +268,6 @@ void __init samsung_clk_of_register_fixed_ext(struct samsung_clk_provider *ctx, samsung_clk_register_fixed_rate(ctx, fixed_rate_clk, nr_fixed_rate_clk); } -/* utility function to get the rate of a specified clock */ -unsigned long _get_rate(const char *clk_name) -{ - struct clk *clk; - - clk = __clk_lookup(clk_name); - if (!clk) { - pr_err("%s: could not find clock %s\n", __func__, clk_name); - return 0; - } - - return clk_get_rate(clk); -} - #ifdef CONFIG_PM_SLEEP static int samsung_clk_suspend(void) { diff --git a/drivers/clk/samsung/clk.h b/drivers/clk/samsung/clk.h index 26499e97275b..b46e83a2581f 100644 --- a/drivers/clk/samsung/clk.h +++ b/drivers/clk/samsung/clk.h @@ -337,54 +337,52 @@ struct samsung_cmu_info { const char *clk_name; }; -extern struct samsung_clk_provider *__init samsung_clk_init( +struct samsung_clk_provider * samsung_clk_init( struct device_node *np, void __iomem *base, unsigned long nr_clks); -extern void __init samsung_clk_of_add_provider(struct device_node *np, +void samsung_clk_of_add_provider(struct device_node *np, struct samsung_clk_provider *ctx); -extern void __init samsung_clk_of_register_fixed_ext( +void samsung_clk_of_register_fixed_ext( struct samsung_clk_provider *ctx, struct samsung_fixed_rate_clock *fixed_rate_clk, unsigned int nr_fixed_rate_clk, const struct of_device_id *clk_matches); -extern void samsung_clk_add_lookup(struct samsung_clk_provider *ctx, +void samsung_clk_add_lookup(struct samsung_clk_provider *ctx, struct clk_hw *clk_hw, unsigned int id); -extern void __init samsung_clk_register_alias(struct samsung_clk_provider *ctx, +void samsung_clk_register_alias(struct samsung_clk_provider *ctx, const struct samsung_clock_alias *list, unsigned int nr_clk); -extern void __init samsung_clk_register_fixed_rate( +void samsung_clk_register_fixed_rate( struct samsung_clk_provider *ctx, const struct samsung_fixed_rate_clock *clk_list, unsigned int nr_clk); -extern void __init samsung_clk_register_fixed_factor( +void samsung_clk_register_fixed_factor( struct samsung_clk_provider *ctx, const struct samsung_fixed_factor_clock *list, unsigned int nr_clk); -extern void __init samsung_clk_register_mux(struct samsung_clk_provider *ctx, +void samsung_clk_register_mux(struct samsung_clk_provider *ctx, const struct samsung_mux_clock *clk_list, unsigned int nr_clk); -extern void __init samsung_clk_register_div(struct samsung_clk_provider *ctx, +void samsung_clk_register_div(struct samsung_clk_provider *ctx, const struct samsung_div_clock *clk_list, unsigned int nr_clk); -extern void __init samsung_clk_register_gate(struct samsung_clk_provider *ctx, +void samsung_clk_register_gate(struct samsung_clk_provider *ctx, const struct samsung_gate_clock *clk_list, unsigned int nr_clk); -extern void __init samsung_clk_register_pll(struct samsung_clk_provider *ctx, +void samsung_clk_register_pll(struct samsung_clk_provider *ctx, const struct samsung_pll_clock *pll_list, unsigned int nr_clk, void __iomem *base); -extern void samsung_clk_register_cpu(struct samsung_clk_provider *ctx, +void samsung_clk_register_cpu(struct samsung_clk_provider *ctx, const struct samsung_cpu_clock *list, unsigned int nr_clk); -extern struct samsung_clk_provider __init *samsung_cmu_register_one( +struct samsung_clk_provider *samsung_cmu_register_one( struct device_node *, const struct samsung_cmu_info *); -extern unsigned long _get_rate(const char *clk_name); - #ifdef CONFIG_PM_SLEEP -extern void samsung_clk_extended_sleep_init(void __iomem *reg_base, +void samsung_clk_extended_sleep_init(void __iomem *reg_base, const unsigned long *rdump, unsigned long nr_rdump, const struct samsung_clk_reg_dump *rsuspend, @@ -399,13 +397,13 @@ static inline void samsung_clk_extended_sleep_init(void __iomem *reg_base, #define samsung_clk_sleep_init(reg_base, rdump, nr_rdump) \ samsung_clk_extended_sleep_init(reg_base, rdump, nr_rdump, NULL, 0) -extern void samsung_clk_save(void __iomem *base, +void samsung_clk_save(void __iomem *base, struct samsung_clk_reg_dump *rd, unsigned int num_regs); -extern void samsung_clk_restore(void __iomem *base, +void samsung_clk_restore(void __iomem *base, const struct samsung_clk_reg_dump *rd, unsigned int num_regs); -extern struct samsung_clk_reg_dump *samsung_clk_alloc_reg_dump( +struct samsung_clk_reg_dump *samsung_clk_alloc_reg_dump( const unsigned long *rdump, unsigned long nr_rdump); diff --git a/drivers/clk/socfpga/clk-agilex.c b/drivers/clk/socfpga/clk-agilex.c index bf8cd928c228..74d21bd82710 100644 --- a/drivers/clk/socfpga/clk-agilex.c +++ b/drivers/clk/socfpga/clk-agilex.c @@ -500,12 +500,10 @@ static int n5x_clkmgr_init(struct platform_device *pdev) struct device_node *np = pdev->dev.of_node; struct device *dev = &pdev->dev; struct stratix10_clock_data *clk_data; - struct resource *res; void __iomem *base; int i, num_clks; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - base = devm_ioremap_resource(dev, res); + base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(base)) return PTR_ERR(base); diff --git a/drivers/clk/socfpga/clk-gate.c b/drivers/clk/socfpga/clk-gate.c index 1ec9678d8cd3..53d6e3ec4309 100644 --- a/drivers/clk/socfpga/clk-gate.c +++ b/drivers/clk/socfpga/clk-gate.c @@ -34,7 +34,7 @@ static u8 socfpga_clk_get_parent(struct clk_hw *hwclk) if (streq(name, SOCFPGA_L4_MP_CLK)) { l4_src = readl(clk_mgr_base_addr + CLKMGR_L4SRC); - return l4_src &= 0x1; + return l4_src & 0x1; } if (streq(name, SOCFPGA_L4_SP_CLK)) { l4_src = readl(clk_mgr_base_addr + CLKMGR_L4SRC); @@ -43,7 +43,7 @@ static u8 socfpga_clk_get_parent(struct clk_hw *hwclk) perpll_src = readl(clk_mgr_base_addr + CLKMGR_PERPLL_SRC); if (streq(name, SOCFPGA_MMC_CLK)) - return perpll_src &= 0x3; + return perpll_src & 0x3; if (streq(name, SOCFPGA_NAND_CLK) || streq(name, SOCFPGA_NAND_X_CLK)) return (perpll_src >> 2) & 3; diff --git a/drivers/clk/socfpga/clk-pll-s10.c b/drivers/clk/socfpga/clk-pll-s10.c index 70076a80149d..e444e4a0ee53 100644 --- a/drivers/clk/socfpga/clk-pll-s10.c +++ b/drivers/clk/socfpga/clk-pll-s10.c @@ -113,7 +113,7 @@ static unsigned long clk_boot_clk_recalc_rate(struct clk_hw *hwclk, SWCTRLBTCLKSEL_MASK) >> SWCTRLBTCLKSEL_SHIFT); div += 1; - return parent_rate /= div; + return parent_rate / div; } diff --git a/drivers/clk/socfpga/clk-s10.c b/drivers/clk/socfpga/clk-s10.c index b532d51faaee..4e508a844b3d 100644 --- a/drivers/clk/socfpga/clk-s10.c +++ b/drivers/clk/socfpga/clk-s10.c @@ -388,12 +388,10 @@ static int s10_clkmgr_init(struct platform_device *pdev) struct device_node *np = pdev->dev.of_node; struct device *dev = &pdev->dev; struct stratix10_clock_data *clk_data; - struct resource *res; void __iomem *base; int i, num_clks; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - base = devm_ioremap_resource(dev, res); + base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(base)) { pr_err("%s: failed to map clock registers\n", __func__); return PTR_ERR(base); diff --git a/drivers/clk/st/clkgen-fsyn.c b/drivers/clk/st/clkgen-fsyn.c index 164285d6be97..582a22c04919 100644 --- a/drivers/clk/st/clkgen-fsyn.c +++ b/drivers/clk/st/clkgen-fsyn.c @@ -988,9 +988,18 @@ static void __init st_of_quadfs_setup(struct device_node *np, void __iomem *reg; spinlock_t *lock; + /* + * First check for reg property within the node to keep backward + * compatibility, then if reg doesn't exist look at the parent node + */ reg = of_iomap(np, 0); - if (!reg) - return; + if (!reg) { + reg = of_iomap(of_get_parent(np), 0); + if (!reg) { + pr_err("%s: Failed to get base address\n", __func__); + return; + } + } clk_parent_name = of_clk_get_parent_name(np, 0); if (!clk_parent_name) diff --git a/drivers/clk/st/clkgen-mux.c b/drivers/clk/st/clkgen-mux.c index ce583ded968a..ee39af7a0b72 100644 --- a/drivers/clk/st/clkgen-mux.c +++ b/drivers/clk/st/clkgen-mux.c @@ -57,10 +57,17 @@ static void __init st_of_clkgen_mux_setup(struct device_node *np, const char **parents; int num_parents = 0; + /* + * First check for reg property within the node to keep backward + * compatibility, then if reg doesn't exist look at the parent node + */ reg = of_iomap(np, 0); if (!reg) { - pr_err("%s: Failed to get base address\n", __func__); - return; + reg = of_iomap(of_get_parent(np), 0); + if (!reg) { + pr_err("%s: Failed to get base address\n", __func__); + return; + } } parents = clkgen_mux_get_parents(np, &num_parents); diff --git a/drivers/clk/sunxi-ng/Kconfig b/drivers/clk/sunxi-ng/Kconfig index e76e1676f0f0..68a94e5af8ed 100644 --- a/drivers/clk/sunxi-ng/Kconfig +++ b/drivers/clk/sunxi-ng/Kconfig @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only config SUNXI_CCU - bool "Clock support for Allwinner SoCs" + tristate "Clock support for Allwinner SoCs" depends on ARCH_SUNXI || COMPILE_TEST select RESET_CONTROLLER default ARCH_SUNXI @@ -8,42 +8,52 @@ config SUNXI_CCU if SUNXI_CCU config SUNIV_F1C100S_CCU - bool "Support for the Allwinner newer F1C100s CCU" + tristate "Support for the Allwinner newer F1C100s CCU" default MACH_SUNIV depends on MACH_SUNIV || COMPILE_TEST +config SUN20I_D1_CCU + tristate "Support for the Allwinner D1 CCU" + default RISCV && ARCH_SUNXI + depends on (RISCV && ARCH_SUNXI) || COMPILE_TEST + +config SUN20I_D1_R_CCU + tristate "Support for the Allwinner D1 PRCM CCU" + default RISCV && ARCH_SUNXI + depends on (RISCV && ARCH_SUNXI) || COMPILE_TEST + config SUN50I_A64_CCU - bool "Support for the Allwinner A64 CCU" + tristate "Support for the Allwinner A64 CCU" default ARM64 && ARCH_SUNXI depends on (ARM64 && ARCH_SUNXI) || COMPILE_TEST config SUN50I_A100_CCU - bool "Support for the Allwinner A100 CCU" + tristate "Support for the Allwinner A100 CCU" default ARM64 && ARCH_SUNXI depends on (ARM64 && ARCH_SUNXI) || COMPILE_TEST config SUN50I_A100_R_CCU - bool "Support for the Allwinner A100 PRCM CCU" + tristate "Support for the Allwinner A100 PRCM CCU" default ARM64 && ARCH_SUNXI depends on (ARM64 && ARCH_SUNXI) || COMPILE_TEST config SUN50I_H6_CCU - bool "Support for the Allwinner H6 CCU" + tristate "Support for the Allwinner H6 CCU" default ARM64 && ARCH_SUNXI depends on (ARM64 && ARCH_SUNXI) || COMPILE_TEST config SUN50I_H616_CCU - bool "Support for the Allwinner H616 CCU" + tristate "Support for the Allwinner H616 CCU" default ARM64 && ARCH_SUNXI depends on (ARM64 && ARCH_SUNXI) || COMPILE_TEST config SUN50I_H6_R_CCU - bool "Support for the Allwinner H6 and H616 PRCM CCU" + tristate "Support for the Allwinner H6 and H616 PRCM CCU" default ARM64 && ARCH_SUNXI depends on (ARM64 && ARCH_SUNXI) || COMPILE_TEST config SUN4I_A10_CCU - bool "Support for the Allwinner A10/A20 CCU" + tristate "Support for the Allwinner A10/A20 CCU" default MACH_SUN4I default MACH_SUN7I depends on MACH_SUN4I || MACH_SUN7I || COMPILE_TEST @@ -52,53 +62,54 @@ config SUN5I_CCU bool "Support for the Allwinner sun5i family CCM" default MACH_SUN5I depends on MACH_SUN5I || COMPILE_TEST + depends on SUNXI_CCU=y config SUN6I_A31_CCU - bool "Support for the Allwinner A31/A31s CCU" + tristate "Support for the Allwinner A31/A31s CCU" default MACH_SUN6I depends on MACH_SUN6I || COMPILE_TEST config SUN8I_A23_CCU - bool "Support for the Allwinner A23 CCU" + tristate "Support for the Allwinner A23 CCU" default MACH_SUN8I depends on MACH_SUN8I || COMPILE_TEST config SUN8I_A33_CCU - bool "Support for the Allwinner A33 CCU" + tristate "Support for the Allwinner A33 CCU" default MACH_SUN8I depends on MACH_SUN8I || COMPILE_TEST config SUN8I_A83T_CCU - bool "Support for the Allwinner A83T CCU" + tristate "Support for the Allwinner A83T CCU" default MACH_SUN8I depends on MACH_SUN8I || COMPILE_TEST config SUN8I_H3_CCU - bool "Support for the Allwinner H3 CCU" + tristate "Support for the Allwinner H3 CCU" default MACH_SUN8I || (ARM64 && ARCH_SUNXI) depends on MACH_SUN8I || (ARM64 && ARCH_SUNXI) || COMPILE_TEST config SUN8I_V3S_CCU - bool "Support for the Allwinner V3s CCU" + tristate "Support for the Allwinner V3s CCU" default MACH_SUN8I depends on MACH_SUN8I || COMPILE_TEST config SUN8I_DE2_CCU - bool "Support for the Allwinner SoCs DE2 CCU" + tristate "Support for the Allwinner SoCs DE2 CCU" default MACH_SUN8I || (ARM64 && ARCH_SUNXI) config SUN8I_R40_CCU - bool "Support for the Allwinner R40 CCU" + tristate "Support for the Allwinner R40 CCU" default MACH_SUN8I depends on MACH_SUN8I || COMPILE_TEST config SUN9I_A80_CCU - bool "Support for the Allwinner A80 CCU" + tristate "Support for the Allwinner A80 CCU" default MACH_SUN9I depends on MACH_SUN9I || COMPILE_TEST config SUN8I_R_CCU - bool "Support for Allwinner SoCs' PRCM CCUs" + tristate "Support for Allwinner SoCs' PRCM CCUs" default MACH_SUN8I || (ARCH_SUNXI && ARM64) endif diff --git a/drivers/clk/sunxi-ng/Makefile b/drivers/clk/sunxi-ng/Makefile index 96c324306d97..ec931cb7aa14 100644 --- a/drivers/clk/sunxi-ng/Makefile +++ b/drivers/clk/sunxi-ng/Makefile @@ -1,44 +1,73 @@ # SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_SUNXI_CCU) += sunxi-ccu.o + # Common objects -obj-y += ccu_common.o -obj-y += ccu_mmc_timing.o -obj-y += ccu_reset.o +sunxi-ccu-y += ccu_common.o +sunxi-ccu-y += ccu_mmc_timing.o +sunxi-ccu-y += ccu_reset.o # Base clock types -obj-y += ccu_div.o -obj-y += ccu_frac.o -obj-y += ccu_gate.o -obj-y += ccu_mux.o -obj-y += ccu_mult.o -obj-y += ccu_phase.o -obj-y += ccu_sdm.o +sunxi-ccu-y += ccu_div.o +sunxi-ccu-y += ccu_frac.o +sunxi-ccu-y += ccu_gate.o +sunxi-ccu-y += ccu_mux.o +sunxi-ccu-y += ccu_mult.o +sunxi-ccu-y += ccu_phase.o +sunxi-ccu-y += ccu_sdm.o # Multi-factor clocks -obj-y += ccu_nk.o -obj-y += ccu_nkm.o -obj-y += ccu_nkmp.o -obj-y += ccu_nm.o -obj-y += ccu_mp.o +sunxi-ccu-y += ccu_nk.o +sunxi-ccu-y += ccu_nkm.o +sunxi-ccu-y += ccu_nkmp.o +sunxi-ccu-y += ccu_nm.o +sunxi-ccu-y += ccu_mp.o # SoC support -obj-$(CONFIG_SUNIV_F1C100S_CCU) += ccu-suniv-f1c100s.o -obj-$(CONFIG_SUN50I_A64_CCU) += ccu-sun50i-a64.o -obj-$(CONFIG_SUN50I_A100_CCU) += ccu-sun50i-a100.o -obj-$(CONFIG_SUN50I_A100_R_CCU) += ccu-sun50i-a100-r.o -obj-$(CONFIG_SUN50I_H6_CCU) += ccu-sun50i-h6.o -obj-$(CONFIG_SUN50I_H616_CCU) += ccu-sun50i-h616.o -obj-$(CONFIG_SUN50I_H6_R_CCU) += ccu-sun50i-h6-r.o -obj-$(CONFIG_SUN4I_A10_CCU) += ccu-sun4i-a10.o -obj-$(CONFIG_SUN5I_CCU) += ccu-sun5i.o -obj-$(CONFIG_SUN6I_A31_CCU) += ccu-sun6i-a31.o -obj-$(CONFIG_SUN8I_A23_CCU) += ccu-sun8i-a23.o -obj-$(CONFIG_SUN8I_A33_CCU) += ccu-sun8i-a33.o -obj-$(CONFIG_SUN8I_A83T_CCU) += ccu-sun8i-a83t.o -obj-$(CONFIG_SUN8I_H3_CCU) += ccu-sun8i-h3.o -obj-$(CONFIG_SUN8I_V3S_CCU) += ccu-sun8i-v3s.o -obj-$(CONFIG_SUN8I_DE2_CCU) += ccu-sun8i-de2.o -obj-$(CONFIG_SUN8I_R_CCU) += ccu-sun8i-r.o -obj-$(CONFIG_SUN8I_R40_CCU) += ccu-sun8i-r40.o -obj-$(CONFIG_SUN9I_A80_CCU) += ccu-sun9i-a80.o -obj-$(CONFIG_SUN9I_A80_CCU) += ccu-sun9i-a80-de.o -obj-$(CONFIG_SUN9I_A80_CCU) += ccu-sun9i-a80-usb.o +obj-$(CONFIG_SUNIV_F1C100S_CCU) += suniv-f1c100s-ccu.o +obj-$(CONFIG_SUN20I_D1_CCU) += sun20i-d1-ccu.o +obj-$(CONFIG_SUN20I_D1_R_CCU) += sun20i-d1-r-ccu.o +obj-$(CONFIG_SUN50I_A64_CCU) += sun50i-a64-ccu.o +obj-$(CONFIG_SUN50I_A100_CCU) += sun50i-a100-ccu.o +obj-$(CONFIG_SUN50I_A100_R_CCU) += sun50i-a100-r-ccu.o +obj-$(CONFIG_SUN50I_H6_CCU) += sun50i-h6-ccu.o +obj-$(CONFIG_SUN50I_H6_R_CCU) += sun50i-h6-r-ccu.o +obj-$(CONFIG_SUN50I_H616_CCU) += sun50i-h616-ccu.o +obj-$(CONFIG_SUN4I_A10_CCU) += sun4i-a10-ccu.o +obj-$(CONFIG_SUN5I_CCU) += sun5i-ccu.o +obj-$(CONFIG_SUN6I_A31_CCU) += sun6i-a31-ccu.o +obj-$(CONFIG_SUN8I_A23_CCU) += sun8i-a23-ccu.o +obj-$(CONFIG_SUN8I_A33_CCU) += sun8i-a33-ccu.o +obj-$(CONFIG_SUN8I_A83T_CCU) += sun8i-a83t-ccu.o +obj-$(CONFIG_SUN8I_H3_CCU) += sun8i-h3-ccu.o +obj-$(CONFIG_SUN8I_R40_CCU) += sun8i-r40-ccu.o +obj-$(CONFIG_SUN8I_V3S_CCU) += sun8i-v3s-ccu.o +obj-$(CONFIG_SUN8I_DE2_CCU) += sun8i-de2-ccu.o +obj-$(CONFIG_SUN8I_R_CCU) += sun8i-r-ccu.o +obj-$(CONFIG_SUN9I_A80_CCU) += sun9i-a80-ccu.o +obj-$(CONFIG_SUN9I_A80_CCU) += sun9i-a80-de-ccu.o +obj-$(CONFIG_SUN9I_A80_CCU) += sun9i-a80-usb-ccu.o + +suniv-f1c100s-ccu-y += ccu-suniv-f1c100s.o +sun20i-d1-ccu-y += ccu-sun20i-d1.o +sun20i-d1-r-ccu-y += ccu-sun20i-d1-r.o +sun50i-a64-ccu-y += ccu-sun50i-a64.o +sun50i-a100-ccu-y += ccu-sun50i-a100.o +sun50i-a100-r-ccu-y += ccu-sun50i-a100-r.o +sun50i-h6-ccu-y += ccu-sun50i-h6.o +sun50i-h6-r-ccu-y += ccu-sun50i-h6-r.o +sun50i-h616-ccu-y += ccu-sun50i-h616.o +sun4i-a10-ccu-y += ccu-sun4i-a10.o +sun5i-ccu-y += ccu-sun5i.o +sun6i-a31-ccu-y += ccu-sun6i-a31.o +sun8i-a23-ccu-y += ccu-sun8i-a23.o +sun8i-a33-ccu-y += ccu-sun8i-a33.o +sun8i-a83t-ccu-y += ccu-sun8i-a83t.o +sun8i-h3-ccu-y += ccu-sun8i-h3.o +sun8i-r40-ccu-y += ccu-sun8i-r40.o +sun8i-v3s-ccu-y += ccu-sun8i-v3s.o +sun8i-de2-ccu-y += ccu-sun8i-de2.o +sun8i-r-ccu-y += ccu-sun8i-r.o +sun9i-a80-ccu-y += ccu-sun9i-a80.o +sun9i-a80-de-ccu-y += ccu-sun9i-a80-de.o +sun9i-a80-usb-ccu-y += ccu-sun9i-a80-usb.o diff --git a/drivers/clk/sunxi-ng/ccu-sun20i-d1-r.c b/drivers/clk/sunxi-ng/ccu-sun20i-d1-r.c new file mode 100644 index 000000000000..9d3ffd3fb2c1 --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu-sun20i-d1-r.c @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2020 huangzhenwei@allwinnertech.com + * Copyright (C) 2021 Samuel Holland + */ + +#include +#include +#include + +#include "ccu_common.h" +#include "ccu_reset.h" + +#include "ccu_gate.h" +#include "ccu_mp.h" + +#include "ccu-sun20i-d1-r.h" + +static const struct clk_parent_data r_ahb_apb0_parents[] = { + { .fw_name = "hosc" }, + { .fw_name = "losc" }, + { .fw_name = "iosc" }, + { .fw_name = "pll-periph" }, +}; +static SUNXI_CCU_MP_DATA_WITH_MUX(r_ahb_clk, "r-ahb", + r_ahb_apb0_parents, 0x000, + 0, 5, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + 0); +static const struct clk_hw *r_ahb_hw = &r_ahb_clk.common.hw; + +static SUNXI_CCU_MP_DATA_WITH_MUX(r_apb0_clk, "r-apb0", + r_ahb_apb0_parents, 0x00c, + 0, 5, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + 0); +static const struct clk_hw *r_apb0_hw = &r_apb0_clk.common.hw; + +static SUNXI_CCU_GATE_HWS(bus_r_timer_clk, "bus-r-timer", &r_apb0_hw, + 0x11c, BIT(0), 0); +static SUNXI_CCU_GATE_HWS(bus_r_twd_clk, "bus-r-twd", &r_apb0_hw, + 0x12c, BIT(0), 0); +static SUNXI_CCU_GATE_HWS(bus_r_ppu_clk, "bus-r-ppu", &r_apb0_hw, + 0x1ac, BIT(0), 0); + +static const struct clk_parent_data r_ir_rx_parents[] = { + { .fw_name = "losc" }, + { .fw_name = "hosc" }, +}; +static SUNXI_CCU_MP_DATA_WITH_MUX_GATE(r_ir_rx_clk, "r-ir-rx", + r_ir_rx_parents, 0x1c0, + 0, 5, /* M */ + 8, 2, /* P */ + 24, 2, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE_HWS(bus_r_ir_rx_clk, "bus-r-ir-rx", &r_apb0_hw, + 0x1cc, BIT(0), 0); +static SUNXI_CCU_GATE_HWS(bus_r_rtc_clk, "bus-r-rtc", &r_ahb_hw, + 0x20c, BIT(0), 0); +static SUNXI_CCU_GATE_HWS(bus_r_cpucfg_clk, "bus-r-cpucfg", &r_apb0_hw, + 0x22c, BIT(0), 0); + +static struct ccu_common *sun20i_d1_r_ccu_clks[] = { + &r_ahb_clk.common, + &r_apb0_clk.common, + &bus_r_timer_clk.common, + &bus_r_twd_clk.common, + &bus_r_ppu_clk.common, + &r_ir_rx_clk.common, + &bus_r_ir_rx_clk.common, + &bus_r_rtc_clk.common, + &bus_r_cpucfg_clk.common, +}; + +static struct clk_hw_onecell_data sun20i_d1_r_hw_clks = { + .num = CLK_NUMBER, + .hws = { + [CLK_R_AHB] = &r_ahb_clk.common.hw, + [CLK_R_APB0] = &r_apb0_clk.common.hw, + [CLK_BUS_R_TIMER] = &bus_r_timer_clk.common.hw, + [CLK_BUS_R_TWD] = &bus_r_twd_clk.common.hw, + [CLK_BUS_R_PPU] = &bus_r_ppu_clk.common.hw, + [CLK_R_IR_RX] = &r_ir_rx_clk.common.hw, + [CLK_BUS_R_IR_RX] = &bus_r_ir_rx_clk.common.hw, + [CLK_BUS_R_RTC] = &bus_r_rtc_clk.common.hw, + [CLK_BUS_R_CPUCFG] = &bus_r_cpucfg_clk.common.hw, + }, +}; + +static struct ccu_reset_map sun20i_d1_r_ccu_resets[] = { + [RST_BUS_R_TIMER] = { 0x11c, BIT(16) }, + [RST_BUS_R_TWD] = { 0x12c, BIT(16) }, + [RST_BUS_R_PPU] = { 0x1ac, BIT(16) }, + [RST_BUS_R_IR_RX] = { 0x1cc, BIT(16) }, + [RST_BUS_R_RTC] = { 0x20c, BIT(16) }, + [RST_BUS_R_CPUCFG] = { 0x22c, BIT(16) }, +}; + +static const struct sunxi_ccu_desc sun20i_d1_r_ccu_desc = { + .ccu_clks = sun20i_d1_r_ccu_clks, + .num_ccu_clks = ARRAY_SIZE(sun20i_d1_r_ccu_clks), + + .hw_clks = &sun20i_d1_r_hw_clks, + + .resets = sun20i_d1_r_ccu_resets, + .num_resets = ARRAY_SIZE(sun20i_d1_r_ccu_resets), +}; + +static int sun20i_d1_r_ccu_probe(struct platform_device *pdev) +{ + void __iomem *reg; + + reg = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(reg)) + return PTR_ERR(reg); + + return devm_sunxi_ccu_probe(&pdev->dev, reg, &sun20i_d1_r_ccu_desc); +} + +static const struct of_device_id sun20i_d1_r_ccu_ids[] = { + { .compatible = "allwinner,sun20i-d1-r-ccu" }, + { } +}; + +static struct platform_driver sun20i_d1_r_ccu_driver = { + .probe = sun20i_d1_r_ccu_probe, + .driver = { + .name = "sun20i-d1-r-ccu", + .suppress_bind_attrs = true, + .of_match_table = sun20i_d1_r_ccu_ids, + }, +}; +module_platform_driver(sun20i_d1_r_ccu_driver); + +MODULE_IMPORT_NS(SUNXI_CCU); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/sunxi-ng/ccu-sun20i-d1-r.h b/drivers/clk/sunxi-ng/ccu-sun20i-d1-r.h new file mode 100644 index 000000000000..afd4342209ee --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu-sun20i-d1-r.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2020 frank@allwinnertech.com + * Copyright (C) 2021 Samuel Holland + */ + +#ifndef _CCU_SUN20I_D1_R_H +#define _CCU_SUN20I_D1_R_H + +#include +#include + +#define CLK_R_APB0 1 + +#define CLK_NUMBER (CLK_BUS_R_CPUCFG + 1) + +#endif /* _CCU_SUN20I_D1_R_H */ diff --git a/drivers/clk/sunxi-ng/ccu-sun20i-d1.c b/drivers/clk/sunxi-ng/ccu-sun20i-d1.c new file mode 100644 index 000000000000..51058ba4db4d --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu-sun20i-d1.c @@ -0,0 +1,1390 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2020 huangzhenwei@allwinnertech.com + * Copyright (C) 2021 Samuel Holland + */ + +#include +#include +#include +#include + +#include "../clk.h" + +#include "ccu_common.h" +#include "ccu_reset.h" + +#include "ccu_div.h" +#include "ccu_gate.h" +#include "ccu_mp.h" +#include "ccu_mult.h" +#include "ccu_nk.h" +#include "ccu_nkm.h" +#include "ccu_nkmp.h" +#include "ccu_nm.h" + +#include "ccu-sun20i-d1.h" + +static const struct clk_parent_data osc24M[] = { + { .fw_name = "hosc" } +}; + +/* + * For the CPU PLL, the output divider is described as "only for testing" + * in the user manual. So it's not modelled and forced to 0. + */ +#define SUN20I_D1_PLL_CPUX_REG 0x000 +static struct ccu_mult pll_cpux_clk = { + .enable = BIT(27), + .lock = BIT(28), + .mult = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .common = { + .reg = 0x000, + .hw.init = CLK_HW_INIT_PARENTS_DATA("pll-cpux", osc24M, + &ccu_mult_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +/* Some PLLs are input * N / div1 / P. Model them as NKMP with no K */ +#define SUN20I_D1_PLL_DDR0_REG 0x010 +static struct ccu_nkmp pll_ddr0_clk = { + .enable = BIT(27), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .p = _SUNXI_CCU_DIV(0, 1), /* output divider */ + .common = { + .reg = 0x010, + .hw.init = CLK_HW_INIT_PARENTS_DATA("pll-ddr0", osc24M, + &ccu_nkmp_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +#define SUN20I_D1_PLL_PERIPH0_REG 0x020 +static struct ccu_nm pll_periph0_4x_clk = { + .enable = BIT(27), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .common = { + .reg = 0x020, + .hw.init = CLK_HW_INIT_PARENTS_DATA("pll-periph0-4x", osc24M, + &ccu_nm_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +static const struct clk_hw *pll_periph0_4x_hws[] = { + &pll_periph0_4x_clk.common.hw +}; +static SUNXI_CCU_M_HWS(pll_periph0_2x_clk, "pll-periph0-2x", + pll_periph0_4x_hws, 0x020, 16, 3, 0); +static SUNXI_CCU_M_HWS(pll_periph0_800M_clk, "pll-periph0-800M", + pll_periph0_4x_hws, 0x020, 20, 3, 0); + +static const struct clk_hw *pll_periph0_2x_hws[] = { + &pll_periph0_2x_clk.common.hw +}; +static CLK_FIXED_FACTOR_HWS(pll_periph0_clk, "pll-periph0", + pll_periph0_2x_hws, 2, 1, 0); + +static const struct clk_hw *pll_periph0_hws[] = { &pll_periph0_clk.hw }; +static CLK_FIXED_FACTOR_HWS(pll_periph0_div3_clk, "pll-periph0-div3", + pll_periph0_2x_hws, 6, 1, 0); + +/* + * For Video PLLs, the output divider is described as "only for testing" + * in the user manual. So it's not modelled and forced to 0. + */ +#define SUN20I_D1_PLL_VIDEO0_REG 0x040 +static struct ccu_nm pll_video0_4x_clk = { + .enable = BIT(27), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .common = { + .reg = 0x040, + .hw.init = CLK_HW_INIT_PARENTS_DATA("pll-video0-4x", osc24M, + &ccu_nm_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +static const struct clk_hw *pll_video0_4x_hws[] = { + &pll_video0_4x_clk.common.hw +}; +static CLK_FIXED_FACTOR_HWS(pll_video0_2x_clk, "pll-video0-2x", + pll_video0_4x_hws, 2, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_video0_clk, "pll-video0", + pll_video0_4x_hws, 4, 1, CLK_SET_RATE_PARENT); + +#define SUN20I_D1_PLL_VIDEO1_REG 0x048 +static struct ccu_nm pll_video1_4x_clk = { + .enable = BIT(27), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .common = { + .reg = 0x048, + .hw.init = CLK_HW_INIT_PARENTS_DATA("pll-video1-4x", osc24M, + &ccu_nm_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +static const struct clk_hw *pll_video1_4x_hws[] = { + &pll_video1_4x_clk.common.hw +}; +static CLK_FIXED_FACTOR_HWS(pll_video1_2x_clk, "pll-video1-2x", + pll_video1_4x_hws, 2, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_video1_clk, "pll-video1", + pll_video1_4x_hws, 4, 1, CLK_SET_RATE_PARENT); + +#define SUN20I_D1_PLL_VE_REG 0x058 +static struct ccu_nkmp pll_ve_clk = { + .enable = BIT(27), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .p = _SUNXI_CCU_DIV(0, 1), /* output divider */ + .common = { + .reg = 0x058, + .hw.init = CLK_HW_INIT_PARENTS_DATA("pll-ve", osc24M, + &ccu_nkmp_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +/* + * PLL_AUDIO0 has m0, m1 dividers in addition to the usual N, M factors. + * Since we only need one frequency from this PLL (22.5792 x 4 == 90.3168 MHz), + * ignore them for now. Enforce the default for them, which is m1 = 0, m0 = 0. + * The M factor must be an even number to produce a 50% duty cycle output. + */ +#define SUN20I_D1_PLL_AUDIO0_REG 0x078 +static struct ccu_sdm_setting pll_audio0_sdm_table[] = { + { .rate = 90316800, .pattern = 0xc001288d, .m = 6, .n = 22 }, +}; + +static struct ccu_nm pll_audio0_4x_clk = { + .enable = BIT(27), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(16, 6), + .sdm = _SUNXI_CCU_SDM(pll_audio0_sdm_table, BIT(24), + 0x178, BIT(31)), + .common = { + .reg = 0x078, + .features = CCU_FEATURE_SIGMA_DELTA_MOD, + .hw.init = CLK_HW_INIT_PARENTS_DATA("pll-audio0-4x", osc24M, + &ccu_nm_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +static const struct clk_hw *pll_audio0_4x_hws[] = { + &pll_audio0_4x_clk.common.hw +}; +static CLK_FIXED_FACTOR_HWS(pll_audio0_2x_clk, "pll-audio0-2x", + pll_audio0_4x_hws, 2, 1, 0); +static CLK_FIXED_FACTOR_HWS(pll_audio0_clk, "pll-audio0", + pll_audio0_4x_hws, 4, 1, 0); + +/* + * PLL_AUDIO1 doesn't need Fractional-N. The output is usually 614.4 MHz for + * audio. The ADC or DAC should divide the PLL output further to 24.576 MHz. + */ +#define SUN20I_D1_PLL_AUDIO1_REG 0x080 +static struct ccu_nm pll_audio1_clk = { + .enable = BIT(27), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), + .common = { + .reg = 0x080, + .hw.init = CLK_HW_INIT_PARENTS_DATA("pll-audio1", osc24M, + &ccu_nm_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +static const struct clk_hw *pll_audio1_hws[] = { + &pll_audio1_clk.common.hw +}; +static SUNXI_CCU_M_HWS(pll_audio1_div2_clk, "pll-audio1-div2", + pll_audio1_hws, 0x080, 16, 3, 0); +static SUNXI_CCU_M_HWS(pll_audio1_div5_clk, "pll-audio1-div5", + pll_audio1_hws, 0x080, 20, 3, 0); + +/* + * The CPUX gate is not modelled - it is in a separate register (0x504) + * and has a special key field. The clock does not need to be ungated anyway. + */ +static const struct clk_parent_data cpux_parents[] = { + { .fw_name = "hosc" }, + { .fw_name = "losc" }, + { .fw_name = "iosc" }, + { .hw = &pll_cpux_clk.common.hw }, + { .hw = &pll_periph0_clk.hw }, + { .hw = &pll_periph0_2x_clk.common.hw }, + { .hw = &pll_periph0_800M_clk.common.hw }, +}; +static SUNXI_CCU_MUX_DATA(cpux_clk, "cpux", cpux_parents, + 0x500, 24, 3, CLK_SET_RATE_PARENT); + +static const struct clk_hw *cpux_hws[] = { &cpux_clk.common.hw }; +static SUNXI_CCU_M_HWS(cpux_axi_clk, "cpux-axi", + cpux_hws, 0x500, 0, 2, 0); +static SUNXI_CCU_M_HWS(cpux_apb_clk, "cpux-apb", + cpux_hws, 0x500, 8, 2, 0); + +static const struct clk_parent_data psi_ahb_parents[] = { + { .fw_name = "hosc" }, + { .fw_name = "losc" }, + { .fw_name = "iosc" }, + { .hw = &pll_periph0_clk.hw }, +}; +static SUNXI_CCU_MP_DATA_WITH_MUX(psi_ahb_clk, "psi-ahb", psi_ahb_parents, 0x510, + 0, 2, /* M */ + 8, 2, /* P */ + 24, 2, /* mux */ + 0); + +static const struct clk_parent_data apb0_apb1_parents[] = { + { .fw_name = "hosc" }, + { .fw_name = "losc" }, + { .hw = &psi_ahb_clk.common.hw }, + { .hw = &pll_periph0_clk.hw }, +}; +static SUNXI_CCU_MP_DATA_WITH_MUX(apb0_clk, "apb0", apb0_apb1_parents, 0x520, + 0, 5, /* M */ + 8, 2, /* P */ + 24, 2, /* mux */ + 0); + +static SUNXI_CCU_MP_DATA_WITH_MUX(apb1_clk, "apb1", apb0_apb1_parents, 0x524, + 0, 5, /* M */ + 8, 2, /* P */ + 24, 2, /* mux */ + 0); + +static const struct clk_hw *psi_ahb_hws[] = { &psi_ahb_clk.common.hw }; +static const struct clk_hw *apb0_hws[] = { &apb0_clk.common.hw }; +static const struct clk_hw *apb1_hws[] = { &apb1_clk.common.hw }; + +static const struct clk_hw *de_di_g2d_parents[] = { + &pll_periph0_2x_clk.common.hw, + &pll_video0_4x_clk.common.hw, + &pll_video1_4x_clk.common.hw, + &pll_audio1_div2_clk.common.hw, +}; +static SUNXI_CCU_M_HW_WITH_MUX_GATE(de_clk, "de", de_di_g2d_parents, 0x600, + 0, 5, /* M */ + 24, 3, /* mux */ + BIT(31), /* gate */ + CLK_SET_RATE_PARENT); + +static SUNXI_CCU_GATE_HWS(bus_de_clk, "bus-de", psi_ahb_hws, + 0x60c, BIT(0), 0); + +static SUNXI_CCU_M_HW_WITH_MUX_GATE(di_clk, "di", de_di_g2d_parents, 0x620, + 0, 5, /* M */ + 24, 3, /* mux */ + BIT(31), /* gate */ + CLK_SET_RATE_PARENT); + +static SUNXI_CCU_GATE_HWS(bus_di_clk, "bus-di", psi_ahb_hws, + 0x62c, BIT(0), 0); + +static SUNXI_CCU_M_HW_WITH_MUX_GATE(g2d_clk, "g2d", de_di_g2d_parents, 0x630, + 0, 5, /* M */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE_HWS(bus_g2d_clk, "bus-g2d", psi_ahb_hws, + 0x63c, BIT(0), 0); + +static const struct clk_parent_data ce_parents[] = { + { .fw_name = "hosc" }, + { .hw = &pll_periph0_2x_clk.common.hw }, + { .hw = &pll_periph0_clk.hw }, +}; +static SUNXI_CCU_MP_DATA_WITH_MUX_GATE(ce_clk, "ce", ce_parents, 0x680, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE_HWS(bus_ce_clk, "bus-ce", psi_ahb_hws, + 0x68c, BIT(0), 0); + +static const struct clk_hw *ve_parents[] = { + &pll_ve_clk.common.hw, + &pll_periph0_2x_clk.common.hw, +}; +static SUNXI_CCU_M_HW_WITH_MUX_GATE(ve_clk, "ve", ve_parents, 0x690, + 0, 5, /* M */ + 24, 1, /* mux */ + BIT(31), /* gate */ + CLK_SET_RATE_PARENT); + +static SUNXI_CCU_GATE_HWS(bus_ve_clk, "bus-ve", psi_ahb_hws, + 0x69c, BIT(0), 0); + +static SUNXI_CCU_GATE_HWS(bus_dma_clk, "bus-dma", psi_ahb_hws, + 0x70c, BIT(0), 0); + +static SUNXI_CCU_GATE_HWS(bus_msgbox0_clk, "bus-msgbox0", psi_ahb_hws, + 0x71c, BIT(0), 0); +static SUNXI_CCU_GATE_HWS(bus_msgbox1_clk, "bus-msgbox1", psi_ahb_hws, + 0x71c, BIT(1), 0); +static SUNXI_CCU_GATE_HWS(bus_msgbox2_clk, "bus-msgbox2", psi_ahb_hws, + 0x71c, BIT(2), 0); + +static SUNXI_CCU_GATE_HWS(bus_spinlock_clk, "bus-spinlock", psi_ahb_hws, + 0x72c, BIT(0), 0); + +static SUNXI_CCU_GATE_HWS(bus_hstimer_clk, "bus-hstimer", psi_ahb_hws, + 0x73c, BIT(0), 0); + +static SUNXI_CCU_GATE_DATA(avs_clk, "avs", osc24M, + 0x740, BIT(31), 0); + +static SUNXI_CCU_GATE_HWS(bus_dbg_clk, "bus-dbg", psi_ahb_hws, + 0x78c, BIT(0), 0); + +static SUNXI_CCU_GATE_HWS(bus_pwm_clk, "bus-pwm", apb0_hws, + 0x7ac, BIT(0), 0); + +static SUNXI_CCU_GATE_HWS(bus_iommu_clk, "bus-iommu", apb0_hws, + 0x7bc, BIT(0), 0); + +static const struct clk_hw *dram_parents[] = { + &pll_ddr0_clk.common.hw, + &pll_audio1_div2_clk.common.hw, + &pll_periph0_2x_clk.common.hw, + &pll_periph0_800M_clk.common.hw, +}; +static SUNXI_CCU_MP_HW_WITH_MUX_GATE(dram_clk, "dram", dram_parents, 0x800, + 0, 2, /* M */ + 8, 2, /* P */ + 24, 2, /* mux */ + BIT(31), CLK_IS_CRITICAL); + +static CLK_FIXED_FACTOR_HW(mbus_clk, "mbus", + &dram_clk.common.hw, 4, 1, 0); + +static const struct clk_hw *mbus_hws[] = { &mbus_clk.hw }; + +static SUNXI_CCU_GATE_HWS(mbus_dma_clk, "mbus-dma", mbus_hws, + 0x804, BIT(0), 0); +static SUNXI_CCU_GATE_HWS(mbus_ve_clk, "mbus-ve", mbus_hws, + 0x804, BIT(1), 0); +static SUNXI_CCU_GATE_HWS(mbus_ce_clk, "mbus-ce", mbus_hws, + 0x804, BIT(2), 0); +static SUNXI_CCU_GATE_HWS(mbus_tvin_clk, "mbus-tvin", mbus_hws, + 0x804, BIT(7), 0); +static SUNXI_CCU_GATE_HWS(mbus_csi_clk, "mbus-csi", mbus_hws, + 0x804, BIT(8), 0); +static SUNXI_CCU_GATE_HWS(mbus_g2d_clk, "mbus-g2d", mbus_hws, + 0x804, BIT(10), 0); +static SUNXI_CCU_GATE_HWS(mbus_riscv_clk, "mbus-riscv", mbus_hws, + 0x804, BIT(11), 0); + +static SUNXI_CCU_GATE_HWS(bus_dram_clk, "bus-dram", psi_ahb_hws, + 0x80c, BIT(0), CLK_IS_CRITICAL); + +static const struct clk_parent_data mmc0_mmc1_parents[] = { + { .fw_name = "hosc" }, + { .hw = &pll_periph0_clk.hw }, + { .hw = &pll_periph0_2x_clk.common.hw }, + { .hw = &pll_audio1_div2_clk.common.hw }, +}; +static SUNXI_CCU_MP_DATA_WITH_MUX_GATE(mmc0_clk, "mmc0", mmc0_mmc1_parents, 0x830, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_MP_DATA_WITH_MUX_GATE(mmc1_clk, "mmc1", mmc0_mmc1_parents, 0x834, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static const struct clk_parent_data mmc2_parents[] = { + { .fw_name = "hosc" }, + { .hw = &pll_periph0_clk.hw }, + { .hw = &pll_periph0_2x_clk.common.hw }, + { .hw = &pll_periph0_800M_clk.common.hw }, + { .hw = &pll_audio1_div2_clk.common.hw }, +}; +static SUNXI_CCU_MP_DATA_WITH_MUX_GATE(mmc2_clk, "mmc2", mmc2_parents, 0x838, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE_HWS(bus_mmc0_clk, "bus-mmc0", psi_ahb_hws, + 0x84c, BIT(0), 0); +static SUNXI_CCU_GATE_HWS(bus_mmc1_clk, "bus-mmc1", psi_ahb_hws, + 0x84c, BIT(1), 0); +static SUNXI_CCU_GATE_HWS(bus_mmc2_clk, "bus-mmc2", psi_ahb_hws, + 0x84c, BIT(2), 0); + +static SUNXI_CCU_GATE_HWS(bus_uart0_clk, "bus-uart0", apb1_hws, + 0x90c, BIT(0), 0); +static SUNXI_CCU_GATE_HWS(bus_uart1_clk, "bus-uart1", apb1_hws, + 0x90c, BIT(1), 0); +static SUNXI_CCU_GATE_HWS(bus_uart2_clk, "bus-uart2", apb1_hws, + 0x90c, BIT(2), 0); +static SUNXI_CCU_GATE_HWS(bus_uart3_clk, "bus-uart3", apb1_hws, + 0x90c, BIT(3), 0); +static SUNXI_CCU_GATE_HWS(bus_uart4_clk, "bus-uart4", apb1_hws, + 0x90c, BIT(4), 0); +static SUNXI_CCU_GATE_HWS(bus_uart5_clk, "bus-uart5", apb1_hws, + 0x90c, BIT(5), 0); + +static SUNXI_CCU_GATE_HWS(bus_i2c0_clk, "bus-i2c0", apb1_hws, + 0x91c, BIT(0), 0); +static SUNXI_CCU_GATE_HWS(bus_i2c1_clk, "bus-i2c1", apb1_hws, + 0x91c, BIT(1), 0); +static SUNXI_CCU_GATE_HWS(bus_i2c2_clk, "bus-i2c2", apb1_hws, + 0x91c, BIT(2), 0); +static SUNXI_CCU_GATE_HWS(bus_i2c3_clk, "bus-i2c3", apb1_hws, + 0x91c, BIT(3), 0); + +static const struct clk_parent_data spi_parents[] = { + { .fw_name = "hosc" }, + { .hw = &pll_periph0_clk.hw }, + { .hw = &pll_periph0_2x_clk.common.hw }, + { .hw = &pll_audio1_div2_clk.common.hw }, + { .hw = &pll_audio1_div5_clk.common.hw }, +}; +static SUNXI_CCU_MP_DATA_WITH_MUX_GATE(spi0_clk, "spi0", spi_parents, 0x940, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_MP_DATA_WITH_MUX_GATE(spi1_clk, "spi1", spi_parents, 0x944, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE_HWS(bus_spi0_clk, "bus-spi0", psi_ahb_hws, + 0x96c, BIT(0), 0); +static SUNXI_CCU_GATE_HWS(bus_spi1_clk, "bus-spi1", psi_ahb_hws, + 0x96c, BIT(1), 0); + +static SUNXI_CCU_GATE_HWS_WITH_PREDIV(emac_25M_clk, "emac-25M", pll_periph0_hws, + 0x970, BIT(31) | BIT(30), 24, 0); + +static SUNXI_CCU_GATE_HWS(bus_emac_clk, "bus-emac", psi_ahb_hws, + 0x97c, BIT(0), 0); + +static const struct clk_parent_data ir_tx_ledc_parents[] = { + { .fw_name = "hosc" }, + { .hw = &pll_periph0_clk.hw }, +}; +static SUNXI_CCU_MP_DATA_WITH_MUX_GATE(ir_tx_clk, "ir-tx", ir_tx_ledc_parents, 0x9c0, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE_HWS(bus_ir_tx_clk, "bus-ir-tx", apb0_hws, + 0x9cc, BIT(0), 0); + +static SUNXI_CCU_GATE_HWS(bus_gpadc_clk, "bus-gpadc", apb0_hws, + 0x9ec, BIT(0), 0); + +static SUNXI_CCU_GATE_HWS(bus_ths_clk, "bus-ths", apb0_hws, + 0x9fc, BIT(0), 0); + +static const struct clk_hw *i2s_spdif_tx_parents[] = { + &pll_audio0_clk.hw, + &pll_audio0_4x_clk.common.hw, + &pll_audio1_div2_clk.common.hw, + &pll_audio1_div5_clk.common.hw, +}; +static SUNXI_CCU_MP_HW_WITH_MUX_GATE(i2s0_clk, "i2s0", i2s_spdif_tx_parents, 0xa10, + 0, 5, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_MP_HW_WITH_MUX_GATE(i2s1_clk, "i2s1", i2s_spdif_tx_parents, 0xa14, + 0, 5, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_MP_HW_WITH_MUX_GATE(i2s2_clk, "i2s2", i2s_spdif_tx_parents, 0xa18, + 0, 5, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static const struct clk_hw *i2s2_asrc_parents[] = { + &pll_audio0_4x_clk.common.hw, + &pll_periph0_clk.hw, + &pll_audio1_div2_clk.common.hw, + &pll_audio1_div5_clk.common.hw, +}; +static SUNXI_CCU_MP_HW_WITH_MUX_GATE(i2s2_asrc_clk, "i2s2-asrc", i2s2_asrc_parents, 0xa1c, + 0, 5, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE_HWS(bus_i2s0_clk, "bus-i2s0", apb0_hws, + 0xa20, BIT(0), 0); +static SUNXI_CCU_GATE_HWS(bus_i2s1_clk, "bus-i2s1", apb0_hws, + 0xa20, BIT(1), 0); +static SUNXI_CCU_GATE_HWS(bus_i2s2_clk, "bus-i2s2", apb0_hws, + 0xa20, BIT(2), 0); + +static SUNXI_CCU_MP_HW_WITH_MUX_GATE(spdif_tx_clk, "spdif-tx", i2s_spdif_tx_parents, 0xa24, + 0, 5, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static const struct clk_hw *spdif_rx_parents[] = { + &pll_periph0_clk.hw, + &pll_audio1_div2_clk.common.hw, + &pll_audio1_div5_clk.common.hw, +}; +static SUNXI_CCU_MP_HW_WITH_MUX_GATE(spdif_rx_clk, "spdif-rx", spdif_rx_parents, 0xa28, + 0, 5, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE_HWS(bus_spdif_clk, "bus-spdif", apb0_hws, + 0xa2c, BIT(0), 0); + +static const struct clk_hw *dmic_codec_parents[] = { + &pll_audio0_clk.hw, + &pll_audio1_div2_clk.common.hw, + &pll_audio1_div5_clk.common.hw, +}; +static SUNXI_CCU_MP_HW_WITH_MUX_GATE(dmic_clk, "dmic", dmic_codec_parents, 0xa40, + 0, 5, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE_HWS(bus_dmic_clk, "bus-dmic", apb0_hws, + 0xa4c, BIT(0), 0); + +static SUNXI_CCU_MP_HW_WITH_MUX_GATE(audio_dac_clk, "audio-dac", dmic_codec_parents, 0xa50, + 0, 5, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_MP_HW_WITH_MUX_GATE(audio_adc_clk, "audio-adc", dmic_codec_parents, 0xa54, + 0, 5, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE_HWS(bus_audio_clk, "bus-audio", apb0_hws, + 0xa5c, BIT(0), 0); + + +/* + * The first parent is a 48 MHz input clock divided by 4. That 48 MHz clock is + * a 2x multiplier from osc24M synchronized by pll-periph0, and is also used by + * the OHCI module. + */ +static const struct clk_parent_data usb_ohci_parents[] = { + { .hw = &pll_periph0_clk.hw }, + { .fw_name = "hosc" }, + { .fw_name = "losc" }, +}; +static const struct ccu_mux_fixed_prediv usb_ohci_predivs[] = { + { .index = 0, .div = 50 }, + { .index = 1, .div = 2 }, +}; + +static struct ccu_mux usb_ohci0_clk = { + .enable = BIT(31), + .mux = { + .shift = 24, + .width = 2, + .fixed_predivs = usb_ohci_predivs, + .n_predivs = ARRAY_SIZE(usb_ohci_predivs), + }, + .common = { + .reg = 0xa70, + .features = CCU_FEATURE_FIXED_PREDIV, + .hw.init = CLK_HW_INIT_PARENTS_DATA("usb-ohci0", + usb_ohci_parents, + &ccu_mux_ops, + 0), + }, +}; + +static struct ccu_mux usb_ohci1_clk = { + .enable = BIT(31), + .mux = { + .shift = 24, + .width = 2, + .fixed_predivs = usb_ohci_predivs, + .n_predivs = ARRAY_SIZE(usb_ohci_predivs), + }, + .common = { + .reg = 0xa74, + .features = CCU_FEATURE_FIXED_PREDIV, + .hw.init = CLK_HW_INIT_PARENTS_DATA("usb-ohci1", + usb_ohci_parents, + &ccu_mux_ops, + 0), + }, +}; + +static SUNXI_CCU_GATE_HWS(bus_ohci0_clk, "bus-ohci0", psi_ahb_hws, + 0xa8c, BIT(0), 0); +static SUNXI_CCU_GATE_HWS(bus_ohci1_clk, "bus-ohci1", psi_ahb_hws, + 0xa8c, BIT(1), 0); +static SUNXI_CCU_GATE_HWS(bus_ehci0_clk, "bus-ehci0", psi_ahb_hws, + 0xa8c, BIT(4), 0); +static SUNXI_CCU_GATE_HWS(bus_ehci1_clk, "bus-ehci1", psi_ahb_hws, + 0xa8c, BIT(5), 0); +static SUNXI_CCU_GATE_HWS(bus_otg_clk, "bus-otg", psi_ahb_hws, + 0xa8c, BIT(8), 0); + +static SUNXI_CCU_GATE_HWS(bus_lradc_clk, "bus-lradc", apb0_hws, + 0xa9c, BIT(0), 0); + +static SUNXI_CCU_GATE_HWS(bus_dpss_top_clk, "bus-dpss-top", psi_ahb_hws, + 0xabc, BIT(0), 0); + +static SUNXI_CCU_GATE_DATA(hdmi_24M_clk, "hdmi-24M", osc24M, + 0xb04, BIT(31), 0); + +static SUNXI_CCU_GATE_HWS_WITH_PREDIV(hdmi_cec_32k_clk, "hdmi-cec-32k", + pll_periph0_2x_hws, + 0xb10, BIT(30), 36621, 0); + +static const struct clk_parent_data hdmi_cec_parents[] = { + { .fw_name = "losc" }, + { .hw = &hdmi_cec_32k_clk.common.hw }, +}; +static SUNXI_CCU_MUX_DATA_WITH_GATE(hdmi_cec_clk, "hdmi-cec", hdmi_cec_parents, 0xb10, + 24, 1, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE_HWS(bus_hdmi_clk, "bus-hdmi", psi_ahb_hws, + 0xb1c, BIT(0), 0); + +static const struct clk_parent_data mipi_dsi_parents[] = { + { .fw_name = "hosc" }, + { .hw = &pll_periph0_clk.hw }, + { .hw = &pll_video0_2x_clk.hw }, + { .hw = &pll_video1_2x_clk.hw }, + { .hw = &pll_audio1_div2_clk.common.hw }, +}; +static SUNXI_CCU_M_DATA_WITH_MUX_GATE(mipi_dsi_clk, "mipi-dsi", mipi_dsi_parents, 0xb24, + 0, 4, /* M */ + 24, 3, /* mux */ + BIT(31), /* gate */ + CLK_SET_RATE_PARENT); + +static SUNXI_CCU_GATE_HWS(bus_mipi_dsi_clk, "bus-mipi-dsi", psi_ahb_hws, + 0xb4c, BIT(0), 0); + +static const struct clk_hw *tcon_tve_parents[] = { + &pll_video0_clk.hw, + &pll_video0_4x_clk.common.hw, + &pll_video1_clk.hw, + &pll_video1_4x_clk.common.hw, + &pll_periph0_2x_clk.common.hw, + &pll_audio1_div2_clk.common.hw, +}; +static SUNXI_CCU_MP_HW_WITH_MUX_GATE(tcon_lcd0_clk, "tcon-lcd0", tcon_tve_parents, 0xb60, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + CLK_SET_RATE_PARENT); + +static SUNXI_CCU_GATE_HWS(bus_tcon_lcd0_clk, "bus-tcon-lcd0", psi_ahb_hws, + 0xb7c, BIT(0), 0); + +static SUNXI_CCU_MP_HW_WITH_MUX_GATE(tcon_tv_clk, "tcon-tv", tcon_tve_parents, 0xb80, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + CLK_SET_RATE_PARENT); + +static SUNXI_CCU_GATE_HWS(bus_tcon_tv_clk, "bus-tcon-tv", psi_ahb_hws, + 0xb9c, BIT(0), 0); + +static SUNXI_CCU_MP_HW_WITH_MUX_GATE(tve_clk, "tve", tcon_tve_parents, 0xbb0, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE_HWS(bus_tve_top_clk, "bus-tve-top", psi_ahb_hws, + 0xbbc, BIT(0), 0); +static SUNXI_CCU_GATE_HWS(bus_tve_clk, "bus-tve", psi_ahb_hws, + 0xbbc, BIT(1), 0); + +static const struct clk_parent_data tvd_parents[] = { + { .fw_name = "hosc" }, + { .hw = &pll_video0_clk.hw }, + { .hw = &pll_video1_clk.hw }, + { .hw = &pll_periph0_clk.hw }, +}; +static SUNXI_CCU_M_DATA_WITH_MUX_GATE(tvd_clk, "tvd", tvd_parents, 0xbc0, + 0, 5, /* M */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE_HWS(bus_tvd_top_clk, "bus-tvd-top", psi_ahb_hws, + 0xbdc, BIT(0), 0); +static SUNXI_CCU_GATE_HWS(bus_tvd_clk, "bus-tvd", psi_ahb_hws, + 0xbdc, BIT(1), 0); + +static SUNXI_CCU_MP_DATA_WITH_MUX_GATE(ledc_clk, "ledc", ir_tx_ledc_parents, 0xbf0, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 1, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE_HWS(bus_ledc_clk, "bus-ledc", psi_ahb_hws, + 0xbfc, BIT(0), 0); + +static const struct clk_hw *csi_top_parents[] = { + &pll_periph0_2x_clk.common.hw, + &pll_video0_2x_clk.hw, + &pll_video1_2x_clk.hw, +}; +static SUNXI_CCU_M_HW_WITH_MUX_GATE(csi_top_clk, "csi-top", csi_top_parents, 0xc04, + 0, 4, /* M */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static const struct clk_parent_data csi_mclk_parents[] = { + { .fw_name = "hosc" }, + { .hw = &pll_periph0_clk.hw }, + { .hw = &pll_video0_clk.hw }, + { .hw = &pll_video1_clk.hw }, + { .hw = &pll_audio1_div2_clk.common.hw }, + { .hw = &pll_audio1_div5_clk.common.hw }, +}; +static SUNXI_CCU_M_DATA_WITH_MUX_GATE(csi_mclk_clk, "csi-mclk", csi_mclk_parents, 0xc08, + 0, 5, /* M */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE_HWS(bus_csi_clk, "bus-csi", psi_ahb_hws, + 0xc1c, BIT(0), 0); + +static const struct clk_parent_data tpadc_parents[] = { + { .fw_name = "hosc" }, + { .hw = &pll_audio0_clk.hw }, +}; +static SUNXI_CCU_MUX_DATA_WITH_GATE(tpadc_clk, "tpadc", tpadc_parents, 0xc50, + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE_HWS(bus_tpadc_clk, "bus-tpadc", apb0_hws, + 0xc5c, BIT(0), 0); + +static SUNXI_CCU_GATE_HWS(bus_tzma_clk, "bus-tzma", apb0_hws, + 0xc6c, BIT(0), 0); + +static const struct clk_parent_data dsp_parents[] = { + { .fw_name = "hosc" }, + { .fw_name = "losc" }, + { .fw_name = "iosc" }, + { .hw = &pll_periph0_2x_clk.common.hw }, + { .hw = &pll_audio1_div2_clk.common.hw }, +}; +static SUNXI_CCU_M_DATA_WITH_MUX_GATE(dsp_clk, "dsp", dsp_parents, 0xc70, + 0, 5, /* M */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE_HWS(bus_dsp_cfg_clk, "bus-dsp-cfg", psi_ahb_hws, + 0xc7c, BIT(1), 0); + +/* + * The RISC-V gate is not modelled - it is in a separate register (0xd04) + * and has a special key field. The clock is critical anyway. + */ +static const struct clk_parent_data riscv_parents[] = { + { .fw_name = "hosc" }, + { .fw_name = "losc" }, + { .fw_name = "iosc" }, + { .hw = &pll_periph0_800M_clk.common.hw }, + { .hw = &pll_periph0_clk.hw }, + { .hw = &pll_cpux_clk.common.hw }, + { .hw = &pll_audio1_div2_clk.common.hw }, +}; +static SUNXI_CCU_M_DATA_WITH_MUX(riscv_clk, "riscv", riscv_parents, 0xd00, + 0, 5, /* M */ + 24, 3, /* mux */ + CLK_SET_RATE_PARENT | CLK_IS_CRITICAL); + +/* The riscv-axi clk must be divided by at least 2. */ +static struct clk_div_table riscv_axi_table[] = { + { .val = 1, .div = 2 }, + { .val = 2, .div = 3 }, + { .val = 3, .div = 4 }, + { /* Sentinel */ } +}; +static SUNXI_CCU_DIV_TABLE_HW(riscv_axi_clk, "riscv-axi", &riscv_clk.common.hw, + 0xd00, 8, 2, riscv_axi_table, 0); + +static SUNXI_CCU_GATE_HWS(bus_riscv_cfg_clk, "bus-riscv-cfg", psi_ahb_hws, + 0xd0c, BIT(0), CLK_IS_CRITICAL); + +static SUNXI_CCU_GATE_DATA(fanout_24M_clk, "fanout-24M", osc24M, + 0xf30, BIT(0), 0); +static SUNXI_CCU_GATE_DATA_WITH_PREDIV(fanout_12M_clk, "fanout-12M", osc24M, + 0xf30, BIT(1), 2, 0); +static SUNXI_CCU_GATE_HWS_WITH_PREDIV(fanout_16M_clk, "fanout-16M", pll_periph0_2x_hws, + 0xf30, BIT(2), 75, 0); +static SUNXI_CCU_GATE_HWS_WITH_PREDIV(fanout_25M_clk, "fanout-25M", pll_periph0_hws, + 0xf30, BIT(3), 24, 0); +static SUNXI_CCU_GATE_HWS_WITH_PREDIV(fanout_32k_clk, "fanout-32k", pll_periph0_2x_hws, + 0xf30, BIT(4), 36621, 0); + +/* This clock has a second divider that is not modelled and forced to 0. */ +#define SUN20I_D1_FANOUT_27M_REG 0xf34 +static const struct clk_hw *fanout_27M_parents[] = { + &pll_video0_clk.hw, + &pll_video1_clk.hw, +}; +static SUNXI_CCU_M_HW_WITH_MUX_GATE(fanout_27M_clk, "fanout-27M", fanout_27M_parents, 0xf34, + 0, 5, /* M */ + 24, 2, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_M_HWS_WITH_GATE(fanout_pclk_clk, "fanout-pclk", apb0_hws, 0xf38, + 0, 5, /* M */ + BIT(31), /* gate */ + 0); + +static const struct clk_hw *fanout_parents[] = { + &fanout_32k_clk.common.hw, + &fanout_12M_clk.common.hw, + &fanout_16M_clk.common.hw, + &fanout_24M_clk.common.hw, + &fanout_25M_clk.common.hw, + &fanout_27M_clk.common.hw, + &fanout_pclk_clk.common.hw, +}; +static SUNXI_CCU_MUX_HW_WITH_GATE(fanout0_clk, "fanout0", fanout_parents, 0xf3c, + 0, 3, /* mux */ + BIT(21), /* gate */ + 0); +static SUNXI_CCU_MUX_HW_WITH_GATE(fanout1_clk, "fanout1", fanout_parents, 0xf3c, + 3, 3, /* mux */ + BIT(22), /* gate */ + 0); +static SUNXI_CCU_MUX_HW_WITH_GATE(fanout2_clk, "fanout2", fanout_parents, 0xf3c, + 6, 3, /* mux */ + BIT(23), /* gate */ + 0); + +static struct ccu_common *sun20i_d1_ccu_clks[] = { + &pll_cpux_clk.common, + &pll_ddr0_clk.common, + &pll_periph0_4x_clk.common, + &pll_periph0_2x_clk.common, + &pll_periph0_800M_clk.common, + &pll_video0_4x_clk.common, + &pll_video1_4x_clk.common, + &pll_ve_clk.common, + &pll_audio0_4x_clk.common, + &pll_audio1_clk.common, + &pll_audio1_div2_clk.common, + &pll_audio1_div5_clk.common, + &cpux_clk.common, + &cpux_axi_clk.common, + &cpux_apb_clk.common, + &psi_ahb_clk.common, + &apb0_clk.common, + &apb1_clk.common, + &de_clk.common, + &bus_de_clk.common, + &di_clk.common, + &bus_di_clk.common, + &g2d_clk.common, + &bus_g2d_clk.common, + &ce_clk.common, + &bus_ce_clk.common, + &ve_clk.common, + &bus_ve_clk.common, + &bus_dma_clk.common, + &bus_msgbox0_clk.common, + &bus_msgbox1_clk.common, + &bus_msgbox2_clk.common, + &bus_spinlock_clk.common, + &bus_hstimer_clk.common, + &avs_clk.common, + &bus_dbg_clk.common, + &bus_pwm_clk.common, + &bus_iommu_clk.common, + &dram_clk.common, + &mbus_dma_clk.common, + &mbus_ve_clk.common, + &mbus_ce_clk.common, + &mbus_tvin_clk.common, + &mbus_csi_clk.common, + &mbus_g2d_clk.common, + &mbus_riscv_clk.common, + &bus_dram_clk.common, + &mmc0_clk.common, + &mmc1_clk.common, + &mmc2_clk.common, + &bus_mmc0_clk.common, + &bus_mmc1_clk.common, + &bus_mmc2_clk.common, + &bus_uart0_clk.common, + &bus_uart1_clk.common, + &bus_uart2_clk.common, + &bus_uart3_clk.common, + &bus_uart4_clk.common, + &bus_uart5_clk.common, + &bus_i2c0_clk.common, + &bus_i2c1_clk.common, + &bus_i2c2_clk.common, + &bus_i2c3_clk.common, + &spi0_clk.common, + &spi1_clk.common, + &bus_spi0_clk.common, + &bus_spi1_clk.common, + &emac_25M_clk.common, + &bus_emac_clk.common, + &ir_tx_clk.common, + &bus_ir_tx_clk.common, + &bus_gpadc_clk.common, + &bus_ths_clk.common, + &i2s0_clk.common, + &i2s1_clk.common, + &i2s2_clk.common, + &i2s2_asrc_clk.common, + &bus_i2s0_clk.common, + &bus_i2s1_clk.common, + &bus_i2s2_clk.common, + &spdif_tx_clk.common, + &spdif_rx_clk.common, + &bus_spdif_clk.common, + &dmic_clk.common, + &bus_dmic_clk.common, + &audio_dac_clk.common, + &audio_adc_clk.common, + &bus_audio_clk.common, + &usb_ohci0_clk.common, + &usb_ohci1_clk.common, + &bus_ohci0_clk.common, + &bus_ohci1_clk.common, + &bus_ehci0_clk.common, + &bus_ehci1_clk.common, + &bus_otg_clk.common, + &bus_lradc_clk.common, + &bus_dpss_top_clk.common, + &hdmi_24M_clk.common, + &hdmi_cec_32k_clk.common, + &hdmi_cec_clk.common, + &bus_hdmi_clk.common, + &mipi_dsi_clk.common, + &bus_mipi_dsi_clk.common, + &tcon_lcd0_clk.common, + &bus_tcon_lcd0_clk.common, + &tcon_tv_clk.common, + &bus_tcon_tv_clk.common, + &tve_clk.common, + &bus_tve_top_clk.common, + &bus_tve_clk.common, + &tvd_clk.common, + &bus_tvd_top_clk.common, + &bus_tvd_clk.common, + &ledc_clk.common, + &bus_ledc_clk.common, + &csi_top_clk.common, + &csi_mclk_clk.common, + &bus_csi_clk.common, + &tpadc_clk.common, + &bus_tpadc_clk.common, + &bus_tzma_clk.common, + &dsp_clk.common, + &bus_dsp_cfg_clk.common, + &riscv_clk.common, + &riscv_axi_clk.common, + &bus_riscv_cfg_clk.common, + &fanout_24M_clk.common, + &fanout_12M_clk.common, + &fanout_16M_clk.common, + &fanout_25M_clk.common, + &fanout_32k_clk.common, + &fanout_27M_clk.common, + &fanout_pclk_clk.common, + &fanout0_clk.common, + &fanout1_clk.common, + &fanout2_clk.common, +}; + +static struct clk_hw_onecell_data sun20i_d1_hw_clks = { + .num = CLK_NUMBER, + .hws = { + [CLK_PLL_CPUX] = &pll_cpux_clk.common.hw, + [CLK_PLL_DDR0] = &pll_ddr0_clk.common.hw, + [CLK_PLL_PERIPH0_4X] = &pll_periph0_4x_clk.common.hw, + [CLK_PLL_PERIPH0_2X] = &pll_periph0_2x_clk.common.hw, + [CLK_PLL_PERIPH0_800M] = &pll_periph0_800M_clk.common.hw, + [CLK_PLL_PERIPH0] = &pll_periph0_clk.hw, + [CLK_PLL_PERIPH0_DIV3] = &pll_periph0_div3_clk.hw, + [CLK_PLL_VIDEO0_4X] = &pll_video0_4x_clk.common.hw, + [CLK_PLL_VIDEO0_2X] = &pll_video0_2x_clk.hw, + [CLK_PLL_VIDEO0] = &pll_video0_clk.hw, + [CLK_PLL_VIDEO1_4X] = &pll_video1_4x_clk.common.hw, + [CLK_PLL_VIDEO1_2X] = &pll_video1_2x_clk.hw, + [CLK_PLL_VIDEO1] = &pll_video1_clk.hw, + [CLK_PLL_VE] = &pll_ve_clk.common.hw, + [CLK_PLL_AUDIO0_4X] = &pll_audio0_4x_clk.common.hw, + [CLK_PLL_AUDIO0_2X] = &pll_audio0_2x_clk.hw, + [CLK_PLL_AUDIO0] = &pll_audio0_clk.hw, + [CLK_PLL_AUDIO1] = &pll_audio1_clk.common.hw, + [CLK_PLL_AUDIO1_DIV2] = &pll_audio1_div2_clk.common.hw, + [CLK_PLL_AUDIO1_DIV5] = &pll_audio1_div5_clk.common.hw, + [CLK_CPUX] = &cpux_clk.common.hw, + [CLK_CPUX_AXI] = &cpux_axi_clk.common.hw, + [CLK_CPUX_APB] = &cpux_apb_clk.common.hw, + [CLK_PSI_AHB] = &psi_ahb_clk.common.hw, + [CLK_APB0] = &apb0_clk.common.hw, + [CLK_APB1] = &apb1_clk.common.hw, + [CLK_MBUS] = &mbus_clk.hw, + [CLK_DE] = &de_clk.common.hw, + [CLK_BUS_DE] = &bus_de_clk.common.hw, + [CLK_DI] = &di_clk.common.hw, + [CLK_BUS_DI] = &bus_di_clk.common.hw, + [CLK_G2D] = &g2d_clk.common.hw, + [CLK_BUS_G2D] = &bus_g2d_clk.common.hw, + [CLK_CE] = &ce_clk.common.hw, + [CLK_BUS_CE] = &bus_ce_clk.common.hw, + [CLK_VE] = &ve_clk.common.hw, + [CLK_BUS_VE] = &bus_ve_clk.common.hw, + [CLK_BUS_DMA] = &bus_dma_clk.common.hw, + [CLK_BUS_MSGBOX0] = &bus_msgbox0_clk.common.hw, + [CLK_BUS_MSGBOX1] = &bus_msgbox1_clk.common.hw, + [CLK_BUS_MSGBOX2] = &bus_msgbox2_clk.common.hw, + [CLK_BUS_SPINLOCK] = &bus_spinlock_clk.common.hw, + [CLK_BUS_HSTIMER] = &bus_hstimer_clk.common.hw, + [CLK_AVS] = &avs_clk.common.hw, + [CLK_BUS_DBG] = &bus_dbg_clk.common.hw, + [CLK_BUS_PWM] = &bus_pwm_clk.common.hw, + [CLK_BUS_IOMMU] = &bus_iommu_clk.common.hw, + [CLK_DRAM] = &dram_clk.common.hw, + [CLK_MBUS_DMA] = &mbus_dma_clk.common.hw, + [CLK_MBUS_VE] = &mbus_ve_clk.common.hw, + [CLK_MBUS_CE] = &mbus_ce_clk.common.hw, + [CLK_MBUS_TVIN] = &mbus_tvin_clk.common.hw, + [CLK_MBUS_CSI] = &mbus_csi_clk.common.hw, + [CLK_MBUS_G2D] = &mbus_g2d_clk.common.hw, + [CLK_MBUS_RISCV] = &mbus_riscv_clk.common.hw, + [CLK_BUS_DRAM] = &bus_dram_clk.common.hw, + [CLK_MMC0] = &mmc0_clk.common.hw, + [CLK_MMC1] = &mmc1_clk.common.hw, + [CLK_MMC2] = &mmc2_clk.common.hw, + [CLK_BUS_MMC0] = &bus_mmc0_clk.common.hw, + [CLK_BUS_MMC1] = &bus_mmc1_clk.common.hw, + [CLK_BUS_MMC2] = &bus_mmc2_clk.common.hw, + [CLK_BUS_UART0] = &bus_uart0_clk.common.hw, + [CLK_BUS_UART1] = &bus_uart1_clk.common.hw, + [CLK_BUS_UART2] = &bus_uart2_clk.common.hw, + [CLK_BUS_UART3] = &bus_uart3_clk.common.hw, + [CLK_BUS_UART4] = &bus_uart4_clk.common.hw, + [CLK_BUS_UART5] = &bus_uart5_clk.common.hw, + [CLK_BUS_I2C0] = &bus_i2c0_clk.common.hw, + [CLK_BUS_I2C1] = &bus_i2c1_clk.common.hw, + [CLK_BUS_I2C2] = &bus_i2c2_clk.common.hw, + [CLK_BUS_I2C3] = &bus_i2c3_clk.common.hw, + [CLK_SPI0] = &spi0_clk.common.hw, + [CLK_SPI1] = &spi1_clk.common.hw, + [CLK_BUS_SPI0] = &bus_spi0_clk.common.hw, + [CLK_BUS_SPI1] = &bus_spi1_clk.common.hw, + [CLK_EMAC_25M] = &emac_25M_clk.common.hw, + [CLK_BUS_EMAC] = &bus_emac_clk.common.hw, + [CLK_IR_TX] = &ir_tx_clk.common.hw, + [CLK_BUS_IR_TX] = &bus_ir_tx_clk.common.hw, + [CLK_BUS_GPADC] = &bus_gpadc_clk.common.hw, + [CLK_BUS_THS] = &bus_ths_clk.common.hw, + [CLK_I2S0] = &i2s0_clk.common.hw, + [CLK_I2S1] = &i2s1_clk.common.hw, + [CLK_I2S2] = &i2s2_clk.common.hw, + [CLK_I2S2_ASRC] = &i2s2_asrc_clk.common.hw, + [CLK_BUS_I2S0] = &bus_i2s0_clk.common.hw, + [CLK_BUS_I2S1] = &bus_i2s1_clk.common.hw, + [CLK_BUS_I2S2] = &bus_i2s2_clk.common.hw, + [CLK_SPDIF_TX] = &spdif_tx_clk.common.hw, + [CLK_SPDIF_RX] = &spdif_rx_clk.common.hw, + [CLK_BUS_SPDIF] = &bus_spdif_clk.common.hw, + [CLK_DMIC] = &dmic_clk.common.hw, + [CLK_BUS_DMIC] = &bus_dmic_clk.common.hw, + [CLK_AUDIO_DAC] = &audio_dac_clk.common.hw, + [CLK_AUDIO_ADC] = &audio_adc_clk.common.hw, + [CLK_BUS_AUDIO] = &bus_audio_clk.common.hw, + [CLK_USB_OHCI0] = &usb_ohci0_clk.common.hw, + [CLK_USB_OHCI1] = &usb_ohci1_clk.common.hw, + [CLK_BUS_OHCI0] = &bus_ohci0_clk.common.hw, + [CLK_BUS_OHCI1] = &bus_ohci1_clk.common.hw, + [CLK_BUS_EHCI0] = &bus_ehci0_clk.common.hw, + [CLK_BUS_EHCI1] = &bus_ehci1_clk.common.hw, + [CLK_BUS_OTG] = &bus_otg_clk.common.hw, + [CLK_BUS_LRADC] = &bus_lradc_clk.common.hw, + [CLK_BUS_DPSS_TOP] = &bus_dpss_top_clk.common.hw, + [CLK_HDMI_24M] = &hdmi_24M_clk.common.hw, + [CLK_HDMI_CEC_32K] = &hdmi_cec_32k_clk.common.hw, + [CLK_HDMI_CEC] = &hdmi_cec_clk.common.hw, + [CLK_BUS_HDMI] = &bus_hdmi_clk.common.hw, + [CLK_MIPI_DSI] = &mipi_dsi_clk.common.hw, + [CLK_BUS_MIPI_DSI] = &bus_mipi_dsi_clk.common.hw, + [CLK_TCON_LCD0] = &tcon_lcd0_clk.common.hw, + [CLK_BUS_TCON_LCD0] = &bus_tcon_lcd0_clk.common.hw, + [CLK_TCON_TV] = &tcon_tv_clk.common.hw, + [CLK_BUS_TCON_TV] = &bus_tcon_tv_clk.common.hw, + [CLK_TVE] = &tve_clk.common.hw, + [CLK_BUS_TVE_TOP] = &bus_tve_top_clk.common.hw, + [CLK_BUS_TVE] = &bus_tve_clk.common.hw, + [CLK_TVD] = &tvd_clk.common.hw, + [CLK_BUS_TVD_TOP] = &bus_tvd_top_clk.common.hw, + [CLK_BUS_TVD] = &bus_tvd_clk.common.hw, + [CLK_LEDC] = &ledc_clk.common.hw, + [CLK_BUS_LEDC] = &bus_ledc_clk.common.hw, + [CLK_CSI_TOP] = &csi_top_clk.common.hw, + [CLK_CSI_MCLK] = &csi_mclk_clk.common.hw, + [CLK_BUS_CSI] = &bus_csi_clk.common.hw, + [CLK_TPADC] = &tpadc_clk.common.hw, + [CLK_BUS_TPADC] = &bus_tpadc_clk.common.hw, + [CLK_BUS_TZMA] = &bus_tzma_clk.common.hw, + [CLK_DSP] = &dsp_clk.common.hw, + [CLK_BUS_DSP_CFG] = &bus_dsp_cfg_clk.common.hw, + [CLK_RISCV] = &riscv_clk.common.hw, + [CLK_RISCV_AXI] = &riscv_axi_clk.common.hw, + [CLK_BUS_RISCV_CFG] = &bus_riscv_cfg_clk.common.hw, + [CLK_FANOUT_24M] = &fanout_24M_clk.common.hw, + [CLK_FANOUT_12M] = &fanout_12M_clk.common.hw, + [CLK_FANOUT_16M] = &fanout_16M_clk.common.hw, + [CLK_FANOUT_25M] = &fanout_25M_clk.common.hw, + [CLK_FANOUT_32K] = &fanout_32k_clk.common.hw, + [CLK_FANOUT_27M] = &fanout_27M_clk.common.hw, + [CLK_FANOUT_PCLK] = &fanout_pclk_clk.common.hw, + [CLK_FANOUT0] = &fanout0_clk.common.hw, + [CLK_FANOUT1] = &fanout1_clk.common.hw, + [CLK_FANOUT2] = &fanout2_clk.common.hw, + }, +}; + +static struct ccu_reset_map sun20i_d1_ccu_resets[] = { + [RST_MBUS] = { 0x540, BIT(30) }, + [RST_BUS_DE] = { 0x60c, BIT(16) }, + [RST_BUS_DI] = { 0x62c, BIT(16) }, + [RST_BUS_G2D] = { 0x63c, BIT(16) }, + [RST_BUS_CE] = { 0x68c, BIT(16) }, + [RST_BUS_VE] = { 0x69c, BIT(16) }, + [RST_BUS_DMA] = { 0x70c, BIT(16) }, + [RST_BUS_MSGBOX0] = { 0x71c, BIT(16) }, + [RST_BUS_MSGBOX1] = { 0x71c, BIT(17) }, + [RST_BUS_MSGBOX2] = { 0x71c, BIT(18) }, + [RST_BUS_SPINLOCK] = { 0x72c, BIT(16) }, + [RST_BUS_HSTIMER] = { 0x73c, BIT(16) }, + [RST_BUS_DBG] = { 0x78c, BIT(16) }, + [RST_BUS_PWM] = { 0x7ac, BIT(16) }, + [RST_BUS_DRAM] = { 0x80c, BIT(16) }, + [RST_BUS_MMC0] = { 0x84c, BIT(16) }, + [RST_BUS_MMC1] = { 0x84c, BIT(17) }, + [RST_BUS_MMC2] = { 0x84c, BIT(18) }, + [RST_BUS_UART0] = { 0x90c, BIT(16) }, + [RST_BUS_UART1] = { 0x90c, BIT(17) }, + [RST_BUS_UART2] = { 0x90c, BIT(18) }, + [RST_BUS_UART3] = { 0x90c, BIT(19) }, + [RST_BUS_UART4] = { 0x90c, BIT(20) }, + [RST_BUS_UART5] = { 0x90c, BIT(21) }, + [RST_BUS_I2C0] = { 0x91c, BIT(16) }, + [RST_BUS_I2C1] = { 0x91c, BIT(17) }, + [RST_BUS_I2C2] = { 0x91c, BIT(18) }, + [RST_BUS_I2C3] = { 0x91c, BIT(19) }, + [RST_BUS_SPI0] = { 0x96c, BIT(16) }, + [RST_BUS_SPI1] = { 0x96c, BIT(17) }, + [RST_BUS_EMAC] = { 0x97c, BIT(16) }, + [RST_BUS_IR_TX] = { 0x9cc, BIT(16) }, + [RST_BUS_GPADC] = { 0x9ec, BIT(16) }, + [RST_BUS_THS] = { 0x9fc, BIT(16) }, + [RST_BUS_I2S0] = { 0xa20, BIT(16) }, + [RST_BUS_I2S1] = { 0xa20, BIT(17) }, + [RST_BUS_I2S2] = { 0xa20, BIT(18) }, + [RST_BUS_SPDIF] = { 0xa2c, BIT(16) }, + [RST_BUS_DMIC] = { 0xa4c, BIT(16) }, + [RST_BUS_AUDIO] = { 0xa5c, BIT(16) }, + [RST_USB_PHY0] = { 0xa70, BIT(30) }, + [RST_USB_PHY1] = { 0xa74, BIT(30) }, + [RST_BUS_OHCI0] = { 0xa8c, BIT(16) }, + [RST_BUS_OHCI1] = { 0xa8c, BIT(17) }, + [RST_BUS_EHCI0] = { 0xa8c, BIT(20) }, + [RST_BUS_EHCI1] = { 0xa8c, BIT(21) }, + [RST_BUS_OTG] = { 0xa8c, BIT(24) }, + [RST_BUS_LRADC] = { 0xa9c, BIT(16) }, + [RST_BUS_DPSS_TOP] = { 0xabc, BIT(16) }, + [RST_BUS_HDMI_MAIN] = { 0xb1c, BIT(16) }, + [RST_BUS_HDMI_SUB] = { 0xb1c, BIT(17) }, + [RST_BUS_MIPI_DSI] = { 0xb4c, BIT(16) }, + [RST_BUS_TCON_LCD0] = { 0xb7c, BIT(16) }, + [RST_BUS_TCON_TV] = { 0xb9c, BIT(16) }, + [RST_BUS_LVDS0] = { 0xbac, BIT(16) }, + [RST_BUS_TVE_TOP] = { 0xbbc, BIT(16) }, + [RST_BUS_TVE] = { 0xbbc, BIT(17) }, + [RST_BUS_TVD_TOP] = { 0xbdc, BIT(16) }, + [RST_BUS_TVD] = { 0xbdc, BIT(17) }, + [RST_BUS_LEDC] = { 0xbfc, BIT(16) }, + [RST_BUS_CSI] = { 0xc1c, BIT(16) }, + [RST_BUS_TPADC] = { 0xc5c, BIT(16) }, + [RST_DSP] = { 0xc7c, BIT(16) }, + [RST_BUS_DSP_CFG] = { 0xc7c, BIT(17) }, + [RST_BUS_DSP_DBG] = { 0xc7c, BIT(18) }, + [RST_BUS_RISCV_CFG] = { 0xd0c, BIT(16) }, +}; + +static const struct sunxi_ccu_desc sun20i_d1_ccu_desc = { + .ccu_clks = sun20i_d1_ccu_clks, + .num_ccu_clks = ARRAY_SIZE(sun20i_d1_ccu_clks), + + .hw_clks = &sun20i_d1_hw_clks, + + .resets = sun20i_d1_ccu_resets, + .num_resets = ARRAY_SIZE(sun20i_d1_ccu_resets), +}; + +static const u32 pll_regs[] = { + SUN20I_D1_PLL_CPUX_REG, + SUN20I_D1_PLL_DDR0_REG, + SUN20I_D1_PLL_PERIPH0_REG, + SUN20I_D1_PLL_VIDEO0_REG, + SUN20I_D1_PLL_VIDEO1_REG, + SUN20I_D1_PLL_VE_REG, + SUN20I_D1_PLL_AUDIO0_REG, + SUN20I_D1_PLL_AUDIO1_REG, +}; + +static const u32 pll_video_regs[] = { + SUN20I_D1_PLL_VIDEO0_REG, + SUN20I_D1_PLL_VIDEO1_REG, +}; + +static struct ccu_mux_nb sun20i_d1_riscv_nb = { + .common = &riscv_clk.common, + .cm = &riscv_clk.mux, + .delay_us = 1, + .bypass_index = 4, /* index of pll-periph0 */ +}; + +static int sun20i_d1_ccu_probe(struct platform_device *pdev) +{ + void __iomem *reg; + u32 val; + int i, ret; + + reg = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(reg)) + return PTR_ERR(reg); + + /* Enable the enable, LDO, and lock bits on all PLLs. */ + for (i = 0; i < ARRAY_SIZE(pll_regs); i++) { + val = readl(reg + pll_regs[i]); + val |= BIT(31) | BIT(30) | BIT(29); + writel(val, reg + pll_regs[i]); + } + + /* Force PLL_CPUX factor M to 0. */ + val = readl(reg + SUN20I_D1_PLL_CPUX_REG); + val &= ~GENMASK(1, 0); + writel(val, reg + SUN20I_D1_PLL_CPUX_REG); + + /* + * Force the output divider of video PLLs to 0. + * + * See the comment before pll-video0 definition for the reason. + */ + for (i = 0; i < ARRAY_SIZE(pll_video_regs); i++) { + val = readl(reg + pll_video_regs[i]); + val &= ~BIT(0); + writel(val, reg + pll_video_regs[i]); + } + + /* Enforce m1 = 0, m0 = 0 for PLL_AUDIO0 */ + val = readl(reg + SUN20I_D1_PLL_AUDIO0_REG); + val &= ~BIT(1) | BIT(0); + writel(val, reg + SUN20I_D1_PLL_AUDIO0_REG); + + /* Force fanout-27M factor N to 0. */ + val = readl(reg + SUN20I_D1_FANOUT_27M_REG); + val &= ~GENMASK(9, 8); + writel(val, reg + SUN20I_D1_FANOUT_27M_REG); + + ret = devm_sunxi_ccu_probe(&pdev->dev, reg, &sun20i_d1_ccu_desc); + if (ret) + return ret; + + /* Reparent CPU during PLL CPUX rate changes */ + ccu_mux_notifier_register(pll_cpux_clk.common.hw.clk, + &sun20i_d1_riscv_nb); + + return 0; +} + +static const struct of_device_id sun20i_d1_ccu_ids[] = { + { .compatible = "allwinner,sun20i-d1-ccu" }, + { } +}; + +static struct platform_driver sun20i_d1_ccu_driver = { + .probe = sun20i_d1_ccu_probe, + .driver = { + .name = "sun20i-d1-ccu", + .suppress_bind_attrs = true, + .of_match_table = sun20i_d1_ccu_ids, + }, +}; +module_platform_driver(sun20i_d1_ccu_driver); + +MODULE_IMPORT_NS(SUNXI_CCU); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/sunxi-ng/ccu-sun20i-d1.h b/drivers/clk/sunxi-ng/ccu-sun20i-d1.h new file mode 100644 index 000000000000..e303176f0d4e --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu-sun20i-d1.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2020 frank@allwinnertech.com + * Copyright (C) 2021 Samuel Holland + */ + +#ifndef _CCU_SUN20I_D1_H_ +#define _CCU_SUN20I_D1_H_ + +#include +#include + +#define CLK_NUMBER (CLK_FANOUT2 + 1) + +#endif /* _CCU_SUN20I_D1_H_ */ diff --git a/drivers/clk/sunxi-ng/ccu-sun4i-a10.c b/drivers/clk/sunxi-ng/ccu-sun4i-a10.c index bd9a8782fec3..c19828f1aa0f 100644 --- a/drivers/clk/sunxi-ng/ccu-sun4i-a10.c +++ b/drivers/clk/sunxi-ng/ccu-sun4i-a10.c @@ -7,7 +7,9 @@ #include #include -#include +#include +#include +#include #include "ccu_common.h" #include "ccu_reset.h" @@ -1425,18 +1427,19 @@ static const struct sunxi_ccu_desc sun7i_a20_ccu_desc = { .num_resets = ARRAY_SIZE(sunxi_a10_a20_ccu_resets), }; -static void __init sun4i_ccu_init(struct device_node *node, - const struct sunxi_ccu_desc *desc) +static int sun4i_a10_ccu_probe(struct platform_device *pdev) { + const struct sunxi_ccu_desc *desc; void __iomem *reg; u32 val; - reg = of_io_request_and_map(node, 0, of_node_full_name(node)); - if (IS_ERR(reg)) { - pr_err("%s: Could not map the clock registers\n", - of_node_full_name(node)); - return; - } + desc = of_device_get_match_data(&pdev->dev); + if (!desc) + return -EINVAL; + + reg = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(reg)) + return PTR_ERR(reg); val = readl(reg + SUN4I_PLL_AUDIO_REG); @@ -1464,19 +1467,30 @@ static void __init sun4i_ccu_init(struct device_node *node, val &= ~GENMASK(7, 6); writel(val | (2 << 6), reg + SUN4I_AHB_REG); - of_sunxi_ccu_probe(node, reg, desc); + return devm_sunxi_ccu_probe(&pdev->dev, reg, desc); } -static void __init sun4i_a10_ccu_setup(struct device_node *node) -{ - sun4i_ccu_init(node, &sun4i_a10_ccu_desc); -} -CLK_OF_DECLARE(sun4i_a10_ccu, "allwinner,sun4i-a10-ccu", - sun4i_a10_ccu_setup); +static const struct of_device_id sun4i_a10_ccu_ids[] = { + { + .compatible = "allwinner,sun4i-a10-ccu", + .data = &sun4i_a10_ccu_desc, + }, + { + .compatible = "allwinner,sun7i-a20-ccu", + .data = &sun7i_a20_ccu_desc, + }, + { } +}; -static void __init sun7i_a20_ccu_setup(struct device_node *node) -{ - sun4i_ccu_init(node, &sun7i_a20_ccu_desc); -} -CLK_OF_DECLARE(sun7i_a20_ccu, "allwinner,sun7i-a20-ccu", - sun7i_a20_ccu_setup); +static struct platform_driver sun4i_a10_ccu_driver = { + .probe = sun4i_a10_ccu_probe, + .driver = { + .name = "sun4i-a10-ccu", + .suppress_bind_attrs = true, + .of_match_table = sun4i_a10_ccu_ids, + }, +}; +module_platform_driver(sun4i_a10_ccu_driver); + +MODULE_IMPORT_NS(SUNXI_CCU); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-a100-r.c b/drivers/clk/sunxi-ng/ccu-sun50i-a100-r.c index 804729e0a208..fddd6c877cec 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-a100-r.c +++ b/drivers/clk/sunxi-ng/ccu-sun50i-a100-r.c @@ -5,7 +5,6 @@ #include #include -#include #include #include "ccu_common.h" @@ -213,3 +212,6 @@ static struct platform_driver sun50i_a100_r_ccu_driver = { }, }; module_platform_driver(sun50i_a100_r_ccu_driver); + +MODULE_IMPORT_NS(SUNXI_CCU); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-a100.c b/drivers/clk/sunxi-ng/ccu-sun50i-a100.c index 1d475d5a3d91..5f93b5526e13 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-a100.c +++ b/drivers/clk/sunxi-ng/ccu-sun50i-a100.c @@ -6,7 +6,6 @@ #include #include #include -#include #include #include "ccu_common.h" @@ -1275,3 +1274,6 @@ static struct platform_driver sun50i_a100_ccu_driver = { }, }; module_platform_driver(sun50i_a100_ccu_driver); + +MODULE_IMPORT_NS(SUNXI_CCU); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-a64.c b/drivers/clk/sunxi-ng/ccu-sun50i-a64.c index a8c5a92b7d0c..41519185600a 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-a64.c +++ b/drivers/clk/sunxi-ng/ccu-sun50i-a64.c @@ -5,7 +5,7 @@ #include #include -#include +#include #include #include "ccu_common.h" @@ -980,4 +980,7 @@ static struct platform_driver sun50i_a64_ccu_driver = { .of_match_table = sun50i_a64_ccu_ids, }, }; -builtin_platform_driver(sun50i_a64_ccu_driver); +module_platform_driver(sun50i_a64_ccu_driver); + +MODULE_IMPORT_NS(SUNXI_CCU); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c b/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c index f30d7eb5424d..712e103382d8 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c +++ b/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c @@ -4,7 +4,8 @@ */ #include -#include +#include +#include #include #include "ccu_common.h" @@ -221,30 +222,43 @@ static const struct sunxi_ccu_desc sun50i_h616_r_ccu_desc = { .num_resets = ARRAY_SIZE(sun50i_h616_r_ccu_resets), }; -static void __init sunxi_r_ccu_init(struct device_node *node, - const struct sunxi_ccu_desc *desc) +static int sun50i_h6_r_ccu_probe(struct platform_device *pdev) { + const struct sunxi_ccu_desc *desc; void __iomem *reg; - reg = of_io_request_and_map(node, 0, of_node_full_name(node)); - if (IS_ERR(reg)) { - pr_err("%pOF: Could not map the clock registers\n", node); - return; - } + desc = of_device_get_match_data(&pdev->dev); + if (!desc) + return -EINVAL; - of_sunxi_ccu_probe(node, reg, desc); + reg = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(reg)) + return PTR_ERR(reg); + + return devm_sunxi_ccu_probe(&pdev->dev, reg, desc); } -static void __init sun50i_h6_r_ccu_setup(struct device_node *node) -{ - sunxi_r_ccu_init(node, &sun50i_h6_r_ccu_desc); -} -CLK_OF_DECLARE(sun50i_h6_r_ccu, "allwinner,sun50i-h6-r-ccu", - sun50i_h6_r_ccu_setup); +static const struct of_device_id sun50i_h6_r_ccu_ids[] = { + { + .compatible = "allwinner,sun50i-h6-r-ccu", + .data = &sun50i_h6_r_ccu_desc, + }, + { + .compatible = "allwinner,sun50i-h616-r-ccu", + .data = &sun50i_h616_r_ccu_desc, + }, + { } +}; -static void __init sun50i_h616_r_ccu_setup(struct device_node *node) -{ - sunxi_r_ccu_init(node, &sun50i_h616_r_ccu_desc); -} -CLK_OF_DECLARE(sun50i_h616_r_ccu, "allwinner,sun50i-h616-r-ccu", - sun50i_h616_r_ccu_setup); +static struct platform_driver sun50i_h6_r_ccu_driver = { + .probe = sun50i_h6_r_ccu_probe, + .driver = { + .name = "sun50i-h6-r-ccu", + .suppress_bind_attrs = true, + .of_match_table = sun50i_h6_r_ccu_ids, + }, +}; +module_platform_driver(sun50i_h6_r_ccu_driver); + +MODULE_IMPORT_NS(SUNXI_CCU); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h6.c b/drivers/clk/sunxi-ng/ccu-sun50i-h6.c index e5672c10d065..1a5e418923f6 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-h6.c +++ b/drivers/clk/sunxi-ng/ccu-sun50i-h6.c @@ -5,7 +5,7 @@ #include #include -#include +#include #include #include "ccu_common.h" @@ -1254,4 +1254,7 @@ static struct platform_driver sun50i_h6_ccu_driver = { .of_match_table = sun50i_h6_ccu_ids, }, }; -builtin_platform_driver(sun50i_h6_ccu_driver); +module_platform_driver(sun50i_h6_ccu_driver); + +MODULE_IMPORT_NS(SUNXI_CCU); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h616.c b/drivers/clk/sunxi-ng/ccu-sun50i-h616.c index 22eb18079a15..49a2474cf314 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-h616.c +++ b/drivers/clk/sunxi-ng/ccu-sun50i-h616.c @@ -7,7 +7,7 @@ #include #include -#include +#include #include #include "ccu_common.h" @@ -1082,17 +1082,15 @@ static const u32 usb2_clk_regs[] = { SUN50I_H616_USB3_CLK_REG, }; -static void __init sun50i_h616_ccu_setup(struct device_node *node) +static int sun50i_h616_ccu_probe(struct platform_device *pdev) { void __iomem *reg; u32 val; int i; - reg = of_io_request_and_map(node, 0, of_node_full_name(node)); - if (IS_ERR(reg)) { - pr_err("%pOF: Could not map clock registers\n", node); - return; - } + reg = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(reg)) + return PTR_ERR(reg); /* Enable the lock bits and the output enable bits on all PLLs */ for (i = 0; i < ARRAY_SIZE(pll_regs); i++) { @@ -1141,8 +1139,23 @@ static void __init sun50i_h616_ccu_setup(struct device_node *node) val |= BIT(24); writel(val, reg + SUN50I_H616_HDMI_CEC_CLK_REG); - of_sunxi_ccu_probe(node, reg, &sun50i_h616_ccu_desc); + return devm_sunxi_ccu_probe(&pdev->dev, reg, &sun50i_h616_ccu_desc); } -CLK_OF_DECLARE(sun50i_h616_ccu, "allwinner,sun50i-h616-ccu", - sun50i_h616_ccu_setup); +static const struct of_device_id sun50i_h616_ccu_ids[] = { + { .compatible = "allwinner,sun50i-h616-ccu" }, + { } +}; + +static struct platform_driver sun50i_h616_ccu_driver = { + .probe = sun50i_h616_ccu_probe, + .driver = { + .name = "sun50i-h616-ccu", + .suppress_bind_attrs = true, + .of_match_table = sun50i_h616_ccu_ids, + }, +}; +module_platform_driver(sun50i_h616_ccu_driver); + +MODULE_IMPORT_NS(SUNXI_CCU); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/sunxi-ng/ccu-sun6i-a31.c b/drivers/clk/sunxi-ng/ccu-sun6i-a31.c index 3df5c0b41580..0762deffb33c 100644 --- a/drivers/clk/sunxi-ng/ccu-sun6i-a31.c +++ b/drivers/clk/sunxi-ng/ccu-sun6i-a31.c @@ -9,7 +9,8 @@ #include #include -#include +#include +#include #include "ccu_common.h" #include "ccu_reset.h" @@ -1226,16 +1227,15 @@ static struct ccu_mux_nb sun6i_a31_cpu_nb = { .bypass_index = 1, /* index of 24 MHz oscillator */ }; -static void __init sun6i_a31_ccu_setup(struct device_node *node) +static int sun6i_a31_ccu_probe(struct platform_device *pdev) { void __iomem *reg; + int ret; u32 val; - reg = of_io_request_and_map(node, 0, of_node_full_name(node)); - if (IS_ERR(reg)) { - pr_err("%pOF: Could not map the clock registers\n", node); - return; - } + reg = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(reg)) + return PTR_ERR(reg); /* Force the PLL-Audio-1x divider to 1 */ val = readl(reg + SUN6I_A31_PLL_AUDIO_REG); @@ -1257,10 +1257,30 @@ static void __init sun6i_a31_ccu_setup(struct device_node *node) val |= 0x3 << 12; writel(val, reg + SUN6I_A31_AHB1_REG); - of_sunxi_ccu_probe(node, reg, &sun6i_a31_ccu_desc); + ret = devm_sunxi_ccu_probe(&pdev->dev, reg, &sun6i_a31_ccu_desc); + if (ret) + return ret; ccu_mux_notifier_register(pll_cpu_clk.common.hw.clk, &sun6i_a31_cpu_nb); + + return 0; } -CLK_OF_DECLARE(sun6i_a31_ccu, "allwinner,sun6i-a31-ccu", - sun6i_a31_ccu_setup); + +static const struct of_device_id sun6i_a31_ccu_ids[] = { + { .compatible = "allwinner,sun6i-a31-ccu" }, + { } +}; + +static struct platform_driver sun6i_a31_ccu_driver = { + .probe = sun6i_a31_ccu_probe, + .driver = { + .name = "sun6i-a31-ccu", + .suppress_bind_attrs = true, + .of_match_table = sun6i_a31_ccu_ids, + }, +}; +module_platform_driver(sun6i_a31_ccu_driver); + +MODULE_IMPORT_NS(SUNXI_CCU); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-a23.c b/drivers/clk/sunxi-ng/ccu-sun8i-a23.c index 577bb235d658..e80cc3864e44 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-a23.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-a23.c @@ -5,7 +5,8 @@ #include #include -#include +#include +#include #include "ccu_common.h" #include "ccu_reset.h" @@ -724,16 +725,14 @@ static const struct sunxi_ccu_desc sun8i_a23_ccu_desc = { .num_resets = ARRAY_SIZE(sun8i_a23_ccu_resets), }; -static void __init sun8i_a23_ccu_setup(struct device_node *node) +static int sun8i_a23_ccu_probe(struct platform_device *pdev) { void __iomem *reg; u32 val; - reg = of_io_request_and_map(node, 0, of_node_full_name(node)); - if (IS_ERR(reg)) { - pr_err("%pOF: Could not map the clock registers\n", node); - return; - } + reg = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(reg)) + return PTR_ERR(reg); /* Force the PLL-Audio-1x divider to 1 */ val = readl(reg + SUN8I_A23_PLL_AUDIO_REG); @@ -745,7 +744,23 @@ static void __init sun8i_a23_ccu_setup(struct device_node *node) val &= ~BIT(16); writel(val, reg + SUN8I_A23_PLL_MIPI_REG); - of_sunxi_ccu_probe(node, reg, &sun8i_a23_ccu_desc); + return devm_sunxi_ccu_probe(&pdev->dev, reg, &sun8i_a23_ccu_desc); } -CLK_OF_DECLARE(sun8i_a23_ccu, "allwinner,sun8i-a23-ccu", - sun8i_a23_ccu_setup); + +static const struct of_device_id sun8i_a23_ccu_ids[] = { + { .compatible = "allwinner,sun8i-a23-ccu" }, + { } +}; + +static struct platform_driver sun8i_a23_ccu_driver = { + .probe = sun8i_a23_ccu_probe, + .driver = { + .name = "sun8i-a23-ccu", + .suppress_bind_attrs = true, + .of_match_table = sun8i_a23_ccu_ids, + }, +}; +module_platform_driver(sun8i_a23_ccu_driver); + +MODULE_IMPORT_NS(SUNXI_CCU); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-a33.c b/drivers/clk/sunxi-ng/ccu-sun8i-a33.c index 8f65cd03f5ac..d12878a1ba9e 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-a33.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-a33.c @@ -5,7 +5,8 @@ #include #include -#include +#include +#include #include "ccu_common.h" #include "ccu_reset.h" @@ -784,16 +785,15 @@ static struct ccu_mux_nb sun8i_a33_cpu_nb = { .bypass_index = 1, /* index of 24 MHz oscillator */ }; -static void __init sun8i_a33_ccu_setup(struct device_node *node) +static int sun8i_a33_ccu_probe(struct platform_device *pdev) { void __iomem *reg; + int ret; u32 val; - reg = of_io_request_and_map(node, 0, of_node_full_name(node)); - if (IS_ERR(reg)) { - pr_err("%pOF: Could not map the clock registers\n", node); - return; - } + reg = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(reg)) + return PTR_ERR(reg); /* Force the PLL-Audio-1x divider to 1 */ val = readl(reg + SUN8I_A33_PLL_AUDIO_REG); @@ -805,7 +805,9 @@ static void __init sun8i_a33_ccu_setup(struct device_node *node) val &= ~BIT(16); writel(val, reg + SUN8I_A33_PLL_MIPI_REG); - of_sunxi_ccu_probe(node, reg, &sun8i_a33_ccu_desc); + ret = devm_sunxi_ccu_probe(&pdev->dev, reg, &sun8i_a33_ccu_desc); + if (ret) + return ret; /* Gate then ungate PLL CPU after any rate changes */ ccu_pll_notifier_register(&sun8i_a33_pll_cpu_nb); @@ -813,6 +815,24 @@ static void __init sun8i_a33_ccu_setup(struct device_node *node) /* Reparent CPU during PLL CPU rate changes */ ccu_mux_notifier_register(pll_cpux_clk.common.hw.clk, &sun8i_a33_cpu_nb); + + return 0; } -CLK_OF_DECLARE(sun8i_a33_ccu, "allwinner,sun8i-a33-ccu", - sun8i_a33_ccu_setup); + +static const struct of_device_id sun8i_a33_ccu_ids[] = { + { .compatible = "allwinner,sun8i-a33-ccu" }, + { } +}; + +static struct platform_driver sun8i_a33_ccu_driver = { + .probe = sun8i_a33_ccu_probe, + .driver = { + .name = "sun8i-a33-ccu", + .suppress_bind_attrs = true, + .of_match_table = sun8i_a33_ccu_ids, + }, +}; +module_platform_driver(sun8i_a33_ccu_driver); + +MODULE_IMPORT_NS(SUNXI_CCU); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-a83t.c b/drivers/clk/sunxi-ng/ccu-sun8i-a83t.c index 3c310aea8cfa..76cbd9e9e89f 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-a83t.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-a83t.c @@ -5,7 +5,7 @@ #include #include -#include +#include #include #include "ccu_common.h" @@ -920,4 +920,7 @@ static struct platform_driver sun8i_a83t_ccu_driver = { .of_match_table = sun8i_a83t_ccu_ids, }, }; -builtin_platform_driver(sun8i_a83t_ccu_driver); +module_platform_driver(sun8i_a83t_ccu_driver); + +MODULE_IMPORT_NS(SUNXI_CCU); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-de2.c b/drivers/clk/sunxi-ng/ccu-sun8i-de2.c index 573b5051d305..e7e3ddf4a227 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-de2.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-de2.c @@ -5,8 +5,8 @@ #include #include -#include -#include +#include +#include #include #include @@ -394,4 +394,7 @@ static struct platform_driver sunxi_de2_clk_driver = { .of_match_table = sunxi_de2_clk_ids, }, }; -builtin_platform_driver(sunxi_de2_clk_driver); +module_platform_driver(sunxi_de2_clk_driver); + +MODULE_IMPORT_NS(SUNXI_CCU); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-h3.c b/drivers/clk/sunxi-ng/ccu-sun8i-h3.c index d2fc2903787d..e058cf691aea 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-h3.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-h3.c @@ -5,7 +5,9 @@ #include #include -#include +#include +#include +#include #include "ccu_common.h" #include "ccu_reset.h" @@ -1137,24 +1139,29 @@ static struct ccu_mux_nb sun8i_h3_cpu_nb = { .bypass_index = 1, /* index of 24 MHz oscillator */ }; -static void __init sunxi_h3_h5_ccu_init(struct device_node *node, - const struct sunxi_ccu_desc *desc) +static int sun8i_h3_ccu_probe(struct platform_device *pdev) { + const struct sunxi_ccu_desc *desc; void __iomem *reg; + int ret; u32 val; - reg = of_io_request_and_map(node, 0, of_node_full_name(node)); - if (IS_ERR(reg)) { - pr_err("%pOF: Could not map the clock registers\n", node); - return; - } + desc = of_device_get_match_data(&pdev->dev); + if (!desc) + return -EINVAL; + + reg = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(reg)) + return PTR_ERR(reg); /* Force the PLL-Audio-1x divider to 1 */ val = readl(reg + SUN8I_H3_PLL_AUDIO_REG); val &= ~GENMASK(19, 16); writel(val | (0 << 16), reg + SUN8I_H3_PLL_AUDIO_REG); - of_sunxi_ccu_probe(node, reg, desc); + ret = devm_sunxi_ccu_probe(&pdev->dev, reg, desc); + if (ret) + return ret; /* Gate then ungate PLL CPU after any rate changes */ ccu_pll_notifier_register(&sun8i_h3_pll_cpu_nb); @@ -1162,18 +1169,31 @@ static void __init sunxi_h3_h5_ccu_init(struct device_node *node, /* Reparent CPU during PLL CPU rate changes */ ccu_mux_notifier_register(pll_cpux_clk.common.hw.clk, &sun8i_h3_cpu_nb); + + return 0; } -static void __init sun8i_h3_ccu_setup(struct device_node *node) -{ - sunxi_h3_h5_ccu_init(node, &sun8i_h3_ccu_desc); -} -CLK_OF_DECLARE(sun8i_h3_ccu, "allwinner,sun8i-h3-ccu", - sun8i_h3_ccu_setup); +static const struct of_device_id sun8i_h3_ccu_ids[] = { + { + .compatible = "allwinner,sun8i-h3-ccu", + .data = &sun8i_h3_ccu_desc, + }, + { + .compatible = "allwinner,sun50i-h5-ccu", + .data = &sun50i_h5_ccu_desc, + }, + { } +}; -static void __init sun50i_h5_ccu_setup(struct device_node *node) -{ - sunxi_h3_h5_ccu_init(node, &sun50i_h5_ccu_desc); -} -CLK_OF_DECLARE(sun50i_h5_ccu, "allwinner,sun50i-h5-ccu", - sun50i_h5_ccu_setup); +static struct platform_driver sun8i_h3_ccu_driver = { + .probe = sun8i_h3_ccu_probe, + .driver = { + .name = "sun8i-h3-ccu", + .suppress_bind_attrs = true, + .of_match_table = sun8i_h3_ccu_ids, + }, +}; +module_platform_driver(sun8i_h3_ccu_driver); + +MODULE_IMPORT_NS(SUNXI_CCU); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-r.c b/drivers/clk/sunxi-ng/ccu-sun8i-r.c index 9e754d1f754a..5b7fab832a52 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-r.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-r.c @@ -4,7 +4,8 @@ */ #include -#include +#include +#include #include #include "ccu_common.h" @@ -254,37 +255,47 @@ static const struct sunxi_ccu_desc sun50i_a64_r_ccu_desc = { .num_resets = ARRAY_SIZE(sun50i_a64_r_ccu_resets), }; -static void __init sunxi_r_ccu_init(struct device_node *node, - const struct sunxi_ccu_desc *desc) +static int sun8i_r_ccu_probe(struct platform_device *pdev) { + const struct sunxi_ccu_desc *desc; void __iomem *reg; - reg = of_io_request_and_map(node, 0, of_node_full_name(node)); - if (IS_ERR(reg)) { - pr_err("%pOF: Could not map the clock registers\n", node); - return; - } + desc = of_device_get_match_data(&pdev->dev); + if (!desc) + return -EINVAL; - of_sunxi_ccu_probe(node, reg, desc); + reg = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(reg)) + return PTR_ERR(reg); + + return devm_sunxi_ccu_probe(&pdev->dev, reg, desc); } -static void __init sun8i_a83t_r_ccu_setup(struct device_node *node) -{ - sunxi_r_ccu_init(node, &sun8i_a83t_r_ccu_desc); -} -CLK_OF_DECLARE(sun8i_a83t_r_ccu, "allwinner,sun8i-a83t-r-ccu", - sun8i_a83t_r_ccu_setup); +static const struct of_device_id sun8i_r_ccu_ids[] = { + { + .compatible = "allwinner,sun8i-a83t-r-ccu", + .data = &sun8i_a83t_r_ccu_desc, + }, + { + .compatible = "allwinner,sun8i-h3-r-ccu", + .data = &sun8i_h3_r_ccu_desc, + }, + { + .compatible = "allwinner,sun50i-a64-r-ccu", + .data = &sun50i_a64_r_ccu_desc, + }, + { } +}; -static void __init sun8i_h3_r_ccu_setup(struct device_node *node) -{ - sunxi_r_ccu_init(node, &sun8i_h3_r_ccu_desc); -} -CLK_OF_DECLARE(sun8i_h3_r_ccu, "allwinner,sun8i-h3-r-ccu", - sun8i_h3_r_ccu_setup); +static struct platform_driver sun8i_r_ccu_driver = { + .probe = sun8i_r_ccu_probe, + .driver = { + .name = "sun8i-r-ccu", + .suppress_bind_attrs = true, + .of_match_table = sun8i_r_ccu_ids, + }, +}; +module_platform_driver(sun8i_r_ccu_driver); -static void __init sun50i_a64_r_ccu_setup(struct device_node *node) -{ - sunxi_r_ccu_init(node, &sun50i_a64_r_ccu_desc); -} -CLK_OF_DECLARE(sun50i_a64_r_ccu, "allwinner,sun50i-a64-r-ccu", - sun50i_a64_r_ccu_setup); +MODULE_IMPORT_NS(SUNXI_CCU); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-r40.c b/drivers/clk/sunxi-ng/ccu-sun8i-r40.c index 8bb18d9add05..31eca0d3bc1e 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-r40.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-r40.c @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -1371,4 +1372,7 @@ static struct platform_driver sun8i_r40_ccu_driver = { .of_match_table = sun8i_r40_ccu_ids, }, }; -builtin_platform_driver(sun8i_r40_ccu_driver); +module_platform_driver(sun8i_r40_ccu_driver); + +MODULE_IMPORT_NS(SUNXI_CCU); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c b/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c index ce150f83ab54..87f87d6ea3ad 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c @@ -8,7 +8,9 @@ #include #include -#include +#include +#include +#include #include "ccu_common.h" #include "ccu_reset.h" @@ -805,38 +807,49 @@ static const struct sunxi_ccu_desc sun8i_v3_ccu_desc = { .num_resets = ARRAY_SIZE(sun8i_v3_ccu_resets), }; -static void __init sun8i_v3_v3s_ccu_init(struct device_node *node, - const struct sunxi_ccu_desc *ccu_desc) +static int sun8i_v3s_ccu_probe(struct platform_device *pdev) { + const struct sunxi_ccu_desc *desc; void __iomem *reg; u32 val; - reg = of_io_request_and_map(node, 0, of_node_full_name(node)); - if (IS_ERR(reg)) { - pr_err("%pOF: Could not map the clock registers\n", node); - return; - } + desc = of_device_get_match_data(&pdev->dev); + if (!desc) + return -EINVAL; + + reg = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(reg)) + return PTR_ERR(reg); /* Force the PLL-Audio-1x divider to 1 */ val = readl(reg + SUN8I_V3S_PLL_AUDIO_REG); val &= ~GENMASK(19, 16); writel(val, reg + SUN8I_V3S_PLL_AUDIO_REG); - of_sunxi_ccu_probe(node, reg, ccu_desc); + return devm_sunxi_ccu_probe(&pdev->dev, reg, desc); } -static void __init sun8i_v3s_ccu_setup(struct device_node *node) -{ - sun8i_v3_v3s_ccu_init(node, &sun8i_v3s_ccu_desc); -} +static const struct of_device_id sun8i_v3s_ccu_ids[] = { + { + .compatible = "allwinner,sun8i-v3-ccu", + .data = &sun8i_v3_ccu_desc, + }, + { + .compatible = "allwinner,sun8i-v3s-ccu", + .data = &sun8i_v3s_ccu_desc, + }, + { } +}; -static void __init sun8i_v3_ccu_setup(struct device_node *node) -{ - sun8i_v3_v3s_ccu_init(node, &sun8i_v3_ccu_desc); -} +static struct platform_driver sun8i_v3s_ccu_driver = { + .probe = sun8i_v3s_ccu_probe, + .driver = { + .name = "sun8i-v3s-ccu", + .suppress_bind_attrs = true, + .of_match_table = sun8i_v3s_ccu_ids, + }, +}; +module_platform_driver(sun8i_v3s_ccu_driver); -CLK_OF_DECLARE(sun8i_v3s_ccu, "allwinner,sun8i-v3s-ccu", - sun8i_v3s_ccu_setup); - -CLK_OF_DECLARE(sun8i_v3_ccu, "allwinner,sun8i-v3-ccu", - sun8i_v3_ccu_setup); +MODULE_IMPORT_NS(SUNXI_CCU); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/sunxi-ng/ccu-sun9i-a80-de.c b/drivers/clk/sunxi-ng/ccu-sun9i-a80-de.c index 3cde2610f467..f2fe0e1cc3c0 100644 --- a/drivers/clk/sunxi-ng/ccu-sun9i-a80-de.c +++ b/drivers/clk/sunxi-ng/ccu-sun9i-a80-de.c @@ -5,7 +5,7 @@ #include #include -#include +#include #include #include @@ -270,4 +270,7 @@ static struct platform_driver sun9i_a80_de_clk_driver = { .of_match_table = sun9i_a80_de_clk_ids, }, }; -builtin_platform_driver(sun9i_a80_de_clk_driver); +module_platform_driver(sun9i_a80_de_clk_driver); + +MODULE_IMPORT_NS(SUNXI_CCU); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c b/drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c index 0740e8978ae8..575ae4ccc65f 100644 --- a/drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c +++ b/drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c @@ -5,7 +5,7 @@ #include #include -#include +#include #include #include "ccu_common.h" @@ -138,4 +138,7 @@ static struct platform_driver sun9i_a80_usb_clk_driver = { .of_match_table = sun9i_a80_usb_clk_ids, }, }; -builtin_platform_driver(sun9i_a80_usb_clk_driver); +module_platform_driver(sun9i_a80_usb_clk_driver); + +MODULE_IMPORT_NS(SUNXI_CCU); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/sunxi-ng/ccu-sun9i-a80.c b/drivers/clk/sunxi-ng/ccu-sun9i-a80.c index d416af29e0d3..730fd8e28014 100644 --- a/drivers/clk/sunxi-ng/ccu-sun9i-a80.c +++ b/drivers/clk/sunxi-ng/ccu-sun9i-a80.c @@ -5,7 +5,7 @@ #include #include -#include +#include #include #include "ccu_common.h" @@ -1245,4 +1245,7 @@ static struct platform_driver sun9i_a80_ccu_driver = { .of_match_table = sun9i_a80_ccu_ids, }, }; -builtin_platform_driver(sun9i_a80_ccu_driver); +module_platform_driver(sun9i_a80_ccu_driver); + +MODULE_IMPORT_NS(SUNXI_CCU); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c b/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c index 61ad7ee91c11..ed097c4f780f 100644 --- a/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c +++ b/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c @@ -6,7 +6,8 @@ #include #include -#include +#include +#include #include "ccu_common.h" #include "ccu_reset.h" @@ -522,23 +523,24 @@ static struct ccu_mux_nb suniv_cpu_nb = { .bypass_index = 1, /* index of 24 MHz oscillator */ }; -static void __init suniv_f1c100s_ccu_setup(struct device_node *node) +static int suniv_f1c100s_ccu_probe(struct platform_device *pdev) { void __iomem *reg; + int ret; u32 val; - reg = of_io_request_and_map(node, 0, of_node_full_name(node)); - if (IS_ERR(reg)) { - pr_err("%pOF: Could not map the clock registers\n", node); - return; - } + reg = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(reg)) + return PTR_ERR(reg); /* Force the PLL-Audio-1x divider to 4 */ val = readl(reg + SUNIV_PLL_AUDIO_REG); val &= ~GENMASK(19, 16); writel(val | (3 << 16), reg + SUNIV_PLL_AUDIO_REG); - of_sunxi_ccu_probe(node, reg, &suniv_ccu_desc); + ret = devm_sunxi_ccu_probe(&pdev->dev, reg, &suniv_ccu_desc); + if (ret) + return ret; /* Gate then ungate PLL CPU after any rate changes */ ccu_pll_notifier_register(&suniv_pll_cpu_nb); @@ -546,6 +548,24 @@ static void __init suniv_f1c100s_ccu_setup(struct device_node *node) /* Reparent CPU during PLL CPU rate changes */ ccu_mux_notifier_register(pll_cpu_clk.common.hw.clk, &suniv_cpu_nb); + + return 0; } -CLK_OF_DECLARE(suniv_f1c100s_ccu, "allwinner,suniv-f1c100s-ccu", - suniv_f1c100s_ccu_setup); + +static const struct of_device_id suniv_f1c100s_ccu_ids[] = { + { .compatible = "allwinner,suniv-f1c100s-ccu" }, + { } +}; + +static struct platform_driver suniv_f1c100s_ccu_driver = { + .probe = suniv_f1c100s_ccu_probe, + .driver = { + .name = "suniv-f1c100s-ccu", + .suppress_bind_attrs = true, + .of_match_table = suniv_f1c100s_ccu_ids, + }, +}; +module_platform_driver(suniv_f1c100s_ccu_driver); + +MODULE_IMPORT_NS(SUNXI_CCU); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/sunxi-ng/ccu_common.c b/drivers/clk/sunxi-ng/ccu_common.c index 31af8b6b5286..8d28a7a079d0 100644 --- a/drivers/clk/sunxi-ng/ccu_common.c +++ b/drivers/clk/sunxi-ng/ccu_common.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include "ccu_common.h" @@ -36,6 +37,7 @@ void ccu_helper_wait_for_lock(struct ccu_common *common, u32 lock) WARN_ON(readl_relaxed_poll_timeout(addr, reg, reg & lock, 100, 70000)); } +EXPORT_SYMBOL_NS_GPL(ccu_helper_wait_for_lock, SUNXI_CCU); /* * This clock notifier is called when the frequency of a PLL clock is @@ -83,6 +85,7 @@ int ccu_pll_notifier_register(struct ccu_pll_nb *pll_nb) return clk_notifier_register(pll_nb->common->hw.clk, &pll_nb->clk_nb); } +EXPORT_SYMBOL_NS_GPL(ccu_pll_notifier_register, SUNXI_CCU); static int sunxi_ccu_probe(struct sunxi_ccu *ccu, struct device *dev, struct device_node *node, void __iomem *reg, @@ -194,6 +197,7 @@ int devm_sunxi_ccu_probe(struct device *dev, void __iomem *reg, return 0; } +EXPORT_SYMBOL_NS_GPL(devm_sunxi_ccu_probe, SUNXI_CCU); void of_sunxi_ccu_probe(struct device_node *node, void __iomem *reg, const struct sunxi_ccu_desc *desc) @@ -211,3 +215,5 @@ void of_sunxi_ccu_probe(struct device_node *node, void __iomem *reg, kfree(ccu); } } + +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/sunxi-ng/ccu_div.c b/drivers/clk/sunxi-ng/ccu_div.c index 4c297089483c..cb10a3ea23f9 100644 --- a/drivers/clk/sunxi-ng/ccu_div.c +++ b/drivers/clk/sunxi-ng/ccu_div.c @@ -141,3 +141,4 @@ const struct clk_ops ccu_div_ops = { .recalc_rate = ccu_div_recalc_rate, .set_rate = ccu_div_set_rate, }; +EXPORT_SYMBOL_NS_GPL(ccu_div_ops, SUNXI_CCU); diff --git a/drivers/clk/sunxi-ng/ccu_div.h b/drivers/clk/sunxi-ng/ccu_div.h index 6682fde6043c..948e2b0c0c3b 100644 --- a/drivers/clk/sunxi-ng/ccu_div.h +++ b/drivers/clk/sunxi-ng/ccu_div.h @@ -108,6 +108,22 @@ struct ccu_div { _shift, _width, _table, 0, \ _flags) +#define SUNXI_CCU_DIV_TABLE_HW(_struct, _name, _parent, _reg, \ + _shift, _width, \ + _table, _flags) \ + struct ccu_div _struct = { \ + .div = _SUNXI_CCU_DIV_TABLE(_shift, _width, \ + _table), \ + .common = { \ + .reg = _reg, \ + .hw.init = CLK_HW_INIT_HW(_name, \ + _parent, \ + &ccu_div_ops, \ + _flags), \ + } \ + } + + #define SUNXI_CCU_M_WITH_MUX_TABLE_GATE(_struct, _name, \ _parents, _table, \ _reg, \ @@ -166,6 +182,68 @@ struct ccu_div { SUNXI_CCU_M_WITH_GATE(_struct, _name, _parent, _reg, \ _mshift, _mwidth, 0, _flags) +#define SUNXI_CCU_M_DATA_WITH_MUX_GATE(_struct, _name, _parents, _reg, \ + _mshift, _mwidth, \ + _muxshift, _muxwidth, \ + _gate, _flags) \ + struct ccu_div _struct = { \ + .enable = _gate, \ + .div = _SUNXI_CCU_DIV(_mshift, _mwidth), \ + .mux = _SUNXI_CCU_MUX(_muxshift, _muxwidth), \ + .common = { \ + .reg = _reg, \ + .hw.init = CLK_HW_INIT_PARENTS_DATA(_name, \ + _parents, \ + &ccu_div_ops, \ + _flags), \ + }, \ + } + +#define SUNXI_CCU_M_DATA_WITH_MUX(_struct, _name, _parents, _reg, \ + _mshift, _mwidth, \ + _muxshift, _muxwidth, \ + _flags) \ + SUNXI_CCU_M_DATA_WITH_MUX_GATE(_struct, _name, _parents, _reg, \ + _mshift, _mwidth, \ + _muxshift, _muxwidth, \ + 0, _flags) + +#define SUNXI_CCU_M_HW_WITH_MUX_GATE(_struct, _name, _parents, _reg, \ + _mshift, _mwidth, _muxshift, _muxwidth, \ + _gate, _flags) \ + struct ccu_div _struct = { \ + .enable = _gate, \ + .div = _SUNXI_CCU_DIV(_mshift, _mwidth), \ + .mux = _SUNXI_CCU_MUX(_muxshift, _muxwidth), \ + .common = { \ + .reg = _reg, \ + .hw.init = CLK_HW_INIT_PARENTS_HW(_name, \ + _parents, \ + &ccu_div_ops, \ + _flags), \ + }, \ + } + +#define SUNXI_CCU_M_HWS_WITH_GATE(_struct, _name, _parent, _reg, \ + _mshift, _mwidth, _gate, \ + _flags) \ + struct ccu_div _struct = { \ + .enable = _gate, \ + .div = _SUNXI_CCU_DIV(_mshift, _mwidth), \ + .common = { \ + .reg = _reg, \ + .hw.init = CLK_HW_INIT_HWS(_name, \ + _parent, \ + &ccu_div_ops, \ + _flags), \ + }, \ + } + +#define SUNXI_CCU_M_HWS(_struct, _name, _parent, _reg, _mshift, \ + _mwidth, _flags) \ + SUNXI_CCU_M_HWS_WITH_GATE(_struct, _name, _parent, _reg, \ + _mshift, _mwidth, 0, _flags) + static inline struct ccu_div *hw_to_ccu_div(struct clk_hw *hw) { struct ccu_common *common = hw_to_ccu_common(hw); diff --git a/drivers/clk/sunxi-ng/ccu_frac.c b/drivers/clk/sunxi-ng/ccu_frac.c index 44fcded8b354..b31f3ad946d6 100644 --- a/drivers/clk/sunxi-ng/ccu_frac.c +++ b/drivers/clk/sunxi-ng/ccu_frac.c @@ -18,6 +18,7 @@ bool ccu_frac_helper_is_enabled(struct ccu_common *common, return !(readl(common->base + common->reg) & cf->enable); } +EXPORT_SYMBOL_NS_GPL(ccu_frac_helper_is_enabled, SUNXI_CCU); void ccu_frac_helper_enable(struct ccu_common *common, struct ccu_frac_internal *cf) @@ -33,6 +34,7 @@ void ccu_frac_helper_enable(struct ccu_common *common, writel(reg & ~cf->enable, common->base + common->reg); spin_unlock_irqrestore(common->lock, flags); } +EXPORT_SYMBOL_NS_GPL(ccu_frac_helper_enable, SUNXI_CCU); void ccu_frac_helper_disable(struct ccu_common *common, struct ccu_frac_internal *cf) @@ -48,6 +50,7 @@ void ccu_frac_helper_disable(struct ccu_common *common, writel(reg | cf->enable, common->base + common->reg); spin_unlock_irqrestore(common->lock, flags); } +EXPORT_SYMBOL_NS_GPL(ccu_frac_helper_disable, SUNXI_CCU); bool ccu_frac_helper_has_rate(struct ccu_common *common, struct ccu_frac_internal *cf, @@ -58,6 +61,7 @@ bool ccu_frac_helper_has_rate(struct ccu_common *common, return (cf->rates[0] == rate) || (cf->rates[1] == rate); } +EXPORT_SYMBOL_NS_GPL(ccu_frac_helper_has_rate, SUNXI_CCU); unsigned long ccu_frac_helper_read_rate(struct ccu_common *common, struct ccu_frac_internal *cf) @@ -79,6 +83,7 @@ unsigned long ccu_frac_helper_read_rate(struct ccu_common *common, return (reg & cf->select) ? cf->rates[1] : cf->rates[0]; } +EXPORT_SYMBOL_NS_GPL(ccu_frac_helper_read_rate, SUNXI_CCU); int ccu_frac_helper_set_rate(struct ccu_common *common, struct ccu_frac_internal *cf, @@ -107,3 +112,4 @@ int ccu_frac_helper_set_rate(struct ccu_common *common, return 0; } +EXPORT_SYMBOL_NS_GPL(ccu_frac_helper_set_rate, SUNXI_CCU); diff --git a/drivers/clk/sunxi-ng/ccu_gate.c b/drivers/clk/sunxi-ng/ccu_gate.c index 3d5ca092b08f..a2115a21807d 100644 --- a/drivers/clk/sunxi-ng/ccu_gate.c +++ b/drivers/clk/sunxi-ng/ccu_gate.c @@ -24,6 +24,7 @@ void ccu_gate_helper_disable(struct ccu_common *common, u32 gate) spin_unlock_irqrestore(common->lock, flags); } +EXPORT_SYMBOL_NS_GPL(ccu_gate_helper_disable, SUNXI_CCU); static void ccu_gate_disable(struct clk_hw *hw) { @@ -49,6 +50,7 @@ int ccu_gate_helper_enable(struct ccu_common *common, u32 gate) return 0; } +EXPORT_SYMBOL_NS_GPL(ccu_gate_helper_enable, SUNXI_CCU); static int ccu_gate_enable(struct clk_hw *hw) { @@ -64,6 +66,7 @@ int ccu_gate_helper_is_enabled(struct ccu_common *common, u32 gate) return readl(common->base + common->reg) & gate; } +EXPORT_SYMBOL_NS_GPL(ccu_gate_helper_is_enabled, SUNXI_CCU); static int ccu_gate_is_enabled(struct clk_hw *hw) { @@ -124,3 +127,4 @@ const struct clk_ops ccu_gate_ops = { .set_rate = ccu_gate_set_rate, .recalc_rate = ccu_gate_recalc_rate, }; +EXPORT_SYMBOL_NS_GPL(ccu_gate_ops, SUNXI_CCU); diff --git a/drivers/clk/sunxi-ng/ccu_gate.h b/drivers/clk/sunxi-ng/ccu_gate.h index c386689a952b..dc05ce06737a 100644 --- a/drivers/clk/sunxi-ng/ccu_gate.h +++ b/drivers/clk/sunxi-ng/ccu_gate.h @@ -53,7 +53,7 @@ struct ccu_gate { } /* - * The following two macros allow the re-use of the data structure + * The following macros allow the re-use of the data structure * holding the parent info. */ #define SUNXI_CCU_GATE_HWS(_struct, _name, _parent, _reg, _gate, _flags) \ @@ -68,6 +68,21 @@ struct ccu_gate { } \ } +#define SUNXI_CCU_GATE_HWS_WITH_PREDIV(_struct, _name, _parent, _reg, \ + _gate, _prediv, _flags) \ + struct ccu_gate _struct = { \ + .enable = _gate, \ + .common = { \ + .reg = _reg, \ + .prediv = _prediv, \ + .features = CCU_FEATURE_ALL_PREDIV, \ + .hw.init = CLK_HW_INIT_HWS(_name, \ + _parent, \ + &ccu_gate_ops, \ + _flags), \ + } \ + } + #define SUNXI_CCU_GATE_DATA(_struct, _name, _data, _reg, _gate, _flags) \ struct ccu_gate _struct = { \ .enable = _gate, \ @@ -81,6 +96,21 @@ struct ccu_gate { } \ } +#define SUNXI_CCU_GATE_DATA_WITH_PREDIV(_struct, _name, _parent, _reg, \ + _gate, _prediv, _flags) \ + struct ccu_gate _struct = { \ + .enable = _gate, \ + .common = { \ + .reg = _reg, \ + .prediv = _prediv, \ + .features = CCU_FEATURE_ALL_PREDIV, \ + .hw.init = CLK_HW_INIT_PARENTS_DATA(_name, \ + _parent, \ + &ccu_gate_ops, \ + _flags), \ + } \ + } + static inline struct ccu_gate *hw_to_ccu_gate(struct clk_hw *hw) { struct ccu_common *common = hw_to_ccu_common(hw); diff --git a/drivers/clk/sunxi-ng/ccu_mp.c b/drivers/clk/sunxi-ng/ccu_mp.c index 9d3a76604d94..57cf2d615148 100644 --- a/drivers/clk/sunxi-ng/ccu_mp.c +++ b/drivers/clk/sunxi-ng/ccu_mp.c @@ -245,6 +245,7 @@ const struct clk_ops ccu_mp_ops = { .recalc_rate = ccu_mp_recalc_rate, .set_rate = ccu_mp_set_rate, }; +EXPORT_SYMBOL_NS_GPL(ccu_mp_ops, SUNXI_CCU); /* * Support for MMC timing mode switching @@ -325,3 +326,4 @@ const struct clk_ops ccu_mp_mmc_ops = { .recalc_rate = ccu_mp_mmc_recalc_rate, .set_rate = ccu_mp_mmc_set_rate, }; +EXPORT_SYMBOL_NS_GPL(ccu_mp_mmc_ops, SUNXI_CCU); diff --git a/drivers/clk/sunxi-ng/ccu_mp.h b/drivers/clk/sunxi-ng/ccu_mp.h index b392e0d575b5..6e50f3728fb5 100644 --- a/drivers/clk/sunxi-ng/ccu_mp.h +++ b/drivers/clk/sunxi-ng/ccu_mp.h @@ -82,6 +82,55 @@ struct ccu_mp { _muxshift, _muxwidth, \ 0, _flags) +#define SUNXI_CCU_MP_DATA_WITH_MUX_GATE(_struct, _name, _parents, _reg, \ + _mshift, _mwidth, \ + _pshift, _pwidth, \ + _muxshift, _muxwidth, \ + _gate, _flags) \ + struct ccu_mp _struct = { \ + .enable = _gate, \ + .m = _SUNXI_CCU_DIV(_mshift, _mwidth), \ + .p = _SUNXI_CCU_DIV(_pshift, _pwidth), \ + .mux = _SUNXI_CCU_MUX(_muxshift, _muxwidth), \ + .common = { \ + .reg = _reg, \ + .hw.init = CLK_HW_INIT_PARENTS_DATA(_name, \ + _parents, \ + &ccu_mp_ops, \ + _flags), \ + } \ + } + +#define SUNXI_CCU_MP_DATA_WITH_MUX(_struct, _name, _parents, _reg, \ + _mshift, _mwidth, \ + _pshift, _pwidth, \ + _muxshift, _muxwidth, \ + _flags) \ + SUNXI_CCU_MP_DATA_WITH_MUX_GATE(_struct, _name, _parents, _reg, \ + _mshift, _mwidth, \ + _pshift, _pwidth, \ + _muxshift, _muxwidth, \ + 0, _flags) + +#define SUNXI_CCU_MP_HW_WITH_MUX_GATE(_struct, _name, _parents, _reg, \ + _mshift, _mwidth, \ + _pshift, _pwidth, \ + _muxshift, _muxwidth, \ + _gate, _flags) \ + struct ccu_mp _struct = { \ + .enable = _gate, \ + .m = _SUNXI_CCU_DIV(_mshift, _mwidth), \ + .p = _SUNXI_CCU_DIV(_pshift, _pwidth), \ + .mux = _SUNXI_CCU_MUX(_muxshift, _muxwidth), \ + .common = { \ + .reg = _reg, \ + .hw.init = CLK_HW_INIT_PARENTS_HW(_name, \ + _parents, \ + &ccu_mp_ops, \ + _flags), \ + } \ + } + static inline struct ccu_mp *hw_to_ccu_mp(struct clk_hw *hw) { struct ccu_common *common = hw_to_ccu_common(hw); diff --git a/drivers/clk/sunxi-ng/ccu_mult.c b/drivers/clk/sunxi-ng/ccu_mult.c index 7c8cf2e04e94..7bee217ef111 100644 --- a/drivers/clk/sunxi-ng/ccu_mult.c +++ b/drivers/clk/sunxi-ng/ccu_mult.c @@ -170,3 +170,4 @@ const struct clk_ops ccu_mult_ops = { .recalc_rate = ccu_mult_recalc_rate, .set_rate = ccu_mult_set_rate, }; +EXPORT_SYMBOL_NS_GPL(ccu_mult_ops, SUNXI_CCU); diff --git a/drivers/clk/sunxi-ng/ccu_mux.c b/drivers/clk/sunxi-ng/ccu_mux.c index 7d75da9a1f2e..2306a1cd83e4 100644 --- a/drivers/clk/sunxi-ng/ccu_mux.c +++ b/drivers/clk/sunxi-ng/ccu_mux.c @@ -64,6 +64,7 @@ unsigned long ccu_mux_helper_apply_prediv(struct ccu_common *common, { return parent_rate / ccu_mux_get_prediv(common, cm, parent_index); } +EXPORT_SYMBOL_NS_GPL(ccu_mux_helper_apply_prediv, SUNXI_CCU); static unsigned long ccu_mux_helper_unapply_prediv(struct ccu_common *common, struct ccu_mux_internal *cm, @@ -152,6 +153,7 @@ out: req->rate = best_rate; return 0; } +EXPORT_SYMBOL_NS_GPL(ccu_mux_helper_determine_rate, SUNXI_CCU); u8 ccu_mux_helper_get_parent(struct ccu_common *common, struct ccu_mux_internal *cm) @@ -174,6 +176,7 @@ u8 ccu_mux_helper_get_parent(struct ccu_common *common, return parent; } +EXPORT_SYMBOL_NS_GPL(ccu_mux_helper_get_parent, SUNXI_CCU); int ccu_mux_helper_set_parent(struct ccu_common *common, struct ccu_mux_internal *cm, @@ -195,6 +198,7 @@ int ccu_mux_helper_set_parent(struct ccu_common *common, return 0; } +EXPORT_SYMBOL_NS_GPL(ccu_mux_helper_set_parent, SUNXI_CCU); static void ccu_mux_disable(struct clk_hw *hw) { @@ -251,6 +255,7 @@ const struct clk_ops ccu_mux_ops = { .determine_rate = __clk_mux_determine_rate, .recalc_rate = ccu_mux_recalc_rate, }; +EXPORT_SYMBOL_NS_GPL(ccu_mux_ops, SUNXI_CCU); /* * This clock notifier is called when the frequency of the of the parent @@ -285,3 +290,4 @@ int ccu_mux_notifier_register(struct clk *clk, struct ccu_mux_nb *mux_nb) return clk_notifier_register(clk, &mux_nb->clk_nb); } +EXPORT_SYMBOL_NS_GPL(ccu_mux_notifier_register, SUNXI_CCU); diff --git a/drivers/clk/sunxi-ng/ccu_mux.h b/drivers/clk/sunxi-ng/ccu_mux.h index e31efc509b3d..2c1811a445b0 100644 --- a/drivers/clk/sunxi-ng/ccu_mux.h +++ b/drivers/clk/sunxi-ng/ccu_mux.h @@ -72,6 +72,39 @@ struct ccu_mux { SUNXI_CCU_MUX_TABLE_WITH_GATE(_struct, _name, _parents, NULL, \ _reg, _shift, _width, 0, _flags) +#define SUNXI_CCU_MUX_DATA_WITH_GATE(_struct, _name, _parents, _reg, \ + _shift, _width, _gate, _flags) \ + struct ccu_mux _struct = { \ + .enable = _gate, \ + .mux = _SUNXI_CCU_MUX(_shift, _width), \ + .common = { \ + .reg = _reg, \ + .hw.init = CLK_HW_INIT_PARENTS_DATA(_name, \ + _parents, \ + &ccu_mux_ops, \ + _flags), \ + } \ + } + +#define SUNXI_CCU_MUX_DATA(_struct, _name, _parents, _reg, \ + _shift, _width, _flags) \ + SUNXI_CCU_MUX_DATA_WITH_GATE(_struct, _name, _parents, _reg, \ + _shift, _width, 0, _flags) + +#define SUNXI_CCU_MUX_HW_WITH_GATE(_struct, _name, _parents, _reg, \ + _shift, _width, _gate, _flags) \ + struct ccu_mux _struct = { \ + .enable = _gate, \ + .mux = _SUNXI_CCU_MUX(_shift, _width), \ + .common = { \ + .reg = _reg, \ + .hw.init = CLK_HW_INIT_PARENTS_HW(_name, \ + _parents, \ + &ccu_mux_ops, \ + _flags), \ + } \ + } + static inline struct ccu_mux *hw_to_ccu_mux(struct clk_hw *hw) { struct ccu_common *common = hw_to_ccu_common(hw); diff --git a/drivers/clk/sunxi-ng/ccu_nk.c b/drivers/clk/sunxi-ng/ccu_nk.c index aee68b00f3b2..c4fb82af97e8 100644 --- a/drivers/clk/sunxi-ng/ccu_nk.c +++ b/drivers/clk/sunxi-ng/ccu_nk.c @@ -157,3 +157,4 @@ const struct clk_ops ccu_nk_ops = { .round_rate = ccu_nk_round_rate, .set_rate = ccu_nk_set_rate, }; +EXPORT_SYMBOL_NS_GPL(ccu_nk_ops, SUNXI_CCU); diff --git a/drivers/clk/sunxi-ng/ccu_nkm.c b/drivers/clk/sunxi-ng/ccu_nkm.c index b9cfee0276ea..67da2c189b53 100644 --- a/drivers/clk/sunxi-ng/ccu_nkm.c +++ b/drivers/clk/sunxi-ng/ccu_nkm.c @@ -206,3 +206,4 @@ const struct clk_ops ccu_nkm_ops = { .recalc_rate = ccu_nkm_recalc_rate, .set_rate = ccu_nkm_set_rate, }; +EXPORT_SYMBOL_NS_GPL(ccu_nkm_ops, SUNXI_CCU); diff --git a/drivers/clk/sunxi-ng/ccu_nkmp.c b/drivers/clk/sunxi-ng/ccu_nkmp.c index bda87b38c45c..39413cb0985c 100644 --- a/drivers/clk/sunxi-ng/ccu_nkmp.c +++ b/drivers/clk/sunxi-ng/ccu_nkmp.c @@ -230,3 +230,4 @@ const struct clk_ops ccu_nkmp_ops = { .round_rate = ccu_nkmp_round_rate, .set_rate = ccu_nkmp_set_rate, }; +EXPORT_SYMBOL_NS_GPL(ccu_nkmp_ops, SUNXI_CCU); diff --git a/drivers/clk/sunxi-ng/ccu_nm.c b/drivers/clk/sunxi-ng/ccu_nm.c index e6bcc0a7170c..9ca9257f4426 100644 --- a/drivers/clk/sunxi-ng/ccu_nm.c +++ b/drivers/clk/sunxi-ng/ccu_nm.c @@ -238,3 +238,4 @@ const struct clk_ops ccu_nm_ops = { .round_rate = ccu_nm_round_rate, .set_rate = ccu_nm_set_rate, }; +EXPORT_SYMBOL_NS_GPL(ccu_nm_ops, SUNXI_CCU); diff --git a/drivers/clk/sunxi-ng/ccu_phase.c b/drivers/clk/sunxi-ng/ccu_phase.c index 92ab8bd66427..e4cae2afe9db 100644 --- a/drivers/clk/sunxi-ng/ccu_phase.c +++ b/drivers/clk/sunxi-ng/ccu_phase.c @@ -121,3 +121,4 @@ const struct clk_ops ccu_phase_ops = { .get_phase = ccu_phase_get_phase, .set_phase = ccu_phase_set_phase, }; +EXPORT_SYMBOL_NS_GPL(ccu_phase_ops, SUNXI_CCU); diff --git a/drivers/clk/sunxi-ng/ccu_reset.c b/drivers/clk/sunxi-ng/ccu_reset.c index 483100e45df3..6577aa18cb01 100644 --- a/drivers/clk/sunxi-ng/ccu_reset.c +++ b/drivers/clk/sunxi-ng/ccu_reset.c @@ -75,3 +75,4 @@ const struct reset_control_ops ccu_reset_ops = { .reset = ccu_reset_reset, .status = ccu_reset_status, }; +EXPORT_SYMBOL_NS_GPL(ccu_reset_ops, SUNXI_CCU); diff --git a/drivers/clk/sunxi-ng/ccu_sdm.c b/drivers/clk/sunxi-ng/ccu_sdm.c index 79581a1c649a..41937ed0766d 100644 --- a/drivers/clk/sunxi-ng/ccu_sdm.c +++ b/drivers/clk/sunxi-ng/ccu_sdm.c @@ -20,6 +20,7 @@ bool ccu_sdm_helper_is_enabled(struct ccu_common *common, return !!(readl(common->base + sdm->tuning_reg) & sdm->tuning_enable); } +EXPORT_SYMBOL_NS_GPL(ccu_sdm_helper_is_enabled, SUNXI_CCU); void ccu_sdm_helper_enable(struct ccu_common *common, struct ccu_sdm_internal *sdm, @@ -49,6 +50,7 @@ void ccu_sdm_helper_enable(struct ccu_common *common, writel(reg | sdm->enable, common->base + common->reg); spin_unlock_irqrestore(common->lock, flags); } +EXPORT_SYMBOL_NS_GPL(ccu_sdm_helper_enable, SUNXI_CCU); void ccu_sdm_helper_disable(struct ccu_common *common, struct ccu_sdm_internal *sdm) @@ -69,6 +71,7 @@ void ccu_sdm_helper_disable(struct ccu_common *common, writel(reg & ~sdm->tuning_enable, common->base + sdm->tuning_reg); spin_unlock_irqrestore(common->lock, flags); } +EXPORT_SYMBOL_NS_GPL(ccu_sdm_helper_disable, SUNXI_CCU); /* * Sigma delta modulation provides a way to do fractional-N frequency @@ -102,6 +105,7 @@ bool ccu_sdm_helper_has_rate(struct ccu_common *common, return false; } +EXPORT_SYMBOL_NS_GPL(ccu_sdm_helper_has_rate, SUNXI_CCU); unsigned long ccu_sdm_helper_read_rate(struct ccu_common *common, struct ccu_sdm_internal *sdm, @@ -132,6 +136,7 @@ unsigned long ccu_sdm_helper_read_rate(struct ccu_common *common, /* We can't calculate the effective clock rate, so just fail. */ return 0; } +EXPORT_SYMBOL_NS_GPL(ccu_sdm_helper_read_rate, SUNXI_CCU); int ccu_sdm_helper_get_factors(struct ccu_common *common, struct ccu_sdm_internal *sdm, @@ -153,3 +158,4 @@ int ccu_sdm_helper_get_factors(struct ccu_common *common, /* nothing found */ return -EINVAL; } +EXPORT_SYMBOL_NS_GPL(ccu_sdm_helper_get_factors, SUNXI_CCU); diff --git a/drivers/clk/tegra/Makefile b/drivers/clk/tegra/Makefile index 7b1816856eb5..a0715cdfc1a4 100644 --- a/drivers/clk/tegra/Makefile +++ b/drivers/clk/tegra/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-y += clk.o obj-y += clk-audio-sync.o +obj-y += clk-device.o obj-y += clk-dfll.o obj-y += clk-divider.o obj-y += clk-periph.o diff --git a/drivers/clk/tegra/clk-device.c b/drivers/clk/tegra/clk-device.c new file mode 100644 index 000000000000..c58beaf8afbc --- /dev/null +++ b/drivers/clk/tegra/clk-device.c @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "clk.h" + +/* + * This driver manages performance state of the core power domain for the + * independent PLLs and system clocks. We created a virtual clock device + * for such clocks, see tegra_clk_dev_register(). + */ + +struct tegra_clk_device { + struct notifier_block clk_nb; + struct device *dev; + struct clk_hw *hw; + struct mutex lock; +}; + +static int tegra_clock_set_pd_state(struct tegra_clk_device *clk_dev, + unsigned long rate) +{ + struct device *dev = clk_dev->dev; + struct dev_pm_opp *opp; + unsigned int pstate; + + opp = dev_pm_opp_find_freq_ceil(dev, &rate); + if (opp == ERR_PTR(-ERANGE)) { + /* + * Some clocks may be unused by a particular board and they + * may have uninitiated clock rate that is overly high. In + * this case clock is expected to be disabled, but still we + * need to set up performance state of the power domain and + * not error out clk initialization. A typical example is + * a PCIe clock on Android tablets. + */ + dev_dbg(dev, "failed to find ceil OPP for %luHz\n", rate); + opp = dev_pm_opp_find_freq_floor(dev, &rate); + } + + if (IS_ERR(opp)) { + dev_err(dev, "failed to find OPP for %luHz: %pe\n", rate, opp); + return PTR_ERR(opp); + } + + pstate = dev_pm_opp_get_required_pstate(opp, 0); + dev_pm_opp_put(opp); + + return dev_pm_genpd_set_performance_state(dev, pstate); +} + +static int tegra_clock_change_notify(struct notifier_block *nb, + unsigned long msg, void *data) +{ + struct clk_notifier_data *cnd = data; + struct tegra_clk_device *clk_dev; + int err = 0; + + clk_dev = container_of(nb, struct tegra_clk_device, clk_nb); + + mutex_lock(&clk_dev->lock); + switch (msg) { + case PRE_RATE_CHANGE: + if (cnd->new_rate > cnd->old_rate) + err = tegra_clock_set_pd_state(clk_dev, cnd->new_rate); + break; + + case ABORT_RATE_CHANGE: + err = tegra_clock_set_pd_state(clk_dev, cnd->old_rate); + break; + + case POST_RATE_CHANGE: + if (cnd->new_rate < cnd->old_rate) + err = tegra_clock_set_pd_state(clk_dev, cnd->new_rate); + break; + + default: + break; + } + mutex_unlock(&clk_dev->lock); + + return notifier_from_errno(err); +} + +static int tegra_clock_sync_pd_state(struct tegra_clk_device *clk_dev) +{ + unsigned long rate; + int ret; + + mutex_lock(&clk_dev->lock); + + rate = clk_hw_get_rate(clk_dev->hw); + ret = tegra_clock_set_pd_state(clk_dev, rate); + + mutex_unlock(&clk_dev->lock); + + return ret; +} + +static int tegra_clock_probe(struct platform_device *pdev) +{ + struct tegra_core_opp_params opp_params = {}; + struct tegra_clk_device *clk_dev; + struct device *dev = &pdev->dev; + struct clk *clk; + int err; + + if (!dev->pm_domain) + return -EINVAL; + + clk_dev = devm_kzalloc(dev, sizeof(*clk_dev), GFP_KERNEL); + if (!clk_dev) + return -ENOMEM; + + clk = devm_clk_get(dev, NULL); + if (IS_ERR(clk)) + return PTR_ERR(clk); + + clk_dev->dev = dev; + clk_dev->hw = __clk_get_hw(clk); + clk_dev->clk_nb.notifier_call = tegra_clock_change_notify; + mutex_init(&clk_dev->lock); + + platform_set_drvdata(pdev, clk_dev); + + /* + * Runtime PM was already enabled for this device by the parent clk + * driver and power domain state should be synced under clk_dev lock, + * hence we don't use the common OPP helper that initializes OPP + * state. For some clocks common OPP helper may fail to find ceil + * rate, it's handled by this driver. + */ + err = devm_tegra_core_dev_init_opp_table(dev, &opp_params); + if (err) + return err; + + err = clk_notifier_register(clk, &clk_dev->clk_nb); + if (err) { + dev_err(dev, "failed to register clk notifier: %d\n", err); + return err; + } + + /* + * The driver is attaching to a potentially active/resumed clock, hence + * we need to sync the power domain performance state in a accordance to + * the clock rate if clock is resumed. + */ + err = tegra_clock_sync_pd_state(clk_dev); + if (err) + goto unreg_clk; + + return 0; + +unreg_clk: + clk_notifier_unregister(clk, &clk_dev->clk_nb); + + return err; +} + +/* + * Tegra GENPD driver enables clocks during NOIRQ phase. It can't be done + * for clocks served by this driver because runtime PM is unavailable in + * NOIRQ phase. We will keep clocks resumed during suspend to mitigate this + * problem. In practice this makes no difference from a power management + * perspective since voltage is kept at a nominal level during suspend anyways. + */ +static const struct dev_pm_ops tegra_clock_pm = { + SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_resume_and_get, pm_runtime_put) +}; + +static const struct of_device_id tegra_clock_match[] = { + { .compatible = "nvidia,tegra20-sclk" }, + { .compatible = "nvidia,tegra30-sclk" }, + { .compatible = "nvidia,tegra30-pllc" }, + { .compatible = "nvidia,tegra30-plle" }, + { .compatible = "nvidia,tegra30-pllm" }, + { } +}; + +static struct platform_driver tegra_clock_driver = { + .driver = { + .name = "tegra-clock", + .of_match_table = tegra_clock_match, + .pm = &tegra_clock_pm, + .suppress_bind_attrs = true, + }, + .probe = tegra_clock_probe, +}; +builtin_platform_driver(tegra_clock_driver); diff --git a/drivers/clk/tegra/clk-pll.c b/drivers/clk/tegra/clk-pll.c index eaa079c177c3..100b5d9b7e26 100644 --- a/drivers/clk/tegra/clk-pll.c +++ b/drivers/clk/tegra/clk-pll.c @@ -1914,7 +1914,7 @@ static struct clk *_tegra_clk_register_pll(struct tegra_clk_pll *pll, /* Data in .init is copied by clk_register(), so stack variable OK */ pll->hw.init = &init; - return clk_register(NULL, &pll->hw); + return tegra_clk_dev_register(&pll->hw); } struct clk *tegra_clk_register_pll(const char *name, const char *parent_name, diff --git a/drivers/clk/tegra/clk-super.c b/drivers/clk/tegra/clk-super.c index 6099c6e9acd4..a98a420398fa 100644 --- a/drivers/clk/tegra/clk-super.c +++ b/drivers/clk/tegra/clk-super.c @@ -226,7 +226,7 @@ struct clk *tegra_clk_register_super_mux(const char *name, /* Data in .init is copied by clk_register(), so stack variable OK */ super->hw.init = &init; - clk = clk_register(NULL, &super->hw); + clk = tegra_clk_dev_register(&super->hw); if (IS_ERR(clk)) kfree(super); diff --git a/drivers/clk/tegra/clk-tegra114.c b/drivers/clk/tegra/clk-tegra114.c index bc9e47a4cb60..ef718c4b3826 100644 --- a/drivers/clk/tegra/clk-tegra114.c +++ b/drivers/clk/tegra/clk-tegra114.c @@ -1158,7 +1158,7 @@ static struct tegra_clk_init_table init_table[] __initdata = { { TEGRA114_CLK_XUSB_HS_SRC, TEGRA114_CLK_XUSB_SS_DIV2, 61200000, 0 }, { TEGRA114_CLK_XUSB_FALCON_SRC, TEGRA114_CLK_PLL_P, 204000000, 0 }, { TEGRA114_CLK_XUSB_HOST_SRC, TEGRA114_CLK_PLL_P, 102000000, 0 }, - { TEGRA114_CLK_VDE, TEGRA114_CLK_CLK_MAX, 600000000, 0 }, + { TEGRA114_CLK_VDE, TEGRA114_CLK_PLL_P, 408000000, 0 }, { TEGRA114_CLK_SPDIF_IN_SYNC, TEGRA114_CLK_CLK_MAX, 24000000, 0 }, { TEGRA114_CLK_I2S0_SYNC, TEGRA114_CLK_CLK_MAX, 24000000, 0 }, { TEGRA114_CLK_I2S1_SYNC, TEGRA114_CLK_CLK_MAX, 24000000, 0 }, diff --git a/drivers/clk/tegra/clk-tegra20.c b/drivers/clk/tegra/clk-tegra20.c index 3664593a5ba4..be3c33441cfc 100644 --- a/drivers/clk/tegra/clk-tegra20.c +++ b/drivers/clk/tegra/clk-tegra20.c @@ -6,8 +6,11 @@ #include #include #include +#include #include #include +#include +#include #include #include #include @@ -414,7 +417,7 @@ static struct tegra_clk_pll_params pll_e_params = { .fixed_rate = 100000000, }; -static struct tegra_devclk devclks[] __initdata = { +static struct tegra_devclk devclks[] = { { .con_id = "pll_c", .dt_id = TEGRA20_CLK_PLL_C }, { .con_id = "pll_c_out1", .dt_id = TEGRA20_CLK_PLL_C_OUT1 }, { .con_id = "pll_p", .dt_id = TEGRA20_CLK_PLL_P }, @@ -710,13 +713,6 @@ static void tegra20_super_clk_init(void) NULL); clks[TEGRA20_CLK_CCLK] = clk; - /* SCLK */ - clk = tegra_clk_register_super_mux("sclk", sclk_parents, - ARRAY_SIZE(sclk_parents), - CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, - clk_base + SCLK_BURST_POLICY, 0, 4, 0, 0, NULL); - clks[TEGRA20_CLK_SCLK] = clk; - /* twd */ clk = clk_register_fixed_factor(NULL, "twd", "cclk", 0, 1, 4); clks[TEGRA20_CLK_TWD] = clk; @@ -1014,7 +1010,7 @@ static struct tegra_cpu_car_ops tegra20_cpu_car_ops = { #endif }; -static struct tegra_clk_init_table init_table[] __initdata = { +static struct tegra_clk_init_table init_table[] = { { TEGRA20_CLK_PLL_P, TEGRA20_CLK_CLK_MAX, 216000000, 1 }, { TEGRA20_CLK_PLL_P_OUT1, TEGRA20_CLK_CLK_MAX, 28800000, 1 }, { TEGRA20_CLK_PLL_P_OUT2, TEGRA20_CLK_CLK_MAX, 48000000, 1 }, @@ -1052,11 +1048,6 @@ static struct tegra_clk_init_table init_table[] __initdata = { { TEGRA20_CLK_CLK_MAX, TEGRA20_CLK_CLK_MAX, 0, 0 }, }; -static void __init tegra20_clock_apply_init_table(void) -{ - tegra_init_from_table(init_table, clks, TEGRA20_CLK_CLK_MAX); -} - /* * Some clocks may be used by different drivers depending on the board * configuration. List those here to register them twice in the clock lookup @@ -1076,6 +1067,8 @@ static const struct of_device_id pmc_match[] __initconst = { { }, }; +static bool tegra20_car_initialized; + static struct clk *tegra20_clk_src_onecell_get(struct of_phandle_args *clkspec, void *data) { @@ -1083,6 +1076,16 @@ static struct clk *tegra20_clk_src_onecell_get(struct of_phandle_args *clkspec, struct clk_hw *hw; struct clk *clk; + /* + * Timer clocks are needed early, the rest of the clocks shouldn't be + * available to device drivers until clock tree is fully initialized. + */ + if (clkspec->args[0] != TEGRA20_CLK_RTC && + clkspec->args[0] != TEGRA20_CLK_TWD && + clkspec->args[0] != TEGRA20_CLK_TIMER && + !tegra20_car_initialized) + return ERR_PTR(-EPROBE_DEFER); + clk = of_clk_src_onecell_get(clkspec, data); if (IS_ERR(clk)) return clk; @@ -1149,10 +1152,48 @@ static void __init tegra20_clock_init(struct device_node *np) tegra_init_dup_clks(tegra_clk_duplicates, clks, TEGRA20_CLK_CLK_MAX); tegra_add_of_provider(np, tegra20_clk_src_onecell_get); - tegra_register_devclks(devclks, ARRAY_SIZE(devclks)); - - tegra_clk_apply_init_table = tegra20_clock_apply_init_table; tegra_cpu_car_ops = &tegra20_cpu_car_ops; } -CLK_OF_DECLARE(tegra20, "nvidia,tegra20-car", tegra20_clock_init); +CLK_OF_DECLARE_DRIVER(tegra20, "nvidia,tegra20-car", tegra20_clock_init); + +/* + * Clocks that use runtime PM can't be created at the tegra20_clock_init + * time because drivers' base isn't initialized yet, and thus platform + * devices can't be created for the clocks. Hence we need to split the + * registration of the clocks into two phases. The first phase registers + * essential clocks which don't require RPM and are actually used during + * early boot. The second phase registers clocks which use RPM and this + * is done when device drivers' core API is ready. + */ +static int tegra20_car_probe(struct platform_device *pdev) +{ + struct clk *clk; + + clk = tegra_clk_register_super_mux("sclk", sclk_parents, + ARRAY_SIZE(sclk_parents), + CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, + clk_base + SCLK_BURST_POLICY, 0, 4, 0, 0, NULL); + clks[TEGRA20_CLK_SCLK] = clk; + + tegra_register_devclks(devclks, ARRAY_SIZE(devclks)); + tegra_init_from_table(init_table, clks, TEGRA20_CLK_CLK_MAX); + tegra20_car_initialized = true; + + return 0; +} + +static const struct of_device_id tegra20_car_match[] = { + { .compatible = "nvidia,tegra20-car" }, + { } +}; + +static struct platform_driver tegra20_car_driver = { + .driver = { + .name = "tegra20-car", + .of_match_table = tegra20_car_match, + .suppress_bind_attrs = true, + }, + .probe = tegra20_car_probe, +}; +builtin_platform_driver(tegra20_car_driver); diff --git a/drivers/clk/tegra/clk-tegra30.c b/drivers/clk/tegra/clk-tegra30.c index 64121bc66d85..04b496123820 100644 --- a/drivers/clk/tegra/clk-tegra30.c +++ b/drivers/clk/tegra/clk-tegra30.c @@ -7,8 +7,11 @@ #include #include #include +#include #include #include +#include +#include #include #include @@ -532,7 +535,7 @@ static unsigned long tegra30_input_freq[] = { [12] = 26000000, }; -static struct tegra_devclk devclks[] __initdata = { +static struct tegra_devclk devclks[] = { { .con_id = "pll_c", .dt_id = TEGRA30_CLK_PLL_C }, { .con_id = "pll_c_out1", .dt_id = TEGRA30_CLK_PLL_C_OUT1 }, { .con_id = "pll_p", .dt_id = TEGRA30_CLK_PLL_P }, @@ -812,11 +815,6 @@ static void __init tegra30_pll_init(void) { struct clk *clk; - /* PLLC */ - clk = tegra_clk_register_pll("pll_c", "pll_ref", clk_base, pmc_base, 0, - &pll_c_params, NULL); - clks[TEGRA30_CLK_PLL_C] = clk; - /* PLLC_OUT1 */ clk = tegra_clk_register_divider("pll_c_out1_div", "pll_c", clk_base + PLLC_OUT, 0, TEGRA_DIVIDER_ROUND_UP, @@ -826,11 +824,6 @@ static void __init tegra30_pll_init(void) 0, NULL); clks[TEGRA30_CLK_PLL_C_OUT1] = clk; - /* PLLM */ - clk = tegra_clk_register_pll("pll_m", "pll_ref", clk_base, pmc_base, - CLK_SET_RATE_GATE, &pll_m_params, NULL); - clks[TEGRA30_CLK_PLL_M] = clk; - /* PLLM_OUT1 */ clk = tegra_clk_register_divider("pll_m_out1_div", "pll_m", clk_base + PLLM_OUT, 0, TEGRA_DIVIDER_ROUND_UP, @@ -880,9 +873,6 @@ static void __init tegra30_pll_init(void) ARRAY_SIZE(pll_e_parents), CLK_SET_RATE_NO_REPARENT, clk_base + PLLE_AUX, 2, 1, 0, NULL); - clk = tegra_clk_register_plle("pll_e", "pll_e_mux", clk_base, pmc_base, - CLK_GET_RATE_NOCACHE, &pll_e_params, NULL); - clks[TEGRA30_CLK_PLL_E] = clk; } static const char *cclk_g_parents[] = { "clk_m", "pll_c", "clk_32k", "pll_m", @@ -971,14 +961,6 @@ static void __init tegra30_super_clk_init(void) NULL); clks[TEGRA30_CLK_CCLK_LP] = clk; - /* SCLK */ - clk = tegra_clk_register_super_mux("sclk", sclk_parents, - ARRAY_SIZE(sclk_parents), - CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, - clk_base + SCLK_BURST_POLICY, - 0, 4, 0, 0, NULL); - clks[TEGRA30_CLK_SCLK] = clk; - /* twd */ clk = clk_register_fixed_factor(NULL, "twd", "cclk_g", CLK_SET_RATE_PARENT, 1, 2); @@ -1214,7 +1196,7 @@ static struct tegra_cpu_car_ops tegra30_cpu_car_ops = { #endif }; -static struct tegra_clk_init_table init_table[] __initdata = { +static struct tegra_clk_init_table init_table[] = { { TEGRA30_CLK_UARTA, TEGRA30_CLK_PLL_P, 408000000, 0 }, { TEGRA30_CLK_UARTB, TEGRA30_CLK_PLL_P, 408000000, 0 }, { TEGRA30_CLK_UARTC, TEGRA30_CLK_PLL_P, 408000000, 0 }, @@ -1259,11 +1241,6 @@ static struct tegra_clk_init_table init_table[] __initdata = { { TEGRA30_CLK_CLK_MAX, TEGRA30_CLK_CLK_MAX, 0, 0 }, }; -static void __init tegra30_clock_apply_init_table(void) -{ - tegra_init_from_table(init_table, clks, TEGRA30_CLK_CLK_MAX); -} - /* * Some clocks may be used by different drivers depending on the board * configuration. List those here to register them twice in the clock lookup @@ -1294,12 +1271,24 @@ static struct tegra_audio_clk_info tegra30_audio_plls[] = { { "pll_a", &pll_a_params, tegra_clk_pll_a, "pll_p_out1" }, }; +static bool tegra30_car_initialized; + static struct clk *tegra30_clk_src_onecell_get(struct of_phandle_args *clkspec, void *data) { struct clk_hw *hw; struct clk *clk; + /* + * Timer clocks are needed early, the rest of the clocks shouldn't be + * available to device drivers until clock tree is fully initialized. + */ + if (clkspec->args[0] != TEGRA30_CLK_RTC && + clkspec->args[0] != TEGRA30_CLK_TWD && + clkspec->args[0] != TEGRA30_CLK_TIMER && + !tegra30_car_initialized) + return ERR_PTR(-EPROBE_DEFER); + clk = of_clk_src_onecell_get(clkspec, data); if (IS_ERR(clk)) return clk; @@ -1357,10 +1346,75 @@ static void __init tegra30_clock_init(struct device_node *np) tegra_init_dup_clks(tegra_clk_duplicates, clks, TEGRA30_CLK_CLK_MAX); tegra_add_of_provider(np, tegra30_clk_src_onecell_get); - tegra_register_devclks(devclks, ARRAY_SIZE(devclks)); - - tegra_clk_apply_init_table = tegra30_clock_apply_init_table; tegra_cpu_car_ops = &tegra30_cpu_car_ops; } -CLK_OF_DECLARE(tegra30, "nvidia,tegra30-car", tegra30_clock_init); +CLK_OF_DECLARE_DRIVER(tegra30, "nvidia,tegra30-car", tegra30_clock_init); + +/* + * Clocks that use runtime PM can't be created at the tegra30_clock_init + * time because drivers' base isn't initialized yet, and thus platform + * devices can't be created for the clocks. Hence we need to split the + * registration of the clocks into two phases. The first phase registers + * essential clocks which don't require RPM and are actually used during + * early boot. The second phase registers clocks which use RPM and this + * is done when device drivers' core API is ready. + */ +static int tegra30_car_probe(struct platform_device *pdev) +{ + struct clk *clk; + + /* PLLC */ + clk = tegra_clk_register_pll("pll_c", "pll_ref", clk_base, pmc_base, 0, + &pll_c_params, NULL); + clks[TEGRA30_CLK_PLL_C] = clk; + + /* PLLE */ + clk = tegra_clk_register_plle("pll_e", "pll_e_mux", clk_base, pmc_base, + CLK_GET_RATE_NOCACHE, &pll_e_params, NULL); + clks[TEGRA30_CLK_PLL_E] = clk; + + /* PLLM */ + clk = tegra_clk_register_pll("pll_m", "pll_ref", clk_base, pmc_base, + CLK_SET_RATE_GATE, &pll_m_params, NULL); + clks[TEGRA30_CLK_PLL_M] = clk; + + /* SCLK */ + clk = tegra_clk_register_super_mux("sclk", sclk_parents, + ARRAY_SIZE(sclk_parents), + CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, + clk_base + SCLK_BURST_POLICY, + 0, 4, 0, 0, NULL); + clks[TEGRA30_CLK_SCLK] = clk; + + tegra_register_devclks(devclks, ARRAY_SIZE(devclks)); + tegra_init_from_table(init_table, clks, TEGRA30_CLK_CLK_MAX); + tegra30_car_initialized = true; + + return 0; +} + +static const struct of_device_id tegra30_car_match[] = { + { .compatible = "nvidia,tegra30-car" }, + { } +}; + +static struct platform_driver tegra30_car_driver = { + .driver = { + .name = "tegra30-car", + .of_match_table = tegra30_car_match, + .suppress_bind_attrs = true, + }, + .probe = tegra30_car_probe, +}; + +/* + * Clock driver must be registered before memory controller driver, + * which doesn't support deferred probing for today and is registered + * from arch init-level. + */ +static int tegra30_car_init(void) +{ + return platform_driver_register(&tegra30_car_driver); +} +postcore_initcall(tegra30_car_init); diff --git a/drivers/clk/tegra/clk.c b/drivers/clk/tegra/clk.c index f6cdce441cf7..26bda45813c0 100644 --- a/drivers/clk/tegra/clk.c +++ b/drivers/clk/tegra/clk.c @@ -9,14 +9,19 @@ #include #include #include +#include #include +#include +#include #include +#include #include #include "clk.h" /* Global data of Tegra CPU CAR ops */ +static struct device_node *tegra_car_np; static struct tegra_cpu_car_ops dummy_car_ops; struct tegra_cpu_car_ops *tegra_cpu_car_ops = &dummy_car_ops; @@ -261,8 +266,8 @@ void __init tegra_init_dup_clks(struct tegra_clk_duplicate *dup_list, } } -void __init tegra_init_from_table(struct tegra_clk_init_table *tbl, - struct clk *clks[], int clk_max) +void tegra_init_from_table(struct tegra_clk_init_table *tbl, + struct clk *clks[], int clk_max) { struct clk *clk; @@ -320,6 +325,8 @@ void __init tegra_add_of_provider(struct device_node *np, { int i; + tegra_car_np = np; + for (i = 0; i < clk_num; i++) { if (IS_ERR(clks[i])) { pr_err @@ -348,7 +355,7 @@ void __init tegra_init_special_resets(unsigned int num, special_reset_deassert = deassert; } -void __init tegra_register_devclks(struct tegra_devclk *dev_clks, int num) +void tegra_register_devclks(struct tegra_devclk *dev_clks, int num) { int i; @@ -372,6 +379,68 @@ struct clk ** __init tegra_lookup_dt_id(int clk_id, return NULL; } +static struct device_node *tegra_clk_get_of_node(struct clk_hw *hw) +{ + struct device_node *np; + char *node_name; + + node_name = kstrdup(hw->init->name, GFP_KERNEL); + if (!node_name) + return NULL; + + strreplace(node_name, '_', '-'); + + for_each_child_of_node(tegra_car_np, np) { + if (!strcmp(np->name, node_name)) + break; + } + + kfree(node_name); + + return np; +} + +struct clk *tegra_clk_dev_register(struct clk_hw *hw) +{ + struct platform_device *pdev, *parent; + const char *dev_name = NULL; + struct device *dev = NULL; + struct device_node *np; + + np = tegra_clk_get_of_node(hw); + + if (!of_device_is_available(np)) + goto put_node; + + dev_name = kasprintf(GFP_KERNEL, "tegra_clk_%s", hw->init->name); + if (!dev_name) + goto put_node; + + parent = of_find_device_by_node(tegra_car_np); + if (parent) { + pdev = of_platform_device_create(np, dev_name, &parent->dev); + put_device(&parent->dev); + + if (!pdev) { + pr_err("%s: failed to create device for %pOF\n", + __func__, np); + goto free_name; + } + + dev = &pdev->dev; + pm_runtime_enable(dev); + } else { + WARN(1, "failed to find device for %pOF\n", tegra_car_np); + } + +free_name: + kfree(dev_name); +put_node: + of_node_put(np); + + return clk_register(dev, hw); +} + tegra_clk_apply_init_table_func tegra_clk_apply_init_table; static int __init tegra_clocks_apply_init_table(void) diff --git a/drivers/clk/tegra/clk.h b/drivers/clk/tegra/clk.h index 0c3ba0ccce1a..5d80d8b79b8e 100644 --- a/drivers/clk/tegra/clk.h +++ b/drivers/clk/tegra/clk.h @@ -927,4 +927,6 @@ struct clk *tegra20_clk_register_emc(void __iomem *ioaddr, bool low_jitter); struct clk *tegra210_clk_register_emc(struct device_node *np, void __iomem *regs); +struct clk *tegra_clk_dev_register(struct clk_hw *hw); + #endif /* TEGRA_CLK_H */ diff --git a/drivers/clk/ti/adpll.c b/drivers/clk/ti/adpll.c index b341cd990be7..962502ca7ff0 100644 --- a/drivers/clk/ti/adpll.c +++ b/drivers/clk/ti/adpll.c @@ -807,7 +807,7 @@ static int ti_adpll_init_registers(struct ti_adpll_data *d) static int ti_adpll_init_inputs(struct ti_adpll_data *d) { - const char *error = "need at least %i inputs"; + static const char error[] = "need at least %i inputs"; struct clk *clock; int nr_inputs; diff --git a/drivers/clk/visconti/Kconfig b/drivers/clk/visconti/Kconfig new file mode 100644 index 000000000000..1661097b0d92 --- /dev/null +++ b/drivers/clk/visconti/Kconfig @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config COMMON_CLK_VISCONTI + bool "Support for Toshiba Visconti5 ARM SoC clock controllers" + depends on ARCH_VISCONTI || COMPILE_TEST + default ARCH_VISCONTI + help + Support for the Toshiba Visconti5 ARM SoC clock controller. + Say Y if you want to include clock support. diff --git a/drivers/clk/visconti/Makefile b/drivers/clk/visconti/Makefile new file mode 100644 index 000000000000..c1254fd52b31 --- /dev/null +++ b/drivers/clk/visconti/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Makefile for Toshiba Visconti clock + +obj-y += clkc.o pll.o reset.o +obj-y += pll-tmpv770x.o clkc-tmpv770x.o diff --git a/drivers/clk/visconti/clkc-tmpv770x.c b/drivers/clk/visconti/clkc-tmpv770x.c new file mode 100644 index 000000000000..c2b2f41a85a4 --- /dev/null +++ b/drivers/clk/visconti/clkc-tmpv770x.c @@ -0,0 +1,291 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Toshiba Visconti clock controller + * + * Copyright (c) 2021 TOSHIBA CORPORATION + * Copyright (c) 2021 Toshiba Electronic Devices & Storage Corporation + * + * Nobuhiro Iwamatsu + */ + +#include +#include + +#include +#include + +#include "clkc.h" +#include "reset.h" + +static DEFINE_SPINLOCK(tmpv770x_clk_lock); +static DEFINE_SPINLOCK(tmpv770x_rst_lock); + +static const struct clk_parent_data clks_parent_data[] = { + { .fw_name = "pipll1", .name = "pipll1", }, +}; + +static const struct clk_parent_data pietherplls_parent_data[] = { + { .fw_name = "pietherpll", .name = "pietherpll", }, +}; + +static const struct visconti_fixed_clk fixed_clk_tables[] = { + /* PLL1 */ + /* PICMPT0/1, PITSC, PIUWDT, PISWDT, PISBUS, PIPMU, PIGPMU, PITMU */ + /* PIEMM, PIMISC, PIGCOMM, PIDCOMM, PIMBUS, PIGPIO, PIPGM */ + { TMPV770X_CLK_PIPLL1_DIV4, "pipll1_div4", "pipll1", 0, 1, 4, }, + /* PISBUS */ + { TMPV770X_CLK_PIPLL1_DIV2, "pipll1_div2", "pipll1", 0, 1, 2, }, + /* PICOBUS_CLK */ + { TMPV770X_CLK_PIPLL1_DIV1, "pipll1_div1", "pipll1", 0, 1, 1, }, + /* PIDNNPLL */ + /* CONN_CLK, PIMBUS, PICRC0/1 */ + { TMPV770X_CLK_PIDNNPLL_DIV1, "pidnnpll_div1", "pidnnpll", 0, 1, 1, }, + { TMPV770X_CLK_PIREFCLK, "pirefclk", "osc2-clk", 0, 1, 1, }, + { TMPV770X_CLK_WDTCLK, "wdtclk", "osc2-clk", 0, 1, 1, }, +}; + +static const struct visconti_clk_gate_table pietherpll_clk_gate_tables[] = { + /* pietherpll */ + { TMPV770X_CLK_PIETHER_2P5M, "piether_2p5m", + pietherplls_parent_data, ARRAY_SIZE(pietherplls_parent_data), + CLK_SET_RATE_PARENT, 0x34, 0x134, 4, 200, + TMPV770X_RESET_PIETHER_2P5M, }, + { TMPV770X_CLK_PIETHER_25M, "piether_25m", + pietherplls_parent_data, ARRAY_SIZE(pietherplls_parent_data), + CLK_SET_RATE_PARENT, 0x34, 0x134, 5, 20, + TMPV770X_RESET_PIETHER_25M, }, + { TMPV770X_CLK_PIETHER_50M, "piether_50m", + pietherplls_parent_data, ARRAY_SIZE(pietherplls_parent_data), + CLK_SET_RATE_PARENT, 0x34, 0x134, 6, 10, + TMPV770X_RESET_PIETHER_50M, }, + { TMPV770X_CLK_PIETHER_125M, "piether_125m", + pietherplls_parent_data, ARRAY_SIZE(pietherplls_parent_data), + CLK_SET_RATE_PARENT, 0x34, 0x134, 7, 4, + TMPV770X_RESET_PIETHER_125M, }, +}; + +static const struct visconti_clk_gate_table clk_gate_tables[] = { + { TMPV770X_CLK_HOX, "hox", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0x4c, 0x14c, 0, 1, + TMPV770X_RESET_HOX, }, + { TMPV770X_CLK_PCIE_MSTR, "pcie_mstr", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0x38, 0x138, 0, 1, + TMPV770X_RESET_PCIE_MSTR, }, + { TMPV770X_CLK_PCIE_AUX, "pcie_aux", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0x38, 0x138, 1, 24, + TMPV770X_RESET_PCIE_AUX, }, + { TMPV770X_CLK_PIINTC, "piintc", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + CLK_IGNORE_UNUSED, 0x8, 0x108, 0, 2, //FIX!! + TMPV770X_RESET_PIINTC,}, + { TMPV770X_CLK_PIETHER_BUS, "piether_bus", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + 0, 0x34, 0x134, 0, 2, + TMPV770X_RESET_PIETHER_BUS, }, /* BUS_CLK */ + { TMPV770X_CLK_PISPI0, "pispi0", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + 0, 0x28, 0x128, 0, 2, + TMPV770X_RESET_PISPI0, }, + { TMPV770X_CLK_PISPI1, "pispi1", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + 0, 0x28, 0x128, 1, 2, + TMPV770X_RESET_PISPI1, }, + { TMPV770X_CLK_PISPI2, "pispi2", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + 0, 0x28, 0x128, 2, 2, + TMPV770X_RESET_PISPI2, }, + { TMPV770X_CLK_PISPI3, "pispi3", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + 0, 0x28, 0x128, 3, 2, + TMPV770X_RESET_PISPI3,}, + { TMPV770X_CLK_PISPI4, "pispi4", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + 0, 0x28, 0x128, 4, 2, + TMPV770X_RESET_PISPI4, }, + { TMPV770X_CLK_PISPI5, "pispi5", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + 0, 0x28, 0x128, 5, 2, + TMPV770X_RESET_PISPI5}, + { TMPV770X_CLK_PISPI6, "pispi6", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + 0, 0x28, 0x128, 6, 2, + TMPV770X_RESET_PISPI6,}, + { TMPV770X_CLK_PIUART0, "piuart0", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + //CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0x2c, 0x12c, 0, 4, + 0, 0x2c, 0x12c, 0, 4, + TMPV770X_RESET_PIUART0,}, + { TMPV770X_CLK_PIUART1, "piuart1", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + //CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0x2c, 0x12c, 1, 4, + 0, 0x2c, 0x12c, 1, 4, + TMPV770X_RESET_PIUART1, }, + { TMPV770X_CLK_PIUART2, "piuart2", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + 0, 0x2c, 0x12c, 2, 4, + TMPV770X_RESET_PIUART2, }, + { TMPV770X_CLK_PIUART3, "piuart3", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + 0, 0x2c, 0x12c, 3, 4, + TMPV770X_RESET_PIUART3, }, + { TMPV770X_CLK_PII2C0, "pii2c0", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + 0, 0x30, 0x130, 0, 4, + TMPV770X_RESET_PII2C0, }, + { TMPV770X_CLK_PII2C1, "pii2c1", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + 0, 0x30, 0x130, 1, 4, + TMPV770X_RESET_PII2C1, }, + { TMPV770X_CLK_PII2C2, "pii2c2", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + 0, 0x30, 0x130, 2, 4, + TMPV770X_RESET_PII2C2, }, + { TMPV770X_CLK_PII2C3, "pii2c3", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + 0, 0x30, 0x130, 3, 4, + TMPV770X_RESET_PII2C3,}, + { TMPV770X_CLK_PII2C4, "pii2c4", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + 0, 0x30, 0x130, 4, 4, + TMPV770X_RESET_PII2C4, }, + { TMPV770X_CLK_PII2C5, "pii2c5", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + 0, 0x30, 0x130, 5, 4, + TMPV770X_RESET_PII2C5, }, + { TMPV770X_CLK_PII2C6, "pii2c6", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + 0, 0x30, 0x130, 6, 4, + TMPV770X_RESET_PII2C6, }, + { TMPV770X_CLK_PII2C7, "pii2c7", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + 0, 0x30, 0x130, 7, 4, + TMPV770X_RESET_PII2C7, }, + { TMPV770X_CLK_PII2C8, "pii2c8", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + 0, 0x30, 0x130, 8, 4, + TMPV770X_RESET_PII2C8, }, + /* PIPCMIF */ + { TMPV770X_CLK_PIPCMIF, "pipcmif", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + 0, 0x64, 0x164, 0, 4, + TMPV770X_RESET_PIPCMIF, }, + /* PISYSTEM */ + { TMPV770X_CLK_WRCK, "wrck", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + 0, 0x68, 0x168, 9, 32, + -1, }, /* No reset */ + { TMPV770X_CLK_PICKMON, "pickmon", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + 0, 0x10, 0x110, 8, 4, + TMPV770X_RESET_PICKMON, }, + { TMPV770X_CLK_SBUSCLK, "sbusclk", + clks_parent_data, ARRAY_SIZE(clks_parent_data), + 0, 0x14, 0x114, 0, 4, + TMPV770X_RESET_SBUSCLK, }, +}; + +static const struct visconti_reset_data clk_reset_data[] = { + [TMPV770X_RESET_PIETHER_2P5M] = { 0x434, 0x534, 4, }, + [TMPV770X_RESET_PIETHER_25M] = { 0x434, 0x534, 5, }, + [TMPV770X_RESET_PIETHER_50M] = { 0x434, 0x534, 6, }, + [TMPV770X_RESET_PIETHER_125M] = { 0x434, 0x534, 7, }, + [TMPV770X_RESET_HOX] = { 0x44c, 0x54c, 0, }, + [TMPV770X_RESET_PCIE_MSTR] = { 0x438, 0x538, 0, }, + [TMPV770X_RESET_PCIE_AUX] = { 0x438, 0x538, 1, }, + [TMPV770X_RESET_PIINTC] = { 0x408, 0x508, 0, }, + [TMPV770X_RESET_PIETHER_BUS] = { 0x434, 0x534, 0, }, + [TMPV770X_RESET_PISPI0] = { 0x428, 0x528, 0, }, + [TMPV770X_RESET_PISPI1] = { 0x428, 0x528, 1, }, + [TMPV770X_RESET_PISPI2] = { 0x428, 0x528, 2, }, + [TMPV770X_RESET_PISPI3] = { 0x428, 0x528, 3, }, + [TMPV770X_RESET_PISPI4] = { 0x428, 0x528, 4, }, + [TMPV770X_RESET_PISPI5] = { 0x428, 0x528, 5, }, + [TMPV770X_RESET_PISPI6] = { 0x428, 0x528, 6, }, + [TMPV770X_RESET_PIUART0] = { 0x42c, 0x52c, 0, }, + [TMPV770X_RESET_PIUART1] = { 0x42c, 0x52c, 1, }, + [TMPV770X_RESET_PIUART2] = { 0x42c, 0x52c, 2, }, + [TMPV770X_RESET_PIUART3] = { 0x42c, 0x52c, 3, }, + [TMPV770X_RESET_PII2C0] = { 0x430, 0x530, 0, }, + [TMPV770X_RESET_PII2C1] = { 0x430, 0x530, 1, }, + [TMPV770X_RESET_PII2C2] = { 0x430, 0x530, 2, }, + [TMPV770X_RESET_PII2C3] = { 0x430, 0x530, 3, }, + [TMPV770X_RESET_PII2C4] = { 0x430, 0x530, 4, }, + [TMPV770X_RESET_PII2C5] = { 0x430, 0x530, 5, }, + [TMPV770X_RESET_PII2C6] = { 0x430, 0x530, 6, }, + [TMPV770X_RESET_PII2C7] = { 0x430, 0x530, 7, }, + [TMPV770X_RESET_PII2C8] = { 0x430, 0x530, 8, }, + [TMPV770X_RESET_PIPCMIF] = { 0x464, 0x564, 0, }, + [TMPV770X_RESET_PICKMON] = { 0x410, 0x510, 8, }, + [TMPV770X_RESET_SBUSCLK] = { 0x414, 0x514, 0, }, +}; + +static int visconti_clk_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct visconti_clk_provider *ctx; + struct device *dev = &pdev->dev; + struct regmap *regmap; + int ret, i; + + regmap = syscon_node_to_regmap(np); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + ctx = visconti_init_clk(dev, regmap, TMPV770X_NR_CLK); + if (IS_ERR(ctx)) + return PTR_ERR(ctx); + + ret = visconti_register_reset_controller(dev, regmap, clk_reset_data, + TMPV770X_NR_RESET, + &visconti_reset_ops, + &tmpv770x_rst_lock); + if (ret) { + dev_err(dev, "Failed to register reset controller: %d\n", ret); + return ret; + } + + for (i = 0; i < (ARRAY_SIZE(fixed_clk_tables)); i++) + ctx->clk_data.hws[fixed_clk_tables[i].id] = + clk_hw_register_fixed_factor(NULL, + fixed_clk_tables[i].name, + fixed_clk_tables[i].parent, + fixed_clk_tables[i].flag, + fixed_clk_tables[i].mult, + fixed_clk_tables[i].div); + + ret = visconti_clk_register_gates(ctx, clk_gate_tables, + ARRAY_SIZE(clk_gate_tables), clk_reset_data, + &tmpv770x_clk_lock); + if (ret) { + dev_err(dev, "Failed to register main clock gate: %d\n", ret); + return ret; + } + + ret = visconti_clk_register_gates(ctx, pietherpll_clk_gate_tables, + ARRAY_SIZE(pietherpll_clk_gate_tables), + clk_reset_data, &tmpv770x_clk_lock); + if (ret) { + dev_err(dev, "Failed to register pietherpll clock gate: %d\n", ret); + return ret; + } + + return of_clk_add_hw_provider(np, of_clk_hw_onecell_get, &ctx->clk_data); +} + +static const struct of_device_id visconti_clk_ids[] = { + { .compatible = "toshiba,tmpv7708-pismu", }, + { } +}; + +static struct platform_driver visconti_clk_driver = { + .probe = visconti_clk_probe, + .driver = { + .name = "visconti-clk", + .of_match_table = visconti_clk_ids, + }, +}; + +builtin_platform_driver(visconti_clk_driver); diff --git a/drivers/clk/visconti/clkc.c b/drivers/clk/visconti/clkc.c new file mode 100644 index 000000000000..56a8a4ffebca --- /dev/null +++ b/drivers/clk/visconti/clkc.c @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Toshiba Visconti clock controller + * + * Copyright (c) 2021 TOSHIBA CORPORATION + * Copyright (c) 2021 Toshiba Electronic Devices & Storage Corporation + * + * Nobuhiro Iwamatsu + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "clkc.h" + +static inline struct visconti_clk_gate *to_visconti_clk_gate(struct clk_hw *hw) +{ + return container_of(hw, struct visconti_clk_gate, hw); +} + +static int visconti_gate_clk_is_enabled(struct clk_hw *hw) +{ + struct visconti_clk_gate *gate = to_visconti_clk_gate(hw); + u32 clk = BIT(gate->ck_idx); + u32 val; + + regmap_read(gate->regmap, gate->ckon_offset, &val); + return (val & clk) ? 1 : 0; +} + +static void visconti_gate_clk_disable(struct clk_hw *hw) +{ + struct visconti_clk_gate *gate = to_visconti_clk_gate(hw); + u32 clk = BIT(gate->ck_idx); + unsigned long flags; + + spin_lock_irqsave(gate->lock, flags); + + if (!visconti_gate_clk_is_enabled(hw)) { + spin_unlock_irqrestore(gate->lock, flags); + return; + } + + regmap_update_bits(gate->regmap, gate->ckoff_offset, clk, clk); + spin_unlock_irqrestore(gate->lock, flags); +} + +static int visconti_gate_clk_enable(struct clk_hw *hw) +{ + struct visconti_clk_gate *gate = to_visconti_clk_gate(hw); + u32 clk = BIT(gate->ck_idx); + unsigned long flags; + + spin_lock_irqsave(gate->lock, flags); + regmap_update_bits(gate->regmap, gate->ckon_offset, clk, clk); + spin_unlock_irqrestore(gate->lock, flags); + + return 0; +} + +static const struct clk_ops visconti_clk_gate_ops = { + .enable = visconti_gate_clk_enable, + .disable = visconti_gate_clk_disable, + .is_enabled = visconti_gate_clk_is_enabled, +}; + +static struct clk_hw *visconti_clk_register_gate(struct device *dev, + const char *name, + const char *parent_name, + struct regmap *regmap, + const struct visconti_clk_gate_table *clks, + u32 rson_offset, + u32 rsoff_offset, + u8 rs_idx, + spinlock_t *lock) +{ + struct visconti_clk_gate *gate; + struct clk_parent_data *pdata; + struct clk_init_data init; + struct clk_hw *hw; + int ret; + + pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL); + if (!pdata) + return ERR_PTR(-ENOMEM); + + pdata->name = pdata->fw_name = parent_name; + + gate = devm_kzalloc(dev, sizeof(*gate), GFP_KERNEL); + if (!gate) + return ERR_PTR(-ENOMEM); + + init.name = name; + init.ops = &visconti_clk_gate_ops; + init.flags = clks->flags; + init.parent_data = pdata; + init.num_parents = 1; + + gate->regmap = regmap; + gate->ckon_offset = clks->ckon_offset; + gate->ckoff_offset = clks->ckoff_offset; + gate->ck_idx = clks->ck_idx; + gate->rson_offset = rson_offset; + gate->rsoff_offset = rsoff_offset; + gate->rs_idx = rs_idx; + gate->lock = lock; + gate->hw.init = &init; + + hw = &gate->hw; + ret = devm_clk_hw_register(dev, hw); + if (ret) + hw = ERR_PTR(ret); + + return hw; +} + +int visconti_clk_register_gates(struct visconti_clk_provider *ctx, + const struct visconti_clk_gate_table *clks, + int num_gate, + const struct visconti_reset_data *reset, + spinlock_t *lock) +{ + struct device *dev = ctx->dev; + int i; + + for (i = 0; i < num_gate; i++) { + const char *parent_div_name = clks[i].parent_data[0].name; + struct clk_parent_data *pdata; + u32 rson_offset, rsoff_offset; + struct clk_hw *gate_clk; + struct clk_hw *div_clk; + char *dev_name; + u8 rs_idx; + + pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL); + if (!pdata) + return -ENOMEM; + + dev_name = devm_kasprintf(dev, GFP_KERNEL, "%s_div", clks[i].name); + if (!dev_name) + return -ENOMEM; + + if (clks[i].rs_id >= 0) { + rson_offset = reset[clks[i].rs_id].rson_offset; + rsoff_offset = reset[clks[i].rs_id].rsoff_offset; + rs_idx = reset[clks[i].rs_id].rs_idx; + } else { + rson_offset = rsoff_offset = rs_idx = -1; + } + + div_clk = devm_clk_hw_register_fixed_factor(dev, + dev_name, + parent_div_name, + 0, 1, + clks[i].div); + if (IS_ERR(div_clk)) + return PTR_ERR(div_clk); + + gate_clk = visconti_clk_register_gate(dev, + clks[i].name, + dev_name, + ctx->regmap, + &clks[i], + rson_offset, + rsoff_offset, + rs_idx, + lock); + if (IS_ERR(gate_clk)) { + dev_err(dev, "%s: failed to register clock %s\n", + __func__, clks[i].name); + return PTR_ERR(gate_clk); + } + + ctx->clk_data.hws[clks[i].id] = gate_clk; + } + + return 0; +} + +struct visconti_clk_provider *visconti_init_clk(struct device *dev, + struct regmap *regmap, + unsigned long nr_clks) +{ + struct visconti_clk_provider *ctx; + int i; + + ctx = devm_kzalloc(dev, struct_size(ctx, clk_data.hws, nr_clks), GFP_KERNEL); + if (!ctx) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < nr_clks; ++i) + ctx->clk_data.hws[i] = ERR_PTR(-ENOENT); + ctx->clk_data.num = nr_clks; + + ctx->dev = dev; + ctx->regmap = regmap; + + return ctx; +} diff --git a/drivers/clk/visconti/clkc.h b/drivers/clk/visconti/clkc.h new file mode 100644 index 000000000000..09ed82ff64e4 --- /dev/null +++ b/drivers/clk/visconti/clkc.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Toshiba Visconti clock controller + * + * Copyright (c) 2021 TOSHIBA CORPORATION + * Copyright (c) 2021 Toshiba Electronic Devices & Storage Corporation + * + * Nobuhiro Iwamatsu + */ + +#ifndef _VISCONTI_CLKC_H_ +#define _VISCONTI_CLKC_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "reset.h" + +struct visconti_clk_provider { + struct device *dev; + struct regmap *regmap; + struct clk_hw_onecell_data clk_data; +}; + +struct visconti_clk_gate_table { + unsigned int id; + const char *name; + const struct clk_parent_data *parent_data; + u8 num_parents; + u8 flags; + u32 ckon_offset; + u32 ckoff_offset; + u8 ck_idx; + unsigned int div; + u8 rs_id; +}; + +struct visconti_fixed_clk { + unsigned int id; + const char *name; + const char *parent; + unsigned long flag; + unsigned int mult; + unsigned int div; +}; + +struct visconti_clk_gate { + struct clk_hw hw; + struct regmap *regmap; + u32 ckon_offset; + u32 ckoff_offset; + u8 ck_idx; + u8 flags; + u32 rson_offset; + u32 rsoff_offset; + u8 rs_idx; + spinlock_t *lock; +}; + +struct visconti_clk_provider *visconti_init_clk(struct device *dev, + struct regmap *regmap, + unsigned long nr_clks); +int visconti_clk_register_gates(struct visconti_clk_provider *data, + const struct visconti_clk_gate_table *clks, + int num_gate, + const struct visconti_reset_data *reset, + spinlock_t *lock); +#endif /* _VISCONTI_CLKC_H_ */ diff --git a/drivers/clk/visconti/pll-tmpv770x.c b/drivers/clk/visconti/pll-tmpv770x.c new file mode 100644 index 000000000000..8360ccf88867 --- /dev/null +++ b/drivers/clk/visconti/pll-tmpv770x.c @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Toshiba Visconti PLL controller + * + * Copyright (c) 2021 TOSHIBA CORPORATION + * Copyright (c) 2021 Toshiba Electronic Devices & Storage Corporation + * + * Nobuhiro Iwamatsu + */ + +#include +#include +#include + +#include + +#include "pll.h" + +static DEFINE_SPINLOCK(tmpv770x_pll_lock); + +static const struct visconti_pll_rate_table pipll0_rates[] __initconst = { + VISCONTI_PLL_RATE(840000000, 0x1, 0x0, 0x1, 0x54, 0x000000, 0x2, 0x1), + VISCONTI_PLL_RATE(780000000, 0x1, 0x0, 0x1, 0x4e, 0x000000, 0x2, 0x1), + VISCONTI_PLL_RATE(600000000, 0x1, 0x0, 0x1, 0x3c, 0x000000, 0x2, 0x1), + { /* sentinel */ }, +}; + +static const struct visconti_pll_rate_table piddrcpll_rates[] __initconst = { + VISCONTI_PLL_RATE(780000000, 0x1, 0x0, 0x1, 0x4e, 0x000000, 0x2, 0x1), + VISCONTI_PLL_RATE(760000000, 0x1, 0x0, 0x1, 0x4c, 0x000000, 0x2, 0x1), + { /* sentinel */ }, +}; + +static const struct visconti_pll_rate_table pivoifpll_rates[] __initconst = { + VISCONTI_PLL_RATE(165000000, 0x1, 0x0, 0x1, 0x42, 0x000000, 0x4, 0x2), + VISCONTI_PLL_RATE(148500000, 0x1, 0x1, 0x1, 0x3b, 0x666666, 0x4, 0x2), + VISCONTI_PLL_RATE(96000000, 0x1, 0x0, 0x1, 0x30, 0x000000, 0x5, 0x2), + VISCONTI_PLL_RATE(74250000, 0x1, 0x1, 0x1, 0x3b, 0x666666, 0x4, 0x4), + VISCONTI_PLL_RATE(54000000, 0x1, 0x0, 0x1, 0x36, 0x000000, 0x5, 0x4), + VISCONTI_PLL_RATE(48000000, 0x1, 0x0, 0x1, 0x30, 0x000000, 0x5, 0x4), + VISCONTI_PLL_RATE(35750000, 0x1, 0x1, 0x1, 0x32, 0x0ccccc, 0x7, 0x4), + { /* sentinel */ }, +}; + +static const struct visconti_pll_rate_table piimgerpll_rates[] __initconst = { + VISCONTI_PLL_RATE(165000000, 0x1, 0x0, 0x1, 0x42, 0x000000, 0x4, 0x2), + VISCONTI_PLL_RATE(96000000, 0x1, 0x0, 0x1, 0x30, 0x000000, 0x5, 0x2), + VISCONTI_PLL_RATE(54000000, 0x1, 0x0, 0x1, 0x36, 0x000000, 0x5, 0x4), + VISCONTI_PLL_RATE(48000000, 0x1, 0x0, 0x1, 0x30, 0x000000, 0x5, 0x4), + { /* sentinel */ }, +}; + +static const struct visconti_pll_info pll_info[] __initconst = { + { TMPV770X_PLL_PIPLL0, "pipll0", "osc2-clk", 0x0, pipll0_rates }, + { TMPV770X_PLL_PIDDRCPLL, "piddrcpll", "osc2-clk", 0x500, piddrcpll_rates }, + { TMPV770X_PLL_PIVOIFPLL, "pivoifpll", "osc2-clk", 0x600, pivoifpll_rates }, + { TMPV770X_PLL_PIIMGERPLL, "piimgerpll", "osc2-clk", 0x700, piimgerpll_rates }, +}; + +static void __init tmpv770x_setup_plls(struct device_node *np) +{ + struct visconti_pll_provider *ctx; + void __iomem *reg_base; + + reg_base = of_iomap(np, 0); + if (!reg_base) + return; + + ctx = visconti_init_pll(np, reg_base, TMPV770X_NR_PLL); + if (IS_ERR(ctx)) { + iounmap(reg_base); + return; + } + + ctx->clk_data.hws[TMPV770X_PLL_PIPLL1] = + clk_hw_register_fixed_rate(NULL, "pipll1", NULL, 0, 600000000); + ctx->clk_data.hws[TMPV770X_PLL_PIDNNPLL] = + clk_hw_register_fixed_rate(NULL, "pidnnpll", NULL, 0, 500000000); + ctx->clk_data.hws[TMPV770X_PLL_PIETHERPLL] = + clk_hw_register_fixed_rate(NULL, "pietherpll", NULL, 0, 500000000); + + visconti_register_plls(ctx, pll_info, ARRAY_SIZE(pll_info), &tmpv770x_pll_lock); +} + +CLK_OF_DECLARE(tmpv770x_plls, "toshiba,tmpv7708-pipllct", tmpv770x_setup_plls); diff --git a/drivers/clk/visconti/pll.c b/drivers/clk/visconti/pll.c new file mode 100644 index 000000000000..a2398bc6c6e4 --- /dev/null +++ b/drivers/clk/visconti/pll.c @@ -0,0 +1,340 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Toshiba Visconti PLL driver + * + * Copyright (c) 2021 TOSHIBA CORPORATION + * Copyright (c) 2021 Toshiba Electronic Devices & Storage Corporation + * + * Nobuhiro Iwamatsu + */ + +#include +#include +#include +#include +#include + +#include "pll.h" + +struct visconti_pll { + struct clk_hw hw; + void __iomem *pll_base; + spinlock_t *lock; + unsigned long flags; + const struct visconti_pll_rate_table *rate_table; + size_t rate_count; + struct visconti_pll_provider *ctx; +}; + +#define PLL_CONF_REG 0x0000 +#define PLL_CTRL_REG 0x0004 +#define PLL_FRACMODE_REG 0x0010 +#define PLL_INTIN_REG 0x0014 +#define PLL_FRACIN_REG 0x0018 +#define PLL_REFDIV_REG 0x001c +#define PLL_POSTDIV_REG 0x0020 + +#define PLL_CONFIG_SEL BIT(0) +#define PLL_PLLEN BIT(4) +#define PLL_BYPASS BIT(16) +#define PLL_INTIN_MASK GENMASK(11, 0) +#define PLL_FRACIN_MASK GENMASK(23, 0) +#define PLL_REFDIV_MASK GENMASK(5, 0) +#define PLL_POSTDIV_MASK GENMASK(2, 0) + +#define PLL0_FRACMODE_DACEN BIT(4) +#define PLL0_FRACMODE_DSMEN BIT(0) + +#define PLL_CREATE_FRACMODE(table) (table->dacen << 4 | table->dsmen) +#define PLL_CREATE_OSTDIV(table) (table->postdiv2 << 4 | table->postdiv1) + +static inline struct visconti_pll *to_visconti_pll(struct clk_hw *hw) +{ + return container_of(hw, struct visconti_pll, hw); +} + +static void visconti_pll_get_params(struct visconti_pll *pll, + struct visconti_pll_rate_table *rate_table) +{ + u32 postdiv, val; + + val = readl(pll->pll_base + PLL_FRACMODE_REG); + + rate_table->dacen = FIELD_GET(PLL0_FRACMODE_DACEN, val); + rate_table->dsmen = FIELD_GET(PLL0_FRACMODE_DSMEN, val); + + rate_table->fracin = readl(pll->pll_base + PLL_FRACIN_REG) & PLL_FRACIN_MASK; + rate_table->intin = readl(pll->pll_base + PLL_INTIN_REG) & PLL_INTIN_MASK; + rate_table->refdiv = readl(pll->pll_base + PLL_REFDIV_REG) & PLL_REFDIV_MASK; + + postdiv = readl(pll->pll_base + PLL_POSTDIV_REG); + rate_table->postdiv1 = postdiv & PLL_POSTDIV_MASK; + rate_table->postdiv2 = (postdiv >> 4) & PLL_POSTDIV_MASK; +} + +static const struct visconti_pll_rate_table *visconti_get_pll_settings(struct visconti_pll *pll, + unsigned long rate) +{ + const struct visconti_pll_rate_table *rate_table = pll->rate_table; + int i; + + for (i = 0; i < pll->rate_count; i++) + if (rate == rate_table[i].rate) + return &rate_table[i]; + + return NULL; +} + +static unsigned long visconti_get_pll_rate_from_data(struct visconti_pll *pll, + const struct visconti_pll_rate_table *rate) +{ + const struct visconti_pll_rate_table *rate_table = pll->rate_table; + int i; + + for (i = 0; i < pll->rate_count; i++) + if (memcmp(&rate_table[i].dacen, &rate->dacen, + sizeof(*rate) - sizeof(unsigned long)) == 0) + return rate_table[i].rate; + + /* set default */ + return rate_table[0].rate; +} + +static long visconti_pll_round_rate(struct clk_hw *hw, + unsigned long rate, unsigned long *prate) +{ + struct visconti_pll *pll = to_visconti_pll(hw); + const struct visconti_pll_rate_table *rate_table = pll->rate_table; + int i; + + /* Assumming rate_table is in descending order */ + for (i = 0; i < pll->rate_count; i++) + if (rate >= rate_table[i].rate) + return rate_table[i].rate; + + /* return minimum supported value */ + return rate_table[i - 1].rate; +} + +static unsigned long visconti_pll_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct visconti_pll *pll = to_visconti_pll(hw); + struct visconti_pll_rate_table rate_table; + + memset(&rate_table, 0, sizeof(rate_table)); + visconti_pll_get_params(pll, &rate_table); + + return visconti_get_pll_rate_from_data(pll, &rate_table); +} + +static int visconti_pll_set_params(struct visconti_pll *pll, + const struct visconti_pll_rate_table *rate_table) +{ + writel(PLL_CREATE_FRACMODE(rate_table), pll->pll_base + PLL_FRACMODE_REG); + writel(PLL_CREATE_OSTDIV(rate_table), pll->pll_base + PLL_POSTDIV_REG); + writel(rate_table->intin, pll->pll_base + PLL_INTIN_REG); + writel(rate_table->fracin, pll->pll_base + PLL_FRACIN_REG); + writel(rate_table->refdiv, pll->pll_base + PLL_REFDIV_REG); + + return 0; +} + +static int visconti_pll_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct visconti_pll *pll = to_visconti_pll(hw); + const struct visconti_pll_rate_table *rate_table; + + rate_table = visconti_get_pll_settings(pll, rate); + if (!rate_table) + return -EINVAL; + + return visconti_pll_set_params(pll, rate_table); +} + +static int visconti_pll_is_enabled(struct clk_hw *hw) +{ + struct visconti_pll *pll = to_visconti_pll(hw); + u32 reg; + + reg = readl(pll->pll_base + PLL_CTRL_REG); + + return (reg & PLL_PLLEN); +} + +static int visconti_pll_enable(struct clk_hw *hw) +{ + struct visconti_pll *pll = to_visconti_pll(hw); + const struct visconti_pll_rate_table *rate_table = pll->rate_table; + unsigned long flags; + u32 reg; + + if (visconti_pll_is_enabled(hw)) + return 0; + + spin_lock_irqsave(pll->lock, flags); + + writel(PLL_CONFIG_SEL, pll->pll_base + PLL_CONF_REG); + + reg = readl(pll->pll_base + PLL_CTRL_REG); + reg |= PLL_BYPASS; + writel(reg, pll->pll_base + PLL_CTRL_REG); + + visconti_pll_set_params(pll, &rate_table[0]); + + reg = readl(pll->pll_base + PLL_CTRL_REG); + reg &= ~PLL_PLLEN; + writel(reg, pll->pll_base + PLL_CTRL_REG); + + udelay(1); + + reg = readl(pll->pll_base + PLL_CTRL_REG); + reg |= PLL_PLLEN; + writel(reg, pll->pll_base + PLL_CTRL_REG); + + udelay(40); + + reg = readl(pll->pll_base + PLL_CTRL_REG); + reg &= ~PLL_BYPASS; + writel(reg, pll->pll_base + PLL_CTRL_REG); + + spin_unlock_irqrestore(pll->lock, flags); + + return 0; +} + +static void visconti_pll_disable(struct clk_hw *hw) +{ + struct visconti_pll *pll = to_visconti_pll(hw); + unsigned long flags; + u32 reg; + + if (!visconti_pll_is_enabled(hw)) + return; + + spin_lock_irqsave(pll->lock, flags); + + writel(PLL_CONFIG_SEL, pll->pll_base + PLL_CONF_REG); + + reg = readl(pll->pll_base + PLL_CTRL_REG); + reg |= PLL_BYPASS; + writel(reg, pll->pll_base + PLL_CTRL_REG); + + reg = readl(pll->pll_base + PLL_CTRL_REG); + reg &= ~PLL_PLLEN; + writel(reg, pll->pll_base + PLL_CTRL_REG); + + spin_unlock_irqrestore(pll->lock, flags); +} + +static const struct clk_ops visconti_pll_ops = { + .enable = visconti_pll_enable, + .disable = visconti_pll_disable, + .is_enabled = visconti_pll_is_enabled, + .round_rate = visconti_pll_round_rate, + .recalc_rate = visconti_pll_recalc_rate, + .set_rate = visconti_pll_set_rate, +}; + +static struct clk_hw *visconti_register_pll(struct visconti_pll_provider *ctx, + const char *name, + const char *parent_name, + int offset, + const struct visconti_pll_rate_table *rate_table, + spinlock_t *lock) +{ + struct clk_init_data init; + struct visconti_pll *pll; + struct clk *pll_clk; + struct clk_hw *pll_hw_clk; + size_t len; + int ret; + + pll = kzalloc(sizeof(*pll), GFP_KERNEL); + if (!pll) + return ERR_PTR(-ENOMEM); + + init.name = name; + init.flags = CLK_IGNORE_UNUSED; + init.parent_names = &parent_name; + init.num_parents = 1; + + for (len = 0; rate_table[len].rate != 0; ) + len++; + pll->rate_count = len; + pll->rate_table = kmemdup(rate_table, + pll->rate_count * sizeof(struct visconti_pll_rate_table), + GFP_KERNEL); + WARN(!pll->rate_table, "%s: could not allocate rate table for %s\n", __func__, name); + + init.ops = &visconti_pll_ops; + pll->hw.init = &init; + pll->pll_base = ctx->reg_base + offset; + pll->lock = lock; + pll->ctx = ctx; + + pll_hw_clk = &pll->hw; + ret = clk_hw_register(NULL, &pll->hw); + if (ret) { + pr_err("failed to register pll clock %s : %ld\n", name, PTR_ERR(pll_clk)); + kfree(pll); + pll_hw_clk = ERR_PTR(ret); + } + + return pll_hw_clk; +} + +static void visconti_pll_add_lookup(struct visconti_pll_provider *ctx, + struct clk_hw *hw_clk, + unsigned int id) +{ + if (id) + ctx->clk_data.hws[id] = hw_clk; +} + +void __init visconti_register_plls(struct visconti_pll_provider *ctx, + const struct visconti_pll_info *list, + unsigned int nr_plls, + spinlock_t *lock) +{ + int idx; + + for (idx = 0; idx < nr_plls; idx++, list++) { + struct clk_hw *clk; + + clk = visconti_register_pll(ctx, + list->name, + list->parent, + list->base_reg, + list->rate_table, + lock); + if (IS_ERR(clk)) { + pr_err("failed to register clock %s\n", list->name); + continue; + } + + visconti_pll_add_lookup(ctx, clk, list->id); + } +} + +struct visconti_pll_provider * __init visconti_init_pll(struct device_node *np, + void __iomem *base, + unsigned long nr_plls) +{ + struct visconti_pll_provider *ctx; + int i; + + ctx = kzalloc(struct_size(ctx, clk_data.hws, nr_plls), GFP_KERNEL); + if (!ctx) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < nr_plls; ++i) + ctx->clk_data.hws[i] = ERR_PTR(-ENOENT); + + ctx->node = np; + ctx->reg_base = base; + ctx->clk_data.num = nr_plls; + + return ctx; +} diff --git a/drivers/clk/visconti/pll.h b/drivers/clk/visconti/pll.h new file mode 100644 index 000000000000..16dae35ab370 --- /dev/null +++ b/drivers/clk/visconti/pll.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2021 TOSHIBA CORPORATION + * Copyright (c) 2021 Toshiba Electronic Devices & Storage Corporation + * + * Nobuhiro Iwamatsu + */ + +#ifndef _VISCONTI_PLL_H_ +#define _VISCONTI_PLL_H_ + +#include +#include +#include + +struct visconti_pll_provider { + void __iomem *reg_base; + struct regmap *regmap; + struct clk_hw_onecell_data clk_data; + struct device_node *node; +}; + +#define VISCONTI_PLL_RATE(_rate, _dacen, _dsmen, \ + _refdiv, _intin, _fracin, _postdiv1, _postdiv2) \ +{ \ + .rate = _rate, \ + .dacen = _dacen, \ + .dsmen = _dsmen, \ + .refdiv = _refdiv, \ + .intin = _intin, \ + .fracin = _fracin, \ + .postdiv1 = _postdiv1, \ + .postdiv2 = _postdiv2 \ +} + +struct visconti_pll_rate_table { + unsigned long rate; + unsigned int dacen; + unsigned int dsmen; + unsigned int refdiv; + unsigned long intin; + unsigned long fracin; + unsigned int postdiv1; + unsigned int postdiv2; +}; + +struct visconti_pll_info { + unsigned int id; + const char *name; + const char *parent; + unsigned long base_reg; + const struct visconti_pll_rate_table *rate_table; +}; + +struct visconti_pll_provider * __init visconti_init_pll(struct device_node *np, + void __iomem *base, + unsigned long nr_plls); +void visconti_register_plls(struct visconti_pll_provider *ctx, + const struct visconti_pll_info *list, + unsigned int nr_plls, spinlock_t *lock); + +#endif /* _VISCONTI_PLL_H_ */ diff --git a/drivers/clk/visconti/reset.c b/drivers/clk/visconti/reset.c new file mode 100644 index 000000000000..e3c3d7804612 --- /dev/null +++ b/drivers/clk/visconti/reset.c @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Toshiba Visconti ARM SoC reset controller + * + * Copyright (c) 2021 TOSHIBA CORPORATION + * Copyright (c) 2021 Toshiba Electronic Devices & Storage Corporation + * + * Nobuhiro Iwamatsu + */ +#include +#include +#include +#include +#include + +#include "reset.h" + +static inline struct visconti_reset *to_visconti_reset(struct reset_controller_dev *rcdev) +{ + return container_of(rcdev, struct visconti_reset, rcdev); +} + +static int visconti_reset_assert(struct reset_controller_dev *rcdev, unsigned long id) +{ + struct visconti_reset *reset = to_visconti_reset(rcdev); + const struct visconti_reset_data *data = &reset->resets[id]; + u32 rst = BIT(data->rs_idx); + unsigned long flags; + int ret; + + spin_lock_irqsave(reset->lock, flags); + ret = regmap_update_bits(reset->regmap, data->rson_offset, rst, rst); + spin_unlock_irqrestore(reset->lock, flags); + + return ret; +} + +static int visconti_reset_deassert(struct reset_controller_dev *rcdev, unsigned long id) +{ + struct visconti_reset *reset = to_visconti_reset(rcdev); + const struct visconti_reset_data *data = &reset->resets[id]; + u32 rst = BIT(data->rs_idx); + unsigned long flags; + int ret; + + spin_lock_irqsave(reset->lock, flags); + ret = regmap_update_bits(reset->regmap, data->rsoff_offset, rst, rst); + spin_unlock_irqrestore(reset->lock, flags); + + return ret; +} + +static int visconti_reset_reset(struct reset_controller_dev *rcdev, unsigned long id) +{ + visconti_reset_assert(rcdev, id); + udelay(1); + visconti_reset_deassert(rcdev, id); + + return 0; +} + +static int visconti_reset_status(struct reset_controller_dev *rcdev, unsigned long id) +{ + struct visconti_reset *reset = to_visconti_reset(rcdev); + const struct visconti_reset_data *data = &reset->resets[id]; + unsigned long flags; + u32 reg; + int ret; + + spin_lock_irqsave(reset->lock, flags); + ret = regmap_read(reset->regmap, data->rson_offset, ®); + spin_unlock_irqrestore(reset->lock, flags); + if (ret) + return ret; + + return !(reg & data->rs_idx); +} + +const struct reset_control_ops visconti_reset_ops = { + .assert = visconti_reset_assert, + .deassert = visconti_reset_deassert, + .reset = visconti_reset_reset, + .status = visconti_reset_status, +}; + +int visconti_register_reset_controller(struct device *dev, + struct regmap *regmap, + const struct visconti_reset_data *resets, + unsigned int num_resets, + const struct reset_control_ops *reset_ops, + spinlock_t *lock) +{ + struct visconti_reset *reset; + + reset = devm_kzalloc(dev, sizeof(*reset), GFP_KERNEL); + if (!reset) + return -ENOMEM; + + reset->regmap = regmap; + reset->resets = resets; + reset->rcdev.ops = reset_ops; + reset->rcdev.nr_resets = num_resets; + reset->rcdev.of_node = dev->of_node; + reset->lock = lock; + + return devm_reset_controller_register(dev, &reset->rcdev); +} diff --git a/drivers/clk/visconti/reset.h b/drivers/clk/visconti/reset.h new file mode 100644 index 000000000000..229dffcbdc98 --- /dev/null +++ b/drivers/clk/visconti/reset.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Toshiba ARM SoC reset controller driver + * + * Copyright (c) 2021 TOSHIBA CORPORATION + * + * Nobuhiro Iwamatsu + */ + +#ifndef _VISCONTI_RESET_H_ +#define _VISCONTI_RESET_H_ + +#include + +struct visconti_reset_data { + u32 rson_offset; + u32 rsoff_offset; + u8 rs_idx; +}; + +struct visconti_reset { + struct reset_controller_dev rcdev; + struct regmap *regmap; + const struct visconti_reset_data *resets; + spinlock_t *lock; +}; + +extern const struct reset_control_ops visconti_reset_ops; + +int visconti_register_reset_controller(struct device *dev, + struct regmap *regmap, + const struct visconti_reset_data *resets, + unsigned int num_resets, + const struct reset_control_ops *reset_ops, + spinlock_t *lock); +#endif /* _VISCONTI_RESET_H_ */ diff --git a/drivers/clk/x86/clk-fch.c b/drivers/clk/x86/clk-fch.c index 8f7c5142b0f0..fdc060e75839 100644 --- a/drivers/clk/x86/clk-fch.c +++ b/drivers/clk/x86/clk-fch.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MIT /* - * clock framework for AMD Stoney based clocks + * clock framework for AMD FCH controller block * * Copyright 2018 Advanced Micro Devices, Inc. */ @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -26,22 +27,37 @@ #define ST_CLK_GATE 3 #define ST_MAX_CLKS 4 -#define RV_CLK_48M 0 -#define RV_CLK_GATE 1 -#define RV_MAX_CLKS 2 +#define CLK_48M_FIXED 0 +#define CLK_GATE_FIXED 1 +#define CLK_MAX_FIXED 2 + +/* List of supported CPU ids for clk mux with 25Mhz clk support */ +#define AMD_CPU_ID_ST 0x1576 static const char * const clk_oscout1_parents[] = { "clk48MHz", "clk25MHz" }; static struct clk_hw *hws[ST_MAX_CLKS]; +static const struct pci_device_id fch_pci_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_AMD, AMD_CPU_ID_ST) }, + { } +}; + static int fch_clk_probe(struct platform_device *pdev) { struct fch_clk_data *fch_data; + struct pci_dev *rdev; fch_data = dev_get_platdata(&pdev->dev); if (!fch_data || !fch_data->base) return -EINVAL; - if (!fch_data->is_rv) { + rdev = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(0, 0)); + if (!rdev) { + dev_err(&pdev->dev, "FCH device not found\n"); + return -ENODEV; + } + + if (pci_match_id(fch_pci_ids, rdev)) { hws[ST_CLK_48M] = clk_hw_register_fixed_rate(NULL, "clk48MHz", NULL, 0, 48000000); hws[ST_CLK_25M] = clk_hw_register_fixed_rate(NULL, "clk25MHz", @@ -59,34 +75,38 @@ static int fch_clk_probe(struct platform_device *pdev) OSCCLKENB, CLK_GATE_SET_TO_DISABLE, NULL); devm_clk_hw_register_clkdev(&pdev->dev, hws[ST_CLK_GATE], - "oscout1", NULL); + fch_data->name, NULL); } else { - hws[RV_CLK_48M] = clk_hw_register_fixed_rate(NULL, "clk48MHz", + hws[CLK_48M_FIXED] = clk_hw_register_fixed_rate(NULL, "clk48MHz", NULL, 0, 48000000); - hws[RV_CLK_GATE] = clk_hw_register_gate(NULL, "oscout1", + hws[CLK_GATE_FIXED] = clk_hw_register_gate(NULL, "oscout1", "clk48MHz", 0, fch_data->base + MISCCLKCNTL1, - OSCCLKENB, CLK_GATE_SET_TO_DISABLE, NULL); + OSCCLKENB, 0, NULL); - devm_clk_hw_register_clkdev(&pdev->dev, hws[RV_CLK_GATE], - "oscout1", NULL); + devm_clk_hw_register_clkdev(&pdev->dev, hws[CLK_GATE_FIXED], + fch_data->name, NULL); } + pci_dev_put(rdev); return 0; } static int fch_clk_remove(struct platform_device *pdev) { int i, clks; - struct fch_clk_data *fch_data; + struct pci_dev *rdev; - fch_data = dev_get_platdata(&pdev->dev); + rdev = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(0, 0)); + if (!rdev) + return -ENODEV; - clks = fch_data->is_rv ? RV_MAX_CLKS : ST_MAX_CLKS; + clks = pci_match_id(fch_pci_ids, rdev) ? CLK_MAX_FIXED : ST_MAX_CLKS; for (i = 0; i < clks; i++) clk_hw_unregister(hws[i]); + pci_dev_put(rdev); return 0; } diff --git a/drivers/clk/zynq/pll.c b/drivers/clk/zynq/pll.c index 54f4184de89a..e5f8fb704df2 100644 --- a/drivers/clk/zynq/pll.c +++ b/drivers/clk/zynq/pll.c @@ -12,7 +12,7 @@ #include /** - * struct zynq_pll + * struct zynq_pll - pll clock * @hw: Handle between common and hardware-specific interfaces * @pll_ctrl: PLL control register * @pll_status: PLL status register @@ -46,7 +46,7 @@ struct zynq_pll { * @hw: Handle between common and hardware-specific interfaces * @rate: Desired clock frequency * @prate: Clock frequency of parent clock - * Returns frequency closest to @rate the hardware can generate. + * Return: frequency closest to @rate the hardware can generate. */ static long zynq_pll_round_rate(struct clk_hw *hw, unsigned long rate, unsigned long *prate) @@ -66,7 +66,7 @@ static long zynq_pll_round_rate(struct clk_hw *hw, unsigned long rate, * zynq_pll_recalc_rate() - Recalculate clock frequency * @hw: Handle between common and hardware-specific interfaces * @parent_rate: Clock frequency of parent clock - * Returns current clock frequency. + * Return: current clock frequency. */ static unsigned long zynq_pll_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) @@ -87,7 +87,7 @@ static unsigned long zynq_pll_recalc_rate(struct clk_hw *hw, /** * zynq_pll_is_enabled - Check if a clock is enabled * @hw: Handle between common and hardware-specific interfaces - * Returns 1 if the clock is enabled, 0 otherwise. + * Return: 1 if the clock is enabled, 0 otherwise. * * Not sure this is a good idea, but since disabled means bypassed for * this clock implementation we say we are always enabled. @@ -110,7 +110,7 @@ static int zynq_pll_is_enabled(struct clk_hw *hw) /** * zynq_pll_enable - Enable clock * @hw: Handle between common and hardware-specific interfaces - * Returns 0 on success + * Return: 0 on success */ static int zynq_pll_enable(struct clk_hw *hw) { @@ -179,7 +179,7 @@ static const struct clk_ops zynq_pll_ops = { * @pll_status: Pointer to PLL status register * @lock_index: Bit index to this PLL's lock status bit in @pll_status * @lock: Register lock - * Returns handle to the registered clock. + * Return: handle to the registered clock. */ struct clk *clk_register_zynq_pll(const char *name, const char *parent, void __iomem *pll_ctrl, void __iomem *pll_status, u8 lock_index, diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index b51995801c6b..0ca3a943875a 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -510,7 +510,8 @@ config SH_TIMER_MTU2 This hardware comes with 16-bit timer registers. config RENESAS_OSTM - bool "Renesas OSTM timer driver" if COMPILE_TEST + bool "Renesas OSTM timer driver" + depends on ARCH_RENESAS || COMPILE_TEST select CLKSRC_MMIO select TIMER_OF help @@ -671,6 +672,15 @@ config MILBEAUT_TIMER help Enables the support for Milbeaut timer driver. +config MSC313E_TIMER + bool "MSC313E timer driver" if COMPILE_TEST + select TIMER_OF + select CLKSRC_MMIO + help + Enables support for the MStar MSC313E timer driver. + This provides access to multiple interrupt generating + programmable 32-bit free running incrementing counters. + config INGENIC_TIMER bool "Clocksource/timer using the TCU in Ingenic JZ SoCs" default MACH_INGENIC diff --git a/drivers/clocksource/Makefile b/drivers/clocksource/Makefile index c17ee32a7151..fa5f624eadb6 100644 --- a/drivers/clocksource/Makefile +++ b/drivers/clocksource/Makefile @@ -88,3 +88,4 @@ obj-$(CONFIG_CSKY_MP_TIMER) += timer-mp-csky.o obj-$(CONFIG_GX6605S_TIMER) += timer-gx6605s.o obj-$(CONFIG_HYPERV_TIMER) += hyperv_timer.o obj-$(CONFIG_MICROCHIP_PIT64B) += timer-microchip-pit64b.o +obj-$(CONFIG_MSC313E_TIMER) += timer-msc313e.o diff --git a/drivers/clocksource/exynos_mct.c b/drivers/clocksource/exynos_mct.c index 5e3e96d3d1b9..6db3d5511b0f 100644 --- a/drivers/clocksource/exynos_mct.c +++ b/drivers/clocksource/exynos_mct.c @@ -467,7 +467,7 @@ static int exynos4_mct_starting_cpu(unsigned int cpu) evt->tick_resume = set_state_shutdown; evt->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_PERCPU; - evt->rating = MCT_CLKEVENTS_RATING, + evt->rating = MCT_CLKEVENTS_RATING; exynos4_mct_write(TICK_BASE_CNT, mevt->base + MCT_L_TCNTB_OFFSET); @@ -504,11 +504,14 @@ static int exynos4_mct_dying_cpu(unsigned int cpu) return 0; } -static int __init exynos4_timer_resources(struct device_node *np, void __iomem *base) +static int __init exynos4_timer_resources(struct device_node *np) { - int err, cpu; struct clk *mct_clk, *tick_clk; + reg_base = of_iomap(np, 0); + if (!reg_base) + panic("%s: unable to ioremap mct address space\n", __func__); + tick_clk = of_clk_get_by_name(np, "fin_pll"); if (IS_ERR(tick_clk)) panic("%s: unable to determine tick clock rate\n", __func__); @@ -519,9 +522,27 @@ static int __init exynos4_timer_resources(struct device_node *np, void __iomem * panic("%s: unable to retrieve mct clock instance\n", __func__); clk_prepare_enable(mct_clk); - reg_base = base; - if (!reg_base) - panic("%s: unable to ioremap mct address space\n", __func__); + return 0; +} + +static int __init exynos4_timer_interrupts(struct device_node *np, + unsigned int int_type) +{ + int nr_irqs, i, err, cpu; + + mct_int_type = int_type; + + /* This driver uses only one global timer interrupt */ + mct_irqs[MCT_G0_IRQ] = irq_of_parse_and_map(np, MCT_G0_IRQ); + + /* + * Find out the number of local irqs specified. The local + * timer irqs are specified after the four global timer + * irqs are specified. + */ + nr_irqs = of_irq_count(np); + for (i = MCT_L0_IRQ; i < nr_irqs; i++) + mct_irqs[i] = irq_of_parse_and_map(np, i); if (mct_int_type == MCT_INT_PPI) { @@ -581,24 +602,13 @@ out_irq: static int __init mct_init_dt(struct device_node *np, unsigned int int_type) { - u32 nr_irqs, i; int ret; - mct_int_type = int_type; + ret = exynos4_timer_resources(np); + if (ret) + return ret; - /* This driver uses only one global timer interrupt */ - mct_irqs[MCT_G0_IRQ] = irq_of_parse_and_map(np, MCT_G0_IRQ); - - /* - * Find out the number of local irqs specified. The local - * timer irqs are specified after the four global timer - * irqs are specified. - */ - nr_irqs = of_irq_count(np); - for (i = MCT_L0_IRQ; i < nr_irqs; i++) - mct_irqs[i] = irq_of_parse_and_map(np, i); - - ret = exynos4_timer_resources(np, of_iomap(np, 0)); + ret = exynos4_timer_interrupts(np, int_type); if (ret) return ret; diff --git a/drivers/clocksource/renesas-ostm.c b/drivers/clocksource/renesas-ostm.c index 3d06ba66008c..21d1392637b8 100644 --- a/drivers/clocksource/renesas-ostm.c +++ b/drivers/clocksource/renesas-ostm.c @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include #include @@ -159,6 +161,7 @@ static int __init ostm_init_clkevt(struct timer_of *to) static int __init ostm_init(struct device_node *np) { + struct reset_control *rstc; struct timer_of *to; int ret; @@ -166,6 +169,14 @@ static int __init ostm_init(struct device_node *np) if (!to) return -ENOMEM; + rstc = of_reset_control_get_optional_exclusive(np, NULL); + if (IS_ERR(rstc)) { + ret = PTR_ERR(rstc); + goto err_free; + } + + reset_control_deassert(rstc); + to->flags = TIMER_OF_BASE | TIMER_OF_CLOCK; if (system_clock) { /* @@ -178,7 +189,7 @@ static int __init ostm_init(struct device_node *np) ret = timer_of_init(np, to); if (ret) - goto err_free; + goto err_reset; /* * First probed device will be used as system clocksource. Any @@ -203,9 +214,35 @@ static int __init ostm_init(struct device_node *np) err_cleanup: timer_of_cleanup(to); +err_reset: + reset_control_assert(rstc); + reset_control_put(rstc); err_free: kfree(to); return ret; } TIMER_OF_DECLARE(ostm, "renesas,ostm", ostm_init); + +#ifdef CONFIG_ARCH_R9A07G044 +static int __init ostm_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + + return ostm_init(dev->of_node); +} + +static const struct of_device_id ostm_of_table[] = { + { .compatible = "renesas,ostm", }, + { /* sentinel */ } +}; + +static struct platform_driver ostm_device_driver = { + .driver = { + .name = "renesas_ostm", + .of_match_table = of_match_ptr(ostm_of_table), + .suppress_bind_attrs = true, + }, +}; +builtin_platform_driver_probe(ostm_device_driver, ostm_probe); +#endif diff --git a/drivers/clocksource/timer-imx-sysctr.c b/drivers/clocksource/timer-imx-sysctr.c index 18b90fc56bfc..55a8e198d2a1 100644 --- a/drivers/clocksource/timer-imx-sysctr.c +++ b/drivers/clocksource/timer-imx-sysctr.c @@ -20,8 +20,8 @@ #define SYS_CTR_CLK_DIV 0x3 -static void __iomem *sys_ctr_base; -static u32 cmpcr; +static void __iomem *sys_ctr_base __ro_after_init; +static u32 cmpcr __ro_after_init; static void sysctr_timer_enable(bool enable) { @@ -119,7 +119,7 @@ static struct timer_of to_sysctr = { static void __init sysctr_clockevent_init(void) { - to_sysctr.clkevt.cpumask = cpumask_of(0); + to_sysctr.clkevt.cpumask = cpu_possible_mask; clockevents_config_and_register(&to_sysctr.clkevt, timer_of_rate(&to_sysctr), diff --git a/drivers/clocksource/timer-msc313e.c b/drivers/clocksource/timer-msc313e.c new file mode 100644 index 000000000000..54c54ca7c786 --- /dev/null +++ b/drivers/clocksource/timer-msc313e.c @@ -0,0 +1,253 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * MStar timer driver + * + * Copyright (C) 2021 Daniel Palmer + * Copyright (C) 2021 Romain Perier + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_ARM +#include +#endif + +#include "timer-of.h" + +#define TIMER_NAME "msc313e_timer" + +#define MSC313E_REG_CTRL 0x00 +#define MSC313E_REG_CTRL_TIMER_EN BIT(0) +#define MSC313E_REG_CTRL_TIMER_TRIG BIT(1) +#define MSC313E_REG_CTRL_TIMER_INT_EN BIT(8) +#define MSC313E_REG_TIMER_MAX_LOW 0x08 +#define MSC313E_REG_TIMER_MAX_HIGH 0x0c +#define MSC313E_REG_COUNTER_LOW 0x10 +#define MSC313E_REG_COUNTER_HIGH 0x14 +#define MSC313E_REG_TIMER_DIVIDE 0x18 + +#define MSC313E_CLK_DIVIDER 9 +#define TIMER_SYNC_TICKS 3 + +#ifdef CONFIG_ARM +struct msc313e_delay { + void __iomem *base; + struct delay_timer delay; +}; +static struct msc313e_delay msc313e_delay; +#endif + +static void __iomem *msc313e_clksrc; + +static void msc313e_timer_stop(void __iomem *base) +{ + writew(0, base + MSC313E_REG_CTRL); +} + +static void msc313e_timer_start(void __iomem *base, bool periodic) +{ + u16 reg; + + reg = readw(base + MSC313E_REG_CTRL); + if (periodic) + reg |= MSC313E_REG_CTRL_TIMER_EN; + else + reg |= MSC313E_REG_CTRL_TIMER_TRIG; + writew(reg | MSC313E_REG_CTRL_TIMER_INT_EN, base + MSC313E_REG_CTRL); +} + +static void msc313e_timer_setup(void __iomem *base, unsigned long delay) +{ + unsigned long flags; + + local_irq_save(flags); + writew(delay >> 16, base + MSC313E_REG_TIMER_MAX_HIGH); + writew(delay & 0xffff, base + MSC313E_REG_TIMER_MAX_LOW); + local_irq_restore(flags); +} + +static unsigned long msc313e_timer_current_value(void __iomem *base) +{ + unsigned long flags; + u16 l, h; + + local_irq_save(flags); + l = readw(base + MSC313E_REG_COUNTER_LOW); + h = readw(base + MSC313E_REG_COUNTER_HIGH); + local_irq_restore(flags); + + return (((u32)h) << 16 | l); +} + +static int msc313e_timer_clkevt_shutdown(struct clock_event_device *evt) +{ + struct timer_of *timer = to_timer_of(evt); + + msc313e_timer_stop(timer_of_base(timer)); + + return 0; +} + +static int msc313e_timer_clkevt_set_oneshot(struct clock_event_device *evt) +{ + struct timer_of *timer = to_timer_of(evt); + + msc313e_timer_stop(timer_of_base(timer)); + msc313e_timer_start(timer_of_base(timer), false); + + return 0; +} + +static int msc313e_timer_clkevt_set_periodic(struct clock_event_device *evt) +{ + struct timer_of *timer = to_timer_of(evt); + + msc313e_timer_stop(timer_of_base(timer)); + msc313e_timer_setup(timer_of_base(timer), timer_of_period(timer)); + msc313e_timer_start(timer_of_base(timer), true); + + return 0; +} + +static int msc313e_timer_clkevt_next_event(unsigned long evt, struct clock_event_device *clkevt) +{ + struct timer_of *timer = to_timer_of(clkevt); + + msc313e_timer_stop(timer_of_base(timer)); + msc313e_timer_setup(timer_of_base(timer), evt); + msc313e_timer_start(timer_of_base(timer), false); + + return 0; +} + +static irqreturn_t msc313e_timer_clkevt_irq(int irq, void *dev_id) +{ + struct clock_event_device *evt = dev_id; + + evt->event_handler(evt); + + return IRQ_HANDLED; +} + +static u64 msc313e_timer_clksrc_read(struct clocksource *cs) +{ + return msc313e_timer_current_value(msc313e_clksrc) & cs->mask; +} + +#ifdef CONFIG_ARM +static unsigned long msc313e_read_delay_timer_read(void) +{ + return msc313e_timer_current_value(msc313e_delay.base); +} +#endif + +static u64 msc313e_timer_sched_clock_read(void) +{ + return msc313e_timer_current_value(msc313e_clksrc); +} + +static struct clock_event_device msc313e_clkevt = { + .name = TIMER_NAME, + .rating = 300, + .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, + .set_state_shutdown = msc313e_timer_clkevt_shutdown, + .set_state_periodic = msc313e_timer_clkevt_set_periodic, + .set_state_oneshot = msc313e_timer_clkevt_set_oneshot, + .tick_resume = msc313e_timer_clkevt_shutdown, + .set_next_event = msc313e_timer_clkevt_next_event, +}; + +static int __init msc313e_clkevt_init(struct device_node *np) +{ + int ret; + struct timer_of *to; + + to = kzalloc(sizeof(*to), GFP_KERNEL); + if (!to) + return -ENOMEM; + + to->flags = TIMER_OF_IRQ | TIMER_OF_CLOCK | TIMER_OF_BASE; + to->of_irq.handler = msc313e_timer_clkevt_irq; + ret = timer_of_init(np, to); + if (ret) + return ret; + + if (of_device_is_compatible(np, "sstar,ssd20xd-timer")) { + to->of_clk.rate = clk_get_rate(to->of_clk.clk) / MSC313E_CLK_DIVIDER; + to->of_clk.period = DIV_ROUND_UP(to->of_clk.rate, HZ); + writew(MSC313E_CLK_DIVIDER - 1, timer_of_base(to) + MSC313E_REG_TIMER_DIVIDE); + } + + msc313e_clkevt.cpumask = cpu_possible_mask; + msc313e_clkevt.irq = to->of_irq.irq; + to->clkevt = msc313e_clkevt; + + clockevents_config_and_register(&to->clkevt, timer_of_rate(to), + TIMER_SYNC_TICKS, 0xffffffff); + return 0; +} + +static int __init msc313e_clksrc_init(struct device_node *np) +{ + struct timer_of to = { 0 }; + int ret; + u16 reg; + + to.flags = TIMER_OF_BASE | TIMER_OF_CLOCK; + ret = timer_of_init(np, &to); + if (ret) + return ret; + + msc313e_clksrc = timer_of_base(&to); + reg = readw(msc313e_clksrc + MSC313E_REG_CTRL); + reg |= MSC313E_REG_CTRL_TIMER_EN; + writew(reg, msc313e_clksrc + MSC313E_REG_CTRL); + +#ifdef CONFIG_ARM + msc313e_delay.base = timer_of_base(&to); + msc313e_delay.delay.read_current_timer = msc313e_read_delay_timer_read; + msc313e_delay.delay.freq = timer_of_rate(&to); + + register_current_timer_delay(&msc313e_delay.delay); +#endif + + sched_clock_register(msc313e_timer_sched_clock_read, 32, timer_of_rate(&to)); + return clocksource_mmio_init(timer_of_base(&to), TIMER_NAME, timer_of_rate(&to), 300, 32, + msc313e_timer_clksrc_read); +} + +static int __init msc313e_timer_init(struct device_node *np) +{ + int ret = 0; + static int num_called; + + switch (num_called) { + case 0: + ret = msc313e_clksrc_init(np); + if (ret) + return ret; + break; + + default: + ret = msc313e_clkevt_init(np); + if (ret) + return ret; + break; + } + + num_called++; + + return 0; +} + +TIMER_OF_DECLARE(msc313, "mstar,msc313e-timer", msc313e_timer_init); +TIMER_OF_DECLARE(ssd20xd, "sstar,ssd20xd-timer", msc313e_timer_init); diff --git a/drivers/clocksource/timer-pistachio.c b/drivers/clocksource/timer-pistachio.c index 6f37181a8c63..69c069e6f0a2 100644 --- a/drivers/clocksource/timer-pistachio.c +++ b/drivers/clocksource/timer-pistachio.c @@ -71,7 +71,8 @@ static u64 notrace pistachio_clocksource_read_cycles(struct clocksource *cs) { struct pistachio_clocksource *pcs = to_pistachio_clocksource(cs); - u32 counter, overflow; + __maybe_unused u32 overflow; + u32 counter; unsigned long flags; /* diff --git a/drivers/dma/mv_xor_v2.c b/drivers/dma/mv_xor_v2.c index 9b0d463f89bb..9c8b4084ba2f 100644 --- a/drivers/dma/mv_xor_v2.c +++ b/drivers/dma/mv_xor_v2.c @@ -149,7 +149,7 @@ struct mv_xor_v2_descriptor { * @desc_size: HW descriptor size * @npendings: number of pending descriptors (for which tx_submit has * @hw_queue_idx: HW queue index - * @msi_desc: local interrupt descriptor information + * @irq: The Linux interrupt number * been called, but not yet issue_pending) */ struct mv_xor_v2_device { @@ -168,7 +168,7 @@ struct mv_xor_v2_device { int desc_size; unsigned int npendings; unsigned int hw_queue_idx; - struct msi_desc *msi_desc; + unsigned int irq; }; /** @@ -718,7 +718,6 @@ static int mv_xor_v2_probe(struct platform_device *pdev) int i, ret = 0; struct dma_device *dma_dev; struct mv_xor_v2_sw_desc *sw_desc; - struct msi_desc *msi_desc; BUILD_BUG_ON(sizeof(struct mv_xor_v2_descriptor) != MV_XOR_V2_EXT_DESC_SIZE); @@ -770,14 +769,9 @@ static int mv_xor_v2_probe(struct platform_device *pdev) if (ret) goto disable_clk; - msi_desc = first_msi_entry(&pdev->dev); - if (!msi_desc) { - ret = -ENODEV; - goto free_msi_irqs; - } - xor_dev->msi_desc = msi_desc; + xor_dev->irq = msi_get_virq(&pdev->dev, 0); - ret = devm_request_irq(&pdev->dev, msi_desc->irq, + ret = devm_request_irq(&pdev->dev, xor_dev->irq, mv_xor_v2_interrupt_handler, 0, dev_name(&pdev->dev), xor_dev); if (ret) @@ -892,7 +886,7 @@ static int mv_xor_v2_remove(struct platform_device *pdev) xor_dev->desc_size * MV_XOR_V2_DESC_NUM, xor_dev->hw_desq_virt, xor_dev->hw_desq); - devm_free_irq(&pdev->dev, xor_dev->msi_desc->irq, xor_dev); + devm_free_irq(&pdev->dev, xor_dev->irq, xor_dev); platform_msi_domain_free_irqs(&pdev->dev); diff --git a/drivers/dma/qcom/hidma.c b/drivers/dma/qcom/hidma.c index 23d64489d25f..65d054bb11aa 100644 --- a/drivers/dma/qcom/hidma.c +++ b/drivers/dma/qcom/hidma.c @@ -666,7 +666,7 @@ static void hidma_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg) struct device *dev = msi_desc_to_dev(desc); struct hidma_dev *dmadev = dev_get_drvdata(dev); - if (!desc->platform.msi_index) { + if (!desc->msi_index) { writel(msg->address_lo, dmadev->dev_evca + 0x118); writel(msg->address_hi, dmadev->dev_evca + 0x11C); writel(msg->data, dmadev->dev_evca + 0x120); @@ -678,11 +678,13 @@ static void hidma_free_msis(struct hidma_dev *dmadev) { #ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN struct device *dev = dmadev->ddev.dev; - struct msi_desc *desc; + int i, virq; - /* free allocated MSI interrupts above */ - for_each_msi_entry(desc, dev) - devm_free_irq(dev, desc->irq, &dmadev->lldev); + for (i = 0; i < HIDMA_MSI_INTS; i++) { + virq = msi_get_virq(dev, i); + if (virq) + devm_free_irq(dev, virq, &dmadev->lldev); + } platform_msi_domain_free_irqs(dev); #endif @@ -692,45 +694,37 @@ static int hidma_request_msi(struct hidma_dev *dmadev, struct platform_device *pdev) { #ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN - int rc; - struct msi_desc *desc; - struct msi_desc *failed_desc = NULL; + int rc, i, virq; rc = platform_msi_domain_alloc_irqs(&pdev->dev, HIDMA_MSI_INTS, hidma_write_msi_msg); if (rc) return rc; - for_each_msi_entry(desc, &pdev->dev) { - if (!desc->platform.msi_index) - dmadev->msi_virqbase = desc->irq; - - rc = devm_request_irq(&pdev->dev, desc->irq, + for (i = 0; i < HIDMA_MSI_INTS; i++) { + virq = msi_get_virq(&pdev->dev, i); + rc = devm_request_irq(&pdev->dev, virq, hidma_chirq_handler_msi, 0, "qcom-hidma-msi", &dmadev->lldev); - if (rc) { - failed_desc = desc; + if (rc) break; - } + if (!i) + dmadev->msi_virqbase = virq; } if (rc) { /* free allocated MSI interrupts above */ - for_each_msi_entry(desc, &pdev->dev) { - if (desc == failed_desc) - break; - devm_free_irq(&pdev->dev, desc->irq, - &dmadev->lldev); + for (--i; i >= 0; i--) { + virq = msi_get_virq(&pdev->dev, i); + devm_free_irq(&pdev->dev, virq, &dmadev->lldev); } + dev_warn(&pdev->dev, + "failed to request MSI irq, falling back to wired IRQ\n"); } else { /* Add callback to free MSIs on teardown */ hidma_ll_setup_irq(dmadev->lldev, true); - } - if (rc) - dev_warn(&pdev->dev, - "failed to request MSI irq, falling back to wired IRQ\n"); return rc; #else return -EINVAL; diff --git a/drivers/dma/ti/k3-udma-private.c b/drivers/dma/ti/k3-udma-private.c index aada84f40723..d4f1e4e9603a 100644 --- a/drivers/dma/ti/k3-udma-private.c +++ b/drivers/dma/ti/k3-udma-private.c @@ -168,8 +168,7 @@ int xudma_pktdma_tflow_get_irq(struct udma_dev *ud, int udma_tflow_id) { const struct udma_oes_offsets *oes = &ud->soc_data->oes; - return ti_sci_inta_msi_get_virq(ud->dev, udma_tflow_id + - oes->pktdma_tchan_flow); + return msi_get_virq(ud->dev, udma_tflow_id + oes->pktdma_tchan_flow); } EXPORT_SYMBOL(xudma_pktdma_tflow_get_irq); @@ -177,7 +176,6 @@ int xudma_pktdma_rflow_get_irq(struct udma_dev *ud, int udma_rflow_id) { const struct udma_oes_offsets *oes = &ud->soc_data->oes; - return ti_sci_inta_msi_get_virq(ud->dev, udma_rflow_id + - oes->pktdma_rchan_flow); + return msi_get_virq(ud->dev, udma_rflow_id + oes->pktdma_rchan_flow); } EXPORT_SYMBOL(xudma_pktdma_rflow_get_irq); diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index 6e56d1cef5ee..c542ba5d383d 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -2313,8 +2313,7 @@ static int udma_alloc_chan_resources(struct dma_chan *chan) /* Event from UDMA (TR events) only needed for slave TR mode channels */ if (is_slave_direction(uc->config.dir) && !uc->config.pkt_mode) { - uc->irq_num_udma = ti_sci_inta_msi_get_virq(ud->dev, - irq_udma_idx); + uc->irq_num_udma = msi_get_virq(ud->dev, irq_udma_idx); if (uc->irq_num_udma <= 0) { dev_err(ud->dev, "Failed to get udma irq (index: %u)\n", irq_udma_idx); @@ -2486,7 +2485,7 @@ static int bcdma_alloc_chan_resources(struct dma_chan *chan) uc->psil_paired = true; } - uc->irq_num_ring = ti_sci_inta_msi_get_virq(ud->dev, irq_ring_idx); + uc->irq_num_ring = msi_get_virq(ud->dev, irq_ring_idx); if (uc->irq_num_ring <= 0) { dev_err(ud->dev, "Failed to get ring irq (index: %u)\n", irq_ring_idx); @@ -2503,8 +2502,7 @@ static int bcdma_alloc_chan_resources(struct dma_chan *chan) /* Event from BCDMA (TR events) only needed for slave channels */ if (is_slave_direction(uc->config.dir)) { - uc->irq_num_udma = ti_sci_inta_msi_get_virq(ud->dev, - irq_udma_idx); + uc->irq_num_udma = msi_get_virq(ud->dev, irq_udma_idx); if (uc->irq_num_udma <= 0) { dev_err(ud->dev, "Failed to get bcdma irq (index: %u)\n", irq_udma_idx); @@ -2672,7 +2670,7 @@ static int pktdma_alloc_chan_resources(struct dma_chan *chan) uc->psil_paired = true; - uc->irq_num_ring = ti_sci_inta_msi_get_virq(ud->dev, irq_ring_idx); + uc->irq_num_ring = msi_get_virq(ud->dev, irq_ring_idx); if (uc->irq_num_ring <= 0) { dev_err(ud->dev, "Failed to get ring irq (index: %u)\n", irq_ring_idx); @@ -5336,9 +5334,9 @@ static int udma_probe(struct platform_device *pdev) if (IS_ERR(ud->ringacc)) return PTR_ERR(ud->ringacc); - dev->msi_domain = of_msi_get_domain(dev, dev->of_node, + dev->msi.domain = of_msi_get_domain(dev, dev->of_node, DOMAIN_BUS_TI_SCI_INTA_MSI); - if (!dev->msi_domain) { + if (!dev->msi.domain) { dev_err(dev, "Failed to get MSI domain\n"); return -EPROBE_DEFER; } diff --git a/drivers/infiniband/core/cache.c b/drivers/infiniband/core/cache.c index b79f816a7203..f6aa1a964573 100644 --- a/drivers/infiniband/core/cache.c +++ b/drivers/infiniband/core/cache.c @@ -956,7 +956,7 @@ int rdma_query_gid(struct ib_device *device, u32 port_num, { struct ib_gid_table *table; unsigned long flags; - int res = -EINVAL; + int res; if (!rdma_is_port_valid(device, port_num)) return -EINVAL; @@ -964,9 +964,15 @@ int rdma_query_gid(struct ib_device *device, u32 port_num, table = rdma_gid_table(device, port_num); read_lock_irqsave(&table->rwlock, flags); - if (index < 0 || index >= table->sz || - !is_gid_entry_valid(table->data_vec[index])) + if (index < 0 || index >= table->sz) { + res = -EINVAL; goto done; + } + + if (!is_gid_entry_valid(table->data_vec[index])) { + res = -ENOENT; + goto done; + } memcpy(gid, &table->data_vec[index]->attr.gid, sizeof(*gid)); res = 0; diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index 835ac54d4a24..27a00ce2e101 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -766,6 +766,7 @@ static int cma_resolve_ib_dev(struct rdma_id_private *id_priv) unsigned int p; u16 pkey, index; enum ib_port_state port_state; + int ret; int i; cma_dev = NULL; @@ -784,9 +785,14 @@ static int cma_resolve_ib_dev(struct rdma_id_private *id_priv) if (ib_get_cached_port_state(cur_dev->device, p, &port_state)) continue; - for (i = 0; !rdma_query_gid(cur_dev->device, - p, i, &gid); - i++) { + + for (i = 0; i < cur_dev->device->port_data[p].immutable.gid_tbl_len; + ++i) { + ret = rdma_query_gid(cur_dev->device, p, i, + &gid); + if (ret) + continue; + if (!memcmp(&gid, dgid, sizeof(gid))) { cma_dev = cur_dev; sgid = gid; @@ -4033,8 +4039,7 @@ static int cma_resolve_ib_udp(struct rdma_id_private *id_priv, memset(&req, 0, sizeof req); offset = cma_user_data_offset(id_priv); - req.private_data_len = offset + conn_param->private_data_len; - if (req.private_data_len < conn_param->private_data_len) + if (check_add_overflow(offset, conn_param->private_data_len, &req.private_data_len)) return -EINVAL; if (req.private_data_len) { @@ -4093,8 +4098,7 @@ static int cma_connect_ib(struct rdma_id_private *id_priv, memset(&req, 0, sizeof req); offset = cma_user_data_offset(id_priv); - req.private_data_len = offset + conn_param->private_data_len; - if (req.private_data_len < conn_param->private_data_len) + if (check_add_overflow(offset, conn_param->private_data_len, &req.private_data_len)) return -EINVAL; if (req.private_data_len) { diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index 22a4adda7981..a311df07b1bd 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -2461,7 +2461,8 @@ int ib_find_gid(struct ib_device *device, union ib_gid *gid, ++i) { ret = rdma_query_gid(device, port, i, &tmp_gid); if (ret) - return ret; + continue; + if (!memcmp(&tmp_gid, gid, sizeof *gid)) { *port_num = port; if (index) diff --git a/drivers/infiniband/core/sysfs.c b/drivers/infiniband/core/sysfs.c index a3f84b50c46a..84c53bd2a52d 100644 --- a/drivers/infiniband/core/sysfs.c +++ b/drivers/infiniband/core/sysfs.c @@ -433,6 +433,7 @@ static struct attribute *port_default_attrs[] = { &ib_port_attr_link_layer.attr, NULL }; +ATTRIBUTE_GROUPS(port_default); static ssize_t print_ndev(const struct ib_gid_attr *gid_attr, char *buf) { @@ -774,7 +775,7 @@ static void ib_port_gid_attr_release(struct kobject *kobj) static struct kobj_type port_type = { .release = ib_port_release, .sysfs_ops = &port_sysfs_ops, - .default_attrs = port_default_attrs + .default_groups = port_default_groups, }; static struct kobj_type gid_attr_type = { diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/umem_odp.c index 7a47343d11f9..aead24c1a682 100644 --- a/drivers/infiniband/core/umem_odp.c +++ b/drivers/infiniband/core/umem_odp.c @@ -227,7 +227,6 @@ struct ib_umem_odp *ib_umem_odp_get(struct ib_device *device, const struct mmu_interval_notifier_ops *ops) { struct ib_umem_odp *umem_odp; - struct mm_struct *mm; int ret; if (WARN_ON_ONCE(!(access & IB_ACCESS_ON_DEMAND))) @@ -241,7 +240,7 @@ struct ib_umem_odp *ib_umem_odp_get(struct ib_device *device, umem_odp->umem.length = size; umem_odp->umem.address = addr; umem_odp->umem.writable = ib_access_writable(access); - umem_odp->umem.owning_mm = mm = current->mm; + umem_odp->umem.owning_mm = current->mm; umem_odp->notifier.ops = ops; umem_odp->page_shift = PAGE_SHIFT; diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index d1345d76d9b1..6b6393176b3c 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -1399,7 +1399,6 @@ static int create_qp(struct uverbs_attr_bundle *attrs, attr.sq_sig_type = cmd->sq_sig_all ? IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR; attr.qp_type = cmd->qp_type; - attr.create_flags = 0; attr.cap.max_send_wr = cmd->max_send_wr; attr.cap.max_recv_wr = cmd->max_recv_wr; diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 29cc0d14399a..3224f18a66e5 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -262,13 +262,12 @@ void bnxt_re_query_fw_str(struct ib_device *ibdev, char *str) int bnxt_re_query_pkey(struct ib_device *ibdev, u32 port_num, u16 index, u16 *pkey) { - struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + if (index > 0) + return -EINVAL; - /* Ignore port_num */ + *pkey = IB_DEFAULT_PKEY_FULL; - memset(pkey, 0, sizeof(*pkey)); - return bnxt_qplib_get_pkey(&rdev->qplib_res, - &rdev->qplib_res.pkey_tbl, index, pkey); + return 0; } int bnxt_re_query_gid(struct ib_device *ibdev, u32 port_num, diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c index b44944fb9b24..3d6834d3d4fb 100644 --- a/drivers/infiniband/hw/bnxt_re/main.c +++ b/drivers/infiniband/hw/bnxt_re/main.c @@ -893,7 +893,6 @@ static int bnxt_re_srqn_handler(struct bnxt_qplib_nq *nq, struct bnxt_re_srq *srq = container_of(handle, struct bnxt_re_srq, qplib_srq); struct ib_event ib_event; - int rc = 0; ib_event.device = &srq->rdev->ibdev; ib_event.element.srq = &srq->ib_srq; @@ -907,7 +906,7 @@ static int bnxt_re_srqn_handler(struct bnxt_qplib_nq *nq, (*srq->ib_srq.event_handler)(&ib_event, srq->ib_srq.srq_context); } - return rc; + return 0; } static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq, diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c index ca88849559bf..96e581ced50e 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c @@ -46,6 +46,7 @@ #include #include #include +#include #include "roce_hsi.h" @@ -1232,7 +1233,7 @@ int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) struct bnxt_qplib_rcfw *rcfw = res->rcfw; struct cmdq_modify_qp req; struct creq_modify_qp_resp resp; - u16 cmd_flags = 0, pkey; + u16 cmd_flags = 0; u32 temp32[4]; u32 bmask; int rc; @@ -1255,11 +1256,9 @@ int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS) req.access = qp->access; - if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY) { - if (!bnxt_qplib_get_pkey(res, &res->pkey_tbl, - qp->pkey_index, &pkey)) - req.pkey = cpu_to_le16(pkey); - } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY) + req.pkey = cpu_to_le16(IB_DEFAULT_PKEY_FULL); + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_QKEY) req.qkey = cpu_to_le32(qp->qkey); diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c index 3de854727460..061b2895dd9b 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c @@ -555,7 +555,7 @@ skip_ctx_setup: void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_rcfw *rcfw) { - kfree(rcfw->cmdq.cmdq_bitmap); + bitmap_free(rcfw->cmdq.cmdq_bitmap); kfree(rcfw->qp_tbl); kfree(rcfw->crsqe_tbl); bnxt_qplib_free_hwq(rcfw->res, &rcfw->cmdq.hwq); @@ -572,7 +572,6 @@ int bnxt_qplib_alloc_rcfw_channel(struct bnxt_qplib_res *res, struct bnxt_qplib_sg_info sginfo = {}; struct bnxt_qplib_cmdq_ctx *cmdq; struct bnxt_qplib_creq_ctx *creq; - u32 bmap_size = 0; rcfw->pdev = res->pdev; cmdq = &rcfw->cmdq; @@ -613,13 +612,10 @@ int bnxt_qplib_alloc_rcfw_channel(struct bnxt_qplib_res *res, if (!rcfw->crsqe_tbl) goto fail; - bmap_size = BITS_TO_LONGS(rcfw->cmdq_depth) * sizeof(unsigned long); - cmdq->cmdq_bitmap = kzalloc(bmap_size, GFP_KERNEL); + cmdq->cmdq_bitmap = bitmap_zalloc(rcfw->cmdq_depth, GFP_KERNEL); if (!cmdq->cmdq_bitmap) goto fail; - cmdq->bmap_size = bmap_size; - /* Allocate one extra to hold the QP1 entries */ rcfw->qp_tbl_size = qp_tbl_sz + 1; rcfw->qp_tbl = kcalloc(rcfw->qp_tbl_size, sizeof(struct bnxt_qplib_qp_node), @@ -667,8 +663,8 @@ void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw) iounmap(cmdq->cmdq_mbox.reg.bar_reg); iounmap(creq->creq_db.reg.bar_reg); - indx = find_first_bit(cmdq->cmdq_bitmap, cmdq->bmap_size); - if (indx != cmdq->bmap_size) + indx = find_first_bit(cmdq->cmdq_bitmap, rcfw->cmdq_depth); + if (indx != rcfw->cmdq_depth) dev_err(&rcfw->pdev->dev, "disabling RCFW with pending cmd-bit %lx\n", indx); diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h index 82faa4e4cda8..0a3d8e7da3d4 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h +++ b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h @@ -152,7 +152,6 @@ struct bnxt_qplib_cmdq_ctx { wait_queue_head_t waitq; unsigned long flags; unsigned long *cmdq_bitmap; - u32 bmap_size; u32 seq_num; }; diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c index bc1ba4b51ba4..126d4f26f75a 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_res.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.c @@ -649,31 +649,6 @@ static void bnxt_qplib_init_sgid_tbl(struct bnxt_qplib_sgid_tbl *sgid_tbl, memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max); } -static void bnxt_qplib_free_pkey_tbl(struct bnxt_qplib_res *res, - struct bnxt_qplib_pkey_tbl *pkey_tbl) -{ - if (!pkey_tbl->tbl) - dev_dbg(&res->pdev->dev, "PKEY tbl not present\n"); - else - kfree(pkey_tbl->tbl); - - pkey_tbl->tbl = NULL; - pkey_tbl->max = 0; - pkey_tbl->active = 0; -} - -static int bnxt_qplib_alloc_pkey_tbl(struct bnxt_qplib_res *res, - struct bnxt_qplib_pkey_tbl *pkey_tbl, - u16 max) -{ - pkey_tbl->tbl = kcalloc(max, sizeof(u16), GFP_KERNEL); - if (!pkey_tbl->tbl) - return -ENOMEM; - - pkey_tbl->max = max; - return 0; -}; - /* PDs */ int bnxt_qplib_alloc_pd(struct bnxt_qplib_pd_tbl *pdt, struct bnxt_qplib_pd *pd) { @@ -843,24 +818,6 @@ unmap_io: return -ENOMEM; } -/* PKEYs */ -static void bnxt_qplib_cleanup_pkey_tbl(struct bnxt_qplib_pkey_tbl *pkey_tbl) -{ - memset(pkey_tbl->tbl, 0, sizeof(u16) * pkey_tbl->max); - pkey_tbl->active = 0; -} - -static void bnxt_qplib_init_pkey_tbl(struct bnxt_qplib_res *res, - struct bnxt_qplib_pkey_tbl *pkey_tbl) -{ - u16 pkey = 0xFFFF; - - memset(pkey_tbl->tbl, 0, sizeof(u16) * pkey_tbl->max); - - /* pkey default = 0xFFFF */ - bnxt_qplib_add_pkey(res, pkey_tbl, &pkey, false); -} - /* Stats */ static void bnxt_qplib_free_stats_ctx(struct pci_dev *pdev, struct bnxt_qplib_stats *stats) @@ -891,21 +848,18 @@ static int bnxt_qplib_alloc_stats_ctx(struct pci_dev *pdev, void bnxt_qplib_cleanup_res(struct bnxt_qplib_res *res) { - bnxt_qplib_cleanup_pkey_tbl(&res->pkey_tbl); bnxt_qplib_cleanup_sgid_tbl(res, &res->sgid_tbl); } int bnxt_qplib_init_res(struct bnxt_qplib_res *res) { bnxt_qplib_init_sgid_tbl(&res->sgid_tbl, res->netdev); - bnxt_qplib_init_pkey_tbl(res, &res->pkey_tbl); return 0; } void bnxt_qplib_free_res(struct bnxt_qplib_res *res) { - bnxt_qplib_free_pkey_tbl(res, &res->pkey_tbl); bnxt_qplib_free_sgid_tbl(res, &res->sgid_tbl); bnxt_qplib_free_pd_tbl(&res->pd_tbl); bnxt_qplib_free_dpi_tbl(res, &res->dpi_tbl); @@ -924,10 +878,6 @@ int bnxt_qplib_alloc_res(struct bnxt_qplib_res *res, struct pci_dev *pdev, if (rc) goto fail; - rc = bnxt_qplib_alloc_pkey_tbl(res, &res->pkey_tbl, dev_attr->max_pkey); - if (rc) - goto fail; - rc = bnxt_qplib_alloc_pd_tbl(res, &res->pd_tbl, dev_attr->max_pd); if (rc) goto fail; diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.h b/drivers/infiniband/hw/bnxt_re/qplib_res.h index e1411a2352a7..982e2c96dac2 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_res.h +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.h @@ -185,12 +185,6 @@ struct bnxt_qplib_sgid_tbl { u8 *vlan; }; -struct bnxt_qplib_pkey_tbl { - u16 *tbl; - u16 max; - u16 active; -}; - struct bnxt_qplib_dpi { u32 dpi; void __iomem *dbr; @@ -258,7 +252,6 @@ struct bnxt_qplib_res { struct bnxt_qplib_rcfw *rcfw; struct bnxt_qplib_pd_tbl pd_tbl; struct bnxt_qplib_sgid_tbl sgid_tbl; - struct bnxt_qplib_pkey_tbl pkey_tbl; struct bnxt_qplib_dpi_tbl dpi_tbl; bool prio; bool is_vf; diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.c b/drivers/infiniband/hw/bnxt_re/qplib_sp.c index 379e715ebd30..b802981b7171 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_sp.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.c @@ -146,17 +146,7 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw, attr->max_srq = le16_to_cpu(sb->max_srq); attr->max_srq_wqes = le32_to_cpu(sb->max_srq_wr) - 1; attr->max_srq_sges = sb->max_srq_sge; - attr->max_pkey = le32_to_cpu(sb->max_pkeys); - /* - * Some versions of FW reports more than 0xFFFF. - * Restrict it for now to 0xFFFF to avoid - * reporting trucated value - */ - if (attr->max_pkey > 0xFFFF) { - /* ib_port_attr::pkey_tbl_len is u16 */ - attr->max_pkey = 0xFFFF; - } - + attr->max_pkey = 1; attr->max_inline_data = le32_to_cpu(sb->max_inline_data); attr->l2_db_size = (sb->l2_db_space_size + 1) * (0x01 << RCFW_DBR_BASE_PAGE_SHIFT); @@ -414,93 +404,6 @@ int bnxt_qplib_update_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, return rc; } -/* pkeys */ -int bnxt_qplib_get_pkey(struct bnxt_qplib_res *res, - struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 index, - u16 *pkey) -{ - if (index == 0xFFFF) { - *pkey = 0xFFFF; - return 0; - } - if (index >= pkey_tbl->max) { - dev_err(&res->pdev->dev, - "Index %d exceeded PKEY table max (%d)\n", - index, pkey_tbl->max); - return -EINVAL; - } - memcpy(pkey, &pkey_tbl->tbl[index], sizeof(*pkey)); - return 0; -} - -int bnxt_qplib_del_pkey(struct bnxt_qplib_res *res, - struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey, - bool update) -{ - int i, rc = 0; - - if (!pkey_tbl) { - dev_err(&res->pdev->dev, "PKEY table not allocated\n"); - return -EINVAL; - } - - /* Do we need a pkey_lock here? */ - if (!pkey_tbl->active) { - dev_err(&res->pdev->dev, "PKEY table has no active entries\n"); - return -ENOMEM; - } - for (i = 0; i < pkey_tbl->max; i++) { - if (!memcmp(&pkey_tbl->tbl[i], pkey, sizeof(*pkey))) - break; - } - if (i == pkey_tbl->max) { - dev_err(&res->pdev->dev, - "PKEY 0x%04x not found in the pkey table\n", *pkey); - return -ENOMEM; - } - memset(&pkey_tbl->tbl[i], 0, sizeof(*pkey)); - pkey_tbl->active--; - - /* unlock */ - return rc; -} - -int bnxt_qplib_add_pkey(struct bnxt_qplib_res *res, - struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey, - bool update) -{ - int i, free_idx, rc = 0; - - if (!pkey_tbl) { - dev_err(&res->pdev->dev, "PKEY table not allocated\n"); - return -EINVAL; - } - - /* Do we need a pkey_lock here? */ - if (pkey_tbl->active == pkey_tbl->max) { - dev_err(&res->pdev->dev, "PKEY table is full\n"); - return -ENOMEM; - } - free_idx = pkey_tbl->max; - for (i = 0; i < pkey_tbl->max; i++) { - if (!memcmp(&pkey_tbl->tbl[i], pkey, sizeof(*pkey))) - return -EALREADY; - else if (!pkey_tbl->tbl[i] && free_idx == pkey_tbl->max) - free_idx = i; - } - if (free_idx == pkey_tbl->max) { - dev_err(&res->pdev->dev, - "PKEY table is FULL but count is not MAX??\n"); - return -ENOMEM; - } - /* Add PKEY to the pkey_tbl */ - memcpy(&pkey_tbl->tbl[free_idx], pkey, sizeof(*pkey)); - pkey_tbl->active++; - - /* unlock */ - return rc; -} - /* AH */ int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah, bool block) diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.h b/drivers/infiniband/hw/bnxt_re/qplib_sp.h index a18f568cb23e..5939e8fc8353 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_sp.h +++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.h @@ -255,15 +255,6 @@ int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, int bnxt_qplib_update_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, struct bnxt_qplib_gid *gid, u16 gid_idx, const u8 *smac); -int bnxt_qplib_get_pkey(struct bnxt_qplib_res *res, - struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 index, - u16 *pkey); -int bnxt_qplib_del_pkey(struct bnxt_qplib_res *res, - struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey, - bool update); -int bnxt_qplib_add_pkey(struct bnxt_qplib_res *res, - struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey, - bool update); int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw, struct bnxt_qplib_dev_attr *attr, bool vf); int bnxt_qplib_set_func_resources(struct bnxt_qplib_res *res, diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c index 913f39ee4416..c16017f6e8db 100644 --- a/drivers/infiniband/hw/cxgb4/cm.c +++ b/drivers/infiniband/hw/cxgb4/cm.c @@ -2471,7 +2471,8 @@ static int accept_cr(struct c4iw_ep *ep, struct sk_buff *skb, skb_get(skb); rpl = cplhdr(skb); if (!is_t4(adapter_type)) { - skb_trim(skb, roundup(sizeof(*rpl5), 16)); + BUILD_BUG_ON(sizeof(*rpl5) != roundup(sizeof(*rpl5), 16)); + skb_trim(skb, sizeof(*rpl5)); rpl5 = (void *)rpl; INIT_TP_WR(rpl5, ep->hwtid); } else { @@ -2487,7 +2488,7 @@ static int accept_cr(struct c4iw_ep *ep, struct sk_buff *skb, opt2 |= CONG_CNTRL_V(CONG_ALG_TAHOE); opt2 |= T5_ISS_F; rpl5 = (void *)rpl; - memset(&rpl5->iss, 0, roundup(sizeof(*rpl5)-sizeof(*rpl), 16)); + memset_after(rpl5, 0, iss); if (peer2peer) isn += 4; rpl5->iss = cpu_to_be32(isn); diff --git a/drivers/infiniband/hw/cxgb4/id_table.c b/drivers/infiniband/hw/cxgb4/id_table.c index 724d23297b35..f64e7e02b129 100644 --- a/drivers/infiniband/hw/cxgb4/id_table.c +++ b/drivers/infiniband/hw/cxgb4/id_table.c @@ -59,7 +59,7 @@ u32 c4iw_id_alloc(struct c4iw_id_table *alloc) alloc->last = obj + 1; if (alloc->last >= alloc->max) alloc->last = 0; - set_bit(obj, alloc->table); + __set_bit(obj, alloc->table); obj += alloc->start; } else obj = -1; @@ -75,37 +75,32 @@ void c4iw_id_free(struct c4iw_id_table *alloc, u32 obj) obj -= alloc->start; spin_lock_irqsave(&alloc->lock, flags); - clear_bit(obj, alloc->table); + __clear_bit(obj, alloc->table); spin_unlock_irqrestore(&alloc->lock, flags); } int c4iw_id_table_alloc(struct c4iw_id_table *alloc, u32 start, u32 num, u32 reserved, u32 flags) { - int i; - alloc->start = start; alloc->flags = flags; if (flags & C4IW_ID_TABLE_F_RANDOM) alloc->last = prandom_u32() % RANDOM_SKIP; else alloc->last = 0; - alloc->max = num; + alloc->max = num; spin_lock_init(&alloc->lock); - alloc->table = kmalloc_array(BITS_TO_LONGS(num), sizeof(long), - GFP_KERNEL); + alloc->table = bitmap_zalloc(num, GFP_KERNEL); if (!alloc->table) return -ENOMEM; - bitmap_zero(alloc->table, num); if (!(alloc->flags & C4IW_ID_TABLE_F_EMPTY)) - for (i = 0; i < reserved; ++i) - set_bit(i, alloc->table); + bitmap_set(alloc->table, 0, reserved); return 0; } void c4iw_id_table_free(struct c4iw_id_table *alloc) { - kfree(alloc->table); + bitmap_free(alloc->table); } diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c index 0c8fd5a85fcb..89f36a3a9af0 100644 --- a/drivers/infiniband/hw/cxgb4/provider.c +++ b/drivers/infiniband/hw/cxgb4/provider.c @@ -41,6 +41,7 @@ #include #include #include +#include #include #include @@ -264,7 +265,8 @@ static int c4iw_query_device(struct ib_device *ibdev, struct ib_device_attr *pro return -EINVAL; dev = to_c4iw_dev(ibdev); - memcpy(&props->sys_image_guid, dev->rdev.lldi.ports[0]->dev_addr, 6); + addrconf_addr_eui48((u8 *)&props->sys_image_guid, + dev->rdev.lldi.ports[0]->dev_addr); props->hw_ver = CHELSIO_CHIP_RELEASE(dev->rdev.lldi.adapter_type); props->fw_ver = dev->rdev.lldi.fw_vers; props->device_cap_flags = dev->device_cap_flags; @@ -525,8 +527,8 @@ void c4iw_register_device(struct work_struct *work) struct c4iw_dev *dev = ctx->dev; pr_debug("c4iw_dev %p\n", dev); - memset(&dev->ibdev.node_guid, 0, sizeof(dev->ibdev.node_guid)); - memcpy(&dev->ibdev.node_guid, dev->rdev.lldi.ports[0]->dev_addr, 6); + addrconf_addr_eui48((u8 *)&dev->ibdev.node_guid, + dev->rdev.lldi.ports[0]->dev_addr); dev->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_WINDOW; if (fastreg_support) dev->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS; diff --git a/drivers/infiniband/hw/cxgb4/qp.c b/drivers/infiniband/hw/cxgb4/qp.c index d20b4ef2c853..ffbd9a89981e 100644 --- a/drivers/infiniband/hw/cxgb4/qp.c +++ b/drivers/infiniband/hw/cxgb4/qp.c @@ -2460,6 +2460,7 @@ int c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, memset(attr, 0, sizeof(*attr)); memset(init_attr, 0, sizeof(*init_attr)); attr->qp_state = to_ib_qp_state(qhp->attr.state); + attr->cur_qp_state = to_ib_qp_state(qhp->attr.state); init_attr->cap.max_send_wr = qhp->attr.sq_num_entries; init_attr->cap.max_recv_wr = qhp->attr.rq_num_entries; init_attr->cap.max_send_sge = qhp->attr.sq_max_sges; diff --git a/drivers/infiniband/hw/hfi1/user_sdma.c b/drivers/infiniband/hw/hfi1/user_sdma.c index 5b11c8282744..a71c5a36ceba 100644 --- a/drivers/infiniband/hw/hfi1/user_sdma.c +++ b/drivers/infiniband/hw/hfi1/user_sdma.c @@ -161,9 +161,7 @@ int hfi1_user_sdma_alloc_queues(struct hfi1_ctxtdata *uctxt, if (!pq->reqs) goto pq_reqs_nomem; - pq->req_in_use = kcalloc(BITS_TO_LONGS(hfi1_sdma_comp_ring_size), - sizeof(*pq->req_in_use), - GFP_KERNEL); + pq->req_in_use = bitmap_zalloc(hfi1_sdma_comp_ring_size, GFP_KERNEL); if (!pq->req_in_use) goto pq_reqs_no_in_use; @@ -210,7 +208,7 @@ cq_comps_nomem: cq_nomem: kmem_cache_destroy(pq->txreq_cache); pq_txreq_nomem: - kfree(pq->req_in_use); + bitmap_free(pq->req_in_use); pq_reqs_no_in_use: kfree(pq->reqs); pq_reqs_nomem: @@ -257,7 +255,7 @@ int hfi1_user_sdma_free_queues(struct hfi1_filedata *fd, pq->wait, !atomic_read(&pq->n_reqs)); kfree(pq->reqs); - kfree(pq->req_in_use); + bitmap_free(pq->req_in_use); kmem_cache_destroy(pq->txreq_cache); flush_pq_iowait(pq); kfree(pq); diff --git a/drivers/infiniband/hw/hns/Kconfig b/drivers/infiniband/hw/hns/Kconfig index 18d10ebf900b..ab3fbba70789 100644 --- a/drivers/infiniband/hw/hns/Kconfig +++ b/drivers/infiniband/hw/hns/Kconfig @@ -5,22 +5,9 @@ config INFINIBAND_HNS depends on ARM64 || (COMPILE_TEST && 64BIT) depends on (HNS_DSAF && HNS_ENET) || HNS3 help - This is a RoCE/RDMA driver for the Hisilicon RoCE engine. The engine - is used in Hisilicon Hip06 and more further ICT SoC based on - platform device. + This is a RoCE/RDMA driver for the Hisilicon RoCE engine. - To compile HIP06 or HIP08 driver as module, choose M here. - -config INFINIBAND_HNS_HIP06 - bool "Hisilicon Hip06 Family RoCE support" - depends on INFINIBAND_HNS && HNS && HNS_DSAF && HNS_ENET - depends on INFINIBAND_HNS=m || (HNS_DSAF=y && HNS_ENET=y) - help - RoCE driver support for Hisilicon RoCE engine in Hisilicon Hip06 and - Hip07 SoC. These RoCE engines are platform devices. - - To compile this driver, choose Y here: if INFINIBAND_HNS is m, this - module will be called hns-roce-hw-v1 + To compile HIP08 driver as module, choose M here. config INFINIBAND_HNS_HIP08 bool "Hisilicon Hip08 Family RoCE support" diff --git a/drivers/infiniband/hw/hns/Makefile b/drivers/infiniband/hw/hns/Makefile index e105945b94a1..9f04f25d9631 100644 --- a/drivers/infiniband/hw/hns/Makefile +++ b/drivers/infiniband/hw/hns/Makefile @@ -9,11 +9,6 @@ hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_pd.o \ hns_roce_ah.o hns_roce_hem.o hns_roce_mr.o hns_roce_qp.o \ hns_roce_cq.o hns_roce_alloc.o hns_roce_db.o hns_roce_srq.o hns_roce_restrack.o -ifdef CONFIG_INFINIBAND_HNS_HIP06 -hns-roce-hw-v1-objs := hns_roce_hw_v1.o $(hns-roce-objs) -obj-$(CONFIG_INFINIBAND_HNS) += hns-roce-hw-v1.o -endif - ifdef CONFIG_INFINIBAND_HNS_HIP08 hns-roce-hw-v2-objs := hns_roce_hw_v2.o hns_roce_hw_v2_dfx.o $(hns-roce-objs) obj-$(CONFIG_INFINIBAND_HNS) += hns-roce-hw-v2.o diff --git a/drivers/infiniband/hw/hns/hns_roce_ah.c b/drivers/infiniband/hw/hns/hns_roce_ah.c index cc258edec331..492b122d0521 100644 --- a/drivers/infiniband/hw/hns/hns_roce_ah.c +++ b/drivers/infiniband/hw/hns/hns_roce_ah.c @@ -30,7 +30,6 @@ * SOFTWARE. */ -#include #include #include #include @@ -61,7 +60,7 @@ int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, struct hns_roce_ah *ah = to_hr_ah(ibah); int ret = 0; - if (hr_dev->pci_dev->revision <= PCI_REVISION_ID_HIP08 && udata) + if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 && udata) return -EOPNOTSUPP; ah->av.port = rdma_ah_get_port_num(ah_attr); @@ -80,7 +79,7 @@ int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, memcpy(ah->av.mac, ah_attr->roce.dmac, ETH_ALEN); /* HIP08 needs to record vlan info in Address Vector */ - if (hr_dev->pci_dev->revision <= PCI_REVISION_ID_HIP08) { + if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) { ret = rdma_read_gid_l2_fields(ah_attr->grh.sgid_attr, &ah->av.vlan_id, NULL); if (ret) diff --git a/drivers/infiniband/hw/hns/hns_roce_alloc.c b/drivers/infiniband/hw/hns/hns_roce_alloc.c index d4fa0fd52294..11a78ceae568 100644 --- a/drivers/infiniband/hw/hns/hns_roce_alloc.c +++ b/drivers/infiniband/hw/hns/hns_roce_alloc.c @@ -31,10 +31,9 @@ * SOFTWARE. */ -#include #include -#include "hns_roce_device.h" #include +#include "hns_roce_device.h" void hns_roce_buf_free(struct hns_roce_dev *hr_dev, struct hns_roce_buf *buf) { diff --git a/drivers/infiniband/hw/hns/hns_roce_cmd.c b/drivers/infiniband/hw/hns/hns_roce_cmd.c index 84f3f2b5f097..4b693d542ace 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cmd.c +++ b/drivers/infiniband/hw/hns/hns_roce_cmd.c @@ -31,7 +31,6 @@ */ #include -#include #include "hns_roce_common.h" #include "hns_roce_device.h" #include "hns_roce_cmd.h" @@ -61,7 +60,7 @@ static int __hns_roce_cmd_mbox_poll(struct hns_roce_dev *hr_dev, u64 in_param, CMD_POLL_TOKEN, 0); if (ret) { dev_err_ratelimited(hr_dev->dev, - "failed to post mailbox %x in poll mode, ret = %d.\n", + "failed to post mailbox 0x%x in poll mode, ret = %d.\n", op, ret); return ret; } @@ -91,7 +90,7 @@ void hns_roce_cmd_event(struct hns_roce_dev *hr_dev, u16 token, u8 status, if (unlikely(token != context->token)) { dev_err_ratelimited(hr_dev->dev, - "[cmd] invalid ae token %x,context token is %x!\n", + "[cmd] invalid ae token 0x%x, context token is 0x%x.\n", token, context->token); return; } @@ -130,14 +129,14 @@ static int __hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param, context->token, 1); if (ret) { dev_err_ratelimited(dev, - "failed to post mailbox %x in event mode, ret = %d.\n", + "failed to post mailbox 0x%x in event mode, ret = %d.\n", op, ret); goto out; } if (!wait_for_completion_timeout(&context->done, msecs_to_jiffies(timeout))) { - dev_err_ratelimited(dev, "[cmd] token %x mailbox %x timeout.\n", + dev_err_ratelimited(dev, "[cmd] token 0x%x mailbox 0x%x timeout.\n", context->token, op); ret = -EBUSY; goto out; @@ -145,7 +144,7 @@ static int __hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param, ret = context->result; if (ret) - dev_err_ratelimited(dev, "[cmd] token %x mailbox %x error %d\n", + dev_err_ratelimited(dev, "[cmd] token 0x%x mailbox 0x%x error %d.\n", context->token, op, ret); out: diff --git a/drivers/infiniband/hw/hns/hns_roce_common.h b/drivers/infiniband/hw/hns/hns_roce_common.h index b73e55de83ac..465d1f914b6c 100644 --- a/drivers/infiniband/hw/hns/hns_roce_common.h +++ b/drivers/infiniband/hw/hns/hns_roce_common.h @@ -104,208 +104,6 @@ #define hr_reg_read(ptr, field) _hr_reg_read(ptr, field) -#define ROCEE_GLB_CFG_ROCEE_DB_SQ_MODE_S 3 -#define ROCEE_GLB_CFG_ROCEE_DB_OTH_MODE_S 4 - -#define ROCEE_GLB_CFG_SQ_EXT_DB_MODE_S 5 - -#define ROCEE_GLB_CFG_OTH_EXT_DB_MODE_S 6 - -#define ROCEE_GLB_CFG_ROCEE_PORT_ST_S 10 -#define ROCEE_GLB_CFG_ROCEE_PORT_ST_M \ - (((1UL << 6) - 1) << ROCEE_GLB_CFG_ROCEE_PORT_ST_S) - -#define ROCEE_GLB_CFG_TRP_RAQ_DROP_EN_S 16 - -#define ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_S 0 -#define ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_M \ - (((1UL << 24) - 1) << ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_S) - -#define ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_S 24 -#define ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_M \ - (((1UL << 4) - 1) << ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_S) - -#define ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_S 0 -#define ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_M \ - (((1UL << 24) - 1) << ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_S) - -#define ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_S 24 -#define ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_M \ - (((1UL << 4) - 1) << ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_S) - -#define ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_S 0 -#define ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_M \ - (((1UL << 16) - 1) << ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_S) - -#define ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_S 16 -#define ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_M \ - (((1UL << 16) - 1) << ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_S) - -#define ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_S 0 -#define ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_M \ - (((1UL << 16) - 1) << ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_S) - -#define ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_S 16 -#define ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_M \ - (((1UL << 16) - 1) << ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_S) - -#define ROCEE_RAQ_WL_ROCEE_RAQ_WL_S 0 -#define ROCEE_RAQ_WL_ROCEE_RAQ_WL_M \ - (((1UL << 8) - 1) << ROCEE_RAQ_WL_ROCEE_RAQ_WL_S) - -#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_S 0 -#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_M \ - (((1UL << 15) - 1) << \ - ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_S) - -#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_S 16 -#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_M \ - (((1UL << 4) - 1) << \ - ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_S) - -#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_EN_S 20 - -#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_EXT_RAQ_MODE 21 - -#define ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_S 0 -#define ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_M \ - (((1UL << 5) - 1) << ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_S) - -#define ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_S 5 -#define ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_M \ - (((1UL << 5) - 1) << ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_S) - -#define ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_S 0 -#define ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_M \ - (((1UL << 5) - 1) << ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_S) - -#define ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_S 5 -#define ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_M \ - (((1UL << 5) - 1) << ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_S) - -#define ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_S 0 -#define ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_M \ - (((1UL << 5) - 1) << ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_S) - -#define ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_S 8 -#define ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_M \ - (((1UL << 5) - 1) << ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_S) - -#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S 0 -#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M \ - (((1UL << 19) - 1) << ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S) - -#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_S 19 - -#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S 20 -#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M \ - (((1UL << 2) - 1) << ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S) - -#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S 22 -#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M \ - (((1UL << 5) - 1) << ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S) - -#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S 31 - -#define ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_S 0 -#define ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_M \ - (((1UL << 3) - 1) << ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_S) - -#define ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_S 0 -#define ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_M \ - (((1UL << 15) - 1) << ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_S) - -#define ROCEE_MB6_ROCEE_MB_CMD_S 0 -#define ROCEE_MB6_ROCEE_MB_CMD_M \ - (((1UL << 8) - 1) << ROCEE_MB6_ROCEE_MB_CMD_S) - -#define ROCEE_MB6_ROCEE_MB_CMD_MDF_S 8 -#define ROCEE_MB6_ROCEE_MB_CMD_MDF_M \ - (((1UL << 4) - 1) << ROCEE_MB6_ROCEE_MB_CMD_MDF_S) - -#define ROCEE_MB6_ROCEE_MB_EVENT_S 14 - -#define ROCEE_MB6_ROCEE_MB_HW_RUN_S 15 - -#define ROCEE_MB6_ROCEE_MB_TOKEN_S 16 -#define ROCEE_MB6_ROCEE_MB_TOKEN_M \ - (((1UL << 16) - 1) << ROCEE_MB6_ROCEE_MB_TOKEN_S) - -#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S 0 -#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M \ - (((1UL << 24) - 1) << ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S) - -#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S 24 -#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M \ - (((1UL << 4) - 1) << ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S) - -#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S 28 -#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M \ - (((1UL << 3) - 1) << ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S) - -#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S 31 - -#define ROCEE_SMAC_H_ROCEE_SMAC_H_S 0 -#define ROCEE_SMAC_H_ROCEE_SMAC_H_M \ - (((1UL << 16) - 1) << ROCEE_SMAC_H_ROCEE_SMAC_H_S) - -#define ROCEE_SMAC_H_ROCEE_PORT_MTU_S 16 -#define ROCEE_SMAC_H_ROCEE_PORT_MTU_M \ - (((1UL << 4) - 1) << ROCEE_SMAC_H_ROCEE_PORT_MTU_S) - -#define ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S 0 -#define ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M \ - (((1UL << 2) - 1) << ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S) - -#define ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_S 8 -#define ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_M \ - (((1UL << 4) - 1) << ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_S) - -#define ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQ_ALM_OVF_INT_ST_S 17 - -#define ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_S 0 -#define ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_M \ - (((1UL << 5) - 1) << ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_S) - -#define ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_S 16 -#define ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_M \ - (((1UL << 16) - 1) << ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_S) - -#define ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_S 0 -#define ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_M \ - (((1UL << 16) - 1) << ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_S) - -#define ROCEE_CAEP_CEQC_SHIFT_CAEP_CEQ_ALM_OVF_INT_ST_S 16 -#define ROCEE_CAEP_CE_IRQ_MASK_CAEP_CEQ_ALM_OVF_MASK_S 1 -#define ROCEE_CAEP_CEQ_ALM_OVF_CAEP_CEQ_ALM_OVF_S 0 - -#define ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S 0 -#define ROCEE_CAEP_AE_MASK_CAEP_AE_IRQ_MASK_S 1 - -#define ROCEE_CAEP_AE_ST_CAEP_AEQ_ALM_OVF_S 0 - -#define ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_S 0 -#define ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_M \ - (((1UL << 28) - 1) << ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_S) - -#define ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S 0 -#define ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M \ - (((1UL << 28) - 1) << ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S) - -#define ROCEE_SDB_INV_CNT_SDB_INV_CNT_S 0 -#define ROCEE_SDB_INV_CNT_SDB_INV_CNT_M \ - (((1UL << 16) - 1) << ROCEE_SDB_INV_CNT_SDB_INV_CNT_S) - -#define ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S 0 -#define ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_M \ - (((1UL << 16) - 1) << ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S) - -#define ROCEE_SDB_CNT_CMP_BITS 16 - -#define ROCEE_TSP_BP_ST_QH_FIFO_ENTRY_S 20 - -#define ROCEE_CNT_CLR_CE_CNT_CLR_CE_S 0 - /*************ROCEE_REG DEFINITION****************/ #define ROCEE_VENDOR_ID_REG 0x0 #define ROCEE_VENDOR_PART_ID_REG 0x4 diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index d763f097599f..55057dcbb2dc 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -30,7 +30,6 @@ * SOFTWARE. */ -#include #include #include #include "hns_roce_device.h" @@ -406,15 +405,6 @@ int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr, goto err_cqn; } - /* - * For the QP created by kernel space, tptr value should be initialized - * to zero; For the QP created by user space, it will cause synchronous - * problems if tptr is set to zero here, so we initialize it in user - * space. - */ - if (!udata && hr_cq->tptr_addr) - *hr_cq->tptr_addr = 0; - if (udata) { resp.cqn = hr_cq->cqn; ret = ib_copy_to_udata(udata, &resp, @@ -441,9 +431,6 @@ int hns_roce_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device); struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq); - if (hr_dev->hw->destroy_cq) - hr_dev->hw->destroy_cq(ib_cq, udata); - free_cqc(hr_dev, hr_cq); free_cqn(hr_dev, hr_cq->cqn); free_cq_db(hr_dev, hr_cq, udata); diff --git a/drivers/infiniband/hw/hns/hns_roce_db.c b/drivers/infiniband/hw/hns/hns_roce_db.c index 751470c7a2ce..5c4c0480832b 100644 --- a/drivers/infiniband/hw/hns/hns_roce_db.c +++ b/drivers/infiniband/hw/hns/hns_roce_db.c @@ -4,7 +4,6 @@ * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. */ -#include #include #include "hns_roce_device.h" diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 43e17d61cb63..1e0bae136997 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -36,36 +36,18 @@ #include #include -#define DRV_NAME "hns_roce" - #define PCI_REVISION_ID_HIP08 0x21 #define PCI_REVISION_ID_HIP09 0x30 -#define HNS_ROCE_HW_VER1 ('h' << 24 | 'i' << 16 | '0' << 8 | '6') - #define HNS_ROCE_MAX_MSG_LEN 0x80000000 #define HNS_ROCE_IB_MIN_SQ_STRIDE 6 #define BA_BYTE_LEN 8 -/* Hardware specification only for v1 engine */ #define HNS_ROCE_MIN_CQE_NUM 0x40 -#define HNS_ROCE_MIN_WQE_NUM 0x20 #define HNS_ROCE_MIN_SRQ_WQE_NUM 1 -/* Hardware specification only for v1 engine */ -#define HNS_ROCE_MAX_INNER_MTPT_NUM 0x7 -#define HNS_ROCE_MAX_MTPT_PBL_NUM 0x100000 - -#define HNS_ROCE_EACH_FREE_CQ_WAIT_MSECS 20 -#define HNS_ROCE_MAX_FREE_CQ_WAIT_CNT \ - (5000 / HNS_ROCE_EACH_FREE_CQ_WAIT_MSECS) -#define HNS_ROCE_CQE_WCMD_EMPTY_BIT 0x2 -#define HNS_ROCE_MIN_CQE_CNT 16 - -#define HNS_ROCE_RESERVED_SGE 1 - #define HNS_ROCE_MAX_IRQ_NUM 128 #define HNS_ROCE_SGE_IN_WQE 2 @@ -102,18 +84,12 @@ #define HNS_ROCE_FRMR_MAX_PA 512 #define PKEY_ID 0xffff -#define GUID_LEN 8 #define NODE_DESC_SIZE 64 #define DB_REG_OFFSET 0x1000 /* Configure to HW for PAGE_SIZE larger than 4KB */ #define PG_SHIFT_OFFSET (PAGE_SHIFT - 12) -#define PAGES_SHIFT_8 8 -#define PAGES_SHIFT_16 16 -#define PAGES_SHIFT_24 24 -#define PAGES_SHIFT_32 32 - #define HNS_ROCE_IDX_QUE_ENTRY_SZ 4 #define SRQ_DB_REG 0x230 @@ -122,11 +98,6 @@ #define CQ_BANKID_SHIFT 2 -/* The chip implementation of the consumer index is calculated - * according to twice the actual EQ depth - */ -#define EQ_DEPTH_COEFF 2 - enum { SERV_TYPE_RC, SERV_TYPE_UC, @@ -182,6 +153,7 @@ enum { HNS_ROCE_CAP_FLAG_FRMR = BIT(8), HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL = BIT(9), HNS_ROCE_CAP_FLAG_ATOMIC = BIT(10), + HNS_ROCE_CAP_FLAG_DIRECT_WQE = BIT(12), HNS_ROCE_CAP_FLAG_SDI_MODE = BIT(14), HNS_ROCE_CAP_FLAG_STASH = BIT(17), }; @@ -227,7 +199,7 @@ struct hns_roce_uar { enum hns_roce_mmap_type { HNS_ROCE_MMAP_TYPE_DB = 1, - HNS_ROCE_MMAP_TYPE_TPTR, + HNS_ROCE_MMAP_TYPE_DWQE, }; struct hns_user_mmap_entry { @@ -242,7 +214,6 @@ struct hns_roce_ucontext { struct list_head page_list; struct mutex page_mutex; struct hns_user_mmap_entry *db_mmap_entry; - struct hns_user_mmap_entry *tptr_mmap_entry; }; struct hns_roce_pd { @@ -345,19 +316,16 @@ struct hns_roce_mw { u32 pbl_buf_pg_sz; }; -/* Only support 4K page size for mr register */ -#define MR_SIZE_4K 0 - struct hns_roce_mr { struct ib_mr ibmr; u64 iova; /* MR's virtual original addr */ u64 size; /* Address range of MR */ u32 key; /* Key of MR */ u32 pd; /* PD num of MR */ - u32 access; /* Access permission of MR */ + u32 access; /* Access permission of MR */ int enabled; /* MR's active status */ - int type; /* MR's register type */ - u32 pbl_hop_num; /* multi-hop number */ + int type; /* MR's register type */ + u32 pbl_hop_num; /* multi-hop number */ struct hns_roce_mtr pbl_mtr; u32 npages; dma_addr_t *page_list; @@ -374,17 +342,17 @@ struct hns_roce_wq { u32 wqe_cnt; /* WQE num */ u32 max_gs; u32 rsv_sge; - int offset; - int wqe_shift; /* WQE size */ + u32 offset; + u32 wqe_shift; /* WQE size */ u32 head; u32 tail; void __iomem *db_reg; }; struct hns_roce_sge { - unsigned int sge_cnt; /* SGE num */ - int offset; - int sge_shift; /* SGE size */ + unsigned int sge_cnt; /* SGE num */ + u32 offset; + u32 sge_shift; /* SGE size */ }; struct hns_roce_buf_list { @@ -453,7 +421,6 @@ struct hns_roce_cq { u32 cons_index; u32 *set_ci_db; void __iomem *db_reg; - u16 *tptr_addr; int arm_sn; int cqe_size; unsigned long cqn; @@ -468,7 +435,7 @@ struct hns_roce_cq { struct hns_roce_idx_que { struct hns_roce_mtr mtr; - int entry_shift; + u32 entry_shift; unsigned long *bitmap; u32 head; u32 tail; @@ -480,7 +447,7 @@ struct hns_roce_srq { u32 wqe_cnt; int max_gs; u32 rsv_sge; - int wqe_shift; + u32 wqe_shift; u32 cqn; u32 xrcdn; void __iomem *db_reg; @@ -539,10 +506,6 @@ struct hns_roce_srq_table { struct hns_roce_hem_table table; }; -struct hns_roce_raq_table { - struct hns_roce_buf_list *e_raq_buf; -}; - struct hns_roce_av { u8 port; u8 gid_index; @@ -627,10 +590,6 @@ struct hns_roce_work { u32 queue_num; }; -enum { - HNS_ROCE_QP_CAP_DIRECT_WQE = BIT(5), -}; - struct hns_roce_qp { struct ib_qp ibqp; struct hns_roce_wq rq; @@ -650,9 +609,7 @@ struct hns_roce_qp { u8 sl; u8 resp_depth; u8 state; - u32 access_flags; u32 atomic_rd_en; - u32 pkey_index; u32 qkey; void (*event)(struct hns_roce_qp *qp, enum hns_roce_event event_type); @@ -672,9 +629,10 @@ struct hns_roce_qp { unsigned long flush_flag; struct hns_roce_work flush_work; struct hns_roce_rinl_buf rq_inl_buf; - struct list_head node; /* all qps are on a list */ - struct list_head rq_node; /* all recv qps are on a list */ - struct list_head sq_node; /* all send qps are on a list */ + struct list_head node; /* all qps are on a list */ + struct list_head rq_node; /* all recv qps are on a list */ + struct list_head sq_node; /* all send qps are on a list */ + struct hns_user_mmap_entry *dwqe_mmap_entry; }; struct hns_roce_ib_iboe { @@ -684,11 +642,6 @@ struct hns_roce_ib_iboe { u8 phy_port[HNS_ROCE_MAX_PORTS]; }; -enum { - HNS_ROCE_EQ_STAT_INVALID = 0, - HNS_ROCE_EQ_STAT_VALID = 2, -}; - struct hns_roce_ceqe { __le32 comp; __le32 rsv[15]; @@ -720,12 +673,9 @@ struct hns_roce_eq { int type_flag; /* Aeq:1 ceq:0 */ int eqn; u32 entries; - u32 log_entries; int eqe_size; int irq; - int log_page_size; u32 cons_index; - struct hns_roce_buf_list *buf_list; int over_ignore; int coalesce; int arm_st; @@ -740,7 +690,6 @@ struct hns_roce_eq { struct hns_roce_eq_table { struct hns_roce_eq *eq; - void __iomem **eqc_base; /* only for hw v1 */ }; enum cong_type { @@ -767,7 +716,7 @@ struct hns_roce_caps { u32 reserved_qps; int num_qpc_timer; int num_cqc_timer; - int num_srqs; + u32 num_srqs; u32 max_wqes; u32 max_srq_wrs; u32 max_srq_sges; @@ -781,7 +730,7 @@ struct hns_roce_caps { u32 min_cqes; u32 min_wqes; u32 reserved_cqs; - int reserved_srqs; + u32 reserved_srqs; int num_aeq_vectors; int num_comp_vectors; int num_other_vectors; @@ -855,7 +804,7 @@ struct hns_roce_caps { u32 cqc_timer_ba_pg_sz; u32 cqc_timer_buf_pg_sz; u32 cqc_timer_hop_num; - u32 cqe_ba_pg_sz; /* page_size = 4K*(2^cqe_ba_pg_sz) */ + u32 cqe_ba_pg_sz; /* page_size = 4K*(2^cqe_ba_pg_sz) */ u32 cqe_buf_pg_sz; u32 cqe_hop_num; u32 srqwqe_ba_pg_sz; @@ -874,7 +823,7 @@ struct hns_roce_caps { u32 gmv_hop_num; u32 sl_num; u32 llm_buf_pg_sz; - u32 chunk_sz; /* chunk size in non multihop mode */ + u32 chunk_sz; /* chunk size in non multihop mode */ u64 flags; u16 default_ceq_max_cnt; u16 default_ceq_period; @@ -897,7 +846,6 @@ enum hns_roce_device_state { }; struct hns_roce_hw { - int (*reset)(struct hns_roce_dev *hr_dev, bool enable); int (*cmq_init)(struct hns_roce_dev *hr_dev); void (*cmq_exit)(struct hns_roce_dev *hr_dev); int (*hw_profile)(struct hns_roce_dev *hr_dev); @@ -909,14 +857,12 @@ struct hns_roce_hw { int (*poll_mbox_done)(struct hns_roce_dev *hr_dev, unsigned int timeout); bool (*chk_mbox_avail)(struct hns_roce_dev *hr_dev, bool *is_busy); - int (*set_gid)(struct hns_roce_dev *hr_dev, u32 port, int gid_index, + int (*set_gid)(struct hns_roce_dev *hr_dev, int gid_index, const union ib_gid *gid, const struct ib_gid_attr *attr); int (*set_mac)(struct hns_roce_dev *hr_dev, u8 phy_port, const u8 *addr); - void (*set_mtu)(struct hns_roce_dev *hr_dev, u8 phy_port, - enum ib_mtu mtu); int (*write_mtpt)(struct hns_roce_dev *hr_dev, void *mb_buf, - struct hns_roce_mr *mr, unsigned long mtpt_idx); + struct hns_roce_mr *mr); int (*rereg_write_mtpt)(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr, int flags, void *mb_buf); @@ -936,9 +882,6 @@ struct hns_roce_hw { enum ib_qp_state new_state); int (*qp_flow_control_init)(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp); - int (*dereg_mr)(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr, - struct ib_udata *udata); - int (*destroy_cq)(struct ib_cq *ibcq, struct ib_udata *udata); int (*init_eq)(struct hns_roce_dev *hr_dev); void (*cleanup_eq)(struct hns_roce_dev *hr_dev); int (*write_srqc)(struct hns_roce_srq *srq, void *mb_buf); @@ -948,13 +891,11 @@ struct hns_roce_hw { struct hns_roce_dev { struct ib_device ib_dev; - struct platform_device *pdev; struct pci_dev *pci_dev; struct device *dev; struct hns_roce_uar priv_uar; const char *irq_names[HNS_ROCE_MAX_IRQ_NUM]; spinlock_t sm_lock; - spinlock_t bt_cmd_lock; bool active; bool is_reset; bool dis_db; @@ -1001,8 +942,6 @@ struct hns_roce_dev { int loop_idc; u32 sdb_offset; u32 odb_offset; - dma_addr_t tptr_dma_addr; /* only for hw v1 */ - u32 tptr_size; /* only for hw v1 */ const struct hns_roce_hw *hw; void *priv; struct workqueue_struct *irq_workq; @@ -1010,6 +949,7 @@ struct hns_roce_dev { u32 func_num; u32 is_vf; u32 cong_algo_tmpl_id; + u64 dwqe_page; }; static inline struct hns_roce_dev *to_hr_dev(struct ib_device *ib_dev) @@ -1158,7 +1098,7 @@ void hns_roce_cmd_use_polling(struct hns_roce_dev *hr_dev); /* hns roce hw need current block and next block addr from mtt */ #define MTT_MIN_COUNT 2 int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, - int offset, u64 *mtt_buf, int mtt_max, u64 *base_addr); + u32 offset, u64 *mtt_buf, int mtt_max, u64 *base_addr); int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, struct hns_roce_buf_attr *buf_attr, unsigned int page_shift, struct ib_udata *udata, diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c index fa15d79eabb3..8917365cc6b8 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.c +++ b/drivers/infiniband/hw/hns/hns_roce_hem.c @@ -31,7 +31,6 @@ * SOFTWARE. */ -#include #include "hns_roce_device.h" #include "hns_roce_hem.h" #include "hns_roce_common.h" diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c deleted file mode 100644 index f4af3992ba95..000000000000 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c +++ /dev/null @@ -1,4675 +0,0 @@ -/* - * Copyright (c) 2016 Hisilicon Limited. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include "hns_roce_common.h" -#include "hns_roce_device.h" -#include "hns_roce_cmd.h" -#include "hns_roce_hem.h" -#include "hns_roce_hw_v1.h" - -/** - * hns_get_gid_index - Get gid index. - * @hr_dev: pointer to structure hns_roce_dev. - * @port: port, value range: 0 ~ MAX - * @gid_index: gid_index, value range: 0 ~ MAX - * Description: - * N ports shared gids, allocation method as follow: - * GID[0][0], GID[1][0],.....GID[N - 1][0], - * GID[0][0], GID[1][0],.....GID[N - 1][0], - * And so on - */ -u8 hns_get_gid_index(struct hns_roce_dev *hr_dev, u32 port, int gid_index) -{ - return gid_index * hr_dev->caps.num_ports + port; -} - -static void set_data_seg(struct hns_roce_wqe_data_seg *dseg, struct ib_sge *sg) -{ - dseg->lkey = cpu_to_le32(sg->lkey); - dseg->addr = cpu_to_le64(sg->addr); - dseg->len = cpu_to_le32(sg->length); -} - -static void set_raddr_seg(struct hns_roce_wqe_raddr_seg *rseg, u64 remote_addr, - u32 rkey) -{ - rseg->raddr = cpu_to_le64(remote_addr); - rseg->rkey = cpu_to_le32(rkey); - rseg->len = 0; -} - -static int hns_roce_v1_post_send(struct ib_qp *ibqp, - const struct ib_send_wr *wr, - const struct ib_send_wr **bad_wr) -{ - struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); - struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah); - struct hns_roce_ud_send_wqe *ud_sq_wqe = NULL; - struct hns_roce_wqe_ctrl_seg *ctrl = NULL; - struct hns_roce_wqe_data_seg *dseg = NULL; - struct hns_roce_qp *qp = to_hr_qp(ibqp); - struct device *dev = &hr_dev->pdev->dev; - struct hns_roce_sq_db sq_db = {}; - int ps_opcode, i; - unsigned long flags = 0; - void *wqe = NULL; - __le32 doorbell[2]; - const u8 *smac; - int ret = 0; - int loopback; - u32 wqe_idx; - int nreq; - - if (unlikely(ibqp->qp_type != IB_QPT_GSI && - ibqp->qp_type != IB_QPT_RC)) { - dev_err(dev, "un-supported QP type\n"); - *bad_wr = NULL; - return -EOPNOTSUPP; - } - - spin_lock_irqsave(&qp->sq.lock, flags); - - for (nreq = 0; wr; ++nreq, wr = wr->next) { - if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) { - ret = -ENOMEM; - *bad_wr = wr; - goto out; - } - - wqe_idx = (qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1); - - if (unlikely(wr->num_sge > qp->sq.max_gs)) { - dev_err(dev, "num_sge=%d > qp->sq.max_gs=%d\n", - wr->num_sge, qp->sq.max_gs); - ret = -EINVAL; - *bad_wr = wr; - goto out; - } - - wqe = hns_roce_get_send_wqe(qp, wqe_idx); - qp->sq.wrid[wqe_idx] = wr->wr_id; - - /* Corresponding to the RC and RD type wqe process separately */ - if (ibqp->qp_type == IB_QPT_GSI) { - ud_sq_wqe = wqe; - roce_set_field(ud_sq_wqe->dmac_h, - UD_SEND_WQE_U32_4_DMAC_0_M, - UD_SEND_WQE_U32_4_DMAC_0_S, - ah->av.mac[0]); - roce_set_field(ud_sq_wqe->dmac_h, - UD_SEND_WQE_U32_4_DMAC_1_M, - UD_SEND_WQE_U32_4_DMAC_1_S, - ah->av.mac[1]); - roce_set_field(ud_sq_wqe->dmac_h, - UD_SEND_WQE_U32_4_DMAC_2_M, - UD_SEND_WQE_U32_4_DMAC_2_S, - ah->av.mac[2]); - roce_set_field(ud_sq_wqe->dmac_h, - UD_SEND_WQE_U32_4_DMAC_3_M, - UD_SEND_WQE_U32_4_DMAC_3_S, - ah->av.mac[3]); - - roce_set_field(ud_sq_wqe->u32_8, - UD_SEND_WQE_U32_8_DMAC_4_M, - UD_SEND_WQE_U32_8_DMAC_4_S, - ah->av.mac[4]); - roce_set_field(ud_sq_wqe->u32_8, - UD_SEND_WQE_U32_8_DMAC_5_M, - UD_SEND_WQE_U32_8_DMAC_5_S, - ah->av.mac[5]); - - smac = (const u8 *)hr_dev->dev_addr[qp->port]; - loopback = ether_addr_equal_unaligned(ah->av.mac, - smac) ? 1 : 0; - roce_set_bit(ud_sq_wqe->u32_8, - UD_SEND_WQE_U32_8_LOOPBACK_INDICATOR_S, - loopback); - - roce_set_field(ud_sq_wqe->u32_8, - UD_SEND_WQE_U32_8_OPERATION_TYPE_M, - UD_SEND_WQE_U32_8_OPERATION_TYPE_S, - HNS_ROCE_WQE_OPCODE_SEND); - roce_set_field(ud_sq_wqe->u32_8, - UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_M, - UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_S, - 2); - roce_set_bit(ud_sq_wqe->u32_8, - UD_SEND_WQE_U32_8_SEND_GL_ROUTING_HDR_FLAG_S, - 1); - - ud_sq_wqe->u32_8 |= (wr->send_flags & IB_SEND_SIGNALED ? - cpu_to_le32(HNS_ROCE_WQE_CQ_NOTIFY) : 0) | - (wr->send_flags & IB_SEND_SOLICITED ? - cpu_to_le32(HNS_ROCE_WQE_SE) : 0) | - ((wr->opcode == IB_WR_SEND_WITH_IMM) ? - cpu_to_le32(HNS_ROCE_WQE_IMM) : 0); - - roce_set_field(ud_sq_wqe->u32_16, - UD_SEND_WQE_U32_16_DEST_QP_M, - UD_SEND_WQE_U32_16_DEST_QP_S, - ud_wr(wr)->remote_qpn); - roce_set_field(ud_sq_wqe->u32_16, - UD_SEND_WQE_U32_16_MAX_STATIC_RATE_M, - UD_SEND_WQE_U32_16_MAX_STATIC_RATE_S, - ah->av.stat_rate); - - roce_set_field(ud_sq_wqe->u32_36, - UD_SEND_WQE_U32_36_FLOW_LABEL_M, - UD_SEND_WQE_U32_36_FLOW_LABEL_S, - ah->av.flowlabel); - roce_set_field(ud_sq_wqe->u32_36, - UD_SEND_WQE_U32_36_PRIORITY_M, - UD_SEND_WQE_U32_36_PRIORITY_S, - ah->av.sl); - roce_set_field(ud_sq_wqe->u32_36, - UD_SEND_WQE_U32_36_SGID_INDEX_M, - UD_SEND_WQE_U32_36_SGID_INDEX_S, - hns_get_gid_index(hr_dev, qp->phy_port, - ah->av.gid_index)); - - roce_set_field(ud_sq_wqe->u32_40, - UD_SEND_WQE_U32_40_HOP_LIMIT_M, - UD_SEND_WQE_U32_40_HOP_LIMIT_S, - ah->av.hop_limit); - roce_set_field(ud_sq_wqe->u32_40, - UD_SEND_WQE_U32_40_TRAFFIC_CLASS_M, - UD_SEND_WQE_U32_40_TRAFFIC_CLASS_S, - ah->av.tclass); - - memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0], GID_LEN); - - ud_sq_wqe->va0_l = - cpu_to_le32((u32)wr->sg_list[0].addr); - ud_sq_wqe->va0_h = - cpu_to_le32((wr->sg_list[0].addr) >> 32); - ud_sq_wqe->l_key0 = - cpu_to_le32(wr->sg_list[0].lkey); - - ud_sq_wqe->va1_l = - cpu_to_le32((u32)wr->sg_list[1].addr); - ud_sq_wqe->va1_h = - cpu_to_le32((wr->sg_list[1].addr) >> 32); - ud_sq_wqe->l_key1 = - cpu_to_le32(wr->sg_list[1].lkey); - } else if (ibqp->qp_type == IB_QPT_RC) { - u32 tmp_len = 0; - - ctrl = wqe; - memset(ctrl, 0, sizeof(struct hns_roce_wqe_ctrl_seg)); - for (i = 0; i < wr->num_sge; i++) - tmp_len += wr->sg_list[i].length; - - ctrl->msg_length = - cpu_to_le32(le32_to_cpu(ctrl->msg_length) + tmp_len); - - ctrl->sgl_pa_h = 0; - ctrl->flag = 0; - - switch (wr->opcode) { - case IB_WR_SEND_WITH_IMM: - case IB_WR_RDMA_WRITE_WITH_IMM: - ctrl->imm_data = wr->ex.imm_data; - break; - case IB_WR_SEND_WITH_INV: - ctrl->inv_key = - cpu_to_le32(wr->ex.invalidate_rkey); - break; - default: - ctrl->imm_data = 0; - break; - } - - /* Ctrl field, ctrl set type: sig, solic, imm, fence */ - /* SO wait for conforming application scenarios */ - ctrl->flag |= (wr->send_flags & IB_SEND_SIGNALED ? - cpu_to_le32(HNS_ROCE_WQE_CQ_NOTIFY) : 0) | - (wr->send_flags & IB_SEND_SOLICITED ? - cpu_to_le32(HNS_ROCE_WQE_SE) : 0) | - ((wr->opcode == IB_WR_SEND_WITH_IMM || - wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) ? - cpu_to_le32(HNS_ROCE_WQE_IMM) : 0) | - (wr->send_flags & IB_SEND_FENCE ? - (cpu_to_le32(HNS_ROCE_WQE_FENCE)) : 0); - - wqe += sizeof(struct hns_roce_wqe_ctrl_seg); - - switch (wr->opcode) { - case IB_WR_RDMA_READ: - ps_opcode = HNS_ROCE_WQE_OPCODE_RDMA_READ; - set_raddr_seg(wqe, rdma_wr(wr)->remote_addr, - rdma_wr(wr)->rkey); - break; - case IB_WR_RDMA_WRITE: - case IB_WR_RDMA_WRITE_WITH_IMM: - ps_opcode = HNS_ROCE_WQE_OPCODE_RDMA_WRITE; - set_raddr_seg(wqe, rdma_wr(wr)->remote_addr, - rdma_wr(wr)->rkey); - break; - case IB_WR_SEND: - case IB_WR_SEND_WITH_INV: - case IB_WR_SEND_WITH_IMM: - ps_opcode = HNS_ROCE_WQE_OPCODE_SEND; - break; - case IB_WR_LOCAL_INV: - case IB_WR_ATOMIC_CMP_AND_SWP: - case IB_WR_ATOMIC_FETCH_AND_ADD: - case IB_WR_LSO: - default: - ps_opcode = HNS_ROCE_WQE_OPCODE_MASK; - break; - } - ctrl->flag |= cpu_to_le32(ps_opcode); - wqe += sizeof(struct hns_roce_wqe_raddr_seg); - - dseg = wqe; - if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) { - if (le32_to_cpu(ctrl->msg_length) > - hr_dev->caps.max_sq_inline) { - ret = -EINVAL; - *bad_wr = wr; - dev_err(dev, "inline len(1-%d)=%d, illegal", - le32_to_cpu(ctrl->msg_length), - hr_dev->caps.max_sq_inline); - goto out; - } - for (i = 0; i < wr->num_sge; i++) { - memcpy(wqe, ((void *) (uintptr_t) - wr->sg_list[i].addr), - wr->sg_list[i].length); - wqe += wr->sg_list[i].length; - } - ctrl->flag |= cpu_to_le32(HNS_ROCE_WQE_INLINE); - } else { - /* sqe num is two */ - for (i = 0; i < wr->num_sge; i++) - set_data_seg(dseg + i, wr->sg_list + i); - - ctrl->flag |= cpu_to_le32(wr->num_sge << - HNS_ROCE_WQE_SGE_NUM_BIT); - } - } - } - -out: - /* Set DB return */ - if (likely(nreq)) { - qp->sq.head += nreq; - - roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_SQ_HEAD_M, - SQ_DOORBELL_U32_4_SQ_HEAD_S, - (qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1))); - roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_SL_M, - SQ_DOORBELL_U32_4_SL_S, qp->sl); - roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_PORT_M, - SQ_DOORBELL_U32_4_PORT_S, qp->phy_port); - roce_set_field(sq_db.u32_8, SQ_DOORBELL_U32_8_QPN_M, - SQ_DOORBELL_U32_8_QPN_S, qp->doorbell_qpn); - roce_set_bit(sq_db.u32_8, SQ_DOORBELL_HW_SYNC_S, 1); - - doorbell[0] = sq_db.u32_4; - doorbell[1] = sq_db.u32_8; - - hns_roce_write64_k(doorbell, qp->sq.db_reg); - } - - spin_unlock_irqrestore(&qp->sq.lock, flags); - - return ret; -} - -static int hns_roce_v1_post_recv(struct ib_qp *ibqp, - const struct ib_recv_wr *wr, - const struct ib_recv_wr **bad_wr) -{ - struct hns_roce_rq_wqe_ctrl *ctrl = NULL; - struct hns_roce_wqe_data_seg *scat = NULL; - struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); - struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); - struct device *dev = &hr_dev->pdev->dev; - struct hns_roce_rq_db rq_db = {}; - __le32 doorbell[2] = {0}; - unsigned long flags = 0; - unsigned int wqe_idx; - int ret = 0; - int nreq; - int i; - u32 reg_val; - - spin_lock_irqsave(&hr_qp->rq.lock, flags); - - for (nreq = 0; wr; ++nreq, wr = wr->next) { - if (hns_roce_wq_overflow(&hr_qp->rq, nreq, - hr_qp->ibqp.recv_cq)) { - ret = -ENOMEM; - *bad_wr = wr; - goto out; - } - - wqe_idx = (hr_qp->rq.head + nreq) & (hr_qp->rq.wqe_cnt - 1); - - if (unlikely(wr->num_sge > hr_qp->rq.max_gs)) { - dev_err(dev, "rq:num_sge=%d > qp->sq.max_gs=%d\n", - wr->num_sge, hr_qp->rq.max_gs); - ret = -EINVAL; - *bad_wr = wr; - goto out; - } - - ctrl = hns_roce_get_recv_wqe(hr_qp, wqe_idx); - - roce_set_field(ctrl->rwqe_byte_12, - RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_M, - RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_S, - wr->num_sge); - - scat = (struct hns_roce_wqe_data_seg *)(ctrl + 1); - - for (i = 0; i < wr->num_sge; i++) - set_data_seg(scat + i, wr->sg_list + i); - - hr_qp->rq.wrid[wqe_idx] = wr->wr_id; - } - -out: - if (likely(nreq)) { - hr_qp->rq.head += nreq; - - if (ibqp->qp_type == IB_QPT_GSI) { - __le32 tmp; - - /* SW update GSI rq header */ - reg_val = roce_read(to_hr_dev(ibqp->device), - ROCEE_QP1C_CFG3_0_REG + - QP1C_CFGN_OFFSET * hr_qp->phy_port); - tmp = cpu_to_le32(reg_val); - roce_set_field(tmp, - ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_M, - ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_S, - hr_qp->rq.head); - reg_val = le32_to_cpu(tmp); - roce_write(to_hr_dev(ibqp->device), - ROCEE_QP1C_CFG3_0_REG + - QP1C_CFGN_OFFSET * hr_qp->phy_port, reg_val); - } else { - roce_set_field(rq_db.u32_4, RQ_DOORBELL_U32_4_RQ_HEAD_M, - RQ_DOORBELL_U32_4_RQ_HEAD_S, - hr_qp->rq.head); - roce_set_field(rq_db.u32_8, RQ_DOORBELL_U32_8_QPN_M, - RQ_DOORBELL_U32_8_QPN_S, hr_qp->qpn); - roce_set_field(rq_db.u32_8, RQ_DOORBELL_U32_8_CMD_M, - RQ_DOORBELL_U32_8_CMD_S, 1); - roce_set_bit(rq_db.u32_8, RQ_DOORBELL_U32_8_HW_SYNC_S, - 1); - - doorbell[0] = rq_db.u32_4; - doorbell[1] = rq_db.u32_8; - - hns_roce_write64_k(doorbell, hr_qp->rq.db_reg); - } - } - spin_unlock_irqrestore(&hr_qp->rq.lock, flags); - - return ret; -} - -static void hns_roce_set_db_event_mode(struct hns_roce_dev *hr_dev, - int sdb_mode, int odb_mode) -{ - __le32 tmp; - u32 val; - - val = roce_read(hr_dev, ROCEE_GLB_CFG_REG); - tmp = cpu_to_le32(val); - roce_set_bit(tmp, ROCEE_GLB_CFG_ROCEE_DB_SQ_MODE_S, sdb_mode); - roce_set_bit(tmp, ROCEE_GLB_CFG_ROCEE_DB_OTH_MODE_S, odb_mode); - val = le32_to_cpu(tmp); - roce_write(hr_dev, ROCEE_GLB_CFG_REG, val); -} - -static int hns_roce_v1_set_hem(struct hns_roce_dev *hr_dev, - struct hns_roce_hem_table *table, int obj, - int step_idx) -{ - spinlock_t *lock = &hr_dev->bt_cmd_lock; - struct device *dev = hr_dev->dev; - struct hns_roce_hem_iter iter; - void __iomem *bt_cmd; - __le32 bt_cmd_val[2]; - __le32 bt_cmd_h = 0; - unsigned long flags; - __le32 bt_cmd_l; - int ret = 0; - u64 bt_ba; - long end; - - /* Find the HEM(Hardware Entry Memory) entry */ - unsigned long i = obj / (table->table_chunk_size / table->obj_size); - - switch (table->type) { - case HEM_TYPE_QPC: - case HEM_TYPE_MTPT: - case HEM_TYPE_CQC: - case HEM_TYPE_SRQC: - roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M, - ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, table->type); - break; - default: - return ret; - } - - roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M, - ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj); - roce_set_bit(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0); - roce_set_bit(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1); - - /* Currently iter only a chunk */ - for (hns_roce_hem_first(table->hem[i], &iter); - !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) { - bt_ba = hns_roce_hem_addr(&iter) >> HNS_HW_PAGE_SHIFT; - - spin_lock_irqsave(lock, flags); - - bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG; - - end = HW_SYNC_TIMEOUT_MSECS; - while (end > 0) { - if (!(readl(bt_cmd) >> BT_CMD_SYNC_SHIFT)) - break; - - mdelay(HW_SYNC_SLEEP_TIME_INTERVAL); - end -= HW_SYNC_SLEEP_TIME_INTERVAL; - } - - if (end <= 0) { - dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n"); - spin_unlock_irqrestore(lock, flags); - return -EBUSY; - } - - bt_cmd_l = cpu_to_le32(bt_ba); - roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M, - ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S, - upper_32_bits(bt_ba)); - - bt_cmd_val[0] = bt_cmd_l; - bt_cmd_val[1] = bt_cmd_h; - hns_roce_write64_k(bt_cmd_val, - hr_dev->reg_base + ROCEE_BT_CMD_L_REG); - spin_unlock_irqrestore(lock, flags); - } - - return ret; -} - -static void hns_roce_set_db_ext_mode(struct hns_roce_dev *hr_dev, u32 sdb_mode, - u32 odb_mode) -{ - __le32 tmp; - u32 val; - - /* Configure SDB/ODB extend mode */ - val = roce_read(hr_dev, ROCEE_GLB_CFG_REG); - tmp = cpu_to_le32(val); - roce_set_bit(tmp, ROCEE_GLB_CFG_SQ_EXT_DB_MODE_S, sdb_mode); - roce_set_bit(tmp, ROCEE_GLB_CFG_OTH_EXT_DB_MODE_S, odb_mode); - val = le32_to_cpu(tmp); - roce_write(hr_dev, ROCEE_GLB_CFG_REG, val); -} - -static void hns_roce_set_sdb(struct hns_roce_dev *hr_dev, u32 sdb_alept, - u32 sdb_alful) -{ - __le32 tmp; - u32 val; - - /* Configure SDB */ - val = roce_read(hr_dev, ROCEE_DB_SQ_WL_REG); - tmp = cpu_to_le32(val); - roce_set_field(tmp, ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_M, - ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_S, sdb_alful); - roce_set_field(tmp, ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_M, - ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_S, sdb_alept); - val = le32_to_cpu(tmp); - roce_write(hr_dev, ROCEE_DB_SQ_WL_REG, val); -} - -static void hns_roce_set_odb(struct hns_roce_dev *hr_dev, u32 odb_alept, - u32 odb_alful) -{ - __le32 tmp; - u32 val; - - /* Configure ODB */ - val = roce_read(hr_dev, ROCEE_DB_OTHERS_WL_REG); - tmp = cpu_to_le32(val); - roce_set_field(tmp, ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_M, - ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_S, odb_alful); - roce_set_field(tmp, ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_M, - ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_S, odb_alept); - val = le32_to_cpu(tmp); - roce_write(hr_dev, ROCEE_DB_OTHERS_WL_REG, val); -} - -static void hns_roce_set_sdb_ext(struct hns_roce_dev *hr_dev, u32 ext_sdb_alept, - u32 ext_sdb_alful) -{ - struct hns_roce_v1_priv *priv = hr_dev->priv; - struct hns_roce_db_table *db = &priv->db_table; - struct device *dev = &hr_dev->pdev->dev; - dma_addr_t sdb_dma_addr; - __le32 tmp; - u32 val; - - /* Configure extend SDB threshold */ - roce_write(hr_dev, ROCEE_EXT_DB_SQ_WL_EMPTY_REG, ext_sdb_alept); - roce_write(hr_dev, ROCEE_EXT_DB_SQ_WL_REG, ext_sdb_alful); - - /* Configure extend SDB base addr */ - sdb_dma_addr = db->ext_db->sdb_buf_list->map; - roce_write(hr_dev, ROCEE_EXT_DB_SQ_REG, (u32)(sdb_dma_addr >> 12)); - - /* Configure extend SDB depth */ - val = roce_read(hr_dev, ROCEE_EXT_DB_SQ_H_REG); - tmp = cpu_to_le32(val); - roce_set_field(tmp, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_M, - ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_S, - db->ext_db->esdb_dep); - /* - * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of - * using 4K page, and shift more 32 because of - * calculating the high 32 bit value evaluated to hardware. - */ - roce_set_field(tmp, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_M, - ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_S, sdb_dma_addr >> 44); - val = le32_to_cpu(tmp); - roce_write(hr_dev, ROCEE_EXT_DB_SQ_H_REG, val); - - dev_dbg(dev, "ext SDB depth: 0x%x\n", db->ext_db->esdb_dep); - dev_dbg(dev, "ext SDB threshold: empty: 0x%x, ful: 0x%x\n", - ext_sdb_alept, ext_sdb_alful); -} - -static void hns_roce_set_odb_ext(struct hns_roce_dev *hr_dev, u32 ext_odb_alept, - u32 ext_odb_alful) -{ - struct hns_roce_v1_priv *priv = hr_dev->priv; - struct hns_roce_db_table *db = &priv->db_table; - struct device *dev = &hr_dev->pdev->dev; - dma_addr_t odb_dma_addr; - __le32 tmp; - u32 val; - - /* Configure extend ODB threshold */ - roce_write(hr_dev, ROCEE_EXT_DB_OTHERS_WL_EMPTY_REG, ext_odb_alept); - roce_write(hr_dev, ROCEE_EXT_DB_OTHERS_WL_REG, ext_odb_alful); - - /* Configure extend ODB base addr */ - odb_dma_addr = db->ext_db->odb_buf_list->map; - roce_write(hr_dev, ROCEE_EXT_DB_OTH_REG, (u32)(odb_dma_addr >> 12)); - - /* Configure extend ODB depth */ - val = roce_read(hr_dev, ROCEE_EXT_DB_OTH_H_REG); - tmp = cpu_to_le32(val); - roce_set_field(tmp, ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_M, - ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_S, - db->ext_db->eodb_dep); - roce_set_field(tmp, ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_M, - ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_S, - db->ext_db->eodb_dep); - val = le32_to_cpu(tmp); - roce_write(hr_dev, ROCEE_EXT_DB_OTH_H_REG, val); - - dev_dbg(dev, "ext ODB depth: 0x%x\n", db->ext_db->eodb_dep); - dev_dbg(dev, "ext ODB threshold: empty: 0x%x, ful: 0x%x\n", - ext_odb_alept, ext_odb_alful); -} - -static int hns_roce_db_ext_init(struct hns_roce_dev *hr_dev, u32 sdb_ext_mod, - u32 odb_ext_mod) -{ - struct hns_roce_v1_priv *priv = hr_dev->priv; - struct hns_roce_db_table *db = &priv->db_table; - struct device *dev = &hr_dev->pdev->dev; - dma_addr_t sdb_dma_addr; - dma_addr_t odb_dma_addr; - int ret = 0; - - db->ext_db = kmalloc(sizeof(*db->ext_db), GFP_KERNEL); - if (!db->ext_db) - return -ENOMEM; - - if (sdb_ext_mod) { - db->ext_db->sdb_buf_list = kmalloc( - sizeof(*db->ext_db->sdb_buf_list), GFP_KERNEL); - if (!db->ext_db->sdb_buf_list) { - ret = -ENOMEM; - goto ext_sdb_buf_fail_out; - } - - db->ext_db->sdb_buf_list->buf = dma_alloc_coherent(dev, - HNS_ROCE_V1_EXT_SDB_SIZE, - &sdb_dma_addr, GFP_KERNEL); - if (!db->ext_db->sdb_buf_list->buf) { - ret = -ENOMEM; - goto alloc_sq_db_buf_fail; - } - db->ext_db->sdb_buf_list->map = sdb_dma_addr; - - db->ext_db->esdb_dep = ilog2(HNS_ROCE_V1_EXT_SDB_DEPTH); - hns_roce_set_sdb_ext(hr_dev, HNS_ROCE_V1_EXT_SDB_ALEPT, - HNS_ROCE_V1_EXT_SDB_ALFUL); - } else - hns_roce_set_sdb(hr_dev, HNS_ROCE_V1_SDB_ALEPT, - HNS_ROCE_V1_SDB_ALFUL); - - if (odb_ext_mod) { - db->ext_db->odb_buf_list = kmalloc( - sizeof(*db->ext_db->odb_buf_list), GFP_KERNEL); - if (!db->ext_db->odb_buf_list) { - ret = -ENOMEM; - goto ext_odb_buf_fail_out; - } - - db->ext_db->odb_buf_list->buf = dma_alloc_coherent(dev, - HNS_ROCE_V1_EXT_ODB_SIZE, - &odb_dma_addr, GFP_KERNEL); - if (!db->ext_db->odb_buf_list->buf) { - ret = -ENOMEM; - goto alloc_otr_db_buf_fail; - } - db->ext_db->odb_buf_list->map = odb_dma_addr; - - db->ext_db->eodb_dep = ilog2(HNS_ROCE_V1_EXT_ODB_DEPTH); - hns_roce_set_odb_ext(hr_dev, HNS_ROCE_V1_EXT_ODB_ALEPT, - HNS_ROCE_V1_EXT_ODB_ALFUL); - } else - hns_roce_set_odb(hr_dev, HNS_ROCE_V1_ODB_ALEPT, - HNS_ROCE_V1_ODB_ALFUL); - - hns_roce_set_db_ext_mode(hr_dev, sdb_ext_mod, odb_ext_mod); - - return 0; - -alloc_otr_db_buf_fail: - kfree(db->ext_db->odb_buf_list); - -ext_odb_buf_fail_out: - if (sdb_ext_mod) { - dma_free_coherent(dev, HNS_ROCE_V1_EXT_SDB_SIZE, - db->ext_db->sdb_buf_list->buf, - db->ext_db->sdb_buf_list->map); - } - -alloc_sq_db_buf_fail: - if (sdb_ext_mod) - kfree(db->ext_db->sdb_buf_list); - -ext_sdb_buf_fail_out: - kfree(db->ext_db); - return ret; -} - -static struct hns_roce_qp *hns_roce_v1_create_lp_qp(struct hns_roce_dev *hr_dev, - struct ib_pd *pd) -{ - struct device *dev = &hr_dev->pdev->dev; - struct ib_qp_init_attr init_attr; - struct ib_qp *qp; - - memset(&init_attr, 0, sizeof(struct ib_qp_init_attr)); - init_attr.qp_type = IB_QPT_RC; - init_attr.sq_sig_type = IB_SIGNAL_ALL_WR; - init_attr.cap.max_recv_wr = HNS_ROCE_MIN_WQE_NUM; - init_attr.cap.max_send_wr = HNS_ROCE_MIN_WQE_NUM; - - qp = ib_create_qp(pd, &init_attr); - if (IS_ERR(qp)) { - dev_err(dev, "Create loop qp for mr free failed!"); - return NULL; - } - - return to_hr_qp(qp); -} - -static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev) -{ - struct hns_roce_v1_priv *priv = hr_dev->priv; - struct hns_roce_free_mr *free_mr = &priv->free_mr; - struct hns_roce_caps *caps = &hr_dev->caps; - struct ib_device *ibdev = &hr_dev->ib_dev; - struct device *dev = &hr_dev->pdev->dev; - struct ib_cq_init_attr cq_init_attr; - struct ib_qp_attr attr = { 0 }; - struct hns_roce_qp *hr_qp; - struct ib_cq *cq; - struct ib_pd *pd; - union ib_gid dgid; - __be64 subnet_prefix; - int attr_mask = 0; - int ret; - int i, j; - u8 queue_en[HNS_ROCE_V1_RESV_QP] = { 0 }; - u8 phy_port; - u32 port = 0; - u8 sl; - - /* Reserved cq for loop qp */ - cq_init_attr.cqe = HNS_ROCE_MIN_WQE_NUM * 2; - cq_init_attr.comp_vector = 0; - - cq = rdma_zalloc_drv_obj(ibdev, ib_cq); - if (!cq) - return -ENOMEM; - - ret = hns_roce_create_cq(cq, &cq_init_attr, NULL); - if (ret) { - dev_err(dev, "Create cq for reserved loop qp failed!"); - goto alloc_cq_failed; - } - free_mr->mr_free_cq = to_hr_cq(cq); - free_mr->mr_free_cq->ib_cq.device = &hr_dev->ib_dev; - free_mr->mr_free_cq->ib_cq.uobject = NULL; - free_mr->mr_free_cq->ib_cq.comp_handler = NULL; - free_mr->mr_free_cq->ib_cq.event_handler = NULL; - free_mr->mr_free_cq->ib_cq.cq_context = NULL; - atomic_set(&free_mr->mr_free_cq->ib_cq.usecnt, 0); - - pd = rdma_zalloc_drv_obj(ibdev, ib_pd); - if (!pd) { - ret = -ENOMEM; - goto alloc_mem_failed; - } - - pd->device = ibdev; - ret = hns_roce_alloc_pd(pd, NULL); - if (ret) - goto alloc_pd_failed; - - free_mr->mr_free_pd = to_hr_pd(pd); - free_mr->mr_free_pd->ibpd.device = &hr_dev->ib_dev; - free_mr->mr_free_pd->ibpd.uobject = NULL; - free_mr->mr_free_pd->ibpd.__internal_mr = NULL; - atomic_set(&free_mr->mr_free_pd->ibpd.usecnt, 0); - - attr.qp_access_flags = IB_ACCESS_REMOTE_WRITE; - attr.pkey_index = 0; - attr.min_rnr_timer = 0; - /* Disable read ability */ - attr.max_dest_rd_atomic = 0; - attr.max_rd_atomic = 0; - /* Use arbitrary values as rq_psn and sq_psn */ - attr.rq_psn = 0x0808; - attr.sq_psn = 0x0808; - attr.retry_cnt = 7; - attr.rnr_retry = 7; - attr.timeout = 0x12; - attr.path_mtu = IB_MTU_256; - attr.ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; - rdma_ah_set_grh(&attr.ah_attr, NULL, 0, 0, 1, 0); - rdma_ah_set_static_rate(&attr.ah_attr, 3); - - subnet_prefix = cpu_to_be64(0xfe80000000000000LL); - for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) { - phy_port = (i >= HNS_ROCE_MAX_PORTS) ? (i - 2) : - (i % HNS_ROCE_MAX_PORTS); - sl = i / HNS_ROCE_MAX_PORTS; - - for (j = 0; j < caps->num_ports; j++) { - if (hr_dev->iboe.phy_port[j] == phy_port) { - queue_en[i] = 1; - port = j; - break; - } - } - - if (!queue_en[i]) - continue; - - free_mr->mr_free_qp[i] = hns_roce_v1_create_lp_qp(hr_dev, pd); - if (!free_mr->mr_free_qp[i]) { - dev_err(dev, "Create loop qp failed!\n"); - ret = -ENOMEM; - goto create_lp_qp_failed; - } - hr_qp = free_mr->mr_free_qp[i]; - - hr_qp->port = port; - hr_qp->phy_port = phy_port; - hr_qp->ibqp.qp_type = IB_QPT_RC; - hr_qp->ibqp.device = &hr_dev->ib_dev; - hr_qp->ibqp.uobject = NULL; - atomic_set(&hr_qp->ibqp.usecnt, 0); - hr_qp->ibqp.pd = pd; - hr_qp->ibqp.recv_cq = cq; - hr_qp->ibqp.send_cq = cq; - - rdma_ah_set_port_num(&attr.ah_attr, port + 1); - rdma_ah_set_sl(&attr.ah_attr, sl); - attr.port_num = port + 1; - - attr.dest_qp_num = hr_qp->qpn; - memcpy(rdma_ah_retrieve_dmac(&attr.ah_attr), - hr_dev->dev_addr[port], - ETH_ALEN); - - memcpy(&dgid.raw, &subnet_prefix, sizeof(u64)); - memcpy(&dgid.raw[8], hr_dev->dev_addr[port], 3); - memcpy(&dgid.raw[13], hr_dev->dev_addr[port] + 3, 3); - dgid.raw[11] = 0xff; - dgid.raw[12] = 0xfe; - dgid.raw[8] ^= 2; - rdma_ah_set_dgid_raw(&attr.ah_attr, dgid.raw); - - ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, &attr, attr_mask, - IB_QPS_RESET, IB_QPS_INIT); - if (ret) { - dev_err(dev, "modify qp failed(%d)!\n", ret); - goto create_lp_qp_failed; - } - - ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, &attr, IB_QP_DEST_QPN, - IB_QPS_INIT, IB_QPS_RTR); - if (ret) { - dev_err(dev, "modify qp failed(%d)!\n", ret); - goto create_lp_qp_failed; - } - - ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, &attr, attr_mask, - IB_QPS_RTR, IB_QPS_RTS); - if (ret) { - dev_err(dev, "modify qp failed(%d)!\n", ret); - goto create_lp_qp_failed; - } - } - - return 0; - -create_lp_qp_failed: - for (i -= 1; i >= 0; i--) { - hr_qp = free_mr->mr_free_qp[i]; - if (ib_destroy_qp(&hr_qp->ibqp)) - dev_err(dev, "Destroy qp %d for mr free failed!\n", i); - } - - hns_roce_dealloc_pd(pd, NULL); - -alloc_pd_failed: - kfree(pd); - -alloc_mem_failed: - hns_roce_destroy_cq(cq, NULL); -alloc_cq_failed: - kfree(cq); - return ret; -} - -static void hns_roce_v1_release_lp_qp(struct hns_roce_dev *hr_dev) -{ - struct hns_roce_v1_priv *priv = hr_dev->priv; - struct hns_roce_free_mr *free_mr = &priv->free_mr; - struct device *dev = &hr_dev->pdev->dev; - struct hns_roce_qp *hr_qp; - int ret; - int i; - - for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) { - hr_qp = free_mr->mr_free_qp[i]; - if (!hr_qp) - continue; - - ret = ib_destroy_qp(&hr_qp->ibqp); - if (ret) - dev_err(dev, "Destroy qp %d for mr free failed(%d)!\n", - i, ret); - } - - hns_roce_destroy_cq(&free_mr->mr_free_cq->ib_cq, NULL); - kfree(&free_mr->mr_free_cq->ib_cq); - hns_roce_dealloc_pd(&free_mr->mr_free_pd->ibpd, NULL); - kfree(&free_mr->mr_free_pd->ibpd); -} - -static int hns_roce_db_init(struct hns_roce_dev *hr_dev) -{ - struct hns_roce_v1_priv *priv = hr_dev->priv; - struct hns_roce_db_table *db = &priv->db_table; - struct device *dev = &hr_dev->pdev->dev; - u32 sdb_ext_mod; - u32 odb_ext_mod; - u32 sdb_evt_mod; - u32 odb_evt_mod; - int ret; - - memset(db, 0, sizeof(*db)); - - /* Default DB mode */ - sdb_ext_mod = HNS_ROCE_SDB_EXTEND_MODE; - odb_ext_mod = HNS_ROCE_ODB_EXTEND_MODE; - sdb_evt_mod = HNS_ROCE_SDB_NORMAL_MODE; - odb_evt_mod = HNS_ROCE_ODB_POLL_MODE; - - db->sdb_ext_mod = sdb_ext_mod; - db->odb_ext_mod = odb_ext_mod; - - /* Init extend DB */ - ret = hns_roce_db_ext_init(hr_dev, sdb_ext_mod, odb_ext_mod); - if (ret) { - dev_err(dev, "Failed in extend DB configuration.\n"); - return ret; - } - - hns_roce_set_db_event_mode(hr_dev, sdb_evt_mod, odb_evt_mod); - - return 0; -} - -static void hns_roce_v1_recreate_lp_qp_work_fn(struct work_struct *work) -{ - struct hns_roce_recreate_lp_qp_work *lp_qp_work; - struct hns_roce_dev *hr_dev; - - lp_qp_work = container_of(work, struct hns_roce_recreate_lp_qp_work, - work); - hr_dev = to_hr_dev(lp_qp_work->ib_dev); - - hns_roce_v1_release_lp_qp(hr_dev); - - if (hns_roce_v1_rsv_lp_qp(hr_dev)) - dev_err(&hr_dev->pdev->dev, "create reserver qp failed\n"); - - if (lp_qp_work->comp_flag) - complete(lp_qp_work->comp); - - kfree(lp_qp_work); -} - -static int hns_roce_v1_recreate_lp_qp(struct hns_roce_dev *hr_dev) -{ - long end = HNS_ROCE_V1_RECREATE_LP_QP_TIMEOUT_MSECS; - struct hns_roce_v1_priv *priv = hr_dev->priv; - struct hns_roce_free_mr *free_mr = &priv->free_mr; - struct hns_roce_recreate_lp_qp_work *lp_qp_work; - struct device *dev = &hr_dev->pdev->dev; - struct completion comp; - - lp_qp_work = kzalloc(sizeof(struct hns_roce_recreate_lp_qp_work), - GFP_KERNEL); - if (!lp_qp_work) - return -ENOMEM; - - INIT_WORK(&(lp_qp_work->work), hns_roce_v1_recreate_lp_qp_work_fn); - - lp_qp_work->ib_dev = &(hr_dev->ib_dev); - lp_qp_work->comp = ∁ - lp_qp_work->comp_flag = 1; - - init_completion(lp_qp_work->comp); - - queue_work(free_mr->free_mr_wq, &(lp_qp_work->work)); - - while (end > 0) { - if (try_wait_for_completion(&comp)) - return 0; - msleep(HNS_ROCE_V1_RECREATE_LP_QP_WAIT_VALUE); - end -= HNS_ROCE_V1_RECREATE_LP_QP_WAIT_VALUE; - } - - lp_qp_work->comp_flag = 0; - if (try_wait_for_completion(&comp)) - return 0; - - dev_warn(dev, "recreate lp qp failed 20s timeout and return failed!\n"); - return -ETIMEDOUT; -} - -static int hns_roce_v1_send_lp_wqe(struct hns_roce_qp *hr_qp) -{ - struct hns_roce_dev *hr_dev = to_hr_dev(hr_qp->ibqp.device); - struct device *dev = &hr_dev->pdev->dev; - struct ib_send_wr send_wr; - const struct ib_send_wr *bad_wr; - int ret; - - memset(&send_wr, 0, sizeof(send_wr)); - send_wr.next = NULL; - send_wr.num_sge = 0; - send_wr.send_flags = 0; - send_wr.sg_list = NULL; - send_wr.wr_id = (unsigned long long)&send_wr; - send_wr.opcode = IB_WR_RDMA_WRITE; - - ret = hns_roce_v1_post_send(&hr_qp->ibqp, &send_wr, &bad_wr); - if (ret) { - dev_err(dev, "Post write wqe for mr free failed(%d)!", ret); - return ret; - } - - return 0; -} - -static void hns_roce_v1_mr_free_work_fn(struct work_struct *work) -{ - unsigned long end = - msecs_to_jiffies(HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS) + jiffies; - struct hns_roce_mr_free_work *mr_work = - container_of(work, struct hns_roce_mr_free_work, work); - struct hns_roce_dev *hr_dev = to_hr_dev(mr_work->ib_dev); - struct hns_roce_v1_priv *priv = hr_dev->priv; - struct hns_roce_free_mr *free_mr = &priv->free_mr; - struct hns_roce_cq *mr_free_cq = free_mr->mr_free_cq; - struct hns_roce_mr *hr_mr = mr_work->mr; - struct device *dev = &hr_dev->pdev->dev; - struct ib_wc wc[HNS_ROCE_V1_RESV_QP]; - struct hns_roce_qp *hr_qp; - int ne = 0; - int ret; - int i; - - for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) { - hr_qp = free_mr->mr_free_qp[i]; - if (!hr_qp) - continue; - ne++; - - ret = hns_roce_v1_send_lp_wqe(hr_qp); - if (ret) { - dev_err(dev, - "Send wqe (qp:0x%lx) for mr free failed(%d)!\n", - hr_qp->qpn, ret); - goto free_work; - } - } - - if (!ne) { - dev_err(dev, "Reserved loop qp is absent!\n"); - goto free_work; - } - - do { - ret = hns_roce_v1_poll_cq(&mr_free_cq->ib_cq, ne, wc); - if (ret < 0 && hr_qp) { - dev_err(dev, - "(qp:0x%lx) starts, Poll cqe failed(%d) for mr 0x%x free! Remain %d cqe\n", - hr_qp->qpn, ret, hr_mr->key, ne); - goto free_work; - } - ne -= ret; - usleep_range(HNS_ROCE_V1_FREE_MR_WAIT_VALUE * 1000, - (1 + HNS_ROCE_V1_FREE_MR_WAIT_VALUE) * 1000); - } while (ne && time_before_eq(jiffies, end)); - - if (ne != 0) - dev_err(dev, - "Poll cqe for mr 0x%x free timeout! Remain %d cqe\n", - hr_mr->key, ne); - -free_work: - if (mr_work->comp_flag) - complete(mr_work->comp); - kfree(mr_work); -} - -static int hns_roce_v1_dereg_mr(struct hns_roce_dev *hr_dev, - struct hns_roce_mr *mr, struct ib_udata *udata) -{ - struct hns_roce_v1_priv *priv = hr_dev->priv; - struct hns_roce_free_mr *free_mr = &priv->free_mr; - long end = HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS; - struct device *dev = &hr_dev->pdev->dev; - struct hns_roce_mr_free_work *mr_work; - unsigned long start = jiffies; - struct completion comp; - int ret = 0; - - if (mr->enabled) { - if (hns_roce_hw_destroy_mpt(hr_dev, NULL, - key_to_hw_index(mr->key) & - (hr_dev->caps.num_mtpts - 1))) - dev_warn(dev, "DESTROY_MPT failed!\n"); - } - - mr_work = kzalloc(sizeof(*mr_work), GFP_KERNEL); - if (!mr_work) { - ret = -ENOMEM; - goto free_mr; - } - - INIT_WORK(&(mr_work->work), hns_roce_v1_mr_free_work_fn); - - mr_work->ib_dev = &(hr_dev->ib_dev); - mr_work->comp = ∁ - mr_work->comp_flag = 1; - mr_work->mr = (void *)mr; - init_completion(mr_work->comp); - - queue_work(free_mr->free_mr_wq, &(mr_work->work)); - - while (end > 0) { - if (try_wait_for_completion(&comp)) - goto free_mr; - msleep(HNS_ROCE_V1_FREE_MR_WAIT_VALUE); - end -= HNS_ROCE_V1_FREE_MR_WAIT_VALUE; - } - - mr_work->comp_flag = 0; - if (try_wait_for_completion(&comp)) - goto free_mr; - - dev_warn(dev, "Free mr work 0x%x over 50s and failed!\n", mr->key); - ret = -ETIMEDOUT; - -free_mr: - dev_dbg(dev, "Free mr 0x%x use 0x%x us.\n", - mr->key, jiffies_to_usecs(jiffies) - jiffies_to_usecs(start)); - - ida_free(&hr_dev->mr_table.mtpt_ida.ida, (int)key_to_hw_index(mr->key)); - hns_roce_mtr_destroy(hr_dev, &mr->pbl_mtr); - kfree(mr); - - return ret; -} - -static void hns_roce_db_free(struct hns_roce_dev *hr_dev) -{ - struct hns_roce_v1_priv *priv = hr_dev->priv; - struct hns_roce_db_table *db = &priv->db_table; - struct device *dev = &hr_dev->pdev->dev; - - if (db->sdb_ext_mod) { - dma_free_coherent(dev, HNS_ROCE_V1_EXT_SDB_SIZE, - db->ext_db->sdb_buf_list->buf, - db->ext_db->sdb_buf_list->map); - kfree(db->ext_db->sdb_buf_list); - } - - if (db->odb_ext_mod) { - dma_free_coherent(dev, HNS_ROCE_V1_EXT_ODB_SIZE, - db->ext_db->odb_buf_list->buf, - db->ext_db->odb_buf_list->map); - kfree(db->ext_db->odb_buf_list); - } - - kfree(db->ext_db); -} - -static int hns_roce_raq_init(struct hns_roce_dev *hr_dev) -{ - struct hns_roce_v1_priv *priv = hr_dev->priv; - struct hns_roce_raq_table *raq = &priv->raq_table; - struct device *dev = &hr_dev->pdev->dev; - dma_addr_t addr; - int raq_shift; - __le32 tmp; - u32 val; - int ret; - - raq->e_raq_buf = kzalloc(sizeof(*(raq->e_raq_buf)), GFP_KERNEL); - if (!raq->e_raq_buf) - return -ENOMEM; - - raq->e_raq_buf->buf = dma_alloc_coherent(dev, HNS_ROCE_V1_RAQ_SIZE, - &addr, GFP_KERNEL); - if (!raq->e_raq_buf->buf) { - ret = -ENOMEM; - goto err_dma_alloc_raq; - } - raq->e_raq_buf->map = addr; - - /* Configure raq extended address. 48bit 4K align */ - roce_write(hr_dev, ROCEE_EXT_RAQ_REG, raq->e_raq_buf->map >> 12); - - /* Configure raq_shift */ - raq_shift = ilog2(HNS_ROCE_V1_RAQ_SIZE / HNS_ROCE_V1_RAQ_ENTRY); - val = roce_read(hr_dev, ROCEE_EXT_RAQ_H_REG); - tmp = cpu_to_le32(val); - roce_set_field(tmp, ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_M, - ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_S, raq_shift); - /* - * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of - * using 4K page, and shift more 32 because of - * calculating the high 32 bit value evaluated to hardware. - */ - roce_set_field(tmp, ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_M, - ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_S, - raq->e_raq_buf->map >> 44); - val = le32_to_cpu(tmp); - roce_write(hr_dev, ROCEE_EXT_RAQ_H_REG, val); - dev_dbg(dev, "Configure raq_shift 0x%x.\n", val); - - /* Configure raq threshold */ - val = roce_read(hr_dev, ROCEE_RAQ_WL_REG); - tmp = cpu_to_le32(val); - roce_set_field(tmp, ROCEE_RAQ_WL_ROCEE_RAQ_WL_M, - ROCEE_RAQ_WL_ROCEE_RAQ_WL_S, - HNS_ROCE_V1_EXT_RAQ_WF); - val = le32_to_cpu(tmp); - roce_write(hr_dev, ROCEE_RAQ_WL_REG, val); - dev_dbg(dev, "Configure raq_wl 0x%x.\n", val); - - /* Enable extend raq */ - val = roce_read(hr_dev, ROCEE_WRMS_POL_TIME_INTERVAL_REG); - tmp = cpu_to_le32(val); - roce_set_field(tmp, - ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_M, - ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_S, - POL_TIME_INTERVAL_VAL); - roce_set_bit(tmp, ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_EXT_RAQ_MODE, 1); - roce_set_field(tmp, - ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_M, - ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_S, - 2); - roce_set_bit(tmp, - ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_EN_S, 1); - val = le32_to_cpu(tmp); - roce_write(hr_dev, ROCEE_WRMS_POL_TIME_INTERVAL_REG, val); - dev_dbg(dev, "Configure WrmsPolTimeInterval 0x%x.\n", val); - - /* Enable raq drop */ - val = roce_read(hr_dev, ROCEE_GLB_CFG_REG); - tmp = cpu_to_le32(val); - roce_set_bit(tmp, ROCEE_GLB_CFG_TRP_RAQ_DROP_EN_S, 1); - val = le32_to_cpu(tmp); - roce_write(hr_dev, ROCEE_GLB_CFG_REG, val); - dev_dbg(dev, "Configure GlbCfg = 0x%x.\n", val); - - return 0; - -err_dma_alloc_raq: - kfree(raq->e_raq_buf); - return ret; -} - -static void hns_roce_raq_free(struct hns_roce_dev *hr_dev) -{ - struct hns_roce_v1_priv *priv = hr_dev->priv; - struct hns_roce_raq_table *raq = &priv->raq_table; - struct device *dev = &hr_dev->pdev->dev; - - dma_free_coherent(dev, HNS_ROCE_V1_RAQ_SIZE, raq->e_raq_buf->buf, - raq->e_raq_buf->map); - kfree(raq->e_raq_buf); -} - -static void hns_roce_port_enable(struct hns_roce_dev *hr_dev, int enable_flag) -{ - __le32 tmp; - u32 val; - - if (enable_flag) { - val = roce_read(hr_dev, ROCEE_GLB_CFG_REG); - /* Open all ports */ - tmp = cpu_to_le32(val); - roce_set_field(tmp, ROCEE_GLB_CFG_ROCEE_PORT_ST_M, - ROCEE_GLB_CFG_ROCEE_PORT_ST_S, - ALL_PORT_VAL_OPEN); - val = le32_to_cpu(tmp); - roce_write(hr_dev, ROCEE_GLB_CFG_REG, val); - } else { - val = roce_read(hr_dev, ROCEE_GLB_CFG_REG); - /* Close all ports */ - tmp = cpu_to_le32(val); - roce_set_field(tmp, ROCEE_GLB_CFG_ROCEE_PORT_ST_M, - ROCEE_GLB_CFG_ROCEE_PORT_ST_S, 0x0); - val = le32_to_cpu(tmp); - roce_write(hr_dev, ROCEE_GLB_CFG_REG, val); - } -} - -static int hns_roce_bt_init(struct hns_roce_dev *hr_dev) -{ - struct hns_roce_v1_priv *priv = hr_dev->priv; - struct device *dev = &hr_dev->pdev->dev; - int ret; - - priv->bt_table.qpc_buf.buf = dma_alloc_coherent(dev, - HNS_ROCE_BT_RSV_BUF_SIZE, &priv->bt_table.qpc_buf.map, - GFP_KERNEL); - if (!priv->bt_table.qpc_buf.buf) - return -ENOMEM; - - priv->bt_table.mtpt_buf.buf = dma_alloc_coherent(dev, - HNS_ROCE_BT_RSV_BUF_SIZE, &priv->bt_table.mtpt_buf.map, - GFP_KERNEL); - if (!priv->bt_table.mtpt_buf.buf) { - ret = -ENOMEM; - goto err_failed_alloc_mtpt_buf; - } - - priv->bt_table.cqc_buf.buf = dma_alloc_coherent(dev, - HNS_ROCE_BT_RSV_BUF_SIZE, &priv->bt_table.cqc_buf.map, - GFP_KERNEL); - if (!priv->bt_table.cqc_buf.buf) { - ret = -ENOMEM; - goto err_failed_alloc_cqc_buf; - } - - return 0; - -err_failed_alloc_cqc_buf: - dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE, - priv->bt_table.mtpt_buf.buf, priv->bt_table.mtpt_buf.map); - -err_failed_alloc_mtpt_buf: - dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE, - priv->bt_table.qpc_buf.buf, priv->bt_table.qpc_buf.map); - - return ret; -} - -static void hns_roce_bt_free(struct hns_roce_dev *hr_dev) -{ - struct hns_roce_v1_priv *priv = hr_dev->priv; - struct device *dev = &hr_dev->pdev->dev; - - dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE, - priv->bt_table.cqc_buf.buf, priv->bt_table.cqc_buf.map); - - dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE, - priv->bt_table.mtpt_buf.buf, priv->bt_table.mtpt_buf.map); - - dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE, - priv->bt_table.qpc_buf.buf, priv->bt_table.qpc_buf.map); -} - -static int hns_roce_tptr_init(struct hns_roce_dev *hr_dev) -{ - struct hns_roce_v1_priv *priv = hr_dev->priv; - struct hns_roce_buf_list *tptr_buf = &priv->tptr_table.tptr_buf; - struct device *dev = &hr_dev->pdev->dev; - - /* - * This buffer will be used for CQ's tptr(tail pointer), also - * named ci(customer index). Every CQ will use 2 bytes to save - * cqe ci in hip06. Hardware will read this area to get new ci - * when the queue is almost full. - */ - tptr_buf->buf = dma_alloc_coherent(dev, HNS_ROCE_V1_TPTR_BUF_SIZE, - &tptr_buf->map, GFP_KERNEL); - if (!tptr_buf->buf) - return -ENOMEM; - - hr_dev->tptr_dma_addr = tptr_buf->map; - hr_dev->tptr_size = HNS_ROCE_V1_TPTR_BUF_SIZE; - - return 0; -} - -static void hns_roce_tptr_free(struct hns_roce_dev *hr_dev) -{ - struct hns_roce_v1_priv *priv = hr_dev->priv; - struct hns_roce_buf_list *tptr_buf = &priv->tptr_table.tptr_buf; - struct device *dev = &hr_dev->pdev->dev; - - dma_free_coherent(dev, HNS_ROCE_V1_TPTR_BUF_SIZE, - tptr_buf->buf, tptr_buf->map); -} - -static int hns_roce_free_mr_init(struct hns_roce_dev *hr_dev) -{ - struct hns_roce_v1_priv *priv = hr_dev->priv; - struct hns_roce_free_mr *free_mr = &priv->free_mr; - struct device *dev = &hr_dev->pdev->dev; - int ret; - - free_mr->free_mr_wq = create_singlethread_workqueue("hns_roce_free_mr"); - if (!free_mr->free_mr_wq) { - dev_err(dev, "Create free mr workqueue failed!\n"); - return -ENOMEM; - } - - ret = hns_roce_v1_rsv_lp_qp(hr_dev); - if (ret) { - dev_err(dev, "Reserved loop qp failed(%d)!\n", ret); - destroy_workqueue(free_mr->free_mr_wq); - } - - return ret; -} - -static void hns_roce_free_mr_free(struct hns_roce_dev *hr_dev) -{ - struct hns_roce_v1_priv *priv = hr_dev->priv; - struct hns_roce_free_mr *free_mr = &priv->free_mr; - - destroy_workqueue(free_mr->free_mr_wq); - - hns_roce_v1_release_lp_qp(hr_dev); -} - -/** - * hns_roce_v1_reset - reset RoCE - * @hr_dev: RoCE device struct pointer - * @dereset: true -- drop reset, false -- reset - * return 0 - success , negative --fail - */ -static int hns_roce_v1_reset(struct hns_roce_dev *hr_dev, bool dereset) -{ - struct device_node *dsaf_node; - struct device *dev = &hr_dev->pdev->dev; - struct device_node *np = dev->of_node; - struct fwnode_handle *fwnode; - int ret; - - /* check if this is DT/ACPI case */ - if (dev_of_node(dev)) { - dsaf_node = of_parse_phandle(np, "dsaf-handle", 0); - if (!dsaf_node) { - dev_err(dev, "could not find dsaf-handle\n"); - return -EINVAL; - } - fwnode = &dsaf_node->fwnode; - } else if (is_acpi_device_node(dev->fwnode)) { - struct fwnode_reference_args args; - - ret = acpi_node_get_property_reference(dev->fwnode, - "dsaf-handle", 0, &args); - if (ret) { - dev_err(dev, "could not find dsaf-handle\n"); - return ret; - } - fwnode = args.fwnode; - } else { - dev_err(dev, "cannot read data from DT or ACPI\n"); - return -ENXIO; - } - - ret = hns_dsaf_roce_reset(fwnode, false); - if (ret) - return ret; - - if (dereset) { - msleep(SLEEP_TIME_INTERVAL); - ret = hns_dsaf_roce_reset(fwnode, true); - } - - return ret; -} - -static int hns_roce_v1_profile(struct hns_roce_dev *hr_dev) -{ - struct hns_roce_caps *caps = &hr_dev->caps; - int i; - - hr_dev->vendor_id = roce_read(hr_dev, ROCEE_VENDOR_ID_REG); - hr_dev->vendor_part_id = roce_read(hr_dev, ROCEE_VENDOR_PART_ID_REG); - hr_dev->sys_image_guid = roce_read(hr_dev, ROCEE_SYS_IMAGE_GUID_L_REG) | - ((u64)roce_read(hr_dev, - ROCEE_SYS_IMAGE_GUID_H_REG) << 32); - hr_dev->hw_rev = HNS_ROCE_HW_VER1; - - caps->num_qps = HNS_ROCE_V1_MAX_QP_NUM; - caps->max_wqes = HNS_ROCE_V1_MAX_WQE_NUM; - caps->min_wqes = HNS_ROCE_MIN_WQE_NUM; - caps->num_cqs = HNS_ROCE_V1_MAX_CQ_NUM; - caps->min_cqes = HNS_ROCE_MIN_CQE_NUM; - caps->max_cqes = HNS_ROCE_V1_MAX_CQE_NUM; - caps->max_sq_sg = HNS_ROCE_V1_SG_NUM; - caps->max_rq_sg = HNS_ROCE_V1_SG_NUM; - caps->max_sq_inline = HNS_ROCE_V1_INLINE_SIZE; - caps->num_uars = HNS_ROCE_V1_UAR_NUM; - caps->phy_num_uars = HNS_ROCE_V1_PHY_UAR_NUM; - caps->num_aeq_vectors = HNS_ROCE_V1_AEQE_VEC_NUM; - caps->num_comp_vectors = HNS_ROCE_V1_COMP_VEC_NUM; - caps->num_other_vectors = HNS_ROCE_V1_ABNORMAL_VEC_NUM; - caps->num_mtpts = HNS_ROCE_V1_MAX_MTPT_NUM; - caps->num_mtt_segs = HNS_ROCE_V1_MAX_MTT_SEGS; - caps->num_pds = HNS_ROCE_V1_MAX_PD_NUM; - caps->max_qp_init_rdma = HNS_ROCE_V1_MAX_QP_INIT_RDMA; - caps->max_qp_dest_rdma = HNS_ROCE_V1_MAX_QP_DEST_RDMA; - caps->max_sq_desc_sz = HNS_ROCE_V1_MAX_SQ_DESC_SZ; - caps->max_rq_desc_sz = HNS_ROCE_V1_MAX_RQ_DESC_SZ; - caps->qpc_sz = HNS_ROCE_V1_QPC_SIZE; - caps->irrl_entry_sz = HNS_ROCE_V1_IRRL_ENTRY_SIZE; - caps->cqc_entry_sz = HNS_ROCE_V1_CQC_ENTRY_SIZE; - caps->mtpt_entry_sz = HNS_ROCE_V1_MTPT_ENTRY_SIZE; - caps->mtt_entry_sz = HNS_ROCE_V1_MTT_ENTRY_SIZE; - caps->cqe_sz = HNS_ROCE_V1_CQE_SIZE; - caps->page_size_cap = HNS_ROCE_V1_PAGE_SIZE_SUPPORT; - caps->reserved_lkey = 0; - caps->reserved_pds = 0; - caps->reserved_mrws = 1; - caps->reserved_uars = 0; - caps->reserved_cqs = 0; - caps->reserved_qps = 12; /* 2 SQP per port, six ports total 12 */ - caps->chunk_sz = HNS_ROCE_V1_TABLE_CHUNK_SIZE; - - for (i = 0; i < caps->num_ports; i++) - caps->pkey_table_len[i] = 1; - - for (i = 0; i < caps->num_ports; i++) { - /* Six ports shared 16 GID in v1 engine */ - if (i >= (HNS_ROCE_V1_GID_NUM % caps->num_ports)) - caps->gid_table_len[i] = HNS_ROCE_V1_GID_NUM / - caps->num_ports; - else - caps->gid_table_len[i] = HNS_ROCE_V1_GID_NUM / - caps->num_ports + 1; - } - - caps->ceqe_depth = HNS_ROCE_V1_COMP_EQE_NUM; - caps->aeqe_depth = HNS_ROCE_V1_ASYNC_EQE_NUM; - caps->local_ca_ack_delay = roce_read(hr_dev, ROCEE_ACK_DELAY_REG); - caps->max_mtu = IB_MTU_2048; - - return 0; -} - -static int hns_roce_v1_init(struct hns_roce_dev *hr_dev) -{ - int ret; - u32 val; - __le32 tmp; - struct device *dev = &hr_dev->pdev->dev; - - /* DMAE user config */ - val = roce_read(hr_dev, ROCEE_DMAE_USER_CFG1_REG); - tmp = cpu_to_le32(val); - roce_set_field(tmp, ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_M, - ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_S, 0xf); - roce_set_field(tmp, ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_M, - ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_S, - 1 << PAGES_SHIFT_16); - val = le32_to_cpu(tmp); - roce_write(hr_dev, ROCEE_DMAE_USER_CFG1_REG, val); - - val = roce_read(hr_dev, ROCEE_DMAE_USER_CFG2_REG); - tmp = cpu_to_le32(val); - roce_set_field(tmp, ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_M, - ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_S, 0xf); - roce_set_field(tmp, ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_M, - ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_S, - 1 << PAGES_SHIFT_16); - - ret = hns_roce_db_init(hr_dev); - if (ret) { - dev_err(dev, "doorbell init failed!\n"); - return ret; - } - - ret = hns_roce_raq_init(hr_dev); - if (ret) { - dev_err(dev, "raq init failed!\n"); - goto error_failed_raq_init; - } - - ret = hns_roce_bt_init(hr_dev); - if (ret) { - dev_err(dev, "bt init failed!\n"); - goto error_failed_bt_init; - } - - ret = hns_roce_tptr_init(hr_dev); - if (ret) { - dev_err(dev, "tptr init failed!\n"); - goto error_failed_tptr_init; - } - - ret = hns_roce_free_mr_init(hr_dev); - if (ret) { - dev_err(dev, "free mr init failed!\n"); - goto error_failed_free_mr_init; - } - - hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_UP); - - return 0; - -error_failed_free_mr_init: - hns_roce_tptr_free(hr_dev); - -error_failed_tptr_init: - hns_roce_bt_free(hr_dev); - -error_failed_bt_init: - hns_roce_raq_free(hr_dev); - -error_failed_raq_init: - hns_roce_db_free(hr_dev); - return ret; -} - -static void hns_roce_v1_exit(struct hns_roce_dev *hr_dev) -{ - hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_DOWN); - hns_roce_free_mr_free(hr_dev); - hns_roce_tptr_free(hr_dev); - hns_roce_bt_free(hr_dev); - hns_roce_raq_free(hr_dev); - hns_roce_db_free(hr_dev); -} - -static int hns_roce_v1_cmd_pending(struct hns_roce_dev *hr_dev) -{ - u32 status = readl(hr_dev->reg_base + ROCEE_MB6_REG); - - return (!!(status & (1 << HCR_GO_BIT))); -} - -static int hns_roce_v1_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param, - u64 out_param, u32 in_modifier, u8 op_modifier, - u16 op, u16 token, int event) -{ - u32 __iomem *hcr = (u32 __iomem *)(hr_dev->reg_base + ROCEE_MB1_REG); - unsigned long end; - u32 val = 0; - __le32 tmp; - - end = msecs_to_jiffies(GO_BIT_TIMEOUT_MSECS) + jiffies; - while (hns_roce_v1_cmd_pending(hr_dev)) { - if (time_after(jiffies, end)) { - dev_err(hr_dev->dev, "jiffies=%d end=%d\n", - (int)jiffies, (int)end); - return -EAGAIN; - } - cond_resched(); - } - - tmp = cpu_to_le32(val); - roce_set_field(tmp, ROCEE_MB6_ROCEE_MB_CMD_M, ROCEE_MB6_ROCEE_MB_CMD_S, - op); - roce_set_field(tmp, ROCEE_MB6_ROCEE_MB_CMD_MDF_M, - ROCEE_MB6_ROCEE_MB_CMD_MDF_S, op_modifier); - roce_set_bit(tmp, ROCEE_MB6_ROCEE_MB_EVENT_S, event); - roce_set_bit(tmp, ROCEE_MB6_ROCEE_MB_HW_RUN_S, 1); - roce_set_field(tmp, ROCEE_MB6_ROCEE_MB_TOKEN_M, - ROCEE_MB6_ROCEE_MB_TOKEN_S, token); - - val = le32_to_cpu(tmp); - writeq(in_param, hcr + 0); - writeq(out_param, hcr + 2); - writel(in_modifier, hcr + 4); - /* Memory barrier */ - wmb(); - - writel(val, hcr + 5); - - return 0; -} - -static int hns_roce_v1_chk_mbox(struct hns_roce_dev *hr_dev, - unsigned int timeout) -{ - u8 __iomem *hcr = hr_dev->reg_base + ROCEE_MB1_REG; - unsigned long end; - u32 status = 0; - - end = msecs_to_jiffies(timeout) + jiffies; - while (hns_roce_v1_cmd_pending(hr_dev) && time_before(jiffies, end)) - cond_resched(); - - if (hns_roce_v1_cmd_pending(hr_dev)) { - dev_err(hr_dev->dev, "[cmd_poll]hw run cmd TIMEDOUT!\n"); - return -ETIMEDOUT; - } - - status = le32_to_cpu((__force __le32) - __raw_readl(hcr + HCR_STATUS_OFFSET)); - if ((status & STATUS_MASK) != 0x1) { - dev_err(hr_dev->dev, "mailbox status 0x%x!\n", status); - return -EBUSY; - } - - return 0; -} - -static int hns_roce_v1_set_gid(struct hns_roce_dev *hr_dev, u32 port, - int gid_index, const union ib_gid *gid, - const struct ib_gid_attr *attr) -{ - unsigned long flags; - u32 *p = NULL; - u8 gid_idx; - - gid_idx = hns_get_gid_index(hr_dev, port, gid_index); - - spin_lock_irqsave(&hr_dev->iboe.lock, flags); - - p = (u32 *)&gid->raw[0]; - roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_L_0_REG + - (HNS_ROCE_V1_GID_NUM * gid_idx)); - - p = (u32 *)&gid->raw[4]; - roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_ML_0_REG + - (HNS_ROCE_V1_GID_NUM * gid_idx)); - - p = (u32 *)&gid->raw[8]; - roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_MH_0_REG + - (HNS_ROCE_V1_GID_NUM * gid_idx)); - - p = (u32 *)&gid->raw[0xc]; - roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_H_0_REG + - (HNS_ROCE_V1_GID_NUM * gid_idx)); - - spin_unlock_irqrestore(&hr_dev->iboe.lock, flags); - - return 0; -} - -static int hns_roce_v1_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port, - const u8 *addr) -{ - u32 reg_smac_l; - u16 reg_smac_h; - __le32 tmp; - u16 *p_h; - u32 *p; - u32 val; - - /* - * When mac changed, loopback may fail - * because of smac not equal to dmac. - * We Need to release and create reserved qp again. - */ - if (hr_dev->hw->dereg_mr) { - int ret; - - ret = hns_roce_v1_recreate_lp_qp(hr_dev); - if (ret && ret != -ETIMEDOUT) - return ret; - } - - p = (u32 *)(&addr[0]); - reg_smac_l = *p; - roce_raw_write(reg_smac_l, hr_dev->reg_base + ROCEE_SMAC_L_0_REG + - PHY_PORT_OFFSET * phy_port); - - val = roce_read(hr_dev, - ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET); - tmp = cpu_to_le32(val); - p_h = (u16 *)(&addr[4]); - reg_smac_h = *p_h; - roce_set_field(tmp, ROCEE_SMAC_H_ROCEE_SMAC_H_M, - ROCEE_SMAC_H_ROCEE_SMAC_H_S, reg_smac_h); - val = le32_to_cpu(tmp); - roce_write(hr_dev, ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET, - val); - - return 0; -} - -static void hns_roce_v1_set_mtu(struct hns_roce_dev *hr_dev, u8 phy_port, - enum ib_mtu mtu) -{ - __le32 tmp; - u32 val; - - val = roce_read(hr_dev, - ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET); - tmp = cpu_to_le32(val); - roce_set_field(tmp, ROCEE_SMAC_H_ROCEE_PORT_MTU_M, - ROCEE_SMAC_H_ROCEE_PORT_MTU_S, mtu); - val = le32_to_cpu(tmp); - roce_write(hr_dev, ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET, - val); -} - -static int hns_roce_v1_write_mtpt(struct hns_roce_dev *hr_dev, void *mb_buf, - struct hns_roce_mr *mr, - unsigned long mtpt_idx) -{ - u64 pages[HNS_ROCE_MAX_INNER_MTPT_NUM] = { 0 }; - struct ib_device *ibdev = &hr_dev->ib_dev; - struct hns_roce_v1_mpt_entry *mpt_entry; - dma_addr_t pbl_ba; - int count; - int i; - - /* MPT filled into mailbox buf */ - mpt_entry = (struct hns_roce_v1_mpt_entry *)mb_buf; - memset(mpt_entry, 0, sizeof(*mpt_entry)); - - roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_KEY_STATE_M, - MPT_BYTE_4_KEY_STATE_S, KEY_VALID); - roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_KEY_M, - MPT_BYTE_4_KEY_S, mr->key); - roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_PAGE_SIZE_M, - MPT_BYTE_4_PAGE_SIZE_S, MR_SIZE_4K); - roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_MW_TYPE_S, 0); - roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_MW_BIND_ENABLE_S, - (mr->access & IB_ACCESS_MW_BIND ? 1 : 0)); - roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_OWN_S, 0); - roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_MEMORY_LOCATION_TYPE_M, - MPT_BYTE_4_MEMORY_LOCATION_TYPE_S, mr->type); - roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_ATOMIC_S, 0); - roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_LOCAL_WRITE_S, - (mr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0)); - roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_WRITE_S, - (mr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0)); - roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_READ_S, - (mr->access & IB_ACCESS_REMOTE_READ ? 1 : 0)); - roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_INVAL_ENABLE_S, - 0); - roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_ADDRESS_TYPE_S, 0); - - roce_set_field(mpt_entry->mpt_byte_12, MPT_BYTE_12_PBL_ADDR_H_M, - MPT_BYTE_12_PBL_ADDR_H_S, 0); - roce_set_field(mpt_entry->mpt_byte_12, MPT_BYTE_12_MW_BIND_COUNTER_M, - MPT_BYTE_12_MW_BIND_COUNTER_S, 0); - - mpt_entry->virt_addr_l = cpu_to_le32((u32)mr->iova); - mpt_entry->virt_addr_h = cpu_to_le32((u32)(mr->iova >> 32)); - mpt_entry->length = cpu_to_le32((u32)mr->size); - - roce_set_field(mpt_entry->mpt_byte_28, MPT_BYTE_28_PD_M, - MPT_BYTE_28_PD_S, mr->pd); - roce_set_field(mpt_entry->mpt_byte_28, MPT_BYTE_28_L_KEY_IDX_L_M, - MPT_BYTE_28_L_KEY_IDX_L_S, mtpt_idx); - roce_set_field(mpt_entry->mpt_byte_64, MPT_BYTE_64_L_KEY_IDX_H_M, - MPT_BYTE_64_L_KEY_IDX_H_S, mtpt_idx >> MTPT_IDX_SHIFT); - - /* DMA memory register */ - if (mr->type == MR_TYPE_DMA) - return 0; - - count = hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, pages, - ARRAY_SIZE(pages), &pbl_ba); - if (count < 1) { - ibdev_err(ibdev, "failed to find PBL mtr, count = %d.", count); - return -ENOBUFS; - } - - /* Register user mr */ - for (i = 0; i < count; i++) { - switch (i) { - case 0: - mpt_entry->pa0_l = cpu_to_le32((u32)(pages[i])); - roce_set_field(mpt_entry->mpt_byte_36, - MPT_BYTE_36_PA0_H_M, - MPT_BYTE_36_PA0_H_S, - (u32)(pages[i] >> PAGES_SHIFT_32)); - break; - case 1: - roce_set_field(mpt_entry->mpt_byte_36, - MPT_BYTE_36_PA1_L_M, - MPT_BYTE_36_PA1_L_S, (u32)(pages[i])); - roce_set_field(mpt_entry->mpt_byte_40, - MPT_BYTE_40_PA1_H_M, - MPT_BYTE_40_PA1_H_S, - (u32)(pages[i] >> PAGES_SHIFT_24)); - break; - case 2: - roce_set_field(mpt_entry->mpt_byte_40, - MPT_BYTE_40_PA2_L_M, - MPT_BYTE_40_PA2_L_S, (u32)(pages[i])); - roce_set_field(mpt_entry->mpt_byte_44, - MPT_BYTE_44_PA2_H_M, - MPT_BYTE_44_PA2_H_S, - (u32)(pages[i] >> PAGES_SHIFT_16)); - break; - case 3: - roce_set_field(mpt_entry->mpt_byte_44, - MPT_BYTE_44_PA3_L_M, - MPT_BYTE_44_PA3_L_S, (u32)(pages[i])); - roce_set_field(mpt_entry->mpt_byte_48, - MPT_BYTE_48_PA3_H_M, - MPT_BYTE_48_PA3_H_S, - (u32)(pages[i] >> PAGES_SHIFT_8)); - break; - case 4: - mpt_entry->pa4_l = cpu_to_le32((u32)(pages[i])); - roce_set_field(mpt_entry->mpt_byte_56, - MPT_BYTE_56_PA4_H_M, - MPT_BYTE_56_PA4_H_S, - (u32)(pages[i] >> PAGES_SHIFT_32)); - break; - case 5: - roce_set_field(mpt_entry->mpt_byte_56, - MPT_BYTE_56_PA5_L_M, - MPT_BYTE_56_PA5_L_S, (u32)(pages[i])); - roce_set_field(mpt_entry->mpt_byte_60, - MPT_BYTE_60_PA5_H_M, - MPT_BYTE_60_PA5_H_S, - (u32)(pages[i] >> PAGES_SHIFT_24)); - break; - case 6: - roce_set_field(mpt_entry->mpt_byte_60, - MPT_BYTE_60_PA6_L_M, - MPT_BYTE_60_PA6_L_S, (u32)(pages[i])); - roce_set_field(mpt_entry->mpt_byte_64, - MPT_BYTE_64_PA6_H_M, - MPT_BYTE_64_PA6_H_S, - (u32)(pages[i] >> PAGES_SHIFT_16)); - break; - default: - break; - } - } - - mpt_entry->pbl_addr_l = cpu_to_le32(pbl_ba); - roce_set_field(mpt_entry->mpt_byte_12, MPT_BYTE_12_PBL_ADDR_H_M, - MPT_BYTE_12_PBL_ADDR_H_S, upper_32_bits(pbl_ba)); - - return 0; -} - -static void *get_cqe(struct hns_roce_cq *hr_cq, int n) -{ - return hns_roce_buf_offset(hr_cq->mtr.kmem, n * HNS_ROCE_V1_CQE_SIZE); -} - -static void *get_sw_cqe(struct hns_roce_cq *hr_cq, int n) -{ - struct hns_roce_cqe *hr_cqe = get_cqe(hr_cq, n & hr_cq->ib_cq.cqe); - - /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */ - return (roce_get_bit(hr_cqe->cqe_byte_4, CQE_BYTE_4_OWNER_S) ^ - !!(n & hr_cq->cq_depth)) ? hr_cqe : NULL; -} - -static struct hns_roce_cqe *next_cqe_sw(struct hns_roce_cq *hr_cq) -{ - return get_sw_cqe(hr_cq, hr_cq->cons_index); -} - -static void hns_roce_v1_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index) -{ - __le32 doorbell[2]; - - doorbell[0] = cpu_to_le32(cons_index & ((hr_cq->cq_depth << 1) - 1)); - doorbell[1] = 0; - roce_set_bit(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S, 1); - roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M, - ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S, 3); - roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M, - ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S, 0); - roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M, - ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S, hr_cq->cqn); - - hns_roce_write64_k(doorbell, hr_cq->db_reg); -} - -static void __hns_roce_v1_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn, - struct hns_roce_srq *srq) -{ - struct hns_roce_cqe *cqe, *dest; - u32 prod_index; - int nfreed = 0; - u8 owner_bit; - - for (prod_index = hr_cq->cons_index; get_sw_cqe(hr_cq, prod_index); - ++prod_index) { - if (prod_index == hr_cq->cons_index + hr_cq->ib_cq.cqe) - break; - } - - /* - * Now backwards through the CQ, removing CQ entries - * that match our QP by overwriting them with next entries. - */ - while ((int) --prod_index - (int) hr_cq->cons_index >= 0) { - cqe = get_cqe(hr_cq, prod_index & hr_cq->ib_cq.cqe); - if ((roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M, - CQE_BYTE_16_LOCAL_QPN_S) & - HNS_ROCE_CQE_QPN_MASK) == qpn) { - /* In v1 engine, not support SRQ */ - ++nfreed; - } else if (nfreed) { - dest = get_cqe(hr_cq, (prod_index + nfreed) & - hr_cq->ib_cq.cqe); - owner_bit = roce_get_bit(dest->cqe_byte_4, - CQE_BYTE_4_OWNER_S); - memcpy(dest, cqe, sizeof(*cqe)); - roce_set_bit(dest->cqe_byte_4, CQE_BYTE_4_OWNER_S, - owner_bit); - } - } - - if (nfreed) { - hr_cq->cons_index += nfreed; - hns_roce_v1_cq_set_ci(hr_cq, hr_cq->cons_index); - } -} - -static void hns_roce_v1_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn, - struct hns_roce_srq *srq) -{ - spin_lock_irq(&hr_cq->lock); - __hns_roce_v1_cq_clean(hr_cq, qpn, srq); - spin_unlock_irq(&hr_cq->lock); -} - -static void hns_roce_v1_write_cqc(struct hns_roce_dev *hr_dev, - struct hns_roce_cq *hr_cq, void *mb_buf, - u64 *mtts, dma_addr_t dma_handle) -{ - struct hns_roce_v1_priv *priv = hr_dev->priv; - struct hns_roce_buf_list *tptr_buf = &priv->tptr_table.tptr_buf; - struct hns_roce_cq_context *cq_context = mb_buf; - dma_addr_t tptr_dma_addr; - int offset; - - memset(cq_context, 0, sizeof(*cq_context)); - - /* Get the tptr for this CQ. */ - offset = hr_cq->cqn * HNS_ROCE_V1_TPTR_ENTRY_SIZE; - tptr_dma_addr = tptr_buf->map + offset; - hr_cq->tptr_addr = (u16 *)(tptr_buf->buf + offset); - - /* Register cq_context members */ - roce_set_field(cq_context->cqc_byte_4, - CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_M, - CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_S, CQ_STATE_VALID); - roce_set_field(cq_context->cqc_byte_4, CQ_CONTEXT_CQC_BYTE_4_CQN_M, - CQ_CONTEXT_CQC_BYTE_4_CQN_S, hr_cq->cqn); - - cq_context->cq_bt_l = cpu_to_le32((u32)dma_handle); - - roce_set_field(cq_context->cqc_byte_12, - CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_M, - CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_S, - ((u64)dma_handle >> 32)); - roce_set_field(cq_context->cqc_byte_12, - CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_M, - CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_S, - ilog2(hr_cq->cq_depth)); - roce_set_field(cq_context->cqc_byte_12, CQ_CONTEXT_CQC_BYTE_12_CEQN_M, - CQ_CONTEXT_CQC_BYTE_12_CEQN_S, hr_cq->vector); - - cq_context->cur_cqe_ba0_l = cpu_to_le32((u32)(mtts[0])); - - roce_set_field(cq_context->cqc_byte_20, - CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_M, - CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_S, (mtts[0]) >> 32); - /* Dedicated hardware, directly set 0 */ - roce_set_field(cq_context->cqc_byte_20, - CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_M, - CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_S, 0); - /** - * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of - * using 4K page, and shift more 32 because of - * calculating the high 32 bit value evaluated to hardware. - */ - roce_set_field(cq_context->cqc_byte_20, - CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_M, - CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_S, - tptr_dma_addr >> 44); - - cq_context->cqe_tptr_addr_l = cpu_to_le32((u32)(tptr_dma_addr >> 12)); - - roce_set_field(cq_context->cqc_byte_32, - CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_M, - CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_S, 0); - roce_set_bit(cq_context->cqc_byte_32, - CQ_CONTEXT_CQC_BYTE_32_SE_FLAG_S, 0); - roce_set_bit(cq_context->cqc_byte_32, - CQ_CONTEXT_CQC_BYTE_32_CE_FLAG_S, 0); - roce_set_bit(cq_context->cqc_byte_32, - CQ_CONTEXT_CQC_BYTE_32_NOTIFICATION_FLAG_S, 0); - roce_set_bit(cq_context->cqc_byte_32, - CQ_CQNTEXT_CQC_BYTE_32_TYPE_OF_COMPLETION_NOTIFICATION_S, - 0); - /* The initial value of cq's ci is 0 */ - roce_set_field(cq_context->cqc_byte_32, - CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_M, - CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_S, 0); -} - -static int hns_roce_v1_req_notify_cq(struct ib_cq *ibcq, - enum ib_cq_notify_flags flags) -{ - struct hns_roce_cq *hr_cq = to_hr_cq(ibcq); - u32 notification_flag; - __le32 doorbell[2] = {}; - - notification_flag = (flags & IB_CQ_SOLICITED_MASK) == - IB_CQ_SOLICITED ? CQ_DB_REQ_NOT : CQ_DB_REQ_NOT_SOL; - /* - * flags = 0; Notification Flag = 1, next - * flags = 1; Notification Flag = 0, solocited - */ - doorbell[0] = - cpu_to_le32(hr_cq->cons_index & ((hr_cq->cq_depth << 1) - 1)); - roce_set_bit(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S, 1); - roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M, - ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S, 3); - roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M, - ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S, 1); - roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M, - ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S, - hr_cq->cqn | notification_flag); - - hns_roce_write64_k(doorbell, hr_cq->db_reg); - - return 0; -} - -static int hns_roce_v1_poll_one(struct hns_roce_cq *hr_cq, - struct hns_roce_qp **cur_qp, struct ib_wc *wc) -{ - int qpn; - int is_send; - u16 wqe_ctr; - u32 status; - u32 opcode; - struct hns_roce_cqe *cqe; - struct hns_roce_qp *hr_qp; - struct hns_roce_wq *wq; - struct hns_roce_wqe_ctrl_seg *sq_wqe; - struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device); - struct device *dev = &hr_dev->pdev->dev; - - /* Find cqe according consumer index */ - cqe = next_cqe_sw(hr_cq); - if (!cqe) - return -EAGAIN; - - ++hr_cq->cons_index; - /* Memory barrier */ - rmb(); - /* 0->SQ, 1->RQ */ - is_send = !(roce_get_bit(cqe->cqe_byte_4, CQE_BYTE_4_SQ_RQ_FLAG_S)); - - /* Local_qpn in UD cqe is always 1, so it needs to compute new qpn */ - if (roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M, - CQE_BYTE_16_LOCAL_QPN_S) <= 1) { - qpn = roce_get_field(cqe->cqe_byte_20, CQE_BYTE_20_PORT_NUM_M, - CQE_BYTE_20_PORT_NUM_S) + - roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M, - CQE_BYTE_16_LOCAL_QPN_S) * - HNS_ROCE_MAX_PORTS; - } else { - qpn = roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M, - CQE_BYTE_16_LOCAL_QPN_S); - } - - if (!*cur_qp || (qpn & HNS_ROCE_CQE_QPN_MASK) != (*cur_qp)->qpn) { - hr_qp = __hns_roce_qp_lookup(hr_dev, qpn); - if (unlikely(!hr_qp)) { - dev_err(dev, "CQ %06lx with entry for unknown QPN %06x\n", - hr_cq->cqn, (qpn & HNS_ROCE_CQE_QPN_MASK)); - return -EINVAL; - } - - *cur_qp = hr_qp; - } - - wc->qp = &(*cur_qp)->ibqp; - wc->vendor_err = 0; - - status = roce_get_field(cqe->cqe_byte_4, - CQE_BYTE_4_STATUS_OF_THE_OPERATION_M, - CQE_BYTE_4_STATUS_OF_THE_OPERATION_S) & - HNS_ROCE_CQE_STATUS_MASK; - switch (status) { - case HNS_ROCE_CQE_SUCCESS: - wc->status = IB_WC_SUCCESS; - break; - case HNS_ROCE_CQE_SYNDROME_LOCAL_LENGTH_ERR: - wc->status = IB_WC_LOC_LEN_ERR; - break; - case HNS_ROCE_CQE_SYNDROME_LOCAL_QP_OP_ERR: - wc->status = IB_WC_LOC_QP_OP_ERR; - break; - case HNS_ROCE_CQE_SYNDROME_LOCAL_PROT_ERR: - wc->status = IB_WC_LOC_PROT_ERR; - break; - case HNS_ROCE_CQE_SYNDROME_WR_FLUSH_ERR: - wc->status = IB_WC_WR_FLUSH_ERR; - break; - case HNS_ROCE_CQE_SYNDROME_MEM_MANAGE_OPERATE_ERR: - wc->status = IB_WC_MW_BIND_ERR; - break; - case HNS_ROCE_CQE_SYNDROME_BAD_RESP_ERR: - wc->status = IB_WC_BAD_RESP_ERR; - break; - case HNS_ROCE_CQE_SYNDROME_LOCAL_ACCESS_ERR: - wc->status = IB_WC_LOC_ACCESS_ERR; - break; - case HNS_ROCE_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR: - wc->status = IB_WC_REM_INV_REQ_ERR; - break; - case HNS_ROCE_CQE_SYNDROME_REMOTE_ACCESS_ERR: - wc->status = IB_WC_REM_ACCESS_ERR; - break; - case HNS_ROCE_CQE_SYNDROME_REMOTE_OP_ERR: - wc->status = IB_WC_REM_OP_ERR; - break; - case HNS_ROCE_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR: - wc->status = IB_WC_RETRY_EXC_ERR; - break; - case HNS_ROCE_CQE_SYNDROME_RNR_RETRY_EXC_ERR: - wc->status = IB_WC_RNR_RETRY_EXC_ERR; - break; - default: - wc->status = IB_WC_GENERAL_ERR; - break; - } - - /* CQE status error, directly return */ - if (wc->status != IB_WC_SUCCESS) - return 0; - - if (is_send) { - /* SQ conrespond to CQE */ - sq_wqe = hns_roce_get_send_wqe(*cur_qp, - roce_get_field(cqe->cqe_byte_4, - CQE_BYTE_4_WQE_INDEX_M, - CQE_BYTE_4_WQE_INDEX_S) & - ((*cur_qp)->sq.wqe_cnt-1)); - switch (le32_to_cpu(sq_wqe->flag) & HNS_ROCE_WQE_OPCODE_MASK) { - case HNS_ROCE_WQE_OPCODE_SEND: - wc->opcode = IB_WC_SEND; - break; - case HNS_ROCE_WQE_OPCODE_RDMA_READ: - wc->opcode = IB_WC_RDMA_READ; - wc->byte_len = le32_to_cpu(cqe->byte_cnt); - break; - case HNS_ROCE_WQE_OPCODE_RDMA_WRITE: - wc->opcode = IB_WC_RDMA_WRITE; - break; - case HNS_ROCE_WQE_OPCODE_LOCAL_INV: - wc->opcode = IB_WC_LOCAL_INV; - break; - case HNS_ROCE_WQE_OPCODE_UD_SEND: - wc->opcode = IB_WC_SEND; - break; - default: - wc->status = IB_WC_GENERAL_ERR; - break; - } - wc->wc_flags = (le32_to_cpu(sq_wqe->flag) & HNS_ROCE_WQE_IMM ? - IB_WC_WITH_IMM : 0); - - wq = &(*cur_qp)->sq; - if ((*cur_qp)->sq_signal_bits) { - /* - * If sg_signal_bit is 1, - * firstly tail pointer updated to wqe - * which current cqe correspond to - */ - wqe_ctr = (u16)roce_get_field(cqe->cqe_byte_4, - CQE_BYTE_4_WQE_INDEX_M, - CQE_BYTE_4_WQE_INDEX_S); - wq->tail += (wqe_ctr - (u16)wq->tail) & - (wq->wqe_cnt - 1); - } - wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; - ++wq->tail; - } else { - /* RQ conrespond to CQE */ - wc->byte_len = le32_to_cpu(cqe->byte_cnt); - opcode = roce_get_field(cqe->cqe_byte_4, - CQE_BYTE_4_OPERATION_TYPE_M, - CQE_BYTE_4_OPERATION_TYPE_S) & - HNS_ROCE_CQE_OPCODE_MASK; - switch (opcode) { - case HNS_ROCE_OPCODE_RDMA_WITH_IMM_RECEIVE: - wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; - wc->wc_flags = IB_WC_WITH_IMM; - wc->ex.imm_data = - cpu_to_be32(le32_to_cpu(cqe->immediate_data)); - break; - case HNS_ROCE_OPCODE_SEND_DATA_RECEIVE: - if (roce_get_bit(cqe->cqe_byte_4, - CQE_BYTE_4_IMM_INDICATOR_S)) { - wc->opcode = IB_WC_RECV; - wc->wc_flags = IB_WC_WITH_IMM; - wc->ex.imm_data = cpu_to_be32( - le32_to_cpu(cqe->immediate_data)); - } else { - wc->opcode = IB_WC_RECV; - wc->wc_flags = 0; - } - break; - default: - wc->status = IB_WC_GENERAL_ERR; - break; - } - - /* Update tail pointer, record wr_id */ - wq = &(*cur_qp)->rq; - wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; - ++wq->tail; - wc->sl = (u8)roce_get_field(cqe->cqe_byte_20, CQE_BYTE_20_SL_M, - CQE_BYTE_20_SL_S); - wc->src_qp = (u8)roce_get_field(cqe->cqe_byte_20, - CQE_BYTE_20_REMOTE_QPN_M, - CQE_BYTE_20_REMOTE_QPN_S); - wc->wc_flags |= (roce_get_bit(cqe->cqe_byte_20, - CQE_BYTE_20_GRH_PRESENT_S) ? - IB_WC_GRH : 0); - wc->pkey_index = (u16)roce_get_field(cqe->cqe_byte_28, - CQE_BYTE_28_P_KEY_IDX_M, - CQE_BYTE_28_P_KEY_IDX_S); - } - - return 0; -} - -int hns_roce_v1_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) -{ - struct hns_roce_cq *hr_cq = to_hr_cq(ibcq); - struct hns_roce_qp *cur_qp = NULL; - unsigned long flags; - int npolled; - int ret; - - spin_lock_irqsave(&hr_cq->lock, flags); - - for (npolled = 0; npolled < num_entries; ++npolled) { - ret = hns_roce_v1_poll_one(hr_cq, &cur_qp, wc + npolled); - if (ret) - break; - } - - if (npolled) { - *hr_cq->tptr_addr = hr_cq->cons_index & - ((hr_cq->cq_depth << 1) - 1); - - hns_roce_v1_cq_set_ci(hr_cq, hr_cq->cons_index); - } - - spin_unlock_irqrestore(&hr_cq->lock, flags); - - if (ret == 0 || ret == -EAGAIN) - return npolled; - else - return ret; -} - -static int hns_roce_v1_clear_hem(struct hns_roce_dev *hr_dev, - struct hns_roce_hem_table *table, int obj, - int step_idx) -{ - struct hns_roce_v1_priv *priv = hr_dev->priv; - struct device *dev = &hr_dev->pdev->dev; - long end = HW_SYNC_TIMEOUT_MSECS; - __le32 bt_cmd_val[2] = {0}; - unsigned long flags = 0; - void __iomem *bt_cmd; - u64 bt_ba = 0; - - switch (table->type) { - case HEM_TYPE_QPC: - bt_ba = priv->bt_table.qpc_buf.map >> 12; - break; - case HEM_TYPE_MTPT: - bt_ba = priv->bt_table.mtpt_buf.map >> 12; - break; - case HEM_TYPE_CQC: - bt_ba = priv->bt_table.cqc_buf.map >> 12; - break; - case HEM_TYPE_SRQC: - dev_dbg(dev, "HEM_TYPE_SRQC not support.\n"); - return -EINVAL; - default: - return 0; - } - roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M, - ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, table->type); - roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M, - ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj); - roce_set_bit(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0); - roce_set_bit(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1); - - spin_lock_irqsave(&hr_dev->bt_cmd_lock, flags); - - bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG; - - while (1) { - if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) { - if (!end) { - dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n"); - spin_unlock_irqrestore(&hr_dev->bt_cmd_lock, - flags); - return -EBUSY; - } - } else { - break; - } - mdelay(HW_SYNC_SLEEP_TIME_INTERVAL); - end -= HW_SYNC_SLEEP_TIME_INTERVAL; - } - - bt_cmd_val[0] = cpu_to_le32(bt_ba); - roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M, - ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S, bt_ba >> 32); - hns_roce_write64_k(bt_cmd_val, hr_dev->reg_base + ROCEE_BT_CMD_L_REG); - - spin_unlock_irqrestore(&hr_dev->bt_cmd_lock, flags); - - return 0; -} - -static int hns_roce_v1_qp_modify(struct hns_roce_dev *hr_dev, - enum hns_roce_qp_state cur_state, - enum hns_roce_qp_state new_state, - struct hns_roce_qp_context *context, - struct hns_roce_qp *hr_qp) -{ - static const u16 - op[HNS_ROCE_QP_NUM_STATE][HNS_ROCE_QP_NUM_STATE] = { - [HNS_ROCE_QP_STATE_RST] = { - [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP, - [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP, - [HNS_ROCE_QP_STATE_INIT] = HNS_ROCE_CMD_RST2INIT_QP, - }, - [HNS_ROCE_QP_STATE_INIT] = { - [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP, - [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP, - /* Note: In v1 engine, HW doesn't support RST2INIT. - * We use RST2INIT cmd instead of INIT2INIT. - */ - [HNS_ROCE_QP_STATE_INIT] = HNS_ROCE_CMD_RST2INIT_QP, - [HNS_ROCE_QP_STATE_RTR] = HNS_ROCE_CMD_INIT2RTR_QP, - }, - [HNS_ROCE_QP_STATE_RTR] = { - [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP, - [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP, - [HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_RTR2RTS_QP, - }, - [HNS_ROCE_QP_STATE_RTS] = { - [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP, - [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP, - [HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_RTS2RTS_QP, - [HNS_ROCE_QP_STATE_SQD] = HNS_ROCE_CMD_RTS2SQD_QP, - }, - [HNS_ROCE_QP_STATE_SQD] = { - [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP, - [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP, - [HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_SQD2RTS_QP, - [HNS_ROCE_QP_STATE_SQD] = HNS_ROCE_CMD_SQD2SQD_QP, - }, - [HNS_ROCE_QP_STATE_ERR] = { - [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP, - [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP, - } - }; - - struct hns_roce_cmd_mailbox *mailbox; - struct device *dev = &hr_dev->pdev->dev; - int ret; - - if (cur_state >= HNS_ROCE_QP_NUM_STATE || - new_state >= HNS_ROCE_QP_NUM_STATE || - !op[cur_state][new_state]) { - dev_err(dev, "[modify_qp]not support state %d to %d\n", - cur_state, new_state); - return -EINVAL; - } - - if (op[cur_state][new_state] == HNS_ROCE_CMD_2RST_QP) - return hns_roce_cmd_mbox(hr_dev, 0, 0, hr_qp->qpn, 2, - HNS_ROCE_CMD_2RST_QP, - HNS_ROCE_CMD_TIMEOUT_MSECS); - - if (op[cur_state][new_state] == HNS_ROCE_CMD_2ERR_QP) - return hns_roce_cmd_mbox(hr_dev, 0, 0, hr_qp->qpn, 2, - HNS_ROCE_CMD_2ERR_QP, - HNS_ROCE_CMD_TIMEOUT_MSECS); - - mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - - memcpy(mailbox->buf, context, sizeof(*context)); - - ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0, - op[cur_state][new_state], - HNS_ROCE_CMD_TIMEOUT_MSECS); - - hns_roce_free_cmd_mailbox(hr_dev, mailbox); - return ret; -} - -static int find_wqe_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, - u64 *sq_ba, u64 *rq_ba, dma_addr_t *bt_ba) -{ - struct ib_device *ibdev = &hr_dev->ib_dev; - int count; - - count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 0, sq_ba, 1, bt_ba); - if (count < 1) { - ibdev_err(ibdev, "Failed to find SQ ba\n"); - return -ENOBUFS; - } - - count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_qp->rq.offset, rq_ba, - 1, NULL); - if (!count) { - ibdev_err(ibdev, "Failed to find RQ ba\n"); - return -ENOBUFS; - } - - return 0; -} - -static int hns_roce_v1_m_sqp(struct ib_qp *ibqp, const struct ib_qp_attr *attr, - int attr_mask, enum ib_qp_state cur_state, - enum ib_qp_state new_state) -{ - struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); - struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); - struct hns_roce_sqp_context *context; - dma_addr_t dma_handle = 0; - u32 __iomem *addr; - u64 sq_ba = 0; - u64 rq_ba = 0; - __le32 tmp; - u32 reg_val; - - context = kzalloc(sizeof(*context), GFP_KERNEL); - if (!context) - return -ENOMEM; - - /* Search QP buf's MTTs */ - if (find_wqe_mtt(hr_dev, hr_qp, &sq_ba, &rq_ba, &dma_handle)) - goto out; - - if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { - roce_set_field(context->qp1c_bytes_4, - QP1C_BYTES_4_SQ_WQE_SHIFT_M, - QP1C_BYTES_4_SQ_WQE_SHIFT_S, - ilog2((unsigned int)hr_qp->sq.wqe_cnt)); - roce_set_field(context->qp1c_bytes_4, - QP1C_BYTES_4_RQ_WQE_SHIFT_M, - QP1C_BYTES_4_RQ_WQE_SHIFT_S, - ilog2((unsigned int)hr_qp->rq.wqe_cnt)); - roce_set_field(context->qp1c_bytes_4, QP1C_BYTES_4_PD_M, - QP1C_BYTES_4_PD_S, to_hr_pd(ibqp->pd)->pdn); - - context->sq_rq_bt_l = cpu_to_le32(dma_handle); - roce_set_field(context->qp1c_bytes_12, - QP1C_BYTES_12_SQ_RQ_BT_H_M, - QP1C_BYTES_12_SQ_RQ_BT_H_S, - upper_32_bits(dma_handle)); - - roce_set_field(context->qp1c_bytes_16, QP1C_BYTES_16_RQ_HEAD_M, - QP1C_BYTES_16_RQ_HEAD_S, hr_qp->rq.head); - roce_set_field(context->qp1c_bytes_16, QP1C_BYTES_16_PORT_NUM_M, - QP1C_BYTES_16_PORT_NUM_S, hr_qp->phy_port); - roce_set_bit(context->qp1c_bytes_16, - QP1C_BYTES_16_SIGNALING_TYPE_S, - hr_qp->sq_signal_bits); - roce_set_bit(context->qp1c_bytes_16, QP1C_BYTES_16_RQ_BA_FLG_S, - 1); - roce_set_bit(context->qp1c_bytes_16, QP1C_BYTES_16_SQ_BA_FLG_S, - 1); - roce_set_bit(context->qp1c_bytes_16, QP1C_BYTES_16_QP1_ERR_S, - 0); - - roce_set_field(context->qp1c_bytes_20, QP1C_BYTES_20_SQ_HEAD_M, - QP1C_BYTES_20_SQ_HEAD_S, hr_qp->sq.head); - roce_set_field(context->qp1c_bytes_20, QP1C_BYTES_20_PKEY_IDX_M, - QP1C_BYTES_20_PKEY_IDX_S, attr->pkey_index); - - context->cur_rq_wqe_ba_l = cpu_to_le32(rq_ba); - - roce_set_field(context->qp1c_bytes_28, - QP1C_BYTES_28_CUR_RQ_WQE_BA_H_M, - QP1C_BYTES_28_CUR_RQ_WQE_BA_H_S, - upper_32_bits(rq_ba)); - roce_set_field(context->qp1c_bytes_28, - QP1C_BYTES_28_RQ_CUR_IDX_M, - QP1C_BYTES_28_RQ_CUR_IDX_S, 0); - - roce_set_field(context->qp1c_bytes_32, - QP1C_BYTES_32_RX_CQ_NUM_M, - QP1C_BYTES_32_RX_CQ_NUM_S, - to_hr_cq(ibqp->recv_cq)->cqn); - roce_set_field(context->qp1c_bytes_32, - QP1C_BYTES_32_TX_CQ_NUM_M, - QP1C_BYTES_32_TX_CQ_NUM_S, - to_hr_cq(ibqp->send_cq)->cqn); - - context->cur_sq_wqe_ba_l = cpu_to_le32(sq_ba); - - roce_set_field(context->qp1c_bytes_40, - QP1C_BYTES_40_CUR_SQ_WQE_BA_H_M, - QP1C_BYTES_40_CUR_SQ_WQE_BA_H_S, - upper_32_bits(sq_ba)); - roce_set_field(context->qp1c_bytes_40, - QP1C_BYTES_40_SQ_CUR_IDX_M, - QP1C_BYTES_40_SQ_CUR_IDX_S, 0); - - /* Copy context to QP1C register */ - addr = (u32 __iomem *)(hr_dev->reg_base + - ROCEE_QP1C_CFG0_0_REG + - hr_qp->phy_port * sizeof(*context)); - - writel(le32_to_cpu(context->qp1c_bytes_4), addr); - writel(le32_to_cpu(context->sq_rq_bt_l), addr + 1); - writel(le32_to_cpu(context->qp1c_bytes_12), addr + 2); - writel(le32_to_cpu(context->qp1c_bytes_16), addr + 3); - writel(le32_to_cpu(context->qp1c_bytes_20), addr + 4); - writel(le32_to_cpu(context->cur_rq_wqe_ba_l), addr + 5); - writel(le32_to_cpu(context->qp1c_bytes_28), addr + 6); - writel(le32_to_cpu(context->qp1c_bytes_32), addr + 7); - writel(le32_to_cpu(context->cur_sq_wqe_ba_l), addr + 8); - writel(le32_to_cpu(context->qp1c_bytes_40), addr + 9); - } - - /* Modify QP1C status */ - reg_val = roce_read(hr_dev, ROCEE_QP1C_CFG0_0_REG + - hr_qp->phy_port * sizeof(*context)); - tmp = cpu_to_le32(reg_val); - roce_set_field(tmp, ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_M, - ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_S, new_state); - reg_val = le32_to_cpu(tmp); - roce_write(hr_dev, ROCEE_QP1C_CFG0_0_REG + - hr_qp->phy_port * sizeof(*context), reg_val); - - hr_qp->state = new_state; - if (new_state == IB_QPS_RESET) { - hns_roce_v1_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn, - ibqp->srq ? to_hr_srq(ibqp->srq) : NULL); - if (ibqp->send_cq != ibqp->recv_cq) - hns_roce_v1_cq_clean(to_hr_cq(ibqp->send_cq), - hr_qp->qpn, NULL); - - hr_qp->rq.head = 0; - hr_qp->rq.tail = 0; - hr_qp->sq.head = 0; - hr_qp->sq.tail = 0; - } - - kfree(context); - return 0; - -out: - kfree(context); - return -EINVAL; -} - -static bool check_qp_state(enum ib_qp_state cur_state, - enum ib_qp_state new_state) -{ - static const bool sm[][IB_QPS_ERR + 1] = { - [IB_QPS_RESET] = { [IB_QPS_RESET] = true, - [IB_QPS_INIT] = true }, - [IB_QPS_INIT] = { [IB_QPS_RESET] = true, - [IB_QPS_INIT] = true, - [IB_QPS_RTR] = true, - [IB_QPS_ERR] = true }, - [IB_QPS_RTR] = { [IB_QPS_RESET] = true, - [IB_QPS_RTS] = true, - [IB_QPS_ERR] = true }, - [IB_QPS_RTS] = { [IB_QPS_RESET] = true, [IB_QPS_ERR] = true }, - [IB_QPS_SQD] = {}, - [IB_QPS_SQE] = {}, - [IB_QPS_ERR] = { [IB_QPS_RESET] = true, [IB_QPS_ERR] = true } - }; - - return sm[cur_state][new_state]; -} - -static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr, - int attr_mask, enum ib_qp_state cur_state, - enum ib_qp_state new_state) -{ - struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); - struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); - struct device *dev = &hr_dev->pdev->dev; - struct hns_roce_qp_context *context; - const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); - dma_addr_t dma_handle_2 = 0; - dma_addr_t dma_handle = 0; - __le32 doorbell[2] = {0}; - u64 *mtts_2 = NULL; - int ret = -EINVAL; - const u8 *smac; - u64 sq_ba = 0; - u64 rq_ba = 0; - u32 port; - u32 port_num; - u8 *dmac; - - if (!check_qp_state(cur_state, new_state)) { - ibdev_err(ibqp->device, - "not support QP(%u) status from %d to %d\n", - ibqp->qp_num, cur_state, new_state); - return -EINVAL; - } - - context = kzalloc(sizeof(*context), GFP_KERNEL); - if (!context) - return -ENOMEM; - - /* Search qp buf's mtts */ - if (find_wqe_mtt(hr_dev, hr_qp, &sq_ba, &rq_ba, &dma_handle)) - goto out; - - /* Search IRRL's mtts */ - mtts_2 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table, - hr_qp->qpn, &dma_handle_2); - if (mtts_2 == NULL) { - dev_err(dev, "qp irrl_table find failed\n"); - goto out; - } - - /* - * Reset to init - * Mandatory param: - * IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS - * Optional param: NA - */ - if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { - roce_set_field(context->qpc_bytes_4, - QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M, - QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S, - to_hr_qp_type(hr_qp->ibqp.qp_type)); - - roce_set_bit(context->qpc_bytes_4, - QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S, 0); - roce_set_bit(context->qpc_bytes_4, - QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S, - !!(attr->qp_access_flags & IB_ACCESS_REMOTE_READ)); - roce_set_bit(context->qpc_bytes_4, - QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S, - !!(attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) - ); - roce_set_bit(context->qpc_bytes_4, - QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S, - !!(attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC) - ); - roce_set_bit(context->qpc_bytes_4, - QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S, 1); - roce_set_field(context->qpc_bytes_4, - QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M, - QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S, - ilog2((unsigned int)hr_qp->sq.wqe_cnt)); - roce_set_field(context->qpc_bytes_4, - QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M, - QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S, - ilog2((unsigned int)hr_qp->rq.wqe_cnt)); - roce_set_field(context->qpc_bytes_4, - QP_CONTEXT_QPC_BYTES_4_PD_M, - QP_CONTEXT_QPC_BYTES_4_PD_S, - to_hr_pd(ibqp->pd)->pdn); - hr_qp->access_flags = attr->qp_access_flags; - roce_set_field(context->qpc_bytes_8, - QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M, - QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S, - to_hr_cq(ibqp->send_cq)->cqn); - roce_set_field(context->qpc_bytes_8, - QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M, - QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S, - to_hr_cq(ibqp->recv_cq)->cqn); - - if (ibqp->srq) - roce_set_field(context->qpc_bytes_12, - QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M, - QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S, - to_hr_srq(ibqp->srq)->srqn); - - roce_set_field(context->qpc_bytes_12, - QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M, - QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S, - attr->pkey_index); - hr_qp->pkey_index = attr->pkey_index; - roce_set_field(context->qpc_bytes_16, - QP_CONTEXT_QPC_BYTES_16_QP_NUM_M, - QP_CONTEXT_QPC_BYTES_16_QP_NUM_S, hr_qp->qpn); - } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) { - roce_set_field(context->qpc_bytes_4, - QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M, - QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S, - to_hr_qp_type(hr_qp->ibqp.qp_type)); - roce_set_bit(context->qpc_bytes_4, - QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S, 0); - if (attr_mask & IB_QP_ACCESS_FLAGS) { - roce_set_bit(context->qpc_bytes_4, - QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S, - !!(attr->qp_access_flags & - IB_ACCESS_REMOTE_READ)); - roce_set_bit(context->qpc_bytes_4, - QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S, - !!(attr->qp_access_flags & - IB_ACCESS_REMOTE_WRITE)); - } else { - roce_set_bit(context->qpc_bytes_4, - QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S, - !!(hr_qp->access_flags & - IB_ACCESS_REMOTE_READ)); - roce_set_bit(context->qpc_bytes_4, - QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S, - !!(hr_qp->access_flags & - IB_ACCESS_REMOTE_WRITE)); - } - - roce_set_bit(context->qpc_bytes_4, - QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S, 1); - roce_set_field(context->qpc_bytes_4, - QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M, - QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S, - ilog2((unsigned int)hr_qp->sq.wqe_cnt)); - roce_set_field(context->qpc_bytes_4, - QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M, - QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S, - ilog2((unsigned int)hr_qp->rq.wqe_cnt)); - roce_set_field(context->qpc_bytes_4, - QP_CONTEXT_QPC_BYTES_4_PD_M, - QP_CONTEXT_QPC_BYTES_4_PD_S, - to_hr_pd(ibqp->pd)->pdn); - - roce_set_field(context->qpc_bytes_8, - QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M, - QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S, - to_hr_cq(ibqp->send_cq)->cqn); - roce_set_field(context->qpc_bytes_8, - QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M, - QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S, - to_hr_cq(ibqp->recv_cq)->cqn); - - if (ibqp->srq) - roce_set_field(context->qpc_bytes_12, - QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M, - QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S, - to_hr_srq(ibqp->srq)->srqn); - if (attr_mask & IB_QP_PKEY_INDEX) - roce_set_field(context->qpc_bytes_12, - QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M, - QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S, - attr->pkey_index); - else - roce_set_field(context->qpc_bytes_12, - QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M, - QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S, - hr_qp->pkey_index); - - roce_set_field(context->qpc_bytes_16, - QP_CONTEXT_QPC_BYTES_16_QP_NUM_M, - QP_CONTEXT_QPC_BYTES_16_QP_NUM_S, hr_qp->qpn); - } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) { - if ((attr_mask & IB_QP_ALT_PATH) || - (attr_mask & IB_QP_ACCESS_FLAGS) || - (attr_mask & IB_QP_PKEY_INDEX) || - (attr_mask & IB_QP_QKEY)) { - dev_err(dev, "INIT2RTR attr_mask error\n"); - goto out; - } - - dmac = (u8 *)attr->ah_attr.roce.dmac; - - context->sq_rq_bt_l = cpu_to_le32(dma_handle); - roce_set_field(context->qpc_bytes_24, - QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_M, - QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_S, - upper_32_bits(dma_handle)); - roce_set_bit(context->qpc_bytes_24, - QP_CONTEXT_QPC_BYTE_24_REMOTE_ENABLE_E2E_CREDITS_S, - 1); - roce_set_field(context->qpc_bytes_24, - QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M, - QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S, - attr->min_rnr_timer); - context->irrl_ba_l = cpu_to_le32((u32)(dma_handle_2)); - roce_set_field(context->qpc_bytes_32, - QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M, - QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_S, - ((u32)(dma_handle_2 >> 32)) & - QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M); - roce_set_field(context->qpc_bytes_32, - QP_CONTEXT_QPC_BYTES_32_MIG_STATE_M, - QP_CONTEXT_QPC_BYTES_32_MIG_STATE_S, 0); - roce_set_bit(context->qpc_bytes_32, - QP_CONTEXT_QPC_BYTE_32_LOCAL_ENABLE_E2E_CREDITS_S, - 1); - roce_set_bit(context->qpc_bytes_32, - QP_CONTEXT_QPC_BYTE_32_SIGNALING_TYPE_S, - hr_qp->sq_signal_bits); - - port = (attr_mask & IB_QP_PORT) ? (attr->port_num - 1) : - hr_qp->port; - smac = (const u8 *)hr_dev->dev_addr[port]; - /* when dmac equals smac or loop_idc is 1, it should loopback */ - if (ether_addr_equal_unaligned(dmac, smac) || - hr_dev->loop_idc == 0x1) - roce_set_bit(context->qpc_bytes_32, - QP_CONTEXT_QPC_BYTE_32_LOOPBACK_INDICATOR_S, 1); - - roce_set_bit(context->qpc_bytes_32, - QP_CONTEXT_QPC_BYTE_32_GLOBAL_HEADER_S, - rdma_ah_get_ah_flags(&attr->ah_attr)); - roce_set_field(context->qpc_bytes_32, - QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M, - QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S, - ilog2((unsigned int)attr->max_dest_rd_atomic)); - - if (attr_mask & IB_QP_DEST_QPN) - roce_set_field(context->qpc_bytes_36, - QP_CONTEXT_QPC_BYTES_36_DEST_QP_M, - QP_CONTEXT_QPC_BYTES_36_DEST_QP_S, - attr->dest_qp_num); - - /* Configure GID index */ - port_num = rdma_ah_get_port_num(&attr->ah_attr); - roce_set_field(context->qpc_bytes_36, - QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M, - QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S, - hns_get_gid_index(hr_dev, - port_num - 1, - grh->sgid_index)); - - memcpy(&(context->dmac_l), dmac, 4); - - roce_set_field(context->qpc_bytes_44, - QP_CONTEXT_QPC_BYTES_44_DMAC_H_M, - QP_CONTEXT_QPC_BYTES_44_DMAC_H_S, - *((u16 *)(&dmac[4]))); - roce_set_field(context->qpc_bytes_44, - QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_M, - QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_S, - rdma_ah_get_static_rate(&attr->ah_attr)); - roce_set_field(context->qpc_bytes_44, - QP_CONTEXT_QPC_BYTES_44_HOPLMT_M, - QP_CONTEXT_QPC_BYTES_44_HOPLMT_S, - grh->hop_limit); - - roce_set_field(context->qpc_bytes_48, - QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M, - QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S, - grh->flow_label); - roce_set_field(context->qpc_bytes_48, - QP_CONTEXT_QPC_BYTES_48_TCLASS_M, - QP_CONTEXT_QPC_BYTES_48_TCLASS_S, - grh->traffic_class); - roce_set_field(context->qpc_bytes_48, - QP_CONTEXT_QPC_BYTES_48_MTU_M, - QP_CONTEXT_QPC_BYTES_48_MTU_S, attr->path_mtu); - - memcpy(context->dgid, grh->dgid.raw, - sizeof(grh->dgid.raw)); - - dev_dbg(dev, "dmac:%x :%lx\n", context->dmac_l, - roce_get_field(context->qpc_bytes_44, - QP_CONTEXT_QPC_BYTES_44_DMAC_H_M, - QP_CONTEXT_QPC_BYTES_44_DMAC_H_S)); - - roce_set_field(context->qpc_bytes_68, - QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_M, - QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_S, - hr_qp->rq.head); - roce_set_field(context->qpc_bytes_68, - QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_M, - QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_S, 0); - - context->cur_rq_wqe_ba_l = cpu_to_le32(rq_ba); - - roce_set_field(context->qpc_bytes_76, - QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_M, - QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_S, - upper_32_bits(rq_ba)); - roce_set_field(context->qpc_bytes_76, - QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_M, - QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_S, 0); - - context->rx_rnr_time = 0; - - roce_set_field(context->qpc_bytes_84, - QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_M, - QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_S, - attr->rq_psn - 1); - roce_set_field(context->qpc_bytes_84, - QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_M, - QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_S, 0); - - roce_set_field(context->qpc_bytes_88, - QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M, - QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S, - attr->rq_psn); - roce_set_bit(context->qpc_bytes_88, - QP_CONTEXT_QPC_BYTES_88_RX_REQ_PSN_ERR_FLAG_S, 0); - roce_set_bit(context->qpc_bytes_88, - QP_CONTEXT_QPC_BYTES_88_RX_LAST_OPCODE_FLG_S, 0); - roce_set_field(context->qpc_bytes_88, - QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_M, - QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_S, - 0); - roce_set_field(context->qpc_bytes_88, - QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_M, - QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_S, - 0); - - context->dma_length = 0; - context->r_key = 0; - context->va_l = 0; - context->va_h = 0; - - roce_set_field(context->qpc_bytes_108, - QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_M, - QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_S, 0); - roce_set_bit(context->qpc_bytes_108, - QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_FLG_S, 0); - roce_set_bit(context->qpc_bytes_108, - QP_CONTEXT_QPC_BYTES_108_TRRL_TDB_PSN_FLG_S, 0); - - roce_set_field(context->qpc_bytes_112, - QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_M, - QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_S, 0); - roce_set_field(context->qpc_bytes_112, - QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_M, - QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_S, 0); - - /* For chip resp ack */ - roce_set_field(context->qpc_bytes_156, - QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M, - QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S, - hr_qp->phy_port); - roce_set_field(context->qpc_bytes_156, - QP_CONTEXT_QPC_BYTES_156_SL_M, - QP_CONTEXT_QPC_BYTES_156_SL_S, - rdma_ah_get_sl(&attr->ah_attr)); - hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr); - } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) { - /* If exist optional param, return error */ - if ((attr_mask & IB_QP_ALT_PATH) || - (attr_mask & IB_QP_ACCESS_FLAGS) || - (attr_mask & IB_QP_QKEY) || - (attr_mask & IB_QP_PATH_MIG_STATE) || - (attr_mask & IB_QP_CUR_STATE) || - (attr_mask & IB_QP_MIN_RNR_TIMER)) { - dev_err(dev, "RTR2RTS attr_mask error\n"); - goto out; - } - - context->rx_cur_sq_wqe_ba_l = cpu_to_le32(sq_ba); - - roce_set_field(context->qpc_bytes_120, - QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_M, - QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_S, - upper_32_bits(sq_ba)); - - roce_set_field(context->qpc_bytes_124, - QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_M, - QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_S, 0); - roce_set_field(context->qpc_bytes_124, - QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_M, - QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_S, 0); - - roce_set_field(context->qpc_bytes_128, - QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_M, - QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_S, - attr->sq_psn); - roce_set_bit(context->qpc_bytes_128, - QP_CONTEXT_QPC_BYTES_128_RX_ACK_PSN_ERR_FLG_S, 0); - roce_set_field(context->qpc_bytes_128, - QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_M, - QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_S, - 0); - roce_set_bit(context->qpc_bytes_128, - QP_CONTEXT_QPC_BYTES_128_IRRL_PSN_VLD_FLG_S, 0); - - roce_set_field(context->qpc_bytes_132, - QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_M, - QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_S, 0); - roce_set_field(context->qpc_bytes_132, - QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_M, - QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_S, 0); - - roce_set_field(context->qpc_bytes_136, - QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_M, - QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_S, - attr->sq_psn); - roce_set_field(context->qpc_bytes_136, - QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_M, - QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_S, - attr->sq_psn); - - roce_set_field(context->qpc_bytes_140, - QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_M, - QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_S, - (attr->sq_psn >> SQ_PSN_SHIFT)); - roce_set_field(context->qpc_bytes_140, - QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_M, - QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_S, 0); - roce_set_bit(context->qpc_bytes_140, - QP_CONTEXT_QPC_BYTES_140_RNR_RETRY_FLG_S, 0); - - roce_set_field(context->qpc_bytes_148, - QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_M, - QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_S, 0); - roce_set_field(context->qpc_bytes_148, - QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M, - QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S, - attr->retry_cnt); - roce_set_field(context->qpc_bytes_148, - QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_M, - QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_S, - attr->rnr_retry); - roce_set_field(context->qpc_bytes_148, - QP_CONTEXT_QPC_BYTES_148_LSN_M, - QP_CONTEXT_QPC_BYTES_148_LSN_S, 0x100); - - context->rnr_retry = 0; - - roce_set_field(context->qpc_bytes_156, - QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_M, - QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_S, - attr->retry_cnt); - if (attr->timeout < 0x12) { - dev_info(dev, "ack timeout value(0x%x) must bigger than 0x12.\n", - attr->timeout); - roce_set_field(context->qpc_bytes_156, - QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M, - QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S, - 0x12); - } else { - roce_set_field(context->qpc_bytes_156, - QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M, - QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S, - attr->timeout); - } - roce_set_field(context->qpc_bytes_156, - QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_M, - QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_S, - attr->rnr_retry); - roce_set_field(context->qpc_bytes_156, - QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M, - QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S, - hr_qp->phy_port); - roce_set_field(context->qpc_bytes_156, - QP_CONTEXT_QPC_BYTES_156_SL_M, - QP_CONTEXT_QPC_BYTES_156_SL_S, - rdma_ah_get_sl(&attr->ah_attr)); - hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr); - roce_set_field(context->qpc_bytes_156, - QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M, - QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S, - ilog2((unsigned int)attr->max_rd_atomic)); - roce_set_field(context->qpc_bytes_156, - QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_M, - QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_S, 0); - context->pkt_use_len = 0; - - roce_set_field(context->qpc_bytes_164, - QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M, - QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S, attr->sq_psn); - roce_set_field(context->qpc_bytes_164, - QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_M, - QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_S, 0); - - roce_set_field(context->qpc_bytes_168, - QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_M, - QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_S, - attr->sq_psn); - roce_set_field(context->qpc_bytes_168, - QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_M, - QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_S, 0); - roce_set_field(context->qpc_bytes_168, - QP_CONTEXT_QPC_BYTES_168_DB_TYPE_M, - QP_CONTEXT_QPC_BYTES_168_DB_TYPE_S, 0); - roce_set_bit(context->qpc_bytes_168, - QP_CONTEXT_QPC_BYTES_168_MSG_LP_IND_S, 0); - roce_set_bit(context->qpc_bytes_168, - QP_CONTEXT_QPC_BYTES_168_CSDB_LP_IND_S, 0); - roce_set_bit(context->qpc_bytes_168, - QP_CONTEXT_QPC_BYTES_168_QP_ERR_FLG_S, 0); - context->sge_use_len = 0; - - roce_set_field(context->qpc_bytes_176, - QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_M, - QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_S, 0); - roce_set_field(context->qpc_bytes_176, - QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_M, - QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_S, - 0); - roce_set_field(context->qpc_bytes_180, - QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_M, - QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_S, 0); - roce_set_field(context->qpc_bytes_180, - QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_M, - QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_S, 0); - - context->tx_cur_sq_wqe_ba_l = cpu_to_le32(sq_ba); - - roce_set_field(context->qpc_bytes_188, - QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_M, - QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_S, - upper_32_bits(sq_ba)); - roce_set_bit(context->qpc_bytes_188, - QP_CONTEXT_QPC_BYTES_188_PKT_RETRY_FLG_S, 0); - roce_set_field(context->qpc_bytes_188, - QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_M, - QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_S, - 0); - } - - /* Every status migrate must change state */ - roce_set_field(context->qpc_bytes_144, - QP_CONTEXT_QPC_BYTES_144_QP_STATE_M, - QP_CONTEXT_QPC_BYTES_144_QP_STATE_S, new_state); - - /* SW pass context to HW */ - ret = hns_roce_v1_qp_modify(hr_dev, to_hns_roce_state(cur_state), - to_hns_roce_state(new_state), context, - hr_qp); - if (ret) { - dev_err(dev, "hns_roce_qp_modify failed\n"); - goto out; - } - - /* - * Use rst2init to instead of init2init with drv, - * need to hw to flash RQ HEAD by DB again - */ - if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) { - roce_set_field(doorbell[0], RQ_DOORBELL_U32_4_RQ_HEAD_M, - RQ_DOORBELL_U32_4_RQ_HEAD_S, hr_qp->rq.head); - roce_set_field(doorbell[1], RQ_DOORBELL_U32_8_QPN_M, - RQ_DOORBELL_U32_8_QPN_S, hr_qp->qpn); - roce_set_field(doorbell[1], RQ_DOORBELL_U32_8_CMD_M, - RQ_DOORBELL_U32_8_CMD_S, 1); - roce_set_bit(doorbell[1], RQ_DOORBELL_U32_8_HW_SYNC_S, 1); - - if (ibqp->uobject) { - hr_qp->rq.db_reg = hr_dev->reg_base + - hr_dev->odb_offset + - DB_REG_OFFSET * hr_dev->priv_uar.index; - } - - hns_roce_write64_k(doorbell, hr_qp->rq.db_reg); - } - - hr_qp->state = new_state; - - if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) - hr_qp->resp_depth = attr->max_dest_rd_atomic; - if (attr_mask & IB_QP_PORT) { - hr_qp->port = attr->port_num - 1; - hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port]; - } - - if (new_state == IB_QPS_RESET && !ibqp->uobject) { - hns_roce_v1_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn, - ibqp->srq ? to_hr_srq(ibqp->srq) : NULL); - if (ibqp->send_cq != ibqp->recv_cq) - hns_roce_v1_cq_clean(to_hr_cq(ibqp->send_cq), - hr_qp->qpn, NULL); - - hr_qp->rq.head = 0; - hr_qp->rq.tail = 0; - hr_qp->sq.head = 0; - hr_qp->sq.tail = 0; - } -out: - kfree(context); - return ret; -} - -static int hns_roce_v1_modify_qp(struct ib_qp *ibqp, - const struct ib_qp_attr *attr, int attr_mask, - enum ib_qp_state cur_state, - enum ib_qp_state new_state) -{ - if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) - return -EOPNOTSUPP; - - if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI) - return hns_roce_v1_m_sqp(ibqp, attr, attr_mask, cur_state, - new_state); - else - return hns_roce_v1_m_qp(ibqp, attr, attr_mask, cur_state, - new_state); -} - -static enum ib_qp_state to_ib_qp_state(enum hns_roce_qp_state state) -{ - switch (state) { - case HNS_ROCE_QP_STATE_RST: - return IB_QPS_RESET; - case HNS_ROCE_QP_STATE_INIT: - return IB_QPS_INIT; - case HNS_ROCE_QP_STATE_RTR: - return IB_QPS_RTR; - case HNS_ROCE_QP_STATE_RTS: - return IB_QPS_RTS; - case HNS_ROCE_QP_STATE_SQD: - return IB_QPS_SQD; - case HNS_ROCE_QP_STATE_ERR: - return IB_QPS_ERR; - default: - return IB_QPS_ERR; - } -} - -static int hns_roce_v1_query_qpc(struct hns_roce_dev *hr_dev, - struct hns_roce_qp *hr_qp, - struct hns_roce_qp_context *hr_context) -{ - struct hns_roce_cmd_mailbox *mailbox; - int ret; - - mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - - ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, hr_qp->qpn, 0, - HNS_ROCE_CMD_QUERY_QP, - HNS_ROCE_CMD_TIMEOUT_MSECS); - if (!ret) - memcpy(hr_context, mailbox->buf, sizeof(*hr_context)); - else - dev_err(&hr_dev->pdev->dev, "QUERY QP cmd process error\n"); - - hns_roce_free_cmd_mailbox(hr_dev, mailbox); - - return ret; -} - -static int hns_roce_v1_q_sqp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, - int qp_attr_mask, - struct ib_qp_init_attr *qp_init_attr) -{ - struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); - struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); - struct hns_roce_sqp_context context; - u32 addr; - - mutex_lock(&hr_qp->mutex); - - if (hr_qp->state == IB_QPS_RESET) { - qp_attr->qp_state = IB_QPS_RESET; - goto done; - } - - addr = ROCEE_QP1C_CFG0_0_REG + - hr_qp->port * sizeof(struct hns_roce_sqp_context); - context.qp1c_bytes_4 = cpu_to_le32(roce_read(hr_dev, addr)); - context.sq_rq_bt_l = cpu_to_le32(roce_read(hr_dev, addr + 1)); - context.qp1c_bytes_12 = cpu_to_le32(roce_read(hr_dev, addr + 2)); - context.qp1c_bytes_16 = cpu_to_le32(roce_read(hr_dev, addr + 3)); - context.qp1c_bytes_20 = cpu_to_le32(roce_read(hr_dev, addr + 4)); - context.cur_rq_wqe_ba_l = cpu_to_le32(roce_read(hr_dev, addr + 5)); - context.qp1c_bytes_28 = cpu_to_le32(roce_read(hr_dev, addr + 6)); - context.qp1c_bytes_32 = cpu_to_le32(roce_read(hr_dev, addr + 7)); - context.cur_sq_wqe_ba_l = cpu_to_le32(roce_read(hr_dev, addr + 8)); - context.qp1c_bytes_40 = cpu_to_le32(roce_read(hr_dev, addr + 9)); - - hr_qp->state = roce_get_field(context.qp1c_bytes_4, - QP1C_BYTES_4_QP_STATE_M, - QP1C_BYTES_4_QP_STATE_S); - qp_attr->qp_state = hr_qp->state; - qp_attr->path_mtu = IB_MTU_256; - qp_attr->path_mig_state = IB_MIG_ARMED; - qp_attr->qkey = QKEY_VAL; - qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; - qp_attr->rq_psn = 0; - qp_attr->sq_psn = 0; - qp_attr->dest_qp_num = 1; - qp_attr->qp_access_flags = 6; - - qp_attr->pkey_index = roce_get_field(context.qp1c_bytes_20, - QP1C_BYTES_20_PKEY_IDX_M, - QP1C_BYTES_20_PKEY_IDX_S); - qp_attr->port_num = hr_qp->port + 1; - qp_attr->sq_draining = 0; - qp_attr->max_rd_atomic = 0; - qp_attr->max_dest_rd_atomic = 0; - qp_attr->min_rnr_timer = 0; - qp_attr->timeout = 0; - qp_attr->retry_cnt = 0; - qp_attr->rnr_retry = 0; - qp_attr->alt_timeout = 0; - -done: - qp_attr->cur_qp_state = qp_attr->qp_state; - qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt; - qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs; - qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt; - qp_attr->cap.max_send_sge = hr_qp->sq.max_gs; - qp_attr->cap.max_inline_data = 0; - qp_init_attr->cap = qp_attr->cap; - qp_init_attr->create_flags = 0; - - mutex_unlock(&hr_qp->mutex); - - return 0; -} - -static int hns_roce_v1_q_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, - int qp_attr_mask, - struct ib_qp_init_attr *qp_init_attr) -{ - struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); - struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); - struct device *dev = &hr_dev->pdev->dev; - struct hns_roce_qp_context *context; - int tmp_qp_state; - int ret = 0; - int state; - - context = kzalloc(sizeof(*context), GFP_KERNEL); - if (!context) - return -ENOMEM; - - memset(qp_attr, 0, sizeof(*qp_attr)); - memset(qp_init_attr, 0, sizeof(*qp_init_attr)); - - mutex_lock(&hr_qp->mutex); - - if (hr_qp->state == IB_QPS_RESET) { - qp_attr->qp_state = IB_QPS_RESET; - goto done; - } - - ret = hns_roce_v1_query_qpc(hr_dev, hr_qp, context); - if (ret) { - dev_err(dev, "query qpc error\n"); - ret = -EINVAL; - goto out; - } - - state = roce_get_field(context->qpc_bytes_144, - QP_CONTEXT_QPC_BYTES_144_QP_STATE_M, - QP_CONTEXT_QPC_BYTES_144_QP_STATE_S); - tmp_qp_state = (int)to_ib_qp_state((enum hns_roce_qp_state)state); - if (tmp_qp_state == -1) { - dev_err(dev, "to_ib_qp_state error\n"); - ret = -EINVAL; - goto out; - } - hr_qp->state = (u8)tmp_qp_state; - qp_attr->qp_state = (enum ib_qp_state)hr_qp->state; - qp_attr->path_mtu = (enum ib_mtu)roce_get_field(context->qpc_bytes_48, - QP_CONTEXT_QPC_BYTES_48_MTU_M, - QP_CONTEXT_QPC_BYTES_48_MTU_S); - qp_attr->path_mig_state = IB_MIG_ARMED; - qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; - if (hr_qp->ibqp.qp_type == IB_QPT_UD) - qp_attr->qkey = QKEY_VAL; - - qp_attr->rq_psn = roce_get_field(context->qpc_bytes_88, - QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M, - QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S); - qp_attr->sq_psn = (u32)roce_get_field(context->qpc_bytes_164, - QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M, - QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S); - qp_attr->dest_qp_num = (u8)roce_get_field(context->qpc_bytes_36, - QP_CONTEXT_QPC_BYTES_36_DEST_QP_M, - QP_CONTEXT_QPC_BYTES_36_DEST_QP_S); - qp_attr->qp_access_flags = ((roce_get_bit(context->qpc_bytes_4, - QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S)) << 2) | - ((roce_get_bit(context->qpc_bytes_4, - QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S)) << 1) | - ((roce_get_bit(context->qpc_bytes_4, - QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S)) << 3); - - if (hr_qp->ibqp.qp_type == IB_QPT_RC) { - struct ib_global_route *grh = - rdma_ah_retrieve_grh(&qp_attr->ah_attr); - - rdma_ah_set_sl(&qp_attr->ah_attr, - roce_get_field(context->qpc_bytes_156, - QP_CONTEXT_QPC_BYTES_156_SL_M, - QP_CONTEXT_QPC_BYTES_156_SL_S)); - rdma_ah_set_ah_flags(&qp_attr->ah_attr, IB_AH_GRH); - grh->flow_label = - roce_get_field(context->qpc_bytes_48, - QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M, - QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S); - grh->sgid_index = - roce_get_field(context->qpc_bytes_36, - QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M, - QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S); - grh->hop_limit = - roce_get_field(context->qpc_bytes_44, - QP_CONTEXT_QPC_BYTES_44_HOPLMT_M, - QP_CONTEXT_QPC_BYTES_44_HOPLMT_S); - grh->traffic_class = - roce_get_field(context->qpc_bytes_48, - QP_CONTEXT_QPC_BYTES_48_TCLASS_M, - QP_CONTEXT_QPC_BYTES_48_TCLASS_S); - - memcpy(grh->dgid.raw, context->dgid, - sizeof(grh->dgid.raw)); - } - - qp_attr->pkey_index = roce_get_field(context->qpc_bytes_12, - QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M, - QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S); - qp_attr->port_num = hr_qp->port + 1; - qp_attr->sq_draining = 0; - qp_attr->max_rd_atomic = 1 << roce_get_field(context->qpc_bytes_156, - QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M, - QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S); - qp_attr->max_dest_rd_atomic = 1 << roce_get_field(context->qpc_bytes_32, - QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M, - QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S); - qp_attr->min_rnr_timer = (u8)(roce_get_field(context->qpc_bytes_24, - QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M, - QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S)); - qp_attr->timeout = (u8)(roce_get_field(context->qpc_bytes_156, - QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M, - QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S)); - qp_attr->retry_cnt = roce_get_field(context->qpc_bytes_148, - QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M, - QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S); - qp_attr->rnr_retry = (u8)le32_to_cpu(context->rnr_retry); - -done: - qp_attr->cur_qp_state = qp_attr->qp_state; - qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt; - qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs; - - if (!ibqp->uobject) { - qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt; - qp_attr->cap.max_send_sge = hr_qp->sq.max_gs; - } else { - qp_attr->cap.max_send_wr = 0; - qp_attr->cap.max_send_sge = 0; - } - - qp_init_attr->cap = qp_attr->cap; - -out: - mutex_unlock(&hr_qp->mutex); - kfree(context); - return ret; -} - -static int hns_roce_v1_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, - int qp_attr_mask, - struct ib_qp_init_attr *qp_init_attr) -{ - struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); - - return hr_qp->doorbell_qpn <= 1 ? - hns_roce_v1_q_sqp(ibqp, qp_attr, qp_attr_mask, qp_init_attr) : - hns_roce_v1_q_qp(ibqp, qp_attr, qp_attr_mask, qp_init_attr); -} - -int hns_roce_v1_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) -{ - struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); - struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); - struct hns_roce_cq *send_cq, *recv_cq; - int ret; - - ret = hns_roce_v1_modify_qp(ibqp, NULL, 0, hr_qp->state, IB_QPS_RESET); - if (ret) - return ret; - - send_cq = hr_qp->ibqp.send_cq ? to_hr_cq(hr_qp->ibqp.send_cq) : NULL; - recv_cq = hr_qp->ibqp.recv_cq ? to_hr_cq(hr_qp->ibqp.recv_cq) : NULL; - - hns_roce_lock_cqs(send_cq, recv_cq); - if (!udata) { - if (recv_cq) - __hns_roce_v1_cq_clean(recv_cq, hr_qp->qpn, - (hr_qp->ibqp.srq ? - to_hr_srq(hr_qp->ibqp.srq) : - NULL)); - - if (send_cq && send_cq != recv_cq) - __hns_roce_v1_cq_clean(send_cq, hr_qp->qpn, NULL); - } - hns_roce_qp_remove(hr_dev, hr_qp); - hns_roce_unlock_cqs(send_cq, recv_cq); - - hns_roce_qp_destroy(hr_dev, hr_qp, udata); - - return 0; -} - -static int hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) -{ - struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device); - struct hns_roce_cq *hr_cq = to_hr_cq(ibcq); - struct device *dev = &hr_dev->pdev->dev; - u32 cqe_cnt_ori; - u32 cqe_cnt_cur; - int wait_time = 0; - - /* - * Before freeing cq buffer, we need to ensure that the outstanding CQE - * have been written by checking the CQE counter. - */ - cqe_cnt_ori = roce_read(hr_dev, ROCEE_SCAEP_WR_CQE_CNT); - while (1) { - if (roce_read(hr_dev, ROCEE_CAEP_CQE_WCMD_EMPTY) & - HNS_ROCE_CQE_WCMD_EMPTY_BIT) - break; - - cqe_cnt_cur = roce_read(hr_dev, ROCEE_SCAEP_WR_CQE_CNT); - if ((cqe_cnt_cur - cqe_cnt_ori) >= HNS_ROCE_MIN_CQE_CNT) - break; - - msleep(HNS_ROCE_EACH_FREE_CQ_WAIT_MSECS); - if (wait_time > HNS_ROCE_MAX_FREE_CQ_WAIT_CNT) { - dev_warn(dev, "Destroy cq 0x%lx timeout!\n", - hr_cq->cqn); - break; - } - wait_time++; - } - return 0; -} - -static void set_eq_cons_index_v1(struct hns_roce_eq *eq, u32 req_not) -{ - roce_raw_write((eq->cons_index & HNS_ROCE_V1_CONS_IDX_M) | - (req_not << eq->log_entries), eq->db_reg); -} - -static void hns_roce_v1_wq_catas_err_handle(struct hns_roce_dev *hr_dev, - struct hns_roce_aeqe *aeqe, int qpn) -{ - struct device *dev = &hr_dev->pdev->dev; - - dev_warn(dev, "Local Work Queue Catastrophic Error.\n"); - switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M, - HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) { - case HNS_ROCE_LWQCE_QPC_ERROR: - dev_warn(dev, "QP %d, QPC error.\n", qpn); - break; - case HNS_ROCE_LWQCE_MTU_ERROR: - dev_warn(dev, "QP %d, MTU error.\n", qpn); - break; - case HNS_ROCE_LWQCE_WQE_BA_ADDR_ERROR: - dev_warn(dev, "QP %d, WQE BA addr error.\n", qpn); - break; - case HNS_ROCE_LWQCE_WQE_ADDR_ERROR: - dev_warn(dev, "QP %d, WQE addr error.\n", qpn); - break; - case HNS_ROCE_LWQCE_SQ_WQE_SHIFT_ERROR: - dev_warn(dev, "QP %d, WQE shift error\n", qpn); - break; - case HNS_ROCE_LWQCE_SL_ERROR: - dev_warn(dev, "QP %d, SL error.\n", qpn); - break; - case HNS_ROCE_LWQCE_PORT_ERROR: - dev_warn(dev, "QP %d, port error.\n", qpn); - break; - default: - break; - } -} - -static void hns_roce_v1_local_wq_access_err_handle(struct hns_roce_dev *hr_dev, - struct hns_roce_aeqe *aeqe, - int qpn) -{ - struct device *dev = &hr_dev->pdev->dev; - - dev_warn(dev, "Local Access Violation Work Queue Error.\n"); - switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M, - HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) { - case HNS_ROCE_LAVWQE_R_KEY_VIOLATION: - dev_warn(dev, "QP %d, R_key violation.\n", qpn); - break; - case HNS_ROCE_LAVWQE_LENGTH_ERROR: - dev_warn(dev, "QP %d, length error.\n", qpn); - break; - case HNS_ROCE_LAVWQE_VA_ERROR: - dev_warn(dev, "QP %d, VA error.\n", qpn); - break; - case HNS_ROCE_LAVWQE_PD_ERROR: - dev_err(dev, "QP %d, PD error.\n", qpn); - break; - case HNS_ROCE_LAVWQE_RW_ACC_ERROR: - dev_warn(dev, "QP %d, rw acc error.\n", qpn); - break; - case HNS_ROCE_LAVWQE_KEY_STATE_ERROR: - dev_warn(dev, "QP %d, key state error.\n", qpn); - break; - case HNS_ROCE_LAVWQE_MR_OPERATION_ERROR: - dev_warn(dev, "QP %d, MR operation error.\n", qpn); - break; - default: - break; - } -} - -static void hns_roce_v1_qp_err_handle(struct hns_roce_dev *hr_dev, - struct hns_roce_aeqe *aeqe, - int event_type) -{ - struct device *dev = &hr_dev->pdev->dev; - int phy_port; - int qpn; - - qpn = roce_get_field(aeqe->event.queue_event.num, - HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_M, - HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S); - phy_port = roce_get_field(aeqe->event.queue_event.num, - HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_M, - HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_S); - if (qpn <= 1) - qpn = HNS_ROCE_MAX_PORTS * qpn + phy_port; - - switch (event_type) { - case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: - dev_warn(dev, "Invalid Req Local Work Queue Error.\n" - "QP %d, phy_port %d.\n", qpn, phy_port); - break; - case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: - hns_roce_v1_wq_catas_err_handle(hr_dev, aeqe, qpn); - break; - case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: - hns_roce_v1_local_wq_access_err_handle(hr_dev, aeqe, qpn); - break; - default: - break; - } - - hns_roce_qp_event(hr_dev, qpn, event_type); -} - -static void hns_roce_v1_cq_err_handle(struct hns_roce_dev *hr_dev, - struct hns_roce_aeqe *aeqe, - int event_type) -{ - struct device *dev = &hr_dev->pdev->dev; - u32 cqn; - - cqn = roce_get_field(aeqe->event.queue_event.num, - HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_M, - HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S); - - switch (event_type) { - case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR: - dev_warn(dev, "CQ 0x%x access err.\n", cqn); - break; - case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW: - dev_warn(dev, "CQ 0x%x overflow\n", cqn); - break; - case HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID: - dev_warn(dev, "CQ 0x%x ID invalid.\n", cqn); - break; - default: - break; - } - - hns_roce_cq_event(hr_dev, cqn, event_type); -} - -static void hns_roce_v1_db_overflow_handle(struct hns_roce_dev *hr_dev, - struct hns_roce_aeqe *aeqe) -{ - struct device *dev = &hr_dev->pdev->dev; - - switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M, - HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) { - case HNS_ROCE_DB_SUBTYPE_SDB_OVF: - dev_warn(dev, "SDB overflow.\n"); - break; - case HNS_ROCE_DB_SUBTYPE_SDB_ALM_OVF: - dev_warn(dev, "SDB almost overflow.\n"); - break; - case HNS_ROCE_DB_SUBTYPE_SDB_ALM_EMP: - dev_warn(dev, "SDB almost empty.\n"); - break; - case HNS_ROCE_DB_SUBTYPE_ODB_OVF: - dev_warn(dev, "ODB overflow.\n"); - break; - case HNS_ROCE_DB_SUBTYPE_ODB_ALM_OVF: - dev_warn(dev, "ODB almost overflow.\n"); - break; - case HNS_ROCE_DB_SUBTYPE_ODB_ALM_EMP: - dev_warn(dev, "SDB almost empty.\n"); - break; - default: - break; - } -} - -static struct hns_roce_aeqe *get_aeqe_v1(struct hns_roce_eq *eq, u32 entry) -{ - unsigned long off = (entry & (eq->entries - 1)) * HNS_ROCE_AEQE_SIZE; - - return (struct hns_roce_aeqe *)((u8 *) - (eq->buf_list[off / HNS_ROCE_BA_SIZE].buf) + - off % HNS_ROCE_BA_SIZE); -} - -static struct hns_roce_aeqe *next_aeqe_sw_v1(struct hns_roce_eq *eq) -{ - struct hns_roce_aeqe *aeqe = get_aeqe_v1(eq, eq->cons_index); - - return (roce_get_bit(aeqe->asyn, HNS_ROCE_AEQE_U32_4_OWNER_S) ^ - !!(eq->cons_index & eq->entries)) ? aeqe : NULL; -} - -static int hns_roce_v1_aeq_int(struct hns_roce_dev *hr_dev, - struct hns_roce_eq *eq) -{ - struct device *dev = &hr_dev->pdev->dev; - struct hns_roce_aeqe *aeqe; - int aeqes_found = 0; - int event_type; - - while ((aeqe = next_aeqe_sw_v1(eq))) { - /* Make sure we read the AEQ entry after we have checked the - * ownership bit - */ - dma_rmb(); - - dev_dbg(dev, "aeqe = %pK, aeqe->asyn.event_type = 0x%lx\n", - aeqe, - roce_get_field(aeqe->asyn, - HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M, - HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S)); - event_type = roce_get_field(aeqe->asyn, - HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M, - HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S); - switch (event_type) { - case HNS_ROCE_EVENT_TYPE_PATH_MIG: - dev_warn(dev, "PATH MIG not supported\n"); - break; - case HNS_ROCE_EVENT_TYPE_COMM_EST: - dev_warn(dev, "COMMUNICATION established\n"); - break; - case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: - dev_warn(dev, "SQ DRAINED not supported\n"); - break; - case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED: - dev_warn(dev, "PATH MIG failed\n"); - break; - case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: - case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: - case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: - hns_roce_v1_qp_err_handle(hr_dev, aeqe, event_type); - break; - case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH: - case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR: - case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH: - dev_warn(dev, "SRQ not support!\n"); - break; - case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR: - case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW: - case HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID: - hns_roce_v1_cq_err_handle(hr_dev, aeqe, event_type); - break; - case HNS_ROCE_EVENT_TYPE_PORT_CHANGE: - dev_warn(dev, "port change.\n"); - break; - case HNS_ROCE_EVENT_TYPE_MB: - hns_roce_cmd_event(hr_dev, - le16_to_cpu(aeqe->event.cmd.token), - aeqe->event.cmd.status, - le64_to_cpu(aeqe->event.cmd.out_param - )); - break; - case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW: - hns_roce_v1_db_overflow_handle(hr_dev, aeqe); - break; - default: - dev_warn(dev, "Unhandled event %d on EQ %d at idx %u.\n", - event_type, eq->eqn, eq->cons_index); - break; - } - - eq->cons_index++; - aeqes_found = 1; - - if (eq->cons_index > 2 * hr_dev->caps.aeqe_depth - 1) - eq->cons_index = 0; - } - - set_eq_cons_index_v1(eq, 0); - - return aeqes_found; -} - -static struct hns_roce_ceqe *get_ceqe_v1(struct hns_roce_eq *eq, u32 entry) -{ - unsigned long off = (entry & (eq->entries - 1)) * HNS_ROCE_CEQE_SIZE; - - return (struct hns_roce_ceqe *)((u8 *) - (eq->buf_list[off / HNS_ROCE_BA_SIZE].buf) + - off % HNS_ROCE_BA_SIZE); -} - -static struct hns_roce_ceqe *next_ceqe_sw_v1(struct hns_roce_eq *eq) -{ - struct hns_roce_ceqe *ceqe = get_ceqe_v1(eq, eq->cons_index); - - return (!!(roce_get_bit(ceqe->comp, - HNS_ROCE_CEQE_CEQE_COMP_OWNER_S))) ^ - (!!(eq->cons_index & eq->entries)) ? ceqe : NULL; -} - -static int hns_roce_v1_ceq_int(struct hns_roce_dev *hr_dev, - struct hns_roce_eq *eq) -{ - struct hns_roce_ceqe *ceqe; - int ceqes_found = 0; - u32 cqn; - - while ((ceqe = next_ceqe_sw_v1(eq))) { - /* Make sure we read CEQ entry after we have checked the - * ownership bit - */ - dma_rmb(); - - cqn = roce_get_field(ceqe->comp, - HNS_ROCE_CEQE_CEQE_COMP_CQN_M, - HNS_ROCE_CEQE_CEQE_COMP_CQN_S); - hns_roce_cq_completion(hr_dev, cqn); - - ++eq->cons_index; - ceqes_found = 1; - - if (eq->cons_index > - EQ_DEPTH_COEFF * hr_dev->caps.ceqe_depth - 1) - eq->cons_index = 0; - } - - set_eq_cons_index_v1(eq, 0); - - return ceqes_found; -} - -static irqreturn_t hns_roce_v1_msix_interrupt_eq(int irq, void *eq_ptr) -{ - struct hns_roce_eq *eq = eq_ptr; - struct hns_roce_dev *hr_dev = eq->hr_dev; - int int_work; - - if (eq->type_flag == HNS_ROCE_CEQ) - /* CEQ irq routine, CEQ is pulse irq, not clear */ - int_work = hns_roce_v1_ceq_int(hr_dev, eq); - else - /* AEQ irq routine, AEQ is pulse irq, not clear */ - int_work = hns_roce_v1_aeq_int(hr_dev, eq); - - return IRQ_RETVAL(int_work); -} - -static irqreturn_t hns_roce_v1_msix_interrupt_abn(int irq, void *dev_id) -{ - struct hns_roce_dev *hr_dev = dev_id; - struct device *dev = &hr_dev->pdev->dev; - int int_work = 0; - u32 caepaemask_val; - u32 cealmovf_val; - u32 caepaest_val; - u32 aeshift_val; - u32 ceshift_val; - u32 cemask_val; - __le32 tmp; - int i; - - /* - * Abnormal interrupt: - * AEQ overflow, ECC multi-bit err, CEQ overflow must clear - * interrupt, mask irq, clear irq, cancel mask operation - */ - aeshift_val = roce_read(hr_dev, ROCEE_CAEP_AEQC_AEQE_SHIFT_REG); - tmp = cpu_to_le32(aeshift_val); - - /* AEQE overflow */ - if (roce_get_bit(tmp, - ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQ_ALM_OVF_INT_ST_S) == 1) { - dev_warn(dev, "AEQ overflow!\n"); - - /* Set mask */ - caepaemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG); - tmp = cpu_to_le32(caepaemask_val); - roce_set_bit(tmp, ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S, - HNS_ROCE_INT_MASK_ENABLE); - caepaemask_val = le32_to_cpu(tmp); - roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, caepaemask_val); - - /* Clear int state(INT_WC : write 1 clear) */ - caepaest_val = roce_read(hr_dev, ROCEE_CAEP_AE_ST_REG); - tmp = cpu_to_le32(caepaest_val); - roce_set_bit(tmp, ROCEE_CAEP_AE_ST_CAEP_AEQ_ALM_OVF_S, 1); - caepaest_val = le32_to_cpu(tmp); - roce_write(hr_dev, ROCEE_CAEP_AE_ST_REG, caepaest_val); - - /* Clear mask */ - caepaemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG); - tmp = cpu_to_le32(caepaemask_val); - roce_set_bit(tmp, ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S, - HNS_ROCE_INT_MASK_DISABLE); - caepaemask_val = le32_to_cpu(tmp); - roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, caepaemask_val); - } - - /* CEQ almost overflow */ - for (i = 0; i < hr_dev->caps.num_comp_vectors; i++) { - ceshift_val = roce_read(hr_dev, ROCEE_CAEP_CEQC_SHIFT_0_REG + - i * CEQ_REG_OFFSET); - tmp = cpu_to_le32(ceshift_val); - - if (roce_get_bit(tmp, - ROCEE_CAEP_CEQC_SHIFT_CAEP_CEQ_ALM_OVF_INT_ST_S) == 1) { - dev_warn(dev, "CEQ[%d] almost overflow!\n", i); - int_work++; - - /* Set mask */ - cemask_val = roce_read(hr_dev, - ROCEE_CAEP_CE_IRQ_MASK_0_REG + - i * CEQ_REG_OFFSET); - tmp = cpu_to_le32(cemask_val); - roce_set_bit(tmp, - ROCEE_CAEP_CE_IRQ_MASK_CAEP_CEQ_ALM_OVF_MASK_S, - HNS_ROCE_INT_MASK_ENABLE); - cemask_val = le32_to_cpu(tmp); - roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG + - i * CEQ_REG_OFFSET, cemask_val); - - /* Clear int state(INT_WC : write 1 clear) */ - cealmovf_val = roce_read(hr_dev, - ROCEE_CAEP_CEQ_ALM_OVF_0_REG + - i * CEQ_REG_OFFSET); - tmp = cpu_to_le32(cealmovf_val); - roce_set_bit(tmp, - ROCEE_CAEP_CEQ_ALM_OVF_CAEP_CEQ_ALM_OVF_S, - 1); - cealmovf_val = le32_to_cpu(tmp); - roce_write(hr_dev, ROCEE_CAEP_CEQ_ALM_OVF_0_REG + - i * CEQ_REG_OFFSET, cealmovf_val); - - /* Clear mask */ - cemask_val = roce_read(hr_dev, - ROCEE_CAEP_CE_IRQ_MASK_0_REG + - i * CEQ_REG_OFFSET); - tmp = cpu_to_le32(cemask_val); - roce_set_bit(tmp, - ROCEE_CAEP_CE_IRQ_MASK_CAEP_CEQ_ALM_OVF_MASK_S, - HNS_ROCE_INT_MASK_DISABLE); - cemask_val = le32_to_cpu(tmp); - roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG + - i * CEQ_REG_OFFSET, cemask_val); - } - } - - /* ECC multi-bit error alarm */ - dev_warn(dev, "ECC UCERR ALARM: 0x%x, 0x%x, 0x%x\n", - roce_read(hr_dev, ROCEE_ECC_UCERR_ALM0_REG), - roce_read(hr_dev, ROCEE_ECC_UCERR_ALM1_REG), - roce_read(hr_dev, ROCEE_ECC_UCERR_ALM2_REG)); - - dev_warn(dev, "ECC CERR ALARM: 0x%x, 0x%x, 0x%x\n", - roce_read(hr_dev, ROCEE_ECC_CERR_ALM0_REG), - roce_read(hr_dev, ROCEE_ECC_CERR_ALM1_REG), - roce_read(hr_dev, ROCEE_ECC_CERR_ALM2_REG)); - - return IRQ_RETVAL(int_work); -} - -static void hns_roce_v1_int_mask_enable(struct hns_roce_dev *hr_dev) -{ - u32 aemask_val; - int masken = 0; - __le32 tmp; - int i; - - /* AEQ INT */ - aemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG); - tmp = cpu_to_le32(aemask_val); - roce_set_bit(tmp, ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S, - masken); - roce_set_bit(tmp, ROCEE_CAEP_AE_MASK_CAEP_AE_IRQ_MASK_S, masken); - aemask_val = le32_to_cpu(tmp); - roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, aemask_val); - - /* CEQ INT */ - for (i = 0; i < hr_dev->caps.num_comp_vectors; i++) { - /* IRQ mask */ - roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG + - i * CEQ_REG_OFFSET, masken); - } -} - -static void hns_roce_v1_free_eq(struct hns_roce_dev *hr_dev, - struct hns_roce_eq *eq) -{ - int npages = (PAGE_ALIGN(eq->eqe_size * eq->entries) + - HNS_ROCE_BA_SIZE - 1) / HNS_ROCE_BA_SIZE; - int i; - - if (!eq->buf_list) - return; - - for (i = 0; i < npages; ++i) - dma_free_coherent(&hr_dev->pdev->dev, HNS_ROCE_BA_SIZE, - eq->buf_list[i].buf, eq->buf_list[i].map); - - kfree(eq->buf_list); -} - -static void hns_roce_v1_enable_eq(struct hns_roce_dev *hr_dev, int eq_num, - int enable_flag) -{ - void __iomem *eqc = hr_dev->eq_table.eqc_base[eq_num]; - __le32 tmp; - u32 val; - - val = readl(eqc); - tmp = cpu_to_le32(val); - - if (enable_flag) - roce_set_field(tmp, - ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M, - ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S, - HNS_ROCE_EQ_STAT_VALID); - else - roce_set_field(tmp, - ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M, - ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S, - HNS_ROCE_EQ_STAT_INVALID); - - val = le32_to_cpu(tmp); - writel(val, eqc); -} - -static int hns_roce_v1_create_eq(struct hns_roce_dev *hr_dev, - struct hns_roce_eq *eq) -{ - void __iomem *eqc = hr_dev->eq_table.eqc_base[eq->eqn]; - struct device *dev = &hr_dev->pdev->dev; - dma_addr_t tmp_dma_addr; - u32 eqcuridx_val; - u32 eqconsindx_val; - u32 eqshift_val; - __le32 tmp2 = 0; - __le32 tmp1 = 0; - __le32 tmp = 0; - int num_bas; - int ret; - int i; - - num_bas = (PAGE_ALIGN(eq->entries * eq->eqe_size) + - HNS_ROCE_BA_SIZE - 1) / HNS_ROCE_BA_SIZE; - - if ((eq->entries * eq->eqe_size) > HNS_ROCE_BA_SIZE) { - dev_err(dev, "[error]eq buf %d gt ba size(%d) need bas=%d\n", - (eq->entries * eq->eqe_size), HNS_ROCE_BA_SIZE, - num_bas); - return -EINVAL; - } - - eq->buf_list = kcalloc(num_bas, sizeof(*eq->buf_list), GFP_KERNEL); - if (!eq->buf_list) - return -ENOMEM; - - for (i = 0; i < num_bas; ++i) { - eq->buf_list[i].buf = dma_alloc_coherent(dev, HNS_ROCE_BA_SIZE, - &tmp_dma_addr, - GFP_KERNEL); - if (!eq->buf_list[i].buf) { - ret = -ENOMEM; - goto err_out_free_pages; - } - - eq->buf_list[i].map = tmp_dma_addr; - } - eq->cons_index = 0; - roce_set_field(tmp, ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M, - ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S, - HNS_ROCE_EQ_STAT_INVALID); - roce_set_field(tmp, ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_M, - ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_S, - eq->log_entries); - eqshift_val = le32_to_cpu(tmp); - writel(eqshift_val, eqc); - - /* Configure eq extended address 12~44bit */ - writel((u32)(eq->buf_list[0].map >> 12), eqc + 4); - - /* - * Configure eq extended address 45~49 bit. - * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of - * using 4K page, and shift more 32 because of - * calculating the high 32 bit value evaluated to hardware. - */ - roce_set_field(tmp1, ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_M, - ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_S, - eq->buf_list[0].map >> 44); - roce_set_field(tmp1, ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_M, - ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_S, 0); - eqcuridx_val = le32_to_cpu(tmp1); - writel(eqcuridx_val, eqc + 8); - - /* Configure eq consumer index */ - roce_set_field(tmp2, ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_M, - ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_S, 0); - eqconsindx_val = le32_to_cpu(tmp2); - writel(eqconsindx_val, eqc + 0xc); - - return 0; - -err_out_free_pages: - for (i -= 1; i >= 0; i--) - dma_free_coherent(dev, HNS_ROCE_BA_SIZE, eq->buf_list[i].buf, - eq->buf_list[i].map); - - kfree(eq->buf_list); - return ret; -} - -static int hns_roce_v1_init_eq_table(struct hns_roce_dev *hr_dev) -{ - struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; - struct device *dev = &hr_dev->pdev->dev; - struct hns_roce_eq *eq; - int irq_num; - int eq_num; - int ret; - int i, j; - - eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors; - irq_num = eq_num + hr_dev->caps.num_other_vectors; - - eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL); - if (!eq_table->eq) - return -ENOMEM; - - eq_table->eqc_base = kcalloc(eq_num, sizeof(*eq_table->eqc_base), - GFP_KERNEL); - if (!eq_table->eqc_base) { - ret = -ENOMEM; - goto err_eqc_base_alloc_fail; - } - - for (i = 0; i < eq_num; i++) { - eq = &eq_table->eq[i]; - eq->hr_dev = hr_dev; - eq->eqn = i; - eq->irq = hr_dev->irq[i]; - eq->log_page_size = PAGE_SHIFT; - - if (i < hr_dev->caps.num_comp_vectors) { - /* CEQ */ - eq_table->eqc_base[i] = hr_dev->reg_base + - ROCEE_CAEP_CEQC_SHIFT_0_REG + - CEQ_REG_OFFSET * i; - eq->type_flag = HNS_ROCE_CEQ; - eq->db_reg = hr_dev->reg_base + - ROCEE_CAEP_CEQC_CONS_IDX_0_REG + - CEQ_REG_OFFSET * i; - eq->entries = hr_dev->caps.ceqe_depth; - eq->log_entries = ilog2(eq->entries); - eq->eqe_size = HNS_ROCE_CEQE_SIZE; - } else { - /* AEQ */ - eq_table->eqc_base[i] = hr_dev->reg_base + - ROCEE_CAEP_AEQC_AEQE_SHIFT_REG; - eq->type_flag = HNS_ROCE_AEQ; - eq->db_reg = hr_dev->reg_base + - ROCEE_CAEP_AEQE_CONS_IDX_REG; - eq->entries = hr_dev->caps.aeqe_depth; - eq->log_entries = ilog2(eq->entries); - eq->eqe_size = HNS_ROCE_AEQE_SIZE; - } - } - - /* Disable irq */ - hns_roce_v1_int_mask_enable(hr_dev); - - /* Configure ce int interval */ - roce_write(hr_dev, ROCEE_CAEP_CE_INTERVAL_CFG_REG, - HNS_ROCE_CEQ_DEFAULT_INTERVAL); - - /* Configure ce int burst num */ - roce_write(hr_dev, ROCEE_CAEP_CE_BURST_NUM_CFG_REG, - HNS_ROCE_CEQ_DEFAULT_BURST_NUM); - - for (i = 0; i < eq_num; i++) { - ret = hns_roce_v1_create_eq(hr_dev, &eq_table->eq[i]); - if (ret) { - dev_err(dev, "eq create failed\n"); - goto err_create_eq_fail; - } - } - - for (j = 0; j < irq_num; j++) { - if (j < eq_num) - ret = request_irq(hr_dev->irq[j], - hns_roce_v1_msix_interrupt_eq, 0, - hr_dev->irq_names[j], - &eq_table->eq[j]); - else - ret = request_irq(hr_dev->irq[j], - hns_roce_v1_msix_interrupt_abn, 0, - hr_dev->irq_names[j], hr_dev); - - if (ret) { - dev_err(dev, "request irq error!\n"); - goto err_request_irq_fail; - } - } - - for (i = 0; i < eq_num; i++) - hns_roce_v1_enable_eq(hr_dev, i, EQ_ENABLE); - - return 0; - -err_request_irq_fail: - for (j -= 1; j >= 0; j--) - free_irq(hr_dev->irq[j], &eq_table->eq[j]); - -err_create_eq_fail: - for (i -= 1; i >= 0; i--) - hns_roce_v1_free_eq(hr_dev, &eq_table->eq[i]); - - kfree(eq_table->eqc_base); - -err_eqc_base_alloc_fail: - kfree(eq_table->eq); - - return ret; -} - -static void hns_roce_v1_cleanup_eq_table(struct hns_roce_dev *hr_dev) -{ - struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; - int irq_num; - int eq_num; - int i; - - eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors; - irq_num = eq_num + hr_dev->caps.num_other_vectors; - for (i = 0; i < eq_num; i++) { - /* Disable EQ */ - hns_roce_v1_enable_eq(hr_dev, i, EQ_DISABLE); - - free_irq(hr_dev->irq[i], &eq_table->eq[i]); - - hns_roce_v1_free_eq(hr_dev, &eq_table->eq[i]); - } - for (i = eq_num; i < irq_num; i++) - free_irq(hr_dev->irq[i], hr_dev); - - kfree(eq_table->eqc_base); - kfree(eq_table->eq); -} - -static const struct ib_device_ops hns_roce_v1_dev_ops = { - .destroy_qp = hns_roce_v1_destroy_qp, - .poll_cq = hns_roce_v1_poll_cq, - .post_recv = hns_roce_v1_post_recv, - .post_send = hns_roce_v1_post_send, - .query_qp = hns_roce_v1_query_qp, - .req_notify_cq = hns_roce_v1_req_notify_cq, -}; - -static const struct hns_roce_hw hns_roce_hw_v1 = { - .reset = hns_roce_v1_reset, - .hw_profile = hns_roce_v1_profile, - .hw_init = hns_roce_v1_init, - .hw_exit = hns_roce_v1_exit, - .post_mbox = hns_roce_v1_post_mbox, - .poll_mbox_done = hns_roce_v1_chk_mbox, - .set_gid = hns_roce_v1_set_gid, - .set_mac = hns_roce_v1_set_mac, - .set_mtu = hns_roce_v1_set_mtu, - .write_mtpt = hns_roce_v1_write_mtpt, - .write_cqc = hns_roce_v1_write_cqc, - .set_hem = hns_roce_v1_set_hem, - .clear_hem = hns_roce_v1_clear_hem, - .modify_qp = hns_roce_v1_modify_qp, - .dereg_mr = hns_roce_v1_dereg_mr, - .destroy_cq = hns_roce_v1_destroy_cq, - .init_eq = hns_roce_v1_init_eq_table, - .cleanup_eq = hns_roce_v1_cleanup_eq_table, - .hns_roce_dev_ops = &hns_roce_v1_dev_ops, -}; - -static const struct of_device_id hns_roce_of_match[] = { - { .compatible = "hisilicon,hns-roce-v1", .data = &hns_roce_hw_v1, }, - {}, -}; -MODULE_DEVICE_TABLE(of, hns_roce_of_match); - -static const struct acpi_device_id hns_roce_acpi_match[] = { - { "HISI00D1", (kernel_ulong_t)&hns_roce_hw_v1 }, - {}, -}; -MODULE_DEVICE_TABLE(acpi, hns_roce_acpi_match); - -static struct -platform_device *hns_roce_find_pdev(struct fwnode_handle *fwnode) -{ - struct device *dev; - - /* get the 'device' corresponding to the matching 'fwnode' */ - dev = bus_find_device_by_fwnode(&platform_bus_type, fwnode); - /* get the platform device */ - return dev ? to_platform_device(dev) : NULL; -} - -static int hns_roce_get_cfg(struct hns_roce_dev *hr_dev) -{ - struct device *dev = &hr_dev->pdev->dev; - struct platform_device *pdev = NULL; - struct net_device *netdev = NULL; - struct device_node *net_node; - int port_cnt = 0; - u8 phy_port; - int ret; - int i; - - /* check if we are compatible with the underlying SoC */ - if (dev_of_node(dev)) { - const struct of_device_id *of_id; - - of_id = of_match_node(hns_roce_of_match, dev->of_node); - if (!of_id) { - dev_err(dev, "device is not compatible!\n"); - return -ENXIO; - } - hr_dev->hw = (const struct hns_roce_hw *)of_id->data; - if (!hr_dev->hw) { - dev_err(dev, "couldn't get H/W specific DT data!\n"); - return -ENXIO; - } - } else if (is_acpi_device_node(dev->fwnode)) { - const struct acpi_device_id *acpi_id; - - acpi_id = acpi_match_device(hns_roce_acpi_match, dev); - if (!acpi_id) { - dev_err(dev, "device is not compatible!\n"); - return -ENXIO; - } - hr_dev->hw = (const struct hns_roce_hw *) acpi_id->driver_data; - if (!hr_dev->hw) { - dev_err(dev, "couldn't get H/W specific ACPI data!\n"); - return -ENXIO; - } - } else { - dev_err(dev, "can't read compatibility data from DT or ACPI\n"); - return -ENXIO; - } - - /* get the mapped register base address */ - hr_dev->reg_base = devm_platform_ioremap_resource(hr_dev->pdev, 0); - if (IS_ERR(hr_dev->reg_base)) - return PTR_ERR(hr_dev->reg_base); - - /* read the node_guid of IB device from the DT or ACPI */ - ret = device_property_read_u8_array(dev, "node-guid", - (u8 *)&hr_dev->ib_dev.node_guid, - GUID_LEN); - if (ret) { - dev_err(dev, "couldn't get node_guid from DT or ACPI!\n"); - return ret; - } - - /* get the RoCE associated ethernet ports or netdevices */ - for (i = 0; i < HNS_ROCE_MAX_PORTS; i++) { - if (dev_of_node(dev)) { - net_node = of_parse_phandle(dev->of_node, "eth-handle", - i); - if (!net_node) - continue; - pdev = of_find_device_by_node(net_node); - } else if (is_acpi_device_node(dev->fwnode)) { - struct fwnode_reference_args args; - - ret = acpi_node_get_property_reference(dev->fwnode, - "eth-handle", - i, &args); - if (ret) - continue; - pdev = hns_roce_find_pdev(args.fwnode); - } else { - dev_err(dev, "cannot read data from DT or ACPI\n"); - return -ENXIO; - } - - if (pdev) { - netdev = platform_get_drvdata(pdev); - phy_port = (u8)i; - if (netdev) { - hr_dev->iboe.netdevs[port_cnt] = netdev; - hr_dev->iboe.phy_port[port_cnt] = phy_port; - } else { - dev_err(dev, "no netdev found with pdev %s\n", - pdev->name); - return -ENODEV; - } - port_cnt++; - } - } - - if (port_cnt == 0) { - dev_err(dev, "unable to get eth-handle for available ports!\n"); - return -EINVAL; - } - - hr_dev->caps.num_ports = port_cnt; - - /* cmd issue mode: 0 is poll, 1 is event */ - hr_dev->cmd_mod = 1; - hr_dev->loop_idc = 0; - hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG; - hr_dev->odb_offset = ROCEE_DB_OTHERS_L_0_REG; - - /* read the interrupt names from the DT or ACPI */ - ret = device_property_read_string_array(dev, "interrupt-names", - hr_dev->irq_names, - HNS_ROCE_V1_MAX_IRQ_NUM); - if (ret < 0) { - dev_err(dev, "couldn't get interrupt names from DT or ACPI!\n"); - return ret; - } - - /* fetch the interrupt numbers */ - for (i = 0; i < HNS_ROCE_V1_MAX_IRQ_NUM; i++) { - hr_dev->irq[i] = platform_get_irq(hr_dev->pdev, i); - if (hr_dev->irq[i] <= 0) - return -EINVAL; - } - - return 0; -} - -/** - * hns_roce_probe - RoCE driver entrance - * @pdev: pointer to platform device - * Return : int - * - */ -static int hns_roce_probe(struct platform_device *pdev) -{ - int ret; - struct hns_roce_dev *hr_dev; - struct device *dev = &pdev->dev; - - hr_dev = ib_alloc_device(hns_roce_dev, ib_dev); - if (!hr_dev) - return -ENOMEM; - - hr_dev->priv = kzalloc(sizeof(struct hns_roce_v1_priv), GFP_KERNEL); - if (!hr_dev->priv) { - ret = -ENOMEM; - goto error_failed_kzalloc; - } - - hr_dev->pdev = pdev; - hr_dev->dev = dev; - platform_set_drvdata(pdev, hr_dev); - - if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64ULL)) && - dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32ULL))) { - dev_err(dev, "Not usable DMA addressing mode\n"); - ret = -EIO; - goto error_failed_get_cfg; - } - - ret = hns_roce_get_cfg(hr_dev); - if (ret) { - dev_err(dev, "Get Configuration failed!\n"); - goto error_failed_get_cfg; - } - - ret = hns_roce_init(hr_dev); - if (ret) { - dev_err(dev, "RoCE engine init failed!\n"); - goto error_failed_get_cfg; - } - - return 0; - -error_failed_get_cfg: - kfree(hr_dev->priv); - -error_failed_kzalloc: - ib_dealloc_device(&hr_dev->ib_dev); - - return ret; -} - -/** - * hns_roce_remove - remove RoCE device - * @pdev: pointer to platform device - */ -static int hns_roce_remove(struct platform_device *pdev) -{ - struct hns_roce_dev *hr_dev = platform_get_drvdata(pdev); - - hns_roce_exit(hr_dev); - kfree(hr_dev->priv); - ib_dealloc_device(&hr_dev->ib_dev); - - return 0; -} - -static struct platform_driver hns_roce_driver = { - .probe = hns_roce_probe, - .remove = hns_roce_remove, - .driver = { - .name = DRV_NAME, - .of_match_table = hns_roce_of_match, - .acpi_match_table = ACPI_PTR(hns_roce_acpi_match), - }, -}; - -module_platform_driver(hns_roce_driver); - -MODULE_LICENSE("Dual BSD/GPL"); -MODULE_AUTHOR("Wei Hu "); -MODULE_AUTHOR("Nenglong Zhao "); -MODULE_AUTHOR("Lijun Ou "); -MODULE_DESCRIPTION("Hisilicon Hip06 Family RoCE Driver"); diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.h b/drivers/infiniband/hw/hns/hns_roce_hw_v1.h deleted file mode 100644 index 60fdcbae6729..000000000000 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.h +++ /dev/null @@ -1,1147 +0,0 @@ -/* - * Copyright (c) 2016 Hisilicon Limited. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef _HNS_ROCE_HW_V1_H -#define _HNS_ROCE_HW_V1_H - -#define CQ_STATE_VALID 2 - -#define HNS_ROCE_V1_MAX_PD_NUM 0x8000 -#define HNS_ROCE_V1_MAX_CQ_NUM 0x10000 -#define HNS_ROCE_V1_MAX_CQE_NUM 0x8000 - -#define HNS_ROCE_V1_MAX_QP_NUM 0x40000 -#define HNS_ROCE_V1_MAX_WQE_NUM 0x4000 - -#define HNS_ROCE_V1_MAX_MTPT_NUM 0x80000 - -#define HNS_ROCE_V1_MAX_MTT_SEGS 0x100000 - -#define HNS_ROCE_V1_MAX_QP_INIT_RDMA 128 -#define HNS_ROCE_V1_MAX_QP_DEST_RDMA 128 - -#define HNS_ROCE_V1_MAX_SQ_DESC_SZ 64 -#define HNS_ROCE_V1_MAX_RQ_DESC_SZ 64 -#define HNS_ROCE_V1_SG_NUM 2 -#define HNS_ROCE_V1_INLINE_SIZE 32 - -#define HNS_ROCE_V1_UAR_NUM 256 -#define HNS_ROCE_V1_PHY_UAR_NUM 8 - -#define HNS_ROCE_V1_GID_NUM 16 -#define HNS_ROCE_V1_RESV_QP 8 - -#define HNS_ROCE_V1_MAX_IRQ_NUM 34 -#define HNS_ROCE_V1_COMP_VEC_NUM 32 -#define HNS_ROCE_V1_AEQE_VEC_NUM 1 -#define HNS_ROCE_V1_ABNORMAL_VEC_NUM 1 - -#define HNS_ROCE_V1_COMP_EQE_NUM 0x8000 -#define HNS_ROCE_V1_ASYNC_EQE_NUM 0x400 - -#define HNS_ROCE_V1_QPC_SIZE 256 -#define HNS_ROCE_V1_IRRL_ENTRY_SIZE 8 -#define HNS_ROCE_V1_CQC_ENTRY_SIZE 64 -#define HNS_ROCE_V1_MTPT_ENTRY_SIZE 64 -#define HNS_ROCE_V1_MTT_ENTRY_SIZE 64 - -#define HNS_ROCE_V1_CQE_SIZE 32 -#define HNS_ROCE_V1_PAGE_SIZE_SUPPORT 0xFFFFF000 - -#define HNS_ROCE_V1_TABLE_CHUNK_SIZE (1 << 17) - -#define HNS_ROCE_V1_EXT_RAQ_WF 8 -#define HNS_ROCE_V1_RAQ_ENTRY 64 -#define HNS_ROCE_V1_RAQ_DEPTH 32768 -#define HNS_ROCE_V1_RAQ_SIZE (HNS_ROCE_V1_RAQ_ENTRY * HNS_ROCE_V1_RAQ_DEPTH) - -#define HNS_ROCE_V1_SDB_DEPTH 0x400 -#define HNS_ROCE_V1_ODB_DEPTH 0x400 - -#define HNS_ROCE_V1_DB_RSVD 0x80 - -#define HNS_ROCE_V1_SDB_ALEPT HNS_ROCE_V1_DB_RSVD -#define HNS_ROCE_V1_SDB_ALFUL (HNS_ROCE_V1_SDB_DEPTH - HNS_ROCE_V1_DB_RSVD) -#define HNS_ROCE_V1_ODB_ALEPT HNS_ROCE_V1_DB_RSVD -#define HNS_ROCE_V1_ODB_ALFUL (HNS_ROCE_V1_ODB_DEPTH - HNS_ROCE_V1_DB_RSVD) - -#define HNS_ROCE_V1_EXT_SDB_DEPTH 0x4000 -#define HNS_ROCE_V1_EXT_ODB_DEPTH 0x4000 -#define HNS_ROCE_V1_EXT_SDB_ENTRY 16 -#define HNS_ROCE_V1_EXT_ODB_ENTRY 16 -#define HNS_ROCE_V1_EXT_SDB_SIZE \ - (HNS_ROCE_V1_EXT_SDB_DEPTH * HNS_ROCE_V1_EXT_SDB_ENTRY) -#define HNS_ROCE_V1_EXT_ODB_SIZE \ - (HNS_ROCE_V1_EXT_ODB_DEPTH * HNS_ROCE_V1_EXT_ODB_ENTRY) - -#define HNS_ROCE_V1_EXT_SDB_ALEPT HNS_ROCE_V1_DB_RSVD -#define HNS_ROCE_V1_EXT_SDB_ALFUL \ - (HNS_ROCE_V1_EXT_SDB_DEPTH - HNS_ROCE_V1_DB_RSVD) -#define HNS_ROCE_V1_EXT_ODB_ALEPT HNS_ROCE_V1_DB_RSVD -#define HNS_ROCE_V1_EXT_ODB_ALFUL \ - (HNS_ROCE_V1_EXT_ODB_DEPTH - HNS_ROCE_V1_DB_RSVD) - -#define HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS 50000 -#define HNS_ROCE_V1_RECREATE_LP_QP_TIMEOUT_MSECS 10000 -#define HNS_ROCE_V1_FREE_MR_WAIT_VALUE 5 -#define HNS_ROCE_V1_RECREATE_LP_QP_WAIT_VALUE 20 - -#define HNS_ROCE_BT_RSV_BUF_SIZE (1 << 17) - -#define HNS_ROCE_V1_TPTR_ENTRY_SIZE 2 -#define HNS_ROCE_V1_TPTR_BUF_SIZE \ - (HNS_ROCE_V1_TPTR_ENTRY_SIZE * HNS_ROCE_V1_MAX_CQ_NUM) - -#define HNS_ROCE_ODB_POLL_MODE 0 - -#define HNS_ROCE_SDB_NORMAL_MODE 0 -#define HNS_ROCE_SDB_EXTEND_MODE 1 - -#define HNS_ROCE_ODB_EXTEND_MODE 1 - -#define KEY_VALID 0x02 - -#define HNS_ROCE_CQE_QPN_MASK 0x3ffff -#define HNS_ROCE_CQE_STATUS_MASK 0x1f -#define HNS_ROCE_CQE_OPCODE_MASK 0xf - -#define HNS_ROCE_CQE_SUCCESS 0x00 -#define HNS_ROCE_CQE_SYNDROME_LOCAL_LENGTH_ERR 0x01 -#define HNS_ROCE_CQE_SYNDROME_LOCAL_QP_OP_ERR 0x02 -#define HNS_ROCE_CQE_SYNDROME_LOCAL_PROT_ERR 0x03 -#define HNS_ROCE_CQE_SYNDROME_WR_FLUSH_ERR 0x04 -#define HNS_ROCE_CQE_SYNDROME_MEM_MANAGE_OPERATE_ERR 0x05 -#define HNS_ROCE_CQE_SYNDROME_BAD_RESP_ERR 0x06 -#define HNS_ROCE_CQE_SYNDROME_LOCAL_ACCESS_ERR 0x07 -#define HNS_ROCE_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR 0x08 -#define HNS_ROCE_CQE_SYNDROME_REMOTE_ACCESS_ERR 0x09 -#define HNS_ROCE_CQE_SYNDROME_REMOTE_OP_ERR 0x0a -#define HNS_ROCE_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR 0x0b -#define HNS_ROCE_CQE_SYNDROME_RNR_RETRY_EXC_ERR 0x0c - -#define QP1C_CFGN_OFFSET 0x28 -#define PHY_PORT_OFFSET 0x8 -#define MTPT_IDX_SHIFT 16 -#define ALL_PORT_VAL_OPEN 0x3f -#define POL_TIME_INTERVAL_VAL 0x80 -#define SLEEP_TIME_INTERVAL 20 -#define SQ_PSN_SHIFT 8 -#define QKEY_VAL 0x80010000 -#define SDB_INV_CNT_OFFSET 8 - -#define HNS_ROCE_CEQ_DEFAULT_INTERVAL 0x10 -#define HNS_ROCE_CEQ_DEFAULT_BURST_NUM 0x10 - -#define HNS_ROCE_INT_MASK_DISABLE 0 -#define HNS_ROCE_INT_MASK_ENABLE 1 - -#define CEQ_REG_OFFSET 0x18 - -#define HNS_ROCE_CEQE_CEQE_COMP_OWNER_S 0 - -#define HNS_ROCE_V1_CONS_IDX_M GENMASK(15, 0) - -#define HNS_ROCE_CEQE_CEQE_COMP_CQN_S 16 -#define HNS_ROCE_CEQE_CEQE_COMP_CQN_M GENMASK(31, 16) - -#define HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S 16 -#define HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M GENMASK(23, 16) - -#define HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S 24 -#define HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M GENMASK(30, 24) - -#define HNS_ROCE_AEQE_U32_4_OWNER_S 31 - -#define HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S 0 -#define HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_M GENMASK(23, 0) - -#define HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_S 25 -#define HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_M GENMASK(27, 25) - -#define HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S 0 -#define HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_M GENMASK(15, 0) - -#define HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_S 0 -#define HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_M GENMASK(4, 0) - -/* Local Work Queue Catastrophic Error,SUBTYPE 0x5 */ -enum { - HNS_ROCE_LWQCE_QPC_ERROR = 1, - HNS_ROCE_LWQCE_MTU_ERROR, - HNS_ROCE_LWQCE_WQE_BA_ADDR_ERROR, - HNS_ROCE_LWQCE_WQE_ADDR_ERROR, - HNS_ROCE_LWQCE_SQ_WQE_SHIFT_ERROR, - HNS_ROCE_LWQCE_SL_ERROR, - HNS_ROCE_LWQCE_PORT_ERROR, -}; - -/* Local Access Violation Work Queue Error,SUBTYPE 0x7 */ -enum { - HNS_ROCE_LAVWQE_R_KEY_VIOLATION = 1, - HNS_ROCE_LAVWQE_LENGTH_ERROR, - HNS_ROCE_LAVWQE_VA_ERROR, - HNS_ROCE_LAVWQE_PD_ERROR, - HNS_ROCE_LAVWQE_RW_ACC_ERROR, - HNS_ROCE_LAVWQE_KEY_STATE_ERROR, - HNS_ROCE_LAVWQE_MR_OPERATION_ERROR, -}; - -/* DOORBELL overflow subtype */ -enum { - HNS_ROCE_DB_SUBTYPE_SDB_OVF = 1, - HNS_ROCE_DB_SUBTYPE_SDB_ALM_OVF, - HNS_ROCE_DB_SUBTYPE_ODB_OVF, - HNS_ROCE_DB_SUBTYPE_ODB_ALM_OVF, - HNS_ROCE_DB_SUBTYPE_SDB_ALM_EMP, - HNS_ROCE_DB_SUBTYPE_ODB_ALM_EMP, -}; - -enum { - /* RQ&SRQ related operations */ - HNS_ROCE_OPCODE_SEND_DATA_RECEIVE = 0x06, - HNS_ROCE_OPCODE_RDMA_WITH_IMM_RECEIVE, -}; - -enum { - HNS_ROCE_PORT_DOWN = 0, - HNS_ROCE_PORT_UP, -}; - -struct hns_roce_cq_context { - __le32 cqc_byte_4; - __le32 cq_bt_l; - __le32 cqc_byte_12; - __le32 cur_cqe_ba0_l; - __le32 cqc_byte_20; - __le32 cqe_tptr_addr_l; - __le32 cur_cqe_ba1_l; - __le32 cqc_byte_32; -}; - -#define CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_S 0 -#define CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_M \ - (((1UL << 2) - 1) << CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_S) - -#define CQ_CONTEXT_CQC_BYTE_4_CQN_S 16 -#define CQ_CONTEXT_CQC_BYTE_4_CQN_M \ - (((1UL << 16) - 1) << CQ_CONTEXT_CQC_BYTE_4_CQN_S) - -#define CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_S 0 -#define CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_M \ - (((1UL << 17) - 1) << CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_S) - -#define CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_S 20 -#define CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_M \ - (((1UL << 4) - 1) << CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_S) - -#define CQ_CONTEXT_CQC_BYTE_12_CEQN_S 24 -#define CQ_CONTEXT_CQC_BYTE_12_CEQN_M \ - (((1UL << 5) - 1) << CQ_CONTEXT_CQC_BYTE_12_CEQN_S) - -#define CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_S 0 -#define CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_M \ - (((1UL << 5) - 1) << CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_S) - -#define CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_S 16 -#define CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_M \ - (((1UL << 16) - 1) << CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_S) - -#define CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_S 8 -#define CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_M \ - (((1UL << 5) - 1) << CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_S) - -#define CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_S 0 -#define CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_M \ - (((1UL << 5) - 1) << CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_S) - -#define CQ_CONTEXT_CQC_BYTE_32_SE_FLAG_S 9 - -#define CQ_CONTEXT_CQC_BYTE_32_CE_FLAG_S 8 -#define CQ_CONTEXT_CQC_BYTE_32_NOTIFICATION_FLAG_S 14 -#define CQ_CQNTEXT_CQC_BYTE_32_TYPE_OF_COMPLETION_NOTIFICATION_S 15 - -#define CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_S 16 -#define CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_M \ - (((1UL << 16) - 1) << CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_S) - -struct hns_roce_cqe { - __le32 cqe_byte_4; - union { - __le32 r_key; - __le32 immediate_data; - }; - __le32 byte_cnt; - __le32 cqe_byte_16; - __le32 cqe_byte_20; - __le32 s_mac_l; - __le32 cqe_byte_28; - __le32 reserved; -}; - -#define CQE_BYTE_4_OWNER_S 7 -#define CQE_BYTE_4_SQ_RQ_FLAG_S 14 - -#define CQE_BYTE_4_STATUS_OF_THE_OPERATION_S 8 -#define CQE_BYTE_4_STATUS_OF_THE_OPERATION_M \ - (((1UL << 5) - 1) << CQE_BYTE_4_STATUS_OF_THE_OPERATION_S) - -#define CQE_BYTE_4_WQE_INDEX_S 16 -#define CQE_BYTE_4_WQE_INDEX_M (((1UL << 14) - 1) << CQE_BYTE_4_WQE_INDEX_S) - -#define CQE_BYTE_4_OPERATION_TYPE_S 0 -#define CQE_BYTE_4_OPERATION_TYPE_M \ - (((1UL << 4) - 1) << CQE_BYTE_4_OPERATION_TYPE_S) - -#define CQE_BYTE_4_IMM_INDICATOR_S 15 - -#define CQE_BYTE_16_LOCAL_QPN_S 0 -#define CQE_BYTE_16_LOCAL_QPN_M (((1UL << 24) - 1) << CQE_BYTE_16_LOCAL_QPN_S) - -#define CQE_BYTE_20_PORT_NUM_S 26 -#define CQE_BYTE_20_PORT_NUM_M (((1UL << 3) - 1) << CQE_BYTE_20_PORT_NUM_S) - -#define CQE_BYTE_20_SL_S 24 -#define CQE_BYTE_20_SL_M (((1UL << 2) - 1) << CQE_BYTE_20_SL_S) - -#define CQE_BYTE_20_REMOTE_QPN_S 0 -#define CQE_BYTE_20_REMOTE_QPN_M \ - (((1UL << 24) - 1) << CQE_BYTE_20_REMOTE_QPN_S) - -#define CQE_BYTE_20_GRH_PRESENT_S 29 - -#define CQE_BYTE_28_P_KEY_IDX_S 16 -#define CQE_BYTE_28_P_KEY_IDX_M (((1UL << 16) - 1) << CQE_BYTE_28_P_KEY_IDX_S) - -#define CQ_DB_REQ_NOT_SOL 0 -#define CQ_DB_REQ_NOT (1 << 16) - -struct hns_roce_v1_mpt_entry { - __le32 mpt_byte_4; - __le32 pbl_addr_l; - __le32 mpt_byte_12; - __le32 virt_addr_l; - __le32 virt_addr_h; - __le32 length; - __le32 mpt_byte_28; - __le32 pa0_l; - __le32 mpt_byte_36; - __le32 mpt_byte_40; - __le32 mpt_byte_44; - __le32 mpt_byte_48; - __le32 pa4_l; - __le32 mpt_byte_56; - __le32 mpt_byte_60; - __le32 mpt_byte_64; -}; - -#define MPT_BYTE_4_KEY_STATE_S 0 -#define MPT_BYTE_4_KEY_STATE_M (((1UL << 2) - 1) << MPT_BYTE_4_KEY_STATE_S) - -#define MPT_BYTE_4_KEY_S 8 -#define MPT_BYTE_4_KEY_M (((1UL << 8) - 1) << MPT_BYTE_4_KEY_S) - -#define MPT_BYTE_4_PAGE_SIZE_S 16 -#define MPT_BYTE_4_PAGE_SIZE_M (((1UL << 2) - 1) << MPT_BYTE_4_PAGE_SIZE_S) - -#define MPT_BYTE_4_MW_TYPE_S 20 - -#define MPT_BYTE_4_MW_BIND_ENABLE_S 21 - -#define MPT_BYTE_4_OWN_S 22 - -#define MPT_BYTE_4_MEMORY_LOCATION_TYPE_S 24 -#define MPT_BYTE_4_MEMORY_LOCATION_TYPE_M \ - (((1UL << 2) - 1) << MPT_BYTE_4_MEMORY_LOCATION_TYPE_S) - -#define MPT_BYTE_4_REMOTE_ATOMIC_S 26 -#define MPT_BYTE_4_LOCAL_WRITE_S 27 -#define MPT_BYTE_4_REMOTE_WRITE_S 28 -#define MPT_BYTE_4_REMOTE_READ_S 29 -#define MPT_BYTE_4_REMOTE_INVAL_ENABLE_S 30 -#define MPT_BYTE_4_ADDRESS_TYPE_S 31 - -#define MPT_BYTE_12_PBL_ADDR_H_S 0 -#define MPT_BYTE_12_PBL_ADDR_H_M \ - (((1UL << 17) - 1) << MPT_BYTE_12_PBL_ADDR_H_S) - -#define MPT_BYTE_12_MW_BIND_COUNTER_S 17 -#define MPT_BYTE_12_MW_BIND_COUNTER_M \ - (((1UL << 15) - 1) << MPT_BYTE_12_MW_BIND_COUNTER_S) - -#define MPT_BYTE_28_PD_S 0 -#define MPT_BYTE_28_PD_M (((1UL << 16) - 1) << MPT_BYTE_28_PD_S) - -#define MPT_BYTE_28_L_KEY_IDX_L_S 16 -#define MPT_BYTE_28_L_KEY_IDX_L_M \ - (((1UL << 16) - 1) << MPT_BYTE_28_L_KEY_IDX_L_S) - -#define MPT_BYTE_36_PA0_H_S 0 -#define MPT_BYTE_36_PA0_H_M (((1UL << 5) - 1) << MPT_BYTE_36_PA0_H_S) - -#define MPT_BYTE_36_PA1_L_S 8 -#define MPT_BYTE_36_PA1_L_M (((1UL << 24) - 1) << MPT_BYTE_36_PA1_L_S) - -#define MPT_BYTE_40_PA1_H_S 0 -#define MPT_BYTE_40_PA1_H_M (((1UL << 13) - 1) << MPT_BYTE_40_PA1_H_S) - -#define MPT_BYTE_40_PA2_L_S 16 -#define MPT_BYTE_40_PA2_L_M (((1UL << 16) - 1) << MPT_BYTE_40_PA2_L_S) - -#define MPT_BYTE_44_PA2_H_S 0 -#define MPT_BYTE_44_PA2_H_M (((1UL << 21) - 1) << MPT_BYTE_44_PA2_H_S) - -#define MPT_BYTE_44_PA3_L_S 24 -#define MPT_BYTE_44_PA3_L_M (((1UL << 8) - 1) << MPT_BYTE_44_PA3_L_S) - -#define MPT_BYTE_48_PA3_H_S 0 -#define MPT_BYTE_48_PA3_H_M (((1UL << 29) - 1) << MPT_BYTE_48_PA3_H_S) - -#define MPT_BYTE_56_PA4_H_S 0 -#define MPT_BYTE_56_PA4_H_M (((1UL << 5) - 1) << MPT_BYTE_56_PA4_H_S) - -#define MPT_BYTE_56_PA5_L_S 8 -#define MPT_BYTE_56_PA5_L_M (((1UL << 24) - 1) << MPT_BYTE_56_PA5_L_S) - -#define MPT_BYTE_60_PA5_H_S 0 -#define MPT_BYTE_60_PA5_H_M (((1UL << 13) - 1) << MPT_BYTE_60_PA5_H_S) - -#define MPT_BYTE_60_PA6_L_S 16 -#define MPT_BYTE_60_PA6_L_M (((1UL << 16) - 1) << MPT_BYTE_60_PA6_L_S) - -#define MPT_BYTE_64_PA6_H_S 0 -#define MPT_BYTE_64_PA6_H_M (((1UL << 21) - 1) << MPT_BYTE_64_PA6_H_S) - -#define MPT_BYTE_64_L_KEY_IDX_H_S 24 -#define MPT_BYTE_64_L_KEY_IDX_H_M \ - (((1UL << 8) - 1) << MPT_BYTE_64_L_KEY_IDX_H_S) - -struct hns_roce_wqe_ctrl_seg { - __le32 sgl_pa_h; - __le32 flag; - union { - __be32 imm_data; - __le32 inv_key; - }; - __le32 msg_length; -}; - -struct hns_roce_wqe_data_seg { - __le64 addr; - __le32 lkey; - __le32 len; -}; - -struct hns_roce_wqe_raddr_seg { - __le32 rkey; - __le32 len; /* reserved */ - __le64 raddr; -}; - -struct hns_roce_rq_wqe_ctrl { - __le32 rwqe_byte_4; - __le32 rocee_sgl_ba_l; - __le32 rwqe_byte_12; - __le32 reserved[5]; -}; - -#define RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_S 16 -#define RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_M \ - (((1UL << 6) - 1) << RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_S) - -#define HNS_ROCE_QP_DESTROY_TIMEOUT_MSECS 10000 - -#define GID_LEN 16 - -struct hns_roce_ud_send_wqe { - __le32 dmac_h; - __le32 u32_8; - __le32 immediate_data; - - __le32 u32_16; - union { - unsigned char dgid[GID_LEN]; - struct { - __le32 u32_20; - __le32 u32_24; - __le32 u32_28; - __le32 u32_32; - }; - }; - - __le32 u32_36; - __le32 u32_40; - - __le32 va0_l; - __le32 va0_h; - __le32 l_key0; - - __le32 va1_l; - __le32 va1_h; - __le32 l_key1; -}; - -#define UD_SEND_WQE_U32_4_DMAC_0_S 0 -#define UD_SEND_WQE_U32_4_DMAC_0_M \ - (((1UL << 8) - 1) << UD_SEND_WQE_U32_4_DMAC_0_S) - -#define UD_SEND_WQE_U32_4_DMAC_1_S 8 -#define UD_SEND_WQE_U32_4_DMAC_1_M \ - (((1UL << 8) - 1) << UD_SEND_WQE_U32_4_DMAC_1_S) - -#define UD_SEND_WQE_U32_4_DMAC_2_S 16 -#define UD_SEND_WQE_U32_4_DMAC_2_M \ - (((1UL << 8) - 1) << UD_SEND_WQE_U32_4_DMAC_2_S) - -#define UD_SEND_WQE_U32_4_DMAC_3_S 24 -#define UD_SEND_WQE_U32_4_DMAC_3_M \ - (((1UL << 8) - 1) << UD_SEND_WQE_U32_4_DMAC_3_S) - -#define UD_SEND_WQE_U32_8_DMAC_4_S 0 -#define UD_SEND_WQE_U32_8_DMAC_4_M \ - (((1UL << 8) - 1) << UD_SEND_WQE_U32_8_DMAC_4_S) - -#define UD_SEND_WQE_U32_8_DMAC_5_S 8 -#define UD_SEND_WQE_U32_8_DMAC_5_M \ - (((1UL << 8) - 1) << UD_SEND_WQE_U32_8_DMAC_5_S) - -#define UD_SEND_WQE_U32_8_LOOPBACK_INDICATOR_S 22 - -#define UD_SEND_WQE_U32_8_OPERATION_TYPE_S 16 -#define UD_SEND_WQE_U32_8_OPERATION_TYPE_M \ - (((1UL << 4) - 1) << UD_SEND_WQE_U32_8_OPERATION_TYPE_S) - -#define UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_S 24 -#define UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_M \ - (((1UL << 6) - 1) << UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_S) - -#define UD_SEND_WQE_U32_8_SEND_GL_ROUTING_HDR_FLAG_S 31 - -#define UD_SEND_WQE_U32_16_DEST_QP_S 0 -#define UD_SEND_WQE_U32_16_DEST_QP_M \ - (((1UL << 24) - 1) << UD_SEND_WQE_U32_16_DEST_QP_S) - -#define UD_SEND_WQE_U32_16_MAX_STATIC_RATE_S 24 -#define UD_SEND_WQE_U32_16_MAX_STATIC_RATE_M \ - (((1UL << 8) - 1) << UD_SEND_WQE_U32_16_MAX_STATIC_RATE_S) - -#define UD_SEND_WQE_U32_36_FLOW_LABEL_S 0 -#define UD_SEND_WQE_U32_36_FLOW_LABEL_M \ - (((1UL << 20) - 1) << UD_SEND_WQE_U32_36_FLOW_LABEL_S) - -#define UD_SEND_WQE_U32_36_PRIORITY_S 20 -#define UD_SEND_WQE_U32_36_PRIORITY_M \ - (((1UL << 4) - 1) << UD_SEND_WQE_U32_36_PRIORITY_S) - -#define UD_SEND_WQE_U32_36_SGID_INDEX_S 24 -#define UD_SEND_WQE_U32_36_SGID_INDEX_M \ - (((1UL << 8) - 1) << UD_SEND_WQE_U32_36_SGID_INDEX_S) - -#define UD_SEND_WQE_U32_40_HOP_LIMIT_S 0 -#define UD_SEND_WQE_U32_40_HOP_LIMIT_M \ - (((1UL << 8) - 1) << UD_SEND_WQE_U32_40_HOP_LIMIT_S) - -#define UD_SEND_WQE_U32_40_TRAFFIC_CLASS_S 8 -#define UD_SEND_WQE_U32_40_TRAFFIC_CLASS_M \ - (((1UL << 8) - 1) << UD_SEND_WQE_U32_40_TRAFFIC_CLASS_S) - -struct hns_roce_sqp_context { - __le32 qp1c_bytes_4; - __le32 sq_rq_bt_l; - __le32 qp1c_bytes_12; - __le32 qp1c_bytes_16; - __le32 qp1c_bytes_20; - __le32 cur_rq_wqe_ba_l; - __le32 qp1c_bytes_28; - __le32 qp1c_bytes_32; - __le32 cur_sq_wqe_ba_l; - __le32 qp1c_bytes_40; -}; - -#define QP1C_BYTES_4_QP_STATE_S 0 -#define QP1C_BYTES_4_QP_STATE_M \ - (((1UL << 3) - 1) << QP1C_BYTES_4_QP_STATE_S) - -#define QP1C_BYTES_4_SQ_WQE_SHIFT_S 8 -#define QP1C_BYTES_4_SQ_WQE_SHIFT_M \ - (((1UL << 4) - 1) << QP1C_BYTES_4_SQ_WQE_SHIFT_S) - -#define QP1C_BYTES_4_RQ_WQE_SHIFT_S 12 -#define QP1C_BYTES_4_RQ_WQE_SHIFT_M \ - (((1UL << 4) - 1) << QP1C_BYTES_4_RQ_WQE_SHIFT_S) - -#define QP1C_BYTES_4_PD_S 16 -#define QP1C_BYTES_4_PD_M (((1UL << 16) - 1) << QP1C_BYTES_4_PD_S) - -#define QP1C_BYTES_12_SQ_RQ_BT_H_S 0 -#define QP1C_BYTES_12_SQ_RQ_BT_H_M \ - (((1UL << 17) - 1) << QP1C_BYTES_12_SQ_RQ_BT_H_S) - -#define QP1C_BYTES_16_RQ_HEAD_S 0 -#define QP1C_BYTES_16_RQ_HEAD_M (((1UL << 15) - 1) << QP1C_BYTES_16_RQ_HEAD_S) - -#define QP1C_BYTES_16_PORT_NUM_S 16 -#define QP1C_BYTES_16_PORT_NUM_M \ - (((1UL << 3) - 1) << QP1C_BYTES_16_PORT_NUM_S) - -#define QP1C_BYTES_16_SIGNALING_TYPE_S 27 -#define QP1C_BYTES_16_LOCAL_ENABLE_E2E_CREDIT_S 28 -#define QP1C_BYTES_16_RQ_BA_FLG_S 29 -#define QP1C_BYTES_16_SQ_BA_FLG_S 30 -#define QP1C_BYTES_16_QP1_ERR_S 31 - -#define QP1C_BYTES_20_SQ_HEAD_S 0 -#define QP1C_BYTES_20_SQ_HEAD_M (((1UL << 15) - 1) << QP1C_BYTES_20_SQ_HEAD_S) - -#define QP1C_BYTES_20_PKEY_IDX_S 16 -#define QP1C_BYTES_20_PKEY_IDX_M \ - (((1UL << 16) - 1) << QP1C_BYTES_20_PKEY_IDX_S) - -#define QP1C_BYTES_28_CUR_RQ_WQE_BA_H_S 0 -#define QP1C_BYTES_28_CUR_RQ_WQE_BA_H_M \ - (((1UL << 5) - 1) << QP1C_BYTES_28_CUR_RQ_WQE_BA_H_S) - -#define QP1C_BYTES_28_RQ_CUR_IDX_S 16 -#define QP1C_BYTES_28_RQ_CUR_IDX_M \ - (((1UL << 15) - 1) << QP1C_BYTES_28_RQ_CUR_IDX_S) - -#define QP1C_BYTES_32_TX_CQ_NUM_S 0 -#define QP1C_BYTES_32_TX_CQ_NUM_M \ - (((1UL << 16) - 1) << QP1C_BYTES_32_TX_CQ_NUM_S) - -#define QP1C_BYTES_32_RX_CQ_NUM_S 16 -#define QP1C_BYTES_32_RX_CQ_NUM_M \ - (((1UL << 16) - 1) << QP1C_BYTES_32_RX_CQ_NUM_S) - -#define QP1C_BYTES_40_CUR_SQ_WQE_BA_H_S 0 -#define QP1C_BYTES_40_CUR_SQ_WQE_BA_H_M \ - (((1UL << 5) - 1) << QP1C_BYTES_40_CUR_SQ_WQE_BA_H_S) - -#define QP1C_BYTES_40_SQ_CUR_IDX_S 16 -#define QP1C_BYTES_40_SQ_CUR_IDX_M \ - (((1UL << 15) - 1) << QP1C_BYTES_40_SQ_CUR_IDX_S) - -#define HNS_ROCE_WQE_INLINE (1UL<<31) -#define HNS_ROCE_WQE_SE (1UL<<30) - -#define HNS_ROCE_WQE_SGE_NUM_BIT 24 -#define HNS_ROCE_WQE_IMM (1UL<<23) -#define HNS_ROCE_WQE_FENCE (1UL<<21) -#define HNS_ROCE_WQE_CQ_NOTIFY (1UL<<20) - -#define HNS_ROCE_WQE_OPCODE_SEND (0<<16) -#define HNS_ROCE_WQE_OPCODE_RDMA_READ (1<<16) -#define HNS_ROCE_WQE_OPCODE_RDMA_WRITE (2<<16) -#define HNS_ROCE_WQE_OPCODE_LOCAL_INV (4<<16) -#define HNS_ROCE_WQE_OPCODE_UD_SEND (7<<16) -#define HNS_ROCE_WQE_OPCODE_MASK (15<<16) - -struct hns_roce_qp_context { - __le32 qpc_bytes_4; - __le32 qpc_bytes_8; - __le32 qpc_bytes_12; - __le32 qpc_bytes_16; - __le32 sq_rq_bt_l; - __le32 qpc_bytes_24; - __le32 irrl_ba_l; - __le32 qpc_bytes_32; - __le32 qpc_bytes_36; - __le32 dmac_l; - __le32 qpc_bytes_44; - __le32 qpc_bytes_48; - u8 dgid[16]; - __le32 qpc_bytes_68; - __le32 cur_rq_wqe_ba_l; - __le32 qpc_bytes_76; - __le32 rx_rnr_time; - __le32 qpc_bytes_84; - __le32 qpc_bytes_88; - union { - __le32 rx_sge_len; - __le32 dma_length; - }; - union { - __le32 rx_sge_num; - __le32 rx_send_pktn; - __le32 r_key; - }; - __le32 va_l; - __le32 va_h; - __le32 qpc_bytes_108; - __le32 qpc_bytes_112; - __le32 rx_cur_sq_wqe_ba_l; - __le32 qpc_bytes_120; - __le32 qpc_bytes_124; - __le32 qpc_bytes_128; - __le32 qpc_bytes_132; - __le32 qpc_bytes_136; - __le32 qpc_bytes_140; - __le32 qpc_bytes_144; - __le32 qpc_bytes_148; - union { - __le32 rnr_retry; - __le32 ack_time; - }; - __le32 qpc_bytes_156; - __le32 pkt_use_len; - __le32 qpc_bytes_164; - __le32 qpc_bytes_168; - union { - __le32 sge_use_len; - __le32 pa_use_len; - }; - __le32 qpc_bytes_176; - __le32 qpc_bytes_180; - __le32 tx_cur_sq_wqe_ba_l; - __le32 qpc_bytes_188; - __le32 rvd21; -}; - -#define QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S 0 -#define QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M \ - (((1UL << 3) - 1) << QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S) - -#define QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S 3 -#define QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S 4 -#define QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S 5 -#define QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S 6 -#define QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S 7 - -#define QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S 8 -#define QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M \ - (((1UL << 4) - 1) << QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S) - -#define QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S 12 -#define QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M \ - (((1UL << 4) - 1) << QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S) - -#define QP_CONTEXT_QPC_BYTES_4_PD_S 16 -#define QP_CONTEXT_QPC_BYTES_4_PD_M \ - (((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_4_PD_S) - -#define QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S 0 -#define QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M \ - (((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S) - -#define QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S 16 -#define QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M \ - (((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S) - -#define QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S 0 -#define QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M \ - (((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S) - -#define QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S 16 -#define QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M \ - (((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S) - -#define QP_CONTEXT_QPC_BYTES_16_QP_NUM_S 0 -#define QP_CONTEXT_QPC_BYTES_16_QP_NUM_M \ - (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_16_QP_NUM_S) - -#define QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_S 0 -#define QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_M \ - (((1UL << 17) - 1) << QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_S) - -#define QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S 18 -#define QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M \ - (((1UL << 5) - 1) << QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S) - -#define QP_CONTEXT_QPC_BYTE_24_REMOTE_ENABLE_E2E_CREDITS_S 23 - -#define QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_S 0 -#define QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M \ - (((1UL << 17) - 1) << QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_S) - -#define QP_CONTEXT_QPC_BYTES_32_MIG_STATE_S 18 -#define QP_CONTEXT_QPC_BYTES_32_MIG_STATE_M \ - (((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_32_MIG_STATE_S) - -#define QP_CONTEXT_QPC_BYTE_32_LOCAL_ENABLE_E2E_CREDITS_S 20 -#define QP_CONTEXT_QPC_BYTE_32_SIGNALING_TYPE_S 21 -#define QP_CONTEXT_QPC_BYTE_32_LOOPBACK_INDICATOR_S 22 -#define QP_CONTEXT_QPC_BYTE_32_GLOBAL_HEADER_S 23 - -#define QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S 24 -#define QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M \ - (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S) - -#define QP_CONTEXT_QPC_BYTES_36_DEST_QP_S 0 -#define QP_CONTEXT_QPC_BYTES_36_DEST_QP_M \ - (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_36_DEST_QP_S) - -#define QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S 24 -#define QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M \ - (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S) - -#define QP_CONTEXT_QPC_BYTES_44_DMAC_H_S 0 -#define QP_CONTEXT_QPC_BYTES_44_DMAC_H_M \ - (((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_44_DMAC_H_S) - -#define QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_S 16 -#define QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_M \ - (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_S) - -#define QP_CONTEXT_QPC_BYTES_44_HOPLMT_S 24 -#define QP_CONTEXT_QPC_BYTES_44_HOPLMT_M \ - (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_44_HOPLMT_S) - -#define QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S 0 -#define QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M \ - (((1UL << 20) - 1) << QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S) - -#define QP_CONTEXT_QPC_BYTES_48_TCLASS_S 20 -#define QP_CONTEXT_QPC_BYTES_48_TCLASS_M \ - (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_48_TCLASS_S) - -#define QP_CONTEXT_QPC_BYTES_48_MTU_S 28 -#define QP_CONTEXT_QPC_BYTES_48_MTU_M \ - (((1UL << 4) - 1) << QP_CONTEXT_QPC_BYTES_48_MTU_S) - -#define QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_S 0 -#define QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_M \ - (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_S) - -#define QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_S 16 -#define QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_M \ - (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_S) - -#define QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_S 0 -#define QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_M \ - (((1UL << 5) - 1) << QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_S) - -#define QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_S 8 -#define QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_M \ - (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_S) - -#define QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_S 0 -#define QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_M \ - (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_S) - -#define QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_S 24 -#define QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_M \ - (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_S) - -#define QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S 0 -#define QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M \ - (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S) - -#define QP_CONTEXT_QPC_BYTES_88_RX_REQ_PSN_ERR_FLAG_S 24 -#define QP_CONTEXT_QPC_BYTES_88_RX_LAST_OPCODE_FLG_S 25 - -#define QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_S 26 -#define QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_M \ - (((1UL << 2) - 1) << \ - QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_S) - -#define QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_S 29 -#define QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_M \ - (((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_S) - -#define QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_S 0 -#define QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_M \ - (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_S) - -#define QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_FLG_S 24 -#define QP_CONTEXT_QPC_BYTES_108_TRRL_TDB_PSN_FLG_S 25 - -#define QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_S 0 -#define QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_M \ - (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_S) - -#define QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_S 24 -#define QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_M \ - (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_S) - -#define QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_S 0 -#define QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_M \ - (((1UL << 5) - 1) << QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_S) - -#define QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_S 0 -#define QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_M \ - (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_S) - -#define QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_S 16 -#define QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_M \ - (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_S) - -#define QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_S 0 -#define QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_M \ - (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_S) - -#define QP_CONTEXT_QPC_BYTES_128_RX_ACK_PSN_ERR_FLG_S 24 - -#define QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_S 25 -#define QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_M \ - (((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_S) - -#define QP_CONTEXT_QPC_BYTES_128_IRRL_PSN_VLD_FLG_S 27 - -#define QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_S 0 -#define QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_M \ - (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_S) - -#define QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_S 24 -#define QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_M \ - (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_S) - -#define QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_S 0 -#define QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_M \ - (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_S) - -#define QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_S 24 -#define QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_M \ - (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_S) - -#define QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_S 0 -#define QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_M \ - (((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_S) - -#define QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_S 16 -#define QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_M \ - (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_S) - -#define QP_CONTEXT_QPC_BYTES_140_RNR_RETRY_FLG_S 31 - -#define QP_CONTEXT_QPC_BYTES_144_QP_STATE_S 0 -#define QP_CONTEXT_QPC_BYTES_144_QP_STATE_M \ - (((1UL << 3) - 1) << QP_CONTEXT_QPC_BYTES_144_QP_STATE_S) - -#define QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_S 0 -#define QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_M \ - (((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_S) - -#define QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S 2 -#define QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M \ - (((1UL << 3) - 1) << QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S) - -#define QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_S 5 -#define QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_M \ - (((1UL << 3) - 1) << QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_S) - -#define QP_CONTEXT_QPC_BYTES_148_LSN_S 8 -#define QP_CONTEXT_QPC_BYTES_148_LSN_M \ - (((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_148_LSN_S) - -#define QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_S 0 -#define QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_M \ - (((1UL << 3) - 1) << QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_S) - -#define QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S 3 -#define QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M \ - (((1UL << 5) - 1) << QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S) - -#define QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_S 8 -#define QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_M \ - (((1UL << 3) - 1) << QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_S) - -#define QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S 11 -#define QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M \ - (((1UL << 3) - 1) << QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S) - -#define QP_CONTEXT_QPC_BYTES_156_SL_S 14 -#define QP_CONTEXT_QPC_BYTES_156_SL_M \ - (((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_156_SL_S) - -#define QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S 16 -#define QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M \ - (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S) - -#define QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_S 24 -#define QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_M \ - (((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_S) - -#define QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S 0 -#define QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M \ - (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S) - -#define QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_S 24 -#define QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_M \ - (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_S) - -#define QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_S 0 -#define QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_M \ - (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_S) - -#define QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_S 24 -#define QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_M \ - (((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_S) - -#define QP_CONTEXT_QPC_BYTES_168_DB_TYPE_S 26 -#define QP_CONTEXT_QPC_BYTES_168_DB_TYPE_M \ - (((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_168_DB_TYPE_S) - -#define QP_CONTEXT_QPC_BYTES_168_MSG_LP_IND_S 28 -#define QP_CONTEXT_QPC_BYTES_168_CSDB_LP_IND_S 29 -#define QP_CONTEXT_QPC_BYTES_168_QP_ERR_FLG_S 30 - -#define QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_S 0 -#define QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_M \ - (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_S) - -#define QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_S 16 -#define QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_M \ - (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_S) - -#define QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_S 0 -#define QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_M \ - (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_S) - -#define QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_S 16 -#define QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_M \ - (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_S) - -#define QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_S 0 -#define QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_M \ - (((1UL << 5) - 1) << QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_S) - -#define QP_CONTEXT_QPC_BYTES_188_PKT_RETRY_FLG_S 8 - -#define QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_S 16 -#define QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_M \ - (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_S) - -#define STATUS_MASK 0xff -#define GO_BIT_TIMEOUT_MSECS 10000 -#define HCR_STATUS_OFFSET 0x18 -#define HCR_GO_BIT 15 - -struct hns_roce_rq_db { - __le32 u32_4; - __le32 u32_8; -}; - -#define RQ_DOORBELL_U32_4_RQ_HEAD_S 0 -#define RQ_DOORBELL_U32_4_RQ_HEAD_M \ - (((1UL << 15) - 1) << RQ_DOORBELL_U32_4_RQ_HEAD_S) - -#define RQ_DOORBELL_U32_8_QPN_S 0 -#define RQ_DOORBELL_U32_8_QPN_M (((1UL << 24) - 1) << RQ_DOORBELL_U32_8_QPN_S) - -#define RQ_DOORBELL_U32_8_CMD_S 28 -#define RQ_DOORBELL_U32_8_CMD_M (((1UL << 3) - 1) << RQ_DOORBELL_U32_8_CMD_S) - -#define RQ_DOORBELL_U32_8_HW_SYNC_S 31 - -struct hns_roce_sq_db { - __le32 u32_4; - __le32 u32_8; -}; - -#define SQ_DOORBELL_U32_4_SQ_HEAD_S 0 -#define SQ_DOORBELL_U32_4_SQ_HEAD_M \ - (((1UL << 15) - 1) << SQ_DOORBELL_U32_4_SQ_HEAD_S) - -#define SQ_DOORBELL_U32_4_SL_S 16 -#define SQ_DOORBELL_U32_4_SL_M \ - (((1UL << 2) - 1) << SQ_DOORBELL_U32_4_SL_S) - -#define SQ_DOORBELL_U32_4_PORT_S 18 -#define SQ_DOORBELL_U32_4_PORT_M (((1UL << 3) - 1) << SQ_DOORBELL_U32_4_PORT_S) - -#define SQ_DOORBELL_U32_8_QPN_S 0 -#define SQ_DOORBELL_U32_8_QPN_M (((1UL << 24) - 1) << SQ_DOORBELL_U32_8_QPN_S) - -#define SQ_DOORBELL_HW_SYNC_S 31 - -struct hns_roce_ext_db { - int esdb_dep; - int eodb_dep; - struct hns_roce_buf_list *sdb_buf_list; - struct hns_roce_buf_list *odb_buf_list; -}; - -struct hns_roce_db_table { - int sdb_ext_mod; - int odb_ext_mod; - struct hns_roce_ext_db *ext_db; -}; - -#define HW_SYNC_SLEEP_TIME_INTERVAL 20 -#define HW_SYNC_TIMEOUT_MSECS (25 * HW_SYNC_SLEEP_TIME_INTERVAL) -#define BT_CMD_SYNC_SHIFT 31 -#define HNS_ROCE_BA_SIZE (32 * 4096) - -struct hns_roce_bt_table { - struct hns_roce_buf_list qpc_buf; - struct hns_roce_buf_list mtpt_buf; - struct hns_roce_buf_list cqc_buf; -}; - -struct hns_roce_tptr_table { - struct hns_roce_buf_list tptr_buf; -}; - -struct hns_roce_qp_work { - struct work_struct work; - struct ib_device *ib_dev; - struct hns_roce_qp *qp; - u32 db_wait_stage; - u32 sdb_issue_ptr; - u32 sdb_inv_cnt; - u32 sche_cnt; -}; - -struct hns_roce_mr_free_work { - struct work_struct work; - struct ib_device *ib_dev; - struct completion *comp; - int comp_flag; - void *mr; -}; - -struct hns_roce_recreate_lp_qp_work { - struct work_struct work; - struct ib_device *ib_dev; - struct completion *comp; - int comp_flag; -}; - -struct hns_roce_free_mr { - struct workqueue_struct *free_mr_wq; - struct hns_roce_qp *mr_free_qp[HNS_ROCE_V1_RESV_QP]; - struct hns_roce_cq *mr_free_cq; - struct hns_roce_pd *mr_free_pd; -}; - -struct hns_roce_v1_priv { - struct hns_roce_db_table db_table; - struct hns_roce_raq_table raq_table; - struct hns_roce_bt_table bt_table; - struct hns_roce_tptr_table tptr_table; - struct hns_roce_free_mr free_mr; -}; - -int hns_dsaf_roce_reset(struct fwnode_handle *dsaf_fwnode, bool dereset); -int hns_roce_v1_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); -int hns_roce_v1_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata); - -#endif diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index eb0defa80d0d..b33e948fd060 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -678,6 +678,7 @@ static void hns_roce_write512(struct hns_roce_dev *hr_dev, u64 *val, static void write_dwqe(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp, void *wqe) { +#define HNS_ROCE_SL_SHIFT 2 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe = wqe; /* All kinds of DirectWQE have the same header field layout */ @@ -685,7 +686,8 @@ static void write_dwqe(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp, roce_set_field(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_DB_SL_L_M, V2_RC_SEND_WQE_BYTE_4_DB_SL_L_S, qp->sl); roce_set_field(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_DB_SL_H_M, - V2_RC_SEND_WQE_BYTE_4_DB_SL_H_S, qp->sl >> 2); + V2_RC_SEND_WQE_BYTE_4_DB_SL_H_S, + qp->sl >> HNS_ROCE_SL_SHIFT); roce_set_field(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_WQE_INDEX_M, V2_RC_SEND_WQE_BYTE_4_WQE_INDEX_S, qp->sq.head); @@ -1305,14 +1307,14 @@ static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev, continue; dev_err_ratelimited(hr_dev->dev, - "Cmdq IO error, opcode = %x, return = %x\n", + "Cmdq IO error, opcode = 0x%x, return = 0x%x.\n", desc->opcode, desc_ret); ret = -EIO; } } else { /* FW/HW reset or incorrect number of desc */ tail = roce_read(hr_dev, ROCEE_TX_CMQ_CI_REG); - dev_warn(hr_dev->dev, "CMDQ move tail from %d to %d\n", + dev_warn(hr_dev->dev, "CMDQ move tail from %u to %u.\n", csq->head, tail); csq->head = tail; @@ -1571,7 +1573,7 @@ static int hns_roce_query_func_info(struct hns_roce_dev *hr_dev) struct hns_roce_cmq_desc desc; int ret; - if (hr_dev->pci_dev->revision < PCI_REVISION_ID_HIP09) { + if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) { hr_dev->func_num = 1; return 0; } @@ -2003,7 +2005,8 @@ static void set_default_caps(struct hns_roce_dev *hr_dev) caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM; if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) { - caps->flags |= HNS_ROCE_CAP_FLAG_STASH; + caps->flags |= HNS_ROCE_CAP_FLAG_STASH | + HNS_ROCE_CAP_FLAG_DIRECT_WQE; caps->max_sq_inline = HNS_ROCE_V3_MAX_SQ_INLINE; } else { caps->max_sq_inline = HNS_ROCE_V2_MAX_SQ_INLINE; @@ -2144,7 +2147,6 @@ static void apply_func_caps(struct hns_roce_dev *hr_dev) caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ; caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ; - caps->eqe_hop_num = HNS_ROCE_EQE_HOP_NUM; caps->pbl_hop_num = HNS_ROCE_PBL_HOP_NUM; caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0; caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0; @@ -2161,6 +2163,7 @@ static void apply_func_caps(struct hns_roce_dev *hr_dev) (u32)priv->handle->rinfo.num_vectors - 2); if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) { + caps->eqe_hop_num = HNS_ROCE_V3_EQE_HOP_NUM; caps->ceqe_size = HNS_ROCE_V3_EQE_SIZE; caps->aeqe_size = HNS_ROCE_V3_EQE_SIZE; @@ -2181,6 +2184,7 @@ static void apply_func_caps(struct hns_roce_dev *hr_dev) } else { u32 func_num = max_t(u32, 1, hr_dev->func_num); + caps->eqe_hop_num = HNS_ROCE_V2_EQE_HOP_NUM; caps->ceqe_size = HNS_ROCE_CEQE_SIZE; caps->aeqe_size = HNS_ROCE_AEQE_SIZE; caps->gid_table_len[0] /= func_num; @@ -2393,7 +2397,7 @@ static int hns_roce_config_entry_size(struct hns_roce_dev *hr_dev) struct hns_roce_caps *caps = &hr_dev->caps; int ret; - if (hr_dev->pci_dev->revision < PCI_REVISION_ID_HIP09) + if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) return 0; ret = config_hem_entry_size(hr_dev, HNS_ROCE_CFG_QPC_SIZE, @@ -2967,8 +2971,8 @@ static int config_gmv_table(struct hns_roce_dev *hr_dev, return hns_roce_cmq_send(hr_dev, desc, 2); } -static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, u32 port, - int gid_index, const union ib_gid *gid, +static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, int gid_index, + const union ib_gid *gid, const struct ib_gid_attr *attr) { enum hns_roce_sgid_type sgid_type = GID_TYPE_FLAG_ROCE_V1; @@ -3063,8 +3067,7 @@ static int set_mtpt_pbl(struct hns_roce_dev *hr_dev, } static int hns_roce_v2_write_mtpt(struct hns_roce_dev *hr_dev, - void *mb_buf, struct hns_roce_mr *mr, - unsigned long mtpt_idx) + void *mb_buf, struct hns_roce_mr *mr) { struct hns_roce_v2_mpt_entry *mpt_entry; int ret; @@ -4488,14 +4491,6 @@ static int modify_qp_rtr_to_rts(struct ib_qp *ibqp, return 0; } -static inline u16 get_udp_sport(u32 fl, u32 lqpn, u32 rqpn) -{ - if (!fl) - fl = rdma_calc_flow_label(lqpn, rqpn); - - return rdma_flow_label_to_udp_sport(fl); -} - static int get_dip_ctx_idx(struct ib_qp *ibqp, const struct ib_qp_attr *attr, u32 *dip_idx) { @@ -4712,8 +4707,9 @@ static int hns_roce_v2_set_path(struct ib_qp *ibqp, } hr_reg_write(context, QPC_UDPSPN, - is_udp ? get_udp_sport(grh->flow_label, ibqp->qp_num, - attr->dest_qp_num) : 0); + is_udp ? rdma_get_udp_sport(grh->flow_label, ibqp->qp_num, + attr->dest_qp_num) : + 0); hr_reg_clear(qpc_mask, QPC_UDPSPN); @@ -4739,7 +4735,7 @@ static int hns_roce_v2_set_path(struct ib_qp *ibqp, hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr); if (unlikely(hr_qp->sl > MAX_SERVICE_LEVEL)) { ibdev_err(ibdev, - "failed to fill QPC, sl (%d) shouldn't be larger than %d.\n", + "failed to fill QPC, sl (%u) shouldn't be larger than %d.\n", hr_qp->sl, MAX_SERVICE_LEVEL); return -EINVAL; } @@ -4768,7 +4764,8 @@ static bool check_qp_state(enum ib_qp_state cur_state, [IB_QPS_ERR] = true }, [IB_QPS_SQD] = {}, [IB_QPS_SQE] = {}, - [IB_QPS_ERR] = { [IB_QPS_RESET] = true, [IB_QPS_ERR] = true } + [IB_QPS_ERR] = { [IB_QPS_RESET] = true, + [IB_QPS_ERR] = true } }; return sm[cur_state][new_state]; @@ -5868,7 +5865,7 @@ static void hns_roce_v2_int_mask_enable(struct hns_roce_dev *hr_dev, roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG, enable_flag); } -static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev, int eqn) +static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev, u32 eqn) { struct device *dev = hr_dev->dev; int ret; @@ -5882,7 +5879,7 @@ static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev, int eqn) 0, HNS_ROCE_CMD_DESTROY_AEQC, HNS_ROCE_CMD_TIMEOUT_MSECS); if (ret) - dev_err(dev, "[mailbox cmd] destroy eqc(%d) failed.\n", eqn); + dev_err(dev, "[mailbox cmd] destroy eqc(%u) failed.\n", eqn); } static void free_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq) @@ -6394,7 +6391,7 @@ static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle) if (!id) return 0; - if (id->driver_data && handle->pdev->revision < PCI_REVISION_ID_HIP09) + if (id->driver_data && handle->pdev->revision == PCI_REVISION_ID_HIP08) return 0; ret = __hns_roce_hw_v2_init_instance(handle); diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h index 35c61da7ba15..12be85f0986e 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -35,26 +35,15 @@ #include -#define HNS_ROCE_VF_QPC_BT_NUM 256 -#define HNS_ROCE_VF_SCCC_BT_NUM 64 -#define HNS_ROCE_VF_SRQC_BT_NUM 64 -#define HNS_ROCE_VF_CQC_BT_NUM 64 -#define HNS_ROCE_VF_MPT_BT_NUM 64 -#define HNS_ROCE_VF_SMAC_NUM 32 -#define HNS_ROCE_VF_SL_NUM 8 -#define HNS_ROCE_VF_GMV_BT_NUM 256 - #define HNS_ROCE_V2_MAX_QP_NUM 0x1000 #define HNS_ROCE_V2_MAX_QPC_TIMER_NUM 0x200 #define HNS_ROCE_V2_MAX_WQE_NUM 0x8000 -#define HNS_ROCE_V2_MAX_SRQ 0x100000 #define HNS_ROCE_V2_MAX_SRQ_WR 0x8000 #define HNS_ROCE_V2_MAX_SRQ_SGE 64 #define HNS_ROCE_V2_MAX_CQ_NUM 0x100000 #define HNS_ROCE_V2_MAX_CQC_TIMER_NUM 0x100 #define HNS_ROCE_V2_MAX_SRQ_NUM 0x100000 #define HNS_ROCE_V2_MAX_CQE_NUM 0x400000 -#define HNS_ROCE_V2_MAX_SRQWQE_NUM 0x8000 #define HNS_ROCE_V2_MAX_RQ_SGE_NUM 64 #define HNS_ROCE_V2_MAX_SQ_SGE_NUM 64 #define HNS_ROCE_V2_MAX_EXTEND_SGE_NUM 0x200000 @@ -63,13 +52,10 @@ #define HNS_ROCE_V2_MAX_RC_INL_INN_SZ 32 #define HNS_ROCE_V2_UAR_NUM 256 #define HNS_ROCE_V2_PHY_UAR_NUM 1 -#define HNS_ROCE_V2_MAX_IRQ_NUM 65 -#define HNS_ROCE_V2_COMP_VEC_NUM 63 #define HNS_ROCE_V2_AEQE_VEC_NUM 1 #define HNS_ROCE_V2_ABNORMAL_VEC_NUM 1 #define HNS_ROCE_V2_MAX_MTPT_NUM 0x100000 #define HNS_ROCE_V2_MAX_MTT_SEGS 0x1000000 -#define HNS_ROCE_V2_MAX_CQE_SEGS 0x1000000 #define HNS_ROCE_V2_MAX_SRQWQE_SEGS 0x1000000 #define HNS_ROCE_V2_MAX_IDX_SEGS 0x1000000 #define HNS_ROCE_V2_MAX_PD_NUM 0x1000000 @@ -81,7 +67,6 @@ #define HNS_ROCE_V2_MAX_RQ_DESC_SZ 16 #define HNS_ROCE_V2_MAX_SRQ_DESC_SZ 64 #define HNS_ROCE_V2_IRRL_ENTRY_SZ 64 -#define HNS_ROCE_V2_TRRL_ENTRY_SZ 48 #define HNS_ROCE_V2_EXT_ATOMIC_TRRL_ENTRY_SZ 100 #define HNS_ROCE_V2_CQC_ENTRY_SZ 64 #define HNS_ROCE_V2_SRQC_ENTRY_SZ 64 @@ -103,7 +88,6 @@ #define HNS_ROCE_INVALID_LKEY 0x0 #define HNS_ROCE_INVALID_SGE_LENGTH 0x80000000 #define HNS_ROCE_CMQ_TX_TIMEOUT 30000 -#define HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE 2 #define HNS_ROCE_V2_RSV_QPS 8 #define HNS_ROCE_V2_HW_RST_TIMEOUT 1000 @@ -117,12 +101,14 @@ #define HNS_ROCE_CQE_HOP_NUM 1 #define HNS_ROCE_SRQWQE_HOP_NUM 1 #define HNS_ROCE_PBL_HOP_NUM 2 -#define HNS_ROCE_EQE_HOP_NUM 2 #define HNS_ROCE_IDX_HOP_NUM 1 #define HNS_ROCE_SQWQE_HOP_NUM 2 #define HNS_ROCE_EXT_SGE_HOP_NUM 1 #define HNS_ROCE_RQWQE_HOP_NUM 2 +#define HNS_ROCE_V2_EQE_HOP_NUM 2 +#define HNS_ROCE_V3_EQE_HOP_NUM 1 + #define HNS_ROCE_BA_PG_SZ_SUPPORTED_256K 6 #define HNS_ROCE_BA_PG_SZ_SUPPORTED_16K 2 #define HNS_ROCE_V2_GID_INDEX_NUM 16 @@ -1441,7 +1427,7 @@ struct hns_roce_v2_priv { struct hns_roce_dip { u8 dgid[GID_LEN_V2]; u32 dip_idx; - struct list_head node; /* all dips are on a list */ + struct list_head node; /* all dips are on a list */ }; /* only for RNR timeout issue of HIP08 */ diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index 4194b626f3c6..f73ba619f375 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -31,7 +31,6 @@ * SOFTWARE. */ #include -#include #include #include #include @@ -70,7 +69,7 @@ static int hns_roce_add_gid(const struct ib_gid_attr *attr, void **context) if (port >= hr_dev->caps.num_ports) return -EINVAL; - ret = hr_dev->hw->set_gid(hr_dev, port, attr->index, &attr->gid, attr); + ret = hr_dev->hw->set_gid(hr_dev, attr->index, &attr->gid, attr); return ret; } @@ -84,7 +83,7 @@ static int hns_roce_del_gid(const struct ib_gid_attr *attr, void **context) if (port >= hr_dev->caps.num_ports) return -EINVAL; - ret = hr_dev->hw->set_gid(hr_dev, port, attr->index, NULL, NULL); + ret = hr_dev->hw->set_gid(hr_dev, attr->index, NULL, NULL); return ret; } @@ -152,9 +151,6 @@ static int hns_roce_setup_mtu_mac(struct hns_roce_dev *hr_dev) u8 i; for (i = 0; i < hr_dev->caps.num_ports; i++) { - if (hr_dev->hw->set_mtu) - hr_dev->hw->set_mtu(hr_dev, hr_dev->iboe.phy_port[i], - hr_dev->caps.max_mtu); ret = hns_roce_set_mac(hr_dev, i, hr_dev->iboe.netdevs[i]->dev_addr); if (ret) @@ -270,6 +266,9 @@ static enum rdma_link_layer hns_roce_get_link_layer(struct ib_device *device, static int hns_roce_query_pkey(struct ib_device *ib_dev, u32 port, u16 index, u16 *pkey) { + if (index > 0) + return -EINVAL; + *pkey = PKEY_ID; return 0; @@ -307,9 +306,22 @@ hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address, entry->address = address; entry->mmap_type = mmap_type; - ret = rdma_user_mmap_entry_insert_exact( - ucontext, &entry->rdma_entry, length, - mmap_type == HNS_ROCE_MMAP_TYPE_DB ? 0 : 1); + switch (mmap_type) { + /* pgoff 0 must be used by DB for compatibility */ + case HNS_ROCE_MMAP_TYPE_DB: + ret = rdma_user_mmap_entry_insert_exact( + ucontext, &entry->rdma_entry, length, 0); + break; + case HNS_ROCE_MMAP_TYPE_DWQE: + ret = rdma_user_mmap_entry_insert_range( + ucontext, &entry->rdma_entry, length, 1, + U32_MAX); + break; + default: + ret = -EINVAL; + break; + } + if (ret) { kfree(entry); return NULL; @@ -323,18 +335,12 @@ static void hns_roce_dealloc_uar_entry(struct hns_roce_ucontext *context) if (context->db_mmap_entry) rdma_user_mmap_entry_remove( &context->db_mmap_entry->rdma_entry); - - if (context->tptr_mmap_entry) - rdma_user_mmap_entry_remove( - &context->tptr_mmap_entry->rdma_entry); } static int hns_roce_alloc_uar_entry(struct ib_ucontext *uctx) { struct hns_roce_ucontext *context = to_hr_ucontext(uctx); - struct hns_roce_dev *hr_dev = to_hr_dev(uctx->device); u64 address; - int ret; address = context->uar.pfn << PAGE_SHIFT; context->db_mmap_entry = hns_roce_user_mmap_entry_insert( @@ -342,27 +348,7 @@ static int hns_roce_alloc_uar_entry(struct ib_ucontext *uctx) if (!context->db_mmap_entry) return -ENOMEM; - if (!hr_dev->tptr_dma_addr || !hr_dev->tptr_size) - return 0; - - /* - * FIXME: using io_remap_pfn_range on the dma address returned - * by dma_alloc_coherent is totally wrong. - */ - context->tptr_mmap_entry = - hns_roce_user_mmap_entry_insert(uctx, hr_dev->tptr_dma_addr, - hr_dev->tptr_size, - HNS_ROCE_MMAP_TYPE_TPTR); - if (!context->tptr_mmap_entry) { - ret = -ENOMEM; - goto err; - } - return 0; - -err: - hns_roce_dealloc_uar_entry(context); - return ret; } static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx, @@ -436,10 +422,15 @@ static int hns_roce_mmap(struct ib_ucontext *uctx, struct vm_area_struct *vma) entry = to_hns_mmap(rdma_entry); pfn = entry->address >> PAGE_SHIFT; - prot = vma->vm_page_prot; - if (entry->mmap_type != HNS_ROCE_MMAP_TYPE_TPTR) - prot = pgprot_noncached(prot); + switch (entry->mmap_type) { + case HNS_ROCE_MMAP_TYPE_DB: + case HNS_ROCE_MMAP_TYPE_DWQE: + prot = pgprot_device(vma->vm_page_prot); + break; + default: + return -EINVAL; + } ret = rdma_user_mmap_io(uctx, vma, pfn, rdma_entry->npages * PAGE_SIZE, prot, rdma_entry); @@ -816,7 +807,6 @@ static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev) int ret; spin_lock_init(&hr_dev->sm_lock); - spin_lock_init(&hr_dev->bt_cmd_lock); if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQ_RECORD_DB || hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) { @@ -907,20 +897,13 @@ int hns_roce_init(struct hns_roce_dev *hr_dev) struct device *dev = hr_dev->dev; int ret; - if (hr_dev->hw->reset) { - ret = hr_dev->hw->reset(hr_dev, true); - if (ret) { - dev_err(dev, "Reset RoCE engine failed!\n"); - return ret; - } - } hr_dev->is_reset = false; if (hr_dev->hw->cmq_init) { ret = hr_dev->hw->cmq_init(hr_dev); if (ret) { dev_err(dev, "Init RoCE Command Queue failed!\n"); - goto error_failed_cmq_init; + return ret; } } @@ -1003,12 +986,6 @@ error_failed_cmd_init: if (hr_dev->hw->cmq_exit) hr_dev->hw->cmq_exit(hr_dev); -error_failed_cmq_init: - if (hr_dev->hw->reset) { - if (hr_dev->hw->reset(hr_dev, false)) - dev_err(dev, "Dereset RoCE engine failed!\n"); - } - return ret; } @@ -1028,8 +1005,6 @@ void hns_roce_exit(struct hns_roce_dev *hr_dev) hns_roce_cmd_cleanup(hr_dev); if (hr_dev->hw->cmq_exit) hr_dev->hw->cmq_exit(hr_dev); - if (hr_dev->hw->reset) - hr_dev->hw->reset(hr_dev, false); } MODULE_LICENSE("Dual BSD/GPL"); diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c index 7089ac780291..2ee06b906b60 100644 --- a/drivers/infiniband/hw/hns/hns_roce_mr.c +++ b/drivers/infiniband/hw/hns/hns_roce_mr.c @@ -31,7 +31,6 @@ * SOFTWARE. */ -#include #include #include #include "hns_roce_device.h" @@ -81,7 +80,7 @@ static int alloc_mr_key(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr) return -ENOMEM; } - mr->key = hw_index_to_key(id); /* MR key */ + mr->key = hw_index_to_key(id); /* MR key */ err = hns_roce_table_get(hr_dev, &hr_dev->mr_table.mtpt_table, (unsigned long)id); @@ -173,8 +172,7 @@ static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev, } if (mr->type != MR_TYPE_FRMR) - ret = hr_dev->hw->write_mtpt(hr_dev, mailbox->buf, mr, - mtpt_idx); + ret = hr_dev->hw->write_mtpt(hr_dev, mailbox->buf, mr); else ret = hr_dev->hw->frmr_write_mtpt(hr_dev, mailbox->buf, mr); if (ret) { @@ -363,12 +361,8 @@ int hns_roce_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) struct hns_roce_mr *mr = to_hr_mr(ibmr); int ret = 0; - if (hr_dev->hw->dereg_mr) { - ret = hr_dev->hw->dereg_mr(hr_dev, mr, udata); - } else { - hns_roce_mr_free(hr_dev, mr); - kfree(mr); - } + hns_roce_mr_free(hr_dev, mr); + kfree(mr); return ret; } @@ -614,10 +608,7 @@ static int mtr_map_region(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, return -ENOBUFS; for (i = 0; i < count && npage < max_count; i++) { - if (hr_dev->hw_rev == HNS_ROCE_HW_VER1) - addr = to_hr_hw_page_addr(pages[npage]); - else - addr = pages[npage]; + addr = pages[npage]; mtts[i] = cpu_to_le64(addr); npage++; @@ -824,11 +815,11 @@ int hns_roce_mtr_map(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, } int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, - int offset, u64 *mtt_buf, int mtt_max, u64 *base_addr) + u32 offset, u64 *mtt_buf, int mtt_max, u64 *base_addr) { struct hns_roce_hem_cfg *cfg = &mtr->hem_cfg; int mtt_count, left; - int start_index; + u32 start_index; int total = 0; __le64 *mtts; u32 npage; @@ -847,10 +838,7 @@ int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, continue; addr = cfg->root_ba + (npage << HNS_HW_PAGE_SHIFT); - if (hr_dev->hw_rev == HNS_ROCE_HW_VER1) - mtt_buf[total] = to_hr_hw_page_addr(addr); - else - mtt_buf[total] = addr; + mtt_buf[total] = addr; total++; } @@ -884,10 +872,10 @@ done: static int mtr_init_buf_cfg(struct hns_roce_dev *hr_dev, struct hns_roce_buf_attr *attr, struct hns_roce_hem_cfg *cfg, - unsigned int *buf_page_shift, int unalinged_size) + unsigned int *buf_page_shift, u64 unalinged_size) { struct hns_roce_buf_region *r; - int first_region_padding; + u64 first_region_padding; int page_cnt, region_cnt; unsigned int page_shift; size_t buf_size; diff --git a/drivers/infiniband/hw/hns/hns_roce_pd.c b/drivers/infiniband/hw/hns/hns_roce_pd.c index 81ffad77ae42..783e71852c50 100644 --- a/drivers/infiniband/hw/hns/hns_roce_pd.c +++ b/drivers/infiniband/hw/hns/hns_roce_pd.c @@ -30,7 +30,6 @@ * SOFTWARE. */ -#include #include #include "hns_roce_device.h" @@ -86,7 +85,6 @@ int hns_roce_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) int hns_roce_uar_alloc(struct hns_roce_dev *hr_dev, struct hns_roce_uar *uar) { struct hns_roce_ida *uar_ida = &hr_dev->uar_ida; - struct resource *res; int id; /* Using bitmap to manager UAR index */ @@ -104,18 +102,9 @@ int hns_roce_uar_alloc(struct hns_roce_dev *hr_dev, struct hns_roce_uar *uar) else uar->index = 0; - if (!dev_is_pci(hr_dev->dev)) { - res = platform_get_resource(hr_dev->pdev, IORESOURCE_MEM, 0); - if (!res) { - ida_free(&uar_ida->ida, id); - dev_err(&hr_dev->pdev->dev, "memory resource not found!\n"); - return -EINVAL; - } - uar->pfn = ((res->start) >> PAGE_SHIFT) + uar->index; - } else { - uar->pfn = ((pci_resource_start(hr_dev->pci_dev, 2)) - >> PAGE_SHIFT); - } + uar->pfn = ((pci_resource_start(hr_dev->pci_dev, 2)) >> PAGE_SHIFT); + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_DIRECT_WQE) + hr_dev->dwqe_page = pci_resource_start(hr_dev->pci_dev, 4); return 0; } diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index 9af4509894e6..d78373e10aab 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -32,7 +32,6 @@ */ #include -#include #include #include #include @@ -110,12 +109,11 @@ void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type) return; } - if (hr_dev->hw_rev != HNS_ROCE_HW_VER1 && - (event_type == HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR || - event_type == HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR || - event_type == HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR || - event_type == HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION || - event_type == HNS_ROCE_EVENT_TYPE_INVALID_XRCETH)) { + if (event_type == HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR || + event_type == HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR || + event_type == HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR || + event_type == HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION || + event_type == HNS_ROCE_EVENT_TYPE_INVALID_XRCETH) { qp->state = IB_QPS_ERR; flush_cqe(hr_dev, qp); @@ -219,13 +217,7 @@ static int alloc_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) int ret; if (hr_qp->ibqp.qp_type == IB_QPT_GSI) { - /* when hw version is v1, the sqpn is allocated */ - if (hr_dev->hw_rev == HNS_ROCE_HW_VER1) - num = HNS_ROCE_MAX_PORTS + - hr_dev->iboe.phy_port[hr_qp->port]; - else - num = 1; - + num = 1; hr_qp->doorbell_qpn = 1; } else { mutex_lock(&qp_table->bank_mutex); @@ -324,11 +316,6 @@ static int alloc_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) if (!hr_qp->qpn) return -EINVAL; - /* In v1 engine, GSI QP context is saved in the RoCE hw's register */ - if (hr_qp->ibqp.qp_type == IB_QPT_GSI && - hr_dev->hw_rev == HNS_ROCE_HW_VER1) - return 0; - /* Alloc memory for QPC */ ret = hns_roce_table_get(hr_dev, &qp_table->qp_table, hr_qp->qpn); if (ret) { @@ -379,6 +366,11 @@ err_out: return ret; } +static void qp_user_mmap_entry_remove(struct hns_roce_qp *hr_qp) +{ + rdma_user_mmap_entry_remove(&hr_qp->dwqe_mmap_entry->rdma_entry); +} + void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) { struct xarray *xa = &hr_dev->qp_table_xa; @@ -402,11 +394,6 @@ static void free_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) { struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; - /* In v1 engine, GSI QP context is saved in the RoCE hw's register */ - if (hr_qp->ibqp.qp_type == IB_QPT_GSI && - hr_dev->hw_rev == HNS_ROCE_HW_VER1) - return; - if (hr_dev->caps.trrl_entry_sz) hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn); hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn); @@ -535,11 +522,6 @@ static void set_ext_sge_param(struct hns_roce_dev *hr_dev, u32 sq_wqe_cnt, hr_qp->sge.sge_shift = HNS_ROCE_SGE_SHIFT; - if (hr_dev->hw_rev == HNS_ROCE_HW_VER1) { - hr_qp->sq.max_gs = HNS_ROCE_SGE_IN_WQE; - return; - } - hr_qp->sq.max_gs = max(1U, cap->max_send_sge); wqe_sge_cnt = get_wqe_ext_sge_cnt(hr_qp); @@ -780,7 +762,11 @@ static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, goto err_inline; } + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_DIRECT_WQE) + hr_qp->en_flags |= HNS_ROCE_QP_CAP_DIRECT_WQE; + return 0; + err_inline: free_rq_inline_buf(hr_qp); @@ -822,6 +808,35 @@ static inline bool kernel_qp_has_rdb(struct hns_roce_dev *hr_dev, hns_roce_qp_has_rq(init_attr)); } +static int qp_mmap_entry(struct hns_roce_qp *hr_qp, + struct hns_roce_dev *hr_dev, + struct ib_udata *udata, + struct hns_roce_ib_create_qp_resp *resp) +{ + struct hns_roce_ucontext *uctx = + rdma_udata_to_drv_context(udata, + struct hns_roce_ucontext, ibucontext); + struct rdma_user_mmap_entry *rdma_entry; + u64 address; + + address = hr_dev->dwqe_page + hr_qp->qpn * HNS_ROCE_DWQE_SIZE; + + hr_qp->dwqe_mmap_entry = + hns_roce_user_mmap_entry_insert(&uctx->ibucontext, address, + HNS_ROCE_DWQE_SIZE, + HNS_ROCE_MMAP_TYPE_DWQE); + + if (!hr_qp->dwqe_mmap_entry) { + ibdev_err(&hr_dev->ib_dev, "failed to get dwqe mmap entry.\n"); + return -ENOMEM; + } + + rdma_entry = &hr_qp->dwqe_mmap_entry->rdma_entry; + resp->dwqe_mmap_key = rdma_user_mmap_get_offset(rdma_entry); + + return 0; +} + static int alloc_user_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, struct ib_qp_init_attr *init_attr, @@ -909,10 +924,16 @@ static int alloc_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, hr_qp->en_flags |= HNS_ROCE_QP_CAP_OWNER_DB; if (udata) { + if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE) { + ret = qp_mmap_entry(hr_qp, hr_dev, udata, resp); + if (ret) + return ret; + } + ret = alloc_user_qp_db(hr_dev, hr_qp, init_attr, udata, ucmd, resp); if (ret) - return ret; + goto err_remove_qp; } else { ret = alloc_kernel_qp_db(hr_dev, hr_qp, init_attr); if (ret) @@ -920,6 +941,12 @@ static int alloc_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, } return 0; + +err_remove_qp: + if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE) + qp_user_mmap_entry_remove(hr_qp); + + return ret; } static void free_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, @@ -933,6 +960,8 @@ static void free_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, hns_roce_db_unmap_user(uctx, &hr_qp->rdb); if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) hns_roce_db_unmap_user(uctx, &hr_qp->sdb); + if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE) + qp_user_mmap_entry_remove(hr_qp); } else { if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) hns_roce_free_db(hr_dev, &hr_qp->rdb); @@ -1158,7 +1187,7 @@ static int check_qp_type(struct hns_roce_dev *hr_dev, enum ib_qp_type type, goto out; break; case IB_QPT_UD: - if (hr_dev->pci_dev->revision <= PCI_REVISION_ID_HIP08 && + if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 && is_user) goto out; break; @@ -1391,7 +1420,7 @@ void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq, } } -static inline void *get_wqe(struct hns_roce_qp *hr_qp, int offset) +static inline void *get_wqe(struct hns_roce_qp *hr_qp, u32 offset) { return hns_roce_buf_offset(hr_qp->mtr.kmem, offset); } diff --git a/drivers/infiniband/hw/irdma/hw.c b/drivers/infiniband/hw/irdma/hw.c index b4c657f5f2f9..4b1b16e7a75b 100644 --- a/drivers/infiniband/hw/irdma/hw.c +++ b/drivers/infiniband/hw/irdma/hw.c @@ -550,7 +550,7 @@ static void irdma_destroy_irq(struct irdma_pci_f *rf, struct irdma_sc_dev *dev = &rf->sc_dev; dev->irq_ops->irdma_dis_irq(dev, msix_vec->idx); - irq_set_affinity_hint(msix_vec->irq, NULL); + irq_update_affinity_hint(msix_vec->irq, NULL); free_irq(msix_vec->irq, dev_id); } @@ -1100,7 +1100,7 @@ irdma_cfg_ceq_vector(struct irdma_pci_f *rf, struct irdma_ceq *iwceq, } cpumask_clear(&msix_vec->mask); cpumask_set_cpu(msix_vec->cpu_affinity, &msix_vec->mask); - irq_set_affinity_hint(msix_vec->irq, &msix_vec->mask); + irq_update_affinity_hint(msix_vec->irq, &msix_vec->mask); if (status) { ibdev_dbg(&rf->iwdev->ibdev, "ERR: ceq irq config fail\n"); return IRDMA_ERR_CFG; diff --git a/drivers/infiniband/hw/irdma/i40iw_if.c b/drivers/infiniband/hw/irdma/i40iw_if.c index d219f64b2c3d..43e962b97d6a 100644 --- a/drivers/infiniband/hw/irdma/i40iw_if.c +++ b/drivers/infiniband/hw/irdma/i40iw_if.c @@ -198,7 +198,7 @@ static void i40iw_remove(struct auxiliary_device *aux_dev) aux_dev); struct i40e_info *cdev_info = i40e_adev->ldev; - return i40e_client_device_unregister(cdev_info); + i40e_client_device_unregister(cdev_info); } static const struct auxiliary_device_id i40iw_auxiliary_id_table[] = { diff --git a/drivers/infiniband/hw/irdma/pble.h b/drivers/infiniband/hw/irdma/pble.h index aa20827dcc9d..d0d4f2b77d34 100644 --- a/drivers/infiniband/hw/irdma/pble.h +++ b/drivers/infiniband/hw/irdma/pble.h @@ -69,7 +69,7 @@ struct irdma_add_page_info { struct irdma_chunk { struct list_head list; struct irdma_dma_info dmainfo; - void *bitmapbuf; + unsigned long *bitmapbuf; u32 sizeofbitmap; u64 size; diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index 8cd5f9261692..460e757d3fe6 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -21,7 +21,8 @@ static int irdma_query_device(struct ib_device *ibdev, return -EINVAL; memset(props, 0, sizeof(*props)); - ether_addr_copy((u8 *)&props->sys_image_guid, iwdev->netdev->dev_addr); + addrconf_addr_eui48((u8 *)&props->sys_image_guid, + iwdev->netdev->dev_addr); props->fw_ver = (u64)irdma_fw_major_ver(&rf->sc_dev) << 32 | irdma_fw_minor_ver(&rf->sc_dev); props->device_cap_flags = iwdev->device_cap_flags; @@ -1170,6 +1171,10 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, udp_info->ttl = attr->ah_attr.grh.hop_limit; udp_info->flow_label = attr->ah_attr.grh.flow_label; udp_info->tos = attr->ah_attr.grh.traffic_class; + udp_info->src_port = + rdma_get_udp_sport(udp_info->flow_label, + ibqp->qp_num, + roce_info->dest_qp); irdma_qp_rem_qos(&iwqp->sc_qp); dev->ws_remove(iwqp->sc_qp.vsi, ctx_info->user_pri); ctx_info->user_pri = rt_tos2priority(udp_info->tos); @@ -4321,24 +4326,6 @@ static enum rdma_link_layer irdma_get_link_layer(struct ib_device *ibdev, return IB_LINK_LAYER_ETHERNET; } -static __be64 irdma_mac_to_guid(struct net_device *ndev) -{ - const unsigned char *mac = ndev->dev_addr; - __be64 guid; - unsigned char *dst = (unsigned char *)&guid; - - dst[0] = mac[0] ^ 2; - dst[1] = mac[1]; - dst[2] = mac[2]; - dst[3] = 0xff; - dst[4] = 0xfe; - dst[5] = mac[3]; - dst[6] = mac[4]; - dst[7] = mac[5]; - - return guid; -} - static const struct ib_device_ops irdma_roce_dev_ops = { .attach_mcast = irdma_attach_mcast, .create_ah = irdma_create_ah, @@ -4408,7 +4395,8 @@ static const struct ib_device_ops irdma_dev_ops = { static void irdma_init_roce_device(struct irdma_device *iwdev) { iwdev->ibdev.node_type = RDMA_NODE_IB_CA; - iwdev->ibdev.node_guid = irdma_mac_to_guid(iwdev->netdev); + addrconf_addr_eui48((u8 *)&iwdev->ibdev.node_guid, + iwdev->netdev->dev_addr); ib_set_device_ops(&iwdev->ibdev, &irdma_roce_dev_ops); } @@ -4421,7 +4409,8 @@ static int irdma_init_iw_device(struct irdma_device *iwdev) struct net_device *netdev = iwdev->netdev; iwdev->ibdev.node_type = RDMA_NODE_RNIC; - ether_addr_copy((u8 *)&iwdev->ibdev.node_guid, netdev->dev_addr); + addrconf_addr_eui48((u8 *)&iwdev->ibdev.node_guid, + netdev->dev_addr); iwdev->ibdev.ops.iw_add_ref = irdma_qp_add_ref; iwdev->ibdev.ops.iw_rem_ref = irdma_qp_rem_ref; iwdev->ibdev.ops.iw_get_qp = irdma_get_qp; diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c index 0d2fa3338784..1c3d97229988 100644 --- a/drivers/infiniband/hw/mlx4/main.c +++ b/drivers/infiniband/hw/mlx4/main.c @@ -85,14 +85,6 @@ static enum rdma_link_layer mlx4_ib_port_link_layer(struct ib_device *device, static struct workqueue_struct *wq; -static void init_query_mad(struct ib_smp *mad) -{ - mad->base_version = 1; - mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED; - mad->class_version = 1; - mad->method = IB_MGMT_METHOD_GET; -} - static int check_flow_steering_support(struct mlx4_dev *dev) { int eth_num_ports = 0; @@ -471,7 +463,7 @@ static int mlx4_ib_query_device(struct ib_device *ibdev, if (!in_mad || !out_mad) goto out; - init_query_mad(in_mad); + ib_init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_NODE_INFO; err = mlx4_MAD_IFC(to_mdev(ibdev), MLX4_MAD_IFC_IGNORE_KEYS, @@ -669,7 +661,7 @@ static int ib_link_query_port(struct ib_device *ibdev, u32 port, if (!in_mad || !out_mad) goto out; - init_query_mad(in_mad); + ib_init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_PORT_INFO; in_mad->attr_mod = cpu_to_be32(port); @@ -721,7 +713,7 @@ static int ib_link_query_port(struct ib_device *ibdev, u32 port, /* If reported active speed is QDR, check if is FDR-10 */ if (props->active_speed == IB_SPEED_QDR) { - init_query_mad(in_mad); + ib_init_query_mad(in_mad); in_mad->attr_id = MLX4_ATTR_EXTENDED_PORT_INFO; in_mad->attr_mod = cpu_to_be32(port); @@ -848,7 +840,7 @@ int __mlx4_ib_query_gid(struct ib_device *ibdev, u32 port, int index, if (!in_mad || !out_mad) goto out; - init_query_mad(in_mad); + ib_init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_PORT_INFO; in_mad->attr_mod = cpu_to_be32(port); @@ -870,7 +862,7 @@ int __mlx4_ib_query_gid(struct ib_device *ibdev, u32 port, int index, } } - init_query_mad(in_mad); + ib_init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_GUID_INFO; in_mad->attr_mod = cpu_to_be32(index / 8); @@ -917,7 +909,7 @@ static int mlx4_ib_query_sl2vl(struct ib_device *ibdev, u32 port, if (!in_mad || !out_mad) goto out; - init_query_mad(in_mad); + ib_init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_SL_TO_VL_TABLE; in_mad->attr_mod = 0; @@ -971,7 +963,7 @@ int __mlx4_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index, if (!in_mad || !out_mad) goto out; - init_query_mad(in_mad); + ib_init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_PKEY_TABLE; in_mad->attr_mod = cpu_to_be32(index / 32); @@ -1990,7 +1982,7 @@ static int init_node_data(struct mlx4_ib_dev *dev) if (!in_mad || !out_mad) goto out; - init_query_mad(in_mad); + ib_init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_NODE_DESC; if (mlx4_is_master(dev->dev)) mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW; @@ -2784,10 +2776,8 @@ static void *mlx4_ib_add(struct mlx4_dev *dev) if (err) goto err_counter; - ibdev->ib_uc_qpns_bitmap = - kmalloc_array(BITS_TO_LONGS(ibdev->steer_qpn_count), - sizeof(long), - GFP_KERNEL); + ibdev->ib_uc_qpns_bitmap = bitmap_alloc(ibdev->steer_qpn_count, + GFP_KERNEL); if (!ibdev->ib_uc_qpns_bitmap) goto err_steer_qp_release; @@ -2875,7 +2865,7 @@ err_diag_counters: mlx4_ib_diag_cleanup(ibdev); err_steer_free_bitmap: - kfree(ibdev->ib_uc_qpns_bitmap); + bitmap_free(ibdev->ib_uc_qpns_bitmap); err_steer_qp_release: mlx4_qp_release_range(dev, ibdev->steer_qpn_base, @@ -2988,7 +2978,7 @@ static void mlx4_ib_remove(struct mlx4_dev *dev, void *ibdev_ptr) mlx4_qp_release_range(dev, ibdev->steer_qpn_base, ibdev->steer_qpn_count); - kfree(ibdev->ib_uc_qpns_bitmap); + bitmap_free(ibdev->ib_uc_qpns_bitmap); iounmap(ibdev->uar_map); for (p = 0; p < ibdev->num_ports; ++p) diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c index a190fb581591..08371a80fdc2 100644 --- a/drivers/infiniband/hw/mlx5/cq.c +++ b/drivers/infiniband/hw/mlx5/cq.c @@ -328,8 +328,11 @@ static void mlx5_handle_error_cqe(struct mlx5_ib_dev *dev, } wc->vendor_err = cqe->vendor_err_synd; - if (dump) + if (dump) { + mlx5_ib_warn(dev, "WC error: %d, Message: %s\n", wc->status, + ib_wc_status_msg(wc->status)); dump_cqe(dev, cqe); + } } static void handle_atomics(struct mlx5_ib_qp *qp, struct mlx5_cqe64 *cqe64, diff --git a/drivers/infiniband/hw/mlx5/mad.c b/drivers/infiniband/hw/mlx5/mad.c index ec242a5a17a3..293ed709e5ed 100644 --- a/drivers/infiniband/hw/mlx5/mad.c +++ b/drivers/infiniband/hw/mlx5/mad.c @@ -291,7 +291,7 @@ int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, unsigned int port) if (!in_mad || !out_mad) goto out; - init_query_mad(in_mad); + ib_init_query_mad(in_mad); in_mad->attr_id = MLX5_ATTR_EXTENDED_PORT_INFO; in_mad->attr_mod = cpu_to_be32(port); @@ -318,7 +318,7 @@ static int mlx5_query_mad_ifc_smp_attr_node_info(struct ib_device *ibdev, if (!in_mad) return -ENOMEM; - init_query_mad(in_mad); + ib_init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_NODE_INFO; err = mlx5_MAD_IFC(to_mdev(ibdev), 1, 1, 1, NULL, NULL, in_mad, @@ -405,7 +405,7 @@ int mlx5_query_mad_ifc_node_desc(struct mlx5_ib_dev *dev, char *node_desc) if (!in_mad || !out_mad) goto out; - init_query_mad(in_mad); + ib_init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_NODE_DESC; err = mlx5_MAD_IFC(dev, 1, 1, 1, NULL, NULL, in_mad, out_mad); @@ -430,7 +430,7 @@ int mlx5_query_mad_ifc_node_guid(struct mlx5_ib_dev *dev, __be64 *node_guid) if (!in_mad || !out_mad) goto out; - init_query_mad(in_mad); + ib_init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_NODE_INFO; err = mlx5_MAD_IFC(dev, 1, 1, 1, NULL, NULL, in_mad, out_mad); @@ -456,7 +456,7 @@ int mlx5_query_mad_ifc_pkey(struct ib_device *ibdev, u32 port, u16 index, if (!in_mad || !out_mad) goto out; - init_query_mad(in_mad); + ib_init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_PKEY_TABLE; in_mad->attr_mod = cpu_to_be32(index / 32); @@ -485,7 +485,7 @@ int mlx5_query_mad_ifc_gids(struct ib_device *ibdev, u32 port, int index, if (!in_mad || !out_mad) goto out; - init_query_mad(in_mad); + ib_init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_PORT_INFO; in_mad->attr_mod = cpu_to_be32(port); @@ -496,7 +496,7 @@ int mlx5_query_mad_ifc_gids(struct ib_device *ibdev, u32 port, int index, memcpy(gid->raw, out_mad->data + 8, 8); - init_query_mad(in_mad); + ib_init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_GUID_INFO; in_mad->attr_mod = cpu_to_be32(index / 8); @@ -530,7 +530,7 @@ int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u32 port, /* props being zeroed by the caller, avoid zeroing it here */ - init_query_mad(in_mad); + ib_init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_PORT_INFO; in_mad->attr_mod = cpu_to_be32(port); @@ -584,6 +584,11 @@ int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u32 port, props->port_cap_flags2 & IB_PORT_LINK_SPEED_HDR_SUP) props->active_speed = IB_SPEED_HDR; break; + case 8: + if (props->port_cap_flags & IB_PORT_CAP_MASK2_SUP && + props->port_cap_flags2 & IB_PORT_LINK_SPEED_NDR_SUP) + props->active_speed = IB_SPEED_NDR; + break; } } @@ -591,7 +596,7 @@ int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u32 port, if (props->active_speed == 4) { if (dev->port_caps[port - 1].ext_port_cap & MLX_EXT_PORT_CAP_FLAG_EXTENDED_PORT_INFO) { - init_query_mad(in_mad); + ib_init_query_mad(in_mad); in_mad->attr_id = MLX5_ATTR_EXTENDED_PORT_INFO; in_mad->attr_mod = cpu_to_be32(port); diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index e3c33be9c5a0..cbc20e400be0 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -665,9 +665,9 @@ struct mlx5_ib_mr { /* User MR data */ struct mlx5_cache_ent *cache_ent; + /* Everything after cache_ent is zero'd when MR allocated */ struct ib_umem *umem; - /* This is zero'd when the MR is allocated */ union { /* Used only while the MR is in the cache */ struct { @@ -719,7 +719,7 @@ struct mlx5_ib_mr { /* Zero the fields in the mr that are variant depending on usage */ static inline void mlx5_clear_mr(struct mlx5_ib_mr *mr) { - memset(mr->out, 0, sizeof(*mr) - offsetof(struct mlx5_ib_mr, out)); + memset_after(mr, 0, cache_ent); } static inline bool is_odp_mr(struct mlx5_ib_mr *mr) @@ -1466,14 +1466,6 @@ extern const struct uapi_definition mlx5_ib_flow_defs[]; extern const struct uapi_definition mlx5_ib_qos_defs[]; extern const struct uapi_definition mlx5_ib_std_types_defs[]; -static inline void init_query_mad(struct ib_smp *mad) -{ - mad->base_version = 1; - mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED; - mad->class_version = 1; - mad->method = IB_MGMT_METHOD_GET; -} - static inline int is_qp1(enum ib_qp_type qp_type) { return qp_type == MLX5_IB_QPT_HW_GSI || qp_type == IB_QPT_GSI; diff --git a/drivers/infiniband/hw/mthca/mthca_allocator.c b/drivers/infiniband/hw/mthca/mthca_allocator.c index aef1d274a14e..9f0f79d02d3c 100644 --- a/drivers/infiniband/hw/mthca/mthca_allocator.c +++ b/drivers/infiniband/hw/mthca/mthca_allocator.c @@ -51,7 +51,7 @@ u32 mthca_alloc(struct mthca_alloc *alloc) } if (obj < alloc->max) { - set_bit(obj, alloc->table); + __set_bit(obj, alloc->table); obj |= alloc->top; } else obj = -1; @@ -69,7 +69,7 @@ void mthca_free(struct mthca_alloc *alloc, u32 obj) spin_lock_irqsave(&alloc->lock, flags); - clear_bit(obj, alloc->table); + __clear_bit(obj, alloc->table); alloc->last = min(alloc->last, obj); alloc->top = (alloc->top + alloc->max) & alloc->mask; @@ -79,8 +79,6 @@ void mthca_free(struct mthca_alloc *alloc, u32 obj) int mthca_alloc_init(struct mthca_alloc *alloc, u32 num, u32 mask, u32 reserved) { - int i; - /* num must be a power of 2 */ if (num != 1 << (ffs(num) - 1)) return -EINVAL; @@ -90,21 +88,18 @@ int mthca_alloc_init(struct mthca_alloc *alloc, u32 num, u32 mask, alloc->max = num; alloc->mask = mask; spin_lock_init(&alloc->lock); - alloc->table = kmalloc_array(BITS_TO_LONGS(num), sizeof(long), - GFP_KERNEL); + alloc->table = bitmap_zalloc(num, GFP_KERNEL); if (!alloc->table) return -ENOMEM; - bitmap_zero(alloc->table, num); - for (i = 0; i < reserved; ++i) - set_bit(i, alloc->table); + bitmap_set(alloc->table, 0, reserved); return 0; } void mthca_alloc_cleanup(struct mthca_alloc *alloc) { - kfree(alloc->table); + bitmap_free(alloc->table); } /* diff --git a/drivers/infiniband/hw/mthca/mthca_mr.c b/drivers/infiniband/hw/mthca/mthca_mr.c index ce0e0867e488..192f83fd7c8a 100644 --- a/drivers/infiniband/hw/mthca/mthca_mr.c +++ b/drivers/infiniband/hw/mthca/mthca_mr.c @@ -101,13 +101,13 @@ static u32 mthca_buddy_alloc(struct mthca_buddy *buddy, int order) return -1; found: - clear_bit(seg, buddy->bits[o]); + __clear_bit(seg, buddy->bits[o]); --buddy->num_free[o]; while (o > order) { --o; seg <<= 1; - set_bit(seg ^ 1, buddy->bits[o]); + __set_bit(seg ^ 1, buddy->bits[o]); ++buddy->num_free[o]; } @@ -125,13 +125,13 @@ static void mthca_buddy_free(struct mthca_buddy *buddy, u32 seg, int order) spin_lock(&buddy->lock); while (test_bit(seg ^ 1, buddy->bits[order])) { - clear_bit(seg ^ 1, buddy->bits[order]); + __clear_bit(seg ^ 1, buddy->bits[order]); --buddy->num_free[order]; seg >>= 1; ++order; } - set_bit(seg, buddy->bits[order]); + __set_bit(seg, buddy->bits[order]); ++buddy->num_free[order]; spin_unlock(&buddy->lock); @@ -139,7 +139,7 @@ static void mthca_buddy_free(struct mthca_buddy *buddy, u32 seg, int order) static int mthca_buddy_init(struct mthca_buddy *buddy, int max_order) { - int i, s; + int i; buddy->max_order = max_order; spin_lock_init(&buddy->lock); @@ -152,22 +152,20 @@ static int mthca_buddy_init(struct mthca_buddy *buddy, int max_order) goto err_out; for (i = 0; i <= buddy->max_order; ++i) { - s = BITS_TO_LONGS(1 << (buddy->max_order - i)); - buddy->bits[i] = kmalloc_array(s, sizeof(long), GFP_KERNEL); + buddy->bits[i] = bitmap_zalloc(1 << (buddy->max_order - i), + GFP_KERNEL); if (!buddy->bits[i]) goto err_out_free; - bitmap_zero(buddy->bits[i], - 1 << (buddy->max_order - i)); } - set_bit(0, buddy->bits[buddy->max_order]); + __set_bit(0, buddy->bits[buddy->max_order]); buddy->num_free[buddy->max_order] = 1; return 0; err_out_free: for (i = 0; i <= buddy->max_order; ++i) - kfree(buddy->bits[i]); + bitmap_free(buddy->bits[i]); err_out: kfree(buddy->bits); @@ -181,7 +179,7 @@ static void mthca_buddy_cleanup(struct mthca_buddy *buddy) int i; for (i = 0; i <= buddy->max_order; ++i) - kfree(buddy->bits[i]); + bitmap_free(buddy->bits[i]); kfree(buddy->bits); kfree(buddy->num_free); @@ -469,8 +467,7 @@ int mthca_mr_alloc(struct mthca_dev *dev, u32 pd, int buffer_size_shift, mpt_entry->start = cpu_to_be64(iova); mpt_entry->length = cpu_to_be64(total_size); - memset(&mpt_entry->lkey, 0, - sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, lkey)); + memset_startat(mpt_entry, 0, lkey); if (mr->mtt) mpt_entry->mtt_seg = diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c index ceee23ebc0f2..c46df53f26cf 100644 --- a/drivers/infiniband/hw/mthca/mthca_provider.c +++ b/drivers/infiniband/hw/mthca/mthca_provider.c @@ -50,14 +50,6 @@ #include #include "mthca_memfree.h" -static void init_query_mad(struct ib_smp *mad) -{ - mad->base_version = 1; - mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED; - mad->class_version = 1; - mad->method = IB_MGMT_METHOD_GET; -} - static int mthca_query_device(struct ib_device *ibdev, struct ib_device_attr *props, struct ib_udata *uhw) { @@ -78,7 +70,7 @@ static int mthca_query_device(struct ib_device *ibdev, struct ib_device_attr *pr props->fw_ver = mdev->fw_ver; - init_query_mad(in_mad); + ib_init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_NODE_INFO; err = mthca_MAD_IFC(mdev, 1, 1, @@ -140,7 +132,7 @@ static int mthca_query_port(struct ib_device *ibdev, /* props being zeroed by the caller, avoid zeroing it here */ - init_query_mad(in_mad); + ib_init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_PORT_INFO; in_mad->attr_mod = cpu_to_be32(port); @@ -234,7 +226,7 @@ static int mthca_query_pkey(struct ib_device *ibdev, if (!in_mad || !out_mad) goto out; - init_query_mad(in_mad); + ib_init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_PKEY_TABLE; in_mad->attr_mod = cpu_to_be32(index / 32); @@ -263,7 +255,7 @@ static int mthca_query_gid(struct ib_device *ibdev, u32 port, if (!in_mad || !out_mad) goto out; - init_query_mad(in_mad); + ib_init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_PORT_INFO; in_mad->attr_mod = cpu_to_be32(port); @@ -274,7 +266,7 @@ static int mthca_query_gid(struct ib_device *ibdev, u32 port, memcpy(gid->raw, out_mad->data + 8, 8); - init_query_mad(in_mad); + ib_init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_GUID_INFO; in_mad->attr_mod = cpu_to_be32(index / 8); @@ -1006,7 +998,7 @@ static int mthca_init_node_data(struct mthca_dev *dev) if (!in_mad || !out_mad) goto out; - init_query_mad(in_mad); + ib_init_query_mad(in_mad); in_mad->attr_id = IB_SMP_ATTR_NODE_DESC; err = mthca_MAD_IFC(dev, 1, 1, diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c index c51c3f40700e..265a581133dc 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c @@ -1506,7 +1506,6 @@ int ocrdma_mbx_dealloc_pd(struct ocrdma_dev *dev, struct ocrdma_pd *pd) static int ocrdma_mbx_alloc_pd_range(struct ocrdma_dev *dev) { int status = -ENOMEM; - size_t pd_bitmap_size; struct ocrdma_alloc_pd_range *cmd; struct ocrdma_alloc_pd_range_rsp *rsp; @@ -1528,10 +1527,8 @@ static int ocrdma_mbx_alloc_pd_range(struct ocrdma_dev *dev) dev->pd_mgr->pd_dpp_start = rsp->dpp_page_pdid & OCRDMA_ALLOC_PD_RNG_RSP_START_PDID_MASK; dev->pd_mgr->max_dpp_pd = rsp->pd_count; - pd_bitmap_size = - BITS_TO_LONGS(rsp->pd_count) * sizeof(long); - dev->pd_mgr->pd_dpp_bitmap = kzalloc(pd_bitmap_size, - GFP_KERNEL); + dev->pd_mgr->pd_dpp_bitmap = bitmap_zalloc(rsp->pd_count, + GFP_KERNEL); } kfree(cmd); } @@ -1547,9 +1544,8 @@ static int ocrdma_mbx_alloc_pd_range(struct ocrdma_dev *dev) dev->pd_mgr->pd_norm_start = rsp->dpp_page_pdid & OCRDMA_ALLOC_PD_RNG_RSP_START_PDID_MASK; dev->pd_mgr->max_normal_pd = rsp->pd_count; - pd_bitmap_size = BITS_TO_LONGS(rsp->pd_count) * sizeof(long); - dev->pd_mgr->pd_norm_bitmap = kzalloc(pd_bitmap_size, - GFP_KERNEL); + dev->pd_mgr->pd_norm_bitmap = bitmap_zalloc(rsp->pd_count, + GFP_KERNEL); } kfree(cmd); @@ -1611,8 +1607,8 @@ void ocrdma_alloc_pd_pool(struct ocrdma_dev *dev) static void ocrdma_free_pd_pool(struct ocrdma_dev *dev) { ocrdma_mbx_dealloc_pd_range(dev); - kfree(dev->pd_mgr->pd_norm_bitmap); - kfree(dev->pd_mgr->pd_dpp_bitmap); + bitmap_free(dev->pd_mgr->pd_norm_bitmap); + bitmap_free(dev->pd_mgr->pd_dpp_bitmap); kfree(dev->pd_mgr); } diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_main.c b/drivers/infiniband/hw/ocrdma/ocrdma_main.c index 7abf6cf1e937..5d4b3bc16493 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_main.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_main.c @@ -62,20 +62,6 @@ MODULE_DESCRIPTION(OCRDMA_ROCE_DRV_DESC " " OCRDMA_ROCE_DRV_VERSION); MODULE_AUTHOR("Emulex Corporation"); MODULE_LICENSE("Dual BSD/GPL"); -void ocrdma_get_guid(struct ocrdma_dev *dev, u8 *guid) -{ - u8 mac_addr[6]; - - memcpy(&mac_addr[0], &dev->nic_info.mac_addr[0], ETH_ALEN); - guid[0] = mac_addr[0] ^ 2; - guid[1] = mac_addr[1]; - guid[2] = mac_addr[2]; - guid[3] = 0xff; - guid[4] = 0xfe; - guid[5] = mac_addr[3]; - guid[6] = mac_addr[4]; - guid[7] = mac_addr[5]; -} static enum rdma_link_layer ocrdma_link_layer(struct ib_device *device, u32 port_num) { @@ -203,7 +189,8 @@ static int ocrdma_register_device(struct ocrdma_dev *dev) { int ret; - ocrdma_get_guid(dev, (u8 *)&dev->ibdev.node_guid); + addrconf_addr_eui48((u8 *)&dev->ibdev.node_guid, + dev->nic_info.mac_addr); BUILD_BUG_ON(sizeof(OCRDMA_NODE_DESC) > IB_DEVICE_NODE_DESC_MAX); memcpy(dev->ibdev.node_desc, OCRDMA_NODE_DESC, sizeof(OCRDMA_NODE_DESC)); diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c index 735123d0e9ec..acf9970ec245 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c @@ -41,6 +41,7 @@ */ #include +#include #include #include #include @@ -74,7 +75,8 @@ int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, memset(attr, 0, sizeof *attr); memcpy(&attr->fw_ver, &dev->attr.fw_ver[0], min(sizeof(dev->attr.fw_ver), sizeof(attr->fw_ver))); - ocrdma_get_guid(dev, (u8 *)&attr->sys_image_guid); + addrconf_addr_eui48((u8 *)&attr->sys_image_guid, + dev->nic_info.mac_addr); attr->max_mr_size = dev->attr.max_mr_size; attr->page_size_cap = 0xffff000; attr->vendor_id = dev->nic_info.pdev->vendor; @@ -245,13 +247,13 @@ static bool ocrdma_search_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr, static u16 _ocrdma_pd_mgr_get_bitmap(struct ocrdma_dev *dev, bool dpp_pool) { u16 pd_bitmap_idx = 0; - const unsigned long *pd_bitmap; + unsigned long *pd_bitmap; if (dpp_pool) { pd_bitmap = dev->pd_mgr->pd_dpp_bitmap; pd_bitmap_idx = find_first_zero_bit(pd_bitmap, dev->pd_mgr->max_dpp_pd); - __set_bit(pd_bitmap_idx, dev->pd_mgr->pd_dpp_bitmap); + __set_bit(pd_bitmap_idx, pd_bitmap); dev->pd_mgr->pd_dpp_count++; if (dev->pd_mgr->pd_dpp_count > dev->pd_mgr->pd_dpp_thrsh) dev->pd_mgr->pd_dpp_thrsh = dev->pd_mgr->pd_dpp_count; @@ -259,7 +261,7 @@ static u16 _ocrdma_pd_mgr_get_bitmap(struct ocrdma_dev *dev, bool dpp_pool) pd_bitmap = dev->pd_mgr->pd_norm_bitmap; pd_bitmap_idx = find_first_zero_bit(pd_bitmap, dev->pd_mgr->max_normal_pd); - __set_bit(pd_bitmap_idx, dev->pd_mgr->pd_norm_bitmap); + __set_bit(pd_bitmap_idx, pd_bitmap); dev->pd_mgr->pd_norm_count++; if (dev->pd_mgr->pd_norm_count > dev->pd_mgr->pd_norm_thrsh) dev->pd_mgr->pd_norm_thrsh = dev->pd_mgr->pd_norm_count; @@ -1844,12 +1846,10 @@ int ocrdma_modify_srq(struct ib_srq *ibsrq, int ocrdma_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) { - int status; struct ocrdma_srq *srq; srq = get_ocrdma_srq(ibsrq); - status = ocrdma_mbx_query_srq(srq, srq_attr); - return status; + return ocrdma_mbx_query_srq(srq, srq_attr); } int ocrdma_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) @@ -1960,7 +1960,6 @@ static int ocrdma_build_inline_sges(struct ocrdma_qp *qp, static int ocrdma_build_send(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, const struct ib_send_wr *wr) { - int status; struct ocrdma_sge *sge; u32 wqe_size = sizeof(*hdr); @@ -1972,8 +1971,7 @@ static int ocrdma_build_send(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, sge = (struct ocrdma_sge *)(hdr + 1); } - status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size); - return status; + return ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size); } static int ocrdma_build_write(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h index b73d742a520c..f860b7fcef33 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h +++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h @@ -59,7 +59,6 @@ int ocrdma_query_port(struct ib_device *ibdev, u32 port, enum rdma_protocol_type ocrdma_query_protocol(struct ib_device *device, u32 port_num); -void ocrdma_get_guid(struct ocrdma_dev *, u8 *guid); int ocrdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey); int ocrdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata); diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c index 9100009f0a23..a53476653b0d 100644 --- a/drivers/infiniband/hw/qedr/verbs.c +++ b/drivers/infiniband/hw/qedr/verbs.c @@ -1931,6 +1931,7 @@ static int qedr_create_user_qp(struct qedr_dev *dev, /* db offset was calculated in copy_qp_uresp, now set in the user q */ if (qedr_qp_has_sq(qp)) { qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset; + qp->sq.max_wr = attrs->cap.max_send_wr; rc = qedr_db_recovery_add(dev, qp->usq.db_addr, &qp->usq.db_rec_data->db_data, DB_REC_WIDTH_32B, @@ -1941,6 +1942,7 @@ static int qedr_create_user_qp(struct qedr_dev *dev, if (qedr_qp_has_rq(qp)) { qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset; + qp->rq.max_wr = attrs->cap.max_recv_wr; rc = qedr_db_recovery_add(dev, qp->urq.db_addr, &qp->urq.db_rec_data->db_data, DB_REC_WIDTH_32B, diff --git a/drivers/infiniband/hw/qib/qib_iba6120.c b/drivers/infiniband/hw/qib/qib_iba6120.c index a9b83bc13f4a..aea571943768 100644 --- a/drivers/infiniband/hw/qib/qib_iba6120.c +++ b/drivers/infiniband/hw/qib/qib_iba6120.c @@ -3030,7 +3030,7 @@ static int qib_6120_ib_updown(struct qib_pportdata *ppd, int ibup, u64 ibcs) /* Does read/modify/write to appropriate registers to * set output and direction bits selected by mask. - * these are in their canonical postions (e.g. lsb of + * these are in their canonical positions (e.g. lsb of * dir will end up in D48 of extctrl on existing chips). * returns contents of GP Inputs. */ diff --git a/drivers/infiniband/hw/qib/qib_iba7220.c b/drivers/infiniband/hw/qib/qib_iba7220.c index d1c0bc31869f..80a8dd6c7814 100644 --- a/drivers/infiniband/hw/qib/qib_iba7220.c +++ b/drivers/infiniband/hw/qib/qib_iba7220.c @@ -3742,7 +3742,7 @@ static int qib_7220_ib_updown(struct qib_pportdata *ppd, int ibup, u64 ibcs) /* * Does read/modify/write to appropriate registers to * set output and direction bits selected by mask. - * these are in their canonical postions (e.g. lsb of + * these are in their canonical positions (e.g. lsb of * dir will end up in D48 of extctrl on existing chips). * returns contents of GP Inputs. */ diff --git a/drivers/infiniband/hw/qib/qib_iba7322.c b/drivers/infiniband/hw/qib/qib_iba7322.c index ab98b6a3ae1e..ceed302cf6a0 100644 --- a/drivers/infiniband/hw/qib/qib_iba7322.c +++ b/drivers/infiniband/hw/qib/qib_iba7322.c @@ -5665,7 +5665,7 @@ static int qib_7322_ib_updown(struct qib_pportdata *ppd, int ibup, u64 ibcs) /* * Does read/modify/write to appropriate registers to * set output and direction bits selected by mask. - * these are in their canonical postions (e.g. lsb of + * these are in their canonical positions (e.g. lsb of * dir will end up in D48 of extctrl on existing chips). * returns contents of GP Inputs. */ diff --git a/drivers/infiniband/hw/usnic/usnic_ib_sysfs.c b/drivers/infiniband/hw/usnic/usnic_ib_sysfs.c index 586b0e52ba7f..7d868f033bbf 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_sysfs.c +++ b/drivers/infiniband/hw/usnic/usnic_ib_sysfs.c @@ -243,10 +243,11 @@ static struct attribute *usnic_ib_qpn_default_attrs[] = { &qpn_attr_summary.attr, NULL }; +ATTRIBUTE_GROUPS(usnic_ib_qpn_default); static struct kobj_type usnic_ib_qpn_type = { .sysfs_ops = &usnic_ib_qpn_sysfs_ops, - .default_attrs = usnic_ib_qpn_default_attrs + .default_groups = usnic_ib_qpn_default_groups, }; int usnic_ib_sysfs_register_usdev(struct usnic_ib_dev *us_ibdev) diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c index 756a83bcff58..5a0e26cd648e 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c +++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c @@ -442,12 +442,10 @@ int usnic_ib_query_gid(struct ib_device *ibdev, u32 port, int index, int usnic_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) { struct usnic_ib_pd *pd = to_upd(ibpd); - void *umem_pd; - umem_pd = pd->umem_pd = usnic_uiom_alloc_pd(); - if (IS_ERR_OR_NULL(umem_pd)) { - return umem_pd ? PTR_ERR(umem_pd) : -ENOMEM; - } + pd->umem_pd = usnic_uiom_alloc_pd(); + if (IS_ERR(pd->umem_pd)) + return PTR_ERR(pd->umem_pd); return 0; } diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_doorbell.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_doorbell.c index bf51357ea3aa..9a4de962e947 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_doorbell.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_doorbell.c @@ -63,12 +63,12 @@ int pvrdma_uar_table_init(struct pvrdma_dev *dev) tbl->max = num; tbl->mask = mask; spin_lock_init(&tbl->lock); - tbl->table = kcalloc(BITS_TO_LONGS(num), sizeof(long), GFP_KERNEL); + tbl->table = bitmap_zalloc(num, GFP_KERNEL); if (!tbl->table) return -ENOMEM; /* 0th UAR is taken by the device. */ - set_bit(0, tbl->table); + __set_bit(0, tbl->table); return 0; } @@ -77,7 +77,7 @@ void pvrdma_uar_table_cleanup(struct pvrdma_dev *dev) { struct pvrdma_id_table *tbl = &dev->uar_table.tbl; - kfree(tbl->table); + bitmap_free(tbl->table); } int pvrdma_uar_alloc(struct pvrdma_dev *dev, struct pvrdma_uar_map *uar) @@ -100,7 +100,7 @@ int pvrdma_uar_alloc(struct pvrdma_dev *dev, struct pvrdma_uar_map *uar) return -ENOMEM; } - set_bit(obj, tbl->table); + __set_bit(obj, tbl->table); obj |= tbl->top; spin_unlock_irqrestore(&tbl->lock, flags); @@ -120,7 +120,7 @@ void pvrdma_uar_free(struct pvrdma_dev *dev, struct pvrdma_uar_map *uar) obj = uar->index & (tbl->max - 1); spin_lock_irqsave(&tbl->lock, flags); - clear_bit(obj, tbl->table); + __clear_bit(obj, tbl->table); tbl->last = min(tbl->last, obj); tbl->top = (tbl->top + tbl->max) & tbl->mask; spin_unlock_irqrestore(&tbl->lock, flags); diff --git a/drivers/infiniband/sw/rxe/Makefile b/drivers/infiniband/sw/rxe/Makefile index 1e24673e9318..5395a581f4bb 100644 --- a/drivers/infiniband/sw/rxe/Makefile +++ b/drivers/infiniband/sw/rxe/Makefile @@ -22,5 +22,4 @@ rdma_rxe-y := \ rxe_mcast.o \ rxe_task.o \ rxe_net.o \ - rxe_sysfs.o \ rxe_hw_counters.o diff --git a/drivers/infiniband/sw/rxe/rxe.c b/drivers/infiniband/sw/rxe/rxe.c index 8e0f9c489cab..fab291245366 100644 --- a/drivers/infiniband/sw/rxe/rxe.c +++ b/drivers/infiniband/sw/rxe/rxe.c @@ -13,8 +13,6 @@ MODULE_AUTHOR("Bob Pearson, Frank Zago, John Groves, Kamal Heib"); MODULE_DESCRIPTION("Soft RDMA transport"); MODULE_LICENSE("Dual BSD/GPL"); -bool rxe_initialized; - /* free resources for a rxe device all objects created for this device must * have been destroyed */ @@ -290,7 +288,6 @@ static int __init rxe_module_init(void) return err; rdma_link_register(&rxe_link_ops); - rxe_initialized = true; pr_info("loaded\n"); return 0; } @@ -301,7 +298,6 @@ static void __exit rxe_module_exit(void) ib_unregister_driver(RDMA_DRIVER_RXE); rxe_net_exit(); - rxe_initialized = false; pr_info("unloaded\n"); } diff --git a/drivers/infiniband/sw/rxe/rxe.h b/drivers/infiniband/sw/rxe/rxe.h index 1bb3fb618bf5..fb9066e6f5f0 100644 --- a/drivers/infiniband/sw/rxe/rxe.h +++ b/drivers/infiniband/sw/rxe/rxe.h @@ -39,8 +39,6 @@ #define RXE_ROCE_V2_SPORT (0xc000) -extern bool rxe_initialized; - void rxe_set_mtu(struct rxe_dev *rxe, unsigned int dev_mtu); int rxe_add(struct rxe_dev *rxe, unsigned int mtu, const char *ibdev_name); diff --git a/drivers/infiniband/sw/rxe/rxe_comp.c b/drivers/infiniband/sw/rxe/rxe_comp.c index d771ba8449a1..f363fe3fa414 100644 --- a/drivers/infiniband/sw/rxe/rxe_comp.c +++ b/drivers/infiniband/sw/rxe/rxe_comp.c @@ -458,8 +458,6 @@ static inline enum comp_state complete_ack(struct rxe_qp *qp, struct rxe_pkt_info *pkt, struct rxe_send_wqe *wqe) { - unsigned long flags; - if (wqe->has_rd_atomic) { wqe->has_rd_atomic = 0; atomic_inc(&qp->req.rd_atomic); @@ -472,11 +470,11 @@ static inline enum comp_state complete_ack(struct rxe_qp *qp, if (unlikely(qp->req.state == QP_STATE_DRAIN)) { /* state_lock used by requester & completer */ - spin_lock_irqsave(&qp->state_lock, flags); + spin_lock_bh(&qp->state_lock); if ((qp->req.state == QP_STATE_DRAIN) && (qp->comp.psn == qp->req.psn)) { qp->req.state = QP_STATE_DRAINED; - spin_unlock_irqrestore(&qp->state_lock, flags); + spin_unlock_bh(&qp->state_lock); if (qp->ibqp.event_handler) { struct ib_event ev; @@ -488,7 +486,7 @@ static inline enum comp_state complete_ack(struct rxe_qp *qp, qp->ibqp.qp_context); } } else { - spin_unlock_irqrestore(&qp->state_lock, flags); + spin_unlock_bh(&qp->state_lock); } } diff --git a/drivers/infiniband/sw/rxe/rxe_cq.c b/drivers/infiniband/sw/rxe/rxe_cq.c index 6848426c074f..6baaaa34458e 100644 --- a/drivers/infiniband/sw/rxe/rxe_cq.c +++ b/drivers/infiniband/sw/rxe/rxe_cq.c @@ -42,14 +42,13 @@ err1: static void rxe_send_complete(struct tasklet_struct *t) { struct rxe_cq *cq = from_tasklet(cq, t, comp_task); - unsigned long flags; - spin_lock_irqsave(&cq->cq_lock, flags); + spin_lock_bh(&cq->cq_lock); if (cq->is_dying) { - spin_unlock_irqrestore(&cq->cq_lock, flags); + spin_unlock_bh(&cq->cq_lock); return; } - spin_unlock_irqrestore(&cq->cq_lock, flags); + spin_unlock_bh(&cq->cq_lock); cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); } @@ -106,15 +105,14 @@ int rxe_cq_resize_queue(struct rxe_cq *cq, int cqe, int rxe_cq_post(struct rxe_cq *cq, struct rxe_cqe *cqe, int solicited) { struct ib_event ev; - unsigned long flags; int full; void *addr; - spin_lock_irqsave(&cq->cq_lock, flags); + spin_lock_bh(&cq->cq_lock); full = queue_full(cq->queue, QUEUE_TYPE_TO_CLIENT); if (unlikely(full)) { - spin_unlock_irqrestore(&cq->cq_lock, flags); + spin_unlock_bh(&cq->cq_lock); if (cq->ibcq.event_handler) { ev.device = cq->ibcq.device; ev.element.cq = &cq->ibcq; @@ -130,7 +128,7 @@ int rxe_cq_post(struct rxe_cq *cq, struct rxe_cqe *cqe, int solicited) queue_advance_producer(cq->queue, QUEUE_TYPE_TO_CLIENT); - spin_unlock_irqrestore(&cq->cq_lock, flags); + spin_unlock_bh(&cq->cq_lock); if ((cq->notify == IB_CQ_NEXT_COMP) || (cq->notify == IB_CQ_SOLICITED && solicited)) { @@ -143,16 +141,14 @@ int rxe_cq_post(struct rxe_cq *cq, struct rxe_cqe *cqe, int solicited) void rxe_cq_disable(struct rxe_cq *cq) { - unsigned long flags; - - spin_lock_irqsave(&cq->cq_lock, flags); + spin_lock_bh(&cq->cq_lock); cq->is_dying = true; - spin_unlock_irqrestore(&cq->cq_lock, flags); + spin_unlock_bh(&cq->cq_lock); } -void rxe_cq_cleanup(struct rxe_pool_entry *arg) +void rxe_cq_cleanup(struct rxe_pool_elem *elem) { - struct rxe_cq *cq = container_of(arg, typeof(*cq), pelem); + struct rxe_cq *cq = container_of(elem, typeof(*cq), elem); if (cq->queue) rxe_queue_cleanup(cq->queue); diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h index 1ca43b859d80..b1e174afb1d4 100644 --- a/drivers/infiniband/sw/rxe/rxe_loc.h +++ b/drivers/infiniband/sw/rxe/rxe_loc.h @@ -37,7 +37,7 @@ int rxe_cq_post(struct rxe_cq *cq, struct rxe_cqe *cqe, int solicited); void rxe_cq_disable(struct rxe_cq *cq); -void rxe_cq_cleanup(struct rxe_pool_entry *arg); +void rxe_cq_cleanup(struct rxe_pool_elem *arg); /* rxe_mcast.c */ int rxe_mcast_get_grp(struct rxe_dev *rxe, union ib_gid *mgid, @@ -51,7 +51,7 @@ int rxe_mcast_drop_grp_elem(struct rxe_dev *rxe, struct rxe_qp *qp, void rxe_drop_all_mcast_groups(struct rxe_qp *qp); -void rxe_mc_cleanup(struct rxe_pool_entry *arg); +void rxe_mc_cleanup(struct rxe_pool_elem *arg); /* rxe_mmap.c */ struct rxe_mmap_info { @@ -89,7 +89,7 @@ int rxe_invalidate_mr(struct rxe_qp *qp, u32 rkey); int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe); int rxe_mr_set_page(struct ib_mr *ibmr, u64 addr); int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata); -void rxe_mr_cleanup(struct rxe_pool_entry *arg); +void rxe_mr_cleanup(struct rxe_pool_elem *arg); /* rxe_mw.c */ int rxe_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata); @@ -97,7 +97,7 @@ int rxe_dealloc_mw(struct ib_mw *ibmw); int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe); int rxe_invalidate_mw(struct rxe_qp *qp, u32 rkey); struct rxe_mw *rxe_lookup_mw(struct rxe_qp *qp, int access, u32 rkey); -void rxe_mw_cleanup(struct rxe_pool_entry *arg); +void rxe_mw_cleanup(struct rxe_pool_elem *arg); /* rxe_net.c */ struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av, @@ -131,7 +131,7 @@ void rxe_qp_error(struct rxe_qp *qp); void rxe_qp_destroy(struct rxe_qp *qp); -void rxe_qp_cleanup(struct rxe_pool_entry *arg); +void rxe_qp_cleanup(struct rxe_pool_elem *elem); static inline int qp_num(struct rxe_qp *qp) { diff --git a/drivers/infiniband/sw/rxe/rxe_mcast.c b/drivers/infiniband/sw/rxe/rxe_mcast.c index 1c1d1b53312d..bd1ac88b8700 100644 --- a/drivers/infiniband/sw/rxe/rxe_mcast.c +++ b/drivers/infiniband/sw/rxe/rxe_mcast.c @@ -40,12 +40,11 @@ int rxe_mcast_get_grp(struct rxe_dev *rxe, union ib_gid *mgid, int err; struct rxe_mc_grp *grp; struct rxe_pool *pool = &rxe->mc_grp_pool; - unsigned long flags; if (rxe->attr.max_mcast_qp_attach == 0) return -EINVAL; - write_lock_irqsave(&pool->pool_lock, flags); + write_lock_bh(&pool->pool_lock); grp = rxe_pool_get_key_locked(pool, mgid); if (grp) @@ -53,13 +52,13 @@ int rxe_mcast_get_grp(struct rxe_dev *rxe, union ib_gid *mgid, grp = create_grp(rxe, pool, mgid); if (IS_ERR(grp)) { - write_unlock_irqrestore(&pool->pool_lock, flags); + write_unlock_bh(&pool->pool_lock); err = PTR_ERR(grp); return err; } done: - write_unlock_irqrestore(&pool->pool_lock, flags); + write_unlock_bh(&pool->pool_lock); *grp_p = grp; return 0; } @@ -169,9 +168,9 @@ void rxe_drop_all_mcast_groups(struct rxe_qp *qp) } } -void rxe_mc_cleanup(struct rxe_pool_entry *arg) +void rxe_mc_cleanup(struct rxe_pool_elem *elem) { - struct rxe_mc_grp *grp = container_of(arg, typeof(*grp), pelem); + struct rxe_mc_grp *grp = container_of(elem, typeof(*grp), elem); struct rxe_dev *rxe = grp->rxe; rxe_drop_key(grp); diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c index bcf717bcf0b3..453ef3c9d535 100644 --- a/drivers/infiniband/sw/rxe/rxe_mr.c +++ b/drivers/infiniband/sw/rxe/rxe_mr.c @@ -50,7 +50,7 @@ int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length) static void rxe_mr_init(int access, struct rxe_mr *mr) { - u32 lkey = mr->pelem.index << 8 | rxe_get_next_key(-1); + u32 lkey = mr->elem.index << 8 | rxe_get_next_key(-1); u32 rkey = (access & IB_ACCESS_REMOTE) ? lkey : 0; /* set ibmr->l/rkey and also copy into private l/rkey @@ -697,9 +697,9 @@ int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) return 0; } -void rxe_mr_cleanup(struct rxe_pool_entry *arg) +void rxe_mr_cleanup(struct rxe_pool_elem *elem) { - struct rxe_mr *mr = container_of(arg, typeof(*mr), pelem); + struct rxe_mr *mr = container_of(elem, typeof(*mr), elem); ib_umem_release(mr->umem); diff --git a/drivers/infiniband/sw/rxe/rxe_mw.c b/drivers/infiniband/sw/rxe/rxe_mw.c index 9534a7fe1a98..32dd8c0b8b9e 100644 --- a/drivers/infiniband/sw/rxe/rxe_mw.c +++ b/drivers/infiniband/sw/rxe/rxe_mw.c @@ -21,7 +21,7 @@ int rxe_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) } rxe_add_index(mw); - mw->rkey = ibmw->rkey = (mw->pelem.index << 8) | rxe_get_next_key(-1); + mw->rkey = ibmw->rkey = (mw->elem.index << 8) | rxe_get_next_key(-1); mw->state = (mw->ibmw.type == IB_MW_TYPE_2) ? RXE_MW_STATE_FREE : RXE_MW_STATE_VALID; spin_lock_init(&mw->lock); @@ -56,11 +56,10 @@ int rxe_dealloc_mw(struct ib_mw *ibmw) { struct rxe_mw *mw = to_rmw(ibmw); struct rxe_pd *pd = to_rpd(ibmw->pd); - unsigned long flags; - spin_lock_irqsave(&mw->lock, flags); + spin_lock_bh(&mw->lock); rxe_do_dealloc_mw(mw); - spin_unlock_irqrestore(&mw->lock, flags); + spin_unlock_bh(&mw->lock); rxe_drop_ref(mw); rxe_drop_ref(pd); @@ -197,7 +196,6 @@ int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe) struct rxe_dev *rxe = to_rdev(qp->ibqp.device); u32 mw_rkey = wqe->wr.wr.mw.mw_rkey; u32 mr_lkey = wqe->wr.wr.mw.mr_lkey; - unsigned long flags; mw = rxe_pool_get_index(&rxe->mw_pool, mw_rkey >> 8); if (unlikely(!mw)) { @@ -225,7 +223,7 @@ int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe) mr = NULL; } - spin_lock_irqsave(&mw->lock, flags); + spin_lock_bh(&mw->lock); ret = rxe_check_bind_mw(qp, wqe, mw, mr); if (ret) @@ -233,7 +231,7 @@ int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe) rxe_do_bind_mw(qp, wqe, mw, mr); err_unlock: - spin_unlock_irqrestore(&mw->lock, flags); + spin_unlock_bh(&mw->lock); err_drop_mr: if (mr) rxe_drop_ref(mr); @@ -280,7 +278,6 @@ static void rxe_do_invalidate_mw(struct rxe_mw *mw) int rxe_invalidate_mw(struct rxe_qp *qp, u32 rkey) { struct rxe_dev *rxe = to_rdev(qp->ibqp.device); - unsigned long flags; struct rxe_mw *mw; int ret; @@ -295,7 +292,7 @@ int rxe_invalidate_mw(struct rxe_qp *qp, u32 rkey) goto err_drop_ref; } - spin_lock_irqsave(&mw->lock, flags); + spin_lock_bh(&mw->lock); ret = rxe_check_invalidate_mw(qp, mw); if (ret) @@ -303,7 +300,7 @@ int rxe_invalidate_mw(struct rxe_qp *qp, u32 rkey) rxe_do_invalidate_mw(mw); err_unlock: - spin_unlock_irqrestore(&mw->lock, flags); + spin_unlock_bh(&mw->lock); err_drop_ref: rxe_drop_ref(mw); err: @@ -333,9 +330,9 @@ struct rxe_mw *rxe_lookup_mw(struct rxe_qp *qp, int access, u32 rkey) return mw; } -void rxe_mw_cleanup(struct rxe_pool_entry *elem) +void rxe_mw_cleanup(struct rxe_pool_elem *elem) { - struct rxe_mw *mw = container_of(elem, typeof(*mw), pelem); + struct rxe_mw *mw = container_of(elem, typeof(*mw), elem); rxe_drop_index(mw); } diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c index 2cb810cb890a..be72bdbfb4ba 100644 --- a/drivers/infiniband/sw/rxe/rxe_net.c +++ b/drivers/infiniband/sw/rxe/rxe_net.c @@ -22,24 +22,20 @@ static struct rxe_recv_sockets recv_sockets; int rxe_mcast_add(struct rxe_dev *rxe, union ib_gid *mgid) { - int err; unsigned char ll_addr[ETH_ALEN]; ipv6_eth_mc_map((struct in6_addr *)mgid->raw, ll_addr); - err = dev_mc_add(rxe->ndev, ll_addr); - return err; + return dev_mc_add(rxe->ndev, ll_addr); } int rxe_mcast_delete(struct rxe_dev *rxe, union ib_gid *mgid) { - int err; unsigned char ll_addr[ETH_ALEN]; ipv6_eth_mc_map((struct in6_addr *)mgid->raw, ll_addr); - err = dev_mc_del(rxe->ndev, ll_addr); - return err; + return dev_mc_del(rxe->ndev, ll_addr); } static struct dst_entry *rxe_find_route4(struct net_device *ndev, @@ -444,7 +440,6 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, else err = rxe_send(skb, pkt); if (err) { - rxe->xmit_errors++; rxe_counter_inc(rxe, RXE_CNT_SEND_ERR); return err; } diff --git a/drivers/infiniband/sw/rxe/rxe_opcode.c b/drivers/infiniband/sw/rxe/rxe_opcode.c index 3ef5a10a6efd..df596ba7527d 100644 --- a/drivers/infiniband/sw/rxe/rxe_opcode.c +++ b/drivers/infiniband/sw/rxe/rxe_opcode.c @@ -108,8 +108,8 @@ struct rxe_wr_opcode_info rxe_wr_opcode_info[] = { struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { [IB_OPCODE_RC_SEND_FIRST] = { .name = "IB_OPCODE_RC_SEND_FIRST", - .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_RWR_MASK - | RXE_SEND_MASK | RXE_START_MASK, + .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_RWR_MASK | + RXE_SEND_MASK | RXE_START_MASK, .length = RXE_BTH_BYTES, .offset = { [RXE_BTH] = 0, @@ -117,9 +117,9 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { } }, [IB_OPCODE_RC_SEND_MIDDLE] = { - .name = "IB_OPCODE_RC_SEND_MIDDLE]", - .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_SEND_MASK - | RXE_MIDDLE_MASK, + .name = "IB_OPCODE_RC_SEND_MIDDLE", + .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_SEND_MASK | + RXE_MIDDLE_MASK, .length = RXE_BTH_BYTES, .offset = { [RXE_BTH] = 0, @@ -128,8 +128,8 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { }, [IB_OPCODE_RC_SEND_LAST] = { .name = "IB_OPCODE_RC_SEND_LAST", - .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_COMP_MASK - | RXE_SEND_MASK | RXE_END_MASK, + .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_COMP_MASK | + RXE_SEND_MASK | RXE_END_MASK, .length = RXE_BTH_BYTES, .offset = { [RXE_BTH] = 0, @@ -138,21 +138,21 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { }, [IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE] = { .name = "IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE", - .mask = RXE_IMMDT_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK - | RXE_COMP_MASK | RXE_SEND_MASK | RXE_END_MASK, + .mask = RXE_IMMDT_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK | + RXE_COMP_MASK | RXE_SEND_MASK | RXE_END_MASK, .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES, .offset = { [RXE_BTH] = 0, [RXE_IMMDT] = RXE_BTH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_IMMDT_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_IMMDT_BYTES, } }, [IB_OPCODE_RC_SEND_ONLY] = { .name = "IB_OPCODE_RC_SEND_ONLY", - .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_COMP_MASK - | RXE_RWR_MASK | RXE_SEND_MASK - | RXE_START_MASK | RXE_END_MASK, + .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_COMP_MASK | + RXE_RWR_MASK | RXE_SEND_MASK | + RXE_START_MASK | RXE_END_MASK, .length = RXE_BTH_BYTES, .offset = { [RXE_BTH] = 0, @@ -161,33 +161,33 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { }, [IB_OPCODE_RC_SEND_ONLY_WITH_IMMEDIATE] = { .name = "IB_OPCODE_RC_SEND_ONLY_WITH_IMMEDIATE", - .mask = RXE_IMMDT_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK - | RXE_COMP_MASK | RXE_RWR_MASK | RXE_SEND_MASK - | RXE_START_MASK | RXE_END_MASK, + .mask = RXE_IMMDT_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK | + RXE_COMP_MASK | RXE_RWR_MASK | RXE_SEND_MASK | + RXE_START_MASK | RXE_END_MASK, .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES, .offset = { [RXE_BTH] = 0, [RXE_IMMDT] = RXE_BTH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_IMMDT_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_IMMDT_BYTES, } }, [IB_OPCODE_RC_RDMA_WRITE_FIRST] = { .name = "IB_OPCODE_RC_RDMA_WRITE_FIRST", - .mask = RXE_RETH_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK - | RXE_WRITE_MASK | RXE_START_MASK, + .mask = RXE_RETH_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK | + RXE_WRITE_MASK | RXE_START_MASK, .length = RXE_BTH_BYTES + RXE_RETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_RETH] = RXE_BTH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_RETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_RETH_BYTES, } }, [IB_OPCODE_RC_RDMA_WRITE_MIDDLE] = { .name = "IB_OPCODE_RC_RDMA_WRITE_MIDDLE", - .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_WRITE_MASK - | RXE_MIDDLE_MASK, + .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_WRITE_MASK | + RXE_MIDDLE_MASK, .length = RXE_BTH_BYTES, .offset = { [RXE_BTH] = 0, @@ -196,8 +196,8 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { }, [IB_OPCODE_RC_RDMA_WRITE_LAST] = { .name = "IB_OPCODE_RC_RDMA_WRITE_LAST", - .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_WRITE_MASK - | RXE_END_MASK, + .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_WRITE_MASK | + RXE_END_MASK, .length = RXE_BTH_BYTES, .offset = { [RXE_BTH] = 0, @@ -206,69 +206,69 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { }, [IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE] = { .name = "IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE", - .mask = RXE_IMMDT_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK - | RXE_WRITE_MASK | RXE_COMP_MASK | RXE_RWR_MASK - | RXE_END_MASK, + .mask = RXE_IMMDT_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK | + RXE_WRITE_MASK | RXE_COMP_MASK | RXE_RWR_MASK | + RXE_END_MASK, .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES, .offset = { [RXE_BTH] = 0, [RXE_IMMDT] = RXE_BTH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_IMMDT_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_IMMDT_BYTES, } }, [IB_OPCODE_RC_RDMA_WRITE_ONLY] = { .name = "IB_OPCODE_RC_RDMA_WRITE_ONLY", - .mask = RXE_RETH_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK - | RXE_WRITE_MASK | RXE_START_MASK - | RXE_END_MASK, + .mask = RXE_RETH_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK | + RXE_WRITE_MASK | RXE_START_MASK | + RXE_END_MASK, .length = RXE_BTH_BYTES + RXE_RETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_RETH] = RXE_BTH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_RETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_RETH_BYTES, } }, [IB_OPCODE_RC_RDMA_WRITE_ONLY_WITH_IMMEDIATE] = { .name = "IB_OPCODE_RC_RDMA_WRITE_ONLY_WITH_IMMEDIATE", - .mask = RXE_RETH_MASK | RXE_IMMDT_MASK | RXE_PAYLOAD_MASK - | RXE_REQ_MASK | RXE_WRITE_MASK - | RXE_COMP_MASK | RXE_RWR_MASK - | RXE_START_MASK | RXE_END_MASK, + .mask = RXE_RETH_MASK | RXE_IMMDT_MASK | RXE_PAYLOAD_MASK | + RXE_REQ_MASK | RXE_WRITE_MASK | + RXE_COMP_MASK | RXE_RWR_MASK | + RXE_START_MASK | RXE_END_MASK, .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES + RXE_RETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_RETH] = RXE_BTH_BYTES, - [RXE_IMMDT] = RXE_BTH_BYTES - + RXE_RETH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_RETH_BYTES - + RXE_IMMDT_BYTES, + [RXE_IMMDT] = RXE_BTH_BYTES + + RXE_RETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_RETH_BYTES + + RXE_IMMDT_BYTES, } }, [IB_OPCODE_RC_RDMA_READ_REQUEST] = { .name = "IB_OPCODE_RC_RDMA_READ_REQUEST", - .mask = RXE_RETH_MASK | RXE_REQ_MASK | RXE_READ_MASK - | RXE_START_MASK | RXE_END_MASK, + .mask = RXE_RETH_MASK | RXE_REQ_MASK | RXE_READ_MASK | + RXE_START_MASK | RXE_END_MASK, .length = RXE_BTH_BYTES + RXE_RETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_RETH] = RXE_BTH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_RETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_RETH_BYTES, } }, [IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST] = { .name = "IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST", - .mask = RXE_AETH_MASK | RXE_PAYLOAD_MASK | RXE_ACK_MASK - | RXE_START_MASK, + .mask = RXE_AETH_MASK | RXE_PAYLOAD_MASK | RXE_ACK_MASK | + RXE_START_MASK, .length = RXE_BTH_BYTES + RXE_AETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_AETH] = RXE_BTH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_AETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_AETH_BYTES, } }, [IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE] = { @@ -282,109 +282,110 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { }, [IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST] = { .name = "IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST", - .mask = RXE_AETH_MASK | RXE_PAYLOAD_MASK | RXE_ACK_MASK - | RXE_END_MASK, + .mask = RXE_AETH_MASK | RXE_PAYLOAD_MASK | RXE_ACK_MASK | + RXE_END_MASK, .length = RXE_BTH_BYTES + RXE_AETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_AETH] = RXE_BTH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_AETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_AETH_BYTES, } }, [IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY] = { .name = "IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY", - .mask = RXE_AETH_MASK | RXE_PAYLOAD_MASK | RXE_ACK_MASK - | RXE_START_MASK | RXE_END_MASK, + .mask = RXE_AETH_MASK | RXE_PAYLOAD_MASK | RXE_ACK_MASK | + RXE_START_MASK | RXE_END_MASK, .length = RXE_BTH_BYTES + RXE_AETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_AETH] = RXE_BTH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_AETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_AETH_BYTES, } }, [IB_OPCODE_RC_ACKNOWLEDGE] = { .name = "IB_OPCODE_RC_ACKNOWLEDGE", - .mask = RXE_AETH_MASK | RXE_ACK_MASK | RXE_START_MASK - | RXE_END_MASK, + .mask = RXE_AETH_MASK | RXE_ACK_MASK | RXE_START_MASK | + RXE_END_MASK, .length = RXE_BTH_BYTES + RXE_AETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_AETH] = RXE_BTH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_AETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_AETH_BYTES, } }, [IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE] = { .name = "IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE", - .mask = RXE_AETH_MASK | RXE_ATMACK_MASK | RXE_ACK_MASK - | RXE_START_MASK | RXE_END_MASK, + .mask = RXE_AETH_MASK | RXE_ATMACK_MASK | RXE_ACK_MASK | + RXE_START_MASK | RXE_END_MASK, .length = RXE_BTH_BYTES + RXE_ATMACK_BYTES + RXE_AETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_AETH] = RXE_BTH_BYTES, - [RXE_ATMACK] = RXE_BTH_BYTES - + RXE_AETH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_ATMACK_BYTES + RXE_AETH_BYTES, + [RXE_ATMACK] = RXE_BTH_BYTES + + RXE_AETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_ATMACK_BYTES + + RXE_AETH_BYTES, } }, [IB_OPCODE_RC_COMPARE_SWAP] = { .name = "IB_OPCODE_RC_COMPARE_SWAP", - .mask = RXE_ATMETH_MASK | RXE_REQ_MASK | RXE_ATOMIC_MASK - | RXE_START_MASK | RXE_END_MASK, + .mask = RXE_ATMETH_MASK | RXE_REQ_MASK | RXE_ATOMIC_MASK | + RXE_START_MASK | RXE_END_MASK, .length = RXE_BTH_BYTES + RXE_ATMETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_ATMETH] = RXE_BTH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_ATMETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_ATMETH_BYTES, } }, [IB_OPCODE_RC_FETCH_ADD] = { .name = "IB_OPCODE_RC_FETCH_ADD", - .mask = RXE_ATMETH_MASK | RXE_REQ_MASK | RXE_ATOMIC_MASK - | RXE_START_MASK | RXE_END_MASK, + .mask = RXE_ATMETH_MASK | RXE_REQ_MASK | RXE_ATOMIC_MASK | + RXE_START_MASK | RXE_END_MASK, .length = RXE_BTH_BYTES + RXE_ATMETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_ATMETH] = RXE_BTH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_ATMETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_ATMETH_BYTES, } }, [IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE] = { .name = "IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE", - .mask = RXE_IETH_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK - | RXE_COMP_MASK | RXE_SEND_MASK | RXE_END_MASK, + .mask = RXE_IETH_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK | + RXE_COMP_MASK | RXE_SEND_MASK | RXE_END_MASK, .length = RXE_BTH_BYTES + RXE_IETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_IETH] = RXE_BTH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_IETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_IETH_BYTES, } }, [IB_OPCODE_RC_SEND_ONLY_WITH_INVALIDATE] = { .name = "IB_OPCODE_RC_SEND_ONLY_INV", - .mask = RXE_IETH_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK - | RXE_COMP_MASK | RXE_RWR_MASK | RXE_SEND_MASK - | RXE_END_MASK | RXE_START_MASK, + .mask = RXE_IETH_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK | + RXE_COMP_MASK | RXE_RWR_MASK | RXE_SEND_MASK | + RXE_END_MASK | RXE_START_MASK, .length = RXE_BTH_BYTES + RXE_IETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_IETH] = RXE_BTH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_IETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_IETH_BYTES, } }, /* UC */ [IB_OPCODE_UC_SEND_FIRST] = { .name = "IB_OPCODE_UC_SEND_FIRST", - .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_RWR_MASK - | RXE_SEND_MASK | RXE_START_MASK, + .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_RWR_MASK | + RXE_SEND_MASK | RXE_START_MASK, .length = RXE_BTH_BYTES, .offset = { [RXE_BTH] = 0, @@ -393,8 +394,8 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { }, [IB_OPCODE_UC_SEND_MIDDLE] = { .name = "IB_OPCODE_UC_SEND_MIDDLE", - .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_SEND_MASK - | RXE_MIDDLE_MASK, + .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_SEND_MASK | + RXE_MIDDLE_MASK, .length = RXE_BTH_BYTES, .offset = { [RXE_BTH] = 0, @@ -403,8 +404,8 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { }, [IB_OPCODE_UC_SEND_LAST] = { .name = "IB_OPCODE_UC_SEND_LAST", - .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_COMP_MASK - | RXE_SEND_MASK | RXE_END_MASK, + .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_COMP_MASK | + RXE_SEND_MASK | RXE_END_MASK, .length = RXE_BTH_BYTES, .offset = { [RXE_BTH] = 0, @@ -413,21 +414,21 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { }, [IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE] = { .name = "IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE", - .mask = RXE_IMMDT_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK - | RXE_COMP_MASK | RXE_SEND_MASK | RXE_END_MASK, + .mask = RXE_IMMDT_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK | + RXE_COMP_MASK | RXE_SEND_MASK | RXE_END_MASK, .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES, .offset = { [RXE_BTH] = 0, [RXE_IMMDT] = RXE_BTH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_IMMDT_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_IMMDT_BYTES, } }, [IB_OPCODE_UC_SEND_ONLY] = { .name = "IB_OPCODE_UC_SEND_ONLY", - .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_COMP_MASK - | RXE_RWR_MASK | RXE_SEND_MASK - | RXE_START_MASK | RXE_END_MASK, + .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_COMP_MASK | + RXE_RWR_MASK | RXE_SEND_MASK | + RXE_START_MASK | RXE_END_MASK, .length = RXE_BTH_BYTES, .offset = { [RXE_BTH] = 0, @@ -436,33 +437,33 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { }, [IB_OPCODE_UC_SEND_ONLY_WITH_IMMEDIATE] = { .name = "IB_OPCODE_UC_SEND_ONLY_WITH_IMMEDIATE", - .mask = RXE_IMMDT_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK - | RXE_COMP_MASK | RXE_RWR_MASK | RXE_SEND_MASK - | RXE_START_MASK | RXE_END_MASK, + .mask = RXE_IMMDT_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK | + RXE_COMP_MASK | RXE_RWR_MASK | RXE_SEND_MASK | + RXE_START_MASK | RXE_END_MASK, .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES, .offset = { [RXE_BTH] = 0, [RXE_IMMDT] = RXE_BTH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_IMMDT_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_IMMDT_BYTES, } }, [IB_OPCODE_UC_RDMA_WRITE_FIRST] = { .name = "IB_OPCODE_UC_RDMA_WRITE_FIRST", - .mask = RXE_RETH_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK - | RXE_WRITE_MASK | RXE_START_MASK, + .mask = RXE_RETH_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK | + RXE_WRITE_MASK | RXE_START_MASK, .length = RXE_BTH_BYTES + RXE_RETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_RETH] = RXE_BTH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_RETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_RETH_BYTES, } }, [IB_OPCODE_UC_RDMA_WRITE_MIDDLE] = { .name = "IB_OPCODE_UC_RDMA_WRITE_MIDDLE", - .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_WRITE_MASK - | RXE_MIDDLE_MASK, + .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_WRITE_MASK | + RXE_MIDDLE_MASK, .length = RXE_BTH_BYTES, .offset = { [RXE_BTH] = 0, @@ -471,8 +472,8 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { }, [IB_OPCODE_UC_RDMA_WRITE_LAST] = { .name = "IB_OPCODE_UC_RDMA_WRITE_LAST", - .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_WRITE_MASK - | RXE_END_MASK, + .mask = RXE_PAYLOAD_MASK | RXE_REQ_MASK | RXE_WRITE_MASK | + RXE_END_MASK, .length = RXE_BTH_BYTES, .offset = { [RXE_BTH] = 0, @@ -481,460 +482,460 @@ struct rxe_opcode_info rxe_opcode[RXE_NUM_OPCODE] = { }, [IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE] = { .name = "IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE", - .mask = RXE_IMMDT_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK - | RXE_WRITE_MASK | RXE_COMP_MASK | RXE_RWR_MASK - | RXE_END_MASK, + .mask = RXE_IMMDT_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK | + RXE_WRITE_MASK | RXE_COMP_MASK | RXE_RWR_MASK | + RXE_END_MASK, .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES, .offset = { [RXE_BTH] = 0, [RXE_IMMDT] = RXE_BTH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_IMMDT_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_IMMDT_BYTES, } }, [IB_OPCODE_UC_RDMA_WRITE_ONLY] = { .name = "IB_OPCODE_UC_RDMA_WRITE_ONLY", - .mask = RXE_RETH_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK - | RXE_WRITE_MASK | RXE_START_MASK - | RXE_END_MASK, + .mask = RXE_RETH_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK | + RXE_WRITE_MASK | RXE_START_MASK | + RXE_END_MASK, .length = RXE_BTH_BYTES + RXE_RETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_RETH] = RXE_BTH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_RETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_RETH_BYTES, } }, [IB_OPCODE_UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE] = { .name = "IB_OPCODE_UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE", - .mask = RXE_RETH_MASK | RXE_IMMDT_MASK | RXE_PAYLOAD_MASK - | RXE_REQ_MASK | RXE_WRITE_MASK - | RXE_COMP_MASK | RXE_RWR_MASK - | RXE_START_MASK | RXE_END_MASK, + .mask = RXE_RETH_MASK | RXE_IMMDT_MASK | RXE_PAYLOAD_MASK | + RXE_REQ_MASK | RXE_WRITE_MASK | + RXE_COMP_MASK | RXE_RWR_MASK | + RXE_START_MASK | RXE_END_MASK, .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES + RXE_RETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_RETH] = RXE_BTH_BYTES, - [RXE_IMMDT] = RXE_BTH_BYTES - + RXE_RETH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_RETH_BYTES - + RXE_IMMDT_BYTES, + [RXE_IMMDT] = RXE_BTH_BYTES + + RXE_RETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_RETH_BYTES + + RXE_IMMDT_BYTES, } }, /* RD */ [IB_OPCODE_RD_SEND_FIRST] = { .name = "IB_OPCODE_RD_SEND_FIRST", - .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_PAYLOAD_MASK - | RXE_REQ_MASK | RXE_RWR_MASK | RXE_SEND_MASK - | RXE_START_MASK, + .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_PAYLOAD_MASK | + RXE_REQ_MASK | RXE_RWR_MASK | RXE_SEND_MASK | + RXE_START_MASK, .length = RXE_BTH_BYTES + RXE_DETH_BYTES + RXE_RDETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_RDETH] = RXE_BTH_BYTES, - [RXE_DETH] = RXE_BTH_BYTES - + RXE_RDETH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_RDETH_BYTES - + RXE_DETH_BYTES, + [RXE_DETH] = RXE_BTH_BYTES + + RXE_RDETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_RDETH_BYTES + + RXE_DETH_BYTES, } }, [IB_OPCODE_RD_SEND_MIDDLE] = { .name = "IB_OPCODE_RD_SEND_MIDDLE", - .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_PAYLOAD_MASK - | RXE_REQ_MASK | RXE_SEND_MASK - | RXE_MIDDLE_MASK, + .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_PAYLOAD_MASK | + RXE_REQ_MASK | RXE_SEND_MASK | + RXE_MIDDLE_MASK, .length = RXE_BTH_BYTES + RXE_DETH_BYTES + RXE_RDETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_RDETH] = RXE_BTH_BYTES, - [RXE_DETH] = RXE_BTH_BYTES - + RXE_RDETH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_RDETH_BYTES - + RXE_DETH_BYTES, + [RXE_DETH] = RXE_BTH_BYTES + + RXE_RDETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_RDETH_BYTES + + RXE_DETH_BYTES, } }, [IB_OPCODE_RD_SEND_LAST] = { .name = "IB_OPCODE_RD_SEND_LAST", - .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_PAYLOAD_MASK - | RXE_REQ_MASK | RXE_COMP_MASK | RXE_SEND_MASK - | RXE_END_MASK, + .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_PAYLOAD_MASK | + RXE_REQ_MASK | RXE_COMP_MASK | RXE_SEND_MASK | + RXE_END_MASK, .length = RXE_BTH_BYTES + RXE_DETH_BYTES + RXE_RDETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_RDETH] = RXE_BTH_BYTES, - [RXE_DETH] = RXE_BTH_BYTES - + RXE_RDETH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_RDETH_BYTES - + RXE_DETH_BYTES, + [RXE_DETH] = RXE_BTH_BYTES + + RXE_RDETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_RDETH_BYTES + + RXE_DETH_BYTES, } }, [IB_OPCODE_RD_SEND_LAST_WITH_IMMEDIATE] = { .name = "IB_OPCODE_RD_SEND_LAST_WITH_IMMEDIATE", - .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_IMMDT_MASK - | RXE_PAYLOAD_MASK | RXE_REQ_MASK - | RXE_COMP_MASK | RXE_SEND_MASK - | RXE_END_MASK, - .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES + RXE_DETH_BYTES - + RXE_RDETH_BYTES, + .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_IMMDT_MASK | + RXE_PAYLOAD_MASK | RXE_REQ_MASK | + RXE_COMP_MASK | RXE_SEND_MASK | + RXE_END_MASK, + .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES + RXE_DETH_BYTES + + RXE_RDETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_RDETH] = RXE_BTH_BYTES, - [RXE_DETH] = RXE_BTH_BYTES - + RXE_RDETH_BYTES, - [RXE_IMMDT] = RXE_BTH_BYTES - + RXE_RDETH_BYTES - + RXE_DETH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_RDETH_BYTES - + RXE_DETH_BYTES - + RXE_IMMDT_BYTES, + [RXE_DETH] = RXE_BTH_BYTES + + RXE_RDETH_BYTES, + [RXE_IMMDT] = RXE_BTH_BYTES + + RXE_RDETH_BYTES + + RXE_DETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_RDETH_BYTES + + RXE_DETH_BYTES + + RXE_IMMDT_BYTES, } }, [IB_OPCODE_RD_SEND_ONLY] = { .name = "IB_OPCODE_RD_SEND_ONLY", - .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_PAYLOAD_MASK - | RXE_REQ_MASK | RXE_COMP_MASK | RXE_RWR_MASK - | RXE_SEND_MASK | RXE_START_MASK | RXE_END_MASK, + .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_PAYLOAD_MASK | + RXE_REQ_MASK | RXE_COMP_MASK | RXE_RWR_MASK | + RXE_SEND_MASK | RXE_START_MASK | RXE_END_MASK, .length = RXE_BTH_BYTES + RXE_DETH_BYTES + RXE_RDETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_RDETH] = RXE_BTH_BYTES, - [RXE_DETH] = RXE_BTH_BYTES - + RXE_RDETH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_RDETH_BYTES - + RXE_DETH_BYTES, + [RXE_DETH] = RXE_BTH_BYTES + + RXE_RDETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_RDETH_BYTES + + RXE_DETH_BYTES, } }, [IB_OPCODE_RD_SEND_ONLY_WITH_IMMEDIATE] = { .name = "IB_OPCODE_RD_SEND_ONLY_WITH_IMMEDIATE", - .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_IMMDT_MASK - | RXE_PAYLOAD_MASK | RXE_REQ_MASK - | RXE_COMP_MASK | RXE_RWR_MASK | RXE_SEND_MASK - | RXE_START_MASK | RXE_END_MASK, - .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES + RXE_DETH_BYTES - + RXE_RDETH_BYTES, + .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_IMMDT_MASK | + RXE_PAYLOAD_MASK | RXE_REQ_MASK | + RXE_COMP_MASK | RXE_RWR_MASK | RXE_SEND_MASK | + RXE_START_MASK | RXE_END_MASK, + .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES + RXE_DETH_BYTES + + RXE_RDETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_RDETH] = RXE_BTH_BYTES, - [RXE_DETH] = RXE_BTH_BYTES - + RXE_RDETH_BYTES, - [RXE_IMMDT] = RXE_BTH_BYTES - + RXE_RDETH_BYTES - + RXE_DETH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_RDETH_BYTES - + RXE_DETH_BYTES - + RXE_IMMDT_BYTES, + [RXE_DETH] = RXE_BTH_BYTES + + RXE_RDETH_BYTES, + [RXE_IMMDT] = RXE_BTH_BYTES + + RXE_RDETH_BYTES + + RXE_DETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_RDETH_BYTES + + RXE_DETH_BYTES + + RXE_IMMDT_BYTES, } }, [IB_OPCODE_RD_RDMA_WRITE_FIRST] = { .name = "IB_OPCODE_RD_RDMA_WRITE_FIRST", - .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_RETH_MASK - | RXE_PAYLOAD_MASK | RXE_REQ_MASK - | RXE_WRITE_MASK | RXE_START_MASK, - .length = RXE_BTH_BYTES + RXE_RETH_BYTES + RXE_DETH_BYTES - + RXE_RDETH_BYTES, + .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_RETH_MASK | + RXE_PAYLOAD_MASK | RXE_REQ_MASK | + RXE_WRITE_MASK | RXE_START_MASK, + .length = RXE_BTH_BYTES + RXE_RETH_BYTES + RXE_DETH_BYTES + + RXE_RDETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_RDETH] = RXE_BTH_BYTES, - [RXE_DETH] = RXE_BTH_BYTES - + RXE_RDETH_BYTES, - [RXE_RETH] = RXE_BTH_BYTES - + RXE_RDETH_BYTES - + RXE_DETH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_RDETH_BYTES - + RXE_DETH_BYTES - + RXE_RETH_BYTES, + [RXE_DETH] = RXE_BTH_BYTES + + RXE_RDETH_BYTES, + [RXE_RETH] = RXE_BTH_BYTES + + RXE_RDETH_BYTES + + RXE_DETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_RDETH_BYTES + + RXE_DETH_BYTES + + RXE_RETH_BYTES, } }, [IB_OPCODE_RD_RDMA_WRITE_MIDDLE] = { .name = "IB_OPCODE_RD_RDMA_WRITE_MIDDLE", - .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_PAYLOAD_MASK - | RXE_REQ_MASK | RXE_WRITE_MASK - | RXE_MIDDLE_MASK, + .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_PAYLOAD_MASK | + RXE_REQ_MASK | RXE_WRITE_MASK | + RXE_MIDDLE_MASK, .length = RXE_BTH_BYTES + RXE_DETH_BYTES + RXE_RDETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_RDETH] = RXE_BTH_BYTES, - [RXE_DETH] = RXE_BTH_BYTES - + RXE_RDETH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_RDETH_BYTES - + RXE_DETH_BYTES, + [RXE_DETH] = RXE_BTH_BYTES + + RXE_RDETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_RDETH_BYTES + + RXE_DETH_BYTES, } }, [IB_OPCODE_RD_RDMA_WRITE_LAST] = { .name = "IB_OPCODE_RD_RDMA_WRITE_LAST", - .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_PAYLOAD_MASK - | RXE_REQ_MASK | RXE_WRITE_MASK - | RXE_END_MASK, + .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_PAYLOAD_MASK | + RXE_REQ_MASK | RXE_WRITE_MASK | + RXE_END_MASK, .length = RXE_BTH_BYTES + RXE_DETH_BYTES + RXE_RDETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_RDETH] = RXE_BTH_BYTES, - [RXE_DETH] = RXE_BTH_BYTES - + RXE_RDETH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_RDETH_BYTES - + RXE_DETH_BYTES, + [RXE_DETH] = RXE_BTH_BYTES + + RXE_RDETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_RDETH_BYTES + + RXE_DETH_BYTES, } }, [IB_OPCODE_RD_RDMA_WRITE_LAST_WITH_IMMEDIATE] = { .name = "IB_OPCODE_RD_RDMA_WRITE_LAST_WITH_IMMEDIATE", - .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_IMMDT_MASK - | RXE_PAYLOAD_MASK | RXE_REQ_MASK - | RXE_WRITE_MASK | RXE_COMP_MASK | RXE_RWR_MASK - | RXE_END_MASK, - .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES + RXE_DETH_BYTES - + RXE_RDETH_BYTES, + .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_IMMDT_MASK | + RXE_PAYLOAD_MASK | RXE_REQ_MASK | + RXE_WRITE_MASK | RXE_COMP_MASK | RXE_RWR_MASK | + RXE_END_MASK, + .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES + RXE_DETH_BYTES + + RXE_RDETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_RDETH] = RXE_BTH_BYTES, - [RXE_DETH] = RXE_BTH_BYTES - + RXE_RDETH_BYTES, - [RXE_IMMDT] = RXE_BTH_BYTES - + RXE_RDETH_BYTES - + RXE_DETH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_RDETH_BYTES - + RXE_DETH_BYTES - + RXE_IMMDT_BYTES, + [RXE_DETH] = RXE_BTH_BYTES + + RXE_RDETH_BYTES, + [RXE_IMMDT] = RXE_BTH_BYTES + + RXE_RDETH_BYTES + + RXE_DETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_RDETH_BYTES + + RXE_DETH_BYTES + + RXE_IMMDT_BYTES, } }, [IB_OPCODE_RD_RDMA_WRITE_ONLY] = { .name = "IB_OPCODE_RD_RDMA_WRITE_ONLY", - .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_RETH_MASK - | RXE_PAYLOAD_MASK | RXE_REQ_MASK - | RXE_WRITE_MASK | RXE_START_MASK - | RXE_END_MASK, - .length = RXE_BTH_BYTES + RXE_RETH_BYTES + RXE_DETH_BYTES - + RXE_RDETH_BYTES, + .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_RETH_MASK | + RXE_PAYLOAD_MASK | RXE_REQ_MASK | + RXE_WRITE_MASK | RXE_START_MASK | + RXE_END_MASK, + .length = RXE_BTH_BYTES + RXE_RETH_BYTES + RXE_DETH_BYTES + + RXE_RDETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_RDETH] = RXE_BTH_BYTES, - [RXE_DETH] = RXE_BTH_BYTES - + RXE_RDETH_BYTES, - [RXE_RETH] = RXE_BTH_BYTES - + RXE_RDETH_BYTES - + RXE_DETH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_RDETH_BYTES - + RXE_DETH_BYTES - + RXE_RETH_BYTES, + [RXE_DETH] = RXE_BTH_BYTES + + RXE_RDETH_BYTES, + [RXE_RETH] = RXE_BTH_BYTES + + RXE_RDETH_BYTES + + RXE_DETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_RDETH_BYTES + + RXE_DETH_BYTES + + RXE_RETH_BYTES, } }, [IB_OPCODE_RD_RDMA_WRITE_ONLY_WITH_IMMEDIATE] = { .name = "IB_OPCODE_RD_RDMA_WRITE_ONLY_WITH_IMMEDIATE", - .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_RETH_MASK - | RXE_IMMDT_MASK | RXE_PAYLOAD_MASK - | RXE_REQ_MASK | RXE_WRITE_MASK - | RXE_COMP_MASK | RXE_RWR_MASK - | RXE_START_MASK | RXE_END_MASK, - .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES + RXE_RETH_BYTES - + RXE_DETH_BYTES + RXE_RDETH_BYTES, + .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_RETH_MASK | + RXE_IMMDT_MASK | RXE_PAYLOAD_MASK | + RXE_REQ_MASK | RXE_WRITE_MASK | + RXE_COMP_MASK | RXE_RWR_MASK | + RXE_START_MASK | RXE_END_MASK, + .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES + RXE_RETH_BYTES + + RXE_DETH_BYTES + RXE_RDETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_RDETH] = RXE_BTH_BYTES, - [RXE_DETH] = RXE_BTH_BYTES - + RXE_RDETH_BYTES, - [RXE_RETH] = RXE_BTH_BYTES - + RXE_RDETH_BYTES - + RXE_DETH_BYTES, - [RXE_IMMDT] = RXE_BTH_BYTES - + RXE_RDETH_BYTES - + RXE_DETH_BYTES - + RXE_RETH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_RDETH_BYTES - + RXE_DETH_BYTES - + RXE_RETH_BYTES - + RXE_IMMDT_BYTES, + [RXE_DETH] = RXE_BTH_BYTES + + RXE_RDETH_BYTES, + [RXE_RETH] = RXE_BTH_BYTES + + RXE_RDETH_BYTES + + RXE_DETH_BYTES, + [RXE_IMMDT] = RXE_BTH_BYTES + + RXE_RDETH_BYTES + + RXE_DETH_BYTES + + RXE_RETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_RDETH_BYTES + + RXE_DETH_BYTES + + RXE_RETH_BYTES + + RXE_IMMDT_BYTES, } }, [IB_OPCODE_RD_RDMA_READ_REQUEST] = { .name = "IB_OPCODE_RD_RDMA_READ_REQUEST", - .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_RETH_MASK - | RXE_REQ_MASK | RXE_READ_MASK - | RXE_START_MASK | RXE_END_MASK, - .length = RXE_BTH_BYTES + RXE_RETH_BYTES + RXE_DETH_BYTES - + RXE_RDETH_BYTES, + .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_RETH_MASK | + RXE_REQ_MASK | RXE_READ_MASK | + RXE_START_MASK | RXE_END_MASK, + .length = RXE_BTH_BYTES + RXE_RETH_BYTES + RXE_DETH_BYTES + + RXE_RDETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_RDETH] = RXE_BTH_BYTES, - [RXE_DETH] = RXE_BTH_BYTES - + RXE_RDETH_BYTES, - [RXE_RETH] = RXE_BTH_BYTES - + RXE_RDETH_BYTES - + RXE_DETH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_RETH_BYTES - + RXE_DETH_BYTES - + RXE_RDETH_BYTES, + [RXE_DETH] = RXE_BTH_BYTES + + RXE_RDETH_BYTES, + [RXE_RETH] = RXE_BTH_BYTES + + RXE_RDETH_BYTES + + RXE_DETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_RETH_BYTES + + RXE_DETH_BYTES + + RXE_RDETH_BYTES, } }, [IB_OPCODE_RD_RDMA_READ_RESPONSE_FIRST] = { .name = "IB_OPCODE_RD_RDMA_READ_RESPONSE_FIRST", - .mask = RXE_RDETH_MASK | RXE_AETH_MASK - | RXE_PAYLOAD_MASK | RXE_ACK_MASK - | RXE_START_MASK, + .mask = RXE_RDETH_MASK | RXE_AETH_MASK | + RXE_PAYLOAD_MASK | RXE_ACK_MASK | + RXE_START_MASK, .length = RXE_BTH_BYTES + RXE_AETH_BYTES + RXE_RDETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_RDETH] = RXE_BTH_BYTES, - [RXE_AETH] = RXE_BTH_BYTES - + RXE_RDETH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_RDETH_BYTES - + RXE_AETH_BYTES, + [RXE_AETH] = RXE_BTH_BYTES + + RXE_RDETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_RDETH_BYTES + + RXE_AETH_BYTES, } }, [IB_OPCODE_RD_RDMA_READ_RESPONSE_MIDDLE] = { .name = "IB_OPCODE_RD_RDMA_READ_RESPONSE_MIDDLE", - .mask = RXE_RDETH_MASK | RXE_PAYLOAD_MASK | RXE_ACK_MASK - | RXE_MIDDLE_MASK, + .mask = RXE_RDETH_MASK | RXE_PAYLOAD_MASK | RXE_ACK_MASK | + RXE_MIDDLE_MASK, .length = RXE_BTH_BYTES + RXE_RDETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_RDETH] = RXE_BTH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_RDETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_RDETH_BYTES, } }, [IB_OPCODE_RD_RDMA_READ_RESPONSE_LAST] = { .name = "IB_OPCODE_RD_RDMA_READ_RESPONSE_LAST", - .mask = RXE_RDETH_MASK | RXE_AETH_MASK | RXE_PAYLOAD_MASK - | RXE_ACK_MASK | RXE_END_MASK, + .mask = RXE_RDETH_MASK | RXE_AETH_MASK | RXE_PAYLOAD_MASK | + RXE_ACK_MASK | RXE_END_MASK, .length = RXE_BTH_BYTES + RXE_AETH_BYTES + RXE_RDETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_RDETH] = RXE_BTH_BYTES, - [RXE_AETH] = RXE_BTH_BYTES - + RXE_RDETH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_RDETH_BYTES - + RXE_AETH_BYTES, + [RXE_AETH] = RXE_BTH_BYTES + + RXE_RDETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_RDETH_BYTES + + RXE_AETH_BYTES, } }, [IB_OPCODE_RD_RDMA_READ_RESPONSE_ONLY] = { .name = "IB_OPCODE_RD_RDMA_READ_RESPONSE_ONLY", - .mask = RXE_RDETH_MASK | RXE_AETH_MASK | RXE_PAYLOAD_MASK - | RXE_ACK_MASK | RXE_START_MASK | RXE_END_MASK, + .mask = RXE_RDETH_MASK | RXE_AETH_MASK | RXE_PAYLOAD_MASK | + RXE_ACK_MASK | RXE_START_MASK | RXE_END_MASK, .length = RXE_BTH_BYTES + RXE_AETH_BYTES + RXE_RDETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_RDETH] = RXE_BTH_BYTES, - [RXE_AETH] = RXE_BTH_BYTES - + RXE_RDETH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_RDETH_BYTES - + RXE_AETH_BYTES, + [RXE_AETH] = RXE_BTH_BYTES + + RXE_RDETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_RDETH_BYTES + + RXE_AETH_BYTES, } }, [IB_OPCODE_RD_ACKNOWLEDGE] = { .name = "IB_OPCODE_RD_ACKNOWLEDGE", - .mask = RXE_RDETH_MASK | RXE_AETH_MASK | RXE_ACK_MASK - | RXE_START_MASK | RXE_END_MASK, + .mask = RXE_RDETH_MASK | RXE_AETH_MASK | RXE_ACK_MASK | + RXE_START_MASK | RXE_END_MASK, .length = RXE_BTH_BYTES + RXE_AETH_BYTES + RXE_RDETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_RDETH] = RXE_BTH_BYTES, - [RXE_AETH] = RXE_BTH_BYTES - + RXE_RDETH_BYTES, + [RXE_AETH] = RXE_BTH_BYTES + + RXE_RDETH_BYTES, } }, [IB_OPCODE_RD_ATOMIC_ACKNOWLEDGE] = { .name = "IB_OPCODE_RD_ATOMIC_ACKNOWLEDGE", - .mask = RXE_RDETH_MASK | RXE_AETH_MASK | RXE_ATMACK_MASK - | RXE_ACK_MASK | RXE_START_MASK | RXE_END_MASK, - .length = RXE_BTH_BYTES + RXE_ATMACK_BYTES + RXE_AETH_BYTES - + RXE_RDETH_BYTES, + .mask = RXE_RDETH_MASK | RXE_AETH_MASK | RXE_ATMACK_MASK | + RXE_ACK_MASK | RXE_START_MASK | RXE_END_MASK, + .length = RXE_BTH_BYTES + RXE_ATMACK_BYTES + RXE_AETH_BYTES + + RXE_RDETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_RDETH] = RXE_BTH_BYTES, - [RXE_AETH] = RXE_BTH_BYTES - + RXE_RDETH_BYTES, - [RXE_ATMACK] = RXE_BTH_BYTES - + RXE_RDETH_BYTES - + RXE_AETH_BYTES, + [RXE_AETH] = RXE_BTH_BYTES + + RXE_RDETH_BYTES, + [RXE_ATMACK] = RXE_BTH_BYTES + + RXE_RDETH_BYTES + + RXE_AETH_BYTES, } }, [IB_OPCODE_RD_COMPARE_SWAP] = { .name = "RD_COMPARE_SWAP", - .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_ATMETH_MASK - | RXE_REQ_MASK | RXE_ATOMIC_MASK - | RXE_START_MASK | RXE_END_MASK, - .length = RXE_BTH_BYTES + RXE_ATMETH_BYTES + RXE_DETH_BYTES - + RXE_RDETH_BYTES, + .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_ATMETH_MASK | + RXE_REQ_MASK | RXE_ATOMIC_MASK | + RXE_START_MASK | RXE_END_MASK, + .length = RXE_BTH_BYTES + RXE_ATMETH_BYTES + RXE_DETH_BYTES + + RXE_RDETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_RDETH] = RXE_BTH_BYTES, - [RXE_DETH] = RXE_BTH_BYTES - + RXE_RDETH_BYTES, - [RXE_ATMETH] = RXE_BTH_BYTES - + RXE_RDETH_BYTES - + RXE_DETH_BYTES, + [RXE_DETH] = RXE_BTH_BYTES + + RXE_RDETH_BYTES, + [RXE_ATMETH] = RXE_BTH_BYTES + + RXE_RDETH_BYTES + + RXE_DETH_BYTES, [RXE_PAYLOAD] = RXE_BTH_BYTES + - + RXE_ATMETH_BYTES - + RXE_DETH_BYTES + - + RXE_RDETH_BYTES, + RXE_ATMETH_BYTES + + RXE_DETH_BYTES + + RXE_RDETH_BYTES, } }, [IB_OPCODE_RD_FETCH_ADD] = { .name = "IB_OPCODE_RD_FETCH_ADD", - .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_ATMETH_MASK - | RXE_REQ_MASK | RXE_ATOMIC_MASK - | RXE_START_MASK | RXE_END_MASK, - .length = RXE_BTH_BYTES + RXE_ATMETH_BYTES + RXE_DETH_BYTES - + RXE_RDETH_BYTES, + .mask = RXE_RDETH_MASK | RXE_DETH_MASK | RXE_ATMETH_MASK | + RXE_REQ_MASK | RXE_ATOMIC_MASK | + RXE_START_MASK | RXE_END_MASK, + .length = RXE_BTH_BYTES + RXE_ATMETH_BYTES + RXE_DETH_BYTES + + RXE_RDETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_RDETH] = RXE_BTH_BYTES, - [RXE_DETH] = RXE_BTH_BYTES - + RXE_RDETH_BYTES, - [RXE_ATMETH] = RXE_BTH_BYTES - + RXE_RDETH_BYTES - + RXE_DETH_BYTES, + [RXE_DETH] = RXE_BTH_BYTES + + RXE_RDETH_BYTES, + [RXE_ATMETH] = RXE_BTH_BYTES + + RXE_RDETH_BYTES + + RXE_DETH_BYTES, [RXE_PAYLOAD] = RXE_BTH_BYTES + - + RXE_ATMETH_BYTES - + RXE_DETH_BYTES + - + RXE_RDETH_BYTES, + RXE_ATMETH_BYTES + + RXE_DETH_BYTES + + RXE_RDETH_BYTES, } }, /* UD */ [IB_OPCODE_UD_SEND_ONLY] = { .name = "IB_OPCODE_UD_SEND_ONLY", - .mask = RXE_DETH_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK - | RXE_COMP_MASK | RXE_RWR_MASK | RXE_SEND_MASK - | RXE_START_MASK | RXE_END_MASK, + .mask = RXE_DETH_MASK | RXE_PAYLOAD_MASK | RXE_REQ_MASK | + RXE_COMP_MASK | RXE_RWR_MASK | RXE_SEND_MASK | + RXE_START_MASK | RXE_END_MASK, .length = RXE_BTH_BYTES + RXE_DETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_DETH] = RXE_BTH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_DETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_DETH_BYTES, } }, [IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE] = { .name = "IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE", - .mask = RXE_DETH_MASK | RXE_IMMDT_MASK | RXE_PAYLOAD_MASK - | RXE_REQ_MASK | RXE_COMP_MASK | RXE_RWR_MASK - | RXE_SEND_MASK | RXE_START_MASK | RXE_END_MASK, + .mask = RXE_DETH_MASK | RXE_IMMDT_MASK | RXE_PAYLOAD_MASK | + RXE_REQ_MASK | RXE_COMP_MASK | RXE_RWR_MASK | + RXE_SEND_MASK | RXE_START_MASK | RXE_END_MASK, .length = RXE_BTH_BYTES + RXE_IMMDT_BYTES + RXE_DETH_BYTES, .offset = { [RXE_BTH] = 0, [RXE_DETH] = RXE_BTH_BYTES, - [RXE_IMMDT] = RXE_BTH_BYTES - + RXE_DETH_BYTES, - [RXE_PAYLOAD] = RXE_BTH_BYTES - + RXE_DETH_BYTES - + RXE_IMMDT_BYTES, + [RXE_IMMDT] = RXE_BTH_BYTES + + RXE_DETH_BYTES, + [RXE_PAYLOAD] = RXE_BTH_BYTES + + RXE_DETH_BYTES + + RXE_IMMDT_BYTES, } }, diff --git a/drivers/infiniband/sw/rxe/rxe_pool.c b/drivers/infiniband/sw/rxe/rxe_pool.c index 2e80bb6aa957..4cb003885e00 100644 --- a/drivers/infiniband/sw/rxe/rxe_pool.c +++ b/drivers/infiniband/sw/rxe/rxe_pool.c @@ -5,13 +5,14 @@ */ #include "rxe.h" -#include "rxe_loc.h" + +#define RXE_POOL_ALIGN (16) static const struct rxe_type_info { const char *name; size_t size; size_t elem_offset; - void (*cleanup)(struct rxe_pool_entry *obj); + void (*cleanup)(struct rxe_pool_elem *obj); enum rxe_pool_flags flags; u32 min_index; u32 max_index; @@ -21,19 +22,19 @@ static const struct rxe_type_info { [RXE_TYPE_UC] = { .name = "rxe-uc", .size = sizeof(struct rxe_ucontext), - .elem_offset = offsetof(struct rxe_ucontext, pelem), + .elem_offset = offsetof(struct rxe_ucontext, elem), .flags = RXE_POOL_NO_ALLOC, }, [RXE_TYPE_PD] = { .name = "rxe-pd", .size = sizeof(struct rxe_pd), - .elem_offset = offsetof(struct rxe_pd, pelem), + .elem_offset = offsetof(struct rxe_pd, elem), .flags = RXE_POOL_NO_ALLOC, }, [RXE_TYPE_AH] = { .name = "rxe-ah", .size = sizeof(struct rxe_ah), - .elem_offset = offsetof(struct rxe_ah, pelem), + .elem_offset = offsetof(struct rxe_ah, elem), .flags = RXE_POOL_INDEX | RXE_POOL_NO_ALLOC, .min_index = RXE_MIN_AH_INDEX, .max_index = RXE_MAX_AH_INDEX, @@ -41,7 +42,7 @@ static const struct rxe_type_info { [RXE_TYPE_SRQ] = { .name = "rxe-srq", .size = sizeof(struct rxe_srq), - .elem_offset = offsetof(struct rxe_srq, pelem), + .elem_offset = offsetof(struct rxe_srq, elem), .flags = RXE_POOL_INDEX | RXE_POOL_NO_ALLOC, .min_index = RXE_MIN_SRQ_INDEX, .max_index = RXE_MAX_SRQ_INDEX, @@ -49,7 +50,7 @@ static const struct rxe_type_info { [RXE_TYPE_QP] = { .name = "rxe-qp", .size = sizeof(struct rxe_qp), - .elem_offset = offsetof(struct rxe_qp, pelem), + .elem_offset = offsetof(struct rxe_qp, elem), .cleanup = rxe_qp_cleanup, .flags = RXE_POOL_INDEX | RXE_POOL_NO_ALLOC, .min_index = RXE_MIN_QP_INDEX, @@ -58,14 +59,14 @@ static const struct rxe_type_info { [RXE_TYPE_CQ] = { .name = "rxe-cq", .size = sizeof(struct rxe_cq), - .elem_offset = offsetof(struct rxe_cq, pelem), + .elem_offset = offsetof(struct rxe_cq, elem), .flags = RXE_POOL_NO_ALLOC, .cleanup = rxe_cq_cleanup, }, [RXE_TYPE_MR] = { .name = "rxe-mr", .size = sizeof(struct rxe_mr), - .elem_offset = offsetof(struct rxe_mr, pelem), + .elem_offset = offsetof(struct rxe_mr, elem), .cleanup = rxe_mr_cleanup, .flags = RXE_POOL_INDEX, .min_index = RXE_MIN_MR_INDEX, @@ -74,7 +75,7 @@ static const struct rxe_type_info { [RXE_TYPE_MW] = { .name = "rxe-mw", .size = sizeof(struct rxe_mw), - .elem_offset = offsetof(struct rxe_mw, pelem), + .elem_offset = offsetof(struct rxe_mw, elem), .cleanup = rxe_mw_cleanup, .flags = RXE_POOL_INDEX | RXE_POOL_NO_ALLOC, .min_index = RXE_MIN_MW_INDEX, @@ -83,7 +84,7 @@ static const struct rxe_type_info { [RXE_TYPE_MC_GRP] = { .name = "rxe-mc_grp", .size = sizeof(struct rxe_mc_grp), - .elem_offset = offsetof(struct rxe_mc_grp, pelem), + .elem_offset = offsetof(struct rxe_mc_grp, elem), .cleanup = rxe_mc_cleanup, .flags = RXE_POOL_KEY, .key_offset = offsetof(struct rxe_mc_grp, mgid), @@ -92,15 +93,10 @@ static const struct rxe_type_info { [RXE_TYPE_MC_ELEM] = { .name = "rxe-mc_elem", .size = sizeof(struct rxe_mc_elem), - .elem_offset = offsetof(struct rxe_mc_elem, pelem), + .elem_offset = offsetof(struct rxe_mc_elem, elem), }, }; -static inline const char *pool_name(struct rxe_pool *pool) -{ - return rxe_type_info[pool->type].name; -} - static int rxe_pool_init_index(struct rxe_pool *pool, u32 max, u32 min) { int err = 0; @@ -130,35 +126,36 @@ int rxe_pool_init( enum rxe_elem_type type, unsigned int max_elem) { + const struct rxe_type_info *info = &rxe_type_info[type]; int err = 0; - size_t size = rxe_type_info[type].size; memset(pool, 0, sizeof(*pool)); pool->rxe = rxe; + pool->name = info->name; pool->type = type; pool->max_elem = max_elem; - pool->elem_size = ALIGN(size, RXE_POOL_ALIGN); - pool->flags = rxe_type_info[type].flags; - pool->index.tree = RB_ROOT; - pool->key.tree = RB_ROOT; - pool->cleanup = rxe_type_info[type].cleanup; + pool->elem_size = ALIGN(info->size, RXE_POOL_ALIGN); + pool->elem_offset = info->elem_offset; + pool->flags = info->flags; + pool->cleanup = info->cleanup; atomic_set(&pool->num_elem, 0); rwlock_init(&pool->pool_lock); - if (rxe_type_info[type].flags & RXE_POOL_INDEX) { - err = rxe_pool_init_index(pool, - rxe_type_info[type].max_index, - rxe_type_info[type].min_index); + if (pool->flags & RXE_POOL_INDEX) { + pool->index.tree = RB_ROOT; + err = rxe_pool_init_index(pool, info->max_index, + info->min_index); if (err) goto out; } - if (rxe_type_info[type].flags & RXE_POOL_KEY) { - pool->key.key_offset = rxe_type_info[type].key_offset; - pool->key.key_size = rxe_type_info[type].key_size; + if (pool->flags & RXE_POOL_KEY) { + pool->key.tree = RB_ROOT; + pool->key.key_offset = info->key_offset; + pool->key.key_size = info->key_size; } out: @@ -169,9 +166,10 @@ void rxe_pool_cleanup(struct rxe_pool *pool) { if (atomic_read(&pool->num_elem) > 0) pr_warn("%s pool destroyed with unfree'd elem\n", - pool_name(pool)); + pool->name); - bitmap_free(pool->index.table); + if (pool->flags & RXE_POOL_INDEX) + bitmap_free(pool->index.table); } static u32 alloc_index(struct rxe_pool *pool) @@ -189,15 +187,15 @@ static u32 alloc_index(struct rxe_pool *pool) return index + pool->index.min_index; } -static int rxe_insert_index(struct rxe_pool *pool, struct rxe_pool_entry *new) +static int rxe_insert_index(struct rxe_pool *pool, struct rxe_pool_elem *new) { struct rb_node **link = &pool->index.tree.rb_node; struct rb_node *parent = NULL; - struct rxe_pool_entry *elem; + struct rxe_pool_elem *elem; while (*link) { parent = *link; - elem = rb_entry(parent, struct rxe_pool_entry, index_node); + elem = rb_entry(parent, struct rxe_pool_elem, index_node); if (elem->index == new->index) { pr_warn("element already exists!\n"); @@ -216,19 +214,20 @@ static int rxe_insert_index(struct rxe_pool *pool, struct rxe_pool_entry *new) return 0; } -static int rxe_insert_key(struct rxe_pool *pool, struct rxe_pool_entry *new) +static int rxe_insert_key(struct rxe_pool *pool, struct rxe_pool_elem *new) { struct rb_node **link = &pool->key.tree.rb_node; struct rb_node *parent = NULL; - struct rxe_pool_entry *elem; + struct rxe_pool_elem *elem; int cmp; while (*link) { parent = *link; - elem = rb_entry(parent, struct rxe_pool_entry, key_node); + elem = rb_entry(parent, struct rxe_pool_elem, key_node); cmp = memcmp((u8 *)elem + pool->key.key_offset, - (u8 *)new + pool->key.key_offset, pool->key.key_size); + (u8 *)new + pool->key.key_offset, + pool->key.key_size); if (cmp == 0) { pr_warn("key already exists!\n"); @@ -247,7 +246,7 @@ static int rxe_insert_key(struct rxe_pool *pool, struct rxe_pool_entry *new) return 0; } -int __rxe_add_key_locked(struct rxe_pool_entry *elem, void *key) +int __rxe_add_key_locked(struct rxe_pool_elem *elem, void *key) { struct rxe_pool *pool = elem->pool; int err; @@ -258,37 +257,35 @@ int __rxe_add_key_locked(struct rxe_pool_entry *elem, void *key) return err; } -int __rxe_add_key(struct rxe_pool_entry *elem, void *key) +int __rxe_add_key(struct rxe_pool_elem *elem, void *key) { struct rxe_pool *pool = elem->pool; - unsigned long flags; int err; - write_lock_irqsave(&pool->pool_lock, flags); + write_lock_bh(&pool->pool_lock); err = __rxe_add_key_locked(elem, key); - write_unlock_irqrestore(&pool->pool_lock, flags); + write_unlock_bh(&pool->pool_lock); return err; } -void __rxe_drop_key_locked(struct rxe_pool_entry *elem) +void __rxe_drop_key_locked(struct rxe_pool_elem *elem) { struct rxe_pool *pool = elem->pool; rb_erase(&elem->key_node, &pool->key.tree); } -void __rxe_drop_key(struct rxe_pool_entry *elem) +void __rxe_drop_key(struct rxe_pool_elem *elem) { struct rxe_pool *pool = elem->pool; - unsigned long flags; - write_lock_irqsave(&pool->pool_lock, flags); + write_lock_bh(&pool->pool_lock); __rxe_drop_key_locked(elem); - write_unlock_irqrestore(&pool->pool_lock, flags); + write_unlock_bh(&pool->pool_lock); } -int __rxe_add_index_locked(struct rxe_pool_entry *elem) +int __rxe_add_index_locked(struct rxe_pool_elem *elem) { struct rxe_pool *pool = elem->pool; int err; @@ -299,20 +296,19 @@ int __rxe_add_index_locked(struct rxe_pool_entry *elem) return err; } -int __rxe_add_index(struct rxe_pool_entry *elem) +int __rxe_add_index(struct rxe_pool_elem *elem) { struct rxe_pool *pool = elem->pool; - unsigned long flags; int err; - write_lock_irqsave(&pool->pool_lock, flags); + write_lock_bh(&pool->pool_lock); err = __rxe_add_index_locked(elem); - write_unlock_irqrestore(&pool->pool_lock, flags); + write_unlock_bh(&pool->pool_lock); return err; } -void __rxe_drop_index_locked(struct rxe_pool_entry *elem) +void __rxe_drop_index_locked(struct rxe_pool_elem *elem) { struct rxe_pool *pool = elem->pool; @@ -320,32 +316,31 @@ void __rxe_drop_index_locked(struct rxe_pool_entry *elem) rb_erase(&elem->index_node, &pool->index.tree); } -void __rxe_drop_index(struct rxe_pool_entry *elem) +void __rxe_drop_index(struct rxe_pool_elem *elem) { struct rxe_pool *pool = elem->pool; - unsigned long flags; - write_lock_irqsave(&pool->pool_lock, flags); + write_lock_bh(&pool->pool_lock); __rxe_drop_index_locked(elem); - write_unlock_irqrestore(&pool->pool_lock, flags); + write_unlock_bh(&pool->pool_lock); } void *rxe_alloc_locked(struct rxe_pool *pool) { - const struct rxe_type_info *info = &rxe_type_info[pool->type]; - struct rxe_pool_entry *elem; - u8 *obj; + struct rxe_pool_elem *elem; + void *obj; if (atomic_inc_return(&pool->num_elem) > pool->max_elem) goto out_cnt; - obj = kzalloc(info->size, GFP_ATOMIC); + obj = kzalloc(pool->elem_size, GFP_ATOMIC); if (!obj) goto out_cnt; - elem = (struct rxe_pool_entry *)(obj + info->elem_offset); + elem = (struct rxe_pool_elem *)((u8 *)obj + pool->elem_offset); elem->pool = pool; + elem->obj = obj; kref_init(&elem->ref_cnt); return obj; @@ -357,20 +352,20 @@ out_cnt: void *rxe_alloc(struct rxe_pool *pool) { - const struct rxe_type_info *info = &rxe_type_info[pool->type]; - struct rxe_pool_entry *elem; - u8 *obj; + struct rxe_pool_elem *elem; + void *obj; if (atomic_inc_return(&pool->num_elem) > pool->max_elem) goto out_cnt; - obj = kzalloc(info->size, GFP_KERNEL); + obj = kzalloc(pool->elem_size, GFP_KERNEL); if (!obj) goto out_cnt; - elem = (struct rxe_pool_entry *)(obj + info->elem_offset); + elem = (struct rxe_pool_elem *)((u8 *)obj + pool->elem_offset); elem->pool = pool; + elem->obj = obj; kref_init(&elem->ref_cnt); return obj; @@ -380,12 +375,13 @@ out_cnt: return NULL; } -int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_entry *elem) +int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem) { if (atomic_inc_return(&pool->num_elem) > pool->max_elem) goto out_cnt; elem->pool = pool; + elem->obj = (u8 *)elem - pool->elem_offset; kref_init(&elem->ref_cnt); return 0; @@ -397,17 +393,16 @@ out_cnt: void rxe_elem_release(struct kref *kref) { - struct rxe_pool_entry *elem = - container_of(kref, struct rxe_pool_entry, ref_cnt); + struct rxe_pool_elem *elem = + container_of(kref, struct rxe_pool_elem, ref_cnt); struct rxe_pool *pool = elem->pool; - const struct rxe_type_info *info = &rxe_type_info[pool->type]; - u8 *obj; + void *obj; if (pool->cleanup) pool->cleanup(elem); if (!(pool->flags & RXE_POOL_NO_ALLOC)) { - obj = (u8 *)elem - info->elem_offset; + obj = elem->obj; kfree(obj); } @@ -416,15 +411,14 @@ void rxe_elem_release(struct kref *kref) void *rxe_pool_get_index_locked(struct rxe_pool *pool, u32 index) { - const struct rxe_type_info *info = &rxe_type_info[pool->type]; struct rb_node *node; - struct rxe_pool_entry *elem; - u8 *obj; + struct rxe_pool_elem *elem; + void *obj; node = pool->index.tree.rb_node; while (node) { - elem = rb_entry(node, struct rxe_pool_entry, index_node); + elem = rb_entry(node, struct rxe_pool_elem, index_node); if (elem->index > index) node = node->rb_left; @@ -436,7 +430,7 @@ void *rxe_pool_get_index_locked(struct rxe_pool *pool, u32 index) if (node) { kref_get(&elem->ref_cnt); - obj = (u8 *)elem - info->elem_offset; + obj = elem->obj; } else { obj = NULL; } @@ -446,28 +440,26 @@ void *rxe_pool_get_index_locked(struct rxe_pool *pool, u32 index) void *rxe_pool_get_index(struct rxe_pool *pool, u32 index) { - u8 *obj; - unsigned long flags; + void *obj; - read_lock_irqsave(&pool->pool_lock, flags); + read_lock_bh(&pool->pool_lock); obj = rxe_pool_get_index_locked(pool, index); - read_unlock_irqrestore(&pool->pool_lock, flags); + read_unlock_bh(&pool->pool_lock); return obj; } void *rxe_pool_get_key_locked(struct rxe_pool *pool, void *key) { - const struct rxe_type_info *info = &rxe_type_info[pool->type]; struct rb_node *node; - struct rxe_pool_entry *elem; - u8 *obj; + struct rxe_pool_elem *elem; + void *obj; int cmp; node = pool->key.tree.rb_node; while (node) { - elem = rb_entry(node, struct rxe_pool_entry, key_node); + elem = rb_entry(node, struct rxe_pool_elem, key_node); cmp = memcmp((u8 *)elem + pool->key.key_offset, key, pool->key.key_size); @@ -482,7 +474,7 @@ void *rxe_pool_get_key_locked(struct rxe_pool *pool, void *key) if (node) { kref_get(&elem->ref_cnt); - obj = (u8 *)elem - info->elem_offset; + obj = elem->obj; } else { obj = NULL; } @@ -492,12 +484,11 @@ void *rxe_pool_get_key_locked(struct rxe_pool *pool, void *key) void *rxe_pool_get_key(struct rxe_pool *pool, void *key) { - u8 *obj; - unsigned long flags; + void *obj; - read_lock_irqsave(&pool->pool_lock, flags); + read_lock_bh(&pool->pool_lock); obj = rxe_pool_get_key_locked(pool, key); - read_unlock_irqrestore(&pool->pool_lock, flags); + read_unlock_bh(&pool->pool_lock); return obj; } diff --git a/drivers/infiniband/sw/rxe/rxe_pool.h b/drivers/infiniband/sw/rxe/rxe_pool.h index 8ecd9f870aea..214279310f4d 100644 --- a/drivers/infiniband/sw/rxe/rxe_pool.h +++ b/drivers/infiniband/sw/rxe/rxe_pool.h @@ -7,9 +7,6 @@ #ifndef RXE_POOL_H #define RXE_POOL_H -#define RXE_POOL_ALIGN (16) -#define RXE_POOL_CACHE_FLAGS (0) - enum rxe_pool_flags { RXE_POOL_INDEX = BIT(1), RXE_POOL_KEY = BIT(2), @@ -30,10 +27,9 @@ enum rxe_elem_type { RXE_NUM_TYPES, /* keep me last */ }; -struct rxe_pool_entry; - -struct rxe_pool_entry { +struct rxe_pool_elem { struct rxe_pool *pool; + void *obj; struct kref ref_cnt; struct list_head list; @@ -47,14 +43,16 @@ struct rxe_pool_entry { struct rxe_pool { struct rxe_dev *rxe; + const char *name; rwlock_t pool_lock; /* protects pool add/del/search */ - size_t elem_size; - void (*cleanup)(struct rxe_pool_entry *obj); + void (*cleanup)(struct rxe_pool_elem *obj); enum rxe_pool_flags flags; enum rxe_elem_type type; unsigned int max_elem; atomic_t num_elem; + size_t elem_size; + size_t elem_offset; /* only used if indexed */ struct { @@ -89,51 +87,51 @@ void *rxe_alloc_locked(struct rxe_pool *pool); void *rxe_alloc(struct rxe_pool *pool); /* connect already allocated object to pool */ -int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_entry *elem); +int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem); -#define rxe_add_to_pool(pool, obj) __rxe_add_to_pool(pool, &(obj)->pelem) +#define rxe_add_to_pool(pool, obj) __rxe_add_to_pool(pool, &(obj)->elem) /* assign an index to an indexed object and insert object into * pool's rb tree holding and not holding the pool_lock */ -int __rxe_add_index_locked(struct rxe_pool_entry *elem); +int __rxe_add_index_locked(struct rxe_pool_elem *elem); -#define rxe_add_index_locked(obj) __rxe_add_index_locked(&(obj)->pelem) +#define rxe_add_index_locked(obj) __rxe_add_index_locked(&(obj)->elem) -int __rxe_add_index(struct rxe_pool_entry *elem); +int __rxe_add_index(struct rxe_pool_elem *elem); -#define rxe_add_index(obj) __rxe_add_index(&(obj)->pelem) +#define rxe_add_index(obj) __rxe_add_index(&(obj)->elem) /* drop an index and remove object from rb tree * holding and not holding the pool_lock */ -void __rxe_drop_index_locked(struct rxe_pool_entry *elem); +void __rxe_drop_index_locked(struct rxe_pool_elem *elem); -#define rxe_drop_index_locked(obj) __rxe_drop_index_locked(&(obj)->pelem) +#define rxe_drop_index_locked(obj) __rxe_drop_index_locked(&(obj)->elem) -void __rxe_drop_index(struct rxe_pool_entry *elem); +void __rxe_drop_index(struct rxe_pool_elem *elem); -#define rxe_drop_index(obj) __rxe_drop_index(&(obj)->pelem) +#define rxe_drop_index(obj) __rxe_drop_index(&(obj)->elem) /* assign a key to a keyed object and insert object into * pool's rb tree holding and not holding pool_lock */ -int __rxe_add_key_locked(struct rxe_pool_entry *elem, void *key); +int __rxe_add_key_locked(struct rxe_pool_elem *elem, void *key); -#define rxe_add_key_locked(obj, key) __rxe_add_key_locked(&(obj)->pelem, key) +#define rxe_add_key_locked(obj, key) __rxe_add_key_locked(&(obj)->elem, key) -int __rxe_add_key(struct rxe_pool_entry *elem, void *key); +int __rxe_add_key(struct rxe_pool_elem *elem, void *key); -#define rxe_add_key(obj, key) __rxe_add_key(&(obj)->pelem, key) +#define rxe_add_key(obj, key) __rxe_add_key(&(obj)->elem, key) /* remove elem from rb tree holding and not holding the pool_lock */ -void __rxe_drop_key_locked(struct rxe_pool_entry *elem); +void __rxe_drop_key_locked(struct rxe_pool_elem *elem); -#define rxe_drop_key_locked(obj) __rxe_drop_key_locked(&(obj)->pelem) +#define rxe_drop_key_locked(obj) __rxe_drop_key_locked(&(obj)->elem) -void __rxe_drop_key(struct rxe_pool_entry *elem); +void __rxe_drop_key(struct rxe_pool_elem *elem); -#define rxe_drop_key(obj) __rxe_drop_key(&(obj)->pelem) +#define rxe_drop_key(obj) __rxe_drop_key(&(obj)->elem) /* lookup an indexed object from index holding and not holding the pool_lock. * takes a reference on object @@ -153,9 +151,9 @@ void *rxe_pool_get_key(struct rxe_pool *pool, void *key); void rxe_elem_release(struct kref *kref); /* take a reference on an object */ -#define rxe_add_ref(elem) kref_get(&(elem)->pelem.ref_cnt) +#define rxe_add_ref(obj) kref_get(&(obj)->elem.ref_cnt) /* drop a reference on an object */ -#define rxe_drop_ref(elem) kref_put(&(elem)->pelem.ref_cnt, rxe_elem_release) +#define rxe_drop_ref(obj) kref_put(&(obj)->elem.ref_cnt, rxe_elem_release) #endif /* RXE_POOL_H */ diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c index 54b8711321c1..afe11f475b8c 100644 --- a/drivers/infiniband/sw/rxe/rxe_qp.c +++ b/drivers/infiniband/sw/rxe/rxe_qp.c @@ -167,7 +167,7 @@ static void rxe_qp_init_misc(struct rxe_dev *rxe, struct rxe_qp *qp, qp->attr.path_mtu = 1; qp->mtu = ib_mtu_enum_to_int(qp->attr.path_mtu); - qpn = qp->pelem.index; + qpn = qp->elem.index; port = &rxe->port; switch (init->qp_type) { @@ -832,9 +832,9 @@ static void rxe_qp_do_cleanup(struct work_struct *work) } /* called when the last reference to the qp is dropped */ -void rxe_qp_cleanup(struct rxe_pool_entry *arg) +void rxe_qp_cleanup(struct rxe_pool_elem *elem) { - struct rxe_qp *qp = container_of(arg, typeof(*qp), pelem); + struct rxe_qp *qp = container_of(elem, typeof(*qp), elem); execute_in_process_context(rxe_qp_do_cleanup, &qp->cleanup_work); } diff --git a/drivers/infiniband/sw/rxe/rxe_queue.c b/drivers/infiniband/sw/rxe/rxe_queue.c index 6e6e023c1b45..a1b283dd2d4c 100644 --- a/drivers/infiniband/sw/rxe/rxe_queue.c +++ b/drivers/infiniband/sw/rxe/rxe_queue.c @@ -151,7 +151,6 @@ int rxe_queue_resize(struct rxe_queue *q, unsigned int *num_elem_p, struct rxe_queue *new_q; unsigned int num_elem = *num_elem_p; int err; - unsigned long flags = 0, flags1; new_q = rxe_queue_init(q->rxe, &num_elem, elem_size, q->type); if (!new_q) @@ -165,17 +164,17 @@ int rxe_queue_resize(struct rxe_queue *q, unsigned int *num_elem_p, goto err1; } - spin_lock_irqsave(consumer_lock, flags1); + spin_lock_bh(consumer_lock); if (producer_lock) { - spin_lock_irqsave(producer_lock, flags); + spin_lock_bh(producer_lock); err = resize_finish(q, new_q, num_elem); - spin_unlock_irqrestore(producer_lock, flags); + spin_unlock_bh(producer_lock); } else { err = resize_finish(q, new_q, num_elem); } - spin_unlock_irqrestore(consumer_lock, flags1); + spin_unlock_bh(consumer_lock); rxe_queue_cleanup(new_q); /* new/old dep on err */ if (err) diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c index 0c9d2af15f3d..5eb89052dd66 100644 --- a/drivers/infiniband/sw/rxe/rxe_req.c +++ b/drivers/infiniband/sw/rxe/rxe_req.c @@ -110,7 +110,6 @@ void rnr_nak_timer(struct timer_list *t) static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp) { struct rxe_send_wqe *wqe; - unsigned long flags; struct rxe_queue *q = qp->sq.queue; unsigned int index = qp->req.wqe_index; unsigned int cons; @@ -124,25 +123,23 @@ static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp) /* check to see if we are drained; * state_lock used by requester and completer */ - spin_lock_irqsave(&qp->state_lock, flags); + spin_lock_bh(&qp->state_lock); do { if (qp->req.state != QP_STATE_DRAIN) { /* comp just finished */ - spin_unlock_irqrestore(&qp->state_lock, - flags); + spin_unlock_bh(&qp->state_lock); break; } if (wqe && ((index != cons) || (wqe->state != wqe_state_posted))) { /* comp not done yet */ - spin_unlock_irqrestore(&qp->state_lock, - flags); + spin_unlock_bh(&qp->state_lock); break; } qp->req.state = QP_STATE_DRAINED; - spin_unlock_irqrestore(&qp->state_lock, flags); + spin_unlock_bh(&qp->state_lock); if (qp->ibqp.event_handler) { struct ib_event ev; @@ -372,7 +369,6 @@ static struct sk_buff *init_req_packet(struct rxe_qp *qp, int pad = (-payload) & 0x3; int paylen; int solicited; - u16 pkey; u32 qp_num; int ack_req; @@ -404,8 +400,6 @@ static struct sk_buff *init_req_packet(struct rxe_qp *qp, (pkt->mask & (RXE_WRITE_MASK | RXE_IMMDT_MASK)) == (RXE_WRITE_MASK | RXE_IMMDT_MASK)); - pkey = IB_DEFAULT_PKEY_FULL; - qp_num = (pkt->mask & RXE_DETH_MASK) ? ibwr->wr.ud.remote_qpn : qp->attr.dest_qp_num; @@ -414,7 +408,7 @@ static struct sk_buff *init_req_packet(struct rxe_qp *qp, if (ack_req) qp->req.noack_pkts = 0; - bth_init(pkt, pkt->opcode, solicited, 0, pad, pkey, qp_num, + bth_init(pkt, pkt->opcode, solicited, 0, pad, IB_DEFAULT_PKEY_FULL, qp_num, ack_req, pkt->psn); /* init optional headers */ diff --git a/drivers/infiniband/sw/rxe/rxe_srq.c b/drivers/infiniband/sw/rxe/rxe_srq.c index eb1c4c3b3a78..0c0721f04357 100644 --- a/drivers/infiniband/sw/rxe/rxe_srq.c +++ b/drivers/infiniband/sw/rxe/rxe_srq.c @@ -83,7 +83,7 @@ int rxe_srq_from_init(struct rxe_dev *rxe, struct rxe_srq *srq, srq->ibsrq.event_handler = init->event_handler; srq->ibsrq.srq_context = init->srq_context; srq->limit = init->attr.srq_limit; - srq->srq_num = srq->pelem.index; + srq->srq_num = srq->elem.index; srq->rq.max_wr = init->attr.max_wr; srq->rq.max_sge = init->attr.max_sge; diff --git a/drivers/infiniband/sw/rxe/rxe_sysfs.c b/drivers/infiniband/sw/rxe/rxe_sysfs.c deleted file mode 100644 index 666202ddff48..000000000000 --- a/drivers/infiniband/sw/rxe/rxe_sysfs.c +++ /dev/null @@ -1,119 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB -/* - * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. - * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. - */ - -#include "rxe.h" -#include "rxe_net.h" - -/* Copy argument and remove trailing CR. Return the new length. */ -static int sanitize_arg(const char *val, char *intf, int intf_len) -{ - int len; - - if (!val) - return 0; - - /* Remove newline. */ - for (len = 0; len < intf_len - 1 && val[len] && val[len] != '\n'; len++) - intf[len] = val[len]; - intf[len] = 0; - - if (len == 0 || (val[len] != 0 && val[len] != '\n')) - return 0; - - return len; -} - -static int rxe_param_set_add(const char *val, const struct kernel_param *kp) -{ - int len; - int err = 0; - char intf[32]; - struct net_device *ndev; - struct rxe_dev *exists; - - if (!rxe_initialized) { - pr_err("Module parameters are not supported, use rdma link add or rxe_cfg\n"); - return -EAGAIN; - } - - len = sanitize_arg(val, intf, sizeof(intf)); - if (!len) { - pr_err("add: invalid interface name\n"); - return -EINVAL; - } - - ndev = dev_get_by_name(&init_net, intf); - if (!ndev) { - pr_err("interface %s not found\n", intf); - return -EINVAL; - } - - if (is_vlan_dev(ndev)) { - pr_err("rxe creation allowed on top of a real device only\n"); - err = -EPERM; - goto err; - } - - exists = rxe_get_dev_from_net(ndev); - if (exists) { - ib_device_put(&exists->ib_dev); - pr_err("already configured on %s\n", intf); - err = -EINVAL; - goto err; - } - - err = rxe_net_add("rxe%d", ndev); - if (err) { - pr_err("failed to add %s\n", intf); - goto err; - } - -err: - dev_put(ndev); - return err; -} - -static int rxe_param_set_remove(const char *val, const struct kernel_param *kp) -{ - int len; - char intf[32]; - struct ib_device *ib_dev; - - len = sanitize_arg(val, intf, sizeof(intf)); - if (!len) { - pr_err("add: invalid interface name\n"); - return -EINVAL; - } - - if (strncmp("all", intf, len) == 0) { - pr_info("rxe_sys: remove all"); - ib_unregister_driver(RDMA_DRIVER_RXE); - return 0; - } - - ib_dev = ib_device_get_by_name(intf, RDMA_DRIVER_RXE); - if (!ib_dev) { - pr_err("not configured on %s\n", intf); - return -EINVAL; - } - - ib_unregister_device_and_put(ib_dev); - - return 0; -} - -static const struct kernel_param_ops rxe_add_ops = { - .set = rxe_param_set_add, -}; - -static const struct kernel_param_ops rxe_remove_ops = { - .set = rxe_param_set_remove, -}; - -module_param_cb(add, &rxe_add_ops, NULL, 0200); -MODULE_PARM_DESC(add, "DEPRECATED. Create RXE device over network interface"); -module_param_cb(remove, &rxe_remove_ops, NULL, 0200); -MODULE_PARM_DESC(remove, "DEPRECATED. Remove RXE device over network interface"); diff --git a/drivers/infiniband/sw/rxe/rxe_task.c b/drivers/infiniband/sw/rxe/rxe_task.c index 6951fdcb31bf..0c4db5bb17d7 100644 --- a/drivers/infiniband/sw/rxe/rxe_task.c +++ b/drivers/infiniband/sw/rxe/rxe_task.c @@ -32,25 +32,24 @@ void rxe_do_task(struct tasklet_struct *t) { int cont; int ret; - unsigned long flags; struct rxe_task *task = from_tasklet(task, t, tasklet); - spin_lock_irqsave(&task->state_lock, flags); + spin_lock_bh(&task->state_lock); switch (task->state) { case TASK_STATE_START: task->state = TASK_STATE_BUSY; - spin_unlock_irqrestore(&task->state_lock, flags); + spin_unlock_bh(&task->state_lock); break; case TASK_STATE_BUSY: task->state = TASK_STATE_ARMED; fallthrough; case TASK_STATE_ARMED: - spin_unlock_irqrestore(&task->state_lock, flags); + spin_unlock_bh(&task->state_lock); return; default: - spin_unlock_irqrestore(&task->state_lock, flags); + spin_unlock_bh(&task->state_lock); pr_warn("%s failed with bad state %d\n", __func__, task->state); return; } @@ -59,7 +58,7 @@ void rxe_do_task(struct tasklet_struct *t) cont = 0; ret = task->func(task->arg); - spin_lock_irqsave(&task->state_lock, flags); + spin_lock_bh(&task->state_lock); switch (task->state) { case TASK_STATE_BUSY: if (ret) @@ -81,7 +80,7 @@ void rxe_do_task(struct tasklet_struct *t) pr_warn("%s failed with bad state %d\n", __func__, task->state); } - spin_unlock_irqrestore(&task->state_lock, flags); + spin_unlock_bh(&task->state_lock); } while (cont); task->ret = ret; @@ -106,7 +105,6 @@ int rxe_init_task(void *obj, struct rxe_task *task, void rxe_cleanup_task(struct rxe_task *task) { - unsigned long flags; bool idle; /* @@ -116,9 +114,9 @@ void rxe_cleanup_task(struct rxe_task *task) task->destroyed = true; do { - spin_lock_irqsave(&task->state_lock, flags); + spin_lock_bh(&task->state_lock); idle = (task->state == TASK_STATE_START); - spin_unlock_irqrestore(&task->state_lock, flags); + spin_unlock_bh(&task->state_lock); } while (!idle); tasklet_kill(&task->tasklet); diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c index 0aa0d7e52773..915ad6664321 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.c +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c @@ -182,7 +182,7 @@ static int rxe_create_ah(struct ib_ah *ibah, /* create index > 0 */ rxe_add_index(ah); - ah->ah_num = ah->pelem.index; + ah->ah_num = ah->elem.index; if (uresp) { /* only if new user provider */ @@ -383,10 +383,9 @@ static int rxe_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_wr) { int err = 0; - unsigned long flags; struct rxe_srq *srq = to_rsrq(ibsrq); - spin_lock_irqsave(&srq->rq.producer_lock, flags); + spin_lock_bh(&srq->rq.producer_lock); while (wr) { err = post_one_recv(&srq->rq, wr); @@ -395,7 +394,7 @@ static int rxe_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, wr = wr->next; } - spin_unlock_irqrestore(&srq->rq.producer_lock, flags); + spin_unlock_bh(&srq->rq.producer_lock); if (err) *bad_wr = wr; @@ -469,6 +468,11 @@ static int rxe_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, if (err) goto err1; + if ((mask & IB_QP_AV) && (attr->ah_attr.ah_flags & IB_AH_GRH)) + qp->src_port = rdma_get_udp_sport(attr->ah_attr.grh.flow_label, + qp->ibqp.qp_num, + qp->attr.dest_qp_num); + return 0; err1: @@ -634,19 +638,18 @@ static int post_one_send(struct rxe_qp *qp, const struct ib_send_wr *ibwr, int err; struct rxe_sq *sq = &qp->sq; struct rxe_send_wqe *send_wqe; - unsigned long flags; int full; err = validate_send_wr(qp, ibwr, mask, length); if (err) return err; - spin_lock_irqsave(&qp->sq.sq_lock, flags); + spin_lock_bh(&qp->sq.sq_lock); full = queue_full(sq->queue, QUEUE_TYPE_TO_DRIVER); if (unlikely(full)) { - spin_unlock_irqrestore(&qp->sq.sq_lock, flags); + spin_unlock_bh(&qp->sq.sq_lock); return -ENOMEM; } @@ -655,7 +658,7 @@ static int post_one_send(struct rxe_qp *qp, const struct ib_send_wr *ibwr, queue_advance_producer(sq->queue, QUEUE_TYPE_TO_DRIVER); - spin_unlock_irqrestore(&qp->sq.sq_lock, flags); + spin_unlock_bh(&qp->sq.sq_lock); return 0; } @@ -735,7 +738,6 @@ static int rxe_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, int err = 0; struct rxe_qp *qp = to_rqp(ibqp); struct rxe_rq *rq = &qp->rq; - unsigned long flags; if (unlikely((qp_state(qp) < IB_QPS_INIT) || !qp->valid)) { *bad_wr = wr; @@ -749,7 +751,7 @@ static int rxe_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, goto err1; } - spin_lock_irqsave(&rq->producer_lock, flags); + spin_lock_bh(&rq->producer_lock); while (wr) { err = post_one_recv(rq, wr); @@ -760,7 +762,7 @@ static int rxe_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, wr = wr->next; } - spin_unlock_irqrestore(&rq->producer_lock, flags); + spin_unlock_bh(&rq->producer_lock); if (qp->resp.state == QP_STATE_ERROR) rxe_run_task(&qp->resp.task, 1); @@ -841,9 +843,8 @@ static int rxe_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) int i; struct rxe_cq *cq = to_rcq(ibcq); struct rxe_cqe *cqe; - unsigned long flags; - spin_lock_irqsave(&cq->cq_lock, flags); + spin_lock_bh(&cq->cq_lock); for (i = 0; i < num_entries; i++) { cqe = queue_head(cq->queue, QUEUE_TYPE_FROM_DRIVER); if (!cqe) @@ -852,7 +853,7 @@ static int rxe_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) memcpy(wc++, &cqe->ibwc, sizeof(*wc)); queue_advance_consumer(cq->queue, QUEUE_TYPE_FROM_DRIVER); } - spin_unlock_irqrestore(&cq->cq_lock, flags); + spin_unlock_bh(&cq->cq_lock); return i; } @@ -870,11 +871,10 @@ static int rxe_peek_cq(struct ib_cq *ibcq, int wc_cnt) static int rxe_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) { struct rxe_cq *cq = to_rcq(ibcq); - unsigned long irq_flags; int ret = 0; int empty; - spin_lock_irqsave(&cq->cq_lock, irq_flags); + spin_lock_bh(&cq->cq_lock); if (cq->notify != IB_CQ_NEXT_COMP) cq->notify = flags & IB_CQ_SOLICITED_MASK; @@ -883,7 +883,7 @@ static int rxe_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) if ((flags & IB_CQ_REPORT_MISSED_EVENTS) && !empty) ret = 1; - spin_unlock_irqrestore(&cq->cq_lock, irq_flags); + spin_unlock_bh(&cq->cq_lock); return ret; } diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h index 35e041450090..e48969e8d4c8 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.h +++ b/drivers/infiniband/sw/rxe/rxe_verbs.h @@ -35,17 +35,17 @@ static inline int psn_compare(u32 psn_a, u32 psn_b) struct rxe_ucontext { struct ib_ucontext ibuc; - struct rxe_pool_entry pelem; + struct rxe_pool_elem elem; }; struct rxe_pd { struct ib_pd ibpd; - struct rxe_pool_entry pelem; + struct rxe_pool_elem elem; }; struct rxe_ah { struct ib_ah ibah; - struct rxe_pool_entry pelem; + struct rxe_pool_elem elem; struct rxe_av av; bool is_user; int ah_num; @@ -60,7 +60,7 @@ struct rxe_cqe { struct rxe_cq { struct ib_cq ibcq; - struct rxe_pool_entry pelem; + struct rxe_pool_elem elem; struct rxe_queue *queue; spinlock_t cq_lock; u8 notify; @@ -95,7 +95,7 @@ struct rxe_rq { struct rxe_srq { struct ib_srq ibsrq; - struct rxe_pool_entry pelem; + struct rxe_pool_elem elem; struct rxe_pd *pd; struct rxe_rq rq; u32 srq_num; @@ -209,7 +209,7 @@ struct rxe_resp_info { struct rxe_qp { struct ib_qp ibqp; - struct rxe_pool_entry pelem; + struct rxe_pool_elem elem; struct ib_qp_attr attr; unsigned int valid; unsigned int mtu; @@ -309,7 +309,7 @@ static inline int rkey_is_mw(u32 rkey) } struct rxe_mr { - struct rxe_pool_entry pelem; + struct rxe_pool_elem elem; struct ib_mr ibmr; struct ib_umem *umem; @@ -342,7 +342,7 @@ enum rxe_mw_state { struct rxe_mw { struct ib_mw ibmw; - struct rxe_pool_entry pelem; + struct rxe_pool_elem elem; spinlock_t lock; enum rxe_mw_state state; struct rxe_qp *qp; /* Type 2 only */ @@ -354,7 +354,7 @@ struct rxe_mw { }; struct rxe_mc_grp { - struct rxe_pool_entry pelem; + struct rxe_pool_elem elem; spinlock_t mcg_lock; /* guard group */ struct rxe_dev *rxe; struct list_head qp_list; @@ -365,7 +365,7 @@ struct rxe_mc_grp { }; struct rxe_mc_elem { - struct rxe_pool_entry pelem; + struct rxe_pool_elem elem; struct list_head qp_list; struct list_head grp_list; struct rxe_qp *qp; @@ -392,8 +392,6 @@ struct rxe_dev { struct net_device *ndev; - int xmit_errors; - struct rxe_pool uc_pool; struct rxe_pool pd_pool; struct rxe_pool ah_pool; @@ -484,6 +482,6 @@ static inline struct rxe_pd *rxe_mw_pd(struct rxe_mw *mw) int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name); -void rxe_mc_cleanup(struct rxe_pool_entry *arg); +void rxe_mc_cleanup(struct rxe_pool_elem *elem); #endif /* RXE_VERBS_H */ diff --git a/drivers/infiniband/sw/siw/siw_verbs.c b/drivers/infiniband/sw/siw/siw_verbs.c index 1b36350601fa..a3dd2cb6d5c9 100644 --- a/drivers/infiniband/sw/siw/siw_verbs.c +++ b/drivers/infiniband/sw/siw/siw_verbs.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -155,7 +156,8 @@ int siw_query_device(struct ib_device *base_dev, struct ib_device_attr *attr, attr->vendor_id = SIW_VENDOR_ID; attr->vendor_part_id = sdev->vendor_part_id; - memcpy(&attr->sys_image_guid, sdev->netdev->dev_addr, 6); + addrconf_addr_eui48((u8 *)&attr->sys_image_guid, + sdev->netdev->dev_addr); return 0; } @@ -660,7 +662,7 @@ static int siw_copy_inline_sgl(const struct ib_send_wr *core_wr, kbuf += core_sge->length; core_sge++; } - sqe->sge[0].length = bytes > 0 ? bytes : 0; + sqe->sge[0].length = max(bytes, 0); sqe->num_sge = bytes > 0 ? 1 : 0; return bytes; diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.c b/drivers/infiniband/ulp/iser/iscsi_iser.c index 776e46ee95da..07e47021a71f 100644 --- a/drivers/infiniband/ulp/iser/iscsi_iser.c +++ b/drivers/infiniband/ulp/iser/iscsi_iser.c @@ -113,10 +113,6 @@ bool iser_pi_enable = false; module_param_named(pi_enable, iser_pi_enable, bool, S_IRUGO); MODULE_PARM_DESC(pi_enable, "Enable T10-PI offload support (default:disabled)"); -int iser_pi_guard; -module_param_named(pi_guard, iser_pi_guard, int, S_IRUGO); -MODULE_PARM_DESC(pi_guard, "T10-PI guard_type [deprecated]"); - static int iscsi_iser_set(const char *val, const struct kernel_param *kp) { int ret; @@ -139,9 +135,8 @@ static int iscsi_iser_set(const char *val, const struct kernel_param *kp) * Notes: In case of data length errors or iscsi PDU completion failures * this routine will signal iscsi layer of connection failure. */ -void -iscsi_iser_recv(struct iscsi_conn *conn, struct iscsi_hdr *hdr, - char *rx_data, int rx_data_len) +void iscsi_iser_recv(struct iscsi_conn *conn, struct iscsi_hdr *hdr, + char *rx_data, int rx_data_len) { int rc = 0; int datalen; @@ -176,8 +171,7 @@ error: * Netes: This routine can't fail, just assign iscsi task * hdr and max hdr size. */ -static int -iscsi_iser_pdu_alloc(struct iscsi_task *task, uint8_t opcode) +static int iscsi_iser_pdu_alloc(struct iscsi_task *task, uint8_t opcode) { struct iscsi_iser_task *iser_task = task->dd_data; @@ -198,9 +192,8 @@ iscsi_iser_pdu_alloc(struct iscsi_task *task, uint8_t opcode) * state mutex to avoid dereferencing the IB device which * may have already been terminated. */ -int -iser_initialize_task_headers(struct iscsi_task *task, - struct iser_tx_desc *tx_desc) +int iser_initialize_task_headers(struct iscsi_task *task, + struct iser_tx_desc *tx_desc) { struct iser_conn *iser_conn = task->conn->dd_data; struct iser_device *device = iser_conn->ib_conn.device; @@ -237,8 +230,7 @@ iser_initialize_task_headers(struct iscsi_task *task, * Return: Returns zero on success or -ENOMEM when failing * to init task headers (dma mapping error). */ -static int -iscsi_iser_task_init(struct iscsi_task *task) +static int iscsi_iser_task_init(struct iscsi_task *task) { struct iscsi_iser_task *iser_task = task->dd_data; int ret; @@ -272,8 +264,8 @@ iscsi_iser_task_init(struct iscsi_task *task) * xmit. * **/ -static int -iscsi_iser_mtask_xmit(struct iscsi_conn *conn, struct iscsi_task *task) +static int iscsi_iser_mtask_xmit(struct iscsi_conn *conn, + struct iscsi_task *task) { int error = 0; @@ -290,9 +282,8 @@ iscsi_iser_mtask_xmit(struct iscsi_conn *conn, struct iscsi_task *task) return error; } -static int -iscsi_iser_task_xmit_unsol_data(struct iscsi_conn *conn, - struct iscsi_task *task) +static int iscsi_iser_task_xmit_unsol_data(struct iscsi_conn *conn, + struct iscsi_task *task) { struct iscsi_r2t_info *r2t = &task->unsol_r2t; struct iscsi_data hdr; @@ -326,8 +317,7 @@ iscsi_iser_task_xmit_unsol_data_exit: * * Return: zero on success or escalates $error on failure. */ -static int -iscsi_iser_task_xmit(struct iscsi_task *task) +static int iscsi_iser_task_xmit(struct iscsi_task *task) { struct iscsi_conn *conn = task->conn; struct iscsi_iser_task *iser_task = task->dd_data; @@ -410,8 +400,7 @@ static void iscsi_iser_cleanup_task(struct iscsi_task *task) * * In addition the error sector is marked. */ -static u8 -iscsi_iser_check_protection(struct iscsi_task *task, sector_t *sector) +static u8 iscsi_iser_check_protection(struct iscsi_task *task, sector_t *sector) { struct iscsi_iser_task *iser_task = task->dd_data; enum iser_data_dir dir = iser_task->dir[ISER_DIR_IN] ? @@ -460,11 +449,9 @@ iscsi_iser_conn_create(struct iscsi_cls_session *cls_session, * -EINVAL in case end-point doesn't exsits anymore or iser connection * state is not UP (teardown already started). */ -static int -iscsi_iser_conn_bind(struct iscsi_cls_session *cls_session, - struct iscsi_cls_conn *cls_conn, - uint64_t transport_eph, - int is_leading) +static int iscsi_iser_conn_bind(struct iscsi_cls_session *cls_session, + struct iscsi_cls_conn *cls_conn, + uint64_t transport_eph, int is_leading) { struct iscsi_conn *conn = cls_conn->dd_data; struct iser_conn *iser_conn; @@ -519,8 +506,7 @@ out: * from this point iscsi must call conn_stop in session/connection * teardown so iser transport must wait for it. */ -static int -iscsi_iser_conn_start(struct iscsi_cls_conn *cls_conn) +static int iscsi_iser_conn_start(struct iscsi_cls_conn *cls_conn) { struct iscsi_conn *iscsi_conn; struct iser_conn *iser_conn; @@ -542,8 +528,7 @@ iscsi_iser_conn_start(struct iscsi_cls_conn *cls_conn) * handle, so we call it under iser the state lock to protect against * this kind of race. */ -static void -iscsi_iser_conn_stop(struct iscsi_cls_conn *cls_conn, int flag) +static void iscsi_iser_conn_stop(struct iscsi_cls_conn *cls_conn, int flag) { struct iscsi_conn *conn = cls_conn->dd_data; struct iser_conn *iser_conn = conn->dd_data; @@ -578,8 +563,7 @@ iscsi_iser_conn_stop(struct iscsi_cls_conn *cls_conn, int flag) * * Removes and free iscsi host. */ -static void -iscsi_iser_session_destroy(struct iscsi_cls_session *cls_session) +static void iscsi_iser_session_destroy(struct iscsi_cls_session *cls_session) { struct Scsi_Host *shost = iscsi_session_to_shost(cls_session); @@ -588,8 +572,7 @@ iscsi_iser_session_destroy(struct iscsi_cls_session *cls_session) iscsi_host_free(shost); } -static inline unsigned int -iser_dif_prot_caps(int prot_caps) +static inline unsigned int iser_dif_prot_caps(int prot_caps) { int ret = 0; @@ -708,9 +691,8 @@ free_host: return NULL; } -static int -iscsi_iser_set_param(struct iscsi_cls_conn *cls_conn, - enum iscsi_param param, char *buf, int buflen) +static int iscsi_iser_set_param(struct iscsi_cls_conn *cls_conn, + enum iscsi_param param, char *buf, int buflen) { int value; @@ -760,8 +742,8 @@ iscsi_iser_set_param(struct iscsi_cls_conn *cls_conn, * * Output connection statistics. */ -static void -iscsi_iser_conn_get_stats(struct iscsi_cls_conn *cls_conn, struct iscsi_stats *stats) +static void iscsi_iser_conn_get_stats(struct iscsi_cls_conn *cls_conn, + struct iscsi_stats *stats) { struct iscsi_conn *conn = cls_conn->dd_data; @@ -812,9 +794,9 @@ static int iscsi_iser_get_ep_param(struct iscsi_endpoint *ep, * Return: iscsi_endpoint created by iscsi layer or ERR_PTR(error) * if fails. */ -static struct iscsi_endpoint * -iscsi_iser_ep_connect(struct Scsi_Host *shost, struct sockaddr *dst_addr, - int non_blocking) +static struct iscsi_endpoint *iscsi_iser_ep_connect(struct Scsi_Host *shost, + struct sockaddr *dst_addr, + int non_blocking) { int err; struct iser_conn *iser_conn; @@ -857,8 +839,7 @@ failure: * or more likely iser connection state transitioned to TEMINATING or * DOWN during the wait period. */ -static int -iscsi_iser_ep_poll(struct iscsi_endpoint *ep, int timeout_ms) +static int iscsi_iser_ep_poll(struct iscsi_endpoint *ep, int timeout_ms) { struct iser_conn *iser_conn = ep->dd_data; int rc; @@ -893,8 +874,7 @@ iscsi_iser_ep_poll(struct iscsi_endpoint *ep, int timeout_ms) * and cleanup or actually call it immediately in case we didn't pass * iscsi conn bind/start stage, thus it is safe. */ -static void -iscsi_iser_ep_disconnect(struct iscsi_endpoint *ep) +static void iscsi_iser_ep_disconnect(struct iscsi_endpoint *ep) { struct iser_conn *iser_conn = ep->dd_data; diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.h b/drivers/infiniband/ulp/iser/iscsi_iser.h index 9f6ac0a09a78..20af46c4e954 100644 --- a/drivers/infiniband/ulp/iser/iscsi_iser.h +++ b/drivers/infiniband/ulp/iser/iscsi_iser.h @@ -119,8 +119,6 @@ #define ISER_QP_MAX_RECV_DTOS (ISER_DEF_XMIT_CMDS_MAX) -#define ISER_MIN_POSTED_RX (ISER_DEF_XMIT_CMDS_MAX >> 2) - /* the max TX (send) WR supported by the iSER QP is defined by * * max_send_wr = T * (1 + D) + C ; D is how many inflight dataouts we expect * * to have at max for SCSI command. The tx posting & completion handling code * @@ -148,8 +146,6 @@ - ISER_MAX_RX_MISC_PDUS) / \ (1 + ISER_INFLIGHT_DATAOUTS)) -#define ISER_SIGNAL_CMD_COUNT 32 - /* Constant PDU lengths calculations */ #define ISER_HEADERS_LEN (sizeof(struct iser_ctrl) + sizeof(struct iscsi_hdr)) @@ -366,9 +362,6 @@ struct iser_fr_pool { * @qp: Connection Queue-pair * @cq: Connection completion queue * @cq_size: The number of max outstanding completions - * @post_recv_buf_count: post receive counter - * @sig_count: send work request signal count - * @rx_wr: receive work request for batch posts * @device: reference to iser device * @fr_pool: connection fast registration poool * @pi_support: Indicate device T10-PI support @@ -379,9 +372,6 @@ struct ib_conn { struct ib_qp *qp; struct ib_cq *cq; u32 cq_size; - int post_recv_buf_count; - u8 sig_count; - struct ib_recv_wr rx_wr[ISER_MIN_POSTED_RX]; struct iser_device *device; struct iser_fr_pool fr_pool; bool pi_support; @@ -397,8 +387,6 @@ struct ib_conn { * @state: connection logical state * @qp_max_recv_dtos: maximum number of data outs, corresponds * to max number of post recvs - * @qp_max_recv_dtos_mask: (qp_max_recv_dtos - 1) - * @min_posted_rx: (qp_max_recv_dtos >> 2) * @max_cmds: maximum cmds allowed for this connection * @name: connection peer portal * @release_work: deffered work for release job @@ -409,7 +397,6 @@ struct ib_conn { * (state is ISER_CONN_UP) * @conn_list: entry in ig conn list * @login_desc: login descriptor - * @rx_desc_head: head of rx_descs cyclic buffer * @rx_descs: rx buffers array (cyclic buffer) * @num_rx_descs: number of rx descriptors * @scsi_sg_tablesize: scsi host sg_tablesize @@ -422,8 +409,6 @@ struct iser_conn { struct iscsi_endpoint *ep; enum iser_conn_state state; unsigned qp_max_recv_dtos; - unsigned qp_max_recv_dtos_mask; - unsigned min_posted_rx; u16 max_cmds; char name[ISER_OBJECT_NAME_SIZE]; struct work_struct release_work; @@ -433,7 +418,6 @@ struct iser_conn { struct completion up_completion; struct list_head conn_list; struct iser_login_desc login_desc; - unsigned int rx_desc_head; struct iser_rx_desc *rx_descs; u32 num_rx_descs; unsigned short scsi_sg_tablesize; @@ -486,7 +470,6 @@ struct iser_global { extern struct iser_global ig; extern int iser_debug_level; extern bool iser_pi_enable; -extern int iser_pi_guard; extern unsigned int iser_max_sectors; extern bool iser_always_reg; @@ -543,9 +526,9 @@ int iser_connect(struct iser_conn *iser_conn, int non_blocking); int iser_post_recvl(struct iser_conn *iser_conn); -int iser_post_recvm(struct iser_conn *iser_conn, int count); -int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc, - bool signal); +int iser_post_recvm(struct iser_conn *iser_conn, + struct iser_rx_desc *rx_desc); +int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc); int iser_dma_map_task_data(struct iscsi_iser_task *iser_task, struct iser_data_buf *data, diff --git a/drivers/infiniband/ulp/iser/iser_initiator.c b/drivers/infiniband/ulp/iser/iser_initiator.c index 27a6f75a9912..2490150d3085 100644 --- a/drivers/infiniband/ulp/iser/iser_initiator.c +++ b/drivers/infiniband/ulp/iser/iser_initiator.c @@ -95,11 +95,8 @@ static int iser_prepare_read_cmd(struct iscsi_task *task) * task->data[ISER_DIR_OUT].data_len, Protection size * is stored at task->prot[ISER_DIR_OUT].data_len */ -static int -iser_prepare_write_cmd(struct iscsi_task *task, - unsigned int imm_sz, - unsigned int unsol_sz, - unsigned int edtl) +static int iser_prepare_write_cmd(struct iscsi_task *task, unsigned int imm_sz, + unsigned int unsol_sz, unsigned int edtl) { struct iscsi_iser_task *iser_task = task->dd_data; struct iser_mem_reg *mem_reg; @@ -160,8 +157,8 @@ iser_prepare_write_cmd(struct iscsi_task *task, } /* creates a new tx descriptor and adds header regd buffer */ -static void iser_create_send_desc(struct iser_conn *iser_conn, - struct iser_tx_desc *tx_desc) +static void iser_create_send_desc(struct iser_conn *iser_conn, + struct iser_tx_desc *tx_desc) { struct iser_device *device = iser_conn->ib_conn.device; @@ -247,8 +244,6 @@ int iser_alloc_rx_descriptors(struct iser_conn *iser_conn, struct iser_device *device = ib_conn->device; iser_conn->qp_max_recv_dtos = session->cmds_max; - iser_conn->qp_max_recv_dtos_mask = session->cmds_max - 1; /* cmds_max is 2^N */ - iser_conn->min_posted_rx = iser_conn->qp_max_recv_dtos >> 2; if (iser_alloc_fastreg_pool(ib_conn, session->scsi_cmds_max, iser_conn->pages_per_mr)) @@ -280,7 +275,6 @@ int iser_alloc_rx_descriptors(struct iser_conn *iser_conn, rx_sg->lkey = device->pd->local_dma_lkey; } - iser_conn->rx_desc_head = 0; return 0; rx_desc_dma_map_failed: @@ -322,37 +316,35 @@ void iser_free_rx_descriptors(struct iser_conn *iser_conn) static int iser_post_rx_bufs(struct iscsi_conn *conn, struct iscsi_hdr *req) { struct iser_conn *iser_conn = conn->dd_data; - struct ib_conn *ib_conn = &iser_conn->ib_conn; struct iscsi_session *session = conn->session; + int err = 0; + int i; iser_dbg("req op %x flags %x\n", req->opcode, req->flags); /* check if this is the last login - going to full feature phase */ if ((req->flags & ISCSI_FULL_FEATURE_PHASE) != ISCSI_FULL_FEATURE_PHASE) - return 0; - - /* - * Check that there is one posted recv buffer - * (for the last login response). - */ - WARN_ON(ib_conn->post_recv_buf_count != 1); + goto out; if (session->discovery_sess) { iser_info("Discovery session, re-using login RX buffer\n"); - return 0; - } else - iser_info("Normal session, posting batch of RX %d buffers\n", - iser_conn->min_posted_rx); + goto out; + } - /* Initial post receive buffers */ - if (iser_post_recvm(iser_conn, iser_conn->min_posted_rx)) - return -ENOMEM; + iser_info("Normal session, posting batch of RX %d buffers\n", + iser_conn->qp_max_recv_dtos - 1); - return 0; -} - -static inline bool iser_signal_comp(u8 sig_count) -{ - return ((sig_count % ISER_SIGNAL_CMD_COUNT) == 0); + /* + * Initial post receive buffers. + * There is one already posted recv buffer (for the last login + * response). Therefore, the first recv buffer is skipped here. + */ + for (i = 1; i < iser_conn->qp_max_recv_dtos; i++) { + err = iser_post_recvm(iser_conn, &iser_conn->rx_descs[i]); + if (err) + goto out; + } +out: + return err; } /** @@ -360,8 +352,7 @@ static inline bool iser_signal_comp(u8 sig_count) * @conn: link to matching iscsi connection * @task: SCSI command task */ -int iser_send_command(struct iscsi_conn *conn, - struct iscsi_task *task) +int iser_send_command(struct iscsi_conn *conn, struct iscsi_task *task) { struct iser_conn *iser_conn = conn->dd_data; struct iscsi_iser_task *iser_task = task->dd_data; @@ -371,7 +362,6 @@ int iser_send_command(struct iscsi_conn *conn, struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)task->hdr; struct scsi_cmnd *sc = task->sc; struct iser_tx_desc *tx_desc = &iser_task->desc; - u8 sig_count = ++iser_conn->ib_conn.sig_count; edtl = ntohl(hdr->data_length); @@ -418,8 +408,7 @@ int iser_send_command(struct iscsi_conn *conn, iser_task->status = ISER_TASK_STATUS_STARTED; - err = iser_post_send(&iser_conn->ib_conn, tx_desc, - iser_signal_comp(sig_count)); + err = iser_post_send(&iser_conn->ib_conn, tx_desc); if (!err) return 0; @@ -434,8 +423,7 @@ send_command_error: * @task: SCSI command task * @hdr: pointer to the LLD's iSCSI message header */ -int iser_send_data_out(struct iscsi_conn *conn, - struct iscsi_task *task, +int iser_send_data_out(struct iscsi_conn *conn, struct iscsi_task *task, struct iscsi_data *hdr) { struct iser_conn *iser_conn = conn->dd_data; @@ -487,7 +475,7 @@ int iser_send_data_out(struct iscsi_conn *conn, itt, buf_offset, data_seg_len); - err = iser_post_send(&iser_conn->ib_conn, tx_desc, true); + err = iser_post_send(&iser_conn->ib_conn, tx_desc); if (!err) return 0; @@ -497,8 +485,7 @@ send_data_out_error: return err; } -int iser_send_control(struct iscsi_conn *conn, - struct iscsi_task *task) +int iser_send_control(struct iscsi_conn *conn, struct iscsi_task *task) { struct iser_conn *iser_conn = conn->dd_data; struct iscsi_iser_task *iser_task = task->dd_data; @@ -550,7 +537,7 @@ int iser_send_control(struct iscsi_conn *conn, goto send_control_error; } - err = iser_post_send(&iser_conn->ib_conn, mdesc, true); + err = iser_post_send(&iser_conn->ib_conn, mdesc); if (!err) return 0; @@ -590,11 +577,14 @@ void iser_login_rsp(struct ib_cq *cq, struct ib_wc *wc) desc->rsp_dma, ISER_RX_LOGIN_SIZE, DMA_FROM_DEVICE); - ib_conn->post_recv_buf_count--; + if (iser_conn->iscsi_conn->session->discovery_sess) + return; + + /* Post the first RX buffer that is skipped in iser_post_rx_bufs() */ + iser_post_recvm(iser_conn, iser_conn->rx_descs); } -static inline int -iser_inv_desc(struct iser_fr_desc *desc, u32 rkey) +static inline int iser_inv_desc(struct iser_fr_desc *desc, u32 rkey) { if (unlikely((!desc->sig_protected && rkey != desc->rsc.mr->rkey) || (desc->sig_protected && rkey != desc->rsc.sig_mr->rkey))) { @@ -607,10 +597,8 @@ iser_inv_desc(struct iser_fr_desc *desc, u32 rkey) return 0; } -static int -iser_check_remote_inv(struct iser_conn *iser_conn, - struct ib_wc *wc, - struct iscsi_hdr *hdr) +static int iser_check_remote_inv(struct iser_conn *iser_conn, struct ib_wc *wc, + struct iscsi_hdr *hdr) { if (wc->wc_flags & IB_WC_WITH_INVALIDATE) { struct iscsi_task *task; @@ -657,8 +645,7 @@ void iser_task_rsp(struct ib_cq *cq, struct ib_wc *wc) struct iser_conn *iser_conn = to_iser_conn(ib_conn); struct iser_rx_desc *desc = iser_rx(wc->wr_cqe); struct iscsi_hdr *hdr; - int length; - int outstanding, count, err; + int length, err; if (unlikely(wc->status != IB_WC_SUCCESS)) { iser_err_comp(wc, "task_rsp"); @@ -687,20 +674,9 @@ void iser_task_rsp(struct ib_cq *cq, struct ib_wc *wc) desc->dma_addr, ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); - /* decrementing conn->post_recv_buf_count only --after-- freeing the * - * task eliminates the need to worry on tasks which are completed in * - * parallel to the execution of iser_conn_term. So the code that waits * - * for the posted rx bufs refcount to become zero handles everything */ - ib_conn->post_recv_buf_count--; - - outstanding = ib_conn->post_recv_buf_count; - if (outstanding + iser_conn->min_posted_rx <= iser_conn->qp_max_recv_dtos) { - count = min(iser_conn->qp_max_recv_dtos - outstanding, - iser_conn->min_posted_rx); - err = iser_post_recvm(iser_conn, count); - if (err) - iser_err("posting %d rx bufs err %d\n", count, err); - } + err = iser_post_recvm(iser_conn, desc); + if (err) + iser_err("posting rx buffer err %d\n", err); } void iser_cmd_comp(struct ib_cq *cq, struct ib_wc *wc) diff --git a/drivers/infiniband/ulp/iser/iser_memory.c b/drivers/infiniband/ulp/iser/iser_memory.c index 9776b755d848..660982625488 100644 --- a/drivers/infiniband/ulp/iser/iser_memory.c +++ b/drivers/infiniband/ulp/iser/iser_memory.c @@ -44,8 +44,7 @@ void iser_reg_comp(struct ib_cq *cq, struct ib_wc *wc) iser_err_comp(wc, "memreg"); } -static struct iser_fr_desc * -iser_reg_desc_get_fr(struct ib_conn *ib_conn) +static struct iser_fr_desc *iser_reg_desc_get_fr(struct ib_conn *ib_conn) { struct iser_fr_pool *fr_pool = &ib_conn->fr_pool; struct iser_fr_desc *desc; @@ -60,9 +59,8 @@ iser_reg_desc_get_fr(struct ib_conn *ib_conn) return desc; } -static void -iser_reg_desc_put_fr(struct ib_conn *ib_conn, - struct iser_fr_desc *desc) +static void iser_reg_desc_put_fr(struct ib_conn *ib_conn, + struct iser_fr_desc *desc) { struct iser_fr_pool *fr_pool = &ib_conn->fr_pool; unsigned long flags; @@ -73,9 +71,9 @@ iser_reg_desc_put_fr(struct ib_conn *ib_conn, } int iser_dma_map_task_data(struct iscsi_iser_task *iser_task, - struct iser_data_buf *data, - enum iser_data_dir iser_dir, - enum dma_data_direction dma_dir) + struct iser_data_buf *data, + enum iser_data_dir iser_dir, + enum dma_data_direction dma_dir) { struct ib_device *dev; @@ -100,9 +98,8 @@ void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task, ib_dma_unmap_sg(dev, data->sg, data->size, dir); } -static int -iser_reg_dma(struct iser_device *device, struct iser_data_buf *mem, - struct iser_mem_reg *reg) +static int iser_reg_dma(struct iser_device *device, struct iser_data_buf *mem, + struct iser_mem_reg *reg) { struct scatterlist *sg = mem->sg; @@ -154,8 +151,8 @@ void iser_unreg_mem_fastreg(struct iscsi_iser_task *iser_task, reg->mem_h = NULL; } -static void -iser_set_dif_domain(struct scsi_cmnd *sc, struct ib_sig_domain *domain) +static void iser_set_dif_domain(struct scsi_cmnd *sc, + struct ib_sig_domain *domain) { domain->sig_type = IB_SIG_TYPE_T10_DIF; domain->sig.dif.pi_interval = scsi_prot_interval(sc); @@ -171,8 +168,8 @@ iser_set_dif_domain(struct scsi_cmnd *sc, struct ib_sig_domain *domain) domain->sig.dif.ref_remap = true; } -static int -iser_set_sig_attrs(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs) +static int iser_set_sig_attrs(struct scsi_cmnd *sc, + struct ib_sig_attrs *sig_attrs) { switch (scsi_get_prot_op(sc)) { case SCSI_PROT_WRITE_INSERT: @@ -205,8 +202,7 @@ iser_set_sig_attrs(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs) return 0; } -static inline void -iser_set_prot_checks(struct scsi_cmnd *sc, u8 *mask) +static inline void iser_set_prot_checks(struct scsi_cmnd *sc, u8 *mask) { *mask = 0; if (sc->prot_flags & SCSI_PROT_REF_CHECK) @@ -215,11 +211,8 @@ iser_set_prot_checks(struct scsi_cmnd *sc, u8 *mask) *mask |= IB_SIG_CHECK_GUARD; } -static inline void -iser_inv_rkey(struct ib_send_wr *inv_wr, - struct ib_mr *mr, - struct ib_cqe *cqe, - struct ib_send_wr *next_wr) +static inline void iser_inv_rkey(struct ib_send_wr *inv_wr, struct ib_mr *mr, + struct ib_cqe *cqe, struct ib_send_wr *next_wr) { inv_wr->opcode = IB_WR_LOCAL_INV; inv_wr->wr_cqe = cqe; @@ -229,12 +222,11 @@ iser_inv_rkey(struct ib_send_wr *inv_wr, inv_wr->next = next_wr; } -static int -iser_reg_sig_mr(struct iscsi_iser_task *iser_task, - struct iser_data_buf *mem, - struct iser_data_buf *sig_mem, - struct iser_reg_resources *rsc, - struct iser_mem_reg *sig_reg) +static int iser_reg_sig_mr(struct iscsi_iser_task *iser_task, + struct iser_data_buf *mem, + struct iser_data_buf *sig_mem, + struct iser_reg_resources *rsc, + struct iser_mem_reg *sig_reg) { struct iser_tx_desc *tx_desc = &iser_task->desc; struct ib_cqe *cqe = &iser_task->iser_conn->ib_conn.reg_cqe; @@ -335,12 +327,10 @@ static int iser_fast_reg_mr(struct iscsi_iser_task *iser_task, return 0; } -static int -iser_reg_data_sg(struct iscsi_iser_task *task, - struct iser_data_buf *mem, - struct iser_fr_desc *desc, - bool use_dma_key, - struct iser_mem_reg *reg) +static int iser_reg_data_sg(struct iscsi_iser_task *task, + struct iser_data_buf *mem, + struct iser_fr_desc *desc, bool use_dma_key, + struct iser_mem_reg *reg) { struct iser_device *device = task->iser_conn->ib_conn.device; diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c index b566f7cb7797..8bf87b073d9b 100644 --- a/drivers/infiniband/ulp/iser/iser_verbs.c +++ b/drivers/infiniband/ulp/iser/iser_verbs.c @@ -265,14 +265,14 @@ static int iser_create_ib_conn_res(struct ib_conn *ib_conn) memset(&init_attr, 0, sizeof(init_attr)); init_attr.event_handler = iser_qp_event_callback; - init_attr.qp_context = (void *)ib_conn; - init_attr.send_cq = ib_conn->cq; - init_attr.recv_cq = ib_conn->cq; - init_attr.cap.max_recv_wr = ISER_QP_MAX_RECV_DTOS; + init_attr.qp_context = (void *)ib_conn; + init_attr.send_cq = ib_conn->cq; + init_attr.recv_cq = ib_conn->cq; + init_attr.cap.max_recv_wr = ISER_QP_MAX_RECV_DTOS; init_attr.cap.max_send_sge = 2; init_attr.cap.max_recv_sge = 1; - init_attr.sq_sig_type = IB_SIGNAL_REQ_WR; - init_attr.qp_type = IB_QPT_RC; + init_attr.sq_sig_type = IB_SIGNAL_REQ_WR; + init_attr.qp_type = IB_QPT_RC; init_attr.cap.max_send_wr = max_send_wr; if (ib_conn->pi_support) init_attr.create_flags |= IB_QP_CREATE_INTEGRITY_EN; @@ -283,9 +283,8 @@ static int iser_create_ib_conn_res(struct ib_conn *ib_conn) goto out_err; ib_conn->qp = ib_conn->cma_id->qp; - iser_info("setting conn %p cma_id %p qp %p max_send_wr %d\n", - ib_conn, ib_conn->cma_id, - ib_conn->cma_id->qp, max_send_wr); + iser_info("setting conn %p cma_id %p qp %p max_send_wr %d\n", ib_conn, + ib_conn->cma_id, ib_conn->cma_id->qp, max_send_wr); return ret; out_err: @@ -313,7 +312,7 @@ struct iser_device *iser_device_find_by_ib_device(struct rdma_cm_id *cma_id) goto inc_refcnt; device = kzalloc(sizeof *device, GFP_KERNEL); - if (device == NULL) + if (!device) goto out; /* assign this device to the device */ @@ -392,8 +391,7 @@ void iser_release_work(struct work_struct *work) * so the cm_id removal is out of here. It is Safe to * be invoked multiple times. */ -static void iser_free_ib_conn_res(struct iser_conn *iser_conn, - bool destroy) +static void iser_free_ib_conn_res(struct iser_conn *iser_conn, bool destroy) { struct ib_conn *ib_conn = &iser_conn->ib_conn; struct iser_device *device = ib_conn->device; @@ -401,7 +399,7 @@ static void iser_free_ib_conn_res(struct iser_conn *iser_conn, iser_info("freeing conn %p cma_id %p qp %p\n", iser_conn, ib_conn->cma_id, ib_conn->qp); - if (ib_conn->qp != NULL) { + if (ib_conn->qp) { rdma_destroy_qp(ib_conn->cma_id); ib_cq_pool_put(ib_conn->cq, ib_conn->cq_size); ib_conn->qp = NULL; @@ -411,7 +409,7 @@ static void iser_free_ib_conn_res(struct iser_conn *iser_conn, if (iser_conn->rx_descs) iser_free_rx_descriptors(iser_conn); - if (device != NULL) { + if (device) { iser_device_try_release(device); ib_conn->device = NULL; } @@ -445,7 +443,7 @@ void iser_conn_release(struct iser_conn *iser_conn) iser_free_ib_conn_res(iser_conn, true); mutex_unlock(&iser_conn->state_mutex); - if (ib_conn->cma_id != NULL) { + if (ib_conn->cma_id) { rdma_destroy_id(ib_conn->cma_id); ib_conn->cma_id = NULL; } @@ -501,13 +499,12 @@ static void iser_connect_error(struct rdma_cm_id *cma_id) { struct iser_conn *iser_conn; - iser_conn = (struct iser_conn *)cma_id->context; + iser_conn = cma_id->context; iser_conn->state = ISER_CONN_TERMINATING; } -static void -iser_calc_scsi_params(struct iser_conn *iser_conn, - unsigned int max_sectors) +static void iser_calc_scsi_params(struct iser_conn *iser_conn, + unsigned int max_sectors) { struct iser_device *device = iser_conn->ib_conn.device; struct ib_device_attr *attr = &device->ib_device->attrs; @@ -545,11 +542,11 @@ iser_calc_scsi_params(struct iser_conn *iser_conn, static void iser_addr_handler(struct rdma_cm_id *cma_id) { struct iser_device *device; - struct iser_conn *iser_conn; - struct ib_conn *ib_conn; + struct iser_conn *iser_conn; + struct ib_conn *ib_conn; int ret; - iser_conn = (struct iser_conn *)cma_id->context; + iser_conn = cma_id->context; if (iser_conn->state != ISER_CONN_PENDING) /* bailout */ return; @@ -593,9 +590,9 @@ static void iser_addr_handler(struct rdma_cm_id *cma_id) static void iser_route_handler(struct rdma_cm_id *cma_id) { struct rdma_conn_param conn_param; - int ret; + int ret; struct iser_cm_hdr req_hdr; - struct iser_conn *iser_conn = (struct iser_conn *)cma_id->context; + struct iser_conn *iser_conn = cma_id->context; struct ib_conn *ib_conn = &iser_conn->ib_conn; struct ib_device *ib_dev = ib_conn->device->ib_device; @@ -609,9 +606,9 @@ static void iser_route_handler(struct rdma_cm_id *cma_id) memset(&conn_param, 0, sizeof conn_param); conn_param.responder_resources = ib_dev->attrs.max_qp_rd_atom; - conn_param.initiator_depth = 1; - conn_param.retry_count = 7; - conn_param.rnr_retry_count = 6; + conn_param.initiator_depth = 1; + conn_param.retry_count = 7; + conn_param.rnr_retry_count = 6; memset(&req_hdr, 0, sizeof(req_hdr)); req_hdr.flags = ISER_ZBVA_NOT_SUP; @@ -638,7 +635,7 @@ static void iser_connected_handler(struct rdma_cm_id *cma_id, struct ib_qp_attr attr; struct ib_qp_init_attr init_attr; - iser_conn = (struct iser_conn *)cma_id->context; + iser_conn = cma_id->context; if (iser_conn->state != ISER_CONN_PENDING) /* bailout */ return; @@ -661,7 +658,7 @@ static void iser_connected_handler(struct rdma_cm_id *cma_id, static void iser_disconnected_handler(struct rdma_cm_id *cma_id) { - struct iser_conn *iser_conn = (struct iser_conn *)cma_id->context; + struct iser_conn *iser_conn = cma_id->context; if (iser_conn_terminate(iser_conn)) { if (iser_conn->iscsi_conn) @@ -675,7 +672,7 @@ static void iser_disconnected_handler(struct rdma_cm_id *cma_id) static void iser_cleanup_handler(struct rdma_cm_id *cma_id, bool destroy) { - struct iser_conn *iser_conn = (struct iser_conn *)cma_id->context; + struct iser_conn *iser_conn = cma_id->context; /* * We are not guaranteed that we visited disconnected_handler @@ -687,12 +684,13 @@ static void iser_cleanup_handler(struct rdma_cm_id *cma_id, complete(&iser_conn->ib_completion); } -static int iser_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) +static int iser_cma_handler(struct rdma_cm_id *cma_id, + struct rdma_cm_event *event) { struct iser_conn *iser_conn; int ret = 0; - iser_conn = (struct iser_conn *)cma_id->context; + iser_conn = cma_id->context; iser_info("%s (%d): status %d conn %p id %p\n", rdma_event_msg(event->event), event->event, event->status, cma_id->context, cma_id); @@ -757,7 +755,6 @@ void iser_conn_init(struct iser_conn *iser_conn) INIT_LIST_HEAD(&iser_conn->conn_list); mutex_init(&iser_conn->state_mutex); - ib_conn->post_recv_buf_count = 0; ib_conn->reg_cqe.done = iser_reg_comp; } @@ -765,10 +762,8 @@ void iser_conn_init(struct iser_conn *iser_conn) * starts the process of connecting to the target * sleeps until the connection is established or rejected */ -int iser_connect(struct iser_conn *iser_conn, - struct sockaddr *src_addr, - struct sockaddr *dst_addr, - int non_blocking) +int iser_connect(struct iser_conn *iser_conn, struct sockaddr *src_addr, + struct sockaddr *dst_addr, int non_blocking) { struct ib_conn *ib_conn = &iser_conn->ib_conn; int err = 0; @@ -785,8 +780,7 @@ int iser_connect(struct iser_conn *iser_conn, iser_conn->state = ISER_CONN_PENDING; ib_conn->cma_id = rdma_create_id(&init_net, iser_cma_handler, - (void *)iser_conn, - RDMA_PS_TCP, IB_QPT_RC); + iser_conn, RDMA_PS_TCP, IB_QPT_RC); if (IS_ERR(ib_conn->cma_id)) { err = PTR_ERR(ib_conn->cma_id); iser_err("rdma_create_id failed: %d\n", err); @@ -829,7 +823,7 @@ int iser_post_recvl(struct iser_conn *iser_conn) struct ib_conn *ib_conn = &iser_conn->ib_conn; struct iser_login_desc *desc = &iser_conn->login_desc; struct ib_recv_wr wr; - int ib_ret; + int ret; desc->sge.addr = desc->rsp_dma; desc->sge.length = ISER_RX_LOGIN_SIZE; @@ -841,46 +835,30 @@ int iser_post_recvl(struct iser_conn *iser_conn) wr.num_sge = 1; wr.next = NULL; - ib_conn->post_recv_buf_count++; - ib_ret = ib_post_recv(ib_conn->qp, &wr, NULL); - if (ib_ret) { - iser_err("ib_post_recv failed ret=%d\n", ib_ret); - ib_conn->post_recv_buf_count--; - } + ret = ib_post_recv(ib_conn->qp, &wr, NULL); + if (unlikely(ret)) + iser_err("ib_post_recv login failed ret=%d\n", ret); - return ib_ret; + return ret; } -int iser_post_recvm(struct iser_conn *iser_conn, int count) +int iser_post_recvm(struct iser_conn *iser_conn, struct iser_rx_desc *rx_desc) { struct ib_conn *ib_conn = &iser_conn->ib_conn; - unsigned int my_rx_head = iser_conn->rx_desc_head; - struct iser_rx_desc *rx_desc; - struct ib_recv_wr *wr; - int i, ib_ret; + struct ib_recv_wr wr; + int ret; - for (wr = ib_conn->rx_wr, i = 0; i < count; i++, wr++) { - rx_desc = &iser_conn->rx_descs[my_rx_head]; - rx_desc->cqe.done = iser_task_rsp; - wr->wr_cqe = &rx_desc->cqe; - wr->sg_list = &rx_desc->rx_sg; - wr->num_sge = 1; - wr->next = wr + 1; - my_rx_head = (my_rx_head + 1) & iser_conn->qp_max_recv_dtos_mask; - } + rx_desc->cqe.done = iser_task_rsp; + wr.wr_cqe = &rx_desc->cqe; + wr.sg_list = &rx_desc->rx_sg; + wr.num_sge = 1; + wr.next = NULL; - wr--; - wr->next = NULL; /* mark end of work requests list */ + ret = ib_post_recv(ib_conn->qp, &wr, NULL); + if (unlikely(ret)) + iser_err("ib_post_recv failed ret=%d\n", ret); - ib_conn->post_recv_buf_count += count; - ib_ret = ib_post_recv(ib_conn->qp, ib_conn->rx_wr, NULL); - if (unlikely(ib_ret)) { - iser_err("ib_post_recv failed ret=%d\n", ib_ret); - ib_conn->post_recv_buf_count -= count; - } else - iser_conn->rx_desc_head = my_rx_head; - - return ib_ret; + return ret; } @@ -888,16 +866,14 @@ int iser_post_recvm(struct iser_conn *iser_conn, int count) * iser_post_send - Initiate a Send DTO operation * @ib_conn: connection RDMA resources * @tx_desc: iSER TX descriptor - * @signal: true to send work request as SIGNALED * * Return: 0 on success, -1 on failure */ -int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc, - bool signal) +int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc) { struct ib_send_wr *wr = &tx_desc->send_wr; struct ib_send_wr *first_wr; - int ib_ret; + int ret; ib_dma_sync_single_for_device(ib_conn->device->ib_device, tx_desc->dma_addr, ISER_HEADERS_LEN, @@ -908,7 +884,7 @@ int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc, wr->sg_list = tx_desc->tx_sg; wr->num_sge = tx_desc->num_sge; wr->opcode = IB_WR_SEND; - wr->send_flags = signal ? IB_SEND_SIGNALED : 0; + wr->send_flags = IB_SEND_SIGNALED; if (tx_desc->inv_wr.next) first_wr = &tx_desc->inv_wr; @@ -917,12 +893,12 @@ int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc, else first_wr = wr; - ib_ret = ib_post_send(ib_conn->qp, first_wr, NULL); - if (unlikely(ib_ret)) + ret = ib_post_send(ib_conn->qp, first_wr, NULL); + if (unlikely(ret)) iser_err("ib_post_send failed, ret:%d opcode:%d\n", - ib_ret, wr->opcode); + ret, wr->opcode); - return ib_ret; + return ret; } u8 iser_check_task_pi_status(struct iscsi_iser_task *iser_task, diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt-stats.c b/drivers/infiniband/ulp/rtrs/rtrs-clt-stats.c index 76e4352fe3f6..385a19846c24 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-clt-stats.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt-stats.c @@ -13,8 +13,8 @@ void rtrs_clt_update_wc_stats(struct rtrs_clt_con *con) { - struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); - struct rtrs_clt_stats *stats = sess->stats; + struct rtrs_clt_path *clt_path = to_clt_path(con->c.path); + struct rtrs_clt_stats *stats = clt_path->stats; struct rtrs_clt_stats_pcpu *s; int cpu; @@ -180,8 +180,8 @@ static inline void rtrs_clt_update_rdma_stats(struct rtrs_clt_stats *stats, void rtrs_clt_update_all_stats(struct rtrs_clt_io_req *req, int dir) { struct rtrs_clt_con *con = req->con; - struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); - struct rtrs_clt_stats *stats = sess->stats; + struct rtrs_clt_path *clt_path = to_clt_path(con->c.path); + struct rtrs_clt_stats *stats = clt_path->stats; unsigned int len; len = req->usr_len + req->data_len; diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt-sysfs.c b/drivers/infiniband/ulp/rtrs/rtrs-clt-sysfs.c index 0e69180c3771..b4fa473b7888 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-clt-sysfs.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt-sysfs.c @@ -16,21 +16,21 @@ #define MIN_MAX_RECONN_ATT -1 #define MAX_MAX_RECONN_ATT 9999 -static void rtrs_clt_sess_release(struct kobject *kobj) +static void rtrs_clt_path_release(struct kobject *kobj) { - struct rtrs_clt_sess *sess; + struct rtrs_clt_path *clt_path; - sess = container_of(kobj, struct rtrs_clt_sess, kobj); + clt_path = container_of(kobj, struct rtrs_clt_path, kobj); - free_sess(sess); + free_path(clt_path); } static struct kobj_type ktype_sess = { .sysfs_ops = &kobj_sysfs_ops, - .release = rtrs_clt_sess_release + .release = rtrs_clt_path_release }; -static void rtrs_clt_sess_stats_release(struct kobject *kobj) +static void rtrs_clt_path_stats_release(struct kobject *kobj) { struct rtrs_clt_stats *stats; @@ -43,14 +43,15 @@ static void rtrs_clt_sess_stats_release(struct kobject *kobj) static struct kobj_type ktype_stats = { .sysfs_ops = &kobj_sysfs_ops, - .release = rtrs_clt_sess_stats_release, + .release = rtrs_clt_path_stats_release, }; static ssize_t max_reconnect_attempts_show(struct device *dev, struct device_attribute *attr, char *page) { - struct rtrs_clt *clt = container_of(dev, struct rtrs_clt, dev); + struct rtrs_clt_sess *clt = container_of(dev, struct rtrs_clt_sess, + dev); return sysfs_emit(page, "%d\n", rtrs_clt_get_max_reconnect_attempts(clt)); @@ -63,7 +64,8 @@ static ssize_t max_reconnect_attempts_store(struct device *dev, { int value; int ret; - struct rtrs_clt *clt = container_of(dev, struct rtrs_clt, dev); + struct rtrs_clt_sess *clt = container_of(dev, struct rtrs_clt_sess, + dev); ret = kstrtoint(buf, 10, &value); if (ret) { @@ -90,9 +92,9 @@ static ssize_t mpath_policy_show(struct device *dev, struct device_attribute *attr, char *page) { - struct rtrs_clt *clt; + struct rtrs_clt_sess *clt; - clt = container_of(dev, struct rtrs_clt, dev); + clt = container_of(dev, struct rtrs_clt_sess, dev); switch (clt->mp_policy) { case MP_POLICY_RR: @@ -114,12 +116,12 @@ static ssize_t mpath_policy_store(struct device *dev, const char *buf, size_t count) { - struct rtrs_clt *clt; + struct rtrs_clt_sess *clt; int value; int ret; size_t len = 0; - clt = container_of(dev, struct rtrs_clt, dev); + clt = container_of(dev, struct rtrs_clt_sess, dev); ret = kstrtoint(buf, 10, &value); if (!ret && (value == MP_POLICY_RR || @@ -169,12 +171,12 @@ static ssize_t add_path_store(struct device *dev, .src = &srcaddr, .dst = &dstaddr }; - struct rtrs_clt *clt; + struct rtrs_clt_sess *clt; const char *nl; size_t len; int err; - clt = container_of(dev, struct rtrs_clt, dev); + clt = container_of(dev, struct rtrs_clt_sess, dev); nl = strchr(buf, '\n'); if (nl) @@ -197,10 +199,10 @@ static DEVICE_ATTR_RW(add_path); static ssize_t rtrs_clt_state_show(struct kobject *kobj, struct kobj_attribute *attr, char *page) { - struct rtrs_clt_sess *sess; + struct rtrs_clt_path *clt_path; - sess = container_of(kobj, struct rtrs_clt_sess, kobj); - if (sess->state == RTRS_CLT_CONNECTED) + clt_path = container_of(kobj, struct rtrs_clt_path, kobj); + if (clt_path->state == RTRS_CLT_CONNECTED) return sysfs_emit(page, "connected\n"); return sysfs_emit(page, "disconnected\n"); @@ -219,16 +221,16 @@ static ssize_t rtrs_clt_reconnect_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { - struct rtrs_clt_sess *sess; + struct rtrs_clt_path *clt_path; int ret; - sess = container_of(kobj, struct rtrs_clt_sess, kobj); + clt_path = container_of(kobj, struct rtrs_clt_path, kobj); if (!sysfs_streq(buf, "1")) { - rtrs_err(sess->clt, "%s: unknown value: '%s'\n", + rtrs_err(clt_path->clt, "%s: unknown value: '%s'\n", attr->attr.name, buf); return -EINVAL; } - ret = rtrs_clt_reconnect_from_sysfs(sess); + ret = rtrs_clt_reconnect_from_sysfs(clt_path); if (ret) return ret; @@ -249,15 +251,15 @@ static ssize_t rtrs_clt_disconnect_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { - struct rtrs_clt_sess *sess; + struct rtrs_clt_path *clt_path; - sess = container_of(kobj, struct rtrs_clt_sess, kobj); + clt_path = container_of(kobj, struct rtrs_clt_path, kobj); if (!sysfs_streq(buf, "1")) { - rtrs_err(sess->clt, "%s: unknown value: '%s'\n", + rtrs_err(clt_path->clt, "%s: unknown value: '%s'\n", attr->attr.name, buf); return -EINVAL; } - rtrs_clt_close_conns(sess, true); + rtrs_clt_close_conns(clt_path, true); return count; } @@ -276,16 +278,16 @@ static ssize_t rtrs_clt_remove_path_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { - struct rtrs_clt_sess *sess; + struct rtrs_clt_path *clt_path; int ret; - sess = container_of(kobj, struct rtrs_clt_sess, kobj); + clt_path = container_of(kobj, struct rtrs_clt_path, kobj); if (!sysfs_streq(buf, "1")) { - rtrs_err(sess->clt, "%s: unknown value: '%s'\n", + rtrs_err(clt_path->clt, "%s: unknown value: '%s'\n", attr->attr.name, buf); return -EINVAL; } - ret = rtrs_clt_remove_path_from_sysfs(sess, &attr->attr); + ret = rtrs_clt_remove_path_from_sysfs(clt_path, &attr->attr); if (ret) return ret; @@ -333,11 +335,11 @@ static ssize_t rtrs_clt_hca_port_show(struct kobject *kobj, struct kobj_attribute *attr, char *page) { - struct rtrs_clt_sess *sess; + struct rtrs_clt_path *clt_path; - sess = container_of(kobj, typeof(*sess), kobj); + clt_path = container_of(kobj, typeof(*clt_path), kobj); - return sysfs_emit(page, "%u\n", sess->hca_port); + return sysfs_emit(page, "%u\n", clt_path->hca_port); } static struct kobj_attribute rtrs_clt_hca_port_attr = @@ -347,11 +349,11 @@ static ssize_t rtrs_clt_hca_name_show(struct kobject *kobj, struct kobj_attribute *attr, char *page) { - struct rtrs_clt_sess *sess; + struct rtrs_clt_path *clt_path; - sess = container_of(kobj, struct rtrs_clt_sess, kobj); + clt_path = container_of(kobj, struct rtrs_clt_path, kobj); - return sysfs_emit(page, "%s\n", sess->hca_name); + return sysfs_emit(page, "%s\n", clt_path->hca_name); } static struct kobj_attribute rtrs_clt_hca_name_attr = @@ -361,12 +363,12 @@ static ssize_t rtrs_clt_cur_latency_show(struct kobject *kobj, struct kobj_attribute *attr, char *page) { - struct rtrs_clt_sess *sess; + struct rtrs_clt_path *clt_path; - sess = container_of(kobj, struct rtrs_clt_sess, kobj); + clt_path = container_of(kobj, struct rtrs_clt_path, kobj); return sysfs_emit(page, "%lld ns\n", - ktime_to_ns(sess->s.hb_cur_latency)); + ktime_to_ns(clt_path->s.hb_cur_latency)); } static struct kobj_attribute rtrs_clt_cur_latency_attr = @@ -376,11 +378,11 @@ static ssize_t rtrs_clt_src_addr_show(struct kobject *kobj, struct kobj_attribute *attr, char *page) { - struct rtrs_clt_sess *sess; + struct rtrs_clt_path *clt_path; int len; - sess = container_of(kobj, struct rtrs_clt_sess, kobj); - len = sockaddr_to_str((struct sockaddr *)&sess->s.src_addr, page, + clt_path = container_of(kobj, struct rtrs_clt_path, kobj); + len = sockaddr_to_str((struct sockaddr *)&clt_path->s.src_addr, page, PAGE_SIZE); len += sysfs_emit_at(page, len, "\n"); return len; @@ -393,11 +395,11 @@ static ssize_t rtrs_clt_dst_addr_show(struct kobject *kobj, struct kobj_attribute *attr, char *page) { - struct rtrs_clt_sess *sess; + struct rtrs_clt_path *clt_path; int len; - sess = container_of(kobj, struct rtrs_clt_sess, kobj); - len = sockaddr_to_str((struct sockaddr *)&sess->s.dst_addr, page, + clt_path = container_of(kobj, struct rtrs_clt_path, kobj); + len = sockaddr_to_str((struct sockaddr *)&clt_path->s.dst_addr, page, PAGE_SIZE); len += sysfs_emit_at(page, len, "\n"); return len; @@ -406,7 +408,7 @@ static ssize_t rtrs_clt_dst_addr_show(struct kobject *kobj, static struct kobj_attribute rtrs_clt_dst_addr_attr = __ATTR(dst_addr, 0444, rtrs_clt_dst_addr_show, NULL); -static struct attribute *rtrs_clt_sess_attrs[] = { +static struct attribute *rtrs_clt_path_attrs[] = { &rtrs_clt_hca_name_attr.attr, &rtrs_clt_hca_port_attr.attr, &rtrs_clt_src_addr_attr.attr, @@ -419,42 +421,43 @@ static struct attribute *rtrs_clt_sess_attrs[] = { NULL, }; -static const struct attribute_group rtrs_clt_sess_attr_group = { - .attrs = rtrs_clt_sess_attrs, +static const struct attribute_group rtrs_clt_path_attr_group = { + .attrs = rtrs_clt_path_attrs, }; -int rtrs_clt_create_sess_files(struct rtrs_clt_sess *sess) +int rtrs_clt_create_path_files(struct rtrs_clt_path *clt_path) { - struct rtrs_clt *clt = sess->clt; + struct rtrs_clt_sess *clt = clt_path->clt; char str[NAME_MAX]; int err; struct rtrs_addr path = { - .src = &sess->s.src_addr, - .dst = &sess->s.dst_addr, + .src = &clt_path->s.src_addr, + .dst = &clt_path->s.dst_addr, }; rtrs_addr_to_str(&path, str, sizeof(str)); - err = kobject_init_and_add(&sess->kobj, &ktype_sess, clt->kobj_paths, + err = kobject_init_and_add(&clt_path->kobj, &ktype_sess, + clt->kobj_paths, "%s", str); if (err) { pr_err("kobject_init_and_add: %d\n", err); - kobject_put(&sess->kobj); + kobject_put(&clt_path->kobj); return err; } - err = sysfs_create_group(&sess->kobj, &rtrs_clt_sess_attr_group); + err = sysfs_create_group(&clt_path->kobj, &rtrs_clt_path_attr_group); if (err) { pr_err("sysfs_create_group(): %d\n", err); goto put_kobj; } - err = kobject_init_and_add(&sess->stats->kobj_stats, &ktype_stats, - &sess->kobj, "stats"); + err = kobject_init_and_add(&clt_path->stats->kobj_stats, &ktype_stats, + &clt_path->kobj, "stats"); if (err) { pr_err("kobject_init_and_add: %d\n", err); - kobject_put(&sess->stats->kobj_stats); + kobject_put(&clt_path->stats->kobj_stats); goto remove_group; } - err = sysfs_create_group(&sess->stats->kobj_stats, + err = sysfs_create_group(&clt_path->stats->kobj_stats, &rtrs_clt_stats_attr_group); if (err) { pr_err("failed to create stats sysfs group, err: %d\n", err); @@ -464,25 +467,25 @@ int rtrs_clt_create_sess_files(struct rtrs_clt_sess *sess) return 0; put_kobj_stats: - kobject_del(&sess->stats->kobj_stats); - kobject_put(&sess->stats->kobj_stats); + kobject_del(&clt_path->stats->kobj_stats); + kobject_put(&clt_path->stats->kobj_stats); remove_group: - sysfs_remove_group(&sess->kobj, &rtrs_clt_sess_attr_group); + sysfs_remove_group(&clt_path->kobj, &rtrs_clt_path_attr_group); put_kobj: - kobject_del(&sess->kobj); - kobject_put(&sess->kobj); + kobject_del(&clt_path->kobj); + kobject_put(&clt_path->kobj); return err; } -void rtrs_clt_destroy_sess_files(struct rtrs_clt_sess *sess, +void rtrs_clt_destroy_path_files(struct rtrs_clt_path *clt_path, const struct attribute *sysfs_self) { - kobject_del(&sess->stats->kobj_stats); - kobject_put(&sess->stats->kobj_stats); + kobject_del(&clt_path->stats->kobj_stats); + kobject_put(&clt_path->stats->kobj_stats); if (sysfs_self) - sysfs_remove_file_self(&sess->kobj, sysfs_self); - kobject_del(&sess->kobj); + sysfs_remove_file_self(&clt_path->kobj, sysfs_self); + kobject_del(&clt_path->kobj); } static struct attribute *rtrs_clt_attrs[] = { @@ -496,12 +499,12 @@ static const struct attribute_group rtrs_clt_attr_group = { .attrs = rtrs_clt_attrs, }; -int rtrs_clt_create_sysfs_root_files(struct rtrs_clt *clt) +int rtrs_clt_create_sysfs_root_files(struct rtrs_clt_sess *clt) { return sysfs_create_group(&clt->dev.kobj, &rtrs_clt_attr_group); } -void rtrs_clt_destroy_sysfs_root(struct rtrs_clt *clt) +void rtrs_clt_destroy_sysfs_root(struct rtrs_clt_sess *clt) { sysfs_remove_group(&clt->dev.kobj, &rtrs_clt_attr_group); diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c index 15c0077dd27e..7c3f98e57889 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c @@ -46,21 +46,21 @@ static struct rtrs_rdma_dev_pd dev_pd = { static struct workqueue_struct *rtrs_wq; static struct class *rtrs_clt_dev_class; -static inline bool rtrs_clt_is_connected(const struct rtrs_clt *clt) +static inline bool rtrs_clt_is_connected(const struct rtrs_clt_sess *clt) { - struct rtrs_clt_sess *sess; + struct rtrs_clt_path *clt_path; bool connected = false; rcu_read_lock(); - list_for_each_entry_rcu(sess, &clt->paths_list, s.entry) - connected |= READ_ONCE(sess->state) == RTRS_CLT_CONNECTED; + list_for_each_entry_rcu(clt_path, &clt->paths_list, s.entry) + connected |= READ_ONCE(clt_path->state) == RTRS_CLT_CONNECTED; rcu_read_unlock(); return connected; } static struct rtrs_permit * -__rtrs_get_permit(struct rtrs_clt *clt, enum rtrs_clt_con_type con_type) +__rtrs_get_permit(struct rtrs_clt_sess *clt, enum rtrs_clt_con_type con_type) { size_t max_depth = clt->queue_depth; struct rtrs_permit *permit; @@ -87,7 +87,7 @@ __rtrs_get_permit(struct rtrs_clt *clt, enum rtrs_clt_con_type con_type) return permit; } -static inline void __rtrs_put_permit(struct rtrs_clt *clt, +static inline void __rtrs_put_permit(struct rtrs_clt_sess *clt, struct rtrs_permit *permit) { clear_bit_unlock(permit->mem_id, clt->permits_map); @@ -107,7 +107,7 @@ static inline void __rtrs_put_permit(struct rtrs_clt *clt, * Context: * Can sleep if @wait == RTRS_PERMIT_WAIT */ -struct rtrs_permit *rtrs_clt_get_permit(struct rtrs_clt *clt, +struct rtrs_permit *rtrs_clt_get_permit(struct rtrs_clt_sess *clt, enum rtrs_clt_con_type con_type, enum wait_type can_wait) { @@ -142,7 +142,8 @@ EXPORT_SYMBOL(rtrs_clt_get_permit); * Context: * Does not matter */ -void rtrs_clt_put_permit(struct rtrs_clt *clt, struct rtrs_permit *permit) +void rtrs_clt_put_permit(struct rtrs_clt_sess *clt, + struct rtrs_permit *permit) { if (WARN_ON(!test_bit(permit->mem_id, clt->permits_map))) return; @@ -163,29 +164,29 @@ EXPORT_SYMBOL(rtrs_clt_put_permit); /** * rtrs_permit_to_clt_con() - returns RDMA connection pointer by the permit - * @sess: client session pointer + * @clt_path: client path pointer * @permit: permit for the allocation of the RDMA buffer * Note: * IO connection starts from 1. * 0 connection is for user messages. */ static -struct rtrs_clt_con *rtrs_permit_to_clt_con(struct rtrs_clt_sess *sess, +struct rtrs_clt_con *rtrs_permit_to_clt_con(struct rtrs_clt_path *clt_path, struct rtrs_permit *permit) { int id = 0; if (permit->con_type == RTRS_IO_CON) - id = (permit->cpu_id % (sess->s.irq_con_num - 1)) + 1; + id = (permit->cpu_id % (clt_path->s.irq_con_num - 1)) + 1; - return to_clt_con(sess->s.con[id]); + return to_clt_con(clt_path->s.con[id]); } /** * rtrs_clt_change_state() - change the session state through session state * machine. * - * @sess: client session to change the state of. + * @clt_path: client path to change the state of. * @new_state: state to change to. * * returns true if sess's state is changed to new state, otherwise return false. @@ -193,15 +194,15 @@ struct rtrs_clt_con *rtrs_permit_to_clt_con(struct rtrs_clt_sess *sess, * Locks: * state_wq lock must be hold. */ -static bool rtrs_clt_change_state(struct rtrs_clt_sess *sess, +static bool rtrs_clt_change_state(struct rtrs_clt_path *clt_path, enum rtrs_clt_state new_state) { enum rtrs_clt_state old_state; bool changed = false; - lockdep_assert_held(&sess->state_wq.lock); + lockdep_assert_held(&clt_path->state_wq.lock); - old_state = sess->state; + old_state = clt_path->state; switch (new_state) { case RTRS_CLT_CONNECTING: switch (old_state) { @@ -275,42 +276,42 @@ static bool rtrs_clt_change_state(struct rtrs_clt_sess *sess, break; } if (changed) { - sess->state = new_state; - wake_up_locked(&sess->state_wq); + clt_path->state = new_state; + wake_up_locked(&clt_path->state_wq); } return changed; } -static bool rtrs_clt_change_state_from_to(struct rtrs_clt_sess *sess, +static bool rtrs_clt_change_state_from_to(struct rtrs_clt_path *clt_path, enum rtrs_clt_state old_state, enum rtrs_clt_state new_state) { bool changed = false; - spin_lock_irq(&sess->state_wq.lock); - if (sess->state == old_state) - changed = rtrs_clt_change_state(sess, new_state); - spin_unlock_irq(&sess->state_wq.lock); + spin_lock_irq(&clt_path->state_wq.lock); + if (clt_path->state == old_state) + changed = rtrs_clt_change_state(clt_path, new_state); + spin_unlock_irq(&clt_path->state_wq.lock); return changed; } static void rtrs_rdma_error_recovery(struct rtrs_clt_con *con) { - struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); + struct rtrs_clt_path *clt_path = to_clt_path(con->c.path); - if (rtrs_clt_change_state_from_to(sess, + if (rtrs_clt_change_state_from_to(clt_path, RTRS_CLT_CONNECTED, RTRS_CLT_RECONNECTING)) { - struct rtrs_clt *clt = sess->clt; + struct rtrs_clt_sess *clt = clt_path->clt; unsigned int delay_ms; /* * Normal scenario, reconnect if we were successfully connected */ delay_ms = clt->reconnect_delay_sec * 1000; - queue_delayed_work(rtrs_wq, &sess->reconnect_dwork, + queue_delayed_work(rtrs_wq, &clt_path->reconnect_dwork, msecs_to_jiffies(delay_ms + prandom_u32() % RTRS_RECONNECT_SEED)); } else { @@ -319,7 +320,7 @@ static void rtrs_rdma_error_recovery(struct rtrs_clt_con *con) * so notify waiter with error state, waiter is responsible * for cleaning the rest and reconnect if needed. */ - rtrs_clt_change_state_from_to(sess, + rtrs_clt_change_state_from_to(clt_path, RTRS_CLT_CONNECTING, RTRS_CLT_CONNECTING_ERR); } @@ -330,7 +331,7 @@ static void rtrs_clt_fast_reg_done(struct ib_cq *cq, struct ib_wc *wc) struct rtrs_clt_con *con = to_clt_con(wc->qp->qp_context); if (wc->status != IB_WC_SUCCESS) { - rtrs_err(con->c.sess, "Failed IB_WR_REG_MR: %s\n", + rtrs_err(con->c.path, "Failed IB_WR_REG_MR: %s\n", ib_wc_status_msg(wc->status)); rtrs_rdma_error_recovery(con); } @@ -350,7 +351,7 @@ static void rtrs_clt_inv_rkey_done(struct ib_cq *cq, struct ib_wc *wc) struct rtrs_clt_con *con = to_clt_con(wc->qp->qp_context); if (wc->status != IB_WC_SUCCESS) { - rtrs_err(con->c.sess, "Failed IB_WR_LOCAL_INV: %s\n", + rtrs_err(con->c.path, "Failed IB_WR_LOCAL_INV: %s\n", ib_wc_status_msg(wc->status)); rtrs_rdma_error_recovery(con); } @@ -380,14 +381,14 @@ static void complete_rdma_req(struct rtrs_clt_io_req *req, int errno, bool notify, bool can_wait) { struct rtrs_clt_con *con = req->con; - struct rtrs_clt_sess *sess; + struct rtrs_clt_path *clt_path; int err; if (WARN_ON(!req->in_use)) return; if (WARN_ON(!req->con)) return; - sess = to_clt_sess(con->c.sess); + clt_path = to_clt_path(con->c.path); if (req->sg_cnt) { if (req->dir == DMA_FROM_DEVICE && req->need_inv) { @@ -417,7 +418,7 @@ static void complete_rdma_req(struct rtrs_clt_io_req *req, int errno, refcount_inc(&req->ref); err = rtrs_inv_rkey(req); if (err) { - rtrs_err(con->c.sess, "Send INV WR key=%#x: %d\n", + rtrs_err(con->c.path, "Send INV WR key=%#x: %d\n", req->mr->rkey, err); } else if (can_wait) { wait_for_completion(&req->inv_comp); @@ -433,21 +434,21 @@ static void complete_rdma_req(struct rtrs_clt_io_req *req, int errno, if (!refcount_dec_and_test(&req->ref)) return; } - ib_dma_unmap_sg(sess->s.dev->ib_dev, req->sglist, + ib_dma_unmap_sg(clt_path->s.dev->ib_dev, req->sglist, req->sg_cnt, req->dir); } if (!refcount_dec_and_test(&req->ref)) return; if (req->mp_policy == MP_POLICY_MIN_INFLIGHT) - atomic_dec(&sess->stats->inflight); + atomic_dec(&clt_path->stats->inflight); req->in_use = false; req->con = NULL; if (errno) { - rtrs_err_rl(con->c.sess, "IO request failed: error=%d path=%s [%s:%u] notify=%d\n", - errno, kobject_name(&sess->kobj), sess->hca_name, - sess->hca_port, notify); + rtrs_err_rl(con->c.path, "IO request failed: error=%d path=%s [%s:%u] notify=%d\n", + errno, kobject_name(&clt_path->kobj), clt_path->hca_name, + clt_path->hca_port, notify); } if (notify) @@ -459,12 +460,12 @@ static int rtrs_post_send_rdma(struct rtrs_clt_con *con, struct rtrs_rbuf *rbuf, u32 off, u32 imm, struct ib_send_wr *wr) { - struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); + struct rtrs_clt_path *clt_path = to_clt_path(con->c.path); enum ib_send_flags flags; struct ib_sge sge; if (!req->sg_size) { - rtrs_wrn(con->c.sess, + rtrs_wrn(con->c.path, "Doing RDMA Write failed, no data supplied\n"); return -EINVAL; } @@ -472,16 +473,17 @@ static int rtrs_post_send_rdma(struct rtrs_clt_con *con, /* user data and user message in the first list element */ sge.addr = req->iu->dma_addr; sge.length = req->sg_size; - sge.lkey = sess->s.dev->ib_pd->local_dma_lkey; + sge.lkey = clt_path->s.dev->ib_pd->local_dma_lkey; /* * From time to time we have to post signalled sends, * or send queue will fill up and only QP reset can help. */ - flags = atomic_inc_return(&con->c.wr_cnt) % sess->s.signal_interval ? + flags = atomic_inc_return(&con->c.wr_cnt) % clt_path->s.signal_interval ? 0 : IB_SEND_SIGNALED; - ib_dma_sync_single_for_device(sess->s.dev->ib_dev, req->iu->dma_addr, + ib_dma_sync_single_for_device(clt_path->s.dev->ib_dev, + req->iu->dma_addr, req->sg_size, DMA_TO_DEVICE); return rtrs_iu_post_rdma_write_imm(&con->c, req->iu, &sge, 1, @@ -489,15 +491,15 @@ static int rtrs_post_send_rdma(struct rtrs_clt_con *con, imm, flags, wr, NULL); } -static void process_io_rsp(struct rtrs_clt_sess *sess, u32 msg_id, +static void process_io_rsp(struct rtrs_clt_path *clt_path, u32 msg_id, s16 errno, bool w_inval) { struct rtrs_clt_io_req *req; - if (WARN_ON(msg_id >= sess->queue_depth)) + if (WARN_ON(msg_id >= clt_path->queue_depth)) return; - req = &sess->reqs[msg_id]; + req = &clt_path->reqs[msg_id]; /* Drop need_inv if server responded with send with invalidation */ req->need_inv &= !w_inval; complete_rdma_req(req, errno, true, false); @@ -507,21 +509,21 @@ static void rtrs_clt_recv_done(struct rtrs_clt_con *con, struct ib_wc *wc) { struct rtrs_iu *iu; int err; - struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); + struct rtrs_clt_path *clt_path = to_clt_path(con->c.path); - WARN_ON((sess->flags & RTRS_MSG_NEW_RKEY_F) == 0); + WARN_ON((clt_path->flags & RTRS_MSG_NEW_RKEY_F) == 0); iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe); err = rtrs_iu_post_recv(&con->c, iu); if (err) { - rtrs_err(con->c.sess, "post iu failed %d\n", err); + rtrs_err(con->c.path, "post iu failed %d\n", err); rtrs_rdma_error_recovery(con); } } static void rtrs_clt_rkey_rsp_done(struct rtrs_clt_con *con, struct ib_wc *wc) { - struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); + struct rtrs_clt_path *clt_path = to_clt_path(con->c.path); struct rtrs_msg_rkey_rsp *msg; u32 imm_type, imm_payload; bool w_inval = false; @@ -529,25 +531,26 @@ static void rtrs_clt_rkey_rsp_done(struct rtrs_clt_con *con, struct ib_wc *wc) u32 buf_id; int err; - WARN_ON((sess->flags & RTRS_MSG_NEW_RKEY_F) == 0); + WARN_ON((clt_path->flags & RTRS_MSG_NEW_RKEY_F) == 0); iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe); if (wc->byte_len < sizeof(*msg)) { - rtrs_err(con->c.sess, "rkey response is malformed: size %d\n", + rtrs_err(con->c.path, "rkey response is malformed: size %d\n", wc->byte_len); goto out; } - ib_dma_sync_single_for_cpu(sess->s.dev->ib_dev, iu->dma_addr, + ib_dma_sync_single_for_cpu(clt_path->s.dev->ib_dev, iu->dma_addr, iu->size, DMA_FROM_DEVICE); msg = iu->buf; if (le16_to_cpu(msg->type) != RTRS_MSG_RKEY_RSP) { - rtrs_err(sess->clt, "rkey response is malformed: type %d\n", + rtrs_err(clt_path->clt, + "rkey response is malformed: type %d\n", le16_to_cpu(msg->type)); goto out; } buf_id = le16_to_cpu(msg->buf_id); - if (WARN_ON(buf_id >= sess->queue_depth)) + if (WARN_ON(buf_id >= clt_path->queue_depth)) goto out; rtrs_from_imm(be32_to_cpu(wc->ex.imm_data), &imm_type, &imm_payload); @@ -560,10 +563,10 @@ static void rtrs_clt_rkey_rsp_done(struct rtrs_clt_con *con, struct ib_wc *wc) if (WARN_ON(buf_id != msg_id)) goto out; - sess->rbufs[buf_id].rkey = le32_to_cpu(msg->rkey); - process_io_rsp(sess, msg_id, err, w_inval); + clt_path->rbufs[buf_id].rkey = le32_to_cpu(msg->rkey); + process_io_rsp(clt_path, msg_id, err, w_inval); } - ib_dma_sync_single_for_device(sess->s.dev->ib_dev, iu->dma_addr, + ib_dma_sync_single_for_device(clt_path->s.dev->ib_dev, iu->dma_addr, iu->size, DMA_FROM_DEVICE); return rtrs_clt_recv_done(con, wc); out: @@ -600,14 +603,14 @@ static int rtrs_post_recv_empty_x2(struct rtrs_con *con, struct ib_cqe *cqe) static void rtrs_clt_rdma_done(struct ib_cq *cq, struct ib_wc *wc) { struct rtrs_clt_con *con = to_clt_con(wc->qp->qp_context); - struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); + struct rtrs_clt_path *clt_path = to_clt_path(con->c.path); u32 imm_type, imm_payload; bool w_inval = false; int err; if (wc->status != IB_WC_SUCCESS) { if (wc->status != IB_WC_WR_FLUSH_ERR) { - rtrs_err(sess->clt, "RDMA failed: %s\n", + rtrs_err(clt_path->clt, "RDMA failed: %s\n", ib_wc_status_msg(wc->status)); rtrs_rdma_error_recovery(con); } @@ -632,21 +635,21 @@ static void rtrs_clt_rdma_done(struct ib_cq *cq, struct ib_wc *wc) w_inval = (imm_type == RTRS_IO_RSP_W_INV_IMM); rtrs_from_io_rsp_imm(imm_payload, &msg_id, &err); - process_io_rsp(sess, msg_id, err, w_inval); + process_io_rsp(clt_path, msg_id, err, w_inval); } else if (imm_type == RTRS_HB_MSG_IMM) { WARN_ON(con->c.cid); - rtrs_send_hb_ack(&sess->s); - if (sess->flags & RTRS_MSG_NEW_RKEY_F) + rtrs_send_hb_ack(&clt_path->s); + if (clt_path->flags & RTRS_MSG_NEW_RKEY_F) return rtrs_clt_recv_done(con, wc); } else if (imm_type == RTRS_HB_ACK_IMM) { WARN_ON(con->c.cid); - sess->s.hb_missed_cnt = 0; - sess->s.hb_cur_latency = - ktime_sub(ktime_get(), sess->s.hb_last_sent); - if (sess->flags & RTRS_MSG_NEW_RKEY_F) + clt_path->s.hb_missed_cnt = 0; + clt_path->s.hb_cur_latency = + ktime_sub(ktime_get(), clt_path->s.hb_last_sent); + if (clt_path->flags & RTRS_MSG_NEW_RKEY_F) return rtrs_clt_recv_done(con, wc); } else { - rtrs_wrn(con->c.sess, "Unknown IMM type %u\n", + rtrs_wrn(con->c.path, "Unknown IMM type %u\n", imm_type); } if (w_inval) @@ -658,7 +661,7 @@ static void rtrs_clt_rdma_done(struct ib_cq *cq, struct ib_wc *wc) else err = rtrs_post_recv_empty(&con->c, &io_comp_cqe); if (err) { - rtrs_err(con->c.sess, "rtrs_post_recv_empty(): %d\n", + rtrs_err(con->c.path, "rtrs_post_recv_empty(): %d\n", err); rtrs_rdma_error_recovery(con); } @@ -670,7 +673,7 @@ static void rtrs_clt_rdma_done(struct ib_cq *cq, struct ib_wc *wc) WARN_ON(!(wc->wc_flags & IB_WC_WITH_INVALIDATE || wc->wc_flags & IB_WC_WITH_IMM)); WARN_ON(wc->wr_cqe->done != rtrs_clt_rdma_done); - if (sess->flags & RTRS_MSG_NEW_RKEY_F) { + if (clt_path->flags & RTRS_MSG_NEW_RKEY_F) { if (wc->wc_flags & IB_WC_WITH_INVALIDATE) return rtrs_clt_recv_done(con, wc); @@ -685,7 +688,7 @@ static void rtrs_clt_rdma_done(struct ib_cq *cq, struct ib_wc *wc) break; default: - rtrs_wrn(sess->clt, "Unexpected WC type: %d\n", wc->opcode); + rtrs_wrn(clt_path->clt, "Unexpected WC type: %d\n", wc->opcode); return; } } @@ -693,10 +696,10 @@ static void rtrs_clt_rdma_done(struct ib_cq *cq, struct ib_wc *wc) static int post_recv_io(struct rtrs_clt_con *con, size_t q_size) { int err, i; - struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); + struct rtrs_clt_path *clt_path = to_clt_path(con->c.path); for (i = 0; i < q_size; i++) { - if (sess->flags & RTRS_MSG_NEW_RKEY_F) { + if (clt_path->flags & RTRS_MSG_NEW_RKEY_F) { struct rtrs_iu *iu = &con->rsp_ius[i]; err = rtrs_iu_post_recv(&con->c, iu); @@ -710,16 +713,16 @@ static int post_recv_io(struct rtrs_clt_con *con, size_t q_size) return 0; } -static int post_recv_sess(struct rtrs_clt_sess *sess) +static int post_recv_path(struct rtrs_clt_path *clt_path) { size_t q_size = 0; int err, cid; - for (cid = 0; cid < sess->s.con_num; cid++) { + for (cid = 0; cid < clt_path->s.con_num; cid++) { if (cid == 0) q_size = SERVICE_CON_QUEUE_DEPTH; else - q_size = sess->queue_depth; + q_size = clt_path->queue_depth; /* * x2 for RDMA read responses + FR key invalidations, @@ -727,9 +730,10 @@ static int post_recv_sess(struct rtrs_clt_sess *sess) */ q_size *= 2; - err = post_recv_io(to_clt_con(sess->s.con[cid]), q_size); + err = post_recv_io(to_clt_con(clt_path->s.con[cid]), q_size); if (err) { - rtrs_err(sess->clt, "post_recv_io(), err: %d\n", err); + rtrs_err(clt_path->clt, "post_recv_io(), err: %d\n", + err); return err; } } @@ -740,8 +744,8 @@ static int post_recv_sess(struct rtrs_clt_sess *sess) struct path_it { int i; struct list_head skip_list; - struct rtrs_clt *clt; - struct rtrs_clt_sess *(*next_path)(struct path_it *it); + struct rtrs_clt_sess *clt; + struct rtrs_clt_path *(*next_path)(struct path_it *it); }; /** @@ -773,11 +777,11 @@ struct path_it { * Locks: * rcu_read_lock() must be hold. */ -static struct rtrs_clt_sess *get_next_path_rr(struct path_it *it) +static struct rtrs_clt_path *get_next_path_rr(struct path_it *it) { - struct rtrs_clt_sess __rcu **ppcpu_path; - struct rtrs_clt_sess *path; - struct rtrs_clt *clt; + struct rtrs_clt_path __rcu **ppcpu_path; + struct rtrs_clt_path *path; + struct rtrs_clt_sess *clt; clt = it->clt; @@ -811,26 +815,26 @@ static struct rtrs_clt_sess *get_next_path_rr(struct path_it *it) * Locks: * rcu_read_lock() must be hold. */ -static struct rtrs_clt_sess *get_next_path_min_inflight(struct path_it *it) +static struct rtrs_clt_path *get_next_path_min_inflight(struct path_it *it) { - struct rtrs_clt_sess *min_path = NULL; - struct rtrs_clt *clt = it->clt; - struct rtrs_clt_sess *sess; + struct rtrs_clt_path *min_path = NULL; + struct rtrs_clt_sess *clt = it->clt; + struct rtrs_clt_path *clt_path; int min_inflight = INT_MAX; int inflight; - list_for_each_entry_rcu(sess, &clt->paths_list, s.entry) { - if (READ_ONCE(sess->state) != RTRS_CLT_CONNECTED) + list_for_each_entry_rcu(clt_path, &clt->paths_list, s.entry) { + if (READ_ONCE(clt_path->state) != RTRS_CLT_CONNECTED) continue; - if (!list_empty(raw_cpu_ptr(sess->mp_skip_entry))) + if (!list_empty(raw_cpu_ptr(clt_path->mp_skip_entry))) continue; - inflight = atomic_read(&sess->stats->inflight); + inflight = atomic_read(&clt_path->stats->inflight); if (inflight < min_inflight) { min_inflight = inflight; - min_path = sess; + min_path = clt_path; } } @@ -862,26 +866,26 @@ static struct rtrs_clt_sess *get_next_path_min_inflight(struct path_it *it) * Therefore the caller MUST check the returned * path is NULL and trigger the IO error. */ -static struct rtrs_clt_sess *get_next_path_min_latency(struct path_it *it) +static struct rtrs_clt_path *get_next_path_min_latency(struct path_it *it) { - struct rtrs_clt_sess *min_path = NULL; - struct rtrs_clt *clt = it->clt; - struct rtrs_clt_sess *sess; - ktime_t min_latency = INT_MAX; + struct rtrs_clt_path *min_path = NULL; + struct rtrs_clt_sess *clt = it->clt; + struct rtrs_clt_path *clt_path; + ktime_t min_latency = KTIME_MAX; ktime_t latency; - list_for_each_entry_rcu(sess, &clt->paths_list, s.entry) { - if (READ_ONCE(sess->state) != RTRS_CLT_CONNECTED) + list_for_each_entry_rcu(clt_path, &clt->paths_list, s.entry) { + if (READ_ONCE(clt_path->state) != RTRS_CLT_CONNECTED) continue; - if (!list_empty(raw_cpu_ptr(sess->mp_skip_entry))) + if (!list_empty(raw_cpu_ptr(clt_path->mp_skip_entry))) continue; - latency = sess->s.hb_cur_latency; + latency = clt_path->s.hb_cur_latency; if (latency < min_latency) { min_latency = latency; - min_path = sess; + min_path = clt_path; } } @@ -895,7 +899,7 @@ static struct rtrs_clt_sess *get_next_path_min_latency(struct path_it *it) return min_path; } -static inline void path_it_init(struct path_it *it, struct rtrs_clt *clt) +static inline void path_it_init(struct path_it *it, struct rtrs_clt_sess *clt) { INIT_LIST_HEAD(&it->skip_list); it->clt = clt; @@ -928,7 +932,7 @@ static inline void path_it_deinit(struct path_it *it) * the corresponding buffer of rtrs_iu (req->iu->buf), which later on will * also hold the control message of rtrs. * @req: an io request holding information about IO. - * @sess: client session + * @clt_path: client path * @conf: conformation callback function to notify upper layer. * @permit: permit for allocation of RDMA remote buffer * @priv: private pointer @@ -940,7 +944,7 @@ static inline void path_it_deinit(struct path_it *it) * @dir: direction of the IO. */ static void rtrs_clt_init_req(struct rtrs_clt_io_req *req, - struct rtrs_clt_sess *sess, + struct rtrs_clt_path *clt_path, void (*conf)(void *priv, int errno), struct rtrs_permit *permit, void *priv, const struct kvec *vec, size_t usr_len, @@ -958,13 +962,13 @@ static void rtrs_clt_init_req(struct rtrs_clt_io_req *req, req->sg_cnt = sg_cnt; req->priv = priv; req->dir = dir; - req->con = rtrs_permit_to_clt_con(sess, permit); + req->con = rtrs_permit_to_clt_con(clt_path, permit); req->conf = conf; req->need_inv = false; req->need_inv_comp = false; req->inv_errno = 0; refcount_set(&req->ref, 1); - req->mp_policy = sess->clt->mp_policy; + req->mp_policy = clt_path->clt->mp_policy; iov_iter_kvec(&iter, READ, vec, 1, usr_len); len = _copy_from_iter(req->iu->buf, usr_len, &iter); @@ -974,7 +978,7 @@ static void rtrs_clt_init_req(struct rtrs_clt_io_req *req, } static struct rtrs_clt_io_req * -rtrs_clt_get_req(struct rtrs_clt_sess *sess, +rtrs_clt_get_req(struct rtrs_clt_path *clt_path, void (*conf)(void *priv, int errno), struct rtrs_permit *permit, void *priv, const struct kvec *vec, size_t usr_len, @@ -983,14 +987,14 @@ rtrs_clt_get_req(struct rtrs_clt_sess *sess, { struct rtrs_clt_io_req *req; - req = &sess->reqs[permit->mem_id]; - rtrs_clt_init_req(req, sess, conf, permit, priv, vec, usr_len, + req = &clt_path->reqs[permit->mem_id]; + rtrs_clt_init_req(req, clt_path, conf, permit, priv, vec, usr_len, sg, sg_cnt, data_len, dir); return req; } static struct rtrs_clt_io_req * -rtrs_clt_get_copy_req(struct rtrs_clt_sess *alive_sess, +rtrs_clt_get_copy_req(struct rtrs_clt_path *alive_path, struct rtrs_clt_io_req *fail_req) { struct rtrs_clt_io_req *req; @@ -999,8 +1003,8 @@ rtrs_clt_get_copy_req(struct rtrs_clt_sess *alive_sess, .iov_len = fail_req->usr_len }; - req = &alive_sess->reqs[fail_req->permit->mem_id]; - rtrs_clt_init_req(req, alive_sess, fail_req->conf, fail_req->permit, + req = &alive_path->reqs[fail_req->permit->mem_id]; + rtrs_clt_init_req(req, alive_path, fail_req->conf, fail_req->permit, fail_req->priv, &vec, fail_req->usr_len, fail_req->sglist, fail_req->sg_cnt, fail_req->data_len, fail_req->dir); @@ -1013,7 +1017,7 @@ static int rtrs_post_rdma_write_sg(struct rtrs_clt_con *con, u32 size, u32 imm, struct ib_send_wr *wr, struct ib_send_wr *tail) { - struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); + struct rtrs_clt_path *clt_path = to_clt_path(con->c.path); struct ib_sge *sge = req->sge; enum ib_send_flags flags; struct scatterlist *sg; @@ -1033,22 +1037,23 @@ static int rtrs_post_rdma_write_sg(struct rtrs_clt_con *con, for_each_sg(req->sglist, sg, req->sg_cnt, i) { sge[i].addr = sg_dma_address(sg); sge[i].length = sg_dma_len(sg); - sge[i].lkey = sess->s.dev->ib_pd->local_dma_lkey; + sge[i].lkey = clt_path->s.dev->ib_pd->local_dma_lkey; } num_sge = 1 + req->sg_cnt; } sge[i].addr = req->iu->dma_addr; sge[i].length = size; - sge[i].lkey = sess->s.dev->ib_pd->local_dma_lkey; + sge[i].lkey = clt_path->s.dev->ib_pd->local_dma_lkey; /* * From time to time we have to post signalled sends, * or send queue will fill up and only QP reset can help. */ - flags = atomic_inc_return(&con->c.wr_cnt) % sess->s.signal_interval ? + flags = atomic_inc_return(&con->c.wr_cnt) % clt_path->s.signal_interval ? 0 : IB_SEND_SIGNALED; - ib_dma_sync_single_for_device(sess->s.dev->ib_dev, req->iu->dma_addr, + ib_dma_sync_single_for_device(clt_path->s.dev->ib_dev, + req->iu->dma_addr, size, DMA_TO_DEVICE); return rtrs_iu_post_rdma_write_imm(&con->c, req->iu, sge, num_sge, @@ -1074,8 +1079,8 @@ static int rtrs_map_sg_fr(struct rtrs_clt_io_req *req, size_t count) static int rtrs_clt_write_req(struct rtrs_clt_io_req *req) { struct rtrs_clt_con *con = req->con; - struct rtrs_sess *s = con->c.sess; - struct rtrs_clt_sess *sess = to_clt_sess(s); + struct rtrs_path *s = con->c.path; + struct rtrs_clt_path *clt_path = to_clt_path(s); struct rtrs_msg_rdma_write *msg; struct rtrs_rbuf *rbuf; @@ -1088,13 +1093,13 @@ static int rtrs_clt_write_req(struct rtrs_clt_io_req *req) const size_t tsize = sizeof(*msg) + req->data_len + req->usr_len; - if (tsize > sess->chunk_size) { + if (tsize > clt_path->chunk_size) { rtrs_wrn(s, "Write request failed, size too big %zu > %d\n", - tsize, sess->chunk_size); + tsize, clt_path->chunk_size); return -EMSGSIZE; } if (req->sg_cnt) { - count = ib_dma_map_sg(sess->s.dev->ib_dev, req->sglist, + count = ib_dma_map_sg(clt_path->s.dev->ib_dev, req->sglist, req->sg_cnt, req->dir); if (!count) { rtrs_wrn(s, "Write request failed, map failed\n"); @@ -1111,7 +1116,7 @@ static int rtrs_clt_write_req(struct rtrs_clt_io_req *req) imm = rtrs_to_io_req_imm(imm); buf_id = req->permit->mem_id; req->sg_size = tsize; - rbuf = &sess->rbufs[buf_id]; + rbuf = &clt_path->rbufs[buf_id]; if (count) { ret = rtrs_map_sg_fr(req, count); @@ -1119,7 +1124,7 @@ static int rtrs_clt_write_req(struct rtrs_clt_io_req *req) rtrs_err_rl(s, "Write request failed, failed to map fast reg. data, err: %d\n", ret); - ib_dma_unmap_sg(sess->s.dev->ib_dev, req->sglist, + ib_dma_unmap_sg(clt_path->s.dev->ib_dev, req->sglist, req->sg_cnt, req->dir); return ret; } @@ -1153,12 +1158,12 @@ static int rtrs_clt_write_req(struct rtrs_clt_io_req *req) if (ret) { rtrs_err_rl(s, "Write request failed: error=%d path=%s [%s:%u]\n", - ret, kobject_name(&sess->kobj), sess->hca_name, - sess->hca_port); + ret, kobject_name(&clt_path->kobj), clt_path->hca_name, + clt_path->hca_port); if (req->mp_policy == MP_POLICY_MIN_INFLIGHT) - atomic_dec(&sess->stats->inflight); + atomic_dec(&clt_path->stats->inflight); if (req->sg_cnt) - ib_dma_unmap_sg(sess->s.dev->ib_dev, req->sglist, + ib_dma_unmap_sg(clt_path->s.dev->ib_dev, req->sglist, req->sg_cnt, req->dir); } @@ -1168,10 +1173,10 @@ static int rtrs_clt_write_req(struct rtrs_clt_io_req *req) static int rtrs_clt_read_req(struct rtrs_clt_io_req *req) { struct rtrs_clt_con *con = req->con; - struct rtrs_sess *s = con->c.sess; - struct rtrs_clt_sess *sess = to_clt_sess(s); + struct rtrs_path *s = con->c.path; + struct rtrs_clt_path *clt_path = to_clt_path(s); struct rtrs_msg_rdma_read *msg; - struct rtrs_ib_dev *dev = sess->s.dev; + struct rtrs_ib_dev *dev = clt_path->s.dev; struct ib_reg_wr rwr; struct ib_send_wr *wr = NULL; @@ -1181,10 +1186,10 @@ static int rtrs_clt_read_req(struct rtrs_clt_io_req *req) const size_t tsize = sizeof(*msg) + req->data_len + req->usr_len; - if (tsize > sess->chunk_size) { + if (tsize > clt_path->chunk_size) { rtrs_wrn(s, "Read request failed, message size is %zu, bigger than CHUNK_SIZE %d\n", - tsize, sess->chunk_size); + tsize, clt_path->chunk_size); return -EMSGSIZE; } @@ -1254,15 +1259,15 @@ static int rtrs_clt_read_req(struct rtrs_clt_io_req *req) */ rtrs_clt_update_all_stats(req, READ); - ret = rtrs_post_send_rdma(req->con, req, &sess->rbufs[buf_id], + ret = rtrs_post_send_rdma(req->con, req, &clt_path->rbufs[buf_id], req->data_len, imm, wr); if (ret) { rtrs_err_rl(s, "Read request failed: error=%d path=%s [%s:%u]\n", - ret, kobject_name(&sess->kobj), sess->hca_name, - sess->hca_port); + ret, kobject_name(&clt_path->kobj), clt_path->hca_name, + clt_path->hca_port); if (req->mp_policy == MP_POLICY_MIN_INFLIGHT) - atomic_dec(&sess->stats->inflight); + atomic_dec(&clt_path->stats->inflight); req->need_inv = false; if (req->sg_cnt) ib_dma_unmap_sg(dev->ib_dev, req->sglist, @@ -1277,21 +1282,21 @@ static int rtrs_clt_read_req(struct rtrs_clt_io_req *req) * @clt: clt context * @fail_req: a failed io request. */ -static int rtrs_clt_failover_req(struct rtrs_clt *clt, +static int rtrs_clt_failover_req(struct rtrs_clt_sess *clt, struct rtrs_clt_io_req *fail_req) { - struct rtrs_clt_sess *alive_sess; + struct rtrs_clt_path *alive_path; struct rtrs_clt_io_req *req; int err = -ECONNABORTED; struct path_it it; rcu_read_lock(); for (path_it_init(&it, clt); - (alive_sess = it.next_path(&it)) && it.i < it.clt->paths_num; + (alive_path = it.next_path(&it)) && it.i < it.clt->paths_num; it.i++) { - if (READ_ONCE(alive_sess->state) != RTRS_CLT_CONNECTED) + if (READ_ONCE(alive_path->state) != RTRS_CLT_CONNECTED) continue; - req = rtrs_clt_get_copy_req(alive_sess, fail_req); + req = rtrs_clt_get_copy_req(alive_path, fail_req); if (req->dir == DMA_TO_DEVICE) err = rtrs_clt_write_req(req); else @@ -1301,7 +1306,7 @@ static int rtrs_clt_failover_req(struct rtrs_clt *clt, continue; } /* Success path */ - rtrs_clt_inc_failover_cnt(alive_sess->stats); + rtrs_clt_inc_failover_cnt(alive_path->stats); break; } path_it_deinit(&it); @@ -1310,16 +1315,16 @@ static int rtrs_clt_failover_req(struct rtrs_clt *clt, return err; } -static void fail_all_outstanding_reqs(struct rtrs_clt_sess *sess) +static void fail_all_outstanding_reqs(struct rtrs_clt_path *clt_path) { - struct rtrs_clt *clt = sess->clt; + struct rtrs_clt_sess *clt = clt_path->clt; struct rtrs_clt_io_req *req; int i, err; - if (!sess->reqs) + if (!clt_path->reqs) return; - for (i = 0; i < sess->queue_depth; ++i) { - req = &sess->reqs[i]; + for (i = 0; i < clt_path->queue_depth; ++i) { + req = &clt_path->reqs[i]; if (!req->in_use) continue; @@ -1337,38 +1342,39 @@ static void fail_all_outstanding_reqs(struct rtrs_clt_sess *sess) } } -static void free_sess_reqs(struct rtrs_clt_sess *sess) +static void free_path_reqs(struct rtrs_clt_path *clt_path) { struct rtrs_clt_io_req *req; int i; - if (!sess->reqs) + if (!clt_path->reqs) return; - for (i = 0; i < sess->queue_depth; ++i) { - req = &sess->reqs[i]; + for (i = 0; i < clt_path->queue_depth; ++i) { + req = &clt_path->reqs[i]; if (req->mr) ib_dereg_mr(req->mr); kfree(req->sge); - rtrs_iu_free(req->iu, sess->s.dev->ib_dev, 1); + rtrs_iu_free(req->iu, clt_path->s.dev->ib_dev, 1); } - kfree(sess->reqs); - sess->reqs = NULL; + kfree(clt_path->reqs); + clt_path->reqs = NULL; } -static int alloc_sess_reqs(struct rtrs_clt_sess *sess) +static int alloc_path_reqs(struct rtrs_clt_path *clt_path) { struct rtrs_clt_io_req *req; int i, err = -ENOMEM; - sess->reqs = kcalloc(sess->queue_depth, sizeof(*sess->reqs), - GFP_KERNEL); - if (!sess->reqs) + clt_path->reqs = kcalloc(clt_path->queue_depth, + sizeof(*clt_path->reqs), + GFP_KERNEL); + if (!clt_path->reqs) return -ENOMEM; - for (i = 0; i < sess->queue_depth; ++i) { - req = &sess->reqs[i]; - req->iu = rtrs_iu_alloc(1, sess->max_hdr_size, GFP_KERNEL, - sess->s.dev->ib_dev, + for (i = 0; i < clt_path->queue_depth; ++i) { + req = &clt_path->reqs[i]; + req->iu = rtrs_iu_alloc(1, clt_path->max_hdr_size, GFP_KERNEL, + clt_path->s.dev->ib_dev, DMA_TO_DEVICE, rtrs_clt_rdma_done); if (!req->iu) @@ -1378,13 +1384,14 @@ static int alloc_sess_reqs(struct rtrs_clt_sess *sess) if (!req->sge) goto out; - req->mr = ib_alloc_mr(sess->s.dev->ib_pd, IB_MR_TYPE_MEM_REG, - sess->max_pages_per_mr); + req->mr = ib_alloc_mr(clt_path->s.dev->ib_pd, + IB_MR_TYPE_MEM_REG, + clt_path->max_pages_per_mr); if (IS_ERR(req->mr)) { err = PTR_ERR(req->mr); req->mr = NULL; - pr_err("Failed to alloc sess->max_pages_per_mr %d\n", - sess->max_pages_per_mr); + pr_err("Failed to alloc clt_path->max_pages_per_mr %d\n", + clt_path->max_pages_per_mr); goto out; } @@ -1394,12 +1401,12 @@ static int alloc_sess_reqs(struct rtrs_clt_sess *sess) return 0; out: - free_sess_reqs(sess); + free_path_reqs(clt_path); return err; } -static int alloc_permits(struct rtrs_clt *clt) +static int alloc_permits(struct rtrs_clt_sess *clt) { unsigned int chunk_bits; int err, i; @@ -1433,7 +1440,7 @@ out_err: return err; } -static void free_permits(struct rtrs_clt *clt) +static void free_permits(struct rtrs_clt_sess *clt) { if (clt->permits_map) { size_t sz = clt->queue_depth; @@ -1447,13 +1454,13 @@ static void free_permits(struct rtrs_clt *clt) clt->permits = NULL; } -static void query_fast_reg_mode(struct rtrs_clt_sess *sess) +static void query_fast_reg_mode(struct rtrs_clt_path *clt_path) { struct ib_device *ib_dev; u64 max_pages_per_mr; int mr_page_shift; - ib_dev = sess->s.dev->ib_dev; + ib_dev = clt_path->s.dev->ib_dev; /* * Use the smallest page size supported by the HCA, down to a @@ -1463,24 +1470,24 @@ static void query_fast_reg_mode(struct rtrs_clt_sess *sess) mr_page_shift = max(12, ffs(ib_dev->attrs.page_size_cap) - 1); max_pages_per_mr = ib_dev->attrs.max_mr_size; do_div(max_pages_per_mr, (1ull << mr_page_shift)); - sess->max_pages_per_mr = - min3(sess->max_pages_per_mr, (u32)max_pages_per_mr, + clt_path->max_pages_per_mr = + min3(clt_path->max_pages_per_mr, (u32)max_pages_per_mr, ib_dev->attrs.max_fast_reg_page_list_len); - sess->clt->max_segments = - min(sess->max_pages_per_mr, sess->clt->max_segments); + clt_path->clt->max_segments = + min(clt_path->max_pages_per_mr, clt_path->clt->max_segments); } -static bool rtrs_clt_change_state_get_old(struct rtrs_clt_sess *sess, +static bool rtrs_clt_change_state_get_old(struct rtrs_clt_path *clt_path, enum rtrs_clt_state new_state, enum rtrs_clt_state *old_state) { bool changed; - spin_lock_irq(&sess->state_wq.lock); + spin_lock_irq(&clt_path->state_wq.lock); if (old_state) - *old_state = sess->state; - changed = rtrs_clt_change_state(sess, new_state); - spin_unlock_irq(&sess->state_wq.lock); + *old_state = clt_path->state; + changed = rtrs_clt_change_state(clt_path, new_state); + spin_unlock_irq(&clt_path->state_wq.lock); return changed; } @@ -1492,9 +1499,9 @@ static void rtrs_clt_hb_err_handler(struct rtrs_con *c) rtrs_rdma_error_recovery(con); } -static void rtrs_clt_init_hb(struct rtrs_clt_sess *sess) +static void rtrs_clt_init_hb(struct rtrs_clt_path *clt_path) { - rtrs_init_hb(&sess->s, &io_comp_cqe, + rtrs_init_hb(&clt_path->s, &io_comp_cqe, RTRS_HB_INTERVAL_MS, RTRS_HB_MISSED_MAX, rtrs_clt_hb_err_handler, @@ -1504,17 +1511,17 @@ static void rtrs_clt_init_hb(struct rtrs_clt_sess *sess) static void rtrs_clt_reconnect_work(struct work_struct *work); static void rtrs_clt_close_work(struct work_struct *work); -static struct rtrs_clt_sess *alloc_sess(struct rtrs_clt *clt, +static struct rtrs_clt_path *alloc_path(struct rtrs_clt_sess *clt, const struct rtrs_addr *path, size_t con_num, u32 nr_poll_queues) { - struct rtrs_clt_sess *sess; + struct rtrs_clt_path *clt_path; int err = -ENOMEM; int cpu; size_t total_con; - sess = kzalloc(sizeof(*sess), GFP_KERNEL); - if (!sess) + clt_path = kzalloc(sizeof(*clt_path), GFP_KERNEL); + if (!clt_path) goto err; /* @@ -1522,20 +1529,21 @@ static struct rtrs_clt_sess *alloc_sess(struct rtrs_clt *clt, * +1: Extra connection for user messages */ total_con = con_num + nr_poll_queues + 1; - sess->s.con = kcalloc(total_con, sizeof(*sess->s.con), GFP_KERNEL); - if (!sess->s.con) - goto err_free_sess; + clt_path->s.con = kcalloc(total_con, sizeof(*clt_path->s.con), + GFP_KERNEL); + if (!clt_path->s.con) + goto err_free_path; - sess->s.con_num = total_con; - sess->s.irq_con_num = con_num + 1; + clt_path->s.con_num = total_con; + clt_path->s.irq_con_num = con_num + 1; - sess->stats = kzalloc(sizeof(*sess->stats), GFP_KERNEL); - if (!sess->stats) + clt_path->stats = kzalloc(sizeof(*clt_path->stats), GFP_KERNEL); + if (!clt_path->stats) goto err_free_con; - mutex_init(&sess->init_mutex); - uuid_gen(&sess->s.uuid); - memcpy(&sess->s.dst_addr, path->dst, + mutex_init(&clt_path->init_mutex); + uuid_gen(&clt_path->s.uuid); + memcpy(&clt_path->s.dst_addr, path->dst, rdma_addr_size((struct sockaddr *)path->dst)); /* @@ -1544,53 +1552,54 @@ static struct rtrs_clt_sess *alloc_sess(struct rtrs_clt *clt, * the sess->src_addr will contain only zeros, which is then fine. */ if (path->src) - memcpy(&sess->s.src_addr, path->src, + memcpy(&clt_path->s.src_addr, path->src, rdma_addr_size((struct sockaddr *)path->src)); - strscpy(sess->s.sessname, clt->sessname, sizeof(sess->s.sessname)); - sess->clt = clt; - sess->max_pages_per_mr = RTRS_MAX_SEGMENTS; - init_waitqueue_head(&sess->state_wq); - sess->state = RTRS_CLT_CONNECTING; - atomic_set(&sess->connected_cnt, 0); - INIT_WORK(&sess->close_work, rtrs_clt_close_work); - INIT_DELAYED_WORK(&sess->reconnect_dwork, rtrs_clt_reconnect_work); - rtrs_clt_init_hb(sess); + strscpy(clt_path->s.sessname, clt->sessname, + sizeof(clt_path->s.sessname)); + clt_path->clt = clt; + clt_path->max_pages_per_mr = RTRS_MAX_SEGMENTS; + init_waitqueue_head(&clt_path->state_wq); + clt_path->state = RTRS_CLT_CONNECTING; + atomic_set(&clt_path->connected_cnt, 0); + INIT_WORK(&clt_path->close_work, rtrs_clt_close_work); + INIT_DELAYED_WORK(&clt_path->reconnect_dwork, rtrs_clt_reconnect_work); + rtrs_clt_init_hb(clt_path); - sess->mp_skip_entry = alloc_percpu(typeof(*sess->mp_skip_entry)); - if (!sess->mp_skip_entry) + clt_path->mp_skip_entry = alloc_percpu(typeof(*clt_path->mp_skip_entry)); + if (!clt_path->mp_skip_entry) goto err_free_stats; for_each_possible_cpu(cpu) - INIT_LIST_HEAD(per_cpu_ptr(sess->mp_skip_entry, cpu)); + INIT_LIST_HEAD(per_cpu_ptr(clt_path->mp_skip_entry, cpu)); - err = rtrs_clt_init_stats(sess->stats); + err = rtrs_clt_init_stats(clt_path->stats); if (err) goto err_free_percpu; - return sess; + return clt_path; err_free_percpu: - free_percpu(sess->mp_skip_entry); + free_percpu(clt_path->mp_skip_entry); err_free_stats: - kfree(sess->stats); + kfree(clt_path->stats); err_free_con: - kfree(sess->s.con); -err_free_sess: - kfree(sess); + kfree(clt_path->s.con); +err_free_path: + kfree(clt_path); err: return ERR_PTR(err); } -void free_sess(struct rtrs_clt_sess *sess) +void free_path(struct rtrs_clt_path *clt_path) { - free_percpu(sess->mp_skip_entry); - mutex_destroy(&sess->init_mutex); - kfree(sess->s.con); - kfree(sess->rbufs); - kfree(sess); + free_percpu(clt_path->mp_skip_entry); + mutex_destroy(&clt_path->init_mutex); + kfree(clt_path->s.con); + kfree(clt_path->rbufs); + kfree(clt_path); } -static int create_con(struct rtrs_clt_sess *sess, unsigned int cid) +static int create_con(struct rtrs_clt_path *clt_path, unsigned int cid) { struct rtrs_clt_con *con; @@ -1601,28 +1610,28 @@ static int create_con(struct rtrs_clt_sess *sess, unsigned int cid) /* Map first two connections to the first CPU */ con->cpu = (cid ? cid - 1 : 0) % nr_cpu_ids; con->c.cid = cid; - con->c.sess = &sess->s; + con->c.path = &clt_path->s; /* Align with srv, init as 1 */ atomic_set(&con->c.wr_cnt, 1); mutex_init(&con->con_mutex); - sess->s.con[cid] = &con->c; + clt_path->s.con[cid] = &con->c; return 0; } static void destroy_con(struct rtrs_clt_con *con) { - struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); + struct rtrs_clt_path *clt_path = to_clt_path(con->c.path); - sess->s.con[con->c.cid] = NULL; + clt_path->s.con[con->c.cid] = NULL; mutex_destroy(&con->con_mutex); kfree(con); } static int create_con_cq_qp(struct rtrs_clt_con *con) { - struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); + struct rtrs_clt_path *clt_path = to_clt_path(con->c.path); u32 max_send_wr, max_recv_wr, cq_num, max_send_sge, wr_limit; int err, cq_vector; struct rtrs_msg_rkey_rsp *rsp; @@ -1631,7 +1640,7 @@ static int create_con_cq_qp(struct rtrs_clt_con *con) if (con->c.cid == 0) { max_send_sge = 1; /* We must be the first here */ - if (WARN_ON(sess->s.dev)) + if (WARN_ON(clt_path->s.dev)) return -EINVAL; /* @@ -1639,16 +1648,16 @@ static int create_con_cq_qp(struct rtrs_clt_con *con) * Be careful not to close user connection before ib dev * is gracefully put. */ - sess->s.dev = rtrs_ib_dev_find_or_add(con->c.cm_id->device, + clt_path->s.dev = rtrs_ib_dev_find_or_add(con->c.cm_id->device, &dev_pd); - if (!sess->s.dev) { - rtrs_wrn(sess->clt, + if (!clt_path->s.dev) { + rtrs_wrn(clt_path->clt, "rtrs_ib_dev_find_get_or_add(): no memory\n"); return -ENOMEM; } - sess->s.dev_ref = 1; - query_fast_reg_mode(sess); - wr_limit = sess->s.dev->ib_dev->attrs.max_qp_wr; + clt_path->s.dev_ref = 1; + query_fast_reg_mode(clt_path); + wr_limit = clt_path->s.dev->ib_dev->attrs.max_qp_wr; /* * Two (request + registration) completion for send * Two for recv if always_invalidate is set on server @@ -1665,27 +1674,28 @@ static int create_con_cq_qp(struct rtrs_clt_con *con) * This is always true if user connection (cid == 0) is * established first. */ - if (WARN_ON(!sess->s.dev)) + if (WARN_ON(!clt_path->s.dev)) return -EINVAL; - if (WARN_ON(!sess->queue_depth)) + if (WARN_ON(!clt_path->queue_depth)) return -EINVAL; - wr_limit = sess->s.dev->ib_dev->attrs.max_qp_wr; + wr_limit = clt_path->s.dev->ib_dev->attrs.max_qp_wr; /* Shared between connections */ - sess->s.dev_ref++; + clt_path->s.dev_ref++; max_send_wr = min_t(int, wr_limit, /* QD * (REQ + RSP + FR REGS or INVS) + drain */ - sess->queue_depth * 3 + 1); + clt_path->queue_depth * 3 + 1); max_recv_wr = min_t(int, wr_limit, - sess->queue_depth * 3 + 1); + clt_path->queue_depth * 3 + 1); max_send_sge = 2; } atomic_set(&con->c.sq_wr_avail, max_send_wr); cq_num = max_send_wr + max_recv_wr; /* alloc iu to recv new rkey reply when server reports flags set */ - if (sess->flags & RTRS_MSG_NEW_RKEY_F || con->c.cid == 0) { + if (clt_path->flags & RTRS_MSG_NEW_RKEY_F || con->c.cid == 0) { con->rsp_ius = rtrs_iu_alloc(cq_num, sizeof(*rsp), - GFP_KERNEL, sess->s.dev->ib_dev, + GFP_KERNEL, + clt_path->s.dev->ib_dev, DMA_FROM_DEVICE, rtrs_clt_rdma_done); if (!con->rsp_ius) @@ -1693,13 +1703,13 @@ static int create_con_cq_qp(struct rtrs_clt_con *con) con->queue_num = cq_num; } cq_num = max_send_wr + max_recv_wr; - cq_vector = con->cpu % sess->s.dev->ib_dev->num_comp_vectors; - if (con->c.cid >= sess->s.irq_con_num) - err = rtrs_cq_qp_create(&sess->s, &con->c, max_send_sge, + cq_vector = con->cpu % clt_path->s.dev->ib_dev->num_comp_vectors; + if (con->c.cid >= clt_path->s.irq_con_num) + err = rtrs_cq_qp_create(&clt_path->s, &con->c, max_send_sge, cq_vector, cq_num, max_send_wr, max_recv_wr, IB_POLL_DIRECT); else - err = rtrs_cq_qp_create(&sess->s, &con->c, max_send_sge, + err = rtrs_cq_qp_create(&clt_path->s, &con->c, max_send_sge, cq_vector, cq_num, max_send_wr, max_recv_wr, IB_POLL_SOFTIRQ); /* @@ -1711,7 +1721,7 @@ static int create_con_cq_qp(struct rtrs_clt_con *con) static void destroy_con_cq_qp(struct rtrs_clt_con *con) { - struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); + struct rtrs_clt_path *clt_path = to_clt_path(con->c.path); /* * Be careful here: destroy_con_cq_qp() can be called even @@ -1720,13 +1730,14 @@ static void destroy_con_cq_qp(struct rtrs_clt_con *con) lockdep_assert_held(&con->con_mutex); rtrs_cq_qp_destroy(&con->c); if (con->rsp_ius) { - rtrs_iu_free(con->rsp_ius, sess->s.dev->ib_dev, con->queue_num); + rtrs_iu_free(con->rsp_ius, clt_path->s.dev->ib_dev, + con->queue_num); con->rsp_ius = NULL; con->queue_num = 0; } - if (sess->s.dev_ref && !--sess->s.dev_ref) { - rtrs_ib_dev_put(sess->s.dev); - sess->s.dev = NULL; + if (clt_path->s.dev_ref && !--clt_path->s.dev_ref) { + rtrs_ib_dev_put(clt_path->s.dev); + clt_path->s.dev = NULL; } } @@ -1745,7 +1756,7 @@ static void destroy_cm(struct rtrs_clt_con *con) static int rtrs_rdma_addr_resolved(struct rtrs_clt_con *con) { - struct rtrs_sess *s = con->c.sess; + struct rtrs_path *s = con->c.path; int err; mutex_lock(&con->con_mutex); @@ -1764,8 +1775,8 @@ static int rtrs_rdma_addr_resolved(struct rtrs_clt_con *con) static int rtrs_rdma_route_resolved(struct rtrs_clt_con *con) { - struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); - struct rtrs_clt *clt = sess->clt; + struct rtrs_clt_path *clt_path = to_clt_path(con->c.path); + struct rtrs_clt_sess *clt = clt_path->clt; struct rtrs_msg_conn_req msg; struct rdma_conn_param param; @@ -1782,11 +1793,11 @@ static int rtrs_rdma_route_resolved(struct rtrs_clt_con *con) .magic = cpu_to_le16(RTRS_MAGIC), .version = cpu_to_le16(RTRS_PROTO_VER), .cid = cpu_to_le16(con->c.cid), - .cid_num = cpu_to_le16(sess->s.con_num), - .recon_cnt = cpu_to_le16(sess->s.recon_cnt), + .cid_num = cpu_to_le16(clt_path->s.con_num), + .recon_cnt = cpu_to_le16(clt_path->s.recon_cnt), }; - msg.first_conn = sess->for_new_clt ? FIRST_CONN : 0; - uuid_copy(&msg.sess_uuid, &sess->s.uuid); + msg.first_conn = clt_path->for_new_clt ? FIRST_CONN : 0; + uuid_copy(&msg.sess_uuid, &clt_path->s.uuid); uuid_copy(&msg.paths_uuid, &clt->paths_uuid); err = rdma_connect_locked(con->c.cm_id, ¶m); @@ -1799,8 +1810,8 @@ static int rtrs_rdma_route_resolved(struct rtrs_clt_con *con) static int rtrs_rdma_conn_established(struct rtrs_clt_con *con, struct rdma_cm_event *ev) { - struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); - struct rtrs_clt *clt = sess->clt; + struct rtrs_clt_path *clt_path = to_clt_path(con->c.path); + struct rtrs_clt_sess *clt = clt_path->clt; const struct rtrs_msg_conn_rsp *msg; u16 version, queue_depth; int errno; @@ -1831,31 +1842,32 @@ static int rtrs_rdma_conn_established(struct rtrs_clt_con *con, if (con->c.cid == 0) { queue_depth = le16_to_cpu(msg->queue_depth); - if (sess->queue_depth > 0 && queue_depth != sess->queue_depth) { + if (clt_path->queue_depth > 0 && queue_depth != clt_path->queue_depth) { rtrs_err(clt, "Error: queue depth changed\n"); /* * Stop any more reconnection attempts */ - sess->reconnect_attempts = -1; + clt_path->reconnect_attempts = -1; rtrs_err(clt, "Disabling auto-reconnect. Trigger a manual reconnect after issue is resolved\n"); return -ECONNRESET; } - if (!sess->rbufs) { - sess->rbufs = kcalloc(queue_depth, sizeof(*sess->rbufs), - GFP_KERNEL); - if (!sess->rbufs) + if (!clt_path->rbufs) { + clt_path->rbufs = kcalloc(queue_depth, + sizeof(*clt_path->rbufs), + GFP_KERNEL); + if (!clt_path->rbufs) return -ENOMEM; } - sess->queue_depth = queue_depth; - sess->s.signal_interval = min_not_zero(queue_depth, + clt_path->queue_depth = queue_depth; + clt_path->s.signal_interval = min_not_zero(queue_depth, (unsigned short) SERVICE_CON_QUEUE_DEPTH); - sess->max_hdr_size = le32_to_cpu(msg->max_hdr_size); - sess->max_io_size = le32_to_cpu(msg->max_io_size); - sess->flags = le32_to_cpu(msg->flags); - sess->chunk_size = sess->max_io_size + sess->max_hdr_size; + clt_path->max_hdr_size = le32_to_cpu(msg->max_hdr_size); + clt_path->max_io_size = le32_to_cpu(msg->max_io_size); + clt_path->flags = le32_to_cpu(msg->flags); + clt_path->chunk_size = clt_path->max_io_size + clt_path->max_hdr_size; /* * Global IO size is always a minimum. @@ -1866,20 +1878,20 @@ static int rtrs_rdma_conn_established(struct rtrs_clt_con *con, * connections in parallel, use lock. */ mutex_lock(&clt->paths_mutex); - clt->queue_depth = sess->queue_depth; - clt->max_io_size = min_not_zero(sess->max_io_size, + clt->queue_depth = clt_path->queue_depth; + clt->max_io_size = min_not_zero(clt_path->max_io_size, clt->max_io_size); mutex_unlock(&clt->paths_mutex); /* * Cache the hca_port and hca_name for sysfs */ - sess->hca_port = con->c.cm_id->port_num; - scnprintf(sess->hca_name, sizeof(sess->hca_name), - sess->s.dev->ib_dev->name); - sess->s.src_addr = con->c.cm_id->route.addr.src_addr; + clt_path->hca_port = con->c.cm_id->port_num; + scnprintf(clt_path->hca_name, sizeof(clt_path->hca_name), + clt_path->s.dev->ib_dev->name); + clt_path->s.src_addr = con->c.cm_id->route.addr.src_addr; /* set for_new_clt, to allow future reconnect on any path */ - sess->for_new_clt = 1; + clt_path->for_new_clt = 1; } return 0; @@ -1887,16 +1899,16 @@ static int rtrs_rdma_conn_established(struct rtrs_clt_con *con, static inline void flag_success_on_conn(struct rtrs_clt_con *con) { - struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); + struct rtrs_clt_path *clt_path = to_clt_path(con->c.path); - atomic_inc(&sess->connected_cnt); + atomic_inc(&clt_path->connected_cnt); con->cm_err = 1; } static int rtrs_rdma_conn_rejected(struct rtrs_clt_con *con, struct rdma_cm_event *ev) { - struct rtrs_sess *s = con->c.sess; + struct rtrs_path *s = con->c.path; const struct rtrs_msg_conn_rsp *msg; const char *rej_msg; int status, errno; @@ -1924,23 +1936,23 @@ static int rtrs_rdma_conn_rejected(struct rtrs_clt_con *con, return -ECONNRESET; } -void rtrs_clt_close_conns(struct rtrs_clt_sess *sess, bool wait) +void rtrs_clt_close_conns(struct rtrs_clt_path *clt_path, bool wait) { - if (rtrs_clt_change_state_get_old(sess, RTRS_CLT_CLOSING, NULL)) - queue_work(rtrs_wq, &sess->close_work); + if (rtrs_clt_change_state_get_old(clt_path, RTRS_CLT_CLOSING, NULL)) + queue_work(rtrs_wq, &clt_path->close_work); if (wait) - flush_work(&sess->close_work); + flush_work(&clt_path->close_work); } static inline void flag_error_on_conn(struct rtrs_clt_con *con, int cm_err) { if (con->cm_err == 1) { - struct rtrs_clt_sess *sess; + struct rtrs_clt_path *clt_path; - sess = to_clt_sess(con->c.sess); - if (atomic_dec_and_test(&sess->connected_cnt)) + clt_path = to_clt_path(con->c.path); + if (atomic_dec_and_test(&clt_path->connected_cnt)) - wake_up(&sess->state_wq); + wake_up(&clt_path->state_wq); } con->cm_err = cm_err; } @@ -1949,8 +1961,8 @@ static int rtrs_clt_rdma_cm_handler(struct rdma_cm_id *cm_id, struct rdma_cm_event *ev) { struct rtrs_clt_con *con = cm_id->context; - struct rtrs_sess *s = con->c.sess; - struct rtrs_clt_sess *sess = to_clt_sess(s); + struct rtrs_path *s = con->c.path; + struct rtrs_clt_path *clt_path = to_clt_path(s); int cm_err = 0; switch (ev->event) { @@ -1968,7 +1980,7 @@ static int rtrs_clt_rdma_cm_handler(struct rdma_cm_id *cm_id, * i.e. wake up without state change, but we set cm_err. */ flag_success_on_conn(con); - wake_up(&sess->state_wq); + wake_up(&clt_path->state_wq); return 0; } break; @@ -1997,7 +2009,7 @@ static int rtrs_clt_rdma_cm_handler(struct rdma_cm_id *cm_id, /* * Device removal is a special case. Queue close and return 0. */ - rtrs_clt_close_conns(sess, false); + rtrs_clt_close_conns(clt_path, false); return 0; default: rtrs_err(s, "Unexpected RDMA CM error (CM event: %s, err: %d)\n", @@ -2020,13 +2032,13 @@ static int rtrs_clt_rdma_cm_handler(struct rdma_cm_id *cm_id, static int create_cm(struct rtrs_clt_con *con) { - struct rtrs_sess *s = con->c.sess; - struct rtrs_clt_sess *sess = to_clt_sess(s); + struct rtrs_path *s = con->c.path; + struct rtrs_clt_path *clt_path = to_clt_path(s); struct rdma_cm_id *cm_id; int err; cm_id = rdma_create_id(&init_net, rtrs_clt_rdma_cm_handler, con, - sess->s.dst_addr.ss_family == AF_IB ? + clt_path->s.dst_addr.ss_family == AF_IB ? RDMA_PS_IB : RDMA_PS_TCP, IB_QPT_RC); if (IS_ERR(cm_id)) { err = PTR_ERR(cm_id); @@ -2042,8 +2054,8 @@ static int create_cm(struct rtrs_clt_con *con) rtrs_err(s, "Set address reuse failed, err: %d\n", err); goto destroy_cm; } - err = rdma_resolve_addr(cm_id, (struct sockaddr *)&sess->s.src_addr, - (struct sockaddr *)&sess->s.dst_addr, + err = rdma_resolve_addr(cm_id, (struct sockaddr *)&clt_path->s.src_addr, + (struct sockaddr *)&clt_path->s.dst_addr, RTRS_CONNECT_TIMEOUT_MS); if (err) { rtrs_err(s, "Failed to resolve address, err: %d\n", err); @@ -2055,8 +2067,8 @@ static int create_cm(struct rtrs_clt_con *con) * or session state was really changed to error by device removal. */ err = wait_event_interruptible_timeout( - sess->state_wq, - con->cm_err || sess->state != RTRS_CLT_CONNECTING, + clt_path->state_wq, + con->cm_err || clt_path->state != RTRS_CLT_CONNECTING, msecs_to_jiffies(RTRS_CONNECT_TIMEOUT_MS)); if (err == 0 || err == -ERESTARTSYS) { if (err == 0) @@ -2068,7 +2080,7 @@ static int create_cm(struct rtrs_clt_con *con) err = con->cm_err; goto errr; } - if (READ_ONCE(sess->state) != RTRS_CLT_CONNECTING) { + if (READ_ONCE(clt_path->state) != RTRS_CLT_CONNECTING) { /* Device removal */ err = -ECONNABORTED; goto errr; @@ -2087,9 +2099,9 @@ destroy_cm: return err; } -static void rtrs_clt_sess_up(struct rtrs_clt_sess *sess) +static void rtrs_clt_path_up(struct rtrs_clt_path *clt_path) { - struct rtrs_clt *clt = sess->clt; + struct rtrs_clt_sess *clt = clt_path->clt; int up; /* @@ -2113,19 +2125,19 @@ static void rtrs_clt_sess_up(struct rtrs_clt_sess *sess) mutex_unlock(&clt->paths_ev_mutex); /* Mark session as established */ - sess->established = true; - sess->reconnect_attempts = 0; - sess->stats->reconnects.successful_cnt++; + clt_path->established = true; + clt_path->reconnect_attempts = 0; + clt_path->stats->reconnects.successful_cnt++; } -static void rtrs_clt_sess_down(struct rtrs_clt_sess *sess) +static void rtrs_clt_path_down(struct rtrs_clt_path *clt_path) { - struct rtrs_clt *clt = sess->clt; + struct rtrs_clt_sess *clt = clt_path->clt; - if (!sess->established) + if (!clt_path->established) return; - sess->established = false; + clt_path->established = false; mutex_lock(&clt->paths_ev_mutex); WARN_ON(!clt->paths_up); if (--clt->paths_up == 0) @@ -2133,19 +2145,19 @@ static void rtrs_clt_sess_down(struct rtrs_clt_sess *sess) mutex_unlock(&clt->paths_ev_mutex); } -static void rtrs_clt_stop_and_destroy_conns(struct rtrs_clt_sess *sess) +static void rtrs_clt_stop_and_destroy_conns(struct rtrs_clt_path *clt_path) { struct rtrs_clt_con *con; unsigned int cid; - WARN_ON(READ_ONCE(sess->state) == RTRS_CLT_CONNECTED); + WARN_ON(READ_ONCE(clt_path->state) == RTRS_CLT_CONNECTED); /* * Possible race with rtrs_clt_open(), when DEVICE_REMOVAL comes * exactly in between. Start destroying after it finishes. */ - mutex_lock(&sess->init_mutex); - mutex_unlock(&sess->init_mutex); + mutex_lock(&clt_path->init_mutex); + mutex_unlock(&clt_path->init_mutex); /* * All IO paths must observe !CONNECTED state before we @@ -2153,7 +2165,7 @@ static void rtrs_clt_stop_and_destroy_conns(struct rtrs_clt_sess *sess) */ synchronize_rcu(); - rtrs_stop_hb(&sess->s); + rtrs_stop_hb(&clt_path->s); /* * The order it utterly crucial: firstly disconnect and complete all @@ -2162,15 +2174,15 @@ static void rtrs_clt_stop_and_destroy_conns(struct rtrs_clt_sess *sess) * eventually notify upper layer about session disconnection. */ - for (cid = 0; cid < sess->s.con_num; cid++) { - if (!sess->s.con[cid]) + for (cid = 0; cid < clt_path->s.con_num; cid++) { + if (!clt_path->s.con[cid]) break; - con = to_clt_con(sess->s.con[cid]); + con = to_clt_con(clt_path->s.con[cid]); stop_cm(con); } - fail_all_outstanding_reqs(sess); - free_sess_reqs(sess); - rtrs_clt_sess_down(sess); + fail_all_outstanding_reqs(clt_path); + free_path_reqs(clt_path); + rtrs_clt_path_down(clt_path); /* * Wait for graceful shutdown, namely when peer side invokes @@ -2180,13 +2192,14 @@ static void rtrs_clt_stop_and_destroy_conns(struct rtrs_clt_sess *sess) * since CM does not fire anything. That is fine, we are not in * hurry. */ - wait_event_timeout(sess->state_wq, !atomic_read(&sess->connected_cnt), + wait_event_timeout(clt_path->state_wq, + !atomic_read(&clt_path->connected_cnt), msecs_to_jiffies(RTRS_CONNECT_TIMEOUT_MS)); - for (cid = 0; cid < sess->s.con_num; cid++) { - if (!sess->s.con[cid]) + for (cid = 0; cid < clt_path->s.con_num; cid++) { + if (!clt_path->s.con[cid]) break; - con = to_clt_con(sess->s.con[cid]); + con = to_clt_con(clt_path->s.con[cid]); mutex_lock(&con->con_mutex); destroy_con_cq_qp(con); mutex_unlock(&con->con_mutex); @@ -2195,26 +2208,26 @@ static void rtrs_clt_stop_and_destroy_conns(struct rtrs_clt_sess *sess) } } -static inline bool xchg_sessions(struct rtrs_clt_sess __rcu **rcu_ppcpu_path, - struct rtrs_clt_sess *sess, - struct rtrs_clt_sess *next) +static inline bool xchg_paths(struct rtrs_clt_path __rcu **rcu_ppcpu_path, + struct rtrs_clt_path *clt_path, + struct rtrs_clt_path *next) { - struct rtrs_clt_sess **ppcpu_path; + struct rtrs_clt_path **ppcpu_path; /* Call cmpxchg() without sparse warnings */ ppcpu_path = (typeof(ppcpu_path))rcu_ppcpu_path; - return sess == cmpxchg(ppcpu_path, sess, next); + return clt_path == cmpxchg(ppcpu_path, clt_path, next); } -static void rtrs_clt_remove_path_from_arr(struct rtrs_clt_sess *sess) +static void rtrs_clt_remove_path_from_arr(struct rtrs_clt_path *clt_path) { - struct rtrs_clt *clt = sess->clt; - struct rtrs_clt_sess *next; + struct rtrs_clt_sess *clt = clt_path->clt; + struct rtrs_clt_path *next; bool wait_for_grace = false; int cpu; mutex_lock(&clt->paths_mutex); - list_del_rcu(&sess->s.entry); + list_del_rcu(&clt_path->s.entry); /* Make sure everybody observes path removal. */ synchronize_rcu(); @@ -2255,7 +2268,7 @@ static void rtrs_clt_remove_path_from_arr(struct rtrs_clt_sess *sess) * removed. If @sess is the last element, then @next is NULL. */ rcu_read_lock(); - next = list_next_or_null_rr_rcu(&clt->paths_list, &sess->s.entry, + next = list_next_or_null_rr_rcu(&clt->paths_list, &clt_path->s.entry, typeof(*next), s.entry); rcu_read_unlock(); @@ -2264,11 +2277,11 @@ static void rtrs_clt_remove_path_from_arr(struct rtrs_clt_sess *sess) * removed, so change the pointer manually. */ for_each_possible_cpu(cpu) { - struct rtrs_clt_sess __rcu **ppcpu_path; + struct rtrs_clt_path __rcu **ppcpu_path; ppcpu_path = per_cpu_ptr(clt->pcpu_path, cpu); if (rcu_dereference_protected(*ppcpu_path, - lockdep_is_held(&clt->paths_mutex)) != sess) + lockdep_is_held(&clt->paths_mutex)) != clt_path) /* * synchronize_rcu() was called just after deleting * entry from the list, thus IO code path cannot @@ -2281,7 +2294,7 @@ static void rtrs_clt_remove_path_from_arr(struct rtrs_clt_sess *sess) * We race with IO code path, which also changes pointer, * thus we have to be careful not to overwrite it. */ - if (xchg_sessions(ppcpu_path, sess, next)) + if (xchg_paths(ppcpu_path, clt_path, next)) /* * @ppcpu_path was successfully replaced with @next, * that means that someone could also pick up the @@ -2296,29 +2309,29 @@ static void rtrs_clt_remove_path_from_arr(struct rtrs_clt_sess *sess) mutex_unlock(&clt->paths_mutex); } -static void rtrs_clt_add_path_to_arr(struct rtrs_clt_sess *sess) +static void rtrs_clt_add_path_to_arr(struct rtrs_clt_path *clt_path) { - struct rtrs_clt *clt = sess->clt; + struct rtrs_clt_sess *clt = clt_path->clt; mutex_lock(&clt->paths_mutex); clt->paths_num++; - list_add_tail_rcu(&sess->s.entry, &clt->paths_list); + list_add_tail_rcu(&clt_path->s.entry, &clt->paths_list); mutex_unlock(&clt->paths_mutex); } static void rtrs_clt_close_work(struct work_struct *work) { - struct rtrs_clt_sess *sess; + struct rtrs_clt_path *clt_path; - sess = container_of(work, struct rtrs_clt_sess, close_work); + clt_path = container_of(work, struct rtrs_clt_path, close_work); - cancel_delayed_work_sync(&sess->reconnect_dwork); - rtrs_clt_stop_and_destroy_conns(sess); - rtrs_clt_change_state_get_old(sess, RTRS_CLT_CLOSED, NULL); + cancel_delayed_work_sync(&clt_path->reconnect_dwork); + rtrs_clt_stop_and_destroy_conns(clt_path); + rtrs_clt_change_state_get_old(clt_path, RTRS_CLT_CLOSED, NULL); } -static int init_conns(struct rtrs_clt_sess *sess) +static int init_conns(struct rtrs_clt_path *clt_path) { unsigned int cid; int err; @@ -2328,31 +2341,31 @@ static int init_conns(struct rtrs_clt_sess *sess) * to avoid clashes with previous sessions not yet closed * sessions on a server side. */ - sess->s.recon_cnt++; + clt_path->s.recon_cnt++; /* Establish all RDMA connections */ - for (cid = 0; cid < sess->s.con_num; cid++) { - err = create_con(sess, cid); + for (cid = 0; cid < clt_path->s.con_num; cid++) { + err = create_con(clt_path, cid); if (err) goto destroy; - err = create_cm(to_clt_con(sess->s.con[cid])); + err = create_cm(to_clt_con(clt_path->s.con[cid])); if (err) { - destroy_con(to_clt_con(sess->s.con[cid])); + destroy_con(to_clt_con(clt_path->s.con[cid])); goto destroy; } } - err = alloc_sess_reqs(sess); + err = alloc_path_reqs(clt_path); if (err) goto destroy; - rtrs_start_hb(&sess->s); + rtrs_start_hb(&clt_path->s); return 0; destroy: while (cid--) { - struct rtrs_clt_con *con = to_clt_con(sess->s.con[cid]); + struct rtrs_clt_con *con = to_clt_con(clt_path->s.con[cid]); stop_cm(con); @@ -2367,7 +2380,7 @@ destroy: * doing rdma_resolve_addr(), switch to CONNECTION_ERR state * manually to keep reconnecting. */ - rtrs_clt_change_state_get_old(sess, RTRS_CLT_CONNECTING_ERR, NULL); + rtrs_clt_change_state_get_old(clt_path, RTRS_CLT_CONNECTING_ERR, NULL); return err; } @@ -2375,31 +2388,32 @@ destroy: static void rtrs_clt_info_req_done(struct ib_cq *cq, struct ib_wc *wc) { struct rtrs_clt_con *con = to_clt_con(wc->qp->qp_context); - struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); + struct rtrs_clt_path *clt_path = to_clt_path(con->c.path); struct rtrs_iu *iu; iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe); - rtrs_iu_free(iu, sess->s.dev->ib_dev, 1); + rtrs_iu_free(iu, clt_path->s.dev->ib_dev, 1); if (wc->status != IB_WC_SUCCESS) { - rtrs_err(sess->clt, "Sess info request send failed: %s\n", + rtrs_err(clt_path->clt, "Path info request send failed: %s\n", ib_wc_status_msg(wc->status)); - rtrs_clt_change_state_get_old(sess, RTRS_CLT_CONNECTING_ERR, NULL); + rtrs_clt_change_state_get_old(clt_path, RTRS_CLT_CONNECTING_ERR, NULL); return; } rtrs_clt_update_wc_stats(con); } -static int process_info_rsp(struct rtrs_clt_sess *sess, +static int process_info_rsp(struct rtrs_clt_path *clt_path, const struct rtrs_msg_info_rsp *msg) { unsigned int sg_cnt, total_len; int i, sgi; sg_cnt = le16_to_cpu(msg->sg_cnt); - if (!sg_cnt || (sess->queue_depth % sg_cnt)) { - rtrs_err(sess->clt, "Incorrect sg_cnt %d, is not multiple\n", + if (!sg_cnt || (clt_path->queue_depth % sg_cnt)) { + rtrs_err(clt_path->clt, + "Incorrect sg_cnt %d, is not multiple\n", sg_cnt); return -EINVAL; } @@ -2408,15 +2422,15 @@ static int process_info_rsp(struct rtrs_clt_sess *sess, * Check if IB immediate data size is enough to hold the mem_id and * the offset inside the memory chunk. */ - if ((ilog2(sg_cnt - 1) + 1) + (ilog2(sess->chunk_size - 1) + 1) > + if ((ilog2(sg_cnt - 1) + 1) + (ilog2(clt_path->chunk_size - 1) + 1) > MAX_IMM_PAYL_BITS) { - rtrs_err(sess->clt, + rtrs_err(clt_path->clt, "RDMA immediate size (%db) not enough to encode %d buffers of size %dB\n", - MAX_IMM_PAYL_BITS, sg_cnt, sess->chunk_size); + MAX_IMM_PAYL_BITS, sg_cnt, clt_path->chunk_size); return -EINVAL; } total_len = 0; - for (sgi = 0, i = 0; sgi < sg_cnt && i < sess->queue_depth; sgi++) { + for (sgi = 0, i = 0; sgi < sg_cnt && i < clt_path->queue_depth; sgi++) { const struct rtrs_sg_desc *desc = &msg->desc[sgi]; u32 len, rkey; u64 addr; @@ -2427,26 +2441,28 @@ static int process_info_rsp(struct rtrs_clt_sess *sess, total_len += len; - if (!len || (len % sess->chunk_size)) { - rtrs_err(sess->clt, "Incorrect [%d].len %d\n", sgi, + if (!len || (len % clt_path->chunk_size)) { + rtrs_err(clt_path->clt, "Incorrect [%d].len %d\n", + sgi, len); return -EINVAL; } - for ( ; len && i < sess->queue_depth; i++) { - sess->rbufs[i].addr = addr; - sess->rbufs[i].rkey = rkey; + for ( ; len && i < clt_path->queue_depth; i++) { + clt_path->rbufs[i].addr = addr; + clt_path->rbufs[i].rkey = rkey; - len -= sess->chunk_size; - addr += sess->chunk_size; + len -= clt_path->chunk_size; + addr += clt_path->chunk_size; } } /* Sanity check */ - if (sgi != sg_cnt || i != sess->queue_depth) { - rtrs_err(sess->clt, "Incorrect sg vector, not fully mapped\n"); + if (sgi != sg_cnt || i != clt_path->queue_depth) { + rtrs_err(clt_path->clt, + "Incorrect sg vector, not fully mapped\n"); return -EINVAL; } - if (total_len != sess->chunk_size * sess->queue_depth) { - rtrs_err(sess->clt, "Incorrect total_len %d\n", total_len); + if (total_len != clt_path->chunk_size * clt_path->queue_depth) { + rtrs_err(clt_path->clt, "Incorrect total_len %d\n", total_len); return -EINVAL; } @@ -2456,7 +2472,7 @@ static int process_info_rsp(struct rtrs_clt_sess *sess, static void rtrs_clt_info_rsp_done(struct ib_cq *cq, struct ib_wc *wc) { struct rtrs_clt_con *con = to_clt_con(wc->qp->qp_context); - struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); + struct rtrs_clt_path *clt_path = to_clt_path(con->c.path); struct rtrs_msg_info_rsp *msg; enum rtrs_clt_state state; struct rtrs_iu *iu; @@ -2468,37 +2484,37 @@ static void rtrs_clt_info_rsp_done(struct ib_cq *cq, struct ib_wc *wc) WARN_ON(con->c.cid); iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe); if (wc->status != IB_WC_SUCCESS) { - rtrs_err(sess->clt, "Sess info response recv failed: %s\n", + rtrs_err(clt_path->clt, "Path info response recv failed: %s\n", ib_wc_status_msg(wc->status)); goto out; } WARN_ON(wc->opcode != IB_WC_RECV); if (wc->byte_len < sizeof(*msg)) { - rtrs_err(sess->clt, "Sess info response is malformed: size %d\n", + rtrs_err(clt_path->clt, "Path info response is malformed: size %d\n", wc->byte_len); goto out; } - ib_dma_sync_single_for_cpu(sess->s.dev->ib_dev, iu->dma_addr, + ib_dma_sync_single_for_cpu(clt_path->s.dev->ib_dev, iu->dma_addr, iu->size, DMA_FROM_DEVICE); msg = iu->buf; if (le16_to_cpu(msg->type) != RTRS_MSG_INFO_RSP) { - rtrs_err(sess->clt, "Sess info response is malformed: type %d\n", + rtrs_err(clt_path->clt, "Path info response is malformed: type %d\n", le16_to_cpu(msg->type)); goto out; } rx_sz = sizeof(*msg); rx_sz += sizeof(msg->desc[0]) * le16_to_cpu(msg->sg_cnt); if (wc->byte_len < rx_sz) { - rtrs_err(sess->clt, "Sess info response is malformed: size %d\n", + rtrs_err(clt_path->clt, "Path info response is malformed: size %d\n", wc->byte_len); goto out; } - err = process_info_rsp(sess, msg); + err = process_info_rsp(clt_path, msg); if (err) goto out; - err = post_recv_sess(sess); + err = post_recv_path(clt_path); if (err) goto out; @@ -2506,25 +2522,25 @@ static void rtrs_clt_info_rsp_done(struct ib_cq *cq, struct ib_wc *wc) out: rtrs_clt_update_wc_stats(con); - rtrs_iu_free(iu, sess->s.dev->ib_dev, 1); - rtrs_clt_change_state_get_old(sess, state, NULL); + rtrs_iu_free(iu, clt_path->s.dev->ib_dev, 1); + rtrs_clt_change_state_get_old(clt_path, state, NULL); } -static int rtrs_send_sess_info(struct rtrs_clt_sess *sess) +static int rtrs_send_path_info(struct rtrs_clt_path *clt_path) { - struct rtrs_clt_con *usr_con = to_clt_con(sess->s.con[0]); + struct rtrs_clt_con *usr_con = to_clt_con(clt_path->s.con[0]); struct rtrs_msg_info_req *msg; struct rtrs_iu *tx_iu, *rx_iu; size_t rx_sz; int err; rx_sz = sizeof(struct rtrs_msg_info_rsp); - rx_sz += sizeof(struct rtrs_sg_desc) * sess->queue_depth; + rx_sz += sizeof(struct rtrs_sg_desc) * clt_path->queue_depth; tx_iu = rtrs_iu_alloc(1, sizeof(struct rtrs_msg_info_req), GFP_KERNEL, - sess->s.dev->ib_dev, DMA_TO_DEVICE, + clt_path->s.dev->ib_dev, DMA_TO_DEVICE, rtrs_clt_info_req_done); - rx_iu = rtrs_iu_alloc(1, rx_sz, GFP_KERNEL, sess->s.dev->ib_dev, + rx_iu = rtrs_iu_alloc(1, rx_sz, GFP_KERNEL, clt_path->s.dev->ib_dev, DMA_FROM_DEVICE, rtrs_clt_info_rsp_done); if (!tx_iu || !rx_iu) { err = -ENOMEM; @@ -2533,33 +2549,34 @@ static int rtrs_send_sess_info(struct rtrs_clt_sess *sess) /* Prepare for getting info response */ err = rtrs_iu_post_recv(&usr_con->c, rx_iu); if (err) { - rtrs_err(sess->clt, "rtrs_iu_post_recv(), err: %d\n", err); + rtrs_err(clt_path->clt, "rtrs_iu_post_recv(), err: %d\n", err); goto out; } rx_iu = NULL; msg = tx_iu->buf; msg->type = cpu_to_le16(RTRS_MSG_INFO_REQ); - memcpy(msg->sessname, sess->s.sessname, sizeof(msg->sessname)); + memcpy(msg->pathname, clt_path->s.sessname, sizeof(msg->pathname)); - ib_dma_sync_single_for_device(sess->s.dev->ib_dev, tx_iu->dma_addr, + ib_dma_sync_single_for_device(clt_path->s.dev->ib_dev, + tx_iu->dma_addr, tx_iu->size, DMA_TO_DEVICE); /* Send info request */ err = rtrs_iu_post_send(&usr_con->c, tx_iu, sizeof(*msg), NULL); if (err) { - rtrs_err(sess->clt, "rtrs_iu_post_send(), err: %d\n", err); + rtrs_err(clt_path->clt, "rtrs_iu_post_send(), err: %d\n", err); goto out; } tx_iu = NULL; /* Wait for state change */ - wait_event_interruptible_timeout(sess->state_wq, - sess->state != RTRS_CLT_CONNECTING, + wait_event_interruptible_timeout(clt_path->state_wq, + clt_path->state != RTRS_CLT_CONNECTING, msecs_to_jiffies( RTRS_CONNECT_TIMEOUT_MS)); - if (READ_ONCE(sess->state) != RTRS_CLT_CONNECTED) { - if (READ_ONCE(sess->state) == RTRS_CLT_CONNECTING_ERR) + if (READ_ONCE(clt_path->state) != RTRS_CLT_CONNECTED) { + if (READ_ONCE(clt_path->state) == RTRS_CLT_CONNECTING_ERR) err = -ECONNRESET; else err = -ETIMEDOUT; @@ -2567,82 +2584,82 @@ static int rtrs_send_sess_info(struct rtrs_clt_sess *sess) out: if (tx_iu) - rtrs_iu_free(tx_iu, sess->s.dev->ib_dev, 1); + rtrs_iu_free(tx_iu, clt_path->s.dev->ib_dev, 1); if (rx_iu) - rtrs_iu_free(rx_iu, sess->s.dev->ib_dev, 1); + rtrs_iu_free(rx_iu, clt_path->s.dev->ib_dev, 1); if (err) /* If we've never taken async path because of malloc problems */ - rtrs_clt_change_state_get_old(sess, RTRS_CLT_CONNECTING_ERR, NULL); + rtrs_clt_change_state_get_old(clt_path, + RTRS_CLT_CONNECTING_ERR, NULL); return err; } /** - * init_sess() - establishes all session connections and does handshake - * @sess: client session. + * init_path() - establishes all path connections and does handshake + * @clt_path: client path. * In case of error full close or reconnect procedure should be taken, * because reconnect or close async works can be started. */ -static int init_sess(struct rtrs_clt_sess *sess) +static int init_path(struct rtrs_clt_path *clt_path) { int err; char str[NAME_MAX]; struct rtrs_addr path = { - .src = &sess->s.src_addr, - .dst = &sess->s.dst_addr, + .src = &clt_path->s.src_addr, + .dst = &clt_path->s.dst_addr, }; rtrs_addr_to_str(&path, str, sizeof(str)); - mutex_lock(&sess->init_mutex); - err = init_conns(sess); + mutex_lock(&clt_path->init_mutex); + err = init_conns(clt_path); if (err) { - rtrs_err(sess->clt, + rtrs_err(clt_path->clt, "init_conns() failed: err=%d path=%s [%s:%u]\n", err, - str, sess->hca_name, sess->hca_port); + str, clt_path->hca_name, clt_path->hca_port); goto out; } - err = rtrs_send_sess_info(sess); + err = rtrs_send_path_info(clt_path); if (err) { - rtrs_err( - sess->clt, - "rtrs_send_sess_info() failed: err=%d path=%s [%s:%u]\n", - err, str, sess->hca_name, sess->hca_port); + rtrs_err(clt_path->clt, + "rtrs_send_path_info() failed: err=%d path=%s [%s:%u]\n", + err, str, clt_path->hca_name, clt_path->hca_port); goto out; } - rtrs_clt_sess_up(sess); + rtrs_clt_path_up(clt_path); out: - mutex_unlock(&sess->init_mutex); + mutex_unlock(&clt_path->init_mutex); return err; } static void rtrs_clt_reconnect_work(struct work_struct *work) { - struct rtrs_clt_sess *sess; - struct rtrs_clt *clt; + struct rtrs_clt_path *clt_path; + struct rtrs_clt_sess *clt; unsigned int delay_ms; int err; - sess = container_of(to_delayed_work(work), struct rtrs_clt_sess, - reconnect_dwork); - clt = sess->clt; + clt_path = container_of(to_delayed_work(work), struct rtrs_clt_path, + reconnect_dwork); + clt = clt_path->clt; - if (READ_ONCE(sess->state) != RTRS_CLT_RECONNECTING) + if (READ_ONCE(clt_path->state) != RTRS_CLT_RECONNECTING) return; - if (sess->reconnect_attempts >= clt->max_reconnect_attempts) { - /* Close a session completely if max attempts is reached */ - rtrs_clt_close_conns(sess, false); + if (clt_path->reconnect_attempts >= clt->max_reconnect_attempts) { + /* Close a path completely if max attempts is reached */ + rtrs_clt_close_conns(clt_path, false); return; } - sess->reconnect_attempts++; + clt_path->reconnect_attempts++; /* Stop everything */ - rtrs_clt_stop_and_destroy_conns(sess); + rtrs_clt_stop_and_destroy_conns(clt_path); msleep(RTRS_RECONNECT_BACKOFF); - if (rtrs_clt_change_state_get_old(sess, RTRS_CLT_CONNECTING, NULL)) { - err = init_sess(sess); + if (rtrs_clt_change_state_get_old(clt_path, RTRS_CLT_CONNECTING, NULL)) { + err = init_path(clt_path); if (err) goto reconnect_again; } @@ -2650,10 +2667,10 @@ static void rtrs_clt_reconnect_work(struct work_struct *work) return; reconnect_again: - if (rtrs_clt_change_state_get_old(sess, RTRS_CLT_RECONNECTING, NULL)) { - sess->stats->reconnects.fail_cnt++; + if (rtrs_clt_change_state_get_old(clt_path, RTRS_CLT_RECONNECTING, NULL)) { + clt_path->stats->reconnects.fail_cnt++; delay_ms = clt->reconnect_delay_sec * 1000; - queue_delayed_work(rtrs_wq, &sess->reconnect_dwork, + queue_delayed_work(rtrs_wq, &clt_path->reconnect_dwork, msecs_to_jiffies(delay_ms + prandom_u32() % RTRS_RECONNECT_SEED)); @@ -2662,19 +2679,20 @@ reconnect_again: static void rtrs_clt_dev_release(struct device *dev) { - struct rtrs_clt *clt = container_of(dev, struct rtrs_clt, dev); + struct rtrs_clt_sess *clt = container_of(dev, struct rtrs_clt_sess, + dev); kfree(clt); } -static struct rtrs_clt *alloc_clt(const char *sessname, size_t paths_num, +static struct rtrs_clt_sess *alloc_clt(const char *sessname, size_t paths_num, u16 port, size_t pdu_sz, void *priv, void (*link_ev)(void *priv, enum rtrs_clt_link_ev ev), unsigned int reconnect_delay_sec, unsigned int max_reconnect_attempts) { - struct rtrs_clt *clt; + struct rtrs_clt_sess *clt; int err; if (!paths_num || paths_num > MAX_PATHS_NUM) @@ -2749,7 +2767,7 @@ err: return ERR_PTR(err); } -static void free_clt(struct rtrs_clt *clt) +static void free_clt(struct rtrs_clt_sess *clt) { free_permits(clt); free_percpu(clt->pcpu_path); @@ -2760,7 +2778,7 @@ static void free_clt(struct rtrs_clt *clt) } /** - * rtrs_clt_open() - Open a session to an RTRS server + * rtrs_clt_open() - Open a path to an RTRS server * @ops: holds the link event callback and the private pointer. * @sessname: name of the session * @paths: Paths to be established defined by their src and dst addresses @@ -2777,24 +2795,24 @@ static void free_clt(struct rtrs_clt *clt) * * Return a valid pointer on success otherwise PTR_ERR. */ -struct rtrs_clt *rtrs_clt_open(struct rtrs_clt_ops *ops, - const char *sessname, +struct rtrs_clt_sess *rtrs_clt_open(struct rtrs_clt_ops *ops, + const char *pathname, const struct rtrs_addr *paths, size_t paths_num, u16 port, size_t pdu_sz, u8 reconnect_delay_sec, s16 max_reconnect_attempts, u32 nr_poll_queues) { - struct rtrs_clt_sess *sess, *tmp; - struct rtrs_clt *clt; + struct rtrs_clt_path *clt_path, *tmp; + struct rtrs_clt_sess *clt; int err, i; - if (strchr(sessname, '/') || strchr(sessname, '.')) { - pr_err("sessname cannot contain / and .\n"); + if (strchr(pathname, '/') || strchr(pathname, '.')) { + pr_err("pathname cannot contain / and .\n"); err = -EINVAL; goto out; } - clt = alloc_clt(sessname, paths_num, port, pdu_sz, ops->priv, + clt = alloc_clt(pathname, paths_num, port, pdu_sz, ops->priv, ops->link_ev, reconnect_delay_sec, max_reconnect_attempts); @@ -2803,49 +2821,49 @@ struct rtrs_clt *rtrs_clt_open(struct rtrs_clt_ops *ops, goto out; } for (i = 0; i < paths_num; i++) { - struct rtrs_clt_sess *sess; + struct rtrs_clt_path *clt_path; - sess = alloc_sess(clt, &paths[i], nr_cpu_ids, + clt_path = alloc_path(clt, &paths[i], nr_cpu_ids, nr_poll_queues); - if (IS_ERR(sess)) { - err = PTR_ERR(sess); - goto close_all_sess; + if (IS_ERR(clt_path)) { + err = PTR_ERR(clt_path); + goto close_all_path; } if (!i) - sess->for_new_clt = 1; - list_add_tail_rcu(&sess->s.entry, &clt->paths_list); + clt_path->for_new_clt = 1; + list_add_tail_rcu(&clt_path->s.entry, &clt->paths_list); - err = init_sess(sess); + err = init_path(clt_path); if (err) { - list_del_rcu(&sess->s.entry); - rtrs_clt_close_conns(sess, true); - free_percpu(sess->stats->pcpu_stats); - kfree(sess->stats); - free_sess(sess); - goto close_all_sess; + list_del_rcu(&clt_path->s.entry); + rtrs_clt_close_conns(clt_path, true); + free_percpu(clt_path->stats->pcpu_stats); + kfree(clt_path->stats); + free_path(clt_path); + goto close_all_path; } - err = rtrs_clt_create_sess_files(sess); + err = rtrs_clt_create_path_files(clt_path); if (err) { - list_del_rcu(&sess->s.entry); - rtrs_clt_close_conns(sess, true); - free_percpu(sess->stats->pcpu_stats); - kfree(sess->stats); - free_sess(sess); - goto close_all_sess; + list_del_rcu(&clt_path->s.entry); + rtrs_clt_close_conns(clt_path, true); + free_percpu(clt_path->stats->pcpu_stats); + kfree(clt_path->stats); + free_path(clt_path); + goto close_all_path; } } err = alloc_permits(clt); if (err) - goto close_all_sess; + goto close_all_path; return clt; -close_all_sess: - list_for_each_entry_safe(sess, tmp, &clt->paths_list, s.entry) { - rtrs_clt_destroy_sess_files(sess, NULL); - rtrs_clt_close_conns(sess, true); - kobject_put(&sess->kobj); +close_all_path: + list_for_each_entry_safe(clt_path, tmp, &clt->paths_list, s.entry) { + rtrs_clt_destroy_path_files(clt_path, NULL); + rtrs_clt_close_conns(clt_path, true); + kobject_put(&clt_path->kobj); } rtrs_clt_destroy_sysfs_root(clt); free_clt(clt); @@ -2856,37 +2874,38 @@ out: EXPORT_SYMBOL(rtrs_clt_open); /** - * rtrs_clt_close() - Close a session + * rtrs_clt_close() - Close a path * @clt: Session handle. Session is freed upon return. */ -void rtrs_clt_close(struct rtrs_clt *clt) +void rtrs_clt_close(struct rtrs_clt_sess *clt) { - struct rtrs_clt_sess *sess, *tmp; + struct rtrs_clt_path *clt_path, *tmp; /* Firstly forbid sysfs access */ rtrs_clt_destroy_sysfs_root(clt); /* Now it is safe to iterate over all paths without locks */ - list_for_each_entry_safe(sess, tmp, &clt->paths_list, s.entry) { - rtrs_clt_close_conns(sess, true); - rtrs_clt_destroy_sess_files(sess, NULL); - kobject_put(&sess->kobj); + list_for_each_entry_safe(clt_path, tmp, &clt->paths_list, s.entry) { + rtrs_clt_close_conns(clt_path, true); + rtrs_clt_destroy_path_files(clt_path, NULL); + kobject_put(&clt_path->kobj); } free_clt(clt); } EXPORT_SYMBOL(rtrs_clt_close); -int rtrs_clt_reconnect_from_sysfs(struct rtrs_clt_sess *sess) +int rtrs_clt_reconnect_from_sysfs(struct rtrs_clt_path *clt_path) { enum rtrs_clt_state old_state; int err = -EBUSY; bool changed; - changed = rtrs_clt_change_state_get_old(sess, RTRS_CLT_RECONNECTING, + changed = rtrs_clt_change_state_get_old(clt_path, + RTRS_CLT_RECONNECTING, &old_state); if (changed) { - sess->reconnect_attempts = 0; - queue_delayed_work(rtrs_wq, &sess->reconnect_dwork, 0); + clt_path->reconnect_attempts = 0; + queue_delayed_work(rtrs_wq, &clt_path->reconnect_dwork, 0); } if (changed || old_state == RTRS_CLT_RECONNECTING) { /* @@ -2894,15 +2913,15 @@ int rtrs_clt_reconnect_from_sysfs(struct rtrs_clt_sess *sess) * execution, so do the flush if we have queued something * right now or work is pending. */ - flush_delayed_work(&sess->reconnect_dwork); - err = (READ_ONCE(sess->state) == + flush_delayed_work(&clt_path->reconnect_dwork); + err = (READ_ONCE(clt_path->state) == RTRS_CLT_CONNECTED ? 0 : -ENOTCONN); } return err; } -int rtrs_clt_remove_path_from_sysfs(struct rtrs_clt_sess *sess, +int rtrs_clt_remove_path_from_sysfs(struct rtrs_clt_path *clt_path, const struct attribute *sysfs_self) { enum rtrs_clt_state old_state; @@ -2918,27 +2937,27 @@ int rtrs_clt_remove_path_from_sysfs(struct rtrs_clt_sess *sess, * removing the path. */ do { - rtrs_clt_close_conns(sess, true); - changed = rtrs_clt_change_state_get_old(sess, + rtrs_clt_close_conns(clt_path, true); + changed = rtrs_clt_change_state_get_old(clt_path, RTRS_CLT_DEAD, &old_state); } while (!changed && old_state != RTRS_CLT_DEAD); if (changed) { - rtrs_clt_remove_path_from_arr(sess); - rtrs_clt_destroy_sess_files(sess, sysfs_self); - kobject_put(&sess->kobj); + rtrs_clt_remove_path_from_arr(clt_path); + rtrs_clt_destroy_path_files(clt_path, sysfs_self); + kobject_put(&clt_path->kobj); } return 0; } -void rtrs_clt_set_max_reconnect_attempts(struct rtrs_clt *clt, int value) +void rtrs_clt_set_max_reconnect_attempts(struct rtrs_clt_sess *clt, int value) { clt->max_reconnect_attempts = (unsigned int)value; } -int rtrs_clt_get_max_reconnect_attempts(const struct rtrs_clt *clt) +int rtrs_clt_get_max_reconnect_attempts(const struct rtrs_clt_sess *clt) { return (int)clt->max_reconnect_attempts; } @@ -2968,12 +2987,12 @@ int rtrs_clt_get_max_reconnect_attempts(const struct rtrs_clt *clt) * On dir=WRITE rtrs client will rdma write data in sg to server side. */ int rtrs_clt_request(int dir, struct rtrs_clt_req_ops *ops, - struct rtrs_clt *clt, struct rtrs_permit *permit, - const struct kvec *vec, size_t nr, size_t data_len, - struct scatterlist *sg, unsigned int sg_cnt) + struct rtrs_clt_sess *clt, struct rtrs_permit *permit, + const struct kvec *vec, size_t nr, size_t data_len, + struct scatterlist *sg, unsigned int sg_cnt) { struct rtrs_clt_io_req *req; - struct rtrs_clt_sess *sess; + struct rtrs_clt_path *clt_path; enum dma_data_direction dma_dir; int err = -ECONNABORTED, i; @@ -2995,19 +3014,19 @@ int rtrs_clt_request(int dir, struct rtrs_clt_req_ops *ops, rcu_read_lock(); for (path_it_init(&it, clt); - (sess = it.next_path(&it)) && it.i < it.clt->paths_num; it.i++) { - if (READ_ONCE(sess->state) != RTRS_CLT_CONNECTED) + (clt_path = it.next_path(&it)) && it.i < it.clt->paths_num; it.i++) { + if (READ_ONCE(clt_path->state) != RTRS_CLT_CONNECTED) continue; - if (usr_len + hdr_len > sess->max_hdr_size) { - rtrs_wrn_rl(sess->clt, + if (usr_len + hdr_len > clt_path->max_hdr_size) { + rtrs_wrn_rl(clt_path->clt, "%s request failed, user message size is %zu and header length %zu, but max size is %u\n", dir == READ ? "Read" : "Write", - usr_len, hdr_len, sess->max_hdr_size); + usr_len, hdr_len, clt_path->max_hdr_size); err = -EMSGSIZE; break; } - req = rtrs_clt_get_req(sess, ops->conf_fn, permit, ops->priv, + req = rtrs_clt_get_req(clt_path, ops->conf_fn, permit, ops->priv, vec, usr_len, sg, sg_cnt, data_len, dma_dir); if (dir == READ) @@ -3028,21 +3047,21 @@ int rtrs_clt_request(int dir, struct rtrs_clt_req_ops *ops, } EXPORT_SYMBOL(rtrs_clt_request); -int rtrs_clt_rdma_cq_direct(struct rtrs_clt *clt, unsigned int index) +int rtrs_clt_rdma_cq_direct(struct rtrs_clt_sess *clt, unsigned int index) { /* If no path, return -1 for block layer not to try again */ int cnt = -1; struct rtrs_con *con; - struct rtrs_clt_sess *sess; + struct rtrs_clt_path *clt_path; struct path_it it; rcu_read_lock(); for (path_it_init(&it, clt); - (sess = it.next_path(&it)) && it.i < it.clt->paths_num; it.i++) { - if (READ_ONCE(sess->state) != RTRS_CLT_CONNECTED) + (clt_path = it.next_path(&it)) && it.i < it.clt->paths_num; it.i++) { + if (READ_ONCE(clt_path->state) != RTRS_CLT_CONNECTED) continue; - con = sess->s.con[index + 1]; + con = clt_path->s.con[index + 1]; cnt = ib_process_cq_direct(con->cq, -1); if (cnt) break; @@ -3062,7 +3081,7 @@ EXPORT_SYMBOL(rtrs_clt_rdma_cq_direct); * 0 on success * -ECOMM no connection to the server */ -int rtrs_clt_query(struct rtrs_clt *clt, struct rtrs_attrs *attr) +int rtrs_clt_query(struct rtrs_clt_sess *clt, struct rtrs_attrs *attr) { if (!rtrs_clt_is_connected(clt)) return -ECOMM; @@ -3077,15 +3096,15 @@ int rtrs_clt_query(struct rtrs_clt *clt, struct rtrs_attrs *attr) } EXPORT_SYMBOL(rtrs_clt_query); -int rtrs_clt_create_path_from_sysfs(struct rtrs_clt *clt, +int rtrs_clt_create_path_from_sysfs(struct rtrs_clt_sess *clt, struct rtrs_addr *addr) { - struct rtrs_clt_sess *sess; + struct rtrs_clt_path *clt_path; int err; - sess = alloc_sess(clt, addr, nr_cpu_ids, 0); - if (IS_ERR(sess)) - return PTR_ERR(sess); + clt_path = alloc_path(clt, addr, nr_cpu_ids, 0); + if (IS_ERR(clt_path)) + return PTR_ERR(clt_path); mutex_lock(&clt->paths_mutex); if (clt->paths_num == 0) { @@ -3094,7 +3113,7 @@ int rtrs_clt_create_path_from_sysfs(struct rtrs_clt *clt, * the addition of the first path is like a new session for * the storage server */ - sess->for_new_clt = 1; + clt_path->for_new_clt = 1; } mutex_unlock(&clt->paths_mutex); @@ -3104,24 +3123,24 @@ int rtrs_clt_create_path_from_sysfs(struct rtrs_clt *clt, * IO will never grab it. Also it is very important to add * path before init, since init fires LINK_CONNECTED event. */ - rtrs_clt_add_path_to_arr(sess); + rtrs_clt_add_path_to_arr(clt_path); - err = init_sess(sess); + err = init_path(clt_path); if (err) - goto close_sess; + goto close_path; - err = rtrs_clt_create_sess_files(sess); + err = rtrs_clt_create_path_files(clt_path); if (err) - goto close_sess; + goto close_path; return 0; -close_sess: - rtrs_clt_remove_path_from_arr(sess); - rtrs_clt_close_conns(sess, true); - free_percpu(sess->stats->pcpu_stats); - kfree(sess->stats); - free_sess(sess); +close_path: + rtrs_clt_remove_path_from_arr(clt_path); + rtrs_clt_close_conns(clt_path, true); + free_percpu(clt_path->stats->pcpu_stats); + kfree(clt_path->stats); + free_path(clt_path); return err; } diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.h b/drivers/infiniband/ulp/rtrs/rtrs-clt.h index 9afffccff973..d1b18a154ae0 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-clt.h +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.h @@ -124,9 +124,9 @@ struct rtrs_rbuf { u32 rkey; }; -struct rtrs_clt_sess { - struct rtrs_sess s; - struct rtrs_clt *clt; +struct rtrs_clt_path { + struct rtrs_path s; + struct rtrs_clt_sess *clt; wait_queue_head_t state_wq; enum rtrs_clt_state state; atomic_t connected_cnt; @@ -153,10 +153,10 @@ struct rtrs_clt_sess { *mp_skip_entry; }; -struct rtrs_clt { +struct rtrs_clt_sess { struct list_head paths_list; /* rcu protected list */ size_t paths_num; - struct rtrs_clt_sess + struct rtrs_clt_path __rcu * __percpu *pcpu_path; uuid_t paths_uuid; int paths_up; @@ -186,31 +186,32 @@ static inline struct rtrs_clt_con *to_clt_con(struct rtrs_con *c) return container_of(c, struct rtrs_clt_con, c); } -static inline struct rtrs_clt_sess *to_clt_sess(struct rtrs_sess *s) +static inline struct rtrs_clt_path *to_clt_path(struct rtrs_path *s) { - return container_of(s, struct rtrs_clt_sess, s); + return container_of(s, struct rtrs_clt_path, s); } -static inline int permit_size(struct rtrs_clt *clt) +static inline int permit_size(struct rtrs_clt_sess *clt) { return sizeof(struct rtrs_permit) + clt->pdu_sz; } -static inline struct rtrs_permit *get_permit(struct rtrs_clt *clt, int idx) +static inline struct rtrs_permit *get_permit(struct rtrs_clt_sess *clt, + int idx) { return (struct rtrs_permit *)(clt->permits + permit_size(clt) * idx); } -int rtrs_clt_reconnect_from_sysfs(struct rtrs_clt_sess *sess); -void rtrs_clt_close_conns(struct rtrs_clt_sess *sess, bool wait); -int rtrs_clt_create_path_from_sysfs(struct rtrs_clt *clt, +int rtrs_clt_reconnect_from_sysfs(struct rtrs_clt_path *path); +void rtrs_clt_close_conns(struct rtrs_clt_path *clt_path, bool wait); +int rtrs_clt_create_path_from_sysfs(struct rtrs_clt_sess *clt, struct rtrs_addr *addr); -int rtrs_clt_remove_path_from_sysfs(struct rtrs_clt_sess *sess, +int rtrs_clt_remove_path_from_sysfs(struct rtrs_clt_path *path, const struct attribute *sysfs_self); -void rtrs_clt_set_max_reconnect_attempts(struct rtrs_clt *clt, int value); -int rtrs_clt_get_max_reconnect_attempts(const struct rtrs_clt *clt); -void free_sess(struct rtrs_clt_sess *sess); +void rtrs_clt_set_max_reconnect_attempts(struct rtrs_clt_sess *clt, int value); +int rtrs_clt_get_max_reconnect_attempts(const struct rtrs_clt_sess *clt); +void free_path(struct rtrs_clt_path *clt_path); /* rtrs-clt-stats.c */ @@ -239,11 +240,11 @@ ssize_t rtrs_clt_reset_all_help(struct rtrs_clt_stats *stats, /* rtrs-clt-sysfs.c */ -int rtrs_clt_create_sysfs_root_files(struct rtrs_clt *clt); -void rtrs_clt_destroy_sysfs_root(struct rtrs_clt *clt); +int rtrs_clt_create_sysfs_root_files(struct rtrs_clt_sess *clt); +void rtrs_clt_destroy_sysfs_root(struct rtrs_clt_sess *clt); -int rtrs_clt_create_sess_files(struct rtrs_clt_sess *sess); -void rtrs_clt_destroy_sess_files(struct rtrs_clt_sess *sess, +int rtrs_clt_create_path_files(struct rtrs_clt_path *clt_path); +void rtrs_clt_destroy_path_files(struct rtrs_clt_path *clt_path, const struct attribute *sysfs_self); #endif /* RTRS_CLT_H */ diff --git a/drivers/infiniband/ulp/rtrs/rtrs-pri.h b/drivers/infiniband/ulp/rtrs/rtrs-pri.h index 78eac9a4f703..9a1e5c2ae55c 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-pri.h +++ b/drivers/infiniband/ulp/rtrs/rtrs-pri.h @@ -90,7 +90,7 @@ struct rtrs_ib_dev { }; struct rtrs_con { - struct rtrs_sess *sess; + struct rtrs_path *path; struct ib_qp *qp; struct ib_cq *cq; struct rdma_cm_id *cm_id; @@ -100,7 +100,7 @@ struct rtrs_con { atomic_t sq_wr_avail; }; -struct rtrs_sess { +struct rtrs_path { struct list_head entry; struct sockaddr_storage dst_addr; struct sockaddr_storage src_addr; @@ -229,11 +229,11 @@ struct rtrs_msg_conn_rsp { /** * struct rtrs_msg_info_req * @type: @RTRS_MSG_INFO_REQ - * @sessname: Session name chosen by client + * @pathname: Path name chosen by client */ struct rtrs_msg_info_req { __le16 type; - u8 sessname[NAME_MAX]; + u8 pathname[NAME_MAX]; u8 reserved[15]; }; @@ -313,19 +313,19 @@ int rtrs_iu_post_rdma_write_imm(struct rtrs_con *con, struct rtrs_iu *iu, int rtrs_post_recv_empty(struct rtrs_con *con, struct ib_cqe *cqe); -int rtrs_cq_qp_create(struct rtrs_sess *sess, struct rtrs_con *con, +int rtrs_cq_qp_create(struct rtrs_path *path, struct rtrs_con *con, u32 max_send_sge, int cq_vector, int nr_cqe, u32 max_send_wr, u32 max_recv_wr, enum ib_poll_context poll_ctx); void rtrs_cq_qp_destroy(struct rtrs_con *con); -void rtrs_init_hb(struct rtrs_sess *sess, struct ib_cqe *cqe, +void rtrs_init_hb(struct rtrs_path *path, struct ib_cqe *cqe, unsigned int interval_ms, unsigned int missed_max, void (*err_handler)(struct rtrs_con *con), struct workqueue_struct *wq); -void rtrs_start_hb(struct rtrs_sess *sess); -void rtrs_stop_hb(struct rtrs_sess *sess); -void rtrs_send_hb_ack(struct rtrs_sess *sess); +void rtrs_start_hb(struct rtrs_path *path); +void rtrs_stop_hb(struct rtrs_path *path); +void rtrs_send_hb_ack(struct rtrs_path *path); void rtrs_rdma_dev_pd_init(enum ib_pd_flags pd_flags, struct rtrs_rdma_dev_pd *pool); diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c b/drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c index 9c43ce5ba1c1..b94ae12c2795 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c @@ -15,10 +15,10 @@ static void rtrs_srv_release(struct kobject *kobj) { - struct rtrs_srv_sess *sess; + struct rtrs_srv_path *srv_path; - sess = container_of(kobj, struct rtrs_srv_sess, kobj); - kfree(sess); + srv_path = container_of(kobj, struct rtrs_srv_path, kobj); + kfree(srv_path); } static struct kobj_type ktype = { @@ -36,24 +36,25 @@ static ssize_t rtrs_srv_disconnect_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { - struct rtrs_srv_sess *sess; - struct rtrs_sess *s; + struct rtrs_srv_path *srv_path; + struct rtrs_path *s; char str[MAXHOSTNAMELEN]; - sess = container_of(kobj, struct rtrs_srv_sess, kobj); - s = &sess->s; + srv_path = container_of(kobj, struct rtrs_srv_path, kobj); + s = &srv_path->s; if (!sysfs_streq(buf, "1")) { rtrs_err(s, "%s: invalid value: '%s'\n", attr->attr.name, buf); return -EINVAL; } - sockaddr_to_str((struct sockaddr *)&sess->s.dst_addr, str, sizeof(str)); + sockaddr_to_str((struct sockaddr *)&srv_path->s.dst_addr, str, + sizeof(str)); rtrs_info(s, "disconnect for path %s requested\n", str); /* first remove sysfs itself to avoid deadlock */ - sysfs_remove_file_self(&sess->kobj, &attr->attr); - close_sess(sess); + sysfs_remove_file_self(&srv_path->kobj, &attr->attr); + close_path(srv_path); return count; } @@ -66,11 +67,11 @@ static ssize_t rtrs_srv_hca_port_show(struct kobject *kobj, struct kobj_attribute *attr, char *page) { - struct rtrs_srv_sess *sess; + struct rtrs_srv_path *srv_path; struct rtrs_con *usr_con; - sess = container_of(kobj, typeof(*sess), kobj); - usr_con = sess->s.con[0]; + srv_path = container_of(kobj, typeof(*srv_path), kobj); + usr_con = srv_path->s.con[0]; return sysfs_emit(page, "%u\n", usr_con->cm_id->port_num); } @@ -82,11 +83,11 @@ static ssize_t rtrs_srv_hca_name_show(struct kobject *kobj, struct kobj_attribute *attr, char *page) { - struct rtrs_srv_sess *sess; + struct rtrs_srv_path *srv_path; - sess = container_of(kobj, struct rtrs_srv_sess, kobj); + srv_path = container_of(kobj, struct rtrs_srv_path, kobj); - return sysfs_emit(page, "%s\n", sess->s.dev->ib_dev->name); + return sysfs_emit(page, "%s\n", srv_path->s.dev->ib_dev->name); } static struct kobj_attribute rtrs_srv_hca_name_attr = @@ -96,11 +97,11 @@ static ssize_t rtrs_srv_src_addr_show(struct kobject *kobj, struct kobj_attribute *attr, char *page) { - struct rtrs_srv_sess *sess; + struct rtrs_srv_path *srv_path; int cnt; - sess = container_of(kobj, struct rtrs_srv_sess, kobj); - cnt = sockaddr_to_str((struct sockaddr *)&sess->s.dst_addr, + srv_path = container_of(kobj, struct rtrs_srv_path, kobj); + cnt = sockaddr_to_str((struct sockaddr *)&srv_path->s.dst_addr, page, PAGE_SIZE); return cnt + sysfs_emit_at(page, cnt, "\n"); } @@ -112,11 +113,11 @@ static ssize_t rtrs_srv_dst_addr_show(struct kobject *kobj, struct kobj_attribute *attr, char *page) { - struct rtrs_srv_sess *sess; + struct rtrs_srv_path *srv_path; int len; - sess = container_of(kobj, struct rtrs_srv_sess, kobj); - len = sockaddr_to_str((struct sockaddr *)&sess->s.src_addr, page, + srv_path = container_of(kobj, struct rtrs_srv_path, kobj); + len = sockaddr_to_str((struct sockaddr *)&srv_path->s.src_addr, page, PAGE_SIZE); len += sysfs_emit_at(page, len, "\n"); return len; @@ -125,7 +126,7 @@ static ssize_t rtrs_srv_dst_addr_show(struct kobject *kobj, static struct kobj_attribute rtrs_srv_dst_addr_attr = __ATTR(dst_addr, 0444, rtrs_srv_dst_addr_show, NULL); -static struct attribute *rtrs_srv_sess_attrs[] = { +static struct attribute *rtrs_srv_path_attrs[] = { &rtrs_srv_hca_name_attr.attr, &rtrs_srv_hca_port_attr.attr, &rtrs_srv_src_addr_attr.attr, @@ -134,8 +135,8 @@ static struct attribute *rtrs_srv_sess_attrs[] = { NULL, }; -static const struct attribute_group rtrs_srv_sess_attr_group = { - .attrs = rtrs_srv_sess_attrs, +static const struct attribute_group rtrs_srv_path_attr_group = { + .attrs = rtrs_srv_path_attrs, }; STAT_ATTR(struct rtrs_srv_stats, rdma, @@ -151,9 +152,9 @@ static const struct attribute_group rtrs_srv_stats_attr_group = { .attrs = rtrs_srv_stats_attrs, }; -static int rtrs_srv_create_once_sysfs_root_folders(struct rtrs_srv_sess *sess) +static int rtrs_srv_create_once_sysfs_root_folders(struct rtrs_srv_path *srv_path) { - struct rtrs_srv *srv = sess->srv; + struct rtrs_srv_sess *srv = srv_path->srv; int err = 0; mutex_lock(&srv->paths_mutex); @@ -164,7 +165,7 @@ static int rtrs_srv_create_once_sysfs_root_folders(struct rtrs_srv_sess *sess) goto unlock; } srv->dev.class = rtrs_dev_class; - err = dev_set_name(&srv->dev, "%s", sess->s.sessname); + err = dev_set_name(&srv->dev, "%s", srv_path->s.sessname); if (err) goto unlock; @@ -196,9 +197,9 @@ unlock: } static void -rtrs_srv_destroy_once_sysfs_root_folders(struct rtrs_srv_sess *sess) +rtrs_srv_destroy_once_sysfs_root_folders(struct rtrs_srv_path *srv_path) { - struct rtrs_srv *srv = sess->srv; + struct rtrs_srv_sess *srv = srv_path->srv; mutex_lock(&srv->paths_mutex); if (!--srv->dev_ref) { @@ -213,7 +214,7 @@ rtrs_srv_destroy_once_sysfs_root_folders(struct rtrs_srv_sess *sess) } } -static void rtrs_srv_sess_stats_release(struct kobject *kobj) +static void rtrs_srv_path_stats_release(struct kobject *kobj) { struct rtrs_srv_stats *stats; @@ -224,22 +225,22 @@ static void rtrs_srv_sess_stats_release(struct kobject *kobj) static struct kobj_type ktype_stats = { .sysfs_ops = &kobj_sysfs_ops, - .release = rtrs_srv_sess_stats_release, + .release = rtrs_srv_path_stats_release, }; -static int rtrs_srv_create_stats_files(struct rtrs_srv_sess *sess) +static int rtrs_srv_create_stats_files(struct rtrs_srv_path *srv_path) { int err; - struct rtrs_sess *s = &sess->s; + struct rtrs_path *s = &srv_path->s; - err = kobject_init_and_add(&sess->stats->kobj_stats, &ktype_stats, - &sess->kobj, "stats"); + err = kobject_init_and_add(&srv_path->stats->kobj_stats, &ktype_stats, + &srv_path->kobj, "stats"); if (err) { rtrs_err(s, "kobject_init_and_add(): %d\n", err); - kobject_put(&sess->stats->kobj_stats); + kobject_put(&srv_path->stats->kobj_stats); return err; } - err = sysfs_create_group(&sess->stats->kobj_stats, + err = sysfs_create_group(&srv_path->stats->kobj_stats, &rtrs_srv_stats_attr_group); if (err) { rtrs_err(s, "sysfs_create_group(): %d\n", err); @@ -249,64 +250,64 @@ static int rtrs_srv_create_stats_files(struct rtrs_srv_sess *sess) return 0; err: - kobject_del(&sess->stats->kobj_stats); - kobject_put(&sess->stats->kobj_stats); + kobject_del(&srv_path->stats->kobj_stats); + kobject_put(&srv_path->stats->kobj_stats); return err; } -int rtrs_srv_create_sess_files(struct rtrs_srv_sess *sess) +int rtrs_srv_create_path_files(struct rtrs_srv_path *srv_path) { - struct rtrs_srv *srv = sess->srv; - struct rtrs_sess *s = &sess->s; + struct rtrs_srv_sess *srv = srv_path->srv; + struct rtrs_path *s = &srv_path->s; char str[NAME_MAX]; int err; struct rtrs_addr path = { - .src = &sess->s.dst_addr, - .dst = &sess->s.src_addr, + .src = &srv_path->s.dst_addr, + .dst = &srv_path->s.src_addr, }; rtrs_addr_to_str(&path, str, sizeof(str)); - err = rtrs_srv_create_once_sysfs_root_folders(sess); + err = rtrs_srv_create_once_sysfs_root_folders(srv_path); if (err) return err; - err = kobject_init_and_add(&sess->kobj, &ktype, srv->kobj_paths, + err = kobject_init_and_add(&srv_path->kobj, &ktype, srv->kobj_paths, "%s", str); if (err) { rtrs_err(s, "kobject_init_and_add(): %d\n", err); goto destroy_root; } - err = sysfs_create_group(&sess->kobj, &rtrs_srv_sess_attr_group); + err = sysfs_create_group(&srv_path->kobj, &rtrs_srv_path_attr_group); if (err) { rtrs_err(s, "sysfs_create_group(): %d\n", err); goto put_kobj; } - err = rtrs_srv_create_stats_files(sess); + err = rtrs_srv_create_stats_files(srv_path); if (err) goto remove_group; return 0; remove_group: - sysfs_remove_group(&sess->kobj, &rtrs_srv_sess_attr_group); + sysfs_remove_group(&srv_path->kobj, &rtrs_srv_path_attr_group); put_kobj: - kobject_del(&sess->kobj); + kobject_del(&srv_path->kobj); destroy_root: - kobject_put(&sess->kobj); - rtrs_srv_destroy_once_sysfs_root_folders(sess); + kobject_put(&srv_path->kobj); + rtrs_srv_destroy_once_sysfs_root_folders(srv_path); return err; } -void rtrs_srv_destroy_sess_files(struct rtrs_srv_sess *sess) +void rtrs_srv_destroy_path_files(struct rtrs_srv_path *srv_path) { - if (sess->kobj.state_in_sysfs) { - kobject_del(&sess->stats->kobj_stats); - kobject_put(&sess->stats->kobj_stats); - sysfs_remove_group(&sess->kobj, &rtrs_srv_sess_attr_group); - kobject_put(&sess->kobj); + if (srv_path->kobj.state_in_sysfs) { + kobject_del(&srv_path->stats->kobj_stats); + kobject_put(&srv_path->stats->kobj_stats); + sysfs_remove_group(&srv_path->kobj, &rtrs_srv_path_attr_group); + kobject_put(&srv_path->kobj); - rtrs_srv_destroy_once_sysfs_root_folders(sess); + rtrs_srv_destroy_once_sysfs_root_folders(srv_path); } } diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.c b/drivers/infiniband/ulp/rtrs/rtrs-srv.c index 7df71f8cf149..24024bce2566 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-srv.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.c @@ -62,19 +62,19 @@ static inline struct rtrs_srv_con *to_srv_con(struct rtrs_con *c) return container_of(c, struct rtrs_srv_con, c); } -static inline struct rtrs_srv_sess *to_srv_sess(struct rtrs_sess *s) +static inline struct rtrs_srv_path *to_srv_path(struct rtrs_path *s) { - return container_of(s, struct rtrs_srv_sess, s); + return container_of(s, struct rtrs_srv_path, s); } -static bool rtrs_srv_change_state(struct rtrs_srv_sess *sess, +static bool rtrs_srv_change_state(struct rtrs_srv_path *srv_path, enum rtrs_srv_state new_state) { enum rtrs_srv_state old_state; bool changed = false; - spin_lock_irq(&sess->state_lock); - old_state = sess->state; + spin_lock_irq(&srv_path->state_lock); + old_state = srv_path->state; switch (new_state) { case RTRS_SRV_CONNECTED: if (old_state == RTRS_SRV_CONNECTING) @@ -93,8 +93,8 @@ static bool rtrs_srv_change_state(struct rtrs_srv_sess *sess, break; } if (changed) - sess->state = new_state; - spin_unlock_irq(&sess->state_lock); + srv_path->state = new_state; + spin_unlock_irq(&srv_path->state_lock); return changed; } @@ -106,16 +106,16 @@ static void free_id(struct rtrs_srv_op *id) kfree(id); } -static void rtrs_srv_free_ops_ids(struct rtrs_srv_sess *sess) +static void rtrs_srv_free_ops_ids(struct rtrs_srv_path *srv_path) { - struct rtrs_srv *srv = sess->srv; + struct rtrs_srv_sess *srv = srv_path->srv; int i; - if (sess->ops_ids) { + if (srv_path->ops_ids) { for (i = 0; i < srv->queue_depth; i++) - free_id(sess->ops_ids[i]); - kfree(sess->ops_ids); - sess->ops_ids = NULL; + free_id(srv_path->ops_ids[i]); + kfree(srv_path->ops_ids); + srv_path->ops_ids = NULL; } } @@ -127,21 +127,24 @@ static struct ib_cqe io_comp_cqe = { static inline void rtrs_srv_inflight_ref_release(struct percpu_ref *ref) { - struct rtrs_srv_sess *sess = container_of(ref, struct rtrs_srv_sess, ids_inflight_ref); + struct rtrs_srv_path *srv_path = container_of(ref, + struct rtrs_srv_path, + ids_inflight_ref); - percpu_ref_exit(&sess->ids_inflight_ref); - complete(&sess->complete_done); + percpu_ref_exit(&srv_path->ids_inflight_ref); + complete(&srv_path->complete_done); } -static int rtrs_srv_alloc_ops_ids(struct rtrs_srv_sess *sess) +static int rtrs_srv_alloc_ops_ids(struct rtrs_srv_path *srv_path) { - struct rtrs_srv *srv = sess->srv; + struct rtrs_srv_sess *srv = srv_path->srv; struct rtrs_srv_op *id; int i, ret; - sess->ops_ids = kcalloc(srv->queue_depth, sizeof(*sess->ops_ids), - GFP_KERNEL); - if (!sess->ops_ids) + srv_path->ops_ids = kcalloc(srv->queue_depth, + sizeof(*srv_path->ops_ids), + GFP_KERNEL); + if (!srv_path->ops_ids) goto err; for (i = 0; i < srv->queue_depth; ++i) { @@ -149,44 +152,44 @@ static int rtrs_srv_alloc_ops_ids(struct rtrs_srv_sess *sess) if (!id) goto err; - sess->ops_ids[i] = id; + srv_path->ops_ids[i] = id; } - ret = percpu_ref_init(&sess->ids_inflight_ref, + ret = percpu_ref_init(&srv_path->ids_inflight_ref, rtrs_srv_inflight_ref_release, 0, GFP_KERNEL); if (ret) { pr_err("Percpu reference init failed\n"); goto err; } - init_completion(&sess->complete_done); + init_completion(&srv_path->complete_done); return 0; err: - rtrs_srv_free_ops_ids(sess); + rtrs_srv_free_ops_ids(srv_path); return -ENOMEM; } -static inline void rtrs_srv_get_ops_ids(struct rtrs_srv_sess *sess) +static inline void rtrs_srv_get_ops_ids(struct rtrs_srv_path *srv_path) { - percpu_ref_get(&sess->ids_inflight_ref); + percpu_ref_get(&srv_path->ids_inflight_ref); } -static inline void rtrs_srv_put_ops_ids(struct rtrs_srv_sess *sess) +static inline void rtrs_srv_put_ops_ids(struct rtrs_srv_path *srv_path) { - percpu_ref_put(&sess->ids_inflight_ref); + percpu_ref_put(&srv_path->ids_inflight_ref); } static void rtrs_srv_reg_mr_done(struct ib_cq *cq, struct ib_wc *wc) { struct rtrs_srv_con *con = to_srv_con(wc->qp->qp_context); - struct rtrs_sess *s = con->c.sess; - struct rtrs_srv_sess *sess = to_srv_sess(s); + struct rtrs_path *s = con->c.path; + struct rtrs_srv_path *srv_path = to_srv_path(s); if (wc->status != IB_WC_SUCCESS) { rtrs_err(s, "REG MR failed: %s\n", ib_wc_status_msg(wc->status)); - close_sess(sess); + close_path(srv_path); return; } } @@ -197,9 +200,9 @@ static struct ib_cqe local_reg_cqe = { static int rdma_write_sg(struct rtrs_srv_op *id) { - struct rtrs_sess *s = id->con->c.sess; - struct rtrs_srv_sess *sess = to_srv_sess(s); - dma_addr_t dma_addr = sess->dma_addr[id->msg_id]; + struct rtrs_path *s = id->con->c.path; + struct rtrs_srv_path *srv_path = to_srv_path(s); + dma_addr_t dma_addr = srv_path->dma_addr[id->msg_id]; struct rtrs_srv_mr *srv_mr; struct ib_send_wr inv_wr; struct ib_rdma_wr imm_wr; @@ -233,7 +236,7 @@ static int rdma_write_sg(struct rtrs_srv_op *id) return -EINVAL; } - plist->lkey = sess->s.dev->ib_pd->local_dma_lkey; + plist->lkey = srv_path->s.dev->ib_pd->local_dma_lkey; offset += plist->length; wr->wr.sg_list = plist; @@ -284,7 +287,7 @@ static int rdma_write_sg(struct rtrs_srv_op *id) if (always_invalidate) { struct rtrs_msg_rkey_rsp *msg; - srv_mr = &sess->mrs[id->msg_id]; + srv_mr = &srv_path->mrs[id->msg_id]; rwr.wr.opcode = IB_WR_REG_MR; rwr.wr.wr_cqe = &local_reg_cqe; rwr.wr.num_sge = 0; @@ -300,11 +303,11 @@ static int rdma_write_sg(struct rtrs_srv_op *id) list.addr = srv_mr->iu->dma_addr; list.length = sizeof(*msg); - list.lkey = sess->s.dev->ib_pd->local_dma_lkey; + list.lkey = srv_path->s.dev->ib_pd->local_dma_lkey; imm_wr.wr.sg_list = &list; imm_wr.wr.num_sge = 1; imm_wr.wr.opcode = IB_WR_SEND_WITH_IMM; - ib_dma_sync_single_for_device(sess->s.dev->ib_dev, + ib_dma_sync_single_for_device(srv_path->s.dev->ib_dev, srv_mr->iu->dma_addr, srv_mr->iu->size, DMA_TO_DEVICE); } else { @@ -317,7 +320,7 @@ static int rdma_write_sg(struct rtrs_srv_op *id) 0, need_inval)); imm_wr.wr.wr_cqe = &io_comp_cqe; - ib_dma_sync_single_for_device(sess->s.dev->ib_dev, dma_addr, + ib_dma_sync_single_for_device(srv_path->s.dev->ib_dev, dma_addr, offset, DMA_BIDIRECTIONAL); err = ib_post_send(id->con->c.qp, &id->tx_wr.wr, NULL); @@ -341,8 +344,8 @@ static int rdma_write_sg(struct rtrs_srv_op *id) static int send_io_resp_imm(struct rtrs_srv_con *con, struct rtrs_srv_op *id, int errno) { - struct rtrs_sess *s = con->c.sess; - struct rtrs_srv_sess *sess = to_srv_sess(s); + struct rtrs_path *s = con->c.path; + struct rtrs_srv_path *srv_path = to_srv_path(s); struct ib_send_wr inv_wr, *wr = NULL; struct ib_rdma_wr imm_wr; struct ib_reg_wr rwr; @@ -402,7 +405,7 @@ static int send_io_resp_imm(struct rtrs_srv_con *con, struct rtrs_srv_op *id, struct ib_sge list; struct rtrs_msg_rkey_rsp *msg; - srv_mr = &sess->mrs[id->msg_id]; + srv_mr = &srv_path->mrs[id->msg_id]; rwr.wr.next = &imm_wr.wr; rwr.wr.opcode = IB_WR_REG_MR; rwr.wr.wr_cqe = &local_reg_cqe; @@ -419,11 +422,11 @@ static int send_io_resp_imm(struct rtrs_srv_con *con, struct rtrs_srv_op *id, list.addr = srv_mr->iu->dma_addr; list.length = sizeof(*msg); - list.lkey = sess->s.dev->ib_pd->local_dma_lkey; + list.lkey = srv_path->s.dev->ib_pd->local_dma_lkey; imm_wr.wr.sg_list = &list; imm_wr.wr.num_sge = 1; imm_wr.wr.opcode = IB_WR_SEND_WITH_IMM; - ib_dma_sync_single_for_device(sess->s.dev->ib_dev, + ib_dma_sync_single_for_device(srv_path->s.dev->ib_dev, srv_mr->iu->dma_addr, srv_mr->iu->size, DMA_TO_DEVICE); } else { @@ -444,11 +447,11 @@ static int send_io_resp_imm(struct rtrs_srv_con *con, struct rtrs_srv_op *id, return err; } -void close_sess(struct rtrs_srv_sess *sess) +void close_path(struct rtrs_srv_path *srv_path) { - if (rtrs_srv_change_state(sess, RTRS_SRV_CLOSING)) - queue_work(rtrs_wq, &sess->close_work); - WARN_ON(sess->state != RTRS_SRV_CLOSING); + if (rtrs_srv_change_state(srv_path, RTRS_SRV_CLOSING)) + queue_work(rtrs_wq, &srv_path->close_work); + WARN_ON(srv_path->state != RTRS_SRV_CLOSING); } static inline const char *rtrs_srv_state_str(enum rtrs_srv_state state) @@ -480,35 +483,35 @@ static inline const char *rtrs_srv_state_str(enum rtrs_srv_state state) */ bool rtrs_srv_resp_rdma(struct rtrs_srv_op *id, int status) { - struct rtrs_srv_sess *sess; + struct rtrs_srv_path *srv_path; struct rtrs_srv_con *con; - struct rtrs_sess *s; + struct rtrs_path *s; int err; if (WARN_ON(!id)) return true; con = id->con; - s = con->c.sess; - sess = to_srv_sess(s); + s = con->c.path; + srv_path = to_srv_path(s); id->status = status; - if (sess->state != RTRS_SRV_CONNECTED) { + if (srv_path->state != RTRS_SRV_CONNECTED) { rtrs_err_rl(s, - "Sending I/O response failed, session %s is disconnected, sess state %s\n", - kobject_name(&sess->kobj), - rtrs_srv_state_str(sess->state)); + "Sending I/O response failed, server path %s is disconnected, path state %s\n", + kobject_name(&srv_path->kobj), + rtrs_srv_state_str(srv_path->state)); goto out; } if (always_invalidate) { - struct rtrs_srv_mr *mr = &sess->mrs[id->msg_id]; + struct rtrs_srv_mr *mr = &srv_path->mrs[id->msg_id]; ib_update_fast_reg_key(mr->mr, ib_inc_rkey(mr->mr->rkey)); } if (atomic_sub_return(1, &con->c.sq_wr_avail) < 0) { - rtrs_err(s, "IB send queue full: sess=%s cid=%d\n", - kobject_name(&sess->kobj), + rtrs_err(s, "IB send queue full: srv_path=%s cid=%d\n", + kobject_name(&srv_path->kobj), con->c.cid); atomic_add(1, &con->c.sq_wr_avail); spin_lock(&con->rsp_wr_wait_lock); @@ -523,12 +526,12 @@ bool rtrs_srv_resp_rdma(struct rtrs_srv_op *id, int status) err = rdma_write_sg(id); if (err) { - rtrs_err_rl(s, "IO response failed: %d: sess=%s\n", err, - kobject_name(&sess->kobj)); - close_sess(sess); + rtrs_err_rl(s, "IO response failed: %d: srv_path=%s\n", err, + kobject_name(&srv_path->kobj)); + close_path(srv_path); } out: - rtrs_srv_put_ops_ids(sess); + rtrs_srv_put_ops_ids(srv_path); return true; } EXPORT_SYMBOL(rtrs_srv_resp_rdma); @@ -538,33 +541,33 @@ EXPORT_SYMBOL(rtrs_srv_resp_rdma); * @srv: Session pointer * @priv: The private pointer that is associated with the session. */ -void rtrs_srv_set_sess_priv(struct rtrs_srv *srv, void *priv) +void rtrs_srv_set_sess_priv(struct rtrs_srv_sess *srv, void *priv) { srv->priv = priv; } EXPORT_SYMBOL(rtrs_srv_set_sess_priv); -static void unmap_cont_bufs(struct rtrs_srv_sess *sess) +static void unmap_cont_bufs(struct rtrs_srv_path *srv_path) { int i; - for (i = 0; i < sess->mrs_num; i++) { + for (i = 0; i < srv_path->mrs_num; i++) { struct rtrs_srv_mr *srv_mr; - srv_mr = &sess->mrs[i]; - rtrs_iu_free(srv_mr->iu, sess->s.dev->ib_dev, 1); + srv_mr = &srv_path->mrs[i]; + rtrs_iu_free(srv_mr->iu, srv_path->s.dev->ib_dev, 1); ib_dereg_mr(srv_mr->mr); - ib_dma_unmap_sg(sess->s.dev->ib_dev, srv_mr->sgt.sgl, + ib_dma_unmap_sg(srv_path->s.dev->ib_dev, srv_mr->sgt.sgl, srv_mr->sgt.nents, DMA_BIDIRECTIONAL); sg_free_table(&srv_mr->sgt); } - kfree(sess->mrs); + kfree(srv_path->mrs); } -static int map_cont_bufs(struct rtrs_srv_sess *sess) +static int map_cont_bufs(struct rtrs_srv_path *srv_path) { - struct rtrs_srv *srv = sess->srv; - struct rtrs_sess *ss = &sess->s; + struct rtrs_srv_sess *srv = srv_path->srv; + struct rtrs_path *ss = &srv_path->s; int i, mri, err, mrs_num; unsigned int chunk_bits; int chunks_per_mr = 1; @@ -581,19 +584,19 @@ static int map_cont_bufs(struct rtrs_srv_sess *sess) mrs_num = srv->queue_depth; } else { chunks_per_mr = - sess->s.dev->ib_dev->attrs.max_fast_reg_page_list_len; + srv_path->s.dev->ib_dev->attrs.max_fast_reg_page_list_len; mrs_num = DIV_ROUND_UP(srv->queue_depth, chunks_per_mr); chunks_per_mr = DIV_ROUND_UP(srv->queue_depth, mrs_num); } - sess->mrs = kcalloc(mrs_num, sizeof(*sess->mrs), GFP_KERNEL); - if (!sess->mrs) + srv_path->mrs = kcalloc(mrs_num, sizeof(*srv_path->mrs), GFP_KERNEL); + if (!srv_path->mrs) return -ENOMEM; - sess->mrs_num = mrs_num; + srv_path->mrs_num = mrs_num; for (mri = 0; mri < mrs_num; mri++) { - struct rtrs_srv_mr *srv_mr = &sess->mrs[mri]; + struct rtrs_srv_mr *srv_mr = &srv_path->mrs[mri]; struct sg_table *sgt = &srv_mr->sgt; struct scatterlist *s; struct ib_mr *mr; @@ -612,13 +615,13 @@ static int map_cont_bufs(struct rtrs_srv_sess *sess) sg_set_page(s, srv->chunks[chunks + i], max_chunk_size, 0); - nr = ib_dma_map_sg(sess->s.dev->ib_dev, sgt->sgl, + nr = ib_dma_map_sg(srv_path->s.dev->ib_dev, sgt->sgl, sgt->nents, DMA_BIDIRECTIONAL); if (nr < sgt->nents) { err = nr < 0 ? nr : -EINVAL; goto free_sg; } - mr = ib_alloc_mr(sess->s.dev->ib_pd, IB_MR_TYPE_MEM_REG, + mr = ib_alloc_mr(srv_path->s.dev->ib_pd, IB_MR_TYPE_MEM_REG, sgt->nents); if (IS_ERR(mr)) { err = PTR_ERR(mr); @@ -634,7 +637,7 @@ static int map_cont_bufs(struct rtrs_srv_sess *sess) if (always_invalidate) { srv_mr->iu = rtrs_iu_alloc(1, sizeof(struct rtrs_msg_rkey_rsp), - GFP_KERNEL, sess->s.dev->ib_dev, + GFP_KERNEL, srv_path->s.dev->ib_dev, DMA_TO_DEVICE, rtrs_srv_rdma_done); if (!srv_mr->iu) { err = -ENOMEM; @@ -644,7 +647,7 @@ static int map_cont_bufs(struct rtrs_srv_sess *sess) } /* Eventually dma addr for each chunk can be cached */ for_each_sg(sgt->sgl, s, sgt->orig_nents, i) - sess->dma_addr[chunks + i] = sg_dma_address(s); + srv_path->dma_addr[chunks + i] = sg_dma_address(s); ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey)); srv_mr->mr = mr; @@ -652,75 +655,75 @@ static int map_cont_bufs(struct rtrs_srv_sess *sess) continue; err: while (mri--) { - srv_mr = &sess->mrs[mri]; + srv_mr = &srv_path->mrs[mri]; sgt = &srv_mr->sgt; mr = srv_mr->mr; - rtrs_iu_free(srv_mr->iu, sess->s.dev->ib_dev, 1); + rtrs_iu_free(srv_mr->iu, srv_path->s.dev->ib_dev, 1); dereg_mr: ib_dereg_mr(mr); unmap_sg: - ib_dma_unmap_sg(sess->s.dev->ib_dev, sgt->sgl, + ib_dma_unmap_sg(srv_path->s.dev->ib_dev, sgt->sgl, sgt->nents, DMA_BIDIRECTIONAL); free_sg: sg_free_table(sgt); } - kfree(sess->mrs); + kfree(srv_path->mrs); return err; } chunk_bits = ilog2(srv->queue_depth - 1) + 1; - sess->mem_bits = (MAX_IMM_PAYL_BITS - chunk_bits); + srv_path->mem_bits = (MAX_IMM_PAYL_BITS - chunk_bits); return 0; } static void rtrs_srv_hb_err_handler(struct rtrs_con *c) { - close_sess(to_srv_sess(c->sess)); + close_path(to_srv_path(c->path)); } -static void rtrs_srv_init_hb(struct rtrs_srv_sess *sess) +static void rtrs_srv_init_hb(struct rtrs_srv_path *srv_path) { - rtrs_init_hb(&sess->s, &io_comp_cqe, + rtrs_init_hb(&srv_path->s, &io_comp_cqe, RTRS_HB_INTERVAL_MS, RTRS_HB_MISSED_MAX, rtrs_srv_hb_err_handler, rtrs_wq); } -static void rtrs_srv_start_hb(struct rtrs_srv_sess *sess) +static void rtrs_srv_start_hb(struct rtrs_srv_path *srv_path) { - rtrs_start_hb(&sess->s); + rtrs_start_hb(&srv_path->s); } -static void rtrs_srv_stop_hb(struct rtrs_srv_sess *sess) +static void rtrs_srv_stop_hb(struct rtrs_srv_path *srv_path) { - rtrs_stop_hb(&sess->s); + rtrs_stop_hb(&srv_path->s); } static void rtrs_srv_info_rsp_done(struct ib_cq *cq, struct ib_wc *wc) { struct rtrs_srv_con *con = to_srv_con(wc->qp->qp_context); - struct rtrs_sess *s = con->c.sess; - struct rtrs_srv_sess *sess = to_srv_sess(s); + struct rtrs_path *s = con->c.path; + struct rtrs_srv_path *srv_path = to_srv_path(s); struct rtrs_iu *iu; iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe); - rtrs_iu_free(iu, sess->s.dev->ib_dev, 1); + rtrs_iu_free(iu, srv_path->s.dev->ib_dev, 1); if (wc->status != IB_WC_SUCCESS) { rtrs_err(s, "Sess info response send failed: %s\n", ib_wc_status_msg(wc->status)); - close_sess(sess); + close_path(srv_path); return; } WARN_ON(wc->opcode != IB_WC_SEND); } -static void rtrs_srv_sess_up(struct rtrs_srv_sess *sess) +static void rtrs_srv_path_up(struct rtrs_srv_path *srv_path) { - struct rtrs_srv *srv = sess->srv; + struct rtrs_srv_sess *srv = srv_path->srv; struct rtrs_srv_ctx *ctx = srv->ctx; int up; @@ -731,18 +734,18 @@ static void rtrs_srv_sess_up(struct rtrs_srv_sess *sess) mutex_unlock(&srv->paths_ev_mutex); /* Mark session as established */ - sess->established = true; + srv_path->established = true; } -static void rtrs_srv_sess_down(struct rtrs_srv_sess *sess) +static void rtrs_srv_path_down(struct rtrs_srv_path *srv_path) { - struct rtrs_srv *srv = sess->srv; + struct rtrs_srv_sess *srv = srv_path->srv; struct rtrs_srv_ctx *ctx = srv->ctx; - if (!sess->established) + if (!srv_path->established) return; - sess->established = false; + srv_path->established = false; mutex_lock(&srv->paths_ev_mutex); WARN_ON(!srv->paths_up); if (--srv->paths_up == 0) @@ -750,11 +753,11 @@ static void rtrs_srv_sess_down(struct rtrs_srv_sess *sess) mutex_unlock(&srv->paths_ev_mutex); } -static bool exist_sessname(struct rtrs_srv_ctx *ctx, - const char *sessname, const uuid_t *path_uuid) +static bool exist_pathname(struct rtrs_srv_ctx *ctx, + const char *pathname, const uuid_t *path_uuid) { - struct rtrs_srv *srv; - struct rtrs_srv_sess *sess; + struct rtrs_srv_sess *srv; + struct rtrs_srv_path *srv_path; bool found = false; mutex_lock(&ctx->srv_mutex); @@ -767,9 +770,9 @@ static bool exist_sessname(struct rtrs_srv_ctx *ctx, continue; } - list_for_each_entry(sess, &srv->paths_list, s.entry) { - if (strlen(sess->s.sessname) == strlen(sessname) && - !strcmp(sess->s.sessname, sessname)) { + list_for_each_entry(srv_path, &srv->paths_list, s.entry) { + if (strlen(srv_path->s.sessname) == strlen(pathname) && + !strcmp(srv_path->s.sessname, pathname)) { found = true; break; } @@ -782,14 +785,14 @@ static bool exist_sessname(struct rtrs_srv_ctx *ctx, return found; } -static int post_recv_sess(struct rtrs_srv_sess *sess); +static int post_recv_path(struct rtrs_srv_path *srv_path); static int rtrs_rdma_do_reject(struct rdma_cm_id *cm_id, int errno); static int process_info_req(struct rtrs_srv_con *con, struct rtrs_msg_info_req *msg) { - struct rtrs_sess *s = con->c.sess; - struct rtrs_srv_sess *sess = to_srv_sess(s); + struct rtrs_path *s = con->c.path; + struct rtrs_srv_path *srv_path = to_srv_path(s); struct ib_send_wr *reg_wr = NULL; struct rtrs_msg_info_rsp *rsp; struct rtrs_iu *tx_iu; @@ -797,31 +800,32 @@ static int process_info_req(struct rtrs_srv_con *con, int mri, err; size_t tx_sz; - err = post_recv_sess(sess); + err = post_recv_path(srv_path); if (err) { - rtrs_err(s, "post_recv_sess(), err: %d\n", err); + rtrs_err(s, "post_recv_path(), err: %d\n", err); return err; } - if (strchr(msg->sessname, '/') || strchr(msg->sessname, '.')) { - rtrs_err(s, "sessname cannot contain / and .\n"); + if (strchr(msg->pathname, '/') || strchr(msg->pathname, '.')) { + rtrs_err(s, "pathname cannot contain / and .\n"); return -EINVAL; } - if (exist_sessname(sess->srv->ctx, - msg->sessname, &sess->srv->paths_uuid)) { - rtrs_err(s, "sessname is duplicated: %s\n", msg->sessname); + if (exist_pathname(srv_path->srv->ctx, + msg->pathname, &srv_path->srv->paths_uuid)) { + rtrs_err(s, "pathname is duplicated: %s\n", msg->pathname); return -EPERM; } - strscpy(sess->s.sessname, msg->sessname, sizeof(sess->s.sessname)); + strscpy(srv_path->s.sessname, msg->pathname, + sizeof(srv_path->s.sessname)); - rwr = kcalloc(sess->mrs_num, sizeof(*rwr), GFP_KERNEL); + rwr = kcalloc(srv_path->mrs_num, sizeof(*rwr), GFP_KERNEL); if (!rwr) return -ENOMEM; tx_sz = sizeof(*rsp); - tx_sz += sizeof(rsp->desc[0]) * sess->mrs_num; - tx_iu = rtrs_iu_alloc(1, tx_sz, GFP_KERNEL, sess->s.dev->ib_dev, + tx_sz += sizeof(rsp->desc[0]) * srv_path->mrs_num; + tx_iu = rtrs_iu_alloc(1, tx_sz, GFP_KERNEL, srv_path->s.dev->ib_dev, DMA_TO_DEVICE, rtrs_srv_info_rsp_done); if (!tx_iu) { err = -ENOMEM; @@ -830,10 +834,10 @@ static int process_info_req(struct rtrs_srv_con *con, rsp = tx_iu->buf; rsp->type = cpu_to_le16(RTRS_MSG_INFO_RSP); - rsp->sg_cnt = cpu_to_le16(sess->mrs_num); + rsp->sg_cnt = cpu_to_le16(srv_path->mrs_num); - for (mri = 0; mri < sess->mrs_num; mri++) { - struct ib_mr *mr = sess->mrs[mri].mr; + for (mri = 0; mri < srv_path->mrs_num; mri++) { + struct ib_mr *mr = srv_path->mrs[mri].mr; rsp->desc[mri].addr = cpu_to_le64(mr->iova); rsp->desc[mri].key = cpu_to_le32(mr->rkey); @@ -854,13 +858,13 @@ static int process_info_req(struct rtrs_srv_con *con, reg_wr = &rwr[mri].wr; } - err = rtrs_srv_create_sess_files(sess); + err = rtrs_srv_create_path_files(srv_path); if (err) goto iu_free; - kobject_get(&sess->kobj); - get_device(&sess->srv->dev); - rtrs_srv_change_state(sess, RTRS_SRV_CONNECTED); - rtrs_srv_start_hb(sess); + kobject_get(&srv_path->kobj); + get_device(&srv_path->srv->dev); + rtrs_srv_change_state(srv_path, RTRS_SRV_CONNECTED); + rtrs_srv_start_hb(srv_path); /* * We do not account number of established connections at the current @@ -868,9 +872,10 @@ static int process_info_req(struct rtrs_srv_con *con, * all connections are successfully established. Thus, simply notify * listener with a proper event if we are the first path. */ - rtrs_srv_sess_up(sess); + rtrs_srv_path_up(srv_path); - ib_dma_sync_single_for_device(sess->s.dev->ib_dev, tx_iu->dma_addr, + ib_dma_sync_single_for_device(srv_path->s.dev->ib_dev, + tx_iu->dma_addr, tx_iu->size, DMA_TO_DEVICE); /* Send info response */ @@ -878,7 +883,7 @@ static int process_info_req(struct rtrs_srv_con *con, if (err) { rtrs_err(s, "rtrs_iu_post_send(), err: %d\n", err); iu_free: - rtrs_iu_free(tx_iu, sess->s.dev->ib_dev, 1); + rtrs_iu_free(tx_iu, srv_path->s.dev->ib_dev, 1); } rwr_free: kfree(rwr); @@ -889,8 +894,8 @@ rwr_free: static void rtrs_srv_info_req_done(struct ib_cq *cq, struct ib_wc *wc) { struct rtrs_srv_con *con = to_srv_con(wc->qp->qp_context); - struct rtrs_sess *s = con->c.sess; - struct rtrs_srv_sess *sess = to_srv_sess(s); + struct rtrs_path *s = con->c.path; + struct rtrs_srv_path *srv_path = to_srv_path(s); struct rtrs_msg_info_req *msg; struct rtrs_iu *iu; int err; @@ -910,7 +915,7 @@ static void rtrs_srv_info_req_done(struct ib_cq *cq, struct ib_wc *wc) wc->byte_len); goto close; } - ib_dma_sync_single_for_cpu(sess->s.dev->ib_dev, iu->dma_addr, + ib_dma_sync_single_for_cpu(srv_path->s.dev->ib_dev, iu->dma_addr, iu->size, DMA_FROM_DEVICE); msg = iu->buf; if (le16_to_cpu(msg->type) != RTRS_MSG_INFO_REQ) { @@ -923,22 +928,22 @@ static void rtrs_srv_info_req_done(struct ib_cq *cq, struct ib_wc *wc) goto close; out: - rtrs_iu_free(iu, sess->s.dev->ib_dev, 1); + rtrs_iu_free(iu, srv_path->s.dev->ib_dev, 1); return; close: - close_sess(sess); + close_path(srv_path); goto out; } static int post_recv_info_req(struct rtrs_srv_con *con) { - struct rtrs_sess *s = con->c.sess; - struct rtrs_srv_sess *sess = to_srv_sess(s); + struct rtrs_path *s = con->c.path; + struct rtrs_srv_path *srv_path = to_srv_path(s); struct rtrs_iu *rx_iu; int err; rx_iu = rtrs_iu_alloc(1, sizeof(struct rtrs_msg_info_req), - GFP_KERNEL, sess->s.dev->ib_dev, + GFP_KERNEL, srv_path->s.dev->ib_dev, DMA_FROM_DEVICE, rtrs_srv_info_req_done); if (!rx_iu) return -ENOMEM; @@ -946,7 +951,7 @@ static int post_recv_info_req(struct rtrs_srv_con *con) err = rtrs_iu_post_recv(&con->c, rx_iu); if (err) { rtrs_err(s, "rtrs_iu_post_recv(), err: %d\n", err); - rtrs_iu_free(rx_iu, sess->s.dev->ib_dev, 1); + rtrs_iu_free(rx_iu, srv_path->s.dev->ib_dev, 1); return err; } @@ -966,20 +971,20 @@ static int post_recv_io(struct rtrs_srv_con *con, size_t q_size) return 0; } -static int post_recv_sess(struct rtrs_srv_sess *sess) +static int post_recv_path(struct rtrs_srv_path *srv_path) { - struct rtrs_srv *srv = sess->srv; - struct rtrs_sess *s = &sess->s; + struct rtrs_srv_sess *srv = srv_path->srv; + struct rtrs_path *s = &srv_path->s; size_t q_size; int err, cid; - for (cid = 0; cid < sess->s.con_num; cid++) { + for (cid = 0; cid < srv_path->s.con_num; cid++) { if (cid == 0) q_size = SERVICE_CON_QUEUE_DEPTH; else q_size = srv->queue_depth; - err = post_recv_io(to_srv_con(sess->s.con[cid]), q_size); + err = post_recv_io(to_srv_con(srv_path->s.con[cid]), q_size); if (err) { rtrs_err(s, "post_recv_io(), err: %d\n", err); return err; @@ -993,9 +998,9 @@ static void process_read(struct rtrs_srv_con *con, struct rtrs_msg_rdma_read *msg, u32 buf_id, u32 off) { - struct rtrs_sess *s = con->c.sess; - struct rtrs_srv_sess *sess = to_srv_sess(s); - struct rtrs_srv *srv = sess->srv; + struct rtrs_path *s = con->c.path; + struct rtrs_srv_path *srv_path = to_srv_path(s); + struct rtrs_srv_sess *srv = srv_path->srv; struct rtrs_srv_ctx *ctx = srv->ctx; struct rtrs_srv_op *id; @@ -1003,10 +1008,10 @@ static void process_read(struct rtrs_srv_con *con, void *data; int ret; - if (sess->state != RTRS_SRV_CONNECTED) { + if (srv_path->state != RTRS_SRV_CONNECTED) { rtrs_err_rl(s, "Processing read request failed, session is disconnected, sess state %s\n", - rtrs_srv_state_str(sess->state)); + rtrs_srv_state_str(srv_path->state)); return; } if (msg->sg_cnt != 1 && msg->sg_cnt != 0) { @@ -1014,9 +1019,9 @@ static void process_read(struct rtrs_srv_con *con, "Processing read request failed, invalid message\n"); return; } - rtrs_srv_get_ops_ids(sess); - rtrs_srv_update_rdma_stats(sess->stats, off, READ); - id = sess->ops_ids[buf_id]; + rtrs_srv_get_ops_ids(srv_path); + rtrs_srv_update_rdma_stats(srv_path->stats, off, READ); + id = srv_path->ops_ids[buf_id]; id->con = con; id->dir = READ; id->msg_id = buf_id; @@ -1042,18 +1047,18 @@ send_err_msg: rtrs_err_rl(s, "Sending err msg for failed RDMA-Write-Req failed, msg_id %d, err: %d\n", buf_id, ret); - close_sess(sess); + close_path(srv_path); } - rtrs_srv_put_ops_ids(sess); + rtrs_srv_put_ops_ids(srv_path); } static void process_write(struct rtrs_srv_con *con, struct rtrs_msg_rdma_write *req, u32 buf_id, u32 off) { - struct rtrs_sess *s = con->c.sess; - struct rtrs_srv_sess *sess = to_srv_sess(s); - struct rtrs_srv *srv = sess->srv; + struct rtrs_path *s = con->c.path; + struct rtrs_srv_path *srv_path = to_srv_path(s); + struct rtrs_srv_sess *srv = srv_path->srv; struct rtrs_srv_ctx *ctx = srv->ctx; struct rtrs_srv_op *id; @@ -1061,15 +1066,15 @@ static void process_write(struct rtrs_srv_con *con, void *data; int ret; - if (sess->state != RTRS_SRV_CONNECTED) { + if (srv_path->state != RTRS_SRV_CONNECTED) { rtrs_err_rl(s, "Processing write request failed, session is disconnected, sess state %s\n", - rtrs_srv_state_str(sess->state)); + rtrs_srv_state_str(srv_path->state)); return; } - rtrs_srv_get_ops_ids(sess); - rtrs_srv_update_rdma_stats(sess->stats, off, WRITE); - id = sess->ops_ids[buf_id]; + rtrs_srv_get_ops_ids(srv_path); + rtrs_srv_update_rdma_stats(srv_path->stats, off, WRITE); + id = srv_path->ops_ids[buf_id]; id->con = con; id->dir = WRITE; id->msg_id = buf_id; @@ -1094,20 +1099,21 @@ send_err_msg: rtrs_err_rl(s, "Processing write request failed, sending I/O response failed, msg_id %d, err: %d\n", buf_id, ret); - close_sess(sess); + close_path(srv_path); } - rtrs_srv_put_ops_ids(sess); + rtrs_srv_put_ops_ids(srv_path); } static void process_io_req(struct rtrs_srv_con *con, void *msg, u32 id, u32 off) { - struct rtrs_sess *s = con->c.sess; - struct rtrs_srv_sess *sess = to_srv_sess(s); + struct rtrs_path *s = con->c.path; + struct rtrs_srv_path *srv_path = to_srv_path(s); struct rtrs_msg_rdma_hdr *hdr; unsigned int type; - ib_dma_sync_single_for_cpu(sess->s.dev->ib_dev, sess->dma_addr[id], + ib_dma_sync_single_for_cpu(srv_path->s.dev->ib_dev, + srv_path->dma_addr[id], max_chunk_size, DMA_BIDIRECTIONAL); hdr = msg; type = le16_to_cpu(hdr->type); @@ -1129,7 +1135,7 @@ static void process_io_req(struct rtrs_srv_con *con, void *msg, return; err: - close_sess(sess); + close_path(srv_path); } static void rtrs_srv_inv_rkey_done(struct ib_cq *cq, struct ib_wc *wc) @@ -1137,16 +1143,16 @@ static void rtrs_srv_inv_rkey_done(struct ib_cq *cq, struct ib_wc *wc) struct rtrs_srv_mr *mr = container_of(wc->wr_cqe, typeof(*mr), inv_cqe); struct rtrs_srv_con *con = to_srv_con(wc->qp->qp_context); - struct rtrs_sess *s = con->c.sess; - struct rtrs_srv_sess *sess = to_srv_sess(s); - struct rtrs_srv *srv = sess->srv; + struct rtrs_path *s = con->c.path; + struct rtrs_srv_path *srv_path = to_srv_path(s); + struct rtrs_srv_sess *srv = srv_path->srv; u32 msg_id, off; void *data; if (wc->status != IB_WC_SUCCESS) { rtrs_err(s, "Failed IB_WR_LOCAL_INV: %s\n", ib_wc_status_msg(wc->status)); - close_sess(sess); + close_path(srv_path); } msg_id = mr->msg_id; off = mr->msg_off; @@ -1194,9 +1200,9 @@ static void rtrs_rdma_process_wr_wait_list(struct rtrs_srv_con *con) static void rtrs_srv_rdma_done(struct ib_cq *cq, struct ib_wc *wc) { struct rtrs_srv_con *con = to_srv_con(wc->qp->qp_context); - struct rtrs_sess *s = con->c.sess; - struct rtrs_srv_sess *sess = to_srv_sess(s); - struct rtrs_srv *srv = sess->srv; + struct rtrs_path *s = con->c.path; + struct rtrs_srv_path *srv_path = to_srv_path(s); + struct rtrs_srv_sess *srv = srv_path->srv; u32 imm_type, imm_payload; int err; @@ -1206,7 +1212,7 @@ static void rtrs_srv_rdma_done(struct ib_cq *cq, struct ib_wc *wc) "%s (wr_cqe: %p, type: %d, vendor_err: 0x%x, len: %u)\n", ib_wc_status_msg(wc->status), wc->wr_cqe, wc->opcode, wc->vendor_err, wc->byte_len); - close_sess(sess); + close_path(srv_path); } return; } @@ -1222,7 +1228,7 @@ static void rtrs_srv_rdma_done(struct ib_cq *cq, struct ib_wc *wc) err = rtrs_post_recv_empty(&con->c, &io_comp_cqe); if (err) { rtrs_err(s, "rtrs_post_recv(), err: %d\n", err); - close_sess(sess); + close_path(srv_path); break; } rtrs_from_imm(be32_to_cpu(wc->ex.imm_data), @@ -1231,16 +1237,16 @@ static void rtrs_srv_rdma_done(struct ib_cq *cq, struct ib_wc *wc) u32 msg_id, off; void *data; - msg_id = imm_payload >> sess->mem_bits; - off = imm_payload & ((1 << sess->mem_bits) - 1); + msg_id = imm_payload >> srv_path->mem_bits; + off = imm_payload & ((1 << srv_path->mem_bits) - 1); if (msg_id >= srv->queue_depth || off >= max_chunk_size) { rtrs_err(s, "Wrong msg_id %u, off %u\n", msg_id, off); - close_sess(sess); + close_path(srv_path); return; } if (always_invalidate) { - struct rtrs_srv_mr *mr = &sess->mrs[msg_id]; + struct rtrs_srv_mr *mr = &srv_path->mrs[msg_id]; mr->msg_off = off; mr->msg_id = msg_id; @@ -1248,7 +1254,7 @@ static void rtrs_srv_rdma_done(struct ib_cq *cq, struct ib_wc *wc) if (err) { rtrs_err(s, "rtrs_post_recv(), err: %d\n", err); - close_sess(sess); + close_path(srv_path); break; } } else { @@ -1257,10 +1263,10 @@ static void rtrs_srv_rdma_done(struct ib_cq *cq, struct ib_wc *wc) } } else if (imm_type == RTRS_HB_MSG_IMM) { WARN_ON(con->c.cid); - rtrs_send_hb_ack(&sess->s); + rtrs_send_hb_ack(&srv_path->s); } else if (imm_type == RTRS_HB_ACK_IMM) { WARN_ON(con->c.cid); - sess->s.hb_missed_cnt = 0; + srv_path->s.hb_missed_cnt = 0; } else { rtrs_wrn(s, "Unknown IMM type %u\n", imm_type); } @@ -1284,22 +1290,23 @@ static void rtrs_srv_rdma_done(struct ib_cq *cq, struct ib_wc *wc) } /** - * rtrs_srv_get_sess_name() - Get rtrs_srv peer hostname. + * rtrs_srv_get_path_name() - Get rtrs_srv peer hostname. * @srv: Session - * @sessname: Sessname buffer + * @pathname: Pathname buffer * @len: Length of sessname buffer */ -int rtrs_srv_get_sess_name(struct rtrs_srv *srv, char *sessname, size_t len) +int rtrs_srv_get_path_name(struct rtrs_srv_sess *srv, char *pathname, + size_t len) { - struct rtrs_srv_sess *sess; + struct rtrs_srv_path *srv_path; int err = -ENOTCONN; mutex_lock(&srv->paths_mutex); - list_for_each_entry(sess, &srv->paths_list, s.entry) { - if (sess->state != RTRS_SRV_CONNECTED) + list_for_each_entry(srv_path, &srv->paths_list, s.entry) { + if (srv_path->state != RTRS_SRV_CONNECTED) continue; - strscpy(sessname, sess->s.sessname, - min_t(size_t, sizeof(sess->s.sessname), len)); + strscpy(pathname, srv_path->s.sessname, + min_t(size_t, sizeof(srv_path->s.sessname), len)); err = 0; break; } @@ -1307,44 +1314,45 @@ int rtrs_srv_get_sess_name(struct rtrs_srv *srv, char *sessname, size_t len) return err; } -EXPORT_SYMBOL(rtrs_srv_get_sess_name); +EXPORT_SYMBOL(rtrs_srv_get_path_name); /** * rtrs_srv_get_queue_depth() - Get rtrs_srv qdepth. * @srv: Session */ -int rtrs_srv_get_queue_depth(struct rtrs_srv *srv) +int rtrs_srv_get_queue_depth(struct rtrs_srv_sess *srv) { return srv->queue_depth; } EXPORT_SYMBOL(rtrs_srv_get_queue_depth); -static int find_next_bit_ring(struct rtrs_srv_sess *sess) +static int find_next_bit_ring(struct rtrs_srv_path *srv_path) { - struct ib_device *ib_dev = sess->s.dev->ib_dev; + struct ib_device *ib_dev = srv_path->s.dev->ib_dev; int v; - v = cpumask_next(sess->cur_cq_vector, &cq_affinity_mask); + v = cpumask_next(srv_path->cur_cq_vector, &cq_affinity_mask); if (v >= nr_cpu_ids || v >= ib_dev->num_comp_vectors) v = cpumask_first(&cq_affinity_mask); return v; } -static int rtrs_srv_get_next_cq_vector(struct rtrs_srv_sess *sess) +static int rtrs_srv_get_next_cq_vector(struct rtrs_srv_path *srv_path) { - sess->cur_cq_vector = find_next_bit_ring(sess); + srv_path->cur_cq_vector = find_next_bit_ring(srv_path); - return sess->cur_cq_vector; + return srv_path->cur_cq_vector; } static void rtrs_srv_dev_release(struct device *dev) { - struct rtrs_srv *srv = container_of(dev, struct rtrs_srv, dev); + struct rtrs_srv_sess *srv = container_of(dev, struct rtrs_srv_sess, + dev); kfree(srv); } -static void free_srv(struct rtrs_srv *srv) +static void free_srv(struct rtrs_srv_sess *srv) { int i; @@ -1358,11 +1366,11 @@ static void free_srv(struct rtrs_srv *srv) put_device(&srv->dev); } -static struct rtrs_srv *get_or_create_srv(struct rtrs_srv_ctx *ctx, +static struct rtrs_srv_sess *get_or_create_srv(struct rtrs_srv_ctx *ctx, const uuid_t *paths_uuid, bool first_conn) { - struct rtrs_srv *srv; + struct rtrs_srv_sess *srv; int i; mutex_lock(&ctx->srv_mutex); @@ -1424,7 +1432,7 @@ err_free_srv: return ERR_PTR(-ENOMEM); } -static void put_srv(struct rtrs_srv *srv) +static void put_srv(struct rtrs_srv_sess *srv) { if (refcount_dec_and_test(&srv->refcount)) { struct rtrs_srv_ctx *ctx = srv->ctx; @@ -1438,23 +1446,23 @@ static void put_srv(struct rtrs_srv *srv) } } -static void __add_path_to_srv(struct rtrs_srv *srv, - struct rtrs_srv_sess *sess) +static void __add_path_to_srv(struct rtrs_srv_sess *srv, + struct rtrs_srv_path *srv_path) { - list_add_tail(&sess->s.entry, &srv->paths_list); + list_add_tail(&srv_path->s.entry, &srv->paths_list); srv->paths_num++; WARN_ON(srv->paths_num >= MAX_PATHS_NUM); } -static void del_path_from_srv(struct rtrs_srv_sess *sess) +static void del_path_from_srv(struct rtrs_srv_path *srv_path) { - struct rtrs_srv *srv = sess->srv; + struct rtrs_srv_sess *srv = srv_path->srv; if (WARN_ON(!srv)) return; mutex_lock(&srv->paths_mutex); - list_del(&sess->s.entry); + list_del(&srv_path->s.entry); WARN_ON(!srv->paths_num); srv->paths_num--; mutex_unlock(&srv->paths_mutex); @@ -1484,47 +1492,47 @@ static int sockaddr_cmp(const struct sockaddr *a, const struct sockaddr *b) } } -static bool __is_path_w_addr_exists(struct rtrs_srv *srv, +static bool __is_path_w_addr_exists(struct rtrs_srv_sess *srv, struct rdma_addr *addr) { - struct rtrs_srv_sess *sess; + struct rtrs_srv_path *srv_path; - list_for_each_entry(sess, &srv->paths_list, s.entry) - if (!sockaddr_cmp((struct sockaddr *)&sess->s.dst_addr, + list_for_each_entry(srv_path, &srv->paths_list, s.entry) + if (!sockaddr_cmp((struct sockaddr *)&srv_path->s.dst_addr, (struct sockaddr *)&addr->dst_addr) && - !sockaddr_cmp((struct sockaddr *)&sess->s.src_addr, + !sockaddr_cmp((struct sockaddr *)&srv_path->s.src_addr, (struct sockaddr *)&addr->src_addr)) return true; return false; } -static void free_sess(struct rtrs_srv_sess *sess) +static void free_path(struct rtrs_srv_path *srv_path) { - if (sess->kobj.state_in_sysfs) { - kobject_del(&sess->kobj); - kobject_put(&sess->kobj); + if (srv_path->kobj.state_in_sysfs) { + kobject_del(&srv_path->kobj); + kobject_put(&srv_path->kobj); } else { - kfree(sess->stats); - kfree(sess); + kfree(srv_path->stats); + kfree(srv_path); } } static void rtrs_srv_close_work(struct work_struct *work) { - struct rtrs_srv_sess *sess; + struct rtrs_srv_path *srv_path; struct rtrs_srv_con *con; int i; - sess = container_of(work, typeof(*sess), close_work); + srv_path = container_of(work, typeof(*srv_path), close_work); - rtrs_srv_destroy_sess_files(sess); - rtrs_srv_stop_hb(sess); + rtrs_srv_destroy_path_files(srv_path); + rtrs_srv_stop_hb(srv_path); - for (i = 0; i < sess->s.con_num; i++) { - if (!sess->s.con[i]) + for (i = 0; i < srv_path->s.con_num; i++) { + if (!srv_path->s.con[i]) continue; - con = to_srv_con(sess->s.con[i]); + con = to_srv_con(srv_path->s.con[i]); rdma_disconnect(con->c.cm_id); ib_drain_qp(con->c.qp); } @@ -1533,41 +1541,41 @@ static void rtrs_srv_close_work(struct work_struct *work) * Degrade ref count to the usual model with a single shared * atomic_t counter */ - percpu_ref_kill(&sess->ids_inflight_ref); + percpu_ref_kill(&srv_path->ids_inflight_ref); /* Wait for all completion */ - wait_for_completion(&sess->complete_done); + wait_for_completion(&srv_path->complete_done); /* Notify upper layer if we are the last path */ - rtrs_srv_sess_down(sess); + rtrs_srv_path_down(srv_path); - unmap_cont_bufs(sess); - rtrs_srv_free_ops_ids(sess); + unmap_cont_bufs(srv_path); + rtrs_srv_free_ops_ids(srv_path); - for (i = 0; i < sess->s.con_num; i++) { - if (!sess->s.con[i]) + for (i = 0; i < srv_path->s.con_num; i++) { + if (!srv_path->s.con[i]) continue; - con = to_srv_con(sess->s.con[i]); + con = to_srv_con(srv_path->s.con[i]); rtrs_cq_qp_destroy(&con->c); rdma_destroy_id(con->c.cm_id); kfree(con); } - rtrs_ib_dev_put(sess->s.dev); + rtrs_ib_dev_put(srv_path->s.dev); - del_path_from_srv(sess); - put_srv(sess->srv); - sess->srv = NULL; - rtrs_srv_change_state(sess, RTRS_SRV_CLOSED); + del_path_from_srv(srv_path); + put_srv(srv_path->srv); + srv_path->srv = NULL; + rtrs_srv_change_state(srv_path, RTRS_SRV_CLOSED); - kfree(sess->dma_addr); - kfree(sess->s.con); - free_sess(sess); + kfree(srv_path->dma_addr); + kfree(srv_path->s.con); + free_path(srv_path); } -static int rtrs_rdma_do_accept(struct rtrs_srv_sess *sess, +static int rtrs_rdma_do_accept(struct rtrs_srv_path *srv_path, struct rdma_cm_id *cm_id) { - struct rtrs_srv *srv = sess->srv; + struct rtrs_srv_sess *srv = srv_path->srv; struct rtrs_msg_conn_rsp msg; struct rdma_conn_param param; int err; @@ -1615,25 +1623,25 @@ static int rtrs_rdma_do_reject(struct rdma_cm_id *cm_id, int errno) return errno; } -static struct rtrs_srv_sess * -__find_sess(struct rtrs_srv *srv, const uuid_t *sess_uuid) +static struct rtrs_srv_path * +__find_path(struct rtrs_srv_sess *srv, const uuid_t *sess_uuid) { - struct rtrs_srv_sess *sess; + struct rtrs_srv_path *srv_path; - list_for_each_entry(sess, &srv->paths_list, s.entry) { - if (uuid_equal(&sess->s.uuid, sess_uuid)) - return sess; + list_for_each_entry(srv_path, &srv->paths_list, s.entry) { + if (uuid_equal(&srv_path->s.uuid, sess_uuid)) + return srv_path; } return NULL; } -static int create_con(struct rtrs_srv_sess *sess, +static int create_con(struct rtrs_srv_path *srv_path, struct rdma_cm_id *cm_id, unsigned int cid) { - struct rtrs_srv *srv = sess->srv; - struct rtrs_sess *s = &sess->s; + struct rtrs_srv_sess *srv = srv_path->srv; + struct rtrs_path *s = &srv_path->s; struct rtrs_srv_con *con; u32 cq_num, max_send_wr, max_recv_wr, wr_limit; @@ -1648,10 +1656,10 @@ static int create_con(struct rtrs_srv_sess *sess, spin_lock_init(&con->rsp_wr_wait_lock); INIT_LIST_HEAD(&con->rsp_wr_wait_list); con->c.cm_id = cm_id; - con->c.sess = &sess->s; + con->c.path = &srv_path->s; con->c.cid = cid; atomic_set(&con->c.wr_cnt, 1); - wr_limit = sess->s.dev->ib_dev->attrs.max_qp_wr; + wr_limit = srv_path->s.dev->ib_dev->attrs.max_qp_wr; if (con->c.cid == 0) { /* @@ -1684,10 +1692,10 @@ static int create_con(struct rtrs_srv_sess *sess, } cq_num = max_send_wr + max_recv_wr; atomic_set(&con->c.sq_wr_avail, max_send_wr); - cq_vector = rtrs_srv_get_next_cq_vector(sess); + cq_vector = rtrs_srv_get_next_cq_vector(srv_path); /* TODO: SOFTIRQ can be faster, but be careful with softirq context */ - err = rtrs_cq_qp_create(&sess->s, &con->c, 1, cq_vector, cq_num, + err = rtrs_cq_qp_create(&srv_path->s, &con->c, 1, cq_vector, cq_num, max_send_wr, max_recv_wr, IB_POLL_WORKQUEUE); if (err) { @@ -1699,8 +1707,8 @@ static int create_con(struct rtrs_srv_sess *sess, if (err) goto free_cqqp; } - WARN_ON(sess->s.con[cid]); - sess->s.con[cid] = &con->c; + WARN_ON(srv_path->s.con[cid]); + srv_path->s.con[cid] = &con->c; /* * Change context from server to current connection. The other @@ -1719,13 +1727,13 @@ err: return err; } -static struct rtrs_srv_sess *__alloc_sess(struct rtrs_srv *srv, +static struct rtrs_srv_path *__alloc_path(struct rtrs_srv_sess *srv, struct rdma_cm_id *cm_id, unsigned int con_num, unsigned int recon_cnt, const uuid_t *uuid) { - struct rtrs_srv_sess *sess; + struct rtrs_srv_path *srv_path; int err = -ENOMEM; char str[NAME_MAX]; struct rtrs_addr path; @@ -1739,74 +1747,76 @@ static struct rtrs_srv_sess *__alloc_sess(struct rtrs_srv *srv, pr_err("Path with same addr exists\n"); goto err; } - sess = kzalloc(sizeof(*sess), GFP_KERNEL); - if (!sess) + srv_path = kzalloc(sizeof(*srv_path), GFP_KERNEL); + if (!srv_path) goto err; - sess->stats = kzalloc(sizeof(*sess->stats), GFP_KERNEL); - if (!sess->stats) + srv_path->stats = kzalloc(sizeof(*srv_path->stats), GFP_KERNEL); + if (!srv_path->stats) goto err_free_sess; - sess->stats->sess = sess; + srv_path->stats->srv_path = srv_path; - sess->dma_addr = kcalloc(srv->queue_depth, sizeof(*sess->dma_addr), - GFP_KERNEL); - if (!sess->dma_addr) + srv_path->dma_addr = kcalloc(srv->queue_depth, + sizeof(*srv_path->dma_addr), + GFP_KERNEL); + if (!srv_path->dma_addr) goto err_free_stats; - sess->s.con = kcalloc(con_num, sizeof(*sess->s.con), GFP_KERNEL); - if (!sess->s.con) + srv_path->s.con = kcalloc(con_num, sizeof(*srv_path->s.con), + GFP_KERNEL); + if (!srv_path->s.con) goto err_free_dma_addr; - sess->state = RTRS_SRV_CONNECTING; - sess->srv = srv; - sess->cur_cq_vector = -1; - sess->s.dst_addr = cm_id->route.addr.dst_addr; - sess->s.src_addr = cm_id->route.addr.src_addr; + srv_path->state = RTRS_SRV_CONNECTING; + srv_path->srv = srv; + srv_path->cur_cq_vector = -1; + srv_path->s.dst_addr = cm_id->route.addr.dst_addr; + srv_path->s.src_addr = cm_id->route.addr.src_addr; /* temporary until receiving session-name from client */ - path.src = &sess->s.src_addr; - path.dst = &sess->s.dst_addr; + path.src = &srv_path->s.src_addr; + path.dst = &srv_path->s.dst_addr; rtrs_addr_to_str(&path, str, sizeof(str)); - strscpy(sess->s.sessname, str, sizeof(sess->s.sessname)); + strscpy(srv_path->s.sessname, str, sizeof(srv_path->s.sessname)); - sess->s.con_num = con_num; - sess->s.irq_con_num = con_num; - sess->s.recon_cnt = recon_cnt; - uuid_copy(&sess->s.uuid, uuid); - spin_lock_init(&sess->state_lock); - INIT_WORK(&sess->close_work, rtrs_srv_close_work); - rtrs_srv_init_hb(sess); + srv_path->s.con_num = con_num; + srv_path->s.irq_con_num = con_num; + srv_path->s.recon_cnt = recon_cnt; + uuid_copy(&srv_path->s.uuid, uuid); + spin_lock_init(&srv_path->state_lock); + INIT_WORK(&srv_path->close_work, rtrs_srv_close_work); + rtrs_srv_init_hb(srv_path); - sess->s.dev = rtrs_ib_dev_find_or_add(cm_id->device, &dev_pd); - if (!sess->s.dev) { + srv_path->s.dev = rtrs_ib_dev_find_or_add(cm_id->device, &dev_pd); + if (!srv_path->s.dev) { err = -ENOMEM; goto err_free_con; } - err = map_cont_bufs(sess); + err = map_cont_bufs(srv_path); if (err) goto err_put_dev; - err = rtrs_srv_alloc_ops_ids(sess); + err = rtrs_srv_alloc_ops_ids(srv_path); if (err) goto err_unmap_bufs; - __add_path_to_srv(srv, sess); + __add_path_to_srv(srv, srv_path); - return sess; + return srv_path; err_unmap_bufs: - unmap_cont_bufs(sess); + unmap_cont_bufs(srv_path); err_put_dev: - rtrs_ib_dev_put(sess->s.dev); + rtrs_ib_dev_put(srv_path->s.dev); err_free_con: - kfree(sess->s.con); + kfree(srv_path->s.con); err_free_dma_addr: - kfree(sess->dma_addr); + kfree(srv_path->dma_addr); err_free_stats: - kfree(sess->stats); + kfree(srv_path->stats); err_free_sess: - kfree(sess); + kfree(srv_path); err: return ERR_PTR(err); } @@ -1816,8 +1826,8 @@ static int rtrs_rdma_connect(struct rdma_cm_id *cm_id, size_t len) { struct rtrs_srv_ctx *ctx = cm_id->context; - struct rtrs_srv_sess *sess; - struct rtrs_srv *srv; + struct rtrs_srv_path *srv_path; + struct rtrs_srv_sess *srv; u16 version, con_num, cid; u16 recon_cnt; @@ -1857,16 +1867,16 @@ static int rtrs_rdma_connect(struct rdma_cm_id *cm_id, goto reject_w_err; } mutex_lock(&srv->paths_mutex); - sess = __find_sess(srv, &msg->sess_uuid); - if (sess) { - struct rtrs_sess *s = &sess->s; + srv_path = __find_path(srv, &msg->sess_uuid); + if (srv_path) { + struct rtrs_path *s = &srv_path->s; /* Session already holds a reference */ put_srv(srv); - if (sess->state != RTRS_SRV_CONNECTING) { + if (srv_path->state != RTRS_SRV_CONNECTING) { rtrs_err(s, "Session in wrong state: %s\n", - rtrs_srv_state_str(sess->state)); + rtrs_srv_state_str(srv_path->state)); mutex_unlock(&srv->paths_mutex); goto reject_w_err; } @@ -1886,19 +1896,19 @@ static int rtrs_rdma_connect(struct rdma_cm_id *cm_id, goto reject_w_err; } } else { - sess = __alloc_sess(srv, cm_id, con_num, recon_cnt, + srv_path = __alloc_path(srv, cm_id, con_num, recon_cnt, &msg->sess_uuid); - if (IS_ERR(sess)) { + if (IS_ERR(srv_path)) { mutex_unlock(&srv->paths_mutex); put_srv(srv); - err = PTR_ERR(sess); + err = PTR_ERR(srv_path); pr_err("RTRS server session allocation failed: %d\n", err); goto reject_w_err; } } - err = create_con(sess, cm_id, cid); + err = create_con(srv_path, cm_id, cid); if (err) { - rtrs_err((&sess->s), "create_con(), error %d\n", err); + rtrs_err((&srv_path->s), "create_con(), error %d\n", err); rtrs_rdma_do_reject(cm_id, err); /* * Since session has other connections we follow normal way @@ -1907,9 +1917,9 @@ static int rtrs_rdma_connect(struct rdma_cm_id *cm_id, */ goto close_and_return_err; } - err = rtrs_rdma_do_accept(sess, cm_id); + err = rtrs_rdma_do_accept(srv_path, cm_id); if (err) { - rtrs_err((&sess->s), "rtrs_rdma_do_accept(), error %d\n", err); + rtrs_err((&srv_path->s), "rtrs_rdma_do_accept(), error %d\n", err); rtrs_rdma_do_reject(cm_id, err); /* * Since current connection was successfully added to the @@ -1929,7 +1939,7 @@ reject_w_err: close_and_return_err: mutex_unlock(&srv->paths_mutex); - close_sess(sess); + close_path(srv_path); return err; } @@ -1937,14 +1947,14 @@ close_and_return_err: static int rtrs_srv_rdma_cm_handler(struct rdma_cm_id *cm_id, struct rdma_cm_event *ev) { - struct rtrs_srv_sess *sess = NULL; - struct rtrs_sess *s = NULL; + struct rtrs_srv_path *srv_path = NULL; + struct rtrs_path *s = NULL; if (ev->event != RDMA_CM_EVENT_CONNECT_REQUEST) { struct rtrs_con *c = cm_id->context; - s = c->sess; - sess = to_srv_sess(s); + s = c->path; + srv_path = to_srv_path(s); } switch (ev->event) { @@ -1968,7 +1978,7 @@ static int rtrs_srv_rdma_cm_handler(struct rdma_cm_id *cm_id, case RDMA_CM_EVENT_ADDR_CHANGE: case RDMA_CM_EVENT_TIMEWAIT_EXIT: case RDMA_CM_EVENT_DEVICE_REMOVAL: - close_sess(sess); + close_path(srv_path); break; default: pr_err("Ignoring unexpected CM event %s, err %d\n", @@ -2176,23 +2186,23 @@ struct rtrs_srv_ctx *rtrs_srv_open(struct rtrs_srv_ops *ops, u16 port) } EXPORT_SYMBOL(rtrs_srv_open); -static void close_sessions(struct rtrs_srv *srv) +static void close_paths(struct rtrs_srv_sess *srv) { - struct rtrs_srv_sess *sess; + struct rtrs_srv_path *srv_path; mutex_lock(&srv->paths_mutex); - list_for_each_entry(sess, &srv->paths_list, s.entry) - close_sess(sess); + list_for_each_entry(srv_path, &srv->paths_list, s.entry) + close_path(srv_path); mutex_unlock(&srv->paths_mutex); } static void close_ctx(struct rtrs_srv_ctx *ctx) { - struct rtrs_srv *srv; + struct rtrs_srv_sess *srv; mutex_lock(&ctx->srv_mutex); list_for_each_entry(srv, &ctx->srv_list, ctx_list) - close_sessions(srv); + close_paths(srv); mutex_unlock(&ctx->srv_mutex); flush_workqueue(rtrs_wq); } diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.h b/drivers/infiniband/ulp/rtrs/rtrs-srv.h index 7d403c12faf3..6292e87f6afd 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-srv.h +++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.h @@ -37,7 +37,7 @@ struct rtrs_srv_stats_rdma_stats { struct rtrs_srv_stats { struct kobject kobj_stats; struct rtrs_srv_stats_rdma_stats rdma_stats; - struct rtrs_srv_sess *sess; + struct rtrs_srv_path *srv_path; }; struct rtrs_srv_con { @@ -71,9 +71,9 @@ struct rtrs_srv_mr { struct rtrs_iu *iu; /* send buffer for new rkey msg */ }; -struct rtrs_srv_sess { - struct rtrs_sess s; - struct rtrs_srv *srv; +struct rtrs_srv_path { + struct rtrs_path s; + struct rtrs_srv_sess *srv; struct work_struct close_work; enum rtrs_srv_state state; spinlock_t state_lock; @@ -90,7 +90,7 @@ struct rtrs_srv_sess { struct rtrs_srv_stats *stats; }; -struct rtrs_srv { +struct rtrs_srv_sess { struct list_head paths_list; int paths_up; struct mutex paths_ev_mutex; @@ -125,7 +125,7 @@ struct rtrs_srv_ib_ctx { extern struct class *rtrs_dev_class; -void close_sess(struct rtrs_srv_sess *sess); +void close_path(struct rtrs_srv_path *srv_path); static inline void rtrs_srv_update_rdma_stats(struct rtrs_srv_stats *s, size_t size, int d) @@ -142,7 +142,7 @@ ssize_t rtrs_srv_reset_all_help(struct rtrs_srv_stats *stats, char *page, size_t len); /* functions which are implemented in rtrs-srv-sysfs.c */ -int rtrs_srv_create_sess_files(struct rtrs_srv_sess *sess); -void rtrs_srv_destroy_sess_files(struct rtrs_srv_sess *sess); +int rtrs_srv_create_path_files(struct rtrs_srv_path *srv_path); +void rtrs_srv_destroy_path_files(struct rtrs_srv_path *srv_path); #endif /* RTRS_SRV_H */ diff --git a/drivers/infiniband/ulp/rtrs/rtrs.c b/drivers/infiniband/ulp/rtrs/rtrs.c index 37952c8e768c..4da889103a5f 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs.c +++ b/drivers/infiniband/ulp/rtrs/rtrs.c @@ -69,16 +69,16 @@ EXPORT_SYMBOL_GPL(rtrs_iu_free); int rtrs_iu_post_recv(struct rtrs_con *con, struct rtrs_iu *iu) { - struct rtrs_sess *sess = con->sess; + struct rtrs_path *path = con->path; struct ib_recv_wr wr; struct ib_sge list; list.addr = iu->dma_addr; list.length = iu->size; - list.lkey = sess->dev->ib_pd->local_dma_lkey; + list.lkey = path->dev->ib_pd->local_dma_lkey; if (list.length == 0) { - rtrs_wrn(con->sess, + rtrs_wrn(con->path, "Posting receive work request failed, sg list is empty\n"); return -EINVAL; } @@ -126,7 +126,7 @@ static int rtrs_post_send(struct ib_qp *qp, struct ib_send_wr *head, int rtrs_iu_post_send(struct rtrs_con *con, struct rtrs_iu *iu, size_t size, struct ib_send_wr *head) { - struct rtrs_sess *sess = con->sess; + struct rtrs_path *path = con->path; struct ib_send_wr wr; struct ib_sge list; @@ -135,7 +135,7 @@ int rtrs_iu_post_send(struct rtrs_con *con, struct rtrs_iu *iu, size_t size, list.addr = iu->dma_addr; list.length = size; - list.lkey = sess->dev->ib_pd->local_dma_lkey; + list.lkey = path->dev->ib_pd->local_dma_lkey; wr = (struct ib_send_wr) { .wr_cqe = &iu->cqe, @@ -188,11 +188,11 @@ static int rtrs_post_rdma_write_imm_empty(struct rtrs_con *con, struct ib_send_wr *head) { struct ib_rdma_wr wr; - struct rtrs_sess *sess = con->sess; + struct rtrs_path *path = con->path; enum ib_send_flags sflags; atomic_dec_if_positive(&con->sq_wr_avail); - sflags = (atomic_inc_return(&con->wr_cnt) % sess->signal_interval) ? + sflags = (atomic_inc_return(&con->wr_cnt) % path->signal_interval) ? 0 : IB_SEND_SIGNALED; wr = (struct ib_rdma_wr) { @@ -211,12 +211,12 @@ static void qp_event_handler(struct ib_event *ev, void *ctx) switch (ev->event) { case IB_EVENT_COMM_EST: - rtrs_info(con->sess, "QP event %s (%d) received\n", + rtrs_info(con->path, "QP event %s (%d) received\n", ib_event_msg(ev->event), ev->event); rdma_notify(con->cm_id, IB_EVENT_COMM_EST); break; default: - rtrs_info(con->sess, "Unhandled QP event %s (%d) received\n", + rtrs_info(con->path, "Unhandled QP event %s (%d) received\n", ib_event_msg(ev->event), ev->event); break; } @@ -224,7 +224,7 @@ static void qp_event_handler(struct ib_event *ev, void *ctx) static bool is_pollqueue(struct rtrs_con *con) { - return con->cid >= con->sess->irq_con_num; + return con->cid >= con->path->irq_con_num; } static int create_cq(struct rtrs_con *con, int cq_vector, int nr_cqe, @@ -240,7 +240,7 @@ static int create_cq(struct rtrs_con *con, int cq_vector, int nr_cqe, cq = ib_cq_pool_get(cm_id->device, nr_cqe, cq_vector, poll_ctx); if (IS_ERR(cq)) { - rtrs_err(con->sess, "Creating completion queue failed, errno: %ld\n", + rtrs_err(con->path, "Creating completion queue failed, errno: %ld\n", PTR_ERR(cq)); return PTR_ERR(cq); } @@ -271,7 +271,7 @@ static int create_qp(struct rtrs_con *con, struct ib_pd *pd, ret = rdma_create_qp(cm_id, pd, &init_attr); if (ret) { - rtrs_err(con->sess, "Creating QP failed, err: %d\n", ret); + rtrs_err(con->path, "Creating QP failed, err: %d\n", ret); return ret; } con->qp = cm_id->qp; @@ -290,7 +290,7 @@ static void destroy_cq(struct rtrs_con *con) con->cq = NULL; } -int rtrs_cq_qp_create(struct rtrs_sess *sess, struct rtrs_con *con, +int rtrs_cq_qp_create(struct rtrs_path *path, struct rtrs_con *con, u32 max_send_sge, int cq_vector, int nr_cqe, u32 max_send_wr, u32 max_recv_wr, enum ib_poll_context poll_ctx) @@ -301,13 +301,13 @@ int rtrs_cq_qp_create(struct rtrs_sess *sess, struct rtrs_con *con, if (err) return err; - err = create_qp(con, sess->dev->ib_pd, max_send_wr, max_recv_wr, + err = create_qp(con, path->dev->ib_pd, max_send_wr, max_recv_wr, max_send_sge); if (err) { destroy_cq(con); return err; } - con->sess = sess; + con->path = path; return 0; } @@ -323,24 +323,24 @@ void rtrs_cq_qp_destroy(struct rtrs_con *con) } EXPORT_SYMBOL_GPL(rtrs_cq_qp_destroy); -static void schedule_hb(struct rtrs_sess *sess) +static void schedule_hb(struct rtrs_path *path) { - queue_delayed_work(sess->hb_wq, &sess->hb_dwork, - msecs_to_jiffies(sess->hb_interval_ms)); + queue_delayed_work(path->hb_wq, &path->hb_dwork, + msecs_to_jiffies(path->hb_interval_ms)); } -void rtrs_send_hb_ack(struct rtrs_sess *sess) +void rtrs_send_hb_ack(struct rtrs_path *path) { - struct rtrs_con *usr_con = sess->con[0]; + struct rtrs_con *usr_con = path->con[0]; u32 imm; int err; imm = rtrs_to_imm(RTRS_HB_ACK_IMM, 0); - err = rtrs_post_rdma_write_imm_empty(usr_con, sess->hb_cqe, imm, + err = rtrs_post_rdma_write_imm_empty(usr_con, path->hb_cqe, imm, NULL); if (err) { - rtrs_err(sess, "send HB ACK failed, errno: %d\n", err); - sess->hb_err_handler(usr_con); + rtrs_err(path, "send HB ACK failed, errno: %d\n", err); + path->hb_err_handler(usr_con); return; } } @@ -349,63 +349,63 @@ EXPORT_SYMBOL_GPL(rtrs_send_hb_ack); static void hb_work(struct work_struct *work) { struct rtrs_con *usr_con; - struct rtrs_sess *sess; + struct rtrs_path *path; u32 imm; int err; - sess = container_of(to_delayed_work(work), typeof(*sess), hb_dwork); - usr_con = sess->con[0]; + path = container_of(to_delayed_work(work), typeof(*path), hb_dwork); + usr_con = path->con[0]; - if (sess->hb_missed_cnt > sess->hb_missed_max) { - rtrs_err(sess, "HB missed max reached.\n"); - sess->hb_err_handler(usr_con); + if (path->hb_missed_cnt > path->hb_missed_max) { + rtrs_err(path, "HB missed max reached.\n"); + path->hb_err_handler(usr_con); return; } - if (sess->hb_missed_cnt++) { + if (path->hb_missed_cnt++) { /* Reschedule work without sending hb */ - schedule_hb(sess); + schedule_hb(path); return; } - sess->hb_last_sent = ktime_get(); + path->hb_last_sent = ktime_get(); imm = rtrs_to_imm(RTRS_HB_MSG_IMM, 0); - err = rtrs_post_rdma_write_imm_empty(usr_con, sess->hb_cqe, imm, + err = rtrs_post_rdma_write_imm_empty(usr_con, path->hb_cqe, imm, NULL); if (err) { - rtrs_err(sess, "HB send failed, errno: %d\n", err); - sess->hb_err_handler(usr_con); + rtrs_err(path, "HB send failed, errno: %d\n", err); + path->hb_err_handler(usr_con); return; } - schedule_hb(sess); + schedule_hb(path); } -void rtrs_init_hb(struct rtrs_sess *sess, struct ib_cqe *cqe, +void rtrs_init_hb(struct rtrs_path *path, struct ib_cqe *cqe, unsigned int interval_ms, unsigned int missed_max, void (*err_handler)(struct rtrs_con *con), struct workqueue_struct *wq) { - sess->hb_cqe = cqe; - sess->hb_interval_ms = interval_ms; - sess->hb_err_handler = err_handler; - sess->hb_wq = wq; - sess->hb_missed_max = missed_max; - sess->hb_missed_cnt = 0; - INIT_DELAYED_WORK(&sess->hb_dwork, hb_work); + path->hb_cqe = cqe; + path->hb_interval_ms = interval_ms; + path->hb_err_handler = err_handler; + path->hb_wq = wq; + path->hb_missed_max = missed_max; + path->hb_missed_cnt = 0; + INIT_DELAYED_WORK(&path->hb_dwork, hb_work); } EXPORT_SYMBOL_GPL(rtrs_init_hb); -void rtrs_start_hb(struct rtrs_sess *sess) +void rtrs_start_hb(struct rtrs_path *path) { - schedule_hb(sess); + schedule_hb(path); } EXPORT_SYMBOL_GPL(rtrs_start_hb); -void rtrs_stop_hb(struct rtrs_sess *sess) +void rtrs_stop_hb(struct rtrs_path *path) { - cancel_delayed_work_sync(&sess->hb_dwork); - sess->hb_missed_cnt = 0; + cancel_delayed_work_sync(&path->hb_dwork); + path->hb_missed_cnt = 0; } EXPORT_SYMBOL_GPL(rtrs_stop_hb); diff --git a/drivers/infiniband/ulp/rtrs/rtrs.h b/drivers/infiniband/ulp/rtrs/rtrs.h index 859c79685daf..5e57a7ccc7fb 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs.h +++ b/drivers/infiniband/ulp/rtrs/rtrs.h @@ -13,9 +13,9 @@ #include struct rtrs_permit; -struct rtrs_clt; +struct rtrs_clt_sess; struct rtrs_srv_ctx; -struct rtrs_srv; +struct rtrs_srv_sess; struct rtrs_srv_op; /* @@ -52,14 +52,14 @@ struct rtrs_clt_ops { void (*link_ev)(void *priv, enum rtrs_clt_link_ev ev); }; -struct rtrs_clt *rtrs_clt_open(struct rtrs_clt_ops *ops, - const char *sessname, +struct rtrs_clt_sess *rtrs_clt_open(struct rtrs_clt_ops *ops, + const char *pathname, const struct rtrs_addr *paths, size_t path_cnt, u16 port, size_t pdu_sz, u8 reconnect_delay_sec, s16 max_reconnect_attempts, u32 nr_poll_queues); -void rtrs_clt_close(struct rtrs_clt *sess); +void rtrs_clt_close(struct rtrs_clt_sess *clt); enum wait_type { RTRS_PERMIT_NOWAIT = 0, @@ -77,11 +77,12 @@ enum rtrs_clt_con_type { RTRS_IO_CON }; -struct rtrs_permit *rtrs_clt_get_permit(struct rtrs_clt *sess, - enum rtrs_clt_con_type con_type, - enum wait_type wait); +struct rtrs_permit *rtrs_clt_get_permit(struct rtrs_clt_sess *sess, + enum rtrs_clt_con_type con_type, + enum wait_type wait); -void rtrs_clt_put_permit(struct rtrs_clt *sess, struct rtrs_permit *permit); +void rtrs_clt_put_permit(struct rtrs_clt_sess *sess, + struct rtrs_permit *permit); /** * rtrs_clt_req_ops - it holds the request confirmation callback @@ -98,10 +99,10 @@ struct rtrs_clt_req_ops { }; int rtrs_clt_request(int dir, struct rtrs_clt_req_ops *ops, - struct rtrs_clt *sess, struct rtrs_permit *permit, + struct rtrs_clt_sess *sess, struct rtrs_permit *permit, const struct kvec *vec, size_t nr, size_t len, struct scatterlist *sg, unsigned int sg_cnt); -int rtrs_clt_rdma_cq_direct(struct rtrs_clt *clt, unsigned int index); +int rtrs_clt_rdma_cq_direct(struct rtrs_clt_sess *clt, unsigned int index); /** * rtrs_attrs - RTRS session attributes @@ -112,7 +113,7 @@ struct rtrs_attrs { u32 max_segments; }; -int rtrs_clt_query(struct rtrs_clt *sess, struct rtrs_attrs *attr); +int rtrs_clt_query(struct rtrs_clt_sess *sess, struct rtrs_attrs *attr); /* * Here goes RTRS server API @@ -163,7 +164,7 @@ struct rtrs_srv_ops { * @priv: Private data from user if previously set with * rtrs_srv_set_sess_priv() */ - int (*link_ev)(struct rtrs_srv *sess, enum rtrs_srv_link_ev ev, + int (*link_ev)(struct rtrs_srv_sess *sess, enum rtrs_srv_link_ev ev, void *priv); }; @@ -173,11 +174,12 @@ void rtrs_srv_close(struct rtrs_srv_ctx *ctx); bool rtrs_srv_resp_rdma(struct rtrs_srv_op *id, int errno); -void rtrs_srv_set_sess_priv(struct rtrs_srv *sess, void *priv); +void rtrs_srv_set_sess_priv(struct rtrs_srv_sess *sess, void *priv); -int rtrs_srv_get_sess_name(struct rtrs_srv *sess, char *sessname, size_t len); +int rtrs_srv_get_path_name(struct rtrs_srv_sess *sess, char *pathname, + size_t len); -int rtrs_srv_get_queue_depth(struct rtrs_srv *sess); +int rtrs_srv_get_queue_depth(struct rtrs_srv_sess *sess); int rtrs_addr_to_sockaddr(const char *str, size_t len, u16 port, struct rtrs_addr *addr); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index f5848b351b19..6dc6d8b6b368 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -3142,7 +3142,7 @@ static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg) phys_addr_t doorbell; struct device *dev = msi_desc_to_dev(desc); struct arm_smmu_device *smmu = dev_get_drvdata(dev); - phys_addr_t *cfg = arm_smmu_msi_cfg[desc->platform.msi_index]; + phys_addr_t *cfg = arm_smmu_msi_cfg[desc->msi_index]; doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo; doorbell &= MSI_CFG0_ADDR_MASK; @@ -3154,7 +3154,6 @@ static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg) static void arm_smmu_setup_msis(struct arm_smmu_device *smmu) { - struct msi_desc *desc; int ret, nvec = ARM_SMMU_MAX_MSIS; struct device *dev = smmu->dev; @@ -3170,7 +3169,7 @@ static void arm_smmu_setup_msis(struct arm_smmu_device *smmu) if (!(smmu->features & ARM_SMMU_FEAT_MSI)) return; - if (!dev->msi_domain) { + if (!dev->msi.domain) { dev_info(smmu->dev, "msi_domain absent - falling back to wired irqs\n"); return; } @@ -3182,21 +3181,9 @@ static void arm_smmu_setup_msis(struct arm_smmu_device *smmu) return; } - for_each_msi_entry(desc, dev) { - switch (desc->platform.msi_index) { - case EVTQ_MSI_INDEX: - smmu->evtq.q.irq = desc->irq; - break; - case GERROR_MSI_INDEX: - smmu->gerr_irq = desc->irq; - break; - case PRIQ_MSI_INDEX: - smmu->priq.q.irq = desc->irq; - break; - default: /* Unknown */ - continue; - } - } + smmu->evtq.q.irq = msi_get_virq(dev, EVTQ_MSI_INDEX); + smmu->gerr_irq = msi_get_virq(dev, GERROR_MSI_INDEX); + smmu->priq.q.irq = msi_get_virq(dev, PRIQ_MSI_INDEX); /* Add callback to free MSIs on teardown */ devm_add_action(dev, arm_smmu_free_msis, dev); diff --git a/drivers/irqchip/irq-gic-v2m.c b/drivers/irqchip/irq-gic-v2m.c index 9349fc68b81a..b249d4df899e 100644 --- a/drivers/irqchip/irq-gic-v2m.c +++ b/drivers/irqchip/irq-gic-v2m.c @@ -88,7 +88,6 @@ static struct irq_chip gicv2m_msi_irq_chip = { .irq_mask = gicv2m_mask_msi_irq, .irq_unmask = gicv2m_unmask_msi_irq, .irq_eoi = irq_chip_eoi_parent, - .irq_write_msi_msg = pci_msi_domain_write_msg, }; static struct msi_domain_info gicv2m_msi_domain_info = { @@ -405,7 +404,7 @@ err_free_v2m: return ret; } -static struct of_device_id gicv2m_device_id[] = { +static const struct of_device_id gicv2m_device_id[] = { { .compatible = "arm,gic-v2m-frame", }, {}, }; diff --git a/drivers/irqchip/irq-gic-v3-its-pci-msi.c b/drivers/irqchip/irq-gic-v3-its-pci-msi.c index ad2810c017ed..93f77a8196da 100644 --- a/drivers/irqchip/irq-gic-v3-its-pci-msi.c +++ b/drivers/irqchip/irq-gic-v3-its-pci-msi.c @@ -28,7 +28,6 @@ static struct irq_chip its_msi_irq_chip = { .irq_unmask = its_unmask_msi_irq, .irq_mask = its_mask_msi_irq, .irq_eoi = irq_chip_eoi_parent, - .irq_write_msi_msg = pci_msi_domain_write_msg, }; static int its_pci_msi_vec_count(struct pci_dev *pdev, void *data) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 0cb584d9815b..d25b7a864bbb 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -46,6 +46,10 @@ #define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING (1 << 0) #define RDIST_FLAGS_RD_TABLES_PREALLOCATED (1 << 1) +#define RD_LOCAL_LPI_ENABLED BIT(0) +#define RD_LOCAL_PENDTABLE_PREALLOCATED BIT(1) +#define RD_LOCAL_MEMRESERVE_DONE BIT(2) + static u32 lpi_id_bits; /* @@ -3044,7 +3048,7 @@ static void its_cpu_init_lpis(void) phys_addr_t paddr; u64 val, tmp; - if (gic_data_rdist()->lpi_enabled) + if (gic_data_rdist()->flags & RD_LOCAL_LPI_ENABLED) return; val = readl_relaxed(rbase + GICR_CTLR); @@ -3063,15 +3067,13 @@ static void its_cpu_init_lpis(void) paddr &= GENMASK_ULL(51, 16); WARN_ON(!gic_check_reserved_range(paddr, LPI_PENDBASE_SZ)); - its_free_pending_table(gic_data_rdist()->pend_page); - gic_data_rdist()->pend_page = NULL; + gic_data_rdist()->flags |= RD_LOCAL_PENDTABLE_PREALLOCATED; goto out; } pend_page = gic_data_rdist()->pend_page; paddr = page_to_phys(pend_page); - WARN_ON(gic_reserve_range(paddr, LPI_PENDBASE_SZ)); /* set PROPBASE */ val = (gic_rdists->prop_table_pa | @@ -3158,10 +3160,11 @@ static void its_cpu_init_lpis(void) /* Make sure the GIC has seen the above */ dsb(sy); out: - gic_data_rdist()->lpi_enabled = true; + gic_data_rdist()->flags |= RD_LOCAL_LPI_ENABLED; pr_info("GICv3: CPU%d: using %s LPI pending table @%pa\n", smp_processor_id(), - gic_data_rdist()->pend_page ? "allocated" : "reserved", + gic_data_rdist()->flags & RD_LOCAL_PENDTABLE_PREALLOCATED ? + "reserved" : "allocated", &paddr); } @@ -5138,7 +5141,7 @@ static int redist_disable_lpis(void) * * If running with preallocated tables, there is nothing to do. */ - if (gic_data_rdist()->lpi_enabled || + if ((gic_data_rdist()->flags & RD_LOCAL_LPI_ENABLED) || (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED)) return 0; @@ -5200,6 +5203,51 @@ int its_cpu_init(void) return 0; } +static void rdist_memreserve_cpuhp_cleanup_workfn(struct work_struct *work) +{ + cpuhp_remove_state_nocalls(gic_rdists->cpuhp_memreserve_state); + gic_rdists->cpuhp_memreserve_state = CPUHP_INVALID; +} + +static DECLARE_WORK(rdist_memreserve_cpuhp_cleanup_work, + rdist_memreserve_cpuhp_cleanup_workfn); + +static int its_cpu_memreserve_lpi(unsigned int cpu) +{ + struct page *pend_page; + int ret = 0; + + /* This gets to run exactly once per CPU */ + if (gic_data_rdist()->flags & RD_LOCAL_MEMRESERVE_DONE) + return 0; + + pend_page = gic_data_rdist()->pend_page; + if (WARN_ON(!pend_page)) { + ret = -ENOMEM; + goto out; + } + /* + * If the pending table was pre-programmed, free the memory we + * preemptively allocated. Otherwise, reserve that memory for + * later kexecs. + */ + if (gic_data_rdist()->flags & RD_LOCAL_PENDTABLE_PREALLOCATED) { + its_free_pending_table(pend_page); + gic_data_rdist()->pend_page = NULL; + } else { + phys_addr_t paddr = page_to_phys(pend_page); + WARN_ON(gic_reserve_range(paddr, LPI_PENDBASE_SZ)); + } + +out: + /* Last CPU being brought up gets to issue the cleanup */ + if (cpumask_equal(&cpus_booted_once_mask, cpu_possible_mask)) + schedule_work(&rdist_memreserve_cpuhp_cleanup_work); + + gic_data_rdist()->flags |= RD_LOCAL_MEMRESERVE_DONE; + return ret; +} + static const struct of_device_id its_device_id[] = { { .compatible = "arm,gic-v3-its", }, {}, @@ -5383,6 +5431,26 @@ static void __init its_acpi_probe(void) static void __init its_acpi_probe(void) { } #endif +int __init its_lpi_memreserve_init(void) +{ + int state; + + if (!efi_enabled(EFI_CONFIG_TABLES)) + return 0; + + gic_rdists->cpuhp_memreserve_state = CPUHP_INVALID; + state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, + "irqchip/arm/gicv3/memreserve:online", + its_cpu_memreserve_lpi, + NULL); + if (state < 0) + return state; + + gic_rdists->cpuhp_memreserve_state = state; + + return 0; +} + int __init its_init(struct fwnode_handle *handle, struct rdists *rdists, struct irq_domain *parent_domain) { diff --git a/drivers/irqchip/irq-gic-v3-mbi.c b/drivers/irqchip/irq-gic-v3-mbi.c index b84c9c2eccdc..a2163d32f17d 100644 --- a/drivers/irqchip/irq-gic-v3-mbi.c +++ b/drivers/irqchip/irq-gic-v3-mbi.c @@ -171,7 +171,6 @@ static struct irq_chip mbi_msi_irq_chip = { .irq_unmask = mbi_unmask_msi_irq, .irq_eoi = irq_chip_eoi_parent, .irq_compose_msi_msg = mbi_compose_msi_msg, - .irq_write_msi_msg = pci_msi_domain_write_msg, }; static struct msi_domain_info mbi_msi_domain_info = { diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index ba8f819089de..c34c0278a2f8 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -931,6 +931,22 @@ static int __gic_update_rdist_properties(struct redist_region *region, { u64 typer = gic_read_typer(ptr + GICR_TYPER); + /* Boot-time cleanip */ + if ((typer & GICR_TYPER_VLPIS) && (typer & GICR_TYPER_RVPEID)) { + u64 val; + + /* Deactivate any present vPE */ + val = gicr_read_vpendbaser(ptr + SZ_128K + GICR_VPENDBASER); + if (val & GICR_VPENDBASER_Valid) + gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast, + ptr + SZ_128K + GICR_VPENDBASER); + + /* Mark the VPE table as invalid */ + val = gicr_read_vpropbaser(ptr + SZ_128K + GICR_VPROPBASER); + val &= ~GICR_VPROPBASER_4_1_VALID; + gicr_write_vpropbaser(val, ptr + SZ_128K + GICR_VPROPBASER); + } + gic_data.rdists.has_vlpis &= !!(typer & GICR_TYPER_VLPIS); /* RVPEID implies some form of DirectLPI, no matter what the doc says... :-/ */ @@ -1837,6 +1853,7 @@ static int __init gic_init_bases(void __iomem *dist_base, if (gic_dist_supports_lpis()) { its_init(handle, &gic_data.rdists, gic_data.domain); its_cpu_init(); + its_lpi_memreserve_init(); } else { if (IS_ENABLED(CONFIG_ARM_GIC_V2M)) gicv2m_init(handle, gic_data.domain); diff --git a/drivers/irqchip/irq-imx-gpcv2.c b/drivers/irqchip/irq-imx-gpcv2.c index 5b5a365dbd5e..b9c22f764b4d 100644 --- a/drivers/irqchip/irq-imx-gpcv2.c +++ b/drivers/irqchip/irq-imx-gpcv2.c @@ -26,7 +26,7 @@ struct gpcv2_irqchip_data { u32 cpu2wakeup; }; -static struct gpcv2_irqchip_data *imx_gpcv2_instance; +static struct gpcv2_irqchip_data *imx_gpcv2_instance __ro_after_init; static void __iomem *gpcv2_idx_to_reg(struct gpcv2_irqchip_data *cd, int i) { diff --git a/drivers/irqchip/irq-ingenic-tcu.c b/drivers/irqchip/irq-ingenic-tcu.c index 34a7d261b710..3363f83bd7e9 100644 --- a/drivers/irqchip/irq-ingenic-tcu.c +++ b/drivers/irqchip/irq-ingenic-tcu.c @@ -28,6 +28,7 @@ static void ingenic_tcu_intc_cascade(struct irq_desc *desc) struct irq_chip_generic *gc = irq_get_domain_generic_chip(domain, 0); struct regmap *map = gc->private; uint32_t irq_reg, irq_mask; + unsigned long bits; unsigned int i; regmap_read(map, TCU_REG_TFR, &irq_reg); @@ -36,8 +37,9 @@ static void ingenic_tcu_intc_cascade(struct irq_desc *desc) chained_irq_enter(irq_chip, desc); irq_reg &= ~irq_mask; + bits = irq_reg; - for_each_set_bit(i, (unsigned long *)&irq_reg, 32) + for_each_set_bit(i, &bits, 32) generic_handle_domain_irq(domain, i); chained_irq_exit(irq_chip, desc); diff --git a/drivers/irqchip/irq-mbigen.c b/drivers/irqchip/irq-mbigen.c index 12df2162108e..f3faf5c99770 100644 --- a/drivers/irqchip/irq-mbigen.c +++ b/drivers/irqchip/irq-mbigen.c @@ -207,7 +207,7 @@ static int mbigen_irq_domain_alloc(struct irq_domain *domain, if (err) return err; - err = platform_msi_domain_alloc(domain, virq, nr_irqs); + err = platform_msi_device_domain_alloc(domain, virq, nr_irqs); if (err) return err; @@ -223,7 +223,7 @@ static int mbigen_irq_domain_alloc(struct irq_domain *domain, static void mbigen_irq_domain_free(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs) { - platform_msi_domain_free(domain, virq, nr_irqs); + platform_msi_device_domain_free(domain, virq, nr_irqs); } static const struct irq_domain_ops mbigen_domain_ops = { diff --git a/drivers/irqchip/irq-mvebu-icu.c b/drivers/irqchip/irq-mvebu-icu.c index 3e7297fc5948..497da344717c 100644 --- a/drivers/irqchip/irq-mvebu-icu.c +++ b/drivers/irqchip/irq-mvebu-icu.c @@ -221,7 +221,7 @@ mvebu_icu_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, icu_irqd->icu_group = msi_data->subset_data->icu_group; icu_irqd->icu = icu; - err = platform_msi_domain_alloc(domain, virq, nr_irqs); + err = platform_msi_device_domain_alloc(domain, virq, nr_irqs); if (err) { dev_err(icu->dev, "failed to allocate ICU interrupt in parent domain\n"); goto free_irqd; @@ -245,7 +245,7 @@ mvebu_icu_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, return 0; free_msi: - platform_msi_domain_free(domain, virq, nr_irqs); + platform_msi_device_domain_free(domain, virq, nr_irqs); free_irqd: kfree(icu_irqd); return err; @@ -260,7 +260,7 @@ mvebu_icu_irq_domain_free(struct irq_domain *domain, unsigned int virq, kfree(icu_irqd); - platform_msi_domain_free(domain, virq, nr_irqs); + platform_msi_device_domain_free(domain, virq, nr_irqs); } static const struct irq_domain_ops mvebu_icu_domain_ops = { @@ -314,12 +314,12 @@ static int mvebu_icu_subset_probe(struct platform_device *pdev) msi_data->subset_data = of_device_get_match_data(dev); } - dev->msi_domain = of_msi_get_domain(dev, dev->of_node, + dev->msi.domain = of_msi_get_domain(dev, dev->of_node, DOMAIN_BUS_PLATFORM_MSI); - if (!dev->msi_domain) + if (!dev->msi.domain) return -EPROBE_DEFER; - msi_parent_dn = irq_domain_get_of_node(dev->msi_domain); + msi_parent_dn = irq_domain_get_of_node(dev->msi.domain); if (!msi_parent_dn) return -ENODEV; diff --git a/drivers/irqchip/irq-renesas-intc-irqpin.c b/drivers/irqchip/irq-renesas-intc-irqpin.c index cb7f60b3b4a9..37f9a4499fdb 100644 --- a/drivers/irqchip/irq-renesas-intc-irqpin.c +++ b/drivers/irqchip/irq-renesas-intc-irqpin.c @@ -375,7 +375,6 @@ static int intc_irqpin_probe(struct platform_device *pdev) struct intc_irqpin_priv *p; struct intc_irqpin_iomem *i; struct resource *io[INTC_IRQPIN_REG_NR]; - struct resource *irq; struct irq_chip *irq_chip; void (*enable_fn)(struct irq_data *d); void (*disable_fn)(struct irq_data *d); @@ -418,12 +417,14 @@ static int intc_irqpin_probe(struct platform_device *pdev) /* allow any number of IRQs between 1 and INTC_IRQPIN_MAX */ for (k = 0; k < INTC_IRQPIN_MAX; k++) { - irq = platform_get_resource(pdev, IORESOURCE_IRQ, k); - if (!irq) + ret = platform_get_irq_optional(pdev, k); + if (ret == -ENXIO) break; + if (ret < 0) + goto err0; p->irq[k].p = p; - p->irq[k].requested_irq = irq->start; + p->irq[k].requested_irq = ret; } nirqs = k; diff --git a/drivers/irqchip/irq-renesas-irqc.c b/drivers/irqchip/irq-renesas-irqc.c index 07a6d8b42b63..909325f88239 100644 --- a/drivers/irqchip/irq-renesas-irqc.c +++ b/drivers/irqchip/irq-renesas-irqc.c @@ -126,7 +126,6 @@ static int irqc_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; const char *name = dev_name(dev); struct irqc_priv *p; - struct resource *irq; int ret; int k; @@ -142,13 +141,15 @@ static int irqc_probe(struct platform_device *pdev) /* allow any number of IRQs between 1 and IRQC_IRQ_MAX */ for (k = 0; k < IRQC_IRQ_MAX; k++) { - irq = platform_get_resource(pdev, IORESOURCE_IRQ, k); - if (!irq) + ret = platform_get_irq_optional(pdev, k); + if (ret == -ENXIO) break; + if (ret < 0) + goto err_runtime_pm_disable; p->irq[k].p = p; p->irq[k].hw_irq = k; - p->irq[k].requested_irq = irq->start; + p->irq[k].requested_irq = ret; } p->number_of_irqs = k; diff --git a/drivers/irqchip/irq-ti-sci-inta.c b/drivers/irqchip/irq-ti-sci-inta.c index 8eba08db33b2..5fdbb4358dd0 100644 --- a/drivers/irqchip/irq-ti-sci-inta.c +++ b/drivers/irqchip/irq-ti-sci-inta.c @@ -595,7 +595,7 @@ static void ti_sci_inta_msi_set_desc(msi_alloc_info_t *arg, struct platform_device *pdev = to_platform_device(desc->dev); arg->desc = desc; - arg->hwirq = TO_HWIRQ(pdev->id, desc->inta.dev_index); + arg->hwirq = TO_HWIRQ(pdev->id, desc->msi_index); } static struct msi_domain_ops ti_sci_inta_msi_ops = { diff --git a/drivers/irqchip/spear-shirq.c b/drivers/irqchip/spear-shirq.c index 1518ba31a80c..7c17a6f643ef 100644 --- a/drivers/irqchip/spear-shirq.c +++ b/drivers/irqchip/spear-shirq.c @@ -149,6 +149,8 @@ static struct spear_shirq spear320_shirq_ras3 = { .offset = 0, .nr_irqs = 7, .mask = ((0x1 << 7) - 1) << 0, + .irq_chip = &dummy_irq_chip, + .status_reg = SPEAR320_INT_STS_MASK_REG, }; static struct spear_shirq spear320_shirq_ras1 = { diff --git a/drivers/leds/Kconfig b/drivers/leds/Kconfig index ac6688d7a3f4..6090e647daee 100644 --- a/drivers/leds/Kconfig +++ b/drivers/leds/Kconfig @@ -260,13 +260,6 @@ config LEDS_NET48XX This option enables support for the Soekris net4801 and net4826 error LED. -config LEDS_FSG - tristate "LED Support for the Freecom FSG-3" - depends on LEDS_CLASS - depends on MACH_FSG - help - This option enables support for the LEDs on the Freecom FSG-3. - config LEDS_WRAP tristate "LED Support for the WRAP series LEDs" depends on LEDS_CLASS diff --git a/drivers/leds/Makefile b/drivers/leds/Makefile index 1a719caf14c0..e58ecb36360f 100644 --- a/drivers/leds/Makefile +++ b/drivers/leds/Makefile @@ -26,7 +26,6 @@ obj-$(CONFIG_LEDS_COBALT_RAQ) += leds-cobalt-raq.o obj-$(CONFIG_LEDS_CPCAP) += leds-cpcap.o obj-$(CONFIG_LEDS_DA903X) += leds-da903x.o obj-$(CONFIG_LEDS_DA9052) += leds-da9052.o -obj-$(CONFIG_LEDS_FSG) += leds-fsg.o obj-$(CONFIG_LEDS_GPIO) += leds-gpio.o obj-$(CONFIG_LEDS_GPIO_REGISTER) += leds-gpio-register.o obj-$(CONFIG_LEDS_HP6XX) += leds-hp6xx.o diff --git a/drivers/leds/blink/leds-lgm-sso.c b/drivers/leds/blink/leds-lgm-sso.c index fd8b7573285a..6f270c0272fb 100644 --- a/drivers/leds/blink/leds-lgm-sso.c +++ b/drivers/leds/blink/leds-lgm-sso.c @@ -477,7 +477,6 @@ static int sso_gpio_gc_init(struct device *dev, struct sso_led_priv *priv) gc->ngpio = priv->gpio.pins; gc->parent = dev; gc->owner = THIS_MODULE; - gc->of_node = dev->of_node; return devm_gpiochip_add_data(dev, gc, priv); } diff --git a/drivers/leds/flash/Kconfig b/drivers/leds/flash/Kconfig index b230f3d65eb0..d3eb689b193c 100644 --- a/drivers/leds/flash/Kconfig +++ b/drivers/leds/flash/Kconfig @@ -48,6 +48,19 @@ config LEDS_MAX77693 multifunction device. It has build in control for two leds in flash and torch mode. +config LEDS_MT6360 + tristate "LED Support for Mediatek MT6360 PMIC" + depends on LEDS_CLASS && OF + depends on LEDS_CLASS_FLASH || !LEDS_CLASS_FLASH + depends on LEDS_CLASS_MULTICOLOR || !LEDS_CLASS_MULTICOLOR + depends on V4L2_FLASH_LED_CLASS || !V4L2_FLASH_LED_CLASS + depends on MFD_MT6360 + help + This option enables support for dual Flash LED drivers found on + Mediatek MT6360 PMIC. + Independent current sources supply for each flash LED support torch + and strobe mode. + config LEDS_RT4505 tristate "LED support for RT4505 flashlight controller" depends on I2C && OF diff --git a/drivers/leds/flash/Makefile b/drivers/leds/flash/Makefile index ebea42f9c37e..0acbddc0b91b 100644 --- a/drivers/leds/flash/Makefile +++ b/drivers/leds/flash/Makefile @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_LEDS_MT6360) += leds-mt6360.o obj-$(CONFIG_LEDS_AAT1290) += leds-aat1290.o obj-$(CONFIG_LEDS_AS3645A) += leds-as3645a.o obj-$(CONFIG_LEDS_KTD2692) += leds-ktd2692.o diff --git a/drivers/leds/flash/leds-ktd2692.c b/drivers/leds/flash/leds-ktd2692.c index f341da1503a4..ed1f20a58bf6 100644 --- a/drivers/leds/flash/leds-ktd2692.c +++ b/drivers/leds/flash/leds-ktd2692.c @@ -274,7 +274,7 @@ static int ktd2692_parse_dt(struct ktd2692_context *led, struct device *dev, struct device_node *child_node; int ret; - if (!dev_of_node(dev)) + if (!np) return -ENXIO; led->ctrl_gpio = devm_gpiod_get(dev, "ctrl", GPIOD_ASIS); diff --git a/drivers/leds/flash/leds-mt6360.c b/drivers/leds/flash/leds-mt6360.c new file mode 100644 index 000000000000..e1066a52d2d2 --- /dev/null +++ b/drivers/leds/flash/leds-mt6360.c @@ -0,0 +1,910 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum { + MT6360_LED_ISNK1 = 0, + MT6360_LED_ISNK2, + MT6360_LED_ISNK3, + MT6360_LED_ISNKML, + MT6360_LED_FLASH1, + MT6360_LED_FLASH2, + MT6360_MAX_LEDS +}; + +#define MT6360_REG_RGBEN 0x380 +#define MT6360_REG_ISNK(_led_no) (0x381 + (_led_no)) +#define MT6360_ISNK_ENMASK(_led_no) BIT(7 - (_led_no)) +#define MT6360_ISNK_MASK GENMASK(4, 0) +#define MT6360_CHRINDSEL_MASK BIT(3) + +/* Virtual definition for multicolor */ +#define MT6360_VIRTUAL_MULTICOLOR (MT6360_MAX_LEDS + 1) +#define MULTICOLOR_NUM_CHANNELS 3 + +#define MT6360_REG_FLEDEN 0x37E +#define MT6360_REG_STRBTO 0x373 +#define MT6360_REG_FLEDBASE(_id) (0x372 + 4 * (_id - MT6360_LED_FLASH1)) +#define MT6360_REG_FLEDISTRB(_id) (MT6360_REG_FLEDBASE(_id) + 2) +#define MT6360_REG_FLEDITOR(_id) (MT6360_REG_FLEDBASE(_id) + 3) +#define MT6360_REG_CHGSTAT2 0x3E1 +#define MT6360_REG_FLEDSTAT1 0x3E9 +#define MT6360_ITORCH_MASK GENMASK(4, 0) +#define MT6360_ISTROBE_MASK GENMASK(6, 0) +#define MT6360_STRBTO_MASK GENMASK(6, 0) +#define MT6360_TORCHEN_MASK BIT(3) +#define MT6360_STROBEN_MASK BIT(2) +#define MT6360_FLCSEN_MASK(_id) BIT(MT6360_LED_FLASH2 - _id) +#define MT6360_FLEDCHGVINOVP_MASK BIT(3) +#define MT6360_FLED1STRBTO_MASK BIT(11) +#define MT6360_FLED2STRBTO_MASK BIT(10) +#define MT6360_FLED1STRB_MASK BIT(9) +#define MT6360_FLED2STRB_MASK BIT(8) +#define MT6360_FLED1SHORT_MASK BIT(7) +#define MT6360_FLED2SHORT_MASK BIT(6) +#define MT6360_FLEDLVF_MASK BIT(3) + +#define MT6360_ISNKRGB_STEPUA 2000 +#define MT6360_ISNKRGB_MAXUA 24000 +#define MT6360_ISNKML_STEPUA 5000 +#define MT6360_ISNKML_MAXUA 150000 + +#define MT6360_ITORCH_MINUA 25000 +#define MT6360_ITORCH_STEPUA 12500 +#define MT6360_ITORCH_MAXUA 400000 +#define MT6360_ISTRB_MINUA 50000 +#define MT6360_ISTRB_STEPUA 12500 +#define MT6360_ISTRB_MAXUA 1500000 +#define MT6360_STRBTO_MINUS 64000 +#define MT6360_STRBTO_STEPUS 32000 +#define MT6360_STRBTO_MAXUS 2432000 + +#define STATE_OFF 0 +#define STATE_KEEP 1 +#define STATE_ON 2 + +struct mt6360_led { + union { + struct led_classdev isnk; + struct led_classdev_mc mc; + struct led_classdev_flash flash; + }; + struct v4l2_flash *v4l2_flash; + struct mt6360_priv *priv; + u32 led_no; + u32 default_state; +}; + +struct mt6360_priv { + struct device *dev; + struct regmap *regmap; + struct mutex lock; + unsigned int fled_strobe_used; + unsigned int fled_torch_used; + unsigned int leds_active; + unsigned int leds_count; + struct mt6360_led leds[]; +}; + +static int mt6360_mc_brightness_set(struct led_classdev *lcdev, + enum led_brightness level) +{ + struct led_classdev_mc *mccdev = lcdev_to_mccdev(lcdev); + struct mt6360_led *led = container_of(mccdev, struct mt6360_led, mc); + struct mt6360_priv *priv = led->priv; + u32 real_bright, enable_mask = 0, enable = 0; + int i, ret; + + mutex_lock(&priv->lock); + + led_mc_calc_color_components(mccdev, level); + + for (i = 0; i < mccdev->num_colors; i++) { + struct mc_subled *subled = mccdev->subled_info + i; + + real_bright = min(lcdev->max_brightness, subled->brightness); + ret = regmap_update_bits(priv->regmap, MT6360_REG_ISNK(i), + MT6360_ISNK_MASK, real_bright); + if (ret) + goto out; + + enable_mask |= MT6360_ISNK_ENMASK(subled->channel); + if (real_bright) + enable |= MT6360_ISNK_ENMASK(subled->channel); + } + + ret = regmap_update_bits(priv->regmap, MT6360_REG_RGBEN, enable_mask, + enable); + +out: + mutex_unlock(&priv->lock); + return ret; +} + +static int mt6360_isnk_brightness_set(struct led_classdev *lcdev, + enum led_brightness level) +{ + struct mt6360_led *led = container_of(lcdev, struct mt6360_led, isnk); + struct mt6360_priv *priv = led->priv; + u32 enable_mask = MT6360_ISNK_ENMASK(led->led_no); + u32 val = level ? MT6360_ISNK_ENMASK(led->led_no) : 0; + int ret; + + mutex_lock(&priv->lock); + + ret = regmap_update_bits(priv->regmap, MT6360_REG_ISNK(led->led_no), + MT6360_ISNK_MASK, level); + if (ret) + goto out; + + ret = regmap_update_bits(priv->regmap, MT6360_REG_RGBEN, enable_mask, + val); + +out: + mutex_unlock(&priv->lock); + return ret; +} + +static int mt6360_torch_brightness_set(struct led_classdev *lcdev, + enum led_brightness level) +{ + struct mt6360_led *led = + container_of(lcdev, struct mt6360_led, flash.led_cdev); + struct mt6360_priv *priv = led->priv; + u32 enable_mask = MT6360_TORCHEN_MASK | MT6360_FLCSEN_MASK(led->led_no); + u32 val = level ? MT6360_FLCSEN_MASK(led->led_no) : 0; + u32 prev = priv->fled_torch_used, curr; + int ret; + + mutex_lock(&priv->lock); + + /* + * Only one set of flash control logic, use the flag to avoid strobe is + * currently used. + */ + if (priv->fled_strobe_used) { + dev_warn(lcdev->dev, "Please disable strobe first [%d]\n", + priv->fled_strobe_used); + ret = -EBUSY; + goto unlock; + } + + if (level) + curr = prev | BIT(led->led_no); + else + curr = prev & ~BIT(led->led_no); + + if (curr) + val |= MT6360_TORCHEN_MASK; + + if (level) { + ret = regmap_update_bits(priv->regmap, + MT6360_REG_FLEDITOR(led->led_no), + MT6360_ITORCH_MASK, level - 1); + if (ret) + goto unlock; + } + + ret = regmap_update_bits(priv->regmap, MT6360_REG_FLEDEN, enable_mask, + val); + if (ret) + goto unlock; + + priv->fled_torch_used = curr; + +unlock: + mutex_unlock(&priv->lock); + return ret; +} + +static int mt6360_flash_brightness_set(struct led_classdev_flash *fl_cdev, + u32 brightness) +{ + /* + * Due to the current spike when turning on flash, let brightness to be + * kept by framework. + * This empty function is used to prevent led_classdev_flash register + * ops check failure. + */ + return 0; +} + +static int _mt6360_flash_brightness_set(struct led_classdev_flash *fl_cdev, + u32 brightness) +{ + struct mt6360_led *led = + container_of(fl_cdev, struct mt6360_led, flash); + struct mt6360_priv *priv = led->priv; + struct led_flash_setting *s = &fl_cdev->brightness; + u32 val = (brightness - s->min) / s->step; + + return regmap_update_bits(priv->regmap, + MT6360_REG_FLEDISTRB(led->led_no), + MT6360_ISTROBE_MASK, val); +} + +static int mt6360_strobe_set(struct led_classdev_flash *fl_cdev, bool state) +{ + struct mt6360_led *led = + container_of(fl_cdev, struct mt6360_led, flash); + struct mt6360_priv *priv = led->priv; + struct led_classdev *lcdev = &fl_cdev->led_cdev; + struct led_flash_setting *s = &fl_cdev->brightness; + u32 enable_mask = MT6360_STROBEN_MASK | MT6360_FLCSEN_MASK(led->led_no); + u32 val = state ? MT6360_FLCSEN_MASK(led->led_no) : 0; + u32 prev = priv->fled_strobe_used, curr; + int ret; + + mutex_lock(&priv->lock); + + /* + * Only one set of flash control logic, use the flag to avoid torch is + * currently used + */ + if (priv->fled_torch_used) { + dev_warn(lcdev->dev, "Please disable torch first [0x%x]\n", + priv->fled_torch_used); + ret = -EBUSY; + goto unlock; + } + + if (state) + curr = prev | BIT(led->led_no); + else + curr = prev & ~BIT(led->led_no); + + if (curr) + val |= MT6360_STROBEN_MASK; + + ret = regmap_update_bits(priv->regmap, MT6360_REG_FLEDEN, enable_mask, + val); + if (ret) { + dev_err(lcdev->dev, "[%d] control current source %d fail\n", + led->led_no, state); + goto unlock; + } + + /* + * If the flash need to be on, config the flash current ramping up to + * the setting value. + * Else, always recover back to the minimum one + */ + ret = _mt6360_flash_brightness_set(fl_cdev, state ? s->val : s->min); + if (ret) + goto unlock; + + /* + * For the flash turn on/off, HW rampping up/down time is 5ms/500us, + * respectively. + */ + if (!prev && curr) + usleep_range(5000, 6000); + else if (prev && !curr) + udelay(500); + + priv->fled_strobe_used = curr; + +unlock: + mutex_unlock(&priv->lock); + return ret; +} + +static int mt6360_strobe_get(struct led_classdev_flash *fl_cdev, bool *state) +{ + struct mt6360_led *led = + container_of(fl_cdev, struct mt6360_led, flash); + struct mt6360_priv *priv = led->priv; + + mutex_lock(&priv->lock); + *state = !!(priv->fled_strobe_used & BIT(led->led_no)); + mutex_unlock(&priv->lock); + + return 0; +} + +static int mt6360_timeout_set(struct led_classdev_flash *fl_cdev, u32 timeout) +{ + struct mt6360_led *led = + container_of(fl_cdev, struct mt6360_led, flash); + struct mt6360_priv *priv = led->priv; + struct led_flash_setting *s = &fl_cdev->timeout; + u32 val = (timeout - s->min) / s->step; + int ret; + + mutex_lock(&priv->lock); + ret = regmap_update_bits(priv->regmap, MT6360_REG_STRBTO, + MT6360_STRBTO_MASK, val); + mutex_unlock(&priv->lock); + + return ret; +} + +static int mt6360_fault_get(struct led_classdev_flash *fl_cdev, u32 *fault) +{ + struct mt6360_led *led = + container_of(fl_cdev, struct mt6360_led, flash); + struct mt6360_priv *priv = led->priv; + u16 fled_stat; + unsigned int chg_stat, strobe_timeout_mask, fled_short_mask; + u32 rfault = 0; + int ret; + + mutex_lock(&priv->lock); + ret = regmap_read(priv->regmap, MT6360_REG_CHGSTAT2, &chg_stat); + if (ret) + goto unlock; + + ret = regmap_raw_read(priv->regmap, MT6360_REG_FLEDSTAT1, &fled_stat, + sizeof(fled_stat)); + if (ret) + goto unlock; + + if (led->led_no == MT6360_LED_FLASH1) { + strobe_timeout_mask = MT6360_FLED1STRBTO_MASK; + fled_short_mask = MT6360_FLED1SHORT_MASK; + } else { + strobe_timeout_mask = MT6360_FLED2STRBTO_MASK; + fled_short_mask = MT6360_FLED2SHORT_MASK; + } + + if (chg_stat & MT6360_FLEDCHGVINOVP_MASK) + rfault |= LED_FAULT_INPUT_VOLTAGE; + + if (fled_stat & strobe_timeout_mask) + rfault |= LED_FAULT_TIMEOUT; + + if (fled_stat & fled_short_mask) + rfault |= LED_FAULT_SHORT_CIRCUIT; + + if (fled_stat & MT6360_FLEDLVF_MASK) + rfault |= LED_FAULT_UNDER_VOLTAGE; + + *fault = rfault; +unlock: + mutex_unlock(&priv->lock); + return ret; +} + +static const struct led_flash_ops mt6360_flash_ops = { + .flash_brightness_set = mt6360_flash_brightness_set, + .strobe_set = mt6360_strobe_set, + .strobe_get = mt6360_strobe_get, + .timeout_set = mt6360_timeout_set, + .fault_get = mt6360_fault_get, +}; + +static int mt6360_isnk_init_default_state(struct mt6360_led *led) +{ + struct mt6360_priv *priv = led->priv; + unsigned int regval; + u32 level; + int ret; + + ret = regmap_read(priv->regmap, MT6360_REG_ISNK(led->led_no), ®val); + if (ret) + return ret; + level = regval & MT6360_ISNK_MASK; + + ret = regmap_read(priv->regmap, MT6360_REG_RGBEN, ®val); + if (ret) + return ret; + + if (!(regval & MT6360_ISNK_ENMASK(led->led_no))) + level = LED_OFF; + + switch (led->default_state) { + case STATE_ON: + led->isnk.brightness = led->isnk.max_brightness; + break; + case STATE_KEEP: + led->isnk.brightness = min(level, led->isnk.max_brightness); + break; + default: + led->isnk.brightness = LED_OFF; + } + + return mt6360_isnk_brightness_set(&led->isnk, led->isnk.brightness); +} + +static int mt6360_flash_init_default_state(struct mt6360_led *led) +{ + struct led_classdev_flash *flash = &led->flash; + struct mt6360_priv *priv = led->priv; + u32 enable_mask = MT6360_TORCHEN_MASK | MT6360_FLCSEN_MASK(led->led_no); + u32 level; + unsigned int regval; + int ret; + + ret = regmap_read(priv->regmap, MT6360_REG_FLEDITOR(led->led_no), + ®val); + if (ret) + return ret; + level = regval & MT6360_ITORCH_MASK; + + ret = regmap_read(priv->regmap, MT6360_REG_FLEDEN, ®val); + if (ret) + return ret; + + if ((regval & enable_mask) == enable_mask) + level += 1; + else + level = LED_OFF; + + switch (led->default_state) { + case STATE_ON: + flash->led_cdev.brightness = flash->led_cdev.max_brightness; + break; + case STATE_KEEP: + flash->led_cdev.brightness = + min(level, flash->led_cdev.max_brightness); + break; + default: + flash->led_cdev.brightness = LED_OFF; + } + + return mt6360_torch_brightness_set(&flash->led_cdev, + flash->led_cdev.brightness); +} + +#if IS_ENABLED(CONFIG_V4L2_FLASH_LED_CLASS) +static int mt6360_flash_external_strobe_set(struct v4l2_flash *v4l2_flash, + bool enable) +{ + struct led_classdev_flash *flash = v4l2_flash->fled_cdev; + struct mt6360_led *led = container_of(flash, struct mt6360_led, flash); + struct mt6360_priv *priv = led->priv; + u32 mask = MT6360_FLCSEN_MASK(led->led_no); + u32 val = enable ? mask : 0; + int ret; + + mutex_lock(&priv->lock); + + ret = regmap_update_bits(priv->regmap, MT6360_REG_FLEDEN, mask, val); + if (ret) + goto unlock; + + if (enable) + priv->fled_strobe_used |= BIT(led->led_no); + else + priv->fled_strobe_used &= ~BIT(led->led_no); + +unlock: + mutex_unlock(&priv->lock); + return ret; +} + +static const struct v4l2_flash_ops v4l2_flash_ops = { + .external_strobe_set = mt6360_flash_external_strobe_set, +}; + +static void mt6360_init_v4l2_flash_config(struct mt6360_led *led, + struct v4l2_flash_config *config) +{ + struct led_classdev *lcdev; + struct led_flash_setting *s = &config->intensity; + + lcdev = &led->flash.led_cdev; + + s->min = MT6360_ITORCH_MINUA; + s->step = MT6360_ITORCH_STEPUA; + s->val = s->max = s->min + (lcdev->max_brightness - 1) * s->step; + + config->has_external_strobe = 1; + strscpy(config->dev_name, lcdev->dev->kobj.name, + sizeof(config->dev_name)); + + config->flash_faults = LED_FAULT_SHORT_CIRCUIT | LED_FAULT_TIMEOUT | + LED_FAULT_INPUT_VOLTAGE | + LED_FAULT_UNDER_VOLTAGE; +} +#else +static const struct v4l2_flash_ops v4l2_flash_ops; +static void mt6360_init_v4l2_flash_config(struct mt6360_led *led, + struct v4l2_flash_config *config) +{ +} +#endif + +static int mt6360_led_register(struct device *parent, struct mt6360_led *led, + struct led_init_data *init_data) +{ + struct mt6360_priv *priv = led->priv; + struct v4l2_flash_config v4l2_config = {0}; + int ret; + + if ((led->led_no == MT6360_LED_ISNK1 || + led->led_no == MT6360_VIRTUAL_MULTICOLOR) && + (priv->leds_active & BIT(MT6360_LED_ISNK1))) { + /* + * Change isink1 to SW control mode, disconnect it with + * charger state + */ + ret = regmap_update_bits(priv->regmap, MT6360_REG_RGBEN, + MT6360_CHRINDSEL_MASK, + MT6360_CHRINDSEL_MASK); + if (ret) { + dev_err(parent, "Failed to config ISNK1 to SW mode\n"); + return ret; + } + } + + switch (led->led_no) { + case MT6360_VIRTUAL_MULTICOLOR: + ret = mt6360_mc_brightness_set(&led->mc.led_cdev, LED_OFF); + if (ret) { + dev_err(parent, + "Failed to init multicolor brightness\n"); + return ret; + } + + ret = devm_led_classdev_multicolor_register_ext(parent, + &led->mc, init_data); + if (ret) { + dev_err(parent, "Couldn't register multicolor\n"); + return ret; + } + break; + case MT6360_LED_ISNK1 ... MT6360_LED_ISNKML: + ret = mt6360_isnk_init_default_state(led); + if (ret) { + dev_err(parent, "Failed to init %d isnk state\n", + led->led_no); + return ret; + } + + ret = devm_led_classdev_register_ext(parent, &led->isnk, + init_data); + if (ret) { + dev_err(parent, "Couldn't register isink %d\n", + led->led_no); + return ret; + } + break; + default: + ret = mt6360_flash_init_default_state(led); + if (ret) { + dev_err(parent, "Failed to init %d flash state\n", + led->led_no); + return ret; + } + + ret = devm_led_classdev_flash_register_ext(parent, &led->flash, + init_data); + if (ret) { + dev_err(parent, "Couldn't register flash %d\n", + led->led_no); + return ret; + } + + mt6360_init_v4l2_flash_config(led, &v4l2_config); + led->v4l2_flash = v4l2_flash_init(parent, init_data->fwnode, + &led->flash, + &v4l2_flash_ops, + &v4l2_config); + if (IS_ERR(led->v4l2_flash)) { + dev_err(parent, "Failed to register %d v4l2 sd\n", + led->led_no); + return PTR_ERR(led->v4l2_flash); + } + } + + return 0; +} + +static u32 clamp_align(u32 val, u32 min, u32 max, u32 step) +{ + u32 retval; + + retval = clamp_val(val, min, max); + if (step > 1) + retval = rounddown(retval - min, step) + min; + + return retval; +} + +static int mt6360_init_isnk_properties(struct mt6360_led *led, + struct led_init_data *init_data) +{ + struct led_classdev *lcdev; + struct mt6360_priv *priv = led->priv; + struct fwnode_handle *child; + u32 step_uA = MT6360_ISNKRGB_STEPUA, max_uA = MT6360_ISNKRGB_MAXUA; + u32 val; + int num_color = 0, ret; + + if (led->led_no == MT6360_VIRTUAL_MULTICOLOR) { + struct mc_subled *sub_led; + + sub_led = devm_kzalloc(priv->dev, + sizeof(*sub_led) * MULTICOLOR_NUM_CHANNELS, GFP_KERNEL); + if (!sub_led) + return -ENOMEM; + + fwnode_for_each_child_node(init_data->fwnode, child) { + u32 reg, color; + + ret = fwnode_property_read_u32(child, "reg", ®); + if (ret || reg > MT6360_LED_ISNK3 || + priv->leds_active & BIT(reg)) + return -EINVAL; + + ret = fwnode_property_read_u32(child, "color", &color); + if (ret) { + dev_err(priv->dev, + "led %d, no color specified\n", + led->led_no); + return ret; + } + + priv->leds_active |= BIT(reg); + sub_led[num_color].color_index = color; + sub_led[num_color].channel = reg; + num_color++; + } + + if (num_color < 2) { + dev_err(priv->dev, + "Multicolor must include 2 or more led channel\n"); + return -EINVAL; + } + + led->mc.num_colors = num_color; + led->mc.subled_info = sub_led; + + lcdev = &led->mc.led_cdev; + lcdev->brightness_set_blocking = mt6360_mc_brightness_set; + } else { + if (led->led_no == MT6360_LED_ISNKML) { + step_uA = MT6360_ISNKML_STEPUA; + max_uA = MT6360_ISNKML_MAXUA; + } + + lcdev = &led->isnk; + lcdev->brightness_set_blocking = mt6360_isnk_brightness_set; + } + + ret = fwnode_property_read_u32(init_data->fwnode, "led-max-microamp", + &val); + if (ret) { + dev_warn(priv->dev, + "Not specified led-max-microamp, config to the minimum\n"); + val = step_uA; + } else + val = clamp_align(val, 0, max_uA, step_uA); + + lcdev->max_brightness = val / step_uA; + + fwnode_property_read_string(init_data->fwnode, "linux,default-trigger", + &lcdev->default_trigger); + + return 0; +} + +static int mt6360_init_flash_properties(struct mt6360_led *led, + struct led_init_data *init_data) +{ + struct led_classdev_flash *flash = &led->flash; + struct led_classdev *lcdev = &flash->led_cdev; + struct mt6360_priv *priv = led->priv; + struct led_flash_setting *s; + u32 val; + int ret; + + ret = fwnode_property_read_u32(init_data->fwnode, "led-max-microamp", + &val); + if (ret) { + dev_warn(priv->dev, + "Not specified led-max-microamp, config to the minimum\n"); + val = MT6360_ITORCH_MINUA; + } else + val = clamp_align(val, MT6360_ITORCH_MINUA, MT6360_ITORCH_MAXUA, + MT6360_ITORCH_STEPUA); + + lcdev->max_brightness = + (val - MT6360_ITORCH_MINUA) / MT6360_ITORCH_STEPUA + 1; + lcdev->brightness_set_blocking = mt6360_torch_brightness_set; + lcdev->flags |= LED_DEV_CAP_FLASH; + + ret = fwnode_property_read_u32(init_data->fwnode, "flash-max-microamp", + &val); + if (ret) { + dev_warn(priv->dev, + "Not specified flash-max-microamp, config to the minimum\n"); + val = MT6360_ISTRB_MINUA; + } else + val = clamp_align(val, MT6360_ISTRB_MINUA, MT6360_ISTRB_MAXUA, + MT6360_ISTRB_STEPUA); + + s = &flash->brightness; + s->min = MT6360_ISTRB_MINUA; + s->step = MT6360_ISTRB_STEPUA; + s->val = s->max = val; + + /* + * Always configure as min level when off to prevent flash current + * spike. + */ + ret = _mt6360_flash_brightness_set(flash, s->min); + if (ret) + return ret; + + ret = fwnode_property_read_u32(init_data->fwnode, + "flash-max-timeout-us", &val); + if (ret) { + dev_warn(priv->dev, + "Not specified flash-max-timeout-us, config to the minimum\n"); + val = MT6360_STRBTO_MINUS; + } else + val = clamp_align(val, MT6360_STRBTO_MINUS, MT6360_STRBTO_MAXUS, + MT6360_STRBTO_STEPUS); + + s = &flash->timeout; + s->min = MT6360_STRBTO_MINUS; + s->step = MT6360_STRBTO_STEPUS; + s->val = s->max = val; + + flash->ops = &mt6360_flash_ops; + + return 0; +} + +static int mt6360_init_common_properties(struct mt6360_led *led, + struct led_init_data *init_data) +{ + const char *const states[] = { "off", "keep", "on" }; + const char *str; + int ret; + + if (!fwnode_property_read_string(init_data->fwnode, + "default-state", &str)) { + ret = match_string(states, ARRAY_SIZE(states), str); + if (ret < 0) + ret = STATE_OFF; + + led->default_state = ret; + } + + return 0; +} + +static void mt6360_v4l2_flash_release(struct mt6360_priv *priv) +{ + int i; + + for (i = 0; i < priv->leds_count; i++) { + struct mt6360_led *led = priv->leds + i; + + if (led->v4l2_flash) + v4l2_flash_release(led->v4l2_flash); + } +} + +static int mt6360_led_probe(struct platform_device *pdev) +{ + struct mt6360_priv *priv; + struct fwnode_handle *child; + size_t count; + int i = 0, ret; + + count = device_get_child_node_count(&pdev->dev); + if (!count || count > MT6360_MAX_LEDS) { + dev_err(&pdev->dev, + "No child node or node count over max led number %zu\n", + count); + return -EINVAL; + } + + priv = devm_kzalloc(&pdev->dev, + struct_size(priv, leds, count), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->leds_count = count; + priv->dev = &pdev->dev; + mutex_init(&priv->lock); + + priv->regmap = dev_get_regmap(pdev->dev.parent, NULL); + if (!priv->regmap) { + dev_err(&pdev->dev, "Failed to get parent regmap\n"); + return -ENODEV; + } + + device_for_each_child_node(&pdev->dev, child) { + struct mt6360_led *led = priv->leds + i; + struct led_init_data init_data = { .fwnode = child, }; + u32 reg, led_color; + + ret = fwnode_property_read_u32(child, "color", &led_color); + if (ret) + goto out_flash_release; + + if (led_color == LED_COLOR_ID_RGB || + led_color == LED_COLOR_ID_MULTI) + reg = MT6360_VIRTUAL_MULTICOLOR; + else { + ret = fwnode_property_read_u32(child, "reg", ®); + if (ret) + goto out_flash_release; + + if (reg >= MT6360_MAX_LEDS) { + ret = -EINVAL; + goto out_flash_release; + } + } + + if (priv->leds_active & BIT(reg)) { + ret = -EINVAL; + goto out_flash_release; + } + priv->leds_active |= BIT(reg); + + led->led_no = reg; + led->priv = priv; + + ret = mt6360_init_common_properties(led, &init_data); + if (ret) + goto out_flash_release; + + if (reg == MT6360_VIRTUAL_MULTICOLOR || + reg <= MT6360_LED_ISNKML) + ret = mt6360_init_isnk_properties(led, &init_data); + else + ret = mt6360_init_flash_properties(led, &init_data); + + if (ret) + goto out_flash_release; + + ret = mt6360_led_register(&pdev->dev, led, &init_data); + if (ret) + goto out_flash_release; + + i++; + } + + platform_set_drvdata(pdev, priv); + return 0; + +out_flash_release: + mt6360_v4l2_flash_release(priv); + return ret; +} + +static int mt6360_led_remove(struct platform_device *pdev) +{ + struct mt6360_priv *priv = platform_get_drvdata(pdev); + + mt6360_v4l2_flash_release(priv); + return 0; +} + +static const struct of_device_id __maybe_unused mt6360_led_of_id[] = { + { .compatible = "mediatek,mt6360-led", }, + {} +}; +MODULE_DEVICE_TABLE(of, mt6360_led_of_id); + +static struct platform_driver mt6360_led_driver = { + .driver = { + .name = "mt6360-led", + .of_match_table = mt6360_led_of_id, + }, + .probe = mt6360_led_probe, + .remove = mt6360_led_remove, +}; +module_platform_driver(mt6360_led_driver); + +MODULE_AUTHOR("Gene Chen "); +MODULE_DESCRIPTION("MT6360 LED Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/leds/led-class.c b/drivers/leds/led-class.c index f4bb02f6e042..6a8ea94834fa 100644 --- a/drivers/leds/led-class.c +++ b/drivers/leds/led-class.c @@ -375,10 +375,8 @@ int led_classdev_register_ext(struct device *parent, mutex_unlock(&led_cdev->led_access); return PTR_ERR(led_cdev->dev); } - if (init_data && init_data->fwnode) { - led_cdev->dev->fwnode = init_data->fwnode; - led_cdev->dev->of_node = to_of_node(init_data->fwnode); - } + if (init_data && init_data->fwnode) + device_set_node(led_cdev->dev, init_data->fwnode); if (ret) dev_warn(parent, "Led %s renamed to %s due to name collision", diff --git a/drivers/leds/leds-fsg.c b/drivers/leds/leds-fsg.c deleted file mode 100644 index bc6b420637d6..000000000000 --- a/drivers/leds/leds-fsg.c +++ /dev/null @@ -1,193 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * LED Driver for the Freecom FSG-3 - * - * Copyright (c) 2008 Rod Whitby - * - * Author: Rod Whitby - * - * Based on leds-spitz.c - * Copyright 2005-2006 Openedhand Ltd. - * Author: Richard Purdie - */ - -#include -#include -#include -#include -#include -#include - -#define FSG_LED_WLAN_BIT 0 -#define FSG_LED_WAN_BIT 1 -#define FSG_LED_SATA_BIT 2 -#define FSG_LED_USB_BIT 4 -#define FSG_LED_RING_BIT 5 -#define FSG_LED_SYNC_BIT 7 - -static short __iomem *latch_address; -static unsigned short latch_value; - - -static void fsg_led_wlan_set(struct led_classdev *led_cdev, - enum led_brightness value) -{ - if (value) { - latch_value &= ~(1 << FSG_LED_WLAN_BIT); - *latch_address = latch_value; - } else { - latch_value |= (1 << FSG_LED_WLAN_BIT); - *latch_address = latch_value; - } -} - -static void fsg_led_wan_set(struct led_classdev *led_cdev, - enum led_brightness value) -{ - if (value) { - latch_value &= ~(1 << FSG_LED_WAN_BIT); - *latch_address = latch_value; - } else { - latch_value |= (1 << FSG_LED_WAN_BIT); - *latch_address = latch_value; - } -} - -static void fsg_led_sata_set(struct led_classdev *led_cdev, - enum led_brightness value) -{ - if (value) { - latch_value &= ~(1 << FSG_LED_SATA_BIT); - *latch_address = latch_value; - } else { - latch_value |= (1 << FSG_LED_SATA_BIT); - *latch_address = latch_value; - } -} - -static void fsg_led_usb_set(struct led_classdev *led_cdev, - enum led_brightness value) -{ - if (value) { - latch_value &= ~(1 << FSG_LED_USB_BIT); - *latch_address = latch_value; - } else { - latch_value |= (1 << FSG_LED_USB_BIT); - *latch_address = latch_value; - } -} - -static void fsg_led_sync_set(struct led_classdev *led_cdev, - enum led_brightness value) -{ - if (value) { - latch_value &= ~(1 << FSG_LED_SYNC_BIT); - *latch_address = latch_value; - } else { - latch_value |= (1 << FSG_LED_SYNC_BIT); - *latch_address = latch_value; - } -} - -static void fsg_led_ring_set(struct led_classdev *led_cdev, - enum led_brightness value) -{ - if (value) { - latch_value &= ~(1 << FSG_LED_RING_BIT); - *latch_address = latch_value; - } else { - latch_value |= (1 << FSG_LED_RING_BIT); - *latch_address = latch_value; - } -} - - -static struct led_classdev fsg_wlan_led = { - .name = "fsg:blue:wlan", - .brightness_set = fsg_led_wlan_set, - .flags = LED_CORE_SUSPENDRESUME, -}; - -static struct led_classdev fsg_wan_led = { - .name = "fsg:blue:wan", - .brightness_set = fsg_led_wan_set, - .flags = LED_CORE_SUSPENDRESUME, -}; - -static struct led_classdev fsg_sata_led = { - .name = "fsg:blue:sata", - .brightness_set = fsg_led_sata_set, - .flags = LED_CORE_SUSPENDRESUME, -}; - -static struct led_classdev fsg_usb_led = { - .name = "fsg:blue:usb", - .brightness_set = fsg_led_usb_set, - .flags = LED_CORE_SUSPENDRESUME, -}; - -static struct led_classdev fsg_sync_led = { - .name = "fsg:blue:sync", - .brightness_set = fsg_led_sync_set, - .flags = LED_CORE_SUSPENDRESUME, -}; - -static struct led_classdev fsg_ring_led = { - .name = "fsg:blue:ring", - .brightness_set = fsg_led_ring_set, - .flags = LED_CORE_SUSPENDRESUME, -}; - - -static int fsg_led_probe(struct platform_device *pdev) -{ - int ret; - - /* Map the LED chip select address space */ - latch_address = (unsigned short *) devm_ioremap(&pdev->dev, - IXP4XX_EXP_BUS_BASE(2), 512); - if (!latch_address) - return -ENOMEM; - - latch_value = 0xffff; - *latch_address = latch_value; - - ret = devm_led_classdev_register(&pdev->dev, &fsg_wlan_led); - if (ret < 0) - return ret; - - ret = devm_led_classdev_register(&pdev->dev, &fsg_wan_led); - if (ret < 0) - return ret; - - ret = devm_led_classdev_register(&pdev->dev, &fsg_sata_led); - if (ret < 0) - return ret; - - ret = devm_led_classdev_register(&pdev->dev, &fsg_usb_led); - if (ret < 0) - return ret; - - ret = devm_led_classdev_register(&pdev->dev, &fsg_sync_led); - if (ret < 0) - return ret; - - ret = devm_led_classdev_register(&pdev->dev, &fsg_ring_led); - if (ret < 0) - return ret; - - return ret; -} - -static struct platform_driver fsg_led_driver = { - .probe = fsg_led_probe, - .driver = { - .name = "fsg-led", - }, -}; - -module_platform_driver(fsg_led_driver); - -MODULE_AUTHOR("Rod Whitby "); -MODULE_DESCRIPTION("Freecom FSG-3 LED driver"); -MODULE_LICENSE("GPL"); diff --git a/drivers/leds/leds-lp50xx.c b/drivers/leds/leds-lp50xx.c index 401df1e2e05d..50b195ff96ca 100644 --- a/drivers/leds/leds-lp50xx.c +++ b/drivers/leds/leds-lp50xx.c @@ -266,7 +266,6 @@ struct lp50xx_led { struct led_classdev_mc mc_cdev; struct lp50xx *priv; unsigned long bank_modules; - int led_intensity[LP50XX_LEDS_PER_MODULE]; u8 ctrl_bank_enabled; int led_number; }; diff --git a/drivers/leds/leds-lp55xx-common.c b/drivers/leds/leds-lp55xx-common.c index d1657c46ee2f..9fdfc1b9a1a0 100644 --- a/drivers/leds/leds-lp55xx-common.c +++ b/drivers/leds/leds-lp55xx-common.c @@ -439,6 +439,8 @@ int lp55xx_init_device(struct lp55xx_chip *chip) return -EINVAL; if (pdata->enable_gpiod) { + gpiod_direction_output(pdata->enable_gpiod, 0); + gpiod_set_consumer_name(pdata->enable_gpiod, "LP55xx enable"); gpiod_set_value(pdata->enable_gpiod, 0); usleep_range(1000, 2000); /* Keep enable down at least 1ms */ @@ -694,7 +696,7 @@ struct lp55xx_platform_data *lp55xx_of_populate_pdata(struct device *dev, of_property_read_u8(np, "clock-mode", &pdata->clock_mode); pdata->enable_gpiod = devm_gpiod_get_optional(dev, "enable", - GPIOD_OUT_LOW); + GPIOD_ASIS); if (IS_ERR(pdata->enable_gpiod)) return ERR_CAST(pdata->enable_gpiod); diff --git a/drivers/leds/leds-tca6507.c b/drivers/leds/leds-tca6507.c index 225b765830bd..1473ced8664c 100644 --- a/drivers/leds/leds-tca6507.c +++ b/drivers/leds/leds-tca6507.c @@ -242,9 +242,7 @@ static int choose_times(int msec, int *c1p, int *c2p) if (diff < 65536) { int actual; if (msec & 1) { - c1 = *c2p; - *c2p = *c1p; - *c1p = c1; + swap(*c2p, *c1p); } actual = time_codes[*c1p] + time_codes[*c2p]; if (*c1p < *c2p) @@ -643,9 +641,6 @@ static int tca6507_probe_gpios(struct device *dev, tca->gpio.direction_output = tca6507_gpio_direction_output; tca->gpio.set = tca6507_gpio_set_value; tca->gpio.parent = dev; -#ifdef CONFIG_OF_GPIO - tca->gpio.of_node = of_node_get(dev_of_node(dev)); -#endif err = gpiochip_add_data(&tca->gpio, tca); if (err) { tca->gpio.ngpio = 0; diff --git a/drivers/mailbox/apple-mailbox.c b/drivers/mailbox/apple-mailbox.c index 72942002a54a..496c4951ccb1 100644 --- a/drivers/mailbox/apple-mailbox.c +++ b/drivers/mailbox/apple-mailbox.c @@ -364,8 +364,8 @@ static const struct apple_mbox_hw apple_mbox_m3_hw = { }; static const struct of_device_id apple_mbox_of_match[] = { - { .compatible = "apple,t8103-asc-mailbox", .data = &apple_mbox_asc_hw }, - { .compatible = "apple,t8103-m3-mailbox", .data = &apple_mbox_m3_hw }, + { .compatible = "apple,asc-mailbox-v4", .data = &apple_mbox_asc_hw }, + { .compatible = "apple,m3-mailbox-v2", .data = &apple_mbox_m3_hw }, {} }; MODULE_DEVICE_TABLE(of, apple_mbox_of_match); diff --git a/drivers/mailbox/bcm-flexrm-mailbox.c b/drivers/mailbox/bcm-flexrm-mailbox.c index 78073ad1f2f1..22acb51531cb 100644 --- a/drivers/mailbox/bcm-flexrm-mailbox.c +++ b/drivers/mailbox/bcm-flexrm-mailbox.c @@ -1298,7 +1298,7 @@ static int flexrm_startup(struct mbox_chan *chan) val = (num_online_cpus() < val) ? val / num_online_cpus() : 1; cpumask_set_cpu((ring->num / val) % num_online_cpus(), &ring->irq_aff_hint); - ret = irq_set_affinity_hint(ring->irq, &ring->irq_aff_hint); + ret = irq_update_affinity_hint(ring->irq, &ring->irq_aff_hint); if (ret) { dev_err(ring->mbox->dev, "failed to set IRQ affinity hint for ring%d\n", @@ -1425,7 +1425,7 @@ static void flexrm_shutdown(struct mbox_chan *chan) /* Release IRQ */ if (ring->irq_requested) { - irq_set_affinity_hint(ring->irq, NULL); + irq_update_affinity_hint(ring->irq, NULL); free_irq(ring->irq, ring); ring->irq_requested = false; } @@ -1484,7 +1484,7 @@ static void flexrm_mbox_msi_write(struct msi_desc *desc, struct msi_msg *msg) { struct device *dev = msi_desc_to_dev(desc); struct flexrm_mbox *mbox = dev_get_drvdata(dev); - struct flexrm_ring *ring = &mbox->rings[desc->platform.msi_index]; + struct flexrm_ring *ring = &mbox->rings[desc->msi_index]; /* Configure per-Ring MSI registers */ writel_relaxed(msg->address_lo, ring->regs + RING_MSI_ADDR_LS); @@ -1497,7 +1497,6 @@ static int flexrm_mbox_probe(struct platform_device *pdev) int index, ret = 0; void __iomem *regs; void __iomem *regs_end; - struct msi_desc *desc; struct resource *iomem; struct flexrm_ring *ring; struct flexrm_mbox *mbox; @@ -1608,10 +1607,8 @@ static int flexrm_mbox_probe(struct platform_device *pdev) goto fail_destroy_cmpl_pool; /* Save alloced IRQ numbers for each ring */ - for_each_msi_entry(desc, dev) { - ring = &mbox->rings[desc->platform.msi_index]; - ring->irq = desc->irq; - } + for (index = 0; index < mbox->num_rings; index++) + mbox->rings[index].irq = msi_get_virq(dev, index); /* Check availability of debugfs */ if (!debugfs_initialized()) diff --git a/drivers/mailbox/hi3660-mailbox.c b/drivers/mailbox/hi3660-mailbox.c index e41bd2f5ea46..ab24e731a782 100644 --- a/drivers/mailbox/hi3660-mailbox.c +++ b/drivers/mailbox/hi3660-mailbox.c @@ -44,14 +44,13 @@ #define MBOX_MSG_LEN 8 /** - * Hi3660 mailbox channel information + * struct hi3660_chan_info - Hi3660 mailbox channel information + * @dst_irq: Interrupt vector for remote processor + * @ack_irq: Interrupt vector for local processor * * A channel can be used for TX or RX, it can trigger remote * processor interrupt to notify remote processor and can receive - * interrupt if has incoming message. - * - * @dst_irq: Interrupt vector for remote processor - * @ack_irq: Interrupt vector for local processor + * interrupt if it has an incoming message. */ struct hi3660_chan_info { unsigned int dst_irq; @@ -59,16 +58,15 @@ struct hi3660_chan_info { }; /** - * Hi3660 mailbox controller data - * - * Mailbox controller includes 32 channels and can allocate - * channel for message transferring. - * + * struct hi3660_mbox - Hi3660 mailbox controller data * @dev: Device to which it is attached * @base: Base address of the register mapping region * @chan: Representation of channels in mailbox controller * @mchan: Representation of channel info * @controller: Representation of a communication channel controller + * + * Mailbox controller includes 32 channels and can allocate + * channel for message transferring. */ struct hi3660_mbox { struct device *dev; diff --git a/drivers/mailbox/imx-mailbox.c b/drivers/mailbox/imx-mailbox.c index ffe36a6bef9e..544de2db6453 100644 --- a/drivers/mailbox/imx-mailbox.c +++ b/drivers/mailbox/imx-mailbox.c @@ -563,8 +563,8 @@ static int imx_mu_probe(struct platform_device *pdev) size = sizeof(struct imx_sc_rpc_msg_max); priv->msg = devm_kzalloc(dev, size, GFP_KERNEL); - if (IS_ERR(priv->msg)) - return PTR_ERR(priv->msg); + if (!priv->msg) + return -ENOMEM; priv->clk = devm_clk_get(dev, NULL); if (IS_ERR(priv->clk)) { diff --git a/drivers/mailbox/mailbox-mpfs.c b/drivers/mailbox/mailbox-mpfs.c index 0d6e2231a2c7..4e34854d1238 100644 --- a/drivers/mailbox/mailbox-mpfs.c +++ b/drivers/mailbox/mailbox-mpfs.c @@ -232,7 +232,7 @@ static int mpfs_mbox_probe(struct platform_device *pdev) } static const struct of_device_id mpfs_mbox_of_match[] = { - {.compatible = "microchip,polarfire-soc-mailbox", }, + {.compatible = "microchip,mpfs-mailbox", }, {}, }; MODULE_DEVICE_TABLE(of, mpfs_mbox_of_match); diff --git a/drivers/mailbox/mtk-cmdq-mailbox.c b/drivers/mailbox/mtk-cmdq-mailbox.c index a8845b162dbf..2578e5aaa935 100644 --- a/drivers/mailbox/mtk-cmdq-mailbox.c +++ b/drivers/mailbox/mtk-cmdq-mailbox.c @@ -573,8 +573,11 @@ static int cmdq_probe(struct platform_device *pdev) cmdq->clocks[alias_id].id = clk_names[alias_id]; cmdq->clocks[alias_id].clk = of_clk_get(node, 0); if (IS_ERR(cmdq->clocks[alias_id].clk)) { - dev_err(dev, "failed to get gce clk: %d\n", alias_id); - return PTR_ERR(cmdq->clocks[alias_id].clk); + of_node_put(node); + return dev_err_probe(dev, + PTR_ERR(cmdq->clocks[alias_id].clk), + "failed to get gce clk: %d\n", + alias_id); } } } @@ -582,8 +585,8 @@ static int cmdq_probe(struct platform_device *pdev) cmdq->clocks[alias_id].id = clk_name; cmdq->clocks[alias_id].clk = devm_clk_get(&pdev->dev, clk_name); if (IS_ERR(cmdq->clocks[alias_id].clk)) { - dev_err(dev, "failed to get gce clk\n"); - return PTR_ERR(cmdq->clocks[alias_id].clk); + return dev_err_probe(dev, PTR_ERR(cmdq->clocks[alias_id].clk), + "failed to get gce clk\n"); } } @@ -658,13 +661,13 @@ static const struct gce_plat gce_plat_v5 = { .thread_nr = 24, .shift = 3, .control_by_sw = true, - .gce_num = 2 + .gce_num = 1 }; static const struct gce_plat gce_plat_v6 = { .thread_nr = 24, .shift = 3, - .control_by_sw = false, + .control_by_sw = true, .gce_num = 2 }; diff --git a/drivers/mailbox/pcc.c b/drivers/mailbox/pcc.c index 887a3704c12e..ed18936b8ce6 100644 --- a/drivers/mailbox/pcc.c +++ b/drivers/mailbox/pcc.c @@ -241,9 +241,11 @@ static irqreturn_t pcc_mbox_irq(int irq, void *p) if (ret) return IRQ_NONE; - val &= pchan->cmd_complete.status_mask; - if (!val) - return IRQ_NONE; + if (val) { /* Ensure GAS exists and value is non-zero */ + val &= pchan->cmd_complete.status_mask; + if (!val) + return IRQ_NONE; + } ret = pcc_chan_reg_read(&pchan->error, &val); if (ret) @@ -289,7 +291,7 @@ pcc_mbox_request_channel(struct mbox_client *cl, int subspace_id) pchan = chan_info + subspace_id; chan = pchan->chan.mchan; if (IS_ERR(chan) || chan->cl) { - dev_err(dev, "Channel not found for idx: %d\n", subspace_id); + pr_err("Channel not found for idx: %d\n", subspace_id); return ERR_PTR(-EBUSY); } dev = chan->mbox->dev; diff --git a/drivers/mailbox/qcom-ipcc.c b/drivers/mailbox/qcom-ipcc.c index f1d4f4679b17..c5d963222014 100644 --- a/drivers/mailbox/qcom-ipcc.c +++ b/drivers/mailbox/qcom-ipcc.c @@ -13,8 +13,6 @@ #include -#define IPCC_MBOX_MAX_CHAN 48 - /* IPCC Register offsets */ #define IPCC_REG_SEND_ID 0x0c #define IPCC_REG_RECV_ID 0x10 @@ -52,9 +50,10 @@ struct qcom_ipcc { struct device *dev; void __iomem *base; struct irq_domain *irq_domain; - struct mbox_chan chan[IPCC_MBOX_MAX_CHAN]; - struct qcom_ipcc_chan_info mchan[IPCC_MBOX_MAX_CHAN]; + struct mbox_chan *chans; + struct qcom_ipcc_chan_info *mchan; struct mbox_controller mbox; + int num_chans; int irq; }; @@ -166,25 +165,37 @@ static struct mbox_chan *qcom_ipcc_mbox_xlate(struct mbox_controller *mbox, struct qcom_ipcc *ipcc = to_qcom_ipcc(mbox); struct qcom_ipcc_chan_info *mchan; struct mbox_chan *chan; - unsigned int i; + struct device *dev; + int chan_id; + + dev = ipcc->dev; if (ph->args_count != 2) return ERR_PTR(-EINVAL); - for (i = 0; i < IPCC_MBOX_MAX_CHAN; i++) { - chan = &ipcc->chan[i]; - if (!chan->con_priv) { - mchan = &ipcc->mchan[i]; - mchan->client_id = ph->args[0]; - mchan->signal_id = ph->args[1]; - chan->con_priv = mchan; - break; - } + for (chan_id = 0; chan_id < mbox->num_chans; chan_id++) { + chan = &ipcc->chans[chan_id]; + mchan = chan->con_priv; - chan = NULL; + if (!mchan) + break; + else if (mchan->client_id == ph->args[0] && + mchan->signal_id == ph->args[1]) + return ERR_PTR(-EBUSY); } - return chan ?: ERR_PTR(-EBUSY); + if (chan_id >= mbox->num_chans) + return ERR_PTR(-EBUSY); + + mchan = devm_kzalloc(dev, sizeof(*mchan), GFP_KERNEL); + if (!mchan) + return ERR_PTR(-ENOMEM); + + mchan->client_id = ph->args[0]; + mchan->signal_id = ph->args[1]; + chan->con_priv = mchan; + + return chan; } static const struct mbox_chan_ops ipcc_mbox_chan_ops = { @@ -192,15 +203,49 @@ static const struct mbox_chan_ops ipcc_mbox_chan_ops = { .shutdown = qcom_ipcc_mbox_shutdown, }; -static int qcom_ipcc_setup_mbox(struct qcom_ipcc *ipcc) +static int qcom_ipcc_setup_mbox(struct qcom_ipcc *ipcc, + struct device_node *controller_dn) { + struct of_phandle_args curr_ph; + struct device_node *client_dn; struct mbox_controller *mbox; struct device *dev = ipcc->dev; + int i, j, ret; + + /* + * Find out the number of clients interested in this mailbox + * and create channels accordingly. + */ + ipcc->num_chans = 0; + for_each_node_with_property(client_dn, "mboxes") { + if (!of_device_is_available(client_dn)) + continue; + i = of_count_phandle_with_args(client_dn, + "mboxes", "#mbox-cells"); + for (j = 0; j < i; j++) { + ret = of_parse_phandle_with_args(client_dn, "mboxes", + "#mbox-cells", j, &curr_ph); + of_node_put(curr_ph.np); + if (!ret && curr_ph.np == controller_dn) { + ipcc->num_chans++; + break; + } + } + } + + /* If no clients are found, skip registering as a mbox controller */ + if (!ipcc->num_chans) + return 0; + + ipcc->chans = devm_kcalloc(dev, ipcc->num_chans, + sizeof(struct mbox_chan), GFP_KERNEL); + if (!ipcc->chans) + return -ENOMEM; mbox = &ipcc->mbox; mbox->dev = dev; - mbox->num_chans = IPCC_MBOX_MAX_CHAN; - mbox->chans = ipcc->chan; + mbox->num_chans = ipcc->num_chans; + mbox->chans = ipcc->chans; mbox->ops = &ipcc_mbox_chan_ops; mbox->of_xlate = qcom_ipcc_mbox_xlate; mbox->txdone_irq = false; @@ -212,6 +257,8 @@ static int qcom_ipcc_setup_mbox(struct qcom_ipcc *ipcc) static int qcom_ipcc_probe(struct platform_device *pdev) { struct qcom_ipcc *ipcc; + static int id; + char *name; int ret; ipcc = devm_kzalloc(&pdev->dev, sizeof(*ipcc), GFP_KERNEL); @@ -228,27 +275,33 @@ static int qcom_ipcc_probe(struct platform_device *pdev) if (ipcc->irq < 0) return ipcc->irq; + name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "ipcc_%d", id++); + if (!name) + return -ENOMEM; + ipcc->irq_domain = irq_domain_add_tree(pdev->dev.of_node, &qcom_ipcc_irq_ops, ipcc); if (!ipcc->irq_domain) return -ENOMEM; - ret = qcom_ipcc_setup_mbox(ipcc); + ret = qcom_ipcc_setup_mbox(ipcc, pdev->dev.of_node); if (ret) goto err_mbox; ret = devm_request_irq(&pdev->dev, ipcc->irq, qcom_ipcc_irq_fn, - IRQF_TRIGGER_HIGH, "ipcc", ipcc); + IRQF_TRIGGER_HIGH | IRQF_NO_SUSPEND, name, ipcc); if (ret < 0) { dev_err(&pdev->dev, "Failed to register the irq: %d\n", ret); - goto err_mbox; + goto err_req_irq; } - enable_irq_wake(ipcc->irq); platform_set_drvdata(pdev, ipcc); return 0; +err_req_irq: + if (ipcc->num_chans) + mbox_controller_unregister(&ipcc->mbox); err_mbox: irq_domain_remove(ipcc->irq_domain); diff --git a/drivers/mailbox/zynqmp-ipi-mailbox.c b/drivers/mailbox/zynqmp-ipi-mailbox.c index f44079d62b1a..31a0fa914274 100644 --- a/drivers/mailbox/zynqmp-ipi-mailbox.c +++ b/drivers/mailbox/zynqmp-ipi-mailbox.c @@ -655,6 +655,7 @@ static int zynqmp_ipi_probe(struct platform_device *pdev) mbox->pdata = pdata; ret = zynqmp_ipi_mbox_probe(mbox, nc); if (ret) { + of_node_put(nc); dev_err(dev, "failed to probe subdev.\n"); ret = -EINVAL; goto free_mbox_dev; diff --git a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig index 5af8494c31b5..52b0b27a6839 100644 --- a/drivers/mmc/host/Kconfig +++ b/drivers/mmc/host/Kconfig @@ -966,6 +966,7 @@ config MMC_REALTEK_USB config MMC_SUNXI tristate "Allwinner sunxi SD/MMC Host Controller support" depends on ARCH_SUNXI || COMPILE_TEST + depends on SUNXI_CCU help This selects support for the SD/MMC Host Controller on Allwinner sunxi SoCs. diff --git a/drivers/mmc/host/renesas_sdhi.h b/drivers/mmc/host/renesas_sdhi.h index 0c45e82ff0de..66d308e73e17 100644 --- a/drivers/mmc/host/renesas_sdhi.h +++ b/drivers/mmc/host/renesas_sdhi.h @@ -18,6 +18,8 @@ struct renesas_sdhi_scc { u32 tap_hs400_4tap; /* sampling clock position for HS400 (4 TAP) */ }; +#define SDHI_FLAG_NEED_CLKH_FALLBACK BIT(0) + struct renesas_sdhi_of_data { unsigned long tmio_flags; u32 tmio_ocr_mask; @@ -31,6 +33,7 @@ struct renesas_sdhi_of_data { int taps_num; unsigned int max_blk_count; unsigned short max_segs; + unsigned long sdhi_flags; }; #define SDHI_CALIB_TABLE_MAX 32 @@ -57,6 +60,7 @@ struct tmio_mmc_dma { struct renesas_sdhi { struct clk *clk; + struct clk *clkh; struct clk *clk_cd; struct tmio_mmc_data mmc_data; struct tmio_mmc_dma dma_priv; diff --git a/drivers/mmc/host/renesas_sdhi_core.c b/drivers/mmc/host/renesas_sdhi_core.c index f5b2684ad805..084c61b2cbec 100644 --- a/drivers/mmc/host/renesas_sdhi_core.c +++ b/drivers/mmc/host/renesas_sdhi_core.c @@ -127,10 +127,12 @@ static int renesas_sdhi_clk_enable(struct tmio_mmc_host *host) } static unsigned int renesas_sdhi_clk_update(struct tmio_mmc_host *host, - unsigned int new_clock) + unsigned int wanted_clock) { struct renesas_sdhi *priv = host_to_priv(host); + struct clk *ref_clk = priv->clk; unsigned int freq, diff, best_freq = 0, diff_min = ~0; + unsigned int new_clock, clkh_shift = 0; int i; /* @@ -141,6 +143,16 @@ static unsigned int renesas_sdhi_clk_update(struct tmio_mmc_host *host, if (!(host->pdata->flags & TMIO_MMC_MIN_RCAR2) || mmc_doing_tune(host->mmc)) return clk_get_rate(priv->clk); + if (priv->clkh) { + bool use_4tap = priv->quirks && priv->quirks->hs400_4taps; + bool need_slow_clkh = (host->mmc->ios.timing == MMC_TIMING_UHS_SDR104) || + (host->mmc->ios.timing == MMC_TIMING_MMC_HS400); + clkh_shift = use_4tap && need_slow_clkh ? 1 : 2; + ref_clk = priv->clkh; + } + + new_clock = wanted_clock << clkh_shift; + /* * We want the bus clock to be as close as possible to, but no * greater than, new_clock. As we can divide by 1 << i for @@ -148,11 +160,10 @@ static unsigned int renesas_sdhi_clk_update(struct tmio_mmc_host *host, * possible, but no greater than, new_clock << i. */ for (i = min(9, ilog2(UINT_MAX / new_clock)); i >= 0; i--) { - freq = clk_round_rate(priv->clk, new_clock << i); + freq = clk_round_rate(ref_clk, new_clock << i); if (freq > (new_clock << i)) { /* Too fast; look for a slightly slower option */ - freq = clk_round_rate(priv->clk, - (new_clock << i) / 4 * 3); + freq = clk_round_rate(ref_clk, (new_clock << i) / 4 * 3); if (freq > (new_clock << i)) continue; } @@ -164,7 +175,10 @@ static unsigned int renesas_sdhi_clk_update(struct tmio_mmc_host *host, } } - clk_set_rate(priv->clk, best_freq); + clk_set_rate(ref_clk, best_freq); + + if (priv->clkh) + clk_set_rate(priv->clk, best_freq >> clkh_shift); return clk_get_rate(priv->clk); } @@ -904,11 +918,12 @@ int renesas_sdhi_probe(struct platform_device *pdev, dma_priv = &priv->dma_priv; priv->clk = devm_clk_get(&pdev->dev, NULL); - if (IS_ERR(priv->clk)) { - ret = PTR_ERR(priv->clk); - dev_err(&pdev->dev, "cannot get clock: %d\n", ret); - return ret; - } + if (IS_ERR(priv->clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(priv->clk), "cannot get clock"); + + priv->clkh = devm_clk_get_optional(&pdev->dev, "clkh"); + if (IS_ERR(priv->clkh)) + return dev_err_probe(&pdev->dev, PTR_ERR(priv->clkh), "cannot get clkh"); /* * Some controllers provide a 2nd clock just to run the internal card @@ -921,9 +936,9 @@ int renesas_sdhi_probe(struct platform_device *pdev, * to the card detect circuit. That leaves us with if separate clocks * are presented, we must treat them both as virtually 1 clock. */ - priv->clk_cd = devm_clk_get(&pdev->dev, "cd"); + priv->clk_cd = devm_clk_get_optional(&pdev->dev, "cd"); if (IS_ERR(priv->clk_cd)) - priv->clk_cd = NULL; + return dev_err_probe(&pdev->dev, PTR_ERR(priv->clk_cd), "cannot get cd clock"); priv->pinctrl = devm_pinctrl_get(&pdev->dev); if (!IS_ERR(priv->pinctrl)) { @@ -947,6 +962,10 @@ int renesas_sdhi_probe(struct platform_device *pdev, mmc_data->max_segs = of_data->max_segs; dma_priv->dma_buswidth = of_data->dma_buswidth; host->bus_shift = of_data->bus_shift; + /* Fallback for old DTs */ + if (!priv->clkh && of_data->sdhi_flags & SDHI_FLAG_NEED_CLKH_FALLBACK) + priv->clkh = clk_get_parent(clk_get_parent(priv->clk)); + } host->write16_hook = renesas_sdhi_write16_hook; @@ -1044,7 +1063,7 @@ int renesas_sdhi_probe(struct platform_device *pdev, host->mmc->caps2 & (MMC_CAP2_HS200_1_8V_SDR | MMC_CAP2_HS400_1_8V))) { const struct renesas_sdhi_scc *taps = of_data->taps; - bool use_4tap = priv->quirks && priv->quirks->hs400_4taps; + bool use_4tap = quirks && quirks->hs400_4taps; bool hit = false; for (i = 0; i < of_data->taps_num; i++) { diff --git a/drivers/mmc/host/renesas_sdhi_internal_dmac.c b/drivers/mmc/host/renesas_sdhi_internal_dmac.c index 7660f7ea74dd..9d2c600fd4ce 100644 --- a/drivers/mmc/host/renesas_sdhi_internal_dmac.c +++ b/drivers/mmc/host/renesas_sdhi_internal_dmac.c @@ -125,6 +125,22 @@ static const struct renesas_sdhi_of_data of_data_rcar_gen3 = { /* DMAC can handle 32bit blk count but only 1 segment */ .max_blk_count = UINT_MAX / TMIO_MAX_BLK_SIZE, .max_segs = 1, + .sdhi_flags = SDHI_FLAG_NEED_CLKH_FALLBACK, +}; + +static const struct renesas_sdhi_of_data of_data_rcar_gen3_no_fallback = { + .tmio_flags = TMIO_MMC_HAS_IDLE_WAIT | TMIO_MMC_CLK_ACTUAL | + TMIO_MMC_HAVE_CBSY | TMIO_MMC_MIN_RCAR2, + .capabilities = MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ | + MMC_CAP_CMD23 | MMC_CAP_WAIT_WHILE_BUSY, + .capabilities2 = MMC_CAP2_NO_WRITE_PROTECT | MMC_CAP2_MERGE_CAPABLE, + .bus_shift = 2, + .scc_offset = 0x1000, + .taps = rcar_gen3_scc_taps, + .taps_num = ARRAY_SIZE(rcar_gen3_scc_taps), + /* DMAC can handle 32bit blk count but only 1 segment */ + .max_blk_count = UINT_MAX / TMIO_MAX_BLK_SIZE, + .max_segs = 1, }; static const u8 r8a7796_es13_calib_table[2][SDHI_CALIB_TABLE_MAX] = { @@ -214,6 +230,10 @@ static const struct renesas_sdhi_of_data_with_quirks of_r8a77965_compatible = { .quirks = &sdhi_quirks_r8a77965, }; +static const struct renesas_sdhi_of_data_with_quirks of_r8a77970_compatible = { + .of_data = &of_data_rcar_gen3_no_fallback, +}; + static const struct renesas_sdhi_of_data_with_quirks of_r8a77980_compatible = { .of_data = &of_data_rcar_gen3, .quirks = &sdhi_quirks_nohs400, @@ -235,6 +255,7 @@ static const struct of_device_id renesas_sdhi_internal_dmac_of_match[] = { { .compatible = "renesas,sdhi-r8a7796", .data = &of_rcar_gen3_compatible, }, { .compatible = "renesas,sdhi-r8a77961", .data = &of_r8a77961_compatible, }, { .compatible = "renesas,sdhi-r8a77965", .data = &of_r8a77965_compatible, }, + { .compatible = "renesas,sdhi-r8a77970", .data = &of_r8a77970_compatible, }, { .compatible = "renesas,sdhi-r8a77980", .data = &of_r8a77980_compatible, }, { .compatible = "renesas,sdhi-r8a77990", .data = &of_r8a77990_compatible, }, { .compatible = "renesas,rcar-gen3-sdhi", .data = &of_rcar_gen3_compatible, }, diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h b/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h index fed5f93bf620..26433a62d7f0 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h +++ b/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h @@ -497,7 +497,7 @@ struct cpl_t5_pass_accept_rpl { __be32 opt2; __be64 opt0; __be32 iss; - __be32 rsvd; + __be32 rsvd[3]; }; struct cpl_act_open_req { diff --git a/drivers/net/ethernet/cisco/enic/enic_main.c b/drivers/net/ethernet/cisco/enic/enic_main.c index 4db6889b79ba..1c81b161de52 100644 --- a/drivers/net/ethernet/cisco/enic/enic_main.c +++ b/drivers/net/ethernet/cisco/enic/enic_main.c @@ -150,10 +150,10 @@ static void enic_set_affinity_hint(struct enic *enic) !cpumask_available(enic->msix[i].affinity_mask) || cpumask_empty(enic->msix[i].affinity_mask)) continue; - err = irq_set_affinity_hint(enic->msix_entry[i].vector, - enic->msix[i].affinity_mask); + err = irq_update_affinity_hint(enic->msix_entry[i].vector, + enic->msix[i].affinity_mask); if (err) - netdev_warn(enic->netdev, "irq_set_affinity_hint failed, err %d\n", + netdev_warn(enic->netdev, "irq_update_affinity_hint failed, err %d\n", err); } @@ -173,7 +173,7 @@ static void enic_unset_affinity_hint(struct enic *enic) int i; for (i = 0; i < enic->intr_count; i++) - irq_set_affinity_hint(enic->msix_entry[i].vector, NULL); + irq_update_affinity_hint(enic->msix_entry[i].vector, NULL); } static int enic_udp_tunnel_set_port(struct net_device *netdev, diff --git a/drivers/net/ethernet/emulex/benet/be_main.c b/drivers/net/ethernet/emulex/benet/be_main.c index ad67b4216079..d0c262f2695a 100644 --- a/drivers/net/ethernet/emulex/benet/be_main.c +++ b/drivers/net/ethernet/emulex/benet/be_main.c @@ -3491,7 +3491,7 @@ static int be_msix_register(struct be_adapter *adapter) if (status) goto err_msix; - irq_set_affinity_hint(vec, eqo->affinity_mask); + irq_update_affinity_hint(vec, eqo->affinity_mask); } return 0; @@ -3552,7 +3552,7 @@ static void be_irq_unregister(struct be_adapter *adapter) /* MSIx */ for_all_evt_queues(adapter, eqo, i) { vec = be_msix_vec_get(adapter, eqo); - irq_set_affinity_hint(vec, NULL); + irq_update_affinity_hint(vec, NULL); free_irq(vec, eqo); } diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c index d21ba70ef4a3..e985ae008a97 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c @@ -4246,7 +4246,7 @@ static int dpaa2_eth_setup_irqs(struct fsl_mc_device *ls_dev) } irq = ls_dev->irqs[0]; - err = devm_request_threaded_irq(&ls_dev->dev, irq->msi_desc->irq, + err = devm_request_threaded_irq(&ls_dev->dev, irq->virq, NULL, dpni_irq0_handler_thread, IRQF_NO_SUSPEND | IRQF_ONESHOT, dev_name(&ls_dev->dev), &ls_dev->dev); @@ -4273,7 +4273,7 @@ static int dpaa2_eth_setup_irqs(struct fsl_mc_device *ls_dev) return 0; free_irq: - devm_free_irq(&ls_dev->dev, irq->msi_desc->irq, &ls_dev->dev); + devm_free_irq(&ls_dev->dev, irq->virq, &ls_dev->dev); free_mc_irq: fsl_mc_free_irqs(ls_dev); diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-ptp.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-ptp.c index 32b5faa87bb8..5f5f8c53c4a0 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-ptp.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-ptp.c @@ -129,7 +129,6 @@ static irqreturn_t dpaa2_ptp_irq_handler_thread(int irq, void *priv) static int dpaa2_ptp_probe(struct fsl_mc_device *mc_dev) { struct device *dev = &mc_dev->dev; - struct fsl_mc_device_irq *irq; struct ptp_qoriq *ptp_qoriq; struct device_node *node; void __iomem *base; @@ -177,8 +176,7 @@ static int dpaa2_ptp_probe(struct fsl_mc_device *mc_dev) goto err_unmap; } - irq = mc_dev->irqs[0]; - ptp_qoriq->irq = irq->msi_desc->irq; + ptp_qoriq->irq = mc_dev->irqs[0]->virq; err = request_threaded_irq(ptp_qoriq->irq, NULL, dpaa2_ptp_irq_handler_thread, diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c index 9b5512b4f15d..9a561072aa4a 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c @@ -1554,8 +1554,7 @@ static int dpaa2_switch_setup_irqs(struct fsl_mc_device *sw_dev) irq = sw_dev->irqs[DPSW_IRQ_INDEX_IF]; - err = devm_request_threaded_irq(dev, irq->msi_desc->irq, - NULL, + err = devm_request_threaded_irq(dev, irq->virq, NULL, dpaa2_switch_irq0_handler_thread, IRQF_NO_SUSPEND | IRQF_ONESHOT, dev_name(dev), dev); @@ -1581,7 +1580,7 @@ static int dpaa2_switch_setup_irqs(struct fsl_mc_device *sw_dev) return 0; free_devm_irq: - devm_free_irq(dev, irq->msi_desc->irq, dev); + devm_free_irq(dev, irq->virq, dev); free_irq: fsl_mc_free_irqs(sw_dev); return err; diff --git a/drivers/net/ethernet/huawei/hinic/hinic_rx.c b/drivers/net/ethernet/huawei/hinic/hinic_rx.c index fed3b6bc0d76..b33ed4d92b71 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_rx.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_rx.c @@ -548,7 +548,7 @@ static int rx_request_irq(struct hinic_rxq *rxq) goto err_req_irq; cpumask_set_cpu(qp->q_id % num_online_cpus(), &rq->affinity_mask); - err = irq_set_affinity_hint(rq->irq, &rq->affinity_mask); + err = irq_set_affinity_and_hint(rq->irq, &rq->affinity_mask); if (err) goto err_irq_affinity; @@ -565,7 +565,7 @@ static void rx_free_irq(struct hinic_rxq *rxq) { struct hinic_rq *rq = rxq->rq; - irq_set_affinity_hint(rq->irq, NULL); + irq_update_affinity_hint(rq->irq, NULL); free_irq(rq->irq, rxq); rx_del_napi(rxq); } diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c index 61afc220fc6c..2a3d8aef7f4e 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_main.c +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c @@ -3915,10 +3915,10 @@ static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename) * * get_cpu_mask returns a static constant mask with * a permanent lifetime so it's ok to pass to - * irq_set_affinity_hint without making a copy. + * irq_update_affinity_hint without making a copy. */ cpu = cpumask_local_spread(q_vector->v_idx, -1); - irq_set_affinity_hint(irq_num, get_cpu_mask(cpu)); + irq_update_affinity_hint(irq_num, get_cpu_mask(cpu)); } vsi->irqs_ready = true; @@ -3929,7 +3929,7 @@ free_queue_irqs: vector--; irq_num = pf->msix_entries[base + vector].vector; irq_set_affinity_notifier(irq_num, NULL); - irq_set_affinity_hint(irq_num, NULL); + irq_update_affinity_hint(irq_num, NULL); free_irq(irq_num, &vsi->q_vectors[vector]); } return err; @@ -4750,7 +4750,7 @@ static void i40e_vsi_free_irq(struct i40e_vsi *vsi) /* clear the affinity notifier in the IRQ descriptor */ irq_set_affinity_notifier(irq_num, NULL); /* remove our suggested affinity mask for this IRQ */ - irq_set_affinity_hint(irq_num, NULL); + irq_update_affinity_hint(irq_num, NULL); synchronize_irq(irq_num); free_irq(irq_num, vsi->q_vectors[i]); diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c b/drivers/net/ethernet/intel/iavf/iavf_main.c index 95116ef2f0ae..8125b9120615 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_main.c +++ b/drivers/net/ethernet/intel/iavf/iavf_main.c @@ -492,10 +492,10 @@ iavf_request_traffic_irqs(struct iavf_adapter *adapter, char *basename) irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify); /* Spread the IRQ affinity hints across online CPUs. Note that * get_cpu_mask returns a mask with a permanent lifetime so - * it's safe to use as a hint for irq_set_affinity_hint. + * it's safe to use as a hint for irq_update_affinity_hint. */ cpu = cpumask_local_spread(q_vector->v_idx, -1); - irq_set_affinity_hint(irq_num, get_cpu_mask(cpu)); + irq_update_affinity_hint(irq_num, get_cpu_mask(cpu)); } return 0; @@ -505,7 +505,7 @@ free_queue_irqs: vector--; irq_num = adapter->msix_entries[vector + NONQ_VECS].vector; irq_set_affinity_notifier(irq_num, NULL); - irq_set_affinity_hint(irq_num, NULL); + irq_update_affinity_hint(irq_num, NULL); free_irq(irq_num, &adapter->q_vectors[vector]); } return err; @@ -557,7 +557,7 @@ static void iavf_free_traffic_irqs(struct iavf_adapter *adapter) for (vector = 0; vector < q_vectors; vector++) { irq_num = adapter->msix_entries[vector + NONQ_VECS].vector; irq_set_affinity_notifier(irq_num, NULL); - irq_set_affinity_hint(irq_num, NULL); + irq_update_affinity_hint(irq_num, NULL); free_irq(irq_num, &adapter->q_vectors[vector]); } } diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index c6ff656b2476..89b467006291 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -3247,8 +3247,8 @@ static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter) /* If Flow Director is enabled, set interrupt affinity */ if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) { /* assign the mask for this irq */ - irq_set_affinity_hint(entry->vector, - &q_vector->affinity_mask); + irq_update_affinity_hint(entry->vector, + &q_vector->affinity_mask); } } @@ -3264,8 +3264,8 @@ static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter) free_queue_irqs: while (vector) { vector--; - irq_set_affinity_hint(adapter->msix_entries[vector].vector, - NULL); + irq_update_affinity_hint(adapter->msix_entries[vector].vector, + NULL); free_irq(adapter->msix_entries[vector].vector, adapter->q_vector[vector]); } @@ -3398,7 +3398,7 @@ static void ixgbe_free_irq(struct ixgbe_adapter *adapter) continue; /* clear the affinity_mask in the IRQ descriptor */ - irq_set_affinity_hint(entry->vector, NULL); + irq_update_affinity_hint(entry->vector, NULL); free_irq(entry->vector, q_vector); } diff --git a/drivers/net/ethernet/mellanox/mlx4/eq.c b/drivers/net/ethernet/mellanox/mlx4/eq.c index 9e48509ed3b2..414e390e6b48 100644 --- a/drivers/net/ethernet/mellanox/mlx4/eq.c +++ b/drivers/net/ethernet/mellanox/mlx4/eq.c @@ -244,9 +244,9 @@ static void mlx4_set_eq_affinity_hint(struct mlx4_priv *priv, int vec) cpumask_empty(eq->affinity_mask)) return; - hint_err = irq_set_affinity_hint(eq->irq, eq->affinity_mask); + hint_err = irq_update_affinity_hint(eq->irq, eq->affinity_mask); if (hint_err) - mlx4_warn(dev, "irq_set_affinity_hint failed, err %d\n", hint_err); + mlx4_warn(dev, "irq_update_affinity_hint failed, err %d\n", hint_err); } #endif @@ -1123,9 +1123,7 @@ static void mlx4_free_irqs(struct mlx4_dev *dev) for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) if (eq_table->eq[i].have_irq) { free_cpumask_var(eq_table->eq[i].affinity_mask); -#if defined(CONFIG_SMP) - irq_set_affinity_hint(eq_table->eq[i].irq, NULL); -#endif + irq_update_affinity_hint(eq_table->eq[i].irq, NULL); free_irq(eq_table->eq[i].irq, eq_table->eq + i); eq_table->eq[i].have_irq = 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c index 90fec0649ef5..41807ef55201 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c @@ -129,11 +129,11 @@ static void irq_release(struct mlx5_irq *irq) struct mlx5_irq_pool *pool = irq->pool; xa_erase(&pool->irqs, irq->index); - /* free_irq requires that affinity and rmap will be cleared + /* free_irq requires that affinity_hint and rmap will be cleared * before calling it. This is why there is asymmetry with set_rmap * which should be called after alloc_irq but before request_irq. */ - irq_set_affinity_hint(irq->irqn, NULL); + irq_update_affinity_hint(irq->irqn, NULL); free_cpumask_var(irq->mask); free_irq(irq->irqn, &irq->nh); kfree(irq); @@ -238,7 +238,7 @@ struct mlx5_irq *mlx5_irq_alloc(struct mlx5_irq_pool *pool, int i, } if (affinity) { cpumask_copy(irq->mask, affinity); - irq_set_affinity_hint(irq->irqn, irq->mask); + irq_set_affinity_and_hint(irq->irqn, irq->mask); } irq->pool = pool; irq->refcount = 1; @@ -251,7 +251,7 @@ struct mlx5_irq *mlx5_irq_alloc(struct mlx5_irq_pool *pool, int i, } return irq; err_xa: - irq_set_affinity_hint(irq->irqn, NULL); + irq_update_affinity_hint(irq->irqn, NULL); free_cpumask_var(irq->mask); err_cpumask: free_irq(irq->irqn, &irq->nh); diff --git a/drivers/net/wireless/ath/ath11k/pci.c b/drivers/net/wireless/ath/ath11k/pci.c index d73b522a0081..de71ad594f34 100644 --- a/drivers/net/wireless/ath/ath11k/pci.c +++ b/drivers/net/wireless/ath/ath11k/pci.c @@ -1014,7 +1014,7 @@ static int ath11k_pci_alloc_msi(struct ath11k_pci *ab_pci) } ab_pci->msi_ep_base_data = msi_desc->msg.data; - if (msi_desc->msi_attrib.is_64) + if (msi_desc->pci.msi_attrib.is_64) set_bit(ATH11K_PCI_FLAG_IS_MSI_64, &ab_pci->flags); ath11k_dbg(ab, ATH11K_DBG_PCI, "msi base data is %d\n", ab_pci->msi_ep_base_data); diff --git a/drivers/ntb/msi.c b/drivers/ntb/msi.c index 3f05cfbc73af..2818bfcf4031 100644 --- a/drivers/ntb/msi.c +++ b/drivers/ntb/msi.c @@ -108,8 +108,10 @@ int ntb_msi_setup_mws(struct ntb_dev *ntb) if (!ntb->msi) return -EINVAL; - desc = first_msi_entry(&ntb->pdev->dev); + msi_lock_descs(&ntb->pdev->dev); + desc = msi_first_desc(&ntb->pdev->dev, MSI_DESC_ASSOCIATED); addr = desc->msg.address_lo + ((uint64_t)desc->msg.address_hi << 32); + msi_unlock_descs(&ntb->pdev->dev); for (peer = 0; peer < ntb_peer_port_count(ntb); peer++) { peer_widx = ntb_peer_highest_mw_idx(ntb, peer); @@ -281,13 +283,15 @@ int ntbm_msi_request_threaded_irq(struct ntb_dev *ntb, irq_handler_t handler, const char *name, void *dev_id, struct ntb_msi_desc *msi_desc) { + struct device *dev = &ntb->pdev->dev; struct msi_desc *entry; int ret; if (!ntb->msi) return -EINVAL; - for_each_pci_msi_entry(entry, ntb->pdev) { + msi_lock_descs(dev); + msi_for_each_desc(entry, dev, MSI_DESC_ASSOCIATED) { if (irq_has_action(entry->irq)) continue; @@ -304,14 +308,17 @@ int ntbm_msi_request_threaded_irq(struct ntb_dev *ntb, irq_handler_t handler, ret = ntbm_msi_setup_callback(ntb, entry, msi_desc); if (ret) { devm_free_irq(&ntb->dev, entry->irq, dev_id); - return ret; + goto unlock; } - - return entry->irq; + ret = entry->irq; + goto unlock; } + ret = -ENODEV; - return -ENODEV; +unlock: + msi_unlock_descs(dev); + return ret; } EXPORT_SYMBOL(ntbm_msi_request_threaded_irq); diff --git a/drivers/pci/Makefile b/drivers/pci/Makefile index d62c4ac4ae1b..37be95adf169 100644 --- a/drivers/pci/Makefile +++ b/drivers/pci/Makefile @@ -5,8 +5,9 @@ obj-$(CONFIG_PCI) += access.o bus.o probe.o host-bridge.o \ remove.o pci.o pci-driver.o search.o \ pci-sysfs.o rom.o setup-res.o irq.o vpd.o \ - setup-bus.o vc.o mmap.o setup-irq.o msi.o + setup-bus.o vc.o mmap.o setup-irq.o +obj-$(CONFIG_PCI) += msi/ obj-$(CONFIG_PCI) += pcie/ ifdef CONFIG_PCI diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c index 6733cb14e775..45b6ff8dc665 100644 --- a/drivers/pci/controller/pci-hyperv.c +++ b/drivers/pci/controller/pci-hyperv.c @@ -3445,18 +3445,23 @@ static int hv_pci_suspend(struct hv_device *hdev) static int hv_pci_restore_msi_msg(struct pci_dev *pdev, void *arg) { - struct msi_desc *entry; struct irq_data *irq_data; + struct msi_desc *entry; + int ret = 0; - for_each_pci_msi_entry(entry, pdev) { + msi_lock_descs(&pdev->dev); + msi_for_each_desc(entry, &pdev->dev, MSI_DESC_ASSOCIATED) { irq_data = irq_get_irq_data(entry->irq); - if (WARN_ON_ONCE(!irq_data)) - return -EINVAL; + if (WARN_ON_ONCE(!irq_data)) { + ret = -EINVAL; + break; + } hv_compose_msi_msg(irq_data, &entry->msg); } + msi_unlock_descs(&pdev->dev); - return 0; + return ret; } /* diff --git a/drivers/pci/msi/Makefile b/drivers/pci/msi/Makefile new file mode 100644 index 000000000000..93ef7b9e404d --- /dev/null +++ b/drivers/pci/msi/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the PCI/MSI +obj-$(CONFIG_PCI) += pcidev_msi.o +obj-$(CONFIG_PCI_MSI) += msi.o +obj-$(CONFIG_PCI_MSI_IRQ_DOMAIN) += irqdomain.o +obj-$(CONFIG_PCI_MSI_ARCH_FALLBACKS) += legacy.o diff --git a/drivers/pci/msi/irqdomain.c b/drivers/pci/msi/irqdomain.c new file mode 100644 index 000000000000..0d63541c4052 --- /dev/null +++ b/drivers/pci/msi/irqdomain.c @@ -0,0 +1,280 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCI Message Signaled Interrupt (MSI) - irqdomain support + */ +#include +#include +#include + +#include "msi.h" + +int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) +{ + struct irq_domain *domain; + + domain = dev_get_msi_domain(&dev->dev); + if (domain && irq_domain_is_hierarchy(domain)) + return msi_domain_alloc_irqs_descs_locked(domain, &dev->dev, nvec); + + return pci_msi_legacy_setup_msi_irqs(dev, nvec, type); +} + +void pci_msi_teardown_msi_irqs(struct pci_dev *dev) +{ + struct irq_domain *domain; + + domain = dev_get_msi_domain(&dev->dev); + if (domain && irq_domain_is_hierarchy(domain)) + msi_domain_free_irqs_descs_locked(domain, &dev->dev); + else + pci_msi_legacy_teardown_msi_irqs(dev); +} + +/** + * pci_msi_domain_write_msg - Helper to write MSI message to PCI config space + * @irq_data: Pointer to interrupt data of the MSI interrupt + * @msg: Pointer to the message + */ +static void pci_msi_domain_write_msg(struct irq_data *irq_data, struct msi_msg *msg) +{ + struct msi_desc *desc = irq_data_get_msi_desc(irq_data); + + /* + * For MSI-X desc->irq is always equal to irq_data->irq. For + * MSI only the first interrupt of MULTI MSI passes the test. + */ + if (desc->irq == irq_data->irq) + __pci_write_msi_msg(desc, msg); +} + +/** + * pci_msi_domain_calc_hwirq - Generate a unique ID for an MSI source + * @desc: Pointer to the MSI descriptor + * + * The ID number is only used within the irqdomain. + */ +static irq_hw_number_t pci_msi_domain_calc_hwirq(struct msi_desc *desc) +{ + struct pci_dev *dev = msi_desc_to_pci_dev(desc); + + return (irq_hw_number_t)desc->msi_index | + pci_dev_id(dev) << 11 | + (pci_domain_nr(dev->bus) & 0xFFFFFFFF) << 27; +} + +static inline bool pci_msi_desc_is_multi_msi(struct msi_desc *desc) +{ + return !desc->pci.msi_attrib.is_msix && desc->nvec_used > 1; +} + +/** + * pci_msi_domain_check_cap - Verify that @domain supports the capabilities + * for @dev + * @domain: The interrupt domain to check + * @info: The domain info for verification + * @dev: The device to check + * + * Returns: + * 0 if the functionality is supported + * 1 if Multi MSI is requested, but the domain does not support it + * -ENOTSUPP otherwise + */ +static int pci_msi_domain_check_cap(struct irq_domain *domain, + struct msi_domain_info *info, + struct device *dev) +{ + struct msi_desc *desc = msi_first_desc(dev, MSI_DESC_ALL); + + /* Special handling to support __pci_enable_msi_range() */ + if (pci_msi_desc_is_multi_msi(desc) && + !(info->flags & MSI_FLAG_MULTI_PCI_MSI)) + return 1; + + if (desc->pci.msi_attrib.is_msix) { + if (!(info->flags & MSI_FLAG_PCI_MSIX)) + return -ENOTSUPP; + + if (info->flags & MSI_FLAG_MSIX_CONTIGUOUS) { + unsigned int idx = 0; + + /* Check for gaps in the entry indices */ + msi_for_each_desc(desc, dev, MSI_DESC_ALL) { + if (desc->msi_index != idx++) + return -ENOTSUPP; + } + } + } + return 0; +} + +static void pci_msi_domain_set_desc(msi_alloc_info_t *arg, + struct msi_desc *desc) +{ + arg->desc = desc; + arg->hwirq = pci_msi_domain_calc_hwirq(desc); +} + +static struct msi_domain_ops pci_msi_domain_ops_default = { + .set_desc = pci_msi_domain_set_desc, + .msi_check = pci_msi_domain_check_cap, +}; + +static void pci_msi_domain_update_dom_ops(struct msi_domain_info *info) +{ + struct msi_domain_ops *ops = info->ops; + + if (ops == NULL) { + info->ops = &pci_msi_domain_ops_default; + } else { + if (ops->set_desc == NULL) + ops->set_desc = pci_msi_domain_set_desc; + if (ops->msi_check == NULL) + ops->msi_check = pci_msi_domain_check_cap; + } +} + +static void pci_msi_domain_update_chip_ops(struct msi_domain_info *info) +{ + struct irq_chip *chip = info->chip; + + BUG_ON(!chip); + if (!chip->irq_write_msi_msg) + chip->irq_write_msi_msg = pci_msi_domain_write_msg; + if (!chip->irq_mask) + chip->irq_mask = pci_msi_mask_irq; + if (!chip->irq_unmask) + chip->irq_unmask = pci_msi_unmask_irq; +} + +/** + * pci_msi_create_irq_domain - Create a MSI interrupt domain + * @fwnode: Optional fwnode of the interrupt controller + * @info: MSI domain info + * @parent: Parent irq domain + * + * Updates the domain and chip ops and creates a MSI interrupt domain. + * + * Returns: + * A domain pointer or NULL in case of failure. + */ +struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode, + struct msi_domain_info *info, + struct irq_domain *parent) +{ + struct irq_domain *domain; + + if (WARN_ON(info->flags & MSI_FLAG_LEVEL_CAPABLE)) + info->flags &= ~MSI_FLAG_LEVEL_CAPABLE; + + if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS) + pci_msi_domain_update_dom_ops(info); + if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS) + pci_msi_domain_update_chip_ops(info); + + info->flags |= MSI_FLAG_ACTIVATE_EARLY | MSI_FLAG_DEV_SYSFS | + MSI_FLAG_FREE_MSI_DESCS; + if (IS_ENABLED(CONFIG_GENERIC_IRQ_RESERVATION_MODE)) + info->flags |= MSI_FLAG_MUST_REACTIVATE; + + /* PCI-MSI is oneshot-safe */ + info->chip->flags |= IRQCHIP_ONESHOT_SAFE; + + domain = msi_create_irq_domain(fwnode, info, parent); + if (!domain) + return NULL; + + irq_domain_update_bus_token(domain, DOMAIN_BUS_PCI_MSI); + return domain; +} +EXPORT_SYMBOL_GPL(pci_msi_create_irq_domain); + +/* + * Users of the generic MSI infrastructure expect a device to have a single ID, + * so with DMA aliases we have to pick the least-worst compromise. Devices with + * DMA phantom functions tend to still emit MSIs from the real function number, + * so we ignore those and only consider topological aliases where either the + * alias device or RID appears on a different bus number. We also make the + * reasonable assumption that bridges are walked in an upstream direction (so + * the last one seen wins), and the much braver assumption that the most likely + * case is that of PCI->PCIe so we should always use the alias RID. This echoes + * the logic from intel_irq_remapping's set_msi_sid(), which presumably works + * well enough in practice; in the face of the horrible PCIe<->PCI-X conditions + * for taking ownership all we can really do is close our eyes and hope... + */ +static int get_msi_id_cb(struct pci_dev *pdev, u16 alias, void *data) +{ + u32 *pa = data; + u8 bus = PCI_BUS_NUM(*pa); + + if (pdev->bus->number != bus || PCI_BUS_NUM(alias) != bus) + *pa = alias; + + return 0; +} + +/** + * pci_msi_domain_get_msi_rid - Get the MSI requester id (RID) + * @domain: The interrupt domain + * @pdev: The PCI device. + * + * The RID for a device is formed from the alias, with a firmware + * supplied mapping applied + * + * Returns: The RID. + */ +u32 pci_msi_domain_get_msi_rid(struct irq_domain *domain, struct pci_dev *pdev) +{ + struct device_node *of_node; + u32 rid = pci_dev_id(pdev); + + pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid); + + of_node = irq_domain_get_of_node(domain); + rid = of_node ? of_msi_map_id(&pdev->dev, of_node, rid) : + iort_msi_map_id(&pdev->dev, rid); + + return rid; +} + +/** + * pci_msi_get_device_domain - Get the MSI domain for a given PCI device + * @pdev: The PCI device + * + * Use the firmware data to find a device-specific MSI domain + * (i.e. not one that is set as a default). + * + * Returns: The corresponding MSI domain or NULL if none has been found. + */ +struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev) +{ + struct irq_domain *dom; + u32 rid = pci_dev_id(pdev); + + pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid); + dom = of_msi_map_get_device_domain(&pdev->dev, rid, DOMAIN_BUS_PCI_MSI); + if (!dom) + dom = iort_get_device_domain(&pdev->dev, rid, + DOMAIN_BUS_PCI_MSI); + return dom; +} + +/** + * pci_dev_has_special_msi_domain - Check whether the device is handled by + * a non-standard PCI-MSI domain + * @pdev: The PCI device to check. + * + * Returns: True if the device irqdomain or the bus irqdomain is + * non-standard PCI/MSI. + */ +bool pci_dev_has_special_msi_domain(struct pci_dev *pdev) +{ + struct irq_domain *dom = dev_get_msi_domain(&pdev->dev); + + if (!dom) + dom = dev_get_msi_domain(&pdev->bus->dev); + + if (!dom) + return true; + + return dom->bus_token != DOMAIN_BUS_PCI_MSI; +} diff --git a/drivers/pci/msi/legacy.c b/drivers/pci/msi/legacy.c new file mode 100644 index 000000000000..cdbb4689db78 --- /dev/null +++ b/drivers/pci/msi/legacy.c @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCI Message Signaled Interrupt (MSI). + * + * Legacy architecture specific setup and teardown mechanism. + */ +#include "msi.h" + +/* Arch hooks */ +int __weak arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc) +{ + return -EINVAL; +} + +void __weak arch_teardown_msi_irq(unsigned int irq) +{ +} + +int __weak arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) +{ + struct msi_desc *desc; + int ret; + + /* + * If an architecture wants to support multiple MSI, it needs to + * override arch_setup_msi_irqs() + */ + if (type == PCI_CAP_ID_MSI && nvec > 1) + return 1; + + msi_for_each_desc(desc, &dev->dev, MSI_DESC_NOTASSOCIATED) { + ret = arch_setup_msi_irq(dev, desc); + if (ret) + return ret < 0 ? ret : -ENOSPC; + } + + return 0; +} + +void __weak arch_teardown_msi_irqs(struct pci_dev *dev) +{ + struct msi_desc *desc; + int i; + + msi_for_each_desc(desc, &dev->dev, MSI_DESC_ASSOCIATED) { + for (i = 0; i < desc->nvec_used; i++) + arch_teardown_msi_irq(desc->irq + i); + } +} + +static int pci_msi_setup_check_result(struct pci_dev *dev, int type, int ret) +{ + struct msi_desc *desc; + int avail = 0; + + if (type != PCI_CAP_ID_MSIX || ret >= 0) + return ret; + + /* Scan the MSI descriptors for successfully allocated ones. */ + msi_for_each_desc(desc, &dev->dev, MSI_DESC_ASSOCIATED) + avail++; + + return avail ? avail : ret; +} + +int pci_msi_legacy_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) +{ + int ret = arch_setup_msi_irqs(dev, nvec, type); + + ret = pci_msi_setup_check_result(dev, type, ret); + if (!ret) + ret = msi_device_populate_sysfs(&dev->dev); + return ret; +} + +void pci_msi_legacy_teardown_msi_irqs(struct pci_dev *dev) +{ + msi_device_destroy_sysfs(&dev->dev); + arch_teardown_msi_irqs(dev); + msi_free_msi_descs(&dev->dev); +} diff --git a/drivers/pci/msi.c b/drivers/pci/msi/msi.c similarity index 56% rename from drivers/pci/msi.c rename to drivers/pci/msi/msi.c index d84cf30bb279..c19c7ca58186 100644 --- a/drivers/pci/msi.c +++ b/drivers/pci/msi/msi.c @@ -6,156 +6,29 @@ * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com) * Copyright (C) 2016 Christoph Hellwig. */ - #include -#include -#include -#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include -#include "pci.h" - -#ifdef CONFIG_PCI_MSI +#include "../pci.h" +#include "msi.h" static int pci_msi_enable = 1; int pci_msi_ignore_mask; -#define msix_table_size(flags) ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) - -#ifdef CONFIG_PCI_MSI_IRQ_DOMAIN -static int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) -{ - struct irq_domain *domain; - - domain = dev_get_msi_domain(&dev->dev); - if (domain && irq_domain_is_hierarchy(domain)) - return msi_domain_alloc_irqs(domain, &dev->dev, nvec); - - return arch_setup_msi_irqs(dev, nvec, type); -} - -static void pci_msi_teardown_msi_irqs(struct pci_dev *dev) -{ - struct irq_domain *domain; - - domain = dev_get_msi_domain(&dev->dev); - if (domain && irq_domain_is_hierarchy(domain)) - msi_domain_free_irqs(domain, &dev->dev); - else - arch_teardown_msi_irqs(dev); -} -#else -#define pci_msi_setup_msi_irqs arch_setup_msi_irqs -#define pci_msi_teardown_msi_irqs arch_teardown_msi_irqs -#endif - -#ifdef CONFIG_PCI_MSI_ARCH_FALLBACKS -/* Arch hooks */ -int __weak arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc) -{ - return -EINVAL; -} - -void __weak arch_teardown_msi_irq(unsigned int irq) -{ -} - -int __weak arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) -{ - struct msi_desc *entry; - int ret; - - /* - * If an architecture wants to support multiple MSI, it needs to - * override arch_setup_msi_irqs() - */ - if (type == PCI_CAP_ID_MSI && nvec > 1) - return 1; - - for_each_pci_msi_entry(entry, dev) { - ret = arch_setup_msi_irq(dev, entry); - if (ret < 0) - return ret; - if (ret > 0) - return -ENOSPC; - } - - return 0; -} - -void __weak arch_teardown_msi_irqs(struct pci_dev *dev) -{ - int i; - struct msi_desc *entry; - - for_each_pci_msi_entry(entry, dev) - if (entry->irq) - for (i = 0; i < entry->nvec_used; i++) - arch_teardown_msi_irq(entry->irq + i); -} -#endif /* CONFIG_PCI_MSI_ARCH_FALLBACKS */ - -static void default_restore_msi_irq(struct pci_dev *dev, int irq) -{ - struct msi_desc *entry; - - entry = NULL; - if (dev->msix_enabled) { - for_each_pci_msi_entry(entry, dev) { - if (irq == entry->irq) - break; - } - } else if (dev->msi_enabled) { - entry = irq_get_msi_desc(irq); - } - - if (entry) - __pci_write_msi_msg(entry, &entry->msg); -} - -void __weak arch_restore_msi_irqs(struct pci_dev *dev) -{ - return default_restore_msi_irqs(dev); -} - -/* - * PCI 2.3 does not specify mask bits for each MSI interrupt. Attempting to - * mask all MSI interrupts by clearing the MSI enable bit does not work - * reliably as devices without an INTx disable bit will then generate a - * level IRQ which will never be cleared. - */ -static inline __attribute_const__ u32 msi_multi_mask(struct msi_desc *desc) -{ - /* Don't shift by >= width of type */ - if (desc->msi_attrib.multi_cap >= 5) - return 0xffffffff; - return (1 << (1 << desc->msi_attrib.multi_cap)) - 1; -} - static noinline void pci_msi_update_mask(struct msi_desc *desc, u32 clear, u32 set) { - raw_spinlock_t *lock = &desc->dev->msi_lock; + raw_spinlock_t *lock = &to_pci_dev(desc->dev)->msi_lock; unsigned long flags; - if (!desc->msi_attrib.can_mask) + if (!desc->pci.msi_attrib.can_mask) return; raw_spin_lock_irqsave(lock, flags); - desc->msi_mask &= ~clear; - desc->msi_mask |= set; - pci_write_config_dword(msi_desc_to_pci_dev(desc), desc->mask_pos, - desc->msi_mask); + desc->pci.msi_mask &= ~clear; + desc->pci.msi_mask |= set; + pci_write_config_dword(msi_desc_to_pci_dev(desc), desc->pci.mask_pos, + desc->pci.msi_mask); raw_spin_unlock_irqrestore(lock, flags); } @@ -171,7 +44,7 @@ static inline void pci_msi_unmask(struct msi_desc *desc, u32 mask) static inline void __iomem *pci_msix_desc_addr(struct msi_desc *desc) { - return desc->mask_base + desc->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE; + return desc->pci.mask_base + desc->msi_index * PCI_MSIX_ENTRY_SIZE; } /* @@ -184,27 +57,27 @@ static void pci_msix_write_vector_ctrl(struct msi_desc *desc, u32 ctrl) { void __iomem *desc_addr = pci_msix_desc_addr(desc); - if (desc->msi_attrib.can_mask) + if (desc->pci.msi_attrib.can_mask) writel(ctrl, desc_addr + PCI_MSIX_ENTRY_VECTOR_CTRL); } static inline void pci_msix_mask(struct msi_desc *desc) { - desc->msix_ctrl |= PCI_MSIX_ENTRY_CTRL_MASKBIT; - pci_msix_write_vector_ctrl(desc, desc->msix_ctrl); + desc->pci.msix_ctrl |= PCI_MSIX_ENTRY_CTRL_MASKBIT; + pci_msix_write_vector_ctrl(desc, desc->pci.msix_ctrl); /* Flush write to device */ - readl(desc->mask_base); + readl(desc->pci.mask_base); } static inline void pci_msix_unmask(struct msi_desc *desc) { - desc->msix_ctrl &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT; - pci_msix_write_vector_ctrl(desc, desc->msix_ctrl); + desc->pci.msix_ctrl &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT; + pci_msix_write_vector_ctrl(desc, desc->pci.msix_ctrl); } static void __pci_msi_mask_desc(struct msi_desc *desc, u32 mask) { - if (desc->msi_attrib.is_msix) + if (desc->pci.msi_attrib.is_msix) pci_msix_mask(desc); else pci_msi_mask(desc, mask); @@ -212,7 +85,7 @@ static void __pci_msi_mask_desc(struct msi_desc *desc, u32 mask) static void __pci_msi_unmask_desc(struct msi_desc *desc, u32 mask) { - if (desc->msi_attrib.is_msix) + if (desc->pci.msi_attrib.is_msix) pci_msix_unmask(desc); else pci_msi_unmask(desc, mask); @@ -242,24 +115,16 @@ void pci_msi_unmask_irq(struct irq_data *data) } EXPORT_SYMBOL_GPL(pci_msi_unmask_irq); -void default_restore_msi_irqs(struct pci_dev *dev) -{ - struct msi_desc *entry; - - for_each_pci_msi_entry(entry, dev) - default_restore_msi_irq(dev, entry->irq); -} - void __pci_read_msi_msg(struct msi_desc *entry, struct msi_msg *msg) { struct pci_dev *dev = msi_desc_to_pci_dev(entry); BUG_ON(dev->current_state != PCI_D0); - if (entry->msi_attrib.is_msix) { + if (entry->pci.msi_attrib.is_msix) { void __iomem *base = pci_msix_desc_addr(entry); - if (WARN_ON_ONCE(entry->msi_attrib.is_virtual)) + if (WARN_ON_ONCE(entry->pci.msi_attrib.is_virtual)) return; msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR); @@ -271,7 +136,7 @@ void __pci_read_msi_msg(struct msi_desc *entry, struct msi_msg *msg) pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_LO, &msg->address_lo); - if (entry->msi_attrib.is_64) { + if (entry->pci.msi_attrib.is_64) { pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_HI, &msg->address_hi); pci_read_config_word(dev, pos + PCI_MSI_DATA_64, &data); @@ -289,12 +154,12 @@ void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg) if (dev->current_state != PCI_D0 || pci_dev_is_disconnected(dev)) { /* Don't touch the hardware now */ - } else if (entry->msi_attrib.is_msix) { + } else if (entry->pci.msi_attrib.is_msix) { void __iomem *base = pci_msix_desc_addr(entry); - u32 ctrl = entry->msix_ctrl; + u32 ctrl = entry->pci.msix_ctrl; bool unmasked = !(ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT); - if (entry->msi_attrib.is_virtual) + if (entry->pci.msi_attrib.is_virtual) goto skip; /* @@ -323,12 +188,12 @@ void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg) pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &msgctl); msgctl &= ~PCI_MSI_FLAGS_QSIZE; - msgctl |= entry->msi_attrib.multiple << 4; + msgctl |= entry->pci.msi_attrib.multiple << 4; pci_write_config_word(dev, pos + PCI_MSI_FLAGS, msgctl); pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_LO, msg->address_lo); - if (entry->msi_attrib.is_64) { + if (entry->pci.msi_attrib.is_64) { pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_HI, msg->address_hi); pci_write_config_word(dev, pos + PCI_MSI_DATA_64, @@ -359,30 +224,11 @@ EXPORT_SYMBOL_GPL(pci_write_msi_msg); static void free_msi_irqs(struct pci_dev *dev) { - struct list_head *msi_list = dev_to_msi_list(&dev->dev); - struct msi_desc *entry, *tmp; - int i; - - for_each_pci_msi_entry(entry, dev) - if (entry->irq) - for (i = 0; i < entry->nvec_used; i++) - BUG_ON(irq_has_action(entry->irq + i)); - - if (dev->msi_irq_groups) { - msi_destroy_sysfs(&dev->dev, dev->msi_irq_groups); - dev->msi_irq_groups = NULL; - } - pci_msi_teardown_msi_irqs(dev); - list_for_each_entry_safe(entry, tmp, msi_list, list) { - if (entry->msi_attrib.is_msix) { - if (list_is_last(&entry->list, msi_list)) - iounmap(entry->mask_base); - } - - list_del(&entry->list); - free_msi_entry(entry); + if (dev->msix_base) { + iounmap(dev->msix_base); + dev->msix_base = NULL; } } @@ -403,10 +249,19 @@ static void pci_msi_set_enable(struct pci_dev *dev, int enable) pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control); } +/* + * Architecture override returns true when the PCI MSI message should be + * written by the generic restore function. + */ +bool __weak arch_restore_msi_irqs(struct pci_dev *dev) +{ + return true; +} + static void __pci_restore_msi_state(struct pci_dev *dev) { - u16 control; struct msi_desc *entry; + u16 control; if (!dev->msi_enabled) return; @@ -415,12 +270,13 @@ static void __pci_restore_msi_state(struct pci_dev *dev) pci_intx_for_msi(dev, 0); pci_msi_set_enable(dev, 0); - arch_restore_msi_irqs(dev); + if (arch_restore_msi_irqs(dev)) + __pci_write_msi_msg(entry, &entry->msg); pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control); pci_msi_update_mask(entry, 0, 0); control &= ~PCI_MSI_FLAGS_QSIZE; - control |= (entry->msi_attrib.multiple << 4) | PCI_MSI_FLAGS_ENABLE; + control |= (entry->pci.msi_attrib.multiple << 4) | PCI_MSI_FLAGS_ENABLE; pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control); } @@ -437,19 +293,25 @@ static void pci_msix_clear_and_set_ctrl(struct pci_dev *dev, u16 clear, u16 set) static void __pci_restore_msix_state(struct pci_dev *dev) { struct msi_desc *entry; + bool write_msg; if (!dev->msix_enabled) return; - BUG_ON(list_empty(dev_to_msi_list(&dev->dev))); /* route the table */ pci_intx_for_msi(dev, 0); pci_msix_clear_and_set_ctrl(dev, 0, PCI_MSIX_FLAGS_ENABLE | PCI_MSIX_FLAGS_MASKALL); - arch_restore_msi_irqs(dev); - for_each_pci_msi_entry(entry, dev) - pci_msix_write_vector_ctrl(entry, entry->msix_ctrl); + write_msg = arch_restore_msi_irqs(dev); + + msi_lock_descs(&dev->dev); + msi_for_each_desc(entry, &dev->dev, MSI_DESC_ALL) { + if (write_msg) + __pci_write_msi_msg(entry, &entry->msg); + pci_msix_write_vector_ctrl(entry, entry->pci.msix_ctrl); + } + msi_unlock_descs(&dev->dev); pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0); } @@ -461,48 +323,79 @@ void pci_restore_msi_state(struct pci_dev *dev) } EXPORT_SYMBOL_GPL(pci_restore_msi_state); -static struct msi_desc * -msi_setup_entry(struct pci_dev *dev, int nvec, struct irq_affinity *affd) +static void pcim_msi_release(void *pcidev) { - struct irq_affinity_desc *masks = NULL; - struct msi_desc *entry; + struct pci_dev *dev = pcidev; + + dev->is_msi_managed = false; + pci_free_irq_vectors(dev); +} + +/* + * Needs to be separate from pcim_release to prevent an ordering problem + * vs. msi_device_data_release() in the MSI core code. + */ +static int pcim_setup_msi_release(struct pci_dev *dev) +{ + int ret; + + if (!pci_is_managed(dev) || dev->is_msi_managed) + return 0; + + ret = devm_add_action(&dev->dev, pcim_msi_release, dev); + if (!ret) + dev->is_msi_managed = true; + return ret; +} + +/* + * Ordering vs. devres: msi device data has to be installed first so that + * pcim_msi_release() is invoked before it on device release. + */ +static int pci_setup_msi_context(struct pci_dev *dev) +{ + int ret = msi_setup_device_data(&dev->dev); + + if (!ret) + ret = pcim_setup_msi_release(dev); + return ret; +} + +static int msi_setup_msi_desc(struct pci_dev *dev, int nvec, + struct irq_affinity_desc *masks) +{ + struct msi_desc desc; u16 control; - if (affd) - masks = irq_create_affinity_masks(nvec, affd); - /* MSI Entry Initialization */ - entry = alloc_msi_entry(&dev->dev, nvec, masks); - if (!entry) - goto out; + memset(&desc, 0, sizeof(desc)); pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control); /* Lies, damned lies, and MSIs */ if (dev->dev_flags & PCI_DEV_FLAGS_HAS_MSI_MASKING) control |= PCI_MSI_FLAGS_MASKBIT; + /* Respect XEN's mask disabling */ + if (pci_msi_ignore_mask) + control &= ~PCI_MSI_FLAGS_MASKBIT; - entry->msi_attrib.is_msix = 0; - entry->msi_attrib.is_64 = !!(control & PCI_MSI_FLAGS_64BIT); - entry->msi_attrib.is_virtual = 0; - entry->msi_attrib.entry_nr = 0; - entry->msi_attrib.can_mask = !pci_msi_ignore_mask && - !!(control & PCI_MSI_FLAGS_MASKBIT); - entry->msi_attrib.default_irq = dev->irq; /* Save IOAPIC IRQ */ - entry->msi_attrib.multi_cap = (control & PCI_MSI_FLAGS_QMASK) >> 1; - entry->msi_attrib.multiple = ilog2(__roundup_pow_of_two(nvec)); + desc.nvec_used = nvec; + desc.pci.msi_attrib.is_64 = !!(control & PCI_MSI_FLAGS_64BIT); + desc.pci.msi_attrib.can_mask = !!(control & PCI_MSI_FLAGS_MASKBIT); + desc.pci.msi_attrib.default_irq = dev->irq; + desc.pci.msi_attrib.multi_cap = (control & PCI_MSI_FLAGS_QMASK) >> 1; + desc.pci.msi_attrib.multiple = ilog2(__roundup_pow_of_two(nvec)); + desc.affinity = masks; if (control & PCI_MSI_FLAGS_64BIT) - entry->mask_pos = dev->msi_cap + PCI_MSI_MASK_64; + desc.pci.mask_pos = dev->msi_cap + PCI_MSI_MASK_64; else - entry->mask_pos = dev->msi_cap + PCI_MSI_MASK_32; + desc.pci.mask_pos = dev->msi_cap + PCI_MSI_MASK_32; /* Save the initial mask status */ - if (entry->msi_attrib.can_mask) - pci_read_config_dword(dev, entry->mask_pos, &entry->msi_mask); + if (desc.pci.msi_attrib.can_mask) + pci_read_config_dword(dev, desc.pci.mask_pos, &desc.pci.msi_mask); -out: - kfree(masks); - return entry; + return msi_add_msi_desc(&dev->dev, &desc); } static int msi_verify_entries(struct pci_dev *dev) @@ -512,14 +405,14 @@ static int msi_verify_entries(struct pci_dev *dev) if (!dev->no_64bit_msi) return 0; - for_each_pci_msi_entry(entry, dev) { + msi_for_each_desc(entry, &dev->dev, MSI_DESC_ALL) { if (entry->msg.address_hi) { pci_err(dev, "arch assigned 64-bit MSI address %#x%08x but device only supports 32 bits\n", entry->msg.address_hi, entry->msg.address_lo); - return -EIO; + break; } } - return 0; + return !entry ? 0 : -EIO; } /** @@ -537,21 +430,29 @@ static int msi_verify_entries(struct pci_dev *dev) static int msi_capability_init(struct pci_dev *dev, int nvec, struct irq_affinity *affd) { - const struct attribute_group **groups; + struct irq_affinity_desc *masks = NULL; struct msi_desc *entry; int ret; - pci_msi_set_enable(dev, 0); /* Disable MSI during set up */ + /* + * Disable MSI during setup in the hardware, but mark it enabled + * so that setup code can evaluate it. + */ + pci_msi_set_enable(dev, 0); + dev->msi_enabled = 1; - entry = msi_setup_entry(dev, nvec, affd); - if (!entry) - return -ENOMEM; + if (affd) + masks = irq_create_affinity_masks(nvec, affd); + + msi_lock_descs(&dev->dev); + ret = msi_setup_msi_desc(dev, nvec, masks); + if (ret) + goto fail; /* All MSIs are unmasked by default; mask them all */ + entry = msi_first_desc(&dev->dev, MSI_DESC_ALL); pci_msi_mask(entry, msi_multi_mask(entry)); - list_add_tail(&entry->list, dev_to_msi_list(&dev->dev)); - /* Configure MSI capability structure */ ret = pci_msi_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSI); if (ret) @@ -561,26 +462,22 @@ static int msi_capability_init(struct pci_dev *dev, int nvec, if (ret) goto err; - groups = msi_populate_sysfs(&dev->dev); - if (IS_ERR(groups)) { - ret = PTR_ERR(groups); - goto err; - } - - dev->msi_irq_groups = groups; - /* Set MSI enabled bits */ pci_intx_for_msi(dev, 0); pci_msi_set_enable(dev, 1); - dev->msi_enabled = 1; pcibios_free_irq(dev); dev->irq = entry->irq; - return 0; + goto unlock; err: pci_msi_unmask(entry, msi_multi_mask(entry)); free_msi_irqs(dev); +fail: + dev->msi_enabled = 0; +unlock: + msi_unlock_descs(&dev->dev); + kfree(masks); return ret; } @@ -605,70 +502,49 @@ static void __iomem *msix_map_region(struct pci_dev *dev, return ioremap(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE); } -static int msix_setup_entries(struct pci_dev *dev, void __iomem *base, - struct msix_entry *entries, int nvec, - struct irq_affinity *affd) +static int msix_setup_msi_descs(struct pci_dev *dev, void __iomem *base, + struct msix_entry *entries, int nvec, + struct irq_affinity_desc *masks) { - struct irq_affinity_desc *curmsk, *masks = NULL; - struct msi_desc *entry; + int ret = 0, i, vec_count = pci_msix_vec_count(dev); + struct irq_affinity_desc *curmsk; + struct msi_desc desc; void __iomem *addr; - int ret, i; - int vec_count = pci_msix_vec_count(dev); - if (affd) - masks = irq_create_affinity_masks(nvec, affd); + memset(&desc, 0, sizeof(desc)); - for (i = 0, curmsk = masks; i < nvec; i++) { - entry = alloc_msi_entry(&dev->dev, 1, curmsk); - if (!entry) { - if (!i) - iounmap(base); - else - free_msi_irqs(dev); - /* No enough memory. Don't try again */ - ret = -ENOMEM; - goto out; + desc.nvec_used = 1; + desc.pci.msi_attrib.is_msix = 1; + desc.pci.msi_attrib.is_64 = 1; + desc.pci.msi_attrib.default_irq = dev->irq; + desc.pci.mask_base = base; + + for (i = 0, curmsk = masks; i < nvec; i++, curmsk++) { + desc.msi_index = entries ? entries[i].entry : i; + desc.affinity = masks ? curmsk : NULL; + desc.pci.msi_attrib.is_virtual = desc.msi_index >= vec_count; + desc.pci.msi_attrib.can_mask = !pci_msi_ignore_mask && + !desc.pci.msi_attrib.is_virtual; + + if (!desc.pci.msi_attrib.can_mask) { + addr = pci_msix_desc_addr(&desc); + desc.pci.msix_ctrl = readl(addr + PCI_MSIX_ENTRY_VECTOR_CTRL); } - entry->msi_attrib.is_msix = 1; - entry->msi_attrib.is_64 = 1; - - if (entries) - entry->msi_attrib.entry_nr = entries[i].entry; - else - entry->msi_attrib.entry_nr = i; - - entry->msi_attrib.is_virtual = - entry->msi_attrib.entry_nr >= vec_count; - - entry->msi_attrib.can_mask = !pci_msi_ignore_mask && - !entry->msi_attrib.is_virtual; - - entry->msi_attrib.default_irq = dev->irq; - entry->mask_base = base; - - if (entry->msi_attrib.can_mask) { - addr = pci_msix_desc_addr(entry); - entry->msix_ctrl = readl(addr + PCI_MSIX_ENTRY_VECTOR_CTRL); - } - - list_add_tail(&entry->list, dev_to_msi_list(&dev->dev)); - if (masks) - curmsk++; + ret = msi_add_msi_desc(&dev->dev, &desc); + if (ret) + break; } - ret = 0; -out: - kfree(masks); return ret; } static void msix_update_entries(struct pci_dev *dev, struct msix_entry *entries) { - struct msi_desc *entry; + struct msi_desc *desc; - for_each_pci_msi_entry(entry, dev) { - if (entries) { - entries->vector = entry->irq; + if (entries) { + msi_for_each_desc(desc, &dev->dev, MSI_DESC_ALL) { + entries->vector = desc->irq; entries++; } } @@ -686,6 +562,41 @@ static void msix_mask_all(void __iomem *base, int tsize) writel(ctrl, base + PCI_MSIX_ENTRY_VECTOR_CTRL); } +static int msix_setup_interrupts(struct pci_dev *dev, void __iomem *base, + struct msix_entry *entries, int nvec, + struct irq_affinity *affd) +{ + struct irq_affinity_desc *masks = NULL; + int ret; + + if (affd) + masks = irq_create_affinity_masks(nvec, affd); + + msi_lock_descs(&dev->dev); + ret = msix_setup_msi_descs(dev, base, entries, nvec, masks); + if (ret) + goto out_free; + + ret = pci_msi_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSIX); + if (ret) + goto out_free; + + /* Check if all MSI entries honor device restrictions */ + ret = msi_verify_entries(dev); + if (ret) + goto out_free; + + msix_update_entries(dev, entries); + goto out_unlock; + +out_free: + free_msi_irqs(dev); +out_unlock: + msi_unlock_descs(&dev->dev); + kfree(masks); + return ret; +} + /** * msix_capability_init - configure device's MSI-X capability * @dev: pointer to the pci_dev data structure of MSI-X device function @@ -700,7 +611,6 @@ static void msix_mask_all(void __iomem *base, int tsize) static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, int nvec, struct irq_affinity *affd) { - const struct attribute_group **groups; void __iomem *base; int ret, tsize; u16 control; @@ -713,6 +623,9 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, pci_msix_clear_and_set_ctrl(dev, 0, PCI_MSIX_FLAGS_MASKALL | PCI_MSIX_FLAGS_ENABLE); + /* Mark it enabled so setup functions can query it */ + dev->msix_enabled = 1; + pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control); /* Request & Map MSI-X table region */ tsize = msix_table_size(control); @@ -722,32 +635,14 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, goto out_disable; } - ret = msix_setup_entries(dev, base, entries, nvec, affd); + dev->msix_base = base; + + ret = msix_setup_interrupts(dev, base, entries, nvec, affd); if (ret) goto out_disable; - ret = pci_msi_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSIX); - if (ret) - goto out_avail; - - /* Check if all MSI entries honor device restrictions */ - ret = msi_verify_entries(dev); - if (ret) - goto out_free; - - msix_update_entries(dev, entries); - - groups = msi_populate_sysfs(&dev->dev); - if (IS_ERR(groups)) { - ret = PTR_ERR(groups); - goto out_free; - } - - dev->msi_irq_groups = groups; - - /* Set MSI-X enabled bits and unmask the function */ + /* Disable INTX */ pci_intx_for_msi(dev, 0); - dev->msix_enabled = 1; /* * Ensure that all table entries are masked to prevent @@ -763,27 +658,8 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, pcibios_free_irq(dev); return 0; -out_avail: - if (ret < 0) { - /* - * If we had some success, report the number of IRQs - * we succeeded in setting up. - */ - struct msi_desc *entry; - int avail = 0; - - for_each_pci_msi_entry(entry, dev) { - if (entry->irq != 0) - avail++; - } - if (avail != 0) - ret = avail; - } - -out_free: - free_msi_irqs(dev); - out_disable: + dev->msix_enabled = 0; pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL | PCI_MSIX_FLAGS_ENABLE, 0); return ret; @@ -870,18 +746,17 @@ static void pci_msi_shutdown(struct pci_dev *dev) if (!pci_msi_enable || !dev || !dev->msi_enabled) return; - BUG_ON(list_empty(dev_to_msi_list(&dev->dev))); - desc = first_pci_msi_entry(dev); - pci_msi_set_enable(dev, 0); pci_intx_for_msi(dev, 1); dev->msi_enabled = 0; /* Return the device with MSI unmasked as initial states */ - pci_msi_unmask(desc, msi_multi_mask(desc)); + desc = msi_first_desc(&dev->dev, MSI_DESC_ALL); + if (!WARN_ON_ONCE(!desc)) + pci_msi_unmask(desc, msi_multi_mask(desc)); /* Restore dev->irq to its default pin-assertion IRQ */ - dev->irq = desc->msi_attrib.default_irq; + dev->irq = desc->pci.msi_attrib.default_irq; pcibios_alloc_irq(dev); } @@ -890,8 +765,10 @@ void pci_disable_msi(struct pci_dev *dev) if (!pci_msi_enable || !dev || !dev->msi_enabled) return; + msi_lock_descs(&dev->dev); pci_msi_shutdown(dev); free_msi_irqs(dev); + msi_unlock_descs(&dev->dev); } EXPORT_SYMBOL(pci_disable_msi); @@ -952,7 +829,7 @@ static int __pci_enable_msix(struct pci_dev *dev, struct msix_entry *entries, static void pci_msix_shutdown(struct pci_dev *dev) { - struct msi_desc *entry; + struct msi_desc *desc; if (!pci_msi_enable || !dev || !dev->msix_enabled) return; @@ -963,8 +840,8 @@ static void pci_msix_shutdown(struct pci_dev *dev) } /* Return the device with MSI-X masked as initial states */ - for_each_pci_msi_entry(entry, dev) - pci_msix_mask(entry); + msi_for_each_desc(desc, &dev->dev, MSI_DESC_ALL) + pci_msix_mask(desc); pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0); pci_intx_for_msi(dev, 1); @@ -977,28 +854,13 @@ void pci_disable_msix(struct pci_dev *dev) if (!pci_msi_enable || !dev || !dev->msix_enabled) return; + msi_lock_descs(&dev->dev); pci_msix_shutdown(dev); free_msi_irqs(dev); + msi_unlock_descs(&dev->dev); } EXPORT_SYMBOL(pci_disable_msix); -void pci_no_msi(void) -{ - pci_msi_enable = 0; -} - -/** - * pci_msi_enabled - is MSI enabled? - * - * Returns true if MSI has not been disabled by the command-line option - * pci=nomsi. - **/ -int pci_msi_enabled(void) -{ - return pci_msi_enable; -} -EXPORT_SYMBOL(pci_msi_enabled); - static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec, struct irq_affinity *affd) { @@ -1029,6 +891,10 @@ static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec, if (nvec > maxvec) nvec = maxvec; + rc = pci_setup_msi_context(dev); + if (rc) + return rc; + for (;;) { if (affd) { nvec = irq_calc_affinity_vectors(minvec, nvec, affd); @@ -1072,6 +938,10 @@ static int __pci_enable_msix_range(struct pci_dev *dev, if (WARN_ON_ONCE(dev->msix_enabled)) return -EINVAL; + rc = pci_setup_msi_context(dev); + if (rc) + return rc; + for (;;) { if (affd) { nvec = irq_calc_affinity_vectors(minvec, nvec, affd); @@ -1194,35 +1064,25 @@ EXPORT_SYMBOL(pci_free_irq_vectors); /** * pci_irq_vector - return Linux IRQ number of a device vector - * @dev: PCI device to operate on - * @nr: device-relative interrupt vector index (0-based). + * @dev: PCI device to operate on + * @nr: Interrupt vector index (0-based) + * + * @nr has the following meanings depending on the interrupt mode: + * MSI-X: The index in the MSI-X vector table + * MSI: The index of the enabled MSI vectors + * INTx: Must be 0 + * + * Return: The Linux interrupt number or -EINVAl if @nr is out of range. */ int pci_irq_vector(struct pci_dev *dev, unsigned int nr) { - if (dev->msix_enabled) { - struct msi_desc *entry; - int i = 0; + unsigned int irq; - for_each_pci_msi_entry(entry, dev) { - if (i == nr) - return entry->irq; - i++; - } - WARN_ON_ONCE(1); - return -EINVAL; - } + if (!dev->msi_enabled && !dev->msix_enabled) + return !nr ? dev->irq : -EINVAL; - if (dev->msi_enabled) { - struct msi_desc *entry = first_pci_msi_entry(dev); - - if (WARN_ON_ONCE(nr >= entry->nvec_used)) - return -EINVAL; - } else { - if (WARN_ON_ONCE(nr > 0)) - return -EINVAL; - } - - return dev->irq + nr; + irq = msi_get_virq(&dev->dev, nr); + return irq ? irq : -EINVAL; } EXPORT_SYMBOL(pci_irq_vector); @@ -1230,31 +1090,36 @@ EXPORT_SYMBOL(pci_irq_vector); * pci_irq_get_affinity - return the affinity of a particular MSI vector * @dev: PCI device to operate on * @nr: device-relative interrupt vector index (0-based). + * + * @nr has the following meanings depending on the interrupt mode: + * MSI-X: The index in the MSI-X vector table + * MSI: The index of the enabled MSI vectors + * INTx: Must be 0 + * + * Return: A cpumask pointer or NULL if @nr is out of range */ const struct cpumask *pci_irq_get_affinity(struct pci_dev *dev, int nr) { - if (dev->msix_enabled) { - struct msi_desc *entry; - int i = 0; + int idx, irq = pci_irq_vector(dev, nr); + struct msi_desc *desc; - for_each_pci_msi_entry(entry, dev) { - if (i == nr) - return &entry->affinity->mask; - i++; - } - WARN_ON_ONCE(1); + if (WARN_ON_ONCE(irq <= 0)) return NULL; - } else if (dev->msi_enabled) { - struct msi_desc *entry = first_pci_msi_entry(dev); - if (WARN_ON_ONCE(!entry || !entry->affinity || - nr >= entry->nvec_used)) - return NULL; - - return &entry->affinity[nr].mask; - } else { + desc = irq_get_msi_desc(irq); + /* Non-MSI does not have the information handy */ + if (!desc) return cpu_possible_mask; - } + + if (WARN_ON_ONCE(!desc->affinity)) + return NULL; + + /* + * MSI has a mask array in the descriptor. + * MSI-X has a single mask. + */ + idx = dev->msi_enabled ? nr : 0; + return &desc->affinity[idx].mask; } EXPORT_SYMBOL(pci_irq_get_affinity); @@ -1264,298 +1129,19 @@ struct pci_dev *msi_desc_to_pci_dev(struct msi_desc *desc) } EXPORT_SYMBOL(msi_desc_to_pci_dev); -void *msi_desc_to_pci_sysdata(struct msi_desc *desc) +void pci_no_msi(void) { - struct pci_dev *dev = msi_desc_to_pci_dev(desc); - - return dev->bus->sysdata; -} -EXPORT_SYMBOL_GPL(msi_desc_to_pci_sysdata); - -#ifdef CONFIG_PCI_MSI_IRQ_DOMAIN -/** - * pci_msi_domain_write_msg - Helper to write MSI message to PCI config space - * @irq_data: Pointer to interrupt data of the MSI interrupt - * @msg: Pointer to the message - */ -void pci_msi_domain_write_msg(struct irq_data *irq_data, struct msi_msg *msg) -{ - struct msi_desc *desc = irq_data_get_msi_desc(irq_data); - - /* - * For MSI-X desc->irq is always equal to irq_data->irq. For - * MSI only the first interrupt of MULTI MSI passes the test. - */ - if (desc->irq == irq_data->irq) - __pci_write_msi_msg(desc, msg); + pci_msi_enable = 0; } /** - * pci_msi_domain_calc_hwirq - Generate a unique ID for an MSI source - * @desc: Pointer to the MSI descriptor + * pci_msi_enabled - is MSI enabled? * - * The ID number is only used within the irqdomain. - */ -static irq_hw_number_t pci_msi_domain_calc_hwirq(struct msi_desc *desc) + * Returns true if MSI has not been disabled by the command-line option + * pci=nomsi. + **/ +int pci_msi_enabled(void) { - struct pci_dev *dev = msi_desc_to_pci_dev(desc); - - return (irq_hw_number_t)desc->msi_attrib.entry_nr | - pci_dev_id(dev) << 11 | - (pci_domain_nr(dev->bus) & 0xFFFFFFFF) << 27; -} - -static inline bool pci_msi_desc_is_multi_msi(struct msi_desc *desc) -{ - return !desc->msi_attrib.is_msix && desc->nvec_used > 1; -} - -/** - * pci_msi_domain_check_cap - Verify that @domain supports the capabilities - * for @dev - * @domain: The interrupt domain to check - * @info: The domain info for verification - * @dev: The device to check - * - * Returns: - * 0 if the functionality is supported - * 1 if Multi MSI is requested, but the domain does not support it - * -ENOTSUPP otherwise - */ -int pci_msi_domain_check_cap(struct irq_domain *domain, - struct msi_domain_info *info, struct device *dev) -{ - struct msi_desc *desc = first_pci_msi_entry(to_pci_dev(dev)); - - /* Special handling to support __pci_enable_msi_range() */ - if (pci_msi_desc_is_multi_msi(desc) && - !(info->flags & MSI_FLAG_MULTI_PCI_MSI)) - return 1; - else if (desc->msi_attrib.is_msix && !(info->flags & MSI_FLAG_PCI_MSIX)) - return -ENOTSUPP; - - return 0; -} - -static int pci_msi_domain_handle_error(struct irq_domain *domain, - struct msi_desc *desc, int error) -{ - /* Special handling to support __pci_enable_msi_range() */ - if (pci_msi_desc_is_multi_msi(desc) && error == -ENOSPC) - return 1; - - return error; -} - -static void pci_msi_domain_set_desc(msi_alloc_info_t *arg, - struct msi_desc *desc) -{ - arg->desc = desc; - arg->hwirq = pci_msi_domain_calc_hwirq(desc); -} - -static struct msi_domain_ops pci_msi_domain_ops_default = { - .set_desc = pci_msi_domain_set_desc, - .msi_check = pci_msi_domain_check_cap, - .handle_error = pci_msi_domain_handle_error, -}; - -static void pci_msi_domain_update_dom_ops(struct msi_domain_info *info) -{ - struct msi_domain_ops *ops = info->ops; - - if (ops == NULL) { - info->ops = &pci_msi_domain_ops_default; - } else { - if (ops->set_desc == NULL) - ops->set_desc = pci_msi_domain_set_desc; - if (ops->msi_check == NULL) - ops->msi_check = pci_msi_domain_check_cap; - if (ops->handle_error == NULL) - ops->handle_error = pci_msi_domain_handle_error; - } -} - -static void pci_msi_domain_update_chip_ops(struct msi_domain_info *info) -{ - struct irq_chip *chip = info->chip; - - BUG_ON(!chip); - if (!chip->irq_write_msi_msg) - chip->irq_write_msi_msg = pci_msi_domain_write_msg; - if (!chip->irq_mask) - chip->irq_mask = pci_msi_mask_irq; - if (!chip->irq_unmask) - chip->irq_unmask = pci_msi_unmask_irq; -} - -/** - * pci_msi_create_irq_domain - Create a MSI interrupt domain - * @fwnode: Optional fwnode of the interrupt controller - * @info: MSI domain info - * @parent: Parent irq domain - * - * Updates the domain and chip ops and creates a MSI interrupt domain. - * - * Returns: - * A domain pointer or NULL in case of failure. - */ -struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode, - struct msi_domain_info *info, - struct irq_domain *parent) -{ - struct irq_domain *domain; - - if (WARN_ON(info->flags & MSI_FLAG_LEVEL_CAPABLE)) - info->flags &= ~MSI_FLAG_LEVEL_CAPABLE; - - if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS) - pci_msi_domain_update_dom_ops(info); - if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS) - pci_msi_domain_update_chip_ops(info); - - info->flags |= MSI_FLAG_ACTIVATE_EARLY; - if (IS_ENABLED(CONFIG_GENERIC_IRQ_RESERVATION_MODE)) - info->flags |= MSI_FLAG_MUST_REACTIVATE; - - /* PCI-MSI is oneshot-safe */ - info->chip->flags |= IRQCHIP_ONESHOT_SAFE; - - domain = msi_create_irq_domain(fwnode, info, parent); - if (!domain) - return NULL; - - irq_domain_update_bus_token(domain, DOMAIN_BUS_PCI_MSI); - return domain; -} -EXPORT_SYMBOL_GPL(pci_msi_create_irq_domain); - -/* - * Users of the generic MSI infrastructure expect a device to have a single ID, - * so with DMA aliases we have to pick the least-worst compromise. Devices with - * DMA phantom functions tend to still emit MSIs from the real function number, - * so we ignore those and only consider topological aliases where either the - * alias device or RID appears on a different bus number. We also make the - * reasonable assumption that bridges are walked in an upstream direction (so - * the last one seen wins), and the much braver assumption that the most likely - * case is that of PCI->PCIe so we should always use the alias RID. This echoes - * the logic from intel_irq_remapping's set_msi_sid(), which presumably works - * well enough in practice; in the face of the horrible PCIe<->PCI-X conditions - * for taking ownership all we can really do is close our eyes and hope... - */ -static int get_msi_id_cb(struct pci_dev *pdev, u16 alias, void *data) -{ - u32 *pa = data; - u8 bus = PCI_BUS_NUM(*pa); - - if (pdev->bus->number != bus || PCI_BUS_NUM(alias) != bus) - *pa = alias; - - return 0; -} - -/** - * pci_msi_domain_get_msi_rid - Get the MSI requester id (RID) - * @domain: The interrupt domain - * @pdev: The PCI device. - * - * The RID for a device is formed from the alias, with a firmware - * supplied mapping applied - * - * Returns: The RID. - */ -u32 pci_msi_domain_get_msi_rid(struct irq_domain *domain, struct pci_dev *pdev) -{ - struct device_node *of_node; - u32 rid = pci_dev_id(pdev); - - pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid); - - of_node = irq_domain_get_of_node(domain); - rid = of_node ? of_msi_map_id(&pdev->dev, of_node, rid) : - iort_msi_map_id(&pdev->dev, rid); - - return rid; -} - -/** - * pci_msi_get_device_domain - Get the MSI domain for a given PCI device - * @pdev: The PCI device - * - * Use the firmware data to find a device-specific MSI domain - * (i.e. not one that is set as a default). - * - * Returns: The corresponding MSI domain or NULL if none has been found. - */ -struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev) -{ - struct irq_domain *dom; - u32 rid = pci_dev_id(pdev); - - pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid); - dom = of_msi_map_get_device_domain(&pdev->dev, rid, DOMAIN_BUS_PCI_MSI); - if (!dom) - dom = iort_get_device_domain(&pdev->dev, rid, - DOMAIN_BUS_PCI_MSI); - return dom; -} - -/** - * pci_dev_has_special_msi_domain - Check whether the device is handled by - * a non-standard PCI-MSI domain - * @pdev: The PCI device to check. - * - * Returns: True if the device irqdomain or the bus irqdomain is - * non-standard PCI/MSI. - */ -bool pci_dev_has_special_msi_domain(struct pci_dev *pdev) -{ - struct irq_domain *dom = dev_get_msi_domain(&pdev->dev); - - if (!dom) - dom = dev_get_msi_domain(&pdev->bus->dev); - - if (!dom) - return true; - - return dom->bus_token != DOMAIN_BUS_PCI_MSI; -} - -#endif /* CONFIG_PCI_MSI_IRQ_DOMAIN */ -#endif /* CONFIG_PCI_MSI */ - -void pci_msi_init(struct pci_dev *dev) -{ - u16 ctrl; - - /* - * Disable the MSI hardware to avoid screaming interrupts - * during boot. This is the power on reset default so - * usually this should be a noop. - */ - dev->msi_cap = pci_find_capability(dev, PCI_CAP_ID_MSI); - if (!dev->msi_cap) - return; - - pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &ctrl); - if (ctrl & PCI_MSI_FLAGS_ENABLE) - pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, - ctrl & ~PCI_MSI_FLAGS_ENABLE); - - if (!(ctrl & PCI_MSI_FLAGS_64BIT)) - dev->no_64bit_msi = 1; -} - -void pci_msix_init(struct pci_dev *dev) -{ - u16 ctrl; - - dev->msix_cap = pci_find_capability(dev, PCI_CAP_ID_MSIX); - if (!dev->msix_cap) - return; - - pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &ctrl); - if (ctrl & PCI_MSIX_FLAGS_ENABLE) - pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, - ctrl & ~PCI_MSIX_FLAGS_ENABLE); + return pci_msi_enable; } +EXPORT_SYMBOL(pci_msi_enabled); diff --git a/drivers/pci/msi/msi.h b/drivers/pci/msi/msi.h new file mode 100644 index 000000000000..dbeff066bedd --- /dev/null +++ b/drivers/pci/msi/msi.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include +#include + +#define msix_table_size(flags) ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) + +extern int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type); +extern void pci_msi_teardown_msi_irqs(struct pci_dev *dev); + +#ifdef CONFIG_PCI_MSI_ARCH_FALLBACKS +extern int pci_msi_legacy_setup_msi_irqs(struct pci_dev *dev, int nvec, int type); +extern void pci_msi_legacy_teardown_msi_irqs(struct pci_dev *dev); +#else +static inline int pci_msi_legacy_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) +{ + WARN_ON_ONCE(1); + return -ENODEV; +} + +static inline void pci_msi_legacy_teardown_msi_irqs(struct pci_dev *dev) +{ + WARN_ON_ONCE(1); +} +#endif + +/* + * PCI 2.3 does not specify mask bits for each MSI interrupt. Attempting to + * mask all MSI interrupts by clearing the MSI enable bit does not work + * reliably as devices without an INTx disable bit will then generate a + * level IRQ which will never be cleared. + */ +static inline __attribute_const__ u32 msi_multi_mask(struct msi_desc *desc) +{ + /* Don't shift by >= width of type */ + if (desc->pci.msi_attrib.multi_cap >= 5) + return 0xffffffff; + return (1 << (1 << desc->pci.msi_attrib.multi_cap)) - 1; +} diff --git a/drivers/pci/msi/pcidev_msi.c b/drivers/pci/msi/pcidev_msi.c new file mode 100644 index 000000000000..5520aff53b56 --- /dev/null +++ b/drivers/pci/msi/pcidev_msi.c @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * MSI[X} related functions which are available unconditionally. + */ +#include "../pci.h" + +/* + * Disable the MSI[X] hardware to avoid screaming interrupts during boot. + * This is the power on reset default so usually this should be a noop. + */ + +void pci_msi_init(struct pci_dev *dev) +{ + u16 ctrl; + + dev->msi_cap = pci_find_capability(dev, PCI_CAP_ID_MSI); + if (!dev->msi_cap) + return; + + pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &ctrl); + if (ctrl & PCI_MSI_FLAGS_ENABLE) { + pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, + ctrl & ~PCI_MSI_FLAGS_ENABLE); + } + + if (!(ctrl & PCI_MSI_FLAGS_64BIT)) + dev->no_64bit_msi = 1; +} + +void pci_msix_init(struct pci_dev *dev) +{ + u16 ctrl; + + dev->msix_cap = pci_find_capability(dev, PCI_CAP_ID_MSIX); + if (!dev->msix_cap) + return; + + pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &ctrl); + if (ctrl & PCI_MSIX_FLAGS_ENABLE) { + pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, + ctrl & ~PCI_MSIX_FLAGS_ENABLE); + } +} diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c index cfe2f85af09e..602f0fb0b007 100644 --- a/drivers/pci/pci-sysfs.c +++ b/drivers/pci/pci-sysfs.c @@ -62,11 +62,8 @@ static ssize_t irq_show(struct device *dev, * For MSI, show the first MSI IRQ; for all other cases including * MSI-X, show the legacy INTx IRQ. */ - if (pdev->msi_enabled) { - struct msi_desc *desc = first_pci_msi_entry(pdev); - - return sysfs_emit(buf, "%u\n", desc->irq); - } + if (pdev->msi_enabled) + return sysfs_emit(buf, "%u\n", pci_irq_vector(pdev, 0)); #endif return sysfs_emit(buf, "%u\n", pdev->irq); diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 3d2fb394986a..f3f606c232a8 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -2024,11 +2024,6 @@ static void pcim_release(struct device *gendev, void *res) struct pci_devres *this = res; int i; - if (dev->msi_enabled) - pci_disable_msi(dev); - if (dev->msix_enabled) - pci_disable_msix(dev); - for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) if (this->region_mask & (1 << i)) pci_release_region(dev, i); diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index 087d3658f75c..443efb00e219 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -2311,7 +2311,9 @@ struct pci_dev *pci_alloc_dev(struct pci_bus *bus) INIT_LIST_HEAD(&dev->bus_list); dev->dev.type = &pci_dev_type; dev->bus = pci_bus_get(bus); - +#ifdef CONFIG_PCI_MSI + raw_spin_lock_init(&dev->msi_lock); +#endif return dev; } EXPORT_SYMBOL(pci_alloc_dev); diff --git a/drivers/pci/xen-pcifront.c b/drivers/pci/xen-pcifront.c index d858d25b6cab..d2a7b9fd678b 100644 --- a/drivers/pci/xen-pcifront.c +++ b/drivers/pci/xen-pcifront.c @@ -262,8 +262,8 @@ static int pci_frontend_enable_msix(struct pci_dev *dev, } i = 0; - for_each_pci_msi_entry(entry, dev) { - op.msix_entries[i].entry = entry->msi_attrib.entry_nr; + msi_for_each_desc(entry, &dev->dev, MSI_DESC_NOTASSOCIATED) { + op.msix_entries[i].entry = entry->msi_index; /* Vector is useless at this point. */ op.msix_entries[i].vector = -1; i++; diff --git a/drivers/perf/arm_smmuv3_pmu.c b/drivers/perf/arm_smmuv3_pmu.c index 1ae19f7301b2..c49108a72865 100644 --- a/drivers/perf/arm_smmuv3_pmu.c +++ b/drivers/perf/arm_smmuv3_pmu.c @@ -703,7 +703,6 @@ static void smmu_pmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg) static void smmu_pmu_setup_msi(struct smmu_pmu *pmu) { - struct msi_desc *desc; struct device *dev = pmu->dev; int ret; @@ -720,9 +719,7 @@ static void smmu_pmu_setup_msi(struct smmu_pmu *pmu) return; } - desc = first_msi_entry(dev); - if (desc) - pmu->irq = desc->irq; + pmu->irq = msi_get_virq(dev, 0); /* Add callback to free MSIs on teardown */ devm_add_action(dev, smmu_pmu_free_msis, dev); diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c index ba17a8f740a9..945755266c49 100644 --- a/drivers/scsi/lpfc/lpfc_init.c +++ b/drivers/scsi/lpfc/lpfc_init.c @@ -12709,7 +12709,7 @@ lpfc_irq_set_aff(struct lpfc_hba_eq_hdl *eqhdl, unsigned int cpu) cpumask_clear(&eqhdl->aff_mask); cpumask_set_cpu(cpu, &eqhdl->aff_mask); irq_set_status_flags(eqhdl->irq, IRQ_NO_BALANCING); - irq_set_affinity_hint(eqhdl->irq, &eqhdl->aff_mask); + irq_set_affinity(eqhdl->irq, &eqhdl->aff_mask); } /** @@ -12998,7 +12998,6 @@ cfg_fail_out: for (--index; index >= 0; index--) { eqhdl = lpfc_get_eq_hdl(index); lpfc_irq_clear_aff(eqhdl); - irq_set_affinity_hint(eqhdl->irq, NULL); free_irq(eqhdl->irq, eqhdl); } @@ -13159,7 +13158,6 @@ lpfc_sli4_disable_intr(struct lpfc_hba *phba) for (index = 0; index < phba->cfg_irq_chann; index++) { eqhdl = lpfc_get_eq_hdl(index); lpfc_irq_clear_aff(eqhdl); - irq_set_affinity_hint(eqhdl->irq, NULL); free_irq(eqhdl->irq, eqhdl); } } else { diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c index aeb95f409826..82e1e24257bc 100644 --- a/drivers/scsi/megaraid/megaraid_sas_base.c +++ b/drivers/scsi/megaraid/megaraid_sas_base.c @@ -5720,7 +5720,7 @@ megasas_setup_irqs_msix(struct megasas_instance *instance, u8 is_probe) "Failed to register IRQ for vector %d.\n", i); for (j = 0; j < i; j++) { if (j < instance->low_latency_index_start) - irq_set_affinity_hint( + irq_update_affinity_hint( pci_irq_vector(pdev, j), NULL); free_irq(pci_irq_vector(pdev, j), &instance->irq_context[j]); @@ -5763,7 +5763,7 @@ megasas_destroy_irqs(struct megasas_instance *instance) { if (instance->msix_vectors) for (i = 0; i < instance->msix_vectors; i++) { if (i < instance->low_latency_index_start) - irq_set_affinity_hint( + irq_update_affinity_hint( pci_irq_vector(instance->pdev, i), NULL); free_irq(pci_irq_vector(instance->pdev, i), &instance->irq_context[i]); @@ -5894,22 +5894,25 @@ int megasas_get_device_list(struct megasas_instance *instance) } /** - * megasas_set_high_iops_queue_affinity_hint - Set affinity hint for high IOPS queues - * @instance: Adapter soft state - * return: void + * megasas_set_high_iops_queue_affinity_and_hint - Set affinity and hint + * for high IOPS queues + * @instance: Adapter soft state + * return: void */ static inline void -megasas_set_high_iops_queue_affinity_hint(struct megasas_instance *instance) +megasas_set_high_iops_queue_affinity_and_hint(struct megasas_instance *instance) { int i; - int local_numa_node; + unsigned int irq; + const struct cpumask *mask; if (instance->perf_mode == MR_BALANCED_PERF_MODE) { - local_numa_node = dev_to_node(&instance->pdev->dev); + mask = cpumask_of_node(dev_to_node(&instance->pdev->dev)); - for (i = 0; i < instance->low_latency_index_start; i++) - irq_set_affinity_hint(pci_irq_vector(instance->pdev, i), - cpumask_of_node(local_numa_node)); + for (i = 0; i < instance->low_latency_index_start; i++) { + irq = pci_irq_vector(instance->pdev, i); + irq_set_affinity_and_hint(irq, mask); + } } } @@ -5998,7 +6001,7 @@ megasas_alloc_irq_vectors(struct megasas_instance *instance) instance->msix_vectors = 0; if (instance->smp_affinity_enable) - megasas_set_high_iops_queue_affinity_hint(instance); + megasas_set_high_iops_queue_affinity_and_hint(instance); } /** diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c b/drivers/scsi/mpt3sas/mpt3sas_base.c index 81dab9b82f79..511726f92d9a 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_base.c +++ b/drivers/scsi/mpt3sas/mpt3sas_base.c @@ -3086,6 +3086,7 @@ _base_check_enable_msix(struct MPT3SAS_ADAPTER *ioc) void mpt3sas_base_free_irq(struct MPT3SAS_ADAPTER *ioc) { + unsigned int irq; struct adapter_reply_queue *reply_q, *next; if (list_empty(&ioc->reply_queue_list)) @@ -3098,9 +3099,10 @@ mpt3sas_base_free_irq(struct MPT3SAS_ADAPTER *ioc) continue; } - if (ioc->smp_affinity_enable) - irq_set_affinity_hint(pci_irq_vector(ioc->pdev, - reply_q->msix_index), NULL); + if (ioc->smp_affinity_enable) { + irq = pci_irq_vector(ioc->pdev, reply_q->msix_index); + irq_update_affinity_hint(irq, NULL); + } free_irq(pci_irq_vector(ioc->pdev, reply_q->msix_index), reply_q); kfree(reply_q); @@ -3167,18 +3169,15 @@ out: * @ioc: per adapter object * * The enduser would need to set the affinity via /proc/irq/#/smp_affinity - * - * It would nice if we could call irq_set_affinity, however it is not - * an exported symbol */ static void _base_assign_reply_queues(struct MPT3SAS_ADAPTER *ioc) { - unsigned int cpu, nr_cpus, nr_msix, index = 0; + unsigned int cpu, nr_cpus, nr_msix, index = 0, irq; struct adapter_reply_queue *reply_q; - int local_numa_node; int iopoll_q_count = ioc->reply_queue_count - ioc->iopoll_q_start_index; + const struct cpumask *mask; if (!_base_is_controller_msix_enabled(ioc)) return; @@ -3201,11 +3200,11 @@ _base_assign_reply_queues(struct MPT3SAS_ADAPTER *ioc) * corresponding to high iops queues. */ if (ioc->high_iops_queues) { - local_numa_node = dev_to_node(&ioc->pdev->dev); + mask = cpumask_of_node(dev_to_node(&ioc->pdev->dev)); for (index = 0; index < ioc->high_iops_queues; index++) { - irq_set_affinity_hint(pci_irq_vector(ioc->pdev, - index), cpumask_of_node(local_numa_node)); + irq = pci_irq_vector(ioc->pdev, index); + irq_set_affinity_and_hint(irq, mask); } } diff --git a/drivers/soc/fsl/dpio/dpio-driver.c b/drivers/soc/fsl/dpio/dpio-driver.c index dd948889eeab..5a2edc48dd79 100644 --- a/drivers/soc/fsl/dpio/dpio-driver.c +++ b/drivers/soc/fsl/dpio/dpio-driver.c @@ -88,7 +88,7 @@ static void unregister_dpio_irq_handlers(struct fsl_mc_device *dpio_dev) irq = dpio_dev->irqs[0]; /* clear the affinity hint */ - irq_set_affinity_hint(irq->msi_desc->irq, NULL); + irq_set_affinity_hint(irq->virq, NULL); } static int register_dpio_irq_handlers(struct fsl_mc_device *dpio_dev, int cpu) @@ -98,7 +98,7 @@ static int register_dpio_irq_handlers(struct fsl_mc_device *dpio_dev, int cpu) irq = dpio_dev->irqs[0]; error = devm_request_irq(&dpio_dev->dev, - irq->msi_desc->irq, + irq->virq, dpio_irq_handler, 0, dev_name(&dpio_dev->dev), @@ -111,10 +111,10 @@ static int register_dpio_irq_handlers(struct fsl_mc_device *dpio_dev, int cpu) } /* set the affinity hint */ - if (irq_set_affinity_hint(irq->msi_desc->irq, cpumask_of(cpu))) + if (irq_set_affinity_hint(irq->virq, cpumask_of(cpu))) dev_err(&dpio_dev->dev, "irq_set_affinity failed irq %d cpu %d\n", - irq->msi_desc->irq, cpu); + irq->virq, cpu); return 0; } diff --git a/drivers/soc/ti/k3-ringacc.c b/drivers/soc/ti/k3-ringacc.c index 312ba0f98ad7..56be39161489 100644 --- a/drivers/soc/ti/k3-ringacc.c +++ b/drivers/soc/ti/k3-ringacc.c @@ -647,7 +647,7 @@ int k3_ringacc_get_ring_irq_num(struct k3_ring *ring) if (!ring) return -EINVAL; - irq_num = ti_sci_inta_msi_get_virq(ring->parent->dev, ring->ring_id); + irq_num = msi_get_virq(ring->parent->dev, ring->ring_id); if (irq_num <= 0) irq_num = -EINVAL; return irq_num; @@ -1356,9 +1356,9 @@ static int k3_ringacc_init(struct platform_device *pdev, struct resource *res; int ret, i; - dev->msi_domain = of_msi_get_domain(dev, dev->of_node, + dev->msi.domain = of_msi_get_domain(dev, dev->of_node, DOMAIN_BUS_TI_SCI_INTA_MSI); - if (!dev->msi_domain) { + if (!dev->msi.domain) { dev_err(dev, "Failed to get MSI domain\n"); return -EPROBE_DEFER; } diff --git a/drivers/soc/ti/ti_sci_inta_msi.c b/drivers/soc/ti/ti_sci_inta_msi.c index a1d9c027022a..991c78b34745 100644 --- a/drivers/soc/ti/ti_sci_inta_msi.c +++ b/drivers/soc/ti/ti_sci_inta_msi.c @@ -51,6 +51,7 @@ struct irq_domain *ti_sci_inta_msi_create_irq_domain(struct fwnode_handle *fwnod struct irq_domain *domain; ti_sci_inta_msi_update_chip_ops(info); + info->flags |= MSI_FLAG_FREE_MSI_DESCS; domain = msi_create_irq_domain(fwnode, info, parent); if (domain) @@ -60,50 +61,32 @@ struct irq_domain *ti_sci_inta_msi_create_irq_domain(struct fwnode_handle *fwnod } EXPORT_SYMBOL_GPL(ti_sci_inta_msi_create_irq_domain); -static void ti_sci_inta_msi_free_descs(struct device *dev) -{ - struct msi_desc *desc, *tmp; - - list_for_each_entry_safe(desc, tmp, dev_to_msi_list(dev), list) { - list_del(&desc->list); - free_msi_entry(desc); - } -} - static int ti_sci_inta_msi_alloc_descs(struct device *dev, struct ti_sci_resource *res) { - struct msi_desc *msi_desc; + struct msi_desc msi_desc; int set, i, count = 0; + memset(&msi_desc, 0, sizeof(msi_desc)); + msi_desc.nvec_used = 1; + for (set = 0; set < res->sets; set++) { - for (i = 0; i < res->desc[set].num; i++) { - msi_desc = alloc_msi_entry(dev, 1, NULL); - if (!msi_desc) { - ti_sci_inta_msi_free_descs(dev); - return -ENOMEM; - } - - msi_desc->inta.dev_index = res->desc[set].start + i; - INIT_LIST_HEAD(&msi_desc->list); - list_add_tail(&msi_desc->list, dev_to_msi_list(dev)); - count++; + for (i = 0; i < res->desc[set].num; i++, count++) { + msi_desc.msi_index = res->desc[set].start + i; + if (msi_add_msi_desc(dev, &msi_desc)) + goto fail; } - for (i = 0; i < res->desc[set].num_sec; i++) { - msi_desc = alloc_msi_entry(dev, 1, NULL); - if (!msi_desc) { - ti_sci_inta_msi_free_descs(dev); - return -ENOMEM; - } - msi_desc->inta.dev_index = res->desc[set].start_sec + i; - INIT_LIST_HEAD(&msi_desc->list); - list_add_tail(&msi_desc->list, dev_to_msi_list(dev)); - count++; + for (i = 0; i < res->desc[set].num_sec; i++, count++) { + msi_desc.msi_index = res->desc[set].start_sec + i; + if (msi_add_msi_desc(dev, &msi_desc)) + goto fail; } } - return count; +fail: + msi_free_msi_descs(dev); + return -ENOMEM; } int ti_sci_inta_msi_domain_alloc_irqs(struct device *dev, @@ -120,39 +103,22 @@ int ti_sci_inta_msi_domain_alloc_irqs(struct device *dev, if (pdev->id < 0) return -ENODEV; - nvec = ti_sci_inta_msi_alloc_descs(dev, res); - if (nvec <= 0) - return nvec; + ret = msi_setup_device_data(dev); + if (ret) + return ret; - ret = msi_domain_alloc_irqs(msi_domain, dev, nvec); - if (ret) { - dev_err(dev, "Failed to allocate IRQs %d\n", ret); - goto cleanup; + msi_lock_descs(dev); + nvec = ti_sci_inta_msi_alloc_descs(dev, res); + if (nvec <= 0) { + ret = nvec; + goto unlock; } - return 0; - -cleanup: - ti_sci_inta_msi_free_descs(&pdev->dev); + ret = msi_domain_alloc_irqs_descs_locked(msi_domain, dev, nvec); + if (ret) + dev_err(dev, "Failed to allocate IRQs %d\n", ret); +unlock: + msi_unlock_descs(dev); return ret; } EXPORT_SYMBOL_GPL(ti_sci_inta_msi_domain_alloc_irqs); - -void ti_sci_inta_msi_domain_free_irqs(struct device *dev) -{ - msi_domain_free_irqs(dev->msi_domain, dev); - ti_sci_inta_msi_free_descs(dev); -} -EXPORT_SYMBOL_GPL(ti_sci_inta_msi_domain_free_irqs); - -unsigned int ti_sci_inta_msi_get_virq(struct device *dev, u32 dev_index) -{ - struct msi_desc *desc; - - for_each_msi_entry(desc, dev) - if (desc->inta.dev_index == dev_index) - return desc->irq; - - return -ENODEV; -} -EXPORT_SYMBOL_GPL(ti_sci_inta_msi_get_virq); diff --git a/drivers/vfio/fsl-mc/vfio_fsl_mc_intr.c b/drivers/vfio/fsl-mc/vfio_fsl_mc_intr.c index 77e584093a23..7b428eac3d3e 100644 --- a/drivers/vfio/fsl-mc/vfio_fsl_mc_intr.c +++ b/drivers/vfio/fsl-mc/vfio_fsl_mc_intr.c @@ -67,7 +67,7 @@ static int vfio_set_trigger(struct vfio_fsl_mc_device *vdev, int hwirq; int ret; - hwirq = vdev->mc_dev->irqs[index]->msi_desc->irq; + hwirq = vdev->mc_dev->irqs[index]->virq; if (irq->trigger) { free_irq(hwirq, irq); kfree(irq->name); @@ -137,7 +137,7 @@ static int vfio_fsl_mc_set_irq_trigger(struct vfio_fsl_mc_device *vdev, return vfio_set_trigger(vdev, index, fd); } - hwirq = vdev->mc_dev->irqs[index]->msi_desc->irq; + hwirq = vdev->mc_dev->irqs[index]->virq; irq = &vdev->mc_irqs[index]; diff --git a/include/dt-bindings/clock/exynos4.h b/include/dt-bindings/clock/exynos4.h index 88ec3968b90a..acbfbab875ec 100644 --- a/include/dt-bindings/clock/exynos4.h +++ b/include/dt-bindings/clock/exynos4.h @@ -209,6 +209,7 @@ #define CLK_ACLK400_MCUISP 395 /* Exynos4x12 only */ #define CLK_MOUT_HDMI 396 #define CLK_MOUT_MIXER 397 +#define CLK_MOUT_VPLLSRC 398 /* gate clocks - ppmu */ #define CLK_PPMULEFT 400 @@ -236,9 +237,10 @@ #define CLK_DIV_C2C 458 /* Exynos4x12 only */ #define CLK_DIV_GDL 459 #define CLK_DIV_GDR 460 +#define CLK_DIV_CORE2 461 /* must be greater than maximal clock id */ -#define CLK_NR_CLKS 461 +#define CLK_NR_CLKS 462 /* Exynos4x12 ISP clocks */ #define CLK_ISP_FIMC_ISP 1 diff --git a/include/dt-bindings/clock/exynos5250.h b/include/dt-bindings/clock/exynos5250.h index e259cc01f22f..4680da7357d3 100644 --- a/include/dt-bindings/clock/exynos5250.h +++ b/include/dt-bindings/clock/exynos5250.h @@ -19,6 +19,7 @@ #define CLK_FOUT_EPLL 7 #define CLK_FOUT_VPLL 8 #define CLK_ARM_CLK 9 +#define CLK_DIV_ARM2 10 /* gate for special clocks (sclk) */ #define CLK_SCLK_CAM_BAYER 128 @@ -174,8 +175,9 @@ #define CLK_MOUT_ACLK300_DISP1_SUB 1027 #define CLK_MOUT_APLL 1028 #define CLK_MOUT_MPLL 1029 +#define CLK_MOUT_VPLLSRC 1030 /* must be greater than maximal clock id */ -#define CLK_NR_CLKS 1030 +#define CLK_NR_CLKS 1031 #endif /* _DT_BINDINGS_CLOCK_EXYNOS_5250_H */ diff --git a/include/dt-bindings/clock/exynos7885.h b/include/dt-bindings/clock/exynos7885.h new file mode 100644 index 000000000000..1f8701691d62 --- /dev/null +++ b/include/dt-bindings/clock/exynos7885.h @@ -0,0 +1,115 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (c) 2021 Dávid Virág + * + * Device Tree binding constants for Exynos7885 clock controller. + */ + +#ifndef _DT_BINDINGS_CLOCK_EXYNOS_7885_H +#define _DT_BINDINGS_CLOCK_EXYNOS_7885_H + +/* CMU_TOP */ +#define CLK_FOUT_SHARED0_PLL 1 +#define CLK_FOUT_SHARED1_PLL 2 +#define CLK_DOUT_SHARED0_DIV2 3 +#define CLK_DOUT_SHARED0_DIV3 4 +#define CLK_DOUT_SHARED0_DIV4 5 +#define CLK_DOUT_SHARED0_DIV5 6 +#define CLK_DOUT_SHARED1_DIV2 7 +#define CLK_DOUT_SHARED1_DIV3 8 +#define CLK_DOUT_SHARED1_DIV4 9 +#define CLK_MOUT_CORE_BUS 10 +#define CLK_MOUT_CORE_CCI 11 +#define CLK_MOUT_CORE_G3D 12 +#define CLK_DOUT_CORE_BUS 13 +#define CLK_DOUT_CORE_CCI 14 +#define CLK_DOUT_CORE_G3D 15 +#define CLK_GOUT_CORE_BUS 16 +#define CLK_GOUT_CORE_CCI 17 +#define CLK_GOUT_CORE_G3D 18 +#define CLK_MOUT_PERI_BUS 19 +#define CLK_MOUT_PERI_SPI0 20 +#define CLK_MOUT_PERI_SPI1 21 +#define CLK_MOUT_PERI_UART0 22 +#define CLK_MOUT_PERI_UART1 23 +#define CLK_MOUT_PERI_UART2 24 +#define CLK_MOUT_PERI_USI0 25 +#define CLK_MOUT_PERI_USI1 26 +#define CLK_MOUT_PERI_USI2 27 +#define CLK_DOUT_PERI_BUS 28 +#define CLK_DOUT_PERI_SPI0 29 +#define CLK_DOUT_PERI_SPI1 30 +#define CLK_DOUT_PERI_UART0 31 +#define CLK_DOUT_PERI_UART1 32 +#define CLK_DOUT_PERI_UART2 33 +#define CLK_DOUT_PERI_USI0 34 +#define CLK_DOUT_PERI_USI1 35 +#define CLK_DOUT_PERI_USI2 36 +#define CLK_GOUT_PERI_BUS 37 +#define CLK_GOUT_PERI_SPI0 38 +#define CLK_GOUT_PERI_SPI1 39 +#define CLK_GOUT_PERI_UART0 40 +#define CLK_GOUT_PERI_UART1 41 +#define CLK_GOUT_PERI_UART2 42 +#define CLK_GOUT_PERI_USI0 43 +#define CLK_GOUT_PERI_USI1 44 +#define CLK_GOUT_PERI_USI2 45 +#define TOP_NR_CLK 46 + +/* CMU_CORE */ +#define CLK_MOUT_CORE_BUS_USER 1 +#define CLK_MOUT_CORE_CCI_USER 2 +#define CLK_MOUT_CORE_G3D_USER 3 +#define CLK_MOUT_CORE_GIC 4 +#define CLK_DOUT_CORE_BUSP 5 +#define CLK_GOUT_CCI_ACLK 6 +#define CLK_GOUT_GIC400_CLK 7 +#define CORE_NR_CLK 8 + +/* CMU_PERI */ +#define CLK_MOUT_PERI_BUS_USER 1 +#define CLK_MOUT_PERI_SPI0_USER 2 +#define CLK_MOUT_PERI_SPI1_USER 3 +#define CLK_MOUT_PERI_UART0_USER 4 +#define CLK_MOUT_PERI_UART1_USER 5 +#define CLK_MOUT_PERI_UART2_USER 6 +#define CLK_MOUT_PERI_USI0_USER 7 +#define CLK_MOUT_PERI_USI1_USER 8 +#define CLK_MOUT_PERI_USI2_USER 9 +#define CLK_GOUT_GPIO_TOP_PCLK 10 +#define CLK_GOUT_HSI2C0_PCLK 11 +#define CLK_GOUT_HSI2C1_PCLK 12 +#define CLK_GOUT_HSI2C2_PCLK 13 +#define CLK_GOUT_HSI2C3_PCLK 14 +#define CLK_GOUT_I2C0_PCLK 15 +#define CLK_GOUT_I2C1_PCLK 16 +#define CLK_GOUT_I2C2_PCLK 17 +#define CLK_GOUT_I2C3_PCLK 18 +#define CLK_GOUT_I2C4_PCLK 19 +#define CLK_GOUT_I2C5_PCLK 20 +#define CLK_GOUT_I2C6_PCLK 21 +#define CLK_GOUT_I2C7_PCLK 22 +#define CLK_GOUT_PWM_MOTOR_PCLK 23 +#define CLK_GOUT_SPI0_PCLK 24 +#define CLK_GOUT_SPI0_EXT_CLK 25 +#define CLK_GOUT_SPI1_PCLK 26 +#define CLK_GOUT_SPI1_EXT_CLK 27 +#define CLK_GOUT_UART0_EXT_UCLK 28 +#define CLK_GOUT_UART0_PCLK 29 +#define CLK_GOUT_UART1_EXT_UCLK 30 +#define CLK_GOUT_UART1_PCLK 31 +#define CLK_GOUT_UART2_EXT_UCLK 32 +#define CLK_GOUT_UART2_PCLK 33 +#define CLK_GOUT_USI0_PCLK 34 +#define CLK_GOUT_USI0_SCLK 35 +#define CLK_GOUT_USI1_PCLK 36 +#define CLK_GOUT_USI1_SCLK 37 +#define CLK_GOUT_USI2_PCLK 38 +#define CLK_GOUT_USI2_SCLK 39 +#define CLK_GOUT_MCT_PCLK 40 +#define CLK_GOUT_SYSREG_PERI_PCLK 41 +#define CLK_GOUT_WDT0_PCLK 42 +#define CLK_GOUT_WDT1_PCLK 43 +#define PERI_NR_CLK 44 + +#endif /* _DT_BINDINGS_CLOCK_EXYNOS_7885_H */ diff --git a/include/dt-bindings/clock/exynos850.h b/include/dt-bindings/clock/exynos850.h index 8999184f94a2..0b6a3c6a7c90 100644 --- a/include/dt-bindings/clock/exynos850.h +++ b/include/dt-bindings/clock/exynos850.h @@ -55,7 +55,55 @@ #define CLK_GOUT_PERI_BUS 43 #define CLK_GOUT_PERI_UART 44 #define CLK_GOUT_PERI_IP 45 -#define TOP_NR_CLK 46 +#define CLK_MOUT_CLKCMU_APM_BUS 46 +#define CLK_DOUT_CLKCMU_APM_BUS 47 +#define CLK_GOUT_CLKCMU_APM_BUS 48 +#define TOP_NR_CLK 49 + +/* CMU_APM */ +#define CLK_RCO_I3C_PMIC 1 +#define OSCCLK_RCO_APM 2 +#define CLK_RCO_APM__ALV 3 +#define CLK_DLL_DCO 4 +#define CLK_MOUT_APM_BUS_USER 5 +#define CLK_MOUT_RCO_APM_I3C_USER 6 +#define CLK_MOUT_RCO_APM_USER 7 +#define CLK_MOUT_DLL_USER 8 +#define CLK_MOUT_CLKCMU_CHUB_BUS 9 +#define CLK_MOUT_APM_BUS 10 +#define CLK_MOUT_APM_I3C 11 +#define CLK_DOUT_CLKCMU_CHUB_BUS 12 +#define CLK_DOUT_APM_BUS 13 +#define CLK_DOUT_APM_I3C 14 +#define CLK_GOUT_CLKCMU_CMGP_BUS 15 +#define CLK_GOUT_CLKCMU_CHUB_BUS 16 +#define CLK_GOUT_RTC_PCLK 17 +#define CLK_GOUT_TOP_RTC_PCLK 18 +#define CLK_GOUT_I3C_PCLK 19 +#define CLK_GOUT_I3C_SCLK 20 +#define CLK_GOUT_SPEEDY_PCLK 21 +#define CLK_GOUT_GPIO_ALIVE_PCLK 22 +#define CLK_GOUT_PMU_ALIVE_PCLK 23 +#define CLK_GOUT_SYSREG_APM_PCLK 24 +#define APM_NR_CLK 25 + +/* CMU_CMGP */ +#define CLK_RCO_CMGP 1 +#define CLK_MOUT_CMGP_ADC 2 +#define CLK_MOUT_CMGP_USI0 3 +#define CLK_MOUT_CMGP_USI1 4 +#define CLK_DOUT_CMGP_ADC 5 +#define CLK_DOUT_CMGP_USI0 6 +#define CLK_DOUT_CMGP_USI1 7 +#define CLK_GOUT_CMGP_ADC_S0_PCLK 8 +#define CLK_GOUT_CMGP_ADC_S1_PCLK 9 +#define CLK_GOUT_CMGP_GPIO_PCLK 10 +#define CLK_GOUT_CMGP_USI0_IPCLK 11 +#define CLK_GOUT_CMGP_USI0_PCLK 12 +#define CLK_GOUT_CMGP_USI1_IPCLK 13 +#define CLK_GOUT_CMGP_USI1_PCLK 14 +#define CLK_GOUT_SYSREG_CMGP_PCLK 15 +#define CMGP_NR_CLK 16 /* CMU_HSI */ #define CLK_MOUT_HSI_BUS_USER 1 @@ -123,7 +171,9 @@ #define CLK_GOUT_MMC_EMBD_SDCLKIN 10 #define CLK_GOUT_SSS_ACLK 11 #define CLK_GOUT_SSS_PCLK 12 -#define CORE_NR_CLK 13 +#define CLK_GOUT_GPIO_CORE_PCLK 13 +#define CLK_GOUT_SYSREG_CORE_PCLK 14 +#define CORE_NR_CLK 15 /* CMU_DPU */ #define CLK_MOUT_DPU_USER 1 diff --git a/include/dt-bindings/clock/imx8mp-clock.h b/include/dt-bindings/clock/imx8mp-clock.h index 43927a1b9e94..235c7a00d379 100644 --- a/include/dt-bindings/clock/imx8mp-clock.h +++ b/include/dt-bindings/clock/imx8mp-clock.h @@ -117,7 +117,6 @@ #define IMX8MP_CLK_AUDIO_AHB 108 #define IMX8MP_CLK_MIPI_DSI_ESC_RX 109 #define IMX8MP_CLK_IPG_ROOT 110 -#define IMX8MP_CLK_IPG_AUDIO_ROOT 111 #define IMX8MP_CLK_DRAM_ALT 112 #define IMX8MP_CLK_DRAM_APB 113 #define IMX8MP_CLK_VPU_G1 114 diff --git a/include/dt-bindings/clock/ingenic,jz4760-cgu.h b/include/dt-bindings/clock/ingenic,jz4760-cgu.h index 4bb2e19c4743..9fb04ebac6de 100644 --- a/include/dt-bindings/clock/ingenic,jz4760-cgu.h +++ b/include/dt-bindings/clock/ingenic,jz4760-cgu.h @@ -50,5 +50,7 @@ #define JZ4760_CLK_LPCLK_DIV 41 #define JZ4760_CLK_TVE 42 #define JZ4760_CLK_LPCLK 43 +#define JZ4760_CLK_MDMA 44 +#define JZ4760_CLK_BDMA 45 #endif /* __DT_BINDINGS_CLOCK_JZ4760_CGU_H__ */ diff --git a/include/dt-bindings/clock/ingenic,jz4770-cgu.h b/include/dt-bindings/clock/ingenic,jz4770-cgu.h index d68a7695a1f8..0b475e8ae321 100644 --- a/include/dt-bindings/clock/ingenic,jz4770-cgu.h +++ b/include/dt-bindings/clock/ingenic,jz4770-cgu.h @@ -54,5 +54,6 @@ #define JZ4770_CLK_OTG_PHY 45 #define JZ4770_CLK_EXT512 46 #define JZ4770_CLK_RTC 47 +#define JZ4770_CLK_BDMA 48 #endif /* __DT_BINDINGS_CLOCK_JZ4770_CGU_H__ */ diff --git a/include/dt-bindings/clock/microchip,lan966x.h b/include/dt-bindings/clock/microchip,lan966x.h new file mode 100644 index 000000000000..6f9d43d76d5a --- /dev/null +++ b/include/dt-bindings/clock/microchip,lan966x.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (c) 2021 Microchip Inc. + * + * Author: Kavyasree Kotagiri + */ + +#ifndef _DT_BINDINGS_CLK_LAN966X_H +#define _DT_BINDINGS_CLK_LAN966X_H + +#define GCK_ID_QSPI0 0 +#define GCK_ID_QSPI1 1 +#define GCK_ID_QSPI2 2 +#define GCK_ID_SDMMC0 3 +#define GCK_ID_PI 4 +#define GCK_ID_MCAN0 5 +#define GCK_ID_MCAN1 6 +#define GCK_ID_FLEXCOM0 7 +#define GCK_ID_FLEXCOM1 8 +#define GCK_ID_FLEXCOM2 9 +#define GCK_ID_FLEXCOM3 10 +#define GCK_ID_FLEXCOM4 11 +#define GCK_ID_TIMER 12 +#define GCK_ID_USB_REFCLK 13 + +/* Gate clocks */ +#define GCK_GATE_UHPHS 14 +#define GCK_GATE_UDPHS 15 +#define GCK_GATE_MCRAMC 16 +#define GCK_GATE_HMATRIX 17 + +#define N_CLOCKS 18 + +#endif diff --git a/include/dt-bindings/clock/mt7986-clk.h b/include/dt-bindings/clock/mt7986-clk.h new file mode 100644 index 000000000000..5a9b169324b0 --- /dev/null +++ b/include/dt-bindings/clock/mt7986-clk.h @@ -0,0 +1,169 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (c) 2021 MediaTek Inc. + * Author: Sam Shih + */ + +#ifndef _DT_BINDINGS_CLK_MT7986_H +#define _DT_BINDINGS_CLK_MT7986_H + +/* APMIXEDSYS */ + +#define CLK_APMIXED_ARMPLL 0 +#define CLK_APMIXED_NET2PLL 1 +#define CLK_APMIXED_MMPLL 2 +#define CLK_APMIXED_SGMPLL 3 +#define CLK_APMIXED_WEDMCUPLL 4 +#define CLK_APMIXED_NET1PLL 5 +#define CLK_APMIXED_MPLL 6 +#define CLK_APMIXED_APLL2 7 + +/* TOPCKGEN */ + +#define CLK_TOP_XTAL 0 +#define CLK_TOP_XTAL_D2 1 +#define CLK_TOP_RTC_32K 2 +#define CLK_TOP_RTC_32P7K 3 +#define CLK_TOP_MPLL_D2 4 +#define CLK_TOP_MPLL_D4 5 +#define CLK_TOP_MPLL_D8 6 +#define CLK_TOP_MPLL_D8_D2 7 +#define CLK_TOP_MPLL_D3_D2 8 +#define CLK_TOP_MMPLL_D2 9 +#define CLK_TOP_MMPLL_D4 10 +#define CLK_TOP_MMPLL_D8 11 +#define CLK_TOP_MMPLL_D8_D2 12 +#define CLK_TOP_MMPLL_D3_D8 13 +#define CLK_TOP_MMPLL_U2PHY 14 +#define CLK_TOP_APLL2_D4 15 +#define CLK_TOP_NET1PLL_D4 16 +#define CLK_TOP_NET1PLL_D5 17 +#define CLK_TOP_NET1PLL_D5_D2 18 +#define CLK_TOP_NET1PLL_D5_D4 19 +#define CLK_TOP_NET1PLL_D8_D2 20 +#define CLK_TOP_NET1PLL_D8_D4 21 +#define CLK_TOP_NET2PLL_D4 22 +#define CLK_TOP_NET2PLL_D4_D2 23 +#define CLK_TOP_NET2PLL_D3_D2 24 +#define CLK_TOP_WEDMCUPLL_D5_D2 25 +#define CLK_TOP_NFI1X_SEL 26 +#define CLK_TOP_SPINFI_SEL 27 +#define CLK_TOP_SPI_SEL 28 +#define CLK_TOP_SPIM_MST_SEL 29 +#define CLK_TOP_UART_SEL 30 +#define CLK_TOP_PWM_SEL 31 +#define CLK_TOP_I2C_SEL 32 +#define CLK_TOP_PEXTP_TL_SEL 33 +#define CLK_TOP_EMMC_250M_SEL 34 +#define CLK_TOP_EMMC_416M_SEL 35 +#define CLK_TOP_F_26M_ADC_SEL 36 +#define CLK_TOP_DRAMC_SEL 37 +#define CLK_TOP_DRAMC_MD32_SEL 38 +#define CLK_TOP_SYSAXI_SEL 39 +#define CLK_TOP_SYSAPB_SEL 40 +#define CLK_TOP_ARM_DB_MAIN_SEL 41 +#define CLK_TOP_ARM_DB_JTSEL 42 +#define CLK_TOP_NETSYS_SEL 43 +#define CLK_TOP_NETSYS_500M_SEL 44 +#define CLK_TOP_NETSYS_MCU_SEL 45 +#define CLK_TOP_NETSYS_2X_SEL 46 +#define CLK_TOP_SGM_325M_SEL 47 +#define CLK_TOP_SGM_REG_SEL 48 +#define CLK_TOP_A1SYS_SEL 49 +#define CLK_TOP_CONN_MCUSYS_SEL 50 +#define CLK_TOP_EIP_B_SEL 51 +#define CLK_TOP_PCIE_PHY_SEL 52 +#define CLK_TOP_USB3_PHY_SEL 53 +#define CLK_TOP_F26M_SEL 54 +#define CLK_TOP_AUD_L_SEL 55 +#define CLK_TOP_A_TUNER_SEL 56 +#define CLK_TOP_U2U3_SEL 57 +#define CLK_TOP_U2U3_SYS_SEL 58 +#define CLK_TOP_U2U3_XHCI_SEL 59 +#define CLK_TOP_DA_U2_REFSEL 60 +#define CLK_TOP_DA_U2_CK_1P_SEL 61 +#define CLK_TOP_AP2CNN_HOST_SEL 62 +#define CLK_TOP_JTAG 63 + +/* INFRACFG */ + +#define CLK_INFRA_SYSAXI_D2 0 +#define CLK_INFRA_UART0_SEL 1 +#define CLK_INFRA_UART1_SEL 2 +#define CLK_INFRA_UART2_SEL 3 +#define CLK_INFRA_SPI0_SEL 4 +#define CLK_INFRA_SPI1_SEL 5 +#define CLK_INFRA_PWM1_SEL 6 +#define CLK_INFRA_PWM2_SEL 7 +#define CLK_INFRA_PWM_BSEL 8 +#define CLK_INFRA_PCIE_SEL 9 +#define CLK_INFRA_GPT_STA 10 +#define CLK_INFRA_PWM_HCK 11 +#define CLK_INFRA_PWM_STA 12 +#define CLK_INFRA_PWM1_CK 13 +#define CLK_INFRA_PWM2_CK 14 +#define CLK_INFRA_CQ_DMA_CK 15 +#define CLK_INFRA_EIP97_CK 16 +#define CLK_INFRA_AUD_BUS_CK 17 +#define CLK_INFRA_AUD_26M_CK 18 +#define CLK_INFRA_AUD_L_CK 19 +#define CLK_INFRA_AUD_AUD_CK 20 +#define CLK_INFRA_AUD_EG2_CK 21 +#define CLK_INFRA_DRAMC_26M_CK 22 +#define CLK_INFRA_DBG_CK 23 +#define CLK_INFRA_AP_DMA_CK 24 +#define CLK_INFRA_SEJ_CK 25 +#define CLK_INFRA_SEJ_13M_CK 26 +#define CLK_INFRA_THERM_CK 27 +#define CLK_INFRA_I2C0_CK 28 +#define CLK_INFRA_UART0_CK 29 +#define CLK_INFRA_UART1_CK 30 +#define CLK_INFRA_UART2_CK 31 +#define CLK_INFRA_NFI1_CK 32 +#define CLK_INFRA_SPINFI1_CK 33 +#define CLK_INFRA_NFI_HCK_CK 34 +#define CLK_INFRA_SPI0_CK 35 +#define CLK_INFRA_SPI1_CK 36 +#define CLK_INFRA_SPI0_HCK_CK 37 +#define CLK_INFRA_SPI1_HCK_CK 38 +#define CLK_INFRA_FRTC_CK 39 +#define CLK_INFRA_MSDC_CK 40 +#define CLK_INFRA_MSDC_HCK_CK 41 +#define CLK_INFRA_MSDC_133M_CK 42 +#define CLK_INFRA_MSDC_66M_CK 43 +#define CLK_INFRA_ADC_26M_CK 44 +#define CLK_INFRA_ADC_FRC_CK 45 +#define CLK_INFRA_FBIST2FPC_CK 46 +#define CLK_INFRA_IUSB_133_CK 47 +#define CLK_INFRA_IUSB_66M_CK 48 +#define CLK_INFRA_IUSB_SYS_CK 49 +#define CLK_INFRA_IUSB_CK 50 +#define CLK_INFRA_IPCIE_CK 51 +#define CLK_INFRA_IPCIE_PIPE_CK 52 +#define CLK_INFRA_IPCIER_CK 53 +#define CLK_INFRA_IPCIEB_CK 54 +#define CLK_INFRA_TRNG_CK 55 + +/* SGMIISYS_0 */ + +#define CLK_SGMII0_TX250M_EN 0 +#define CLK_SGMII0_RX250M_EN 1 +#define CLK_SGMII0_CDR_REF 2 +#define CLK_SGMII0_CDR_FB 3 + +/* SGMIISYS_1 */ + +#define CLK_SGMII1_TX250M_EN 0 +#define CLK_SGMII1_RX250M_EN 1 +#define CLK_SGMII1_CDR_REF 2 +#define CLK_SGMII1_CDR_FB 3 + +/* ETHSYS */ + +#define CLK_ETH_FE_EN 0 +#define CLK_ETH_GP2_EN 1 +#define CLK_ETH_GP1_EN 2 +#define CLK_ETH_WOCPU1_EN 3 +#define CLK_ETH_WOCPU0_EN 4 + +#endif /* _DT_BINDINGS_CLK_MT7986_H */ diff --git a/include/dt-bindings/clock/qcom,gcc-msm8976.h b/include/dt-bindings/clock/qcom,gcc-msm8976.h new file mode 100644 index 000000000000..51955fd49426 --- /dev/null +++ b/include/dt-bindings/clock/qcom,gcc-msm8976.h @@ -0,0 +1,240 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (C) 2016, The Linux Foundation. All rights reserved. + * Copyright (C) 2016-2021, AngeloGioacchino Del Regno + * + */ + +#ifndef _DT_BINDINGS_CLK_MSM_GCC_8976_H +#define _DT_BINDINGS_CLK_MSM_GCC_8976_H + +#define GPLL0 0 +#define GPLL2 1 +#define GPLL3 2 +#define GPLL4 3 +#define GPLL6 4 +#define GPLL0_CLK_SRC 5 +#define GPLL2_CLK_SRC 6 +#define GPLL3_CLK_SRC 7 +#define GPLL4_CLK_SRC 8 +#define GPLL6_CLK_SRC 9 +#define GCC_BLSP1_QUP1_SPI_APPS_CLK 10 +#define GCC_BLSP1_QUP1_I2C_APPS_CLK 11 +#define GCC_BLSP1_QUP2_I2C_APPS_CLK 12 +#define GCC_BLSP1_QUP2_SPI_APPS_CLK 13 +#define GCC_BLSP1_QUP3_I2C_APPS_CLK 14 +#define GCC_BLSP1_QUP3_SPI_APPS_CLK 15 +#define GCC_BLSP1_QUP4_I2C_APPS_CLK 16 +#define GCC_BLSP1_QUP4_SPI_APPS_CLK 17 +#define GCC_BLSP1_UART1_APPS_CLK 18 +#define GCC_BLSP1_UART2_APPS_CLK 19 +#define GCC_BLSP2_QUP1_I2C_APPS_CLK 20 +#define GCC_BLSP2_QUP1_SPI_APPS_CLK 21 +#define GCC_BLSP2_QUP2_I2C_APPS_CLK 22 +#define GCC_BLSP2_QUP2_SPI_APPS_CLK 23 +#define GCC_BLSP2_QUP3_I2C_APPS_CLK 24 +#define GCC_BLSP2_QUP3_SPI_APPS_CLK 25 +#define GCC_BLSP2_QUP4_I2C_APPS_CLK 26 +#define GCC_BLSP2_QUP4_SPI_APPS_CLK 27 +#define GCC_BLSP2_UART1_APPS_CLK 28 +#define GCC_BLSP2_UART2_APPS_CLK 29 +#define GCC_CAMSS_CCI_AHB_CLK 30 +#define GCC_CAMSS_CCI_CLK 31 +#define GCC_CAMSS_CPP_AHB_CLK 32 +#define GCC_CAMSS_CPP_AXI_CLK 33 +#define GCC_CAMSS_CPP_CLK 34 +#define GCC_CAMSS_CSI0_AHB_CLK 35 +#define GCC_CAMSS_CSI0_CLK 36 +#define GCC_CAMSS_CSI0PHY_CLK 37 +#define GCC_CAMSS_CSI0PIX_CLK 38 +#define GCC_CAMSS_CSI0RDI_CLK 39 +#define GCC_CAMSS_CSI1_AHB_CLK 40 +#define GCC_CAMSS_CSI1_CLK 41 +#define GCC_CAMSS_CSI1PHY_CLK 42 +#define GCC_CAMSS_CSI1PIX_CLK 43 +#define GCC_CAMSS_CSI1RDI_CLK 44 +#define GCC_CAMSS_CSI2_AHB_CLK 45 +#define GCC_CAMSS_CSI2_CLK 46 +#define GCC_CAMSS_CSI2PHY_CLK 47 +#define GCC_CAMSS_CSI2PIX_CLK 48 +#define GCC_CAMSS_CSI2RDI_CLK 49 +#define GCC_CAMSS_CSI_VFE0_CLK 50 +#define GCC_CAMSS_CSI_VFE1_CLK 51 +#define GCC_CAMSS_GP0_CLK 52 +#define GCC_CAMSS_GP1_CLK 53 +#define GCC_CAMSS_ISPIF_AHB_CLK 54 +#define GCC_CAMSS_JPEG0_CLK 55 +#define GCC_CAMSS_JPEG_AHB_CLK 56 +#define GCC_CAMSS_JPEG_AXI_CLK 57 +#define GCC_CAMSS_MCLK0_CLK 58 +#define GCC_CAMSS_MCLK1_CLK 59 +#define GCC_CAMSS_MCLK2_CLK 60 +#define GCC_CAMSS_MICRO_AHB_CLK 61 +#define GCC_CAMSS_CSI0PHYTIMER_CLK 62 +#define GCC_CAMSS_CSI1PHYTIMER_CLK 63 +#define GCC_CAMSS_AHB_CLK 64 +#define GCC_CAMSS_TOP_AHB_CLK 65 +#define GCC_CAMSS_VFE0_CLK 66 +#define GCC_CAMSS_VFE_AHB_CLK 67 +#define GCC_CAMSS_VFE_AXI_CLK 68 +#define GCC_CAMSS_VFE1_AHB_CLK 69 +#define GCC_CAMSS_VFE1_AXI_CLK 70 +#define GCC_CAMSS_VFE1_CLK 71 +#define GCC_DCC_CLK 72 +#define GCC_GP1_CLK 73 +#define GCC_GP2_CLK 74 +#define GCC_GP3_CLK 75 +#define GCC_MDSS_AHB_CLK 76 +#define GCC_MDSS_AXI_CLK 77 +#define GCC_MDSS_ESC0_CLK 78 +#define GCC_MDSS_ESC1_CLK 79 +#define GCC_MDSS_MDP_CLK 80 +#define GCC_MDSS_VSYNC_CLK 81 +#define GCC_MSS_CFG_AHB_CLK 82 +#define GCC_MSS_Q6_BIMC_AXI_CLK 83 +#define GCC_PDM2_CLK 84 +#define GCC_PRNG_AHB_CLK 85 +#define GCC_PDM_AHB_CLK 86 +#define GCC_RBCPR_GFX_AHB_CLK 87 +#define GCC_RBCPR_GFX_CLK 88 +#define GCC_SDCC1_AHB_CLK 89 +#define GCC_SDCC1_APPS_CLK 90 +#define GCC_SDCC1_ICE_CORE_CLK 91 +#define GCC_SDCC2_AHB_CLK 92 +#define GCC_SDCC2_APPS_CLK 93 +#define GCC_SDCC3_AHB_CLK 94 +#define GCC_SDCC3_APPS_CLK 95 +#define GCC_USB2A_PHY_SLEEP_CLK 96 +#define GCC_USB_HS_PHY_CFG_AHB_CLK 97 +#define GCC_USB_FS_AHB_CLK 98 +#define GCC_USB_FS_IC_CLK 99 +#define GCC_USB_FS_SYSTEM_CLK 100 +#define GCC_USB_HS_AHB_CLK 101 +#define GCC_USB_HS_SYSTEM_CLK 102 +#define GCC_VENUS0_AHB_CLK 103 +#define GCC_VENUS0_AXI_CLK 104 +#define GCC_VENUS0_CORE0_VCODEC0_CLK 105 +#define GCC_VENUS0_CORE1_VCODEC0_CLK 106 +#define GCC_VENUS0_VCODEC0_CLK 107 +#define GCC_APSS_AHB_CLK 108 +#define GCC_APSS_AXI_CLK 109 +#define GCC_BLSP1_AHB_CLK 110 +#define GCC_BLSP2_AHB_CLK 111 +#define GCC_BOOT_ROM_AHB_CLK 112 +#define GCC_CRYPTO_AHB_CLK 113 +#define GCC_CRYPTO_AXI_CLK 114 +#define GCC_CRYPTO_CLK 115 +#define GCC_CPP_TBU_CLK 116 +#define GCC_APSS_TCU_CLK 117 +#define GCC_JPEG_TBU_CLK 118 +#define GCC_MDP_RT_TBU_CLK 119 +#define GCC_MDP_TBU_CLK 120 +#define GCC_SMMU_CFG_CLK 121 +#define GCC_VENUS_1_TBU_CLK 122 +#define GCC_VENUS_TBU_CLK 123 +#define GCC_VFE1_TBU_CLK 124 +#define GCC_VFE_TBU_CLK 125 +#define GCC_APS_0_CLK 126 +#define GCC_APS_1_CLK 127 +#define APS_0_CLK_SRC 128 +#define APS_1_CLK_SRC 129 +#define APSS_AHB_CLK_SRC 130 +#define BLSP1_QUP1_I2C_APPS_CLK_SRC 131 +#define BLSP1_QUP1_SPI_APPS_CLK_SRC 132 +#define BLSP1_QUP2_I2C_APPS_CLK_SRC 133 +#define BLSP1_QUP2_SPI_APPS_CLK_SRC 134 +#define BLSP1_QUP3_I2C_APPS_CLK_SRC 135 +#define BLSP1_QUP3_SPI_APPS_CLK_SRC 136 +#define BLSP1_QUP4_I2C_APPS_CLK_SRC 137 +#define BLSP1_QUP4_SPI_APPS_CLK_SRC 138 +#define BLSP1_UART1_APPS_CLK_SRC 139 +#define BLSP1_UART2_APPS_CLK_SRC 140 +#define BLSP2_QUP1_I2C_APPS_CLK_SRC 141 +#define BLSP2_QUP1_SPI_APPS_CLK_SRC 142 +#define BLSP2_QUP2_I2C_APPS_CLK_SRC 143 +#define BLSP2_QUP2_SPI_APPS_CLK_SRC 144 +#define BLSP2_QUP3_I2C_APPS_CLK_SRC 145 +#define BLSP2_QUP3_SPI_APPS_CLK_SRC 146 +#define BLSP2_QUP4_I2C_APPS_CLK_SRC 147 +#define BLSP2_QUP4_SPI_APPS_CLK_SRC 148 +#define BLSP2_UART1_APPS_CLK_SRC 149 +#define BLSP2_UART2_APPS_CLK_SRC 150 +#define CCI_CLK_SRC 151 +#define CPP_CLK_SRC 152 +#define CSI0_CLK_SRC 153 +#define CSI1_CLK_SRC 154 +#define CSI2_CLK_SRC 155 +#define CAMSS_GP0_CLK_SRC 156 +#define CAMSS_GP1_CLK_SRC 157 +#define JPEG0_CLK_SRC 158 +#define MCLK0_CLK_SRC 159 +#define MCLK1_CLK_SRC 160 +#define MCLK2_CLK_SRC 161 +#define CSI0PHYTIMER_CLK_SRC 162 +#define CSI1PHYTIMER_CLK_SRC 163 +#define CAMSS_TOP_AHB_CLK_SRC 164 +#define VFE0_CLK_SRC 165 +#define VFE1_CLK_SRC 166 +#define CRYPTO_CLK_SRC 167 +#define GP1_CLK_SRC 168 +#define GP2_CLK_SRC 169 +#define GP3_CLK_SRC 170 +#define ESC0_CLK_SRC 171 +#define ESC1_CLK_SRC 172 +#define MDP_CLK_SRC 173 +#define VSYNC_CLK_SRC 174 +#define PDM2_CLK_SRC 175 +#define RBCPR_GFX_CLK_SRC 176 +#define SDCC1_APPS_CLK_SRC 177 +#define SDCC1_ICE_CORE_CLK_SRC 178 +#define SDCC2_APPS_CLK_SRC 179 +#define SDCC3_APPS_CLK_SRC 180 +#define USB_FS_IC_CLK_SRC 181 +#define USB_FS_SYSTEM_CLK_SRC 182 +#define USB_HS_SYSTEM_CLK_SRC 183 +#define VCODEC0_CLK_SRC 184 +#define GCC_MDSS_BYTE0_CLK_SRC 185 +#define GCC_MDSS_BYTE1_CLK_SRC 186 +#define GCC_MDSS_BYTE0_CLK 187 +#define GCC_MDSS_BYTE1_CLK 188 +#define GCC_MDSS_PCLK0_CLK_SRC 189 +#define GCC_MDSS_PCLK1_CLK_SRC 190 +#define GCC_MDSS_PCLK0_CLK 191 +#define GCC_MDSS_PCLK1_CLK 192 +#define GCC_GFX3D_CLK_SRC 193 +#define GCC_GFX3D_OXILI_CLK 194 +#define GCC_GFX3D_BIMC_CLK 195 +#define GCC_GFX3D_OXILI_AHB_CLK 196 +#define GCC_GFX3D_OXILI_AON_CLK 197 +#define GCC_GFX3D_OXILI_GMEM_CLK 198 +#define GCC_GFX3D_OXILI_TIMER_CLK 199 +#define GCC_GFX3D_TBU0_CLK 200 +#define GCC_GFX3D_TBU1_CLK 201 +#define GCC_GFX3D_TCU_CLK 202 +#define GCC_GFX3D_GTCU_AHB_CLK 203 + +/* GCC block resets */ +#define RST_CAMSS_MICRO_BCR 0 +#define RST_USB_HS_BCR 1 +#define RST_QUSB2_PHY_BCR 2 +#define RST_USB2_HS_PHY_ONLY_BCR 3 +#define RST_USB_HS_PHY_CFG_AHB_BCR 4 +#define RST_USB_FS_BCR 5 +#define RST_CAMSS_CSI1PIX_BCR 6 +#define RST_CAMSS_CSI_VFE1_BCR 7 +#define RST_CAMSS_VFE1_BCR 8 +#define RST_CAMSS_CPP_BCR 9 + +/* GDSCs */ +#define VENUS_GDSC 0 +#define VENUS_CORE0_GDSC 1 +#define VENUS_CORE1_GDSC 2 +#define MDSS_GDSC 3 +#define JPEG_GDSC 4 +#define VFE0_GDSC 5 +#define VFE1_GDSC 6 +#define CPP_GDSC 7 +#define OXILI_GX_GDSC 8 +#define OXILI_CX_GDSC 9 + +#endif /* _DT_BINDINGS_CLK_MSM_GCC_8976_H */ diff --git a/include/dt-bindings/clock/sun20i-d1-ccu.h b/include/dt-bindings/clock/sun20i-d1-ccu.h new file mode 100644 index 000000000000..e3ac53315e1a --- /dev/null +++ b/include/dt-bindings/clock/sun20i-d1-ccu.h @@ -0,0 +1,156 @@ +/* SPDX-License-Identifier: (GPL-2.0+ or MIT) */ +/* + * Copyright (C) 2020 huangzhenwei@allwinnertech.com + * Copyright (C) 2021 Samuel Holland + */ + +#ifndef _DT_BINDINGS_CLK_SUN20I_D1_CCU_H_ +#define _DT_BINDINGS_CLK_SUN20I_D1_CCU_H_ + +#define CLK_PLL_CPUX 0 +#define CLK_PLL_DDR0 1 +#define CLK_PLL_PERIPH0_4X 2 +#define CLK_PLL_PERIPH0_2X 3 +#define CLK_PLL_PERIPH0_800M 4 +#define CLK_PLL_PERIPH0 5 +#define CLK_PLL_PERIPH0_DIV3 6 +#define CLK_PLL_VIDEO0_4X 7 +#define CLK_PLL_VIDEO0_2X 8 +#define CLK_PLL_VIDEO0 9 +#define CLK_PLL_VIDEO1_4X 10 +#define CLK_PLL_VIDEO1_2X 11 +#define CLK_PLL_VIDEO1 12 +#define CLK_PLL_VE 13 +#define CLK_PLL_AUDIO0_4X 14 +#define CLK_PLL_AUDIO0_2X 15 +#define CLK_PLL_AUDIO0 16 +#define CLK_PLL_AUDIO1 17 +#define CLK_PLL_AUDIO1_DIV2 18 +#define CLK_PLL_AUDIO1_DIV5 19 +#define CLK_CPUX 20 +#define CLK_CPUX_AXI 21 +#define CLK_CPUX_APB 22 +#define CLK_PSI_AHB 23 +#define CLK_APB0 24 +#define CLK_APB1 25 +#define CLK_MBUS 26 +#define CLK_DE 27 +#define CLK_BUS_DE 28 +#define CLK_DI 29 +#define CLK_BUS_DI 30 +#define CLK_G2D 31 +#define CLK_BUS_G2D 32 +#define CLK_CE 33 +#define CLK_BUS_CE 34 +#define CLK_VE 35 +#define CLK_BUS_VE 36 +#define CLK_BUS_DMA 37 +#define CLK_BUS_MSGBOX0 38 +#define CLK_BUS_MSGBOX1 39 +#define CLK_BUS_MSGBOX2 40 +#define CLK_BUS_SPINLOCK 41 +#define CLK_BUS_HSTIMER 42 +#define CLK_AVS 43 +#define CLK_BUS_DBG 44 +#define CLK_BUS_PWM 45 +#define CLK_BUS_IOMMU 46 +#define CLK_DRAM 47 +#define CLK_MBUS_DMA 48 +#define CLK_MBUS_VE 49 +#define CLK_MBUS_CE 50 +#define CLK_MBUS_TVIN 51 +#define CLK_MBUS_CSI 52 +#define CLK_MBUS_G2D 53 +#define CLK_MBUS_RISCV 54 +#define CLK_BUS_DRAM 55 +#define CLK_MMC0 56 +#define CLK_MMC1 57 +#define CLK_MMC2 58 +#define CLK_BUS_MMC0 59 +#define CLK_BUS_MMC1 60 +#define CLK_BUS_MMC2 61 +#define CLK_BUS_UART0 62 +#define CLK_BUS_UART1 63 +#define CLK_BUS_UART2 64 +#define CLK_BUS_UART3 65 +#define CLK_BUS_UART4 66 +#define CLK_BUS_UART5 67 +#define CLK_BUS_I2C0 68 +#define CLK_BUS_I2C1 69 +#define CLK_BUS_I2C2 70 +#define CLK_BUS_I2C3 71 +#define CLK_SPI0 72 +#define CLK_SPI1 73 +#define CLK_BUS_SPI0 74 +#define CLK_BUS_SPI1 75 +#define CLK_EMAC_25M 76 +#define CLK_BUS_EMAC 77 +#define CLK_IR_TX 78 +#define CLK_BUS_IR_TX 79 +#define CLK_BUS_GPADC 80 +#define CLK_BUS_THS 81 +#define CLK_I2S0 82 +#define CLK_I2S1 83 +#define CLK_I2S2 84 +#define CLK_I2S2_ASRC 85 +#define CLK_BUS_I2S0 86 +#define CLK_BUS_I2S1 87 +#define CLK_BUS_I2S2 88 +#define CLK_SPDIF_TX 89 +#define CLK_SPDIF_RX 90 +#define CLK_BUS_SPDIF 91 +#define CLK_DMIC 92 +#define CLK_BUS_DMIC 93 +#define CLK_AUDIO_DAC 94 +#define CLK_AUDIO_ADC 95 +#define CLK_BUS_AUDIO 96 +#define CLK_USB_OHCI0 97 +#define CLK_USB_OHCI1 98 +#define CLK_BUS_OHCI0 99 +#define CLK_BUS_OHCI1 100 +#define CLK_BUS_EHCI0 101 +#define CLK_BUS_EHCI1 102 +#define CLK_BUS_OTG 103 +#define CLK_BUS_LRADC 104 +#define CLK_BUS_DPSS_TOP 105 +#define CLK_HDMI_24M 106 +#define CLK_HDMI_CEC_32K 107 +#define CLK_HDMI_CEC 108 +#define CLK_BUS_HDMI 109 +#define CLK_MIPI_DSI 110 +#define CLK_BUS_MIPI_DSI 111 +#define CLK_TCON_LCD0 112 +#define CLK_BUS_TCON_LCD0 113 +#define CLK_TCON_TV 114 +#define CLK_BUS_TCON_TV 115 +#define CLK_TVE 116 +#define CLK_BUS_TVE_TOP 117 +#define CLK_BUS_TVE 118 +#define CLK_TVD 119 +#define CLK_BUS_TVD_TOP 120 +#define CLK_BUS_TVD 121 +#define CLK_LEDC 122 +#define CLK_BUS_LEDC 123 +#define CLK_CSI_TOP 124 +#define CLK_CSI_MCLK 125 +#define CLK_BUS_CSI 126 +#define CLK_TPADC 127 +#define CLK_BUS_TPADC 128 +#define CLK_BUS_TZMA 129 +#define CLK_DSP 130 +#define CLK_BUS_DSP_CFG 131 +#define CLK_RISCV 132 +#define CLK_RISCV_AXI 133 +#define CLK_BUS_RISCV_CFG 134 +#define CLK_FANOUT_24M 135 +#define CLK_FANOUT_12M 136 +#define CLK_FANOUT_16M 137 +#define CLK_FANOUT_25M 138 +#define CLK_FANOUT_32K 139 +#define CLK_FANOUT_27M 140 +#define CLK_FANOUT_PCLK 141 +#define CLK_FANOUT0 142 +#define CLK_FANOUT1 143 +#define CLK_FANOUT2 144 + +#endif /* _DT_BINDINGS_CLK_SUN20I_D1_CCU_H_ */ diff --git a/include/dt-bindings/clock/sun20i-d1-r-ccu.h b/include/dt-bindings/clock/sun20i-d1-r-ccu.h new file mode 100644 index 000000000000..4c2697fd32b0 --- /dev/null +++ b/include/dt-bindings/clock/sun20i-d1-r-ccu.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: (GPL-2.0+ or MIT) */ +/* + * Copyright (C) 2021 Samuel Holland + */ + +#ifndef _DT_BINDINGS_CLK_SUN20I_D1_R_CCU_H_ +#define _DT_BINDINGS_CLK_SUN20I_D1_R_CCU_H_ + +#define CLK_R_AHB 0 + +#define CLK_BUS_R_TIMER 2 +#define CLK_BUS_R_TWD 3 +#define CLK_BUS_R_PPU 4 +#define CLK_R_IR_RX 5 +#define CLK_BUS_R_IR_RX 6 +#define CLK_BUS_R_RTC 7 +#define CLK_BUS_R_CPUCFG 8 + +#endif /* _DT_BINDINGS_CLK_SUN20I_D1_R_CCU_H_ */ diff --git a/include/dt-bindings/clock/toshiba,tmpv770x.h b/include/dt-bindings/clock/toshiba,tmpv770x.h new file mode 100644 index 000000000000..5fce713001fd --- /dev/null +++ b/include/dt-bindings/clock/toshiba,tmpv770x.h @@ -0,0 +1,181 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ + +#ifndef _DT_BINDINGS_CLOCK_TOSHIBA_TMPV770X_H_ +#define _DT_BINDINGS_CLOCK_TOSHIBA_TMPV770X_H_ + +/* PLL */ +#define TMPV770X_PLL_PIPLL0 0 +#define TMPV770X_PLL_PIPLL1 1 +#define TMPV770X_PLL_PIDNNPLL 2 +#define TMPV770X_PLL_PIETHERPLL 3 +#define TMPV770X_PLL_PIDDRCPLL 4 +#define TMPV770X_PLL_PIVOIFPLL 5 +#define TMPV770X_PLL_PIIMGERPLL 6 +#define TMPV770X_NR_PLL 7 + +/* Clocks */ +#define TMPV770X_CLK_PIPLL1_DIV1 0 +#define TMPV770X_CLK_PIPLL1_DIV2 1 +#define TMPV770X_CLK_PIPLL1_DIV4 2 +#define TMPV770X_CLK_PIDNNPLL_DIV1 3 +#define TMPV770X_CLK_DDRC_PHY_PLL0 4 +#define TMPV770X_CLK_DDRC_PHY_PLL1 5 +#define TMPV770X_CLK_D_PHYPLL 6 +#define TMPV770X_CLK_PHY_PCIEPLL 7 +#define TMPV770X_CLK_CA53CL0 8 +#define TMPV770X_CLK_CA53CL1 9 +#define TMPV770X_CLK_PISDMAC 10 +#define TMPV770X_CLK_PIPDMAC0 11 +#define TMPV770X_CLK_PIPDMAC1 12 +#define TMPV770X_CLK_PIWRAM 13 +#define TMPV770X_CLK_DDRC0 14 +#define TMPV770X_CLK_DDRC0_SCLK 15 +#define TMPV770X_CLK_DDRC0_NCLK 16 +#define TMPV770X_CLK_DDRC0_MCLK 17 +#define TMPV770X_CLK_DDRC0_APBCLK 18 +#define TMPV770X_CLK_DDRC1 19 +#define TMPV770X_CLK_DDRC1_SCLK 20 +#define TMPV770X_CLK_DDRC1_NCLK 21 +#define TMPV770X_CLK_DDRC1_MCLK 22 +#define TMPV770X_CLK_DDRC1_APBCLK 23 +#define TMPV770X_CLK_HOX 24 +#define TMPV770X_CLK_PCIE_MSTR 25 +#define TMPV770X_CLK_PCIE_AUX 26 +#define TMPV770X_CLK_PIINTC 27 +#define TMPV770X_CLK_PIETHER_BUS 28 +#define TMPV770X_CLK_PISPI0 29 +#define TMPV770X_CLK_PISPI1 30 +#define TMPV770X_CLK_PISPI2 31 +#define TMPV770X_CLK_PISPI3 32 +#define TMPV770X_CLK_PISPI4 33 +#define TMPV770X_CLK_PISPI5 34 +#define TMPV770X_CLK_PISPI6 35 +#define TMPV770X_CLK_PIUART0 36 +#define TMPV770X_CLK_PIUART1 37 +#define TMPV770X_CLK_PIUART2 38 +#define TMPV770X_CLK_PIUART3 39 +#define TMPV770X_CLK_PII2C0 40 +#define TMPV770X_CLK_PII2C1 41 +#define TMPV770X_CLK_PII2C2 42 +#define TMPV770X_CLK_PII2C3 43 +#define TMPV770X_CLK_PII2C4 44 +#define TMPV770X_CLK_PII2C5 45 +#define TMPV770X_CLK_PII2C6 46 +#define TMPV770X_CLK_PII2C7 47 +#define TMPV770X_CLK_PII2C8 48 +#define TMPV770X_CLK_PIGPIO 49 +#define TMPV770X_CLK_PIPGM 50 +#define TMPV770X_CLK_PIPCMIF 51 +#define TMPV770X_CLK_PIPCMIF_AUDIO_O 52 +#define TMPV770X_CLK_PIPCMIF_AUDIO_I 53 +#define TMPV770X_CLK_PICMPT0 54 +#define TMPV770X_CLK_PICMPT1 55 +#define TMPV770X_CLK_PITSC 56 +#define TMPV770X_CLK_PIUWDT 57 +#define TMPV770X_CLK_PISWDT 58 +#define TMPV770X_CLK_WDTCLK 59 +#define TMPV770X_CLK_PISUBUS_150M 60 +#define TMPV770X_CLK_PISUBUS_300M 61 +#define TMPV770X_CLK_PIPMU 62 +#define TMPV770X_CLK_PIGPMU 63 +#define TMPV770X_CLK_PITMU 64 +#define TMPV770X_CLK_WRCK 65 +#define TMPV770X_CLK_PIEMM 66 +#define TMPV770X_CLK_PIMISC 67 +#define TMPV770X_CLK_PIGCOMM 68 +#define TMPV770X_CLK_PIDCOMM 69 +#define TMPV770X_CLK_PICKMON 70 +#define TMPV770X_CLK_PIMBUS 71 +#define TMPV770X_CLK_SBUSCLK 72 +#define TMPV770X_CLK_DDR0_APBCLKCLK 73 +#define TMPV770X_CLK_DDR1_APBCLKCLK 74 +#define TMPV770X_CLK_DSP0_PBCLK 75 +#define TMPV770X_CLK_DSP1_PBCLK 76 +#define TMPV770X_CLK_DSP2_PBCLK 77 +#define TMPV770X_CLK_DSP3_PBCLK 78 +#define TMPV770X_CLK_DSVIIF0_APBCLK 79 +#define TMPV770X_CLK_VIIF0_APBCLK 80 +#define TMPV770X_CLK_VIIF0_CFGCLK 81 +#define TMPV770X_CLK_VIIF1_APBCLK 82 +#define TMPV770X_CLK_VIIF1_CFGCLK 83 +#define TMPV770X_CLK_VIIF2_APBCLK 84 +#define TMPV770X_CLK_VIIF2_CFGCLK 85 +#define TMPV770X_CLK_VIIF3_APBCLK 86 +#define TMPV770X_CLK_VIIF3_CFGCLK 87 +#define TMPV770X_CLK_VIIF4_APBCLK 88 +#define TMPV770X_CLK_VIIF4_CFGCLK 89 +#define TMPV770X_CLK_VIIF5_APBCLK 90 +#define TMPV770X_CLK_VIIF5_CFGCLK 91 +#define TMPV770X_CLK_VOIF_SBUSCLK 92 +#define TMPV770X_CLK_VOIF_PROCCLK 93 +#define TMPV770X_CLK_VOIF_DPHYCFGCLK 94 +#define TMPV770X_CLK_DNN0 95 +#define TMPV770X_CLK_STMAT 96 +#define TMPV770X_CLK_HWA0 97 +#define TMPV770X_CLK_AFFINE0 98 +#define TMPV770X_CLK_HAMAT 99 +#define TMPV770X_CLK_SMLDB 100 +#define TMPV770X_CLK_HWA0_ASYNC 101 +#define TMPV770X_CLK_HWA2 102 +#define TMPV770X_CLK_FLMAT 103 +#define TMPV770X_CLK_PYRAMID 104 +#define TMPV770X_CLK_HWA2_ASYNC 105 +#define TMPV770X_CLK_DSP0 106 +#define TMPV770X_CLK_VIIFBS0 107 +#define TMPV770X_CLK_VIIFBS0_L2ISP 108 +#define TMPV770X_CLK_VIIFBS0_L1ISP 109 +#define TMPV770X_CLK_VIIFBS0_PROC 110 +#define TMPV770X_CLK_VIIFBS1 111 +#define TMPV770X_CLK_VIIFBS2 112 +#define TMPV770X_CLK_VIIFOP_MBUS 113 +#define TMPV770X_CLK_VIIFOP0_PROC 114 +#define TMPV770X_CLK_PIETHER_2P5M 115 +#define TMPV770X_CLK_PIETHER_25M 116 +#define TMPV770X_CLK_PIETHER_50M 117 +#define TMPV770X_CLK_PIETHER_125M 118 +#define TMPV770X_CLK_VOIF0_DPHYCFG 119 +#define TMPV770X_CLK_VOIF0_PROC 120 +#define TMPV770X_CLK_VOIF0_SBUS 121 +#define TMPV770X_CLK_VOIF0_DSIREF 122 +#define TMPV770X_CLK_VOIF0_PIXEL 123 +#define TMPV770X_CLK_PIREFCLK 124 +#define TMPV770X_CLK_SBUS 125 +#define TMPV770X_CLK_BUSLCK 126 +#define TMPV770X_NR_CLK 127 + +/* Reset */ +#define TMPV770X_RESET_PIETHER_2P5M 0 +#define TMPV770X_RESET_PIETHER_25M 1 +#define TMPV770X_RESET_PIETHER_50M 2 +#define TMPV770X_RESET_PIETHER_125M 3 +#define TMPV770X_RESET_HOX 4 +#define TMPV770X_RESET_PCIE_MSTR 5 +#define TMPV770X_RESET_PCIE_AUX 6 +#define TMPV770X_RESET_PIINTC 7 +#define TMPV770X_RESET_PIETHER_BUS 8 +#define TMPV770X_RESET_PISPI0 9 +#define TMPV770X_RESET_PISPI1 10 +#define TMPV770X_RESET_PISPI2 11 +#define TMPV770X_RESET_PISPI3 12 +#define TMPV770X_RESET_PISPI4 13 +#define TMPV770X_RESET_PISPI5 14 +#define TMPV770X_RESET_PISPI6 15 +#define TMPV770X_RESET_PIUART0 16 +#define TMPV770X_RESET_PIUART1 17 +#define TMPV770X_RESET_PIUART2 18 +#define TMPV770X_RESET_PIUART3 19 +#define TMPV770X_RESET_PII2C0 20 +#define TMPV770X_RESET_PII2C1 21 +#define TMPV770X_RESET_PII2C2 22 +#define TMPV770X_RESET_PII2C3 23 +#define TMPV770X_RESET_PII2C4 24 +#define TMPV770X_RESET_PII2C5 25 +#define TMPV770X_RESET_PII2C6 26 +#define TMPV770X_RESET_PII2C7 27 +#define TMPV770X_RESET_PII2C8 28 +#define TMPV770X_RESET_PIPCMIF 29 +#define TMPV770X_RESET_PICKMON 30 +#define TMPV770X_RESET_SBUSCLK 31 +#define TMPV770X_NR_RESET 32 + +#endif /*_DT_BINDINGS_CLOCK_TOSHIBA_TMPV770X_H_ */ diff --git a/include/dt-bindings/mailbox/qcom-ipcc.h b/include/dt-bindings/mailbox/qcom-ipcc.h index eb91a6c05b71..9296d0bb5f34 100644 --- a/include/dt-bindings/mailbox/qcom-ipcc.h +++ b/include/dt-bindings/mailbox/qcom-ipcc.h @@ -8,6 +8,7 @@ /* Signal IDs for MPROC protocol */ #define IPCC_MPROC_SIGNAL_GLINK_QMP 0 +#define IPCC_MPROC_SIGNAL_TZ 1 #define IPCC_MPROC_SIGNAL_SMP2P 2 #define IPCC_MPROC_SIGNAL_PING 3 @@ -29,6 +30,7 @@ #define IPCC_CLIENT_PCIE1 14 #define IPCC_CLIENT_PCIE2 15 #define IPCC_CLIENT_SPSS 16 +#define IPCC_CLIENT_TME 23 #define IPCC_CLIENT_WPSS 24 #endif diff --git a/include/dt-bindings/reset/sun20i-d1-ccu.h b/include/dt-bindings/reset/sun20i-d1-ccu.h new file mode 100644 index 000000000000..de9ff5203239 --- /dev/null +++ b/include/dt-bindings/reset/sun20i-d1-ccu.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: (GPL-2.0+ or MIT) */ +/* + * Copyright (c) 2020 huangzhenwei@allwinnertech.com + * Copyright (C) 2021 Samuel Holland + */ + +#ifndef _DT_BINDINGS_RST_SUN20I_D1_CCU_H_ +#define _DT_BINDINGS_RST_SUN20I_D1_CCU_H_ + +#define RST_MBUS 0 +#define RST_BUS_DE 1 +#define RST_BUS_DI 2 +#define RST_BUS_G2D 3 +#define RST_BUS_CE 4 +#define RST_BUS_VE 5 +#define RST_BUS_DMA 6 +#define RST_BUS_MSGBOX0 7 +#define RST_BUS_MSGBOX1 8 +#define RST_BUS_MSGBOX2 9 +#define RST_BUS_SPINLOCK 10 +#define RST_BUS_HSTIMER 11 +#define RST_BUS_DBG 12 +#define RST_BUS_PWM 13 +#define RST_BUS_DRAM 14 +#define RST_BUS_MMC0 15 +#define RST_BUS_MMC1 16 +#define RST_BUS_MMC2 17 +#define RST_BUS_UART0 18 +#define RST_BUS_UART1 19 +#define RST_BUS_UART2 20 +#define RST_BUS_UART3 21 +#define RST_BUS_UART4 22 +#define RST_BUS_UART5 23 +#define RST_BUS_I2C0 24 +#define RST_BUS_I2C1 25 +#define RST_BUS_I2C2 26 +#define RST_BUS_I2C3 27 +#define RST_BUS_SPI0 28 +#define RST_BUS_SPI1 29 +#define RST_BUS_EMAC 30 +#define RST_BUS_IR_TX 31 +#define RST_BUS_GPADC 32 +#define RST_BUS_THS 33 +#define RST_BUS_I2S0 34 +#define RST_BUS_I2S1 35 +#define RST_BUS_I2S2 36 +#define RST_BUS_SPDIF 37 +#define RST_BUS_DMIC 38 +#define RST_BUS_AUDIO 39 +#define RST_USB_PHY0 40 +#define RST_USB_PHY1 41 +#define RST_BUS_OHCI0 42 +#define RST_BUS_OHCI1 43 +#define RST_BUS_EHCI0 44 +#define RST_BUS_EHCI1 45 +#define RST_BUS_OTG 46 +#define RST_BUS_LRADC 47 +#define RST_BUS_DPSS_TOP 48 +#define RST_BUS_HDMI_SUB 49 +#define RST_BUS_HDMI_MAIN 50 +#define RST_BUS_MIPI_DSI 51 +#define RST_BUS_TCON_LCD0 52 +#define RST_BUS_TCON_TV 53 +#define RST_BUS_LVDS0 54 +#define RST_BUS_TVE 55 +#define RST_BUS_TVE_TOP 56 +#define RST_BUS_TVD 57 +#define RST_BUS_TVD_TOP 58 +#define RST_BUS_LEDC 59 +#define RST_BUS_CSI 60 +#define RST_BUS_TPADC 61 +#define RST_DSP 62 +#define RST_BUS_DSP_CFG 63 +#define RST_BUS_DSP_DBG 64 +#define RST_BUS_RISCV_CFG 65 + +#endif /* _DT_BINDINGS_RST_SUN20I_D1_CCU_H_ */ diff --git a/include/dt-bindings/reset/sun20i-d1-r-ccu.h b/include/dt-bindings/reset/sun20i-d1-r-ccu.h new file mode 100644 index 000000000000..d93d6423d283 --- /dev/null +++ b/include/dt-bindings/reset/sun20i-d1-r-ccu.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: (GPL-2.0+ or MIT) */ +/* + * Copyright (C) 2021 Samuel Holland + */ + +#ifndef _DT_BINDINGS_RST_SUN20I_D1_R_CCU_H_ +#define _DT_BINDINGS_RST_SUN20I_D1_R_CCU_H_ + +#define RST_BUS_R_TIMER 0 +#define RST_BUS_R_TWD 1 +#define RST_BUS_R_PPU 2 +#define RST_BUS_R_IR_RX 3 +#define RST_BUS_R_RTC 4 +#define RST_BUS_R_CPUCFG 5 + +#endif /* _DT_BINDINGS_RST_SUN20I_D1_R_CCU_H_ */ diff --git a/include/dt-bindings/reset/toshiba,tmpv770x.h b/include/dt-bindings/reset/toshiba,tmpv770x.h new file mode 100644 index 000000000000..c1007acb1941 --- /dev/null +++ b/include/dt-bindings/reset/toshiba,tmpv770x.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ + +#ifndef _DT_BINDINGS_RESET_TOSHIBA_TMPV770X_H_ +#define _DT_BINDINGS_RESET_TOSHIBA_TMPV770X_H_ + +/* Reset */ +#define TMPV770X_RESET_PIETHER_2P5M 0 +#define TMPV770X_RESET_PIETHER_25M 1 +#define TMPV770X_RESET_PIETHER_50M 2 +#define TMPV770X_RESET_PIETHER_125M 3 +#define TMPV770X_RESET_HOX 4 +#define TMPV770X_RESET_PCIE_MSTR 5 +#define TMPV770X_RESET_PCIE_AUX 6 +#define TMPV770X_RESET_PIINTC 7 +#define TMPV770X_RESET_PIETHER_BUS 8 +#define TMPV770X_RESET_PISPI0 9 +#define TMPV770X_RESET_PISPI1 10 +#define TMPV770X_RESET_PISPI2 11 +#define TMPV770X_RESET_PISPI3 12 +#define TMPV770X_RESET_PISPI4 13 +#define TMPV770X_RESET_PISPI5 14 +#define TMPV770X_RESET_PISPI6 15 +#define TMPV770X_RESET_PIUART0 16 +#define TMPV770X_RESET_PIUART1 17 +#define TMPV770X_RESET_PIUART2 18 +#define TMPV770X_RESET_PIUART3 19 +#define TMPV770X_RESET_PII2C0 20 +#define TMPV770X_RESET_PII2C1 21 +#define TMPV770X_RESET_PII2C2 22 +#define TMPV770X_RESET_PII2C3 23 +#define TMPV770X_RESET_PII2C4 24 +#define TMPV770X_RESET_PII2C5 25 +#define TMPV770X_RESET_PII2C6 26 +#define TMPV770X_RESET_PII2C7 27 +#define TMPV770X_RESET_PII2C8 28 +#define TMPV770X_RESET_PIPCMIF 29 +#define TMPV770X_RESET_PICKMON 30 +#define TMPV770X_RESET_SBUSCLK 31 +#define TMPV770X_NR_RESET 32 + +#endif /*_DT_BINDINGS_RESET_TOSHIBA_TMPV770X_H_ */ diff --git a/include/linux/clk-provider.h b/include/linux/clk-provider.h index f72c3728811f..c22936ebb06a 100644 --- a/include/linux/clk-provider.h +++ b/include/linux/clk-provider.h @@ -504,6 +504,13 @@ struct clk_hw *__clk_hw_register_gate(struct device *dev, unsigned long flags, void __iomem *reg, u8 bit_idx, u8 clk_gate_flags, spinlock_t *lock); +struct clk_hw *__devm_clk_hw_register_gate(struct device *dev, + struct device_node *np, const char *name, + const char *parent_name, const struct clk_hw *parent_hw, + const struct clk_parent_data *parent_data, + unsigned long flags, + void __iomem *reg, u8 bit_idx, + u8 clk_gate_flags, spinlock_t *lock); struct clk *clk_register_gate(struct device *dev, const char *name, const char *parent_name, unsigned long flags, void __iomem *reg, u8 bit_idx, @@ -558,6 +565,22 @@ struct clk *clk_register_gate(struct device *dev, const char *name, __clk_hw_register_gate((dev), NULL, (name), NULL, NULL, (parent_data), \ (flags), (reg), (bit_idx), \ (clk_gate_flags), (lock)) +/** + * devm_clk_hw_register_gate - register a gate clock with the clock framework + * @dev: device that is registering this clock + * @name: name of this clock + * @parent_name: name of this clock's parent + * @flags: framework-specific flags for this clock + * @reg: register address to control gating of this clock + * @bit_idx: which bit in the register controls gating of this clock + * @clk_gate_flags: gate-specific flags for this clock + * @lock: shared register lock for this clock + */ +#define devm_clk_hw_register_gate(dev, name, parent_name, flags, reg, bit_idx,\ + clk_gate_flags, lock) \ + __devm_clk_hw_register_gate((dev), NULL, (name), (parent_name), NULL, \ + NULL, (flags), (reg), (bit_idx), \ + (clk_gate_flags), (lock)) void clk_unregister_gate(struct clk *clk); void clk_hw_unregister_gate(struct clk_hw *hw); int clk_gate_is_enabled(struct clk_hw *hw); diff --git a/include/linux/clk/sunxi-ng.h b/include/linux/clk/sunxi-ng.h index 3cd14acde0a1..cf32123b39f5 100644 --- a/include/linux/clk/sunxi-ng.h +++ b/include/linux/clk/sunxi-ng.h @@ -6,22 +6,7 @@ #ifndef _LINUX_CLK_SUNXI_NG_H_ #define _LINUX_CLK_SUNXI_NG_H_ -#include - -#ifdef CONFIG_SUNXI_CCU int sunxi_ccu_set_mmc_timing_mode(struct clk *clk, bool new_mode); int sunxi_ccu_get_mmc_timing_mode(struct clk *clk); -#else -static inline int sunxi_ccu_set_mmc_timing_mode(struct clk *clk, - bool new_mode) -{ - return -ENOTSUPP; -} - -static inline int sunxi_ccu_get_mmc_timing_mode(struct clk *clk) -{ - return -ENOTSUPP; -} -#endif #endif diff --git a/include/linux/device.h b/include/linux/device.h index e270cb740b9e..93459724dcde 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -45,6 +45,7 @@ struct iommu_ops; struct iommu_group; struct dev_pin_info; struct dev_iommu; +struct msi_device_data; /** * struct subsys_interface - interfaces to device functions @@ -371,6 +372,20 @@ struct dev_links_info { enum dl_dev_state status; }; +/** + * struct dev_msi_info - Device data related to MSI + * @domain: The MSI interrupt domain associated to the device + * @data: Pointer to MSI device data + */ +struct dev_msi_info { +#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN + struct irq_domain *domain; +#endif +#ifdef CONFIG_GENERIC_MSI_IRQ + struct msi_device_data *data; +#endif +}; + /** * struct device - The basic device structure * @parent: The device's "parent" device, the device to which it is attached. @@ -407,9 +422,7 @@ struct dev_links_info { * @em_pd: device's energy model performance domain * @pins: For device pin management. * See Documentation/driver-api/pin-control.rst for details. - * @msi_lock: Lock to protect MSI mask cache and mask register - * @msi_list: Hosts MSI descriptors - * @msi_domain: The generic MSI domain this device is using. + * @msi: MSI related data * @numa_node: NUMA node this device is close to. * @dma_ops: DMA mapping operations for this device. * @dma_mask: Dma mask (if dma'ble device). @@ -501,16 +514,10 @@ struct device { struct em_perf_domain *em_pd; #endif -#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN - struct irq_domain *msi_domain; -#endif #ifdef CONFIG_PINCTRL struct dev_pin_info *pins; #endif -#ifdef CONFIG_GENERIC_MSI_IRQ - raw_spinlock_t msi_lock; - struct list_head msi_list; -#endif + struct dev_msi_info msi; #ifdef CONFIG_DMA_OPS const struct dma_map_ops *dma_ops; #endif @@ -668,7 +675,7 @@ static inline void set_dev_node(struct device *dev, int node) static inline struct irq_domain *dev_get_msi_domain(const struct device *dev) { #ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN - return dev->msi_domain; + return dev->msi.domain; #else return NULL; #endif @@ -677,7 +684,7 @@ static inline struct irq_domain *dev_get_msi_domain(const struct device *dev) static inline void dev_set_msi_domain(struct device *dev, struct irq_domain *d) { #ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN - dev->msi_domain = d; + dev->msi.domain = d; #endif } diff --git a/include/linux/fsl/mc.h b/include/linux/fsl/mc.h index e026f6c48b49..7b6c42bfb660 100644 --- a/include/linux/fsl/mc.h +++ b/include/linux/fsl/mc.h @@ -91,13 +91,13 @@ struct fsl_mc_resource { /** * struct fsl_mc_device_irq - MC object device message-based interrupt - * @msi_desc: pointer to MSI descriptor allocated by fsl_mc_msi_alloc_descs() + * @virq: Linux virtual interrupt number * @mc_dev: MC object device that owns this interrupt * @dev_irq_index: device-relative IRQ index * @resource: MC generic resource associated with the interrupt */ struct fsl_mc_device_irq { - struct msi_desc *msi_desc; + unsigned int virq; struct fsl_mc_device *mc_dev; u8 dev_irq_index; struct fsl_mc_resource resource; diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index 48a6365d63b5..7e163627149c 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h @@ -329,7 +329,46 @@ extern int irq_force_affinity(unsigned int irq, const struct cpumask *cpumask); extern int irq_can_set_affinity(unsigned int irq); extern int irq_select_affinity(unsigned int irq); -extern int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m); +extern int __irq_apply_affinity_hint(unsigned int irq, const struct cpumask *m, + bool setaffinity); + +/** + * irq_update_affinity_hint - Update the affinity hint + * @irq: Interrupt to update + * @m: cpumask pointer (NULL to clear the hint) + * + * Updates the affinity hint, but does not change the affinity of the interrupt. + */ +static inline int +irq_update_affinity_hint(unsigned int irq, const struct cpumask *m) +{ + return __irq_apply_affinity_hint(irq, m, false); +} + +/** + * irq_set_affinity_and_hint - Update the affinity hint and apply the provided + * cpumask to the interrupt + * @irq: Interrupt to update + * @m: cpumask pointer (NULL to clear the hint) + * + * Updates the affinity hint and if @m is not NULL it applies it as the + * affinity of that interrupt. + */ +static inline int +irq_set_affinity_and_hint(unsigned int irq, const struct cpumask *m) +{ + return __irq_apply_affinity_hint(irq, m, true); +} + +/* + * Deprecated. Use irq_update_affinity_hint() or irq_set_affinity_and_hint() + * instead. + */ +static inline int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m) +{ + return irq_set_affinity_and_hint(irq, m); +} + extern int irq_update_affinity_desc(unsigned int irq, struct irq_affinity_desc *affinity); @@ -361,6 +400,18 @@ static inline int irq_can_set_affinity(unsigned int irq) static inline int irq_select_affinity(unsigned int irq) { return 0; } +static inline int irq_update_affinity_hint(unsigned int irq, + const struct cpumask *m) +{ + return -EINVAL; +} + +static inline int irq_set_affinity_and_hint(unsigned int irq, + const struct cpumask *m) +{ + return -EINVAL; +} + static inline int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m) { diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 6bb55caa13e9..221f9a517a47 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -615,7 +615,7 @@ struct rdists { void __iomem *rd_base; struct page *pend_page; phys_addr_t phys_base; - bool lpi_enabled; + u64 flags; cpumask_t *vpe_table_mask; void *vpe_l1_base; } __percpu *rdist; @@ -624,6 +624,7 @@ struct rdists { u64 flags; u32 gicd_typer; u32 gicd_typer2; + int cpuhp_memreserve_state; bool has_vlpis; bool has_rvpeid; bool has_direct_lpi; @@ -632,6 +633,7 @@ struct rdists { struct irq_domain; struct fwnode_handle; +int __init its_lpi_memreserve_init(void); int its_cpu_init(void); int its_init(struct fwnode_handle *handle, struct rdists *rdists, struct irq_domain *domain); diff --git a/include/linux/irqdomain.h b/include/linux/irqdomain.h index 553da4899f55..d476405802e9 100644 --- a/include/linux/irqdomain.h +++ b/include/linux/irqdomain.h @@ -131,7 +131,7 @@ struct irq_domain_ops { #endif }; -extern struct irq_domain_ops irq_generic_chip_ops; +extern const struct irq_domain_ops irq_generic_chip_ops; struct irq_domain_chip_generic; diff --git a/include/linux/mfd/tps68470.h b/include/linux/mfd/tps68470.h index ffe81127d91c..7807fa329db0 100644 --- a/include/linux/mfd/tps68470.h +++ b/include/linux/mfd/tps68470.h @@ -75,6 +75,17 @@ #define TPS68470_CLKCFG1_MODE_A_MASK GENMASK(1, 0) #define TPS68470_CLKCFG1_MODE_B_MASK GENMASK(3, 2) +#define TPS68470_CLKCFG2_DRV_STR_2MA 0x05 +#define TPS68470_PLL_OUTPUT_ENABLE 0x02 +#define TPS68470_CLK_SRC_XTAL BIT(0) +#define TPS68470_PLLSWR_DEFAULT GENMASK(1, 0) +#define TPS68470_OSC_EXT_CAP_DEFAULT 0x05 + +#define TPS68470_OUTPUT_A_SHIFT 0x00 +#define TPS68470_OUTPUT_B_SHIFT 0x02 +#define TPS68470_CLK_SRC_SHIFT GENMASK(2, 0) +#define TPS68470_OSC_EXT_CAP_SHIFT BIT(2) + #define TPS68470_GPIO_CTL_REG_A(x) (TPS68470_REG_GPCTL0A + (x) * 2) #define TPS68470_GPIO_CTL_REG_B(x) (TPS68470_REG_GPCTL0B + (x) * 2) #define TPS68470_GPIO_MODE_MASK GENMASK(1, 0) diff --git a/include/linux/msi.h b/include/linux/msi.h index e616f94c7c58..fc918a658d48 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -2,7 +2,23 @@ #ifndef LINUX_MSI_H #define LINUX_MSI_H -#include +/* + * This header file contains MSI data structures and functions which are + * only relevant for: + * - Interrupt core code + * - PCI/MSI core code + * - MSI interrupt domain implementations + * - IOMMU, low level VFIO, NTB and other justified exceptions + * dealing with low level MSI details. + * + * Regular device drivers have no business with any of these functions and + * especially storing MSI descriptor pointers in random code is considered + * abuse. The only function which is relevant for drivers is msi_get_virq(). + */ + +#include +#include +#include #include #include @@ -56,6 +72,8 @@ struct irq_data; struct msi_desc; struct pci_dev; struct platform_msi_priv_data; +struct device_attribute; + void __get_cached_msi_msg(struct msi_desc *entry, struct msi_msg *msg); #ifdef CONFIG_GENERIC_MSI_IRQ void get_cached_msi_msg(unsigned int irq, struct msi_msg *msg); @@ -69,62 +87,59 @@ typedef void (*irq_write_msi_msg_t)(struct msi_desc *desc, struct msi_msg *msg); /** - * platform_msi_desc - Platform device specific msi descriptor data - * @msi_priv_data: Pointer to platform private data - * @msi_index: The index of the MSI descriptor for multi MSI - */ -struct platform_msi_desc { - struct platform_msi_priv_data *msi_priv_data; - u16 msi_index; -}; - -/** - * fsl_mc_msi_desc - FSL-MC device specific msi descriptor data - * @msi_index: The index of the MSI descriptor - */ -struct fsl_mc_msi_desc { - u16 msi_index; -}; - -/** - * ti_sci_inta_msi_desc - TISCI based INTA specific msi descriptor data - * @dev_index: TISCI device index - */ -struct ti_sci_inta_msi_desc { - u16 dev_index; -}; - -/** - * struct msi_desc - Descriptor structure for MSI based interrupts - * @list: List head for management - * @irq: The base interrupt number - * @nvec_used: The number of vectors used - * @dev: Pointer to the device which uses this descriptor - * @msg: The last set MSI message cached for reuse - * @affinity: Optional pointer to a cpu affinity mask for this descriptor - * - * @write_msi_msg: Callback that may be called when the MSI message - * address or data changes - * @write_msi_msg_data: Data parameter for the callback. + * pci_msi_desc - PCI/MSI specific MSI descriptor data * * @msi_mask: [PCI MSI] MSI cached mask bits * @msix_ctrl: [PCI MSI-X] MSI-X cached per vector control bits * @is_msix: [PCI MSI/X] True if MSI-X * @multiple: [PCI MSI/X] log2 num of messages allocated * @multi_cap: [PCI MSI/X] log2 num of messages supported - * @maskbit: [PCI MSI/X] Mask-Pending bit supported? + * @can_mask: [PCI MSI/X] Masking supported? * @is_64: [PCI MSI/X] Address size: 0=32bit 1=64bit - * @entry_nr: [PCI MSI/X] Entry which is described by this descriptor * @default_irq:[PCI MSI/X] The default pre-assigned non-MSI irq * @mask_pos: [PCI MSI] Mask register position * @mask_base: [PCI MSI-X] Mask register base address - * @platform: [platform] Platform device specific msi descriptor data - * @fsl_mc: [fsl-mc] FSL MC device specific msi descriptor data - * @inta: [INTA] TISCI based INTA specific msi descriptor data + */ +struct pci_msi_desc { + union { + u32 msi_mask; + u32 msix_ctrl; + }; + struct { + u8 is_msix : 1; + u8 multiple : 3; + u8 multi_cap : 3; + u8 can_mask : 1; + u8 is_64 : 1; + u8 is_virtual : 1; + unsigned default_irq; + } msi_attrib; + union { + u8 mask_pos; + void __iomem *mask_base; + }; +}; + +#define MSI_MAX_INDEX ((unsigned int)USHRT_MAX) + +/** + * struct msi_desc - Descriptor structure for MSI based interrupts + * @irq: The base interrupt number + * @nvec_used: The number of vectors used + * @dev: Pointer to the device which uses this descriptor + * @msg: The last set MSI message cached for reuse + * @affinity: Optional pointer to a cpu affinity mask for this descriptor + * @sysfs_attr: Pointer to sysfs device attribute + * + * @write_msi_msg: Callback that may be called when the MSI message + * address or data changes + * @write_msi_msg_data: Data parameter for the callback. + * + * @msi_index: Index of the msi descriptor + * @pci: PCI specific msi descriptor data */ struct msi_desc { /* Shared device/bus type independent data */ - struct list_head list; unsigned int irq; unsigned int nvec_used; struct device *dev; @@ -133,61 +148,71 @@ struct msi_desc { #ifdef CONFIG_IRQ_MSI_IOMMU const void *iommu_cookie; #endif +#ifdef CONFIG_SYSFS + struct device_attribute *sysfs_attrs; +#endif void (*write_msi_msg)(struct msi_desc *entry, void *data); void *write_msi_msg_data; - union { - /* PCI MSI/X specific data */ - struct { - union { - u32 msi_mask; - u32 msix_ctrl; - }; - struct { - u8 is_msix : 1; - u8 multiple : 3; - u8 multi_cap : 3; - u8 can_mask : 1; - u8 is_64 : 1; - u8 is_virtual : 1; - u16 entry_nr; - unsigned default_irq; - } msi_attrib; - union { - u8 mask_pos; - void __iomem *mask_base; - }; - }; - - /* - * Non PCI variants add their data structure here. New - * entries need to use a named structure. We want - * proper name spaces for this. The PCI part is - * anonymous for now as it would require an immediate - * tree wide cleanup. - */ - struct platform_msi_desc platform; - struct fsl_mc_msi_desc fsl_mc; - struct ti_sci_inta_msi_desc inta; - }; + u16 msi_index; + struct pci_msi_desc pci; }; -/* Helpers to hide struct msi_desc implementation details */ +/* + * Filter values for the MSI descriptor iterators and accessor functions. + */ +enum msi_desc_filter { + /* All descriptors */ + MSI_DESC_ALL, + /* Descriptors which have no interrupt associated */ + MSI_DESC_NOTASSOCIATED, + /* Descriptors which have an interrupt associated */ + MSI_DESC_ASSOCIATED, +}; + +/** + * msi_device_data - MSI per device data + * @properties: MSI properties which are interesting to drivers + * @platform_data: Platform-MSI specific data + * @mutex: Mutex protecting the MSI descriptor store + * @__store: Xarray for storing MSI descriptor pointers + * @__iter_idx: Index to search the next entry for iterators + */ +struct msi_device_data { + unsigned long properties; + struct platform_msi_priv_data *platform_data; + struct mutex mutex; + struct xarray __store; + unsigned long __iter_idx; +}; + +int msi_setup_device_data(struct device *dev); + +unsigned int msi_get_virq(struct device *dev, unsigned int index); +void msi_lock_descs(struct device *dev); +void msi_unlock_descs(struct device *dev); + +struct msi_desc *msi_first_desc(struct device *dev, enum msi_desc_filter filter); +struct msi_desc *msi_next_desc(struct device *dev, enum msi_desc_filter filter); + +/** + * msi_for_each_desc - Iterate the MSI descriptors + * + * @desc: struct msi_desc pointer used as iterator + * @dev: struct device pointer - device to iterate + * @filter: Filter for descriptor selection + * + * Notes: + * - The loop must be protected with a msi_lock_descs()/msi_unlock_descs() + * pair. + * - It is safe to remove a retrieved MSI descriptor in the loop. + */ +#define msi_for_each_desc(desc, dev, filter) \ + for ((desc) = msi_first_desc((dev), (filter)); (desc); \ + (desc) = msi_next_desc((dev), (filter))) + #define msi_desc_to_dev(desc) ((desc)->dev) -#define dev_to_msi_list(dev) (&(dev)->msi_list) -#define first_msi_entry(dev) \ - list_first_entry(dev_to_msi_list((dev)), struct msi_desc, list) -#define for_each_msi_entry(desc, dev) \ - list_for_each_entry((desc), dev_to_msi_list((dev)), list) -#define for_each_msi_entry_safe(desc, tmp, dev) \ - list_for_each_entry_safe((desc), (tmp), dev_to_msi_list((dev)), list) -#define for_each_msi_vector(desc, __irq, dev) \ - for_each_msi_entry((desc), (dev)) \ - if ((desc)->irq) \ - for (__irq = (desc)->irq; \ - __irq < ((desc)->irq + (desc)->nvec_used); \ - __irq++) #ifdef CONFIG_IRQ_MSI_IOMMU static inline const void *msi_desc_get_iommu_cookie(struct msi_desc *desc) @@ -213,36 +238,33 @@ static inline void msi_desc_set_iommu_cookie(struct msi_desc *desc, #endif #ifdef CONFIG_PCI_MSI -#define first_pci_msi_entry(pdev) first_msi_entry(&(pdev)->dev) -#define for_each_pci_msi_entry(desc, pdev) \ - for_each_msi_entry((desc), &(pdev)->dev) - struct pci_dev *msi_desc_to_pci_dev(struct msi_desc *desc); -void *msi_desc_to_pci_sysdata(struct msi_desc *desc); void pci_write_msi_msg(unsigned int irq, struct msi_msg *msg); #else /* CONFIG_PCI_MSI */ -static inline void *msi_desc_to_pci_sysdata(struct msi_desc *desc) -{ - return NULL; -} static inline void pci_write_msi_msg(unsigned int irq, struct msi_msg *msg) { } #endif /* CONFIG_PCI_MSI */ -struct msi_desc *alloc_msi_entry(struct device *dev, int nvec, - const struct irq_affinity_desc *affinity); -void free_msi_entry(struct msi_desc *entry); +int msi_add_msi_desc(struct device *dev, struct msi_desc *init_desc); +void msi_free_msi_descs_range(struct device *dev, enum msi_desc_filter filter, + unsigned int first_index, unsigned int last_index); + +/** + * msi_free_msi_descs - Free MSI descriptors of a device + * @dev: Device to free the descriptors + */ +static inline void msi_free_msi_descs(struct device *dev) +{ + msi_free_msi_descs_range(dev, MSI_DESC_ALL, 0, MSI_MAX_INDEX); +} + void __pci_read_msi_msg(struct msi_desc *entry, struct msi_msg *msg); void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg); void pci_msi_mask_irq(struct irq_data *data); void pci_msi_unmask_irq(struct irq_data *data); -const struct attribute_group **msi_populate_sysfs(struct device *dev); -void msi_destroy_sysfs(struct device *dev, - const struct attribute_group **msi_irq_groups); - /* * The arch hooks to setup up msi irqs. Default functions are implemented * as weak symbols so that they /can/ be overriden by architecture specific @@ -256,25 +278,20 @@ int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc); void arch_teardown_msi_irq(unsigned int irq); int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type); void arch_teardown_msi_irqs(struct pci_dev *dev); -#else -static inline int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) -{ - WARN_ON_ONCE(1); - return -ENODEV; -} - -static inline void arch_teardown_msi_irqs(struct pci_dev *dev) -{ - WARN_ON_ONCE(1); -} -#endif +#ifdef CONFIG_SYSFS +int msi_device_populate_sysfs(struct device *dev); +void msi_device_destroy_sysfs(struct device *dev); +#else /* CONFIG_SYSFS */ +static inline int msi_device_populate_sysfs(struct device *dev) { return 0; } +static inline void msi_device_destroy_sysfs(struct device *dev) { } +#endif /* !CONFIG_SYSFS */ +#endif /* CONFIG_PCI_MSI_ARCH_FALLBACKS */ /* - * The restore hooks are still available as they are useful even - * for fully irq domain based setups. Courtesy to XEN/X86. + * The restore hook is still available even for fully irq domain based + * setups. Courtesy to XEN/X86. */ -void arch_restore_msi_irqs(struct pci_dev *dev); -void default_restore_msi_irqs(struct pci_dev *dev); +bool arch_restore_msi_irqs(struct pci_dev *dev); #ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN @@ -294,20 +311,17 @@ struct msi_domain_info; * @msi_free: Domain specific function to free a MSI interrupts * @msi_check: Callback for verification of the domain/info/dev data * @msi_prepare: Prepare the allocation of the interrupts in the domain - * @msi_finish: Optional callback to finalize the allocation * @set_desc: Set the msi descriptor for an interrupt - * @handle_error: Optional error handler if the allocation fails * @domain_alloc_irqs: Optional function to override the default allocation * function. * @domain_free_irqs: Optional function to override the default free * function. * - * @get_hwirq, @msi_init and @msi_free are callbacks used by - * msi_create_irq_domain() and related interfaces + * @get_hwirq, @msi_init and @msi_free are callbacks used by the underlying + * irqdomain. * - * @msi_check, @msi_prepare, @msi_finish, @set_desc and @handle_error - * are callbacks used by msi_domain_alloc_irqs() and related - * interfaces which are based on msi_desc. + * @msi_check, @msi_prepare and @set_desc are callbacks used by + * msi_domain_alloc/free_irqs(). * * @domain_alloc_irqs, @domain_free_irqs can be used to override the * default allocation/free functions (__msi_domain_alloc/free_irqs). This @@ -341,11 +355,8 @@ struct msi_domain_ops { int (*msi_prepare)(struct irq_domain *domain, struct device *dev, int nvec, msi_alloc_info_t *arg); - void (*msi_finish)(msi_alloc_info_t *arg, int retval); void (*set_desc)(msi_alloc_info_t *arg, struct msi_desc *desc); - int (*handle_error)(struct irq_domain *domain, - struct msi_desc *desc, int error); int (*domain_alloc_irqs)(struct irq_domain *domain, struct device *dev, int nvec); void (*domain_free_irqs)(struct irq_domain *domain, @@ -399,6 +410,14 @@ enum { MSI_FLAG_MUST_REACTIVATE = (1 << 5), /* Is level-triggered capable, using two messages */ MSI_FLAG_LEVEL_CAPABLE = (1 << 6), + /* Populate sysfs on alloc() and destroy it on free() */ + MSI_FLAG_DEV_SYSFS = (1 << 7), + /* MSI-X entries must be contiguous */ + MSI_FLAG_MSIX_CONTIGUOUS = (1 << 8), + /* Allocate simple MSI descriptors */ + MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS = (1 << 9), + /* Free MSI descriptors */ + MSI_FLAG_FREE_MSI_DESCS = (1 << 10), }; int msi_domain_set_affinity(struct irq_data *data, const struct cpumask *mask, @@ -409,9 +428,12 @@ struct irq_domain *msi_create_irq_domain(struct fwnode_handle *fwnode, struct irq_domain *parent); int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, int nvec); +int msi_domain_alloc_irqs_descs_locked(struct irq_domain *domain, struct device *dev, + int nvec); int msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, int nvec); void __msi_domain_free_irqs(struct irq_domain *domain, struct device *dev); +void msi_domain_free_irqs_descs_locked(struct irq_domain *domain, struct device *dev); void msi_domain_free_irqs(struct irq_domain *domain, struct device *dev); struct msi_domain_info *msi_get_domain_info(struct irq_domain *domain); @@ -440,20 +462,17 @@ __platform_msi_create_device_domain(struct device *dev, #define platform_msi_create_device_tree_domain(dev, nvec, write, ops, data) \ __platform_msi_create_device_domain(dev, nvec, true, write, ops, data) -int platform_msi_domain_alloc(struct irq_domain *domain, unsigned int virq, - unsigned int nr_irqs); -void platform_msi_domain_free(struct irq_domain *domain, unsigned int virq, - unsigned int nvec); +int platform_msi_device_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs); +void platform_msi_device_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nvec); void *platform_msi_get_host_data(struct irq_domain *domain); #endif /* CONFIG_GENERIC_MSI_IRQ_DOMAIN */ #ifdef CONFIG_PCI_MSI_IRQ_DOMAIN -void pci_msi_domain_write_msg(struct irq_data *irq_data, struct msi_msg *msg); struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode, struct msi_domain_info *info, struct irq_domain *parent); -int pci_msi_domain_check_cap(struct irq_domain *domain, - struct msi_domain_info *info, struct device *dev); u32 pci_msi_domain_get_msi_rid(struct irq_domain *domain, struct pci_dev *pdev); struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev); bool pci_dev_has_special_msi_domain(struct pci_dev *pdev); diff --git a/include/linux/pci.h b/include/linux/pci.h index 18a75c8e615c..0a7b6b2f163b 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -425,7 +425,8 @@ struct pci_dev { unsigned int ats_enabled:1; /* Address Translation Svc */ unsigned int pasid_enabled:1; /* Process Address Space ID */ unsigned int pri_enabled:1; /* Page Request Interface */ - unsigned int is_managed:1; + unsigned int is_managed:1; /* Managed via devres */ + unsigned int is_msi_managed:1; /* MSI release via devres installed */ unsigned int needs_freset:1; /* Requires fundamental reset */ unsigned int state_saved:1; unsigned int is_physfn:1; @@ -473,7 +474,8 @@ struct pci_dev { u8 ptm_granularity; #endif #ifdef CONFIG_PCI_MSI - const struct attribute_group **msi_irq_groups; + void __iomem *msix_base; + raw_spinlock_t msi_lock; #endif struct pci_vpd vpd; #ifdef CONFIG_PCIE_DPC diff --git a/include/linux/platform_data/clk-fch.h b/include/linux/platform_data/clk-fch.h index b9f682459f08..11a2a23fd9b2 100644 --- a/include/linux/platform_data/clk-fch.h +++ b/include/linux/platform_data/clk-fch.h @@ -12,7 +12,7 @@ struct fch_clk_data { void __iomem *base; - u32 is_rv; + char *name; }; #endif /* __CLK_FCH_H */ diff --git a/include/linux/reset.h b/include/linux/reset.h index db0e6115a2f6..8a21b5756c3e 100644 --- a/include/linux/reset.h +++ b/include/linux/reset.h @@ -454,6 +454,26 @@ static inline struct reset_control *of_reset_control_get_exclusive( return __of_reset_control_get(node, id, 0, false, false, true); } +/** + * of_reset_control_get_optional_exclusive - Lookup and obtain an optional exclusive + * reference to a reset controller. + * @node: device to be reset by the controller + * @id: reset line name + * + * Optional variant of of_reset_control_get_exclusive(). If the requested reset + * is not specified in the device tree, this function returns NULL instead of + * an error. + * + * Returns a struct reset_control or IS_ERR() condition containing errno. + * + * Use of id names is optional. + */ +static inline struct reset_control *of_reset_control_get_optional_exclusive( + struct device_node *node, const char *id) +{ + return __of_reset_control_get(node, id, 0, false, true, true); +} + /** * of_reset_control_get_shared - Lookup and obtain a shared reference * to a reset controller. diff --git a/include/linux/soc/ti/ti_sci_inta_msi.h b/include/linux/soc/ti/ti_sci_inta_msi.h index e3aa8b14612e..4dba2f2aff6f 100644 --- a/include/linux/soc/ti/ti_sci_inta_msi.h +++ b/include/linux/soc/ti/ti_sci_inta_msi.h @@ -18,6 +18,4 @@ struct irq_domain struct irq_domain *parent); int ti_sci_inta_msi_domain_alloc_irqs(struct device *dev, struct ti_sci_resource *res); -unsigned int ti_sci_inta_msi_get_virq(struct device *dev, u32 index); -void ti_sci_inta_msi_domain_free_irqs(struct device *dev); #endif /* __INCLUDE_LINUX_IRQCHIP_TI_SCI_INTA_H */ diff --git a/include/rdma/ib_mad.h b/include/rdma/ib_mad.h index 465b0d0bdaf8..2e3843b761e8 100644 --- a/include/rdma/ib_mad.h +++ b/include/rdma/ib_mad.h @@ -276,6 +276,7 @@ enum ib_port_capability_mask2_bits { IB_PORT_SWITCH_PORT_STATE_TABLE_SUP = 1 << 3, IB_PORT_LINK_WIDTH_2X_SUP = 1 << 4, IB_PORT_LINK_SPEED_HDR_SUP = 1 << 5, + IB_PORT_LINK_SPEED_NDR_SUP = 1 << 10, }; #define OPA_CLASS_PORT_INFO_PR_SUPPORT BIT(26) diff --git a/include/rdma/ib_smi.h b/include/rdma/ib_smi.h index fdb8633cbaff..fc16b826b2c1 100644 --- a/include/rdma/ib_smi.h +++ b/include/rdma/ib_smi.h @@ -144,5 +144,15 @@ ib_get_smp_direction(struct ib_smp *smp) #define IB_NOTICE_TRAP_DR_NOTICE 0x80 #define IB_NOTICE_TRAP_DR_TRUNC 0x40 - +/** + * ib_init_query_mad - Initialize query MAD. + * @mad: MAD to initialize. + */ +static inline void ib_init_query_mad(struct ib_smp *mad) +{ + mad->base_version = IB_MGMT_BASE_VERSION; + mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED; + mad->class_version = 1; + mad->method = IB_MGMT_METHOD_GET; +} #endif /* IB_SMI_H */ diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 6e9ad656ecb7..69d883f7fb41 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -4749,6 +4749,23 @@ static inline u32 rdma_calc_flow_label(u32 lqpn, u32 rqpn) return (u32)(v & IB_GRH_FLOWLABEL_MASK); } +/** + * rdma_get_udp_sport - Calculate and set UDP source port based on the flow + * label. If flow label is not defined in GRH then + * calculate it based on lqpn/rqpn. + * + * @fl: flow label from GRH + * @lqpn: local qp number + * @rqpn: remote qp number + */ +static inline u16 rdma_get_udp_sport(u32 fl, u32 lqpn, u32 rqpn) +{ + if (!fl) + fl = rdma_calc_flow_label(lqpn, rqpn); + + return rdma_flow_label_to_udp_sport(fl); +} + const struct ib_port_immutable* ib_port_immutable_read(struct ib_device *dev, unsigned int port); #endif /* IB_VERBS_H */ diff --git a/include/uapi/rdma/hns-abi.h b/include/uapi/rdma/hns-abi.h index 42b177655560..f6fde06db4b4 100644 --- a/include/uapi/rdma/hns-abi.h +++ b/include/uapi/rdma/hns-abi.h @@ -77,10 +77,12 @@ enum hns_roce_qp_cap_flags { HNS_ROCE_QP_CAP_RQ_RECORD_DB = 1 << 0, HNS_ROCE_QP_CAP_SQ_RECORD_DB = 1 << 1, HNS_ROCE_QP_CAP_OWNER_DB = 1 << 2, + HNS_ROCE_QP_CAP_DIRECT_WQE = 1 << 5, }; struct hns_roce_ib_create_qp_resp { __aligned_u64 cap_flags; + __aligned_u64 dwqe_mmap_key; }; struct hns_roce_ib_alloc_ucontext_resp { diff --git a/kernel/irq/generic-chip.c b/kernel/irq/generic-chip.c index 6f29bf4c8515..f0862eb6b506 100644 --- a/kernel/irq/generic-chip.c +++ b/kernel/irq/generic-chip.c @@ -451,7 +451,7 @@ static void irq_unmap_generic_chip(struct irq_domain *d, unsigned int virq) } -struct irq_domain_ops irq_generic_chip_ops = { +const struct irq_domain_ops irq_generic_chip_ops = { .map = irq_map_generic_chip, .unmap = irq_unmap_generic_chip, .xlate = irq_domain_xlate_onetwocell, diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index 7405e384e5ed..f23ffd30385b 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -486,7 +486,8 @@ int irq_force_affinity(unsigned int irq, const struct cpumask *cpumask) } EXPORT_SYMBOL_GPL(irq_force_affinity); -int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m) +int __irq_apply_affinity_hint(unsigned int irq, const struct cpumask *m, + bool setaffinity) { unsigned long flags; struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL); @@ -495,12 +496,11 @@ int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m) return -EINVAL; desc->affinity_hint = m; irq_put_desc_unlock(desc, flags); - /* set the initial affinity to prevent every interrupt being on CPU0 */ - if (m) + if (m && setaffinity) __irq_set_affinity(irq, m, false); return 0; } -EXPORT_SYMBOL_GPL(irq_set_affinity_hint); +EXPORT_SYMBOL_GPL(__irq_apply_affinity_hint); static void irq_affinity_notify(struct work_struct *work) { diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index 7f350ae59c5f..2bdfce5edafd 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -14,12 +14,15 @@ #include #include #include +#include #include #include "internals.h" +static inline int msi_sysfs_create_group(struct device *dev); + /** - * alloc_msi_entry - Allocate an initialized msi_desc + * msi_alloc_desc - Allocate an initialized msi_desc * @dev: Pointer to the device for which this is allocated * @nvec: The number of vectors used in this entry * @affinity: Optional pointer to an affinity mask array size of @nvec @@ -29,34 +32,134 @@ * * Return: pointer to allocated &msi_desc on success or %NULL on failure */ -struct msi_desc *alloc_msi_entry(struct device *dev, int nvec, - const struct irq_affinity_desc *affinity) +static struct msi_desc *msi_alloc_desc(struct device *dev, int nvec, + const struct irq_affinity_desc *affinity) { - struct msi_desc *desc; + struct msi_desc *desc = kzalloc(sizeof(*desc), GFP_KERNEL); - desc = kzalloc(sizeof(*desc), GFP_KERNEL); if (!desc) return NULL; - INIT_LIST_HEAD(&desc->list); desc->dev = dev; desc->nvec_used = nvec; if (affinity) { - desc->affinity = kmemdup(affinity, - nvec * sizeof(*desc->affinity), GFP_KERNEL); + desc->affinity = kmemdup(affinity, nvec * sizeof(*desc->affinity), GFP_KERNEL); if (!desc->affinity) { kfree(desc); return NULL; } } - return desc; } -void free_msi_entry(struct msi_desc *entry) +static void msi_free_desc(struct msi_desc *desc) { - kfree(entry->affinity); - kfree(entry); + kfree(desc->affinity); + kfree(desc); +} + +static int msi_insert_desc(struct msi_device_data *md, struct msi_desc *desc, unsigned int index) +{ + int ret; + + desc->msi_index = index; + ret = xa_insert(&md->__store, index, desc, GFP_KERNEL); + if (ret) + msi_free_desc(desc); + return ret; +} + +/** + * msi_add_msi_desc - Allocate and initialize a MSI descriptor + * @dev: Pointer to the device for which the descriptor is allocated + * @init_desc: Pointer to an MSI descriptor to initialize the new descriptor + * + * Return: 0 on success or an appropriate failure code. + */ +int msi_add_msi_desc(struct device *dev, struct msi_desc *init_desc) +{ + struct msi_desc *desc; + + lockdep_assert_held(&dev->msi.data->mutex); + + desc = msi_alloc_desc(dev, init_desc->nvec_used, init_desc->affinity); + if (!desc) + return -ENOMEM; + + /* Copy type specific data to the new descriptor. */ + desc->pci = init_desc->pci; + return msi_insert_desc(dev->msi.data, desc, init_desc->msi_index); +} + +/** + * msi_add_simple_msi_descs - Allocate and initialize MSI descriptors + * @dev: Pointer to the device for which the descriptors are allocated + * @index: Index for the first MSI descriptor + * @ndesc: Number of descriptors to allocate + * + * Return: 0 on success or an appropriate failure code. + */ +static int msi_add_simple_msi_descs(struct device *dev, unsigned int index, unsigned int ndesc) +{ + unsigned int idx, last = index + ndesc - 1; + struct msi_desc *desc; + int ret; + + lockdep_assert_held(&dev->msi.data->mutex); + + for (idx = index; idx <= last; idx++) { + desc = msi_alloc_desc(dev, 1, NULL); + if (!desc) + goto fail_mem; + ret = msi_insert_desc(dev->msi.data, desc, idx); + if (ret) + goto fail; + } + return 0; + +fail_mem: + ret = -ENOMEM; +fail: + msi_free_msi_descs_range(dev, MSI_DESC_NOTASSOCIATED, index, last); + return ret; +} + +static bool msi_desc_match(struct msi_desc *desc, enum msi_desc_filter filter) +{ + switch (filter) { + case MSI_DESC_ALL: + return true; + case MSI_DESC_NOTASSOCIATED: + return !desc->irq; + case MSI_DESC_ASSOCIATED: + return !!desc->irq; + } + WARN_ON_ONCE(1); + return false; +} + +/** + * msi_free_msi_descs_range - Free MSI descriptors of a device + * @dev: Device to free the descriptors + * @filter: Descriptor state filter + * @first_index: Index to start freeing from + * @last_index: Last index to be freed + */ +void msi_free_msi_descs_range(struct device *dev, enum msi_desc_filter filter, + unsigned int first_index, unsigned int last_index) +{ + struct xarray *xa = &dev->msi.data->__store; + struct msi_desc *desc; + unsigned long idx; + + lockdep_assert_held(&dev->msi.data->mutex); + + xa_for_each_range(xa, idx, desc, first_index, last_index) { + if (msi_desc_match(desc, filter)) { + xa_erase(xa, idx); + msi_free_desc(desc); + } + } } void __get_cached_msi_msg(struct msi_desc *entry, struct msi_msg *msg) @@ -72,138 +175,289 @@ void get_cached_msi_msg(unsigned int irq, struct msi_msg *msg) } EXPORT_SYMBOL_GPL(get_cached_msi_msg); +static void msi_device_data_release(struct device *dev, void *res) +{ + struct msi_device_data *md = res; + + WARN_ON_ONCE(!xa_empty(&md->__store)); + xa_destroy(&md->__store); + dev->msi.data = NULL; +} + +/** + * msi_setup_device_data - Setup MSI device data + * @dev: Device for which MSI device data should be set up + * + * Return: 0 on success, appropriate error code otherwise + * + * This can be called more than once for @dev. If the MSI device data is + * already allocated the call succeeds. The allocated memory is + * automatically released when the device is destroyed. + */ +int msi_setup_device_data(struct device *dev) +{ + struct msi_device_data *md; + int ret; + + if (dev->msi.data) + return 0; + + md = devres_alloc(msi_device_data_release, sizeof(*md), GFP_KERNEL); + if (!md) + return -ENOMEM; + + ret = msi_sysfs_create_group(dev); + if (ret) { + devres_free(md); + return ret; + } + + xa_init(&md->__store); + mutex_init(&md->mutex); + dev->msi.data = md; + devres_add(dev, md); + return 0; +} + +/** + * msi_lock_descs - Lock the MSI descriptor storage of a device + * @dev: Device to operate on + */ +void msi_lock_descs(struct device *dev) +{ + mutex_lock(&dev->msi.data->mutex); +} +EXPORT_SYMBOL_GPL(msi_lock_descs); + +/** + * msi_unlock_descs - Unlock the MSI descriptor storage of a device + * @dev: Device to operate on + */ +void msi_unlock_descs(struct device *dev) +{ + /* Invalidate the index wich was cached by the iterator */ + dev->msi.data->__iter_idx = MSI_MAX_INDEX; + mutex_unlock(&dev->msi.data->mutex); +} +EXPORT_SYMBOL_GPL(msi_unlock_descs); + +static struct msi_desc *msi_find_desc(struct msi_device_data *md, enum msi_desc_filter filter) +{ + struct msi_desc *desc; + + xa_for_each_start(&md->__store, md->__iter_idx, desc, md->__iter_idx) { + if (msi_desc_match(desc, filter)) + return desc; + } + md->__iter_idx = MSI_MAX_INDEX; + return NULL; +} + +/** + * msi_first_desc - Get the first MSI descriptor of a device + * @dev: Device to operate on + * @filter: Descriptor state filter + * + * Must be called with the MSI descriptor mutex held, i.e. msi_lock_descs() + * must be invoked before the call. + * + * Return: Pointer to the first MSI descriptor matching the search + * criteria, NULL if none found. + */ +struct msi_desc *msi_first_desc(struct device *dev, enum msi_desc_filter filter) +{ + struct msi_device_data *md = dev->msi.data; + + if (WARN_ON_ONCE(!md)) + return NULL; + + lockdep_assert_held(&md->mutex); + + md->__iter_idx = 0; + return msi_find_desc(md, filter); +} +EXPORT_SYMBOL_GPL(msi_first_desc); + +/** + * msi_next_desc - Get the next MSI descriptor of a device + * @dev: Device to operate on + * + * The first invocation of msi_next_desc() has to be preceeded by a + * successful invocation of __msi_first_desc(). Consecutive invocations are + * only valid if the previous one was successful. All these operations have + * to be done within the same MSI mutex held region. + * + * Return: Pointer to the next MSI descriptor matching the search + * criteria, NULL if none found. + */ +struct msi_desc *msi_next_desc(struct device *dev, enum msi_desc_filter filter) +{ + struct msi_device_data *md = dev->msi.data; + + if (WARN_ON_ONCE(!md)) + return NULL; + + lockdep_assert_held(&md->mutex); + + if (md->__iter_idx >= (unsigned long)MSI_MAX_INDEX) + return NULL; + + md->__iter_idx++; + return msi_find_desc(md, filter); +} +EXPORT_SYMBOL_GPL(msi_next_desc); + +/** + * msi_get_virq - Return Linux interrupt number of a MSI interrupt + * @dev: Device to operate on + * @index: MSI interrupt index to look for (0-based) + * + * Return: The Linux interrupt number on success (> 0), 0 if not found + */ +unsigned int msi_get_virq(struct device *dev, unsigned int index) +{ + struct msi_desc *desc; + unsigned int ret = 0; + bool pcimsi; + + if (!dev->msi.data) + return 0; + + pcimsi = dev_is_pci(dev) ? to_pci_dev(dev)->msi_enabled : false; + + msi_lock_descs(dev); + desc = xa_load(&dev->msi.data->__store, pcimsi ? 0 : index); + if (desc && desc->irq) { + /* + * PCI-MSI has only one descriptor for multiple interrupts. + * PCI-MSIX and platform MSI use a descriptor per + * interrupt. + */ + if (pcimsi) { + if (index < desc->nvec_used) + ret = desc->irq + index; + } else { + ret = desc->irq; + } + } + msi_unlock_descs(dev); + return ret; +} +EXPORT_SYMBOL_GPL(msi_get_virq); + +#ifdef CONFIG_SYSFS +static struct attribute *msi_dev_attrs[] = { + NULL +}; + +static const struct attribute_group msi_irqs_group = { + .name = "msi_irqs", + .attrs = msi_dev_attrs, +}; + +static inline int msi_sysfs_create_group(struct device *dev) +{ + return devm_device_add_group(dev, &msi_irqs_group); +} + static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct msi_desc *entry; - bool is_msix = false; - unsigned long irq; - int retval; - - retval = kstrtoul(attr->attr.name, 10, &irq); - if (retval) - return retval; - - entry = irq_get_msi_desc(irq); - if (!entry) - return -ENODEV; - - if (dev_is_pci(dev)) - is_msix = entry->msi_attrib.is_msix; + /* MSI vs. MSIX is per device not per interrupt */ + bool is_msix = dev_is_pci(dev) ? to_pci_dev(dev)->msix_enabled : false; return sysfs_emit(buf, "%s\n", is_msix ? "msix" : "msi"); } -/** - * msi_populate_sysfs - Populate msi_irqs sysfs entries for devices - * @dev: The device(PCI, platform etc) who will get sysfs entries - * - * Return attribute_group ** so that specific bus MSI can save it to - * somewhere during initilizing msi irqs. If devices has no MSI irq, - * return NULL; if it fails to populate sysfs, return ERR_PTR - */ -const struct attribute_group **msi_populate_sysfs(struct device *dev) +static void msi_sysfs_remove_desc(struct device *dev, struct msi_desc *desc) { - const struct attribute_group **msi_irq_groups; - struct attribute **msi_attrs, *msi_attr; - struct device_attribute *msi_dev_attr; - struct attribute_group *msi_irq_group; - struct msi_desc *entry; - int ret = -ENOMEM; - int num_msi = 0; - int count = 0; + struct device_attribute *attrs = desc->sysfs_attrs; int i; - /* Determine how many msi entries we have */ - for_each_msi_entry(entry, dev) - num_msi += entry->nvec_used; - if (!num_msi) - return NULL; + if (!attrs) + return; - /* Dynamically create the MSI attributes for the device */ - msi_attrs = kcalloc(num_msi + 1, sizeof(void *), GFP_KERNEL); - if (!msi_attrs) - return ERR_PTR(-ENOMEM); + desc->sysfs_attrs = NULL; + for (i = 0; i < desc->nvec_used; i++) { + if (attrs[i].show) + sysfs_remove_file_from_group(&dev->kobj, &attrs[i].attr, msi_irqs_group.name); + kfree(attrs[i].attr.name); + } + kfree(attrs); +} - for_each_msi_entry(entry, dev) { - for (i = 0; i < entry->nvec_used; i++) { - msi_dev_attr = kzalloc(sizeof(*msi_dev_attr), GFP_KERNEL); - if (!msi_dev_attr) - goto error_attrs; - msi_attrs[count] = &msi_dev_attr->attr; +static int msi_sysfs_populate_desc(struct device *dev, struct msi_desc *desc) +{ + struct device_attribute *attrs; + int ret, i; - sysfs_attr_init(&msi_dev_attr->attr); - msi_dev_attr->attr.name = kasprintf(GFP_KERNEL, "%d", - entry->irq + i); - if (!msi_dev_attr->attr.name) - goto error_attrs; - msi_dev_attr->attr.mode = 0444; - msi_dev_attr->show = msi_mode_show; - ++count; + attrs = kcalloc(desc->nvec_used, sizeof(*attrs), GFP_KERNEL); + if (!attrs) + return -ENOMEM; + + desc->sysfs_attrs = attrs; + for (i = 0; i < desc->nvec_used; i++) { + sysfs_attr_init(&attrs[i].attr); + attrs[i].attr.name = kasprintf(GFP_KERNEL, "%d", desc->irq + i); + if (!attrs[i].attr.name) { + ret = -ENOMEM; + goto fail; + } + + attrs[i].attr.mode = 0444; + attrs[i].show = msi_mode_show; + + ret = sysfs_add_file_to_group(&dev->kobj, &attrs[i].attr, msi_irqs_group.name); + if (ret) { + attrs[i].show = NULL; + goto fail; } } + return 0; - msi_irq_group = kzalloc(sizeof(*msi_irq_group), GFP_KERNEL); - if (!msi_irq_group) - goto error_attrs; - msi_irq_group->name = "msi_irqs"; - msi_irq_group->attrs = msi_attrs; +fail: + msi_sysfs_remove_desc(dev, desc); + return ret; +} - msi_irq_groups = kcalloc(2, sizeof(void *), GFP_KERNEL); - if (!msi_irq_groups) - goto error_irq_group; - msi_irq_groups[0] = msi_irq_group; +#ifdef CONFIG_PCI_MSI_ARCH_FALLBACKS +/** + * msi_device_populate_sysfs - Populate msi_irqs sysfs entries for a device + * @dev: The device (PCI, platform etc) which will get sysfs entries + */ +int msi_device_populate_sysfs(struct device *dev) +{ + struct msi_desc *desc; + int ret; - ret = sysfs_create_groups(&dev->kobj, msi_irq_groups); - if (ret) - goto error_irq_groups; - - return msi_irq_groups; - -error_irq_groups: - kfree(msi_irq_groups); -error_irq_group: - kfree(msi_irq_group); -error_attrs: - count = 0; - msi_attr = msi_attrs[count]; - while (msi_attr) { - msi_dev_attr = container_of(msi_attr, struct device_attribute, attr); - kfree(msi_attr->name); - kfree(msi_dev_attr); - ++count; - msi_attr = msi_attrs[count]; + msi_for_each_desc(desc, dev, MSI_DESC_ASSOCIATED) { + if (desc->sysfs_attrs) + continue; + ret = msi_sysfs_populate_desc(dev, desc); + if (ret) + return ret; } - kfree(msi_attrs); - return ERR_PTR(ret); + return 0; } /** - * msi_destroy_sysfs - Destroy msi_irqs sysfs entries for devices - * @dev: The device(PCI, platform etc) who will remove sysfs entries - * @msi_irq_groups: attribute_group for device msi_irqs entries + * msi_device_destroy_sysfs - Destroy msi_irqs sysfs entries for a device + * @dev: The device (PCI, platform etc) for which to remove + * sysfs entries */ -void msi_destroy_sysfs(struct device *dev, const struct attribute_group **msi_irq_groups) +void msi_device_destroy_sysfs(struct device *dev) { - struct device_attribute *dev_attr; - struct attribute **msi_attrs; - int count = 0; + struct msi_desc *desc; - if (msi_irq_groups) { - sysfs_remove_groups(&dev->kobj, msi_irq_groups); - msi_attrs = msi_irq_groups[0]->attrs; - while (msi_attrs[count]) { - dev_attr = container_of(msi_attrs[count], - struct device_attribute, attr); - kfree(dev_attr->attr.name); - kfree(dev_attr); - ++count; - } - kfree(msi_attrs); - kfree(msi_irq_groups[0]); - kfree(msi_irq_groups); - } + msi_for_each_desc(desc, dev, MSI_DESC_ALL) + msi_sysfs_remove_desc(dev, desc); } +#endif /* CONFIG_PCI_MSI_ARCH_FALLBACK */ +#else /* CONFIG_SYSFS */ +static inline int msi_sysfs_create_group(struct device *dev) { return 0; } +static inline int msi_sysfs_populate_desc(struct device *dev, struct msi_desc *desc) { return 0; } +static inline void msi_sysfs_remove_desc(struct device *dev, struct msi_desc *desc) { } +#endif /* !CONFIG_SYSFS */ #ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN static inline void irq_chip_write_msi_msg(struct irq_data *data, @@ -456,43 +710,38 @@ int msi_domain_prepare_irqs(struct irq_domain *domain, struct device *dev, } int msi_domain_populate_irqs(struct irq_domain *domain, struct device *dev, - int virq, int nvec, msi_alloc_info_t *arg) + int virq_base, int nvec, msi_alloc_info_t *arg) { struct msi_domain_info *info = domain->host_data; struct msi_domain_ops *ops = info->ops; struct msi_desc *desc; - int ret = 0; + int ret, virq; - for_each_msi_entry(desc, dev) { - /* Don't even try the multi-MSI brain damage. */ - if (WARN_ON(!desc->irq || desc->nvec_used != 1)) { - ret = -EINVAL; - break; - } + msi_lock_descs(dev); + ret = msi_add_simple_msi_descs(dev, virq_base, nvec); + if (ret) + goto unlock; - if (!(desc->irq >= virq && desc->irq < (virq + nvec))) - continue; + for (virq = virq_base; virq < virq_base + nvec; virq++) { + desc = xa_load(&dev->msi.data->__store, virq); + desc->irq = virq; ops->set_desc(arg, desc); - /* Assumes the domain mutex is held! */ - ret = irq_domain_alloc_irqs_hierarchy(domain, desc->irq, 1, - arg); + ret = irq_domain_alloc_irqs_hierarchy(domain, virq, 1, arg); if (ret) - break; + goto fail; - irq_set_msi_desc_off(desc->irq, 0, desc); - } - - if (ret) { - /* Mop up the damage */ - for_each_msi_entry(desc, dev) { - if (!(desc->irq >= virq && desc->irq < (virq + nvec))) - continue; - - irq_domain_free_irqs_common(domain, desc->irq, 1); - } + irq_set_msi_desc(virq, desc); } + msi_unlock_descs(dev); + return 0; +fail: + for (--virq; virq >= virq_base; virq--) + irq_domain_free_irqs_common(domain, virq, 1); + msi_free_msi_descs_range(dev, MSI_DESC_ALL, virq_base, virq_base + nvec - 1); +unlock: + msi_unlock_descs(dev); return ret; } @@ -531,8 +780,59 @@ static bool msi_check_reservation_mode(struct irq_domain *domain, * Checking the first MSI descriptor is sufficient. MSIX supports * masking and MSI does so when the can_mask attribute is set. */ - desc = first_msi_entry(dev); - return desc->msi_attrib.is_msix || desc->msi_attrib.can_mask; + desc = msi_first_desc(dev, MSI_DESC_ALL); + return desc->pci.msi_attrib.is_msix || desc->pci.msi_attrib.can_mask; +} + +static int msi_handle_pci_fail(struct irq_domain *domain, struct msi_desc *desc, + int allocated) +{ + switch(domain->bus_token) { + case DOMAIN_BUS_PCI_MSI: + case DOMAIN_BUS_VMD_MSI: + if (IS_ENABLED(CONFIG_PCI_MSI)) + break; + fallthrough; + default: + return -ENOSPC; + } + + /* Let a failed PCI multi MSI allocation retry */ + if (desc->nvec_used > 1) + return 1; + + /* If there was a successful allocation let the caller know */ + return allocated ? allocated : -ENOSPC; +} + +#define VIRQ_CAN_RESERVE 0x01 +#define VIRQ_ACTIVATE 0x02 +#define VIRQ_NOMASK_QUIRK 0x04 + +static int msi_init_virq(struct irq_domain *domain, int virq, unsigned int vflags) +{ + struct irq_data *irqd = irq_domain_get_irq_data(domain, virq); + int ret; + + if (!(vflags & VIRQ_CAN_RESERVE)) { + irqd_clr_can_reserve(irqd); + if (vflags & VIRQ_NOMASK_QUIRK) + irqd_set_msi_nomask_quirk(irqd); + } + + if (!(vflags & VIRQ_ACTIVATE)) + return 0; + + ret = irq_domain_activate_irq(irqd, vflags & VIRQ_CAN_RESERVE); + if (ret) + return ret; + /* + * If the interrupt uses reservation mode, clear the activated bit + * so request_irq() will assign the final vector. + */ + if (vflags & VIRQ_CAN_RESERVE) + irqd_clr_activated(irqd); + return 0; } int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, @@ -540,83 +840,103 @@ int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, { struct msi_domain_info *info = domain->host_data; struct msi_domain_ops *ops = info->ops; - struct irq_data *irq_data; - struct msi_desc *desc; msi_alloc_info_t arg = { }; + unsigned int vflags = 0; + struct msi_desc *desc; + int allocated = 0; int i, ret, virq; - bool can_reserve; ret = msi_domain_prepare_irqs(domain, dev, nvec, &arg); if (ret) return ret; - for_each_msi_entry(desc, dev) { - ops->set_desc(&arg, desc); - - virq = __irq_domain_alloc_irqs(domain, -1, desc->nvec_used, - dev_to_node(dev), &arg, false, - desc->affinity); - if (virq < 0) { - ret = -ENOSPC; - if (ops->handle_error) - ret = ops->handle_error(domain, desc, ret); - if (ops->msi_finish) - ops->msi_finish(&arg, ret); - return ret; - } - - for (i = 0; i < desc->nvec_used; i++) { - irq_set_msi_desc_off(virq, i, desc); - irq_debugfs_copy_devname(virq + i, dev); - } - } - - if (ops->msi_finish) - ops->msi_finish(&arg, 0); - - can_reserve = msi_check_reservation_mode(domain, info, dev); - /* * This flag is set by the PCI layer as we need to activate * the MSI entries before the PCI layer enables MSI in the * card. Otherwise the card latches a random msi message. */ - if (!(info->flags & MSI_FLAG_ACTIVATE_EARLY)) - goto skip_activate; + if (info->flags & MSI_FLAG_ACTIVATE_EARLY) + vflags |= VIRQ_ACTIVATE; - for_each_msi_vector(desc, i, dev) { - if (desc->irq == i) { - virq = desc->irq; - dev_dbg(dev, "irq [%d-%d] for MSI\n", - virq, virq + desc->nvec_used - 1); - } - - irq_data = irq_domain_get_irq_data(domain, i); - if (!can_reserve) { - irqd_clr_can_reserve(irq_data); - if (domain->flags & IRQ_DOMAIN_MSI_NOMASK_QUIRK) - irqd_set_msi_nomask_quirk(irq_data); - } - ret = irq_domain_activate_irq(irq_data, can_reserve); - if (ret) - goto cleanup; + /* + * Interrupt can use a reserved vector and will not occupy + * a real device vector until the interrupt is requested. + */ + if (msi_check_reservation_mode(domain, info, dev)) { + vflags |= VIRQ_CAN_RESERVE; + /* + * MSI affinity setting requires a special quirk (X86) when + * reservation mode is active. + */ + if (domain->flags & IRQ_DOMAIN_MSI_NOMASK_QUIRK) + vflags |= VIRQ_NOMASK_QUIRK; } -skip_activate: - /* - * If these interrupts use reservation mode, clear the activated bit - * so request_irq() will assign the final vector. - */ - if (can_reserve) { - for_each_msi_vector(desc, i, dev) { - irq_data = irq_domain_get_irq_data(domain, i); - irqd_clr_activated(irq_data); + msi_for_each_desc(desc, dev, MSI_DESC_NOTASSOCIATED) { + ops->set_desc(&arg, desc); + + virq = __irq_domain_alloc_irqs(domain, -1, desc->nvec_used, + dev_to_node(dev), &arg, false, + desc->affinity); + if (virq < 0) + return msi_handle_pci_fail(domain, desc, allocated); + + for (i = 0; i < desc->nvec_used; i++) { + irq_set_msi_desc_off(virq, i, desc); + irq_debugfs_copy_devname(virq + i, dev); + ret = msi_init_virq(domain, virq + i, vflags); + if (ret) + return ret; } + if (info->flags & MSI_FLAG_DEV_SYSFS) { + ret = msi_sysfs_populate_desc(dev, desc); + if (ret) + return ret; + } + allocated++; } return 0; +} -cleanup: - msi_domain_free_irqs(domain, dev); +static int msi_domain_add_simple_msi_descs(struct msi_domain_info *info, + struct device *dev, + unsigned int num_descs) +{ + if (!(info->flags & MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS)) + return 0; + + return msi_add_simple_msi_descs(dev, 0, num_descs); +} + +/** + * msi_domain_alloc_irqs_descs_locked - Allocate interrupts from a MSI interrupt domain + * @domain: The domain to allocate from + * @dev: Pointer to device struct of the device for which the interrupts + * are allocated + * @nvec: The number of interrupts to allocate + * + * Must be invoked from within a msi_lock_descs() / msi_unlock_descs() + * pair. Use this for MSI irqdomains which implement their own vector + * allocation/free. + * + * Return: %0 on success or an error code. + */ +int msi_domain_alloc_irqs_descs_locked(struct irq_domain *domain, struct device *dev, + int nvec) +{ + struct msi_domain_info *info = domain->host_data; + struct msi_domain_ops *ops = info->ops; + int ret; + + lockdep_assert_held(&dev->msi.data->mutex); + + ret = msi_domain_add_simple_msi_descs(info, dev, nvec); + if (ret) + return ret; + + ret = ops->domain_alloc_irqs(domain, dev, nvec); + if (ret) + msi_domain_free_irqs_descs_locked(domain, dev); return ret; } @@ -629,40 +949,67 @@ cleanup: * * Return: %0 on success or an error code. */ -int msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, - int nvec) +int msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, int nvec) { - struct msi_domain_info *info = domain->host_data; - struct msi_domain_ops *ops = info->ops; + int ret; - return ops->domain_alloc_irqs(domain, dev, nvec); + msi_lock_descs(dev); + ret = msi_domain_alloc_irqs_descs_locked(domain, dev, nvec); + msi_unlock_descs(dev); + return ret; } void __msi_domain_free_irqs(struct irq_domain *domain, struct device *dev) { - struct irq_data *irq_data; + struct msi_domain_info *info = domain->host_data; + struct irq_data *irqd; struct msi_desc *desc; int i; - for_each_msi_vector(desc, i, dev) { - irq_data = irq_domain_get_irq_data(domain, i); - if (irqd_is_activated(irq_data)) - irq_domain_deactivate_irq(irq_data); - } - - for_each_msi_entry(desc, dev) { - /* - * We might have failed to allocate an MSI early - * enough that there is no IRQ associated to this - * entry. If that's the case, don't do anything. - */ - if (desc->irq) { - irq_domain_free_irqs(desc->irq, desc->nvec_used); - desc->irq = 0; + /* Only handle MSI entries which have an interrupt associated */ + msi_for_each_desc(desc, dev, MSI_DESC_ASSOCIATED) { + /* Make sure all interrupts are deactivated */ + for (i = 0; i < desc->nvec_used; i++) { + irqd = irq_domain_get_irq_data(domain, desc->irq + i); + if (irqd && irqd_is_activated(irqd)) + irq_domain_deactivate_irq(irqd); } + + irq_domain_free_irqs(desc->irq, desc->nvec_used); + if (info->flags & MSI_FLAG_DEV_SYSFS) + msi_sysfs_remove_desc(dev, desc); + desc->irq = 0; } } +static void msi_domain_free_msi_descs(struct msi_domain_info *info, + struct device *dev) +{ + if (info->flags & MSI_FLAG_FREE_MSI_DESCS) + msi_free_msi_descs(dev); +} + +/** + * msi_domain_free_irqs_descs_locked - Free interrupts from a MSI interrupt @domain associated to @dev + * @domain: The domain to managing the interrupts + * @dev: Pointer to device struct of the device for which the interrupts + * are free + * + * Must be invoked from within a msi_lock_descs() / msi_unlock_descs() + * pair. Use this for MSI irqdomains which implement their own vector + * allocation. + */ +void msi_domain_free_irqs_descs_locked(struct irq_domain *domain, struct device *dev) +{ + struct msi_domain_info *info = domain->host_data; + struct msi_domain_ops *ops = info->ops; + + lockdep_assert_held(&dev->msi.data->mutex); + + ops->domain_free_irqs(domain, dev); + msi_domain_free_msi_descs(info, dev); +} + /** * msi_domain_free_irqs - Free interrupts from a MSI interrupt @domain associated to @dev * @domain: The domain to managing the interrupts @@ -671,10 +1018,9 @@ void __msi_domain_free_irqs(struct irq_domain *domain, struct device *dev) */ void msi_domain_free_irqs(struct irq_domain *domain, struct device *dev) { - struct msi_domain_info *info = domain->host_data; - struct msi_domain_ops *ops = info->ops; - - return ops->domain_free_irqs(domain, dev); + msi_lock_descs(dev); + msi_domain_free_irqs_descs_locked(domain, dev); + msi_unlock_descs(dev); } /** diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index b8a14d2fb5ba..b7e52a642948 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -107,7 +107,7 @@ static u64 suspend_start; * This delay could be due to SMIs, NMIs, or to VCPU preemptions. Used as * a lower bound for cs->uncertainty_margin values when registering clocks. */ -#define WATCHDOG_MAX_SKEW (50 * NSEC_PER_USEC) +#define WATCHDOG_MAX_SKEW (100 * NSEC_PER_USEC) #ifdef CONFIG_CLOCKSOURCE_WATCHDOG static void clocksource_watchdog_work(struct work_struct *work); @@ -199,23 +199,30 @@ void clocksource_mark_unstable(struct clocksource *cs) spin_unlock_irqrestore(&watchdog_lock, flags); } -ulong max_cswd_read_retries = 3; +ulong max_cswd_read_retries = 2; module_param(max_cswd_read_retries, ulong, 0644); EXPORT_SYMBOL_GPL(max_cswd_read_retries); static int verify_n_cpus = 8; module_param(verify_n_cpus, int, 0644); -static bool cs_watchdog_read(struct clocksource *cs, u64 *csnow, u64 *wdnow) +enum wd_read_status { + WD_READ_SUCCESS, + WD_READ_UNSTABLE, + WD_READ_SKIP +}; + +static enum wd_read_status cs_watchdog_read(struct clocksource *cs, u64 *csnow, u64 *wdnow) { unsigned int nretries; - u64 wd_end, wd_delta; - int64_t wd_delay; + u64 wd_end, wd_end2, wd_delta; + int64_t wd_delay, wd_seq_delay; for (nretries = 0; nretries <= max_cswd_read_retries; nretries++) { local_irq_disable(); *wdnow = watchdog->read(watchdog); *csnow = cs->read(cs); wd_end = watchdog->read(watchdog); + wd_end2 = watchdog->read(watchdog); local_irq_enable(); wd_delta = clocksource_delta(wd_end, *wdnow, watchdog->mask); @@ -226,13 +233,34 @@ static bool cs_watchdog_read(struct clocksource *cs, u64 *csnow, u64 *wdnow) pr_warn("timekeeping watchdog on CPU%d: %s retried %d times before success\n", smp_processor_id(), watchdog->name, nretries); } - return true; + return WD_READ_SUCCESS; } + + /* + * Now compute delay in consecutive watchdog read to see if + * there is too much external interferences that cause + * significant delay in reading both clocksource and watchdog. + * + * If consecutive WD read-back delay > WATCHDOG_MAX_SKEW/2, + * report system busy, reinit the watchdog and skip the current + * watchdog test. + */ + wd_delta = clocksource_delta(wd_end2, wd_end, watchdog->mask); + wd_seq_delay = clocksource_cyc2ns(wd_delta, watchdog->mult, watchdog->shift); + if (wd_seq_delay > WATCHDOG_MAX_SKEW/2) + goto skip_test; } pr_warn("timekeeping watchdog on CPU%d: %s read-back delay of %lldns, attempt %d, marking unstable\n", smp_processor_id(), watchdog->name, wd_delay, nretries); - return false; + return WD_READ_UNSTABLE; + +skip_test: + pr_info("timekeeping watchdog on CPU%d: %s wd-wd read-back delay of %lldns\n", + smp_processor_id(), watchdog->name, wd_seq_delay); + pr_info("wd-%s-wd read-back delay of %lldns, clock-skew test skipped!\n", + cs->name, wd_delay); + return WD_READ_SKIP; } static u64 csnow_mid; @@ -356,6 +384,7 @@ static void clocksource_watchdog(struct timer_list *unused) int next_cpu, reset_pending; int64_t wd_nsec, cs_nsec; struct clocksource *cs; + enum wd_read_status read_ret; u32 md; spin_lock(&watchdog_lock); @@ -373,9 +402,12 @@ static void clocksource_watchdog(struct timer_list *unused) continue; } - if (!cs_watchdog_read(cs, &csnow, &wdnow)) { - /* Clock readout unreliable, so give it up. */ - __clocksource_unstable(cs); + read_ret = cs_watchdog_read(cs, &csnow, &wdnow); + + if (read_ret != WD_READ_SUCCESS) { + if (read_ret == WD_READ_UNSTABLE) + /* Clock readout unreliable, so give it up. */ + __clocksource_unstable(cs); continue; }