shinys000114/linux
1
0
Fork 0
mirror of https://github.com/hardkernel/linux.git synced 2026-06-07 11:26:02 +09:00
Code Issues Packages Projects Releases Wiki Activity

Merge 6daa90439e ("Merge tag 'dmaengine-5.11-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine") into android-mainline

Browse Source 
Steps on the way to 5.11-rc1

Signed-off-by: Greg Kroah-Hartman <gregkh@google.com>
Change-Id: I2a1e3d1ffe32d541d151892a7042721a902eb05e
This commit is contained in:
Greg Kroah-Hartman
2020-12-21 12:20:43 +01:00
parent 68d0da6f3e 6daa90439e
commit 7494e31ef6
160 changed files with 18284 additions and 4015 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
Documentation/ABI/stable/sysfs-driver-dma-idxd
View File

@@ -77,6 +77,13 @@ Contact: dmaengine@vger.kernel.org
Description: The operation capability bit mask specify the operation types
supported by the this device.
What: /sys/bus/dsa/devices/dsa<m>/pasid_enabled
Date: Oct 27, 2020
KernelVersion: 5.11.0
Contact: dmaengine@vger.kernel.org
Description: To indicate if PASID (process address space identifier) is
enabled or not for this device.
What: /sys/bus/dsa/devices/dsa<m>/state
Date: Oct 25, 2019
KernelVersion: 5.6.0
@@ -122,6 +129,13 @@ KernelVersion: 5.10.0
Contact: dmaengine@vger.kernel.org
Description: The last executed device administrative command's status/error.
What: /sys/bus/dsa/devices/wq<m>.<n>/block_on_fault
Date: Oct 27, 2020
KernelVersion: 5.11.0
Contact: dmaengine@vger.kernel.org
Description: To indicate block on fault is allowed or not for the work queue
to support on demand paging.
What: /sys/bus/dsa/devices/wq<m>.<n>/group_id
Date: Oct 25, 2019
KernelVersion: 5.6.0
@@ -190,6 +204,13 @@ Contact: dmaengine@vger.kernel.org
Description: The max batch size for this workqueue. Cannot exceed device
max batch size. Configurable parameter.
What: /sys/bus/dsa/devices/wq<m>.<n>/ats_disable
Date: Nov 13, 2020
KernelVersion: 5.11.0
Contact: dmaengine@vger.kernel.org
Description: Indicate whether ATS disable is turned on for the workqueue.
0 indicates ATS is on, and 1 indicates ATS is off for the workqueue.
What: /sys/bus/dsa/devices/engine<m>.<n>/group_id
Date: Oct 25, 2019
KernelVersion: 5.6.0

5
Documentation/devicetree/bindings/dma/allwinner,sun50i-a64-dma.yaml
View File

@@ -21,6 +21,7 @@ properties:
compatible:
oneOf:
- const: allwinner,sun50i-a64-dma
- const: allwinner,sun50i-a100-dma
- const: allwinner,sun50i-h6-dma
- items:
- const: allwinner,sun8i-r40-dma
@@ -56,7 +57,9 @@ required:
if:
properties:
compatible:
const: allwinner,sun50i-h6-dma
enum:
- allwinner,sun50i-a100-dma
- allwinner,sun50i-h6-dma
then:
properties:

3
Documentation/devicetree/bindings/dma/atmel-xdma.txt
View File

@@ -2,7 +2,8 @@
* XDMA Controller
Required properties:
- compatible: Should be "atmel,sama5d4-dma" or "microchip,sam9x60-dma".
- compatible: Should be "atmel,sama5d4-dma", "microchip,sam9x60-dma" or
"microchip,sama7g5-dma".
- reg: Should contain DMA registers location and length.
- interrupts: Should contain DMA interrupt.
- #dma-cells: Must be <1>, used to represent the number of integer cells in

1
Documentation/devicetree/bindings/dma/mtk-uart-apdma.txt
View File

@@ -4,6 +4,7 @@ Required properties:
- compatible should contain:
* "mediatek,mt2712-uart-dma" for MT2712 compatible APDMA
* "mediatek,mt6577-uart-dma" for MT6577 and all of the above
* "mediatek,mt8516-uart-dma", "mediatek,mt6577" for MT8516 SoC
- reg: The base address of the APDMA register bank.

88
Documentation/devicetree/bindings/dma/qcom,gpi.yaml Normal file
View File

@@ -0,0 +1,88 @@
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
%YAML 1.2
---
$id: http://devicetree.org/schemas/dma/qcom,gpi.yaml#
$schema: http://devicetree.org/meta-schemas/core.yaml#
title: Qualcomm Technologies Inc GPI DMA controller
maintainers:
- Vinod Koul <vkoul@kernel.org>
description: |
QCOM GPI DMA controller provides DMA capabilities for
peripheral buses such as I2C, UART, and SPI.
allOf:
- $ref: "dma-controller.yaml#"
properties:
compatible:
enum:
- qcom,sdm845-gpi-dma
reg:
maxItems: 1
interrupts:
description:
Interrupt lines for each GPI instance
maxItems: 13
"#dma-cells":
const: 3
description: >
DMA clients must use the format described in dma.txt, giving a phandle
to the DMA controller plus the following 3 integer cells:
- channel: if set to 0xffffffff, any available channel will be allocated
for the client. Otherwise, the exact channel specified will be used.
- seid: serial id of the client as defined in the SoC documentation.
- client: type of the client as defined in dt-bindings/dma/qcom-gpi.h
iommus:
maxItems: 1
dma-channels:
maximum: 31
dma-channel-mask:
maxItems: 1
required:
- compatible
- reg
- interrupts
- "#dma-cells"
- iommus
- dma-channels
- dma-channel-mask
additionalProperties: false
examples:
- |
#include <dt-bindings/interrupt-controller/arm-gic.h>
#include <dt-bindings/dma/qcom-gpi.h>
gpi_dma0: dma-controller@800000 {
compatible = "qcom,gpi-dma";
#dma-cells = <3>;
reg = <0x00800000 0x60000>;
iommus = <&apps_smmu 0x0016 0x0>;
dma-channels = <13>;
dma-channel-mask = <0xfa>;
interrupts = <GIC_SPI 244 IRQ_TYPE_LEVEL_HIGH>,
<GIC_SPI 245 IRQ_TYPE_LEVEL_HIGH>,
<GIC_SPI 246 IRQ_TYPE_LEVEL_HIGH>,
<GIC_SPI 247 IRQ_TYPE_LEVEL_HIGH>,
<GIC_SPI 248 IRQ_TYPE_LEVEL_HIGH>,
<GIC_SPI 249 IRQ_TYPE_LEVEL_HIGH>,
<GIC_SPI 250 IRQ_TYPE_LEVEL_HIGH>,
<GIC_SPI 251 IRQ_TYPE_LEVEL_HIGH>,
<GIC_SPI 252 IRQ_TYPE_LEVEL_HIGH>,
<GIC_SPI 253 IRQ_TYPE_LEVEL_HIGH>,
<GIC_SPI 254 IRQ_TYPE_LEVEL_HIGH>,
<GIC_SPI 255 IRQ_TYPE_LEVEL_HIGH>,
<GIC_SPI 256 IRQ_TYPE_LEVEL_HIGH>;
};
...

164
Documentation/devicetree/bindings/dma/ti/k3-bcdma.yaml Normal file
View File

@@ -0,0 +1,164 @@
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
%YAML 1.2
---
$id: http://devicetree.org/schemas/dma/ti/k3-bcdma.yaml#
$schema: http://devicetree.org/meta-schemas/core.yaml#
title: Texas Instruments K3 DMSS BCDMA Device Tree Bindings
maintainers:
- Peter Ujfalusi <peter.ujfalusi@ti.com>
description: |
The Block Copy DMA (BCDMA) is intended to perform similar functions as the TR
mode channels of K3 UDMA-P.
BCDMA includes block copy channels and Split channels.
Block copy channels mainly used for memory to memory transfers, but with
optional triggers a block copy channel can service peripherals by accessing
directly to memory mapped registers or area.
Split channels can be used to service PSI-L based peripherals.
The peripherals can be PSI-L native or legacy, non PSI-L native peripherals
with PDMAs. PDMA is tasked to act as a bridge between the PSI-L fabric and the
legacy peripheral.
PDMAs can be configured via BCDMA split channel's peer registers to match with
the configuration of the legacy peripheral.
allOf:
- $ref: /schemas/dma/dma-controller.yaml#
properties:
compatible:
const: ti,am64-dmss-bcdma
"#dma-cells":
const: 3
description: |
cell 1: type of the BCDMA channel to be used to service the peripheral:
0 - split channel
1 - block copy channel using global trigger 1
2 - block copy channel using global trigger 2
3 - block copy channel using local trigger
cell 2: parameter for the channel:
if cell 1 is 0 (split channel):
PSI-L thread ID of the remote (to BCDMA) end.
Valid ranges for thread ID depends on the data movement direction:
for source thread IDs (rx): 0 - 0x7fff
for destination thread IDs (tx): 0x8000 - 0xffff
Please refer to the device documentation for the PSI-L thread map and
also the PSI-L peripheral chapter for the correct thread ID.
if cell 1 is 1 or 2 (block copy channel using global trigger):
Unused, ignored
The trigger must be configured for the channel externally to BCDMA,
channels using global triggers should not be requested directly, but
via DMA event router.
if cell 1 is 3 (block copy channel using local trigger):
bchan number of the locally triggered channel
cell 3: ASEL value for the channel
reg:
maxItems: 5
reg-names:
items:
- const: gcfg
- const: bchanrt
- const: rchanrt
- const: tchanrt
- const: ringrt
msi-parent: true
ti,asel:
$ref: /schemas/types.yaml#/definitions/uint32
description: ASEL value for non slave channels
ti,sci-rm-range-bchan:
$ref: /schemas/types.yaml#/definitions/uint32-array
description: |
Array of BCDMA block-copy channel resource subtypes for resource
allocation for this host
minItems: 1
# Should be enough
maxItems: 255
items:
maximum: 0x3f
ti,sci-rm-range-tchan:
$ref: /schemas/types.yaml#/definitions/uint32-array
description: |
Array of BCDMA split tx channel resource subtypes for resource allocation
for this host
minItems: 1
# Should be enough
maxItems: 255
items:
maximum: 0x3f
ti,sci-rm-range-rchan:
$ref: /schemas/types.yaml#/definitions/uint32-array
description: |
Array of BCDMA split rx channel resource subtypes for resource allocation
for this host
minItems: 1
# Should be enough
maxItems: 255
items:
maximum: 0x3f
required:
- compatible
- "#dma-cells"
- reg
- reg-names
- msi-parent
- ti,sci
- ti,sci-dev-id
- ti,sci-rm-range-bchan
- ti,sci-rm-range-tchan
- ti,sci-rm-range-rchan
unevaluatedProperties: false
examples:
- |+
cbass_main {
#address-cells = <2>;
#size-cells = <2>;
main_dmss {
compatible = "simple-mfd";
#address-cells = <2>;
#size-cells = <2>;
dma-ranges;
ranges;
ti,sci-dev-id = <25>;
main_bcdma: dma-controller@485c0100 {
compatible = "ti,am64-dmss-bcdma";
reg = <0x0 0x485c0100 0x0 0x100>,
<0x0 0x4c000000 0x0 0x20000>,
<0x0 0x4a820000 0x0 0x20000>,
<0x0 0x4aa40000 0x0 0x20000>,
<0x0 0x4bc00000 0x0 0x100000>;
reg-names = "gcfg", "bchanrt", "rchanrt", "tchanrt", "ringrt";
msi-parent = <&inta_main_dmss>;
#dma-cells = <3>;
ti,sci = <&dmsc>;
ti,sci-dev-id = <26>;
ti,sci-rm-range-bchan = <0x20>; /* BLOCK_COPY_CHAN */
ti,sci-rm-range-rchan = <0x21>; /* SPLIT_TR_RX_CHAN */
ti,sci-rm-range-tchan = <0x22>; /* SPLIT_TR_TX_CHAN */
};
};
};

172
Documentation/devicetree/bindings/dma/ti/k3-pktdma.yaml Normal file
View File

@@ -0,0 +1,172 @@
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
%YAML 1.2
---
$id: http://devicetree.org/schemas/dma/ti/k3-pktdma.yaml#
$schema: http://devicetree.org/meta-schemas/core.yaml#
title: Texas Instruments K3 DMSS PKTDMA Device Tree Bindings
maintainers:
- Peter Ujfalusi <peter.ujfalusi@ti.com>
description: |
The Packet DMA (PKTDMA) is intended to perform similar functions as the packet
mode channels of K3 UDMA-P.
PKTDMA only includes Split channels to service PSI-L based peripherals.
The peripherals can be PSI-L native or legacy, non PSI-L native peripherals
with PDMAs. PDMA is tasked to act as a bridge between the PSI-L fabric and the
legacy peripheral.
PDMAs can be configured via PKTDMA split channel's peer registers to match
with the configuration of the legacy peripheral.
allOf:
- $ref: /schemas/dma/dma-controller.yaml#
properties:
compatible:
const: ti,am64-dmss-pktdma
"#dma-cells":
const: 2
description: |
The first cell is the PSI-L thread ID of the remote (to PKTDMA) end.
Valid ranges for thread ID depends on the data movement direction:
for source thread IDs (rx): 0 - 0x7fff
for destination thread IDs (tx): 0x8000 - 0xffff
Please refer to the device documentation for the PSI-L thread map and also
the PSI-L peripheral chapter for the correct thread ID.
The second cell is the ASEL value for the channel
reg:
maxItems: 4
reg-names:
items:
- const: gcfg
- const: rchanrt
- const: tchanrt
- const: ringrt
msi-parent: true
ti,sci-rm-range-tchan:
$ref: /schemas/types.yaml#/definitions/uint32-array
description: |
Array of PKTDMA split tx channel resource subtypes for resource allocation
for this host
minItems: 1
# Should be enough
maxItems: 255
items:
maximum: 0x3f
ti,sci-rm-range-tflow:
$ref: /schemas/types.yaml#/definitions/uint32-array
description: |
Array of PKTDMA split tx flow resource subtypes for resource allocation
for this host
minItems: 1
# Should be enough
maxItems: 255
items:
maximum: 0x3f
ti,sci-rm-range-rchan:
$ref: /schemas/types.yaml#/definitions/uint32-array
description: |
Array of PKTDMA split rx channel resource subtypes for resource allocation
for this host
minItems: 1
# Should be enough
maxItems: 255
items:
maximum: 0x3f
ti,sci-rm-range-rflow:
$ref: /schemas/types.yaml#/definitions/uint32-array
description: |
Array of PKTDMA split rx flow resource subtypes for resource allocation
for this host
minItems: 1
# Should be enough
maxItems: 255
items:
maximum: 0x3f
required:
- compatible
- "#dma-cells"
- reg
- reg-names
- msi-parent
- ti,sci
- ti,sci-dev-id
- ti,sci-rm-range-tchan
- ti,sci-rm-range-tflow
- ti,sci-rm-range-rchan
- ti,sci-rm-range-rflow
unevaluatedProperties: false
examples:
- |+
cbass_main {
#address-cells = <2>;
#size-cells = <2>;
main_dmss {
compatible = "simple-mfd";
#address-cells = <2>;
#size-cells = <2>;
dma-ranges;
ranges;
ti,sci-dev-id = <25>;
main_pktdma: dma-controller@485c0000 {
compatible = "ti,am64-dmss-pktdma";
reg = <0x0 0x485c0000 0x0 0x100>,
<0x0 0x4a800000 0x0 0x20000>,
<0x0 0x4aa00000 0x0 0x40000>,
<0x0 0x4b800000 0x0 0x400000>;
reg-names = "gcfg", "rchanrt", "tchanrt", "ringrt";
msi-parent = <&inta_main_dmss>;
#dma-cells = <2>;
ti,sci = <&dmsc>;
ti,sci-dev-id = <30>;
ti,sci-rm-range-tchan = <0x23>, /* UNMAPPED_TX_CHAN */
<0x24>, /* CPSW_TX_CHAN */
<0x25>, /* SAUL_TX_0_CHAN */
<0x26>, /* SAUL_TX_1_CHAN */
<0x27>, /* ICSSG_0_TX_CHAN */
<0x28>; /* ICSSG_1_TX_CHAN */
ti,sci-rm-range-tflow = <0x10>, /* RING_UNMAPPED_TX_CHAN */
<0x11>, /* RING_CPSW_TX_CHAN */
<0x12>, /* RING_SAUL_TX_0_CHAN */
<0x13>, /* RING_SAUL_TX_1_CHAN */
<0x14>, /* RING_ICSSG_0_TX_CHAN */
<0x15>; /* RING_ICSSG_1_TX_CHAN */
ti,sci-rm-range-rchan = <0x29>, /* UNMAPPED_RX_CHAN */
<0x2b>, /* CPSW_RX_CHAN */
<0x2d>, /* SAUL_RX_0_CHAN */
<0x2f>, /* SAUL_RX_1_CHAN */
<0x31>, /* SAUL_RX_2_CHAN */
<0x33>, /* SAUL_RX_3_CHAN */
<0x35>, /* ICSSG_0_RX_CHAN */
<0x37>; /* ICSSG_1_RX_CHAN */
ti,sci-rm-range-rflow = <0x2a>, /* FLOW_UNMAPPED_RX_CHAN */
<0x2c>, /* FLOW_CPSW_RX_CHAN */
<0x2e>, /* FLOW_SAUL_RX_0/1_CHAN */
<0x32>, /* FLOW_SAUL_RX_2/3_CHAN */
<0x36>, /* FLOW_ICSSG_0_RX_CHAN */
<0x38>; /* FLOW_ICSSG_1_RX_CHAN */
};
};
};

209
Documentation/devicetree/bindings/mailbox/arm,mhuv2.yaml Normal file
View File

@@ -0,0 +1,209 @@
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
%YAML 1.2
---
$id: http://devicetree.org/schemas/mailbox/arm,mhuv2.yaml#
$schema: http://devicetree.org/meta-schemas/core.yaml#
title: ARM MHUv2 Mailbox Controller
maintainers:
- Tushar Khandelwal <tushar.khandelwal@arm.com>
- Viresh Kumar <viresh.kumar@linaro.org>
description: |
The Arm Message Handling Unit (MHU) Version 2 is a mailbox controller that has
between 1 and 124 channel windows (each 32-bit wide) to provide unidirectional
communication with remote processor(s), where the number of channel windows
are implementation dependent.
Given the unidirectional nature of the controller, an MHUv2 mailbox may only
be written to or read from. If a pair of MHU controllers is implemented
between two processing elements to provide bidirectional communication, these
must be specified as two separate mailboxes.
If the interrupts property is present in device tree node, then its treated as
a "receiver" mailbox, otherwise a "sender".
An MHU controller must be specified along with the supported transport
protocols. The transport protocols determine the method of data transmission
as well as the number of provided mailbox channels.
Following are the possible transport protocols.
- Data-transfer: Each transfer is made of one or more words, using one or more
channel windows.
- Doorbell: Each transfer is made up of single bit flag, using any one of the
bits in a channel window. A channel window can support up to 32 doorbells
and the entire window shall be used in doorbell protocol. Optionally, data
may be transmitted through a shared memory region, wherein the MHU is used
strictly as an interrupt generation mechanism but that is out of the scope
of these bindings.
# We need a select here so we don't match all nodes with 'arm,primecell'
select:
properties:
compatible:
contains:
enum:
- arm,mhuv2-tx
- arm,mhuv2-rx
required:
- compatible
properties:
compatible:
oneOf:
- description: Sender mode
items:
- const: arm,mhuv2-tx
- const: arm,primecell
- description: Receiver-mode
items:
- const: arm,mhuv2-rx
- const: arm,primecell
reg:
maxItems: 1
interrupts:
description: |
The MHUv2 controller always implements an interrupt in the "receiver"
mode, while the interrupt in the "sender" mode was not available in the
version MHUv2.0, but the later versions do have it.
maxItems: 1
clocks:
maxItems: 1
clock-names:
maxItems: 1
arm,mhuv2-protocols:
$ref: /schemas/types.yaml#/definitions/uint32-matrix
description: |
The MHUv2 controller may contain up to 124 channel windows (each 32-bit
wide). The hardware and the DT bindings allows any combination of those to
be used for various transport protocols.
This property allows a platform to describe how these channel windows are
used in various transport protocols. The entries in this property shall be
present as an array of tuples, where each tuple describes details about
one of the transport protocol being implemented over some channel
window(s).
The first field of a tuple signifies the transfer protocol, 0 is reserved
for doorbell protocol, and 1 is reserved for data-transfer protocol.
Using any other value in the first field of a tuple makes it invalid.
The second field of a tuple signifies the number of channel windows where
the protocol would be used and should be set to a non zero value. For
doorbell protocol this field signifies the number of 32-bit channel
windows that implement the doorbell protocol. For data-transfer protocol,
this field signifies the number of 32-bit channel windows that implement
the data-transfer protocol.
The total number of channel windows specified here shouldn't be more than
the ones implemented by the platform, though one can specify lesser number
of windows here than what the platform implements.
mhu: mailbox@2b1f0000 {
...
arm,mhuv2-protocols = <0 2>, <1 1>, <1 5>, <1 7>;
}
The above example defines the protocols of an ARM MHUv2 mailbox
controller, where a total of 15 channel windows are used. The first two
windows are used in doorbell protocol (64 doorbells), followed by 1, 5 and
7 windows (separately) used in data-transfer protocol.
minItems: 1
maxItems: 124
items:
items:
- enum: [ 0, 1 ]
- minimum: 0
maximum: 124
'#mbox-cells':
description: |
It is always set to 2. The first argument in the consumers 'mboxes'
property represents the channel window group, which may be used in
doorbell, or data-transfer protocol, and the second argument (only
relevant in doorbell protocol, should be 0 otherwise) represents the
doorbell number within the 32 bit wide channel window.
From the example given above for arm,mhuv2-protocols, here is how a client
node can reference them.
mboxes = <&mhu 0 5>; // Channel Window Group 0, doorbell 5.
mboxes = <&mhu 1 7>; // Channel Window Group 1, doorbell 7.
mboxes = <&mhu 2 0>; // Channel Window Group 2, data transfer protocol with 1 window.
mboxes = <&mhu 3 0>; // Channel Window Group 3, data transfer protocol with 5 windows.
mboxes = <&mhu 4 0>; // Channel Window Group 4, data transfer protocol with 7 windows.
const: 2
if:
# Interrupt is compulsory for receiver
properties:
compatible:
contains:
const: arm,mhuv2-rx
then:
required:
- interrupts
required:
- compatible
- reg
- '#mbox-cells'
- arm,mhuv2-protocols
additionalProperties: false
examples:
# Multiple transport protocols implemented by the mailbox controllers
- |
soc {
#address-cells = <2>;
#size-cells = <2>;
mhu_tx: mailbox@2b1f0000 {
#mbox-cells = <2>;
compatible = "arm,mhuv2-tx", "arm,primecell";
reg = <0 0x2b1f0000 0 0x1000>;
clocks = <&clock 0>;
clock-names = "apb_pclk";
interrupts = <0 45 4>;
arm,mhuv2-protocols = <1 5>, <1 2>, <1 5>, <1 7>, <0 2>;
};
mhu_rx: mailbox@2b1f1000 {
#mbox-cells = <2>;
compatible = "arm,mhuv2-rx", "arm,primecell";
reg = <0 0x2b1f1000 0 0x1000>;
clocks = <&clock 0>;
clock-names = "apb_pclk";
interrupts = <0 46 4>;
arm,mhuv2-protocols = <1 1>, <1 7>, <0 2>;
};
mhu_client: scb@2e000000 {
compatible = "fujitsu,mb86s70-scb-1.0";
reg = <0 0x2e000000 0 0x4000>;
mboxes =
//data-transfer protocol with 5 windows, mhu-tx
<&mhu_tx 2 0>,
//data-transfer protocol with 7 windows, mhu-tx
<&mhu_tx 3 0>,
//doorbell protocol channel 4, doorbell 27, mhu-tx
<&mhu_tx 4 27>,
//data-transfer protocol with 1 window, mhu-rx
<&mhu_rx 0 0>;
};
};

4
Documentation/driver-api/dmaengine/client.rst
View File

@@ -120,7 +120,9 @@ The details of these operations are:
.. code-block:: c
nr_sg = dma_map_sg(chan->device->dev, sgl, sg_len);
struct device *dma_dev = dmaengine_get_dma_device(chan);
nr_sg = dma_map_sg(dma_dev, sgl, sg_len);
if (nr_sg == 0)
/* error */

9
MAINTAINERS
View File

@@ -10531,6 +10531,15 @@ F: drivers/mailbox/
F: include/linux/mailbox_client.h
F: include/linux/mailbox_controller.h
MAILBOX ARM MHUv2
M: Viresh Kumar <viresh.kumar@linaro.org>
M: Tushar Khandelwal <Tushar.Khandelwal@arm.com>
L: linux-kernel@vger.kernel.org
S: Maintained
F: drivers/mailbox/arm_mhuv2.c
F: include/linux/mailbox/arm_mhuv2_message.h
F: Documentation/devicetree/bindings/mailbox/arm,mhuv2.yaml
MAN-PAGES: MANUAL PAGES FOR LINUX -- Sections 2, 3, 4, 5, and 7
M: Michael Kerrisk <mtk.manpages@gmail.com>
L: linux-man@vger.kernel.org

8
drivers/block/rbd.c
View File

@@ -3925,8 +3925,12 @@ static int find_watcher(struct rbd_device *rbd_dev,
sscanf(locker->id.cookie, RBD_LOCK_COOKIE_PREFIX " %llu", &cookie);
for (i = 0; i < num_watchers; i++) {
if (!memcmp(&watchers[i].addr, &locker->info.addr,
sizeof(locker->info.addr)) &&
/*
* Ignore addr->type while comparing. This mimics
* entity_addr_t::get_legacy_str() + strcmp().
*/
if (ceph_addr_equal_no_type(&watchers[i].addr,
&locker->info.addr) &&
watchers[i].cookie == cookie) {
struct rbd_client_id cid = {
.gid = le64_to_cpu(watchers[i].name.num),

10
drivers/dma/Kconfig
View File

@@ -296,6 +296,16 @@ config INTEL_IDXD
If unsure, say N.
# Config symbol that collects all the dependencies that's necessary to
# support shared virtual memory for the devices supported by idxd.
config INTEL_IDXD_SVM
bool "Accelerator Shared Virtual Memory Support"
depends on INTEL_IDXD
depends on INTEL_IOMMU_SVM
depends on PCI_PRI
depends on PCI_PASID
depends on PCI_IOV
config INTEL_IOATDMA
tristate "Intel I/OAT DMA support"
depends on PCI && X86_64

163
drivers/dma/at_xdmac.c
View File

@@ -30,7 +30,24 @@
#define AT_XDMAC_FIFO_SZ(i) (((i) >> 5) & 0x7FF) /* Number of Bytes */
#define AT_XDMAC_NB_REQ(i) ((((i) >> 16) & 0x3F) + 1) /* Number of Peripheral Requests Minus One */
#define AT_XDMAC_GCFG 0x04 /* Global Configuration Register */
#define AT_XDMAC_WRHP(i) (((i) & 0xF) << 4)
#define AT_XDMAC_WRMP(i) (((i) & 0xF) << 8)
#define AT_XDMAC_WRLP(i) (((i) & 0xF) << 12)
#define AT_XDMAC_RDHP(i) (((i) & 0xF) << 16)
#define AT_XDMAC_RDMP(i) (((i) & 0xF) << 20)
#define AT_XDMAC_RDLP(i) (((i) & 0xF) << 24)
#define AT_XDMAC_RDSG(i) (((i) & 0xF) << 28)
#define AT_XDMAC_GCFG_M2M (AT_XDMAC_RDLP(0xF) | AT_XDMAC_WRLP(0xF))
#define AT_XDMAC_GCFG_P2M (AT_XDMAC_RDSG(0x1) | AT_XDMAC_RDHP(0x3) | \
AT_XDMAC_WRHP(0x5))
#define AT_XDMAC_GWAC 0x08 /* Global Weighted Arbiter Configuration Register */
#define AT_XDMAC_PW0(i) (((i) & 0xF) << 0)
#define AT_XDMAC_PW1(i) (((i) & 0xF) << 4)
#define AT_XDMAC_PW2(i) (((i) & 0xF) << 8)
#define AT_XDMAC_PW3(i) (((i) & 0xF) << 12)
#define AT_XDMAC_GWAC_M2M 0
#define AT_XDMAC_GWAC_P2M (AT_XDMAC_PW0(0xF) | AT_XDMAC_PW2(0xF))
#define AT_XDMAC_GIE 0x0C /* Global Interrupt Enable Register */
#define AT_XDMAC_GID 0x10 /* Global Interrupt Disable Register */
#define AT_XDMAC_GIM 0x14 /* Global Interrupt Mask Register */
@@ -38,13 +55,6 @@
#define AT_XDMAC_GE 0x1C /* Global Channel Enable Register */
#define AT_XDMAC_GD 0x20 /* Global Channel Disable Register */
#define AT_XDMAC_GS 0x24 /* Global Channel Status Register */
#define AT_XDMAC_GRS 0x28 /* Global Channel Read Suspend Register */
#define AT_XDMAC_GWS 0x2C /* Global Write Suspend Register */
#define AT_XDMAC_GRWS 0x30 /* Global Channel Read Write Suspend Register */
#define AT_XDMAC_GRWR 0x34 /* Global Channel Read Write Resume Register */
#define AT_XDMAC_GSWR 0x38 /* Global Channel Software Request Register */
#define AT_XDMAC_GSWS 0x3C /* Global channel Software Request Status Register */
#define AT_XDMAC_GSWF 0x40 /* Global Channel Software Flush Request Register */
#define AT_XDMAC_VERSION 0xFFC /* XDMAC Version Register */
/* Channel relative registers offsets */
@@ -150,8 +160,6 @@
#define AT_XDMAC_CSUS 0x30 /* Channel Source Microblock Stride */
#define AT_XDMAC_CDUS 0x34 /* Channel Destination Microblock Stride */
#define AT_XDMAC_CHAN_REG_BASE 0x50 /* Channel registers base address */
/* Microblock control members */
#define AT_XDMAC_MBR_UBC_UBLEN_MAX 0xFFFFFFUL /* Maximum Microblock Length */
#define AT_XDMAC_MBR_UBC_NDE (0x1 << 24) /* Next Descriptor Enable */
@@ -179,6 +187,29 @@ enum atc_status {
AT_XDMAC_CHAN_IS_PAUSED,
};
struct at_xdmac_layout {
/* Global Channel Read Suspend Register */
u8 grs;
/* Global Write Suspend Register */
u8 gws;
/* Global Channel Read Write Suspend Register */
u8 grws;
/* Global Channel Read Write Resume Register */
u8 grwr;
/* Global Channel Software Request Register */
u8 gswr;
/* Global channel Software Request Status Register */
u8 gsws;
/* Global Channel Software Flush Request Register */
u8 gswf;
/* Channel reg base */
u8 chan_cc_reg_base;
/* Source/Destination Interface must be specified or not */
bool sdif;
/* AXI queue priority configuration supported */
bool axi_config;
};
/* ----- Channels ----- */
struct at_xdmac_chan {
struct dma_chan chan;
@@ -212,6 +243,7 @@ struct at_xdmac {
struct clk *clk;
u32 save_gim;
struct dma_pool *at_xdmac_desc_pool;
const struct at_xdmac_layout *layout;
struct at_xdmac_chan chan[];
};
@@ -244,9 +276,35 @@ struct at_xdmac_desc {
struct list_head xfer_node;
} __aligned(sizeof(u64));
static const struct at_xdmac_layout at_xdmac_sama5d4_layout = {
.grs = 0x28,
.gws = 0x2C,
.grws = 0x30,
.grwr = 0x34,
.gswr = 0x38,
.gsws = 0x3C,
.gswf = 0x40,
.chan_cc_reg_base = 0x50,
.sdif = true,
.axi_config = false,
};
static const struct at_xdmac_layout at_xdmac_sama7g5_layout = {
.grs = 0x30,
.gws = 0x38,
.grws = 0x40,
.grwr = 0x44,
.gswr = 0x48,
.gsws = 0x4C,
.gswf = 0x50,
.chan_cc_reg_base = 0x60,
.sdif = false,
.axi_config = true,
};
static inline void __iomem *at_xdmac_chan_reg_base(struct at_xdmac *atxdmac, unsigned int chan_nb)
{
return atxdmac->regs + (AT_XDMAC_CHAN_REG_BASE + chan_nb * 0x40);
return atxdmac->regs + (atxdmac->layout->chan_cc_reg_base + chan_nb * 0x40);
}
#define at_xdmac_read(atxdmac, reg) readl_relaxed((atxdmac)->regs + (reg))
@@ -345,8 +403,10 @@ static void at_xdmac_start_xfer(struct at_xdmac_chan *atchan,
first->active_xfer = true;
/* Tell xdmac where to get the first descriptor. */
reg = AT_XDMAC_CNDA_NDA(first->tx_dma_desc.phys)
| AT_XDMAC_CNDA_NDAIF(atchan->memif);
reg = AT_XDMAC_CNDA_NDA(first->tx_dma_desc.phys);
if (atxdmac->layout->sdif)
reg |= AT_XDMAC_CNDA_NDAIF(atchan->memif);
at_xdmac_chan_write(atchan, AT_XDMAC_CNDA, reg);
/*
@@ -541,6 +601,7 @@ static int at_xdmac_compute_chan_conf(struct dma_chan *chan,
enum dma_transfer_direction direction)
{
struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device);
int csize, dwidth;
if (direction == DMA_DEV_TO_MEM) {
@@ -548,12 +609,14 @@ static int at_xdmac_compute_chan_conf(struct dma_chan *chan,
AT91_XDMAC_DT_PERID(atchan->perid)
| AT_XDMAC_CC_DAM_INCREMENTED_AM
| AT_XDMAC_CC_SAM_FIXED_AM
| AT_XDMAC_CC_DIF(atchan->memif)
| AT_XDMAC_CC_SIF(atchan->perif)
| AT_XDMAC_CC_SWREQ_HWR_CONNECTED
| AT_XDMAC_CC_DSYNC_PER2MEM
| AT_XDMAC_CC_MBSIZE_SIXTEEN
| AT_XDMAC_CC_TYPE_PER_TRAN;
if (atxdmac->layout->sdif)
atchan->cfg |= AT_XDMAC_CC_DIF(atchan->memif) |
AT_XDMAC_CC_SIF(atchan->perif);
csize = ffs(atchan->sconfig.src_maxburst) - 1;
if (csize < 0) {
dev_err(chan2dev(chan), "invalid src maxburst value\n");
@@ -571,12 +634,14 @@ static int at_xdmac_compute_chan_conf(struct dma_chan *chan,
AT91_XDMAC_DT_PERID(atchan->perid)
| AT_XDMAC_CC_DAM_FIXED_AM
| AT_XDMAC_CC_SAM_INCREMENTED_AM
| AT_XDMAC_CC_DIF(atchan->perif)
| AT_XDMAC_CC_SIF(atchan->memif)
| AT_XDMAC_CC_SWREQ_HWR_CONNECTED
| AT_XDMAC_CC_DSYNC_MEM2PER
| AT_XDMAC_CC_MBSIZE_SIXTEEN
| AT_XDMAC_CC_TYPE_PER_TRAN;
if (atxdmac->layout->sdif)
atchan->cfg |= AT_XDMAC_CC_DIF(atchan->perif) |
AT_XDMAC_CC_SIF(atchan->memif);
csize = ffs(atchan->sconfig.dst_maxburst) - 1;
if (csize < 0) {
dev_err(chan2dev(chan), "invalid src maxburst value\n");
@@ -866,10 +931,12 @@ at_xdmac_interleaved_queue_desc(struct dma_chan *chan,
* ERRATA: Even if useless for memory transfers, the PERID has to not
* match the one of another channel. If not, it could lead to spurious
* flag status.
* For SAMA7G5x case, the SIF and DIF fields are no longer used.
* Thus, no need to have the SIF/DIF interfaces here.
* For SAMA5D4x and SAMA5D2x the SIF and DIF are already configured as
* zero.
*/
u32 chan_cc = AT_XDMAC_CC_PERID(0x3f)
| AT_XDMAC_CC_DIF(0)
| AT_XDMAC_CC_SIF(0)
u32 chan_cc = AT_XDMAC_CC_PERID(0x7f)
| AT_XDMAC_CC_MBSIZE_SIXTEEN
| AT_XDMAC_CC_TYPE_MEM_TRAN;
@@ -1048,12 +1115,14 @@ at_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
* ERRATA: Even if useless for memory transfers, the PERID has to not
* match the one of another channel. If not, it could lead to spurious
* flag status.
* For SAMA7G5x case, the SIF and DIF fields are no longer used.
* Thus, no need to have the SIF/DIF interfaces here.
* For SAMA5D4x and SAMA5D2x the SIF and DIF are already configured as
* zero.
*/
u32 chan_cc = AT_XDMAC_CC_PERID(0x3f)
u32 chan_cc = AT_XDMAC_CC_PERID(0x7f)
| AT_XDMAC_CC_DAM_INCREMENTED_AM
| AT_XDMAC_CC_SAM_INCREMENTED_AM
| AT_XDMAC_CC_DIF(0)
| AT_XDMAC_CC_SIF(0)
| AT_XDMAC_CC_MBSIZE_SIXTEEN
| AT_XDMAC_CC_TYPE_MEM_TRAN;
unsigned long irqflags;
@@ -1154,12 +1223,14 @@ static struct at_xdmac_desc *at_xdmac_memset_create_desc(struct dma_chan *chan,
* ERRATA: Even if useless for memory transfers, the PERID has to not
* match the one of another channel. If not, it could lead to spurious
* flag status.
* For SAMA7G5x case, the SIF and DIF fields are no longer used.
* Thus, no need to have the SIF/DIF interfaces here.
* For SAMA5D4x and SAMA5D2x the SIF and DIF are already configured as
* zero.
*/
u32 chan_cc = AT_XDMAC_CC_PERID(0x3f)
u32 chan_cc = AT_XDMAC_CC_PERID(0x7f)
| AT_XDMAC_CC_DAM_UBS_AM
| AT_XDMAC_CC_SAM_INCREMENTED_AM
| AT_XDMAC_CC_DIF(0)
| AT_XDMAC_CC_SIF(0)
| AT_XDMAC_CC_MBSIZE_SIXTEEN
| AT_XDMAC_CC_MEMSET_HW_MODE
| AT_XDMAC_CC_TYPE_MEM_TRAN;
@@ -1438,7 +1509,7 @@ at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
mask = AT_XDMAC_CC_TYPE | AT_XDMAC_CC_DSYNC;
value = AT_XDMAC_CC_TYPE_PER_TRAN | AT_XDMAC_CC_DSYNC_PER2MEM;
if ((desc->lld.mbr_cfg & mask) == value) {
at_xdmac_write(atxdmac, AT_XDMAC_GSWF, atchan->mask);
at_xdmac_write(atxdmac, atxdmac->layout->gswf, atchan->mask);
while (!(at_xdmac_chan_read(atchan, AT_XDMAC_CIS) & AT_XDMAC_CIS_FIS))
cpu_relax();
}
@@ -1496,7 +1567,7 @@ at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
* FIFO flush ensures that data are really written.
*/
if ((desc->lld.mbr_cfg & mask) == value) {
at_xdmac_write(atxdmac, AT_XDMAC_GSWF, atchan->mask);
at_xdmac_write(atxdmac, atxdmac->layout->gswf, atchan->mask);
while (!(at_xdmac_chan_read(atchan, AT_XDMAC_CIS) & AT_XDMAC_CIS_FIS))
cpu_relax();
}
@@ -1761,7 +1832,7 @@ static int at_xdmac_device_pause(struct dma_chan *chan)
return 0;
spin_lock_irqsave(&atchan->lock, flags);
at_xdmac_write(atxdmac, AT_XDMAC_GRWS, atchan->mask);
at_xdmac_write(atxdmac, atxdmac->layout->grws, atchan->mask);
while (at_xdmac_chan_read(atchan, AT_XDMAC_CC)
& (AT_XDMAC_CC_WRIP | AT_XDMAC_CC_RDIP))
cpu_relax();
@@ -1784,7 +1855,7 @@ static int at_xdmac_device_resume(struct dma_chan *chan)
return 0;
}
at_xdmac_write(atxdmac, AT_XDMAC_GRWR, atchan->mask);
at_xdmac_write(atxdmac, atxdmac->layout->grwr, atchan->mask);
clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status);
spin_unlock_irqrestore(&atchan->lock, flags);
@@ -1947,6 +2018,30 @@ static int atmel_xdmac_resume(struct device *dev)
}
#endif /* CONFIG_PM_SLEEP */
static void at_xdmac_axi_config(struct platform_device *pdev)
{
struct at_xdmac *atxdmac = (struct at_xdmac *)platform_get_drvdata(pdev);
bool dev_m2m = false;
u32 dma_requests;
if (!atxdmac->layout->axi_config)
return; /* Not supported */
if (!of_property_read_u32(pdev->dev.of_node, "dma-requests",
&dma_requests)) {
dev_info(&pdev->dev, "controller in mem2mem mode.\n");
dev_m2m = true;
}
if (dev_m2m) {
at_xdmac_write(atxdmac, AT_XDMAC_GCFG, AT_XDMAC_GCFG_M2M);
at_xdmac_write(atxdmac, AT_XDMAC_GWAC, AT_XDMAC_GWAC_M2M);
} else {
at_xdmac_write(atxdmac, AT_XDMAC_GCFG, AT_XDMAC_GCFG_P2M);
at_xdmac_write(atxdmac, AT_XDMAC_GWAC, AT_XDMAC_GWAC_P2M);
}
}
static int at_xdmac_probe(struct platform_device *pdev)
{
struct at_xdmac *atxdmac;
@@ -1986,6 +2081,10 @@ static int at_xdmac_probe(struct platform_device *pdev)
atxdmac->regs = base;
atxdmac->irq = irq;
atxdmac->layout = of_device_get_match_data(&pdev->dev);
if (!atxdmac->layout)
return -ENODEV;
atxdmac->clk = devm_clk_get(&pdev->dev, "dma_clk");
if (IS_ERR(atxdmac->clk)) {
dev_err(&pdev->dev, "can't get dma_clk\n");
@@ -2087,6 +2186,8 @@ static int at_xdmac_probe(struct platform_device *pdev)
dev_info(&pdev->dev, "%d channels, mapped at 0x%p\n",
nr_channels, atxdmac->regs);
at_xdmac_axi_config(pdev);
return 0;
err_dma_unregister:
@@ -2128,6 +2229,10 @@ static const struct dev_pm_ops atmel_xdmac_dev_pm_ops = {
static const struct of_device_id atmel_xdmac_dt_ids[] = {
{
.compatible = "atmel,sama5d4-dma",
.data = &at_xdmac_sama5d4_layout,
}, {
.compatible = "microchip,sama7g5-dma",
.data = &at_xdmac_sama7g5_layout,
}, {
/* sentinel */
}

2
drivers/dma/dma-jz4780.c
View File

@@ -1044,7 +1044,7 @@ static struct platform_driver jz4780_dma_driver = {
.remove = jz4780_dma_remove,
.driver = {
.name = "jz4780-dma",
.of_match_table = of_match_ptr(jz4780_dma_dt_match),
.of_match_table = jz4780_dma_dt_match,
},
};

13
drivers/dma/dmatest.c
View File

@@ -573,6 +573,7 @@ static int dmatest_func(void *data)
struct dmatest_params *params;
struct dma_chan *chan;
struct dma_device *dev;
struct device *dma_dev;
unsigned int error_count;
unsigned int failed_tests = 0;
unsigned int total_tests = 0;
@@ -606,6 +607,8 @@ static int dmatest_func(void *data)
params = &info->params;
chan = thread->chan;
dev = chan->device;
dma_dev = dmaengine_get_dma_device(chan);
src = &thread->src;
dst = &thread->dst;
if (thread->type == DMA_MEMCPY) {
@@ -730,7 +733,7 @@ static int dmatest_func(void *data)
filltime = ktime_add(filltime, diff);
}
um = dmaengine_get_unmap_data(dev->dev, src->cnt + dst->cnt,
um = dmaengine_get_unmap_data(dma_dev, src->cnt + dst->cnt,
GFP_KERNEL);
if (!um) {
failed_tests++;
@@ -745,10 +748,10 @@ static int dmatest_func(void *data)
struct page *pg = virt_to_page(buf);
unsigned long pg_off = offset_in_page(buf);
um->addr[i] = dma_map_page(dev->dev, pg, pg_off,
um->addr[i] = dma_map_page(dma_dev, pg, pg_off,
um->len, DMA_TO_DEVICE);
srcs[i] = um->addr[i] + src->off;
ret = dma_mapping_error(dev->dev, um->addr[i]);
ret = dma_mapping_error(dma_dev, um->addr[i]);
if (ret) {
result("src mapping error", total_tests,
src->off, dst->off, len, ret);
@@ -763,9 +766,9 @@ static int dmatest_func(void *data)
struct page *pg = virt_to_page(buf);
unsigned long pg_off = offset_in_page(buf);
dsts[i] = dma_map_page(dev->dev, pg, pg_off, um->len,
dsts[i] = dma_map_page(dma_dev, pg, pg_off, um->len,
DMA_BIDIRECTIONAL);
ret = dma_mapping_error(dev->dev, dsts[i]);
ret = dma_mapping_error(dma_dev, dsts[i]);
if (ret) {
result("dst mapping error", total_tests,
src->off, dst->off, len, ret);

2
drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
View File

@@ -992,7 +992,7 @@ static struct platform_driver dw_driver = {
.remove = dw_remove,
.driver = {
.name = KBUILD_MODNAME,
.of_match_table = of_match_ptr(dw_dma_of_id_table),
.of_match_table = dw_dma_of_id_table,
.pm = &dw_axi_dma_pm_ops,
},
};

6
drivers/dma/dw/core.c
View File

@@ -982,8 +982,11 @@ static int dwc_alloc_chan_resources(struct dma_chan *chan)
dev_vdbg(chan2dev(chan), "%s\n", __func__);
pm_runtime_get_sync(dw->dma.dev);
/* ASSERT: channel is idle */
if (dma_readl(dw, CH_EN) & dwc->mask) {
pm_runtime_put_sync_suspend(dw->dma.dev);
dev_dbg(chan2dev(chan), "DMA channel not idle?\n");
return -EIO;
}
@@ -1000,6 +1003,7 @@ static int dwc_alloc_chan_resources(struct dma_chan *chan)
* We need controller-specific data to set up slave transfers.
*/
if (chan->private && !dw_dma_filter(chan, chan->private)) {
pm_runtime_put_sync_suspend(dw->dma.dev);
dev_warn(chan2dev(chan), "Wrong controller-specific data\n");
return -EINVAL;
}
@@ -1043,6 +1047,8 @@ static void dwc_free_chan_resources(struct dma_chan *chan)
if (!dw->in_use)
do_dw_dma_off(dw);
pm_runtime_put_sync_suspend(dw->dma.dev);
dev_vdbg(chan2dev(chan), "%s: done\n", __func__);
}

5
drivers/dma/hisi_dma.c
View File

@@ -431,9 +431,8 @@ static irqreturn_t hisi_dma_irq(int irq, void *data)
struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
struct hisi_dma_desc *desc;
struct hisi_dma_cqe *cqe;
unsigned long flags;
spin_lock_irqsave(&chan->vc.lock, flags);
spin_lock(&chan->vc.lock);
desc = chan->desc;
cqe = chan->cq + chan->cq_head;
@@ -452,7 +451,7 @@ static irqreturn_t hisi_dma_irq(int irq, void *data)
chan->desc = NULL;
}
spin_unlock_irqrestore(&chan->vc.lock, flags);
spin_unlock(&chan->vc.lock);
return IRQ_HANDLED;
}

8
drivers/dma/idma64.c
View File

@@ -667,9 +667,7 @@ static int idma64_platform_remove(struct platform_device *pdev)
return idma64_remove(chip);
}
#ifdef CONFIG_PM_SLEEP
static int idma64_pm_suspend(struct device *dev)
static int __maybe_unused idma64_pm_suspend(struct device *dev)
{
struct idma64_chip *chip = dev_get_drvdata(dev);
@@ -677,7 +675,7 @@ static int idma64_pm_suspend(struct device *dev)
return 0;
}
static int idma64_pm_resume(struct device *dev)
static int __maybe_unused idma64_pm_resume(struct device *dev)
{
struct idma64_chip *chip = dev_get_drvdata(dev);
@@ -685,8 +683,6 @@ static int idma64_pm_resume(struct device *dev)
return 0;
}
#endif /* CONFIG_PM_SLEEP */
static const struct dev_pm_ops idma64_dev_pm_ops = {
SET_SYSTEM_SLEEP_PM_OPS(idma64_pm_suspend, idma64_pm_resume)
};

50
drivers/dma/idxd/cdev.c
View File

@@ -11,6 +11,7 @@
#include <linux/cdev.h>
#include <linux/fs.h>
#include <linux/poll.h>
#include <linux/iommu.h>
#include <uapi/linux/idxd.h>
#include "registers.h"
#include "idxd.h"
@@ -27,12 +28,15 @@ struct idxd_cdev_context {
*/
static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = {
{ .name = "dsa" },
{ .name = "iax" }
};
struct idxd_user_context {
struct idxd_wq *wq;
struct task_struct *task;
unsigned int pasid;
unsigned int flags;
struct iommu_sva *sva;
};
enum idxd_cdev_cleanup {
@@ -75,6 +79,8 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp)
struct idxd_wq *wq;
struct device *dev;
int rc = 0;
struct iommu_sva *sva;
unsigned int pasid;
wq = inode_wq(inode);
idxd = wq->idxd;
@@ -95,6 +101,34 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp)
ctx->wq = wq;
filp->private_data = ctx;
if (device_pasid_enabled(idxd)) {
sva = iommu_sva_bind_device(dev, current->mm, NULL);
if (IS_ERR(sva)) {
rc = PTR_ERR(sva);
dev_err(dev, "pasid allocation failed: %d\n", rc);
goto failed;
}
pasid = iommu_sva_get_pasid(sva);
if (pasid == IOMMU_PASID_INVALID) {
iommu_sva_unbind_device(sva);
goto failed;
}
ctx->sva = sva;
ctx->pasid = pasid;
if (wq_dedicated(wq)) {
rc = idxd_wq_set_pasid(wq, pasid);
if (rc < 0) {
iommu_sva_unbind_device(sva);
dev_err(dev, "wq set pasid failed: %d\n", rc);
goto failed;
}
}
}
idxd_wq_get(wq);
mutex_unlock(&wq->wq_lock);
return 0;
@@ -111,13 +145,27 @@ static int idxd_cdev_release(struct inode *node, struct file *filep)
struct idxd_wq *wq = ctx->wq;
struct idxd_device *idxd = wq->idxd;
struct device *dev = &idxd->pdev->dev;
int rc;
dev_dbg(dev, "%s called\n", __func__);
filep->private_data = NULL;
/* Wait for in-flight operations to complete. */
idxd_wq_drain(wq);
if (wq_shared(wq)) {
idxd_device_drain_pasid(idxd, ctx->pasid);
} else {
if (device_pasid_enabled(idxd)) {
/* The wq disable in the disable pasid function will drain the wq */
rc = idxd_wq_disable_pasid(wq);
if (rc < 0)
dev_err(dev, "wq disable pasid failed.\n");
} else {
idxd_wq_drain(wq);
}
}
if (ctx->sva)
iommu_sva_unbind_device(ctx->sva);
kfree(ctx);
mutex_lock(&wq->wq_lock);
idxd_wq_put(wq);

143
drivers/dma/idxd/device.c
View File

@@ -131,6 +131,8 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq)
struct idxd_device *idxd = wq->idxd;
struct device *dev = &idxd->pdev->dev;
int rc, num_descs, i;
int align;
u64 tmp;
if (wq->type != IDXD_WQT_KERNEL)
return 0;
@@ -142,14 +144,27 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq)
if (rc < 0)
return rc;
wq->compls_size = num_descs * sizeof(struct dsa_completion_record);
wq->compls = dma_alloc_coherent(dev, wq->compls_size,
&wq->compls_addr, GFP_KERNEL);
if (!wq->compls) {
if (idxd->type == IDXD_TYPE_DSA)
align = 32;
else if (idxd->type == IDXD_TYPE_IAX)
align = 64;
else
return -ENODEV;
wq->compls_size = num_descs * idxd->compl_size + align;
wq->compls_raw = dma_alloc_coherent(dev, wq->compls_size,
&wq->compls_addr_raw, GFP_KERNEL);
if (!wq->compls_raw) {
rc = -ENOMEM;
goto fail_alloc_compls;
}
/* Adjust alignment */
wq->compls_addr = (wq->compls_addr_raw + (align - 1)) & ~(align - 1);
tmp = (u64)wq->compls_raw;
tmp = (tmp + (align - 1)) & ~(align - 1);
wq->compls = (struct dsa_completion_record *)tmp;
rc = alloc_descs(wq, num_descs);
if (rc < 0)
goto fail_alloc_descs;
@@ -163,9 +178,11 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq)
struct idxd_desc *desc = wq->descs[i];
desc->hw = wq->hw_descs[i];
desc->completion = &wq->compls[i];
desc->compl_dma = wq->compls_addr +
sizeof(struct dsa_completion_record) * i;
if (idxd->type == IDXD_TYPE_DSA)
desc->completion = &wq->compls[i];
else if (idxd->type == IDXD_TYPE_IAX)
desc->iax_completion = &wq->iax_compls[i];
desc->compl_dma = wq->compls_addr + idxd->compl_size * i;
desc->id = i;
desc->wq = wq;
desc->cpu = -1;
@@ -178,7 +195,8 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq)
fail_sbitmap_init:
free_descs(wq);
fail_alloc_descs:
dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr);
dma_free_coherent(dev, wq->compls_size, wq->compls_raw,
wq->compls_addr_raw);
fail_alloc_compls:
free_hw_descs(wq);
return rc;
@@ -193,7 +211,8 @@ void idxd_wq_free_resources(struct idxd_wq *wq)
free_hw_descs(wq);
free_descs(wq);
dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr);
dma_free_coherent(dev, wq->compls_size, wq->compls_raw,
wq->compls_addr_raw);
sbitmap_queue_free(&wq->sbq);
}
@@ -273,10 +292,9 @@ int idxd_wq_map_portal(struct idxd_wq *wq)
start = pci_resource_start(pdev, IDXD_WQ_BAR);
start += idxd_get_wq_portal_full_offset(wq->id, IDXD_PORTAL_LIMITED);
wq->dportal = devm_ioremap(dev, start, IDXD_PORTAL_SIZE);
if (!wq->dportal)
wq->portal = devm_ioremap(dev, start, IDXD_PORTAL_SIZE);
if (!wq->portal)
return -ENOMEM;
dev_dbg(dev, "wq %d portal mapped at %p\n", wq->id, wq->dportal);
return 0;
}
@@ -285,7 +303,61 @@ void idxd_wq_unmap_portal(struct idxd_wq *wq)
{
struct device *dev = &wq->idxd->pdev->dev;
devm_iounmap(dev, wq->dportal);
devm_iounmap(dev, wq->portal);
}
int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid)
{
struct idxd_device *idxd = wq->idxd;
int rc;
union wqcfg wqcfg;
unsigned int offset;
unsigned long flags;
rc = idxd_wq_disable(wq);
if (rc < 0)
return rc;
offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX);
spin_lock_irqsave(&idxd->dev_lock, flags);
wqcfg.bits[WQCFG_PASID_IDX] = ioread32(idxd->reg_base + offset);
wqcfg.pasid_en = 1;
wqcfg.pasid = pasid;
iowrite32(wqcfg.bits[WQCFG_PASID_IDX], idxd->reg_base + offset);
spin_unlock_irqrestore(&idxd->dev_lock, flags);
rc = idxd_wq_enable(wq);
if (rc < 0)
return rc;
return 0;
}
int idxd_wq_disable_pasid(struct idxd_wq *wq)
{
struct idxd_device *idxd = wq->idxd;
int rc;
union wqcfg wqcfg;
unsigned int offset;
unsigned long flags;
rc = idxd_wq_disable(wq);
if (rc < 0)
return rc;
offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX);
spin_lock_irqsave(&idxd->dev_lock, flags);
wqcfg.bits[WQCFG_PASID_IDX] = ioread32(idxd->reg_base + offset);
wqcfg.pasid_en = 0;
wqcfg.pasid = 0;
iowrite32(wqcfg.bits[WQCFG_PASID_IDX], idxd->reg_base + offset);
spin_unlock_irqrestore(&idxd->dev_lock, flags);
rc = idxd_wq_enable(wq);
if (rc < 0)
return rc;
return 0;
}
void idxd_wq_disable_cleanup(struct idxd_wq *wq)
@@ -301,6 +373,7 @@ void idxd_wq_disable_cleanup(struct idxd_wq *wq)
wq->group = NULL;
wq->threshold = 0;
wq->priority = 0;
wq->ats_dis = 0;
clear_bit(WQ_FLAG_DEDICATED, &wq->flags);
memset(wq->name, 0, WQ_NAME_SIZE);
@@ -468,6 +541,17 @@ void idxd_device_reset(struct idxd_device *idxd)
spin_unlock_irqrestore(&idxd->dev_lock, flags);
}
void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid)
{
struct device *dev = &idxd->pdev->dev;
u32 operand;
operand = pasid;
dev_dbg(dev, "cmd: %u operand: %#x\n", IDXD_CMD_DRAIN_PASID, operand);
idxd_cmd_exec(idxd, IDXD_CMD_DRAIN_PASID, operand, NULL);
dev_dbg(dev, "pasid %d drained\n", pasid);
}
/* Device configuration bits */
static void idxd_group_config_write(struct idxd_group *group)
{
@@ -479,24 +563,22 @@ static void idxd_group_config_write(struct idxd_group *group)
dev_dbg(dev, "Writing group %d cfg registers\n", group->id);
/* setup GRPWQCFG */
for (i = 0; i < 4; i++) {
grpcfg_offset = idxd->grpcfg_offset +
group->id * 64 + i * sizeof(u64);
iowrite64(group->grpcfg.wqs[i],
idxd->reg_base + grpcfg_offset);
for (i = 0; i < GRPWQCFG_STRIDES; i++) {
grpcfg_offset = GRPWQCFG_OFFSET(idxd, group->id, i);
iowrite64(group->grpcfg.wqs[i], idxd->reg_base + grpcfg_offset);
dev_dbg(dev, "GRPCFG wq[%d:%d: %#x]: %#llx\n",
group->id, i, grpcfg_offset,
ioread64(idxd->reg_base + grpcfg_offset));
}
/* setup GRPENGCFG */
grpcfg_offset = idxd->grpcfg_offset + group->id * 64 + 32;
grpcfg_offset = GRPENGCFG_OFFSET(idxd, group->id);
iowrite64(group->grpcfg.engines, idxd->reg_base + grpcfg_offset);
dev_dbg(dev, "GRPCFG engs[%d: %#x]: %#llx\n", group->id,
grpcfg_offset, ioread64(idxd->reg_base + grpcfg_offset));
/* setup GRPFLAGS */
grpcfg_offset = idxd->grpcfg_offset + group->id * 64 + 40;
grpcfg_offset = GRPFLGCFG_OFFSET(idxd, group->id);
iowrite32(group->grpcfg.flags.bits, idxd->reg_base + grpcfg_offset);
dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#x\n",
group->id, grpcfg_offset,
@@ -554,9 +636,24 @@ static int idxd_wq_config_write(struct idxd_wq *wq)
/* byte 8-11 */
wq->wqcfg->priv = !!(wq->type == IDXD_WQT_KERNEL);
wq->wqcfg->mode = 1;
if (wq_dedicated(wq))
wq->wqcfg->mode = 1;
if (device_pasid_enabled(idxd)) {
wq->wqcfg->pasid_en = 1;
if (wq->type == IDXD_WQT_KERNEL && wq_dedicated(wq))
wq->wqcfg->pasid = idxd->pasid;
}
wq->wqcfg->priority = wq->priority;
if (idxd->hw.gen_cap.block_on_fault &&
test_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags))
wq->wqcfg->bof = 1;
if (idxd->hw.wq_cap.wq_ats_support)
wq->wqcfg->wq_ats_disable = wq->ats_dis;
/* bytes 12-15 */
wq->wqcfg->max_xfer_shift = ilog2(wq->max_xfer_bytes);
wq->wqcfg->max_batch_shift = ilog2(wq->max_batch_size);
@@ -664,8 +761,8 @@ static int idxd_wqs_setup(struct idxd_device *idxd)
if (!wq->size)
continue;
if (!wq_dedicated(wq)) {
dev_warn(dev, "No shared workqueue support.\n");
if (wq_shared(wq) && !device_swq_supported(idxd)) {
dev_warn(dev, "No shared wq support but configured.\n");
return -EINVAL;
}

9
drivers/dma/idxd/dma.c
View File

@@ -61,8 +61,6 @@ static inline void idxd_prep_desc_common(struct idxd_wq *wq,
u64 addr_f1, u64 addr_f2, u64 len,
u64 compl, u32 flags)
{
struct idxd_device *idxd = wq->idxd;
hw->flags = flags;
hw->opcode = opcode;
hw->src_addr = addr_f1;
@@ -70,13 +68,6 @@ static inline void idxd_prep_desc_common(struct idxd_wq *wq,
hw->xfer_size = len;
hw->priv = !!(wq->type == IDXD_WQT_KERNEL);
hw->completion_addr = compl;
/*
* Descriptor completion vectors are 1-8 for MSIX. We will round
* robin through the 8 vectors.
*/
wq->vec_ptr = (wq->vec_ptr % idxd->num_wq_irqs) + 1;
hw->int_handle = wq->vec_ptr;
}
static struct dma_async_tx_descriptor *

58
drivers/dma/idxd/idxd.h
View File

@@ -20,7 +20,8 @@ extern struct kmem_cache *idxd_desc_pool;
enum idxd_type {
IDXD_TYPE_UNKNOWN = -1,
IDXD_TYPE_DSA = 0,
IDXD_TYPE_MAX
IDXD_TYPE_IAX,
IDXD_TYPE_MAX,
};
#define IDXD_NAME_SIZE 128
@@ -34,6 +35,11 @@ struct idxd_irq_entry {
int id;
struct llist_head pending_llist;
struct list_head work_list;
/*
* Lock to protect access between irq thread process descriptor
* and irq thread processing error descriptor.
*/
spinlock_t list_lock;
};
struct idxd_group {
@@ -59,6 +65,7 @@ enum idxd_wq_state {
enum idxd_wq_flag {
WQ_FLAG_DEDICATED = 0,
WQ_FLAG_BLOCK_ON_FAULT,
};
enum idxd_wq_type {
@@ -86,10 +93,11 @@ enum idxd_op_type {
enum idxd_complete_type {
IDXD_COMPLETE_NORMAL = 0,
IDXD_COMPLETE_ABORT,
IDXD_COMPLETE_DEV_FAIL,
};
struct idxd_wq {
void __iomem *dportal;
void __iomem *portal;
struct device conf_dev;
struct idxd_cdev idxd_cdev;
struct idxd_device *idxd;
@@ -107,8 +115,13 @@ struct idxd_wq {
u32 vec_ptr; /* interrupt steering */
struct dsa_hw_desc **hw_descs;
int num_descs;
struct dsa_completion_record *compls;
union {
struct dsa_completion_record *compls;
struct iax_completion_record *iax_compls;
};
void *compls_raw;
dma_addr_t compls_addr;
dma_addr_t compls_addr_raw;
int compls_size;
struct idxd_desc **descs;
struct sbitmap_queue sbq;
@@ -116,6 +129,7 @@ struct idxd_wq {
char name[WQ_NAME_SIZE + 1];
u64 max_xfer_bytes;
u32 max_batch_size;
bool ats_dis;
};
struct idxd_engine {
@@ -145,6 +159,7 @@ enum idxd_device_state {
enum idxd_device_flag {
IDXD_FLAG_CONFIGURABLE = 0,
IDXD_FLAG_CMD_RUNNING,
IDXD_FLAG_PASID_ENABLED,
};
struct idxd_device {
@@ -167,6 +182,9 @@ struct idxd_device {
struct idxd_wq *wqs;
struct idxd_engine *engines;
struct iommu_sva *sva;
unsigned int pasid;
int num_groups;
u32 msix_perm_offset;
@@ -184,6 +202,7 @@ struct idxd_device {
int token_limit;
int nr_tokens; /* non-reserved tokens */
unsigned int wqcfg_size;
int compl_size;
union sw_err_reg sw_err;
wait_queue_head_t cmd_waitq;
@@ -198,9 +217,15 @@ struct idxd_device {
/* IDXD software descriptor */
struct idxd_desc {
struct dsa_hw_desc *hw;
union {
struct dsa_hw_desc *hw;
struct iax_hw_desc *iax_hw;
};
dma_addr_t desc_dma;
struct dsa_completion_record *completion;
union {
struct dsa_completion_record *completion;
struct iax_completion_record *iax_completion;
};
dma_addr_t compl_dma;
struct dma_async_tx_descriptor txd;
struct llist_node llnode;
@@ -214,12 +239,30 @@ struct idxd_desc {
#define confdev_to_wq(dev) container_of(dev, struct idxd_wq, conf_dev)
extern struct bus_type dsa_bus_type;
extern struct bus_type iax_bus_type;
extern bool support_enqcmd;
static inline bool wq_dedicated(struct idxd_wq *wq)
{
return test_bit(WQ_FLAG_DEDICATED, &wq->flags);
}
static inline bool wq_shared(struct idxd_wq *wq)
{
return !test_bit(WQ_FLAG_DEDICATED, &wq->flags);
}
static inline bool device_pasid_enabled(struct idxd_device *idxd)
{
return test_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
}
static inline bool device_swq_supported(struct idxd_device *idxd)
{
return (support_enqcmd && device_pasid_enabled(idxd));
}
enum idxd_portal_prot {
IDXD_PORTAL_UNLIMITED = 0,
IDXD_PORTAL_LIMITED,
@@ -242,6 +285,8 @@ static inline void idxd_set_type(struct idxd_device *idxd)
if (pdev->device == PCI_DEVICE_ID_INTEL_DSA_SPR0)
idxd->type = IDXD_TYPE_DSA;
else if (pdev->device == PCI_DEVICE_ID_INTEL_IAX_SPR0)
idxd->type = IDXD_TYPE_IAX;
else
idxd->type = IDXD_TYPE_UNKNOWN;
}
@@ -288,6 +333,7 @@ void idxd_device_reset(struct idxd_device *idxd);
void idxd_device_cleanup(struct idxd_device *idxd);
int idxd_device_config(struct idxd_device *idxd);
void idxd_device_wqs_clear_state(struct idxd_device *idxd);
void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid);
/* work queue control */
int idxd_wq_alloc_resources(struct idxd_wq *wq);
@@ -298,6 +344,8 @@ void idxd_wq_drain(struct idxd_wq *wq);
int idxd_wq_map_portal(struct idxd_wq *wq);
void idxd_wq_unmap_portal(struct idxd_wq *wq);
void idxd_wq_disable_cleanup(struct idxd_wq *wq);
int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid);
int idxd_wq_disable_pasid(struct idxd_wq *wq);
/* submission */
int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc);

123
drivers/dma/idxd/init.c
View File

@@ -14,6 +14,8 @@
#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/device.h>
#include <linux/idr.h>
#include <linux/intel-svm.h>
#include <linux/iommu.h>
#include <uapi/linux/idxd.h>
#include <linux/dmaengine.h>
#include "../dmaengine.h"
@@ -26,18 +28,24 @@ MODULE_AUTHOR("Intel Corporation");
#define DRV_NAME "idxd"
bool support_enqcmd;
static struct idr idxd_idrs[IDXD_TYPE_MAX];
static struct mutex idxd_idr_lock;
static struct pci_device_id idxd_pci_tbl[] = {
/* DSA ver 1.0 platforms */
{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_DSA_SPR0) },
/* IAX ver 1.0 platforms */
{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IAX_SPR0) },
{ 0, }
};
MODULE_DEVICE_TABLE(pci, idxd_pci_tbl);
static char *idxd_name[] = {
"dsa",
"iax"
};
const char *idxd_get_dev_name(struct idxd_device *idxd)
@@ -53,6 +61,7 @@ static int idxd_setup_interrupts(struct idxd_device *idxd)
struct idxd_irq_entry *irq_entry;
int i, msixcnt;
int rc = 0;
union msix_perm mperm;
msixcnt = pci_msix_vec_count(pdev);
if (msixcnt < 0) {
@@ -92,6 +101,7 @@ static int idxd_setup_interrupts(struct idxd_device *idxd)
for (i = 0; i < msixcnt; i++) {
idxd->irq_entries[i].id = i;
idxd->irq_entries[i].idxd = idxd;
spin_lock_init(&idxd->irq_entries[i].list_lock);
}
msix = &idxd->msix_entries[0];
@@ -131,6 +141,13 @@ static int idxd_setup_interrupts(struct idxd_device *idxd)
idxd_unmask_error_interrupts(idxd);
/* Setup MSIX permission table */
mperm.bits = 0;
mperm.pasid = idxd->pasid;
mperm.pasid_en = device_pasid_enabled(idxd);
for (i = 1; i < msixcnt; i++)
iowrite32(mperm.bits, idxd->reg_base + idxd->msix_perm_offset + i * 8);
return 0;
err_no_irq:
@@ -201,17 +218,14 @@ static void idxd_read_table_offsets(struct idxd_device *idxd)
struct device *dev = &idxd->pdev->dev;
offsets.bits[0] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET);
offsets.bits[1] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET
+ sizeof(u64));
idxd->grpcfg_offset = offsets.grpcfg * 0x100;
offsets.bits[1] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET + sizeof(u64));
idxd->grpcfg_offset = offsets.grpcfg * IDXD_TABLE_MULT;
dev_dbg(dev, "IDXD Group Config Offset: %#x\n", idxd->grpcfg_offset);
idxd->wqcfg_offset = offsets.wqcfg * 0x100;
dev_dbg(dev, "IDXD Work Queue Config Offset: %#x\n",
idxd->wqcfg_offset);
idxd->msix_perm_offset = offsets.msix_perm * 0x100;
dev_dbg(dev, "IDXD MSIX Permission Offset: %#x\n",
idxd->msix_perm_offset);
idxd->perfmon_offset = offsets.perfmon * 0x100;
idxd->wqcfg_offset = offsets.wqcfg * IDXD_TABLE_MULT;
dev_dbg(dev, "IDXD Work Queue Config Offset: %#x\n", idxd->wqcfg_offset);
idxd->msix_perm_offset = offsets.msix_perm * IDXD_TABLE_MULT;
dev_dbg(dev, "IDXD MSIX Permission Offset: %#x\n", idxd->msix_perm_offset);
idxd->perfmon_offset = offsets.perfmon * IDXD_TABLE_MULT;
dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset);
}
@@ -265,8 +279,7 @@ static void idxd_read_caps(struct idxd_device *idxd)
}
}
static struct idxd_device *idxd_alloc(struct pci_dev *pdev,
void __iomem * const *iomap)
static struct idxd_device *idxd_alloc(struct pci_dev *pdev)
{
struct device *dev = &pdev->dev;
struct idxd_device *idxd;
@@ -276,12 +289,45 @@ static struct idxd_device *idxd_alloc(struct pci_dev *pdev,
return NULL;
idxd->pdev = pdev;
idxd->reg_base = iomap[IDXD_MMIO_BAR];
spin_lock_init(&idxd->dev_lock);
return idxd;
}
static int idxd_enable_system_pasid(struct idxd_device *idxd)
{
int flags;
unsigned int pasid;
struct iommu_sva *sva;
flags = SVM_FLAG_SUPERVISOR_MODE;
sva = iommu_sva_bind_device(&idxd->pdev->dev, NULL, &flags);
if (IS_ERR(sva)) {
dev_warn(&idxd->pdev->dev,
"iommu sva bind failed: %ld\n", PTR_ERR(sva));
return PTR_ERR(sva);
}
pasid = iommu_sva_get_pasid(sva);
if (pasid == IOMMU_PASID_INVALID) {
iommu_sva_unbind_device(sva);
return -ENODEV;
}
idxd->sva = sva;
idxd->pasid = pasid;
dev_dbg(&idxd->pdev->dev, "system pasid: %u\n", pasid);
return 0;
}
static void idxd_disable_system_pasid(struct idxd_device *idxd)
{
iommu_sva_unbind_device(idxd->sva);
idxd->sva = NULL;
}
static int idxd_probe(struct idxd_device *idxd)
{
struct pci_dev *pdev = idxd->pdev;
@@ -292,6 +338,14 @@ static int idxd_probe(struct idxd_device *idxd)
idxd_device_init_reset(idxd);
dev_dbg(dev, "IDXD reset complete\n");
if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM)) {
rc = idxd_enable_system_pasid(idxd);
if (rc < 0)
dev_warn(dev, "Failed to enable PASID. No SVA support: %d\n", rc);
else
set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
}
idxd_read_caps(idxd);
idxd_read_table_offsets(idxd);
@@ -322,29 +376,37 @@ static int idxd_probe(struct idxd_device *idxd)
idxd_mask_error_interrupts(idxd);
idxd_mask_msix_vectors(idxd);
err_setup:
if (device_pasid_enabled(idxd))
idxd_disable_system_pasid(idxd);
return rc;
}
static void idxd_type_init(struct idxd_device *idxd)
{
if (idxd->type == IDXD_TYPE_DSA)
idxd->compl_size = sizeof(struct dsa_completion_record);
else if (idxd->type == IDXD_TYPE_IAX)
idxd->compl_size = sizeof(struct iax_completion_record);
}
static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
void __iomem * const *iomap;
struct device *dev = &pdev->dev;
struct idxd_device *idxd;
int rc;
unsigned int mask;
rc = pcim_enable_device(pdev);
if (rc)
return rc;
dev_dbg(dev, "Mapping BARs\n");
mask = (1 << IDXD_MMIO_BAR);
rc = pcim_iomap_regions(pdev, mask, DRV_NAME);
if (rc)
return rc;
dev_dbg(dev, "Alloc IDXD context\n");
idxd = idxd_alloc(pdev);
if (!idxd)
return -ENOMEM;
iomap = pcim_iomap_table(pdev);
if (!iomap)
dev_dbg(dev, "Mapping BARs\n");
idxd->reg_base = pcim_iomap(pdev, IDXD_MMIO_BAR, 0);
if (!idxd->reg_base)
return -ENOMEM;
dev_dbg(dev, "Set DMA masks\n");
@@ -360,13 +422,10 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
if (rc)
return rc;
dev_dbg(dev, "Alloc IDXD context\n");
idxd = idxd_alloc(pdev, iomap);
if (!idxd)
return -ENOMEM;
idxd_set_type(idxd);
idxd_type_init(idxd);
dev_dbg(dev, "Set PCI master\n");
pci_set_master(pdev);
pci_set_drvdata(pdev, idxd);
@@ -452,6 +511,8 @@ static void idxd_remove(struct pci_dev *pdev)
dev_dbg(&pdev->dev, "%s called\n", __func__);
idxd_cleanup_sysfs(idxd);
idxd_shutdown(pdev);
if (device_pasid_enabled(idxd))
idxd_disable_system_pasid(idxd);
mutex_lock(&idxd_idr_lock);
idr_remove(&idxd_idrs[idxd->type], idxd->id);
mutex_unlock(&idxd_idr_lock);
@@ -470,7 +531,7 @@ static int __init idxd_init_module(void)
int err, i;
/*
* If the CPU does not support write512, there's no point in
* If the CPU does not support MOVDIR64B or ENQCMDS, there's no point in
* enumerating the device. We can not utilize it.
*/
if (!boot_cpu_has(X86_FEATURE_MOVDIR64B)) {
@@ -478,8 +539,10 @@ static int __init idxd_init_module(void)
return -ENODEV;
}
pr_info("%s: Intel(R) Accelerator Devices Driver %s\n",
DRV_NAME, IDXD_DRIVER_VERSION);
if (!boot_cpu_has(X86_FEATURE_ENQCMD))
pr_warn("Platform does not have ENQCMD(S) support.\n");
else
support_enqcmd = true;
mutex_init(&idxd_idr_lock);
for (i = 0; i < IDXD_TYPE_MAX; i++)

146
drivers/dma/idxd/irq.c
View File

@@ -11,6 +11,24 @@
#include "idxd.h"
#include "registers.h"
enum irq_work_type {
IRQ_WORK_NORMAL = 0,
IRQ_WORK_PROCESS_FAULT,
};
struct idxd_fault {
struct work_struct work;
u64 addr;
struct idxd_device *idxd;
};
static int irq_process_work_list(struct idxd_irq_entry *irq_entry,
enum irq_work_type wtype,
int *processed, u64 data);
static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry,
enum irq_work_type wtype,
int *processed, u64 data);
static void idxd_device_reinit(struct work_struct *work)
{
struct idxd_device *idxd = container_of(work, struct idxd_device, work);
@@ -44,6 +62,46 @@ static void idxd_device_reinit(struct work_struct *work)
idxd_device_wqs_clear_state(idxd);
}
static void idxd_device_fault_work(struct work_struct *work)
{
struct idxd_fault *fault = container_of(work, struct idxd_fault, work);
struct idxd_irq_entry *ie;
int i;
int processed;
int irqcnt = fault->idxd->num_wq_irqs + 1;
for (i = 1; i < irqcnt; i++) {
ie = &fault->idxd->irq_entries[i];
irq_process_work_list(ie, IRQ_WORK_PROCESS_FAULT,
&processed, fault->addr);
if (processed)
break;
irq_process_pending_llist(ie, IRQ_WORK_PROCESS_FAULT,
&processed, fault->addr);
if (processed)
break;
}
kfree(fault);
}
static int idxd_device_schedule_fault_process(struct idxd_device *idxd,
u64 fault_addr)
{
struct idxd_fault *fault;
fault = kmalloc(sizeof(*fault), GFP_ATOMIC);
if (!fault)
return -ENOMEM;
fault->addr = fault_addr;
fault->idxd = idxd;
INIT_WORK(&fault->work, idxd_device_fault_work);
queue_work(idxd->wq, &fault->work);
return 0;
}
irqreturn_t idxd_irq_handler(int vec, void *data)
{
struct idxd_irq_entry *irq_entry = data;
@@ -125,6 +183,15 @@ irqreturn_t idxd_misc_thread(int vec, void *data)
if (!err)
goto out;
/*
* This case should rarely happen and typically is due to software
* programming error by the driver.
*/
if (idxd->sw_err.valid &&
idxd->sw_err.desc_valid &&
idxd->sw_err.fault_addr)
idxd_device_schedule_fault_process(idxd, idxd->sw_err.fault_addr);
gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET);
if (gensts.state == IDXD_DEVICE_STATE_HALT) {
idxd->state = IDXD_DEV_HALTED;
@@ -152,57 +219,110 @@ irqreturn_t idxd_misc_thread(int vec, void *data)
return IRQ_HANDLED;
}
static bool process_fault(struct idxd_desc *desc, u64 fault_addr)
{
/*
* Completion address can be bad as well. Check fault address match for descriptor
* and completion address.
*/
if ((u64)desc->hw == fault_addr ||
(u64)desc->completion == fault_addr) {
idxd_dma_complete_txd(desc, IDXD_COMPLETE_DEV_FAIL);
return true;
}
return false;
}
static bool complete_desc(struct idxd_desc *desc)
{
if (desc->completion->status) {
idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL);
return true;
}
return false;
}
static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry,
int *processed)
enum irq_work_type wtype,
int *processed, u64 data)
{
struct idxd_desc *desc, *t;
struct llist_node *head;
int queued = 0;
bool completed = false;
unsigned long flags;
*processed = 0;
head = llist_del_all(&irq_entry->pending_llist);
if (!head)
return 0;
goto out;
llist_for_each_entry_safe(desc, t, head, llnode) {
if (desc->completion->status) {
idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL);
if (wtype == IRQ_WORK_NORMAL)
completed = complete_desc(desc);
else if (wtype == IRQ_WORK_PROCESS_FAULT)
completed = process_fault(desc, data);
if (completed) {
idxd_free_desc(desc->wq, desc);
(*processed)++;
if (wtype == IRQ_WORK_PROCESS_FAULT)
break;
} else {
list_add_tail(&desc->list, &irq_entry->work_list);
spin_lock_irqsave(&irq_entry->list_lock, flags);
list_add_tail(&desc->list,
&irq_entry->work_list);
spin_unlock_irqrestore(&irq_entry->list_lock, flags);
queued++;
}
}
out:
return queued;
}
static int irq_process_work_list(struct idxd_irq_entry *irq_entry,
int *processed)
enum irq_work_type wtype,
int *processed, u64 data)
{
struct list_head *node, *next;
int queued = 0;
bool completed = false;
unsigned long flags;
*processed = 0;
spin_lock_irqsave(&irq_entry->list_lock, flags);
if (list_empty(&irq_entry->work_list))
return 0;
goto out;
list_for_each_safe(node, next, &irq_entry->work_list) {
struct idxd_desc *desc =
container_of(node, struct idxd_desc, list);
if (desc->completion->status) {
spin_unlock_irqrestore(&irq_entry->list_lock, flags);
if (wtype == IRQ_WORK_NORMAL)
completed = complete_desc(desc);
else if (wtype == IRQ_WORK_PROCESS_FAULT)
completed = process_fault(desc, data);
if (completed) {
spin_lock_irqsave(&irq_entry->list_lock, flags);
list_del(&desc->list);
/* process and callback */
idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL);
spin_unlock_irqrestore(&irq_entry->list_lock, flags);
idxd_free_desc(desc->wq, desc);
(*processed)++;
if (wtype == IRQ_WORK_PROCESS_FAULT)
return queued;
} else {
queued++;
}
spin_lock_irqsave(&irq_entry->list_lock, flags);
}
out:
spin_unlock_irqrestore(&irq_entry->list_lock, flags);
return queued;
}
@@ -230,12 +350,14 @@ static int idxd_desc_process(struct idxd_irq_entry *irq_entry)
* 5. Repeat until no more descriptors.
*/
do {
rc = irq_process_work_list(irq_entry, &processed);
rc = irq_process_work_list(irq_entry, IRQ_WORK_NORMAL,
&processed, 0);
total += processed;
if (rc != 0)
continue;
rc = irq_process_pending_llist(irq_entry, &processed);
rc = irq_process_pending_llist(irq_entry, IRQ_WORK_NORMAL,
&processed, 0);
total += processed;
} while (rc != 0);

28
drivers/dma/idxd/registers.h
View File

@@ -5,6 +5,7 @@
/* PCI Config */
#define PCI_DEVICE_ID_INTEL_DSA_SPR0 0x0b25
#define PCI_DEVICE_ID_INTEL_IAX_SPR0 0x0cfe
#define IDXD_MMIO_BAR 0
#define IDXD_WQ_BAR 2
@@ -47,7 +48,7 @@ union wq_cap_reg {
u64 rsvd:20;
u64 shared_mode:1;
u64 dedicated_mode:1;
u64 rsvd2:1;
u64 wq_ats_support:1;
u64 priority:1;
u64 occupancy:1;
u64 occupancy_int:1;
@@ -102,6 +103,8 @@ union offsets_reg {
u64 bits[2];
} __packed;
#define IDXD_TABLE_MULT 0x100
#define IDXD_GENCFG_OFFSET 0x80
union gencfg_reg {
struct {
@@ -301,7 +304,8 @@ union wqcfg {
/* bytes 8-11 */
u32 mode:1; /* shared or dedicated */
u32 bof:1; /* block on fault */
u32 rsvd2:2;
u32 wq_ats_disable:1;
u32 rsvd2:1;
u32 priority:4;
u32 pasid:20;
u32 pasid_en:1;
@@ -336,6 +340,8 @@ union wqcfg {
u32 bits[8];
} __packed;
#define WQCFG_PASID_IDX 2
/*
* This macro calculates the offset into the WQCFG register
* idxd - struct idxd *
@@ -354,4 +360,22 @@ union wqcfg {
#define WQCFG_STRIDES(_idxd_dev) ((_idxd_dev)->wqcfg_size / sizeof(u32))
#define GRPCFG_SIZE 64
#define GRPWQCFG_STRIDES 4
/*
* This macro calculates the offset into the GRPCFG register
* idxd - struct idxd *
* n - wq id
* ofs - the index of the 32b dword for the config register
*
* The WQCFG register block is divided into groups per each wq. The n index
* allows us to move to the register group that's for that particular wq.
* Each register is 32bits. The ofs gives us the number of register to access.
*/
#define GRPWQCFG_OFFSET(idxd_dev, n, ofs) ((idxd_dev)->grpcfg_offset +\
(n) * GRPCFG_SIZE + sizeof(u64) * (ofs))
#define GRPENGCFG_OFFSET(idxd_dev, n) ((idxd_dev)->grpcfg_offset + (n) * GRPCFG_SIZE + 32)
#define GRPFLGCFG_OFFSET(idxd_dev, n) ((idxd_dev)->grpcfg_offset + (n) * GRPCFG_SIZE + 40)
#endif

37
drivers/dma/idxd/submit.c
View File

@@ -11,11 +11,22 @@
static struct idxd_desc *__get_desc(struct idxd_wq *wq, int idx, int cpu)
{
struct idxd_desc *desc;
struct idxd_device *idxd = wq->idxd;
desc = wq->descs[idx];
memset(desc->hw, 0, sizeof(struct dsa_hw_desc));
memset(desc->completion, 0, sizeof(struct dsa_completion_record));
memset(desc->completion, 0, idxd->compl_size);
desc->cpu = cpu;
if (device_pasid_enabled(idxd))
desc->hw->pasid = idxd->pasid;
/*
* Descriptor completion vectors are 1-8 for MSIX. We will round
* robin through the 8 vectors.
*/
wq->vec_ptr = (wq->vec_ptr % idxd->num_wq_irqs) + 1;
desc->hw->int_handle = wq->vec_ptr;
return desc;
}
@@ -70,18 +81,32 @@ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc)
struct idxd_device *idxd = wq->idxd;
int vec = desc->hw->int_handle;
void __iomem *portal;
int rc;
if (idxd->state != IDXD_DEV_ENABLED)
return -EIO;
portal = wq->dportal;
portal = wq->portal;
/*
* The wmb() flushes writes to coherent DMA data before possibly
* triggering a DMA read. The wmb() is necessary even on UP because
* the recipient is a device.
* The wmb() flushes writes to coherent DMA data before
* possibly triggering a DMA read. The wmb() is necessary
* even on UP because the recipient is a device.
*/
wmb();
iosubmit_cmds512(portal, desc->hw, 1);
if (wq_dedicated(wq)) {
iosubmit_cmds512(portal, desc->hw, 1);
} else {
/*
* It's not likely that we would receive queue full rejection
* since the descriptor allocation gates at wq size. If we
* receive a -EAGAIN, that means something went wrong such as the
* device is not accepting descriptor at all.
*/
rc = enqcmds(portal, desc->hw);
if (rc < 0)
return rc;
}
/*
* Pending the descriptor to the lockless list for the irq_entry

207
drivers/dma/idxd/sysfs.c
View File

@@ -41,14 +41,24 @@ static struct device_type dsa_device_type = {
.release = idxd_conf_device_release,
};
static struct device_type iax_device_type = {
.name = "iax",
.release = idxd_conf_device_release,
};
static inline bool is_dsa_dev(struct device *dev)
{
return dev ? dev->type == &dsa_device_type : false;
}
static inline bool is_iax_dev(struct device *dev)
{
return dev ? dev->type == &iax_device_type : false;
}
static inline bool is_idxd_dev(struct device *dev)
{
return is_dsa_dev(dev);
return is_dsa_dev(dev) || is_iax_dev(dev);
}
static inline bool is_idxd_wq_dev(struct device *dev)
@@ -175,6 +185,30 @@ static int idxd_config_bus_probe(struct device *dev)
return -EINVAL;
}
/* Shared WQ checks */
if (wq_shared(wq)) {
if (!device_swq_supported(idxd)) {
dev_warn(dev,
"PASID not enabled and shared WQ.\n");
mutex_unlock(&wq->wq_lock);
return -ENXIO;
}
/*
* Shared wq with the threshold set to 0 means the user
* did not set the threshold or transitioned from a
* dedicated wq but did not set threshold. A value
* of 0 would effectively disable the shared wq. The
* driver does not allow a value of 0 to be set for
* threshold via sysfs.
*/
if (wq->threshold == 0) {
dev_warn(dev,
"Shared WQ and threshold 0.\n");
mutex_unlock(&wq->wq_lock);
return -EINVAL;
}
}
rc = idxd_wq_alloc_resources(wq);
if (rc < 0) {
mutex_unlock(&wq->wq_lock);
@@ -335,8 +369,17 @@ struct bus_type dsa_bus_type = {
.shutdown = idxd_config_bus_shutdown,
};
struct bus_type iax_bus_type = {
.name = "iax",
.match = idxd_config_bus_match,
.probe = idxd_config_bus_probe,
.remove = idxd_config_bus_remove,
.shutdown = idxd_config_bus_shutdown,
};
static struct bus_type *idxd_bus_types[] = {
&dsa_bus_type
&dsa_bus_type,
&iax_bus_type
};
static struct idxd_device_driver dsa_drv = {
@@ -348,8 +391,18 @@ static struct idxd_device_driver dsa_drv = {
},
};
static struct idxd_device_driver iax_drv = {
.drv = {
.name = "iax",
.bus = &iax_bus_type,
.owner = THIS_MODULE,
.mod_name = KBUILD_MODNAME,
},
};
static struct idxd_device_driver *idxd_drvs[] = {
&dsa_drv
&dsa_drv,
&iax_drv
};
struct bus_type *idxd_get_bus_type(struct idxd_device *idxd)
@@ -361,6 +414,8 @@ static struct device_type *idxd_get_device_type(struct idxd_device *idxd)
{
if (idxd->type == IDXD_TYPE_DSA)
return &dsa_device_type;
else if (idxd->type == IDXD_TYPE_IAX)
return &iax_device_type;
else
return NULL;
}
@@ -501,6 +556,9 @@ static ssize_t group_tokens_reserved_store(struct device *dev,
if (rc < 0)
return -EINVAL;
if (idxd->type == IDXD_TYPE_IAX)
return -EOPNOTSUPP;
if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
return -EPERM;
@@ -546,6 +604,9 @@ static ssize_t group_tokens_allowed_store(struct device *dev,
if (rc < 0)
return -EINVAL;
if (idxd->type == IDXD_TYPE_IAX)
return -EOPNOTSUPP;
if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
return -EPERM;
@@ -588,6 +649,9 @@ static ssize_t group_use_token_limit_store(struct device *dev,
if (rc < 0)
return -EINVAL;
if (idxd->type == IDXD_TYPE_IAX)
return -EOPNOTSUPP;
if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
return -EPERM;
@@ -875,6 +939,8 @@ static ssize_t wq_mode_store(struct device *dev,
if (sysfs_streq(buf, "dedicated")) {
set_bit(WQ_FLAG_DEDICATED, &wq->flags);
wq->threshold = 0;
} else if (sysfs_streq(buf, "shared") && device_swq_supported(idxd)) {
clear_bit(WQ_FLAG_DEDICATED, &wq->flags);
} else {
return -EINVAL;
}
@@ -973,6 +1039,87 @@ static ssize_t wq_priority_store(struct device *dev,
static struct device_attribute dev_attr_wq_priority =
__ATTR(priority, 0644, wq_priority_show, wq_priority_store);
static ssize_t wq_block_on_fault_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
return sprintf(buf, "%u\n",
test_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags));
}
static ssize_t wq_block_on_fault_store(struct device *dev,
struct device_attribute *attr,
const char *buf, size_t count)
{
struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
struct idxd_device *idxd = wq->idxd;
bool bof;
int rc;
if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
return -EPERM;
if (wq->state != IDXD_WQ_DISABLED)
return -ENXIO;
rc = kstrtobool(buf, &bof);
if (rc < 0)
return rc;
if (bof)
set_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags);
else
clear_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags);
return count;
}
static struct device_attribute dev_attr_wq_block_on_fault =
__ATTR(block_on_fault, 0644, wq_block_on_fault_show,
wq_block_on_fault_store);
static ssize_t wq_threshold_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
return sprintf(buf, "%u\n", wq->threshold);
}
static ssize_t wq_threshold_store(struct device *dev,
struct device_attribute *attr,
const char *buf, size_t count)
{
struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
struct idxd_device *idxd = wq->idxd;
unsigned int val;
int rc;
rc = kstrtouint(buf, 0, &val);
if (rc < 0)
return -EINVAL;
if (val > wq->size || val <= 0)
return -EINVAL;
if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
return -EPERM;
if (wq->state != IDXD_WQ_DISABLED)
return -ENXIO;
if (test_bit(WQ_FLAG_DEDICATED, &wq->flags))
return -EINVAL;
wq->threshold = val;
return count;
}
static struct device_attribute dev_attr_wq_threshold =
__ATTR(threshold, 0644, wq_threshold_show, wq_threshold_store);
static ssize_t wq_type_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
@@ -1044,6 +1191,13 @@ static ssize_t wq_name_store(struct device *dev,
if (strlen(buf) > WQ_NAME_SIZE || strlen(buf) == 0)
return -EINVAL;
/*
* This is temporarily placed here until we have SVM support for
* dmaengine.
*/
if (wq->type == IDXD_WQT_KERNEL && device_pasid_enabled(wq->idxd))
return -EOPNOTSUPP;
memset(wq->name, 0, WQ_NAME_SIZE + 1);
strncpy(wq->name, buf, WQ_NAME_SIZE);
strreplace(wq->name, '\n', '\0');
@@ -1147,6 +1301,39 @@ static ssize_t wq_max_batch_size_store(struct device *dev, struct device_attribu
static struct device_attribute dev_attr_wq_max_batch_size =
__ATTR(max_batch_size, 0644, wq_max_batch_size_show, wq_max_batch_size_store);
static ssize_t wq_ats_disable_show(struct device *dev, struct device_attribute *attr, char *buf)
{
struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
return sprintf(buf, "%u\n", wq->ats_dis);
}
static ssize_t wq_ats_disable_store(struct device *dev, struct device_attribute *attr,
const char *buf, size_t count)
{
struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
struct idxd_device *idxd = wq->idxd;
bool ats_dis;
int rc;
if (wq->state != IDXD_WQ_DISABLED)
return -EPERM;
if (!idxd->hw.wq_cap.wq_ats_support)
return -EOPNOTSUPP;
rc = kstrtobool(buf, &ats_dis);
if (rc < 0)
return rc;
wq->ats_dis = ats_dis;
return count;
}
static struct device_attribute dev_attr_wq_ats_disable =
__ATTR(ats_disable, 0644, wq_ats_disable_show, wq_ats_disable_store);
static struct attribute *idxd_wq_attributes[] = {
&dev_attr_wq_clients.attr,
&dev_attr_wq_state.attr,
@@ -1154,11 +1341,14 @@ static struct attribute *idxd_wq_attributes[] = {
&dev_attr_wq_mode.attr,
&dev_attr_wq_size.attr,
&dev_attr_wq_priority.attr,
&dev_attr_wq_block_on_fault.attr,
&dev_attr_wq_threshold.attr,
&dev_attr_wq_type.attr,
&dev_attr_wq_name.attr,
&dev_attr_wq_cdev_minor.attr,
&dev_attr_wq_max_transfer_size.attr,
&dev_attr_wq_max_batch_size.attr,
&dev_attr_wq_ats_disable.attr,
NULL,
};
@@ -1305,6 +1495,16 @@ static ssize_t clients_show(struct device *dev,
}
static DEVICE_ATTR_RO(clients);
static ssize_t pasid_enabled_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct idxd_device *idxd =
container_of(dev, struct idxd_device, conf_dev);
return sprintf(buf, "%u\n", device_pasid_enabled(idxd));
}
static DEVICE_ATTR_RO(pasid_enabled);
static ssize_t state_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
@@ -1424,6 +1624,7 @@ static struct attribute *idxd_device_attributes[] = {
&dev_attr_gen_cap.attr,
&dev_attr_configurable.attr,
&dev_attr_clients.attr,
&dev_attr_pasid_enabled.attr,
&dev_attr_state.attr,
&dev_attr_errors.attr,
&dev_attr_max_tokens.attr,

33
drivers/dma/imx-dma.c
View File

@@ -191,32 +191,13 @@ struct imxdma_filter_data {
int request;
};
static const struct platform_device_id imx_dma_devtype[] = {
{
.name = "imx1-dma",
.driver_data = IMX1_DMA,
}, {
.name = "imx21-dma",
.driver_data = IMX21_DMA,
}, {
.name = "imx27-dma",
.driver_data = IMX27_DMA,
}, {
/* sentinel */
}
};
MODULE_DEVICE_TABLE(platform, imx_dma_devtype);
static const struct of_device_id imx_dma_of_dev_id[] = {
{
.compatible = "fsl,imx1-dma",
.data = &imx_dma_devtype[IMX1_DMA],
.compatible = "fsl,imx1-dma", .data = (const void *)IMX1_DMA,
}, {
.compatible = "fsl,imx21-dma",
.data = &imx_dma_devtype[IMX21_DMA],
.compatible = "fsl,imx21-dma", .data = (const void *)IMX21_DMA,
}, {
.compatible = "fsl,imx27-dma",
.data = &imx_dma_devtype[IMX27_DMA],
.compatible = "fsl,imx27-dma", .data = (const void *)IMX27_DMA,
}, {
/* sentinel */
}
@@ -1056,20 +1037,15 @@ static int __init imxdma_probe(struct platform_device *pdev)
{
struct imxdma_engine *imxdma;
struct resource *res;
const struct of_device_id *of_id;
int ret, i;
int irq, irq_err;
of_id = of_match_device(imx_dma_of_dev_id, &pdev->dev);
if (of_id)
pdev->id_entry = of_id->data;
imxdma = devm_kzalloc(&pdev->dev, sizeof(*imxdma), GFP_KERNEL);
if (!imxdma)
return -ENOMEM;
imxdma->dev = &pdev->dev;
imxdma->devtype = pdev->id_entry->driver_data;
imxdma->devtype = (enum imx_dma_type)of_device_get_match_data(&pdev->dev);
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
imxdma->base = devm_ioremap_resource(&pdev->dev, res);
@@ -1263,7 +1239,6 @@ static struct platform_driver imxdma_driver = {
.name = "imx-dma",
.of_match_table = imx_dma_of_dev_id,
},
.id_table = imx_dma_devtype,
.remove = imxdma_remove,
};

38
drivers/dma/imx-sdma.c
View File

@@ -566,37 +566,6 @@ static struct sdma_driver_data sdma_imx8mq = {
.check_ratio = 1,
};
static const struct platform_device_id sdma_devtypes[] = {
{
.name = "imx25-sdma",
.driver_data = (unsigned long)&sdma_imx25,
}, {
.name = "imx31-sdma",
.driver_data = (unsigned long)&sdma_imx31,
}, {
.name = "imx35-sdma",
.driver_data = (unsigned long)&sdma_imx35,
}, {
.name = "imx51-sdma",
.driver_data = (unsigned long)&sdma_imx51,
}, {
.name = "imx53-sdma",
.driver_data = (unsigned long)&sdma_imx53,
}, {
.name = "imx6q-sdma",
.driver_data = (unsigned long)&sdma_imx6q,
}, {
.name = "imx7d-sdma",
.driver_data = (unsigned long)&sdma_imx7d,
}, {
.name = "imx8mq-sdma",
.driver_data = (unsigned long)&sdma_imx8mq,
}, {
/* sentinel */
}
};
MODULE_DEVICE_TABLE(platform, sdma_devtypes);
static const struct of_device_id sdma_dt_ids[] = {
{ .compatible = "fsl,imx6q-sdma", .data = &sdma_imx6q, },
{ .compatible = "fsl,imx53-sdma", .data = &sdma_imx53, },
@@ -1998,11 +1967,7 @@ static int sdma_probe(struct platform_device *pdev)
s32 *saddr_arr;
const struct sdma_driver_data *drvdata = NULL;
if (of_id)
drvdata = of_id->data;
else if (pdev->id_entry)
drvdata = (void *)pdev->id_entry->driver_data;
drvdata = of_id->data;
if (!drvdata) {
dev_err(&pdev->dev, "unable to find driver data\n");
return -EINVAL;
@@ -2211,7 +2176,6 @@ static struct platform_driver sdma_driver = {
.name = "imx-sdma",
.of_match_table = sdma_dt_ids,
},
.id_table = sdma_devtypes,
.remove = sdma_remove,
.probe = sdma_probe,
};

11
drivers/dma/ipu/ipu_idmac.c
View File

@@ -1160,14 +1160,13 @@ static irqreturn_t idmac_interrupt(int irq, void *dev_id)
struct idmac_tx_desc *desc, *descnew;
bool done = false;
u32 ready0, ready1, curbuf, err;
unsigned long flags;
struct dmaengine_desc_callback cb;
/* IDMAC has cleared the respective BUFx_RDY bit, we manage the buffer */
dev_dbg(dev, "IDMAC irq %d, buf %d\n", irq, ichan->active_buffer);
spin_lock_irqsave(&ipu_data.lock, flags);
spin_lock(&ipu_data.lock);
ready0 = idmac_read_ipureg(&ipu_data, IPU_CHA_BUF0_RDY);
ready1 = idmac_read_ipureg(&ipu_data, IPU_CHA_BUF1_RDY);
@@ -1176,7 +1175,7 @@ static irqreturn_t idmac_interrupt(int irq, void *dev_id)
if (err & (1 << chan_id)) {
idmac_write_ipureg(&ipu_data, 1 << chan_id, IPU_INT_STAT_4);
spin_unlock_irqrestore(&ipu_data.lock, flags);
spin_unlock(&ipu_data.lock);
/*
* Doing this
* ichan->sg[0] = ichan->sg[1] = NULL;
@@ -1188,7 +1187,7 @@ static irqreturn_t idmac_interrupt(int irq, void *dev_id)
chan_id, ready0, ready1, curbuf);
return IRQ_HANDLED;
}
spin_unlock_irqrestore(&ipu_data.lock, flags);
spin_unlock(&ipu_data.lock);
/* Other interrupts do not interfere with this channel */
spin_lock(&ichan->lock);
@@ -1251,9 +1250,9 @@ static irqreturn_t idmac_interrupt(int irq, void *dev_id)
if (unlikely(sgnew)) {
ipu_submit_buffer(ichan, descnew, sgnew, !ichan->active_buffer);
} else {
spin_lock_irqsave(&ipu_data.lock, flags);
spin_lock(&ipu_data.lock);
ipu_ic_disable_task(&ipu_data, chan_id);
spin_unlock_irqrestore(&ipu_data.lock, flags);
spin_unlock(&ipu_data.lock);
ichan->status = IPU_CHANNEL_READY;
/* Continue to check for complete descriptor */
}

9
drivers/dma/k3dma.c
View File

@@ -223,24 +223,23 @@ static irqreturn_t k3_dma_int_handler(int irq, void *dev_id)
i = __ffs(stat);
stat &= ~BIT(i);
if (likely(tc1 & BIT(i)) || (tc2 & BIT(i))) {
unsigned long flags;
p = &d->phy[i];
c = p->vchan;
if (c && (tc1 & BIT(i))) {
spin_lock_irqsave(&c->vc.lock, flags);
spin_lock(&c->vc.lock);
if (p->ds_run != NULL) {
vchan_cookie_complete(&p->ds_run->vd);
p->ds_done = p->ds_run;
p->ds_run = NULL;
}
spin_unlock_irqrestore(&c->vc.lock, flags);
spin_unlock(&c->vc.lock);
}
if (c && (tc2 & BIT(i))) {
spin_lock_irqsave(&c->vc.lock, flags);
spin_lock(&c->vc.lock);
if (p->ds_run != NULL)
vchan_cyclic_callback(&p->ds_run->vd);
spin_unlock_irqrestore(&c->vc.lock, flags);
spin_unlock(&c->vc.lock);
}
irq_chan |= BIT(i);
}

5
drivers/dma/milbeaut-xdmac.c
View File

@@ -160,10 +160,9 @@ static irqreturn_t milbeaut_xdmac_interrupt(int irq, void *dev_id)
{
struct milbeaut_xdmac_chan *mc = dev_id;
struct milbeaut_xdmac_desc *md;
unsigned long flags;
u32 val;
spin_lock_irqsave(&mc->vc.lock, flags);
spin_lock(&mc->vc.lock);
/* Ack and Stop */
val = FIELD_PREP(M10V_XDDSD_IS_MASK, 0x0);
@@ -177,7 +176,7 @@ static irqreturn_t milbeaut_xdmac_interrupt(int irq, void *dev_id)
milbeaut_xdmac_start(mc);
out:
spin_unlock_irqrestore(&mc->vc.lock, flags);
spin_unlock(&mc->vc.lock);
return IRQ_HANDLED;
}

5
drivers/dma/moxart-dma.c
View File

@@ -524,7 +524,6 @@ static irqreturn_t moxart_dma_interrupt(int irq, void *devid)
struct moxart_dmadev *mc = devid;
struct moxart_chan *ch = &mc->slave_chans[0];
unsigned int i;
unsigned long flags;
u32 ctrl;
dev_dbg(chan2dev(&ch->vc.chan), "%s\n", __func__);
@@ -541,14 +540,14 @@ static irqreturn_t moxart_dma_interrupt(int irq, void *devid)
if (ctrl & APB_DMA_FIN_INT_STS) {
ctrl &= ~APB_DMA_FIN_INT_STS;
if (ch->desc) {
spin_lock_irqsave(&ch->vc.lock, flags);
spin_lock(&ch->vc.lock);
if (++ch->sgidx < ch->desc->sglen) {
moxart_dma_start_sg(ch, ch->sgidx);
} else {
vchan_cookie_complete(&ch->desc->vd);
moxart_dma_start_desc(&ch->vc.chan);
}
spin_unlock_irqrestore(&ch->vc.lock, flags);
spin_unlock(&ch->vc.lock);
}
}

2
drivers/dma/mv_xor.c
View File

@@ -1455,7 +1455,7 @@ static struct platform_driver mv_xor_driver = {
.resume = mv_xor_resume,
.driver = {
.name = MV_XOR_NAME,
.of_match_table = of_match_ptr(mv_xor_dt_ids),
.of_match_table = mv_xor_dt_ids,
},
};

4
drivers/dma/mv_xor_v2.c
View File

@@ -771,8 +771,10 @@ static int mv_xor_v2_probe(struct platform_device *pdev)
goto disable_clk;
msi_desc = first_msi_entry(&pdev->dev);
if (!msi_desc)
if (!msi_desc) {
ret = -ENODEV;
goto free_msi_irqs;
}
xor_dev->msi_desc = msi_desc;
ret = devm_request_irq(&pdev->dev, msi_desc->irq,

37
drivers/dma/mxs-dma.c
View File

@@ -167,29 +167,11 @@ static struct mxs_dma_type mxs_dma_types[] = {
}
};
static const struct platform_device_id mxs_dma_ids[] = {
{
.name = "imx23-dma-apbh",
.driver_data = (kernel_ulong_t) &mxs_dma_types[0],
}, {
.name = "imx23-dma-apbx",
.driver_data = (kernel_ulong_t) &mxs_dma_types[1],
}, {
.name = "imx28-dma-apbh",
.driver_data = (kernel_ulong_t) &mxs_dma_types[2],
}, {
.name = "imx28-dma-apbx",
.driver_data = (kernel_ulong_t) &mxs_dma_types[3],
}, {
/* end of list */
}
};
static const struct of_device_id mxs_dma_dt_ids[] = {
{ .compatible = "fsl,imx23-dma-apbh", .data = &mxs_dma_ids[0], },
{ .compatible = "fsl,imx23-dma-apbx", .data = &mxs_dma_ids[1], },
{ .compatible = "fsl,imx28-dma-apbh", .data = &mxs_dma_ids[2], },
{ .compatible = "fsl,imx28-dma-apbx", .data = &mxs_dma_ids[3], },
{ .compatible = "fsl,imx23-dma-apbh", .data = &mxs_dma_types[0], },
{ .compatible = "fsl,imx23-dma-apbx", .data = &mxs_dma_types[1], },
{ .compatible = "fsl,imx28-dma-apbh", .data = &mxs_dma_types[2], },
{ .compatible = "fsl,imx28-dma-apbx", .data = &mxs_dma_types[3], },
{ /* sentinel */ }
};
MODULE_DEVICE_TABLE(of, mxs_dma_dt_ids);
@@ -762,8 +744,6 @@ static struct dma_chan *mxs_dma_xlate(struct of_phandle_args *dma_spec,
static int __init mxs_dma_probe(struct platform_device *pdev)
{
struct device_node *np = pdev->dev.of_node;
const struct platform_device_id *id_entry;
const struct of_device_id *of_id;
const struct mxs_dma_type *dma_type;
struct mxs_dma_engine *mxs_dma;
struct resource *iores;
@@ -779,13 +759,7 @@ static int __init mxs_dma_probe(struct platform_device *pdev)
return ret;
}
of_id = of_match_device(mxs_dma_dt_ids, &pdev->dev);
if (of_id)
id_entry = of_id->data;
else
id_entry = platform_get_device_id(pdev);
dma_type = (struct mxs_dma_type *)id_entry->driver_data;
dma_type = (struct mxs_dma_type *)of_device_get_match_data(&pdev->dev);
mxs_dma->type = dma_type->type;
mxs_dma->dev_id = dma_type->id;
@@ -865,7 +839,6 @@ static struct platform_driver mxs_dma_driver = {
.name = "mxs-dma",
.of_match_table = mxs_dma_dt_ids,
},
.id_table = mxs_dma_ids,
};
static int __init mxs_dma_module_init(void)

10
drivers/dma/of-dma.c
View File

@@ -75,8 +75,18 @@ static struct dma_chan *of_dma_router_xlate(struct of_phandle_args *dma_spec,
ofdma->dma_router->route_free(ofdma->dma_router->dev,
route_data);
} else {
int ret = 0;
chan->router = ofdma->dma_router;
chan->route_data = route_data;
if (chan->device->device_router_config)
ret = chan->device->device_router_config(chan);
if (ret) {
dma_release_channel(chan);
chan = ERR_PTR(ret);
}
}
/*

2
drivers/dma/pl330.c
View File

@@ -1527,8 +1527,6 @@ static int pl330_submit_req(struct pl330_thread *thrd,
/* First dry run to check if req is acceptable */
ret = _setup_req(pl330, 1, thrd, idx, &xs);
if (ret < 0)
goto xfer_exit;
if (ret > pl330->mcbufsz / 2) {
dev_info(pl330->ddma.dev, "%s:%d Try increasing mcbufsz (%i/%i)\n",

4
drivers/dma/ppc4xx/adma.c
View File

@@ -69,7 +69,7 @@ struct ppc_dma_chan_ref {
};
/* The list of channels exported by ppc440spe ADMA */
struct list_head
static struct list_head
ppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list);
/* This flag is set when want to refetch the xor chain in the interrupt
@@ -559,7 +559,6 @@ static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc,
int sg_index, unsigned char mult_value)
{
struct dma_cdb *dma_hw_desc;
struct xor_cb *xor_hw_desc;
u32 *psgu;
switch (chan->device->id) {
@@ -590,7 +589,6 @@ static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc,
*psgu |= cpu_to_le32(mult_value << mult_index);
break;
case PPC440SPE_XOR_ID:
xor_hw_desc = desc->hw_desc;
break;
default:
BUG();

5
drivers/dma/pxa_dma.c
View File

@@ -606,7 +606,6 @@ static irqreturn_t pxad_chan_handler(int irq, void *dev_id)
struct pxad_chan *chan = phy->vchan;
struct virt_dma_desc *vd, *tmp;
unsigned int dcsr;
unsigned long flags;
bool vd_completed;
dma_cookie_t last_started = 0;
@@ -616,7 +615,7 @@ static irqreturn_t pxad_chan_handler(int irq, void *dev_id)
if (dcsr & PXA_DCSR_RUN)
return IRQ_NONE;
spin_lock_irqsave(&chan->vc.lock, flags);
spin_lock(&chan->vc.lock);
list_for_each_entry_safe(vd, tmp, &chan->vc.desc_issued, node) {
vd_completed = is_desc_completed(vd);
dev_dbg(&chan->vc.chan.dev->device,
@@ -658,7 +657,7 @@ static irqreturn_t pxad_chan_handler(int irq, void *dev_id)
pxad_launch_chan(chan, to_pxad_sw_desc(vd));
}
}
spin_unlock_irqrestore(&chan->vc.lock, flags);
spin_unlock(&chan->vc.lock);
wake_up(&chan->wq_state);
return IRQ_HANDLED;

23
drivers/dma/qcom/Kconfig
View File

@@ -1,4 +1,15 @@
# SPDX-License-Identifier: GPL-2.0-only
config QCOM_ADM
tristate "Qualcomm ADM support"
depends on (ARCH_QCOM || COMPILE_TEST) && !PHYS_ADDR_T_64BIT
select DMA_ENGINE
select DMA_VIRTUAL_CHANNELS
help
Enable support for the Qualcomm Application Data Mover (ADM) DMA
controller, as present on MSM8x60, APQ8064, and IPQ8064 devices.
This controller provides DMA capabilities for both general purpose
and on-chip peripheral devices.
config QCOM_BAM_DMA
tristate "QCOM BAM DMA support"
depends on ARCH_QCOM || (COMPILE_TEST && OF && ARM)
@@ -8,6 +19,18 @@ config QCOM_BAM_DMA
Enable support for the QCOM BAM DMA controller. This controller
provides DMA capabilities for a variety of on-chip devices.
config QCOM_GPI_DMA
tristate "Qualcomm Technologies GPI DMA support"
depends on ARCH_QCOM
select DMA_ENGINE
select DMA_VIRTUAL_CHANNELS
help
Enable support for the QCOM GPI DMA controller. This controller
provides DMA capabilities for a variety of peripheral buses such
as I2C, UART, and SPI. By using GPI dmaengine driver, bus drivers
can use a standardize interface that is protocol independent to
transfer data between DDR and peripheral.
config QCOM_HIDMA_MGMT
tristate "Qualcomm Technologies HIDMA Management support"
select DMA_ENGINE

2
drivers/dma/qcom/Makefile
View File

@@ -1,5 +1,7 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_QCOM_ADM) += qcom_adm.o
obj-$(CONFIG_QCOM_BAM_DMA) += bam_dma.o
obj-$(CONFIG_QCOM_GPI_DMA) += gpi.o
obj-$(CONFIG_QCOM_HIDMA_MGMT) += hdma_mgmt.o
hdma_mgmt-objs := hidma_mgmt.o hidma_mgmt_sys.o
obj-$(CONFIG_QCOM_HIDMA) += hdma.o

2
drivers/dma/qcom/bam_dma.c
View File

@@ -875,7 +875,7 @@ static irqreturn_t bam_dma_irq(int irq, void *data)
ret = bam_pm_runtime_get_sync(bdev->dev);
if (ret < 0)
return ret;
return IRQ_NONE;
if (srcs & BAM_IRQ) {
clr_mask = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_STTS));

2303
drivers/dma/qcom/gpi.c Normal file
View File

File diff suppressed because it is too large Load Diff

905
drivers/dma/qcom/qcom_adm.c Normal file
View File

@@ -0,0 +1,905 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2013-2015, The Linux Foundation. All rights reserved.
*/
#include <linux/clk.h>
#include <linux/delay.h>
#include <linux/device.h>
#include <linux/dmaengine.h>
#include <linux/dma-mapping.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/of_address.h>
#include <linux/of_irq.h>
#include <linux/of_dma.h>
#include <linux/platform_device.h>
#include <linux/reset.h>
#include <linux/scatterlist.h>
#include <linux/slab.h>
#include "../dmaengine.h"
#include "../virt-dma.h"
/* ADM registers - calculated from channel number and security domain */
#define ADM_CHAN_MULTI 0x4
#define ADM_CI_MULTI 0x4
#define ADM_CRCI_MULTI 0x4
#define ADM_EE_MULTI 0x800
#define ADM_CHAN_OFFS(chan) (ADM_CHAN_MULTI * (chan))
#define ADM_EE_OFFS(ee) (ADM_EE_MULTI * (ee))
#define ADM_CHAN_EE_OFFS(chan, ee) (ADM_CHAN_OFFS(chan) + ADM_EE_OFFS(ee))
#define ADM_CHAN_OFFS(chan) (ADM_CHAN_MULTI * (chan))
#define ADM_CI_OFFS(ci) (ADM_CHAN_OFF(ci))
#define ADM_CH_CMD_PTR(chan, ee) (ADM_CHAN_EE_OFFS(chan, ee))
#define ADM_CH_RSLT(chan, ee) (0x40 + ADM_CHAN_EE_OFFS(chan, ee))
#define ADM_CH_FLUSH_STATE0(chan, ee) (0x80 + ADM_CHAN_EE_OFFS(chan, ee))
#define ADM_CH_STATUS_SD(chan, ee) (0x200 + ADM_CHAN_EE_OFFS(chan, ee))
#define ADM_CH_CONF(chan) (0x240 + ADM_CHAN_OFFS(chan))
#define ADM_CH_RSLT_CONF(chan, ee) (0x300 + ADM_CHAN_EE_OFFS(chan, ee))
#define ADM_SEC_DOMAIN_IRQ_STATUS(ee) (0x380 + ADM_EE_OFFS(ee))
#define ADM_CI_CONF(ci) (0x390 + (ci) * ADM_CI_MULTI)
#define ADM_GP_CTL 0x3d8
#define ADM_CRCI_CTL(crci, ee) (0x400 + (crci) * ADM_CRCI_MULTI + \
ADM_EE_OFFS(ee))
/* channel status */
#define ADM_CH_STATUS_VALID BIT(1)
/* channel result */
#define ADM_CH_RSLT_VALID BIT(31)
#define ADM_CH_RSLT_ERR BIT(3)
#define ADM_CH_RSLT_FLUSH BIT(2)
#define ADM_CH_RSLT_TPD BIT(1)
/* channel conf */
#define ADM_CH_CONF_SHADOW_EN BIT(12)
#define ADM_CH_CONF_MPU_DISABLE BIT(11)
#define ADM_CH_CONF_PERM_MPU_CONF BIT(9)
#define ADM_CH_CONF_FORCE_RSLT_EN BIT(7)
#define ADM_CH_CONF_SEC_DOMAIN(ee) ((((ee) & 0x3) << 4) | (((ee) & 0x4) << 11))
/* channel result conf */
#define ADM_CH_RSLT_CONF_FLUSH_EN BIT(1)
#define ADM_CH_RSLT_CONF_IRQ_EN BIT(0)
/* CRCI CTL */
#define ADM_CRCI_CTL_MUX_SEL BIT(18)
#define ADM_CRCI_CTL_RST BIT(17)
/* CI configuration */
#define ADM_CI_RANGE_END(x) ((x) << 24)
#define ADM_CI_RANGE_START(x) ((x) << 16)
#define ADM_CI_BURST_4_WORDS BIT(2)
#define ADM_CI_BURST_8_WORDS BIT(3)
/* GP CTL */
#define ADM_GP_CTL_LP_EN BIT(12)
#define ADM_GP_CTL_LP_CNT(x) ((x) << 8)
/* Command pointer list entry */
#define ADM_CPLE_LP BIT(31)
#define ADM_CPLE_CMD_PTR_LIST BIT(29)
/* Command list entry */
#define ADM_CMD_LC BIT(31)
#define ADM_CMD_DST_CRCI(n) (((n) & 0xf) << 7)
#define ADM_CMD_SRC_CRCI(n) (((n) & 0xf) << 3)
#define ADM_CMD_TYPE_SINGLE 0x0
#define ADM_CMD_TYPE_BOX 0x3
#define ADM_CRCI_MUX_SEL BIT(4)
#define ADM_DESC_ALIGN 8
#define ADM_MAX_XFER (SZ_64K - 1)
#define ADM_MAX_ROWS (SZ_64K - 1)
#define ADM_MAX_CHANNELS 16
struct adm_desc_hw_box {
u32 cmd;
u32 src_addr;
u32 dst_addr;
u32 row_len;
u32 num_rows;
u32 row_offset;
};
struct adm_desc_hw_single {
u32 cmd;
u32 src_addr;
u32 dst_addr;
u32 len;
};
struct adm_async_desc {
struct virt_dma_desc vd;
struct adm_device *adev;
size_t length;
enum dma_transfer_direction dir;
dma_addr_t dma_addr;
size_t dma_len;
void *cpl;
dma_addr_t cp_addr;
u32 crci;
u32 mux;
u32 blk_size;
};
struct adm_chan {
struct virt_dma_chan vc;
struct adm_device *adev;
/* parsed from DT */
u32 id; /* channel id */
struct adm_async_desc *curr_txd;
struct dma_slave_config slave;
struct list_head node;
int error;
int initialized;
};
static inline struct adm_chan *to_adm_chan(struct dma_chan *common)
{
return container_of(common, struct adm_chan, vc.chan);
}
struct adm_device {
void __iomem *regs;
struct device *dev;
struct dma_device common;
struct device_dma_parameters dma_parms;
struct adm_chan *channels;
u32 ee;
struct clk *core_clk;
struct clk *iface_clk;
struct reset_control *clk_reset;
struct reset_control *c0_reset;
struct reset_control *c1_reset;
struct reset_control *c2_reset;
int irq;
};
/**
* adm_free_chan - Frees dma resources associated with the specific channel
*
* @chan: dma channel
*
* Free all allocated descriptors associated with this channel
*/
static void adm_free_chan(struct dma_chan *chan)
{
/* free all queued descriptors */
vchan_free_chan_resources(to_virt_chan(chan));
}
/**
* adm_get_blksize - Get block size from burst value
*
* @burst: Burst size of transaction
*/
static int adm_get_blksize(unsigned int burst)
{
int ret;
switch (burst) {
case 16:
case 32:
case 64:
case 128:
ret = ffs(burst >> 4) - 1;
break;
case 192:
ret = 4;
break;
case 256:
ret = 5;
break;
default:
ret = -EINVAL;
break;
}
return ret;
}
/**
* adm_process_fc_descriptors - Process descriptors for flow controlled xfers
*
* @achan: ADM channel
* @desc: Descriptor memory pointer
* @sg: Scatterlist entry
* @crci: CRCI value
* @burst: Burst size of transaction
* @direction: DMA transfer direction
*/
static void *adm_process_fc_descriptors(struct adm_chan *achan, void *desc,
struct scatterlist *sg, u32 crci,
u32 burst,
enum dma_transfer_direction direction)
{
struct adm_desc_hw_box *box_desc = NULL;
struct adm_desc_hw_single *single_desc;
u32 remainder = sg_dma_len(sg);
u32 rows, row_offset, crci_cmd;
u32 mem_addr = sg_dma_address(sg);
u32 *incr_addr = &mem_addr;
u32 *src, *dst;
if (direction == DMA_DEV_TO_MEM) {
crci_cmd = ADM_CMD_SRC_CRCI(crci);
row_offset = burst;
src = &achan->slave.src_addr;
dst = &mem_addr;
} else {
crci_cmd = ADM_CMD_DST_CRCI(crci);
row_offset = burst << 16;
src = &mem_addr;
dst = &achan->slave.dst_addr;
}
while (remainder >= burst) {
box_desc = desc;
box_desc->cmd = ADM_CMD_TYPE_BOX | crci_cmd;
box_desc->row_offset = row_offset;
box_desc->src_addr = *src;
box_desc->dst_addr = *dst;
rows = remainder / burst;
rows = min_t(u32, rows, ADM_MAX_ROWS);
box_desc->num_rows = rows << 16 | rows;
box_desc->row_len = burst << 16 | burst;
*incr_addr += burst * rows;
remainder -= burst * rows;
desc += sizeof(*box_desc);
}
/* if leftover bytes, do one single descriptor */
if (remainder) {
single_desc = desc;
single_desc->cmd = ADM_CMD_TYPE_SINGLE | crci_cmd;
single_desc->len = remainder;
single_desc->src_addr = *src;
single_desc->dst_addr = *dst;
desc += sizeof(*single_desc);
if (sg_is_last(sg))
single_desc->cmd |= ADM_CMD_LC;
} else {
if (box_desc && sg_is_last(sg))
box_desc->cmd |= ADM_CMD_LC;
}
return desc;
}
/**
* adm_process_non_fc_descriptors - Process descriptors for non-fc xfers
*
* @achan: ADM channel
* @desc: Descriptor memory pointer
* @sg: Scatterlist entry
* @direction: DMA transfer direction
*/
static void *adm_process_non_fc_descriptors(struct adm_chan *achan, void *desc,
struct scatterlist *sg,
enum dma_transfer_direction direction)
{
struct adm_desc_hw_single *single_desc;
u32 remainder = sg_dma_len(sg);
u32 mem_addr = sg_dma_address(sg);
u32 *incr_addr = &mem_addr;
u32 *src, *dst;
if (direction == DMA_DEV_TO_MEM) {
src = &achan->slave.src_addr;
dst = &mem_addr;
} else {
src = &mem_addr;
dst = &achan->slave.dst_addr;
}
do {
single_desc = desc;
single_desc->cmd = ADM_CMD_TYPE_SINGLE;
single_desc->src_addr = *src;
single_desc->dst_addr = *dst;
single_desc->len = (remainder > ADM_MAX_XFER) ?
ADM_MAX_XFER : remainder;
remainder -= single_desc->len;
*incr_addr += single_desc->len;
desc += sizeof(*single_desc);
} while (remainder);
/* set last command if this is the end of the whole transaction */
if (sg_is_last(sg))
single_desc->cmd |= ADM_CMD_LC;
return desc;
}
/**
* adm_prep_slave_sg - Prep slave sg transaction
*
* @chan: dma channel
* @sgl: scatter gather list
* @sg_len: length of sg
* @direction: DMA transfer direction
* @flags: DMA flags
* @context: transfer context (unused)
*/
static struct dma_async_tx_descriptor *adm_prep_slave_sg(struct dma_chan *chan,
struct scatterlist *sgl,
unsigned int sg_len,
enum dma_transfer_direction direction,
unsigned long flags,
void *context)
{
struct adm_chan *achan = to_adm_chan(chan);
struct adm_device *adev = achan->adev;
struct adm_async_desc *async_desc;
struct scatterlist *sg;
dma_addr_t cple_addr;
u32 i, burst;
u32 single_count = 0, box_count = 0, crci = 0;
void *desc;
u32 *cple;
int blk_size = 0;
if (!is_slave_direction(direction)) {
dev_err(adev->dev, "invalid dma direction\n");
return NULL;
}
/*
* get burst value from slave configuration
*/
burst = (direction == DMA_MEM_TO_DEV) ?
achan->slave.dst_maxburst :
achan->slave.src_maxburst;
/* if using flow control, validate burst and crci values */
if (achan->slave.device_fc) {
blk_size = adm_get_blksize(burst);
if (blk_size < 0) {
dev_err(adev->dev, "invalid burst value: %d\n",
burst);
return ERR_PTR(-EINVAL);
}
crci = achan->slave.slave_id & 0xf;
if (!crci || achan->slave.slave_id > 0x1f) {
dev_err(adev->dev, "invalid crci value\n");
return ERR_PTR(-EINVAL);
}
}
/* iterate through sgs and compute allocation size of structures */
for_each_sg(sgl, sg, sg_len, i) {
if (achan->slave.device_fc) {
box_count += DIV_ROUND_UP(sg_dma_len(sg) / burst,
ADM_MAX_ROWS);
if (sg_dma_len(sg) % burst)
single_count++;
} else {
single_count += DIV_ROUND_UP(sg_dma_len(sg),
ADM_MAX_XFER);
}
}
async_desc = kzalloc(sizeof(*async_desc), GFP_NOWAIT);
if (!async_desc)
return ERR_PTR(-ENOMEM);
if (crci)
async_desc->mux = achan->slave.slave_id & ADM_CRCI_MUX_SEL ?
ADM_CRCI_CTL_MUX_SEL : 0;
async_desc->crci = crci;
async_desc->blk_size = blk_size;
async_desc->dma_len = single_count * sizeof(struct adm_desc_hw_single) +
box_count * sizeof(struct adm_desc_hw_box) +
sizeof(*cple) + 2 * ADM_DESC_ALIGN;
async_desc->cpl = kzalloc(async_desc->dma_len, GFP_NOWAIT);
if (!async_desc->cpl)
goto free;
async_desc->adev = adev;
/* both command list entry and descriptors must be 8 byte aligned */
cple = PTR_ALIGN(async_desc->cpl, ADM_DESC_ALIGN);
desc = PTR_ALIGN(cple + 1, ADM_DESC_ALIGN);
for_each_sg(sgl, sg, sg_len, i) {
async_desc->length += sg_dma_len(sg);
if (achan->slave.device_fc)
desc = adm_process_fc_descriptors(achan, desc, sg, crci,
burst, direction);
else
desc = adm_process_non_fc_descriptors(achan, desc, sg,
direction);
}
async_desc->dma_addr = dma_map_single(adev->dev, async_desc->cpl,
async_desc->dma_len,
DMA_TO_DEVICE);
if (dma_mapping_error(adev->dev, async_desc->dma_addr))
goto free;
cple_addr = async_desc->dma_addr + ((void *)cple - async_desc->cpl);
/* init cmd list */
dma_sync_single_for_cpu(adev->dev, cple_addr, sizeof(*cple),
DMA_TO_DEVICE);
*cple = ADM_CPLE_LP;
*cple |= (async_desc->dma_addr + ADM_DESC_ALIGN) >> 3;
dma_sync_single_for_device(adev->dev, cple_addr, sizeof(*cple),
DMA_TO_DEVICE);
return vchan_tx_prep(&achan->vc, &async_desc->vd, flags);
free:
kfree(async_desc);
return ERR_PTR(-ENOMEM);
}
/**
* adm_terminate_all - terminate all transactions on a channel
* @chan: dma channel
*
* Dequeues and frees all transactions, aborts current transaction
* No callbacks are done
*
*/
static int adm_terminate_all(struct dma_chan *chan)
{
struct adm_chan *achan = to_adm_chan(chan);
struct adm_device *adev = achan->adev;
unsigned long flags;
LIST_HEAD(head);
spin_lock_irqsave(&achan->vc.lock, flags);
vchan_get_all_descriptors(&achan->vc, &head);
/* send flush command to terminate current transaction */
writel_relaxed(0x0,
adev->regs + ADM_CH_FLUSH_STATE0(achan->id, adev->ee));
spin_unlock_irqrestore(&achan->vc.lock, flags);
vchan_dma_desc_free_list(&achan->vc, &head);
return 0;
}
static int adm_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg)
{
struct adm_chan *achan = to_adm_chan(chan);
unsigned long flag;
spin_lock_irqsave(&achan->vc.lock, flag);
memcpy(&achan->slave, cfg, sizeof(struct dma_slave_config));
spin_unlock_irqrestore(&achan->vc.lock, flag);
return 0;
}
/**
* adm_start_dma - start next transaction
* @achan: ADM dma channel
*/
static void adm_start_dma(struct adm_chan *achan)
{
struct virt_dma_desc *vd = vchan_next_desc(&achan->vc);
struct adm_device *adev = achan->adev;
struct adm_async_desc *async_desc;
lockdep_assert_held(&achan->vc.lock);
if (!vd)
return;
list_del(&vd->node);
/* write next command list out to the CMD FIFO */
async_desc = container_of(vd, struct adm_async_desc, vd);
achan->curr_txd = async_desc;
/* reset channel error */
achan->error = 0;
if (!achan->initialized) {
/* enable interrupts */
writel(ADM_CH_CONF_SHADOW_EN |
ADM_CH_CONF_PERM_MPU_CONF |
ADM_CH_CONF_MPU_DISABLE |
ADM_CH_CONF_SEC_DOMAIN(adev->ee),
adev->regs + ADM_CH_CONF(achan->id));
writel(ADM_CH_RSLT_CONF_IRQ_EN | ADM_CH_RSLT_CONF_FLUSH_EN,
adev->regs + ADM_CH_RSLT_CONF(achan->id, adev->ee));
achan->initialized = 1;
}
/* set the crci block size if this transaction requires CRCI */
if (async_desc->crci) {
writel(async_desc->mux | async_desc->blk_size,
adev->regs + ADM_CRCI_CTL(async_desc->crci, adev->ee));
}
/* make sure IRQ enable doesn't get reordered */
wmb();
/* write next command list out to the CMD FIFO */
writel(ALIGN(async_desc->dma_addr, ADM_DESC_ALIGN) >> 3,
adev->regs + ADM_CH_CMD_PTR(achan->id, adev->ee));
}
/**
* adm_dma_irq - irq handler for ADM controller
* @irq: IRQ of interrupt
* @data: callback data
*
* IRQ handler for the bam controller
*/
static irqreturn_t adm_dma_irq(int irq, void *data)
{
struct adm_device *adev = data;
u32 srcs, i;
struct adm_async_desc *async_desc;
unsigned long flags;
srcs = readl_relaxed(adev->regs +
ADM_SEC_DOMAIN_IRQ_STATUS(adev->ee));
for (i = 0; i < ADM_MAX_CHANNELS; i++) {
struct adm_chan *achan = &adev->channels[i];
u32 status, result;
if (srcs & BIT(i)) {
status = readl_relaxed(adev->regs +
ADM_CH_STATUS_SD(i, adev->ee));
/* if no result present, skip */
if (!(status & ADM_CH_STATUS_VALID))
continue;
result = readl_relaxed(adev->regs +
ADM_CH_RSLT(i, adev->ee));
/* no valid results, skip */
if (!(result & ADM_CH_RSLT_VALID))
continue;
/* flag error if transaction was flushed or failed */
if (result & (ADM_CH_RSLT_ERR | ADM_CH_RSLT_FLUSH))
achan->error = 1;
spin_lock_irqsave(&achan->vc.lock, flags);
async_desc = achan->curr_txd;
achan->curr_txd = NULL;
if (async_desc) {
vchan_cookie_complete(&async_desc->vd);
/* kick off next DMA */
adm_start_dma(achan);
}
spin_unlock_irqrestore(&achan->vc.lock, flags);
}
}
return IRQ_HANDLED;
}
/**
* adm_tx_status - returns status of transaction
* @chan: dma channel
* @cookie: transaction cookie
* @txstate: DMA transaction state
*
* Return status of dma transaction
*/
static enum dma_status adm_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
struct dma_tx_state *txstate)
{
struct adm_chan *achan = to_adm_chan(chan);
struct virt_dma_desc *vd;
enum dma_status ret;
unsigned long flags;
size_t residue = 0;
ret = dma_cookie_status(chan, cookie, txstate);
if (ret == DMA_COMPLETE || !txstate)
return ret;
spin_lock_irqsave(&achan->vc.lock, flags);
vd = vchan_find_desc(&achan->vc, cookie);
if (vd)
residue = container_of(vd, struct adm_async_desc, vd)->length;
spin_unlock_irqrestore(&achan->vc.lock, flags);
/*
* residue is either the full length if it is in the issued list, or 0
* if it is in progress. We have no reliable way of determining
* anything inbetween
*/
dma_set_residue(txstate, residue);
if (achan->error)
return DMA_ERROR;
return ret;
}
/**
* adm_issue_pending - starts pending transactions
* @chan: dma channel
*
* Issues all pending transactions and starts DMA
*/
static void adm_issue_pending(struct dma_chan *chan)
{
struct adm_chan *achan = to_adm_chan(chan);
unsigned long flags;
spin_lock_irqsave(&achan->vc.lock, flags);
if (vchan_issue_pending(&achan->vc) && !achan->curr_txd)
adm_start_dma(achan);
spin_unlock_irqrestore(&achan->vc.lock, flags);
}
/**
* adm_dma_free_desc - free descriptor memory
* @vd: virtual descriptor
*
*/
static void adm_dma_free_desc(struct virt_dma_desc *vd)
{
struct adm_async_desc *async_desc = container_of(vd,
struct adm_async_desc, vd);
dma_unmap_single(async_desc->adev->dev, async_desc->dma_addr,
async_desc->dma_len, DMA_TO_DEVICE);
kfree(async_desc->cpl);
kfree(async_desc);
}
static void adm_channel_init(struct adm_device *adev, struct adm_chan *achan,
u32 index)
{
achan->id = index;
achan->adev = adev;
vchan_init(&achan->vc, &adev->common);
achan->vc.desc_free = adm_dma_free_desc;
}
static int adm_dma_probe(struct platform_device *pdev)
{
struct adm_device *adev;
int ret;
u32 i;
adev = devm_kzalloc(&pdev->dev, sizeof(*adev), GFP_KERNEL);
if (!adev)
return -ENOMEM;
adev->dev = &pdev->dev;
adev->regs = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(adev->regs))
return PTR_ERR(adev->regs);
adev->irq = platform_get_irq(pdev, 0);
if (adev->irq < 0)
return adev->irq;
ret = of_property_read_u32(pdev->dev.of_node, "qcom,ee", &adev->ee);
if (ret) {
dev_err(adev->dev, "Execution environment unspecified\n");
return ret;
}
adev->core_clk = devm_clk_get(adev->dev, "core");
if (IS_ERR(adev->core_clk))
return PTR_ERR(adev->core_clk);
adev->iface_clk = devm_clk_get(adev->dev, "iface");
if (IS_ERR(adev->iface_clk))
return PTR_ERR(adev->iface_clk);
adev->clk_reset = devm_reset_control_get_exclusive(&pdev->dev, "clk");
if (IS_ERR(adev->clk_reset)) {
dev_err(adev->dev, "failed to get ADM0 reset\n");
return PTR_ERR(adev->clk_reset);
}
adev->c0_reset = devm_reset_control_get_exclusive(&pdev->dev, "c0");
if (IS_ERR(adev->c0_reset)) {
dev_err(adev->dev, "failed to get ADM0 C0 reset\n");
return PTR_ERR(adev->c0_reset);
}
adev->c1_reset = devm_reset_control_get_exclusive(&pdev->dev, "c1");
if (IS_ERR(adev->c1_reset)) {
dev_err(adev->dev, "failed to get ADM0 C1 reset\n");
return PTR_ERR(adev->c1_reset);
}
adev->c2_reset = devm_reset_control_get_exclusive(&pdev->dev, "c2");
if (IS_ERR(adev->c2_reset)) {
dev_err(adev->dev, "failed to get ADM0 C2 reset\n");
return PTR_ERR(adev->c2_reset);
}
ret = clk_prepare_enable(adev->core_clk);
if (ret) {
dev_err(adev->dev, "failed to prepare/enable core clock\n");
return ret;
}
ret = clk_prepare_enable(adev->iface_clk);
if (ret) {
dev_err(adev->dev, "failed to prepare/enable iface clock\n");
goto err_disable_core_clk;
}
reset_control_assert(adev->clk_reset);
reset_control_assert(adev->c0_reset);
reset_control_assert(adev->c1_reset);
reset_control_assert(adev->c2_reset);
udelay(2);
reset_control_deassert(adev->clk_reset);
reset_control_deassert(adev->c0_reset);
reset_control_deassert(adev->c1_reset);
reset_control_deassert(adev->c2_reset);
adev->channels = devm_kcalloc(adev->dev, ADM_MAX_CHANNELS,
sizeof(*adev->channels), GFP_KERNEL);
if (!adev->channels) {
ret = -ENOMEM;
goto err_disable_clks;
}
/* allocate and initialize channels */
INIT_LIST_HEAD(&adev->common.channels);
for (i = 0; i < ADM_MAX_CHANNELS; i++)
adm_channel_init(adev, &adev->channels[i], i);
/* reset CRCIs */
for (i = 0; i < 16; i++)
writel(ADM_CRCI_CTL_RST, adev->regs +
ADM_CRCI_CTL(i, adev->ee));
/* configure client interfaces */
writel(ADM_CI_RANGE_START(0x40) | ADM_CI_RANGE_END(0xb0) |
ADM_CI_BURST_8_WORDS, adev->regs + ADM_CI_CONF(0));
writel(ADM_CI_RANGE_START(0x2a) | ADM_CI_RANGE_END(0x2c) |
ADM_CI_BURST_8_WORDS, adev->regs + ADM_CI_CONF(1));
writel(ADM_CI_RANGE_START(0x12) | ADM_CI_RANGE_END(0x28) |
ADM_CI_BURST_8_WORDS, adev->regs + ADM_CI_CONF(2));
writel(ADM_GP_CTL_LP_EN | ADM_GP_CTL_LP_CNT(0xf),
adev->regs + ADM_GP_CTL);
ret = devm_request_irq(adev->dev, adev->irq, adm_dma_irq,
0, "adm_dma", adev);
if (ret)
goto err_disable_clks;
platform_set_drvdata(pdev, adev);
adev->common.dev = adev->dev;
adev->common.dev->dma_parms = &adev->dma_parms;
/* set capabilities */
dma_cap_zero(adev->common.cap_mask);
dma_cap_set(DMA_SLAVE, adev->common.cap_mask);
dma_cap_set(DMA_PRIVATE, adev->common.cap_mask);
/* initialize dmaengine apis */
adev->common.directions = BIT(DMA_DEV_TO_MEM | DMA_MEM_TO_DEV);
adev->common.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
adev->common.src_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES;
adev->common.dst_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES;
adev->common.device_free_chan_resources = adm_free_chan;
adev->common.device_prep_slave_sg = adm_prep_slave_sg;
adev->common.device_issue_pending = adm_issue_pending;
adev->common.device_tx_status = adm_tx_status;
adev->common.device_terminate_all = adm_terminate_all;
adev->common.device_config = adm_slave_config;
ret = dma_async_device_register(&adev->common);
if (ret) {
dev_err(adev->dev, "failed to register dma async device\n");
goto err_disable_clks;
}
ret = of_dma_controller_register(pdev->dev.of_node,
of_dma_xlate_by_chan_id,
&adev->common);
if (ret)
goto err_unregister_dma;
return 0;
err_unregister_dma:
dma_async_device_unregister(&adev->common);
err_disable_clks:
clk_disable_unprepare(adev->iface_clk);
err_disable_core_clk:
clk_disable_unprepare(adev->core_clk);
return ret;
}
static int adm_dma_remove(struct platform_device *pdev)
{
struct adm_device *adev = platform_get_drvdata(pdev);
struct adm_chan *achan;
u32 i;
of_dma_controller_free(pdev->dev.of_node);
dma_async_device_unregister(&adev->common);
for (i = 0; i < ADM_MAX_CHANNELS; i++) {
achan = &adev->channels[i];
/* mask IRQs for this channel/EE pair */
writel(0, adev->regs + ADM_CH_RSLT_CONF(achan->id, adev->ee));
tasklet_kill(&adev->channels[i].vc.task);
adm_terminate_all(&adev->channels[i].vc.chan);
}
devm_free_irq(adev->dev, adev->irq, adev);
clk_disable_unprepare(adev->core_clk);
clk_disable_unprepare(adev->iface_clk);
return 0;
}
static const struct of_device_id adm_of_match[] = {
{ .compatible = "qcom,adm", },
{}
};
MODULE_DEVICE_TABLE(of, adm_of_match);
static struct platform_driver adm_dma_driver = {
.probe = adm_dma_probe,
.remove = adm_dma_remove,
.driver = {
.name = "adm-dma-engine",
.of_match_table = adm_of_match,
},
};
module_platform_driver(adm_dma_driver);
MODULE_AUTHOR("Andy Gross <agross@codeaurora.org>");
MODULE_DESCRIPTION("QCOM ADM DMA engine driver");
MODULE_LICENSE("GPL v2");

12
drivers/dma/sf-pdma/sf-pdma.c
View File

@@ -326,10 +326,9 @@ static irqreturn_t sf_pdma_done_isr(int irq, void *dev_id)
{
struct sf_pdma_chan *chan = dev_id;
struct pdma_regs *regs = &chan->regs;
unsigned long flags;
u64 residue;
spin_lock_irqsave(&chan->vchan.lock, flags);
spin_lock(&chan->vchan.lock);
writel((readl(regs->ctrl)) & ~PDMA_DONE_STATUS_MASK, regs->ctrl);
residue = readq(regs->residue);
@@ -346,7 +345,7 @@ static irqreturn_t sf_pdma_done_isr(int irq, void *dev_id)
sf_pdma_xfer_desc(chan);
}
spin_unlock_irqrestore(&chan->vchan.lock, flags);
spin_unlock(&chan->vchan.lock);
return IRQ_HANDLED;
}
@@ -355,11 +354,10 @@ static irqreturn_t sf_pdma_err_isr(int irq, void *dev_id)
{
struct sf_pdma_chan *chan = dev_id;
struct pdma_regs *regs = &chan->regs;
unsigned long flags;
spin_lock_irqsave(&chan->lock, flags);
spin_lock(&chan->lock);
writel((readl(regs->ctrl)) & ~PDMA_ERR_STATUS_MASK, regs->ctrl);
spin_unlock_irqrestore(&chan->lock, flags);
spin_unlock(&chan->lock);
tasklet_schedule(&chan->err_tasklet);
@@ -584,7 +582,7 @@ static struct platform_driver sf_pdma_driver = {
.remove = sf_pdma_remove,
.driver = {
.name = "sf-pdma",
.of_match_table = of_match_ptr(sf_pdma_dt_ids),
.of_match_table = sf_pdma_dt_ids,
},
};

5
drivers/dma/ste_dma40.c
View File

@@ -1643,13 +1643,12 @@ static irqreturn_t d40_handle_interrupt(int irq, void *data)
u32 row;
long chan = -1;
struct d40_chan *d40c;
unsigned long flags;
struct d40_base *base = data;
u32 *regs = base->regs_interrupt;
struct d40_interrupt_lookup *il = base->gen_dmac.il;
u32 il_size = base->gen_dmac.il_size;
spin_lock_irqsave(&base->interrupt_lock, flags);
spin_lock(&base->interrupt_lock);
/* Read interrupt status of both logical and physical channels */
for (i = 0; i < il_size; i++)
@@ -1694,7 +1693,7 @@ static irqreturn_t d40_handle_interrupt(int irq, void *data)
spin_unlock(&d40c->lock);
}
spin_unlock_irqrestore(&base->interrupt_lock, flags);
spin_unlock(&base->interrupt_lock);
return IRQ_HANDLED;
}

47
drivers/dma/stm32-dma.c
View File

@@ -264,9 +264,11 @@ static int stm32_dma_get_width(struct stm32_dma_chan *chan,
}
static enum dma_slave_buswidth stm32_dma_get_max_width(u32 buf_len,
dma_addr_t buf_addr,
u32 threshold)
{
enum dma_slave_buswidth max_width;
u64 addr = buf_addr;
if (threshold == STM32_DMA_FIFO_THRESHOLD_FULL)
max_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
@@ -277,6 +279,9 @@ static enum dma_slave_buswidth stm32_dma_get_max_width(u32 buf_len,
max_width > DMA_SLAVE_BUSWIDTH_1_BYTE)
max_width = max_width >> 1;
if (do_div(addr, max_width))
max_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
return max_width;
}
@@ -648,21 +653,12 @@ static irqreturn_t stm32_dma_chan_irq(int irq, void *devid)
scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id));
sfcr = stm32_dma_read(dmadev, STM32_DMA_SFCR(chan->id));
if (status & STM32_DMA_TCI) {
stm32_dma_irq_clear(chan, STM32_DMA_TCI);
if (scr & STM32_DMA_SCR_TCIE)
stm32_dma_handle_chan_done(chan);
status &= ~STM32_DMA_TCI;
}
if (status & STM32_DMA_HTI) {
stm32_dma_irq_clear(chan, STM32_DMA_HTI);
status &= ~STM32_DMA_HTI;
}
if (status & STM32_DMA_FEI) {
stm32_dma_irq_clear(chan, STM32_DMA_FEI);
status &= ~STM32_DMA_FEI;
if (sfcr & STM32_DMA_SFCR_FEIE) {
if (!(scr & STM32_DMA_SCR_EN))
if (!(scr & STM32_DMA_SCR_EN) &&
!(status & STM32_DMA_TCI))
dev_err(chan2dev(chan), "FIFO Error\n");
else
dev_dbg(chan2dev(chan), "FIFO over/underrun\n");
@@ -674,6 +670,19 @@ static irqreturn_t stm32_dma_chan_irq(int irq, void *devid)
if (sfcr & STM32_DMA_SCR_DMEIE)
dev_dbg(chan2dev(chan), "Direct mode overrun\n");
}
if (status & STM32_DMA_TCI) {
stm32_dma_irq_clear(chan, STM32_DMA_TCI);
if (scr & STM32_DMA_SCR_TCIE)
stm32_dma_handle_chan_done(chan);
status &= ~STM32_DMA_TCI;
}
if (status & STM32_DMA_HTI) {
stm32_dma_irq_clear(chan, STM32_DMA_HTI);
status &= ~STM32_DMA_HTI;
}
if (status) {
stm32_dma_irq_clear(chan, status);
dev_err(chan2dev(chan), "DMA error: status=0x%08x\n", status);
@@ -703,7 +712,7 @@ static void stm32_dma_issue_pending(struct dma_chan *c)
static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan,
enum dma_transfer_direction direction,
enum dma_slave_buswidth *buswidth,
u32 buf_len)
u32 buf_len, dma_addr_t buf_addr)
{
enum dma_slave_buswidth src_addr_width, dst_addr_width;
int src_bus_width, dst_bus_width;
@@ -735,7 +744,8 @@ static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan,
return dst_burst_size;
/* Set memory data size */
src_addr_width = stm32_dma_get_max_width(buf_len, fifoth);
src_addr_width = stm32_dma_get_max_width(buf_len, buf_addr,
fifoth);
chan->mem_width = src_addr_width;
src_bus_width = stm32_dma_get_width(chan, src_addr_width);
if (src_bus_width < 0)
@@ -784,7 +794,8 @@ static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan,
return src_burst_size;
/* Set memory data size */
dst_addr_width = stm32_dma_get_max_width(buf_len, fifoth);
dst_addr_width = stm32_dma_get_max_width(buf_len, buf_addr,
fifoth);
chan->mem_width = dst_addr_width;
dst_bus_width = stm32_dma_get_width(chan, dst_addr_width);
if (dst_bus_width < 0)
@@ -872,7 +883,8 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_slave_sg(
for_each_sg(sgl, sg, sg_len, i) {
ret = stm32_dma_set_xfer_param(chan, direction, &buswidth,
sg_dma_len(sg));
sg_dma_len(sg),
sg_dma_address(sg));
if (ret < 0)
goto err;
@@ -940,7 +952,8 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_dma_cyclic(
return NULL;
}
ret = stm32_dma_set_xfer_param(chan, direction, &buswidth, period_len);
ret = stm32_dma_set_xfer_param(chan, direction, &buswidth, period_len,
buf_addr);
if (ret < 0)
return NULL;
@@ -1216,6 +1229,8 @@ static void stm32_dma_free_chan_resources(struct dma_chan *c)
pm_runtime_put(dmadev->ddev.dev);
vchan_free_chan_resources(to_virt_chan(c));
stm32_dma_clear_reg(&chan->chan_reg);
chan->threshold = 0;
}
static void stm32_dma_desc_free(struct virt_dma_desc *vdesc)

2
drivers/dma/stm32-dmamux.c
View File

@@ -168,7 +168,7 @@ error_chan_id:
return ERR_PTR(ret);
}
static const struct of_device_id stm32_stm32dma_master_match[] = {
static const struct of_device_id stm32_stm32dma_master_match[] __maybe_unused = {
{ .compatible = "st,stm32-dma", },
{},
};

66
drivers/dma/stm32-mdma.c
View File

@@ -339,7 +339,7 @@ static struct stm32_mdma_desc *stm32_mdma_alloc_desc(
struct stm32_mdma_desc *desc;
int i;
desc = kzalloc(offsetof(typeof(*desc), node[count]), GFP_NOWAIT);
desc = kzalloc(struct_size(desc, node, count), GFP_NOWAIT);
if (!desc)
return NULL;
@@ -1346,7 +1346,7 @@ static irqreturn_t stm32_mdma_irq_handler(int irq, void *devid)
{
struct stm32_mdma_device *dmadev = devid;
struct stm32_mdma_chan *chan = devid;
u32 reg, id, ien, status, flag;
u32 reg, id, ccr, ien, status;
/* Find out which channel generates the interrupt */
status = readl_relaxed(dmadev->base + STM32_MDMA_GISR0);
@@ -1368,67 +1368,71 @@ static irqreturn_t stm32_mdma_irq_handler(int irq, void *devid)
chan = &dmadev->chan[id];
if (!chan) {
dev_dbg(mdma2dev(dmadev), "MDMA channel not initialized\n");
goto exit;
dev_warn(mdma2dev(dmadev), "MDMA channel not initialized\n");
return IRQ_NONE;
}
/* Handle interrupt for the channel */
spin_lock(&chan->vchan.lock);
status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(chan->id));
ien = stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id));
ien &= STM32_MDMA_CCR_IRQ_MASK;
ien >>= 1;
status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(id));
/* Mask Channel ReQuest Active bit which can be set in case of MEM2MEM */
status &= ~STM32_MDMA_CISR_CRQA;
ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(id));
ien = (ccr & STM32_MDMA_CCR_IRQ_MASK) >> 1;
if (!(status & ien)) {
spin_unlock(&chan->vchan.lock);
dev_dbg(chan2dev(chan),
"spurious it (status=0x%04x, ien=0x%04x)\n",
status, ien);
dev_warn(chan2dev(chan),
"spurious it (status=0x%04x, ien=0x%04x)\n",
status, ien);
return IRQ_NONE;
}
flag = __ffs(status & ien);
reg = STM32_MDMA_CIFCR(chan->id);
reg = STM32_MDMA_CIFCR(id);
switch (1 << flag) {
case STM32_MDMA_CISR_TEIF:
id = chan->id;
status = readl_relaxed(dmadev->base + STM32_MDMA_CESR(id));
dev_err(chan2dev(chan), "Transfer Err: stat=0x%08x\n", status);
if (status & STM32_MDMA_CISR_TEIF) {
dev_err(chan2dev(chan), "Transfer Err: stat=0x%08x\n",
readl_relaxed(dmadev->base + STM32_MDMA_CESR(id)));
stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CTEIF);
break;
status &= ~STM32_MDMA_CISR_TEIF;
}
case STM32_MDMA_CISR_CTCIF:
if (status & STM32_MDMA_CISR_CTCIF) {
stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CCTCIF);
status &= ~STM32_MDMA_CISR_CTCIF;
stm32_mdma_xfer_end(chan);
break;
}
case STM32_MDMA_CISR_BRTIF:
if (status & STM32_MDMA_CISR_BRTIF) {
stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CBRTIF);
break;
status &= ~STM32_MDMA_CISR_BRTIF;
}
case STM32_MDMA_CISR_BTIF:
if (status & STM32_MDMA_CISR_BTIF) {
stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CBTIF);
status &= ~STM32_MDMA_CISR_BTIF;
chan->curr_hwdesc++;
if (chan->desc && chan->desc->cyclic) {
if (chan->curr_hwdesc == chan->desc->count)
chan->curr_hwdesc = 0;
vchan_cyclic_callback(&chan->desc->vdesc);
}
break;
}
case STM32_MDMA_CISR_TCIF:
if (status & STM32_MDMA_CISR_TCIF) {
stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CLTCIF);
break;
status &= ~STM32_MDMA_CISR_TCIF;
}
default:
dev_err(chan2dev(chan), "it %d unhandled (status=0x%04x)\n",
1 << flag, status);
if (status) {
stm32_mdma_set_bits(dmadev, reg, status);
dev_err(chan2dev(chan), "DMA error: status=0x%08x\n", status);
if (!(ccr & STM32_MDMA_CCR_EN))
dev_err(chan2dev(chan), "chan disabled by HW\n");
}
spin_unlock(&chan->vchan.lock);
exit:
return IRQ_HANDLED;
}

25
drivers/dma/sun6i-dma.c
View File

@@ -1173,6 +1173,30 @@ static struct sun6i_dma_config sun50i_a64_dma_cfg = {
BIT(DMA_SLAVE_BUSWIDTH_8_BYTES),
};
/*
* TODO: Add support for more than 4g physical addressing.
*
* The A100 binding uses the number of dma channels from the
* device tree node.
*/
static struct sun6i_dma_config sun50i_a100_dma_cfg = {
.clock_autogate_enable = sun6i_enable_clock_autogate_h3,
.set_burst_length = sun6i_set_burst_length_h3,
.set_drq = sun6i_set_drq_h6,
.set_mode = sun6i_set_mode_h6,
.src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16),
.dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16),
.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
BIT(DMA_SLAVE_BUSWIDTH_8_BYTES),
.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
BIT(DMA_SLAVE_BUSWIDTH_8_BYTES),
.has_mbus_clk = true,
};
/*
* The H6 binding uses the number of dma channels from the
* device tree node.
@@ -1225,6 +1249,7 @@ static const struct of_device_id sun6i_dma_match[] = {
{ .compatible = "allwinner,sun8i-h3-dma", .data = &sun8i_h3_dma_cfg },
{ .compatible = "allwinner,sun8i-v3s-dma", .data = &sun8i_v3s_dma_cfg },
{ .compatible = "allwinner,sun50i-a64-dma", .data = &sun50i_a64_dma_cfg },
{ .compatible = "allwinner,sun50i-a100-dma", .data = &sun50i_a100_dma_cfg },
{ .compatible = "allwinner,sun50i-h6-dma", .data = &sun50i_h6_dma_cfg },
{ /* sentinel */ }
};

7
drivers/dma/tegra210-adma.c
View File

@@ -408,19 +408,18 @@ static irqreturn_t tegra_adma_isr(int irq, void *dev_id)
{
struct tegra_adma_chan *tdc = dev_id;
unsigned long status;
unsigned long flags;
spin_lock_irqsave(&tdc->vc.lock, flags);
spin_lock(&tdc->vc.lock);
status = tegra_adma_irq_clear(tdc);
if (status == 0 || !tdc->desc) {
spin_unlock_irqrestore(&tdc->vc.lock, flags);
spin_unlock(&tdc->vc.lock);
return IRQ_NONE;
}
vchan_cyclic_callback(&tdc->desc->vd);
spin_unlock_irqrestore(&tdc->vc.lock, flags);
spin_unlock(&tdc->vc.lock);
return IRQ_HANDLED;
}

3
drivers/dma/ti/Makefile
View File

@@ -7,5 +7,6 @@ obj-$(CONFIG_TI_K3_UDMA_GLUE_LAYER) += k3-udma-glue.o
obj-$(CONFIG_TI_K3_PSIL) += k3-psil.o \
k3-psil-am654.o \
k3-psil-j721e.o \
k3-psil-j7200.o
k3-psil-j7200.o \
k3-psil-am64.o
obj-$(CONFIG_TI_DMA_CROSSBAR) += dma-crossbar.o

6
drivers/dma/ti/dma-crossbar.c
View File

@@ -122,7 +122,7 @@ static void *ti_am335x_xbar_route_allocate(struct of_phandle_args *dma_spec,
return map;
}
static const struct of_device_id ti_am335x_master_match[] = {
static const struct of_device_id ti_am335x_master_match[] __maybe_unused = {
{ .compatible = "ti,edma3-tpcc", },
{},
};
@@ -292,7 +292,7 @@ static const u32 ti_dma_offset[] = {
[TI_XBAR_SDMA_OFFSET] = 1,
};
static const struct of_device_id ti_dra7_master_match[] = {
static const struct of_device_id ti_dra7_master_match[] __maybe_unused = {
{
.compatible = "ti,omap4430-sdma",
.data = &ti_dma_offset[TI_XBAR_SDMA_OFFSET],
@@ -460,7 +460,7 @@ static int ti_dma_xbar_probe(struct platform_device *pdev)
static struct platform_driver ti_dma_xbar_driver = {
.driver = {
.name = "ti-dma-crossbar",
.of_match_table = of_match_ptr(ti_dma_xbar_match),
.of_match_table = ti_dma_xbar_match,
},
.probe = ti_dma_xbar_probe,
};

158
drivers/dma/ti/k3-psil-am64.c Normal file
View File

@@ -0,0 +1,158 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2020 Texas Instruments Incorporated - https://www.ti.com
* Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
*/
#include <linux/kernel.h>
#include "k3-psil-priv.h"
#define PSIL_PDMA_XY_TR(x) \
{ \
.thread_id = x, \
.ep_config = { \
.ep_type = PSIL_EP_PDMA_XY, \
.mapped_channel_id = -1, \
.default_flow_id = -1, \
}, \
}
#define PSIL_PDMA_XY_PKT(x) \
{ \
.thread_id = x, \
.ep_config = { \
.ep_type = PSIL_EP_PDMA_XY, \
.mapped_channel_id = -1, \
.default_flow_id = -1, \
.pkt_mode = 1, \
}, \
}
#define PSIL_ETHERNET(x, ch, flow_base, flow_cnt) \
{ \
.thread_id = x, \
.ep_config = { \
.ep_type = PSIL_EP_NATIVE, \
.pkt_mode = 1, \
.needs_epib = 1, \
.psd_size = 16, \
.mapped_channel_id = ch, \
.flow_start = flow_base, \
.flow_num = flow_cnt, \
.default_flow_id = flow_base, \
}, \
}
#define PSIL_SAUL(x, ch, flow_base, flow_cnt, default_flow, tx) \
{ \
.thread_id = x, \
.ep_config = { \
.ep_type = PSIL_EP_NATIVE, \
.pkt_mode = 1, \
.needs_epib = 1, \
.psd_size = 64, \
.mapped_channel_id = ch, \
.flow_start = flow_base, \
.flow_num = flow_cnt, \
.default_flow_id = default_flow, \
.notdpkt = tx, \
}, \
}
/* PSI-L source thread IDs, used for RX (DMA_DEV_TO_MEM) */
static struct psil_ep am64_src_ep_map[] = {
/* SAUL */
PSIL_SAUL(0x4000, 17, 32, 8, 32, 0),
PSIL_SAUL(0x4001, 18, 32, 8, 33, 0),
PSIL_SAUL(0x4002, 19, 40, 8, 40, 0),
PSIL_SAUL(0x4003, 20, 40, 8, 41, 0),
/* ICSS_G0 */
PSIL_ETHERNET(0x4100, 21, 48, 16),
PSIL_ETHERNET(0x4101, 22, 64, 16),
PSIL_ETHERNET(0x4102, 23, 80, 16),
PSIL_ETHERNET(0x4103, 24, 96, 16),
/* ICSS_G1 */
PSIL_ETHERNET(0x4200, 25, 112, 16),
PSIL_ETHERNET(0x4201, 26, 128, 16),
PSIL_ETHERNET(0x4202, 27, 144, 16),
PSIL_ETHERNET(0x4203, 28, 160, 16),
/* PDMA_MAIN0 - SPI0-3 */
PSIL_PDMA_XY_PKT(0x4300),
PSIL_PDMA_XY_PKT(0x4301),
PSIL_PDMA_XY_PKT(0x4302),
PSIL_PDMA_XY_PKT(0x4303),
PSIL_PDMA_XY_PKT(0x4304),
PSIL_PDMA_XY_PKT(0x4305),
PSIL_PDMA_XY_PKT(0x4306),
PSIL_PDMA_XY_PKT(0x4307),
PSIL_PDMA_XY_PKT(0x4308),
PSIL_PDMA_XY_PKT(0x4309),
PSIL_PDMA_XY_PKT(0x430a),
PSIL_PDMA_XY_PKT(0x430b),
PSIL_PDMA_XY_PKT(0x430c),
PSIL_PDMA_XY_PKT(0x430d),
PSIL_PDMA_XY_PKT(0x430e),
PSIL_PDMA_XY_PKT(0x430f),
/* PDMA_MAIN0 - USART0-1 */
PSIL_PDMA_XY_PKT(0x4310),
PSIL_PDMA_XY_PKT(0x4311),
/* PDMA_MAIN1 - SPI4 */
PSIL_PDMA_XY_PKT(0x4400),
PSIL_PDMA_XY_PKT(0x4401),
PSIL_PDMA_XY_PKT(0x4402),
PSIL_PDMA_XY_PKT(0x4403),
/* PDMA_MAIN1 - USART2-6 */
PSIL_PDMA_XY_PKT(0x4404),
PSIL_PDMA_XY_PKT(0x4405),
PSIL_PDMA_XY_PKT(0x4406),
PSIL_PDMA_XY_PKT(0x4407),
PSIL_PDMA_XY_PKT(0x4408),
/* PDMA_MAIN1 - ADCs */
PSIL_PDMA_XY_TR(0x440f),
PSIL_PDMA_XY_TR(0x4410),
/* CPSW2 */
PSIL_ETHERNET(0x4500, 16, 16, 16),
};
/* PSI-L destination thread IDs, used for TX (DMA_MEM_TO_DEV) */
static struct psil_ep am64_dst_ep_map[] = {
/* SAUL */
PSIL_SAUL(0xc000, 24, 80, 8, 80, 1),
PSIL_SAUL(0xc001, 25, 88, 8, 88, 1),
/* ICSS_G0 */
PSIL_ETHERNET(0xc100, 26, 96, 1),
PSIL_ETHERNET(0xc101, 27, 97, 1),
PSIL_ETHERNET(0xc102, 28, 98, 1),
PSIL_ETHERNET(0xc103, 29, 99, 1),
PSIL_ETHERNET(0xc104, 30, 100, 1),
PSIL_ETHERNET(0xc105, 31, 101, 1),
PSIL_ETHERNET(0xc106, 32, 102, 1),
PSIL_ETHERNET(0xc107, 33, 103, 1),
/* ICSS_G1 */
PSIL_ETHERNET(0xc200, 34, 104, 1),
PSIL_ETHERNET(0xc201, 35, 105, 1),
PSIL_ETHERNET(0xc202, 36, 106, 1),
PSIL_ETHERNET(0xc203, 37, 107, 1),
PSIL_ETHERNET(0xc204, 38, 108, 1),
PSIL_ETHERNET(0xc205, 39, 109, 1),
PSIL_ETHERNET(0xc206, 40, 110, 1),
PSIL_ETHERNET(0xc207, 41, 111, 1),
/* CPSW2 */
PSIL_ETHERNET(0xc500, 16, 16, 8),
PSIL_ETHERNET(0xc501, 17, 24, 8),
PSIL_ETHERNET(0xc502, 18, 32, 8),
PSIL_ETHERNET(0xc503, 19, 40, 8),
PSIL_ETHERNET(0xc504, 20, 48, 8),
PSIL_ETHERNET(0xc505, 21, 56, 8),
PSIL_ETHERNET(0xc506, 22, 64, 8),
PSIL_ETHERNET(0xc507, 23, 72, 8),
};
struct psil_ep_map am64_ep_map = {
.name = "am64",
.src = am64_src_ep_map,
.src_count = ARRAY_SIZE(am64_src_ep_map),
.dst = am64_dst_ep_map,
.dst_count = ARRAY_SIZE(am64_dst_ep_map),
};

1
drivers/dma/ti/k3-psil-priv.h
View File

@@ -40,5 +40,6 @@ struct psil_endpoint_config *psil_get_ep_config(u32 thread_id);
extern struct psil_ep_map am654_ep_map;
extern struct psil_ep_map j721e_ep_map;
extern struct psil_ep_map j7200_ep_map;
extern struct psil_ep_map am64_ep_map;
#endif /* K3_PSIL_PRIV_H_ */

1
drivers/dma/ti/k3-psil.c
View File

@@ -20,6 +20,7 @@ static const struct soc_device_attribute k3_soc_devices[] = {
{ .family = "AM65X", .data = &am654_ep_map },
{ .family = "J721E", .data = &j721e_ep_map },
{ .family = "J7200", .data = &j7200_ep_map },
{ .family = "AM64X", .data = &am64_ep_map },
{ /* sentinel */ }
};

383
drivers/dma/ti/k3-udma-glue.c
View File

@@ -22,6 +22,7 @@
struct k3_udma_glue_common {
struct device *dev;
struct device chan_dev;
struct udma_dev *udmax;
const struct udma_tisci_rm *tisci_rm;
struct k3_ringacc *ringacc;
@@ -32,7 +33,8 @@ struct k3_udma_glue_common {
bool epib;
u32 psdata_size;
u32 swdata_size;
u32 atype;
u32 atype_asel;
struct psil_endpoint_config *ep_config;
};
struct k3_udma_glue_tx_channel {
@@ -53,6 +55,8 @@ struct k3_udma_glue_tx_channel {
bool tx_filt_einfo;
bool tx_filt_pswords;
bool tx_supr_tdpkt;
int udma_tflow_id;
};
struct k3_udma_glue_rx_flow {
@@ -81,20 +85,26 @@ struct k3_udma_glue_rx_channel {
u32 flows_ready;
};
static void k3_udma_chan_dev_release(struct device *dev)
{
/* The struct containing the device is devm managed */
}
static struct class k3_udma_glue_devclass = {
.name = "k3_udma_glue_chan",
.dev_release = k3_udma_chan_dev_release,
};
#define K3_UDMAX_TDOWN_TIMEOUT_US 1000
static int of_k3_udma_glue_parse(struct device_node *udmax_np,
struct k3_udma_glue_common *common)
{
common->ringacc = of_k3_ringacc_get_by_phandle(udmax_np,
"ti,ringacc");
if (IS_ERR(common->ringacc))
return PTR_ERR(common->ringacc);
common->udmax = of_xudma_dev_get(udmax_np, NULL);
if (IS_ERR(common->udmax))
return PTR_ERR(common->udmax);
common->ringacc = xudma_get_ringacc(common->udmax);
common->tisci_rm = xudma_dev_get_tisci_rm(common->udmax);
return 0;
@@ -104,7 +114,6 @@ static int of_k3_udma_glue_parse_chn(struct device_node *chn_np,
const char *name, struct k3_udma_glue_common *common,
bool tx_chn)
{
struct psil_endpoint_config *ep_config;
struct of_phandle_args dma_spec;
u32 thread_id;
int ret = 0;
@@ -121,15 +130,26 @@ static int of_k3_udma_glue_parse_chn(struct device_node *chn_np,
&dma_spec))
return -ENOENT;
ret = of_k3_udma_glue_parse(dma_spec.np, common);
if (ret)
goto out_put_spec;
thread_id = dma_spec.args[0];
if (dma_spec.args_count == 2) {
if (dma_spec.args[1] > 2) {
if (dma_spec.args[1] > 2 && !xudma_is_pktdma(common->udmax)) {
dev_err(common->dev, "Invalid channel atype: %u\n",
dma_spec.args[1]);
ret = -EINVAL;
goto out_put_spec;
}
common->atype = dma_spec.args[1];
if (dma_spec.args[1] > 15 && xudma_is_pktdma(common->udmax)) {
dev_err(common->dev, "Invalid channel asel: %u\n",
dma_spec.args[1]);
ret = -EINVAL;
goto out_put_spec;
}
common->atype_asel = dma_spec.args[1];
}
if (tx_chn && !(thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) {
@@ -143,25 +163,23 @@ static int of_k3_udma_glue_parse_chn(struct device_node *chn_np,
}
/* get psil endpoint config */
ep_config = psil_get_ep_config(thread_id);
if (IS_ERR(ep_config)) {
common->ep_config = psil_get_ep_config(thread_id);
if (IS_ERR(common->ep_config)) {
dev_err(common->dev,
"No configuration for psi-l thread 0x%04x\n",
thread_id);
ret = PTR_ERR(ep_config);
ret = PTR_ERR(common->ep_config);
goto out_put_spec;
}
common->epib = ep_config->needs_epib;
common->psdata_size = ep_config->psd_size;
common->epib = common->ep_config->needs_epib;
common->psdata_size = common->ep_config->psd_size;
if (tx_chn)
common->dst_thread = thread_id;
else
common->src_thread = thread_id;
ret = of_k3_udma_glue_parse(dma_spec.np, common);
out_put_spec:
of_node_put(dma_spec.np);
return ret;
@@ -227,7 +245,7 @@ static int k3_udma_glue_cfg_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
req.tx_supr_tdpkt = 1;
req.tx_fetch_size = tx_chn->common.hdesc_size >> 2;
req.txcq_qnum = k3_ringacc_get_ring_id(tx_chn->ringtxcq);
req.tx_atype = tx_chn->common.atype;
req.tx_atype = tx_chn->common.atype_asel;
return tisci_rm->tisci_udmap_ops->tx_ch_cfg(tisci_rm->tisci, &req);
}
@@ -259,8 +277,14 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev,
tx_chn->common.psdata_size,
tx_chn->common.swdata_size);
if (xudma_is_pktdma(tx_chn->common.udmax))
tx_chn->udma_tchan_id = tx_chn->common.ep_config->mapped_channel_id;
else
tx_chn->udma_tchan_id = -1;
/* request and cfg UDMAP TX channel */
tx_chn->udma_tchanx = xudma_tchan_get(tx_chn->common.udmax, -1);
tx_chn->udma_tchanx = xudma_tchan_get(tx_chn->common.udmax,
tx_chn->udma_tchan_id);
if (IS_ERR(tx_chn->udma_tchanx)) {
ret = PTR_ERR(tx_chn->udma_tchanx);
dev_err(dev, "UDMAX tchanx get err %d\n", ret);
@@ -268,11 +292,34 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev,
}
tx_chn->udma_tchan_id = xudma_tchan_get_id(tx_chn->udma_tchanx);
tx_chn->common.chan_dev.class = &k3_udma_glue_devclass;
tx_chn->common.chan_dev.parent = xudma_get_device(tx_chn->common.udmax);
dev_set_name(&tx_chn->common.chan_dev, "tchan%d-0x%04x",
tx_chn->udma_tchan_id, tx_chn->common.dst_thread);
ret = device_register(&tx_chn->common.chan_dev);
if (ret) {
dev_err(dev, "Channel Device registration failed %d\n", ret);
tx_chn->common.chan_dev.parent = NULL;
goto err;
}
if (xudma_is_pktdma(tx_chn->common.udmax)) {
/* prepare the channel device as coherent */
tx_chn->common.chan_dev.dma_coherent = true;
dma_coerce_mask_and_coherent(&tx_chn->common.chan_dev,
DMA_BIT_MASK(48));
}
atomic_set(&tx_chn->free_pkts, cfg->txcq_cfg.size);
if (xudma_is_pktdma(tx_chn->common.udmax))
tx_chn->udma_tflow_id = tx_chn->common.ep_config->default_flow_id;
else
tx_chn->udma_tflow_id = tx_chn->udma_tchan_id;
/* request and cfg rings */
ret = k3_ringacc_request_rings_pair(tx_chn->common.ringacc,
tx_chn->udma_tchan_id, -1,
tx_chn->udma_tflow_id, -1,
&tx_chn->ringtx,
&tx_chn->ringtxcq);
if (ret) {
@@ -280,6 +327,16 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev,
goto err;
}
/* Set the dma_dev for the rings to be configured */
cfg->tx_cfg.dma_dev = k3_udma_glue_tx_get_dma_device(tx_chn);
cfg->txcq_cfg.dma_dev = cfg->tx_cfg.dma_dev;
/* Set the ASEL value for DMA rings of PKTDMA */
if (xudma_is_pktdma(tx_chn->common.udmax)) {
cfg->tx_cfg.asel = tx_chn->common.atype_asel;
cfg->txcq_cfg.asel = tx_chn->common.atype_asel;
}
ret = k3_ringacc_ring_cfg(tx_chn->ringtx, &cfg->tx_cfg);
if (ret) {
dev_err(dev, "Failed to cfg ringtx %d\n", ret);
@@ -303,19 +360,6 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev,
goto err;
}
ret = xudma_navss_psil_pair(tx_chn->common.udmax,
tx_chn->common.src_thread,
tx_chn->common.dst_thread);
if (ret) {
dev_err(dev, "PSI-L request err %d\n", ret);
goto err;
}
tx_chn->psil_paired = true;
/* reset TX RT registers */
k3_udma_glue_disable_tx_chn(tx_chn);
k3_udma_glue_dump_tx_chn(tx_chn);
return tx_chn;
@@ -344,6 +388,11 @@ void k3_udma_glue_release_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
if (tx_chn->ringtx)
k3_ringacc_ring_free(tx_chn->ringtx);
if (tx_chn->common.chan_dev.parent) {
device_unregister(&tx_chn->common.chan_dev);
tx_chn->common.chan_dev.parent = NULL;
}
}
EXPORT_SYMBOL_GPL(k3_udma_glue_release_tx_chn);
@@ -378,6 +427,18 @@ EXPORT_SYMBOL_GPL(k3_udma_glue_pop_tx_chn);
int k3_udma_glue_enable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
{
int ret;
ret = xudma_navss_psil_pair(tx_chn->common.udmax,
tx_chn->common.src_thread,
tx_chn->common.dst_thread);
if (ret) {
dev_err(tx_chn->common.dev, "PSI-L request err %d\n", ret);
return ret;
}
tx_chn->psil_paired = true;
xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_PEER_RT_EN_REG,
UDMA_PEER_RT_EN_ENABLE);
@@ -398,6 +459,13 @@ void k3_udma_glue_disable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
xudma_tchanrt_write(tx_chn->udma_tchanx,
UDMA_CHAN_RT_PEER_RT_EN_REG, 0);
k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn dis2");
if (tx_chn->psil_paired) {
xudma_navss_psil_unpair(tx_chn->common.udmax,
tx_chn->common.src_thread,
tx_chn->common.dst_thread);
tx_chn->psil_paired = false;
}
}
EXPORT_SYMBOL_GPL(k3_udma_glue_disable_tx_chn);
@@ -437,13 +505,10 @@ void k3_udma_glue_reset_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
void *data,
void (*cleanup)(void *data, dma_addr_t desc_dma))
{
struct device *dev = tx_chn->common.dev;
dma_addr_t desc_dma;
int occ_tx, i, ret;
/* reset TXCQ as it is not input for udma - expected to be empty */
if (tx_chn->ringtxcq)
k3_ringacc_ring_reset(tx_chn->ringtxcq);
/*
* TXQ reset need to be special way as it is input for udma and its
* state cached by udma, so:
@@ -452,17 +517,20 @@ void k3_udma_glue_reset_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
* 3) reset TXQ in a special way
*/
occ_tx = k3_ringacc_ring_get_occ(tx_chn->ringtx);
dev_dbg(tx_chn->common.dev, "TX reset occ_tx %u\n", occ_tx);
dev_dbg(dev, "TX reset occ_tx %u\n", occ_tx);
for (i = 0; i < occ_tx; i++) {
ret = k3_ringacc_ring_pop(tx_chn->ringtx, &desc_dma);
if (ret) {
dev_err(tx_chn->common.dev, "TX reset pop %d\n", ret);
if (ret != -ENODATA)
dev_err(dev, "TX reset pop %d\n", ret);
break;
}
cleanup(data, desc_dma);
}
/* reset TXCQ as it is not input for udma - expected to be empty */
k3_ringacc_ring_reset(tx_chn->ringtxcq);
k3_ringacc_ring_reset_dma(tx_chn->ringtx, occ_tx);
}
EXPORT_SYMBOL_GPL(k3_udma_glue_reset_tx_chn);
@@ -481,12 +549,50 @@ EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_txcq_id);
int k3_udma_glue_tx_get_irq(struct k3_udma_glue_tx_channel *tx_chn)
{
tx_chn->virq = k3_ringacc_get_ring_irq_num(tx_chn->ringtxcq);
if (xudma_is_pktdma(tx_chn->common.udmax)) {
tx_chn->virq = xudma_pktdma_tflow_get_irq(tx_chn->common.udmax,
tx_chn->udma_tflow_id);
} else {
tx_chn->virq = k3_ringacc_get_ring_irq_num(tx_chn->ringtxcq);
}
return tx_chn->virq;
}
EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_irq);
struct device *
k3_udma_glue_tx_get_dma_device(struct k3_udma_glue_tx_channel *tx_chn)
{
if (xudma_is_pktdma(tx_chn->common.udmax) &&
(tx_chn->common.atype_asel == 14 || tx_chn->common.atype_asel == 15))
return &tx_chn->common.chan_dev;
return xudma_get_device(tx_chn->common.udmax);
}
EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_dma_device);
void k3_udma_glue_tx_dma_to_cppi5_addr(struct k3_udma_glue_tx_channel *tx_chn,
dma_addr_t *addr)
{
if (!xudma_is_pktdma(tx_chn->common.udmax) ||
!tx_chn->common.atype_asel)
return;
*addr |= (u64)tx_chn->common.atype_asel << K3_ADDRESS_ASEL_SHIFT;
}
EXPORT_SYMBOL_GPL(k3_udma_glue_tx_dma_to_cppi5_addr);
void k3_udma_glue_tx_cppi5_to_dma_addr(struct k3_udma_glue_tx_channel *tx_chn,
dma_addr_t *addr)
{
if (!xudma_is_pktdma(tx_chn->common.udmax) ||
!tx_chn->common.atype_asel)
return;
*addr &= (u64)GENMASK(K3_ADDRESS_ASEL_SHIFT - 1, 0);
}
EXPORT_SYMBOL_GPL(k3_udma_glue_tx_cppi5_to_dma_addr);
static int k3_udma_glue_cfg_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
{
const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
@@ -498,8 +604,6 @@ static int k3_udma_glue_cfg_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |
TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |
TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |
TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID |
TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID |
TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID;
req.nav_id = tisci_rm->tisci_dev_id;
@@ -511,13 +615,16 @@ static int k3_udma_glue_cfg_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
* req.rxcq_qnum = k3_ringacc_get_ring_id(rx_chn->flows[0].ringrx);
*/
req.rxcq_qnum = 0xFFFF;
if (rx_chn->flow_num && rx_chn->flow_id_base != rx_chn->udma_rchan_id) {
if (!xudma_is_pktdma(rx_chn->common.udmax) && rx_chn->flow_num &&
rx_chn->flow_id_base != rx_chn->udma_rchan_id) {
/* Default flow + extra ones */
req.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID |
TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID;
req.flowid_start = rx_chn->flow_id_base;
req.flowid_cnt = rx_chn->flow_num;
}
req.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
req.rx_atype = rx_chn->common.atype;
req.rx_atype = rx_chn->common.atype_asel;
ret = tisci_rm->tisci_udmap_ops->rx_ch_cfg(tisci_rm->tisci, &req);
if (ret)
@@ -571,10 +678,18 @@ static int k3_udma_glue_cfg_rx_flow(struct k3_udma_glue_rx_channel *rx_chn,
goto err_rflow_put;
}
if (xudma_is_pktdma(rx_chn->common.udmax)) {
rx_ringfdq_id = flow->udma_rflow_id +
xudma_get_rflow_ring_offset(rx_chn->common.udmax);
rx_ring_id = 0;
} else {
rx_ring_id = flow_cfg->ring_rxq_id;
rx_ringfdq_id = flow_cfg->ring_rxfdq0_id;
}
/* request and cfg rings */
ret = k3_ringacc_request_rings_pair(rx_chn->common.ringacc,
flow_cfg->ring_rxfdq0_id,
flow_cfg->ring_rxq_id,
rx_ringfdq_id, rx_ring_id,
&flow->ringrxfdq,
&flow->ringrx);
if (ret) {
@@ -582,6 +697,16 @@ static int k3_udma_glue_cfg_rx_flow(struct k3_udma_glue_rx_channel *rx_chn,
goto err_rflow_put;
}
/* Set the dma_dev for the rings to be configured */
flow_cfg->rx_cfg.dma_dev = k3_udma_glue_rx_get_dma_device(rx_chn);
flow_cfg->rxfdq_cfg.dma_dev = flow_cfg->rx_cfg.dma_dev;
/* Set the ASEL value for DMA rings of PKTDMA */
if (xudma_is_pktdma(rx_chn->common.udmax)) {
flow_cfg->rx_cfg.asel = rx_chn->common.atype_asel;
flow_cfg->rxfdq_cfg.asel = rx_chn->common.atype_asel;
}
ret = k3_ringacc_ring_cfg(flow->ringrx, &flow_cfg->rx_cfg);
if (ret) {
dev_err(dev, "Failed to cfg ringrx %d\n", ret);
@@ -740,6 +865,7 @@ k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name,
struct k3_udma_glue_rx_channel_cfg *cfg)
{
struct k3_udma_glue_rx_channel *rx_chn;
struct psil_endpoint_config *ep_cfg;
int ret, i;
if (cfg->flow_id_num <= 0)
@@ -767,8 +893,16 @@ k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name,
rx_chn->common.psdata_size,
rx_chn->common.swdata_size);
ep_cfg = rx_chn->common.ep_config;
if (xudma_is_pktdma(rx_chn->common.udmax))
rx_chn->udma_rchan_id = ep_cfg->mapped_channel_id;
else
rx_chn->udma_rchan_id = -1;
/* request and cfg UDMAP RX channel */
rx_chn->udma_rchanx = xudma_rchan_get(rx_chn->common.udmax, -1);
rx_chn->udma_rchanx = xudma_rchan_get(rx_chn->common.udmax,
rx_chn->udma_rchan_id);
if (IS_ERR(rx_chn->udma_rchanx)) {
ret = PTR_ERR(rx_chn->udma_rchanx);
dev_err(dev, "UDMAX rchanx get err %d\n", ret);
@@ -776,12 +910,48 @@ k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name,
}
rx_chn->udma_rchan_id = xudma_rchan_get_id(rx_chn->udma_rchanx);
rx_chn->flow_num = cfg->flow_id_num;
rx_chn->flow_id_base = cfg->flow_id_base;
rx_chn->common.chan_dev.class = &k3_udma_glue_devclass;
rx_chn->common.chan_dev.parent = xudma_get_device(rx_chn->common.udmax);
dev_set_name(&rx_chn->common.chan_dev, "rchan%d-0x%04x",
rx_chn->udma_rchan_id, rx_chn->common.src_thread);
ret = device_register(&rx_chn->common.chan_dev);
if (ret) {
dev_err(dev, "Channel Device registration failed %d\n", ret);
rx_chn->common.chan_dev.parent = NULL;
goto err;
}
/* Use RX channel id as flow id: target dev can't generate flow_id */
if (cfg->flow_id_use_rxchan_id)
rx_chn->flow_id_base = rx_chn->udma_rchan_id;
if (xudma_is_pktdma(rx_chn->common.udmax)) {
/* prepare the channel device as coherent */
rx_chn->common.chan_dev.dma_coherent = true;
dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev,
DMA_BIT_MASK(48));
}
if (xudma_is_pktdma(rx_chn->common.udmax)) {
int flow_start = cfg->flow_id_base;
int flow_end;
if (flow_start == -1)
flow_start = ep_cfg->flow_start;
flow_end = flow_start + cfg->flow_id_num - 1;
if (flow_start < ep_cfg->flow_start ||
flow_end > (ep_cfg->flow_start + ep_cfg->flow_num - 1)) {
dev_err(dev, "Invalid flow range requested\n");
ret = -EINVAL;
goto err;
}
rx_chn->flow_id_base = flow_start;
} else {
rx_chn->flow_id_base = cfg->flow_id_base;
/* Use RX channel id as flow id: target dev can't generate flow_id */
if (cfg->flow_id_use_rxchan_id)
rx_chn->flow_id_base = rx_chn->udma_rchan_id;
}
rx_chn->flow_num = cfg->flow_id_num;
rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num,
sizeof(*rx_chn->flows), GFP_KERNEL);
@@ -815,19 +985,6 @@ k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name,
goto err;
}
ret = xudma_navss_psil_pair(rx_chn->common.udmax,
rx_chn->common.src_thread,
rx_chn->common.dst_thread);
if (ret) {
dev_err(dev, "PSI-L request err %d\n", ret);
goto err;
}
rx_chn->psil_paired = true;
/* reset RX RT registers */
k3_udma_glue_disable_rx_chn(rx_chn);
k3_udma_glue_dump_rx_chn(rx_chn);
return rx_chn;
@@ -884,6 +1041,24 @@ k3_udma_glue_request_remote_rx_chn(struct device *dev, const char *name,
goto err;
}
rx_chn->common.chan_dev.class = &k3_udma_glue_devclass;
rx_chn->common.chan_dev.parent = xudma_get_device(rx_chn->common.udmax);
dev_set_name(&rx_chn->common.chan_dev, "rchan_remote-0x%04x",
rx_chn->common.src_thread);
ret = device_register(&rx_chn->common.chan_dev);
if (ret) {
dev_err(dev, "Channel Device registration failed %d\n", ret);
rx_chn->common.chan_dev.parent = NULL;
goto err;
}
if (xudma_is_pktdma(rx_chn->common.udmax)) {
/* prepare the channel device as coherent */
rx_chn->common.chan_dev.dma_coherent = true;
dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev,
DMA_BIT_MASK(48));
}
ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg);
if (ret)
goto err;
@@ -936,6 +1111,11 @@ void k3_udma_glue_release_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
if (!IS_ERR_OR_NULL(rx_chn->udma_rchanx))
xudma_rchan_put(rx_chn->common.udmax,
rx_chn->udma_rchanx);
if (rx_chn->common.chan_dev.parent) {
device_unregister(&rx_chn->common.chan_dev);
rx_chn->common.chan_dev.parent = NULL;
}
}
EXPORT_SYMBOL_GPL(k3_udma_glue_release_rx_chn);
@@ -1052,12 +1232,24 @@ EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_disable);
int k3_udma_glue_enable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
{
int ret;
if (rx_chn->remote)
return -EINVAL;
if (rx_chn->flows_ready < rx_chn->flow_num)
return -EINVAL;
ret = xudma_navss_psil_pair(rx_chn->common.udmax,
rx_chn->common.src_thread,
rx_chn->common.dst_thread);
if (ret) {
dev_err(rx_chn->common.dev, "PSI-L request err %d\n", ret);
return ret;
}
rx_chn->psil_paired = true;
xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG,
UDMA_CHAN_RT_CTL_EN);
@@ -1078,6 +1270,13 @@ void k3_udma_glue_disable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG, 0);
k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt dis2");
if (rx_chn->psil_paired) {
xudma_navss_psil_unpair(rx_chn->common.udmax,
rx_chn->common.src_thread,
rx_chn->common.dst_thread);
rx_chn->psil_paired = false;
}
}
EXPORT_SYMBOL_GPL(k3_udma_glue_disable_rx_chn);
@@ -1128,12 +1327,10 @@ void k3_udma_glue_reset_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
/* reset RXCQ as it is not input for udma - expected to be empty */
occ_rx = k3_ringacc_ring_get_occ(flow->ringrx);
dev_dbg(dev, "RX reset flow %u occ_rx %u\n", flow_num, occ_rx);
if (flow->ringrx)
k3_ringacc_ring_reset(flow->ringrx);
/* Skip RX FDQ in case one FDQ is used for the set of flows */
if (skip_fdq)
return;
goto do_reset;
/*
* RX FDQ reset need to be special way as it is input for udma and its
@@ -1148,13 +1345,17 @@ void k3_udma_glue_reset_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
for (i = 0; i < occ_rx; i++) {
ret = k3_ringacc_ring_pop(flow->ringrxfdq, &desc_dma);
if (ret) {
dev_err(dev, "RX reset pop %d\n", ret);
if (ret != -ENODATA)
dev_err(dev, "RX reset pop %d\n", ret);
break;
}
cleanup(data, desc_dma);
}
k3_ringacc_ring_reset_dma(flow->ringrxfdq, occ_rx);
do_reset:
k3_ringacc_ring_reset(flow->ringrx);
}
EXPORT_SYMBOL_GPL(k3_udma_glue_reset_rx_chn);
@@ -1184,8 +1385,52 @@ int k3_udma_glue_rx_get_irq(struct k3_udma_glue_rx_channel *rx_chn,
flow = &rx_chn->flows[flow_num];
flow->virq = k3_ringacc_get_ring_irq_num(flow->ringrx);
if (xudma_is_pktdma(rx_chn->common.udmax)) {
flow->virq = xudma_pktdma_rflow_get_irq(rx_chn->common.udmax,
flow->udma_rflow_id);
} else {
flow->virq = k3_ringacc_get_ring_irq_num(flow->ringrx);
}
return flow->virq;
}
EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_irq);
struct device *
k3_udma_glue_rx_get_dma_device(struct k3_udma_glue_rx_channel *rx_chn)
{
if (xudma_is_pktdma(rx_chn->common.udmax) &&
(rx_chn->common.atype_asel == 14 || rx_chn->common.atype_asel == 15))
return &rx_chn->common.chan_dev;
return xudma_get_device(rx_chn->common.udmax);
}
EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_dma_device);
void k3_udma_glue_rx_dma_to_cppi5_addr(struct k3_udma_glue_rx_channel *rx_chn,
dma_addr_t *addr)
{
if (!xudma_is_pktdma(rx_chn->common.udmax) ||
!rx_chn->common.atype_asel)
return;
*addr |= (u64)rx_chn->common.atype_asel << K3_ADDRESS_ASEL_SHIFT;
}
EXPORT_SYMBOL_GPL(k3_udma_glue_rx_dma_to_cppi5_addr);
void k3_udma_glue_rx_cppi5_to_dma_addr(struct k3_udma_glue_rx_channel *rx_chn,
dma_addr_t *addr)
{
if (!xudma_is_pktdma(rx_chn->common.udmax) ||
!rx_chn->common.atype_asel)
return;
*addr &= (u64)GENMASK(K3_ADDRESS_ASEL_SHIFT - 1, 0);
}
EXPORT_SYMBOL_GPL(k3_udma_glue_rx_cppi5_to_dma_addr);
static int __init k3_udma_glue_class_init(void)
{
return class_register(&k3_udma_glue_devclass);
}
arch_initcall(k3_udma_glue_class_init);

45
drivers/dma/ti/k3-udma-private.c
View File

@@ -50,6 +50,18 @@ struct udma_dev *of_xudma_dev_get(struct device_node *np, const char *property)
}
EXPORT_SYMBOL(of_xudma_dev_get);
struct device *xudma_get_device(struct udma_dev *ud)
{
return ud->dev;
}
EXPORT_SYMBOL(xudma_get_device);
struct k3_ringacc *xudma_get_ringacc(struct udma_dev *ud)
{
return ud->ringacc;
}
EXPORT_SYMBOL(xudma_get_ringacc);
u32 xudma_dev_get_psil_base(struct udma_dev *ud)
{
return ud->psil_base;
@@ -76,6 +88,9 @@ EXPORT_SYMBOL(xudma_free_gp_rflow_range);
bool xudma_rflow_is_gp(struct udma_dev *ud, int id)
{
if (!ud->rflow_gp_map)
return false;
return !test_bit(id, ud->rflow_gp_map);
}
EXPORT_SYMBOL(xudma_rflow_is_gp);
@@ -107,6 +122,12 @@ void xudma_rflow_put(struct udma_dev *ud, struct udma_rflow *p)
}
EXPORT_SYMBOL(xudma_rflow_put);
int xudma_get_rflow_ring_offset(struct udma_dev *ud)
{
return ud->tflow_cnt;
}
EXPORT_SYMBOL(xudma_get_rflow_ring_offset);
#define XUDMA_GET_RESOURCE_ID(res) \
int xudma_##res##_get_id(struct udma_##res *p) \
{ \
@@ -136,3 +157,27 @@ void xudma_##res##rt_write(struct udma_##res *p, int reg, u32 val) \
EXPORT_SYMBOL(xudma_##res##rt_write)
XUDMA_RT_IO_FUNCTIONS(tchan);
XUDMA_RT_IO_FUNCTIONS(rchan);
int xudma_is_pktdma(struct udma_dev *ud)
{
return ud->match_data->type == DMA_TYPE_PKTDMA;
}
EXPORT_SYMBOL(xudma_is_pktdma);
int xudma_pktdma_tflow_get_irq(struct udma_dev *ud, int udma_tflow_id)
{
const struct udma_oes_offsets *oes = &ud->soc_data->oes;
return ti_sci_inta_msi_get_virq(ud->dev, udma_tflow_id +
oes->pktdma_tchan_flow);
}
EXPORT_SYMBOL(xudma_pktdma_tflow_get_irq);
int xudma_pktdma_rflow_get_irq(struct udma_dev *ud, int udma_rflow_id)
{
const struct udma_oes_offsets *oes = &ud->soc_data->oes;
return ti_sci_inta_msi_get_virq(ud->dev, udma_rflow_id +
oes->pktdma_rchan_flow);
}
EXPORT_SYMBOL(xudma_pktdma_rflow_get_irq);

1936
drivers/dma/ti/k3-udma.c
View File

File diff suppressed because it is too large Load Diff

28
drivers/dma/ti/k3-udma.h
View File

@@ -18,7 +18,7 @@
#define UDMA_RX_FLOW_ID_FW_OES_REG 0x80
#define UDMA_RX_FLOW_ID_FW_STATUS_REG 0x88
/* TCHANRT/RCHANRT registers */
/* BCHANRT/TCHANRT/RCHANRT registers */
#define UDMA_CHAN_RT_CTL_REG 0x0
#define UDMA_CHAN_RT_SWTRIG_REG 0x8
#define UDMA_CHAN_RT_STDATA_REG 0x80
@@ -45,6 +45,18 @@
#define UDMA_CAP3_HCHAN_CNT(val) (((val) >> 14) & 0x1ff)
#define UDMA_CAP3_UCHAN_CNT(val) (((val) >> 23) & 0x1ff)
#define BCDMA_CAP2_BCHAN_CNT(val) ((val) & 0x1ff)
#define BCDMA_CAP2_TCHAN_CNT(val) (((val) >> 9) & 0x1ff)
#define BCDMA_CAP2_RCHAN_CNT(val) (((val) >> 18) & 0x1ff)
#define BCDMA_CAP3_HBCHAN_CNT(val) (((val) >> 14) & 0x1ff)
#define BCDMA_CAP3_UBCHAN_CNT(val) (((val) >> 23) & 0x1ff)
#define BCDMA_CAP4_HRCHAN_CNT(val) ((val) & 0xff)
#define BCDMA_CAP4_URCHAN_CNT(val) (((val) >> 8) & 0xff)
#define BCDMA_CAP4_HTCHAN_CNT(val) (((val) >> 16) & 0xff)
#define BCDMA_CAP4_UTCHAN_CNT(val) (((val) >> 24) & 0xff)
#define PKTDMA_CAP4_TFLOW_CNT(val) ((val) & 0x3fff)
/* UDMA_CHAN_RT_CTL_REG */
#define UDMA_CHAN_RT_CTL_EN BIT(31)
#define UDMA_CHAN_RT_CTL_TDOWN BIT(30)
@@ -82,15 +94,20 @@
*/
#define PDMA_STATIC_TR_Z(x, mask) ((x) & (mask))
/* Address Space Select */
#define K3_ADDRESS_ASEL_SHIFT 48
struct udma_dev;
struct udma_tchan;
struct udma_rchan;
struct udma_rflow;
enum udma_rm_range {
RM_RANGE_TCHAN = 0,
RM_RANGE_BCHAN = 0,
RM_RANGE_TCHAN,
RM_RANGE_RCHAN,
RM_RANGE_RFLOW,
RM_RANGE_TFLOW,
RM_RANGE_LAST,
};
@@ -112,6 +129,8 @@ int xudma_navss_psil_unpair(struct udma_dev *ud, u32 src_thread,
u32 dst_thread);
struct udma_dev *of_xudma_dev_get(struct device_node *np, const char *property);
struct device *xudma_get_device(struct udma_dev *ud);
struct k3_ringacc *xudma_get_ringacc(struct udma_dev *ud);
void xudma_dev_put(struct udma_dev *ud);
u32 xudma_dev_get_psil_base(struct udma_dev *ud);
struct udma_tisci_rm *xudma_dev_get_tisci_rm(struct udma_dev *ud);
@@ -136,5 +155,10 @@ void xudma_tchanrt_write(struct udma_tchan *tchan, int reg, u32 val);
u32 xudma_rchanrt_read(struct udma_rchan *rchan, int reg);
void xudma_rchanrt_write(struct udma_rchan *rchan, int reg, u32 val);
bool xudma_rflow_is_gp(struct udma_dev *ud, int id);
int xudma_get_rflow_ring_offset(struct udma_dev *ud);
int xudma_is_pktdma(struct udma_dev *ud);
int xudma_pktdma_tflow_get_irq(struct udma_dev *ud, int udma_tflow_id);
int xudma_pktdma_rflow_get_irq(struct udma_dev *ud, int udma_rflow_id);
#endif /* K3_UDMA_H_ */

9
drivers/mailbox/Kconfig
View File

@@ -16,6 +16,13 @@ config ARM_MHU
The controller has 3 mailbox channels, the last of which can be
used in Secure mode only.
config ARM_MHU_V2
tristate "ARM MHUv2 Mailbox"
depends on ARM_AMBA
help
Say Y here if you want to build the ARM MHUv2 controller driver,
which provides unidirectional mailboxes between processing elements.
config IMX_MBOX
tristate "i.MX Mailbox"
depends on ARCH_MXC || COMPILE_TEST
@@ -201,7 +208,7 @@ config BCM_FLEXRM_MBOX
config STM32_IPCC
tristate "STM32 IPCC Mailbox"
depends on MACH_STM32MP157
depends on MACH_STM32MP157 || COMPILE_TEST
help
Mailbox implementation for STMicroelectonics STM32 family chips
with hardware for Inter-Processor Communication Controller (IPCC)

2
drivers/mailbox/Makefile
View File

@@ -7,6 +7,8 @@ obj-$(CONFIG_MAILBOX_TEST) += mailbox-test.o
obj-$(CONFIG_ARM_MHU) += arm_mhu.o arm_mhu_db.o
obj-$(CONFIG_ARM_MHU_V2) += arm_mhuv2.o
obj-$(CONFIG_IMX_MBOX) += imx-mailbox.o
obj-$(CONFIG_ARMADA_37XX_RWTM_MBOX) += armada-37xx-rwtm-mailbox.o

2
drivers/mailbox/arm_mhu_db.c
View File

@@ -180,7 +180,7 @@ static void mhu_db_shutdown(struct mbox_chan *chan)
/* Reset channel */
mhu_db_mbox_clear_irq(chan);
kfree(chan->con_priv);
devm_kfree(mbox->dev, chan->con_priv);
chan->con_priv = NULL;
}

1136
drivers/mailbox/arm_mhuv2.c Normal file
View File

File diff suppressed because it is too large Load Diff

15
drivers/mailbox/stm32-ipcc.c
View File

@@ -144,11 +144,11 @@ static irqreturn_t stm32_ipcc_tx_irq(int irq, void *data)
static int stm32_ipcc_send_data(struct mbox_chan *link, void *data)
{
unsigned int chan = (unsigned int)link->con_priv;
unsigned long chan = (unsigned long)link->con_priv;
struct stm32_ipcc *ipcc = container_of(link->mbox, struct stm32_ipcc,
controller);
dev_dbg(ipcc->controller.dev, "%s: chan:%d\n", __func__, chan);
dev_dbg(ipcc->controller.dev, "%s: chan:%lu\n", __func__, chan);
/* set channel n occupied */
stm32_ipcc_set_bits(&ipcc->lock, ipcc->reg_proc + IPCC_XSCR,
@@ -163,7 +163,7 @@ static int stm32_ipcc_send_data(struct mbox_chan *link, void *data)
static int stm32_ipcc_startup(struct mbox_chan *link)
{
unsigned int chan = (unsigned int)link->con_priv;
unsigned long chan = (unsigned long)link->con_priv;
struct stm32_ipcc *ipcc = container_of(link->mbox, struct stm32_ipcc,
controller);
int ret;
@@ -183,7 +183,7 @@ static int stm32_ipcc_startup(struct mbox_chan *link)
static void stm32_ipcc_shutdown(struct mbox_chan *link)
{
unsigned int chan = (unsigned int)link->con_priv;
unsigned long chan = (unsigned long)link->con_priv;
struct stm32_ipcc *ipcc = container_of(link->mbox, struct stm32_ipcc,
controller);
@@ -206,7 +206,7 @@ static int stm32_ipcc_probe(struct platform_device *pdev)
struct device_node *np = dev->of_node;
struct stm32_ipcc *ipcc;
struct resource *res;
unsigned int i;
unsigned long i;
int ret;
u32 ip_ver;
static const char * const irq_name[] = {"rx", "tx"};
@@ -257,9 +257,6 @@ static int stm32_ipcc_probe(struct platform_device *pdev)
for (i = 0; i < IPCC_IRQ_NUM; i++) {
ipcc->irqs[i] = platform_get_irq_byname(pdev, irq_name[i]);
if (ipcc->irqs[i] < 0) {
if (ipcc->irqs[i] != -EPROBE_DEFER)
dev_err(dev, "no IRQ specified %s\n",
irq_name[i]);
ret = ipcc->irqs[i];
goto err_clk;
}
@@ -268,7 +265,7 @@ static int stm32_ipcc_probe(struct platform_device *pdev)
irq_thread[i], IRQF_ONESHOT,
dev_name(dev), ipcc);
if (ret) {
dev_err(dev, "failed to request irq %d (%d)\n", i, ret);
dev_err(dev, "failed to request irq %lu (%d)\n", i, ret);
goto err_clk;
}
}

325
drivers/soc/ti/k3-ringacc.c
View File

@@ -11,6 +11,7 @@
#include <linux/of.h>
#include <linux/platform_device.h>
#include <linux/sys_soc.h>
#include <linux/dma/ti-cppi5.h>
#include <linux/soc/ti/k3-ringacc.h>
#include <linux/soc/ti/ti_sci_protocol.h>
#include <linux/soc/ti/ti_sci_inta_msi.h>
@@ -21,6 +22,7 @@ static LIST_HEAD(k3_ringacc_list);
static DEFINE_MUTEX(k3_ringacc_list_lock);
#define K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK GENMASK(19, 0)
#define K3_DMARING_CFG_RING_SIZE_ELCNT_MASK GENMASK(15, 0)
/**
* struct k3_ring_rt_regs - The RA realtime Control/Status Registers region
@@ -43,7 +45,13 @@ struct k3_ring_rt_regs {
u32 hwindx;
};
#define K3_RINGACC_RT_REGS_STEP 0x1000
#define K3_RINGACC_RT_REGS_STEP 0x1000
#define K3_DMARING_RT_REGS_STEP 0x2000
#define K3_DMARING_RT_REGS_REVERSE_OFS 0x1000
#define K3_RINGACC_RT_OCC_MASK GENMASK(20, 0)
#define K3_DMARING_RT_OCC_TDOWN_COMPLETE BIT(31)
#define K3_DMARING_RT_DB_ENTRY_MASK GENMASK(7, 0)
#define K3_DMARING_RT_DB_TDOWN_ACK BIT(31)
/**
* struct k3_ring_fifo_regs - The Ring Accelerator Queues Registers region
@@ -122,6 +130,7 @@ struct k3_ring_state {
u32 occ;
u32 windex;
u32 rindex;
u32 tdown_complete:1;
};
/**
@@ -143,6 +152,7 @@ struct k3_ring_state {
* @use_count: Use count for shared rings
* @proxy_id: RA Ring Proxy Id (only if @K3_RINGACC_RING_USE_PROXY)
* @dma_dev: device to be used for DMA API (allocation, mapping)
* @asel: Address Space Select value for physical addresses
*/
struct k3_ring {
struct k3_ring_rt_regs __iomem *rt;
@@ -157,12 +167,15 @@ struct k3_ring {
u32 flags;
#define K3_RING_FLAG_BUSY BIT(1)
#define K3_RING_FLAG_SHARED BIT(2)
#define K3_RING_FLAG_REVERSE BIT(3)
struct k3_ring_state state;
u32 ring_id;
struct k3_ringacc *parent;
u32 use_count;
int proxy_id;
struct device *dma_dev;
u32 asel;
#define K3_ADDRESS_ASEL_SHIFT 48
};
struct k3_ringacc_ops {
@@ -188,6 +201,7 @@ struct k3_ringacc_ops {
* @tisci_ring_ops: ti-sci rings ops
* @tisci_dev_id: ti-sci device id
* @ops: SoC specific ringacc operation
* @dma_rings: indicate DMA ring (dual ring within BCDMA/PKTDMA)
*/
struct k3_ringacc {
struct device *dev;
@@ -210,6 +224,7 @@ struct k3_ringacc {
u32 tisci_dev_id;
const struct k3_ringacc_ops *ops;
bool dma_rings;
};
/**
@@ -221,6 +236,21 @@ struct k3_ringacc_soc_data {
unsigned dma_ring_reset_quirk:1;
};
static int k3_ringacc_ring_read_occ(struct k3_ring *ring)
{
return readl(&ring->rt->occ) & K3_RINGACC_RT_OCC_MASK;
}
static void k3_ringacc_ring_update_occ(struct k3_ring *ring)
{
u32 val;
val = readl(&ring->rt->occ);
ring->state.occ = val & K3_RINGACC_RT_OCC_MASK;
ring->state.tdown_complete = !!(val & K3_DMARING_RT_OCC_TDOWN_COMPLETE);
}
static long k3_ringacc_ring_get_fifo_pos(struct k3_ring *ring)
{
return K3_RINGACC_FIFO_WINDOW_SIZE_BYTES -
@@ -234,12 +264,24 @@ static void *k3_ringacc_get_elm_addr(struct k3_ring *ring, u32 idx)
static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem);
static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem);
static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem);
static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem);
static struct k3_ring_ops k3_ring_mode_ring_ops = {
.push_tail = k3_ringacc_ring_push_mem,
.pop_head = k3_ringacc_ring_pop_mem,
};
static struct k3_ring_ops k3_dmaring_fwd_ops = {
.push_tail = k3_ringacc_ring_push_mem,
.pop_head = k3_dmaring_fwd_pop,
};
static struct k3_ring_ops k3_dmaring_reverse_ops = {
/* Reverse side of the DMA ring can only be popped by SW */
.pop_head = k3_dmaring_reverse_pop,
};
static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem);
static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem);
static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem);
@@ -342,6 +384,40 @@ error:
}
EXPORT_SYMBOL_GPL(k3_ringacc_request_ring);
static int k3_dmaring_request_dual_ring(struct k3_ringacc *ringacc, int fwd_id,
struct k3_ring **fwd_ring,
struct k3_ring **compl_ring)
{
int ret = 0;
/*
* DMA rings must be requested by ID, completion ring is the reverse
* side of the forward ring
*/
if (fwd_id < 0)
return -EINVAL;
mutex_lock(&ringacc->req_lock);
if (test_bit(fwd_id, ringacc->rings_inuse)) {
ret = -EBUSY;
goto error;
}
*fwd_ring = &ringacc->rings[fwd_id];
*compl_ring = &ringacc->rings[fwd_id + ringacc->num_rings];
set_bit(fwd_id, ringacc->rings_inuse);
ringacc->rings[fwd_id].use_count++;
dev_dbg(ringacc->dev, "Giving ring#%d\n", fwd_id);
mutex_unlock(&ringacc->req_lock);
return 0;
error:
mutex_unlock(&ringacc->req_lock);
return ret;
}
int k3_ringacc_request_rings_pair(struct k3_ringacc *ringacc,
int fwd_id, int compl_id,
struct k3_ring **fwd_ring,
@@ -352,6 +428,10 @@ int k3_ringacc_request_rings_pair(struct k3_ringacc *ringacc,
if (!fwd_ring || !compl_ring)
return -EINVAL;
if (ringacc->dma_rings)
return k3_dmaring_request_dual_ring(ringacc, fwd_id,
fwd_ring, compl_ring);
*fwd_ring = k3_ringacc_request_ring(ringacc, fwd_id, 0);
if (!(*fwd_ring))
return -ENODEV;
@@ -421,7 +501,7 @@ void k3_ringacc_ring_reset_dma(struct k3_ring *ring, u32 occ)
goto reset;
if (!occ)
occ = readl(&ring->rt->occ);
occ = k3_ringacc_ring_read_occ(ring);
if (occ) {
u32 db_ring_cnt, db_ring_cnt_cur;
@@ -496,6 +576,13 @@ int k3_ringacc_ring_free(struct k3_ring *ring)
ringacc = ring->parent;
/*
* DMA rings: rings shared memory and configuration, only forward ring
* is configured and reverse ring considered as slave.
*/
if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE))
return 0;
dev_dbg(ring->parent->dev, "flags: 0x%08x\n", ring->flags);
if (!test_bit(ring->ring_id, ringacc->rings_inuse))
@@ -517,6 +604,8 @@ int k3_ringacc_ring_free(struct k3_ring *ring)
ring->flags = 0;
ring->ops = NULL;
ring->dma_dev = NULL;
ring->asel = 0;
if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) {
clear_bit(ring->proxy_id, ringacc->proxy_inuse);
ring->proxy = NULL;
@@ -581,6 +670,7 @@ static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring)
ring_cfg.count = ring->size;
ring_cfg.mode = ring->mode;
ring_cfg.size = ring->elm_size;
ring_cfg.asel = ring->asel;
ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg);
if (ret)
@@ -590,6 +680,90 @@ static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring)
return ret;
}
static int k3_dmaring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg)
{
struct k3_ringacc *ringacc;
struct k3_ring *reverse_ring;
int ret = 0;
if (cfg->elm_size != K3_RINGACC_RING_ELSIZE_8 ||
cfg->mode != K3_RINGACC_RING_MODE_RING ||
cfg->size & ~K3_DMARING_CFG_RING_SIZE_ELCNT_MASK)
return -EINVAL;
ringacc = ring->parent;
/*
* DMA rings: rings shared memory and configuration, only forward ring
* is configured and reverse ring considered as slave.
*/
if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE))
return 0;
if (!test_bit(ring->ring_id, ringacc->rings_inuse))
return -EINVAL;
ring->size = cfg->size;
ring->elm_size = cfg->elm_size;
ring->mode = cfg->mode;
ring->asel = cfg->asel;
ring->dma_dev = cfg->dma_dev;
if (!ring->dma_dev) {
dev_warn(ringacc->dev, "dma_dev is not provided for ring%d\n",
ring->ring_id);
ring->dma_dev = ringacc->dev;
}
memset(&ring->state, 0, sizeof(ring->state));
ring->ops = &k3_dmaring_fwd_ops;
ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev,
ring->size * (4 << ring->elm_size),
&ring->ring_mem_dma, GFP_KERNEL);
if (!ring->ring_mem_virt) {
dev_err(ringacc->dev, "Failed to alloc ring mem\n");
ret = -ENOMEM;
goto err_free_ops;
}
ret = k3_ringacc_ring_cfg_sci(ring);
if (ret)
goto err_free_mem;
ring->flags |= K3_RING_FLAG_BUSY;
k3_ringacc_ring_dump(ring);
/* DMA rings: configure reverse ring */
reverse_ring = &ringacc->rings[ring->ring_id + ringacc->num_rings];
reverse_ring->size = cfg->size;
reverse_ring->elm_size = cfg->elm_size;
reverse_ring->mode = cfg->mode;
reverse_ring->asel = cfg->asel;
memset(&reverse_ring->state, 0, sizeof(reverse_ring->state));
reverse_ring->ops = &k3_dmaring_reverse_ops;
reverse_ring->ring_mem_virt = ring->ring_mem_virt;
reverse_ring->ring_mem_dma = ring->ring_mem_dma;
reverse_ring->flags |= K3_RING_FLAG_BUSY;
k3_ringacc_ring_dump(reverse_ring);
return 0;
err_free_mem:
dma_free_coherent(ring->dma_dev,
ring->size * (4 << ring->elm_size),
ring->ring_mem_virt,
ring->ring_mem_dma);
err_free_ops:
ring->ops = NULL;
ring->proxy = NULL;
ring->dma_dev = NULL;
ring->asel = 0;
return ret;
}
int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg)
{
struct k3_ringacc *ringacc;
@@ -597,8 +771,12 @@ int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg)
if (!ring || !cfg)
return -EINVAL;
ringacc = ring->parent;
if (ringacc->dma_rings)
return k3_dmaring_cfg(ring, cfg);
if (cfg->elm_size > K3_RINGACC_RING_ELSIZE_256 ||
cfg->mode >= K3_RINGACC_RING_MODE_INVALID ||
cfg->size & ~K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK ||
@@ -705,7 +883,7 @@ u32 k3_ringacc_ring_get_free(struct k3_ring *ring)
return -EINVAL;
if (!ring->state.free)
ring->state.free = ring->size - readl(&ring->rt->occ);
ring->state.free = ring->size - k3_ringacc_ring_read_occ(ring);
return ring->state.free;
}
@@ -716,7 +894,7 @@ u32 k3_ringacc_ring_get_occ(struct k3_ring *ring)
if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
return -EINVAL;
return readl(&ring->rt->occ);
return k3_ringacc_ring_read_occ(ring);
}
EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_occ);
@@ -892,6 +1070,72 @@ static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem)
K3_RINGACC_ACCESS_MODE_POP_HEAD);
}
/*
* The element is 48 bits of address + ASEL bits in the ring.
* ASEL is used by the DMAs and should be removed for the kernel as it is not
* part of the physical memory address.
*/
static void k3_dmaring_remove_asel_from_elem(u64 *elem)
{
*elem &= GENMASK_ULL(K3_ADDRESS_ASEL_SHIFT - 1, 0);
}
static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem)
{
void *elem_ptr;
u32 elem_idx;
/*
* DMA rings: forward ring is always tied DMA channel and HW does not
* maintain any state data required for POP operation and its unknown
* how much elements were consumed by HW. So, to actually
* do POP, the read pointer has to be recalculated every time.
*/
ring->state.occ = k3_ringacc_ring_read_occ(ring);
if (ring->state.windex >= ring->state.occ)
elem_idx = ring->state.windex - ring->state.occ;
else
elem_idx = ring->size - (ring->state.occ - ring->state.windex);
elem_ptr = k3_ringacc_get_elm_addr(ring, elem_idx);
memcpy(elem, elem_ptr, (4 << ring->elm_size));
k3_dmaring_remove_asel_from_elem(elem);
ring->state.occ--;
writel(-1, &ring->rt->db);
dev_dbg(ring->parent->dev, "%s: occ%d Windex%d Rindex%d pos_ptr%px\n",
__func__, ring->state.occ, ring->state.windex, elem_idx,
elem_ptr);
return 0;
}
static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem)
{
void *elem_ptr;
elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex);
if (ring->state.occ) {
memcpy(elem, elem_ptr, (4 << ring->elm_size));
k3_dmaring_remove_asel_from_elem(elem);
ring->state.rindex = (ring->state.rindex + 1) % ring->size;
ring->state.occ--;
writel(-1 & K3_DMARING_RT_DB_ENTRY_MASK, &ring->rt->db);
} else if (ring->state.tdown_complete) {
dma_addr_t *value = elem;
*value = CPPI5_TDCM_MARKER;
writel(K3_DMARING_RT_DB_TDOWN_ACK, &ring->rt->db);
ring->state.tdown_complete = false;
}
dev_dbg(ring->parent->dev, "%s: occ%d index%d pos_ptr%px\n",
__func__, ring->state.occ, ring->state.rindex, elem_ptr);
return 0;
}
static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem)
{
void *elem_ptr;
@@ -899,6 +1143,11 @@ static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem)
elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.windex);
memcpy(elem_ptr, elem, (4 << ring->elm_size));
if (ring->parent->dma_rings) {
u64 *addr = elem_ptr;
*addr |= ((u64)ring->asel << K3_ADDRESS_ASEL_SHIFT);
}
ring->state.windex = (ring->state.windex + 1) % ring->size;
ring->state.free--;
@@ -975,12 +1224,12 @@ int k3_ringacc_ring_pop(struct k3_ring *ring, void *elem)
return -EINVAL;
if (!ring->state.occ)
ring->state.occ = k3_ringacc_ring_get_occ(ring);
k3_ringacc_ring_update_occ(ring);
dev_dbg(ring->parent->dev, "ring_pop: occ%d index%d\n", ring->state.occ,
ring->state.rindex);
if (!ring->state.occ)
if (!ring->state.occ && !ring->state.tdown_complete)
return -ENODATA;
if (ring->ops && ring->ops->pop_head)
@@ -998,7 +1247,7 @@ int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem)
return -EINVAL;
if (!ring->state.occ)
ring->state.occ = k3_ringacc_ring_get_occ(ring);
k3_ringacc_ring_update_occ(ring);
dev_dbg(ring->parent->dev, "ring_pop_tail: occ%d index%d\n",
ring->state.occ, ring->state.rindex);
@@ -1203,6 +1452,68 @@ static const struct of_device_id k3_ringacc_of_match[] = {
{},
};
struct k3_ringacc *k3_ringacc_dmarings_init(struct platform_device *pdev,
struct k3_ringacc_init_data *data)
{
struct device *dev = &pdev->dev;
struct k3_ringacc *ringacc;
void __iomem *base_rt;
struct resource *res;
int i;
ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL);
if (!ringacc)
return ERR_PTR(-ENOMEM);
ringacc->dev = dev;
ringacc->dma_rings = true;
ringacc->num_rings = data->num_rings;
ringacc->tisci = data->tisci;
ringacc->tisci_dev_id = data->tisci_dev_id;
mutex_init(&ringacc->req_lock);
res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ringrt");
base_rt = devm_ioremap_resource(dev, res);
if (IS_ERR(base_rt))
return ERR_CAST(base_rt);
ringacc->rings = devm_kzalloc(dev,
sizeof(*ringacc->rings) *
ringacc->num_rings * 2,
GFP_KERNEL);
ringacc->rings_inuse = devm_kcalloc(dev,
BITS_TO_LONGS(ringacc->num_rings),
sizeof(unsigned long), GFP_KERNEL);
if (!ringacc->rings || !ringacc->rings_inuse)
return ERR_PTR(-ENOMEM);
for (i = 0; i < ringacc->num_rings; i++) {
struct k3_ring *ring = &ringacc->rings[i];
ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i;
ring->parent = ringacc;
ring->ring_id = i;
ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
ring = &ringacc->rings[ringacc->num_rings + i];
ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i +
K3_DMARING_RT_REGS_REVERSE_OFS;
ring->parent = ringacc;
ring->ring_id = i;
ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
ring->flags = K3_RING_FLAG_REVERSE;
}
ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;
dev_info(dev, "Number of rings: %u\n", ringacc->num_rings);
return ringacc;
}
EXPORT_SYMBOL_GPL(k3_ringacc_dmarings_init);
static int k3_ringacc_probe(struct platform_device *pdev)
{
const struct ringacc_match_data *match_data;

6
fs/ceph/addr.c
View File

@@ -840,7 +840,7 @@ static int ceph_writepages_start(struct address_space *mapping,
wbc->sync_mode == WB_SYNC_NONE ? "NONE" :
(wbc->sync_mode == WB_SYNC_ALL ? "ALL" : "HOLD"));
if (READ_ONCE(fsc->mount_state) == CEPH_MOUNT_SHUTDOWN) {
if (READ_ONCE(fsc->mount_state) >= CEPH_MOUNT_SHUTDOWN) {
if (ci->i_wrbuffer_ref > 0) {
pr_warn_ratelimited(
"writepage_start %p %lld forced umount\n",
@@ -1264,7 +1264,7 @@ ceph_find_incompatible(struct page *page)
struct ceph_fs_client *fsc = ceph_inode_to_client(inode);
struct ceph_inode_info *ci = ceph_inode(inode);
if (READ_ONCE(fsc->mount_state) == CEPH_MOUNT_SHUTDOWN) {
if (READ_ONCE(fsc->mount_state) >= CEPH_MOUNT_SHUTDOWN) {
dout(" page %p forced umount\n", page);
return ERR_PTR(-EIO);
}
@@ -1321,7 +1321,7 @@ static int ceph_write_begin(struct file *file, struct address_space *mapping,
dout("write_begin file %p inode %p page %p %d~%d\n", file, inode, page, (int)pos, (int)len);
for (;;) {
page = grab_cache_page_write_begin(mapping, index, 0);
page = grab_cache_page_write_begin(mapping, index, flags);
if (!page) {
r = -ENOMEM;
break;

27
fs/ceph/caps.c
View File

@@ -1140,16 +1140,24 @@ void __ceph_remove_cap(struct ceph_cap *cap, bool queue_release)
{
struct ceph_mds_session *session = cap->session;
struct ceph_inode_info *ci = cap->ci;
struct ceph_mds_client *mdsc =
ceph_sb_to_client(ci->vfs_inode.i_sb)->mdsc;
struct ceph_mds_client *mdsc;
int removed = 0;
/* 'ci' being NULL means the remove have already occurred */
if (!ci) {
dout("%s: cap inode is NULL\n", __func__);
return;
}
dout("__ceph_remove_cap %p from %p\n", cap, &ci->vfs_inode);
mdsc = ceph_inode_to_client(&ci->vfs_inode)->mdsc;
/* remove from inode's cap rbtree, and clear auth cap */
rb_erase(&cap->ci_node, &ci->i_caps);
if (ci->i_auth_cap == cap) {
WARN_ON_ONCE(!list_empty(&ci->i_dirty_item));
WARN_ON_ONCE(!list_empty(&ci->i_dirty_item) &&
!mdsc->fsc->blocklisted);
ci->i_auth_cap = NULL;
}
@@ -2746,7 +2754,7 @@ again:
goto out_unlock;
}
if (READ_ONCE(mdsc->fsc->mount_state) == CEPH_MOUNT_SHUTDOWN) {
if (READ_ONCE(mdsc->fsc->mount_state) >= CEPH_MOUNT_SHUTDOWN) {
dout("get_cap_refs %p forced umount\n", inode);
ret = -EIO;
goto out_unlock;
@@ -4027,15 +4035,13 @@ void ceph_handle_caps(struct ceph_mds_session *session,
}
if (msg_version >= 8) {
u64 flush_tid;
u32 caller_uid, caller_gid;
u32 pool_ns_len;
/* version >= 6 */
ceph_decode_64_safe(&p, end, flush_tid, bad);
ceph_decode_skip_64(&p, end, bad); // flush_tid
/* version >= 7 */
ceph_decode_32_safe(&p, end, caller_uid, bad);
ceph_decode_32_safe(&p, end, caller_gid, bad);
ceph_decode_skip_32(&p, end, bad); // caller_uid
ceph_decode_skip_32(&p, end, bad); // caller_gid
/* version >= 8 */
ceph_decode_32_safe(&p, end, pool_ns_len, bad);
if (pool_ns_len > 0) {
@@ -4058,9 +4064,8 @@ void ceph_handle_caps(struct ceph_mds_session *session,
}
if (msg_version >= 11) {
u32 flags;
/* version >= 10 */
ceph_decode_32_safe(&p, end, flags, bad);
ceph_decode_skip_32(&p, end, bad); // flags
/* version >= 11 */
extra_info.dirstat_valid = true;
ceph_decode_64_safe(&p, end, extra_info.nfiles, bad);

20
fs/ceph/debugfs.c
View File

@@ -304,11 +304,25 @@ static int mds_sessions_show(struct seq_file *s, void *ptr)
return 0;
}
static int status_show(struct seq_file *s, void *p)
{
struct ceph_fs_client *fsc = s->private;
struct ceph_entity_inst *inst = &fsc->client->msgr.inst;
struct ceph_entity_addr *client_addr = ceph_client_addr(fsc->client);
seq_printf(s, "instance: %s.%lld %s/%u\n", ENTITY_NAME(inst->name),
ceph_pr_addr(client_addr), le32_to_cpu(client_addr->nonce));
seq_printf(s, "blocklisted: %s\n", fsc->blocklisted ? "true" : "false");
return 0;
}
DEFINE_SHOW_ATTRIBUTE(mdsmap);
DEFINE_SHOW_ATTRIBUTE(mdsc);
DEFINE_SHOW_ATTRIBUTE(caps);
DEFINE_SHOW_ATTRIBUTE(mds_sessions);
DEFINE_SHOW_ATTRIBUTE(metric);
DEFINE_SHOW_ATTRIBUTE(status);
/*
@@ -394,6 +408,12 @@ void ceph_fs_debugfs_init(struct ceph_fs_client *fsc)
fsc->client->debugfs_dir,
fsc,
&caps_fops);
fsc->debugfs_status = debugfs_create_file("status",
0400,
fsc->client->debugfs_dir,
fsc,
&status_fops);
}

9
fs/ceph/dir.c
View File

@@ -1202,12 +1202,11 @@ static int ceph_rename(struct inode *old_dir, struct dentry *old_dentry,
op = CEPH_MDS_OP_RENAMESNAP;
else
return -EROFS;
} else if (old_dir != new_dir) {
err = ceph_quota_check_rename(mdsc, d_inode(old_dentry),
new_dir);
if (err)
return err;
}
/* don't allow cross-quota renames */
if ((old_dir != new_dir) &&
(!ceph_quota_is_same_realm(old_dir, new_dir)))
return -EXDEV;
dout("rename dir %p dentry %p to dir %p dentry %p\n",
old_dir, old_dentry, new_dir, new_dentry);

41
fs/ceph/inode.c
View File

@@ -1315,15 +1315,10 @@ retry_lookup:
}
if (rinfo->head->is_target) {
tvino.ino = le64_to_cpu(rinfo->targeti.in->ino);
tvino.snap = le64_to_cpu(rinfo->targeti.in->snapid);
in = ceph_get_inode(sb, tvino);
if (IS_ERR(in)) {
err = PTR_ERR(in);
goto done;
}
/* Should be filled in by handle_reply */
BUG_ON(!req->r_target_inode);
in = req->r_target_inode;
err = ceph_fill_inode(in, req->r_locked_page, &rinfo->targeti,
NULL, session,
(!test_bit(CEPH_MDS_R_ABORTED, &req->r_req_flags) &&
@@ -1333,11 +1328,13 @@ retry_lookup:
if (err < 0) {
pr_err("ceph_fill_inode badness %p %llx.%llx\n",
in, ceph_vinop(in));
req->r_target_inode = NULL;
if (in->i_state & I_NEW)
discard_new_inode(in);
else
iput(in);
goto done;
}
req->r_target_inode = in;
if (in->i_state & I_NEW)
unlock_new_inode(in);
}
@@ -1597,8 +1594,7 @@ int ceph_readdir_prepopulate(struct ceph_mds_request *req,
struct dentry *dn;
struct inode *in;
int err = 0, skipped = 0, ret, i;
struct ceph_mds_request_head *rhead = req->r_request->front.iov_base;
u32 frag = le32_to_cpu(rhead->args.readdir.frag);
u32 frag = le32_to_cpu(req->r_args.readdir.frag);
u32 last_hash = 0;
u32 fpos_offset;
struct ceph_readdir_cache_control cache_ctl = {};
@@ -1615,7 +1611,7 @@ int ceph_readdir_prepopulate(struct ceph_mds_request *req,
} else if (rinfo->offset_hash) {
/* mds understands offset_hash */
WARN_ON_ONCE(req->r_readdir_offset != 2);
last_hash = le32_to_cpu(rhead->args.readdir.offset_hash);
last_hash = le32_to_cpu(req->r_args.readdir.offset_hash);
}
}
@@ -1888,7 +1884,7 @@ static void ceph_do_invalidate_pages(struct inode *inode)
mutex_lock(&ci->i_truncate_mutex);
if (READ_ONCE(fsc->mount_state) == CEPH_MOUNT_SHUTDOWN) {
if (READ_ONCE(fsc->mount_state) >= CEPH_MOUNT_SHUTDOWN) {
pr_warn_ratelimited("invalidate_pages %p %lld forced umount\n",
inode, ceph_ino(inode));
mapping_set_error(inode->i_mapping, -EIO);
@@ -2340,15 +2336,23 @@ int ceph_permission(struct inode *inode, int mask)
}
/* Craft a mask of needed caps given a set of requested statx attrs. */
static int statx_to_caps(u32 want)
static int statx_to_caps(u32 want, umode_t mode)
{
int mask = 0;
if (want & (STATX_MODE|STATX_UID|STATX_GID|STATX_CTIME|STATX_BTIME))
mask |= CEPH_CAP_AUTH_SHARED;
if (want & (STATX_NLINK|STATX_CTIME))
mask |= CEPH_CAP_LINK_SHARED;
if (want & (STATX_NLINK|STATX_CTIME)) {
/*
* The link count for directories depends on inode->i_subdirs,
* and that is only updated when Fs caps are held.
*/
if (S_ISDIR(mode))
mask |= CEPH_CAP_FILE_SHARED;
else
mask |= CEPH_CAP_LINK_SHARED;
}
if (want & (STATX_ATIME|STATX_MTIME|STATX_CTIME|STATX_SIZE|
STATX_BLOCKS))
@@ -2374,8 +2378,9 @@ int ceph_getattr(const struct path *path, struct kstat *stat,
/* Skip the getattr altogether if we're asked not to sync */
if (!(flags & AT_STATX_DONT_SYNC)) {
err = ceph_do_getattr(inode, statx_to_caps(request_mask),
flags & AT_STATX_FORCE_SYNC);
err = ceph_do_getattr(inode,
statx_to_caps(request_mask, inode->i_mode),
flags & AT_STATX_FORCE_SYNC);
if (err)
return err;
}

8
fs/ceph/locks.c
View File

@@ -57,7 +57,7 @@ static const struct file_lock_operations ceph_fl_lock_ops = {
.fl_release_private = ceph_fl_release_lock,
};
/**
/*
* Implement fcntl and flock locking functions.
*/
static int ceph_lock_message(u8 lock_type, u16 operation, struct inode *inode,
@@ -225,7 +225,7 @@ static int try_unlock_file(struct file *file, struct file_lock *fl)
return 1;
}
/**
/*
* Attempt to set an fcntl lock.
* For now, this just goes away to the server. Later it may be more awesome.
*/
@@ -408,7 +408,7 @@ static int lock_to_ceph_filelock(struct file_lock *lock,
return err;
}
/**
/*
* Encode the flock and fcntl locks for the given inode into the ceph_filelock
* array. Must be called with inode->i_lock already held.
* If we encounter more of a specific lock type than expected, return -ENOSPC.
@@ -458,7 +458,7 @@ fail:
return err;
}
/**
/*
* Copy the encoded flock and fcntl locks into the pagelist.
* Format is: #fcntl locks, sequential fcntl locks, #flock locks,
* sequential flock locks.

280
fs/ceph/mds_client.c
View File

@@ -516,13 +516,9 @@ static int parse_reply_info_create(void **p, void *end,
/* Malformed reply? */
info->has_create_ino = false;
} else if (test_bit(CEPHFS_FEATURE_DELEG_INO, &s->s_features)) {
u8 struct_v, struct_compat;
u32 len;
info->has_create_ino = true;
ceph_decode_8_safe(p, end, struct_v, bad);
ceph_decode_8_safe(p, end, struct_compat, bad);
ceph_decode_32_safe(p, end, len, bad);
/* struct_v, struct_compat, and len */
ceph_decode_skip_n(p, end, 2 + sizeof(u32), bad);
ceph_decode_64_safe(p, end, info->ino, bad);
ret = ceph_parse_deleg_inos(p, end, s);
if (ret)
@@ -837,6 +833,7 @@ void ceph_mdsc_release_request(struct kref *kref)
}
kfree(req->r_path1);
kfree(req->r_path2);
put_cred(req->r_cred);
if (req->r_pagelist)
ceph_pagelist_release(req->r_pagelist);
put_request_session(req);
@@ -892,8 +889,7 @@ static void __register_request(struct ceph_mds_client *mdsc,
ceph_mdsc_get_request(req);
insert_request(&mdsc->request_tree, req);
req->r_uid = current_fsuid();
req->r_gid = current_fsgid();
req->r_cred = get_current_cred();
if (mdsc->oldest_tid == 0 && req->r_op != CEPH_MDS_OP_SETFILELOCK)
mdsc->oldest_tid = req->r_tid;
@@ -1243,7 +1239,7 @@ static struct ceph_msg *create_session_open_msg(struct ceph_mds_client *mdsc, u6
{
struct ceph_msg *msg;
struct ceph_mds_session_head *h;
int i = -1;
int i;
int extra_bytes = 0;
int metadata_key_count = 0;
struct ceph_options *opt = mdsc->fsc->client->options;
@@ -1595,7 +1591,7 @@ static int remove_session_caps_cb(struct inode *inode, struct ceph_cap *cap,
struct ceph_cap_flush *cf;
struct ceph_mds_client *mdsc = fsc->mdsc;
if (READ_ONCE(fsc->mount_state) == CEPH_MOUNT_SHUTDOWN) {
if (READ_ONCE(fsc->mount_state) >= CEPH_MOUNT_SHUTDOWN) {
if (inode->i_data.nrpages > 0)
invalidate = true;
if (ci->i_wrbuffer_ref > 0)
@@ -2482,21 +2478,24 @@ static int set_request_path_attr(struct inode *rinode, struct dentry *rdentry,
/*
* called under mdsc->mutex
*/
static struct ceph_msg *create_request_message(struct ceph_mds_client *mdsc,
static struct ceph_msg *create_request_message(struct ceph_mds_session *session,
struct ceph_mds_request *req,
int mds, bool drop_cap_releases)
bool drop_cap_releases)
{
int mds = session->s_mds;
struct ceph_mds_client *mdsc = session->s_mdsc;
struct ceph_msg *msg;
struct ceph_mds_request_head *head;
struct ceph_mds_request_head_old *head;
const char *path1 = NULL;
const char *path2 = NULL;
u64 ino1 = 0, ino2 = 0;
int pathlen1 = 0, pathlen2 = 0;
bool freepath1 = false, freepath2 = false;
int len;
int len, i;
u16 releases;
void *p, *end;
int ret;
bool legacy = !(session->s_con.peer_features & CEPH_FEATURE_FS_BTIME);
ret = set_request_path_attr(req->r_inode, req->r_dentry,
req->r_parent, req->r_path1, req->r_ino1.ino,
@@ -2518,14 +2517,23 @@ static struct ceph_msg *create_request_message(struct ceph_mds_client *mdsc,
goto out_free1;
}
len = sizeof(*head) +
pathlen1 + pathlen2 + 2*(1 + sizeof(u32) + sizeof(u64)) +
if (legacy) {
/* Old style */
len = sizeof(*head);
} else {
/* New style: add gid_list and any later fields */
len = sizeof(struct ceph_mds_request_head) + sizeof(u32) +
(sizeof(u64) * req->r_cred->group_info->ngroups);
}
len += pathlen1 + pathlen2 + 2*(1 + sizeof(u32) + sizeof(u64)) +
sizeof(struct ceph_timespec);
/* calculate (max) length for cap releases */
len += sizeof(struct ceph_mds_request_release) *
(!!req->r_inode_drop + !!req->r_dentry_drop +
!!req->r_old_inode_drop + !!req->r_old_dentry_drop);
if (req->r_dentry_drop)
len += pathlen1;
if (req->r_old_dentry_drop)
@@ -2537,17 +2545,33 @@ static struct ceph_msg *create_request_message(struct ceph_mds_client *mdsc,
goto out_free2;
}
msg->hdr.version = cpu_to_le16(2);
msg->hdr.tid = cpu_to_le64(req->r_tid);
head = msg->front.iov_base;
p = msg->front.iov_base + sizeof(*head);
/*
* The old ceph_mds_request_header didn't contain a version field, and
* one was added when we moved the message version from 3->4.
*/
if (legacy) {
msg->hdr.version = cpu_to_le16(3);
head = msg->front.iov_base;
p = msg->front.iov_base + sizeof(*head);
} else {
struct ceph_mds_request_head *new_head = msg->front.iov_base;
msg->hdr.version = cpu_to_le16(4);
new_head->version = cpu_to_le16(CEPH_MDS_REQUEST_HEAD_VERSION);
head = (struct ceph_mds_request_head_old *)&new_head->oldest_client_tid;
p = msg->front.iov_base + sizeof(*new_head);
}
end = msg->front.iov_base + msg->front.iov_len;
head->mdsmap_epoch = cpu_to_le32(mdsc->mdsmap->m_epoch);
head->op = cpu_to_le32(req->r_op);
head->caller_uid = cpu_to_le32(from_kuid(&init_user_ns, req->r_uid));
head->caller_gid = cpu_to_le32(from_kgid(&init_user_ns, req->r_gid));
head->caller_uid = cpu_to_le32(from_kuid(&init_user_ns,
req->r_cred->fsuid));
head->caller_gid = cpu_to_le32(from_kgid(&init_user_ns,
req->r_cred->fsgid));
head->ino = cpu_to_le64(req->r_deleg_ino);
head->args = req->r_args;
@@ -2592,6 +2616,14 @@ static struct ceph_msg *create_request_message(struct ceph_mds_client *mdsc,
ceph_encode_copy(&p, &ts, sizeof(ts));
}
/* gid list */
if (!legacy) {
ceph_encode_32(&p, req->r_cred->group_info->ngroups);
for (i = 0; i < req->r_cred->group_info->ngroups; i++)
ceph_encode_64(&p, from_kgid(&init_user_ns,
req->r_cred->group_info->gid[i]));
}
if (WARN_ON_ONCE(p > end)) {
ceph_msg_put(msg);
msg = ERR_PTR(-ERANGE);
@@ -2635,14 +2667,28 @@ static void complete_request(struct ceph_mds_client *mdsc,
complete_all(&req->r_completion);
}
static struct ceph_mds_request_head_old *
find_old_request_head(void *p, u64 features)
{
bool legacy = !(features & CEPH_FEATURE_FS_BTIME);
struct ceph_mds_request_head *new_head;
if (legacy)
return (struct ceph_mds_request_head_old *)p;
new_head = (struct ceph_mds_request_head *)p;
return (struct ceph_mds_request_head_old *)&new_head->oldest_client_tid;
}
/*
* called under mdsc->mutex
*/
static int __prepare_send_request(struct ceph_mds_client *mdsc,
static int __prepare_send_request(struct ceph_mds_session *session,
struct ceph_mds_request *req,
int mds, bool drop_cap_releases)
bool drop_cap_releases)
{
struct ceph_mds_request_head *rhead;
int mds = session->s_mds;
struct ceph_mds_client *mdsc = session->s_mdsc;
struct ceph_mds_request_head_old *rhead;
struct ceph_msg *msg;
int flags = 0;
@@ -2661,6 +2707,7 @@ static int __prepare_send_request(struct ceph_mds_client *mdsc,
if (test_bit(CEPH_MDS_R_GOT_UNSAFE, &req->r_req_flags)) {
void *p;
/*
* Replay. Do not regenerate message (and rebuild
* paths, etc.); just use the original message.
@@ -2668,7 +2715,8 @@ static int __prepare_send_request(struct ceph_mds_client *mdsc,
* d_move mangles the src name.
*/
msg = req->r_request;
rhead = msg->front.iov_base;
rhead = find_old_request_head(msg->front.iov_base,
session->s_con.peer_features);
flags = le32_to_cpu(rhead->flags);
flags |= CEPH_MDS_FLAG_REPLAY;
@@ -2699,14 +2747,15 @@ static int __prepare_send_request(struct ceph_mds_client *mdsc,
ceph_msg_put(req->r_request);
req->r_request = NULL;
}
msg = create_request_message(mdsc, req, mds, drop_cap_releases);
msg = create_request_message(session, req, drop_cap_releases);
if (IS_ERR(msg)) {
req->r_err = PTR_ERR(msg);
return PTR_ERR(msg);
}
req->r_request = msg;
rhead = msg->front.iov_base;
rhead = find_old_request_head(msg->front.iov_base,
session->s_con.peer_features);
rhead->oldest_client_tid = cpu_to_le64(__get_oldest_tid(mdsc));
if (test_bit(CEPH_MDS_R_GOT_UNSAFE, &req->r_req_flags))
flags |= CEPH_MDS_FLAG_REPLAY;
@@ -2725,15 +2774,13 @@ static int __prepare_send_request(struct ceph_mds_client *mdsc,
/*
* called under mdsc->mutex
*/
static int __send_request(struct ceph_mds_client *mdsc,
struct ceph_mds_session *session,
static int __send_request(struct ceph_mds_session *session,
struct ceph_mds_request *req,
bool drop_cap_releases)
{
int err;
err = __prepare_send_request(mdsc, req, session->s_mds,
drop_cap_releases);
err = __prepare_send_request(session, req, drop_cap_releases);
if (!err) {
ceph_msg_get(req->r_request);
ceph_con_send(&session->s_con, req->r_request);
@@ -2818,10 +2865,6 @@ static void __do_request(struct ceph_mds_client *mdsc,
ceph_session_state_name(session->s_state));
if (session->s_state != CEPH_MDS_SESSION_OPEN &&
session->s_state != CEPH_MDS_SESSION_HUNG) {
if (session->s_state == CEPH_MDS_SESSION_REJECTED) {
err = -EACCES;
goto out_session;
}
/*
* We cannot queue async requests since the caps and delegated
* inodes are bound to the session. Just return -EJUKEBOX and
@@ -2831,6 +2874,20 @@ static void __do_request(struct ceph_mds_client *mdsc,
err = -EJUKEBOX;
goto out_session;
}
/*
* If the session has been REJECTED, then return a hard error,
* unless it's a CLEANRECOVER mount, in which case we'll queue
* it to the mdsc queue.
*/
if (session->s_state == CEPH_MDS_SESSION_REJECTED) {
if (ceph_test_mount_opt(mdsc->fsc, CLEANRECOVER))
list_add(&req->r_wait, &mdsc->waiting_for_map);
else
err = -EACCES;
goto out_session;
}
if (session->s_state == CEPH_MDS_SESSION_NEW ||
session->s_state == CEPH_MDS_SESSION_CLOSING) {
err = __open_session(mdsc, session);
@@ -2850,7 +2907,7 @@ static void __do_request(struct ceph_mds_client *mdsc,
if (req->r_request_started == 0) /* note request start time */
req->r_request_started = jiffies;
err = __send_request(mdsc, session, req, false);
err = __send_request(session, req, false);
out_session:
ceph_put_mds_session(session);
@@ -3173,6 +3230,23 @@ static void handle_reply(struct ceph_mds_session *session, struct ceph_msg *msg)
err = parse_reply_info(session, msg, rinfo, session->s_con.peer_features);
mutex_unlock(&mdsc->mutex);
/* Must find target inode outside of mutexes to avoid deadlocks */
if ((err >= 0) && rinfo->head->is_target) {
struct inode *in;
struct ceph_vino tvino = {
.ino = le64_to_cpu(rinfo->targeti.in->ino),
.snap = le64_to_cpu(rinfo->targeti.in->snapid)
};
in = ceph_get_inode(mdsc->fsc->sb, tvino);
if (IS_ERR(in)) {
err = PTR_ERR(in);
mutex_lock(&session->s_mutex);
goto out_err;
}
req->r_target_inode = in;
}
mutex_lock(&session->s_mutex);
if (err < 0) {
pr_err("mdsc_handle_reply got corrupt reply mds%d(tid:%lld)\n", mds, tid);
@@ -3514,7 +3588,7 @@ static void replay_unsafe_requests(struct ceph_mds_client *mdsc,
mutex_lock(&mdsc->mutex);
list_for_each_entry_safe(req, nreq, &session->s_unsafe, r_unsafe_item)
__send_request(mdsc, session, req, true);
__send_request(session, req, true);
/*
* also re-send old requests when MDS enters reconnect stage. So that MDS
@@ -3535,7 +3609,7 @@ static void replay_unsafe_requests(struct ceph_mds_client *mdsc,
ceph_mdsc_release_dir_caps_no_check(req);
__send_request(mdsc, session, req, true);
__send_request(session, req, true);
}
mutex_unlock(&mdsc->mutex);
}
@@ -4374,12 +4448,7 @@ static void maybe_recover_session(struct ceph_mds_client *mdsc)
if (!READ_ONCE(fsc->blocklisted))
return;
if (fsc->last_auto_reconnect &&
time_before(jiffies, fsc->last_auto_reconnect + HZ * 60 * 30))
return;
pr_info("auto reconnect after blocklisted\n");
fsc->last_auto_reconnect = jiffies;
ceph_force_reconnect(fsc->sb);
}
@@ -4678,7 +4747,7 @@ void ceph_mdsc_sync(struct ceph_mds_client *mdsc)
{
u64 want_tid, want_flush;
if (READ_ONCE(mdsc->fsc->mount_state) == CEPH_MOUNT_SHUTDOWN)
if (READ_ONCE(mdsc->fsc->mount_state) >= CEPH_MOUNT_SHUTDOWN)
return;
dout("sync\n");
@@ -4855,10 +4924,8 @@ void ceph_mdsc_handle_fsmap(struct ceph_mds_client *mdsc, struct ceph_msg *msg)
void *p = msg->front.iov_base;
void *end = p + msg->front.iov_len;
u32 epoch;
u32 map_len;
u32 num_fs;
u32 mount_fscid = (u32)-1;
u8 struct_v, struct_cv;
int err = -EINVAL;
ceph_decode_need(&p, end, sizeof(u32), bad);
@@ -4866,24 +4933,17 @@ void ceph_mdsc_handle_fsmap(struct ceph_mds_client *mdsc, struct ceph_msg *msg)
dout("handle_fsmap epoch %u\n", epoch);
ceph_decode_need(&p, end, 2 + sizeof(u32), bad);
struct_v = ceph_decode_8(&p);
struct_cv = ceph_decode_8(&p);
map_len = ceph_decode_32(&p);
/* struct_v, struct_cv, map_len, epoch, legacy_client_fscid */
ceph_decode_skip_n(&p, end, 2 + sizeof(u32) * 3, bad);
ceph_decode_need(&p, end, sizeof(u32) * 3, bad);
p += sizeof(u32) * 2; /* skip epoch and legacy_client_fscid */
num_fs = ceph_decode_32(&p);
ceph_decode_32_safe(&p, end, num_fs, bad);
while (num_fs-- > 0) {
void *info_p, *info_end;
u32 info_len;
u8 info_v, info_cv;
u32 fscid, namelen;
ceph_decode_need(&p, end, 2 + sizeof(u32), bad);
info_v = ceph_decode_8(&p);
info_cv = ceph_decode_8(&p);
p += 2; // info_v, info_cv
info_len = ceph_decode_32(&p);
ceph_decode_need(&p, end, info_len, bad);
info_p = p;
@@ -4954,7 +5014,7 @@ void ceph_mdsc_handle_mdsmap(struct ceph_mds_client *mdsc, struct ceph_msg *msg)
return;
}
newmap = ceph_mdsmap_decode(&p, end);
newmap = ceph_mdsmap_decode(&p, end, ceph_msgr2(mdsc->fsc->client));
if (IS_ERR(newmap)) {
err = PTR_ERR(newmap);
goto bad_unlock;
@@ -5081,23 +5141,12 @@ static struct ceph_auth_handshake *get_authorizer(struct ceph_connection *con,
struct ceph_mds_client *mdsc = s->s_mdsc;
struct ceph_auth_client *ac = mdsc->fsc->client->monc.auth;
struct ceph_auth_handshake *auth = &s->s_auth;
int ret;
if (force_new && auth->authorizer) {
ceph_auth_destroy_authorizer(auth->authorizer);
auth->authorizer = NULL;
}
if (!auth->authorizer) {
int ret = ceph_auth_create_authorizer(ac, CEPH_ENTITY_TYPE_MDS,
auth);
if (ret)
return ERR_PTR(ret);
} else {
int ret = ceph_auth_update_authorizer(ac, CEPH_ENTITY_TYPE_MDS,
auth);
if (ret)
return ERR_PTR(ret);
}
*proto = ac->protocol;
ret = __ceph_auth_get_authorizer(ac, auth, CEPH_ENTITY_TYPE_MDS,
force_new, proto, NULL, NULL);
if (ret)
return ERR_PTR(ret);
return auth;
}
@@ -5118,8 +5167,11 @@ static int verify_authorizer_reply(struct ceph_connection *con)
struct ceph_mds_session *s = con->private;
struct ceph_mds_client *mdsc = s->s_mdsc;
struct ceph_auth_client *ac = mdsc->fsc->client->monc.auth;
struct ceph_auth_handshake *auth = &s->s_auth;
return ceph_auth_verify_authorizer_reply(ac, s->s_auth.authorizer);
return ceph_auth_verify_authorizer_reply(ac, auth->authorizer,
auth->authorizer_reply_buf, auth->authorizer_reply_buf_len,
NULL, NULL, NULL, NULL);
}
static int invalidate_authorizer(struct ceph_connection *con)
@@ -5133,6 +5185,80 @@ static int invalidate_authorizer(struct ceph_connection *con)
return ceph_monc_validate_auth(&mdsc->fsc->client->monc);
}
static int mds_get_auth_request(struct ceph_connection *con,
void *buf, int *buf_len,
void **authorizer, int *authorizer_len)
{
struct ceph_mds_session *s = con->private;
struct ceph_auth_client *ac = s->s_mdsc->fsc->client->monc.auth;
struct ceph_auth_handshake *auth = &s->s_auth;
int ret;
ret = ceph_auth_get_authorizer(ac, auth, CEPH_ENTITY_TYPE_MDS,
buf, buf_len);
if (ret)
return ret;
*authorizer = auth->authorizer_buf;
*authorizer_len = auth->authorizer_buf_len;
return 0;
}
static int mds_handle_auth_reply_more(struct ceph_connection *con,
void *reply, int reply_len,
void *buf, int *buf_len,
void **authorizer, int *authorizer_len)
{
struct ceph_mds_session *s = con->private;
struct ceph_auth_client *ac = s->s_mdsc->fsc->client->monc.auth;
struct ceph_auth_handshake *auth = &s->s_auth;
int ret;
ret = ceph_auth_handle_svc_reply_more(ac, auth, reply, reply_len,
buf, buf_len);
if (ret)
return ret;
*authorizer = auth->authorizer_buf;
*authorizer_len = auth->authorizer_buf_len;
return 0;
}
static int mds_handle_auth_done(struct ceph_connection *con,
u64 global_id, void *reply, int reply_len,
u8 *session_key, int *session_key_len,
u8 *con_secret, int *con_secret_len)
{
struct ceph_mds_session *s = con->private;
struct ceph_auth_client *ac = s->s_mdsc->fsc->client->monc.auth;
struct ceph_auth_handshake *auth = &s->s_auth;
return ceph_auth_handle_svc_reply_done(ac, auth, reply, reply_len,
session_key, session_key_len,
con_secret, con_secret_len);
}
static int mds_handle_auth_bad_method(struct ceph_connection *con,
int used_proto, int result,
const int *allowed_protos, int proto_cnt,
const int *allowed_modes, int mode_cnt)
{
struct ceph_mds_session *s = con->private;
struct ceph_mon_client *monc = &s->s_mdsc->fsc->client->monc;
int ret;
if (ceph_auth_handle_bad_authorizer(monc->auth, CEPH_ENTITY_TYPE_MDS,
used_proto, result,
allowed_protos, proto_cnt,
allowed_modes, mode_cnt)) {
ret = ceph_monc_validate_auth(monc);
if (ret)
return ret;
}
return -EACCES;
}
static struct ceph_msg *mds_alloc_msg(struct ceph_connection *con,
struct ceph_msg_header *hdr, int *skip)
{
@@ -5182,6 +5308,10 @@ static const struct ceph_connection_operations mds_con_ops = {
.alloc_msg = mds_alloc_msg,
.sign_message = mds_sign_message,
.check_message_signature = mds_check_message_signature,
.get_auth_request = mds_get_auth_request,
.handle_auth_reply_more = mds_handle_auth_reply_more,
.handle_auth_done = mds_handle_auth_done,
.handle_auth_bad_method = mds_handle_auth_bad_method,
};
/* eof */

3
fs/ceph/mds_client.h
View File

@@ -275,8 +275,7 @@ struct ceph_mds_request {
union ceph_mds_request_args r_args;
int r_fmode; /* file mode, if expecting cap */
kuid_t r_uid;
kgid_t r_gid;
const struct cred *r_cred;
int r_request_release_offset;
struct timespec64 r_stamp;

25
fs/ceph/mdsmap.c
View File

@@ -114,7 +114,7 @@ bad:
* Ignore any fields we don't care about (there are quite a few of
* them).
*/
struct ceph_mdsmap *ceph_mdsmap_decode(void **p, void *end)
struct ceph_mdsmap *ceph_mdsmap_decode(void **p, void *end, bool msgr2)
{
struct ceph_mdsmap *m;
const void *start = *p;
@@ -201,18 +201,19 @@ struct ceph_mdsmap *ceph_mdsmap_decode(void **p, void *end)
namelen = ceph_decode_32(p); /* skip mds name */
*p += namelen;
ceph_decode_need(p, end,
4*sizeof(u32) + sizeof(u64) +
sizeof(addr) + sizeof(struct ceph_timespec),
bad);
mds = ceph_decode_32(p);
inc = ceph_decode_32(p);
state = ceph_decode_32(p);
ceph_decode_32_safe(p, end, mds, bad);
ceph_decode_32_safe(p, end, inc, bad);
ceph_decode_32_safe(p, end, state, bad);
*p += sizeof(u64); /* state_seq */
err = ceph_decode_entity_addr(p, end, &addr);
if (info_v >= 8)
err = ceph_decode_entity_addrvec(p, end, msgr2, &addr);
else
err = ceph_decode_entity_addr(p, end, &addr);
if (err)
goto corrupt;
ceph_decode_copy(p, &laggy_since, sizeof(laggy_since));
ceph_decode_copy_safe(p, end, &laggy_since, sizeof(laggy_since),
bad);
laggy = laggy_since.tv_sec != 0 || laggy_since.tv_nsec != 0;
*p += sizeof(u32);
ceph_decode_32_safe(p, end, namelen, bad);
@@ -243,8 +244,8 @@ struct ceph_mdsmap *ceph_mdsmap_decode(void **p, void *end)
}
if (state <= 0) {
pr_warn("mdsmap_decode got incorrect state(%s)\n",
ceph_mds_state_name(state));
dout("mdsmap_decode got incorrect state(%s)\n",
ceph_mds_state_name(state));
continue;
}

18
fs/ceph/metric.c
View File

@@ -16,6 +16,7 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
struct ceph_metric_read_latency *read;
struct ceph_metric_write_latency *write;
struct ceph_metric_metadata_latency *meta;
struct ceph_metric_dlease *dlease;
struct ceph_client_metric *m = &mdsc->metric;
u64 nr_caps = atomic64_read(&m->total_caps);
struct ceph_msg *msg;
@@ -25,7 +26,7 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
s32 len;
len = sizeof(*head) + sizeof(*cap) + sizeof(*read) + sizeof(*write)
+ sizeof(*meta);
+ sizeof(*meta) + sizeof(*dlease);
msg = ceph_msg_new(CEPH_MSG_CLIENT_METRICS, len, GFP_NOFS, true);
if (!msg) {
@@ -42,8 +43,8 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
cap->ver = 1;
cap->compat = 1;
cap->data_len = cpu_to_le32(sizeof(*cap) - 10);
cap->hit = cpu_to_le64(percpu_counter_sum(&mdsc->metric.i_caps_hit));
cap->mis = cpu_to_le64(percpu_counter_sum(&mdsc->metric.i_caps_mis));
cap->hit = cpu_to_le64(percpu_counter_sum(&m->i_caps_hit));
cap->mis = cpu_to_le64(percpu_counter_sum(&m->i_caps_mis));
cap->total = cpu_to_le64(nr_caps);
items++;
@@ -83,6 +84,17 @@ static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
meta->nsec = cpu_to_le32(ts.tv_nsec);
items++;
/* encode the dentry lease metric */
dlease = (struct ceph_metric_dlease *)(meta + 1);
dlease->type = cpu_to_le32(CLIENT_METRIC_TYPE_DENTRY_LEASE);
dlease->ver = 1;
dlease->compat = 1;
dlease->data_len = cpu_to_le32(sizeof(*dlease) - 10);
dlease->hit = cpu_to_le64(percpu_counter_sum(&m->d_lease_hit));
dlease->mis = cpu_to_le64(percpu_counter_sum(&m->d_lease_mis));
dlease->total = cpu_to_le64(atomic64_read(&m->total_dentries));
items++;
put_unaligned_le32(items, &head->num);
msg->front.iov_len = len;
msg->hdr.version = cpu_to_le16(1);

14
fs/ceph/metric.h
View File

@@ -27,6 +27,7 @@ enum ceph_metric_type {
CLIENT_METRIC_TYPE_READ_LATENCY, \
CLIENT_METRIC_TYPE_WRITE_LATENCY, \
CLIENT_METRIC_TYPE_METADATA_LATENCY, \
CLIENT_METRIC_TYPE_DENTRY_LEASE, \
\
CLIENT_METRIC_TYPE_MAX, \
}
@@ -80,6 +81,19 @@ struct ceph_metric_metadata_latency {
__le32 nsec;
} __packed;
/* metric dentry lease header */
struct ceph_metric_dlease {
__le32 type; /* ceph metric type */
__u8 ver;
__u8 compat;
__le32 data_len; /* length of sizeof(hit + mis + total) */
__le64 hit;
__le64 mis;
__le64 total;
} __packed;
struct ceph_metric_head {
__le32 num; /* the number of metrics that will be sent */
} __packed;

58
fs/ceph/quota.c
View File

@@ -264,7 +264,7 @@ restart:
return NULL;
}
static bool ceph_quota_is_same_realm(struct inode *old, struct inode *new)
bool ceph_quota_is_same_realm(struct inode *old, struct inode *new)
{
struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(old->i_sb);
struct ceph_snap_realm *old_realm, *new_realm;
@@ -516,59 +516,3 @@ bool ceph_quota_update_statfs(struct ceph_fs_client *fsc, struct kstatfs *buf)
return is_updated;
}
/*
* ceph_quota_check_rename - check if a rename can be executed
* @mdsc: MDS client instance
* @old: inode to be copied
* @new: destination inode (directory)
*
* This function verifies if a rename (e.g. moving a file or directory) can be
* executed. It forces an rstat update in the @new target directory (and in the
* source @old as well, if it's a directory). The actual check is done both for
* max_files and max_bytes.
*
* This function returns 0 if it's OK to do the rename, or, if quotas are
* exceeded, -EXDEV (if @old is a directory) or -EDQUOT.
*/
int ceph_quota_check_rename(struct ceph_mds_client *mdsc,
struct inode *old, struct inode *new)
{
struct ceph_inode_info *ci_old = ceph_inode(old);
int ret = 0;
if (ceph_quota_is_same_realm(old, new))
return 0;
/*
* Get the latest rstat for target directory (and for source, if a
* directory)
*/
ret = ceph_do_getattr(new, CEPH_STAT_RSTAT, false);
if (ret)
return ret;
if (S_ISDIR(old->i_mode)) {
ret = ceph_do_getattr(old, CEPH_STAT_RSTAT, false);
if (ret)
return ret;
ret = check_quota_exceeded(new, QUOTA_CHECK_MAX_BYTES_OP,
ci_old->i_rbytes);
if (!ret)
ret = check_quota_exceeded(new,
QUOTA_CHECK_MAX_FILES_OP,
ci_old->i_rfiles +
ci_old->i_rsubdirs);
if (ret)
ret = -EXDEV;
} else {
ret = check_quota_exceeded(new, QUOTA_CHECK_MAX_BYTES_OP,
i_size_read(old));
if (!ret)
ret = check_quota_exceeded(new,
QUOTA_CHECK_MAX_FILES_OP, 1);
if (ret)
ret = -EDQUOT;
}
return ret;
}

14
fs/ceph/super.c
View File

@@ -831,6 +831,13 @@ static void destroy_caches(void)
ceph_fscache_unregister();
}
static void __ceph_umount_begin(struct ceph_fs_client *fsc)
{
ceph_osdc_abort_requests(&fsc->client->osdc, -EIO);
ceph_mdsc_force_umount(fsc->mdsc);
fsc->filp_gen++; // invalidate open files
}
/*
* ceph_umount_begin - initiate forced umount. Tear down the
* mount, skipping steps that may hang while waiting for server(s).
@@ -843,9 +850,7 @@ static void ceph_umount_begin(struct super_block *sb)
if (!fsc)
return;
fsc->mount_state = CEPH_MOUNT_SHUTDOWN;
ceph_osdc_abort_requests(&fsc->client->osdc, -EIO);
ceph_mdsc_force_umount(fsc->mdsc);
fsc->filp_gen++; // invalidate open files
__ceph_umount_begin(fsc);
}
static const struct super_operations ceph_super_ops = {
@@ -1234,7 +1239,8 @@ int ceph_force_reconnect(struct super_block *sb)
struct ceph_fs_client *fsc = ceph_sb_to_client(sb);
int err = 0;
ceph_umount_begin(sb);
fsc->mount_state = CEPH_MOUNT_RECOVER;
__ceph_umount_begin(fsc);
/* Make sure all page caches get invalidated.
* see remove_session_caps_cb() */

7
fs/ceph/super.h
View File

@@ -106,9 +106,8 @@ struct ceph_fs_client {
struct ceph_mount_options *mount_options;
struct ceph_client *client;
unsigned long mount_state;
int mount_state;
unsigned long last_auto_reconnect;
bool blocklisted;
bool have_copy_from2;
@@ -129,6 +128,7 @@ struct ceph_fs_client {
struct dentry *debugfs_bdi;
struct dentry *debugfs_mdsc, *debugfs_mdsmap;
struct dentry *debugfs_metric;
struct dentry *debugfs_status;
struct dentry *debugfs_mds_sessions;
#endif
@@ -1222,14 +1222,13 @@ extern void ceph_handle_quota(struct ceph_mds_client *mdsc,
struct ceph_mds_session *session,
struct ceph_msg *msg);
extern bool ceph_quota_is_max_files_exceeded(struct inode *inode);
extern bool ceph_quota_is_same_realm(struct inode *old, struct inode *new);
extern bool ceph_quota_is_max_bytes_exceeded(struct inode *inode,
loff_t newlen);
extern bool ceph_quota_is_max_bytes_approaching(struct inode *inode,
loff_t newlen);
extern bool ceph_quota_update_statfs(struct ceph_fs_client *fsc,
struct kstatfs *buf);
extern int ceph_quota_check_rename(struct ceph_mds_client *mdsc,
struct inode *old, struct inode *new);
extern void ceph_cleanup_quotarealms_inodes(struct ceph_mds_client *mdsc);
#endif /* _FS_CEPH_SUPER_H */

81
fs/ceph/xattr.c
View File

@@ -42,6 +42,7 @@ struct ceph_vxattr {
#define VXATTR_FLAG_READONLY (1<<0)
#define VXATTR_FLAG_HIDDEN (1<<1)
#define VXATTR_FLAG_RSTAT (1<<2)
#define VXATTR_FLAG_DIRSTAT (1<<3)
/* layouts */
@@ -303,6 +304,36 @@ static ssize_t ceph_vxattrcb_snap_btime(struct ceph_inode_info *ci, char *val,
ci->i_snap_btime.tv_nsec);
}
static ssize_t ceph_vxattrcb_cluster_fsid(struct ceph_inode_info *ci,
char *val, size_t size)
{
struct ceph_fs_client *fsc = ceph_sb_to_client(ci->vfs_inode.i_sb);
return ceph_fmt_xattr(val, size, "%pU", &fsc->client->fsid);
}
static ssize_t ceph_vxattrcb_client_id(struct ceph_inode_info *ci,
char *val, size_t size)
{
struct ceph_fs_client *fsc = ceph_sb_to_client(ci->vfs_inode.i_sb);
return ceph_fmt_xattr(val, size, "client%lld",
ceph_client_gid(fsc->client));
}
static ssize_t ceph_vxattrcb_caps(struct ceph_inode_info *ci, char *val,
size_t size)
{
int issued;
spin_lock(&ci->i_ceph_lock);
issued = __ceph_caps_issued(ci, NULL);
spin_unlock(&ci->i_ceph_lock);
return ceph_fmt_xattr(val, size, "%s/0x%x",
ceph_cap_string(issued), issued);
}
#define CEPH_XATTR_NAME(_type, _name) XATTR_CEPH_PREFIX #_type "." #_name
#define CEPH_XATTR_NAME2(_type, _name, _name2) \
XATTR_CEPH_PREFIX #_type "." #_name "." #_name2
@@ -347,9 +378,9 @@ static struct ceph_vxattr ceph_dir_vxattrs[] = {
XATTR_LAYOUT_FIELD(dir, layout, object_size),
XATTR_LAYOUT_FIELD(dir, layout, pool),
XATTR_LAYOUT_FIELD(dir, layout, pool_namespace),
XATTR_NAME_CEPH(dir, entries, 0),
XATTR_NAME_CEPH(dir, files, 0),
XATTR_NAME_CEPH(dir, subdirs, 0),
XATTR_NAME_CEPH(dir, entries, VXATTR_FLAG_DIRSTAT),
XATTR_NAME_CEPH(dir, files, VXATTR_FLAG_DIRSTAT),
XATTR_NAME_CEPH(dir, subdirs, VXATTR_FLAG_DIRSTAT),
XATTR_RSTAT_FIELD(dir, rentries),
XATTR_RSTAT_FIELD(dir, rfiles),
XATTR_RSTAT_FIELD(dir, rsubdirs),
@@ -378,6 +409,13 @@ static struct ceph_vxattr ceph_dir_vxattrs[] = {
.exists_cb = ceph_vxattrcb_snap_btime_exists,
.flags = VXATTR_FLAG_READONLY,
},
{
.name = "ceph.caps",
.name_size = sizeof("ceph.caps"),
.getxattr_cb = ceph_vxattrcb_caps,
.exists_cb = NULL,
.flags = VXATTR_FLAG_HIDDEN,
},
{ .name = NULL, 0 } /* Required table terminator */
};
@@ -403,6 +441,31 @@ static struct ceph_vxattr ceph_file_vxattrs[] = {
.exists_cb = ceph_vxattrcb_snap_btime_exists,
.flags = VXATTR_FLAG_READONLY,
},
{
.name = "ceph.caps",
.name_size = sizeof("ceph.caps"),
.getxattr_cb = ceph_vxattrcb_caps,
.exists_cb = NULL,
.flags = VXATTR_FLAG_HIDDEN,
},
{ .name = NULL, 0 } /* Required table terminator */
};
static struct ceph_vxattr ceph_common_vxattrs[] = {
{
.name = "ceph.cluster_fsid",
.name_size = sizeof("ceph.cluster_fsid"),
.getxattr_cb = ceph_vxattrcb_cluster_fsid,
.exists_cb = NULL,
.flags = VXATTR_FLAG_READONLY,
},
{
.name = "ceph.client_id",
.name_size = sizeof("ceph.client_id"),
.getxattr_cb = ceph_vxattrcb_client_id,
.exists_cb = NULL,
.flags = VXATTR_FLAG_READONLY,
},
{ .name = NULL, 0 } /* Required table terminator */
};
@@ -428,6 +491,13 @@ static struct ceph_vxattr *ceph_match_vxattr(struct inode *inode,
}
}
vxattr = ceph_common_vxattrs;
while (vxattr->name) {
if (!strcmp(vxattr->name, name))
return vxattr;
vxattr++;
}
return NULL;
}
@@ -837,6 +907,8 @@ ssize_t __ceph_getxattr(struct inode *inode, const char *name, void *value,
int mask = 0;
if (vxattr->flags & VXATTR_FLAG_RSTAT)
mask |= CEPH_STAT_RSTAT;
if (vxattr->flags & VXATTR_FLAG_DIRSTAT)
mask |= CEPH_CAP_FILE_SHARED;
err = ceph_do_getattr(inode, mask, true);
if (err)
return err;
@@ -950,6 +1022,7 @@ static int ceph_sync_setxattr(struct inode *inode, const char *name,
struct ceph_inode_info *ci = ceph_inode(inode);
struct ceph_mds_request *req;
struct ceph_mds_client *mdsc = fsc->mdsc;
struct ceph_osd_client *osdc = &fsc->client->osdc;
struct ceph_pagelist *pagelist = NULL;
int op = CEPH_MDS_OP_SETXATTR;
int err;
@@ -988,6 +1061,8 @@ static int ceph_sync_setxattr(struct inode *inode, const char *name,
if (op == CEPH_MDS_OP_SETXATTR) {
req->r_args.setxattr.flags = cpu_to_le32(flags);
req->r_args.setxattr.osdmap_epoch =
cpu_to_le32(osdc->osdmap->epoch);
req->r_pagelist = pagelist;
pagelist = NULL;
}

22
fs/lockd/host.c
View File

@@ -163,7 +163,7 @@ static struct nlm_host *nlm_alloc_host(struct nlm_lookup_host_info *ni,
host->h_nsmhandle = nsm;
host->h_addrbuf = nsm->sm_addrbuf;
host->net = ni->net;
host->h_cred = get_cred(ni->cred),
host->h_cred = get_cred(ni->cred);
strlcpy(host->nodename, utsname()->nodename, sizeof(host->nodename));
out:
@@ -439,12 +439,7 @@ nlm_bind_host(struct nlm_host *host)
* RPC rebind is required
*/
if ((clnt = host->h_rpcclnt) != NULL) {
if (time_after_eq(jiffies, host->h_nextrebind)) {
rpc_force_rebind(clnt);
host->h_nextrebind = jiffies + NLM_HOST_REBIND;
dprintk("lockd: next rebind in %lu jiffies\n",
host->h_nextrebind - jiffies);
}
nlm_rebind_host(host);
} else {
unsigned long increment = nlmsvc_timeout;
struct rpc_timeout timeparms = {
@@ -494,13 +489,20 @@ nlm_bind_host(struct nlm_host *host)
return clnt;
}
/*
* Force a portmap lookup of the remote lockd port
/**
* nlm_rebind_host - If needed, force a portmap lookup of the peer's lockd port
* @host: NLM host handle for peer
*
* This is not needed when using a connection-oriented protocol, such as TCP.
* The existing autobind mechanism is sufficient to force a rebind when
* required, e.g. on connection state transitions.
*/
void
nlm_rebind_host(struct nlm_host *host)
{
dprintk("lockd: rebind host %s\n", host->h_name);
if (host->h_proto != IPPROTO_UDP)
return;
if (host->h_rpcclnt && time_after_eq(jiffies, host->h_nextrebind)) {
rpc_force_rebind(host->h_rpcclnt);
host->h_nextrebind = jiffies + NLM_HOST_REBIND;

8
fs/nfs/client.c
View File

@@ -571,7 +571,7 @@ static int nfs_start_lockd(struct nfs_server *server)
1 : 0,
.net = clp->cl_net,
.nlmclnt_ops = clp->cl_nfs_mod->rpc_ops->nlmclnt_ops,
.cred = current_cred(),
.cred = server->cred,
};
if (nlm_init.nfs_version > 3)
@@ -781,8 +781,8 @@ static void nfs_server_set_fsinfo(struct nfs_server *server,
server->wtmult = nfs_block_bits(fsinfo->wtmult, NULL);
server->dtsize = nfs_block_size(fsinfo->dtpref, NULL);
if (server->dtsize > PAGE_SIZE * NFS_MAX_READDIR_PAGES)
server->dtsize = PAGE_SIZE * NFS_MAX_READDIR_PAGES;
if (server->dtsize > NFS_MAX_FILE_IO_SIZE)
server->dtsize = NFS_MAX_FILE_IO_SIZE;
if (server->dtsize > server->rsize)
server->dtsize = server->rsize;
@@ -985,7 +985,7 @@ struct nfs_server *nfs_create_server(struct fs_context *fc)
if (!server)
return ERR_PTR(-ENOMEM);
server->cred = get_cred(current_cred());
server->cred = get_cred(fc->cred);
error = -ENOMEM;
fattr = nfs_alloc_fattr();

713
fs/nfs/dir.c
View File

File diff suppressed because it is too large Load Diff

17
fs/nfs/flexfilelayout/flexfilelayout.c
View File

@@ -740,16 +740,12 @@ ff_layout_choose_ds_for_read(struct pnfs_layout_segment *lseg,
struct nfs4_ff_layout_segment *fls = FF_LAYOUT_LSEG(lseg);
struct nfs4_ff_layout_mirror *mirror;
struct nfs4_pnfs_ds *ds;
bool fail_return = false;
u32 idx;
/* mirrors are initially sorted by efficiency */
for (idx = start_idx; idx < fls->mirror_array_cnt; idx++) {
if (idx+1 == fls->mirror_array_cnt)
fail_return = !check_device;
mirror = FF_LAYOUT_COMP(lseg, idx);
ds = nfs4_ff_layout_prepare_ds(lseg, mirror, fail_return);
ds = nfs4_ff_layout_prepare_ds(lseg, mirror, false);
if (!ds)
continue;
@@ -1056,7 +1052,7 @@ static void ff_layout_resend_pnfs_read(struct nfs_pgio_header *hdr)
u32 idx = hdr->pgio_mirror_idx + 1;
u32 new_idx = 0;
if (ff_layout_choose_any_ds_for_read(hdr->lseg, idx + 1, &new_idx))
if (ff_layout_choose_any_ds_for_read(hdr->lseg, idx, &new_idx))
ff_layout_send_layouterror(hdr->lseg);
else
pnfs_error_mark_layout_for_return(hdr->inode, hdr->lseg);
@@ -2284,7 +2280,6 @@ ff_layout_encode_netaddr(struct xdr_stream *xdr, struct nfs4_pnfs_ds_addr *da)
struct sockaddr *sap = (struct sockaddr *)&da->da_addr;
char portbuf[RPCBIND_MAXUADDRPLEN];
char addrbuf[RPCBIND_MAXUADDRLEN];
char *netid;
unsigned short port;
int len, netid_len;
__be32 *p;
@@ -2294,18 +2289,13 @@ ff_layout_encode_netaddr(struct xdr_stream *xdr, struct nfs4_pnfs_ds_addr *da)
if (ff_layout_ntop4(sap, addrbuf, sizeof(addrbuf)) == 0)
return;
port = ntohs(((struct sockaddr_in *)sap)->sin_port);
netid = "tcp";
netid_len = 3;
break;
case AF_INET6:
if (ff_layout_ntop6_noscopeid(sap, addrbuf, sizeof(addrbuf)) == 0)
return;
port = ntohs(((struct sockaddr_in6 *)sap)->sin6_port);
netid = "tcp6";
netid_len = 4;
break;
default:
/* we only support tcp and tcp6 */
WARN_ON_ONCE(1);
return;
}
@@ -2313,8 +2303,9 @@ ff_layout_encode_netaddr(struct xdr_stream *xdr, struct nfs4_pnfs_ds_addr *da)
snprintf(portbuf, sizeof(portbuf), ".%u.%u", port >> 8, port & 0xff);
len = strlcat(addrbuf, portbuf, sizeof(addrbuf));
netid_len = strlen(da->da_netid);
p = xdr_reserve_space(xdr, 4 + netid_len);
xdr_encode_opaque(p, netid, netid_len);
xdr_encode_opaque(p, da->da_netid, netid_len);
p = xdr_reserve_space(xdr, 4 + len);
xdr_encode_opaque(p, addrbuf, len);

21
fs/nfs/fs_context.c
View File

@@ -510,13 +510,12 @@ static int nfs_fs_context_parse_param(struct fs_context *fc,
ctx->nfs_server.protocol = XPRT_TRANSPORT_UDP;
break;
case Opt_tcp:
ctx->flags |= NFS_MOUNT_TCP;
ctx->nfs_server.protocol = XPRT_TRANSPORT_TCP;
break;
case Opt_rdma:
ctx->flags |= NFS_MOUNT_TCP; /* for side protocols */
ctx->nfs_server.protocol = XPRT_TRANSPORT_RDMA;
xprt_load_transport(param->key);
ret = xprt_find_transport_ident(param->key);
if (ret < 0)
goto out_bad_transport;
ctx->nfs_server.protocol = ret;
break;
case Opt_acl:
if (result.negated)
@@ -670,11 +669,13 @@ static int nfs_fs_context_parse_param(struct fs_context *fc,
case Opt_xprt_rdma:
/* vector side protocols to TCP */
ctx->flags |= NFS_MOUNT_TCP;
ctx->nfs_server.protocol = XPRT_TRANSPORT_RDMA;
xprt_load_transport(param->string);
ret = xprt_find_transport_ident(param->string);
if (ret < 0)
goto out_bad_transport;
ctx->nfs_server.protocol = ret;
break;
default:
return nfs_invalf(fc, "NFS: Unrecognized transport protocol");
goto out_bad_transport;
}
ctx->protofamily = protofamily;
@@ -697,7 +698,7 @@ static int nfs_fs_context_parse_param(struct fs_context *fc,
break;
case Opt_xprt_rdma: /* not used for side protocols */
default:
return nfs_invalf(fc, "NFS: Unrecognized transport protocol");
goto out_bad_transport;
}
ctx->mountfamily = mountfamily;
break;
@@ -787,6 +788,8 @@ out_invalid_address:
return nfs_invalf(fc, "NFS: Bad IP address specified");
out_of_bounds:
return nfs_invalf(fc, "NFS: Value for '%s' out of range", param->key);
out_bad_transport:
return nfs_invalf(fc, "NFS: Unrecognized transport protocol");
}
/*

9
fs/nfs/inode.c
View File

@@ -229,7 +229,6 @@ static void nfs_zap_caches_locked(struct inode *inode)
nfsi->attrtimeo = NFS_MINATTRTIMEO(inode);
nfsi->attrtimeo_timestamp = jiffies;
memset(NFS_I(inode)->cookieverf, 0, sizeof(NFS_I(inode)->cookieverf));
if (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode)) {
nfs_set_cache_invalid(inode, NFS_INO_INVALID_ATTR
| NFS_INO_INVALID_DATA
@@ -1237,7 +1236,6 @@ EXPORT_SYMBOL_GPL(nfs_revalidate_inode);
static int nfs_invalidate_mapping(struct inode *inode, struct address_space *mapping)
{
struct nfs_inode *nfsi = NFS_I(inode);
int ret;
if (mapping->nrpages != 0) {
@@ -1250,11 +1248,6 @@ static int nfs_invalidate_mapping(struct inode *inode, struct address_space *map
if (ret < 0)
return ret;
}
if (S_ISDIR(inode->i_mode)) {
spin_lock(&inode->i_lock);
memset(nfsi->cookieverf, 0, sizeof(nfsi->cookieverf));
spin_unlock(&inode->i_lock);
}
nfs_inc_stats(inode, NFSIOS_DATAINVALIDATE);
nfs_fscache_wait_on_invalidate(inode);
@@ -2180,7 +2173,7 @@ static int nfsiod_start(void)
{
struct workqueue_struct *wq;
dprintk("RPC: creating workqueue nfsiod\n");
wq = alloc_workqueue("nfsiod", WQ_MEM_RECLAIM, 0);
wq = alloc_workqueue("nfsiod", WQ_MEM_RECLAIM | WQ_UNBOUND, 0);
if (wq == NULL)
return -ENOMEM;
nfsiod_workqueue = wq;

6
fs/nfs/internal.h
View File

@@ -56,12 +56,6 @@ static inline bool nfs_lookup_is_soft_revalidate(const struct dentry *dentry)
#define NFS_UNSPEC_RETRANS (UINT_MAX)
#define NFS_UNSPEC_TIMEO (UINT_MAX)
/*
* Maximum number of pages that readdir can use for creating
* a vmapped array of pages.
*/
#define NFS_MAX_READDIR_PAGES 8
struct nfs_client_initdata {
unsigned long init_flags;
const char *hostname; /* Hostname of the server */

19
fs/nfs/nfs2xdr.c
View File

@@ -34,6 +34,7 @@
* Declare the space requirements for NFS arguments and replies as
* number of 32bit-words
*/
#define NFS_pagepad_sz (1) /* Page padding */
#define NFS_fhandle_sz (8)
#define NFS_sattr_sz (8)
#define NFS_filename_sz (1+(NFS2_MAXNAMLEN>>2))
@@ -56,11 +57,11 @@
#define NFS_attrstat_sz (1+NFS_fattr_sz)
#define NFS_diropres_sz (1+NFS_fhandle_sz+NFS_fattr_sz)
#define NFS_readlinkres_sz (2+1)
#define NFS_readres_sz (1+NFS_fattr_sz+1+1)
#define NFS_readlinkres_sz (2+NFS_pagepad_sz)
#define NFS_readres_sz (1+NFS_fattr_sz+1+NFS_pagepad_sz)
#define NFS_writeres_sz (NFS_attrstat_sz)
#define NFS_stat_sz (1)
#define NFS_readdirres_sz (1+1)
#define NFS_readdirres_sz (1+NFS_pagepad_sz)
#define NFS_statfsres_sz (1+NFS_info_sz)
static int nfs_stat_to_errno(enum nfs_stat);
@@ -592,8 +593,8 @@ static void nfs2_xdr_enc_readlinkargs(struct rpc_rqst *req,
const struct nfs_readlinkargs *args = data;
encode_fhandle(xdr, args->fh);
rpc_prepare_reply_pages(req, args->pages, args->pgbase,
args->pglen, NFS_readlinkres_sz);
rpc_prepare_reply_pages(req, args->pages, args->pgbase, args->pglen,
NFS_readlinkres_sz - NFS_pagepad_sz);
}
/*
@@ -628,8 +629,8 @@ static void nfs2_xdr_enc_readargs(struct rpc_rqst *req,
const struct nfs_pgio_args *args = data;
encode_readargs(xdr, args);
rpc_prepare_reply_pages(req, args->pages, args->pgbase,
args->count, NFS_readres_sz);
rpc_prepare_reply_pages(req, args->pages, args->pgbase, args->count,
NFS_readres_sz - NFS_pagepad_sz);
req->rq_rcv_buf.flags |= XDRBUF_READ;
}
@@ -786,8 +787,8 @@ static void nfs2_xdr_enc_readdirargs(struct rpc_rqst *req,
const struct nfs_readdirargs *args = data;
encode_readdirargs(xdr, args);
rpc_prepare_reply_pages(req, args->pages, 0,
args->count, NFS_readdirres_sz);
rpc_prepare_reply_pages(req, args->pages, 0, args->count,
NFS_readdirres_sz - NFS_pagepad_sz);
}
/*

83
fs/nfs/nfs3proc.c
View File

@@ -154,14 +154,14 @@ nfs3_proc_setattr(struct dentry *dentry, struct nfs_fattr *fattr,
}
static int
nfs3_proc_lookup(struct inode *dir, struct dentry *dentry,
struct nfs_fh *fhandle, struct nfs_fattr *fattr,
struct nfs4_label *label)
__nfs3_proc_lookup(struct inode *dir, const char *name, size_t len,
struct nfs_fh *fhandle, struct nfs_fattr *fattr,
unsigned short task_flags)
{
struct nfs3_diropargs arg = {
.fh = NFS_FH(dir),
.name = dentry->d_name.name,
.len = dentry->d_name.len
.name = name,
.len = len
};
struct nfs3_diropres res = {
.fh = fhandle,
@@ -173,17 +173,11 @@ nfs3_proc_lookup(struct inode *dir, struct dentry *dentry,
.rpc_resp = &res,
};
int status;
unsigned short task_flags = 0;
/* Is this is an attribute revalidation, subject to softreval? */
if (nfs_lookup_is_soft_revalidate(dentry))
task_flags |= RPC_TASK_TIMEOUT;
res.dir_attr = nfs_alloc_fattr();
if (res.dir_attr == NULL)
return -ENOMEM;
dprintk("NFS call lookup %pd2\n", dentry);
nfs_fattr_init(fattr);
status = rpc_call_sync(NFS_CLIENT(dir), &msg, task_flags);
nfs_refresh_inode(dir, res.dir_attr);
@@ -198,6 +192,37 @@ nfs3_proc_lookup(struct inode *dir, struct dentry *dentry,
return status;
}
static int
nfs3_proc_lookup(struct inode *dir, struct dentry *dentry,
struct nfs_fh *fhandle, struct nfs_fattr *fattr,
struct nfs4_label *label)
{
unsigned short task_flags = 0;
/* Is this is an attribute revalidation, subject to softreval? */
if (nfs_lookup_is_soft_revalidate(dentry))
task_flags |= RPC_TASK_TIMEOUT;
dprintk("NFS call lookup %pd2\n", dentry);
return __nfs3_proc_lookup(dir, dentry->d_name.name,
dentry->d_name.len, fhandle, fattr,
task_flags);
}
static int nfs3_proc_lookupp(struct inode *inode, struct nfs_fh *fhandle,
struct nfs_fattr *fattr, struct nfs4_label *label)
{
const char dotdot[] = "..";
const size_t len = strlen(dotdot);
unsigned short task_flags = 0;
if (NFS_SERVER(inode)->flags & NFS_MOUNT_SOFTREVAL)
task_flags |= RPC_TASK_TIMEOUT;
return __nfs3_proc_lookup(inode, dotdot, len, fhandle, fattr,
task_flags);
}
static int nfs3_proc_access(struct inode *inode, struct nfs_access_entry *entry)
{
struct nfs3_accessargs arg = {
@@ -637,37 +662,36 @@ out:
* Also note that this implementation handles both plain readdir and
* readdirplus.
*/
static int
nfs3_proc_readdir(struct dentry *dentry, const struct cred *cred,
u64 cookie, struct page **pages, unsigned int count, bool plus)
static int nfs3_proc_readdir(struct nfs_readdir_arg *nr_arg,
struct nfs_readdir_res *nr_res)
{
struct inode *dir = d_inode(dentry);
__be32 *verf = NFS_I(dir)->cookieverf;
struct inode *dir = d_inode(nr_arg->dentry);
struct nfs3_readdirargs arg = {
.fh = NFS_FH(dir),
.cookie = cookie,
.verf = {verf[0], verf[1]},
.plus = plus,
.count = count,
.pages = pages
.cookie = nr_arg->cookie,
.plus = nr_arg->plus,
.count = nr_arg->page_len,
.pages = nr_arg->pages
};
struct nfs3_readdirres res = {
.verf = verf,
.plus = plus
.verf = nr_res->verf,
.plus = nr_arg->plus,
};
struct rpc_message msg = {
.rpc_proc = &nfs3_procedures[NFS3PROC_READDIR],
.rpc_argp = &arg,
.rpc_resp = &res,
.rpc_cred = cred,
.rpc_cred = nr_arg->cred,
};
int status = -ENOMEM;
if (plus)
if (nr_arg->plus)
msg.rpc_proc = &nfs3_procedures[NFS3PROC_READDIRPLUS];
if (arg.cookie)
memcpy(arg.verf, nr_arg->verf, sizeof(arg.verf));
dprintk("NFS call readdir%s %d\n",
plus? "plus" : "", (unsigned int) cookie);
dprintk("NFS call readdir%s %llu\n", nr_arg->plus ? "plus" : "",
(unsigned long long)nr_arg->cookie);
res.dir_attr = nfs_alloc_fattr();
if (res.dir_attr == NULL)
@@ -680,8 +704,8 @@ nfs3_proc_readdir(struct dentry *dentry, const struct cred *cred,
nfs_free_fattr(res.dir_attr);
out:
dprintk("NFS reply readdir%s: %d\n",
plus? "plus" : "", status);
dprintk("NFS reply readdir%s: %d\n", nr_arg->plus ? "plus" : "",
status);
return status;
}
@@ -1004,6 +1028,7 @@ const struct nfs_rpc_ops nfs_v3_clientops = {
.getattr = nfs3_proc_getattr,
.setattr = nfs3_proc_setattr,
.lookup = nfs3_proc_lookup,
.lookupp = nfs3_proc_lookupp,
.access = nfs3_proc_access,
.readlink = nfs3_proc_readlink,
.create = nfs3_proc_create,

29
fs/nfs/nfs3xdr.c
View File

@@ -33,6 +33,7 @@
* Declare the space requirements for NFS arguments and replies as
* number of 32bit-words
*/
#define NFS3_pagepad_sz (1) /* Page padding */
#define NFS3_fhandle_sz (1+16)
#define NFS3_fh_sz (NFS3_fhandle_sz) /* shorthand */
#define NFS3_sattr_sz (15)
@@ -69,13 +70,13 @@
#define NFS3_removeres_sz (NFS3_setattrres_sz)
#define NFS3_lookupres_sz (1+NFS3_fh_sz+(2 * NFS3_post_op_attr_sz))
#define NFS3_accessres_sz (1+NFS3_post_op_attr_sz+1)
#define NFS3_readlinkres_sz (1+NFS3_post_op_attr_sz+1+1)
#define NFS3_readres_sz (1+NFS3_post_op_attr_sz+3+1)
#define NFS3_readlinkres_sz (1+NFS3_post_op_attr_sz+1+NFS3_pagepad_sz)
#define NFS3_readres_sz (1+NFS3_post_op_attr_sz+3+NFS3_pagepad_sz)
#define NFS3_writeres_sz (1+NFS3_wcc_data_sz+4)
#define NFS3_createres_sz (1+NFS3_fh_sz+NFS3_post_op_attr_sz+NFS3_wcc_data_sz)
#define NFS3_renameres_sz (1+(2 * NFS3_wcc_data_sz))
#define NFS3_linkres_sz (1+NFS3_post_op_attr_sz+NFS3_wcc_data_sz)
#define NFS3_readdirres_sz (1+NFS3_post_op_attr_sz+2+1)
#define NFS3_readdirres_sz (1+NFS3_post_op_attr_sz+2+NFS3_pagepad_sz)
#define NFS3_fsstatres_sz (1+NFS3_post_op_attr_sz+13)
#define NFS3_fsinfores_sz (1+NFS3_post_op_attr_sz+12)
#define NFS3_pathconfres_sz (1+NFS3_post_op_attr_sz+6)
@@ -85,7 +86,8 @@
#define ACL3_setaclargs_sz (NFS3_fh_sz+1+ \
XDR_QUADLEN(NFS_ACL_INLINE_BUFSIZE))
#define ACL3_getaclres_sz (1+NFS3_post_op_attr_sz+1+ \
XDR_QUADLEN(NFS_ACL_INLINE_BUFSIZE)+1)
XDR_QUADLEN(NFS_ACL_INLINE_BUFSIZE)+\
NFS3_pagepad_sz)
#define ACL3_setaclres_sz (1+NFS3_post_op_attr_sz)
static int nfs3_stat_to_errno(enum nfs_stat);
@@ -909,8 +911,8 @@ static void nfs3_xdr_enc_readlink3args(struct rpc_rqst *req,
const struct nfs3_readlinkargs *args = data;
encode_nfs_fh3(xdr, args->fh);
rpc_prepare_reply_pages(req, args->pages, args->pgbase,
args->pglen, NFS3_readlinkres_sz);
rpc_prepare_reply_pages(req, args->pages, args->pgbase, args->pglen,
NFS3_readlinkres_sz - NFS3_pagepad_sz);
}
/*
@@ -939,7 +941,8 @@ static void nfs3_xdr_enc_read3args(struct rpc_rqst *req,
const void *data)
{
const struct nfs_pgio_args *args = data;
unsigned int replen = args->replen ? args->replen : NFS3_readres_sz;
unsigned int replen = args->replen ? args->replen :
NFS3_readres_sz - NFS3_pagepad_sz;
encode_read3args(xdr, args);
rpc_prepare_reply_pages(req, args->pages, args->pgbase,
@@ -1239,8 +1242,8 @@ static void nfs3_xdr_enc_readdir3args(struct rpc_rqst *req,
const struct nfs3_readdirargs *args = data;
encode_readdir3args(xdr, args);
rpc_prepare_reply_pages(req, args->pages, 0,
args->count, NFS3_readdirres_sz);
rpc_prepare_reply_pages(req, args->pages, 0, args->count,
NFS3_readdirres_sz - NFS3_pagepad_sz);
}
/*
@@ -1281,8 +1284,8 @@ static void nfs3_xdr_enc_readdirplus3args(struct rpc_rqst *req,
const struct nfs3_readdirargs *args = data;
encode_readdirplus3args(xdr, args);
rpc_prepare_reply_pages(req, args->pages, 0,
args->count, NFS3_readdirres_sz);
rpc_prepare_reply_pages(req, args->pages, 0, args->count,
NFS3_readdirres_sz - NFS3_pagepad_sz);
}
/*
@@ -1328,7 +1331,7 @@ static void nfs3_xdr_enc_getacl3args(struct rpc_rqst *req,
if (args->mask & (NFS_ACL | NFS_DFACL)) {
rpc_prepare_reply_pages(req, args->pages, 0,
NFSACL_MAXPAGES << PAGE_SHIFT,
ACL3_getaclres_sz);
ACL3_getaclres_sz - NFS3_pagepad_sz);
req->rq_rcv_buf.flags |= XDRBUF_SPARSE_PAGES;
}
}
@@ -1648,7 +1651,7 @@ static int nfs3_xdr_dec_read3res(struct rpc_rqst *req, struct xdr_stream *xdr,
result->op_status = status;
if (status != NFS3_OK)
goto out_status;
result->replen = 4 + ((xdr_stream_pos(xdr) - pos) >> 2);
result->replen = 3 + ((xdr_stream_pos(xdr) - pos) >> 2);
error = decode_read3resok(xdr, result);
out:
return error;

48
fs/nfs/nfs42proc.c
View File

@@ -1173,14 +1173,12 @@ static int _nfs42_proc_setxattr(struct inode *inode, const char *name,
}
static ssize_t _nfs42_proc_getxattr(struct inode *inode, const char *name,
void *buf, size_t buflen)
void *buf, size_t buflen, struct page **pages,
size_t plen)
{
struct nfs_server *server = NFS_SERVER(inode);
struct page *pages[NFS4XATTR_MAXPAGES] = {};
struct nfs42_getxattrargs arg = {
.fh = NFS_FH(inode),
.xattr_pages = pages,
.xattr_len = buflen,
.xattr_name = name,
};
struct nfs42_getxattrres res;
@@ -1189,7 +1187,10 @@ static ssize_t _nfs42_proc_getxattr(struct inode *inode, const char *name,
.rpc_argp = &arg,
.rpc_resp = &res,
};
int ret, np;
ssize_t ret;
arg.xattr_len = plen;
arg.xattr_pages = pages;
ret = nfs4_call_sync(server->client, server, &msg, &arg.seq_args,
&res.seq_res, 0);
@@ -1214,10 +1215,6 @@ static ssize_t _nfs42_proc_getxattr(struct inode *inode, const char *name,
_copy_from_pages(buf, pages, 0, res.xattr_len);
}
np = DIV_ROUND_UP(res.xattr_len, PAGE_SIZE);
while (--np >= 0)
__free_page(pages[np]);
return res.xattr_len;
}
@@ -1292,16 +1289,45 @@ ssize_t nfs42_proc_getxattr(struct inode *inode, const char *name,
void *buf, size_t buflen)
{
struct nfs4_exception exception = { };
ssize_t err;
ssize_t err, np, i;
struct page **pages;
np = nfs_page_array_len(0, buflen ?: XATTR_SIZE_MAX);
pages = kmalloc_array(np, sizeof(*pages), GFP_KERNEL);
if (!pages)
return -ENOMEM;
for (i = 0; i < np; i++) {
pages[i] = alloc_page(GFP_KERNEL);
if (!pages[i]) {
np = i + 1;
err = -ENOMEM;
goto out;
}
}
/*
* The GETXATTR op has no length field in the call, and the
* xattr data is at the end of the reply.
*
* There is no downside in using the page-aligned length. It will
* allow receiving and caching xattrs that are too large for the
* caller but still fit in the page-rounded value.
*/
do {
err = _nfs42_proc_getxattr(inode, name, buf, buflen);
err = _nfs42_proc_getxattr(inode, name, buf, buflen,
pages, np * PAGE_SIZE);
if (err >= 0)
break;
err = nfs4_handle_exception(NFS_SERVER(inode), err,
&exception);
} while (exception.retry);
out:
while (--np >= 0)
__free_page(pages[np]);
kfree(pages);
return err;
}

101
fs/nfs/nfs42xdr.c
View File

@@ -191,7 +191,7 @@
#define encode_getxattr_maxsz (op_encode_hdr_maxsz + 1 + \
nfs4_xattr_name_maxsz)
#define decode_getxattr_maxsz (op_decode_hdr_maxsz + 1 + 1)
#define decode_getxattr_maxsz (op_decode_hdr_maxsz + 1 + pagepad_maxsz)
#define encode_setxattr_maxsz (op_encode_hdr_maxsz + \
1 + nfs4_xattr_name_maxsz + 1)
#define decode_setxattr_maxsz (op_decode_hdr_maxsz + decode_change_info_maxsz)
@@ -489,6 +489,12 @@ static int decode_getxattr(struct xdr_stream *xdr,
return -EIO;
len = be32_to_cpup(p);
/*
* Only check against the page length here. The actual
* requested length may be smaller, but that is only
* checked against after possibly caching a valid reply.
*/
if (len > req->rq_rcv_buf.page_len)
return -ERANGE;
@@ -1019,56 +1025,80 @@ static int decode_deallocate(struct xdr_stream *xdr, struct nfs42_falloc_res *re
return decode_op_hdr(xdr, OP_DEALLOCATE);
}
static int decode_read_plus_data(struct xdr_stream *xdr, struct nfs_pgio_res *res,
uint32_t *eof)
static int decode_read_plus_data(struct xdr_stream *xdr,
struct nfs_pgio_args *args,
struct nfs_pgio_res *res)
{
uint32_t count, recvd;
uint64_t offset;
__be32 *p;
p = xdr_inline_decode(xdr, 8 + 4);
if (unlikely(!p))
return -EIO;
if (!p)
return 1;
p = xdr_decode_hyper(p, &offset);
count = be32_to_cpup(p);
recvd = xdr_align_data(xdr, res->count, count);
res->count += recvd;
if (count > recvd) {
dprintk("NFS: server cheating in read reply: "
"count %u > recvd %u\n", count, recvd);
*eof = 0;
recvd = xdr_align_data(xdr, res->count, xdr_align_size(count));
if (recvd > count)
recvd = count;
if (res->count + recvd > args->count) {
if (args->count > res->count)
res->count += args->count - res->count;
return 1;
}
res->count += recvd;
if (count > recvd)
return 1;
return 0;
}
static int decode_read_plus_hole(struct xdr_stream *xdr, struct nfs_pgio_res *res,
uint32_t *eof)
static int decode_read_plus_hole(struct xdr_stream *xdr,
struct nfs_pgio_args *args,
struct nfs_pgio_res *res, uint32_t *eof)
{
uint64_t offset, length, recvd;
__be32 *p;
p = xdr_inline_decode(xdr, 8 + 8);
if (unlikely(!p))
return -EIO;
if (!p)
return 1;
p = xdr_decode_hyper(p, &offset);
p = xdr_decode_hyper(p, &length);
if (offset != args->offset + res->count) {
/* Server returned an out-of-sequence extent */
if (offset > args->offset + res->count ||
offset + length < args->offset + res->count) {
dprintk("NFS: server returned out of sequence extent: "
"offset/size = %llu/%llu != expected %llu\n",
(unsigned long long)offset,
(unsigned long long)length,
(unsigned long long)(args->offset +
res->count));
return 1;
}
length -= args->offset + res->count - offset;
}
if (length + res->count > args->count) {
*eof = 0;
if (unlikely(res->count >= args->count))
return 1;
length = args->count - res->count;
}
recvd = xdr_expand_hole(xdr, res->count, length);
res->count += recvd;
if (recvd < length) {
*eof = 0;
if (recvd < length)
return 1;
}
return 0;
}
static int decode_read_plus(struct xdr_stream *xdr, struct nfs_pgio_res *res)
{
struct nfs_pgio_header *hdr =
container_of(res, struct nfs_pgio_header, res);
struct nfs_pgio_args *args = &hdr->args;
uint32_t eof, segments, type;
int status, i;
__be32 *p;
@@ -1081,6 +1111,7 @@ static int decode_read_plus(struct xdr_stream *xdr, struct nfs_pgio_res *res)
if (unlikely(!p))
return -EIO;
res->count = 0;
eof = be32_to_cpup(p++);
segments = be32_to_cpup(p++);
if (segments == 0)
@@ -1088,26 +1119,31 @@ static int decode_read_plus(struct xdr_stream *xdr, struct nfs_pgio_res *res)
for (i = 0; i < segments; i++) {
p = xdr_inline_decode(xdr, 4);
if (unlikely(!p))
return -EIO;
if (!p)
goto early_out;
type = be32_to_cpup(p++);
if (type == NFS4_CONTENT_DATA)
status = decode_read_plus_data(xdr, res, &eof);
status = decode_read_plus_data(xdr, args, res);
else if (type == NFS4_CONTENT_HOLE)
status = decode_read_plus_hole(xdr, res, &eof);
status = decode_read_plus_hole(xdr, args, res, &eof);
else
return -EINVAL;
if (status < 0)
return status;
if (status > 0)
break;
goto early_out;
}
out:
res->eof = eof;
return 0;
early_out:
if (unlikely(!i))
return -EIO;
res->eof = 0;
return 0;
}
static int decode_seek(struct xdr_stream *xdr, struct nfs42_seek_res *res)
@@ -1476,18 +1512,16 @@ static void nfs4_xdr_enc_getxattr(struct rpc_rqst *req, struct xdr_stream *xdr,
struct compound_hdr hdr = {
.minorversion = nfs4_xdr_minorversion(&args->seq_args),
};
size_t plen;
uint32_t replen;
encode_compound_hdr(xdr, req, &hdr);
encode_sequence(xdr, &args->seq_args, &hdr);
encode_putfh(xdr, args->fh, &hdr);
replen = hdr.replen + op_decode_hdr_maxsz + 1;
encode_getxattr(xdr, args->xattr_name, &hdr);
plen = args->xattr_len ? args->xattr_len : XATTR_SIZE_MAX;
rpc_prepare_reply_pages(req, args->xattr_pages, 0, plen,
hdr.replen);
req->rq_rcv_buf.flags |= XDRBUF_SPARSE_PAGES;
rpc_prepare_reply_pages(req, args->xattr_pages, 0, args->xattr_len,
replen);
encode_nops(&hdr);
}
@@ -1520,14 +1554,15 @@ static void nfs4_xdr_enc_listxattrs(struct rpc_rqst *req,
struct compound_hdr hdr = {
.minorversion = nfs4_xdr_minorversion(&args->seq_args),
};
uint32_t replen;
encode_compound_hdr(xdr, req, &hdr);
encode_sequence(xdr, &args->seq_args, &hdr);
encode_putfh(xdr, args->fh, &hdr);
replen = hdr.replen + op_decode_hdr_maxsz + 2 + 1;
encode_listxattrs(xdr, args, &hdr);
rpc_prepare_reply_pages(req, args->xattr_pages, 0, args->count,
hdr.replen);
rpc_prepare_reply_pages(req, args->xattr_pages, 0, args->count, replen);
encode_nops(&hdr);
}

2
fs/nfs/nfs4client.c
View File

@@ -1153,7 +1153,7 @@ struct nfs_server *nfs4_create_server(struct fs_context *fc)
if (!server)
return ERR_PTR(-ENOMEM);
server->cred = get_cred(current_cred());
server->cred = get_cred(fc->cred);
auth_probe = ctx->auth_info.flavor_len < 1;

71
fs/nfs/nfs4proc.c
View File

@@ -184,6 +184,8 @@ static int nfs4_map_errors(int err)
return -EPROTONOSUPPORT;
case -NFS4ERR_FILE_OPEN:
return -EBUSY;
case -NFS4ERR_NOT_SAME:
return -ENOTSYNC;
default:
dprintk("%s could not handle NFSv4 error %d\n",
__func__, -err);
@@ -4397,6 +4399,10 @@ static int _nfs4_proc_lookupp(struct inode *inode,
.rpc_argp = &args,
.rpc_resp = &res,
};
unsigned short task_flags = 0;
if (NFS_SERVER(inode)->flags & NFS_MOUNT_SOFTREVAL)
task_flags |= RPC_TASK_TIMEOUT;
args.bitmask = nfs4_bitmask(server, label);
@@ -4404,7 +4410,7 @@ static int _nfs4_proc_lookupp(struct inode *inode,
dprintk("NFS call lookupp ino=0x%lx\n", inode->i_ino);
status = nfs4_call_sync(clnt, server, &msg, &args.seq_args,
&res.seq_res, 0);
&res.seq_res, task_flags);
dprintk("NFS reply lookupp: %d\n", status);
return status;
}
@@ -4957,35 +4963,40 @@ static int nfs4_proc_mkdir(struct inode *dir, struct dentry *dentry,
return err;
}
static int _nfs4_proc_readdir(struct dentry *dentry, const struct cred *cred,
u64 cookie, struct page **pages, unsigned int count, bool plus)
static int _nfs4_proc_readdir(struct nfs_readdir_arg *nr_arg,
struct nfs_readdir_res *nr_res)
{
struct inode *dir = d_inode(dentry);
struct inode *dir = d_inode(nr_arg->dentry);
struct nfs_server *server = NFS_SERVER(dir);
struct nfs4_readdir_arg args = {
.fh = NFS_FH(dir),
.pages = pages,
.pages = nr_arg->pages,
.pgbase = 0,
.count = count,
.bitmask = NFS_SERVER(d_inode(dentry))->attr_bitmask,
.plus = plus,
.count = nr_arg->page_len,
.plus = nr_arg->plus,
};
struct nfs4_readdir_res res;
struct rpc_message msg = {
.rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_READDIR],
.rpc_argp = &args,
.rpc_resp = &res,
.rpc_cred = cred,
.rpc_cred = nr_arg->cred,
};
int status;
dprintk("%s: dentry = %pd2, cookie = %Lu\n", __func__,
dentry,
(unsigned long long)cookie);
nfs4_setup_readdir(cookie, NFS_I(dir)->cookieverf, dentry, &args);
dprintk("%s: dentry = %pd2, cookie = %llu\n", __func__,
nr_arg->dentry, (unsigned long long)nr_arg->cookie);
if (!(server->caps & NFS_CAP_SECURITY_LABEL))
args.bitmask = server->attr_bitmask_nl;
else
args.bitmask = server->attr_bitmask;
nfs4_setup_readdir(nr_arg->cookie, nr_arg->verf, nr_arg->dentry, &args);
res.pgbase = args.pgbase;
status = nfs4_call_sync(NFS_SERVER(dir)->client, NFS_SERVER(dir), &msg, &args.seq_args, &res.seq_res, 0);
status = nfs4_call_sync(server->client, server, &msg, &args.seq_args,
&res.seq_res, 0);
if (status >= 0) {
memcpy(NFS_I(dir)->cookieverf, res.verifier.data, NFS4_VERIFIER_SIZE);
memcpy(nr_res->verf, res.verifier.data, NFS4_VERIFIER_SIZE);
status += args.pgbase;
}
@@ -4995,19 +5006,18 @@ static int _nfs4_proc_readdir(struct dentry *dentry, const struct cred *cred,
return status;
}
static int nfs4_proc_readdir(struct dentry *dentry, const struct cred *cred,
u64 cookie, struct page **pages, unsigned int count, bool plus)
static int nfs4_proc_readdir(struct nfs_readdir_arg *arg,
struct nfs_readdir_res *res)
{
struct nfs4_exception exception = {
.interruptible = true,
};
int err;
do {
err = _nfs4_proc_readdir(dentry, cred, cookie,
pages, count, plus);
trace_nfs4_readdir(d_inode(dentry), err);
err = nfs4_handle_exception(NFS_SERVER(d_inode(dentry)), err,
&exception);
err = _nfs4_proc_readdir(arg, res);
trace_nfs4_readdir(d_inode(arg->dentry), err);
err = nfs4_handle_exception(NFS_SERVER(d_inode(arg->dentry)),
err, &exception);
} while (exception.retry);
return err;
}
@@ -5310,17 +5320,17 @@ static int nfs4_read_done(struct rpc_task *task, struct nfs_pgio_header *hdr)
}
#if defined CONFIG_NFS_V4_2 && defined CONFIG_NFS_V4_2_READ_PLUS
static void nfs42_read_plus_support(struct nfs_server *server, struct rpc_message *msg)
static void nfs42_read_plus_support(struct nfs_pgio_header *hdr,
struct rpc_message *msg)
{
if (server->caps & NFS_CAP_READ_PLUS)
/* Note: We don't use READ_PLUS with pNFS yet */
if (nfs_server_capable(hdr->inode, NFS_CAP_READ_PLUS) && !hdr->ds_clp)
msg->rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_READ_PLUS];
else
msg->rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_READ];
}
#else
static void nfs42_read_plus_support(struct nfs_server *server, struct rpc_message *msg)
static void nfs42_read_plus_support(struct nfs_pgio_header *hdr,
struct rpc_message *msg)
{
msg->rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_READ];
}
#endif /* CONFIG_NFS_V4_2 */
@@ -5330,7 +5340,8 @@ static void nfs4_proc_read_setup(struct nfs_pgio_header *hdr,
hdr->timestamp = jiffies;
if (!hdr->pgio_done_cb)
hdr->pgio_done_cb = nfs4_read_done_cb;
nfs42_read_plus_support(NFS_SERVER(hdr->inode), msg);
msg->rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_READ];
nfs42_read_plus_support(hdr, msg);
nfs4_init_sequence(&hdr->args.seq_args, &hdr->res.seq_res, 0, 0);
}
@@ -9654,6 +9665,8 @@ _nfs4_proc_getdeviceinfo(struct nfs_server *server,
if (res.notification != args.notify_types)
pdev->nocache = 1;
trace_nfs4_getdeviceinfo(server, &pdev->dev_id, status);
dprintk("<-- %s status=%d\n", __func__, status);
return status;

2
fs/nfs/nfs4session.h
View File

@@ -34,7 +34,7 @@ enum nfs4_slot_tbl_state {
NFS4_SLOT_TBL_DRAINING,
};
#define SLOT_TABLE_SZ DIV_ROUND_UP(NFS4_MAX_SLOT_TABLE, 8*sizeof(long))
#define SLOT_TABLE_SZ DIV_ROUND_UP(NFS4_MAX_SLOT_TABLE, BITS_PER_LONG)
struct nfs4_slot_table {
struct nfs4_session *session; /* Parent session */
struct nfs4_slot *slots; /* seqid per slot */

2
fs/nfs/nfs4super.c
View File

@@ -67,7 +67,7 @@ static void nfs4_evict_inode(struct inode *inode)
nfs_inode_evict_delegation(inode);
/* Note that above delegreturn would trigger pnfs return-on-close */
pnfs_return_layout(inode);
pnfs_destroy_layout(NFS_I(inode));
pnfs_destroy_layout_final(NFS_I(inode));
/* First call standard NFS clear_inode() code */
nfs_clear_inode(inode);
nfs4_xattr_cache_zap(inode);

75
fs/nfs/nfs4trace.h
View File

@@ -2189,6 +2189,81 @@ DEFINE_PNFS_LAYOUT_EVENT(pnfs_mds_fallback_write_done);
DEFINE_PNFS_LAYOUT_EVENT(pnfs_mds_fallback_read_pagelist);
DEFINE_PNFS_LAYOUT_EVENT(pnfs_mds_fallback_write_pagelist);
DECLARE_EVENT_CLASS(nfs4_deviceid_event,
TP_PROTO(
const struct nfs_client *clp,
const struct nfs4_deviceid *deviceid
),
TP_ARGS(clp, deviceid),
TP_STRUCT__entry(
__string(dstaddr, clp->cl_hostname)
__array(unsigned char, deviceid, NFS4_DEVICEID4_SIZE)
),
TP_fast_assign(
__assign_str(dstaddr, clp->cl_hostname);
memcpy(__entry->deviceid, deviceid->data,
NFS4_DEVICEID4_SIZE);
),
TP_printk(
"deviceid=%s, dstaddr=%s",
__print_hex(__entry->deviceid, NFS4_DEVICEID4_SIZE),
__get_str(dstaddr)
)
);
#define DEFINE_PNFS_DEVICEID_EVENT(name) \
DEFINE_EVENT(nfs4_deviceid_event, name, \
TP_PROTO(const struct nfs_client *clp, \
const struct nfs4_deviceid *deviceid \
), \
TP_ARGS(clp, deviceid))
DEFINE_PNFS_DEVICEID_EVENT(nfs4_deviceid_free);
DECLARE_EVENT_CLASS(nfs4_deviceid_status,
TP_PROTO(
const struct nfs_server *server,
const struct nfs4_deviceid *deviceid,
int status
),
TP_ARGS(server, deviceid, status),
TP_STRUCT__entry(
__field(dev_t, dev)
__field(int, status)
__string(dstaddr, server->nfs_client->cl_hostname)
__array(unsigned char, deviceid, NFS4_DEVICEID4_SIZE)
),
TP_fast_assign(
__entry->dev = server->s_dev;
__entry->status = status;
__assign_str(dstaddr, server->nfs_client->cl_hostname);
memcpy(__entry->deviceid, deviceid->data,
NFS4_DEVICEID4_SIZE);
),
TP_printk(
"dev=%02x:%02x: deviceid=%s, dstaddr=%s, status=%d",
MAJOR(__entry->dev), MINOR(__entry->dev),
__print_hex(__entry->deviceid, NFS4_DEVICEID4_SIZE),
__get_str(dstaddr),
__entry->status
)
);
#define DEFINE_PNFS_DEVICEID_STATUS(name) \
DEFINE_EVENT(nfs4_deviceid_status, name, \
TP_PROTO(const struct nfs_server *server, \
const struct nfs4_deviceid *deviceid, \
int status \
), \
TP_ARGS(server, deviceid, status))
DEFINE_PNFS_DEVICEID_STATUS(nfs4_getdeviceinfo);
DEFINE_PNFS_DEVICEID_STATUS(nfs4_find_deviceid);
DECLARE_EVENT_CLASS(nfs4_flexfiles_io_event,
TP_PROTO(
const struct nfs_pgio_header *hdr

Some files were not shown because too many files have changed in this diff Show More