shinys000114/linux
1
0
Fork 0
mirror of https://github.com/hardkernel/linux.git synced 2026-06-05 10:31:46 +09:00
Code Issues Packages Projects Releases Wiki Activity

Merge 2dacc1e57b ("Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma") into android-mainline

Browse Source 
Steps on the way to 5.18-rc1

Signed-off-by: Greg Kroah-Hartman <gregkh@google.com>
Change-Id: Id0ee951aba8f477e4d8b5b051aa5d8843a80c965
This commit is contained in:
Greg Kroah-Hartman
2022-04-08 07:21:05 +02:00
parent 4b7cbdd012 2dacc1e57b
commit a9954d2db7
187 changed files with 7535 additions and 4704 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
Documentation/ABI/testing/sysfs-bus-cxl
View File

@@ -1,3 +1,12 @@
What: /sys/bus/cxl/flush
Date: Januarry, 2022
KernelVersion: v5.18
Contact: linux-cxl@vger.kernel.org
Description:
(WO) If userspace manually unbinds a port the kernel schedules
all descendant memdevs for unbind. Writing '1' to this attribute
flushes that work.
What: /sys/bus/cxl/devices/memX/firmware_version
Date: December, 2020
KernelVersion: v5.12
@@ -25,6 +34,24 @@ Description:
identically named field in the Identify Memory Device Output
Payload in the CXL-2.0 specification.
What: /sys/bus/cxl/devices/memX/serial
Date: January, 2022
KernelVersion: v5.18
Contact: linux-cxl@vger.kernel.org
Description:
(RO) 64-bit serial number per the PCIe Device Serial Number
capability. Mandatory for CXL devices, see CXL 2.0 8.1.12.2
Memory Device PCIe Capabilities and Extended Capabilities.
What: /sys/bus/cxl/devices/memX/numa_node
Date: January, 2022
KernelVersion: v5.18
Contact: linux-cxl@vger.kernel.org
Description:
(RO) If NUMA is enabled and the platform has affinitized the
host PCI device for this memory device, emit the CPU node
affinity for this device.
What: /sys/bus/cxl/devices/*/devtype
Date: June, 2021
KernelVersion: v5.14
@@ -34,6 +61,15 @@ Description:
the same value communicated in the DEVTYPE environment variable
for uevents for devices on the "cxl" bus.
What: /sys/bus/cxl/devices/*/modalias
Date: December, 2021
KernelVersion: v5.18
Contact: linux-cxl@vger.kernel.org
Description:
CXL device objects export the modalias attribute which mirrors
the same value communicated in the MODALIAS environment variable
for uevents for devices on the "cxl" bus.
What: /sys/bus/cxl/devices/portX/uport
Date: June, 2021
KernelVersion: v5.14

315
Documentation/driver-api/cxl/memory-devices.rst
View File

@@ -14,6 +14,303 @@ that optionally define a device's contribution to an interleaved address
range across multiple devices underneath a host-bridge or interleaved
across host-bridges.
CXL Bus: Theory of Operation
============================
Similar to how a RAID driver takes disk objects and assembles them into a new
logical device, the CXL subsystem is tasked to take PCIe and ACPI objects and
assemble them into a CXL.mem decode topology. The need for runtime configuration
of the CXL.mem topology is also similar to RAID in that different environments
with the same hardware configuration may decide to assemble the topology in
contrasting ways. One may choose performance (RAID0) striping memory across
multiple Host Bridges and endpoints while another may opt for fault tolerance
and disable any striping in the CXL.mem topology.
Platform firmware enumerates a menu of interleave options at the "CXL root port"
(Linux term for the top of the CXL decode topology). From there, PCIe topology
dictates which endpoints can participate in which Host Bridge decode regimes.
Each PCIe Switch in the path between the root and an endpoint introduces a point
at which the interleave can be split. For example platform firmware may say at a
given range only decodes to 1 one Host Bridge, but that Host Bridge may in turn
interleave cycles across multiple Root Ports. An intervening Switch between a
port and an endpoint may interleave cycles across multiple Downstream Switch
Ports, etc.
Here is a sample listing of a CXL topology defined by 'cxl_test'. The 'cxl_test'
module generates an emulated CXL topology of 2 Host Bridges each with 2 Root
Ports. Each of those Root Ports are connected to 2-way switches with endpoints
connected to those downstream ports for a total of 8 endpoints::
# cxl list -BEMPu -b cxl_test
{
"bus":"root3",
"provider":"cxl_test",
"ports:root3":[
{
"port":"port5",
"host":"cxl_host_bridge.1",
"ports:port5":[
{
"port":"port8",
"host":"cxl_switch_uport.1",
"endpoints:port8":[
{
"endpoint":"endpoint9",
"host":"mem2",
"memdev":{
"memdev":"mem2",
"pmem_size":"256.00 MiB (268.44 MB)",
"ram_size":"256.00 MiB (268.44 MB)",
"serial":"0x1",
"numa_node":1,
"host":"cxl_mem.1"
}
},
{
"endpoint":"endpoint15",
"host":"mem6",
"memdev":{
"memdev":"mem6",
"pmem_size":"256.00 MiB (268.44 MB)",
"ram_size":"256.00 MiB (268.44 MB)",
"serial":"0x5",
"numa_node":1,
"host":"cxl_mem.5"
}
}
]
},
{
"port":"port12",
"host":"cxl_switch_uport.3",
"endpoints:port12":[
{
"endpoint":"endpoint17",
"host":"mem8",
"memdev":{
"memdev":"mem8",
"pmem_size":"256.00 MiB (268.44 MB)",
"ram_size":"256.00 MiB (268.44 MB)",
"serial":"0x7",
"numa_node":1,
"host":"cxl_mem.7"
}
},
{
"endpoint":"endpoint13",
"host":"mem4",
"memdev":{
"memdev":"mem4",
"pmem_size":"256.00 MiB (268.44 MB)",
"ram_size":"256.00 MiB (268.44 MB)",
"serial":"0x3",
"numa_node":1,
"host":"cxl_mem.3"
}
}
]
}
]
},
{
"port":"port4",
"host":"cxl_host_bridge.0",
"ports:port4":[
{
"port":"port6",
"host":"cxl_switch_uport.0",
"endpoints:port6":[
{
"endpoint":"endpoint7",
"host":"mem1",
"memdev":{
"memdev":"mem1",
"pmem_size":"256.00 MiB (268.44 MB)",
"ram_size":"256.00 MiB (268.44 MB)",
"serial":"0",
"numa_node":0,
"host":"cxl_mem.0"
}
},
{
"endpoint":"endpoint14",
"host":"mem5",
"memdev":{
"memdev":"mem5",
"pmem_size":"256.00 MiB (268.44 MB)",
"ram_size":"256.00 MiB (268.44 MB)",
"serial":"0x4",
"numa_node":0,
"host":"cxl_mem.4"
}
}
]
},
{
"port":"port10",
"host":"cxl_switch_uport.2",
"endpoints:port10":[
{
"endpoint":"endpoint16",
"host":"mem7",
"memdev":{
"memdev":"mem7",
"pmem_size":"256.00 MiB (268.44 MB)",
"ram_size":"256.00 MiB (268.44 MB)",
"serial":"0x6",
"numa_node":0,
"host":"cxl_mem.6"
}
},
{
"endpoint":"endpoint11",
"host":"mem3",
"memdev":{
"memdev":"mem3",
"pmem_size":"256.00 MiB (268.44 MB)",
"ram_size":"256.00 MiB (268.44 MB)",
"serial":"0x2",
"numa_node":0,
"host":"cxl_mem.2"
}
}
]
}
]
}
]
}
In that listing each "root", "port", and "endpoint" object correspond a kernel
'struct cxl_port' object. A 'cxl_port' is a device that can decode CXL.mem to
its descendants. So "root" claims non-PCIe enumerable platform decode ranges and
decodes them to "ports", "ports" decode to "endpoints", and "endpoints"
represent the decode from SPA (System Physical Address) to DPA (Device Physical
Address).
Continuing the RAID analogy, disks have both topology metadata and on device
metadata that determine RAID set assembly. CXL Port topology and CXL Port link
status is metadata for CXL.mem set assembly. The CXL Port topology is enumerated
by the arrival of a CXL.mem device. I.e. unless and until the PCIe core attaches
the cxl_pci driver to a CXL Memory Expander there is no role for CXL Port
objects. Conversely for hot-unplug / removal scenarios, there is no need for
the Linux PCI core to tear down switch-level CXL resources because the endpoint
->remove() event cleans up the port data that was established to support that
Memory Expander.
The port metadata and potential decode schemes that a give memory device may
participate can be determined via a command like::
# cxl list -BDMu -d root -m mem3
{
"bus":"root3",
"provider":"cxl_test",
"decoders:root3":[
{
"decoder":"decoder3.1",
"resource":"0x8030000000",
"size":"512.00 MiB (536.87 MB)",
"volatile_capable":true,
"nr_targets":2
},
{
"decoder":"decoder3.3",
"resource":"0x8060000000",
"size":"512.00 MiB (536.87 MB)",
"pmem_capable":true,
"nr_targets":2
},
{
"decoder":"decoder3.0",
"resource":"0x8020000000",
"size":"256.00 MiB (268.44 MB)",
"volatile_capable":true,
"nr_targets":1
},
{
"decoder":"decoder3.2",
"resource":"0x8050000000",
"size":"256.00 MiB (268.44 MB)",
"pmem_capable":true,
"nr_targets":1
}
],
"memdevs:root3":[
{
"memdev":"mem3",
"pmem_size":"256.00 MiB (268.44 MB)",
"ram_size":"256.00 MiB (268.44 MB)",
"serial":"0x2",
"numa_node":0,
"host":"cxl_mem.2"
}
]
}
...which queries the CXL topology to ask "given CXL Memory Expander with a kernel
device name of 'mem3' which platform level decode ranges may this device
participate". A given expander can participate in multiple CXL.mem interleave
sets simultaneously depending on how many decoder resource it has. In this
example mem3 can participate in one or more of a PMEM interleave that spans to
Host Bridges, a PMEM interleave that targets a single Host Bridge, a Volatile
memory interleave that spans 2 Host Bridges, and a Volatile memory interleave
that only targets a single Host Bridge.
Conversely the memory devices that can participate in a given platform level
decode scheme can be determined via a command like the following::
# cxl list -MDu -d 3.2
[
{
"memdevs":[
{
"memdev":"mem1",
"pmem_size":"256.00 MiB (268.44 MB)",
"ram_size":"256.00 MiB (268.44 MB)",
"serial":"0",
"numa_node":0,
"host":"cxl_mem.0"
},
{
"memdev":"mem5",
"pmem_size":"256.00 MiB (268.44 MB)",
"ram_size":"256.00 MiB (268.44 MB)",
"serial":"0x4",
"numa_node":0,
"host":"cxl_mem.4"
},
{
"memdev":"mem7",
"pmem_size":"256.00 MiB (268.44 MB)",
"ram_size":"256.00 MiB (268.44 MB)",
"serial":"0x6",
"numa_node":0,
"host":"cxl_mem.6"
},
{
"memdev":"mem3",
"pmem_size":"256.00 MiB (268.44 MB)",
"ram_size":"256.00 MiB (268.44 MB)",
"serial":"0x2",
"numa_node":0,
"host":"cxl_mem.2"
}
]
},
{
"root decoders":[
{
"decoder":"decoder3.2",
"resource":"0x8050000000",
"size":"256.00 MiB (268.44 MB)",
"pmem_capable":true,
"nr_targets":1
}
]
}
]
...where the naming scheme for decoders is "decoder<port_id>.<instance_id>".
Driver Infrastructure
=====================
@@ -28,6 +325,14 @@ CXL Memory Device
.. kernel-doc:: drivers/cxl/pci.c
:internal:
.. kernel-doc:: drivers/cxl/mem.c
:doc: cxl mem
CXL Port
--------
.. kernel-doc:: drivers/cxl/port.c
:doc: cxl port
CXL Core
--------
.. kernel-doc:: drivers/cxl/cxl.h
@@ -36,10 +341,16 @@ CXL Core
.. kernel-doc:: drivers/cxl/cxl.h
:internal:
.. kernel-doc:: drivers/cxl/core/bus.c
.. kernel-doc:: drivers/cxl/core/port.c
:doc: cxl core
.. kernel-doc:: drivers/cxl/core/bus.c
.. kernel-doc:: drivers/cxl/core/port.c
:identifiers:
.. kernel-doc:: drivers/cxl/core/pci.c
:doc: cxl core pci
.. kernel-doc:: drivers/cxl/core/pci.c
:identifiers:
.. kernel-doc:: drivers/cxl/core/pmem.c

2
MAINTAINERS
View File

@@ -4456,6 +4456,7 @@ F: drivers/power/supply/cw2015_battery.c
CEPH COMMON CODE (LIBCEPH)
M: Ilya Dryomov <idryomov@gmail.com>
M: Jeff Layton <jlayton@kernel.org>
M: Xiubo Li <xiubli@redhat.com>
L: ceph-devel@vger.kernel.org
S: Supported
W: http://ceph.com/
@@ -4466,6 +4467,7 @@ F: net/ceph/
CEPH DISTRIBUTED FILE SYSTEM CLIENT (CEPH)
M: Jeff Layton <jlayton@kernel.org>
M: Xiubo Li <xiubli@redhat.com>
M: Ilya Dryomov <idryomov@gmail.com>
L: ceph-devel@vger.kernel.org
S: Supported

44
drivers/cxl/Kconfig
View File

@@ -13,25 +13,26 @@ menuconfig CXL_BUS
if CXL_BUS
config CXL_MEM
tristate "CXL.mem: Memory Devices"
config CXL_PCI
tristate "PCI manageability"
default CXL_BUS
help
The CXL.mem protocol allows a device to act as a provider of
"System RAM" and/or "Persistent Memory" that is fully coherent
as if the memory was attached to the typical CPU memory
controller.
The CXL specification defines a "CXL memory device" sub-class in the
PCI "memory controller" base class of devices. Device's identified by
this class code provide support for volatile and / or persistent
memory to be mapped into the system address map (Host-managed Device
Memory (HDM)).
Say 'y/m' to enable a driver that will attach to CXL.mem devices for
configuration and management primarily via the mailbox interface. See
Chapter 2.3 Type 3 CXL Device in the CXL 2.0 specification for more
details.
Say 'y/m' to enable a driver that will attach to CXL memory expander
devices enumerated by the memory device class code for configuration
and management primarily via the mailbox interface. See Chapter 2.3
Type 3 CXL Device in the CXL 2.0 specification for more details.
If unsure say 'm'.
config CXL_MEM_RAW_COMMANDS
bool "RAW Command Interface for Memory Devices"
depends on CXL_MEM
depends on CXL_PCI
help
Enable CXL RAW command interface.
@@ -76,4 +77,25 @@ config CXL_PMEM
provisioning the persistent memory capacity of CXL memory expanders.
If unsure say 'm'.
config CXL_MEM
tristate "CXL: Memory Expansion"
depends on CXL_PCI
default CXL_BUS
help
The CXL.mem protocol allows a device to act as a provider of "System
RAM" and/or "Persistent Memory" that is fully coherent as if the
memory were attached to the typical CPU memory controller. This is
known as HDM "Host-managed Device Memory".
Say 'y/m' to enable a driver that will attach to CXL.mem devices for
memory expansion and control of HDM. See Chapter 9.13 in the CXL 2.0
specification for a detailed description of HDM.
If unsure say 'm'.
config CXL_PORT
default CXL_BUS
tristate
endif

6
drivers/cxl/Makefile
View File

@@ -1,9 +1,13 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_CXL_BUS) += core/
obj-$(CONFIG_CXL_MEM) += cxl_pci.o
obj-$(CONFIG_CXL_PCI) += cxl_pci.o
obj-$(CONFIG_CXL_MEM) += cxl_mem.o
obj-$(CONFIG_CXL_ACPI) += cxl_acpi.o
obj-$(CONFIG_CXL_PMEM) += cxl_pmem.o
obj-$(CONFIG_CXL_PORT) += cxl_port.o
cxl_mem-y := mem.o
cxl_pci-y := pci.o
cxl_acpi-y := acpi.o
cxl_pmem-y := pmem.o
cxl_port-y := port.o

154
drivers/cxl/acpi.c
View File

@@ -6,6 +6,7 @@
#include <linux/kernel.h>
#include <linux/acpi.h>
#include <linux/pci.h>
#include "cxlpci.h"
#include "cxl.h"
/* Encode defined in CXL 2.0 8.2.5.12.7 HDM Decoder Control Register */
@@ -14,7 +15,7 @@
static unsigned long cfmws_to_decoder_flags(int restrictions)
{
unsigned long flags = 0;
unsigned long flags = CXL_DECODER_F_ENABLE;
if (restrictions & ACPI_CEDT_CFMWS_RESTRICT_TYPE2)
flags |= CXL_DECODER_F_TYPE2;
@@ -101,16 +102,14 @@ static int cxl_parse_cfmws(union acpi_subtable_headers *header, void *arg,
for (i = 0; i < CFMWS_INTERLEAVE_WAYS(cfmws); i++)
target_map[i] = cfmws->interleave_targets[i];
cxld = cxl_decoder_alloc(root_port, CFMWS_INTERLEAVE_WAYS(cfmws));
cxld = cxl_root_decoder_alloc(root_port, CFMWS_INTERLEAVE_WAYS(cfmws));
if (IS_ERR(cxld))
return 0;
cxld->flags = cfmws_to_decoder_flags(cfmws->restrictions);
cxld->target_type = CXL_DECODER_EXPANDER;
cxld->range = (struct range){
.start = cfmws->base_hpa,
.end = cfmws->base_hpa + cfmws->window_size - 1,
};
cxld->platform_res = (struct resource)DEFINE_RES_MEM(cfmws->base_hpa,
cfmws->window_size);
cxld->interleave_ways = CFMWS_INTERLEAVE_WAYS(cfmws);
cxld->interleave_granularity = CFMWS_INTERLEAVE_GRANULARITY(cfmws);
@@ -120,67 +119,17 @@ static int cxl_parse_cfmws(union acpi_subtable_headers *header, void *arg,
else
rc = cxl_decoder_autoremove(dev, cxld);
if (rc) {
dev_err(dev, "Failed to add decoder for %#llx-%#llx\n",
cfmws->base_hpa,
cfmws->base_hpa + cfmws->window_size - 1);
dev_err(dev, "Failed to add decoder for %pr\n",
&cxld->platform_res);
return 0;
}
dev_dbg(dev, "add: %s node: %d range %#llx-%#llx\n",
dev_name(&cxld->dev), phys_to_target_node(cxld->range.start),
cfmws->base_hpa, cfmws->base_hpa + cfmws->window_size - 1);
dev_dbg(dev, "add: %s node: %d range %pr\n", dev_name(&cxld->dev),
phys_to_target_node(cxld->platform_res.start),
&cxld->platform_res);
return 0;
}
__mock int match_add_root_ports(struct pci_dev *pdev, void *data)
{
struct cxl_walk_context *ctx = data;
struct pci_bus *root_bus = ctx->root;
struct cxl_port *port = ctx->port;
int type = pci_pcie_type(pdev);
struct device *dev = ctx->dev;
u32 lnkcap, port_num;
int rc;
if (pdev->bus != root_bus)
return 0;
if (!pci_is_pcie(pdev))
return 0;
if (type != PCI_EXP_TYPE_ROOT_PORT)
return 0;
if (pci_read_config_dword(pdev, pci_pcie_cap(pdev) + PCI_EXP_LNKCAP,
&lnkcap) != PCIBIOS_SUCCESSFUL)
return 0;
/* TODO walk DVSEC to find component register base */
port_num = FIELD_GET(PCI_EXP_LNKCAP_PN, lnkcap);
rc = cxl_add_dport(port, &pdev->dev, port_num, CXL_RESOURCE_NONE);
if (rc) {
ctx->error = rc;
return rc;
}
ctx->count++;
dev_dbg(dev, "add dport%d: %s\n", port_num, dev_name(&pdev->dev));
return 0;
}
static struct cxl_dport *find_dport_by_dev(struct cxl_port *port, struct device *dev)
{
struct cxl_dport *dport;
device_lock(&port->dev);
list_for_each_entry(dport, &port->dports, list)
if (dport->dport == dev) {
device_unlock(&port->dev);
return dport;
}
device_unlock(&port->dev);
return NULL;
}
__mock struct acpi_device *to_cxl_host_bridge(struct device *host,
struct device *dev)
{
@@ -204,83 +153,35 @@ static int add_host_bridge_uport(struct device *match, void *arg)
struct device *host = root_port->dev.parent;
struct acpi_device *bridge = to_cxl_host_bridge(host, match);
struct acpi_pci_root *pci_root;
struct cxl_walk_context ctx;
int single_port_map[1], rc;
struct cxl_decoder *cxld;
struct cxl_dport *dport;
struct cxl_port *port;
int rc;
if (!bridge)
return 0;
dport = find_dport_by_dev(root_port, match);
dport = cxl_find_dport_by_dev(root_port, match);
if (!dport) {
dev_dbg(host, "host bridge expected and not found\n");
return 0;
}
/*
* Note that this lookup already succeeded in
* to_cxl_host_bridge(), so no need to check for failure here
*/
pci_root = acpi_pci_find_root(bridge->handle);
rc = devm_cxl_register_pci_bus(host, match, pci_root->bus);
if (rc)
return rc;
port = devm_cxl_add_port(host, match, dport->component_reg_phys,
root_port);
if (IS_ERR(port))
return PTR_ERR(port);
dev_dbg(host, "%s: add: %s\n", dev_name(match), dev_name(&port->dev));
/*
* Note that this lookup already succeeded in
* to_cxl_host_bridge(), so no need to check for failure here
*/
pci_root = acpi_pci_find_root(bridge->handle);
ctx = (struct cxl_walk_context){
.dev = host,
.root = pci_root->bus,
.port = port,
};
pci_walk_bus(pci_root->bus, match_add_root_ports, &ctx);
if (ctx.count == 0)
return -ENODEV;
if (ctx.error)
return ctx.error;
if (ctx.count > 1)
return 0;
/* TODO: Scan CHBCR for HDM Decoder resources */
/*
* Per the CXL specification (8.2.5.12 CXL HDM Decoder Capability
* Structure) single ported host-bridges need not publish a decoder
* capability when a passthrough decode can be assumed, i.e. all
* transactions that the uport sees are claimed and passed to the single
* dport. Disable the range until the first CXL region is enumerated /
* activated.
*/
cxld = cxl_decoder_alloc(port, 1);
if (IS_ERR(cxld))
return PTR_ERR(cxld);
cxld->interleave_ways = 1;
cxld->interleave_granularity = PAGE_SIZE;
cxld->target_type = CXL_DECODER_EXPANDER;
cxld->range = (struct range) {
.start = 0,
.end = -1,
};
device_lock(&port->dev);
dport = list_first_entry(&port->dports, typeof(*dport), list);
device_unlock(&port->dev);
single_port_map[0] = dport->port_id;
rc = cxl_decoder_add(cxld, single_port_map);
if (rc)
put_device(&cxld->dev);
else
rc = cxl_decoder_autoremove(host, cxld);
if (rc == 0)
dev_dbg(host, "add: %s\n", dev_name(&cxld->dev));
return rc;
return 0;
}
struct cxl_chbs_context {
@@ -309,9 +210,9 @@ static int cxl_get_chbcr(union acpi_subtable_headers *header, void *arg,
static int add_host_bridge_dport(struct device *match, void *arg)
{
int rc;
acpi_status status;
unsigned long long uid;
struct cxl_dport *dport;
struct cxl_chbs_context ctx;
struct cxl_port *root_port = arg;
struct device *host = root_port->dev.parent;
@@ -340,11 +241,11 @@ static int add_host_bridge_dport(struct device *match, void *arg)
return 0;
}
rc = cxl_add_dport(root_port, match, uid, ctx.chbcr);
if (rc) {
dport = devm_cxl_add_dport(root_port, match, uid, ctx.chbcr);
if (IS_ERR(dport)) {
dev_err(host, "failed to add downstream port: %s\n",
dev_name(match));
return rc;
return PTR_ERR(dport);
}
dev_dbg(host, "add dport%llu: %s\n", uid, dev_name(match));
return 0;
@@ -413,7 +314,8 @@ static int cxl_acpi_probe(struct platform_device *pdev)
if (rc < 0)
return rc;
return 0;
/* In case PCI is scanned before ACPI re-trigger memdev attach */
return cxl_bus_rescan();
}
static const struct acpi_device_id cxl_acpi_ids[] = {

4
drivers/cxl/core/Makefile
View File

@@ -2,8 +2,10 @@
obj-$(CONFIG_CXL_BUS) += cxl_core.o
ccflags-y += -I$(srctree)/drivers/cxl
cxl_core-y := bus.o
cxl_core-y := port.o
cxl_core-y += pmem.o
cxl_core-y += regs.o
cxl_core-y += memdev.o
cxl_core-y += mbox.o
cxl_core-y += pci.o
cxl_core-y += hdm.o

675
drivers/cxl/core/bus.c
View File

@@ -1,675 +0,0 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2020 Intel Corporation. All rights reserved. */
#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/device.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/slab.h>
#include <linux/idr.h>
#include <cxlmem.h>
#include <cxl.h>
#include "core.h"
/**
* DOC: cxl core
*
* The CXL core provides a set of interfaces that can be consumed by CXL aware
* drivers. The interfaces allow for creation, modification, and destruction of
* regions, memory devices, ports, and decoders. CXL aware drivers must register
* with the CXL core via these interfaces in order to be able to participate in
* cross-device interleave coordination. The CXL core also establishes and
* maintains the bridge to the nvdimm subsystem.
*
* CXL core introduces sysfs hierarchy to control the devices that are
* instantiated by the core.
*/
static DEFINE_IDA(cxl_port_ida);
static ssize_t devtype_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
return sysfs_emit(buf, "%s\n", dev->type->name);
}
static DEVICE_ATTR_RO(devtype);
static struct attribute *cxl_base_attributes[] = {
&dev_attr_devtype.attr,
NULL,
};
struct attribute_group cxl_base_attribute_group = {
.attrs = cxl_base_attributes,
};
static ssize_t start_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
struct cxl_decoder *cxld = to_cxl_decoder(dev);
return sysfs_emit(buf, "%#llx\n", cxld->range.start);
}
static DEVICE_ATTR_RO(start);
static ssize_t size_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
struct cxl_decoder *cxld = to_cxl_decoder(dev);
return sysfs_emit(buf, "%#llx\n", range_len(&cxld->range));
}
static DEVICE_ATTR_RO(size);
#define CXL_DECODER_FLAG_ATTR(name, flag) \
static ssize_t name##_show(struct device *dev, \
struct device_attribute *attr, char *buf) \
{ \
struct cxl_decoder *cxld = to_cxl_decoder(dev); \
\
return sysfs_emit(buf, "%s\n", \
(cxld->flags & (flag)) ? "1" : "0"); \
} \
static DEVICE_ATTR_RO(name)
CXL_DECODER_FLAG_ATTR(cap_pmem, CXL_DECODER_F_PMEM);
CXL_DECODER_FLAG_ATTR(cap_ram, CXL_DECODER_F_RAM);
CXL_DECODER_FLAG_ATTR(cap_type2, CXL_DECODER_F_TYPE2);
CXL_DECODER_FLAG_ATTR(cap_type3, CXL_DECODER_F_TYPE3);
CXL_DECODER_FLAG_ATTR(locked, CXL_DECODER_F_LOCK);
static ssize_t target_type_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct cxl_decoder *cxld = to_cxl_decoder(dev);
switch (cxld->target_type) {
case CXL_DECODER_ACCELERATOR:
return sysfs_emit(buf, "accelerator\n");
case CXL_DECODER_EXPANDER:
return sysfs_emit(buf, "expander\n");
}
return -ENXIO;
}
static DEVICE_ATTR_RO(target_type);
static ssize_t target_list_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct cxl_decoder *cxld = to_cxl_decoder(dev);
ssize_t offset = 0;
int i, rc = 0;
device_lock(dev);
for (i = 0; i < cxld->interleave_ways; i++) {
struct cxl_dport *dport = cxld->target[i];
struct cxl_dport *next = NULL;
if (!dport)
break;
if (i + 1 < cxld->interleave_ways)
next = cxld->target[i + 1];
rc = sysfs_emit_at(buf, offset, "%d%s", dport->port_id,
next ? "," : "");
if (rc < 0)
break;
offset += rc;
}
device_unlock(dev);
if (rc < 0)
return rc;
rc = sysfs_emit_at(buf, offset, "\n");
if (rc < 0)
return rc;
return offset + rc;
}
static DEVICE_ATTR_RO(target_list);
static struct attribute *cxl_decoder_base_attrs[] = {
&dev_attr_start.attr,
&dev_attr_size.attr,
&dev_attr_locked.attr,
&dev_attr_target_list.attr,
NULL,
};
static struct attribute_group cxl_decoder_base_attribute_group = {
.attrs = cxl_decoder_base_attrs,
};
static struct attribute *cxl_decoder_root_attrs[] = {
&dev_attr_cap_pmem.attr,
&dev_attr_cap_ram.attr,
&dev_attr_cap_type2.attr,
&dev_attr_cap_type3.attr,
NULL,
};
static struct attribute_group cxl_decoder_root_attribute_group = {
.attrs = cxl_decoder_root_attrs,
};
static const struct attribute_group *cxl_decoder_root_attribute_groups[] = {
&cxl_decoder_root_attribute_group,
&cxl_decoder_base_attribute_group,
&cxl_base_attribute_group,
NULL,
};
static struct attribute *cxl_decoder_switch_attrs[] = {
&dev_attr_target_type.attr,
NULL,
};
static struct attribute_group cxl_decoder_switch_attribute_group = {
.attrs = cxl_decoder_switch_attrs,
};
static const struct attribute_group *cxl_decoder_switch_attribute_groups[] = {
&cxl_decoder_switch_attribute_group,
&cxl_decoder_base_attribute_group,
&cxl_base_attribute_group,
NULL,
};
static void cxl_decoder_release(struct device *dev)
{
struct cxl_decoder *cxld = to_cxl_decoder(dev);
struct cxl_port *port = to_cxl_port(dev->parent);
ida_free(&port->decoder_ida, cxld->id);
kfree(cxld);
}
static const struct device_type cxl_decoder_switch_type = {
.name = "cxl_decoder_switch",
.release = cxl_decoder_release,
.groups = cxl_decoder_switch_attribute_groups,
};
static const struct device_type cxl_decoder_root_type = {
.name = "cxl_decoder_root",
.release = cxl_decoder_release,
.groups = cxl_decoder_root_attribute_groups,
};
bool is_root_decoder(struct device *dev)
{
return dev->type == &cxl_decoder_root_type;
}
EXPORT_SYMBOL_NS_GPL(is_root_decoder, CXL);
struct cxl_decoder *to_cxl_decoder(struct device *dev)
{
if (dev_WARN_ONCE(dev, dev->type->release != cxl_decoder_release,
"not a cxl_decoder device\n"))
return NULL;
return container_of(dev, struct cxl_decoder, dev);
}
EXPORT_SYMBOL_NS_GPL(to_cxl_decoder, CXL);
static void cxl_dport_release(struct cxl_dport *dport)
{
list_del(&dport->list);
put_device(dport->dport);
kfree(dport);
}
static void cxl_port_release(struct device *dev)
{
struct cxl_port *port = to_cxl_port(dev);
struct cxl_dport *dport, *_d;
device_lock(dev);
list_for_each_entry_safe(dport, _d, &port->dports, list)
cxl_dport_release(dport);
device_unlock(dev);
ida_free(&cxl_port_ida, port->id);
kfree(port);
}
static const struct attribute_group *cxl_port_attribute_groups[] = {
&cxl_base_attribute_group,
NULL,
};
static const struct device_type cxl_port_type = {
.name = "cxl_port",
.release = cxl_port_release,
.groups = cxl_port_attribute_groups,
};
struct cxl_port *to_cxl_port(struct device *dev)
{
if (dev_WARN_ONCE(dev, dev->type != &cxl_port_type,
"not a cxl_port device\n"))
return NULL;
return container_of(dev, struct cxl_port, dev);
}
static void unregister_port(void *_port)
{
struct cxl_port *port = _port;
struct cxl_dport *dport;
device_lock(&port->dev);
list_for_each_entry(dport, &port->dports, list) {
char link_name[CXL_TARGET_STRLEN];
if (snprintf(link_name, CXL_TARGET_STRLEN, "dport%d",
dport->port_id) >= CXL_TARGET_STRLEN)
continue;
sysfs_remove_link(&port->dev.kobj, link_name);
}
device_unlock(&port->dev);
device_unregister(&port->dev);
}
static void cxl_unlink_uport(void *_port)
{
struct cxl_port *port = _port;
sysfs_remove_link(&port->dev.kobj, "uport");
}
static int devm_cxl_link_uport(struct device *host, struct cxl_port *port)
{
int rc;
rc = sysfs_create_link(&port->dev.kobj, &port->uport->kobj, "uport");
if (rc)
return rc;
return devm_add_action_or_reset(host, cxl_unlink_uport, port);
}
static struct cxl_port *cxl_port_alloc(struct device *uport,
resource_size_t component_reg_phys,
struct cxl_port *parent_port)
{
struct cxl_port *port;
struct device *dev;
int rc;
port = kzalloc(sizeof(*port), GFP_KERNEL);
if (!port)
return ERR_PTR(-ENOMEM);
rc = ida_alloc(&cxl_port_ida, GFP_KERNEL);
if (rc < 0)
goto err;
port->id = rc;
/*
* The top-level cxl_port "cxl_root" does not have a cxl_port as
* its parent and it does not have any corresponding component
* registers as its decode is described by a fixed platform
* description.
*/
dev = &port->dev;
if (parent_port)
dev->parent = &parent_port->dev;
else
dev->parent = uport;
port->uport = uport;
port->component_reg_phys = component_reg_phys;
ida_init(&port->decoder_ida);
INIT_LIST_HEAD(&port->dports);
device_initialize(dev);
device_set_pm_not_required(dev);
dev->bus = &cxl_bus_type;
dev->type = &cxl_port_type;
return port;
err:
kfree(port);
return ERR_PTR(rc);
}
/**
* devm_cxl_add_port - register a cxl_port in CXL memory decode hierarchy
* @host: host device for devm operations
* @uport: "physical" device implementing this upstream port
* @component_reg_phys: (optional) for configurable cxl_port instances
* @parent_port: next hop up in the CXL memory decode hierarchy
*/
struct cxl_port *devm_cxl_add_port(struct device *host, struct device *uport,
resource_size_t component_reg_phys,
struct cxl_port *parent_port)
{
struct cxl_port *port;
struct device *dev;
int rc;
port = cxl_port_alloc(uport, component_reg_phys, parent_port);
if (IS_ERR(port))
return port;
dev = &port->dev;
if (parent_port)
rc = dev_set_name(dev, "port%d", port->id);
else
rc = dev_set_name(dev, "root%d", port->id);
if (rc)
goto err;
rc = device_add(dev);
if (rc)
goto err;
rc = devm_add_action_or_reset(host, unregister_port, port);
if (rc)
return ERR_PTR(rc);
rc = devm_cxl_link_uport(host, port);
if (rc)
return ERR_PTR(rc);
return port;
err:
put_device(dev);
return ERR_PTR(rc);
}
EXPORT_SYMBOL_NS_GPL(devm_cxl_add_port, CXL);
static struct cxl_dport *find_dport(struct cxl_port *port, int id)
{
struct cxl_dport *dport;
device_lock_assert(&port->dev);
list_for_each_entry (dport, &port->dports, list)
if (dport->port_id == id)
return dport;
return NULL;
}
static int add_dport(struct cxl_port *port, struct cxl_dport *new)
{
struct cxl_dport *dup;
device_lock(&port->dev);
dup = find_dport(port, new->port_id);
if (dup)
dev_err(&port->dev,
"unable to add dport%d-%s non-unique port id (%s)\n",
new->port_id, dev_name(new->dport),
dev_name(dup->dport));
else
list_add_tail(&new->list, &port->dports);
device_unlock(&port->dev);
return dup ? -EEXIST : 0;
}
/**
* cxl_add_dport - append downstream port data to a cxl_port
* @port: the cxl_port that references this dport
* @dport_dev: firmware or PCI device representing the dport
* @port_id: identifier for this dport in a decoder's target list
* @component_reg_phys: optional location of CXL component registers
*
* Note that all allocations and links are undone by cxl_port deletion
* and release.
*/
int cxl_add_dport(struct cxl_port *port, struct device *dport_dev, int port_id,
resource_size_t component_reg_phys)
{
char link_name[CXL_TARGET_STRLEN];
struct cxl_dport *dport;
int rc;
if (snprintf(link_name, CXL_TARGET_STRLEN, "dport%d", port_id) >=
CXL_TARGET_STRLEN)
return -EINVAL;
dport = kzalloc(sizeof(*dport), GFP_KERNEL);
if (!dport)
return -ENOMEM;
INIT_LIST_HEAD(&dport->list);
dport->dport = get_device(dport_dev);
dport->port_id = port_id;
dport->component_reg_phys = component_reg_phys;
dport->port = port;
rc = add_dport(port, dport);
if (rc)
goto err;
rc = sysfs_create_link(&port->dev.kobj, &dport_dev->kobj, link_name);
if (rc)
goto err;
return 0;
err:
cxl_dport_release(dport);
return rc;
}
EXPORT_SYMBOL_NS_GPL(cxl_add_dport, CXL);
static int decoder_populate_targets(struct cxl_decoder *cxld,
struct cxl_port *port, int *target_map)
{
int rc = 0, i;
if (!target_map)
return 0;
device_lock(&port->dev);
if (list_empty(&port->dports)) {
rc = -EINVAL;
goto out_unlock;
}
for (i = 0; i < cxld->nr_targets; i++) {
struct cxl_dport *dport = find_dport(port, target_map[i]);
if (!dport) {
rc = -ENXIO;
goto out_unlock;
}
cxld->target[i] = dport;
}
out_unlock:
device_unlock(&port->dev);
return rc;
}
struct cxl_decoder *cxl_decoder_alloc(struct cxl_port *port, int nr_targets)
{
struct cxl_decoder *cxld;
struct device *dev;
int rc = 0;
if (nr_targets > CXL_DECODER_MAX_INTERLEAVE || nr_targets < 1)
return ERR_PTR(-EINVAL);
cxld = kzalloc(struct_size(cxld, target, nr_targets), GFP_KERNEL);
if (!cxld)
return ERR_PTR(-ENOMEM);
rc = ida_alloc(&port->decoder_ida, GFP_KERNEL);
if (rc < 0)
goto err;
cxld->id = rc;
cxld->nr_targets = nr_targets;
dev = &cxld->dev;
device_initialize(dev);
device_set_pm_not_required(dev);
dev->parent = &port->dev;
dev->bus = &cxl_bus_type;
/* root ports do not have a cxl_port_type parent */
if (port->dev.parent->type == &cxl_port_type)
dev->type = &cxl_decoder_switch_type;
else
dev->type = &cxl_decoder_root_type;
return cxld;
err:
kfree(cxld);
return ERR_PTR(rc);
}
EXPORT_SYMBOL_NS_GPL(cxl_decoder_alloc, CXL);
int cxl_decoder_add(struct cxl_decoder *cxld, int *target_map)
{
struct cxl_port *port;
struct device *dev;
int rc;
if (WARN_ON_ONCE(!cxld))
return -EINVAL;
if (WARN_ON_ONCE(IS_ERR(cxld)))
return PTR_ERR(cxld);
if (cxld->interleave_ways < 1)
return -EINVAL;
port = to_cxl_port(cxld->dev.parent);
rc = decoder_populate_targets(cxld, port, target_map);
if (rc)
return rc;
dev = &cxld->dev;
rc = dev_set_name(dev, "decoder%d.%d", port->id, cxld->id);
if (rc)
return rc;
return device_add(dev);
}
EXPORT_SYMBOL_NS_GPL(cxl_decoder_add, CXL);
static void cxld_unregister(void *dev)
{
device_unregister(dev);
}
int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld)
{
return devm_add_action_or_reset(host, cxld_unregister, &cxld->dev);
}
EXPORT_SYMBOL_NS_GPL(cxl_decoder_autoremove, CXL);
/**
* __cxl_driver_register - register a driver for the cxl bus
* @cxl_drv: cxl driver structure to attach
* @owner: owning module/driver
* @modname: KBUILD_MODNAME for parent driver
*/
int __cxl_driver_register(struct cxl_driver *cxl_drv, struct module *owner,
const char *modname)
{
if (!cxl_drv->probe) {
pr_debug("%s ->probe() must be specified\n", modname);
return -EINVAL;
}
if (!cxl_drv->name) {
pr_debug("%s ->name must be specified\n", modname);
return -EINVAL;
}
if (!cxl_drv->id) {
pr_debug("%s ->id must be specified\n", modname);
return -EINVAL;
}
cxl_drv->drv.bus = &cxl_bus_type;
cxl_drv->drv.owner = owner;
cxl_drv->drv.mod_name = modname;
cxl_drv->drv.name = cxl_drv->name;
return driver_register(&cxl_drv->drv);
}
EXPORT_SYMBOL_NS_GPL(__cxl_driver_register, CXL);
void cxl_driver_unregister(struct cxl_driver *cxl_drv)
{
driver_unregister(&cxl_drv->drv);
}
EXPORT_SYMBOL_NS_GPL(cxl_driver_unregister, CXL);
static int cxl_device_id(struct device *dev)
{
if (dev->type == &cxl_nvdimm_bridge_type)
return CXL_DEVICE_NVDIMM_BRIDGE;
if (dev->type == &cxl_nvdimm_type)
return CXL_DEVICE_NVDIMM;
return 0;
}
static int cxl_bus_uevent(struct device *dev, struct kobj_uevent_env *env)
{
return add_uevent_var(env, "MODALIAS=" CXL_MODALIAS_FMT,
cxl_device_id(dev));
}
static int cxl_bus_match(struct device *dev, struct device_driver *drv)
{
return cxl_device_id(dev) == to_cxl_drv(drv)->id;
}
static int cxl_bus_probe(struct device *dev)
{
return to_cxl_drv(dev->driver)->probe(dev);
}
static void cxl_bus_remove(struct device *dev)
{
struct cxl_driver *cxl_drv = to_cxl_drv(dev->driver);
if (cxl_drv->remove)
cxl_drv->remove(dev);
}
struct bus_type cxl_bus_type = {
.name = "cxl",
.uevent = cxl_bus_uevent,
.match = cxl_bus_match,
.probe = cxl_bus_probe,
.remove = cxl_bus_remove,
};
EXPORT_SYMBOL_NS_GPL(cxl_bus_type, CXL);
static __init int cxl_core_init(void)
{
int rc;
cxl_mbox_init();
rc = cxl_memdev_init();
if (rc)
return rc;
rc = bus_register(&cxl_bus_type);
if (rc)
goto err;
return 0;
err:
cxl_memdev_exit();
cxl_mbox_exit();
return rc;
}
static void cxl_core_exit(void)
{
bus_unregister(&cxl_bus_type);
cxl_memdev_exit();
cxl_mbox_exit();
}
module_init(cxl_core_init);
module_exit(cxl_core_exit);
MODULE_LICENSE("GPL v2");

2
drivers/cxl/core/core.h
View File

@@ -14,6 +14,8 @@ struct cxl_mem_query_commands;
int cxl_query_cmd(struct cxl_memdev *cxlmd,
struct cxl_mem_query_commands __user *q);
int cxl_send_cmd(struct cxl_memdev *cxlmd, struct cxl_send_command __user *s);
void __iomem *devm_cxl_iomap_block(struct device *dev, resource_size_t addr,
resource_size_t length);
int cxl_memdev_init(void);
void cxl_memdev_exit(void);

276
drivers/cxl/core/hdm.c Normal file
View File

@@ -0,0 +1,276 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2022 Intel Corporation. All rights reserved. */
#include <linux/io-64-nonatomic-hi-lo.h>
#include <linux/device.h>
#include <linux/delay.h>
#include "cxlmem.h"
#include "core.h"
/**
* DOC: cxl core hdm
*
* Compute Express Link Host Managed Device Memory, starting with the
* CXL 2.0 specification, is managed by an array of HDM Decoder register
* instances per CXL port and per CXL endpoint. Define common helpers
* for enumerating these registers and capabilities.
*/
static int add_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
int *target_map)
{
int rc;
rc = cxl_decoder_add_locked(cxld, target_map);
if (rc) {
put_device(&cxld->dev);
dev_err(&port->dev, "Failed to add decoder\n");
return rc;
}
rc = cxl_decoder_autoremove(&port->dev, cxld);
if (rc)
return rc;
dev_dbg(&cxld->dev, "Added to port %s\n", dev_name(&port->dev));
return 0;
}
/*
* Per the CXL specification (8.2.5.12 CXL HDM Decoder Capability Structure)
* single ported host-bridges need not publish a decoder capability when a
* passthrough decode can be assumed, i.e. all transactions that the uport sees
* are claimed and passed to the single dport. Disable the range until the first
* CXL region is enumerated / activated.
*/
int devm_cxl_add_passthrough_decoder(struct cxl_port *port)
{
struct cxl_decoder *cxld;
struct cxl_dport *dport;
int single_port_map[1];
cxld = cxl_switch_decoder_alloc(port, 1);
if (IS_ERR(cxld))
return PTR_ERR(cxld);
device_lock_assert(&port->dev);
dport = list_first_entry(&port->dports, typeof(*dport), list);
single_port_map[0] = dport->port_id;
return add_hdm_decoder(port, cxld, single_port_map);
}
EXPORT_SYMBOL_NS_GPL(devm_cxl_add_passthrough_decoder, CXL);
static void parse_hdm_decoder_caps(struct cxl_hdm *cxlhdm)
{
u32 hdm_cap;
hdm_cap = readl(cxlhdm->regs.hdm_decoder + CXL_HDM_DECODER_CAP_OFFSET);
cxlhdm->decoder_count = cxl_hdm_decoder_count(hdm_cap);
cxlhdm->target_count =
FIELD_GET(CXL_HDM_DECODER_TARGET_COUNT_MASK, hdm_cap);
if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_11_8, hdm_cap))
cxlhdm->interleave_mask |= GENMASK(11, 8);
if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_14_12, hdm_cap))
cxlhdm->interleave_mask |= GENMASK(14, 12);
}
static void __iomem *map_hdm_decoder_regs(struct cxl_port *port,
void __iomem *crb)
{
struct cxl_component_reg_map map;
cxl_probe_component_regs(&port->dev, crb, &map);
if (!map.hdm_decoder.valid) {
dev_err(&port->dev, "HDM decoder registers invalid\n");
return IOMEM_ERR_PTR(-ENXIO);
}
return crb + map.hdm_decoder.offset;
}
/**
* devm_cxl_setup_hdm - map HDM decoder component registers
* @port: cxl_port to map
*/
struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port)
{
struct device *dev = &port->dev;
void __iomem *crb, *hdm;
struct cxl_hdm *cxlhdm;
cxlhdm = devm_kzalloc(dev, sizeof(*cxlhdm), GFP_KERNEL);
if (!cxlhdm)
return ERR_PTR(-ENOMEM);
cxlhdm->port = port;
crb = devm_cxl_iomap_block(dev, port->component_reg_phys,
CXL_COMPONENT_REG_BLOCK_SIZE);
if (!crb) {
dev_err(dev, "No component registers mapped\n");
return ERR_PTR(-ENXIO);
}
hdm = map_hdm_decoder_regs(port, crb);
if (IS_ERR(hdm))
return ERR_CAST(hdm);
cxlhdm->regs.hdm_decoder = hdm;
parse_hdm_decoder_caps(cxlhdm);
if (cxlhdm->decoder_count == 0) {
dev_err(dev, "Spec violation. Caps invalid\n");
return ERR_PTR(-ENXIO);
}
return cxlhdm;
}
EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_hdm, CXL);
static int to_interleave_granularity(u32 ctrl)
{
int val = FIELD_GET(CXL_HDM_DECODER0_CTRL_IG_MASK, ctrl);
return 256 << val;
}
static int to_interleave_ways(u32 ctrl)
{
int val = FIELD_GET(CXL_HDM_DECODER0_CTRL_IW_MASK, ctrl);
switch (val) {
case 0 ... 4:
return 1 << val;
case 8 ... 10:
return 3 << (val - 8);
default:
return 0;
}
}
static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
int *target_map, void __iomem *hdm, int which)
{
u64 size, base;
u32 ctrl;
int i;
union {
u64 value;
unsigned char target_id[8];
} target_list;
ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
base = ioread64_hi_lo(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(which));
size = ioread64_hi_lo(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(which));
if (!(ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED))
size = 0;
if (base == U64_MAX || size == U64_MAX) {
dev_warn(&port->dev, "decoder%d.%d: Invalid resource range\n",
port->id, cxld->id);
return -ENXIO;
}
cxld->decoder_range = (struct range) {
.start = base,
.end = base + size - 1,
};
/* switch decoders are always enabled if committed */
if (ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED) {
cxld->flags |= CXL_DECODER_F_ENABLE;
if (ctrl & CXL_HDM_DECODER0_CTRL_LOCK)
cxld->flags |= CXL_DECODER_F_LOCK;
}
cxld->interleave_ways = to_interleave_ways(ctrl);
if (!cxld->interleave_ways) {
dev_warn(&port->dev,
"decoder%d.%d: Invalid interleave ways (ctrl: %#x)\n",
port->id, cxld->id, ctrl);
return -ENXIO;
}
cxld->interleave_granularity = to_interleave_granularity(ctrl);
if (FIELD_GET(CXL_HDM_DECODER0_CTRL_TYPE, ctrl))
cxld->target_type = CXL_DECODER_EXPANDER;
else
cxld->target_type = CXL_DECODER_ACCELERATOR;
if (is_cxl_endpoint(to_cxl_port(cxld->dev.parent)))
return 0;
target_list.value =
ioread64_hi_lo(hdm + CXL_HDM_DECODER0_TL_LOW(which));
for (i = 0; i < cxld->interleave_ways; i++)
target_map[i] = target_list.target_id[i];
return 0;
}
/**
* devm_cxl_enumerate_decoders - add decoder objects per HDM register set
* @cxlhdm: Structure to populate with HDM capabilities
*/
int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm)
{
void __iomem *hdm = cxlhdm->regs.hdm_decoder;
struct cxl_port *port = cxlhdm->port;
int i, committed, failed;
u32 ctrl;
/*
* Since the register resource was recently claimed via request_region()
* be careful about trusting the "not-committed" status until the commit
* timeout has elapsed. The commit timeout is 10ms (CXL 2.0
* 8.2.5.12.20), but double it to be tolerant of any clock skew between
* host and target.
*/
for (i = 0, committed = 0; i < cxlhdm->decoder_count; i++) {
ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i));
if (ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED)
committed++;
}
/* ensure that future checks of committed can be trusted */
if (committed != cxlhdm->decoder_count)
msleep(20);
for (i = 0, failed = 0; i < cxlhdm->decoder_count; i++) {
int target_map[CXL_DECODER_MAX_INTERLEAVE] = { 0 };
int rc, target_count = cxlhdm->target_count;
struct cxl_decoder *cxld;
if (is_cxl_endpoint(port))
cxld = cxl_endpoint_decoder_alloc(port);
else
cxld = cxl_switch_decoder_alloc(port, target_count);
if (IS_ERR(cxld)) {
dev_warn(&port->dev,
"Failed to allocate the decoder\n");
return PTR_ERR(cxld);
}
rc = init_hdm_decoder(port, cxld, target_map,
cxlhdm->regs.hdm_decoder, i);
if (rc) {
put_device(&cxld->dev);
failed++;
continue;
}
rc = add_hdm_decoder(port, cxld, target_map);
if (rc) {
dev_warn(&port->dev,
"Failed to add decoder to port\n");
return rc;
}
}
if (failed == cxlhdm->decoder_count) {
dev_err(&port->dev, "No valid decoders found\n");
return -ENXIO;
}
return 0;
}
EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_decoders, CXL);

44
drivers/cxl/core/memdev.c
View File

@@ -89,10 +89,29 @@ static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr,
static struct device_attribute dev_attr_pmem_size =
__ATTR(size, 0444, pmem_size_show, NULL);
static ssize_t serial_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
struct cxl_dev_state *cxlds = cxlmd->cxlds;
return sysfs_emit(buf, "%#llx\n", cxlds->serial);
}
static DEVICE_ATTR_RO(serial);
static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
return sprintf(buf, "%d\n", dev_to_node(dev));
}
static DEVICE_ATTR_RO(numa_node);
static struct attribute *cxl_memdev_attributes[] = {
&dev_attr_serial.attr,
&dev_attr_firmware_version.attr,
&dev_attr_payload_max.attr,
&dev_attr_label_storage_size.attr,
&dev_attr_numa_node.attr,
NULL,
};
@@ -106,8 +125,17 @@ static struct attribute *cxl_memdev_ram_attributes[] = {
NULL,
};
static umode_t cxl_memdev_visible(struct kobject *kobj, struct attribute *a,
int n)
{
if (!IS_ENABLED(CONFIG_NUMA) && a == &dev_attr_numa_node.attr)
return 0;
return a->mode;
}
static struct attribute_group cxl_memdev_attribute_group = {
.attrs = cxl_memdev_attributes,
.is_visible = cxl_memdev_visible,
};
static struct attribute_group cxl_memdev_ram_attribute_group = {
@@ -134,6 +162,12 @@ static const struct device_type cxl_memdev_type = {
.groups = cxl_memdev_attribute_groups,
};
bool is_cxl_memdev(struct device *dev)
{
return dev->type == &cxl_memdev_type;
}
EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, CXL);
/**
* set_exclusive_cxl_commands() - atomically disable user cxl commands
* @cxlds: The device state to operate on
@@ -185,6 +219,15 @@ static void cxl_memdev_unregister(void *_cxlmd)
put_device(dev);
}
static void detach_memdev(struct work_struct *work)
{
struct cxl_memdev *cxlmd;
cxlmd = container_of(work, typeof(*cxlmd), detach_work);
device_release_driver(&cxlmd->dev);
put_device(&cxlmd->dev);
}
static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds,
const struct file_operations *fops)
{
@@ -209,6 +252,7 @@ static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds,
dev->devt = MKDEV(cxl_mem_major, cxlmd->id);
dev->type = &cxl_memdev_type;
device_set_pm_not_required(dev);
INIT_WORK(&cxlmd->detach_work, detach_memdev);
cdev = &cxlmd->cdev;
cdev_init(cdev, fops);

96
drivers/cxl/core/pci.c Normal file
View File

@@ -0,0 +1,96 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2021 Intel Corporation. All rights reserved. */
#include <linux/device.h>
#include <linux/pci.h>
#include <cxlpci.h>
#include <cxl.h>
#include "core.h"
/**
* DOC: cxl core pci
*
* Compute Express Link protocols are layered on top of PCIe. CXL core provides
* a set of helpers for CXL interactions which occur via PCIe.
*/
struct cxl_walk_context {
struct pci_bus *bus;
struct cxl_port *port;
int type;
int error;
int count;
};
static int match_add_dports(struct pci_dev *pdev, void *data)
{
struct cxl_walk_context *ctx = data;
struct cxl_port *port = ctx->port;
int type = pci_pcie_type(pdev);
struct cxl_register_map map;
struct cxl_dport *dport;
u32 lnkcap, port_num;
int rc;
if (pdev->bus != ctx->bus)
return 0;
if (!pci_is_pcie(pdev))
return 0;
if (type != ctx->type)
return 0;
if (pci_read_config_dword(pdev, pci_pcie_cap(pdev) + PCI_EXP_LNKCAP,
&lnkcap))
return 0;
rc = cxl_find_regblock(pdev, CXL_REGLOC_RBI_COMPONENT, &map);
if (rc)
dev_dbg(&port->dev, "failed to find component registers\n");
port_num = FIELD_GET(PCI_EXP_LNKCAP_PN, lnkcap);
dport = devm_cxl_add_dport(port, &pdev->dev, port_num,
cxl_regmap_to_base(pdev, &map));
if (IS_ERR(dport)) {
ctx->error = PTR_ERR(dport);
return PTR_ERR(dport);
}
ctx->count++;
dev_dbg(&port->dev, "add dport%d: %s\n", port_num, dev_name(&pdev->dev));
return 0;
}
/**
* devm_cxl_port_enumerate_dports - enumerate downstream ports of the upstream port
* @port: cxl_port whose ->uport is the upstream of dports to be enumerated
*
* Returns a positive number of dports enumerated or a negative error
* code.
*/
int devm_cxl_port_enumerate_dports(struct cxl_port *port)
{
struct pci_bus *bus = cxl_port_to_pci_bus(port);
struct cxl_walk_context ctx;
int type;
if (!bus)
return -ENXIO;
if (pci_is_root_bus(bus))
type = PCI_EXP_TYPE_ROOT_PORT;
else
type = PCI_EXP_TYPE_DOWNSTREAM;
ctx = (struct cxl_walk_context) {
.port = port,
.bus = bus,
.type = type,
};
pci_walk_bus(bus, match_add_dports, &ctx);
if (ctx.count == 0)
return -ENODEV;
if (ctx.error)
return ctx.error;
return ctx.count;
}
EXPORT_SYMBOL_NS_GPL(devm_cxl_port_enumerate_dports, CXL);

18
drivers/cxl/core/pmem.c
View File

@@ -57,24 +57,30 @@ bool is_cxl_nvdimm_bridge(struct device *dev)
}
EXPORT_SYMBOL_NS_GPL(is_cxl_nvdimm_bridge, CXL);
__mock int match_nvdimm_bridge(struct device *dev, const void *data)
static int match_nvdimm_bridge(struct device *dev, void *data)
{
return is_cxl_nvdimm_bridge(dev);
}
struct cxl_nvdimm_bridge *cxl_find_nvdimm_bridge(struct cxl_nvdimm *cxl_nvd)
{
struct cxl_port *port = find_cxl_root(&cxl_nvd->dev);
struct device *dev;
dev = bus_find_device(&cxl_bus_type, NULL, cxl_nvd, match_nvdimm_bridge);
if (!port)
return NULL;
dev = device_find_child(&port->dev, NULL, match_nvdimm_bridge);
put_device(&port->dev);
if (!dev)
return NULL;
return to_cxl_nvdimm_bridge(dev);
}
EXPORT_SYMBOL_NS_GPL(cxl_find_nvdimm_bridge, CXL);
static struct cxl_nvdimm_bridge *
cxl_nvdimm_bridge_alloc(struct cxl_port *port)
static struct cxl_nvdimm_bridge *cxl_nvdimm_bridge_alloc(struct cxl_port *port)
{
struct cxl_nvdimm_bridge *cxl_nvb;
struct device *dev;
@@ -115,10 +121,10 @@ static void unregister_nvb(void *_cxl_nvb)
* work to flush. Once the state has been changed to 'dead' then no new
* work can be queued by user-triggered bind.
*/
device_lock(&cxl_nvb->dev);
cxl_device_lock(&cxl_nvb->dev);
flush = cxl_nvb->state != CXL_NVB_NEW;
cxl_nvb->state = CXL_NVB_DEAD;
device_unlock(&cxl_nvb->dev);
cxl_device_unlock(&cxl_nvb->dev);
/*
* Even though the device core will trigger device_release_driver()

1568
drivers/cxl/core/port.c Normal file
View File

File diff suppressed because it is too large Load Diff

67
drivers/cxl/core/regs.c
View File

@@ -5,6 +5,7 @@
#include <linux/slab.h>
#include <linux/pci.h>
#include <cxlmem.h>
#include <cxlpci.h>
/**
* DOC: cxl registers
@@ -35,7 +36,7 @@ void cxl_probe_component_regs(struct device *dev, void __iomem *base,
struct cxl_component_reg_map *map)
{
int cap, cap_count;
u64 cap_array;
u32 cap_array;
*map = (struct cxl_component_reg_map) { 0 };
@@ -45,11 +46,11 @@ void cxl_probe_component_regs(struct device *dev, void __iomem *base,
*/
base += CXL_CM_OFFSET;
cap_array = readq(base + CXL_CM_CAP_HDR_OFFSET);
cap_array = readl(base + CXL_CM_CAP_HDR_OFFSET);
if (FIELD_GET(CXL_CM_CAP_HDR_ID_MASK, cap_array) != CM_CAP_HDR_CAP_ID) {
dev_err(dev,
"Couldn't locate the CXL.cache and CXL.mem capability array header./n");
"Couldn't locate the CXL.cache and CXL.mem capability array header.\n");
return;
}
@@ -158,9 +159,8 @@ void cxl_probe_device_regs(struct device *dev, void __iomem *base,
}
EXPORT_SYMBOL_NS_GPL(cxl_probe_device_regs, CXL);
static void __iomem *devm_cxl_iomap_block(struct device *dev,
resource_size_t addr,
resource_size_t length)
void __iomem *devm_cxl_iomap_block(struct device *dev, resource_size_t addr,
resource_size_t length)
{
void __iomem *ret_val;
struct resource *res;
@@ -247,3 +247,58 @@ int cxl_map_device_regs(struct pci_dev *pdev,
return 0;
}
EXPORT_SYMBOL_NS_GPL(cxl_map_device_regs, CXL);
static void cxl_decode_regblock(u32 reg_lo, u32 reg_hi,
struct cxl_register_map *map)
{
map->block_offset = ((u64)reg_hi << 32) |
(reg_lo & CXL_DVSEC_REG_LOCATOR_BLOCK_OFF_LOW_MASK);
map->barno = FIELD_GET(CXL_DVSEC_REG_LOCATOR_BIR_MASK, reg_lo);
map->reg_type = FIELD_GET(CXL_DVSEC_REG_LOCATOR_BLOCK_ID_MASK, reg_lo);
}
/**
* cxl_find_regblock() - Locate register blocks by type
* @pdev: The CXL PCI device to enumerate.
* @type: Register Block Indicator id
* @map: Enumeration output, clobbered on error
*
* Return: 0 if register block enumerated, negative error code otherwise
*
* A CXL DVSEC may point to one or more register blocks, search for them
* by @type.
*/
int cxl_find_regblock(struct pci_dev *pdev, enum cxl_regloc_type type,
struct cxl_register_map *map)
{
u32 regloc_size, regblocks;
int regloc, i;
map->block_offset = U64_MAX;
regloc = pci_find_dvsec_capability(pdev, PCI_DVSEC_VENDOR_ID_CXL,
CXL_DVSEC_REG_LOCATOR);
if (!regloc)
return -ENXIO;
pci_read_config_dword(pdev, regloc + PCI_DVSEC_HEADER1, &regloc_size);
regloc_size = FIELD_GET(PCI_DVSEC_HEADER1_LENGTH_MASK, regloc_size);
regloc += CXL_DVSEC_REG_LOCATOR_BLOCK1_OFFSET;
regblocks = (regloc_size - CXL_DVSEC_REG_LOCATOR_BLOCK1_OFFSET) / 8;
for (i = 0; i < regblocks; i++, regloc += 8) {
u32 reg_lo, reg_hi;
pci_read_config_dword(pdev, regloc, &reg_lo);
pci_read_config_dword(pdev, regloc + 4, &reg_hi);
cxl_decode_regblock(reg_lo, reg_hi, map);
if (map->reg_type == type)
return 0;
}
map->block_offset = U64_MAX;
return -ENODEV;
}
EXPORT_SYMBOL_NS_GPL(cxl_find_regblock, CXL);

195
drivers/cxl/cxl.h
View File

@@ -17,6 +17,9 @@
* (port-driver, region-driver, nvdimm object-drivers... etc).
*/
/* CXL 2.0 8.2.4 CXL Component Register Layout and Definition */
#define CXL_COMPONENT_REG_BLOCK_SIZE SZ_64K
/* CXL 2.0 8.2.5 CXL.cache and CXL.mem Registers*/
#define CXL_CM_OFFSET 0x1000
#define CXL_CM_CAP_HDR_OFFSET 0x0
@@ -36,11 +39,23 @@
#define CXL_HDM_DECODER_CAP_OFFSET 0x0
#define CXL_HDM_DECODER_COUNT_MASK GENMASK(3, 0)
#define CXL_HDM_DECODER_TARGET_COUNT_MASK GENMASK(7, 4)
#define CXL_HDM_DECODER0_BASE_LOW_OFFSET 0x10
#define CXL_HDM_DECODER0_BASE_HIGH_OFFSET 0x14
#define CXL_HDM_DECODER0_SIZE_LOW_OFFSET 0x18
#define CXL_HDM_DECODER0_SIZE_HIGH_OFFSET 0x1c
#define CXL_HDM_DECODER0_CTRL_OFFSET 0x20
#define CXL_HDM_DECODER_INTERLEAVE_11_8 BIT(8)
#define CXL_HDM_DECODER_INTERLEAVE_14_12 BIT(9)
#define CXL_HDM_DECODER_CTRL_OFFSET 0x4
#define CXL_HDM_DECODER_ENABLE BIT(1)
#define CXL_HDM_DECODER0_BASE_LOW_OFFSET(i) (0x20 * (i) + 0x10)
#define CXL_HDM_DECODER0_BASE_HIGH_OFFSET(i) (0x20 * (i) + 0x14)
#define CXL_HDM_DECODER0_SIZE_LOW_OFFSET(i) (0x20 * (i) + 0x18)
#define CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(i) (0x20 * (i) + 0x1c)
#define CXL_HDM_DECODER0_CTRL_OFFSET(i) (0x20 * (i) + 0x20)
#define CXL_HDM_DECODER0_CTRL_IG_MASK GENMASK(3, 0)
#define CXL_HDM_DECODER0_CTRL_IW_MASK GENMASK(7, 4)
#define CXL_HDM_DECODER0_CTRL_LOCK BIT(8)
#define CXL_HDM_DECODER0_CTRL_COMMIT BIT(9)
#define CXL_HDM_DECODER0_CTRL_COMMITTED BIT(10)
#define CXL_HDM_DECODER0_CTRL_TYPE BIT(12)
#define CXL_HDM_DECODER0_TL_LOW(i) (0x20 * (i) + 0x24)
#define CXL_HDM_DECODER0_TL_HIGH(i) (0x20 * (i) + 0x28)
static inline int cxl_hdm_decoder_count(u32 cap_hdr)
{
@@ -145,6 +160,12 @@ int cxl_map_device_regs(struct pci_dev *pdev,
struct cxl_device_regs *regs,
struct cxl_register_map *map);
enum cxl_regloc_type;
int cxl_find_regblock(struct pci_dev *pdev, enum cxl_regloc_type type,
struct cxl_register_map *map);
void __iomem *devm_cxl_iomap_block(struct device *dev, resource_size_t addr,
resource_size_t length);
#define CXL_RESOURCE_NONE ((resource_size_t) -1)
#define CXL_TARGET_STRLEN 20
@@ -158,7 +179,8 @@ int cxl_map_device_regs(struct pci_dev *pdev,
#define CXL_DECODER_F_TYPE2 BIT(2)
#define CXL_DECODER_F_TYPE3 BIT(3)
#define CXL_DECODER_F_LOCK BIT(4)
#define CXL_DECODER_F_MASK GENMASK(4, 0)
#define CXL_DECODER_F_ENABLE BIT(5)
#define CXL_DECODER_F_MASK GENMASK(5, 0)
enum cxl_decoder_type {
CXL_DECODER_ACCELERATOR = 2,
@@ -175,22 +197,28 @@ enum cxl_decoder_type {
* struct cxl_decoder - CXL address range decode configuration
* @dev: this decoder's device
* @id: kernel device name id
* @range: address range considered by this decoder
* @platform_res: address space resources considered by root decoder
* @decoder_range: address space resources considered by midlevel decoder
* @interleave_ways: number of cxl_dports in this decode
* @interleave_granularity: data stride per dport
* @target_type: accelerator vs expander (type2 vs type3) selector
* @flags: memory type capabilities and locking
* @target_lock: coordinate coherent reads of the target list
* @nr_targets: number of elements in @target
* @target: active ordered target list in current decoder configuration
*/
struct cxl_decoder {
struct device dev;
int id;
struct range range;
union {
struct resource platform_res;
struct range decoder_range;
};
int interleave_ways;
int interleave_granularity;
enum cxl_decoder_type target_type;
unsigned long flags;
seqlock_t target_lock;
int nr_targets;
struct cxl_dport *target[];
};
@@ -226,14 +254,6 @@ struct cxl_nvdimm {
struct nvdimm *nvdimm;
};
struct cxl_walk_context {
struct device *dev;
struct pci_bus *root;
struct cxl_port *port;
int error;
int count;
};
/**
* struct cxl_port - logical collection of upstream port devices and
* downstream port devices to construct a CXL memory
@@ -242,16 +262,22 @@ struct cxl_walk_context {
* @uport: PCI or platform device implementing the upstream port capability
* @id: id for port device-name
* @dports: cxl_dport instances referenced by decoders
* @endpoints: cxl_ep instances, endpoints that are a descendant of this port
* @decoder_ida: allocator for decoder ids
* @component_reg_phys: component register capability base address (optional)
* @dead: last ep has been removed, force port re-creation
* @depth: How deep this port is relative to the root. depth 0 is the root.
*/
struct cxl_port {
struct device dev;
struct device *uport;
int id;
struct list_head dports;
struct list_head endpoints;
struct ida decoder_ida;
resource_size_t component_reg_phys;
bool dead;
unsigned int depth;
};
/**
@@ -270,19 +296,65 @@ struct cxl_dport {
struct list_head list;
};
/**
* struct cxl_ep - track an endpoint's interest in a port
* @ep: device that hosts a generic CXL endpoint (expander or accelerator)
* @list: node on port->endpoints list
*/
struct cxl_ep {
struct device *ep;
struct list_head list;
};
/*
* The platform firmware device hosting the root is also the top of the
* CXL port topology. All other CXL ports have another CXL port as their
* parent and their ->uport / host device is out-of-line of the port
* ancestry.
*/
static inline bool is_cxl_root(struct cxl_port *port)
{
return port->uport == port->dev.parent;
}
bool is_cxl_port(struct device *dev);
struct cxl_port *to_cxl_port(struct device *dev);
struct pci_bus;
int devm_cxl_register_pci_bus(struct device *host, struct device *uport,
struct pci_bus *bus);
struct pci_bus *cxl_port_to_pci_bus(struct cxl_port *port);
struct cxl_port *devm_cxl_add_port(struct device *host, struct device *uport,
resource_size_t component_reg_phys,
struct cxl_port *parent_port);
struct cxl_port *find_cxl_root(struct device *dev);
int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd);
int cxl_bus_rescan(void);
struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd);
bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd);
int cxl_add_dport(struct cxl_port *port, struct device *dport, int port_id,
resource_size_t component_reg_phys);
struct cxl_dport *devm_cxl_add_dport(struct cxl_port *port,
struct device *dport, int port_id,
resource_size_t component_reg_phys);
struct cxl_dport *cxl_find_dport_by_dev(struct cxl_port *port,
const struct device *dev);
struct cxl_decoder *to_cxl_decoder(struct device *dev);
bool is_root_decoder(struct device *dev);
struct cxl_decoder *cxl_decoder_alloc(struct cxl_port *port, int nr_targets);
bool is_cxl_decoder(struct device *dev);
struct cxl_decoder *cxl_root_decoder_alloc(struct cxl_port *port,
unsigned int nr_targets);
struct cxl_decoder *cxl_switch_decoder_alloc(struct cxl_port *port,
unsigned int nr_targets);
int cxl_decoder_add(struct cxl_decoder *cxld, int *target_map);
struct cxl_decoder *cxl_endpoint_decoder_alloc(struct cxl_port *port);
int cxl_decoder_add_locked(struct cxl_decoder *cxld, int *target_map);
int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld);
int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint);
struct cxl_hdm;
struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port);
int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm);
int devm_cxl_add_passthrough_decoder(struct cxl_port *port);
extern struct bus_type cxl_bus_type;
@@ -304,8 +376,14 @@ int __cxl_driver_register(struct cxl_driver *cxl_drv, struct module *owner,
#define cxl_driver_register(x) __cxl_driver_register(x, THIS_MODULE, KBUILD_MODNAME)
void cxl_driver_unregister(struct cxl_driver *cxl_drv);
#define module_cxl_driver(__cxl_driver) \
module_driver(__cxl_driver, cxl_driver_register, cxl_driver_unregister)
#define CXL_DEVICE_NVDIMM_BRIDGE 1
#define CXL_DEVICE_NVDIMM 2
#define CXL_DEVICE_PORT 3
#define CXL_DEVICE_ROOT 4
#define CXL_DEVICE_MEMORY_EXPANDER 5
#define MODULE_ALIAS_CXL(type) MODULE_ALIAS("cxl:t" __stringify(type) "*")
#define CXL_MODALIAS_FMT "cxl:t%d"
@@ -326,4 +404,83 @@ struct cxl_nvdimm_bridge *cxl_find_nvdimm_bridge(struct cxl_nvdimm *cxl_nvd);
#ifndef __mock
#define __mock static
#endif
#ifdef CONFIG_PROVE_CXL_LOCKING
enum cxl_lock_class {
CXL_ANON_LOCK,
CXL_NVDIMM_LOCK,
CXL_NVDIMM_BRIDGE_LOCK,
CXL_PORT_LOCK,
/*
* Be careful to add new lock classes here, CXL_PORT_LOCK is
* extended by the port depth, so a maximum CXL port topology
* depth would need to be defined first.
*/
};
static inline void cxl_nested_lock(struct device *dev)
{
if (is_cxl_port(dev)) {
struct cxl_port *port = to_cxl_port(dev);
mutex_lock_nested(&dev->lockdep_mutex,
CXL_PORT_LOCK + port->depth);
} else if (is_cxl_decoder(dev)) {
struct cxl_port *port = to_cxl_port(dev->parent);
/*
* A decoder is the immediate child of a port, so set
* its lock class equal to other child device siblings.
*/
mutex_lock_nested(&dev->lockdep_mutex,
CXL_PORT_LOCK + port->depth + 1);
} else if (is_cxl_nvdimm_bridge(dev))
mutex_lock_nested(&dev->lockdep_mutex, CXL_NVDIMM_BRIDGE_LOCK);
else if (is_cxl_nvdimm(dev))
mutex_lock_nested(&dev->lockdep_mutex, CXL_NVDIMM_LOCK);
else
mutex_lock_nested(&dev->lockdep_mutex, CXL_ANON_LOCK);
}
static inline void cxl_nested_unlock(struct device *dev)
{
mutex_unlock(&dev->lockdep_mutex);
}
static inline void cxl_device_lock(struct device *dev)
{
/*
* For double lock errors the lockup will happen before lockdep
* warns at cxl_nested_lock(), so assert explicitly.
*/
lockdep_assert_not_held(&dev->lockdep_mutex);
device_lock(dev);
cxl_nested_lock(dev);
}
static inline void cxl_device_unlock(struct device *dev)
{
cxl_nested_unlock(dev);
device_unlock(dev);
}
#else
static inline void cxl_nested_lock(struct device *dev)
{
}
static inline void cxl_nested_unlock(struct device *dev)
{
}
static inline void cxl_device_lock(struct device *dev)
{
device_lock(dev);
}
static inline void cxl_device_unlock(struct device *dev)
{
device_unlock(dev);
}
#endif
#endif /* __CXL_H__ */

39
drivers/cxl/cxlmem.h
View File

@@ -34,12 +34,14 @@
* @dev: driver core device object
* @cdev: char dev core object for ioctl operations
* @cxlds: The device state backing this device
* @detach_work: active memdev lost a port in its ancestry
* @id: id number of this memdev instance.
*/
struct cxl_memdev {
struct device dev;
struct cdev cdev;
struct cxl_dev_state *cxlds;
struct work_struct detach_work;
int id;
};
@@ -48,6 +50,12 @@ static inline struct cxl_memdev *to_cxl_memdev(struct device *dev)
return container_of(dev, struct cxl_memdev, dev);
}
bool is_cxl_memdev(struct device *dev);
static inline bool is_cxl_endpoint(struct cxl_port *port)
{
return is_cxl_memdev(port->uport);
}
struct cxl_memdev *devm_cxl_add_memdev(struct cxl_dev_state *cxlds);
/**
@@ -89,6 +97,18 @@ struct cxl_mbox_cmd {
*/
#define CXL_CAPACITY_MULTIPLIER SZ_256M
/**
* struct cxl_endpoint_dvsec_info - Cached DVSEC info
* @mem_enabled: cached value of mem_enabled in the DVSEC, PCIE_DEVICE
* @ranges: Number of active HDM ranges this device uses.
* @dvsec_range: cached attributes of the ranges in the DVSEC, PCIE_DEVICE
*/
struct cxl_endpoint_dvsec_info {
bool mem_enabled;
int ranges;
struct range dvsec_range[2];
};
/**
* struct cxl_dev_state - The driver device state
*
@@ -98,6 +118,7 @@ struct cxl_mbox_cmd {
*
* @dev: The device associated with this CXL state
* @regs: Parsed register blocks
* @cxl_dvsec: Offset to the PCIe device DVSEC
* @payload_size: Size of space for payload
* (CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register)
* @lsa_size: Size of Label Storage Area
@@ -116,7 +137,11 @@ struct cxl_mbox_cmd {
* @active_persistent_bytes: sum of hard + soft persistent
* @next_volatile_bytes: volatile capacity change pending device reset
* @next_persistent_bytes: persistent capacity change pending device reset
* @component_reg_phys: register base of component registers
* @info: Cached DVSEC information about the device.
* @serial: PCIe Device Serial Number
* @mbox_send: @dev specific transport for transmitting mailbox commands
* @wait_media_ready: @dev specific method to await media ready
*
* See section 8.2.9.5.2 Capacity Configuration and Label Storage for
* details on capacity parameters.
@@ -125,6 +150,7 @@ struct cxl_dev_state {
struct device *dev;
struct cxl_regs regs;
int cxl_dvsec;
size_t payload_size;
size_t lsa_size;
@@ -145,7 +171,12 @@ struct cxl_dev_state {
u64 next_volatile_bytes;
u64 next_persistent_bytes;
resource_size_t component_reg_phys;
struct cxl_endpoint_dvsec_info info;
u64 serial;
int (*mbox_send)(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd);
int (*wait_media_ready)(struct cxl_dev_state *cxlds);
};
enum cxl_opcode {
@@ -264,4 +295,12 @@ int cxl_mem_create_range_info(struct cxl_dev_state *cxlds);
struct cxl_dev_state *cxl_dev_state_create(struct device *dev);
void set_exclusive_cxl_commands(struct cxl_dev_state *cxlds, unsigned long *cmds);
void clear_exclusive_cxl_commands(struct cxl_dev_state *cxlds, unsigned long *cmds);
struct cxl_hdm {
struct cxl_component_regs regs;
unsigned int decoder_count;
unsigned int target_count;
unsigned int interleave_mask;
struct cxl_port *port;
};
#endif /* __CXL_MEM_H__ */

75
drivers/cxl/cxlpci.h Normal file
View File

@@ -0,0 +1,75 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/* Copyright(c) 2020 Intel Corporation. All rights reserved. */
#ifndef __CXL_PCI_H__
#define __CXL_PCI_H__
#include <linux/pci.h>
#include "cxl.h"
#define CXL_MEMORY_PROGIF 0x10
/*
* See section 8.1 Configuration Space Registers in the CXL 2.0
* Specification. Names are taken straight from the specification with "CXL" and
* "DVSEC" redundancies removed. When obvious, abbreviations may be used.
*/
#define PCI_DVSEC_HEADER1_LENGTH_MASK GENMASK(31, 20)
#define PCI_DVSEC_VENDOR_ID_CXL 0x1E98
/* CXL 2.0 8.1.3: PCIe DVSEC for CXL Device */
#define CXL_DVSEC_PCIE_DEVICE 0
#define CXL_DVSEC_CAP_OFFSET 0xA
#define CXL_DVSEC_MEM_CAPABLE BIT(2)
#define CXL_DVSEC_HDM_COUNT_MASK GENMASK(5, 4)
#define CXL_DVSEC_CTRL_OFFSET 0xC
#define CXL_DVSEC_MEM_ENABLE BIT(2)
#define CXL_DVSEC_RANGE_SIZE_HIGH(i) (0x18 + (i * 0x10))
#define CXL_DVSEC_RANGE_SIZE_LOW(i) (0x1C + (i * 0x10))
#define CXL_DVSEC_MEM_INFO_VALID BIT(0)
#define CXL_DVSEC_MEM_ACTIVE BIT(1)
#define CXL_DVSEC_MEM_SIZE_LOW_MASK GENMASK(31, 28)
#define CXL_DVSEC_RANGE_BASE_HIGH(i) (0x20 + (i * 0x10))
#define CXL_DVSEC_RANGE_BASE_LOW(i) (0x24 + (i * 0x10))
#define CXL_DVSEC_MEM_BASE_LOW_MASK GENMASK(31, 28)
/* CXL 2.0 8.1.4: Non-CXL Function Map DVSEC */
#define CXL_DVSEC_FUNCTION_MAP 2
/* CXL 2.0 8.1.5: CXL 2.0 Extensions DVSEC for Ports */
#define CXL_DVSEC_PORT_EXTENSIONS 3
/* CXL 2.0 8.1.6: GPF DVSEC for CXL Port */
#define CXL_DVSEC_PORT_GPF 4
/* CXL 2.0 8.1.7: GPF DVSEC for CXL Device */
#define CXL_DVSEC_DEVICE_GPF 5
/* CXL 2.0 8.1.8: PCIe DVSEC for Flex Bus Port */
#define CXL_DVSEC_PCIE_FLEXBUS_PORT 7
/* CXL 2.0 8.1.9: Register Locator DVSEC */
#define CXL_DVSEC_REG_LOCATOR 8
#define CXL_DVSEC_REG_LOCATOR_BLOCK1_OFFSET 0xC
#define CXL_DVSEC_REG_LOCATOR_BIR_MASK GENMASK(2, 0)
#define CXL_DVSEC_REG_LOCATOR_BLOCK_ID_MASK GENMASK(15, 8)
#define CXL_DVSEC_REG_LOCATOR_BLOCK_OFF_LOW_MASK GENMASK(31, 16)
/* Register Block Identifier (RBI) */
enum cxl_regloc_type {
CXL_REGLOC_RBI_EMPTY = 0,
CXL_REGLOC_RBI_COMPONENT,
CXL_REGLOC_RBI_VIRT,
CXL_REGLOC_RBI_MEMDEV,
CXL_REGLOC_RBI_TYPES
};
static inline resource_size_t cxl_regmap_to_base(struct pci_dev *pdev,
struct cxl_register_map *map)
{
if (map->block_offset == U64_MAX)
return CXL_RESOURCE_NONE;
return pci_resource_start(pdev, map->barno) + map->block_offset;
}
int devm_cxl_port_enumerate_dports(struct cxl_port *port);
#endif /* __CXL_PCI_H__ */

228
drivers/cxl/mem.c Normal file
View File

@@ -0,0 +1,228 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2022 Intel Corporation. All rights reserved. */
#include <linux/device.h>
#include <linux/module.h>
#include <linux/pci.h>
#include "cxlmem.h"
#include "cxlpci.h"
/**
* DOC: cxl mem
*
* CXL memory endpoint devices and switches are CXL capable devices that are
* participating in CXL.mem protocol. Their functionality builds on top of the
* CXL.io protocol that allows enumerating and configuring components via
* standard PCI mechanisms.
*
* The cxl_mem driver owns kicking off the enumeration of this CXL.mem
* capability. With the detection of a CXL capable endpoint, the driver will
* walk up to find the platform specific port it is connected to, and determine
* if there are intervening switches in the path. If there are switches, a
* secondary action is to enumerate those (implemented in cxl_core). Finally the
* cxl_mem driver adds the device it is bound to as a CXL endpoint-port for use
* in higher level operations.
*/
static int wait_for_media(struct cxl_memdev *cxlmd)
{
struct cxl_dev_state *cxlds = cxlmd->cxlds;
struct cxl_endpoint_dvsec_info *info = &cxlds->info;
int rc;
if (!info->mem_enabled)
return -EBUSY;
rc = cxlds->wait_media_ready(cxlds);
if (rc)
return rc;
/*
* We know the device is active, and enabled, if any ranges are non-zero
* we'll need to check later before adding the port since that owns the
* HDM decoder registers.
*/
return 0;
}
static int create_endpoint(struct cxl_memdev *cxlmd,
struct cxl_port *parent_port)
{
struct cxl_dev_state *cxlds = cxlmd->cxlds;
struct cxl_port *endpoint;
endpoint = devm_cxl_add_port(&parent_port->dev, &cxlmd->dev,
cxlds->component_reg_phys, parent_port);
if (IS_ERR(endpoint))
return PTR_ERR(endpoint);
dev_dbg(&cxlmd->dev, "add: %s\n", dev_name(&endpoint->dev));
if (!endpoint->dev.driver) {
dev_err(&cxlmd->dev, "%s failed probe\n",
dev_name(&endpoint->dev));
return -ENXIO;
}
return cxl_endpoint_autoremove(cxlmd, endpoint);
}
/**
* cxl_dvsec_decode_init() - Setup HDM decoding for the endpoint
* @cxlds: Device state
*
* Additionally, enables global HDM decoding. Warning: don't call this outside
* of probe. Once probe is complete, the port driver owns all access to the HDM
* decoder registers.
*
* Returns: false if DVSEC Ranges are being used instead of HDM
* decoders, or if it can not be determined if DVSEC Ranges are in use.
* Otherwise, returns true.
*/
__mock bool cxl_dvsec_decode_init(struct cxl_dev_state *cxlds)
{
struct cxl_endpoint_dvsec_info *info = &cxlds->info;
struct cxl_register_map map;
struct cxl_component_reg_map *cmap = &map.component_map;
bool global_enable, do_hdm_init = false;
void __iomem *crb;
u32 global_ctrl;
/* map hdm decoder */
crb = ioremap(cxlds->component_reg_phys, CXL_COMPONENT_REG_BLOCK_SIZE);
if (!crb) {
dev_dbg(cxlds->dev, "Failed to map component registers\n");
return false;
}
cxl_probe_component_regs(cxlds->dev, crb, cmap);
if (!cmap->hdm_decoder.valid) {
dev_dbg(cxlds->dev, "Invalid HDM decoder registers\n");
goto out;
}
global_ctrl = readl(crb + cmap->hdm_decoder.offset +
CXL_HDM_DECODER_CTRL_OFFSET);
global_enable = global_ctrl & CXL_HDM_DECODER_ENABLE;
if (!global_enable && info->ranges) {
dev_dbg(cxlds->dev,
"DVSEC ranges already programmed and HDM decoders not enabled.\n");
goto out;
}
do_hdm_init = true;
/*
* Permanently (for this boot at least) opt the device into HDM
* operation. Individual HDM decoders still need to be enabled after
* this point.
*/
if (!global_enable) {
dev_dbg(cxlds->dev, "Enabling HDM decode\n");
writel(global_ctrl | CXL_HDM_DECODER_ENABLE,
crb + cmap->hdm_decoder.offset +
CXL_HDM_DECODER_CTRL_OFFSET);
}
out:
iounmap(crb);
return do_hdm_init;
}
static int cxl_mem_probe(struct device *dev)
{
struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
struct cxl_dev_state *cxlds = cxlmd->cxlds;
struct cxl_port *parent_port;
int rc;
/*
* Someone is trying to reattach this device after it lost its port
* connection (an endpoint port previously registered by this memdev was
* disabled). This racy check is ok because if the port is still gone,
* no harm done, and if the port hierarchy comes back it will re-trigger
* this probe. Port rescan and memdev detach work share the same
* single-threaded workqueue.
*/
if (work_pending(&cxlmd->detach_work))
return -EBUSY;
rc = wait_for_media(cxlmd);
if (rc) {
dev_err(dev, "Media not active (%d)\n", rc);
return rc;
}
/*
* If DVSEC ranges are being used instead of HDM decoder registers there
* is no use in trying to manage those.
*/
if (!cxl_dvsec_decode_init(cxlds)) {
struct cxl_endpoint_dvsec_info *info = &cxlds->info;
int i;
/* */
for (i = 0; i < 2; i++) {
u64 base, size;
/*
* Give a nice warning to the user that BIOS has really
* botched things for them if it didn't place DVSEC
* ranges in the memory map.
*/
base = info->dvsec_range[i].start;
size = range_len(&info->dvsec_range[i]);
if (size && !region_intersects(base, size,
IORESOURCE_SYSTEM_RAM,
IORES_DESC_NONE)) {
dev_err(dev,
"DVSEC range %#llx-%#llx must be reserved by BIOS, but isn't\n",
base, base + size - 1);
}
}
dev_err(dev,
"Active DVSEC range registers in use. Will not bind.\n");
return -EBUSY;
}
rc = devm_cxl_enumerate_ports(cxlmd);
if (rc)
return rc;
parent_port = cxl_mem_find_port(cxlmd);
if (!parent_port) {
dev_err(dev, "CXL port topology not found\n");
return -ENXIO;
}
cxl_device_lock(&parent_port->dev);
if (!parent_port->dev.driver) {
dev_err(dev, "CXL port topology %s not enabled\n",
dev_name(&parent_port->dev));
rc = -ENXIO;
goto out;
}
rc = create_endpoint(cxlmd, parent_port);
out:
cxl_device_unlock(&parent_port->dev);
put_device(&parent_port->dev);
return rc;
}
static struct cxl_driver cxl_mem_driver = {
.name = "cxl_mem",
.probe = cxl_mem_probe,
.id = CXL_DEVICE_MEMORY_EXPANDER,
};
module_cxl_driver(cxl_mem_driver);
MODULE_LICENSE("GPL v2");
MODULE_IMPORT_NS(CXL);
MODULE_ALIAS_CXL(CXL_DEVICE_MEMORY_EXPANDER);
/*
* create_endpoint() wants to validate port driver attach immediately after
* endpoint registration.
*/
MODULE_SOFTDEP("pre: cxl_port");

397
drivers/cxl/pci.c
View File

@@ -1,14 +1,16 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2020 Intel Corporation. All rights reserved. */
#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/moduleparam.h>
#include <linux/module.h>
#include <linux/delay.h>
#include <linux/sizes.h>
#include <linux/mutex.h>
#include <linux/list.h>
#include <linux/pci.h>
#include <linux/io.h>
#include "cxlmem.h"
#include "pci.h"
#include "cxlpci.h"
#include "cxl.h"
/**
@@ -35,6 +37,20 @@
/* CXL 2.0 - 8.2.8.4 */
#define CXL_MAILBOX_TIMEOUT_MS (2 * HZ)
/*
* CXL 2.0 ECN "Add Mailbox Ready Time" defines a capability field to
* dictate how long to wait for the mailbox to become ready. The new
* field allows the device to tell software the amount of time to wait
* before mailbox ready. This field per the spec theoretically allows
* for up to 255 seconds. 255 seconds is unreasonably long, its longer
* than the maximum SATA port link recovery wait. Default to 60 seconds
* until someone builds a CXL device that needs more time in practice.
*/
static unsigned short mbox_ready_timeout = 60;
module_param(mbox_ready_timeout, ushort, 0644);
MODULE_PARM_DESC(mbox_ready_timeout,
"seconds to wait for mailbox ready / memory active status");
static int cxl_pci_mbox_wait_for_doorbell(struct cxl_dev_state *cxlds)
{
const unsigned long start = jiffies;
@@ -57,14 +73,16 @@ static int cxl_pci_mbox_wait_for_doorbell(struct cxl_dev_state *cxlds)
return 0;
}
static void cxl_pci_mbox_timeout(struct cxl_dev_state *cxlds,
struct cxl_mbox_cmd *mbox_cmd)
{
struct device *dev = cxlds->dev;
#define cxl_err(dev, status, msg) \
dev_err_ratelimited(dev, msg ", device state %s%s\n", \
status & CXLMDEV_DEV_FATAL ? " fatal" : "", \
status & CXLMDEV_FW_HALT ? " firmware-halt" : "")
dev_dbg(dev, "Mailbox command (opcode: %#x size: %zub) timed out\n",
mbox_cmd->opcode, mbox_cmd->size_in);
}
#define cxl_cmd_err(dev, cmd, status, msg) \
dev_err_ratelimited(dev, msg " (opcode: %#x), device state %s%s\n", \
(cmd)->opcode, \
status & CXLMDEV_DEV_FATAL ? " fatal" : "", \
status & CXLMDEV_FW_HALT ? " firmware-halt" : "")
/**
* __cxl_pci_mbox_send_cmd() - Execute a mailbox command
@@ -118,7 +136,11 @@ static int __cxl_pci_mbox_send_cmd(struct cxl_dev_state *cxlds,
/* #1 */
if (cxl_doorbell_busy(cxlds)) {
dev_err_ratelimited(dev, "Mailbox re-busy after acquiring\n");
u64 md_status =
readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);
cxl_cmd_err(cxlds->dev, mbox_cmd, md_status,
"mailbox queue busy");
return -EBUSY;
}
@@ -144,7 +166,9 @@ static int __cxl_pci_mbox_send_cmd(struct cxl_dev_state *cxlds,
/* #5 */
rc = cxl_pci_mbox_wait_for_doorbell(cxlds);
if (rc == -ETIMEDOUT) {
cxl_pci_mbox_timeout(cxlds, mbox_cmd);
u64 md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);
cxl_cmd_err(cxlds->dev, mbox_cmd, md_status, "mailbox timeout");
return rc;
}
@@ -182,98 +206,13 @@ static int __cxl_pci_mbox_send_cmd(struct cxl_dev_state *cxlds,
return 0;
}
/**
* cxl_pci_mbox_get() - Acquire exclusive access to the mailbox.
* @cxlds: The device state to gain access to.
*
* Context: Any context. Takes the mbox_mutex.
* Return: 0 if exclusive access was acquired.
*/
static int cxl_pci_mbox_get(struct cxl_dev_state *cxlds)
{
struct device *dev = cxlds->dev;
u64 md_status;
int rc;
mutex_lock_io(&cxlds->mbox_mutex);
/*
* XXX: There is some amount of ambiguity in the 2.0 version of the spec
* around the mailbox interface ready (8.2.8.5.1.1). The purpose of the
* bit is to allow firmware running on the device to notify the driver
* that it's ready to receive commands. It is unclear if the bit needs
* to be read for each transaction mailbox, ie. the firmware can switch
* it on and off as needed. Second, there is no defined timeout for
* mailbox ready, like there is for the doorbell interface.
*
* Assumptions:
* 1. The firmware might toggle the Mailbox Interface Ready bit, check
* it for every command.
*
* 2. If the doorbell is clear, the firmware should have first set the
* Mailbox Interface Ready bit. Therefore, waiting for the doorbell
* to be ready is sufficient.
*/
rc = cxl_pci_mbox_wait_for_doorbell(cxlds);
if (rc) {
dev_warn(dev, "Mailbox interface not ready\n");
goto out;
}
md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);
if (!(md_status & CXLMDEV_MBOX_IF_READY && CXLMDEV_READY(md_status))) {
dev_err(dev, "mbox: reported doorbell ready, but not mbox ready\n");
rc = -EBUSY;
goto out;
}
/*
* Hardware shouldn't allow a ready status but also have failure bits
* set. Spit out an error, this should be a bug report
*/
rc = -EFAULT;
if (md_status & CXLMDEV_DEV_FATAL) {
dev_err(dev, "mbox: reported ready, but fatal\n");
goto out;
}
if (md_status & CXLMDEV_FW_HALT) {
dev_err(dev, "mbox: reported ready, but halted\n");
goto out;
}
if (CXLMDEV_RESET_NEEDED(md_status)) {
dev_err(dev, "mbox: reported ready, but reset needed\n");
goto out;
}
/* with lock held */
return 0;
out:
mutex_unlock(&cxlds->mbox_mutex);
return rc;
}
/**
* cxl_pci_mbox_put() - Release exclusive access to the mailbox.
* @cxlds: The device state to communicate with.
*
* Context: Any context. Expects mbox_mutex to be held.
*/
static void cxl_pci_mbox_put(struct cxl_dev_state *cxlds)
{
mutex_unlock(&cxlds->mbox_mutex);
}
static int cxl_pci_mbox_send(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd)
{
int rc;
rc = cxl_pci_mbox_get(cxlds);
if (rc)
return rc;
mutex_lock_io(&cxlds->mbox_mutex);
rc = __cxl_pci_mbox_send_cmd(cxlds, cmd);
cxl_pci_mbox_put(cxlds);
mutex_unlock(&cxlds->mbox_mutex);
return rc;
}
@@ -281,6 +220,34 @@ static int cxl_pci_mbox_send(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *c
static int cxl_pci_setup_mailbox(struct cxl_dev_state *cxlds)
{
const int cap = readl(cxlds->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET);
unsigned long timeout;
u64 md_status;
timeout = jiffies + mbox_ready_timeout * HZ;
do {
md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);
if (md_status & CXLMDEV_MBOX_IF_READY)
break;
if (msleep_interruptible(100))
break;
} while (!time_after(jiffies, timeout));
if (!(md_status & CXLMDEV_MBOX_IF_READY)) {
cxl_err(cxlds->dev, md_status,
"timeout awaiting mailbox ready");
return -ETIMEDOUT;
}
/*
* A command may be in flight from a previous driver instance,
* think kexec, do one doorbell wait so that
* __cxl_pci_mbox_send_cmd() can assume that it is the only
* source for future doorbell busy events.
*/
if (cxl_pci_mbox_wait_for_doorbell(cxlds) != 0) {
cxl_err(cxlds->dev, md_status, "timeout awaiting mailbox idle");
return -ETIMEDOUT;
}
cxlds->mbox_send = cxl_pci_mbox_send;
cxlds->payload_size =
@@ -400,58 +367,6 @@ static int cxl_map_regs(struct cxl_dev_state *cxlds, struct cxl_register_map *ma
return 0;
}
static void cxl_decode_regblock(u32 reg_lo, u32 reg_hi,
struct cxl_register_map *map)
{
map->block_offset =
((u64)reg_hi << 32) | (reg_lo & CXL_REGLOC_ADDR_MASK);
map->barno = FIELD_GET(CXL_REGLOC_BIR_MASK, reg_lo);
map->reg_type = FIELD_GET(CXL_REGLOC_RBI_MASK, reg_lo);
}
/**
* cxl_find_regblock() - Locate register blocks by type
* @pdev: The CXL PCI device to enumerate.
* @type: Register Block Indicator id
* @map: Enumeration output, clobbered on error
*
* Return: 0 if register block enumerated, negative error code otherwise
*
* A CXL DVSEC may point to one or more register blocks, search for them
* by @type.
*/
static int cxl_find_regblock(struct pci_dev *pdev, enum cxl_regloc_type type,
struct cxl_register_map *map)
{
u32 regloc_size, regblocks;
int regloc, i;
regloc = pci_find_dvsec_capability(pdev, PCI_DVSEC_VENDOR_ID_CXL,
PCI_DVSEC_ID_CXL_REGLOC_DVSEC_ID);
if (!regloc)
return -ENXIO;
pci_read_config_dword(pdev, regloc + PCI_DVSEC_HEADER1, &regloc_size);
regloc_size = FIELD_GET(PCI_DVSEC_HEADER1_LENGTH_MASK, regloc_size);
regloc += PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET;
regblocks = (regloc_size - PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET) / 8;
for (i = 0; i < regblocks; i++, regloc += 8) {
u32 reg_lo, reg_hi;
pci_read_config_dword(pdev, regloc, &reg_lo);
pci_read_config_dword(pdev, regloc + 4, &reg_hi);
cxl_decode_regblock(reg_lo, reg_hi, map);
if (map->reg_type == type)
return 0;
}
return -ENODEV;
}
static int cxl_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
struct cxl_register_map *map)
{
@@ -471,6 +386,165 @@ static int cxl_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
return rc;
}
static int wait_for_valid(struct cxl_dev_state *cxlds)
{
struct pci_dev *pdev = to_pci_dev(cxlds->dev);
int d = cxlds->cxl_dvsec, rc;
u32 val;
/*
* Memory_Info_Valid: When set, indicates that the CXL Range 1 Size high
* and Size Low registers are valid. Must be set within 1 second of
* deassertion of reset to CXL device. Likely it is already set by the
* time this runs, but otherwise give a 1.5 second timeout in case of
* clock skew.
*/
rc = pci_read_config_dword(pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(0), &val);
if (rc)
return rc;
if (val & CXL_DVSEC_MEM_INFO_VALID)
return 0;
msleep(1500);
rc = pci_read_config_dword(pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(0), &val);
if (rc)
return rc;
if (val & CXL_DVSEC_MEM_INFO_VALID)
return 0;
return -ETIMEDOUT;
}
/*
* Wait up to @mbox_ready_timeout for the device to report memory
* active.
*/
static int wait_for_media_ready(struct cxl_dev_state *cxlds)
{
struct pci_dev *pdev = to_pci_dev(cxlds->dev);
int d = cxlds->cxl_dvsec;
bool active = false;
u64 md_status;
int rc, i;
rc = wait_for_valid(cxlds);
if (rc)
return rc;
for (i = mbox_ready_timeout; i; i--) {
u32 temp;
int rc;
rc = pci_read_config_dword(
pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(0), &temp);
if (rc)
return rc;
active = FIELD_GET(CXL_DVSEC_MEM_ACTIVE, temp);
if (active)
break;
msleep(1000);
}
if (!active) {
dev_err(&pdev->dev,
"timeout awaiting memory active after %d seconds\n",
mbox_ready_timeout);
return -ETIMEDOUT;
}
md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);
if (!CXLMDEV_READY(md_status))
return -EIO;
return 0;
}
static int cxl_dvsec_ranges(struct cxl_dev_state *cxlds)
{
struct cxl_endpoint_dvsec_info *info = &cxlds->info;
struct pci_dev *pdev = to_pci_dev(cxlds->dev);
int d = cxlds->cxl_dvsec;
int hdm_count, rc, i;
u16 cap, ctrl;
if (!d)
return -ENXIO;
rc = pci_read_config_word(pdev, d + CXL_DVSEC_CAP_OFFSET, &cap);
if (rc)
return rc;
rc = pci_read_config_word(pdev, d + CXL_DVSEC_CTRL_OFFSET, &ctrl);
if (rc)
return rc;
if (!(cap & CXL_DVSEC_MEM_CAPABLE))
return -ENXIO;
/*
* It is not allowed by spec for MEM.capable to be set and have 0 legacy
* HDM decoders (values > 2 are also undefined as of CXL 2.0). As this
* driver is for a spec defined class code which must be CXL.mem
* capable, there is no point in continuing to enable CXL.mem.
*/
hdm_count = FIELD_GET(CXL_DVSEC_HDM_COUNT_MASK, cap);
if (!hdm_count || hdm_count > 2)
return -EINVAL;
rc = wait_for_valid(cxlds);
if (rc)
return rc;
info->mem_enabled = FIELD_GET(CXL_DVSEC_MEM_ENABLE, ctrl);
for (i = 0; i < hdm_count; i++) {
u64 base, size;
u32 temp;
rc = pci_read_config_dword(
pdev, d + CXL_DVSEC_RANGE_SIZE_HIGH(i), &temp);
if (rc)
return rc;
size = (u64)temp << 32;
rc = pci_read_config_dword(
pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(i), &temp);
if (rc)
return rc;
size |= temp & CXL_DVSEC_MEM_SIZE_LOW_MASK;
rc = pci_read_config_dword(
pdev, d + CXL_DVSEC_RANGE_BASE_HIGH(i), &temp);
if (rc)
return rc;
base = (u64)temp << 32;
rc = pci_read_config_dword(
pdev, d + CXL_DVSEC_RANGE_BASE_LOW(i), &temp);
if (rc)
return rc;
base |= temp & CXL_DVSEC_MEM_BASE_LOW_MASK;
info->dvsec_range[i] = (struct range) {
.start = base,
.end = base + size - 1
};
if (size)
info->ranges++;
}
return 0;
}
static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
struct cxl_register_map map;
@@ -493,6 +567,15 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
if (IS_ERR(cxlds))
return PTR_ERR(cxlds);
cxlds->serial = pci_get_dsn(pdev);
cxlds->cxl_dvsec = pci_find_dvsec_capability(
pdev, PCI_DVSEC_VENDOR_ID_CXL, CXL_DVSEC_PCIE_DEVICE);
if (!cxlds->cxl_dvsec)
dev_warn(&pdev->dev,
"Device DVSEC not present, skip CXL.mem init\n");
cxlds->wait_media_ready = wait_for_media_ready;
rc = cxl_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map);
if (rc)
return rc;
@@ -501,6 +584,17 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
if (rc)
return rc;
/*
* If the component registers can't be found, the cxl_pci driver may
* still be useful for management functions so don't return an error.
*/
cxlds->component_reg_phys = CXL_RESOURCE_NONE;
rc = cxl_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT, &map);
if (rc)
dev_warn(&pdev->dev, "No component registers (%d)\n", rc);
cxlds->component_reg_phys = cxl_regmap_to_base(pdev, &map);
rc = cxl_pci_setup_mailbox(cxlds);
if (rc)
return rc;
@@ -517,6 +611,11 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
if (rc)
return rc;
rc = cxl_dvsec_ranges(cxlds);
if (rc)
dev_warn(&pdev->dev,
"Failed to get DVSEC range information (%d)\n", rc);
cxlmd = devm_cxl_add_memdev(cxlds);
if (IS_ERR(cxlmd))
return PTR_ERR(cxlmd);

34
drivers/cxl/pci.h
View File

@@ -1,34 +0,0 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/* Copyright(c) 2020 Intel Corporation. All rights reserved. */
#ifndef __CXL_PCI_H__
#define __CXL_PCI_H__
#define CXL_MEMORY_PROGIF 0x10
/*
* See section 8.1 Configuration Space Registers in the CXL 2.0
* Specification
*/
#define PCI_DVSEC_HEADER1_LENGTH_MASK GENMASK(31, 20)
#define PCI_DVSEC_VENDOR_ID_CXL 0x1E98
#define PCI_DVSEC_ID_CXL 0x0
#define PCI_DVSEC_ID_CXL_REGLOC_DVSEC_ID 0x8
#define PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET 0xC
/* BAR Indicator Register (BIR) */
#define CXL_REGLOC_BIR_MASK GENMASK(2, 0)
/* Register Block Identifier (RBI) */
enum cxl_regloc_type {
CXL_REGLOC_RBI_EMPTY = 0,
CXL_REGLOC_RBI_COMPONENT,
CXL_REGLOC_RBI_VIRT,
CXL_REGLOC_RBI_MEMDEV,
CXL_REGLOC_RBI_TYPES
};
#define CXL_REGLOC_RBI_MASK GENMASK(15, 8)
#define CXL_REGLOC_ADDR_MASK GENMASK(31, 16)
#endif /* __CXL_PCI_H__ */

12
drivers/cxl/pmem.c
View File

@@ -43,7 +43,7 @@ static int cxl_nvdimm_probe(struct device *dev)
if (!cxl_nvb)
return -ENXIO;
device_lock(&cxl_nvb->dev);
cxl_device_lock(&cxl_nvb->dev);
if (!cxl_nvb->nvdimm_bus) {
rc = -ENXIO;
goto out;
@@ -68,7 +68,7 @@ static int cxl_nvdimm_probe(struct device *dev)
dev_set_drvdata(dev, nvdimm);
rc = devm_add_action_or_reset(dev, unregister_nvdimm, nvdimm);
out:
device_unlock(&cxl_nvb->dev);
cxl_device_unlock(&cxl_nvb->dev);
put_device(&cxl_nvb->dev);
return rc;
@@ -233,7 +233,7 @@ static void cxl_nvb_update_state(struct work_struct *work)
struct nvdimm_bus *victim_bus = NULL;
bool release = false, rescan = false;
device_lock(&cxl_nvb->dev);
cxl_device_lock(&cxl_nvb->dev);
switch (cxl_nvb->state) {
case CXL_NVB_ONLINE:
if (!online_nvdimm_bus(cxl_nvb)) {
@@ -251,7 +251,7 @@ static void cxl_nvb_update_state(struct work_struct *work)
default:
break;
}
device_unlock(&cxl_nvb->dev);
cxl_device_unlock(&cxl_nvb->dev);
if (release)
device_release_driver(&cxl_nvb->dev);
@@ -327,9 +327,9 @@ static int cxl_nvdimm_bridge_reset(struct device *dev, void *data)
return 0;
cxl_nvb = to_cxl_nvdimm_bridge(dev);
device_lock(dev);
cxl_device_lock(dev);
cxl_nvb->state = CXL_NVB_NEW;
device_unlock(dev);
cxl_device_unlock(dev);
return 0;
}

76
drivers/cxl/port.c Normal file
View File

@@ -0,0 +1,76 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2022 Intel Corporation. All rights reserved. */
#include <linux/device.h>
#include <linux/module.h>
#include <linux/slab.h>
#include "cxlmem.h"
#include "cxlpci.h"
/**
* DOC: cxl port
*
* The port driver enumerates dport via PCI and scans for HDM
* (Host-managed-Device-Memory) decoder resources via the
* @component_reg_phys value passed in by the agent that registered the
* port. All descendant ports of a CXL root port (described by platform
* firmware) are managed in this drivers context. Each driver instance
* is responsible for tearing down the driver context of immediate
* descendant ports. The locking for this is validated by
* CONFIG_PROVE_CXL_LOCKING.
*
* The primary service this driver provides is presenting APIs to other
* drivers to utilize the decoders, and indicating to userspace (via bind
* status) the connectivity of the CXL.mem protocol throughout the
* PCIe topology.
*/
static void schedule_detach(void *cxlmd)
{
schedule_cxl_memdev_detach(cxlmd);
}
static int cxl_port_probe(struct device *dev)
{
struct cxl_port *port = to_cxl_port(dev);
struct cxl_hdm *cxlhdm;
int rc;
if (is_cxl_endpoint(port)) {
struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport);
get_device(&cxlmd->dev);
rc = devm_add_action_or_reset(dev, schedule_detach, cxlmd);
if (rc)
return rc;
} else {
rc = devm_cxl_port_enumerate_dports(port);
if (rc < 0)
return rc;
if (rc == 1)
return devm_cxl_add_passthrough_decoder(port);
}
cxlhdm = devm_cxl_setup_hdm(port);
if (IS_ERR(cxlhdm))
return PTR_ERR(cxlhdm);
rc = devm_cxl_enumerate_decoders(cxlhdm);
if (rc) {
dev_err(dev, "Couldn't enumerate decoders (%d)\n", rc);
return rc;
}
return 0;
}
static struct cxl_driver cxl_port_driver = {
.name = "cxl_port",
.probe = cxl_port_probe,
.id = CXL_DEVICE_PORT,
};
module_cxl_driver(cxl_port_driver);
MODULE_LICENSE("GPL v2");
MODULE_IMPORT_NS(CXL);
MODULE_ALIAS_CXL(CXL_DEVICE_PORT);

2
drivers/dax/super.c
View File

@@ -21,6 +21,7 @@
* @cdev: optional character interface for "device dax"
* @private: dax driver private data
* @flags: state and boolean properties
* @ops: operations for this device
*/
struct dax_device {
struct inode inode;
@@ -476,6 +477,7 @@ static int dax_fs_init(void)
static void dax_fs_exit(void)
{
kern_unmount(dax_mnt);
rcu_barrier();
kmem_cache_destroy(dax_cache);
}

1
drivers/infiniband/core/addr.c
View File

@@ -37,7 +37,6 @@
#include <linux/inetdevice.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/module.h>
#include <net/arp.h>
#include <net/neighbour.h>
#include <net/route.h>

1
drivers/infiniband/core/cache.c
View File

@@ -34,7 +34,6 @@
*/
#include <linux/if_vlan.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/slab.h>
#include <linux/workqueue.h>

2
drivers/infiniband/core/cma.c
View File

@@ -2642,7 +2642,7 @@ int rdma_set_ack_timeout(struct rdma_cm_id *id, u8 timeout)
{
struct rdma_id_private *id_priv;
if (id->qp_type != IB_QPT_RC)
if (id->qp_type != IB_QPT_RC && id->qp_type != IB_QPT_XRC_INI)
return -EINVAL;
id_priv = container_of(id, struct rdma_id_private, id);

1
drivers/infiniband/core/cma_configfs.c
View File

@@ -30,7 +30,6 @@
* SOFTWARE.
*/
#include <linux/module.h>
#include <linux/configfs.h>
#include <rdma/ib_verbs.h>
#include <rdma/rdma_cm.h>

1
drivers/infiniband/core/cq.c
View File

@@ -2,7 +2,6 @@
/*
* Copyright (c) 2015 HGST, a Western Digital Company.
*/
#include <linux/module.h>
#include <linux/err.h>
#include <linux/slab.h>
#include <rdma/ib_verbs.h>

1
drivers/infiniband/core/iwpm_util.h
View File

@@ -33,7 +33,6 @@
#ifndef _IWPM_UTIL_H
#define _IWPM_UTIL_H
#include <linux/module.h>
#include <linux/io.h>
#include <linux/in.h>
#include <linux/in6.h>

3
drivers/infiniband/core/nldev.c
View File

@@ -1951,9 +1951,10 @@ static int nldev_stat_set_counter_dynamic_doit(struct nlattr *tb[],
u32 port)
{
struct rdma_hw_stats *stats;
int rem, i, index, ret = 0;
struct nlattr *entry_attr;
unsigned long *target;
int rem, i, ret = 0;
u32 index;
stats = ib_get_hw_stats_port(device, port);
if (!stats)

1
drivers/infiniband/core/sa_query.c
View File

@@ -32,7 +32,6 @@
* SOFTWARE.
*/
#include <linux/module.h>
#include <linux/init.h>
#include <linux/err.h>
#include <linux/random.h>

9
drivers/infiniband/core/verbs.c
View File

@@ -268,9 +268,6 @@ struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags,
return ERR_PTR(-ENOMEM);
pd->device = device;
pd->uobject = NULL;
pd->__internal_mr = NULL;
atomic_set(&pd->usecnt, 0);
pd->flags = flags;
rdma_restrack_new(&pd->res, RDMA_RESTRACK_PD);
@@ -341,11 +338,6 @@ int ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata)
pd->__internal_mr = NULL;
}
/* uverbs manipulates usecnt with proper locking, while the kabi
* requires the caller to guarantee we can't race here.
*/
WARN_ON(atomic_read(&pd->usecnt));
ret = pd->device->ops.dealloc_pd(pd, udata);
if (ret)
return ret;
@@ -2153,6 +2145,7 @@ struct ib_mr *ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
return mr;
mr->device = pd->device;
mr->type = IB_MR_TYPE_USER;
mr->pd = pd;
mr->dm = NULL;
atomic_inc(&pd->usecnt);

5
drivers/infiniband/hw/hfi1/affinity.c
View File

@@ -5,7 +5,6 @@
#include <linux/topology.h>
#include <linux/cpumask.h>
#include <linux/module.h>
#include <linux/interrupt.h>
#include <linux/numa.h>
@@ -667,7 +666,7 @@ int hfi1_dev_affinity_init(struct hfi1_devdata *dd)
* engines, use the same CPU cores as general/control
* context.
*/
if (cpumask_weight(&entry->def_intr.mask) == 0)
if (cpumask_empty(&entry->def_intr.mask))
cpumask_copy(&entry->def_intr.mask,
&entry->general_intr_mask);
}
@@ -687,7 +686,7 @@ int hfi1_dev_affinity_init(struct hfi1_devdata *dd)
* vectors, use the same CPU core as the general/control
* context.
*/
if (cpumask_weight(&entry->comp_vect_mask) == 0)
if (cpumask_empty(&entry->comp_vect_mask))
cpumask_copy(&entry->comp_vect_mask,
&entry->general_intr_mask);
}

1
drivers/infiniband/hw/hfi1/debugfs.c
View File

@@ -7,7 +7,6 @@
#include <linux/seq_file.h>
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/module.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/ratelimit.h>

1
drivers/infiniband/hw/hfi1/device.c
View File

@@ -4,7 +4,6 @@
*/
#include <linux/cdev.h>
#include <linux/module.h>
#include <linux/device.h>
#include <linux/fs.h>

1
drivers/infiniband/hw/hfi1/fault.c
View File

@@ -6,7 +6,6 @@
#include <linux/debugfs.h>
#include <linux/seq_file.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/bitmap.h>

1
drivers/infiniband/hw/hfi1/firmware.c
View File

@@ -5,7 +5,6 @@
#include <linux/firmware.h>
#include <linux/mutex.h>
#include <linux/module.h>
#include <linux/delay.h>
#include <linux/crc32.h>

3
drivers/infiniband/hw/hfi1/verbs.c
View File

@@ -1397,8 +1397,7 @@ static int query_port(struct rvt_dev_info *rdi, u32 port_num,
4096 : hfi1_max_mtu), IB_MTU_4096);
props->active_mtu = !valid_ib_mtu(ppd->ibmtu) ? props->max_mtu :
mtu_to_enum(ppd->ibmtu, IB_MTU_4096);
props->phys_mtu = HFI1_CAP_IS_KSET(AIP) ? hfi1_max_mtu :
ib_mtu_enum_to_int(props->max_mtu);
props->phys_mtu = hfi1_max_mtu;
return 0;
}

97
drivers/infiniband/hw/hns/hns_roce_cmd.c
View File

@@ -38,45 +38,36 @@
#define CMD_POLL_TOKEN 0xffff
#define CMD_MAX_NUM 32
static int hns_roce_cmd_mbox_post_hw(struct hns_roce_dev *hr_dev, u64 in_param,
u64 out_param, u32 in_modifier,
u8 op_modifier, u16 op, u16 token,
int event)
static int hns_roce_cmd_mbox_post_hw(struct hns_roce_dev *hr_dev,
struct hns_roce_mbox_msg *mbox_msg)
{
return hr_dev->hw->post_mbox(hr_dev, in_param, out_param, in_modifier,
op_modifier, op, token, event);
return hr_dev->hw->post_mbox(hr_dev, mbox_msg);
}
/* this should be called with "poll_sem" */
static int __hns_roce_cmd_mbox_poll(struct hns_roce_dev *hr_dev, u64 in_param,
u64 out_param, unsigned long in_modifier,
u8 op_modifier, u16 op,
unsigned int timeout)
static int __hns_roce_cmd_mbox_poll(struct hns_roce_dev *hr_dev,
struct hns_roce_mbox_msg *mbox_msg)
{
int ret;
ret = hns_roce_cmd_mbox_post_hw(hr_dev, in_param, out_param,
in_modifier, op_modifier, op,
CMD_POLL_TOKEN, 0);
ret = hns_roce_cmd_mbox_post_hw(hr_dev, mbox_msg);
if (ret) {
dev_err_ratelimited(hr_dev->dev,
"failed to post mailbox 0x%x in poll mode, ret = %d.\n",
op, ret);
mbox_msg->cmd, ret);
return ret;
}
return hr_dev->hw->poll_mbox_done(hr_dev, timeout);
return hr_dev->hw->poll_mbox_done(hr_dev);
}
static int hns_roce_cmd_mbox_poll(struct hns_roce_dev *hr_dev, u64 in_param,
u64 out_param, unsigned long in_modifier,
u8 op_modifier, u16 op, unsigned int timeout)
static int hns_roce_cmd_mbox_poll(struct hns_roce_dev *hr_dev,
struct hns_roce_mbox_msg *mbox_msg)
{
int ret;
down(&hr_dev->cmd.poll_sem);
ret = __hns_roce_cmd_mbox_poll(hr_dev, in_param, out_param, in_modifier,
op_modifier, op, timeout);
ret = __hns_roce_cmd_mbox_poll(hr_dev, mbox_msg);
up(&hr_dev->cmd.poll_sem);
return ret;
@@ -100,10 +91,8 @@ void hns_roce_cmd_event(struct hns_roce_dev *hr_dev, u16 token, u8 status,
complete(&context->done);
}
static int __hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param,
u64 out_param, unsigned long in_modifier,
u8 op_modifier, u16 op,
unsigned int timeout)
static int __hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev,
struct hns_roce_mbox_msg *mbox_msg)
{
struct hns_roce_cmdq *cmd = &hr_dev->cmd;
struct hns_roce_cmd_context *context;
@@ -124,20 +113,19 @@ static int __hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param,
reinit_completion(&context->done);
ret = hns_roce_cmd_mbox_post_hw(hr_dev, in_param, out_param,
in_modifier, op_modifier, op,
context->token, 1);
mbox_msg->token = context->token;
ret = hns_roce_cmd_mbox_post_hw(hr_dev, mbox_msg);
if (ret) {
dev_err_ratelimited(dev,
"failed to post mailbox 0x%x in event mode, ret = %d.\n",
op, ret);
mbox_msg->cmd, ret);
goto out;
}
if (!wait_for_completion_timeout(&context->done,
msecs_to_jiffies(timeout))) {
msecs_to_jiffies(HNS_ROCE_CMD_TIMEOUT_MSECS))) {
dev_err_ratelimited(dev, "[cmd] token 0x%x mailbox 0x%x timeout.\n",
context->token, op);
context->token, mbox_msg->cmd);
ret = -EBUSY;
goto out;
}
@@ -145,45 +133,50 @@ static int __hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param,
ret = context->result;
if (ret)
dev_err_ratelimited(dev, "[cmd] token 0x%x mailbox 0x%x error %d.\n",
context->token, op, ret);
context->token, mbox_msg->cmd, ret);
out:
context->busy = 0;
return ret;
}
static int hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param,
u64 out_param, unsigned long in_modifier,
u8 op_modifier, u16 op, unsigned int timeout)
static int hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev,
struct hns_roce_mbox_msg *mbox_msg)
{
int ret;
down(&hr_dev->cmd.event_sem);
ret = __hns_roce_cmd_mbox_wait(hr_dev, in_param, out_param, in_modifier,
op_modifier, op, timeout);
ret = __hns_roce_cmd_mbox_wait(hr_dev, mbox_msg);
up(&hr_dev->cmd.event_sem);
return ret;
}
int hns_roce_cmd_mbox(struct hns_roce_dev *hr_dev, u64 in_param, u64 out_param,
unsigned long in_modifier, u8 op_modifier, u16 op,
unsigned int timeout)
u8 cmd, unsigned long tag)
{
struct hns_roce_mbox_msg mbox_msg = {};
bool is_busy;
if (hr_dev->hw->chk_mbox_avail)
if (!hr_dev->hw->chk_mbox_avail(hr_dev, &is_busy))
return is_busy ? -EBUSY : 0;
if (hr_dev->cmd.use_events)
return hns_roce_cmd_mbox_wait(hr_dev, in_param, out_param,
in_modifier, op_modifier, op,
timeout);
else
return hns_roce_cmd_mbox_poll(hr_dev, in_param, out_param,
in_modifier, op_modifier, op,
timeout);
mbox_msg.in_param = in_param;
mbox_msg.out_param = out_param;
mbox_msg.cmd = cmd;
mbox_msg.tag = tag;
if (hr_dev->cmd.use_events) {
mbox_msg.event_en = 1;
return hns_roce_cmd_mbox_wait(hr_dev, &mbox_msg);
} else {
mbox_msg.event_en = 0;
mbox_msg.token = CMD_POLL_TOKEN;
return hns_roce_cmd_mbox_poll(hr_dev, &mbox_msg);
}
}
int hns_roce_cmd_init(struct hns_roce_dev *hr_dev)
@@ -269,3 +262,15 @@ void hns_roce_free_cmd_mailbox(struct hns_roce_dev *hr_dev,
dma_pool_free(hr_dev->cmd.pool, mailbox->buf, mailbox->dma);
kfree(mailbox);
}
int hns_roce_create_hw_ctx(struct hns_roce_dev *dev,
struct hns_roce_cmd_mailbox *mailbox,
u8 cmd, unsigned long idx)
{
return hns_roce_cmd_mbox(dev, mailbox->dma, 0, cmd, idx);
}
int hns_roce_destroy_hw_ctx(struct hns_roce_dev *dev, u8 cmd, unsigned long idx)
{
return hns_roce_cmd_mbox(dev, 0, 0, cmd, idx);
}

8
drivers/infiniband/hw/hns/hns_roce_cmd.h
View File

@@ -140,12 +140,16 @@ enum {
};
int hns_roce_cmd_mbox(struct hns_roce_dev *hr_dev, u64 in_param, u64 out_param,
unsigned long in_modifier, u8 op_modifier, u16 op,
unsigned int timeout);
u8 cmd, unsigned long tag);
struct hns_roce_cmd_mailbox *
hns_roce_alloc_cmd_mailbox(struct hns_roce_dev *hr_dev);
void hns_roce_free_cmd_mailbox(struct hns_roce_dev *hr_dev,
struct hns_roce_cmd_mailbox *mailbox);
int hns_roce_create_hw_ctx(struct hns_roce_dev *dev,
struct hns_roce_cmd_mailbox *mailbox,
u8 cmd, unsigned long idx);
int hns_roce_destroy_hw_ctx(struct hns_roce_dev *dev, u8 cmd,
unsigned long idx);
#endif /* _HNS_ROCE_CMD_H */

71
drivers/infiniband/hw/hns/hns_roce_cq.c
View File

@@ -100,12 +100,39 @@ static void free_cqn(struct hns_roce_dev *hr_dev, unsigned long cqn)
mutex_unlock(&cq_table->bank_mutex);
}
static int hns_roce_create_cqc(struct hns_roce_dev *hr_dev,
struct hns_roce_cq *hr_cq,
u64 *mtts, dma_addr_t dma_handle)
{
struct ib_device *ibdev = &hr_dev->ib_dev;
struct hns_roce_cmd_mailbox *mailbox;
int ret;
mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
if (IS_ERR(mailbox)) {
ibdev_err(ibdev, "failed to alloc mailbox for CQC.\n");
return PTR_ERR(mailbox);
}
hr_dev->hw->write_cqc(hr_dev, hr_cq, mailbox->buf, mtts, dma_handle);
ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_CQC,
hr_cq->cqn);
if (ret)
ibdev_err(ibdev,
"failed to send create cmd for CQ(0x%lx), ret = %d.\n",
hr_cq->cqn, ret);
hns_roce_free_cmd_mailbox(hr_dev, mailbox);
return ret;
}
static int alloc_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)
{
struct hns_roce_cq_table *cq_table = &hr_dev->cq_table;
struct ib_device *ibdev = &hr_dev->ib_dev;
struct hns_roce_cmd_mailbox *mailbox;
u64 mtts[MTT_MIN_COUNT] = { 0 };
u64 mtts[MTT_MIN_COUNT] = {};
dma_addr_t dma_handle;
int ret;
@@ -121,7 +148,7 @@ static int alloc_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)
if (ret) {
ibdev_err(ibdev, "failed to get CQ(0x%lx) context, ret = %d.\n",
hr_cq->cqn, ret);
goto err_out;
return ret;
}
ret = xa_err(xa_store(&cq_table->array, hr_cq->cqn, hr_cq, GFP_KERNEL));
@@ -130,41 +157,17 @@ static int alloc_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)
goto err_put;
}
/* Allocate mailbox memory */
mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
if (IS_ERR(mailbox)) {
ret = PTR_ERR(mailbox);
ret = hns_roce_create_cqc(hr_dev, hr_cq, mtts, dma_handle);
if (ret)
goto err_xa;
}
hr_dev->hw->write_cqc(hr_dev, hr_cq, mailbox->buf, mtts, dma_handle);
/* Send mailbox to hw */
ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_cq->cqn, 0,
HNS_ROCE_CMD_CREATE_CQC, HNS_ROCE_CMD_TIMEOUT_MSECS);
hns_roce_free_cmd_mailbox(hr_dev, mailbox);
if (ret) {
ibdev_err(ibdev,
"failed to send create cmd for CQ(0x%lx), ret = %d.\n",
hr_cq->cqn, ret);
goto err_xa;
}
hr_cq->cons_index = 0;
hr_cq->arm_sn = 1;
refcount_set(&hr_cq->refcount, 1);
init_completion(&hr_cq->free);
return 0;
err_xa:
xa_erase(&cq_table->array, hr_cq->cqn);
err_put:
hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn);
err_out:
return ret;
}
@@ -174,9 +177,8 @@ static void free_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)
struct device *dev = hr_dev->dev;
int ret;
ret = hns_roce_cmd_mbox(hr_dev, 0, 0, hr_cq->cqn, 1,
HNS_ROCE_CMD_DESTROY_CQC,
HNS_ROCE_CMD_TIMEOUT_MSECS);
ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_CQC,
hr_cq->cqn);
if (ret)
dev_err(dev, "DESTROY_CQ failed (%d) for CQN %06lx\n", ret,
hr_cq->cqn);
@@ -413,6 +415,11 @@ int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr,
goto err_cqc;
}
hr_cq->cons_index = 0;
hr_cq->arm_sn = 1;
refcount_set(&hr_cq->refcount, 1);
init_completion(&hr_cq->free);
return 0;
err_cqc:

26
drivers/infiniband/hw/hns/hns_roce_device.h
View File

@@ -561,6 +561,15 @@ struct hns_roce_cmd_mailbox {
dma_addr_t dma;
};
struct hns_roce_mbox_msg {
u64 in_param;
u64 out_param;
u8 cmd;
u32 tag;
u16 token;
u8 event_en;
};
struct hns_roce_dev;
struct hns_roce_rinl_sge {
@@ -624,6 +633,7 @@ struct hns_roce_qp {
u32 next_sge;
enum ib_mtu path_mtu;
u32 max_inline_data;
u8 free_mr_en;
/* 0: flush needed, 1: unneeded */
unsigned long flush_flag;
@@ -851,11 +861,9 @@ struct hns_roce_hw {
int (*hw_profile)(struct hns_roce_dev *hr_dev);
int (*hw_init)(struct hns_roce_dev *hr_dev);
void (*hw_exit)(struct hns_roce_dev *hr_dev);
int (*post_mbox)(struct hns_roce_dev *hr_dev, u64 in_param,
u64 out_param, u32 in_modifier, u8 op_modifier, u16 op,
u16 token, int event);
int (*poll_mbox_done)(struct hns_roce_dev *hr_dev,
unsigned int timeout);
int (*post_mbox)(struct hns_roce_dev *hr_dev,
struct hns_roce_mbox_msg *mbox_msg);
int (*poll_mbox_done)(struct hns_roce_dev *hr_dev);
bool (*chk_mbox_avail)(struct hns_roce_dev *hr_dev, bool *is_busy);
int (*set_gid)(struct hns_roce_dev *hr_dev, int gid_index,
const union ib_gid *gid, const struct ib_gid_attr *attr);
@@ -873,15 +881,16 @@ struct hns_roce_hw {
struct hns_roce_cq *hr_cq, void *mb_buf, u64 *mtts,
dma_addr_t dma_handle);
int (*set_hem)(struct hns_roce_dev *hr_dev,
struct hns_roce_hem_table *table, int obj, int step_idx);
struct hns_roce_hem_table *table, int obj, u32 step_idx);
int (*clear_hem)(struct hns_roce_dev *hr_dev,
struct hns_roce_hem_table *table, int obj,
int step_idx);
u32 step_idx);
int (*modify_qp)(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
int attr_mask, enum ib_qp_state cur_state,
enum ib_qp_state new_state);
int (*qp_flow_control_init)(struct hns_roce_dev *hr_dev,
struct hns_roce_qp *hr_qp);
void (*dereg_mr)(struct hns_roce_dev *hr_dev);
int (*init_eq)(struct hns_roce_dev *hr_dev);
void (*cleanup_eq)(struct hns_roce_dev *hr_dev);
int (*write_srqc)(struct hns_roce_srq *srq, void *mb_buf);
@@ -1145,9 +1154,6 @@ struct ib_mr *hns_roce_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
int hns_roce_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
unsigned int *sg_offset);
int hns_roce_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
int hns_roce_hw_destroy_mpt(struct hns_roce_dev *hr_dev,
struct hns_roce_cmd_mailbox *mailbox,
unsigned long mpt_index);
unsigned long key_to_hw_index(u32 key);
int hns_roce_alloc_mw(struct ib_mw *mw, struct ib_udata *udata);

4
drivers/infiniband/hw/hns/hns_roce_hem.c
View File

@@ -488,7 +488,7 @@ static int set_mhop_hem(struct hns_roce_dev *hr_dev,
struct hns_roce_hem_index *index)
{
struct ib_device *ibdev = &hr_dev->ib_dev;
int step_idx;
u32 step_idx;
int ret = 0;
if (index->inited & HEM_INDEX_L0) {
@@ -618,7 +618,7 @@ static void clear_mhop_hem(struct hns_roce_dev *hr_dev,
struct ib_device *ibdev = &hr_dev->ib_dev;
u32 hop_num = mhop->hop_num;
u32 chunk_ba_num;
int step_idx;
u32 step_idx;
index->inited = HEM_INDEX_BUF;
chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;

458
drivers/infiniband/hw/hns/hns_roce_hw_v2.c
View File

@@ -1344,17 +1344,17 @@ static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
return ret;
}
static int config_hem_ba_to_hw(struct hns_roce_dev *hr_dev, unsigned long obj,
dma_addr_t base_addr, u16 op)
static int config_hem_ba_to_hw(struct hns_roce_dev *hr_dev,
dma_addr_t base_addr, u8 cmd, unsigned long tag)
{
struct hns_roce_cmd_mailbox *mbox = hns_roce_alloc_cmd_mailbox(hr_dev);
struct hns_roce_cmd_mailbox *mbox;
int ret;
mbox = hns_roce_alloc_cmd_mailbox(hr_dev);
if (IS_ERR(mbox))
return PTR_ERR(mbox);
ret = hns_roce_cmd_mbox(hr_dev, base_addr, mbox->dma, obj, 0, op,
HNS_ROCE_CMD_TIMEOUT_MSECS);
ret = hns_roce_cmd_mbox(hr_dev, base_addr, mbox->dma, cmd, tag);
hns_roce_free_cmd_mailbox(hr_dev, mbox);
return ret;
}
@@ -2664,6 +2664,194 @@ static void free_dip_list(struct hns_roce_dev *hr_dev)
spin_unlock_irqrestore(&hr_dev->dip_list_lock, flags);
}
static void free_mr_exit(struct hns_roce_dev *hr_dev)
{
struct hns_roce_v2_priv *priv = hr_dev->priv;
struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
int ret;
int i;
for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) {
if (free_mr->rsv_qp[i]) {
ret = ib_destroy_qp(free_mr->rsv_qp[i]);
if (ret)
ibdev_err(&hr_dev->ib_dev,
"failed to destroy qp in free mr.\n");
free_mr->rsv_qp[i] = NULL;
}
}
if (free_mr->rsv_cq) {
ib_destroy_cq(free_mr->rsv_cq);
free_mr->rsv_cq = NULL;
}
if (free_mr->rsv_pd) {
ib_dealloc_pd(free_mr->rsv_pd);
free_mr->rsv_pd = NULL;
}
}
static int free_mr_alloc_res(struct hns_roce_dev *hr_dev)
{
struct hns_roce_v2_priv *priv = hr_dev->priv;
struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
struct ib_device *ibdev = &hr_dev->ib_dev;
struct ib_cq_init_attr cq_init_attr = {};
struct ib_qp_init_attr qp_init_attr = {};
struct ib_pd *pd;
struct ib_cq *cq;
struct ib_qp *qp;
int ret;
int i;
pd = ib_alloc_pd(ibdev, 0);
if (IS_ERR(pd)) {
ibdev_err(ibdev, "failed to create pd for free mr.\n");
return PTR_ERR(pd);
}
free_mr->rsv_pd = pd;
cq_init_attr.cqe = HNS_ROCE_FREE_MR_USED_CQE_NUM;
cq = ib_create_cq(ibdev, NULL, NULL, NULL, &cq_init_attr);
if (IS_ERR(cq)) {
ibdev_err(ibdev, "failed to create cq for free mr.\n");
ret = PTR_ERR(cq);
goto create_failed;
}
free_mr->rsv_cq = cq;
qp_init_attr.qp_type = IB_QPT_RC;
qp_init_attr.sq_sig_type = IB_SIGNAL_ALL_WR;
qp_init_attr.send_cq = free_mr->rsv_cq;
qp_init_attr.recv_cq = free_mr->rsv_cq;
for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) {
qp_init_attr.cap.max_send_wr = HNS_ROCE_FREE_MR_USED_SQWQE_NUM;
qp_init_attr.cap.max_send_sge = HNS_ROCE_FREE_MR_USED_SQSGE_NUM;
qp_init_attr.cap.max_recv_wr = HNS_ROCE_FREE_MR_USED_RQWQE_NUM;
qp_init_attr.cap.max_recv_sge = HNS_ROCE_FREE_MR_USED_RQSGE_NUM;
qp = ib_create_qp(free_mr->rsv_pd, &qp_init_attr);
if (IS_ERR(qp)) {
ibdev_err(ibdev, "failed to create qp for free mr.\n");
ret = PTR_ERR(qp);
goto create_failed;
}
free_mr->rsv_qp[i] = qp;
}
return 0;
create_failed:
free_mr_exit(hr_dev);
return ret;
}
static int free_mr_modify_rsv_qp(struct hns_roce_dev *hr_dev,
struct ib_qp_attr *attr, int sl_num)
{
struct hns_roce_v2_priv *priv = hr_dev->priv;
struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
struct ib_device *ibdev = &hr_dev->ib_dev;
struct hns_roce_qp *hr_qp;
int loopback;
int mask;
int ret;
hr_qp = to_hr_qp(free_mr->rsv_qp[sl_num]);
hr_qp->free_mr_en = 1;
mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS;
attr->qp_state = IB_QPS_INIT;
attr->port_num = 1;
attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE;
ret = ib_modify_qp(&hr_qp->ibqp, attr, mask);
if (ret) {
ibdev_err(ibdev, "failed to modify qp to init, ret = %d.\n",
ret);
return ret;
}
loopback = hr_dev->loop_idc;
/* Set qpc lbi = 1 incidate loopback IO */
hr_dev->loop_idc = 1;
mask = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU | IB_QP_DEST_QPN |
IB_QP_RQ_PSN | IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER;
attr->qp_state = IB_QPS_RTR;
attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
attr->path_mtu = IB_MTU_256;
attr->dest_qp_num = hr_qp->qpn;
attr->rq_psn = HNS_ROCE_FREE_MR_USED_PSN;
rdma_ah_set_sl(&attr->ah_attr, (u8)sl_num);
ret = ib_modify_qp(&hr_qp->ibqp, attr, mask);
hr_dev->loop_idc = loopback;
if (ret) {
ibdev_err(ibdev, "failed to modify qp to rtr, ret = %d.\n",
ret);
return ret;
}
mask = IB_QP_STATE | IB_QP_SQ_PSN | IB_QP_RETRY_CNT | IB_QP_TIMEOUT |
IB_QP_RNR_RETRY | IB_QP_MAX_QP_RD_ATOMIC;
attr->qp_state = IB_QPS_RTS;
attr->sq_psn = HNS_ROCE_FREE_MR_USED_PSN;
attr->retry_cnt = HNS_ROCE_FREE_MR_USED_QP_RETRY_CNT;
attr->timeout = HNS_ROCE_FREE_MR_USED_QP_TIMEOUT;
ret = ib_modify_qp(&hr_qp->ibqp, attr, mask);
if (ret)
ibdev_err(ibdev, "failed to modify qp to rts, ret = %d.\n",
ret);
return ret;
}
static int free_mr_modify_qp(struct hns_roce_dev *hr_dev)
{
struct hns_roce_v2_priv *priv = hr_dev->priv;
struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
struct ib_qp_attr attr = {};
int ret;
int i;
rdma_ah_set_grh(&attr.ah_attr, NULL, 0, 0, 1, 0);
rdma_ah_set_static_rate(&attr.ah_attr, 3);
rdma_ah_set_port_num(&attr.ah_attr, 1);
for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) {
ret = free_mr_modify_rsv_qp(hr_dev, &attr, i);
if (ret)
return ret;
}
return 0;
}
static int free_mr_init(struct hns_roce_dev *hr_dev)
{
int ret;
ret = free_mr_alloc_res(hr_dev);
if (ret)
return ret;
ret = free_mr_modify_qp(hr_dev);
if (ret)
goto err_modify_qp;
return 0;
err_modify_qp:
free_mr_exit(hr_dev);
return ret;
}
static int get_hem_table(struct hns_roce_dev *hr_dev)
{
unsigned int qpc_count;
@@ -2780,21 +2968,21 @@ static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev)
free_dip_list(hr_dev);
}
static int hns_roce_mbox_post(struct hns_roce_dev *hr_dev, u64 in_param,
u64 out_param, u32 in_modifier, u8 op_modifier,
u16 op, u16 token, int event)
static int hns_roce_mbox_post(struct hns_roce_dev *hr_dev,
struct hns_roce_mbox_msg *mbox_msg)
{
struct hns_roce_cmq_desc desc;
struct hns_roce_post_mbox *mb = (struct hns_roce_post_mbox *)desc.data;
hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_POST_MB, false);
mb->in_param_l = cpu_to_le32(in_param);
mb->in_param_h = cpu_to_le32(in_param >> 32);
mb->out_param_l = cpu_to_le32(out_param);
mb->out_param_h = cpu_to_le32(out_param >> 32);
mb->cmd_tag = cpu_to_le32(in_modifier << 8 | op);
mb->token_event_en = cpu_to_le32(event << 16 | token);
mb->in_param_l = cpu_to_le32(mbox_msg->in_param);
mb->in_param_h = cpu_to_le32(mbox_msg->in_param >> 32);
mb->out_param_l = cpu_to_le32(mbox_msg->out_param);
mb->out_param_h = cpu_to_le32(mbox_msg->out_param >> 32);
mb->cmd_tag = cpu_to_le32(mbox_msg->tag << 8 | mbox_msg->cmd);
mb->token_event_en = cpu_to_le32(mbox_msg->event_en << 16 |
mbox_msg->token);
return hns_roce_cmq_send(hr_dev, &desc, 1);
}
@@ -2847,9 +3035,8 @@ static int v2_wait_mbox_complete(struct hns_roce_dev *hr_dev, u32 timeout,
return ret;
}
static int v2_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param,
u64 out_param, u32 in_modifier, u8 op_modifier,
u16 op, u16 token, int event)
static int v2_post_mbox(struct hns_roce_dev *hr_dev,
struct hns_roce_mbox_msg *mbox_msg)
{
u8 status = 0;
int ret;
@@ -2865,8 +3052,7 @@ static int v2_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param,
}
/* Post new message to mbox */
ret = hns_roce_mbox_post(hr_dev, in_param, out_param, in_modifier,
op_modifier, op, token, event);
ret = hns_roce_mbox_post(hr_dev, mbox_msg);
if (ret)
dev_err_ratelimited(hr_dev->dev,
"failed to post mailbox, ret = %d.\n", ret);
@@ -2874,12 +3060,13 @@ static int v2_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param,
return ret;
}
static int v2_poll_mbox_done(struct hns_roce_dev *hr_dev, unsigned int timeout)
static int v2_poll_mbox_done(struct hns_roce_dev *hr_dev)
{
u8 status = 0;
int ret;
ret = v2_wait_mbox_complete(hr_dev, timeout, &status);
ret = v2_wait_mbox_complete(hr_dev, HNS_ROCE_CMD_TIMEOUT_MSECS,
&status);
if (!ret) {
if (status != MB_ST_COMPLETE_SUCC)
return -EBUSY;
@@ -3245,6 +3432,98 @@ static int hns_roce_v2_mw_write_mtpt(void *mb_buf, struct hns_roce_mw *mw)
return 0;
}
static int free_mr_post_send_lp_wqe(struct hns_roce_qp *hr_qp)
{
struct hns_roce_dev *hr_dev = to_hr_dev(hr_qp->ibqp.device);
struct ib_device *ibdev = &hr_dev->ib_dev;
const struct ib_send_wr *bad_wr;
struct ib_rdma_wr rdma_wr = {};
struct ib_send_wr *send_wr;
int ret;
send_wr = &rdma_wr.wr;
send_wr->opcode = IB_WR_RDMA_WRITE;
ret = hns_roce_v2_post_send(&hr_qp->ibqp, send_wr, &bad_wr);
if (ret) {
ibdev_err(ibdev, "failed to post wqe for free mr, ret = %d.\n",
ret);
return ret;
}
return 0;
}
static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries,
struct ib_wc *wc);
static void free_mr_send_cmd_to_hw(struct hns_roce_dev *hr_dev)
{
struct hns_roce_v2_priv *priv = hr_dev->priv;
struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
struct ib_wc wc[ARRAY_SIZE(free_mr->rsv_qp)];
struct ib_device *ibdev = &hr_dev->ib_dev;
struct hns_roce_qp *hr_qp;
unsigned long end;
int cqe_cnt = 0;
int npolled;
int ret;
int i;
/*
* If the device initialization is not complete or in the uninstall
* process, then there is no need to execute free mr.
*/
if (priv->handle->rinfo.reset_state == HNS_ROCE_STATE_RST_INIT ||
priv->handle->rinfo.instance_state == HNS_ROCE_STATE_INIT ||
hr_dev->state == HNS_ROCE_DEVICE_STATE_UNINIT)
return;
mutex_lock(&free_mr->mutex);
for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) {
hr_qp = to_hr_qp(free_mr->rsv_qp[i]);
ret = free_mr_post_send_lp_wqe(hr_qp);
if (ret) {
ibdev_err(ibdev,
"failed to send wqe (qp:0x%lx) for free mr, ret = %d.\n",
hr_qp->qpn, ret);
break;
}
cqe_cnt++;
}
end = msecs_to_jiffies(HNS_ROCE_V2_FREE_MR_TIMEOUT) + jiffies;
while (cqe_cnt) {
npolled = hns_roce_v2_poll_cq(free_mr->rsv_cq, cqe_cnt, wc);
if (npolled < 0) {
ibdev_err(ibdev,
"failed to poll cqe for free mr, remain %d cqe.\n",
cqe_cnt);
goto out;
}
if (time_after(jiffies, end)) {
ibdev_err(ibdev,
"failed to poll cqe for free mr and timeout, remain %d cqe.\n",
cqe_cnt);
goto out;
}
cqe_cnt -= npolled;
}
out:
mutex_unlock(&free_mr->mutex);
}
static void hns_roce_v2_dereg_mr(struct hns_roce_dev *hr_dev)
{
if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08)
free_mr_send_cmd_to_hw(hr_dev);
}
static void *get_cqe_v2(struct hns_roce_cq *hr_cq, int n)
{
return hns_roce_buf_offset(hr_cq->mtr.kmem, n * hr_cq->cqe_size);
@@ -3818,38 +4097,38 @@ out:
}
static int get_op_for_set_hem(struct hns_roce_dev *hr_dev, u32 type,
int step_idx, u16 *mbox_op)
u32 step_idx, u8 *mbox_cmd)
{
u16 op;
u8 cmd;
switch (type) {
case HEM_TYPE_QPC:
op = HNS_ROCE_CMD_WRITE_QPC_BT0;
cmd = HNS_ROCE_CMD_WRITE_QPC_BT0;
break;
case HEM_TYPE_MTPT:
op = HNS_ROCE_CMD_WRITE_MPT_BT0;
cmd = HNS_ROCE_CMD_WRITE_MPT_BT0;
break;
case HEM_TYPE_CQC:
op = HNS_ROCE_CMD_WRITE_CQC_BT0;
cmd = HNS_ROCE_CMD_WRITE_CQC_BT0;
break;
case HEM_TYPE_SRQC:
op = HNS_ROCE_CMD_WRITE_SRQC_BT0;
cmd = HNS_ROCE_CMD_WRITE_SRQC_BT0;
break;
case HEM_TYPE_SCCC:
op = HNS_ROCE_CMD_WRITE_SCCC_BT0;
cmd = HNS_ROCE_CMD_WRITE_SCCC_BT0;
break;
case HEM_TYPE_QPC_TIMER:
op = HNS_ROCE_CMD_WRITE_QPC_TIMER_BT0;
cmd = HNS_ROCE_CMD_WRITE_QPC_TIMER_BT0;
break;
case HEM_TYPE_CQC_TIMER:
op = HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0;
cmd = HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0;
break;
default:
dev_warn(hr_dev->dev, "failed to check hem type %u.\n", type);
return -EINVAL;
}
*mbox_op = op + step_idx;
*mbox_cmd = cmd + step_idx;
return 0;
}
@@ -3872,10 +4151,10 @@ static int config_gmv_ba_to_hw(struct hns_roce_dev *hr_dev, unsigned long obj,
}
static int set_hem_to_hw(struct hns_roce_dev *hr_dev, int obj,
dma_addr_t base_addr, u32 hem_type, int step_idx)
dma_addr_t base_addr, u32 hem_type, u32 step_idx)
{
int ret;
u16 op;
u8 cmd;
if (unlikely(hem_type == HEM_TYPE_GMV))
return config_gmv_ba_to_hw(hr_dev, obj, base_addr);
@@ -3883,16 +4162,16 @@ static int set_hem_to_hw(struct hns_roce_dev *hr_dev, int obj,
if (unlikely(hem_type == HEM_TYPE_SCCC && step_idx))
return 0;
ret = get_op_for_set_hem(hr_dev, hem_type, step_idx, &op);
ret = get_op_for_set_hem(hr_dev, hem_type, step_idx, &cmd);
if (ret < 0)
return ret;
return config_hem_ba_to_hw(hr_dev, obj, base_addr, op);
return config_hem_ba_to_hw(hr_dev, base_addr, cmd, obj);
}
static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev,
struct hns_roce_hem_table *table, int obj,
int step_idx)
u32 step_idx)
{
struct hns_roce_hem_iter iter;
struct hns_roce_hem_mhop mhop;
@@ -3950,29 +4229,29 @@ static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev,
}
static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev,
struct hns_roce_hem_table *table, int obj,
int step_idx)
struct hns_roce_hem_table *table,
int tag, u32 step_idx)
{
struct device *dev = hr_dev->dev;
struct hns_roce_cmd_mailbox *mailbox;
struct device *dev = hr_dev->dev;
u8 cmd = 0xff;
int ret;
u16 op = 0xff;
if (!hns_roce_check_whether_mhop(hr_dev, table->type))
return 0;
switch (table->type) {
case HEM_TYPE_QPC:
op = HNS_ROCE_CMD_DESTROY_QPC_BT0;
cmd = HNS_ROCE_CMD_DESTROY_QPC_BT0;
break;
case HEM_TYPE_MTPT:
op = HNS_ROCE_CMD_DESTROY_MPT_BT0;
cmd = HNS_ROCE_CMD_DESTROY_MPT_BT0;
break;
case HEM_TYPE_CQC:
op = HNS_ROCE_CMD_DESTROY_CQC_BT0;
cmd = HNS_ROCE_CMD_DESTROY_CQC_BT0;
break;
case HEM_TYPE_SRQC:
op = HNS_ROCE_CMD_DESTROY_SRQC_BT0;
cmd = HNS_ROCE_CMD_DESTROY_SRQC_BT0;
break;
case HEM_TYPE_SCCC:
case HEM_TYPE_QPC_TIMER:
@@ -3985,15 +4264,13 @@ static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev,
return 0;
}
op += step_idx;
cmd += step_idx;
mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
if (IS_ERR(mailbox))
return PTR_ERR(mailbox);
/* configure the tag and op */
ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, obj, 0, op,
HNS_ROCE_CMD_TIMEOUT_MSECS);
ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, cmd, tag);
hns_roce_free_cmd_mailbox(hr_dev, mailbox);
return ret;
@@ -4017,9 +4294,8 @@ static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev,
memcpy(mailbox->buf, context, qpc_size);
memcpy(mailbox->buf + qpc_size, qpc_mask, qpc_size);
ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0,
HNS_ROCE_CMD_MODIFY_QPC,
HNS_ROCE_CMD_TIMEOUT_MSECS);
ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0,
HNS_ROCE_CMD_MODIFY_QPC, hr_qp->qpn);
hns_roce_free_cmd_mailbox(hr_dev, mailbox);
@@ -4667,6 +4943,18 @@ static int hns_roce_v2_set_path(struct ib_qp *ibqp,
u8 hr_port;
int ret;
/*
* If free_mr_en of qp is set, it means that this qp comes from
* free mr. This qp will perform the loopback operation.
* In the loopback scenario, only sl needs to be set.
*/
if (hr_qp->free_mr_en) {
hr_reg_write(context, QPC_SL, rdma_ah_get_sl(&attr->ah_attr));
hr_reg_clear(qpc_mask, QPC_SL);
hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr);
return 0;
}
ib_port = (attr_mask & IB_QP_PORT) ? attr->port_num : hr_qp->port + 1;
hr_port = ib_port - 1;
is_roce_protocol = rdma_cap_eth_ah(&hr_dev->ib_dev, ib_port) &&
@@ -5092,9 +5380,8 @@ static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev,
if (IS_ERR(mailbox))
return PTR_ERR(mailbox);
ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, hr_qp->qpn, 0,
HNS_ROCE_CMD_QUERY_QPC,
HNS_ROCE_CMD_TIMEOUT_MSECS);
ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_QPC,
hr_qp->qpn);
if (ret)
goto out;
@@ -5460,9 +5747,8 @@ static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq,
hr_reg_write(srq_context, SRQC_LIMIT_WL, srq_attr->srq_limit);
hr_reg_clear(srqc_mask, SRQC_LIMIT_WL);
ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, srq->srqn, 0,
HNS_ROCE_CMD_MODIFY_SRQC,
HNS_ROCE_CMD_TIMEOUT_MSECS);
ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0,
HNS_ROCE_CMD_MODIFY_SRQC, srq->srqn);
hns_roce_free_cmd_mailbox(hr_dev, mailbox);
if (ret) {
ibdev_err(&hr_dev->ib_dev,
@@ -5488,9 +5774,8 @@ static int hns_roce_v2_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
return PTR_ERR(mailbox);
srq_context = mailbox->buf;
ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, srq->srqn, 0,
HNS_ROCE_CMD_QUERY_SRQC,
HNS_ROCE_CMD_TIMEOUT_MSECS);
ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma,
HNS_ROCE_CMD_QUERY_SRQC, srq->srqn);
if (ret) {
ibdev_err(&hr_dev->ib_dev,
"failed to process cmd of querying SRQ, ret = %d.\n",
@@ -5540,9 +5825,8 @@ static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
hr_reg_write(cq_context, CQC_CQ_PERIOD, cq_period);
hr_reg_clear(cqc_mask, CQC_CQ_PERIOD);
ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_cq->cqn, 1,
HNS_ROCE_CMD_MODIFY_CQC,
HNS_ROCE_CMD_TIMEOUT_MSECS);
ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0,
HNS_ROCE_CMD_MODIFY_CQC, hr_cq->cqn);
hns_roce_free_cmd_mailbox(hr_dev, mailbox);
if (ret)
ibdev_err(&hr_dev->ib_dev,
@@ -5869,15 +6153,14 @@ static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev, u32 eqn)
{
struct device *dev = hr_dev->dev;
int ret;
u8 cmd;
if (eqn < hr_dev->caps.num_comp_vectors)
ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M,
0, HNS_ROCE_CMD_DESTROY_CEQC,
HNS_ROCE_CMD_TIMEOUT_MSECS);
cmd = HNS_ROCE_CMD_DESTROY_CEQC;
else
ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M,
0, HNS_ROCE_CMD_DESTROY_AEQC,
HNS_ROCE_CMD_TIMEOUT_MSECS);
cmd = HNS_ROCE_CMD_DESTROY_AEQC;
ret = hns_roce_destroy_hw_ctx(hr_dev, cmd, eqn & HNS_ROCE_V2_EQN_M);
if (ret)
dev_err(dev, "[mailbox cmd] destroy eqc(%u) failed.\n", eqn);
}
@@ -5983,16 +6266,15 @@ static int alloc_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
}
static int hns_roce_v2_create_eq(struct hns_roce_dev *hr_dev,
struct hns_roce_eq *eq,
unsigned int eq_cmd)
struct hns_roce_eq *eq, u8 eq_cmd)
{
struct hns_roce_cmd_mailbox *mailbox;
int ret;
/* Allocate mailbox memory */
mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
if (IS_ERR_OR_NULL(mailbox))
return -ENOMEM;
if (IS_ERR(mailbox))
return PTR_ERR(mailbox);
ret = alloc_eq_buf(hr_dev, eq);
if (ret)
@@ -6002,8 +6284,7 @@ static int hns_roce_v2_create_eq(struct hns_roce_dev *hr_dev,
if (ret)
goto err_cmd_mbox;
ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, eq->eqn, 0,
eq_cmd, HNS_ROCE_CMD_TIMEOUT_MSECS);
ret = hns_roce_create_hw_ctx(hr_dev, mailbox, eq_cmd, eq->eqn);
if (ret) {
dev_err(hr_dev->dev, "[mailbox cmd] create eqc failed.\n");
goto err_cmd_mbox;
@@ -6114,14 +6395,14 @@ static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev)
struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
struct device *dev = hr_dev->dev;
struct hns_roce_eq *eq;
unsigned int eq_cmd;
int irq_num;
int eq_num;
int other_num;
int comp_num;
int aeq_num;
int i;
int irq_num;
int eq_num;
u8 eq_cmd;
int ret;
int i;
other_num = hr_dev->caps.num_other_vectors;
comp_num = hr_dev->caps.num_comp_vectors;
@@ -6258,6 +6539,7 @@ static const struct hns_roce_hw hns_roce_hw_v2 = {
.set_hem = hns_roce_v2_set_hem,
.clear_hem = hns_roce_v2_clear_hem,
.modify_qp = hns_roce_v2_modify_qp,
.dereg_mr = hns_roce_v2_dereg_mr,
.qp_flow_control_init = hns_roce_v2_qp_flow_control_init,
.init_eq = hns_roce_v2_init_eq_table,
.cleanup_eq = hns_roce_v2_cleanup_eq_table,
@@ -6339,14 +6621,25 @@ static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
ret = hns_roce_init(hr_dev);
if (ret) {
dev_err(hr_dev->dev, "RoCE Engine init failed!\n");
goto error_failed_get_cfg;
goto error_failed_cfg;
}
if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {
ret = free_mr_init(hr_dev);
if (ret) {
dev_err(hr_dev->dev, "failed to init free mr!\n");
goto error_failed_roce_init;
}
}
handle->priv = hr_dev;
return 0;
error_failed_get_cfg:
error_failed_roce_init:
hns_roce_exit(hr_dev);
error_failed_cfg:
kfree(hr_dev->priv);
error_failed_kzalloc:
@@ -6368,6 +6661,9 @@ static void __hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
hr_dev->state = HNS_ROCE_DEVICE_STATE_UNINIT;
hns_roce_handle_device_err(hr_dev);
if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08)
free_mr_exit(hr_dev);
hns_roce_exit(hr_dev);
kfree(hr_dev->priv);
ib_dealloc_device(&hr_dev->ib_dev);

20
drivers/infiniband/hw/hns/hns_roce_hw_v2.h
View File

@@ -139,6 +139,18 @@ enum {
#define CMD_CSQ_DESC_NUM 1024
#define CMD_CRQ_DESC_NUM 1024
/* Free mr used parameters */
#define HNS_ROCE_FREE_MR_USED_CQE_NUM 128
#define HNS_ROCE_FREE_MR_USED_QP_NUM 0x8
#define HNS_ROCE_FREE_MR_USED_PSN 0x0808
#define HNS_ROCE_FREE_MR_USED_QP_RETRY_CNT 0x7
#define HNS_ROCE_FREE_MR_USED_QP_TIMEOUT 0x12
#define HNS_ROCE_FREE_MR_USED_SQWQE_NUM 128
#define HNS_ROCE_FREE_MR_USED_SQSGE_NUM 0x2
#define HNS_ROCE_FREE_MR_USED_RQWQE_NUM 128
#define HNS_ROCE_FREE_MR_USED_RQSGE_NUM 0x2
#define HNS_ROCE_V2_FREE_MR_TIMEOUT 4500
enum {
NO_ARMED = 0x0,
REG_NXT_CEQE = 0x2,
@@ -1418,10 +1430,18 @@ struct hns_roce_link_table {
#define HNS_ROCE_EXT_LLM_ENTRY(addr, id) (((id) << (64 - 12)) | ((addr) >> 12))
#define HNS_ROCE_EXT_LLM_MIN_PAGES(que_num) ((que_num) * 4 + 2)
struct hns_roce_v2_free_mr {
struct ib_qp *rsv_qp[HNS_ROCE_FREE_MR_USED_QP_NUM];
struct ib_cq *rsv_cq;
struct ib_pd *rsv_pd;
struct mutex mutex;
};
struct hns_roce_v2_priv {
struct hnae3_handle *handle;
struct hns_roce_v2_cmq cmq;
struct hns_roce_link_table ext_llm;
struct hns_roce_v2_free_mr free_mr;
};
struct hns_roce_dip {

5
drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c
View File

@@ -18,9 +18,8 @@ int hns_roce_v2_query_cqc_info(struct hns_roce_dev *hr_dev, u32 cqn,
return PTR_ERR(mailbox);
cq_context = mailbox->buf;
ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, cqn, 0,
HNS_ROCE_CMD_QUERY_CQC,
HNS_ROCE_CMD_TIMEOUT_MSECS);
ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_CQC,
cqn);
if (ret) {
dev_err(hr_dev->dev, "QUERY cqc cmd process error\n");
goto err_mailbox;

50
drivers/infiniband/hw/hns/hns_roce_mr.c
View File

@@ -47,24 +47,6 @@ unsigned long key_to_hw_index(u32 key)
return (key << 24) | (key >> 8);
}
static int hns_roce_hw_create_mpt(struct hns_roce_dev *hr_dev,
struct hns_roce_cmd_mailbox *mailbox,
unsigned long mpt_index)
{
return hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, mpt_index, 0,
HNS_ROCE_CMD_CREATE_MPT,
HNS_ROCE_CMD_TIMEOUT_MSECS);
}
int hns_roce_hw_destroy_mpt(struct hns_roce_dev *hr_dev,
struct hns_roce_cmd_mailbox *mailbox,
unsigned long mpt_index)
{
return hns_roce_cmd_mbox(hr_dev, 0, mailbox ? mailbox->dma : 0,
mpt_index, !mailbox, HNS_ROCE_CMD_DESTROY_MPT,
HNS_ROCE_CMD_TIMEOUT_MSECS);
}
static int alloc_mr_key(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr)
{
struct hns_roce_ida *mtpt_ida = &hr_dev->mr_table.mtpt_ida;
@@ -137,14 +119,13 @@ static void free_mr_pbl(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr)
hns_roce_mtr_destroy(hr_dev, &mr->pbl_mtr);
}
static void hns_roce_mr_free(struct hns_roce_dev *hr_dev,
struct hns_roce_mr *mr)
static void hns_roce_mr_free(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr)
{
struct ib_device *ibdev = &hr_dev->ib_dev;
int ret;
if (mr->enabled) {
ret = hns_roce_hw_destroy_mpt(hr_dev, NULL,
ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_MPT,
key_to_hw_index(mr->key) &
(hr_dev->caps.num_mtpts - 1));
if (ret)
@@ -166,10 +147,8 @@ static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev,
/* Allocate mailbox memory */
mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
if (IS_ERR(mailbox)) {
ret = PTR_ERR(mailbox);
return ret;
}
if (IS_ERR(mailbox))
return PTR_ERR(mailbox);
if (mr->type != MR_TYPE_FRMR)
ret = hr_dev->hw->write_mtpt(hr_dev, mailbox->buf, mr);
@@ -180,7 +159,7 @@ static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev,
goto err_page;
}
ret = hns_roce_hw_create_mpt(hr_dev, mailbox,
ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_MPT,
mtpt_idx & (hr_dev->caps.num_mtpts - 1));
if (ret) {
dev_err(dev, "failed to create mpt, ret = %d.\n", ret);
@@ -303,13 +282,14 @@ struct ib_mr *hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start,
return ERR_CAST(mailbox);
mtpt_idx = key_to_hw_index(mr->key) & (hr_dev->caps.num_mtpts - 1);
ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, mtpt_idx, 0,
HNS_ROCE_CMD_QUERY_MPT,
HNS_ROCE_CMD_TIMEOUT_MSECS);
ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_MPT,
mtpt_idx);
if (ret)
goto free_cmd_mbox;
ret = hns_roce_hw_destroy_mpt(hr_dev, NULL, mtpt_idx);
ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_MPT,
mtpt_idx);
if (ret)
ibdev_warn(ib_dev, "failed to destroy MPT, ret = %d.\n", ret);
@@ -339,7 +319,8 @@ struct ib_mr *hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start,
goto free_cmd_mbox;
}
ret = hns_roce_hw_create_mpt(hr_dev, mailbox, mtpt_idx);
ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_MPT,
mtpt_idx);
if (ret) {
ibdev_err(ib_dev, "failed to create MPT, ret = %d.\n", ret);
goto free_cmd_mbox;
@@ -361,6 +342,9 @@ int hns_roce_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
struct hns_roce_mr *mr = to_hr_mr(ibmr);
int ret = 0;
if (hr_dev->hw->dereg_mr)
hr_dev->hw->dereg_mr(hr_dev);
hns_roce_mr_free(hr_dev, mr);
kfree(mr);
@@ -480,7 +464,7 @@ static void hns_roce_mw_free(struct hns_roce_dev *hr_dev,
int ret;
if (mw->enabled) {
ret = hns_roce_hw_destroy_mpt(hr_dev, NULL,
ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_MPT,
key_to_hw_index(mw->rkey) &
(hr_dev->caps.num_mtpts - 1));
if (ret)
@@ -520,7 +504,7 @@ static int hns_roce_mw_enable(struct hns_roce_dev *hr_dev,
goto err_page;
}
ret = hns_roce_hw_create_mpt(hr_dev, mailbox,
ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_MPT,
mtpt_idx & (hr_dev->caps.num_mtpts - 1));
if (ret) {
dev_err(dev, "MW CREATE_MPT failed (%d)\n", ret);

110
drivers/infiniband/hw/hns/hns_roce_srq.c
View File

@@ -59,58 +59,39 @@ static void hns_roce_ib_srq_event(struct hns_roce_srq *srq,
}
}
static int hns_roce_hw_create_srq(struct hns_roce_dev *dev,
struct hns_roce_cmd_mailbox *mailbox,
unsigned long srq_num)
static int alloc_srqn(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)
{
return hns_roce_cmd_mbox(dev, mailbox->dma, 0, srq_num, 0,
HNS_ROCE_CMD_CREATE_SRQ,
HNS_ROCE_CMD_TIMEOUT_MSECS);
}
static int hns_roce_hw_destroy_srq(struct hns_roce_dev *dev,
struct hns_roce_cmd_mailbox *mailbox,
unsigned long srq_num)
{
return hns_roce_cmd_mbox(dev, 0, mailbox ? mailbox->dma : 0, srq_num,
mailbox ? 0 : 1, HNS_ROCE_CMD_DESTROY_SRQ,
HNS_ROCE_CMD_TIMEOUT_MSECS);
}
static int alloc_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)
{
struct hns_roce_srq_table *srq_table = &hr_dev->srq_table;
struct hns_roce_ida *srq_ida = &hr_dev->srq_table.srq_ida;
struct ib_device *ibdev = &hr_dev->ib_dev;
struct hns_roce_cmd_mailbox *mailbox;
int ret;
int id;
id = ida_alloc_range(&srq_ida->ida, srq_ida->min, srq_ida->max,
GFP_KERNEL);
if (id < 0) {
ibdev_err(ibdev, "failed to alloc srq(%d).\n", id);
ibdev_err(&hr_dev->ib_dev, "failed to alloc srq(%d).\n", id);
return -ENOMEM;
}
srq->srqn = (unsigned long)id;
ret = hns_roce_table_get(hr_dev, &srq_table->table, srq->srqn);
if (ret) {
ibdev_err(ibdev, "failed to get SRQC table, ret = %d.\n", ret);
goto err_out;
}
srq->srqn = id;
ret = xa_err(xa_store(&srq_table->xa, srq->srqn, srq, GFP_KERNEL));
if (ret) {
ibdev_err(ibdev, "failed to store SRQC, ret = %d.\n", ret);
goto err_put;
}
return 0;
}
static void free_srqn(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)
{
ida_free(&hr_dev->srq_table.srq_ida.ida, (int)srq->srqn);
}
static int hns_roce_create_srqc(struct hns_roce_dev *hr_dev,
struct hns_roce_srq *srq)
{
struct ib_device *ibdev = &hr_dev->ib_dev;
struct hns_roce_cmd_mailbox *mailbox;
int ret;
mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
if (IS_ERR_OR_NULL(mailbox)) {
if (IS_ERR(mailbox)) {
ibdev_err(ibdev, "failed to alloc mailbox for SRQC.\n");
ret = -ENOMEM;
goto err_xa;
return PTR_ERR(mailbox);
}
ret = hr_dev->hw->write_srqc(srq, mailbox->buf);
@@ -119,24 +100,44 @@ static int alloc_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)
goto err_mbox;
}
ret = hns_roce_hw_create_srq(hr_dev, mailbox, srq->srqn);
if (ret) {
ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_SRQ,
srq->srqn);
if (ret)
ibdev_err(ibdev, "failed to config SRQC, ret = %d.\n", ret);
goto err_mbox;
}
hns_roce_free_cmd_mailbox(hr_dev, mailbox);
return 0;
err_mbox:
hns_roce_free_cmd_mailbox(hr_dev, mailbox);
return ret;
}
static int alloc_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)
{
struct hns_roce_srq_table *srq_table = &hr_dev->srq_table;
struct ib_device *ibdev = &hr_dev->ib_dev;
int ret;
ret = hns_roce_table_get(hr_dev, &srq_table->table, srq->srqn);
if (ret) {
ibdev_err(ibdev, "failed to get SRQC table, ret = %d.\n", ret);
return ret;
}
ret = xa_err(xa_store(&srq_table->xa, srq->srqn, srq, GFP_KERNEL));
if (ret) {
ibdev_err(ibdev, "failed to store SRQC, ret = %d.\n", ret);
goto err_put;
}
ret = hns_roce_create_srqc(hr_dev, srq);
if (ret)
goto err_xa;
return 0;
err_xa:
xa_erase(&srq_table->xa, srq->srqn);
err_put:
hns_roce_table_put(hr_dev, &srq_table->table, srq->srqn);
err_out:
ida_free(&srq_ida->ida, id);
return ret;
}
@@ -146,7 +147,8 @@ static void free_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)
struct hns_roce_srq_table *srq_table = &hr_dev->srq_table;
int ret;
ret = hns_roce_hw_destroy_srq(hr_dev, NULL, srq->srqn);
ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_SRQ,
srq->srqn);
if (ret)
dev_err(hr_dev->dev, "DESTROY_SRQ failed (%d) for SRQN %06lx\n",
ret, srq->srqn);
@@ -158,7 +160,6 @@ static void free_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)
wait_for_completion(&srq->free);
hns_roce_table_put(hr_dev, &srq_table->table, srq->srqn);
ida_free(&srq_table->srq_ida.ida, (int)srq->srqn);
}
static int alloc_srq_idx(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq,
@@ -406,10 +407,14 @@ int hns_roce_create_srq(struct ib_srq *ib_srq,
if (ret)
return ret;
ret = alloc_srqc(hr_dev, srq);
ret = alloc_srqn(hr_dev, srq);
if (ret)
goto err_srq_buf;
ret = alloc_srqc(hr_dev, srq);
if (ret)
goto err_srqn;
if (udata) {
resp.srqn = srq->srqn;
if (ib_copy_to_udata(udata, &resp,
@@ -428,6 +433,8 @@ int hns_roce_create_srq(struct ib_srq *ib_srq,
err_srqc:
free_srqc(hr_dev, srq);
err_srqn:
free_srqn(hr_dev, srq);
err_srq_buf:
free_srq_buf(hr_dev, srq);
@@ -440,6 +447,7 @@ int hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
struct hns_roce_srq *srq = to_hr_srq(ibsrq);
free_srqc(hr_dev, srq);
free_srqn(hr_dev, srq);
free_srq_buf(hr_dev, srq);
return 0;
}

68
drivers/infiniband/hw/irdma/cm.c
View File

@@ -1501,15 +1501,14 @@ irdma_find_listener(struct irdma_cm_core *cm_core, u32 *dst_addr, u16 dst_port,
* @cm_info: CM info for parent listen node
* @cm_parent_listen_node: The parent listen node
*/
static enum irdma_status_code
irdma_del_multiple_qhash(struct irdma_device *iwdev,
struct irdma_cm_info *cm_info,
struct irdma_cm_listener *cm_parent_listen_node)
static int irdma_del_multiple_qhash(struct irdma_device *iwdev,
struct irdma_cm_info *cm_info,
struct irdma_cm_listener *cm_parent_listen_node)
{
struct irdma_cm_listener *child_listen_node;
enum irdma_status_code ret = IRDMA_ERR_CFG;
struct list_head *pos, *tpos;
unsigned long flags;
int ret = -EINVAL;
spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags);
list_for_each_safe (pos, tpos,
@@ -1618,16 +1617,16 @@ u16 irdma_get_vlan_ipv4(u32 *addr)
* Adds a qhash and a child listen node for every IPv6 address
* on the adapter and adds the associated qhash filter
*/
static enum irdma_status_code
irdma_add_mqh_6(struct irdma_device *iwdev, struct irdma_cm_info *cm_info,
struct irdma_cm_listener *cm_parent_listen_node)
static int irdma_add_mqh_6(struct irdma_device *iwdev,
struct irdma_cm_info *cm_info,
struct irdma_cm_listener *cm_parent_listen_node)
{
struct net_device *ip_dev;
struct inet6_dev *idev;
struct inet6_ifaddr *ifp, *tmp;
enum irdma_status_code ret = 0;
struct irdma_cm_listener *child_listen_node;
unsigned long flags;
int ret = 0;
rtnl_lock();
for_each_netdev(&init_net, ip_dev) {
@@ -1653,7 +1652,7 @@ irdma_add_mqh_6(struct irdma_device *iwdev, struct irdma_cm_info *cm_info,
child_listen_node);
if (!child_listen_node) {
ibdev_dbg(&iwdev->ibdev, "CM: listener memory allocation\n");
ret = IRDMA_ERR_NO_MEMORY;
ret = -ENOMEM;
goto exit;
}
@@ -1700,16 +1699,16 @@ exit:
* Adds a qhash and a child listen node for every IPv4 address
* on the adapter and adds the associated qhash filter
*/
static enum irdma_status_code
irdma_add_mqh_4(struct irdma_device *iwdev, struct irdma_cm_info *cm_info,
struct irdma_cm_listener *cm_parent_listen_node)
static int irdma_add_mqh_4(struct irdma_device *iwdev,
struct irdma_cm_info *cm_info,
struct irdma_cm_listener *cm_parent_listen_node)
{
struct net_device *ip_dev;
struct in_device *idev;
struct irdma_cm_listener *child_listen_node;
enum irdma_status_code ret = 0;
unsigned long flags;
const struct in_ifaddr *ifa;
int ret = 0;
rtnl_lock();
for_each_netdev(&init_net, ip_dev) {
@@ -1734,7 +1733,7 @@ irdma_add_mqh_4(struct irdma_device *iwdev, struct irdma_cm_info *cm_info,
if (!child_listen_node) {
ibdev_dbg(&iwdev->ibdev, "CM: listener memory allocation\n");
in_dev_put(idev);
ret = IRDMA_ERR_NO_MEMORY;
ret = -ENOMEM;
goto exit;
}
@@ -1781,9 +1780,9 @@ exit:
* @cm_info: CM info for parent listen node
* @cm_listen_node: The parent listen node
*/
static enum irdma_status_code
irdma_add_mqh(struct irdma_device *iwdev, struct irdma_cm_info *cm_info,
struct irdma_cm_listener *cm_listen_node)
static int irdma_add_mqh(struct irdma_device *iwdev,
struct irdma_cm_info *cm_info,
struct irdma_cm_listener *cm_listen_node)
{
if (cm_info->ipv4)
return irdma_add_mqh_4(iwdev, cm_info, cm_listen_node);
@@ -2200,7 +2199,7 @@ irdma_make_cm_node(struct irdma_cm_core *cm_core, struct irdma_device *iwdev,
/* set our node specific transport info */
cm_node->ipv4 = cm_info->ipv4;
cm_node->vlan_id = cm_info->vlan_id;
if (cm_node->vlan_id >= VLAN_N_VID && iwdev->dcb)
if (cm_node->vlan_id >= VLAN_N_VID && iwdev->dcb_vlan_mode)
cm_node->vlan_id = 0;
cm_node->tos = cm_info->tos;
cm_node->user_pri = cm_info->user_pri;
@@ -2209,8 +2208,12 @@ irdma_make_cm_node(struct irdma_cm_core *cm_core, struct irdma_device *iwdev,
ibdev_warn(&iwdev->ibdev,
"application TOS[%d] and remote client TOS[%d] mismatch\n",
listener->tos, cm_info->tos);
cm_node->tos = max(listener->tos, cm_info->tos);
cm_node->user_pri = rt_tos2priority(cm_node->tos);
if (iwdev->vsi.dscp_mode) {
cm_node->user_pri = listener->user_pri;
} else {
cm_node->tos = max(listener->tos, cm_info->tos);
cm_node->user_pri = rt_tos2priority(cm_node->tos);
}
ibdev_dbg(&iwdev->ibdev,
"DCB: listener: TOS:[%d] UP:[%d]\n", cm_node->tos,
cm_node->user_pri);
@@ -3201,8 +3204,7 @@ static void irdma_cm_free_ah_nop(struct irdma_cm_node *cm_node)
* @iwdev: iwarp device structure
* @rdma_ver: HW version
*/
enum irdma_status_code irdma_setup_cm_core(struct irdma_device *iwdev,
u8 rdma_ver)
int irdma_setup_cm_core(struct irdma_device *iwdev, u8 rdma_ver)
{
struct irdma_cm_core *cm_core = &iwdev->cm_core;
@@ -3212,7 +3214,7 @@ enum irdma_status_code irdma_setup_cm_core(struct irdma_device *iwdev,
/* Handles CM event work items send to Iwarp core */
cm_core->event_wq = alloc_ordered_workqueue("iwarp-event-wq", 0);
if (!cm_core->event_wq)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
INIT_LIST_HEAD(&cm_core->listen_list);
@@ -3835,7 +3837,11 @@ int irdma_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
cm_info.cm_id = cm_id;
cm_info.qh_qpid = iwdev->vsi.ilq->qp_id;
cm_info.tos = cm_id->tos;
cm_info.user_pri = rt_tos2priority(cm_id->tos);
if (iwdev->vsi.dscp_mode)
cm_info.user_pri =
iwqp->sc_qp.vsi->dscp_map[irdma_tos2dscp(cm_info.tos)];
else
cm_info.user_pri = rt_tos2priority(cm_id->tos);
if (iwqp->sc_qp.dev->ws_add(iwqp->sc_qp.vsi, cm_info.user_pri))
return -ENOMEM;
@@ -3915,10 +3921,10 @@ int irdma_create_listen(struct iw_cm_id *cm_id, int backlog)
struct irdma_device *iwdev;
struct irdma_cm_listener *cm_listen_node;
struct irdma_cm_info cm_info = {};
enum irdma_status_code err;
struct sockaddr_in *laddr;
struct sockaddr_in6 *laddr6;
bool wildcard = false;
int err;
iwdev = to_iwdev(cm_id->device);
if (!iwdev)
@@ -3959,7 +3965,7 @@ int irdma_create_listen(struct iw_cm_id *cm_id, int backlog)
}
}
if (cm_info.vlan_id >= VLAN_N_VID && iwdev->dcb)
if (cm_info.vlan_id >= VLAN_N_VID && iwdev->dcb_vlan_mode)
cm_info.vlan_id = 0;
cm_info.backlog = backlog;
cm_info.cm_id = cm_id;
@@ -3977,7 +3983,11 @@ int irdma_create_listen(struct iw_cm_id *cm_id, int backlog)
cm_id->provider_data = cm_listen_node;
cm_listen_node->tos = cm_id->tos;
cm_listen_node->user_pri = rt_tos2priority(cm_id->tos);
if (iwdev->vsi.dscp_mode)
cm_listen_node->user_pri =
iwdev->vsi.dscp_map[irdma_tos2dscp(cm_id->tos)];
else
cm_listen_node->user_pri = rt_tos2priority(cm_id->tos);
cm_info.user_pri = cm_listen_node->user_pri;
if (!cm_listen_node->reused_node) {
if (wildcard) {
@@ -4325,11 +4335,11 @@ static void irdma_qhash_ctrl(struct irdma_device *iwdev,
struct list_head *child_listen_list = &parent_listen_node->child_listen_list;
struct irdma_cm_listener *child_listen_node;
struct list_head *pos, *tpos;
enum irdma_status_code err;
bool node_allocated = false;
enum irdma_quad_hash_manage_type op = ifup ?
IRDMA_QHASH_MANAGE_TYPE_ADD :
IRDMA_QHASH_MANAGE_TYPE_DELETE;
int err;
list_for_each_safe (pos, tpos, child_listen_list) {
child_listen_node = list_entry(pos, struct irdma_cm_listener,

7
drivers/infiniband/hw/irdma/cm.h
View File

@@ -384,6 +384,13 @@ int irdma_schedule_cm_timer(struct irdma_cm_node *cm_node,
struct irdma_puda_buf *sqbuf,
enum irdma_timer_type type, int send_retrans,
int close_when_complete);
static inline u8 irdma_tos2dscp(u8 tos)
{
#define IRDMA_DSCP_VAL GENMASK(7, 2)
return (u8)FIELD_GET(IRDMA_DSCP_VAL, tos);
}
int irdma_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);
int irdma_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len);
int irdma_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);

602
drivers/infiniband/hw/irdma/ctrl.c
View File

File diff suppressed because it is too large Load Diff

8
drivers/infiniband/hw/irdma/defs.h
View File

@@ -964,7 +964,7 @@ enum irdma_cqp_op_type {
(_ring).head = ((_ring).head + 1) % size; \
(_retcode) = 0; \
} else { \
(_retcode) = IRDMA_ERR_RING_FULL; \
(_retcode) = -ENOMEM; \
} \
}
#define IRDMA_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \
@@ -975,7 +975,7 @@ enum irdma_cqp_op_type {
(_ring).head = ((_ring).head + (_count)) % size; \
(_retcode) = 0; \
} else { \
(_retcode) = IRDMA_ERR_RING_FULL; \
(_retcode) = -ENOMEM; \
} \
}
#define IRDMA_SQ_RING_MOVE_HEAD(_ring, _retcode) \
@@ -986,7 +986,7 @@ enum irdma_cqp_op_type {
(_ring).head = ((_ring).head + 1) % size; \
(_retcode) = 0; \
} else { \
(_retcode) = IRDMA_ERR_RING_FULL; \
(_retcode) = -ENOMEM; \
} \
}
#define IRDMA_SQ_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \
@@ -997,7 +997,7 @@ enum irdma_cqp_op_type {
(_ring).head = ((_ring).head + (_count)) % size; \
(_retcode) = 0; \
} else { \
(_retcode) = IRDMA_ERR_RING_FULL; \
(_retcode) = -ENOMEM; \
} \
}
#define IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(_ring, _count) \

105
drivers/infiniband/hw/irdma/hmc.c
View File

@@ -1,7 +1,6 @@
// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
/* Copyright (c) 2015 - 2021 Intel Corporation */
#include "osdep.h"
#include "status.h"
#include "hmc.h"
#include "defs.h"
#include "type.h"
@@ -121,10 +120,8 @@ static inline void irdma_invalidate_pf_hmc_pd(struct irdma_sc_dev *dev, u32 sd_i
* @type: paged or direct sd
* @setsd: flag to set or clear sd
*/
enum irdma_status_code irdma_hmc_sd_one(struct irdma_sc_dev *dev, u8 hmc_fn_id,
u64 pa, u32 sd_idx,
enum irdma_sd_entry_type type,
bool setsd)
int irdma_hmc_sd_one(struct irdma_sc_dev *dev, u8 hmc_fn_id, u64 pa, u32 sd_idx,
enum irdma_sd_entry_type type, bool setsd)
{
struct irdma_update_sds_info sdinfo;
@@ -145,16 +142,15 @@ enum irdma_status_code irdma_hmc_sd_one(struct irdma_sc_dev *dev, u8 hmc_fn_id,
* @sd_cnt: number of sd entries
* @setsd: flag to set or clear sd
*/
static enum irdma_status_code irdma_hmc_sd_grp(struct irdma_sc_dev *dev,
struct irdma_hmc_info *hmc_info,
u32 sd_index, u32 sd_cnt,
bool setsd)
static int irdma_hmc_sd_grp(struct irdma_sc_dev *dev,
struct irdma_hmc_info *hmc_info, u32 sd_index,
u32 sd_cnt, bool setsd)
{
struct irdma_hmc_sd_entry *sd_entry;
struct irdma_update_sds_info sdinfo = {};
u64 pa;
u32 i;
enum irdma_status_code ret_code = 0;
int ret_code = 0;
sdinfo.hmc_fn_id = hmc_info->hmc_fn_id;
for (i = sd_index; i < sd_index + sd_cnt; i++) {
@@ -196,16 +192,15 @@ static enum irdma_status_code irdma_hmc_sd_grp(struct irdma_sc_dev *dev,
* @dev: pointer to the device structure
* @info: create obj info
*/
static enum irdma_status_code
irdma_hmc_finish_add_sd_reg(struct irdma_sc_dev *dev,
struct irdma_hmc_create_obj_info *info)
static int irdma_hmc_finish_add_sd_reg(struct irdma_sc_dev *dev,
struct irdma_hmc_create_obj_info *info)
{
if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt)
return IRDMA_ERR_INVALID_HMC_OBJ_INDEX;
return -EINVAL;
if ((info->start_idx + info->count) >
info->hmc_info->hmc_obj[info->rsrc_type].cnt)
return IRDMA_ERR_INVALID_HMC_OBJ_COUNT;
return -EINVAL;
if (!info->add_sd_cnt)
return 0;
@@ -222,9 +217,8 @@ irdma_hmc_finish_add_sd_reg(struct irdma_sc_dev *dev,
* This will allocate memory for PDs and backing pages and populate
* the sd and pd entries.
*/
enum irdma_status_code
irdma_sc_create_hmc_obj(struct irdma_sc_dev *dev,
struct irdma_hmc_create_obj_info *info)
int irdma_sc_create_hmc_obj(struct irdma_sc_dev *dev,
struct irdma_hmc_create_obj_info *info)
{
struct irdma_hmc_sd_entry *sd_entry;
u32 sd_idx, sd_lmt;
@@ -232,10 +226,10 @@ irdma_sc_create_hmc_obj(struct irdma_sc_dev *dev,
u32 pd_idx1 = 0, pd_lmt1 = 0;
u32 i, j;
bool pd_error = false;
enum irdma_status_code ret_code = 0;
int ret_code = 0;
if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt)
return IRDMA_ERR_INVALID_HMC_OBJ_INDEX;
return -EINVAL;
if ((info->start_idx + info->count) >
info->hmc_info->hmc_obj[info->rsrc_type].cnt) {
@@ -243,7 +237,7 @@ irdma_sc_create_hmc_obj(struct irdma_sc_dev *dev,
"HMC: error type %u, start = %u, req cnt %u, cnt = %u\n",
info->rsrc_type, info->start_idx, info->count,
info->hmc_info->hmc_obj[info->rsrc_type].cnt);
return IRDMA_ERR_INVALID_HMC_OBJ_COUNT;
return -EINVAL;
}
irdma_find_sd_index_limit(info->hmc_info, info->rsrc_type,
@@ -251,7 +245,7 @@ irdma_sc_create_hmc_obj(struct irdma_sc_dev *dev,
&sd_lmt);
if (sd_idx >= info->hmc_info->sd_table.sd_cnt ||
sd_lmt > info->hmc_info->sd_table.sd_cnt) {
return IRDMA_ERR_INVALID_SD_INDEX;
return -EINVAL;
}
irdma_find_pd_index_limit(info->hmc_info, info->rsrc_type,
@@ -312,7 +306,7 @@ exit_sd_error:
irdma_prep_remove_pd_page(info->hmc_info, (j - 1));
break;
default:
ret_code = IRDMA_ERR_INVALID_SD_TYPE;
ret_code = -EINVAL;
break;
}
j--;
@@ -327,12 +321,12 @@ exit_sd_error:
* @info: dele obj info
* @reset: true if called before reset
*/
static enum irdma_status_code
irdma_finish_del_sd_reg(struct irdma_sc_dev *dev,
struct irdma_hmc_del_obj_info *info, bool reset)
static int irdma_finish_del_sd_reg(struct irdma_sc_dev *dev,
struct irdma_hmc_del_obj_info *info,
bool reset)
{
struct irdma_hmc_sd_entry *sd_entry;
enum irdma_status_code ret_code = 0;
int ret_code = 0;
u32 i, sd_idx;
struct irdma_dma_mem *mem;
@@ -373,22 +367,21 @@ irdma_finish_del_sd_reg(struct irdma_sc_dev *dev,
* caller should deallocate memory allocated previously for
* book-keeping information about PDs and backing storage.
*/
enum irdma_status_code irdma_sc_del_hmc_obj(struct irdma_sc_dev *dev,
struct irdma_hmc_del_obj_info *info,
bool reset)
int irdma_sc_del_hmc_obj(struct irdma_sc_dev *dev,
struct irdma_hmc_del_obj_info *info, bool reset)
{
struct irdma_hmc_pd_table *pd_table;
u32 sd_idx, sd_lmt;
u32 pd_idx, pd_lmt, rel_pd_idx;
u32 i, j;
enum irdma_status_code ret_code = 0;
int ret_code = 0;
if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt) {
ibdev_dbg(to_ibdev(dev),
"HMC: error start_idx[%04d] >= [type %04d].cnt[%04d]\n",
info->start_idx, info->rsrc_type,
info->hmc_info->hmc_obj[info->rsrc_type].cnt);
return IRDMA_ERR_INVALID_HMC_OBJ_INDEX;
return -EINVAL;
}
if ((info->start_idx + info->count) >
@@ -397,7 +390,7 @@ enum irdma_status_code irdma_sc_del_hmc_obj(struct irdma_sc_dev *dev,
"HMC: error start_idx[%04d] + count %04d >= [type %04d].cnt[%04d]\n",
info->start_idx, info->count, info->rsrc_type,
info->hmc_info->hmc_obj[info->rsrc_type].cnt);
return IRDMA_ERR_INVALID_HMC_OBJ_COUNT;
return -EINVAL;
}
irdma_find_pd_index_limit(info->hmc_info, info->rsrc_type,
@@ -433,7 +426,7 @@ enum irdma_status_code irdma_sc_del_hmc_obj(struct irdma_sc_dev *dev,
if (sd_idx >= info->hmc_info->sd_table.sd_cnt ||
sd_lmt > info->hmc_info->sd_table.sd_cnt) {
ibdev_dbg(to_ibdev(dev), "HMC: invalid sd_idx\n");
return IRDMA_ERR_INVALID_SD_INDEX;
return -EINVAL;
}
for (i = sd_idx; i < sd_lmt; i++) {
@@ -477,11 +470,9 @@ enum irdma_status_code irdma_sc_del_hmc_obj(struct irdma_sc_dev *dev,
* @type: what type of segment descriptor we're manipulating
* @direct_mode_sz: size to alloc in direct mode
*/
enum irdma_status_code irdma_add_sd_table_entry(struct irdma_hw *hw,
struct irdma_hmc_info *hmc_info,
u32 sd_index,
enum irdma_sd_entry_type type,
u64 direct_mode_sz)
int irdma_add_sd_table_entry(struct irdma_hw *hw,
struct irdma_hmc_info *hmc_info, u32 sd_index,
enum irdma_sd_entry_type type, u64 direct_mode_sz)
{
struct irdma_hmc_sd_entry *sd_entry;
struct irdma_dma_mem dma_mem;
@@ -499,7 +490,7 @@ enum irdma_status_code irdma_add_sd_table_entry(struct irdma_hw *hw,
dma_mem.va = dma_alloc_coherent(hw->device, dma_mem.size,
&dma_mem.pa, GFP_KERNEL);
if (!dma_mem.va)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
if (type == IRDMA_SD_TYPE_PAGED) {
struct irdma_virt_mem *vmem =
&sd_entry->u.pd_table.pd_entry_virt_mem;
@@ -510,7 +501,7 @@ enum irdma_status_code irdma_add_sd_table_entry(struct irdma_hw *hw,
dma_free_coherent(hw->device, dma_mem.size,
dma_mem.va, dma_mem.pa);
dma_mem.va = NULL;
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
}
sd_entry->u.pd_table.pd_entry = vmem->va;
@@ -549,10 +540,9 @@ enum irdma_status_code irdma_add_sd_table_entry(struct irdma_hw *hw,
* aligned on 4K boundary and zeroed memory.
* 2. It should be 4K in size.
*/
enum irdma_status_code irdma_add_pd_table_entry(struct irdma_sc_dev *dev,
struct irdma_hmc_info *hmc_info,
u32 pd_index,
struct irdma_dma_mem *rsrc_pg)
int irdma_add_pd_table_entry(struct irdma_sc_dev *dev,
struct irdma_hmc_info *hmc_info, u32 pd_index,
struct irdma_dma_mem *rsrc_pg)
{
struct irdma_hmc_pd_table *pd_table;
struct irdma_hmc_pd_entry *pd_entry;
@@ -563,7 +553,7 @@ enum irdma_status_code irdma_add_pd_table_entry(struct irdma_sc_dev *dev,
u64 page_desc;
if (pd_index / IRDMA_HMC_PD_CNT_IN_SD >= hmc_info->sd_table.sd_cnt)
return IRDMA_ERR_INVALID_PAGE_DESC_INDEX;
return -EINVAL;
sd_idx = (pd_index / IRDMA_HMC_PD_CNT_IN_SD);
if (hmc_info->sd_table.sd_entry[sd_idx].entry_type !=
@@ -584,7 +574,7 @@ enum irdma_status_code irdma_add_pd_table_entry(struct irdma_sc_dev *dev,
page->size, &page->pa,
GFP_KERNEL);
if (!page->va)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
pd_entry->rsrc_pg = false;
}
@@ -621,9 +611,8 @@ enum irdma_status_code irdma_add_pd_table_entry(struct irdma_sc_dev *dev,
* 1. Caller can deallocate the memory used by backing storage after this
* function returns.
*/
enum irdma_status_code irdma_remove_pd_bp(struct irdma_sc_dev *dev,
struct irdma_hmc_info *hmc_info,
u32 idx)
int irdma_remove_pd_bp(struct irdma_sc_dev *dev,
struct irdma_hmc_info *hmc_info, u32 idx)
{
struct irdma_hmc_pd_entry *pd_entry;
struct irdma_hmc_pd_table *pd_table;
@@ -635,11 +624,11 @@ enum irdma_status_code irdma_remove_pd_bp(struct irdma_sc_dev *dev,
sd_idx = idx / IRDMA_HMC_PD_CNT_IN_SD;
rel_pd_idx = idx % IRDMA_HMC_PD_CNT_IN_SD;
if (sd_idx >= hmc_info->sd_table.sd_cnt)
return IRDMA_ERR_INVALID_PAGE_DESC_INDEX;
return -EINVAL;
sd_entry = &hmc_info->sd_table.sd_entry[sd_idx];
if (sd_entry->entry_type != IRDMA_SD_TYPE_PAGED)
return IRDMA_ERR_INVALID_SD_TYPE;
return -EINVAL;
pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
pd_entry = &pd_table->pd_entry[rel_pd_idx];
@@ -656,7 +645,7 @@ enum irdma_status_code irdma_remove_pd_bp(struct irdma_sc_dev *dev,
if (!pd_entry->rsrc_pg) {
mem = &pd_entry->bp.addr;
if (!mem || !mem->va)
return IRDMA_ERR_PARAM;
return -EINVAL;
dma_free_coherent(dev->hw->device, mem->size, mem->va,
mem->pa);
@@ -673,14 +662,13 @@ enum irdma_status_code irdma_remove_pd_bp(struct irdma_sc_dev *dev,
* @hmc_info: pointer to the HMC configuration information structure
* @idx: the page index
*/
enum irdma_status_code irdma_prep_remove_sd_bp(struct irdma_hmc_info *hmc_info,
u32 idx)
int irdma_prep_remove_sd_bp(struct irdma_hmc_info *hmc_info, u32 idx)
{
struct irdma_hmc_sd_entry *sd_entry;
sd_entry = &hmc_info->sd_table.sd_entry[idx];
if (--sd_entry->u.bp.use_cnt)
return IRDMA_ERR_NOT_READY;
return -EBUSY;
hmc_info->sd_table.use_cnt--;
sd_entry->valid = false;
@@ -693,15 +681,14 @@ enum irdma_status_code irdma_prep_remove_sd_bp(struct irdma_hmc_info *hmc_info,
* @hmc_info: pointer to the HMC configuration information structure
* @idx: segment descriptor index to find the relevant page descriptor
*/
enum irdma_status_code
irdma_prep_remove_pd_page(struct irdma_hmc_info *hmc_info, u32 idx)
int irdma_prep_remove_pd_page(struct irdma_hmc_info *hmc_info, u32 idx)
{
struct irdma_hmc_sd_entry *sd_entry;
sd_entry = &hmc_info->sd_table.sd_entry[idx];
if (sd_entry->u.pd_table.use_cnt)
return IRDMA_ERR_NOT_READY;
return -EBUSY;
sd_entry->valid = false;
hmc_info->sd_table.use_cnt--;

53
drivers/infiniband/hw/irdma/hmc.h
View File

@@ -141,40 +141,29 @@ struct irdma_hmc_del_obj_info {
bool privileged;
};
enum irdma_status_code irdma_copy_dma_mem(struct irdma_hw *hw, void *dest_buf,
struct irdma_dma_mem *src_mem,
u64 src_offset, u64 size);
enum irdma_status_code
irdma_sc_create_hmc_obj(struct irdma_sc_dev *dev,
struct irdma_hmc_create_obj_info *info);
enum irdma_status_code irdma_sc_del_hmc_obj(struct irdma_sc_dev *dev,
struct irdma_hmc_del_obj_info *info,
bool reset);
enum irdma_status_code irdma_hmc_sd_one(struct irdma_sc_dev *dev, u8 hmc_fn_id,
u64 pa, u32 sd_idx,
enum irdma_sd_entry_type type,
bool setsd);
enum irdma_status_code
irdma_update_sds_noccq(struct irdma_sc_dev *dev,
struct irdma_update_sds_info *info);
int irdma_copy_dma_mem(struct irdma_hw *hw, void *dest_buf,
struct irdma_dma_mem *src_mem, u64 src_offset, u64 size);
int irdma_sc_create_hmc_obj(struct irdma_sc_dev *dev,
struct irdma_hmc_create_obj_info *info);
int irdma_sc_del_hmc_obj(struct irdma_sc_dev *dev,
struct irdma_hmc_del_obj_info *info, bool reset);
int irdma_hmc_sd_one(struct irdma_sc_dev *dev, u8 hmc_fn_id, u64 pa, u32 sd_idx,
enum irdma_sd_entry_type type,
bool setsd);
int irdma_update_sds_noccq(struct irdma_sc_dev *dev,
struct irdma_update_sds_info *info);
struct irdma_vfdev *irdma_vfdev_from_fpm(struct irdma_sc_dev *dev,
u8 hmc_fn_id);
struct irdma_hmc_info *irdma_vf_hmcinfo_from_fpm(struct irdma_sc_dev *dev,
u8 hmc_fn_id);
enum irdma_status_code irdma_add_sd_table_entry(struct irdma_hw *hw,
struct irdma_hmc_info *hmc_info,
u32 sd_index,
enum irdma_sd_entry_type type,
u64 direct_mode_sz);
enum irdma_status_code irdma_add_pd_table_entry(struct irdma_sc_dev *dev,
struct irdma_hmc_info *hmc_info,
u32 pd_index,
struct irdma_dma_mem *rsrc_pg);
enum irdma_status_code irdma_remove_pd_bp(struct irdma_sc_dev *dev,
struct irdma_hmc_info *hmc_info,
u32 idx);
enum irdma_status_code irdma_prep_remove_sd_bp(struct irdma_hmc_info *hmc_info,
u32 idx);
enum irdma_status_code
irdma_prep_remove_pd_page(struct irdma_hmc_info *hmc_info, u32 idx);
int irdma_add_sd_table_entry(struct irdma_hw *hw,
struct irdma_hmc_info *hmc_info, u32 sd_index,
enum irdma_sd_entry_type type, u64 direct_mode_sz);
int irdma_add_pd_table_entry(struct irdma_sc_dev *dev,
struct irdma_hmc_info *hmc_info, u32 pd_index,
struct irdma_dma_mem *rsrc_pg);
int irdma_remove_pd_bp(struct irdma_sc_dev *dev,
struct irdma_hmc_info *hmc_info, u32 idx);
int irdma_prep_remove_sd_bp(struct irdma_hmc_info *hmc_info, u32 idx);
int irdma_prep_remove_pd_page(struct irdma_hmc_info *hmc_info, u32 idx);
#endif /* IRDMA_HMC_H */

192
drivers/infiniband/hw/irdma/hw.c
View File

@@ -75,12 +75,12 @@ static void irdma_puda_ce_handler(struct irdma_pci_f *rf,
struct irdma_sc_cq *cq)
{
struct irdma_sc_dev *dev = &rf->sc_dev;
enum irdma_status_code status;
u32 compl_error;
int status;
do {
status = irdma_puda_poll_cmpl(dev, cq, &compl_error);
if (status == IRDMA_ERR_Q_EMPTY)
if (status == -ENOENT)
break;
if (status) {
ibdev_dbg(to_ibdev(dev), "ERR: puda status = %d\n", status);
@@ -456,7 +456,7 @@ static void irdma_ceq_dpc(struct tasklet_struct *t)
* Allocate iwdev msix table and copy the msix info to the table
* Return 0 if successful, otherwise return error
*/
static enum irdma_status_code irdma_save_msix_info(struct irdma_pci_f *rf)
static int irdma_save_msix_info(struct irdma_pci_f *rf)
{
struct irdma_qvlist_info *iw_qvlist;
struct irdma_qv_info *iw_qvinfo;
@@ -466,13 +466,13 @@ static enum irdma_status_code irdma_save_msix_info(struct irdma_pci_f *rf)
size_t size;
if (!rf->msix_count)
return IRDMA_ERR_NO_INTR;
return -EINVAL;
size = sizeof(struct irdma_msix_vector) * rf->msix_count;
size += struct_size(iw_qvlist, qv_info, rf->msix_count);
rf->iw_msixtbl = kzalloc(size, GFP_KERNEL);
if (!rf->iw_msixtbl)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
rf->iw_qvlist = (struct irdma_qvlist_info *)
(&rf->iw_msixtbl[rf->msix_count]);
@@ -564,9 +564,9 @@ static void irdma_destroy_irq(struct irdma_pci_f *rf,
*/
static void irdma_destroy_cqp(struct irdma_pci_f *rf, bool free_hwcqp)
{
enum irdma_status_code status = 0;
struct irdma_sc_dev *dev = &rf->sc_dev;
struct irdma_cqp *cqp = &rf->cqp;
int status = 0;
if (rf->cqp_cmpl_wq)
destroy_workqueue(rf->cqp_cmpl_wq);
@@ -606,9 +606,9 @@ static void irdma_destroy_virt_aeq(struct irdma_pci_f *rf)
*/
static void irdma_destroy_aeq(struct irdma_pci_f *rf)
{
enum irdma_status_code status = IRDMA_ERR_NOT_READY;
struct irdma_sc_dev *dev = &rf->sc_dev;
struct irdma_aeq *aeq = &rf->aeq;
int status = -EBUSY;
if (!rf->msix_shared) {
rf->sc_dev.irq_ops->irdma_cfg_aeq(&rf->sc_dev, rf->iw_msixtbl->idx, false);
@@ -642,8 +642,8 @@ exit:
*/
static void irdma_destroy_ceq(struct irdma_pci_f *rf, struct irdma_ceq *iwceq)
{
enum irdma_status_code status;
struct irdma_sc_dev *dev = &rf->sc_dev;
int status;
if (rf->reset)
goto exit;
@@ -733,7 +733,7 @@ static void irdma_destroy_ccq(struct irdma_pci_f *rf)
{
struct irdma_sc_dev *dev = &rf->sc_dev;
struct irdma_ccq *ccq = &rf->ccq;
enum irdma_status_code status = 0;
int status = 0;
if (!rf->reset)
status = irdma_sc_ccq_destroy(dev->ccq, 0, true);
@@ -796,9 +796,8 @@ static void irdma_del_hmc_objects(struct irdma_sc_dev *dev,
* @dev: hardware control device structure
* @info: information for the hmc object to create
*/
static enum irdma_status_code
irdma_create_hmc_obj_type(struct irdma_sc_dev *dev,
struct irdma_hmc_create_obj_info *info)
static int irdma_create_hmc_obj_type(struct irdma_sc_dev *dev,
struct irdma_hmc_create_obj_info *info)
{
return irdma_sc_create_hmc_obj(dev, info);
}
@@ -812,13 +811,12 @@ irdma_create_hmc_obj_type(struct irdma_sc_dev *dev,
* Create the device hmc objects and allocate hmc pages
* Return 0 if successful, otherwise clean up and return error
*/
static enum irdma_status_code
irdma_create_hmc_objs(struct irdma_pci_f *rf, bool privileged, enum irdma_vers vers)
static int irdma_create_hmc_objs(struct irdma_pci_f *rf, bool privileged,
enum irdma_vers vers)
{
struct irdma_sc_dev *dev = &rf->sc_dev;
struct irdma_hmc_create_obj_info info = {};
enum irdma_status_code status = 0;
int i;
int i, status = 0;
info.hmc_info = dev->hmc_info;
info.privileged = privileged;
@@ -868,9 +866,9 @@ irdma_create_hmc_objs(struct irdma_pci_f *rf, bool privileged, enum irdma_vers v
* update the memptr to point to the new aligned memory
* Return 0 if successful, otherwise return no memory error
*/
static enum irdma_status_code
irdma_obj_aligned_mem(struct irdma_pci_f *rf, struct irdma_dma_mem *memptr,
u32 size, u32 mask)
static int irdma_obj_aligned_mem(struct irdma_pci_f *rf,
struct irdma_dma_mem *memptr, u32 size,
u32 mask)
{
unsigned long va, newva;
unsigned long extra;
@@ -884,7 +882,7 @@ irdma_obj_aligned_mem(struct irdma_pci_f *rf, struct irdma_dma_mem *memptr,
memptr->pa = rf->obj_next.pa + extra;
memptr->size = size;
if (((u8 *)memptr->va + size) > ((u8 *)rf->obj_mem.va + rf->obj_mem.size))
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
rf->obj_next.va = (u8 *)memptr->va + size;
rf->obj_next.pa = memptr->pa + size;
@@ -899,25 +897,24 @@ irdma_obj_aligned_mem(struct irdma_pci_f *rf, struct irdma_dma_mem *memptr,
* Return 0, if the cqp and all the resources associated with it
* are successfully created, otherwise return error
*/
static enum irdma_status_code irdma_create_cqp(struct irdma_pci_f *rf)
static int irdma_create_cqp(struct irdma_pci_f *rf)
{
enum irdma_status_code status;
u32 sqsize = IRDMA_CQP_SW_SQSIZE_2048;
struct irdma_dma_mem mem;
struct irdma_sc_dev *dev = &rf->sc_dev;
struct irdma_cqp_init_info cqp_init_info = {};
struct irdma_cqp *cqp = &rf->cqp;
u16 maj_err, min_err;
int i;
int i, status;
cqp->cqp_requests = kcalloc(sqsize, sizeof(*cqp->cqp_requests), GFP_KERNEL);
if (!cqp->cqp_requests)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
cqp->scratch_array = kcalloc(sqsize, sizeof(*cqp->scratch_array), GFP_KERNEL);
if (!cqp->scratch_array) {
kfree(cqp->cqp_requests);
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
}
dev->cqp = &cqp->sc_cqp;
@@ -929,7 +926,7 @@ static enum irdma_status_code irdma_create_cqp(struct irdma_pci_f *rf)
if (!cqp->sq.va) {
kfree(cqp->scratch_array);
kfree(cqp->cqp_requests);
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
}
status = irdma_obj_aligned_mem(rf, &mem, sizeof(struct irdma_cqp_ctx),
@@ -999,12 +996,12 @@ exit:
* Return 0, if the ccq and the resources associated with it
* are successfully created, otherwise return error
*/
static enum irdma_status_code irdma_create_ccq(struct irdma_pci_f *rf)
static int irdma_create_ccq(struct irdma_pci_f *rf)
{
struct irdma_sc_dev *dev = &rf->sc_dev;
enum irdma_status_code status;
struct irdma_ccq_init_info info = {};
struct irdma_ccq *ccq = &rf->ccq;
int status;
dev->ccq = &ccq->sc_cq;
dev->ccq->dev = dev;
@@ -1015,7 +1012,7 @@ static enum irdma_status_code irdma_create_ccq(struct irdma_pci_f *rf)
ccq->mem_cq.va = dma_alloc_coherent(dev->hw->device, ccq->mem_cq.size,
&ccq->mem_cq.pa, GFP_KERNEL);
if (!ccq->mem_cq.va)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
status = irdma_obj_aligned_mem(rf, &ccq->shadow_area,
ccq->shadow_area.size,
@@ -1054,9 +1051,9 @@ exit:
* Allocate a mac ip entry and add it to the hw table Return 0
* if successful, otherwise return error
*/
static enum irdma_status_code irdma_alloc_set_mac(struct irdma_device *iwdev)
static int irdma_alloc_set_mac(struct irdma_device *iwdev)
{
enum irdma_status_code status;
int status;
status = irdma_alloc_local_mac_entry(iwdev->rf,
&iwdev->mac_ip_table_idx);
@@ -1082,9 +1079,8 @@ static enum irdma_status_code irdma_alloc_set_mac(struct irdma_device *iwdev)
* Allocate interrupt resources and enable irq handling
* Return 0 if successful, otherwise return error
*/
static enum irdma_status_code
irdma_cfg_ceq_vector(struct irdma_pci_f *rf, struct irdma_ceq *iwceq,
u32 ceq_id, struct irdma_msix_vector *msix_vec)
static int irdma_cfg_ceq_vector(struct irdma_pci_f *rf, struct irdma_ceq *iwceq,
u32 ceq_id, struct irdma_msix_vector *msix_vec)
{
int status;
@@ -1103,7 +1099,7 @@ irdma_cfg_ceq_vector(struct irdma_pci_f *rf, struct irdma_ceq *iwceq,
irq_update_affinity_hint(msix_vec->irq, &msix_vec->mask);
if (status) {
ibdev_dbg(&rf->iwdev->ibdev, "ERR: ceq irq config fail\n");
return IRDMA_ERR_CFG;
return status;
}
msix_vec->ceq_id = ceq_id;
@@ -1119,7 +1115,7 @@ irdma_cfg_ceq_vector(struct irdma_pci_f *rf, struct irdma_ceq *iwceq,
* Allocate interrupt resources and enable irq handling
* Return 0 if successful, otherwise return error
*/
static enum irdma_status_code irdma_cfg_aeq_vector(struct irdma_pci_f *rf)
static int irdma_cfg_aeq_vector(struct irdma_pci_f *rf)
{
struct irdma_msix_vector *msix_vec = rf->iw_msixtbl;
u32 ret = 0;
@@ -1131,7 +1127,7 @@ static enum irdma_status_code irdma_cfg_aeq_vector(struct irdma_pci_f *rf)
}
if (ret) {
ibdev_dbg(&rf->iwdev->ibdev, "ERR: aeq irq config fail\n");
return IRDMA_ERR_CFG;
return -EINVAL;
}
rf->sc_dev.irq_ops->irdma_cfg_aeq(&rf->sc_dev, msix_vec->idx, true);
@@ -1149,12 +1145,10 @@ static enum irdma_status_code irdma_cfg_aeq_vector(struct irdma_pci_f *rf)
* Return 0, if the ceq and the resources associated with it
* are successfully created, otherwise return error
*/
static enum irdma_status_code irdma_create_ceq(struct irdma_pci_f *rf,
struct irdma_ceq *iwceq,
u32 ceq_id,
struct irdma_sc_vsi *vsi)
static int irdma_create_ceq(struct irdma_pci_f *rf, struct irdma_ceq *iwceq,
u32 ceq_id, struct irdma_sc_vsi *vsi)
{
enum irdma_status_code status;
int status;
struct irdma_ceq_init_info info = {};
struct irdma_sc_dev *dev = &rf->sc_dev;
u64 scratch;
@@ -1169,7 +1163,7 @@ static enum irdma_status_code irdma_create_ceq(struct irdma_pci_f *rf,
iwceq->mem.va = dma_alloc_coherent(dev->hw->device, iwceq->mem.size,
&iwceq->mem.pa, GFP_KERNEL);
if (!iwceq->mem.va)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
info.ceq_id = ceq_id;
info.ceqe_base = iwceq->mem.va;
@@ -1205,18 +1199,18 @@ static enum irdma_status_code irdma_create_ceq(struct irdma_pci_f *rf,
* Create the ceq 0 and configure it's msix interrupt vector
* Return 0, if successfully set up, otherwise return error
*/
static enum irdma_status_code irdma_setup_ceq_0(struct irdma_pci_f *rf)
static int irdma_setup_ceq_0(struct irdma_pci_f *rf)
{
struct irdma_ceq *iwceq;
struct irdma_msix_vector *msix_vec;
u32 i;
enum irdma_status_code status = 0;
int status = 0;
u32 num_ceqs;
num_ceqs = min(rf->msix_count, rf->sc_dev.hmc_fpm_misc.max_ceqs);
rf->ceqlist = kcalloc(num_ceqs, sizeof(*rf->ceqlist), GFP_KERNEL);
if (!rf->ceqlist) {
status = IRDMA_ERR_NO_MEMORY;
status = -ENOMEM;
goto exit;
}
@@ -1262,14 +1256,13 @@ exit:
* Create the ceq's and configure their msix interrupt vectors
* Return 0, if ceqs are successfully set up, otherwise return error
*/
static enum irdma_status_code irdma_setup_ceqs(struct irdma_pci_f *rf,
struct irdma_sc_vsi *vsi)
static int irdma_setup_ceqs(struct irdma_pci_f *rf, struct irdma_sc_vsi *vsi)
{
u32 i;
u32 ceq_id;
struct irdma_ceq *iwceq;
struct irdma_msix_vector *msix_vec;
enum irdma_status_code status;
int status;
u32 num_ceqs;
num_ceqs = min(rf->msix_count, rf->sc_dev.hmc_fpm_misc.max_ceqs);
@@ -1303,22 +1296,21 @@ del_ceqs:
return status;
}
static enum irdma_status_code irdma_create_virt_aeq(struct irdma_pci_f *rf,
u32 size)
static int irdma_create_virt_aeq(struct irdma_pci_f *rf, u32 size)
{
enum irdma_status_code status = IRDMA_ERR_NO_MEMORY;
struct irdma_aeq *aeq = &rf->aeq;
dma_addr_t *pg_arr;
u32 pg_cnt;
int status;
if (rf->rdma_ver < IRDMA_GEN_2)
return IRDMA_NOT_SUPPORTED;
return -EOPNOTSUPP;
aeq->mem.size = sizeof(struct irdma_sc_aeqe) * size;
aeq->mem.va = vzalloc(aeq->mem.size);
if (!aeq->mem.va)
return status;
return -ENOMEM;
pg_cnt = DIV_ROUND_UP(aeq->mem.size, PAGE_SIZE);
status = irdma_get_pble(rf->pble_rsrc, &aeq->palloc, pg_cnt, true);
@@ -1345,15 +1337,15 @@ static enum irdma_status_code irdma_create_virt_aeq(struct irdma_pci_f *rf,
* Return 0, if the aeq and the resources associated with it
* are successfully created, otherwise return error
*/
static enum irdma_status_code irdma_create_aeq(struct irdma_pci_f *rf)
static int irdma_create_aeq(struct irdma_pci_f *rf)
{
enum irdma_status_code status;
struct irdma_aeq_init_info info = {};
struct irdma_sc_dev *dev = &rf->sc_dev;
struct irdma_aeq *aeq = &rf->aeq;
struct irdma_hmc_info *hmc_info = rf->sc_dev.hmc_info;
u32 aeq_size;
u8 multiplier = (rf->protocol_used == IRDMA_IWARP_PROTOCOL_ONLY) ? 2 : 1;
int status;
aeq_size = multiplier * hmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt +
hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt;
@@ -1412,10 +1404,10 @@ err:
* Create the aeq and configure its msix interrupt vector
* Return 0 if successful, otherwise return error
*/
static enum irdma_status_code irdma_setup_aeq(struct irdma_pci_f *rf)
static int irdma_setup_aeq(struct irdma_pci_f *rf)
{
struct irdma_sc_dev *dev = &rf->sc_dev;
enum irdma_status_code status;
int status;
status = irdma_create_aeq(rf);
if (status)
@@ -1439,10 +1431,10 @@ static enum irdma_status_code irdma_setup_aeq(struct irdma_pci_f *rf)
*
* Return 0 if successful, otherwise return error
*/
static enum irdma_status_code irdma_initialize_ilq(struct irdma_device *iwdev)
static int irdma_initialize_ilq(struct irdma_device *iwdev)
{
struct irdma_puda_rsrc_info info = {};
enum irdma_status_code status;
int status;
info.type = IRDMA_PUDA_RSRC_TYPE_ILQ;
info.cq_id = 1;
@@ -1469,10 +1461,10 @@ static enum irdma_status_code irdma_initialize_ilq(struct irdma_device *iwdev)
*
* Return 0 if successful, otherwise return error
*/
static enum irdma_status_code irdma_initialize_ieq(struct irdma_device *iwdev)
static int irdma_initialize_ieq(struct irdma_device *iwdev)
{
struct irdma_puda_rsrc_info info = {};
enum irdma_status_code status;
int status;
info.type = IRDMA_PUDA_RSRC_TYPE_IEQ;
info.cq_id = 2;
@@ -1515,9 +1507,9 @@ void irdma_reinitialize_ieq(struct irdma_sc_vsi *vsi)
* the hmc objects and create the objects
* Return 0 if successful, otherwise return error
*/
static enum irdma_status_code irdma_hmc_setup(struct irdma_pci_f *rf)
static int irdma_hmc_setup(struct irdma_pci_f *rf)
{
enum irdma_status_code status;
int status;
u32 qpcnt;
if (rf->rdma_ver == IRDMA_GEN_1)
@@ -1570,9 +1562,9 @@ static void irdma_del_init_mem(struct irdma_pci_f *rf)
* Return 0 if successful, otherwise clean up the resources
* and return error
*/
static enum irdma_status_code irdma_initialize_dev(struct irdma_pci_f *rf)
static int irdma_initialize_dev(struct irdma_pci_f *rf)
{
enum irdma_status_code status;
int status;
struct irdma_sc_dev *dev = &rf->sc_dev;
struct irdma_device_init_info info = {};
struct irdma_dma_mem mem;
@@ -1584,7 +1576,7 @@ static enum irdma_status_code irdma_initialize_dev(struct irdma_pci_f *rf)
rf->hmc_info_mem = kzalloc(size, GFP_KERNEL);
if (!rf->hmc_info_mem)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
rf->pble_rsrc = (struct irdma_hmc_pble_rsrc *)rf->hmc_info_mem;
dev->hmc_info = &rf->hw.hmc;
@@ -1608,7 +1600,7 @@ static enum irdma_status_code irdma_initialize_dev(struct irdma_pci_f *rf)
info.fpm_commit_buf = mem.va;
info.bar0 = rf->hw.hw_addr;
info.hmc_fn_id = PCI_FUNC(rf->pcidev->devfn);
info.hmc_fn_id = rf->pf_id;
info.hw = &rf->hw;
status = irdma_sc_dev_init(rf->rdma_ver, &rf->sc_dev, &info);
if (status)
@@ -1667,9 +1659,9 @@ void irdma_rt_deinit_hw(struct irdma_device *iwdev)
destroy_workqueue(iwdev->cleanup_wq);
}
static enum irdma_status_code irdma_setup_init_state(struct irdma_pci_f *rf)
static int irdma_setup_init_state(struct irdma_pci_f *rf)
{
enum irdma_status_code status;
int status;
status = irdma_save_msix_info(rf);
if (status)
@@ -1680,7 +1672,7 @@ static enum irdma_status_code irdma_setup_init_state(struct irdma_pci_f *rf)
rf->obj_mem.va = dma_alloc_coherent(rf->hw.device, rf->obj_mem.size,
&rf->obj_mem.pa, GFP_KERNEL);
if (!rf->obj_mem.va) {
status = IRDMA_ERR_NO_MEMORY;
status = -ENOMEM;
goto clean_msixtbl;
}
@@ -1763,14 +1755,14 @@ void irdma_ctrl_deinit_hw(struct irdma_pci_f *rf)
* Create device queues ILQ, IEQ, CEQs and PBLEs. Setup irdma
* device resource objects.
*/
enum irdma_status_code irdma_rt_init_hw(struct irdma_device *iwdev,
struct irdma_l2params *l2params)
int irdma_rt_init_hw(struct irdma_device *iwdev,
struct irdma_l2params *l2params)
{
struct irdma_pci_f *rf = iwdev->rf;
struct irdma_sc_dev *dev = &rf->sc_dev;
enum irdma_status_code status;
struct irdma_vsi_init_info vsi_info = {};
struct irdma_vsi_stats_info stats_info = {};
int status;
vsi_info.dev = dev;
vsi_info.back_vsi = iwdev;
@@ -1788,7 +1780,7 @@ enum irdma_status_code irdma_rt_init_hw(struct irdma_device *iwdev,
stats_info.pestat = kzalloc(sizeof(*stats_info.pestat), GFP_KERNEL);
if (!stats_info.pestat) {
irdma_cleanup_cm_core(&iwdev->cm_core);
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
}
stats_info.fcn_id = dev->hmc_fn_id;
status = irdma_vsi_stats_init(&iwdev->vsi, &stats_info);
@@ -1850,7 +1842,7 @@ enum irdma_status_code irdma_rt_init_hw(struct irdma_device *iwdev,
iwdev->cleanup_wq = alloc_workqueue("irdma-cleanup-wq",
WQ_UNBOUND, WQ_UNBOUND_MAX_ACTIVE);
if (!iwdev->cleanup_wq)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
irdma_get_used_rsrc(iwdev);
init_waitqueue_head(&iwdev->suspend_wq);
@@ -1870,10 +1862,10 @@ enum irdma_status_code irdma_rt_init_hw(struct irdma_device *iwdev,
*
* Create admin queues, HMC obejcts and RF resource objects
*/
enum irdma_status_code irdma_ctrl_init_hw(struct irdma_pci_f *rf)
int irdma_ctrl_init_hw(struct irdma_pci_f *rf)
{
struct irdma_sc_dev *dev = &rf->sc_dev;
enum irdma_status_code status;
int status;
do {
status = irdma_setup_init_state(rf);
if (status)
@@ -1915,7 +1907,7 @@ enum irdma_status_code irdma_ctrl_init_hw(struct irdma_pci_f *rf)
rf->cqp_cmpl_wq = alloc_ordered_workqueue("cqp_cmpl_wq",
WQ_HIGHPRI | WQ_UNBOUND);
if (!rf->cqp_cmpl_wq) {
status = IRDMA_ERR_NO_MEMORY;
status = -ENOMEM;
break;
}
INIT_WORK(&rf->cqp_cmpl_work, cqp_compl_worker);
@@ -2202,11 +2194,11 @@ int irdma_add_local_mac_entry(struct irdma_pci_f *rf, const u8 *mac_addr, u16 id
struct irdma_cqp *iwcqp = &rf->cqp;
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
enum irdma_status_code status;
int status;
cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
if (!cqp_request)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
cqp_info = &cqp_request->info;
cqp_info->post_sq = 1;
@@ -2238,11 +2230,11 @@ int irdma_alloc_local_mac_entry(struct irdma_pci_f *rf, u16 *mac_tbl_idx)
struct irdma_cqp *iwcqp = &rf->cqp;
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
enum irdma_status_code status = 0;
int status = 0;
cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
if (!cqp_request)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
cqp_info = &cqp_request->info;
cqp_info->cqp_cmd = IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY;
@@ -2264,18 +2256,17 @@ int irdma_alloc_local_mac_entry(struct irdma_pci_f *rf, u16 *mac_tbl_idx)
* @accel_local_port: port for apbvt
* @add_port: add ordelete port
*/
static enum irdma_status_code
irdma_cqp_manage_apbvt_cmd(struct irdma_device *iwdev, u16 accel_local_port,
bool add_port)
static int irdma_cqp_manage_apbvt_cmd(struct irdma_device *iwdev,
u16 accel_local_port, bool add_port)
{
struct irdma_apbvt_info *info;
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
enum irdma_status_code status;
int status;
cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, add_port);
if (!cqp_request)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
cqp_info = &cqp_request->info;
info = &cqp_info->in.u.manage_apbvt_entry.info;
@@ -2429,22 +2420,21 @@ static void irdma_send_syn_cqp_callback(struct irdma_cqp_request *cqp_request)
* @cmnode: cmnode associated with connection
* @wait: wait for completion
*/
enum irdma_status_code
irdma_manage_qhash(struct irdma_device *iwdev, struct irdma_cm_info *cminfo,
enum irdma_quad_entry_type etype,
enum irdma_quad_hash_manage_type mtype, void *cmnode,
bool wait)
int irdma_manage_qhash(struct irdma_device *iwdev, struct irdma_cm_info *cminfo,
enum irdma_quad_entry_type etype,
enum irdma_quad_hash_manage_type mtype, void *cmnode,
bool wait)
{
struct irdma_qhash_table_info *info;
enum irdma_status_code status;
struct irdma_cqp *iwcqp = &iwdev->rf->cqp;
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
struct irdma_cm_node *cm_node = cmnode;
int status;
cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait);
if (!cqp_request)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
cqp_info = &cqp_request->info;
info = &cqp_info->in.u.manage_qhash_table_entry.info;
@@ -2558,12 +2548,10 @@ static void irdma_hw_flush_wqes_callback(struct irdma_cqp_request *cqp_request)
* @info: info for flush
* @wait: flag wait for completion
*/
enum irdma_status_code irdma_hw_flush_wqes(struct irdma_pci_f *rf,
struct irdma_sc_qp *qp,
struct irdma_qp_flush_info *info,
bool wait)
int irdma_hw_flush_wqes(struct irdma_pci_f *rf, struct irdma_sc_qp *qp,
struct irdma_qp_flush_info *info, bool wait)
{
enum irdma_status_code status;
int status;
struct irdma_qp_flush_info *hw_info;
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
@@ -2571,7 +2559,7 @@ enum irdma_status_code irdma_hw_flush_wqes(struct irdma_pci_f *rf,
cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait);
if (!cqp_request)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
cqp_info = &cqp_request->info;
if (!wait)
@@ -2619,7 +2607,7 @@ enum irdma_status_code irdma_hw_flush_wqes(struct irdma_pci_f *rf,
info->sq = true;
new_req = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
if (!new_req) {
status = IRDMA_ERR_NO_MEMORY;
status = -ENOMEM;
goto put_cqp;
}
cqp_info = &new_req->info;

1
drivers/infiniband/hw/irdma/i40iw_hw.c
View File

@@ -3,7 +3,6 @@
#include "osdep.h"
#include "type.h"
#include "i40iw_hw.h"
#include "status.h"
#include "protos.h"
static u32 i40iw_regs[IRDMA_MAX_REGS] = {

3
drivers/infiniband/hw/irdma/i40iw_if.c
View File

@@ -77,6 +77,7 @@ static void i40iw_fill_device_info(struct irdma_device *iwdev, struct i40e_info
rf->rdma_ver = IRDMA_GEN_1;
rf->gen_ops.request_reset = i40iw_request_reset;
rf->pcidev = cdev_info->pcidev;
rf->pf_id = cdev_info->fid;
rf->hw.hw_addr = cdev_info->hw_addr;
rf->cdev = cdev_info;
rf->msix_count = cdev_info->msix_count;
@@ -138,7 +139,7 @@ static int i40iw_open(struct i40e_info *cdev_info, struct i40e_client *client)
if (last_qset == IRDMA_NO_QSET)
last_qset = qset;
else if ((qset != last_qset) && (qset != IRDMA_NO_QSET))
iwdev->dcb = true;
iwdev->dcb_vlan_mode = true;
}
if (irdma_rt_init_hw(iwdev, &l2params)) {

29
drivers/infiniband/hw/irdma/main.c
View File

@@ -79,6 +79,10 @@ static void irdma_fill_qos_info(struct irdma_l2params *l2params,
}
for (i = 0; i < IIDC_MAX_USER_PRIORITY; i++)
l2params->up2tc[i] = qos_info->up2tc[i];
if (qos_info->pfc_mode == IIDC_DSCP_PFC_MODE) {
l2params->dscp_mode = true;
memcpy(l2params->dscp_map, qos_info->dscp_map, sizeof(l2params->dscp_map));
}
}
static void irdma_iidc_event_handler(struct ice_pf *pf, struct iidc_event *event)
@@ -108,8 +112,9 @@ static void irdma_iidc_event_handler(struct ice_pf *pf, struct iidc_event *event
l2params.tc_changed = true;
ibdev_dbg(&iwdev->ibdev, "CLNT: TC Change\n");
ice_get_qos_params(pf, &qos_info);
iwdev->dcb = qos_info.num_tc > 1;
irdma_fill_qos_info(&l2params, &qos_info);
if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY)
iwdev->dcb_vlan_mode = qos_info.num_tc > 1 && !l2params.dscp_mode;
irdma_change_l2params(&iwdev->vsi, &l2params);
} else if (*event->type & BIT(IIDC_EVENT_CRIT_ERR)) {
ibdev_warn(&iwdev->ibdev, "ICE OICR event notification: oicr = 0x%08x\n",
@@ -157,8 +162,8 @@ static void irdma_request_reset(struct irdma_pci_f *rf)
* @vsi: vsi structure
* @tc_node: Traffic class node
*/
static enum irdma_status_code irdma_lan_register_qset(struct irdma_sc_vsi *vsi,
struct irdma_ws_node *tc_node)
static int irdma_lan_register_qset(struct irdma_sc_vsi *vsi,
struct irdma_ws_node *tc_node)
{
struct irdma_device *iwdev = vsi->back_vsi;
struct ice_pf *pf = iwdev->rf->cdev;
@@ -171,7 +176,7 @@ static enum irdma_status_code irdma_lan_register_qset(struct irdma_sc_vsi *vsi,
ret = ice_add_rdma_qset(pf, &qset);
if (ret) {
ibdev_dbg(&iwdev->ibdev, "WS: LAN alloc_res for rdma qset failed.\n");
return IRDMA_ERR_REG_QSET;
return ret;
}
tc_node->l2_sched_node_id = qset.teid;
@@ -226,6 +231,7 @@ static void irdma_fill_device_info(struct irdma_device *iwdev, struct ice_pf *pf
rf->hw.hw_addr = pf->hw.hw_addr;
rf->pcidev = pf->pdev;
rf->msix_count = pf->num_rdma_msix;
rf->pf_id = pf->hw.pf_id;
rf->msix_entries = &pf->msix_entries[pf->rdma_base_vector];
rf->default_vsi.vsi_idx = vsi->vsi_num;
rf->protocol_used = pf->rdma_mode & IIDC_RDMA_PROTOCOL_ROCEV2 ?
@@ -236,7 +242,7 @@ static void irdma_fill_device_info(struct irdma_device *iwdev, struct ice_pf *pf
rf->gen_ops.request_reset = irdma_request_reset;
rf->limits_sel = 7;
rf->iwdev = iwdev;
mutex_init(&iwdev->ah_tbl_lock);
iwdev->netdev = vsi->netdev;
iwdev->vsi_num = vsi->vsi_num;
iwdev->init_state = INITIAL_STATE;
@@ -275,18 +281,19 @@ static int irdma_probe(struct auxiliary_device *aux_dev, const struct auxiliary_
irdma_fill_device_info(iwdev, pf, vsi);
rf = iwdev->rf;
if (irdma_ctrl_init_hw(rf)) {
err = -EIO;
err = irdma_ctrl_init_hw(rf);
if (err)
goto err_ctrl_init;
}
l2params.mtu = iwdev->netdev->mtu;
ice_get_qos_params(pf, &qos_info);
irdma_fill_qos_info(&l2params, &qos_info);
if (irdma_rt_init_hw(iwdev, &l2params)) {
err = -EIO;
if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY)
iwdev->dcb_vlan_mode = l2params.num_tc > 1 && !l2params.dscp_mode;
err = irdma_rt_init_hw(iwdev, &l2params);
if (err)
goto err_rt_init;
}
err = irdma_ib_register_device(iwdev);
if (err)

47
drivers/infiniband/hw/irdma/main.h
View File

@@ -40,7 +40,6 @@
#include <rdma/ib_umem.h>
#include <rdma/ib_cache.h>
#include <rdma/uverbs_ioctl.h>
#include "status.h"
#include "osdep.h"
#include "defs.h"
#include "hmc.h"
@@ -242,8 +241,8 @@ struct irdma_qvlist_info {
struct irdma_gen_ops {
void (*request_reset)(struct irdma_pci_f *rf);
enum irdma_status_code (*register_qset)(struct irdma_sc_vsi *vsi,
struct irdma_ws_node *tc_node);
int (*register_qset)(struct irdma_sc_vsi *vsi,
struct irdma_ws_node *tc_node);
void (*unregister_qset)(struct irdma_sc_vsi *vsi,
struct irdma_ws_node *tc_node);
};
@@ -257,6 +256,7 @@ struct irdma_pci_f {
u8 *mem_rsrc;
u8 rdma_ver;
u8 rst_to;
u8 pf_id;
enum irdma_protocol_used protocol_used;
u32 sd_type;
u32 msix_count;
@@ -332,6 +332,8 @@ struct irdma_device {
struct workqueue_struct *cleanup_wq;
struct irdma_sc_vsi vsi;
struct irdma_cm_core cm_core;
DECLARE_HASHTABLE(ah_hash_tbl, 8);
struct mutex ah_tbl_lock; /* protect AH hash table access */
u32 roce_cwnd;
u32 roce_ackcreds;
u32 vendor_id;
@@ -345,7 +347,7 @@ struct irdma_device {
u8 iw_status;
bool roce_mode:1;
bool roce_dcqcn_en:1;
bool dcb:1;
bool dcb_vlan_mode:1;
bool iw_ooo:1;
enum init_completion_state init_state;
@@ -457,10 +459,10 @@ static inline void irdma_free_rsrc(struct irdma_pci_f *rf,
spin_unlock_irqrestore(&rf->rsrc_lock, flags);
}
enum irdma_status_code irdma_ctrl_init_hw(struct irdma_pci_f *rf);
int irdma_ctrl_init_hw(struct irdma_pci_f *rf);
void irdma_ctrl_deinit_hw(struct irdma_pci_f *rf);
enum irdma_status_code irdma_rt_init_hw(struct irdma_device *iwdev,
struct irdma_l2params *l2params);
int irdma_rt_init_hw(struct irdma_device *iwdev,
struct irdma_l2params *l2params);
void irdma_rt_deinit_hw(struct irdma_device *iwdev);
void irdma_qp_add_ref(struct ib_qp *ibqp);
void irdma_qp_rem_ref(struct ib_qp *ibqp);
@@ -489,9 +491,8 @@ void irdma_cm_disconn(struct irdma_qp *qp);
bool irdma_cqp_crit_err(struct irdma_sc_dev *dev, u8 cqp_cmd,
u16 maj_err_code, u16 min_err_code);
enum irdma_status_code
irdma_handle_cqp_op(struct irdma_pci_f *rf,
struct irdma_cqp_request *cqp_request);
int irdma_handle_cqp_op(struct irdma_pci_f *rf,
struct irdma_cqp_request *cqp_request);
int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
struct ib_udata *udata);
@@ -500,21 +501,17 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr,
void irdma_cq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_cq *cq);
void irdma_cleanup_pending_cqp_op(struct irdma_pci_f *rf);
enum irdma_status_code irdma_hw_modify_qp(struct irdma_device *iwdev,
struct irdma_qp *iwqp,
struct irdma_modify_qp_info *info,
bool wait);
enum irdma_status_code irdma_qp_suspend_resume(struct irdma_sc_qp *qp,
bool suspend);
enum irdma_status_code
irdma_manage_qhash(struct irdma_device *iwdev, struct irdma_cm_info *cminfo,
enum irdma_quad_entry_type etype,
enum irdma_quad_hash_manage_type mtype, void *cmnode,
bool wait);
int irdma_hw_modify_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp,
struct irdma_modify_qp_info *info, bool wait);
int irdma_qp_suspend_resume(struct irdma_sc_qp *qp, bool suspend);
int irdma_manage_qhash(struct irdma_device *iwdev, struct irdma_cm_info *cminfo,
enum irdma_quad_entry_type etype,
enum irdma_quad_hash_manage_type mtype, void *cmnode,
bool wait);
void irdma_receive_ilq(struct irdma_sc_vsi *vsi, struct irdma_puda_buf *rbuf);
void irdma_free_sqbuf(struct irdma_sc_vsi *vsi, void *bufp);
void irdma_free_qp_rsrc(struct irdma_qp *iwqp);
enum irdma_status_code irdma_setup_cm_core(struct irdma_device *iwdev, u8 ver);
int irdma_setup_cm_core(struct irdma_device *iwdev, u8 ver);
void irdma_cleanup_cm_core(struct irdma_cm_core *cm_core);
void irdma_next_iw_state(struct irdma_qp *iwqp, u8 state, u8 del_hash, u8 term,
u8 term_len);
@@ -523,10 +520,8 @@ int irdma_send_reset(struct irdma_cm_node *cm_node);
struct irdma_cm_node *irdma_find_node(struct irdma_cm_core *cm_core,
u16 rem_port, u32 *rem_addr, u16 loc_port,
u32 *loc_addr, u16 vlan_id);
enum irdma_status_code irdma_hw_flush_wqes(struct irdma_pci_f *rf,
struct irdma_sc_qp *qp,
struct irdma_qp_flush_info *info,
bool wait);
int irdma_hw_flush_wqes(struct irdma_pci_f *rf, struct irdma_sc_qp *qp,
struct irdma_qp_flush_info *info, bool wait);
void irdma_gen_ae(struct irdma_pci_f *rf, struct irdma_sc_qp *qp,
struct irdma_gen_ae_info *info, bool wait);
void irdma_copy_ip_ntohl(u32 *dst, __be32 *src);

41
drivers/infiniband/hw/irdma/osdep.h
View File

@@ -5,6 +5,7 @@
#include <linux/pci.h>
#include <linux/bitfield.h>
#include <linux/net/intel/iidc.h>
#include <crypto/hash.h>
#include <rdma/ib_verbs.h>
@@ -42,32 +43,28 @@ enum irdma_status_code irdma_vf_wait_vchnl_resp(struct irdma_sc_dev *dev);
bool irdma_vf_clear_to_send(struct irdma_sc_dev *dev);
void irdma_add_dev_ref(struct irdma_sc_dev *dev);
void irdma_put_dev_ref(struct irdma_sc_dev *dev);
enum irdma_status_code irdma_ieq_check_mpacrc(struct shash_desc *desc,
void *addr, u32 len, u32 val);
int irdma_ieq_check_mpacrc(struct shash_desc *desc, void *addr, u32 len,
u32 val);
struct irdma_sc_qp *irdma_ieq_get_qp(struct irdma_sc_dev *dev,
struct irdma_puda_buf *buf);
void irdma_send_ieq_ack(struct irdma_sc_qp *qp);
void irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len,
u32 seqnum);
void irdma_free_hash_desc(struct shash_desc *hash_desc);
enum irdma_status_code irdma_init_hash_desc(struct shash_desc **hash_desc);
enum irdma_status_code
irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
struct irdma_puda_buf *buf);
enum irdma_status_code irdma_cqp_sds_cmd(struct irdma_sc_dev *dev,
struct irdma_update_sds_info *info);
enum irdma_status_code
irdma_cqp_manage_hmc_fcn_cmd(struct irdma_sc_dev *dev,
struct irdma_hmc_fcn_info *hmcfcninfo,
u16 *pmf_idx);
enum irdma_status_code
irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev *dev,
struct irdma_dma_mem *val_mem, u8 hmc_fn_id);
enum irdma_status_code
irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev *dev,
struct irdma_dma_mem *val_mem, u8 hmc_fn_id);
enum irdma_status_code irdma_alloc_query_fpm_buf(struct irdma_sc_dev *dev,
struct irdma_dma_mem *mem);
int irdma_init_hash_desc(struct shash_desc **hash_desc);
int irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
struct irdma_puda_buf *buf);
int irdma_cqp_sds_cmd(struct irdma_sc_dev *dev,
struct irdma_update_sds_info *info);
int irdma_cqp_manage_hmc_fcn_cmd(struct irdma_sc_dev *dev,
struct irdma_hmc_fcn_info *hmcfcninfo,
u16 *pmf_idx);
int irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev *dev,
struct irdma_dma_mem *val_mem, u8 hmc_fn_id);
int irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev *dev,
struct irdma_dma_mem *val_mem, u8 hmc_fn_id);
int irdma_alloc_query_fpm_buf(struct irdma_sc_dev *dev,
struct irdma_dma_mem *mem);
void *irdma_remove_cqp_head(struct irdma_sc_dev *dev);
void irdma_term_modify_qp(struct irdma_sc_qp *qp, u8 next_state, u8 term,
u8 term_len);
@@ -79,7 +76,7 @@ void irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi);
void wr32(struct irdma_hw *hw, u32 reg, u32 val);
u32 rd32(struct irdma_hw *hw, u32 reg);
u64 rd64(struct irdma_hw *hw, u32 reg);
enum irdma_status_code irdma_map_vm_page_list(struct irdma_hw *hw, void *va,
dma_addr_t *pg_dma, u32 pg_cnt);
int irdma_map_vm_page_list(struct irdma_hw *hw, void *va, dma_addr_t *pg_dma,
u32 pg_cnt);
void irdma_unmap_vm_page_list(struct irdma_hw *hw, dma_addr_t *pg_dma, u32 pg_cnt);
#endif /* IRDMA_OSDEP_H */

77
drivers/infiniband/hw/irdma/pble.c
View File

@@ -1,15 +1,13 @@
// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
/* Copyright (c) 2015 - 2021 Intel Corporation */
#include "osdep.h"
#include "status.h"
#include "hmc.h"
#include "defs.h"
#include "type.h"
#include "protos.h"
#include "pble.h"
static enum irdma_status_code
add_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc);
static int add_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc);
/**
* irdma_destroy_pble_prm - destroy prm during module unload
@@ -35,13 +33,12 @@ void irdma_destroy_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc)
* @dev: irdma_sc_dev struct
* @pble_rsrc: pble resources
*/
enum irdma_status_code
irdma_hmc_init_pble(struct irdma_sc_dev *dev,
struct irdma_hmc_pble_rsrc *pble_rsrc)
int irdma_hmc_init_pble(struct irdma_sc_dev *dev,
struct irdma_hmc_pble_rsrc *pble_rsrc)
{
struct irdma_hmc_info *hmc_info;
u32 fpm_idx = 0;
enum irdma_status_code status = 0;
int status = 0;
hmc_info = dev->hmc_info;
pble_rsrc->dev = dev;
@@ -60,7 +57,7 @@ irdma_hmc_init_pble(struct irdma_sc_dev *dev,
INIT_LIST_HEAD(&pble_rsrc->pinfo.clist);
if (add_pble_prm(pble_rsrc)) {
irdma_destroy_pble_prm(pble_rsrc);
status = IRDMA_ERR_NO_MEMORY;
status = -ENOMEM;
}
return status;
@@ -84,12 +81,11 @@ static void get_sd_pd_idx(struct irdma_hmc_pble_rsrc *pble_rsrc,
* @pble_rsrc: pble resource ptr
* @info: page info for sd
*/
static enum irdma_status_code
add_sd_direct(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_add_page_info *info)
static int add_sd_direct(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_add_page_info *info)
{
struct irdma_sc_dev *dev = pble_rsrc->dev;
enum irdma_status_code ret_code = 0;
int ret_code = 0;
struct sd_pd_idx *idx = &info->idx;
struct irdma_chunk *chunk = info->chunk;
struct irdma_hmc_info *hmc_info = info->hmc_info;
@@ -137,9 +133,8 @@ static u32 fpm_to_idx(struct irdma_hmc_pble_rsrc *pble_rsrc, u64 addr)
* @pble_rsrc: pble resource management
* @info: page info for sd
*/
static enum irdma_status_code
add_bp_pages(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_add_page_info *info)
static int add_bp_pages(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_add_page_info *info)
{
struct irdma_sc_dev *dev = pble_rsrc->dev;
u8 *addr;
@@ -148,13 +143,13 @@ add_bp_pages(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_hmc_sd_entry *sd_entry = info->sd_entry;
struct irdma_hmc_info *hmc_info = info->hmc_info;
struct irdma_chunk *chunk = info->chunk;
enum irdma_status_code status = 0;
int status = 0;
u32 rel_pd_idx = info->idx.rel_pd_idx;
u32 pd_idx = info->idx.pd_idx;
u32 i;
if (irdma_pble_get_paged_mem(chunk, info->pages))
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
status = irdma_add_sd_table_entry(dev->hw, hmc_info, info->idx.sd_idx,
IRDMA_SD_TYPE_PAGED,
@@ -207,8 +202,7 @@ static enum irdma_sd_entry_type irdma_get_type(struct irdma_sc_dev *dev,
* add_pble_prm - add a sd entry for pble resoure
* @pble_rsrc: pble resource management
*/
static enum irdma_status_code
add_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc)
static int add_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc)
{
struct irdma_sc_dev *dev = pble_rsrc->dev;
struct irdma_hmc_sd_entry *sd_entry;
@@ -216,22 +210,22 @@ add_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc)
struct irdma_chunk *chunk;
struct irdma_add_page_info info;
struct sd_pd_idx *idx = &info.idx;
enum irdma_status_code ret_code = 0;
int ret_code = 0;
enum irdma_sd_entry_type sd_entry_type;
u64 sd_reg_val = 0;
struct irdma_virt_mem chunkmem;
u32 pages;
if (pble_rsrc->unallocated_pble < PBLE_PER_PAGE)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
if (pble_rsrc->next_fpm_addr & 0xfff)
return IRDMA_ERR_INVALID_PAGE_DESC_INDEX;
return -EINVAL;
chunkmem.size = sizeof(*chunk);
chunkmem.va = kzalloc(chunkmem.size, GFP_KERNEL);
if (!chunkmem.va)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
chunk = chunkmem.va;
chunk->chunkmem = chunkmem;
@@ -337,9 +331,8 @@ static void free_lvl2(struct irdma_hmc_pble_rsrc *pble_rsrc,
* @pble_rsrc: pble resource management
* @palloc: level 2 pble allocation
*/
static enum irdma_status_code
get_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_pble_alloc *palloc)
static int get_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_pble_alloc *palloc)
{
u32 lf4k, lflast, total, i;
u32 pblcnt = PBLE_PER_PAGE;
@@ -347,7 +340,7 @@ get_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_pble_level2 *lvl2 = &palloc->level2;
struct irdma_pble_info *root = &lvl2->root;
struct irdma_pble_info *leaf;
enum irdma_status_code ret_code;
int ret_code;
u64 fpm_addr;
/* number of full 512 (4K) leafs) */
@@ -359,7 +352,7 @@ get_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
lvl2->leafmem.size = (sizeof(*leaf) * total);
lvl2->leafmem.va = kzalloc(lvl2->leafmem.size, GFP_KERNEL);
if (!lvl2->leafmem.va)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
lvl2->leaf = lvl2->leafmem.va;
leaf = lvl2->leaf;
@@ -368,7 +361,7 @@ get_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
if (ret_code) {
kfree(lvl2->leafmem.va);
lvl2->leaf = NULL;
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
}
root->idx = fpm_to_idx(pble_rsrc, fpm_addr);
@@ -397,7 +390,7 @@ get_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
error:
free_lvl2(pble_rsrc, palloc);
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
}
/**
@@ -405,11 +398,10 @@ error:
* @pble_rsrc: pble resource management
* @palloc: level 1 pble allocation
*/
static enum irdma_status_code
get_lvl1_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_pble_alloc *palloc)
static int get_lvl1_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_pble_alloc *palloc)
{
enum irdma_status_code ret_code;
int ret_code;
u64 fpm_addr;
struct irdma_pble_info *lvl1 = &palloc->level1;
@@ -417,7 +409,7 @@ get_lvl1_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
palloc->total_cnt << 3, &lvl1->addr,
&fpm_addr);
if (ret_code)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
palloc->level = PBLE_LEVEL_1;
lvl1->idx = fpm_to_idx(pble_rsrc, fpm_addr);
@@ -433,11 +425,10 @@ get_lvl1_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
* @palloc: contains all inforamtion regarding pble (idx + pble addr)
* @level1_only: flag for a level 1 PBLE
*/
static enum irdma_status_code
get_lvl1_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_pble_alloc *palloc, bool level1_only)
static int get_lvl1_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_pble_alloc *palloc, bool level1_only)
{
enum irdma_status_code status = 0;
int status = 0;
status = get_lvl1_pble(pble_rsrc, palloc);
if (!status || level1_only || palloc->total_cnt <= PBLE_PER_PAGE)
@@ -455,11 +446,11 @@ get_lvl1_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
* @pble_cnt: #of pbles requested
* @level1_only: true if only pble level 1 to acquire
*/
enum irdma_status_code irdma_get_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_pble_alloc *palloc,
u32 pble_cnt, bool level1_only)
int irdma_get_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_pble_alloc *palloc, u32 pble_cnt,
bool level1_only)
{
enum irdma_status_code status = 0;
int status = 0;
int max_sds = 0;
int i;

25
drivers/infiniband/hw/irdma/pble.h
View File

@@ -108,20 +108,18 @@ struct irdma_hmc_pble_rsrc {
};
void irdma_destroy_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc);
enum irdma_status_code
irdma_hmc_init_pble(struct irdma_sc_dev *dev,
struct irdma_hmc_pble_rsrc *pble_rsrc);
int irdma_hmc_init_pble(struct irdma_sc_dev *dev,
struct irdma_hmc_pble_rsrc *pble_rsrc);
void irdma_free_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_pble_alloc *palloc);
enum irdma_status_code irdma_get_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_pble_alloc *palloc,
u32 pble_cnt, bool level1_only);
enum irdma_status_code irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm,
struct irdma_chunk *pchunk);
enum irdma_status_code
irdma_prm_get_pbles(struct irdma_pble_prm *pprm,
struct irdma_pble_chunkinfo *chunkinfo, u64 mem_size,
u64 **vaddr, u64 *fpm_addr);
int irdma_get_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_pble_alloc *palloc, u32 pble_cnt,
bool level1_only);
int irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm,
struct irdma_chunk *pchunk);
int irdma_prm_get_pbles(struct irdma_pble_prm *pprm,
struct irdma_pble_chunkinfo *chunkinfo, u64 mem_size,
u64 **vaddr, u64 *fpm_addr);
void irdma_prm_return_pbles(struct irdma_pble_prm *pprm,
struct irdma_pble_chunkinfo *chunkinfo);
void irdma_pble_acquire_lock(struct irdma_hmc_pble_rsrc *pble_rsrc,
@@ -129,7 +127,6 @@ void irdma_pble_acquire_lock(struct irdma_hmc_pble_rsrc *pble_rsrc,
void irdma_pble_release_lock(struct irdma_hmc_pble_rsrc *pble_rsrc,
unsigned long *flags);
void irdma_pble_free_paged_mem(struct irdma_chunk *chunk);
enum irdma_status_code irdma_pble_get_paged_mem(struct irdma_chunk *chunk,
u32 pg_cnt);
int irdma_pble_get_paged_mem(struct irdma_chunk *chunk, u32 pg_cnt);
void irdma_prm_rem_bitmapmem(struct irdma_hw *hw, struct irdma_chunk *chunk);
#endif /* IRDMA_PBLE_H */

90
drivers/infiniband/hw/irdma/protos.h
View File

@@ -12,58 +12,51 @@
#define CQP_TIMEOUT_THRESHOLD 500
/* init operations */
enum irdma_status_code irdma_sc_dev_init(enum irdma_vers ver,
struct irdma_sc_dev *dev,
struct irdma_device_init_info *info);
int irdma_sc_dev_init(enum irdma_vers ver, struct irdma_sc_dev *dev,
struct irdma_device_init_info *info);
void irdma_sc_rt_init(struct irdma_sc_dev *dev);
void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp);
__le64 *irdma_sc_cqp_get_next_send_wqe(struct irdma_sc_cqp *cqp, u64 scratch);
enum irdma_status_code
irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,
struct irdma_fast_reg_stag_info *info, bool post_sq);
int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,
struct irdma_fast_reg_stag_info *info,
bool post_sq);
/* HMC/FPM functions */
enum irdma_status_code irdma_sc_init_iw_hmc(struct irdma_sc_dev *dev,
u8 hmc_fn_id);
int irdma_sc_init_iw_hmc(struct irdma_sc_dev *dev, u8 hmc_fn_id);
/* stats misc */
enum irdma_status_code
irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev,
struct irdma_vsi_pestat *pestat, bool wait);
int irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev,
struct irdma_vsi_pestat *pestat, bool wait);
void irdma_cqp_gather_stats_gen1(struct irdma_sc_dev *dev,
struct irdma_vsi_pestat *pestat);
void irdma_hw_stats_read_all(struct irdma_vsi_pestat *stats,
struct irdma_dev_hw_stats *stats_values,
u64 *hw_stats_regs_32, u64 *hw_stats_regs_64,
u8 hw_rev);
enum irdma_status_code
irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd,
struct irdma_ws_node_info *node_info);
enum irdma_status_code irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev,
struct irdma_sc_ceq *sc_ceq, u8 op);
enum irdma_status_code irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev,
struct irdma_sc_aeq *sc_aeq, u8 op);
enum irdma_status_code
irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi, u8 cmd,
struct irdma_stats_inst_info *stats_info);
int irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd,
struct irdma_ws_node_info *node_info);
int irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_ceq *sc_ceq,
u8 op);
int irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_aeq *sc_aeq,
u8 op);
int irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi, u8 cmd,
struct irdma_stats_inst_info *stats_info);
u16 irdma_alloc_ws_node_id(struct irdma_sc_dev *dev);
void irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id);
void irdma_update_stats(struct irdma_dev_hw_stats *hw_stats,
struct irdma_gather_stats *gather_stats,
struct irdma_gather_stats *last_gather_stats);
/* vsi functions */
enum irdma_status_code irdma_vsi_stats_init(struct irdma_sc_vsi *vsi,
struct irdma_vsi_stats_info *info);
int irdma_vsi_stats_init(struct irdma_sc_vsi *vsi,
struct irdma_vsi_stats_info *info);
void irdma_vsi_stats_free(struct irdma_sc_vsi *vsi);
void irdma_sc_vsi_init(struct irdma_sc_vsi *vsi,
struct irdma_vsi_init_info *info);
enum irdma_status_code irdma_sc_add_cq_ctx(struct irdma_sc_ceq *ceq,
struct irdma_sc_cq *cq);
int irdma_sc_add_cq_ctx(struct irdma_sc_ceq *ceq, struct irdma_sc_cq *cq);
void irdma_sc_remove_cq_ctx(struct irdma_sc_ceq *ceq, struct irdma_sc_cq *cq);
/* misc L2 param change functions */
void irdma_change_l2params(struct irdma_sc_vsi *vsi,
struct irdma_l2params *l2params);
void irdma_sc_suspend_resume_qps(struct irdma_sc_vsi *vsi, u8 suspend);
enum irdma_status_code irdma_cqp_qp_suspend_resume(struct irdma_sc_qp *qp,
u8 cmd);
int irdma_cqp_qp_suspend_resume(struct irdma_sc_qp *qp, u8 cmd);
void irdma_qp_add_qos(struct irdma_sc_qp *qp);
void irdma_qp_rem_qos(struct irdma_sc_qp *qp);
struct irdma_sc_qp *irdma_get_qp_from_list(struct list_head *head,
@@ -81,31 +74,26 @@ void irdma_terminate_received(struct irdma_sc_qp *qp,
/* misc */
u8 irdma_get_encoded_wqe_size(u32 wqsize, enum irdma_queue_type queue_type);
void irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp);
enum irdma_status_code
irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch,
u8 hmc_fn_id, bool post_sq,
bool poll_registers);
enum irdma_status_code irdma_cfg_fpm_val(struct irdma_sc_dev *dev,
u32 qp_count);
enum irdma_status_code irdma_get_rdma_features(struct irdma_sc_dev *dev);
int irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch,
u8 hmc_fn_id, bool post_sq,
bool poll_registers);
int irdma_cfg_fpm_val(struct irdma_sc_dev *dev, u32 qp_count);
int irdma_get_rdma_features(struct irdma_sc_dev *dev);
void free_sd_mem(struct irdma_sc_dev *dev);
enum irdma_status_code irdma_process_cqp_cmd(struct irdma_sc_dev *dev,
struct cqp_cmds_info *pcmdinfo);
enum irdma_status_code irdma_process_bh(struct irdma_sc_dev *dev);
enum irdma_status_code irdma_cqp_sds_cmd(struct irdma_sc_dev *dev,
struct irdma_update_sds_info *info);
enum irdma_status_code
irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev *dev,
struct irdma_dma_mem *val_mem, u8 hmc_fn_id);
enum irdma_status_code
irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev *dev,
struct irdma_dma_mem *val_mem, u8 hmc_fn_id);
enum irdma_status_code irdma_alloc_query_fpm_buf(struct irdma_sc_dev *dev,
struct irdma_dma_mem *mem);
enum irdma_status_code
irdma_cqp_manage_hmc_fcn_cmd(struct irdma_sc_dev *dev,
struct irdma_hmc_fcn_info *hmcfcninfo,
u16 *pmf_idx);
int irdma_process_cqp_cmd(struct irdma_sc_dev *dev,
struct cqp_cmds_info *pcmdinfo);
int irdma_process_bh(struct irdma_sc_dev *dev);
int irdma_cqp_sds_cmd(struct irdma_sc_dev *dev,
struct irdma_update_sds_info *info);
int irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev *dev,
struct irdma_dma_mem *val_mem, u8 hmc_fn_id);
int irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev *dev,
struct irdma_dma_mem *val_mem, u8 hmc_fn_id);
int irdma_alloc_query_fpm_buf(struct irdma_sc_dev *dev,
struct irdma_dma_mem *mem);
int irdma_cqp_manage_hmc_fcn_cmd(struct irdma_sc_dev *dev,
struct irdma_hmc_fcn_info *hmcfcninfo,
u16 *pmf_idx);
void irdma_add_dev_ref(struct irdma_sc_dev *dev);
void irdma_put_dev_ref(struct irdma_sc_dev *dev);
void *irdma_remove_cqp_head(struct irdma_sc_dev *dev);

132
drivers/infiniband/hw/irdma/puda.c
View File

@@ -1,7 +1,6 @@
// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
/* Copyright (c) 2015 - 2021 Intel Corporation */
#include "osdep.h"
#include "status.h"
#include "hmc.h"
#include "defs.h"
#include "type.h"
@@ -114,8 +113,7 @@ static void irdma_puda_post_recvbuf(struct irdma_puda_rsrc *rsrc, u32 wqe_idx,
* @rsrc: resource to use for buffer
* @initial: flag if during init time
*/
static enum irdma_status_code
irdma_puda_replenish_rq(struct irdma_puda_rsrc *rsrc, bool initial)
static int irdma_puda_replenish_rq(struct irdma_puda_rsrc *rsrc, bool initial)
{
u32 i;
u32 invalid_cnt = rsrc->rxq_invalid_cnt;
@@ -124,7 +122,7 @@ irdma_puda_replenish_rq(struct irdma_puda_rsrc *rsrc, bool initial)
for (i = 0; i < invalid_cnt; i++) {
buf = irdma_puda_get_bufpool(rsrc);
if (!buf)
return IRDMA_ERR_list_empty;
return -ENOBUFS;
irdma_puda_post_recvbuf(rsrc, rsrc->rx_wqe_idx, buf, initial);
rsrc->rx_wqe_idx = ((rsrc->rx_wqe_idx + 1) % rsrc->rq_size);
rsrc->rxq_invalid_cnt--;
@@ -194,7 +192,7 @@ static __le64 *irdma_puda_get_next_send_wqe(struct irdma_qp_uk *qp,
u32 *wqe_idx)
{
__le64 *wqe = NULL;
enum irdma_status_code ret_code = 0;
int ret_code = 0;
*wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring);
if (!*wqe_idx)
@@ -213,8 +211,8 @@ static __le64 *irdma_puda_get_next_send_wqe(struct irdma_qp_uk *qp,
* @cq: cq for poll
* @info: info return for successful completion
*/
static enum irdma_status_code
irdma_puda_poll_info(struct irdma_sc_cq *cq, struct irdma_puda_cmpl_info *info)
static int irdma_puda_poll_info(struct irdma_sc_cq *cq,
struct irdma_puda_cmpl_info *info)
{
struct irdma_cq_uk *cq_uk = &cq->cq_uk;
u64 qword0, qword2, qword3, qword6;
@@ -233,7 +231,7 @@ irdma_puda_poll_info(struct irdma_sc_cq *cq, struct irdma_puda_cmpl_info *info)
get_64bit_val(cqe, 24, &qword3);
valid_bit = (bool)FIELD_GET(IRDMA_CQ_VALID, qword3);
if (valid_bit != cq_uk->polarity)
return IRDMA_ERR_Q_EMPTY;
return -ENOENT;
if (cq->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
ext_valid = (bool)FIELD_GET(IRDMA_CQ_EXTCQE, qword3);
@@ -246,7 +244,7 @@ irdma_puda_poll_info(struct irdma_sc_cq *cq, struct irdma_puda_cmpl_info *info)
if (!peek_head)
polarity ^= 1;
if (polarity != cq_uk->polarity)
return IRDMA_ERR_Q_EMPTY;
return -ENOENT;
IRDMA_RING_MOVE_HEAD_NOCHECK(cq_uk->cq_ring);
if (!IRDMA_RING_CURRENT_HEAD(cq_uk->cq_ring))
@@ -267,7 +265,7 @@ irdma_puda_poll_info(struct irdma_sc_cq *cq, struct irdma_puda_cmpl_info *info)
major_err = (u32)(FIELD_GET(IRDMA_CQ_MAJERR, qword3));
minor_err = (u32)(FIELD_GET(IRDMA_CQ_MINERR, qword3));
info->compl_error = major_err << 16 | minor_err;
return IRDMA_ERR_CQ_COMPL_ERROR;
return -EIO;
}
get_64bit_val(cqe, 0, &qword0);
@@ -319,14 +317,13 @@ irdma_puda_poll_info(struct irdma_sc_cq *cq, struct irdma_puda_cmpl_info *info)
* @cq: cq getting interrupt
* @compl_err: return any completion err
*/
enum irdma_status_code irdma_puda_poll_cmpl(struct irdma_sc_dev *dev,
struct irdma_sc_cq *cq,
u32 *compl_err)
int irdma_puda_poll_cmpl(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq,
u32 *compl_err)
{
struct irdma_qp_uk *qp;
struct irdma_cq_uk *cq_uk = &cq->cq_uk;
struct irdma_puda_cmpl_info info = {};
enum irdma_status_code ret = 0;
int ret = 0;
struct irdma_puda_buf *buf;
struct irdma_puda_rsrc *rsrc;
u8 cq_type = cq->cq_type;
@@ -337,24 +334,24 @@ enum irdma_status_code irdma_puda_poll_cmpl(struct irdma_sc_dev *dev,
cq->vsi->ieq;
} else {
ibdev_dbg(to_ibdev(dev), "PUDA: qp_type error\n");
return IRDMA_ERR_BAD_PTR;
return -EINVAL;
}
ret = irdma_puda_poll_info(cq, &info);
*compl_err = info.compl_error;
if (ret == IRDMA_ERR_Q_EMPTY)
if (ret == -ENOENT)
return ret;
if (ret)
goto done;
qp = info.qp;
if (!qp || !rsrc) {
ret = IRDMA_ERR_BAD_PTR;
ret = -EFAULT;
goto done;
}
if (qp->qp_id != rsrc->qp_id) {
ret = IRDMA_ERR_BAD_PTR;
ret = -EFAULT;
goto done;
}
@@ -422,8 +419,7 @@ done:
* @qp: puda qp for send
* @info: buffer information for transmit
*/
enum irdma_status_code irdma_puda_send(struct irdma_sc_qp *qp,
struct irdma_puda_send_info *info)
int irdma_puda_send(struct irdma_sc_qp *qp, struct irdma_puda_send_info *info)
{
__le64 *wqe;
u32 iplen, l4len;
@@ -443,7 +439,7 @@ enum irdma_status_code irdma_puda_send(struct irdma_sc_qp *qp,
wqe = irdma_puda_get_next_send_wqe(&qp->qp_uk, &wqe_idx);
if (!wqe)
return IRDMA_ERR_QP_TOOMANY_WRS_POSTED;
return -ENOMEM;
qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid = (uintptr_t)info->scratch;
/* Third line of WQE descriptor */
@@ -503,7 +499,7 @@ void irdma_puda_send_buf(struct irdma_puda_rsrc *rsrc,
struct irdma_puda_buf *buf)
{
struct irdma_puda_send_info info;
enum irdma_status_code ret = 0;
int ret = 0;
unsigned long flags;
spin_lock_irqsave(&rsrc->bufpool_lock, flags);
@@ -603,19 +599,18 @@ static void irdma_puda_qp_setctx(struct irdma_puda_rsrc *rsrc)
* @dev: Device
* @qp: Resource qp
*/
static enum irdma_status_code irdma_puda_qp_wqe(struct irdma_sc_dev *dev,
struct irdma_sc_qp *qp)
static int irdma_puda_qp_wqe(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
{
struct irdma_sc_cqp *cqp;
__le64 *wqe;
u64 hdr;
struct irdma_ccq_cqe_info compl_info;
enum irdma_status_code status = 0;
int status = 0;
cqp = dev->cqp;
wqe = irdma_sc_cqp_get_next_send_wqe(cqp, 0);
if (!wqe)
return IRDMA_ERR_RING_FULL;
return -ENOMEM;
set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
set_64bit_val(wqe, 40, qp->shadow_area_pa);
@@ -643,11 +638,11 @@ static enum irdma_status_code irdma_puda_qp_wqe(struct irdma_sc_dev *dev,
* irdma_puda_qp_create - create qp for resource
* @rsrc: resource to use for buffer
*/
static enum irdma_status_code irdma_puda_qp_create(struct irdma_puda_rsrc *rsrc)
static int irdma_puda_qp_create(struct irdma_puda_rsrc *rsrc)
{
struct irdma_sc_qp *qp = &rsrc->qp;
struct irdma_qp_uk *ukqp = &qp->qp_uk;
enum irdma_status_code ret = 0;
int ret = 0;
u32 sq_size, rq_size;
struct irdma_dma_mem *mem;
@@ -659,7 +654,7 @@ static enum irdma_status_code irdma_puda_qp_create(struct irdma_puda_rsrc *rsrc)
rsrc->qpmem.size, &rsrc->qpmem.pa,
GFP_KERNEL);
if (!rsrc->qpmem.va)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
mem = &rsrc->qpmem;
memset(mem->va, 0, rsrc->qpmem.size);
@@ -722,19 +717,18 @@ static enum irdma_status_code irdma_puda_qp_create(struct irdma_puda_rsrc *rsrc)
* @dev: Device
* @cq: resource for cq
*/
static enum irdma_status_code irdma_puda_cq_wqe(struct irdma_sc_dev *dev,
struct irdma_sc_cq *cq)
static int irdma_puda_cq_wqe(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)
{
__le64 *wqe;
struct irdma_sc_cqp *cqp;
u64 hdr;
struct irdma_ccq_cqe_info compl_info;
enum irdma_status_code status = 0;
int status = 0;
cqp = dev->cqp;
wqe = irdma_sc_cqp_get_next_send_wqe(cqp, 0);
if (!wqe)
return IRDMA_ERR_RING_FULL;
return -ENOMEM;
set_64bit_val(wqe, 0, cq->cq_uk.cq_size);
set_64bit_val(wqe, 8, (uintptr_t)cq >> 1);
@@ -775,11 +769,11 @@ static enum irdma_status_code irdma_puda_cq_wqe(struct irdma_sc_dev *dev,
* irdma_puda_cq_create - create cq for resource
* @rsrc: resource for which cq to create
*/
static enum irdma_status_code irdma_puda_cq_create(struct irdma_puda_rsrc *rsrc)
static int irdma_puda_cq_create(struct irdma_puda_rsrc *rsrc)
{
struct irdma_sc_dev *dev = rsrc->dev;
struct irdma_sc_cq *cq = &rsrc->cq;
enum irdma_status_code ret = 0;
int ret = 0;
u32 cqsize;
struct irdma_dma_mem *mem;
struct irdma_cq_init_info info = {};
@@ -792,7 +786,7 @@ static enum irdma_status_code irdma_puda_cq_create(struct irdma_puda_rsrc *rsrc)
rsrc->cqmem.va = dma_alloc_coherent(dev->hw->device, rsrc->cqmem.size,
&rsrc->cqmem.pa, GFP_KERNEL);
if (!rsrc->cqmem.va)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
mem = &rsrc->cqmem;
info.dev = dev;
@@ -833,7 +827,7 @@ error:
*/
static void irdma_puda_free_qp(struct irdma_puda_rsrc *rsrc)
{
enum irdma_status_code ret;
int ret;
struct irdma_ccq_cqe_info compl_info;
struct irdma_sc_dev *dev = rsrc->dev;
@@ -865,7 +859,7 @@ static void irdma_puda_free_qp(struct irdma_puda_rsrc *rsrc)
*/
static void irdma_puda_free_cq(struct irdma_puda_rsrc *rsrc)
{
enum irdma_status_code ret;
int ret;
struct irdma_ccq_cqe_info compl_info;
struct irdma_sc_dev *dev = rsrc->dev;
@@ -967,8 +961,7 @@ void irdma_puda_dele_rsrc(struct irdma_sc_vsi *vsi, enum puda_rsrc_type type,
* @rsrc: resource for buffer allocation
* @count: number of buffers to create
*/
static enum irdma_status_code irdma_puda_allocbufs(struct irdma_puda_rsrc *rsrc,
u32 count)
static int irdma_puda_allocbufs(struct irdma_puda_rsrc *rsrc, u32 count)
{
u32 i;
struct irdma_puda_buf *buf;
@@ -978,7 +971,7 @@ static enum irdma_status_code irdma_puda_allocbufs(struct irdma_puda_rsrc *rsrc,
buf = irdma_puda_alloc_buf(rsrc->dev, rsrc->buf_size);
if (!buf) {
rsrc->stats_buf_alloc_fail++;
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
}
irdma_puda_ret_bufpool(rsrc, buf);
rsrc->alloc_buf_count++;
@@ -1001,11 +994,11 @@ static enum irdma_status_code irdma_puda_allocbufs(struct irdma_puda_rsrc *rsrc,
* @vsi: sc VSI struct
* @info: resource information
*/
enum irdma_status_code irdma_puda_create_rsrc(struct irdma_sc_vsi *vsi,
struct irdma_puda_rsrc_info *info)
int irdma_puda_create_rsrc(struct irdma_sc_vsi *vsi,
struct irdma_puda_rsrc_info *info)
{
struct irdma_sc_dev *dev = vsi->dev;
enum irdma_status_code ret = 0;
int ret = 0;
struct irdma_puda_rsrc *rsrc;
u32 pudasize;
u32 sqwridsize, rqwridsize;
@@ -1023,12 +1016,12 @@ enum irdma_status_code irdma_puda_create_rsrc(struct irdma_sc_vsi *vsi,
vmem = &vsi->ieq_mem;
break;
default:
return IRDMA_NOT_SUPPORTED;
return -EOPNOTSUPP;
}
vmem->size = pudasize + sqwridsize + rqwridsize;
vmem->va = kzalloc(vmem->size, GFP_KERNEL);
if (!vmem->va)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
rsrc = vmem->va;
spin_lock_init(&rsrc->bufpool_lock);
@@ -1046,7 +1039,7 @@ enum irdma_status_code irdma_puda_create_rsrc(struct irdma_sc_vsi *vsi,
rsrc->xmit_complete = irdma_ieq_tx_compl;
break;
default:
return IRDMA_NOT_SUPPORTED;
return -EOPNOTSUPP;
}
rsrc->type = info->type;
@@ -1323,12 +1316,12 @@ static void irdma_ieq_compl_pfpdu(struct irdma_puda_rsrc *ieq,
* @buf: first receive buffer
* @fpdu_len: total length of fpdu
*/
static enum irdma_status_code
irdma_ieq_create_pbufl(struct irdma_pfpdu *pfpdu, struct list_head *rxlist,
struct list_head *pbufl, struct irdma_puda_buf *buf,
u16 fpdu_len)
static int irdma_ieq_create_pbufl(struct irdma_pfpdu *pfpdu,
struct list_head *rxlist,
struct list_head *pbufl,
struct irdma_puda_buf *buf, u16 fpdu_len)
{
enum irdma_status_code status = 0;
int status = 0;
struct irdma_puda_buf *nextbuf;
u32 nextseqnum;
u16 plen = fpdu_len - buf->datalen;
@@ -1338,13 +1331,13 @@ irdma_ieq_create_pbufl(struct irdma_pfpdu *pfpdu, struct list_head *rxlist,
do {
nextbuf = irdma_puda_get_listbuf(rxlist);
if (!nextbuf) {
status = IRDMA_ERR_list_empty;
status = -ENOBUFS;
break;
}
list_add_tail(&nextbuf->list, pbufl);
if (nextbuf->seqnum != nextseqnum) {
pfpdu->bad_seq_num++;
status = IRDMA_ERR_SEQ_NUM;
status = -ERANGE;
break;
}
if (nextbuf->datalen >= plen) {
@@ -1366,11 +1359,11 @@ irdma_ieq_create_pbufl(struct irdma_pfpdu *pfpdu, struct list_head *rxlist,
* @buf: receive buffer
* @fpdu_len: fpdu len in the buffer
*/
static enum irdma_status_code
irdma_ieq_handle_partial(struct irdma_puda_rsrc *ieq, struct irdma_pfpdu *pfpdu,
struct irdma_puda_buf *buf, u16 fpdu_len)
static int irdma_ieq_handle_partial(struct irdma_puda_rsrc *ieq,
struct irdma_pfpdu *pfpdu,
struct irdma_puda_buf *buf, u16 fpdu_len)
{
enum irdma_status_code status = 0;
int status = 0;
u8 *crcptr;
u32 mpacrc;
u32 seqnum = buf->seqnum;
@@ -1390,7 +1383,7 @@ irdma_ieq_handle_partial(struct irdma_puda_rsrc *ieq, struct irdma_pfpdu *pfpdu,
txbuf = irdma_puda_get_bufpool(ieq);
if (!txbuf) {
pfpdu->no_tx_bufs++;
status = IRDMA_ERR_NO_TXBUFS;
status = -ENOBUFS;
goto error;
}
@@ -1434,9 +1427,9 @@ error:
* @pfpdu: partial management per user qp
* @buf: receive buffer
*/
static enum irdma_status_code irdma_ieq_process_buf(struct irdma_puda_rsrc *ieq,
struct irdma_pfpdu *pfpdu,
struct irdma_puda_buf *buf)
static int irdma_ieq_process_buf(struct irdma_puda_rsrc *ieq,
struct irdma_pfpdu *pfpdu,
struct irdma_puda_buf *buf)
{
u16 fpdu_len = 0;
u16 datalen = buf->datalen;
@@ -1450,7 +1443,7 @@ static enum irdma_status_code irdma_ieq_process_buf(struct irdma_puda_rsrc *ieq,
bool partial = false;
struct irdma_puda_buf *txbuf;
struct list_head *rxlist = &pfpdu->rxlist;
enum irdma_status_code ret = 0;
int ret = 0;
ioffset = (u16)(buf->data - (u8 *)buf->mem.va);
while (datalen) {
@@ -1459,7 +1452,7 @@ static enum irdma_status_code irdma_ieq_process_buf(struct irdma_puda_rsrc *ieq,
ibdev_dbg(to_ibdev(ieq->dev),
"IEQ: error bad fpdu len\n");
list_add(&buf->list, rxlist);
return IRDMA_ERR_MPA_CRC;
return -EINVAL;
}
if (datalen < fpdu_len) {
@@ -1475,7 +1468,7 @@ static enum irdma_status_code irdma_ieq_process_buf(struct irdma_puda_rsrc *ieq,
list_add(&buf->list, rxlist);
ibdev_dbg(to_ibdev(ieq->dev),
"ERR: IRDMA_ERR_MPA_CRC\n");
return IRDMA_ERR_MPA_CRC;
return -EINVAL;
}
full++;
pfpdu->fpdu_processed++;
@@ -1490,7 +1483,7 @@ static enum irdma_status_code irdma_ieq_process_buf(struct irdma_puda_rsrc *ieq,
if (!txbuf) {
pfpdu->no_tx_bufs++;
list_add(&buf->list, rxlist);
return IRDMA_ERR_NO_TXBUFS;
return -ENOBUFS;
}
/* modify txbuf's buffer header */
irdma_ieq_setup_tx_buf(buf, txbuf);
@@ -1539,7 +1532,7 @@ void irdma_ieq_process_fpdus(struct irdma_sc_qp *qp,
struct irdma_pfpdu *pfpdu = &qp->pfpdu;
struct list_head *rxlist = &pfpdu->rxlist;
struct irdma_puda_buf *buf;
enum irdma_status_code status;
int status;
do {
if (list_empty(rxlist))
@@ -1557,7 +1550,7 @@ void irdma_ieq_process_fpdus(struct irdma_sc_qp *qp,
}
/* keep processing buffers from the head of the list */
status = irdma_ieq_process_buf(ieq, pfpdu, buf);
if (status == IRDMA_ERR_MPA_CRC) {
if (status == -EINVAL) {
pfpdu->mpa_crc_err = true;
while (!list_empty(rxlist)) {
buf = irdma_puda_get_listbuf(rxlist);
@@ -1576,8 +1569,7 @@ void irdma_ieq_process_fpdus(struct irdma_sc_qp *qp,
* @qp: qp pointer
* @buf: buf received on IEQ used to create AH
*/
static enum irdma_status_code irdma_ieq_create_ah(struct irdma_sc_qp *qp,
struct irdma_puda_buf *buf)
static int irdma_ieq_create_ah(struct irdma_sc_qp *qp, struct irdma_puda_buf *buf)
{
struct irdma_ah_info ah_info = {};

43
drivers/infiniband/hw/irdma/puda.h
View File

@@ -151,42 +151,33 @@ void irdma_puda_ret_bufpool(struct irdma_puda_rsrc *rsrc,
struct irdma_puda_buf *buf);
void irdma_puda_send_buf(struct irdma_puda_rsrc *rsrc,
struct irdma_puda_buf *buf);
enum irdma_status_code irdma_puda_send(struct irdma_sc_qp *qp,
struct irdma_puda_send_info *info);
enum irdma_status_code
irdma_puda_create_rsrc(struct irdma_sc_vsi *vsi,
struct irdma_puda_rsrc_info *info);
int irdma_puda_send(struct irdma_sc_qp *qp, struct irdma_puda_send_info *info);
int irdma_puda_create_rsrc(struct irdma_sc_vsi *vsi,
struct irdma_puda_rsrc_info *info);
void irdma_puda_dele_rsrc(struct irdma_sc_vsi *vsi, enum puda_rsrc_type type,
bool reset);
enum irdma_status_code irdma_puda_poll_cmpl(struct irdma_sc_dev *dev,
struct irdma_sc_cq *cq,
u32 *compl_err);
int irdma_puda_poll_cmpl(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq,
u32 *compl_err);
struct irdma_sc_qp *irdma_ieq_get_qp(struct irdma_sc_dev *dev,
struct irdma_puda_buf *buf);
enum irdma_status_code
irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
struct irdma_puda_buf *buf);
enum irdma_status_code irdma_ieq_check_mpacrc(struct shash_desc *desc,
void *addr, u32 len, u32 val);
enum irdma_status_code irdma_init_hash_desc(struct shash_desc **desc);
int irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
struct irdma_puda_buf *buf);
int irdma_ieq_check_mpacrc(struct shash_desc *desc, void *addr, u32 len, u32 val);
int irdma_init_hash_desc(struct shash_desc **desc);
void irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp);
void irdma_free_hash_desc(struct shash_desc *desc);
void irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len,
u32 seqnum);
enum irdma_status_code irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev,
struct irdma_sc_qp *qp);
enum irdma_status_code irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev,
struct irdma_sc_cq *cq);
enum irdma_status_code irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp);
void irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len, u32 seqnum);
int irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp);
int irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq);
int irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp);
void irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq);
void irdma_puda_ieq_get_ah_info(struct irdma_sc_qp *qp,
struct irdma_ah_info *ah_info);
enum irdma_status_code irdma_puda_create_ah(struct irdma_sc_dev *dev,
struct irdma_ah_info *ah_info,
bool wait, enum puda_rsrc_type type,
void *cb_param,
struct irdma_sc_ah **ah);
int irdma_puda_create_ah(struct irdma_sc_dev *dev,
struct irdma_ah_info *ah_info, bool wait,
enum puda_rsrc_type type, void *cb_param,
struct irdma_sc_ah **ah);
void irdma_puda_free_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah);
void irdma_ieq_process_fpdus(struct irdma_sc_qp *qp,
struct irdma_puda_rsrc *ieq);

71
drivers/infiniband/hw/irdma/status.h
View File

@@ -1,71 +0,0 @@
/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */
/* Copyright (c) 2015 - 2020 Intel Corporation */
#ifndef IRDMA_STATUS_H
#define IRDMA_STATUS_H
/* Error Codes */
enum irdma_status_code {
IRDMA_SUCCESS = 0,
IRDMA_ERR_NVM = -1,
IRDMA_ERR_NVM_CHECKSUM = -2,
IRDMA_ERR_CFG = -4,
IRDMA_ERR_PARAM = -5,
IRDMA_ERR_DEVICE_NOT_SUPPORTED = -6,
IRDMA_ERR_RESET_FAILED = -7,
IRDMA_ERR_SWFW_SYNC = -8,
IRDMA_ERR_NO_MEMORY = -9,
IRDMA_ERR_BAD_PTR = -10,
IRDMA_ERR_INVALID_PD_ID = -11,
IRDMA_ERR_INVALID_QP_ID = -12,
IRDMA_ERR_INVALID_CQ_ID = -13,
IRDMA_ERR_INVALID_CEQ_ID = -14,
IRDMA_ERR_INVALID_AEQ_ID = -15,
IRDMA_ERR_INVALID_SIZE = -16,
IRDMA_ERR_INVALID_ARP_INDEX = -17,
IRDMA_ERR_INVALID_FPM_FUNC_ID = -18,
IRDMA_ERR_QP_INVALID_MSG_SIZE = -19,
IRDMA_ERR_QP_TOOMANY_WRS_POSTED = -20,
IRDMA_ERR_INVALID_FRAG_COUNT = -21,
IRDMA_ERR_Q_EMPTY = -22,
IRDMA_ERR_INVALID_ALIGNMENT = -23,
IRDMA_ERR_FLUSHED_Q = -24,
IRDMA_ERR_INVALID_PUSH_PAGE_INDEX = -25,
IRDMA_ERR_INVALID_INLINE_DATA_SIZE = -26,
IRDMA_ERR_TIMEOUT = -27,
IRDMA_ERR_OPCODE_MISMATCH = -28,
IRDMA_ERR_CQP_COMPL_ERROR = -29,
IRDMA_ERR_INVALID_VF_ID = -30,
IRDMA_ERR_INVALID_HMCFN_ID = -31,
IRDMA_ERR_BACKING_PAGE_ERROR = -32,
IRDMA_ERR_NO_PBLCHUNKS_AVAILABLE = -33,
IRDMA_ERR_INVALID_PBLE_INDEX = -34,
IRDMA_ERR_INVALID_SD_INDEX = -35,
IRDMA_ERR_INVALID_PAGE_DESC_INDEX = -36,
IRDMA_ERR_INVALID_SD_TYPE = -37,
IRDMA_ERR_MEMCPY_FAILED = -38,
IRDMA_ERR_INVALID_HMC_OBJ_INDEX = -39,
IRDMA_ERR_INVALID_HMC_OBJ_COUNT = -40,
IRDMA_ERR_BUF_TOO_SHORT = -43,
IRDMA_ERR_BAD_IWARP_CQE = -44,
IRDMA_ERR_NVM_BLANK_MODE = -45,
IRDMA_ERR_NOT_IMPL = -46,
IRDMA_ERR_PE_DOORBELL_NOT_ENA = -47,
IRDMA_ERR_NOT_READY = -48,
IRDMA_NOT_SUPPORTED = -49,
IRDMA_ERR_FIRMWARE_API_VER = -50,
IRDMA_ERR_RING_FULL = -51,
IRDMA_ERR_MPA_CRC = -61,
IRDMA_ERR_NO_TXBUFS = -62,
IRDMA_ERR_SEQ_NUM = -63,
IRDMA_ERR_list_empty = -64,
IRDMA_ERR_INVALID_MAC_ADDR = -65,
IRDMA_ERR_BAD_STAG = -66,
IRDMA_ERR_CQ_COMPL_ERROR = -67,
IRDMA_ERR_Q_DESTROYED = -68,
IRDMA_ERR_INVALID_FEAT_CNT = -69,
IRDMA_ERR_REG_CQ_FULL = -70,
IRDMA_ERR_VF_MSG_ERROR = -71,
IRDMA_ERR_NO_INTR = -72,
IRDMA_ERR_REG_QSET = -73,
};
#endif /* IRDMA_STATUS_H */

109
drivers/infiniband/hw/irdma/type.h
View File

@@ -2,7 +2,6 @@
/* Copyright (c) 2015 - 2021 Intel Corporation */
#ifndef IRDMA_TYPE_H
#define IRDMA_TYPE_H
#include "status.h"
#include "osdep.h"
#include "irdma.h"
#include "user.h"
@@ -402,8 +401,8 @@ struct irdma_sc_cqp {
u64 host_ctx_pa;
void *back_cqp;
struct irdma_sc_dev *dev;
enum irdma_status_code (*process_cqp_sds)(struct irdma_sc_dev *dev,
struct irdma_update_sds_info *info);
int (*process_cqp_sds)(struct irdma_sc_dev *dev,
struct irdma_update_sds_info *info);
struct irdma_dma_mem sdbuf;
struct irdma_ring sq_ring;
struct irdma_cqp_quanta *sq_base;
@@ -605,12 +604,14 @@ struct irdma_sc_vsi {
struct irdma_qos qos[IRDMA_MAX_USER_PRIORITY];
struct irdma_vsi_pestat *pestat;
atomic_t qp_suspend_reqs;
enum irdma_status_code (*register_qset)(struct irdma_sc_vsi *vsi,
struct irdma_ws_node *tc_node);
int (*register_qset)(struct irdma_sc_vsi *vsi,
struct irdma_ws_node *tc_node);
void (*unregister_qset)(struct irdma_sc_vsi *vsi,
struct irdma_ws_node *tc_node);
u8 qos_rel_bw;
u8 qos_prio_type;
u8 dscp_map[IIDC_MAX_DSCP_MAPPING];
bool dscp_mode:1;
};
struct irdma_sc_dev {
@@ -655,7 +656,7 @@ struct irdma_sc_dev {
bool vchnl_up:1;
bool ceq_valid:1;
u8 pci_rev;
enum irdma_status_code (*ws_add)(struct irdma_sc_vsi *vsi, u8 user_pri);
int (*ws_add)(struct irdma_sc_vsi *vsi, u8 user_pri);
void (*ws_remove)(struct irdma_sc_vsi *vsi, u8 user_pri);
void (*ws_reset)(struct irdma_sc_vsi *vsi);
};
@@ -735,11 +736,13 @@ struct irdma_l2params {
u16 qs_handle_list[IRDMA_MAX_USER_PRIORITY];
u16 mtu;
u8 up2tc[IRDMA_MAX_USER_PRIORITY];
u8 dscp_map[IIDC_MAX_DSCP_MAPPING];
u8 num_tc;
u8 vsi_rel_bw;
u8 vsi_prio_type;
bool mtu_changed:1;
bool tc_changed:1;
bool dscp_mode:1;
};
struct irdma_vsi_init_info {
@@ -750,8 +753,8 @@ struct irdma_vsi_init_info {
u16 pf_data_vsi_num;
enum irdma_vm_vf_type vm_vf_type;
u16 vm_id;
enum irdma_status_code (*register_qset)(struct irdma_sc_vsi *vsi,
struct irdma_ws_node *tc_node);
int (*register_qset)(struct irdma_sc_vsi *vsi,
struct irdma_ws_node *tc_node);
void (*unregister_qset)(struct irdma_sc_vsi *vsi,
struct irdma_ws_node *tc_node);
};
@@ -1198,29 +1201,27 @@ struct irdma_irq_ops {
};
void irdma_sc_ccq_arm(struct irdma_sc_cq *ccq);
enum irdma_status_code irdma_sc_ccq_create(struct irdma_sc_cq *ccq, u64 scratch,
bool check_overflow, bool post_sq);
enum irdma_status_code irdma_sc_ccq_destroy(struct irdma_sc_cq *ccq, u64 scratch,
bool post_sq);
enum irdma_status_code irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq *ccq,
struct irdma_ccq_cqe_info *info);
enum irdma_status_code irdma_sc_ccq_init(struct irdma_sc_cq *ccq,
struct irdma_ccq_init_info *info);
int irdma_sc_ccq_create(struct irdma_sc_cq *ccq, u64 scratch,
bool check_overflow, bool post_sq);
int irdma_sc_ccq_destroy(struct irdma_sc_cq *ccq, u64 scratch, bool post_sq);
int irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq *ccq,
struct irdma_ccq_cqe_info *info);
int irdma_sc_ccq_init(struct irdma_sc_cq *ccq,
struct irdma_ccq_init_info *info);
enum irdma_status_code irdma_sc_cceq_create(struct irdma_sc_ceq *ceq, u64 scratch);
enum irdma_status_code irdma_sc_cceq_destroy_done(struct irdma_sc_ceq *ceq);
int irdma_sc_cceq_create(struct irdma_sc_ceq *ceq, u64 scratch);
int irdma_sc_cceq_destroy_done(struct irdma_sc_ceq *ceq);
enum irdma_status_code irdma_sc_ceq_destroy(struct irdma_sc_ceq *ceq, u64 scratch,
bool post_sq);
enum irdma_status_code irdma_sc_ceq_init(struct irdma_sc_ceq *ceq,
struct irdma_ceq_init_info *info);
int irdma_sc_ceq_destroy(struct irdma_sc_ceq *ceq, u64 scratch, bool post_sq);
int irdma_sc_ceq_init(struct irdma_sc_ceq *ceq,
struct irdma_ceq_init_info *info);
void irdma_sc_cleanup_ceqes(struct irdma_sc_cq *cq, struct irdma_sc_ceq *ceq);
void *irdma_sc_process_ceq(struct irdma_sc_dev *dev, struct irdma_sc_ceq *ceq);
enum irdma_status_code irdma_sc_aeq_init(struct irdma_sc_aeq *aeq,
struct irdma_aeq_init_info *info);
enum irdma_status_code irdma_sc_get_next_aeqe(struct irdma_sc_aeq *aeq,
struct irdma_aeqe_info *info);
int irdma_sc_aeq_init(struct irdma_sc_aeq *aeq,
struct irdma_aeq_init_info *info);
int irdma_sc_get_next_aeqe(struct irdma_sc_aeq *aeq,
struct irdma_aeqe_info *info);
void irdma_sc_repost_aeq_entries(struct irdma_sc_dev *dev, u32 count);
void irdma_sc_pd_init(struct irdma_sc_dev *dev, struct irdma_sc_pd *pd, u32 pd_id,
@@ -1228,31 +1229,27 @@ void irdma_sc_pd_init(struct irdma_sc_dev *dev, struct irdma_sc_pd *pd, u32 pd_i
void irdma_cfg_aeq(struct irdma_sc_dev *dev, u32 idx, bool enable);
void irdma_check_cqp_progress(struct irdma_cqp_timeout *cqp_timeout,
struct irdma_sc_dev *dev);
enum irdma_status_code irdma_sc_cqp_create(struct irdma_sc_cqp *cqp, u16 *maj_err,
u16 *min_err);
enum irdma_status_code irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp);
enum irdma_status_code irdma_sc_cqp_init(struct irdma_sc_cqp *cqp,
struct irdma_cqp_init_info *info);
int irdma_sc_cqp_create(struct irdma_sc_cqp *cqp, u16 *maj_err, u16 *min_err);
int irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp);
int irdma_sc_cqp_init(struct irdma_sc_cqp *cqp,
struct irdma_cqp_init_info *info);
void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp);
enum irdma_status_code irdma_sc_poll_for_cqp_op_done(struct irdma_sc_cqp *cqp, u8 opcode,
struct irdma_ccq_cqe_info *cmpl_info);
enum irdma_status_code irdma_sc_fast_register(struct irdma_sc_qp *qp,
struct irdma_fast_reg_stag_info *info,
bool post_sq);
enum irdma_status_code irdma_sc_qp_create(struct irdma_sc_qp *qp,
struct irdma_create_qp_info *info,
u64 scratch, bool post_sq);
enum irdma_status_code irdma_sc_qp_destroy(struct irdma_sc_qp *qp,
u64 scratch, bool remove_hash_idx,
bool ignore_mw_bnd, bool post_sq);
enum irdma_status_code irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp,
struct irdma_qp_flush_info *info,
u64 scratch, bool post_sq);
enum irdma_status_code irdma_sc_qp_init(struct irdma_sc_qp *qp,
struct irdma_qp_init_info *info);
enum irdma_status_code irdma_sc_qp_modify(struct irdma_sc_qp *qp,
struct irdma_modify_qp_info *info,
u64 scratch, bool post_sq);
int irdma_sc_poll_for_cqp_op_done(struct irdma_sc_cqp *cqp, u8 opcode,
struct irdma_ccq_cqe_info *cmpl_info);
int irdma_sc_fast_register(struct irdma_sc_qp *qp,
struct irdma_fast_reg_stag_info *info, bool post_sq);
int irdma_sc_qp_create(struct irdma_sc_qp *qp,
struct irdma_create_qp_info *info, u64 scratch,
bool post_sq);
int irdma_sc_qp_destroy(struct irdma_sc_qp *qp, u64 scratch,
bool remove_hash_idx, bool ignore_mw_bnd, bool post_sq);
int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp,
struct irdma_qp_flush_info *info, u64 scratch,
bool post_sq);
int irdma_sc_qp_init(struct irdma_sc_qp *qp, struct irdma_qp_init_info *info);
int irdma_sc_qp_modify(struct irdma_sc_qp *qp,
struct irdma_modify_qp_info *info, u64 scratch,
bool post_sq);
void irdma_sc_send_lsmm(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size,
irdma_stag stag);
@@ -1261,14 +1258,12 @@ void irdma_sc_qp_setctx(struct irdma_sc_qp *qp, __le64 *qp_ctx,
struct irdma_qp_host_ctx_info *info);
void irdma_sc_qp_setctx_roce(struct irdma_sc_qp *qp, __le64 *qp_ctx,
struct irdma_qp_host_ctx_info *info);
enum irdma_status_code irdma_sc_cq_destroy(struct irdma_sc_cq *cq, u64 scratch,
bool post_sq);
enum irdma_status_code irdma_sc_cq_init(struct irdma_sc_cq *cq,
struct irdma_cq_init_info *info);
int irdma_sc_cq_destroy(struct irdma_sc_cq *cq, u64 scratch, bool post_sq);
int irdma_sc_cq_init(struct irdma_sc_cq *cq, struct irdma_cq_init_info *info);
void irdma_sc_cq_resize(struct irdma_sc_cq *cq, struct irdma_modify_cq_info *info);
enum irdma_status_code irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp,
u64 scratch, u8 hmc_fn_id,
bool post_sq, bool poll_registers);
int irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch,
u8 hmc_fn_id, bool post_sq,
bool poll_registers);
void sc_vsi_update_stats(struct irdma_sc_vsi *vsi);
struct cqp_info {

40
drivers/infiniband/hw/irdma/uda.c
View File

@@ -3,7 +3,6 @@
#include <linux/etherdevice.h>
#include "osdep.h"
#include "status.h"
#include "hmc.h"
#include "defs.h"
#include "type.h"
@@ -18,16 +17,15 @@
* @op: Operation
* @scratch: u64 saved to be used during cqp completion
*/
enum irdma_status_code irdma_sc_access_ah(struct irdma_sc_cqp *cqp,
struct irdma_ah_info *info,
u32 op, u64 scratch)
int irdma_sc_access_ah(struct irdma_sc_cqp *cqp, struct irdma_ah_info *info,
u32 op, u64 scratch)
{
__le64 *wqe;
u64 qw1, qw2;
wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
if (!wqe)
return IRDMA_ERR_RING_FULL;
return -ENOMEM;
set_64bit_val(wqe, 0, ether_addr_to_u64(info->mac_addr) << 16);
qw1 = FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_PDINDEXLO, info->pd_idx) |
@@ -86,8 +84,7 @@ enum irdma_status_code irdma_sc_access_ah(struct irdma_sc_cqp *cqp,
* irdma_create_mg_ctx() - create a mcg context
* @info: multicast group context info
*/
static enum irdma_status_code
irdma_create_mg_ctx(struct irdma_mcast_grp_info *info)
static void irdma_create_mg_ctx(struct irdma_mcast_grp_info *info)
{
struct irdma_mcast_grp_ctx_entry_info *entry_info = NULL;
u8 idx = 0; /* index in the array */
@@ -106,8 +103,6 @@ irdma_create_mg_ctx(struct irdma_mcast_grp_info *info)
ctx_idx++;
}
}
return 0;
}
/**
@@ -117,27 +112,24 @@ irdma_create_mg_ctx(struct irdma_mcast_grp_info *info)
* @op: operation to perform
* @scratch: u64 saved to be used during cqp completion
*/
enum irdma_status_code irdma_access_mcast_grp(struct irdma_sc_cqp *cqp,
struct irdma_mcast_grp_info *info,
u32 op, u64 scratch)
int irdma_access_mcast_grp(struct irdma_sc_cqp *cqp,
struct irdma_mcast_grp_info *info, u32 op,
u64 scratch)
{
__le64 *wqe;
enum irdma_status_code ret_code = 0;
if (info->mg_id >= IRDMA_UDA_MAX_FSI_MGS) {
ibdev_dbg(to_ibdev(cqp->dev), "WQE: mg_id out of range\n");
return IRDMA_ERR_PARAM;
return -EINVAL;
}
wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
if (!wqe) {
ibdev_dbg(to_ibdev(cqp->dev), "WQE: ring full\n");
return IRDMA_ERR_RING_FULL;
return -ENOMEM;
}
ret_code = irdma_create_mg_ctx(info);
if (ret_code)
return ret_code;
irdma_create_mg_ctx(info);
set_64bit_val(wqe, 32, info->dma_mem_mc.pa);
set_64bit_val(wqe, 16,
@@ -198,8 +190,8 @@ static bool irdma_compare_mgs(struct irdma_mcast_grp_ctx_entry_info *entry1,
* @ctx: Multcast group context
* @mg: Multcast group info
*/
enum irdma_status_code irdma_sc_add_mcast_grp(struct irdma_mcast_grp_info *ctx,
struct irdma_mcast_grp_ctx_entry_info *mg)
int irdma_sc_add_mcast_grp(struct irdma_mcast_grp_info *ctx,
struct irdma_mcast_grp_ctx_entry_info *mg)
{
u32 idx;
bool free_entry_found = false;
@@ -228,7 +220,7 @@ enum irdma_status_code irdma_sc_add_mcast_grp(struct irdma_mcast_grp_info *ctx,
return 0;
}
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
}
/**
@@ -239,8 +231,8 @@ enum irdma_status_code irdma_sc_add_mcast_grp(struct irdma_mcast_grp_info *ctx,
* Finds and removes a specific mulicast group from context, all
* parameters must match to remove a multicast group.
*/
enum irdma_status_code irdma_sc_del_mcast_grp(struct irdma_mcast_grp_info *ctx,
struct irdma_mcast_grp_ctx_entry_info *mg)
int irdma_sc_del_mcast_grp(struct irdma_mcast_grp_info *ctx,
struct irdma_mcast_grp_ctx_entry_info *mg)
{
u32 idx;
@@ -269,5 +261,5 @@ enum irdma_status_code irdma_sc_del_mcast_grp(struct irdma_mcast_grp_info *ctx,
}
}
return IRDMA_ERR_PARAM;
return -EINVAL;
}

46
drivers/infiniband/hw/irdma/uda.h
View File

@@ -32,56 +32,54 @@ struct irdma_sc_ah {
struct irdma_ah_info ah_info;
};
enum irdma_status_code irdma_sc_add_mcast_grp(struct irdma_mcast_grp_info *ctx,
struct irdma_mcast_grp_ctx_entry_info *mg);
enum irdma_status_code irdma_sc_del_mcast_grp(struct irdma_mcast_grp_info *ctx,
struct irdma_mcast_grp_ctx_entry_info *mg);
enum irdma_status_code irdma_sc_access_ah(struct irdma_sc_cqp *cqp, struct irdma_ah_info *info,
u32 op, u64 scratch);
enum irdma_status_code irdma_access_mcast_grp(struct irdma_sc_cqp *cqp,
struct irdma_mcast_grp_info *info,
u32 op, u64 scratch);
int irdma_sc_add_mcast_grp(struct irdma_mcast_grp_info *ctx,
struct irdma_mcast_grp_ctx_entry_info *mg);
int irdma_sc_del_mcast_grp(struct irdma_mcast_grp_info *ctx,
struct irdma_mcast_grp_ctx_entry_info *mg);
int irdma_sc_access_ah(struct irdma_sc_cqp *cqp, struct irdma_ah_info *info,
u32 op, u64 scratch);
int irdma_access_mcast_grp(struct irdma_sc_cqp *cqp,
struct irdma_mcast_grp_info *info, u32 op,
u64 scratch);
static inline void irdma_sc_init_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah)
{
ah->dev = dev;
}
static inline enum irdma_status_code irdma_sc_create_ah(struct irdma_sc_cqp *cqp,
struct irdma_ah_info *info,
u64 scratch)
static inline int irdma_sc_create_ah(struct irdma_sc_cqp *cqp,
struct irdma_ah_info *info, u64 scratch)
{
return irdma_sc_access_ah(cqp, info, IRDMA_CQP_OP_CREATE_ADDR_HANDLE,
scratch);
}
static inline enum irdma_status_code irdma_sc_destroy_ah(struct irdma_sc_cqp *cqp,
struct irdma_ah_info *info,
u64 scratch)
static inline int irdma_sc_destroy_ah(struct irdma_sc_cqp *cqp,
struct irdma_ah_info *info, u64 scratch)
{
return irdma_sc_access_ah(cqp, info, IRDMA_CQP_OP_DESTROY_ADDR_HANDLE,
scratch);
}
static inline enum irdma_status_code irdma_sc_create_mcast_grp(struct irdma_sc_cqp *cqp,
struct irdma_mcast_grp_info *info,
u64 scratch)
static inline int irdma_sc_create_mcast_grp(struct irdma_sc_cqp *cqp,
struct irdma_mcast_grp_info *info,
u64 scratch)
{
return irdma_access_mcast_grp(cqp, info, IRDMA_CQP_OP_CREATE_MCAST_GRP,
scratch);
}
static inline enum irdma_status_code irdma_sc_modify_mcast_grp(struct irdma_sc_cqp *cqp,
struct irdma_mcast_grp_info *info,
u64 scratch)
static inline int irdma_sc_modify_mcast_grp(struct irdma_sc_cqp *cqp,
struct irdma_mcast_grp_info *info,
u64 scratch)
{
return irdma_access_mcast_grp(cqp, info, IRDMA_CQP_OP_MODIFY_MCAST_GRP,
scratch);
}
static inline enum irdma_status_code irdma_sc_destroy_mcast_grp(struct irdma_sc_cqp *cqp,
struct irdma_mcast_grp_info *info,
u64 scratch)
static inline int irdma_sc_destroy_mcast_grp(struct irdma_sc_cqp *cqp,
struct irdma_mcast_grp_info *info,
u64 scratch)
{
return irdma_access_mcast_grp(cqp, info, IRDMA_CQP_OP_DESTROY_MCAST_GRP,
scratch);

122
drivers/infiniband/hw/irdma/uk.c
View File

@@ -1,7 +1,6 @@
// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
/* Copyright (c) 2015 - 2021 Intel Corporation */
#include "osdep.h"
#include "status.h"
#include "defs.h"
#include "user.h"
#include "irdma.h"
@@ -56,7 +55,7 @@ static void irdma_set_fragment_gen_1(__le64 *wqe, u32 offset,
* irdma_nop_1 - insert a NOP wqe
* @qp: hw qp ptr
*/
static enum irdma_status_code irdma_nop_1(struct irdma_qp_uk *qp)
static int irdma_nop_1(struct irdma_qp_uk *qp)
{
u64 hdr;
__le64 *wqe;
@@ -64,7 +63,7 @@ static enum irdma_status_code irdma_nop_1(struct irdma_qp_uk *qp)
bool signaled = false;
if (!qp->sq_ring.head)
return IRDMA_ERR_PARAM;
return -EINVAL;
wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring);
wqe = qp->sq_base[wqe_idx].elem;
@@ -245,7 +244,7 @@ __le64 *irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx,
__le64 *irdma_qp_get_next_recv_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx)
{
__le64 *wqe;
enum irdma_status_code ret_code;
int ret_code;
if (IRDMA_RING_FULL_ERR(qp->rq_ring))
return NULL;
@@ -268,16 +267,15 @@ __le64 *irdma_qp_get_next_recv_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx)
* @info: post sq information
* @post_sq: flag to post sq
*/
enum irdma_status_code irdma_uk_rdma_write(struct irdma_qp_uk *qp,
struct irdma_post_sq_info *info,
bool post_sq)
int irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
bool post_sq)
{
u64 hdr;
__le64 *wqe;
struct irdma_rdma_write *op_info;
u32 i, wqe_idx;
u32 total_size = 0, byte_off;
enum irdma_status_code ret_code;
int ret_code;
u32 frag_cnt, addl_frag_cnt;
bool read_fence = false;
u16 quanta;
@@ -286,7 +284,7 @@ enum irdma_status_code irdma_uk_rdma_write(struct irdma_qp_uk *qp,
op_info = &info->op.rdma_write;
if (op_info->num_lo_sges > qp->max_sq_frag_cnt)
return IRDMA_ERR_INVALID_FRAG_COUNT;
return -EINVAL;
for (i = 0; i < op_info->num_lo_sges; i++)
total_size += op_info->lo_sg_list[i].length;
@@ -305,7 +303,7 @@ enum irdma_status_code irdma_uk_rdma_write(struct irdma_qp_uk *qp,
wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
info);
if (!wqe)
return IRDMA_ERR_QP_TOOMANY_WRS_POSTED;
return -ENOMEM;
irdma_clr_wqes(qp, wqe_idx);
@@ -370,12 +368,11 @@ enum irdma_status_code irdma_uk_rdma_write(struct irdma_qp_uk *qp,
* @inv_stag: flag for inv_stag
* @post_sq: flag to post sq
*/
enum irdma_status_code irdma_uk_rdma_read(struct irdma_qp_uk *qp,
struct irdma_post_sq_info *info,
bool inv_stag, bool post_sq)
int irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
bool inv_stag, bool post_sq)
{
struct irdma_rdma_read *op_info;
enum irdma_status_code ret_code;
int ret_code;
u32 i, byte_off, total_size = 0;
bool local_fence = false;
u32 addl_frag_cnt;
@@ -388,7 +385,7 @@ enum irdma_status_code irdma_uk_rdma_read(struct irdma_qp_uk *qp,
op_info = &info->op.rdma_read;
if (qp->max_sq_frag_cnt < op_info->num_lo_sges)
return IRDMA_ERR_INVALID_FRAG_COUNT;
return -EINVAL;
for (i = 0; i < op_info->num_lo_sges; i++)
total_size += op_info->lo_sg_list[i].length;
@@ -400,7 +397,7 @@ enum irdma_status_code irdma_uk_rdma_read(struct irdma_qp_uk *qp,
wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
info);
if (!wqe)
return IRDMA_ERR_QP_TOOMANY_WRS_POSTED;
return -ENOMEM;
irdma_clr_wqes(qp, wqe_idx);
@@ -457,15 +454,14 @@ enum irdma_status_code irdma_uk_rdma_read(struct irdma_qp_uk *qp,
* @info: post sq information
* @post_sq: flag to post sq
*/
enum irdma_status_code irdma_uk_send(struct irdma_qp_uk *qp,
struct irdma_post_sq_info *info,
bool post_sq)
int irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
bool post_sq)
{
__le64 *wqe;
struct irdma_post_send *op_info;
u64 hdr;
u32 i, wqe_idx, total_size = 0, byte_off;
enum irdma_status_code ret_code;
int ret_code;
u32 frag_cnt, addl_frag_cnt;
bool read_fence = false;
u16 quanta;
@@ -474,7 +470,7 @@ enum irdma_status_code irdma_uk_send(struct irdma_qp_uk *qp,
op_info = &info->op.send;
if (qp->max_sq_frag_cnt < op_info->num_sges)
return IRDMA_ERR_INVALID_FRAG_COUNT;
return -EINVAL;
for (i = 0; i < op_info->num_sges; i++)
total_size += op_info->sg_list[i].length;
@@ -490,7 +486,7 @@ enum irdma_status_code irdma_uk_send(struct irdma_qp_uk *qp,
wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
info);
if (!wqe)
return IRDMA_ERR_QP_TOOMANY_WRS_POSTED;
return -ENOMEM;
irdma_clr_wqes(qp, wqe_idx);
@@ -678,9 +674,8 @@ static u16 irdma_inline_data_size_to_quanta(u32 data_size)
* @info: post sq information
* @post_sq: flag to post sq
*/
enum irdma_status_code
irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
bool post_sq)
int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
struct irdma_post_sq_info *info, bool post_sq)
{
__le64 *wqe;
struct irdma_inline_rdma_write *op_info;
@@ -693,13 +688,13 @@ irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *in
op_info = &info->op.inline_rdma_write;
if (op_info->len > qp->max_inline_data)
return IRDMA_ERR_INVALID_INLINE_DATA_SIZE;
return -EINVAL;
quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(op_info->len);
wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, op_info->len,
info);
if (!wqe)
return IRDMA_ERR_QP_TOOMANY_WRS_POSTED;
return -ENOMEM;
irdma_clr_wqes(qp, wqe_idx);
@@ -745,9 +740,8 @@ irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *in
* @info: post sq information
* @post_sq: flag to post sq
*/
enum irdma_status_code irdma_uk_inline_send(struct irdma_qp_uk *qp,
struct irdma_post_sq_info *info,
bool post_sq)
int irdma_uk_inline_send(struct irdma_qp_uk *qp,
struct irdma_post_sq_info *info, bool post_sq)
{
__le64 *wqe;
struct irdma_post_inline_send *op_info;
@@ -760,13 +754,13 @@ enum irdma_status_code irdma_uk_inline_send(struct irdma_qp_uk *qp,
op_info = &info->op.inline_send;
if (op_info->len > qp->max_inline_data)
return IRDMA_ERR_INVALID_INLINE_DATA_SIZE;
return -EINVAL;
quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(op_info->len);
wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, op_info->len,
info);
if (!wqe)
return IRDMA_ERR_QP_TOOMANY_WRS_POSTED;
return -ENOMEM;
irdma_clr_wqes(qp, wqe_idx);
@@ -817,9 +811,9 @@ enum irdma_status_code irdma_uk_inline_send(struct irdma_qp_uk *qp,
* @info: post sq information
* @post_sq: flag to post sq
*/
enum irdma_status_code
irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
struct irdma_post_sq_info *info, bool post_sq)
int irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
struct irdma_post_sq_info *info,
bool post_sq)
{
__le64 *wqe;
struct irdma_inv_local_stag *op_info;
@@ -835,7 +829,7 @@ irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, IRDMA_QP_WQE_MIN_QUANTA,
0, info);
if (!wqe)
return IRDMA_ERR_QP_TOOMANY_WRS_POSTED;
return -ENOMEM;
irdma_clr_wqes(qp, wqe_idx);
@@ -871,8 +865,8 @@ irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
* @qp: hw qp ptr
* @info: post rq information
*/
enum irdma_status_code irdma_uk_post_receive(struct irdma_qp_uk *qp,
struct irdma_post_rq_info *info)
int irdma_uk_post_receive(struct irdma_qp_uk *qp,
struct irdma_post_rq_info *info)
{
u32 wqe_idx, i, byte_off;
u32 addl_frag_cnt;
@@ -880,11 +874,11 @@ enum irdma_status_code irdma_uk_post_receive(struct irdma_qp_uk *qp,
u64 hdr;
if (qp->max_rq_frag_cnt < info->num_sges)
return IRDMA_ERR_INVALID_FRAG_COUNT;
return -EINVAL;
wqe = irdma_qp_get_next_recv_wqe(qp, &wqe_idx);
if (!wqe)
return IRDMA_ERR_QP_TOOMANY_WRS_POSTED;
return -ENOMEM;
qp->rq_wrid_array[wqe_idx] = info->wr_id;
addl_frag_cnt = info->num_sges > 1 ? (info->num_sges - 1) : 0;
@@ -1000,15 +994,15 @@ void irdma_uk_cq_request_notification(struct irdma_cq_uk *cq,
* @cq: hw cq
* @info: cq poll information returned
*/
enum irdma_status_code
irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq, struct irdma_cq_poll_info *info)
int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq,
struct irdma_cq_poll_info *info)
{
u64 comp_ctx, qword0, qword2, qword3;
__le64 *cqe;
struct irdma_qp_uk *qp;
struct irdma_ring *pring = NULL;
u32 wqe_idx, q_type;
enum irdma_status_code ret_code;
int ret_code;
bool move_cq_head = true;
u8 polarity;
bool ext_valid;
@@ -1022,7 +1016,7 @@ irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq, struct irdma_cq_poll_info *info)
get_64bit_val(cqe, 24, &qword3);
polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3);
if (polarity != cq->polarity)
return IRDMA_ERR_Q_EMPTY;
return -ENOENT;
/* Ensure CQE contents are read after valid bit is checked */
dma_rmb();
@@ -1045,7 +1039,7 @@ irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq, struct irdma_cq_poll_info *info)
polarity ^= 1;
}
if (polarity != cq->polarity)
return IRDMA_ERR_Q_EMPTY;
return -ENOENT;
/* Ensure ext CQE contents are read after ext valid bit is checked */
dma_rmb();
@@ -1112,7 +1106,7 @@ irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq, struct irdma_cq_poll_info *info)
info->solicited_event = (bool)FIELD_GET(IRDMACQ_SOEVENT, qword3);
qp = (struct irdma_qp_uk *)(unsigned long)comp_ctx;
if (!qp || qp->destroy_pending) {
ret_code = IRDMA_ERR_Q_DESTROYED;
ret_code = -EFAULT;
goto exit;
}
wqe_idx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, qword3);
@@ -1126,7 +1120,7 @@ irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq, struct irdma_cq_poll_info *info)
if (info->comp_status == IRDMA_COMPL_STATUS_FLUSHED ||
info->comp_status == IRDMA_COMPL_STATUS_UNKNOWN) {
if (!IRDMA_RING_MORE_WORK(qp->rq_ring)) {
ret_code = IRDMA_ERR_Q_EMPTY;
ret_code = -ENOENT;
goto exit;
}
@@ -1186,7 +1180,7 @@ irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq, struct irdma_cq_poll_info *info)
wqe_idx + qp->sq_wrtrk_array[wqe_idx].quanta);
} else {
if (!IRDMA_RING_MORE_WORK(qp->sq_ring)) {
ret_code = IRDMA_ERR_Q_EMPTY;
ret_code = -ENOENT;
goto exit;
}
@@ -1303,15 +1297,15 @@ void irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge,
* @sqdepth: depth of SQ
*
*/
enum irdma_status_code irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs,
u32 sq_size, u8 shift, u32 *sqdepth)
int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift,
u32 *sqdepth)
{
*sqdepth = irdma_qp_round_up((sq_size << shift) + IRDMA_SQ_RSVD);
if (*sqdepth < (IRDMA_QP_SW_MIN_WQSIZE << shift))
*sqdepth = IRDMA_QP_SW_MIN_WQSIZE << shift;
else if (*sqdepth > uk_attrs->max_hw_wq_quanta)
return IRDMA_ERR_INVALID_SIZE;
return -EINVAL;
return 0;
}
@@ -1323,15 +1317,15 @@ enum irdma_status_code irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs,
* @shift: shift which determines size of WQE
* @rqdepth: depth of RQ
*/
enum irdma_status_code irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs,
u32 rq_size, u8 shift, u32 *rqdepth)
int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift,
u32 *rqdepth)
{
*rqdepth = irdma_qp_round_up((rq_size << shift) + IRDMA_RQ_RSVD);
if (*rqdepth < (IRDMA_QP_SW_MIN_WQSIZE << shift))
*rqdepth = IRDMA_QP_SW_MIN_WQSIZE << shift;
else if (*rqdepth > uk_attrs->max_hw_rq_quanta)
return IRDMA_ERR_INVALID_SIZE;
return -EINVAL;
return 0;
}
@@ -1381,17 +1375,16 @@ static void irdma_setup_connection_wqes(struct irdma_qp_uk *qp,
* allowed. Then size of wqe * the number of wqes should be the
* amount of memory allocated for sq and rq.
*/
enum irdma_status_code irdma_uk_qp_init(struct irdma_qp_uk *qp,
struct irdma_qp_uk_init_info *info)
int irdma_uk_qp_init(struct irdma_qp_uk *qp, struct irdma_qp_uk_init_info *info)
{
enum irdma_status_code ret_code = 0;
int ret_code = 0;
u32 sq_ring_size;
u8 sqshift, rqshift;
qp->uk_attrs = info->uk_attrs;
if (info->max_sq_frag_cnt > qp->uk_attrs->max_hw_wq_frags ||
info->max_rq_frag_cnt > qp->uk_attrs->max_hw_wq_frags)
return IRDMA_ERR_INVALID_FRAG_COUNT;
return -EINVAL;
irdma_get_wqe_shift(qp->uk_attrs, info->max_rq_frag_cnt, 0, &rqshift);
if (qp->uk_attrs->hw_rev == IRDMA_GEN_1) {
@@ -1502,8 +1495,7 @@ void irdma_uk_clean_cq(void *q, struct irdma_cq_uk *cq)
* @signaled: signaled for completion
* @post_sq: ring doorbell
*/
enum irdma_status_code irdma_nop(struct irdma_qp_uk *qp, u64 wr_id,
bool signaled, bool post_sq)
int irdma_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled, bool post_sq)
{
__le64 *wqe;
u64 hdr;
@@ -1515,7 +1507,7 @@ enum irdma_status_code irdma_nop(struct irdma_qp_uk *qp, u64 wr_id,
wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, IRDMA_QP_WQE_MIN_QUANTA,
0, &info);
if (!wqe)
return IRDMA_ERR_QP_TOOMANY_WRS_POSTED;
return -ENOMEM;
irdma_clr_wqes(qp, wqe_idx);
@@ -1541,7 +1533,7 @@ enum irdma_status_code irdma_nop(struct irdma_qp_uk *qp, u64 wr_id,
* @frag_cnt: number of fragments
* @quanta: quanta for frag_cnt
*/
enum irdma_status_code irdma_fragcnt_to_quanta_sq(u32 frag_cnt, u16 *quanta)
int irdma_fragcnt_to_quanta_sq(u32 frag_cnt, u16 *quanta)
{
switch (frag_cnt) {
case 0:
@@ -1577,7 +1569,7 @@ enum irdma_status_code irdma_fragcnt_to_quanta_sq(u32 frag_cnt, u16 *quanta)
*quanta = 8;
break;
default:
return IRDMA_ERR_INVALID_FRAG_COUNT;
return -EINVAL;
}
return 0;
@@ -1588,7 +1580,7 @@ enum irdma_status_code irdma_fragcnt_to_quanta_sq(u32 frag_cnt, u16 *quanta)
* @frag_cnt: number of fragments
* @wqe_size: size in bytes given frag_cnt
*/
enum irdma_status_code irdma_fragcnt_to_wqesize_rq(u32 frag_cnt, u16 *wqe_size)
int irdma_fragcnt_to_wqesize_rq(u32 frag_cnt, u16 *wqe_size)
{
switch (frag_cnt) {
case 0:
@@ -1615,7 +1607,7 @@ enum irdma_status_code irdma_fragcnt_to_wqesize_rq(u32 frag_cnt, u16 *wqe_size)
*wqe_size = 256;
break;
default:
return IRDMA_ERR_INVALID_FRAG_COUNT;
return -EINVAL;
}
return 0;

62
drivers/infiniband/hw/irdma/user.h
View File

@@ -270,29 +270,24 @@ struct irdma_cq_poll_info {
bool imm_valid:1;
};
enum irdma_status_code irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
struct irdma_post_sq_info *info,
bool post_sq);
enum irdma_status_code irdma_uk_inline_send(struct irdma_qp_uk *qp,
struct irdma_post_sq_info *info,
bool post_sq);
enum irdma_status_code irdma_uk_post_nop(struct irdma_qp_uk *qp, u64 wr_id,
bool signaled, bool post_sq);
enum irdma_status_code irdma_uk_post_receive(struct irdma_qp_uk *qp,
struct irdma_post_rq_info *info);
int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
struct irdma_post_sq_info *info, bool post_sq);
int irdma_uk_inline_send(struct irdma_qp_uk *qp,
struct irdma_post_sq_info *info, bool post_sq);
int irdma_uk_post_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled,
bool post_sq);
int irdma_uk_post_receive(struct irdma_qp_uk *qp,
struct irdma_post_rq_info *info);
void irdma_uk_qp_post_wr(struct irdma_qp_uk *qp);
enum irdma_status_code irdma_uk_rdma_read(struct irdma_qp_uk *qp,
struct irdma_post_sq_info *info,
bool inv_stag, bool post_sq);
enum irdma_status_code irdma_uk_rdma_write(struct irdma_qp_uk *qp,
struct irdma_post_sq_info *info,
bool post_sq);
enum irdma_status_code irdma_uk_send(struct irdma_qp_uk *qp,
struct irdma_post_sq_info *info, bool post_sq);
enum irdma_status_code irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
struct irdma_post_sq_info *info,
bool post_sq);
int irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
bool inv_stag, bool post_sq);
int irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
bool post_sq);
int irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
bool post_sq);
int irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
struct irdma_post_sq_info *info,
bool post_sq);
struct irdma_wqe_uk_ops {
void (*iw_copy_inline_data)(u8 *dest, u8 *src, u32 len, u8 polarity);
@@ -303,16 +298,16 @@ struct irdma_wqe_uk_ops {
struct irdma_bind_window *op_info);
};
enum irdma_status_code irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq,
struct irdma_cq_poll_info *info);
int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq,
struct irdma_cq_poll_info *info);
void irdma_uk_cq_request_notification(struct irdma_cq_uk *cq,
enum irdma_cmpl_notify cq_notify);
void irdma_uk_cq_resize(struct irdma_cq_uk *cq, void *cq_base, int size);
void irdma_uk_cq_set_resized_cnt(struct irdma_cq_uk *qp, u16 cnt);
void irdma_uk_cq_init(struct irdma_cq_uk *cq,
struct irdma_cq_uk_init_info *info);
enum irdma_status_code irdma_uk_qp_init(struct irdma_qp_uk *qp,
struct irdma_qp_uk_init_info *info);
int irdma_uk_qp_init(struct irdma_qp_uk *qp,
struct irdma_qp_uk_init_info *info);
struct irdma_sq_uk_wr_trk_info {
u64 wrid;
u32 wr_len;
@@ -413,16 +408,15 @@ __le64 *irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx,
struct irdma_post_sq_info *info);
__le64 *irdma_qp_get_next_recv_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx);
void irdma_uk_clean_cq(void *q, struct irdma_cq_uk *cq);
enum irdma_status_code irdma_nop(struct irdma_qp_uk *qp, u64 wr_id,
bool signaled, bool post_sq);
enum irdma_status_code irdma_fragcnt_to_quanta_sq(u32 frag_cnt, u16 *quanta);
enum irdma_status_code irdma_fragcnt_to_wqesize_rq(u32 frag_cnt, u16 *wqe_size);
int irdma_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled, bool post_sq);
int irdma_fragcnt_to_quanta_sq(u32 frag_cnt, u16 *quanta);
int irdma_fragcnt_to_wqesize_rq(u32 frag_cnt, u16 *wqe_size);
void irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge,
u32 inline_data, u8 *shift);
enum irdma_status_code irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs,
u32 sq_size, u8 shift, u32 *wqdepth);
enum irdma_status_code irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs,
u32 rq_size, u8 shift, u32 *wqdepth);
int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift,
u32 *wqdepth);
int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift,
u32 *wqdepth);
void irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 *wqe, u16 quanta,
u32 wqe_idx, bool post_sq);
void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx);

247
drivers/infiniband/hw/irdma/utils.c
View File

@@ -150,31 +150,35 @@ int irdma_inetaddr_event(struct notifier_block *notifier, unsigned long event,
void *ptr)
{
struct in_ifaddr *ifa = ptr;
struct net_device *netdev = ifa->ifa_dev->dev;
struct net_device *real_dev, *netdev = ifa->ifa_dev->dev;
struct irdma_device *iwdev;
struct ib_device *ibdev;
u32 local_ipaddr;
ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_IRDMA);
real_dev = rdma_vlan_dev_real_dev(netdev);
if (!real_dev)
real_dev = netdev;
ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA);
if (!ibdev)
return NOTIFY_DONE;
iwdev = to_iwdev(ibdev);
local_ipaddr = ntohl(ifa->ifa_address);
ibdev_dbg(&iwdev->ibdev,
"DEV: netdev %p event %lu local_ip=%pI4 MAC=%pM\n", netdev,
event, &local_ipaddr, netdev->dev_addr);
"DEV: netdev %p event %lu local_ip=%pI4 MAC=%pM\n", real_dev,
event, &local_ipaddr, real_dev->dev_addr);
switch (event) {
case NETDEV_DOWN:
irdma_manage_arp_cache(iwdev->rf, netdev->dev_addr,
irdma_manage_arp_cache(iwdev->rf, real_dev->dev_addr,
&local_ipaddr, true, IRDMA_ARP_DELETE);
irdma_if_notify(iwdev, netdev, &local_ipaddr, true, false);
irdma_if_notify(iwdev, real_dev, &local_ipaddr, true, false);
irdma_gid_change_event(&iwdev->ibdev);
break;
case NETDEV_UP:
case NETDEV_CHANGEADDR:
irdma_add_arp(iwdev->rf, &local_ipaddr, true, netdev->dev_addr);
irdma_if_notify(iwdev, netdev, &local_ipaddr, true, true);
irdma_add_arp(iwdev->rf, &local_ipaddr, true, real_dev->dev_addr);
irdma_if_notify(iwdev, real_dev, &local_ipaddr, true, true);
irdma_gid_change_event(&iwdev->ibdev);
break;
default:
@@ -196,32 +200,36 @@ int irdma_inet6addr_event(struct notifier_block *notifier, unsigned long event,
void *ptr)
{
struct inet6_ifaddr *ifa = ptr;
struct net_device *netdev = ifa->idev->dev;
struct net_device *real_dev, *netdev = ifa->idev->dev;
struct irdma_device *iwdev;
struct ib_device *ibdev;
u32 local_ipaddr6[4];
ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_IRDMA);
real_dev = rdma_vlan_dev_real_dev(netdev);
if (!real_dev)
real_dev = netdev;
ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA);
if (!ibdev)
return NOTIFY_DONE;
iwdev = to_iwdev(ibdev);
irdma_copy_ip_ntohl(local_ipaddr6, ifa->addr.in6_u.u6_addr32);
ibdev_dbg(&iwdev->ibdev,
"DEV: netdev %p event %lu local_ip=%pI6 MAC=%pM\n", netdev,
event, local_ipaddr6, netdev->dev_addr);
"DEV: netdev %p event %lu local_ip=%pI6 MAC=%pM\n", real_dev,
event, local_ipaddr6, real_dev->dev_addr);
switch (event) {
case NETDEV_DOWN:
irdma_manage_arp_cache(iwdev->rf, netdev->dev_addr,
irdma_manage_arp_cache(iwdev->rf, real_dev->dev_addr,
local_ipaddr6, false, IRDMA_ARP_DELETE);
irdma_if_notify(iwdev, netdev, local_ipaddr6, false, false);
irdma_if_notify(iwdev, real_dev, local_ipaddr6, false, false);
irdma_gid_change_event(&iwdev->ibdev);
break;
case NETDEV_UP:
case NETDEV_CHANGEADDR:
irdma_add_arp(iwdev->rf, local_ipaddr6, false,
netdev->dev_addr);
irdma_if_notify(iwdev, netdev, local_ipaddr6, false, true);
real_dev->dev_addr);
irdma_if_notify(iwdev, real_dev, local_ipaddr6, false, true);
irdma_gid_change_event(&iwdev->ibdev);
break;
default:
@@ -243,14 +251,18 @@ int irdma_net_event(struct notifier_block *notifier, unsigned long event,
void *ptr)
{
struct neighbour *neigh = ptr;
struct net_device *real_dev, *netdev = (struct net_device *)neigh->dev;
struct irdma_device *iwdev;
struct ib_device *ibdev;
__be32 *p;
u32 local_ipaddr[4] = {};
bool ipv4 = true;
ibdev = ib_device_get_by_netdev((struct net_device *)neigh->dev,
RDMA_DRIVER_IRDMA);
real_dev = rdma_vlan_dev_real_dev(netdev);
if (!real_dev)
real_dev = netdev;
ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA);
if (!ibdev)
return NOTIFY_DONE;
@@ -551,12 +563,12 @@ void irdma_cleanup_pending_cqp_op(struct irdma_pci_f *rf)
* @rf: RDMA PCI function
* @cqp_request: cqp request to wait
*/
static enum irdma_status_code irdma_wait_event(struct irdma_pci_f *rf,
struct irdma_cqp_request *cqp_request)
static int irdma_wait_event(struct irdma_pci_f *rf,
struct irdma_cqp_request *cqp_request)
{
struct irdma_cqp_timeout cqp_timeout = {};
bool cqp_error = false;
enum irdma_status_code err_code = 0;
int err_code = 0;
cqp_timeout.compl_cqp_cmds = rf->sc_dev.cqp_cmd_stats[IRDMA_OP_CMPL_CMDS];
do {
@@ -575,12 +587,12 @@ static enum irdma_status_code irdma_wait_event(struct irdma_pci_f *rf,
rf->reset = true;
rf->gen_ops.request_reset(rf);
}
return IRDMA_ERR_TIMEOUT;
return -ETIMEDOUT;
} while (1);
cqp_error = cqp_request->compl_info.error;
if (cqp_error) {
err_code = IRDMA_ERR_CQP_COMPL_ERROR;
err_code = -EIO;
if (cqp_request->compl_info.maj_err_code == 0xFFFF &&
cqp_request->compl_info.min_err_code == 0x8029) {
if (!rf->reset) {
@@ -680,16 +692,16 @@ bool irdma_cqp_crit_err(struct irdma_sc_dev *dev, u8 cqp_cmd,
* @rf: RDMA PCI function
* @cqp_request: cqp request to process
*/
enum irdma_status_code irdma_handle_cqp_op(struct irdma_pci_f *rf,
struct irdma_cqp_request *cqp_request)
int irdma_handle_cqp_op(struct irdma_pci_f *rf,
struct irdma_cqp_request *cqp_request)
{
struct irdma_sc_dev *dev = &rf->sc_dev;
struct cqp_cmds_info *info = &cqp_request->info;
enum irdma_status_code status;
int status;
bool put_cqp_request = true;
if (rf->reset)
return IRDMA_ERR_NOT_READY;
return -EBUSY;
irdma_get_cqp_request(cqp_request);
status = irdma_process_cqp_cmd(dev, info);
@@ -791,17 +803,17 @@ void *irdma_remove_cqp_head(struct irdma_sc_dev *dev)
* @sdinfo: information for sd cqp
*
*/
enum irdma_status_code irdma_cqp_sds_cmd(struct irdma_sc_dev *dev,
struct irdma_update_sds_info *sdinfo)
int irdma_cqp_sds_cmd(struct irdma_sc_dev *dev,
struct irdma_update_sds_info *sdinfo)
{
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
struct irdma_pci_f *rf = dev_to_rf(dev);
enum irdma_status_code status;
int status;
cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
if (!cqp_request)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
cqp_info = &cqp_request->info;
memcpy(&cqp_info->in.u.update_pe_sds.info, sdinfo,
@@ -822,19 +834,18 @@ enum irdma_status_code irdma_cqp_sds_cmd(struct irdma_sc_dev *dev,
* @qp: hardware control qp
* @op: suspend or resume
*/
enum irdma_status_code irdma_cqp_qp_suspend_resume(struct irdma_sc_qp *qp,
u8 op)
int irdma_cqp_qp_suspend_resume(struct irdma_sc_qp *qp, u8 op)
{
struct irdma_sc_dev *dev = qp->dev;
struct irdma_cqp_request *cqp_request;
struct irdma_sc_cqp *cqp = dev->cqp;
struct cqp_cmds_info *cqp_info;
struct irdma_pci_f *rf = dev_to_rf(dev);
enum irdma_status_code status;
int status;
cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false);
if (!cqp_request)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
cqp_info = &cqp_request->info;
cqp_info->cqp_cmd = op;
@@ -940,18 +951,17 @@ void irdma_terminate_del_timer(struct irdma_sc_qp *qp)
* @val_mem: buffer for fpm
* @hmc_fn_id: function id for fpm
*/
enum irdma_status_code
irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev *dev,
struct irdma_dma_mem *val_mem, u8 hmc_fn_id)
int irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev *dev,
struct irdma_dma_mem *val_mem, u8 hmc_fn_id)
{
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
struct irdma_pci_f *rf = dev_to_rf(dev);
enum irdma_status_code status;
int status;
cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
if (!cqp_request)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
cqp_info = &cqp_request->info;
cqp_request->param = NULL;
@@ -975,18 +985,17 @@ irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev *dev,
* @val_mem: buffer with fpm values
* @hmc_fn_id: function id for fpm
*/
enum irdma_status_code
irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev *dev,
struct irdma_dma_mem *val_mem, u8 hmc_fn_id)
int irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev *dev,
struct irdma_dma_mem *val_mem, u8 hmc_fn_id)
{
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
struct irdma_pci_f *rf = dev_to_rf(dev);
enum irdma_status_code status;
int status;
cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
if (!cqp_request)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
cqp_info = &cqp_request->info;
cqp_request->param = NULL;
@@ -1009,18 +1018,17 @@ irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev *dev,
* @dev: device pointer
* @cq: pointer to created cq
*/
enum irdma_status_code irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev,
struct irdma_sc_cq *cq)
int irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)
{
struct irdma_pci_f *rf = dev_to_rf(dev);
struct irdma_cqp *iwcqp = &rf->cqp;
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
enum irdma_status_code status;
int status;
cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
if (!cqp_request)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
cqp_info = &cqp_request->info;
cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE;
@@ -1039,19 +1047,18 @@ enum irdma_status_code irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev,
* @dev: device pointer
* @qp: pointer to created qp
*/
enum irdma_status_code irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev,
struct irdma_sc_qp *qp)
int irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
{
struct irdma_pci_f *rf = dev_to_rf(dev);
struct irdma_cqp *iwcqp = &rf->cqp;
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
struct irdma_create_qp_info *qp_info;
enum irdma_status_code status;
int status;
cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
if (!cqp_request)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
cqp_info = &cqp_request->info;
qp_info = &cqp_request->info.in.u.qp_create.info;
@@ -1079,7 +1086,7 @@ static void irdma_dealloc_push_page(struct irdma_pci_f *rf,
{
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
enum irdma_status_code status;
int status;
if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX)
return;
@@ -1179,12 +1186,10 @@ static void irdma_hw_modify_qp_callback(struct irdma_cqp_request *cqp_request)
* @info: info for modify qp
* @wait: flag to wait or not for modify qp completion
*/
enum irdma_status_code irdma_hw_modify_qp(struct irdma_device *iwdev,
struct irdma_qp *iwqp,
struct irdma_modify_qp_info *info,
bool wait)
int irdma_hw_modify_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp,
struct irdma_modify_qp_info *info, bool wait)
{
enum irdma_status_code status;
int status;
struct irdma_pci_f *rf = iwdev->rf;
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
@@ -1192,7 +1197,7 @@ enum irdma_status_code irdma_hw_modify_qp(struct irdma_device *iwdev,
cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait);
if (!cqp_request)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
if (!wait) {
cqp_request->callback_fcn = irdma_hw_modify_qp_callback;
@@ -1230,7 +1235,7 @@ enum irdma_status_code irdma_hw_modify_qp(struct irdma_device *iwdev,
cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp,
wait);
if (!cqp_request)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
cqp_info = &cqp_request->info;
m_info = &cqp_info->in.u.qp_modify.info;
@@ -1271,17 +1276,17 @@ void irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)
* @dev: device pointer
* @qp: pointer to qp
*/
enum irdma_status_code irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
int irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
{
struct irdma_pci_f *rf = dev_to_rf(dev);
struct irdma_cqp *iwcqp = &rf->cqp;
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
enum irdma_status_code status;
int status;
cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
if (!cqp_request)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
cqp_info = &cqp_request->info;
memset(cqp_info, 0, sizeof(*cqp_info));
@@ -1317,20 +1322,20 @@ void irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
* irdma_init_hash_desc - initialize hash for crc calculation
* @desc: cryption type
*/
enum irdma_status_code irdma_init_hash_desc(struct shash_desc **desc)
int irdma_init_hash_desc(struct shash_desc **desc)
{
struct crypto_shash *tfm;
struct shash_desc *tdesc;
tfm = crypto_alloc_shash("crc32c", 0, 0);
if (IS_ERR(tfm))
return IRDMA_ERR_MPA_CRC;
return -EINVAL;
tdesc = kzalloc(sizeof(*tdesc) + crypto_shash_descsize(tfm),
GFP_KERNEL);
if (!tdesc) {
crypto_free_shash(tfm);
return IRDMA_ERR_MPA_CRC;
return -EINVAL;
}
tdesc->tfm = tfm;
@@ -1358,19 +1363,19 @@ void irdma_free_hash_desc(struct shash_desc *desc)
* @len: length of buffer
* @val: value to be compared
*/
enum irdma_status_code irdma_ieq_check_mpacrc(struct shash_desc *desc,
void *addr, u32 len, u32 val)
int irdma_ieq_check_mpacrc(struct shash_desc *desc, void *addr, u32 len,
u32 val)
{
u32 crc = 0;
int ret;
enum irdma_status_code ret_code = 0;
int ret_code = 0;
crypto_shash_init(desc);
ret = crypto_shash_update(desc, addr, len);
if (!ret)
crypto_shash_final(desc, (u8 *)&crc);
if (crc != val)
ret_code = IRDMA_ERR_MPA_CRC;
ret_code = -EINVAL;
return ret_code;
}
@@ -1524,9 +1529,8 @@ void irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len,
* @info: to get information
* @buf: puda buffer
*/
static enum irdma_status_code
irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
struct irdma_puda_buf *buf)
static int irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
struct irdma_puda_buf *buf)
{
struct iphdr *iph;
struct ipv6hdr *ip6h;
@@ -1563,7 +1567,7 @@ irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
ibdev_dbg(to_ibdev(buf->vsi->dev),
"ERR: payload_len = 0x%x totallen expected0x%x\n",
info->payload_len, buf->totallen);
return IRDMA_ERR_INVALID_SIZE;
return -EINVAL;
}
buf->tcphlen = tcph->doff << 2;
@@ -1580,9 +1584,8 @@ irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
* @info: to get information
* @buf: puda buffer
*/
enum irdma_status_code
irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
struct irdma_puda_buf *buf)
int irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
struct irdma_puda_buf *buf)
{
struct tcphdr *tcph;
u32 pkt_len;
@@ -1861,20 +1864,19 @@ static void irdma_process_cqp_stats(struct irdma_cqp_request *cqp_request)
* @pestat: pointer to stats info
* @wait: flag to wait or not wait for stats
*/
enum irdma_status_code
irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev,
struct irdma_vsi_pestat *pestat, bool wait)
int irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev,
struct irdma_vsi_pestat *pestat, bool wait)
{
struct irdma_pci_f *rf = dev_to_rf(dev);
struct irdma_cqp *iwcqp = &rf->cqp;
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
enum irdma_status_code status;
int status;
cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait);
if (!cqp_request)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
cqp_info = &cqp_request->info;
memset(cqp_info, 0, sizeof(*cqp_info));
@@ -1900,22 +1902,21 @@ irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev,
* @cmd: command to allocate or free
* @stats_info: pointer to allocate stats info
*/
enum irdma_status_code
irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi, u8 cmd,
struct irdma_stats_inst_info *stats_info)
int irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi, u8 cmd,
struct irdma_stats_inst_info *stats_info)
{
struct irdma_pci_f *rf = dev_to_rf(vsi->dev);
struct irdma_cqp *iwcqp = &rf->cqp;
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
enum irdma_status_code status;
int status;
bool wait = false;
if (cmd == IRDMA_OP_STATS_ALLOCATE)
wait = true;
cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait);
if (!cqp_request)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
cqp_info = &cqp_request->info;
memset(cqp_info, 0, sizeof(*cqp_info));
@@ -1938,17 +1939,17 @@ irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi, u8 cmd,
* @sc_ceq: pointer to ceq structure
* @op: Create or Destroy
*/
enum irdma_status_code irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev,
struct irdma_sc_ceq *sc_ceq, u8 op)
int irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_ceq *sc_ceq,
u8 op)
{
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
struct irdma_pci_f *rf = dev_to_rf(dev);
enum irdma_status_code status;
int status;
cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
if (!cqp_request)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
cqp_info = &cqp_request->info;
cqp_info->post_sq = 1;
@@ -1968,17 +1969,17 @@ enum irdma_status_code irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev,
* @sc_aeq: pointer to aeq structure
* @op: Create or Destroy
*/
enum irdma_status_code irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev,
struct irdma_sc_aeq *sc_aeq, u8 op)
int irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_aeq *sc_aeq,
u8 op)
{
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
struct irdma_pci_f *rf = dev_to_rf(dev);
enum irdma_status_code status;
int status;
cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
if (!cqp_request)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
cqp_info = &cqp_request->info;
cqp_info->post_sq = 1;
@@ -1998,16 +1999,15 @@ enum irdma_status_code irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev,
* @cmd: Add, modify or delete
* @node_info: pointer to ws node info
*/
enum irdma_status_code
irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd,
struct irdma_ws_node_info *node_info)
int irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd,
struct irdma_ws_node_info *node_info)
{
struct irdma_pci_f *rf = dev_to_rf(dev);
struct irdma_cqp *iwcqp = &rf->cqp;
struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp;
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
enum irdma_status_code status;
int status;
bool poll;
if (!rf->sc_dev.ceq_valid)
@@ -2017,7 +2017,7 @@ irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd,
cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, !poll);
if (!cqp_request)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
cqp_info = &cqp_request->info;
memset(cqp_info, 0, sizeof(*cqp_info));
@@ -2066,7 +2066,7 @@ int irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd,
{
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
enum irdma_status_code status;
int status;
if (cmd != IRDMA_OP_AH_CREATE && cmd != IRDMA_OP_AH_DESTROY)
return -EINVAL;
@@ -2148,11 +2148,10 @@ static void irdma_ilq_ah_cb(struct irdma_cqp_request *cqp_request)
* @ah_ret: Returned pointer to address handle if created
*
*/
enum irdma_status_code irdma_puda_create_ah(struct irdma_sc_dev *dev,
struct irdma_ah_info *ah_info,
bool wait, enum puda_rsrc_type type,
void *cb_param,
struct irdma_sc_ah **ah_ret)
int irdma_puda_create_ah(struct irdma_sc_dev *dev,
struct irdma_ah_info *ah_info, bool wait,
enum puda_rsrc_type type, void *cb_param,
struct irdma_sc_ah **ah_ret)
{
struct irdma_sc_ah *ah;
struct irdma_pci_f *rf = dev_to_rf(dev);
@@ -2161,7 +2160,7 @@ enum irdma_status_code irdma_puda_create_ah(struct irdma_sc_dev *dev,
ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
*ah_ret = ah;
if (!ah)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah,
&ah_info->ah_idx, &rf->next_ah);
@@ -2187,7 +2186,7 @@ error:
err_free:
kfree(ah);
*ah_ret = NULL;
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
}
/**
@@ -2229,19 +2228,19 @@ void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request)
* @pprm: pble resource manager
* @pchunk: chunk of memory to add
*/
enum irdma_status_code irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm,
struct irdma_chunk *pchunk)
int irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm,
struct irdma_chunk *pchunk)
{
u64 sizeofbitmap;
if (pchunk->size & 0xfff)
return IRDMA_ERR_PARAM;
return -EINVAL;
sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift;
pchunk->bitmapbuf = bitmap_zalloc(sizeofbitmap, GFP_KERNEL);
if (!pchunk->bitmapbuf)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
pchunk->sizeofbitmap = sizeofbitmap;
/* each pble is 8 bytes hence shift by 3 */
@@ -2259,10 +2258,9 @@ enum irdma_status_code irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm,
* @vaddr: returns virtual address of pble memory
* @fpm_addr: returns fpm address of pble memory
*/
enum irdma_status_code
irdma_prm_get_pbles(struct irdma_pble_prm *pprm,
struct irdma_pble_chunkinfo *chunkinfo, u64 mem_size,
u64 **vaddr, u64 *fpm_addr)
int irdma_prm_get_pbles(struct irdma_pble_prm *pprm,
struct irdma_pble_chunkinfo *chunkinfo, u64 mem_size,
u64 **vaddr, u64 *fpm_addr)
{
u64 bits_needed;
u64 bit_idx = PBLE_INVALID_IDX;
@@ -2290,7 +2288,7 @@ irdma_prm_get_pbles(struct irdma_pble_prm *pprm,
if (!pchunk || bit_idx >= pchunk->sizeofbitmap) {
spin_unlock_irqrestore(&pprm->prm_lock, flags);
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
}
bitmap_set(pchunk->bitmapbuf, bit_idx, bits_needed);
@@ -2325,8 +2323,8 @@ void irdma_prm_return_pbles(struct irdma_pble_prm *pprm,
spin_unlock_irqrestore(&pprm->prm_lock, flags);
}
enum irdma_status_code irdma_map_vm_page_list(struct irdma_hw *hw, void *va,
dma_addr_t *pg_dma, u32 pg_cnt)
int irdma_map_vm_page_list(struct irdma_hw *hw, void *va, dma_addr_t *pg_dma,
u32 pg_cnt)
{
struct page *vm_page;
int i;
@@ -2350,7 +2348,7 @@ enum irdma_status_code irdma_map_vm_page_list(struct irdma_hw *hw, void *va,
err:
irdma_unmap_vm_page_list(hw, pg_dma, i);
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
}
void irdma_unmap_vm_page_list(struct irdma_hw *hw, dma_addr_t *pg_dma, u32 pg_cnt)
@@ -2386,15 +2384,14 @@ done:
* @chunk: chunk to add for paged memory
* @pg_cnt: number of pages needed
*/
enum irdma_status_code irdma_pble_get_paged_mem(struct irdma_chunk *chunk,
u32 pg_cnt)
int irdma_pble_get_paged_mem(struct irdma_chunk *chunk, u32 pg_cnt)
{
u32 size;
void *va;
chunk->dmainfo.dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL);
if (!chunk->dmainfo.dmaaddrs)
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
size = PAGE_SIZE * pg_cnt;
va = vmalloc(size);
@@ -2416,7 +2413,7 @@ err:
kfree(chunk->dmainfo.dmaaddrs);
chunk->dmainfo.dmaaddrs = NULL;
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
}
/**

389
drivers/infiniband/hw/irdma/verbs.c
View File

@@ -256,7 +256,7 @@ static void irdma_alloc_push_page(struct irdma_qp *iwqp)
struct cqp_cmds_info *cqp_info;
struct irdma_device *iwdev = iwqp->iwdev;
struct irdma_sc_qp *qp = &iwqp->sc_qp;
enum irdma_status_code status;
int status;
cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true);
if (!cqp_request)
@@ -592,7 +592,7 @@ static int irdma_setup_kmode_qp(struct irdma_device *iwdev,
u32 sqdepth, rqdepth;
u8 sqshift, rqshift;
u32 size;
enum irdma_status_code status;
int status;
struct irdma_qp_uk_init_info *ukinfo = &info->qp_uk_init_info;
struct irdma_uk_attrs *uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs;
@@ -603,7 +603,7 @@ static int irdma_setup_kmode_qp(struct irdma_device *iwdev,
status = irdma_get_sqdepth(uk_attrs, ukinfo->sq_size, sqshift,
&sqdepth);
if (status)
return -ENOMEM;
return status;
if (uk_attrs->hw_rev == IRDMA_GEN_1)
rqshift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1;
@@ -614,7 +614,7 @@ static int irdma_setup_kmode_qp(struct irdma_device *iwdev,
status = irdma_get_rqdepth(uk_attrs, ukinfo->rq_size, rqshift,
&rqdepth);
if (status)
return -ENOMEM;
return status;
iwqp->kqp.sq_wrid_mem =
kcalloc(sqdepth, sizeof(*iwqp->kqp.sq_wrid_mem), GFP_KERNEL);
@@ -668,7 +668,7 @@ static int irdma_cqp_create_qp_cmd(struct irdma_qp *iwqp)
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
struct irdma_create_qp_info *qp_info;
enum irdma_status_code status;
int status;
cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
if (!cqp_request)
@@ -688,7 +688,7 @@ static int irdma_cqp_create_qp_cmd(struct irdma_qp *iwqp)
status = irdma_handle_cqp_op(rf, cqp_request);
irdma_put_cqp_request(&rf->cqp, cqp_request);
return status ? -ENOMEM : 0;
return status;
}
static void irdma_roce_fill_and_set_qpctx_info(struct irdma_qp *iwqp,
@@ -806,7 +806,6 @@ static int irdma_create_qp(struct ib_qp *ibqp,
struct irdma_create_qp_req req;
struct irdma_create_qp_resp uresp = {};
u32 qp_num = 0;
enum irdma_status_code ret;
int err_code;
int sq_size;
int rq_size;
@@ -936,9 +935,8 @@ static int irdma_create_qp(struct ib_qp *ibqp,
if (dev->hw_attrs.uk_attrs.hw_rev > IRDMA_GEN_1)
init_info.qp_uk_init_info.qp_caps |= IRDMA_PUSH_MODE;
ret = irdma_sc_qp_init(qp, &init_info);
if (ret) {
err_code = -EPROTO;
err_code = irdma_sc_qp_init(qp, &init_info);
if (err_code) {
ibdev_dbg(&iwdev->ibdev, "VERBS: qp_init fail\n");
goto error;
}
@@ -1189,7 +1187,7 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr,
if (ret)
return ret;
if (vlan_id >= VLAN_N_VID && iwdev->dcb)
if (vlan_id >= VLAN_N_VID && iwdev->dcb_vlan_mode)
vlan_id = 0;
if (vlan_id < VLAN_N_VID) {
udp_info->insert_vlan_tag = true;
@@ -1202,7 +1200,7 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr,
av->attrs = attr->ah_attr;
rdma_gid2ip((struct sockaddr *)&av->sgid_addr, &sgid_attr->gid);
rdma_gid2ip((struct sockaddr *)&av->dgid_addr, &attr->ah_attr.grh.dgid);
if (av->sgid_addr.saddr.sa_family == AF_INET6) {
if (av->net_type == RDMA_NETWORK_IPV6) {
__be32 *daddr =
av->dgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32;
__be32 *saddr =
@@ -1218,7 +1216,7 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr,
&local_ip[0],
false, NULL,
IRDMA_ARP_RESOLVE);
} else {
} else if (av->net_type == RDMA_NETWORK_IPV4) {
__be32 saddr = av->sgid_addr.saddr_in.sin_addr.s_addr;
__be32 daddr = av->dgid_addr.saddr_in.sin_addr.s_addr;
@@ -1792,7 +1790,6 @@ static int irdma_resize_cq(struct ib_cq *ibcq, int entries,
struct irdma_device *iwdev;
struct irdma_pci_f *rf;
struct irdma_cq_buf *cq_buf = NULL;
enum irdma_status_code status = 0;
unsigned long flags;
int ret;
@@ -1885,12 +1882,10 @@ static int irdma_resize_cq(struct ib_cq *ibcq, int entries,
cqp_info->in.u.cq_modify.cq = &iwcq->sc_cq;
cqp_info->in.u.cq_modify.scratch = (uintptr_t)cqp_request;
cqp_info->post_sq = 1;
status = irdma_handle_cqp_op(rf, cqp_request);
ret = irdma_handle_cqp_op(rf, cqp_request);
irdma_put_cqp_request(&rf->cqp, cqp_request);
if (status) {
ret = -EPROTO;
if (ret)
goto error;
}
spin_lock_irqsave(&iwcq->lock, flags);
if (cq_buf) {
@@ -1945,7 +1940,6 @@ static int irdma_create_cq(struct ib_cq *ibcq,
struct irdma_sc_cq *cq;
struct irdma_sc_dev *dev = &rf->sc_dev;
struct irdma_cq_init_info info = {};
enum irdma_status_code status;
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
struct irdma_cq_uk_init_info *ukinfo = &info.cq_uk_init_info;
@@ -2095,12 +2089,10 @@ static int irdma_create_cq(struct ib_cq *ibcq,
cqp_info->in.u.cq_create.cq = cq;
cqp_info->in.u.cq_create.check_overflow = true;
cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request;
status = irdma_handle_cqp_op(rf, cqp_request);
err_code = irdma_handle_cqp_op(rf, cqp_request);
irdma_put_cqp_request(&rf->cqp, cqp_request);
if (status) {
err_code = -ENOMEM;
if (err_code)
goto cq_free_rsrc;
}
if (udata) {
struct irdma_create_cq_resp resp = {};
@@ -2309,14 +2301,14 @@ static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr,
struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc;
struct irdma_pble_info *pinfo;
u64 *pbl;
enum irdma_status_code status;
int status;
enum irdma_pble_level level = PBLE_LEVEL_1;
if (use_pbles) {
status = irdma_get_pble(rf->pble_rsrc, palloc, iwmr->page_cnt,
false);
if (status)
return -ENOMEM;
return status;
iwpbl->pbl_allocated = true;
level = palloc->level;
@@ -2434,7 +2426,7 @@ static int irdma_hw_alloc_mw(struct irdma_device *iwdev, struct irdma_mr *iwmr)
struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd);
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
enum irdma_status_code status;
int status;
cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true);
if (!cqp_request)
@@ -2457,7 +2449,7 @@ static int irdma_hw_alloc_mw(struct irdma_device *iwdev, struct irdma_mr *iwmr)
status = irdma_handle_cqp_op(iwdev->rf, cqp_request);
irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request);
return status ? -ENOMEM : 0;
return status;
}
/**
@@ -2509,7 +2501,7 @@ static int irdma_dealloc_mw(struct ib_mw *ibmw)
cqp_info = &cqp_request->info;
info = &cqp_info->in.u.dealloc_stag.info;
memset(info, 0, sizeof(*info));
info->pd_id = iwpd->sc_pd.pd_id & 0x00007fff;
info->pd_id = iwpd->sc_pd.pd_id;
info->stag_idx = ibmw->rkey >> IRDMA_CQPSQ_STAG_IDX_S;
info->mr = false;
cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG;
@@ -2533,8 +2525,7 @@ static int irdma_hw_alloc_stag(struct irdma_device *iwdev,
{
struct irdma_allocate_stag_info *info;
struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd);
enum irdma_status_code status;
int err = 0;
int status;
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
@@ -2556,10 +2547,8 @@ static int irdma_hw_alloc_stag(struct irdma_device *iwdev,
cqp_info->in.u.alloc_stag.scratch = (uintptr_t)cqp_request;
status = irdma_handle_cqp_op(iwdev->rf, cqp_request);
irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request);
if (status)
err = -ENOMEM;
return err;
return status;
}
/**
@@ -2575,9 +2564,8 @@ static struct ib_mr *irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
struct irdma_pble_alloc *palloc;
struct irdma_pbl *iwpbl;
struct irdma_mr *iwmr;
enum irdma_status_code status;
u32 stag;
int err_code = -ENOMEM;
int err_code;
iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
if (!iwmr)
@@ -2599,9 +2587,9 @@ static struct ib_mr *irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
iwmr->type = IRDMA_MEMREG_TYPE_MEM;
palloc = &iwpbl->pble_alloc;
iwmr->page_cnt = max_num_sg;
status = irdma_get_pble(iwdev->rf->pble_rsrc, palloc, iwmr->page_cnt,
true);
if (status)
err_code = irdma_get_pble(iwdev->rf->pble_rsrc, palloc, iwmr->page_cnt,
true);
if (err_code)
goto err_get_pble;
err_code = irdma_hw_alloc_stag(iwdev, iwmr);
@@ -2672,10 +2660,9 @@ static int irdma_hwreg_mr(struct irdma_device *iwdev, struct irdma_mr *iwmr,
struct irdma_reg_ns_stag_info *stag_info;
struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd);
struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc;
enum irdma_status_code status;
int err = 0;
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
int ret;
cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true);
if (!cqp_request)
@@ -2712,12 +2699,10 @@ static int irdma_hwreg_mr(struct irdma_device *iwdev, struct irdma_mr *iwmr,
cqp_info->post_sq = 1;
cqp_info->in.u.mr_reg_non_shared.dev = &iwdev->rf->sc_dev;
cqp_info->in.u.mr_reg_non_shared.scratch = (uintptr_t)cqp_request;
status = irdma_handle_cqp_op(iwdev->rf, cqp_request);
ret = irdma_handle_cqp_op(iwdev->rf, cqp_request);
irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request);
if (status)
err = -ENOMEM;
return err;
return ret;
}
/**
@@ -2897,7 +2882,6 @@ struct ib_mr *irdma_reg_phys_mr(struct ib_pd *pd, u64 addr, u64 size, int access
struct irdma_device *iwdev = to_iwdev(pd->device);
struct irdma_pbl *iwpbl;
struct irdma_mr *iwmr;
enum irdma_status_code status;
u32 stag;
int ret;
@@ -2925,10 +2909,9 @@ struct ib_mr *irdma_reg_phys_mr(struct ib_pd *pd, u64 addr, u64 size, int access
iwmr->pgaddrmem[0] = addr;
iwmr->len = size;
iwmr->page_size = SZ_4K;
status = irdma_hwreg_mr(iwdev, iwmr, access);
if (status) {
ret = irdma_hwreg_mr(iwdev, iwmr, access);
if (ret) {
irdma_free_stag(iwdev, stag);
ret = -ENOMEM;
goto err;
}
@@ -3021,7 +3004,7 @@ static int irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
cqp_info = &cqp_request->info;
info = &cqp_info->in.u.dealloc_stag.info;
memset(info, 0, sizeof(*info));
info->pd_id = iwpd->sc_pd.pd_id & 0x00007fff;
info->pd_id = iwpd->sc_pd.pd_id;
info->stag_idx = ib_mr->rkey >> IRDMA_CQPSQ_STAG_IDX_S;
info->mr = true;
if (iwpbl->pbl_allocated)
@@ -3057,7 +3040,6 @@ static int irdma_post_send(struct ib_qp *ibqp,
struct irdma_qp_uk *ukqp;
struct irdma_sc_dev *dev;
struct irdma_post_sq_info info;
enum irdma_status_code ret;
int err = 0;
unsigned long flags;
bool inv_stag;
@@ -3116,7 +3098,7 @@ static int irdma_post_send(struct ib_qp *ibqp,
info.op.inline_send.qkey = ud_wr(ib_wr)->remote_qkey;
info.op.inline_send.dest_qp = ud_wr(ib_wr)->remote_qpn;
}
ret = irdma_uk_inline_send(ukqp, &info, false);
err = irdma_uk_inline_send(ukqp, &info, false);
} else {
info.op.send.num_sges = ib_wr->num_sge;
info.op.send.sg_list = ib_wr->sg_list;
@@ -3127,14 +3109,7 @@ static int irdma_post_send(struct ib_qp *ibqp,
info.op.send.qkey = ud_wr(ib_wr)->remote_qkey;
info.op.send.dest_qp = ud_wr(ib_wr)->remote_qpn;
}
ret = irdma_uk_send(ukqp, &info, false);
}
if (ret) {
if (ret == IRDMA_ERR_QP_TOOMANY_WRS_POSTED)
err = -ENOMEM;
else
err = -EINVAL;
err = irdma_uk_send(ukqp, &info, false);
}
break;
case IB_WR_RDMA_WRITE_WITH_IMM:
@@ -3160,20 +3135,13 @@ static int irdma_post_send(struct ib_qp *ibqp,
rdma_wr(ib_wr)->remote_addr;
info.op.inline_rdma_write.rem_addr.lkey =
rdma_wr(ib_wr)->rkey;
ret = irdma_uk_inline_rdma_write(ukqp, &info, false);
err = irdma_uk_inline_rdma_write(ukqp, &info, false);
} else {
info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list;
info.op.rdma_write.num_lo_sges = ib_wr->num_sge;
info.op.rdma_write.rem_addr.addr = rdma_wr(ib_wr)->remote_addr;
info.op.rdma_write.rem_addr.lkey = rdma_wr(ib_wr)->rkey;
ret = irdma_uk_rdma_write(ukqp, &info, false);
}
if (ret) {
if (ret == IRDMA_ERR_QP_TOOMANY_WRS_POSTED)
err = -ENOMEM;
else
err = -EINVAL;
err = irdma_uk_rdma_write(ukqp, &info, false);
}
break;
case IB_WR_RDMA_READ_WITH_INV:
@@ -3190,21 +3158,12 @@ static int irdma_post_send(struct ib_qp *ibqp,
info.op.rdma_read.rem_addr.lkey = rdma_wr(ib_wr)->rkey;
info.op.rdma_read.lo_sg_list = (void *)ib_wr->sg_list;
info.op.rdma_read.num_lo_sges = ib_wr->num_sge;
ret = irdma_uk_rdma_read(ukqp, &info, inv_stag, false);
if (ret) {
if (ret == IRDMA_ERR_QP_TOOMANY_WRS_POSTED)
err = -ENOMEM;
else
err = -EINVAL;
}
err = irdma_uk_rdma_read(ukqp, &info, inv_stag, false);
break;
case IB_WR_LOCAL_INV:
info.op_type = IRDMA_OP_TYPE_INV_STAG;
info.op.inv_local_stag.target_stag = ib_wr->ex.invalidate_rkey;
ret = irdma_uk_stag_local_invalidate(ukqp, &info, true);
if (ret)
err = -ENOMEM;
err = irdma_uk_stag_local_invalidate(ukqp, &info, true);
break;
case IB_WR_REG_MR: {
struct irdma_mr *iwmr = to_iwmr(reg_wr(ib_wr)->mr);
@@ -3226,10 +3185,8 @@ static int irdma_post_send(struct ib_qp *ibqp,
stag_info.local_fence = ib_wr->send_flags & IB_SEND_FENCE;
if (iwmr->npages > IRDMA_MIN_PAGES_PER_FMR)
stag_info.chunk_size = 1;
ret = irdma_sc_mr_fast_register(&iwqp->sc_qp, &stag_info,
err = irdma_sc_mr_fast_register(&iwqp->sc_qp, &stag_info,
true);
if (ret)
err = -ENOMEM;
break;
}
default:
@@ -3274,7 +3231,6 @@ static int irdma_post_recv(struct ib_qp *ibqp,
struct irdma_qp *iwqp;
struct irdma_qp_uk *ukqp;
struct irdma_post_rq_info post_recv = {};
enum irdma_status_code ret = 0;
unsigned long flags;
int err = 0;
bool reflush = false;
@@ -3289,14 +3245,10 @@ static int irdma_post_recv(struct ib_qp *ibqp,
post_recv.num_sges = ib_wr->num_sge;
post_recv.wr_id = ib_wr->wr_id;
post_recv.sg_list = ib_wr->sg_list;
ret = irdma_uk_post_receive(ukqp, &post_recv);
if (ret) {
err = irdma_uk_post_receive(ukqp, &post_recv);
if (err) {
ibdev_dbg(&iwqp->iwdev->ibdev,
"VERBS: post_recv err %d\n", ret);
if (ret == IRDMA_ERR_QP_TOOMANY_WRS_POSTED)
err = -ENOMEM;
else
err = -EINVAL;
"VERBS: post_recv err %d\n", err);
goto out;
}
@@ -3483,7 +3435,7 @@ static int __irdma_poll_cq(struct irdma_cq *iwcq, int num_entries, struct ib_wc
struct irdma_cq_buf *last_buf = NULL;
struct irdma_cq_poll_info *cur_cqe = &iwcq->cur_cqe;
struct irdma_cq_buf *cq_buf;
enum irdma_status_code ret;
int ret;
struct irdma_device *iwdev;
struct irdma_cq_uk *ukcq;
bool cq_new_cqe = false;
@@ -3503,10 +3455,10 @@ static int __irdma_poll_cq(struct irdma_cq *iwcq, int num_entries, struct ib_wc
cq_new_cqe = true;
continue;
}
if (ret == IRDMA_ERR_Q_EMPTY)
if (ret == -ENOENT)
break;
/* QP using the CQ is destroyed. Skip reporting this CQE */
if (ret == IRDMA_ERR_Q_DESTROYED) {
if (ret == -EFAULT) {
cq_new_cqe = true;
continue;
}
@@ -3528,10 +3480,10 @@ static int __irdma_poll_cq(struct irdma_cq *iwcq, int num_entries, struct ib_wc
continue;
}
if (ret == IRDMA_ERR_Q_EMPTY)
if (ret == -ENOENT)
break;
/* QP using the CQ is destroyed. Skip reporting this CQE */
if (ret == IRDMA_ERR_Q_DESTROYED) {
if (ret == -EFAULT) {
cq_new_cqe = true;
continue;
}
@@ -3553,7 +3505,7 @@ error:
ibdev_dbg(&iwdev->ibdev, "%s: Error polling CQ, irdma_err: %d\n",
__func__, ret);
return -EINVAL;
return ret;
}
/**
@@ -3859,7 +3811,7 @@ static int irdma_mcast_cqp_op(struct irdma_device *iwdev,
{
struct cqp_cmds_info *cqp_info;
struct irdma_cqp_request *cqp_request;
enum irdma_status_code status;
int status;
cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true);
if (!cqp_request)
@@ -3873,10 +3825,8 @@ static int irdma_mcast_cqp_op(struct irdma_device *iwdev,
cqp_info->in.u.mc_create.cqp = &iwdev->rf->cqp.sc_cqp;
status = irdma_handle_cqp_op(iwdev->rf, cqp_request);
irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request);
if (status)
return -ENOMEM;
return 0;
return status;
}
/**
@@ -3932,11 +3882,7 @@ static int irdma_attach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid)
int ret = 0;
bool ipv4;
u16 vlan_id;
union {
struct sockaddr saddr;
struct sockaddr_in saddr_in;
struct sockaddr_in6 saddr_in6;
} sgid_addr;
union irdma_sockaddr sgid_addr;
unsigned char dmac[ETH_ALEN];
rdma_gid2ip((struct sockaddr *)&sgid_addr, ibgid);
@@ -4072,11 +4018,7 @@ static int irdma_detach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid)
struct irdma_mcast_grp_ctx_entry_info mcg_info = {};
int ret;
unsigned long flags;
union {
struct sockaddr saddr;
struct sockaddr_in saddr_in;
struct sockaddr_in6 saddr_in6;
} sgid_addr;
union irdma_sockaddr sgid_addr;
rdma_gid2ip((struct sockaddr *)&sgid_addr, ibgid);
if (!ipv6_addr_v4mapped((struct in6_addr *)ibgid))
@@ -4132,17 +4074,47 @@ static int irdma_detach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid)
return 0;
}
/**
* irdma_create_ah - create address handle
* @ibah: address handle
* @attr: address handle attributes
* @udata: User data
*
* returns 0 on success, error otherwise
*/
static int irdma_create_ah(struct ib_ah *ibah,
struct rdma_ah_init_attr *attr,
struct ib_udata *udata)
static int irdma_create_hw_ah(struct irdma_device *iwdev, struct irdma_ah *ah, bool sleep)
{
struct irdma_pci_f *rf = iwdev->rf;
int err;
err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah, &ah->sc_ah.ah_info.ah_idx,
&rf->next_ah);
if (err)
return err;
err = irdma_ah_cqp_op(rf, &ah->sc_ah, IRDMA_OP_AH_CREATE, sleep,
irdma_gsi_ud_qp_ah_cb, &ah->sc_ah);
if (err) {
ibdev_dbg(&iwdev->ibdev, "VERBS: CQP-OP Create AH fail");
goto err_ah_create;
}
if (!sleep) {
int cnt = CQP_COMPL_WAIT_TIME_MS * CQP_TIMEOUT_THRESHOLD;
do {
irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq);
mdelay(1);
} while (!ah->sc_ah.ah_info.ah_valid && --cnt);
if (!cnt) {
ibdev_dbg(&iwdev->ibdev, "VERBS: CQP create AH timed out");
err = -ETIMEDOUT;
goto err_ah_create;
}
}
return 0;
err_ah_create:
irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, ah->sc_ah.ah_info.ah_idx);
return err;
}
static int irdma_setup_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *attr)
{
struct irdma_pd *pd = to_iwpd(ibah->pd);
struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah);
@@ -4151,25 +4123,13 @@ static int irdma_create_ah(struct ib_ah *ibah,
struct irdma_device *iwdev = to_iwdev(ibah->pd->device);
struct irdma_pci_f *rf = iwdev->rf;
struct irdma_sc_ah *sc_ah;
u32 ah_id = 0;
struct irdma_ah_info *ah_info;
struct irdma_create_ah_resp uresp;
union {
struct sockaddr saddr;
struct sockaddr_in saddr_in;
struct sockaddr_in6 saddr_in6;
} sgid_addr, dgid_addr;
union irdma_sockaddr sgid_addr, dgid_addr;
int err;
u8 dmac[ETH_ALEN];
err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah, &ah_id,
&rf->next_ah);
if (err)
return err;
ah->pd = pd;
sc_ah = &ah->sc_ah;
sc_ah->ah_info.ah_idx = ah_id;
sc_ah->ah_info.vsi = &iwdev->vsi;
irdma_sc_init_ah(&rf->sc_dev, sc_ah);
ah->sgid_index = ah_attr->grh.sgid_index;
@@ -4179,10 +4139,7 @@ static int irdma_create_ah(struct ib_ah *ibah,
rdma_gid2ip((struct sockaddr *)&dgid_addr, &ah_attr->grh.dgid);
ah->av.attrs = *ah_attr;
ah->av.net_type = rdma_gid_attr_network_type(sgid_attr);
ah->av.sgid_addr.saddr = sgid_addr.saddr;
ah->av.dgid_addr.saddr = dgid_addr.saddr;
ah_info = &sc_ah->ah_info;
ah_info->ah_idx = ah_id;
ah_info->pd_idx = pd->sc_pd.pd_id;
if (ah_attr->ah_flags & IB_AH_GRH) {
ah_info->flow_label = ah_attr->grh.flow_label;
@@ -4191,7 +4148,7 @@ static int irdma_create_ah(struct ib_ah *ibah,
}
ether_addr_copy(dmac, ah_attr->roce.dmac);
if (rdma_gid_attr_network_type(sgid_attr) == RDMA_NETWORK_IPV4) {
if (ah->av.net_type == RDMA_NETWORK_IPV4) {
ah_info->ipv4_valid = true;
ah_info->dest_ip_addr[0] =
ntohl(dgid_addr.saddr_in.sin_addr.s_addr);
@@ -4219,17 +4176,15 @@ static int irdma_create_ah(struct ib_ah *ibah,
err = rdma_read_gid_l2_fields(sgid_attr, &ah_info->vlan_tag,
ah_info->mac_addr);
if (err)
goto error;
return err;
ah_info->dst_arpindex = irdma_add_arp(iwdev->rf, ah_info->dest_ip_addr,
ah_info->ipv4_valid, dmac);
if (ah_info->dst_arpindex == -1) {
err = -EINVAL;
goto error;
}
if (ah_info->dst_arpindex == -1)
return -EINVAL;
if (ah_info->vlan_tag >= VLAN_N_VID && iwdev->dcb)
if (ah_info->vlan_tag >= VLAN_N_VID && iwdev->dcb_vlan_mode)
ah_info->vlan_tag = 0;
if (ah_info->vlan_tag < VLAN_N_VID) {
@@ -4238,43 +4193,38 @@ static int irdma_create_ah(struct ib_ah *ibah,
rt_tos2priority(ah_info->tc_tos) << VLAN_PRIO_SHIFT;
}
err = irdma_ah_cqp_op(iwdev->rf, sc_ah, IRDMA_OP_AH_CREATE,
attr->flags & RDMA_CREATE_AH_SLEEPABLE,
irdma_gsi_ud_qp_ah_cb, sc_ah);
return 0;
}
if (err) {
ibdev_dbg(&iwdev->ibdev,
"VERBS: CQP-OP Create AH fail");
goto error;
}
/**
* irdma_ah_exists - Check for existing identical AH
* @iwdev: irdma device
* @new_ah: AH to check for
*
* returns true if AH is found, false if not found.
*/
static bool irdma_ah_exists(struct irdma_device *iwdev,
struct irdma_ah *new_ah)
{
struct irdma_ah *ah;
u32 key = new_ah->sc_ah.ah_info.dest_ip_addr[0] ^
new_ah->sc_ah.ah_info.dest_ip_addr[1] ^
new_ah->sc_ah.ah_info.dest_ip_addr[2] ^
new_ah->sc_ah.ah_info.dest_ip_addr[3];
if (!(attr->flags & RDMA_CREATE_AH_SLEEPABLE)) {
int cnt = CQP_COMPL_WAIT_TIME_MS * CQP_TIMEOUT_THRESHOLD;
do {
irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq);
mdelay(1);
} while (!sc_ah->ah_info.ah_valid && --cnt);
if (!cnt) {
ibdev_dbg(&iwdev->ibdev,
"VERBS: CQP create AH timed out");
err = -ETIMEDOUT;
goto error;
hash_for_each_possible(iwdev->ah_hash_tbl, ah, list, key) {
/* Set ah_valid and ah_id the same so memcmp can work */
new_ah->sc_ah.ah_info.ah_idx = ah->sc_ah.ah_info.ah_idx;
new_ah->sc_ah.ah_info.ah_valid = ah->sc_ah.ah_info.ah_valid;
if (!memcmp(&ah->sc_ah.ah_info, &new_ah->sc_ah.ah_info,
sizeof(ah->sc_ah.ah_info))) {
refcount_inc(&ah->refcnt);
new_ah->parent_ah = ah;
return true;
}
}
if (udata) {
uresp.ah_id = ah->sc_ah.ah_info.ah_idx;
err = ib_copy_to_udata(udata, &uresp,
min(sizeof(uresp), udata->outlen));
}
return 0;
error:
irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, ah_id);
return err;
return false;
}
/**
@@ -4287,6 +4237,17 @@ static int irdma_destroy_ah(struct ib_ah *ibah, u32 ah_flags)
struct irdma_device *iwdev = to_iwdev(ibah->device);
struct irdma_ah *ah = to_iwah(ibah);
if ((ah_flags & RDMA_DESTROY_AH_SLEEPABLE) && ah->parent_ah) {
mutex_lock(&iwdev->ah_tbl_lock);
if (!refcount_dec_and_test(&ah->parent_ah->refcnt)) {
mutex_unlock(&iwdev->ah_tbl_lock);
return 0;
}
hash_del(&ah->parent_ah->list);
kfree(ah->parent_ah);
mutex_unlock(&iwdev->ah_tbl_lock);
}
irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, IRDMA_OP_AH_DESTROY,
false, NULL, ah);
@@ -4296,6 +4257,80 @@ static int irdma_destroy_ah(struct ib_ah *ibah, u32 ah_flags)
return 0;
}
/**
* irdma_create_user_ah - create user address handle
* @ibah: address handle
* @attr: address handle attributes
* @udata: User data
*
* returns 0 on success, error otherwise
*/
static int irdma_create_user_ah(struct ib_ah *ibah,
struct rdma_ah_init_attr *attr,
struct ib_udata *udata)
{
struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah);
struct irdma_device *iwdev = to_iwdev(ibah->pd->device);
struct irdma_create_ah_resp uresp;
struct irdma_ah *parent_ah;
int err;
err = irdma_setup_ah(ibah, attr);
if (err)
return err;
mutex_lock(&iwdev->ah_tbl_lock);
if (!irdma_ah_exists(iwdev, ah)) {
err = irdma_create_hw_ah(iwdev, ah, true);
if (err) {
mutex_unlock(&iwdev->ah_tbl_lock);
return err;
}
/* Add new AH to list */
parent_ah = kmemdup(ah, sizeof(*ah), GFP_KERNEL);
if (parent_ah) {
u32 key = parent_ah->sc_ah.ah_info.dest_ip_addr[0] ^
parent_ah->sc_ah.ah_info.dest_ip_addr[1] ^
parent_ah->sc_ah.ah_info.dest_ip_addr[2] ^
parent_ah->sc_ah.ah_info.dest_ip_addr[3];
ah->parent_ah = parent_ah;
hash_add(iwdev->ah_hash_tbl, &parent_ah->list, key);
refcount_set(&parent_ah->refcnt, 1);
}
}
mutex_unlock(&iwdev->ah_tbl_lock);
uresp.ah_id = ah->sc_ah.ah_info.ah_idx;
err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen));
if (err)
irdma_destroy_ah(ibah, attr->flags);
return err;
}
/**
* irdma_create_ah - create address handle
* @ibah: address handle
* @attr: address handle attributes
* @udata: NULL
*
* returns 0 on success, error otherwise
*/
static int irdma_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *attr,
struct ib_udata *udata)
{
struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah);
struct irdma_device *iwdev = to_iwdev(ibah->pd->device);
int err;
err = irdma_setup_ah(ibah, attr);
if (err)
return err;
err = irdma_create_hw_ah(iwdev, ah, attr->flags & RDMA_CREATE_AH_SLEEPABLE);
return err;
}
/**
* irdma_query_ah - Query address handle
* @ibah: pointer to address handle
@@ -4329,7 +4364,7 @@ static enum rdma_link_layer irdma_get_link_layer(struct ib_device *ibdev,
static const struct ib_device_ops irdma_roce_dev_ops = {
.attach_mcast = irdma_attach_mcast,
.create_ah = irdma_create_ah,
.create_user_ah = irdma_create_ah,
.create_user_ah = irdma_create_user_ah,
.destroy_ah = irdma_destroy_ah,
.detach_mcast = irdma_detach_mcast,
.get_link_layer = irdma_get_link_layer,

15
drivers/infiniband/hw/irdma/verbs.h
View File

@@ -25,14 +25,16 @@ struct irdma_pd {
struct irdma_sc_pd sc_pd;
};
union irdma_sockaddr {
struct sockaddr_in saddr_in;
struct sockaddr_in6 saddr_in6;
};
struct irdma_av {
u8 macaddr[16];
struct rdma_ah_attr attrs;
union {
struct sockaddr saddr;
struct sockaddr_in saddr_in;
struct sockaddr_in6 saddr_in6;
} sgid_addr, dgid_addr;
union irdma_sockaddr sgid_addr;
union irdma_sockaddr dgid_addr;
u8 net_type;
};
@@ -43,6 +45,9 @@ struct irdma_ah {
struct irdma_av av;
u8 sgid_index;
union ib_gid dgid;
struct hlist_node list;
refcount_t refcnt;
struct irdma_ah *parent_ah; /* AH from cached list */
};
struct irdma_hmc_pble {

19
drivers/infiniband/hw/irdma/ws.c
View File

@@ -1,7 +1,6 @@
// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
/* Copyright (c) 2017 - 2021 Intel Corporation */
#include "osdep.h"
#include "status.h"
#include "hmc.h"
#include "defs.h"
#include "type.h"
@@ -87,8 +86,8 @@ static void irdma_free_node(struct irdma_sc_vsi *vsi,
* @node: pointer to node
* @cmd: add, remove or modify
*/
static enum irdma_status_code
irdma_ws_cqp_cmd(struct irdma_sc_vsi *vsi, struct irdma_ws_node *node, u8 cmd)
static int irdma_ws_cqp_cmd(struct irdma_sc_vsi *vsi,
struct irdma_ws_node *node, u8 cmd)
{
struct irdma_ws_node_info node_info = {};
@@ -106,7 +105,7 @@ irdma_ws_cqp_cmd(struct irdma_sc_vsi *vsi, struct irdma_ws_node *node, u8 cmd)
node_info.enable = node->enable;
if (irdma_cqp_ws_node_cmd(vsi->dev, cmd, &node_info)) {
ibdev_dbg(to_ibdev(vsi->dev), "WS: CQP WS CMD failed\n");
return IRDMA_ERR_NO_MEMORY;
return -ENOMEM;
}
if (node->type_leaf && cmd == IRDMA_OP_WS_ADD_NODE) {
@@ -234,18 +233,18 @@ static void irdma_remove_leaf(struct irdma_sc_vsi *vsi, u8 user_pri)
* @vsi: vsi pointer
* @user_pri: user priority
*/
enum irdma_status_code irdma_ws_add(struct irdma_sc_vsi *vsi, u8 user_pri)
int irdma_ws_add(struct irdma_sc_vsi *vsi, u8 user_pri)
{
struct irdma_ws_node *ws_tree_root;
struct irdma_ws_node *vsi_node;
struct irdma_ws_node *tc_node;
u16 traffic_class;
enum irdma_status_code ret = 0;
int ret = 0;
int i;
mutex_lock(&vsi->dev->ws_mutex);
if (vsi->tc_change_pending) {
ret = IRDMA_ERR_NOT_READY;
ret = -EBUSY;
goto exit;
}
@@ -258,7 +257,7 @@ enum irdma_status_code irdma_ws_add(struct irdma_sc_vsi *vsi, u8 user_pri)
ws_tree_root = irdma_alloc_node(vsi, user_pri,
WS_NODE_TYPE_PARENT, NULL);
if (!ws_tree_root) {
ret = IRDMA_ERR_NO_MEMORY;
ret = -ENOMEM;
goto exit;
}
@@ -283,7 +282,7 @@ enum irdma_status_code irdma_ws_add(struct irdma_sc_vsi *vsi, u8 user_pri)
vsi_node = irdma_alloc_node(vsi, user_pri, WS_NODE_TYPE_PARENT,
ws_tree_root);
if (!vsi_node) {
ret = IRDMA_ERR_NO_MEMORY;
ret = -ENOMEM;
goto vsi_add_err;
}
@@ -310,7 +309,7 @@ enum irdma_status_code irdma_ws_add(struct irdma_sc_vsi *vsi, u8 user_pri)
tc_node = irdma_alloc_node(vsi, user_pri, WS_NODE_TYPE_LEAF,
vsi_node);
if (!tc_node) {
ret = IRDMA_ERR_NO_MEMORY;
ret = -ENOMEM;
goto leaf_add_err;
}

2
drivers/infiniband/hw/irdma/ws.h
View File

@@ -34,7 +34,7 @@ struct irdma_ws_node {
};
struct irdma_sc_vsi;
enum irdma_status_code irdma_ws_add(struct irdma_sc_vsi *vsi, u8 user_pri);
int irdma_ws_add(struct irdma_sc_vsi *vsi, u8 user_pri);
void irdma_ws_remove(struct irdma_sc_vsi *vsi, u8 user_pri);
void irdma_ws_reset(struct irdma_sc_vsi *vsi);

1
drivers/infiniband/hw/mlx4/alias_GUID.c
View File

@@ -38,7 +38,6 @@
#include <rdma/ib_sa.h>
#include <rdma/ib_pack.h>
#include <linux/mlx4/cmd.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <rdma/ib_user_verbs.h>

1
drivers/infiniband/hw/mlx4/srq.c
View File

@@ -320,7 +320,6 @@ int mlx4_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
if (mdev->dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) {
err = -EIO;
*bad_wr = wr;
nreq = 0;
goto out;
}

4
drivers/infiniband/hw/mlx5/devx.c
View File

@@ -1895,8 +1895,10 @@ subscribe_event_xa_alloc(struct mlx5_devx_event_table *devx_event_table,
key_level2,
obj_event,
GFP_KERNEL);
if (err)
if (err) {
kfree(obj_event);
return err;
}
INIT_LIST_HEAD(&obj_event->obj_sub_list);
}

1
drivers/infiniband/hw/mlx5/ib_virt.c
View File

@@ -30,7 +30,6 @@
* SOFTWARE.
*/
#include <linux/module.h>
#include <linux/mlx5/vport.h>
#include "mlx5_ib.h"

1
drivers/infiniband/hw/mlx5/mem.c
View File

@@ -30,7 +30,6 @@
* SOFTWARE.
*/
#include <linux/module.h>
#include <rdma/ib_umem.h>
#include <rdma/ib_umem_odp.h>
#include "mlx5_ib.h"

12
drivers/infiniband/hw/mlx5/mlx5_ib.h
View File

@@ -763,9 +763,9 @@ struct mlx5_cache_ent {
char name[4];
u32 order;
u32 xlt;
u32 access_mode;
u32 page;
unsigned int ndescs;
u8 disabled:1;
u8 fill_to_high_water:1;
@@ -788,7 +788,6 @@ struct mlx5_cache_ent {
u32 miss;
struct mlx5_ib_dev *dev;
struct work_struct work;
struct delayed_work dwork;
};
@@ -1344,7 +1343,8 @@ int mlx5_mr_cache_init(struct mlx5_ib_dev *dev);
int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev);
struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev,
unsigned int entry, int access_flags);
struct mlx5_cache_ent *ent,
int access_flags);
int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
struct ib_mr_status *mr_status);
@@ -1539,12 +1539,6 @@ static inline int get_uars_per_sys_page(struct mlx5_ib_dev *dev, bool lib_suppor
MLX5_UARS_IN_PAGE : 1;
}
static inline int get_num_static_uars(struct mlx5_ib_dev *dev,
struct mlx5_bfreg_info *bfregi)
{
return get_uars_per_sys_page(dev, bfregi->lib_uar_4k) * bfregi->num_static_sys_pages;
}
extern void *xlt_emergency_page;
int bfregn_to_uar_index(struct mlx5_ib_dev *dev,

104
drivers/infiniband/hw/mlx5/mr.c
View File

@@ -68,7 +68,6 @@ static void set_mkc_access_pd_addr_fields(void *mkc, int acc, u64 start_addr,
struct ib_pd *pd)
{
struct mlx5_ib_dev *dev = to_mdev(pd->device);
bool ro_pci_enabled = pcie_relaxed_ordering_enabled(dev->mdev->pdev);
MLX5_SET(mkc, mkc, a, !!(acc & IB_ACCESS_REMOTE_ATOMIC));
MLX5_SET(mkc, mkc, rw, !!(acc & IB_ACCESS_REMOTE_WRITE));
@@ -76,12 +75,13 @@ static void set_mkc_access_pd_addr_fields(void *mkc, int acc, u64 start_addr,
MLX5_SET(mkc, mkc, lw, !!(acc & IB_ACCESS_LOCAL_WRITE));
MLX5_SET(mkc, mkc, lr, 1);
if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write))
MLX5_SET(mkc, mkc, relaxed_ordering_write,
(acc & IB_ACCESS_RELAXED_ORDERING) && ro_pci_enabled);
if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read))
MLX5_SET(mkc, mkc, relaxed_ordering_read,
(acc & IB_ACCESS_RELAXED_ORDERING) && ro_pci_enabled);
if ((acc & IB_ACCESS_RELAXED_ORDERING) &&
pcie_relaxed_ordering_enabled(dev->mdev->pdev)) {
if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write))
MLX5_SET(mkc, mkc, relaxed_ordering_write, 1);
if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read))
MLX5_SET(mkc, mkc, relaxed_ordering_read, 1);
}
MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
MLX5_SET(mkc, mkc, qpn, 0xffffff);
@@ -189,6 +189,25 @@ static void create_mkey_callback(int status, struct mlx5_async_work *context)
spin_unlock_irqrestore(&ent->lock, flags);
}
static int get_mkc_octo_size(unsigned int access_mode, unsigned int ndescs)
{
int ret = 0;
switch (access_mode) {
case MLX5_MKC_ACCESS_MODE_MTT:
ret = DIV_ROUND_UP(ndescs, MLX5_IB_UMR_OCTOWORD /
sizeof(struct mlx5_mtt));
break;
case MLX5_MKC_ACCESS_MODE_KSM:
ret = DIV_ROUND_UP(ndescs, MLX5_IB_UMR_OCTOWORD /
sizeof(struct mlx5_klm));
break;
default:
WARN_ON(1);
}
return ret;
}
static struct mlx5_ib_mr *alloc_cache_mr(struct mlx5_cache_ent *ent, void *mkc)
{
struct mlx5_ib_mr *mr;
@@ -204,7 +223,8 @@ static struct mlx5_ib_mr *alloc_cache_mr(struct mlx5_cache_ent *ent, void *mkc)
MLX5_SET(mkc, mkc, access_mode_1_0, ent->access_mode & 0x3);
MLX5_SET(mkc, mkc, access_mode_4_2, (ent->access_mode >> 2) & 0x7);
MLX5_SET(mkc, mkc, translations_octword_size, ent->xlt);
MLX5_SET(mkc, mkc, translations_octword_size,
get_mkc_octo_size(ent->access_mode, ent->ndescs));
MLX5_SET(mkc, mkc, log_page_size, ent->page);
return mr;
}
@@ -478,14 +498,14 @@ static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent)
return;
if (ent->available_mrs < ent->limit) {
ent->fill_to_high_water = true;
queue_work(ent->dev->cache.wq, &ent->work);
mod_delayed_work(ent->dev->cache.wq, &ent->dwork, 0);
} else if (ent->fill_to_high_water &&
ent->available_mrs + ent->pending < 2 * ent->limit) {
/*
* Once we start populating due to hitting a low water mark
* continue until we pass the high water mark.
*/
queue_work(ent->dev->cache.wq, &ent->work);
mod_delayed_work(ent->dev->cache.wq, &ent->dwork, 0);
} else if (ent->available_mrs == 2 * ent->limit) {
ent->fill_to_high_water = false;
} else if (ent->available_mrs > 2 * ent->limit) {
@@ -495,7 +515,7 @@ static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent)
queue_delayed_work(ent->dev->cache.wq, &ent->dwork,
msecs_to_jiffies(1000));
else
queue_work(ent->dev->cache.wq, &ent->work);
mod_delayed_work(ent->dev->cache.wq, &ent->dwork, 0);
}
}
@@ -571,33 +591,20 @@ static void delayed_cache_work_func(struct work_struct *work)
__cache_work_func(ent);
}
static void cache_work_func(struct work_struct *work)
{
struct mlx5_cache_ent *ent;
ent = container_of(work, struct mlx5_cache_ent, work);
__cache_work_func(ent);
}
/* Allocate a special entry from the cache */
struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev,
unsigned int entry, int access_flags)
struct mlx5_cache_ent *ent,
int access_flags)
{
struct mlx5_mr_cache *cache = &dev->cache;
struct mlx5_cache_ent *ent;
struct mlx5_ib_mr *mr;
if (WARN_ON(entry <= MR_CACHE_LAST_STD_ENTRY ||
entry >= ARRAY_SIZE(cache->ent)))
return ERR_PTR(-EINVAL);
/* Matches access in alloc_cache_mr() */
if (!mlx5_ib_can_reconfig_with_umr(dev, 0, access_flags))
return ERR_PTR(-EOPNOTSUPP);
ent = &cache->ent[entry];
spin_lock_irq(&ent->lock);
if (list_empty(&ent->head)) {
queue_adjust_cache_locked(ent);
ent->miss++;
spin_unlock_irq(&ent->lock);
mr = create_cache_mr(ent);
if (IS_ERR(mr))
@@ -611,32 +618,9 @@ struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev,
mlx5_clear_mr(mr);
}
mr->access_flags = access_flags;
return mr;
}
/* Return a MR already available in the cache */
static struct mlx5_ib_mr *get_cache_mr(struct mlx5_cache_ent *req_ent)
{
struct mlx5_ib_mr *mr = NULL;
struct mlx5_cache_ent *ent = req_ent;
spin_lock_irq(&ent->lock);
if (!list_empty(&ent->head)) {
mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list);
list_del(&mr->list);
ent->available_mrs--;
queue_adjust_cache_locked(ent);
spin_unlock_irq(&ent->lock);
mlx5_clear_mr(mr);
return mr;
}
queue_adjust_cache_locked(ent);
spin_unlock_irq(&ent->lock);
req_ent->miss++;
return NULL;
}
static void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
{
struct mlx5_cache_ent *ent = mr->cache_ent;
@@ -739,7 +723,6 @@ int mlx5_mr_cache_init(struct mlx5_ib_dev *dev)
ent->dev = dev;
ent->limit = 0;
INIT_WORK(&ent->work, cache_work_func);
INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func);
if (i > MR_CACHE_LAST_STD_ENTRY) {
@@ -751,8 +734,7 @@ int mlx5_mr_cache_init(struct mlx5_ib_dev *dev)
continue;
ent->page = PAGE_SHIFT;
ent->xlt = (1 << ent->order) * sizeof(struct mlx5_mtt) /
MLX5_IB_UMR_OCTOWORD;
ent->ndescs = 1 << ent->order;
ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT;
if ((dev->mdev->profile.mask & MLX5_PROF_MASK_MR_CACHE) &&
!dev->is_rep && mlx5_core_is_pf(dev->mdev) &&
@@ -783,7 +765,6 @@ int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev)
spin_lock_irq(&ent->lock);
ent->disabled = true;
spin_unlock_irq(&ent->lock);
cancel_work_sync(&ent->work);
cancel_delayed_work_sync(&ent->dwork);
}
@@ -972,16 +953,9 @@ static struct mlx5_ib_mr *alloc_cacheable_mr(struct ib_pd *pd,
return mr;
}
mr = get_cache_mr(ent);
if (!mr) {
mr = create_cache_mr(ent);
/*
* The above already tried to do the same stuff as reg_create(),
* no reason to try it again.
*/
if (IS_ERR(mr))
return mr;
}
mr = mlx5_mr_cache_alloc(dev, ent, access_flags);
if (IS_ERR(mr))
return mr;
mr->ibmr.pd = pd;
mr->umem = umem;

19
drivers/infiniband/hw/mlx5/odp.c
View File

@@ -407,6 +407,7 @@ static void mlx5_ib_page_fault_resume(struct mlx5_ib_dev *dev,
static struct mlx5_ib_mr *implicit_get_child_mr(struct mlx5_ib_mr *imr,
unsigned long idx)
{
struct mlx5_ib_dev *dev = mr_to_mdev(imr);
struct ib_umem_odp *odp;
struct mlx5_ib_mr *mr;
struct mlx5_ib_mr *ret;
@@ -418,13 +419,14 @@ static struct mlx5_ib_mr *implicit_get_child_mr(struct mlx5_ib_mr *imr,
if (IS_ERR(odp))
return ERR_CAST(odp);
mr = mlx5_mr_cache_alloc(
mr_to_mdev(imr), MLX5_IMR_MTT_CACHE_ENTRY, imr->access_flags);
mr = mlx5_mr_cache_alloc(dev, &dev->cache.ent[MLX5_IMR_MTT_CACHE_ENTRY],
imr->access_flags);
if (IS_ERR(mr)) {
ib_umem_odp_release(odp);
return mr;
}
mr->access_flags = imr->access_flags;
mr->ibmr.pd = imr->ibmr.pd;
mr->ibmr.device = &mr_to_mdev(imr)->ib_dev;
mr->umem = &odp->umem;
@@ -493,12 +495,15 @@ struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd,
if (IS_ERR(umem_odp))
return ERR_CAST(umem_odp);
imr = mlx5_mr_cache_alloc(dev, MLX5_IMR_KSM_CACHE_ENTRY, access_flags);
imr = mlx5_mr_cache_alloc(dev,
&dev->cache.ent[MLX5_IMR_KSM_CACHE_ENTRY],
access_flags);
if (IS_ERR(imr)) {
ib_umem_odp_release(umem_odp);
return imr;
}
imr->access_flags = access_flags;
imr->ibmr.pd = &pd->ibpd;
imr->ibmr.iova = 0;
imr->umem = &umem_odp->umem;
@@ -1593,18 +1598,14 @@ void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent *ent)
switch (ent->order - 2) {
case MLX5_IMR_MTT_CACHE_ENTRY:
ent->page = PAGE_SHIFT;
ent->xlt = MLX5_IMR_MTT_ENTRIES *
sizeof(struct mlx5_mtt) /
MLX5_IB_UMR_OCTOWORD;
ent->ndescs = MLX5_IMR_MTT_ENTRIES;
ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT;
ent->limit = 0;
break;
case MLX5_IMR_KSM_CACHE_ENTRY:
ent->page = MLX5_KSM_PAGE_SHIFT;
ent->xlt = mlx5_imr_ksm_entries *
sizeof(struct mlx5_klm) /
MLX5_IB_UMR_OCTOWORD;
ent->ndescs = mlx5_imr_ksm_entries;
ent->access_mode = MLX5_MKC_ACCESS_MODE_KSM;
ent->limit = 0;
break;

4
drivers/infiniband/hw/mlx5/qp.c
View File

@@ -31,7 +31,6 @@
*/
#include <linux/etherdevice.h>
#include <linux/module.h>
#include <rdma/ib_umem.h>
#include <rdma/ib_cache.h>
#include <rdma/ib_user_verbs.h>
@@ -615,7 +614,8 @@ enum {
static int max_bfregs(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi)
{
return get_num_static_uars(dev, bfregi) * MLX5_NON_FP_BFREGS_PER_UAR;
return get_uars_per_sys_page(dev, bfregi->lib_uar_4k) *
bfregi->num_static_sys_pages * MLX5_NON_FP_BFREGS_PER_UAR;
}
static int num_med_bfreg(struct mlx5_ib_dev *dev,

1
drivers/infiniband/hw/mlx5/srq.c
View File

@@ -3,7 +3,6 @@
* Copyright (c) 2013-2018, Mellanox Technologies inc. All rights reserved.
*/
#include <linux/module.h>
#include <linux/mlx5/qp.h>
#include <linux/slab.h>
#include <rdma/ib_umem.h>

8
drivers/infiniband/hw/mthca/mthca_main.c
View File

@@ -939,12 +939,8 @@ static int __mthca_init_one(struct pci_dev *pdev, int hca_type)
err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
if (err) {
dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask.\n");
err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
if (err) {
dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting.\n");
goto err_free_res;
}
dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting.\n");
goto err_free_res;
}
/* We can handle large RDMA requests, so allow larger segments. */

2
drivers/infiniband/hw/mthca/mthca_profile.c
View File

@@ -31,8 +31,6 @@
* SOFTWARE.
*/
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/string.h>
#include <linux/slab.h>

1
drivers/infiniband/hw/qib/qib_fs.c
View File

@@ -32,7 +32,6 @@
* SOFTWARE.
*/
#include <linux/module.h>
#include <linux/fs.h>
#include <linux/fs_context.h>
#include <linux/mount.h>

4
drivers/infiniband/hw/qib/qib_iba7220.c
View File

@@ -634,7 +634,7 @@ static const struct qib_hwerror_msgs qib_7220_hwerror_msgs[] = {
QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIECPLTIMEOUT,
"PCIe completion timeout"),
/*
* In practice, it's unlikely wthat we'll see PCIe PLL, or bus
* In practice, it's unlikely that we'll see PCIe PLL, or bus
* parity or memory parity error failures, because most likely we
* won't be able to talk to the core of the chip. Nonetheless, we
* might see them, if they are in parts of the PCIe core that aren't
@@ -2988,7 +2988,7 @@ done:
* the utility. Names need to be 12 chars or less (w/o newline), for proper
* display by utility.
* Non-error counters are first.
* Start of "error" conters is indicated by a leading "E " on the first
* Start of "error" counters is indicated by a leading "E " on the first
* "error" counter, and doesn't count in label length.
* The EgrOvfl list needs to be last so we truncate them at the configured
* context count for the device.

1
drivers/infiniband/hw/usnic/usnic_debugfs.c
View File

@@ -32,7 +32,6 @@
*/
#include <linux/debugfs.h>
#include <linux/module.h>
#include "usnic.h"
#include "usnic_log.h"

1
drivers/infiniband/hw/usnic/usnic_ib_qp_grp.c
View File

@@ -32,7 +32,6 @@
*/
#include <linux/bug.h>
#include <linux/errno.h>
#include <linux/module.h>
#include <linux/spinlock.h>
#include "usnic_log.h"

1
drivers/infiniband/hw/usnic/usnic_ib_sysfs.c
View File

@@ -31,7 +31,6 @@
*
*/
#include <linux/module.h>
#include <linux/init.h>
#include <linux/errno.h>

1
drivers/infiniband/hw/usnic/usnic_ib_verbs.c
View File

@@ -30,7 +30,6 @@
* SOFTWARE.
*
*/
#include <linux/module.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/errno.h>

1
drivers/infiniband/hw/usnic/usnic_transport.c
View File

@@ -32,7 +32,6 @@
*/
#include <linux/bitmap.h>
#include <linux/file.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <net/inet_sock.h>

1
drivers/infiniband/hw/usnic/usnic_vnic.c
View File

@@ -31,7 +31,6 @@
*
*/
#include <linux/errno.h>
#include <linux/module.h>
#include <linux/pci.h>
#include "usnic_ib.h"

10
drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c
View File

@@ -811,12 +811,10 @@ static int pvrdma_pci_probe(struct pci_dev *pdev,
}
/* Enable 64-Bit DMA */
if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0) {
ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
if (ret != 0) {
dev_err(&pdev->dev, "dma_set_mask failed\n");
goto err_free_resource;
}
ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
if (ret) {
dev_err(&pdev->dev, "dma_set_mask failed\n");
goto err_free_resource;
}
dma_set_max_seg_size(&pdev->dev, UINT_MAX);
pci_set_master(pdev);

110
drivers/infiniband/sw/rxe/rxe.c
View File

@@ -28,8 +28,8 @@ void rxe_dealloc(struct ib_device *ib_dev)
rxe_pool_cleanup(&rxe->cq_pool);
rxe_pool_cleanup(&rxe->mr_pool);
rxe_pool_cleanup(&rxe->mw_pool);
rxe_pool_cleanup(&rxe->mc_grp_pool);
rxe_pool_cleanup(&rxe->mc_elem_pool);
WARN_ON(!RB_EMPTY_ROOT(&rxe->mcg_tree));
if (rxe->tfm)
crypto_free_shash(rxe->tfm);
@@ -114,106 +114,37 @@ static void rxe_init_ports(struct rxe_dev *rxe)
}
/* init pools of managed objects */
static int rxe_init_pools(struct rxe_dev *rxe)
static void rxe_init_pools(struct rxe_dev *rxe)
{
int err;
err = rxe_pool_init(rxe, &rxe->uc_pool, RXE_TYPE_UC,
rxe->max_ucontext);
if (err)
goto err1;
err = rxe_pool_init(rxe, &rxe->pd_pool, RXE_TYPE_PD,
rxe->attr.max_pd);
if (err)
goto err2;
err = rxe_pool_init(rxe, &rxe->ah_pool, RXE_TYPE_AH,
rxe->attr.max_ah);
if (err)
goto err3;
err = rxe_pool_init(rxe, &rxe->srq_pool, RXE_TYPE_SRQ,
rxe->attr.max_srq);
if (err)
goto err4;
err = rxe_pool_init(rxe, &rxe->qp_pool, RXE_TYPE_QP,
rxe->attr.max_qp);
if (err)
goto err5;
err = rxe_pool_init(rxe, &rxe->cq_pool, RXE_TYPE_CQ,
rxe->attr.max_cq);
if (err)
goto err6;
err = rxe_pool_init(rxe, &rxe->mr_pool, RXE_TYPE_MR,
rxe->attr.max_mr);
if (err)
goto err7;
err = rxe_pool_init(rxe, &rxe->mw_pool, RXE_TYPE_MW,
rxe->attr.max_mw);
if (err)
goto err8;
err = rxe_pool_init(rxe, &rxe->mc_grp_pool, RXE_TYPE_MC_GRP,
rxe->attr.max_mcast_grp);
if (err)
goto err9;
err = rxe_pool_init(rxe, &rxe->mc_elem_pool, RXE_TYPE_MC_ELEM,
rxe->attr.max_total_mcast_qp_attach);
if (err)
goto err10;
return 0;
err10:
rxe_pool_cleanup(&rxe->mc_grp_pool);
err9:
rxe_pool_cleanup(&rxe->mw_pool);
err8:
rxe_pool_cleanup(&rxe->mr_pool);
err7:
rxe_pool_cleanup(&rxe->cq_pool);
err6:
rxe_pool_cleanup(&rxe->qp_pool);
err5:
rxe_pool_cleanup(&rxe->srq_pool);
err4:
rxe_pool_cleanup(&rxe->ah_pool);
err3:
rxe_pool_cleanup(&rxe->pd_pool);
err2:
rxe_pool_cleanup(&rxe->uc_pool);
err1:
return err;
rxe_pool_init(rxe, &rxe->uc_pool, RXE_TYPE_UC);
rxe_pool_init(rxe, &rxe->pd_pool, RXE_TYPE_PD);
rxe_pool_init(rxe, &rxe->ah_pool, RXE_TYPE_AH);
rxe_pool_init(rxe, &rxe->srq_pool, RXE_TYPE_SRQ);
rxe_pool_init(rxe, &rxe->qp_pool, RXE_TYPE_QP);
rxe_pool_init(rxe, &rxe->cq_pool, RXE_TYPE_CQ);
rxe_pool_init(rxe, &rxe->mr_pool, RXE_TYPE_MR);
rxe_pool_init(rxe, &rxe->mw_pool, RXE_TYPE_MW);
}
/* initialize rxe device state */
static int rxe_init(struct rxe_dev *rxe)
static void rxe_init(struct rxe_dev *rxe)
{
int err;
/* init default device parameters */
rxe_init_device_param(rxe);
rxe_init_ports(rxe);
err = rxe_init_pools(rxe);
if (err)
return err;
rxe_init_pools(rxe);
/* init pending mmap list */
spin_lock_init(&rxe->mmap_offset_lock);
spin_lock_init(&rxe->pending_lock);
INIT_LIST_HEAD(&rxe->pending_mmaps);
mutex_init(&rxe->usdev_lock);
/* init multicast support */
spin_lock_init(&rxe->mcg_lock);
rxe->mcg_tree = RB_ROOT;
return 0;
mutex_init(&rxe->usdev_lock);
}
void rxe_set_mtu(struct rxe_dev *rxe, unsigned int ndev_mtu)
@@ -235,12 +166,7 @@ void rxe_set_mtu(struct rxe_dev *rxe, unsigned int ndev_mtu)
*/
int rxe_add(struct rxe_dev *rxe, unsigned int mtu, const char *ibdev_name)
{
int err;
err = rxe_init(rxe);
if (err)
return err;
rxe_init(rxe);
rxe_set_mtu(rxe, mtu);
return rxe_register_device(rxe, ibdev_name);

1
drivers/infiniband/sw/rxe/rxe.h
View File

@@ -12,7 +12,6 @@
#endif
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/module.h>
#include <linux/skbuff.h>
#include <rdma/ib_verbs.h>

Some files were not shown because too many files have changed in this diff Show More