diff --git a/Documentation/ABI/testing/sysfs-driver-intel-xe-sriov b/Documentation/ABI/testing/sysfs-driver-intel-xe-sriov index 2fd7e9b7bacc08..7f5ef9eada531d 100644 --- a/Documentation/ABI/testing/sysfs-driver-intel-xe-sriov +++ b/Documentation/ABI/testing/sysfs-driver-intel-xe-sriov @@ -119,7 +119,7 @@ Description: The GT preemption timeout (PT) in [us] to be applied to all functions. See sriov_admin/{pf,vf}/profile/preempt_timeout_us for more details. - sched_priority: (RW/RO) string + sched_priority: (WO) string The GT scheduling priority to be applied for all functions. See sriov_admin/{pf,vf}/profile/sched_priority for more details. diff --git a/Documentation/PCI/endpoint/pci-vntb-howto.rst b/Documentation/PCI/endpoint/pci-vntb-howto.rst index 9a7a2f0a68498e..3679f5c3025494 100644 --- a/Documentation/PCI/endpoint/pci-vntb-howto.rst +++ b/Documentation/PCI/endpoint/pci-vntb-howto.rst @@ -52,14 +52,14 @@ pci-epf-vntb device, the following commands can be used:: # cd /sys/kernel/config/pci_ep/ # mkdir functions/pci_epf_vntb/func1 -The "mkdir func1" above creates the pci-epf-ntb function device that will +The "mkdir func1" above creates the pci-epf-vntb function device that will be probed by pci_epf_vntb driver. The PCI endpoint framework populates the directory with the following configurable fields:: - # ls functions/pci_epf_ntb/func1 - baseclass_code deviceid msi_interrupts pci-epf-ntb.0 + # ls functions/pci_epf_vntb/func1 + baseclass_code deviceid msi_interrupts pci-epf-vntb.0 progif_code secondary subsys_id vendorid cache_line_size interrupt_pin msix_interrupts primary revid subclass_code subsys_vendor_id @@ -111,13 +111,13 @@ A sample configuration for virtual NTB driver for virtual PCI bus:: # echo 0x080A > functions/pci_epf_vntb/func1/pci_epf_vntb.0/vntb_pid # echo 0x10 > functions/pci_epf_vntb/func1/pci_epf_vntb.0/vbus_number -Binding pci-epf-ntb Device to EP Controller +Binding pci-epf-vntb Device to EP Controller -------------------------------------------- NTB function device should be attached to PCI endpoint controllers connected to the host. - # ln -s controllers/5f010000.pcie_ep functions/pci-epf-ntb/func1/primary + # ln -s controllers/5f010000.pcie_ep functions/pci_epf_vntb/func1/primary Once the above step is completed, the PCI endpoint controllers are ready to establish a link with the host. @@ -139,7 +139,7 @@ lspci Output at Host side ------------------------- Note that the devices listed here correspond to the values populated in -"Creating pci-epf-ntb Device" section above:: +"Creating pci-epf-vntb Device" section above:: # lspci 00:00.0 PCI bridge: Freescale Semiconductor Inc Device 0000 (rev 01) @@ -152,7 +152,7 @@ lspci Output at EP Side / Virtual PCI bus ----------------------------------------- Note that the devices listed here correspond to the values populated in -"Creating pci-epf-ntb Device" section above:: +"Creating pci-epf-vntb Device" section above:: # lspci 10:00.0 Unassigned class [ffff]: Dawicontrol Computersysteme GmbH Device 1234 (rev ff) diff --git a/Documentation/devicetree/bindings/arm/apple.yaml b/Documentation/devicetree/bindings/arm/apple.yaml index 5c2629ec3d4cbc..13e0a79ca5b3b2 100644 --- a/Documentation/devicetree/bindings/arm/apple.yaml +++ b/Documentation/devicetree/bindings/arm/apple.yaml @@ -96,6 +96,13 @@ description: | - MacBook Pro (13-inch, M2, 2022) - Mac mini (M2, 2023) + Devices based on the "M3" SoC: + + - MacBook Air (13-inch, M3, 2024) + - MacBook Air (15-inch, M3, 2024) + - MacBook Pro (14-inch, M3, 2023) + - iMac (24-inch, M3, 2023) + Devices based on the "M1 Pro", "M1 Max" and "M1 Ultra" SoCs: - MacBook Pro (14-inch, M1 Pro, 2021) @@ -116,6 +123,14 @@ description: | - Mac Studio (M2 Ultra, 2023) - Mac Pro (M2 Ultra, 2023) + Devices based on the "M3 Pro", "M3 Max" and "M3 Ultra" SoCs: + + - MacBook Pro (14-inch, M3 Pro, 2023) + - MacBook Pro (14-inch, M3 Max, 2023) + - MacBook Pro (16-inch, M3 Pro, 2023) + - MacBook Pro (16-inch, M3 Max, 2023) + - Mac Studio (M3 Ultra, 2025) + The compatible property should follow this format: compatible = "apple,", "apple,", "apple,arm-platform"; @@ -297,6 +312,17 @@ properties: - const: apple,t8112 - const: apple,arm-platform + - description: Apple M2 SoC based platforms + items: + - enum: + - apple,j433 # iMac (24-inch, 2x USB-C, M3, 2023) + - apple,j434 # iMac (24-inch, 4x USB-C, M3, 2023) + - apple,j504 # MacBook Pro (13-inch, M3, 2023) + - apple,j613 # MacBook Air (13-inch, M3, 2024) + - apple,j615 # MacBook Air (15-inch, M3, 2024) + - const: apple,t8122 + - const: apple,arm-platform + - description: Apple M1 Pro SoC based platforms items: - enum: @@ -347,6 +373,36 @@ properties: - const: apple,t6022 - const: apple,arm-platform + - description: Apple M3 Pro SoC based platforms + items: + - enum: + - apple,j514s # MacBook Pro (14-inch, M3 Pro, 2023) + - apple,j516s # MacBook Pro (16-inch, M3 Pro, 2023) + - const: apple,t6030 + - const: apple,arm-platform + + - description: Apple M3 Max SoC based platforms + oneOf: + - items: + - enum: + - apple,j514c # MacBook Pro (14-inch, M3 Max, 16 cores, 2023) + - apple,j516c # MacBook Pro (16-inch, M3 Max, 16 cores, 2023) + - const: apple,t6031 + - const: apple,arm-platform + - items: + - enum: + - apple,j514m # MacBook Pro (14-inch, M3 Max, 14 cores, 2023) + - apple,j516m # MacBook Pro (16-inch, M3 Max, 14 cores, 2023) + - const: apple,t6034 + - const: apple,arm-platform + + - description: Apple M3 Ultra SoC based platforms + items: + - enum: + - apple,j575d # Mac Studio (M3 Ultra, 2025) + - const: apple,t6032 + - const: apple,arm-platform + additionalProperties: true ... diff --git a/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml b/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml index b88f41a225a385..44f1bd3e9e2491 100644 --- a/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml +++ b/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml @@ -36,7 +36,10 @@ properties: - const: syscon - const: simple-mfd - items: - - const: apple,t6020-pmgr + - enum: + - apple,t6020-pmgr + - apple,t6030-pmgr + - apple,t8122-pmgr - const: apple,t8103-pmgr - const: syscon - const: simple-mfd diff --git a/Documentation/devicetree/bindings/arm/cpus.yaml b/Documentation/devicetree/bindings/arm/cpus.yaml index 736b7ab1bd0a02..e1fd73281657df 100644 --- a/Documentation/devicetree/bindings/arm/cpus.yaml +++ b/Documentation/devicetree/bindings/arm/cpus.yaml @@ -85,11 +85,13 @@ properties: - apple,avalanche - apple,blizzard - apple,cyclone + - apple,everest - apple,firestorm - apple,hurricane-zephyr - apple,icestorm - apple,mistral - apple,monsoon + - apple,sawtooth - apple,twister - apple,typhoon - arm,arm710t diff --git a/Documentation/devicetree/bindings/dma/apple,sio.yaml b/Documentation/devicetree/bindings/dma/apple,sio.yaml new file mode 100644 index 00000000000000..0e3780ad9dd79a --- /dev/null +++ b/Documentation/devicetree/bindings/dma/apple,sio.yaml @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/dma/apple,sio.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple SIO Coprocessor + +description: + SIO is a coprocessor on Apple M1 and later chips (and maybe also on earlier + chips). Its role is to offload SPI, UART and DisplayPort audio transfers, + being a pretend DMA controller. + +maintainers: + - Martin Povišer + +allOf: + - $ref: dma-controller.yaml# + +properties: + compatible: + items: + - enum: + - apple,t6000-sio + - apple,t8103-sio + - const: apple,sio + + reg: + maxItems: 1 + + '#dma-cells': + const: 1 + description: + DMA clients specify a single cell that corresponds to the RTKit endpoint + number used for arranging the transfers in question + + dma-channels: + maximum: 128 + + mboxes: + maxItems: 1 + + iommus: + maxItems: 1 + + power-domains: + maxItems: 1 + + memory-region: + minItems: 2 + maxItems: 8 + description: + A number of references to reserved memory regions among which are the DATA/TEXT + sections of coprocessor executable firmware and also auxiliary firmware data + describing the available DMA-enabled peripherals + + apple,sio-firmware-params: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: | + Parameters in the form of opaque key/value pairs that are to be sent to the SIO + coprocesssor once it boots. These parameters can point into the reserved memory + regions (in device address space). + + Note that unlike Apple's firmware, we treat the parameters, and the data they + refer to, as opaque. Apple embed short data blobs into their SIO devicetree node + that describe the DMA-enabled peripherals (presumably with defined semantics). + Their driver processes those blobs and sets up data structure in mapped device + memory, then references this memory in the parameters sent to the SIO. At the + level of description we are opting for in this binding, we assume the job of + constructing those data structures has been done in advance, leaving behind an + opaque list of key/value parameter pairs to be sent by a prospective driver. + + This approach is chosen for two reasons: + + - It means we don't need to try to understand the semantics of Apple's blobs + as long as we know the transformation we need to do from Apple's devicetree + data to SIO data (which can be shoved away into a loader). It also means the + semantics of Apple's blobs (or of something to replace them) need not be part + of the binding and be kept up with Apple's firmware changes in the future. + + - It leaves less work for the driver attaching on this binding. Instead the work + is done upfront in the loader which can be better suited for keeping up with + Apple's firmware changes. + +required: + - compatible + - reg + - '#dma-cells' + - dma-channels + - mboxes + - iommus + - power-domains + +additionalProperties: false + +examples: + - | + sio: dma-controller@36400000 { + compatible = "apple,t8103-sio", "apple,sio"; + reg = <0x36400000 0x8000>; + dma-channels = <128>; + #dma-cells = <1>; + mboxes = <&sio_mbox>; + iommus = <&sio_dart 0>; + power-domains = <&ps_sio_cpu>; + memory-region = <&sio_text>, <&sio_data>, + <&sio_auxdata1>, <&sio_auxdata2>; /* Filled by loader */ + apple,sio-firmware-params = <0xb 0x10>, <0xc 0x1b80>, <0xf 0x14>, + <0x10 0x1e000>, <0x30d 0x34>, <0x30e 0x4000>, + <0x1a 0x38>, <0x1b 0x50>; /* Filled by loader */ + }; diff --git a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml new file mode 100644 index 00000000000000..2eec317bc4b3e6 --- /dev/null +++ b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml @@ -0,0 +1,86 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/hwmon/apple,smc-hwmon.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple SMC Hardware Monitoring + +description: + Apple's System Management Controller (SMC) exposes a vast array of + hardware monitoring sensors, including temperature probes, current and + voltage sense, power meters, and fan speeds. It also provides endpoints + to manually control the speed of each fan individually. Each Apple + Silicon device exposes a different set of endpoints via SMC keys. This + is true even when two machines share an SoC. The CPU core temperature + sensor keys on an M1 Mac mini are different to those on an M1 MacBook + Pro, for example. + +maintainers: + - James Calligeros + +$defs: + sensor: + type: object + + properties: + apple,key-id: + $ref: /schemas/types.yaml#/definitions/string + pattern: "^[A-Za-z0-9]{4}$" + description: The SMC FourCC key of the desired sensor. + Must match the node's suffix. + + label: + description: Human-readable name for the sensor + + required: + - apple,key-id + +properties: + compatible: + const: apple,smc-hwmon + +patternProperties: + "^current-[A-Za-z0-9]{4}$": + $ref: "#/$defs/sensor" + unevaluatedProperties: false + + "^fan-[A-Za-z0-9]{4}$": + $ref: "#/$defs/sensor" + unevaluatedProperties: false + + properties: + apple,fan-minimum: + $ref: /schemas/types.yaml#/definitions/string + pattern: "^[A-Za-z0-9]{4}$" + description: SMC key containing the fan's minimum speed + + apple,fan-maximum: + $ref: /schemas/types.yaml#/definitions/string + pattern: "^[A-Za-z0-9]{4}$" + description: SMC key containing the fan's maximum speed + + apple,fan-target: + $ref: /schemas/types.yaml#/definitions/string + pattern: "^[A-Za-z0-9]{4}$" + description: Writeable endpoint for setting desired fan speed + + apple,fan-mode: + $ref: /schemas/types.yaml#/definitions/string + pattern: "^[A-Za-z0-9]{4}$" + description: Writeable key to enable/disable manual fan control + + + "^power-[A-Za-z0-9]{4}$": + $ref: "#/$defs/sensor" + unevaluatedProperties: false + + "^temperature-[A-Za-z0-9]{4}$": + $ref: "#/$defs/sensor" + unevaluatedProperties: false + + "^voltage-[A-Za-z0-9]{4}$": + $ref: "#/$defs/sensor" + unevaluatedProperties: false + +additionalProperties: false diff --git a/Documentation/devicetree/bindings/i2c/apple,i2c.yaml b/Documentation/devicetree/bindings/i2c/apple,i2c.yaml index 500a965bdb7a84..ea320e8a22b02e 100644 --- a/Documentation/devicetree/bindings/i2c/apple,i2c.yaml +++ b/Documentation/devicetree/bindings/i2c/apple,i2c.yaml @@ -22,7 +22,10 @@ properties: compatible: oneOf: - items: - - const: apple,t6020-i2c + - enum: + - apple,t6020-i2c + - apple,t6030-i2c + - apple,t8122-i2c - const: apple,t8103-i2c - items: - enum: diff --git a/Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml b/Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml index ee5a0dfff43781..d0d9a90e96e7ba 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml @@ -4,10 +4,10 @@ $id: http://devicetree.org/schemas/interrupt-controller/apple,aic2.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Apple Interrupt Controller 2 +title: Apple Interrupt Controller 2 and 3 maintainers: - - Hector Martin + - Janne Grunau description: | The Apple Interrupt Controller 2 is a simple interrupt controller present on @@ -28,14 +28,24 @@ description: | which do not go through a discrete interrupt controller. It also handles FIQ-based Fast IPIs. + The Apple Interrupt Controller 3 is in its base functionality very similar to + the Apple Interrupt Controller 2 and uses the same device tree bindings. It is + found on Apple ARM SoCs platforms starting with t8122 (M3). + properties: compatible: - items: - - enum: - - apple,t8112-aic - - apple,t6000-aic - - apple,t6020-aic - - const: apple,aic2 + oneOf: + - items: + - enum: + - apple,t6000-aic + - apple,t6020-aic + - apple,t8112-aic + - const: apple,aic2 + - items: + - enum: + - apple,t6030-aic3 + - const: apple,t8122-aic3 + - const: apple,t8122-aic3 interrupt-controller: true @@ -117,7 +127,9 @@ allOf: properties: compatible: contains: - const: apple,t8112-aic + enum: + - apple,t8112-aic + - apple,t8122-aic3 then: properties: '#interrupt-cells': diff --git a/Documentation/devicetree/bindings/media/qcom,qcs8300-camss.yaml b/Documentation/devicetree/bindings/media/qcom,qcs8300-camss.yaml index 80a4540a22dc23..e5f170aa4d9ee7 100644 --- a/Documentation/devicetree/bindings/media/qcom,qcs8300-camss.yaml +++ b/Documentation/devicetree/bindings/media/qcom,qcs8300-camss.yaml @@ -120,6 +120,14 @@ properties: items: - const: top + vdda-phy-supply: + description: + Phandle to a 0.88V regulator supply to CSI PHYs. + + vdda-pll-supply: + description: + Phandle to 1.2V regulator supply to CSI PHYs pll block. + ports: $ref: /schemas/graph.yaml#/properties/ports @@ -160,6 +168,8 @@ required: - power-domains - power-domain-names - ports + - vdda-phy-supply + - vdda-pll-supply additionalProperties: false @@ -328,6 +338,9 @@ examples: power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>; power-domain-names = "top"; + vdda-phy-supply = <&vreg_l4a_0p88>; + vdda-pll-supply = <&vreg_l1c_1p2>; + ports { #address-cells = <1>; #size-cells = <0>; diff --git a/Documentation/devicetree/bindings/mfd/apple,smc.yaml b/Documentation/devicetree/bindings/mfd/apple,smc.yaml index 0410e712c900a7..34ce048619f5f7 100644 --- a/Documentation/devicetree/bindings/mfd/apple,smc.yaml +++ b/Documentation/devicetree/bindings/mfd/apple,smc.yaml @@ -49,6 +49,9 @@ properties: rtc: $ref: /schemas/rtc/apple,smc-rtc.yaml + hwmon: + $ref: /schemas/hwmon/apple,smc-hwmon.yaml + additionalProperties: false required: @@ -89,5 +92,38 @@ examples: nvmem-cells = <&rtc_offset>; nvmem-cell-names = "rtc_offset"; }; + + hwmon { + compatible = "apple,smc-hwmon"; + + current-ID0R { + apple,key-id = "ID0R"; + label = "AC Input Current"; + }; + + fan-F0Ac { + apple,key-id = "F0Ac"; + apple,fan-minimum = "F0Mn"; + apple,fan-maximum = "F0Mx"; + apple,fan-target = "F0Tg"; + apple,fan-mode = "F0Md"; + label = "Fan 1"; + }; + + power-PSTR { + apple,key-id = "PSTR"; + label = "Total System Power"; + }; + + temperature-TW0P { + apple,key-id = "TW0P"; + label = "WiFi/BT Module Temperature"; + }; + + voltage-VD0R { + apple,key-id = "VD0R"; + label = "AC Input Voltage"; + }; + }; }; }; diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index c0852be04f6ded..c0eb0ec87f946e 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -82,6 +82,27 @@ properties: power-domains: maxItems: 1 +patternProperties: + "^pci@": + $ref: /schemas/pci/pci-bus.yaml# + type: object + description: A single PCI root port + + properties: + reg: + maxItems: 1 + + pwren-gpios: + description: Optional GPIO to power on the device + maxItems: 1 + + required: + - reset-gpios + - interrupt-controller + - "#interrupt-cells" + - interrupt-map-mask + - interrupt-map + required: - compatible - reg @@ -161,7 +182,7 @@ examples: pinctrl-0 = <&pcie_pins>; pinctrl-names = "default"; - pci@0,0 { + port00: pci@0,0 { device_type = "pci"; reg = <0x0 0x0 0x0 0x0 0x0>; reset-gpios = <&pinctrl_ap 152 0>; @@ -169,9 +190,17 @@ examples: #address-cells = <3>; #size-cells = <2>; ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port00 0 0 0 0>, + <0 0 0 2 &port00 0 0 0 1>, + <0 0 0 3 &port00 0 0 0 2>, + <0 0 0 4 &port00 0 0 0 3>; }; - pci@1,0 { + port01: pci@1,0 { device_type = "pci"; reg = <0x800 0x0 0x0 0x0 0x0>; reset-gpios = <&pinctrl_ap 153 0>; @@ -179,9 +208,17 @@ examples: #address-cells = <3>; #size-cells = <2>; ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port01 0 0 0 0>, + <0 0 0 2 &port01 0 0 0 1>, + <0 0 0 3 &port01 0 0 0 2>, + <0 0 0 4 &port01 0 0 0 3>; }; - pci@2,0 { + port02: pci@2,0 { device_type = "pci"; reg = <0x1000 0x0 0x0 0x0 0x0>; reset-gpios = <&pinctrl_ap 33 0>; @@ -189,6 +226,14 @@ examples: #address-cells = <3>; #size-cells = <2>; ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port02 0 0 0 0>, + <0 0 0 2 &port02 0 0 0 1>, + <0 0 0 3 &port02 0 0 0 2>, + <0 0 0 4 &port02 0 0 0 3>; }; }; }; diff --git a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml new file mode 100644 index 00000000000000..0acac7e3ee67ef --- /dev/null +++ b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml @@ -0,0 +1,222 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/apple,atcphy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple Type-C PHY (ATCPHY) + +maintainers: + - Sven Peter + +description: > + The Apple Type-C PHY (ATCPHY) is a combined PHY for USB 2.0, USB 3.x, + USB4/Thunderbolt, and DisplayPort connectivity via Type-C ports found in + Apple Silicon SoCs. + + The PHY handles muxing between these different protocols and also provides the + reset controller for the attached DWC3 USB controller. + + It is designed for USB4 operation and does not handle individual differential + pairs as distinct DisplayPort lanes. Any reference to lane in this binding + hence refers to two differential pairs (RX and TX) as used in USB terminology. + + In order to correctly setup these lanes for the various modes calibration + values copied from Apple's firmware and converted to the format described + below by our bootloader m1n1 are required. Without these only USB2 operation + is possible. + +allOf: + - $ref: /schemas/usb/usb-switch.yaml# + +$defs: + apple,tunable: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: Register offset + - description: Mask to be applied to the register value + - description: Bits to be set after applying the mask + description: > + List of (register offset, mask, value) tuples copied from Apple's Device + Tree by our bootloader m1n1 and used to configure the PHY. These values + even vary for a single product/device and likely contain calibration + values determined by Apple at manufacturing time. + Unless otherwise noted these tunables are always applied to the core + register region. + +properties: + compatible: + oneOf: + - items: + - enum: + - apple,t6000-atcphy + - apple,t6020-atcphy + - apple,t8112-atcphy + - const: apple,t8103-atcphy + - const: apple,t8103-atcphy + + reg: + items: + - description: Common controls for all PHYs (USB2/3/4, DisplayPort, TBT) + - description: DisplayPort Alternate Mode PHY specific controls + - description: Type-C PHY AXI to Apple Fabric interconnect controls + - description: USB2 PHY specific controls + - description: USB3 PIPE interface controls + + reg-names: + items: + - const: core + - const: lpdptx + - const: axi2af + - const: usb2phy + - const: pipehandler + + "#phy-cells": + const: 1 + + "#reset-cells": + const: 0 + + mode-switch: true + orientation-switch: true + + power-domains: + maxItems: 1 + + ports: + $ref: /schemas/graph.yaml#/properties/ports + properties: + port@0: + $ref: /schemas/graph.yaml#/properties/port + description: Outgoing connection to the SS port of the Type-C connector. + + port@1: + $ref: /schemas/graph.yaml#/properties/port + description: Incoming endpoint from the USB3 controller. + + port@2: + $ref: /schemas/graph.yaml#/properties/port + description: Incoming endpoint from the DisplayPort controller. + + port@3: + $ref: /schemas/graph.yaml#/properties/port + description: Incoming endpoint from the USB4/Thunderbolt controller. + + apple,tunable-common-a: + $ref: "#/$defs/apple,tunable" + description: > + Common tunables required for all modes, applied before tunable-axi2af. + + apple,tunable-axi2af: + $ref: "#/$defs/apple,tunable" + description: > + AXI to Apple Fabric tunables, required for all modes. Unlike all other + tunables these are applied to the axi2af region. + + apple,tunable-common-b: + $ref: "#/$defs/apple,tunable" + description: > + Common tunables required for all modes, applied after tunable-axi2af. + + apple,tunable-lane0-usb: + $ref: "#/$defs/apple,tunable" + description: USB3 tunables for lane 0. + + apple,tunable-lane1-usb: + $ref: "#/$defs/apple,tunable" + description: USB3 tunables for lane 1. + + apple,tunable-lane0-cio: + $ref: "#/$defs/apple,tunable" + description: USB4/Thunderbolt ("Converged IO") tunables for lane 0. + + apple,tunable-lane1-cio: + $ref: "#/$defs/apple,tunable" + description: USB4/Thunderbolt ("Converged IO") tunables for lane 1. + + apple,tunable-lane0-dp: + $ref: "#/$defs/apple,tunable" + description: > + DisplayPort tunables for lane 0. + + Note that lane here refers to a USB RX and TX pair re-used for DisplayPort + and not to an individual DisplayPort differential lane. + + apple,tunable-lane1-dp: + $ref: "#/$defs/apple,tunable" + description: > + DisplayPort tunables for lane 1. + + Note that lane here refers to a USB RX and TX pair re-used for DisplayPort + and not to an individual DisplayPort differential lane. + +required: + - compatible + - reg + - reg-names + - "#phy-cells" + - "#reset-cells" + - orientation-switch + - mode-switch + - power-domains + - ports + +additionalProperties: false + +examples: + - | + phy@83000000 { + compatible = "apple,t8103-atcphy"; + reg = <0x83000000 0x4c000>, + <0x83050000 0x8000>, + <0x80000000 0x4000>, + <0x82a90000 0x4000>, + <0x82a84000 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&ps_atc0_usb>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + endpoint { + remote-endpoint = <&typec_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + + endpoint { + remote-endpoint = <&dwc3_ss_out>; + }; + }; + + port@2 { + reg = <2>; + + endpoint { + remote-endpoint = <&dcp_dp_out>; + }; + }; + + port@3 { + reg = <3>; + + endpoint { + remote-endpoint = <&acio_tbt_out>; + }; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/phy/qcom,edp-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,edp-phy.yaml index eb97181cbb9579..bfc4d75f50ff9e 100644 --- a/Documentation/devicetree/bindings/phy/qcom,edp-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,edp-phy.yaml @@ -37,12 +37,15 @@ properties: - description: PLL register block clocks: - maxItems: 2 + minItems: 2 + maxItems: 3 clock-names: + minItems: 2 items: - const: aux - const: cfg_ahb + - const: ref "#clock-cells": const: 1 @@ -64,6 +67,29 @@ required: - "#clock-cells" - "#phy-cells" +allOf: + - if: + properties: + compatible: + enum: + - qcom,x1e80100-dp-phy + then: + properties: + clocks: + minItems: 3 + maxItems: 3 + clock-names: + minItems: 3 + maxItems: 3 + else: + properties: + clocks: + minItems: 2 + maxItems: 2 + clock-names: + minItems: 2 + maxItems: 2 + additionalProperties: false examples: diff --git a/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml index 665ec79a69f1db..1d9e101745fbc7 100644 --- a/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml @@ -18,7 +18,10 @@ properties: compatible: oneOf: - items: - - const: apple,t6020-pinctrl + - enum: + - apple,t6020-pinctrl + - apple,t6030-pinctrl + - apple,t8122-pinctrl - const: apple,t8103-pinctrl - items: # Do not add additional SoC to this list. diff --git a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml index caf15188099921..9e3b394bf64d00 100644 --- a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml @@ -43,7 +43,10 @@ properties: - apple,t6000-pmgr-pwrstate - const: apple,pmgr-pwrstate - items: - - const: apple,t6020-pmgr-pwrstate + - enum: + - apple,t6020-pmgr-pwrstate + - apple,t6030-pmgr-pwrstate + - apple,t8122-pmgr-pwrstate - const: apple,t8103-pmgr-pwrstate reg: @@ -80,6 +83,18 @@ properties: minimum: 0 maximum: 15 + apple,force-disable: + description: + Forces this device to be disabled (bus access blocked) when the power + domain is powered down. + type: boolean + + apple,force-reset: + description: + Forces a reset/error recovery of the power control logic when the power + domain is powered down. + type: boolean + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml b/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml index 04519b0c581d0e..dffb72896da47b 100644 --- a/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml +++ b/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml @@ -18,8 +18,10 @@ properties: - enum: - apple,t8103-fpwm - apple,t8112-fpwm + - apple,t8122-fpwm - apple,t6000-fpwm - apple,t6020-fpwm + - apple,t6030-fpwm - const: apple,s5l-fpwm reg: diff --git a/Documentation/devicetree/bindings/sound/apple,macaudio.yaml b/Documentation/devicetree/bindings/sound/apple,macaudio.yaml new file mode 100644 index 00000000000000..8fe22dec3015d6 --- /dev/null +++ b/Documentation/devicetree/bindings/sound/apple,macaudio.yaml @@ -0,0 +1,162 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/sound/apple,macaudio.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple Silicon Macs integrated sound peripherals + +description: + This binding represents the overall machine-level integration of sound + peripherals on 'Apple Silicon' machines by Apple. + +maintainers: + - Martin Povišer + +properties: + compatible: + items: + - enum: + - apple,j274-macaudio + - apple,j293-macaudio + - apple,j314-macaudio + - const: apple,macaudio + + "#address-cells": + const: 1 + + "#size-cells": + const: 0 + + model: + description: + Model name for presentation to users + $ref: /schemas/types.yaml#/definitions/string + +patternProperties: + "^dai-link(@[0-9a-f]+)?$": + description: | + Node for each sound peripheral such as the speaker array, headphones jack, + or microphone. + type: object + + additionalProperties: false + + properties: + reg: + maxItems: 1 + + link-name: + description: | + Name for the peripheral, expecting 'Speaker' or 'Speakers' if this is + the speaker array. + $ref: /schemas/types.yaml#/definitions/string + + cpu: + type: object + + properties: + sound-dai: + description: | + DAI list with CPU-side I2S ports involved in this peripheral. + minItems: 1 + maxItems: 2 + + required: + - sound-dai + + codec: + type: object + + properties: + sound-dai: + minItems: 1 + maxItems: 8 + description: | + DAI list with the CODEC-side DAIs connected to the above CPU-side + DAIs and involved in this sound peripheral. + + The list is in left/right order if applicable. If there are more + than one CPU-side DAIs (there can be two), the CODECs must be + listed first those connected to the first CPU, then those + connected to the second. + + In addition, on some machines with many speaker codecs, the CODECs + are listed in this fixed order: + + J293: Left Front, Left Rear, Right Front, Right Rear + J314: Left Woofer 1, Left Tweeter, Left Woofer 2, + Right Woofer 1, Right Tweeter, Right Woofer 2 + + required: + - sound-dai + + required: + - reg + - cpu + - codec + +required: + - compatible + - model + +additionalProperties: false + +examples: + - | + mca: mca@9b600000 { + compatible = "apple,t6000-mca", "apple,mca"; + reg = <0x9b600000 0x10000>, + <0x9b500000 0x20000>; + + clocks = <&nco 0>, <&nco 1>, <&nco 2>, <&nco 3>; + power-domains = <&ps_audio_p>, <&ps_mca0>, <&ps_mca1>, + <&ps_mca2>, <&ps_mca3>; + dmas = <&admac 0>, <&admac 1>, <&admac 2>, <&admac 3>, + <&admac 4>, <&admac 5>, <&admac 6>, <&admac 7>, + <&admac 8>, <&admac 9>, <&admac 10>, <&admac 11>, + <&admac 12>, <&admac 13>, <&admac 14>, <&admac 15>; + dma-names = "tx0a", "rx0a", "tx0b", "rx0b", + "tx1a", "rx1a", "tx1b", "rx1b", + "tx2a", "rx2a", "tx2b", "rx2b", + "tx3a", "rx3a", "tx3b", "rx3b"; + + #sound-dai-cells = <1>; + }; + + sound { + compatible = "apple,j314-macaudio", "apple,macaudio"; + model = "MacBook Pro J314 integrated audio"; + + #address-cells = <1>; + #size-cells = <0>; + + dai-link@0 { + reg = <0>; + link-name = "Speakers"; + + cpu { + sound-dai = <&mca 0>, <&mca 1>; + }; + codec { + sound-dai = <&speaker_left_woof1>, + <&speaker_left_tweet>, + <&speaker_left_woof2>, + <&speaker_right_woof1>, + <&speaker_right_tweet>, + <&speaker_right_woof2>; + }; + }; + + dai-link@1 { + reg = <1>; + link-name = "Headphones Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/sound/asahi-kasei,ak4458.yaml b/Documentation/devicetree/bindings/sound/asahi-kasei,ak4458.yaml index 1fdbeecc5eff9d..3a3313ea0890a4 100644 --- a/Documentation/devicetree/bindings/sound/asahi-kasei,ak4458.yaml +++ b/Documentation/devicetree/bindings/sound/asahi-kasei,ak4458.yaml @@ -21,10 +21,10 @@ properties: reg: maxItems: 1 - avdd-supply: + AVDD-supply: description: Analog power supply - dvdd-supply: + DVDD-supply: description: Digital power supply reset-gpios: @@ -60,7 +60,7 @@ allOf: properties: dsd-path: false -additionalProperties: false +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/sound/asahi-kasei,ak5558.yaml b/Documentation/devicetree/bindings/sound/asahi-kasei,ak5558.yaml index d3d494ae8abfeb..dc8f85f266bf30 100644 --- a/Documentation/devicetree/bindings/sound/asahi-kasei,ak5558.yaml +++ b/Documentation/devicetree/bindings/sound/asahi-kasei,ak5558.yaml @@ -19,10 +19,10 @@ properties: reg: maxItems: 1 - avdd-supply: + AVDD-supply: description: A 1.8V supply that powers up the AVDD pin. - dvdd-supply: + DVDD-supply: description: A 1.2V supply that powers up the DVDD pin. reset-gpios: diff --git a/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml b/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml index 05602678c070d7..db0d11aa0a3e15 100644 --- a/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml @@ -16,7 +16,10 @@ properties: compatible: oneOf: - items: - - const: apple,t6020-wdt + - enum: + - apple,t6020-wdt + - apple,t6030-wdt + - apple,t8122-wdt - const: apple,t8103-wdt - items: - enum: diff --git a/Documentation/networking/ip-sysctl.rst b/Documentation/networking/ip-sysctl.rst index bc9a01606daf5a..2c65d57103fb1e 100644 --- a/Documentation/networking/ip-sysctl.rst +++ b/Documentation/networking/ip-sysctl.rst @@ -3232,12 +3232,13 @@ enhanced_dad - BOOLEAN =========== ratelimit - INTEGER - Limit the maximal rates for sending ICMPv6 messages. + Limit the maximal rates for sending ICMPv6 messages to a particular + peer. 0 to disable any limiting, - otherwise the minimal space between responses in milliseconds. + otherwise the space between responses in milliseconds. - Default: 1000 + Default: 100 ratemask - list of comma separated ranges For ICMPv6 message types matching the ranges in the ratemask, limit diff --git a/Documentation/trace/events-pci.rst b/Documentation/trace/events-pci.rst new file mode 100644 index 00000000000000..03ff4ad30ddfa1 --- /dev/null +++ b/Documentation/trace/events-pci.rst @@ -0,0 +1,74 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=========================== +Subsystem Trace Points: PCI +=========================== + +Overview +======== +The PCI tracing system provides tracepoints to monitor critical hardware events +that can impact system performance and reliability. These events normally show +up here: + + /sys/kernel/tracing/events/pci + +Cf. include/trace/events/pci.h for the events definitions. + +Available Tracepoints +===================== + +pci_hp_event +------------ + +Monitors PCI hotplug events including card insertion/removal and link +state changes. +:: + + pci_hp_event "%s slot:%s, event:%s\n" + +**Event Types**: + +* ``LINK_UP`` - PCIe link established +* ``LINK_DOWN`` - PCIe link lost +* ``CARD_PRESENT`` - Card detected in slot +* ``CARD_NOT_PRESENT`` - Card removed from slot + +**Example Usage**:: + + # Enable the tracepoint + echo 1 > /sys/kernel/debug/tracing/events/pci/pci_hp_event/enable + + # Monitor events (the following output is generated when a device is hotplugged) + cat /sys/kernel/debug/tracing/trace_pipe + irq/51-pciehp-88 [001] ..... 1311.177459: pci_hp_event: 0000:00:02.0 slot:10, event:CARD_PRESENT + + irq/51-pciehp-88 [001] ..... 1311.177566: pci_hp_event: 0000:00:02.0 slot:10, event:LINK_UP + +pcie_link_event +--------------- + +Monitors PCIe link speed changes and provides detailed link status information. +:: + + pcie_link_event "%s type:%d, reason:%d, cur_bus_speed:%d, max_bus_speed:%d, width:%u, flit_mode:%u, status:%s\n" + +**Parameters**: + +* ``type`` - PCIe device type (4=Root Port, etc.) +* ``reason`` - Reason for link change: + + - ``0`` - Link retrain + - ``1`` - Bus enumeration + - ``2`` - Bandwidth notification enable + - ``3`` - Bandwidth notification IRQ + - ``4`` - Hotplug event + + +**Example Usage**:: + + # Enable the tracepoint + echo 1 > /sys/kernel/debug/tracing/events/pci/pcie_link_event/enable + + # Monitor events (the following output is generated when a device is hotplugged) + cat /sys/kernel/debug/tracing/trace_pipe + irq/51-pciehp-88 [001] ..... 381.545386: pcie_link_event: 0000:00:02.0 type:4, reason:4, cur_bus_speed:20, max_bus_speed:23, width:1, flit_mode:0, status:DLLLA diff --git a/Documentation/trace/index.rst b/Documentation/trace/index.rst index b4a429dc4f7ad6..0a40bfabcf19bf 100644 --- a/Documentation/trace/index.rst +++ b/Documentation/trace/index.rst @@ -54,6 +54,7 @@ applications. events-power events-nmi events-msr + events-pci boottime-trace histogram histogram-design diff --git a/MAINTAINERS b/MAINTAINERS index 4dc3be39bbb372..f55ba7631faa4e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2441,9 +2441,11 @@ M: Martin Povišer L: asahi@lists.linux.dev L: linux-sound@vger.kernel.org S: Maintained +F: Documentation/devicetree/bindings/dma/apple,sio.yaml F: Documentation/devicetree/bindings/sound/apple,* F: Documentation/devicetree/bindings/sound/cirrus,cs42l84.yaml F: Documentation/devicetree/bindings/sound/trivial-codec.yaml +F: drivers/dma/apple-sio.c F: sound/soc/apple/* F: sound/soc/codecs/cs42l83-i2c.c F: sound/soc/codecs/cs42l84.* @@ -2467,6 +2469,7 @@ F: Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml F: Documentation/devicetree/bindings/dma/apple,admac.yaml F: Documentation/devicetree/bindings/gpio/apple,smc-gpio.yaml F: Documentation/devicetree/bindings/gpu/apple,agx.yaml +F: Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml F: Documentation/devicetree/bindings/i2c/apple,i2c.yaml F: Documentation/devicetree/bindings/input/touchscreen/apple,z2-multitouch.yaml F: Documentation/devicetree/bindings/interrupt-controller/apple,* @@ -2480,6 +2483,7 @@ F: Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml F: Documentation/devicetree/bindings/nvmem/apple,efuses.yaml F: Documentation/devicetree/bindings/nvmem/apple,spmi-nvmem.yaml F: Documentation/devicetree/bindings/pci/apple,pcie.yaml +F: Documentation/devicetree/bindings/phy/apple,atcphy.yaml F: Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml F: Documentation/devicetree/bindings/power/apple* F: Documentation/devicetree/bindings/power/reset/apple,smc-reboot.yaml @@ -2500,6 +2504,7 @@ F: drivers/hwmon/macsmc-hwmon.c F: drivers/pmdomain/apple/ F: drivers/i2c/busses/i2c-pasemi-core.c F: drivers/i2c/busses/i2c-pasemi-platform.c +F: drivers/input/misc/macsmc-input.c F: drivers/input/touchscreen/apple_z2.c F: drivers/iommu/apple-dart.c F: drivers/iommu/io-pgtable-dart.c @@ -2508,6 +2513,7 @@ F: drivers/mfd/macsmc.c F: drivers/nvme/host/apple.c F: drivers/nvmem/apple-efuses.c F: drivers/nvmem/apple-spmi-nvmem.c +F: drivers/phy/apple/ F: drivers/pinctrl/pinctrl-apple-gpio.c F: drivers/power/reset/macsmc-reboot.c F: drivers/pwm/pwm-apple.c diff --git a/Makefile b/Makefile index d3a8482bdbd0de..f486050e0bee49 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 VERSION = 6 PATCHLEVEL = 19 -SUBLEVEL = 0 +SUBLEVEL = 5 EXTRAVERSION = NAME = Baby Opossum Posse @@ -1474,6 +1474,15 @@ ifneq ($(wildcard $(resolve_btfids_O)),) $(Q)$(MAKE) -sC $(srctree)/tools/bpf/resolve_btfids O=$(resolve_btfids_O) clean endif +PHONY += objtool_clean + +objtool_O = $(abspath $(objtree))/tools/objtool + +objtool_clean: +ifneq ($(wildcard $(objtool_O)),) + $(Q)$(MAKE) -sC $(abs_srctree)/tools/objtool O=$(objtool_O) srctree=$(abs_srctree) clean +endif + tools/: FORCE $(Q)mkdir -p $(objtree)/tools $(Q)$(MAKE) O=$(abspath $(objtree)) subdir=tools -C $(srctree)/tools/ @@ -1637,7 +1646,7 @@ vmlinuxclean: $(Q)$(CONFIG_SHELL) $(srctree)/scripts/link-vmlinux.sh clean $(Q)$(if $(ARCH_POSTLINK), $(MAKE) -f $(ARCH_POSTLINK) clean) -clean: archclean vmlinuxclean resolve_btfids_clean +clean: archclean vmlinuxclean resolve_btfids_clean objtool_clean # mrproper - Delete all generated files, including .config # diff --git a/arch/arm/boot/dts/allwinner/sun5i-a13-utoo-p66.dts b/arch/arm/boot/dts/allwinner/sun5i-a13-utoo-p66.dts index be486d28d04fae..428cab5a0e906e 100644 --- a/arch/arm/boot/dts/allwinner/sun5i-a13-utoo-p66.dts +++ b/arch/arm/boot/dts/allwinner/sun5i-a13-utoo-p66.dts @@ -102,6 +102,7 @@ /* The P66 uses a different EINT then the reference design */ interrupts = <6 9 IRQ_TYPE_EDGE_FALLING>; /* EINT9 (PG9) */ /* The icn8318 binding expects wake-gpios instead of power-gpios */ + /delete-property/ power-gpios; wake-gpios = <&pio 1 3 GPIO_ACTIVE_HIGH>; /* PB3 */ touchscreen-size-x = <800>; touchscreen-size-y = <480>; diff --git a/arch/arm/boot/dts/nxp/lpc/lpc32xx.dtsi b/arch/arm/boot/dts/nxp/lpc/lpc32xx.dtsi index 2236901a003130..8e9ed93da129e8 100644 --- a/arch/arm/boot/dts/nxp/lpc/lpc32xx.dtsi +++ b/arch/arm/boot/dts/nxp/lpc/lpc32xx.dtsi @@ -302,6 +302,7 @@ mpwm: pwm@400e8000 { compatible = "nxp,lpc3220-motor-pwm"; reg = <0x400e8000 0x78>; + clocks = <&clk LPC32XX_CLK_MCPWM>; #pwm-cells = <3>; status = "disabled"; }; diff --git a/arch/arm/kernel/vdso.c b/arch/arm/kernel/vdso.c index e38a30477f3d70..566c40f0f7c77b 100644 --- a/arch/arm/kernel/vdso.c +++ b/arch/arm/kernel/vdso.c @@ -161,6 +161,7 @@ static void __init patch_vdso(void *ehdr) vdso_nullpatch_one(&einfo, "__vdso_gettimeofday"); vdso_nullpatch_one(&einfo, "__vdso_clock_gettime"); vdso_nullpatch_one(&einfo, "__vdso_clock_gettime64"); + vdso_nullpatch_one(&einfo, "__vdso_clock_getres"); } } diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 93173f0a09c7de..5fede9c28b4897 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -444,6 +444,9 @@ config KASAN_SHADOW_OFFSET config UNWIND_TABLES bool +config ARM64_ACTLR_STATE + bool + source "arch/arm64/Kconfig.platforms" menu "Kernel Features" @@ -2294,6 +2297,16 @@ config ARM64_DEBUG_PRIORITY_MASKING If unsure, say N endif # ARM64_PSEUDO_NMI +config ARM64_MEMORY_MODEL_CONTROL + bool "Runtime memory model control" + select ARM64_ACTLR_STATE + help + Some ARM64 CPUs support runtime switching of the CPU memory + model, which can be useful to emulate other CPU architectures + which have different memory models. Say Y to enable support + for the PR_SET_MEM_MODEL/PR_GET_MEM_MODEL prctl() calls on + CPUs with this feature. + config RELOCATABLE bool "Build a relocatable kernel image" if EXPERT select ARCH_HAS_RELR diff --git a/arch/arm64/boot/dts/amlogic/amlogic-c3.dtsi b/arch/arm64/boot/dts/amlogic/amlogic-c3.dtsi index 13b7ac03f9b201..885d9d64e9e4ec 100644 --- a/arch/arm64/boot/dts/amlogic/amlogic-c3.dtsi +++ b/arch/arm64/boot/dts/amlogic/amlogic-c3.dtsi @@ -969,6 +969,10 @@ no-sd; resets = <&reset RESET_SD_EMMC_A>; status = "disabled"; + + assigned-clocks = <&clkc_periphs CLKID_SD_EMMC_A>; + assigned-clock-rates = <24000000>; + }; sd: mmc@8a000 { @@ -984,6 +988,9 @@ no-sdio; resets = <&reset RESET_SD_EMMC_B>; status = "disabled"; + + assigned-clocks = <&clkc_periphs CLKID_SD_EMMC_B>; + assigned-clock-rates = <24000000>; }; nand: nand-controller@8d000 { diff --git a/arch/arm64/boot/dts/amlogic/meson-axg.dtsi b/arch/arm64/boot/dts/amlogic/meson-axg.dtsi index e95c91894968b2..cc72491eaf6f52 100644 --- a/arch/arm64/boot/dts/amlogic/meson-axg.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-axg.dtsi @@ -1960,6 +1960,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_B>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_B_CLK0>; + assigned-clock-rates = <24000000>; }; sd_emmc_c: mmc@7000 { @@ -1972,6 +1975,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_C>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_C_CLK0>; + assigned-clock-rates = <24000000>; }; nfc: nand-controller@7800 { diff --git a/arch/arm64/boot/dts/amlogic/meson-g12-common.dtsi b/arch/arm64/boot/dts/amlogic/meson-g12-common.dtsi index ca455f634834b5..00609d2da67437 100644 --- a/arch/arm64/boot/dts/amlogic/meson-g12-common.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-g12-common.dtsi @@ -2431,6 +2431,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_A>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_A_CLK0>; + assigned-clock-rates = <24000000>; }; sd_emmc_b: mmc@ffe05000 { @@ -2443,6 +2446,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_B>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_B_CLK0>; + assigned-clock-rates = <24000000>; }; sd_emmc_c: mmc@ffe07000 { @@ -2455,6 +2461,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_C>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_C_CLK0>; + assigned-clock-rates = <24000000>; }; usb: usb@ffe09000 { diff --git a/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi b/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi index f69923da07febd..a9c830a570cc6c 100644 --- a/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi @@ -824,6 +824,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_A>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_A_CLK0>; + assigned-clock-rates = <24000000>; }; &sd_emmc_b { @@ -832,6 +835,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_B>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_B_CLK0>; + assigned-clock-rates = <24000000>; }; &sd_emmc_c { @@ -840,6 +846,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_C>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_C_CLK0>; + assigned-clock-rates = <24000000>; }; &simplefb_hdmi { diff --git a/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi b/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi index ba535010a3c91d..e202d84f067205 100644 --- a/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi @@ -894,6 +894,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_A>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_A_CLK0>; + assigned-clock-rates = <24000000>; }; &sd_emmc_b { @@ -902,6 +905,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_B>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_B_CLK0>; + assigned-clock-rates = <24000000>; }; &sd_emmc_c { @@ -910,6 +916,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_C>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_C_CLK0>; + assigned-clock-rates = <24000000>; }; &simplefb_hdmi { diff --git a/arch/arm64/boot/dts/amlogic/meson-s4.dtsi b/arch/arm64/boot/dts/amlogic/meson-s4.dtsi index 9d99ed2994dfa2..dfc0a30a6e61be 100644 --- a/arch/arm64/boot/dts/amlogic/meson-s4.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-s4.dtsi @@ -819,13 +819,16 @@ reg = <0x0 0xfe088000 0x0 0x800>; interrupts = ; clocks = <&clkc_periphs CLKID_SDEMMC_A>, - <&xtal>, + <&clkc_periphs CLKID_SD_EMMC_A>, <&clkc_pll CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_A>; cap-sdio-irq; keep-power-in-suspend; status = "disabled"; + + assigned-clocks = <&clkc_periphs CLKID_SD_EMMC_A>; + assigned-clock-rates = <24000000>; }; sd: mmc@fe08a000 { @@ -838,6 +841,9 @@ clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_B>; status = "disabled"; + + assigned-clocks = <&clkc_periphs CLKID_SD_EMMC_B>; + assigned-clock-rates = <24000000>; }; emmc: mmc@fe08c000 { @@ -845,13 +851,16 @@ reg = <0x0 0xfe08c000 0x0 0x800>; interrupts = ; clocks = <&clkc_periphs CLKID_NAND>, - <&xtal>, + <&clkc_periphs CLKID_SD_EMMC_C>, <&clkc_pll CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_NAND_EMMC>; no-sdio; no-sd; status = "disabled"; + + assigned-clocks = <&clkc_periphs CLKID_SD_EMMC_C>; + assigned-clock-rates = <24000000>; }; }; }; diff --git a/arch/arm64/boot/dts/amlogic/meson-sm1-odroid-hc4.dts b/arch/arm64/boot/dts/amlogic/meson-sm1-odroid-hc4.dts index 0170139b8d32f4..3ece30a0a1fff7 100644 --- a/arch/arm64/boot/dts/amlogic/meson-sm1-odroid-hc4.dts +++ b/arch/arm64/boot/dts/amlogic/meson-sm1-odroid-hc4.dts @@ -52,6 +52,7 @@ gpio = <&gpio GPIOH_8 GPIO_OPEN_DRAIN>; enable-active-high; + regulator-boot-on; regulator-always-on; }; @@ -65,6 +66,7 @@ gpio = <&gpio GPIOH_8 GPIO_OPEN_DRAIN>; enable-active-high; + regulator-boot-on; regulator-always-on; }; diff --git a/arch/arm64/boot/dts/amlogic/meson-sm1-odroid.dtsi b/arch/arm64/boot/dts/amlogic/meson-sm1-odroid.dtsi index c4524eb4f0996d..0bce4e8d965f2c 100644 --- a/arch/arm64/boot/dts/amlogic/meson-sm1-odroid.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-sm1-odroid.dtsi @@ -37,6 +37,7 @@ gpio = <&gpio_ao GPIOAO_3 GPIO_OPEN_DRAIN>; enable-active-high; + regulator-boot-on; regulator-always-on; }; @@ -50,6 +51,7 @@ enable-gpios = <&gpio_ao GPIOE_2 GPIO_OPEN_DRAIN>; enable-active-high; + regulator-boot-on; regulator-always-on; gpios = <&gpio_ao GPIOAO_6 GPIO_OPEN_SOURCE>; @@ -81,6 +83,7 @@ regulator-name = "5V"; regulator-min-microvolt = <5000000>; regulator-max-microvolt = <5000000>; + regulator-boot-on; regulator-always-on; vin-supply = <&main_12v>; gpio = <&gpio GPIOH_8 GPIO_OPEN_DRAIN>; diff --git a/arch/arm64/boot/dts/apple/Makefile b/arch/arm64/boot/dts/apple/Makefile index 4eebcd85c90fcf..d253f08c98329e 100644 --- a/arch/arm64/boot/dts/apple/Makefile +++ b/arch/arm64/boot/dts/apple/Makefile @@ -91,3 +91,6 @@ dtb-$(CONFIG_ARCH_APPLE) += t8112-j413.dtb dtb-$(CONFIG_ARCH_APPLE) += t8112-j415.dtb dtb-$(CONFIG_ARCH_APPLE) += t8112-j473.dtb dtb-$(CONFIG_ARCH_APPLE) += t8112-j493.dtb +dtb-$(CONFIG_ARCH_APPLE) += t8122-j433.dtb +dtb-$(CONFIG_ARCH_APPLE) += t8122-j434.dtb +dtb-$(CONFIG_ARCH_APPLE) += t8122-j504.dtb diff --git a/arch/arm64/boot/dts/apple/hwmon-common.dtsi b/arch/arm64/boot/dts/apple/hwmon-common.dtsi new file mode 100644 index 00000000000000..2a74d9a114abb6 --- /dev/null +++ b/arch/arm64/boot/dts/apple/hwmon-common.dtsi @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * hwmon sensors expected on all systems + * + * Copyright The Asahi Linux Contributors + */ + +&smc_hwmon { + power-PSTR { + apple,key-id = "PSTR"; + label = "Total System Power"; + }; + power-PDTR { + apple,key-id = "PDTR"; + label = "AC Input Power"; + }; + power-PMVR { + apple,key-id = "PMVR"; + label = "3.8 V Rail Power"; + }; + temperature-TH0x { + apple,key-id = "TH0x"; + label = "NAND Flash Temperature"; + }; + voltage-VD0R { + apple,key-id = "VD0R"; + label = "AC Input Voltage"; + }; + current-ID0R { + apple,key-id = "ID0R"; + label = "AC Input Current"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/hwmon-fan-dual.dtsi b/arch/arm64/boot/dts/apple/hwmon-fan-dual.dtsi new file mode 100644 index 00000000000000..61c34692f1cd5a --- /dev/null +++ b/arch/arm64/boot/dts/apple/hwmon-fan-dual.dtsi @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Copyright The Asahi Linux Contributors + * + * Fan hwmon sensors for machines with 2 fan. + */ + +#include "hwmon-fan.dtsi" + +&smc_hwmon { + fan-F0Ac { + label = "Fan 1"; + }; + fan-F1Ac { + apple,key-id = "F1Ac"; + label = "Fan 2"; + apple,fan-minimum = "F1Mn"; + apple,fan-maximum = "F1Mx"; + apple,fan-target = "F1Tg"; + apple,fan-mode = "F1Md"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/hwmon-fan.dtsi b/arch/arm64/boot/dts/apple/hwmon-fan.dtsi new file mode 100644 index 00000000000000..180eb8d7441f44 --- /dev/null +++ b/arch/arm64/boot/dts/apple/hwmon-fan.dtsi @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Copyright The Asahi Linux Contributors + * + * Fan hwmon sensors for machines with a single fan. + */ + +&smc_hwmon { + fan-F0Ac { + apple,key-id = "F0Ac"; + label = "Fan"; + apple,fan-minimum = "F0Mn"; + apple,fan-maximum = "F0Mx"; + apple,fan-target = "F0Tg"; + apple,fan-mode = "F0Md"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/hwmon-laptop.dtsi b/arch/arm64/boot/dts/apple/hwmon-laptop.dtsi new file mode 100644 index 00000000000000..4afb91ee69fe76 --- /dev/null +++ b/arch/arm64/boot/dts/apple/hwmon-laptop.dtsi @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * hwmon sensors expected on all laptops + * + * Copyright The Asahi Linux Contributors + */ + +&smc_hwmon { + power-PHPC { + apple,key-id = "PHPC"; + label = "Heatpipe Power"; + }; + temperature-TB0T { + apple,key-id = "TB0T"; + label = "Battery Hotspot"; + }; + temperature-TCHP { + apple,key-id = "TCHP"; + label = "Charge Regulator Temp"; + }; + temperature-TW0P { + apple,key-id = "TW0P"; + label = "WiFi/BT Module Temp"; + }; + voltage-SBAV { + apple,key-id = "SBAV"; + label = "Battery Voltage"; + }; + voltage-VD0R { + apple,key-id = "VD0R"; + label = "Charger Input Voltage"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/hwmon-mini.dtsi b/arch/arm64/boot/dts/apple/hwmon-mini.dtsi new file mode 100644 index 00000000000000..7fd86e911acfe7 --- /dev/null +++ b/arch/arm64/boot/dts/apple/hwmon-mini.dtsi @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * hwmon sensors common to the Mac mini desktop + * models, but not the Studio or Pro. + * + * Copyright The Asahi Linux Contributors + */ + +#include "hwmon-fan.dtsi" + +&smc_hwmon { + temperature-TW0P { + apple,key-id = "TW0P"; + label = "WiFi/BT Module Temp"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/isp-common.dtsi b/arch/arm64/boot/dts/apple/isp-common.dtsi new file mode 100644 index 00000000000000..739e6e9e66e740 --- /dev/null +++ b/arch/arm64/boot/dts/apple/isp-common.dtsi @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Common ISP configuration for Apple silicon platforms. + * + * Copyright The Asahi Linux Contributors + */ + +/ { + aliases { + isp = &isp; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + isp_heap: isp-heap { + compatible = "apple,asc-mem"; + /* Filled in by bootloder */ + reg = <0 0 0 0>; + no-map; + }; + }; +}; + +&isp { + memory-region = <&isp_heap>; + memory-region-names = "heap"; + status = "okay"; +}; + +&isp_dart0 { + status = "okay"; +}; + +&isp_dart1 { + status = "okay"; +}; + +&isp_dart2 { + status = "okay"; +}; + +&ps_isp_sys { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/apple/isp-imx248.dtsi b/arch/arm64/boot/dts/apple/isp-imx248.dtsi new file mode 100644 index 00000000000000..0a4ac1a0152c2c --- /dev/null +++ b/arch/arm64/boot/dts/apple/isp-imx248.dtsi @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * ISP configuration for platforms with IMX248 sensor. + * + * Copyright The Asahi Linux Contributors + */ + +#include "isp-common.dtsi" + +&isp { + apple,temporal-filter = <0>; + + sensor-presets { + /* 1280x720 */ + preset0 { + apple,config-index = <0>; + apple,input-size = <1296 736>; + apple,output-size = <1280 720>; + apple,crop = <8 8 1280 720>; + }; + /* 960x720 (4:3) */ + preset1 { + apple,config-index = <0>; + apple,input-size = <1296 736>; + apple,output-size = <960 720>; + apple,crop = <168 8 960 720>; + }; + /* 960x540 (16:9) */ + preset2 { + apple,config-index = <0>; + apple,input-size = <1296 736>; + apple,output-size = <960 540>; + apple,crop = <8 8 1280 720>; + }; + /* 640x480 (4:3) */ + preset3 { + apple,config-index = <0>; + apple,input-size = <1296 736>; + apple,output-size = <640 480>; + apple,crop = <168 8 960 720>; + }; + /* 640x360 (16:9) */ + preset4 { + apple,config-index = <0>; + apple,input-size = <1296 736>; + apple,output-size = <640 360>; + apple,crop = <8 8 1280 720>; + }; + /* 320x180 (16:9) */ + preset5 { + apple,config-index = <0>; + apple,input-size = <1296 736>; + apple,output-size = <320 180>; + apple,crop = <8 8 1280 720>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/isp-imx364.dtsi b/arch/arm64/boot/dts/apple/isp-imx364.dtsi new file mode 100644 index 00000000000000..55484d86523657 --- /dev/null +++ b/arch/arm64/boot/dts/apple/isp-imx364.dtsi @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * ISP configuration for platforms with IMX364 sensor. + * + * Copyright The Asahi Linux Contributors + */ + +#include "isp-common.dtsi" + +&isp { + apple,temporal-filter = <0>; + + sensor-presets { + /* 1920x1080 */ + preset0 { + apple,config-index = <0>; + apple,input-size = <1920 1080>; + apple,output-size = <1920 1080>; + apple,crop = <0 0 1920 1080>; + }; + /* 1440x720 (4:3) */ + preset1 { + apple,config-index = <0>; + apple,input-size = <1920 1080>; + apple,output-size = <1440 1080>; + apple,crop = <240 0 1440 1080>; + }; + /* 1280x720 (16:9) */ + preset2 { + apple,config-index = <0>; + apple,input-size = <1920 1080>; + apple,output-size = <1280 720>; + apple,crop = <0 0 1920 1080>; + }; + /* 960x720 (4:3) */ + preset3{ + apple,config-index = <0>; + apple,input-size = <1920 1080>; + apple,output-size = <960 720>; + apple,crop = <240 0 1440 1080>; + }; + /* 960x540 (16:9) */ + preset4 { + apple,config-index = <0>; + apple,input-size = <1920 1080>; + apple,output-size = <960 540>; + apple,crop = <0 0 1920 1080>; + }; + /* 640x480 (4:3) */ + preset5 { + apple,config-index = <0>; + apple,input-size = <1920 1080>; + apple,output-size = <640 480>; + apple,crop = <240 0 1440 1080>; + }; + /* 640x360 (16:9) */ + preset6 { + apple,config-index = <0>; + apple,input-size = <1920 1080>; + apple,output-size = <640 360>; + apple,crop = <0 0 1920 1080>; + }; + /* 320x180 (16:9) */ + preset7 { + apple,config-index = <0>; + apple,input-size = <1920 1080>; + apple,output-size = <320 180>; + apple,crop = <0 0 1920 1080>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/isp-imx558-cfg0.dtsi b/arch/arm64/boot/dts/apple/isp-imx558-cfg0.dtsi new file mode 100644 index 00000000000000..729b97829cbb7e --- /dev/null +++ b/arch/arm64/boot/dts/apple/isp-imx558-cfg0.dtsi @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * ISP configuration for platforms with IMX558 sensor in + * config #0 mode. + * + * These platforms enable MLVNR for all configs except + * #0, which we don't support. Config #0 is an uncropped + * square 1920x1920 sensor, with dark corners. + * Therefore, we synthesize common resolutions by using + * crop/scale while always choosing config #0. + * + * Copyright The Asahi Linux Contributors + */ + +#include "isp-common.dtsi" + +&isp { + apple,temporal-filter = <0>; + + sensor-presets { + /* 1920x1080 */ + preset0 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <1920 1080>; + apple,crop = <0 420 1920 1080>; + }; + /* 1080x1920 */ + preset1 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <1080 1920>; + apple,crop = <420 0 1080 1920>; + }; + /* 1920x1440 */ + preset2 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <1920 1440>; + apple,crop = <0 240 1920 1440>; + }; + /* 1440x1920 */ + preset3 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <1440 1920>; + apple,crop = <240 0 1440 1920>; + }; + /* 1280x720 */ + preset4 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <1280 720>; + apple,crop = <0 420 1920 1080>; + }; + /* 720x1280 */ + preset5 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <720 1280>; + apple,crop = <420 0 1080 1920>; + }; + /* 1280x960 */ + preset6 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <1280 960>; + apple,crop = <0 240 1920 1440>; + }; + /* 960x1280 */ + preset7 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <960 1280>; + apple,crop = <240 0 1440 1920>; + }; + /* 640x480 */ + preset8 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <640 480>; + apple,crop = <0 240 1920 1440>; + }; + /* 480x640 */ + preset9 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <480 640>; + apple,crop = <240 0 1440 1920>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/isp-imx558.dtsi b/arch/arm64/boot/dts/apple/isp-imx558.dtsi new file mode 100644 index 00000000000000..d55854c883f5b6 --- /dev/null +++ b/arch/arm64/boot/dts/apple/isp-imx558.dtsi @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * ISP configuration for platforms with IMX558 sensor. + * + * Copyright The Asahi Linux Contributors + */ + +#include "isp-common.dtsi" + +&isp { + apple,temporal-filter = <0>; + + sensor-presets { + /* 1920x1080 */ + preset0 { + apple,config-index = <1>; + apple,input-size = <1920 1080>; + apple,output-size = <1920 1080>; + apple,crop = <0 0 1920 1080>; + }; + /* 1080x1920 */ + preset1 { + apple,config-index = <2>; + apple,input-size = <1080 1920>; + apple,output-size = <1080 1920>; + apple,crop = <0 0 1080 1920>; + }; + /* 1760x1328 */ + preset2 { + apple,config-index = <3>; + apple,input-size = <1760 1328>; + apple,output-size = <1760 1328>; + apple,crop = <0 0 1760 1328>; + }; + /* 1328x1760 */ + preset3 { + apple,config-index = <4>; + apple,input-size = <1328 1760>; + apple,output-size = < 1328 1760>; + apple,crop = <0 0 1328 1760>; + }; + /* 1152x1152 */ + preset4 { + apple,config-index = <5>; + apple,input-size = <1152 1152>; + apple,output-size = <1152 1152>; + apple,crop = <0 0 1152 1152>; + }; + /* 1280x720 */ + preset5 { + apple,config-index = <1>; + apple,input-size = <1920 1080>; + apple,output-size = <1280 720>; + apple,crop = <0 0 1920 1080>; + }; + /* 720x1280 */ + preset6 { + apple,config-index = <2>; + apple,input-size = <1080 1920>; + apple,output-size = <720 1280>; + apple,crop = <0 0 1080 1920>; + }; + /* 1280x960 */ + preset7 { + apple,config-index = <3>; + apple,input-size = <1760 1328>; + apple,output-size = <1280 960>; + apple,crop = <0 4 1760 1320>; + }; + /* 960x1280 */ + preset8 { + apple,config-index = <4>; + apple,input-size = <1328 1760>; + apple,output-size = <960 1280>; + apple,crop = <4 0 1320 1760>; + }; + /* 640x480 */ + preset9 { + apple,config-index = <3>; + apple,input-size = <1760 1328>; + apple,output-size = <640 480>; + apple,crop = <0 4 1760 1320>; + }; + /* 480x640 */ + preset10 { + apple,config-index = <4>; + apple,input-size = <1328 1760>; + apple,output-size = <480 640>; + apple,crop = <4 0 1320 1760>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/s8001-j98a-j99a.dtsi b/arch/arm64/boot/dts/apple/s8001-j98a-j99a.dtsi index e66a4c1c138fe8..67633c56a72389 100644 --- a/arch/arm64/boot/dts/apple/s8001-j98a-j99a.dtsi +++ b/arch/arm64/boot/dts/apple/s8001-j98a-j99a.dtsi @@ -9,6 +9,10 @@ * Copyright (c) 2024, Nick Chan */ +&dwi_bl { + status = "okay"; +}; + &ps_dcs4 { apple,always-on; /* LPDDR4 interface */ }; diff --git a/arch/arm64/boot/dts/apple/s8001.dtsi b/arch/arm64/boot/dts/apple/s8001.dtsi index b5b00dca6ffa4c..209c7dd19b7c28 100644 --- a/arch/arm64/boot/dts/apple/s8001.dtsi +++ b/arch/arm64/boot/dts/apple/s8001.dtsi @@ -209,6 +209,13 @@ power-domains = <&ps_aic>; }; + dwi_bl: backlight@20e200080 { + compatible = "apple,s8000-dwi-bl", "apple,dwi-bl"; + reg = <0x2 0x0e200080 0x0 0x8>; + power-domains = <&ps_dwi>; + status = "disabled"; + }; + pinctrl_ap: pinctrl@20f100000 { compatible = "apple,s8000-pinctrl", "apple,pinctrl"; reg = <0x2 0x0f100000 0x0 0x100000>; diff --git a/arch/arm64/boot/dts/apple/t6000-j314s.dts b/arch/arm64/boot/dts/apple/t6000-j314s.dts index 1430b91ff1b152..afa86668440f04 100644 --- a/arch/arm64/boot/dts/apple/t6000-j314s.dts +++ b/arch/arm64/boot/dts/apple/t6000-j314s.dts @@ -24,3 +24,20 @@ &bluetooth0 { brcm,board-type = "apple,maldives"; }; + +&panel { + compatible = "apple,panel-j314", "apple,panel-mini-led", "apple,panel"; + width-mm = <302>; + height-mm = <196>; + adj-height-mm = <189>; +}; + +&aop_audio { + apple,chassis-name = "J314"; + apple,machine-kind = "MacBook Pro"; +}; + +&sound { + compatible = "apple,j314-macaudio", "apple,macaudio"; + model = "MacBook Pro J314"; +}; diff --git a/arch/arm64/boot/dts/apple/t6000-j316s.dts b/arch/arm64/boot/dts/apple/t6000-j316s.dts index da0cbe7d96736b..ddfc3c530923c7 100644 --- a/arch/arm64/boot/dts/apple/t6000-j316s.dts +++ b/arch/arm64/boot/dts/apple/t6000-j316s.dts @@ -24,3 +24,20 @@ &bluetooth0 { brcm,board-type = "apple,madagascar"; }; + +&panel { + compatible = "apple,panel-j316", "apple,panel-mini-led", "apple,panel"; + width-mm = <346>; + height-mm = <223>; + adj-height-mm = <216>; +}; + +&aop_audio { + apple,chassis-name = "J316"; + apple,machine-kind = "MacBook Pro"; +}; + +&sound { + compatible = "apple,j316-macaudio", "apple,macaudio"; + model = "MacBook Pro J316"; +}; diff --git a/arch/arm64/boot/dts/apple/t6000.dtsi b/arch/arm64/boot/dts/apple/t6000.dtsi index 0ad77c98073fe6..9b97b2325ec503 100644 --- a/arch/arm64/boot/dts/apple/t6000.dtsi +++ b/arch/arm64/boot/dts/apple/t6000.dtsi @@ -9,6 +9,8 @@ /* This chip is just a cut down version of t6001, so include it and disable the missing parts */ +#define GPU_REPEAT(x) + #include "t6001.dtsi" / { @@ -16,7 +18,22 @@ }; /delete-node/ &pmgr_south; +/delete-node/ &pmp_report_dispext2; +/delete-node/ &pmp_report_dispext3; +/delete-node/ &pmp_report_venc1; +/delete-node/ &pmp_report_msr1; +/delete-node/ &pmp_report_prores; + +&pmp { + apple,pio-ranges = <0x2 0x82000000 0x0 0x1000000>, + <0x3 0x4000000 0x0 0x1000000>, + <0x3 0x83000000 0x0 0x1000000>, + <0x4 0x2000000 0x0 0x1000000>, + <0x2 0x10e70000 0x0 0x90000>, + <0x2 0x11e70000 0x0 0x90000>, + <0x2 0x12e70000 0x0 0x90000>; +}; &gpu { - compatible = "apple,agx-g13s"; + compatible = "apple,agx-t6000", "apple,agx-g13x"; }; diff --git a/arch/arm64/boot/dts/apple/t6001-j314c.dts b/arch/arm64/boot/dts/apple/t6001-j314c.dts index c37097dcfdb304..245df6d03ee422 100644 --- a/arch/arm64/boot/dts/apple/t6001-j314c.dts +++ b/arch/arm64/boot/dts/apple/t6001-j314c.dts @@ -24,3 +24,20 @@ &bluetooth0 { brcm,board-type = "apple,maldives"; }; + +&panel { + compatible = "apple,panel-j314", "apple,panel-mini-led", "apple,panel"; + width-mm = <302>; + height-mm = <196>; + adj-height-mm = <189>; +}; + +&aop_audio { + apple,chassis-name = "J314"; + apple,machine-kind = "MacBook Pro"; +}; + +&sound { + compatible = "apple,j314-macaudio", "apple,macaudio"; + model = "MacBook Pro J314"; +}; diff --git a/arch/arm64/boot/dts/apple/t6001-j316c.dts b/arch/arm64/boot/dts/apple/t6001-j316c.dts index 3bc6e0c3294cf9..a000d497b705fa 100644 --- a/arch/arm64/boot/dts/apple/t6001-j316c.dts +++ b/arch/arm64/boot/dts/apple/t6001-j316c.dts @@ -24,3 +24,20 @@ &bluetooth0 { brcm,board-type = "apple,madagascar"; }; + +&panel { + compatible = "apple,panel-j316", "apple,panel-mini-led", "apple,panel"; + width-mm = <346>; + height-mm = <223>; + adj-height-mm = <216>; +}; + +&aop_audio { + apple,chassis-name = "J316"; + apple,machine-kind = "MacBook Pro"; +}; + +&sound { + compatible = "apple,j316-macaudio", "apple,macaudio"; + model = "MacBook Pro J316"; +}; diff --git a/arch/arm64/boot/dts/apple/t6001-j375c.dts b/arch/arm64/boot/dts/apple/t6001-j375c.dts index 2e7c23714d4d00..f3f98f03800908 100644 --- a/arch/arm64/boot/dts/apple/t6001-j375c.dts +++ b/arch/arm64/boot/dts/apple/t6001-j375c.dts @@ -24,3 +24,39 @@ &bluetooth0 { brcm,board-type = "apple,okinawa"; }; + +&dpaudio0 { + status = "okay"; +}; + +&sound { + compatible = "apple,j375-macaudio", "apple,macaudio"; + model = "Mac Studio J375"; +}; + +&pinctrl_ap { + usb_hub_oe-hog { + gpio-hog; + gpios = <230 0>; + input; + line-name = "usb-hub-oe"; + }; + + usb_hub_rst-hog { + gpio-hog; + gpios = <231 GPIO_ACTIVE_LOW>; + output-low; + line-name = "usb-hub-rst"; + }; +}; + +&gpu { + apple,avg-power-ki-only = <0.6375>; + apple,avg-power-kp = <0.58>; + apple,avg-power-target-filter-tc = <1>; + apple,perf-base-pstate = <3>; + apple,ppm-ki = <5.8>; + apple,ppm-kp = <0.355>; +}; + +#include "hwmon-fan-dual.dtsi" diff --git a/arch/arm64/boot/dts/apple/t6001.dtsi b/arch/arm64/boot/dts/apple/t6001.dtsi index ffbe823b71bc8d..73053599d71d81 100644 --- a/arch/arm64/boot/dts/apple/t6001.dtsi +++ b/arch/arm64/boot/dts/apple/t6001.dtsi @@ -11,10 +11,15 @@ #include #include #include +#include #include #include "multi-die-cpp.h" +#ifndef GPU_REPEAT +# define GPU_REPEAT(x) +#endif + #include "t600x-common.dtsi" / { @@ -27,6 +32,8 @@ ranges; nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; // filled via templated includes at the end of the file }; @@ -63,6 +70,18 @@ }; }; +&pmp { + apple,pio-ranges = <0x2 0x82000000 0x0 0x1000000>, + <0x3 0x4000000 0x0 0x1000000>, + <0x3 0x83000000 0x0 0x1000000>, + <0x4 0x2000000 0x0 0x1000000>, + <0x2 0x10e70000 0x0 0x90000>, + <0x2 0x11e70000 0x0 0x90000>, + <0x2 0x12e70000 0x0 0x90000>, + <0x4 0x82000000 0x0 0x1000000>, + <0x5 0x2000000 0x0 0x1000000>; +}; + &gpu { - compatible = "apple,agx-g13c", "apple,agx-g13s"; + compatible = "apple,agx-t6001", "apple,agx-g13c", "apple,agx-g13s"; }; diff --git a/arch/arm64/boot/dts/apple/t6002-j375d.dts b/arch/arm64/boot/dts/apple/t6002-j375d.dts index 2b7f80119618ad..5cf30cd162b679 100644 --- a/arch/arm64/boot/dts/apple/t6002-j375d.dts +++ b/arch/arm64/boot/dts/apple/t6002-j375d.dts @@ -15,6 +15,19 @@ / { compatible = "apple,j375d", "apple,t6002", "apple,arm-platform"; model = "Apple Mac Studio (M1 Ultra, 2022)"; + aliases { + atcphy4 = &atcphy0_die1; + atcphy5 = &atcphy1_die1; + }; +}; + +&dpaudio0 { + status = "okay"; +}; + +&sound { + compatible = "apple,j375-macaudio", "apple,macaudio"; + model = "Mac Studio J375"; }; /* USB Type C */ @@ -26,6 +39,30 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <174 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec4: connector { + compatible = "usb-c-connector"; + label = "USB-C Front Right"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec4_connector_hs: endpoint { + remote-endpoint = <&dwc3_4_hs>; + }; + }; + port@1 { + reg = <1>; + typec4_connector_ss: endpoint { + remote-endpoint = <&atcphy4_typec_lanes>; + }; + }; + }; + }; }; /* front-left */ @@ -35,6 +72,30 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <174 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec5: connector { + compatible = "usb-c-connector"; + label = "USB-C Front Left"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec5_connector_hs: endpoint { + remote-endpoint = <&dwc3_5_hs>; + }; + }; + port@1 { + reg = <1>; + typec5_connector_ss: endpoint { + remote-endpoint = <&atcphy5_typec_lanes>; + }; + }; + }; + }; }; }; @@ -46,6 +107,106 @@ brcm,board-type = "apple,okinawa"; }; +/* USB controllers on die 1 */ +&dwc3_0_die1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_4_hs: endpoint { + remote-endpoint = <&typec4_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_4_ss: endpoint { + remote-endpoint = <&atcphy4_usb3>; + }; + }; + }; +}; + +&dwc3_1_die1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_5_hs: endpoint { + remote-endpoint = <&typec5_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_5_ss: endpoint { + remote-endpoint = <&atcphy5_usb3>; + }; + }; + }; +}; + +/* Type-C PHYs */ +&atcphy0_die1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy4_typec_lanes: endpoint { + remote-endpoint = <&typec4_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy4_usb3: endpoint { + remote-endpoint = <&dwc3_4_ss>; + }; + }; + }; +}; + +&atcphy1_die1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy5_typec_lanes: endpoint { + remote-endpoint = <&typec5_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy5_usb3: endpoint { + remote-endpoint = <&dwc3_5_ss>; + }; + }; + }; +}; + +/* delete unused USB nodes on die 1 */ + +/delete-node/ &dwc3_2_dart_0_die1; +/delete-node/ &dwc3_2_dart_1_die1; +/delete-node/ &dwc3_2_die1; +/delete-node/ &atcphy2_die1; +/delete-node/ &atcphy2_xbar_die1; + +/delete-node/ &dwc3_3_dart_0_die1; +/delete-node/ &dwc3_3_dart_1_die1; +/delete-node/ &dwc3_3_die1; +/delete-node/ &atcphy3_die1; +/delete-node/ &atcphy3_xbar_die1; + /* delete unused always-on power-domains on die 1 */ /delete-node/ &ps_atc2_usb_aon_die1; @@ -56,3 +217,14 @@ /delete-node/ &ps_disp0_cpu0_die1; /delete-node/ &ps_disp0_fe_die1; + +&gpu { + apple,avg-power-ki-only = <0.6375>; + apple,avg-power-kp = <0.58>; + apple,avg-power-target-filter-tc = <1>; + apple,perf-base-pstate = <3>; + apple,ppm-ki = <5.8>; + apple,ppm-kp = <0.355>; +}; + +#include "hwmon-fan-dual.dtsi" diff --git a/arch/arm64/boot/dts/apple/t6002.dtsi b/arch/arm64/boot/dts/apple/t6002.dtsi index 8fb648836b538b..b5f175d71ee69a 100644 --- a/arch/arm64/boot/dts/apple/t6002.dtsi +++ b/arch/arm64/boot/dts/apple/t6002.dtsi @@ -11,10 +11,15 @@ #include #include #include +#include #include #include "multi-die-cpp.h" +#ifndef GPU_REPEAT +# define GPU_REPEAT(x) +#endif + #include "t600x-common.dtsi" / { @@ -235,6 +240,8 @@ <0x5 0x80000000 0x5 0x80000000 0x1 0x80000000>, <0x7 0x0 0x7 0x0 0xf 0x80000000>; nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; // filled via templated includes at the end of the file }; @@ -246,6 +253,8 @@ ranges = <0x2 0x0 0x22 0x0 0x4 0x0>, <0x7 0x0 0x27 0x0 0xf 0x80000000>; nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; // filled via templated includes at the end of the file }; @@ -296,11 +305,78 @@ }; }; +&dcpext0_die1 { + // TODO: verify + apple,bw-scratch = <&pmgr_dcp 0 4 0x9c0>; +}; + +&dcpext1_die1 { + // TODO: verify + apple,bw-scratch = <&pmgr_dcp 0 4 0x9c8>; +}; + &ps_gfx { // On t6002, the die0 GPU power domain needs both AFR power domains power-domains = <&ps_afr>, <&ps_afr_die1>; }; +&pmp { + apple,pio-ranges = <0x2 0x82000000 0x0 0x1000000>, + <0x3 0x4000000 0x0 0x1000000>, + <0x3 0x83000000 0x0 0x1000000>, + <0x4 0x2000000 0x0 0x1000000>, + <0x2 0x10e70000 0x0 0x90000>, + <0x2 0x11e70000 0x0 0x90000>, + <0x2 0x12e70000 0x0 0x90000>, + <0x4 0x82000000 0x0 0x1000000>, + <0x5 0x2000000 0x0 0x1000000>, + <0x22 0x82000000 0x0 0x1000000>, + <0x23 0x4000000 0x0 0x1000000>, + <0x23 0x83000000 0x0 0x1000000>, + <0x24 0x2000000 0x0 0x1000000>, + <0x24 0x82000000 0x0 0x1000000>, + <0x25 0x2000000 0x0 0x1000000>, + <0x22 0x10e70000 0x0 0x90000>, + <0x22 0x11e70000 0x0 0x90000>, + <0x22 0x12e70000 0x0 0x90000>; +}; + +&pmp_report { + pmp_report_dispext0_die1: report@33 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x33>; + label = "pmp-dispext0_die1"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext0_cpu0_die1>; + }; + + pmp_report_dispext1_die1: report@34 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x34>; + label = "pmp-dispext1_die1"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext1_cpu0_die1>; + }; + + pmp_report_dispext2_die1: report@35 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x35>; + label = "pmp-dispext2_die1"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext1_cpu0_die1>; + status = "disabled"; + }; + + pmp_report_dispext3_die1: report@36 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x36>; + label = "pmp-dispext3_die1"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext1_cpu0_die1>; + status = "disabled"; + }; +}; + &gpu { - compatible = "apple,agx-g13d", "apple,agx-g13s"; + compatible = "apple,agx-t6002", "apple,agx-g13d", "apple,agx-g13s"; }; diff --git a/arch/arm64/boot/dts/apple/t600x-common.dtsi b/arch/arm64/boot/dts/apple/t600x-common.dtsi index e20234ef213538..318914dffef856 100644 --- a/arch/arm64/boot/dts/apple/t600x-common.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-common.dtsi @@ -13,6 +13,7 @@ aliases { gpu = &gpu; + pmp = &pmp; }; cpus { @@ -229,26 +230,31 @@ opp-hz = /bits/ 64 <600000000>; opp-level = <1>; clock-latency-ns = <7500>; + opp-microwatt = <47296>; }; opp02 { opp-hz = /bits/ 64 <972000000>; opp-level = <2>; clock-latency-ns = <23000>; + opp-microwatt = <99715>; }; opp03 { opp-hz = /bits/ 64 <1332000000>; opp-level = <3>; clock-latency-ns = <29000>; + opp-microwatt = <188860>; }; opp04 { opp-hz = /bits/ 64 <1704000000>; opp-level = <4>; clock-latency-ns = <40000>; + opp-microwatt = <288891>; }; opp05 { opp-hz = /bits/ 64 <2064000000>; opp-level = <5>; clock-latency-ns = <50000>; + opp-microwatt = <412979>; }; }; @@ -259,82 +265,139 @@ opp-hz = /bits/ 64 <600000000>; opp-level = <1>; clock-latency-ns = <8000>; + opp-microwatt = <290230>; }; opp02 { opp-hz = /bits/ 64 <828000000>; opp-level = <2>; clock-latency-ns = <18000>; + opp-microwatt = <449013>; }; opp03 { opp-hz = /bits/ 64 <1056000000>; opp-level = <3>; clock-latency-ns = <19000>; + opp-microwatt = <647097>; }; opp04 { opp-hz = /bits/ 64 <1296000000>; opp-level = <4>; clock-latency-ns = <23000>; + opp-microwatt = <865620>; }; opp05 { opp-hz = /bits/ 64 <1524000000>; opp-level = <5>; clock-latency-ns = <24000>; + opp-microwatt = <1112838>; }; opp06 { opp-hz = /bits/ 64 <1752000000>; opp-level = <6>; clock-latency-ns = <28000>; + opp-microwatt = <1453271>; }; opp07 { opp-hz = /bits/ 64 <1980000000>; opp-level = <7>; clock-latency-ns = <31000>; + opp-microwatt = <1776667>; }; opp08 { opp-hz = /bits/ 64 <2208000000>; opp-level = <8>; clock-latency-ns = <45000>; + opp-microwatt = <2366690>; }; opp09 { opp-hz = /bits/ 64 <2448000000>; opp-level = <9>; clock-latency-ns = <49000>; + opp-microwatt = <2892193>; }; opp10 { opp-hz = /bits/ 64 <2676000000>; opp-level = <10>; clock-latency-ns = <53000>; + opp-microwatt = <3475417>; }; opp11 { opp-hz = /bits/ 64 <2904000000>; opp-level = <11>; clock-latency-ns = <56000>; + opp-microwatt = <3959410>; }; opp12 { opp-hz = /bits/ 64 <3036000000>; opp-level = <12>; clock-latency-ns = <56000>; + opp-microwatt = <4540620>; }; - /* Not available until CPU deep sleep is implemented opp13 { opp-hz = /bits/ 64 <3132000000>; opp-level = <13>; clock-latency-ns = <56000>; + opp-microwatt = <4745031>; turbo-mode; }; opp14 { opp-hz = /bits/ 64 <3168000000>; opp-level = <14>; clock-latency-ns = <56000>; + opp-microwatt = <4822390>; turbo-mode; }; opp15 { opp-hz = /bits/ 64 <3228000000>; opp-level = <15>; clock-latency-ns = <56000>; + opp-microwatt = <4951324>; turbo-mode; }; - */ + }; + + gpu_opp: opp-table-gpu { + compatible = "operating-points-v2"; + + /* + * NOTE: The voltage and power values are device-specific and + * must be filled in by the bootloader. + */ + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = GPU_REPEAT(400000); + opp-microwatt = <0>; + }; + opp01 { + opp-hz = /bits/ 64 <388800000>; + opp-microvolt = GPU_REPEAT(634000); + opp-microwatt = <25011450>; + }; + opp02 { + opp-hz = /bits/ 64 <486000000>; + opp-microvolt = GPU_REPEAT(650000); + opp-microwatt = <31681170>; + }; + opp03 { + opp-hz = /bits/ 64 <648000000>; + opp-microvolt = GPU_REPEAT(668000); + opp-microwatt = <41685750>; + }; + opp04 { + opp-hz = /bits/ 64 <777600000>; + opp-microvolt = GPU_REPEAT(715000); + opp-microwatt = <56692620>; + }; + opp05 { + opp-hz = /bits/ 64 <972000000>; + opp-microvolt = GPU_REPEAT(778000); + opp-microwatt = <83371500>; + }; + opp06 { + opp-hz = /bits/ 64 <1296000000>; + opp-microvolt = GPU_REPEAT(903000); + opp-microwatt = <166743000>; + }; }; pmu-e { @@ -373,6 +436,40 @@ clock-output-names = "clk_200m"; }; + clk_disp0: clock-disp0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <237333328>; + clock-output-names = "clk_disp0"; + }; + + clk_dispext0: clock-dispext0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext0"; + }; + + clk_dispext0_die1: clock-dispext0_die1 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext0_die1"; + }; + + clk_dispext1: clock-dispext1 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext1"; + }; + + clk_dispext1_die1: clock-dispext1_die1 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext1_die1"; + }; /* * This is a fabulated representation of the input clock * to NCO since we don't know the true clock tree. @@ -401,15 +498,15 @@ }; uat_handoff: uat-handoff { - status = "disabled"; + reg = <0 0 0 0>; }; uat_pagetables: uat-pagetables { - status = "disabled"; + reg = <0 0 0 0>; }; uat_ttbs: uat-ttbs { - status = "disabled"; + reg = <0 0 0 0>; }; }; }; diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi index 3603b276a2abcf..fad3b908ed8d00 100644 --- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi @@ -24,6 +24,206 @@ power-domains = <&ps_aic>; }; + pmgr_misc: power-management@28e20c000 { + compatible = "apple,t6000-pmgr-misc"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x2 0x8e20c000 0 0x400>, + <0x2 0x8e20c800 0 0x400>; + reg-names = "fabric-ps", "dcs-ps"; + apple,dcs-min-ps = <7>; + }; + + pmgr_dcp: power-management@28e3d0000 { + reg = <0x2 0x8e3d0000 0x0 0x4000>; + reg-names = "dcp-fw-pmgr"; + #apple,bw-scratch-cells = <3>; + }; + + pmp_dart: iommu@28e300000 { + compatible = "apple,t6000-dart"; + reg = <0x2 0x8e300000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_pmp>; + }; + + pmp_report: pmp_report@28e3c0000 { + compatible = "apple,t6000-pmp-v2-report"; + reg = <0x2 0x8e3c0000 0x0 0x20000>; + power-domains = <&ps_pms_sram>; + #address-cells = <1>; + #size-cells = <0>; + + pmp_report_gfx: report@9 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x9>; + label = "pmp-gfx"; + #power-domain-cells = <0>; + power-domains = <&ps_gfx>; + }; + + pmp_report_ane_sys: report@a { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0xa>; + label = "pmp-ane-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_ane_sys>; + status = "disabled"; + }; + + pmp_report_isp_sys: report@b { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0xb>; + label = "pmp-isp-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_isp_sys>; + }; + + pmp_report_disp0: report@c { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0xc>; + label = "pmp-disp0"; + #power-domain-cells = <0>; + power-domains = <&ps_disp0_cpu0>; + apple,always-on; + }; + + pmp_report_dispext0: report@d { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0xd>; + label = "pmp-dispext0"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext0_cpu0>; + }; + + pmp_report_dispext1: report@e { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0xe>; + label = "pmp-dispext1"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext1_cpu0>; + }; + + pmp_report_venc_sys: report@10 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x10>; + label = "pmp-venc-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_venc_sys>; + status = "disabled"; + }; + + pmp_report_avd_sys: report@11 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x11>; + label = "pmp-avd-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_avd_sys>; + status = "disabled"; + }; + + pmp_report_msr0: report@12 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x12>; + label = "pmp-msr0"; + #power-domain-cells = <0>; + power-domains = <&ps_msr0>; + status = "disabled"; + }; + + pmp_report_jpg: report@13 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x13>; + label = "pmp-jpg"; + #power-domain-cells = <0>; + power-domains = <&ps_jpg>; + status = "disabled"; + }; + + pmp_report_scodec: report@14 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x14>; + label = "pmp-scodec"; + #power-domain-cells = <0>; + power-domains = <&ps_scodec>; + status = "disabled"; + }; + + pmp_report_dispext2: report@20 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x20>; + label = "pmp-dispext2"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext2_cpu0>; + status = "disabled"; + }; + + pmp_report_dispext3: report@21 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x21>; + label = "pmp-dispext3"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext3_cpu0>; + status = "disabled"; + }; + + pmp_report_venc1: report@22 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x22>; + label = "pmp-venc1"; + #power-domain-cells = <0>; + power-domains = <&ps_venc1_sys>; + status = "disabled"; + }; + + pmp_report_msr1: report@23 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x23>; + label = "pmp-msr1"; + #power-domain-cells = <0>; + power-domains = <&ps_msr1>; + status = "disabled"; + }; + + pmp_report_prores: report@24 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x24>; + label = "pmp-prores"; + #power-domain-cells = <0>; + power-domains = <&ps_prores>; + status = "disabled"; + }; + }; + + pmp: pmp@28e700000 { + compatible = "apple,t6000-pmp-v2"; + reg = <0x2 0x8e700000 0x0 0x100000>, + <0x2 0x8ec00000 0x0 0x4000>; + reg-names = "pmp", "asc"; + mboxes = <&pmp_mbox>; + mbox-names = "mbox"; + iommus = <&pmp_dart 0>; + power-domains = <&ps_pmp>; + tunables { + }; + }; + + pmp_mbox: mbox@28ec08000 { + compatible = "apple,t6000-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x8ec08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_pmp>, <&ps_pms_sram>; + }; + smc: smc@290400000 { compatible = "apple,t6000-smc", "apple,smc"; reg = <0x2 0x90400000 0x0 0x4000>, @@ -37,12 +237,22 @@ #gpio-cells = <2>; }; + smc_hwmon: hwmon { + compatible = "apple,smc-hwmon"; + }; + smc_reboot: reboot { compatible = "apple,smc-reboot"; nvmem-cells = <&shutdown_flag>, <&boot_stage>, - <&boot_error_count>, <&panic_count>; + <&boot_error_count>, <&panic_count>, <&pm_setting>; nvmem-cell-names = "shutdown_flag", "boot_stage", - "boot_error_count", "panic_count"; + "boot_error_count", "panic_count", "pm_setting"; + }; + + rtc { + compatible = "apple,smc-rtc"; + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; }; }; @@ -145,22 +355,198 @@ interrupts = ; }; - sio_dart_0: iommu@39b004000 { + aop_mbox: mbox@293408000 { + compatible = "apple,t6000-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x93408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + status = "disabled"; + }; + + aop_dart: iommu@293808000 { compatible = "apple,t6000-dart"; - reg = <0x3 0x9b004000 0x0 0x4000>; + reg = <0x2 0x93808000 0x0 0x4000>; + #iommu-cells = <1>; interrupt-parent = <&aic>; - interrupts = ; + interrupts = ; + status = "disabled"; + }; + + aop_admac: dma-controller@293980000 { + compatible = "apple,t6000-admac", "apple,admac"; + reg = <0x2 0x93980000 0x0 0x34000>; + #dma-cells = <1>; + dma-channels = <16>; + interrupts-extended = <0>, + <0>, + <&aic AIC_IRQ 0 600 IRQ_TYPE_LEVEL_HIGH>, + <0>; + iommus = <&aop_dart 7>; + status = "disabled"; + }; + + aop: aop@293c00000 { + compatible = "apple,t6000-aop"; + reg = <0x2 0x93c00000 0x0 0x250000>, + <0x2 0x93400000 0x0 0x6c000>; + mboxes = <&aop_mbox>; + mbox-names = "mbox"; + iommus = <&aop_dart 0>; + + status = "disabled"; + + aop_audio: audio { + compatible = "apple,t6000-aop-audio", "apple,aop-audio"; + dmas = <&aop_admac 1>; + dma-names = "dma"; + }; + + aop_als: als { + compatible = "apple,t6000-aop-als", "apple,aop-als"; + // intentionally empty + }; + + las { + compatible = "apple,t6000-aop-las", "apple,aop-las"; + }; + }; + + disp0_dart: iommu@38b304000 { + compatible = "apple,t6000-dart"; + reg = <0x3 0x8b304000 0x0 0x4000>; #iommu-cells = <1>; - power-domains = <&ps_sio_cpu>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + power-domains = <&ps_disp0_cpu0>; + apple,dma-range = <0x0 0x0 0x0 0xfc000000>; }; - sio_dart_1: iommu@39b008000 { + dcp_dart: iommu@38b30c000 { compatible = "apple,t6000-dart"; - reg = <0x3 0x9b008000 0x0 0x8000>; + reg = <0x3 0x8b30c000 0x0 0x4000>; + #iommu-cells = <1>; interrupt-parent = <&aic>; - interrupts = ; + interrupts = ; + power-domains = <&ps_disp0_cpu0>; + apple,dma-range = <0x1f0 0x0 0x0 0xfc000000>; + }; + + sep_dart: iommu@3952c0000 { + compatible = "apple,t6000-dart"; + reg = <0x3 0x952c0000 0x0 0x4000>; #iommu-cells = <1>; - power-domains = <&ps_sio_cpu>; + interrupt-parent = <&aic>; + interrupts = ; + }; + + sep: sep@396400000 { + compatible = "apple,sep"; + reg = <0x3 0x96400000 0x0 0x6C000>; + mboxes = <&sep_mbox>; + mbox-names = "mbox"; + iommus = <&sep_dart 0>; + power-domains = <&ps_sep>; + status = "disabled"; + }; + + sep_mbox: mbox@396408000 { + compatible = "apple,t6000-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x96408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + dpaudio0: audio-controller@39b500000 { + compatible = "apple,t6000-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b500000 0x0 0x4000>; + dmas = <&sio 0x64>; + dma-names = "tx"; + power-domains = <&ps_dpa0>; + reset-domains = <&ps_dpa0>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dpaudio0_dcp: endpoint { + remote-endpoint = <&dcp_audio>; + }; + }; + }; + }; + + dcp_mbox: mbox@38bc08000 { + compatible = "apple,t6000-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x8bc08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_disp0_cpu0>; + }; + + dcp: dcp@38bc00000 { + compatible = "apple,t6000-dcp", "apple,dcp"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 0>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x3 0x8bc00000 0x0 0x4000>, + <0x3 0x8a000000 0x0 0x3000000>, + <0x3 0x8b320000 0x0 0x4000>, + <0x3 0x8b344000 0x0 0x4000>, + <0x3 0x8b800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x988>; + power-domains = <&pmp_report_disp0>; + resets = <&ps_disp0_cpu0>; + clocks = <&clk_disp0>; + phandle = <&dcp>; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + disp0_piodma: piodma { + iommus = <&disp0_dart 4>; + phandle = <&disp0_piodma>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dcp_audio: endpoint { + remote-endpoint = <&dpaudio0_dcp>; + }; + }; + }; + }; + + display: display-subsystem { + compatible = "apple,display-subsystem"; + iommus = <&disp0_dart 0>; + /* generate phandle explicitly for use in loader */ + phandle = <&display>; }; fpwm0: pwm@39b030000 { @@ -327,10 +713,10 @@ "tx2a", "rx2a", "tx2b", "rx2b", "tx3a", "rx3a", "tx3b", "rx3b"; interrupt-parent = <&aic>; - interrupts = , + interrupts = , + , , - , - ; + ; power-domains = <&ps_audio_p>, <&ps_mca0>, <&ps_mca1>, <&ps_mca2>, <&ps_mca3>; resets = <&ps_audio_p>; @@ -343,13 +729,50 @@ <0x4 0x4000000 0 0x1000000>; reg-names = "asc", "sgx"; mboxes = <&agx_mbox>; - power-domains = <&ps_gfx>; + power-domains = <&pmp_report_gfx>; memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>, <&gpu_hw_cal_a>, <&gpu_hw_cal_b>, <&gpu_globals>; memory-region-names = "ttbs", "pagetables", "handoff", "hw-cal-a", "hw-cal-b", "globals"; apple,firmware-abi = <0 0 0>; + + apple,firmware-version = <12 3 0>; + apple,firmware-compat = <12 3 0>; + + operating-points-v2 = <&gpu_opp>; + apple,perf-base-pstate = <1>; + apple,min-sram-microvolt = <790000>; + apple,avg-power-filter-tc-ms = <1000>; + apple,avg-power-ki-only = <2.4>; + apple,avg-power-kp = <1.5>; + apple,avg-power-min-duty-cycle = <40>; + apple,avg-power-target-filter-tc = <125>; + apple,fast-die0-integral-gain = <500.0>; + apple,fast-die0-proportional-gain = <72.0>; + apple,perf-boost-ce-step = <50>; + apple,perf-boost-min-util = <90>; + apple,perf-filter-drop-threshold = <0>; + apple,perf-filter-time-constant = <5>; + apple,perf-filter-time-constant2 = <50>; + apple,perf-integral-gain = <6.3>; + apple,perf-integral-gain2 = <0.197392>; + apple,perf-integral-min-clamp = <0>; + apple,perf-proportional-gain = <15.75>; + apple,perf-proportional-gain2 = <6.853981>; + apple,perf-tgt-utilization = <85>; + apple,power-sample-period = <8>; + apple,ppm-filter-time-constant-ms = <100>; + apple,ppm-ki = <30.0>; + apple,ppm-kp = <1.5>; + apple,pwr-filter-time-constant = <313>; + apple,pwr-integral-gain = <0.0202129>; + apple,pwr-integral-min-clamp = <0>; + apple,pwr-min-duty-cycle = <40>; + apple,pwr-proportional-gain = <5.2831855>; + + apple,core-leak-coef = GPU_REPEAT(1200.0); + apple,sram-leak-coef = GPU_REPEAT(20.0); }; agx_mbox: mbox@406408000 { @@ -365,6 +788,55 @@ #mbox-cells = <0>; }; + isp_dart0: iommu@3860e8000 { + compatible = "apple,t6000-dart"; + reg = <0x3 0x860e8000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + status = "disabled"; + }; + + isp_dart1: iommu@3860f4000 { + compatible = "apple,t6000-dart"; + reg = <0x3 0x860f4000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + status = "disabled"; + }; + + isp_dart2: iommu@3860fc000 { + compatible = "apple,t6000-dart"; + reg = <0x3 0x860fc000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + status = "disabled"; + }; + + isp: isp@384000000 { + compatible = "apple,t6000-isp", "apple,isp"; + iommus = <&isp_dart0 0>, <&isp_dart1 0>, <&isp_dart2 0>; + reg-names = "coproc", "mbox", "gpio", "mbox2"; + reg = <0x3 0x84000000 0x0 0x2000000>, + <0x3 0x86104000 0x0 0x100>, + <0x3 0x86104170 0x0 0x100>, + <0x3 0x861043f0 0x0 0x100>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&pmp_report_isp_sys>, <&ps_isp_set0>, + <&ps_isp_set1>, <&ps_isp_fe>, <&ps_isp_set3>, + <&ps_isp_set4>, <&ps_isp_set5>, <&ps_isp_set6>, + <&ps_isp_set7>, <&ps_isp_set8>; + apple,dart-vm-size = <0x0 0xa0000000>; + + status = "disabled"; + }; + pcie0_dart_0: iommu@581008000 { compatible = "apple,t6000-dart"; reg = <0x5 0x81008000 0x0 0x4000>; @@ -442,6 +914,8 @@ pinctrl-0 = <&pcie_pins>; pinctrl-names = "default"; + dma-coherent; + port00: pci@0,0 { device_type = "pci"; reg = <0x0 0x0 0x0 0x0 0x0>; diff --git a/arch/arm64/boot/dts/apple/t600x-dieX.dtsi b/arch/arm64/boot/dts/apple/t600x-dieX.dtsi index a32ff0c9d7b0c2..bde7c93c3b4814 100644 --- a/arch/arm64/boot/dts/apple/t600x-dieX.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-dieX.dtsi @@ -24,6 +24,160 @@ #performance-domain-cells = <0>; }; + DIE_NODE(dispext0_dart): iommu@289304000 { + compatible = "apple,t6000-dart"; + reg = <0x2 0x89304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_dispext0_cpu0)>; + apple,dma-range = <0x0 0x0 0x0 0xfc000000>; + status = "disabled"; + }; + + DIE_NODE(dcpext0_dart): iommu@28930c000 { + compatible = "apple,t6000-dart"; + reg = <0x2 0x8930c000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_dispext0_cpu0)>; + apple,dma-range = <0x1f0 0x0 0x0 0xfc000000>; + status = "disabled"; + }; + + DIE_NODE(dcpext0_mbox): mbox@289c08000 { + compatible = "apple,t6000-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x89c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&DIE_NODE(ps_dispext0_cpu0)>; + resets = <&DIE_NODE(ps_dispext0_cpu0)>; + status = "disabled"; + }; + + DIE_NODE(dcpext0): dcp@289c00000 { + compatible = "apple,t6000-dcpext", "apple,dcpext"; + mboxes = <&DIE_NODE(dcpext0_mbox)>; + mbox-names = "mbox"; + iommus = <&DIE_NODE(dcpext0_dart) 0>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x2 0x89c00000 0x0 0x4000>, + <0x2 0x88000000 0x0 0x3000000>, + <0x2 0x89320000 0x0 0x4000>, + <0x2 0x89344000 0x0 0x4000>, + <0x2 0x89800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x990>; + power-domains = <&DIE_NODE(pmp_report_dispext0)>; + resets = <&DIE_NODE(ps_dispext0_cpu0)>; + clocks = <&DIE_NODE(clk_dispext0)>; + phandle = <&DIE_NODE(dcpext0)>; + apple,dcp-index = <1>; + status = "disabled"; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + piodma { + iommus = <&DIE_NODE(dispext0_dart) 4>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dcpext0_audio): endpoint { + remote-endpoint = <&DIE_NODE(dpaudio1_dcp)>; + }; + }; + }; + }; + + DIE_NODE(dispext1_dart): iommu@28c304000 { + compatible = "apple,t6000-dart", "apple,t8110-dart"; + reg = <0x2 0x8c304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_dispext1_cpu0)>; + apple,dma-range = <0x0 0x0 0x0 0xfc000000>; + status = "disabled"; + }; + + DIE_NODE(dcpext1_dart): iommu@28c30c000 { + compatible = "apple,t6000-dart", "apple,t8110-dart"; + reg = <0x2 0x8c30c000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_dispext1_cpu0)>; + apple,dma-range = <0x1f0 0x0 0x0 0xfc000000>; + status = "disabled"; + }; + + DIE_NODE(dcpext1_mbox): mbox@28cc08000 { + compatible = "apple,t6000-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x8cc08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&DIE_NODE(ps_dispext1_cpu0)>; + resets = <&DIE_NODE(ps_dispext1_cpu0)>; + status = "disabled"; + }; + + DIE_NODE(dcpext1): dcp@28cc00000 { + compatible = "apple,t6000-dcpext", "apple,dcpext"; + mboxes = <&DIE_NODE(dcpext1_mbox)>; + mbox-names = "mbox"; + iommus = <&DIE_NODE(dcpext1_dart) 0>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x2 0x8cc00000 0x0 0x4000>, + <0x2 0x8b000000 0x0 0x3000000>, + <0x2 0x8c320000 0x0 0x4000>, + <0x2 0x8c344000 0x0 0x4000>, + <0x2 0x8c800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x998>; + power-domains = <&DIE_NODE(pmp_report_dispext1)>; + resets = <&DIE_NODE(ps_dispext1_cpu0)>; + clocks = <&DIE_NODE(clk_dispext1)>; + phandle = <&DIE_NODE(dcpext1)>; + apple,dcp-index = <2>; + status = "disabled"; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + piodma { + iommus = <&DIE_NODE(dispext1_dart) 4>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dcpext1_audio): endpoint { + remote-endpoint = <&DIE_NODE(dpaudio2_dcp)>; + }; + }; + }; + }; + DIE_NODE(pmgr): power-management@28e080000 { compatible = "apple,t6000-pmgr", "apple,pmgr", "syscon", "simple-mfd"; #address-cells = <1>; @@ -95,6 +249,24 @@ ; }; + DIE_NODE(sio_dart_0): iommu@39b004000 { + compatible = "apple,t6000-dart"; + reg = <0x3 0x9b004000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&DIE_NODE(ps_sio_cpu)>; + }; + + DIE_NODE(sio_dart_1): iommu@39b008000 { + compatible = "apple,t6000-dart"; + reg = <0x3 0x9b008000 0x0 0x8000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&DIE_NODE(ps_sio_cpu)>; + }; + DIE_NODE(pinctrl_ap): pinctrl@39b028000 { compatible = "apple,t6000-pinctrl", "apple,pinctrl"; reg = <0x3 0x9b028000 0x0 0x4000>; @@ -119,3 +291,361 @@ interrupt-controller; #interrupt-cells = <2>; }; + + DIE_NODE(sio_mbox): mbox@39bc08000 { + compatible = "apple,t6000-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x9bc08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&DIE_NODE(ps_sio_cpu)>; + }; + + DIE_NODE(sio): sio@39bc00000 { + compatible = "apple,t6000-sio", "apple,sio"; + reg = <0x3 0x9bc00000 0x0 0x8000>; + dma-channels = <128>; + #dma-cells = <1>; + mboxes = <&DIE_NODE(sio_mbox)>; + iommus = <&DIE_NODE(sio_dart_0) 0>, <&DIE_NODE(sio_dart_1) 0>; + power-domains = <&DIE_NODE(ps_sio_cpu)>; + resets = <&DIE_NODE(ps_sio)>; /* TODO: verify reset does something */ + status = "disabled"; + }; + + DIE_NODE(dpaudio1): audio-controller@39b504000 { + compatible = "apple,t6000-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b540000 0x0 0x4000>; + dmas = <&DIE_NODE(sio) 0x66>; + dma-names = "tx"; + power-domains = <&DIE_NODE(ps_dpa1)>; + reset-domains = <&DIE_NODE(ps_dpa1)>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dpaudio1_dcp): endpoint { + remote-endpoint = <&DIE_NODE(dcpext0_audio)>; + }; + }; + }; + }; + + DIE_NODE(dpaudio2): audio-controller@39b508000 { + compatible = "apple,t6000-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b580000 0x0 0x4000>; + dmas = <&DIE_NODE(sio) 0x68>; + dma-names = "tx"; + power-domains = <&DIE_NODE(ps_dpa2)>; + reset-domains = <&DIE_NODE(ps_dpa2)>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dpaudio2_dcp): endpoint { + remote-endpoint = <&DIE_NODE(dcpext1_audio)>; + }; + }; + }; + }; + + /* + * omit dpaudio3 / 4 as long as the linked dcpext nodes don't exist + * + DIE_NODE(dpaudio3): audio-controller@39b50c000 { + compatible = "apple,t6000-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b5c0000 0x0 0x4000>; + dmas = <&DIE_NODE(sio) 0x6a>; + dma-names = "tx"; + power-domains = <&DIE_NODE(ps_dpa3)>; + reset-domains = <&DIE_NODE(ps_dpa3)>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dpaudio3_dcp): endpoint { + remote-endpoint = <&DIE_NODE(dcpext2_audio)>; + }; + }; + }; + }; + + DIE_NODE(dpaudio4): audio-controller@39b510000 { + compatible = "apple,t6000-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b500000 0x0 0x4000>; + dmas = <&DIE_NODE(sio) 0x6c>; + dma-names = "tx"; + power-domains = <&DIE_NODE(ps_dpa4)>; + reset-domains = <&DIE_NODE(ps_dpa4)>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dpaudio4_dcp): endpoint { + remote-endpoint = <&DIE_NODE(dcpext3_audio)>; + }; + }; + }; + }; + */ + + DIE_NODE(dwc3_0): usb@702280000 { + compatible = "apple,t6000-dwc3", "apple,t8103-dwc3"; + reg = <0x7 0x02280000 0x0 0xcd00>, <0x7 0x0228cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&DIE_NODE(dwc3_0_dart_0) 0>, + <&DIE_NODE(dwc3_0_dart_1) 1>; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + resets = <&DIE_NODE(atcphy0)>; + phys = <&DIE_NODE(atcphy0) PHY_TYPE_USB2>, <&DIE_NODE(atcphy0) PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + DIE_NODE(dwc3_0_dart_0): iommu@702f00000 { + compatible = "apple,t6000-dart"; + reg = <0x7 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_0_dart_1): iommu@702f80000 { + compatible = "apple,t6000-dart"; + reg = <0x7 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(atcphy0): phy@703000000 { + compatible = "apple,t6000-atcphy", "apple,t8103-atcphy"; + reg = <0x7 0x03000000 0x0 0x4c000>, + <0x7 0x03050000 0x0 0x8000>, + <0x7 0x00000000 0x0 0x4000>, + <0x7 0x02a90000 0x0 0x4000>, + <0x7 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + }; + + DIE_NODE(atcphy0_xbar): mux@70304c000 { + compatible = "apple,t6000-display-crossbar"; + reg = <0x7 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + status = "disabled"; + }; + + DIE_NODE(dwc3_1): usb@b02280000 { + compatible = "apple,t6000-dwc3", "apple,t8103-dwc3"; + reg = <0xb 0x02280000 0x0 0xcd00>, <0xb 0x0228cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&DIE_NODE(dwc3_1_dart_0) 0>, + <&DIE_NODE(dwc3_1_dart_1) 1>; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + resets = <&DIE_NODE(atcphy1)>; + phys = <&DIE_NODE(atcphy1) PHY_TYPE_USB2>, <&DIE_NODE(atcphy1) PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + DIE_NODE(dwc3_1_dart_0): iommu@b02f00000 { + compatible = "apple,t6000-dart"; + reg = <0xb 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_1_dart_1): iommu@b02f80000 { + compatible = "apple,t6000-dart"; + reg = <0xb 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(atcphy1): phy@b03000000 { + compatible = "apple,t6000-atcphy", "apple,t8103-atcphy"; + reg = <0xb 0x03000000 0x0 0x4c000>, + <0xb 0x03050000 0x0 0x8000>, + <0xb 0x00000000 0x0 0x4000>, + <0xb 0x02a90000 0x0 0x4000>, + <0xb 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + }; + + DIE_NODE(atcphy1_xbar): mux@b0304c000 { + compatible = "apple,t6000-display-crossbar"; + reg = <0xb 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + status = "disabled"; + }; + + DIE_NODE(dwc3_2): usb@f02280000 { + compatible = "apple,t6000-dwc3", "apple,t8103-dwc3"; + reg = <0xf 0x02280000 0x0 0xcd00>, <0xf 0x0228cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&DIE_NODE(dwc3_2_dart_0) 0>, + <&DIE_NODE(dwc3_2_dart_1) 1>; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + resets = <&DIE_NODE(atcphy2)>; + phys = <&DIE_NODE(atcphy2) PHY_TYPE_USB2>, <&DIE_NODE(atcphy2) PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + DIE_NODE(dwc3_2_dart_0): iommu@f02f00000 { + compatible = "apple,t6000-dart"; + reg = <0xf 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_2_dart_1): iommu@f02f80000 { + compatible = "apple,t6000-dart"; + reg = <0xf 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(atcphy2): phy@f03000000 { + compatible = "apple,t6000-atcphy", "apple,t8103-atcphy"; + reg = <0xf 0x03000000 0x0 0x4c000>, + <0xf 0x03050000 0x0 0x8000>, + <0xf 0x00000000 0x0 0x4000>, + <0xf 0x02a90000 0x0 0x4000>, + <0xf 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + }; + + DIE_NODE(atcphy2_xbar): mux@f0304c000 { + compatible = "apple,t6000-display-crossbar"; + reg = <0xf 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + status = "disabled"; + }; + + DIE_NODE(dwc3_3): usb@1302280000 { + compatible = "apple,t6000-dwc3", "apple,t8103-dwc3"; + reg = <0x13 0x02280000 0x0 0xcd00>, <0x13 0x0228cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&DIE_NODE(dwc3_3_dart_0) 0>, + <&DIE_NODE(dwc3_3_dart_1) 1>; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + resets = <&DIE_NODE(atcphy3)>; + phys = <&DIE_NODE(atcphy3) PHY_TYPE_USB2>, <&DIE_NODE(atcphy3) PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + DIE_NODE(dwc3_3_dart_0): iommu@1302f00000 { + compatible = "apple,t6000-dart"; + reg = <0x13 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_3_dart_1): iommu@1302f80000 { + compatible = "apple,t6000-dart"; + reg = <0x13 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(atcphy3): phy@1303000000 { + compatible = "apple,t6000-atcphy", "apple,t8103-atcphy"; + reg = <0x13 0x03000000 0x0 0x4c000>, + <0x13 0x03050000 0x0 0x8000>, + <0x13 0x00000000 0x0 0x4000>, + <0x13 0x02a90000 0x0 0x4000>, + <0x13 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + }; + + DIE_NODE(atcphy3_xbar): mux@130304c000 { + compatible = "apple,t6000-display-crossbar"; + reg = <0x13 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + status = "disabled"; + }; diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi index c0aac59a6fae4f..c6e33954c03067 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi @@ -12,9 +12,20 @@ #include / { + chassis-type = "laptop"; + aliases { + atcphy0 = &atcphy0; + atcphy1 = &atcphy1; + atcphy2 = &atcphy2; + atcphy3 = &atcphy3; bluetooth0 = &bluetooth0; + dcp = &dcp; + dcpext0 = &dcpext0; + disp0 = &display; + disp0_piodma = &disp0_piodma; serial0 = &serial0; + sio = &sio; wifi0 = &wifi0; }; @@ -30,6 +41,9 @@ reg = <0 0 0 0>; /* To be filled by loader */ /* Format properties will be added by loader */ status = "disabled"; + panel = <&panel>; + post-init-providers = <&panel>; + power-domains = <&ps_disp0_cpu0>; }; }; @@ -55,6 +69,76 @@ status = "okay"; }; +&dcp { + panel: panel { + apple,max-brightness = <500>; + }; +}; + +&display { + iommus = <&disp0_dart 0>, <&dispext0_dart 0>; +}; + +&dispext0_dart { + status = "okay"; +}; + +&dcpext0_dart { + status = "okay"; +}; + +&dcpext0_mbox { + status = "okay"; +}; + +&dcpext0 { + /* enabled by the loader */ + apple,connector-type = "HDMI-A"; + + /* HDMI HPD gpio, used as interrupt*/ + hdmi-hpd-gpios = <&pinctrl_nub 15 GPIO_ACTIVE_HIGH>; + + hdmi-pwren-gpios = <&smc_gpio 23 GPIO_ACTIVE_HIGH>; + dp2hdmi-pwren-gpios = <&smc_gpio 6 GPIO_ACTIVE_HIGH>; + + phys = <&atcphy3 PHY_TYPE_DP>; + phy-names = "dp-phy"; + mux-controls = <&atcphy3_xbar 0>; + mux-control-names = "dp-xbar"; + mux-index = <0>; + apple,dptx-phy = <3>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + + decpext0_dpout: endpoint { + remote-endpoint = <&atcphy3_dp>; + }; + }; + }; +}; + +/* remove once m1n1 enables sio nodes after setup */ +&sio { + status = "okay"; +}; + +&dpaudio1 { + status = "okay"; +}; + +&atcphy3_xbar { + status = "okay"; +}; + +&ps_atc3_common { + apple,always-on; /* Needs to stay on for HDMI resume */ +}; + /* USB Type C */ &i2c0 { hpm0: usb-pd@38 { @@ -63,6 +147,30 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <174 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec0: connector { + compatible = "usb-c-connector"; + label = "USB-C Left Rear"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec0_connector_hs: endpoint { + remote-endpoint = <&dwc3_0_hs>; + }; + }; + port@1 { + reg = <1>; + typec0_connector_ss: endpoint { + remote-endpoint = <&atcphy0_typec_lanes>; + }; + }; + }; + }; }; hpm1: usb-pd@3f { @@ -71,6 +179,30 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <174 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec1: connector { + compatible = "usb-c-connector"; + label = "USB-C Left Front"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec1_connector_hs: endpoint { + remote-endpoint = <&dwc3_1_hs>; + }; + }; + port@1 { + reg = <1>; + typec1_connector_ss: endpoint { + remote-endpoint = <&atcphy1_typec_lanes>; + }; + }; + }; + }; }; hpm2: usb-pd@3b { @@ -79,6 +211,30 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <174 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec2: connector { + compatible = "usb-c-connector"; + label = "USB-C Right"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec2_connector_hs: endpoint { + remote-endpoint = <&dwc3_2_hs>; + }; + }; + port@1 { + reg = <1>; + typec2_connector_ss: endpoint { + remote-endpoint = <&atcphy2_typec_lanes>; + }; + }; + }; + }; }; /* MagSafe port */ @@ -91,14 +247,138 @@ }; }; +/* Virtual regulator representing the shared shutdown GPIO */ +/ { + speaker_sdz: fixed-regulator-sn012776-sdz { + compatible = "regulator-fixed"; + regulator-name = "sn012776-sdz"; + startup-delay-us = <5000>; + gpios = <&pinctrl_ap 178 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&i2c1 { + status = "okay"; + + speaker_left_tweet: codec@3a { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3a>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Tweeter"; + interrupts-extended = <&pinctrl_ap 179 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <8>; + ti,vmon-slot-no = <10>; + }; + + speaker_left_woof1: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Woofer 1"; + interrupts-extended = <&pinctrl_ap 179 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + ti,sdout-force-zero-mask = <0xf0f0f0>; + }; + + speaker_left_woof2: codec@39 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x39>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Woofer 2"; + interrupts-extended = <&pinctrl_ap 179 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <16>; + ti,vmon-slot-no = <18>; + }; +}; + +&i2c2 { + status = "okay"; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 4 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupts-extended = <&pinctrl_ap 180 IRQ_TYPE_LEVEL_LOW>; + sound-name-prefix = "Jack"; + }; +}; + +&i2c3 { + status = "okay"; + + speaker_right_tweet: codec@3d { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3d>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Tweeter"; + interrupts-extended = <&pinctrl_ap 179 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <12>; + ti,vmon-slot-no = <14>; + }; + + speaker_right_woof1: codec@3b { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3b>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Woofer 1"; + interrupts-extended = <&pinctrl_ap 179 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <4>; + ti,vmon-slot-no = <6>; + ti,sdout-force-zero-mask = <0x0f0f0f>; + }; + + speaker_right_woof2: codec@3c { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3c>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Woofer 2"; + interrupts-extended = <&pinctrl_ap 179 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <20>; + ti,vmon-slot-no = <22>; + }; +}; + &nco_clkref { clock-frequency = <1068000000>; }; +#ifndef NO_SPI_TRACKPAD +&spi3 { + status = "okay"; + + hid-transport@0 { + compatible = "apple,spi-hid-transport"; + reg = <0>; + spi-max-frequency = <8000000>; + /* + * Apple's ADT specifies 20us CS change delays, and the + * SPI HID interface metadata specifies 45us. Using either + * seems not to be reliable, but adding both works, so + * best guess is they are cumulative. + */ + spi-cs-setup-delay-ns = <65000>; + spi-cs-hold-delay-ns = <65000>; + spi-cs-inactive-delay-ns = <250000>; + spien-gpios = <&pinctrl_ap 194 0>; + interrupts-extended = <&pinctrl_nub 6 IRQ_TYPE_LEVEL_LOW>; + }; +}; +#endif + /* PCIe devices */ &port00 { /* WLAN */ bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; wifi0: wifi@0,0 { compatible = "pci14e4,4433"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -118,6 +398,8 @@ &port01 { /* SD card reader */ bus-range = <2 2>; + pwren-gpios = <&smc_gpio 26 GPIO_ACTIVE_HIGH>; + status = "okay"; sdhci0: mmc@0,0 { compatible = "pci17a0,9755"; reg = <0x20000 0x0 0x0 0x0 0x0>; @@ -130,4 +412,235 @@ status = "okay"; }; +&pcie0_dart_1 { + status = "okay"; +}; + +/* USB controllers */ +&dwc3_0 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_0_hs: endpoint { + remote-endpoint = <&typec0_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_0_ss: endpoint { + remote-endpoint = <&atcphy0_usb3>; + }; + }; + }; +}; + +&dwc3_1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_1_hs: endpoint { + remote-endpoint = <&typec1_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_1_ss: endpoint { + remote-endpoint = <&atcphy1_usb3>; + }; + }; + }; +}; + +&dwc3_2 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_2_hs: endpoint { + remote-endpoint = <&typec2_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_2_ss: endpoint { + remote-endpoint = <&atcphy2_usb3>; + }; + }; + }; +}; + +/* + * ps_atc3_usb_aon power-domain is always-on to keep dwc3 working over suspend. + * atc3 is used exclusively for the DP-to-HDMI so do not keep this always on. + */ +&ps_atc3_usb_aon { + /delete-property/ apple,always-on; +}; + +/* ATC3 is used for DisplayPort -> HDMI only */ +&dwc3_3_dart_0 { + status = "disabled"; +}; + +&dwc3_3_dart_1 { + status = "disabled"; +}; + +&dwc3_3 { + status = "disabled"; +}; + +/* Delete unused dwc3_3 to prevent dt_disable_missing_devs() from disabling + * atcphy3 via phandle references from a disablecd device. + */ +/delete-node/ &dwc3_3; + +/* Type-C PHYs */ +&atcphy0 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy0_typec_lanes: endpoint { + remote-endpoint = <&typec0_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy0_usb3: endpoint { + remote-endpoint = <&dwc3_0_ss>; + }; + }; + }; +}; + +&atcphy1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy1_typec_lanes: endpoint { + remote-endpoint = <&typec1_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy1_usb3: endpoint { + remote-endpoint = <&dwc3_1_ss>; + }; + }; + }; +}; + +&atcphy2 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy2_typec_lanes: endpoint { + remote-endpoint = <&typec2_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy2_usb3: endpoint { + remote-endpoint = <&dwc3_2_ss>; + }; + }; + }; +}; + +&atcphy3 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + reg = <2>; + atcphy3_dp: endpoint { + remote-endpoint = <&decpext0_dpout>; + }; + }; + }; +}; + +&aop_mbox { + status = "okay"; +}; + +&aop_dart { + status = "okay"; +}; + +&aop_admac { + status = "okay"; +}; + +&aop { + status = "okay"; +}; + +/ { + sound: sound { + /* compatible is set per machine */ + + dai-link@0 { + link-name = "Speakers"; + + cpu { + sound-dai = <&mca 0>, <&mca 1>; + }; + codec { + sound-dai = <&speaker_left_woof1>, + <&speaker_left_tweet>, + <&speaker_left_woof2>, + <&speaker_right_woof1>, + <&speaker_right_tweet>, + <&speaker_right_woof2>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + #include "spi1-nvram.dtsi" + +#include "isp-imx558.dtsi" + +&isp { + apple,platform-id = <3>; +}; + +#include "hwmon-common.dtsi" +#include "hwmon-fan-dual.dtsi" +#include "hwmon-laptop.dtsi" diff --git a/arch/arm64/boot/dts/apple/t600x-j375.dtsi b/arch/arm64/boot/dts/apple/t600x-j375.dtsi index c0fb93ae72f4d4..a5ee97241214d6 100644 --- a/arch/arm64/boot/dts/apple/t600x-j375.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j375.dtsi @@ -10,10 +10,22 @@ */ / { + chassis-type = "desktop"; + aliases { + atcphy0 = &atcphy0; + atcphy1 = &atcphy1; + atcphy2 = &atcphy2; + atcphy3 = &atcphy3; bluetooth0 = &bluetooth0; + #ifndef NO_DCP + dcp = &dcp; + disp0 = &display; + disp0_piodma = &disp0_piodma; + #endif ethernet0 = ðernet0; serial0 = &serial0; + sio = &sio; wifi0 = &wifi0; }; @@ -29,6 +41,7 @@ reg = <0 0 0 0>; /* To be filled by loader */ /* Format properties will be added by loader */ status = "disabled"; + power-domains = <&ps_disp0_cpu0>; }; }; @@ -42,6 +55,15 @@ status = "okay"; }; +&dcp { + apple,connector-type = "HDMI-A"; +}; + +/* remove once m1n1 enables sio nodes after setup */ +&sio { + status = "okay"; +}; + /* USB Type C */ &i2c0 { hpm0: usb-pd@38 { @@ -50,6 +72,30 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <174 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec0: connector { + compatible = "usb-c-connector"; + label = "USB-C Back Left"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec0_connector_hs: endpoint { + remote-endpoint = <&dwc3_0_hs>; + }; + }; + port@1 { + reg = <1>; + typec0_connector_ss: endpoint { + remote-endpoint = <&atcphy0_typec_lanes>; + }; + }; + }; + }; }; hpm1: usb-pd@3f { @@ -58,6 +104,30 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <174 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec1: connector { + compatible = "usb-c-connector"; + label = "USB-C Back Left Middle"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec1_connector_hs: endpoint { + remote-endpoint = <&dwc3_1_hs>; + }; + }; + port@1 { + reg = <1>; + typec1_connector_ss: endpoint { + remote-endpoint = <&atcphy1_typec_lanes>; + }; + }; + }; + }; }; hpm2: usb-pd@3b { @@ -66,6 +136,30 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <174 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec2: connector { + compatible = "usb-c-connector"; + label = "USB-C Back Right Middle"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec2_connector_hs: endpoint { + remote-endpoint = <&dwc3_2_hs>; + }; + }; + port@1 { + reg = <1>; + typec2_connector_ss: endpoint { + remote-endpoint = <&atcphy2_typec_lanes>; + }; + }; + }; + }; }; hpm3: usb-pd@3c { @@ -74,6 +168,228 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <174 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec3: connector { + compatible = "usb-c-connector"; + label = "USB-C Back Right"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec3_connector_hs: endpoint { + remote-endpoint = <&dwc3_3_hs>; + }; + }; + port@1 { + reg = <1>; + typec3_connector_ss: endpoint { + remote-endpoint = <&atcphy3_typec_lanes>; + }; + }; + }; + }; + }; +}; + +/* USB controllers */ +&dwc3_0 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_0_hs: endpoint { + remote-endpoint = <&typec0_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_0_ss: endpoint { + remote-endpoint = <&atcphy0_usb3>; + }; + }; + }; +}; + +&dwc3_1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_1_hs: endpoint { + remote-endpoint = <&typec1_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_1_ss: endpoint { + remote-endpoint = <&atcphy1_usb3>; + }; + }; + }; +}; + +&dwc3_2 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_2_hs: endpoint { + remote-endpoint = <&typec2_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_2_ss: endpoint { + remote-endpoint = <&atcphy2_usb3>; + }; + }; + }; +}; + +&dwc3_3 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_3_hs: endpoint { + remote-endpoint = <&typec3_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_3_ss: endpoint { + remote-endpoint = <&atcphy3_usb3>; + }; + }; + }; +}; + +/* Type-C PHYs */ +&atcphy0 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy0_typec_lanes: endpoint { + remote-endpoint = <&typec0_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy0_usb3: endpoint { + remote-endpoint = <&dwc3_0_ss>; + }; + }; + }; +}; + +&atcphy1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy1_typec_lanes: endpoint { + remote-endpoint = <&typec1_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy1_usb3: endpoint { + remote-endpoint = <&dwc3_1_ss>; + }; + }; + }; +}; + +&atcphy2 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy2_typec_lanes: endpoint { + remote-endpoint = <&typec2_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy2_usb3: endpoint { + remote-endpoint = <&dwc3_2_ss>; + }; + }; + }; +}; + +&atcphy3 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy3_typec_lanes: endpoint { + remote-endpoint = <&typec3_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy3_usb3: endpoint { + remote-endpoint = <&dwc3_3_ss>; + }; + }; + }; +}; + +/* Audio */ +&i2c1 { + status = "okay"; + + speaker: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + shutdown-gpios = <&pinctrl_ap 178 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupts-extended = <&pinctrl_ap 179 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + }; +}; + +&i2c2 { + status = "okay"; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 4 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupts-extended = <&pinctrl_ap 180 IRQ_TYPE_LEVEL_LOW>; + sound-name-prefix = "Jack"; }; }; @@ -81,10 +397,39 @@ clock-frequency = <1068000000>; }; +/ { + sound: sound { + /* compatible is set per machine */ + + dai-link@0 { + link-name = "Speaker"; + + cpu { + sound-dai = <&mca 0>; + }; + codec { + sound-dai = <&speaker>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + /* PCIe devices */ &port00 { /* WLAN */ bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; wifi0: wifi@0,0 { compatible = "pci14e4,4433"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -104,6 +449,7 @@ &port01 { /* SD card reader */ bus-range = <2 2>; + pwren-gpios = <&smc_gpio 26 GPIO_ACTIVE_HIGH>; sdhci0: mmc@0,0 { compatible = "pci17a0,9755"; reg = <0x20000 0x0 0x0 0x0 0x0>; @@ -126,6 +472,7 @@ &port03 { /* USB xHCI */ bus-range = <4 4>; + pwren-gpios = <&smc_gpio 20 GPIO_ACTIVE_HIGH>; status = "okay"; }; @@ -139,3 +486,5 @@ }; #include "spi1-nvram.dtsi" + +#include "hwmon-common.dtsi" diff --git a/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi b/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi index 0bd44753b76a0c..1429554ed54505 100644 --- a/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi @@ -396,6 +396,7 @@ #reset-cells = <0>; label = DIE_LABEL(dispext0_cpu0); power-domains = <&DIE_NODE(ps_dispext0_fe)>; + apple,min-state = <4>; }; DIE_NODE(ps_dispext1_cpu0): power-controller@2a8 { @@ -405,6 +406,7 @@ #reset-cells = <0>; label = DIE_LABEL(dispext1_cpu0); power-domains = <&DIE_NODE(ps_dispext1_fe)>; + apple,min-state = <4>; }; DIE_NODE(ps_ane_sys_cpu): power-controller@2c8 { @@ -824,7 +826,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(sio_cpu); - power-domains = <&DIE_NODE(ps_sio)>; + power-domains = <&DIE_NODE(ps_sio) &DIE_NODE(ps_uart_p) &DIE_NODE(ps_spi_p) &DIE_NODE(ps_audio_p)>; }; DIE_NODE(ps_fpwm0): power-controller@190 { @@ -1113,6 +1115,7 @@ #reset-cells = <0>; label = DIE_LABEL(mca0); power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + apple,externally-clocked; }; DIE_NODE(ps_mca1): power-controller@290 { @@ -1122,6 +1125,7 @@ #reset-cells = <0>; label = DIE_LABEL(mca1); power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + apple,externally-clocked; }; DIE_NODE(ps_mca2): power-controller@298 { @@ -1131,6 +1135,7 @@ #reset-cells = <0>; label = DIE_LABEL(mca2); power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + apple,externally-clocked; }; DIE_NODE(ps_mca3): power-controller@2a0 { @@ -1140,6 +1145,7 @@ #reset-cells = <0>; label = DIE_LABEL(mca3); power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + apple,externally-clocked; }; DIE_NODE(ps_dpa0): power-controller@2a8 { @@ -1293,7 +1299,6 @@ #reset-cells = <0>; label = DIE_LABEL(disp0_fe); power-domains = <&DIE_NODE(ps_afnc2_lw0)>; - apple,always-on; /* TODO: figure out if we can enable PM here */ }; DIE_NODE(ps_disp0_cpu0): power-controller@350 { @@ -1303,7 +1308,6 @@ #reset-cells = <0>; label = DIE_LABEL(disp0_cpu0); power-domains = <&DIE_NODE(ps_disp0_fe)>; - apple,always-on; /* TODO: figure out if we can enable PM here */ apple,min-state = <4>; }; @@ -1368,6 +1372,7 @@ #reset-cells = <0>; label = DIE_LABEL(isp_sys); power-domains = <&DIE_NODE(ps_afnc2_lw1)>; + status = "disabled"; }; DIE_NODE(ps_venc_sys): power-controller@3b0 { @@ -1385,12 +1390,6 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(ans2); - /* - * The ADT makes ps_apcie_st[1]_sys depend on ps_ans2 instead, - * but we'd rather have a single power domain for the downstream - * device to depend on, so use this node as the child. - * This makes more sense anyway (since ANS2 uses APCIE_ST). - */ power-domains = <&DIE_NODE(ps_afnc2_lw0)>; }; @@ -1456,6 +1455,86 @@ label = DIE_LABEL(venc_me1); power-domains = <&DIE_NODE(ps_venc_me0)>; }; + + /* There is a dependency tree involved with these PDs, + * but we do not express it here since the ISP driver + * is supposed to sequence them in the right order anyway + * (and we do not know the exact tree structure). + * + * This also works around spurious parent PD activation + * on machines with ISP disabled (desktops). + */ + DIE_NODE(ps_isp_set0): power-controller@4000 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_set0); + }; + + DIE_NODE(ps_isp_set1): power-controller@4010 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_set1); + }; + + DIE_NODE(ps_isp_fe): power-controller@4008 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ps_isp_fe); + }; + + DIE_NODE(ps_isp_set3): power-controller@4028 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4028 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_set3); + }; + + DIE_NODE(ps_isp_set4): power-controller@4020 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_set4); + }; + + DIE_NODE(ps_isp_set5): power-controller@4030 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4030 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_set5); + }; + + DIE_NODE(ps_isp_set6): power-controller@4018 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_set6); + }; + + DIE_NODE(ps_isp_set7): power-controller@4038 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4038 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_set7); + }; + + DIE_NODE(ps_isp_set8): power-controller@4040 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4040 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_set8); + }; }; &DIE_NODE(pmgr_south) { @@ -1715,6 +1794,7 @@ #reset-cells = <0>; label = DIE_LABEL(dispext2_cpu0); power-domains = <&DIE_NODE(ps_dispext2_fe)>; + apple,min-state = <4>; }; DIE_NODE(ps_dispext3_fe): power-controller@210 { @@ -1733,6 +1813,7 @@ #reset-cells = <0>; label = DIE_LABEL(dispext3_cpu0); power-domains = <&DIE_NODE(ps_dispext3_fe)>; + apple,min-state = <4>; }; DIE_NODE(ps_msr1): power-controller@250 { @@ -1881,6 +1962,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(msg); + apple,always-on; /* Core AON device? */ }; DIE_NODE(ps_nub_gpio): power-controller@80 { diff --git a/arch/arm64/boot/dts/apple/t6020-j414s.dts b/arch/arm64/boot/dts/apple/t6020-j414s.dts index 631c54c5f03dee..a227069727dd8f 100644 --- a/arch/arm64/boot/dts/apple/t6020-j414s.dts +++ b/arch/arm64/boot/dts/apple/t6020-j414s.dts @@ -24,3 +24,24 @@ &bluetooth0 { brcm,board-type = "apple,tokara"; }; + +&panel { + compatible = "apple,panel-j414", "apple,panel-mini-led", "apple,panel"; + width-mm = <302>; + height-mm = <196>; + adj-height-mm = <189>; +}; + +&aop_audio { + apple,chassis-name = "J414"; + apple,machine-kind = "MacBook Pro"; +}; + +&sound { + compatible = "apple,j414-macaudio", "apple,j314-macaudio", "apple,macaudio"; + model = "MacBook Pro J414"; +}; + +&mtp_mt { + firmware-name = "apple/tpmtfw-j414s.bin"; +}; diff --git a/arch/arm64/boot/dts/apple/t6020-j416s.dts b/arch/arm64/boot/dts/apple/t6020-j416s.dts index c277ed5889a214..3ea2b1d52593e2 100644 --- a/arch/arm64/boot/dts/apple/t6020-j416s.dts +++ b/arch/arm64/boot/dts/apple/t6020-j416s.dts @@ -24,3 +24,24 @@ &bluetooth0 { brcm,board-type = "apple,amami"; }; + +&panel { + compatible = "apple,panel-j416", "apple,panel-mini-led", "apple,panel"; + width-mm = <346>; + height-mm = <223>; + adj-height-mm = <216>; +}; + +&aop_audio { + apple,chassis-name = "J416"; + apple,machine-kind = "MacBook Pro"; +}; + +&sound { + compatible = "apple,j416-macaudio", "apple,j316-macaudio", "apple,macaudio"; + model = "MacBook Pro J416"; +}; + +&mtp_mt { + firmware-name = "apple/tpmtfw-j416s.bin"; +}; diff --git a/arch/arm64/boot/dts/apple/t6020-j474s.dts b/arch/arm64/boot/dts/apple/t6020-j474s.dts index 7c7ad5b8ad189e..12dfe9693502ad 100644 --- a/arch/arm64/boot/dts/apple/t6020-j474s.dts +++ b/arch/arm64/boot/dts/apple/t6020-j474s.dts @@ -45,3 +45,84 @@ <0x200 &pcie0_dart_2 1 1>, <0x300 &pcie0_dart_3 1 1>; }; + +&port02 { + bus-range = <2 2>; +}; + +ðernet0 { + reg = <0x20000 0x0 0x0 0x0 0x0>; +}; + +&port03 { + bus-range = <3 3>; +}; + +&sound { + compatible = "apple,j474-macaudio", "apple,j473-macaudio", "apple,macaudio"; + model = "Mac mini J474"; +}; + +&lpdptxphy { + status = "okay"; +}; + +#define USE_DCPEXT0 1 + +#if USE_DCPEXT0 +/ { + aliases { + dcpext0 = &dcpext0; + /delete-property/ dcp; + }; +}; + +&framebuffer0 { + power-domains = <&ps_dispext0_cpu0>, <&ps_dptx_phy_ps>; +}; + +&dcp { + status = "disabled"; +}; +&display { + iommus = <&dispext0_dart 0>; +}; +&dispext0_dart { + status = "okay"; +}; +&dcpext0_dart { + status = "okay"; +}; +&dcpext0_mbox { + status = "okay"; +}; +&dpaudio1 { + status = "okay"; +}; +&dcpext0 { +#else +&dpaudio0 { + status = "okay"; +}; +&dcp { +#endif + status = "okay"; + apple,connector-type = "HDMI-A"; + + /* HDMI HPD gpio, used as interrupt*/ + hdmi-hpd-gpios = <&pinctrl_aop 25 GPIO_ACTIVE_HIGH>; + + hdmi-pwren-gpios = <&smc_gpio 23 GPIO_ACTIVE_HIGH>; + dp2hdmi-pwren-gpios = <&smc_gpio 25 GPIO_ACTIVE_HIGH>; + + phys = <&lpdptxphy>; + phy-names = "dp-phy"; + apple,dptx-phy = <4>; +}; + +&gpu { + /* Apple does not do this, but they probably should */ + apple,perf-base-pstate = <3>; +}; + +#include "hwmon-mini.dtsi" diff --git a/arch/arm64/boot/dts/apple/t6020.dtsi b/arch/arm64/boot/dts/apple/t6020.dtsi index bffa66a3ffff3f..82c38593873386 100644 --- a/arch/arm64/boot/dts/apple/t6020.dtsi +++ b/arch/arm64/boot/dts/apple/t6020.dtsi @@ -9,6 +9,8 @@ /* This chip is just a cut down version of t6021, so include it and disable the missing parts */ +#define GPU_REPEAT(x) + #include "t6021.dtsi" / { @@ -16,7 +18,30 @@ }; /delete-node/ &pmgr_south; +/delete-node/ &pmp_report_dispext2; +/delete-node/ &pmp_report_dispext3; +/delete-node/ &pmp_report_venc1; +/delete-node/ &pmp_report_msr1; +/delete-node/ &pmp_report_prores; + +&pmp { + apple,pio-ranges = <0x2 0x80000000 0x0 0x1000000>, + <0x3 0x0 0x0 0x1000000>, + <0x3 0x80000000 0x0 0x1000000>, + <0x4 0x0 0x0 0x1000000>, + <0x3 0x40000000 0x0 0x1000000>, + <0x2 0x10e70000 0x0 0x90000>, + <0x2 0x11e70000 0x0 0x90000>, + <0x2 0x12e70000 0x0 0x90000>; +}; &gpu { - compatible = "apple,agx-g14s"; + compatible = "apple,agx-t6020", "apple,agx-g14x", "apple,agx-g14s"; + + apple,avg-power-filter-tc-ms = <302>; + apple,avg-power-ki-only = <2.6375>; + apple,avg-power-kp = <0.18>; + apple,fast-die0-integral-gain = <1350.0>; + apple,ppm-filter-time-constant-ms = <32>; + apple,ppm-ki = <28.0>; }; diff --git a/arch/arm64/boot/dts/apple/t6021-j414c.dts b/arch/arm64/boot/dts/apple/t6021-j414c.dts index cdcf0740714dcf..fab3b03ff3c452 100644 --- a/arch/arm64/boot/dts/apple/t6021-j414c.dts +++ b/arch/arm64/boot/dts/apple/t6021-j414c.dts @@ -24,3 +24,24 @@ &bluetooth0 { brcm,board-type = "apple,tokara"; }; + +&panel { + compatible = "apple,panel-j414", "apple,panel-mini-led", "apple,panel"; + width-mm = <302>; + height-mm = <196>; + adj-height-mm = <189>; +}; + +&aop_audio { + apple,chassis-name = "J414"; + apple,machine-kind = "MacBook Pro"; +}; + +&sound { + compatible = "apple,j414-macaudio", "apple,j314-macaudio", "apple,macaudio"; + model = "MacBook Pro J414"; +}; + +&mtp_mt { + firmware-name = "apple/tpmtfw-j414c.bin"; +}; diff --git a/arch/arm64/boot/dts/apple/t6021-j416c.dts b/arch/arm64/boot/dts/apple/t6021-j416c.dts index 6d8146b9417036..b476e235639ffc 100644 --- a/arch/arm64/boot/dts/apple/t6021-j416c.dts +++ b/arch/arm64/boot/dts/apple/t6021-j416c.dts @@ -17,6 +17,26 @@ model = "Apple MacBook Pro (16-inch, M2 Max, 2023)"; }; +/* This machine model (only) has two extra boost CPU P-states * + * Disabled: Only the highest CPU bin (38 GPU cores) has this. + * Keep this disabled until m1n1 learns how to remove these OPPs + * for unsupported machines, otherwise it breaks cpufreq. +&avalanche_opp { + opp18 { + opp-hz = /bits/ 64 <3528000000>; + opp-level = <18>; + clock-latency-ns = <67000>; + turbo-mode; + }; + opp19 { + opp-hz = /bits/ 64 <3696000000>; + opp-level = <19>; + clock-latency-ns = <67000>; + turbo-mode; + }; +}; +*/ + &wifi0 { brcm,board-type = "apple,amami"; }; @@ -24,3 +44,24 @@ &bluetooth0 { brcm,board-type = "apple,amami"; }; + +&panel { + compatible = "apple,panel-j416", "apple,panel-mini-led", "apple,panel"; + width-mm = <346>; + height-mm = <223>; + adj-height-mm = <216>; +}; + +&aop_audio { + apple,chassis-name = "J416"; + apple,machine-kind = "MacBook Pro"; +}; + +&sound { + compatible = "apple,j416-macaudio", "apple,j316-macaudio", "apple,macaudio"; + model = "MacBook Pro J416"; +}; + +&mtp_mt { + firmware-name = "apple/tpmtfw-j416c.bin"; +}; diff --git a/arch/arm64/boot/dts/apple/t6021-j475c.dts b/arch/arm64/boot/dts/apple/t6021-j475c.dts index 533e3577487469..e4321cfc556838 100644 --- a/arch/arm64/boot/dts/apple/t6021-j475c.dts +++ b/arch/arm64/boot/dts/apple/t6021-j475c.dts @@ -29,9 +29,99 @@ /* enable PCIe port01 with SDHCI */ &port01 { + pwren-gpios = <&smc_gpio 22 GPIO_ACTIVE_HIGH>; status = "okay"; }; &pcie0_dart_1 { status = "okay"; }; + +&pinctrl_ap { + usb_hub_oe-hog { + gpio-hog; + gpios = <231 0>; + input; + line-name = "usb-hub-oe"; + }; + + usb_hub_rst-hog { + gpio-hog; + gpios = <232 GPIO_ACTIVE_LOW>; + output-low; + line-name = "usb-hub-rst"; + }; +}; + +&sound { + compatible = "apple,j475-macaudio", "apple,j375-macaudio", "apple,macaudio"; + model = "Mac Studio J475"; +}; + +&lpdptxphy { + status = "okay"; +}; + + +#define USE_DCPEXT0 1 + +#if USE_DCPEXT0 +/ { + aliases { + dcpext0 = &dcpext0; + /delete-property/ dcp; + }; +}; + +&framebuffer0 { + power-domains = <&ps_dispext0_cpu0>, <&ps_dptx_phy_ps>; +}; + +&dcp { + status = "disabled"; +}; +&display { + iommus = <&dispext0_dart 0>; +}; +&dispext0_dart { + status = "okay"; +}; +&dcpext0_dart { + status = "okay"; +}; +&dcpext0_mbox { + status = "okay"; +}; +&dpaudio1 { + status = "okay"; +}; +&dcpext0 { +#else +&dpaudio0 { + status = "okay"; +}; +&dcp { +#endif + status = "okay"; + apple,connector-type = "HDMI-A"; + + /* HDMI HPD gpio, used as interrupt*/ + hdmi-hpd-gpios = <&pinctrl_aop 25 GPIO_ACTIVE_HIGH>; + + hdmi-pwren-gpios = <&smc_gpio 23 GPIO_ACTIVE_HIGH>; + dp2hdmi-pwren-gpios = <&smc_gpio 25 GPIO_ACTIVE_HIGH>; + + phys = <&lpdptxphy>; + phy-names = "dp-phy"; + apple,dptx-phy = <4>; +}; + +&gpu { + apple,idleoff-standby-timer = <3000>; + apple,perf-base-pstate = <5>; + apple,perf-boost-ce-step = <100>; + apple,perf-boost-min-util = <75>; + apple,perf-tgt-utilization = <70>; +}; + +#include "hwmon-fan-dual.dtsi" diff --git a/arch/arm64/boot/dts/apple/t6021.dtsi b/arch/arm64/boot/dts/apple/t6021.dtsi index 62907ad6a54683..44c3b09ad46c51 100644 --- a/arch/arm64/boot/dts/apple/t6021.dtsi +++ b/arch/arm64/boot/dts/apple/t6021.dtsi @@ -16,6 +16,13 @@ #include "multi-die-cpp.h" +#ifndef GPU_REPEAT +# define GPU_REPEAT(x) +#endif +#ifndef GPU_DIE_REPEAT +# define GPU_DIE_REPEAT(x) +#endif + #include "t602x-common.dtsi" / { @@ -28,6 +35,8 @@ ranges; nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; // filled via templated includes at the end of the file }; @@ -64,6 +73,26 @@ }; }; +&pmp { + apple,pio-ranges = <0x2 0x80000000 0x0 0x1000000>, + <0x3 0x0 0x0 0x1000000>, + <0x3 0x80000000 0x0 0x1000000>, + <0x4 0x0 0x0 0x1000000>, + <0x3 0x40000000 0x0 0x1000000>, + <0x2 0x10e70000 0x0 0x90000>, + <0x2 0x11e70000 0x0 0x90000>, + <0x2 0x12e70000 0x0 0x90000>, + <0x4 0x80000000 0x0 0x1000000>, + <0x5 0x0 0x0 0x1000000>; +}; + &gpu { - compatible = "apple,agx-g14c", "apple,agx-g14s"; + compatible = "apple,agx-t6021", "apple,agx-g14x", "apple,agx-g14c", "apple,agx-g14s"; + + apple,avg-power-filter-tc-ms = <300>; + apple,avg-power-ki-only = <1.5125>; + apple,avg-power-kp = <0.38>; + apple,fast-die0-integral-gain = <700.0>; + apple,ppm-filter-time-constant-ms = <34>; + apple,ppm-ki = <18.0>; }; diff --git a/arch/arm64/boot/dts/apple/t6022-j180d.dts b/arch/arm64/boot/dts/apple/t6022-j180d.dts index dca6bd167c225a..4a528c28031805 100644 --- a/arch/arm64/boot/dts/apple/t6022-j180d.dts +++ b/arch/arm64/boot/dts/apple/t6022-j180d.dts @@ -11,13 +11,28 @@ #include "t6022.dtsi" #include "t6022-jxxxd.dtsi" +#include "t6022-pcie-ge.dtsi" / { compatible = "apple,j180d", "apple,t6022", "apple,arm-platform"; model = "Apple Mac Pro (M2 Ultra, 2023)"; + chassis-type = "server"; + aliases { - nvram = &nvram; + atcphy0 = &atcphy0; + atcphy1 = &atcphy1; + atcphy2 = &atcphy2; + atcphy3 = &atcphy3; + atcphy4 = &atcphy0_die1; + atcphy5 = &atcphy1_die1; + atcphy6 = &atcphy2_die1; + atcphy7 = &atcphy3_die1; + bluetooth0 = &bluetooth0; + dcpext0 = &dcpext0; + ethernet0 = ðernet0; + ethernet1 = ðernet1; serial0 = &serial0; + wifi0 = &wifi0; }; chosen { @@ -36,6 +51,13 @@ }; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + /* To be filled by loader */ + }; + memory@10000000000 { device_type = "memory"; reg = <0x100 0 0x2 0>; /* To be filled by loader */ @@ -46,6 +68,46 @@ status = "okay"; }; +&lpdptxphy { + status = "okay"; +}; + +&display { + iommus = <&dispext0_dart_die1 0>, <&dispext0_dart 0>; +}; + +&dispext0_dart { + status = "okay"; +}; + +&dcpext0_dart { + status = "okay"; +}; + +&dcpext0_mbox { + status = "okay"; +}; + +&dcpext0 { + status = "okay"; + apple,connector-type = "HDMI-A"; + + /* HDMI HPD gpio, used as interrupt*/ + hdmi-hpd-gpios = <&pinctrl_aop 25 GPIO_ACTIVE_HIGH>; + + // shared between dp2hdmi-gpio0 / dp2hdmi-gpio1 + // hdmi-pwren-gpios = <&smc_gpio 23 GPIO_ACTIVE_HIGH>; + + phys = <&lpdptxphy>; + phy-names = "dp-phy"; + apple,dptx-phy = <4>; + apple,dptx-die = <0>; +}; + +&dpaudio1 { + status = "okay"; +}; + /* USB Type C Rear */ &i2c0 { hpm2: usb-pd@3b { @@ -54,6 +116,30 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <44 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec2: connector { + compatible = "usb-c-connector"; + label = "USB-C Back 1"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec2_connector_hs: endpoint { + remote-endpoint = <&dwc3_2_hs>; + }; + }; + port@1 { + reg = <1>; + typec2_connector_ss: endpoint { + remote-endpoint = <&atcphy2_typec_lanes>; + }; + }; + }; + }; }; hpm3: usb-pd@3c { @@ -62,6 +148,30 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <44 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec3: connector { + compatible = "usb-c-connector"; + label = "USB-C Back 2"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec3_connector_hs: endpoint { + remote-endpoint = <&dwc3_3_hs>; + }; + }; + port@1 { + reg = <1>; + typec3_connector_ss: endpoint { + remote-endpoint = <&atcphy3_typec_lanes>; + }; + }; + }; + }; }; /* hpm4 and hpm5 included from t6022-jxxxd.dtsi */ @@ -72,6 +182,30 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <44 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec6: connector { + compatible = "usb-c-connector"; + label = "USB-C Back 5"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec6_connector_hs: endpoint { + remote-endpoint = <&dwc3_6_hs>; + }; + }; + port@1 { + reg = <1>; + typec6_connector_ss: endpoint { + remote-endpoint = <&atcphy6_typec_lanes>; + }; + }; + }; + }; }; hpm7: usb-pd@3e { @@ -80,9 +214,41 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <44 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec7: connector { + compatible = "usb-c-connector"; + label = "USB-C Back 6"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec7_connector_hs: endpoint { + remote-endpoint = <&dwc3_7_hs>; + }; + }; + port@1 { + reg = <1>; + typec7_connector_ss: endpoint { + remote-endpoint = <&atcphy7_typec_lanes>; + }; + }; + }; + }; }; }; +&typec4 { + label = "USB-C Back 3"; +}; + +&typec5 { + label = "USB-C Back 4"; +}; + /* USB Type C Front */ &i2c3 { status = "okay"; @@ -93,6 +259,30 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <60 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec0: connector { + compatible = "usb-c-connector"; + label = "USB-C Top Right"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec0_connector_hs: endpoint { + remote-endpoint = <&dwc3_0_hs>; + }; + }; + port@1 { + reg = <1>; + typec0_connector_ss: endpoint { + remote-endpoint = <&atcphy0_typec_lanes>; + }; + }; + }; + }; }; hpm1: usb-pd@3f { @@ -101,9 +291,711 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <60 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec1: connector { + compatible = "usb-c-connector"; + label = "USB-C Top Left"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec1_connector_hs: endpoint { + remote-endpoint = <&dwc3_1_hs>; + }; + }; + port@1 { + reg = <1>; + typec1_connector_ss: endpoint { + remote-endpoint = <&atcphy1_typec_lanes>; + }; + }; + }; + }; + }; +}; + +/* USB controllers */ +&dwc3_0 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_0_hs: endpoint { + remote-endpoint = <&typec0_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_0_ss: endpoint { + remote-endpoint = <&atcphy0_usb3>; + }; + }; + }; +}; + +&dwc3_1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_1_hs: endpoint { + remote-endpoint = <&typec1_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_1_ss: endpoint { + remote-endpoint = <&atcphy1_usb3>; + }; + }; + }; +}; + +&dwc3_2 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_2_hs: endpoint { + remote-endpoint = <&typec2_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_2_ss: endpoint { + remote-endpoint = <&atcphy2_usb3>; + }; + }; + }; +}; + +&dwc3_3 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_3_hs: endpoint { + remote-endpoint = <&typec3_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_3_ss: endpoint { + remote-endpoint = <&atcphy3_usb3>; + }; + }; }; }; +/* USB controllers on die 1 */ +&dwc3_2_die1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_6_hs: endpoint { + remote-endpoint = <&typec6_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_6_ss: endpoint { + remote-endpoint = <&atcphy6_usb3>; + }; + }; + }; +}; + +&dwc3_3_die1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_7_hs: endpoint { + remote-endpoint = <&typec7_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_7_ss: endpoint { + remote-endpoint = <&atcphy7_usb3>; + }; + }; + }; +}; + +/* Type-C PHYs */ +&atcphy0 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy0_typec_lanes: endpoint { + remote-endpoint = <&typec0_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy0_usb3: endpoint { + remote-endpoint = <&dwc3_0_ss>; + }; + }; + }; +}; + +&atcphy1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy1_typec_lanes: endpoint { + remote-endpoint = <&typec1_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy1_usb3: endpoint { + remote-endpoint = <&dwc3_1_ss>; + }; + }; + }; +}; + +&atcphy2 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy2_typec_lanes: endpoint { + remote-endpoint = <&typec2_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy2_usb3: endpoint { + remote-endpoint = <&dwc3_2_ss>; + }; + }; + }; +}; + +&atcphy3 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy3_typec_lanes: endpoint { + remote-endpoint = <&typec3_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy3_usb3: endpoint { + remote-endpoint = <&dwc3_3_ss>; + }; + }; + }; +}; + +&atcphy2_die1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy6_typec_lanes: endpoint { + remote-endpoint = <&typec6_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy6_usb3: endpoint { + remote-endpoint = <&dwc3_6_ss>; + }; + }; + }; +}; + +&atcphy3_die1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy7_typec_lanes: endpoint { + remote-endpoint = <&typec7_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy7_usb3: endpoint { + remote-endpoint = <&dwc3_7_ss>; + }; + }; + }; +}; + +/* Audio */ +&i2c1 { + status = "okay"; + + speaker_tweeter: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + shutdown-gpios = <&pinctrl_ap 57 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Tweeter"; + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <4>; + ti,vmon-slot-no = <6>; + }; + + speaker_woofer: codec@39 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x39>; + shutdown-gpios = <&pinctrl_ap 57 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Woofer"; + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + }; +}; + +&i2c2 { + status = "okay"; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 8 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupts-extended = <&pinctrl_ap 59 IRQ_TYPE_LEVEL_LOW>; + sound-name-prefix = "Jack"; + }; +}; + +&nco_clkref { + clock-frequency = <1068000000>; +}; + +/ { + sound: sound { + compatible = "apple,j180-macaudio", "apple,macaudio"; + model = "Mac Pro J180"; + + dai-link@0 { + link-name = "Speakers"; + /* + * DANGER ZONE: You can blow your speakers! + * + * The drivers are not ready, and unless you are careful + * to attenuate the audio stream, you run the risk of + * blowing your speakers. + */ + status = "disabled"; + cpu { + sound-dai = <&mca 0>; + }; + codec { + sound-dai = <&speaker_woofer>, <&speaker_tweeter>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + +/* PCIe devices */ +&port_ge00 { + bus-range = <0x01 0x09>; + + pci@0,0 { + // compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x10000 0x00 0x00 0x00 0x00>; + bus-range = <0x02 0x07>; + + #address-cells = <0x03>; + #size-cells = <0x02>; + ranges = <0x82010000 0x00 0x80000000 0x82010000 0x00 0x80000000 0x00 0x500000>, + <0xc3010000 0x18 0x00000000 0xc3010000 0x18 0x00000000 0x00 0x500000>; + + #interrupt-cells = <0x01>; + interrupt-map-mask = <0xffff00 0x00 0x00 0x07>; + interrupt-map = <0x20000 0x00 0x00 0x01 &port_ge00 0x00 0x00 0x00 0x00>, + <0x20000 0x00 0x00 0x02 &port_ge00 0x00 0x00 0x00 0x01>, + <0x20000 0x00 0x00 0x03 &port_ge00 0x00 0x00 0x00 0x02>, + <0x20000 0x00 0x00 0x04 &port_ge00 0x00 0x00 0x00 0x03>, + <0x20800 0x00 0x00 0x01 &port_ge00 0x00 0x00 0x00 0x01>, + <0x20800 0x00 0x00 0x02 &port_ge00 0x00 0x00 0x00 0x02>, + <0x20800 0x00 0x00 0x03 &port_ge00 0x00 0x00 0x00 0x03>, + <0x20800 0x00 0x00 0x04 &port_ge00 0x00 0x00 0x00 0x00>, + <0x21000 0x00 0x00 0x01 &port_ge00 0x00 0x00 0x00 0x02>, + <0x21000 0x00 0x00 0x02 &port_ge00 0x00 0x00 0x00 0x03>, + <0x21000 0x00 0x00 0x03 &port_ge00 0x00 0x00 0x00 0x00>, + <0x21000 0x00 0x00 0x04 &port_ge00 0x00 0x00 0x00 0x01>, + <0x21800 0x00 0x00 0x01 &port_ge00 0x00 0x00 0x00 0x03>, + <0x21800 0x00 0x00 0x02 &port_ge00 0x00 0x00 0x00 0x00>, + <0x21800 0x00 0x00 0x03 &port_ge00 0x00 0x00 0x00 0x01>, + <0x21800 0x00 0x00 0x04 &port_ge00 0x00 0x00 0x00 0x02>, + <0x22000 0x00 0x00 0x01 &port_ge00 0x00 0x00 0x00 0x00>, + <0x22000 0x00 0x00 0x02 &port_ge00 0x00 0x00 0x00 0x01>, + <0x22000 0x00 0x00 0x03 &port_ge00 0x00 0x00 0x00 0x02>, + <0x22000 0x00 0x00 0x04 &port_ge00 0x00 0x00 0x00 0x03>; + + /* pci-slot1-dsp, PCIe slot-1 */ + pci@0,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x20000 0x00 0x00 0x00 0x00>; + bus-range = <0x03 0x03>; + + #address-cells = <0x03>; + #size-cells = <0x02>; + ranges; + }; + + /* pci-slot2-dsp, PCIe slot-2 */ + pci@1,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x20800 0x00 0x00 0x00 0x00>; + bus-range = <0x04 0x04>; + + #address-cells = <0x03>; + #size-cells = <0x02>; + ranges; + }; + + /* pci-slot3-dsp, PCIe slot-3 */ + pci@2,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x21000 0x00 0x00 0x00 0x00>; + bus-range = <0x05 0x05>; + + #address-cells = <0x03>; + #size-cells = <0x02>; + ranges; + }; + + /* pci-slot4-dsp, PCIe slot-4 */ + pci@3,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x21800 0x00 0x00 0x00 0x00>; + bus-range = <0x06 0x06>; + + #address-cells = <0x03>; + #size-cells = <0x02>; + ranges; + }; + + /* pci-slot5-dsp, PCIe slot-5 */ + pci@4,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x22000 0x00 0x00 0x00 0x00>; + bus-range = <0x07 0x07>; + + #address-cells = <0x03>; + #size-cells = <0x02>; + ranges; + }; + + }; +}; + +&port_ge00_die1 { + bus-range = <0x01 0x09>; + + /* + * Add mulptiple "reset-gpios" since there is no mechanismen to access + * PERST# for devices behind the PCIe switch. + * The "pwren" GPIO is from the wifi/bt chip which faces the same + * problem without pci-pwrctrl integration. + */ + reset-gpios = <&pinctrl_ap 4 GPIO_ACTIVE_LOW>, + <&pinctrl_ap 6 GPIO_ACTIVE_LOW>, + <&pinctrl_ap 7 GPIO_ACTIVE_LOW>, + <&pinctrl_ap_die1 9 GPIO_ACTIVE_LOW>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; + + pci@0,0 { + device_type = "pci"; + reg = <0x10000 0x00 0x00 0x00 0x00>; + bus-range = <0x02 0x09>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0xffff00 0x00 0x00 0x07>; + interrupt-map = <0x20000 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x20000 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x20000 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x20000 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x03>, + <0x20800 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x20800 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x20800 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x03>, + <0x20800 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x21000 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x21000 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x03>, + <0x21000 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x21000 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x21800 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x03>, + <0x21800 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x21800 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x21800 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x22000 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x22000 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x22000 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x22000 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x03>, + <0x22800 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x22800 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x22800 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x03>, + <0x22800 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x23000 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x23000 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x03>, + <0x23000 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x23000 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x01>; + + /* pci-usba-dsp, internal USB-A port */ + pci@0,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x20000 0x00 0x00 0x00 0x00>; + bus-range = <0x03 0x03>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0x30000 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x30000 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x30000 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x02>; + + /* temporarily handled in the root port */ + // reset-gpios = <&pinctrl_ap 6 GPIO_ACTIVE_LOW>; + }; + + /* pci-sata-dsp, internal AHCI controller */ + pci@1,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x20800 0x00 0x00 0x00 0x00>; + bus-range = <0x04 0x04>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0x40000 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x40000 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x40000 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x02>; + }; + + /* pci-bio-dsp, I/O board USB-A ports */ + pci@2,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x21000 0x00 0x00 0x00 0x00>; + bus-range = <0x05 0x05>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0x50000 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x50000 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x50000 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x00>; + + /* temporarily handled in the root port */ + // reset-gpios = <&pinctrl_ap 7 GPIO_ACTIVE_LOW>; + }; + + /* pci-lan-dsp, Qtion AQC113 10G etherner controller (0) */ + pci@3,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x21800 0x00 0x00 0x00 0x00>; + bus-range = <0x06 0x06>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0x60000 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x60000 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x60000 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x01>; + + ethernet0: ethernet@0,0 { + reg = <0x60000 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-mac-address = [00 10 18 00 00 00]; + }; + }; + + /* pci-lan-b-dsp, Qtion AQC113 10G etherner controller (1) */ + pci@4,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x22000 0x00 0x00 0x00 0x00>; + bus-range = <0x07 0x07>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0x70000 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x70000 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x70000 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x02>; + + ethernet1: ethernet@0,0 { + reg = <0x70000 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-mac-address = [00 10 18 00 00 00]; + }; + }; + + /* pci-wifibt-dsp, Broadcom BCM4388 Wlan/BT */ + pci@5,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x22800 0x00 0x00 0x00 0x00>; + bus-range = <0x08 0x08>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0x80000 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x80000 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x80000 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x80100 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x80100 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x80100 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x02>; + + /* temporarily handled in the root port */ + // reset-gpios = <&pinctrl_ap 4 GPIO_ACTIVE_LOW>; + // pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; + + wifi0: wifi@0,0 { + reg = <0x80000 0x0 0x0 0x0 0x0>; + compatible = "pci14e4,4433"; + brcm,board-type = "apple,sumatra"; + apple,antenna-sku = "XX"; + /* To be filled by the loader */ + local-mac-address = [00 10 18 00 00 10]; + }; + + bluetooth0: network@0,1 { + compatible = "pci14e4,5f71"; + brcm,board-type = "apple,sumatra"; + // reg = <0x80100 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-bd-address = [00 00 00 00 00 00]; + }; + }; + + /* pci-slot6-dsp, PCIe slot-6 */ + pci@6,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x23000 0x00 0x00 0x00 0x00>; + bus-range = <0x09 0x09>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + }; + }; +}; + +&pcie_ge { + status = "ok"; +}; + +&pcie_ge_dart { + status = "ok"; +}; + +&pcie_ge_die1 { + status = "ok"; +}; + +&pcie_ge_dart_die1 { + status = "ok"; +}; + /* * Delete unused PCIe nodes, the Mac Pro uses slightly different PCIe * controllers with a single port connected to a PM40100 PCIe switch diff --git a/arch/arm64/boot/dts/apple/t6022-j475d.dts b/arch/arm64/boot/dts/apple/t6022-j475d.dts index 736594544f79b5..cdfc78a1703c7a 100644 --- a/arch/arm64/boot/dts/apple/t6022-j475d.dts +++ b/arch/arm64/boot/dts/apple/t6022-j475d.dts @@ -9,6 +9,8 @@ /dts-v1/; +#define NO_DCP + #include "t6022.dtsi" #include "t602x-j474-j475.dtsi" #include "t6022-jxxxd.dtsi" @@ -16,14 +18,31 @@ / { compatible = "apple,j475d", "apple,t6022", "apple,arm-platform"; model = "Apple Mac Studio (M2 Ultra, 2023)"; + + aliases { + atcphy4 = &atcphy0_die1; + atcphy5 = &atcphy1_die1; + /delete-property/ dcp; + /delete-property/ sio; + }; +}; + +&sio { + status = "disabled"; }; &framebuffer0 { power-domains = <&ps_dispext0_cpu0_die1>, <&ps_dptx_phy_ps_die1>; }; +&dcpext0_die1 { + // J180 misses "function-dp2hdmi_pwr_en" + dp2hdmi-pwren-gpios = <&smc_gpio 25 GPIO_ACTIVE_HIGH>; +}; + /* enable PCIe port01 with SDHCI */ &port01 { + pwren-gpios = <&smc_gpio 22 GPIO_ACTIVE_HIGH>; status = "okay"; }; @@ -31,6 +50,34 @@ status = "okay"; }; +&typec4 { + label = "USB-C Front Right"; +}; + +&typec5 { + label = "USB-C Front Left"; +}; + +/* delete unused USB nodes on die 1 */ +/delete-node/ &dwc3_2_dart_0_die1; +/delete-node/ &dwc3_2_dart_1_die1; +/delete-node/ &dwc3_2_die1; +/delete-node/ &atcphy2_die1; +/delete-node/ &atcphy2_xbar_die1; + +/delete-node/ &dwc3_3_dart_0_die1; +/delete-node/ &dwc3_3_dart_1_die1; +/delete-node/ &dwc3_3_die1; +/delete-node/ &atcphy3_die1; +/delete-node/ &atcphy3_xbar_die1; + +/* delete unused always-on power-domains on die 1 */ +/delete-node/ &ps_atc2_usb_aon_die1; +/delete-node/ &ps_atc2_usb_die1; + +/delete-node/ &ps_atc3_usb_aon_die1; +/delete-node/ &ps_atc3_usb_die1; + &wifi0 { compatible = "pci14e4,4434"; brcm,board-type = "apple,canary"; @@ -40,3 +87,10 @@ compatible = "pci14e4,5f72"; brcm,board-type = "apple,canary"; }; + +&sound { + compatible = "apple,j475-macaudio", "apple,j375-macaudio", "apple,macaudio"; + model = "Mac Studio J475"; +}; + +#include "hwmon-fan-dual.dtsi" diff --git a/arch/arm64/boot/dts/apple/t6022-jxxxd.dtsi b/arch/arm64/boot/dts/apple/t6022-jxxxd.dtsi index 4f7bf2ebfe397d..fa0183441d791b 100644 --- a/arch/arm64/boot/dts/apple/t6022-jxxxd.dtsi +++ b/arch/arm64/boot/dts/apple/t6022-jxxxd.dtsi @@ -9,9 +9,66 @@ * Copyright The Asahi Linux Contributors */ +/ { + aliases { + dcpext4 = &dcpext0_die1; + disp0 = &display; + sio1 = &sio_die1; + }; +}; + +&lpdptxphy_die1 { + status = "okay"; +}; + +&display { + iommus = <&dispext0_dart_die1 0>; +}; + +&dispext0_dart_die1 { + status = "okay"; +}; + +&dcpext0_dart_die1 { + status = "okay"; +}; + +&dcpext0_mbox_die1 { + status = "okay"; +}; + +&dcpext0_die1 { + status = "okay"; + apple,connector-type = "HDMI-A"; + + /* HDMI HPD gpio, used as interrupt*/ + hdmi-hpd-gpios = <&pinctrl_aop 41 GPIO_ACTIVE_HIGH>; + + hdmi-pwren-gpios = <&smc_gpio 23 GPIO_ACTIVE_HIGH>; + // J180 misses "function-dp2hdmi_pwr_en" + // dp2hdmi-pwren-gpios = <&smc_gpio 25 GPIO_ACTIVE_HIGH>; + + phys = <&lpdptxphy_die1>; + phy-names = "dp-phy"; + apple,dptx-phy = <4>; + apple,dptx-die = <1>; +}; + +&dpaudio1_die1 { + status = "okay"; +}; + +/* delete missing dcp0/disp0 */ +/delete-node/ &disp0_dart; +/delete-node/ &dcp_dart; +/delete-node/ &dcp_mbox; +/delete-node/ &dcp; +/delete-node/ &dpaudio0; + /* delete power-domains for missing disp0 / disp0_die1 */ /delete-node/ &ps_disp0_cpu0; /delete-node/ &ps_disp0_fe; +/delete-node/ &pmp_report_disp0; /delete-node/ &ps_disp0_cpu0_die1; /delete-node/ &ps_disp0_fe_die1; @@ -25,6 +82,29 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <44 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec4: connector { + compatible = "usb-c-connector"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec4_connector_hs: endpoint { + remote-endpoint = <&dwc3_4_hs>; + }; + }; + port@1 { + reg = <1>; + typec4_connector_ss: endpoint { + remote-endpoint = <&atcphy4_typec_lanes>; + }; + }; + }; + }; }; /* front-left */ @@ -34,5 +114,115 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <44 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec5: connector { + compatible = "usb-c-connector"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec5_connector_hs: endpoint { + remote-endpoint = <&dwc3_5_hs>; + }; + }; + port@1 { + reg = <1>; + typec5_connector_ss: endpoint { + remote-endpoint = <&atcphy5_typec_lanes>; + }; + }; + }; + }; + }; +}; + + +/* USB controllers on die 1 */ +&dwc3_0_die1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_4_hs: endpoint { + remote-endpoint = <&typec4_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_4_ss: endpoint { + remote-endpoint = <&atcphy4_usb3>; + }; + }; + }; +}; + +&dwc3_1_die1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_5_hs: endpoint { + remote-endpoint = <&typec5_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_5_ss: endpoint { + remote-endpoint = <&atcphy5_usb3>; + }; + }; + }; +}; + +/* Type-C PHYs */ +&atcphy0_die1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy4_typec_lanes: endpoint { + remote-endpoint = <&typec4_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy4_usb3: endpoint { + remote-endpoint = <&dwc3_4_ss>; + }; + }; + }; +}; + +&atcphy1_die1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy5_typec_lanes: endpoint { + remote-endpoint = <&typec5_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy5_usb3: endpoint { + remote-endpoint = <&dwc3_5_ss>; + }; + }; }; }; diff --git a/arch/arm64/boot/dts/apple/t6022-pcie-ge.dtsi b/arch/arm64/boot/dts/apple/t6022-pcie-ge.dtsi new file mode 100644 index 00000000000000..f78c483c29133f --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6022-pcie-ge.dtsi @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Include PCIe-GE nodes presen on both dies of T6022 (M2 Ultra) in the + * Mac Pro (2023). + * + * Copyright The Asahi Linux Contributors + */ + +#define DIE +#define DIE_NO 0 + +&die0 { + #include "t602x-pcie-ge.dtsi" +}; + +#undef DIE +#undef DIE_NO + +#define DIE _die1 +#define DIE_NO 1 + +&die1 { + #include "t602x-pcie-ge.dtsi" +}; + +#undef DIE +#undef DIE_NO diff --git a/arch/arm64/boot/dts/apple/t6022.dtsi b/arch/arm64/boot/dts/apple/t6022.dtsi index e73bf2f7510ae2..788bbf078f3894 100644 --- a/arch/arm64/boot/dts/apple/t6022.dtsi +++ b/arch/arm64/boot/dts/apple/t6022.dtsi @@ -16,6 +16,13 @@ #include "multi-die-cpp.h" +#ifndef GPU_REPEAT +# define GPU_REPEAT(x) +#endif +#ifndef GPU_DIE_REPEAT +# define GPU_DIE_REPEAT(x) +#endif + #include "t602x-common.dtsi" / { @@ -339,11 +346,81 @@ }; }; +&dcpext0_die1 { + apple,bw-scratch = <&pmgr_dcp 0 4 0x1240>; +}; + +&dcpext1_die1 { + apple,bw-scratch = <&pmgr_dcp 0 4 0x1248>; +}; + &ps_gfx { // On t6022, the die0 GPU power domain needs both AFR power domains power-domains = <&ps_afr>, <&ps_afr_die1>; }; +&pmp { + apple,pio-ranges = <0x2 0x80000000 0x0 0x1000000>, + <0x3 0x0 0x0 0x1000000>, + <0x3 0x80000000 0x0 0x1000000>, + <0x4 0x0 0x0 0x1000000>, + <0x3 0x40000000 0x0 0x1000000>, + <0x2 0x10e70000 0x0 0x90000>, + <0x2 0x11e70000 0x0 0x90000>, + <0x2 0x12e70000 0x0 0x90000>, + <0x4 0x80000000 0x0 0x1000000>, + <0x5 0x0 0x0 0x1000000>, + <0x22 0x80000000 0x0 0x1000000>, + <0x23 0x0 0x0 0x1000000>, + <0x23 0x80000000 0x0 0x1000000>, + <0x24 0x0 0x0 0x1000000>, + <0x23 0x40000000 0x0 0x1000000>, + <0x24 0x80000000 0x0 0x1000000>, + <0x25 0x0 0x0 0x1000000>, + <0x22 0x10e70000 0x0 0x90000>, + <0x22 0x11e70000 0x0 0x90000>, + <0x22 0x12e70000 0x0 0x90000>; +}; + +&pmp_report { + pmp_report_dispext0_die1: report@1f { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x1f>; + label = "pmp-dispext0_die1"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext0_cpu0_die1>; + }; + + pmp_report_dispext1_die1: report@20 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x20>; + label = "pmp-dispext1_die1"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext1_cpu0_die1>; + }; +}; + &gpu { - compatible = "apple,agx-g14d", "apple,agx-g14s"; + compatible = "apple,agx-t6022", "apple,agx-g14x", "apple,agx-g14d", "apple,agx-g14s"; + + apple,avg-power-filter-tc-ms = <302>; + apple,avg-power-ki-only = <1.0125>; + apple,avg-power-kp = <0.15>; + apple,fast-die0-integral-gain = <9.6>; + apple,fast-die0-proportional-gain = <24.0>; + apple,idleoff-standby-timer = <3000>; + apple,perf-base-pstate = <5>; + apple,perf-boost-ce-step = <100>; + apple,perf-boost-min-util = <75>; + apple,perf-tgt-utilization = <70>; + apple,ppm-ki = <11.0>; + apple,ppm-kp = <0.15>; +}; + +&pinctrl_ap_die1 { + pcie_ge_pins_die1: pcie-ge1-pins { + pinmux = ; + }; }; diff --git a/arch/arm64/boot/dts/apple/t602x-common.dtsi b/arch/arm64/boot/dts/apple/t602x-common.dtsi index 9c800a391e7e87..317f913bf13a12 100644 --- a/arch/arm64/boot/dts/apple/t602x-common.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-common.dtsi @@ -13,6 +13,7 @@ aliases { gpu = &gpu; + pmp = &pmp; }; cpus { @@ -387,6 +388,134 @@ }; }; + gpu_opp: opp-table-gpu { + compatible = "operating-points-v2"; + + /* + * NOTE: The voltage and power values are device-specific and + * must be filled in by the bootloader. + */ + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = GPU_REPEAT(400000); + opp-microwatt = <0>; + }; + opp01 { + opp-hz = /bits/ 64 <444000000>; + opp-microvolt = GPU_REPEAT(637000); + opp-microwatt = <4295000>; + }; + opp02 { + opp-hz = /bits/ 64 <612000000>; + opp-microvolt = GPU_REPEAT(656000); + opp-microwatt = <6251000>; + }; + opp03 { + opp-hz = /bits/ 64 <808000000>; + opp-microvolt = GPU_REPEAT(687000); + opp-microwatt = <8625000>; + }; + opp04 { + opp-hz = /bits/ 64 <968000000>; + opp-microvolt = GPU_REPEAT(725000); + opp-microwatt = <11948000>; + }; + opp05 { + opp-hz = /bits/ 64 <1110000000>; + opp-microvolt = GPU_REPEAT(790000); + opp-microwatt = <15071000>; + }; + opp06 { + opp-hz = /bits/ 64 <1236000000>; + opp-microvolt = GPU_REPEAT(843000); + opp-microwatt = <18891000>; + }; + opp07 { + opp-hz = /bits/ 64 <1338000000>; + opp-microvolt = GPU_REPEAT(887000); + opp-microwatt = <21960000>; + }; + opp08 { + opp-hz = /bits/ 64 <1398000000>; + opp-microvolt = GPU_REPEAT(918000); + opp-microwatt = <22800000>; + }; + }; + + gpu_cs_opp: opp-table-gpu-cs { + compatible = "operating-points-v2"; + + /* + * NOTE: The voltage and power values are device-specific and + * must be filled in by the bootloader. + */ + opp00 { + opp-hz = /bits/ 64 <24>; + opp-microvolt = GPU_DIE_REPEAT(668000); + }; + opp01 { + opp-hz = /bits/ 64 <444000000>; + opp-microvolt = GPU_DIE_REPEAT(668000); + }; + opp02 { + opp-hz = /bits/ 64 <612000000>; + opp-microvolt = GPU_DIE_REPEAT(678000); + }; + opp03 { + opp-hz = /bits/ 64 <808000000>; + opp-microvolt = GPU_DIE_REPEAT(737000); + }; + opp04 { + opp-hz = /bits/ 64 <1024000000>; + opp-microvolt = GPU_DIE_REPEAT(815000); + }; + opp05 { + opp-hz = /bits/ 64 <1140000000>; + opp-microvolt = GPU_DIE_REPEAT(862000); + }; + opp06 { + opp-hz = /bits/ 64 <1236000000>; + opp-microvolt = GPU_DIE_REPEAT(893000); + }; + }; + + gpu_afr_opp: opp-table-gpu-afr { + compatible = "operating-points-v2"; + + /* + * NOTE: The voltage and power values are device-specific and + * must be filled in by the bootloader. + */ + opp00 { + opp-hz = /bits/ 64 <24>; + opp-microvolt = GPU_DIE_REPEAT(668000); + }; + opp01 { + opp-hz = /bits/ 64 <400000000>; + opp-microvolt = GPU_DIE_REPEAT(668000); + }; + opp02 { + opp-hz = /bits/ 64 <552000000>; + opp-microvolt = GPU_DIE_REPEAT(678000); + }; + opp03 { + opp-hz = /bits/ 64 <760000000>; + opp-microvolt = GPU_DIE_REPEAT(737000); + }; + opp04 { + opp-hz = /bits/ 64 <980000000>; + opp-microvolt = GPU_DIE_REPEAT(815000); + }; + opp05 { + opp-hz = /bits/ 64 <1098000000>; + opp-microvolt = GPU_DIE_REPEAT(862000); + }; + opp06 { + opp-hz = /bits/ 64 <1200000000>; + opp-microvolt = GPU_DIE_REPEAT(893000); + }; + }; + pmu-e { compatible = "apple,blizzard-pmu"; interrupt-parent = <&aic>; @@ -423,6 +552,41 @@ clock-output-names = "clk_200m"; }; + clk_disp0: clock-disp0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <257142848>; /* TODO: check */ + clock-output-names = "clk_disp0"; + }; + + clk_dispext0: clock-dispext0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext0"; + }; + + clk_dispext0_die1: clock-dispext0_die1 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext0_die1"; + }; + + clk_dispext1: clock-dispext1 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext1"; + }; + + clk_dispext1_die1: clock-dispext1_die1 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext1_die1"; + }; + /* * This is a fabulated representation of the input clock * to NCO since we don't know the true clock tree. @@ -451,15 +615,15 @@ }; uat_handoff: uat-handoff { - status = "disabled"; + reg = <0 0 0 0>; }; uat_pagetables: uat-pagetables { - status = "disabled"; + reg = <0 0 0 0>; }; uat_ttbs: uat-ttbs { - status = "disabled"; + reg = <0 0 0 0>; }; }; }; diff --git a/arch/arm64/boot/dts/apple/t602x-die0.dtsi b/arch/arm64/boot/dts/apple/t602x-die0.dtsi index 2e7d2bf08ddc82..ff5664bbe0542d 100644 --- a/arch/arm64/boot/dts/apple/t602x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-die0.dtsi @@ -7,7 +7,7 @@ */ nco: clock-controller@28e03c000 { - compatible = "apple,t6020-nco", "apple,t8103-nco"; + compatible = "apple,t6020-nco", "apple,t8103-nco", "apple,nco"; reg = <0x2 0x8e03c000 0x0 0x14000>; clocks = <&nco_clkref>; #clock-cells = <1>; @@ -23,8 +23,223 @@ power-domains = <&ps_aic>; }; + pmp_dart: iommu@28e300000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x2 0x8e300000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_pmp>; + }; + + pmp_report: pmp_report@28e3c0000 { + compatible = "apple,t6020-pmp-v2-report"; + reg = <0x2 0x8e3c0000 0x0 0x20000>; + power-domains = <&ps_pms_sram>; + #address-cells = <1>; + #size-cells = <0>; + + pmp_report_gfx: report@a { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xa>; + label = "pmp-gfx"; + #power-domain-cells = <0>; + power-domains = <&ps_gfx>; + }; + + pmp_report_ane_sys: report@b { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xb>; + label = "pmp-ane-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_ane_sys>; + status = "disabled"; + }; + + pmp_report_isp_sys: report@c { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xc>; + label = "pmp-isp-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_isp_sys>; + }; + + pmp_report_disp0: report@d { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xd>; + label = "pmp-disp0"; + #power-domain-cells = <0>; + power-domains = <&ps_disp0_cpu0>; + apple,always-on; + }; + + pmp_report_dispext0: report@e { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xe>; + label = "pmp-dispext0"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext0_cpu0>; + }; + + pmp_report_dispext1: report@f { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xe>; + label = "pmp-dispext1"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext1_cpu0>; + }; + + pmp_report_venc_sys: report@10 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x10>; + label = "pmp-venc-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_venc_sys>; + status = "disabled"; + }; + + pmp_report_avd_sys: report@11 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x11>; + label = "pmp-avd-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_avd_sys>; + status = "disabled"; + }; + + pmp_report_msr0: report@12 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x12>; + label = "pmp-msr0"; + #power-domain-cells = <0>; + power-domains = <&ps_msr0>; + status = "disabled"; + }; + + pmp_report_jpg: report@13 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x13>; + label = "pmp-jpg"; + #power-domain-cells = <0>; + power-domains = <&ps_jpg>; + status = "disabled"; + }; + + pmp_report_scodec: report@14 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x14>; + label = "pmp-scodec"; + #power-domain-cells = <0>; + power-domains = <&ps_scodec>; + status = "disabled"; + }; + + pmp_report_dispext2: report@1f { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x1f>; + label = "pmp-dispext2"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext2_cpu0>; + status = "disabled"; + }; + + pmp_report_dispext3: report@20 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x20>; + label = "pmp-dispext3"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext3_cpu0>; + status = "disabled"; + }; + + pmp_report_venc1: report@21 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x21>; + label = "pmp-venc1"; + #power-domain-cells = <0>; + power-domains = <&ps_venc1_sys>; + status = "disabled"; + }; + + pmp_report_msr1: report@22 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x22>; + label = "pmp-msr1"; + #power-domain-cells = <0>; + power-domains = <&ps_msr1>; + status = "disabled"; + }; + + pmp_report_prores: report@23 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x23>; + label = "pmp-prores"; + #power-domain-cells = <0>; + power-domains = <&ps_prores>; + status = "disabled"; + }; + }; + + pmp: pmp@28e700000 { + compatible = "apple,t6020-pmp-v2", "apple,t6000-pmp-v2"; + reg = <0x2 0x8e700000 0x0 0x100000>, + <0x2 0x8ec00000 0x0 0x4000>; + reg-names = "pmp", "asc"; + mboxes = <&pmp_mbox>; + mbox-names = "mbox"; + iommus = <&pmp_dart 0>; + power-domains = <&ps_pmp>; + tunables { + }; + }; + + pmp_mbox: mbox@28ec08000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x8ec08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_pmp>, <&ps_pms_sram>; + }; + + pmgr_misc: power-management@28e20c000 { + compatible = "apple,t6020-pmgr-misc", "apple,t6000-pmgr-misc"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x2 0x8e20c000 0 0x400>, + <0x2 0x8e20c400 0 0x400>; + reg-names = "fabric-ps", "dcs-ps"; + }; + + pmgr_dcp: power-management@28e3d0000 { + reg = <0x2 0x8e3d0000 0x0 0x4000>; + reg-names = "dcp-fw-pmgr"; + #apple,bw-scratch-cells = <3>; + }; + nub_spmi0: spmi@29e114000 { - compatible = "apple,t6020-spmi", "apple,t8103-spmi"; + compatible = "apple,t6020-spmi", "apple,t8103-spmi", "apple,spmi"; reg = <0x2 0x9e114000 0x0 0x100>; #address-cells = <2>; #size-cells = <0>; @@ -81,7 +296,7 @@ }; wdt: watchdog@29e2c4000 { - compatible = "apple,t6020-wdt", "apple,t8103-wdt"; + compatible = "apple,t6020-wdt", "apple,t8103-wdt", "apple,wdt"; reg = <0x2 0x9e2c4000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -102,7 +317,7 @@ }; smc: smc@2a2400000 { - compatible = "apple,t6020-smc", "apple,t8103-smc"; + compatible = "apple,t6020-smc", "apple,t8103-smc", "apple,smc"; reg = <0x2 0xa2400000 0x0 0x4000>, <0x2 0xa3e00000 0x0 0x100000>; reg-names = "smc", "sram"; @@ -114,17 +329,27 @@ #gpio-cells = <2>; }; + smc_hwmon: hwmon { + compatible = "apple,smc-hwmon"; + }; + smc_reboot: reboot { compatible = "apple,smc-reboot"; nvmem-cells = <&shutdown_flag>, <&boot_stage>, - <&boot_error_count>, <&panic_count>; + <&boot_error_count>, <&panic_count>, <&pm_setting>; nvmem-cell-names = "shutdown_flag", "boot_stage", - "boot_error_count", "panic_count"; + "boot_error_count", "panic_count", "pm_setting"; + }; + + rtc { + compatible = "apple,smc-rtc"; + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; }; }; pinctrl_smc: pinctrl@2a2820000 { - compatible = "apple,t6020-pinctrl", "apple,t8103-pinctrl"; + compatible = "apple,t6020-pinctrl", "apple,t8103-pinctrl", "apple,pinctrl"; reg = <0x2 0xa2820000 0x0 0x4000>; gpio-controller; @@ -144,13 +369,309 @@ ; }; - sio_dart: iommu@39b008000 { + aop_mbox: mbox@2a6408000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0xa6408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + status = "disabled"; + }; + + aop_dart: iommu@2a6808000 { compatible = "apple,t6020-dart", "apple,t8110-dart"; - reg = <0x3 0x9b008000 0x0 0x8000>; + reg = <0x2 0xa6808000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + // apple,dma-range = <0x100 0x0 0x300 0x0>; + }; + + aop_admac: dma-controller@2a6980000 { + compatible = "apple,t6020-admac", "apple,admac"; + reg = <0x2 0xa6980000 0x0 0x34000>; + #dma-cells = <1>; + dma-channels = <16>; + interrupts-extended = <0>, + <0>, + <&aic AIC_IRQ 0 631 IRQ_TYPE_LEVEL_HIGH>, + <0>; + iommus = <&aop_dart 10>; + status = "disabled"; + }; + + aop: aop@2a6c00000 { + compatible = "apple,t6020-aop"; + reg = <0x2 0xa6c00000 0x0 0x250000>, + <0x2 0xa6400000 0x0 0x6c000>; + mboxes = <&aop_mbox>; + mbox-names = "mbox"; + iommus = <&aop_dart 0>; + + status = "disabled"; + + aop_audio: audio { + compatible = "apple,t6020-aop-audio", "apple,aop-audio"; + dmas = <&aop_admac 1>; + dma-names = "dma"; + }; + + aop_als: als { + compatible = "apple,t6020-aop-als", "apple,aop-als"; + // intentionally empty + }; + + las { + compatible = "apple,t6020-aop-las", "apple,aop-las"; + }; + }; + + mtp: mtp@2a9400000 { + compatible = "apple,t6020-mtp", "apple,t6020-rtk-helper-asc4", "apple,mtp", "apple,rtk-helper-asc4"; + reg = <0x2 0xa9400000 0x0 0x4000>, + <0x2 0xa9c00000 0x0 0x100000>; + reg-names = "asc", "sram"; + mboxes = <&mtp_mbox>; + iommus = <&mtp_dart 1>; + #helper-cells = <0>; + + status = "disabled"; + }; + + mtp_mbox: mbox@2a9408000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0xa9408000 0x0 0x4000>; interrupt-parent = <&aic>; - interrupts = ; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + + status = "disabled"; + }; + + mtp_dart: iommu@2a9808000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x2 0xa9808000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + + apple,dma-range = <0x100 0x0 0x1 0x0>; + + status = "disabled"; + }; + + mtp_dockchannel: fifo@2a9b14000 { + compatible = "apple,t6020-dockchannel", "apple,dockchannel"; + reg = <0x2 0xa9b14000 0x0 0x4000>; + reg-names = "irq"; + interrupt-parent = <&aic>; + interrupts = ; + + ranges = <0 0x2 0xa9b28000 0x20000>; + nonposted-mmio; + #address-cells = <1>; + #size-cells = <1>; + + interrupt-controller; + #interrupt-cells = <2>; + + status = "disabled"; + + mtp_hid: input@8000 { + compatible = "apple,dockchannel-hid"; + reg = <0x8000 0x4000>, + <0xc000 0x4000>, + <0x0000 0x4000>, + <0x4000 0x4000>; + reg-names = "config", "data", + "rmt-config", "rmt-data"; + iommus = <&mtp_dart 1>; + interrupt-parent = <&mtp_dockchannel>; + interrupts = <2 IRQ_TYPE_LEVEL_HIGH>, + <3 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "tx", "rx"; + + apple,fifo-size = <0x800>; + apple,helper-cpu = <&mtp>; + }; + + }; + + isp_dart0: iommu@3860e8000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x860e8000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + + apple,dma-range = <0x100 0x0 0x1 0x0>; + status = "disabled"; + }; + + isp_dart1: iommu@3860f4000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x860f4000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + + apple,dma-range = <0x100 0x0 0x1 0x0>; + status = "disabled"; + }; + + isp_dart2: iommu@3860fc000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x860fc000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&pmp_report_isp_sys>; + + apple,dma-range = <0x100 0x0 0x1 0x0>; + status = "disabled"; + }; + + isp: isp@384000000 { + compatible = "apple,t6020-isp", "apple,isp"; + iommus = <&isp_dart0 0>, <&isp_dart1 0>, <&isp_dart2 0>; + reg-names = "coproc", "mbox", "gpio", "mbox2"; + reg = <0x3 0x84000000 0x0 0x2000000>, + <0x3 0x86104000 0x0 0x100>, + <0x3 0x86104170 0x0 0x100>, + <0x3 0x861043f0 0x0 0x100>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_cpu>, <&ps_isp_fe>, + <&ps_dprx>, <&ps_isp_vis>, <&ps_isp_be>, + <&ps_isp_clr>, <&ps_isp_raw>; + apple,dart-vm-size = <0x0 0xa0000000>; + + status = "disabled"; + }; + + disp0_dart: iommu@389304000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x89304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + power-domains = <&ps_disp0_cpu0>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + }; + + dcp_dart: iommu@38930c000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x8930c000 0x0 0x4000>; #iommu-cells = <1>; - power-domains = <&ps_sio_cpu>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_disp0_cpu0>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + }; + + dcp_mbox: mbox@389c08000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x89c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_disp0_cpu0>; + }; + + dcp: dcp@389c00000 { + compatible = "apple,t6020-dcp", "apple,dcp"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 5>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x3 0x89c00000 0x0 0x4000>, // check? + <0x3 0x88000000 0x0 0x61c000>, + <0x3 0x89320000 0x0 0x4000>, + <0x3 0x89344000 0x0 0x4000>, + <0x3 0x89800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x1208>; + power-domains = <&pmp_report_disp0>; + resets = <&ps_disp0_cpu0>; + clocks = <&clk_disp0>; + phandle = <&dcp>; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + disp0_piodma: piodma { + iommus = <&disp0_dart 4>; + phandle = <&disp0_piodma>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dcp_audio: endpoint { + remote-endpoint = <&dpaudio0_dcp>; + }; + }; + }; + }; + + display: display-subsystem { + compatible = "apple,display-subsystem"; + iommus = <&disp0_dart 0>; + /* generate phandle explicitly for use in loader */ + phandle = <&display>; + }; + + sep_dart: iommu@394ac0000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x94ac0000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + }; + + sep: sep@396400000 { + compatible = "apple,sep"; + reg = <0x3 0x96400000 0x0 0x6C000>; + mboxes = <&sep_mbox>; + mbox-names = "mbox"; + iommus = <&sep_dart 0>; + power-domains = <&ps_sep>; + status = "disabled"; + }; + + sep_mbox: mbox@396408000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x96408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; }; fpwm0: pwm@39b030000 { @@ -163,7 +684,7 @@ }; i2c0: i2c@39b040000 { - compatible = "apple,t6020-i2c", "apple,t8103-i2c"; + compatible = "apple,t6020-i2c", "apple,t8103-i2c", "apple,i2c"; reg = <0x3 0x9b040000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -176,7 +697,7 @@ }; i2c1: i2c@39b044000 { - compatible = "apple,t6020-i2c", "apple,t8103-i2c"; + compatible = "apple,t6020-i2c", "apple,t8103-i2c", "apple,i2c"; reg = <0x3 0x9b044000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -190,7 +711,7 @@ }; i2c2: i2c@39b048000 { - compatible = "apple,t6020-i2c", "apple,t8103-i2c"; + compatible = "apple,t6020-i2c", "apple,t8103-i2c", "apple,i2c"; reg = <0x3 0x9b048000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -204,7 +725,7 @@ }; i2c3: i2c@39b04c000 { - compatible = "apple,t6020-i2c", "apple,t8103-i2c"; + compatible = "apple,t6020-i2c", "apple,t8103-i2c", "apple,i2c"; reg = <0x3 0x9b04c000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -218,7 +739,7 @@ }; i2c4: i2c@39b050000 { - compatible = "apple,t6020-i2c", "apple,t8103-i2c"; + compatible = "apple,t6020-i2c", "apple,t8103-i2c", "apple,i2c"; reg = <0x3 0x9b050000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -232,7 +753,7 @@ }; i2c5: i2c@39b054000 { - compatible = "apple,t6020-i2c", "apple,t8103-i2c"; + compatible = "apple,t6020-i2c", "apple,t8103-i2c", "apple,i2c"; reg = <0x3 0x9b054000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -246,7 +767,7 @@ }; i2c6: i2c@39b054000 { - compatible = "apple,t6020-i2c", "apple,t8103-i2c"; + compatible = "apple,t6020-i2c", "apple,t8103-i2c", "apple,i2c"; reg = <0x3 0x9b054000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -260,7 +781,7 @@ }; i2c7: i2c@39b054000 { - compatible = "apple,t6020-i2c", "apple,t8103-i2c"; + compatible = "apple,t6020-i2c", "apple,t8103-i2c", "apple,i2c"; reg = <0x3 0x9b054000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -274,7 +795,7 @@ }; i2c8: i2c@39b054000 { - compatible = "apple,t6020-i2c", "apple,t8103-i2c"; + compatible = "apple,t6020-i2c", "apple,t8103-i2c", "apple,i2c"; reg = <0x3 0x9b054000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -288,7 +809,7 @@ }; spi1: spi@39b104000 { - compatible = "apple,t6020-spi", "apple,t8103-spi"; + compatible = "apple,t6020-spi", "apple,t8103-spi", "apple,spi"; reg = <0x3 0x9b104000 0x0 0x4000>; interrupt-parent = <&aic>; interrupts = ; @@ -302,7 +823,7 @@ }; spi2: spi@39b108000 { - compatible = "apple,t6020-spi", "apple,t8103-spi"; + compatible = "apple,t6020-spi", "apple,t8103-spi", "apple,spi"; reg = <0x3 0x9b108000 0x0 0x4000>; interrupt-parent = <&aic>; interrupts = ; @@ -316,7 +837,7 @@ }; spi4: spi@39b110000 { - compatible = "apple,t6020-spi", "apple,t8103-spi"; + compatible = "apple,t6020-spi", "apple,t8103-spi", "apple,spi"; reg = <0x3 0x9b110000 0x0 0x4000>; interrupt-parent = <&aic>; interrupts = ; @@ -346,7 +867,7 @@ }; admac: dma-controller@39b400000 { - compatible = "apple,t6020-admac", "apple,t8103-admac"; + compatible = "apple,t6020-admac", "apple,t8103-admac", "apple,admac"; reg = <0x3 0x9b400000 0x0 0x34000>; #dma-cells = <1>; dma-channels = <16>; @@ -359,8 +880,29 @@ resets = <&ps_audio_p>; }; + dpaudio0: audio-controller@39b500000 { + compatible = "apple,t6020-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b500000 0x0 0x4000>; + dmas = <&sio 0x64>; + dma-names = "tx"; + power-domains = <&ps_dpa0>; + reset-domains = <&ps_dpa0>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dpaudio0_dcp: endpoint { + remote-endpoint = <&dcp_audio>; + }; + }; + }; + }; + mca: mca@39b600000 { - compatible = "apple,t6020-mca", "apple,t8103-mca"; + compatible = "apple,t6020-mca", "apple,t8103-mca", "apple,mca"; reg = <0x3 0x9b600000 0x0 0x10000>, <0x3 0x9b500000 0x0 0x20000>; clocks = <&nco 0>, <&nco 1>, <&nco 2>, <&nco 3>; @@ -388,14 +930,71 @@ reg = <0x4 0x6400000 0 0x40000>, <0x4 0x4000000 0 0x1000000>; reg-names = "asc", "sgx"; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; mboxes = <&agx_mbox>; - power-domains = <&ps_gfx>; + power-domains = <&pmp_report_gfx>; memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>, <&gpu_hw_cal_a>, <&gpu_hw_cal_b>, <&gpu_globals>; memory-region-names = "ttbs", "pagetables", "handoff", "hw-cal-a", "hw-cal-b", "globals"; apple,firmware-abi = <0 0 0>; + apple,firmware-version = <0 0 0>; + apple,firmware-compat = <0 0 0>; + + operating-points-v2 = <&gpu_opp>; + apple,cs-opp = <&gpu_cs_opp>; + apple,afr-opp = <&gpu_afr_opp>; + + apple,min-sram-microvolt = <790000>; + apple,csafr-min-sram-microvolt = <812000>; + apple,perf-base-pstate = <1>; + + apple,avg-power-min-duty-cycle = <40>; + apple,avg-power-target-filter-tc = <1>; + apple,fast-die0-proportional-gain = <34.0>; + apple,perf-boost-ce-step = <50>; + apple,perf-boost-min-util = <90>; + apple,perf-filter-drop-threshold = <0>; + apple,perf-filter-time-constant = <5>; + apple,perf-filter-time-constant2 = <200>; + apple,perf-integral-gain = <1.62>; + apple,perf-integral-gain2 = <1.62>; + apple,perf-integral-min-clamp = <0>; + apple,perf-proportional-gain2 = <5.4>; + apple,perf-proportional-gain = <5.4>; + apple,perf-tgt-utilization = <85>; + apple,power-sample-period = <8>; + apple,ppm-filter-time-constant-ms = <34>; + apple,ppm-ki = <18.0>; + apple,ppm-kp = <0.1>; + apple,pwr-filter-time-constant = <313>; + apple,pwr-integral-gain = <0.0202129>; + apple,pwr-integral-min-clamp = <0>; + apple,pwr-min-duty-cycle = <40>; + apple,pwr-proportional-gain = <5.2831855>; + apple,pwr-sample-period-aic-clks = <200000>; + apple,se-engagement-criteria = <700>; + apple,se-filter-time-constant = <9>; + apple,se-filter-time-constant-1 = <3>; + apple,se-inactive-threshold = <2500>; + apple,se-ki = <-50.0>; + apple,se-ki-1 = <-100.0>; + apple,se-kp = <-5.0>; + apple,se-kp-1 = <-10.0>; + apple,se-reset-criteria = <50>; + + apple,core-leak-coef = GPU_REPEAT(1200.0); + apple,sram-leak-coef = GPU_REPEAT(20.0); + apple,cs-leak-coef = GPU_DIE_REPEAT(400.0); + apple,afr-leak-coef = GPU_DIE_REPEAT(200.0); }; agx_mbox: mbox@406408000 { @@ -455,6 +1054,8 @@ pinctrl-0 = <&pcie_pins>; pinctrl-names = "default"; + dma-coherent; + port00: pci@0,0 { device_type = "pci"; reg = <0x0 0x0 0x0 0x0 0x0>; diff --git a/arch/arm64/boot/dts/apple/t602x-dieX.dtsi b/arch/arm64/boot/dts/apple/t602x-dieX.dtsi index cb07fd82b32e67..b3fafb91348fc0 100644 --- a/arch/arm64/boot/dts/apple/t602x-dieX.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-dieX.dtsi @@ -23,6 +23,83 @@ #performance-domain-cells = <0>; }; + DIE_NODE(dispext0_dart): iommu@289304000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x2 0x89304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_dispext0_cpu0)>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "disabled"; + }; + + DIE_NODE(dcpext0_dart): iommu@28930c000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x2 0x8930c000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_dispext0_cpu0)>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "disabled"; + }; + + DIE_NODE(dcpext0_mbox): mbox@289c08000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x89c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&DIE_NODE(ps_dispext0_cpu0)>; + resets = <&DIE_NODE(ps_dispext0_cpu0)>; + status = "disabled"; + }; + + DIE_NODE(dcpext0): dcp@289c00000 { + compatible = "apple,t6020-dcpext", "apple,dcpext"; + mboxes = <&DIE_NODE(dcpext0_mbox)>; + mbox-names = "mbox"; + iommus = <&DIE_NODE(dcpext0_dart) 5>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x2 0x89c00000 0x0 0x4000>, + <0x2 0x88000000 0x0 0x4000000>, + <0x2 0x89320000 0x0 0x4000>, + <0x2 0x89344000 0x0 0x4000>, + <0x2 0x89800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x1210>; + power-domains = <&DIE_NODE(pmp_report_dispext0)>; + resets = <&DIE_NODE(ps_dispext0_cpu0)>; + clocks = <&DIE_NODE(clk_dispext0)>; + phandle = <&DIE_NODE(dcpext0)>; + apple,dcp-index = <1>; + status = "disabled"; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + piodma { + iommus = <&DIE_NODE(dispext0_dart) 4>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dcpext0_audio): endpoint { + remote-endpoint = <&DIE_NODE(dpaudio1_dcp)>; + }; + }; + }; + }; + DIE_NODE(pmgr): power-management@28e080000 { compatible = "apple,t6020-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; #address-cells = <1>; @@ -45,7 +122,7 @@ }; DIE_NODE(pinctrl_nub): pinctrl@29e1f0000 { - compatible = "apple,t6020-pinctrl", "apple,t8103-pinctrl"; + compatible = "apple,t6020-pinctrl", "apple,t8103-pinctrl", "apple,pinctrl"; reg = <0x2 0x9e1f0000 0x0 0x4000>; power-domains = <&DIE_NODE(ps_nub_gpio)>; @@ -74,7 +151,7 @@ }; DIE_NODE(pinctrl_aop): pinctrl@2a6820000 { - compatible = "apple,t6020-pinctrl", "apple,t8103-pinctrl"; + compatible = "apple,t6020-pinctrl", "apple,t8103-pinctrl", "apple,pinctrl"; reg = <0x2 0xa6820000 0x0 0x4000>; gpio-controller; @@ -94,8 +171,95 @@ ; }; + DIE_NODE(dispext1_dart): iommu@315304000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x15304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_dispext1_cpu0)>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "disabled"; + }; + + DIE_NODE(dcpext1_dart): iommu@31530c000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x1530c000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_dispext1_cpu0)>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "disabled"; + }; + + DIE_NODE(dcpext1_mbox): mbox@315c08000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x15c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&DIE_NODE(ps_dispext1_cpu0)>; + resets = <&DIE_NODE(ps_dispext1_cpu0)>; + status = "disabled"; + }; + + DIE_NODE(dcpext1): dcp@315c00000 { + compatible = "apple,t6020-dcpext", "apple,dcpext"; + mboxes = <&DIE_NODE(dcpext1_mbox)>; + mbox-names = "mbox"; + iommus = <&DIE_NODE(dcpext1_dart) 5>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x3 0x15c00000 0x0 0x4000>, + <0x3 0x14000000 0x0 0x4000000>, + <0x3 0x15320000 0x0 0x4000>, + <0x3 0x15344000 0x0 0x4000>, + <0x3 0x15800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x1218>; + power-domains = <&DIE_NODE(pmp_report_dispext1)>; + resets = <&DIE_NODE(ps_dispext1_cpu0)>; + clocks = <&DIE_NODE(clk_dispext1)>; + phandle = <&DIE_NODE(dcpext1)>; + apple,dcp-index = <2>; + status = "disabled"; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + piodma { + iommus = <&DIE_NODE(dispext1_dart) 4>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dcpext1_audio): endpoint { + remote-endpoint = <&DIE_NODE(dpaudio2_dcp)>; + }; + }; + }; + }; + + DIE_NODE(sio_dart): iommu@39b008000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x9b008000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&DIE_NODE(ps_sio)>; + //apple,dma-range = <0x100 0x0001c000 0x2ff 0xfffe4000>; + }; + DIE_NODE(pinctrl_ap): pinctrl@39b028000 { - compatible = "apple,t6020-pinctrl", "apple,t8103-pinctrl"; + compatible = "apple,t6020-pinctrl", "apple,t8103-pinctrl", "apple,pinctrl"; reg = <0x3 0x9b028000 0x0 0x4000>; interrupt-parent = <&aic>; @@ -119,6 +283,131 @@ #interrupt-cells = <2>; }; + DIE_NODE(sio_mbox): mbox@39bc08000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x9bc08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&DIE_NODE(ps_sio_cpu)>; + }; + + DIE_NODE(sio): sio@39bc00000 { + compatible = "apple,t6020-sio", "apple,sio"; + reg = <0x3 0x9bc00000 0x0 0x8000>; + dma-channels = <128>; + #dma-cells = <1>; + mboxes = <&DIE_NODE(sio_mbox)>; + iommus = <&DIE_NODE(sio_dart) 0>; + power-domains = <&DIE_NODE(ps_sio_cpu)>; + resets = <&DIE_NODE(ps_sio_cpu)>; + status = "disabled"; + }; + + DIE_NODE(dpaudio1): audio-controller@39b504000 { + compatible = "apple,t6020-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b540000 0x0 0x4000>; + dmas = <&DIE_NODE(sio) 0x66>; + dma-names = "tx"; + power-domains = <&DIE_NODE(ps_dpa1)>; + reset-domains = <&DIE_NODE(ps_dpa1)>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dpaudio1_dcp): endpoint { + remote-endpoint = <&DIE_NODE(dcpext0_audio)>; + }; + }; + }; + }; + + DIE_NODE(dpaudio2): audio-controller@39b508000 { + compatible = "apple,t6020-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b580000 0x0 0x4000>; + dmas = <&DIE_NODE(sio) 0x68>; + dma-names = "tx"; + power-domains = <&DIE_NODE(ps_dpa2)>; + reset-domains = <&DIE_NODE(ps_dpa2)>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dpaudio2_dcp): endpoint { + remote-endpoint = <&DIE_NODE(dcpext1_audio)>; + }; + }; + }; + }; + + /* + * omit dpaudio3 / 4 as long as the linked dcpext nodes don't exist + * + DIE_NODE(dpaudio3): audio-controller@39b50c000 { + compatible = "apple,t6020-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b5c0000 0x0 0x4000>; + dmas = <&DIE_NODE(sio) 0x6a>; + dma-names = "tx"; + power-domains = <&DIE_NODE(ps_dpa3)>; + reset-domains = <&DIE_NODE(ps_dpa3)>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dpaudio3_dcp): endpoint { + remote-endpoint = <&DIE_NODE(dcpext2_audio)>; + }; + }; + }; + }; + + DIE_NODE(dpaudio4): audio-controller@39b510000 { + compatible = "apple,t6020-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b500000 0x0 0x4000>; + dmas = <&DIE_NODE(sio) 0x6c>; + dma-names = "tx"; + power-domains = <&DIE_NODE(ps_dpa4)>; + reset-domains = <&DIE_NODE(ps_dpa4)>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dpaudio4_dcp): endpoint { + remote-endpoint = <&DIE_NODE(dcpext3_audio)>; + }; + }; + }; + }; + */ + + DIE_NODE(lpdptxphy): phy@39c000000 { + compatible = "apple,t6020-dptx-phy", "apple,dptx-phy"; + reg = <0x3 0x9c000000 0x0 0x4000>, + <0x3 0x9c040000 0x0 0xc000>; + reg-names = "core", "dptx"; + power-domains = <&DIE_NODE(ps_dptx_phy_ps)>; + #phy-cells = <0>; + #reset-cells = <0>; + status = "disabled"; /* only exposed on desktop devices */ + }; + DIE_NODE(pmgr_gfx): power-management@404e80000 { compatible = "apple,t6020-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; #address-cells = <1>; @@ -126,3 +415,247 @@ reg = <0x4 0x4e80000 0 0x4000>; }; + + DIE_NODE(dwc3_0): usb@702280000 { + compatible = "apple,t6020-dwc3", "apple,t8103-dwc3"; + reg = <0x7 0x02280000 0x0 0xcd00>, <0x7 0x0228cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&DIE_NODE(dwc3_0_dart_0) 0>, + <&DIE_NODE(dwc3_0_dart_1) 1>; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + resets = <&DIE_NODE(atcphy0)>; + phys = <&DIE_NODE(atcphy0) PHY_TYPE_USB2>, <&DIE_NODE(atcphy0) PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + DIE_NODE(dwc3_0_dart_0): iommu@702f00000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x7 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_0_dart_1): iommu@702f80000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x7 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(atcphy0): phy@703000000 { + compatible = "apple,t6020-atcphy", "apple,t8103-atcphy"; + reg = <0x7 0x03000000 0x0 0x4c000>, + <0x7 0x03050000 0x0 0x8000>, + <0x7 0x00000000 0x0 0x4000>, + <0x7 0x02a90000 0x0 0x4000>, + <0x7 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + }; + + DIE_NODE(atcphy0_xbar): mux@70304c000 { + compatible = "apple,t6020-display-crossbar"; + reg = <0x7 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + status = "disabled"; + }; + + DIE_NODE(dwc3_1): usb@b02280000 { + compatible = "apple,t6020-dwc3", "apple,t8103-dwc3"; + reg = <0xb 0x02280000 0x0 0xcd00>, <0xb 0x0228cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&DIE_NODE(dwc3_1_dart_0) 0>, + <&DIE_NODE(dwc3_1_dart_1) 1>; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + resets = <&DIE_NODE(atcphy1)>; + phys = <&DIE_NODE(atcphy1) PHY_TYPE_USB2>, <&DIE_NODE(atcphy1) PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + DIE_NODE(dwc3_1_dart_0): iommu@b02f00000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0xb 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_1_dart_1): iommu@b02f80000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0xb 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(atcphy1): phy@b03000000 { + compatible = "apple,t6020-atcphy", "apple,t8103-atcphy"; + reg = <0xb 0x03000000 0x0 0x4c000>, + <0xb 0x03050000 0x0 0x8000>, + <0xb 0x00000000 0x0 0x4000>, + <0xb 0x02a90000 0x0 0x4000>, + <0xb 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + }; + + DIE_NODE(atcphy1_xbar): mux@b0304c000 { + compatible = "apple,t6020-display-crossbar"; + reg = <0xb 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + status = "disabled"; + }; + + DIE_NODE(dwc3_2): usb@f02280000 { + compatible = "apple,t6020-dwc3", "apple,t8103-dwc3"; + reg = <0xf 0x02280000 0x0 0xcd00>, <0xf 0x0228cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&DIE_NODE(dwc3_2_dart_0) 0>, + <&DIE_NODE(dwc3_2_dart_1) 1>; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + resets = <&DIE_NODE(atcphy2)>; + phys = <&DIE_NODE(atcphy2) PHY_TYPE_USB2>, <&DIE_NODE(atcphy2) PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + DIE_NODE(dwc3_2_dart_0): iommu@f02f00000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0xf 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_2_dart_1): iommu@f02f80000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0xf 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(atcphy2): phy@f03000000 { + compatible = "apple,t6020-atcphy", "apple,t8103-atcphy"; + reg = <0xf 0x03000000 0x0 0x4c000>, + <0xf 0x03050000 0x0 0x8000>, + <0xf 0x00000000 0x0 0x4000>, + <0xf 0x02a90000 0x0 0x4000>, + <0xf 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + }; + + DIE_NODE(atcphy2_xbar): mux@f0304c000 { + compatible = "apple,t6020-display-crossbar"; + reg = <0xf 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + status = "disabled"; + }; + + DIE_NODE(dwc3_3): usb@1302280000 { + compatible = "apple,t6020-dwc3", "apple,t8103-dwc3"; + reg = <0x13 0x02280000 0x0 0xcd00>, <0x13 0x0228cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&DIE_NODE(dwc3_3_dart_0) 0>, + <&DIE_NODE(dwc3_3_dart_1) 1>; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + resets = <&DIE_NODE(atcphy3)>; + phys = <&DIE_NODE(atcphy3) PHY_TYPE_USB2>, <&DIE_NODE(atcphy3) PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + DIE_NODE(dwc3_3_dart_0): iommu@1302f00000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x13 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_3_dart_1): iommu@1302f80000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x13 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(atcphy3): phy@1303000000 { + compatible = "apple,t6020-atcphy", "apple,t8103-atcphy"; + reg = <0x13 0x03000000 0x0 0x4c000>, + <0x13 0x03050000 0x0 0x8000>, + <0x13 0x00000000 0x0 0x4000>, + <0x13 0x02a90000 0x0 0x4000>, + <0x13 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + }; + + DIE_NODE(atcphy3_xbar): mux@130304c000 { + compatible = "apple,t6020-display-crossbar"; + reg = <0x13 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + status = "disabled"; + }; diff --git a/arch/arm64/boot/dts/apple/t602x-gpio-pins.dtsi b/arch/arm64/boot/dts/apple/t602x-gpio-pins.dtsi index e41b6475f79218..c5de99bd2e5cf3 100644 --- a/arch/arm64/boot/dts/apple/t602x-gpio-pins.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-gpio-pins.dtsi @@ -78,4 +78,8 @@ , ; }; + + pcie_ge_pins: pcie-ge-pins { + pinmux = ; + }; }; diff --git a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi index 0e806d8ddf81b1..0057e6a9465f9d 100644 --- a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi @@ -14,12 +14,31 @@ * the GPIO indices. */ +#define NO_SPI_TRACKPAD #include "t600x-j314-j316.dtsi" +/ { + aliases { + keyboard = &keyboard; + }; +}; + &framebuffer0 { power-domains = <&ps_disp0_cpu0>, <&ps_dptx_phy_ps>; }; +/* HACK: keep dptx_phy_ps power-domain always-on + * it is unclear how to sequence with dcp for the integrated display + */ +&ps_dptx_phy_ps { + apple,always-on; +}; + +&dcpext0 { + /* HDMI HPD gpio, used as interrupt*/ + hdmi-hpd-gpios = <&pinctrl_aop 25 GPIO_ACTIVE_HIGH>; +}; + &hpm0 { interrupts = <44 IRQ_TYPE_LEVEL_LOW>; }; @@ -36,6 +55,40 @@ interrupts = <44 IRQ_TYPE_LEVEL_LOW>; }; +/* Redefine GPIO for SDZ */ +&speaker_sdz { + gpios = <&pinctrl_ap 57 GPIO_ACTIVE_HIGH>; +}; + +&speaker_left_tweet { + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; +}; + +&speaker_left_woof1 { + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; +}; + +&speaker_left_woof2 { + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; +}; + +&speaker_right_tweet { + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; +}; + +&speaker_right_woof1 { + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; +}; + +&speaker_right_woof2 { + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; +}; + +&jack_codec { + reset-gpios = <&pinctrl_nub 8 GPIO_ACTIVE_HIGH>; + interrupts-extended = <&pinctrl_ap 59 IRQ_TYPE_LEVEL_LOW>; +}; + &wifi0 { compatible = "pci14e4,4434"; }; @@ -43,3 +96,56 @@ &bluetooth0 { compatible = "pci14e4,5f72"; }; + +&port01 { + pwren-gpios = <&smc_gpio 22 GPIO_ACTIVE_HIGH>; +}; + +&ps_mtp_fabric { + status = "okay"; +}; + +&mtp { + status = "okay"; +}; + +&mtp_mbox { + status = "okay"; +}; + +&mtp_dart { + status = "okay"; +}; + +&mtp_dockchannel { + status = "okay"; +}; + +&mtp_hid { + apple,afe-reset-gpios = <&smc_gpio 25 GPIO_ACTIVE_LOW>; + apple,stm-reset-gpios = <&smc_gpio 26 GPIO_ACTIVE_LOW>; + + mtp_mt: multi-touch { + }; + + keyboard: keyboard { + hid-country-code = <0>; + apple,keyboard-layout-id = <0>; + }; + + stm { + }; + + actuator { + }; + + tp_accel { + }; +}; + +&isp { + apple,platform-id = <7>; + /delete-node/ sensor-presets; /* Override j31[46] below */ +}; + +#include "isp-imx558-cfg0.dtsi" diff --git a/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi b/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi index ee12fea5b12cb3..287348628eb177 100644 --- a/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi @@ -36,3 +36,23 @@ &hpm3 { interrupts = <44 IRQ_TYPE_LEVEL_LOW>; }; + +/* PCIe devices */ +&port00 { + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; +}; + +&port03 { + /* USB xHCI */ + pwren-gpios = <&smc_gpio 19 GPIO_ACTIVE_HIGH>; +}; + +&speaker { + shutdown-gpios = <&pinctrl_ap 57 GPIO_ACTIVE_HIGH>; + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; +}; + +&jack_codec { + reset-gpios = <&pinctrl_nub 8 GPIO_ACTIVE_HIGH>; + interrupts-extended = <&pinctrl_ap 59 IRQ_TYPE_LEVEL_LOW>; +}; diff --git a/arch/arm64/boot/dts/apple/t602x-nvme.dtsi b/arch/arm64/boot/dts/apple/t602x-nvme.dtsi index 590cec8ac804c0..eb8c4e359079e5 100644 --- a/arch/arm64/boot/dts/apple/t602x-nvme.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-nvme.dtsi @@ -26,7 +26,7 @@ }; DIE_NODE(nvme): nvme@34bcc0000 { - compatible = "apple,t6020-nvme-ans2", "apple,t8103-nvme-ans2"; + compatible = "apple,t6020-nvme-ans2", "apple,t8103-nvme-ans2", "apple,nvme-ans2"; reg = <0x3 0x4bcc0000 0x0 0x40000>, <0x3 0x47400000 0x0 0x4000>; reg-names = "nvme", "ans"; interrupt-parent = <&aic>; diff --git a/arch/arm64/boot/dts/apple/t602x-pcie-ge.dtsi b/arch/arm64/boot/dts/apple/t602x-pcie-ge.dtsi new file mode 100644 index 00000000000000..4a509cae0e5766 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t602x-pcie-ge.dtsi @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * PCIe-GE Nodes present on both dies of a T6022 (M2 Ultra) and M2 Pro/Max but + * only used on T6022 in the Mac Pro (2023). + * + * Copyright The Asahi Linux Contributors + */ + + DIE_NODE(pcie_ge): pcie@1680000000 { + compatible = "apple,t6020-pcie-ge", "apple,t6020-pcie"; + device_type = "pci"; + + reg = <0x16 0x80000000 0x0 0x1000000>, /* config */ + <0x16 0x91000000 0x0 0x4000>, /* rc */ + <0x16 0x94008000 0x0 0x4000>, /* port0 */ + <0x16 0x9e01c000 0x0 0x4000>, /* phy0 */ + <0x16 0x9401c000 0x0 0x1000>; /* ltssm0 */ + reg-names = "config", "rc", "port0", "phy0", "ltssm0"; + + interrupt-parent = <&aic>; + interrupts = ; + + msi-controller; + msi-parent = <&DIE_NODE(pcie_ge)>; + msi-ranges = <&aic AIC_IRQ DIE_NO 1672 IRQ_TYPE_EDGE_RISING 128>; + + iommu-map = <0x000 &DIE_NODE(pcie_ge_dart) 0 0>, + <0x100 &DIE_NODE(pcie_ge_dart) 1 1>, + <0x200 &DIE_NODE(pcie_ge_dart) 2 2>, + <0x300 &DIE_NODE(pcie_ge_dart) 3 3>, + <0x400 &DIE_NODE(pcie_ge_dart) 4 4>, + <0x500 &DIE_NODE(pcie_ge_dart) 5 5>, + <0x600 &DIE_NODE(pcie_ge_dart) 6 6>, + <0x700 &DIE_NODE(pcie_ge_dart) 7 7>, + <0x800 &DIE_NODE(pcie_ge_dart) 8 8>, + <0x900 &DIE_NODE(pcie_ge_dart) 9 9>, + <0xa00 &DIE_NODE(pcie_ge_dart) 10 10>, + <0xb00 &DIE_NODE(pcie_ge_dart) 11 11>, + <0xc00 &DIE_NODE(pcie_ge_dart) 12 12>, + <0xd00 &DIE_NODE(pcie_ge_dart) 13 13>, + <0xe00 &DIE_NODE(pcie_ge_dart) 14 14>, + <0xf00 &DIE_NODE(pcie_ge_dart) 15 15>; + iommu-map-mask = <0xff00>; + + bus-range = <0 15>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x43000000 0x18 0x00000000 0x18 0x00000000 0x4 0x00000000>, + <0x02000000 0x00 0x80000000 0x17 0x80000000 0x0 0x80000000>; + + power-domains = <&DIE_NODE(ps_apcie_ge_sys)>; + pinctrl-0 = <&DIE_NODE(pcie_ge_pins)>; + pinctrl-names = "default"; + + dma-coherent; + + status = "disabled"; + + DIE_NODE(port_ge00): pci@0,0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + reset-gpios = <&DIE_NODE(pinctrl_ap) 9 GPIO_ACTIVE_LOW>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &DIE_NODE(port_ge00) 0 0 0 0>, + <0 0 0 2 &DIE_NODE(port_ge00) 0 0 0 1>, + <0 0 0 3 &DIE_NODE(port_ge00) 0 0 0 2>, + <0 0 0 4 &DIE_NODE(port_ge00) 0 0 0 3>; + }; + }; + + DIE_NODE(pcie_ge_dart): iommu@1694000000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x16 0x94000000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_apcie_ge_sys)>; + status = "disabled"; + }; diff --git a/arch/arm64/boot/dts/apple/t602x-pmgr.dtsi b/arch/arm64/boot/dts/apple/t602x-pmgr.dtsi index f5382a2faf0b25..b9233f252e6ca7 100644 --- a/arch/arm64/boot/dts/apple/t602x-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-pmgr.dtsi @@ -7,7 +7,7 @@ &DIE_NODE(pmgr) { DIE_NODE(ps_afi): power-controller@100 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x100 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -16,7 +16,7 @@ }; DIE_NODE(ps_aic): power-controller@108 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x108 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -25,7 +25,7 @@ }; DIE_NODE(ps_dwi): power-controller@110 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x110 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -33,7 +33,7 @@ }; DIE_NODE(ps_pms): power-controller@118 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x118 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -42,7 +42,7 @@ }; DIE_NODE(ps_gpio): power-controller@120 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x120 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -51,7 +51,7 @@ }; DIE_NODE(ps_soc_dpe): power-controller@128 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x128 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -60,7 +60,7 @@ }; DIE_NODE(ps_pms_c1ppt): power-controller@130 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x130 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -69,7 +69,7 @@ }; DIE_NODE(ps_pmgr_soc_ocla): power-controller@138 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x138 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -78,7 +78,7 @@ }; DIE_NODE(ps_amcc0): power-controller@168 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x168 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -87,7 +87,7 @@ }; DIE_NODE(ps_amcc2): power-controller@170 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x170 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -96,7 +96,7 @@ }; DIE_NODE(ps_dcs_00): power-controller@178 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x178 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -105,7 +105,7 @@ }; DIE_NODE(ps_dcs_01): power-controller@180 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x180 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -114,7 +114,7 @@ }; DIE_NODE(ps_dcs_02): power-controller@188 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x188 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -123,7 +123,7 @@ }; DIE_NODE(ps_dcs_03): power-controller@190 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x190 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -132,7 +132,7 @@ }; DIE_NODE(ps_dcs_08): power-controller@198 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x198 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -141,7 +141,7 @@ }; DIE_NODE(ps_dcs_09): power-controller@1a0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1a0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -150,7 +150,7 @@ }; DIE_NODE(ps_dcs_10): power-controller@1a8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1a8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -159,7 +159,7 @@ }; DIE_NODE(ps_dcs_11): power-controller@1b0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1b0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -168,7 +168,7 @@ }; DIE_NODE(ps_afnc1_ioa): power-controller@1b8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1b8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -178,7 +178,7 @@ }; DIE_NODE(ps_afc): power-controller@1d0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1d0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -187,7 +187,7 @@ }; DIE_NODE(ps_afnc0_ioa): power-controller@1e8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1e8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -197,7 +197,7 @@ }; DIE_NODE(ps_afnc1_ls): power-controller@1f0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1f0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -207,7 +207,7 @@ }; DIE_NODE(ps_afnc0_ls): power-controller@1f8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1f8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -217,7 +217,7 @@ }; DIE_NODE(ps_afnc1_lw0): power-controller@200 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x200 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -227,7 +227,7 @@ }; DIE_NODE(ps_afnc1_lw1): power-controller@208 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x208 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -237,7 +237,7 @@ }; DIE_NODE(ps_afnc1_lw2): power-controller@210 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x210 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -247,7 +247,7 @@ }; DIE_NODE(ps_afnc0_lw0): power-controller@218 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x218 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -257,7 +257,7 @@ }; DIE_NODE(ps_scodec): power-controller@220 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x220 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -266,7 +266,7 @@ }; DIE_NODE(ps_atc0_common): power-controller@228 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x228 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -275,7 +275,7 @@ }; DIE_NODE(ps_atc1_common): power-controller@230 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x230 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -284,7 +284,7 @@ }; DIE_NODE(ps_atc2_common): power-controller@238 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x238 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -293,7 +293,7 @@ }; DIE_NODE(ps_atc3_common): power-controller@240 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x240 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -302,7 +302,7 @@ }; DIE_NODE(ps_dispext1_sys): power-controller@248 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x248 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -311,7 +311,7 @@ }; DIE_NODE(ps_pms_bridge): power-controller@250 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x250 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -321,7 +321,7 @@ }; DIE_NODE(ps_dispext0_sys): power-controller@258 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x258 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -330,7 +330,7 @@ }; DIE_NODE(ps_ane_sys): power-controller@260 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x260 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -339,7 +339,7 @@ }; DIE_NODE(ps_avd_sys): power-controller@268 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x268 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -348,7 +348,7 @@ }; DIE_NODE(ps_atc0_cio): power-controller@270 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x270 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -357,7 +357,7 @@ }; DIE_NODE(ps_atc0_pcie): power-controller@278 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x278 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -366,7 +366,7 @@ }; DIE_NODE(ps_atc1_cio): power-controller@280 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x280 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -375,7 +375,7 @@ }; DIE_NODE(ps_atc1_pcie): power-controller@288 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x288 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -384,7 +384,7 @@ }; DIE_NODE(ps_atc2_cio): power-controller@290 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x290 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -393,7 +393,7 @@ }; DIE_NODE(ps_atc2_pcie): power-controller@298 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x298 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -402,7 +402,7 @@ }; DIE_NODE(ps_atc3_cio): power-controller@2a0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2a0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -411,7 +411,7 @@ }; DIE_NODE(ps_atc3_pcie): power-controller@2a8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2a8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -420,7 +420,7 @@ }; DIE_NODE(ps_dispext1_fe): power-controller@2b0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2b0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -429,7 +429,7 @@ }; DIE_NODE(ps_dispext1_cpu0): power-controller@2b8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2b8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -439,7 +439,7 @@ }; DIE_NODE(ps_dispext0_fe): power-controller@2c0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2c0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -447,8 +447,9 @@ power-domains = <&DIE_NODE(ps_dispext0_sys)>; }; + /* PMP is only present on die 0 of the M2 Ultra */ DIE_NODE(ps_pmp): power-controller@2c8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2c8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -456,7 +457,7 @@ }; DIE_NODE(ps_pms_sram): power-controller@2d0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2d0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -464,7 +465,7 @@ }; DIE_NODE(ps_dispext0_cpu0): power-controller@2d8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2d8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -474,7 +475,7 @@ }; DIE_NODE(ps_ane_cpu): power-controller@2e0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2e0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -483,7 +484,7 @@ }; DIE_NODE(ps_atc0_cio_pcie): power-controller@2e8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2e8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -492,7 +493,7 @@ }; DIE_NODE(ps_atc0_cio_usb): power-controller@2f0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2f0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -501,7 +502,7 @@ }; DIE_NODE(ps_atc1_cio_pcie): power-controller@2f8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2f8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -510,7 +511,7 @@ }; DIE_NODE(ps_atc1_cio_usb): power-controller@300 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x300 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -519,7 +520,7 @@ }; DIE_NODE(ps_atc2_cio_pcie): power-controller@308 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x308 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -528,7 +529,7 @@ }; DIE_NODE(ps_atc2_cio_usb): power-controller@310 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x310 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -537,7 +538,7 @@ }; DIE_NODE(ps_atc3_cio_pcie): power-controller@318 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x318 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -546,7 +547,7 @@ }; DIE_NODE(ps_atc3_cio_usb): power-controller@320 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x320 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -555,7 +556,7 @@ }; DIE_NODE(ps_trace_fab): power-controller@390 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x390 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -563,7 +564,7 @@ }; DIE_NODE(ps_ane_sys_mpm): power-controller@4000 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4000 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -572,7 +573,7 @@ }; DIE_NODE(ps_ane_td): power-controller@4008 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4008 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -581,7 +582,7 @@ }; DIE_NODE(ps_ane_base): power-controller@4010 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4010 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -590,7 +591,7 @@ }; DIE_NODE(ps_ane_set1): power-controller@4018 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4018 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -599,7 +600,7 @@ }; DIE_NODE(ps_ane_set2): power-controller@4020 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4020 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -608,7 +609,7 @@ }; DIE_NODE(ps_ane_set3): power-controller@4028 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4028 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -617,7 +618,7 @@ }; DIE_NODE(ps_ane_set4): power-controller@4030 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4030 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -628,7 +629,7 @@ &DIE_NODE(pmgr_south) { DIE_NODE(ps_amcc4): power-controller@100 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x100 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -637,7 +638,7 @@ }; DIE_NODE(ps_amcc5): power-controller@108 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x108 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -646,7 +647,7 @@ }; DIE_NODE(ps_amcc6): power-controller@110 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x110 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -655,7 +656,7 @@ }; DIE_NODE(ps_amcc7): power-controller@118 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x118 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -664,7 +665,7 @@ }; DIE_NODE(ps_dcs_16): power-controller@120 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x120 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -673,7 +674,7 @@ }; DIE_NODE(ps_dcs_17): power-controller@128 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x128 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -682,7 +683,7 @@ }; DIE_NODE(ps_dcs_18): power-controller@130 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x130 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -691,7 +692,7 @@ }; DIE_NODE(ps_dcs_19): power-controller@138 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x138 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -700,7 +701,7 @@ }; DIE_NODE(ps_dcs_20): power-controller@140 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x140 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -709,7 +710,7 @@ }; DIE_NODE(ps_dcs_21): power-controller@148 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x148 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -718,7 +719,7 @@ }; DIE_NODE(ps_dcs_22): power-controller@150 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x150 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -727,7 +728,7 @@ }; DIE_NODE(ps_dcs_23): power-controller@158 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x158 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -736,7 +737,7 @@ }; DIE_NODE(ps_dcs_24): power-controller@160 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x160 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -745,7 +746,7 @@ }; DIE_NODE(ps_dcs_25): power-controller@168 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x168 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -754,7 +755,7 @@ }; DIE_NODE(ps_dcs_26): power-controller@170 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x170 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -763,7 +764,7 @@ }; DIE_NODE(ps_dcs_27): power-controller@178 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x178 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -772,7 +773,7 @@ }; DIE_NODE(ps_dcs_28): power-controller@180 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x180 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -781,7 +782,7 @@ }; DIE_NODE(ps_dcs_29): power-controller@188 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x188 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -790,7 +791,7 @@ }; DIE_NODE(ps_dcs_30): power-controller@190 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x190 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -799,7 +800,7 @@ }; DIE_NODE(ps_dcs_31): power-controller@198 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x198 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -808,7 +809,7 @@ }; DIE_NODE(ps_afnc4_ioa): power-controller@1a0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1a0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -818,7 +819,7 @@ }; DIE_NODE(ps_afnc4_ls): power-controller@1a8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1a8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -828,7 +829,7 @@ }; DIE_NODE(ps_afnc4_lw0): power-controller@1b0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1b0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -838,7 +839,7 @@ }; DIE_NODE(ps_afnc5_ioa): power-controller@1b8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1b8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -848,7 +849,7 @@ }; DIE_NODE(ps_afnc5_ls): power-controller@1c0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1c0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -858,7 +859,7 @@ }; DIE_NODE(ps_afnc5_lw0): power-controller@1c8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1c8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -868,7 +869,7 @@ }; DIE_NODE(ps_dispext2_sys): power-controller@1d0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1d0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -876,7 +877,7 @@ }; DIE_NODE(ps_msr1): power-controller@1d8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1d8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -884,7 +885,7 @@ }; DIE_NODE(ps_dispext2_fe): power-controller@1e0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1e0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -893,7 +894,7 @@ }; DIE_NODE(ps_dispext2_cpu0): power-controller@1e8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1e8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -903,7 +904,7 @@ }; DIE_NODE(ps_msr1_ase_core): power-controller@1f0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1f0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -912,7 +913,7 @@ }; DIE_NODE(ps_dispext3_sys): power-controller@220 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x220 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -920,7 +921,7 @@ }; DIE_NODE(ps_venc1_sys): power-controller@228 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x228 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -928,7 +929,7 @@ }; DIE_NODE(ps_dispext3_fe): power-controller@230 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x230 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -937,7 +938,7 @@ }; DIE_NODE(ps_dispext3_cpu0): power-controller@238 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x238 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -947,7 +948,7 @@ }; DIE_NODE(ps_venc1_dma): power-controller@4000 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4000 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -956,7 +957,7 @@ }; DIE_NODE(ps_venc1_pipe4): power-controller@4008 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4008 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -965,7 +966,7 @@ }; DIE_NODE(ps_venc1_pipe5): power-controller@4010 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4010 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -974,7 +975,7 @@ }; DIE_NODE(ps_venc1_me0): power-controller@4018 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4018 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -983,7 +984,7 @@ }; DIE_NODE(ps_venc1_me1): power-controller@4020 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4020 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -994,7 +995,7 @@ &DIE_NODE(pmgr_east) { DIE_NODE(ps_clvr_spmi0): power-controller@100 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x100 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1003,7 +1004,7 @@ }; DIE_NODE(ps_clvr_spmi1): power-controller@108 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x108 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1012,7 +1013,7 @@ }; DIE_NODE(ps_clvr_spmi2): power-controller@110 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x110 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1021,7 +1022,7 @@ }; DIE_NODE(ps_clvr_spmi3): power-controller@118 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x118 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1030,7 +1031,7 @@ }; DIE_NODE(ps_clvr_spmi4): power-controller@120 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x120 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1039,7 +1040,7 @@ }; DIE_NODE(ps_ispsens0): power-controller@128 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x128 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1047,7 +1048,7 @@ }; DIE_NODE(ps_ispsens1): power-controller@130 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x130 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1055,7 +1056,7 @@ }; DIE_NODE(ps_ispsens2): power-controller@138 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x138 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1063,7 +1064,7 @@ }; DIE_NODE(ps_ispsens3): power-controller@140 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x140 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1071,7 +1072,7 @@ }; DIE_NODE(ps_afnc6_ioa): power-controller@148 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x148 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1081,7 +1082,7 @@ }; DIE_NODE(ps_afnc6_ls): power-controller@150 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x150 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1091,7 +1092,7 @@ }; DIE_NODE(ps_afnc6_lw0): power-controller@158 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x158 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1101,7 +1102,7 @@ }; DIE_NODE(ps_afnc2_ioa): power-controller@160 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x160 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1111,7 +1112,7 @@ }; DIE_NODE(ps_afnc2_ls): power-controller@168 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x168 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1121,7 +1122,7 @@ }; DIE_NODE(ps_afnc2_lw0): power-controller@170 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x170 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1131,7 +1132,7 @@ }; DIE_NODE(ps_afnc2_lw1): power-controller@178 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x178 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1141,7 +1142,7 @@ }; DIE_NODE(ps_afnc3_ioa): power-controller@180 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x180 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1151,7 +1152,7 @@ }; DIE_NODE(ps_afnc3_ls): power-controller@188 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x188 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1161,7 +1162,7 @@ }; DIE_NODE(ps_afnc3_lw0): power-controller@190 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x190 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1171,7 +1172,7 @@ }; DIE_NODE(ps_apcie_gp): power-controller@198 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x198 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1180,7 +1181,7 @@ }; DIE_NODE(ps_apcie_st): power-controller@1a0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1a0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1189,7 +1190,7 @@ }; DIE_NODE(ps_ans2): power-controller@1a8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1a8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1198,7 +1199,7 @@ }; DIE_NODE(ps_disp0_sys): power-controller@1b0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1b0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1207,7 +1208,7 @@ }; DIE_NODE(ps_jpg): power-controller@1b8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1b8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1216,7 +1217,7 @@ }; DIE_NODE(ps_sio): power-controller@1c0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1c0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1225,7 +1226,7 @@ }; DIE_NODE(ps_isp_sys): power-controller@1c8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1c8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1235,7 +1236,7 @@ }; DIE_NODE(ps_disp0_fe): power-controller@1d0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1d0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1244,7 +1245,7 @@ }; DIE_NODE(ps_disp0_cpu0): power-controller@1d8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1d8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1254,16 +1255,16 @@ }; DIE_NODE(ps_sio_cpu): power-controller@1e0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1e0 4>; #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(sio_cpu); - power-domains = <&DIE_NODE(ps_sio)>; + power-domains = <&DIE_NODE(ps_sio) &DIE_NODE(ps_uart_p) &DIE_NODE(ps_spi_p) &DIE_NODE(ps_audio_p)>; }; DIE_NODE(ps_fpwm0): power-controller@1e8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1e8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1272,7 +1273,7 @@ }; DIE_NODE(ps_fpwm1): power-controller@1f0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1f0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1281,7 +1282,7 @@ }; DIE_NODE(ps_fpwm2): power-controller@1f8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1f8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1290,7 +1291,7 @@ }; DIE_NODE(ps_i2c0): power-controller@200 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x200 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1299,7 +1300,7 @@ }; DIE_NODE(ps_i2c1): power-controller@208 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x208 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1308,7 +1309,7 @@ }; DIE_NODE(ps_i2c2): power-controller@210 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x210 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1317,7 +1318,7 @@ }; DIE_NODE(ps_i2c3): power-controller@218 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x218 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1326,7 +1327,7 @@ }; DIE_NODE(ps_i2c4): power-controller@220 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x220 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1335,7 +1336,7 @@ }; DIE_NODE(ps_i2c5): power-controller@228 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x228 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1344,7 +1345,7 @@ }; DIE_NODE(ps_i2c6): power-controller@230 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x230 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1353,7 +1354,7 @@ }; DIE_NODE(ps_i2c7): power-controller@238 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x238 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1362,7 +1363,7 @@ }; DIE_NODE(ps_i2c8): power-controller@240 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x240 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1371,7 +1372,7 @@ }; DIE_NODE(ps_spi_p): power-controller@248 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x248 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1380,7 +1381,7 @@ }; DIE_NODE(ps_sio_spmi0): power-controller@250 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x250 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1389,7 +1390,7 @@ }; DIE_NODE(ps_sio_spmi1): power-controller@258 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x258 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1398,7 +1399,7 @@ }; DIE_NODE(ps_sio_spmi2): power-controller@260 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x260 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1407,7 +1408,7 @@ }; DIE_NODE(ps_uart_p): power-controller@268 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x268 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1416,7 +1417,7 @@ }; DIE_NODE(ps_audio_p): power-controller@270 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x270 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1425,7 +1426,7 @@ }; DIE_NODE(ps_sio_adma): power-controller@278 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x278 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1434,7 +1435,7 @@ }; DIE_NODE(ps_aes): power-controller@280 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x280 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1444,7 +1445,7 @@ }; DIE_NODE(ps_dptx_phy_ps): power-controller@288 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x288 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1453,7 +1454,7 @@ }; DIE_NODE(ps_spi0): power-controller@2d8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2d8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1462,7 +1463,7 @@ }; DIE_NODE(ps_spi1): power-controller@2e0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2e0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1471,7 +1472,7 @@ }; DIE_NODE(ps_spi2): power-controller@2e8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2e8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1480,7 +1481,7 @@ }; DIE_NODE(ps_spi3): power-controller@2f0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2f0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1489,7 +1490,7 @@ }; DIE_NODE(ps_spi4): power-controller@2f8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2f8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1498,7 +1499,7 @@ }; DIE_NODE(ps_spi5): power-controller@300 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x300 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1507,7 +1508,7 @@ }; DIE_NODE(ps_uart_n): power-controller@308 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x308 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1516,7 +1517,7 @@ }; DIE_NODE(ps_uart0): power-controller@310 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x310 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1525,7 +1526,7 @@ }; DIE_NODE(ps_amcc1): power-controller@318 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x318 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1534,7 +1535,7 @@ }; DIE_NODE(ps_amcc3): power-controller@320 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x320 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1543,7 +1544,7 @@ }; DIE_NODE(ps_dcs_04): power-controller@328 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x328 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1552,7 +1553,7 @@ }; DIE_NODE(ps_dcs_05): power-controller@330 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x330 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1561,7 +1562,7 @@ }; DIE_NODE(ps_dcs_06): power-controller@338 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x338 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1570,7 +1571,7 @@ }; DIE_NODE(ps_dcs_07): power-controller@340 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x340 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1579,7 +1580,7 @@ }; DIE_NODE(ps_dcs_12): power-controller@348 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x348 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1588,7 +1589,7 @@ }; DIE_NODE(ps_dcs_13): power-controller@350 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x350 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1597,7 +1598,7 @@ }; DIE_NODE(ps_dcs_14): power-controller@358 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x358 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1606,7 +1607,7 @@ }; DIE_NODE(ps_dcs_15): power-controller@360 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x360 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1615,7 +1616,7 @@ }; DIE_NODE(ps_uart1): power-controller@368 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x368 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1624,7 +1625,7 @@ }; DIE_NODE(ps_uart2): power-controller@370 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x370 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1633,7 +1634,7 @@ }; DIE_NODE(ps_uart3): power-controller@378 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x378 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1642,7 +1643,7 @@ }; DIE_NODE(ps_uart4): power-controller@380 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x380 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1651,7 +1652,7 @@ }; DIE_NODE(ps_uart5): power-controller@388 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x388 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1660,7 +1661,7 @@ }; DIE_NODE(ps_uart6): power-controller@390 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x390 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1669,43 +1670,47 @@ }; DIE_NODE(ps_mca0): power-controller@398 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x398 4>; #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(mca0); power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + apple,externally-clocked; }; DIE_NODE(ps_mca1): power-controller@3a0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3a0 4>; #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(mca1); power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + apple,externally-clocked; }; DIE_NODE(ps_mca2): power-controller@3a8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3a8 4>; #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(mca2); power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + apple,externally-clocked; }; DIE_NODE(ps_mca3): power-controller@3b0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3b0 4>; #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(mca3); power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + apple,externally-clocked; }; DIE_NODE(ps_dpa0): power-controller@3b8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3b8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1714,7 +1719,7 @@ }; DIE_NODE(ps_dpa1): power-controller@3c0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3c0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1723,7 +1728,7 @@ }; DIE_NODE(ps_dpa2): power-controller@3c8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3c8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1732,7 +1737,7 @@ }; DIE_NODE(ps_dpa3): power-controller@3d0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3d0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1741,7 +1746,7 @@ }; DIE_NODE(ps_msr0): power-controller@3d8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3d8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1749,7 +1754,7 @@ }; DIE_NODE(ps_venc_sys): power-controller@3e0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3e0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1757,7 +1762,7 @@ }; DIE_NODE(ps_dpa4): power-controller@3e8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3e8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1766,7 +1771,7 @@ }; DIE_NODE(ps_msr0_ase_core): power-controller@3f0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3f0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1775,7 +1780,7 @@ }; DIE_NODE(ps_apcie_gpshr_sys): power-controller@3f8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3f8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1784,7 +1789,7 @@ }; DIE_NODE(ps_apcie_st_sys): power-controller@408 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x408 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1793,7 +1798,7 @@ }; DIE_NODE(ps_apcie_st1_sys): power-controller@410 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x410 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1802,7 +1807,7 @@ }; DIE_NODE(ps_apcie_gp_sys): power-controller@418 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x418 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1812,7 +1817,7 @@ }; DIE_NODE(ps_apcie_ge_sys): power-controller@420 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x420 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1821,7 +1826,7 @@ }; DIE_NODE(ps_apcie_phy_sw): power-controller@428 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x428 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1830,7 +1835,7 @@ }; DIE_NODE(ps_sep): power-controller@c00 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xc00 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1847,16 +1852,17 @@ * have to enable/disable everything in the per-model DTs. */ DIE_NODE(ps_isp_cpu): power-controller@4000 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4000 4>; #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(isp_cpu); /* power-domains = <&DIE_NODE(ps_isp_sys)>; */ + apple,force-disable; }; DIE_NODE(ps_isp_fe): power-controller@4008 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4008 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1865,7 +1871,7 @@ }; DIE_NODE(ps_dprx): power-controller@4010 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4010 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1874,7 +1880,7 @@ }; DIE_NODE(ps_isp_vis): power-controller@4018 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4018 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1883,7 +1889,7 @@ }; DIE_NODE(ps_isp_be): power-controller@4020 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4020 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1892,7 +1898,7 @@ }; DIE_NODE(ps_isp_raw): power-controller@4028 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4028 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1901,7 +1907,7 @@ }; DIE_NODE(ps_isp_clr): power-controller@4030 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4030 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1910,7 +1916,7 @@ }; DIE_NODE(ps_venc_dma): power-controller@8000 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x8000 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1919,7 +1925,7 @@ }; DIE_NODE(ps_venc_pipe4): power-controller@8008 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x8008 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1928,7 +1934,7 @@ }; DIE_NODE(ps_venc_pipe5): power-controller@8010 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x8010 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1937,7 +1943,7 @@ }; DIE_NODE(ps_venc_me0): power-controller@8018 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x8018 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1946,7 +1952,7 @@ }; DIE_NODE(ps_venc_me1): power-controller@8020 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x8020 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1955,7 +1961,7 @@ }; DIE_NODE(ps_prores): power-controller@c000 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xc000 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1966,7 +1972,7 @@ &DIE_NODE(pmgr_mini) { DIE_NODE(ps_debug): power-controller@58 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x58 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1975,7 +1981,7 @@ }; DIE_NODE(ps_nub_spmi0): power-controller@60 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x60 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1984,7 +1990,7 @@ }; DIE_NODE(ps_nub_spmi1): power-controller@68 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x68 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1993,7 +1999,7 @@ }; DIE_NODE(ps_nub_aon): power-controller@70 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x70 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2002,7 +2008,7 @@ }; DIE_NODE(ps_msg): power-controller@78 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x78 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2011,7 +2017,7 @@ }; DIE_NODE(ps_nub_gpio): power-controller@80 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x80 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2020,7 +2026,7 @@ }; DIE_NODE(ps_nub_fabric): power-controller@88 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x88 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2029,7 +2035,7 @@ }; DIE_NODE(ps_atc0_usb_aon): power-controller@90 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x90 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2038,7 +2044,7 @@ }; DIE_NODE(ps_atc1_usb_aon): power-controller@98 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x98 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2047,7 +2053,7 @@ }; DIE_NODE(ps_atc2_usb_aon): power-controller@a0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xa0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2056,7 +2062,7 @@ }; DIE_NODE(ps_atc3_usb_aon): power-controller@a8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xa8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2065,7 +2071,7 @@ }; DIE_NODE(ps_mtp_fabric): power-controller@b0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xb0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2076,7 +2082,7 @@ }; DIE_NODE(ps_nub_sram): power-controller@b8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xb8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2085,7 +2091,7 @@ }; DIE_NODE(ps_debug_switch): power-controller@c0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xc0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2094,7 +2100,7 @@ }; DIE_NODE(ps_atc0_usb): power-controller@c8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xc8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2103,7 +2109,7 @@ }; DIE_NODE(ps_atc1_usb): power-controller@d0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xd0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2112,7 +2118,7 @@ }; DIE_NODE(ps_atc2_usb): power-controller@d8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xd8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2121,7 +2127,7 @@ }; DIE_NODE(ps_atc3_usb): power-controller@e0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xe0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2132,7 +2138,7 @@ #if 0 /* MTP stuff is self-managed */ DIE_NODE(ps_mtp_gpio): power-controller@e8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xe8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2142,7 +2148,7 @@ }; DIE_NODE(ps_mtp_base): power-controller@f0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xf0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2152,7 +2158,7 @@ }; DIE_NODE(ps_mtp_periph): power-controller@f8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xf8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2162,7 +2168,7 @@ }; DIE_NODE(ps_mtp_spi0): power-controller@100 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x100 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2172,7 +2178,7 @@ }; DIE_NODE(ps_mtp_i2cm0): power-controller@108 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x108 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2182,7 +2188,7 @@ }; DIE_NODE(ps_mtp_uart0): power-controller@110 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x110 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2192,7 +2198,7 @@ }; DIE_NODE(ps_mtp_cpu): power-controller@118 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x118 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2202,7 +2208,7 @@ }; DIE_NODE(ps_mtp_scm_fabric): power-controller@120 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x120 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2212,7 +2218,7 @@ }; DIE_NODE(ps_mtp_sram): power-controller@128 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x128 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2222,7 +2228,7 @@ }; DIE_NODE(ps_mtp_dma): power-controller@130 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x130 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2235,7 +2241,7 @@ &DIE_NODE(pmgr_gfx) { DIE_NODE(ps_gpx): power-controller@0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2245,7 +2251,7 @@ }; DIE_NODE(ps_afr): power-controller@100 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x100 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2255,7 +2261,7 @@ }; DIE_NODE(ps_gfx): power-controller@108 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x108 4>; #power-domain-cells = <0>; #reset-cells = <0>; diff --git a/arch/arm64/boot/dts/apple/t8103-j274.dts b/arch/arm64/boot/dts/apple/t8103-j274.dts index 1c3e37f86d46d7..f55683c48784b8 100644 --- a/arch/arm64/boot/dts/apple/t8103-j274.dts +++ b/arch/arm64/boot/dts/apple/t8103-j274.dts @@ -15,12 +15,27 @@ / { compatible = "apple,j274", "apple,t8103", "apple,arm-platform"; model = "Apple Mac mini (M1, 2020)"; + chassis-type = "desktop"; aliases { ethernet0 = ðernet0; + sio = &sio; }; }; +&dcp { + apple,connector-type = "HDMI-A"; +}; + +/* remove once m1n1 enables sio nodes after setup */ +&sio { + status = "okay"; +}; + +&dpaudio0 { + status = "okay"; +}; + &bluetooth0 { brcm,board-type = "apple,atlantisb"; }; @@ -29,6 +44,18 @@ brcm,board-type = "apple,atlantisb"; }; +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Back-left"; +}; + +&typec1 { + label = "USB-C Back-right"; +}; + /* * Force the bus number assignments so that we can declare some of the * on-board devices and properties that are populated by the bootloader @@ -58,6 +85,65 @@ status = "okay"; }; +&i2c1 { + speaker_amp: codec@31 { + compatible = "ti,tas5770l", "ti,tas2770"; + reg = <0x31>; + shutdown-gpios = <&pinctrl_ap 181 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + ti,sdout-zero-fill; + }; +}; + &i2c2 { status = "okay"; + + jack_codec: codec@48 { + compatible = "cirrus,cs42l83"; + reg = <0x48>; + reset-gpios = <&pinctrl_nub 11 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <183 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + cirrus,ts-inv = <1>; + sound-name-prefix = "Jack"; + }; }; + +/ { + sound { + compatible = "apple,j274-macaudio", "apple,macaudio"; + model = "Mac mini J274"; + + dai-link@0 { + link-name = "Speaker"; + + cpu { + sound-dai = <&mca 0>; + }; + codec { + sound-dai = <&speaker_amp>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + + }; +}; + +&gpu { + apple,perf-base-pstate = <3>; +}; + +#include "hwmon-mini.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8103-j293.dts b/arch/arm64/boot/dts/apple/t8103-j293.dts index 5b3c42e9f0e677..6fa04626b1d08b 100644 --- a/arch/arm64/boot/dts/apple/t8103-j293.dts +++ b/arch/arm64/boot/dts/apple/t8103-j293.dts @@ -16,6 +16,7 @@ / { compatible = "apple,j293", "apple,t8103", "apple,arm-platform"; model = "Apple MacBook Pro (13-inch, M1, 2020)"; + chassis-type = "laptop"; /* * All of those are used by the bootloader to pass calibration @@ -23,6 +24,7 @@ */ aliases { touchbar0 = &touchbar0; + sep = &sep; }; led-controller { @@ -38,6 +40,20 @@ }; }; +&dcp { + panel: panel { + compatible = "apple,panel-j293", "apple,panel"; + width-mm = <286>; + height-mm = <179>; + apple,max-brightness = <525>; + }; +}; + +&framebuffer0 { + panel = <&panel>; + post-init-providers = <&panel>; +}; + &bluetooth0 { brcm,board-type = "apple,honshu"; }; @@ -46,8 +62,117 @@ brcm,board-type = "apple,honshu"; }; +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Left-back"; +}; + +&typec1 { + label = "USB-C Left-front"; +}; + +&spi3 { + status = "okay"; + + hid-transport@0 { + compatible = "apple,spi-hid-transport"; + reg = <0>; + spi-max-frequency = <8000000>; + /* + * Apple's ADT specifies 20us CS change delays, and the + * SPI HID interface metadata specifies 45us. Using either + * seems not to be reliable, but adding both works, so + * best guess is they are cumulative. + */ + spi-cs-setup-delay-ns = <65000>; + spi-cs-hold-delay-ns = <65000>; + spi-cs-inactive-delay-ns = <250000>; + spien-gpios = <&pinctrl_ap 195 0>; + interrupts-extended = <&pinctrl_nub 13 IRQ_TYPE_LEVEL_LOW>; + }; +}; + +/* Virtual regulator representing the shared shutdown GPIO */ +/ { + speaker_sdz: fixed-regulator-tas5770-sdz { + compatible = "regulator-fixed"; + regulator-name = "tas5770-sdz"; + startup-delay-us = <5000>; + gpios = <&pinctrl_ap 181 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&i2c1 { + speaker_left_rear: codec@31 { + compatible = "ti,tas5770l", "ti,tas2770"; + reg = <0x31>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Rear"; + interrupts-extended = <&pinctrl_ap 182 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <8>; + ti,vmon-slot-no = <10>; + ti,pdm-slot-no = <12>; + }; + + speaker_left_front: codec@32 { + compatible = "ti,tas5770l", "ti,tas2770"; + reg = <0x32>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Front"; + interrupts-extended = <&pinctrl_ap 182 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + ti,pdm-slot-no = <4>; + ti,sdout-pull-down; + }; +}; + &i2c2 { status = "okay"; + + jack_codec: codec@48 { + compatible = "cirrus,cs42l83"; + reg = <0x48>; + reset-gpios = <&pinctrl_nub 11 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <183 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + cirrus,ts-inv = <1>; + sound-name-prefix = "Jack"; + }; +}; + +&i2c3 { + speaker_right_rear: codec@34 { + compatible = "ti,tas5770l", "ti,tas2770"; + reg = <0x34>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Rear"; + interrupts-extended = <&pinctrl_ap 182 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <12>; + ti,vmon-slot-no = <14>; + ti,pdm-slot-no = <16>; + }; + + speaker_right_front: codec@35 { + compatible = "ti,tas5770l", "ti,tas2770"; + reg = <0x35>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Front"; + interrupts-extended = <&pinctrl_ap 182 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <4>; + ti,vmon-slot-no = <6>; + ti,pdm-slot-no = <8>; + ti,sdout-pull-down; + }; }; &i2c4 { @@ -119,3 +244,67 @@ &displaydfr_dart { status = "okay"; }; + +&aop_mbox { + status = "okay"; +}; + +&aop_dart { + status = "okay"; +}; + +&aop_admac { + status = "okay"; +}; + +&aop { + status = "okay"; +}; + +&sep { + status = "okay"; +}; + +&aop_audio { + apple,chassis-name = "J293"; + apple,machine-kind = "MacBook Pro"; +}; + +/ { + sound { + compatible = "apple,j293-macaudio", "apple,macaudio"; + model = "MacBook Pro J293"; + + dai-link@0 { + link-name = "Speakers"; + + cpu { + sound-dai = <&mca 0>, <&mca 1>; + }; + codec { + sound-dai = <&speaker_left_front>, <&speaker_left_rear>, + <&speaker_right_front>, <&speaker_right_rear>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + +#include "isp-imx248.dtsi" + +&isp { + apple,platform-id = <1>; +}; + +#include "hwmon-fan.dtsi" +#include "hwmon-laptop.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8103-j313.dts b/arch/arm64/boot/dts/apple/t8103-j313.dts index 97a4344d8dca68..883ba4a1f0100a 100644 --- a/arch/arm64/boot/dts/apple/t8103-j313.dts +++ b/arch/arm64/boot/dts/apple/t8103-j313.dts @@ -16,6 +16,11 @@ / { compatible = "apple,j313", "apple,t8103", "apple,arm-platform"; model = "Apple MacBook Air (M1, 2020)"; + chassis-type = "laptop"; + + aliases { + sep = &sep; + }; led-controller { compatible = "pwm-leds"; @@ -30,6 +35,20 @@ }; }; +&dcp { + panel: panel { + compatible = "apple,panel-j313", "apple,panel"; + width-mm = <286>; + height-mm = <179>; + apple,max-brightness = <420>; + }; +}; + +&framebuffer0 { + panel = <&panel>; + post-init-providers = <&panel>; +}; + &bluetooth0 { brcm,board-type = "apple,shikoku"; }; @@ -41,3 +60,148 @@ &fpwm1 { status = "okay"; }; + +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Left-back"; +}; + +&typec1 { + label = "USB-C Left-front"; +}; + +&spi3 { + status = "okay"; + + hid-transport@0 { + compatible = "apple,spi-hid-transport"; + reg = <0>; + spi-max-frequency = <8000000>; + /* + * Apple's ADT specifies 20us CS change delays, and the + * SPI HID interface metadata specifies 45us. Using either + * seems not to be reliable, but adding both works, so + * best guess is they are cumulative. + */ + spi-cs-setup-delay-ns = <65000>; + spi-cs-hold-delay-ns = <65000>; + spi-cs-inactive-delay-ns = <250000>; + spien-gpios = <&pinctrl_ap 195 0>; + interrupts-extended = <&pinctrl_nub 13 IRQ_TYPE_LEVEL_LOW>; + }; +}; + +/* Virtual regulator representing the shared shutdown GPIO */ +/ { + speaker_sdz: fixed-regulator-tas5770-sdz { + compatible = "regulator-fixed"; + regulator-name = "tas5770-sdz"; + startup-delay-us = <5000>; + gpios = <&pinctrl_ap 181 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&i2c1 { + speaker_left: codec@31 { + compatible = "ti,tas5770l", "ti,tas2770"; + reg = <0x31>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left"; + interrupts-extended = <&pinctrl_ap 182 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + ti,sdout-zero-fill; + }; +}; + +&i2c3 { + speaker_right: codec@34 { + compatible = "ti,tas5770l", "ti,tas2770"; + reg = <0x34>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right"; + interrupts-extended = <&pinctrl_ap 182 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <4>; + ti,vmon-slot-no = <6>; + ti,sdout-zero-fill; + }; + + jack_codec: codec@48 { + compatible = "cirrus,cs42l83"; + reg = <0x48>; + reset-gpios = <&pinctrl_nub 11 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <183 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + cirrus,ts-inv = <1>; + sound-name-prefix = "Jack"; + }; +}; + +&aop_mbox { + status = "okay"; +}; + +&aop_dart { + status = "okay"; +}; + +&aop_admac { + status = "okay"; +}; + +&aop { + status = "okay"; +}; + +&sep { + status = "okay"; +}; + +&aop_audio { + apple,chassis-name = "J313"; + apple,machine-kind = "MacBook Air"; +}; + +/ { + sound { + compatible = "apple,j313-macaudio", "apple,macaudio"; + model = "MacBook Air J313"; + + dai-link@0 { + link-name = "Speakers"; + + cpu { + sound-dai = <&mca 0>, <&mca 1>; + }; + codec { + sound-dai = <&speaker_left>, <&speaker_right>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + +#include "isp-imx248.dtsi" + +&isp { + apple,platform-id = <1>; +}; + +#include "hwmon-laptop.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8103-j456.dts b/arch/arm64/boot/dts/apple/t8103-j456.dts index 58c8e43789b486..c7da4815fb94c0 100644 --- a/arch/arm64/boot/dts/apple/t8103-j456.dts +++ b/arch/arm64/boot/dts/apple/t8103-j456.dts @@ -15,12 +15,27 @@ / { compatible = "apple,j456", "apple,t8103", "apple,arm-platform"; model = "Apple iMac (24-inch, 4x USB-C, M1, 2021)"; + chassis-type = "all-in-one"; aliases { ethernet0 = ðernet0; }; }; +&dcp { + panel: panel { + compatible = "apple,panel-j456", "apple,panel"; + width-mm = <522>; + height-mm = <294>; + apple,max-brightness = <525>; + }; +}; + +&framebuffer0 { + panel = <&panel>; + post-init-providers = <&panel>; +}; + &bluetooth0 { brcm,board-type = "apple,capri"; }; @@ -47,6 +62,18 @@ }; }; +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Back-right"; +}; + +&typec1 { + label = "USB-C Back-right-middle"; +}; + /* * Force the bus number assignments so that we can declare some of the * on-board devices and properties that are populated by the bootloader @@ -75,3 +102,72 @@ &pcie0_dart_2 { status = "okay"; }; + +&i2c1 { + jack_codec: codec@48 { + compatible = "cirrus,cs42l83"; + reg = <0x48>; + reset-gpios = <&pinctrl_nub 11 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <183 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + cirrus,ts-inv = <1>; + sound-name-prefix = "Jack"; + }; +}; + +&aop_mbox { + status = "okay"; +}; + +&aop_dart { + status = "okay"; +}; + +&aop_admac { + status = "okay"; +}; + +&aop { + status = "okay"; +}; + +&sep { + status = "okay"; +}; + +&aop_audio { + apple,chassis-name = "J456"; + apple,machine-kind = "iMac"; + apple,no-beamforming; +}; + +/ { + sound { + compatible = "apple,j456-macaudio", "apple,macaudio"; + model = "iMac J456"; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + +&gpu { + apple,perf-base-pstate = <3>; +}; + +#include "isp-imx364.dtsi" + +&isp { + apple,platform-id = <2>; +}; + +#include "hwmon-fan-dual.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8103-j457.dts b/arch/arm64/boot/dts/apple/t8103-j457.dts index 7089ccf3ce5566..fc0f28fb1c4367 100644 --- a/arch/arm64/boot/dts/apple/t8103-j457.dts +++ b/arch/arm64/boot/dts/apple/t8103-j457.dts @@ -15,12 +15,27 @@ / { compatible = "apple,j457", "apple,t8103", "apple,arm-platform"; model = "Apple iMac (24-inch, 2x USB-C, M1, 2021)"; + chassis-type = "all-in-one"; aliases { ethernet0 = ðernet0; }; }; +&dcp { + panel: panel { + compatible = "apple,panel-j457", "apple,panel"; + width-mm = <522>; + height-mm = <294>; + apple,max-brightness = <525>; + }; +}; + +&framebuffer0 { + panel = <&panel>; + post-init-providers = <&panel>; +}; + /* * Adjust pcie0's iommu-map to account for the disabled port01. */ @@ -37,6 +52,18 @@ brcm,board-type = "apple,santorini"; }; +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Back-right"; +}; + +&typec1 { + label = "USB-C Back-left"; +}; + /* * Force the bus number assignments so that we can declare some of the * on-board devices and properties that are populated by the bootloader @@ -56,3 +83,72 @@ &pcie0_dart_2 { status = "okay"; }; + +&i2c1 { + jack_codec: codec@48 { + compatible = "cirrus,cs42l83"; + reg = <0x48>; + reset-gpios = <&pinctrl_nub 11 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <183 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + cirrus,ts-inv = <1>; + sound-name-prefix = "Jack"; + }; +}; + +&aop_mbox { + status = "okay"; +}; + +&aop_dart { + status = "okay"; +}; + +&aop_admac { + status = "okay"; +}; + +&aop { + status = "okay"; +}; + +&sep { + status = "okay"; +}; + +&aop_audio { + apple,chassis-name = "J457"; + apple,machine-kind = "iMac"; + apple,no-beamforming; +}; + +/ { + sound { + compatible = "apple,j457-macaudio", "apple,macaudio"; + model = "iMac J457"; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + +&gpu { + apple,perf-base-pstate = <3>; +}; + +#include "isp-imx364.dtsi" + +&isp { + apple,platform-id = <2>; +}; + +#include "hwmon-fan.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi index 0c8206156bfefd..67a57fc507df92 100644 --- a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi @@ -12,9 +12,14 @@ / { aliases { bluetooth0 = &bluetooth0; + dcp = &dcp; + disp0 = &display; + disp0_piodma = &disp0_piodma; serial0 = &serial0; serial2 = &serial2; wifi0 = &wifi0; + atcphy0 = &atcphy0; + atcphy1 = &atcphy1; }; chosen { @@ -27,11 +32,19 @@ framebuffer0: framebuffer@0 { compatible = "apple,simple-framebuffer", "simple-framebuffer"; reg = <0 0 0 0>; /* To be filled by loader */ + power-domains = <&ps_disp0_cpu0>; /* Format properties will be added by loader */ status = "disabled"; }; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + /* To be filled by loader */ + }; + memory@800000000 { device_type = "memory"; reg = <0x8 0 0x2 0>; /* To be filled by loader */ @@ -53,6 +66,29 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <106 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec0: connector { + compatible = "usb-c-connector"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec0_connector_hs: endpoint { + remote-endpoint = <&dwc3_0_hs>; + }; + }; + port@1 { + reg = <1>; + typec0_connector_ss: endpoint { + remote-endpoint = <&atcphy0_typec_lanes>; + }; + }; + }; + }; }; hpm1: usb-pd@3f { @@ -61,6 +97,115 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <106 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec1: connector { + compatible = "usb-c-connector"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec1_connector_hs: endpoint { + remote-endpoint = <&dwc3_1_hs>; + }; + }; + port@1 { + reg = <1>; + typec1_connector_ss: endpoint { + remote-endpoint = <&atcphy1_typec_lanes>; + }; + }; + }; + }; + }; +}; + +/* USB controllers */ +&dwc3_0 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_0_hs: endpoint { + remote-endpoint = <&typec0_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_0_ss: endpoint { + remote-endpoint = <&atcphy0_usb3>; + }; + }; + }; +}; + +&dwc3_1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_1_hs: endpoint { + remote-endpoint = <&typec1_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_1_ss: endpoint { + remote-endpoint = <&atcphy1_usb3>; + }; + }; + }; +}; + +/* Type-C PHYs */ +&atcphy0 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy0_typec_lanes: endpoint { + remote-endpoint = <&typec0_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy0_usb3: endpoint { + remote-endpoint = <&dwc3_0_ss>; + }; + }; + }; +}; + +&atcphy1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy1_typec_lanes: endpoint { + remote-endpoint = <&typec1_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy1_usb3: endpoint { + remote-endpoint = <&dwc3_1_ss>; + }; + }; }; }; @@ -71,6 +216,7 @@ */ &port00 { bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; wifi0: wifi@0,0 { compatible = "pci14e4,4425"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -91,4 +237,6 @@ clock-frequency = <900000000>; }; +#include "hwmon-common.dtsi" + #include "spi1-nvram.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi index c41c57d63997a5..5d3846d44e3578 100644 --- a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi @@ -234,7 +234,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "sio_cpu"; - power-domains = <&ps_sio>; + power-domains = <&ps_sio &ps_uart_p &ps_spi_p &ps_dpa0>; }; ps_fpwm0: power-controller@1d8 { @@ -493,6 +493,7 @@ #reset-cells = <0>; label = "mca0"; power-domains = <&ps_audio_p>, <&ps_sio_adma>; + apple,externally-clocked; }; ps_mca1: power-controller@2c0 { @@ -502,6 +503,7 @@ #reset-cells = <0>; label = "mca1"; power-domains = <&ps_audio_p>, <&ps_sio_adma>; + apple,externally-clocked; }; ps_mca2: power-controller@2c8 { @@ -511,6 +513,7 @@ #reset-cells = <0>; label = "mca2"; power-domains = <&ps_audio_p>, <&ps_sio_adma>; + apple,externally-clocked; }; ps_mca3: power-controller@2d0 { @@ -520,6 +523,7 @@ #reset-cells = <0>; label = "mca3"; power-domains = <&ps_audio_p>, <&ps_sio_adma>; + apple,externally-clocked; }; ps_mca4: power-controller@2d8 { @@ -529,6 +533,7 @@ #reset-cells = <0>; label = "mca4"; power-domains = <&ps_audio_p>, <&ps_sio_adma>; + apple,externally-clocked; }; ps_mca5: power-controller@2e0 { @@ -538,6 +543,7 @@ #reset-cells = <0>; label = "mca5"; power-domains = <&ps_audio_p>, <&ps_sio_adma>; + apple,externally-clocked; }; ps_dpa0: power-controller@2e8 { @@ -645,8 +651,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "disp0_fe"; - power-domains = <&ps_rmx>; - apple,always-on; /* TODO: figure out if we can enable PM here */ + power-domains = <&ps_rmx>, <&ps_pmp>; }; ps_dispext_fe: power-controller@368 { @@ -655,7 +660,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "dispext_fe"; - power-domains = <&ps_rmx>; + power-domains = <&ps_rmx>, <&ps_pmp>; }; ps_dispext_cpu0: power-controller@378 { @@ -717,6 +722,7 @@ #reset-cells = <0>; label = "apcie_gp"; power-domains = <&ps_apcie>; + apple,always-on; /* Breaks things if shut down */ }; ps_ans2: power-controller@3f0 { @@ -733,6 +739,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "gfx"; + power-domains = <&ps_pmp>; }; ps_dcs4: power-controller@320 { @@ -805,6 +812,7 @@ #reset-cells = <0>; label = "isp_sys"; power-domains = <&ps_rmx>; + status = "disabled"; }; ps_venc_sys: power-controller@408 { @@ -1000,9 +1008,125 @@ #reset-cells = <0>; label = "disp0_cpu0"; power-domains = <&ps_disp0_fe>; - apple,always-on; /* TODO: figure out if we can enable PM here */ apple,min-state = <4>; }; + + /* There is a dependency tree involved with these PDs, + * but we do not express it here since the ISP driver + * is supposed to sequence them in the right order anyway + * (and we do not know the exact tree structure). + * + * This also works around spurious parent PD activation + * on machines with ISP disabled (desktops). + */ + ps_isp_set0: power-controller@4000 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set0"; + apple,force-disable; + }; + + ps_isp_set1: power-controller@4008 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set1"; + apple,force-disable; + apple,force-reset; + }; + + ps_isp_set2: power-controller@4010 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set2"; + apple,force-disable; + apple,force-reset; + }; + + ps_isp_fe: power-controller@4018 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_fe"; + }; + + ps_isp_set4: power-controller@4020 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set4"; + }; + + ps_isp_set5: power-controller@4028 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4028 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set5"; + }; + + ps_isp_set6: power-controller@4030 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4030 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set6"; + }; + + ps_isp_set7: power-controller@4038 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4038 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set7"; + }; + + ps_isp_set8: power-controller@4040 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4040 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set8"; + }; + + ps_isp_set9: power-controller@4048 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4048 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set9"; + }; + + ps_isp_set10: power-controller@4050 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4050 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set10"; + }; + + ps_isp_set11: power-controller@4058 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4058 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set11"; + }; + + ps_isp_set12: power-controller@4060 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4060 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set12"; + }; }; &pmgr_mini { @@ -1095,6 +1219,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "msg"; + apple,always-on; /* Core AON device? */ }; ps_atc0_usb_aon: power-controller@88 { @@ -1103,6 +1228,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "atc0_usb_aon"; + apple,always-on; /* Needs to stay on for dwc3 to work */ }; ps_atc1_usb_aon: power-controller@90 { @@ -1111,6 +1237,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "atc1_usb_aon"; + apple,always-on; /* Needs to stay on for dwc3 to work */ }; ps_atc0_usb: power-controller@98 { diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index 8b7b2788796874..d3fc50b8f901b5 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -12,6 +12,7 @@ #include #include #include +#include / { compatible = "apple,t8103", "apple,arm-platform"; @@ -193,26 +194,31 @@ opp-hz = /bits/ 64 <600000000>; opp-level = <1>; clock-latency-ns = <7500>; + opp-microwatt = <47296>; }; opp02 { opp-hz = /bits/ 64 <972000000>; opp-level = <2>; clock-latency-ns = <22000>; + opp-microwatt = <99715>; }; opp03 { opp-hz = /bits/ 64 <1332000000>; opp-level = <3>; clock-latency-ns = <27000>; + opp-microwatt = <188860>; }; opp04 { opp-hz = /bits/ 64 <1704000000>; opp-level = <4>; clock-latency-ns = <33000>; + opp-microwatt = <288891>; }; opp05 { opp-hz = /bits/ 64 <2064000000>; opp-level = <5>; clock-latency-ns = <50000>; + opp-microwatt = <412979>; }; }; @@ -223,83 +229,140 @@ opp-hz = /bits/ 64 <600000000>; opp-level = <1>; clock-latency-ns = <8000>; + opp-microwatt = <290230>; }; opp02 { opp-hz = /bits/ 64 <828000000>; opp-level = <2>; clock-latency-ns = <19000>; + opp-microwatt = <449013>; }; opp03 { opp-hz = /bits/ 64 <1056000000>; opp-level = <3>; clock-latency-ns = <21000>; + opp-microwatt = <647097>; }; opp04 { opp-hz = /bits/ 64 <1284000000>; opp-level = <4>; clock-latency-ns = <23000>; + opp-microwatt = <865620>; }; opp05 { opp-hz = /bits/ 64 <1500000000>; opp-level = <5>; clock-latency-ns = <24000>; + opp-microwatt = <1112838>; }; opp06 { opp-hz = /bits/ 64 <1728000000>; opp-level = <6>; clock-latency-ns = <29000>; + opp-microwatt = <1453271>; }; opp07 { opp-hz = /bits/ 64 <1956000000>; opp-level = <7>; clock-latency-ns = <31000>; + opp-microwatt = <1776667>; }; opp08 { opp-hz = /bits/ 64 <2184000000>; opp-level = <8>; clock-latency-ns = <34000>; + opp-microwatt = <2366690>; }; opp09 { opp-hz = /bits/ 64 <2388000000>; opp-level = <9>; clock-latency-ns = <36000>; + opp-microwatt = <2892193>; }; opp10 { opp-hz = /bits/ 64 <2592000000>; opp-level = <10>; clock-latency-ns = <51000>; + opp-microwatt = <3475417>; }; opp11 { opp-hz = /bits/ 64 <2772000000>; opp-level = <11>; clock-latency-ns = <54000>; + opp-microwatt = <3959410>; }; opp12 { opp-hz = /bits/ 64 <2988000000>; opp-level = <12>; clock-latency-ns = <55000>; + opp-microwatt = <4540620>; }; -#if 0 /* Not available until CPU deep sleep is implemented */ opp13 { opp-hz = /bits/ 64 <3096000000>; opp-level = <13>; clock-latency-ns = <55000>; + opp-microwatt = <4745031>; turbo-mode; }; opp14 { opp-hz = /bits/ 64 <3144000000>; opp-level = <14>; clock-latency-ns = <56000>; + opp-microwatt = <4822390>; turbo-mode; }; opp15 { opp-hz = /bits/ 64 <3204000000>; opp-level = <15>; clock-latency-ns = <56000>; + opp-microwatt = <4951324>; turbo-mode; }; -#endif + }; + + gpu_opp: opp-table-gpu { + compatible = "operating-points-v2"; + + /* + * NOTE: The voltage and power values are device-specific and + * must be filled in by the bootloader. + */ + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <400000>; + opp-microwatt = <0>; + }; + opp01 { + opp-hz = /bits/ 64 <396000000>; + opp-microvolt = <603000>; + opp-microwatt = <3714690>; + }; + opp02 { + opp-hz = /bits/ 64 <528000000>; + opp-microvolt = <640000>; + opp-microwatt = <5083260>; + }; + opp03 { + opp-hz = /bits/ 64 <720000000>; + opp-microvolt = <690000>; + opp-microwatt = <7429380>; + }; + opp04 { + opp-hz = /bits/ 64 <924000000>; + opp-microvolt = <784000>; + opp-microwatt = <11730600>; + }; + opp05 { + opp-hz = /bits/ 64 <1128000000>; + opp-microvolt = <862000>; + opp-microwatt = <17009370>; + }; + opp06 { + opp-hz = /bits/ 64 <1278000000>; + opp-microvolt = <931000>; + opp-microwatt = <19551000>; + }; }; timer { @@ -345,6 +408,22 @@ clock-output-names = "clk_200m"; }; + /* Pixel clock? frequency in Hz (compare: 4K@60 VGA clock 533.250 MHz) */ + clk_disp0: clock-disp0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <533333328>; + clock-output-names = "clk_disp0"; + }; + + /* Pixel clock? frequency in Hz (compare: 4K@60 VGA clock 533.250 MHz) */ + clk_dispext0: clock-dispext0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext0"; + }; + /* * This is a fabulated representation of the input clock * to NCO since we don't know the true clock tree. @@ -373,15 +452,15 @@ }; uat_handoff: uat-handoff { - status = "disabled"; + reg = <0 0 0 0>; }; uat_pagetables: uat-pagetables { - status = "disabled"; + reg = <0 0 0 0>; }; uat_ttbs: uat-ttbs { - status = "disabled"; + reg = <0 0 0 0>; }; }; @@ -392,9 +471,11 @@ ranges; nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; gpu: gpu@206400000 { - compatible = "apple,agx-g13g"; + compatible = "apple,agx-t8103", "apple,agx-g13g"; reg = <0x2 0x6400000 0 0x40000>, <0x2 0x4000000 0 0x1000000>; reg-names = "asc", "sgx"; @@ -406,6 +487,40 @@ "hw-cal-a", "hw-cal-b", "globals"; apple,firmware-abi = <0 0 0>; + + apple,firmware-version = <12 3 0>; + apple,firmware-compat = <12 3 0>; + + operating-points-v2 = <&gpu_opp>; + apple,perf-base-pstate = <1>; + apple,min-sram-microvolt = <850000>; + apple,avg-power-filter-tc-ms = <1000>; + apple,avg-power-ki-only = <7.5>; + apple,avg-power-kp = <4.0>; + apple,avg-power-min-duty-cycle = <40>; + apple,avg-power-target-filter-tc = <125>; + apple,fast-die0-integral-gain = <200.0>; + apple,fast-die0-proportional-gain = <5.0>; + apple,perf-filter-drop-threshold = <0>; + apple,perf-filter-time-constant = <5>; + apple,perf-filter-time-constant2 = <50>; + apple,perf-integral-gain2 = <0.197392>; + apple,perf-integral-min-clamp = <0>; + apple,perf-proportional-gain2 = <6.853981>; + apple,perf-tgt-utilization = <85>; + apple,power-sample-period = <8>; + apple,power-zones = <30000 100 6875>; + apple,ppm-filter-time-constant-ms = <100>; + apple,ppm-ki = <91.5>; + apple,ppm-kp = <6.9>; + apple,pwr-filter-time-constant = <313>; + apple,pwr-integral-gain = <0.0202129>; + apple,pwr-integral-min-clamp = <0>; + apple,pwr-min-duty-cycle = <40>; + apple,pwr-proportional-gain = <5.2831855>; + + apple,core-leak-coef = <1000.0>; + apple,sram-leak-coef = <45.0>; }; agx_mbox: mbox@206408000 { @@ -492,6 +607,146 @@ }; }; + disp0_dart: iommu@231304000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x31304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_disp0_cpu0>; + apple,dma-range = <0x0 0x0 0x0 0xfc000000>; + status = "disabled"; + }; + + dcp_dart: iommu@23130c000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x3130c000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + apple,dma-range = <0xf 0x00000000 0x0 0xfc000000>; + power-domains = <&ps_disp0_cpu0>; + }; + + dcp_mbox: mbox@231c08000 { + compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x31c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_disp0_cpu0>; + resets = <&ps_disp0_cpu0>; + }; + + dcp: dcp@231c00000 { + compatible = "apple,t8103-dcp", "apple,dcp"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 0>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", + "disp-3", "disp-4"; + reg = <0x2 0x31c00000 0x0 0x4000>, + <0x2 0x30000000 0x0 0x3e8000>, + <0x2 0x31320000 0x0 0x4000>, + <0x2 0x31344000 0x0 0x4000>, + <0x2 0x31800000 0x0 0x800000>, + <0x2 0x3b3d0000 0x0 0x4000>; + apple,bw-scratch = <&pmgr_dcp 0 5 0x14>; + apple,bw-doorbell = <&pmgr_dcp 1 6>; + power-domains = <&ps_disp0_cpu0>; + resets = <&ps_disp0_cpu0>; + clocks = <&clk_disp0>; + phandle = <&dcp>; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + disp0_piodma: piodma { + iommus = <&disp0_dart 4>; + phandle = <&disp0_piodma>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dcp_audio: endpoint { + remote-endpoint = <&dpaudio0_dcp>; + }; + }; + }; + }; + + display: display-subsystem { + compatible = "apple,display-subsystem"; + /* disp_dart0 must be 1st since it is locked */ + iommus = <&disp0_dart 0>; + /* generate phandle explicitly for use in loader */ + phandle = <&display>; + }; + + isp_dart0: iommu@22c0e8000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x2c0e8000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + + status = "disabled"; + }; + + isp_dart1: iommu@22c0f4000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x2c0f4000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + + status = "disabled"; + }; + + isp_dart2: iommu@22c0fc000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x2c0fc000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + + status = "disabled"; + }; + + isp: isp@22a000000 { + compatible = "apple,t8103-isp", "apple,isp"; + iommus = <&isp_dart0 0>, <&isp_dart1 0>, <&isp_dart2 0>; + reg-names = "coproc", "mbox", "gpio", "mbox2"; + reg = <0x2 0x2a000000 0x0 0x2000000>, + <0x2 0x2c104000 0x0 0x100>, + <0x2 0x2c104170 0x0 0x100>, + <0x2 0x2c1043f0 0x0 0x100>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>, <&ps_isp_set0>, + <&ps_isp_set1>, <&ps_isp_set2>, <&ps_isp_fe>, + <&ps_isp_set4>, <&ps_isp_set5>, <&ps_isp_set6>, + <&ps_isp_set7>, <&ps_isp_set8>, <&ps_isp_set9>, + <&ps_isp_set10>, <&ps_isp_set11>, + <&ps_isp_set12>; + + apple,dart-vm-size = <0x0 0xa0000000>; + + status = "disabled"; + }; + sio_dart: iommu@235004000 { compatible = "apple,t8103-dart"; reg = <0x2 0x35004000 0x0 0x4000>; @@ -647,6 +902,32 @@ status = "disabled"; }; + sio_mbox: mbox@236408000 { + compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x36408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_sio>; + }; + + sio: sio@236400000 { + compatible = "apple,t8103-sio", "apple,sio"; + reg = <0x2 0x36400000 0x0 0x8000>; + dma-channels = <128>; + #dma-cells = <1>; + mboxes = <&sio_mbox>; + iommus = <&sio_dart 0>; + power-domains = <&ps_sio_cpu>; + resets = <&ps_sio>; /* TODO: verify reset does something */ + status = "disabled"; + }; + admac: dma-controller@238200000 { compatible = "apple,t8103-admac", "apple,admac"; reg = <0x2 0x38200000 0x0 0x34000>; @@ -661,6 +942,48 @@ resets = <&ps_audio_p>; }; + dpaudio0: audio-controller@238330000 { + compatible = "apple,t8103-dpaudio", "apple,dpaudio"; + reg = <0x2 0x38330000 0x0 0x4000>; + dmas = <&sio 0x64>; + dma-names = "tx"; + power-domains = <&ps_dpa0>; + reset-domains = <&ps_dpa0>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dpaudio0_dcp: endpoint { + remote-endpoint = <&dcp_audio>; + }; + }; + }; + }; + + dpaudio1: audio-controller@238334000 { + compatible = "apple,t8103-dpaudio", "apple,dpaudio"; + reg = <0x2 0x38334000 0x0 0x4000>; + dmas = <&sio 0x66>; + dma-names = "tx"; + power-domains = <&ps_dpa1>; + reset-domains = <&ps_dpa1>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dpaudio1_dcp: endpoint { + remote-endpoint = <&dcpext_audio>; + }; + }; + }; + }; + mca: i2s@238400000 { compatible = "apple,t8103-mca", "apple,mca"; reg = <0x2 0x38400000 0x0 0x18000>, @@ -729,6 +1052,14 @@ reg = <0x2 0x3b700000 0 0x14000>; }; + pmgr_dcp: power-management@23b738000 { + reg = <0x2 0x3b738000 0x0 0x1000>, + <0x2 0x3bc3c000 0x0 0x1000>; + reg-names = "dcp-bw-scratch", "dcp-bw-doorbell"; + #apple,bw-scratch-cells = <3>; + #apple,bw-doorbell-cells = <2>; + }; + pinctrl_ap: pinctrl@23c100000 { compatible = "apple,t8103-pinctrl", "apple,pinctrl"; reg = <0x2 0x3c100000 0x0 0x100000>; @@ -909,12 +1240,22 @@ #gpio-cells = <2>; }; + smc_hwmon: hwmon { + compatible = "apple,smc-hwmon"; + }; + smc_reboot: reboot { compatible = "apple,smc-reboot"; nvmem-cells = <&shutdown_flag>, <&boot_stage>, - <&boot_error_count>, <&panic_count>; + <&boot_error_count>, <&panic_count>, <&pm_setting>; nvmem-cell-names = "shutdown_flag", "boot_stage", - "boot_error_count", "panic_count"; + "boot_error_count", "panic_count", "pm_setting"; + }; + + rtc { + compatible = "apple,smc-rtc"; + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; }; }; @@ -952,6 +1293,36 @@ ; }; + sep_dart: iommu@2412c0000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x412c0000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + }; + + sep: sep@242400000 { + compatible = "apple,sep"; + reg = <0x2 0x42400000 0x0 0x6C000>; + mboxes = <&sep_mbox>; + mbox-names = "mbox"; + iommus = <&sep_dart 0>; + status = "disabled"; + }; + + sep_mbox: mbox@242408000 { + compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x42408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + pinctrl_aop: pinctrl@24a820000 { compatible = "apple,t8103-pinctrl", "apple,pinctrl"; reg = <0x2 0x4a820000 0x0 0x4000>; @@ -973,6 +1344,147 @@ ; }; + aop_mbox: mbox@24a408000 { + compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x4a408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + status = "disabled"; + }; + + aop_dart: iommu@24a808000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x4a808000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + }; + + aop_admac: dma-controller@24a980000 { + compatible = "apple,t8103-admac", "apple,admac"; + reg = <0x2 0x4a980000 0x0 0x34000>; + #dma-cells = <1>; + dma-channels = <16>; + interrupts-extended = <0>, + <0>, + <&aic AIC_IRQ 321 IRQ_TYPE_LEVEL_HIGH>, + <0>; + iommus = <&aop_dart 7>; + status = "disabled"; + }; + + aop: aop@24ac00000 { + compatible = "apple,t8103-aop"; + reg = <0x2 0x4ac00000 0x0 0x1e0000>, + <0x2 0x4a400000 0x0 0x6c000>; + mboxes = <&aop_mbox>; + mbox-names = "mbox"; + iommus = <&aop_dart 0>; + + status = "disabled"; + + aop_audio: audio { + compatible = "apple,t8103-aop-audio", "apple,aop-audio"; + dmas = <&aop_admac 1>; + dma-names = "dma"; + }; + + aop_als: als { + compatible = "apple,t8103-aop-als", "apple,aop-als"; + // intentionally empty + }; + + las { + compatible = "apple,t8103-aop-las", "apple,aop-las"; + }; + }; + + dispext0_dart: iommu@271304000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x71304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_dispext_cpu0>; + apple,dma-range = <0x0 0x0 0x0 0xfc000000>; + status = "disabled"; + }; + + dcpext_dart: iommu@27130c000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x7130c000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_dispext_cpu0>; + apple,dma-range = <0xf 0x00000000 0x0 0xfc000000>; + status = "disabled"; + }; + + dcpext_mbox: mbox@271c08000 { + compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x71c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_dispext_cpu0>; + resets = <&ps_dispext_cpu0>; + status = "disabled"; + }; + + dcpext: dcp@271c00000 { + compatible = "apple,t8103-dcpext", "apple,dcpext"; + mboxes = <&dcpext_mbox>; + mbox-names = "mbox"; + iommus = <&dcpext_dart 0>; + phandle = <&dcpext>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", + "disp-3", "disp-4"; + reg = <0x2 0x71c00000 0x0 0x4000>, + <0x2 0x70000000 0x0 0x118000>, + <0x2 0x71320000 0x0 0x4000>, + <0x2 0x71344000 0x0 0x4000>, + <0x2 0x71800000 0x0 0x800000>, + <0x2 0x3b3d0000 0x0 0x4000>; + apple,bw-scratch = <&pmgr_dcp 0 5 0x18>; + apple,bw-doorbell = <&pmgr_dcp 1 6>; + power-domains = <&ps_dispext_cpu0>; + resets = <&ps_dispext_cpu0>; + clocks = <&clk_dispext0>; + status = "disabled"; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + piodma { + iommus = <&dispext0_dart 4>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dcpext_audio: endpoint { + remote-endpoint = <&dpaudio1_dcp>; + }; + }; + }; + }; + ans_mbox: mbox@277408000 { compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; reg = <0x2 0x77408000 0x0 0x4000>; @@ -1007,6 +1519,124 @@ resets = <&ps_ans2>; }; + dwc3_0: usb@382280000 { + compatible = "apple,t8103-dwc3"; + reg = <0x3 0x82280000 0x0 0xcd00>, <0x3 0x8228cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&dwc3_0_dart_0 0>, <&dwc3_0_dart_1 1>; + power-domains = <&ps_atc0_usb>; + resets = <&atcphy0>; + phys = <&atcphy0 PHY_TYPE_USB2>, <&atcphy0 PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + dwc3_0_dart_0: iommu@382f00000 { + compatible = "apple,t8103-dart"; + reg = <0x3 0x82f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&ps_atc0_usb>; + }; + + dwc3_0_dart_1: iommu@382f80000 { + compatible = "apple,t8103-dart"; + reg = <0x3 0x82f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&ps_atc0_usb>; + }; + + atcphy0: phy@383000000 { + compatible = "apple,t8103-atcphy"; + reg = <0x3 0x83000000 0x0 0x4c000>, + <0x3 0x83050000 0x0 0x8000>, + <0x3 0x80000000 0x0 0x4000>, + <0x3 0x82a90000 0x0 0x4000>, + <0x3 0x82a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&ps_atc0_usb>; + }; + + atcphy0_xbar: mux@38304c000 { + compatible = "apple,t8103-display-crossbar"; + reg = <0x3 0x8304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&ps_atc0_usb>; + }; + + dwc3_1: usb@502280000 { + compatible = "apple,t8103-dwc3"; + reg = <0x5 0x02280000 0x0 0xcd00>, <0x5 0x0228cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&dwc3_1_dart_0 0>, <&dwc3_1_dart_1 1>; + power-domains = <&ps_atc1_usb>; + resets = <&atcphy1>; + phys = <&atcphy1 PHY_TYPE_USB2>, <&atcphy1 PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + dwc3_1_dart_0: iommu@502f00000 { + compatible = "apple,t8103-dart"; + reg = <0x5 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&ps_atc1_usb>; + }; + + dwc3_1_dart_1: iommu@502f80000 { + compatible = "apple,t8103-dart"; + reg = <0x5 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&ps_atc1_usb>; + }; + + atcphy1: phy@503000000 { + compatible = "apple,t8103-atcphy"; + reg = <0x5 0x03000000 0x0 0x4c000>, + <0x5 0x03050000 0x0 0x8000>, + <0x5 0x0 0x0 0x4000>, + <0x5 0x02a90000 0x0 0x4000>, + <0x5 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&ps_atc1_usb>; + }; + + atcphy1_xbar: mux@50304c000 { + compatible = "apple,t8103-display-crossbar"; + reg = <0x5 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&ps_atc1_usb>; + }; + pcie0_dart_0: iommu@681008000 { compatible = "apple,t8103-dart"; reg = <0x6 0x81008000 0x0 0x4000>; @@ -1072,6 +1702,8 @@ pinctrl-0 = <&pcie_pins>; pinctrl-names = "default"; + dma-coherent; + port00: pci@0,0 { device_type = "pci"; reg = <0x0 0x0 0x0 0x0 0x0>; diff --git a/arch/arm64/boot/dts/apple/t8112-j413.dts b/arch/arm64/boot/dts/apple/t8112-j413.dts index 6f69658623bf89..20285be747d965 100644 --- a/arch/arm64/boot/dts/apple/t8112-j413.dts +++ b/arch/arm64/boot/dts/apple/t8112-j413.dts @@ -16,9 +16,11 @@ / { compatible = "apple,j413", "apple,t8112", "apple,arm-platform"; model = "Apple MacBook Air (13-inch, M2, 2022)"; + chassis-type = "laptop"; aliases { bluetooth0 = &bluetooth0; + keyboard = &keyboard; wifi0 = &wifi0; }; @@ -35,6 +37,21 @@ }; }; +&dcp { + panel: panel { + compatible = "apple,panel-j413", "apple,panel"; + width-mm = <290>; + height-mm = <189>; + adj-height-mm = <181>; + apple,max-brightness = <525>; + }; +}; + +&framebuffer0 { + panel = <&panel>; + post-init-providers = <&panel>; +}; + /* * Force the bus number assignments so that we can declare some of the * on-board devices and properties that are populated by the bootloader @@ -42,6 +59,7 @@ */ &port00 { bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; wifi0: wifi@0,0 { compatible = "pci14e4,4433"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -60,6 +78,18 @@ }; }; +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Left-back"; +}; + +&typec1 { + label = "USB-C Left-front"; +}; + &i2c0 { /* MagSafe port */ hpm5: usb-pd@3a { @@ -71,6 +101,76 @@ }; }; +/* Virtual regulator representing the shared shutdown GPIO */ +/ { + speaker_sdz: fixed-regulator-sn012776-sdz { + compatible = "regulator-fixed"; + regulator-name = "sn012776-sdz"; + startup-delay-us = <5000>; + gpios = <&pinctrl_ap 88 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&i2c1 { + speaker_left_woof: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Woofer"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + ti,sdout-force-zero-mask = <0xf0f0>; + }; + + speaker_left_tweet: codec@39 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x39>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Tweeter"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <8>; + ti,vmon-slot-no = <10>; + }; +}; + +&i2c3 { + speaker_right_woof: codec@3b { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3b>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Woofer"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <4>; + ti,vmon-slot-no = <6>; + ti,sdout-force-zero-mask = <0x0f0f>; + }; + + speaker_right_tweet: codec@3c { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3c>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Tweeter"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <12>; + ti,vmon-slot-no = <14>; + }; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 12 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupts-extended = <&pinctrl_ap 149 IRQ_TYPE_LEVEL_LOW>; + sound-name-prefix = "Jack"; + }; +}; + &i2c4 { status = "okay"; }; @@ -78,3 +178,98 @@ &fpwm1 { status = "okay"; }; + +&aop_mbox { + status = "okay"; +}; + +&aop_dart { + status = "okay"; +}; + +&aop_admac { + status = "okay"; +}; + +&aop { + status = "okay"; +}; + +&aop_audio { + apple,chassis-name = "J413"; + apple,machine-kind = "MacBook Air"; +}; + +/ { + sound { + compatible = "apple,j413-macaudio", "apple,macaudio"; + model = "MacBook Air J413"; + + dai-link@0 { + link-name = "Speakers"; + + cpu { + sound-dai = <&mca 0>, <&mca 1>; + }; + codec { + sound-dai = <&speaker_left_woof>, <&speaker_left_tweet>, + <&speaker_right_woof>, <&speaker_right_tweet>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + +&mtp { + status = "okay"; +}; +&mtp_mbox { + status = "okay"; +}; +&mtp_dart { + status = "okay"; +}; +&mtp_dockchannel { + status = "okay"; +}; +&mtp_hid { + apple,afe-reset-gpios = <&smc_gpio 8 GPIO_ACTIVE_LOW>; + apple,stm-reset-gpios = <&smc_gpio 24 GPIO_ACTIVE_LOW>; + + multi-touch { + firmware-name = "apple/tpmtfw-j413.bin"; + }; + + keyboard: keyboard { + hid-country-code = <0>; + apple,keyboard-layout-id = <0>; + }; + + stm { + }; + + actuator { + }; + + tp_accel { + }; +}; + +#include "isp-imx558-cfg0.dtsi" + +&isp { + apple,platform-id = <14>; + apple,temporal-filter = <1>; +}; + +#include "hwmon-laptop.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8112-j415.dts b/arch/arm64/boot/dts/apple/t8112-j415.dts index b54e218e5384ca..c2c32ca5577eff 100644 --- a/arch/arm64/boot/dts/apple/t8112-j415.dts +++ b/arch/arm64/boot/dts/apple/t8112-j415.dts @@ -16,9 +16,11 @@ / { compatible = "apple,j415", "apple,t8112", "apple,arm-platform"; model = "Apple MacBook Air (15-inch, M2, 2023)"; + chassis-type = "laptop"; aliases { bluetooth0 = &bluetooth0; + keyboard = &keyboard; wifi0 = &wifi0; }; @@ -35,6 +37,21 @@ }; }; +&dcp { + panel: panel { + compatible = "apple,panel-j415", "apple,panel"; + width-mm = <327>; + height-mm = <211>; + adj-height-mm = <204>; + apple,max-brightness = <500>; + }; +}; + +&framebuffer0 { + panel = <&panel>; + post-init-providers = <&panel>; +}; + /* * Force the bus number assignments so that we can declare some of the * on-board devices and properties that are populated by the bootloader @@ -42,6 +59,7 @@ */ &port00 { bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; wifi0: wifi@0,0 { compatible = "pci14e4,4433"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -60,6 +78,18 @@ }; }; +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Left-back"; +}; + +&typec1 { + label = "USB-C Left-front"; +}; + &i2c0 { /* MagSafe port */ hpm5: usb-pd@3a { @@ -71,6 +101,98 @@ }; }; +/* Virtual regulator representing the shared shutdown GPIO */ +/ { + speaker_sdz: fixed-regulator-sn012776-sdz { + compatible = "regulator-fixed"; + regulator-name = "sn012776-sdz"; + startup-delay-us = <5000>; + gpios = <&pinctrl_ap 88 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&i2c1 { + speaker_left_woof1: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Woofer 1"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + ti,sdout-force-zero-mask = <0xf0f0f0>; + }; + + speaker_left_tweet: codec@39 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x39>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Tweeter"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <8>; + ti,vmon-slot-no = <10>; + }; + + speaker_left_woof2: codec@3a { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3a>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Woofer 2"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <16>; + ti,vmon-slot-no = <18>; + }; +}; + +&i2c3 { + speaker_right_woof1: codec@3b { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3b>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Woofer 1"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <4>; + ti,vmon-slot-no = <6>; + ti,sdout-force-zero-mask = <0x0f0f0f>; + }; + + speaker_right_tweet: codec@3c { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3c>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Tweeter"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <12>; + ti,vmon-slot-no = <14>; + }; + + speaker_right_woof2: codec@3d { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3d>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Woofer 2"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <20>; + ti,vmon-slot-no = <22>; + }; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 12 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupts-extended = <&pinctrl_ap 149 IRQ_TYPE_LEVEL_LOW>; + sound-name-prefix = "Jack"; + }; +}; + &i2c4 { status = "okay"; }; @@ -78,3 +200,102 @@ &fpwm1 { status = "okay"; }; + +&aop_mbox { + status = "okay"; +}; + +&aop_dart { + status = "okay"; +}; + +&aop_admac { + status = "okay"; +}; + +&aop { + status = "okay"; +}; + +&aop_audio { + apple,chassis-name = "J415"; + apple,machine-kind = "MacBook Air"; +}; + +/ { + sound { + compatible = "apple,j415-macaudio", "apple,macaudio"; + model = "MacBook Air J415"; + + dai-link@0 { + link-name = "Speakers"; + + cpu { + sound-dai = <&mca 0>, <&mca 1>; + }; + codec { + sound-dai = <&speaker_left_woof1>, + <&speaker_left_tweet>, + <&speaker_left_woof2>, + <&speaker_right_woof1>, + <&speaker_right_tweet>, + <&speaker_right_woof2>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + +&mtp { + status = "okay"; +}; +&mtp_mbox { + status = "okay"; +}; +&mtp_dart { + status = "okay"; +}; +&mtp_dockchannel { + status = "okay"; +}; +&mtp_hid { + apple,afe-reset-gpios = <&smc_gpio 8 GPIO_ACTIVE_LOW>; + apple,stm-reset-gpios = <&smc_gpio 24 GPIO_ACTIVE_LOW>; + + multi-touch { + firmware-name = "apple/tpmtfw-j415.bin"; + }; + + keyboard: keyboard { + hid-country-code = <0>; + apple,keyboard-layout-id = <0>; + }; + + stm { + }; + + actuator { + }; + + tp_accel { + }; +}; + +#include "isp-imx558-cfg0.dtsi" + +&isp { + apple,platform-id = <15>; + apple,temporal-filter = <1>; +}; + +#include "hwmon-laptop.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8112-j473.dts b/arch/arm64/boot/dts/apple/t8112-j473.dts index 06fe257f08be49..4fc96779806ea3 100644 --- a/arch/arm64/boot/dts/apple/t8112-j473.dts +++ b/arch/arm64/boot/dts/apple/t8112-j473.dts @@ -15,12 +15,85 @@ / { compatible = "apple,j473", "apple,t8112", "apple,arm-platform"; model = "Apple Mac mini (M2, 2023)"; + chassis-type = "desktop"; aliases { + bluetooth0 = &bluetooth0; + /delete-property/ dcp; + dcpext = &dcpext; ethernet0 = ðernet0; + sio = &sio; + wifi0 = &wifi0; }; }; +&framebuffer0 { + power-domains = <&ps_dispext_cpu0>, <&ps_dptx_ext_phy>; +}; + +&dptxphy { + status = "okay"; +}; + +&dcp { + status = "disabled"; +}; + +&display { + iommus = <&dispext0_dart 0>; +}; +&dispext0_dart { + status = "okay"; +}; +&dcpext_dart { + status = "okay"; +}; +&dcpext_mbox { + status = "okay"; +}; +&dcpext { + status = "okay"; + apple,connector-type = "HDMI-A"; + + /* HDMI HPD gpio, used as interrupt*/ + hdmi-hpd-gpios = <&pinctrl_aop 49 GPIO_ACTIVE_HIGH>; + + hdmi-pwren-gpios = <&smc_gpio 21 GPIO_ACTIVE_HIGH>; + dp2hdmi-pwren-gpios = <&smc_gpio 22 GPIO_ACTIVE_HIGH>; + + phys = <&dptxphy>; + phy-names = "dp-phy"; + apple,dptx-phy = <5>; +}; + +/* remove once m1n1 enables sio nodes after setup */ +&sio { + status = "okay"; +}; + +&dpaudio1 { + status = "okay"; +}; + +/* + * Keep the power-domains used for the HDMI port on. + */ +&framebuffer0 { + power-domains = <&ps_dispext_cpu0>, <&ps_dptx_ext_phy>; +}; + +/* + * The M2 Mac mini uses dispext for the HDMI output so it's not necessary to + * keep disp0 power-domains always-on. + */ +&ps_disp0_sys { + /delete-property/ apple,always-on; +}; + +&ps_disp0_fe { + /delete-property/ apple,always-on; +}; + /* * Force the bus number assignments so that we can declare some of the * on-board devices and properties that are populated by the bootloader @@ -28,10 +101,28 @@ */ &port00 { bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; + wifi0: wifi@0,0 { + compatible = "pci14e4,4434"; + reg = <0x10000 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-mac-address = [00 10 18 00 00 10]; + apple,antenna-sku = "XX"; + brcm,board-type = "apple,miyake"; + }; + + bluetooth0: bluetooth@0,1 { + compatible = "pci14e4,5f72"; + reg = <0x10100 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-bd-address = [00 00 00 00 00 00]; + brcm,board-type = "apple,miyake"; + }; }; &port01 { bus-range = <2 2>; + pwren-gpios = <&smc_gpio 24 GPIO_ACTIVE_HIGH>; status = "okay"; }; @@ -52,3 +143,75 @@ &pcie2_dart { status = "okay"; }; + +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Back-left"; +}; + +&typec1 { + label = "USB-C Back-right"; +}; + +&i2c1 { + speaker_amp: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + shutdown-gpios = <&pinctrl_ap 88 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + }; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 12 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <149 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + cirrus,ts-inv = <1>; + sound-name-prefix = "Jack"; + }; +}; + +/ { + sound { + compatible = "apple,j473-macaudio", "apple,macaudio"; + model = "Mac mini J473"; + + dai-link@0 { + link-name = "Speaker"; + + cpu { + sound-dai = <&mca 0>; + }; + codec { + sound-dai = <&speaker_amp>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + + }; +}; + +&gpu { + apple,perf-base-pstate = <3>; +}; + +#include "hwmon-mini.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8112-j493.dts b/arch/arm64/boot/dts/apple/t8112-j493.dts index fb8ad7d4c65a8f..368c4a9cc01758 100644 --- a/arch/arm64/boot/dts/apple/t8112-j493.dts +++ b/arch/arm64/boot/dts/apple/t8112-j493.dts @@ -16,6 +16,7 @@ / { compatible = "apple,j493", "apple,t8112", "apple,arm-platform"; model = "Apple MacBook Pro (13-inch, M2, 2022)"; + chassis-type = "laptop"; /* * All of those are used by the bootloader to pass calibration @@ -23,6 +24,7 @@ */ aliases { bluetooth0 = &bluetooth0; + keyboard = &keyboard; touchbar0 = &touchbar0; wifi0 = &wifi0; }; @@ -50,6 +52,20 @@ apple,always-on; }; +&dcp { + panel: panel { + compatible = "apple,panel-j493", "apple,panel"; + width-mm = <286>; + height-mm = <179>; + apple,max-brightness = <525>; + }; +}; + +&framebuffer0 { + panel = <&panel>; + post-init-providers = <&panel>; +}; + &display_dfr { status = "okay"; }; @@ -90,6 +106,7 @@ */ &port00 { bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; wifi0: wifi@0,0 { compatible = "pci14e4,4425"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -108,6 +125,88 @@ }; }; +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Left-back"; +}; + +&typec1 { + label = "USB-C Left-front"; +}; + +/* Virtual regulator representing the shared shutdown GPIO */ +/ { + speaker_sdz: fixed-regulator-sn012776-sdz { + compatible = "regulator-fixed"; + regulator-name = "sn012776-sdz"; + startup-delay-us = <5000>; + gpios = <&pinctrl_ap 88 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&i2c1 { + speaker_left_rear: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Rear"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <8>; + ti,vmon-slot-no = <10>; + }; + + speaker_left_front: codec@39 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x39>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Front"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + ti,sdout-force-zero-mask = <0xf0f0>; + }; +}; + +&i2c3 { + speaker_right_rear: codec@3b { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3b>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Rear"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <12>; + ti,vmon-slot-no = <14>; + }; + + speaker_right_front: codec@3c { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3c>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Front"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <4>; + ti,vmon-slot-no = <6>; + ti,sdout-force-zero-mask = <0x0f0f>; + }; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 12 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupts-extended = <&pinctrl_ap 149 IRQ_TYPE_LEVEL_LOW>; + sound-name-prefix = "Jack"; + }; +}; + &i2c4 { status = "okay"; }; @@ -133,3 +232,98 @@ touchscreen-inverted-y; }; }; + +&aop_mbox { + status = "okay"; +}; + +&aop_dart { + status = "okay"; +}; + +&aop_admac { + status = "okay"; +}; + +&aop { + status = "okay"; +}; + +&aop_audio { + apple,chassis-name = "J493"; + apple,machine-kind = "MacBook Pro"; +}; + +/ { + sound { + compatible = "apple,j493-macaudio", "apple,macaudio"; + model = "MacBook Pro J493"; + + dai-link@0 { + link-name = "Speakers"; + + cpu { + sound-dai = <&mca 0>, <&mca 1>; + }; + codec { + sound-dai = <&speaker_left_front>, <&speaker_left_rear>, + <&speaker_right_front>, <&speaker_right_rear>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + +&mtp { + status = "okay"; +}; +&mtp_mbox { + status = "okay"; +}; +&mtp_dart { + status = "okay"; +}; +&mtp_dockchannel { + status = "okay"; +}; +&mtp_hid { + apple,afe-reset-gpios = <&smc_gpio 8 GPIO_ACTIVE_LOW>; + apple,stm-reset-gpios = <&smc_gpio 24 GPIO_ACTIVE_LOW>; + + multi-touch { + firmware-name = "apple/tpmtfw-j493.bin"; + }; + + keyboard: keyboard { + hid-country-code = <0>; + apple,keyboard-layout-id = <0>; + }; + + stm { + }; + + actuator { + }; + + tp_accel { + }; +}; + +#include "isp-imx248.dtsi" + +&isp { + apple,platform-id = <6>; +}; + +#include "hwmon-fan.dtsi" +#include "hwmon-laptop.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi index 6da35496a4c88d..fb957f785d82c5 100644 --- a/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi @@ -11,6 +11,11 @@ / { aliases { + atcphy0 = &atcphy0; + atcphy1 = &atcphy1; + dcp = &dcp; + disp0 = &display; + disp0_piodma = &disp0_piodma; serial0 = &serial0; serial2 = &serial2; }; @@ -25,6 +30,7 @@ framebuffer0: framebuffer@0 { compatible = "apple,simple-framebuffer", "simple-framebuffer"; reg = <0 0 0 0>; /* To be filled by loader */ + power-domains = <&ps_disp0_cpu0>; /* Format properties will be added by loader */ status = "disabled"; }; @@ -53,6 +59,29 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <8 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec0: connector { + compatible = "usb-c-connector"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec0_connector_hs: endpoint { + remote-endpoint = <&dwc3_0_hs>; + }; + }; + port@1 { + reg = <1>; + typec0_connector_ss: endpoint { + remote-endpoint = <&atcphy0_typec_lanes>; + }; + }; + }; + }; }; hpm1: usb-pd@3f { @@ -61,6 +90,115 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <8 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec1: connector { + compatible = "usb-c-connector"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec1_connector_hs: endpoint { + remote-endpoint = <&dwc3_1_hs>; + }; + }; + port@1 { + reg = <1>; + typec1_connector_ss: endpoint { + remote-endpoint = <&atcphy1_typec_lanes>; + }; + }; + }; + }; + }; +}; + +/* USB controllers */ +&dwc3_0 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_0_hs: endpoint { + remote-endpoint = <&typec0_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_0_ss: endpoint { + remote-endpoint = <&atcphy0_usb3>; + }; + }; + }; +}; + +&dwc3_1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_1_hs: endpoint { + remote-endpoint = <&typec1_connector_hs>; + }; + }; + + port@1 { + reg = <1>; + dwc3_1_ss: endpoint { + remote-endpoint = <&atcphy1_usb3>; + }; + }; + }; +}; + +/* Type-C PHYs */ +&atcphy0 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy0_typec_lanes: endpoint { + remote-endpoint = <&typec0_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy0_usb3: endpoint { + remote-endpoint = <&dwc3_0_ss>; + }; + }; + }; +}; + +&atcphy1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy1_typec_lanes: endpoint { + remote-endpoint = <&typec1_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy1_usb3: endpoint { + remote-endpoint = <&dwc3_1_ss>; + }; + }; }; }; @@ -80,4 +218,6 @@ clock-frequency = <900000000>; }; +#include "hwmon-common.dtsi" + #include "spi1-nvram.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi index 7c050c6f2707a1..ab8ec9bd4e4401 100644 --- a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi @@ -176,7 +176,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "sio_cpu"; - power-domains = <&ps_sio>; + power-domains = <&ps_sio &ps_uart_p &ps_spi_p &ps_dpa0>; }; ps_fpwm0: power-controller@1c8 { @@ -465,6 +465,7 @@ #reset-cells = <0>; label = "mca0"; power-domains = <&ps_sio_adma>, <&ps_audio_p>; + apple,externally-clocked; }; ps_mca1: power-controller@2c8 { @@ -474,6 +475,7 @@ #reset-cells = <0>; label = "mca1"; power-domains = <&ps_sio_adma>, <&ps_audio_p>; + apple,externally-clocked; }; ps_mca2: power-controller@2d0 { @@ -483,6 +485,7 @@ #reset-cells = <0>; label = "mca2"; power-domains = <&ps_sio_adma>, <&ps_audio_p>; + apple,externally-clocked; }; ps_mca3: power-controller@2d8 { @@ -492,6 +495,7 @@ #reset-cells = <0>; label = "mca3"; power-domains = <&ps_sio_adma>, <&ps_audio_p>; + apple,externally-clocked; }; ps_mca4: power-controller@2e0 { @@ -501,6 +505,7 @@ #reset-cells = <0>; label = "mca4"; power-domains = <&ps_sio_adma>, <&ps_audio_p>; + apple,externally-clocked; }; ps_mca5: power-controller@2e8 { @@ -510,6 +515,7 @@ #reset-cells = <0>; label = "mca5"; power-domains = <&ps_sio_adma>, <&ps_audio_p>; + apple,externally-clocked; }; ps_mcc: power-controller@2f0 { @@ -663,7 +669,6 @@ #reset-cells = <0>; label = "disp0_sys"; power-domains = <&ps_rmx1>; - apple,always-on; /* TODO: figure out if we can enable PM here */ }; ps_disp0_fe: power-controller@378 { @@ -672,8 +677,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "disp0_fe"; - power-domains = <&ps_disp0_sys>; - apple,always-on; /* TODO: figure out if we can enable PM here */ + power-domains = <&ps_disp0_sys>, <&ps_pmp>; }; ps_dispext_sys: power-controller@380 { @@ -691,7 +695,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "dispext_fe"; - power-domains = <&ps_dispext_sys>; + power-domains = <&ps_dispext_sys>, <&ps_pmp>; }; ps_dispext_cpu0: power-controller@3c8 { @@ -773,7 +777,6 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "pmp"; - apple,always-on; }; ps_pms_sram: power-controller@418 { @@ -818,6 +821,7 @@ #reset-cells = <0>; label = "isp_sys"; power-domains = <&ps_rmx1>; + status = "disabled"; }; ps_venc_sys: power-controller@440 { @@ -964,6 +968,123 @@ apple,always-on; }; + /* There is a dependency tree involved with these PDs, + * but we do not express it here since the ISP driver + * is supposed to sequence them in the right order anyway + * (and we do not know the exact tree structure). + * + * This also works around spurious parent PD activation + * on machines with ISP disabled (desktops). + */ + ps_isp_set0: power-controller@4000 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set0"; + apple,force-disable; + }; + + ps_isp_set1: power-controller@4008 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set1"; + apple,force-disable; + apple,force-reset; + }; + + ps_isp_set2: power-controller@4010 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set2"; + apple,force-disable; + apple,force-reset; + }; + + ps_isp_fe: power-controller@4018 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_fe"; + }; + + ps_isp_set4: power-controller@4020 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set4"; + }; + + ps_isp_set5: power-controller@4028 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4028 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set5"; + }; + + ps_isp_set6: power-controller@4030 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4030 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set6"; + }; + + ps_isp_set7: power-controller@4038 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4038 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set7"; + }; + + ps_isp_set8: power-controller@4040 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4040 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set8"; + }; + + ps_isp_set9: power-controller@4048 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4048 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set9"; + }; + + ps_isp_set12: power-controller@4050 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4050 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set10"; + }; + + ps_isp_set10: power-controller@4058 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4058 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set11"; + }; + + ps_isp_set11: power-controller@4060 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4060 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set12"; + }; + ps_venc_dma: power-controller@8000 { compatible = "apple,t8112-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x8000 4>; @@ -1064,6 +1185,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "msg"; + apple,always-on; /* Core AON device? */ }; ps_nub_gpio: power-controller@80 { diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index 3f79878b25af1f..c05d10a47b25fa 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -11,6 +11,7 @@ #include #include #include +#include #include / { @@ -21,6 +22,7 @@ aliases { gpu = &gpu; + pmp = &pmp; }; cpus { @@ -194,36 +196,43 @@ opp-hz = /bits/ 64 <600000000>; opp-level = <1>; clock-latency-ns = <7500>; + opp-microwatt = <26000>; }; opp02 { opp-hz = /bits/ 64 <912000000>; opp-level = <2>; clock-latency-ns = <20000>; + opp-microwatt = <56000>; }; opp03 { opp-hz = /bits/ 64 <1284000000>; opp-level = <3>; clock-latency-ns = <22000>; + opp-microwatt = <88000>; }; opp04 { opp-hz = /bits/ 64 <1752000000>; opp-level = <4>; clock-latency-ns = <30000>; + opp-microwatt = <155000>; }; opp05 { opp-hz = /bits/ 64 <2004000000>; opp-level = <5>; clock-latency-ns = <35000>; + opp-microwatt = <231000>; }; opp06 { opp-hz = /bits/ 64 <2256000000>; opp-level = <6>; clock-latency-ns = <39000>; + opp-microwatt = <254000>; }; opp07 { opp-hz = /bits/ 64 <2424000000>; opp-level = <7>; clock-latency-ns = <53000>; + opp-microwatt = <351000>; }; }; @@ -235,93 +244,161 @@ opp-hz = /bits/ 64 <660000000>; opp-level = <1>; clock-latency-ns = <9000>; + opp-microwatt = <133000>; }; opp02 { opp-hz = /bits/ 64 <924000000>; opp-level = <2>; clock-latency-ns = <19000>; + opp-microwatt = <212000>; }; opp03 { opp-hz = /bits/ 64 <1188000000>; opp-level = <3>; clock-latency-ns = <22000>; + opp-microwatt = <261000>; }; opp04 { opp-hz = /bits/ 64 <1452000000>; opp-level = <4>; clock-latency-ns = <24000>; + opp-microwatt = <345000>; }; opp05 { opp-hz = /bits/ 64 <1704000000>; opp-level = <5>; clock-latency-ns = <26000>; + opp-microwatt = <441000>; }; opp06 { opp-hz = /bits/ 64 <1968000000>; opp-level = <6>; clock-latency-ns = <28000>; + opp-microwatt = <619000>; }; opp07 { opp-hz = /bits/ 64 <2208000000>; opp-level = <7>; clock-latency-ns = <30000>; + opp-microwatt = <740000>; }; opp08 { opp-hz = /bits/ 64 <2400000000>; opp-level = <8>; clock-latency-ns = <33000>; + opp-microwatt = <855000>; }; opp09 { opp-hz = /bits/ 64 <2568000000>; opp-level = <9>; clock-latency-ns = <34000>; + opp-microwatt = <1006000>; }; opp10 { opp-hz = /bits/ 64 <2724000000>; opp-level = <10>; clock-latency-ns = <36000>; + opp-microwatt = <1217000>; }; opp11 { opp-hz = /bits/ 64 <2868000000>; opp-level = <11>; clock-latency-ns = <41000>; + opp-microwatt = <1534000>; }; opp12 { opp-hz = /bits/ 64 <2988000000>; opp-level = <12>; clock-latency-ns = <42000>; + opp-microwatt = <1714000>; }; opp13 { opp-hz = /bits/ 64 <3096000000>; opp-level = <13>; clock-latency-ns = <44000>; + opp-microwatt = <1877000>; }; opp14 { opp-hz = /bits/ 64 <3204000000>; opp-level = <14>; clock-latency-ns = <46000>; + opp-microwatt = <2159000>; }; - /* Not available until CPU deep sleep is implemented */ -#if 0 opp15 { opp-hz = /bits/ 64 <3324000000>; opp-level = <15>; clock-latency-ns = <62000>; + opp-microwatt = <2393000>; turbo-mode; }; opp16 { opp-hz = /bits/ 64 <3408000000>; opp-level = <16>; clock-latency-ns = <62000>; + opp-microwatt = <2497000>; turbo-mode; }; opp17 { opp-hz = /bits/ 64 <3504000000>; opp-level = <17>; clock-latency-ns = <62000>; + opp-microwatt = <2648000>; turbo-mode; }; -#endif + }; + + gpu_opp: opp-table-gpu { + compatible = "operating-points-v2"; + + /* + * NOTE: The voltage and power values are device-specific and + * must be filled in by the bootloader. + */ + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <400000>; + opp-microwatt = <0>; + }; + opp01 { + opp-hz = /bits/ 64 <444000000>; + opp-microvolt = <603000>; + opp-microwatt = <4295000>; + }; + opp02 { + opp-hz = /bits/ 64 <612000000>; + opp-microvolt = <675000>; + opp-microwatt = <6251000>; + }; + opp03 { + opp-hz = /bits/ 64 <808000000>; + opp-microvolt = <710000>; + opp-microwatt = <8625000>; + }; + opp04 { + opp-hz = /bits/ 64 <968000000>; + opp-microvolt = <775000>; + opp-microwatt = <11948000>; + }; + opp05 { + opp-hz = /bits/ 64 <1110000000>; + opp-microvolt = <820000>; + opp-microwatt = <15071000>; + }; + opp06 { + opp-hz = /bits/ 64 <1236000000>; + opp-microvolt = <875000>; + opp-microwatt = <18891000>; + }; + opp07 { + opp-hz = /bits/ 64 <1338000000>; + opp-microvolt = <915000>; + opp-microwatt = <21960000>; + }; + opp08 { + opp-hz = /bits/ 64 <1398000000>; + opp-microvolt = <950000>; + opp-microwatt = <22800000>; + }; }; timer { @@ -370,6 +447,22 @@ clock-output-names = "nco_ref"; }; + /* Pixel clock? frequency in Hz (compare: 4K@60 VGA clock 533.250 MHz) */ + clk_disp0: clock-disp0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <533333328>; + clock-output-names = "clk_disp0"; + }; + + /* Pixel clock? frequency in Hz (compare: 4K@60 VGA clock 533.250 MHz) */ + clk_dispext0: clock-dispext0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext0"; + }; + reserved-memory { #address-cells = <2>; #size-cells = <2>; @@ -388,15 +481,15 @@ }; uat_handoff: uat-handoff { - status = "disabled"; + reg = <0 0 0 0>; }; uat_pagetables: uat-pagetables { - status = "disabled"; + reg = <0 0 0 0>; }; uat_ttbs: uat-ttbs { - status = "disabled"; + reg = <0 0 0 0>; }; }; @@ -407,20 +500,53 @@ ranges; nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; gpu: gpu@206400000 { - compatible = "apple,agx-g14g"; + compatible = "apple,agx-t8112", "apple,agx-g14g"; reg = <0x2 0x6400000 0 0x40000>, <0x2 0x4000000 0 0x1000000>; reg-names = "asc", "sgx"; mboxes = <&agx_mbox>; - power-domains = <&ps_gfx>; + power-domains = <&pmp_report_gfx>; memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>, <&gpu_hw_cal_a>, <&gpu_hw_cal_b>, <&gpu_globals>; memory-region-names = "ttbs", "pagetables", "handoff", "hw-cal-a", "hw-cal-b", "globals"; apple,firmware-abi = <0 0 0>; + apple,firmware-version = <12 4 0>; + apple,firmware-compat = <12 4 0>; + + operating-points-v2 = <&gpu_opp>; + apple,perf-base-pstate = <1>; + apple,min-sram-microvolt = <780000>; + apple,avg-power-filter-tc-ms = <300>; + apple,avg-power-ki-only = <9.375>; + apple,avg-power-kp = <3.22>; + apple,avg-power-min-duty-cycle = <40>; + apple,avg-power-target-filter-tc = <1>; + apple,fast-die0-integral-gain = <200.0>; + apple,fast-die0-proportional-gain = <5.0>; + apple,perf-boost-ce-step = <50>; + apple,perf-boost-min-util = <90>; + apple,perf-filter-drop-threshold = <0>; + apple,perf-filter-time-constant = <5>; + apple,perf-filter-time-constant2 = <200>; + apple,perf-integral-gain = <5.94>; + apple,perf-integral-gain2 = <5.94>; + apple,perf-integral-min-clamp = <0>; + apple,perf-proportional-gain = <14.85>; + apple,perf-proportional-gain2 = <14.85>; + apple,perf-tgt-utilization = <85>; + apple,power-sample-period = <8>; + apple,ppm-filter-time-constant-ms = <34>; + apple,ppm-ki = <205.0>; + apple,ppm-kp = <0.75>; + apple,pwr-min-duty-cycle = <40>; + apple,core-leak-coef = <1920.0>; + apple,sram-leak-coef = <74.0>; }; agx_mbox: mbox@206408000 { @@ -453,7 +579,7 @@ reg = <0x2 0x28200000 0x0 0xc000>, <0x2 0x28400000 0x0 0x4000>; reg-names = "be", "fe"; - power-domains = <&ps_dispdfr_fe>, <&ps_dispdfr_be>; + power-domains = <&pmp_report_dispdfr>, <&ps_dispdfr_be>; interrupt-parent = <&aic>; interrupts = , ; @@ -507,6 +633,140 @@ }; }; + isp_dart0: iommu@22c4a8000 { + compatible = "apple,t8110-dart"; + reg = <0x2 0x2c4a8000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + status = "disabled"; + }; + + isp_dart1: iommu@22c4b4000 { + compatible = "apple,t8110-dart"; + reg = <0x2 0x2c4b4000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + status = "disabled"; + }; + + isp_dart2: iommu@22c4bc000 { + compatible = "apple,t8110-dart"; + reg = <0x2 0x2c4bc000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + status = "disabled"; + }; + + isp: isp@22a000000 { + compatible = "apple,t8112-isp", "apple,isp"; + iommus = <&isp_dart0 0>, <&isp_dart1 0>, <&isp_dart2 0>; + reg-names = "coproc", "mbox", "gpio", "mbox2"; + reg = <0x2 0x2a000000 0x0 0x2000000>, + <0x2 0x2c4c4000 0x0 0x100>, + <0x2 0x2c4c41b0 0x0 0x100>, + <0x2 0x2c4c4430 0x0 0x100>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&pmp_report_isp_sys>, <&ps_isp_set0>, + <&ps_isp_set1>, <&ps_isp_set2>, <&ps_isp_fe>, + <&ps_isp_set4>, <&ps_isp_set5>, <&ps_isp_set6>, + <&ps_isp_set7>, <&ps_isp_set8>, <&ps_isp_set9>, + <&ps_isp_set10>, <&ps_isp_set11>, + <&ps_isp_set12>; + + apple,dart-vm-size = <0x0 0xa0000000>; + status = "disabled"; + }; + + disp0_dart: iommu@231304000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x31304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_disp0_cpu0>; + apple,dma-range = <0x0 0x0 0xf 0xffff0000>; + status = "disabled"; + }; + + dcp_dart: iommu@23130c000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x3130c000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_disp0_cpu0>; + apple,dma-range = <0x8 0x00000000 0x7 0xffff0000>; + }; + + dcp_mbox: mbox@231c08000 { + compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x31c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_disp0_cpu0>; + resets = <&ps_disp0_cpu0>; + }; + + dcp: dcp@231c00000 { + compatible = "apple,t8112-dcp", "apple,dcp"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 5>; + + /* the ADT has 2 additional regs which seems to be unused */ + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x2 0x31c00000 0x0 0x4000>, + <0x2 0x30000000 0x0 0x61c000>, + <0x2 0x31320000 0x0 0x4000>, + <0x2 0x31344000 0x0 0x4000>, + <0x2 0x31800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x5d8>; + power-domains = <&pmp_report_disp0>; + resets = <&ps_disp0_cpu0>; + clocks = <&clk_disp0>; + phandle = <&dcp>; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + disp0_piodma: piodma { + iommus = <&disp0_dart 4>; + phandle = <&disp0_piodma>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dcp_audio: endpoint { + remote-endpoint = <&dpaudio0_dcp>; + }; + }; + }; + }; + + display: display-subsystem { + compatible = "apple,display-subsystem"; + /* disp_dart0 must be 1st since it is locked */ + iommus = <&disp0_dart 0>; + /* generate phandle explicitly for use in loader */ + phandle = <&display>; + }; + sio_dart: iommu@235004000 { compatible = "apple,t8110-dart"; reg = <0x2 0x35004000 0x0 0x4000>; @@ -651,6 +911,32 @@ status = "disabled"; }; + sio_mbox: mbox@236408000 { + compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x36408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_sio_cpu>; + }; + + sio: sio@236400000 { + compatible = "apple,t8112-sio", "apple,sio"; + reg = <0x2 0x36400000 0x0 0x8000>; + dma-channels = <128>; + #dma-cells = <1>; + mboxes = <&sio_mbox>; + iommus = <&sio_dart 0>; + power-domains = <&ps_sio_cpu>; + resets = <&ps_sio>; /* TODO: verify reset does something */ + status = "disabled"; + }; + admac: dma-controller@238200000 { compatible = "apple,t8112-admac", "apple,admac"; reg = <0x2 0x38200000 0x0 0x34000>; @@ -665,6 +951,48 @@ resets = <&ps_audio_p>; }; + dpaudio0: audio-controller@238330000 { + compatible = "apple,t8112-dpaudio", "apple,dpaudio"; + reg = <0x2 0x38330000 0x0 0x4000>; + dmas = <&sio 0x64>; + dma-names = "tx"; + power-domains = <&ps_dpa0>; + reset-domains = <&ps_dpa0>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dpaudio0_dcp: endpoint { + remote-endpoint = <&dcp_audio>; + }; + }; + }; + }; + + dpaudio1: audio-controller@238334000 { + compatible = "apple,t8112-dpaudio", "apple,dpaudio"; + reg = <0x2 0x38334000 0x0 0x4000>; + dmas = <&sio 0x66>; + dma-names = "tx"; + power-domains = <&ps_dpa1>; + reset-domains = <&ps_dpa1>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dpaudio1_dcp: endpoint { + remote-endpoint = <&dcpext_audio>; + }; + }; + }; + }; + mca: i2s@238400000 { compatible = "apple,t8112-mca", "apple,mca"; reg = <0x2 0x38400000 0x0 0x18000>, @@ -736,6 +1064,163 @@ /* child nodes are added in t8103-pmgr.dtsi */ }; + pmp_dart: iommu@23b300000 { + compatible = "apple,t8110-dart"; + reg = <0x2 0x3b300000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_pmp>; + }; + + pmp_report: pmp_report@23b3c0000 { + compatible = "apple,t8112-pmp-v2-report"; + reg = <0x2 0x3b3c0000 0x0 0x20000>; + power-domains = <&ps_pms_sram>; + #address-cells = <1>; + #size-cells = <0>; + + pmp_report_gfx: report@4 { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x4>; + label = "pmp-gfx"; + #power-domain-cells = <0>; + power-domains = <&ps_gfx>; + }; + + pmp_report_ane_sys: report@5 { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x5>; + label = "pmp-ane-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_ane_sys>; + status = "disabled"; + }; + + pmp_report_isp_sys: report@6 { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x6>; + label = "pmp-isp-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_isp_sys>; + }; + + pmp_report_disp0: report@7 { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x7>; + label = "pmp-disp0"; + #power-domain-cells = <0>; + power-domains = <&ps_disp0_cpu0>; + apple,always-on; + }; + + pmp_report_dispext: report@8 { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x8>; + label = "pmp-dispext"; + #power-domain-cells = <0>; + power-domains = <&pmp_report_dispext>; + }; + + pmp_report_venc_sys: report@9 { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x9>; + label = "pmp-venc-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_venc_sys>; + status = "disabled"; + }; + + pmp_report_avd_sys: report@a { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xa>; + label = "pmp-avd-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_avd_sys>; + status = "disabled"; + }; + + pmp_report_msr: report@b { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xb>; + label = "pmp-msr"; + #power-domain-cells = <0>; + power-domains = <&ps_msr>; + status = "disabled"; + }; + + pmp_report_jpg: report@c { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xc>; + label = "pmp-jpg"; + #power-domain-cells = <0>; + power-domains = <&ps_jpg>; + status = "disabled"; + }; + + pmp_report_scodec: report@d { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xd>; + label = "pmp-scodec"; + #power-domain-cells = <0>; + power-domains = <&ps_scodec>; + status = "disabled"; + }; + + pmp_report_dispdfr: report@11 { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x11>; + label = "pmp-dispdfr"; + #power-domain-cells = <0>; + power-domains = <&ps_dispdfr_fe>; + }; + }; + + pmp: pmp@23b500000 { + compatible = "apple,t8112-pmp-v2", "apple,t6000-pmp-v2"; + reg = <0x2 0x3b500000 0x0 0x80000>, + <0x2 0x3bc00000 0x0 0x4000>; + reg-names = "pmp", "asc"; + mboxes = <&pmp_mbox>; + mbox-names = "mbox"; + iommus = <&pmp_dart 0>; + power-domains = <&ps_pmp>; + tunables { + }; + }; + + pmp_mbox: mbox@23bc08000 { + compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x3bc08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_pmp>, <&ps_pms_sram>; + }; + + + pmgr_dcp: power-management@23b3d0000 { + reg = <0x2 0x3b3d0000 0x0 0x4000>; + reg-names = "dcp-bw-scratch"; + #apple,bw-scratch-cells = <3>; + }; + pinctrl_ap: pinctrl@23c100000 { compatible = "apple,t8112-pinctrl", "apple,pinctrl"; reg = <0x2 0x3c100000 0x0 0x100000>; @@ -804,6 +1289,17 @@ }; }; + dptxphy: phy@23c500000 { + compatible = "apple,t8112-dptx-phy", "apple,dptx-phy"; + reg = <0x2 0x3c500000 0x0 0x4000>, + <0x2 0x3c540000 0x0 0xc000>; + reg-names = "core", "dptx"; + power-domains = <&ps_dptx_ext_phy>; + #phy-cells = <0>; + #reset-cells = <0>; + status = "disabled"; /* only used on j473 */ + }; + pinctrl_nub: pinctrl@23d1f0000 { compatible = "apple,t8112-pinctrl", "apple,pinctrl"; reg = <0x2 0x3d1f0000 0x0 0x4000>; @@ -912,12 +1408,22 @@ #gpio-cells = <2>; }; + smc_hwmon: hwmon { + compatible = "apple,smc-hwmon"; + }; + smc_reboot: reboot { compatible = "apple,smc-reboot"; nvmem-cells = <&shutdown_flag>, <&boot_stage>, - <&boot_error_count>, <&panic_count>; + <&boot_error_count>, <&panic_count>, <&pm_setting>; nvmem-cell-names = "shutdown_flag", "boot_stage", - "boot_error_count", "panic_count"; + "boot_error_count", "panic_count", "pm_setting"; + }; + + rtc { + compatible = "apple,smc-rtc"; + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; }; }; @@ -976,6 +1482,250 @@ ; }; + aop_mbox: mbox@24a408000 { + compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x4a408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + status = "disabled"; + }; + + aop_dart: iommu@24a808000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x4a808000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + }; + + aop_admac: dma-controller@24a980000 { + compatible = "apple,t8112-admac", "apple,admac"; + reg = <0x2 0x4a980000 0x0 0x34000>; + #dma-cells = <1>; + dma-channels = <16>; + interrupts-extended = <0>, + <0>, + <&aic AIC_IRQ 359 IRQ_TYPE_LEVEL_HIGH>, + <0>; + iommus = <&aop_dart 10>; + status = "disabled"; + }; + + aop: aop@24ac00000 { + compatible = "apple,t8112-aop"; + reg = <0x2 0x4ac00000 0x0 0x1e0000>, + <0x2 0x4a400000 0x0 0x6c000>; + mboxes = <&aop_mbox>; + mbox-names = "mbox"; + iommus = <&aop_dart 0>; + + status = "disabled"; + + aop_audio: audio { + compatible = "apple,t8112-aop-audio", "apple,aop-audio"; + dmas = <&aop_admac 1>; + dma-names = "dma"; + }; + + aop_als: als { + compatible = "apple,t8112-aop-als", "apple,aop-als"; + // intentionally empty + }; + + las { + compatible = "apple,t8112-aop-las", "apple,aop-las"; + }; + }; + + mtp: mtp@24e400000 { + compatible = "apple,t8112-mtp", "apple,t8112-rtk-helper-asc4", "apple,mtp", "apple,rtk-helper-asc4"; + reg = <0x2 0x4e400000 0x0 0x4000>, + <0x2 0x4ec00000 0x0 0x100000>; + reg-names = "asc", "sram"; + mboxes = <&mtp_mbox>; + iommus = <&mtp_dart 1>; + #helper-cells = <0>; + + status = "disabled"; + }; + + mtp_mbox: mbox@24e408000 { + compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x4e408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + + status = "disabled"; + }; + + mtp_dart: iommu@24e808000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x4e808000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + + status = "disabled"; + }; + + mtp_dockchannel: fifo@24eb14000 { + compatible = "apple,t8112-dockchannel", "apple,dockchannel"; + reg = <0x2 0x4eb14000 0x0 0x4000>; + reg-names = "irq"; + interrupt-parent = <&aic>; + interrupts = ; + + ranges = <0 0x2 0x4eb28000 0x20000>; + nonposted-mmio; + #address-cells = <1>; + #size-cells = <1>; + + interrupt-controller; + #interrupt-cells = <2>; + + status = "disabled"; + + mtp_hid: input@8000 { + compatible = "apple,dockchannel-hid"; + reg = <0x8000 0x4000>, + <0xc000 0x4000>, + <0x0000 0x4000>, + <0x4000 0x4000>; + reg-names = "config", "data", + "rmt-config", "rmt-data"; + iommus = <&mtp_dart 1>; + interrupt-parent = <&mtp_dockchannel>; + interrupts = <2 IRQ_TYPE_LEVEL_HIGH>, + <3 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "tx", "rx"; + + apple,fifo-size = <0x800>; + apple,helper-cpu = <&mtp>; + }; + + }; + + sep_dart: iommu@25d2c0000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x5d2c0000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + }; + + sep: sep@25e400000 { + compatible = "apple,sep"; + reg = <0x2 0x5e400000 0x0 0x6C000>; + mboxes = <&sep_mbox>; + mbox-names = "mbox"; + iommus = <&sep_dart 0>; + status = "disabled"; + }; + + sep_mbox: mbox@25e408000 { + compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x5e408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + dispext0_dart: iommu@271304000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x71304000 0x0 0x4000>; + #iommu-cells = <1>; + apple,dma-range = <0x0 0x0 0xf 0xffff0000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_dispext_cpu0>; + status = "disabled"; + }; + + dcpext_dart: iommu@27130c000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x7130c000 0x0 0x4000>; + #iommu-cells = <1>; + apple,dma-range = <0x8 0x0 0x7 0xffff0000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_dispext_cpu0>; + status = "disabled"; + }; + + dcpext_mbox: mbox@271c08000 { + compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x71c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_dispext_cpu0>; + resets = <&ps_dispext_cpu0>; + status = "disabled"; + }; + + dcpext: dcp@271c00000 { + compatible = "apple,t8112-dcpext", "apple,dcpext"; + mboxes = <&dcpext_mbox>; + mbox-names = "mbox"; + iommus = <&dcpext_dart 5>; + phandle = <&dcpext>; + + /* the ADT has 2 additional regs which seems to be unused */ + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x2 0x71c00000 0x0 0x4000>, + <0x2 0x70000000 0x0 0x61C000>, + <0x2 0x71320000 0x0 0x4000>, + <0x2 0x71344000 0x0 0x4000>, + <0x2 0x71800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x5e0>; + power-domains = <&ps_dispext_cpu0>; + resets = <&ps_dispext_cpu0>; + clocks = <&clk_dispext0>; + apple,dcp-index = <1>; + status = "disabled"; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + piodma { + iommus = <&dispext0_dart 4>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dcpext_audio: endpoint { + remote-endpoint = <&dpaudio1_dcp>; + }; + }; + }; + }; + ans_mbox: mbox@277408000 { compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; reg = <0x2 0x77408000 0x0 0x4000>; @@ -1010,6 +1760,124 @@ resets = <&ps_ans>; }; + dwc3_0: usb@382280000 { + compatible = "apple,t8112-dwc3", "apple,t8103-dwc3"; + reg = <0x3 0x82280000 0x0 0xcd00>, <0x3 0x8228cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&dwc3_0_dart_0 0>, <&dwc3_0_dart_1 1>; + power-domains = <&ps_atc0_usb>; + resets = <&atcphy0>; + phys = <&atcphy0 PHY_TYPE_USB2>, <&atcphy0 PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + dwc3_0_dart_0: iommu@382f00000 { + compatible = "apple,t8110-dart"; + reg = <0x3 0x82f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&ps_atc0_usb>; + }; + + dwc3_0_dart_1: iommu@382f80000 { + compatible = "apple,t8110-dart"; + reg = <0x3 0x82f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&ps_atc0_usb>; + }; + + atcphy0: phy@383000000 { + compatible = "apple,t8112-atcphy", "apple,t8103-atcphy"; + reg = <0x3 0x83000000 0x0 0x4c000>, + <0x3 0x83050000 0x0 0x8000>, + <0x3 0x80000000 0x0 0x4000>, + <0x3 0x82a90000 0x0 0x4000>, + <0x3 0x82a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&ps_atc0_usb>; + }; + + atcphy0_xbar: mux@38304c000 { + compatible = "apple,t8112-display-crossbar", "apple,t8103-display-crossbar"; + reg = <0x3 0x8304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&ps_atc0_usb>; + }; + + dwc3_1: usb@502280000 { + compatible = "apple,t8112-dwc3", "apple,t8103-dwc3"; + reg = <0x5 0x02280000 0x0 0xcd00>, <0x5 0x0228cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&dwc3_1_dart_0 0>, <&dwc3_1_dart_1 1>; + power-domains = <&ps_atc1_usb>; + resets = <&atcphy1>; + phys = <&atcphy1 PHY_TYPE_USB2>, <&atcphy1 PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + dwc3_1_dart_0: iommu@502f00000 { + compatible = "apple,t8110-dart"; + reg = <0x5 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&ps_atc1_usb>; + }; + + dwc3_1_dart_1: iommu@502f80000 { + compatible = "apple,t8110-dart"; + reg = <0x5 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&ps_atc1_usb>; + }; + + atcphy1: phy@503000000 { + compatible = "apple,t8112-atcphy", "apple,t8103-atcphy"; + reg = <0x5 0x03000000 0x0 0x4c000>, + <0x5 0x03050000 0x0 0x8000>, + <0x5 0x0 0x0 0x4000>, + <0x5 0x02a90000 0x0 0x4000>, + <0x5 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&ps_atc1_usb>; + }; + + atcphy1_xbar: mux@50304c000 { + compatible = "apple,t8112-display-crossbar", "apple,t8103-display-crossbar"; + reg = <0x5 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&ps_atc1_usb>; + }; + pcie0_dart: iommu@681008000 { compatible = "apple,t8110-dart"; reg = <0x6 0x81008000 0x0 0x4000>; @@ -1087,6 +1955,8 @@ pinctrl-0 = <&pcie_pins>; pinctrl-names = "default"; + dma-coherent; + port00: pci@0,0 { device_type = "pci"; reg = <0x0 0x0 0x0 0x0 0x0>; diff --git a/arch/arm64/boot/dts/apple/t8122-j433.dts b/arch/arm64/boot/dts/apple/t8122-j433.dts new file mode 100644 index 00000000000000..61fff420571574 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8122-j433.dts @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple iMac (24-inch, 2x USB-C, M3, 2023) + * + * target-type: J433 + * + * Copyright The Asahi Linux Contributors + */ + +/dts-v1/; + +#include "t8122.dtsi" +#include "t8122-usbpd-i2c.dtsi" +#include "t8122-jxxx.dtsi" + +/ { + compatible = "apple,j433", "apple,t8122", "apple,arm-platform"; + model = "Apple iMac (24-inch, 2x USB-C, M3, 2023)"; +}; + + diff --git a/arch/arm64/boot/dts/apple/t8122-j434.dts b/arch/arm64/boot/dts/apple/t8122-j434.dts new file mode 100644 index 00000000000000..16ef556a07037e --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8122-j434.dts @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple iMac (24-inch, 4x USB-C, M3, 2023) + * + * target-type: J434 + * + * Copyright The Asahi Linux Contributors + */ + +/dts-v1/; + +#include "t8122.dtsi" +#include "t8122-usbpd-i2c.dtsi" +#include "t8122-jxxx.dtsi" + +/ { + compatible = "apple,j434", "apple,t8122", "apple,arm-platform"; + model = "Apple iMac (24-inch, 4x USB-C, M3, 2023)"; +}; + + diff --git a/arch/arm64/boot/dts/apple/t8122-j504.dts b/arch/arm64/boot/dts/apple/t8122-j504.dts new file mode 100644 index 00000000000000..464491b55b019c --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8122-j504.dts @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple MacBook Pro (14-inch, M3, 2023) + * + * target-type: J504 + * + * Copyright The Asahi Linux Contributors + */ + +/dts-v1/; + +#include "t8122.dtsi" +#include "t8122-usbpd-i2c.dtsi" +#include "t8122-jxxx.dtsi" +#include + +/ { + compatible = "apple,j504", "apple,t8122", "apple,arm-platform"; + model = "Apple MacBook Pro (14-inch, M3, 2023)"; + + led-controller { + compatible = "pwm-leds"; + led-0 { + pwms = <&fpwm1 0 40000>; + label = "kbd_backlight"; + function = LED_FUNCTION_KBD_BACKLIGHT; + color = ; + max-brightness = <255>; + default-state = "keep"; + }; + }; +}; + +&fpwm1 { + status = "okay"; +}; + diff --git a/arch/arm64/boot/dts/apple/t8122-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8122-jxxx.dtsi new file mode 100644 index 00000000000000..dd85f0c9fb1e11 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8122-jxxx.dtsi @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple M3 MacBook Air/Pro and iMac (M3, 2023/2024) + * + * This file contains parts common to all Apple M3 devices using the t8122. + * + * target-type: J433, J434, J504, J613, J615 + * + * Copyright The Asahi Linux Contributors + */ + +/ { + aliases { + serial0 = &serial0; + }; + + chosen { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + stdout-path = "serial0"; + + framebuffer0: framebuffer@0 { + compatible = "apple,simple-framebuffer", "simple-framebuffer"; + reg = <0 0 0 0>; /* To be filled by loader */ + power-domains = <&ps_disp_cpu>, <&ps_dptx_ext_phy>; + /* Format properties will be added by loader */ + status = "disabled"; + }; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + /* To be filled by loader */ + }; + + memory@800000000 { + device_type = "memory"; + reg = <0x8 0 0x2 0>; /* To be filled by loader */ + }; +}; + +&serial0 { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/apple/t8122-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8122-pmgr.dtsi new file mode 100644 index 00000000000000..e5c5602b85fe66 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8122-pmgr.dtsi @@ -0,0 +1,1165 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * PMGR Power domains for the Apple T8122 "M3" SoC + * + * Copyright The Asahi Linux Contributors + */ + +&pmgr { + ps_ecpm: power-controller@40 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x40 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ecpm"; + }; + + ps_pcpm: power-controller@48 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x48 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pcpm"; + }; + + ps_sbr: power-controller@100 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x100 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "sbr"; + apple,always-on; /* Core device */ + }; + + ps_msg: power-controller@108 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x108 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "msg"; + }; + + ps_aic: power-controller@110 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x110 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aic"; + apple,always-on; /* Core device */ + }; + + ps_dwi: power-controller@118 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x118 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dwi"; + }; + + ps_gpio: power-controller@120 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x120 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "gpio"; + }; + + ps_pms_busif: power-controller@128 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x128 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_busif"; + apple,always-on; /* Core device */ + }; + + ps_pms: power-controller@130 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x130 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms"; + apple,always-on; /* Core device */ + }; + + ps_pms_fpwm0: power-controller@138 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x138 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm0"; + power-domains = <&ps_pms>; + }; + + ps_pms_fpwm1: power-controller@140 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x140 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm1"; + power-domains = <&ps_pms>; + }; + + ps_pms_fpwm2: power-controller@148 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x148 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm2"; + power-domains = <&ps_pms>; + }; + + ps_pms_fpwm3: power-controller@150 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x150 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm3"; + power-domains = <&ps_pms>; + }; + + ps_pms_fpwm4: power-controller@158 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x158 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm4"; + power-domains = <&ps_pms>; + }; + + ps_pms_c1ppt: power-controller@160 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x160 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_c1ppt"; + }; + + ps_soc_rc: power-controller@168 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x168 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "soc_rc"; + }; + + ps_soc_dpe: power-controller@170 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x170 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "soc_dpe"; + apple,always-on; + }; + + ps_pmgr_soc_ocla: power-controller@178 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x178 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pmgr_soc_ocla"; + power-domains = <&ps_pms>; + }; + + ps_ispsens0: power-controller@180 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x180 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ispsens0"; + }; + + ps_ispsens1: power-controller@188 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x188 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ispsens1"; + }; + + ps_ispsens2: power-controller@190 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x190 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ispsens2"; + }; + + ps_ispsens3: power-controller@198 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x198 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ispsens3"; + }; + + ps_aft0: power-controller@1a8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aft0"; + }; + + ps_ioa0: power-controller@1b0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ioa0"; + apple,always-on; + }; + + ps_ap_tmm: power-controller@1b8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ap_tmm"; + }; + + ps_disp_sys: power-controller@1d8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1d8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "disp_sys"; + apple,always-on; /* TODO: figure out if we can enable PM here */ + }; + + ps_gfx: power-controller@1e0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "gfx"; + }; + + ps_isp_sys: power-controller@1e8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_sys"; + }; + + ps_avd_sys: power-controller@1f0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "avd_sys"; + }; + + ps_jpg: power-controller@200 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x200 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "jpg"; + }; + + ps_disp_fe: power-controller@208 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x208 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "disp_fe"; + power-domains = <&ps_disp_sys>; + apple,always-on; /* TODO: figure out if we can enable PM here */ + }; + + ps_sio_cpu: power-controller@210 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x210 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "sio_cpu"; + }; + + ps_fpwm0: power-controller@218 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x218 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fpwm0"; + }; + + ps_fpwm1: power-controller@220 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x220 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fpwm1"; + }; + + ps_fpwm2: power-controller@228 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x228 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fpwm2"; + }; + + ps_i2c0: power-controller@230 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x230 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c0"; + }; + + ps_i2c1: power-controller@238 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x238 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c1"; + }; + + ps_i2c2: power-controller@240 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x240 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c2"; + }; + + ps_i2c3: power-controller@248 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x248 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c3"; + }; + + ps_i2c4: power-controller@250 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x250 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c4"; + }; + + ps_i2c5: power-controller@258 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x258 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c5"; + }; + + ps_i2c6: power-controller@260 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x260 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c6"; + }; + + ps_i2c7: power-controller@268 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x268 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c7"; + }; + + ps_i2c8: power-controller@270 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x270 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c8"; + }; + + ps_spi_p: power-controller@278 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x278 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi_p"; + }; + + ps_uart_p: power-controller@280 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x280 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart_p"; + }; + + ps_audio_p: power-controller@288 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x288 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_p"; + }; + + ps_aes: power-controller@290 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x290 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aes"; + }; + + ps_spi0: power-controller@298 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x298 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi0"; + power-domains = <&ps_spi_p>; + }; + + ps_spi1: power-controller@2a0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi1"; + power-domains = <&ps_spi_p>; + }; + + ps_spi2: power-controller@2a8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi2"; + power-domains = <&ps_spi_p>; + }; + + ps_spi3: power-controller@2b0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi3"; + power-domains = <&ps_spi_p>; + }; + + ps_spi4: power-controller@2b8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi4"; + power-domains = <&ps_spi_p>; + }; + + ps_spi5: power-controller@2c0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi5"; + power-domains = <&ps_spi_p>; + }; + + ps_qspi: power-controller@2c8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2c8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "qspi"; + power-domains = <&ps_spi_p>; + }; + + ps_uart_n: power-controller@2d0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart_n"; + power-domains = <&ps_uart_p>; + }; + + ps_uart0: power-controller@2d8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2d8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart0"; + power-domains = <&ps_uart_p>; + }; + + ps_uart1: power-controller@2e0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart1"; + power-domains = <&ps_uart_p>; + }; + + ps_uart2: power-controller@2e8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart2"; + power-domains = <&ps_uart_p>; + }; + + ps_uart3: power-controller@2f0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart3"; + power-domains = <&ps_uart_p>; + }; + + ps_uart4: power-controller@2f8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart4"; + power-domains = <&ps_uart_p>; + }; + + ps_uart5: power-controller@300 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x300 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart5"; + power-domains = <&ps_uart_p>; + }; + + ps_uart6: power-controller@308 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x308 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart6"; + power-domains = <&ps_uart_p>; + }; + + ps_sio_adma: power-controller@310 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x310 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "sio_adma"; + power-domains = <&ps_fpwm0>; + }; + + ps_dpa0: power-controller@318 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x318 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa0"; + power-domains = <&ps_audio_p>; + }; + + ps_dcs0: power-controller@330 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x330 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs0"; + apple,always-on; + }; + + ps_dcs2: power-controller@338 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x338 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs2"; + apple,always-on; + }; + + ps_dcs1: power-controller@340 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x340 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs1"; + apple,always-on; + }; + + ps_dcs3: power-controller@348 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x348 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs3"; + apple,always-on; + }; + + ps_dcs4: power-controller@358 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x358 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs4"; + apple,always-on; + }; + + ps_dcs5: power-controller@360 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x360 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs5"; + apple,always-on; + }; + + ps_dcs6: power-controller@368 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x368 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs6"; + apple,always-on; + }; + + ps_dcs7: power-controller@370 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x370 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs7"; + apple,always-on; + }; + + ps_dpa1: power-controller@378 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x378 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa1"; + power-domains = <&ps_audio_p>; + }; + + ps_dpa2: power-controller@380 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x380 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa2"; + power-domains = <&ps_audio_p>; + }; + + ps_dpa3: power-controller@388 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x388 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa3"; + power-domains = <&ps_audio_p>; + }; + + ps_dpa4: power-controller@390 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x390 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa4"; + power-domains = <&ps_audio_p>; + }; + + ps_mca0: power-controller@398 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x398 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "mca0"; + power-domains = <&ps_sio_adma>, <&ps_audio_p>; + }; + + ps_mca1: power-controller@3a0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "mca1"; + power-domains = <&ps_sio_adma>, <&ps_audio_p>; + }; + + ps_mca2: power-controller@3a8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "mca2"; + power-domains = <&ps_sio_adma>, <&ps_audio_p>; + }; + + ps_trace_fab: power-controller@3b0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "trace_fab"; + }; + + ps_mca3: power-controller@3b8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "mca3"; + power-domains = <&ps_sio_adma>, <&ps_audio_p>; + }; + + ps_ioa1: power-controller@3c0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ioa1"; + apple,always-on; + }; + + ps_apcie: power-controller@3f0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie"; + }; + + ps_ans: power-controller@3f8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3f8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ans"; + }; + + ps_atc0_common: power-controller@400 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x400 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_common"; + }; + + ps_atc1_common: power-controller@408 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x408 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1_common"; + }; + + ps_dispext_sys: power-controller@410 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x410 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext_sys"; + }; + + ps_venc_sys: power-controller@418 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x418 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc_sys"; + }; + + ps_scodec: power-controller@420 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x420 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "scodec"; + }; + + ps_msr: power-controller@428 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x428 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "msr"; + power-domains = <&ps_aft0>; + }; + + ps_dptx_ext_phy: power-controller@430 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x430 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dptx_ext_phy"; + }; + + ps_ane_sys: power-controller@438 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x438 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_sys"; + }; + + ps_apcie_gp: power-controller@440 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x440 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_gp"; + power-domains = <&ps_apcie>; + }; + + ps_apcie_st: power-controller@448 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x448 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_st"; + power-domains = <&ps_ans>, <&ps_apcie>; + }; + + ps_pmp: power-controller@450 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x450 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pmp"; + apple,always-on; + }; + + ps_pms_sram: power-controller@458 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x458 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_sram"; + apple,always-on; + }; + + ps_atc0_pcie: power-controller@460 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x460 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_pcie"; + power-domains = <&ps_atc0_common>; + }; + + ps_atc0_cio: power-controller@468 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x468 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_cio"; + power-domains = <&ps_atc0_common>; + }; + + ps_atc1_pcie: power-controller@470 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x470 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1_pcie"; + power-domains = <&ps_atc1_common>; + }; + + ps_atc1_cio: power-controller@478 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x478 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1_cio"; + power-domains = <&ps_atc1_common>; + }; + + ps_dispext_fe: power-controller@480 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x480 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext_fe"; + power-domains = <&ps_dispext_sys>; + }; + + ps_dispext_cpu: power-controller@488 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x488 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext_cpu"; + power-domains = <&ps_dispext_fe>; + apple,min-state = <4>; + }; + + ps_scodec_stream: power-controller@490 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x490 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "scodec_stream"; + power-domains = <&ps_scodec>; + }; + + ps_msr_ase_core: power-controller@498 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x498 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "msr_ase_core"; + power-domains = <&ps_msr>; + }; + + ps_apcie_phy_sw: power-controller@4a0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_phy_sw"; + power-domains = <&ps_apcie_st>, <&ps_apcie_gp>; + }; + + ps_atc0_cio_pcie: power-controller@4a8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_cio_pcie"; + power-domains = <&ps_atc0_cio>; + }; + + ps_atc0_cio_usb: power-controller@4b0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_cio_usb"; + power-domains = <&ps_atc0_cio>; + }; + + ps_atc1_cio_pcie: power-controller@4b8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1_cio_pcie"; + power-domains = <&ps_atc1_cio>; + }; + + ps_atc1_cio_usb: power-controller@4c0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1_cio_usb"; + power-domains = <&ps_atc1_cio>; + }; + + ps_sep: power-controller@c00 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xc00 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "sep"; + apple,always-on; + }; + + ps_venc_dma: power-controller@8000 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc_dma"; + power-domains = <&ps_venc_sys>; + }; + + ps_venc_pipe4: power-controller@8008 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc_pipe4"; + power-domains = <&ps_venc_dma>; + }; + + ps_venc_pipe5: power-controller@8010 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc_pipe5"; + power-domains = <&ps_venc_dma>; + }; + + ps_venc_me0: power-controller@8018 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc_me0"; + power-domains = <&ps_venc_dma>; + }; + + ps_venc_me1: power-controller@8020 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc_me1"; + power-domains = <&ps_venc_me0>; + }; + + ps_disp_cpu: power-controller@10000 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x10000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "disp_cpu"; + power-domains = <&ps_disp_fe>; + apple,min-state = <4>; + }; +}; + +&pmgr_mini { + + ps_debug_gated: power-controller@0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "debug_gated"; + apple,always-on; + }; + + ps_nub_spmi0: power-controller@58 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x58 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi0"; + apple,always-on; + }; + + ps_nub_spmi1: power-controller@60 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x60 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi1"; + apple,always-on; + }; + + ps_nub_spmi2: power-controller@68 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x68 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi2"; + apple,always-on; + }; + + ps_nub_spmi_a0: power-controller@70 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x70 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi_a0"; + apple,always-on; + }; + + ps_nub_aon: power-controller@78 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x78 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_aon"; + apple,always-on; + }; + + ps_nub_spi0: power-controller@80 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x80 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spi0"; + apple,always-on; + }; + + ps_nub_ocla: power-controller@88 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x88 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_ocla"; + apple,always-on; + }; + + ps_nub_gpio: power-controller@90 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x90 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_gpio"; + apple,always-on; + }; + + ps_nub_fabric: power-controller@98 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x98 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_fabric"; + apple,always-on; + }; + + ps_nub_sram: power-controller@a0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xa0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_sram"; + apple,always-on; + }; + + ps_debug_switch: power-controller@a8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xa8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "debug_switch"; + apple,always-on; + }; + + ps_atc0_usb_aon: power-controller@b0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xb0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_usb_aon"; + }; + + ps_atc1_usb_aon: power-controller@b8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xb8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1_usb_aon"; + }; + + ps_atc0_usb: power-controller@c0 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xc0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_usb"; + power-domains = <&ps_atc0_usb_aon>, <&ps_atc0_common>; + }; + + ps_atc1_usb: power-controller@c8 { + compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xc8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1_usb"; + power-domains = <&ps_atc1_usb_aon>, <&ps_atc1_common>; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8122-usbpd-i2c.dtsi b/arch/arm64/boot/dts/apple/t8122-usbpd-i2c.dtsi new file mode 100644 index 00000000000000..112c5199cabdd4 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8122-usbpd-i2c.dtsi @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple M3 MacBook Pro and iMac (M3, 2023) I2C based USB PD controller nodes + * + * This file contains nodes for t8122 devices using I2C based cd321x USB Type-C + * port controllers. The are used in the M3 MacBook Pro and iMacs but not in the + * M3 Macbook Airs. + * + * target-type: J433, J434, J504 + * + * Copyright The Asahi Linux Contributors + */ + +&i2c0 { + status = "okay"; + + hpm0: usb-pd@38 { + compatible = "apple,cd321x"; + reg = <0x38>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <8 IRQ_TYPE_LEVEL_LOW>; + interrupt-names = "irq"; + }; + + hpm1: usb-pd@3f { + compatible = "apple,cd321x"; + reg = <0x3f>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <8 IRQ_TYPE_LEVEL_LOW>; + interrupt-names = "irq"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8122.dtsi b/arch/arm64/boot/dts/apple/t8122.dtsi new file mode 100644 index 00000000000000..2e2cf30f3cd510 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8122.dtsi @@ -0,0 +1,451 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple T8122 "M3" SoC + * + * Other names: H15G + * + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include + +/ { + compatible = "apple,t8122", "apple,arm-platform"; + + #address-cells = <2>; + #size-cells = <2>; + + cpus { + #address-cells = <2>; + #size-cells = <0>; + + cpu-map { + cluster0 { + core0 { + cpu = <&cpu_e0>; + }; + core1 { + cpu = <&cpu_e1>; + }; + core2 { + cpu = <&cpu_e2>; + }; + core3 { + cpu = <&cpu_e3>; + }; + }; + + cluster1 { + core0 { + cpu = <&cpu_p0>; + }; + core1 { + cpu = <&cpu_p1>; + }; + core2 { + cpu = <&cpu_p2>; + }; + core3 { + cpu = <&cpu_p3>; + }; + }; + }; + + cpu_e0: cpu@0 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x0>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + }; + + cpu_e1: cpu@1 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x1>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + }; + + cpu_e2: cpu@2 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x2>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + }; + + cpu_e3: cpu@3 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x3>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + }; + + cpu_p0: cpu@10100 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10100>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p1: cpu@10101 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10101>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p2: cpu@10102 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10102>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p3: cpu@10103 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10103>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + l2_cache_0: l2-cache-0 { + compatible = "cache"; + cache-level = <2>; + cache-unified; + cache-size = <0x400000>; + }; + + l2_cache_1: l2-cache-1 { + compatible = "cache"; + cache-level = <2>; + cache-unified; + cache-size = <0x1000000>; + }; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupt-parent = <&aic>; + interrupt-names = "phys", "virt", "hyp-phys", "hyp-virt"; + interrupts = , + , + , + ; + }; + + clkref: clock-ref { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <24000000>; + clock-output-names = "clkref"; + }; + + // clk_200m: clock-200m { + // compatible = "fixed-clock"; + // #clock-cells = <0>; + // clock-frequency = <200000000>; + // clock-output-names = "clk_200m"; + // }; + + soc { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + + ranges; + nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; + + i2c0: i2c@235010000 { + compatible = "apple,t8122-i2c", "apple,t8103-i2c"; + reg = <0x2 0x35010000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c0_pins>; + pinctrl-names = "default"; + #address-cells = <0x1>; + #size-cells = <0x0>; + power-domains = <&ps_i2c0>; + status = "disabled"; + }; + + i2c1: i2c@235014000 { + compatible = "apple,t8122-i2c", "apple,t8103-i2c"; + reg = <0x2 0x35014000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c1_pins>; + pinctrl-names = "default"; + #address-cells = <0x1>; + #size-cells = <0x0>; + power-domains = <&ps_i2c1>; + status = "disabled"; + }; + + i2c2: i2c@235018000 { + compatible = "apple,t8122-i2c", "apple,t8103-i2c"; + reg = <0x2 0x35018000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c2_pins>; + pinctrl-names = "default"; + #address-cells = <0x1>; + #size-cells = <0x0>; + power-domains = <&ps_i2c2>; + status = "disabled"; + }; + + i2c3: i2c@23501c000 { + compatible = "apple,t8122-i2c", "apple,t8103-i2c"; + reg = <0x2 0x3501c000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c3_pins>; + pinctrl-names = "default"; + #address-cells = <0x1>; + #size-cells = <0x0>; + power-domains = <&ps_i2c3>; + status = "disabled"; + }; + + i2c4: i2c@235020000 { + compatible = "apple,t8122-i2c", "apple,t8103-i2c"; + reg = <0x2 0x35020000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c4_pins>; + pinctrl-names = "default"; + #address-cells = <0x1>; + #size-cells = <0x0>; + power-domains = <&ps_i2c4>; + status = "disabled"; + }; + + fpwm1: pwm@2a1044000 { + compatible = "apple,t8122-fpwm", "apple,s5l-fpwm"; + reg = <0x2 0xa1044000 0x0 0x4000>; + power-domains = <&ps_fpwm1>; + clocks = <&clkref>; + #pwm-cells = <2>; + status = "disabled"; + }; + + serial0: serial@2a1200000 { + compatible = "apple,s5l-uart"; + reg = <0x2 0xa1200000 0x0 0x1000>; + reg-io-width = <4>; + interrupt-parent = <&aic>; + interrupts = ; + /* + * TODO: figure out the clocking properly, there may + * be a third selectable clock. + */ + clocks = <&clkref>, <&clkref>; + clock-names = "uart", "clk_uart_baud0"; + power-domains = <&ps_uart0>; + status = "disabled"; + }; + + aic: interrupt-controller@2d1000000 { + compatible = "apple,t8122-aic3"; + #interrupt-cells = <3>; + interrupt-controller; + reg = <0x2 0xd1000000 0x0 0x184000>, + <0x2 0xd1040000 0x0 0x4>; + reg-names = "core", "event"; + power-domains = <&ps_aic>; + + affinities { + e-core-pmu-affinity { + apple,fiq-index = ; + cpus = <&cpu_e0 &cpu_e1 &cpu_e2 &cpu_e3>; + }; + + p-core-pmu-affinity { + apple,fiq-index = ; + cpus = <&cpu_p0 &cpu_p1 &cpu_p2 &cpu_p3>; + }; + }; + }; + + pmgr: power-management@2d0700000 { + compatible = "apple,t8122-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x2 0xd0700000 0 0x10000>; + /* child nodes are added in t8122-pmgr.dtsi */ + }; + + pinctrl_ap: pinctrl@2c7100000 { + compatible = "apple,t8122-pinctrl", "apple,t8103-pinctrl"; + reg = <0x2 0xc7100000 0x0 0x100000>; + power-domains = <&ps_gpio>; + + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_ap 0 0 224>; + apple,npins = <224>; + + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + + i2c0_pins: i2c0-pins { + pinmux = , + ; + }; + + i2c1_pins: i2c1-pins { + pinmux = , + ; + }; + + i2c2_pins: i2c2-pins { + pinmux = , + ; + }; + + i2c3_pins: i2c3-pins { + pinmux = , + ; + }; + + i2c4_pins: i2c4-pins { + pinmux = , + ; + }; + + }; + + pinctrl_nub: pinctrl@2e41f0000 { + compatible = "apple,t8122-pinctrl", "apple,t8103-pinctrl"; + reg = <0x2 0xe41f0000 0x0 0x4000>; + power-domains = <&ps_nub_gpio>; + + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_nub 0 0 32>; + apple,npins = <32>; + + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + }; + + pmgr_mini: power-management@2e4280000 { + compatible = "apple,t8122-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x2 0xe4280000 0 0x4000>; + /* child nodes are added in t8122-pmgr.dtsi */ + }; + + wdt: watchdog@2e42b0000 { + compatible = "apple,t8122-wdt", "apple,t8103-wdt"; + reg = <0x2 0xe42b0000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + }; + + pinctrl_smc: pinctrl@2ec820000 { + compatible = "apple,t8122-pinctrl", "apple,t8103-pinctrl"; + reg = <0x2 0xec820000 0x0 0x4000>; + + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_smc 0 0 18>; + apple,npins = <18>; + + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + }; + + pinctrl_aop: pinctrl@2f4824000 { + compatible = "apple,t8122-pinctrl", "apple,t8103-pinctrl"; + reg = <0x2 0xf4824000 0x0 0x4000>; + + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_aop 0 0 54>; + apple,npins = <54>; + + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + }; + }; +}; + +#include "t8122-pmgr.dtsi" diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mp-ras314.dts b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mp-ras314.dts index f7346b3d35fe53..a122f2ed5f5314 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mp-ras314.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mp-ras314.dts @@ -704,7 +704,7 @@ fsl,pins = , , , - ; + ; }; pinctrl_gpt1: gpt1grp { diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts index 59642a8a2c445d..c73d40fb789f69 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts @@ -867,7 +867,7 @@ fsl,pins = , , , - ; + ; }; pinctrl_hoggpio2: hoggpio2grp { diff --git a/arch/arm64/boot/dts/freescale/imx95-clock.h b/arch/arm64/boot/dts/freescale/imx95-clock.h index e1f91203e79470..22311612e44033 100644 --- a/arch/arm64/boot/dts/freescale/imx95-clock.h +++ b/arch/arm64/boot/dts/freescale/imx95-clock.h @@ -183,5 +183,6 @@ #define IMX95_CLK_SEL_A55P (IMX95_CCM_NUM_CLK_SRC + 123 + 7) #define IMX95_CLK_SEL_DRAM (IMX95_CCM_NUM_CLK_SRC + 123 + 8) #define IMX95_CLK_SEL_TEMPSENSE (IMX95_CCM_NUM_CLK_SRC + 123 + 9) +#define IMX95_CLK_GPU_CGC (IMX95_CCM_NUM_CLK_SRC + 123 + 10) #endif /* __CLOCK_IMX95_H */ diff --git a/arch/arm64/boot/dts/freescale/imx95.dtsi b/arch/arm64/boot/dts/freescale/imx95.dtsi index a4d85481755947..55e2da094c889f 100644 --- a/arch/arm64/boot/dts/freescale/imx95.dtsi +++ b/arch/arm64/boot/dts/freescale/imx95.dtsi @@ -2164,7 +2164,7 @@ gpu: gpu@4d900000 { compatible = "nxp,imx95-mali", "arm,mali-valhall-csf"; reg = <0 0x4d900000 0 0x480000>; - clocks = <&scmi_clk IMX95_CLK_GPU>, <&scmi_clk IMX95_CLK_GPUAPB>; + clocks = <&scmi_clk IMX95_CLK_GPU_CGC>, <&scmi_clk IMX95_CLK_GPUAPB>; clock-names = "core", "coregroup"; interrupts = , , diff --git a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-pico6.dts b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-pico6.dts index cce326aec1aa59..40af5656d6f15f 100644 --- a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-pico6.dts +++ b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-pico6.dts @@ -91,7 +91,7 @@ &pio { bt_pins_wakeup: bt-pins-wakeup { - piins-bt-wakeup { + pins-bt-wakeup { pinmux = ; input-enable; }; diff --git a/arch/arm64/boot/dts/mediatek/mt8183.dtsi b/arch/arm64/boot/dts/mediatek/mt8183.dtsi index 4e20a8f2eb2580..95cc067995336c 100644 --- a/arch/arm64/boot/dts/mediatek/mt8183.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8183.dtsi @@ -1812,15 +1812,23 @@ #size-cells = <0>; port@0 { + #address-cells = <1>; + #size-cells = <0>; reg = <0>; - ovl_2l1_in: endpoint { + + ovl_2l1_in: endpoint@1 { + reg = <1>; remote-endpoint = <&mmsys_ep_ext>; }; }; port@1 { + #address-cells = <1>; + #size-cells = <0>; reg = <1>; - ovl_2l1_out: endpoint { + + ovl_2l1_out: endpoint@1 { + reg = <1>; remote-endpoint = <&rdma1_in>; }; }; @@ -1872,15 +1880,23 @@ #size-cells = <0>; port@0 { + #address-cells = <1>; + #size-cells = <0>; reg = <0>; - rdma1_in: endpoint { + + rdma1_in: endpoint@1 { + reg = <1>; remote-endpoint = <&ovl_2l1_out>; }; }; port@1 { + #address-cells = <1>; + #size-cells = <0>; reg = <1>; - rdma1_out: endpoint { + + rdma1_out: endpoint@1 { + reg = <1>; remote-endpoint = <&dpi_in>; }; }; @@ -2076,15 +2092,24 @@ #size-cells = <0>; port@0 { + #address-cells = <1>; + #size-cells = <0>; reg = <0>; - dpi_in: endpoint { + + dpi_in: endpoint@1 { + reg = <1>; remote-endpoint = <&rdma1_out>; }; }; port@1 { + #address-cells = <1>; + #size-cells = <0>; reg = <1>; - dpi_out: endpoint { }; + + dpi_out: endpoint@1 { + reg = <1>; + }; }; }; }; diff --git a/arch/arm64/boot/dts/qcom/agatti.dtsi b/arch/arm64/boot/dts/qcom/agatti.dtsi index 8bf5c5583fc22e..969ae1378db2ce 100644 --- a/arch/arm64/boot/dts/qcom/agatti.dtsi +++ b/arch/arm64/boot/dts/qcom/agatti.dtsi @@ -1591,8 +1591,12 @@ gpu: gpu@5900000 { compatible = "qcom,adreno-07000200", "qcom,adreno"; - reg = <0x0 0x05900000 0x0 0x40000>; - reg-names = "kgsl_3d0_reg_memory"; + reg = <0x0 0x05900000 0x0 0x40000>, + <0x0 0x0599e000 0x0 0x1000>, + <0x0 0x05961000 0x0 0x800>; + reg-names = "kgsl_3d0_reg_memory", + "cx_mem", + "cx_dbgc"; interrupts = ; diff --git a/arch/arm64/boot/dts/qcom/hamoa.dtsi b/arch/arm64/boot/dts/qcom/hamoa.dtsi index a17900eacb2039..83a0a0c3239d21 100644 --- a/arch/arm64/boot/dts/qcom/hamoa.dtsi +++ b/arch/arm64/boot/dts/qcom/hamoa.dtsi @@ -791,8 +791,8 @@ #address-cells = <2>; #size-cells = <2>; - dma-ranges = <0 0 0 0 0x10 0>; - ranges = <0 0 0 0 0x10 0>; + dma-ranges = <0 0 0 0 0x100 0>; + ranges = <0 0 0 0 0x100 0>; gcc: clock-controller@100000 { compatible = "qcom,x1e80100-gcc"; @@ -2937,7 +2937,7 @@ reg = <0 0x00fda000 0 0x4000>; clocks = <&gcc GCC_USB3_SEC_PHY_AUX_CLK>, - <&rpmhcc RPMH_CXO_CLK>, + <&tcsr TCSR_USB4_1_CLKREF_EN>, <&gcc GCC_USB3_SEC_PHY_COM_AUX_CLK>, <&gcc GCC_USB3_SEC_PHY_PIPE_CLK>; clock-names = "aux", @@ -3008,7 +3008,7 @@ reg = <0 0x00fdf000 0 0x4000>; clocks = <&gcc GCC_USB3_TERT_PHY_AUX_CLK>, - <&rpmhcc RPMH_CXO_CLK>, + <&tcsr TCSR_USB4_2_CLKREF_EN>, <&gcc GCC_USB3_TERT_PHY_COM_AUX_CLK>, <&gcc GCC_USB3_TERT_PHY_PIPE_CLK>; clock-names = "aux", diff --git a/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon.dtsi b/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon.dtsi index 4c983b10dd9252..7ace3540ef0a0e 100644 --- a/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon.dtsi @@ -378,7 +378,7 @@ status = "okay"; sideinteraction: touch@2c { - compatible = "ad,ad7147_captouch"; + compatible = "adi,ad7147_captouch"; reg = <0x2c>; pinctrl-names = "default", "sleep"; diff --git a/arch/arm64/boot/dts/qcom/qrb4210-rb2.dts b/arch/arm64/boot/dts/qcom/qrb4210-rb2.dts index 0cd36c54632fa3..5f8613150bdd29 100644 --- a/arch/arm64/boot/dts/qcom/qrb4210-rb2.dts +++ b/arch/arm64/boot/dts/qcom/qrb4210-rb2.dts @@ -694,7 +694,7 @@ &uart3 { interrupts-extended = <&intc GIC_SPI 330 IRQ_TYPE_LEVEL_HIGH>, - <&tlmm 11 IRQ_TYPE_LEVEL_HIGH>; + <&tlmm 11 IRQ_TYPE_EDGE_FALLING>; pinctrl-0 = <&uart3_default>; pinctrl-1 = <&uart3_sleep>; pinctrl-names = "default", "sleep"; diff --git a/arch/arm64/boot/dts/qcom/sdm630.dtsi b/arch/arm64/boot/dts/qcom/sdm630.dtsi index 8b1a45a4e56ed1..b383e480a394d4 100644 --- a/arch/arm64/boot/dts/qcom/sdm630.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm630.dtsi @@ -598,8 +598,8 @@ }; gpu_speed_bin: gpu-speed-bin@41a0 { - reg = <0x41a2 0x1>; - bits = <5 7>; + reg = <0x41a2 0x2>; + bits = <5 8>; }; }; diff --git a/arch/arm64/boot/dts/qcom/sdm845-db845c.dts b/arch/arm64/boot/dts/qcom/sdm845-db845c.dts index ce23f87e0316b6..5118b776a9bb37 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-db845c.dts +++ b/arch/arm64/boot/dts/qcom/sdm845-db845c.dts @@ -379,6 +379,12 @@ regulator-initial-mode = ; }; + vreg_l23a_3p3: ldo23 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3312000>; + regulator-initial-mode = ; + }; + vreg_l24a_3p075: ldo24 { regulator-min-microvolt = <3088000>; regulator-max-microvolt = <3088000>; @@ -850,7 +856,6 @@ status = "okay"; pinctrl-names = "default"; pinctrl-0 = <&qup_spi0_default>; - cs-gpios = <&tlmm 3 GPIO_ACTIVE_LOW>; can@0 { compatible = "microchip,mcp2517fd"; @@ -1156,6 +1161,7 @@ vdd-1.8-xo-supply = <&vreg_l7a_1p8>; vdd-1.3-rfa-supply = <&vreg_l17a_1p3>; vdd-3.3-ch0-supply = <&vreg_l25a_3p3>; + vdd-3.3-ch1-supply = <&vreg_l23a_3p3>; qcom,snoc-host-cap-8bit-quirk; qcom,calibration-variant = "Thundercomm_DB845C"; diff --git a/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi b/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi index db6dd04c51bb5f..8251f5a2f94757 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi @@ -148,7 +148,6 @@ gpio = <&tlmm 88 0>; enable-active-high; - regulator-boot-on; }; panel_vci_3v3: panel-vci-3v3-regulator { @@ -273,7 +272,7 @@ regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; regulator-initial-mode = ; - regulator-always-on; + regulator-boot-on; }; vreg_l17a_1p3: ldo17 { diff --git a/arch/arm64/boot/dts/qcom/sdm850-huawei-matebook-e-2019.dts b/arch/arm64/boot/dts/qcom/sdm850-huawei-matebook-e-2019.dts index 0ef9ea38a424a5..f048653818702a 100644 --- a/arch/arm64/boot/dts/qcom/sdm850-huawei-matebook-e-2019.dts +++ b/arch/arm64/boot/dts/qcom/sdm850-huawei-matebook-e-2019.dts @@ -30,9 +30,7 @@ /delete-node/ &ipa_fw_mem; /delete-node/ &ipa_gsi_mem; /delete-node/ &gpu_mem; -/delete-node/ &adsp_mem; /delete-node/ &wlan_msa_mem; -/delete-node/ &slpi_mem; / { model = "Huawei MateBook E 2019"; @@ -145,22 +143,13 @@ no-map; }; - adsp_mem: adsp@8c500000 { - reg = <0 0x8c500000 0 0x1a00000>; - no-map; - }; - ipa_fw_mem: ipa-fw@8df00000 { - reg = <0 0x8df00000 0 0x100000>; + reg = <0 0x8df00000 0 0x5a000>; no-map; }; - slpi_mem: slpi@96700000 { - reg = <0 0x96700000 0 0x1200000>; - }; - - gpu_mem: gpu@97900000 { - reg = <0 0x97900000 0 0x5000>; + gpu_mem: gpu@8df5a000 { + reg = <0 0x8df5a000 0 0x5000>; no-map; }; diff --git a/arch/arm64/boot/dts/qcom/sm6115.dtsi b/arch/arm64/boot/dts/qcom/sm6115.dtsi index 5e2032c26ea388..4dba724f2c756e 100644 --- a/arch/arm64/boot/dts/qcom/sm6115.dtsi +++ b/arch/arm64/boot/dts/qcom/sm6115.dtsi @@ -1715,8 +1715,12 @@ gpu: gpu@5900000 { compatible = "qcom,adreno-610.0", "qcom,adreno"; - reg = <0x0 0x05900000 0x0 0x40000>; - reg-names = "kgsl_3d0_reg_memory"; + reg = <0x0 0x05900000 0x0 0x40000>, + <0x0 0x0599e000 0x0 0x1000>, + <0x0 0x05961000 0x0 0x800>; + reg-names = "kgsl_3d0_reg_memory", + "cx_mem", + "cx_dbgc"; /* There's no (real) GMU, so we have to handle quite a bunch of clocks! */ clocks = <&gpucc GPU_CC_GX_GFX3D_CLK>, diff --git a/arch/arm64/boot/dts/qcom/sm8150-hdk.dts b/arch/arm64/boot/dts/qcom/sm8150-hdk.dts index 0339a572f34d01..1eea9c5c668473 100644 --- a/arch/arm64/boot/dts/qcom/sm8150-hdk.dts +++ b/arch/arm64/boot/dts/qcom/sm8150-hdk.dts @@ -387,6 +387,10 @@ status = "okay"; }; +&gpu_zap_shader { + firmware-name = "qcom/sm8150/a640_zap.mbn"; +}; + &i2c4 { clock-frequency = <100000>; diff --git a/arch/arm64/boot/dts/qcom/sm8150-mtp.dts b/arch/arm64/boot/dts/qcom/sm8150-mtp.dts index 12e8e1ada6d8bd..0f2d511624a8bf 100644 --- a/arch/arm64/boot/dts/qcom/sm8150-mtp.dts +++ b/arch/arm64/boot/dts/qcom/sm8150-mtp.dts @@ -358,6 +358,10 @@ status = "okay"; }; +&gpu_zap_shader { + firmware-name = "qcom/sm8150/a640_zap.mbn"; +}; + &pon { mode-bootloader = <0x2>; mode-recovery = <0x1>; diff --git a/arch/arm64/boot/dts/qcom/sm8250-hdk.dts b/arch/arm64/boot/dts/qcom/sm8250-hdk.dts index f5c193c6c5f9b4..3ea9d2b1a7d581 100644 --- a/arch/arm64/boot/dts/qcom/sm8250-hdk.dts +++ b/arch/arm64/boot/dts/qcom/sm8250-hdk.dts @@ -373,6 +373,10 @@ status = "okay"; }; +&gpu_zap_shader { + firmware-name = "qcom/sm8250/a650_zap.mbn"; +}; + &pon { mode-bootloader = <0x2>; mode-recovery = <0x1>; diff --git a/arch/arm64/boot/dts/qcom/talos.dtsi b/arch/arm64/boot/dts/qcom/talos.dtsi index 95d26e3136229f..3ef21bae317423 100644 --- a/arch/arm64/boot/dts/qcom/talos.dtsi +++ b/arch/arm64/boot/dts/qcom/talos.dtsi @@ -537,7 +537,6 @@ qup_opp_table: opp-table-qup { compatible = "operating-points-v2"; - opp-shared; opp-75000000 { opp-hz = /bits/ 64 <75000000>; diff --git a/arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts b/arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts index 08e814c03fa855..ed6fcdc337a0bd 100644 --- a/arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts +++ b/arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts @@ -8,7 +8,6 @@ /dts-v1/; /* Switch selection settings */ -#define SW_LCD_EN 0 #define SW_GPIO8_CAN0_STB 0 #define SW_GPIO9_CAN1_STB 0 #define SW_LCD_EN 0 diff --git a/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-common.dtsi b/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-common.dtsi index 3eed1f3948e8ea..63bd91690b5401 100644 --- a/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-common.dtsi +++ b/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-common.dtsi @@ -224,8 +224,7 @@ ctrl-pins { pinmux = , /* SD0_CLK */ , /* SD0_CMD */ - , /* SD0_CD */ - ; /* SD0_WP */ + ; /* SD0_CD */ }; }; @@ -282,6 +281,7 @@ pinctrl-names = "default", "state_uhs"; vmmc-supply = <®_3p3v>; vqmmc-supply = <&vqmmc_sdhi0>; + wp-gpios = <&pinctrl RZT2H_GPIO(22, 6) GPIO_ACTIVE_HIGH>; bus-width = <4>; sd-uhs-sdr50; sd-uhs-sdr104; diff --git a/arch/arm64/boot/dts/ti/k3-am67a-kontron-sa67-base.dts b/arch/arm64/boot/dts/ti/k3-am67a-kontron-sa67-base.dts index 7169d934adac58..95234c8460ed0d 100644 --- a/arch/arm64/boot/dts/ti/k3-am67a-kontron-sa67-base.dts +++ b/arch/arm64/boot/dts/ti/k3-am67a-kontron-sa67-base.dts @@ -85,8 +85,7 @@ linux,cma { compatible = "shared-dma-pool"; reusable; - size = <0x10000000>; - alignment = <0x2000>; + size = <0x00 0x10000000>; linux,cma-default; }; @@ -174,6 +173,7 @@ regulator-max-microvolt = <3300000>; vin-supply = <&vcc_3p3_s0>; regulator-boot-on; + enable-active-high; enable-gpios = <&main_gpio0 7 GPIO_ACTIVE_HIGH>; gpios = <&main_gpio0 8 GPIO_ACTIVE_HIGH>; states = <3300000 0x0>, diff --git a/arch/arm64/boot/dts/ti/k3-am69-aquila-clover.dts b/arch/arm64/boot/dts/ti/k3-am69-aquila-clover.dts index 55fd214a82e44c..c816ba3bfbdf78 100644 --- a/arch/arm64/boot/dts/ti/k3-am69-aquila-clover.dts +++ b/arch/arm64/boot/dts/ti/k3-am69-aquila-clover.dts @@ -280,8 +280,8 @@ try-power-role = "sink"; self-powered; source-pdos = ; - sink-pdos = ; - op-sink-microwatt = <1000000>; + sink-pdos = ; + op-sink-microwatt = <0>; ports { #address-cells = <1>; diff --git a/arch/arm64/boot/dts/ti/k3-am69-aquila-dev.dts b/arch/arm64/boot/dts/ti/k3-am69-aquila-dev.dts index c7ce804eac7038..f48601ae38b7c7 100644 --- a/arch/arm64/boot/dts/ti/k3-am69-aquila-dev.dts +++ b/arch/arm64/boot/dts/ti/k3-am69-aquila-dev.dts @@ -399,8 +399,8 @@ try-power-role = "sink"; self-powered; source-pdos = ; - sink-pdos = ; - op-sink-microwatt = <1000000>; + sink-pdos = ; + op-sink-microwatt = <0>; ports { #address-cells = <1>; diff --git a/arch/arm64/boot/dts/ti/k3-j784s4-j742s2-main-common.dtsi b/arch/arm64/boot/dts/ti/k3-j784s4-j742s2-main-common.dtsi index 9cc0901d58fbf9..c2636e624f18ba 100644 --- a/arch/arm64/boot/dts/ti/k3-j784s4-j742s2-main-common.dtsi +++ b/arch/arm64/boot/dts/ti/k3-j784s4-j742s2-main-common.dtsi @@ -2378,42 +2378,6 @@ assigned-clock-parents = <&k3_clks 351 4>; }; - watchdog4: watchdog@2240000 { - compatible = "ti,j7-rti-wdt"; - reg = <0x00 0x2240000 0x00 0x100>; - clocks = <&k3_clks 352 0>; - power-domains = <&k3_pds 352 TI_SCI_PD_EXCLUSIVE>; - assigned-clocks = <&k3_clks 352 0>; - assigned-clock-parents = <&k3_clks 352 4>; - }; - - watchdog5: watchdog@2250000 { - compatible = "ti,j7-rti-wdt"; - reg = <0x00 0x2250000 0x00 0x100>; - clocks = <&k3_clks 353 0>; - power-domains = <&k3_pds 353 TI_SCI_PD_EXCLUSIVE>; - assigned-clocks = <&k3_clks 353 0>; - assigned-clock-parents = <&k3_clks 353 4>; - }; - - watchdog6: watchdog@2260000 { - compatible = "ti,j7-rti-wdt"; - reg = <0x00 0x2260000 0x00 0x100>; - clocks = <&k3_clks 354 0>; - power-domains = <&k3_pds 354 TI_SCI_PD_EXCLUSIVE>; - assigned-clocks = <&k3_clks 354 0>; - assigned-clock-parents = <&k3_clks 354 4>; - }; - - watchdog7: watchdog@2270000 { - compatible = "ti,j7-rti-wdt"; - reg = <0x00 0x2270000 0x00 0x100>; - clocks = <&k3_clks 355 0>; - power-domains = <&k3_pds 355 TI_SCI_PD_EXCLUSIVE>; - assigned-clocks = <&k3_clks 355 0>; - assigned-clock-parents = <&k3_clks 355 4>; - }; - /* * The following RTI instances are coupled with MCU R5Fs, c7x and * GPU so keeping them reserved as these will be used by their diff --git a/arch/arm64/boot/dts/ti/k3-j784s4-main.dtsi b/arch/arm64/boot/dts/ti/k3-j784s4-main.dtsi index 0160fe0da98388..78fcd0c40abcfa 100644 --- a/arch/arm64/boot/dts/ti/k3-j784s4-main.dtsi +++ b/arch/arm64/boot/dts/ti/k3-j784s4-main.dtsi @@ -6,17 +6,40 @@ */ &cbass_main { - c71_3: dsp@67800000 { - compatible = "ti,j721s2-c71-dsp"; - reg = <0x00 0x67800000 0x00 0x00080000>, - <0x00 0x67e00000 0x00 0x0000c000>; - reg-names = "l2sram", "l1dram"; - resets = <&k3_reset 40 1>; - firmware-name = "j784s4-c71_3-fw"; - ti,sci = <&sms>; - ti,sci-dev-id = <40>; - ti,sci-proc-ids = <0x33 0xff>; - status = "disabled"; + watchdog4: watchdog@2240000 { + compatible = "ti,j7-rti-wdt"; + reg = <0x00 0x2240000 0x00 0x100>; + clocks = <&k3_clks 352 0>; + power-domains = <&k3_pds 352 TI_SCI_PD_EXCLUSIVE>; + assigned-clocks = <&k3_clks 352 0>; + assigned-clock-parents = <&k3_clks 352 4>; + }; + + watchdog5: watchdog@2250000 { + compatible = "ti,j7-rti-wdt"; + reg = <0x00 0x2250000 0x00 0x100>; + clocks = <&k3_clks 353 0>; + power-domains = <&k3_pds 353 TI_SCI_PD_EXCLUSIVE>; + assigned-clocks = <&k3_clks 353 0>; + assigned-clock-parents = <&k3_clks 353 4>; + }; + + watchdog6: watchdog@2260000 { + compatible = "ti,j7-rti-wdt"; + reg = <0x00 0x2260000 0x00 0x100>; + clocks = <&k3_clks 354 0>; + power-domains = <&k3_pds 354 TI_SCI_PD_EXCLUSIVE>; + assigned-clocks = <&k3_clks 354 0>; + assigned-clock-parents = <&k3_clks 354 4>; + }; + + watchdog7: watchdog@2270000 { + compatible = "ti,j7-rti-wdt"; + reg = <0x00 0x2270000 0x00 0x100>; + clocks = <&k3_clks 355 0>; + power-domains = <&k3_pds 355 TI_SCI_PD_EXCLUSIVE>; + assigned-clocks = <&k3_clks 355 0>; + assigned-clock-parents = <&k3_clks 355 4>; }; pcie2_rc: pcie@2920000 { @@ -113,6 +136,19 @@ status = "disabled"; }; }; + + c71_3: dsp@67800000 { + compatible = "ti,j721s2-c71-dsp"; + reg = <0x00 0x67800000 0x00 0x00080000>, + <0x00 0x67e00000 0x00 0x0000c000>; + reg-names = "l2sram", "l1dram"; + resets = <&k3_reset 40 1>; + firmware-name = "j784s4-c71_3-fw"; + ti,sci = <&sms>; + ti,sci-dev-id = <40>; + ti,sci-proc-ids = <0x33 0xff>; + status = "disabled"; + }; }; &scm_conf { diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config new file mode 100644 index 00000000000000..7ed0a173544025 --- /dev/null +++ b/arch/arm64/configs/asahi.config @@ -0,0 +1,74 @@ +CONFIG_RUST=y +CONFIG_ARM64_ACTLR_STATE=y +CONFIG_ARCH_APPLE=y +# CONFIG_ARM64_4K_PAGES is not set +CONFIG_ARM64_16K_PAGES=y +# CONFIG_ARM64_64K_PAGES is not set +CONFIG_ARM64_MEMORY_MODEL_CONTROL=y +CONFIG_ARM_APPLE_CPUIDLE=y +CONFIG_ARM_APPLE_SOC_CPUFREQ=m +CONFIG_BT_HCIBCM4377=m +CONFIG_PCIE_APPLE=m +CONFIG_NVME_APPLE=m +CONFIG_BRCMFMAC=m +CONFIG_BRCMFMAC_PCIE=y +CONFIG_TOUCHSCREEN_APPLE_Z2=m +CONFIG_INPUT_MACSMC_INPUT=m +CONFIG_I2C_APPLE=m +CONFIG_SPI_APPLE=m +CONFIG_SPMI_APPLE=m +CONFIG_PINCTRL_APPLE_GPIO=m +CONFIG_GPIO_MACSMC=m +CONFIG_POWER_RESET_MACSMC=m +CONFIG_CHARGER_MACSMC=m +CONFIG_SENSORS_MACSMC_HWMON=m +CONFIG_APPLE_WATCHDOG=m +CONFIG_VIDEO_APPLE_ISP=m +CONFIG_DRM=y +CONFIG_DRM_ASAHI=m +CONFIG_DRM_ADP=m +CONFIG_DRM_APPLE=m +CONFIG_DRM_APPLE_AUDIO=y +CONFIG_SND_SOC_APPLE_AOP_AUDIO=m +CONFIG_SND_SOC_APPLE_MCA=m +CONFIG_SND_SOC_APPLE_MACAUDIO=m +CONFIG_SND_SOC_CS42L83=m +CONFIG_SND_SOC_CS42L84=m +CONFIG_SND_SOC_TAS2764=m +CONFIG_SND_SOC_TAS2770=m +CONFIG_HID_APPLE=m +CONFIG_HID_MAGICMOUSE=m +CONFIG_SERIAL_SAMSUNG=y +CONFIG_SERIAL_SAMSUNG_CONSOLE=y +CONFIG_HID_DOCKCHANNEL=m +CONFIG_SPI_HID_APPLE_OF=m +CONFIG_SPI_HID_APPLE_CORE=m +CONFIG_USB_DWC3_APPLE=m +CONFIG_USB_XHCI_PCI_ASMEDIA=y +CONFIG_RTC_DRV_MACSMC=m +CONFIG_APPLE_ADMAC=m +CONFIG_APPLE_SIO=m +CONFIG_MFD_MACSMC=m +CONFIG_COMMON_CLK_APPLE_NCO=m +CONFIG_APPLE_DART=m +CONFIG_APPLE_DOCKCHANNEL=m +CONFIG_APPLE_MAILBOX=y +CONFIG_APPLE_PMGR_MISC=y +CONFIG_APPLE_RTKIT=y +CONFIG_APPLE_RTKIT_HELPER=m +CONFIG_APPLE_SART=m +CONFIG_RUST_APPLE_RTKIT=y +CONFIG_APPLE_AOP=m +CONFIG_APPLE_SEP=m +CONFIG_APPLE_PMGR_PWRSTATE=y +CONFIG_IIO_AOP_SENSOR_LAS=m +CONFIG_IIO_AOP_SENSOR_ALS=m +CONFIG_RUST_FW_LOADER_ABSTRACTIONS=y +CONFIG_PWM_APPLE=m +CONFIG_APPLE_AIC=y +CONFIG_PHY_APPLE_ATC=m +CONFIG_PHY_APPLE_DPTX=m +CONFIG_APPLE_M1_CPU_PMU=y +CONFIG_NVMEM_APPLE_EFUSES=m +CONFIG_NVMEM_APPLE_SPMI=m +CONFIG_MUX_APPLE_DPXBAR=m diff --git a/arch/arm64/include/asm/apple_cpufeature.h b/arch/arm64/include/asm/apple_cpufeature.h new file mode 100644 index 00000000000000..4370d91ffa3ec9 --- /dev/null +++ b/arch/arm64/include/asm/apple_cpufeature.h @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 + +#ifndef __ASM_APPLE_CPUFEATURES_H +#define __ASM_APPLE_CPUFEATURES_H + +#include +#include + +#define AIDR_APPLE_TSO_SHIFT 9 +#define AIDR_APPLE_TSO BIT(9) + +#define ACTLR_APPLE_TSO_SHIFT 1 +#define ACTLR_APPLE_TSO BIT(1) + +#endif diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 4de51f8d92cbac..a69eae348990bf 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -955,6 +955,12 @@ static inline unsigned int get_vmid_bits(u64 mmfr1) return 8; } +static __always_inline bool system_has_actlr_state(void) +{ + return IS_ENABLED(CONFIG_ARM64_ACTLR_STATE) && + alternative_has_cap_unlikely(ARM64_HAS_TSO_APPLE); +} + s64 arm64_ftr_safe_value(const struct arm64_ftr_bits *ftrp, s64 new, s64 cur); struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id); @@ -1078,6 +1084,10 @@ static inline bool cpu_has_lpa2(void) #endif } +void __init init_cpucap_indirect_list_impdef(void); +void __init init_cpucap_indirect_list_from_array(const struct arm64_cpu_capabilities *caps); +bool cpufeature_matches(u64 reg, const struct arm64_cpu_capabilities *entry); + #endif /* __ASSEMBLER__ */ #endif diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 55d34192a8de17..b50c32b10f7829 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -103,6 +103,11 @@ static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu) { if (!vcpu_has_run_once(vcpu)) vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS; + if (IS_ENABLED(CONFIG_ARM64_ACTLR_STATE) && ( + alternative_has_cap_unlikely(ARM64_HAS_ACTLR_VIRT) || + alternative_has_cap_unlikely(ARM64_HAS_ACTLR_VIRT_APPLE) + )) + vcpu->arch.hcr_el2 &= ~HCR_TACR; /* * For non-FWB CPUs, we trap VM ops (HCR_EL2.TVM) until M+C diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h index 9d54b2ea49d66b..66383e3009feae 100644 --- a/arch/arm64/include/asm/memory.h +++ b/arch/arm64/include/asm/memory.h @@ -112,7 +112,7 @@ #define DIRECT_MAP_PHYSMEM_END __pa(PAGE_END - 1) -#define MIN_THREAD_SHIFT (14 + KASAN_THREAD_SHIFT) +#define MIN_THREAD_SHIFT (15 + KASAN_THREAD_SHIFT) /* * VMAP'd stacks are allocated at page granularity, so we must ensure that such diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index e30c4c8e3a7a7d..499e0a85c08828 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -199,6 +199,9 @@ struct thread_struct { u64 gcs_base; u64 gcs_size; #endif +#ifdef CONFIG_ARM64_ACTLR_STATE + u64 actlr; +#endif }; static inline unsigned int thread_get_vl(struct thread_struct *thread, diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 76f32e424065e5..70bf2f40a0e4dd 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -34,6 +34,7 @@ obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ cpufeature.o alternative.o cacheinfo.o \ smp.o smp_spin_table.o topology.o smccc-call.o \ syscall.o proton-pack.o idle.o patching.o pi/ \ + cpufeature_impdef.o \ rsi.o jump_label.o obj-$(CONFIG_COMPAT) += sys32.o signal32.o \ diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index c840a93b9ef95b..4f06f909887d84 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -1080,7 +1080,7 @@ static void init_cpu_ftr_reg(u32 sys_reg, u64 new) extern const struct arm64_cpu_capabilities arm64_errata[]; static const struct arm64_cpu_capabilities arm64_features[]; -static void __init +void __init init_cpucap_indirect_list_from_array(const struct arm64_cpu_capabilities *caps) { for (; caps->matches; caps++) { @@ -1592,8 +1592,8 @@ has_always(const struct arm64_cpu_capabilities *entry, int scope) return true; } -static bool -feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry) +bool +cpufeature_matches(u64 reg, const struct arm64_cpu_capabilities *entry) { int val, min, max; u64 tmp; @@ -1646,14 +1646,14 @@ has_user_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope) if (!mask) return false; - return feature_matches(val, entry); + return cpufeature_matches(val, entry); } static bool has_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope) { u64 val = read_scoped_sysreg(entry, scope); - return feature_matches(val, entry); + return cpufeature_matches(val, entry); } const struct cpumask *system_32bit_el0_cpumask(void) @@ -3888,6 +3888,7 @@ void __init setup_boot_cpu_features(void) * handle the boot CPU. */ init_cpucap_indirect_list(); + init_cpucap_indirect_list_impdef(); /* * Detect broken pseudo-NMI. Must be called _before_ the call to diff --git a/arch/arm64/kernel/cpufeature_impdef.c b/arch/arm64/kernel/cpufeature_impdef.c new file mode 100644 index 00000000000000..aee7571fbadb84 --- /dev/null +++ b/arch/arm64/kernel/cpufeature_impdef.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Contains implementation-defined CPU feature definitions. + */ + +#define pr_fmt(fmt) "CPU features: " fmt + +#include +#include +#include +#include +#include + +#ifdef CONFIG_ARM64_MEMORY_MODEL_CONTROL +static bool has_apple_feature(const struct arm64_cpu_capabilities *entry, int scope) +{ + u64 val; + WARN_ON(scope == SCOPE_LOCAL_CPU && preemptible()); + + if (read_cpuid_implementor() != ARM_CPU_IMP_APPLE) + return false; + + val = read_sysreg(aidr_el1); + return cpufeature_matches(val, entry); +} + +static bool has_apple_tso(const struct arm64_cpu_capabilities *entry, int scope) +{ + u64 val; + + if (!has_apple_feature(entry, scope)) + return false; + + /* + * KVM and old versions of the macOS hypervisor will advertise TSO in + * AIDR_EL1, but then ignore writes to ACTLR_EL1. Test that the bit is + * actually writable before enabling TSO. + */ + + val = read_sysreg(actlr_el1); + write_sysreg(val ^ ACTLR_APPLE_TSO, actlr_el1); + if (!((val ^ read_sysreg(actlr_el1)) & ACTLR_APPLE_TSO)) { + pr_info_once("CPU advertises Apple TSO but it is broken, ignoring\n"); + return false; + } + + write_sysreg(val, actlr_el1); + return true; +} + +static bool has_tso_fixed(const struct arm64_cpu_capabilities *entry, int scope) +{ + /* List of CPUs that always use the TSO memory model */ + static const struct midr_range fixed_tso_list[] = { + MIDR_ALL_VERSIONS(MIDR_NVIDIA_DENVER), + MIDR_ALL_VERSIONS(MIDR_NVIDIA_CARMEL), + MIDR_ALL_VERSIONS(MIDR_FUJITSU_A64FX), + { /* sentinel */ } + }; + + return is_midr_in_range_list(fixed_tso_list); +} +#endif + +static bool has_apple_actlr_virt_impdef(const struct arm64_cpu_capabilities *entry, int scope) +{ + u64 midr = read_cpuid_id() & MIDR_CPU_MODEL_MASK; + + return midr >= MIDR_APPLE_M1_ICESTORM && midr <= MIDR_APPLE_M1_FIRESTORM_MAX; +} + +static bool has_apple_actlr_virt(const struct arm64_cpu_capabilities *entry, int scope) +{ + u64 midr = read_cpuid_id() & MIDR_CPU_MODEL_MASK; + + return midr >= MIDR_APPLE_M2_BLIZZARD && midr <= MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, 0xfff); +} + +static const struct arm64_cpu_capabilities arm64_impdef_features[] = { +#ifdef CONFIG_ARM64_MEMORY_MODEL_CONTROL + { + .desc = "TSO memory model (Apple)", + .capability = ARM64_HAS_TSO_APPLE, + .type = ARM64_CPUCAP_EARLY_LOCAL_CPU_FEATURE, + .matches = has_apple_tso, + .field_pos = AIDR_APPLE_TSO_SHIFT, + .field_width = 1, + .sign = FTR_UNSIGNED, + .min_field_value = 1, + .max_field_value = 1, + }, + { + .desc = "TSO memory model (Fixed)", + .capability = ARM64_HAS_TSO_FIXED, + .type = ARM64_CPUCAP_EARLY_LOCAL_CPU_FEATURE, + .matches = has_tso_fixed, + }, +#endif + { + .desc = "ACTLR virtualization (IMPDEF, Apple)", + .capability = ARM64_HAS_ACTLR_VIRT_APPLE, + .type = ARM64_CPUCAP_EARLY_LOCAL_CPU_FEATURE, + .matches = has_apple_actlr_virt_impdef, + }, + { + .desc = "ACTLR virtualization (architectural?)", + .capability = ARM64_HAS_ACTLR_VIRT, + .type = ARM64_CPUCAP_EARLY_LOCAL_CPU_FEATURE, + .matches = has_apple_actlr_virt, + }, + {}, +}; + +void __init init_cpucap_indirect_list_impdef(void) +{ + init_cpucap_indirect_list_from_array(arm64_impdef_features); +} diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 489554931231e6..59c416cfcdd2a4 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -41,8 +41,10 @@ #include #include #include +#include #include +#include #include #include #include @@ -442,6 +444,11 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) if (system_supports_poe()) p->thread.por_el0 = read_sysreg_s(SYS_POR_EL0); +#ifdef CONFIG_ARM64_ACTLR_STATE + if (system_has_actlr_state()) + p->thread.actlr = read_sysreg(actlr_el1); +#endif + if (stack_start) { if (is_compat_thread(task_thread_info(p))) childregs->compat_sp = stack_start; @@ -699,6 +706,65 @@ void update_sctlr_el1(u64 sctlr) isb(); } +#ifdef CONFIG_ARM64_MEMORY_MODEL_CONTROL +int arch_prctl_mem_model_get(struct task_struct *t) +{ + if (alternative_has_cap_unlikely(ARM64_HAS_TSO_APPLE) && + t->thread.actlr & ACTLR_APPLE_TSO) + return PR_SET_MEM_MODEL_TSO; + + return PR_SET_MEM_MODEL_DEFAULT; +} + +int arch_prctl_mem_model_set(struct task_struct *t, unsigned long val) +{ + if (alternative_has_cap_unlikely(ARM64_HAS_TSO_FIXED) && + val == PR_SET_MEM_MODEL_TSO) + return 0; + + if (alternative_has_cap_unlikely(ARM64_HAS_TSO_APPLE)) { + WARN_ON(!system_has_actlr_state()); + + switch (val) { + case PR_SET_MEM_MODEL_TSO: + t->thread.actlr |= ACTLR_APPLE_TSO; + break; + case PR_SET_MEM_MODEL_DEFAULT: + t->thread.actlr &= ~ACTLR_APPLE_TSO; + break; + default: + return -EINVAL; + } + write_sysreg(t->thread.actlr, actlr_el1); + return 0; + } + + if (val == PR_SET_MEM_MODEL_DEFAULT) + return 0; + + return -EINVAL; +} +#endif + +#ifdef CONFIG_ARM64_ACTLR_STATE +/* + * IMPDEF control register ACTLR_EL1 handling. Some CPUs use this to + * expose features that can be controlled by userspace. + */ +static void actlr_thread_switch(struct task_struct *next) +{ + if (!system_has_actlr_state()) + return; + + current->thread.actlr = read_sysreg(actlr_el1); + write_sysreg(next->thread.actlr, actlr_el1); +} +#else +static inline void actlr_thread_switch(struct task_struct *next) +{ +} +#endif + /* * Thread switching. */ @@ -718,6 +784,7 @@ struct task_struct *__switch_to(struct task_struct *prev, ptrauth_thread_switch_user(next); permission_overlay_switch(next); gcs_thread_switch(next); + actlr_thread_switch(next); /* * Complete any pending TLB or cache maintenance on this CPU in case the @@ -840,6 +907,10 @@ void arch_setup_new_exec(void) arch_prctl_spec_ctrl_set(current, PR_SPEC_STORE_BYPASS, PR_SPEC_ENABLE); } + +#ifdef CONFIG_ARM64_MEMORY_MODEL_CONTROL + arch_prctl_mem_model_set(current, PR_SET_MEM_MODEL_DEFAULT); +#endif } #ifdef CONFIG_ARM64_TAGGED_ADDR_ABI diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 23c05dc7a8f2ac..0fa2403c6fc0e6 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -368,6 +368,14 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) */ init_task.thread_info.ttbr0 = phys_to_ttbr(__pa_symbol(reserved_pg_dir)); #endif +#ifdef CONFIG_ARM64_ACTLR_STATE + /* Store the boot CPU ACTLR_EL1 value as the default. This will only + * be actually restored during context switching iff the platform is + * known to use ACTLR_EL1 for exposable features and its layout is + * known to be the same on all CPUs. + */ + init_task.thread.actlr = read_sysreg(actlr_el1); +#endif if (boot_args[1] || boot_args[2] || boot_args[3]) { pr_err("WARNING: x1-x3 nonzero in violation of boot protocol:\n" diff --git a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h index a17cbe7582de90..7c8383c809ea36 100644 --- a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h +++ b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h @@ -16,6 +16,9 @@ #include #include +#define SYS_IMP_APL_ACTLR_EL12 sys_reg(3, 6, 15, 14, 6) +#define SYS_ACTLR_EL12 sys_reg(3, 5, 1, 0, 1) + static inline bool ctxt_has_s1poe(struct kvm_cpu_context *ctxt); static inline struct kvm_vcpu *ctxt_to_vcpu(struct kvm_cpu_context *ctxt) @@ -172,6 +175,13 @@ static inline void __sysreg_save_el1_state(struct kvm_cpu_context *ctxt) if (ctxt_has_sctlr2(ctxt)) ctxt_sys_reg(ctxt, SCTLR2_EL1) = read_sysreg_el1(SYS_SCTLR2); + + if (IS_ENABLED(CONFIG_ARM64_ACTLR_STATE)) { + if (alternative_has_cap_unlikely(ARM64_HAS_ACTLR_VIRT)) + ctxt_sys_reg(ctxt, ACTLR_EL1) = read_sysreg_s(SYS_ACTLR_EL12); + else if (alternative_has_cap_unlikely(ARM64_HAS_ACTLR_VIRT_APPLE)) + ctxt_sys_reg(ctxt, ACTLR_EL1) = read_sysreg_s(SYS_IMP_APL_ACTLR_EL12); + } } static inline void __sysreg_save_el2_return_state(struct kvm_cpu_context *ctxt) @@ -256,6 +266,13 @@ static inline void __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt, write_sysreg(ctxt_sys_reg(ctxt, PAR_EL1), par_el1); write_sysreg(ctxt_sys_reg(ctxt, TPIDR_EL1), tpidr_el1); + if (IS_ENABLED(CONFIG_ARM64_ACTLR_STATE)) { + if (alternative_has_cap_unlikely(ARM64_HAS_ACTLR_VIRT)) + write_sysreg_s(ctxt_sys_reg(ctxt, ACTLR_EL1), SYS_ACTLR_EL12); + else if (alternative_has_cap_unlikely(ARM64_HAS_ACTLR_VIRT_APPLE)) + write_sysreg_s(ctxt_sys_reg(ctxt, ACTLR_EL1), SYS_IMP_APL_ACTLR_EL12); + } + if (ctxt_has_mte(ctxt)) { write_sysreg_el1(ctxt_sys_reg(ctxt, TFSR_EL1), SYS_TFSR); write_sysreg_s(ctxt_sys_reg(ctxt, TFSRE0_EL1), SYS_TFSRE0_EL1); diff --git a/arch/arm64/mm/gcs.c b/arch/arm64/mm/gcs.c index 6e93f78de79b17..04a23a497f2051 100644 --- a/arch/arm64/mm/gcs.c +++ b/arch/arm64/mm/gcs.c @@ -199,8 +199,8 @@ int arch_set_shadow_stack_status(struct task_struct *task, unsigned long arg) size = gcs_size(0); gcs = alloc_gcs(0, size); - if (!gcs) - return -ENOMEM; + if (IS_ERR_VALUE(gcs)) + return gcs; task->thread.gcspr_el0 = gcs + size - sizeof(u64); task->thread.gcs_base = gcs; diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index b6eb7a465ad248..1d657bd3ce6552 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -2951,7 +2951,7 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, u64 plt_target = 0ULL; bool poking_bpf_entry; - if (!__bpf_address_lookup((unsigned long)ip, &size, &offset, namebuf)) + if (!bpf_address_lookup((unsigned long)ip, &size, &offset, namebuf)) /* Only poking bpf text is supported. Since kernel function * entry is set up by ftrace, we reply on ftrace to poke kernel * functions. diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 0fac75f0153439..ee29e7c1c50465 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -8,6 +8,8 @@ BTI # Unreliable: use system_supports_32bit_el0() instead. HAS_32BIT_EL0_DO_NOT_USE HAS_32BIT_EL1 +HAS_ACTLR_VIRT +HAS_ACTLR_VIRT_APPLE HAS_ADDRESS_AUTH HAS_ADDRESS_AUTH_ARCH_QARMA3 HAS_ADDRESS_AUTH_ARCH_QARMA5 @@ -62,6 +64,8 @@ HAS_STAGE2_FWB HAS_TCR2 HAS_TIDCP1 HAS_TLB_RANGE +HAS_TSO_APPLE +HAS_TSO_FIXED HAS_VA52 HAS_VIRT_HOST_EXTN HAS_WFXT diff --git a/arch/loongarch/mm/kasan_init.c b/arch/loongarch/mm/kasan_init.c index 170da98ad4f551..0fc02ca0645738 100644 --- a/arch/loongarch/mm/kasan_init.c +++ b/arch/loongarch/mm/kasan_init.c @@ -40,39 +40,43 @@ static pgd_t kasan_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE); #define __pte_none(early, pte) (early ? pte_none(pte) : \ ((pte_val(pte) & _PFN_MASK) == (unsigned long)__pa(kasan_early_shadow_page))) -void *kasan_mem_to_shadow(const void *addr) +static void *mem_to_shadow(const void *addr) { - if (!kasan_enabled()) { + unsigned long offset = 0; + unsigned long maddr = (unsigned long)addr; + unsigned long xrange = (maddr >> XRANGE_SHIFT) & 0xffff; + + if (maddr >= FIXADDR_START) return (void *)(kasan_early_shadow_page); - } else { - unsigned long maddr = (unsigned long)addr; - unsigned long xrange = (maddr >> XRANGE_SHIFT) & 0xffff; - unsigned long offset = 0; - - if (maddr >= FIXADDR_START) - return (void *)(kasan_early_shadow_page); - - maddr &= XRANGE_SHADOW_MASK; - switch (xrange) { - case XKPRANGE_CC_SEG: - offset = XKPRANGE_CC_SHADOW_OFFSET; - break; - case XKPRANGE_UC_SEG: - offset = XKPRANGE_UC_SHADOW_OFFSET; - break; - case XKPRANGE_WC_SEG: - offset = XKPRANGE_WC_SHADOW_OFFSET; - break; - case XKVRANGE_VC_SEG: - offset = XKVRANGE_VC_SHADOW_OFFSET; - break; - default: - WARN_ON(1); - return NULL; - } - return (void *)((maddr >> KASAN_SHADOW_SCALE_SHIFT) + offset); + maddr &= XRANGE_SHADOW_MASK; + switch (xrange) { + case XKPRANGE_CC_SEG: + offset = XKPRANGE_CC_SHADOW_OFFSET; + break; + case XKPRANGE_UC_SEG: + offset = XKPRANGE_UC_SHADOW_OFFSET; + break; + case XKPRANGE_WC_SEG: + offset = XKPRANGE_WC_SHADOW_OFFSET; + break; + case XKVRANGE_VC_SEG: + offset = XKVRANGE_VC_SHADOW_OFFSET; + break; + default: + WARN_ON(1); + return NULL; } + + return (void *)((maddr >> KASAN_SHADOW_SCALE_SHIFT) + offset); +} + +void *kasan_mem_to_shadow(const void *addr) +{ + if (kasan_enabled()) + return mem_to_shadow(addr); + else + return (void *)(kasan_early_shadow_page); } const void *kasan_shadow_to_mem(const void *shadow_addr) @@ -293,11 +297,8 @@ void __init kasan_init(void) /* Maps everything to a single page of zeroes */ kasan_pgd_populate(KASAN_SHADOW_START, KASAN_SHADOW_END, NUMA_NO_NODE, true); - kasan_populate_early_shadow(kasan_mem_to_shadow((void *)VMALLOC_START), - kasan_mem_to_shadow((void *)KFENCE_AREA_END)); - - /* Enable KASAN here before kasan_mem_to_shadow(). */ - kasan_init_generic(); + kasan_populate_early_shadow(mem_to_shadow((void *)VMALLOC_START), + mem_to_shadow((void *)KFENCE_AREA_END)); /* Populate the linear mapping */ for_each_mem_range(i, &pa_start, &pa_end) { @@ -307,13 +308,13 @@ void __init kasan_init(void) if (start >= end) break; - kasan_map_populate((unsigned long)kasan_mem_to_shadow(start), - (unsigned long)kasan_mem_to_shadow(end), NUMA_NO_NODE); + kasan_map_populate((unsigned long)mem_to_shadow(start), + (unsigned long)mem_to_shadow(end), NUMA_NO_NODE); } /* Populate modules mapping */ - kasan_map_populate((unsigned long)kasan_mem_to_shadow((void *)MODULES_VADDR), - (unsigned long)kasan_mem_to_shadow((void *)MODULES_END), NUMA_NO_NODE); + kasan_map_populate((unsigned long)mem_to_shadow((void *)MODULES_VADDR), + (unsigned long)mem_to_shadow((void *)MODULES_END), NUMA_NO_NODE); /* * KAsan may reuse the contents of kasan_early_shadow_pte directly, so we * should make sure that it maps the zero page read-only. @@ -328,4 +329,5 @@ void __init kasan_init(void) /* At this point kasan is fully initialized. Enable error messages */ init_task.kasan_depth = 0; + kasan_init_generic(); } diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index d1d5a65308b9eb..3b63bc5b99d9a4 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -1319,7 +1319,7 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, /* Only poking bpf text is supported. Since kernel function entry * is set up by ftrace, we rely on ftrace to poke kernel functions. */ - if (!__bpf_address_lookup((unsigned long)ip, &size, &offset, namebuf)) + if (!bpf_address_lookup((unsigned long)ip, &size, &offset, namebuf)) return -ENOTSUPP; image = ip - offset; diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index b88b97139fa8e1..d87db7c535ea1d 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -1408,7 +1408,6 @@ config CPU_LOONGSON32 select CPU_MIPS32 select CPU_MIPSR2 select CPU_HAS_PREFETCH - select CPU_HAS_LOAD_STORE_LR select CPU_SUPPORTS_32BIT_KERNEL select CPU_SUPPORTS_HIGHMEM select CPU_SUPPORTS_CPUFREQ diff --git a/arch/mips/kernel/relocate.c b/arch/mips/kernel/relocate.c index 7f1c136ad85062..59833210542fff 100644 --- a/arch/mips/kernel/relocate.c +++ b/arch/mips/kernel/relocate.c @@ -420,7 +420,20 @@ void *__init relocate_kernel(void) goto out; /* The current thread is now within the relocated image */ +#ifndef CONFIG_CC_IS_CLANG __current_thread_info = RELOCATED(&init_thread_union); +#else + /* + * LLVM may wrongly restore $gp ($28) in epilog even if it's + * intentionally modified. Work around this by using inline + * assembly to assign $gp. $gp couldn't be listed as output or + * clobber, or LLVM will still restore its original value. + * See also LLVM upstream issue + * https://github.com/llvm/llvm-project/issues/176546 + */ + asm volatile("move $28, %0" : : + "r" (RELOCATED(&init_thread_union))); +#endif /* Return the new kernel's entry point */ kernel_entry = RELOCATED(start_kernel); diff --git a/arch/powerpc/include/asm/eeh.h b/arch/powerpc/include/asm/eeh.h index 5e34611de9ef40..b7ebb4ac2c7103 100644 --- a/arch/powerpc/include/asm/eeh.h +++ b/arch/powerpc/include/asm/eeh.h @@ -289,6 +289,8 @@ void eeh_pe_dev_traverse(struct eeh_pe *root, void eeh_pe_restore_bars(struct eeh_pe *pe); const char *eeh_pe_loc_get(struct eeh_pe *pe); struct pci_bus *eeh_pe_bus_get(struct eeh_pe *pe); +const char *eeh_pe_loc_get_bus(struct pci_bus *bus); +struct pci_bus *eeh_pe_bus_get_nolock(struct eeh_pe *pe); void eeh_show_enabled(void); int __init eeh_init(struct eeh_ops *ops); diff --git a/arch/powerpc/include/asm/kup.h b/arch/powerpc/include/asm/kup.h index dab63b82a8d4f3..f2009d7c8cfa75 100644 --- a/arch/powerpc/include/asm/kup.h +++ b/arch/powerpc/include/asm/kup.h @@ -134,7 +134,6 @@ static __always_inline void kuap_assert_locked(void) static __always_inline void allow_read_from_user(const void __user *from, unsigned long size) { - barrier_nospec(); allow_user_access(NULL, from, size, KUAP_READ); } @@ -146,7 +145,6 @@ static __always_inline void allow_write_to_user(void __user *to, unsigned long s static __always_inline void allow_read_write_user(void __user *to, const void __user *from, unsigned long size) { - barrier_nospec(); allow_user_access(to, from, size, KUAP_READ_WRITE); } diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h index 784a00e681fa37..3e622e647d6226 100644 --- a/arch/powerpc/include/asm/uaccess.h +++ b/arch/powerpc/include/asm/uaccess.h @@ -301,6 +301,7 @@ do { \ __typeof__(sizeof(*(ptr))) __gu_size = sizeof(*(ptr)); \ \ might_fault(); \ + barrier_nospec(); \ allow_read_from_user(__gu_addr, __gu_size); \ __get_user_size_allowed(__gu_val, __gu_addr, __gu_size, __gu_err); \ prevent_read_from_user(__gu_addr, __gu_size); \ @@ -329,6 +330,7 @@ raw_copy_in_user(void __user *to, const void __user *from, unsigned long n) { unsigned long ret; + barrier_nospec(); allow_read_write_user(to, from, n); ret = __copy_tofrom_user(to, from, n); prevent_read_write_user(to, from, n); @@ -415,6 +417,7 @@ static __must_check __always_inline bool user_access_begin(const void __user *pt might_fault(); + barrier_nospec(); allow_read_write_user((void __user *)ptr, ptr, len); return true; } @@ -431,6 +434,7 @@ user_read_access_begin(const void __user *ptr, size_t len) might_fault(); + barrier_nospec(); allow_read_from_user(ptr, len); return true; } diff --git a/arch/powerpc/kernel/eeh_driver.c b/arch/powerpc/kernel/eeh_driver.c index ef78ff77cf8f21..028f6915853234 100644 --- a/arch/powerpc/kernel/eeh_driver.c +++ b/arch/powerpc/kernel/eeh_driver.c @@ -846,7 +846,7 @@ void eeh_handle_normal_event(struct eeh_pe *pe) pci_lock_rescan_remove(); - bus = eeh_pe_bus_get(pe); + bus = eeh_pe_bus_get_nolock(pe); if (!bus) { pr_err("%s: Cannot find PCI bus for PHB#%x-PE#%x\n", __func__, pe->phb->global_number, pe->addr); @@ -886,14 +886,15 @@ void eeh_handle_normal_event(struct eeh_pe *pe) /* Log the event */ if (pe->type & EEH_PE_PHB) { pr_err("EEH: Recovering PHB#%x, location: %s\n", - pe->phb->global_number, eeh_pe_loc_get(pe)); + pe->phb->global_number, eeh_pe_loc_get_bus(bus)); } else { struct eeh_pe *phb_pe = eeh_phb_pe_get(pe->phb); pr_err("EEH: Recovering PHB#%x-PE#%x\n", pe->phb->global_number, pe->addr); pr_err("EEH: PE location: %s, PHB location: %s\n", - eeh_pe_loc_get(pe), eeh_pe_loc_get(phb_pe)); + eeh_pe_loc_get_bus(bus), + eeh_pe_loc_get_bus(eeh_pe_bus_get_nolock(phb_pe))); } #ifdef CONFIG_STACKTRACE @@ -1098,7 +1099,7 @@ void eeh_handle_normal_event(struct eeh_pe *pe) eeh_pe_state_clear(pe, EEH_PE_PRI_BUS, true); eeh_pe_dev_mode_mark(pe, EEH_DEV_REMOVED); - bus = eeh_pe_bus_get(pe); + bus = eeh_pe_bus_get_nolock(pe); if (bus) pci_hp_remove_devices(bus); else @@ -1222,7 +1223,7 @@ void eeh_handle_special_event(void) (phb_pe->state & EEH_PE_RECOVERING)) continue; - bus = eeh_pe_bus_get(phb_pe); + bus = eeh_pe_bus_get_nolock(phb_pe); if (!bus) { pr_err("%s: Cannot find PCI bus for " "PHB#%x-PE#%x\n", diff --git a/arch/powerpc/kernel/eeh_pe.c b/arch/powerpc/kernel/eeh_pe.c index e740101fadf3b1..040e8f69a4aa86 100644 --- a/arch/powerpc/kernel/eeh_pe.c +++ b/arch/powerpc/kernel/eeh_pe.c @@ -812,6 +812,24 @@ void eeh_pe_restore_bars(struct eeh_pe *pe) const char *eeh_pe_loc_get(struct eeh_pe *pe) { struct pci_bus *bus = eeh_pe_bus_get(pe); + return eeh_pe_loc_get_bus(bus); +} + +/** + * eeh_pe_loc_get_bus - Retrieve location code binding to the given PCI bus + * @bus: PCI bus + * + * Retrieve the location code associated with the given PCI bus. If the bus + * is a root bus, the location code is fetched from the PHB device tree node + * or root port. Otherwise, the location code is obtained from the device + * tree node of the upstream bridge of the bus. The function walks up the + * bus hierarchy if necessary, checking each node for the appropriate + * location code property ("ibm,io-base-loc-code" for root buses, + * "ibm,slot-location-code" for others). If no location code is found, + * returns "N/A". + */ +const char *eeh_pe_loc_get_bus(struct pci_bus *bus) +{ struct device_node *dn; const char *loc = NULL; @@ -838,8 +856,9 @@ const char *eeh_pe_loc_get(struct eeh_pe *pe) } /** - * eeh_pe_bus_get - Retrieve PCI bus according to the given PE + * _eeh_pe_bus_get - Retrieve PCI bus according to the given PE * @pe: EEH PE + * @do_lock: Is the caller already held the pci_lock_rescan_remove? * * Retrieve the PCI bus according to the given PE. Basically, * there're 3 types of PEs: PHB/Bus/Device. For PHB PE, the @@ -847,7 +866,7 @@ const char *eeh_pe_loc_get(struct eeh_pe *pe) * returned for BUS PE. However, we don't have associated PCI * bus for DEVICE PE. */ -struct pci_bus *eeh_pe_bus_get(struct eeh_pe *pe) +static struct pci_bus *_eeh_pe_bus_get(struct eeh_pe *pe, bool do_lock) { struct eeh_dev *edev; struct pci_dev *pdev; @@ -862,11 +881,58 @@ struct pci_bus *eeh_pe_bus_get(struct eeh_pe *pe) /* Retrieve the parent PCI bus of first (top) PCI device */ edev = list_first_entry_or_null(&pe->edevs, struct eeh_dev, entry); - pci_lock_rescan_remove(); + if (do_lock) + pci_lock_rescan_remove(); pdev = eeh_dev_to_pci_dev(edev); if (pdev) bus = pdev->bus; - pci_unlock_rescan_remove(); + if (do_lock) + pci_unlock_rescan_remove(); return bus; } + +/** + * eeh_pe_bus_get - Retrieve PCI bus associated with the given EEH PE, locking + * if needed + * @pe: Pointer to the EEH PE + * + * This function is a wrapper around _eeh_pe_bus_get(), which retrieves the PCI + * bus associated with the provided EEH PE structure. It acquires the PCI + * rescans lock to ensure safe access to shared data during the retrieval + * process. This function should be used when the caller requires the PCI bus + * while holding the rescan/remove lock, typically during operations that modify + * or inspect PCIe device state in a safe manner. + * + * RETURNS: + * A pointer to the PCI bus associated with the EEH PE, or NULL if none found. + */ + +struct pci_bus *eeh_pe_bus_get(struct eeh_pe *pe) +{ + return _eeh_pe_bus_get(pe, true); +} + +/** + * eeh_pe_bus_get_nolock - Retrieve PCI bus associated with the given EEH PE + * without locking + * @pe: Pointer to the EEH PE + * + * This function is a variant of _eeh_pe_bus_get() that retrieves the PCI bus + * associated with the specified EEH PE without acquiring the + * pci_lock_rescan_remove lock. It should only be used when the caller can + * guarantee safe access to PE structures without the need for that lock, + * typically in contexts where the lock is already held locking is otherwise + * managed. + * + * RETURNS: + * pointer to the PCI bus associated with the EEH PE, or NULL if none is found. + * + * NOTE: + * Use this function carefully to avoid race conditions and data corruption. + */ + +struct pci_bus *eeh_pe_bus_get_nolock(struct eeh_pe *pe) +{ + return _eeh_pe_bus_get(pe, false); +} diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 5e976730b2f5f5..e199976e410a19 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -1122,7 +1122,7 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, bpf_func = (unsigned long)ip; /* We currently only support poking bpf programs */ - if (!__bpf_address_lookup(bpf_func, &size, &offset, name)) { + if (!bpf_address_lookup(bpf_func, &size, &offset, name)) { pr_err("%s (0x%lx): kernel/modules are not supported\n", __func__, bpf_func); return -EOPNOTSUPP; } diff --git a/arch/riscv/boot/dts/sophgo/cv180x.dtsi b/arch/riscv/boot/dts/sophgo/cv180x.dtsi index 1b2b1969a6484e..06b0ce5a2db7af 100644 --- a/arch/riscv/boot/dts/sophgo/cv180x.dtsi +++ b/arch/riscv/boot/dts/sophgo/cv180x.dtsi @@ -438,8 +438,8 @@ clocks = <&clk CLK_AXI4_USB>, <&clk CLK_APB_USB>; clock-names = "otg", "utmi"; g-np-tx-fifo-size = <32>; - g-rx-fifo-size = <536>; - g-tx-fifo-size = <768 512 512 384 128 128>; + g-rx-fifo-size = <1536>; + g-tx-fifo-size = <128 128 64 64 64 64 32 32>; interrupts = ; phys = <&usbphy>; phy-names = "usb2-phy"; diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 0e5fad5f06ca11..783be50f38f2b3 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -275,6 +275,7 @@ config S390 select SPARSE_IRQ select SWIOTLB select SYSCTL_EXCEPTION_TRACE + select SYSTEM_DATA_VERIFICATION if KEXEC_SIG select THREAD_INFO_IN_TASK select TRACE_IRQFLAGS_SUPPORT select TTY @@ -301,7 +302,7 @@ config ARCH_SUPPORTS_KEXEC_FILE def_bool y config ARCH_SUPPORTS_KEXEC_SIG - def_bool MODULE_SIG_FORMAT + def_bool y config ARCH_SUPPORTS_KEXEC_PURGATORY def_bool y diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 576baa9a52c5bd..af1329ae9f82aa 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -3073,8 +3073,8 @@ void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap) cap->version = x86_pmu.version; cap->num_counters_gp = x86_pmu_num_counters(NULL); cap->num_counters_fixed = x86_pmu_num_counters_fixed(NULL); - cap->bit_width_gp = x86_pmu.cntval_bits; - cap->bit_width_fixed = x86_pmu.cntval_bits; + cap->bit_width_gp = cap->num_counters_gp ? x86_pmu.cntval_bits : 0; + cap->bit_width_fixed = cap->num_counters_fixed ? x86_pmu.cntval_bits : 0; cap->events_mask = (unsigned int)x86_pmu.events_maskl; cap->events_mask_len = x86_pmu.events_mask_len; cap->pebs_ept = x86_pmu.pebs_ept; diff --git a/arch/x86/hyperv/hv_crash.c b/arch/x86/hyperv/hv_crash.c index c0e22921ace1a4..a78e4fed572032 100644 --- a/arch/x86/hyperv/hv_crash.c +++ b/arch/x86/hyperv/hv_crash.c @@ -628,7 +628,9 @@ void hv_root_crash_init(void) if (rc) goto err_out; +#ifdef CONFIG_SMP smp_ops.crash_stop_other_cpus = hv_crash_stop_other_cpus; +#endif crash_kexec_post_notifiers = true; hv_crash_enabled = true; diff --git a/arch/x86/hyperv/hv_vtl.c b/arch/x86/hyperv/hv_vtl.c index c0edaed0efb303..9b6a9bc4ab760e 100644 --- a/arch/x86/hyperv/hv_vtl.c +++ b/arch/x86/hyperv/hv_vtl.c @@ -110,7 +110,7 @@ static void hv_vtl_ap_entry(void) static int hv_vtl_bringup_vcpu(u32 target_vp_index, int cpu, u64 eip_ignored) { - u64 status; + u64 status, rsp, rip; int ret = 0; struct hv_enable_vp_vtl *input; unsigned long irq_flags; @@ -123,9 +123,11 @@ static int hv_vtl_bringup_vcpu(u32 target_vp_index, int cpu, u64 eip_ignored) struct desc_struct *gdt; struct task_struct *idle = idle_thread_get(cpu); - u64 rsp = (unsigned long)idle->thread.sp; + if (IS_ERR(idle)) + return PTR_ERR(idle); - u64 rip = (u64)&hv_vtl_ap_entry; + rsp = (unsigned long)idle->thread.sp; + rip = (u64)&hv_vtl_ap_entry; native_store_gdt(&gdt_ptr); store_idt(&idt_ptr); diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h index b08c95872eed9f..c56e1e63b89324 100644 --- a/arch/x86/include/asm/ftrace.h +++ b/arch/x86/include/asm/ftrace.h @@ -57,7 +57,7 @@ arch_ftrace_get_regs(struct ftrace_regs *fregs) } #define arch_ftrace_partial_regs(regs) do { \ - regs->flags &= ~X86_EFLAGS_FIXED; \ + regs->flags |= X86_EFLAGS_FIXED; \ regs->cs = __KERNEL_CS; \ } while (0) diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index bc94ff1e250ad2..86059f2c0fcd45 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -951,26 +951,14 @@ static void init_amd_zen1(struct cpuinfo_x86 *c) } } -static bool cpu_has_zenbleed_microcode(void) -{ - u32 good_rev = 0; - - switch (boot_cpu_data.x86_model) { - case 0x30 ... 0x3f: good_rev = 0x0830107b; break; - case 0x60 ... 0x67: good_rev = 0x0860010c; break; - case 0x68 ... 0x6f: good_rev = 0x08608107; break; - case 0x70 ... 0x7f: good_rev = 0x08701033; break; - case 0xa0 ... 0xaf: good_rev = 0x08a00009; break; - - default: - return false; - } - - if (boot_cpu_data.microcode < good_rev) - return false; - - return true; -} +static const struct x86_cpu_id amd_zenbleed_microcode[] = { + ZEN_MODEL_STEP_UCODE(0x17, 0x31, 0x0, 0x0830107b), + ZEN_MODEL_STEP_UCODE(0x17, 0x60, 0x1, 0x0860010c), + ZEN_MODEL_STEP_UCODE(0x17, 0x68, 0x1, 0x08608107), + ZEN_MODEL_STEP_UCODE(0x17, 0x71, 0x0, 0x08701033), + ZEN_MODEL_STEP_UCODE(0x17, 0xa0, 0x0, 0x08a00009), + {} +}; static void zen2_zenbleed_check(struct cpuinfo_x86 *c) { @@ -980,7 +968,7 @@ static void zen2_zenbleed_check(struct cpuinfo_x86 *c) if (!cpu_has(c, X86_FEATURE_AVX)) return; - if (!cpu_has_zenbleed_microcode()) { + if (!x86_match_min_microcode_rev(amd_zenbleed_microcode)) { pr_notice_once("Zenbleed: please update your microcode for the most optimal fix\n"); msr_set_bit(MSR_AMD64_DE_CFG, MSR_AMD64_DE_CFG_ZEN2_FP_BACKUP_FIX_BIT); } else { diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S index a132608265f6c0..62c1c93aa1c6a6 100644 --- a/arch/x86/kernel/ftrace_64.S +++ b/arch/x86/kernel/ftrace_64.S @@ -364,6 +364,9 @@ SYM_CODE_START(return_to_handler) UNWIND_HINT_UNDEFINED ANNOTATE_NOENDBR + /* Store original rsp for pt_regs.sp value. */ + movq %rsp, %rdi + /* Restore return_to_handler value that got eaten by previous ret instruction. */ subq $8, %rsp UNWIND_HINT_FUNC @@ -374,7 +377,7 @@ SYM_CODE_START(return_to_handler) movq %rax, RAX(%rsp) movq %rdx, RDX(%rsp) movq %rbp, RBP(%rsp) - movq %rsp, RSP(%rsp) + movq %rdi, RSP(%rsp) movq %rsp, %rdi call ftrace_return_to_handler diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c index 53282dc7d5ac5b..23b91bf9b66303 100644 --- a/arch/x86/xen/enlighten.c +++ b/arch/x86/xen/enlighten.c @@ -470,7 +470,7 @@ int __init arch_xen_unpopulated_init(struct resource **res) * driver to know how much of the physmap is unpopulated and * set an accurate initial memory target. */ - xen_released_pages += xen_extra_mem[i].n_pfns; + xen_unpopulated_pages += xen_extra_mem[i].n_pfns; /* Zero so region is not also added to the balloon driver. */ xen_extra_mem[i].n_pfns = 0; } diff --git a/block/ioctl.c b/block/ioctl.c index 344478348a54e3..337e4c3b65b2cc 100644 --- a/block/ioctl.c +++ b/block/ioctl.c @@ -318,7 +318,13 @@ int blkdev_compat_ptr_ioctl(struct block_device *bdev, blk_mode_t mode, EXPORT_SYMBOL(blkdev_compat_ptr_ioctl); #endif -static bool blkdev_pr_allowed(struct block_device *bdev, blk_mode_t mode) +enum pr_direction { + PR_IN, /* read from device */ + PR_OUT, /* write to device */ +}; + +static bool blkdev_pr_allowed(struct block_device *bdev, blk_mode_t mode, + enum pr_direction dir) { /* no sense to make reservations for partitions */ if (bdev_is_partition(bdev)) @@ -326,11 +332,17 @@ static bool blkdev_pr_allowed(struct block_device *bdev, blk_mode_t mode) if (capable(CAP_SYS_ADMIN)) return true; + /* - * Only allow unprivileged reservations if the file descriptor is open - * for writing. + * Only allow unprivileged reservation _out_ commands if the file + * descriptor is open for writing. Allow reservation _in_ commands if + * the file descriptor is open for reading since they do not modify the + * device. */ - return mode & BLK_OPEN_WRITE; + if (dir == PR_IN) + return mode & BLK_OPEN_READ; + else + return mode & BLK_OPEN_WRITE; } static int blkdev_pr_register(struct block_device *bdev, blk_mode_t mode, @@ -339,7 +351,7 @@ static int blkdev_pr_register(struct block_device *bdev, blk_mode_t mode, const struct pr_ops *ops = bdev->bd_disk->fops->pr_ops; struct pr_registration reg; - if (!blkdev_pr_allowed(bdev, mode)) + if (!blkdev_pr_allowed(bdev, mode, PR_OUT)) return -EPERM; if (!ops || !ops->pr_register) return -EOPNOTSUPP; @@ -357,7 +369,7 @@ static int blkdev_pr_reserve(struct block_device *bdev, blk_mode_t mode, const struct pr_ops *ops = bdev->bd_disk->fops->pr_ops; struct pr_reservation rsv; - if (!blkdev_pr_allowed(bdev, mode)) + if (!blkdev_pr_allowed(bdev, mode, PR_OUT)) return -EPERM; if (!ops || !ops->pr_reserve) return -EOPNOTSUPP; @@ -375,7 +387,7 @@ static int blkdev_pr_release(struct block_device *bdev, blk_mode_t mode, const struct pr_ops *ops = bdev->bd_disk->fops->pr_ops; struct pr_reservation rsv; - if (!blkdev_pr_allowed(bdev, mode)) + if (!blkdev_pr_allowed(bdev, mode, PR_OUT)) return -EPERM; if (!ops || !ops->pr_release) return -EOPNOTSUPP; @@ -393,7 +405,7 @@ static int blkdev_pr_preempt(struct block_device *bdev, blk_mode_t mode, const struct pr_ops *ops = bdev->bd_disk->fops->pr_ops; struct pr_preempt p; - if (!blkdev_pr_allowed(bdev, mode)) + if (!blkdev_pr_allowed(bdev, mode, PR_OUT)) return -EPERM; if (!ops || !ops->pr_preempt) return -EOPNOTSUPP; @@ -411,7 +423,7 @@ static int blkdev_pr_clear(struct block_device *bdev, blk_mode_t mode, const struct pr_ops *ops = bdev->bd_disk->fops->pr_ops; struct pr_clear c; - if (!blkdev_pr_allowed(bdev, mode)) + if (!blkdev_pr_allowed(bdev, mode, PR_OUT)) return -EPERM; if (!ops || !ops->pr_clear) return -EOPNOTSUPP; @@ -434,7 +446,7 @@ static int blkdev_pr_read_keys(struct block_device *bdev, blk_mode_t mode, size_t keys_copy_len; int ret; - if (!blkdev_pr_allowed(bdev, mode)) + if (!blkdev_pr_allowed(bdev, mode, PR_IN)) return -EPERM; if (!ops || !ops->pr_read_keys) return -EOPNOTSUPP; @@ -486,7 +498,7 @@ static int blkdev_pr_read_reservation(struct block_device *bdev, struct pr_read_reservation out = {}; int ret; - if (!blkdev_pr_allowed(bdev, mode)) + if (!blkdev_pr_allowed(bdev, mode, PR_IN)) return -EPERM; if (!ops || !ops->pr_read_reservation) return -EOPNOTSUPP; diff --git a/drivers/accel/amdxdna/aie2_ctx.c b/drivers/accel/amdxdna/aie2_ctx.c index 42d876a427c595..37d05f2e986f9a 100644 --- a/drivers/accel/amdxdna/aie2_ctx.c +++ b/drivers/accel/amdxdna/aie2_ctx.c @@ -47,17 +47,6 @@ static void aie2_job_put(struct amdxdna_sched_job *job) kref_put(&job->refcnt, aie2_job_release); } -static void aie2_hwctx_status_shift_stop(struct amdxdna_hwctx *hwctx) -{ - hwctx->old_status = hwctx->status; - hwctx->status = HWCTX_STAT_STOP; -} - -static void aie2_hwctx_status_restore(struct amdxdna_hwctx *hwctx) -{ - hwctx->status = hwctx->old_status; -} - /* The bad_job is used in aie2_sched_job_timedout, otherwise, set it to NULL */ static void aie2_hwctx_stop(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx, struct drm_sched_job *bad_job) @@ -84,11 +73,6 @@ static int aie2_hwctx_restart(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hw goto out; } - if (hwctx->status != HWCTX_STAT_READY) { - XDNA_DBG(xdna, "hwctx is not ready, status %d", hwctx->status); - goto out; - } - ret = aie2_config_cu(hwctx, NULL); if (ret) { XDNA_ERR(xdna, "Config cu failed, ret %d", ret); @@ -140,7 +124,6 @@ static int aie2_hwctx_suspend_cb(struct amdxdna_hwctx *hwctx, void *arg) aie2_hwctx_wait_for_idle(hwctx); aie2_hwctx_stop(xdna, hwctx, NULL); - aie2_hwctx_status_shift_stop(hwctx); return 0; } @@ -162,7 +145,6 @@ static int aie2_hwctx_resume_cb(struct amdxdna_hwctx *hwctx, void *arg) { struct amdxdna_dev *xdna = hwctx->client->xdna; - aie2_hwctx_status_restore(hwctx); return aie2_hwctx_restart(xdna, hwctx); } @@ -292,7 +274,7 @@ aie2_sched_cmdlist_resp_handler(void *handle, void __iomem *data, size_t size) ret = -EINVAL; goto out; } - amdxdna_cmd_set_state(cmd_abo, fail_cmd_status); + amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_ERROR); if (amdxdna_cmd_get_op(cmd_abo) == ERT_CMD_CHAIN) { struct amdxdna_cmd_chain *cc = amdxdna_cmd_get_payload(cmd_abo, NULL); @@ -315,12 +297,19 @@ aie2_sched_job_run(struct drm_sched_job *sched_job) struct dma_fence *fence; int ret; + if (!hwctx->priv->mbox_chann) + return NULL; + if (!mmget_not_zero(job->mm)) return ERR_PTR(-ESRCH); kref_get(&job->refcnt); fence = dma_fence_get(job->fence); + ret = amdxdna_pm_resume_get(hwctx->client->xdna); + if (ret) + goto out; + if (job->drv_cmd) { switch (job->drv_cmd->opcode) { case SYNC_DEBUG_BO: @@ -347,6 +336,7 @@ aie2_sched_job_run(struct drm_sched_job *sched_job) out: if (ret) { + amdxdna_pm_suspend_put(hwctx->client->xdna); dma_fence_put(job->fence); aie2_job_put(job); mmput(job->mm); @@ -468,6 +458,12 @@ static int aie2_alloc_resource(struct amdxdna_hwctx *hwctx) struct alloc_requests *xrs_req; int ret; + if (AIE2_FEATURE_ON(xdna->dev_handle, AIE2_TEMPORAL_ONLY)) { + hwctx->num_unused_col = xdna->dev_handle->total_col - hwctx->num_col; + hwctx->num_col = xdna->dev_handle->total_col; + return aie2_create_context(xdna->dev_handle, hwctx); + } + xrs_req = kzalloc(sizeof(*xrs_req), GFP_KERNEL); if (!xrs_req) return -ENOMEM; @@ -499,9 +495,15 @@ static void aie2_release_resource(struct amdxdna_hwctx *hwctx) struct amdxdna_dev *xdna = hwctx->client->xdna; int ret; - ret = xrs_release_resource(xdna->xrs_hdl, (uintptr_t)hwctx); - if (ret) - XDNA_ERR(xdna, "Release AIE resource failed, ret %d", ret); + if (AIE2_FEATURE_ON(xdna->dev_handle, AIE2_TEMPORAL_ONLY)) { + ret = aie2_destroy_context(xdna->dev_handle, hwctx); + if (ret) + XDNA_ERR(xdna, "Destroy temporal only context failed, ret %d", ret); + } else { + ret = xrs_release_resource(xdna->xrs_hdl, (uintptr_t)hwctx); + if (ret) + XDNA_ERR(xdna, "Release AIE resource failed, ret %d", ret); + } } static int aie2_ctx_syncobj_create(struct amdxdna_hwctx *hwctx) @@ -651,7 +653,6 @@ int aie2_hwctx_init(struct amdxdna_hwctx *hwctx) } amdxdna_pm_suspend_put(xdna); - hwctx->status = HWCTX_STAT_INIT; init_waitqueue_head(&priv->job_free_wq); XDNA_DBG(xdna, "hwctx %s init completed", hwctx->name); @@ -693,7 +694,9 @@ void aie2_hwctx_fini(struct amdxdna_hwctx *hwctx) aie2_hwctx_wait_for_idle(hwctx); /* Request fw to destroy hwctx and cancel the rest pending requests */ + drm_sched_stop(&hwctx->priv->sched, NULL); aie2_release_resource(hwctx); + drm_sched_start(&hwctx->priv->sched, 0); mutex_unlock(&xdna->dev_lock); drm_sched_entity_destroy(&hwctx->priv->entity); @@ -737,7 +740,7 @@ static int aie2_hwctx_cu_config(struct amdxdna_hwctx *hwctx, void *buf, u32 size if (XDNA_MBZ_DBG(xdna, config->pad, sizeof(config->pad))) return -EINVAL; - if (hwctx->status != HWCTX_STAT_INIT) { + if (hwctx->cus) { XDNA_ERR(xdna, "Not support re-config CU"); return -EINVAL; } @@ -768,7 +771,6 @@ static int aie2_hwctx_cu_config(struct amdxdna_hwctx *hwctx, void *buf, u32 size } wmb(); /* To avoid locking in command submit when check status */ - hwctx->status = HWCTX_STAT_READY; return 0; @@ -991,15 +993,11 @@ int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, goto free_chain; } - ret = amdxdna_pm_resume_get(xdna); - if (ret) - goto cleanup_job; - retry: ret = drm_gem_lock_reservations(job->bos, job->bo_cnt, &acquire_ctx); if (ret) { XDNA_WARN(xdna, "Failed to lock BOs, ret %d", ret); - goto suspend_put; + goto cleanup_job; } for (i = 0; i < job->bo_cnt; i++) { @@ -1007,7 +1005,7 @@ int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, if (ret) { XDNA_WARN(xdna, "Failed to reserve fences %d", ret); drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); - goto suspend_put; + goto cleanup_job; } } @@ -1022,12 +1020,12 @@ int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); } else if (time_after(jiffies, timeout)) { ret = -ETIME; - goto suspend_put; + goto cleanup_job; } ret = aie2_populate_range(abo); if (ret) - goto suspend_put; + goto cleanup_job; goto retry; } } @@ -1053,8 +1051,6 @@ int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, return 0; -suspend_put: - amdxdna_pm_suspend_put(xdna); cleanup_job: drm_sched_job_cleanup(&job->base); free_chain: diff --git a/drivers/accel/amdxdna/aie2_message.c b/drivers/accel/amdxdna/aie2_message.c index d493bb1c33606f..2c5b27d90563ea 100644 --- a/drivers/accel/amdxdna/aie2_message.c +++ b/drivers/accel/amdxdna/aie2_message.c @@ -39,7 +39,6 @@ static int aie2_send_mgmt_msg_wait(struct amdxdna_dev_hdl *ndev, if (!ndev->mgmt_chann) return -ENODEV; - drm_WARN_ON(&xdna->ddev, xdna->rpm_on && !mutex_is_locked(&xdna->dev_lock)); ret = xdna_send_msg_wait(xdna, ndev->mgmt_chann, msg); if (ret == -ETIME) { xdna_mailbox_stop_channel(ndev->mgmt_chann); @@ -186,6 +185,19 @@ int aie2_query_firmware_version(struct amdxdna_dev_hdl *ndev, return 0; } +static int aie2_destroy_context_req(struct amdxdna_dev_hdl *ndev, u32 id) +{ + DECLARE_AIE2_MSG(destroy_ctx, MSG_OP_DESTROY_CONTEXT); + struct amdxdna_dev *xdna = ndev->xdna; + int ret; + + req.context_id = id; + ret = aie2_send_mgmt_msg_wait(ndev, &msg); + if (ret) + XDNA_WARN(xdna, "Destroy context failed, ret %d", ret); + + return ret; +} int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx) { DECLARE_AIE2_MSG(create_ctx, MSG_OP_CREATE_CONTEXT); @@ -199,6 +211,7 @@ int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwct req.aie_type = 1; req.start_col = hwctx->start_col; req.num_col = hwctx->num_col; + req.num_unused_col = hwctx->num_unused_col; req.num_cq_pairs_requested = 1; req.pasid = hwctx->client->pasid; req.context_priority = 2; @@ -208,13 +221,14 @@ int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwct return ret; hwctx->fw_ctx_id = resp.context_id; - WARN_ONCE(hwctx->fw_ctx_id == -1, "Unexpected context id"); + if (WARN_ON_ONCE(hwctx->fw_ctx_id == -1)) + return -EINVAL; if (ndev->force_preempt_enabled) { ret = aie2_runtime_cfg(ndev, AIE2_RT_CFG_FORCE_PREEMPT, &hwctx->fw_ctx_id); if (ret) { XDNA_ERR(xdna, "failed to enable force preempt %d", ret); - return ret; + goto del_ctx_req; } } @@ -231,51 +245,39 @@ int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwct ret = pci_irq_vector(to_pci_dev(xdna->ddev.dev), resp.msix_id); if (ret == -EINVAL) { - XDNA_ERR(xdna, "not able to create channel"); - goto out_destroy_context; + XDNA_ERR(xdna, "Alloc IRQ failed %d", ret); + goto del_ctx_req; } intr_reg = i2x.mb_head_ptr_reg + 4; hwctx->priv->mbox_chann = xdna_mailbox_create_channel(ndev->mbox, &x2i, &i2x, intr_reg, ret); if (!hwctx->priv->mbox_chann) { - XDNA_ERR(xdna, "not able to create channel"); + XDNA_ERR(xdna, "Not able to create channel"); ret = -EINVAL; - goto out_destroy_context; + goto del_ctx_req; } ndev->hwctx_num++; - XDNA_DBG(xdna, "%s mailbox channel irq: %d, msix_id: %d", - hwctx->name, ret, resp.msix_id); - XDNA_DBG(xdna, "%s created fw ctx %d pasid %d", hwctx->name, - hwctx->fw_ctx_id, hwctx->client->pasid); + XDNA_DBG(xdna, "Mailbox channel irq: %d, msix_id: %d", ret, resp.msix_id); + XDNA_DBG(xdna, "Created fw ctx %d pasid %d", hwctx->fw_ctx_id, hwctx->client->pasid); return 0; -out_destroy_context: - aie2_destroy_context(ndev, hwctx); +del_ctx_req: + aie2_destroy_context_req(ndev, hwctx->fw_ctx_id); return ret; } int aie2_destroy_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx) { - DECLARE_AIE2_MSG(destroy_ctx, MSG_OP_DESTROY_CONTEXT); struct amdxdna_dev *xdna = ndev->xdna; int ret; - if (hwctx->fw_ctx_id == -1) - return 0; - xdna_mailbox_stop_channel(hwctx->priv->mbox_chann); - - req.context_id = hwctx->fw_ctx_id; - ret = aie2_send_mgmt_msg_wait(ndev, &msg); - if (ret) - XDNA_WARN(xdna, "%s destroy context failed, ret %d", hwctx->name, ret); - + ret = aie2_destroy_context_req(ndev, hwctx->fw_ctx_id); xdna_mailbox_destroy_channel(hwctx->priv->mbox_chann); - XDNA_DBG(xdna, "%s destroyed fw ctx %d", hwctx->name, - hwctx->fw_ctx_id); + XDNA_DBG(xdna, "Destroyed fw ctx %d", hwctx->fw_ctx_id); hwctx->priv->mbox_chann = NULL; hwctx->fw_ctx_id = -1; ndev->hwctx_num--; @@ -448,6 +450,9 @@ int aie2_config_cu(struct amdxdna_hwctx *hwctx, if (!chann) return -ENODEV; + if (!hwctx->cus) + return 0; + if (hwctx->cus->num_cus > MAX_NUM_CUS) { XDNA_DBG(xdna, "Exceed maximum CU %d", MAX_NUM_CUS); return -EINVAL; @@ -646,6 +651,7 @@ aie2_cmdlist_fill_npu_cf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *siz u32 cmd_len; void *cmd; + memset(npu_slot, 0, sizeof(*npu_slot)); cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); if (*size < sizeof(*npu_slot) + cmd_len) return -EINVAL; @@ -654,7 +660,6 @@ aie2_cmdlist_fill_npu_cf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *siz if (npu_slot->cu_idx == INVALID_CU_IDX) return -EINVAL; - memset(npu_slot, 0, sizeof(*npu_slot)); npu_slot->type = EXEC_NPU_TYPE_NON_ELF; npu_slot->arg_cnt = cmd_len / sizeof(u32); memcpy(npu_slot->args, cmd, cmd_len); @@ -671,6 +676,7 @@ aie2_cmdlist_fill_npu_dpu(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *si u32 cmd_len; u32 arg_sz; + memset(npu_slot, 0, sizeof(*npu_slot)); sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); arg_sz = cmd_len - sizeof(*sn); if (cmd_len < sizeof(*sn) || arg_sz > MAX_NPU_ARGS_SIZE) @@ -683,7 +689,6 @@ aie2_cmdlist_fill_npu_dpu(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *si if (npu_slot->cu_idx == INVALID_CU_IDX) return -EINVAL; - memset(npu_slot, 0, sizeof(*npu_slot)); npu_slot->type = EXEC_NPU_TYPE_PARTIAL_ELF; npu_slot->inst_buf_addr = sn->buffer; npu_slot->inst_size = sn->buffer_size; @@ -703,6 +708,7 @@ aie2_cmdlist_fill_npu_preempt(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t u32 cmd_len; u32 arg_sz; + memset(npu_slot, 0, sizeof(*npu_slot)); pd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); arg_sz = cmd_len - sizeof(*pd); if (cmd_len < sizeof(*pd) || arg_sz > MAX_NPU_ARGS_SIZE) @@ -715,7 +721,6 @@ aie2_cmdlist_fill_npu_preempt(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t if (npu_slot->cu_idx == INVALID_CU_IDX) return -EINVAL; - memset(npu_slot, 0, sizeof(*npu_slot)); npu_slot->type = EXEC_NPU_TYPE_PREEMPT; npu_slot->inst_buf_addr = pd->inst_buf; npu_slot->save_buf_addr = pd->save_buf; @@ -739,6 +744,7 @@ aie2_cmdlist_fill_npu_elf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *si u32 cmd_len; u32 arg_sz; + memset(npu_slot, 0, sizeof(*npu_slot)); pd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); arg_sz = cmd_len - sizeof(*pd); if (cmd_len < sizeof(*pd) || arg_sz > MAX_NPU_ARGS_SIZE) @@ -747,7 +753,6 @@ aie2_cmdlist_fill_npu_elf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *si if (*size < sizeof(*npu_slot) + arg_sz) return -EINVAL; - memset(npu_slot, 0, sizeof(*npu_slot)); npu_slot->type = EXEC_NPU_TYPE_ELF; npu_slot->inst_buf_addr = pd->inst_buf; npu_slot->save_buf_addr = pd->save_buf; diff --git a/drivers/accel/amdxdna/aie2_msg_priv.h b/drivers/accel/amdxdna/aie2_msg_priv.h index 1c957a6298d39d..cc912b7899ce5a 100644 --- a/drivers/accel/amdxdna/aie2_msg_priv.h +++ b/drivers/accel/amdxdna/aie2_msg_priv.h @@ -112,7 +112,8 @@ struct create_ctx_req { __u32 aie_type; __u8 start_col; __u8 num_col; - __u16 reserved; + __u8 num_unused_col; + __u8 reserved; __u8 num_cq_pairs_requested; __u8 reserved1; __u16 pasid; diff --git a/drivers/accel/amdxdna/aie2_pci.c b/drivers/accel/amdxdna/aie2_pci.c index 8141d8e5163605..ec1c3ad57d490e 100644 --- a/drivers/accel/amdxdna/aie2_pci.c +++ b/drivers/accel/amdxdna/aie2_pci.c @@ -322,7 +322,7 @@ static int aie2_xrs_set_dft_dpm_level(struct drm_device *ddev, u32 dpm_level) if (ndev->pw_mode != POWER_MODE_DEFAULT || ndev->dpm_level == dpm_level) return 0; - return ndev->priv->hw_ops.set_dpm(ndev, dpm_level); + return aie2_pm_set_dpm(ndev, dpm_level); } static struct xrs_action_ops aie2_xrs_actions = { diff --git a/drivers/accel/amdxdna/aie2_pci.h b/drivers/accel/amdxdna/aie2_pci.h index a5f9c42155d178..4fdc032bc171bd 100644 --- a/drivers/accel/amdxdna/aie2_pci.h +++ b/drivers/accel/amdxdna/aie2_pci.h @@ -231,6 +231,7 @@ struct aie2_hw_ops { enum aie2_fw_feature { AIE2_NPU_COMMAND, AIE2_PREEMPT, + AIE2_TEMPORAL_ONLY, AIE2_FEATURE_MAX }; @@ -285,6 +286,7 @@ int npu4_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level); /* aie2_pm.c */ int aie2_pm_init(struct amdxdna_dev_hdl *ndev); int aie2_pm_set_mode(struct amdxdna_dev_hdl *ndev, enum amdxdna_power_mode_type target); +int aie2_pm_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level); /* aie2_psp.c */ struct psp_device *aie2m_psp_create(struct drm_device *ddev, struct psp_config *conf); diff --git a/drivers/accel/amdxdna/aie2_pm.c b/drivers/accel/amdxdna/aie2_pm.c index 426c38fce84829..579b8be13b1804 100644 --- a/drivers/accel/amdxdna/aie2_pm.c +++ b/drivers/accel/amdxdna/aie2_pm.c @@ -10,6 +10,7 @@ #include "aie2_pci.h" #include "amdxdna_pci_drv.h" +#include "amdxdna_pm.h" #define AIE2_CLK_GATING_ENABLE 1 #define AIE2_CLK_GATING_DISABLE 0 @@ -26,6 +27,22 @@ static int aie2_pm_set_clk_gating(struct amdxdna_dev_hdl *ndev, u32 val) return 0; } +int aie2_pm_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level) +{ + int ret; + + ret = amdxdna_pm_resume_get(ndev->xdna); + if (ret) + return ret; + + ret = ndev->priv->hw_ops.set_dpm(ndev, dpm_level); + if (!ret) + ndev->dpm_level = dpm_level; + amdxdna_pm_suspend_put(ndev->xdna); + + return ret; +} + int aie2_pm_init(struct amdxdna_dev_hdl *ndev) { int ret; @@ -50,6 +67,7 @@ int aie2_pm_init(struct amdxdna_dev_hdl *ndev) ret = ndev->priv->hw_ops.set_dpm(ndev, ndev->max_dpm_level); if (ret) return ret; + ndev->dpm_level = ndev->max_dpm_level; ret = aie2_pm_set_clk_gating(ndev, AIE2_CLK_GATING_ENABLE); if (ret) @@ -94,7 +112,7 @@ int aie2_pm_set_mode(struct amdxdna_dev_hdl *ndev, enum amdxdna_power_mode_type return -EOPNOTSUPP; } - ret = ndev->priv->hw_ops.set_dpm(ndev, dpm_level); + ret = aie2_pm_set_dpm(ndev, dpm_level); if (ret) return ret; diff --git a/drivers/accel/amdxdna/aie2_smu.c b/drivers/accel/amdxdna/aie2_smu.c index bd94ee96c2bc06..d8c31924e501ba 100644 --- a/drivers/accel/amdxdna/aie2_smu.c +++ b/drivers/accel/amdxdna/aie2_smu.c @@ -11,7 +11,6 @@ #include "aie2_pci.h" #include "amdxdna_pci_drv.h" -#include "amdxdna_pm.h" #define SMU_RESULT_OK 1 @@ -67,16 +66,12 @@ int npu1_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level) u32 freq; int ret; - ret = amdxdna_pm_resume_get(ndev->xdna); - if (ret) - return ret; - ret = aie2_smu_exec(ndev, AIE2_SMU_SET_MPNPUCLK_FREQ, ndev->priv->dpm_clk_tbl[dpm_level].npuclk, &freq); if (ret) { XDNA_ERR(ndev->xdna, "Set npu clock to %d failed, ret %d\n", ndev->priv->dpm_clk_tbl[dpm_level].npuclk, ret); - goto suspend_put; + return ret; } ndev->npuclk_freq = freq; @@ -85,12 +80,10 @@ int npu1_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level) if (ret) { XDNA_ERR(ndev->xdna, "Set h clock to %d failed, ret %d\n", ndev->priv->dpm_clk_tbl[dpm_level].hclk, ret); - goto suspend_put; + return ret; } - amdxdna_pm_suspend_put(ndev->xdna); ndev->hclk_freq = freq; - ndev->dpm_level = dpm_level; ndev->max_tops = 2 * ndev->total_col; ndev->curr_tops = ndev->max_tops * freq / 1028; @@ -98,38 +91,28 @@ int npu1_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level) ndev->npuclk_freq, ndev->hclk_freq); return 0; - -suspend_put: - amdxdna_pm_suspend_put(ndev->xdna); - return ret; } int npu4_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level) { int ret; - ret = amdxdna_pm_resume_get(ndev->xdna); - if (ret) - return ret; - ret = aie2_smu_exec(ndev, AIE2_SMU_SET_HARD_DPMLEVEL, dpm_level, NULL); if (ret) { XDNA_ERR(ndev->xdna, "Set hard dpm level %d failed, ret %d ", dpm_level, ret); - goto suspend_put; + return ret; } ret = aie2_smu_exec(ndev, AIE2_SMU_SET_SOFT_DPMLEVEL, dpm_level, NULL); if (ret) { XDNA_ERR(ndev->xdna, "Set soft dpm level %d failed, ret %d", dpm_level, ret); - goto suspend_put; + return ret; } - amdxdna_pm_suspend_put(ndev->xdna); ndev->npuclk_freq = ndev->priv->dpm_clk_tbl[dpm_level].npuclk; ndev->hclk_freq = ndev->priv->dpm_clk_tbl[dpm_level].hclk; - ndev->dpm_level = dpm_level; ndev->max_tops = NPU4_DPM_TOPS(ndev, ndev->max_dpm_level); ndev->curr_tops = NPU4_DPM_TOPS(ndev, dpm_level); @@ -137,10 +120,6 @@ int npu4_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level) ndev->npuclk_freq, ndev->hclk_freq); return 0; - -suspend_put: - amdxdna_pm_suspend_put(ndev->xdna); - return ret; } int aie2_smu_init(struct amdxdna_dev_hdl *ndev) diff --git a/drivers/accel/amdxdna/amdxdna_ctx.h b/drivers/accel/amdxdna/amdxdna_ctx.h index b6151244d64fe2..16c85f08f03c6e 100644 --- a/drivers/accel/amdxdna/amdxdna_ctx.h +++ b/drivers/accel/amdxdna/amdxdna_ctx.h @@ -98,11 +98,7 @@ struct amdxdna_hwctx { u32 *col_list; u32 start_col; u32 num_col; -#define HWCTX_STAT_INIT 0 -#define HWCTX_STAT_READY 1 -#define HWCTX_STAT_STOP 2 - u32 status; - u32 old_status; + u32 num_unused_col; struct amdxdna_qos_info qos; struct amdxdna_hwctx_param_config_cu *cus; diff --git a/drivers/accel/amdxdna/amdxdna_mailbox.c b/drivers/accel/amdxdna/amdxdna_mailbox.c index 858df97cd3fbd4..8b72cf6bd6e4d3 100644 --- a/drivers/accel/amdxdna/amdxdna_mailbox.c +++ b/drivers/accel/amdxdna/amdxdna_mailbox.c @@ -207,26 +207,34 @@ mailbox_send_msg(struct mailbox_channel *mb_chann, struct mailbox_msg *mb_msg) u32 head, tail; u32 start_addr; u32 tmp_tail; + int ret; head = mailbox_get_headptr(mb_chann, CHAN_RES_X2I); tail = mb_chann->x2i_tail; - ringbuf_size = mailbox_get_ringbuf_size(mb_chann, CHAN_RES_X2I); + ringbuf_size = mailbox_get_ringbuf_size(mb_chann, CHAN_RES_X2I) - sizeof(u32); start_addr = mb_chann->res[CHAN_RES_X2I].rb_start_addr; tmp_tail = tail + mb_msg->pkg_size; - if (tail < head && tmp_tail >= head) - goto no_space; - - if (tail >= head && (tmp_tail > ringbuf_size - sizeof(u32) && - mb_msg->pkg_size >= head)) - goto no_space; - if (tail >= head && tmp_tail > ringbuf_size - sizeof(u32)) { +check_again: + if (tail >= head && tmp_tail > ringbuf_size) { write_addr = mb_chann->mb->res.ringbuf_base + start_addr + tail; writel(TOMBSTONE, write_addr); /* tombstone is set. Write from the start of the ringbuf */ tail = 0; + tmp_tail = tail + mb_msg->pkg_size; + } + + if (tail < head && tmp_tail >= head) { + ret = read_poll_timeout(mailbox_get_headptr, head, + tmp_tail < head || tail >= head, + 1, 100, false, mb_chann, CHAN_RES_X2I); + if (ret) + return ret; + + if (tail >= head) + goto check_again; } write_addr = mb_chann->mb->res.ringbuf_base + start_addr + tail; @@ -238,9 +246,6 @@ mailbox_send_msg(struct mailbox_channel *mb_chann, struct mailbox_msg *mb_msg) mb_msg->pkg.header.id); return 0; - -no_space: - return -ENOSPC; } static int diff --git a/drivers/accel/amdxdna/amdxdna_pci_drv.c b/drivers/accel/amdxdna/amdxdna_pci_drv.c index 1973ab67721be0..bcb0d77b63cbb5 100644 --- a/drivers/accel/amdxdna/amdxdna_pci_drv.c +++ b/drivers/accel/amdxdna/amdxdna_pci_drv.c @@ -83,6 +83,8 @@ static int amdxdna_drm_open(struct drm_device *ddev, struct drm_file *filp) ret = -ENODEV; goto unbind_sva; } + client->mm = current->mm; + mmgrab(client->mm); init_srcu_struct(&client->hwctx_srcu); xa_init_flags(&client->hwctx_xa, XA_FLAGS_ALLOC); mutex_init(&client->mm_lock); @@ -119,6 +121,7 @@ static void amdxdna_drm_close(struct drm_device *ddev, struct drm_file *filp) drm_gem_object_put(to_gobj(client->dev_heap)); iommu_sva_unbind_device(client->sva); + mmdrop(client->mm); XDNA_DBG(xdna, "pid %d closed", client->pid); kfree(client); @@ -282,7 +285,7 @@ static int amdxdna_probe(struct pci_dev *pdev, const struct pci_device_id *id) fs_reclaim_release(GFP_KERNEL); } - xdna->notifier_wq = alloc_ordered_workqueue("notifier_wq", 0); + xdna->notifier_wq = alloc_ordered_workqueue("notifier_wq", WQ_MEM_RECLAIM); if (!xdna->notifier_wq) return -ENOMEM; diff --git a/drivers/accel/amdxdna/amdxdna_pci_drv.h b/drivers/accel/amdxdna/amdxdna_pci_drv.h index c99477f5e454f0..ec21cb378a472a 100644 --- a/drivers/accel/amdxdna/amdxdna_pci_drv.h +++ b/drivers/accel/amdxdna/amdxdna_pci_drv.h @@ -101,7 +101,6 @@ struct amdxdna_dev { struct amdxdna_fw_ver fw_ver; struct rw_semaphore notifier_lock; /* for mmu notifier*/ struct workqueue_struct *notifier_wq; - bool rpm_on; }; /* @@ -131,6 +130,7 @@ struct amdxdna_client { struct iommu_sva *sva; int pasid; + struct mm_struct *mm; }; #define amdxdna_for_each_hwctx(client, hwctx_id, entry) \ diff --git a/drivers/accel/amdxdna/amdxdna_pm.c b/drivers/accel/amdxdna/amdxdna_pm.c index fa38e65d617c4b..d024d480521c49 100644 --- a/drivers/accel/amdxdna/amdxdna_pm.c +++ b/drivers/accel/amdxdna/amdxdna_pm.c @@ -15,14 +15,9 @@ int amdxdna_pm_suspend(struct device *dev) { struct amdxdna_dev *xdna = to_xdna_dev(dev_get_drvdata(dev)); int ret = -EOPNOTSUPP; - bool rpm; - if (xdna->dev_info->ops->suspend) { - rpm = xdna->rpm_on; - xdna->rpm_on = false; + if (xdna->dev_info->ops->suspend) ret = xdna->dev_info->ops->suspend(xdna); - xdna->rpm_on = rpm; - } XDNA_DBG(xdna, "Suspend done ret %d", ret); return ret; @@ -32,14 +27,9 @@ int amdxdna_pm_resume(struct device *dev) { struct amdxdna_dev *xdna = to_xdna_dev(dev_get_drvdata(dev)); int ret = -EOPNOTSUPP; - bool rpm; - if (xdna->dev_info->ops->resume) { - rpm = xdna->rpm_on; - xdna->rpm_on = false; + if (xdna->dev_info->ops->resume) ret = xdna->dev_info->ops->resume(xdna); - xdna->rpm_on = rpm; - } XDNA_DBG(xdna, "Resume done ret %d", ret); return ret; @@ -50,9 +40,6 @@ int amdxdna_pm_resume_get(struct amdxdna_dev *xdna) struct device *dev = xdna->ddev.dev; int ret; - if (!xdna->rpm_on) - return 0; - ret = pm_runtime_resume_and_get(dev); if (ret) { XDNA_ERR(xdna, "Resume failed: %d", ret); @@ -66,9 +53,6 @@ void amdxdna_pm_suspend_put(struct amdxdna_dev *xdna) { struct device *dev = xdna->ddev.dev; - if (!xdna->rpm_on) - return; - pm_runtime_put_autosuspend(dev); } @@ -81,14 +65,12 @@ void amdxdna_pm_init(struct amdxdna_dev *xdna) pm_runtime_use_autosuspend(dev); pm_runtime_allow(dev); pm_runtime_put_autosuspend(dev); - xdna->rpm_on = true; } void amdxdna_pm_fini(struct amdxdna_dev *xdna) { struct device *dev = xdna->ddev.dev; - xdna->rpm_on = false; pm_runtime_get_noresume(dev); pm_runtime_forbid(dev); } diff --git a/drivers/accel/amdxdna/amdxdna_ubuf.c b/drivers/accel/amdxdna/amdxdna_ubuf.c index 077b2261cf2a04..9e3b3b055caa86 100644 --- a/drivers/accel/amdxdna/amdxdna_ubuf.c +++ b/drivers/accel/amdxdna/amdxdna_ubuf.c @@ -34,15 +34,21 @@ static struct sg_table *amdxdna_ubuf_map(struct dma_buf_attachment *attach, ret = sg_alloc_table_from_pages(sg, ubuf->pages, ubuf->nr_pages, 0, ubuf->nr_pages << PAGE_SHIFT, GFP_KERNEL); if (ret) - return ERR_PTR(ret); + goto err_free_sg; if (ubuf->flags & AMDXDNA_UBUF_FLAG_MAP_DMA) { ret = dma_map_sgtable(attach->dev, sg, direction, 0); if (ret) - return ERR_PTR(ret); + goto err_free_table; } return sg; + +err_free_table: + sg_free_table(sg); +err_free_sg: + kfree(sg); + return ERR_PTR(ret); } static void amdxdna_ubuf_unmap(struct dma_buf_attachment *attach, diff --git a/drivers/accel/amdxdna/npu4_regs.c b/drivers/accel/amdxdna/npu4_regs.c index 986a5f28ba2452..2ceedfe583a8c9 100644 --- a/drivers/accel/amdxdna/npu4_regs.c +++ b/drivers/accel/amdxdna/npu4_regs.c @@ -89,6 +89,7 @@ const struct dpm_clk_freq npu4_dpm_clk_table[] = { const struct aie2_fw_feature_tbl npu4_fw_feature_table[] = { { .feature = AIE2_NPU_COMMAND, .min_minor = 15 }, { .feature = AIE2_PREEMPT, .min_minor = 12 }, + { .feature = AIE2_TEMPORAL_ONLY, .min_minor = 12 }, { 0 } }; diff --git a/drivers/acpi/acpica/evregion.c b/drivers/acpi/acpica/evregion.c index fa3475da7ea9b6..b6198f73c81dfa 100644 --- a/drivers/acpi/acpica/evregion.c +++ b/drivers/acpi/acpica/evregion.c @@ -163,7 +163,9 @@ acpi_ev_address_space_dispatch(union acpi_operand_object *region_obj, return_ACPI_STATUS(AE_NOT_EXIST); } - if (region_obj->region.space_id == ACPI_ADR_SPACE_PLATFORM_COMM) { + if (field_obj + && region_obj->region.space_id == + ACPI_ADR_SPACE_PLATFORM_COMM) { struct acpi_pcc_info *ctx = handler_desc->address_space.context; diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index e66e20d1f31b76..b59b0100d03c51 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -362,7 +362,7 @@ static int send_pcc_cmd(int pcc_ss_id, u16 cmd) end: if (cmd == CMD_WRITE) { if (unlikely(ret)) { - for_each_possible_cpu(i) { + for_each_online_cpu(i) { struct cpc_desc *desc = per_cpu(cpc_desc_ptr, i); if (!desc) @@ -524,7 +524,7 @@ int acpi_get_psd_map(unsigned int cpu, struct cppc_cpudata *cpu_data) else if (pdomain->coord_type == DOMAIN_COORD_TYPE_SW_ANY) cpu_data->shared_type = CPUFREQ_SHARED_TYPE_ANY; - for_each_possible_cpu(i) { + for_each_online_cpu(i) { if (i == cpu) continue; diff --git a/drivers/acpi/power.c b/drivers/acpi/power.c index 361a7721a6a873..7da5ae5594a727 100644 --- a/drivers/acpi/power.c +++ b/drivers/acpi/power.c @@ -1113,6 +1113,19 @@ static const struct dmi_system_id dmi_leave_unused_power_resources_on[] = { DMI_MATCH(DMI_PRODUCT_NAME, "SATELLITE Click Mini L9W-B"), }, }, + { + /* + * THUNDEROBOT ZERO laptop: Due to its SSDT table bug, power + * resource 'PXP' will be shut down on initialization, making + * the NVMe #2 and the NVIDIA dGPU both unavailable (they're + * both controlled by 'PXP'). + */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "THUNDEROBOT"), + DMI_MATCH(DMI_PRODUCT_NAME, "ZERO"), + } + + }, {} }; diff --git a/drivers/acpi/processor_driver.c b/drivers/acpi/processor_driver.c index 65e779be64ffcc..7644de24d2faa4 100644 --- a/drivers/acpi/processor_driver.c +++ b/drivers/acpi/processor_driver.c @@ -166,7 +166,7 @@ static int __acpi_processor_start(struct acpi_device *device) if (result && !IS_ENABLED(CONFIG_ACPI_CPU_FREQ_PSS)) dev_dbg(&device->dev, "CPPC data invalid or not present\n"); - if (!cpuidle_get_driver() || cpuidle_get_driver() == &acpi_idle_driver) + if (cpuidle_get_driver() == &acpi_idle_driver) acpi_processor_power_init(pr); acpi_pss_perf_init(pr); diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index ddf9a7b28a5943..50dfce8d8bba09 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -77,6 +77,7 @@ static unsigned int ata_dev_init_params(struct ata_device *dev, static unsigned int ata_dev_set_xfermode(struct ata_device *dev); static void ata_dev_xfermask(struct ata_device *dev); static unsigned int ata_dev_quirks(const struct ata_device *dev); +static u64 ata_dev_get_quirk_value(struct ata_device *dev, unsigned int quirk); static DEFINE_IDA(ata_ida); @@ -3152,9 +3153,10 @@ int ata_dev_configure(struct ata_device *dev) dev->max_sectors = min_t(unsigned int, ATA_MAX_SECTORS_1024, dev->max_sectors); - if (dev->quirks & ATA_QUIRK_MAX_SEC_8191) - dev->max_sectors = min_t(unsigned int, ATA_MAX_SECTORS_8191, - dev->max_sectors); + if (dev->quirks & ATA_QUIRK_MAX_SEC) + dev->max_sectors = min_t(unsigned int, dev->max_sectors, + ata_dev_get_quirk_value(dev, + ATA_QUIRK_MAX_SEC)); if (dev->quirks & ATA_QUIRK_MAX_SEC_LBA48) dev->max_sectors = ATA_MAX_SECTORS_LBA48; @@ -4008,7 +4010,7 @@ static const char * const ata_quirk_names[] = { [__ATA_QUIRK_NO_DMA_LOG] = "nodmalog", [__ATA_QUIRK_NOTRIM] = "notrim", [__ATA_QUIRK_MAX_SEC_1024] = "maxsec1024", - [__ATA_QUIRK_MAX_SEC_8191] = "maxsec8191", + [__ATA_QUIRK_MAX_SEC] = "maxsec", [__ATA_QUIRK_MAX_TRIM_128M] = "maxtrim128m", [__ATA_QUIRK_NO_NCQ_ON_ATI] = "noncqonati", [__ATA_QUIRK_NO_LPM_ON_ATI] = "nolpmonati", @@ -4053,6 +4055,22 @@ static void ata_dev_print_quirks(const struct ata_device *dev, kfree(str); } +struct ata_dev_quirk_value { + const char *model_num; + const char *model_rev; + u64 val; +}; + +static const struct ata_dev_quirk_value __ata_dev_max_sec_quirks[] = { + { "TORiSAN DVD-ROM DRD-N216", NULL, 128 }, + { "ST380013AS", "3.20", 1024 }, + { "LITEON CX1-JB*-HP", NULL, 1024 }, + { "LITEON EP1-*", NULL, 1024 }, + { "DELLBOSS VD", "MV.R00-0", 8191 }, + { "INTEL SSDSC2KG480G8", "XCV10120", 8191 }, + { }, +}; + struct ata_dev_quirks_entry { const char *model_num; const char *model_rev; @@ -4097,7 +4115,7 @@ static const struct ata_dev_quirks_entry __ata_dev_quirks[] = { { "ASMT109x- Config", NULL, ATA_QUIRK_DISABLE }, /* Weird ATAPI devices */ - { "TORiSAN DVD-ROM DRD-N216", NULL, ATA_QUIRK_MAX_SEC_128 }, + { "TORiSAN DVD-ROM DRD-N216", NULL, ATA_QUIRK_MAX_SEC }, { "QUANTUM DAT DAT72-000", NULL, ATA_QUIRK_ATAPI_MOD16_DMA }, { "Slimtype DVD A DS8A8SH", NULL, ATA_QUIRK_MAX_SEC_LBA48 }, { "Slimtype DVD A DS8A9SH", NULL, ATA_QUIRK_MAX_SEC_LBA48 }, @@ -4106,20 +4124,20 @@ static const struct ata_dev_quirks_entry __ata_dev_quirks[] = { * Causes silent data corruption with higher max sects. * http://lkml.kernel.org/g/x49wpy40ysk.fsf@segfault.boston.devel.redhat.com */ - { "ST380013AS", "3.20", ATA_QUIRK_MAX_SEC_1024 }, + { "ST380013AS", "3.20", ATA_QUIRK_MAX_SEC }, /* * These devices time out with higher max sects. * https://bugzilla.kernel.org/show_bug.cgi?id=121671 */ - { "LITEON CX1-JB*-HP", NULL, ATA_QUIRK_MAX_SEC_1024 }, - { "LITEON EP1-*", NULL, ATA_QUIRK_MAX_SEC_1024 }, + { "LITEON CX1-JB*-HP", NULL, ATA_QUIRK_MAX_SEC }, + { "LITEON EP1-*", NULL, ATA_QUIRK_MAX_SEC }, /* * These devices time out with higher max sects. * https://bugzilla.kernel.org/show_bug.cgi?id=220693 */ - { "DELLBOSS VD", "MV.R00-0", ATA_QUIRK_MAX_SEC_8191 }, + { "DELLBOSS VD", "MV.R00-0", ATA_QUIRK_MAX_SEC }, /* Devices we expect to fail diagnostics */ @@ -4307,6 +4325,8 @@ static const struct ata_dev_quirks_entry __ata_dev_quirks[] = { { "Micron*", NULL, ATA_QUIRK_ZERO_AFTER_TRIM }, { "Crucial*", NULL, ATA_QUIRK_ZERO_AFTER_TRIM }, + { "INTEL SSDSC2KG480G8", "XCV10120", ATA_QUIRK_ZERO_AFTER_TRIM | + ATA_QUIRK_MAX_SEC }, { "INTEL*SSD*", NULL, ATA_QUIRK_ZERO_AFTER_TRIM }, { "SSD*INTEL*", NULL, ATA_QUIRK_ZERO_AFTER_TRIM }, { "Samsung*SSD*", NULL, ATA_QUIRK_ZERO_AFTER_TRIM }, @@ -4372,6 +4392,39 @@ static unsigned int ata_dev_quirks(const struct ata_device *dev) return 0; } +static u64 ata_dev_get_max_sec_quirk_value(struct ata_device *dev) +{ + unsigned char model_num[ATA_ID_PROD_LEN + 1]; + unsigned char model_rev[ATA_ID_FW_REV_LEN + 1]; + const struct ata_dev_quirk_value *ad = __ata_dev_max_sec_quirks; + u64 val = 0; + + ata_id_c_string(dev->id, model_num, ATA_ID_PROD, sizeof(model_num)); + ata_id_c_string(dev->id, model_rev, ATA_ID_FW_REV, sizeof(model_rev)); + + while (ad->model_num) { + if (glob_match(ad->model_num, model_num) && + (!ad->model_rev || glob_match(ad->model_rev, model_rev))) { + val = ad->val; + break; + } + ad++; + } + + ata_dev_warn(dev, "%s quirk is using value: %llu\n", + ata_quirk_names[__ATA_QUIRK_MAX_SEC], val); + + return val; +} + +static u64 ata_dev_get_quirk_value(struct ata_device *dev, unsigned int quirk) +{ + if (quirk == ATA_QUIRK_MAX_SEC) + return ata_dev_get_max_sec_quirk_value(dev); + + return 0; +} + static bool ata_dev_nodma(const struct ata_device *dev) { /* @@ -5567,6 +5620,7 @@ struct ata_port *ata_port_alloc(struct ata_host *host) mutex_init(&ap->scsi_scan_mutex); INIT_DELAYED_WORK(&ap->hotplug_task, ata_scsi_hotplug); INIT_DELAYED_WORK(&ap->scsi_rescan_task, ata_scsi_dev_rescan); + INIT_WORK(&ap->deferred_qc_work, ata_scsi_deferred_qc_work); INIT_LIST_HEAD(&ap->eh_done_q); init_waitqueue_head(&ap->eh_wait_q); init_completion(&ap->park_req_pending); @@ -6189,9 +6243,11 @@ static void ata_port_detach(struct ata_port *ap) /* wait till EH commits suicide */ ata_port_wait_eh(ap); - /* it better be dead now */ + /* It better be dead now and not have any remaining deferred qc. */ WARN_ON(!(ap->pflags & ATA_PFLAG_UNLOADED)); + WARN_ON(ap->deferred_qc); + cancel_work_sync(&ap->deferred_qc_work); cancel_delayed_work_sync(&ap->hotplug_task); cancel_delayed_work_sync(&ap->scsi_rescan_task); diff --git a/drivers/ata/libata-eh.c b/drivers/ata/libata-eh.c index 2586e77ebf45d0..258e657f3527c2 100644 --- a/drivers/ata/libata-eh.c +++ b/drivers/ata/libata-eh.c @@ -640,12 +640,28 @@ void ata_scsi_cmd_error_handler(struct Scsi_Host *host, struct ata_port *ap, set_host_byte(scmd, DID_OK); ata_qc_for_each_raw(ap, qc, i) { - if (qc->flags & ATA_QCFLAG_ACTIVE && - qc->scsicmd == scmd) + if (qc->scsicmd != scmd) + continue; + if ((qc->flags & ATA_QCFLAG_ACTIVE) || + qc == ap->deferred_qc) break; } - if (i < ATA_MAX_QUEUE) { + if (qc == ap->deferred_qc) { + /* + * This is a deferred command that timed out while + * waiting for the command queue to drain. Since the qc + * is not active yet (deferred_qc is still set, so the + * deferred qc work has not issued the command yet), + * simply signal the timeout by finishing the SCSI + * command and clear the deferred qc to prevent the + * deferred qc work from issuing this qc. + */ + WARN_ON_ONCE(qc->flags & ATA_QCFLAG_ACTIVE); + ap->deferred_qc = NULL; + set_host_byte(scmd, DID_TIME_OUT); + scsi_eh_finish_cmd(scmd, &ap->eh_done_q); + } else if (i < ATA_MAX_QUEUE) { /* the scmd has an associated qc */ if (!(qc->flags & ATA_QCFLAG_EH)) { /* which hasn't failed yet, timeout */ @@ -917,6 +933,12 @@ static void ata_eh_set_pending(struct ata_port *ap, bool fastdrain) ap->pflags |= ATA_PFLAG_EH_PENDING; + /* + * If we have a deferred qc, requeue it so that it is retried once EH + * completes. + */ + ata_scsi_requeue_deferred_qc(ap); + if (!fastdrain) return; diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c index 721d3f270c8ec6..5f9abeb7b2a882 100644 --- a/drivers/ata/libata-scsi.c +++ b/drivers/ata/libata-scsi.c @@ -1658,8 +1658,77 @@ static void ata_qc_done(struct ata_queued_cmd *qc) done(cmd); } +void ata_scsi_deferred_qc_work(struct work_struct *work) +{ + struct ata_port *ap = + container_of(work, struct ata_port, deferred_qc_work); + struct ata_queued_cmd *qc; + unsigned long flags; + + spin_lock_irqsave(ap->lock, flags); + + /* + * If we still have a deferred qc and we are not in EH, issue it. In + * such case, we should not need any more deferring the qc, so warn if + * qc_defer() says otherwise. + */ + qc = ap->deferred_qc; + if (qc && !ata_port_eh_scheduled(ap)) { + WARN_ON_ONCE(ap->ops->qc_defer(qc)); + ap->deferred_qc = NULL; + ata_qc_issue(qc); + } + + spin_unlock_irqrestore(ap->lock, flags); +} + +void ata_scsi_requeue_deferred_qc(struct ata_port *ap) +{ + struct ata_queued_cmd *qc = ap->deferred_qc; + struct scsi_cmnd *scmd; + + lockdep_assert_held(ap->lock); + + /* + * If we have a deferred qc when a reset occurs or NCQ commands fail, + * do not try to be smart about what to do with this deferred command + * and simply retry it by completing it with DID_SOFT_ERROR. + */ + if (!qc) + return; + + scmd = qc->scsicmd; + ap->deferred_qc = NULL; + ata_qc_free(qc); + scmd->result = (DID_SOFT_ERROR << 16); + scsi_done(scmd); +} + +static void ata_scsi_schedule_deferred_qc(struct ata_port *ap) +{ + struct ata_queued_cmd *qc = ap->deferred_qc; + + lockdep_assert_held(ap->lock); + + /* + * If we have a deferred qc, then qc_defer() is defined and we can use + * this callback to determine if this qc is good to go, unless EH has + * been scheduled. + */ + if (!qc) + return; + + if (ata_port_eh_scheduled(ap)) { + ata_scsi_requeue_deferred_qc(ap); + return; + } + if (!ap->ops->qc_defer(qc)) + queue_work(system_highpri_wq, &ap->deferred_qc_work); +} + static void ata_scsi_qc_complete(struct ata_queued_cmd *qc) { + struct ata_port *ap = qc->ap; struct scsi_cmnd *cmd = qc->scsicmd; u8 *cdb = cmd->cmnd; bool have_sense = qc->flags & ATA_QCFLAG_SENSE_VALID; @@ -1689,6 +1758,66 @@ static void ata_scsi_qc_complete(struct ata_queued_cmd *qc) } ata_qc_done(qc); + + ata_scsi_schedule_deferred_qc(ap); +} + +static int ata_scsi_qc_issue(struct ata_port *ap, struct ata_queued_cmd *qc) +{ + int ret; + + if (!ap->ops->qc_defer) + goto issue; + + /* + * If we already have a deferred qc, then rely on the SCSI layer to + * requeue and defer all incoming commands until the deferred qc is + * processed, once all on-going commands complete. + */ + if (ap->deferred_qc) { + ata_qc_free(qc); + return SCSI_MLQUEUE_DEVICE_BUSY; + } + + /* Check if the command needs to be deferred. */ + ret = ap->ops->qc_defer(qc); + switch (ret) { + case 0: + break; + case ATA_DEFER_LINK: + ret = SCSI_MLQUEUE_DEVICE_BUSY; + break; + case ATA_DEFER_PORT: + ret = SCSI_MLQUEUE_HOST_BUSY; + break; + default: + WARN_ON_ONCE(1); + ret = SCSI_MLQUEUE_HOST_BUSY; + break; + } + + if (ret) { + /* + * We must defer this qc: if this is not an NCQ command, keep + * this qc as a deferred one and report to the SCSI layer that + * we issued it so that it is not requeued. The deferred qc will + * be issued with the port deferred_qc_work once all on-going + * commands complete. + */ + if (!ata_is_ncq(qc->tf.protocol)) { + ap->deferred_qc = qc; + return 0; + } + + /* Force a requeue of the command to defer its execution. */ + ata_qc_free(qc); + return ret; + } + +issue: + ata_qc_issue(qc); + + return 0; } /** @@ -1714,66 +1843,49 @@ static void ata_scsi_qc_complete(struct ata_queued_cmd *qc) * spin_lock_irqsave(host lock) * * RETURNS: - * 0 on success, SCSI_ML_QUEUE_DEVICE_BUSY if the command - * needs to be deferred. + * 0 on success, SCSI_ML_QUEUE_DEVICE_BUSY or SCSI_MLQUEUE_HOST_BUSY if the + * command needs to be deferred. */ static int ata_scsi_translate(struct ata_device *dev, struct scsi_cmnd *cmd, ata_xlat_func_t xlat_func) { struct ata_port *ap = dev->link->ap; struct ata_queued_cmd *qc; - int rc; + lockdep_assert_held(ap->lock); + + /* + * ata_scsi_qc_new() calls scsi_done(cmd) in case of failure. So we + * have nothing further to do when allocating a qc fails. + */ qc = ata_scsi_qc_new(dev, cmd); if (!qc) - goto err_mem; + return 0; /* data is present; dma-map it */ if (cmd->sc_data_direction == DMA_FROM_DEVICE || cmd->sc_data_direction == DMA_TO_DEVICE) { if (unlikely(scsi_bufflen(cmd) < 1)) { ata_dev_warn(dev, "WARNING: zero len r/w req\n"); - goto err_did; + cmd->result = (DID_ERROR << 16); + goto done; } ata_sg_init(qc, scsi_sglist(cmd), scsi_sg_count(cmd)); - qc->dma_dir = cmd->sc_data_direction; } qc->complete_fn = ata_scsi_qc_complete; if (xlat_func(qc)) - goto early_finish; + goto done; - if (ap->ops->qc_defer) { - if ((rc = ap->ops->qc_defer(qc))) - goto defer; - } - - /* select device, send command to hardware */ - ata_qc_issue(qc); + return ata_scsi_qc_issue(ap, qc); - return 0; - -early_finish: - ata_qc_free(qc); - scsi_done(cmd); - return 0; - -err_did: +done: ata_qc_free(qc); - cmd->result = (DID_ERROR << 16); scsi_done(cmd); -err_mem: return 0; - -defer: - ata_qc_free(qc); - if (rc == ATA_DEFER_LINK) - return SCSI_MLQUEUE_DEVICE_BUSY; - else - return SCSI_MLQUEUE_HOST_BUSY; } /** diff --git a/drivers/ata/libata.h b/drivers/ata/libata.h index 0e7ecac7368098..60a675df61dc77 100644 --- a/drivers/ata/libata.h +++ b/drivers/ata/libata.h @@ -165,6 +165,8 @@ void ata_scsi_sdev_config(struct scsi_device *sdev); int ata_scsi_dev_config(struct scsi_device *sdev, struct queue_limits *lim, struct ata_device *dev); int __ata_scsi_queuecmd(struct scsi_cmnd *scmd, struct ata_device *dev); +void ata_scsi_deferred_qc_work(struct work_struct *work); +void ata_scsi_requeue_deferred_qc(struct ata_port *ap); /* libata-eh.c */ extern unsigned int ata_internal_cmd_timeout(struct ata_device *dev, u8 cmd); diff --git a/drivers/ata/pata_ftide010.c b/drivers/ata/pata_ftide010.c index c3a8384c3e04d4..c41da296eb389d 100644 --- a/drivers/ata/pata_ftide010.c +++ b/drivers/ata/pata_ftide010.c @@ -122,10 +122,10 @@ static const u8 mwdma_50_active_time[3] = {6, 2, 2}; static const u8 mwdma_50_recovery_time[3] = {6, 2, 1}; static const u8 mwdma_66_active_time[3] = {8, 3, 3}; static const u8 mwdma_66_recovery_time[3] = {8, 2, 1}; -static const u8 udma_50_setup_time[6] = {3, 3, 2, 2, 1, 1}; +static const u8 udma_50_setup_time[6] = {3, 3, 2, 2, 1, 9}; static const u8 udma_50_hold_time[6] = {3, 1, 1, 1, 1, 1}; -static const u8 udma_66_setup_time[7] = {4, 4, 3, 2, }; -static const u8 udma_66_hold_time[7] = {}; +static const u8 udma_66_setup_time[7] = {4, 4, 3, 2, 1, 9, 9}; +static const u8 udma_66_hold_time[7] = {4, 2, 1, 1, 1, 1, 1}; /* * We set 66 MHz for all MWDMA modes diff --git a/drivers/auxdisplay/arm-charlcd.c b/drivers/auxdisplay/arm-charlcd.c index a7eae99a48f77d..4e22882f57c9c2 100644 --- a/drivers/auxdisplay/arm-charlcd.c +++ b/drivers/auxdisplay/arm-charlcd.c @@ -323,7 +323,7 @@ static int __init charlcd_probe(struct platform_device *pdev) out_no_irq: iounmap(lcd->virtbase); out_no_memregion: - release_mem_region(lcd->phybase, SZ_4K); + release_mem_region(lcd->phybase, lcd->physize); out_no_resource: kfree(lcd); return ret; diff --git a/drivers/base/core.c b/drivers/base/core.c index 40de2f51a1b1ab..4ad9b291153119 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -2328,6 +2328,32 @@ static void fw_devlink_link_device(struct device *dev) __fw_devlink_link_to_suppliers(dev, fwnode); } +/** + * fw_devlink_count_absent_consumers - Return how many consumers have + * either not been created yet, or do not yet have a driver attached. + * @fwnode: fwnode of the supplier + */ +int fw_devlink_count_absent_consumers(struct fwnode_handle *fwnode) +{ + struct fwnode_link *link, *tmp; + struct device_link *dlink, *dtmp; + struct device *sup_dev = get_dev_from_fwnode(fwnode); + int count = 0; + + list_for_each_entry_safe(link, tmp, &fwnode->consumers, s_hook) + count++; + + if (!sup_dev) + return count; + + list_for_each_entry_safe(dlink, dtmp, &sup_dev->links.consumers, s_node) + if (dlink->consumer->links.status != DL_DEV_DRIVER_BOUND) + count++; + + return count; +} +EXPORT_SYMBOL_GPL(fw_devlink_count_absent_consumers); + /* Device links support end. */ static struct kobject *dev_kobj; diff --git a/drivers/base/firmware_loader/main.c b/drivers/base/firmware_loader/main.c index 4ebdca9e4da47f..b6aabfc70732de 100644 --- a/drivers/base/firmware_loader/main.c +++ b/drivers/base/firmware_loader/main.c @@ -471,6 +471,8 @@ static int fw_decompress_xz(struct device *dev, struct fw_priv *fw_priv, static char fw_path_para[256]; static const char * const fw_path[] = { fw_path_para, + "/lib/firmware/vendor/" UTS_RELEASE, + "/lib/firmware/vendor", "/lib/firmware/updates/" UTS_RELEASE, "/lib/firmware/updates", "/lib/firmware/" UTS_RELEASE, diff --git a/drivers/base/power/wakeirq.c b/drivers/base/power/wakeirq.c index 8aa28c08b2891f..c0809d18fc5406 100644 --- a/drivers/base/power/wakeirq.c +++ b/drivers/base/power/wakeirq.c @@ -83,13 +83,16 @@ EXPORT_SYMBOL_GPL(dev_pm_set_wake_irq); */ void dev_pm_clear_wake_irq(struct device *dev) { - struct wake_irq *wirq = dev->power.wakeirq; + struct wake_irq *wirq; unsigned long flags; - if (!wirq) + spin_lock_irqsave(&dev->power.lock, flags); + wirq = dev->power.wakeirq; + if (!wirq) { + spin_unlock_irqrestore(&dev->power.lock, flags); return; + } - spin_lock_irqsave(&dev->power.lock, flags); device_wakeup_detach_irq(dev); dev->power.wakeirq = NULL; spin_unlock_irqrestore(&dev->power.lock, flags); diff --git a/drivers/base/power/wakeup.c b/drivers/base/power/wakeup.c index 1e1a0e7eeac5fa..e69033d16fba04 100644 --- a/drivers/base/power/wakeup.c +++ b/drivers/base/power/wakeup.c @@ -275,9 +275,7 @@ EXPORT_SYMBOL_GPL(wakeup_sources_read_unlock); */ struct wakeup_source *wakeup_sources_walk_start(void) { - struct list_head *ws_head = &wakeup_sources; - - return list_entry_rcu(ws_head->next, struct wakeup_source, entry); + return list_first_or_null_rcu(&wakeup_sources, struct wakeup_source, entry); } EXPORT_SYMBOL_GPL(wakeup_sources_walk_start); diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c index c73376886e7a51..1f6ac9202b66a1 100644 --- a/drivers/block/drbd/drbd_main.c +++ b/drivers/block/drbd/drbd_main.c @@ -2659,9 +2659,6 @@ enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsig * connect. */ .max_hw_sectors = DRBD_MAX_BIO_SIZE_SAFE >> 8, - .features = BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA | - BLK_FEAT_ROTATIONAL | - BLK_FEAT_STABLE_WRITES, }; device = minor_to_device(minor); diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c index 91f3b8afb63ce6..b502038be0a923 100644 --- a/drivers/block/drbd/drbd_nl.c +++ b/drivers/block/drbd/drbd_nl.c @@ -1296,6 +1296,8 @@ void drbd_reconsider_queue_parameters(struct drbd_device *device, lim.max_segments = drbd_backing_dev_max_segments(device); } else { lim.max_segments = BLK_MAX_SEGMENTS; + lim.features = BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA | + BLK_FEAT_ROTATIONAL | BLK_FEAT_STABLE_WRITES; } lim.max_hw_sectors = new >> SECTOR_SHIFT; @@ -1318,8 +1320,24 @@ void drbd_reconsider_queue_parameters(struct drbd_device *device, lim.max_hw_discard_sectors = 0; } - if (bdev) + if (bdev) { blk_stack_limits(&lim, &b->limits, 0); + /* + * blk_set_stacking_limits() cleared the features, and + * blk_stack_limits() may or may not have inherited + * BLK_FEAT_STABLE_WRITES from the backing device. + * + * DRBD always requires stable writes because: + * 1. The same bio data is read for both local disk I/O and + * network transmission. If the page changes mid-flight, + * the local and remote copies could diverge. + * 2. When data integrity is enabled, DRBD calculates a + * checksum before sending the data. If the page changes + * between checksum calculation and transmission, the + * receiver will detect a checksum mismatch. + */ + lim.features |= BLK_FEAT_STABLE_WRITES; + } /* * If we can handle "zeroes" efficiently on the protocol, we want to do diff --git a/drivers/block/rnbd/rnbd-srv.c b/drivers/block/rnbd/rnbd-srv.c index 2df8941a6b1463..9b3fdc202e1521 100644 --- a/drivers/block/rnbd/rnbd-srv.c +++ b/drivers/block/rnbd/rnbd-srv.c @@ -145,18 +145,30 @@ static int process_rdma(struct rnbd_srv_session *srv_sess, priv->sess_dev = sess_dev; priv->id = id; - bio = bio_alloc(file_bdev(sess_dev->bdev_file), 1, + bio = bio_alloc(file_bdev(sess_dev->bdev_file), !!datalen, rnbd_to_bio_flags(le32_to_cpu(msg->rw)), GFP_KERNEL); - bio_add_virt_nofail(bio, data, datalen); - - bio->bi_opf = rnbd_to_bio_flags(le32_to_cpu(msg->rw)); - if (bio_has_data(bio) && - bio->bi_iter.bi_size != le32_to_cpu(msg->bi_size)) { - rnbd_srv_err_rl(sess_dev, "Datalen mismatch: bio bi_size (%u), bi_size (%u)\n", - bio->bi_iter.bi_size, msg->bi_size); - err = -EINVAL; - goto bio_put; + if (unlikely(!bio)) { + err = -ENOMEM; + goto put_sess_dev; } + + if (!datalen) { + /* + * For special requests like DISCARD and WRITE_ZEROES, the datalen is zero. + */ + bio->bi_iter.bi_size = le32_to_cpu(msg->bi_size); + } else { + bio_add_virt_nofail(bio, data, datalen); + bio->bi_opf = rnbd_to_bio_flags(le32_to_cpu(msg->rw)); + if (bio->bi_iter.bi_size != le32_to_cpu(msg->bi_size)) { + rnbd_srv_err_rl(sess_dev, + "Datalen mismatch: bio bi_size (%u), bi_size (%u)\n", + bio->bi_iter.bi_size, msg->bi_size); + err = -EINVAL; + goto bio_put; + } + } + bio->bi_end_io = rnbd_dev_bi_end_io; bio->bi_private = priv; bio->bi_iter.bi_sector = le64_to_cpu(msg->sector); @@ -170,6 +182,7 @@ static int process_rdma(struct rnbd_srv_session *srv_sess, bio_put: bio_put(bio); +put_sess_dev: rnbd_put_sess_dev(sess_dev); err: kfree(priv); diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index cd1e84653002d5..06e0790150d1d1 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -2462,11 +2462,11 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd, io->res = result; req = ublk_fill_io_cmd(io, cmd); ret = ublk_config_io_buf(ub, io, cmd, addr, &buf_idx); + if (buf_idx != UBLK_INVALID_BUF_IDX) + io_buffer_unregister_bvec(cmd, buf_idx, issue_flags); compl = ublk_need_complete_req(ub, io); /* can't touch 'ublk_io' any more */ - if (buf_idx != UBLK_INVALID_BUF_IDX) - io_buffer_unregister_bvec(cmd, buf_idx, issue_flags); if (req_op(req) == REQ_OP_ZONE_APPEND) req->__sector = addr; if (compl) @@ -3311,12 +3311,11 @@ static int ublk_ctrl_del_dev(struct ublk_device **p_ub, bool wait) return 0; } -static inline void ublk_ctrl_cmd_dump(struct io_uring_cmd *cmd) +static inline void ublk_ctrl_cmd_dump(u32 cmd_op, + const struct ublksrv_ctrl_cmd *header) { - const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe); - pr_devel("%s: cmd_op %x, dev id %d qid %d data %llx buf %llx len %u\n", - __func__, cmd->cmd_op, header->dev_id, header->queue_id, + __func__, cmd_op, header->dev_id, header->queue_id, header->data[0], header->addr, header->len); } @@ -3685,9 +3684,8 @@ static int ublk_char_dev_permission(struct ublk_device *ub, } static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub, - struct io_uring_cmd *cmd) + u32 cmd_op, struct ublksrv_ctrl_cmd *header) { - struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)io_uring_sqe_cmd(cmd->sqe); bool unprivileged = ub->dev_info.flags & UBLK_F_UNPRIVILEGED_DEV; void __user *argp = (void __user *)(unsigned long)header->addr; char *dev_path = NULL; @@ -3703,7 +3701,7 @@ static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub, * know if the specified device is created as unprivileged * mode. */ - if (_IOC_NR(cmd->cmd_op) != UBLK_CMD_GET_DEV_INFO2) + if (_IOC_NR(cmd_op) != UBLK_CMD_GET_DEV_INFO2) return 0; } @@ -3724,7 +3722,7 @@ static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub, return PTR_ERR(dev_path); ret = -EINVAL; - switch (_IOC_NR(cmd->cmd_op)) { + switch (_IOC_NR(cmd_op)) { case UBLK_CMD_GET_DEV_INFO: case UBLK_CMD_GET_DEV_INFO2: case UBLK_CMD_GET_QUEUE_AFFINITY: @@ -3753,7 +3751,7 @@ static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub, header->addr += header->dev_path_len; } pr_devel("%s: dev id %d cmd_op %x uid %d gid %d path %s ret %d\n", - __func__, ub->ub_number, cmd->cmd_op, + __func__, ub->ub_number, cmd_op, ub->dev_info.owner_uid, ub->dev_info.owner_gid, dev_path, ret); exit: @@ -3777,7 +3775,9 @@ static bool ublk_ctrl_uring_cmd_may_sleep(u32 cmd_op) static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) { - const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe); + /* May point to userspace-mapped memory */ + const struct ublksrv_ctrl_cmd *ub_src = io_uring_sqe_cmd(cmd->sqe); + struct ublksrv_ctrl_cmd header; struct ublk_device *ub = NULL; u32 cmd_op = cmd->cmd_op; int ret = -EINVAL; @@ -3786,44 +3786,50 @@ static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd, issue_flags & IO_URING_F_NONBLOCK) return -EAGAIN; - ublk_ctrl_cmd_dump(cmd); - if (!(issue_flags & IO_URING_F_SQE128)) - goto out; + return -EINVAL; + + header.dev_id = READ_ONCE(ub_src->dev_id); + header.queue_id = READ_ONCE(ub_src->queue_id); + header.len = READ_ONCE(ub_src->len); + header.addr = READ_ONCE(ub_src->addr); + header.data[0] = READ_ONCE(ub_src->data[0]); + header.dev_path_len = READ_ONCE(ub_src->dev_path_len); + ublk_ctrl_cmd_dump(cmd_op, &header); ret = ublk_check_cmd_op(cmd_op); if (ret) goto out; if (cmd_op == UBLK_U_CMD_GET_FEATURES) { - ret = ublk_ctrl_get_features(header); + ret = ublk_ctrl_get_features(&header); goto out; } if (_IOC_NR(cmd_op) != UBLK_CMD_ADD_DEV) { ret = -ENODEV; - ub = ublk_get_device_from_id(header->dev_id); + ub = ublk_get_device_from_id(header.dev_id); if (!ub) goto out; - ret = ublk_ctrl_uring_cmd_permission(ub, cmd); + ret = ublk_ctrl_uring_cmd_permission(ub, cmd_op, &header); if (ret) goto put_dev; } switch (_IOC_NR(cmd_op)) { case UBLK_CMD_START_DEV: - ret = ublk_ctrl_start_dev(ub, header); + ret = ublk_ctrl_start_dev(ub, &header); break; case UBLK_CMD_STOP_DEV: ret = ublk_ctrl_stop_dev(ub); break; case UBLK_CMD_GET_DEV_INFO: case UBLK_CMD_GET_DEV_INFO2: - ret = ublk_ctrl_get_dev_info(ub, header); + ret = ublk_ctrl_get_dev_info(ub, &header); break; case UBLK_CMD_ADD_DEV: - ret = ublk_ctrl_add_dev(header); + ret = ublk_ctrl_add_dev(&header); break; case UBLK_CMD_DEL_DEV: ret = ublk_ctrl_del_dev(&ub, true); @@ -3832,26 +3838,26 @@ static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd, ret = ublk_ctrl_del_dev(&ub, false); break; case UBLK_CMD_GET_QUEUE_AFFINITY: - ret = ublk_ctrl_get_queue_affinity(ub, header); + ret = ublk_ctrl_get_queue_affinity(ub, &header); break; case UBLK_CMD_GET_PARAMS: - ret = ublk_ctrl_get_params(ub, header); + ret = ublk_ctrl_get_params(ub, &header); break; case UBLK_CMD_SET_PARAMS: - ret = ublk_ctrl_set_params(ub, header); + ret = ublk_ctrl_set_params(ub, &header); break; case UBLK_CMD_START_USER_RECOVERY: - ret = ublk_ctrl_start_recovery(ub, header); + ret = ublk_ctrl_start_recovery(ub, &header); break; case UBLK_CMD_END_USER_RECOVERY: - ret = ublk_ctrl_end_recovery(ub, header); + ret = ublk_ctrl_end_recovery(ub, &header); break; case UBLK_CMD_UPDATE_SIZE: - ublk_ctrl_set_size(ub, header); + ublk_ctrl_set_size(ub, &header); ret = 0; break; case UBLK_CMD_QUIESCE_DEV: - ret = ublk_ctrl_quiesce_dev(ub, header); + ret = ublk_ctrl_quiesce_dev(ub, &header); break; default: ret = -EOPNOTSUPP; @@ -3863,7 +3869,7 @@ static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd, ublk_put_device(ub); out: pr_devel("%s: cmd done ret %d cmd_op %x, dev id %d qid %d\n", - __func__, ret, cmd->cmd_op, header->dev_id, header->queue_id); + __func__, ret, cmd_op, header.dev_id, header.queue_id); return ret; } diff --git a/drivers/bluetooth/btintel_pcie.c b/drivers/bluetooth/btintel_pcie.c index 2936b535479f26..704767b334b98b 100644 --- a/drivers/bluetooth/btintel_pcie.c +++ b/drivers/bluetooth/btintel_pcie.c @@ -1431,11 +1431,6 @@ static void btintel_pcie_msix_rx_handle(struct btintel_pcie_data *data) } } -static irqreturn_t btintel_pcie_msix_isr(int irq, void *data) -{ - return IRQ_WAKE_THREAD; -} - static inline bool btintel_pcie_is_rxq_empty(struct btintel_pcie_data *data) { return data->ia.cr_hia[BTINTEL_PCIE_RXQ_NUM] == data->ia.cr_tia[BTINTEL_PCIE_RXQ_NUM]; @@ -1537,9 +1532,9 @@ static int btintel_pcie_setup_irq(struct btintel_pcie_data *data) err = devm_request_threaded_irq(&data->pdev->dev, msix_entry->vector, - btintel_pcie_msix_isr, + NULL, btintel_pcie_irq_msix_handler, - IRQF_SHARED, + IRQF_ONESHOT | IRQF_SHARED, KBUILD_MODNAME, msix_entry); if (err) { diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c index ded09e94d296df..80ccfa8fd982a4 100644 --- a/drivers/bluetooth/btusb.c +++ b/drivers/bluetooth/btusb.c @@ -527,6 +527,8 @@ static const struct usb_device_id quirks_table[] = { BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x2001, 0x332a), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x7392, 0xe611), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, /* Realtek 8852AE Bluetooth devices */ { USB_DEVICE(0x0bda, 0x2852), .driver_info = BTUSB_REALTEK | diff --git a/drivers/bus/fsl-mc/fsl-mc-bus.c b/drivers/bus/fsl-mc/fsl-mc-bus.c index 25845c04e5620a..a97baf2cbcdd51 100644 --- a/drivers/bus/fsl-mc/fsl-mc-bus.c +++ b/drivers/bus/fsl-mc/fsl-mc-bus.c @@ -202,8 +202,12 @@ static ssize_t driver_override_show(struct device *dev, struct device_attribute *attr, char *buf) { struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev); + ssize_t len; - return sysfs_emit(buf, "%s\n", mc_dev->driver_override); + device_lock(dev); + len = sysfs_emit(buf, "%s\n", mc_dev->driver_override); + device_unlock(dev); + return len; } static DEVICE_ATTR_RW(driver_override); diff --git a/drivers/bus/simple-pm-bus.c b/drivers/bus/simple-pm-bus.c index 3f00d953fb9a0e..518c10673cdec3 100644 --- a/drivers/bus/simple-pm-bus.c +++ b/drivers/bus/simple-pm-bus.c @@ -142,6 +142,15 @@ static const struct of_device_id simple_pm_bus_of_match[] = { { .compatible = "simple-mfd", .data = ONLY_BUS }, { .compatible = "isa", .data = ONLY_BUS }, { .compatible = "arm,amba-bus", .data = ONLY_BUS }, + { .compatible = "apple,s5l8960x-pmgr", .data = ONLY_BUS }, + { .compatible = "apple,t7000-pmgr", .data = ONLY_BUS }, + { .compatible = "apple,s8000-pmgr", .data = ONLY_BUS }, + { .compatible = "apple,t8010-pmgr", .data = ONLY_BUS }, + { .compatible = "apple,t8015-pmgr", .data = ONLY_BUS }, + { .compatible = "apple,t8103-pmgr", .data = ONLY_BUS }, + { .compatible = "apple,t8112-pmgr", .data = ONLY_BUS }, + { .compatible = "apple,t6000-pmgr", .data = ONLY_BUS }, + { .compatible = "apple,t6020-pmgr", .data = ONLY_BUS },\ { .compatible = "fsl,ls1021a-scfg", }, { .compatible = "fsl,ls1043a-scfg", }, { .compatible = "fsl,ls1046a-scfg", }, diff --git a/drivers/char/hw_random/airoha-trng.c b/drivers/char/hw_random/airoha-trng.c index 1dbfa9505c214f..9a648f6d9fd409 100644 --- a/drivers/char/hw_random/airoha-trng.c +++ b/drivers/char/hw_random/airoha-trng.c @@ -212,6 +212,7 @@ static int airoha_trng_probe(struct platform_device *pdev) trng->rng.init = airoha_trng_init; trng->rng.cleanup = airoha_trng_cleanup; trng->rng.read = airoha_trng_read; + trng->rng.quality = 900; ret = devm_hwrng_register(dev, &trng->rng); if (ret) { diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index 96d7fe41b373d5..aba92d777f7260 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -20,23 +20,25 @@ #include #include #include +#include #include #include #include #include #include +#include #define RNG_MODULE_NAME "hw_random" #define RNG_BUFFER_SIZE (SMP_CACHE_BYTES < 32 ? 32 : SMP_CACHE_BYTES) -static struct hwrng *current_rng; +static struct hwrng __rcu *current_rng; /* the current rng has been explicitly chosen by user via sysfs */ static int cur_rng_set_by_user; static struct task_struct *hwrng_fill; /* list of registered rngs */ static LIST_HEAD(rng_list); -/* Protects rng_list and current_rng */ +/* Protects rng_list, hwrng_fill and updating on current_rng */ static DEFINE_MUTEX(rng_mutex); /* Protects rng read functions, data_avail, rng_buffer and rng_fillbuf */ static DEFINE_MUTEX(reading_mutex); @@ -64,18 +66,39 @@ static size_t rng_buffer_size(void) return RNG_BUFFER_SIZE; } -static inline void cleanup_rng(struct kref *kref) +static void cleanup_rng_work(struct work_struct *work) { - struct hwrng *rng = container_of(kref, struct hwrng, ref); + struct hwrng *rng = container_of(work, struct hwrng, cleanup_work); + + /* + * Hold rng_mutex here so we serialize in case they set_current_rng + * on rng again immediately. + */ + mutex_lock(&rng_mutex); + + /* Skip if rng has been reinitialized. */ + if (kref_read(&rng->ref)) { + mutex_unlock(&rng_mutex); + return; + } if (rng->cleanup) rng->cleanup(rng); complete(&rng->cleanup_done); + mutex_unlock(&rng_mutex); +} + +static inline void cleanup_rng(struct kref *kref) +{ + struct hwrng *rng = container_of(kref, struct hwrng, ref); + + schedule_work(&rng->cleanup_work); } static int set_current_rng(struct hwrng *rng) { + struct hwrng *old_rng; int err; BUG_ON(!mutex_is_locked(&rng_mutex)); @@ -84,8 +107,14 @@ static int set_current_rng(struct hwrng *rng) if (err) return err; - drop_current_rng(); - current_rng = rng; + old_rng = rcu_dereference_protected(current_rng, + lockdep_is_held(&rng_mutex)); + rcu_assign_pointer(current_rng, rng); + + if (old_rng) { + synchronize_rcu(); + kref_put(&old_rng->ref, cleanup_rng); + } /* if necessary, start hwrng thread */ if (!hwrng_fill) { @@ -101,47 +130,56 @@ static int set_current_rng(struct hwrng *rng) static void drop_current_rng(void) { - BUG_ON(!mutex_is_locked(&rng_mutex)); - if (!current_rng) + struct hwrng *rng; + + rng = rcu_dereference_protected(current_rng, + lockdep_is_held(&rng_mutex)); + if (!rng) return; + RCU_INIT_POINTER(current_rng, NULL); + synchronize_rcu(); + + if (hwrng_fill) { + kthread_stop(hwrng_fill); + hwrng_fill = NULL; + } + /* decrease last reference for triggering the cleanup */ - kref_put(¤t_rng->ref, cleanup_rng); - current_rng = NULL; + kref_put(&rng->ref, cleanup_rng); } -/* Returns ERR_PTR(), NULL or refcounted hwrng */ +/* Returns NULL or refcounted hwrng */ static struct hwrng *get_current_rng_nolock(void) { - if (current_rng) - kref_get(¤t_rng->ref); + struct hwrng *rng; + + rng = rcu_dereference_protected(current_rng, + lockdep_is_held(&rng_mutex)); + if (rng) + kref_get(&rng->ref); - return current_rng; + return rng; } static struct hwrng *get_current_rng(void) { struct hwrng *rng; - if (mutex_lock_interruptible(&rng_mutex)) - return ERR_PTR(-ERESTARTSYS); + rcu_read_lock(); + rng = rcu_dereference(current_rng); + if (rng) + kref_get(&rng->ref); - rng = get_current_rng_nolock(); + rcu_read_unlock(); - mutex_unlock(&rng_mutex); return rng; } static void put_rng(struct hwrng *rng) { - /* - * Hold rng_mutex here so we serialize in case they set_current_rng - * on rng again immediately. - */ - mutex_lock(&rng_mutex); if (rng) kref_put(&rng->ref, cleanup_rng); - mutex_unlock(&rng_mutex); } static int hwrng_init(struct hwrng *rng) @@ -213,10 +251,6 @@ static ssize_t rng_dev_read(struct file *filp, char __user *buf, while (size) { rng = get_current_rng(); - if (IS_ERR(rng)) { - err = PTR_ERR(rng); - goto out; - } if (!rng) { err = -ENODEV; goto out; @@ -303,7 +337,7 @@ static struct miscdevice rng_miscdev = { static int enable_best_rng(void) { - struct hwrng *rng, *new_rng = NULL; + struct hwrng *rng, *cur_rng, *new_rng = NULL; int ret = -ENODEV; BUG_ON(!mutex_is_locked(&rng_mutex)); @@ -321,7 +355,9 @@ static int enable_best_rng(void) new_rng = rng; } - ret = ((new_rng == current_rng) ? 0 : set_current_rng(new_rng)); + cur_rng = rcu_dereference_protected(current_rng, + lockdep_is_held(&rng_mutex)); + ret = ((new_rng == cur_rng) ? 0 : set_current_rng(new_rng)); if (!ret) cur_rng_set_by_user = 0; @@ -371,8 +407,6 @@ static ssize_t rng_current_show(struct device *dev, struct hwrng *rng; rng = get_current_rng(); - if (IS_ERR(rng)) - return PTR_ERR(rng); ret = sysfs_emit(buf, "%s\n", rng ? rng->name : "none"); put_rng(rng); @@ -416,8 +450,6 @@ static ssize_t rng_quality_show(struct device *dev, struct hwrng *rng; rng = get_current_rng(); - if (IS_ERR(rng)) - return PTR_ERR(rng); if (!rng) /* no need to put_rng */ return -ENODEV; @@ -432,6 +464,7 @@ static ssize_t rng_quality_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t len) { + struct hwrng *rng; u16 quality; int ret = -EINVAL; @@ -448,12 +481,13 @@ static ssize_t rng_quality_store(struct device *dev, goto out; } - if (!current_rng) { + rng = rcu_dereference_protected(current_rng, lockdep_is_held(&rng_mutex)); + if (!rng) { ret = -ENODEV; goto out; } - current_rng->quality = quality; + rng->quality = quality; current_quality = quality; /* obsolete */ /* the best available RNG may have changed */ @@ -489,8 +523,20 @@ static int hwrng_fillfn(void *unused) struct hwrng *rng; rng = get_current_rng(); - if (IS_ERR(rng) || !rng) + if (!rng) { + /* + * Keep the task_struct alive until kthread_stop() + * is called to avoid UAF in drop_current_rng(). + */ + while (!kthread_should_stop()) { + set_current_state(TASK_INTERRUPTIBLE); + if (!kthread_should_stop()) + schedule(); + } + set_current_state(TASK_RUNNING); break; + } + mutex_lock(&reading_mutex); rc = rng_get_data(rng, rng_fillbuf, rng_buffer_size(), 1); @@ -518,14 +564,13 @@ static int hwrng_fillfn(void *unused) add_hwgenerator_randomness((void *)rng_fillbuf, rc, entropy >> 10, true); } - hwrng_fill = NULL; return 0; } int hwrng_register(struct hwrng *rng) { int err = -EINVAL; - struct hwrng *tmp; + struct hwrng *cur_rng, *tmp; if (!rng->name || (!rng->data_read && !rng->read)) goto out; @@ -540,6 +585,7 @@ int hwrng_register(struct hwrng *rng) } list_add_tail(&rng->list, &rng_list); + INIT_WORK(&rng->cleanup_work, cleanup_rng_work); init_completion(&rng->cleanup_done); complete(&rng->cleanup_done); init_completion(&rng->dying); @@ -547,16 +593,19 @@ int hwrng_register(struct hwrng *rng) /* Adjust quality field to always have a proper value */ rng->quality = min3(default_quality, 1024, rng->quality ?: 1024); - if (!cur_rng_set_by_user && - (!current_rng || rng->quality > current_rng->quality)) { - /* - * Set new rng as current as the new rng source - * provides better entropy quality and was not - * chosen by userspace. - */ - err = set_current_rng(rng); - if (err) - goto out_unlock; + if (!cur_rng_set_by_user) { + cur_rng = rcu_dereference_protected(current_rng, + lockdep_is_held(&rng_mutex)); + if (!cur_rng || rng->quality > cur_rng->quality) { + /* + * Set new rng as current as the new rng source + * provides better entropy quality and was not + * chosen by userspace. + */ + err = set_current_rng(rng); + if (err) + goto out_unlock; + } } mutex_unlock(&rng_mutex); return 0; @@ -569,14 +618,17 @@ EXPORT_SYMBOL_GPL(hwrng_register); void hwrng_unregister(struct hwrng *rng) { - struct hwrng *new_rng; + struct hwrng *cur_rng; int err; mutex_lock(&rng_mutex); list_del(&rng->list); complete_all(&rng->dying); - if (current_rng == rng) { + + cur_rng = rcu_dereference_protected(current_rng, + lockdep_is_held(&rng_mutex)); + if (cur_rng == rng) { err = enable_best_rng(); if (err) { drop_current_rng(); @@ -584,17 +636,7 @@ void hwrng_unregister(struct hwrng *rng) } } - new_rng = get_current_rng_nolock(); - if (list_empty(&rng_list)) { - mutex_unlock(&rng_mutex); - if (hwrng_fill) - kthread_stop(hwrng_fill); - } else - mutex_unlock(&rng_mutex); - - if (new_rng) - put_rng(new_rng); - + mutex_unlock(&rng_mutex); wait_for_completion(&rng->cleanup_done); } EXPORT_SYMBOL_GPL(hwrng_unregister); @@ -682,7 +724,7 @@ static int __init hwrng_modinit(void) static void __exit hwrng_modexit(void) { mutex_lock(&rng_mutex); - BUG_ON(current_rng); + WARN_ON(rcu_access_pointer(current_rng)); kfree(rng_buffer); kfree(rng_fillbuf); mutex_unlock(&rng_mutex); diff --git a/drivers/char/misc_minor_kunit.c b/drivers/char/misc_minor_kunit.c index 6fc8b05169c575..e930c78e1ef97e 100644 --- a/drivers/char/misc_minor_kunit.c +++ b/drivers/char/misc_minor_kunit.c @@ -166,7 +166,7 @@ static void __init miscdev_test_can_open(struct kunit *test, struct miscdevice * KUNIT_FAIL(test, "failed to create node\n"); filp = filp_open(devname, O_RDONLY, 0); - if (IS_ERR_OR_NULL(filp)) + if (IS_ERR(filp)) KUNIT_FAIL(test, "failed to open misc device: %ld\n", PTR_ERR(filp)); else fput(filp); diff --git a/drivers/char/tpm/st33zp24/st33zp24.c b/drivers/char/tpm/st33zp24/st33zp24.c index 2ed7815e4899b7..e2b7451ea7ccd3 100644 --- a/drivers/char/tpm/st33zp24/st33zp24.c +++ b/drivers/char/tpm/st33zp24/st33zp24.c @@ -328,8 +328,10 @@ static int st33zp24_send(struct tpm_chip *chip, unsigned char *buf, for (i = 0; i < len - 1;) { burstcnt = get_burstcount(chip); - if (burstcnt < 0) - return burstcnt; + if (burstcnt < 0) { + ret = burstcnt; + goto out_err; + } size = min_t(int, len - i - 1, burstcnt); ret = tpm_dev->ops->send(tpm_dev->phy_id, TPM_DATA_FIFO, buf + i, size); diff --git a/drivers/char/tpm/tpm_i2c_infineon.c b/drivers/char/tpm/tpm_i2c_infineon.c index bdf1f329a67946..8b7d32de0b2ef9 100644 --- a/drivers/char/tpm/tpm_i2c_infineon.c +++ b/drivers/char/tpm/tpm_i2c_infineon.c @@ -544,8 +544,10 @@ static int tpm_tis_i2c_send(struct tpm_chip *chip, u8 *buf, size_t bufsiz, burstcnt = get_burstcount(chip); /* burstcnt < 0 = TPM is busy */ - if (burstcnt < 0) - return burstcnt; + if (burstcnt < 0) { + rc = burstcnt; + goto out_err; + } if (burstcnt > (len - 1 - count)) burstcnt = len - 1 - count; diff --git a/drivers/clk/actions/owl-composite.c b/drivers/clk/actions/owl-composite.c index 00b74f8bc4375a..9540444307d6c9 100644 --- a/drivers/clk/actions/owl-composite.c +++ b/drivers/clk/actions/owl-composite.c @@ -57,15 +57,10 @@ static int owl_comp_div_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) { struct owl_composite *comp = hw_to_owl_comp(hw); - long rate; - - rate = owl_divider_helper_round_rate(&comp->common, &comp->rate.div_hw, - req->rate, &req->best_parent_rate); - if (rate < 0) - return rate; + struct owl_divider_hw *div = &comp->rate.div_hw; - req->rate = rate; - return 0; + return divider_determine_rate(&comp->common.hw, req, div->table, + div->width, div->div_flags); } static unsigned long owl_comp_div_recalc_rate(struct clk_hw *hw, diff --git a/drivers/clk/actions/owl-divider.c b/drivers/clk/actions/owl-divider.c index 118f1393c6780f..316ace80e87e3b 100644 --- a/drivers/clk/actions/owl-divider.c +++ b/drivers/clk/actions/owl-divider.c @@ -13,26 +13,13 @@ #include "owl-divider.h" -long owl_divider_helper_round_rate(struct owl_clk_common *common, - const struct owl_divider_hw *div_hw, - unsigned long rate, - unsigned long *parent_rate) -{ - return divider_round_rate(&common->hw, rate, parent_rate, - div_hw->table, div_hw->width, - div_hw->div_flags); -} - static int owl_divider_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) { struct owl_divider *div = hw_to_owl_divider(hw); - req->rate = owl_divider_helper_round_rate(&div->common, &div->div_hw, - req->rate, - &req->best_parent_rate); - - return 0; + return divider_determine_rate(hw, req, div->div_hw.table, + div->div_hw.width, div->div_hw.div_flags); } unsigned long owl_divider_helper_recalc_rate(struct owl_clk_common *common, diff --git a/drivers/clk/actions/owl-divider.h b/drivers/clk/actions/owl-divider.h index d76f58782c5288..1d3bb4e5898a3c 100644 --- a/drivers/clk/actions/owl-divider.h +++ b/drivers/clk/actions/owl-divider.h @@ -56,11 +56,6 @@ static inline struct owl_divider *hw_to_owl_divider(struct clk_hw *hw) return container_of(common, struct owl_divider, common); } -long owl_divider_helper_round_rate(struct owl_clk_common *common, - const struct owl_divider_hw *div_hw, - unsigned long rate, - unsigned long *parent_rate); - unsigned long owl_divider_helper_recalc_rate(struct owl_clk_common *common, const struct owl_divider_hw *div_hw, unsigned long parent_rate); diff --git a/drivers/clk/clk-apple-nco.c b/drivers/clk/clk-apple-nco.c index d3ced4a0f029ec..434c067968bbc1 100644 --- a/drivers/clk/clk-apple-nco.c +++ b/drivers/clk/clk-apple-nco.c @@ -320,6 +320,7 @@ static int applnco_probe(struct platform_device *pdev) } static const struct of_device_id applnco_ids[] = { + { .compatible = "apple,t8103-nco" }, { .compatible = "apple,nco" }, { } }; diff --git a/drivers/clk/clk-bm1880.c b/drivers/clk/clk-bm1880.c index dac190bc6e19a6..d2617fe16d2e42 100644 --- a/drivers/clk/clk-bm1880.c +++ b/drivers/clk/clk-bm1880.c @@ -629,10 +629,7 @@ static int bm1880_clk_div_determine_rate(struct clk_hw *hw, return 0; } - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - div->table, div->width, div->flags); - - return 0; + return divider_determine_rate(hw, req, div->table, div->width, div->flags); } static int bm1880_clk_div_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/clk/clk-loongson1.c b/drivers/clk/clk-loongson1.c index f9f060d08a5fae..1674181a1107dc 100644 --- a/drivers/clk/clk-loongson1.c +++ b/drivers/clk/clk-loongson1.c @@ -99,10 +99,7 @@ static int ls1x_divider_determine_rate(struct clk_hw *hw, struct ls1x_clk *ls1x_clk = to_ls1x_clk(hw); const struct ls1x_clk_div_data *d = ls1x_clk->data; - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - d->table, d->width, d->flags); - - return 0; + return divider_determine_rate(hw, req, d->table, d->width, d->flags); } static int ls1x_divider_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/clk/clk-milbeaut.c b/drivers/clk/clk-milbeaut.c index b4f9b7143eaa65..bb94d02a76cf10 100644 --- a/drivers/clk/clk-milbeaut.c +++ b/drivers/clk/clk-milbeaut.c @@ -407,10 +407,7 @@ static int m10v_clk_divider_determine_rate(struct clk_hw *hw, return 0; } - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - divider->table, divider->width, divider->flags); - - return 0; + return divider_determine_rate(hw, req, divider->table, divider->width, divider->flags); } static int m10v_clk_divider_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/clk/clk-versaclock3.c b/drivers/clk/clk-versaclock3.c index 1849863dbd673f..27b6cf70f3ae1b 100644 --- a/drivers/clk/clk-versaclock3.c +++ b/drivers/clk/clk-versaclock3.c @@ -523,11 +523,8 @@ static int vc3_div_determine_rate(struct clk_hw *hw, return 0; } - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - div_data->table, - div_data->width, div_data->flags); - - return 0; + return divider_determine_rate(hw, req, div_data->table, div_data->width, + div_data->flags); } static int vc3_div_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/clk/hisilicon/clkdivider-hi6220.c b/drivers/clk/hisilicon/clkdivider-hi6220.c index 6bae18a84cb6c0..fd7ceb92d65157 100644 --- a/drivers/clk/hisilicon/clkdivider-hi6220.c +++ b/drivers/clk/hisilicon/clkdivider-hi6220.c @@ -60,10 +60,8 @@ static int hi6220_clkdiv_determine_rate(struct clk_hw *hw, { struct hi6220_clk_divider *dclk = to_hi6220_clk_divider(hw); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, dclk->table, - dclk->width, CLK_DIVIDER_ROUND_CLOSEST); - - return 0; + return divider_determine_rate(hw, req, dclk->table, dclk->width, + CLK_DIVIDER_ROUND_CLOSEST); } static int hi6220_clkdiv_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/clk/mediatek/clk-mt7981-eth.c b/drivers/clk/mediatek/clk-mt7981-eth.c index 906aec9ddff54b..0655ebb6c561f6 100644 --- a/drivers/clk/mediatek/clk-mt7981-eth.c +++ b/drivers/clk/mediatek/clk-mt7981-eth.c @@ -31,7 +31,7 @@ static const struct mtk_gate_regs sgmii0_cg_regs = { .ops = &mtk_clk_gate_ops_no_setclr_inv, \ } -static const struct mtk_gate sgmii0_clks[] __initconst = { +static const struct mtk_gate sgmii0_clks[] = { GATE_SGMII0(CLK_SGM0_TX_EN, "sgm0_tx_en", "usb_tx250m", 2), GATE_SGMII0(CLK_SGM0_RX_EN, "sgm0_rx_en", "usb_eq_rx250m", 3), GATE_SGMII0(CLK_SGM0_CK0_EN, "sgm0_ck0_en", "usb_ln0", 4), @@ -53,7 +53,7 @@ static const struct mtk_gate_regs sgmii1_cg_regs = { .ops = &mtk_clk_gate_ops_no_setclr_inv, \ } -static const struct mtk_gate sgmii1_clks[] __initconst = { +static const struct mtk_gate sgmii1_clks[] = { GATE_SGMII1(CLK_SGM1_TX_EN, "sgm1_tx_en", "usb_tx250m", 2), GATE_SGMII1(CLK_SGM1_RX_EN, "sgm1_rx_en", "usb_eq_rx250m", 3), GATE_SGMII1(CLK_SGM1_CK1_EN, "sgm1_ck1_en", "usb_ln0", 4), @@ -75,7 +75,7 @@ static const struct mtk_gate_regs eth_cg_regs = { .ops = &mtk_clk_gate_ops_no_setclr_inv, \ } -static const struct mtk_gate eth_clks[] __initconst = { +static const struct mtk_gate eth_clks[] = { GATE_ETH(CLK_ETH_FE_EN, "eth_fe_en", "netsys_2x", 6), GATE_ETH(CLK_ETH_GP2_EN, "eth_gp2_en", "sgm_325m", 7), GATE_ETH(CLK_ETH_GP1_EN, "eth_gp1_en", "sgm_325m", 8), diff --git a/drivers/clk/mediatek/clk-mt8196-mfg.c b/drivers/clk/mediatek/clk-mt8196-mfg.c index ae1eb9de79ae29..f40795b47ff1fd 100644 --- a/drivers/clk/mediatek/clk-mt8196-mfg.c +++ b/drivers/clk/mediatek/clk-mt8196-mfg.c @@ -58,24 +58,25 @@ .pcw_shift = _pcw_shift, \ .pcwbits = _pcwbits, \ .pcwibits = MT8196_INTEGER_BITS, \ + .parent_name = "mfg_eb", \ } static const struct mtk_pll_data mfg_ao_plls[] = { - PLL(CLK_MFG_AO_MFGPLL, "mfgpll", MFGPLL_CON0, MFGPLL_CON0, 0, 0, 0, - BIT(0), MFGPLL_CON1, 24, 0, 0, 0, + PLL(CLK_MFG_AO_MFGPLL, "mfgpll", MFGPLL_CON0, MFGPLL_CON0, 0, 0, + PLL_PARENT_EN, BIT(0), MFGPLL_CON1, 24, 0, 0, 0, MFGPLL_CON1, 0, 22), }; static const struct mtk_pll_data mfgsc0_ao_plls[] = { PLL(CLK_MFGSC0_AO_MFGPLL_SC0, "mfgpll-sc0", MFGPLL_SC0_CON0, - MFGPLL_SC0_CON0, 0, 0, 0, BIT(0), MFGPLL_SC0_CON1, 24, 0, 0, 0, - MFGPLL_SC0_CON1, 0, 22), + MFGPLL_SC0_CON0, 0, 0, PLL_PARENT_EN, BIT(0), MFGPLL_SC0_CON1, 24, + 0, 0, 0, MFGPLL_SC0_CON1, 0, 22), }; static const struct mtk_pll_data mfgsc1_ao_plls[] = { PLL(CLK_MFGSC1_AO_MFGPLL_SC1, "mfgpll-sc1", MFGPLL_SC1_CON0, - MFGPLL_SC1_CON0, 0, 0, 0, BIT(0), MFGPLL_SC1_CON1, 24, 0, 0, 0, - MFGPLL_SC1_CON1, 0, 22), + MFGPLL_SC1_CON0, 0, 0, PLL_PARENT_EN, BIT(0), MFGPLL_SC1_CON1, 24, + 0, 0, 0, MFGPLL_SC1_CON1, 0, 22), }; static const struct of_device_id of_match_clk_mt8196_mfg[] = { diff --git a/drivers/clk/mediatek/clk-mt8516.c b/drivers/clk/mediatek/clk-mt8516.c index 21eb052b0a539c..342a59019fea9b 100644 --- a/drivers/clk/mediatek/clk-mt8516.c +++ b/drivers/clk/mediatek/clk-mt8516.c @@ -544,7 +544,7 @@ static const struct mtk_gate_regs top5_cg_regs = { #define GATE_TOP5(_id, _name, _parent, _shift) \ GATE_MTK(_id, _name, _parent, &top5_cg_regs, _shift, &mtk_clk_gate_ops_setclr) -static const struct mtk_gate top_clks[] __initconst = { +static const struct mtk_gate top_clks[] = { /* TOP1 */ GATE_TOP1(CLK_TOP_THEM, "them", "ahb_infra_sel", 1), GATE_TOP1(CLK_TOP_APDMA, "apdma", "ahb_infra_sel", 2), diff --git a/drivers/clk/mediatek/clk-mtk.c b/drivers/clk/mediatek/clk-mtk.c index 19cd27941747aa..deafe55a96cb1d 100644 --- a/drivers/clk/mediatek/clk-mtk.c +++ b/drivers/clk/mediatek/clk-mtk.c @@ -497,14 +497,16 @@ static int __mtk_clk_simple_probe(struct platform_device *pdev, if (mcd->need_runtime_pm) { - devm_pm_runtime_enable(&pdev->dev); + r = devm_pm_runtime_enable(&pdev->dev); + if (r) + goto unmap_io; /* * Do a pm_runtime_resume_and_get() to workaround a possible * deadlock between clk_register() and the genpd framework. */ r = pm_runtime_resume_and_get(&pdev->dev); if (r) - return r; + goto unmap_io; } /* Calculate how many clk_hw_onecell_data entries to allocate */ @@ -618,11 +620,11 @@ static int __mtk_clk_simple_probe(struct platform_device *pdev, free_data: mtk_free_clk_data(clk_data); free_base: - if (mcd->shared_io && base) - iounmap(base); - if (mcd->need_runtime_pm) pm_runtime_put(&pdev->dev); +unmap_io: + if (mcd->shared_io && base) + iounmap(base); return r; } diff --git a/drivers/clk/mediatek/clk-pll.c b/drivers/clk/mediatek/clk-pll.c index cd2b6ce551c6b0..de3eb02670554f 100644 --- a/drivers/clk/mediatek/clk-pll.c +++ b/drivers/clk/mediatek/clk-pll.c @@ -358,6 +358,9 @@ struct clk_hw *mtk_clk_register_pll_ops(struct mtk_clk_pll *pll, init.name = data->name; init.flags = (data->flags & PLL_AO) ? CLK_IS_CRITICAL : 0; + if (data->flags & PLL_PARENT_EN) + init.flags |= CLK_OPS_PARENT_ENABLE; + init.ops = pll_ops; if (data->parent_name) init.parent_names = &data->parent_name; diff --git a/drivers/clk/mediatek/clk-pll.h b/drivers/clk/mediatek/clk-pll.h index d71c150ce83e4b..de5a8fb7cbcfe9 100644 --- a/drivers/clk/mediatek/clk-pll.h +++ b/drivers/clk/mediatek/clk-pll.h @@ -21,6 +21,7 @@ struct mtk_pll_div_table { #define HAVE_RST_BAR BIT(0) #define PLL_AO BIT(1) +#define PLL_PARENT_EN BIT(2) #define POSTDIV_MASK GENMASK(2, 0) struct mtk_pll_data { diff --git a/drivers/clk/meson/g12a.c b/drivers/clk/meson/g12a.c index 185b6348251dbd..d0d4c7b6dc8277 100644 --- a/drivers/clk/meson/g12a.c +++ b/drivers/clk/meson/g12a.c @@ -777,12 +777,23 @@ static struct clk_regmap g12a_hdmi_pll_dco = { }, }; +/* + * G12/SM1 hdmi OD dividers are POWER_OF_TWO dividers but limited to /4. + * A divider value of 3 should map to /8 but instead map /4 so ignore it. + */ +static const struct clk_div_table g12a_hdmi_pll_od_div_table[] = { + { .val = 0, .div = 1 }, + { .val = 1, .div = 2 }, + { .val = 2, .div = 4 }, + { /* sentinel */ } +}; + static struct clk_regmap g12a_hdmi_pll_od = { .data = &(struct clk_regmap_div_data){ .offset = HHI_HDMI_PLL_CNTL0, .shift = 16, .width = 2, - .flags = CLK_DIVIDER_POWER_OF_TWO, + .table = g12a_hdmi_pll_od_div_table, }, .hw.init = &(struct clk_init_data){ .name = "hdmi_pll_od", @@ -800,7 +811,7 @@ static struct clk_regmap g12a_hdmi_pll_od2 = { .offset = HHI_HDMI_PLL_CNTL0, .shift = 18, .width = 2, - .flags = CLK_DIVIDER_POWER_OF_TWO, + .table = g12a_hdmi_pll_od_div_table, }, .hw.init = &(struct clk_init_data){ .name = "hdmi_pll_od2", @@ -818,7 +829,7 @@ static struct clk_regmap g12a_hdmi_pll = { .offset = HHI_HDMI_PLL_CNTL0, .shift = 20, .width = 2, - .flags = CLK_DIVIDER_POWER_OF_TWO, + .table = g12a_hdmi_pll_od_div_table, }, .hw.init = &(struct clk_init_data){ .name = "hdmi_pll", diff --git a/drivers/clk/meson/gxbb.c b/drivers/clk/meson/gxbb.c index 5a229c4ffae105..ec9a3414875ac3 100644 --- a/drivers/clk/meson/gxbb.c +++ b/drivers/clk/meson/gxbb.c @@ -349,12 +349,23 @@ static struct clk_regmap gxbb_hdmi_pll = { }, }; +/* + * GXL hdmi OD dividers are POWER_OF_TWO dividers but limited to /4. + * A divider value of 3 should map to /8 but instead map /4 so ignore it. + */ +static const struct clk_div_table gxl_hdmi_pll_od_div_table[] = { + { .val = 0, .div = 1 }, + { .val = 1, .div = 2 }, + { .val = 2, .div = 4 }, + { /* sentinel */ } +}; + static struct clk_regmap gxl_hdmi_pll_od = { .data = &(struct clk_regmap_div_data){ .offset = HHI_HDMI_PLL_CNTL + 8, .shift = 21, .width = 2, - .flags = CLK_DIVIDER_POWER_OF_TWO, + .table = gxl_hdmi_pll_od_div_table, }, .hw.init = &(struct clk_init_data){ .name = "hdmi_pll_od", @@ -372,7 +383,7 @@ static struct clk_regmap gxl_hdmi_pll_od2 = { .offset = HHI_HDMI_PLL_CNTL + 8, .shift = 23, .width = 2, - .flags = CLK_DIVIDER_POWER_OF_TWO, + .table = gxl_hdmi_pll_od_div_table, }, .hw.init = &(struct clk_init_data){ .name = "hdmi_pll_od2", @@ -390,7 +401,7 @@ static struct clk_regmap gxl_hdmi_pll = { .offset = HHI_HDMI_PLL_CNTL + 8, .shift = 19, .width = 2, - .flags = CLK_DIVIDER_POWER_OF_TWO, + .table = gxl_hdmi_pll_od_div_table, }, .hw.init = &(struct clk_init_data){ .name = "hdmi_pll", diff --git a/drivers/clk/microchip/clk-core.c b/drivers/clk/microchip/clk-core.c index b34348d491f3e1..a0163441dfe5c1 100644 --- a/drivers/clk/microchip/clk-core.c +++ b/drivers/clk/microchip/clk-core.c @@ -780,15 +780,6 @@ static unsigned long sclk_get_rate(struct clk_hw *hw, unsigned long parent_rate) return parent_rate / div; } -static int sclk_determine_rate(struct clk_hw *hw, - struct clk_rate_request *req) -{ - req->rate = calc_best_divided_rate(req->rate, req->best_parent_rate, - SLEW_SYSDIV, 1); - - return 0; -} - static int sclk_set_rate(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate) { @@ -912,7 +903,6 @@ static int sclk_init(struct clk_hw *hw) const struct clk_ops pic32_sclk_ops = { .get_parent = sclk_get_parent, .set_parent = sclk_set_parent, - .determine_rate = sclk_determine_rate, .set_rate = sclk_set_rate, .recalc_rate = sclk_get_rate, .init = sclk_init, diff --git a/drivers/clk/nuvoton/clk-ma35d1-divider.c b/drivers/clk/nuvoton/clk-ma35d1-divider.c index e39f53d5bf4578..e992e7c3034197 100644 --- a/drivers/clk/nuvoton/clk-ma35d1-divider.c +++ b/drivers/clk/nuvoton/clk-ma35d1-divider.c @@ -44,11 +44,8 @@ static int ma35d1_clkdiv_determine_rate(struct clk_hw *hw, { struct ma35d1_adc_clk_div *dclk = to_ma35d1_adc_clk_div(hw); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - dclk->table, dclk->width, - CLK_DIVIDER_ROUND_CLOSEST); - - return 0; + return divider_determine_rate(hw, req, dclk->table, dclk->width, + CLK_DIVIDER_ROUND_CLOSEST); } static int ma35d1_clkdiv_set_rate(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate) diff --git a/drivers/clk/nxp/clk-lpc32xx.c b/drivers/clk/nxp/clk-lpc32xx.c index 23f980cf6a2b59..ae2fa5341a2e4f 100644 --- a/drivers/clk/nxp/clk-lpc32xx.c +++ b/drivers/clk/nxp/clk-lpc32xx.c @@ -975,10 +975,8 @@ static int clk_divider_determine_rate(struct clk_hw *hw, return 0; } - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - divider->table, divider->width, divider->flags); - - return 0; + return divider_determine_rate(hw, req, divider->table, divider->width, + divider->flags); } static int clk_divider_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/clk/qcom/clk-alpha-pll.c b/drivers/clk/qcom/clk-alpha-pll.c index 6aeba40358c11e..a84e8bee653462 100644 --- a/drivers/clk/qcom/clk-alpha-pll.c +++ b/drivers/clk/qcom/clk-alpha-pll.c @@ -1257,11 +1257,8 @@ static int clk_alpha_pll_postdiv_determine_rate(struct clk_hw *hw, else table = clk_alpha_div_table; - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - table, pll->width, - CLK_DIVIDER_POWER_OF_TWO); - - return 0; + return divider_determine_rate(hw, req, table, pll->width, + CLK_DIVIDER_POWER_OF_TWO); } static int clk_alpha_pll_postdiv_ro_determine_rate(struct clk_hw *hw, @@ -1617,11 +1614,8 @@ static int clk_trion_pll_postdiv_determine_rate(struct clk_hw *hw, { struct clk_alpha_pll_postdiv *pll = to_clk_alpha_pll_postdiv(hw); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - pll->post_div_table, - pll->width, CLK_DIVIDER_ROUND_CLOSEST); - - return 0; + return divider_determine_rate(hw, req, pll->post_div_table, pll->width, + CLK_DIVIDER_ROUND_CLOSEST); }; static int @@ -1657,11 +1651,8 @@ static int clk_alpha_pll_postdiv_fabia_determine_rate(struct clk_hw *hw, { struct clk_alpha_pll_postdiv *pll = to_clk_alpha_pll_postdiv(hw); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - pll->post_div_table, - pll->width, CLK_DIVIDER_ROUND_CLOSEST); - - return 0; + return divider_determine_rate(hw, req, pll->post_div_table, pll->width, + CLK_DIVIDER_ROUND_CLOSEST); } static int clk_alpha_pll_postdiv_fabia_set_rate(struct clk_hw *hw, diff --git a/drivers/clk/qcom/clk-rcg2.c b/drivers/clk/qcom/clk-rcg2.c index e18cb8807d7353..d0a5847f911144 100644 --- a/drivers/clk/qcom/clk-rcg2.c +++ b/drivers/clk/qcom/clk-rcg2.c @@ -755,7 +755,7 @@ static int clk_rcg2_get_duty_cycle(struct clk_hw *hw, struct clk_duty *duty) static int clk_rcg2_set_duty_cycle(struct clk_hw *hw, struct clk_duty *duty) { struct clk_rcg2 *rcg = to_clk_rcg2(hw); - u32 notn_m, n, m, d, not2d, mask, duty_per, cfg; + u32 notn_m, n, m, d, not2d, mask, cfg; int ret; /* Duty-cycle cannot be modified for non-MND RCGs */ @@ -774,10 +774,8 @@ static int clk_rcg2_set_duty_cycle(struct clk_hw *hw, struct clk_duty *duty) n = (~(notn_m) + m) & mask; - duty_per = (duty->num * 100) / duty->den; - /* Calculate 2d value */ - d = DIV_ROUND_CLOSEST(n * duty_per * 2, 100); + d = DIV_ROUND_CLOSEST(n * duty->num * 2, duty->den); /* * Check bit widths of 2d. If D is too big reduce duty cycle. @@ -1266,6 +1264,7 @@ static int clk_gfx3d_determine_rate(struct clk_hw *hw, if (req->max_rate < parent_req.max_rate) parent_req.max_rate = req->max_rate; + parent_req.best_parent_hw = req->best_parent_hw; ret = __clk_determine_rate(req->best_parent_hw, &parent_req); if (ret) return ret; diff --git a/drivers/clk/qcom/clk-regmap-divider.c b/drivers/clk/qcom/clk-regmap-divider.c index 4f5395f0ab6d0e..672e82caf20504 100644 --- a/drivers/clk/qcom/clk-regmap-divider.c +++ b/drivers/clk/qcom/clk-regmap-divider.c @@ -26,24 +26,16 @@ static int div_ro_determine_rate(struct clk_hw *hw, val >>= divider->shift; val &= BIT(divider->width) - 1; - req->rate = divider_ro_round_rate(hw, req->rate, - &req->best_parent_rate, NULL, - divider->width, - CLK_DIVIDER_ROUND_CLOSEST, val); - - return 0; + return divider_ro_determine_rate(hw, req, NULL, divider->width, + CLK_DIVIDER_ROUND_CLOSEST, val); } static int div_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) { struct clk_regmap_div *divider = to_clk_regmap_div(hw); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - NULL, - divider->width, - CLK_DIVIDER_ROUND_CLOSEST); - - return 0; + return divider_determine_rate(hw, req, NULL, divider->width, + CLK_DIVIDER_ROUND_CLOSEST); } static int div_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/clk/qcom/common.c b/drivers/clk/qcom/common.c index 12159188677418..eec369d2173b5c 100644 --- a/drivers/clk/qcom/common.c +++ b/drivers/clk/qcom/common.c @@ -454,7 +454,7 @@ int qcom_cc_probe_by_index(struct platform_device *pdev, int index, base = devm_platform_ioremap_resource(pdev, index); if (IS_ERR(base)) - return -ENOMEM; + return PTR_ERR(base); regmap = devm_regmap_init_mmio(&pdev->dev, base, desc->config); if (IS_ERR(regmap)) diff --git a/drivers/clk/qcom/dispcc-sdm845.c b/drivers/clk/qcom/dispcc-sdm845.c index 2f9e9665d7e93f..78e43f6d750260 100644 --- a/drivers/clk/qcom/dispcc-sdm845.c +++ b/drivers/clk/qcom/dispcc-sdm845.c @@ -280,7 +280,7 @@ static struct clk_rcg2 disp_cc_mdss_pclk0_clk_src = { .name = "disp_cc_mdss_pclk0_clk_src", .parent_data = disp_cc_parent_data_4, .num_parents = ARRAY_SIZE(disp_cc_parent_data_4), - .flags = CLK_SET_RATE_PARENT, + .flags = CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, .ops = &clk_pixel_ops, }, }; @@ -295,7 +295,7 @@ static struct clk_rcg2 disp_cc_mdss_pclk1_clk_src = { .name = "disp_cc_mdss_pclk1_clk_src", .parent_data = disp_cc_parent_data_4, .num_parents = ARRAY_SIZE(disp_cc_parent_data_4), - .flags = CLK_SET_RATE_PARENT, + .flags = CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, .ops = &clk_pixel_ops, }, }; diff --git a/drivers/clk/qcom/dispcc-sm7150.c b/drivers/clk/qcom/dispcc-sm7150.c index 811d380a8e9f9b..ed8e34ffd69b05 100644 --- a/drivers/clk/qcom/dispcc-sm7150.c +++ b/drivers/clk/qcom/dispcc-sm7150.c @@ -371,7 +371,7 @@ static struct clk_rcg2 dispcc_mdss_pclk1_clk_src = { .name = "dispcc_mdss_pclk1_clk_src", .parent_data = dispcc_parent_data_4, .num_parents = ARRAY_SIZE(dispcc_parent_data_4), - .flags = CLK_SET_RATE_PARENT, + .flags = CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, .ops = &clk_pixel_ops, }, }; diff --git a/drivers/clk/qcom/gcc-glymur.c b/drivers/clk/qcom/gcc-glymur.c index deab819576d0e1..238e205735ed59 100644 --- a/drivers/clk/qcom/gcc-glymur.c +++ b/drivers/clk/qcom/gcc-glymur.c @@ -2317,7 +2317,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .parent_data = gcc_parent_data_17, .num_parents = ARRAY_SIZE(gcc_parent_data_17), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -2339,7 +2339,7 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { .parent_data = gcc_parent_data_3, .num_parents = ARRAY_SIZE(gcc_parent_data_3), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-ipq5018.c b/drivers/clk/qcom/gcc-ipq5018.c index dcda2be8c1a519..64792cda062021 100644 --- a/drivers/clk/qcom/gcc-ipq5018.c +++ b/drivers/clk/qcom/gcc-ipq5018.c @@ -1340,6 +1340,7 @@ static struct clk_branch gcc_sleep_clk_src = { .name = "gcc_sleep_clk_src", .parent_data = gcc_sleep_clk_data, .num_parents = ARRAY_SIZE(gcc_sleep_clk_data), + .flags = CLK_IS_CRITICAL, .ops = &clk_branch2_ops, }, }, diff --git a/drivers/clk/qcom/gcc-milos.c b/drivers/clk/qcom/gcc-milos.c index c9d61b05bafa16..81fa09ec55d7f6 100644 --- a/drivers/clk/qcom/gcc-milos.c +++ b/drivers/clk/qcom/gcc-milos.c @@ -917,7 +917,7 @@ static struct clk_rcg2 gcc_sdcc1_apps_clk_src = { .name = "gcc_sdcc1_apps_clk_src", .parent_data = gcc_parent_data_9, .num_parents = ARRAY_SIZE(gcc_parent_data_9), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -938,7 +938,7 @@ static struct clk_rcg2 gcc_sdcc1_ice_core_clk_src = { .name = "gcc_sdcc1_ice_core_clk_src", .parent_data = gcc_parent_data_10, .num_parents = ARRAY_SIZE(gcc_parent_data_10), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -962,7 +962,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .name = "gcc_sdcc2_apps_clk_src", .parent_data = gcc_parent_data_11, .num_parents = ARRAY_SIZE(gcc_parent_data_11), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-msm8917.c b/drivers/clk/qcom/gcc-msm8917.c index 0a1aa623cd49af..9d1c5a9953e2c6 100644 --- a/drivers/clk/qcom/gcc-msm8917.c +++ b/drivers/clk/qcom/gcc-msm8917.c @@ -3409,7 +3409,6 @@ static struct gdsc cpp_gdsc = { .pd = { .name = "cpp_gdsc", }, - .flags = ALWAYS_ON, .pwrsts = PWRSTS_OFF_ON, }; diff --git a/drivers/clk/qcom/gcc-msm8953.c b/drivers/clk/qcom/gcc-msm8953.c index 8f29ecc74c50bf..8fe1d3e421440c 100644 --- a/drivers/clk/qcom/gcc-msm8953.c +++ b/drivers/clk/qcom/gcc-msm8953.c @@ -3946,7 +3946,6 @@ static struct gdsc cpp_gdsc = { .pd = { .name = "cpp_gdsc", }, - .flags = ALWAYS_ON, .pwrsts = PWRSTS_OFF_ON, }; diff --git a/drivers/clk/qcom/gcc-qdu1000.c b/drivers/clk/qcom/gcc-qdu1000.c index dbe9e9437939af..915bb9b4ff8130 100644 --- a/drivers/clk/qcom/gcc-qdu1000.c +++ b/drivers/clk/qcom/gcc-qdu1000.c @@ -904,7 +904,7 @@ static struct clk_rcg2 gcc_sdcc5_apps_clk_src = { .name = "gcc_sdcc5_apps_clk_src", .parent_data = gcc_parent_data_8, .num_parents = ARRAY_SIZE(gcc_parent_data_8), - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -923,7 +923,7 @@ static struct clk_rcg2 gcc_sdcc5_ice_core_clk_src = { .name = "gcc_sdcc5_ice_core_clk_src", .parent_data = gcc_parent_data_2, .num_parents = ARRAY_SIZE(gcc_parent_data_2), - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-sdx75.c b/drivers/clk/qcom/gcc-sdx75.c index 453a6bf8e87863..1f3cd58483a2d6 100644 --- a/drivers/clk/qcom/gcc-sdx75.c +++ b/drivers/clk/qcom/gcc-sdx75.c @@ -1033,7 +1033,7 @@ static struct clk_rcg2 gcc_sdcc1_apps_clk_src = { .name = "gcc_sdcc1_apps_clk_src", .parent_data = gcc_parent_data_17, .num_parents = ARRAY_SIZE(gcc_parent_data_17), - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -1057,7 +1057,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .name = "gcc_sdcc2_apps_clk_src", .parent_data = gcc_parent_data_18, .num_parents = ARRAY_SIZE(gcc_parent_data_18), - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-sm4450.c b/drivers/clk/qcom/gcc-sm4450.c index e2d9e4691c5b71..023d840e9f4ef0 100644 --- a/drivers/clk/qcom/gcc-sm4450.c +++ b/drivers/clk/qcom/gcc-sm4450.c @@ -769,7 +769,7 @@ static struct clk_rcg2 gcc_sdcc1_apps_clk_src = { .parent_data = gcc_parent_data_4, .num_parents = ARRAY_SIZE(gcc_parent_data_4), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -791,7 +791,7 @@ static struct clk_rcg2 gcc_sdcc1_ice_core_clk_src = { .parent_data = gcc_parent_data_4, .num_parents = ARRAY_SIZE(gcc_parent_data_4), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -815,7 +815,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .parent_data = gcc_parent_data_6, .num_parents = ARRAY_SIZE(gcc_parent_data_6), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-sm8450.c b/drivers/clk/qcom/gcc-sm8450.c index 65d7d52bce0343..b18bb34889ab28 100644 --- a/drivers/clk/qcom/gcc-sm8450.c +++ b/drivers/clk/qcom/gcc-sm8450.c @@ -1034,7 +1034,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .parent_data = gcc_parent_data_7, .num_parents = ARRAY_SIZE(gcc_parent_data_7), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -1057,7 +1057,7 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { .parent_data = gcc_parent_data_0, .num_parents = ARRAY_SIZE(gcc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-sm8550.c b/drivers/clk/qcom/gcc-sm8550.c index 862a9bf73bcb5d..36a5b7de5b55d2 100644 --- a/drivers/clk/qcom/gcc-sm8550.c +++ b/drivers/clk/qcom/gcc-sm8550.c @@ -1025,7 +1025,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .parent_data = gcc_parent_data_9, .num_parents = ARRAY_SIZE(gcc_parent_data_9), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -1048,7 +1048,7 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { .parent_data = gcc_parent_data_0, .num_parents = ARRAY_SIZE(gcc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-sm8650.c b/drivers/clk/qcom/gcc-sm8650.c index 24f98062b9dd50..2dd6444ce0365f 100644 --- a/drivers/clk/qcom/gcc-sm8650.c +++ b/drivers/clk/qcom/gcc-sm8650.c @@ -1257,7 +1257,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .parent_data = gcc_parent_data_11, .num_parents = ARRAY_SIZE(gcc_parent_data_11), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -1279,7 +1279,7 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { .parent_data = gcc_parent_data_0, .num_parents = ARRAY_SIZE(gcc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-sm8750.c b/drivers/clk/qcom/gcc-sm8750.c index def86b71a3da53..db81569dd4b17d 100644 --- a/drivers/clk/qcom/gcc-sm8750.c +++ b/drivers/clk/qcom/gcc-sm8750.c @@ -1030,7 +1030,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .parent_data = gcc_parent_data_8, .num_parents = ARRAY_SIZE(gcc_parent_data_8), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -1052,7 +1052,7 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { .parent_data = gcc_parent_data_0, .num_parents = ARRAY_SIZE(gcc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-x1e80100.c b/drivers/clk/qcom/gcc-x1e80100.c index b63c8abdd2fc24..e46e65e631513e 100644 --- a/drivers/clk/qcom/gcc-x1e80100.c +++ b/drivers/clk/qcom/gcc-x1e80100.c @@ -1516,7 +1516,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .parent_data = gcc_parent_data_9, .num_parents = ARRAY_SIZE(gcc_parent_data_9), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -1538,7 +1538,7 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { .parent_data = gcc_parent_data_0, .num_parents = ARRAY_SIZE(gcc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/rockchip/clk.c b/drivers/clk/rockchip/clk.c index 2601df3b1066b1..9ac9d13e87de0f 100644 --- a/drivers/clk/rockchip/clk.c +++ b/drivers/clk/rockchip/clk.c @@ -693,7 +693,7 @@ void rockchip_clk_register_late_branches(struct device *dev, break; } - if (!pdev) + if (IS_ERR_OR_NULL(pdev)) dev_err(dev, "failed to register device for clock %s\n", list->name); } } diff --git a/drivers/clk/sophgo/clk-sg2042-clkgen.c b/drivers/clk/sophgo/clk-sg2042-clkgen.c index 683661b71787c9..9725ac4e050a4e 100644 --- a/drivers/clk/sophgo/clk-sg2042-clkgen.c +++ b/drivers/clk/sophgo/clk-sg2042-clkgen.c @@ -180,7 +180,6 @@ static int sg2042_clk_divider_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) { struct sg2042_divider_clock *divider = to_sg2042_clk_divider(hw); - unsigned long ret_rate; u32 bestdiv; /* if read only, just return current value */ @@ -191,17 +190,13 @@ static int sg2042_clk_divider_determine_rate(struct clk_hw *hw, bestdiv = readl(divider->reg) >> divider->shift; bestdiv &= clk_div_mask(divider->width); } - ret_rate = DIV_ROUND_UP_ULL((u64)req->best_parent_rate, bestdiv); - } else { - ret_rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, NULL, - divider->width, divider->div_flags); - } + req->rate = DIV_ROUND_UP_ULL((u64)req->best_parent_rate, bestdiv); - pr_debug("--> %s: divider_round_rate: val = %ld\n", - clk_hw_get_name(hw), ret_rate); - req->rate = ret_rate; + return 0; + } - return 0; + return divider_determine_rate(hw, req, NULL, divider->width, + divider->div_flags); } static int sg2042_clk_divider_set_rate(struct clk_hw *hw, diff --git a/drivers/clk/spacemit/Makefile b/drivers/clk/spacemit/Makefile index 5ec6da61db98e0..ad2bf315109b80 100644 --- a/drivers/clk/spacemit/Makefile +++ b/drivers/clk/spacemit/Makefile @@ -1,5 +1,10 @@ # SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_SPACEMIT_K1_CCU) = spacemit-ccu-k1.o -spacemit-ccu-k1-y = ccu_pll.o ccu_mix.o ccu_ddn.o +obj-$(CONFIG_SPACEMIT_CCU) += spacemit-ccu.o +spacemit-ccu-y += ccu_common.o +spacemit-ccu-y += ccu_pll.o +spacemit-ccu-y += ccu_mix.o +spacemit-ccu-y += ccu_ddn.o + +obj-$(CONFIG_SPACEMIT_K1_CCU) += spacemit-ccu-k1.o spacemit-ccu-k1-y += ccu-k1.o diff --git a/drivers/clk/spacemit/ccu-k1.c b/drivers/clk/spacemit/ccu-k1.c index 4761bc1e3b6e60..01d9485b615d32 100644 --- a/drivers/clk/spacemit/ccu-k1.c +++ b/drivers/clk/spacemit/ccu-k1.c @@ -1204,6 +1204,7 @@ static struct platform_driver k1_ccu_driver = { }; module_platform_driver(k1_ccu_driver); +MODULE_IMPORT_NS("CLK_SPACEMIT"); MODULE_DESCRIPTION("SpacemiT K1 CCU driver"); MODULE_AUTHOR("Haylen Chu "); MODULE_LICENSE("GPL"); diff --git a/drivers/clk/spacemit/ccu_common.c b/drivers/clk/spacemit/ccu_common.c new file mode 100644 index 00000000000000..4412c4104dabb3 --- /dev/null +++ b/drivers/clk/spacemit/ccu_common.c @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include + +MODULE_DESCRIPTION("SpacemiT CCU common clock driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/spacemit/ccu_ddn.c b/drivers/clk/spacemit/ccu_ddn.c index 5b16e273bee5b1..b5540e0781ffa3 100644 --- a/drivers/clk/spacemit/ccu_ddn.c +++ b/drivers/clk/spacemit/ccu_ddn.c @@ -84,3 +84,4 @@ const struct clk_ops spacemit_ccu_ddn_ops = { .determine_rate = ccu_ddn_determine_rate, .set_rate = ccu_ddn_set_rate, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_ddn_ops, "CLK_SPACEMIT"); diff --git a/drivers/clk/spacemit/ccu_mix.c b/drivers/clk/spacemit/ccu_mix.c index 7b799087537231..67f8b12b4f5b7d 100644 --- a/drivers/clk/spacemit/ccu_mix.c +++ b/drivers/clk/spacemit/ccu_mix.c @@ -198,24 +198,28 @@ const struct clk_ops spacemit_ccu_gate_ops = { .enable = ccu_gate_enable, .is_enabled = ccu_gate_is_enabled, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_gate_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_factor_ops = { .determine_rate = ccu_factor_determine_rate, .recalc_rate = ccu_factor_recalc_rate, .set_rate = ccu_factor_set_rate, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_factor_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_mux_ops = { .determine_rate = ccu_mix_determine_rate, .get_parent = ccu_mux_get_parent, .set_parent = ccu_mux_set_parent, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_mux_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_div_ops = { .determine_rate = ccu_mix_determine_rate, .recalc_rate = ccu_div_recalc_rate, .set_rate = ccu_mix_set_rate, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_div_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_factor_gate_ops = { .disable = ccu_gate_disable, @@ -226,6 +230,7 @@ const struct clk_ops spacemit_ccu_factor_gate_ops = { .recalc_rate = ccu_factor_recalc_rate, .set_rate = ccu_factor_set_rate, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_factor_gate_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_mux_gate_ops = { .disable = ccu_gate_disable, @@ -236,6 +241,7 @@ const struct clk_ops spacemit_ccu_mux_gate_ops = { .get_parent = ccu_mux_get_parent, .set_parent = ccu_mux_set_parent, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_mux_gate_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_div_gate_ops = { .disable = ccu_gate_disable, @@ -246,6 +252,7 @@ const struct clk_ops spacemit_ccu_div_gate_ops = { .recalc_rate = ccu_div_recalc_rate, .set_rate = ccu_mix_set_rate, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_div_gate_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_mux_div_gate_ops = { .disable = ccu_gate_disable, @@ -259,6 +266,7 @@ const struct clk_ops spacemit_ccu_mux_div_gate_ops = { .recalc_rate = ccu_div_recalc_rate, .set_rate = ccu_mix_set_rate, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_mux_div_gate_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_mux_div_ops = { .get_parent = ccu_mux_get_parent, @@ -268,3 +276,4 @@ const struct clk_ops spacemit_ccu_mux_div_ops = { .recalc_rate = ccu_div_recalc_rate, .set_rate = ccu_mix_set_rate, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_mux_div_ops, "CLK_SPACEMIT"); diff --git a/drivers/clk/spacemit/ccu_pll.c b/drivers/clk/spacemit/ccu_pll.c index d92f0dae65a490..76d0244873d87c 100644 --- a/drivers/clk/spacemit/ccu_pll.c +++ b/drivers/clk/spacemit/ccu_pll.c @@ -157,3 +157,4 @@ const struct clk_ops spacemit_ccu_pll_ops = { .determine_rate = ccu_pll_determine_rate, .is_enabled = ccu_pll_is_enabled, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_pll_ops, "CLK_SPACEMIT"); diff --git a/drivers/clk/sprd/div.c b/drivers/clk/sprd/div.c index 01342388196800..cd57163a7204c0 100644 --- a/drivers/clk/sprd/div.c +++ b/drivers/clk/sprd/div.c @@ -14,11 +14,7 @@ static int sprd_div_determine_rate(struct clk_hw *hw, { struct sprd_div *cd = hw_to_sprd_div(hw); - req->rate = divider_round_rate(&cd->common.hw, req->rate, - &req->best_parent_rate, - NULL, cd->div.width, 0); - - return 0; + return divider_determine_rate(&cd->common.hw, req, NULL, cd->div.width, 0); } unsigned long sprd_div_helper_recalc_rate(struct sprd_clk_common *common, diff --git a/drivers/clk/stm32/clk-stm32-core.c b/drivers/clk/stm32/clk-stm32-core.c index 72825b9c36a4d3..e921c25a929c30 100644 --- a/drivers/clk/stm32/clk-stm32-core.c +++ b/drivers/clk/stm32/clk-stm32-core.c @@ -369,22 +369,14 @@ static int clk_stm32_divider_determine_rate(struct clk_hw *hw, val = readl(div->base + divider->offset) >> divider->shift; val &= clk_div_mask(divider->width); - req->rate = divider_ro_round_rate(hw, req->rate, - &req->best_parent_rate, - divider->table, - divider->width, - divider->flags, val); - - return 0; + return divider_ro_determine_rate(hw, req, + divider->table, + divider->width, + divider->flags, val); } - req->rate = divider_round_rate_parent(hw, clk_hw_get_parent(hw), - req->rate, - &req->best_parent_rate, - divider->table, - divider->width, divider->flags); - - return 0; + return divider_determine_rate(hw, req, divider->table, divider->width, + divider->flags); } static unsigned long clk_stm32_divider_recalc_rate(struct clk_hw *hw, @@ -441,7 +433,6 @@ static int clk_stm32_composite_determine_rate(struct clk_hw *hw, { struct clk_stm32_composite *composite = to_clk_stm32_composite(hw); const struct stm32_div_cfg *divider; - long rate; if (composite->div_id == NO_STM32_DIV) return 0; @@ -455,24 +446,13 @@ static int clk_stm32_composite_determine_rate(struct clk_hw *hw, val = readl(composite->base + divider->offset) >> divider->shift; val &= clk_div_mask(divider->width); - rate = divider_ro_round_rate(hw, req->rate, &req->best_parent_rate, - divider->table, divider->width, divider->flags, - val); - if (rate < 0) - return rate; - - req->rate = rate; - return 0; + return divider_ro_determine_rate(hw, req, divider->table, + divider->width, divider->flags, + val); } - rate = divider_round_rate_parent(hw, clk_hw_get_parent(hw), - req->rate, &req->best_parent_rate, - divider->table, divider->width, divider->flags); - if (rate < 0) - return rate; - - req->rate = rate; - return 0; + return divider_determine_rate(hw, req, divider->table, divider->width, + divider->flags); } static u8 clk_stm32_composite_get_parent(struct clk_hw *hw) diff --git a/drivers/clk/thead/clk-th1520-ap.c b/drivers/clk/thead/clk-th1520-ap.c index 71ad03a998e8e1..d870f0c665f8a4 100644 --- a/drivers/clk/thead/clk-th1520-ap.c +++ b/drivers/clk/thead/clk-th1520-ap.c @@ -8,11 +8,14 @@ #include #include #include +#include #include #include #include #include +#define TH1520_PLL_STS 0x80 + #define TH1520_PLL_POSTDIV2 GENMASK(26, 24) #define TH1520_PLL_POSTDIV1 GENMASK(22, 20) #define TH1520_PLL_FBDIV GENMASK(19, 8) @@ -23,6 +26,13 @@ #define TH1520_PLL_FRAC GENMASK(23, 0) #define TH1520_PLL_FRAC_BITS 24 +/* + * All PLLs in TH1520 take 21250ns at maximum to lock, let's take its double + * for safety. + */ +#define TH1520_PLL_LOCK_TIMEOUT_US 44 +#define TH1520_PLL_STABLE_DELAY_US 30 + struct ccu_internal { u8 shift; u8 width; @@ -64,6 +74,7 @@ struct ccu_div { struct ccu_pll { struct ccu_common common; + u32 lock_sts_mask; }; #define TH_CCU_ARG(_shift, _width) \ @@ -299,9 +310,21 @@ static void ccu_pll_disable(struct clk_hw *hw) static int ccu_pll_enable(struct clk_hw *hw) { struct ccu_pll *pll = hw_to_ccu_pll(hw); + u32 reg; + int ret; - return regmap_clear_bits(pll->common.map, pll->common.cfg1, - TH1520_PLL_VCO_RST); + regmap_clear_bits(pll->common.map, pll->common.cfg1, + TH1520_PLL_VCO_RST); + + ret = regmap_read_poll_timeout_atomic(pll->common.map, TH1520_PLL_STS, + reg, reg & pll->lock_sts_mask, + 5, TH1520_PLL_LOCK_TIMEOUT_US); + if (ret) + return ret; + + udelay(TH1520_PLL_STABLE_DELAY_US); + + return 0; } static int ccu_pll_is_enabled(struct clk_hw *hw) @@ -389,6 +412,7 @@ static struct ccu_pll cpu_pll0_clk = { &clk_pll_ops, CLK_IS_CRITICAL), }, + .lock_sts_mask = BIT(1), }; static struct ccu_pll cpu_pll1_clk = { @@ -401,6 +425,7 @@ static struct ccu_pll cpu_pll1_clk = { &clk_pll_ops, CLK_IS_CRITICAL), }, + .lock_sts_mask = BIT(4), }; static struct ccu_pll gmac_pll_clk = { @@ -413,6 +438,7 @@ static struct ccu_pll gmac_pll_clk = { &clk_pll_ops, CLK_IS_CRITICAL), }, + .lock_sts_mask = BIT(3), }; static const struct clk_hw *gmac_pll_clk_parent[] = { @@ -433,6 +459,7 @@ static struct ccu_pll video_pll_clk = { &clk_pll_ops, CLK_IS_CRITICAL), }, + .lock_sts_mask = BIT(7), }; static const struct clk_hw *video_pll_clk_parent[] = { @@ -453,6 +480,7 @@ static struct ccu_pll dpu0_pll_clk = { &clk_pll_ops, 0), }, + .lock_sts_mask = BIT(8), }; static const struct clk_hw *dpu0_pll_clk_parent[] = { @@ -469,6 +497,7 @@ static struct ccu_pll dpu1_pll_clk = { &clk_pll_ops, 0), }, + .lock_sts_mask = BIT(9), }; static const struct clk_hw *dpu1_pll_clk_parent[] = { @@ -485,6 +514,7 @@ static struct ccu_pll tee_pll_clk = { &clk_pll_ops, CLK_IS_CRITICAL), }, + .lock_sts_mask = BIT(10), }; static const struct clk_parent_data c910_i0_parents[] = { diff --git a/drivers/clk/x86/clk-cgu.c b/drivers/clk/x86/clk-cgu.c index d099667355f8d8..92ee05d75af2b2 100644 --- a/drivers/clk/x86/clk-cgu.c +++ b/drivers/clk/x86/clk-cgu.c @@ -137,10 +137,8 @@ static int lgm_clk_divider_determine_rate(struct clk_hw *hw, { struct lgm_clk_divider *divider = to_lgm_clk_divider(hw); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, divider->table, - divider->width, divider->flags); - - return 0; + return divider_determine_rate(hw, req, divider->table, divider->width, + divider->flags); } static int diff --git a/drivers/clk/zynqmp/divider.c b/drivers/clk/zynqmp/divider.c index c824eeacd8ebd4..984e577ea67115 100644 --- a/drivers/clk/zynqmp/divider.c +++ b/drivers/clk/zynqmp/divider.c @@ -111,10 +111,9 @@ static unsigned long zynqmp_clk_divider_recalc_rate(struct clk_hw *hw, } /** - * zynqmp_clk_divider_round_rate() - Round rate of divider clock + * zynqmp_clk_divider_determine_rate() - Determine rate of divider clock * @hw: handle between common and hardware-specific interfaces - * @rate: rate of clock to be set - * @prate: rate of parent clock + * @req: rate of clock to be set * * Return: 0 on success else error+reason */ @@ -151,8 +150,9 @@ static int zynqmp_clk_divider_determine_rate(struct clk_hw *hw, width = fls(divider->max_div); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - NULL, width, divider->flags); + ret = divider_determine_rate(hw, req, NULL, width, divider->flags); + if (ret != 0) + return ret; if (divider->is_frac && (clk_hw_get_flags(hw) & CLK_SET_RATE_PARENT) && (req->rate % req->best_parent_rate)) diff --git a/drivers/clk/zynqmp/pll.c b/drivers/clk/zynqmp/pll.c index 630a3936c97c34..6bc2c3934f564f 100644 --- a/drivers/clk/zynqmp/pll.c +++ b/drivers/clk/zynqmp/pll.c @@ -91,10 +91,9 @@ static inline void zynqmp_pll_set_mode(struct clk_hw *hw, bool on) } /** - * zynqmp_pll_round_rate() - Round a clock frequency + * zynqmp_pll_determine_rate() - Round a clock frequency * @hw: Handle between common and hardware-specific interfaces - * @rate: Desired clock frequency - * @prate: Clock frequency of parent clock + * @req: Desired clock frequency * * Return: Frequency closest to @rate the hardware can generate */ diff --git a/drivers/clocksource/timer-sp804.c b/drivers/clocksource/timer-sp804.c index e82a95ea472478..d698584273596b 100644 --- a/drivers/clocksource/timer-sp804.c +++ b/drivers/clocksource/timer-sp804.c @@ -106,21 +106,25 @@ static u64 notrace sp804_read(void) return ~readl_relaxed(sched_clkevt->value); } +/* Register delay timer backed by the hardware counter */ #ifdef CONFIG_ARM static struct delay_timer delay; +static struct sp804_clkevt *delay_clkevt; + static unsigned long sp804_read_delay_timer_read(void) { - return sp804_read(); + return ~readl_relaxed(delay_clkevt->value); } -static void sp804_register_delay_timer(int freq) +static void sp804_register_delay_timer(struct sp804_clkevt *clk, int freq) { + delay_clkevt = clk; delay.freq = freq; delay.read_current_timer = sp804_read_delay_timer_read; register_current_timer_delay(&delay); } #else -static inline void sp804_register_delay_timer(int freq) {} +static inline void sp804_register_delay_timer(struct sp804_clkevt *clk, int freq) {} #endif static int __init sp804_clocksource_and_sched_clock_init(void __iomem *base, @@ -135,8 +139,6 @@ static int __init sp804_clocksource_and_sched_clock_init(void __iomem *base, if (rate < 0) return -EINVAL; - sp804_register_delay_timer(rate); - clkevt = sp804_clkevt_get(base); writel(0, clkevt->ctrl); @@ -152,6 +154,8 @@ static int __init sp804_clocksource_and_sched_clock_init(void __iomem *base, clocksource_mmio_init(clkevt->value, name, rate, 200, 32, clocksource_mmio_readl_down); + sp804_register_delay_timer(clkevt, rate); + if (use_sched_clock) { sched_clkevt = clkevt; sched_clock_register(sp804_read, 32, rate); diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index ec4abe3745736b..1625ec2d0d06a1 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1161,7 +1161,7 @@ static void hybrid_init_cpu_capacity_scaling(bool refresh) * the capacity of SMT threads is not deterministic even approximately, * do not do that when SMT is in use. */ - if (hwp_is_hybrid && !sched_smt_active() && arch_enable_hybrid_capacity_scale()) { + if (hwp_is_hybrid && !cpu_smt_possible() && arch_enable_hybrid_capacity_scale()) { hybrid_refresh_cpu_capacity_scaling(); /* * Disabling ITMT causes sched domains to be rebuilt to disable asym diff --git a/drivers/cpufreq/scmi-cpufreq.c b/drivers/cpufreq/scmi-cpufreq.c index d2a110079f5fd5..c450cf9c881d49 100644 --- a/drivers/cpufreq/scmi-cpufreq.c +++ b/drivers/cpufreq/scmi-cpufreq.c @@ -101,6 +101,7 @@ static int scmi_cpu_domain_id(struct device *cpu_dev) return -EINVAL; } + of_node_put(domain_id.np); return domain_id.args[0]; } diff --git a/drivers/cpuidle/Kconfig.arm b/drivers/cpuidle/Kconfig.arm index a1ee475d180dac..c6870f08457632 100644 --- a/drivers/cpuidle/Kconfig.arm +++ b/drivers/cpuidle/Kconfig.arm @@ -130,3 +130,11 @@ config ARM_QCOM_SPM_CPUIDLE The Subsystem Power Manager (SPM) controls low power modes for the CPU and L2 cores. It interface with various system drivers to put the cores in low power modes. + +config ARM_APPLE_CPUIDLE + bool "Apple SoC CPU idle driver" + depends on ARM64 + default ARCH_APPLE + select CPU_IDLE_MULTIPLE_DRIVERS + help + Select this to enable cpuidle on Apple SoCs. diff --git a/drivers/cpuidle/Makefile b/drivers/cpuidle/Makefile index 1de9e92c5b0fc9..f9e7a71d52c13f 100644 --- a/drivers/cpuidle/Makefile +++ b/drivers/cpuidle/Makefile @@ -29,6 +29,7 @@ obj-$(CONFIG_ARM_PSCI_CPUIDLE) += cpuidle-psci.o obj-$(CONFIG_ARM_PSCI_CPUIDLE_DOMAIN) += cpuidle-psci-domain.o obj-$(CONFIG_ARM_TEGRA_CPUIDLE) += cpuidle-tegra.o obj-$(CONFIG_ARM_QCOM_SPM_CPUIDLE) += cpuidle-qcom-spm.o +obj-$(CONFIG_ARM_APPLE_CPUIDLE) += cpuidle-apple.o ############################################################################### # MIPS drivers diff --git a/drivers/cpuidle/cpuidle-apple.c b/drivers/cpuidle/cpuidle-apple.c new file mode 100644 index 00000000000000..b7504066d75aa8 --- /dev/null +++ b/drivers/cpuidle/cpuidle-apple.c @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Copyright The Asahi Linux Contributors + * + * CPU idle support for Apple SoCs + */ + +#include +#include +#include +#include +#include +#include + +enum idle_state { + STATE_WFI, + STATE_PWRDOWN, + STATE_COUNT +}; + +asm( + ".pushsection .cpuidle.text, \"ax\"\n" + ".type apple_cpu_deep_wfi, @function\n" + "apple_cpu_deep_wfi:\n" + "str x30, [sp, #-16]!\n" + "stp x28, x29, [sp, #-16]!\n" + "stp x26, x27, [sp, #-16]!\n" + "stp x24, x25, [sp, #-16]!\n" + "stp x22, x23, [sp, #-16]!\n" + "stp x20, x21, [sp, #-16]!\n" + "stp x18, x19, [sp, #-16]!\n" + + "mrs x0, s3_5_c15_c5_0\n" + "orr x0, x0, #(3L << 24)\n" + "msr s3_5_c15_c5_0, x0\n" + + "1:\n" + "dsb sy\n" + "wfi\n" + + "mrs x0, ISR_EL1\n" + "cbz x0, 1b\n" + + "mrs x0, s3_5_c15_c5_0\n" + "bic x0, x0, #(1L << 24)\n" + "msr s3_5_c15_c5_0, x0\n" + + "ldp x18, x19, [sp], #16\n" + "ldp x20, x21, [sp], #16\n" + "ldp x22, x23, [sp], #16\n" + "ldp x24, x25, [sp], #16\n" + "ldp x26, x27, [sp], #16\n" + "ldp x28, x29, [sp], #16\n" + "ldr x30, [sp], #16\n" + + "ret\n" + ".popsection\n" +); + +void apple_cpu_deep_wfi(void); + +static __cpuidle int apple_enter_wfi(struct cpuidle_device *dev, struct cpuidle_driver *drv, int index) +{ + cpu_do_idle(); + return index; +} + +static __cpuidle int apple_enter_idle(struct cpuidle_device *dev, struct cpuidle_driver *drv, int index) +{ + /* + * Deep WFI will clobber FP state, among other things. + * The CPU PM notifier will take care of saving that and anything else + * that needs to be notified of the CPU powering down. + */ + if (cpu_pm_enter()) + return -1; + + ct_cpuidle_enter(); + + switch(index) { + case STATE_PWRDOWN: + apple_cpu_deep_wfi(); + break; + default: + WARN_ON(1); + break; + } + + ct_cpuidle_exit(); + + cpu_pm_exit(); + + return index; +} + +static struct cpuidle_driver apple_idle_driver = { + .name = "apple_idle", + .owner = THIS_MODULE, + .states = { + [STATE_WFI] = { + .enter = apple_enter_wfi, + .enter_s2idle = apple_enter_wfi, + .exit_latency = 1, + .target_residency = 1, + .power_usage = UINT_MAX, + .name = "WFI", + .desc = "CPU clock-gated", + .flags = 0, + }, + [STATE_PWRDOWN] = { + .enter = apple_enter_idle, + .enter_s2idle = apple_enter_idle, + .exit_latency = 10, + .target_residency = 10000, + .power_usage = 0, + .name = "CPU PD", + .desc = "CPU/cluster powered down", + .flags = CPUIDLE_FLAG_RCU_IDLE, + }, + }, + .safe_state_index = STATE_WFI, + .state_count = STATE_COUNT, +}; + +static int apple_cpuidle_probe(struct platform_device *pdev) +{ + return cpuidle_register(&apple_idle_driver, NULL); +} + +static struct platform_driver apple_cpuidle_driver = { + .driver = { + .name = "cpuidle-apple", + }, + .probe = apple_cpuidle_probe, +}; + +static int __init apple_cpuidle_init(void) +{ + struct platform_device *pdev; + int ret; + + ret = platform_driver_register(&apple_cpuidle_driver); + if (ret) + return ret; + + if (!of_machine_is_compatible("apple,arm-platform")) + return 0; + + if (!(of_machine_is_compatible("apple,t8103") || + of_machine_is_compatible("apple,t8112") || + of_machine_is_compatible("apple,t6000") || + of_machine_is_compatible("apple,t6001") || + of_machine_is_compatible("apple,t6002") || + of_machine_is_compatible("apple,t6020") || + of_machine_is_compatible("apple,t6021") || + of_machine_is_compatible("apple,t6022"))) + return 0; + + pdev = platform_device_register_simple("cpuidle-apple", -1, NULL, 0); + if (IS_ERR(pdev)) { + platform_driver_unregister(&apple_cpuidle_driver); + return PTR_ERR(pdev); + } + + return 0; +} +device_initcall(apple_cpuidle_init); diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c index c7876e9e024f90..65fbb8e807b977 100644 --- a/drivers/cpuidle/cpuidle.c +++ b/drivers/cpuidle/cpuidle.c @@ -359,6 +359,16 @@ noinstr int cpuidle_enter_state(struct cpuidle_device *dev, int cpuidle_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, bool *stop_tick) { + /* + * If there is only a single idle state (or none), there is nothing + * meaningful for the governor to choose. Skip the governor and + * always use state 0 with the tick running. + */ + if (drv->state_count <= 1) { + *stop_tick = false; + return 0; + } + return cpuidle_curr_governor->select(drv, dev, stop_tick); } diff --git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c index 64d6f7a1c77663..ca863ba03d4544 100644 --- a/drivers/cpuidle/governors/menu.c +++ b/drivers/cpuidle/governors/menu.c @@ -239,7 +239,7 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, /* Find the shortest expected idle interval. */ predicted_ns = get_typical_interval(data) * NSEC_PER_USEC; - if (predicted_ns > RESIDENCY_THRESHOLD_NS) { + if (predicted_ns > RESIDENCY_THRESHOLD_NS || tick_nohz_tick_stopped()) { unsigned int timer_us; /* Determine the time till the closest timer. */ @@ -259,6 +259,16 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, RESOLUTION * DECAY * NSEC_PER_USEC); /* Use the lowest expected idle interval to pick the idle state. */ predicted_ns = min((u64)timer_us * NSEC_PER_USEC, predicted_ns); + /* + * If the tick is already stopped, the cost of possible short + * idle duration misprediction is much higher, because the CPU + * may be stuck in a shallow idle state for a long time as a + * result of it. In that case, say we might mispredict and use + * the known time till the closest timer event for the idle + * state selection. + */ + if (tick_nohz_tick_stopped() && predicted_ns < TICK_NSEC) + predicted_ns = data->next_timer_ns; } else { /* * Because the next timer event is not going to be determined @@ -284,16 +294,6 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, return 0; } - /* - * If the tick is already stopped, the cost of possible short idle - * duration misprediction is much higher, because the CPU may be stuck - * in a shallow idle state for a long time as a result of it. In that - * case, say we might mispredict and use the known time till the closest - * timer event for the idle state selection. - */ - if (tick_nohz_tick_stopped() && predicted_ns < TICK_NSEC) - predicted_ns = data->next_timer_ns; - /* * Find the idle state with the lowest power while satisfying * our constraints. diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c index 107ccb2ade4205..c6117c23eb25b0 100644 --- a/drivers/crypto/caam/caamalg_qi2.c +++ b/drivers/crypto/caam/caamalg_qi2.c @@ -4814,7 +4814,8 @@ static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv) { struct device *dev = priv->dev; struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); - int err; + struct dpaa2_caam_priv_per_cpu *ppriv; + int i, err; if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) { err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle); @@ -4822,6 +4823,12 @@ static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv) dev_err(dev, "dpseci_reset() failed\n"); } + for_each_cpu(i, priv->clean_mask) { + ppriv = per_cpu_ptr(priv->ppriv, i); + free_netdev(ppriv->net_dev); + } + free_cpumask_var(priv->clean_mask); + dpaa2_dpseci_congestion_free(priv); dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); } @@ -5007,16 +5014,15 @@ static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev) struct device *dev = &ls_dev->dev; struct dpaa2_caam_priv *priv; struct dpaa2_caam_priv_per_cpu *ppriv; - cpumask_var_t clean_mask; int err, cpu; u8 i; err = -ENOMEM; - if (!zalloc_cpumask_var(&clean_mask, GFP_KERNEL)) - goto err_cpumask; - priv = dev_get_drvdata(dev); + if (!zalloc_cpumask_var(&priv->clean_mask, GFP_KERNEL)) + goto err_cpumask; + priv->dev = dev; priv->dpsec_id = ls_dev->obj_desc.id; @@ -5118,7 +5124,7 @@ static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev) err = -ENOMEM; goto err_alloc_netdev; } - cpumask_set_cpu(cpu, clean_mask); + cpumask_set_cpu(cpu, priv->clean_mask); ppriv->net_dev->dev = *dev; netif_napi_add_tx_weight(ppriv->net_dev, &ppriv->napi, @@ -5126,18 +5132,16 @@ static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev) DPAA2_CAAM_NAPI_WEIGHT); } - err = 0; - goto free_cpumask; + return 0; err_alloc_netdev: - free_dpaa2_pcpu_netdev(priv, clean_mask); + free_dpaa2_pcpu_netdev(priv, priv->clean_mask); err_get_rx_queue: dpaa2_dpseci_congestion_free(priv); err_get_vers: dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); err_open: -free_cpumask: - free_cpumask_var(clean_mask); + free_cpumask_var(priv->clean_mask); err_cpumask: return err; } @@ -5182,7 +5186,6 @@ static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv) ppriv = per_cpu_ptr(priv->ppriv, i); napi_disable(&ppriv->napi); netif_napi_del(&ppriv->napi); - free_netdev(ppriv->net_dev); } return 0; diff --git a/drivers/crypto/caam/caamalg_qi2.h b/drivers/crypto/caam/caamalg_qi2.h index 61d1219a202fcb..8e65b4b28c7bab 100644 --- a/drivers/crypto/caam/caamalg_qi2.h +++ b/drivers/crypto/caam/caamalg_qi2.h @@ -42,6 +42,7 @@ * @mc_io: pointer to MC portal's I/O object * @domain: IOMMU domain * @ppriv: per CPU pointers to privata data + * @clean_mask: CPU mask of CPUs that have allocated netdevs */ struct dpaa2_caam_priv { int dpsec_id; @@ -65,6 +66,7 @@ struct dpaa2_caam_priv { struct dpaa2_caam_priv_per_cpu __percpu *ppriv; struct dentry *dfs_root; + cpumask_var_t clean_mask; }; /** diff --git a/drivers/crypto/cavium/cpt/cptvf_main.c b/drivers/crypto/cavium/cpt/cptvf_main.c index c246920e6f540c..bccd680c7f7ee4 100644 --- a/drivers/crypto/cavium/cpt/cptvf_main.c +++ b/drivers/crypto/cavium/cpt/cptvf_main.c @@ -180,7 +180,8 @@ static void free_command_queues(struct cpt_vf *cptvf, hlist_for_each_entry_safe(chunk, node, &cqinfo->queue[i].chead, nextchunk) { - dma_free_coherent(&pdev->dev, chunk->size, + dma_free_coherent(&pdev->dev, + chunk->size + CPT_NEXT_CHUNK_PTR_SIZE, chunk->head, chunk->dma_addr); chunk->head = NULL; diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c index d78865d9d5f09c..d0412e58476253 100644 --- a/drivers/crypto/ccp/ccp-ops.c +++ b/drivers/crypto/ccp/ccp-ops.c @@ -642,7 +642,7 @@ ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) struct ccp_data dst; struct ccp_data aad; struct ccp_op op; - } *wa __cleanup(kfree) = kzalloc(sizeof *wa, GFP_KERNEL); + } *wa __free(kfree) = kzalloc(sizeof(*wa), GFP_KERNEL); unsigned int dm_offset; unsigned int authsize; unsigned int jobid; diff --git a/drivers/crypto/ccp/psp-dev.c b/drivers/crypto/ccp/psp-dev.c index 9e21da0e298ad7..5c7f7e02a7d8ab 100644 --- a/drivers/crypto/ccp/psp-dev.c +++ b/drivers/crypto/ccp/psp-dev.c @@ -351,6 +351,17 @@ struct psp_device *psp_get_master_device(void) return sp ? sp->psp_data : NULL; } +int psp_restore(struct sp_device *sp) +{ + struct psp_device *psp = sp->psp_data; + int ret = 0; + + if (psp->tee_data) + ret = tee_restore(psp); + + return ret; +} + void psp_pci_init(void) { psp_master = psp_get_master_device(); diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 956ea609d0cce1..1cdadddb744edf 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -127,13 +127,6 @@ static size_t sev_es_tmr_size = SEV_TMR_SIZE; #define NV_LENGTH (32 * 1024) static void *sev_init_ex_buffer; -/* - * SEV_DATA_RANGE_LIST: - * Array containing range of pages that firmware transitions to HV-fixed - * page state. - */ -static struct sev_data_range_list *snp_range_list; - static void __sev_firmware_shutdown(struct sev_device *sev, bool panic); static int snp_shutdown_on_panic(struct notifier_block *nb, @@ -1361,6 +1354,7 @@ static int snp_filter_reserved_mem_regions(struct resource *rs, void *arg) static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid) { + struct sev_data_range_list *snp_range_list __free(kfree) = NULL; struct psp_device *psp = psp_master; struct sev_data_snp_init_ex data; struct sev_device *sev; @@ -2378,11 +2372,10 @@ static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable) static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp) { struct sev_device *sev = psp_master->sev_data; - bool shutdown_required = false; struct sev_data_snp_addr buf; struct page *status_page; - int ret, error; void *data; + int ret; if (!argp->data) return -EINVAL; @@ -2393,31 +2386,35 @@ static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp) data = page_address(status_page); - if (!sev->snp_initialized) { - ret = snp_move_to_init_state(argp, &shutdown_required); - if (ret) - goto cleanup; - } - /* - * Firmware expects status page to be in firmware-owned state, otherwise - * it will report firmware error code INVALID_PAGE_STATE (0x1A). + * SNP_PLATFORM_STATUS can be executed in any SNP state. But if executed + * when SNP has been initialized, the status page must be firmware-owned. */ - if (rmp_mark_pages_firmware(__pa(data), 1, true)) { - ret = -EFAULT; - goto cleanup; + if (sev->snp_initialized) { + /* + * Firmware expects the status page to be in Firmware state, + * otherwise it will report an error INVALID_PAGE_STATE. + */ + if (rmp_mark_pages_firmware(__pa(data), 1, true)) { + ret = -EFAULT; + goto cleanup; + } } buf.address = __psp_pa(data); ret = __sev_do_cmd_locked(SEV_CMD_SNP_PLATFORM_STATUS, &buf, &argp->error); - /* - * Status page will be transitioned to Reclaim state upon success, or - * left in Firmware state in failure. Use snp_reclaim_pages() to - * transition either case back to Hypervisor-owned state. - */ - if (snp_reclaim_pages(__pa(data), 1, true)) - return -EFAULT; + if (sev->snp_initialized) { + /* + * The status page will be in Reclaim state on success, or left + * in Firmware state on failure. Use snp_reclaim_pages() to + * transition either case back to Hypervisor-owned state. + */ + if (snp_reclaim_pages(__pa(data), 1, true)) { + snp_leak_pages(__page_to_pfn(status_page), 1); + return -EFAULT; + } + } if (ret) goto cleanup; @@ -2427,9 +2424,6 @@ static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp) ret = -EFAULT; cleanup: - if (shutdown_required) - __sev_snp_shutdown_locked(&error, false); - __free_pages(status_page, 0); return ret; } @@ -2780,11 +2774,6 @@ static void __sev_firmware_shutdown(struct sev_device *sev, bool panic) sev_init_ex_buffer = NULL; } - if (snp_range_list) { - kfree(snp_range_list); - snp_range_list = NULL; - } - __sev_snp_shutdown_locked(&error, panic); } diff --git a/drivers/crypto/ccp/sp-dev.c b/drivers/crypto/ccp/sp-dev.c index 3467f6db4f505a..f204aa5df96e24 100644 --- a/drivers/crypto/ccp/sp-dev.c +++ b/drivers/crypto/ccp/sp-dev.c @@ -230,6 +230,18 @@ int sp_resume(struct sp_device *sp) return 0; } +int sp_restore(struct sp_device *sp) +{ + if (sp->psp_data) { + int ret = psp_restore(sp); + + if (ret) + return ret; + } + + return sp_resume(sp); +} + struct sp_device *sp_get_psp_master_device(void) { struct sp_device *i, *ret = NULL; diff --git a/drivers/crypto/ccp/sp-dev.h b/drivers/crypto/ccp/sp-dev.h index 1335a83fe052ea..a83751cfd00603 100644 --- a/drivers/crypto/ccp/sp-dev.h +++ b/drivers/crypto/ccp/sp-dev.h @@ -141,6 +141,7 @@ void sp_destroy(struct sp_device *sp); int sp_suspend(struct sp_device *sp); int sp_resume(struct sp_device *sp); +int sp_restore(struct sp_device *sp); int sp_request_ccp_irq(struct sp_device *sp, irq_handler_t handler, const char *name, void *data); void sp_free_ccp_irq(struct sp_device *sp, void *data); @@ -174,6 +175,7 @@ int psp_dev_init(struct sp_device *sp); void psp_pci_init(void); void psp_dev_destroy(struct sp_device *sp); void psp_pci_exit(void); +int psp_restore(struct sp_device *sp); #else /* !CONFIG_CRYPTO_DEV_SP_PSP */ @@ -181,6 +183,7 @@ static inline int psp_dev_init(struct sp_device *sp) { return 0; } static inline void psp_pci_init(void) { } static inline void psp_dev_destroy(struct sp_device *sp) { } static inline void psp_pci_exit(void) { } +static inline int psp_restore(struct sp_device *sp) { return 0; } #endif /* CONFIG_CRYPTO_DEV_SP_PSP */ diff --git a/drivers/crypto/ccp/sp-pci.c b/drivers/crypto/ccp/sp-pci.c index 8891ceee1d7d05..6ac805d99ccb30 100644 --- a/drivers/crypto/ccp/sp-pci.c +++ b/drivers/crypto/ccp/sp-pci.c @@ -353,6 +353,13 @@ static int __maybe_unused sp_pci_resume(struct device *dev) return sp_resume(sp); } +static int __maybe_unused sp_pci_restore(struct device *dev) +{ + struct sp_device *sp = dev_get_drvdata(dev); + + return sp_restore(sp); +} + #ifdef CONFIG_CRYPTO_DEV_SP_PSP static const struct sev_vdata sevv1 = { .cmdresp_reg = 0x10580, /* C2PMSG_32 */ @@ -563,7 +570,14 @@ static const struct pci_device_id sp_pci_table[] = { }; MODULE_DEVICE_TABLE(pci, sp_pci_table); -static SIMPLE_DEV_PM_OPS(sp_pci_pm_ops, sp_pci_suspend, sp_pci_resume); +static const struct dev_pm_ops sp_pci_pm_ops = { + .suspend = pm_sleep_ptr(sp_pci_suspend), + .resume = pm_sleep_ptr(sp_pci_resume), + .freeze = pm_sleep_ptr(sp_pci_suspend), + .thaw = pm_sleep_ptr(sp_pci_resume), + .poweroff = pm_sleep_ptr(sp_pci_suspend), + .restore_early = pm_sleep_ptr(sp_pci_restore), +}; static struct pci_driver sp_pci_driver = { .name = "ccp", diff --git a/drivers/crypto/ccp/tee-dev.c b/drivers/crypto/ccp/tee-dev.c index 5e1d80724678d0..92ffa412622a24 100644 --- a/drivers/crypto/ccp/tee-dev.c +++ b/drivers/crypto/ccp/tee-dev.c @@ -86,10 +86,34 @@ static inline void tee_free_cmd_buffer(struct tee_init_ring_cmd *cmd) kfree(cmd); } +static bool tee_send_destroy_cmd(struct psp_tee_device *tee) +{ + unsigned int reg; + int ret; + + ret = psp_mailbox_command(tee->psp, PSP_CMD_TEE_RING_DESTROY, NULL, + TEE_DEFAULT_CMD_TIMEOUT, ®); + if (ret) { + dev_err(tee->dev, "tee: ring destroy command timed out, disabling TEE support\n"); + psp_dead = true; + return false; + } + + if (FIELD_GET(PSP_CMDRESP_STS, reg)) { + dev_err(tee->dev, "tee: ring destroy command failed (%#010lx)\n", + FIELD_GET(PSP_CMDRESP_STS, reg)); + psp_dead = true; + return false; + } + + return true; +} + static int tee_init_ring(struct psp_tee_device *tee) { int ring_size = MAX_RING_BUFFER_ENTRIES * sizeof(struct tee_ring_cmd); struct tee_init_ring_cmd *cmd; + bool retry = false; unsigned int reg; int ret; @@ -112,6 +136,7 @@ static int tee_init_ring(struct psp_tee_device *tee) /* Send command buffer details to Trusted OS by writing to * CPU-PSP message registers */ +retry_init: ret = psp_mailbox_command(tee->psp, PSP_CMD_TEE_RING_INIT, cmd, TEE_DEFAULT_CMD_TIMEOUT, ®); if (ret) { @@ -122,9 +147,22 @@ static int tee_init_ring(struct psp_tee_device *tee) } if (FIELD_GET(PSP_CMDRESP_STS, reg)) { + /* + * During the hibernate resume sequence driver may have gotten loaded + * but the ring not properly destroyed. If the ring doesn't work, try + * to destroy and re-init once. + */ + if (!retry && FIELD_GET(PSP_CMDRESP_STS, reg) == PSP_TEE_STS_RING_BUSY) { + dev_info(tee->dev, "tee: ring init command failed with busy status, retrying\n"); + if (tee_send_destroy_cmd(tee)) { + retry = true; + goto retry_init; + } + } dev_err(tee->dev, "tee: ring init command failed (%#010lx)\n", FIELD_GET(PSP_CMDRESP_STS, reg)); tee_free_ring(tee); + psp_dead = true; ret = -EIO; } @@ -136,24 +174,13 @@ static int tee_init_ring(struct psp_tee_device *tee) static void tee_destroy_ring(struct psp_tee_device *tee) { - unsigned int reg; - int ret; - if (!tee->rb_mgr.ring_start) return; if (psp_dead) goto free_ring; - ret = psp_mailbox_command(tee->psp, PSP_CMD_TEE_RING_DESTROY, NULL, - TEE_DEFAULT_CMD_TIMEOUT, ®); - if (ret) { - dev_err(tee->dev, "tee: ring destroy command timed out, disabling TEE support\n"); - psp_dead = true; - } else if (FIELD_GET(PSP_CMDRESP_STS, reg)) { - dev_err(tee->dev, "tee: ring destroy command failed (%#010lx)\n", - FIELD_GET(PSP_CMDRESP_STS, reg)); - } + tee_send_destroy_cmd(tee); free_ring: tee_free_ring(tee); @@ -365,3 +392,8 @@ int psp_check_tee_status(void) return 0; } EXPORT_SYMBOL(psp_check_tee_status); + +int tee_restore(struct psp_device *psp) +{ + return tee_init_ring(psp->tee_data); +} diff --git a/drivers/crypto/ccp/tee-dev.h b/drivers/crypto/ccp/tee-dev.h index ea9a2b7c05f577..c23416cb7bb37f 100644 --- a/drivers/crypto/ccp/tee-dev.h +++ b/drivers/crypto/ccp/tee-dev.h @@ -111,5 +111,6 @@ struct tee_ring_cmd { int tee_dev_init(struct psp_device *psp); void tee_dev_destroy(struct psp_device *psp); +int tee_restore(struct psp_device *psp); #endif /* __TEE_DEV_H__ */ diff --git a/drivers/crypto/hisilicon/Kconfig b/drivers/crypto/hisilicon/Kconfig index 4835bdebdbb381..a0cb1a8186ac9d 100644 --- a/drivers/crypto/hisilicon/Kconfig +++ b/drivers/crypto/hisilicon/Kconfig @@ -57,6 +57,7 @@ config CRYPTO_DEV_HISI_ZIP depends on UACCE || UACCE=n depends on ACPI select CRYPTO_DEV_HISI_QM + select CRYPTO_DEFLATE help Support for HiSilicon ZIP Driver diff --git a/drivers/crypto/hisilicon/hpre/hpre.h b/drivers/crypto/hisilicon/hpre/hpre.h index 0f3ddbadbcf991..021dbd9a1d48fb 100644 --- a/drivers/crypto/hisilicon/hpre/hpre.h +++ b/drivers/crypto/hisilicon/hpre/hpre.h @@ -94,9 +94,8 @@ struct hpre_sqe { __le64 key; __le64 in; __le64 out; - __le16 tag; - __le16 resv2; -#define _HPRE_SQE_ALIGN_EXT 7 + __le64 tag; +#define _HPRE_SQE_ALIGN_EXT 6 __le32 rsvd1[_HPRE_SQE_ALIGN_EXT]; }; diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index 21ccf879f70c55..839c1f67714369 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -93,6 +93,7 @@ struct hpre_dh_ctx { char *g; /* m */ dma_addr_t dma_g; + struct crypto_kpp *soft_tfm; }; struct hpre_ecdh_ctx { @@ -103,17 +104,15 @@ struct hpre_ecdh_ctx { /* low address: x->y */ unsigned char *g; dma_addr_t dma_g; + struct crypto_kpp *soft_tfm; }; struct hpre_ctx { struct hisi_qp *qp; struct device *dev; - struct hpre_asym_request **req_list; struct hpre *hpre; - spinlock_t req_lock; unsigned int key_sz; bool crt_g2_mode; - struct idr req_idr; union { struct hpre_rsa_ctx rsa; struct hpre_dh_ctx dh; @@ -123,6 +122,7 @@ struct hpre_ctx { unsigned int curve_id; /* for high performance core */ u8 enable_hpcore; + bool fallback; }; struct hpre_asym_request { @@ -136,7 +136,6 @@ struct hpre_asym_request { struct kpp_request *ecdh; } areq; int err; - int req_id; hpre_cb cb; struct timespec64 req_time; }; @@ -151,79 +150,13 @@ static inline unsigned int hpre_align_pd(void) return (hpre_align_sz() - 1) & ~(crypto_tfm_ctx_alignment() - 1); } -static int hpre_alloc_req_id(struct hpre_ctx *ctx) +static void hpre_dfx_add_req_time(struct hpre_asym_request *hpre_req) { - unsigned long flags; - int id; - - spin_lock_irqsave(&ctx->req_lock, flags); - id = idr_alloc(&ctx->req_idr, NULL, 0, ctx->qp->sq_depth, GFP_ATOMIC); - spin_unlock_irqrestore(&ctx->req_lock, flags); - - return id; -} - -static void hpre_free_req_id(struct hpre_ctx *ctx, int req_id) -{ - unsigned long flags; - - spin_lock_irqsave(&ctx->req_lock, flags); - idr_remove(&ctx->req_idr, req_id); - spin_unlock_irqrestore(&ctx->req_lock, flags); -} - -static int hpre_add_req_to_ctx(struct hpre_asym_request *hpre_req) -{ - struct hpre_ctx *ctx; - struct hpre_dfx *dfx; - int id; - - ctx = hpre_req->ctx; - id = hpre_alloc_req_id(ctx); - if (unlikely(id < 0)) - return -EINVAL; - - ctx->req_list[id] = hpre_req; - hpre_req->req_id = id; + struct hpre_ctx *ctx = hpre_req->ctx; + struct hpre_dfx *dfx = ctx->hpre->debug.dfx; - dfx = ctx->hpre->debug.dfx; if (atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value)) ktime_get_ts64(&hpre_req->req_time); - - return id; -} - -static void hpre_rm_req_from_ctx(struct hpre_asym_request *hpre_req) -{ - struct hpre_ctx *ctx = hpre_req->ctx; - int id = hpre_req->req_id; - - if (hpre_req->req_id >= 0) { - hpre_req->req_id = HPRE_INVLD_REQ_ID; - ctx->req_list[id] = NULL; - hpre_free_req_id(ctx, id); - } -} - -static struct hisi_qp *hpre_get_qp_and_start(u8 type) -{ - struct hisi_qp *qp; - int ret; - - qp = hpre_create_qp(type); - if (!qp) { - pr_err("Can not create hpre qp!\n"); - return ERR_PTR(-ENODEV); - } - - ret = hisi_qm_start_qp(qp, 0); - if (ret < 0) { - hisi_qm_free_qps(&qp, 1); - pci_err(qp->qm->pdev, "Can not start qp!\n"); - return ERR_PTR(-EINVAL); - } - - return qp; } static int hpre_get_data_dma_addr(struct hpre_asym_request *hpre_req, @@ -340,26 +273,19 @@ static void hpre_hw_data_clr_all(struct hpre_ctx *ctx, static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe, void **kreq) { - struct hpre_asym_request *req; unsigned int err, done, alg; - int id; #define HPRE_NO_HW_ERR 0 #define HPRE_HW_TASK_DONE 3 #define HREE_HW_ERR_MASK GENMASK(10, 0) #define HREE_SQE_DONE_MASK GENMASK(1, 0) #define HREE_ALG_TYPE_MASK GENMASK(4, 0) - id = (int)le16_to_cpu(sqe->tag); - req = ctx->req_list[id]; - hpre_rm_req_from_ctx(req); - *kreq = req; + *kreq = (void *)le64_to_cpu(sqe->tag); err = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_ALG_BITS) & HREE_HW_ERR_MASK; - done = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_DONE_SHIFT) & HREE_SQE_DONE_MASK; - if (likely(err == HPRE_NO_HW_ERR && done == HPRE_HW_TASK_DONE)) return 0; @@ -370,36 +296,10 @@ static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe, return -EINVAL; } -static int hpre_ctx_set(struct hpre_ctx *ctx, struct hisi_qp *qp, int qlen) -{ - struct hpre *hpre; - - if (!ctx || !qp || qlen < 0) - return -EINVAL; - - spin_lock_init(&ctx->req_lock); - ctx->qp = qp; - ctx->dev = &qp->qm->pdev->dev; - - hpre = container_of(ctx->qp->qm, struct hpre, qm); - ctx->hpre = hpre; - ctx->req_list = kcalloc(qlen, sizeof(void *), GFP_KERNEL); - if (!ctx->req_list) - return -ENOMEM; - ctx->key_sz = 0; - ctx->crt_g2_mode = false; - idr_init(&ctx->req_idr); - - return 0; -} - static void hpre_ctx_clear(struct hpre_ctx *ctx, bool is_clear_all) { - if (is_clear_all) { - idr_destroy(&ctx->req_idr); - kfree(ctx->req_list); + if (is_clear_all) hisi_qm_free_qps(&ctx->qp, 1); - } ctx->crt_g2_mode = false; ctx->key_sz = 0; @@ -467,49 +367,44 @@ static void hpre_rsa_cb(struct hpre_ctx *ctx, void *resp) static void hpre_alg_cb(struct hisi_qp *qp, void *resp) { - struct hpre_ctx *ctx = qp->qp_ctx; - struct hpre_dfx *dfx = ctx->hpre->debug.dfx; + struct hpre_asym_request *h_req; struct hpre_sqe *sqe = resp; - struct hpre_asym_request *req = ctx->req_list[le16_to_cpu(sqe->tag)]; - if (unlikely(!req)) { - atomic64_inc(&dfx[HPRE_INVALID_REQ_CNT].value); + h_req = (struct hpre_asym_request *)le64_to_cpu(sqe->tag); + if (unlikely(!h_req)) { + pr_err("Failed to get request, and qp_id is %u\n", qp->qp_id); return; } - req->cb(ctx, resp); -} - -static void hpre_stop_qp_and_put(struct hisi_qp *qp) -{ - hisi_qm_stop_qp(qp); - hisi_qm_free_qps(&qp, 1); + h_req->cb(h_req->ctx, resp); } static int hpre_ctx_init(struct hpre_ctx *ctx, u8 type) { struct hisi_qp *qp; - int ret; + struct hpre *hpre; - qp = hpre_get_qp_and_start(type); - if (IS_ERR(qp)) - return PTR_ERR(qp); + qp = hpre_create_qp(type); + if (!qp) { + ctx->qp = NULL; + return -ENODEV; + } - qp->qp_ctx = ctx; qp->req_cb = hpre_alg_cb; + ctx->qp = qp; + ctx->dev = &qp->qm->pdev->dev; + hpre = container_of(ctx->qp->qm, struct hpre, qm); + ctx->hpre = hpre; + ctx->key_sz = 0; + ctx->crt_g2_mode = false; - ret = hpre_ctx_set(ctx, qp, qp->sq_depth); - if (ret) - hpre_stop_qp_and_put(qp); - - return ret; + return 0; } static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa) { struct hpre_asym_request *h_req; struct hpre_sqe *msg; - int req_id; void *tmp; if (is_rsa) { @@ -549,11 +444,8 @@ static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa) msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; h_req->ctx = ctx; - req_id = hpre_add_req_to_ctx(h_req); - if (req_id < 0) - return -EBUSY; - - msg->tag = cpu_to_le16((u16)req_id); + hpre_dfx_add_req_time(h_req); + msg->tag = cpu_to_le64((uintptr_t)h_req); return 0; } @@ -566,9 +458,7 @@ static int hpre_send(struct hpre_ctx *ctx, struct hpre_sqe *msg) do { atomic64_inc(&dfx[HPRE_SEND_CNT].value); - spin_lock_bh(&ctx->req_lock); ret = hisi_qp_send(ctx->qp, msg); - spin_unlock_bh(&ctx->req_lock); if (ret != -EBUSY) break; atomic64_inc(&dfx[HPRE_SEND_BUSY_CNT].value); @@ -619,12 +509,53 @@ static int hpre_dh_compute_value(struct kpp_request *req) return -EINPROGRESS; clear_all: - hpre_rm_req_from_ctx(hpre_req); hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); return ret; } +static struct kpp_request *hpre_dh_prepare_fb_req(struct kpp_request *req) +{ + struct kpp_request *fb_req = kpp_request_ctx(req); + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + + kpp_request_set_tfm(fb_req, ctx->dh.soft_tfm); + kpp_request_set_callback(fb_req, req->base.flags, req->base.complete, req->base.data); + kpp_request_set_input(fb_req, req->src, req->src_len); + kpp_request_set_output(fb_req, req->dst, req->dst_len); + + return fb_req; +} + +static int hpre_dh_generate_public_key(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + struct kpp_request *fb_req; + + if (ctx->fallback) { + fb_req = hpre_dh_prepare_fb_req(req); + return crypto_kpp_generate_public_key(fb_req); + } + + return hpre_dh_compute_value(req); +} + +static int hpre_dh_compute_shared_secret(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + struct kpp_request *fb_req; + + if (ctx->fallback) { + fb_req = hpre_dh_prepare_fb_req(req); + return crypto_kpp_compute_shared_secret(fb_req); + } + + return hpre_dh_compute_value(req); +} + static int hpre_is_dh_params_length_valid(unsigned int key_sz) { #define _HPRE_DH_GRP1 768 @@ -651,13 +582,6 @@ static int hpre_dh_set_params(struct hpre_ctx *ctx, struct dh *params) struct device *dev = ctx->dev; unsigned int sz; - if (params->p_size > HPRE_DH_MAX_P_SZ) - return -EINVAL; - - if (hpre_is_dh_params_length_valid(params->p_size << - HPRE_BITS_2_BYTES_SHIFT)) - return -EINVAL; - sz = ctx->key_sz = params->p_size; ctx->dh.xa_p = dma_alloc_coherent(dev, sz << 1, &ctx->dh.dma_xa_p, GFP_KERNEL); @@ -690,8 +614,8 @@ static void hpre_dh_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) struct device *dev = ctx->dev; unsigned int sz = ctx->key_sz; - if (is_clear_all) - hisi_qm_stop_qp(ctx->qp); + if (!ctx->qp) + return; if (ctx->dh.g) { dma_free_coherent(dev, sz, ctx->dh.g, ctx->dh.dma_g); @@ -718,6 +642,13 @@ static int hpre_dh_set_secret(struct crypto_kpp *tfm, const void *buf, if (crypto_dh_decode_key(buf, len, ¶ms) < 0) return -EINVAL; + if (!ctx->qp) + goto set_soft_secret; + + if (hpre_is_dh_params_length_valid(params.p_size << + HPRE_BITS_2_BYTES_SHIFT)) + goto set_soft_secret; + /* Free old secret if any */ hpre_dh_clear_ctx(ctx, false); @@ -728,27 +659,55 @@ static int hpre_dh_set_secret(struct crypto_kpp *tfm, const void *buf, memcpy(ctx->dh.xa_p + (ctx->key_sz - params.key_size), params.key, params.key_size); + ctx->fallback = false; return 0; err_clear_ctx: hpre_dh_clear_ctx(ctx, false); return ret; +set_soft_secret: + ctx->fallback = true; + return crypto_kpp_set_secret(ctx->dh.soft_tfm, buf, len); } static unsigned int hpre_dh_max_size(struct crypto_kpp *tfm) { struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + if (ctx->fallback) + return crypto_kpp_maxsize(ctx->dh.soft_tfm); + return ctx->key_sz; } static int hpre_dh_init_tfm(struct crypto_kpp *tfm) { struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + const char *alg = kpp_alg_name(tfm); + unsigned int reqsize; + int ret; + + ctx->dh.soft_tfm = crypto_alloc_kpp(alg, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->dh.soft_tfm)) { + pr_err("Failed to alloc dh tfm!\n"); + return PTR_ERR(ctx->dh.soft_tfm); + } - kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); + crypto_kpp_set_flags(ctx->dh.soft_tfm, crypto_kpp_get_flags(tfm)); - return hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE); + reqsize = max(sizeof(struct hpre_asym_request) + hpre_align_pd(), + sizeof(struct kpp_request) + crypto_kpp_reqsize(ctx->dh.soft_tfm)); + kpp_set_reqsize(tfm, reqsize); + + ret = hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE); + if (ret && ret != -ENODEV) { + crypto_free_kpp(ctx->dh.soft_tfm); + return ret; + } else if (ret == -ENODEV) { + ctx->fallback = true; + } + + return 0; } static void hpre_dh_exit_tfm(struct crypto_kpp *tfm) @@ -756,6 +715,7 @@ static void hpre_dh_exit_tfm(struct crypto_kpp *tfm) struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); hpre_dh_clear_ctx(ctx, true); + crypto_free_kpp(ctx->dh.soft_tfm); } static void hpre_rsa_drop_leading_zeros(const char **ptr, size_t *len) @@ -795,9 +755,8 @@ static int hpre_rsa_enc(struct akcipher_request *req) struct hpre_sqe *msg = &hpre_req->req; int ret; - /* For 512 and 1536 bits key size, use soft tfm instead */ - if (ctx->key_sz == HPRE_RSA_512BITS_KSZ || - ctx->key_sz == HPRE_RSA_1536BITS_KSZ) { + /* For unsupported key size and unavailable devices, use soft tfm instead */ + if (ctx->fallback) { akcipher_request_set_tfm(req, ctx->rsa.soft_tfm); ret = crypto_akcipher_encrypt(req); akcipher_request_set_tfm(req, tfm); @@ -828,7 +787,6 @@ static int hpre_rsa_enc(struct akcipher_request *req) return -EINPROGRESS; clear_all: - hpre_rm_req_from_ctx(hpre_req); hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); return ret; @@ -843,9 +801,8 @@ static int hpre_rsa_dec(struct akcipher_request *req) struct hpre_sqe *msg = &hpre_req->req; int ret; - /* For 512 and 1536 bits key size, use soft tfm instead */ - if (ctx->key_sz == HPRE_RSA_512BITS_KSZ || - ctx->key_sz == HPRE_RSA_1536BITS_KSZ) { + /* For unsupported key size and unavailable devices, use soft tfm instead */ + if (ctx->fallback) { akcipher_request_set_tfm(req, ctx->rsa.soft_tfm); ret = crypto_akcipher_decrypt(req); akcipher_request_set_tfm(req, tfm); @@ -883,7 +840,6 @@ static int hpre_rsa_dec(struct akcipher_request *req) return -EINPROGRESS; clear_all: - hpre_rm_req_from_ctx(hpre_req); hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); return ret; @@ -899,8 +855,10 @@ static int hpre_rsa_set_n(struct hpre_ctx *ctx, const char *value, ctx->key_sz = vlen; /* if invalid key size provided, we use software tfm */ - if (!hpre_rsa_key_size_is_support(ctx->key_sz)) + if (!hpre_rsa_key_size_is_support(ctx->key_sz)) { + ctx->fallback = true; return 0; + } ctx->rsa.pubkey = dma_alloc_coherent(ctx->dev, vlen << 1, &ctx->rsa.dma_pubkey, @@ -1035,8 +993,8 @@ static void hpre_rsa_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) unsigned int half_key_sz = ctx->key_sz >> 1; struct device *dev = ctx->dev; - if (is_clear_all) - hisi_qm_stop_qp(ctx->qp); + if (!ctx->qp) + return; if (ctx->rsa.pubkey) { dma_free_coherent(dev, ctx->key_sz << 1, @@ -1117,6 +1075,7 @@ static int hpre_rsa_setkey(struct hpre_ctx *ctx, const void *key, goto free; } + ctx->fallback = false; return 0; free: @@ -1134,6 +1093,9 @@ static int hpre_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key, if (ret) return ret; + if (!ctx->qp) + return 0; + return hpre_rsa_setkey(ctx, key, keylen, false); } @@ -1147,6 +1109,9 @@ static int hpre_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key, if (ret) return ret; + if (!ctx->qp) + return 0; + return hpre_rsa_setkey(ctx, key, keylen, true); } @@ -1154,9 +1119,8 @@ static unsigned int hpre_rsa_max_size(struct crypto_akcipher *tfm) { struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); - /* For 512 and 1536 bits key size, use soft tfm instead */ - if (ctx->key_sz == HPRE_RSA_512BITS_KSZ || - ctx->key_sz == HPRE_RSA_1536BITS_KSZ) + /* For unsupported key size and unavailable devices, use soft tfm instead */ + if (ctx->fallback) return crypto_akcipher_maxsize(ctx->rsa.soft_tfm); return ctx->key_sz; @@ -1177,10 +1141,14 @@ static int hpre_rsa_init_tfm(struct crypto_akcipher *tfm) hpre_align_pd()); ret = hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE); - if (ret) + if (ret && ret != -ENODEV) { crypto_free_akcipher(ctx->rsa.soft_tfm); + return ret; + } else if (ret == -ENODEV) { + ctx->fallback = true; + } - return ret; + return 0; } static void hpre_rsa_exit_tfm(struct crypto_akcipher *tfm) @@ -1207,9 +1175,6 @@ static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) unsigned int sz = ctx->key_sz; unsigned int shift = sz << 1; - if (is_clear_all) - hisi_qm_stop_qp(ctx->qp); - if (ctx->ecdh.p) { /* ecdh: p->a->k->b */ memzero_explicit(ctx->ecdh.p + shift, sz); @@ -1346,7 +1311,7 @@ static int hpre_ecdh_set_param(struct hpre_ctx *ctx, struct ecdh *params) return 0; } -static bool hpre_key_is_zero(char *key, unsigned short key_sz) +static bool hpre_key_is_zero(const char *key, unsigned short key_sz) { int i; @@ -1387,6 +1352,9 @@ static int hpre_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, struct ecdh params; int ret; + if (ctx->fallback) + return crypto_kpp_set_secret(ctx->ecdh.soft_tfm, buf, len); + if (crypto_ecdh_decode_key(buf, len, ¶ms) < 0) { dev_err(dev, "failed to decode ecdh key!\n"); return -EINVAL; @@ -1488,7 +1456,6 @@ static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx, { struct hpre_asym_request *h_req; struct hpre_sqe *msg; - int req_id; void *tmp; if (req->dst_len < ctx->key_sz << 1) { @@ -1510,11 +1477,8 @@ static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx, msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; h_req->ctx = ctx; - req_id = hpre_add_req_to_ctx(h_req); - if (req_id < 0) - return -EBUSY; - - msg->tag = cpu_to_le16((u16)req_id); + hpre_dfx_add_req_time(h_req); + msg->tag = cpu_to_le64((uintptr_t)h_req); return 0; } @@ -1612,28 +1576,86 @@ static int hpre_ecdh_compute_value(struct kpp_request *req) return -EINPROGRESS; clear_all: - hpre_rm_req_from_ctx(hpre_req); hpre_ecdh_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); return ret; } +static int hpre_ecdh_generate_public_key(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + int ret; + + if (ctx->fallback) { + kpp_request_set_tfm(req, ctx->ecdh.soft_tfm); + ret = crypto_kpp_generate_public_key(req); + kpp_request_set_tfm(req, tfm); + return ret; + } + + return hpre_ecdh_compute_value(req); +} + +static int hpre_ecdh_compute_shared_secret(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + int ret; + + if (ctx->fallback) { + kpp_request_set_tfm(req, ctx->ecdh.soft_tfm); + ret = crypto_kpp_compute_shared_secret(req); + kpp_request_set_tfm(req, tfm); + return ret; + } + + return hpre_ecdh_compute_value(req); +} + static unsigned int hpre_ecdh_max_size(struct crypto_kpp *tfm) { struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + if (ctx->fallback) + return crypto_kpp_maxsize(ctx->ecdh.soft_tfm); + /* max size is the pub_key_size, include x and y */ return ctx->key_sz << 1; } +static int hpre_ecdh_init_tfm(struct crypto_kpp *tfm) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + const char *alg = kpp_alg_name(tfm); + int ret; + + ret = hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); + if (!ret) { + kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); + return 0; + } else if (ret && ret != -ENODEV) { + return ret; + } + + ctx->ecdh.soft_tfm = crypto_alloc_kpp(alg, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->ecdh.soft_tfm)) { + pr_err("Failed to alloc %s tfm!\n", alg); + return PTR_ERR(ctx->ecdh.soft_tfm); + } + + crypto_kpp_set_flags(ctx->ecdh.soft_tfm, crypto_kpp_get_flags(tfm)); + ctx->fallback = true; + + return 0; +} + static int hpre_ecdh_nist_p192_init_tfm(struct crypto_kpp *tfm) { struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); ctx->curve_id = ECC_CURVE_NIST_P192; - kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); - - return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); + return hpre_ecdh_init_tfm(tfm); } static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm) @@ -1643,9 +1665,7 @@ static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm) ctx->curve_id = ECC_CURVE_NIST_P256; ctx->enable_hpcore = 1; - kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); - - return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); + return hpre_ecdh_init_tfm(tfm); } static int hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm) @@ -1654,15 +1674,18 @@ static int hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm) ctx->curve_id = ECC_CURVE_NIST_P384; - kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); - - return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); + return hpre_ecdh_init_tfm(tfm); } static void hpre_ecdh_exit_tfm(struct crypto_kpp *tfm) { struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + if (ctx->fallback) { + crypto_free_kpp(ctx->ecdh.soft_tfm); + return; + } + hpre_ecc_clear_ctx(ctx, true); } @@ -1680,13 +1703,14 @@ static struct akcipher_alg rsa = { .cra_name = "rsa", .cra_driver_name = "hpre-rsa", .cra_module = THIS_MODULE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, }, }; static struct kpp_alg dh = { .set_secret = hpre_dh_set_secret, - .generate_public_key = hpre_dh_compute_value, - .compute_shared_secret = hpre_dh_compute_value, + .generate_public_key = hpre_dh_generate_public_key, + .compute_shared_secret = hpre_dh_compute_shared_secret, .max_size = hpre_dh_max_size, .init = hpre_dh_init_tfm, .exit = hpre_dh_exit_tfm, @@ -1696,14 +1720,15 @@ static struct kpp_alg dh = { .cra_name = "dh", .cra_driver_name = "hpre-dh", .cra_module = THIS_MODULE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, }, }; static struct kpp_alg ecdh_curves[] = { { .set_secret = hpre_ecdh_set_secret, - .generate_public_key = hpre_ecdh_compute_value, - .compute_shared_secret = hpre_ecdh_compute_value, + .generate_public_key = hpre_ecdh_generate_public_key, + .compute_shared_secret = hpre_ecdh_compute_shared_secret, .max_size = hpre_ecdh_max_size, .init = hpre_ecdh_nist_p192_init_tfm, .exit = hpre_ecdh_exit_tfm, @@ -1713,11 +1738,12 @@ static struct kpp_alg ecdh_curves[] = { .cra_name = "ecdh-nist-p192", .cra_driver_name = "hpre-ecdh-nist-p192", .cra_module = THIS_MODULE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, }, }, { .set_secret = hpre_ecdh_set_secret, - .generate_public_key = hpre_ecdh_compute_value, - .compute_shared_secret = hpre_ecdh_compute_value, + .generate_public_key = hpre_ecdh_generate_public_key, + .compute_shared_secret = hpre_ecdh_compute_shared_secret, .max_size = hpre_ecdh_max_size, .init = hpre_ecdh_nist_p256_init_tfm, .exit = hpre_ecdh_exit_tfm, @@ -1727,11 +1753,12 @@ static struct kpp_alg ecdh_curves[] = { .cra_name = "ecdh-nist-p256", .cra_driver_name = "hpre-ecdh-nist-p256", .cra_module = THIS_MODULE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, }, }, { .set_secret = hpre_ecdh_set_secret, - .generate_public_key = hpre_ecdh_compute_value, - .compute_shared_secret = hpre_ecdh_compute_value, + .generate_public_key = hpre_ecdh_generate_public_key, + .compute_shared_secret = hpre_ecdh_compute_shared_secret, .max_size = hpre_ecdh_max_size, .init = hpre_ecdh_nist_p384_init_tfm, .exit = hpre_ecdh_exit_tfm, @@ -1741,6 +1768,7 @@ static struct kpp_alg ecdh_curves[] = { .cra_name = "ecdh-nist-p384", .cra_driver_name = "hpre-ecdh-nist-p384", .cra_module = THIS_MODULE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, }, } }; diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index b94fecd765eebd..884d5d0afaf41f 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -465,7 +465,7 @@ struct hisi_qp *hpre_create_qp(u8 type) * type: 0 - RSA/DH. algorithm supported in V2, * 1 - ECC algorithm in V3. */ - ret = hisi_qm_alloc_qps_node(&hpre_devices, 1, type, node, &qp); + ret = hisi_qm_alloc_qps_node(&hpre_devices, 1, &type, node, &qp); if (!ret) return qp; diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index d47bf06a90f7d3..b8e59f99f7007c 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -2219,6 +2219,7 @@ static void qp_stop_fail_cb(struct hisi_qp *qp) for (i = 0; i < qp_used; i++) { pos = (i + cur_head) % sq_depth; qp->req_cb(qp, qp->sqe + (u32)(qm->sqe_size * pos)); + qm_cq_head_update(qp); atomic_dec(&qp->qp_status.used); } } @@ -2368,25 +2369,33 @@ EXPORT_SYMBOL_GPL(hisi_qm_stop_qp); int hisi_qp_send(struct hisi_qp *qp, const void *msg) { struct hisi_qp_status *qp_status = &qp->qp_status; - u16 sq_tail = qp_status->sq_tail; - u16 sq_tail_next = (sq_tail + 1) % qp->sq_depth; - void *sqe = qm_get_avail_sqe(qp); + u16 sq_tail, sq_tail_next; + void *sqe; + spin_lock_bh(&qp->qp_lock); if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP || atomic_read(&qp->qm->status.flags) == QM_STOP || qp->is_resetting)) { + spin_unlock_bh(&qp->qp_lock); dev_info_ratelimited(&qp->qm->pdev->dev, "QP is stopped or resetting\n"); return -EAGAIN; } - if (!sqe) + sqe = qm_get_avail_sqe(qp); + if (!sqe) { + spin_unlock_bh(&qp->qp_lock); return -EBUSY; + } + sq_tail = qp_status->sq_tail; + sq_tail_next = (sq_tail + 1) % qp->sq_depth; memcpy(sqe, msg, qp->qm->sqe_size); + qp->msg[sq_tail] = msg; qm_db(qp->qm, qp->qp_id, QM_DOORBELL_CMD_SQ, sq_tail_next, 0); atomic_inc(&qp->qp_status.used); qp_status->sq_tail = sq_tail_next; + spin_unlock_bh(&qp->qp_lock); return 0; } @@ -2919,12 +2928,13 @@ EXPORT_SYMBOL_GPL(hisi_qm_wait_task_finish); static void hisi_qp_memory_uninit(struct hisi_qm *qm, int num) { struct device *dev = &qm->pdev->dev; - struct qm_dma *qdma; + struct hisi_qp *qp; int i; for (i = num - 1; i >= 0; i--) { - qdma = &qm->qp_array[i].qdma; - dma_free_coherent(dev, qdma->size, qdma->va, qdma->dma); + qp = &qm->qp_array[i]; + dma_free_coherent(dev, qp->qdma.size, qp->qdma.va, qp->qdma.dma); + kfree(qp->msg); kfree(qm->poll_data[i].qp_finish_id); } @@ -2946,10 +2956,14 @@ static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id, return -ENOMEM; qp = &qm->qp_array[id]; + qp->msg = kmalloc_array(sq_depth, sizeof(void *), GFP_KERNEL); + if (!qp->msg) + goto err_free_qp_finish_id; + qp->qdma.va = dma_alloc_coherent(dev, dma_size, &qp->qdma.dma, GFP_KERNEL); if (!qp->qdma.va) - goto err_free_qp_finish_id; + goto err_free_qp_msg; qp->sqe = qp->qdma.va; qp->sqe_dma = qp->qdma.dma; @@ -2961,8 +2975,14 @@ static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id, qp->qm = qm; qp->qp_id = id; + spin_lock_init(&qp->qp_lock); + spin_lock_init(&qp->backlog.lock); + INIT_LIST_HEAD(&qp->backlog.list); + return 0; +err_free_qp_msg: + kfree(qp->msg); err_free_qp_finish_id: kfree(qm->poll_data[id].qp_finish_id); return ret; @@ -3533,6 +3553,14 @@ void hisi_qm_dev_err_uninit(struct hisi_qm *qm) } EXPORT_SYMBOL_GPL(hisi_qm_dev_err_uninit); +static void qm_release_qp_nolock(struct hisi_qp *qp) +{ + struct hisi_qm *qm = qp->qm; + + qm->qp_in_used--; + idr_remove(&qm->qp_idr, qp->qp_id); +} + /** * hisi_qm_free_qps() - free multiple queue pairs. * @qps: The queue pairs need to be freed. @@ -3545,8 +3573,15 @@ void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num) if (!qps || qp_num <= 0) return; - for (i = qp_num - 1; i >= 0; i--) - hisi_qm_release_qp(qps[i]); + down_write(&qps[0]->qm->qps_lock); + + for (i = qp_num - 1; i >= 0; i--) { + qm_stop_qp_nolock(qps[i]); + qm_release_qp_nolock(qps[i]); + } + + up_write(&qps[0]->qm->qps_lock); + qm_pm_put_sync(qps[0]->qm); } EXPORT_SYMBOL_GPL(hisi_qm_free_qps); @@ -3560,6 +3595,43 @@ static void free_list(struct list_head *head) } } +static int qm_get_and_start_qp(struct hisi_qm *qm, int qp_num, struct hisi_qp **qps, u8 *alg_type) +{ + int i, ret; + + ret = qm_pm_get_sync(qm); + if (ret) + return ret; + + down_write(&qm->qps_lock); + for (i = 0; i < qp_num; i++) { + qps[i] = qm_create_qp_nolock(qm, alg_type[i]); + if (IS_ERR(qps[i])) { + ret = -ENODEV; + goto stop_and_free; + } + + ret = qm_start_qp_nolock(qps[i], 0); + if (ret) { + qm_release_qp_nolock(qps[i]); + goto stop_and_free; + } + } + up_write(&qm->qps_lock); + + return 0; + +stop_and_free: + for (i--; i >= 0; i--) { + qm_stop_qp_nolock(qps[i]); + qm_release_qp_nolock(qps[i]); + } + up_write(&qm->qps_lock); + qm_pm_put_sync(qm); + + return ret; +} + static int hisi_qm_sort_devices(int node, struct list_head *head, struct hisi_qm_list *qm_list) { @@ -3608,12 +3680,11 @@ static int hisi_qm_sort_devices(int node, struct list_head *head, * not meet the requirements will return error. */ int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num, - u8 alg_type, int node, struct hisi_qp **qps) + u8 *alg_type, int node, struct hisi_qp **qps) { struct hisi_qm_resource *tmp; int ret = -ENODEV; LIST_HEAD(head); - int i; if (!qps || !qm_list || qp_num <= 0) return -EINVAL; @@ -3625,24 +3696,15 @@ int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num, } list_for_each_entry(tmp, &head, list) { - for (i = 0; i < qp_num; i++) { - qps[i] = hisi_qm_create_qp(tmp->qm, alg_type); - if (IS_ERR(qps[i])) { - hisi_qm_free_qps(qps, i); - break; - } - } - - if (i == qp_num) { - ret = 0; + ret = qm_get_and_start_qp(tmp->qm, qp_num, qps, alg_type); + if (!ret) break; - } } mutex_unlock(&qm_list->lock); if (ret) - pr_info("Failed to create qps, node[%d], alg[%u], qp[%d]!\n", - node, alg_type, qp_num); + pr_info("Failed to create qps, node[%d], qp[%d]!\n", + node, qp_num); err: free_list(&head); diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h index 81d0beda93b2bb..0710977861f323 100644 --- a/drivers/crypto/hisilicon/sec2/sec.h +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -82,11 +82,6 @@ struct sec_aead_req { __u8 out_mac_buf[SEC_MAX_MAC_LEN]; }; -struct sec_instance_backlog { - struct list_head list; - spinlock_t lock; -}; - /* SEC request of Crypto */ struct sec_req { union { @@ -112,7 +107,6 @@ struct sec_req { bool use_pbuf; struct list_head list; - struct sec_instance_backlog *backlog; struct sec_request_buf buf; }; @@ -172,7 +166,6 @@ struct sec_qp_ctx { spinlock_t id_lock; struct hisi_acc_sgl_pool *c_in_pool; struct hisi_acc_sgl_pool *c_out_pool; - struct sec_instance_backlog backlog; u16 send_head; }; diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 31590d01139a37..c462b58d303435 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -54,7 +54,6 @@ #define SEC_AUTH_CIPHER_V3 0x40 #define SEC_FLAG_OFFSET 7 #define SEC_FLAG_MASK 0x0780 -#define SEC_TYPE_MASK 0x0F #define SEC_DONE_MASK 0x0001 #define SEC_ICV_MASK 0x000E @@ -148,7 +147,7 @@ static void sec_free_req_id(struct sec_req *req) spin_unlock_bh(&qp_ctx->id_lock); } -static u8 pre_parse_finished_bd(struct bd_status *status, void *resp) +static void pre_parse_finished_bd(struct bd_status *status, void *resp) { struct sec_sqe *bd = resp; @@ -158,11 +157,9 @@ static u8 pre_parse_finished_bd(struct bd_status *status, void *resp) SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; status->tag = le16_to_cpu(bd->type2.tag); status->err_type = bd->type2.error_type; - - return bd->type_cipher_auth & SEC_TYPE_MASK; } -static u8 pre_parse_finished_bd3(struct bd_status *status, void *resp) +static void pre_parse_finished_bd3(struct bd_status *status, void *resp) { struct sec_sqe3 *bd3 = resp; @@ -172,8 +169,6 @@ static u8 pre_parse_finished_bd3(struct bd_status *status, void *resp) SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; status->tag = le64_to_cpu(bd3->tag); status->err_type = bd3->error_type; - - return le32_to_cpu(bd3->bd_param) & SEC_TYPE_MASK; } static int sec_cb_status_check(struct sec_req *req, @@ -244,7 +239,7 @@ static void sec_alg_send_backlog_soft(struct sec_ctx *ctx, struct sec_qp_ctx *qp struct sec_req *req, *tmp; int ret; - list_for_each_entry_safe(req, tmp, &qp_ctx->backlog.list, list) { + list_for_each_entry_safe(req, tmp, &qp_ctx->qp->backlog.list, list) { list_del(&req->list); ctx->req_op->buf_unmap(ctx, req); if (req->req_id >= 0) @@ -265,11 +260,12 @@ static void sec_alg_send_backlog_soft(struct sec_ctx *ctx, struct sec_qp_ctx *qp static void sec_alg_send_backlog(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) { + struct hisi_qp *qp = qp_ctx->qp; struct sec_req *req, *tmp; int ret; - spin_lock_bh(&qp_ctx->backlog.lock); - list_for_each_entry_safe(req, tmp, &qp_ctx->backlog.list, list) { + spin_lock_bh(&qp->backlog.lock); + list_for_each_entry_safe(req, tmp, &qp->backlog.list, list) { ret = qp_send_message(req); switch (ret) { case -EINPROGRESS: @@ -287,42 +283,46 @@ static void sec_alg_send_backlog(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) } unlock: - spin_unlock_bh(&qp_ctx->backlog.lock); + spin_unlock_bh(&qp->backlog.lock); } static void sec_req_cb(struct hisi_qp *qp, void *resp) { - struct sec_qp_ctx *qp_ctx = qp->qp_ctx; - struct sec_dfx *dfx = &qp_ctx->ctx->sec->debug.dfx; - u8 type_supported = qp_ctx->ctx->type_supported; + const struct sec_sqe *sqe = qp->msg[qp->qp_status.cq_head]; + struct sec_req *req = container_of(sqe, struct sec_req, sec_sqe); + struct sec_ctx *ctx = req->ctx; + struct sec_dfx *dfx = &ctx->sec->debug.dfx; struct bd_status status; - struct sec_ctx *ctx; - struct sec_req *req; int err; - u8 type; - if (type_supported == SEC_BD_TYPE2) { - type = pre_parse_finished_bd(&status, resp); - req = qp_ctx->req_list[status.tag]; - } else { - type = pre_parse_finished_bd3(&status, resp); - req = (void *)(uintptr_t)status.tag; - } + pre_parse_finished_bd(&status, resp); - if (unlikely(type != type_supported)) { - atomic64_inc(&dfx->err_bd_cnt); - pr_err("err bd type [%u]\n", type); - return; - } + req->err_type = status.err_type; + err = sec_cb_status_check(req, &status); + if (err) + atomic64_inc(&dfx->done_flag_cnt); - if (unlikely(!req)) { - atomic64_inc(&dfx->invalid_req_cnt); - atomic_inc(&qp->qp_status.used); - return; - } + atomic64_inc(&dfx->recv_cnt); + ctx->req_op->buf_unmap(ctx, req); + ctx->req_op->callback(ctx, req, err); +} + +static void sec_req_cb3(struct hisi_qp *qp, void *resp) +{ + struct bd_status status; + struct sec_ctx *ctx; + struct sec_dfx *dfx; + struct sec_req *req; + int err; + + pre_parse_finished_bd3(&status, resp); + + req = (void *)(uintptr_t)status.tag; req->err_type = status.err_type; ctx = req->ctx; + dfx = &ctx->sec->debug.dfx; + err = sec_cb_status_check(req, &status); if (err) atomic64_inc(&dfx->done_flag_cnt); @@ -330,7 +330,6 @@ static void sec_req_cb(struct hisi_qp *qp, void *resp) atomic64_inc(&dfx->recv_cnt); ctx->req_op->buf_unmap(ctx, req); - ctx->req_op->callback(ctx, req, err); } @@ -348,8 +347,10 @@ static int sec_alg_send_message_retry(struct sec_req *req) static int sec_alg_try_enqueue(struct sec_req *req) { + struct hisi_qp *qp = req->qp_ctx->qp; + /* Check if any request is already backlogged */ - if (!list_empty(&req->backlog->list)) + if (!list_empty(&qp->backlog.list)) return -EBUSY; /* Try to enqueue to HW ring */ @@ -359,17 +360,18 @@ static int sec_alg_try_enqueue(struct sec_req *req) static int sec_alg_send_message_maybacklog(struct sec_req *req) { + struct hisi_qp *qp = req->qp_ctx->qp; int ret; ret = sec_alg_try_enqueue(req); if (ret != -EBUSY) return ret; - spin_lock_bh(&req->backlog->lock); + spin_lock_bh(&qp->backlog.lock); ret = sec_alg_try_enqueue(req); if (ret == -EBUSY) - list_add_tail(&req->list, &req->backlog->list); - spin_unlock_bh(&req->backlog->lock); + list_add_tail(&req->list, &qp->backlog.list); + spin_unlock_bh(&qp->backlog.lock); return ret; } @@ -624,32 +626,25 @@ static int sec_create_qp_ctx(struct sec_ctx *ctx, int qp_ctx_id) qp_ctx = &ctx->qp_ctx[qp_ctx_id]; qp = ctx->qps[qp_ctx_id]; - qp->req_type = 0; - qp->qp_ctx = qp_ctx; qp_ctx->qp = qp; qp_ctx->ctx = ctx; - qp->req_cb = sec_req_cb; + if (ctx->type_supported == SEC_BD_TYPE3) + qp->req_cb = sec_req_cb3; + else + qp->req_cb = sec_req_cb; spin_lock_init(&qp_ctx->req_lock); idr_init(&qp_ctx->req_idr); - spin_lock_init(&qp_ctx->backlog.lock); spin_lock_init(&qp_ctx->id_lock); - INIT_LIST_HEAD(&qp_ctx->backlog.list); qp_ctx->send_head = 0; ret = sec_alloc_qp_ctx_resource(ctx, qp_ctx); if (ret) goto err_destroy_idr; - ret = hisi_qm_start_qp(qp, 0); - if (ret < 0) - goto err_resource_free; - return 0; -err_resource_free: - sec_free_qp_ctx_resource(ctx, qp_ctx); err_destroy_idr: idr_destroy(&qp_ctx->req_idr); return ret; @@ -658,7 +653,6 @@ static int sec_create_qp_ctx(struct sec_ctx *ctx, int qp_ctx_id) static void sec_release_qp_ctx(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) { - hisi_qm_stop_qp(qp_ctx->qp); sec_free_qp_ctx_resource(ctx, qp_ctx); idr_destroy(&qp_ctx->req_idr); } @@ -669,10 +663,8 @@ static int sec_ctx_base_init(struct sec_ctx *ctx) int i, ret; ctx->qps = sec_create_qps(); - if (!ctx->qps) { - pr_err("Can not create sec qps!\n"); + if (!ctx->qps) return -ENODEV; - } sec = container_of(ctx->qps[0]->qm, struct sec_dev, qm); ctx->sec = sec; @@ -708,6 +700,9 @@ static void sec_ctx_base_uninit(struct sec_ctx *ctx) { int i; + if (!ctx->qps) + return; + for (i = 0; i < ctx->sec->ctx_q_num; i++) sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); @@ -719,6 +714,9 @@ static int sec_cipher_init(struct sec_ctx *ctx) { struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + if (!ctx->qps) + return 0; + c_ctx->c_key = dma_alloc_coherent(ctx->dev, SEC_MAX_KEY_SIZE, &c_ctx->c_key_dma, GFP_KERNEL); if (!c_ctx->c_key) @@ -731,6 +729,9 @@ static void sec_cipher_uninit(struct sec_ctx *ctx) { struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + if (!ctx->qps) + return; + memzero_explicit(c_ctx->c_key, SEC_MAX_KEY_SIZE); dma_free_coherent(ctx->dev, SEC_MAX_KEY_SIZE, c_ctx->c_key, c_ctx->c_key_dma); @@ -752,6 +753,9 @@ static void sec_auth_uninit(struct sec_ctx *ctx) { struct sec_auth_ctx *a_ctx = &ctx->a_ctx; + if (!ctx->qps) + return; + memzero_explicit(a_ctx->a_key, SEC_MAX_AKEY_SIZE); dma_free_coherent(ctx->dev, SEC_MAX_AKEY_SIZE, a_ctx->a_key, a_ctx->a_key_dma); @@ -789,7 +793,7 @@ static int sec_skcipher_init(struct crypto_skcipher *tfm) } ret = sec_ctx_base_init(ctx); - if (ret) + if (ret && ret != -ENODEV) return ret; ret = sec_cipher_init(ctx); @@ -898,6 +902,9 @@ static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, struct device *dev = ctx->dev; int ret; + if (!ctx->qps) + goto set_soft_key; + if (c_mode == SEC_CMODE_XTS) { ret = xts_verify_key(tfm, key, keylen); if (ret) { @@ -928,13 +935,14 @@ static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, } memcpy(c_ctx->c_key, key, keylen); - if (c_ctx->fbtfm) { - ret = crypto_sync_skcipher_setkey(c_ctx->fbtfm, key, keylen); - if (ret) { - dev_err(dev, "failed to set fallback skcipher key!\n"); - return ret; - } + +set_soft_key: + ret = crypto_sync_skcipher_setkey(c_ctx->fbtfm, key, keylen); + if (ret) { + dev_err(dev, "failed to set fallback skcipher key!\n"); + return ret; } + return 0; } @@ -1398,6 +1406,9 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, struct crypto_authenc_keys keys; int ret; + if (!ctx->qps) + return sec_aead_fallback_setkey(a_ctx, tfm, key, keylen); + ctx->a_ctx.a_alg = a_alg; ctx->c_ctx.c_alg = c_alg; c_ctx->c_mode = c_mode; @@ -1952,7 +1963,6 @@ static int sec_request_init(struct sec_ctx *ctx, struct sec_req *req) } while (req->req_id < 0 && ++i < ctx->sec->ctx_q_num); req->qp_ctx = qp_ctx; - req->backlog = &qp_ctx->backlog; return 0; } @@ -2055,6 +2065,9 @@ static int sec_skcipher_ctx_init(struct crypto_skcipher *tfm) if (ret) return ret; + if (!ctx->qps) + return 0; + if (ctx->sec->qm.ver < QM_HW_V3) { ctx->type_supported = SEC_BD_TYPE2; ctx->req_op = &sec_skcipher_req_ops; @@ -2063,7 +2076,7 @@ static int sec_skcipher_ctx_init(struct crypto_skcipher *tfm) ctx->req_op = &sec_skcipher_req_ops_v3; } - return ret; + return 0; } static void sec_skcipher_ctx_exit(struct crypto_skcipher *tfm) @@ -2131,7 +2144,7 @@ static int sec_aead_ctx_init(struct crypto_aead *tfm, const char *hash_name) int ret; ret = sec_aead_init(tfm); - if (ret) { + if (ret && ret != -ENODEV) { pr_err("hisi_sec2: aead init error!\n"); return ret; } @@ -2173,7 +2186,7 @@ static int sec_aead_xcm_ctx_init(struct crypto_aead *tfm) int ret; ret = sec_aead_init(tfm); - if (ret) { + if (ret && ret != -ENODEV) { dev_err(ctx->dev, "hisi_sec2: aead xcm init error!\n"); return ret; } @@ -2318,6 +2331,9 @@ static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) bool need_fallback = false; int ret; + if (!ctx->qps) + goto soft_crypto; + if (!sk_req->cryptlen) { if (ctx->c_ctx.c_mode == SEC_CMODE_XTS) return -EINVAL; @@ -2335,9 +2351,12 @@ static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) return -EINVAL; if (unlikely(ctx->c_ctx.fallback || need_fallback)) - return sec_skcipher_soft_crypto(ctx, sk_req, encrypt); + goto soft_crypto; return ctx->req_op->process(ctx, req); + +soft_crypto: + return sec_skcipher_soft_crypto(ctx, sk_req, encrypt); } static int sec_skcipher_encrypt(struct skcipher_request *sk_req) @@ -2545,6 +2564,9 @@ static int sec_aead_crypto(struct aead_request *a_req, bool encrypt) bool need_fallback = false; int ret; + if (!ctx->qps) + goto soft_crypto; + req->flag = a_req->base.flags; req->aead_req.aead_req = a_req; req->c_req.encrypt = encrypt; @@ -2555,11 +2577,14 @@ static int sec_aead_crypto(struct aead_request *a_req, bool encrypt) ret = sec_aead_param_check(ctx, req, &need_fallback); if (unlikely(ret)) { if (need_fallback) - return sec_aead_soft_crypto(ctx, a_req, encrypt); + goto soft_crypto; return -EINVAL; } return ctx->req_op->process(ctx, req); + +soft_crypto: + return sec_aead_soft_crypto(ctx, a_req, encrypt); } static int sec_aead_encrypt(struct aead_request *a_req) diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 5eb2d68207426e..7dd125f5f511fa 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -417,18 +417,29 @@ struct hisi_qp **sec_create_qps(void) int node = cpu_to_node(raw_smp_processor_id()); u32 ctx_num = ctx_q_num; struct hisi_qp **qps; + u8 *type; int ret; qps = kcalloc(ctx_num, sizeof(struct hisi_qp *), GFP_KERNEL); if (!qps) return NULL; - ret = hisi_qm_alloc_qps_node(&sec_devices, ctx_num, 0, node, qps); - if (!ret) - return qps; + /* The type of SEC is all 0, so just allocated by kcalloc */ + type = kcalloc(ctx_num, sizeof(u8), GFP_KERNEL); + if (!type) { + kfree(qps); + return NULL; + } - kfree(qps); - return NULL; + ret = hisi_qm_alloc_qps_node(&sec_devices, ctx_num, type, node, qps); + if (ret) { + kfree(type); + kfree(qps); + return NULL; + } + + kfree(type); + return qps; } u64 sec_get_alg_bitmap(struct hisi_qm *qm, u32 high, u32 low) diff --git a/drivers/crypto/hisilicon/sgl.c b/drivers/crypto/hisilicon/sgl.c index 24c7b6ab285ba6..d41b34405c21c4 100644 --- a/drivers/crypto/hisilicon/sgl.c +++ b/drivers/crypto/hisilicon/sgl.c @@ -260,7 +260,7 @@ hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev, struct scatterlist *sgl, return curr_hw_sgl; err_unmap: - dma_unmap_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL); + dma_unmap_sg(dev, sgl, sg_n, dir); return ERR_PTR(ret); } diff --git a/drivers/crypto/hisilicon/trng/trng.c b/drivers/crypto/hisilicon/trng/trng.c index ac74df4a947126..5ca0b90859a81e 100644 --- a/drivers/crypto/hisilicon/trng/trng.c +++ b/drivers/crypto/hisilicon/trng/trng.c @@ -40,6 +40,7 @@ #define SEED_SHIFT_24 24 #define SEED_SHIFT_16 16 #define SEED_SHIFT_8 8 +#define SW_MAX_RANDOM_BYTES 65520 struct hisi_trng_list { struct mutex lock; @@ -53,8 +54,10 @@ struct hisi_trng { struct list_head list; struct hwrng rng; u32 ver; - bool is_used; - struct mutex mutex; + u32 ctx_num; + /* The bytes of the random number generated since the last seeding. */ + u32 random_bytes; + struct mutex lock; }; struct hisi_trng_ctx { @@ -63,10 +66,14 @@ struct hisi_trng_ctx { static atomic_t trng_active_devs; static struct hisi_trng_list trng_devices; +static int hisi_trng_read(struct hwrng *rng, void *buf, size_t max, bool wait); -static void hisi_trng_set_seed(struct hisi_trng *trng, const u8 *seed) +static int hisi_trng_set_seed(struct hisi_trng *trng, const u8 *seed) { u32 val, seed_reg, i; + int ret; + + writel(0x0, trng->base + SW_DRBG_BLOCKS); for (i = 0; i < SW_DRBG_SEED_SIZE; i += SW_DRBG_SEED_SIZE / SW_DRBG_SEED_REGS_NUM) { @@ -78,6 +85,20 @@ static void hisi_trng_set_seed(struct hisi_trng *trng, const u8 *seed) seed_reg = (i >> SW_DRBG_NUM_SHIFT) % SW_DRBG_SEED_REGS_NUM; writel(val, trng->base + SW_DRBG_SEED(seed_reg)); } + + writel(SW_DRBG_BLOCKS_NUM | (0x1 << SW_DRBG_ENABLE_SHIFT), + trng->base + SW_DRBG_BLOCKS); + writel(0x1, trng->base + SW_DRBG_INIT); + ret = readl_relaxed_poll_timeout(trng->base + SW_DRBG_STATUS, + val, val & BIT(0), SLEEP_US, TIMEOUT_US); + if (ret) { + pr_err("failed to init trng(%d)\n", ret); + return -EIO; + } + + trng->random_bytes = 0; + + return 0; } static int hisi_trng_seed(struct crypto_rng *tfm, const u8 *seed, @@ -85,8 +106,7 @@ static int hisi_trng_seed(struct crypto_rng *tfm, const u8 *seed, { struct hisi_trng_ctx *ctx = crypto_rng_ctx(tfm); struct hisi_trng *trng = ctx->trng; - u32 val = 0; - int ret = 0; + int ret; if (slen < SW_DRBG_SEED_SIZE) { pr_err("slen(%u) is not matched with trng(%d)\n", slen, @@ -94,43 +114,45 @@ static int hisi_trng_seed(struct crypto_rng *tfm, const u8 *seed, return -EINVAL; } - writel(0x0, trng->base + SW_DRBG_BLOCKS); - hisi_trng_set_seed(trng, seed); + mutex_lock(&trng->lock); + ret = hisi_trng_set_seed(trng, seed); + mutex_unlock(&trng->lock); - writel(SW_DRBG_BLOCKS_NUM | (0x1 << SW_DRBG_ENABLE_SHIFT), - trng->base + SW_DRBG_BLOCKS); - writel(0x1, trng->base + SW_DRBG_INIT); + return ret; +} - ret = readl_relaxed_poll_timeout(trng->base + SW_DRBG_STATUS, - val, val & BIT(0), SLEEP_US, TIMEOUT_US); - if (ret) - pr_err("fail to init trng(%d)\n", ret); +static int hisi_trng_reseed(struct hisi_trng *trng) +{ + u8 seed[SW_DRBG_SEED_SIZE]; + int size; - return ret; + if (!trng->random_bytes) + return 0; + + size = hisi_trng_read(&trng->rng, seed, SW_DRBG_SEED_SIZE, false); + if (size != SW_DRBG_SEED_SIZE) + return -EIO; + + return hisi_trng_set_seed(trng, seed); } -static int hisi_trng_generate(struct crypto_rng *tfm, const u8 *src, - unsigned int slen, u8 *dstn, unsigned int dlen) +static int hisi_trng_get_bytes(struct hisi_trng *trng, u8 *dstn, unsigned int dlen) { - struct hisi_trng_ctx *ctx = crypto_rng_ctx(tfm); - struct hisi_trng *trng = ctx->trng; u32 data[SW_DRBG_DATA_NUM]; u32 currsize = 0; u32 val = 0; int ret; u32 i; - if (dlen > SW_DRBG_BLOCKS_NUM * SW_DRBG_BYTES || dlen == 0) { - pr_err("dlen(%u) exceeds limit(%d)!\n", dlen, - SW_DRBG_BLOCKS_NUM * SW_DRBG_BYTES); - return -EINVAL; - } + ret = hisi_trng_reseed(trng); + if (ret) + return ret; do { ret = readl_relaxed_poll_timeout(trng->base + SW_DRBG_STATUS, - val, val & BIT(1), SLEEP_US, TIMEOUT_US); + val, val & BIT(1), SLEEP_US, TIMEOUT_US); if (ret) { - pr_err("fail to generate random number(%d)!\n", ret); + pr_err("failed to generate random number(%d)!\n", ret); break; } @@ -145,30 +167,57 @@ static int hisi_trng_generate(struct crypto_rng *tfm, const u8 *src, currsize = dlen; } + trng->random_bytes += SW_DRBG_BYTES; writel(0x1, trng->base + SW_DRBG_GEN); } while (currsize < dlen); return ret; } +static int hisi_trng_generate(struct crypto_rng *tfm, const u8 *src, + unsigned int slen, u8 *dstn, unsigned int dlen) +{ + struct hisi_trng_ctx *ctx = crypto_rng_ctx(tfm); + struct hisi_trng *trng = ctx->trng; + unsigned int currsize = 0; + unsigned int block_size; + int ret; + + if (!dstn || !dlen) { + pr_err("output is error, dlen %u!\n", dlen); + return -EINVAL; + } + + do { + block_size = min_t(unsigned int, dlen - currsize, SW_MAX_RANDOM_BYTES); + mutex_lock(&trng->lock); + ret = hisi_trng_get_bytes(trng, dstn + currsize, block_size); + mutex_unlock(&trng->lock); + if (ret) + return ret; + currsize += block_size; + } while (currsize < dlen); + + return 0; +} + static int hisi_trng_init(struct crypto_tfm *tfm) { struct hisi_trng_ctx *ctx = crypto_tfm_ctx(tfm); struct hisi_trng *trng; - int ret = -EBUSY; + u32 ctx_num = ~0; mutex_lock(&trng_devices.lock); list_for_each_entry(trng, &trng_devices.list, list) { - if (!trng->is_used) { - trng->is_used = true; + if (trng->ctx_num < ctx_num) { + ctx_num = trng->ctx_num; ctx->trng = trng; - ret = 0; - break; } } + ctx->trng->ctx_num++; mutex_unlock(&trng_devices.lock); - return ret; + return 0; } static void hisi_trng_exit(struct crypto_tfm *tfm) @@ -176,7 +225,7 @@ static void hisi_trng_exit(struct crypto_tfm *tfm) struct hisi_trng_ctx *ctx = crypto_tfm_ctx(tfm); mutex_lock(&trng_devices.lock); - ctx->trng->is_used = false; + ctx->trng->ctx_num--; mutex_unlock(&trng_devices.lock); } @@ -238,7 +287,7 @@ static int hisi_trng_del_from_list(struct hisi_trng *trng) int ret = -EBUSY; mutex_lock(&trng_devices.lock); - if (!trng->is_used) { + if (!trng->ctx_num) { list_del(&trng->list); ret = 0; } @@ -262,7 +311,9 @@ static int hisi_trng_probe(struct platform_device *pdev) if (IS_ERR(trng->base)) return PTR_ERR(trng->base); - trng->is_used = false; + trng->ctx_num = 0; + trng->random_bytes = SW_MAX_RANDOM_BYTES; + mutex_init(&trng->lock); trng->ver = readl(trng->base + HISI_TRNG_VERSION); if (!trng_devices.is_init) { INIT_LIST_HEAD(&trng_devices.list); diff --git a/drivers/crypto/hisilicon/zip/zip.h b/drivers/crypto/hisilicon/zip/zip.h index 9fb2a9c01132b4..b83f228281ab1e 100644 --- a/drivers/crypto/hisilicon/zip/zip.h +++ b/drivers/crypto/hisilicon/zip/zip.h @@ -99,7 +99,7 @@ enum zip_cap_table_type { ZIP_CORE5_BITMAP, }; -int zip_create_qps(struct hisi_qp **qps, int qp_num, int node); +int zip_create_qps(struct hisi_qp **qps, int qp_num, int node, u8 *alg_type); int hisi_zip_register_to_crypto(struct hisi_qm *qm); void hisi_zip_unregister_from_crypto(struct hisi_qm *qm); bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg); diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.c b/drivers/crypto/hisilicon/zip/zip_crypto.c index b97513981a3b79..e140d4f8afe0e8 100644 --- a/drivers/crypto/hisilicon/zip/zip_crypto.c +++ b/drivers/crypto/hisilicon/zip/zip_crypto.c @@ -39,6 +39,7 @@ enum { HZIP_CTX_Q_NUM }; +#define GET_REQ_FROM_SQE(sqe) ((u64)(sqe)->dw26 | (u64)(sqe)->dw27 << 32) #define COMP_NAME_TO_TYPE(alg_name) \ (!strcmp((alg_name), "deflate") ? HZIP_ALG_TYPE_DEFLATE : 0) @@ -48,6 +49,7 @@ struct hisi_zip_req { struct hisi_acc_hw_sgl *hw_dst; dma_addr_t dma_src; dma_addr_t dma_dst; + struct hisi_zip_qp_ctx *qp_ctx; u16 req_id; }; @@ -64,6 +66,7 @@ struct hisi_zip_qp_ctx { struct hisi_acc_sgl_pool *sgl_pool; struct hisi_zip *zip_dev; struct hisi_zip_ctx *ctx; + u8 req_type; }; struct hisi_zip_sqe_ops { @@ -74,7 +77,6 @@ struct hisi_zip_sqe_ops { void (*fill_req_type)(struct hisi_zip_sqe *sqe, u8 req_type); void (*fill_tag)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req); void (*fill_sqe_type)(struct hisi_zip_sqe *sqe, u8 sqe_type); - u32 (*get_tag)(struct hisi_zip_sqe *sqe); u32 (*get_status)(struct hisi_zip_sqe *sqe); u32 (*get_dstlen)(struct hisi_zip_sqe *sqe); }; @@ -82,6 +84,7 @@ struct hisi_zip_sqe_ops { struct hisi_zip_ctx { struct hisi_zip_qp_ctx qp_ctx[HZIP_CTX_Q_NUM]; const struct hisi_zip_sqe_ops *ops; + bool fallback; }; static int sgl_sge_nr_set(const char *val, const struct kernel_param *kp) @@ -108,6 +111,24 @@ static u16 sgl_sge_nr = HZIP_SGL_SGE_NR; module_param_cb(sgl_sge_nr, &sgl_sge_nr_ops, &sgl_sge_nr, 0444); MODULE_PARM_DESC(sgl_sge_nr, "Number of sge in sgl(1-255)"); +static int hisi_zip_fallback_do_work(struct acomp_req *acomp_req, bool is_decompress) +{ + ACOMP_FBREQ_ON_STACK(fbreq, acomp_req); + int ret; + + if (!is_decompress) + ret = crypto_acomp_compress(fbreq); + else + ret = crypto_acomp_decompress(fbreq); + if (ret) { + pr_err("failed to do fallback work, ret=%d\n", ret); + return ret; + } + + acomp_req->dlen = fbreq->dlen; + return ret; +} + static struct hisi_zip_req *hisi_zip_create_req(struct hisi_zip_qp_ctx *qp_ctx, struct acomp_req *req) { @@ -131,6 +152,7 @@ static struct hisi_zip_req *hisi_zip_create_req(struct hisi_zip_qp_ctx *qp_ctx, req_cache = q + req_id; req_cache->req_id = req_id; req_cache->req = req; + req_cache->qp_ctx = qp_ctx; return req_cache; } @@ -181,7 +203,8 @@ static void hisi_zip_fill_req_type(struct hisi_zip_sqe *sqe, u8 req_type) static void hisi_zip_fill_tag(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req) { - sqe->dw26 = req->req_id; + sqe->dw26 = lower_32_bits((u64)req); + sqe->dw27 = upper_32_bits((u64)req); } static void hisi_zip_fill_sqe_type(struct hisi_zip_sqe *sqe, u8 sqe_type) @@ -213,7 +236,6 @@ static int hisi_zip_do_work(struct hisi_zip_qp_ctx *qp_ctx, { struct hisi_acc_sgl_pool *pool = qp_ctx->sgl_pool; struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx; - struct hisi_zip_req_q *req_q = &qp_ctx->req_q; struct acomp_req *a_req = req->req; struct hisi_qp *qp = qp_ctx->qp; struct device *dev = &qp->qm->pdev->dev; @@ -237,18 +259,16 @@ static int hisi_zip_do_work(struct hisi_zip_qp_ctx *qp_ctx, &req->dma_dst, DMA_FROM_DEVICE); if (IS_ERR(req->hw_dst)) { ret = PTR_ERR(req->hw_dst); - dev_err(dev, "failed to map the dst buffer to hw slg (%d)!\n", + dev_err(dev, "failed to map the dst buffer to hw sgl (%d)!\n", ret); goto err_unmap_input; } - hisi_zip_fill_sqe(qp_ctx->ctx, &zip_sqe, qp->req_type, req); + hisi_zip_fill_sqe(qp_ctx->ctx, &zip_sqe, qp_ctx->req_type, req); /* send command to start a task */ atomic64_inc(&dfx->send_cnt); - spin_lock_bh(&req_q->req_lock); ret = hisi_qp_send(qp, &zip_sqe); - spin_unlock_bh(&req_q->req_lock); if (unlikely(ret < 0)) { atomic64_inc(&dfx->send_busy_cnt); ret = -EAGAIN; @@ -265,11 +285,6 @@ static int hisi_zip_do_work(struct hisi_zip_qp_ctx *qp_ctx, return ret; } -static u32 hisi_zip_get_tag(struct hisi_zip_sqe *sqe) -{ - return sqe->dw26; -} - static u32 hisi_zip_get_status(struct hisi_zip_sqe *sqe) { return sqe->dw3 & HZIP_BD_STATUS_M; @@ -282,14 +297,12 @@ static u32 hisi_zip_get_dstlen(struct hisi_zip_sqe *sqe) static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data) { - struct hisi_zip_qp_ctx *qp_ctx = qp->qp_ctx; + struct hisi_zip_sqe *sqe = data; + struct hisi_zip_req *req = (struct hisi_zip_req *)GET_REQ_FROM_SQE(sqe); + struct hisi_zip_qp_ctx *qp_ctx = req->qp_ctx; const struct hisi_zip_sqe_ops *ops = qp_ctx->ctx->ops; struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx; - struct hisi_zip_req_q *req_q = &qp_ctx->req_q; struct device *dev = &qp->qm->pdev->dev; - struct hisi_zip_sqe *sqe = data; - u32 tag = ops->get_tag(sqe); - struct hisi_zip_req *req = req_q->q + tag; struct acomp_req *acomp_req = req->req; int err = 0; u32 status; @@ -319,10 +332,15 @@ static int hisi_zip_acompress(struct acomp_req *acomp_req) { struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_COMP]; - struct device *dev = &qp_ctx->qp->qm->pdev->dev; struct hisi_zip_req *req; + struct device *dev; int ret; + if (ctx->fallback) + return hisi_zip_fallback_do_work(acomp_req, 0); + + dev = &qp_ctx->qp->qm->pdev->dev; + req = hisi_zip_create_req(qp_ctx, acomp_req); if (IS_ERR(req)) return PTR_ERR(req); @@ -340,10 +358,15 @@ static int hisi_zip_adecompress(struct acomp_req *acomp_req) { struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_DECOMP]; - struct device *dev = &qp_ctx->qp->qm->pdev->dev; struct hisi_zip_req *req; + struct device *dev; int ret; + if (ctx->fallback) + return hisi_zip_fallback_do_work(acomp_req, 1); + + dev = &qp_ctx->qp->qm->pdev->dev; + req = hisi_zip_create_req(qp_ctx, acomp_req); if (IS_ERR(req)) return PTR_ERR(req); @@ -358,33 +381,6 @@ static int hisi_zip_adecompress(struct acomp_req *acomp_req) return ret; } -static int hisi_zip_start_qp(struct hisi_qp *qp, struct hisi_zip_qp_ctx *qp_ctx, - int alg_type, int req_type) -{ - struct device *dev = &qp->qm->pdev->dev; - int ret; - - qp->req_type = req_type; - qp->alg_type = alg_type; - qp->qp_ctx = qp_ctx; - - ret = hisi_qm_start_qp(qp, 0); - if (ret < 0) { - dev_err(dev, "failed to start qp (%d)!\n", ret); - return ret; - } - - qp_ctx->qp = qp; - - return 0; -} - -static void hisi_zip_release_qp(struct hisi_zip_qp_ctx *qp_ctx) -{ - hisi_qm_stop_qp(qp_ctx->qp); - hisi_qm_free_qps(&qp_ctx->qp, 1); -} - static const struct hisi_zip_sqe_ops hisi_zip_ops = { .sqe_type = 0x3, .fill_addr = hisi_zip_fill_addr, @@ -393,7 +389,6 @@ static const struct hisi_zip_sqe_ops hisi_zip_ops = { .fill_req_type = hisi_zip_fill_req_type, .fill_tag = hisi_zip_fill_tag, .fill_sqe_type = hisi_zip_fill_sqe_type, - .get_tag = hisi_zip_get_tag, .get_status = hisi_zip_get_status, .get_dstlen = hisi_zip_get_dstlen, }; @@ -402,10 +397,15 @@ static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int { struct hisi_qp *qps[HZIP_CTX_Q_NUM] = { NULL }; struct hisi_zip_qp_ctx *qp_ctx; + u8 alg_type[HZIP_CTX_Q_NUM]; struct hisi_zip *hisi_zip; - int ret, i, j; + int ret, i; - ret = zip_create_qps(qps, HZIP_CTX_Q_NUM, node); + /* alg_type = 0 for compress, 1 for decompress in hw sqe */ + for (i = 0; i < HZIP_CTX_Q_NUM; i++) + alg_type[i] = i; + + ret = zip_create_qps(qps, HZIP_CTX_Q_NUM, node, alg_type); if (ret) { pr_err("failed to create zip qps (%d)!\n", ret); return -ENODEV; @@ -414,19 +414,11 @@ static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int hisi_zip = container_of(qps[0]->qm, struct hisi_zip, qm); for (i = 0; i < HZIP_CTX_Q_NUM; i++) { - /* alg_type = 0 for compress, 1 for decompress in hw sqe */ qp_ctx = &hisi_zip_ctx->qp_ctx[i]; qp_ctx->ctx = hisi_zip_ctx; - ret = hisi_zip_start_qp(qps[i], qp_ctx, i, req_type); - if (ret) { - for (j = i - 1; j >= 0; j--) - hisi_qm_stop_qp(hisi_zip_ctx->qp_ctx[j].qp); - - hisi_qm_free_qps(qps, HZIP_CTX_Q_NUM); - return ret; - } - qp_ctx->zip_dev = hisi_zip; + qp_ctx->req_type = req_type; + qp_ctx->qp = qps[i]; } hisi_zip_ctx->ops = &hisi_zip_ops; @@ -436,10 +428,13 @@ static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int static void hisi_zip_ctx_exit(struct hisi_zip_ctx *hisi_zip_ctx) { + struct hisi_qp *qps[HZIP_CTX_Q_NUM] = { NULL }; int i; for (i = 0; i < HZIP_CTX_Q_NUM; i++) - hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[i]); + qps[i] = hisi_zip_ctx->qp_ctx[i].qp; + + hisi_qm_free_qps(qps, HZIP_CTX_Q_NUM); } static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx) @@ -549,7 +544,7 @@ static int hisi_zip_acomp_init(struct crypto_acomp *tfm) ret = hisi_zip_ctx_init(ctx, COMP_NAME_TO_TYPE(alg_name), tfm->base.node); if (ret) { pr_err("failed to init ctx (%d)!\n", ret); - return ret; + goto switch_to_soft; } dev = &ctx->qp_ctx[0].qp->qm->pdev->dev; @@ -574,17 +569,20 @@ static int hisi_zip_acomp_init(struct crypto_acomp *tfm) hisi_zip_release_req_q(ctx); err_ctx_exit: hisi_zip_ctx_exit(ctx); - return ret; +switch_to_soft: + ctx->fallback = true; + return 0; } static void hisi_zip_acomp_exit(struct crypto_acomp *tfm) { struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base); - hisi_zip_set_acomp_cb(ctx, NULL); - hisi_zip_release_sgl_pool(ctx); - hisi_zip_release_req_q(ctx); - hisi_zip_ctx_exit(ctx); + if (!ctx->fallback) { + hisi_zip_release_sgl_pool(ctx); + hisi_zip_release_req_q(ctx); + hisi_zip_ctx_exit(ctx); + } } static struct acomp_alg hisi_zip_acomp_deflate = { @@ -595,7 +593,8 @@ static struct acomp_alg hisi_zip_acomp_deflate = { .base = { .cra_name = "deflate", .cra_driver_name = "hisi-deflate-acomp", - .cra_flags = CRYPTO_ALG_ASYNC, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, .cra_module = THIS_MODULE, .cra_priority = HZIP_ALG_PRIORITY, .cra_ctxsize = sizeof(struct hisi_zip_ctx), diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 4fcbe6bada0663..85b26ef1754855 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -446,12 +446,12 @@ static const struct pci_device_id hisi_zip_dev_ids[] = { }; MODULE_DEVICE_TABLE(pci, hisi_zip_dev_ids); -int zip_create_qps(struct hisi_qp **qps, int qp_num, int node) +int zip_create_qps(struct hisi_qp **qps, int qp_num, int node, u8 *alg_type) { if (node == NUMA_NO_NODE) node = cpu_to_node(raw_smp_processor_id()); - return hisi_qm_alloc_qps_node(&zip_devices, qp_num, 0, node, qps); + return hisi_qm_alloc_qps_node(&zip_devices, qp_num, alg_type, node, qps); } bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg) diff --git a/drivers/crypto/inside-secure/eip93/eip93-main.c b/drivers/crypto/inside-secure/eip93/eip93-main.c index 0b38a567da0e01..b7fd9795062d4c 100644 --- a/drivers/crypto/inside-secure/eip93/eip93-main.c +++ b/drivers/crypto/inside-secure/eip93/eip93-main.c @@ -77,11 +77,44 @@ inline void eip93_irq_clear(struct eip93_device *eip93, u32 mask) __raw_writel(mask, eip93->base + EIP93_REG_INT_CLR); } -static void eip93_unregister_algs(unsigned int i) +static int eip93_algo_is_supported(u32 alg_flags, u32 supported_algo_flags) +{ + if ((IS_DES(alg_flags) || IS_3DES(alg_flags)) && + !(supported_algo_flags & EIP93_PE_OPTION_TDES)) + return 0; + + if (IS_AES(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_AES)) + return 0; + + if (IS_HASH_MD5(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_MD5)) + return 0; + + if (IS_HASH_SHA1(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_SHA_1)) + return 0; + + if (IS_HASH_SHA224(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_SHA_224)) + return 0; + + if (IS_HASH_SHA256(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_SHA_256)) + return 0; + + return 1; +} + +static void eip93_unregister_algs(u32 supported_algo_flags, unsigned int i) { unsigned int j; for (j = 0; j < i; j++) { + if (!eip93_algo_is_supported(eip93_algs[j]->flags, + supported_algo_flags)) + continue; + switch (eip93_algs[j]->type) { case EIP93_ALG_TYPE_SKCIPHER: crypto_unregister_skcipher(&eip93_algs[j]->alg.skcipher); @@ -90,7 +123,7 @@ static void eip93_unregister_algs(unsigned int i) crypto_unregister_aead(&eip93_algs[j]->alg.aead); break; case EIP93_ALG_TYPE_HASH: - crypto_unregister_ahash(&eip93_algs[i]->alg.ahash); + crypto_unregister_ahash(&eip93_algs[j]->alg.ahash); break; } } @@ -106,49 +139,27 @@ static int eip93_register_algs(struct eip93_device *eip93, u32 supported_algo_fl eip93_algs[i]->eip93 = eip93; - if ((IS_DES(alg_flags) || IS_3DES(alg_flags)) && - !(supported_algo_flags & EIP93_PE_OPTION_TDES)) + if (!eip93_algo_is_supported(alg_flags, supported_algo_flags)) continue; - if (IS_AES(alg_flags)) { - if (!(supported_algo_flags & EIP93_PE_OPTION_AES)) - continue; + if (IS_AES(alg_flags) && !IS_HMAC(alg_flags)) { + if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY128) + eip93_algs[i]->alg.skcipher.max_keysize = + AES_KEYSIZE_128; - if (!IS_HMAC(alg_flags)) { - if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY128) - eip93_algs[i]->alg.skcipher.max_keysize = - AES_KEYSIZE_128; + if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY192) + eip93_algs[i]->alg.skcipher.max_keysize = + AES_KEYSIZE_192; - if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY192) - eip93_algs[i]->alg.skcipher.max_keysize = - AES_KEYSIZE_192; + if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY256) + eip93_algs[i]->alg.skcipher.max_keysize = + AES_KEYSIZE_256; - if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY256) - eip93_algs[i]->alg.skcipher.max_keysize = - AES_KEYSIZE_256; - - if (IS_RFC3686(alg_flags)) - eip93_algs[i]->alg.skcipher.max_keysize += - CTR_RFC3686_NONCE_SIZE; - } + if (IS_RFC3686(alg_flags)) + eip93_algs[i]->alg.skcipher.max_keysize += + CTR_RFC3686_NONCE_SIZE; } - if (IS_HASH_MD5(alg_flags) && - !(supported_algo_flags & EIP93_PE_OPTION_MD5)) - continue; - - if (IS_HASH_SHA1(alg_flags) && - !(supported_algo_flags & EIP93_PE_OPTION_SHA_1)) - continue; - - if (IS_HASH_SHA224(alg_flags) && - !(supported_algo_flags & EIP93_PE_OPTION_SHA_224)) - continue; - - if (IS_HASH_SHA256(alg_flags) && - !(supported_algo_flags & EIP93_PE_OPTION_SHA_256)) - continue; - switch (eip93_algs[i]->type) { case EIP93_ALG_TYPE_SKCIPHER: ret = crypto_register_skcipher(&eip93_algs[i]->alg.skcipher); @@ -167,7 +178,7 @@ static int eip93_register_algs(struct eip93_device *eip93, u32 supported_algo_fl return 0; fail: - eip93_unregister_algs(i); + eip93_unregister_algs(supported_algo_flags, i); return ret; } @@ -469,8 +480,11 @@ static int eip93_crypto_probe(struct platform_device *pdev) static void eip93_crypto_remove(struct platform_device *pdev) { struct eip93_device *eip93 = platform_get_drvdata(pdev); + u32 algo_flags; + + algo_flags = readl(eip93->base + EIP93_REG_PE_OPTION_1); - eip93_unregister_algs(ARRAY_SIZE(eip93_algs)); + eip93_unregister_algs(algo_flags, ARRAY_SIZE(eip93_algs)); eip93_cleanup(eip93); } diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index d0058757b0000d..da9b2bc515194f 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -221,15 +221,13 @@ static struct iaa_compression_mode *iaa_compression_modes[IAA_COMP_MODES_MAX]; static int find_empty_iaa_compression_mode(void) { - int i = -EINVAL; + int i; - for (i = 0; i < IAA_COMP_MODES_MAX; i++) { - if (iaa_compression_modes[i]) - continue; - break; - } + for (i = 0; i < IAA_COMP_MODES_MAX; i++) + if (!iaa_compression_modes[i]) + return i; - return i; + return -EINVAL; } static struct iaa_compression_mode *find_iaa_compression_mode(const char *name, int *idx) diff --git a/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.c b/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.c index b9b5e744a3f163..af8dbc7517cf82 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.c +++ b/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.c @@ -148,6 +148,16 @@ static struct pfvf_message handle_blkmsg_req(struct adf_accel_vf_info *vf_info, blk_byte = FIELD_GET(ADF_VF2PF_SMALL_BLOCK_BYTE_MASK, req.data); byte_max = ADF_VF2PF_SMALL_BLOCK_BYTE_MAX; break; + default: + dev_err(&GET_DEV(vf_info->accel_dev), + "Invalid BlockMsg type 0x%.4x received from VF%u\n", + req.type, vf_info->vf_nr); + resp.type = ADF_PF2VF_MSGTYPE_BLKMSG_RESP; + resp.data = FIELD_PREP(ADF_PF2VF_BLKMSG_RESP_TYPE_MASK, + ADF_PF2VF_BLKMSG_RESP_TYPE_ERROR) | + FIELD_PREP(ADF_PF2VF_BLKMSG_RESP_DATA_MASK, + ADF_PF2VF_UNSPECIFIED_ERROR); + return resp; } /* Is this a request for CRC or data? */ diff --git a/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c b/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c index 85c682e248fb91..e09b9edfce423d 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c +++ b/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c @@ -255,8 +255,8 @@ static int qat_dh_compute_value(struct kpp_request *req) qat_req->areq.dh = req; msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE; msg->pke_hdr.comn_req_flags = - ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_PTR_TYPE_FLAT, - QAT_COMN_CD_FLD_TYPE_64BIT_ADR); + ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR, + QAT_COMN_PTR_TYPE_FLAT); /* * If no source is provided use g as base @@ -731,8 +731,8 @@ static int qat_rsa_enc(struct akcipher_request *req) qat_req->areq.rsa = req; msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE; msg->pke_hdr.comn_req_flags = - ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_PTR_TYPE_FLAT, - QAT_COMN_CD_FLD_TYPE_64BIT_ADR); + ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR, + QAT_COMN_PTR_TYPE_FLAT); qat_req->in.rsa.enc.e = ctx->dma_e; qat_req->in.rsa.enc.n = ctx->dma_n; @@ -867,8 +867,8 @@ static int qat_rsa_dec(struct akcipher_request *req) qat_req->areq.rsa = req; msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE; msg->pke_hdr.comn_req_flags = - ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_PTR_TYPE_FLAT, - QAT_COMN_CD_FLD_TYPE_64BIT_ADR); + ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR, + QAT_COMN_PTR_TYPE_FLAT); if (ctx->crt_mode) { qat_req->in.rsa.dec_crt.p = ctx->dma_p; diff --git a/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c index 9f5601c0280bf1..417a48f4135053 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c +++ b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c @@ -1326,7 +1326,7 @@ static ssize_t ucode_load_store(struct device *dev, int del_grp_idx = -1; int ucode_idx = 0; - if (strlen(buf) > OTX_CPT_UCODE_NAME_LENGTH) + if (count >= OTX_CPT_UCODE_NAME_LENGTH) return -EINVAL; eng_grps = container_of(attr, struct otx_cpt_eng_grps, ucode_load_attr); diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_main.c b/drivers/crypto/marvell/octeontx/otx_cptvf_main.c index 88a41d1ca5f644..6c0bfb3ea1c9f2 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptvf_main.c +++ b/drivers/crypto/marvell/octeontx/otx_cptvf_main.c @@ -168,7 +168,8 @@ static void free_command_queues(struct otx_cptvf *cptvf, chunk = list_first_entry(&cqinfo->queue[i].chead, struct otx_cpt_cmd_chunk, nextchunk); - dma_free_coherent(&pdev->dev, chunk->size, + dma_free_coherent(&pdev->dev, + chunk->size + OTX_CPT_NEXT_CHUNK_PTR_SIZE, chunk->head, chunk->dma_addr); chunk->head = NULL; diff --git a/drivers/crypto/omap-crypto.c b/drivers/crypto/omap-crypto.c index a4cc6bf146ec09..0345c9383d5097 100644 --- a/drivers/crypto/omap-crypto.c +++ b/drivers/crypto/omap-crypto.c @@ -21,7 +21,7 @@ static int omap_crypto_copy_sg_lists(int total, int bs, struct scatterlist *tmp; if (!(flags & OMAP_CRYPTO_FORCE_SINGLE_ENTRY)) { - new_sg = kmalloc_array(n, sizeof(*sg), GFP_KERNEL); + new_sg = kmalloc_array(n, sizeof(*new_sg), GFP_KERNEL); if (!new_sg) return -ENOMEM; diff --git a/drivers/crypto/starfive/jh7110-aes.c b/drivers/crypto/starfive/jh7110-aes.c index 426b24889af853..01195664cc7cd9 100644 --- a/drivers/crypto/starfive/jh7110-aes.c +++ b/drivers/crypto/starfive/jh7110-aes.c @@ -669,8 +669,10 @@ static int starfive_aes_aead_do_one_req(struct crypto_engine *engine, void *areq return -ENOMEM; if (sg_copy_to_buffer(req->src, sg_nents_for_len(req->src, cryp->assoclen), - rctx->adata, cryp->assoclen) != cryp->assoclen) + rctx->adata, cryp->assoclen) != cryp->assoclen) { + kfree(rctx->adata); return -EINVAL; + } } if (cryp->total_in) @@ -681,8 +683,11 @@ static int starfive_aes_aead_do_one_req(struct crypto_engine *engine, void *areq ctx->rctx = rctx; ret = starfive_aes_hw_init(ctx); - if (ret) + if (ret) { + if (cryp->assoclen) + kfree(rctx->adata); return ret; + } if (!cryp->assoclen) goto write_text; diff --git a/drivers/crypto/virtio/virtio_crypto_core.c b/drivers/crypto/virtio/virtio_crypto_core.c index 3d241446099cc9..ccc6b5c1b24b37 100644 --- a/drivers/crypto/virtio/virtio_crypto_core.c +++ b/drivers/crypto/virtio/virtio_crypto_core.c @@ -75,15 +75,20 @@ static void virtcrypto_done_task(unsigned long data) struct data_queue *data_vq = (struct data_queue *)data; struct virtqueue *vq = data_vq->vq; struct virtio_crypto_request *vc_req; + unsigned long flags; unsigned int len; + spin_lock_irqsave(&data_vq->lock, flags); do { virtqueue_disable_cb(vq); while ((vc_req = virtqueue_get_buf(vq, &len)) != NULL) { + spin_unlock_irqrestore(&data_vq->lock, flags); if (vc_req->alg_cb) vc_req->alg_cb(vc_req, len); + spin_lock_irqsave(&data_vq->lock, flags); } } while (!virtqueue_enable_cb(vq)); + spin_unlock_irqrestore(&data_vq->lock, flags); } static void virtcrypto_dataq_callback(struct virtqueue *vq) diff --git a/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c b/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c index 1b3fb21a2a7de2..11053d1786d4d2 100644 --- a/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c +++ b/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c @@ -541,8 +541,6 @@ int virtio_crypto_skcipher_crypt_req( if (ret < 0) return ret; - virtqueue_kick(data_vq->vq); - return 0; } diff --git a/drivers/cxl/core/edac.c b/drivers/cxl/core/edac.c index 79994ca9bc9f37..81160260e26b7e 100644 --- a/drivers/cxl/core/edac.c +++ b/drivers/cxl/core/edac.c @@ -1988,6 +1988,40 @@ static int cxl_memdev_soft_ppr_init(struct cxl_memdev *cxlmd, return 0; } +static void err_rec_free(void *_cxlmd) +{ + struct cxl_memdev *cxlmd = _cxlmd; + struct cxl_mem_err_rec *array_rec = cxlmd->err_rec_array; + struct cxl_event_gen_media *rec_gen_media; + struct cxl_event_dram *rec_dram; + unsigned long index; + + cxlmd->err_rec_array = NULL; + xa_for_each(&array_rec->rec_dram, index, rec_dram) + kfree(rec_dram); + xa_destroy(&array_rec->rec_dram); + + xa_for_each(&array_rec->rec_gen_media, index, rec_gen_media) + kfree(rec_gen_media); + xa_destroy(&array_rec->rec_gen_media); + kfree(array_rec); +} + +static int devm_cxl_memdev_setup_err_rec(struct cxl_memdev *cxlmd) +{ + struct cxl_mem_err_rec *array_rec = + kzalloc(sizeof(*array_rec), GFP_KERNEL); + + if (!array_rec) + return -ENOMEM; + + xa_init(&array_rec->rec_gen_media); + xa_init(&array_rec->rec_dram); + cxlmd->err_rec_array = array_rec; + + return devm_add_action_or_reset(&cxlmd->dev, err_rec_free, cxlmd); +} + int devm_cxl_memdev_edac_register(struct cxl_memdev *cxlmd) { struct edac_dev_feature ras_features[CXL_NR_EDAC_DEV_FEATURES]; @@ -2038,15 +2072,9 @@ int devm_cxl_memdev_edac_register(struct cxl_memdev *cxlmd) } if (repair_inst) { - struct cxl_mem_err_rec *array_rec = - devm_kzalloc(&cxlmd->dev, sizeof(*array_rec), - GFP_KERNEL); - if (!array_rec) - return -ENOMEM; - - xa_init(&array_rec->rec_gen_media); - xa_init(&array_rec->rec_dram); - cxlmd->err_rec_array = array_rec; + rc = devm_cxl_memdev_setup_err_rec(cxlmd); + if (rc) + return rc; } } @@ -2088,22 +2116,4 @@ int devm_cxl_region_edac_register(struct cxl_region *cxlr) } EXPORT_SYMBOL_NS_GPL(devm_cxl_region_edac_register, "CXL"); -void devm_cxl_memdev_edac_release(struct cxl_memdev *cxlmd) -{ - struct cxl_mem_err_rec *array_rec = cxlmd->err_rec_array; - struct cxl_event_gen_media *rec_gen_media; - struct cxl_event_dram *rec_dram; - unsigned long index; - - if (!IS_ENABLED(CONFIG_CXL_EDAC_MEM_REPAIR) || !array_rec) - return; - - xa_for_each(&array_rec->rec_dram, index, rec_dram) - kfree(rec_dram); - xa_destroy(&array_rec->rec_dram); - xa_for_each(&array_rec->rec_gen_media, index, rec_gen_media) - kfree(rec_gen_media); - xa_destroy(&array_rec->rec_gen_media); -} -EXPORT_SYMBOL_NS_GPL(devm_cxl_memdev_edac_release, "CXL"); diff --git a/drivers/cxl/core/hdm.c b/drivers/cxl/core/hdm.c index eb5a3a7640c601..bc4b0c86072589 100644 --- a/drivers/cxl/core/hdm.c +++ b/drivers/cxl/core/hdm.c @@ -844,14 +844,13 @@ static int cxl_decoder_commit(struct cxl_decoder *cxld) scoped_guard(rwsem_read, &cxl_rwsem.dpa) setup_hw_decoder(cxld, hdm); - port->commit_end++; rc = cxld_await_commit(hdm, cxld->id); if (rc) { dev_dbg(&port->dev, "%s: error %d committing decoder\n", dev_name(&cxld->dev), rc); - cxld->reset(cxld); return rc; } + port->commit_end++; cxld->flags |= CXL_DECODER_F_ENABLE; return 0; @@ -966,7 +965,7 @@ static int cxl_setup_hdm_decoder_from_dvsec( rc = devm_cxl_dpa_reserve(cxled, *dpa_base, len, 0); if (rc) { dev_err(&port->dev, - "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)", + "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx: %d\n", port->id, cxld->id, *dpa_base, *dpa_base + len - 1, rc); return rc; } @@ -1117,7 +1116,7 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, rc = devm_cxl_dpa_reserve(cxled, *dpa_base + skip, dpa_size, skip); if (rc) { dev_err(&port->dev, - "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)", + "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx: %d\n", port->id, cxld->id, *dpa_base, *dpa_base + dpa_size + skip - 1, rc); return rc; diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c index e370d733e44001..4dff7f44d908e0 100644 --- a/drivers/cxl/core/memdev.c +++ b/drivers/cxl/core/memdev.c @@ -27,7 +27,6 @@ static void cxl_memdev_release(struct device *dev) struct cxl_memdev *cxlmd = to_cxl_memdev(dev); ida_free(&cxl_memdev_ida, cxlmd->id); - devm_cxl_memdev_edac_release(cxlmd); kfree(cxlmd); } diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 3310dbfae9d663..4717dcff264be6 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -822,16 +822,18 @@ DEFINE_DEBUGFS_ATTRIBUTE(cxl_einj_inject_fops, NULL, cxl_einj_inject, static void cxl_debugfs_create_dport_dir(struct cxl_dport *dport) { + struct cxl_port *parent = parent_port_of(dport->port); struct dentry *dir; if (!einj_cxl_is_initialized()) return; /* - * dport_dev needs to be a PCIe port for CXL 2.0+ ports because - * EINJ expects a dport SBDF to be specified for 2.0 error injection. + * Protocol error injection is only available for CXL 2.0+ root ports + * and CXL 1.1 downstream ports */ - if (!dport->rch && !dev_is_pci(dport->dport_dev)) + if (!dport->rch && + !(dev_is_pci(dport->dport_dev) && parent && is_cxl_root(parent))) return; dir = cxl_debugfs_create_dir(dev_name(dport->dport_dev)); diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h index 434031a0c1f74a..c12ab4fc951238 100644 --- a/drivers/cxl/cxlmem.h +++ b/drivers/cxl/cxlmem.h @@ -63,7 +63,7 @@ struct cxl_memdev { int depth; u8 scrub_cycle; int scrub_region_id; - void *err_rec_array; + struct cxl_mem_err_rec *err_rec_array; }; static inline struct cxl_memdev *to_cxl_memdev(struct device *dev) @@ -877,7 +877,6 @@ int devm_cxl_memdev_edac_register(struct cxl_memdev *cxlmd); int devm_cxl_region_edac_register(struct cxl_region *cxlr); int cxl_store_rec_gen_media(struct cxl_memdev *cxlmd, union cxl_event *evt); int cxl_store_rec_dram(struct cxl_memdev *cxlmd, union cxl_event *evt); -void devm_cxl_memdev_edac_release(struct cxl_memdev *cxlmd); #else static inline int devm_cxl_memdev_edac_register(struct cxl_memdev *cxlmd) { return 0; } @@ -889,8 +888,6 @@ static inline int cxl_store_rec_gen_media(struct cxl_memdev *cxlmd, static inline int cxl_store_rec_dram(struct cxl_memdev *cxlmd, union cxl_event *evt) { return 0; } -static inline void devm_cxl_memdev_edac_release(struct cxl_memdev *cxlmd) -{ return; } #endif #ifdef CONFIG_CXL_SUSPEND diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 8bb0a119ecd48a..3ba2aedaae79d2 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -89,9 +89,20 @@ config APPLE_ADMAC tristate "Apple ADMAC support" depends on ARCH_APPLE || COMPILE_TEST select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS help Enable support for Audio DMA Controller found on Apple Silicon SoCs. +config APPLE_SIO + tristate "Apple SIO support" + depends on ARCH_APPLE || COMPILE_TEST + depends on APPLE_RTKIT + depends on OF_ADDRESS + select DMA_ENGINE + help + Enable support for the SIO coprocessor found on Apple Silicon SoCs + where it provides DMA services. + config ARM_DMA350 tristate "Arm DMA-350 support" depends on ARM || ARM64 || COMPILE_TEST diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index a54d7688392b1a..1c11fdc02692cc 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -17,6 +17,7 @@ obj-$(CONFIG_ALTERA_MSGDMA) += altera-msgdma.o obj-$(CONFIG_AMBA_PL08X) += amba-pl08x.o obj-$(CONFIG_AMCC_PPC440SPE_ADMA) += ppc4xx/ obj-$(CONFIG_APPLE_ADMAC) += apple-admac.o +obj-$(CONFIG_APPLE_SIO) += apple-sio.o obj-$(CONFIG_ARM_DMA350) += arm-dma350.o obj-$(CONFIG_AT_HDMAC) += at_hdmac.o obj-$(CONFIG_AT_XDMAC) += at_xdmac.o diff --git a/drivers/dma/apple-sio.c b/drivers/dma/apple-sio.c new file mode 100644 index 00000000000000..511f91999ed3de --- /dev/null +++ b/drivers/dma/apple-sio.c @@ -0,0 +1,942 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Driver for SIO coprocessor on t8103 (M1) and other Apple SoCs + * + * Copyright (C) The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dmaengine.h" +#include "virt-dma.h" + +#define NCHANNELS_MAX 0x80 + +#define REG_CPU_CONTROL 0x44 +#define CPU_CONTROL_RUN BIT(4) + +#define SIOMSG_DATA GENMASK(63, 32) +#define SIOMSG_TYPE GENMASK(23, 16) +#define SIOMSG_PARAM GENMASK(31, 24) +#define SIOMSG_TAG GENMASK(13, 8) +#define SIOMSG_EP GENMASK(7, 0) + +#define EP_SIO 0x20 + +#define MSG_START 0x2 +#define MSG_SETUP 0x3 +#define MSG_CONFIGURE 0x5 +#define MSG_ISSUE 0x6 +#define MSG_TERMINATE 0x8 +#define MSG_ACK 0x65 +#define MSG_NACK 0x66 +#define MSG_STARTED 0x67 +#define MSG_REPORT 0x68 + +#define SIO_CALL_TIMEOUT_MS 100 +#define SIO_SHMEM_SIZE 0x1000 +#define SIO_NO_DESC_SLOTS 64 + +/* + * There are two kinds of 'transaction descriptors' in play here. + * + * There's the struct sio_tx, and the struct dma_async_tx_descriptor embedded + * inside, which jointly represent a transaction to the dmaengine subsystem. + * At this time we only support those transactions to be cyclic. + * + * Then there are the coprocessor descriptors, which is what the coprocessor + * knows and understands. These don't seem to have a cyclic regime, so we can't + * map the dmaengine transaction on an exact coprocessor counterpart. Instead + * we continually queue up many coprocessor descriptors to implement a cyclic + * transaction. + * + * The number below is the maximum of how far ahead (how many) coprocessor + * descriptors we should be queuing up, per channel, for a cyclic transaction. + * Basically it's a made-up number. + */ +#define SIO_MAX_NINFLIGHT 4 + +struct sio_coproc_desc { + u32 pad1; + u32 flag; + u64 unk; + u64 iova; + u64 size; + u64 pad2; + u64 pad3; +} __packed; +static_assert(sizeof(struct sio_coproc_desc) == 48); + +struct sio_shmem_chan_config { + u32 datashape; + u32 timeout; + u32 fifo; + u32 threshold; + u32 limit; +} __packed; + +struct sio_data; +struct sio_tx; + +struct sio_chan { + unsigned int no; + struct sio_data *host; + struct virt_dma_chan vc; + struct work_struct terminate_wq; + + bool configured; + struct sio_shmem_chan_config cfg; + + struct sio_tx *current_tx; +}; + +#define SIO_NTAGS 16 + +typedef void (*sio_ack_callback)(struct sio_chan *, void *, bool); + +struct sio_data { + void __iomem *base; + struct dma_device dma; + struct device *dev; + struct apple_rtkit *rtk; + void *shmem; + struct sio_coproc_desc *shmem_desc_base; + unsigned long *desc_allocated; + + struct sio_tagdata { + DECLARE_BITMAP(allocated, SIO_NTAGS); + int last_tag; + + struct completion completions[SIO_NTAGS]; + bool atomic[SIO_NTAGS]; + bool acked[SIO_NTAGS]; + + sio_ack_callback ack_callback[SIO_NTAGS]; + void *cookie[SIO_NTAGS]; + } tags; + + int nchannels; + struct sio_chan channels[]; +}; + +struct sio_tx { + struct virt_dma_desc vd; + struct completion done; + + bool terminated; + size_t period_len; + int nperiods; + int ninflight; + int next; + + struct sio_coproc_desc *siodesc[]; +}; + +static int sio_send_siomsg(struct sio_data *sio, u64 msg); +static int sio_send_siomsg_atomic(struct sio_data *sio, u64 msg, + sio_ack_callback ack_callback, + void *cookie); +static int sio_call(struct sio_data *sio, u64 msg); + +static struct sio_chan *to_sio_chan(struct dma_chan *chan) +{ + return container_of(chan, struct sio_chan, vc.chan); +} + +static struct sio_tx *to_sio_tx(struct dma_async_tx_descriptor *tx) +{ + return container_of(tx, struct sio_tx, vd.tx); +} + +static int sio_alloc_tag(struct sio_data *sio) +{ + struct sio_tagdata *tags = &sio->tags; + int tag, i; + + /* + * Because tag number 0 is special, the usable tag range + * is 1...(SIO_NTAGS - 1). So, to pick the next usable tag, + * we do modulo (SIO_NTAGS - 1) *then* plus one. + */ + +#define SIO_USABLE_TAGS (SIO_NTAGS - 1) + tag = (READ_ONCE(tags->last_tag) % SIO_USABLE_TAGS) + 1; + + for (i = 0; i < SIO_USABLE_TAGS; i++) { + if (!test_and_set_bit(tag, tags->allocated)) + break; + + tag = (tag % SIO_USABLE_TAGS) + 1; + } + + WRITE_ONCE(tags->last_tag, tag); + + if (i < SIO_USABLE_TAGS) + return tag; + else + return -EBUSY; +#undef SIO_USABLE_TAGS +} + +static void sio_free_tag(struct sio_data *sio, int tag) +{ + struct sio_tagdata *tags = &sio->tags; + + if (WARN_ON(tag >= SIO_NTAGS)) + return; + + tags->atomic[tag] = false; + tags->ack_callback[tag] = NULL; + + WARN_ON(!test_and_clear_bit(tag, tags->allocated)); +} + +static void sio_set_tag_atomic(struct sio_data *sio, int tag, + sio_ack_callback ack_callback, + void *cookie) +{ + struct sio_tagdata *tags = &sio->tags; + + tags->atomic[tag] = true; + tags->ack_callback[tag] = ack_callback; + tags->cookie[tag] = cookie; +} + +static struct sio_coproc_desc *sio_alloc_desc(struct sio_data *sio) +{ + int i; + + for (i = 0; i < SIO_NO_DESC_SLOTS; i++) + if (!test_and_set_bit(i, sio->desc_allocated)) + return sio->shmem_desc_base + i; + + return NULL; +} + +static void sio_free_desc(struct sio_data *sio, struct sio_coproc_desc *desc) +{ + clear_bit(desc - sio->shmem_desc_base, sio->desc_allocated); +} + +static int sio_coproc_desc_slot(struct sio_data *sio, struct sio_coproc_desc *desc) +{ + return (desc - sio->shmem_desc_base) * 4; +} + +static enum dma_transfer_direction sio_chan_direction(int channo) +{ + /* Channel directions are fixed based on channel number */ + return (channo & 1) ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV; +} + +static void sio_tx_free(struct virt_dma_desc *vd) +{ + struct sio_data *sio = to_sio_chan(vd->tx.chan)->host; + struct sio_tx *siotx = to_sio_tx(&vd->tx); + int i; + + for (i = 0; i < siotx->nperiods; i++) + if (siotx->siodesc[i]) + sio_free_desc(sio, siotx->siodesc[i]); + kfree(siotx); +} + +static struct dma_async_tx_descriptor *sio_prep_dma_cyclic( + struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, + size_t period_len, enum dma_transfer_direction direction, + unsigned long flags) +{ + struct sio_chan *siochan = to_sio_chan(chan); + struct sio_tx *siotx = NULL; + int i, nperiods = buf_len / period_len; + + if (direction != sio_chan_direction(siochan->no)) + return NULL; + + siotx = kzalloc(struct_size(siotx, siodesc, nperiods), GFP_NOWAIT); + if (!siotx) + return NULL; + + init_completion(&siotx->done); + siotx->period_len = period_len; + siotx->nperiods = nperiods; + + for (i = 0; i < nperiods; i++) { + struct sio_coproc_desc *d; + + siotx->siodesc[i] = d = sio_alloc_desc(siochan->host); + if (!d) { + siotx->vd.tx.chan = &siochan->vc.chan; + sio_tx_free(&siotx->vd); + return NULL; + } + + d->flag = 1; /* not sure what's up with this */ + d->iova = buf_addr + period_len * i; + d->size = period_len; + } + dma_wmb(); + + return vchan_tx_prep(&siochan->vc, &siotx->vd, flags); +} + +static enum dma_status sio_tx_status(struct dma_chan *chan, dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct sio_chan *siochan = to_sio_chan(chan); + struct virt_dma_desc *vd; + struct sio_tx *siotx; + enum dma_status ret; + unsigned long flags; + int periods_residue; + size_t residue; + + ret = dma_cookie_status(chan, cookie, txstate); + if (ret == DMA_COMPLETE || !txstate) + return ret; + + spin_lock_irqsave(&siochan->vc.lock, flags); + siotx = siochan->current_tx; + + if (siotx && siotx->vd.tx.cookie == cookie) { + ret = DMA_IN_PROGRESS; + periods_residue = siotx->next - siotx->ninflight; + while (periods_residue < 0) + periods_residue += siotx->nperiods; + residue = (siotx->nperiods - periods_residue) * siotx->period_len; + } else { + ret = DMA_IN_PROGRESS; + residue = 0; + vd = vchan_find_desc(&siochan->vc, cookie); + if (vd) { + siotx = to_sio_tx(&vd->tx); + residue = siotx->period_len * siotx->nperiods; + } + } + spin_unlock_irqrestore(&siochan->vc.lock, flags); + dma_set_residue(txstate, residue); + + return ret; +} + +static bool sio_fill_in_locked(struct sio_chan *siochan); + +static void sio_handle_issue_ack(struct sio_chan *siochan, void *cookie, bool ok) +{ + dma_cookie_t tx_cookie = (unsigned long) cookie; + unsigned long flags; + struct sio_tx *tx; + + if (!ok) { + dev_err(siochan->host->dev, "nacked issue on chan %d\n", siochan->no); + return; + } + + spin_lock_irqsave(&siochan->vc.lock, flags); + if (!siochan->current_tx || tx_cookie != siochan->current_tx->vd.tx.cookie || + siochan->current_tx->terminated) + goto out; + + tx = siochan->current_tx; + tx->next = (tx->next + 1) % tx->nperiods; + tx->ninflight++; + sio_fill_in_locked(siochan); + +out: + spin_unlock_irqrestore(&siochan->vc.lock, flags); +} + +static bool sio_fill_in_locked(struct sio_chan *siochan) +{ + struct sio_data *sio = siochan->host; + struct sio_tx *tx = siochan->current_tx; + struct sio_coproc_desc *d = tx->siodesc[tx->next]; + int ret; + + if (tx->ninflight >= SIO_MAX_NINFLIGHT || tx->terminated) + return false; + + static_assert(sizeof(dma_cookie_t) <= sizeof(void *)); + ret = sio_send_siomsg_atomic(sio, FIELD_PREP(SIOMSG_EP, siochan->no) | + FIELD_PREP(SIOMSG_TYPE, MSG_ISSUE) | + FIELD_PREP(SIOMSG_DATA, sio_coproc_desc_slot(sio, d)), + sio_handle_issue_ack, (void *) (uintptr_t) tx->vd.tx.cookie); + if (ret < 0) + dev_err_ratelimited(sio->dev, "can't issue on chan %d ninflight %d: %d\n", + siochan->no, tx->ninflight, ret); + return true; +} + +static void sio_update_current_tx_locked(struct sio_chan *siochan) +{ + struct virt_dma_desc *vd = vchan_next_desc(&siochan->vc); + + if (vd && !siochan->current_tx) { + list_del(&vd->node); + siochan->current_tx = to_sio_tx(&vd->tx); + sio_fill_in_locked(siochan); + } +} + +static void sio_issue_pending(struct dma_chan *chan) +{ + struct sio_chan *siochan = to_sio_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&siochan->vc.lock, flags); + vchan_issue_pending(&siochan->vc); + sio_update_current_tx_locked(siochan); + spin_unlock_irqrestore(&siochan->vc.lock, flags); +} + +static int sio_terminate_all(struct dma_chan *chan) +{ + struct sio_chan *siochan = to_sio_chan(chan); + unsigned long flags; + LIST_HEAD(to_free); + + spin_lock_irqsave(&siochan->vc.lock, flags); + if (siochan->current_tx && !siochan->current_tx->terminated) { + dma_cookie_complete(&siochan->current_tx->vd.tx); + siochan->current_tx->terminated = true; + schedule_work(&siochan->terminate_wq); + } + vchan_get_all_descriptors(&siochan->vc, &to_free); + spin_unlock_irqrestore(&siochan->vc.lock, flags); + + vchan_dma_desc_free_list(&siochan->vc, &to_free); + + return 0; +} + +static void sio_terminate_work(struct work_struct *wq) +{ + struct sio_chan *siochan = container_of(wq, struct sio_chan, terminate_wq); + struct sio_tx *tx; + unsigned long flags; + int ret; + + spin_lock_irqsave(&siochan->vc.lock, flags); + tx = siochan->current_tx; + spin_unlock_irqrestore(&siochan->vc.lock, flags); + + if (WARN_ON(!tx)) + return; + + ret = sio_call(siochan->host, FIELD_PREP(SIOMSG_EP, siochan->no) | + FIELD_PREP(SIOMSG_TYPE, MSG_TERMINATE)); + if (ret < 0) + dev_err(siochan->host->dev, "terminate call on chan %d failed: %d\n", + siochan->no, ret); + + ret = wait_for_completion_timeout(&tx->done, msecs_to_jiffies(500)); + if (!ret) + dev_err(siochan->host->dev, "terminate descriptor wait timed out\n"); + + tasklet_kill(&siochan->vc.task); + + spin_lock_irqsave(&siochan->vc.lock, flags); + WARN_ON(siochan->current_tx != tx); + siochan->current_tx = NULL; + sio_update_current_tx_locked(siochan); + spin_unlock_irqrestore(&siochan->vc.lock, flags); + + sio_tx_free(&tx->vd); +} + +static void sio_synchronize(struct dma_chan *chan) +{ + struct sio_chan *siochan = to_sio_chan(chan); + + flush_work(&siochan->terminate_wq); +} + +static void sio_free_chan_resources(struct dma_chan *chan) +{ + sio_terminate_all(chan); + sio_synchronize(chan); + vchan_free_chan_resources(&to_sio_chan(chan)->vc); +} + +static struct dma_chan *sio_dma_of_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct sio_data *sio = (struct sio_data *) ofdma->of_dma_data; + unsigned int index = dma_spec->args[0]; + + if (dma_spec->args_count != 1 || index >= sio->nchannels) + return ERR_PTR(-EINVAL); + + return dma_get_slave_channel(&sio->channels[index].vc.chan); +} + +static void sio_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_size) +{ + struct sio_data *sio = cookie; + + dev_err(sio->dev, "SIO down (crashed)"); +} + +static void sio_process_report(struct sio_chan *siochan) +{ + unsigned long flags; + + spin_lock_irqsave(&siochan->vc.lock, flags); + if (siochan->current_tx) { + struct sio_tx *tx = siochan->current_tx; + + if (tx->ninflight) + tx->ninflight--; + vchan_cyclic_callback(&tx->vd); + if (!sio_fill_in_locked(siochan) && !tx->ninflight) + complete(&tx->done); + } + spin_unlock_irqrestore(&siochan->vc.lock, flags); +} + +static void sio_recv_msg(void *cookie, u8 ep, u64 msg) +{ + struct sio_data *sio = cookie; + struct sio_tagdata *tags = &sio->tags; + u32 data; + u8 type, tag, sioep; + + if (ep != EP_SIO) + goto unknown; + + data = FIELD_GET(SIOMSG_DATA, msg); + // param = FIELD_GET(SIOMSG_PARAM, msg); + type = FIELD_GET(SIOMSG_TYPE, msg); + tag = FIELD_GET(SIOMSG_TAG, msg); + sioep = FIELD_GET(SIOMSG_EP, msg); + + switch (type) { + case MSG_STARTED: + dev_info(sio->dev, "SIO protocol v%u\n", data); + type = MSG_ACK; /* Pretend this is an ACK */ + fallthrough; + case MSG_ACK: + case MSG_NACK: + if (WARN_ON(tag >= SIO_NTAGS)) + break; + + if (tags->atomic[tag]) { + sio_ack_callback callback = tags->ack_callback[tag]; + + if (callback && !WARN_ON(sioep >= sio->nchannels)) + callback(&sio->channels[sioep], + tags->cookie[tag], type == MSG_ACK); + if (type == MSG_NACK) + dev_err(sio->dev, "got a NACK on channel %d\n", sioep); + sio_free_tag(sio, tag); + } else { + tags->acked[tag] = (type == MSG_ACK); + complete(&tags->completions[tag]); + } + break; + + case MSG_REPORT: + if (WARN_ON(sioep >= sio->nchannels)) + break; + + sio_process_report(&sio->channels[sioep]); + break; + + default: + goto unknown; + } + return; + +unknown: + dev_warn(sio->dev, "received unknown message: ep %x data %016llx\n", + ep, msg); +} + +static int _sio_send_siomsg(struct sio_data *sio, u64 msg, bool atomic, + sio_ack_callback ack_callback, void *cookie) +{ + int tag, ret; + + tag = sio_alloc_tag(sio); + if (tag < 0) + return tag; + + if (atomic) + sio_set_tag_atomic(sio, tag, ack_callback, cookie); + else + reinit_completion(&sio->tags.completions[tag]); + + msg &= ~SIOMSG_TAG; + msg |= FIELD_PREP(SIOMSG_TAG, tag); + ret = apple_rtkit_send_message(sio->rtk, EP_SIO, msg, NULL, + atomic); + if (ret < 0) { + sio_free_tag(sio, tag); + return ret; + } + + return tag; +} + +static int sio_send_siomsg(struct sio_data *sio, u64 msg) +{ + return _sio_send_siomsg(sio, msg, false, NULL, NULL); +} + +static int sio_send_siomsg_atomic(struct sio_data *sio, u64 msg, + sio_ack_callback ack_callback, + void *cookie) +{ + return _sio_send_siomsg(sio, msg, true, ack_callback, cookie); +} + +static int sio_call(struct sio_data *sio, u64 msg) +{ + int tag, ret; + + tag = sio_send_siomsg(sio, msg); + if (tag < 0) + return tag; + + ret = wait_for_completion_timeout(&sio->tags.completions[tag], + msecs_to_jiffies(SIO_CALL_TIMEOUT_MS)); + if (!ret) { + dev_warn(sio->dev, "call %8llx timed out\n", msg); + sio_free_tag(sio, tag); + return -ETIME; + } + + ret = sio->tags.acked[tag]; + sio_free_tag(sio, tag); + + return ret; +} + +static const struct apple_rtkit_ops sio_rtkit_ops = { + .crashed = sio_rtk_crashed, + .recv_message = sio_recv_msg, +}; + +static int sio_device_config(struct dma_chan *chan, + struct dma_slave_config *config) +{ + struct sio_chan *siochan = to_sio_chan(chan); + struct sio_data *sio = siochan->host; + bool is_tx = sio_chan_direction(siochan->no) == DMA_MEM_TO_DEV; + struct sio_shmem_chan_config *cfg_shmem = sio->shmem; + struct sio_shmem_chan_config cfg; + int ret; + + switch (is_tx ? config->dst_addr_width : config->src_addr_width) { + case DMA_SLAVE_BUSWIDTH_1_BYTE: + cfg.datashape = 0; + break; + case DMA_SLAVE_BUSWIDTH_2_BYTES: + cfg.datashape = 1; + break; + case DMA_SLAVE_BUSWIDTH_4_BYTES: + cfg.datashape = 2; + break; + default: + return -EINVAL; + } + + cfg.timeout = 0; + cfg.fifo = 0x800; + cfg.limit = 0x800; + cfg.threshold = 0x800; + + /* + * Dmaengine prescribes we ought to apply the new configuration only + * to newly-queued descriptors. + * + * To comply with dmaengine's interface we take the lazy path here: + * we apply the configuration right away, we only allow the channel + * to be configured once, which means subsequent calls to `device_config` + * either return -EBUSY if the configuration differs, or they are + * a no-op if the configuration is the same as the starting one. + * + * This is the reasonable thing to do given that these sio channels + * are tied to fixed peripherals, and what's more given that the + * only planned consumer of this dmaengine driver in the kernel is + * diplayport audio support, where the DMA configuration is fixed, + * and no more than a single descriptor (a cyclic one) gets ever issued + * at the same time. + * + * The code complexity cost of tracking to which descriptor + * the configuration relates would be significant here, especially + * since we need to do a non-atomic operation to apply it (a call to + * the coprocessor) and dmaengine has its bunch of atomicity + * restrictions. And this complexity would be for naught since it + * doesn't even get exercised by the only planned consumer. + */ + if (siochan->configured && memcmp(&siochan->cfg, &cfg, sizeof(cfg))) + return -EBUSY; + + *cfg_shmem = cfg; + dma_wmb(); + + ret = sio_call(sio, FIELD_PREP(SIOMSG_TYPE, MSG_CONFIGURE) | + FIELD_PREP(SIOMSG_EP, siochan->no)); + + if (ret == 1) + ret = 0; + else if (ret == 0) + ret = -EINVAL; + + if (ret == 0) { + siochan->configured = true; + siochan->cfg = cfg; + } + + return ret; +} + +static int sio_alloc_shmem(struct sio_data *sio) +{ + dma_addr_t iova; + int err; + + sio->shmem = dma_alloc_coherent(sio->dev, SIO_SHMEM_SIZE, + &iova, GFP_KERNEL | __GFP_ZERO); + if (!sio->shmem) + return -ENOMEM; + + sio->shmem_desc_base = (struct sio_coproc_desc *) (sio->shmem + 56); + sio->desc_allocated = devm_kzalloc(sio->dev, SIO_NO_DESC_SLOTS / 32, + GFP_KERNEL); + if (!sio->desc_allocated) + return -ENOMEM; + + err = sio_call(sio, FIELD_PREP(SIOMSG_TYPE, MSG_SETUP) | + FIELD_PREP(SIOMSG_PARAM, 1) | + FIELD_PREP(SIOMSG_DATA, iova >> 12)); + if (err != 1) { + if (err == 0) + err = -EINVAL; + return err; + } + + err = sio_call(sio, FIELD_PREP(SIOMSG_TYPE, MSG_SETUP) | + FIELD_PREP(SIOMSG_PARAM, 2) | + FIELD_PREP(SIOMSG_DATA, SIO_SHMEM_SIZE)); + if (err != 1) { + if (err == 0) + err = -EINVAL; + return err; + } + + return 0; +} + +static int sio_send_dt_params(struct sio_data *sio) +{ + struct device_node *np = sio->dev->of_node; + const char *propname = "apple,sio-firmware-params"; + int nparams, err, i; + + nparams = of_property_count_u32_elems(np, propname); + if (nparams < 0) { + err = nparams; + goto badprop; + } + + for (i = 0; i < nparams / 2; i++) { + u32 key, val; + + err = of_property_read_u32_index(np, propname, 2 * i, &key); + if (err) + goto badprop; + err = of_property_read_u32_index(np, propname, 2 * i + 1, &val); + if (err) + goto badprop; + + err = sio_call(sio, FIELD_PREP(SIOMSG_TYPE, MSG_SETUP) | + FIELD_PREP(SIOMSG_PARAM, key & 0xff) | + FIELD_PREP(SIOMSG_EP, key >> 8) | + FIELD_PREP(SIOMSG_DATA, val)); + if (err < 1) { + if (err == 0) + err = -ENXIO; + return dev_err_probe(sio->dev, err, "sending SIO parameter %#x value %#x\n", + key, val); + } + } + + return 0; + +badprop: + return dev_err_probe(sio->dev, err, "failed to read '%s'\n", propname); +} + +static int sio_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct sio_data *sio; + struct dma_device *dma; + int nchannels; + int err, i; + + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(42)); + if (err) + return dev_err_probe(&pdev->dev, err, "Failed to set DMA mask\n"); + + err = of_property_read_u32(np, "dma-channels", &nchannels); + if (err || nchannels > NCHANNELS_MAX) + return dev_err_probe(&pdev->dev, -EINVAL, + "missing or invalid dma-channels property\n"); + + sio = devm_kzalloc(&pdev->dev, struct_size(sio, channels, nchannels), GFP_KERNEL); + if (!sio) + return -ENOMEM; + + platform_set_drvdata(pdev, sio); + sio->dev = &pdev->dev; + sio->nchannels = nchannels; + + sio->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(sio->base)) + return PTR_ERR(sio->base); + + pm_runtime_get_noresume(&pdev->dev); + pm_runtime_set_active(&pdev->dev); + err = devm_pm_runtime_enable(&pdev->dev); + if (err < 0) + return dev_err_probe(&pdev->dev, err, + "pm_runtime_enable failed: %d\n", err); + + sio->rtk = devm_apple_rtkit_init(&pdev->dev, sio, NULL, 0, &sio_rtkit_ops); + if (IS_ERR(sio->rtk)) { + err = PTR_ERR(sio->rtk); + dev_err(&pdev->dev, "couldn't initialize rtkit\n"); + goto rpm_put; + } + for (i = 1; i < SIO_NTAGS; i++) + init_completion(&sio->tags.completions[i]); + + dma = &sio->dma; + dma_cap_set(DMA_PRIVATE, dma->cap_mask); + dma_cap_set(DMA_CYCLIC, dma->cap_mask); + + dma->dev = &pdev->dev; + dma->device_free_chan_resources = sio_free_chan_resources; + dma->device_tx_status = sio_tx_status; + dma->device_issue_pending = sio_issue_pending; + dma->device_terminate_all = sio_terminate_all; + dma->device_synchronize = sio_synchronize; + dma->device_prep_dma_cyclic = sio_prep_dma_cyclic; + dma->device_config = sio_device_config; + + dma->directions = BIT(DMA_MEM_TO_DEV); + dma->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; + dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); + + INIT_LIST_HEAD(&dma->channels); + for (i = 0; i < nchannels; i++) { + struct sio_chan *siochan = &sio->channels[i]; + + siochan->host = sio; + siochan->no = i; + siochan->vc.desc_free = sio_tx_free; + INIT_WORK(&siochan->terminate_wq, sio_terminate_work); + vchan_init(&siochan->vc, dma); + } + + writel(CPU_CONTROL_RUN, sio->base + REG_CPU_CONTROL); + + err = apple_rtkit_boot(sio->rtk); + if (err) + return dev_err_probe(&pdev->dev, err, "SIO did not boot\n"); + + err = apple_rtkit_start_ep(sio->rtk, EP_SIO); + if (err) + return dev_err_probe(&pdev->dev, err, "starting SIO endpoint\n"); + + err = sio_call(sio, FIELD_PREP(SIOMSG_TYPE, MSG_START)); + if (err < 1) { + if (err == 0) + err = -ENXIO; + return dev_err_probe(&pdev->dev, err, "starting SIO service\n"); + } + + err = sio_send_dt_params(sio); + if (err < 0) + return dev_err_probe(&pdev->dev, err, "failed to send boot-up parameters\n"); + + err = sio_alloc_shmem(sio); + if (err < 0) + return err; + + err = dma_async_device_register(&sio->dma); + if (err) + return dev_err_probe(&pdev->dev, err, "failed to register DMA device\n"); + + err = of_dma_controller_register(pdev->dev.of_node, sio_dma_of_xlate, sio); + if (err) { + dma_async_device_unregister(&sio->dma); + return dev_err_probe(&pdev->dev, err, "failed to register with OF\n"); + } + +rpm_put: + pm_runtime_put(&pdev->dev); + + return err; +} + +static void sio_remove(struct platform_device *pdev) +{ + struct sio_data *sio = platform_get_drvdata(pdev); + + of_dma_controller_free(pdev->dev.of_node); + dma_async_device_unregister(&sio->dma); +} + +static const struct of_device_id sio_of_match[] = { + { .compatible = "apple,sio", }, + { } +}; +MODULE_DEVICE_TABLE(of, sio_of_match); + +static __maybe_unused int sio_suspend(struct device *dev) +{ + /* + * TODO: SIO coproc sleep state + */ + return 0; +} + +static __maybe_unused int sio_resume(struct device *dev) +{ + return 0; +} + +static DEFINE_RUNTIME_DEV_PM_OPS(sio_pm_ops, sio_suspend, sio_resume, NULL); + +static struct platform_driver apple_sio_driver = { + .driver = { + .name = "apple-sio", + .of_match_table = sio_of_match, + .pm = pm_ptr(&sio_pm_ops), + }, + .probe = sio_probe, + .remove = sio_remove, +}; +module_platform_driver(apple_sio_driver); + +MODULE_AUTHOR("Martin Povišer "); +MODULE_DESCRIPTION("Driver for SIO coprocessor on Apple SoCs"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/dma/dma-axi-dmac.c b/drivers/dma/dma-axi-dmac.c index 5b06b0dc67ee12..0f25f6d8ae71fa 100644 --- a/drivers/dma/dma-axi-dmac.c +++ b/drivers/dma/dma-axi-dmac.c @@ -233,11 +233,9 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) unsigned int flags = 0; unsigned int val; - if (!chan->hw_sg) { - val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER); - if (val) /* Queue is full, wait for the next SOT IRQ */ - return; - } + val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER); + if (val) /* Queue is full, wait for the next SOT IRQ */ + return; desc = chan->next_desc; @@ -247,6 +245,7 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) return; list_move_tail(&vdesc->node, &chan->active_descs); desc = to_axi_dmac_desc(vdesc); + chan->next_desc = desc; } sg = &desc->sg[desc->num_submitted]; @@ -265,8 +264,6 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) else chan->next_desc = NULL; flags |= AXI_DMAC_FLAG_LAST; - } else { - chan->next_desc = desc; } sg->hw->id = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_ID); diff --git a/drivers/dma/fsl-edma-main.c b/drivers/dma/fsl-edma-main.c index a753b7cbfa7a33..dbcdd1e6831900 100644 --- a/drivers/dma/fsl-edma-main.c +++ b/drivers/dma/fsl-edma-main.c @@ -915,7 +915,6 @@ static void fsl_edma_remove(struct platform_device *pdev) of_dma_controller_free(np); dma_async_device_unregister(&fsl_edma->dma_dev); fsl_edma_cleanup_vchan(&fsl_edma->dma_dev); - fsl_disable_clocks(fsl_edma, fsl_edma->drvdata->dmamuxs); } static int fsl_edma_suspend_late(struct device *dev) diff --git a/drivers/dma/mediatek/mtk-uart-apdma.c b/drivers/dma/mediatek/mtk-uart-apdma.c index 08e15177427b94..96c18c815f1df6 100644 --- a/drivers/dma/mediatek/mtk-uart-apdma.c +++ b/drivers/dma/mediatek/mtk-uart-apdma.c @@ -41,7 +41,7 @@ #define VFF_STOP_CLR_B 0 #define VFF_EN_CLR_B 0 #define VFF_INT_EN_CLR_B 0 -#define VFF_4G_SUPPORT_CLR_B 0 +#define VFF_ADDR2_CLR_B 0 /* * interrupt trigger level for tx @@ -72,7 +72,7 @@ /* TX: the buffer size SW can write. RX: the buffer size HW can write. */ #define VFF_LEFT_SIZE 0x40 #define VFF_DEBUG_STATUS 0x50 -#define VFF_4G_SUPPORT 0x54 +#define VFF_ADDR2 0x54 struct mtk_uart_apdmadev { struct dma_device ddev; @@ -149,7 +149,7 @@ static void mtk_uart_apdma_start_tx(struct mtk_chan *c) mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_TX_INT_CLR_B); if (mtkd->support_33bits) - mtk_uart_apdma_write(c, VFF_4G_SUPPORT, VFF_4G_EN_B); + mtk_uart_apdma_write(c, VFF_ADDR2, upper_32_bits(d->addr)); } mtk_uart_apdma_write(c, VFF_EN, VFF_EN_B); @@ -192,7 +192,7 @@ static void mtk_uart_apdma_start_rx(struct mtk_chan *c) mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_RX_INT_CLR_B); if (mtkd->support_33bits) - mtk_uart_apdma_write(c, VFF_4G_SUPPORT, VFF_4G_EN_B); + mtk_uart_apdma_write(c, VFF_ADDR2, upper_32_bits(d->addr)); } mtk_uart_apdma_write(c, VFF_INT_EN, VFF_RX_INT_EN_B); @@ -298,7 +298,7 @@ static int mtk_uart_apdma_alloc_chan_resources(struct dma_chan *chan) } if (mtkd->support_33bits) - mtk_uart_apdma_write(c, VFF_4G_SUPPORT, VFF_4G_SUPPORT_CLR_B); + mtk_uart_apdma_write(c, VFF_ADDR2, VFF_ADDR2_CLR_B); err_pm: pm_runtime_put_noidle(mtkd->ddev.dev); diff --git a/drivers/dpll/zl3073x/dpll.c b/drivers/dpll/zl3073x/dpll.c index 9879d85d29af0f..a8001c9760382d 100644 --- a/drivers/dpll/zl3073x/dpll.c +++ b/drivers/dpll/zl3073x/dpll.c @@ -1039,10 +1039,8 @@ zl3073x_dpll_output_pin_phase_adjust_get(const struct dpll_pin *dpll_pin, out_id = zl3073x_output_pin_out_get(pin->id); out = zl3073x_out_state_get(zldev, out_id); - /* Convert value to ps and reverse two's complement negation applied - * during 'set' - */ - *phase_adjust = -out->phase_comp * pin->phase_gran; + /* The value in the register is expressed in half synth clock cycles. */ + *phase_adjust = out->phase_comp * pin->phase_gran; return 0; } @@ -1064,10 +1062,8 @@ zl3073x_dpll_output_pin_phase_adjust_set(const struct dpll_pin *dpll_pin, out_id = zl3073x_output_pin_out_get(pin->id); out = *zl3073x_out_state_get(zldev, out_id); - /* The value in the register is stored as two's complement negation - * of requested value and expressed in half synth clock cycles. - */ - out.phase_comp = -phase_adjust / pin->phase_gran; + /* The value in the register is expressed in half synth clock cycles. */ + out.phase_comp = phase_adjust / pin->phase_gran; /* Update output configuration from mailbox */ return zl3073x_out_state_set(zldev, out_id, &out); diff --git a/drivers/dpll/zl3073x/ref.h b/drivers/dpll/zl3073x/ref.h index efc7f59cd9f9c6..0d8618f5ce8df5 100644 --- a/drivers/dpll/zl3073x/ref.h +++ b/drivers/dpll/zl3073x/ref.h @@ -91,6 +91,8 @@ zl3073x_ref_freq_set(struct zl3073x_ref *ref, u32 freq) ref->freq_base = base; ref->freq_mult = mult; + ref->freq_ratio_m = 1; + ref->freq_ratio_n = 1; return 0; } diff --git a/drivers/edac/altera_edac.c b/drivers/edac/altera_edac.c index 0c5b94e64ea157..4edd2088c2db6f 100644 --- a/drivers/edac/altera_edac.c +++ b/drivers/edac/altera_edac.c @@ -1563,8 +1563,7 @@ static int altr_portb_setup(struct altr_edac_device_dev *device) goto err_release_group_1; } rc = devm_request_irq(&altdev->ddev, altdev->sb_irq, - prv->ecc_irq_handler, - IRQF_ONESHOT | IRQF_TRIGGER_HIGH, + prv->ecc_irq_handler, IRQF_TRIGGER_HIGH, ecc_name, altdev); if (rc) { edac_printk(KERN_ERR, EDAC_DEVICE, "PortB SBERR IRQ error\n"); @@ -1587,8 +1586,7 @@ static int altr_portb_setup(struct altr_edac_device_dev *device) goto err_release_group_1; } rc = devm_request_irq(&altdev->ddev, altdev->db_irq, - prv->ecc_irq_handler, - IRQF_ONESHOT | IRQF_TRIGGER_HIGH, + prv->ecc_irq_handler, IRQF_TRIGGER_HIGH, ecc_name, altdev); if (rc) { edac_printk(KERN_ERR, EDAC_DEVICE, "PortB DBERR IRQ error\n"); @@ -1970,8 +1968,7 @@ static int altr_edac_a10_device_add(struct altr_arria10_edac *edac, goto err_release_group1; } rc = devm_request_irq(edac->dev, altdev->sb_irq, prv->ecc_irq_handler, - IRQF_ONESHOT | IRQF_TRIGGER_HIGH, - ecc_name, altdev); + IRQF_TRIGGER_HIGH, ecc_name, altdev); if (rc) { edac_printk(KERN_ERR, EDAC_DEVICE, "No SBERR IRQ resource\n"); goto err_release_group1; @@ -1993,7 +1990,7 @@ static int altr_edac_a10_device_add(struct altr_arria10_edac *edac, goto err_release_group1; } rc = devm_request_irq(edac->dev, altdev->db_irq, prv->ecc_irq_handler, - IRQF_ONESHOT | IRQF_TRIGGER_HIGH, + IRQF_TRIGGER_HIGH, ecc_name, altdev); if (rc) { edac_printk(KERN_ERR, EDAC_DEVICE, "No DBERR IRQ resource\n"); diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 2391f3469961ab..63fca0ee2c23bc 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -3911,7 +3911,7 @@ static int per_family_init(struct amd64_pvt *pvt) } if (tmp_name) - scnprintf(pvt->ctl_name, sizeof(pvt->ctl_name), tmp_name); + scnprintf(pvt->ctl_name, sizeof(pvt->ctl_name), "%s", tmp_name); else scnprintf(pvt->ctl_name, sizeof(pvt->ctl_name), "F%02Xh_M%02Xh", pvt->fam, pvt->model); diff --git a/drivers/edac/i5000_edac.c b/drivers/edac/i5000_edac.c index 4a1bebc1ff14d4..471b8540d18b0f 100644 --- a/drivers/edac/i5000_edac.c +++ b/drivers/edac/i5000_edac.c @@ -1111,6 +1111,7 @@ static void calculate_dimm_size(struct i5000_pvt *pvt) n = snprintf(p, space, " "); p += n; + space -= n; for (branch = 0; branch < MAX_BRANCHES; branch++) { n = snprintf(p, space, " branch %d | ", branch); p += n; diff --git a/drivers/edac/i5400_edac.c b/drivers/edac/i5400_edac.c index b5cf25905b059d..fb49a1d1df1120 100644 --- a/drivers/edac/i5400_edac.c +++ b/drivers/edac/i5400_edac.c @@ -1026,13 +1026,13 @@ static void calculate_dimm_size(struct i5400_pvt *pvt) space -= n; } - space -= n; edac_dbg(2, "%s\n", mem_buffer); p = mem_buffer; space = PAGE_SIZE; n = snprintf(p, space, " "); p += n; + space -= n; for (branch = 0; branch < MAX_BRANCHES; branch++) { n = snprintf(p, space, " branch %d | ", branch); p += n; diff --git a/drivers/firmware/arm_ffa/driver.c b/drivers/firmware/arm_ffa/driver.c index c72ee475658569..c501c3104b3a4f 100644 --- a/drivers/firmware/arm_ffa/driver.c +++ b/drivers/firmware/arm_ffa/driver.c @@ -981,10 +981,27 @@ static void __do_sched_recv_cb(u16 part_id, u16 vcpu, bool is_per_vcpu) } } +/* + * Map logical ID index to the u16 index within the packed ID list. + * + * For native responses (FF-A width == kernel word size), IDs are + * tightly packed: idx -> idx. + * + * For 32-bit responses on a 64-bit kernel, each 64-bit register + * contributes 4 x u16 values but only the lower 2 are defined; the + * upper 2 are garbage. This mapping skips those upper halves: + * 0,1,2,3,4,5,... -> 0,1,4,5,8,9,... + */ +static int list_idx_to_u16_idx(int idx, bool is_native_resp) +{ + return is_native_resp ? idx : idx + 2 * (idx >> 1); +} + static void ffa_notification_info_get(void) { - int idx, list, max_ids, lists_cnt, ids_processed, ids_count[MAX_IDS_64]; - bool is_64b_resp; + int ids_processed, ids_count[MAX_IDS_64]; + int idx, list, max_ids, lists_cnt; + bool is_64b_resp, is_native_resp; ffa_value_t ret; u64 id_list; @@ -1001,6 +1018,7 @@ static void ffa_notification_info_get(void) } is_64b_resp = (ret.a0 == FFA_FN64_SUCCESS); + is_native_resp = (ret.a0 == FFA_FN_NATIVE(SUCCESS)); ids_processed = 0; lists_cnt = FIELD_GET(NOTIFICATION_INFO_GET_ID_COUNT, ret.a2); @@ -1017,12 +1035,16 @@ static void ffa_notification_info_get(void) /* Process IDs */ for (list = 0; list < lists_cnt; list++) { + int u16_idx; u16 vcpu_id, part_id, *packed_id_list = (u16 *)&ret.a3; if (ids_processed >= max_ids - 1) break; - part_id = packed_id_list[ids_processed++]; + u16_idx = list_idx_to_u16_idx(ids_processed, + is_native_resp); + part_id = packed_id_list[u16_idx]; + ids_processed++; if (ids_count[list] == 1) { /* Global Notification */ __do_sched_recv_cb(part_id, 0, false); @@ -1034,7 +1056,10 @@ static void ffa_notification_info_get(void) if (ids_processed >= max_ids - 1) break; - vcpu_id = packed_id_list[ids_processed++]; + u16_idx = list_idx_to_u16_idx(ids_processed, + is_native_resp); + vcpu_id = packed_id_list[u16_idx]; + ids_processed++; __do_sched_recv_cb(part_id, vcpu_id, true); } diff --git a/drivers/firmware/cirrus/cs_dsp.c b/drivers/firmware/cirrus/cs_dsp.c index 73d201e7d9927a..abed96fa5853ae 100644 --- a/drivers/firmware/cirrus/cs_dsp.c +++ b/drivers/firmware/cirrus/cs_dsp.c @@ -412,18 +412,23 @@ static ssize_t cs_dsp_debugfs_string_read(struct cs_dsp *dsp, size_t count, loff_t *ppos, const char **pstr) { - const char *str __free(kfree) = NULL; + const char *str; + ssize_t ret = 0; scoped_guard(mutex, &dsp->pwr_lock) { - if (!*pstr) - return 0; - - str = kasprintf(GFP_KERNEL, "%s\n", *pstr); - if (!str) - return -ENOMEM; - - return simple_read_from_buffer(user_buf, count, ppos, str, strlen(str)); + if (*pstr) { + str = kasprintf(GFP_KERNEL, "%s\n", *pstr); + if (str) { + ret = simple_read_from_buffer(user_buf, count, + ppos, str, strlen(str)); + kfree(str); + } else { + ret = -ENOMEM; + } + } } + + return ret; } static ssize_t cs_dsp_debugfs_wmfw_read(struct file *file, @@ -1483,7 +1488,7 @@ static int cs_dsp_load(struct cs_dsp *dsp, const struct firmware *firmware, const struct wmfw_region *region; const struct cs_dsp_region *mem; const char *region_name; - u8 *buf __free(kfree) = NULL; + u8 *buf = NULL; size_t buf_len = 0; size_t region_len; unsigned int reg; @@ -1638,6 +1643,8 @@ static int cs_dsp_load(struct cs_dsp *dsp, const struct firmware *firmware, ret = 0; out_fw: + kfree(buf); + if (ret == -EOVERFLOW) cs_dsp_err(dsp, "%s: file content overflows file data\n", file); @@ -2169,7 +2176,7 @@ static int cs_dsp_load_coeff(struct cs_dsp *dsp, const struct firmware *firmware struct cs_dsp_alg_region *alg_region; const char *region_name; int ret, pos, blocks, type, offset, reg, version; - u8 *buf __free(kfree) = NULL; + u8 *buf = NULL; size_t buf_len = 0; size_t region_len; @@ -2348,6 +2355,8 @@ static int cs_dsp_load_coeff(struct cs_dsp *dsp, const struct firmware *firmware ret = 0; out_fw: + kfree(buf); + if (ret == -EOVERFLOW) cs_dsp_err(dsp, "%s: file content overflows file data\n", file); diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c index 17b5f3415465ed..92e91c3eb4690e 100644 --- a/drivers/firmware/efi/efi.c +++ b/drivers/firmware/efi/efi.c @@ -692,13 +692,13 @@ static __init int match_config_table(const efi_guid_t *guid, static __init void reserve_unaccepted(struct efi_unaccepted_memory *unaccepted) { - phys_addr_t start, size; + phys_addr_t start, end; start = PAGE_ALIGN_DOWN(efi.unaccepted); - size = PAGE_ALIGN(sizeof(*unaccepted) + unaccepted->size); + end = PAGE_ALIGN(efi.unaccepted + sizeof(*unaccepted) + unaccepted->size); - memblock_add(start, size); - memblock_reserve(start, size); + memblock_add(start, end - start); + memblock_reserve(start, end - start); } int __init efi_config_parse_tables(const efi_config_table_t *config_tables, diff --git a/drivers/gpib/common/iblib.c b/drivers/gpib/common/iblib.c index 7cbb6a467177da..b672dd6aad25f2 100644 --- a/drivers/gpib/common/iblib.c +++ b/drivers/gpib/common/iblib.c @@ -227,11 +227,10 @@ int ibonline(struct gpib_board *board) #ifndef CONFIG_NIOS2 board->autospoll_task = kthread_run(&autospoll_thread, board, "gpib%d_autospoll_kthread", board->minor); - retval = IS_ERR(board->autospoll_task); - if (retval) { + if (IS_ERR(board->autospoll_task)) { dev_err(board->gpib_dev, "failed to create autospoll thread\n"); board->interface->detach(board); - return retval; + return PTR_ERR(board->autospoll_task); } #endif board->online = 1; diff --git a/drivers/gpib/ni_usb/ni_usb_gpib.c b/drivers/gpib/ni_usb/ni_usb_gpib.c index 1f8412de9fa329..b6fddb437f5524 100644 --- a/drivers/gpib/ni_usb/ni_usb_gpib.c +++ b/drivers/gpib/ni_usb/ni_usb_gpib.c @@ -566,7 +566,7 @@ static int ni_usb_write_registers(struct ni_usb_priv *ni_priv, retval, bytes_read); ni_usb_dump_raw_block(in_data, bytes_read); kfree(in_data); - return retval; + return retval ?: -EINVAL; } mutex_unlock(&ni_priv->addressed_transfer_lock); @@ -1780,7 +1780,7 @@ static int ni_usb_setup_init(struct gpib_board *board, struct ni_usb_register *w i++; if (i > NUM_INIT_WRITES) { dev_err(&usb_dev->dev, "bug!, buffer overrun, i=%i\n", i); - return 0; + return -EINVAL; } return i; } @@ -1799,10 +1799,12 @@ static int ni_usb_init(struct gpib_board *board) return -ENOMEM; writes_len = ni_usb_setup_init(board, writes); - if (writes_len) - retval = ni_usb_write_registers(ni_priv, writes, writes_len, &ibsta); - else - return -EFAULT; + if (writes_len < 0) { + kfree(writes); + return writes_len; + } + + retval = ni_usb_write_registers(ni_priv, writes, writes_len, &ibsta); kfree(writes); if (retval) { dev_err(&usb_dev->dev, "register write failed, retval=%i\n", retval); diff --git a/drivers/gpio/gpio-amd-fch.c b/drivers/gpio/gpio-amd-fch.c index e6c6c3ec7656e9..9f329938202bff 100644 --- a/drivers/gpio/gpio-amd-fch.c +++ b/drivers/gpio/gpio-amd-fch.c @@ -8,6 +8,7 @@ * */ +#include #include #include #include @@ -120,15 +121,15 @@ static int amd_fch_gpio_get(struct gpio_chip *gc, unsigned int offset) { unsigned long flags; - int ret; + u32 val; struct amd_fch_gpio_priv *priv = gpiochip_get_data(gc); void __iomem *ptr = amd_fch_gpio_addr(priv, offset); spin_lock_irqsave(&priv->lock, flags); - ret = (readl_relaxed(ptr) & AMD_FCH_GPIO_FLAG_READ); + val = readl_relaxed(ptr); spin_unlock_irqrestore(&priv->lock, flags); - return ret; + return FIELD_GET(AMD_FCH_GPIO_FLAG_READ, val); } static int amd_fch_gpio_request(struct gpio_chip *chip, diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c index 2adc3c07090822..189127721e3832 100644 --- a/drivers/gpio/gpiolib-cdev.c +++ b/drivers/gpio/gpiolib-cdev.c @@ -388,7 +388,7 @@ static int linehandle_create(struct gpio_device *gdev, void __user *ip) fd_publish(fdf); dev_dbg(&gdev->dev, "registered chardev handle for %d lines\n", - lh->num_descs); + handlereq.lines); return 0; } diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index 266d69b004b3b7..77190fc4fbd690 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -294,6 +294,8 @@ config DRM_VGEM source "drivers/gpu/drm/vkms/Kconfig" +source "drivers/gpu/drm/asahi/Kconfig" + source "drivers/gpu/drm/exynos/Kconfig" source "drivers/gpu/drm/rockchip/Kconfig" diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile index 16c63acda55cea..ed5c544198a3e2 100644 --- a/drivers/gpu/drm/Makefile +++ b/drivers/gpu/drm/Makefile @@ -220,6 +220,7 @@ obj-y += tiny/ obj-$(CONFIG_DRM_PL111) += pl111/ obj-$(CONFIG_DRM_TVE200) += tve200/ obj-$(CONFIG_DRM_ADP) += adp/ +obj-$(CONFIG_DRM_ASAHI) += asahi/ obj-$(CONFIG_DRM_XEN) += xen/ obj-$(CONFIG_DRM_VBOXVIDEO) += vboxvideo/ obj-$(CONFIG_DRM_LIMA) += lima/ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c index d31460a9e95829..7c9d8a6d0bfdb9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c @@ -1135,8 +1135,10 @@ static int amdgpu_acpi_enumerate_xcc(void) if (!dev_info) ret = amdgpu_acpi_dev_init(&dev_info, xcc_info, sbdf); - if (ret == -ENOMEM) + if (ret == -ENOMEM) { + kfree(xcc_info); return ret; + } if (!dev_info) { kfree(xcc_info); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index ecdfe6cb36ccdc..dac0b15823f2a6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -892,8 +892,10 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, struct amdgpu_bo *bo = e->bo; e->range = amdgpu_hmm_range_alloc(NULL); - if (unlikely(!e->range)) - return -ENOMEM; + if (unlikely(!e->range)) { + r = -ENOMEM; + goto out_free_user_pages; + } r = amdgpu_ttm_tt_get_user_pages(bo, e->range); if (r) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index 39387da8586b4f..83567ade842987 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -3172,7 +3172,6 @@ static int __init amdgpu_init(void) if (r) goto error_fence; - DRM_INFO("amdgpu kernel modesetting enabled.\n"); amdgpu_register_atpx_handler(); amdgpu_acpi_detect(); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c index 3e38c5db298712..5a93cbadc4f44a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c @@ -458,9 +458,6 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data, /* always clear VRAM */ flags |= AMDGPU_GEM_CREATE_VRAM_CLEARED; - if (args->in.domains & AMDGPU_GEM_DOMAIN_MMIO_REMAP) - return -EINVAL; - /* create a gem object to contain this object in */ if (args->in.domains & (AMDGPU_GEM_DOMAIN_GDS | AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c index d9c7ad297293bd..2b37398337afc3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c @@ -1387,7 +1387,7 @@ int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev, if (!*exp_ranges) *exp_ranges = range_cnt; err: - kfree(ranges); + kvfree(ranges); return ret; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c index 44f230d67da242..bfa64cd7a62d43 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c @@ -229,7 +229,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs, r = amdgpu_vm_flush(ring, job, need_pipe_sync); if (r) { amdgpu_ring_undo(ring); - return r; + goto free_fence; } } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index e08f58de4b17f2..f51ad28d0cbd5c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -153,14 +153,6 @@ void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain) c++; } - if (domain & AMDGPU_GEM_DOMAIN_MMIO_REMAP) { - places[c].fpfn = 0; - places[c].lpfn = 0; - places[c].mem_type = AMDGPU_PL_MMIO_REMAP; - places[c].flags = 0; - c++; - } - if (domain & AMDGPU_GEM_DOMAIN_GTT) { places[c].fpfn = 0; places[c].lpfn = 0; @@ -1534,8 +1526,17 @@ u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo) */ uint32_t amdgpu_bo_mem_stats_placement(struct amdgpu_bo *bo) { - uint32_t domain = bo->preferred_domains & AMDGPU_GEM_DOMAIN_MASK; + u32 domain; + /* + * MMIO_REMAP is internal now, so it no longer maps from a userspace + * domain bit. Keep fdinfo/mem-stats visibility by checking the actual + * TTM placement. + */ + if (bo->tbo.resource && bo->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP) + return AMDGPU_PL_MMIO_REMAP; + + domain = bo->preferred_domains & AMDGPU_GEM_DOMAIN_MASK; if (!domain) return TTM_PL_SYSTEM; @@ -1554,8 +1555,6 @@ uint32_t amdgpu_bo_mem_stats_placement(struct amdgpu_bo *bo) return AMDGPU_PL_OA; case AMDGPU_GEM_DOMAIN_DOORBELL: return AMDGPU_PL_DOORBELL; - case AMDGPU_GEM_DOMAIN_MMIO_REMAP: - return AMDGPU_PL_MMIO_REMAP; default: return TTM_PL_SYSTEM; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h index 52c2d1731aabf6..912c9afaf9e11e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h @@ -168,8 +168,6 @@ static inline unsigned amdgpu_mem_type_to_domain(u32 mem_type) return AMDGPU_GEM_DOMAIN_OA; case AMDGPU_PL_DOORBELL: return AMDGPU_GEM_DOMAIN_DOORBELL; - case AMDGPU_PL_MMIO_REMAP: - return AMDGPU_GEM_DOMAIN_MMIO_REMAP; default: break; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c index 2a6cf7963dde22..8de9f68f7bea66 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c @@ -4343,7 +4343,7 @@ int amdgpu_ras_init(struct amdgpu_device *adev) * to handle fatal error */ r = amdgpu_nbio_ras_sw_init(adev); if (r) - return r; + goto release_con; if (adev->nbio.ras && adev->nbio.ras->init_ras_controller_interrupt) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 2b931e855abd9d..c200d5dc7b6bd4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -1812,10 +1812,10 @@ static void amdgpu_ttm_pools_fini(struct amdgpu_device *adev) } /** - * amdgpu_ttm_mmio_remap_bo_init - Allocate the singleton 4K MMIO_REMAP BO + * amdgpu_ttm_mmio_remap_bo_init - Allocate the singleton MMIO_REMAP BO * @adev: amdgpu device * - * Allocates a one-page (4K) GEM BO in AMDGPU_GEM_DOMAIN_MMIO_REMAP when the + * Allocates a global BO with backing AMDGPU_PL_MMIO_REMAP when the * hardware exposes a remap base (adev->rmmio_remap.bus_addr) and the host * PAGE_SIZE is <= AMDGPU_GPU_PAGE_SIZE (4K). The BO is created as a regular * GEM object (amdgpu_bo_create). @@ -1824,42 +1824,95 @@ static void amdgpu_ttm_pools_fini(struct amdgpu_device *adev) * * 0 on success or intentional skip (feature not present/unsupported) * * negative errno on allocation failure */ -static int amdgpu_ttm_mmio_remap_bo_init(struct amdgpu_device *adev) +static int amdgpu_ttm_alloc_mmio_remap_bo(struct amdgpu_device *adev) { + struct ttm_operation_ctx ctx = { false, false }; + struct ttm_placement placement; + struct ttm_buffer_object *tbo; + struct ttm_place placements; struct amdgpu_bo_param bp; + struct ttm_resource *tmp; int r; /* Skip if HW doesn't expose remap, or if PAGE_SIZE > AMDGPU_GPU_PAGE_SIZE (4K). */ if (!adev->rmmio_remap.bus_addr || PAGE_SIZE > AMDGPU_GPU_PAGE_SIZE) return 0; + /* + * Allocate a BO first and then move it to AMDGPU_PL_MMIO_REMAP. + * The initial TTM resource assigned by amdgpu_bo_create() is + * replaced below with a fixed MMIO_REMAP placement. + */ memset(&bp, 0, sizeof(bp)); - - /* Create exactly one GEM BO in the MMIO_REMAP domain. */ - bp.type = ttm_bo_type_device; /* userspace-mappable GEM */ - bp.size = AMDGPU_GPU_PAGE_SIZE; /* 4K */ + bp.type = ttm_bo_type_device; + bp.size = AMDGPU_GPU_PAGE_SIZE; bp.byte_align = AMDGPU_GPU_PAGE_SIZE; - bp.domain = AMDGPU_GEM_DOMAIN_MMIO_REMAP; + bp.domain = 0; bp.flags = 0; bp.resv = NULL; bp.bo_ptr_size = sizeof(struct amdgpu_bo); - r = amdgpu_bo_create(adev, &bp, &adev->rmmio_remap.bo); if (r) return r; + r = amdgpu_bo_reserve(adev->rmmio_remap.bo, true); + if (r) + goto err_unref; + + tbo = &adev->rmmio_remap.bo->tbo; + + /* + * MMIO_REMAP is a fixed I/O placement (AMDGPU_PL_MMIO_REMAP). + */ + placement.num_placement = 1; + placement.placement = &placements; + placements.fpfn = 0; + placements.lpfn = 0; + placements.mem_type = AMDGPU_PL_MMIO_REMAP; + placements.flags = 0; + /* Force the BO into the fixed MMIO_REMAP placement */ + r = ttm_bo_mem_space(tbo, &placement, &tmp, &ctx); + if (unlikely(r)) + goto err_unlock; + + ttm_resource_free(tbo, &tbo->resource); + ttm_bo_assign_mem(tbo, tmp); + ttm_bo_pin(tbo); + + amdgpu_bo_unreserve(adev->rmmio_remap.bo); return 0; + +err_unlock: + amdgpu_bo_unreserve(adev->rmmio_remap.bo); + +err_unref: + amdgpu_bo_unref(&adev->rmmio_remap.bo); + adev->rmmio_remap.bo = NULL; + return r; } /** - * amdgpu_ttm_mmio_remap_bo_fini - Free the singleton MMIO_REMAP BO + * amdgpu_ttm_free_mmio_remap_bo - Free the singleton MMIO_REMAP BO * @adev: amdgpu device * * Frees the kernel-owned MMIO_REMAP BO if it was allocated by * amdgpu_ttm_mmio_remap_bo_init(). */ -static void amdgpu_ttm_mmio_remap_bo_fini(struct amdgpu_device *adev) +static void amdgpu_ttm_free_mmio_remap_bo(struct amdgpu_device *adev) { + if (!adev->rmmio_remap.bo) + return; + + if (!amdgpu_bo_reserve(adev->rmmio_remap.bo, true)) { + ttm_bo_unpin(&adev->rmmio_remap.bo->tbo); + amdgpu_bo_unreserve(adev->rmmio_remap.bo); + } + + /* + * At this point we rely on normal DRM teardown ordering: + * no new user ioctls can access the global MMIO_REMAP BO + * once TTM teardown begins. + */ amdgpu_bo_unref(&adev->rmmio_remap.bo); adev->rmmio_remap.bo = NULL; } @@ -2043,8 +2096,8 @@ int amdgpu_ttm_init(struct amdgpu_device *adev) return r; } - /* Allocate the singleton MMIO_REMAP BO (4K) if supported */ - r = amdgpu_ttm_mmio_remap_bo_init(adev); + /* Allocate the singleton MMIO_REMAP BO if supported */ + r = amdgpu_ttm_alloc_mmio_remap_bo(adev); if (r) return r; @@ -2111,7 +2164,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev) amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL, &adev->mman.sdma_access_ptr); - amdgpu_ttm_mmio_remap_bo_fini(adev); + amdgpu_ttm_free_mmio_remap_bo(adev); amdgpu_ttm_fw_reserve_vram_fini(adev); amdgpu_ttm_drv_reserve_vram_fini(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c index 8ddc4df06a1fde..45e2933214a802 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c @@ -1424,18 +1424,9 @@ static int sdma_v5_0_sw_init(struct amdgpu_ip_block *ip_block) adev->sdma.supported_reset = amdgpu_get_soft_full_reset_mask(&adev->sdma.instance[0].ring); - switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) { - case IP_VERSION(5, 0, 0): - case IP_VERSION(5, 0, 2): - case IP_VERSION(5, 0, 5): - if ((adev->sdma.instance[0].fw_version >= 35) && - !amdgpu_sriov_vf(adev) && - !adev->debug_disable_gpu_ring_reset) - adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; - break; - default: - break; - } + if (!amdgpu_sriov_vf(adev) && + !adev->debug_disable_gpu_ring_reset) + adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; /* Allocate memory for SDMA IP Dump buffer */ ptr = kcalloc(adev->sdma.num_instances * reg_count, sizeof(uint32_t), GFP_KERNEL); diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c index 51101b0aa2fabf..82b1d34a6533ef 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c @@ -1342,25 +1342,9 @@ static int sdma_v5_2_sw_init(struct amdgpu_ip_block *ip_block) adev->sdma.supported_reset = amdgpu_get_soft_full_reset_mask(&adev->sdma.instance[0].ring); - switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) { - case IP_VERSION(5, 2, 0): - case IP_VERSION(5, 2, 2): - case IP_VERSION(5, 2, 3): - case IP_VERSION(5, 2, 4): - if ((adev->sdma.instance[0].fw_version >= 76) && - !amdgpu_sriov_vf(adev) && - !adev->debug_disable_gpu_ring_reset) - adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; - break; - case IP_VERSION(5, 2, 5): - if ((adev->sdma.instance[0].fw_version >= 34) && - !amdgpu_sriov_vf(adev) && - !adev->debug_disable_gpu_ring_reset) - adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; - break; - default: - break; - } + if (!amdgpu_sriov_vf(adev) && + !adev->debug_disable_gpu_ring_reset) + adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; /* Allocate memory for SDMA IP Dump buffer */ ptr = kcalloc(adev->sdma.num_instances * reg_count, sizeof(uint32_t), GFP_KERNEL); diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c index 21704004498728..6809c6d4be5b1b 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c @@ -1351,18 +1351,9 @@ static int sdma_v6_0_sw_init(struct amdgpu_ip_block *ip_block) adev->sdma.supported_reset = amdgpu_get_soft_full_reset_mask(&adev->sdma.instance[0].ring); - switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) { - case IP_VERSION(6, 0, 0): - case IP_VERSION(6, 0, 2): - case IP_VERSION(6, 0, 3): - if ((adev->sdma.instance[0].fw_version >= 21) && - !amdgpu_sriov_vf(adev) && - !adev->debug_disable_gpu_ring_reset) - adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; - break; - default: - break; - } + if (!amdgpu_sriov_vf(adev) && + !adev->debug_disable_gpu_ring_reset) + adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; if (amdgpu_sdma_ras_sw_init(adev)) { dev_err(adev->dev, "Failed to initialize sdma ras block!\n"); diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c index 8897dcc9c1a0a9..e35fae9cdaf66e 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c @@ -1964,7 +1964,8 @@ static int vcn_v2_0_start_sriov(struct amdgpu_device *adev) struct mmsch_v2_0_cmd_end end = { {0} }; struct mmsch_v2_0_init_header *header; uint32_t *init_table = adev->virt.mm_table.cpu_addr; - uint8_t i = 0; + + /* This path only programs VCN instance 0. */ header = (struct mmsch_v2_0_init_header *)init_table; direct_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_WRITE; @@ -1983,93 +1984,93 @@ static int vcn_v2_0_start_sriov(struct amdgpu_device *adev) size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.inst[0].fw->size + 4); MMSCH_V2_0_INSERT_DIRECT_RD_MOD_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_STATUS), + SOC15_REG_OFFSET(UVD, 0, mmUVD_STATUS), 0xFFFFFFFF, 0x00000004); /* mc resume*/ if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_lo); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_hi); offset = 0; } else { MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), lower_32_bits(adev->vcn.inst->gpu_addr)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), upper_32_bits(adev->vcn.inst->gpu_addr)); offset = size; } MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_OFFSET0), + SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0), 0); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_SIZE0), + SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_SIZE0), size); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), lower_32_bits(adev->vcn.inst->gpu_addr + offset)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), upper_32_bits(adev->vcn.inst->gpu_addr + offset)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_OFFSET1), + SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_OFFSET1), 0); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_SIZE1), + SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW), lower_32_bits(adev->vcn.inst->gpu_addr + offset + AMDGPU_VCN_STACK_SIZE)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH), upper_32_bits(adev->vcn.inst->gpu_addr + offset + AMDGPU_VCN_STACK_SIZE)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_OFFSET2), + SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_OFFSET2), 0); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_SIZE2), + SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE); for (r = 0; r < adev->vcn.inst[0].num_enc_rings; ++r) { ring = &adev->vcn.inst->ring_enc[r]; ring->wptr = 0; MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_RB_BASE_LO), + SOC15_REG_OFFSET(UVD, 0, mmUVD_RB_BASE_LO), lower_32_bits(ring->gpu_addr)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_RB_BASE_HI), + SOC15_REG_OFFSET(UVD, 0, mmUVD_RB_BASE_HI), upper_32_bits(ring->gpu_addr)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_RB_SIZE), + SOC15_REG_OFFSET(UVD, 0, mmUVD_RB_SIZE), ring->ring_size / 4); } ring = &adev->vcn.inst->ring_dec; ring->wptr = 0; MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW), lower_32_bits(ring->gpu_addr)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH), upper_32_bits(ring->gpu_addr)); /* force RBC into idle state */ @@ -2080,7 +2081,7 @@ static int vcn_v2_0_start_sriov(struct amdgpu_device *adev) tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1); tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_RBC_RB_CNTL), tmp); + SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_RB_CNTL), tmp); /* add end packet */ tmp = sizeof(struct mmsch_v2_0_cmd_end); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c index ba99e0f258aee2..986cb297de8f83 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c @@ -401,27 +401,25 @@ static int kfd_dbg_get_dev_watch_id(struct kfd_process_device *pdd, int *watch_i return -ENOMEM; } -static void kfd_dbg_clear_dev_watch_id(struct kfd_process_device *pdd, int watch_id) +static void kfd_dbg_clear_dev_watch_id(struct kfd_process_device *pdd, u32 watch_id) { spin_lock(&pdd->dev->watch_points_lock); /* process owns device watch point so safe to clear */ - if ((pdd->alloc_watch_ids >> watch_id) & 0x1) { - pdd->alloc_watch_ids &= ~(0x1 << watch_id); - pdd->dev->alloc_watch_ids &= ~(0x1 << watch_id); + if (pdd->alloc_watch_ids & BIT(watch_id)) { + pdd->alloc_watch_ids &= ~BIT(watch_id); + pdd->dev->alloc_watch_ids &= ~BIT(watch_id); } spin_unlock(&pdd->dev->watch_points_lock); } -static bool kfd_dbg_owns_dev_watch_id(struct kfd_process_device *pdd, int watch_id) +static bool kfd_dbg_owns_dev_watch_id(struct kfd_process_device *pdd, u32 watch_id) { bool owns_watch_id = false; spin_lock(&pdd->dev->watch_points_lock); - owns_watch_id = watch_id < MAX_WATCH_ADDRESSES && - ((pdd->alloc_watch_ids >> watch_id) & 0x1); - + owns_watch_id = pdd->alloc_watch_ids & BIT(watch_id); spin_unlock(&pdd->dev->watch_points_lock); return owns_watch_id; @@ -432,6 +430,9 @@ int kfd_dbg_trap_clear_dev_address_watch(struct kfd_process_device *pdd, { int r; + if (watch_id >= MAX_WATCH_ADDRESSES) + return -EINVAL; + if (!kfd_dbg_owns_dev_watch_id(pdd, watch_id)) return -EINVAL; @@ -469,6 +470,9 @@ int kfd_dbg_trap_set_dev_address_watch(struct kfd_process_device *pdd, if (r) return r; + if (*watch_id >= MAX_WATCH_ADDRESSES) + return -EINVAL; + if (!pdd->dev->kfd->shared_resources.enable_mes) { r = debug_lock_and_unmap(pdd->dev->dqm); if (r) { diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c index a085faac9fe1a4..5c15168f4faf6b 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c @@ -1996,7 +1996,7 @@ static int signal_eviction_fence(struct kfd_process *p) ef = dma_fence_get_rcu_safe(&p->ef); rcu_read_unlock(); if (!ef) - return -EINVAL; + return true; ret = dma_fence_signal(ef); dma_fence_put(ef); diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index a8a59126b2d2b0..62622aa622066e 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -7434,7 +7434,7 @@ amdgpu_dm_connector_poll(struct amdgpu_dm_connector *aconnector, bool force) * * Only allow to poll such a connector again when forcing. */ - if (!force && link->local_sink && link->type == dc_connection_dac_load) + if (!force && link->local_sink && link->type == dc_connection_analog_load) return connector->status; mutex_lock(&aconnector->hpd_lock); @@ -12275,10 +12275,9 @@ static int dm_crtc_get_cursor_mode(struct amdgpu_device *adev, /* Overlay cursor not supported on HW before DCN * DCN401 does not have the cursor-on-scaled-plane or cursor-on-yuv-plane restrictions - * as previous DCN generations, so enable native mode on DCN401 in addition to DCE + * as previous DCN generations, so enable native mode on DCN401 */ - if (amdgpu_ip_version(adev, DCE_HWIP, 0) == 0 || - amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 0, 1)) { + if (amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 0, 1)) { *cursor_mode = DM_CURSOR_NATIVE_MODE; return 0; } @@ -12598,6 +12597,12 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev, * need to be added for DC to not disable a plane by mistake */ if (dm_new_crtc_state->cursor_mode == DM_CURSOR_OVERLAY_MODE) { + if (amdgpu_ip_version(adev, DCE_HWIP, 0) == 0) { + drm_dbg(dev, "Overlay cursor not supported on DCE\n"); + ret = -EINVAL; + goto fail; + } + ret = drm_atomic_add_affected_planes(state, crtc); if (ret) goto fail; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c index 1dcc79b35225f5..20a76d81d532d2 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c @@ -23,6 +23,9 @@ * Authors: AMD * */ + +#include + #include "amdgpu.h" #include "amdgpu_mode.h" #include "amdgpu_dm.h" diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c index e7b0928bd3db79..5948e2a6219e37 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c @@ -919,16 +919,15 @@ void amdgpu_dm_hpd_init(struct amdgpu_device *adev) continue; amdgpu_dm_connector = to_amdgpu_dm_connector(connector); + dc_link = amdgpu_dm_connector->dc_link; + if (!dc_link) + continue; /* * Analog connectors may be hot-plugged unlike other connector * types that don't support HPD. Only poll analog connectors. */ - use_polling |= - amdgpu_dm_connector->dc_link && - dc_connector_supports_analog(amdgpu_dm_connector->dc_link->link_id.id); - - dc_link = amdgpu_dm_connector->dc_link; + use_polling |= dc_connector_supports_analog(dc_link->link_id.id); /* * Get a base driver irq reference for hpd ints for the lifetime diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c index 7c4496fb4b9d48..f0946e67aef972 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c @@ -1060,10 +1060,15 @@ static void amdgpu_dm_plane_get_min_max_dc_plane_scaling(struct drm_device *dev, *min_downscale = plane_cap->max_downscale_factor.nv12; break; + /* All 64 bpp formats have the same fp16 scaling limits */ case DRM_FORMAT_XRGB16161616F: case DRM_FORMAT_ARGB16161616F: case DRM_FORMAT_XBGR16161616F: case DRM_FORMAT_ABGR16161616F: + case DRM_FORMAT_XRGB16161616: + case DRM_FORMAT_ARGB16161616: + case DRM_FORMAT_XBGR16161616: + case DRM_FORMAT_ABGR16161616: *max_upscale = plane_cap->max_upscale_factor.fp16; *min_downscale = plane_cap->max_downscale_factor.fp16; break; diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table.c b/drivers/gpu/drm/amd/display/dc/bios/command_table.c index 76a3559f0ddc15..b692fa37402d9f 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/command_table.c +++ b/drivers/gpu/drm/amd/display/dc/bios/command_table.c @@ -1874,8 +1874,7 @@ static void dac_encoder_control_prepare_params( uint8_t dac_standard) { params->ucDacStandard = dac_standard; - if (action == ENCODER_CONTROL_SETUP || - action == ENCODER_CONTROL_INIT) + if (action == ENCODER_CONTROL_INIT) params->ucAction = ATOM_ENCODER_INIT; else if (action == ENCODER_CONTROL_ENABLE) params->ucAction = ATOM_ENABLE; diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index 29edfa51ea2cc0..0a9758a042586f 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -2793,7 +2793,7 @@ void dc_get_underflow_debug_data_for_otg(struct dc *dc, int primary_otg_inst, st void dc_get_power_feature_status(struct dc *dc, int primary_otg_inst, struct power_features *out_data); -/** +/* * Software state variables used to program register fields across the display pipeline */ struct dc_register_software_state { diff --git a/drivers/gpu/drm/amd/display/dc/dc_types.h b/drivers/gpu/drm/amd/display/dc/dc_types.h index f46039f642034f..3e63d7bda1661c 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_types.h +++ b/drivers/gpu/drm/amd/display/dc/dc_types.h @@ -354,7 +354,7 @@ enum dc_connection_type { dc_connection_single, dc_connection_mst_branch, dc_connection_sst_branch, - dc_connection_dac_load + dc_connection_analog_load }; struct dc_csc_adjustments { diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c index 87dbb8d7ed27d8..bec8dab156eecb 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c @@ -102,6 +102,7 @@ static const struct link_encoder_funcs dce110_lnk_enc_funcs = { .enable_dp_output = dce110_link_encoder_enable_dp_output, .enable_dp_mst_output = dce110_link_encoder_enable_dp_mst_output, .enable_lvds_output = dce110_link_encoder_enable_lvds_output, + .enable_analog_output = dce110_link_encoder_enable_analog_output, .disable_output = dce110_link_encoder_disable_output, .dp_set_lane_settings = dce110_link_encoder_dp_set_lane_settings, .dp_set_phy_pattern = dce110_link_encoder_dp_set_phy_pattern, @@ -131,6 +132,21 @@ static enum bp_result link_transmitter_control( return result; } +static enum bp_result link_dac_encoder_control( + struct dce110_link_encoder *link_enc, + enum bp_encoder_control_action action, + uint32_t pix_clk_100hz) +{ + struct dc_bios *bios = link_enc->base.ctx->dc_bios; + struct bp_encoder_control encoder_control = {0}; + + encoder_control.action = action; + encoder_control.engine_id = link_enc->base.analog_engine; + encoder_control.pixel_clock = pix_clk_100hz / 10; + + return bios->funcs->encoder_control(bios, &encoder_control); +} + static void enable_phy_bypass_mode( struct dce110_link_encoder *enc110, bool enable) @@ -850,6 +866,7 @@ void dce110_link_encoder_construct( enc110->base.funcs = &dce110_lnk_enc_funcs; enc110->base.ctx = init_data->ctx; enc110->base.id = init_data->encoder; + enc110->base.analog_id = init_data->analog_encoder; enc110->base.hpd_source = init_data->hpd_source; enc110->base.connector = init_data->connector; @@ -1017,6 +1034,16 @@ void dce110_link_encoder_hw_init( cntl.coherent = false; cntl.hpd_sel = enc110->base.hpd_source; + if (enc110->base.analog_engine != ENGINE_ID_UNKNOWN) { + result = link_dac_encoder_control(enc110, ENCODER_CONTROL_INIT, 0); + if (result != BP_RESULT_OK) { + DC_LOG_ERROR("%s: Failed to execute VBIOS command table for DAC!\n", + __func__); + BREAK_TO_DEBUGGER(); + return; + } + } + /* The code below is only applicable to encoders with a digital transmitter. */ if (enc110->base.transmitter == TRANSMITTER_UNKNOWN) return; @@ -1166,6 +1193,22 @@ void dce110_link_encoder_enable_lvds_output( } } +void dce110_link_encoder_enable_analog_output( + struct link_encoder *enc, + uint32_t pixel_clock) +{ + struct dce110_link_encoder *enc110 = TO_DCE110_LINK_ENC(enc); + enum bp_result result; + + result = link_dac_encoder_control(enc110, ENCODER_CONTROL_ENABLE, pixel_clock); + + if (result != BP_RESULT_OK) { + DC_LOG_ERROR("%s: Failed to execute VBIOS command table!\n", + __func__); + BREAK_TO_DEBUGGER(); + } +} + /* enables DP PHY output */ void dce110_link_encoder_enable_dp_output( struct link_encoder *enc, @@ -1336,19 +1379,8 @@ void dce110_link_encoder_disable_output( struct bp_transmitter_control cntl = { 0 }; enum bp_result result; - switch (enc->analog_engine) { - case ENGINE_ID_DACA: - REG_UPDATE(DAC_ENABLE, DAC_ENABLE, 0); - break; - case ENGINE_ID_DACB: - /* DACB doesn't seem to be present on DCE6+, - * although there are references to it in the register file. - */ - DC_LOG_ERROR("%s DACB is unsupported\n", __func__); - break; - default: - break; - } + if (enc->analog_engine != ENGINE_ID_UNKNOWN) + link_dac_encoder_control(enc110, ENCODER_CONTROL_DISABLE, 0); /* The code below only applies to connectors that support digital signals. */ if (enc->transmitter == TRANSMITTER_UNKNOWN) @@ -1761,6 +1793,7 @@ static const struct link_encoder_funcs dce60_lnk_enc_funcs = { .enable_dp_output = dce60_link_encoder_enable_dp_output, .enable_dp_mst_output = dce60_link_encoder_enable_dp_mst_output, .enable_lvds_output = dce110_link_encoder_enable_lvds_output, + .enable_analog_output = dce110_link_encoder_enable_analog_output, .disable_output = dce110_link_encoder_disable_output, .dp_set_lane_settings = dce110_link_encoder_dp_set_lane_settings, .dp_set_phy_pattern = dce60_link_encoder_dp_set_phy_pattern, @@ -1793,6 +1826,7 @@ void dce60_link_encoder_construct( enc110->base.funcs = &dce60_lnk_enc_funcs; enc110->base.ctx = init_data->ctx; enc110->base.id = init_data->encoder; + enc110->base.analog_id = init_data->analog_encoder; enc110->base.hpd_source = init_data->hpd_source; enc110->base.connector = init_data->connector; diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.h b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.h index c58b69bc319b73..6870cb619d208e 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.h +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.h @@ -273,6 +273,11 @@ void dce110_link_encoder_enable_lvds_output( enum clock_source_id clock_source, uint32_t pixel_clock); +/* enables analog output from the DAC */ +void dce110_link_encoder_enable_analog_output( + struct link_encoder *enc, + uint32_t pixel_clock); + /* disable PHY output */ void dce110_link_encoder_disable_output( struct link_encoder *enc, diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c index 574618d5d4a4e0..87c19f17c799fc 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c @@ -1498,7 +1498,10 @@ static void dig_connect_to_otg( { struct dce110_stream_encoder *enc110 = DCE110STRENC_FROM_STRENC(enc); - REG_UPDATE(DIG_FE_CNTL, DIG_SOURCE_SELECT, tg_inst); + if (enc->id == ENGINE_ID_DACA || enc->id == ENGINE_ID_DACB) + REG_UPDATE(DAC_SOURCE_SELECT, DAC_SOURCE_SELECT, tg_inst); + else + REG_UPDATE(DIG_FE_CNTL, DIG_SOURCE_SELECT, tg_inst); } static unsigned int dig_source_otg( @@ -1507,7 +1510,10 @@ static unsigned int dig_source_otg( uint32_t tg_inst = 0; struct dce110_stream_encoder *enc110 = DCE110STRENC_FROM_STRENC(enc); - REG_GET(DIG_FE_CNTL, DIG_SOURCE_SELECT, &tg_inst); + if (enc->id == ENGINE_ID_DACA || enc->id == ENGINE_ID_DACB) + REG_GET(DAC_SOURCE_SELECT, DAC_SOURCE_SELECT, &tg_inst); + else + REG_GET(DIG_FE_CNTL, DIG_SOURCE_SELECT, &tg_inst); return tg_inst; } @@ -1568,16 +1574,25 @@ void dce110_stream_encoder_construct( enc110->se_mask = se_mask; } -static const struct stream_encoder_funcs dce110_an_str_enc_funcs = {}; +static const struct stream_encoder_funcs dce110_an_str_enc_funcs = { + .dig_connect_to_otg = dig_connect_to_otg, + .dig_source_otg = dig_source_otg, +}; void dce110_analog_stream_encoder_construct( struct dce110_stream_encoder *enc110, struct dc_context *ctx, struct dc_bios *bp, - enum engine_id eng_id) + enum engine_id eng_id, + const struct dce110_stream_enc_registers *regs, + const struct dce_stream_encoder_shift *se_shift, + const struct dce_stream_encoder_mask *se_mask) { enc110->base.funcs = &dce110_an_str_enc_funcs; enc110->base.ctx = ctx; enc110->base.id = eng_id; enc110->base.bp = bp; + enc110->regs = regs; + enc110->se_shift = se_shift; + enc110->se_mask = se_mask; } diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.h b/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.h index 068de1392121ed..342c0afe6a9494 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.h +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.h @@ -65,6 +65,7 @@ SRI(AFMT_60958_1, DIG, id), \ SRI(AFMT_60958_2, DIG, id), \ SRI(DIG_FE_CNTL, DIG, id), \ + SR(DAC_SOURCE_SELECT), \ SRI(HDMI_CONTROL, DIG, id), \ SRI(HDMI_GC, DIG, id), \ SRI(HDMI_GENERIC_PACKET_CONTROL0, DIG, id), \ @@ -290,7 +291,8 @@ #define SE_COMMON_MASK_SH_LIST_DCE80_100(mask_sh)\ SE_COMMON_MASK_SH_LIST_DCE_COMMON(mask_sh),\ SE_SF(TMDS_CNTL, TMDS_PIXEL_ENCODING, mask_sh),\ - SE_SF(TMDS_CNTL, TMDS_COLOR_FORMAT, mask_sh) + SE_SF(TMDS_CNTL, TMDS_COLOR_FORMAT, mask_sh),\ + SE_SF(DAC_SOURCE_SELECT, DAC_SOURCE_SELECT, mask_sh) #define SE_COMMON_MASK_SH_LIST_DCE110(mask_sh)\ SE_COMMON_MASK_SH_LIST_DCE_COMMON(mask_sh),\ @@ -494,6 +496,7 @@ struct dce_stream_encoder_shift { uint8_t DP_VID_N_MUL; uint8_t DP_VID_M_DOUBLE_VALUE_EN; uint8_t DIG_SOURCE_SELECT; + uint8_t DAC_SOURCE_SELECT; }; struct dce_stream_encoder_mask { @@ -626,6 +629,7 @@ struct dce_stream_encoder_mask { uint32_t DP_VID_N_MUL; uint32_t DP_VID_M_DOUBLE_VALUE_EN; uint32_t DIG_SOURCE_SELECT; + uint32_t DAC_SOURCE_SELECT; }; struct dce110_stream_enc_registers { @@ -653,6 +657,7 @@ struct dce110_stream_enc_registers { uint32_t AFMT_60958_1; uint32_t AFMT_60958_2; uint32_t DIG_FE_CNTL; + uint32_t DAC_SOURCE_SELECT; uint32_t DP_MSE_RATE_CNTL; uint32_t DP_MSE_RATE_UPDATE; uint32_t DP_PIXEL_FORMAT; @@ -712,7 +717,10 @@ void dce110_analog_stream_encoder_construct( struct dce110_stream_encoder *enc110, struct dc_context *ctx, struct dc_bios *bp, - enum engine_id eng_id); + enum engine_id eng_id, + const struct dce110_stream_enc_registers *regs, + const struct dce_stream_encoder_shift *se_shift, + const struct dce_stream_encoder_mask *se_mask); void dce110_se_audio_mute_control( struct stream_encoder *enc, bool mute); diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c index ebd74b43e935ee..5896ce5511ab1e 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c @@ -659,20 +659,6 @@ void dce110_update_info_frame(struct pipe_ctx *pipe_ctx) } } -static void -dce110_dac_encoder_control(struct pipe_ctx *pipe_ctx, bool enable) -{ - struct dc_link *link = pipe_ctx->stream->link; - struct dc_bios *bios = link->ctx->dc_bios; - struct bp_encoder_control encoder_control = {0}; - - encoder_control.action = enable ? ENCODER_CONTROL_ENABLE : ENCODER_CONTROL_DISABLE; - encoder_control.engine_id = link->link_enc->analog_engine; - encoder_control.pixel_clock = pipe_ctx->stream->timing.pix_clk_100hz / 10; - - bios->funcs->encoder_control(bios, &encoder_control); -} - void dce110_enable_stream(struct pipe_ctx *pipe_ctx) { enum dc_lane_count lane_count = @@ -703,8 +689,6 @@ void dce110_enable_stream(struct pipe_ctx *pipe_ctx) tg->funcs->set_early_control(tg, early_control); - if (dc_is_rgb_signal(pipe_ctx->stream->signal)) - dce110_dac_encoder_control(pipe_ctx, true); } static enum bp_result link_transmitter_control( @@ -1218,9 +1202,6 @@ void dce110_disable_stream(struct pipe_ctx *pipe_ctx) dccg->funcs->disable_symclk_se(dccg, stream_enc->stream_enc_inst, link_enc->transmitter - TRANSMITTER_UNIPHY_A); } - - if (dc_is_rgb_signal(pipe_ctx->stream->signal)) - dce110_dac_encoder_control(pipe_ctx, false); } void dce110_unblank_stream(struct pipe_ctx *pipe_ctx, @@ -1603,25 +1584,6 @@ static enum dc_status dce110_enable_stream_timing( return DC_OK; } -static void -dce110_select_crtc_source(struct pipe_ctx *pipe_ctx) -{ - struct dc_link *link = pipe_ctx->stream->link; - struct dc_bios *bios = link->ctx->dc_bios; - struct bp_crtc_source_select crtc_source_select = {0}; - enum engine_id engine_id = link->link_enc->preferred_engine; - - if (dc_is_rgb_signal(pipe_ctx->stream->signal)) - engine_id = link->link_enc->analog_engine; - - crtc_source_select.controller_id = CONTROLLER_ID_D0 + pipe_ctx->stream_res.tg->inst; - crtc_source_select.color_depth = pipe_ctx->stream->timing.display_color_depth; - crtc_source_select.engine_id = engine_id; - crtc_source_select.sink_signal = pipe_ctx->stream->signal; - - bios->funcs->select_crtc_source(bios, &crtc_source_select); -} - enum dc_status dce110_apply_single_controller_ctx_to_hw( struct pipe_ctx *pipe_ctx, struct dc_state *context, @@ -1641,10 +1603,6 @@ enum dc_status dce110_apply_single_controller_ctx_to_hw( hws->funcs.disable_stream_gating(dc, pipe_ctx); } - if (pipe_ctx->stream->signal == SIGNAL_TYPE_RGB) { - dce110_select_crtc_source(pipe_ctx); - } - if (pipe_ctx->stream_res.audio != NULL) { struct audio_output audio_output = {0}; @@ -1724,8 +1682,7 @@ enum dc_status dce110_apply_single_controller_ctx_to_hw( pipe_ctx->stream_res.tg->funcs->set_static_screen_control( pipe_ctx->stream_res.tg, event_triggers, 2); - if (!dc_is_virtual_signal(pipe_ctx->stream->signal) && - !dc_is_rgb_signal(pipe_ctx->stream->signal)) + if (!dc_is_virtual_signal(pipe_ctx->stream->signal)) pipe_ctx->stream_res.stream_enc->funcs->dig_connect_to_otg( pipe_ctx->stream_res.stream_enc, pipe_ctx->stream_res.tg->inst); @@ -3312,6 +3269,15 @@ void dce110_enable_tmds_link_output(struct dc_link *link, link->phy_state.symclk_state = SYMCLK_ON_TX_ON; } +static void dce110_enable_analog_link_output( + struct dc_link *link, + uint32_t pix_clk_100hz) +{ + link->link_enc->funcs->enable_analog_output( + link->link_enc, + pix_clk_100hz); +} + void dce110_enable_dp_link_output( struct dc_link *link, const struct link_resource *link_res, @@ -3449,6 +3415,7 @@ static const struct hw_sequencer_funcs dce110_funcs = { .enable_lvds_link_output = dce110_enable_lvds_link_output, .enable_tmds_link_output = dce110_enable_tmds_link_output, .enable_dp_link_output = dce110_enable_dp_link_output, + .enable_analog_link_output = dce110_enable_analog_link_output, .disable_link_output = dce110_disable_link_output, }; diff --git a/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h index 8ed9eea40c5646..4f6bd365e055a6 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h +++ b/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h @@ -1184,6 +1184,8 @@ struct hw_sequencer_funcs { const struct link_resource *link_res, enum clock_source_id clock_source, uint32_t pixel_clock); + void (*enable_analog_link_output)(struct dc_link *link, + uint32_t pixel_clock); void (*disable_link_output)(struct dc_link *link, const struct link_resource *link_res, enum signal_type signal); diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h b/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h index df512920a9fabe..b1a88618c5bf85 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h @@ -47,6 +47,7 @@ struct encoder_init_data { enum hpd_source_id hpd_source; /* TODO: in DAL2, here was pointer to EventManagerInterface */ struct graphics_object_id encoder; + struct graphics_object_id analog_encoder; enum engine_id analog_engine; struct dc_context *ctx; enum transmitter transmitter; @@ -81,6 +82,7 @@ struct link_encoder { int32_t aux_channel_offset; struct dc_context *ctx; struct graphics_object_id id; + struct graphics_object_id analog_id; struct graphics_object_id connector; uint32_t output_signals; enum engine_id preferred_engine; @@ -128,6 +130,8 @@ struct link_encoder_funcs { void (*enable_lvds_output)(struct link_encoder *enc, enum clock_source_id clock_source, uint32_t pixel_clock); + void (*enable_analog_output)(struct link_encoder *enc, + uint32_t pixel_clock); void (*disable_output)(struct link_encoder *link_enc, enum signal_type signal); void (*dp_set_lane_settings)(struct link_encoder *enc, diff --git a/drivers/gpu/drm/amd/display/dc/link/link_detection.c b/drivers/gpu/drm/amd/display/dc/link/link_detection.c index 7fa6bc97a91935..f986b57381bab8 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_detection.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_detection.c @@ -1173,11 +1173,10 @@ static bool detect_link_and_local_sink(struct dc_link *link, * - cheap DVI-A cable or adapter that doesn't connect DDC */ if (dc_connector_supports_analog(link->link_id.id)) { - /* If we didn't do DAC load detection yet, do it now - * to verify there really is a display connected. + /* If we didn't already detect a display using + * DAC load detection, we know it isn't connected. */ - if (link->type != dc_connection_dac_load && - !link_detect_dac_load_detect(link)) { + if (link->type != dc_connection_analog_load) { if (prev_sink) dc_sink_release(prev_sink); link_disconnect_sink(link); @@ -1185,7 +1184,7 @@ static bool detect_link_and_local_sink(struct dc_link *link, } DC_LOG_INFO("%s detected analog display without EDID\n", __func__); - link->type = dc_connection_dac_load; + link->type = dc_connection_analog_load; sink->edid_caps.analog = true; break; } @@ -1372,7 +1371,7 @@ static bool detect_link_and_local_sink(struct dc_link *link, * @link: DC link to evaluate (must support analog signalling). * @type: Updated with the detected connection type: * dc_connection_single (analog via DDC), - * dc_connection_dac_load (via load-detect), + * dc_connection_analog_load (via load-detect), * or dc_connection_none. * * Return: true if detection completed. @@ -1388,7 +1387,7 @@ static bool link_detect_analog(struct dc_link *link, enum dc_connection_type *ty } if (link_detect_dac_load_detect(link)) { - *type = dc_connection_dac_load; + *type = dc_connection_analog_load; return true; } diff --git a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c index 6ae13414761716..635f614c06734c 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c @@ -2208,6 +2208,18 @@ static enum dc_status enable_link_dp_mst( return enable_link_dp(state, pipe_ctx); } +static enum dc_status enable_link_analog( + struct dc_state *state, + struct pipe_ctx *pipe_ctx) +{ + struct dc_link *link = pipe_ctx->stream->link; + + link->dc->hwss.enable_analog_link_output( + link, pipe_ctx->stream->timing.pix_clk_100hz); + + return DC_OK; +} + static enum dc_status enable_link_virtual(struct pipe_ctx *pipe_ctx) { struct dc_link *link = pipe_ctx->stream->link; @@ -2263,7 +2275,7 @@ static enum dc_status enable_link( status = DC_OK; break; case SIGNAL_TYPE_RGB: - status = DC_OK; + status = enable_link_analog(state, pipe_ctx); break; case SIGNAL_TYPE_VIRTUAL: status = enable_link_virtual(pipe_ctx); diff --git a/drivers/gpu/drm/amd/display/dc/link/link_factory.c b/drivers/gpu/drm/amd/display/dc/link/link_factory.c index a6e2b0821969b9..9003e0d314e00b 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_factory.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_factory.c @@ -350,24 +350,6 @@ static enum transmitter translate_encoder_to_transmitter( return TRANSMITTER_UNKNOWN; } break; - case ENCODER_ID_EXTERNAL_NUTMEG: - switch (encoder.enum_id) { - case ENUM_ID_1: - return TRANSMITTER_NUTMEG_CRT; - default: - return TRANSMITTER_UNKNOWN; - } - break; - case ENCODER_ID_EXTERNAL_TRAVIS: - switch (encoder.enum_id) { - case ENUM_ID_1: - return TRANSMITTER_TRAVIS_CRT; - case ENUM_ID_2: - return TRANSMITTER_TRAVIS_LCD; - default: - return TRANSMITTER_UNKNOWN; - } - break; default: return TRANSMITTER_UNKNOWN; } @@ -451,20 +433,19 @@ static enum channel_id get_ddc_line(struct dc_link *link) return channel; } -static enum engine_id find_analog_engine(struct dc_link *link) +static enum engine_id find_analog_engine(struct dc_link *link, struct graphics_object_id *enc) { struct dc_bios *bp = link->ctx->dc_bios; - struct graphics_object_id encoder = {0}; enum bp_result bp_result = BP_RESULT_OK; int i; for (i = 0; i < 3; i++) { - bp_result = bp->funcs->get_src_obj(bp, link->link_id, i, &encoder); + bp_result = bp->funcs->get_src_obj(bp, link->link_id, i, enc); if (bp_result != BP_RESULT_OK) return ENGINE_ID_UNKNOWN; - switch (encoder.id) { + switch (enc->id) { case ENCODER_ID_INTERNAL_DAC1: case ENCODER_ID_INTERNAL_KLDSCP_DAC1: return ENGINE_ID_DACA; @@ -474,17 +455,10 @@ static enum engine_id find_analog_engine(struct dc_link *link) } } + memset(enc, 0, sizeof(*enc)); return ENGINE_ID_UNKNOWN; } -static bool transmitter_supported(const enum transmitter transmitter) -{ - return transmitter != TRANSMITTER_UNKNOWN && - transmitter != TRANSMITTER_NUTMEG_CRT && - transmitter != TRANSMITTER_TRAVIS_CRT && - transmitter != TRANSMITTER_TRAVIS_LCD; -} - static bool analog_engine_supported(const enum engine_id engine_id) { return engine_id == ENGINE_ID_DACA || @@ -502,6 +476,9 @@ static bool construct_phy(struct dc_link *link, struct dc_bios *bios = init_params->dc->ctx->dc_bios; const struct dc_vbios_funcs *bp_funcs = bios->funcs; struct bp_disp_connector_caps_info disp_connect_caps_info = { 0 }; + struct graphics_object_id link_encoder = { 0 }; + enum transmitter transmitter_from_encoder; + enum engine_id link_analog_engine; DC_LOGGER_INIT(dc_ctx->logger); @@ -522,21 +499,21 @@ static bool construct_phy(struct dc_link *link, link->link_id = bios->funcs->get_connector_id(bios, init_params->connector_index); - /* Determine early if the link has any supported encoders, - * so that we avoid initializing DDC and HPD, etc. - */ - bp_funcs->get_src_obj(bios, link->link_id, 0, &enc_init_data.encoder); - enc_init_data.transmitter = translate_encoder_to_transmitter(enc_init_data.encoder); - enc_init_data.analog_engine = find_analog_engine(link); - link->ep_type = DISPLAY_ENDPOINT_PHY; DC_LOG_DC("BIOS object table - link_id: %d", link->link_id.id); - if (!transmitter_supported(enc_init_data.transmitter) && - !analog_engine_supported(enc_init_data.analog_engine)) { + /* Determine early if the link has any supported encoders, + * so that we avoid initializing DDC and HPD, etc. + */ + bp_funcs->get_src_obj(bios, link->link_id, 0, &link_encoder); + transmitter_from_encoder = translate_encoder_to_transmitter(link_encoder); + link_analog_engine = find_analog_engine(link, &enc_init_data.analog_encoder); + + if (transmitter_from_encoder == TRANSMITTER_UNKNOWN && + !analog_engine_supported(link_analog_engine)) { DC_LOG_WARNING("link_id %d has unsupported encoder\n", link->link_id.id); - goto unsupported_fail; + goto create_fail; } if (bios->funcs->get_disp_connector_caps_info) { @@ -670,6 +647,9 @@ static bool construct_phy(struct dc_link *link, enc_init_data.connector = link->link_id; enc_init_data.channel = get_ddc_line(link); enc_init_data.hpd_source = get_hpd_line(link); + enc_init_data.transmitter = transmitter_from_encoder; + enc_init_data.encoder = link_encoder; + enc_init_data.analog_engine = link_analog_engine; link->hpd_src = enc_init_data.hpd_source; @@ -806,7 +786,6 @@ static bool construct_phy(struct dc_link *link, link->hpd_gpio = NULL; } -unsupported_fail: DC_LOG_DC("BIOS object table - %s failed.\n", __func__); return false; } diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c index d40d91ec2035ff..b78bb595d69ed6 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c @@ -242,7 +242,8 @@ static const struct dce110_stream_enc_registers stream_enc_regs[] = { stream_enc_regs(3), stream_enc_regs(4), stream_enc_regs(5), - stream_enc_regs(6) + stream_enc_regs(6), + {SR(DAC_SOURCE_SELECT),} /* DACA */ }; static const struct dce_stream_encoder_shift se_shift = { @@ -491,7 +492,8 @@ static struct stream_encoder *dce100_stream_encoder_create( return NULL; if (eng_id == ENGINE_ID_DACA || eng_id == ENGINE_ID_DACB) { - dce110_analog_stream_encoder_construct(enc110, ctx, ctx->dc_bios, eng_id); + dce110_analog_stream_encoder_construct(enc110, ctx, ctx->dc_bios, eng_id, + &stream_enc_regs[eng_id], &se_shift, &se_mask); return &enc110->base; } @@ -638,7 +640,8 @@ static struct link_encoder *dce100_link_encoder_create( if (!enc110) return NULL; - if (enc_init_data->connector.id == CONNECTOR_ID_VGA) { + if (enc_init_data->connector.id == CONNECTOR_ID_VGA && + enc_init_data->analog_engine != ENGINE_ID_UNKNOWN) { dce110_link_encoder_construct(enc110, enc_init_data, &link_enc_feature, @@ -978,7 +981,10 @@ struct stream_encoder *dce100_find_first_free_match_stream_enc_for_link( struct dc_link *link = stream->link; enum engine_id preferred_engine = link->link_enc->preferred_engine; - if (dc_is_rgb_signal(stream->signal)) + /* Prefer analog engine if the link encoder has one. + * Otherwise, it's an external encoder. + */ + if (dc_is_rgb_signal(stream->signal) && link->link_enc->analog_engine != ENGINE_ID_UNKNOWN) preferred_engine = link->link_enc->analog_engine; for (i = 0; i < pool->stream_enc_count; i++) { diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce110/dce110_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce110/dce110_resource.c index cd54382c0af3ee..7c09825cd9bd3b 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dce110/dce110_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dce110/dce110_resource.c @@ -895,6 +895,8 @@ static void get_pixel_clock_parameters( */ pixel_clk_params->requested_pix_clk_100hz = stream->timing.pix_clk_100hz; pixel_clk_params->encoder_object_id = stream->link->link_enc->id; + if (dc_is_rgb_signal(pipe_ctx->stream->signal)) + pixel_clk_params->encoder_object_id = stream->link->link_enc->analog_id; pixel_clk_params->signal_type = pipe_ctx->stream->signal; pixel_clk_params->controller_id = pipe_ctx->stream_res.tg->inst + 1; /* TODO: un-hardcode*/ diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c index f0152933bee2c0..6cf2faffc961b8 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c @@ -258,7 +258,9 @@ static const struct dce110_stream_enc_registers stream_enc_regs[] = { stream_enc_regs(2), stream_enc_regs(3), stream_enc_regs(4), - stream_enc_regs(5) + stream_enc_regs(5), + {0}, + {SR(DAC_SOURCE_SELECT),} /* DACA */ }; static const struct dce_stream_encoder_shift se_shift = { @@ -607,7 +609,8 @@ static struct stream_encoder *dce60_stream_encoder_create( return NULL; if (eng_id == ENGINE_ID_DACA || eng_id == ENGINE_ID_DACB) { - dce110_analog_stream_encoder_construct(enc110, ctx, ctx->dc_bios, eng_id); + dce110_analog_stream_encoder_construct(enc110, ctx, ctx->dc_bios, eng_id, + &stream_enc_regs[eng_id], &se_shift, &se_mask); return &enc110->base; } @@ -733,8 +736,9 @@ static struct link_encoder *dce60_link_encoder_create( if (!enc110) return NULL; - if (enc_init_data->connector.id == CONNECTOR_ID_VGA) { - dce110_link_encoder_construct(enc110, + if (enc_init_data->connector.id == CONNECTOR_ID_VGA && + enc_init_data->analog_engine != ENGINE_ID_UNKNOWN) { + dce60_link_encoder_construct(enc110, enc_init_data, &link_enc_feature, &link_enc_regs[ENGINE_ID_DACA], diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c index 8687104cabb723..066dbf8125a870 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c @@ -258,7 +258,8 @@ static const struct dce110_stream_enc_registers stream_enc_regs[] = { stream_enc_regs(3), stream_enc_regs(4), stream_enc_regs(5), - stream_enc_regs(6) + stream_enc_regs(6), + {SR(DAC_SOURCE_SELECT),} /* DACA */ }; static const struct dce_stream_encoder_shift se_shift = { @@ -614,7 +615,8 @@ static struct stream_encoder *dce80_stream_encoder_create( return NULL; if (eng_id == ENGINE_ID_DACA || eng_id == ENGINE_ID_DACB) { - dce110_analog_stream_encoder_construct(enc110, ctx, ctx->dc_bios, eng_id); + dce110_analog_stream_encoder_construct(enc110, ctx, ctx->dc_bios, eng_id, + &stream_enc_regs[eng_id], &se_shift, &se_mask); return &enc110->base; } @@ -740,7 +742,8 @@ static struct link_encoder *dce80_link_encoder_create( if (!enc110) return NULL; - if (enc_init_data->connector.id == CONNECTOR_ID_VGA) { + if (enc_init_data->connector.id == CONNECTOR_ID_VGA && + enc_init_data->analog_engine != ENGINE_ID_UNKNOWN) { dce110_link_encoder_construct(enc110, enc_init_data, &link_enc_feature, diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c index 4e962f522f1bee..228ae665c7893f 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c @@ -1230,12 +1230,12 @@ static struct stream_encoder *dcn315_stream_encoder_create( /*PHYB is wired off in HW, allow front end to remapping, otherwise needs more changes*/ /* Mapping of VPG, AFMT, DME register blocks to DIO block instance */ - if (eng_id <= ENGINE_ID_DIGF) { - vpg_inst = eng_id; - afmt_inst = eng_id; - } else + if (eng_id < 0 || eng_id >= ARRAY_SIZE(stream_enc_regs)) return NULL; + vpg_inst = eng_id; + afmt_inst = eng_id; + enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL); vpg = dcn31_vpg_create(ctx, vpg_inst); afmt = dcn31_afmt_create(ctx, afmt_inst); diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c index 5a95dd54cb4298..45abf3b2eb2c48 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c @@ -1223,12 +1223,12 @@ static struct stream_encoder *dcn316_stream_encoder_create( int afmt_inst; /* Mapping of VPG, AFMT, DME register blocks to DIO block instance */ - if (eng_id <= ENGINE_ID_DIGF) { - vpg_inst = eng_id; - afmt_inst = eng_id; - } else + if (eng_id < 0 || eng_id >= ARRAY_SIZE(stream_enc_regs)) return NULL; + vpg_inst = eng_id; + afmt_inst = eng_id; + enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL); vpg = dcn31_vpg_create(ctx, vpg_inst); afmt = dcn31_afmt_create(ctx, afmt_inst); diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c index b276fec3e479a2..d39a0f9c78c92c 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c @@ -1211,12 +1211,12 @@ static struct stream_encoder *dcn32_stream_encoder_create( int afmt_inst; /* Mapping of VPG, AFMT, DME register blocks to DIO block instance */ - if (eng_id <= ENGINE_ID_DIGF) { - vpg_inst = eng_id; - afmt_inst = eng_id; - } else + if (eng_id < 0 || eng_id >= ARRAY_SIZE(stream_enc_regs)) return NULL; + vpg_inst = eng_id; + afmt_inst = eng_id; + enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL); vpg = dcn32_vpg_create(ctx, vpg_inst); afmt = dcn32_afmt_create(ctx, afmt_inst); diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c index 3466ca34c93fed..c72c6dbc0cb4d7 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c @@ -1192,12 +1192,12 @@ static struct stream_encoder *dcn321_stream_encoder_create( int afmt_inst; /* Mapping of VPG, AFMT, DME register blocks to DIO block instance */ - if (eng_id <= ENGINE_ID_DIGF) { - vpg_inst = eng_id; - afmt_inst = eng_id; - } else + if (eng_id < 0 || eng_id >= ARRAY_SIZE(stream_enc_regs)) return NULL; + vpg_inst = eng_id; + afmt_inst = eng_id; + enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL); vpg = dcn321_vpg_create(ctx, vpg_inst); afmt = dcn321_afmt_create(ctx, afmt_inst); diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c index d056e5fd545871..9edabd8ceca9bf 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c @@ -1274,12 +1274,12 @@ static struct stream_encoder *dcn35_stream_encoder_create( int afmt_inst; /* Mapping of VPG, AFMT, DME register blocks to DIO block instance */ - if (eng_id <= ENGINE_ID_DIGF) { - vpg_inst = eng_id; - afmt_inst = eng_id; - } else + if (eng_id < 0 || eng_id >= ARRAY_SIZE(stream_enc_regs)) return NULL; + vpg_inst = eng_id; + afmt_inst = eng_id; + enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL); vpg = dcn31_vpg_create(ctx, vpg_inst); afmt = dcn31_afmt_create(ctx, afmt_inst); diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c index 9fab3169069c40..43ece2bbcd64f6 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c @@ -1254,12 +1254,12 @@ static struct stream_encoder *dcn35_stream_encoder_create( int afmt_inst; /* Mapping of VPG, AFMT, DME register blocks to DIO block instance */ - if (eng_id <= ENGINE_ID_DIGF) { - vpg_inst = eng_id; - afmt_inst = eng_id; - } else + if (eng_id < 0 || eng_id >= ARRAY_SIZE(stream_enc_regs)) return NULL; + vpg_inst = eng_id; + afmt_inst = eng_id; + enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL); vpg = dcn31_vpg_create(ctx, vpg_inst); afmt = dcn31_afmt_create(ctx, afmt_inst); diff --git a/drivers/gpu/drm/amd/include/amd_shared.h b/drivers/gpu/drm/amd/include/amd_shared.h index 17945094a13834..ac2d3701e2bdcc 100644 --- a/drivers/gpu/drm/amd/include/amd_shared.h +++ b/drivers/gpu/drm/amd/include/amd_shared.h @@ -89,6 +89,7 @@ enum amd_apu_flags { * @AMD_IP_BLOCK_TYPE_VPE: Video Processing Engine * @AMD_IP_BLOCK_TYPE_UMSCH_MM: User Mode Scheduler for Multimedia * @AMD_IP_BLOCK_TYPE_ISP: Image Signal Processor +* @AMD_IP_BLOCK_TYPE_RAS: Reliability, Availability, Serviceability * @AMD_IP_BLOCK_TYPE_NUM: Total number of IP block types */ enum amd_ip_block_type { diff --git a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c index 302af1fb6901e7..9c9e96655c4be2 100644 --- a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c +++ b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c @@ -1589,7 +1589,7 @@ int amdgpu_dpm_get_power_limit(struct amdgpu_device *adev, int ret = 0; if (!pp_funcs->get_power_limit) - return -ENODATA; + return -EOPNOTSUPP; mutex_lock(&adev->pm.mutex); ret = pp_funcs->get_power_limit(adev->powerplay.pp_handle, diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c index 9e635f733fbfdf..370326ec65d948 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c @@ -922,7 +922,7 @@ void smu_v13_0_12_get_gpu_metrics(struct smu_context *smu, void **table, gpu_metrics->gfx_below_host_limit_total_acc [i] = SMUQ10_ROUND( metrics->GfxclkBelowHostLimitTotalAcc[inst]); - }; + } } gpu_metrics->xgmi_link_width = metrics->XgmiWidth; diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig new file mode 100644 index 00000000000000..ab98d29b6b4a84 --- /dev/null +++ b/drivers/gpu/drm/asahi/Kconfig @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: GPL-2.0 + +config RUST_DRM_SCHED + bool + select DRM_SCHED + +config RUST_DRM_GEM_SHMEM_HELPER + bool + select DRM_GEM_SHMEM_HELPER + +config RUST_DRM_GPUVM + bool + select DRM_GPUVM + +config DRM_ASAHI + tristate "Asahi (DRM support for Apple AGX GPUs)" + depends on RUST + depends on DRM=y + depends on (ARM64 && ARCH_APPLE) || (COMPILE_TEST && !GENERIC_ATOMIC64) + depends on MMU + depends on IOMMU_SUPPORT + depends on PAGE_SIZE_16KB + select RUST_DRM_SCHED + select RUST_DRM_GEM_SHMEM_HELPER + select RUST_DRM_GPUVM + select RUST_APPLE_RTKIT + select WANT_DEV_COREDUMP + help + DRM driver for Apple AGX GPUs (G13x, found in the M1 SoC family) + +config DRM_ASAHI_DEBUG_ALLOCATOR + bool "Use debug allocator" + depends on DRM_ASAHI + help + Use an alternate, simpler allocator which significantly reduces + performance, but can help find firmware- or GPU-side memory safety + issues. However, it can also trigger firmware bugs more easily, + so expect GPU crashes. + + Say N unless you are debugging firmware structures or porting to a + new firmware version. diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile new file mode 100644 index 00000000000000..e6724866798760 --- /dev/null +++ b/drivers/gpu/drm/asahi/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_DRM_ASAHI) += asahi.o diff --git a/drivers/gpu/drm/asahi/alloc.rs b/drivers/gpu/drm/asahi/alloc.rs new file mode 100644 index 00000000000000..2711b784843300 --- /dev/null +++ b/drivers/gpu/drm/asahi/alloc.rs @@ -0,0 +1,1087 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU kernel object allocator. +//! +//! This kernel driver needs to manage a large number of GPU objects, in both firmware/kernel +//! address space and user address space. This module implements a simple grow-only heap allocator +//! based on the DRM MM range allocator, and a debug allocator that allocates each object as a +//! separate GEM object. +//! +//! Allocations may optionally have debugging enabled, which adds preambles that store metadata +//! about the allocation. This is useful for live debugging using the hypervisor or postmortem +//! debugging with a GPU memory snapshot, since it makes it easier to identify use-after-free and +//! caching issues. + +use kernel::{ + drm::mm, + error::Result, + prelude::*, + str::CString, // +}; + +use crate::debug::*; +use crate::driver::{ + AsahiDevRef, + AsahiDevice, // +}; +use crate::fw::types::Zeroable; +use crate::mmu; +use crate::object::{ + GpuArray, + GpuObject, + GpuOnlyArray, + GpuStruct, + GpuWeakPointer, // +}; +use crate::util::RangeExt; + +use core::cmp::Ordering; +use core::fmt; +use core::fmt::{ + Debug, + Formatter, // +}; +use core::marker::PhantomData; +use core::mem; +use core::ops::Range; +use core::ptr::NonNull; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Alloc; + +#[cfg(not(CONFIG_DRM_ASAHI_DEBUG_ALLOCATOR))] +/// The driver-global allocator type +pub(crate) type DefaultAllocator = HeapAllocator; + +#[cfg(not(CONFIG_DRM_ASAHI_DEBUG_ALLOCATOR))] +/// The driver-global allocation type +pub(crate) type DefaultAllocation = HeapAllocation; + +#[cfg(CONFIG_DRM_ASAHI_DEBUG_ALLOCATOR)] +/// The driver-global allocator type +pub(crate) type DefaultAllocator = SimpleAllocator; + +#[cfg(CONFIG_DRM_ASAHI_DEBUG_ALLOCATOR)] +/// The driver-global allocation type +pub(crate) type DefaultAllocation = SimpleAllocation; + +/// Represents a raw allocation (without any type information). +pub(crate) trait RawAllocation { + /// Returns the CPU-side pointer (if CPU mapping is enabled) as a byte non-null pointer. + fn ptr(&self) -> Option>; + /// Returns the GPU VA pointer as a u64. + fn gpu_ptr(&self) -> u64; + /// Returns the AsahiDevice that owns this allocation. + fn device(&self) -> &AsahiDevice; +} + +/// Represents a typed allocation. +pub(crate) trait Allocation: Debug { + /// Returns the typed CPU-side pointer (if CPU mapping is enabled). + fn ptr(&self) -> Option>; + /// Returns the GPU VA pointer as a u64. + fn gpu_ptr(&self) -> u64; + /// Returns the size of the allocation in bytes. + fn size(&self) -> usize; + /// Returns the AsahiDevice that owns this allocation. + fn device(&self) -> &AsahiDevice; +} + +/// A generic typed allocation wrapping a RawAllocation. +/// +/// This is currently the only Allocation implementation, since it is shared by all allocators. +/// +/// # Invariants +/// The alloaction at `alloc` must have a size equal or greater than `alloc_size` plus `debug_offset` plus `padding`. +pub(crate) struct GenericAlloc { + alloc: U, + alloc_size: usize, + debug_offset: usize, + padding: usize, + tag: u32, + pad_word: u32, + _p: PhantomData, +} + +impl Allocation for GenericAlloc { + /// Returns a pointer to the inner (usable) part of the allocation. + fn ptr(&self) -> Option> { + // SAFETY: self.debug_offset is always within the allocation per the invariant, so is safe to add + // to the base pointer. + unsafe { self.alloc.ptr().map(|p| p.add(self.debug_offset).cast()) } + } + /// Returns the GPU pointer to the inner (usable) part of the allocation. + fn gpu_ptr(&self) -> u64 { + self.alloc.gpu_ptr() + self.debug_offset as u64 + } + /// Returns the size of the inner (usable) part of the allocation. + fn size(&self) -> usize { + self.alloc_size + } + fn device(&self) -> &AsahiDevice { + self.alloc.device() + } +} + +impl Debug for GenericAlloc { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct(core::any::type_name::>()) + .field("ptr", &format_args!("{:?}", self.ptr())) + .field("gpu_ptr", &format_args!("{:#X?}", self.gpu_ptr())) + .field("size", &format_args!("{:#X?}", self.size())) + .finish() + } +} + +/// Debugging data associated with an allocation, when debugging is enabled. +#[repr(C)] +struct AllocDebugData { + state: u32, + tag: u32, + size: u64, + base_gpuva: u64, + obj_gpuva: u64, + name: [u8; 0x20], +} + +/// Magic flag indicating a live allocation. +const STATE_LIVE: u32 = u32::from_le_bytes(*b"LIVE"); +/// Magic flag indicating a freed allocation. +const STATE_DEAD: u32 = u32::from_le_bytes(*b"DEAD"); + +/// Marker byte to identify when firmware/GPU write beyond the end of an allocation. +const GUARD_MARKER: u32 = 0x93939393; + +impl Drop for GenericAlloc { + fn drop(&mut self) { + let debug_len = mem::size_of::(); + if self.debug_offset >= debug_len { + if let Some(p) = self.alloc.ptr() { + // SAFETY: self.debug_offset is always greater than the alloc size per + // the invariant, and greater than debug_len as checked above. + unsafe { + let p = p.as_ptr().add(self.debug_offset - debug_len); + (p as *mut u32).write(STATE_DEAD); + } + } + } + if debug_enabled(DebugFlags::FillAllocations) { + if let Some(p) = self.ptr() { + // SAFETY: Writing to our inner base pointer with our known inner size is safe. + unsafe { (p.as_ptr() as *mut u8).write_bytes(0xde, self.size()) }; + } + } + if self.padding != 0 { + if let Some(p) = self.ptr() { + // SAFETY: Per the invariant, we have at least `self.padding` bytes trailing + // the inner base pointer, after `size()` bytes. + let guard = unsafe { + core::slice::from_raw_parts( + (p.as_ptr() as *mut u8 as *const u8).add(self.size()), + self.padding, + ) + }; + let mut first_err = None; + let mut last_err = 0; + for (i, p) in guard.iter().enumerate() { + if *p != (self.pad_word >> (8 * (i & 3))) as u8 { + if first_err.is_none() { + first_err = Some(i); + } + last_err = i; + } + } + if let Some(start) = first_err { + dev_warn!( + self.device().as_ref(), + "Allocator: Corruption after object of type {}/{:#x} at {:#x}:{:#x} + {:#x}..={:#x}\n", + core::any::type_name::(), + self.tag, + self.gpu_ptr(), + self.size(), + start, + last_err, + ); + } + } + } + } +} + +static_assert!(mem::size_of::() == 0x40); + +/// A trait representing an allocator. +pub(crate) trait Allocator { + /// The raw allocation type used by this allocator. + type Raw: RawAllocation; + // TODO: Needs associated_type_defaults + // type Allocation = GenericAlloc; + + /// Returns whether CPU-side mapping is enabled. + fn cpu_maps(&self) -> bool; + /// Returns the minimum alignment for allocations. + fn min_align(&self) -> usize; + /// Allocate an object of the given size in bytes with the given alignment. + fn alloc(&mut self, size: usize, align: usize) -> Result; + + /// Returns a tuple of (count, size) of how much garbage (freed but not yet reusable objects) + /// exists in this allocator. Optional. + fn garbage(&self) -> (usize, usize) { + (0, 0) + } + /// Collect garbage for this allocator, up to the given object count. Optional. + fn collect_garbage(&mut self, _count: usize) {} + + /// Allocate a new GpuStruct object. See [`GpuObject::new`]. + #[inline(never)] + fn new_object( + &mut self, + inner: T, + callback: impl for<'a> FnOnce(&'a T) -> T::Raw<'a>, + ) -> Result>> { + GpuObject::>::new(self.alloc_object()?, inner, callback) + } + + /// Allocate a new GpuStruct object. See [`GpuObject::new_default`]. + #[inline(never)] + fn new_default( + &mut self, + ) -> Result>> + where + for<'a> ::Raw<'a>: Default + Zeroable, + { + GpuObject::>::new_default(self.alloc_object()?) + } + + /// Allocate a new GpuStruct object. See [`GpuObject::new_init`]. + #[inline(never)] + fn new_init<'a, T: GpuStruct, R: PinInit, F>, E, F>( + &mut self, + inner_init: impl Init, + raw_init: impl FnOnce(&'a T, GpuWeakPointer) -> R, + ) -> Result>> + where + kernel::error::Error: core::convert::From, + kernel::error::Error: core::convert::From, + { + GpuObject::>::new_init_prealloc( + self.alloc_object()?, + |_p| inner_init, + raw_init, + ) + } + + /// Allocate a generic buffer of the given size and alignment, applying the debug features if + /// enabled to tag it and detect overflows. + fn alloc_generic( + &mut self, + size: usize, + align: usize, + tag: Option, + ) -> Result> { + let padding = if debug_enabled(DebugFlags::DetectOverflows) { + size + } else { + 0 + }; + + let ret: GenericAlloc = + if self.cpu_maps() && debug_enabled(debug::DebugFlags::DebugAllocations) { + let debug_align = self.min_align().max(align); + let debug_len = mem::size_of::(); + let debug_offset = (debug_len * 2 + debug_align - 1) & !(debug_align - 1); + + let alloc = self.alloc(size + debug_offset + padding, align)?; + + let mut debug = AllocDebugData { + state: STATE_LIVE, + tag: tag.unwrap_or(0), + size: size as u64, + base_gpuva: alloc.gpu_ptr(), + obj_gpuva: alloc.gpu_ptr() + debug_offset as u64, + name: [0; 0x20], + }; + + let name = core::any::type_name::().as_bytes(); + let len = name.len().min(debug.name.len() - 1); + debug.name[..len].copy_from_slice(&name[..len]); + + if let Some(p) = alloc.ptr() { + // SAFETY: Per the size calculations above, this pointer math and the + // writes never exceed the allocation size. + unsafe { + let p = p.as_ptr(); + p.write_bytes(0x42, debug_offset - 2 * debug_len); + let cur = p.add(debug_offset - debug_len) as *mut AllocDebugData; + let prev = p.add(debug_offset - 2 * debug_len) as *mut AllocDebugData; + prev.copy_from(cur, 1); + cur.copy_from(&debug, 1); + }; + } + + GenericAlloc { + alloc, + alloc_size: size, + debug_offset, + tag: tag.unwrap_or(0), + pad_word: tag.unwrap_or(GUARD_MARKER) | 0x81818181, + padding, + _p: PhantomData, + } + } else { + GenericAlloc { + alloc: self.alloc(size + padding, align)?, + alloc_size: size, + debug_offset: 0, + tag: tag.unwrap_or(0), + pad_word: tag.unwrap_or(GUARD_MARKER) | 0x81818181, + padding, + _p: PhantomData, + } + }; + + if debug_enabled(DebugFlags::FillAllocations) { + if let Some(p) = ret.ptr() { + // SAFETY: Writing to our inner base pointer with our known inner size is safe. + unsafe { (p.as_ptr() as *mut u8).write_bytes(0xaa, ret.size()) }; + } + } + + if padding != 0 { + if let Some(p) = ret.ptr() { + // SAFETY: Per the invariant, we have at least `self.padding` bytes trailing + // the inner base pointer, after `size()` bytes. + let guard = unsafe { + core::slice::from_raw_parts_mut( + (p.as_ptr() as *mut u8).add(ret.size()), + padding, + ) + }; + for (i, p) in guard.iter_mut().enumerate() { + *p = (ret.pad_word >> (8 * (i & 3))) as u8; + } + } + } + + Ok(ret) + } + + /// Allocate an object of a given type, without actually initializing the allocation. + /// + /// This is useful to directly call [`GpuObject::new_*`], without borrowing a reference to the + /// allocator for the entire duration (e.g. if further allocations need to happen inside the + /// callbacks). + fn alloc_object(&mut self) -> Result> { + let size = mem::size_of::>(); + let align = mem::align_of::>(); + + self.alloc_generic(size, align, None) + } + + /// Allocate an empty `GpuArray` of a given type and length. + fn array_empty( + &mut self, + count: usize, + ) -> Result>> { + let size = mem::size_of::() * count; + let align = mem::align_of::(); + + let alloc = self.alloc_generic(size, align, None)?; + GpuArray::>::empty(alloc, count) + } + + /// Allocate an empty `GpuArray` of a given type and length. + fn array_empty_tagged( + &mut self, + count: usize, + tag: &[u8; 4], + ) -> Result>> { + let size = mem::size_of::() * count; + let align = mem::align_of::(); + + let alloc = self.alloc_generic(size, align, Some(u32::from_le_bytes(*tag)))?; + GpuArray::>::empty(alloc, count) + } + + /// Allocate an empty `GpuOnlyArray` of a given type and length. + fn array_gpuonly( + &mut self, + count: usize, + ) -> Result>> { + let size = mem::size_of::() * count; + let align = mem::align_of::(); + + let alloc = self.alloc_generic(size, align, None)?; + GpuOnlyArray::>::new(alloc, count) + } +} + +/// A simple allocation backed by a separate GEM object. +/// +/// # Invariants +/// `ptr` is either None or a valid, non-null pointer to the CPU view of the object. +/// `gpu_ptr` is the GPU-side VA of the object. +pub(crate) struct SimpleAllocation { + dev: AsahiDevRef, + ptr: Option>, + gpu_ptr: u64, + _mapping: mmu::KernelMapping, + obj: crate::gem::ObjectRef, +} + +/// SAFETY: `SimpleAllocation` just points to raw memory and should be safe to send across threads. +unsafe impl Send for SimpleAllocation {} +/// SAFETY: `SimpleAllocation` just points to raw memory and should be safe to share across threads. +unsafe impl Sync for SimpleAllocation {} + +impl Drop for SimpleAllocation { + fn drop(&mut self) { + mod_dev_dbg!( + self.device(), + "SimpleAllocator: drop object @ {:#x}\n", + self.gpu_ptr() + ); + if debug_enabled(DebugFlags::FillAllocations) { + if let Ok(mut vmap) = self.obj.vmap() { + vmap.memset(0x42); + } + } + } +} + +impl RawAllocation for SimpleAllocation { + fn ptr(&self) -> Option> { + self.ptr + } + fn gpu_ptr(&self) -> u64 { + self.gpu_ptr + } + fn device(&self) -> &AsahiDevice { + &self.dev + } +} + +/// A simple allocator that allocates each object as its own GEM object, aligned to the end of a +/// page. +/// +/// This is very slow, but it has the advantage that over-reads by the firmware or GPU will fault on +/// the guard page after the allocation, which can be useful to validate that the firmware's or +/// GPU's idea of object size what we expect. +pub(crate) struct SimpleAllocator { + dev: AsahiDevRef, + range: Range, + prot: mmu::Prot, + vm: mmu::Vm, + min_align: usize, + cpu_maps: bool, +} + +impl SimpleAllocator { + /// Create a new `SimpleAllocator` for a given address range and `Vm`. + #[allow(dead_code)] + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + dev: &AsahiDevice, + vm: &mmu::Vm, + range: Range, + min_align: usize, + prot: mmu::Prot, + _block_size: usize, + mut cpu_maps: bool, + _name: fmt::Arguments<'_>, + _keep_garbage: bool, + ) -> Result { + if debug_enabled(DebugFlags::ForceCPUMaps) { + cpu_maps = true; + } + Ok(SimpleAllocator { + dev: dev.into(), + vm: vm.clone(), + range, + prot, + min_align, + cpu_maps, + }) + } +} + +impl Allocator for SimpleAllocator { + type Raw = SimpleAllocation; + + fn cpu_maps(&self) -> bool { + self.cpu_maps + } + + fn min_align(&self) -> usize { + self.min_align + } + + #[inline(never)] + fn alloc(&mut self, size: usize, align: usize) -> Result { + let size_aligned = (size + mmu::UAT_PGSZ - 1) & !mmu::UAT_PGMSK; + let align = self.min_align.max(align); + let offset = (size_aligned - size) & !(align - 1); + + mod_dev_dbg!( + &self.dev, + "SimpleAllocator::new: size={:#x} size_al={:#x} al={:#x} off={:#x}\n", + size, + size_aligned, + align, + offset + ); + + let mut obj = crate::gem::new_kernel_object(&self.dev, size_aligned)?; + let p = obj.vmap()?.as_mut_ptr() as *mut u8; + if debug_enabled(DebugFlags::FillAllocations) { + obj.vmap()?.memset(0xde); + } + let mapping = obj.map_into_range( + &self.vm, + self.range.clone(), + self.min_align.max(mmu::UAT_PGSZ) as u64, + self.prot, + true, + )?; + + let iova = mapping.iova(); + + // SAFETY: Per the math above to calculate `size_aligned`, this can never overflow. + let ptr = unsafe { p.add(offset) }; + let gpu_ptr = iova + offset as u64; + + mod_dev_dbg!( + &self.dev, + "SimpleAllocator::new -> {:#?} / {:#?} | {:#x} / {:#x}\n", + p, + ptr, + iova, + gpu_ptr + ); + + Ok(SimpleAllocation { + dev: self.dev.clone(), + ptr: NonNull::new(ptr), + gpu_ptr, + _mapping: mapping, + obj, + }) + } +} + +/// Inner data for an allocation from the heap allocator. +/// +/// This is wrapped in an `mm::Node`. +pub(crate) struct HeapAllocationInner { + dev: AsahiDevRef, + ptr: Option>, + real_size: usize, +} + +/// SAFETY: `HeapAllocationInner` just points to raw memory and should be safe to send across threads. +unsafe impl Send for HeapAllocationInner {} +/// SAFETY: `HeapAllocationInner` just points to raw memory and should be safe to share between threads. +unsafe impl Sync for HeapAllocationInner {} + +/// Outer view of a heap allocation. +/// +/// This uses an Option<> so we can move the internal `Node` into the garbage pool when it gets +/// dropped. +/// +/// # Invariants +/// The `Option` must always be `Some(...)` while this object is alive. +pub(crate) struct HeapAllocation(Option>); + +impl Drop for HeapAllocation { + fn drop(&mut self) { + let node = self.0.take().unwrap(); + let size = node.size(); + let alloc = node.alloc_ref(); + + alloc.with(|a| { + if let Some(garbage) = a.garbage.as_mut() { + if garbage.push(node, GFP_KERNEL).is_err() { + dev_err!( + &a.dev.as_ref(), + "HeapAllocation[{}]::drop: Failed to keep garbage\n", + &*a.name, + ); + } + a.total_garbage += size as usize; + None + } else { + // We need to ensure node survives this scope, since dropping it + // will try to take the mm lock and deadlock us + Some(node) + } + }); + } +} + +impl mm::AllocInner for HeapAllocatorInner { + fn drop_object( + &mut self, + start: u64, + _size: u64, + _color: usize, + obj: &mut HeapAllocationInner, + ) { + /* real_size == 0 means it's a guard node */ + if obj.real_size > 0 { + mod_dev_dbg!( + obj.dev, + "HeapAllocator[{}]: drop object @ {:#x} ({} bytes)\n", + &*self.name, + start, + obj.real_size, + ); + self.allocated -= obj.real_size; + } + } +} + +impl RawAllocation for HeapAllocation { + // SAFETY: This function must always return a valid pointer. + // Since the HeapAllocation contains a reference to the + // backing_objects array that contains the object backing this pointer, + // and objects are only ever added to it, this pointer is guaranteed to + // remain valid for the lifetime of the HeapAllocation. + fn ptr(&self) -> Option> { + self.0.as_ref().unwrap().ptr + } + // SAFETY: This function must always return a valid GPU pointer. + // See the explanation in ptr(). + fn gpu_ptr(&self) -> u64 { + self.0.as_ref().unwrap().start() + } + fn device(&self) -> &AsahiDevice { + &self.0.as_ref().unwrap().dev + } +} + +/// Inner data for a heap allocator which uses the DRM MM range allocator to manage the heap. +/// +/// This is wrapped by an `mm::Allocator`. +struct HeapAllocatorInner { + dev: AsahiDevRef, + allocated: usize, + backing_objects: KVec<(crate::gem::ObjectRef, mmu::KernelMapping, u64)>, + garbage: Option>>, + total_garbage: usize, + name: CString, +} + +/// A heap allocator which uses the DRM MM range allocator to manage its objects. +/// +/// The heap is composed of a series of GEM objects. This implementation only ever grows the heap, +/// never shrinks it. +pub(crate) struct HeapAllocator { + dev: AsahiDevRef, + range: Range, + top: u64, + prot: mmu::Prot, + vm: mmu::Vm, + min_align: usize, + block_size: usize, + cpu_maps: bool, + guard_nodes: KVec>, + mm: mm::Allocator, + name: CString, + garbage: Option>>, +} + +impl HeapAllocator { + /// Create a new HeapAllocator for a given `Vm` and address range. + #[allow(dead_code)] + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + dev: &AsahiDevice, + vm: &mmu::Vm, + range: Range, + min_align: usize, + prot: mmu::Prot, + block_size: usize, + mut cpu_maps: bool, + name: fmt::Arguments<'_>, + keep_garbage: bool, + ) -> Result { + if !min_align.is_power_of_two() { + return Err(EINVAL); + } + if debug_enabled(DebugFlags::ForceCPUMaps) { + cpu_maps = true; + } + + let name = CString::try_from_fmt(name)?; + + let inner = HeapAllocatorInner { + dev: dev.into(), + allocated: 0, + backing_objects: KVec::new(), + // TODO: This clearly needs a try_clone() or similar + name: CString::try_from_fmt(fmt!("{}", &*name))?, + garbage: if keep_garbage { + Some(KVec::new()) + } else { + None + }, + total_garbage: 0, + }; + + let mm = mm::Allocator::new(range.start, range.range(), inner)?; + + Ok(HeapAllocator { + dev: dev.into(), + vm: vm.clone(), + top: range.start, + range, + prot, + min_align, + block_size: block_size.max(min_align), + cpu_maps, + guard_nodes: KVec::new(), + mm, + name, + garbage: if keep_garbage { + Some({ + let mut v = KVec::new(); + v.reserve(128, GFP_KERNEL)?; + v + }) + } else { + None + }, + }) + } + + /// Add a new backing block of the given size to this heap. + /// + /// If CPU mapping is enabled, this also adds a guard node to the range allocator to ensure that + /// objects cannot straddle backing block boundaries, since we cannot easily create a contiguous + /// CPU VA mapping for them. This can create some fragmentation. If CPU mapping is disabled, we + /// skip the guard blocks, since the GPU view of the heap is always contiguous. + #[inline(never)] + fn add_block(&mut self, size: usize) -> Result { + let size_aligned = (size + mmu::UAT_PGSZ - 1) & !mmu::UAT_PGMSK; + + mod_dev_dbg!( + &self.dev, + "HeapAllocator[{}]::add_block: size={:#x} size_al={:#x}\n", + &*self.name, + size, + size_aligned, + ); + + if self.top.saturating_add(size_aligned as u64) > self.range.end { + dev_err!( + self.dev.as_ref(), + "HeapAllocator[{}]::add_block: Exhausted VA space\n", + &*self.name, + ); + } + + let mut obj = crate::gem::new_kernel_object(&self.dev, size_aligned)?; + if self.cpu_maps && debug_enabled(DebugFlags::FillAllocations) { + obj.vmap()?.memset(0xde); + } + + let gpu_ptr = self.top; + let mapping = obj + .map_at(&self.vm, gpu_ptr, self.prot, self.cpu_maps) + .inspect_err(|err| { + dev_err!( + self.dev.as_ref(), + "HeapAllocator[{}]::add_block: Failed to map at {:#x} ({:?})\n", + &*self.name, + gpu_ptr, + err + ); + })?; + + if self.cpu_maps { + // Create virtual mapping here ahead of time so that the vmap() in + // alloc_inner() does not take the the object's dma_resv lock while + // the mm lock is locked. mmu::Vm requires the opposite lock order. + obj.vmap()?; + } + + self.mm + .with_inner(|inner| inner.backing_objects.reserve(1, GFP_KERNEL))?; + + let mut new_top = self.top + size_aligned as u64; + if self.cpu_maps { + let guard = self.min_align.max(mmu::UAT_PGSZ); + mod_dev_dbg!( + self.dev, + "HeapAllocator[{}]::add_block: Adding guard node {:#x}:{:#x}\n", + &*self.name, + new_top, + guard + ); + + let inner = HeapAllocationInner { + dev: self.dev.clone(), + ptr: None, + real_size: 0, + }; + + let node = match self.mm.reserve_node(inner, new_top, guard as u64, 0) { + Ok(a) => a, + Err(a) => { + dev_err!( + self.dev.as_ref(), + "HeapAllocator[{}]::add_block: Failed to reserve guard node {:#x}:{:#x}: {:?}\n", + &*self.name, + guard, + new_top, + a + ); + return Err(EIO); + } + }; + + self.guard_nodes.push(node, GFP_KERNEL)?; + + new_top += guard as u64; + } + mod_dev_dbg!( + &self.dev, + "HeapAllocator[{}]::add_block: top={:#x}\n", + &*self.name, + new_top + ); + + self.mm.with_inner(|inner| { + inner + .backing_objects + .push((obj, mapping, gpu_ptr), GFP_KERNEL) + })?; + + self.top = new_top; + + cls_dev_dbg!( + MemStats, + &self.dev, + "{} Heap: grow to {} bytes\n", + &*self.name, + self.top - self.range.start + ); + + Ok(()) + } + + /// Find the backing object index that backs a given GPU address. + fn find_obj(&mut self, addr: u64) -> Result { + self.mm.with_inner(|inner| { + inner + .backing_objects + .binary_search_by(|obj| { + let start = obj.2; + let end = obj.2 + obj.0.size() as u64; + if start > addr { + Ordering::Greater + } else if end <= addr { + Ordering::Less + } else { + Ordering::Equal + } + }) + .or(Err(ENOENT)) + }) + } + + fn alloc_inner(&mut self, size: usize, align: usize) -> Result { + if align != 0 && !align.is_power_of_two() { + return Err(EINVAL); + } + let align = self.min_align.max(align); + let size_aligned = (size + align - 1) & !(align - 1); + + mod_dev_dbg!( + &self.dev, + "HeapAllocator[{}]::new: size={:#x} size_al={:#x}\n", + &*self.name, + size, + size_aligned, + ); + + let inner = HeapAllocationInner { + dev: self.dev.clone(), + ptr: None, + real_size: size, + }; + + let mut node = match self.mm.insert_node_generic( + inner, + size_aligned as u64, + align as u64, + 0, + mm::InsertMode::Best, + ) { + Ok(a) => a, + Err(a) => { + dev_err!( + &self.dev.as_ref(), + "HeapAllocator[{}]::new: Failed to insert node of size {:#x} / align {:#x}: {:?}\n", + &*self.name, size_aligned, align, a + ); + return Err(a); + } + }; + + self.mm.with_inner(|inner| inner.allocated += size); + + let mut new_object = false; + let start = node.start(); + let end = start + node.size(); + if end > self.top { + if start > self.top { + dev_warn!( + self.dev.as_ref(), + "HeapAllocator[{}]::alloc: top={:#x}, start={:#x}\n", + &*self.name, + self.top, + start + ); + } + let block_size = self.block_size.max((end - self.top) as usize); + self.add_block(block_size)?; + new_object = true; + } + assert!(end <= self.top); + + if self.cpu_maps { + mod_dev_dbg!( + self.dev, + "HeapAllocator[{}]::alloc: mapping to CPU\n", + &*self.name + ); + + let idx = if new_object { + None + } else { + Some(match self.find_obj(start) { + Ok(a) => a, + Err(_) => { + dev_warn!( + self.dev.as_ref(), + "HeapAllocator[{}]::alloc: Failed to find object at {:#x}\n", + &*self.name, + start + ); + return Err(EIO); + } + }) + }; + let (obj_start, obj_size, p) = self.mm.with_inner(|inner| -> Result<_> { + let idx = idx.unwrap_or(inner.backing_objects.len() - 1); + let obj = &mut inner.backing_objects[idx]; + let p = obj.0.vmap()?.as_mut_ptr() as *mut u8; + Ok((obj.2, obj.0.size(), p)) + })?; + assert!(obj_start <= start); + assert!(obj_start + obj_size as u64 >= end); + node.as_mut().inner_mut().ptr = + // SAFETY: Per the asserts above, this offset is always within the allocation. + NonNull::new(unsafe { p.add((start - obj_start) as usize) }); + mod_dev_dbg!( + self.dev, + "HeapAllocator[{}]::alloc: CPU pointer = {:?}\n", + &*self.name, + node.ptr + ); + } + + mod_dev_dbg!( + self.dev, + "HeapAllocator[{}]::alloc: Allocated {:#x} bytes @ {:#x}\n", + &*self.name, + end - start, + start + ); + + Ok(HeapAllocation(Some(node))) + } +} + +impl Allocator for HeapAllocator { + type Raw = HeapAllocation; + + fn cpu_maps(&self) -> bool { + self.cpu_maps + } + + fn min_align(&self) -> usize { + self.min_align + } + + fn alloc(&mut self, size: usize, align: usize) -> Result { + let ret = self.alloc_inner(size, align); + + if ret.is_err() { + dev_warn!( + self.dev.as_ref(), + "HeapAllocator[{}]::alloc: Allocation of {:#x}({:#x}) size object failed\n", + &*self.name, + size, + align + ); + } + ret + } + + fn garbage(&self) -> (usize, usize) { + self.mm.with_inner(|inner| { + if let Some(g) = inner.garbage.as_ref() { + (g.len(), inner.total_garbage) + } else { + (0, 0) + } + }) + } + + fn collect_garbage(&mut self, mut count: usize) { + if let Some(garbage) = self.garbage.as_mut() { + garbage.clear(); + + while count > 0 { + let block = count.min(garbage.capacity()); + assert!(block > 0); + + // Take the garbage out of the inner block, so we can safely drop it without deadlocking + self.mm.with_inner(|inner| { + if let Some(g) = inner.garbage.as_mut() { + for node in g.drain(0..block) { + inner.total_garbage -= node.size() as usize; + garbage + .push(node, GFP_KERNEL) + .expect("push() failed after reserve()"); + } + } + }); + + count -= block; + // Now drop it + garbage.clear(); + } + } + } +} + +impl Drop for HeapAllocatorInner { + fn drop(&mut self) { + mod_dev_dbg!( + self.dev, + "HeapAllocator[{}]: dropping allocator\n", + &*self.name + ); + if self.allocated > 0 { + // This should never happen + dev_crit!( + self.dev.as_ref(), + "HeapAllocator[{}]: dropping with {} bytes allocated\n", + &*self.name, + self.allocated + ); + } + } +} diff --git a/drivers/gpu/drm/asahi/asahi.rs b/drivers/gpu/drm/asahi/asahi.rs new file mode 100644 index 00000000000000..296091dced554d --- /dev/null +++ b/drivers/gpu/drm/asahi/asahi.rs @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![recursion_limit = "2048"] + +//! Driver for the Apple AGX GPUs found in Apple Silicon SoCs. + +mod alloc; +mod buffer; +mod channel; +#[cfg(CONFIG_DEV_COREDUMP)] +mod crashdump; +mod debug; +mod driver; +mod event; +mod file; +mod float; +mod fw; +mod gem; +mod gpu; +mod hw; +mod initdata; +mod mem; +mod microseq; +mod mmu; +mod object; +mod pgtable; +mod queue; +mod regs; +mod slotalloc; +mod util; +#[cfg(CONFIG_DRM_ASAHI_MAPLE_TREE)] +mod vm; +mod workqueue; + +kernel::module_platform_driver! { + type: driver::AsahiDriver, + name: "asahi", + description: "AGX GPU driver for Apple silicon SoCs", + license: "Dual MIT/GPL", + params: { + debug_flags: u64 { + default: 0, + // permissions: 0o644, + description: "Debug flags", + }, + fault_control: u32 { + default: 0xb, + // permissions: 0, + description: "Fault control (0x0: hard faults, 0xb: macOS default)", + }, + initial_tvb_size: usize { + default: 0x8, + // permissions: 0o644, + description: "Initial TVB size in blocks", + }, + robust_isolation: u32 { + default: 0, + // permissions: 0o644, + description: "Fully isolate GPU contexts (limits performance)", + }, + }, +} diff --git a/drivers/gpu/drm/asahi/buffer.rs b/drivers/gpu/drm/asahi/buffer.rs new file mode 100644 index 00000000000000..309482441062d9 --- /dev/null +++ b/drivers/gpu/drm/asahi/buffer.rs @@ -0,0 +1,809 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Tiled Vertex Buffer management +//! +//! This module manages the Tiled Vertex Buffer, also known as the Parameter Buffer (in imgtec +//! parlance) or the tiler heap (on other architectures). This buffer holds transformed primitive +//! data between the vertex/tiling stage and the fragment stage. +//! +//! On AGX, the buffer is a heap of 128K blocks split into 32K pages (which must be aligned to a +//! multiple of 32K in VA space). The buffer can be shared between multiple render jobs, and each +//! will allocate pages from it during vertex processing and return them during fragment processing. +//! +//! If the buffer runs out of free pages, the vertex pass stops and a partial fragment pass occurs, +//! spilling the intermediate render target state to RAM (a partial render). This is all managed +//! transparently by the firmware. Since partial renders are less efficient, the kernel must grow +//! the heap in response to feedback from the firmware to avoid partial renders in the future. +//! Currently, we only ever grow the heap, and never shrink it. +//! +//! AGX also supports memoryless render targets, which can be used for intermediate results within +//! a render pass. To support partial renders, it seems the GPU/firmware has the ability to borrow +//! pages from the TVB buffer as a temporary render target buffer. Since this happens during a +//! partial render itself, if the buffer runs out of space, it requires synchronous growth in +//! response to a firmware interrupt. This is not currently supported, but may be in the future, +//! though it is unclear whether it is worth the effort. +//! +//! This module is also in charge of managing the temporary objects associated with a single render +//! pass, which includes the top-level tile array, the tail pointer cache, preemption buffers, and +//! other miscellaneous structures collectively managed as a "scene". +//! +//! To avoid runaway memory usage, there is a maximum size for buffers (at that point it's unlikely +//! that partial renders will incur much overhead over the buffer data access itself). This is +//! different depending on whether memoryless render targets are in use, and is currently hardcoded. +//! to the most common value used by macOS. + +use crate::debug::*; +use crate::fw::buffer; +use crate::fw::types::*; +use crate::util::*; +use crate::{ + alloc, + fw, + gpu, + hw, + mmu, + slotalloc, // +}; +use core::sync::atomic::Ordering; +use kernel::new_mutex; +use kernel::prelude::*; +use kernel::sync::{ + Arc, + Mutex, // +}; +use kernel::{ + c_str, + static_lock_class, // +}; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Buffer; + +/// There are 127 GPU/firmware-side buffer manager slots (yes, 127, not 128). +const NUM_BUFFERS: u32 = 127; + +/// Page size bits for buffer pages (32K). VAs must be aligned to this size. +pub(crate) const PAGE_SHIFT: usize = 15; +/// Page size for buffer pages. +pub(crate) const PAGE_SIZE: usize = 1 << PAGE_SHIFT; +/// Number of pages in a buffer block, which should be contiguous in VA space. +pub(crate) const PAGES_PER_BLOCK: usize = 4; +/// Size of a buffer block. +pub(crate) const BLOCK_SIZE: usize = PAGE_SIZE * PAGES_PER_BLOCK; + +/// Metadata about the tiling configuration for a scene. This is computed in the `render` module. +/// based on dimensions, tile size, and other info. +pub(crate) struct TileInfo { + /// Tile count in the X dimension. Tiles are always 32x32. + pub(crate) tiles_x: u32, + /// Tile count in the Y dimension. Tiles are always 32x32. + pub(crate) tiles_y: u32, + /// Total tile count. + pub(crate) tiles: u32, + /// Micro-tile width (16 or 32). + pub(crate) utile_width: u32, + /// Micro-tile height (16 or 32). + pub(crate) utile_height: u32, + // Macro-tiles in the X dimension. Always 4. + //pub(crate) mtiles_x: u32, + // Macro-tiles in the Y dimension. Always 4. + //pub(crate) mtiles_y: u32, + /// Tiles per macro-tile in the X dimension. + pub(crate) tiles_per_mtile_x: u32, + /// Tiles per macro-tile in the Y dimension. + pub(crate) tiles_per_mtile_y: u32, + // Total tiles per macro-tile. + //pub(crate) tiles_per_mtile: u32, + /// Micro-tiles per macro-tile in the X dimension. + pub(crate) utiles_per_mtile_x: u32, + /// Micro-tiles per macro-tile in the Y dimension. + pub(crate) utiles_per_mtile_y: u32, + // Total micro-tiles per macro-tile. + //pub(crate) utiles_per_mtile: u32, + /// Size of the top-level tilemap, in bytes (for all layers, one cluster). + pub(crate) tilemap_size: usize, + /// Size of the Tail Pointer Cache, in bytes (for all layers * clusters). + pub(crate) tpc_size: usize, + /// Number of blocks in the clustering meta buffer (for clustering) per layer. + pub(crate) meta1_layer_stride: u32, + /// Number of blocks in the clustering meta buffer (for clustering). + pub(crate) meta1_blocks: u32, + /// Layering metadata size. + pub(crate) layermeta_size: usize, + /// Minimum number of TVB blocks for this render. + pub(crate) min_tvb_blocks: usize, + /// Tiling parameter structure passed to firmware. + pub(crate) params: fw::vertex::raw::TilingParameters, +} + +/// A single scene, representing a render pass and its required buffers. +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct Scene { + object: GpuObject, + slot: u32, + rebind: bool, + preempt2_off: usize, + preempt3_off: usize, + // Note: these are dead code only on some version variants. + // It's easier to do this than to propagate the version conditionals everywhere. + #[allow(dead_code)] + meta1_off: usize, + #[allow(dead_code)] + meta2_off: usize, + #[allow(dead_code)] + meta3_off: usize, + #[allow(dead_code)] + meta4_off: usize, +} + +#[versions(AGX)] +impl Scene::ver { + /// Returns true if the buffer was bound to a fresh manager slot, and therefore needs an init + /// command before a render. + pub(crate) fn rebind(&self) -> bool { + self.rebind + } + + /// Returns the buffer manager slot this scene's buffer was bound to. + pub(crate) fn slot(&self) -> u32 { + self.slot + } + + /// Returns the GPU pointer to the [`buffer::Scene::ver`]. + pub(crate) fn gpu_pointer(&self) -> GpuPointer<'_, buffer::Scene::ver> { + self.object.gpu_pointer() + } + + /// Returns the GPU weak pointer to the [`buffer::Scene::ver`]. + pub(crate) fn weak_pointer(&self) -> GpuWeakPointer { + self.object.weak_pointer() + } + + /// Returns the GPU weak pointer to the kernel-side temp buffer. + /// (purpose unknown...) + pub(crate) fn kernel_buffer_pointer(&self) -> GpuWeakPointer<[u8]> { + self.object.buffer.inner.lock().kernel_buffer.weak_pointer() + } + + /// Returns the GPU pointer to the `buffer::Info::ver` object associated with this Scene. + pub(crate) fn buffer_pointer(&self) -> GpuPointer<'_, buffer::Info::ver> { + // SAFETY: We can't return the strong pointer directly since its lifetime crosses a lock, + // but we know its lifetime will be valid as long as &self since we hold a reference to the + // buffer, so just construct the strong pointer with the right lifetime here. + unsafe { self.weak_buffer_pointer().upgrade() } + } + + /// Returns the GPU weak pointer to the `buffer::Info::ver` object associated with this Scene. + pub(crate) fn weak_buffer_pointer(&self) -> GpuWeakPointer { + self.object.buffer.inner.lock().info.weak_pointer() + } + + /// Returns the GPU pointer to the TVB heap metadata buffer. + pub(crate) fn tvb_heapmeta_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { + self.object.tvb_heapmeta.gpu_pointer() + } + + /// Returns the GPU pointer to the layer metadata buffer. + pub(crate) fn tvb_layermeta_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { + self.object.tvb_heapmeta.gpu_offset_pointer(0x200) + } + + /// Returns the GPU pointer to the top-level TVB tilemap buffer. + pub(crate) fn tvb_tilemap_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { + self.object.tvb_tilemap.gpu_pointer() + } + + /// Returns the GPU pointer to the Tail Pointer Cache buffer. + pub(crate) fn tpc_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { + self.object.tpc.gpu_pointer() + } + + /// Returns the GPU pointer to the first preemption scratch buffer. + pub(crate) fn preempt_buf_1_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { + self.object.preempt_buf.gpu_pointer() + } + + /// Returns the GPU pointer to the second preemption scratch buffer. + pub(crate) fn preempt_buf_2_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { + self.object + .preempt_buf + .gpu_offset_pointer(self.preempt2_off) + } + + /// Returns the GPU pointer to the third preemption scratch buffer. + pub(crate) fn preempt_buf_3_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { + self.object + .preempt_buf + .gpu_offset_pointer(self.preempt3_off) + } + + /// Returns the GPU pointer to the per-cluster tilemap buffer, if clustering is enabled. + #[allow(dead_code)] + pub(crate) fn cluster_tilemaps_pointer(&self) -> Option> { + self.object + .clustering + .as_ref() + .map(|c| c.tilemaps.gpu_pointer()) + } + + /// Returns the GPU pointer to the clustering layer metadata buffer, if clustering is enabled. + #[allow(dead_code)] + pub(crate) fn tvb_cluster_layermeta_pointer(&self) -> Option> { + self.object + .clustering + .as_ref() + .map(|c| c.meta.gpu_pointer()) + } + + /// Returns the GPU pointer to the clustering metadata 1 buffer, if clustering is enabled. + #[allow(dead_code)] + pub(crate) fn meta_1_pointer(&self) -> Option> { + self.object + .clustering + .as_ref() + .map(|c| c.meta.gpu_offset_pointer(self.meta1_off)) + } + + /// Returns the GPU pointer to the clustering metadata 2 buffer, if clustering is enabled. + #[allow(dead_code)] + pub(crate) fn meta_2_pointer(&self) -> Option> { + self.object + .clustering + .as_ref() + .map(|c| c.meta.gpu_offset_pointer(self.meta2_off)) + } + + /// Returns the GPU pointer to the clustering metadata 3 buffer, if clustering is enabled. + #[allow(dead_code)] + pub(crate) fn meta_3_pointer(&self) -> Option> { + self.object + .clustering + .as_ref() + .map(|c| c.meta.gpu_offset_pointer(self.meta3_off)) + } + + /// Returns the GPU pointer to the clustering metadata 4 buffer, if clustering is enabled. + #[allow(dead_code)] + pub(crate) fn meta_4_pointer(&self) -> Option> { + self.object + .clustering + .as_ref() + .map(|c| c.meta.gpu_offset_pointer(self.meta4_off)) + } +} + +#[versions(AGX)] +impl Drop for Scene::ver { + fn drop(&mut self) { + let mut inner = self.object.buffer.inner.lock(); + assert_ne!(inner.active_scenes, 0); + inner.active_scenes -= 1; + + if inner.active_scenes == 0 { + mod_pr_debug!( + "Buffer: no scenes left, dropping slot {}", + inner.active_slot.take().unwrap().slot() + ); + inner.active_slot = None; + } + } +} + +/// Inner data for a single TVB buffer object. +#[versions(AGX)] +struct BufferInner { + info: GpuObject, + ualloc: Arc>, + ualloc_priv: Arc>, + blocks: KVec>, + max_blocks: usize, + max_blocks_nomemless: usize, + mgr: BufferManager::ver, + active_scenes: usize, + active_slot: Option>, + last_token: Option, + tpc: Option>>, + kernel_buffer: GpuArray, + stats: GpuObject, + cfg: &'static hw::HwConfig, + preempt1_size: usize, + preempt2_size: usize, + preempt3_size: usize, + num_clusters: usize, +} + +/// Locked and reference counted TVB buffer. +#[versions(AGX)] +pub(crate) struct Buffer { + inner: Arc>, +} + +#[versions(AGX)] +impl Buffer::ver { + /// Create a new Buffer for a given VM, given the per-VM allocators. + pub(crate) fn new( + gpu: &dyn gpu::GpuManager, + alloc: &mut gpu::KernelAllocators, + ualloc: Arc>, + ualloc_priv: Arc>, + mgr: &BufferManager::ver, + ) -> Result { + // These are the typical max numbers on macOS. + // 8GB machines have this halved. + let max_size: usize = 862_322_688; // bytes + let max_size_nomemless = max_size / 3; + + let max_blocks = max_size / BLOCK_SIZE; + let max_blocks_nomemless = max_size_nomemless / BLOCK_SIZE; + let max_pages = max_blocks * PAGES_PER_BLOCK; + let max_pages_nomemless = max_blocks_nomemless * PAGES_PER_BLOCK; + + let num_clusters = gpu.get_dyncfg().id.num_clusters as usize; + let num_clusters_adj = if num_clusters > 1 { + num_clusters + 1 + } else { + 1 + }; + + let preempt1_size = num_clusters_adj * gpu.get_cfg().preempt1_size; + let preempt2_size = num_clusters_adj * gpu.get_cfg().preempt2_size; + let preempt3_size = num_clusters_adj * gpu.get_cfg().preempt3_size; + + let shared = &mut alloc.shared; + let info = alloc.private.new_init( + { + let ualloc_priv = &ualloc_priv; + try_init!(buffer::Info::ver { + block_ctl: shared.new_default::()?, + counter: shared.new_default::()?, + page_list: ualloc_priv.lock().array_empty_tagged(max_pages, b"PLST")?, + block_list: ualloc_priv + .lock() + .array_empty_tagged(max_blocks * 2, b"BLST")?, + }) + }, + |inner, _p| { + try_init!(buffer::raw::Info::ver { + gpu_counter: 0x0, + unk_4: 0, + last_id: 0x0, + cur_id: -1, + unk_10: 0x0, + gpu_counter2: 0x0, + unk_18: 0x0, + #[ver(V < V13_0B4 || G >= G14X)] + unk_1c: 0x0, + page_list: inner.page_list.gpu_pointer(), + page_list_size: (4 * max_pages).try_into()?, + page_count: AtomicU32::new(0), + max_blocks: max_blocks.try_into()?, + block_count: AtomicU32::new(0), + unk_38: 0x0, + block_list: inner.block_list.gpu_pointer(), + block_ctl: inner.block_ctl.gpu_pointer(), + last_page: AtomicU32::new(0), + gpu_page_ptr1: 0x0, + gpu_page_ptr2: 0x0, + unk_58: 0x0, + block_size: BLOCK_SIZE as u32, + unk_60: U64(0x0), + counter: inner.counter.gpu_pointer(), + unk_70: 0x0, + unk_74: 0x0, + unk_78: 0x0, + unk_7c: 0x0, + unk_80: 0x1, + max_pages: max_pages.try_into()?, + max_pages_nomemless: max_pages_nomemless.try_into()?, + unk_8c: 0x0, + unk_90: Default::default(), + }) + }, + )?; + + // Technically similar to Scene below, let's play it safe. + let kernel_buffer = alloc.shared.array_empty_tagged(0x40, b"KBUF")?; + let stats = alloc + .shared + .new_object(Default::default(), |_inner| buffer::raw::Stats { + reset: AtomicU32::from(1), + ..Default::default() + })?; + + Ok(Buffer::ver { + inner: Arc::pin_init( + new_mutex!(BufferInner::ver { + info, + ualloc, + ualloc_priv, + blocks: KVec::new(), + max_blocks, + max_blocks_nomemless, + mgr: mgr.clone(), + active_scenes: 0, + active_slot: None, + last_token: None, + tpc: None, + kernel_buffer, + stats, + cfg: gpu.get_cfg(), + preempt1_size, + preempt2_size, + preempt3_size, + num_clusters, + }), + GFP_KERNEL, + )?, + }) + } + + /// Returns the total block count allocated to this Buffer. + pub(crate) fn block_count(&self) -> u32 { + self.inner.lock().blocks.len() as u32 + } + + /// Automatically grow the Buffer based on feedback from the statistics. + pub(crate) fn auto_grow(&self) -> Result { + let inner = self.inner.lock(); + + let used_pages = inner.stats.with(|raw, _inner| { + let used = raw.max_pages.load(Ordering::Relaxed); + raw.reset.store(1, Ordering::Release); + used as usize + }); + + let need_blocks = (used_pages * 2) + .div_ceil(PAGES_PER_BLOCK) + .min(inner.max_blocks_nomemless); + let want_blocks = (used_pages * 3) + .div_ceil(PAGES_PER_BLOCK) + .min(inner.max_blocks_nomemless); + + let cur_count = inner.blocks.len(); + + if need_blocks <= cur_count { + Ok(false) + } else { + // Grow to 3x requested size (same logic as macOS) + core::mem::drop(inner); + self.ensure_blocks(want_blocks)?; + Ok(true) + } + } + + /// Synchronously grow the Buffer. + pub(crate) fn sync_grow(&self) { + let inner = self.inner.lock(); + + let cur_count = inner.blocks.len(); + core::mem::drop(inner); + if self.ensure_blocks(cur_count + 10).is_err() { + pr_err!("BufferManager: Failed to grow buffer synchronously\n"); + } + } + + /// Ensure that the buffer has at least a certain minimum size in blocks. + pub(crate) fn ensure_blocks(&self, min_blocks: usize) -> Result { + let mut inner = self.inner.lock(); + + let cur_count = inner.blocks.len(); + if cur_count >= min_blocks { + return Ok(false); + } + if min_blocks > inner.max_blocks { + return Err(ENOMEM); + } + + let add_blocks = min_blocks - cur_count; + let new_count = min_blocks; + + let mut new_blocks: KVec> = KVec::new(); + + // Allocate the new blocks first, so if it fails they will be dropped + let mut ualloc = inner.ualloc.lock(); + for _i in 0..add_blocks { + new_blocks.push(ualloc.array_gpuonly(BLOCK_SIZE)?, GFP_KERNEL)?; + } + core::mem::drop(ualloc); + + // Then actually commit them + inner.blocks.reserve(add_blocks, GFP_KERNEL)?; + + for (i, block) in new_blocks.into_iter().enumerate() { + let page_num = (block.gpu_va().get() >> PAGE_SHIFT) as u32; + + inner + .blocks + .push(block, GFP_KERNEL) + .expect("push() failed after reserve()"); + inner.info.block_list[2 * (cur_count + i)] = page_num; + for j in 0..PAGES_PER_BLOCK { + inner.info.page_list[(cur_count + i) * PAGES_PER_BLOCK + j] = page_num + j as u32; + } + } + + inner.info.block_ctl.with(|raw, _inner| { + raw.total.store(new_count as u32, Ordering::SeqCst); + raw.wptr.store(new_count as u32, Ordering::SeqCst); + }); + + /* Only do this update if the buffer manager is idle (which means we own it) */ + if inner.active_scenes == 0 { + let page_count = (new_count * PAGES_PER_BLOCK) as u32; + inner.info.with(|raw, _inner| { + raw.page_count.store(page_count, Ordering::Relaxed); + raw.block_count.store(new_count as u32, Ordering::Relaxed); + raw.last_page.store(page_count - 1, Ordering::Relaxed); + }); + } + + Ok(true) + } + + /// Create a new [`Scene::ver`] (render pass) using this buffer. + pub(crate) fn new_scene( + &self, + alloc: &mut gpu::KernelAllocators, + tile_info: &TileInfo, + ) -> Result { + let mut inner = self.inner.lock(); + + let tilemap_size = tile_info.tilemap_size; + let tpc_size = tile_info.tpc_size; + + // TODO: what is this exactly? + mod_pr_debug!("Buffer: Allocating TVB buffers\n"); + + // This seems to be a list, with 4x2 bytes of headers and 8 bytes per entry. + // On single-cluster devices, the used length always seems to be 1. + // On M1 Ultra, it can grow and usually doesn't exceed 64 entries. + // macOS allocates a whole 64K * 0x80 for this, so let's go with + // that to be safe... + let user_buffer = inner.ualloc.lock().array_empty_tagged( + if inner.num_clusters > 1 { + 0x10080 + } else { + 0x80 + }, + b"UBUF", + )?; + + let tvb_heapmeta = inner + .ualloc + .lock() + .array_empty_tagged(0x200 + tile_info.layermeta_size, b"HMTA")?; + let tvb_tilemap = inner + .ualloc + .lock() + .array_empty_tagged(tilemap_size, b"TMAP")?; + + mod_pr_debug!("Buffer: Allocating misc buffers\n"); + let preempt_buf = inner.ualloc.lock().array_empty_tagged( + inner.preempt1_size + inner.preempt2_size + inner.preempt3_size, + b"PRMT", + )?; + + let tpc = match inner.tpc.as_ref() { + Some(buf) if buf.len() >= tpc_size => buf.clone(), + _ => { + // MacOS allocates this as shared GPU+FW, but + // priv seems to work and might be faster? + // Needs to be FW-writable anyway, so ualloc + // won't work. + let buf = Arc::new( + inner.ualloc_priv.lock().array_empty_tagged( + (tpc_size + mmu::UAT_PGMSK) & !mmu::UAT_PGMSK, + b"TPC ", + )?, + GFP_KERNEL, + )?; + inner.tpc = Some(buf.clone()); + buf + } + }; + + let mut clmeta_size = 0; + let mut meta1_size = 0; + let mut meta2_size = 0; + let mut meta3_size = 0; + + let clustering = if inner.num_clusters > 1 { + let cfg = inner.cfg.clustering.as_ref().unwrap(); + + clmeta_size = tile_info.layermeta_size * cfg.max_splits; + // Maybe: (4x4 macro tiles + 1 global page)*n, 32bit each (17*4*n) + // Unused on t602x? + meta1_size = align(tile_info.meta1_blocks as usize * cfg.meta1_blocksize, 0x80); + meta2_size = align(cfg.meta2_size, 0x80); + meta3_size = align(cfg.meta3_size, 0x80); + let meta4_size = cfg.meta4_size; + + let meta_size = clmeta_size + meta1_size + meta2_size + meta3_size + meta4_size; + + mod_pr_debug!("Buffer: Allocating clustering buffers\n"); + let tilemaps = inner + .ualloc + .lock() + .array_empty_tagged(cfg.max_splits * tilemap_size, b"CTMP")?; + let meta = inner.ualloc.lock().array_empty_tagged(meta_size, b"CMTA")?; + Some(buffer::ClusterBuffers { tilemaps, meta }) + } else { + None + }; + + // Could be made strong, but we wind up with a deadlock if we try to grab the + // pointer through the inner.buffer path inside the closure. + let stats_pointer = inner.stats.weak_pointer(); + + let _gpu = &mut alloc.gpu; + + // macOS allocates this as private. However, the firmware does not + // DC CIVAC this before reading it (like it does most other things), + // which causes odd cache incoherency bugs when combined with + // speculation on the firmware side (maybe). This doesn't happen + // on macOS because these structs are a circular pool that is mapped + // already initialized. Just mark this shared for now. + let scene = alloc.shared.new_init( + try_init!(buffer::Scene::ver { + user_buffer: user_buffer, + buffer: self.clone(), + tvb_heapmeta: tvb_heapmeta, + tvb_tilemap: tvb_tilemap, + tpc: tpc, + clustering: clustering, + preempt_buf: preempt_buf, + #[ver(G >= G14X)] + control_word: _gpu.array_empty_tagged(1, b"CWRD")?, + }), + |inner, _p| { + try_init!(buffer::raw::Scene::ver { + #[ver(G >= G14X)] + control_word: inner.control_word.gpu_pointer(), + #[ver(G >= G14X)] + control_word2: inner.control_word.gpu_pointer(), + pass_page_count: AtomicU32::new(0), + unk_4: 0, + unk_8: U64(0), + unk_10: U64(0), + user_buffer: inner.user_buffer.gpu_pointer(), + unk_20: 0, + #[ver(V >= V13_3)] + unk_28: U64(0), + stats: stats_pointer, + total_page_count: AtomicU32::new(0), + #[ver(G < G14X)] + unk_30: U64(0), + #[ver(G < G14X)] + unk_38: U64(0), + }) + }, + )?; + + let mut rebind = false; + + if inner.active_slot.is_none() { + assert_eq!(inner.active_scenes, 0); + + let slot = inner.mgr.0.get_inner(inner.last_token, |inner, mgr| { + inner.owners[mgr.slot() as usize] = Some(self.clone()); + Ok(()) + })?; + rebind = slot.changed(); + + mod_pr_debug!("Buffer: assigning slot {} (rebind={})", slot.slot(), rebind); + + inner.last_token = Some(slot.token()); + inner.active_slot = Some(slot); + } + + inner.active_scenes += 1; + + Ok(Scene::ver { + object: scene, + slot: inner.active_slot.as_ref().unwrap().slot(), + rebind, + preempt2_off: inner.preempt1_size, + preempt3_off: inner.preempt1_size + inner.preempt2_size, + meta1_off: clmeta_size, + meta2_off: clmeta_size + meta1_size, + meta3_off: clmeta_size + meta1_size + meta2_size, + meta4_off: clmeta_size + meta1_size + meta2_size + meta3_size, + }) + } + + /// Increment the buffer manager usage count. Should we done once we know the Scene is ready + /// to be committed and used in commands submitted to the GPU. + pub(crate) fn increment(&self) { + let inner = self.inner.lock(); + inner.info.counter.with(|raw, _inner| { + // We could use fetch_add, but the non-LSE atomic + // sequence Rust produces confuses the hypervisor. + // We have inner locked anyway, so this is not racy. + let v = raw.count.load(Ordering::Relaxed); + raw.count.store(v + 1, Ordering::Relaxed); + }); + } + + pub(crate) fn any_ref(&self) -> Arc { + self.inner.clone() + } +} + +#[versions(AGX)] +impl Clone for Buffer::ver { + fn clone(&self) -> Self { + Buffer::ver { + inner: self.inner.clone(), + } + } +} + +#[versions(AGX)] +struct BufferSlotInner(); + +#[versions(AGX)] +impl slotalloc::SlotItem for BufferSlotInner::ver { + type Data = BufferManagerInner::ver; + + fn release(&mut self, data: &mut Self::Data, slot: u32) { + mod_pr_debug!("BufferManager: Released slot {}\n", slot); + data.owners[slot as usize] = None; + } +} + +/// Inner data for the buffer manager, to be protected by the SlotAllocator lock. +#[versions(AGX)] +pub(crate) struct BufferManagerInner { + owners: KVec>, +} + +/// The GPU-global buffer manager, used to allocate and release buffer slots from the pool. +#[versions(AGX)] +pub(crate) struct BufferManager(slotalloc::SlotAllocator); + +#[versions(AGX)] +impl BufferManager::ver { + pub(crate) fn new() -> Result { + let mut owners = KVec::new(); + for _i in 0..(NUM_BUFFERS as usize) { + owners.push(None, GFP_KERNEL)?; + } + Ok(BufferManager::ver(slotalloc::SlotAllocator::new( + NUM_BUFFERS, + BufferManagerInner::ver { owners }, + |_inner, _slot| Some(BufferSlotInner::ver()), + c_str!("BufferManager::SlotAllocator"), + static_lock_class!(), + static_lock_class!(), + )?)) + } + + /// Signals a Buffer to synchronously grow. + pub(crate) fn grow(&self, slot: u32) { + match self + .0 + .with_inner(|inner| inner.owners[slot as usize].as_ref().cloned()) + { + Some(owner) => { + pr_err!( + "BufferManager: Unexpected grow request for slot {}. This might deadlock. Please report this bug.\n", + slot + ); + owner.sync_grow(); + } + None => { + pr_err!( + "BufferManager: Received grow request for empty slot {}\n", + slot + ); + } + } + } +} + +#[versions(AGX)] +impl Clone for BufferManager::ver { + fn clone(&self) -> Self { + BufferManager::ver(self.0.clone()) + } +} diff --git a/drivers/gpu/drm/asahi/channel.rs b/drivers/gpu/drm/asahi/channel.rs new file mode 100644 index 00000000000000..30cc0efbf3ce22 --- /dev/null +++ b/drivers/gpu/drm/asahi/channel.rs @@ -0,0 +1,631 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU ring buffer channels +//! +//! The GPU firmware use a set of ring buffer channels to receive commands from the driver and send +//! it notifications and status messages. +//! +//! These ring buffers mostly follow uniform conventions, so they share the same base +//! implementation. + +use crate::debug::*; +use crate::driver::{ + AsahiDevRef, + AsahiDevice, // +}; +use crate::fw::channels::*; +use crate::fw::initdata::{ + raw, + ChannelRing, // +}; +use crate::fw::types::*; +use crate::{ + buffer, + event, + gpu, + mem, // +}; +use kernel::{ + c_str, + prelude::*, + sync::Arc, + time::{ + delay::fsleep, + Delta, + Instant, + Monotonic, // + }, +}; + +pub(crate) use crate::fw::channels::PipeType; + +/// A receive (FW->driver) channel. +pub(crate) struct RxChannel +where + for<'a> ::Raw<'a>: Debug + Default + Zeroable, +{ + ring: ChannelRing, + // FIXME: needs feature(generic_const_exprs) + //rptr: [u32; T::SUB_CHANNELS], + rptr: [u32; 6], + count: u32, +} + +impl RxChannel +where + for<'a> ::Raw<'a>: Debug + Default + Zeroable, +{ + /// Allocates a new receive channel with a given message count. + pub(crate) fn new(alloc: &mut gpu::KernelAllocators, count: usize) -> Result> { + Ok(RxChannel { + ring: ChannelRing { + state: alloc.shared.new_default()?, + ring: alloc.shared.array_empty(T::SUB_CHANNELS * count)?, + }, + rptr: Default::default(), + count: count as u32, + }) + } + + /// Receives a message on the specified sub-channel index, optionally leaving in the ring + /// buffer. + /// + /// Returns None if the channel is empty. + fn get_or_peek(&mut self, index: usize, peek: bool) -> Option { + self.ring.state.with(|raw, _inner| { + let wptr = T::wptr(raw, index); + let rptr = &mut self.rptr[index]; + if wptr == *rptr { + None + } else { + let off = self.count as usize * index; + let msg = self.ring.ring[off + *rptr as usize]; + if !peek { + *rptr = (*rptr + 1) % self.count; + T::set_rptr(raw, index, *rptr); + } + Some(msg) + } + }) + } + + /// Receives a message on the specified sub-channel index, and dequeues it from the ring buffer. + /// + /// Returns None if the channel is empty. + pub(crate) fn get(&mut self, index: usize) -> Option { + self.get_or_peek(index, false) + } + + /// Peeks a message on the specified sub-channel index, leaving it in the ring buffer. + /// + /// Returns None if the channel is empty. + pub(crate) fn peek(&mut self, index: usize) -> Option { + self.get_or_peek(index, true) + } +} + +/// A transmit (driver->FW) channel. +pub(crate) struct TxChannel +where + for<'a> ::Raw<'a>: Debug + Default + Zeroable, +{ + ring: ChannelRing, + wptr: u32, + count: u32, +} + +impl TxChannel +where + for<'a> ::Raw<'a>: Debug + Default + Zeroable, +{ + /// Allocates a new cached transmit channel with a given message count. + pub(crate) fn new(alloc: &mut gpu::KernelAllocators, count: usize) -> Result> { + Ok(TxChannel { + ring: ChannelRing { + state: alloc.shared.new_default()?, + ring: alloc.private.array_empty(count)?, + }, + wptr: 0, + count: count as u32, + }) + } + + /// Allocates a new uncached transmit channel with a given message count. + pub(crate) fn new_uncached( + alloc: &mut gpu::KernelAllocators, + count: usize, + ) -> Result> { + Ok(TxChannel { + ring: ChannelRing { + state: alloc.shared.new_default()?, + ring: alloc.shared.array_empty(count)?, + }, + wptr: 0, + count: count as u32, + }) + } + + /// Send a message to the ring, returning a cookie with the ring buffer position. + /// + /// This will poll/block if the ring is full, which we don't really expect to happen. + pub(crate) fn put(&mut self, msg: &U) -> u32 { + self.ring.state.with(|raw, _inner| { + let next_wptr = (self.wptr + 1) % self.count; + let mut rptr = T::rptr(raw); + if next_wptr == rptr { + pr_err!( + "TX ring buffer is full! Waiting... ({}, {})\n", + next_wptr, + rptr + ); + // TODO: block properly on incoming messages? + while next_wptr == rptr { + fsleep(Delta::from_millis(8)); + rptr = T::rptr(raw); + } + } + self.ring.ring[self.wptr as usize] = *msg; + mem::sync(); + T::set_wptr(raw, next_wptr); + self.wptr = next_wptr; + }); + self.wptr + } + + /// Wait for a previously submitted message to be popped off of the ring by the GPU firmware. + /// + /// This busy-loops, and is intended to be used for rare cases when we need to block for + /// completion of a cache management or invalidation operation synchronously (which + /// the firmware normally completes fast enough not to be worth sleeping for). + /// If the poll takes longer than 10ms, this switches to sleeping between polls. + pub(crate) fn wait_for(&mut self, wptr: u32, timeout_ms: i64) -> Result { + const MAX_FAST_POLL: i64 = 10; + let start = Instant::::now(); + let timeout_ms = timeout_ms.max(1); + let timeout_fast = Delta::from_millis(timeout_ms.min(MAX_FAST_POLL)); + let timeout_slow = Delta::from_millis(timeout_ms); + self.ring.state.with(|raw, _inner| { + while start.elapsed() < timeout_fast { + if T::rptr(raw) == wptr { + return Ok(()); + } + mem::sync(); + } + while start.elapsed() < timeout_slow { + if T::rptr(raw) == wptr { + return Ok(()); + } + fsleep(Delta::from_millis(5)); + mem::sync(); + } + Err(ETIMEDOUT) + }) + } +} + +/// Device Control channel for global device management commands. +#[versions(AGX)] +pub(crate) struct DeviceControlChannel { + dev: AsahiDevRef, + ch: TxChannel, +} + +#[versions(AGX)] +impl DeviceControlChannel::ver { + const COMMAND_TIMEOUT_MS: i64 = 1000; + + /// Allocate a new Device Control channel. + pub(crate) fn new( + dev: &AsahiDevice, + alloc: &mut gpu::KernelAllocators, + ) -> Result { + Ok(DeviceControlChannel::ver { + dev: dev.into(), + ch: TxChannel::::new(alloc, 0x100)?, + }) + } + + /// Returns the raw `ChannelRing` structure to pass to firmware. + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + self.ch.ring.to_raw() + } + + /// Submits a Device Control command. + pub(crate) fn send(&mut self, msg: &DeviceControlMsg::ver) -> u32 { + cls_dev_dbg!(DeviceControlCh, self.dev, "DeviceControl: {:?}\n", msg); + self.ch.put(msg) + } + + /// Waits for a previously submitted Device Control command to complete. + pub(crate) fn wait_for(&mut self, wptr: u32) -> Result { + self.ch.wait_for(wptr, Self::COMMAND_TIMEOUT_MS) + } +} + +/// Pipe channel to submit WorkQueue execution requests. +#[versions(AGX)] +pub(crate) struct PipeChannel { + dev: AsahiDevRef, + ch: TxChannel, +} + +#[versions(AGX)] +impl PipeChannel::ver { + /// Allocate a new Pipe submission channel. + pub(crate) fn new( + dev: &AsahiDevice, + alloc: &mut gpu::KernelAllocators, + ) -> Result { + Ok(PipeChannel::ver { + dev: dev.into(), + ch: TxChannel::::new(alloc, 0x100)?, + }) + } + + /// Returns the raw `ChannelRing` structure to pass to firmware. + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + self.ch.ring.to_raw() + } + + /// Submits a Pipe kick command to the firmware. + pub(crate) fn send(&mut self, msg: &PipeMsg::ver) { + cls_dev_dbg!(PipeCh, self.dev, "Pipe: {:?}\n", msg); + self.ch.put(msg); + } +} + +/// Firmware Control channel, used for secure cache flush requests. +pub(crate) struct FwCtlChannel { + dev: AsahiDevRef, + ch: TxChannel, +} + +impl FwCtlChannel { + const COMMAND_TIMEOUT_MS: i64 = 1000; + + /// Allocate a new Firmware Control channel. + pub(crate) fn new( + dev: &AsahiDevice, + alloc: &mut gpu::KernelAllocators, + ) -> Result { + Ok(FwCtlChannel { + dev: dev.into(), + ch: TxChannel::::new_uncached(alloc, 0x100)?, + }) + } + + /// Returns the raw `ChannelRing` structure to pass to firmware. + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + self.ch.ring.to_raw() + } + + /// Submits a Firmware Control command to the firmware. + pub(crate) fn send(&mut self, msg: &FwCtlMsg) -> u32 { + cls_dev_dbg!(FwCtlCh, self.dev, "FwCtl: {:?}\n", msg); + self.ch.put(msg) + } + + /// Waits for a previously submitted Firmware Control command to complete. + pub(crate) fn wait_for(&mut self, wptr: u32) -> Result { + self.ch.wait_for(wptr, Self::COMMAND_TIMEOUT_MS) + } +} + +/// Event channel, used to notify the driver of command completions, GPU faults and errors, and +/// other events. +#[versions(AGX)] +pub(crate) struct EventChannel { + dev: AsahiDevRef, + ch: RxChannel, + ev_mgr: Arc, + buf_mgr: buffer::BufferManager::ver, + gpu: Option>, +} + +#[versions(AGX)] +impl EventChannel::ver { + /// Allocate a new Event channel. + pub(crate) fn new( + dev: &AsahiDevice, + alloc: &mut gpu::KernelAllocators, + ev_mgr: Arc, + buf_mgr: buffer::BufferManager::ver, + ) -> Result { + Ok(EventChannel::ver { + dev: dev.into(), + ch: RxChannel::::new(alloc, 0x100)?, + ev_mgr, + buf_mgr, + gpu: None, + }) + } + + /// Registers the managing `Gpu` instance that will handle events on this channel. + pub(crate) fn set_manager(&mut self, gpu: Arc) { + self.gpu = Some(gpu); + } + + /// Returns the raw `ChannelRing` structure to pass to firmware. + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + self.ch.ring.to_raw() + } + + /// Polls for new Event messages on this ring. + pub(crate) fn poll(&mut self) { + while let Some(msg) = self.ch.get(0) { + // SAFETY: The raw view is always valid for all bit patterns. + let tag = unsafe { msg.raw.0 }; + match tag { + 0..=EVENT_MAX => { + // SAFETY: Since we have checked the tag to be in range, + // accessing the enum view is valid. + let msg = unsafe { msg.msg }; + + cls_dev_dbg!(EventCh, self.dev, "Event: {:?}\n", msg); + match msg { + EventMsg::Fault => match self.gpu.as_ref() { + Some(gpu) => gpu.handle_fault(), + None => { + dev_crit!( + self.dev.as_ref(), + "EventChannel: No GPU manager available!\n" + ) + } + }, + EventMsg::Timeout { + counter, + unk_8, + event_slot, + } => match self.gpu.as_ref() { + Some(gpu) => gpu.handle_timeout(counter, event_slot, unk_8), + None => { + dev_crit!( + self.dev.as_ref(), + "EventChannel: No GPU manager available!\n" + ) + } + }, + EventMsg::Flag { firing, .. } => { + for (i, flags) in firing.iter().enumerate() { + for j in 0..32 { + if flags & (1u32 << j) != 0 { + self.ev_mgr.signal((i * 32 + j) as u32); + } + } + } + } + EventMsg::GrowTVB { + vm_slot, + buffer_slot, + counter, + } => match self.gpu.as_ref() { + Some(gpu) => { + self.buf_mgr.grow(buffer_slot); + gpu.ack_grow(buffer_slot, vm_slot, counter); + } + None => { + dev_crit!( + self.dev.as_ref(), + "EventChannel: No GPU manager available!\n" + ) + } + }, + EventMsg::ChannelError { + error_type, + pipe_type, + event_slot, + event_value, + } => match self.gpu.as_ref() { + Some(gpu) => { + let error_type = match error_type { + 0 => ChannelErrorType::MemoryError, + 1 => ChannelErrorType::DMKill, + 2 => ChannelErrorType::Aborted, + 3 => ChannelErrorType::Unk3, + a => ChannelErrorType::Unknown(a), + }; + gpu.handle_channel_error( + error_type, + pipe_type, + event_slot, + event_value, + ); + } + None => { + dev_crit!( + self.dev.as_ref(), + "EventChannel: No GPU manager available!\n" + ) + } + }, + msg => { + dev_crit!(self.dev.as_ref(), "Unknown event message: {:?}\n", msg); + } + } + } + _ => { + // SAFETY: The raw view is always valid for all bit patterns. + dev_warn!(self.dev.as_ref(), "Unknown event message: {:?}\n", unsafe { + msg.raw + }); + } + } + } + } +} + +/// Firmware Log channel. This one is pretty special, since it has 6 sub-channels (for different log +/// levels), and it also uses a side buffer to actually hold the log messages, only passing around +/// pointers in the main buffer. +pub(crate) struct FwLogChannel { + dev: AsahiDevRef, + ch: RxChannel, + payload_buf: GpuArray, +} + +impl FwLogChannel { + const RING_SIZE: usize = 0x100; + const BUF_SIZE: usize = 0x100; + + /// Allocate a new Firmware Log channel. + pub(crate) fn new( + dev: &AsahiDevice, + alloc: &mut gpu::KernelAllocators, + ) -> Result { + Ok(FwLogChannel { + dev: dev.into(), + ch: RxChannel::::new(alloc, Self::RING_SIZE)?, + payload_buf: alloc + .shared + .array_empty(Self::BUF_SIZE * FwLogChannelState::SUB_CHANNELS)?, + }) + } + + /// Returns the raw `ChannelRing` structure to pass to firmware. + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + self.ch.ring.to_raw() + } + + /// Returns the GPU pointers to the firmware log payload buffer. + pub(crate) fn get_buf(&self) -> GpuWeakPointer<[RawFwLogPayloadMsg]> { + self.payload_buf.weak_pointer() + } + + /// Polls for new log messages on all sub-rings. + pub(crate) fn poll(&mut self) { + for i in 0..=FwLogChannelState::SUB_CHANNELS - 1 { + while let Some(msg) = self.ch.peek(i) { + cls_dev_dbg!(FwLogCh, self.dev, "FwLog{}: {:?}\n", i, msg); + if msg.msg_type != 2 { + dev_warn!(self.dev.as_ref(), "Unknown FWLog{} message: {:?}\n", i, msg); + self.ch.get(i); + continue; + } + if msg.msg_index.0 as usize >= Self::BUF_SIZE { + dev_warn!( + self.dev.as_ref(), + "FWLog{} message index out of bounds: {:?}\n", + i, + msg + ); + self.ch.get(i); + continue; + } + let index = Self::BUF_SIZE * i + msg.msg_index.0 as usize; + let payload = &self.payload_buf.as_slice()[index]; + if payload.msg_type != 3 { + dev_warn!( + self.dev.as_ref(), + "Unknown FWLog{} payload: {:?}\n", + i, + payload + ); + self.ch.get(i); + continue; + } + let msg = if let Some(end) = payload.msg.iter().position(|&r| r == 0) { + CStr::from_bytes_with_nul(&(*payload.msg)[..end + 1]) + .unwrap_or(c_str!("cstr_err")) + } else { + dev_warn!( + self.dev.as_ref(), + "FWLog{} payload not NUL-terminated: {:?}\n", + i, + payload + ); + self.ch.get(i); + continue; + }; + match i { + 0 => dev_dbg!(self.dev.as_ref(), "FWLog: {}\n", msg), + 1 => dev_info!(self.dev.as_ref(), "FWLog: {}\n", msg), + 2 => dev_notice!(self.dev.as_ref(), "FWLog: {}\n", msg), + 3 => dev_warn!(self.dev.as_ref(), "FWLog: {}\n", msg), + 4 => dev_err!(self.dev.as_ref(), "FWLog: {}\n", msg), + 5 => dev_crit!(self.dev.as_ref(), "FWLog: {}\n", msg), + _ => (), + }; + self.ch.get(i); + } + } + } +} + +pub(crate) struct KTraceChannel { + dev: AsahiDevRef, + ch: RxChannel, +} + +/// KTrace channel, used to receive detailed execution trace markers from the firmware. +/// We currently disable this in initdata, so no messages are expected here at this time. +impl KTraceChannel { + /// Allocate a new KTrace channel. + pub(crate) fn new( + dev: &AsahiDevice, + alloc: &mut gpu::KernelAllocators, + ) -> Result { + Ok(KTraceChannel { + dev: dev.into(), + ch: RxChannel::::new(alloc, 0x200)?, + }) + } + + /// Returns the raw `ChannelRing` structure to pass to firmware. + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + self.ch.ring.to_raw() + } + + /// Polls for new KTrace messages on this ring. + pub(crate) fn poll(&mut self) { + while let Some(msg) = self.ch.get(0) { + cls_dev_dbg!(KTraceCh, self.dev, "KTrace: {:?}\n", msg); + } + } +} + +/// Statistics channel, reporting power-related statistics to the driver. +/// Not really implemented other than debug logs yet... +#[versions(AGX)] +pub(crate) struct StatsChannel { + dev: AsahiDevRef, + ch: RxChannel, +} + +#[versions(AGX)] +impl StatsChannel::ver { + /// Allocate a new Statistics channel. + pub(crate) fn new( + dev: &AsahiDevice, + alloc: &mut gpu::KernelAllocators, + ) -> Result { + Ok(StatsChannel::ver { + dev: dev.into(), + ch: RxChannel::::new(alloc, 0x100)?, + }) + } + + /// Returns the raw `ChannelRing` structure to pass to firmware. + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + self.ch.ring.to_raw() + } + + /// Polls for new statistics messages on this ring. + pub(crate) fn poll(&mut self) { + while let Some(msg) = self.ch.get(0) { + // SAFETY: The raw view is always valid for all bit patterns. + let tag = unsafe { msg.raw.0 }; + match tag { + 0..=STATS_MAX::ver => { + // SAFETY: Since we have checked the tag to be in range, + // accessing the enum view is valid. + let msg = unsafe { msg.msg }; + cls_dev_dbg!(StatsCh, self.dev, "Stats: {:?}\n", msg); + } + _ => { + // SAFETY: The raw view is always valid for all bit patterns. + pr_warn!("Unknown stats message: {:?}\n", unsafe { msg.raw }); + } + } + } + } +} diff --git a/drivers/gpu/drm/asahi/crashdump.rs b/drivers/gpu/drm/asahi/crashdump.rs new file mode 100644 index 00000000000000..bd9f2f1649584f --- /dev/null +++ b/drivers/gpu/drm/asahi/crashdump.rs @@ -0,0 +1,305 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU crash dump formatter +//! +//! Takes a raw dump of firmware/kernel mapped pages from `pgtable` and formats it into +//! an ELF core dump suitable for dumping into userspace. + +use core::mem::size_of; + +use kernel::{ + devcoredump::DevCoreDump, + error::Result, + page::{Page, PAGE_MASK, PAGE_SHIFT, PAGE_SIZE}, + prelude::*, + types::Owned, + uapi, +}; + +use crate::hw; +use crate::pgtable::{self, DumpedPage, Prot, UAT_PGSZ}; +use crate::util::align; + +pub(crate) struct CrashDump { + headers: KVVec, + pages: KVVec>, +} + +const NOTE_NAME_AGX: &str = "AGX"; +const NOTE_AGX_DUMP_INFO: u32 = 1; + +const NOTE_NAME_RTKIT: &str = "RTKIT"; +const NOTE_RTKIT_CRASHLOG: u32 = 1; + +#[repr(C)] +pub(crate) struct AGXDumpInfo { + initdata_address: u64, + chip_id: u32, + gpu_gen: hw::GpuGen, + gpu_variant: hw::GpuVariant, + gpu_rev: hw::GpuRevision, + total_active_cores: u32, + firmware_version: [u32; 6], +} + +struct ELFNote { + name: &'static str, + ty: u32, + data: KVVec, +} + +pub(crate) struct CrashDumpBuilder { + page_dump: KVVec, + notes: KVec, +} + +/// Helper to convert ELF headers into byte slices +/// TODO: Hook this up into kernel::AsBytes somehow +/// +/// # Safety +/// +/// Types implementing this trait must have no padding bytes. +unsafe trait AsBytes: Sized { + fn as_bytes(&self) -> &[u8] { + // SAFETY: This trait is only implemented for types with no padding bytes + unsafe { core::slice::from_raw_parts(self as *const _ as *const u8, size_of::()) } + } + fn slice_as_bytes(slice: &[Self]) -> &[u8] { + // SAFETY: This trait is only implemented for types with no padding bytes + unsafe { + core::slice::from_raw_parts(slice.as_ptr() as *const u8, core::mem::size_of_val(slice)) + } + } +} + +// SAFETY: This type has no padding +unsafe impl AsBytes for uapi::Elf64_Ehdr {} +// SAFETY: This type has no padding +unsafe impl AsBytes for uapi::Elf64_Phdr {} +// SAFETY: This type has no padding +unsafe impl AsBytes for uapi::Elf64_Nhdr {} +// SAFETY: This type has no padding +unsafe impl AsBytes for AGXDumpInfo {} + +const FIRMWARE_ENTRYPOINT: u64 = 0xFFFFFF8000000000u64; + +impl CrashDumpBuilder { + pub(crate) fn new(page_dump: KVVec) -> Result { + Ok(CrashDumpBuilder { + page_dump, + notes: KVec::new(), + }) + } + + pub(crate) fn add_agx_info( + &mut self, + cfg: &hw::HwConfig, + dyncfg: &hw::DynConfig, + initdata_address: u64, + ) -> Result { + let mut info = AGXDumpInfo { + chip_id: cfg.chip_id, + gpu_gen: dyncfg.id.gpu_gen, + gpu_variant: dyncfg.id.gpu_variant, + gpu_rev: dyncfg.id.gpu_rev, + total_active_cores: dyncfg.id.total_active_cores, + firmware_version: [0; 6], + initdata_address, + }; + info.firmware_version[..dyncfg.firmware_version.len().min(6)] + .copy_from_slice(&dyncfg.firmware_version); + + let mut data = KVVec::new(); + data.extend_from_slice(info.as_bytes(), GFP_KERNEL)?; + + self.notes.push( + ELFNote { + name: NOTE_NAME_AGX, + ty: NOTE_AGX_DUMP_INFO, + data, + }, + GFP_KERNEL, + )?; + Ok(()) + } + + pub(crate) fn add_crashlog(&mut self, crashlog: &[u8]) -> Result { + let mut data = KVVec::new(); + data.extend_from_slice(crashlog, GFP_KERNEL)?; + + self.notes.push( + ELFNote { + name: NOTE_NAME_RTKIT, + ty: NOTE_RTKIT_CRASHLOG, + data, + }, + GFP_KERNEL, + )?; + + Ok(()) + } + + pub(crate) fn finalize(self) -> Result { + let CrashDumpBuilder { page_dump, notes } = self; + + let mut ehdr: uapi::Elf64_Ehdr = Default::default(); + + ehdr.e_ident[uapi::EI_MAG0 as usize..=uapi::EI_MAG3 as usize].copy_from_slice(b"\x7fELF"); + ehdr.e_ident[uapi::EI_CLASS as usize] = uapi::ELFCLASS64 as u8; + ehdr.e_ident[uapi::EI_DATA as usize] = uapi::ELFDATA2LSB as u8; + ehdr.e_ident[uapi::EI_VERSION as usize] = uapi::EV_CURRENT as u8; + ehdr.e_type = uapi::ET_CORE as u16; + ehdr.e_machine = uapi::EM_AARCH64 as u16; + ehdr.e_version = uapi::EV_CURRENT; + ehdr.e_entry = FIRMWARE_ENTRYPOINT; + ehdr.e_ehsize = core::mem::size_of::() as u16; + ehdr.e_phentsize = core::mem::size_of::() as u16; + + let phdr_offset = core::mem::size_of::(); + + // PHDRs come after the ELF header + ehdr.e_phoff = phdr_offset as u64; + + let mut phdrs = KVVec::new(); + + // First PHDR is the NOTE section + phdrs.push( + uapi::Elf64_Phdr { + p_type: uapi::PT_NOTE, + p_flags: uapi::PF_R, + p_align: 1, + ..Default::default() + }, + GFP_KERNEL, + )?; + + // Generate the page phdrs. The offset will be fixed up later. + let mut off: usize = 0; + let mut next = None; + let mut pages: KVVec> = KVVec::new(); + + for mut page in page_dump { + let vaddr = page.iova; + let paddr = page.pte & pgtable::PTE_ADDR_BITS; + let flags = Prot::from_pte(page.pte).elf_flags(); + let valid = page.data.is_some(); + let cur = (vaddr, paddr, flags, valid); + if Some(cur) != next { + phdrs.push( + uapi::Elf64_Phdr { + p_type: uapi::PT_LOAD, + p_offset: if valid { off as u64 } else { 0 }, + p_vaddr: vaddr, + p_paddr: paddr, + p_filesz: if valid { UAT_PGSZ as u64 } else { 0 }, + p_memsz: UAT_PGSZ as u64, + p_flags: flags, + p_align: UAT_PGSZ as u64, + }, + GFP_KERNEL, + )?; + if valid { + off += UAT_PGSZ; + } + } else { + let ph = phdrs.last_mut().unwrap(); + ph.p_memsz += UAT_PGSZ as u64; + if valid { + ph.p_filesz += UAT_PGSZ as u64; + off += UAT_PGSZ; + } + } + if let Some(data_page) = page.data.take() { + pages.push(data_page, GFP_KERNEL)?; + } + next = Some(( + vaddr + UAT_PGSZ as u64, + paddr + UAT_PGSZ as u64, + flags, + valid, + )); + } + + ehdr.e_phnum = phdrs.len() as u16; + + let note_offset = phdr_offset + size_of::() * phdrs.len(); + + let mut note_data: KVVec = KVVec::new(); + + for note in notes { + let hdr = uapi::Elf64_Nhdr { + n_namesz: note.name.len() as u32 + 1, + n_descsz: note.data.len() as u32, + n_type: note.ty, + }; + note_data.extend_from_slice(hdr.as_bytes(), GFP_KERNEL)?; + note_data.extend_from_slice(note.name.as_bytes(), GFP_KERNEL)?; + note_data.push(0, GFP_KERNEL)?; + while note_data.len() & 3 != 0 { + note_data.push(0, GFP_KERNEL)?; + } + note_data.extend_from_slice(¬e.data, GFP_KERNEL)?; + while note_data.len() & 3 != 0 { + note_data.push(0, GFP_KERNEL)?; + } + } + + // NOTE section comes after the PHDRs + phdrs[0].p_offset = note_offset as u64; + phdrs[0].p_filesz = note_data.len() as u64; + + // Align data section to the page size + let data_offset = align(note_offset + note_data.len(), UAT_PGSZ); + + // Fix up data PHDR offsets + for phdr in &mut phdrs[1..] { + phdr.p_offset += data_offset as u64; + } + + // Build ELF header buffer + let mut headers: KVVec = KVVec::from_elem(0, data_offset, GFP_KERNEL)?; + + headers[0..size_of::()].copy_from_slice(ehdr.as_bytes()); + headers[phdr_offset..phdr_offset + phdrs.len() * size_of::()] + .copy_from_slice(AsBytes::slice_as_bytes(&phdrs)); + headers[note_offset..note_offset + note_data.len()].copy_from_slice(¬e_data); + + Ok(CrashDump { headers, pages }) + } +} + +impl DevCoreDump for CrashDump { + fn read(&self, buf: &mut [u8], mut offset: usize) -> Result { + let mut read = 0; + let mut left = buf.len(); + if offset < self.headers.len() { + let block = left.min(self.headers.len() - offset); + buf[..block].copy_from_slice(&self.headers[offset..offset + block]); + read += block; + offset += block; + left -= block; + } + if left == 0 { + return Ok(read); + } + offset -= self.headers.len(); // Offset from the page area + + while left > 0 { + let page_index = offset >> PAGE_SHIFT; + let page_offset = offset & !PAGE_MASK; + let block = left.min(PAGE_SIZE - page_offset); + let Some(page) = self.pages.get(page_index) else { + break; + }; + let slice = &mut buf[read..read + block]; + // SAFETY: We own the page, and the slice guarantees the + // dst length is sufficient. + unsafe { page.read_raw(slice.as_mut_ptr(), page_offset, slice.len())? }; + read += block; + offset += block; + left -= block; + } + + Ok(read) + } +} diff --git a/drivers/gpu/drm/asahi/debug.rs b/drivers/gpu/drm/asahi/debug.rs new file mode 100644 index 00000000000000..50628ade5ab8e4 --- /dev/null +++ b/drivers/gpu/drm/asahi/debug.rs @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![allow(dead_code)] + +//! Debug enable/disable flags and convenience macros + +#[allow(unused_imports)] +pub(crate) use super::{ + cls_dev_dbg, + cls_pr_debug, + debug, + mod_dev_dbg, + mod_pr_debug, // +}; +use crate::module_parameters; +use core::sync::atomic::{ + AtomicU64, + Ordering, // +}; + +static DEBUG_FLAGS: AtomicU64 = AtomicU64::new(0); + +/// Debug flag bit indices +pub(crate) enum DebugFlags { + // 0-4: Memory-related debug + Mmu = 0, + PgTable = 1, + Alloc = 2, + Gem = 3, + Object = 4, + + // 5-7: Firmware objects and resources + Event = 5, + Buffer = 6, + WorkQueue = 7, + + // 8-13: DRM interface, rendering, compute, GPU globals + Gpu = 8, + File = 9, + Queue = 10, + Render = 11, + Compute = 12, + Errors = 13, + + // 14-15: Misc stats + MemStats = 14, + TVBStats = 15, + + // 16-22: Channels + FwLogCh = 16, + KTraceCh = 17, + StatsCh = 18, + EventCh = 19, + PipeCh = 20, + DeviceControlCh = 21, + FwCtlCh = 22, + + // 32-35: Allocator debugging + FillAllocations = 32, + DebugAllocations = 33, + DetectOverflows = 34, + ForceCPUMaps = 35, + + // 36-: Behavior flags + ConservativeTlbi = 36, + KeepGpuPowered = 37, + WaitForPowerOff = 38, + NoGpuRecovery = 39, + DisableClustering = 40, + + // 48-: Misc + Debug0 = 48, + Debug1 = 49, + Debug2 = 50, + Debug3 = 51, + Debug4 = 52, + Debug5 = 53, + Debug6 = 54, + Debug7 = 55, + + VerboseFaults = 61, + AllowUnknownOverrides = 62, + OopsOnGpuCrash = 63, +} + +/// Update the cached global debug flags from the module parameter +pub(crate) fn update_debug_flags() { + let flags = *module_parameters::debug_flags.value(); + + DEBUG_FLAGS.store(flags, Ordering::Relaxed); +} + +/// Check whether debug is enabled for a given flag +#[inline(always)] +pub(crate) fn debug_enabled(flag: DebugFlags) -> bool { + DEBUG_FLAGS.load(Ordering::Relaxed) & 1 << (flag as usize) != 0 +} + +/// Run some code only if debug is enabled for the calling module +#[macro_export] +macro_rules! debug { + ($($arg:tt)*) => { + if $crate::debug::debug_enabled(DEBUG_CLASS) { + $($arg)* + } + }; +} + +/// pr_info!() if debug is enabled for the calling module +#[macro_export] +macro_rules! mod_pr_debug ( + ($($arg:tt)*) => ( + $crate::debug! { ::kernel::pr_info! ( $($arg)* ); } + ) +); + +/// dev_info!() if debug is enabled for the calling module +#[macro_export] +macro_rules! mod_dev_dbg ( + ($dev:expr, $($arg:tt)*) => ( + $crate::debug! { ::kernel::dev_info! ( $dev.as_ref(), $($arg)* ); } + ) +); + +/// pr_info!() if debug is enabled for a specific module +#[macro_export] +macro_rules! cls_pr_debug ( + ($cls:ident, $($arg:tt)*) => ( + if $crate::debug::debug_enabled($crate::debug::DebugFlags::$cls) { + ::kernel::pr_info! ( $($arg)* ); + } + ) +); + +/// dev_info!() if debug is enabled for a specific module +#[macro_export] +macro_rules! cls_dev_dbg ( + ($cls:ident, $dev:expr, $($arg:tt)*) => ( + if $crate::debug::debug_enabled($crate::debug::DebugFlags::$cls) { + ::kernel::dev_info! ( $dev.as_ref(), $($arg)* ); + } + ) +); diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs new file mode 100644 index 00000000000000..03245a4826be71 --- /dev/null +++ b/drivers/gpu/drm/asahi/driver.rs @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Top-level GPU driver implementation. + +use kernel::{ + c_str, + device::Core, + dma::{ + Device, + DmaMask, // + }, + drm, + drm::ioctl, + of, + platform, + prelude::*, + sync::Arc, // +}; + +use crate::{ + debug, + file, + gem::AsahiObject, + gpu, + hw, + regs, // +}; + +use kernel::macros::vtable; +use kernel::types::ARef; + +/// Holds a reference to the top-level `GpuManager` object. +#[pin_data] +pub(crate) struct AsahiData { + #[pin] + pub(crate) gpu: Arc, + pub(crate) pdev: ARef, + pub(crate) resources: regs::Resources, +} + +unsafe impl Send for AsahiData {} +unsafe impl Sync for AsahiData {} + +pub(crate) struct AsahiDriver { + #[expect(unused)] + drm: ARef>, +} + +unsafe impl Send for AsahiDriver {} +unsafe impl Sync for AsahiDriver {} + +/// Convenience type alias for the DRM device type for this driver. +pub(crate) type AsahiDevice = drm::device::Device; +pub(crate) type AsahiDevRef = ARef; + +/// DRM Driver metadata +const INFO: drm::driver::DriverInfo = drm::driver::DriverInfo { + major: 0, + minor: 0, + patchlevel: 0, + name: c_str!("asahi"), + desc: c_str!("Apple AGX Graphics"), +}; + +/// DRM Driver implementation for `AsahiDriver`. +#[vtable] +impl drm::driver::Driver for AsahiDriver { + /// Our `DeviceData` type, reference-counted + type Data = AsahiData; + /// Our `File` type. + type File = file::File; + /// Our `Object` type. + type Object = drm::gem::shmem::Object; + + const INFO: drm::driver::DriverInfo = INFO; + const FEATURES: u32 = drm::driver::FEAT_GEM + | drm::driver::FEAT_RENDER + | drm::driver::FEAT_SYNCOBJ + | drm::driver::FEAT_SYNCOBJ_TIMELINE + | drm::driver::FEAT_GEM_GPUVA; + + kernel::declare_drm_ioctls! { + (ASAHI_GET_PARAMS, drm_asahi_get_params, + ioctl::RENDER_ALLOW, crate::file::File::get_params), + (ASAHI_GET_TIME, drm_asahi_get_time, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::get_time), + (ASAHI_VM_CREATE, drm_asahi_vm_create, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::vm_create), + (ASAHI_VM_DESTROY, drm_asahi_vm_destroy, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::vm_destroy), + (ASAHI_VM_BIND, drm_asahi_vm_bind, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::vm_bind), + (ASAHI_GEM_CREATE, drm_asahi_gem_create, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::gem_create), + (ASAHI_GEM_MMAP_OFFSET, drm_asahi_gem_mmap_offset, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::gem_mmap_offset), + (ASAHI_GEM_BIND_OBJECT, drm_asahi_gem_bind_object, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::gem_bind_object), + (ASAHI_QUEUE_CREATE, drm_asahi_queue_create, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::queue_create), + (ASAHI_QUEUE_DESTROY, drm_asahi_queue_destroy, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::queue_destroy), + (ASAHI_SUBMIT, drm_asahi_submit, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::submit), + } +} + +// OF Device ID table.s +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + ::IdInfo, + [ + ( + of::DeviceId::new(c_str!("apple,agx-t8103")), + &hw::t8103::HWCONFIG + ), + ( + of::DeviceId::new(c_str!("apple,agx-t8112")), + &hw::t8112::HWCONFIG + ), + ( + of::DeviceId::new(c_str!("apple,agx-t6000")), + &hw::t600x::HWCONFIG_T6000 + ), + ( + of::DeviceId::new(c_str!("apple,agx-t6001")), + &hw::t600x::HWCONFIG_T6001 + ), + ( + of::DeviceId::new(c_str!("apple,agx-t6002")), + &hw::t600x::HWCONFIG_T6002 + ), + ( + of::DeviceId::new(c_str!("apple,agx-t6020")), + &hw::t602x::HWCONFIG_T6020 + ), + ( + of::DeviceId::new(c_str!("apple,agx-t6021")), + &hw::t602x::HWCONFIG_T6021 + ), + ( + of::DeviceId::new(c_str!("apple,agx-t6022")), + &hw::t602x::HWCONFIG_T6022 + ), + ] +); + +/// Platform Driver implementation for `AsahiDriver`. +impl platform::Driver for AsahiDriver { + type IdInfo = &'static hw::HwConfig; + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + /// Device probe function. + fn probe( + pdev: &platform::Device, + info: Option<&Self::IdInfo>, + ) -> impl PinInit { + debug::update_debug_flags(); + + dev_info!(pdev.as_ref(), "Probing...\n"); + + let cfg = info.ok_or(ENODEV)?; + + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; + + let res = regs::Resources::new(pdev)?; + + // Initialize misc MMIO + res.init_mmio()?; + + // Start the coprocessor CPU, so UAT can initialize the handoff + regs::Resources::start_cpu(pdev)?; + + let fwnode = pdev.as_ref().fwnode().ok_or(EIO)?; + let compat: KVec = fwnode + .property_read_array_vec(c_str!("apple,firmware-compat"), 3)? + .required_by(pdev.as_ref())?; + + // TODO: This is very temporary + // SAFETY: This should be safe as data is not touched by the driver + // untill it gets fully initialised. + // Additionally drm::device::Device::release() will not drop data and + // leaks instead. + let uninit = unsafe { + pin_init::pin_init_from_closure::(|_slot| Ok(())) + }; + let drm: ARef = drm::device::Device::new(pdev.as_ref(), uninit)?; + + let gpu = match (cfg.gpu_gen, cfg.gpu_variant, compat.as_slice()) { + (hw::GpuGen::G13, _, &[12, 3, 0]) => { + gpu::GpuManagerG13V12_3::new(&drm.clone(), &res, cfg)? as Arc + } + (hw::GpuGen::G14, hw::GpuVariant::G, &[12, 4, 0]) => { + gpu::GpuManagerG14V12_4::new(&drm.clone(), &res, cfg)? as Arc + } + (hw::GpuGen::G13, _, &[13, 5, 0]) => { + gpu::GpuManagerG13V13_5::new(&drm.clone(), &res, cfg)? as Arc + } + (hw::GpuGen::G14, hw::GpuVariant::G, &[13, 5, 0]) => { + gpu::GpuManagerG14V13_5::new(&drm.clone(), &res, cfg)? as Arc + } + (hw::GpuGen::G14, _, &[13, 5, 0]) => { + gpu::GpuManagerG14XV13_5::new(&drm.clone(), &res, cfg)? as Arc + } + _ => { + dev_info!( + pdev.as_ref(), + "Unsupported GPU/firmware combination ({:?}, {:?}, {:?})\n", + cfg.gpu_gen, + cfg.gpu_variant, + compat + ); + return Err(ENODEV); + } + }; + + let data = try_pin_init!(AsahiData { + gpu, + pdev: pdev.into(), + resources: res, + }); + + let ptr: *const AsahiData = &raw const **drm; + unsafe { + data.__pinned_init(ptr as *mut AsahiData)?; + } + + (*drm).gpu.init()?; + + drm::driver::Registration::new_foreign_owned(&drm, pdev.as_ref(), 0)?; + + Ok(Self { drm }) + } +} diff --git a/drivers/gpu/drm/asahi/event.rs b/drivers/gpu/drm/asahi/event.rs new file mode 100644 index 00000000000000..edd7d701e665cd --- /dev/null +++ b/drivers/gpu/drm/asahi/event.rs @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU event manager +//! +//! The GPU firmware manages work completion by using event objects (Apple calls them "stamps"), +//! which are monotonically incrementing counters. There are a fixed number of objects, and +//! they are managed with a `SlotAllocator`. +//! +//! This module manages the set of available events and lets users compute expected values. +//! It also manages signaling owners when the GPU firmware reports that an event fired. + +use crate::debug::*; +use crate::fw::types::*; +use crate::{ + gpu, + slotalloc, + workqueue, // +}; +use core::cmp; +use core::sync::atomic::Ordering; +use kernel::prelude::*; +use kernel::sync::Arc; +use kernel::{ + c_str, + static_lock_class, // +}; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Event; + +/// Number of events managed by the firmware. +const NUM_EVENTS: u32 = 128; + +/// Inner data associated with a given event slot. +pub(crate) struct EventInner { + /// CPU pointer to the driver notification event stamp + stamp: *const AtomicU32, + /// GPU pointer to the driver notification event stamp + gpu_stamp: GpuWeakPointer, + /// GPU pointer to the firmware-internal event stamp + gpu_fw_stamp: GpuWeakPointer, +} + +/// SAFETY: The event slots are safe to send across threads. +unsafe impl Send for EventInner {} + +/// Alias for an event token, which allows requesting the same event. +pub(crate) type Token = slotalloc::SlotToken; +/// Alias for an allocated `Event` that has a slot. +pub(crate) type Event = slotalloc::Guard; + +/// Represents a given stamp value for an event. +#[derive(Eq, PartialEq, Copy, Clone, Debug)] +#[repr(transparent)] +pub(crate) struct EventValue(u32); + +impl EventValue { + /// Returns the `EventValue` that succeeds this one. + pub(crate) fn next(&self) -> EventValue { + EventValue(self.0.wrapping_add(0x100)) + } + + /// Increments this `EventValue` in place. + pub(crate) fn increment(&mut self) { + self.0 = self.0.wrapping_add(0x100); + } + + /* Not used + /// Increments this `EventValue` in place by a certain count. + pub(crate) fn add(&mut self, val: u32) { + self.0 = self + .0 + .wrapping_add(val.checked_mul(0x100).expect("Adding too many events")); + } + */ + + /// Increments this `EventValue` in place by a certain count. + pub(crate) fn sub(&mut self, val: u32) { + self.0 = self + .0 + .wrapping_sub(val.checked_mul(0x100).expect("Subtracting too many events")); + } + + /// Computes the delta between this event and another event. + pub(crate) fn delta(&self, other: &EventValue) -> i32 { + (self.0.wrapping_sub(other.0) as i32) >> 8 + } +} + +impl PartialOrd for EventValue { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for EventValue { + fn cmp(&self, other: &Self) -> cmp::Ordering { + self.delta(other).cmp(&0) + } +} + +impl EventInner { + /// Returns the GPU pointer to the driver notification stamp + pub(crate) fn stamp_pointer(&self) -> GpuWeakPointer { + self.gpu_stamp + } + + /// Returns the GPU pointer to the firmware internal stamp + pub(crate) fn fw_stamp_pointer(&self) -> GpuWeakPointer { + self.gpu_fw_stamp + } + + /// Fetches the current event value from shared memory + pub(crate) fn current(&self) -> EventValue { + // SAFETY: The pointer is always valid as constructed in + // EventManager below, and outside users cannot construct + // new EventInners, nor move or copy them, and Guards as + // returned by the SlotAllocator hold a reference to the + // SlotAllocator containing the EventManagerInner, which + // keeps the GpuObject the stamp is contained within alive. + EventValue(unsafe { &*self.stamp }.load(Ordering::Acquire)) + } +} + +impl slotalloc::SlotItem for EventInner { + type Data = EventManagerInner; + + fn release(&mut self, data: &mut Self::Data, slot: u32) { + mod_pr_debug!("EventManager: Released slot {}\n", slot); + data.owners[slot as usize] = None; + } +} + +/// Inner data for the event manager, to be protected by the SlotAllocator lock. +pub(crate) struct EventManagerInner { + stamps: GpuArray, + fw_stamps: GpuArray, + // Note: Use dyn to avoid having to version this entire module. + owners: KVec>>, +} + +/// Top-level EventManager object. +pub(crate) struct EventManager { + alloc: slotalloc::SlotAllocator, +} + +impl EventManager { + /// Create a new EventManager. + #[inline(never)] + pub(crate) fn new(alloc: &mut gpu::KernelAllocators) -> Result { + let mut owners = KVec::new(); + for _i in 0..(NUM_EVENTS as usize) { + owners.push(None, GFP_KERNEL)?; + } + let inner = EventManagerInner { + stamps: alloc.shared.array_empty(NUM_EVENTS as usize)?, + fw_stamps: alloc.private.array_empty(NUM_EVENTS as usize)?, + owners, + }; + + for slot in 0..NUM_EVENTS { + inner.stamps[slot as usize] + .0 + .store(slot << 24, Ordering::Relaxed); + } + + Ok(EventManager { + alloc: slotalloc::SlotAllocator::new( + NUM_EVENTS, + inner, + |inner: &mut EventManagerInner, slot| { + Some(EventInner { + stamp: &inner.stamps[slot as usize].0, + gpu_stamp: inner.stamps.weak_item_pointer(slot as usize), + gpu_fw_stamp: inner.fw_stamps.weak_item_pointer(slot as usize), + }) + }, + c_str!("EventManager::SlotAllocator"), + static_lock_class!(), + static_lock_class!(), + )?, + }) + } + + /// Gets a free `Event`, optionally trying to reuse the last one allocated by this caller. + pub(crate) fn get( + &self, + token: Option, + owner: Arc, + ) -> Result { + let ev = self.alloc.get_inner(token, |inner, ev| { + mod_pr_debug!( + "EventManager: Registered owner {:p} on slot {}\n", + &*owner, + ev.slot() + ); + inner.owners[ev.slot() as usize] = Some(owner); + Ok(()) + })?; + Ok(ev) + } + + /// Signals an event by slot, indicating completion (of one or more commands). + pub(crate) fn signal(&self, slot: u32) { + match self + .alloc + .with_inner(|inner| inner.owners[slot as usize].as_ref().cloned()) + { + Some(owner) => { + owner.signal(); + } + None => { + mod_pr_debug!("EventManager: Received event for empty slot {}\n", slot); + } + } + } + + /// Marks the owner of an event as having lost its work due to a GPU error. + pub(crate) fn mark_error(&self, slot: u32, wait_value: u32, error: workqueue::WorkError) { + match self + .alloc + .with_inner(|inner| inner.owners[slot as usize].as_ref().cloned()) + { + Some(owner) => { + owner.mark_error(EventValue(wait_value), error); + } + None => { + pr_err!("Received error for empty slot {}\n", slot); + } + } + } + + /// Returns a reference to the workqueue owning an event. + pub(crate) fn get_owner( + &self, + slot: u32, + ) -> Option> { + self.alloc + .with_inner(|inner| inner.owners[slot as usize].as_ref().cloned()) + } + + /// Fail all commands, used when the GPU crashes. + pub(crate) fn fail_all(&self, error: workqueue::WorkError) { + let mut owners: KVec> = KVec::new(); + + self.alloc.with_inner(|inner| { + for wq in inner.owners.iter().filter_map(|o| o.as_ref()).cloned() { + if owners.push(wq, GFP_KERNEL).is_err() { + pr_err!("Failed to signal failure to WorkQueue\n"); + } + } + }); + + for wq in owners { + wq.fail_all(error); + } + } +} diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs new file mode 100644 index 00000000000000..02a4ce874a92c8 --- /dev/null +++ b/drivers/gpu/drm/asahi/file.rs @@ -0,0 +1,1092 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![allow(clippy::unusual_byte_groupings)] + +//! File implementation, which represents a single DRM client. +//! +//! This is in charge of managing the resources associated with one GPU client, including an +//! arbitrary number of submission queues and Vm objects, and reporting hardware/driver +//! information to userspace and accepting submissions. + +use crate::debug::*; +use crate::driver::AsahiDevice; +use crate::{ + alloc, + buffer, + driver, + gem, + mmu, + module_parameters, + queue, + util::{ + align, + align_down, + gcd, + AnyBitPattern, + RangeExt, + Reader, // + }, // +}; +use core::mem::MaybeUninit; +use core::ops::Deref; +use core::ops::Range; +use core::ptr::addr_of_mut; +use kernel::bindings; +use kernel::dma_fence::RawDmaFence; +use kernel::drm::gem::BaseObject; +use kernel::error::code::*; +use kernel::new_mutex; +use kernel::prelude::*; +use kernel::sync::{ + Arc, + Mutex, // +}; +use kernel::time::NSEC_PER_SEC; +use kernel::uaccess::{ + UserPtr, + UserSlice, // +}; +use kernel::{ + dma_fence, + drm, + uapi, + xarray, // +}; + +const DEBUG_CLASS: DebugFlags = DebugFlags::File; + +pub(crate) const MAX_COMMANDS_PER_SUBMISSION: u32 = 64; + +/// A client instance of an `mmu::Vm` address space. +struct Vm { + ualloc: Arc>, + ualloc_priv: Arc>, + vm: mmu::Vm, + kernel_range: Range, + _dummy_mapping: mmu::KernelMapping, +} + +impl Drop for Vm { + fn drop(&mut self) { + // When the user Vm is dropped, unmap everything in the user range + let left_range = VM_USER_RANGE.start..self.kernel_range.start; + let right_range = self.kernel_range.end..VM_USER_RANGE.end; + + if !left_range.is_empty() + && self + .vm + .unmap_range(left_range.start, left_range.range()) + .is_err() + { + pr_err!("Vm::Drop: vm.unmap_range() failed\n"); + } + if !right_range.is_empty() + && self + .vm + .unmap_range(right_range.start, right_range.range()) + .is_err() + { + pr_err!("Vm::Drop: vm.unmap_range() failed\n"); + } + } +} + +/// Sync object from userspace. +pub(crate) struct SyncItem { + pub(crate) syncobj: drm::syncobj::SyncObj, + pub(crate) fence: Option, + pub(crate) chain_fence: Option, + pub(crate) timeline_value: u64, +} + +impl SyncItem { + fn parse_one(file: &DrmFile, data: uapi::drm_asahi_sync, out: bool) -> Result { + match data.sync_type { + uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_SYNCOBJ => { + if data.timeline_value != 0 { + cls_pr_debug!(Errors, "Non-timeline sync object with a nonzero value\n"); + return Err(EINVAL); + } + let syncobj = drm::syncobj::SyncObj::lookup_handle(file, data.handle)?; + + Ok(SyncItem { + fence: if out { + None + } else { + Some(syncobj.fence_get().ok_or_else(|| { + cls_pr_debug!(Errors, "Failed to get fence from sync object\n"); + EINVAL + })?) + }, + syncobj, + chain_fence: None, + timeline_value: data.timeline_value, + }) + } + uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ => { + let syncobj = drm::syncobj::SyncObj::lookup_handle(file, data.handle)?; + let fence = if out { + None + } else { + syncobj + .fence_get() + .ok_or_else(|| { + cls_pr_debug!( + Errors, + "Failed to get fence from timeline sync object\n" + ); + EINVAL + })? + .chain_find_seqno(data.timeline_value)? + }; + + Ok(SyncItem { + fence, + syncobj, + chain_fence: if out { + Some(dma_fence::FenceChain::new()?) + } else { + None + }, + timeline_value: data.timeline_value, + }) + } + _ => { + cls_pr_debug!(Errors, "Invalid sync type {}\n", data.sync_type); + Err(EINVAL) + } + } + } + + fn parse_array( + file: &DrmFile, + ptr: u64, + in_count: u32, + out_count: u32, + ) -> Result> { + let count = in_count + out_count; + let mut vec = KVec::with_capacity(count as usize, GFP_KERNEL)?; + + const STRIDE: usize = core::mem::size_of::(); + let size = STRIDE * count as usize; + + // SAFETY: We only read this once, so there are no TOCTOU issues. + let mut reader = UserSlice::new(UserPtr::from_addr(ptr as _), size).reader(); + + for i in 0..count { + let mut sync: MaybeUninit = MaybeUninit::uninit(); + + // SAFETY: The size of `sync` is STRIDE + reader.read_raw(unsafe { + core::slice::from_raw_parts_mut(sync.as_mut_ptr() as *mut MaybeUninit, STRIDE) + })?; + + // SAFETY: All bit patterns in the struct are valid + let sync = unsafe { sync.assume_init() }; + + vec.push(SyncItem::parse_one(file, sync, i >= in_count)?, GFP_KERNEL)?; + } + + Ok(vec) + } +} + +#[derive(Clone)] +pub(crate) enum Object { + TimestampBuffer(Arc), +} + +/// State associated with a client. +// #[pin_data] +pub(crate) struct File { + id: u64, + // #[pin] + vms: xarray::XArray>, + // #[pin] + queues: xarray::XArray>>>, + // #[pin] + objects: xarray::XArray>, +} + +/// Convenience type alias for our DRM `File` type. +pub(crate) type DrmFile = drm::File; + +/// Available VM range for the user +const VM_USER_RANGE: Range = mmu::IOVA_USER_USABLE_RANGE; + +/// Minimum reserved AS for kernel mappings +const VM_KERNEL_MIN_SIZE: u64 = 0x20000000; + +impl drm::file::DriverFile for File { + type Driver = driver::AsahiDriver; + + /// Create a new `File` instance for a fresh client. + fn open(device: &AsahiDevice) -> Result>> { + debug::update_debug_flags(); + + let gpu = &device.gpu; + let id = gpu.ids().file.next(); + + mod_dev_dbg!(device, "[File {}]: DRM device opened\n", id); + Ok(KBox::pin_init(File::new(id), GFP_KERNEL)?) + } + + fn as_raw(&self) -> *mut bindings::drm_file { + todo!() + } +} + +// SAFETY: All bit patterns are valid by construction. +unsafe impl AnyBitPattern for uapi::drm_asahi_gem_bind_op {} + +impl File { + fn new(id: u64) -> impl PinInit { + unsafe { + pin_init::pin_init_from_closure(move |slot: *mut Self| { + let raw_vms = addr_of_mut!((*slot).vms); + xarray::XArray::>::new(xarray::AllocKind::Alloc1) + .__pinned_init(raw_vms)?; + + let raw_queues = addr_of_mut!((*slot).queues); + xarray::XArray::>>>::new( + xarray::AllocKind::Alloc1, + ) + .__pinned_init(raw_queues)?; + + let raw_objects = addr_of_mut!((*slot).objects); + xarray::XArray::>::new(xarray::AllocKind::Alloc1) + .__pinned_init(raw_objects)?; + + (*slot).id = id; + Ok(()) + }) + } + } + + fn vms(self: Pin<&Self>) -> Pin<&xarray::XArray>> { + // SAFETY: Structural pinned projection for vms. + // We never move out of this field. + unsafe { self.map_unchecked(|s| &s.vms) } + } + + #[allow(clippy::type_complexity)] + fn queues(self: Pin<&Self>) -> Pin<&xarray::XArray>>>> { + // SAFETY: Structural pinned projection for queues. + // We never move out of this field. + unsafe { self.map_unchecked(|s| &s.queues) } + } + + fn objects(self: Pin<&Self>) -> Pin<&xarray::XArray>> { + // SAFETY: Structural pinned projection for objects. + // We never move out of this field. + unsafe { self.map_unchecked(|s| &s.objects) } + } + + /// IOCTL: get_param: Get a driver parameter value. + pub(crate) fn get_params( + device: &AsahiDevice, + data: &uapi::drm_asahi_get_params, + file: &DrmFile, + ) -> Result { + mod_dev_dbg!(device, "[File {}]: IOCTL: get_params\n", file.inner().id); + + let gpu = &device.gpu; + + if data.param_group != 0 || data.pad != 0 { + cls_pr_debug!(Errors, "get_params: Invalid arguments\n"); + return Err(EINVAL); + } + + if gpu.is_crashed() { + return Err(ENODEV); + } + + let mut params = uapi::drm_asahi_params_global { + features: 0, + + gpu_generation: gpu.get_dyncfg().id.gpu_gen as u32, + gpu_variant: gpu.get_dyncfg().id.gpu_variant as u32, + gpu_revision: gpu.get_dyncfg().id.gpu_rev as u32, + chip_id: gpu.get_cfg().chip_id, + + num_dies: gpu.get_cfg().num_dies, + num_clusters_total: gpu.get_dyncfg().id.num_clusters, + num_cores_per_cluster: gpu.get_dyncfg().id.num_cores, + core_masks: [0; uapi::DRM_ASAHI_MAX_CLUSTERS as usize], + + vm_start: VM_USER_RANGE.start, + vm_end: VM_USER_RANGE.end, + vm_kernel_min_size: VM_KERNEL_MIN_SIZE, + + max_commands_per_submission: MAX_COMMANDS_PER_SUBMISSION, + max_attachments: crate::microseq::MAX_ATTACHMENTS as u32, + max_frequency_khz: gpu.get_dyncfg().pwr.max_frequency_khz(), + + command_timestamp_frequency_hz: 1_000_000_000, // User timestamps always in nanoseconds + }; + + for (i, mask) in gpu.get_dyncfg().id.core_masks.iter().enumerate() { + *(params.core_masks.get_mut(i).ok_or(EIO)?) = (*mask).into(); + } + + if *module_parameters::fault_control.value() == 0xb { + params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS as u64; + } + + let size = core::mem::size_of::().min(data.size.try_into()?); + + // SAFETY: We only write to this userptr once, so there are no TOCTOU issues. + let mut params_writer = + UserSlice::new(UserPtr::from_addr(data.pointer as _), size).writer(); + + // SAFETY: `size` is at most the sizeof of `params` + params_writer.write_slice(unsafe { + core::slice::from_raw_parts(¶ms as *const _ as *const u8, size) + })?; + + Ok(0) + } + + /// IOCTL: vm_create: Create a new `Vm`. + pub(crate) fn vm_create( + device: &AsahiDevice, + data: &mut uapi::drm_asahi_vm_create, + file: &DrmFile, + ) -> Result { + let kernel_range = data.kernel_start..data.kernel_end; + + // Validate requested kernel range + if !VM_USER_RANGE.is_superset(kernel_range.clone()) + || kernel_range.range() < VM_KERNEL_MIN_SIZE + || kernel_range.start & (mmu::UAT_PGMSK as u64) != 0 + || kernel_range.end & (mmu::UAT_PGMSK as u64) != 0 + { + cls_pr_debug!(Errors, "vm_create: Invalid kernel range\n"); + return Err(EINVAL); + } + + // Align to buffer::PAGE_SIZE so the allocators are happy + let kernel_range = align(kernel_range.start, buffer::PAGE_SIZE as u64) + ..align_down(kernel_range.end, buffer::PAGE_SIZE as u64); + + let kernel_half_size = align_down(kernel_range.range() >> 1, buffer::PAGE_SIZE as u64); + let kernel_gpu_range = kernel_range.start..(kernel_range.start + kernel_half_size); + let kernel_gpufw_range = kernel_gpu_range.end..kernel_range.end; + + let gpu = &device.gpu; + let file_id = file.inner().id; + let vm = gpu.new_vm(kernel_range.clone())?; + + let vm_xa = file.inner().vms(); + let resv = vm_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; + let id: u32 = resv.index().try_into()?; + + mod_dev_dbg!(device, "[File {} VM {}]: VM Create\n", file_id, id); + mod_dev_dbg!( + device, + "[File {} VM {}]: Creating allocators\n", + file_id, + id + ); + let ualloc = Arc::pin_init( + new_mutex!(alloc::DefaultAllocator::new( + device, + &vm, + kernel_gpu_range, + buffer::PAGE_SIZE, + mmu::PROT_GPU_SHARED_RW, + 512 * 1024, + true, + fmt!("File {} VM {} GPU Shared", file_id, id), + false, + )?), + GFP_KERNEL, + )?; + let ualloc_priv = Arc::pin_init( + new_mutex!(alloc::DefaultAllocator::new( + device, + &vm, + kernel_gpufw_range, + buffer::PAGE_SIZE, + mmu::PROT_GPU_FW_PRIV_RW, + 64 * 1024, + true, + fmt!("File {} VM {} GPU FW Private", file_id, id), + false, + )?), + GFP_KERNEL, + )?; + + mod_dev_dbg!( + device, + "[File {} VM {}]: Creating dummy object\n", + file_id, + id + ); + let mut dummy_obj = gem::new_kernel_object(device, 0x4000)?; + dummy_obj.vmap()?.memset(0); + let dummy_mapping = + dummy_obj.map_at(&vm, mmu::IOVA_UNK_PAGE, mmu::PROT_GPU_SHARED_RW, true)?; + + mod_dev_dbg!(device, "[File {} VM {}]: VM created\n", file_id, id); + resv.fill(KBox::new( + Vm { + ualloc, + ualloc_priv, + vm, + kernel_range, + _dummy_mapping: dummy_mapping, + }, + GFP_KERNEL, + )?)?; + + data.vm_id = id; + + Ok(0) + } + + /// IOCTL: vm_destroy: Destroy a `Vm`. + pub(crate) fn vm_destroy( + _device: &AsahiDevice, + data: &mut uapi::drm_asahi_vm_destroy, + file: &DrmFile, + ) -> Result { + let vm = file.inner().vms().remove(data.vm_id as usize); + if vm.is_none() { + Err(ENOENT) + } else { + Ok(0) + } + } + + /// IOCTL: gem_create: Create a new GEM object. + pub(crate) fn gem_create( + device: &AsahiDevice, + data: &mut uapi::drm_asahi_gem_create, + file: &DrmFile, + ) -> Result { + mod_dev_dbg!( + device, + "[File {}]: IOCTL: gem_create size={:#x?}\n", + file.inner().id, + data.size + ); + + if (data.flags + & !(uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_WRITEBACK + | uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE)) + != 0 + || (data.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE == 0 + && data.vm_id != 0) + { + cls_pr_debug!(Errors, "gem_create: Invalid arguments\n"); + return Err(EINVAL); + } + + let resv_gem; + let resv_obj = if data.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 { + resv_gem = file + .inner() + .vms() + .lock() + .get(data.vm_id.try_into()?) + .ok_or(ENOENT)? + .vm + .get_resv_obj(); + Some(resv_gem.deref()) + } else { + None + }; + + let gem = gem::new_object(device, data.size.try_into()?, data.flags, resv_obj)?; + + let handle = gem.create_handle(file)?; + data.handle = handle; + + mod_dev_dbg!( + device, + "[File {}]: IOCTL: gem_create size={:#x} handle={:#x?}\n", + file.inner().id, + data.size, + data.handle + ); + + Ok(0) + } + + /// IOCTL: gem_mmap_offset: Assign an mmap offset to a GEM object. + pub(crate) fn gem_mmap_offset( + device: &AsahiDevice, + data: &mut uapi::drm_asahi_gem_mmap_offset, + file: &DrmFile, + ) -> Result { + mod_dev_dbg!( + device, + "[File {}]: IOCTL: gem_mmap_offset handle={:#x?}\n", + file.inner().id, + data.handle + ); + + if data.flags != 0 { + cls_pr_debug!(Errors, "gem_mmap_offset: Unexpected flags\n"); + return Err(EINVAL); + } + + let gem = gem::Object::lookup_handle(file, data.handle)?; + data.offset = gem.create_mmap_offset()?; + Ok(0) + } + + /// IOCTL: vm_bind: Map or unmap memory into a Vm. + pub(crate) fn vm_bind( + device: &AsahiDevice, + data: &uapi::drm_asahi_vm_bind, + file: &DrmFile, + ) -> Result { + mod_dev_dbg!( + device, + "[File {} VM {}]: IOCTL: vm_bind\n", + file.inner().id, + data.vm_id, + ); + + if data.stride == 0 || data.pad != 0 { + cls_pr_debug!(Errors, "vm_bind: Unexpected headers\n"); + return Err(EINVAL); + } + + let vm_id = data.vm_id.try_into()?; + + let mut vec = KVec::new(); + let size = (data.stride * data.num_binds) as usize; + let reader = UserSlice::new(UserPtr::from_addr(data.userptr as _), size).reader(); + reader.read_all(&mut vec, GFP_KERNEL)?; + let mut reader = Reader::new(&vec); + + for _i in 0..data.num_binds { + let bind: uapi::drm_asahi_gem_bind_op = reader.read_up_to(data.stride as usize)?; + Self::do_gem_bind_unbind(vm_id, &bind, file)?; + } + + Ok(0) + } + + pub(crate) fn do_gem_bind_unbind( + vm_id: usize, + data: &uapi::drm_asahi_gem_bind_op, + file: &DrmFile, + ) -> Result { + if (data.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_UNBIND) != 0 { + Self::do_gem_unbind(vm_id, data, file) + } else { + Self::do_gem_bind(vm_id, data, file) + } + } + + pub(crate) fn do_gem_bind( + vm_id: usize, + data: &uapi::drm_asahi_gem_bind_op, + file: &DrmFile, + ) -> Result { + if (data.addr | data.range | data.offset) as usize & mmu::UAT_PGMSK != 0 { + cls_pr_debug!( + Errors, + "gem_bind: Addr/range/offset not page aligned: {:#x} {:#x}\n", + data.addr, + data.range + ); + return Err(EINVAL); // Must be page aligned + } + + if (data.flags + & !(uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_READ + | uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_WRITE + | uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_SINGLE_PAGE)) + != 0 + { + cls_pr_debug!(Errors, "gem_bind: Invalid flags {:#x}\n", data.flags); + return Err(EINVAL); + } + + let single_page = data.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_SINGLE_PAGE != 0; + + let bo = gem::Object::lookup_handle(file, data.handle)?; + + let start = data.addr; + let end = data.addr.checked_add(data.range).ok_or(EINVAL)?; + let range = start..end; + + let bo_accessed_size = if single_page { + mmu::UAT_PGMSK as u64 + } else { + data.range + }; + let end_off = data.offset.checked_add(bo_accessed_size).ok_or(EINVAL)?; + if end_off as usize > bo.size() { + return Err(EINVAL); + } + + if !VM_USER_RANGE.is_superset(range.clone()) { + cls_pr_debug!( + Errors, + "gem_bind: Invalid map range {:#x}..{:#x} (not contained in user range)\n", + start, + end + ); + return Err(EINVAL); // Invalid map range + } + + let prot = if data.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_READ != 0 { + if data.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_WRITE != 0 { + mmu::PROT_GPU_SHARED_RW + } else { + mmu::PROT_GPU_SHARED_RO + } + } else if data.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_WRITE != 0 { + mmu::PROT_GPU_SHARED_WO + } else { + cls_pr_debug!( + Errors, + "gem_bind: Must specify read or write (flags: {:#x})\n", + data.flags + ); + return Err(EINVAL); // Must specify one of DRM_ASAHI_BIND_{READ,WRITE} + }; + + let vms_xa = file.inner().vms(); + let guard = vms_xa.lock(); + let guarded_vm = guard.get(vm_id).ok_or(ENOENT)?; + + // Clone it immediately so we aren't holding the XArray lock + let vm = guarded_vm.vm.clone(); + let kernel_range = guarded_vm.kernel_range.clone(); + let _ = guarded_vm; + core::mem::drop(guard); + + if kernel_range.overlaps(range) { + cls_pr_debug!( + Errors, + "gem_bind: Invalid map range {:#x}..{:#x} (intrudes in kernel range)\n", + start, + end + ); + return Err(EINVAL); + } + + vm.bind_object(&bo, data.addr, data.range, data.offset, prot, single_page)?; + + vm.bo_deferred_cleanup(); + + Ok(0) + } + + pub(crate) fn do_gem_unbind( + vm_id: usize, + data: &uapi::drm_asahi_gem_bind_op, + file: &DrmFile, + ) -> Result { + if data.offset != 0 + || data.flags != uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_UNBIND + || data.handle != 0 + { + cls_pr_debug!(Errors, "gem_unbind: offset/flags/handle not zero\n"); + return Err(EINVAL); + } + + if (data.addr | data.range) as usize & mmu::UAT_PGMSK != 0 { + cls_pr_debug!( + Errors, + "gem_bind: Addr/range/offset not page aligned: {:#x} {:#x}\n", + data.addr, + data.range + ); + return Err(EINVAL); // Must be page aligned + } + + let start = data.addr; + let end = data.addr.checked_add(data.range).ok_or(EINVAL)?; + let range = start..end; + + if !VM_USER_RANGE.is_superset(range.clone()) { + cls_pr_debug!( + Errors, + "gem_bind: Invalid unmap range {:#x}..{:#x} (not contained in user range)\n", + start, + end + ); + return Err(EINVAL); // Invalid map range + } + + let vms_xa = file.inner().vms(); + let guard = vms_xa.lock(); + let guarded_vm = guard.get(vm_id).ok_or(ENOENT)?; + + // Clone it immediately so we aren't holding the XArray lock + let vm = guarded_vm.vm.clone(); + let kernel_range = guarded_vm.kernel_range.clone(); + let _ = guarded_vm; + core::mem::drop(guard); + + if kernel_range.overlaps(range.clone()) { + cls_pr_debug!( + Errors, + "gem_bind: Invalid unmap range {:#x}..{:#x} (intrudes in kernel range)\n", + start, + end + ); + return Err(EINVAL); + } + + vm.unmap_range(range.start, range.range())?; + + vm.bo_deferred_cleanup(); + + Ok(0) + } + + pub(crate) fn unbind_gem_object(file: &DrmFile, bo: &gem::Object) -> Result { + // TODO: use iter() + let mut index = 0; + loop { + let vms = file.inner().vms(); + let item = vms.find(index, usize::MAX); + match item { + Some((idx, file_vm)) => { + // Clone since we can't hold the xarray spinlock while + // calling drop_mappings() + let vm = file_vm.borrow().vm.clone(); + core::mem::drop(file_vm); + vm.drop_mappings(bo)?; + if idx == usize::MAX { + break; + } + index = idx + 1; + } + None => break, + } + } + Ok(()) + } + + /// IOCTL: gem_bind_object: Map or unmap a GEM object as a special object. + pub(crate) fn gem_bind_object( + device: &AsahiDevice, + data: &mut uapi::drm_asahi_gem_bind_object, + file: &DrmFile, + ) -> Result { + mod_dev_dbg!( + device, + "[File {} VM {}]: IOCTL: gem_bind_object op={:?} handle={:#x?} flags={:#x?} {:#x?}:{:#x?} object_handle={:#x?}\n", + file.inner().id, + data.vm_id, + data.op, + data.handle, + data.flags, + data.offset, + data.range, + data.object_handle + ); + + if data.pad != 0 { + cls_pr_debug!(Errors, "gem_bind_object: Unexpected pad\n"); + return Err(EINVAL); + } + + if data.vm_id != 0 { + cls_pr_debug!(Errors, "gem_bind_object: Unexpected vm_id\n"); + return Err(EINVAL); + } + + match data.op { + uapi::drm_asahi_bind_object_op_DRM_ASAHI_BIND_OBJECT_OP_BIND => { + Self::do_gem_bind_object(device, data, file) + } + uapi::drm_asahi_bind_object_op_DRM_ASAHI_BIND_OBJECT_OP_UNBIND => { + Self::do_gem_unbind_object(device, data, file) + } + _ => { + cls_pr_debug!(Errors, "gem_bind_object: Invalid op {}\n", data.op); + Err(EINVAL) + } + } + } + + pub(crate) fn do_gem_bind_object( + device: &AsahiDevice, + data: &mut uapi::drm_asahi_gem_bind_object, + file: &DrmFile, + ) -> Result { + if (data.range | data.offset) as usize & mmu::UAT_PGMSK != 0 { + cls_pr_debug!( + Errors, + "gem_bind_object: Range/offset not page aligned: {:#x} {:#x}\n", + data.range, + data.offset + ); + return Err(EINVAL); // Must be page aligned + } + + if data.flags != uapi::drm_asahi_bind_object_flags_DRM_ASAHI_BIND_OBJECT_USAGE_TIMESTAMPS { + cls_pr_debug!(Errors, "gem_bind_object: Invalid flags {:#x}\n", data.flags); + return Err(EINVAL); + } + + let offset = data.offset.try_into()?; + let end_offset = data + .offset + .checked_add(data.range) + .ok_or(EINVAL)? + .try_into()?; + let bo = gem::ObjectRef::new(gem::Object::lookup_handle(file, data.handle)?); + + let mapping = Arc::new( + device.gpu.map_timestamp_buffer(bo, offset..end_offset)?, + GFP_KERNEL, + )?; + let obj = KBox::new(Object::TimestampBuffer(mapping), GFP_KERNEL)?; + let handle = file + .inner() + .objects() + .lock() + .insert_limit(1..=u32::MAX, obj, GFP_KERNEL)? as u64; + + data.object_handle = handle as u32; + Ok(0) + } + + pub(crate) fn do_gem_unbind_object( + _device: &AsahiDevice, + data: &mut uapi::drm_asahi_gem_bind_object, + file: &DrmFile, + ) -> Result { + if data.range != 0 || data.offset != 0 { + cls_pr_debug!( + Errors, + "gem_unbind_object: Range/offset not zero: {:#x} {:#x}\n", + data.range, + data.offset + ); + return Err(EINVAL); + } + + if data.flags != 0 { + cls_pr_debug!( + Errors, + "gem_unbind_object: Invalid flags {:#x}\n", + data.flags + ); + return Err(EINVAL); + } + + if data.handle != 0 { + cls_pr_debug!( + Errors, + "gem_unbind_object: Invalid handle {}\n", + data.handle + ); + return Err(EINVAL); + } + + let object = file.inner().objects().remove(data.object_handle as usize); + if object.is_none() { + Err(ENOENT) + } else { + Ok(0) + } + } + + /// IOCTL: queue_create: Create a new command submission queue of a given type. + pub(crate) fn queue_create( + device: &AsahiDevice, + data: &mut uapi::drm_asahi_queue_create, + file: &DrmFile, + ) -> Result { + let file_id = file.inner().id; + + mod_dev_dbg!( + device, + "[File {} VM {}]: Creating queue prio={:?} flags={:#x?}\n", + file_id, + data.vm_id, + data.priority, + data.flags, + ); + + if data.flags != 0 || data.priority > uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME { + cls_pr_debug!(Errors, "queue_create: Invalid arguments\n"); + return Err(EINVAL); + } + + // TODO: Allow with CAP_SYS_NICE + if data.priority >= uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH { + cls_pr_debug!(Errors, "queue_create: Invalid priority\n"); + return Err(EINVAL); + } + + let queues_xa = file.inner().queues(); + let resv = queues_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; + let vms_xa = file.inner().vms(); + let guard = vms_xa.lock(); + let file_vm = guard.get(data.vm_id.try_into()?).ok_or(ENOENT)?; + let vm = file_vm.vm.clone(); + let ualloc = file_vm.ualloc.clone(); + let ualloc_priv = file_vm.ualloc_priv.clone(); + // Drop the vms lock eagerly + let _ = file_vm; + core::mem::drop(guard); + + let queue = device.gpu.new_queue( + vm, + ualloc, + ualloc_priv, + // TODO: Plumb deeper the enum + uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME - data.priority, + data.usc_exec_base, + )?; + + data.queue_id = resv.index().try_into()?; + resv.fill(Arc::pin_init(new_mutex!(queue), GFP_KERNEL)?)?; + + Ok(0) + } + + /// IOCTL: queue_destroy: Destroy a command submission queue. + pub(crate) fn queue_destroy( + _device: &AsahiDevice, + data: &mut uapi::drm_asahi_queue_destroy, + file: &DrmFile, + ) -> Result { + // grab the queue so the xarray spinlock is dropped first + let queue = file.inner().queues().remove(data.queue_id as usize); + if queue.is_none() { + Err(ENOENT) + } else { + Ok(0) + } + } + + /// IOCTL: submit: Submit GPU work to a command submission queue. + pub(crate) fn submit( + device: &AsahiDevice, + data: &mut uapi::drm_asahi_submit, + file: &DrmFile, + ) -> Result { + debug::update_debug_flags(); + + if data.flags != 0 || data.pad != 0 { + cls_pr_debug!(Errors, "submit: Invalid arguments\n"); + return Err(EINVAL); + } + + let gpu = &device.gpu; + gpu.update_globals(); + + // Upgrade to Arc to drop the XArray lock early + let queue: Arc>> = file + .inner() + .queues() + .lock() + .get(data.queue_id.try_into()?) + .ok_or(ENOENT)? + .into(); + + let id = gpu.ids().submission.next(); + mod_dev_dbg!( + device, + "[File {} Queue {}]: IOCTL: submit (submission ID: {})\n", + file.inner().id, + data.queue_id, + id + ); + + mod_dev_dbg!( + device, + "[File {} Queue {}]: IOCTL: submit({}): Parsing syncs\n", + file.inner().id, + data.queue_id, + id + ); + let syncs = + SyncItem::parse_array(file, data.syncs, data.in_sync_count, data.out_sync_count)?; + + mod_dev_dbg!( + device, + "[File {} Queue {}]: IOCTL: submit({}): Parsing commands\n", + file.inner().id, + data.queue_id, + id + ); + + let mut vec = KVec::new(); + + // Copy the command buffer into the kernel. Because we need to iterate + // the command buffer twice, we do this in one big copy_from_user to + // avoid TOCTOU issues. + let reader = UserSlice::new( + UserPtr::from_addr(data.cmdbuf as _), + data.cmdbuf_size as usize, + ) + .reader(); + reader.read_all(&mut vec, GFP_KERNEL)?; + + let objects = file.inner().objects(); + let ret = queue + .lock() + .submit(id, syncs, data.in_sync_count as usize, &vec, objects); + + match ret { + Err(ERESTARTSYS) => Err(ERESTARTSYS), + Err(e) => { + dev_info!( + device.as_ref(), + "[File {} Queue {}]: IOCTL: submit failed! (submission ID: {} err: {:?})\n", + file.inner().id, + data.queue_id, + id, + e + ); + Err(e) + } + Ok(()) => Ok(0), + } + } + + /// IOCTL: get_time: Get the current GPU timer value. + pub(crate) fn get_time( + device: &AsahiDevice, + data: &mut uapi::drm_asahi_get_time, + _file: &DrmFile, + ) -> Result { + if data.flags != 0 { + cls_pr_debug!(Errors, "get_time: Unexpected flags\n"); + return Err(EINVAL); + } + + // TODO: Do this on device-init for perf. + let gpu = &device.gpu; + let frequency_hz = gpu.get_cfg().base_clock_hz as u64; + let ts_gcd = gcd(frequency_hz, NSEC_PER_SEC as u64); + + let num = (NSEC_PER_SEC as u64) / ts_gcd; + let den = frequency_hz / ts_gcd; + + let raw: u64; + + // SAFETY: Assembly only loads the timer + unsafe { + core::arch::asm!( + "mrs {x}, CNTPCT_EL0", + x = out(reg) raw + ); + } + + data.gpu_timestamp = (raw * num) / den; + + Ok(0) + } +} + +impl Drop for File { + fn drop(&mut self) { + mod_pr_debug!("[File {}]: Closing...\n", self.id); + } +} diff --git a/drivers/gpu/drm/asahi/float.rs b/drivers/gpu/drm/asahi/float.rs new file mode 100644 index 00000000000000..d58a3d284da124 --- /dev/null +++ b/drivers/gpu/drm/asahi/float.rs @@ -0,0 +1,392 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Basic soft floating-point support +//! +//! The GPU firmware requires a large number of power-related configuration values, many of which +//! are IEEE 754 32-bit floating point values. These values change not only between GPU/SoC +//! variants, but also between specific hardware platforms using these SoCs, so they must be +//! derived from device tree properties. There are many redundant values computed from the same +//! inputs with simple add/sub/mul/div calculations, plus a few values that are actually specific +//! to each individual device depending on its binning and fused voltage configuration, so it +//! doesn't make sense to store the final values to be passed to the firmware in the device tree. +//! +//! Therefore, we need a way to perform floating-point calculations in the kernel. +//! +//! Using the actual FPU from kernel mode is asking for trouble, since there is no way to bound +//! the execution of FPU instructions to a controlled section of code without outright putting it +//! in its own compilation unit, which is quite painful for Rust. Since these calculations only +//! have to happen at initialization time and there is no need for performance, let's use a simple +//! software float implementation instead. +//! +//! This implementation makes no attempt to be fully IEEE754 compliant, but it's good enough and +//! gives bit-identical results to macOS in the vast majority of cases, with one or two exceptions +//! related to slightly non-compliant rounding. + +use core::ops; +use kernel::{ + of, + prelude::*, // +}; + +/// An IEEE754-compatible floating point number implemented in software. +#[derive(Default, Debug, Copy, Clone)] +#[repr(transparent)] +pub(crate) struct F32(u32); + +// SAFETY: F32 is a transparent repr of `u32` and therefore zeroable +unsafe impl Zeroable for F32 {} + +#[derive(Default, Debug, Copy, Clone)] +struct F32U { + sign: bool, + exp: i32, + frac: i64, +} + +impl F32 { + /// Convert a raw 32-bit representation into an F32 + pub(crate) const fn from_bits(u: u32) -> F32 { + F32(u) + } + + // Convert a `f32` value into an F32 + // + // This must ONLY be used in const context. Use the `f32!{}` macro to do it safely. + #[doc(hidden)] + pub(crate) const fn from_f32(v: f32) -> F32 { + // Replace with to_bits() after kernel Rust minreq is >= 1.83.0 + #[allow(clippy::transmute_float_to_int)] + #[allow(unnecessary_transmutes)] + // SAFETY: Transmuting f32 to u32 is always safe + F32(unsafe { core::mem::transmute::(v) }) + } + + // Convert an F32 into a `f32` value + // + // For testing only. + #[doc(hidden)] + #[cfg(test)] + pub(crate) fn to_f32(self) -> f32 { + f32::from_bits(self.0) + } + + const fn unpack(&self) -> F32U { + F32U { + sign: self.0 & (1 << 31) != 0, + exp: ((self.0 >> 23) & 0xff) as i32 - 127, + frac: (((self.0 & 0x7fffff) | 0x800000) as i64) << 9, + } + .norm() + } +} + +/// Safely construct an `F32` out of a constant floating-point value. +/// +/// This ensures that the conversion happens in const context, so no floating point operations are +/// emitted. +#[macro_export] +macro_rules! f32 { + ([$($val:expr),*]) => {{ + [$(f32!($val)),*] + }}; + ($val:expr) => {{ + const _K: $crate::float::F32 = $crate::float::F32::from_f32($val); + _K + }}; +} + +impl ops::Neg for F32 { + type Output = F32; + + fn neg(self) -> F32 { + F32(self.0 ^ (1 << 31)) + } +} + +impl ops::Add for F32 { + type Output = F32; + + fn add(self, rhs: F32) -> F32 { + self.unpack().add(rhs.unpack()).pack() + } +} + +impl ops::Sub for F32 { + type Output = F32; + + fn sub(self, rhs: F32) -> F32 { + self.unpack().add((-rhs).unpack()).pack() + } +} + +impl ops::Mul for F32 { + type Output = F32; + + fn mul(self, rhs: F32) -> F32 { + self.unpack().mul(rhs.unpack()).pack() + } +} + +impl ops::Div for F32 { + type Output = F32; + + fn div(self, rhs: F32) -> F32 { + self.unpack().div(rhs.unpack()).pack() + } +} + +macro_rules! from_ints { + ($u:ty, $i:ty) => { + impl From<$i> for F32 { + fn from(v: $i) -> F32 { + F32U::from_i64(v as i64).pack() + } + } + impl From<$u> for F32 { + fn from(v: $u) -> F32 { + F32U::from_u64(v as u64).pack() + } + } + }; +} + +from_ints!(u8, i8); +from_ints!(u16, i16); +from_ints!(u32, i32); +from_ints!(u64, i64); + +impl F32U { + const INFINITY: F32U = f32!(f32::INFINITY).unpack(); + const NEG_INFINITY: F32U = f32!(f32::NEG_INFINITY).unpack(); + + fn from_i64(v: i64) -> F32U { + F32U { + sign: v < 0, + exp: 32, + frac: v.abs(), + } + .norm() + } + + fn from_u64(mut v: u64) -> F32U { + let mut exp = 32; + if v >= (1 << 63) { + exp = 31; + v >>= 1; + } + F32U { + sign: false, + exp, + frac: v as i64, + } + .norm() + } + + fn shr(&mut self, shift: i32) { + if shift > 63 { + self.exp = 0; + self.frac = 0; + } else { + self.frac >>= shift; + } + } + + fn align(a: &mut F32U, b: &mut F32U) { + if a.exp > b.exp { + b.shr(a.exp - b.exp); + b.exp = a.exp; + } else { + a.shr(b.exp - a.exp); + a.exp = b.exp; + } + } + + fn mul(self, other: F32U) -> F32U { + F32U { + sign: self.sign != other.sign, + exp: self.exp + other.exp, + frac: ((self.frac >> 8) * (other.frac >> 8)) >> 16, + } + } + + fn div(self, other: F32U) -> F32U { + if other.frac == 0 || self.is_inf() { + if self.sign { + F32U::NEG_INFINITY + } else { + F32U::INFINITY + } + } else { + F32U { + sign: self.sign != other.sign, + exp: self.exp - other.exp, + frac: ((self.frac << 24) / (other.frac >> 8)), + } + } + } + + fn add(mut self, mut other: F32U) -> F32U { + F32U::align(&mut self, &mut other); + if self.sign == other.sign { + self.frac += other.frac; + } else { + self.frac -= other.frac; + } + if self.frac < 0 { + self.sign = !self.sign; + self.frac = -self.frac; + } + self + } + + const fn norm(mut self) -> F32U { + let lz = self.frac.leading_zeros() as i32; + if lz > 31 { + self.frac <<= lz - 31; + self.exp -= lz - 31; + } else if lz < 31 { + self.frac >>= 31 - lz; + self.exp += 31 - lz; + } + + if self.is_zero() { + return F32U { + sign: self.sign, + frac: 0, + exp: 0, + }; + } + self + } + + const fn is_zero(&self) -> bool { + self.frac == 0 || self.exp < -126 + } + + const fn is_inf(&self) -> bool { + self.exp > 127 + } + + const fn pack(mut self) -> F32 { + self = self.norm(); + if !self.is_zero() { + self.frac += 0x100; + self = self.norm(); + } + + if self.is_inf() { + if self.sign { + return f32!(f32::NEG_INFINITY); + } else { + return f32!(f32::INFINITY); + } + } else if self.is_zero() { + if self.sign { + return f32!(-0.0); + } else { + return f32!(0.0); + } + } + + F32(if self.sign { 1u32 << 31 } else { 0u32 } + | ((self.exp + 127) as u32) << 23 + | ((self.frac >> 9) & 0x7fffff) as u32) + } +} + +impl<'a> TryFrom> for F32 { + type Error = Error; + + fn try_from(p: of::Property<'_>) -> core::result::Result { + let bits: u32 = p.try_into()?; + Ok(F32::from_bits(bits)) + } +} + +impl of::PropertyUnit for F32 { + const UNIT_SIZE: usize = 4; + + fn from_bytes(data: &[u8]) -> Result { + Ok(F32::from_bits(::from_bytes(data)?)) + } +} + +// TODO: Make this an actual test and figure out how to make it run. +#[cfg(test)] +mod tests { + #[test] + fn test_all() { + fn add(a: f32, b: f32) { + println!( + "{} + {} = {} {}", + a, + b, + (F32::from_f32(a) + F32::from_f32(b)).to_f32(), + a + b + ); + } + fn sub(a: f32, b: f32) { + println!( + "{} - {} = {} {}", + a, + b, + (F32::from_f32(a) - F32::from_f32(b)).to_f32(), + a - b + ); + } + fn mul(a: f32, b: f32) { + println!( + "{} * {} = {} {}", + a, + b, + (F32::from_f32(a) * F32::from_f32(b)).to_f32(), + a * b + ); + } + fn div(a: f32, b: f32) { + println!( + "{} / {} = {} {}", + a, + b, + (F32::from_f32(a) / F32::from_f32(b)).to_f32(), + a / b + ); + } + + fn test(a: f32, b: f32) { + add(a, b); + sub(a, b); + mul(a, b); + div(a, b); + } + + test(1.123, 7.567); + test(1.123, 1.456); + test(7.567, 1.123); + test(1.123, -7.567); + test(1.123, -1.456); + test(7.567, -1.123); + test(-1.123, -7.567); + test(-1.123, -1.456); + test(-7.567, -1.123); + test(1000.123, 0.001); + test(1000.123, 0.0000001); + test(0.0012, 1000.123); + test(0.0000001, 1000.123); + test(0., 0.); + test(0., 1.); + test(1., 0.); + test(1., 1.); + test(2., f32::INFINITY); + test(2., f32::NEG_INFINITY); + test(f32::INFINITY, 2.); + test(f32::NEG_INFINITY, 2.); + test(f32::NEG_INFINITY, 2.); + test(f32::MAX, 2.); + test(f32::MIN, 2.); + test(f32::MIN_POSITIVE, 2.); + test(2., f32::MAX); + test(2., f32::MIN); + test(2., f32::MIN_POSITIVE); + } +} diff --git a/drivers/gpu/drm/asahi/fw/buffer.rs b/drivers/gpu/drm/asahi/fw/buffer.rs new file mode 100644 index 00000000000000..b1f4974fd02902 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/buffer.rs @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU tiled vertex buffer control firmware structures + +use super::types::*; +use super::workqueue; +use crate::{ + default_zeroed, + no_debug, + trivial_gpustruct, // +}; +use kernel::sync::Arc; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct BlockControl { + pub(crate) total: AtomicU32, + pub(crate) wptr: AtomicU32, + pub(crate) unk: AtomicU32, + pub(crate) pad: Pad<0x34>, + } + default_zeroed!(BlockControl); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct Counter { + pub(crate) count: AtomicU32, + __pad: Pad<0x3c>, + } + default_zeroed!(Counter); + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct Stats { + pub(crate) max_pages: AtomicU32, + pub(crate) max_b: AtomicU32, + pub(crate) overflow_count: AtomicU32, + pub(crate) gpu_c: AtomicU32, + pub(crate) __pad0: Pad<0x10>, + pub(crate) reset: AtomicU32, + pub(crate) __pad1: Pad<0x1c>, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct Info<'a> { + pub(crate) gpu_counter: u32, + pub(crate) unk_4: u32, + pub(crate) last_id: i32, + pub(crate) cur_id: i32, + pub(crate) unk_10: u32, + pub(crate) gpu_counter2: u32, + pub(crate) unk_18: u32, + + #[ver(V < V13_0B4 || G >= G14X)] + pub(crate) unk_1c: u32, + + pub(crate) page_list: GpuPointer<'a, &'a [u32]>, + pub(crate) page_list_size: u32, + pub(crate) page_count: AtomicU32, + pub(crate) max_blocks: u32, + pub(crate) block_count: AtomicU32, + pub(crate) unk_38: u32, + pub(crate) block_list: GpuPointer<'a, &'a [u32]>, + pub(crate) block_ctl: GpuPointer<'a, super::BlockControl>, + pub(crate) last_page: AtomicU32, + pub(crate) gpu_page_ptr1: u32, + pub(crate) gpu_page_ptr2: u32, + pub(crate) unk_58: u32, + pub(crate) block_size: u32, + pub(crate) unk_60: U64, + pub(crate) counter: GpuPointer<'a, super::Counter>, + pub(crate) unk_70: u32, + pub(crate) unk_74: u32, + pub(crate) unk_78: u32, + pub(crate) unk_7c: u32, + pub(crate) unk_80: u32, + pub(crate) max_pages: u32, + pub(crate) max_pages_nomemless: u32, + pub(crate) unk_8c: u32, + pub(crate) unk_90: Array<0x30, u8>, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct Scene<'a> { + #[ver(G >= G14X)] + pub(crate) control_word: GpuPointer<'a, &'a [u32]>, + #[ver(G >= G14X)] + pub(crate) control_word2: GpuPointer<'a, &'a [u32]>, + pub(crate) pass_page_count: AtomicU32, + pub(crate) unk_4: u32, + pub(crate) unk_8: U64, + pub(crate) unk_10: U64, + pub(crate) user_buffer: GpuPointer<'a, &'a [u8]>, + pub(crate) unk_20: u32, + #[ver(V >= V13_3)] + pub(crate) unk_28: U64, + pub(crate) stats: GpuWeakPointer, + pub(crate) total_page_count: AtomicU32, + #[ver(G < G14X)] + pub(crate) unk_30: U64, // pad + #[ver(G < G14X)] + pub(crate) unk_38: U64, // pad + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct InitBuffer<'a> { + pub(crate) tag: workqueue::CommandType, + pub(crate) vm_slot: u32, + pub(crate) buffer_slot: u32, + pub(crate) unk_c: u32, + pub(crate) block_count: u32, + pub(crate) buffer: GpuPointer<'a, super::Info::ver>, + pub(crate) stamp_value: EventValue, + } +} + +trivial_gpustruct!(BlockControl); +trivial_gpustruct!(Counter); +trivial_gpustruct!(Stats); + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct Info { + pub(crate) block_ctl: GpuObject, + pub(crate) counter: GpuObject, + pub(crate) page_list: GpuArray, + pub(crate) block_list: GpuArray, +} + +#[versions(AGX)] +impl GpuStruct for Info::ver { + type Raw<'a> = raw::Info::ver<'a>; +} + +pub(crate) struct ClusterBuffers { + pub(crate) tilemaps: GpuArray, + pub(crate) meta: GpuArray, +} + +#[versions(AGX)] +pub(crate) struct Scene { + pub(crate) user_buffer: GpuArray, + pub(crate) buffer: crate::buffer::Buffer::ver, + pub(crate) tvb_heapmeta: GpuArray, + pub(crate) tvb_tilemap: GpuArray, + pub(crate) tpc: Arc>, + pub(crate) clustering: Option, + pub(crate) preempt_buf: GpuArray, + #[ver(G >= G14X)] + pub(crate) control_word: GpuArray, +} + +#[versions(AGX)] +no_debug!(Scene::ver); + +#[versions(AGX)] +impl GpuStruct for Scene::ver { + type Raw<'a> = raw::Scene::ver<'a>; +} + +#[versions(AGX)] +pub(crate) struct InitBuffer { + pub(crate) scene: Arc, +} + +#[versions(AGX)] +no_debug!(InitBuffer::ver); + +#[versions(AGX)] +impl workqueue::Command for InitBuffer::ver {} + +#[versions(AGX)] +impl GpuStruct for InitBuffer::ver { + type Raw<'a> = raw::InitBuffer::ver<'a>; +} diff --git a/drivers/gpu/drm/asahi/fw/channels.rs b/drivers/gpu/drm/asahi/fw/channels.rs new file mode 100644 index 00000000000000..c1a7ec82aad1e2 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/channels.rs @@ -0,0 +1,443 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU communication channel firmware structures (ring buffers) + +use super::types::*; +use crate::default_zeroed; +use core::sync::atomic::Ordering; +use kernel::static_assert; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct ChannelState<'a> { + pub(crate) read_ptr: AtomicU32, + __pad0: Pad<0x1c>, + pub(crate) write_ptr: AtomicU32, + __pad1: Pad<0xc>, + _p: PhantomData<&'a ()>, + } + default_zeroed!(<'a>, ChannelState<'a>); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct FwCtlChannelState<'a> { + pub(crate) read_ptr: AtomicU32, + __pad0: Pad<0xc>, + pub(crate) write_ptr: AtomicU32, + __pad1: Pad<0xc>, + _p: PhantomData<&'a ()>, + } + default_zeroed!(<'a>, FwCtlChannelState<'a>); +} + +pub(crate) trait RxChannelState: GpuStruct + Debug + Default +where + for<'a> ::Raw<'a>: Default + Zeroable, +{ + const SUB_CHANNELS: usize; + + fn wptr(raw: &Self::Raw<'_>, index: usize) -> u32; + fn set_rptr(raw: &Self::Raw<'_>, index: usize, rptr: u32); +} + +#[derive(Debug, Default)] +pub(crate) struct ChannelState {} + +impl GpuStruct for ChannelState { + type Raw<'a> = raw::ChannelState<'a>; +} + +impl RxChannelState for ChannelState { + const SUB_CHANNELS: usize = 1; + + fn wptr(raw: &Self::Raw<'_>, _index: usize) -> u32 { + raw.write_ptr.load(Ordering::Acquire) + } + + fn set_rptr(raw: &Self::Raw<'_>, _index: usize, rptr: u32) { + raw.read_ptr.store(rptr, Ordering::Release); + } +} + +#[derive(Debug, Default)] +pub(crate) struct FwLogChannelState {} + +impl GpuStruct for FwLogChannelState { + type Raw<'a> = Array<6, raw::ChannelState<'a>>; +} + +impl RxChannelState for FwLogChannelState { + const SUB_CHANNELS: usize = 6; + + fn wptr(raw: &Self::Raw<'_>, index: usize) -> u32 { + raw[index].write_ptr.load(Ordering::Acquire) + } + + fn set_rptr(raw: &Self::Raw<'_>, index: usize, rptr: u32) { + raw[index].read_ptr.store(rptr, Ordering::Release); + } +} + +#[derive(Debug, Default)] +pub(crate) struct FwCtlChannelState {} + +impl GpuStruct for FwCtlChannelState { + type Raw<'a> = raw::FwCtlChannelState<'a>; +} + +pub(crate) trait TxChannelState: GpuStruct + Debug + Default { + fn rptr(raw: &Self::Raw<'_>) -> u32; + fn set_wptr(raw: &Self::Raw<'_>, wptr: u32); +} + +impl TxChannelState for ChannelState { + fn rptr(raw: &Self::Raw<'_>) -> u32 { + raw.read_ptr.load(Ordering::Acquire) + } + + fn set_wptr(raw: &Self::Raw<'_>, wptr: u32) { + raw.write_ptr.store(wptr, Ordering::Release); + } +} + +impl TxChannelState for FwCtlChannelState { + fn rptr(raw: &Self::Raw<'_>) -> u32 { + raw.read_ptr.load(Ordering::Acquire) + } + + fn set_wptr(raw: &Self::Raw<'_>, wptr: u32) { + raw.write_ptr.store(wptr, Ordering::Release); + } +} + +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum PipeType { + #[default] + Vertex = 0, + Fragment = 1, + Compute = 2, +} + +#[versions(AGX)] +#[derive(Debug, Copy, Clone, Default)] +#[repr(C)] +pub(crate) struct RunWorkQueueMsg { + pub(crate) pipe_type: PipeType, + pub(crate) work_queue: Option>, + pub(crate) wptr: u32, + pub(crate) event_slot: u32, + pub(crate) is_new: bool, + #[ver(V >= V13_2 && G == G14)] + pub(crate) __pad: Pad<0x2b>, + #[ver(V < V13_2 || G != G14)] + pub(crate) __pad: Pad<0x1b>, +} + +#[versions(AGX)] +pub(crate) type PipeMsg = RunWorkQueueMsg::ver; + +#[versions(AGX)] +pub(crate) const DEVICECONTROL_SZ: usize = { + #[ver(V < V13_2 || G != G14)] + { + 0x2c + } + #[ver(V >= V13_2 && G == G14)] + { + 0x3c + } +}; + +// TODO: clean up when arbitrary_enum_discriminant is stable +// https://github.com/rust-lang/rust/issues/60553 + +#[versions(AGX)] +#[derive(Debug, Copy, Clone)] +#[repr(C, u32)] +#[allow(dead_code)] +pub(crate) enum DeviceControlMsg { + Unk00(Array), + Unk01(Array), + Unk02(Array), + Unk03(Array), + Unk04(Array), + Unk05(Array), + Unk06(Array), + Unk07(Array), + Unk08(Array), + Unk09(Array), + Unk0a(Array), + Unk0b(Array), + Unk0c(Array), + #[ver(V >= V13_3)] + Unk0d(Array), + GrowTVBAck { + unk_4: u32, + buffer_slot: u32, + vm_slot: u32, + counter: u32, + subpipe: u32, + halt_count: U64, + __pad: Pad<{ DEVICECONTROL_SZ::ver - 0x1c }>, + }, + RecoverChannel { + pipe_type: u32, + work_queue: GpuWeakPointer, + event_value: u32, + __pad: Pad<{ DEVICECONTROL_SZ::ver - 0x10 }>, + }, + IdlePowerOff { + val: u32, + __pad: Pad<{ DEVICECONTROL_SZ::ver - 0x4 }>, + }, + Unk10(Array), + Unk11(Array), + Unk12(Array), + Unk13(Array), + Unk14(Array), // Init? + Unk15(Array), // Enable something + Unk16(Array), // Disable something + DestroyContext { + unk_4: u32, + ctx_23: u8, + #[ver(V < V13_3)] + __pad0: Pad<3>, + unk_c: U32, + unk_10: U32, + ctx_0: u8, + ctx_1: u8, + ctx_4: u8, + #[ver(V < V13_3)] + __pad1: Pad<1>, + #[ver(V < V13_3)] + unk_18: u32, + gpu_context: Option>, + #[ver(V < V13_3)] + __pad2: Pad<{ DEVICECONTROL_SZ::ver - 0x20 }>, + #[ver(V >= V13_3)] + __pad2: Pad<{ DEVICECONTROL_SZ::ver - 0x18 }>, + }, + Unk18(Array), + Initialize(Pad), // Update RegionC +} + +#[versions(AGX)] +static_assert!(core::mem::size_of::() == 4 + DEVICECONTROL_SZ::ver); + +#[versions(AGX)] +default_zeroed!(DeviceControlMsg::ver); + +#[derive(Copy, Clone, Default, Debug)] +#[repr(C)] +#[allow(dead_code)] +pub(crate) struct FwCtlMsg { + pub(crate) addr: U64, + pub(crate) unk_8: u32, + pub(crate) slot: u32, + pub(crate) page_count: u16, + pub(crate) unk_12: u16, +} + +pub(crate) const EVENT_SZ: usize = 0x34; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(C, u32)] +#[allow(dead_code)] +pub(crate) enum ChannelErrorType { + MemoryError, + DMKill, + Aborted, + Unk3, + Unknown(u32), +} + +#[derive(Debug, Copy, Clone)] +#[repr(C, u32)] +#[allow(dead_code)] +pub(crate) enum EventMsg { + Fault, + Flag { + firing: [u32; 4], + unk_14: u16, + }, + Unk2(Array), + Unk3(Array), + Timeout { + counter: u32, + unk_8: u32, + event_slot: i32, + }, + Unk5(Array), + Unk6(Array), + GrowTVB { + vm_slot: u32, + buffer_slot: u32, + counter: u32, + }, + ChannelError { + error_type: u32, + pipe_type: u32, + event_slot: u32, + event_value: u32, + }, + // Max discriminant: 0x8 +} + +static_assert!(core::mem::size_of::() == 4 + EVENT_SZ); + +pub(crate) const EVENT_MAX: u32 = 0x8; + +#[derive(Copy, Clone)] +#[repr(C)] +pub(crate) union RawEventMsg { + pub(crate) raw: (u32, Array), + pub(crate) msg: EventMsg, +} + +default_zeroed!(RawEventMsg); + +#[derive(Debug, Copy, Clone, Default)] +#[repr(C)] +pub(crate) struct RawFwLogMsg { + pub(crate) msg_type: u32, + __pad0: u32, + pub(crate) msg_index: U64, + __pad1: Pad<0x28>, +} + +#[derive(Debug, Copy, Clone, Default)] +#[repr(C)] +pub(crate) struct RawFwLogPayloadMsg { + pub(crate) msg_type: u32, + pub(crate) seq_no: u32, + pub(crate) timestamp: U64, + pub(crate) msg: Array<0xc8, u8>, +} + +#[derive(Debug, Copy, Clone, Default)] +#[repr(C)] +pub(crate) struct RawKTraceMsg { + pub(crate) msg_type: u32, + pub(crate) timestamp: U64, + pub(crate) args: Array<4, U64>, + pub(crate) code: u8, + pub(crate) channel: u8, + __pad: Pad<1>, + pub(crate) thread: u8, + pub(crate) unk_flag: U64, +} + +#[versions(AGX)] +pub(crate) const STATS_SZ: usize = { + #[ver(V < V13_0B4)] + { + 0x2c + } + #[ver(V >= V13_0B4)] + { + 0x3c + } +}; + +#[versions(AGX)] +#[derive(Debug, Copy, Clone)] +#[repr(C, u32)] +#[allow(dead_code)] +pub(crate) enum StatsMsg { + Power { + // 0x00 + __pad: Pad<0x18>, + power: U64, + }, + Unk1(Array<{ STATS_SZ::ver }, u8>), + PowerOn { + // 0x02 + off_time: U64, + }, + PowerOff { + // 0x03 + on_time: U64, + }, + Utilization { + // 0x04 + timestamp: U64, + util1: u32, + util2: u32, + util3: u32, + util4: u32, + }, + Unk5(Array<{ STATS_SZ::ver }, u8>), + Unk6(Array<{ STATS_SZ::ver }, u8>), + Unk7(Array<{ STATS_SZ::ver }, u8>), + Unk8(Array<{ STATS_SZ::ver }, u8>), + AvgPower { + // 0x09 + active_cs: U64, + unk2: u32, + unk3: u32, + unk4: u32, + avg_power: u32, + }, + Temperature { + // 0x0a + __pad: Pad<0x8>, + raw_value: u32, + scale: u32, + tmin: u32, + tmax: u32, + }, + PowerState { + // 0x0b + timestamp: U64, + last_busy_ts: U64, + active: u32, + poweroff: u32, + unk1: u32, + pstate: u32, + unk2: u32, + unk3: u32, + }, + FwBusy { + // 0x0c + timestamp: U64, + busy: u32, + }, + PState { + // 0x0d + __pad: Pad<0x8>, + ps_min: u32, + unk1: u32, + ps_max: u32, + unk2: u32, + }, + TempSensor { + // 0x0e + __pad: Pad<0x4>, + sensor_id: u32, + raw_value: u32, + scale: u32, + tmin: u32, + tmax: u32, + }, // Max discriminant: 0xe +} + +#[versions(AGX)] +static_assert!(core::mem::size_of::() == 4 + STATS_SZ::ver); + +#[versions(AGX)] +pub(crate) const STATS_MAX: u32 = 0xe; + +#[versions(AGX)] +#[derive(Copy, Clone)] +#[repr(C)] +pub(crate) union RawStatsMsg { + pub(crate) raw: (u32, Array<{ STATS_SZ::ver }, u8>), + pub(crate) msg: StatsMsg::ver, +} + +#[versions(AGX)] +default_zeroed!(RawStatsMsg::ver); diff --git a/drivers/gpu/drm/asahi/fw/compute.rs b/drivers/gpu/drm/asahi/fw/compute.rs new file mode 100644 index 00000000000000..f5f6ffa9d8d0d8 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/compute.rs @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU compute job firmware structures + +use super::types::*; +use super::{ + event, + job, + workqueue, // +}; +use crate::{ + microseq, + mmu, // +}; +use kernel::sync::Arc; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobParameters1<'a> { + pub(crate) preempt_buf1: GpuPointer<'a, &'a [u8]>, + pub(crate) cdm_ctrl_stream_base: U64, + pub(crate) preempt_buf2: GpuPointer<'a, &'a [u8]>, + pub(crate) preempt_buf3: GpuPointer<'a, &'a [u8]>, + pub(crate) preempt_buf4: GpuPointer<'a, &'a [u8]>, + pub(crate) preempt_buf5: GpuPointer<'a, &'a [u8]>, + pub(crate) usc_exec_base_cp: U64, + pub(crate) unk_38: U64, + pub(crate) helper_program: u32, + pub(crate) unk_44: u32, + pub(crate) helper_arg: U64, + pub(crate) helper_cfg: u32, + pub(crate) unk_54: u32, + pub(crate) unk_58: u32, + pub(crate) unk_5c: u32, + pub(crate) iogpu_unk_40: u32, + pub(crate) __pad: Pad<0xfc>, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobParameters2<'a> { + #[ver(V >= V13_0B4)] + pub(crate) unk_0_0: u32, + pub(crate) unk_0: Array<0x24, u8>, + pub(crate) preempt_buf1: GpuPointer<'a, &'a [u8]>, + pub(crate) cdm_ctrl_stream_end: U64, + pub(crate) unk_34: Array<0x20, u8>, + pub(crate) unk_g14x: u32, + pub(crate) unk_58: u32, + #[ver(V < V13_0B4)] + pub(crate) unk_5c: u32, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct RunCompute<'a> { + pub(crate) tag: workqueue::CommandType, + + #[ver(V >= V13_0B4)] + pub(crate) counter: U64, + + pub(crate) unk_4: u32, + pub(crate) vm_slot: u32, + pub(crate) notifier: GpuPointer<'a, event::Notifier::ver>, + pub(crate) unk_pointee: u32, + #[ver(G < G14X)] + pub(crate) __pad0: Array<0x50, u8>, + #[ver(G < G14X)] + pub(crate) job_params1: JobParameters1<'a>, + #[ver(G >= G14X)] + pub(crate) registers: job::raw::RegisterArray, + pub(crate) __pad1: Array<0x20, u8>, + pub(crate) microsequence: GpuPointer<'a, &'a [u8]>, + pub(crate) microsequence_size: u32, + pub(crate) job_params2: JobParameters2::ver<'a>, + pub(crate) encoder_params: job::raw::EncoderParams, + pub(crate) meta: job::raw::JobMeta, + pub(crate) command_time: U64, + pub(crate) timestamp_pointers: job::raw::TimestampPointers<'a>, + pub(crate) user_timestamp_pointers: job::raw::TimestampPointers<'a>, + pub(crate) client_sequence: u8, + pub(crate) pad_2d1: Array<3, u8>, + pub(crate) unk_2d4: u32, + pub(crate) unk_2d8: u8, + #[ver(V >= V13_0B4)] + pub(crate) context_store_req: U64, + #[ver(V >= V13_0B4)] + pub(crate) context_store_compl: U64, + #[ver(V >= V13_0B4)] + pub(crate) unk_2e9: Array<0x14, u8>, + #[ver(V >= V13_0B4)] + pub(crate) unk_flag: U32, + #[ver(V >= V13_0B4)] + pub(crate) unk_pad: Array<0x10, u8>, + } +} + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct RunCompute { + pub(crate) notifier: Arc>, + pub(crate) preempt_buf: GpuArray, + pub(crate) micro_seq: microseq::MicroSequence, + pub(crate) vm_bind: mmu::VmBind, + pub(crate) timestamps: Arc>, + pub(crate) user_timestamps: job::UserTimestamps, +} + +#[versions(AGX)] +impl GpuStruct for RunCompute::ver { + type Raw<'a> = raw::RunCompute::ver<'a>; +} + +#[versions(AGX)] +impl workqueue::Command for RunCompute::ver {} diff --git a/drivers/gpu/drm/asahi/fw/event.rs b/drivers/gpu/drm/asahi/fw/event.rs new file mode 100644 index 00000000000000..52bc456f58707d --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/event.rs @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU events control structures & stamps + +use super::types::*; +use crate::{ + default_zeroed, + trivial_gpustruct, // +}; +use core::sync::atomic::Ordering; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug, Clone, Copy, Default)] + #[repr(C)] + pub(crate) struct LinkedListHead { + pub(crate) prev: Option>, + pub(crate) next: Option>, + } + + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct NotifierList { + pub(crate) list_head: LinkedListHead, + pub(crate) unkptr_10: U64, + } + default_zeroed!(NotifierList); + + #[versions(AGX)] + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct NotifierState { + unk_14: u32, + unk_18: U64, + unk_20: u32, + vm_slot: u32, + has_vtx: u32, + pstamp_vtx: Array<4, U64>, + has_frag: u32, + pstamp_frag: Array<4, U64>, + has_comp: u32, + pstamp_comp: Array<4, U64>, + #[ver(G >= G14 && V < V13_0B4)] + unk_98_g14_0: Array<0x14, u8>, + in_list: u32, + list_head: LinkedListHead, + #[ver(G >= G14 && V < V13_0B4)] + unk_a8_g14_0: Pad<4>, + #[ver(V >= V13_0B4)] + pub(crate) unk_buf: Array<0x8, u8>, // Init to all-ff + } + + #[versions(AGX)] + impl Default for NotifierState::ver { + fn default() -> Self { + #[allow(unused_mut)] + // SAFETY: All bit patterns are valid for this type. + let mut s: Self = unsafe { core::mem::zeroed() }; + #[ver(V >= V13_0B4)] + s.unk_buf = Array::new([0xff; 0x8]); + s + } + } + + #[derive(Debug)] + #[repr(transparent)] + pub(crate) struct Threshold(AtomicU64); + default_zeroed!(Threshold); + + impl Threshold { + pub(crate) fn increase(&self, amount: u32) { + // We could use fetch_add, but the non-LSE atomic + // sequence Rust produces confuses the hypervisor. + let v = self.0.load(Ordering::Relaxed); + self.0.store(v + (amount as u64), Ordering::Relaxed); + } + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct Notifier<'a> { + pub(crate) threshold: GpuPointer<'a, super::Threshold>, + pub(crate) generation: AtomicU32, + pub(crate) cur_count: AtomicU32, + pub(crate) unk_10: AtomicU32, + pub(crate) state: NotifierState::ver, + } +} + +trivial_gpustruct!(Threshold); +trivial_gpustruct!(NotifierList); + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct Notifier { + pub(crate) threshold: GpuObject, +} + +#[versions(AGX)] +impl GpuStruct for Notifier::ver { + type Raw<'a> = raw::Notifier::ver<'a>; +} diff --git a/drivers/gpu/drm/asahi/fw/fragment.rs b/drivers/gpu/drm/asahi/fw/fragment.rs new file mode 100644 index 00000000000000..3daad1ae4db671 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/fragment.rs @@ -0,0 +1,296 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU fragment job firmware structures + +use super::types::*; +use super::{ + event, + job, + workqueue, // +}; +use crate::{ + buffer, + fw, + microseq, + mmu, // +}; +use kernel::sync::Arc; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct BackgroundProgram { + pub(crate) rsrc_spec: U64, + pub(crate) address: U64, + } + + #[derive(Debug, Clone, Copy, Default)] + #[repr(C)] + pub(crate) struct EotProgram { + pub(crate) unk_0: U64, + pub(crate) unk_8: u32, + pub(crate) rsrc_spec: u32, + pub(crate) unk_10: u32, + pub(crate) address: u32, + pub(crate) unk_18: u32, + pub(crate) unk_1c_padding: u32, + } + + impl EotProgram { + pub(crate) fn new(rsrc_spec: u32, address: u32) -> EotProgram { + EotProgram { + rsrc_spec, + address, + ..Default::default() + } + } + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct ArrayAddr { + pub(crate) ptr: U64, + pub(crate) unk_padding: U64, + } + + #[versions(AGX)] + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct AuxFBInfo { + pub(crate) isp_ctl: u32, + pub(crate) unk2: u32, + pub(crate) width: u32, + pub(crate) height: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk3: U64, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobParameters1<'a> { + pub(crate) utile_config: u32, + pub(crate) unk_4: u32, + pub(crate) bg: BackgroundProgram, + pub(crate) ppp_multisamplectl: U64, + pub(crate) isp_scissor_base: U64, + pub(crate) isp_dbias_base: U64, + pub(crate) aux_fb_info: AuxFBInfo::ver, + pub(crate) isp_zls_pixels: U64, + pub(crate) isp_oclqry_base: U64, + pub(crate) zls_ctrl: U64, + + #[ver(G >= G14)] + pub(crate) unk_58_g14_0: U64, + #[ver(G >= G14)] + pub(crate) unk_58_g14_8: U64, + + pub(crate) z_load: U64, + pub(crate) z_store: U64, + pub(crate) s_load: U64, + pub(crate) s_store: U64, + + #[ver(G >= G14)] + pub(crate) unk_68_g14_0: Array<0x20, u8>, + + pub(crate) z_load_stride: U64, + pub(crate) z_store_stride: U64, + pub(crate) s_load_stride: U64, + pub(crate) s_store_stride: U64, + pub(crate) z_load_comp: U64, + pub(crate) z_load_comp_stride: U64, + pub(crate) z_store_comp: U64, + pub(crate) z_store_comp_stride: U64, + pub(crate) s_load_comp: U64, + pub(crate) s_load_comp_stride: U64, + pub(crate) s_store_comp: U64, + pub(crate) s_store_comp_stride: U64, + pub(crate) tvb_tilemap: GpuPointer<'a, &'a [u8]>, + pub(crate) tvb_layermeta: GpuPointer<'a, &'a [u8]>, + pub(crate) mtile_stride_dwords: U64, + pub(crate) tvb_heapmeta: GpuPointer<'a, &'a [u8]>, + pub(crate) tile_config: U64, + pub(crate) aux_fb: GpuPointer<'a, &'a [u8]>, + pub(crate) unk_108: Array<0x6, U64>, + pub(crate) usc_exec_base_isp: U64, + pub(crate) unk_140: U64, + pub(crate) helper_program: u32, + pub(crate) unk_14c: u32, + pub(crate) helper_arg: U64, + pub(crate) unk_158: U64, + pub(crate) unk_160: U64, + + #[ver(G < G14)] + pub(crate) __pad: Pad<0x1d8>, + #[ver(G >= G14)] + pub(crate) __pad: Pad<0x1a8>, + #[ver(V < V13_0B4)] + pub(crate) __pad1: Pad<0x8>, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobParameters2 { + pub(crate) eot_rsrc_spec: u32, + pub(crate) eot_usc: u32, + pub(crate) unk_8: u32, + pub(crate) unk_c: u32, + pub(crate) isp_merge_upper_x: F32, + pub(crate) isp_merge_upper_y: F32, + pub(crate) unk_18: U64, + pub(crate) utiles_per_mtile_y: u16, + pub(crate) utiles_per_mtile_x: u16, + pub(crate) unk_24: u32, + pub(crate) tile_counts: u32, + pub(crate) tib_blocks: u32, + pub(crate) isp_bgobjdepth: u32, + pub(crate) isp_bgobjvals: u32, + pub(crate) unk_38: u32, + pub(crate) unk_3c: u32, + pub(crate) helper_cfg: u32, + pub(crate) __pad: Pad<0xac>, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobParameters3 { + pub(crate) isp_dbias_base: ArrayAddr, + pub(crate) isp_scissor_base: ArrayAddr, + pub(crate) isp_oclqry_base: U64, + pub(crate) unk_118: U64, + pub(crate) unk_120: Array<0x25, U64>, + pub(crate) unk_partial_bg: BackgroundProgram, + pub(crate) unk_258: U64, + pub(crate) unk_260: U64, + pub(crate) unk_268: U64, + pub(crate) unk_270: U64, + pub(crate) partial_bg: BackgroundProgram, + pub(crate) zls_ctrl: U64, + pub(crate) unk_290: U64, + pub(crate) z_load: U64, + pub(crate) z_partial_stride: U64, + pub(crate) z_partial_comp_stride: U64, + pub(crate) z_store: U64, + pub(crate) z_partial: U64, + pub(crate) z_partial_comp: U64, + pub(crate) s_load: U64, + pub(crate) s_partial_stride: U64, + pub(crate) s_partial_comp_stride: U64, + pub(crate) s_store: U64, + pub(crate) s_partial: U64, + pub(crate) s_partial_comp: U64, + pub(crate) unk_2f8: Array<2, U64>, + pub(crate) tib_blocks: u32, + pub(crate) unk_30c: u32, + pub(crate) aux_fb_info: AuxFBInfo::ver, + pub(crate) tile_config: U64, + pub(crate) unk_328_padding: Array<0x8, u8>, + pub(crate) unk_partial_eot: EotProgram, + pub(crate) partial_eot: EotProgram, + pub(crate) isp_bgobjdepth: u32, + pub(crate) isp_bgobjvals: u32, + pub(crate) sample_size: u32, + pub(crate) unk_37c: u32, + pub(crate) unk_380: U64, + pub(crate) unk_388: U64, + + #[ver(V >= V13_0B4)] + pub(crate) unk_390_0: U64, + + pub(crate) isp_zls_pixels: U64, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct RunFragment<'a> { + pub(crate) tag: workqueue::CommandType, + + #[ver(V >= V13_0B4)] + pub(crate) counter: U64, + + pub(crate) vm_slot: u32, + pub(crate) unk_8: u32, + pub(crate) microsequence: GpuPointer<'a, &'a [u8]>, + pub(crate) microsequence_size: u32, + pub(crate) notifier: GpuPointer<'a, event::Notifier::ver>, + pub(crate) buffer: GpuPointer<'a, fw::buffer::Info::ver>, + pub(crate) scene: GpuPointer<'a, fw::buffer::Scene::ver>, + pub(crate) unk_buffer_buf: GpuWeakPointer<[u8]>, + pub(crate) tvb_tilemap: GpuPointer<'a, &'a [u8]>, + pub(crate) ppp_multisamplectl: U64, + pub(crate) samples: u32, + pub(crate) tiles_per_mtile_y: u16, + pub(crate) tiles_per_mtile_x: u16, + pub(crate) unk_50: U64, + pub(crate) unk_58: U64, + pub(crate) isp_merge_upper_x: F32, + pub(crate) isp_merge_upper_y: F32, + pub(crate) unk_68: U64, + pub(crate) tile_count: U64, + + #[ver(G < G14X)] + pub(crate) job_params1: JobParameters1::ver<'a>, + #[ver(G < G14X)] + pub(crate) job_params2: JobParameters2, + #[ver(G >= G14X)] + pub(crate) registers: job::raw::RegisterArray, + + pub(crate) job_params3: JobParameters3::ver, + pub(crate) unk_758_flag: u32, + pub(crate) unk_75c_flag: u32, + pub(crate) unk_buf: Array<0x110, u8>, + pub(crate) busy_flag: u32, + pub(crate) tvb_overflow_count: u32, + pub(crate) unk_878: u32, + pub(crate) encoder_params: job::raw::EncoderParams, + pub(crate) process_empty_tiles: u32, + pub(crate) no_clear_pipeline_textures: u32, + pub(crate) msaa_zs: u32, + pub(crate) unk_pointee: u32, + #[ver(V >= V13_3)] + pub(crate) unk_v13_3: u32, + pub(crate) meta: job::raw::JobMeta, + pub(crate) unk_after_meta: u32, + pub(crate) unk_buf_0: U64, + pub(crate) unk_buf_8: U64, + pub(crate) unk_buf_10: U64, + pub(crate) command_time: U64, + pub(crate) timestamp_pointers: job::raw::TimestampPointers<'a>, + pub(crate) user_timestamp_pointers: job::raw::TimestampPointers<'a>, + pub(crate) client_sequence: u8, + pub(crate) pad_925: Array<3, u8>, + pub(crate) unk_928: u32, + pub(crate) unk_92c: u8, + + #[ver(V >= V13_0B4)] + pub(crate) unk_ts: U64, + + #[ver(V >= V13_0B4)] + pub(crate) unk_92d_8: Array<0x1b, u8>, + } +} + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct RunFragment { + pub(crate) notifier: Arc>, + pub(crate) scene: Arc, + pub(crate) micro_seq: microseq::MicroSequence, + pub(crate) vm_bind: mmu::VmBind, + pub(crate) aux_fb: GpuArray, + pub(crate) timestamps: Arc>, + pub(crate) user_timestamps: job::UserTimestamps, +} + +#[versions(AGX)] +impl GpuStruct for RunFragment::ver { + type Raw<'a> = raw::RunFragment::ver<'a>; +} + +#[versions(AGX)] +impl workqueue::Command for RunFragment::ver {} diff --git a/drivers/gpu/drm/asahi/fw/initdata.rs b/drivers/gpu/drm/asahi/fw/initdata.rs new file mode 100644 index 00000000000000..43cd3f0dfc2560 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/initdata.rs @@ -0,0 +1,1351 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU initialization / global structures + +use super::channels; +use super::types::*; +use crate::{ + default_zeroed, + gem, + mmu, + no_debug, + trivial_gpustruct, // +}; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct ChannelRing { + pub(crate) state: Option>, + pub(crate) ring: Option>, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct PipeChannels { + pub(crate) vtx: ChannelRing, + pub(crate) frag: ChannelRing, + pub(crate) comp: ChannelRing, + } + #[versions(AGX)] + default_zeroed!(PipeChannels::ver); + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct FwStatusFlags { + pub(crate) halt_count: AtomicU64, + __pad0: Pad<0x8>, + pub(crate) halted: AtomicU32, + __pad1: Pad<0xc>, + pub(crate) resume: AtomicU32, + __pad2: Pad<0xc>, + pub(crate) unk_40: u32, + __pad3: Pad<0xc>, + pub(crate) unk_ctr: u32, + __pad4: Pad<0xc>, + pub(crate) unk_60: u32, + __pad5: Pad<0xc>, + pub(crate) unk_70: u32, + __pad6: Pad<0xc>, + } + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct FwStatus { + pub(crate) fwctl_channel: ChannelRing, + pub(crate) flags: FwStatusFlags, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataShared1 { + pub(crate) table: Array<16, i32>, + pub(crate) unk_44: Array<0x60, u8>, + pub(crate) unk_a4: u32, + pub(crate) unk_a8: u32, + } + default_zeroed!(HwDataShared1); + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct HwDataShared2Curve { + pub(crate) unk_0: u32, + pub(crate) unk_4: u32, + pub(crate) t1: Array<16, u16>, + pub(crate) t2: Array<16, i16>, + pub(crate) t3: Array<8, Array<16, i32>>, + } + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct HwDataShared2G14 { + pub(crate) unk_0: Array<5, u32>, + pub(crate) unk_14: u32, + pub(crate) unk_18: Array<8, u32>, + pub(crate) curve1: HwDataShared2Curve, + pub(crate) curve2: HwDataShared2Curve, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataShared2 { + pub(crate) table: Array<10, i32>, + pub(crate) unk_28: Array<0x10, u8>, + pub(crate) g14: HwDataShared2G14, + pub(crate) unk_500: u32, + pub(crate) unk_504: u32, + pub(crate) unk_508: u32, + pub(crate) unk_50c: u32, + } + default_zeroed!(HwDataShared2); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataShared3 { + pub(crate) unk_0: u32, + pub(crate) unk_4: u32, + pub(crate) unk_8: u32, + pub(crate) table: Array<16, u32>, + pub(crate) unk_4c: u32, + } + default_zeroed!(HwDataShared3); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataA130Extra { + pub(crate) unk_0: Array<0x38, u8>, + pub(crate) unk_38: u32, + pub(crate) unk_3c: u32, + pub(crate) gpu_se_inactive_threshold: u32, + pub(crate) unk_44: u32, + pub(crate) gpu_se_engagement_criteria: i32, + pub(crate) gpu_se_reset_criteria: u32, + pub(crate) unk_50: u32, + pub(crate) unk_54: u32, + pub(crate) unk_58: u32, + pub(crate) unk_5c: u32, + pub(crate) gpu_se_filter_a_neg: F32, + pub(crate) gpu_se_filter_1_a_neg: F32, + pub(crate) gpu_se_filter_a: F32, + pub(crate) gpu_se_filter_1_a: F32, + pub(crate) gpu_se_ki_dt: F32, + pub(crate) gpu_se_ki_1_dt: F32, + pub(crate) unk_78: F32, + pub(crate) unk_7c: F32, + pub(crate) gpu_se_kp: F32, + pub(crate) gpu_se_kp_1: F32, + pub(crate) unk_88: u32, + pub(crate) unk_8c: u32, + pub(crate) max_pstate_scaled_1: u32, + pub(crate) unk_94: u32, + pub(crate) unk_98: u32, + pub(crate) unk_9c: F32, + pub(crate) unk_a0: u32, + pub(crate) unk_a4: u32, + pub(crate) gpu_se_filter_time_constant_ms: u32, + pub(crate) gpu_se_filter_time_constant_1_ms: u32, + pub(crate) gpu_se_filter_time_constant_clks: U64, + pub(crate) gpu_se_filter_time_constant_1_clks: U64, + pub(crate) unk_c0: u32, + pub(crate) unk_c4: F32, + pub(crate) unk_c8: Array<0x4c, u8>, + pub(crate) unk_114: F32, + pub(crate) unk_118: u32, + pub(crate) unk_11c: u32, + pub(crate) unk_120: u32, + pub(crate) unk_124: u32, + pub(crate) max_pstate_scaled_2: u32, + pub(crate) unk_12c: Array<0x8c, u8>, + } + default_zeroed!(HwDataA130Extra); + + #[repr(C)] + pub(crate) struct T81xxData { + pub(crate) unk_d8c: u32, + pub(crate) unk_d90: u32, + pub(crate) unk_d94: u32, + pub(crate) unk_d98: u32, + pub(crate) unk_d9c: F32, + pub(crate) unk_da0: u32, + pub(crate) unk_da4: F32, + pub(crate) unk_da8: u32, + pub(crate) unk_dac: F32, + pub(crate) unk_db0: u32, + pub(crate) unk_db4: u32, + pub(crate) unk_db8: F32, + pub(crate) unk_dbc: F32, + pub(crate) unk_dc0: u32, + pub(crate) unk_dc4: u32, + pub(crate) unk_dc8: u32, + pub(crate) max_pstate_scaled: u32, + } + default_zeroed!(T81xxData); + + #[versions(AGX)] + #[derive(Default, Copy, Clone)] + #[repr(C)] + pub(crate) struct PowerZone { + pub(crate) val: F32, + pub(crate) target: u32, + pub(crate) target_off: u32, + pub(crate) filter_tc_x4: u32, + pub(crate) filter_tc_xperiod: u32, + #[ver(V >= V13_0B4)] + pub(crate) unk_10: u32, + #[ver(V >= V13_0B4)] + pub(crate) unk_14: u32, + pub(crate) filter_a_neg: F32, + pub(crate) filter_a: F32, + pub(crate) pad: u32, + } + + #[versions(AGX)] + const MAX_CORES_PER_CLUSTER: usize = { + #[ver(G >= G14X)] + { + 16 + } + #[ver(G < G14X)] + { + 8 + } + }; + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct AuxLeakCoef { + pub(crate) afr_1: Array<2, F32>, + pub(crate) cs_1: Array<2, F32>, + pub(crate) afr_2: Array<2, F32>, + pub(crate) cs_2: Array<2, F32>, + } + + #[versions(AGX)] + #[repr(C)] + pub(crate) struct HwDataA { + pub(crate) unk_0: u32, + pub(crate) clocks_per_period: u32, + + #[ver(V >= V13_0B4)] + pub(crate) clocks_per_period_2: u32, + + pub(crate) unk_8: u32, + pub(crate) pwr_status: AtomicU32, + pub(crate) unk_10: F32, + pub(crate) unk_14: u32, + pub(crate) unk_18: u32, + pub(crate) unk_1c: u32, + pub(crate) unk_20: u32, + pub(crate) unk_24: u32, + pub(crate) actual_pstate: u32, + pub(crate) tgt_pstate: u32, + pub(crate) unk_30: u32, + pub(crate) cur_pstate: u32, + pub(crate) unk_38: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_3c_0: u32, + + pub(crate) base_pstate_scaled: u32, + pub(crate) unk_40: u32, + pub(crate) max_pstate_scaled: u32, + pub(crate) unk_48: u32, + pub(crate) min_pstate_scaled: u32, + pub(crate) freq_mhz: F32, + pub(crate) unk_54: Array<0x20, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_74_0: u32, + + pub(crate) sram_k: Array<0x10, F32>, + pub(crate) unk_b4: Array<0x100, u8>, + pub(crate) unk_1b4: u32, + pub(crate) temp_c: u32, + pub(crate) avg_power_mw: u32, + pub(crate) update_ts: U64, + pub(crate) unk_1c8: u32, + pub(crate) unk_1cc: Array<0x478, u8>, + pub(crate) pad_644: Pad<0x8>, + pub(crate) unk_64c: u32, + pub(crate) unk_650: u32, + pub(crate) pad_654: u32, + pub(crate) pwr_filter_a_neg: F32, + pub(crate) pad_65c: u32, + pub(crate) pwr_filter_a: F32, + pub(crate) pad_664: u32, + pub(crate) pwr_integral_gain: F32, + pub(crate) pad_66c: u32, + pub(crate) pwr_integral_min_clamp: F32, + pub(crate) max_power_1: F32, + pub(crate) pwr_proportional_gain: F32, + pub(crate) pad_67c: u32, + pub(crate) pwr_pstate_related_k: F32, + pub(crate) pwr_pstate_max_dc_offset: i32, + pub(crate) unk_688: u32, + pub(crate) max_pstate_scaled_2: u32, + pub(crate) pad_690: u32, + pub(crate) unk_694: u32, + pub(crate) max_power_2: u32, + pub(crate) pad_69c: Pad<0x18>, + pub(crate) unk_6b4: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_6b8_0: Array<0x10, u8>, + + pub(crate) max_pstate_scaled_3: u32, + pub(crate) unk_6bc: u32, + pub(crate) pad_6c0: Pad<0x14>, + pub(crate) ppm_filter_tc_periods_x4: u32, + pub(crate) unk_6d8: u32, + pub(crate) pad_6dc: u32, + pub(crate) ppm_filter_a_neg: F32, + pub(crate) pad_6e4: u32, + pub(crate) ppm_filter_a: F32, + pub(crate) pad_6ec: u32, + pub(crate) ppm_ki_dt: F32, + pub(crate) pad_6f4: u32, + pub(crate) pwr_integral_min_clamp_2: u32, + pub(crate) unk_6fc: F32, + pub(crate) ppm_kp: F32, + pub(crate) pad_704: u32, + pub(crate) unk_708: u32, + pub(crate) pwr_min_duty_cycle: u32, + pub(crate) max_pstate_scaled_4: u32, + pub(crate) unk_714: u32, + pub(crate) pad_718: u32, + pub(crate) unk_71c: F32, + pub(crate) max_power_3: u32, + pub(crate) cur_power_mw_2: u32, + pub(crate) ppm_filter_tc_ms: u32, + pub(crate) unk_72c: u32, + + #[ver(V >= V13_0B4)] + pub(crate) ppm_filter_tc_clks: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_730_4: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_730_8: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_730_c: u32, + + pub(crate) unk_730: F32, + pub(crate) unk_734: u32, + pub(crate) unk_738: u32, + pub(crate) unk_73c: u32, + pub(crate) unk_740: u32, + pub(crate) unk_744: u32, + pub(crate) unk_748: Array<0x4, F32>, + pub(crate) unk_758: u32, + pub(crate) perf_tgt_utilization: u32, + pub(crate) pad_760: u32, + pub(crate) perf_boost_min_util: u32, + pub(crate) perf_boost_ce_step: u32, + pub(crate) perf_reset_iters: u32, + pub(crate) pad_770: u32, + pub(crate) unk_774: u32, + pub(crate) unk_778: u32, + pub(crate) perf_filter_drop_threshold: u32, + pub(crate) perf_filter_a_neg: F32, + pub(crate) perf_filter_a2_neg: F32, + pub(crate) perf_filter_a: F32, + pub(crate) perf_filter_a2: F32, + pub(crate) perf_ki: F32, + pub(crate) perf_ki2: F32, + pub(crate) perf_integral_min_clamp: F32, + pub(crate) unk_79c: F32, + pub(crate) perf_kp: F32, + pub(crate) perf_kp2: F32, + pub(crate) boost_state_unk_k: F32, + pub(crate) base_pstate_scaled_2: u32, + pub(crate) max_pstate_scaled_5: u32, + pub(crate) base_pstate_scaled_3: u32, + pub(crate) pad_7b8: u32, + pub(crate) perf_cur_utilization: F32, + pub(crate) perf_tgt_utilization_2: u32, + pub(crate) pad_7c4: Pad<0x18>, + pub(crate) unk_7dc: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_7e0_0: Array<0x10, u8>, + + pub(crate) base_pstate_scaled_4: u32, + pub(crate) pad_7e4: u32, + pub(crate) unk_7e8: Array<0x14, u8>, + pub(crate) unk_7fc: F32, + pub(crate) pwr_min_duty_cycle_2: F32, + pub(crate) max_pstate_scaled_6: F32, + pub(crate) max_freq_mhz: u32, + pub(crate) pad_80c: u32, + pub(crate) unk_810: u32, + pub(crate) pad_814: u32, + pub(crate) pwr_min_duty_cycle_3: u32, + pub(crate) unk_81c: u32, + pub(crate) pad_820: u32, + pub(crate) min_pstate_scaled_4: F32, + pub(crate) max_pstate_scaled_7: u32, + pub(crate) unk_82c: u32, + pub(crate) unk_alpha_neg: F32, + pub(crate) unk_alpha: F32, + pub(crate) unk_838: u32, + pub(crate) unk_83c: u32, + pub(crate) pad_840: Pad<0x2c>, + pub(crate) unk_86c: u32, + pub(crate) fast_die0_sensor_mask: U64, + #[ver(G >= G14X)] + pub(crate) fast_die1_sensor_mask: U64, + pub(crate) fast_die0_release_temp_cc: u32, + pub(crate) unk_87c: i32, + pub(crate) unk_880: u32, + pub(crate) unk_884: u32, + pub(crate) pad_888: u32, + pub(crate) unk_88c: u32, + pub(crate) pad_890: u32, + pub(crate) unk_894: F32, + pub(crate) pad_898: u32, + pub(crate) fast_die0_ki_dt: F32, + pub(crate) pad_8a0: u32, + pub(crate) unk_8a4: u32, + pub(crate) unk_8a8: F32, + pub(crate) fast_die0_kp: F32, + pub(crate) pad_8b0: u32, + pub(crate) unk_8b4: u32, + pub(crate) pwr_min_duty_cycle_4: u32, + pub(crate) max_pstate_scaled_8: u32, + pub(crate) max_pstate_scaled_9: u32, + pub(crate) fast_die0_prop_tgt_delta: u32, + pub(crate) unk_8c8: u32, + pub(crate) unk_8cc: u32, + pub(crate) pad_8d0: Pad<0x14>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_8e4_0: Array<0x10, u8>, + + pub(crate) unk_8e4: u32, + pub(crate) unk_8e8: u32, + pub(crate) max_pstate_scaled_10: u32, + pub(crate) unk_8f0: u32, + pub(crate) unk_8f4: u32, + pub(crate) pad_8f8: u32, + pub(crate) pad_8fc: u32, + pub(crate) unk_900: Array<0x24, u8>, + + pub(crate) unk_coef_a1: Array<8, Array>, + pub(crate) unk_coef_a2: Array<8, Array>, + + pub(crate) pad_b24: Pad<0x70>, + pub(crate) max_pstate_scaled_11: u32, + pub(crate) freq_with_off: u32, + pub(crate) unk_b9c: u32, + pub(crate) unk_ba0: U64, + pub(crate) unk_ba8: U64, + pub(crate) unk_bb0: u32, + pub(crate) unk_bb4: u32, + + #[ver(V >= V13_3)] + pub(crate) pad_bb8_0: Pad<0x200>, + #[ver(V >= V13_5)] + pub(crate) pad_bb8_200: Pad<0x8>, + + pub(crate) pad_bb8: Pad<0x74>, + pub(crate) unk_c2c: u32, + pub(crate) power_zone_count: u32, + pub(crate) max_power_4: u32, + pub(crate) max_power_5: u32, + pub(crate) max_power_6: u32, + pub(crate) unk_c40: u32, + pub(crate) unk_c44: F32, + pub(crate) avg_power_target_filter_a_neg: F32, + pub(crate) avg_power_target_filter_a: F32, + pub(crate) avg_power_target_filter_tc_x4: u32, + pub(crate) avg_power_target_filter_tc_xperiod: u32, + + #[ver(V >= V13_0B4)] + pub(crate) avg_power_target_filter_tc_clks: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_c58_4: u32, + + pub(crate) power_zones: Array<5, PowerZone::ver>, + pub(crate) avg_power_filter_tc_periods_x4: u32, + pub(crate) unk_cfc: u32, + pub(crate) unk_d00: u32, + pub(crate) avg_power_filter_a_neg: F32, + pub(crate) unk_d08: u32, + pub(crate) avg_power_filter_a: F32, + pub(crate) unk_d10: u32, + pub(crate) avg_power_ki_dt: F32, + pub(crate) unk_d18: u32, + pub(crate) unk_d1c: u32, + pub(crate) unk_d20: F32, + pub(crate) avg_power_kp: F32, + pub(crate) unk_d28: u32, + pub(crate) unk_d2c: u32, + pub(crate) avg_power_min_duty_cycle: u32, + pub(crate) max_pstate_scaled_12: u32, + pub(crate) max_pstate_scaled_13: u32, + pub(crate) unk_d3c: u32, + pub(crate) max_power_7: F32, + pub(crate) max_power_8: u32, + pub(crate) unk_d48: u32, + pub(crate) avg_power_filter_tc_ms: u32, + pub(crate) unk_d50: u32, + + #[ver(V >= V13_0B4)] + pub(crate) avg_power_filter_tc_clks: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_d54_4: Array<0xc, u8>, + + pub(crate) unk_d54: Array<0x10, u8>, + pub(crate) max_pstate_scaled_14: u32, + pub(crate) unk_d68: Array<0x24, u8>, + + pub(crate) t81xx_data: T81xxData, + + pub(crate) unk_dd0: Array<0x40, u8>, + + #[ver(V >= V13_2)] + pub(crate) unk_e10_pad: Array<0x10, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_e10_0: HwDataA130Extra, + + pub(crate) unk_e10: Array<0xc, u8>, + + pub(crate) fast_die0_sensor_mask_2: U64, + #[ver(G >= G14X)] + pub(crate) fast_die1_sensor_mask_2: U64, + + pub(crate) unk_e24: u32, + pub(crate) unk_e28: u32, + pub(crate) unk_e2c: Pad<0x1c>, + pub(crate) unk_coef_b1: Array<8, Array>, + pub(crate) unk_coef_b2: Array<8, Array>, + + #[ver(G >= G14X)] + pub(crate) pad_1048_0: Pad<0x600>, + + pub(crate) pad_1048: Pad<0x5e4>, + + pub(crate) fast_die0_sensor_mask_alt: U64, + #[ver(G >= G14X)] + pub(crate) fast_die1_sensor_mask_alt: U64, + #[ver(V < V13_0B4)] + pub(crate) fast_die0_sensor_present: U64, + + pub(crate) unk_163c: u32, + + pub(crate) unk_1640: Array<0x2000, u8>, + + #[ver(G >= G14X)] + pub(crate) unk_3640_0: Array<0x2000, u8>, + + pub(crate) unk_3640: u32, + pub(crate) unk_3644: u32, + pub(crate) hws1: HwDataShared1, + + #[ver(V >= V13_0B4)] + pub(crate) unk_hws2: Array<16, u16>, + + pub(crate) hws2: HwDataShared2, + pub(crate) unk_3c00: u32, + pub(crate) unk_3c04: u32, + pub(crate) hws3: HwDataShared3, + pub(crate) unk_3c58: Array<0x3c, u8>, + pub(crate) unk_3c94: u32, + pub(crate) unk_3c98: U64, + pub(crate) unk_3ca0: U64, + pub(crate) unk_3ca8: U64, + pub(crate) unk_3cb0: U64, + pub(crate) ts_last_idle: U64, + pub(crate) ts_last_poweron: U64, + pub(crate) ts_last_poweroff: U64, + pub(crate) unk_3cd0: U64, + pub(crate) unk_3cd8: U64, + + #[ver(V >= V13_0B4)] + pub(crate) unk_3ce0_0: u32, + + pub(crate) unk_3ce0: u32, + pub(crate) unk_3ce4: u32, + pub(crate) unk_3ce8: u32, + pub(crate) unk_3cec: u32, + pub(crate) unk_3cf0: u32, + pub(crate) core_leak_coef: Array<8, F32>, + pub(crate) sram_leak_coef: Array<8, F32>, + + #[ver(V >= V13_0B4)] + pub(crate) aux_leak_coef: AuxLeakCoef, + #[ver(V >= V13_0B4)] + pub(crate) unk_3d34_0: Array<0x18, u8>, + + pub(crate) unk_3d34: Array<0x38, u8>, + } + #[versions(AGX)] + default_zeroed!(HwDataA::ver); + #[versions(AGX)] + no_debug!(HwDataA::ver); + + #[derive(Debug, Default, Clone, Copy)] + #[repr(C)] + pub(crate) struct IOMapping { + pub(crate) phys_addr: U64, + pub(crate) virt_addr: U64, + pub(crate) total_size: u32, + pub(crate) element_size: u32, + pub(crate) readwrite: U64, + } + + #[versions(AGX)] + const IO_MAPPING_COUNT: usize = { + #[ver(V < V13_0B4)] + { + 0x14 + } + #[ver(V >= V13_0B4 && V < V13_3)] + { + 0x17 + } + #[ver(V >= V13_3 && V < V13_5)] + { + 0x18 + } + #[ver(V >= V13_5)] + { + 0x19 + } + }; + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataBAuxPStates { + pub(crate) cs_max_pstate: u32, + pub(crate) cs_frequencies: Array<0x10, u32>, + pub(crate) cs_voltages: Array<0x10, Array<0x2, u32>>, + pub(crate) cs_voltages_sram: Array<0x10, Array<0x2, u32>>, + pub(crate) cs_unkpad: u32, + pub(crate) afr_max_pstate: u32, + pub(crate) afr_frequencies: Array<0x8, u32>, + pub(crate) afr_voltages: Array<0x8, Array<0x2, u32>>, + pub(crate) afr_voltages_sram: Array<0x8, Array<0x2, u32>>, + pub(crate) afr_unkpad: u32, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataB { + #[ver(V < V13_0B4)] + pub(crate) unk_0: U64, + + pub(crate) unk_8: U64, + + #[ver(V < V13_0B4)] + pub(crate) unk_10: U64, + + pub(crate) unk_18: U64, + pub(crate) unk_20: U64, + pub(crate) unk_28: U64, + pub(crate) unk_30: U64, + pub(crate) timestamp_area_base: U64, + pub(crate) pad_40: Pad<0x20>, + + #[ver(V < V13_0B4)] + pub(crate) yuv_matrices: Array<0xf, Array<3, Array<4, i16>>>, + + #[ver(V >= V13_0B4)] + pub(crate) yuv_matrices: Array<0x3f, Array<3, Array<4, i16>>>, + + pub(crate) pad_1c8: Pad<0x8>, + pub(crate) io_mappings: Array, + + #[ver(V >= V13_0B4)] + pub(crate) sgx_sram_ptr: U64, + + pub(crate) chip_id: u32, + pub(crate) unk_454: u32, + pub(crate) unk_458: u32, + pub(crate) unk_45c: u32, + pub(crate) unk_460: u32, + pub(crate) unk_464: u32, + pub(crate) unk_468: u32, + pub(crate) unk_46c: u32, + pub(crate) unk_470: u32, + pub(crate) unk_474: u32, + pub(crate) unk_478: u32, + pub(crate) unk_47c: u32, + pub(crate) unk_480: u32, + pub(crate) unk_484: u32, + pub(crate) unk_488: u32, + pub(crate) unk_48c: u32, + pub(crate) base_clock_khz: u32, + pub(crate) power_sample_period: u32, + pub(crate) pad_498: Pad<0x4>, + pub(crate) unk_49c: u32, + pub(crate) unk_4a0: u32, + pub(crate) unk_4a4: u32, + pub(crate) pad_4a8: Pad<0x4>, + pub(crate) unk_4ac: u32, + pub(crate) pad_4b0: Pad<0x8>, + pub(crate) unk_4b8: u32, + pub(crate) unk_4bc: Array<0x4, u8>, + pub(crate) unk_4c0: u32, + pub(crate) unk_4c4: u32, + pub(crate) unk_4c8: u32, + pub(crate) unk_4cc: u32, + pub(crate) unk_4d0: u32, + pub(crate) unk_4d4: u32, + pub(crate) unk_4d8: Array<0x4, u8>, + pub(crate) unk_4dc: u32, + pub(crate) unk_4e0: U64, + pub(crate) unk_4e8: u32, + pub(crate) unk_4ec: u32, + pub(crate) unk_4f0: u32, + pub(crate) unk_4f4: u32, + pub(crate) unk_4f8: u32, + pub(crate) unk_4fc: u32, + pub(crate) unk_500: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_504_0: u32, + + pub(crate) unk_504: u32, + pub(crate) unk_508: u32, + pub(crate) unk_50c: u32, + pub(crate) unk_510: u32, + pub(crate) unk_514: u32, + pub(crate) unk_518: u32, + pub(crate) unk_51c: u32, + pub(crate) unk_520: u32, + pub(crate) unk_524: u32, + pub(crate) unk_528: u32, + pub(crate) unk_52c: u32, + pub(crate) unk_530: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_534_0: u32, + + pub(crate) unk_534: u32, + pub(crate) unk_538: u32, + + pub(crate) num_frags: u32, + pub(crate) unk_540: u32, + pub(crate) unk_544: u32, + pub(crate) unk_548: u32, + pub(crate) unk_54c: u32, + pub(crate) unk_550: u32, + pub(crate) unk_554: u32, + pub(crate) uat_ttb_base: U64, + pub(crate) gpu_core_id: u32, + pub(crate) gpu_rev_id: u32, + pub(crate) num_cores: u32, + pub(crate) max_pstate: u32, + + #[ver(V < V13_0B4)] + pub(crate) num_pstates: u32, + + pub(crate) frequencies: Array<0x10, u32>, + pub(crate) voltages: Array<0x10, [u32; 0x8]>, + pub(crate) voltages_sram: Array<0x10, [u32; 0x8]>, + + #[ver(V >= V13_3)] + pub(crate) unk_9f4_0: Pad<64>, + + pub(crate) sram_k: Array<0x10, F32>, + pub(crate) unk_9f4: Array<0x10, u32>, + pub(crate) rel_max_powers: Array<0x10, u32>, + pub(crate) rel_boost_freqs: Array<0x10, u32>, + + #[ver(V >= V13_3)] + pub(crate) unk_arr_0: Array<32, u32>, + + #[ver(V < V13_0B4)] + pub(crate) min_sram_volt: u32, + + #[ver(V < V13_0B4)] + pub(crate) unk_ab8: u32, + + #[ver(V < V13_0B4)] + pub(crate) unk_abc: u32, + + #[ver(V < V13_0B4)] + pub(crate) unk_ac0: u32, + + #[ver(V >= V13_0B4)] + pub(crate) aux_ps: HwDataBAuxPStates, + + #[ver(V >= V13_3)] + pub(crate) pad_ac4_0: Array<0x44c, u8>, + + pub(crate) pad_ac4: Pad<0x8>, + pub(crate) unk_acc: u32, + pub(crate) unk_ad0: u32, + pub(crate) pad_ad4: Pad<0x10>, + pub(crate) unk_ae4: Array<0x4, u32>, + pub(crate) pad_af4: Pad<0x4>, + pub(crate) unk_af8: u32, + pub(crate) pad_afc: Pad<0x8>, + pub(crate) unk_b04: u32, + pub(crate) unk_b08: u32, + pub(crate) unk_b0c: u32, + + #[ver(G >= G14X)] + pub(crate) pad_b10_0: Array<0x8, u8>, + + pub(crate) unk_b10: u32, + pub(crate) timer_offset: U64, + pub(crate) unk_b1c: u32, + pub(crate) unk_b20: u32, + pub(crate) unk_b24: u32, + pub(crate) unk_b28: u32, + pub(crate) unk_b2c: u32, + pub(crate) unk_b30: u32, + pub(crate) unk_b34: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_b38_0: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_b38_4: u32, + + #[ver(V >= V13_3)] + pub(crate) unk_b38_8: u32, + + pub(crate) unk_b38: Array<0xc, u32>, + pub(crate) unk_b68: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_b6c: Array<0xd0, u8>, + + #[ver(G >= G14X)] + pub(crate) unk_c3c_0: Array<0x8, u8>, + + #[ver(G < G14X && V >= V13_5)] + pub(crate) unk_c3c_8: Array<0x10, u8>, + + #[ver(V >= V13_5)] + pub(crate) unk_c3c_18: Array<0x20, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_c3c: u32, + } + #[versions(AGX)] + default_zeroed!(HwDataB::ver); + + #[derive(Debug)] + #[repr(C, packed)] + pub(crate) struct GpuStatsVtx { + // This changes all the time and we don't use it, let's just make it a big buffer + pub(crate) opaque: Array<0x3000, u8>, + } + default_zeroed!(GpuStatsVtx); + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct GpuStatsFrag { + // This changes all the time and we don't use it, let's just make it a big buffer + // except for these two fields which may need init. + #[ver(G >= G14X)] + pub(crate) unk1_0: Array<0x910, u8>, + pub(crate) unk1: Array<0x100, u8>, + pub(crate) cur_stamp_id: i32, + pub(crate) unk2: Array<0x14, u8>, + pub(crate) unk_id: i32, + pub(crate) unk3: Array<0x1000, u8>, + } + + #[versions(AGX)] + impl Default for GpuStatsFrag::ver { + fn default() -> Self { + Self { + #[ver(G >= G14X)] + unk1_0: Default::default(), + unk1: Default::default(), + cur_stamp_id: -1, + unk2: Default::default(), + unk_id: -1, + unk3: Default::default(), + } + } + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct GpuGlobalStatsVtx { + pub(crate) total_cmds: u32, + pub(crate) stats: GpuStatsVtx, + } + default_zeroed!(GpuGlobalStatsVtx); + + #[versions(AGX)] + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct GpuGlobalStatsFrag { + pub(crate) total_cmds: u32, + pub(crate) unk_4: u32, + pub(crate) stats: GpuStatsFrag::ver, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct GpuStatsComp { + // This changes all the time and we don't use it, let's just make it a big buffer + pub(crate) opaque: Array<0x3000, u8>, + } + default_zeroed!(GpuStatsComp); + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct RuntimeScratch { + pub(crate) unk_280: Array<0x6800, u8>, + pub(crate) unk_6a80: u32, + pub(crate) gpu_idle: u32, + pub(crate) unkpad_6a88: Pad<0x14>, + pub(crate) unk_6a9c: u32, + pub(crate) unk_ctr0: u32, + pub(crate) unk_ctr1: u32, + pub(crate) unk_6aa8: u32, + pub(crate) unk_6aac: u32, + pub(crate) unk_ctr2: u32, + pub(crate) unk_6ab4: u32, + pub(crate) unk_6ab8: u32, + pub(crate) unk_6abc: u32, + pub(crate) unk_6ac0: u32, + pub(crate) unk_6ac4: u32, + pub(crate) unk_ctr3: u32, + pub(crate) unk_6acc: u32, + pub(crate) unk_6ad0: u32, + pub(crate) unk_6ad4: u32, + pub(crate) unk_6ad8: u32, + pub(crate) unk_6adc: u32, + pub(crate) unk_6ae0: u32, + pub(crate) unk_6ae4: u32, + pub(crate) unk_6ae8: u32, + pub(crate) unk_6aec: u32, + pub(crate) unk_6af0: u32, + pub(crate) unk_ctr4: u32, + pub(crate) unk_ctr5: u32, + pub(crate) unk_6afc: u32, + pub(crate) pad_6b00: Pad<0x38>, + + #[ver(G >= G14X)] + pub(crate) pad_6b00_extra: Array<0x4800, u8>, + + pub(crate) unk_6b38: u32, + pub(crate) pad_6b3c: Pad<0x84>, + } + #[versions(AGX)] + default_zeroed!(RuntimeScratch::ver); + + #[versions(AGX)] + #[repr(C)] + pub(crate) struct RuntimePointers<'a> { + pub(crate) pipes: Array<4, PipeChannels::ver>, + + pub(crate) device_control: + ChannelRing, + pub(crate) event: ChannelRing, + pub(crate) fw_log: ChannelRing, + pub(crate) ktrace: ChannelRing, + pub(crate) stats: ChannelRing, + + pub(crate) __pad0: Pad<0x50>, + pub(crate) unk_160: U64, + pub(crate) unk_168: U64, + pub(crate) stats_vtx: GpuPointer<'a, super::GpuGlobalStatsVtx>, + pub(crate) stats_frag: GpuPointer<'a, super::GpuGlobalStatsFrag::ver>, + pub(crate) stats_comp: GpuPointer<'a, super::GpuStatsComp>, + pub(crate) hwdata_a: GpuPointer<'a, super::HwDataA::ver>, + pub(crate) unkptr_190: GpuPointer<'a, &'a [u8]>, + pub(crate) unkptr_198: GpuPointer<'a, &'a [u8]>, + pub(crate) hwdata_b: GpuPointer<'a, super::HwDataB::ver>, + pub(crate) hwdata_b_2: GpuPointer<'a, super::HwDataB::ver>, + pub(crate) fwlog_buf: Option>, + pub(crate) unkptr_1b8: GpuPointer<'a, &'a [u8]>, + + #[ver(G < G14X)] + pub(crate) unkptr_1c0: GpuPointer<'a, &'a [u8]>, + #[ver(G < G14X)] + pub(crate) unkptr_1c8: GpuPointer<'a, &'a [u8]>, + + pub(crate) unk_1d0: u32, + pub(crate) unk_1d4: u32, + pub(crate) unk_1d8: Array<0x3c, u8>, + pub(crate) buffer_mgr_ctl_gpu_addr: U64, + pub(crate) buffer_mgr_ctl_fw_addr: U64, + pub(crate) __pad1: Pad<0x5c>, + pub(crate) gpu_scratch: RuntimeScratch::ver, + } + #[versions(AGX)] + no_debug!(RuntimePointers::ver<'_>); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct PendingStamp { + pub(crate) info: AtomicU32, + pub(crate) wait_value: AtomicU32, + } + default_zeroed!(PendingStamp); + + #[derive(Debug, Clone, Copy)] + #[repr(C, packed)] + pub(crate) struct FaultInfo { + pub(crate) unk_0: u32, + pub(crate) unk_4: u32, + pub(crate) queue_uuid: u32, + pub(crate) unk_c: u32, + pub(crate) unk_10: u32, + pub(crate) unk_14: u32, + } + default_zeroed!(FaultInfo); + + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct PowerZoneGlobal { + pub(crate) target: u32, + pub(crate) target_off: u32, + pub(crate) filter_tc: u32, + } + default_zeroed!(PowerZoneGlobal); + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct Globals { + pub(crate) ktrace_enable: u32, + pub(crate) unk_4: Array<0x20, u8>, + + #[ver(V >= V13_2)] + pub(crate) unk_24_0: u32, + + pub(crate) unk_24: u32, + + #[ver(V >= V13_0B4)] + pub(crate) debug: u32, + + #[ver(V >= V13_3)] + pub(crate) unk_28_4: u32, + + pub(crate) unk_28: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_2c_0: u32, + + pub(crate) unk_2c: u32, + pub(crate) unk_30: u32, + pub(crate) unk_34: u32, + pub(crate) unk_38: Array<0x1c, u8>, + + // pub(crate) sub: GlobalsSub::ver, + pub(crate) unk_54: u16, + pub(crate) unk_56: u16, + pub(crate) unk_58: u16, + pub(crate) unk_5a: U32, + pub(crate) unk_5e: U32, + pub(crate) unk_62: U32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_66_0: Array<0xc, u8>, + + pub(crate) unk_66: U32, + pub(crate) unk_6a: Array<0x16, u8>, + // end GlobalsSub::ver + pub(crate) unk_80: Array<0xf80, u8>, + pub(crate) unk_1000: Array<0x7000, u8>, + pub(crate) unk_8000: Array<0x900, u8>, + + #[ver(G >= G14X)] + pub(crate) unk_8900_pad: Array<0x484c, u8>, + + #[ver(V >= V13_3)] + pub(crate) unk_8900_pad2: Array<0x54, u8>, + + pub(crate) unk_8900: u32, + pub(crate) pending_submissions: AtomicU32, + pub(crate) max_power: u32, + pub(crate) max_pstate_scaled: u32, + pub(crate) max_pstate_scaled_2: u32, + pub(crate) unk_8914: u32, + pub(crate) unk_8918: u32, + pub(crate) max_pstate_scaled_3: u32, + pub(crate) unk_8920: u32, + pub(crate) power_zone_count: u32, + pub(crate) avg_power_filter_tc_periods: u32, + pub(crate) avg_power_ki_dt: F32, + pub(crate) avg_power_kp: F32, + pub(crate) avg_power_min_duty_cycle: u32, + pub(crate) avg_power_target_filter_tc: u32, + pub(crate) power_zones: Array<5, PowerZoneGlobal>, + pub(crate) unk_8978: Array<0x44, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_89bc_0: Array<0x3c, u8>, + + pub(crate) unk_89bc: u32, + pub(crate) fast_die0_release_temp: u32, + pub(crate) unk_89c4: i32, + pub(crate) fast_die0_prop_tgt_delta: u32, + pub(crate) fast_die0_kp: F32, + pub(crate) fast_die0_ki_dt: F32, + pub(crate) unk_89d4: Array<0xc, u8>, + pub(crate) unk_89e0: u32, + pub(crate) max_power_2: u32, + pub(crate) ppm_kp: F32, + pub(crate) ppm_ki_dt: F32, + pub(crate) unk_89f0: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_89f4_0: Array<0x8, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_89f4_8: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_89f4_c: Array<0x50, u8>, + + #[ver(V >= V13_3)] + pub(crate) unk_89f4_5c: Array<0xc, u8>, + + pub(crate) unk_89f4: u32, + pub(crate) hws1: HwDataShared1, + pub(crate) hws2: HwDataShared2, + + #[ver(V >= V13_0B4)] + pub(crate) idle_off_standby_timer: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_hws2_4: Array<0x8, F32>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_hws2_24: u32, + + pub(crate) unk_hws2_28: u32, + + pub(crate) hws3: HwDataShared3, + pub(crate) unk_9004: Array<8, u8>, + pub(crate) unk_900c: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_9010_0: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_9010_4: Array<0x14, u8>, + + pub(crate) unk_9010: Array<0x2c, u8>, + pub(crate) unk_903c: u32, + pub(crate) unk_9040: Array<0xc0, u8>, + pub(crate) unk_9100: Array<0x6f00, u8>, + pub(crate) unk_10000: Array<0xe50, u8>, + pub(crate) unk_10e50: u32, + pub(crate) unk_10e54: Array<0x2c, u8>, + + #[ver((G >= G14X && V < V13_3) || (G <= G14 && V >= V13_3))] + pub(crate) unk_x_pad: Array<0x4, u8>, + + // bit 0: sets sgx_reg 0x17620 + // bit 1: sets sgx_reg 0x17630 + pub(crate) fault_control: u32, + pub(crate) do_init: u32, + pub(crate) unk_10e88: Array<0x188, u8>, + pub(crate) idle_ts: U64, + pub(crate) idle_unk: U64, + pub(crate) progress_check_interval_3d: u32, + pub(crate) progress_check_interval_ta: u32, + pub(crate) progress_check_interval_cl: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_1102c_0: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_1102c_4: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_1102c_8: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_1102c_c: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_1102c_10: u32, + + pub(crate) unk_1102c: u32, + pub(crate) idle_off_delay_ms: AtomicU32, + pub(crate) fender_idle_off_delay_ms: u32, + pub(crate) fw_early_wake_timeout_ms: u32, + #[ver(V == V13_3)] + pub(crate) ps_pad_0: Pad<0x8>, + pub(crate) pending_stamps: Array<0x100, PendingStamp>, + #[ver(V != V13_3)] + pub(crate) ps_pad_0: Pad<0x8>, + pub(crate) unkpad_ps: Pad<0x78>, + pub(crate) unk_117bc: u32, + pub(crate) fault_info: FaultInfo, + pub(crate) counter: u32, + pub(crate) unk_118dc: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_118e0_0: Array<0x9c, u8>, + + #[ver(G >= G14X)] + pub(crate) unk_118e0_9c: Array<0x580, u8>, + + #[ver(V >= V13_3)] + pub(crate) unk_118e0_9c_x: Array<0x8, u8>, + + pub(crate) cl_context_switch_timeout_ms: u32, + + #[ver(V >= V13_0B4)] + pub(crate) cl_kill_timeout_ms: u32, + + pub(crate) cdm_context_store_latency_threshold: u32, + pub(crate) unk_118e8: u32, + pub(crate) unk_118ec: Array<0x400, u8>, + pub(crate) unk_11cec: Array<0x54, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_11d40: Array<0x19c, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_11edc: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_11ee0: Array<0x1c, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_11efc: u32, + + #[ver(V >= V13_3)] + pub(crate) unk_11f00: Array<0x280, u8>, + } + #[versions(AGX)] + default_zeroed!(Globals::ver); + + #[derive(Debug, Default, Clone, Copy)] + #[repr(C, packed)] + pub(crate) struct UatLevelInfo { + pub(crate) unk_3: u8, + pub(crate) unk_1: u8, + pub(crate) unk_2: u8, + pub(crate) index_shift: u8, + pub(crate) num_entries: u16, + pub(crate) unk_4: u16, + pub(crate) unk_8: U64, + pub(crate) unk_10: U64, + pub(crate) index_mask: U64, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct InitData<'a> { + #[ver(V >= V13_0B4)] + pub(crate) ver_info: Array<0x4, u16>, + + pub(crate) unk_buf: GpuPointer<'a, &'a [u8]>, + pub(crate) unk_8: u32, + pub(crate) unk_c: u32, + pub(crate) runtime_pointers: GpuPointer<'a, super::RuntimePointers::ver>, + pub(crate) globals: GpuPointer<'a, super::Globals::ver>, + pub(crate) fw_status: GpuPointer<'a, super::FwStatus>, + pub(crate) uat_page_size: u16, + pub(crate) uat_page_bits: u8, + pub(crate) uat_num_levels: u8, + pub(crate) uat_level_info: Array<0x3, UatLevelInfo>, + pub(crate) __pad0: Pad<0x14>, + pub(crate) host_mapped_fw_allocations: u32, + pub(crate) unk_ac: u32, + pub(crate) unk_b0: u32, + pub(crate) unk_b4: u32, + pub(crate) unk_b8: u32, + } +} + +#[derive(Debug)] +pub(crate) struct ChannelRing +where + for<'a> ::Raw<'a>: Debug, +{ + pub(crate) state: GpuObject, + pub(crate) ring: GpuArray, +} + +impl ChannelRing +where + for<'a> ::Raw<'a>: Debug, +{ + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + raw::ChannelRing { + state: Some(self.state.weak_pointer()), + ring: Some(self.ring.weak_pointer()), + } + } +} + +trivial_gpustruct!(FwStatus); +trivial_gpustruct!(GpuGlobalStatsVtx); +#[versions(AGX)] +trivial_gpustruct!(GpuGlobalStatsFrag::ver); +trivial_gpustruct!(GpuStatsComp); + +#[versions(AGX)] +trivial_gpustruct!(HwDataA::ver); + +#[versions(AGX)] +trivial_gpustruct!(HwDataB::ver); + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct Stats { + pub(crate) vtx: GpuObject, + pub(crate) frag: GpuObject, + pub(crate) comp: GpuObject, +} + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct RuntimePointers { + pub(crate) stats: Stats::ver, + + pub(crate) hwdata_a: GpuObject, + pub(crate) unkptr_190: GpuArray, + pub(crate) unkptr_198: GpuArray, + pub(crate) hwdata_b: GpuObject, + + pub(crate) unkptr_1b8: GpuArray, + pub(crate) unkptr_1c0: GpuArray, + pub(crate) unkptr_1c8: GpuArray, + + pub(crate) buffer_mgr_ctl: gem::ObjectRef, + pub(crate) buffer_mgr_ctl_low_mapping: Option, + pub(crate) buffer_mgr_ctl_high_mapping: Option, +} + +#[versions(AGX)] +impl GpuStruct for RuntimePointers::ver { + type Raw<'a> = raw::RuntimePointers::ver<'a>; +} + +#[versions(AGX)] +trivial_gpustruct!(Globals::ver); + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct InitData { + pub(crate) unk_buf: GpuArray, + pub(crate) runtime_pointers: GpuObject, + pub(crate) globals: GpuObject, + pub(crate) fw_status: GpuObject, +} + +#[versions(AGX)] +impl GpuStruct for InitData::ver { + type Raw<'a> = raw::InitData::ver<'a>; +} diff --git a/drivers/gpu/drm/asahi/fw/job.rs b/drivers/gpu/drm/asahi/fw/job.rs new file mode 100644 index 00000000000000..e4f2f9225ea050 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/job.rs @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Common GPU job firmware structures + +use super::types::*; +use crate::{ + default_zeroed, + mmu, + trivial_gpustruct, // +}; +use kernel::prelude::Result; +use kernel::sync::Arc; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct JobMeta { + pub(crate) unk_0: u16, + pub(crate) unk_2: u8, + pub(crate) no_preemption: u8, + pub(crate) stamp: GpuWeakPointer, + pub(crate) fw_stamp: GpuWeakPointer, + pub(crate) stamp_value: EventValue, + pub(crate) stamp_slot: u32, + pub(crate) evctl_index: u32, + pub(crate) flush_stamps: u32, + pub(crate) uuid: u32, + pub(crate) event_seq: u32, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct EncoderParams { + pub(crate) unk_8: u32, + pub(crate) sync_grow: u32, + pub(crate) unk_10: u32, + pub(crate) encoder_id: u32, + pub(crate) unk_18: u32, + pub(crate) unk_mask: u32, + pub(crate) sampler_array: U64, + pub(crate) sampler_count: u32, + pub(crate) sampler_max: u32, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobTimestamps { + pub(crate) start: AtomicU64, + pub(crate) end: AtomicU64, + } + default_zeroed!(JobTimestamps); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct RenderTimestamps { + pub(crate) vtx: JobTimestamps, + pub(crate) frag: JobTimestamps, + } + default_zeroed!(RenderTimestamps); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct Register { + pub(crate) number: u32, + pub(crate) value: U64, + } + default_zeroed!(Register); + + impl Register { + fn new(number: u32, value: u64) -> Register { + Register { + number, + value: U64(value), + } + } + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct RegisterArray { + pub(crate) registers: Array<128, Register>, + pub(crate) pad: Array<0x100, u8>, + + pub(crate) addr: GpuWeakPointer>, + pub(crate) count: u16, + pub(crate) length: u16, + pub(crate) unk_pad: u32, + } + + impl RegisterArray { + pub(crate) fn new( + self_ptr: GpuWeakPointer>, + cb: impl FnOnce(&mut RegisterArray), + ) -> RegisterArray { + let mut array = RegisterArray { + registers: Default::default(), + pad: Default::default(), + addr: self_ptr, + count: 0, + length: 0, + unk_pad: 0, + }; + + cb(&mut array); + + array + } + + pub(crate) fn add(&mut self, number: u32, value: u64) { + self.registers[self.count as usize] = Register::new(number, value); + self.count += 1; + self.length += core::mem::size_of::() as u16; + } + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct TimestampPointers<'a> { + pub(crate) start_addr: Option>, + pub(crate) end_addr: Option>, + } +} + +trivial_gpustruct!(JobTimestamps); +trivial_gpustruct!(RenderTimestamps); + +#[derive(Debug)] +pub(crate) struct UserTimestamp { + pub(crate) mapping: Arc, + pub(crate) offset: usize, +} + +#[derive(Debug, Default)] +pub(crate) struct UserTimestamps { + pub(crate) start: Option, + pub(crate) end: Option, +} + +impl UserTimestamps { + pub(crate) fn any(&self) -> bool { + self.start.is_some() || self.end.is_some() + } + + pub(crate) fn pointers(&self) -> Result> { + Ok(raw::TimestampPointers { + start_addr: self + .start + .as_ref() + .map(|a| GpuPointer::from_mapping(&a.mapping, a.offset)) + .transpose()?, + end_addr: self + .end + .as_ref() + .map(|a| GpuPointer::from_mapping(&a.mapping, a.offset)) + .transpose()?, + }) + } +} diff --git a/drivers/gpu/drm/asahi/fw/microseq.rs b/drivers/gpu/drm/asahi/fw/microseq.rs new file mode 100644 index 00000000000000..ff59deda6be615 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/microseq.rs @@ -0,0 +1,412 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU firmware microsequence operations + +use super::types::*; +use super::{ + buffer, + compute, + fragment, + initdata, + job, + vertex, + workqueue, // +}; +use crate::default_zeroed; + +pub(crate) trait Operation {} + +#[derive(Debug, Copy, Clone)] +#[repr(u32)] +enum OpCode { + WaitForIdle = 0x01, + WaitForIdle2 = 0x02, + RetireStamp = 0x18, + #[allow(dead_code)] + Timestamp = 0x19, + StartVertex = 0x22, + FinalizeVertex = 0x23, + StartFragment = 0x24, + FinalizeFragment = 0x25, + StartCompute = 0x29, + FinalizeCompute = 0x2a, +} + +#[derive(Debug, Copy, Clone)] +#[repr(u32)] +pub(crate) enum Pipe { + Vertex = 1 << 0, + Fragment = 1 << 8, + Compute = 1 << 15, +} + +pub(crate) const MAX_ATTACHMENTS: usize = 16; + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub(crate) struct Attachment { + pub(crate) address: U64, + pub(crate) size: u32, + pub(crate) unk_c: u16, + pub(crate) unk_e: u16, +} +default_zeroed!(Attachment); + +#[derive(Debug, Clone, Copy, Default)] +#[repr(C)] +pub(crate) struct Attachments { + pub(crate) list: Array, + pub(crate) count: u32, +} + +#[derive(Debug, Copy, Clone)] +#[repr(transparent)] +pub(crate) struct OpHeader(u32); + +impl OpHeader { + const fn new(opcode: OpCode) -> OpHeader { + OpHeader(opcode as u32) + } + const fn with_args(opcode: OpCode, args: u32) -> OpHeader { + OpHeader(opcode as u32 | args) + } +} + +macro_rules! simple_op { + ($name:ident) => { + #[allow(dead_code)] + #[derive(Debug, Copy, Clone)] + pub(crate) struct $name(OpHeader); + + impl $name { + pub(crate) const HEADER: $name = $name(OpHeader::new(OpCode::$name)); + } + }; +} + +pub(crate) mod op { + use super::*; + + simple_op!(StartVertex); + simple_op!(FinalizeVertex); + simple_op!(StartFragment); + simple_op!(FinalizeFragment); + simple_op!(StartCompute); + simple_op!(FinalizeCompute); + simple_op!(WaitForIdle2); + + #[allow(dead_code)] + #[derive(Debug, Copy, Clone)] + pub(crate) struct RetireStamp(OpHeader); + impl RetireStamp { + pub(crate) const HEADER: RetireStamp = + RetireStamp(OpHeader::with_args(OpCode::RetireStamp, 0x40000000)); + } + + #[allow(dead_code)] + #[derive(Debug, Copy, Clone)] + pub(crate) struct WaitForIdle(OpHeader); + impl WaitForIdle { + pub(crate) const fn new(pipe: Pipe) -> WaitForIdle { + WaitForIdle(OpHeader::with_args(OpCode::WaitForIdle, (pipe as u32) << 8)) + } + } + + #[allow(dead_code)] + #[derive(Debug, Copy, Clone)] + pub(crate) struct Timestamp(OpHeader); + impl Timestamp { + #[allow(dead_code)] + pub(crate) const fn new(flag: bool) -> Timestamp { + Timestamp(OpHeader::with_args(OpCode::Timestamp, (flag as u32) << 31)) + } + } +} + +#[derive(Debug)] +#[repr(C)] +pub(crate) struct WaitForIdle { + pub(crate) header: op::WaitForIdle, +} + +impl Operation for WaitForIdle {} + +#[derive(Debug)] +#[repr(C)] +pub(crate) struct WaitForIdle2 { + pub(crate) header: op::WaitForIdle2, +} + +impl Operation for WaitForIdle2 {} + +#[derive(Debug)] +#[repr(C)] +pub(crate) struct RetireStamp { + pub(crate) header: op::RetireStamp, +} + +impl Operation for RetireStamp {} + +#[versions(AGX)] +#[derive(Debug)] +#[repr(C)] +pub(crate) struct Timestamp<'a> { + pub(crate) header: op::Timestamp, + pub(crate) command_time: GpuWeakPointer, + pub(crate) ts_pointers: GpuWeakPointer>, + // Unused? + pub(crate) update_ts: GpuWeakPointer>>, + pub(crate) work_queue: GpuWeakPointer, + pub(crate) user_ts_pointers: GpuWeakPointer>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_ts: GpuWeakPointer, + + pub(crate) uuid: u32, + pub(crate) unk_30_padding: u32, +} + +#[versions(AGX)] +impl<'a> Operation for Timestamp::ver<'a> {} + +#[versions(AGX)] +#[derive(Debug)] +#[repr(C)] +pub(crate) struct StartVertex<'a> { + pub(crate) header: op::StartVertex, + pub(crate) tiling_params: Option>, + pub(crate) job_params1: Option>>, + #[ver(G >= G14X)] + pub(crate) registers: GpuWeakPointer, + pub(crate) buffer: GpuWeakPointer, + pub(crate) scene: GpuWeakPointer, + pub(crate) stats: GpuWeakPointer, + pub(crate) work_queue: GpuWeakPointer, + pub(crate) vm_slot: u32, + pub(crate) unk_38: u32, + pub(crate) event_generation: u32, + pub(crate) buffer_slot: u32, + pub(crate) unk_44: u32, + pub(crate) event_seq: U64, + pub(crate) unk_50: u32, + pub(crate) unk_pointer: GpuWeakPointer, + pub(crate) unk_job_buf: GpuWeakPointer, + pub(crate) unk_64: u32, + pub(crate) unk_68: u32, + pub(crate) uuid: u32, + pub(crate) attachments: Attachments, + pub(crate) padding: u32, + + #[ver(V >= V13_0B4)] + pub(crate) counter: U64, + + #[ver(V >= V13_0B4)] + pub(crate) notifier_buf: GpuWeakPointer>, + + pub(crate) unk_178: u32, +} + +#[versions(AGX)] +impl<'a> Operation for StartVertex::ver<'a> {} + +#[versions(AGX)] +#[derive(Debug)] +#[repr(C)] +pub(crate) struct FinalizeVertex { + pub(crate) header: op::FinalizeVertex, + pub(crate) scene: GpuWeakPointer, + pub(crate) buffer: GpuWeakPointer, + pub(crate) stats: GpuWeakPointer, + pub(crate) work_queue: GpuWeakPointer, + pub(crate) vm_slot: u32, + pub(crate) unk_28: u32, + pub(crate) unk_pointer: GpuWeakPointer, + pub(crate) unk_34: u32, + pub(crate) uuid: u32, + pub(crate) fw_stamp: GpuWeakPointer, + pub(crate) stamp_value: EventValue, + pub(crate) unk_48: U64, + pub(crate) unk_50: u32, + pub(crate) unk_54: u32, + pub(crate) unk_58: U64, + pub(crate) unk_60: u32, + pub(crate) unk_64: u32, + pub(crate) unk_68: u32, + + #[ver(G >= G14 && V < V13_0B4)] + pub(crate) unk_68_g14: U64, + + pub(crate) restart_branch_offset: i32, + pub(crate) has_attachments: u32, // Check DCMP errors bits 2,3 1=ktrace 2=log 3=panic + + #[ver(V >= V13_0B4)] + pub(crate) unk_74: Array<0x10, u8>, +} + +#[versions(AGX)] +impl Operation for FinalizeVertex::ver {} + +#[versions(AGX)] +#[derive(Debug)] +#[repr(C)] +pub(crate) struct StartFragment<'a> { + pub(crate) header: op::StartFragment, + pub(crate) job_params2: Option>, + pub(crate) job_params1: Option>>, + #[ver(G >= G14X)] + pub(crate) registers: GpuWeakPointer, + pub(crate) scene: GpuPointer<'a, buffer::Scene::ver>, + pub(crate) stats: GpuWeakPointer, + pub(crate) busy_flag: GpuWeakPointer, + pub(crate) tvb_overflow_count: GpuWeakPointer, + pub(crate) unk_pointer: GpuWeakPointer, + pub(crate) work_queue: GpuWeakPointer, + pub(crate) work_item: GpuWeakPointer, + pub(crate) vm_slot: u32, + pub(crate) unk_50: u32, + pub(crate) event_generation: u32, + pub(crate) buffer_slot: u32, + pub(crate) sync_grow: u32, + pub(crate) event_seq: U64, + pub(crate) unk_68: u32, + pub(crate) unk_758_flag: GpuWeakPointer, + pub(crate) unk_job_buf: GpuWeakPointer, + #[ver(V >= V13_3)] + pub(crate) unk_7c_0: U64, + pub(crate) unk_7c: u32, + pub(crate) unk_80: u32, + pub(crate) unk_84: u32, + pub(crate) uuid: u32, + pub(crate) attachments: Attachments, + pub(crate) padding: u32, + + #[ver(V >= V13_0B4)] + pub(crate) counter: U64, + + #[ver(V >= V13_0B4)] + pub(crate) notifier_buf: GpuWeakPointer>, +} + +#[versions(AGX)] +impl<'a> Operation for StartFragment::ver<'a> {} + +#[versions(AGX)] +#[derive(Debug)] +#[repr(C)] +pub(crate) struct FinalizeFragment { + pub(crate) header: op::FinalizeFragment, + pub(crate) uuid: u32, + pub(crate) unk_8: u32, + pub(crate) fw_stamp: GpuWeakPointer, + pub(crate) stamp_value: EventValue, + pub(crate) unk_18: u32, + pub(crate) scene: GpuWeakPointer, + pub(crate) buffer: GpuWeakPointer, + pub(crate) unk_2c: U64, + pub(crate) stats: GpuWeakPointer, + pub(crate) unk_pointer: GpuWeakPointer, + pub(crate) busy_flag: GpuWeakPointer, + pub(crate) work_queue: GpuWeakPointer, + pub(crate) work_item: GpuWeakPointer, + pub(crate) vm_slot: u32, + pub(crate) unk_60: u32, + pub(crate) unk_758_flag: GpuWeakPointer, + #[ver(V >= V13_3)] + pub(crate) unk_6c_0: U64, + pub(crate) unk_6c: U64, + pub(crate) unk_74: U64, + pub(crate) unk_7c: U64, + pub(crate) unk_84: U64, + pub(crate) unk_8c: U64, + + #[ver(G == G14 && V < V13_0B4)] + pub(crate) unk_8c_g14: U64, + + pub(crate) restart_branch_offset: i32, + pub(crate) has_attachments: u32, // Check DCMP errors bits 2,3 1=ktrace 2=log 3=panic + + #[ver(V >= V13_0B4)] + pub(crate) unk_9c: Array<0x10, u8>, +} + +#[versions(AGX)] +impl Operation for FinalizeFragment::ver {} + +#[versions(AGX)] +#[derive(Debug)] +#[repr(C)] +pub(crate) struct StartCompute<'a> { + pub(crate) header: op::StartCompute, + pub(crate) unk_pointer: GpuWeakPointer, + pub(crate) job_params1: Option>>, + #[ver(G >= G14X)] + pub(crate) registers: GpuWeakPointer, + pub(crate) stats: GpuWeakPointer, + pub(crate) work_queue: GpuWeakPointer, + pub(crate) vm_slot: u32, + pub(crate) unk_28: u32, + pub(crate) event_generation: u32, + pub(crate) event_seq: U64, + pub(crate) unk_38: u32, + pub(crate) job_params2: GpuWeakPointer>, + pub(crate) unk_44: u32, + pub(crate) uuid: u32, + pub(crate) attachments: Attachments, + pub(crate) padding: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_flag: GpuWeakPointer, + + #[ver(V >= V13_0B4)] + pub(crate) counter: U64, + + #[ver(V >= V13_0B4)] + pub(crate) notifier_buf: GpuWeakPointer>, +} + +#[versions(AGX)] +impl<'a> Operation for StartCompute::ver<'a> {} + +#[versions(AGX)] +#[derive(Debug)] +#[repr(C)] +pub(crate) struct FinalizeCompute<'a> { + pub(crate) header: op::FinalizeCompute, + pub(crate) stats: GpuWeakPointer, + pub(crate) work_queue: GpuWeakPointer, + pub(crate) vm_slot: u32, + #[ver(V < V13_0B4)] + pub(crate) unk_18: u32, + pub(crate) job_params2: GpuWeakPointer>, + pub(crate) unk_24: u32, + pub(crate) uuid: u32, + pub(crate) fw_stamp: GpuWeakPointer, + pub(crate) stamp_value: EventValue, + pub(crate) unk_38: u32, + pub(crate) unk_3c: u32, + pub(crate) unk_40: u32, + pub(crate) unk_44: u32, + pub(crate) unk_48: u32, + pub(crate) unk_4c: u32, + pub(crate) unk_50: u32, + pub(crate) unk_54: u32, + pub(crate) unk_58: u32, + + #[ver(G == G14 && V < V13_0B4)] + pub(crate) unk_5c_g14: U64, + + pub(crate) restart_branch_offset: i32, + pub(crate) has_attachments: u32, // Check DCMP errors bits 2,3 1=ktrace 2=log 3=panic + + #[ver(V >= V13_0B4)] + pub(crate) unk_64: Array<0xd, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_flag: GpuWeakPointer, + + #[ver(V >= V13_0B4)] + pub(crate) unk_79: Array<0x7, u8>, +} + +#[versions(AGX)] +impl<'a> Operation for FinalizeCompute::ver<'a> {} diff --git a/drivers/gpu/drm/asahi/fw/mod.rs b/drivers/gpu/drm/asahi/fw/mod.rs new file mode 100644 index 00000000000000..a5649aa20d3a8e --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/mod.rs @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Firmware structures for Apple AGX GPUs + +pub(crate) mod buffer; +pub(crate) mod channels; +pub(crate) mod compute; +pub(crate) mod event; +pub(crate) mod fragment; +pub(crate) mod initdata; +pub(crate) mod job; +pub(crate) mod microseq; +pub(crate) mod types; +pub(crate) mod vertex; +pub(crate) mod workqueue; diff --git a/drivers/gpu/drm/asahi/fw/types.rs b/drivers/gpu/drm/asahi/fw/types.rs new file mode 100644 index 00000000000000..f55d6cec6b8ca3 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/types.rs @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Common types for firmware structure definitions + +use crate::{alloc, object}; +use core::fmt; +use core::ops::{Deref, DerefMut, Index, IndexMut}; + +pub(crate) use crate::event::EventValue; +pub(crate) use crate::object::{ + GpuPointer, + GpuStruct, + GpuWeakPointer, // +}; +pub(crate) use crate::{ + f32, + float::F32, // +}; + +pub(crate) use core::fmt::Debug; +pub(crate) use core::marker::PhantomData; +pub(crate) use core::sync::atomic::{ + AtomicI32, + AtomicU32, + AtomicU64, // +}; +pub(crate) use kernel::macros::versions; +pub(crate) use kernel::prelude::Zeroable; + +// Make the trait visible +pub(crate) use crate::alloc::Allocator as _Allocator; + +/// General allocator type used for the driver +pub(crate) type Allocator = alloc::DefaultAllocator; + +/// General GpuObject type used for the driver +pub(crate) type GpuObject = + object::GpuObject>; + +/// General GpuArray type used for the driver +pub(crate) type GpuArray = object::GpuArray>; + +/// General GpuOnlyArray type used for the driver +pub(crate) type GpuOnlyArray = + object::GpuOnlyArray>; + +/// A stamp slot that is shared between firmware and the driver. +#[derive(Debug, Default)] +#[repr(transparent)] +pub(crate) struct Stamp(pub(crate) AtomicU32); + +/// A stamp slot that is for private firmware use. +/// +/// This is a separate type to guard against pointer type confusion. +#[derive(Debug, Default)] +#[repr(transparent)] +pub(crate) struct FwStamp(pub(crate) AtomicU32); + +/// An unaligned u64 type. +/// +/// This is useful to avoid having to pack firmware structures entirely, since that is incompatible +/// with `#[derive(Debug)]` and atomics. +#[derive(Copy, Clone, Default)] +#[repr(C, packed(1))] +pub(crate) struct U64(pub(crate) u64); + +// SAFETY: U64 is zeroable just like u64 +unsafe impl Zeroable for U64 {} + +impl fmt::Debug for U64 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let v = self.0; + f.write_fmt(format_args!("{:#x}", v)) + } +} + +/// An unaligned u32 type. +/// +/// This is useful to avoid having to pack firmware structures entirely, since that is incompatible +/// with `#[derive(Debug)]` and atomics. +#[derive(Copy, Clone, Default)] +#[repr(C, packed(1))] +pub(crate) struct U32(pub(crate) u32); + +// SAFETY: U32 is zeroable just like u32 +unsafe impl Zeroable for U32 {} + +impl fmt::Debug for U32 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let v = self.0; + f.write_fmt(format_args!("{:#x}", v)) + } +} + +/// Create a dummy `Debug` implementation, for when we need it but it's too painful to write by +/// hand or not very useful. +#[macro_export] +macro_rules! no_debug { + ($type:ty) => { + impl ::core::fmt::Debug for $type { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "...") + } + } + }; +} + +/// Implement Zeroable for a given type (and Default along with it). +/// +/// # Safety +/// +/// This macro must only be used if a type only contains primitive types which can be +/// zero-initialized, FFI structs intended to be zero-initialized, or other types which +/// impl Zeroable. +#[macro_export] +macro_rules! default_zeroed { + (<$($lt:lifetime),*>, $type:ty) => { + impl<$($lt),*> Default for $type { + fn default() -> $type { + unsafe { core::mem::zeroed() } + } + } + // SAFETY: The user is responsible for ensuring this is safe. + unsafe impl<$($lt),*> ::pin_init::Zeroable for $type {} + }; + ($type:ty) => { + impl Default for $type { + fn default() -> $type { + unsafe { core::mem::zeroed() } + } + } + // SAFETY: The user is responsible for ensuring this is safe. + unsafe impl ::pin_init::Zeroable for $type {} + }; +} + +/// A convenience type for a number of padding bytes. Hidden from Debug formatting. +#[derive(Copy, Clone)] +#[repr(C, packed)] +pub(crate) struct Pad([u8; N]); + +/// SAFETY: Primitive type, safe to zero-init. +unsafe impl Zeroable for Pad {} + +impl Default for Pad { + fn default() -> Self { + unsafe { core::mem::zeroed() } + } +} + +impl fmt::Debug for Pad { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_fmt(format_args!("")) + } +} + +/// A convenience type for a fixed-sized array with Default/Zeroable impls. +#[derive(Copy, Clone)] +#[repr(C)] +pub(crate) struct Array([T; N]); + +impl Array { + pub(crate) fn new(data: [T; N]) -> Self { + Self(data) + } +} + +// SAFETY: Arrays of Zeroable values can be safely Zeroable. +unsafe impl Zeroable for Array {} + +impl Default for Array { + fn default() -> Self { + unsafe { core::mem::zeroed() } + } +} + +impl Index for Array { + type Output = T; + + fn index(&self, index: usize) -> &Self::Output { + &self.0[index] + } +} + +impl IndexMut for Array { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + &mut self.0[index] + } +} + +impl Deref for Array { + type Target = [T; N]; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for Array { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl fmt::Debug for Array { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +/// Convenience macro to define an identically-named trivial GpuStruct with no inner fields for a +/// given raw type name. +#[macro_export] +macro_rules! trivial_gpustruct { + ($type:ident) => { + #[derive(Debug)] + pub(crate) struct $type {} + + impl GpuStruct for $type { + type Raw<'a> = raw::$type; + } + $crate::default_zeroed!($type); + }; +} diff --git a/drivers/gpu/drm/asahi/fw/vertex.rs b/drivers/gpu/drm/asahi/fw/vertex.rs new file mode 100644 index 00000000000000..07a0e05c72112c --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/vertex.rs @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU vertex job firmware structures + +use super::types::*; +use super::{ + event, + job, + workqueue, // +}; +use crate::{ + buffer, + fw, + microseq, + mmu, // +}; +use kernel::sync::Arc; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug, Default, Copy, Clone)] + #[repr(C)] + pub(crate) struct TilingParameters { + pub(crate) rgn_size: u32, + pub(crate) unk_4: u32, + pub(crate) ppp_ctrl: u32, + pub(crate) x_max: u16, + pub(crate) y_max: u16, + pub(crate) te_screen: u32, + pub(crate) te_mtile1: u32, + pub(crate) te_mtile2: u32, + pub(crate) tiles_per_mtile: u32, + pub(crate) tpc_stride: u32, + pub(crate) unk_24: u32, + pub(crate) unk_28: u32, + pub(crate) helper_cfg: u32, + pub(crate) __pad: Pad<0x70>, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobParameters1<'a> { + pub(crate) unk_0: U64, + pub(crate) unk_8: F32, + pub(crate) unk_c: F32, + pub(crate) tvb_tilemap: GpuPointer<'a, &'a [u8]>, + #[ver(G < G14)] + pub(crate) tvb_cluster_tilemaps: Option>, + pub(crate) tpc: GpuPointer<'a, &'a [u8]>, + pub(crate) tvb_heapmeta: GpuPointer<'a, &'a [u8]>, + pub(crate) iogpu_unk_54: U64, + pub(crate) iogpu_unk_56: U64, + #[ver(G < G14)] + pub(crate) tvb_cluster_meta1: Option>, + pub(crate) utile_config: u32, + pub(crate) unk_4c: u32, + pub(crate) ppp_multisamplectl: U64, + pub(crate) tvb_layermeta: GpuPointer<'a, &'a [u8]>, + #[ver(G < G14)] + pub(crate) tvb_cluster_layermeta: Option>, + #[ver(G < G14)] + pub(crate) core_mask: Array<2, u32>, + pub(crate) preempt_buf1: GpuPointer<'a, &'a [u8]>, + pub(crate) preempt_buf2: GpuPointer<'a, &'a [u8]>, + pub(crate) unk_80: U64, + pub(crate) preempt_buf3: GpuPointer<'a, &'a [u8]>, + pub(crate) vdm_ctrl_stream_base: U64, + #[ver(G < G14)] + pub(crate) tvb_cluster_meta2: Option>, + #[ver(G < G14)] + pub(crate) tvb_cluster_meta3: Option>, + #[ver(G < G14)] + pub(crate) tiling_control: u32, + #[ver(G < G14)] + pub(crate) unk_ac: u32, + pub(crate) unk_b0: Array<6, U64>, + pub(crate) usc_exec_base_ta: U64, + #[ver(G < G14)] + pub(crate) tvb_cluster_meta4: Option>, + #[ver(G < G14)] + pub(crate) unk_f0: U64, + pub(crate) unk_f8: U64, + pub(crate) helper_program: u32, + pub(crate) unk_104: u32, + pub(crate) helper_arg: U64, + pub(crate) unk_110: U64, + pub(crate) unk_118: u32, + #[ver(G >= G14)] + pub(crate) __pad: Pad<{ 8 * 9 + 0x268 }>, + #[ver(G < G14)] + pub(crate) __pad: Pad<0x268>, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobParameters2<'a> { + pub(crate) unk_480: Array<4, u32>, + pub(crate) unk_498: U64, + pub(crate) unk_4a0: u32, + pub(crate) preempt_buf1: GpuPointer<'a, &'a [u8]>, + pub(crate) unk_4ac: u32, + pub(crate) unk_4b0: U64, + pub(crate) unk_4b8: u32, + pub(crate) unk_4bc: U64, + pub(crate) unk_4c4_padding: Array<0x48, u8>, + pub(crate) unk_50c: u32, + pub(crate) unk_510: U64, + pub(crate) unk_518: U64, + pub(crate) unk_520: U64, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct RunVertex<'a> { + pub(crate) tag: workqueue::CommandType, + + #[ver(V >= V13_0B4)] + pub(crate) counter: U64, + + pub(crate) vm_slot: u32, + pub(crate) unk_8: u32, + pub(crate) notifier: GpuPointer<'a, event::Notifier::ver>, + pub(crate) buffer_slot: u32, + pub(crate) unk_1c: u32, + pub(crate) buffer: GpuPointer<'a, fw::buffer::Info::ver>, + pub(crate) scene: GpuPointer<'a, fw::buffer::Scene::ver>, + pub(crate) unk_buffer_buf: GpuWeakPointer<[u8]>, + pub(crate) unk_34: u32, + + #[ver(G < G14X)] + pub(crate) job_params1: JobParameters1::ver<'a>, + #[ver(G < G14X)] + pub(crate) tiling_params: TilingParameters, + #[ver(G >= G14X)] + pub(crate) registers: job::raw::RegisterArray, + + pub(crate) tpc: GpuPointer<'a, &'a [u8]>, + pub(crate) tpc_size: U64, + pub(crate) microsequence: GpuPointer<'a, &'a [u8]>, + pub(crate) microsequence_size: u32, + pub(crate) fragment_stamp_slot: u32, + pub(crate) fragment_stamp_value: EventValue, + pub(crate) unk_pointee: u32, + pub(crate) unk_pad: u32, + pub(crate) job_params2: JobParameters2<'a>, + pub(crate) encoder_params: job::raw::EncoderParams, + pub(crate) unk_55c: u32, + pub(crate) unk_560: u32, + pub(crate) sync_grow: u32, + pub(crate) unk_568: u32, + pub(crate) uses_scratch: u32, + pub(crate) meta: job::raw::JobMeta, + pub(crate) unk_after_meta: u32, + pub(crate) unk_buf_0: U64, + pub(crate) unk_buf_8: U64, + pub(crate) unk_buf_10: U64, + pub(crate) command_time: U64, + pub(crate) timestamp_pointers: job::raw::TimestampPointers<'a>, + pub(crate) user_timestamp_pointers: job::raw::TimestampPointers<'a>, + pub(crate) client_sequence: u8, + pub(crate) pad_5d5: Array<3, u8>, + pub(crate) unk_5d8: u32, + pub(crate) unk_5dc: u8, + + #[ver(V >= V13_0B4)] + pub(crate) unk_ts: U64, + + #[ver(V >= V13_0B4)] + pub(crate) unk_5dd_8: Array<0x1b, u8>, + } +} + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct RunVertex { + pub(crate) notifier: Arc>, + pub(crate) scene: Arc, + pub(crate) micro_seq: microseq::MicroSequence, + pub(crate) vm_bind: mmu::VmBind, + pub(crate) timestamps: Arc>, + pub(crate) user_timestamps: job::UserTimestamps, +} + +#[versions(AGX)] +impl GpuStruct for RunVertex::ver { + type Raw<'a> = raw::RunVertex::ver<'a>; +} + +#[versions(AGX)] +impl workqueue::Command for RunVertex::ver {} diff --git a/drivers/gpu/drm/asahi/fw/workqueue.rs b/drivers/gpu/drm/asahi/fw/workqueue.rs new file mode 100644 index 00000000000000..b86bd3ff3757af --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/workqueue.rs @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU work queue firmware structes + +use super::event; +use super::types::*; +use crate::event::EventValue; +use crate::{ + default_zeroed, + trivial_gpustruct, // +}; +use kernel::sync::Arc; + +#[derive(Debug)] +#[repr(u32)] +pub(crate) enum CommandType { + RunVertex = 0, + RunFragment = 1, + #[allow(dead_code)] + RunBlitter = 2, + RunCompute = 3, + Barrier = 4, + InitBuffer = 6, +} + +pub(crate) trait Command: GpuStruct + Send + Sync {} + +pub(crate) mod raw { + use super::*; + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct Barrier { + pub(crate) tag: CommandType, + pub(crate) wait_stamp: GpuWeakPointer, + pub(crate) wait_value: EventValue, + pub(crate) wait_slot: u32, + pub(crate) stamp_self: EventValue, + pub(crate) uuid: u32, + pub(crate) external_barrier: u32, + // G14X addition + pub(crate) internal_barrier_type: u32, + pub(crate) padding: Pad<0x1c>, + } + + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct GpuContextData { + pub(crate) unk_0: u8, + pub(crate) unk_1: u8, + unk_2: Array<0x2, u8>, + pub(crate) unk_4: u8, + pub(crate) unk_5: u8, + unk_6: Array<0x18, u8>, + pub(crate) unk_1e: u8, + pub(crate) unk_1f: u8, + unk_20: Array<0x3, u8>, + pub(crate) unk_23: u8, + unk_24: Array<0x1c, u8>, + } + + impl Default for GpuContextData { + fn default() -> Self { + Self { + unk_0: 0xff, + unk_1: 0xff, + unk_2: Default::default(), + unk_4: 0, + unk_5: 1, + unk_6: Default::default(), + unk_1e: 0xff, + unk_1f: 0, + unk_20: Default::default(), + unk_23: 2, + unk_24: Default::default(), + } + } + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct RingState { + pub(crate) gpu_doneptr: AtomicU32, + __pad0: Pad<0xc>, + pub(crate) unk_10: AtomicU32, + __pad1: Pad<0xc>, + pub(crate) unk_20: AtomicU32, + __pad2: Pad<0xc>, + pub(crate) gpu_rptr: AtomicU32, + __pad3: Pad<0xc>, + pub(crate) cpu_wptr: AtomicU32, + __pad4: Pad<0xc>, + pub(crate) rb_size: u32, + __pad5: Pad<0xc>, + // This isn't part of the structure, but it's here as a + // debugging hack so we can inspect what ring position + // the driver considered complete and freeable. + pub(crate) cpu_freeptr: AtomicU32, + __pad6: Pad<0xc>, + } + default_zeroed!(RingState); + + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct Priority( + pub(crate) u32, + pub(crate) u32, + pub(crate) U64, + pub(crate) u32, + pub(crate) u32, + pub(crate) u32, + ); + + pub(crate) const PRIORITY: [Priority; 4] = [ + Priority(0, 0, U64(0xffff_ffff_ffff_0000), 1, 0, 1), + Priority(1, 1, U64(0xffff_ffff_0000_0000), 0, 0, 0), + Priority(2, 2, U64(0xffff_0000_0000_0000), 0, 0, 2), + Priority(3, 3, U64(0x0000_0000_0000_0000), 0, 0, 3), + ]; + + impl Default for Priority { + fn default() -> Priority { + PRIORITY[2] + } + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct QueueInfo<'a> { + pub(crate) state: GpuPointer<'a, super::RingState>, + pub(crate) ring: GpuPointer<'a, &'a [u64]>, + pub(crate) notifier_list: GpuPointer<'a, event::NotifierList>, + pub(crate) gpu_buf: GpuPointer<'a, &'a [u8]>, + pub(crate) gpu_rptr1: AtomicU32, + pub(crate) gpu_rptr2: AtomicU32, + pub(crate) gpu_rptr3: AtomicU32, + pub(crate) event_id: AtomicI32, + pub(crate) priority: Priority, + pub(crate) unk_4c: i32, + pub(crate) uuid: u32, + pub(crate) unk_54: i32, + pub(crate) unk_58: U64, + pub(crate) busy: AtomicU32, + pub(crate) __pad: Pad<0x20>, + #[ver(V >= V13_2 && G < G14X)] + pub(crate) unk_84_0: u32, + pub(crate) unk_84_state: AtomicU32, + pub(crate) error_count: AtomicU32, + pub(crate) unk_8c: u32, + pub(crate) unk_90: u32, + pub(crate) unk_94: u32, + pub(crate) pending: AtomicU32, + pub(crate) unk_9c: u32, + pub(crate) gpu_context: GpuPointer<'a, super::GpuContextData>, + pub(crate) unk_a8: U64, + #[ver(V >= V13_2 && G < G14X)] + pub(crate) unk_b0: u32, + } +} + +trivial_gpustruct!(Barrier); +trivial_gpustruct!(RingState); + +impl Command for Barrier {} + +pub(crate) struct GpuContextData { + pub(crate) _buffer: Arc, +} +impl GpuStruct for GpuContextData { + type Raw<'a> = raw::GpuContextData; +} + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct QueueInfo { + pub(crate) state: GpuObject, + pub(crate) ring: GpuArray, + pub(crate) gpu_buf: GpuArray, + pub(crate) notifier_list: Arc>, + pub(crate) gpu_context: Arc, +} + +#[versions(AGX)] +impl GpuStruct for QueueInfo::ver { + type Raw<'a> = raw::QueueInfo::ver<'a>; +} diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs new file mode 100644 index 00000000000000..8affba257d956f --- /dev/null +++ b/drivers/gpu/drm/asahi/gem.rs @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Asahi driver GEM object implementation +//! +//! Basic wrappers and adaptations between generic GEM shmem objects and this driver's +//! view of what a GPU buffer object is. It is in charge of keeping track of all mappings for +//! each GEM object so we can remove them when a client (File) or a Vm are destroyed, as well as +//! implementing RTKit buffers on top of GEM objects for firmware use. + +use kernel::{ + drm, + drm::gem::{ + shmem, + shmem::VMap, + BaseObject, + DriverObject, // + }, + error::Result, + prelude::*, + types::ARef, + uapi, +}; + +use core::ops::Range; +use core::sync::atomic::{ + AtomicU64, + Ordering, // +}; + +use crate::{ + debug::*, + driver::{ + AsahiDevice, + AsahiDriver, // + }, + file, + mmu, + util::*, // +}; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Gem; + +/// Represents the inner data of a GEM object for this driver. +#[pin_data] +pub(crate) struct AsahiObject { + /// ID for debug + id: u64, + /// Object creation flags. + flags: u32, + /// Whether this object can be exported. + exportable: bool, + /// Whether this is a kernel-created object. + kernel: bool, +} + +/// Type alias for the shmem GEM object type for this driver. +pub(crate) type Object = shmem::Object; + +unsafe impl Send for AsahiObject {} +unsafe impl Sync for AsahiObject {} + +// /// Type alias for the SGTable type for this driver. +// pub(crate) type SGTable = shmem::SGTable; + +/// A shared reference to a GEM object for this driver. +pub(crate) struct ObjectRef { + /// The underlying GEM object reference + pub(crate) gem: ARef, + /// The kernel-side VMap of this object, if needed + vmap: Option>, +} + +crate::no_debug!(ObjectRef); + +static GEM_ID: AtomicU64 = AtomicU64::new(0); + +impl ObjectRef { + /// Create a new wrapper for a raw GEM object reference. + pub(crate) fn new(gem: ARef) -> ObjectRef { + ObjectRef { gem, vmap: None } + } + + /// Return the `VMap` for this object, creating it if necessary. + pub(crate) fn vmap(&mut self) -> Result> { + if self.vmap.is_none() { + self.vmap = Some(self.gem.owned_vmap()?); + } + self.gem.vmap() + } + + /// Returns the size of an object in bytes + pub(crate) fn size(&self) -> usize { + self.gem.size() + } + + /// Maps an object into a given `Vm` at any free address within a given range. + pub(crate) fn map_into_range( + &mut self, + vm: &crate::mmu::Vm, + range: Range, + alignment: u64, + prot: mmu::Prot, + guard: bool, + ) -> Result { + // Only used for kernel objects now + if !self.gem.kernel { + return Err(EINVAL); + } + vm.map_in_range(&self.gem, 0..self.gem.size(), alignment, range, prot, guard) + } + + /// Maps a range within an object into a given `Vm` at any free address within a given range. + pub(crate) fn map_range_into_range( + &mut self, + vm: &crate::mmu::Vm, + obj_range: Range, + range: Range, + alignment: u64, + prot: mmu::Prot, + guard: bool, + ) -> Result { + if obj_range.end > self.gem.size() { + return Err(EINVAL); + } + if self.gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 + && vm.is_extobj(&*self.gem) + { + return Err(EINVAL); + } + vm.map_in_range(&self.gem, obj_range, alignment, range, prot, guard) + } + + /// Maps an object into a given `Vm` at a specific address. + /// + /// Returns Err(ENOSPC) if the requested address is already busy. + pub(crate) fn map_at( + &mut self, + vm: &crate::mmu::Vm, + addr: u64, + prot: mmu::Prot, + guard: bool, + ) -> Result { + if self.gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 + && vm.is_extobj(&*self.gem) + { + return Err(EINVAL); + } + + vm.map_at(addr, self.gem.size(), self.gem.clone(), prot, guard) + } +} + +pub(crate) struct AsahiObjConfig { + flags: u32, + exportable: bool, + kernel: bool, +} + +/// Create a new kernel-owned GEM object. +pub(crate) fn new_kernel_object(dev: &AsahiDevice, size: usize) -> Result { + let gem = shmem::Object::::new( + dev, + align(size, mmu::UAT_PGSZ), + shmem::ObjectConfig:: { + map_wc: false, + parent_resv_obj: None, + }, + AsahiObjConfig { + flags: 0, + exportable: false, + kernel: true, + }, + )?; + + mod_pr_debug!("AsahiObject new kernel object id={}\n", gem.id); + Ok(ObjectRef::new(gem)) +} + +/// Create a new user-owned GEM object with the given flags. +pub(crate) fn new_object( + dev: &AsahiDevice, + size: usize, + flags: u32, + parent_object: Option<&shmem::Object>, +) -> Result> { + if (flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0) != parent_object.is_some() + { + return Err(EINVAL); + } + + let gem = shmem::Object::::new( + dev, + align(size, mmu::UAT_PGSZ), + shmem::ObjectConfig:: { + map_wc: flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_WRITEBACK == 0, + parent_resv_obj: parent_object, + }, + AsahiObjConfig { + flags, + exportable: parent_object.is_none(), + kernel: false, + }, + )?; + + mod_pr_debug!("AsahiObject new user object: id={}\n", gem.id); + Ok(gem) +} + +#[vtable] +impl DriverObject for AsahiObject { + type Driver = AsahiDriver; + type Args = AsahiObjConfig; + + const HAS_EXPORT: bool = true; + + /// Callback to create the inner data of a GEM object + fn new(_dev: &AsahiDevice, _size: usize, args: Self::Args) -> impl PinInit { + let id = GEM_ID.fetch_add(1, Ordering::Relaxed); + mod_pr_debug!("AsahiObject::new id={}\n", id); + try_pin_init!(AsahiObject { + id, + flags: args.flags, + exportable: args.exportable, + kernel: args.kernel, + }) + } + + /// Callback to drop all mappings for a GEM object owned by a given `File` + fn close(obj: &::Object, file: &drm::gem::DriverFile) { + // fn close(obj: &Object, file: &DrmFile) { + mod_pr_debug!("AsahiObject::close id={}\n", obj.id); + if file::File::unbind_gem_object(file, obj).is_err() { + pr_err!("AsahiObject::close: Failed to unbind GEM object\n"); + } + } + + /// Optional handle for exporting a gem object. + fn export( + obj: &::Object, + flags: u32, + ) -> Result> { + if !obj.exportable { + return Err(EINVAL); + } + + obj.prime_export(flags) + } +} diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs new file mode 100644 index 00000000000000..b6c1c64c503671 --- /dev/null +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -0,0 +1,1642 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Top-level GPU manager +//! +//! This module is the root of all GPU firmware management for a given driver instance. It is +//! responsible for initialization, owning the top-level managers (events, UAT, etc.), and +//! communicating with the raw RtKit endpoints to send and receive messages to/from the GPU +//! firmware. +//! +//! It is also the point where diverging driver firmware/GPU variants (using the versions macro) +//! are unified, so that the top level of the driver itself (in `driver`) does not have to concern +//! itself with version dependence. + +use core::any::Any; +use core::ops::Range; +use core::slice; +use core::sync::atomic::{ + AtomicBool, + AtomicU64, + Ordering, // +}; + +use kernel::{ + c_str, + drm::gem::shmem, + error::code::*, + io::mem::{ + Mem, + MemFlag, // + }, + iosys_map::IoSysMapRef, + macros::versions, + new_mutex, + prelude::*, + soc::apple::rtkit, + sync::{ + lock::{ + mutex::MutexBackend, + Guard, // + }, + Arc, + Mutex, + UniqueArc, // + }, + time::{ + Delta, + Instant, + Monotonic, // + }, + types::ForeignOwnable, // +}; +#[cfg(CONFIG_DEV_COREDUMP)] +use kernel::{ + devcoredump, + time::msecs_to_jiffies, // +}; + +use crate::alloc::Allocator; +use crate::debug::*; +use crate::driver::{ + AsahiDevRef, + AsahiDevice, // +}; +use crate::fw::channels::{ + ChannelErrorType, + PipeType, // +}; +use crate::fw::types::{ + U32, + U64, // +}; +use crate::{ + alloc, + buffer, + channel, + event, + fw, + gem, + hw, + initdata, + mem, + mmu, + queue, + regs, + workqueue, // +}; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Gpu; + +/// Firmware endpoint for init & incoming notifications. +const EP_FIRMWARE: u8 = 0x20; + +/// Doorbell endpoint for work/message submissions. +const EP_DOORBELL: u8 = 0x21; + +/// Initialize the GPU firmware. +const MSG_INIT: u64 = 0x81 << 48; +const INIT_DATA_MASK: u64 = (1 << 44) - 1; + +/// TX channel doorbell. +const MSG_TX_DOORBELL: u64 = 0x83 << 48; +/// Firmware control channel doorbell. +const MSG_FWCTL: u64 = 0x84 << 48; +// /// Halt the firmware (?). +// const MSG_HALT: u64 = 0x85 << 48; + +/// Receive channel doorbell notification. +const MSG_RX_DOORBELL: u64 = 0x42 << 48; + +/// Doorbell number for firmware kicks/wakeups. +const DOORBELL_KICKFW: u64 = 0x10; +/// Doorbell number for device control channel kicks. +const DOORBELL_DEVCTRL: u64 = 0x11; + +// Upper kernel half VA address ranges. +/// Private (cached) firmware structure VA range base. +const IOVA_KERN_PRIV_RANGE: Range = 0xffffffa000000000..0xffffffa600000000; +/// Private (cached) GPU-RO firmware structure VA range base. +const IOVA_KERN_GPU_RO_RANGE: Range = 0xffffffa600000000..0xffffffa800000000; +/// Shared (uncached) firmware structure VA range base. +const IOVA_KERN_SHARED_RANGE: Range = 0xffffffa800000000..0xffffffaa00000000; +/// Shared (uncached) read-only firmware structure VA range base. +const IOVA_KERN_SHARED_RO_RANGE: Range = 0xffffffaa00000000..0xffffffac00000000; +/// GPU/FW shared structure VA range base. +const IOVA_KERN_GPU_RANGE: Range = 0xffffffac00000000..0xffffffae00000000; +/// GPU/FW shared structure VA range base. +const IOVA_KERN_RTKIT_RANGE: Range = 0xffffffae00000000..0xffffffae10000000; +/// Shared (uncached) timestamp region. +pub(crate) const IOVA_KERN_TIMESTAMP_RANGE: Range = 0xffffffae10000000..0xffffffae14000000; +/// FW MMIO VA range base. +const IOVA_KERN_MMIO_RANGE: Range = 0xffffffaf00000000..0xffffffb000000000; + +/// GPU/FW buffer manager control address (context 0 low) +pub(crate) const IOVA_KERN_GPU_BUFMGR_LOW: u64 = 0x20_0000_0000; +/// GPU/FW buffer manager control address (context 0 high) +pub(crate) const IOVA_KERN_GPU_BUFMGR_HIGH: u64 = 0xffffffaeffff0000; + +/// Timeout for entering the halt state after a fault or request. +const HALT_ENTER_TIMEOUT: Delta = Delta::from_millis(100); + +/// Maximum amount of firmware-private memory garbage allowed before collection. +/// Collection flushes the FW cache and is expensive, so this needs to be +/// reasonably high. +const MAX_FW_ALLOC_GARBAGE_BYTES: usize = 16 * 1024 * 1024; +/// Maximum count of firmware-private memory garbage objects allowed before collection. +/// This works out to 16K of memory in the garbage list (8 bytes each), which keeps us +/// within the safe range for kmalloc (on 16K page systems). +const MAX_FW_ALLOC_GARBAGE_OBJECTS: usize = 2048; + +/// Global allocators used for kernel-half structures. +pub(crate) struct KernelAllocators { + pub(crate) private: alloc::DefaultAllocator, + pub(crate) shared: alloc::DefaultAllocator, + pub(crate) shared_ro: alloc::DefaultAllocator, + #[allow(dead_code)] + pub(crate) gpu: alloc::DefaultAllocator, + pub(crate) gpu_ro: alloc::DefaultAllocator, +} + +/// Receive (GPU->driver) ring buffer channels. +#[versions(AGX)] +#[pin_data] +struct RxChannels { + event: channel::EventChannel::ver, + fw_log: channel::FwLogChannel, + ktrace: channel::KTraceChannel, + stats: channel::StatsChannel::ver, +} + +/// GPU work submission pipe channels (driver->GPU). +#[versions(AGX)] +struct PipeChannels { + pub(crate) vtx: KVec>>>, + pub(crate) frag: KVec>>>, + pub(crate) comp: KVec>>>, +} + +/// Misc command transmit (driver->GPU) channels. +#[versions(AGX)] +#[pin_data] +struct TxChannels { + pub(crate) device_control: channel::DeviceControlChannel::ver, +} + +/// Number of work submission pipes per type, one for each priority level. +const NUM_PIPES: usize = 4; + +/// A generic monotonically incrementing ID used to uniquely identify object instances within the +/// driver. +pub(crate) struct ID(AtomicU64); + +impl ID { + /// Create a new ID counter with a given value. + fn new(val: u64) -> ID { + ID(AtomicU64::new(val)) + } + + /// Fetch the next unique ID. + pub(crate) fn next(&self) -> u64 { + self.0.fetch_add(1, Ordering::Relaxed) + } +} + +impl Default for ID { + /// IDs default to starting at 2, as 0/1 are considered reserved for the system. + fn default() -> Self { + Self::new(2) + } +} + +/// A guard representing one active submission on the GPU. When dropped, decrements the active +/// submission count. +pub(crate) struct OpGuard(Arc); + +impl Drop for OpGuard { + fn drop(&mut self) { + self.0.end_op(); + } +} + +/// Set of global sequence IDs used in the driver. +#[derive(Default)] +pub(crate) struct SequenceIDs { + /// `File` instance ID. + pub(crate) file: ID, + /// `Vm` instance ID. + pub(crate) vm: ID, + /// Submission instance ID. + pub(crate) submission: ID, + /// `Queue` instance ID. + pub(crate) queue: ID, +} + +/// Top-level GPU manager that owns all the global state relevant to the driver instance. +#[versions(AGX)] +#[pin_data] +pub(crate) struct GpuManager { + dev: AsahiDevRef, + cfg: &'static hw::HwConfig, + dyncfg: hw::DynConfig, + pub(crate) initdata: fw::types::GpuObject, + uat: mmu::Uat, + crashed: AtomicBool, + #[pin] + alloc: Mutex, + io_mappings: KVec, + next_mmio_iova: u64, + #[pin] + rtkit: Mutex>>, + #[pin] + rx_channels: Mutex, + #[pin] + tx_channels: Mutex, + #[pin] + fwctl_channel: Mutex, + pipes: PipeChannels::ver, + event_manager: Arc, + buffer_mgr: buffer::BufferManager::ver, + ids: SequenceIDs, + #[allow(clippy::vec_box)] + #[pin] + garbage_contexts: Mutex>>>, +} + +/// Trait used to abstract the firmware/GPU-dependent variants of the GpuManager. +pub(crate) trait GpuManager: Send + Sync { + /// Cast as an Any type. + fn as_any(&self) -> &dyn Any; + /// Cast Arc as an Any type. + fn arc_as_any(self: Arc) -> Arc; + /// Initialize the GPU. + fn init(&self) -> Result; + /// Update the GPU globals from global info + /// + /// TODO: Unclear what can and cannot be updated like this. + fn update_globals(&self); + /// Get a reference to the KernelAllocators. + fn alloc(&self) -> Guard<'_, KernelAllocators, MutexBackend>; + /// Create a new `Vm` given a unique `File` ID. + fn new_vm(&self, kernel_range: Range) -> Result; + /// Bind a `Vm` to an available slot and return the `VmBind`. + fn bind_vm(&self, vm: &mmu::Vm) -> Result; + /// Create a new user command queue. + fn new_queue( + &self, + vm: mmu::Vm, + ualloc: Arc>, + ualloc_priv: Arc>, + priority: u32, + usc_exec_base: u64, + ) -> Result>; + /// Return a reference to the global `SequenceIDs` instance. + fn ids(&self) -> &SequenceIDs; + /// Kick the firmware (wake it up if asleep). + /// + /// This should be useful to reduce latency on work submission, so we can ask the firmware to + /// wake up while we do some preparatory work for the work submission. + fn kick_firmware(&self) -> Result; + /// Flush the entire firmware cache. + /// + /// TODO: Does this actually work? + fn flush_fw_cache(&self) -> Result; + /// Handle a GPU work timeout event. + fn handle_timeout(&self, counter: u32, event_slot: i32, unk: u32); + /// Handle a GPU fault event. + fn handle_fault(&self); + /// Handle a channel error event. + fn handle_channel_error( + &self, + error_type: ChannelErrorType, + pipe_type: u32, + event_slot: u32, + event_value: u32, + ); + /// Acknowledge a Buffer grow op. + fn ack_grow(&self, buffer_slot: u32, vm_slot: u32, counter: u32); + /// Send a firmware control command (secure cache flush). + fn fwctl(&self, msg: fw::channels::FwCtlMsg) -> Result; + /// Get the static GPU configuration for this SoC. + fn get_cfg(&self) -> &'static hw::HwConfig; + /// Get the dynamic GPU configuration for this SoC. + fn get_dyncfg(&self) -> &hw::DynConfig; + /// Register an unused context as garbage + fn free_context(&self, data: KBox>); + /// Check whether the GPU is crashed + fn is_crashed(&self) -> bool; + /// Map a BO as a timestamp buffer + fn map_timestamp_buffer( + &self, + bo: gem::ObjectRef, + range: Range, + ) -> Result; +} + +/// Private generic trait for functions that don't need to escape this module. +trait GpuManagerPriv { + /// Decrement the pending submission counter. + fn end_op(&self); +} + +pub(crate) struct RtkitObject { + vmap: shmem::VMap, + mapping: mmu::KernelMapping, +} + +impl rtkit::Buffer for RtkitObject { + fn iova(&self) -> Result { + Ok(self.mapping.iova() as usize) + } + fn buf(&mut self) -> Result> { + Ok(self.vmap.get()) + } +} + +#[versions(AGX)] +#[vtable] +impl rtkit::Operations for GpuManager::ver { + type Data = Arc; + type Buffer = RtkitObject; + + fn recv_message(data: ::Borrowed<'_>, ep: u8, msg: u64) { + let dev = &data.dev; + //dev_info!(dev.as_ref(), "RtKit message: {:#x}:{:#x}\n", ep, msg); + + if ep != EP_FIRMWARE || msg != MSG_RX_DOORBELL { + dev_err!(dev.as_ref(), "Unknown message: {:#x}:{:#x}\n", ep, msg); + return; + } + + let mut ch = data.rx_channels.lock(); + + ch.fw_log.poll(); + ch.ktrace.poll(); + ch.stats.poll(); + ch.event.poll(); + } + + fn crashed(data: ::Borrowed<'_>, crashlog: Option<&[u8]>) { + let dev = &data.dev; + + data.crashed.store(true, Ordering::Relaxed); + + #[cfg(CONFIG_DEV_COREDUMP)] + if let Err(e) = data.generate_crashdump(crashlog) { + dev_err!(dev.as_ref(), "Could not generate crashdump: {:?}\n", e); + } + #[cfg(not(CONFIG_DEV_COREDUMP))] + let _ = crashlog; + + if debug_enabled(DebugFlags::OopsOnGpuCrash) { + panic!("GPU firmware crashed"); + } else { + dev_err!(dev.as_ref(), "GPU firmware crashed, failing all jobs\n"); + data.event_manager.fail_all(workqueue::WorkError::NoDevice); + } + } + + fn shmem_alloc( + data: ::Borrowed<'_>, + size: usize, + ) -> Result { + let dev = &data.dev; + mod_dev_dbg!(dev, "shmem_alloc() {:#x} bytes\n", size); + + let mut obj = gem::new_kernel_object(dev, size)?; + let vmap = obj.gem.owned_vmap()?; + let mapping = obj.map_into_range( + data.uat.kernel_vm(), + IOVA_KERN_RTKIT_RANGE, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + true, + )?; + mod_dev_dbg!(dev, "shmem_alloc() -> VA {:#x}\n", mapping.iova()); + Ok(RtkitObject { vmap, mapping }) + } +} + +#[versions(AGX)] +impl GpuManager::ver { + /// Create a new GpuManager of this version/GPU combination. + #[inline(never)] + pub(crate) fn new( + dev: &AsahiDevice, + res: ®s::Resources, + cfg: &'static hw::HwConfig, + ) -> Result> { + let uat = Self::make_uat(dev, cfg)?; + let dyncfg = Self::make_dyncfg(dev, res, cfg, &uat)?; + + let mut alloc = KernelAllocators { + private: alloc::DefaultAllocator::new( + dev, + uat.kernel_vm(), + IOVA_KERN_PRIV_RANGE, + 0x80, + mmu::PROT_FW_PRIV_RW, + 1024 * 1024, + true, + fmt!("Kernel Private"), + true, + )?, + shared: alloc::DefaultAllocator::new( + dev, + uat.kernel_vm(), + IOVA_KERN_SHARED_RANGE, + 0x80, + mmu::PROT_FW_SHARED_RW, + 1024 * 1024, + true, + fmt!("Kernel Shared"), + false, + )?, + shared_ro: alloc::DefaultAllocator::new( + dev, + uat.kernel_vm(), + IOVA_KERN_SHARED_RO_RANGE, + 0x80, + mmu::PROT_FW_SHARED_RO, + 64 * 1024, + true, + fmt!("Kernel RO Shared"), + false, + )?, + gpu: alloc::DefaultAllocator::new( + dev, + uat.kernel_vm(), + IOVA_KERN_GPU_RANGE, + 0x80, + mmu::PROT_GPU_FW_SHARED_RW, + 64 * 1024, + true, + fmt!("Kernel GPU Shared"), + false, + )?, + gpu_ro: alloc::DefaultAllocator::new( + dev, + uat.kernel_vm(), + IOVA_KERN_GPU_RO_RANGE, + 0x80, + mmu::PROT_GPU_RO_FW_PRIV_RW, + 1024 * 1024, + true, + fmt!("Kernel GPU RO Shared"), + true, + )?, + }; + + let event_manager = Self::make_event_manager(&mut alloc)?; + let mut initdata = Self::make_initdata(dev, cfg, &dyncfg, &mut alloc)?; + + initdata.runtime_pointers.buffer_mgr_ctl_low_mapping = + Some(initdata.runtime_pointers.buffer_mgr_ctl.map_at( + uat.kernel_lower_vm(), + IOVA_KERN_GPU_BUFMGR_LOW, + mmu::PROT_GPU_SHARED_RW, + false, + )?); + initdata.runtime_pointers.buffer_mgr_ctl_high_mapping = + Some(initdata.runtime_pointers.buffer_mgr_ctl.map_at( + uat.kernel_vm(), + IOVA_KERN_GPU_BUFMGR_HIGH, + mmu::PROT_FW_SHARED_RW, + false, + )?); + + let mut mgr = Self::make_mgr(dev, cfg, dyncfg, uat, alloc, event_manager, initdata)?; + + { + let fwctl = mgr.fwctl_channel.lock(); + let p_fwctl = fwctl.to_raw(); + core::mem::drop(fwctl); + + mgr.as_mut() + .initdata_mut() + .fw_status + .with_mut(|raw, _inner| { + raw.fwctl_channel = p_fwctl; + }); + } + + { + let txc = mgr.tx_channels.lock(); + let p_device_control = txc.device_control.to_raw(); + core::mem::drop(txc); + + let rxc = mgr.rx_channels.lock(); + let p_event = rxc.event.to_raw(); + let p_fw_log = rxc.fw_log.to_raw(); + let p_ktrace = rxc.ktrace.to_raw(); + let p_stats = rxc.stats.to_raw(); + let p_fwlog_buf = rxc.fw_log.get_buf(); + core::mem::drop(rxc); + + mgr.as_mut() + .initdata_mut() + .runtime_pointers + .with_mut(|raw, _inner| { + raw.device_control = p_device_control; + raw.event = p_event; + raw.fw_log = p_fw_log; + raw.ktrace = p_ktrace; + raw.stats = p_stats; + raw.fwlog_buf = Some(p_fwlog_buf); + }); + } + + let mut p_pipes: KVec = KVec::new(); + + for ((v, f), c) in mgr + .pipes + .vtx + .iter() + .zip(&mgr.pipes.frag) + .zip(&mgr.pipes.comp) + { + p_pipes.push( + fw::initdata::raw::PipeChannels::ver { + vtx: v.lock().to_raw(), + frag: f.lock().to_raw(), + comp: c.lock().to_raw(), + }, + GFP_KERNEL, + )?; + } + + mgr.as_mut() + .initdata_mut() + .runtime_pointers + .with_mut(|raw, _inner| { + for (i, p) in p_pipes.into_iter().enumerate() { + raw.pipes[i].vtx = p.vtx; + raw.pipes[i].frag = p.frag; + raw.pipes[i].comp = p.comp; + } + }); + + for (i, map) in cfg.io_mappings.iter().enumerate() { + if let Some(map) = map.as_ref() { + Self::iomap(&mut mgr, cfg, i, map)?; + } + } + + #[ver(V >= V13_0B4)] + if let Some(base) = cfg.sram_base { + let size = cfg.sram_size.unwrap(); + let iova = mgr.as_mut().alloc_mmio_iova(size); + + let mapping = mgr + .uat + .kernel_vm() + .map_io(iova, base, size, mmu::PROT_FW_SHARED_RW)?; + + mgr.as_mut() + .initdata_mut() + .runtime_pointers + .hwdata_b + .with_mut(|raw, _| { + raw.sgx_sram_ptr = U64(mapping.iova()); + }); + + mgr.as_mut().io_mappings_mut().push(mapping, GFP_KERNEL)?; + } + + let mgr = Arc::from(mgr); + + let rtkit = rtkit::RtKit::::new(dev.as_ref(), None, 0, mgr.clone())?; + + *mgr.rtkit.lock() = Some(rtkit); + + { + let mut rxc = mgr.rx_channels.lock(); + rxc.event.set_manager(mgr.clone()); + } + + Ok(mgr) + } + + /// Return a mutable reference to the initdata member + fn initdata_mut( + self: Pin<&mut Self>, + ) -> &mut fw::types::GpuObject { + // SAFETY: initdata does not require structural pinning. + unsafe { &mut self.get_unchecked_mut().initdata } + } + + /// Return a mutable reference to the io_mappings member + fn io_mappings_mut(self: Pin<&mut Self>) -> &mut KVec { + // SAFETY: io_mappings does not require structural pinning. + unsafe { &mut self.get_unchecked_mut().io_mappings } + } + + /// Allocate an MMIO iova range + fn alloc_mmio_iova(self: Pin<&mut Self>, size: usize) -> u64 { + // SAFETY: next_mmio_iova does not require structural pinning. + let next_ref = unsafe { &mut self.get_unchecked_mut().next_mmio_iova }; + + let addr = *next_ref; + let next = addr + (size + mmu::UAT_PGSZ) as u64; + + assert!(next <= IOVA_KERN_MMIO_RANGE.end); + + *next_ref = next; + + addr + } + + /// Build the entire GPU InitData structure tree and return it as a boxed GpuObject. + fn make_initdata( + dev: &AsahiDevice, + cfg: &'static hw::HwConfig, + dyncfg: &hw::DynConfig, + alloc: &mut KernelAllocators, + ) -> Result>> { + let mut builder = initdata::InitDataBuilder::ver::new(dev, alloc, cfg, dyncfg); + builder.build() + } + + /// Create a fresh boxed Uat instance. + /// + /// Force disable inlining to avoid blowing up the stack. + #[inline(never)] + fn make_uat(dev: &AsahiDevice, cfg: &'static hw::HwConfig) -> Result> { + // G14X has a new thing in the Scene structure that unfortunately requires + // write access from user contexts. Hopefully it's not security-sensitive. + #[ver(G >= G14X)] + let map_kernel_to_user = true; + #[ver(G < G14X)] + let map_kernel_to_user = false; + + Ok(KBox::new( + mmu::Uat::new(dev, cfg, map_kernel_to_user)?, + GFP_KERNEL, + )?) + } + + /// Actually create the final GpuManager instance, as a UniqueArc. + /// + /// Force disable inlining to avoid blowing up the stack. + #[inline(never)] + fn make_mgr( + dev: &AsahiDevice, + cfg: &'static hw::HwConfig, + dyncfg: KBox, + uat: KBox, + mut alloc: KernelAllocators, + event_manager: Arc, + initdata: KBox>, + ) -> Result>> { + let mut pipes = PipeChannels::ver { + vtx: KVec::new(), + frag: KVec::new(), + comp: KVec::new(), + }; + + for _i in 0..=NUM_PIPES - 1 { + pipes.vtx.push( + KBox::pin_init( + new_mutex!(channel::PipeChannel::ver::new(dev, &mut alloc)?, "pipe_vtx",), + GFP_KERNEL, + )?, + GFP_KERNEL, + )?; + pipes.frag.push( + KBox::pin_init( + new_mutex!( + channel::PipeChannel::ver::new(dev, &mut alloc)?, + "pipe_frag", + ), + GFP_KERNEL, + )?, + GFP_KERNEL, + )?; + pipes.comp.push( + KBox::pin_init( + new_mutex!( + channel::PipeChannel::ver::new(dev, &mut alloc)?, + "pipe_comp", + ), + GFP_KERNEL, + )?, + GFP_KERNEL, + )?; + } + + let fwctl_channel = channel::FwCtlChannel::new(dev, &mut alloc)?; + + let buffer_mgr = buffer::BufferManager::ver::new()?; + let event_manager_clone = event_manager.clone(); + let buffer_mgr_clone = buffer_mgr.clone(); + let alloc_ref = &mut alloc; + let rx_channels = KBox::init( + try_init!(RxChannels::ver { + event: channel::EventChannel::ver::new( + dev, + alloc_ref, + event_manager_clone, + buffer_mgr_clone, + )?, + fw_log: channel::FwLogChannel::new(dev, alloc_ref)?, + ktrace: channel::KTraceChannel::new(dev, alloc_ref)?, + stats: channel::StatsChannel::ver::new(dev, alloc_ref)?, + }), + GFP_KERNEL, + )?; + + let alloc_ref = &mut alloc; + let tx_channels = KBox::init( + try_init!(TxChannels::ver { + device_control: channel::DeviceControlChannel::ver::new(dev, alloc_ref)?, + }), + GFP_KERNEL, + )?; + + let x = UniqueArc::pin_init( + try_pin_init!(GpuManager::ver { + dev: dev.into(), + cfg, + dyncfg: KBox::::into_inner(dyncfg), + initdata: KBox::>::into_inner(initdata), + uat: KBox::::into_inner(uat), + io_mappings: KVec::new(), + next_mmio_iova: IOVA_KERN_MMIO_RANGE.start, + rtkit <- new_mutex!(None, "rtkit"), + crashed: AtomicBool::new(false), + event_manager, + alloc <- new_mutex!(alloc, "alloc"), + fwctl_channel <- new_mutex!(fwctl_channel, "fwctl_channel"), + rx_channels <- new_mutex!(KBox::::into_inner(rx_channels), "rx_channels"), + tx_channels <- new_mutex!(KBox::::into_inner(tx_channels), "tx_channels"), + pipes, + buffer_mgr, + ids: Default::default(), + garbage_contexts <- new_mutex!(KVec::new(), "garbage_contexts"), + }), + GFP_KERNEL, + )?; + + Ok(x) + } + + fn load_hwdata_blob(dev: &AsahiDevice, name: &CStr, size_name: &CStr) -> Result> { + let of_node = dev.as_ref().of_node().ok_or(EINVAL)?; + let size: usize = dev + .as_ref() + .fwnode() + .ok_or(ENOENT)? + .property_read::(size_name) + .or(0) + .try_into()?; + let res = of_node.reserved_mem_region_to_resource_byname(name)?; + // SAFETY: No dma here, just loading init data. + let mem = unsafe { Mem::try_new(res, (MemFlag::WB).into())? }; + if size > mem.size() { + return Err(ENOENT); + } + // SAFETY: trusting the bootloader to fill it out correctly + let blob_sl = unsafe { slice::from_raw_parts(mem.ptr(), size) }; + let mut blob = KVVec::new(); + blob.extend_from_slice(blob_sl, GFP_KERNEL)?; + Ok(blob) + } + + /// Fetch and validate the GPU dynamic configuration from the device tree and hardware. + /// + /// Force disable inlining to avoid blowing up the stack. + #[inline(never)] + fn make_dyncfg( + dev: &AsahiDevice, + res: ®s::Resources, + cfg: &'static hw::HwConfig, + uat: &mmu::Uat, + ) -> Result> { + let gpu_id = res.get_gpu_id()?; + + dev_info!(dev.as_ref(), "GPU Information:\n"); + dev_info!( + dev.as_ref(), + " Type: {:?}{:?}\n", + gpu_id.gpu_gen, + gpu_id.gpu_variant + ); + dev_info!(dev.as_ref(), " Clusters: {}\n", gpu_id.num_clusters); + dev_info!( + dev.as_ref(), + " Cores: {} ({})\n", + gpu_id.num_cores, + gpu_id.num_cores * gpu_id.num_clusters + ); + dev_info!( + dev.as_ref(), + " Frags: {} ({})\n", + gpu_id.num_frags, + gpu_id.num_frags * gpu_id.num_clusters + ); + dev_info!( + dev.as_ref(), + " GPs: {} ({})\n", + gpu_id.num_gps, + gpu_id.num_gps * gpu_id.num_clusters + ); + dev_info!(dev.as_ref(), " Core masks: {:#x?}\n", gpu_id.core_masks); + dev_info!( + dev.as_ref(), + " Active cores: {}\n", + gpu_id.total_active_cores + ); + + dev_info!(dev.as_ref(), "Getting configuration from device tree...\n"); + let pwr_cfg = hw::PwrConfig::load(dev, cfg)?; + dev_info!(dev.as_ref(), "Dynamic configuration fetched\n"); + + if gpu_id.gpu_gen != cfg.gpu_gen || gpu_id.gpu_variant != cfg.gpu_variant { + dev_err!( + dev.as_ref(), + "GPU type mismatch (expected {:?}{:?}, found {:?}{:?})\n", + cfg.gpu_gen, + cfg.gpu_variant, + gpu_id.gpu_gen, + gpu_id.gpu_variant + ); + return Err(EIO); + } + if gpu_id.num_clusters > cfg.max_num_clusters { + dev_err!( + dev.as_ref(), + "Too many clusters ({} > {})\n", + gpu_id.num_clusters, + cfg.max_num_clusters + ); + return Err(EIO); + } + if gpu_id.num_cores > cfg.max_num_cores { + dev_err!( + dev.as_ref(), + "Too many cores ({} > {})\n", + gpu_id.num_cores, + cfg.max_num_cores + ); + return Err(EIO); + } + if gpu_id.num_frags > cfg.max_num_frags { + dev_err!( + dev.as_ref(), + "Too many frags ({} > {})\n", + gpu_id.num_frags, + cfg.max_num_frags + ); + return Err(EIO); + } + if gpu_id.num_gps > cfg.max_num_gps { + dev_err!( + dev.as_ref(), + "Too many GPs ({} > {})\n", + gpu_id.num_gps, + cfg.max_num_gps + ); + return Err(EIO); + } + + let fwnode = dev.as_ref().fwnode().ok_or(ENOENT)?; + + Ok(KBox::new( + hw::DynConfig { + pwr: pwr_cfg, + uat_ttb_base: uat.ttb_base(), + id: gpu_id, + firmware_version: fwnode + .property_read_array_vec(c_str!("apple,firmware-version"), 3)? + .or(kernel::kvec![0; 3]?), + + hw_data_a: Self::load_hwdata_blob( + dev, + c_str!("hw-cal-a"), + c_str!("debug,hw-cal-a-size"), + ) + .unwrap_or(KVVec::new()), + hw_data_b: Self::load_hwdata_blob( + dev, + c_str!("hw-cal-b"), + c_str!("debug,hw-cal-b-size"), + ) + .unwrap_or(KVVec::new()), + hw_globals: Self::load_hwdata_blob( + dev, + c_str!("globals"), + c_str!("debug,globals-size"), + ) + .unwrap_or(KVVec::new()), + }, + GFP_KERNEL, + )?) + } + + /// Create the global GPU event manager, and return an `Arc<>` to it. + fn make_event_manager(alloc: &mut KernelAllocators) -> Result> { + Ok(Arc::new(event::EventManager::new(alloc)?, GFP_KERNEL)?) + } + + /// Create a new MMIO mapping and add it to the mappings list in initdata at the specified + /// index. + fn iomap( + this: &mut Pin>, + cfg: &'static hw::HwConfig, + index: usize, + map: &hw::IOMapping, + ) -> Result { + let dies = if map.per_die { + cfg.num_dies as usize + } else { + 1 + }; + + let off = map.base & mmu::UAT_PGMSK; + let base = map.base - off; + let end = (map.base + map.size + mmu::UAT_PGMSK) & !mmu::UAT_PGMSK; + let map_size = end - base; + + // Array mappings must be aligned + assert!((off == 0 && map_size == map.size) || (map.count == 1 && !map.per_die)); + assert!(map.count > 0); + + let iova = this.as_mut().alloc_mmio_iova(map_size * map.count * dies); + let mut cur_iova = iova; + + for die in 0..dies { + for i in 0..map.count { + let phys_off = die * 0x20_0000_0000 + i * map.stride; + + let mapping = this.uat.kernel_vm().map_io( + cur_iova, + base + phys_off, + map_size, + if map.writable { + mmu::PROT_FW_MMIO_RW + } else { + mmu::PROT_FW_MMIO_RO + }, + )?; + + this.as_mut().io_mappings_mut().push(mapping, GFP_KERNEL)?; + cur_iova += map_size as u64; + } + } + + this.as_mut() + .initdata_mut() + .runtime_pointers + .hwdata_b + .with_mut(|raw, _| { + raw.io_mappings[index] = fw::initdata::raw::IOMapping { + phys_addr: U64(map.base as u64), + virt_addr: U64(iova + off as u64), + total_size: (map.size * map.count * dies) as u32, + element_size: map.size as u32, + readwrite: U64(map.writable as u64), + }; + }); + + Ok(()) + } + + /// Mark work associated with currently in-progress event slots as failed, after a fault or + /// timeout. + fn mark_pending_events(&self, culprit_slot: Option, error: workqueue::WorkError) { + dev_err!(self.dev.as_ref(), " Pending events:\n"); + + self.initdata.globals.with(|raw, _inner| { + for (index, i) in raw.pending_stamps.iter().enumerate() { + let info = i.info.load(Ordering::Relaxed); + let wait_value = i.wait_value.load(Ordering::Relaxed); + + if info & 1 != 0 { + #[ver(V >= V13_5)] + let slot = (info >> 4) & 0x7f; + #[ver(V < V13_5)] + let slot = (info >> 3) & 0x7f; + #[ver(V >= V13_5)] + let flags = info & 0xf; + #[ver(V < V13_5)] + let flags = info & 0x7; + dev_err!( + self.dev.as_ref(), + " [{}:{}] flags={} value={:#x}\n", + index, + slot, + flags, + wait_value + ); + let error = if culprit_slot.is_some() && culprit_slot != Some(slot) { + workqueue::WorkError::Killed + } else { + error + }; + self.event_manager.mark_error(slot, wait_value, error); + i.info.store(0, Ordering::Relaxed); + i.wait_value.store(0, Ordering::Relaxed); + } + } + }); + } + + /// Fetch the GPU MMU fault information from the hardware registers. + fn get_fault_info(&self) -> Option { + let res = &(*self.dev).resources; + + let info = res.get_fault_info(self.cfg); + if info.is_some() { + dev_err!( + self.dev.as_ref(), + " Fault info: {:#x?}\n", + info.as_ref().unwrap() + ); + } + info + } + + /// Resume the GPU firmware after it halts (due to a timeout, fault, or request). + fn recover(&self) { + self.initdata.fw_status.with(|raw, _inner| { + let halt_count = raw.flags.halt_count.load(Ordering::Relaxed); + let mut halted = raw.flags.halted.load(Ordering::Relaxed); + dev_err!(self.dev.as_ref(), " Halt count: {}\n", halt_count); + dev_err!(self.dev.as_ref(), " Halted: {}\n", halted); + + if halted == 0 { + let start = Instant::::now(); + while start.elapsed() < HALT_ENTER_TIMEOUT { + halted = raw.flags.halted.load(Ordering::Relaxed); + if halted != 0 { + break; + } + mem::sync(); + } + halted = raw.flags.halted.load(Ordering::Relaxed); + } + + if debug_enabled(DebugFlags::NoGpuRecovery) { + dev_crit!( + self.dev.as_ref(), + " GPU recovery is disabled, wedging forever!\n" + ); + } else if halted != 0 { + dev_err!(self.dev.as_ref(), " Attempting recovery...\n"); + raw.flags.halted.store(0, Ordering::SeqCst); + raw.flags.resume.store(1, Ordering::SeqCst); + } else { + dev_err!(self.dev.as_ref(), " Cannot recover.\n"); + } + }); + } + + /// Return the packed GPU enabled core masks. + // Only used for some versions + #[allow(dead_code)] + pub(crate) fn core_masks_packed(&self) -> &[u32] { + self.dyncfg.id.core_masks_packed.as_slice() + } + + /// Kick a submission pipe for a submitted job to tell the firmware to start processing it. + pub(crate) fn run_job(&self, job: workqueue::JobSubmission::ver<'_>) -> Result { + mod_dev_dbg!(self.dev, "GPU: run_job\n"); + + let pipe_type = job.pipe_type(); + mod_dev_dbg!(self.dev, "GPU: run_job: pipe_type={:?}\n", pipe_type); + + let pipes = match pipe_type { + PipeType::Vertex => &self.pipes.vtx, + PipeType::Fragment => &self.pipes.frag, + PipeType::Compute => &self.pipes.comp, + }; + + let index: usize = job.priority() as usize; + let mut pipe = pipes.get(index).ok_or(EIO)?.lock(); + + mod_dev_dbg!(self.dev, "GPU: run_job: run()\n"); + job.run(&mut pipe); + mod_dev_dbg!(self.dev, "GPU: run_job: ring doorbell\n"); + + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().unwrap(); + rtk.send_message( + EP_DOORBELL, + MSG_TX_DOORBELL | pipe_type as u64 | ((index as u64) << 2), + )?; + mod_dev_dbg!(self.dev, "GPU: run_job: done\n"); + + Ok(()) + } + + pub(crate) fn start_op(self: &Arc) -> Result { + if self.is_crashed() { + return Err(ENODEV); + } + + let val = self + .initdata + .globals + .with(|raw, _inner| raw.pending_submissions.fetch_add(1, Ordering::Acquire)); + + mod_dev_dbg!(self.dev, "OP start (pending: {})\n", val + 1); + self.kick_firmware()?; + Ok(OpGuard(self.clone())) + } + + fn invalidate_context( + &self, + context: &fw::types::GpuObject, + ) -> Result { + mod_dev_dbg!( + self.dev, + "Invalidating GPU context @ {:?}\n", + context.weak_pointer() + ); + + if self.is_crashed() { + return Err(ENODEV); + } + + let mut guard = self.alloc.lock(); + let (garbage_count, _) = guard.private.garbage(); + let (garbage_count_gpuro, _) = guard.gpu_ro.garbage(); + + let dc = context.with( + |raw, _inner| fw::channels::DeviceControlMsg::ver::DestroyContext { + unk_4: 0, + ctx_23: raw.unk_23, + #[ver(V < V13_3)] + __pad0: Default::default(), + unk_c: U32(0), + unk_10: U32(0), + ctx_0: raw.unk_0, + ctx_1: raw.unk_1, + ctx_4: raw.unk_4, + #[ver(V < V13_3)] + __pad1: Default::default(), + #[ver(V < V13_3)] + unk_18: 0, + gpu_context: Some(context.weak_pointer()), + __pad2: Default::default(), + }, + ); + + mod_dev_dbg!(self.dev, "Context invalidation command: {:?}\n", &dc); + + let mut txch = self.tx_channels.lock(); + + let token = txch.device_control.send(&dc); + + { + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().unwrap(); + rtk.send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_DEVCTRL)?; + } + + txch.device_control.wait_for(token)?; + + mod_dev_dbg!( + self.dev, + "GPU context invalidated: {:?}\n", + context.weak_pointer() + ); + + // The invalidation does a cache flush, so it is okay to collect garbage + guard.private.collect_garbage(garbage_count); + guard.gpu_ro.collect_garbage(garbage_count_gpuro); + + Ok(()) + } + + #[cfg(CONFIG_DEV_COREDUMP)] + fn generate_crashdump(&self, crashlog: Option<&[u8]>) -> Result { + // Lock the allocators, to block kernel/FW memory mutations (mostly) + let kalloc = self.alloc(); + let pages = self.uat.dump_kernel_pages()?; + core::mem::drop(kalloc); + + let mut crashdump = crate::crashdump::CrashDumpBuilder::new(pages)?; + let initdata_addr = self.initdata.gpu_va().get(); + crashdump.add_agx_info(self.cfg, &self.dyncfg, initdata_addr)?; + if let Some(crashlog) = crashlog { + crashdump.add_crashlog(crashlog)?; + } + let crashdump = KBox::new(crashdump.finalize()?, GFP_KERNEL)?; + + devcoredump::dev_coredump( + self.dev.as_ref(), + &crate::THIS_MODULE, + crashdump, + GFP_KERNEL, + msecs_to_jiffies(60 * 60 * 1000), + ); + + Ok(()) + } +} + +#[versions(AGX)] +impl GpuManager for GpuManager::ver { + fn as_any(&self) -> &dyn Any { + self + } + + fn arc_as_any(self: Arc) -> Arc { + self as Arc + } + + fn init(&self) -> Result { + self.tx_channels.lock().device_control.send( + &fw::channels::DeviceControlMsg::ver::Initialize(Default::default()), + ); + + let initdata = self.initdata.gpu_va().get(); + let mut guard = self.rtkit.lock(); + let mut rtk = guard.as_mut().as_pin_mut().unwrap(); + + rtk.as_mut().boot()?; + rtk.as_mut().start_endpoint(EP_FIRMWARE)?; + rtk.as_mut().start_endpoint(EP_DOORBELL)?; + rtk.as_mut() + .send_message(EP_FIRMWARE, MSG_INIT | (initdata & INIT_DATA_MASK))?; + rtk.as_mut() + .send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_DEVCTRL)?; + core::mem::drop(guard); + + self.kick_firmware()?; + Ok(()) + } + + fn update_globals(&self) { + let mut timeout: u32 = 2; + if debug_enabled(DebugFlags::WaitForPowerOff) { + timeout = 0; + } else if debug_enabled(DebugFlags::KeepGpuPowered) { + timeout = 5000; + } + + self.initdata.globals.with(|raw, _inner| { + raw.idle_off_delay_ms.store(timeout, Ordering::Relaxed); + }); + } + + fn alloc(&self) -> Guard<'_, KernelAllocators, MutexBackend> { + /* Clean up idle contexts */ + let mut garbage_ctx = KVec::new(); + core::mem::swap(&mut *self.garbage_contexts.lock(), &mut garbage_ctx); + + for ctx in garbage_ctx { + if self.invalidate_context(&ctx).is_err() { + dev_err!( + self.dev.as_ref(), + "GpuContext: Failed to invalidate GPU context!\n" + ); + if debug_enabled(DebugFlags::OopsOnGpuCrash) { + panic!("GPU firmware timed out"); + } + } + } + + let mut guard = self.alloc.lock(); + let (garbage_count, garbage_bytes) = guard.private.garbage(); + let (ro_garbage_count, ro_garbage_bytes) = guard.gpu_ro.garbage(); + + if garbage_bytes > MAX_FW_ALLOC_GARBAGE_BYTES + || ro_garbage_bytes > MAX_FW_ALLOC_GARBAGE_BYTES + || garbage_count > MAX_FW_ALLOC_GARBAGE_OBJECTS + || ro_garbage_count > MAX_FW_ALLOC_GARBAGE_OBJECTS + { + mod_dev_dbg!( + self.dev, + "Collecting kalloc garbage (private: {} objects, {} bytes, gpuro: {} objects, {} bytes)\n", + garbage_count, + garbage_bytes, + ro_garbage_count, + ro_garbage_bytes + ); + if self.flush_fw_cache().is_err() { + dev_err!(self.dev.as_ref(), "Failed to flush FW cache\n"); + } else { + guard.private.collect_garbage(garbage_count); + guard.gpu_ro.collect_garbage(ro_garbage_count); + } + } + + guard + } + + fn new_vm(&self, kernel_range: Range) -> Result { + self.uat.new_vm(self.ids.vm.next(), kernel_range) + } + + fn bind_vm(&self, vm: &mmu::Vm) -> Result { + self.uat.bind(vm) + } + + fn new_queue( + &self, + vm: mmu::Vm, + ualloc: Arc>, + ualloc_priv: Arc>, + priority: u32, + usc_exec_base: u64, + ) -> Result> { + let mut kalloc = self.alloc(); + let id = self.ids.queue.next(); + Ok(KBox::new( + queue::Queue::ver::new( + &self.dev, + vm, + &mut kalloc, + ualloc, + ualloc_priv, + self.event_manager.clone(), + &self.buffer_mgr, + id, + priority, + usc_exec_base, + )?, + GFP_KERNEL, + )?) + } + + fn kick_firmware(&self) -> Result { + if self.is_crashed() { + return Err(ENODEV); + } + + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().unwrap(); + rtk.send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_KICKFW)?; + + Ok(()) + } + + fn flush_fw_cache(&self) -> Result { + mod_dev_dbg!(self.dev, "Flushing coprocessor data cache\n"); + + if self.is_crashed() { + return Err(ENODEV); + } + + // ctx_0 == 0xff or ctx_1 == 0xff cause no effect on context, + // but this command does a full cache flush too, so abuse it + // for that. + + let dc = fw::channels::DeviceControlMsg::ver::DestroyContext { + unk_4: 0, + + ctx_23: 0, + #[ver(V < V13_3)] + __pad0: Default::default(), + unk_c: U32(0), + unk_10: U32(0), + ctx_0: 0xff, + ctx_1: 0xff, + ctx_4: 0, + #[ver(V < V13_3)] + __pad1: Default::default(), + #[ver(V < V13_3)] + unk_18: 0, + gpu_context: None, + __pad2: Default::default(), + }; + + let mut txch = self.tx_channels.lock(); + + let token = txch.device_control.send(&dc); + { + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().unwrap(); + rtk.send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_DEVCTRL)?; + } + + txch.device_control.wait_for(token)?; + Ok(()) + } + + fn ids(&self) -> &SequenceIDs { + &self.ids + } + + fn handle_timeout(&self, counter: u32, event_slot: i32, unk: u32) { + dev_err!(self.dev.as_ref(), " (\\________/) \n"); + dev_err!(self.dev.as_ref(), " | | \n"); + dev_err!(self.dev.as_ref(), "'.| \\ , / |.'\n"); + dev_err!(self.dev.as_ref(), "--| / (( \\ |--\n"); + dev_err!(self.dev.as_ref(), ".'| _-_- |'.\n"); + dev_err!(self.dev.as_ref(), " |________| \n"); + dev_err!(self.dev.as_ref(), "** GPU timeout nya~!!!!! **\n"); + dev_err!(self.dev.as_ref(), " Event slot: {}\n", event_slot); + dev_err!(self.dev.as_ref(), " Timeout count: {}\n", counter); + dev_err!(self.dev.as_ref(), " Unk: {}\n", unk); + + // If we have fault info, consider it a fault. + let error = match self.get_fault_info() { + Some(info) => workqueue::WorkError::Fault(info), + None => workqueue::WorkError::Timeout, + }; + self.mark_pending_events(event_slot.try_into().ok(), error); + self.recover(); + } + + fn handle_fault(&self) { + dev_err!(self.dev.as_ref(), " (\\________/) \n"); + dev_err!(self.dev.as_ref(), " | | \n"); + dev_err!(self.dev.as_ref(), "'.| \\ , / |.'\n"); + dev_err!(self.dev.as_ref(), "--| / (( \\ |--\n"); + dev_err!(self.dev.as_ref(), ".'| _-_- |'.\n"); + dev_err!(self.dev.as_ref(), " |________| \n"); + dev_err!(self.dev.as_ref(), "GPU fault nya~!!!!!\n"); + let error = match self.get_fault_info() { + Some(info) => workqueue::WorkError::Fault(info), + None => workqueue::WorkError::Unknown, + }; + self.mark_pending_events(None, error); + self.recover(); + } + + fn handle_channel_error( + &self, + error_type: ChannelErrorType, + pipe_type: u32, + event_slot: u32, + event_value: u32, + ) { + dev_err!(self.dev.as_ref(), " (\\________/) \n"); + dev_err!(self.dev.as_ref(), " | | \n"); + dev_err!(self.dev.as_ref(), "'.| \\ , / |.'\n"); + dev_err!(self.dev.as_ref(), "--| / (( \\ |--\n"); + dev_err!(self.dev.as_ref(), ".'| _-_- |'.\n"); + dev_err!(self.dev.as_ref(), " |________| \n"); + dev_err!(self.dev.as_ref(), "GPU channel error nya~!!!!!\n"); + dev_err!(self.dev.as_ref(), " Error type: {:?}\n", error_type); + dev_err!(self.dev.as_ref(), " Pipe type: {}\n", pipe_type); + dev_err!(self.dev.as_ref(), " Event slot: {}\n", event_slot); + dev_err!(self.dev.as_ref(), " Event value: {:#x?}\n", event_value); + + self.event_manager.mark_error( + event_slot, + event_value, + workqueue::WorkError::ChannelError(error_type), + ); + + let wq = match self.event_manager.get_owner(event_slot) { + Some(wq) => wq, + None => { + dev_err!( + self.dev.as_ref(), + "Workqueue not found for this event slot!\n" + ); + return; + } + }; + + let wq = match wq.as_any().downcast_ref::() { + Some(wq) => wq, + None => { + dev_crit!(self.dev.as_ref(), "GpuManager mismatched with WorkQueue!\n"); + return; + } + }; + + if debug_enabled(DebugFlags::VerboseFaults) { + wq.dump_info(); + } + + let dc = fw::channels::DeviceControlMsg::ver::RecoverChannel { + pipe_type, + work_queue: wq.info_pointer(), + event_value, + __pad: Default::default(), + }; + + mod_dev_dbg!(self.dev, "Recover Channel command: {:?}\n", &dc); + let mut txch = self.tx_channels.lock(); + + let token = txch.device_control.send(&dc); + { + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().unwrap(); + if rtk + .send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_DEVCTRL) + .is_err() + { + dev_err!( + self.dev.as_ref(), + "Failed to send Recover Channel command\n" + ); + } + } + + if txch.device_control.wait_for(token).is_err() { + dev_err!( + self.dev.as_ref(), + "Timed out waiting for Recover Channel command\n" + ); + } + + if debug_enabled(DebugFlags::VerboseFaults) { + wq.dump_info(); + } + } + + fn ack_grow(&self, buffer_slot: u32, vm_slot: u32, counter: u32) { + let halt_count = self + .initdata + .fw_status + .with(|raw, _inner| raw.flags.halt_count.load(Ordering::Relaxed)); + + let dc = fw::channels::DeviceControlMsg::ver::GrowTVBAck { + unk_4: 1, + buffer_slot, + vm_slot, + counter, + subpipe: 0, // TODO + halt_count: U64(halt_count), + __pad: Default::default(), + }; + + mod_dev_dbg!(self.dev, "TVB Grow Ack command: {:?}\n", &dc); + + let mut txch = self.tx_channels.lock(); + + txch.device_control.send(&dc); + { + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().unwrap(); + if rtk + .send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_DEVCTRL) + .is_err() + { + dev_err!(self.dev.as_ref(), "Failed to send TVB Grow Ack command\n"); + } + } + } + + fn fwctl(&self, msg: fw::channels::FwCtlMsg) -> Result { + if self.is_crashed() { + return Err(ENODEV); + } + + let mut fwctl = self.fwctl_channel.lock(); + let token = fwctl.send(&msg); + { + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().unwrap(); + rtk.send_message(EP_DOORBELL, MSG_FWCTL)?; + } + fwctl.wait_for(token)?; + Ok(()) + } + + fn get_cfg(&self) -> &'static hw::HwConfig { + self.cfg + } + + fn get_dyncfg(&self) -> &hw::DynConfig { + &self.dyncfg + } + + fn free_context(&self, ctx: KBox>) { + let mut garbage = self.garbage_contexts.lock(); + + if garbage.push(ctx, GFP_KERNEL).is_err() { + dev_err!( + self.dev.as_ref(), + "Failed to reserve space for freed context, deadlock possible.\n" + ); + } + } + + fn is_crashed(&self) -> bool { + self.crashed.load(Ordering::Relaxed) + } + + fn map_timestamp_buffer( + &self, + mut bo: gem::ObjectRef, + range: Range, + ) -> Result { + bo.map_range_into_range( + self.uat.kernel_vm(), + range, + IOVA_KERN_TIMESTAMP_RANGE, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + false, + ) + } +} + +#[versions(AGX)] +impl GpuManagerPriv for GpuManager::ver { + fn end_op(&self) { + let val = self + .initdata + .globals + .with(|raw, _inner| raw.pending_submissions.fetch_sub(1, Ordering::Release)); + + mod_dev_dbg!(self.dev, "OP end (pending: {})\n", val - 1); + } +} diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs new file mode 100644 index 00000000000000..611764b5463b59 --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -0,0 +1,657 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Per-SoC hardware configuration structures +//! +//! This module contains the definitions used to store per-GPU and per-SoC configuration data. + +use crate::driver::AsahiDevice; +use crate::fw::types::*; +use kernel::c_str; +use kernel::prelude::*; + +const MAX_POWERZONES: usize = 5; + +pub(crate) mod t600x; +pub(crate) mod t602x; +pub(crate) mod t8103; +pub(crate) mod t8112; + +/// GPU generation enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuGen { + G13 = 13, + G14 = 14, +} + +/// GPU variant enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuVariant { + P = 'P' as u32, + G = 'G' as u32, + S = 'S' as u32, + C = 'C' as u32, + D = 'D' as u32, +} + +/// GPU revision enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuRevision { + A0 = 0x00, + A1 = 0x01, + B0 = 0x10, + B1 = 0x11, + C0 = 0x20, + C1 = 0x21, +} + +/// GPU core type enumeration. Note: Part of the firmware ABI. +#[derive(Debug, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuCore { + // Unknown = 0, + // G5P = 1, + // G5G = 2, + // G9P = 3, + // G9G = 4, + // G10P = 5, + // G11P = 6, + // G11M = 7, + // G11G = 8, + // G12P = 9, + // G13P = 10, + G13G = 11, + G13S = 12, + G13C = 13, + // G14P = 14, + G14G = 15, + G14S = 16, + G14C = 17, + G14D = 18, // Split out, unlike G13D +} + +/// GPU revision ID. Note: Part of the firmware ABI. +#[derive(Debug, PartialEq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuRevisionID { + // Unknown = 0, + A0 = 1, + A1 = 2, + B0 = 3, + B1 = 4, + C0 = 5, + C1 = 6, +} + +/// A single performance state of the GPU. +#[derive(Debug)] +pub(crate) struct PState { + /// Voltage in millivolts, per GPU cluster. + pub(crate) volt_mv: KVec, + /// Frequency in hertz. + pub(crate) freq_hz: u32, + /// Maximum power consumption of the GPU at this pstate, in milliwatts. + pub(crate) pwr_mw: u32, +} + +impl PState { + pub(crate) fn max_volt_mv(&self) -> u32 { + *self.volt_mv.iter().max().expect("No voltages") + } +} + +/// A power zone definition (we have no idea what this is but Apple puts them in the DT). +#[allow(missing_docs)] +#[derive(Debug, Copy, Clone)] +pub(crate) struct PowerZone { + pub(crate) target: u32, + pub(crate) target_offset: u32, + pub(crate) filter_tc: u32, +} + +/// An MMIO mapping used by the firmware. +#[derive(Debug, Copy, Clone)] +pub(crate) struct IOMapping { + /// Base physical address of the mapping. + pub(crate) base: usize, + /// Whether this mapping should be replicated to all dies + pub(crate) per_die: bool, + /// Number of mappings. + pub(crate) count: usize, + /// Size of one mapping. + pub(crate) size: usize, + /// Stride between mappings. + pub(crate) stride: usize, + /// Whether the mapping should be writable. + pub(crate) writable: bool, +} + +impl IOMapping { + /// Convenience constructor for a new IOMapping. + pub(crate) const fn new( + base: usize, + per_die: bool, + count: usize, + size: usize, + stride: usize, + writable: bool, + ) -> IOMapping { + IOMapping { + base, + per_die, + count, + size, + stride, + writable, + } + } +} + +/// Unknown HwConfigA fields that vary from SoC to SoC. +#[allow(missing_docs)] +#[derive(Debug, Copy, Clone)] +pub(crate) struct HwConfigA { + pub(crate) unk_87c: i32, + pub(crate) unk_8cc: u32, + pub(crate) unk_e24: u32, +} + +/// Unknown HwConfigB fields that vary from SoC to SoC. +#[allow(missing_docs)] +#[derive(Debug, Copy, Clone)] +pub(crate) struct HwConfigB { + pub(crate) unk_454: u32, + pub(crate) unk_4e0: u64, + pub(crate) unk_534: u32, + pub(crate) unk_ab8: u32, + pub(crate) unk_abc: u32, + pub(crate) unk_b30: u32, +} + +/// Render command configs that vary from SoC to SoC. +#[derive(Debug, Copy, Clone)] +pub(crate) struct HwRenderConfig { + /// Vertex/tiling-related configuration register (lsb: disable clustering) + pub(crate) tiling_control: u32, +} + +#[derive(Debug)] +pub(crate) struct HwConfigShared2Curves { + pub(crate) t1_coef: u32, + pub(crate) t2: &'static [i16], + pub(crate) t3_coefs: &'static [u32], + pub(crate) t3_scales: &'static [u32], +} + +/// Static hardware clustering configuration for multi-cluster SoCs. +#[derive(Debug)] +pub(crate) struct HwClusteringConfig { + pub(crate) meta1_blocksize: usize, + pub(crate) meta2_size: usize, + pub(crate) meta3_size: usize, + pub(crate) meta4_size: usize, + pub(crate) max_splits: usize, +} + +/// Static hardware configuration for a given SoC model. +#[derive(Debug)] +pub(crate) struct HwConfig { + /// Chip ID in hex format (e.g. 0x8103 for t8103). + pub(crate) chip_id: u32, + /// GPU generation. + pub(crate) gpu_gen: GpuGen, + /// GPU variant type. + pub(crate) gpu_variant: GpuVariant, + /// GPU core type ID (as known by the firmware). + pub(crate) gpu_core: GpuCore, + + /// Base clock used used for timekeeping. + pub(crate) base_clock_hz: u32, + /// Output address space for the UAT on this SoC. + pub(crate) uat_oas: u32, + /// Number of dies on this SoC. + pub(crate) num_dies: u32, + /// Maximum number of clusters on this SoC. + pub(crate) max_num_clusters: u32, + /// Maximum number of cores per cluster for this GPU. + pub(crate) max_num_cores: u32, + /// Maximum number of frags per cluster for this GPU. + pub(crate) max_num_frags: u32, + /// Maximum number of GPs per cluster for this GPU. + pub(crate) max_num_gps: u32, + + /// Required size of the first preemption buffer. + pub(crate) preempt1_size: usize, + /// Required size of the second preemption buffer. + pub(crate) preempt2_size: usize, + /// Required size of the third preemption buffer. + pub(crate) preempt3_size: usize, + + /// Required size of the compute preemption buffer. + pub(crate) compute_preempt1_size: usize, + + pub(crate) clustering: Option, + + /// Rendering-relevant configuration. + pub(crate) render: HwRenderConfig, + + /// Misc HWDataA field values. + pub(crate) da: HwConfigA, + /// Misc HWDataB field values. + pub(crate) db: HwConfigB, + /// HwDataShared1.table. + pub(crate) shared1_tab: &'static [i32], + /// HwDataShared1.unk_a4. + pub(crate) shared1_a4: u32, + /// HwDataShared2.table. + pub(crate) shared2_tab: &'static [i32], + /// HwDataShared2.unk_508. + pub(crate) shared2_unk_508: u32, + /// HwDataShared2.unk_508. + pub(crate) shared2_curves: Option, + + /// HwDataShared3.unk_8. + pub(crate) shared3_unk: u32, + /// HwDataShared3.table. + pub(crate) shared3_tab: &'static [u32], + + /// Globals.idle_off_standby_timer. + pub(crate) idle_off_standby_timer_default: u32, + /// Globals.unk_hws2_4. + pub(crate) unk_hws2_4: Option<[F32; 8]>, + /// Globals.unk_hws2_24. + pub(crate) unk_hws2_24: u32, + /// Globals.unk_54 + pub(crate) global_unk_54: u16, + + /// Constant related to SRAM voltages. + pub(crate) sram_k: F32, + /// Unknown per-cluster coefficients 1. + pub(crate) unk_coef_a: &'static [&'static [F32]], + /// Unknown per-cluster coefficients 2. + pub(crate) unk_coef_b: &'static [&'static [F32]], + /// Unknown table in Global struct. + pub(crate) global_tab: Option<&'static [u8]>, + /// Whether this GPU has CS/AFR performance states + pub(crate) has_csafr: bool, + + /// Temperature sensor list (8 bits per sensor). + pub(crate) fast_sensor_mask: [u64; 2], + /// Temperature sensor list (alternate). + pub(crate) fast_sensor_mask_alt: [u64; 2], + /// Temperature sensor present bitmask. + pub(crate) fast_die0_sensor_present: u32, + /// Required MMIO mappings for this GPU/firmware. + pub(crate) io_mappings: &'static [Option], + /// SRAM base + pub(crate) sram_base: Option, + /// SRAM size + pub(crate) sram_size: Option, +} + +/// Dynamic (fetched from hardware/DT) configuration. +#[derive(Debug)] +pub(crate) struct DynConfig { + /// Base physical address of the UAT TTB (from DT reserved memory region). + pub(crate) uat_ttb_base: u64, + /// GPU ID configuration read from hardware. + pub(crate) id: GpuIdConfig, + /// Power calibration configuration for this specific chip/device. + pub(crate) pwr: PwrConfig, + /// Firmware version. + #[allow(dead_code)] + pub(crate) firmware_version: KVec, + + pub(crate) hw_data_a: KVVec, + pub(crate) hw_data_b: KVVec, + pub(crate) hw_globals: KVVec, +} + +/// Specific GPU ID configuration fetched from SGX MMIO registers. +#[derive(Debug)] +pub(crate) struct GpuIdConfig { + /// GPU generation (should match static config). + pub(crate) gpu_gen: GpuGen, + /// GPU variant type (should match static config). + pub(crate) gpu_variant: GpuVariant, + /// GPU silicon revision. + pub(crate) gpu_rev: GpuRevision, + /// GPU silicon revision ID (firmware enum). + pub(crate) gpu_rev_id: GpuRevisionID, + /// Total number of GPU clusters. + pub(crate) num_clusters: u32, + /// Maximum number of GPU cores per cluster. + pub(crate) num_cores: u32, + /// Number of frags per cluster. + pub(crate) num_frags: u32, + /// Number of GPs per cluster. + pub(crate) num_gps: u32, + /// Total number of active cores for the whole GPU. + pub(crate) total_active_cores: u32, + /// Mask of active cores per cluster. + pub(crate) core_masks: KVec, + /// Packed mask of all active cores. + pub(crate) core_masks_packed: KVec, +} + +/// Configurable CS/AFR GPU power settings from the device tree. +#[derive(Debug)] +pub(crate) struct CsAfrPwrConfig { + /// GPU CS performance state list. + pub(crate) perf_states_cs: KVec, + /// GPU AFR performance state list. + pub(crate) perf_states_afr: KVec, + + /// CS leakage coefficient per die. + pub(crate) leak_coef_cs: KVec, + /// AFR leakage coefficient per die. + pub(crate) leak_coef_afr: KVec, + + /// Minimum voltage for the CS/AFR SRAM power domain in microvolts. + pub(crate) min_sram_microvolt: u32, +} + +/// Configurable GPU power settings from the device tree. +#[derive(Debug)] +pub(crate) struct PwrConfig { + /// GPU performance state list. + pub(crate) perf_states: KVec, + /// GPU power zone list. + pub(crate) power_zones: KVec, + + /// Core leakage coefficient per cluster. + pub(crate) core_leak_coef: KVec, + /// SRAM leakage coefficient per cluster. + pub(crate) sram_leak_coef: KVec, + + pub(crate) csafr: Option, + + /// Maximum total power of the GPU in milliwatts. + pub(crate) max_power_mw: u32, + /// Maximum frequency of the GPU in megahertz. + pub(crate) max_freq_mhz: u32, + + /// Minimum performance state to start at. + pub(crate) perf_base_pstate: u32, + /// Maximum enabled performance state. + pub(crate) perf_max_pstate: u32, + + /// Minimum voltage for the SRAM power domain in microvolts. + pub(crate) min_sram_microvolt: u32, + + // Most of these fields are just named after Apple ADT property names and we don't fully + // understand them. They configure various power-related PID loops and filters. + /// Average power filter time constant in milliseconds. + pub(crate) avg_power_filter_tc_ms: u32, + /// Average power filter PID integral gain? + pub(crate) avg_power_ki_only: F32, + /// Average power filter PID proportional gain? + pub(crate) avg_power_kp: F32, + pub(crate) avg_power_min_duty_cycle: u32, + /// Average power target filter time constant in periods. + pub(crate) avg_power_target_filter_tc: u32, + /// "Fast die0" (temperature?) PID integral gain. + pub(crate) fast_die0_integral_gain: F32, + /// "Fast die0" (temperature?) PID proportional gain. + pub(crate) fast_die0_proportional_gain: F32, + pub(crate) fast_die0_prop_tgt_delta: u32, + pub(crate) fast_die0_release_temp: u32, + /// Delay from the fender (?) becoming idle to powerdown + pub(crate) fender_idle_off_delay_ms: u32, + /// Timeout from firmware early wake to sleep if no work was submitted (?) + pub(crate) fw_early_wake_timeout_ms: u32, + /// Delay from the GPU becoming idle to powerdown + pub(crate) idle_off_delay_ms: u32, + /// Related to the above? + pub(crate) idle_off_standby_timer: u32, + /// Percent? + pub(crate) perf_boost_ce_step: u32, + /// Minimum utilization before performance state is increased in %. + pub(crate) perf_boost_min_util: u32, + pub(crate) perf_filter_drop_threshold: u32, + /// Performance PID filter time constant? (periods?) + pub(crate) perf_filter_time_constant: u32, + /// Performance PID filter time constant 2? (periods?) + pub(crate) perf_filter_time_constant2: u32, + /// Performance PID integral gain. + pub(crate) perf_integral_gain: F32, + /// Performance PID integral gain 2 (?). + pub(crate) perf_integral_gain2: F32, + pub(crate) perf_integral_min_clamp: u32, + /// Performance PID proportional gain. + pub(crate) perf_proportional_gain: F32, + /// Performance PID proportional gain 2 (?). + pub(crate) perf_proportional_gain2: F32, + pub(crate) perf_reset_iters: u32, + /// Target GPU utilization for the performance controller in %. + pub(crate) perf_tgt_utilization: u32, + /// Power sampling period in milliseconds. + pub(crate) power_sample_period: u32, + /// PPM (?) filter time constant in milliseconds. + pub(crate) ppm_filter_time_constant_ms: u32, + /// PPM (?) filter PID integral gain. + pub(crate) ppm_ki: F32, + /// PPM (?) filter PID proportional gain. + pub(crate) ppm_kp: F32, + /// Power consumption filter time constant (periods?) + pub(crate) pwr_filter_time_constant: u32, + /// Power consumption filter PID integral gain. + pub(crate) pwr_integral_gain: F32, + pub(crate) pwr_integral_min_clamp: u32, + pub(crate) pwr_min_duty_cycle: u32, + pub(crate) pwr_proportional_gain: F32, + /// Power sample period in base clocks, used when not an integer number of ms + pub(crate) pwr_sample_period_aic_clks: u32, + + pub(crate) se_engagement_criteria: i32, + pub(crate) se_filter_time_constant: u32, + pub(crate) se_filter_time_constant_1: u32, + pub(crate) se_inactive_threshold: u32, + pub(crate) se_ki: F32, + pub(crate) se_ki_1: F32, + pub(crate) se_kp: F32, + pub(crate) se_kp_1: F32, + pub(crate) se_reset_criteria: u32, +} + +impl PwrConfig { + fn load_opp( + dev: &AsahiDevice, + name: &CStr, + cfg: &HwConfig, + is_main: bool, + ) -> Result> { + let mut perf_states = KVec::new(); + + let node = dev.as_ref().of_node().ok_or(EIO)?; + let opps = node.parse_phandle(name, 0).ok_or(EIO)?; + + for opp in opps.children() { + let freq_hz: u64 = opp.get_property(c_str!("opp-hz"))?; + let mut volt_uv: KVec = opp.get_property(c_str!("opp-microvolt"))?; + let pwr_uw: u32 = if is_main { + opp.get_property(c_str!("opp-microwatt"))? + } else { + 0 + }; + + let voltage_count = if is_main { + cfg.max_num_clusters + } else { + cfg.num_dies + }; + + if volt_uv.len() != voltage_count as usize { + dev_err!( + dev.as_ref(), + "Invalid opp-microvolt length (expected {}, got {})\n", + voltage_count, + volt_uv.len() + ); + return Err(EINVAL); + } + + volt_uv.iter_mut().for_each(|a| *a /= 1000); + let volt_mv = volt_uv; + + let pwr_mw = pwr_uw / 1000; + + perf_states.push( + PState { + freq_hz: freq_hz.try_into()?, + volt_mv, + pwr_mw, + }, + GFP_KERNEL, + )?; + } + + if perf_states.is_empty() { + Err(EINVAL) + } else { + Ok(perf_states) + } + } + + /// Load the GPU power configuration from the device tree. + pub(crate) fn load(dev: &AsahiDevice, cfg: &HwConfig) -> Result { + let perf_states = Self::load_opp(dev, c_str!("operating-points-v2"), cfg, true)?; + let node = dev.as_ref().of_node().ok_or(EIO)?; + + macro_rules! prop { + ($prop:expr, $default:expr) => {{ + node.get_opt_property(c_str!($prop)) + .map_err(|e| { + dev_err!(dev.as_ref(), "Error reading property {}: {:?}\n", $prop, e); + e + })? + .unwrap_or($default) + }}; + ($prop:expr) => {{ + node.get_property(c_str!($prop)).map_err(|e| { + dev_err!(dev.as_ref(), "Error reading property {}: {:?}\n", $prop, e); + e + })? + }}; + } + + let pz_data = prop!("apple,power-zones", KVec::new()); + + if pz_data.len() > 3 * MAX_POWERZONES || pz_data.len() % 3 != 0 { + dev_err!(dev.as_ref(), "Invalid apple,power-zones value\n"); + return Err(EINVAL); + } + + let pz_count = pz_data.len() / 3; + let mut power_zones = KVec::new(); + for i in (0..pz_count).step_by(3) { + power_zones.push( + PowerZone { + target: pz_data[i], + target_offset: pz_data[i + 1], + filter_tc: pz_data[i + 2], + }, + GFP_KERNEL, + )?; + } + + let core_leak_coef: KVec = prop!("apple,core-leak-coef"); + let sram_leak_coef: KVec = prop!("apple,sram-leak-coef"); + + if core_leak_coef.len() != cfg.max_num_clusters as usize { + dev_err!(dev.as_ref(), "Invalid apple,core-leak-coef\n"); + return Err(EINVAL); + } + if sram_leak_coef.len() != cfg.max_num_clusters as usize { + dev_err!(dev.as_ref(), "Invalid apple,sram_leak_coef\n"); + return Err(EINVAL); + } + + let csafr = if cfg.has_csafr { + Some(CsAfrPwrConfig { + perf_states_cs: Self::load_opp(dev, c_str!("apple,cs-opp"), cfg, false)?, + perf_states_afr: Self::load_opp(dev, c_str!("apple,afr-opp"), cfg, false)?, + leak_coef_cs: prop!("apple,cs-leak-coef"), + leak_coef_afr: prop!("apple,afr-leak-coef"), + min_sram_microvolt: prop!("apple,csafr-min-sram-microvolt"), + }) + } else { + None + }; + + let power_sample_period: u32 = prop!("apple,power-sample-period"); + + Ok(PwrConfig { + core_leak_coef, + sram_leak_coef, + + max_power_mw: perf_states.iter().map(|a| a.pwr_mw).max().unwrap(), + max_freq_mhz: perf_states.iter().map(|a| a.freq_hz).max().unwrap() / 1_000_000, + + perf_base_pstate: prop!("apple,perf-base-pstate", 1), + perf_max_pstate: perf_states.len() as u32 - 1, + min_sram_microvolt: prop!("apple,min-sram-microvolt"), + + avg_power_filter_tc_ms: prop!("apple,avg-power-filter-tc-ms"), + avg_power_ki_only: prop!("apple,avg-power-ki-only"), + avg_power_kp: prop!("apple,avg-power-kp"), + avg_power_min_duty_cycle: prop!("apple,avg-power-min-duty-cycle"), + avg_power_target_filter_tc: prop!("apple,avg-power-target-filter-tc"), + fast_die0_integral_gain: prop!("apple,fast-die0-integral-gain"), + fast_die0_proportional_gain: prop!("apple,fast-die0-proportional-gain"), + fast_die0_prop_tgt_delta: prop!("apple,fast-die0-prop-tgt-delta", 0), + fast_die0_release_temp: prop!("apple,fast-die0-release-temp", 80), + fender_idle_off_delay_ms: prop!("apple,fender-idle-off-delay-ms", 40), + fw_early_wake_timeout_ms: prop!("apple,fw-early-wake-timeout-ms", 5), + idle_off_delay_ms: prop!("apple,idle-off-delay-ms", 2), + idle_off_standby_timer: prop!( + "apple,idleoff-standby-timer", + cfg.idle_off_standby_timer_default + ), + perf_boost_ce_step: prop!("apple,perf-boost-ce-step", 25), + perf_boost_min_util: prop!("apple,perf-boost-min-util", 100), + perf_filter_drop_threshold: prop!("apple,perf-filter-drop-threshold"), + perf_filter_time_constant2: prop!("apple,perf-filter-time-constant2"), + perf_filter_time_constant: prop!("apple,perf-filter-time-constant"), + perf_integral_gain2: prop!("apple,perf-integral-gain2"), + perf_integral_gain: prop!("apple,perf-integral-gain", f32!(7.8956833)), + perf_integral_min_clamp: prop!("apple,perf-integral-min-clamp"), + perf_proportional_gain2: prop!("apple,perf-proportional-gain2"), + perf_proportional_gain: prop!("apple,perf-proportional-gain", f32!(14.707963)), + perf_reset_iters: prop!("apple,perf-reset-iters", 6), + perf_tgt_utilization: prop!("apple,perf-tgt-utilization"), + power_sample_period, + ppm_filter_time_constant_ms: prop!("apple,ppm-filter-time-constant-ms"), + ppm_ki: prop!("apple,ppm-ki"), + ppm_kp: prop!("apple,ppm-kp"), + pwr_filter_time_constant: prop!("apple,pwr-filter-time-constant", 313), + pwr_integral_gain: prop!("apple,pwr-integral-gain", f32!(0.0202129)), + pwr_integral_min_clamp: prop!("apple,pwr-integral-min-clamp", 0), + pwr_min_duty_cycle: prop!("apple,pwr-min-duty-cycle"), + pwr_proportional_gain: prop!("apple,pwr-proportional-gain", f32!(5.2831855)), + pwr_sample_period_aic_clks: prop!( + "apple,pwr-sample-period-aic-clks", + cfg.base_clock_hz / 1000 * power_sample_period + ), + se_engagement_criteria: prop!("apple,se-engagement-criteria", -1), + se_filter_time_constant: prop!("apple,se-filter-time-constant", 9), + se_filter_time_constant_1: prop!("apple,se-filter-time-constant-1", 3), + se_inactive_threshold: prop!("apple,se-inactive-threshold", 2500), + se_ki: prop!("apple,se-ki", f32!(-50.0)), + se_ki_1: prop!("apple,se-ki-1", f32!(-100.0)), + se_kp: prop!("apple,se-kp", f32!(-5.0)), + se_kp_1: prop!("apple,se-kp-1", f32!(-10.0)), + se_reset_criteria: prop!("apple,se-reset-criteria", 50), + + perf_states, + power_zones, + csafr, + }) + } + + pub(crate) fn max_frequency_khz(&self) -> u32 { + self.perf_states[self.perf_max_pstate as usize].freq_hz / 1000 + } +} diff --git a/drivers/gpu/drm/asahi/hw/t600x.rs b/drivers/gpu/drm/asahi/hw/t600x.rs new file mode 100644 index 00000000000000..58665f985ec38e --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t600x.rs @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for t600x (M1 Pro/Max/Ultra) platforms. + +use crate::f32; + +use super::*; + +const fn iomaps(mcc_count: usize, has_die1: bool) -> [Option; 20] { + [ + Some(IOMapping::new(0x404d00000, false, 1, 0x1c000, 0, true)), // Fender + Some(IOMapping::new(0x20e100000, false, 1, 0x4000, 0, false)), // AICTimer + Some(IOMapping::new(0x28e104000, false, 1, 0x4000, 0, true)), // AICSWInt + Some(IOMapping::new(0x404000000, false, 1, 0x20000, 0, true)), // RGX + None, // UVD + None, // unused + None, // DisplayUnderrunWA + Some(IOMapping::new(0x28e494000, true, 1, 0x4000, 0, false)), // AnalogTempSensorControllerRegs + None, // PMPDoorbell + Some(IOMapping::new(0x404d80000, false, 1, 0x8000, 0, true)), // MetrologySensorRegs + Some(IOMapping::new(0x204d61000, false, 1, 0x1000, 0, true)), // GMGIFAFRegs + Some(IOMapping::new( + 0x200000000, + true, + mcc_count, + 0xd8000, + 0x1000000, + true, + )), // MCache registers + None, // AICBankedRegisters + None, // PMGRScratch + Some(IOMapping::new(0x2643c4000, false, 1, 0x1000, 0, true)), // NIA Special agent idle register die 0 + if has_die1 { + // NIA Special agent idle register die 1 + Some(IOMapping::new(0x22643c4000, false, 1, 0x1000, 0, true)) + } else { + None + }, + None, // CRE registers + None, // Streaming codec registers + Some(IOMapping::new(0x28e3d0000, false, 1, 0x1000, 0, true)), // ? + Some(IOMapping::new(0x28e3c0000, false, 1, 0x2000, 0, false)), // ? + ] +} + +pub(crate) const HWCONFIG_T6002: super::HwConfig = HwConfig { + chip_id: 0x6002, + gpu_gen: GpuGen::G13, + gpu_variant: GpuVariant::D, + gpu_core: GpuCore::G13C, + + base_clock_hz: 24_000_000, + uat_oas: 42, + num_dies: 2, + max_num_clusters: 8, + max_num_cores: 8, + max_num_frags: 8, + max_num_gps: 4, + + preempt1_size: 0x540, + preempt2_size: 0x280, + preempt3_size: 0x20, + compute_preempt1_size: 0x3bd00, + clustering: Some(HwClusteringConfig { + meta1_blocksize: 0x44, + meta2_size: 0xc0 * 8, + meta3_size: 0x280 * 8, + meta4_size: 0x30 * 16, + max_splits: 16, + }), + + render: HwRenderConfig { + tiling_control: 0xa540, + }, + + da: HwConfigA { + unk_87c: 900, + unk_8cc: 11000, + unk_e24: 125, + }, + db: HwConfigB { + unk_454: 1, + unk_4e0: 4, + unk_534: 1, + unk_ab8: 0x2084, + unk_abc: 0x80, + unk_b30: 0, + }, + shared1_tab: &[ + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + ], + shared1_a4: 0xffff, + shared2_tab: &[-1, -1, -1, -1, 0x2aa, 0xaaa, -1, -1, 0, 0], + shared2_unk_508: 0xcc00001, + shared2_curves: None, + shared3_unk: 0, + shared3_tab: &[], + idle_off_standby_timer_default: 0, + unk_hws2_4: None, + unk_hws2_24: 0, + global_unk_54: 0xffff, + sram_k: f32!(1.02), + unk_coef_a: &[ + &f32!([9.838]), + &f32!([9.819]), + &f32!([9.826]), + &f32!([9.799]), + &f32!([9.799]), + &f32!([9.826]), + &f32!([9.819]), + &f32!([9.838]), + ], + unk_coef_b: &[ + &f32!([13.0]), + &f32!([13.0]), + &f32!([13.0]), + &f32!([13.0]), + &f32!([13.0]), + &f32!([13.0]), + &f32!([13.0]), + &f32!([13.0]), + ], + global_tab: Some(&[ + 0, 1, 2, 1, 1, 90, 75, 1, 1, 1, 2, 90, 75, 1, 1, 1, 1, 90, 75, 1, 1, + ]), + has_csafr: false, + fast_sensor_mask: [0x8080808080808080, 0], + fast_sensor_mask_alt: [0x9090909090909090, 0], + fast_die0_sensor_present: 0xff, + io_mappings: &iomaps(8, true), + sram_base: None, + sram_size: None, +}; + +pub(crate) const HWCONFIG_T6001: super::HwConfig = HwConfig { + chip_id: 0x6001, + gpu_variant: GpuVariant::C, + gpu_core: GpuCore::G13C, + + num_dies: 1, + max_num_clusters: 4, + fast_sensor_mask: [0x80808080, 0], + fast_sensor_mask_alt: [0x90909090, 0], + fast_die0_sensor_present: 0x0f, + io_mappings: &iomaps(8, false), + ..HWCONFIG_T6002 +}; + +pub(crate) const HWCONFIG_T6000: super::HwConfig = HwConfig { + chip_id: 0x6000, + gpu_variant: GpuVariant::S, + gpu_core: GpuCore::G13S, + + max_num_clusters: 2, + fast_sensor_mask: [0x8080, 0], + fast_sensor_mask_alt: [0x9090, 0], + fast_die0_sensor_present: 0x03, + io_mappings: &iomaps(4, false), + ..HWCONFIG_T6001 +}; diff --git a/drivers/gpu/drm/asahi/hw/t602x.rs b/drivers/gpu/drm/asahi/hw/t602x.rs new file mode 100644 index 00000000000000..98a7ac2b76e571 --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t602x.rs @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for t600x (M1 Pro/Max/Ultra) platforms. + +use crate::f32; + +use super::*; + +const fn iomaps(chip_id: u32, mcc_count: usize) -> [Option; 24] { + [ + Some(IOMapping::new(0x404d00000, false, 1, 0x144000, 0, true)), // Fender + Some(IOMapping::new(0x20e100000, false, 1, 0x4000, 0, false)), // AICTimer + Some(IOMapping::new(0x28e106000, false, 1, 0x4000, 0, true)), // AICSWInt + Some(IOMapping::new(0x404000000, false, 1, 0x20000, 0, true)), // RGX + None, // UVD + None, // unused + None, // DisplayUnderrunWA + Some(match chip_id { + 0x6020 => IOMapping::new(0x28e460000, true, 1, 0x4000, 0, false), + _ => IOMapping::new(0x28e478000, true, 1, 0x4000, 0, false), + }), // AnalogTempSensorControllerRegs + None, // PMPDoorbell + Some(IOMapping::new(0x404e08000, false, 1, 0x8000, 0, true)), // MetrologySensorRegs + None, // GMGIFAFRegs + Some(IOMapping::new( + 0x200000000, + true, + mcc_count, + 0xd8000, + 0x1000000, + true, + )), // MCache registers + Some(IOMapping::new(0x28e118000, false, 1, 0x4000, 0, false)), // AICBankedRegisters + None, // PMGRScratch + None, // NIA Special agent idle register die 0 + None, // NIA Special agent idle register die 1 + None, // CRE registers + None, // Streaming codec registers + Some(IOMapping::new(0x28e3d0000, false, 1, 0x4000, 0, true)), // ? + Some(IOMapping::new(0x28e3c0000, false, 1, 0x4000, 0, false)), // ? + Some(IOMapping::new(0x28e3d8000, false, 1, 0x4000, 0, true)), // ? + Some(IOMapping::new(0x404eac000, true, 1, 0x4000, 0, true)), // ? + None, + None, + ] +} + +// TODO: Tentative +pub(crate) const HWCONFIG_T6022: super::HwConfig = HwConfig { + chip_id: 0x6022, + gpu_gen: GpuGen::G14, + gpu_variant: GpuVariant::D, + gpu_core: GpuCore::G14D, + + base_clock_hz: 24_000_000, + uat_oas: 42, + num_dies: 2, + max_num_clusters: 8, + max_num_cores: 10, + max_num_frags: 10, + max_num_gps: 4, + + preempt1_size: 0x540, + preempt2_size: 0x280, + preempt3_size: 0x40, + compute_preempt1_size: 0x25980 * 2, // Conservative guess + clustering: Some(HwClusteringConfig { + meta1_blocksize: 0x44, + meta2_size: 0xc0 * 16, + meta3_size: 0x280 * 16, + meta4_size: 0x10 * 128, + max_splits: 64, + }), + + render: HwRenderConfig { + tiling_control: 0x180340, + }, + + da: HwConfigA { + unk_87c: 500, + unk_8cc: 11000, + unk_e24: 125, + }, + db: HwConfigB { + unk_454: 1, + unk_4e0: 4, + unk_534: 0, + unk_ab8: 0, // Unused + unk_abc: 0, // Unused + unk_b30: 0, + }, + shared1_tab: &[ + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + ], + shared1_a4: 0, + shared2_tab: &[0x800, 0x1555, -1, -1, -1, -1, -1, -1, 0xaaaaa, 0], + shared2_unk_508: 0xc00007, + shared2_curves: Some(HwConfigShared2Curves { + t1_coef: 11000, + t2: &[ + 0xf07, 0x4c0, 0x680, 0x8c0, 0xa80, 0xc40, 0xd80, 0xec0, 0xf40, + ], + t3_coefs: &[0, 20, 27, 36, 43, 50, 55, 60, 62], + t3_scales: &[9, 3209, 10400], + }), + shared3_unk: 8, + shared3_tab: &[ + 125, 125, 125, 125, 125, 125, 125, 125, 7500, 125, 125, 125, 125, 125, 125, 125, + ], + idle_off_standby_timer_default: 700, + unk_hws2_4: Some(f32!([1.0, 0.8, 0.2, 0.9, 0.1, 0.25, 0.5, 0.9])), + unk_hws2_24: 6, + global_unk_54: 4000, + sram_k: f32!(1.02), + unk_coef_a: &[ + &f32!([0.0, 8.2, 0.0, 6.9, 6.9]), + &f32!([0.0, 0.0, 0.0, 6.9, 6.9]), + &f32!([0.0, 8.2, 0.0, 6.9, 0.0]), + &f32!([0.0, 0.0, 0.0, 6.9, 0.0]), + &f32!([0.0, 0.0, 0.0, 6.9, 0.0]), + &f32!([0.0, 8.2, 0.0, 6.9, 0.0]), + &f32!([0.0, 0.0, 0.0, 6.9, 6.9]), + &f32!([0.0, 8.2, 0.0, 6.9, 6.9]), + ], + unk_coef_b: &[ + &f32!([0.0, 9.0, 0.0, 8.0, 8.0]), + &f32!([0.0, 0.0, 0.0, 8.0, 8.0]), + &f32!([0.0, 9.0, 0.0, 8.0, 0.0]), + &f32!([0.0, 0.0, 0.0, 8.0, 0.0]), + &f32!([0.0, 0.0, 0.0, 8.0, 0.0]), + &f32!([0.0, 9.0, 0.0, 8.0, 0.0]), + &f32!([0.0, 0.0, 0.0, 8.0, 8.0]), + &f32!([0.0, 9.0, 0.0, 8.0, 8.0]), + ], + global_tab: Some(&[ + 0, 2, 2, 1, 1, 90, 75, 1, 1, 1, 2, 90, 75, 1, 1, 1, 2, 90, 75, 1, 1, 1, 1, 90, 75, 1, 1, + ]), + has_csafr: true, + fast_sensor_mask: [0x40005000c000d00, 0xd000c0005000400], + // Apple typo? Should probably be 0x140015001c001d00 + fast_sensor_mask_alt: [0x140015001d001d00, 0x1d001c0015001400], + fast_die0_sensor_present: 0, // Unused + io_mappings: &iomaps(0x6022, 8), + sram_base: Some(0x404d60000), + sram_size: Some(0x20000), +}; + +pub(crate) const HWCONFIG_T6021: super::HwConfig = HwConfig { + chip_id: 0x6021, + gpu_variant: GpuVariant::C, + gpu_core: GpuCore::G14C, + + num_dies: 1, + max_num_clusters: 4, + compute_preempt1_size: 0x25980, + unk_hws2_4: Some(f32!([1.0, 0.8, 0.2, 0.9, 0.1, 0.25, 0.7, 0.9])), + fast_sensor_mask: [0x40005000c000d00, 0], + fast_sensor_mask_alt: [0x140015001d001d00, 0], + io_mappings: &iomaps(0x6021, 8), + ..HWCONFIG_T6022 +}; + +pub(crate) const HWCONFIG_T6020: super::HwConfig = HwConfig { + chip_id: 0x6020, + gpu_variant: GpuVariant::S, + gpu_core: GpuCore::G14S, + + db: HwConfigB { + unk_454: 0, + ..HWCONFIG_T6021.db + }, + + max_num_clusters: 2, + fast_sensor_mask: [0xc000d00, 0], + fast_sensor_mask_alt: [0x1d001d00, 0], + io_mappings: &iomaps(0x6020, 4), + ..HWCONFIG_T6021 +}; diff --git a/drivers/gpu/drm/asahi/hw/t8103.rs b/drivers/gpu/drm/asahi/hw/t8103.rs new file mode 100644 index 00000000000000..484bf6c3414f2f --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t8103.rs @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for t8103 platforms (M1). + +use crate::f32; + +use super::*; + +pub(crate) const HWCONFIG: super::HwConfig = HwConfig { + chip_id: 0x8103, + gpu_gen: GpuGen::G13, + gpu_variant: GpuVariant::G, + gpu_core: GpuCore::G13G, + + base_clock_hz: 24_000_000, + uat_oas: 40, + num_dies: 1, + max_num_clusters: 1, + max_num_cores: 8, + max_num_frags: 8, + max_num_gps: 4, + + preempt1_size: 0x540, + preempt2_size: 0x280, + preempt3_size: 0x20, + compute_preempt1_size: 0x7f80, + clustering: None, + + render: HwRenderConfig { + // bit 0: disable clustering (always) + tiling_control: 0xa041, + }, + + da: HwConfigA { + unk_87c: -220, + unk_8cc: 9880, + unk_e24: 112, + }, + db: HwConfigB { + unk_454: 1, + unk_4e0: 0, + unk_534: 0, + unk_ab8: 0x48, + unk_abc: 0x8, + unk_b30: 0, + }, + shared1_tab: &[ + -1, 0x7282, 0x50ea, 0x370a, 0x25be, 0x1c1f, 0x16fb, -1, -1, -1, -1, -1, -1, -1, -1, -1, + ], + shared1_a4: 0xffff, + shared2_tab: &[0x800, 0x1555, -1, -1, -1, -1, -1, -1, 0, 0], + shared2_unk_508: 0xc00007, + shared2_curves: None, + shared3_unk: 0, + shared3_tab: &[], + idle_off_standby_timer_default: 0, + unk_hws2_4: None, + unk_hws2_24: 0, + global_unk_54: 0xffff, + sram_k: f32!(1.02), + unk_coef_a: &[], + unk_coef_b: &[], + global_tab: None, + has_csafr: false, + fast_sensor_mask: [0x12, 0], + fast_sensor_mask_alt: [0x12, 0], + fast_die0_sensor_present: 0x01, + io_mappings: &[ + Some(IOMapping::new(0x204d00000, false, 1, 0x1c000, 0, true)), // Fender + Some(IOMapping::new(0x20e100000, false, 1, 0x4000, 0, false)), // AICTimer + Some(IOMapping::new(0x23b104000, false, 1, 0x4000, 0, true)), // AICSWInt + Some(IOMapping::new(0x204000000, false, 1, 0x20000, 0, true)), // RGX + None, // UVD + None, // unused + None, // DisplayUnderrunWA + Some(IOMapping::new(0x23b2e8000, false, 1, 0x1000, 0, false)), // AnalogTempSensorControllerRegs + Some(IOMapping::new(0x23bc00000, false, 1, 0x1000, 0, true)), // PMPDoorbell + Some(IOMapping::new(0x204d80000, false, 1, 0x5000, 0, true)), // MetrologySensorRegs + Some(IOMapping::new(0x204d61000, false, 1, 0x1000, 0, true)), // GMGIFAFRegs + Some(IOMapping::new(0x200000000, false, 1, 0xd6400, 0, true)), // MCache registers + None, // AICBankedRegisters + Some(IOMapping::new(0x23b738000, false, 1, 0x1000, 0, true)), // PMGRScratch + None, // NIA Special agent idle register die 0 + None, // NIA Special agent idle register die 1 + None, // CRE registers + None, // Streaming codec registers + None, // + None, // + ], + sram_base: None, + sram_size: None, +}; diff --git a/drivers/gpu/drm/asahi/hw/t8112.rs b/drivers/gpu/drm/asahi/hw/t8112.rs new file mode 100644 index 00000000000000..3eba0457d76ac9 --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t8112.rs @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for t8112 platforms (M2). + +use crate::f32; + +use super::*; + +pub(crate) const HWCONFIG: super::HwConfig = HwConfig { + chip_id: 0x8112, + gpu_gen: GpuGen::G14, + gpu_variant: GpuVariant::G, + gpu_core: GpuCore::G14G, + + base_clock_hz: 24_000_000, + uat_oas: 40, + num_dies: 1, + max_num_clusters: 1, + max_num_cores: 10, + max_num_frags: 10, + max_num_gps: 4, + + preempt1_size: 0x540, + preempt2_size: 0x280, + preempt3_size: 0x20, + compute_preempt1_size: 0x10000, // TODO: Check + clustering: None, + + render: HwRenderConfig { + // TODO: this is unused here, may be present in newer FW + tiling_control: 0xa041, + }, + + da: HwConfigA { + unk_87c: 900, + unk_8cc: 11000, + unk_e24: 125, + }, + db: HwConfigB { + unk_454: 1, + unk_4e0: 4, + unk_534: 0, + unk_ab8: 0x2048, + unk_abc: 0x4000, + unk_b30: 1, + }, + shared1_tab: &[ + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + ], + shared1_a4: 0, + shared2_tab: &[-1, -1, -1, -1, -1, -1, -1, -1, 0xaa5aa, 0], + shared2_unk_508: 0xc00000, + shared2_curves: Some(HwConfigShared2Curves { + t1_coef: 7200, + t2: &[ + 0xf07, 0x4c0, 0x6c0, 0x8c0, 0xac0, 0xc40, 0xdc0, 0xec0, 0xf80, + ], + t3_coefs: &[0, 20, 28, 36, 44, 50, 56, 60, 63], + t3_scales: &[9, 3209, 10400], + }), + shared3_unk: 5, + shared3_tab: &[ + 10700, 10700, 10700, 10700, 10700, 6000, 1000, 1000, 1000, 10700, 10700, 10700, 10700, + 10700, 10700, 10700, + ], + idle_off_standby_timer_default: 0, + unk_hws2_4: None, + unk_hws2_24: 0, + global_unk_54: 0xffff, + + sram_k: f32!(1.02), + // 13.2: last coef changed from 6.6 to 5.3, assuming that was a fix we can backport + unk_coef_a: &[&f32!([0.0, 0.0, 0.0, 0.0, 5.3, 0.0, 5.3, /*6.6*/ 5.3])], + unk_coef_b: &[&f32!([0.0, 0.0, 0.0, 0.0, 5.3, 0.0, 5.3, /*6.6*/ 5.3])], + global_tab: None, + has_csafr: false, + fast_sensor_mask: [0x6800, 0], + fast_sensor_mask_alt: [0x6800, 0], + fast_die0_sensor_present: 0x02, + io_mappings: &[ + Some(IOMapping::new(0x204d00000, false, 1, 0x14000, 0, true)), // Fender + Some(IOMapping::new(0x20e100000, false, 1, 0x4000, 0, false)), // AICTimer + Some(IOMapping::new(0x23b0c4000, false, 1, 0x4000, 0, true)), // AICSWInt + Some(IOMapping::new(0x204000000, false, 1, 0x20000, 0, true)), // RGX + None, // UVD + None, // unused + None, // DisplayUnderrunWA + Some(IOMapping::new(0x23b2c0000, false, 1, 0x1000, 0, false)), // AnalogTempSensorControllerRegs + None, // PMPDoorbell + Some(IOMapping::new(0x204d80000, false, 1, 0x8000, 0, true)), // MetrologySensorRegs + Some(IOMapping::new(0x204d61000, false, 1, 0x1000, 0, true)), // GMGIFAFRegs + Some(IOMapping::new(0x200000000, false, 1, 0xd6400, 0, true)), // MCache registers + None, // AICBankedRegisters + None, // PMGRScratch + None, // NIA Special agent idle register die 0 + None, // NIA Special agent idle register die 1 + Some(IOMapping::new(0x204e00000, false, 1, 0x10000, 0, true)), // CRE registers + Some(IOMapping::new(0x27d050000, false, 1, 0x4000, 0, true)), // Streaming codec registers + Some(IOMapping::new(0x23b3d0000, false, 1, 0x1000, 0, true)), // + Some(IOMapping::new(0x23b3c0000, false, 1, 0x1000, 0, false)), // + ], + sram_base: None, + sram_size: None, +}; diff --git a/drivers/gpu/drm/asahi/initdata.rs b/drivers/gpu/drm/asahi/initdata.rs new file mode 100644 index 00000000000000..b190a7da6fe85f --- /dev/null +++ b/drivers/gpu/drm/asahi/initdata.rs @@ -0,0 +1,1021 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![allow(clippy::unusual_byte_groupings)] + +//! GPU initialization data builder. +//! +//! The root of all interaction between the GPU firmware and the host driver is a complex set of +//! nested structures that we call InitData. This includes both GPU hardware/firmware configuration +//! and the pointers to the ring buffers and global data fields that are used for communication at +//! runtime. +//! +//! Many of these structures are poorly understood, so there are lots of hardcoded unknown values +//! derived from observing the InitData structures that macOS generates. + +use crate::f32; +use crate::fw::initdata::*; +use crate::fw::types::*; +use crate::module_parameters; +use crate::{ + driver::AsahiDevice, + gem, + gpu, + hw, + mmu, // +}; +use kernel::error::{ + Error, + Result, // +}; +use kernel::macros::versions; +use kernel::prelude::*; +use kernel::try_init; + +use ::pin_init; +use ::pin_init::Init; + +/// Builder helper for the global GPU InitData. +#[versions(AGX)] +pub(crate) struct InitDataBuilder<'a> { + dev: &'a AsahiDevice, + alloc: &'a mut gpu::KernelAllocators, + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, +} + +#[versions(AGX)] +impl<'a> InitDataBuilder::ver<'a> { + /// Create a new InitData builder + pub(crate) fn new( + dev: &'a AsahiDevice, + alloc: &'a mut gpu::KernelAllocators, + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + ) -> InitDataBuilder::ver<'a> { + InitDataBuilder::ver { + dev, + alloc, + cfg, + dyncfg, + } + } + + /// Create the HwDataShared1 structure, which is used in two places in InitData. + fn hw_shared1(cfg: &'static hw::HwConfig) -> impl Init { + init!(raw::HwDataShared1 { + unk_a4: cfg.shared1_a4, + ..Zeroable::init_zeroed() + }) + .chain(|ret| { + for (i, val) in cfg.shared1_tab.iter().enumerate() { + ret.table[i] = *val; + } + Ok(()) + }) + } + + fn init_curve( + curve: &mut raw::HwDataShared2Curve, + unk_0: u32, + unk_4: u32, + t1: &[u16], + t2: &[i16], + t3: &[KVec], + ) { + curve.unk_0 = unk_0; + curve.unk_4 = unk_4; + (*curve.t1)[..t1.len()].copy_from_slice(t1); + (*curve.t1)[t1.len()..].fill(t1[0]); + (*curve.t2)[..t2.len()].copy_from_slice(t2); + (*curve.t2)[t2.len()..].fill(t2[0]); + for (i, a) in curve.t3.iter_mut().enumerate() { + a.fill(0x3ffffff); + if i < t3.len() { + let b = &t3[i]; + (**a)[..b.len()].copy_from_slice(b); + } + } + } + + /// Create the HwDataShared2 structure, which is used in two places in InitData. + fn hw_shared2( + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + ) -> impl Init + 'a { + try_init!(raw::HwDataShared2 { + unk_28: Array::new([0xff; 16]), + g14: Default::default(), + unk_508: cfg.shared2_unk_508, + ..Zeroable::init_zeroed() + }) + .chain(|ret| { + for (i, val) in cfg.shared2_tab.iter().enumerate() { + ret.table[i] = *val; + } + + let curve_cfg = match cfg.shared2_curves.as_ref() { + None => return Ok(()), + Some(a) => a, + }; + + let mut t1 = KVec::new(); + let mut t3 = KVec::new(); + + for _ in 0..curve_cfg.t3_scales.len() { + t3.push(KVec::new(), GFP_KERNEL)?; + } + + for (i, ps) in dyncfg.pwr.perf_states.iter().enumerate() { + let t3_coef = curve_cfg.t3_coefs[i]; + if t3_coef == 0 { + t1.push(0xffff, GFP_KERNEL)?; + for j in t3.iter_mut() { + j.push(0x3ffffff, GFP_KERNEL)?; + } + continue; + } + + let f_khz = (ps.freq_hz / 1000) as u64; + let v_max = ps.max_volt_mv() as u64; + + t1.push( + (1000000000 * (curve_cfg.t1_coef as u64) / (f_khz * v_max)) + .try_into() + .unwrap(), + GFP_KERNEL, + )?; + + for (j, scale) in curve_cfg.t3_scales.iter().enumerate() { + t3[j].push( + (t3_coef as u64 * 1000000100 * *scale as u64 / (f_khz * v_max * 6)) + .try_into() + .unwrap(), + GFP_KERNEL, + )?; + } + } + + ret.g14.unk_14 = 0x6000000; + Self::init_curve( + &mut ret.g14.curve1, + 0, + 0x20000000, + &[0xffff], + &[0x0f07], + &[], + ); + Self::init_curve(&mut ret.g14.curve2, 7, 0x80000000, &t1, curve_cfg.t2, &t3); + + Ok(()) + }) + } + + /// Create the HwDataShared3 structure, which is used in two places in InitData. + fn hw_shared3(cfg: &'static hw::HwConfig) -> impl Init { + pin_init::init_zeroed::().chain(|ret| { + if !cfg.shared3_tab.is_empty() { + ret.unk_0 = 1; + ret.unk_4 = 500; + ret.unk_8 = cfg.shared3_unk; + ret.table.copy_from_slice(cfg.shared3_tab); + ret.unk_4c = 1; + } + Ok(()) + }) + } + + /// Create an unknown T81xx-specific data structure. + fn t81xx_data( + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + ) -> impl Init { + let _perf_max_pstate = dyncfg.pwr.perf_max_pstate; + + pin_init::init_zeroed::().chain(move |_ret| { + match cfg.chip_id { + 0x8103 | 0x8112 => { + #[ver(V < V13_3)] + { + _ret.unk_d8c = 0x80000000; + _ret.unk_d90 = 4; + _ret.unk_d9c = f32!(0.6); + _ret.unk_da4 = f32!(0.4); + _ret.unk_dac = f32!(0.38552); + _ret.unk_db8 = f32!(65536.0); + _ret.unk_dbc = f32!(13.56); + _ret.max_pstate_scaled = 100 * _perf_max_pstate; + } + } + _ => (), + } + Ok(()) + }) + } + + /// Create the HwDataA structure. This mostly contains power-related configuration. + fn hwdata_a(&mut self) -> Result> { + let pwr = &self.dyncfg.pwr; + let period_ms = pwr.power_sample_period; + let period_s = F32::from(period_ms) / f32!(1000.0); + let ppm_filter_tc_periods = pwr.ppm_filter_time_constant_ms / period_ms; + #[ver(V >= V13_0B4)] + let ppm_filter_tc_ms_rounded = ppm_filter_tc_periods * period_ms; + let ppm_filter_a = f32!(1.0) / ppm_filter_tc_periods.into(); + let perf_filter_a = f32!(1.0) / pwr.perf_filter_time_constant.into(); + let perf_filter_a2 = f32!(1.0) / pwr.perf_filter_time_constant2.into(); + let avg_power_target_filter_a = f32!(1.0) / pwr.avg_power_target_filter_tc.into(); + let avg_power_filter_tc_periods = pwr.avg_power_filter_tc_ms / period_ms; + #[ver(V >= V13_0B4)] + let avg_power_filter_tc_ms_rounded = avg_power_filter_tc_periods * period_ms; + let avg_power_filter_a = f32!(1.0) / avg_power_filter_tc_periods.into(); + let pwr_filter_a = f32!(1.0) / pwr.pwr_filter_time_constant.into(); + + let base_ps = pwr.perf_base_pstate; + let base_ps_scaled = 100 * base_ps; + let max_ps = pwr.perf_max_pstate; + let max_ps_scaled = 100 * max_ps; + let boost_ps_count = max_ps - base_ps; + + #[allow(unused_variables)] + let base_clock_khz = self.cfg.base_clock_hz / 1000; + let v_clocks_per_period = pwr.pwr_sample_period_aic_clks; + + #[allow(unused_variables)] + let clocks_per_period_coarse = self.cfg.base_clock_hz / 1000 * pwr.power_sample_period; + + self.alloc + .private + .new_init(pin_init::init_zeroed(), |_inner, _ptr| { + let cfg = &self.cfg; + let dyncfg = &self.dyncfg; + try_init!(raw::HwDataA::ver { + clocks_per_period: v_clocks_per_period, + #[ver(V >= V13_0B4)] + clocks_per_period_2: v_clocks_per_period, + pwr_status: AtomicU32::new(4), + unk_10: f32!(1.0), + actual_pstate: 1, + tgt_pstate: 1, + base_pstate_scaled: base_ps_scaled, + unk_40: 1, + max_pstate_scaled: max_ps_scaled, + min_pstate_scaled: 100, + unk_64c: 625, + pwr_filter_a_neg: f32!(1.0) - pwr_filter_a, + pwr_filter_a: pwr_filter_a, + pwr_integral_gain: pwr.pwr_integral_gain, + pwr_integral_min_clamp: pwr.pwr_integral_min_clamp.into(), + max_power_1: pwr.max_power_mw.into(), + pwr_proportional_gain: pwr.pwr_proportional_gain, + pwr_pstate_related_k: -F32::from(max_ps_scaled) / pwr.max_power_mw.into(), + pwr_pstate_max_dc_offset: pwr.pwr_min_duty_cycle as i32 - max_ps_scaled as i32, + max_pstate_scaled_2: max_ps_scaled, + max_power_2: pwr.max_power_mw, + max_pstate_scaled_3: max_ps_scaled, + ppm_filter_tc_periods_x4: ppm_filter_tc_periods * 4, + ppm_filter_a_neg: f32!(1.0) - ppm_filter_a, + ppm_filter_a: ppm_filter_a, + ppm_ki_dt: pwr.ppm_ki * period_s, + unk_6fc: f32!(65536.0), + ppm_kp: pwr.ppm_kp, + pwr_min_duty_cycle: pwr.pwr_min_duty_cycle, + max_pstate_scaled_4: max_ps_scaled, + unk_71c: f32!(0.0), + max_power_3: pwr.max_power_mw, + cur_power_mw_2: 0x0, + ppm_filter_tc_ms: pwr.ppm_filter_time_constant_ms, + #[ver(V >= V13_0B4)] + ppm_filter_tc_clks: ppm_filter_tc_ms_rounded * base_clock_khz, + perf_tgt_utilization: pwr.perf_tgt_utilization, + perf_boost_min_util: pwr.perf_boost_min_util, + perf_boost_ce_step: pwr.perf_boost_ce_step, + perf_reset_iters: pwr.perf_reset_iters, + unk_774: 6, + unk_778: 1, + perf_filter_drop_threshold: pwr.perf_filter_drop_threshold, + perf_filter_a_neg: f32!(1.0) - perf_filter_a, + perf_filter_a2_neg: f32!(1.0) - perf_filter_a2, + perf_filter_a: perf_filter_a, + perf_filter_a2: perf_filter_a2, + perf_ki: pwr.perf_integral_gain, + perf_ki2: pwr.perf_integral_gain2, + perf_integral_min_clamp: pwr.perf_integral_min_clamp.into(), + unk_79c: f32!(95.0), + perf_kp: pwr.perf_proportional_gain, + perf_kp2: pwr.perf_proportional_gain2, + boost_state_unk_k: F32::from(boost_ps_count) / f32!(0.95), + base_pstate_scaled_2: base_ps_scaled, + max_pstate_scaled_5: max_ps_scaled, + base_pstate_scaled_3: base_ps_scaled, + perf_tgt_utilization_2: pwr.perf_tgt_utilization, + base_pstate_scaled_4: base_ps_scaled, + unk_7fc: f32!(65536.0), + pwr_min_duty_cycle_2: pwr.pwr_min_duty_cycle.into(), + max_pstate_scaled_6: max_ps_scaled.into(), + max_freq_mhz: pwr.max_freq_mhz, + pwr_min_duty_cycle_3: pwr.pwr_min_duty_cycle, + min_pstate_scaled_4: f32!(100.0), + max_pstate_scaled_7: max_ps_scaled, + unk_alpha_neg: f32!(0.8), + unk_alpha: f32!(0.2), + fast_die0_sensor_mask: U64(cfg.fast_sensor_mask[0]), + #[ver(G >= G14X)] + fast_die1_sensor_mask: U64(cfg.fast_sensor_mask[1]), + fast_die0_release_temp_cc: 100 * pwr.fast_die0_release_temp, + unk_87c: cfg.da.unk_87c, + unk_880: 0x4, + unk_894: f32!(1.0), + + fast_die0_ki_dt: pwr.fast_die0_integral_gain * period_s, + unk_8a8: f32!(65536.0), + fast_die0_kp: pwr.fast_die0_proportional_gain, + pwr_min_duty_cycle_4: pwr.pwr_min_duty_cycle, + max_pstate_scaled_8: max_ps_scaled, + max_pstate_scaled_9: max_ps_scaled, + fast_die0_prop_tgt_delta: 100 * pwr.fast_die0_prop_tgt_delta, + unk_8cc: cfg.da.unk_8cc, + max_pstate_scaled_10: max_ps_scaled, + max_pstate_scaled_11: max_ps_scaled, + unk_c2c: 1, + power_zone_count: pwr.power_zones.len() as u32, + max_power_4: pwr.max_power_mw, + max_power_5: pwr.max_power_mw, + max_power_6: pwr.max_power_mw, + avg_power_target_filter_a_neg: f32!(1.0) - avg_power_target_filter_a, + avg_power_target_filter_a: avg_power_target_filter_a, + avg_power_target_filter_tc_x4: 4 * pwr.avg_power_target_filter_tc, + avg_power_target_filter_tc_xperiod: period_ms * pwr.avg_power_target_filter_tc, + #[ver(V >= V13_0B4)] + avg_power_target_filter_tc_clks: period_ms + * pwr.avg_power_target_filter_tc + * base_clock_khz, + avg_power_filter_tc_periods_x4: 4 * avg_power_filter_tc_periods, + avg_power_filter_a_neg: f32!(1.0) - avg_power_filter_a, + avg_power_filter_a: avg_power_filter_a, + avg_power_ki_dt: pwr.avg_power_ki_only * period_s, + unk_d20: f32!(65536.0), + avg_power_kp: pwr.avg_power_kp, + avg_power_min_duty_cycle: pwr.avg_power_min_duty_cycle, + max_pstate_scaled_12: max_ps_scaled, + max_pstate_scaled_13: max_ps_scaled, + max_power_7: pwr.max_power_mw.into(), + max_power_8: pwr.max_power_mw, + avg_power_filter_tc_ms: pwr.avg_power_filter_tc_ms, + #[ver(V >= V13_0B4)] + avg_power_filter_tc_clks: avg_power_filter_tc_ms_rounded * base_clock_khz, + max_pstate_scaled_14: max_ps_scaled, + t81xx_data <- Self::t81xx_data(cfg, dyncfg), + #[ver(V >= V13_0B4)] + unk_e10_0 <- { + let filter_a = f32!(1.0) / pwr.se_filter_time_constant.into(); + let filter_1_a = f32!(1.0) / pwr.se_filter_time_constant_1.into(); + try_init!(raw::HwDataA130Extra { + unk_38: 4, + unk_3c: 8000, + gpu_se_inactive_threshold: pwr.se_inactive_threshold, + gpu_se_engagement_criteria: pwr.se_engagement_criteria, + gpu_se_reset_criteria: pwr.se_reset_criteria, + unk_54: 50, + unk_58: 0x1, + gpu_se_filter_a_neg: f32!(1.0) - filter_a, + gpu_se_filter_1_a_neg: f32!(1.0) - filter_1_a, + gpu_se_filter_a: filter_a, + gpu_se_filter_1_a: filter_1_a, + gpu_se_ki_dt: pwr.se_ki * period_s, + gpu_se_ki_1_dt: pwr.se_ki_1 * period_s, + unk_7c: f32!(65536.0), + gpu_se_kp: pwr.se_kp, + gpu_se_kp_1: pwr.se_kp_1, + + #[ver(V >= V13_3)] + unk_8c: 100, + #[ver(V < V13_3)] + unk_8c: 40, + + max_pstate_scaled_1: max_ps_scaled, + unk_9c: f32!(8000.0), + unk_a0: 1400, + gpu_se_filter_time_constant_ms: pwr.se_filter_time_constant * period_ms, + gpu_se_filter_time_constant_1_ms: pwr.se_filter_time_constant_1 + * period_ms, + gpu_se_filter_time_constant_clks: U64((pwr.se_filter_time_constant + * clocks_per_period_coarse) + .into()), + gpu_se_filter_time_constant_1_clks: U64((pwr + .se_filter_time_constant_1 + * clocks_per_period_coarse) + .into()), + unk_c4: f32!(65536.0), + unk_114: f32!(65536.0), + unk_124: 40, + max_pstate_scaled_2: max_ps_scaled, + ..Zeroable::init_zeroed() + }) + }, + fast_die0_sensor_mask_2: U64(cfg.fast_sensor_mask[0]), + #[ver(G >= G14X)] + fast_die1_sensor_mask_2: U64(cfg.fast_sensor_mask[1]), + unk_e24: cfg.da.unk_e24, + unk_e28: 1, + fast_die0_sensor_mask_alt: U64(cfg.fast_sensor_mask_alt[0]), + #[ver(G >= G14X)] + fast_die1_sensor_mask_alt: U64(cfg.fast_sensor_mask_alt[1]), + #[ver(V < V13_0B4)] + fast_die0_sensor_present: U64(cfg.fast_die0_sensor_present as u64), + unk_163c: 1, + unk_3644: 0, + hws1 <- Self::hw_shared1(cfg), + hws2 <- Self::hw_shared2(cfg, dyncfg), + hws3 <- Self::hw_shared3(cfg), + unk_3ce8: 1, + ..Zeroable::init_zeroed() + }) + .chain(|raw| { + for i in 0..self.dyncfg.pwr.perf_states.len() { + raw.sram_k[i] = self.cfg.sram_k; + } + + for (i, coef) in pwr.core_leak_coef.iter().enumerate() { + raw.core_leak_coef[i] = *coef; + } + + for (i, coef) in pwr.sram_leak_coef.iter().enumerate() { + raw.sram_leak_coef[i] = *coef; + } + + #[ver(V >= V13_0B4)] + if let Some(csafr) = pwr.csafr.as_ref() { + for (i, coef) in csafr.leak_coef_afr.iter().enumerate() { + raw.aux_leak_coef.cs_1[i] = *coef; + raw.aux_leak_coef.cs_2[i] = *coef; + } + + for (i, coef) in csafr.leak_coef_cs.iter().enumerate() { + raw.aux_leak_coef.afr_1[i] = *coef; + raw.aux_leak_coef.afr_2[i] = *coef; + } + } + + for i in 0..self.dyncfg.id.num_clusters as usize { + if let Some(coef_a) = self.cfg.unk_coef_a.get(i) { + (*raw.unk_coef_a1[i])[..coef_a.len()].copy_from_slice(coef_a); + (*raw.unk_coef_a2[i])[..coef_a.len()].copy_from_slice(coef_a); + } + if let Some(coef_b) = self.cfg.unk_coef_b.get(i) { + (*raw.unk_coef_b1[i])[..coef_b.len()].copy_from_slice(coef_b); + (*raw.unk_coef_b2[i])[..coef_b.len()].copy_from_slice(coef_b); + } + } + + for (i, pz) in pwr.power_zones.iter().enumerate() { + raw.power_zones[i].target = pz.target; + raw.power_zones[i].target_off = pz.target - pz.target_offset; + raw.power_zones[i].filter_tc_x4 = 4 * pz.filter_tc; + raw.power_zones[i].filter_tc_xperiod = period_ms * pz.filter_tc; + let filter_a = f32!(1.0) / pz.filter_tc.into(); + raw.power_zones[i].filter_a = filter_a; + raw.power_zones[i].filter_a_neg = f32!(1.0) - filter_a; + #[ver(V >= V13_0B4)] + raw.power_zones[i].unk_10 = 1320000000; + } + + #[ver(V >= V13_0B4 && G >= G14X)] + for (i, j) in raw.hws2.g14.curve2.t1.iter().enumerate() { + raw.unk_hws2[i] = if *j == 0xffff { 0 } else { j / 2 }; + } + + if !dyncfg.hw_data_b.is_empty() { + unsafe { + let mut matches: bool = true; + let sla = core::slice::from_raw_parts( + raw as *const raw::HwDataA::ver as *const u8, + core::mem::size_of::(), + ); + if sla.len() != dyncfg.hw_data_a.len() { + matches = false; + dev_err!( + self.dev.as_ref(), + "!!! Hwdata A size mismatch: {} {}", + sla.len(), + dyncfg.hw_data_a.len(), + ); + } + for i in 0..core::cmp::min(sla.len(), dyncfg.hw_data_a.len()) { + if sla[i] != dyncfg.hw_data_a[i] { + matches = false; + dev_err!(self.dev.as_ref(), "!!! Hwdata A first mismatch: {i}"); + break; + } + } + if matches { + dev_info!(self.dev.as_ref(), "!!! Hwdata A match"); + } + } + } + + Ok(()) + }) + }) + } + + /// Create the HwDataB structure. This mostly contains GPU-related configuration. + fn hwdata_b(&mut self) -> Result> { + self.alloc + .private + .new_init(pin_init::init_zeroed(), |_inner, _ptr| { + let cfg = &self.cfg; + let dyncfg = &self.dyncfg; + try_init!(raw::HwDataB::ver { + // Userspace VA map related + #[ver(V < V13_0B4)] + unk_0: U64(0x13_00000000), + unk_8: U64(0x14_00000000), + #[ver(V < V13_0B4)] + unk_10: U64(0x1_00000000), + unk_18: U64(0xffc00000), + // USC start + unk_20: U64(0), // U64(0x11_00000000), + unk_28: U64(0), // U64(0x11_00000000), + // Unknown page + //unk_30: U64(0x6f_ffff8000), + unk_30: U64(mmu::IOVA_UNK_PAGE), + timestamp_area_base: U64(gpu::IOVA_KERN_TIMESTAMP_RANGE.start), + // TODO: yuv matrices + chip_id: cfg.chip_id, + unk_454: cfg.db.unk_454, + unk_458: 0x1, + unk_460: 0x1, + unk_464: 0x1, + unk_468: 0x1, + unk_47c: 0x1, + unk_484: 0x1, + unk_48c: 0x1, + base_clock_khz: cfg.base_clock_hz / 1000, + power_sample_period: dyncfg.pwr.power_sample_period, + unk_49c: 0x1, + unk_4a0: 0x1, + unk_4a4: 0x1, + unk_4c0: 0x1f, + unk_4e0: U64(cfg.db.unk_4e0), + unk_4f0: 0x1, + unk_4f4: 0x1, + unk_504: 0x31, + unk_524: 0x1, // use_secure_cache_flush + unk_534: cfg.db.unk_534, + num_frags: dyncfg.id.num_frags * dyncfg.id.num_clusters, + unk_554: 0x1, + uat_ttb_base: U64(dyncfg.uat_ttb_base), + gpu_core_id: cfg.gpu_core as u32, + gpu_rev_id: dyncfg.id.gpu_rev_id as u32, + num_cores: dyncfg.id.num_cores * dyncfg.id.num_clusters, + max_pstate: dyncfg.pwr.perf_states.len() as u32 - 1, + #[ver(V < V13_0B4)] + num_pstates: dyncfg.pwr.perf_states.len() as u32, + #[ver(V < V13_0B4)] + min_sram_volt: dyncfg.pwr.min_sram_microvolt / 1000, + #[ver(V < V13_0B4)] + unk_ab8: cfg.db.unk_ab8, + #[ver(V < V13_0B4)] + unk_abc: cfg.db.unk_abc, + #[ver(V < V13_0B4)] + unk_ac0: 0x1020, + + #[ver(V >= V13_0B4)] + unk_ae4: Array::new([0x0, 0x3, 0x7, 0x7]), + #[ver(V < V13_0B4)] + unk_ae4: Array::new([0x0, 0xf, 0x3f, 0x3f]), + unk_b10: 0x1, + timer_offset: U64(0), + unk_b24: 0x1, + unk_b28: 0x1, + unk_b2c: 0x1, + unk_b30: cfg.db.unk_b30, + #[ver(V >= V13_0B4)] + unk_b38_0: 1, + #[ver(V >= V13_0B4)] + unk_b38_4: 1, + unk_b38: Array::new([0xffffffff; 12]), + #[ver(V >= V13_0B4 && V < V13_3)] + unk_c3c: 0x19, + #[ver(V >= V13_3)] + unk_c3c: 0x1a, + ..Zeroable::init_zeroed() + }) + .chain(|raw| { + #[ver(V >= V13_3)] + for i in 0..16 { + raw.unk_arr_0[i] = i as u32; + } + + let base_ps = self.dyncfg.pwr.perf_base_pstate as usize; + let max_ps = self.dyncfg.pwr.perf_max_pstate as usize; + let base_freq = self.dyncfg.pwr.perf_states[base_ps].freq_hz; + let max_freq = self.dyncfg.pwr.perf_states[max_ps].freq_hz; + + for (i, ps) in self.dyncfg.pwr.perf_states.iter().enumerate() { + raw.frequencies[i] = ps.freq_hz / 1000000; + for (j, mv) in ps.volt_mv.iter().enumerate() { + let sram_mv = (*mv).max(self.dyncfg.pwr.min_sram_microvolt / 1000); + raw.voltages[i][j] = *mv; + raw.voltages_sram[i][j] = sram_mv; + } + for j in ps.volt_mv.len()..raw.voltages[i].len() { + raw.voltages[i][j] = raw.voltages[i][0]; + raw.voltages_sram[i][j] = raw.voltages_sram[i][0]; + } + raw.sram_k[i] = self.cfg.sram_k; + raw.rel_max_powers[i] = ps.pwr_mw * 100 / self.dyncfg.pwr.max_power_mw; + raw.rel_boost_freqs[i] = if i > base_ps { + (ps.freq_hz - base_freq) / ((max_freq - base_freq) / 100) + } else { + 0 + }; + } + + #[ver(V >= V13_0B4)] + if let Some(csafr) = self.dyncfg.pwr.csafr.as_ref() { + let aux = &mut raw.aux_ps; + aux.cs_max_pstate = (csafr.perf_states_cs.len() - 1).try_into()?; + aux.afr_max_pstate = (csafr.perf_states_afr.len() - 1).try_into()?; + + for (i, ps) in csafr.perf_states_cs.iter().enumerate() { + aux.cs_frequencies[i] = ps.freq_hz / 1000000; + for (j, mv) in ps.volt_mv.iter().enumerate() { + let sram_mv = (*mv).max(csafr.min_sram_microvolt / 1000); + aux.cs_voltages[i][j] = *mv; + aux.cs_voltages_sram[i][j] = sram_mv; + } + } + + for (i, ps) in csafr.perf_states_afr.iter().enumerate() { + aux.afr_frequencies[i] = ps.freq_hz / 1000000; + for (j, mv) in ps.volt_mv.iter().enumerate() { + let sram_mv = (*mv).max(csafr.min_sram_microvolt / 1000); + aux.afr_voltages[i][j] = *mv; + aux.afr_voltages_sram[i][j] = sram_mv; + } + } + } + + // Special case override for T602x + #[ver(G == G14X)] + if dyncfg.id.gpu_rev_id == hw::GpuRevisionID::B1 { + raw.gpu_rev_id = hw::GpuRevisionID::B0 as u32; + } + + if !dyncfg.hw_data_b.is_empty() { + unsafe { + let mut matches: bool = true; + let sla = core::slice::from_raw_parts( + raw as *const raw::HwDataB::ver as *const u8, + core::mem::size_of::(), + ); + if sla.len() != dyncfg.hw_data_b.len() { + matches = false; + dev_err!( + self.dev.as_ref(), + "!!! Hwdata B size mismatch: {} {}", + sla.len(), + dyncfg.hw_data_b.len(), + ); + } + for i in 0..core::cmp::min(sla.len(), dyncfg.hw_data_b.len()) { + if sla[i] != dyncfg.hw_data_b[i] { + matches = false; + dev_err!(self.dev.as_ref(), "!!! Hwdata B first mismatch: {i}"); + break; + } + } + if matches { + dev_info!(self.dev.as_ref(), "!!! Hwdata B match"); + } + } + } + + Ok(()) + }) + }) + } + + /// Create the Globals structure, which contains global firmware config including more power + /// configuration data and globals used to exchange state between the firmware and driver. + fn globals(&mut self) -> Result> { + self.alloc + .private + .new_init(pin_init::init_zeroed(), |_inner, _ptr| { + let cfg = &self.cfg; + let dyncfg = &self.dyncfg; + let pwr = &dyncfg.pwr; + let period_ms = pwr.power_sample_period; + let period_s = F32::from(period_ms) / f32!(1000.0); + let avg_power_filter_tc_periods = pwr.avg_power_filter_tc_ms / period_ms; + + let max_ps = pwr.perf_max_pstate; + let max_ps_scaled = 100 * max_ps; + + try_init!(raw::Globals::ver { + //ktrace_enable: 0xffffffff, + ktrace_enable: 0, + #[ver(V >= V13_2)] + unk_24_0: 3000, + unk_24: 0, + #[ver(V >= V13_0B4)] + debug: 0, + unk_28: 1, + #[ver(G >= G14X)] + unk_2c_0: 1, + #[ver(V >= V13_0B4 && G < G14X)] + unk_2c_0: 0, + unk_2c: 1, + unk_30: 0, + unk_34: 120, + // sub <- try_init!(raw::GlobalsSub::ver { + unk_54: cfg.global_unk_54, + unk_56: 40, + unk_58: 0xffff, + unk_5e: U32(1), + unk_66: U32(1), + // ..Zeroable::init_zeroed() + // }), + unk_8900: 1, + pending_submissions: AtomicU32::new(0), + max_power: pwr.max_power_mw, + max_pstate_scaled: max_ps_scaled, + max_pstate_scaled_2: max_ps_scaled, + max_pstate_scaled_3: max_ps_scaled, + power_zone_count: pwr.power_zones.len() as u32, + avg_power_filter_tc_periods: avg_power_filter_tc_periods, + avg_power_ki_dt: pwr.avg_power_ki_only * period_s, + avg_power_kp: pwr.avg_power_kp, + avg_power_min_duty_cycle: pwr.avg_power_min_duty_cycle, + avg_power_target_filter_tc: pwr.avg_power_target_filter_tc, + unk_89bc: cfg.da.unk_8cc, + fast_die0_release_temp: 100 * pwr.fast_die0_release_temp, + unk_89c4: cfg.da.unk_87c, + fast_die0_prop_tgt_delta: 100 * pwr.fast_die0_prop_tgt_delta, + fast_die0_kp: pwr.fast_die0_proportional_gain, + fast_die0_ki_dt: pwr.fast_die0_integral_gain * period_s, + unk_89e0: 1, + max_power_2: pwr.max_power_mw, + ppm_kp: pwr.ppm_kp, + ppm_ki_dt: pwr.ppm_ki * period_s, + #[ver(V >= V13_0B4)] + unk_89f4_8: 1, + unk_89f4: 0, + hws1 <- Self::hw_shared1(cfg), + hws2 <- Self::hw_shared2(cfg, dyncfg), + hws3 <- Self::hw_shared3(cfg), + #[ver(V >= V13_0B4)] + idle_off_standby_timer: pwr.idle_off_standby_timer, + #[ver(V >= V13_0B4)] + unk_hws2_4: cfg.unk_hws2_4.map(Array::new).unwrap_or_default(), + #[ver(V >= V13_0B4)] + unk_hws2_24: cfg.unk_hws2_24, + unk_900c: 1, + #[ver(V >= V13_0B4)] + unk_9010_0: 1, + #[ver(V >= V13_0B4)] + unk_903c: 1, + #[ver(V < V13_0B4)] + unk_903c: 0, + fault_control: *module_parameters::fault_control.value(), + do_init: 1, + progress_check_interval_3d: 40, + progress_check_interval_ta: 10, + progress_check_interval_cl: 250, + #[ver(V >= V13_0B4)] + unk_1102c_0: 1, + #[ver(V >= V13_0B4)] + unk_1102c_4: 1, + #[ver(V >= V13_0B4)] + unk_1102c_8: 100, + #[ver(V >= V13_0B4)] + unk_1102c_c: 1, + idle_off_delay_ms: AtomicU32::new(pwr.idle_off_delay_ms), + fender_idle_off_delay_ms: pwr.fender_idle_off_delay_ms, + fw_early_wake_timeout_ms: pwr.fw_early_wake_timeout_ms, + cl_context_switch_timeout_ms: 40, + #[ver(V >= V13_0B4)] + cl_kill_timeout_ms: 50, + #[ver(V >= V13_0B4)] + unk_11edc: 0, + #[ver(V >= V13_0B4)] + unk_11efc: 0, + ..Zeroable::init_zeroed() + }) + .chain(|raw| { + for (i, pz) in self.dyncfg.pwr.power_zones.iter().enumerate() { + raw.power_zones[i].target = pz.target; + raw.power_zones[i].target_off = pz.target - pz.target_offset; + raw.power_zones[i].filter_tc = pz.filter_tc; + } + + if let Some(tab) = self.cfg.global_tab.as_ref() { + for (i, x) in tab.iter().enumerate() { + raw.unk_118ec[i] = *x; + } + raw.unk_118e8 = 1; + } + + if !dyncfg.hw_globals.is_empty() { + unsafe { + let mut matches: bool = true; + let sla = core::slice::from_raw_parts( + raw as *const raw::Globals::ver as *const u8, + core::mem::size_of::(), + ); + if sla.len() != dyncfg.hw_globals.len() { + matches = false; + dev_err!( + self.dev.as_ref(), + "!!! Globals size mismatch: {} {}", + sla.len(), + dyncfg.hw_globals.len(), + ); + } + for i in 0..core::cmp::min(sla.len(), dyncfg.hw_globals.len()) { + if sla[i] != dyncfg.hw_globals[i] { + matches = false; + dev_err!(self.dev.as_ref(), "!!! Globals first mismatch: {i}"); + break; + } + } + if matches { + dev_info!(self.dev.as_ref(), "!!! Globals match"); + } + } + } + + Ok(()) + }) + }) + } + + /// Create the RuntimePointers structure, which contains pointers to most of the other + /// structures including the ring buffer channels, statistics structures, and HwDataA/HwDataB. + fn runtime_pointers(&mut self) -> Result> { + let hwa = self.hwdata_a()?; + let hwb = self.hwdata_b()?; + + let mut buffer_mgr_ctl = gem::new_kernel_object(self.dev, 0x4000)?; + buffer_mgr_ctl.vmap()?.memset(0); + + GpuObject::new_init_prealloc( + self.alloc.private.alloc_object()?, + |_ptr| { + let alloc = &mut *self.alloc; + try_init!(RuntimePointers::ver { + stats <- { + let alloc = &mut *alloc; + try_init!(Stats::ver { + vtx: alloc.private.new_default::()?, + frag: alloc.private.new_init( + pin_init::init_zeroed::(), + |_inner, _ptr| { + try_init!(raw::GpuGlobalStatsFrag::ver { + total_cmds: 0, + unk_4: 0, + stats: Default::default(), + }) + } + )?, + comp: alloc.private.new_default::()?, + }) + }, + + hwdata_a: hwa, + unkptr_190: alloc.private.array_empty_tagged(0x80, b"I190")?, + unkptr_198: alloc.private.array_empty_tagged(0xc0, b"I198")?, + hwdata_b: hwb, + + unkptr_1b8: alloc.private.array_empty_tagged(0x1000, b"I1B8")?, + unkptr_1c0: alloc.private.array_empty_tagged(0x300, b"I1C0")?, + unkptr_1c8: alloc.private.array_empty_tagged(0x1000, b"I1C8")?, + + buffer_mgr_ctl, + buffer_mgr_ctl_low_mapping: None, + buffer_mgr_ctl_high_mapping: None, + }) + }, + |inner, _ptr| { + try_init!(raw::RuntimePointers::ver { + pipes: Default::default(), + device_control: Default::default(), + event: Default::default(), + fw_log: Default::default(), + ktrace: Default::default(), + stats: Default::default(), + + stats_vtx: inner.stats.vtx.gpu_pointer(), + stats_frag: inner.stats.frag.gpu_pointer(), + stats_comp: inner.stats.comp.gpu_pointer(), + + hwdata_a: inner.hwdata_a.gpu_pointer(), + unkptr_190: inner.unkptr_190.gpu_pointer(), + unkptr_198: inner.unkptr_198.gpu_pointer(), + hwdata_b: inner.hwdata_b.gpu_pointer(), + hwdata_b_2: inner.hwdata_b.gpu_pointer(), + + fwlog_buf: None, + + unkptr_1b8: inner.unkptr_1b8.gpu_pointer(), + + #[ver(G < G14X)] + unkptr_1c0: inner.unkptr_1c0.gpu_pointer(), + #[ver(G < G14X)] + unkptr_1c8: inner.unkptr_1c8.gpu_pointer(), + + buffer_mgr_ctl_gpu_addr: U64(gpu::IOVA_KERN_GPU_BUFMGR_LOW), + buffer_mgr_ctl_fw_addr: U64(gpu::IOVA_KERN_GPU_BUFMGR_HIGH), + + __pad0: Default::default(), + unk_160: U64(0), + unk_168: U64(0), + unk_1d0: 0, + unk_1d4: 0, + unk_1d8: Default::default(), + + __pad1: Default::default(), + gpu_scratch: raw::RuntimeScratch::ver { + unk_6b38: 0xff, + ..Default::default() + }, + }) + }, + ) + } + + /// Create the FwStatus structure, which is used to coordinate the firmware halt state between + /// the firmware and the driver. + fn fw_status(&mut self) -> Result> { + self.alloc + .shared + .new_object(Default::default(), |_inner| Default::default()) + } + + /// Create one UatLevelInfo structure, which describes one level of translation for the UAT MMU. + fn uat_level_info( + cfg: &'static hw::HwConfig, + index_shift: usize, + num_entries: usize, + ) -> raw::UatLevelInfo { + raw::UatLevelInfo { + index_shift: index_shift as _, + unk_1: 14, + unk_2: 14, + unk_3: 8, + unk_4: 0x4000, + num_entries: num_entries as _, + unk_8: U64(1), + unk_10: U64(((1u64 << cfg.uat_oas) - 1) & !(mmu::UAT_PGMSK as u64)), + index_mask: U64(((num_entries - 1) << index_shift) as u64), + } + } + + /// Build the top-level InitData object. + #[inline(never)] + pub(crate) fn build(&mut self) -> Result>> { + let runtime_pointers = self.runtime_pointers()?; + let globals = self.globals()?; + let fw_status = self.fw_status()?; + let shared_ro = &mut self.alloc.shared_ro; + + let obj = self.alloc.private.new_init( + try_init!(InitData::ver { + unk_buf: shared_ro.array_empty_tagged(0x4000, b"IDTA")?, + runtime_pointers, + globals, + fw_status, + }), + |inner, _ptr| { + let cfg = &self.cfg; + try_init!(raw::InitData::ver { + #[ver(V == V13_5 && G != G14X)] + ver_info: Array::new([0x6ba0, 0x1f28, 0x601, 0xb0]), + #[ver(V == V13_5 && G == G14X)] + ver_info: Array::new([0xb390, 0x70f8, 0x601, 0xb0]), + unk_buf: inner.unk_buf.gpu_pointer(), + unk_8: 0, + unk_c: 0, + runtime_pointers: inner.runtime_pointers.gpu_pointer(), + globals: inner.globals.gpu_pointer(), + fw_status: inner.fw_status.gpu_pointer(), + uat_page_size: 0x4000, + uat_page_bits: 14, + uat_num_levels: 3, + uat_level_info: Array::new([ + Self::uat_level_info(cfg, 36, 8), + Self::uat_level_info(cfg, 25, 2048), + Self::uat_level_info(cfg, 14, 2048), + ]), + __pad0: Default::default(), + host_mapped_fw_allocations: 1, + unk_ac: 0, + unk_b0: 0, + unk_b4: 0, + unk_b8: 0, + }) + }, + )?; + Ok(KBox::new(obj, GFP_KERNEL)?) + } +} diff --git a/drivers/gpu/drm/asahi/mem.rs b/drivers/gpu/drm/asahi/mem.rs new file mode 100644 index 00000000000000..60a64e23a161c5 --- /dev/null +++ b/drivers/gpu/drm/asahi/mem.rs @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! ARM64 low level memory operations. +//! +//! This GPU uses CPU-side `tlbi` outer-shareable instructions to manage its TLBs. +//! Yes, really. Even though the VA address spaces are unrelated. +//! +//! Right now we pick our own ASIDs and don't coordinate with the CPU. This might result +//! in needless TLB shootdowns on the CPU side... TODO: fix this. + +use core::arch::asm; +use core::cmp::min; + +use crate::debug::*; +use crate::mmu; + +type Asid = u8; + +/// Invalidate the entire GPU TLB. +#[inline(always)] +pub(crate) fn tlbi_all() { + // SAFETY: tlbi is always safe by definition + unsafe { + asm!(".arch armv8.4-a", "tlbi vmalle1os",); + } +} + +/// Invalidate all TLB entries for a given ASID. +#[inline(always)] +pub(crate) fn tlbi_asid(asid: Asid) { + if debug_enabled(DebugFlags::ConservativeTlbi) { + tlbi_all(); + sync(); + return; + } + + // SAFETY: tlbi is always safe by definition + unsafe { + asm!( + ".arch armv8.4-a", + "tlbi aside1os, {x}", + x = in(reg) ((asid as u64) << 48) + ); + } +} + +/// Invalidate a single page for a given ASID. +#[inline(always)] +pub(crate) fn tlbi_page(asid: Asid, va: usize) { + if debug_enabled(DebugFlags::ConservativeTlbi) { + tlbi_all(); + sync(); + return; + } + + let val: u64 = ((asid as u64) << 48) | ((va as u64 >> 12) & 0xffffffffffc); + // SAFETY: tlbi is always safe by definition + unsafe { + asm!( + ".arch armv8.4-a", + "tlbi vae1os, {x}", + x = in(reg) val + ); + } +} + +/// Invalidate a range of pages for a given ASID. +#[inline(always)] +pub(crate) fn tlbi_range(asid: Asid, va: usize, len: usize) { + if debug_enabled(DebugFlags::ConservativeTlbi) { + tlbi_all(); + sync(); + return; + } + + if len == 0 { + return; + } + + let start_pg = va >> mmu::UAT_PGBIT; + let end_pg = (va + len + mmu::UAT_PGMSK) >> mmu::UAT_PGBIT; + + let mut val: u64 = ((asid as u64) << 48) | (2 << 46) | (start_pg as u64 & 0x1fffffffff); + let pages = end_pg - start_pg; + + // Guess? It's possible that the page count is in terms of 4K pages + // when the CPU is in 4K mode... + #[cfg(CONFIG_ARM64_4K_PAGES)] + let pages = 4 * pages; + + if pages == 1 { + tlbi_page(asid, va); + return; + } + + // Page count is always in units of 2 + let num = ((pages + 1) >> 1) as u64; + // base: 5 bits + // exp: 2 bits + // pages = (base + 1) << (5 * exp + 1) + // 0:00000 -> 2 pages = 2 << 0 + // 0:11111 -> 32 * 2 pages = 2 << 5 + // 1:00000 -> 1 * 32 * 2 pages = 2 << 5 + // 1:11111 -> 32 * 32 * 2 pages = 2 << 10 + // 2:00000 -> 1 * 32 * 32 * 2 pages = 2 << 10 + // 2:11111 -> 32 * 32 * 32 * 2 pages = 2 << 15 + // 3:00000 -> 1 * 32 * 32 * 32 * 2 pages = 2 << 15 + // 3:11111 -> 32 * 32 * 32 * 32 * 2 pages = 2 << 20 + let exp = min(3, (64 - num.leading_zeros()) / 5); + let bits = 5 * exp; + let mut base = (num + (1 << bits) - 1) >> bits; + + val |= (exp as u64) << 44; + + while base > 32 { + // SAFETY: tlbi is always safe by definition + unsafe { + asm!( + ".arch armv8.4-a", + "tlbi rvae1os, {x}", + x = in(reg) val | (31 << 39) + ); + } + base -= 32; + } + + // SAFETY: tlbi is always safe by definition + unsafe { + asm!( + ".arch armv8.4-a", + "tlbi rvae1os, {x}", + x = in(reg) val | ((base - 1) << 39) + ); + } +} + +/// Issue a memory barrier (`dsb sy`). +#[inline(always)] +pub(crate) fn sync() { + // SAFETY: Barriers are always safe + unsafe { + asm!("dsb sy"); + } +} diff --git a/drivers/gpu/drm/asahi/microseq.rs b/drivers/gpu/drm/asahi/microseq.rs new file mode 100644 index 00000000000000..cbdb5de62e9218 --- /dev/null +++ b/drivers/gpu/drm/asahi/microseq.rs @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU Micro operation sequence builder +//! +//! As part of a single job submisssion to the GPU, the GPU firmware interprets a sequence of +//! commands that we call a "microsequence". These are responsible for setting up the job execution, +//! timestamping the process, waiting for completion, tearing up any resources, and signaling +//! completion to the driver via the event stamp mechanism. +//! +//! Although the microsequences used by the macOS driver are usually quite uniform and simple, the +//! firmware actually implements enough operations to make this interpreter Turing-complete (!). +//! Most of those aren't implemented yet, since we don't need them, but they could come in handy in +//! the future to do strange things or work around firmware bugs... +//! +//! This module simply implements a collection of microsequence operations that can be appended to +//! and later concatenated into one buffer, ready for firmware execution. + +use crate::fw::microseq; +pub(crate) use crate::fw::microseq::*; +use crate::fw::types::*; +use kernel::prelude::*; + +/// MicroSequence object type, which is just an opaque byte array. +pub(crate) type MicroSequence = GpuArray; + +/// MicroSequence builder. +pub(crate) struct Builder { + ops: KVec, +} + +impl Builder { + /// Create a new Builder object + pub(crate) fn new() -> Builder { + Builder { ops: KVec::new() } + } + + /// Get the relative offset from the current pointer to a given target offset. + /// + /// Used for relative jumps. + pub(crate) fn offset_to(&self, target: i32) -> i32 { + target - self.ops.len() as i32 + } + + /// Add an operation to the end of the sequence. + pub(crate) fn add(&mut self, op: T) -> Result { + let off = self.ops.len(); + let p: *const T = &op; + let p: *const u8 = p as *const u8; + // SAFETY: Microseq operations always have no padding bytes, so it is safe to + // access them as a byte slice. + let s: &[u8] = unsafe { core::slice::from_raw_parts(p, core::mem::size_of::()) }; + self.ops.extend_from_slice(s, GFP_KERNEL)?; + Ok(off as i32) + } + + /// Collect all submitted operations into a finalized GPU object. + pub(crate) fn build(self, alloc: &mut Allocator) -> Result { + let mut array = alloc.array_empty::(self.ops.len())?; + + array.as_mut_slice().clone_from_slice(self.ops.as_slice()); + Ok(array) + } +} diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs new file mode 100644 index 00000000000000..93692e27f2061f --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -0,0 +1,1602 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU UAT (MMU) management +//! +//! AGX GPUs use an MMU called the UAT, which is largely compatible with the ARM64 page table +//! format. This module manages the global MMU structures, including a shared handoff structure +//! that is used to coordinate VM management operations with the firmware, the TTBAT which points +//! to currently active GPU VM contexts, as well as the individual `Vm` operations to map and +//! unmap buffer objects into a single user or kernel address space. +//! +//! The actual page table management is in the `pt` module. + +use core::fmt::Debug; +use core::mem::size_of; +use core::num::NonZeroUsize; +use core::ops::Range; +use core::sync::atomic::{ + fence, + AtomicU32, + AtomicU64, + AtomicU8, + Ordering, // +}; + +use kernel::{ + addr::PhysicalAddr, + bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE, + c_str, + device, + drm::{ + gem::shmem, + gpuvm, + mm, // + }, + error::Result, + io, + new_mutex, + prelude::*, + static_lock_class, + sync::{ + lock::{ + mutex::MutexBackend, + Guard, // + }, + Arc, Mutex, + }, + time::{ + delay::fsleep, + Delta, + Instant, + Monotonic, // + }, + types::ARef, // +}; + +use crate::debug::*; +use crate::module_parameters; +use crate::no_debug; +use crate::{ + driver, + fw, + gem, + hw, + mem, + pgtable, + slotalloc, + util::RangeExt, // +}; + +// KernelMapping protection types +pub(crate) use crate::pgtable::Prot; +pub(crate) use pgtable::prot::*; +pub(crate) use pgtable::{ + UatPageTable, + UAT_PGBIT, + UAT_PGMSK, + UAT_PGSZ, // +}; + +use pgtable::UAT_IAS; + +use pin_init; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; + +/// PPL magic number for the handoff region +const PPL_MAGIC: u64 = 0x4b1d000000000002; + +/// Number of supported context entries in the TTBAT +const UAT_NUM_CTX: usize = 64; +/// First context available for users +const UAT_USER_CTX_START: usize = 1; +/// Number of available user contexts +const UAT_USER_CTX: usize = UAT_NUM_CTX - UAT_USER_CTX_START; + +/// Lower/user base VA +pub(crate) const IOVA_USER_BASE: u64 = UAT_PGSZ as u64; +/// Lower/user top VA +pub(crate) const IOVA_USER_TOP: u64 = 1 << (UAT_IAS as u64); +/// Lower/user VA range +pub(crate) const IOVA_USER_RANGE: Range = IOVA_USER_BASE..IOVA_USER_TOP; + +/// Upper/kernel base VA +#[cfg(CONFIG_DEV_COREDUMP)] +const IOVA_TTBR1_BASE: u64 = 0xffffff8000000000; +/// Driver-managed kernel base VA +const IOVA_KERN_BASE: u64 = 0xffffffa000000000; +/// Driver-managed kernel top VA +const IOVA_KERN_TOP: u64 = 0xffffffb000000000; +/// Driver-managed kernel VA range +const IOVA_KERN_RANGE: Range = IOVA_KERN_BASE..IOVA_KERN_TOP; +/// Full kernel VA range +#[cfg(CONFIG_DEV_COREDUMP)] +const IOVA_KERN_FULL_RANGE: Range = IOVA_TTBR1_BASE..(!UAT_PGMSK as u64); + +const TTBR_VALID: u64 = 0x1; // BIT(0) +const TTBR_ASID_SHIFT: usize = 48; + +/// Address of a special dummy page? +//const IOVA_UNK_PAGE: u64 = 0x6f_ffff8000; +pub(crate) const IOVA_UNK_PAGE: u64 = IOVA_USER_TOP - 2 * UAT_PGSZ as u64; +/// User VA range excluding the unk page +pub(crate) const IOVA_USER_USABLE_RANGE: Range = IOVA_USER_BASE..IOVA_UNK_PAGE; + +/// A pre-allocated memory region for UAT management +struct UatRegion { + base: PhysicalAddr, + map: io::mem::Mem, +} + +/// SAFETY: It's safe to share UAT region records across threads. +unsafe impl Send for UatRegion {} +/// SAFETY: It's safe to share UAT region records across threads. +unsafe impl Sync for UatRegion {} + +/// Handoff region flush info structure +#[repr(C)] +struct FlushInfo { + state: AtomicU64, + addr: AtomicU64, + size: AtomicU64, +} + +/// UAT Handoff region layout +#[repr(C)] +struct Handoff { + magic_ap: AtomicU64, + magic_fw: AtomicU64, + + lock_ap: AtomicU8, + lock_fw: AtomicU8, + // Implicit padding: 2 bytes + turn: AtomicU32, + cur_slot: AtomicU32, + // Implicit padding: 4 bytes + flush: [FlushInfo; UAT_NUM_CTX + 1], + + unk2: AtomicU8, + // Implicit padding: 7 bytes + unk3: AtomicU64, +} + +const HANDOFF_SIZE: usize = size_of::(); + +/// One VM slot in the TTBAT +#[repr(C)] +struct SlotTTBS { + ttb0: AtomicU64, + ttb1: AtomicU64, +} + +const SLOTS_SIZE: usize = UAT_NUM_CTX * size_of::(); + +// We need at least page 0 (ttb0) +const PAGETABLES_SIZE: usize = UAT_PGSZ; + +/// Inner data for a Vm instance. This is reference-counted by the outer Vm object. +struct VmInner { + dev: driver::AsahiDevRef, + is_kernel: bool, + va_range: Range, + page_table: UatPageTable, + mm: mm::Allocator<(), KernelMappingInner>, + uat_inner: Arc, + binding: Arc>, + id: u64, +} + +/// Slot binding-related inner data for a Vm instance. +struct VmBinding { + active_users: usize, + binding: Option>, + bind_token: Option, + ttb: u64, +} + +/// Data associated with a VM <=> BO pairing +#[pin_data] +struct VmBo { + #[pin] + sgt: Mutex>>, +} + +impl gpuvm::DriverGpuVmBo for VmBo { + fn new() -> impl PinInit { + pin_init!(VmBo { + sgt <- new_mutex!(None, "VmBinding"), + }) + } +} + +#[derive(Default)] +struct StepContext { + new_va: Option>>>, + prev_va: Option>>>, + next_va: Option>>>, + vm_bo: Option>>, + prot: Prot, +} + +impl gpuvm::DriverGpuVm for VmInner { + type Driver = driver::AsahiDriver; + type GpuVmBo = VmBo; + type StepContext = StepContext; + + fn step_map( + self: &mut gpuvm::UpdatingGpuVm<'_, Self>, + op: &mut gpuvm::OpMap, + ctx: &mut Self::StepContext, + ) -> Result { + let mut iova = op.addr(); + let mut left = op.range() as usize; + let mut offset = op.offset() as usize; + + let bo = ctx.vm_bo.as_ref().expect("step_map with no BO"); + + let one_page = op.flags().contains(gpuvm::GpuVaFlags::REPEAT); + + let guard = bo.inner().sgt.lock(); + for range in guard.as_ref().expect("step_map with no SGT").iter() { + // TODO: proper DMA address/length handling + let mut addr = range.dma_address() as usize; + let mut len: usize = range.dma_len() as usize; + + if left == 0 { + break; + } + + if offset > 0 { + let skip = len.min(offset); + addr += skip; + len -= skip; + offset -= skip; + } + + if len == 0 { + continue; + } + + assert!(offset == 0); + + if one_page { + len = left; + } else { + len = len.min(left); + } + + mod_dev_dbg!( + self.dev, + "MMU: map: {:#x}:{:#x} -> {:#x} [OP={}]\n", + addr, + len, + iova, + one_page + ); + + self.page_table.map_pages( + iova..(iova + len as u64), + addr as PhysicalAddr, + ctx.prot, + one_page, + )?; + + left -= len; + iova += len as u64; + } + + let gpuva = ctx.new_va.take().expect("Multiple step_map calls"); + + if op + .map_and_link_va( + self, + gpuva, + ctx.vm_bo.as_ref().expect("step_map with no BO"), + ) + .is_err() + { + dev_err!( + self.dev.as_ref(), + "map_and_link_va failed: {:#x} [{:#x}] -> {:#x}\n", + op.offset(), + op.range(), + op.addr() + ); + return Err(EINVAL); + } + Ok(()) + } + fn step_unmap( + self: &mut gpuvm::UpdatingGpuVm<'_, Self>, + op: &mut gpuvm::OpUnMap, + _ctx: &mut Self::StepContext, + ) -> Result { + let va = op.va().expect("step_unmap: missing VA"); + + mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", va.addr(), va.range()); + + self.page_table + .unmap_pages(va.addr()..(va.addr() + va.range()))?; + + if let Some(asid) = self.slot() { + fence(Ordering::SeqCst); + mem::tlbi_range(asid as u8, va.addr() as usize, va.range() as usize); + mod_dev_dbg!( + self.dev, + "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", + asid, + va.addr(), + va.range(), + ); + mem::sync(); + } + + if op.unmap_and_unlink_va_defer().is_none() { + dev_err!(self.dev.as_ref(), "step_unmap: could not unlink gpuva"); + } + Ok(()) + } + fn step_remap( + self: &mut gpuvm::UpdatingGpuVm<'_, Self>, + op: &mut gpuvm::OpReMap, + vm_bo: &gpuvm::GpuVmBo, + ctx: &mut Self::StepContext, + ) -> Result { + let va = op.unmap().va().expect("No previous VA"); + let orig_addr = va.addr(); + let orig_range = va.range(); + + // Only unmap the hole between prev/next, if they exist + let unmap_start = if let Some(op) = op.prev_map() { + op.addr() + op.range() + } else { + orig_addr + }; + + let unmap_end = if let Some(op) = op.next_map() { + op.addr() + } else { + orig_addr + orig_range + }; + + mod_dev_dbg!( + self.dev, + "MMU: unmap for remap: {:#x}..{:#x} (from {:#x}:{:#x})\n", + unmap_start, + unmap_end, + orig_addr, + orig_range + ); + + let unmap_range = unmap_end - unmap_start; + + self.page_table.unmap_pages(unmap_start..unmap_end)?; + + if let Some(asid) = self.slot() { + fence(Ordering::SeqCst); + mem::tlbi_range(asid as u8, unmap_start as usize, unmap_range as usize); + mod_dev_dbg!( + self.dev, + "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", + asid, + unmap_start, + unmap_range, + ); + mem::sync(); + } + + if op.unmap().unmap_and_unlink_va_defer().is_none() { + dev_err!(self.dev.as_ref(), "step_unmap: could not unlink gpuva"); + } + + if let Some(prev_op) = op.prev_map() { + let prev_gpuva = ctx + .prev_va + .take() + .expect("Multiple step_remap calls with prev_op"); + if prev_op.map_and_link_va(self, prev_gpuva, vm_bo).is_err() { + dev_err!(self.dev.as_ref(), "step_remap: could not relink prev gpuva"); + return Err(EINVAL); + } + } + + if let Some(next_op) = op.next_map() { + let next_gpuva = ctx + .next_va + .take() + .expect("Multiple step_remap calls with next_op"); + if next_op.map_and_link_va(self, next_gpuva, vm_bo).is_err() { + dev_err!(self.dev.as_ref(), "step_remap: could not relink next gpuva"); + return Err(EINVAL); + } + } + + Ok(()) + } +} + +impl VmInner { + /// Returns the slot index, if this VM is bound. + fn slot(&self) -> Option { + if self.is_kernel { + // The GFX ASC does not care about the ASID. Pick an arbitrary one. + // TODO: This needs to be a persistently reserved ASID once we integrate + // with the ARM64 kernel ASID machinery to avoid overlap. + Some(0) + } else { + // We don't check whether we lost the slot, which could cause unnecessary + // invalidations against another Vm. However, this situation should be very + // rare (e.g. a Vm lost its slot, which means 63 other Vms bound in the + // interim, and then it gets killed / drops its mappings without doing any + // final rendering). Anything doing active maps/unmaps is probably also + // rendering and therefore likely bound. + self.binding + .lock() + .bind_token + .as_ref() + .map(|token| token.last_slot() + UAT_USER_CTX_START as u32) + } + } + + /// Returns the translation table base for this Vm + fn ttb(&self) -> u64 { + self.page_table.ttb() + } + + /// Map an `mm::Node` representing an mapping in VA space. + fn map_node(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { + let mut iova = node.start(); + let guard = node.bo.as_ref().ok_or(EINVAL)?.inner().sgt.lock(); + let sgt = guard.as_ref().ok_or(EINVAL)?; + let mut offset = node.offset; + let mut left = node.mapped_size; + + for range in sgt.iter() { + if left == 0 { + break; + } + + // TODO: proper DMA address/length handling + let mut addr = range.dma_address() as usize; + let mut len: usize = range.dma_len() as usize; + + if (offset | addr | len | iova as usize) & UAT_PGMSK != 0 { + dev_err!( + self.dev.as_ref(), + "MMU: KernelMapping {:#x}:{:#x} -> {:#x} is not page-aligned\n", + addr, + len, + iova + ); + return Err(EINVAL); + } + + if offset > 0 { + let skip = len.min(offset); + addr += skip; + len -= skip; + offset -= skip; + } + + len = len.min(left); + + if len == 0 { + continue; + } + + mod_dev_dbg!( + self.dev, + "MMU: map: {:#x}:{:#x} -> {:#x}\n", + addr, + len, + iova + ); + + self.page_table.map_pages( + iova..(iova + len as u64), + addr as PhysicalAddr, + prot, + false, + )?; + + iova += len as u64; + left -= len; + } + Ok(()) + } +} + +/// Shared reference to a virtual memory address space ([`Vm`]). +#[derive(Clone)] +pub(crate) struct Vm { + id: u64, + inner: ARef>, + dummy_obj: ARef, + binding: Arc>, +} +no_debug!(Vm); + +/// Slot data for a [`Vm`] slot (nothing, we only care about the indices). +pub(crate) struct SlotInner(); + +impl slotalloc::SlotItem for SlotInner { + type Data = (); +} + +/// Represents a single user of a binding of a [`Vm`] to a slot. +/// +/// The number of users is counted, and the slot will be freed when it drops to 0. +#[derive(Debug)] +pub(crate) struct VmBind(Vm, u32); + +impl VmBind { + /// Returns the slot that this `Vm` is bound to. + pub(crate) fn slot(&self) -> u32 { + self.1 + } +} + +impl Drop for VmBind { + fn drop(&mut self) { + let mut binding = self.0.binding.lock(); + + assert_ne!(binding.active_users, 0); + binding.active_users -= 1; + mod_pr_debug!( + "MMU: slot {} active users {}\n", + self.1, + binding.active_users + ); + if binding.active_users == 0 { + binding.binding = None; + } + } +} + +impl Clone for VmBind { + fn clone(&self) -> VmBind { + let mut binding = self.0.binding.lock(); + + binding.active_users += 1; + mod_pr_debug!( + "MMU: slot {} active users {}\n", + self.1, + binding.active_users + ); + VmBind(self.0.clone(), self.1) + } +} + +/// Inner data required for an object mapping into a [`Vm`]. +pub(crate) struct KernelMappingInner { + // Drop order matters: + // - Drop the GpuVmBo first, which resv locks its BO and drops a GpuVm reference + // - Drop the GEM BO next, since BO free can take the resv lock itself + // - Drop the owner GpuVm last, since that again can take resv locks when the refcount drops to 0 + bo: Option>>, + _gem: Option>, + owner: ARef>, + uat_inner: Arc, + prot: Prot, + offset: usize, + mapped_size: usize, +} + +/// An object mapping into a [`Vm`], which reserves the address range from use by other mappings. +pub(crate) struct KernelMapping(mm::Node<(), KernelMappingInner>); + +impl KernelMapping { + /// Returns the IOVA base of this mapping + pub(crate) fn iova(&self) -> u64 { + self.0.start() + } + + /// Returns the size of this mapping in bytes + pub(crate) fn size(&self) -> usize { + self.0.mapped_size + } + + /// Returns the IOVA base of this mapping + pub(crate) fn iova_range(&self) -> Range { + self.0.start()..(self.0.start() + self.0.mapped_size as u64) + } + + /// Remap a cached mapping as uncached, then synchronously flush that range of VAs from the + /// coprocessor cache. This is required to safely unmap cached/private mappings. + fn remap_uncached_and_flush(&mut self) { + let mut owner = self + .0 + .owner + .exec_lock(None, false) + .expect("Failed to exec_lock in remap_uncached_and_flush"); + + mod_dev_dbg!( + owner.dev, + "MMU: remap as uncached {:#x}:{:#x}\n", + self.iova(), + self.size() + ); + + // Remap in-place as uncached. + // Do not try to unmap the guard page (-1) + let prot = self.0.prot.as_uncached(); + if owner + .page_table + .reprot_pages(self.iova_range(), prot) + .is_err() + { + dev_err!( + owner.dev.as_ref(), + "MMU: remap {:#x}:{:#x} failed\n", + self.iova(), + self.size() + ); + } + fence(Ordering::SeqCst); + + // If we don't have (and have never had) a VM slot, just return + let slot = match owner.slot() { + None => return, + Some(slot) => slot, + }; + + let flush_slot = if owner.is_kernel { + // If this is a kernel mapping, always flush on index 64 + UAT_NUM_CTX as u32 + } else { + // Otherwise, check if this slot is the active one, otherwise return + // Also check that we actually own this slot + let ttb = owner.ttb() | TTBR_VALID | (slot as u64) << TTBR_ASID_SHIFT; + + let uat_inner = self.0.uat_inner.lock(); + uat_inner.handoff().lock(); + let cur_slot = uat_inner.handoff().current_slot(); + let ttb_cur = uat_inner.ttbs()[slot as usize].ttb0.load(Ordering::Relaxed); + uat_inner.handoff().unlock(); + if cur_slot == Some(slot) && ttb_cur == ttb { + slot + } else { + return; + } + }; + + // FIXME: There is a race here, though it'll probably never happen in practice. + // In theory, it's possible for the ASC to finish using our slot, whatever command + // it was processing to complete, the slot to be lost to another context, and the ASC + // to begin using it again with a different page table, thus faulting when it gets a + // flush request here. In practice, the chance of this happening is probably vanishingly + // small, as all 62 other slots would have to be recycled or in use before that slot can + // be reused, and the ASC using user contexts at all is very rare. + + // Still, the locking around UAT/Handoff/TTBs should probably be redesigned to better + // model the interactions with the firmware and avoid these races. + // Possibly TTB changes should be tied to slot locks: + + // Flush: + // - Can early check handoff here (no need to lock). + // If user slot and it doesn't match the active ASC slot, + // we can elide the flush as the ASC guarantees it flushes + // TLBs/caches when it switches context. We just need a + // barrier to ensure ordering. + // - Lock TTB slot + // - If user ctx: + // - Lock handoff AP-side + // - Lock handoff dekker + // - Check TTB & handoff cur ctx + // - Perform flush if necessary + // - This implies taking the fwring lock + // + // TTB change: + // - lock TTB slot + // - lock handoff AP-side + // - lock handoff dekker + // change TTB + + // Lock this flush slot, and write the range to it + let flush = self.0.uat_inner.lock_flush(flush_slot); + let pages = self.size() >> UAT_PGBIT; + flush.begin_flush(self.iova(), self.size() as u64); + if pages >= 0x10000 { + dev_err!( + owner.dev.as_ref(), + "MMU: Flush too big ({:#x} pages))\n", + pages + ); + } + + let cmd = fw::channels::FwCtlMsg { + addr: fw::types::U64(self.iova()), + unk_8: 0, + slot: flush_slot, + page_count: pages as u16, + unk_12: 2, // ? + }; + + // Tell the firmware to do a cache flush + if let Err(e) = (*owner.dev).gpu.fwctl(cmd) { + dev_err!( + owner.dev.as_ref(), + "MMU: ASC cache flush {:#x}:{:#x} failed (err: {:?})\n", + self.iova(), + self.size(), + e + ); + } + + // Finish the flush + flush.end_flush(); + + // Slot is unlocked here + } +} +no_debug!(KernelMapping); + +impl Drop for KernelMapping { + fn drop(&mut self) { + // This is the main unmap function for UAT mappings. + // The sequence of operations here is finicky, due to the interaction + // between cached GFX ASC mappings and the page tables. These mappings + // always have to be flushed from the cache before being unmapped. + + // For uncached mappings, just unmapping and flushing the TLB is sufficient. + + // For cached mappings, this is the required sequence: + // 1. Remap it as uncached + // 2. Flush the TLB range + // 3. If kernel VA mapping OR user VA mapping and handoff.current_slot() == slot: + // a. Take a lock for this slot + // b. Write the flush range to the right context slot in handoff area + // c. Issue a cache invalidation request via FwCtl queue + // d. Poll for completion via queue + // e. Check for completion flag in the handoff area + // f. Drop the lock + // 4. Unmap + // 5. Flush the TLB range again + + if self.0.prot.is_cached_noncoherent() { + mod_pr_debug!( + "MMU: remap as uncached {:#x}:{:#x}\n", + self.iova(), + self.size() + ); + self.remap_uncached_and_flush(); + } + + let mut owner = self + .0 + .owner + .exec_lock(None, false) + .expect("exec_lock failed in KernelMapping::drop"); + mod_dev_dbg!( + owner.dev, + "MMU: unmap {:#x}:{:#x}\n", + self.iova(), + self.size() + ); + + if owner.page_table.unmap_pages(self.iova_range()).is_err() { + dev_err!( + owner.dev.as_ref(), + "MMU: unmap {:#x}:{:#x} failed\n", + self.iova(), + self.size() + ); + } + + if let Some(asid) = owner.slot() { + fence(Ordering::SeqCst); + mem::tlbi_range(asid as u8, self.iova() as usize, self.size()); + mod_dev_dbg!( + owner.dev, + "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", + asid, + self.iova(), + self.size() + ); + mem::sync(); + } + } +} + +/// Shared UAT global data structures +struct UatShared { + kernel_ttb1: u64, + map_kernel_to_user: bool, + handoff_rgn: UatRegion, + ttbs_rgn: UatRegion, +} + +impl UatShared { + /// Returns the handoff region area + fn handoff(&self) -> &Handoff { + // SAFETY: pointer is non-null per the type invariant + unsafe { (self.handoff_rgn.map.ptr() as *mut Handoff).as_ref() }.unwrap() + } + + /// Returns the TTBAT area + fn ttbs(&self) -> &[SlotTTBS; UAT_NUM_CTX] { + // SAFETY: pointer is non-null per the type invariant + unsafe { (self.ttbs_rgn.map.ptr() as *mut [SlotTTBS; UAT_NUM_CTX]).as_ref() }.unwrap() + } +} + +// SAFETY: Nothing here is unsafe to send across threads. +unsafe impl Send for UatShared {} + +/// Inner data for the top-level UAT instance. +#[pin_data] +struct UatInner { + #[pin] + shared: Mutex, + #[pin] + handoff_flush: [Mutex; UAT_NUM_CTX + 1], +} + +impl UatInner { + /// Take the lock on the shared data and return the guard. + fn lock(&self) -> Guard<'_, UatShared, MutexBackend> { + self.shared.lock() + } + + /// Take a lock on a handoff flush slot and return the guard. + fn lock_flush(&self, slot: u32) -> Guard<'_, HandoffFlush, MutexBackend> { + self.handoff_flush[slot as usize].lock() + } +} + +/// Top-level UAT manager object +pub(crate) struct Uat { + dev: driver::AsahiDevRef, + cfg: &'static hw::HwConfig, + + inner: Arc, + slots: slotalloc::SlotAllocator, + + kernel_vm: Vm, + kernel_lower_vm: Vm, +} + +impl Handoff { + /// Lock the handoff region from firmware access + fn lock(&self) { + self.lock_ap.store(1, Ordering::Relaxed); + fence(Ordering::SeqCst); + + while self.lock_fw.load(Ordering::Relaxed) != 0 { + if self.turn.load(Ordering::Relaxed) != 0 { + self.lock_ap.store(0, Ordering::Relaxed); + while self.turn.load(Ordering::Relaxed) != 0 {} + self.lock_ap.store(1, Ordering::Relaxed); + fence(Ordering::SeqCst); + } + } + fence(Ordering::Acquire); + } + + /// Unlock the handoff region, allowing firmware access + fn unlock(&self) { + self.turn.store(1, Ordering::Relaxed); + self.lock_ap.store(0, Ordering::Release); + } + + /// Returns the current Vm slot mapped by the firmware for lower/unprivileged access, if any. + fn current_slot(&self) -> Option { + let slot = self.cur_slot.load(Ordering::Relaxed); + if slot == 0 || slot == u32::MAX { + None + } else { + Some(slot) + } + } + + /// Initialize the handoff region + fn init(&self) -> Result { + self.magic_ap.store(PPL_MAGIC, Ordering::Relaxed); + self.cur_slot.store(0, Ordering::Relaxed); + self.unk3.store(0, Ordering::Relaxed); + fence(Ordering::SeqCst); + + let start = Instant::::now(); + const TIMEOUT: Delta = Delta::from_millis(1000); + + self.lock(); + while start.elapsed() < TIMEOUT { + if self.magic_fw.load(Ordering::Relaxed) == PPL_MAGIC { + break; + } else { + self.unlock(); + fsleep(Delta::from_millis(10)); + self.lock(); + } + } + + if self.magic_fw.load(Ordering::Relaxed) != PPL_MAGIC { + self.unlock(); + pr_err!("Handoff: Failed to initialize (firmware not running?)\n"); + return Err(EIO); + } + + self.unlock(); + + for i in 0..=UAT_NUM_CTX { + self.flush[i].state.store(0, Ordering::Relaxed); + self.flush[i].addr.store(0, Ordering::Relaxed); + self.flush[i].size.store(0, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + Ok(()) + } +} + +/// Represents a single flush info slot in the handoff region. +/// +/// # Invariants +/// The pointer is valid and there is no aliasing HandoffFlush instance. +struct HandoffFlush(*const FlushInfo); + +// SAFETY: These pointers are safe to send across threads. +unsafe impl Send for HandoffFlush {} + +impl HandoffFlush { + /// Set up a flush operation for the coprocessor + fn begin_flush(&self, start: u64, size: u64) { + // SAFETY: Per the type invariant, this is safe + let flush = unsafe { self.0.as_ref().unwrap() }; + + let state = flush.state.load(Ordering::Relaxed); + if state != 0 { + pr_err!("Handoff: expected flush state 0, got {}\n", state); + } + flush.addr.store(start, Ordering::Relaxed); + flush.size.store(size, Ordering::Relaxed); + flush.state.store(1, Ordering::Relaxed); + } + + /// Complete a flush operation for the coprocessor + fn end_flush(&self) { + // SAFETY: Per the type invariant, this is safe + let flush = unsafe { self.0.as_ref().unwrap() }; + let state = flush.state.load(Ordering::Relaxed); + if state != 2 { + pr_err!("Handoff: expected flush state 2, got {}\n", state); + } + flush.state.store(0, Ordering::Relaxed); + } +} + +impl Vm { + /// Create a new virtual memory address space + fn new( + dev: &driver::AsahiDevice, + uat_inner: Arc, + kernel_range: Range, + cfg: &'static hw::HwConfig, + ttb: Option, + id: u64, + ) -> Result { + let dummy_obj = gem::new_kernel_object(dev, UAT_PGSZ)?; + let is_kernel = ttb.is_some(); + + let page_table = if let Some(ttb) = ttb { + UatPageTable::new_with_ttb(ttb, IOVA_KERN_RANGE, cfg.uat_oas)? + } else { + UatPageTable::new(cfg.uat_oas)? + }; + + let (va_range, gpuvm_range) = if is_kernel { + (IOVA_KERN_RANGE, kernel_range.clone()) + } else { + (IOVA_USER_RANGE, IOVA_USER_USABLE_RANGE) + }; + + let mm = mm::Allocator::new(va_range.start, va_range.range(), ())?; + + let binding = Arc::pin_init( + new_mutex!( + VmBinding { + binding: None, + bind_token: None, + active_users: 0, + ttb: page_table.ttb(), + }, + "VmBinding", + ), + GFP_KERNEL, + )?; + + let binding_clone = binding.clone(); + Ok(Vm { + id, + dummy_obj: dummy_obj.gem.clone(), + inner: gpuvm::GpuVm::new( + c_str!("Asahi::GpuVm"), + // TODO: should we using DRM_GPUVM_RESV_PROTECTED as well? + drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE, + dev, + dummy_obj.gem.clone(), + gpuvm_range, + kernel_range, + init!(VmInner { + dev: dev.into(), + va_range, + is_kernel, + page_table, + mm, + uat_inner, + binding: binding_clone, + id, + }), + )?, + binding, + }) + } + + /// Get the translation table base for this Vm + fn ttb(&self) -> u64 { + self.binding.lock().ttb + } + + /// Map a GEM object (using its `SGTable`) into this Vm at a free address in a given range. + #[allow(clippy::too_many_arguments)] + pub(crate) fn map_in_range( + &self, + gem: &gem::Object, + object_range: Range, + alignment: u64, + range: Range, + prot: Prot, + guard: bool, + ) -> Result { + let size = object_range.range(); + let sgt = gem.owned_sg_table()?; + let mut inner = self.inner.exec_lock(Some(gem), false)?; + let vm_bo = self.inner.obtain_bo(gem)?; + + let mut vm_bo_guard = vm_bo.inner().sgt.lock(); + if vm_bo_guard.is_none() { + vm_bo_guard.replace(sgt); + } + core::mem::drop(vm_bo_guard); + + let uat_inner = inner.uat_inner.clone(); + let node = inner.mm.insert_node_in_range( + KernelMappingInner { + owner: self.inner.clone(), + uat_inner, + prot, + bo: Some(vm_bo), + _gem: Some(gem.into()), + offset: object_range.start, + mapped_size: size, + }, + (size + if guard { UAT_PGSZ } else { 0 }) as u64, // Add guard page + alignment, + 0, + range.start, + range.end, + mm::InsertMode::Best, + )?; + + let ret = inner.map_node(&node, prot); + // Drop the exec_lock first, so that if map_node failed the + // KernelMappingInner destructur does not deadlock. + core::mem::drop(inner); + ret?; + Ok(KernelMapping(node)) + } + + /// Map a GEM object into this Vm at a specific address. + #[allow(clippy::too_many_arguments)] + pub(crate) fn map_at( + &self, + addr: u64, + size: usize, + gem: ARef, + prot: Prot, + guard: bool, + ) -> Result { + let sgt = gem.owned_sg_table()?; + let mut inner = self.inner.exec_lock(Some(&gem), false)?; + + let vm_bo = self.inner.obtain_bo(&gem)?; + + let mut vm_bo_guard = vm_bo.inner().sgt.lock(); + if vm_bo_guard.is_none() { + vm_bo_guard.replace(sgt); + } + core::mem::drop(vm_bo_guard); + + let uat_inner = inner.uat_inner.clone(); + let node = inner.mm.reserve_node( + KernelMappingInner { + owner: self.inner.clone(), + uat_inner, + prot, + bo: Some(vm_bo), + _gem: Some(gem.clone()), + offset: 0, + mapped_size: size, + }, + addr, + (size + if guard { UAT_PGSZ } else { 0 }) as u64, // Add guard page + 0, + )?; + + let ret = inner.map_node(&node, prot); + // Drop the exec_lock first, so that if map_node failed the + // KernelMappingInner destructur does not deadlock. + core::mem::drop(inner); + ret?; + Ok(KernelMapping(node)) + } + + /// Map a range of a GEM object into this Vm using GPUVM. + #[allow(clippy::too_many_arguments)] + pub(crate) fn bind_object( + &self, + gem: &gem::Object, + addr: u64, + size: u64, + offset: u64, + prot: Prot, + single_page: bool, + ) -> Result { + // Mapping needs a complete context + let mut ctx = StepContext { + new_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + prev_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + next_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + prot, + ..Default::default() + }; + + let sgt = gem.owned_sg_table()?; + let mut inner = self.inner.exec_lock(Some(gem), true)?; + + // Preallocate the page tables, to fail early if we ENOMEM + inner.page_table.alloc_pages(addr..(addr + size))?; + + let vm_bo = self.inner.obtain_bo(gem)?; + + let mut vm_bo_guard = vm_bo.inner().sgt.lock(); + if vm_bo_guard.is_none() { + vm_bo_guard.replace(sgt); + } + core::mem::drop(vm_bo_guard); + + ctx.vm_bo = Some(vm_bo); + + if (addr | size | offset) & (UAT_PGMSK as u64) != 0 { + dev_err!( + inner.dev.as_ref(), + "MMU: Map step {:#x} [{:#x}] -> {:#x} is not page-aligned\n", + offset, + size, + addr + ); + return Err(EINVAL); + } + + let (flags, gem_range) = if single_page { + (gpuvm::GpuVaFlags::REPEAT, UAT_PGSZ as u32) + } else { + (gpuvm::GpuVaFlags::NONE, 0u32) + }; + + mod_dev_dbg!( + inner.dev, + "MMU: sm_map: {:#x} [{:#x}] -> {:#x}\n", + offset, + size, + addr + ); + inner.sm_map(&mut ctx, addr, size, offset, gem_range, flags) + } + + /// Add a direct MMIO mapping to this Vm at a free address. + pub(crate) fn map_io( + &self, + iova: u64, + phys: usize, + size: usize, + prot: Prot, + ) -> Result { + let mut inner = self.inner.exec_lock(None, false)?; + + if (iova as usize | phys | size) & UAT_PGMSK != 0 { + dev_err!( + inner.dev.as_ref(), + "MMU: KernelMapping {:#x}:{:#x} -> {:#x} is not page-aligned\n", + phys, + size, + iova + ); + return Err(EINVAL); + } + + dev_info!( + inner.dev.as_ref(), + "MMU: IO map: {:#x}:{:#x} -> {:#x}\n", + phys, + size, + iova + ); + + let uat_inner = inner.uat_inner.clone(); + let node = inner.mm.reserve_node( + KernelMappingInner { + owner: self.inner.clone(), + uat_inner, + prot, + bo: None, + _gem: None, + offset: 0, + mapped_size: size, + }, + iova, + size as u64, + 0, + )?; + + let ret = inner.page_table.map_pages( + iova..(iova + size as u64), + phys as PhysicalAddr, + prot, + false, + ); + // Drop the exec_lock first, so that if map_node failed the + // KernelMappingInner destructur does not deadlock. + core::mem::drop(inner); + ret?; + Ok(KernelMapping(node)) + } + + /// Unmap everything in an address range. + pub(crate) fn unmap_range(&self, iova: u64, size: u64) -> Result { + // Unmapping a range can only do a single split, so just preallocate + // the prev and next GpuVas + let mut ctx = StepContext { + prev_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + next_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + ..Default::default() + }; + + let mut inner = self.inner.exec_lock(None, false)?; + + mod_dev_dbg!(inner.dev, "MMU: sm_unmap: {:#x}:{:#x}\n", iova, size); + inner.sm_unmap(&mut ctx, iova, size) + } + + /// Drop mappings for a given bo. + pub(crate) fn drop_mappings(&self, gem: &gem::Object) -> Result { + // Removing whole mappings only does unmaps, so no preallocated VAs + let mut ctx = Default::default(); + + let inner = self.inner.exec_lock(Some(gem), false)?; + + if let Some(bo) = self.inner.find_bo(gem) { + mod_dev_dbg!(inner.dev, "MMU: bo_unmap\n"); + self.inner.bo_unmap(&mut ctx, &bo)?; + mod_dev_dbg!(inner.dev, "MMU: bo_unmap done\n"); + // We need to drop the exec_lock first, then the GpuVmBo since that will take the lock itself. + core::mem::drop(inner); + core::mem::drop(bo); + } + + Ok(()) + } + + /// Returns the dummy GEM object used to hold the shared DMA reservation locks + pub(crate) fn get_resv_obj(&self) -> ARef { + self.dummy_obj.clone() + } + + /// Check whether an object is external to this GpuVm + pub(crate) fn is_extobj(&self, gem: &gem::Object) -> bool { + self.inner.is_extobj(gem) + } + + /// Check whether an object is external to this GpuVm + pub(crate) fn bo_deferred_cleanup(&self) { + self.inner.bo_deferred_cleanup() + } +} + +impl Drop for VmInner { + fn drop(&mut self) { + let mut binding = self.binding.lock(); + assert_eq!(binding.active_users, 0); + + mod_pr_debug!( + "VmInner::Drop [{}]: bind_token={:?}\n", + self.id, + binding.bind_token + ); + + // Make sure this VM is not mapped to a TTB if it was + if let Some(token) = binding.bind_token.take() { + let idx = (token.last_slot() as usize) + UAT_USER_CTX_START; + let ttb = self.ttb() | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; + + let uat_inner = self.uat_inner.lock(); + uat_inner.handoff().lock(); + let handoff_cur = uat_inner.handoff().current_slot(); + let ttb_cur = uat_inner.ttbs()[idx].ttb0.load(Ordering::SeqCst); + let inval = ttb_cur == ttb; + if inval { + if handoff_cur == Some(idx as u32) { + pr_err!( + "VmInner::drop owning slot {}, but it is currently in use by the ASC?\n", + idx + ); + } + uat_inner.ttbs()[idx].ttb0.store(0, Ordering::SeqCst); + uat_inner.ttbs()[idx].ttb1.store(0, Ordering::SeqCst); + } + uat_inner.handoff().unlock(); + core::mem::drop(uat_inner); + + // In principle we dropped all the KernelMappings already, but we might as + // well play it safe and invalidate the whole ASID. + if inval { + mod_pr_debug!( + "VmInner::Drop [{}]: need inval for ASID {:#x}\n", + self.id, + idx + ); + mem::tlbi_asid(idx as u8); + mem::sync(); + } + } + } +} + +impl Uat { + /// Map a bootloader-preallocated memory region + fn map_region( + dev: &device::Device, + name: &CStr, + size: usize, + cached: bool, + ) -> Result { + let of_node = dev.of_node().ok_or(EINVAL)?; + let res = of_node.reserved_mem_region_to_resource_byname(name)?; + let base = res.start(); + let res_size = res.size().try_into()?; + + if size > res_size { + dev_err!( + dev, + "Region {} is too small (expected {}, got {})\n", + name, + size, + res_size + ); + return Err(ENOMEM); + } + + let flags = if cached { + io::mem::MemFlag::WB + } else { + io::mem::MemFlag::WC + }; + + // SAFETY: The safety of this operation hinges on the correctness of + // much of this file and also the `pgtable` module, so it is difficult + // to prove in a single safety comment. Such is life with raw GPU + // page table management... + let map = unsafe { io::mem::Mem::try_new(res, flags.into()) }.inspect_err(|_| { + dev_err!(dev, "Failed to remap {} mem resource\n", name); + })?; + + Ok(UatRegion { base, map }) + } + + /// Returns a reference to the global kernel (upper half) `Vm` + pub(crate) fn kernel_vm(&self) -> &Vm { + &self.kernel_vm + } + + /// Returns a reference to the local kernel (lower half) `Vm` + pub(crate) fn kernel_lower_vm(&self) -> &Vm { + &self.kernel_lower_vm + } + + #[cfg(CONFIG_DEV_COREDUMP)] + pub(crate) fn dump_kernel_pages(&self) -> Result> { + let mut inner = self.kernel_vm.inner.exec_lock(None, false)?; + inner.page_table.dump_pages(IOVA_KERN_FULL_RANGE) + } + + /// Returns the base physical address of the TTBAT region. + pub(crate) fn ttb_base(&self) -> u64 { + let inner = self.inner.lock(); + + inner.ttbs_rgn.base + } + + /// Binds a `Vm` to a slot, preferring the last used one. + pub(crate) fn bind(&self, vm: &Vm) -> Result { + let mut binding = vm.binding.lock(); + + if binding.binding.is_none() { + assert_eq!(binding.active_users, 0); + + let isolation = *module_parameters::robust_isolation.value() != 0; + + self.slots.set_limit(if isolation { + NonZeroUsize::new(1) + } else { + None + }); + + let slot = self.slots.get(binding.bind_token)?; + if slot.changed() { + mod_pr_debug!("Vm Bind [{}]: bind_token={:?}\n", vm.id, slot.token(),); + let idx = (slot.slot() as usize) + UAT_USER_CTX_START; + let ttb = binding.ttb | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; + + let uat_inner = self.inner.lock(); + + let ttb1 = if uat_inner.map_kernel_to_user { + uat_inner.kernel_ttb1 | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT + } else { + 0 + }; + + let ttbs = uat_inner.ttbs(); + uat_inner.handoff().lock(); + if uat_inner.handoff().current_slot() == Some(idx as u32) { + pr_err!( + "Vm::bind to slot {}, but it is currently in use by the ASC?\n", + idx + ); + } + ttbs[idx].ttb0.store(ttb, Ordering::Release); + ttbs[idx].ttb1.store(ttb1, Ordering::Release); + uat_inner.handoff().unlock(); + core::mem::drop(uat_inner); + + // Make sure all TLB entries from the previous owner of this ASID are gone + mem::tlbi_asid(idx as u8); + mem::sync(); + } + + binding.bind_token = Some(slot.token()); + binding.binding = Some(slot); + } + + binding.active_users += 1; + + let slot = binding.binding.as_ref().unwrap().slot() + UAT_USER_CTX_START as u32; + mod_pr_debug!("MMU: slot {} active users {}\n", slot, binding.active_users); + Ok(VmBind(vm.clone(), slot)) + } + + /// Creates a new `Vm` linked to this UAT. + pub(crate) fn new_vm(&self, id: u64, kernel_range: Range) -> Result { + Vm::new( + &self.dev, + self.inner.clone(), + kernel_range, + self.cfg, + None, + id, + ) + } + + /// Creates the reference-counted inner data for a new `Uat` instance. + #[inline(never)] + fn make_inner(dev: &driver::AsahiDevice) -> Result> { + let handoff_rgn = Self::map_region(dev.as_ref(), c_str!("handoff"), HANDOFF_SIZE, true)?; + let ttbs_rgn = Self::map_region(dev.as_ref(), c_str!("ttbs"), SLOTS_SIZE, true)?; + + // SAFETY: The Handoff struct layout matches the firmware's view of memory at this address, + // and the region is at least large enough per the size specified above. + let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; + + dev_info!(dev.as_ref(), "MMU: Initializing kernel page table\n"); + + Arc::pin_init( + try_pin_init!(UatInner { + handoff_flush <- pin_init::pin_init_array_from_fn(|i| { + new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") + }), + shared <- new_mutex!( + UatShared { + kernel_ttb1: 0, + map_kernel_to_user: false, + handoff_rgn, + ttbs_rgn, + }, + "uat_shared" + ), + }), + GFP_KERNEL, + ) + } + + /// Creates a new `Uat` instance given the relevant hardware config. + #[inline(never)] + pub(crate) fn new( + dev: &driver::AsahiDevice, + cfg: &'static hw::HwConfig, + map_kernel_to_user: bool, + ) -> Result { + dev_info!(dev.as_ref(), "MMU: Initializing...\n"); + + let inner = Self::make_inner(dev)?; + + let of_node = dev.as_ref().of_node().ok_or(EINVAL)?; + let res = of_node.reserved_mem_region_to_resource_byname(c_str!("pagetables"))?; + let ttb1 = res.start(); + let ttb1size: usize = res.size().try_into()?; + + if ttb1size < PAGETABLES_SIZE { + dev_err!(dev.as_ref(), "MMU: Pagetables region is too small\n"); + return Err(ENOMEM); + } + + dev_info!(dev.as_ref(), "MMU: Creating kernel page tables\n"); + let kernel_lower_vm = Vm::new(dev, inner.clone(), IOVA_USER_RANGE, cfg, None, 1)?; + let kernel_vm = Vm::new(dev, inner.clone(), IOVA_KERN_RANGE, cfg, Some(ttb1), 0)?; + + dev_info!(dev.as_ref(), "MMU: Kernel page tables created\n"); + + let ttb0 = kernel_lower_vm.ttb(); + + let uat = Self { + dev: dev.into(), + cfg, + kernel_vm, + kernel_lower_vm, + inner, + slots: slotalloc::SlotAllocator::new( + UAT_USER_CTX as u32, + (), + |_inner, _slot| Some(SlotInner()), + c_str!("Uat::SlotAllocator"), + static_lock_class!(), + static_lock_class!(), + )?, + }; + + let mut inner = uat.inner.lock(); + + inner.map_kernel_to_user = map_kernel_to_user; + inner.kernel_ttb1 = ttb1; + + inner.handoff().init()?; + + dev_info!(dev.as_ref(), "MMU: Initializing TTBs\n"); + + inner.handoff().lock(); + + let ttbs = inner.ttbs(); + + ttbs[0].ttb0.store(ttb0 | TTBR_VALID, Ordering::SeqCst); + ttbs[0].ttb1.store(ttb1 | TTBR_VALID, Ordering::SeqCst); + + for ctx in &ttbs[1..] { + ctx.ttb0.store(0, Ordering::Relaxed); + ctx.ttb1.store(0, Ordering::Relaxed); + } + + inner.handoff().unlock(); + + core::mem::drop(inner); + + dev_info!(dev.as_ref(), "MMU: initialized\n"); + + Ok(uat) + } +} + +impl Drop for Uat { + fn drop(&mut self) { + // Make sure we flush the TLBs + fence(Ordering::SeqCst); + mem::tlbi_all(); + mem::sync(); + } +} diff --git a/drivers/gpu/drm/asahi/object.rs b/drivers/gpu/drm/asahi/object.rs new file mode 100644 index 00000000000000..38a2268137effb --- /dev/null +++ b/drivers/gpu/drm/asahi/object.rs @@ -0,0 +1,733 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Asahi GPU object model +//! +//! The AGX GPU includes a coprocessor that uses a large number of shared memory structures to +//! communicate with the driver. These structures contain GPU VA pointers to each other, which are +//! directly dereferenced by the firmware and are expected to always be valid for the usage +//! lifetime of the containing struct (which is an implicit contract, not explicitly managed). +//! Any faults cause an unrecoverable firmware crash, requiring a full system reboot. +//! +//! In order to manage this complexity safely, we implement a GPU object model using Rust's type +//! system to enforce GPU object lifetime relationships. GPU objects represent an allocated piece +//! of memory of a given type, mapped to the GPU (and usually also the CPU). On the CPU side, +//! these objects are associated with a pure Rust structure that contains the objects it depends +//! on (or references to them). This allows us to map Rust lifetimes into the GPU object model +//! system. Then, GPU VA pointers also inherit those lifetimes, which means the Rust borrow checker +//! can ensure that all pointers are assigned an address that is guaranteed to outlive the GPU +//! object it points to. +//! +//! Since the firmware object model does have self-referencing pointers (and there is of course no +//! underlying revocability mechanism to make it safe), we must have an escape hatch. GPU pointers +//! can be weak pointers, which do not enforce lifetimes. In those cases, it is the user's +//! responsibility to ensure that lifetime requirements are met. +//! +//! In other words, the model is necessarily leaky and there is no way to fully map Rust safety to +//! GPU firmware object safety. The goal of the model is to make it easy to model the lifetimes of +//! GPU objects and have the compiler help in avoiding mistakes, rather than to guarantee safety +//! 100% of the time as would be the case for CPU-side Rust code. + +// TODO: There is a fundamental soundness issue with sharing memory with the GPU (that even affects +// C code too). Since the GPU is free to mutate that memory at any time, normal reference invariants +// cannot be enforced on the CPU side. For example, the compiler could perform an optimization that +// assumes that a given memory location does not change between two reads, and causes UB otherwise, +// and then the GPU could mutate that memory out from under the CPU. +// +// For cases where we *expect* this to happen, we use atomic types, which avoid this issue. However, +// doing so for every single field of every type is a non-starter. Right now, there seems to be no +// good solution for this that does not come with significant performance or ergonomics downsides. +// +// In *practice* we are almost always only writing GPU memory, and only reading from atomics, so the +// chances of this actually triggering UB (e.g. a security issue that can be triggered from the GPU +// side) due to a compiler optimization are very slim. +// +// Further discussion: https://github.com/rust-lang/unsafe-code-guidelines/issues/152 + +use kernel::{ + error::code::*, + prelude::*, + sync::Arc, // +}; + +use core::fmt; +use core::fmt::Debug; +use core::fmt::Formatter; +use core::marker::PhantomData; +use core::mem::MaybeUninit; +use core::num::NonZeroU64; +use core::ops::{ + Deref, + DerefMut, + Index, + IndexMut, // +}; +use core::{mem, ptr, slice}; + +use crate::alloc::Allocation; +use crate::debug::*; +use crate::fw::types::Zeroable; +use crate::mmu; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Object; + +/// A GPU-side strong pointer, which is a 64-bit non-zero VA with an associated lifetime. +/// +/// In rare cases these pointers are not aligned, so this is `packed(1)`. +#[repr(C, packed(1))] +pub(crate) struct GpuPointer<'a, T: ?Sized>(NonZeroU64, PhantomData<&'a T>); + +impl<'a, T: ?Sized> GpuPointer<'a, T> { + /// Logical OR the pointer with an arbitrary `u64`. This is used when GPU struct fields contain + /// misc flag fields in the upper bits. The lifetime is retained. This is GPU-unsafe in + /// principle, but we assert that only non-implemented address bits are touched, which is safe + /// for pointers used by the GPU (not by firmware). + pub(crate) fn or(&self, other: u64) -> GpuPointer<'a, T> { + // This will fail for kernel-half pointers, which should not be ORed. + assert_eq!(self.0.get() & other, 0); + // Assert that we only touch the high bits. + assert_eq!(other & 0xffffffffff, 0); + GpuPointer(self.0 | other, PhantomData) + } + + /// Add an arbitrary offset to the pointer. This is not safe (from the GPU perspective), and + /// should only be used via the `inner_ptr` macro to get pointers to inner fields, hence we mark + /// it `unsafe` to discourage direct use. + /// + /// # Safety + /// Do not use directly, only via `inner_ptr`. + // NOTE: The third argument is a type inference hack. + pub(crate) unsafe fn offset(&self, off: usize, _: *const U) -> GpuPointer<'a, U> { + GpuPointer::<'a, U>( + NonZeroU64::new(self.0.get() + (off as u64)).unwrap(), + PhantomData, + ) + } +} + +impl<'a, T> GpuPointer<'a, T> { + /// Create a GPU pointer from a KernelMapping and an offset. + /// TODO: Change all GPU pointers to point to the raw types so size_of here is GPU-sound. + pub(crate) fn from_mapping( + mapping: &'a Arc, + offset: usize, + ) -> Result> { + let addr = mapping.iova().checked_add(offset as u64).ok_or(EINVAL)?; + let end = offset + .checked_add(core::mem::size_of::()) + .ok_or(EINVAL)?; + if end > mapping.size() { + Err(ERANGE) + } else { + Ok(Self(addr.try_into().unwrap(), PhantomData)) + } + } +} + +impl<'a, T: ?Sized> Debug for GpuPointer<'a, T> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let val = self.0; + f.write_fmt(format_args!("{:#x} ({})", val, core::any::type_name::())) + } +} + +impl<'a, T: ?Sized> From> for u64 { + fn from(value: GpuPointer<'a, T>) -> Self { + value.0.get() + } +} + +/// Take a pointer to a sub-field within a structure pointed to by a GpuPointer, keeping the +/// lifetime. +#[macro_export] +macro_rules! inner_ptr { + ($gpuva:expr, $($f:tt)*) => ({ + // This mirrors kernel::offset_of(), except we use type inference to avoid having to know + // the type of the pointer explicitly. + fn uninit_from(_: GpuPointer<'_, T>) -> core::mem::MaybeUninit> { + core::mem::MaybeUninit::uninit() + } + let tmp = uninit_from($gpuva); + let outer = tmp.as_ptr(); + // SAFETY: The pointer is valid and aligned, just not initialised; `addr_of` ensures that + // we don't actually read from `outer` (which would be UB) nor create an intermediate + // reference. + let p: *const _ = unsafe { core::ptr::addr_of!((*outer).$($f)*) }; + let inner = p as *const u8; + // SAFETY: The two pointers are within the same allocation block. + let off = unsafe { inner.offset_from(outer as *const u8) }; + // SAFETY: The resulting pointer is guaranteed to point to valid memory within the outer + // object. + unsafe { $gpuva.offset(off.try_into().unwrap(), p) } + }) +} + +/// A GPU-side weak pointer, which is a 64-bit non-zero VA with no lifetime. +/// +/// In rare cases these pointers are not aligned, so this is `packed(1)`. +#[repr(C, packed(1))] +pub(crate) struct GpuWeakPointer(NonZeroU64, PhantomData<*const T>); + +/// SAFETY: GPU weak pointers are always safe to share between threads. +unsafe impl Send for GpuWeakPointer {} +/// SAFETY: GPU weak pointers are always safe to share between threads. +unsafe impl Sync for GpuWeakPointer {} + +// Weak pointers can be copied/cloned regardless of their target type. +impl Copy for GpuWeakPointer {} + +impl Clone for GpuWeakPointer { + fn clone(&self) -> Self { + *self + } +} + +impl GpuWeakPointer { + /// Add an arbitrary offset to the pointer. This is not safe (from the GPU perspective), and + /// should only be used via the `inner_weak_ptr` macro to get pointers to inner fields, hence we + /// mark it `unsafe` to discourage direct use. + /// + /// # Safety + /// Do not use directly, only via `inner_weak_ptr`. + // NOTE: The third argument is a type inference hack. + pub(crate) unsafe fn offset(&self, off: usize, _: *const U) -> GpuWeakPointer { + GpuWeakPointer::( + NonZeroU64::new(self.0.get() + (off as u64)).unwrap(), + PhantomData, + ) + } + + /// Upgrade a weak pointer into a strong pointer. This is not considered safe from the GPU + /// perspective. + /// + /// # Safety + /// The caller must ensure tht the data pointed to lives in the GPU at least as long as the + /// returned lifetime. + pub(crate) unsafe fn upgrade<'a>(&self) -> GpuPointer<'a, T> { + GpuPointer(self.0, PhantomData) + } +} + +impl From> for u64 { + fn from(value: GpuWeakPointer) -> Self { + value.0.get() + } +} + +impl Debug for GpuWeakPointer { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let val = self.0; + f.write_fmt(format_args!("{:#x} ({})", val, core::any::type_name::())) + } +} + +/// Take a pointer to a sub-field within a structure pointed to by a GpuWeakPointer. +#[macro_export] +macro_rules! inner_weak_ptr { + ($gpuva:expr, $($f:tt)*) => ({ + // See inner_ptr() + fn uninit_from(_: GpuWeakPointer) -> core::mem::MaybeUninit> { + core::mem::MaybeUninit::uninit() + } + let tmp = uninit_from($gpuva); + let outer = tmp.as_ptr(); + // SAFETY: The pointer is valid and aligned, just not initialised; `addr_of` ensures that + // we don't actually read from `outer` (which would be UB) nor create an intermediate + // reference. + let p: *const _ = unsafe { core::ptr::addr_of!((*outer).$($f)*) }; + let inner = p as *const u8; + // SAFETY: The two pointers are within the same allocation block. + let off = unsafe { inner.offset_from(outer as *const u8) }; + // SAFETY: The resulting pointer is guaranteed to point to valid memory within the outer + // object. + unsafe { $gpuva.offset(off.try_into().unwrap(), p) } + }) +} + +/// Types that implement this trait represent a GPU structure from the CPU side. +/// +/// The `Raw` type represents the actual raw structure definition on the GPU side. +/// +/// Types implementing [`GpuStruct`] must have fields owning any objects (or strong references +/// to them) that GPU pointers in the `Raw` structure point to. This mechanism is used to enforce +/// lifetimes. +pub(crate) trait GpuStruct: 'static { + /// The type of the GPU-side structure definition representing the firmware struct layout. + type Raw<'a>; +} + +/// An instance of a GPU object in memory. +/// +/// # Invariants +/// `raw` must point to a valid mapping of the `T::Raw` type associated with the `alloc` allocation. +/// `gpu_ptr` must be the GPU address of the same object. +pub(crate) struct GpuObject> { + raw: *mut T::Raw<'static>, + alloc: U, + gpu_ptr: GpuWeakPointer, + inner: KBox, +} + +impl> GpuObject { + /// Create a new GpuObject given an allocator and the inner data (a type implementing + /// GpuStruct). + /// + /// The caller passes a closure that constructs the `T::Raw` type given a reference to the + /// `GpuStruct`. This is the mechanism used to enforce lifetimes. + pub(crate) fn new( + alloc: U, + inner: T, + callback: impl for<'a> FnOnce(&'a T) -> T::Raw<'a>, + ) -> Result { + let size = mem::size_of::>(); + if size > 0x1000 { + dev_crit!( + alloc.device().as_ref(), + "Allocating {} of size {:#x}, with new, please use new_boxed!\n", + core::any::type_name::(), + size + ); + } + if alloc.size() < size { + return Err(ENOMEM); + } + let gpu_ptr = + GpuWeakPointer::(NonZeroU64::new(alloc.gpu_ptr()).ok_or(EINVAL)?, PhantomData); + mod_dev_dbg!( + alloc.device(), + "Allocating {} @ {:#x}\n", + core::any::type_name::(), + alloc.gpu_ptr() + ); + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut T::Raw<'static>; + let mut raw = callback(&inner); + // SAFETY: `p` is guaranteed to be valid per the Allocation invariant, and the type is + // identical to the type of `raw` other than the lifetime. + unsafe { p.copy_from(&mut raw as *mut _ as *mut u8 as *mut _, 1) }; + mem::forget(raw); + Ok(Self { + raw: p, + gpu_ptr, + alloc, + inner: KBox::new(inner, GFP_KERNEL)?, + }) + } + + /// Create a new GpuObject given an allocator and the boxed inner data (a type implementing + /// GpuStruct). + /// + /// The caller passes a closure that initializes the `T::Raw` type given a reference to the + /// `GpuStruct` and a `MaybeUninit`. This is intended to be used with the place!() + /// macro to avoid constructing the whole `T::Raw` object on the stack. + pub(crate) fn new_boxed( + alloc: U, + inner: KBox, + callback: impl for<'a> FnOnce( + &'a T, + &'a mut MaybeUninit>, + ) -> Result<&'a mut T::Raw<'a>>, + ) -> Result { + if alloc.size() < mem::size_of::>() { + return Err(ENOMEM); + } + let gpu_ptr = + GpuWeakPointer::(NonZeroU64::new(alloc.gpu_ptr()).ok_or(EINVAL)?, PhantomData); + mod_dev_dbg!( + alloc.device(), + "Allocating {} @ {:#x}\n", + core::any::type_name::(), + alloc.gpu_ptr() + ); + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut MaybeUninit>; + // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. + let raw = callback(&inner, unsafe { &mut *p })?; + if p as *mut T::Raw<'_> != raw as *mut _ { + dev_err!( + alloc.device().as_ref(), + "Allocation callback returned a mismatched reference ({})\n", + core::any::type_name::(), + ); + return Err(EINVAL); + } + Ok(Self { + raw: p as *mut u8 as *mut T::Raw<'static>, + gpu_ptr, + alloc, + inner, + }) + } + + /// Create a new GpuObject given an allocator and the inner data (a type implementing + /// GpuStruct). + /// + /// The caller passes a closure that initializes the `T::Raw` type given a reference to the + /// `GpuStruct` and a `MaybeUninit`. This is intended to be used with the place!() + /// macro to avoid constructing the whole `T::Raw` object on the stack. + pub(crate) fn new_inplace( + alloc: U, + inner: T, + callback: impl for<'a> FnOnce( + &'a T, + &'a mut MaybeUninit>, + ) -> Result<&'a mut T::Raw<'a>>, + ) -> Result { + GpuObject::::new_boxed(alloc, KBox::new(inner, GFP_KERNEL)?, callback) + } + + /// Create a new GpuObject given an allocator and the boxed inner data (a type implementing + /// GpuStruct). + /// + /// The caller passes a closure that initializes the `T::Raw` type given a reference to the + /// `GpuStruct` and a `MaybeUninit`. This is intended to be used with the place!() + /// macro to avoid constructing the whole `T::Raw` object on the stack. + pub(crate) fn new_init_prealloc<'a, I: Init, R: PinInit, F>, E, F>( + alloc: U, + inner_init: impl FnOnce(GpuWeakPointer) -> I, + raw_init: impl FnOnce(&'a T, GpuWeakPointer) -> R, + ) -> Result + where + kernel::error::Error: core::convert::From, + kernel::error::Error: core::convert::From, + { + if alloc.size() < mem::size_of::>() { + return Err(ENOMEM); + } + let gpu_ptr = + GpuWeakPointer::(NonZeroU64::new(alloc.gpu_ptr()).ok_or(EINVAL)?, PhantomData); + mod_dev_dbg!( + alloc.device(), + "Allocating {} @ {:#x}\n", + core::any::type_name::(), + alloc.gpu_ptr() + ); + let inner = inner_init(gpu_ptr); + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut T::Raw<'_>; + let ret = Self { + raw: p as *mut u8 as *mut T::Raw<'static>, + gpu_ptr, + alloc, + inner: KBox::init(inner, GFP_KERNEL)?, + }; + let q = &*ret.inner as *const T; + // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. + unsafe { raw_init(&*q, gpu_ptr).__pinned_init(p) }?; + Ok(ret) + } + + /// Returns the GPU VA of this object (as a raw [`NonZeroU64`]) + pub(crate) fn gpu_va(&self) -> NonZeroU64 { + self.gpu_ptr.0 + } + + /// Returns a strong GPU pointer to this object, with a lifetime. + pub(crate) fn gpu_pointer(&self) -> GpuPointer<'_, T> { + GpuPointer(self.gpu_ptr.0, PhantomData) + } + + /// Returns a weak GPU pointer to this object, with no lifetime. + pub(crate) fn weak_pointer(&self) -> GpuWeakPointer { + GpuWeakPointer(self.gpu_ptr.0, PhantomData) + } + + /// Perform a mutation to the inner `Raw` data given a user-supplied callback. + /// + /// The callback gets a mutable reference to the `GpuStruct` type. + pub(crate) fn with_mut( + &mut self, + callback: impl for<'a> FnOnce(&'a mut ::Raw<'a>, &'a mut T) -> RetVal, + ) -> RetVal { + // SAFETY: `self.raw` is valid per the type invariant, and the second half is just + // converting lifetimes. + unsafe { callback(&mut *self.raw, &mut *(&mut *self.inner as *mut _)) } + } + + /// Access the inner `Raw` data given a user-supplied callback. + /// + /// The callback gets a reference to the `GpuStruct` type. + pub(crate) fn with( + &self, + callback: impl for<'a> FnOnce(&'a ::Raw<'a>, &'a T) -> RetVal, + ) -> RetVal { + // SAFETY: `self.raw` is valid per the type invariant, and the second half is just + // converting lifetimes. + unsafe { callback(&*self.raw, &*(&*self.inner as *const _)) } + } +} + +impl> Deref for GpuObject { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl> DerefMut for GpuObject { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +impl> Debug for GpuObject +where + ::Raw<'static>: Debug, +{ + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct(core::any::type_name::()) + // SAFETY: `self.raw` is valid per the type invariant. + .field("raw", &format_args!("{:#X?}", unsafe { &*self.raw })) + .field("inner", &format_args!("{:#X?}", &self.inner)) + .field("alloc", &format_args!("{:?}", &self.alloc)) + .finish() + } +} + +impl> GpuObject +where + for<'a> ::Raw<'a>: Default + Zeroable, +{ + /// Create a new GpuObject with default data. `T` must implement `Default` and `T::Raw` must + /// implement `Zeroable`, since the GPU-side memory is initialized by zeroing. + pub(crate) fn new_default(alloc: U) -> Result { + GpuObject::::new_inplace(alloc, Default::default(), |_inner, raw| { + // SAFETY: `raw` is valid here, and `T::Raw` implements `Zeroable`. + Ok(unsafe { + ptr::write_bytes(raw, 0, 1); + (*raw).assume_init_mut() + }) + }) + } +} + +impl> Drop for GpuObject { + fn drop(&mut self) { + mod_dev_dbg!( + self.alloc.device(), + "Dropping {} @ {:?}\n", + core::any::type_name::(), + self.gpu_pointer() + ); + } +} + +// SAFETY: GpuObjects are Send as long as the GpuStruct itself is Send +unsafe impl> Send for GpuObject {} +// SAFETY: GpuObjects are Send as long as the GpuStruct itself is Send +unsafe impl> Sync for GpuObject {} + +/// Trait used to erase the type of a GpuObject, used when we need to keep a list of heterogenous +/// objects around. +pub(crate) trait OpaqueGpuObject: Send + Sync { + fn gpu_va(&self) -> NonZeroU64; +} + +impl> OpaqueGpuObject for GpuObject { + fn gpu_va(&self) -> NonZeroU64 { + Self::gpu_va(self) + } +} + +/// An array of raw GPU objects that is only accessible to the GPU (no CPU-side mapping required). +/// +/// This must necessarily be uninitialized as far as the GPU is concerned, so it cannot be used +/// when initialization is required. +/// +/// # Invariants +/// +/// `alloc` is valid and at least as large as `len` times the size of one `T`. +/// `gpu_ptr` is valid and points to the allocation start. +pub(crate) struct GpuOnlyArray> { + len: usize, + alloc: U, + gpu_ptr: NonZeroU64, + _p: PhantomData, +} + +impl> GpuOnlyArray { + /// Allocate a new GPU-only array with the given length. + pub(crate) fn new(alloc: U, count: usize) -> Result> { + let bytes = count * mem::size_of::(); + let gpu_ptr = NonZeroU64::new(alloc.gpu_ptr()).ok_or(EINVAL)?; + if alloc.size() < bytes { + return Err(ENOMEM); + } + Ok(Self { + len: count, + alloc, + gpu_ptr, + _p: PhantomData, + }) + } + + /// Returns the GPU VA of this arraw (as a raw [`NonZeroU64`]) + pub(crate) fn gpu_va(&self) -> NonZeroU64 { + self.gpu_ptr + } + + /// Returns a strong GPU pointer to this array, with a lifetime. + pub(crate) fn gpu_pointer(&self) -> GpuPointer<'_, &'_ [T]> { + GpuPointer(self.gpu_ptr, PhantomData) + } + + /// Returns a weak GPU pointer to this array, with no lifetime. + pub(crate) fn weak_pointer(&self) -> GpuWeakPointer<[T]> { + GpuWeakPointer(self.gpu_ptr, PhantomData) + } + + /// Returns a pointer to an offset within the array (as a subslice). + pub(crate) fn gpu_offset_pointer(&self, offset: usize) -> GpuPointer<'_, &'_ [T]> { + if offset > self.len { + panic!("Index {} out of bounds (len: {})", offset, self.len); + } + GpuPointer( + NonZeroU64::new(self.gpu_ptr.get() + (offset * mem::size_of::()) as u64).unwrap(), + PhantomData, + ) + } + + /* Not used yet + /// Returns a weak pointer to an offset within the array (as a subslice). + pub(crate) fn weak_offset_pointer(&self, offset: usize) -> GpuWeakPointer<[T]> { + if offset > self.len { + panic!("Index {} out of bounds (len: {})", offset, self.len); + } + GpuWeakPointer( + NonZeroU64::new(self.gpu_ptr.get() + (offset * mem::size_of::()) as u64).unwrap(), + PhantomData, + ) + } + + /// Returns a pointer to an element within the array. + pub(crate) fn gpu_item_pointer(&self, index: usize) -> GpuPointer<'_, &'_ T> { + if index >= self.len { + panic!("Index {} out of bounds (len: {})", index, self.len); + } + GpuPointer( + NonZeroU64::new(self.gpu_ptr.get() + (index * mem::size_of::()) as u64).unwrap(), + PhantomData, + ) + } + */ + + /// Returns a weak pointer to an element within the array. + pub(crate) fn weak_item_pointer(&self, index: usize) -> GpuWeakPointer { + if index >= self.len { + panic!("Index {} out of bounds (len: {})", index, self.len); + } + GpuWeakPointer( + NonZeroU64::new(self.gpu_ptr.get() + (index * mem::size_of::()) as u64).unwrap(), + PhantomData, + ) + } + + /// Returns the length of the array. + pub(crate) fn len(&self) -> usize { + self.len + } +} + +impl> Debug for GpuOnlyArray { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct(core::any::type_name::()) + .field("len", &format_args!("{:#X?}", self.len())) + .finish() + } +} + +impl> Drop for GpuOnlyArray { + fn drop(&mut self) { + mod_dev_dbg!( + self.alloc.device(), + "Dropping {} @ {:?}\n", + core::any::type_name::(), + self.gpu_pointer() + ); + } +} + +/// An array of raw GPU objects that is also CPU-accessible. +/// +/// # Invariants +/// +/// `raw` is valid and points to the CPU-side view of the array (which must have one). +pub(crate) struct GpuArray> { + raw: *mut T, + array: GpuOnlyArray, +} + +impl> GpuArray { + /// Allocate a new GPU array, initializing each element to its default. + pub(crate) fn empty(alloc: U, count: usize) -> Result> { + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr(); + let inner = GpuOnlyArray::new(alloc, count)?; + let mut pi = p; + for _i in 0..count { + // SAFETY: `pi` is valid per the Allocation type invariant, and GpuOnlyArray guarantees + // that it can never iterate beyond the buffer length. + unsafe { + pi.write(Default::default()); + pi = pi.add(1); + } + } + Ok(Self { + raw: p, + array: inner, + }) + } +} + +impl> GpuArray { + /// Get a slice view of the array contents. + pub(crate) fn as_slice(&self) -> &[T] { + // SAFETY: self.raw / self.len are valid per the type invariant + unsafe { slice::from_raw_parts(self.raw, self.len) } + } + + /// Get a mutable slice view of the array contents. + pub(crate) fn as_mut_slice(&mut self) -> &mut [T] { + // SAFETY: self.raw / self.len are valid per the type invariant + unsafe { slice::from_raw_parts_mut(self.raw, self.len) } + } +} + +impl> Deref for GpuArray { + type Target = GpuOnlyArray; + + fn deref(&self) -> &GpuOnlyArray { + &self.array + } +} + +impl> Index for GpuArray { + type Output = T; + + fn index(&self, index: usize) -> &T { + if index >= self.len { + panic!("Index {} out of bounds (len: {})", index, self.len); + } + // SAFETY: This is bounds checked above + unsafe { &*(self.raw.add(index)) } + } +} + +impl> IndexMut for GpuArray { + fn index_mut(&mut self, index: usize) -> &mut T { + if index >= self.len { + panic!("Index {} out of bounds (len: {})", index, self.len); + } + // SAFETY: This is bounds checked above + unsafe { &mut *(self.raw.add(index)) } + } +} + +// SAFETY: GpuArray are Send as long as the contained type itself is Send +unsafe impl> Send for GpuArray {} +// SAFETY: GpuArray are Sync as long as the contained type itself is Sync +unsafe impl> Sync for GpuArray {} + +impl> Debug for GpuArray { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct(core::any::type_name::()) + .field("array", &format_args!("{:#X?}", self.as_slice())) + .finish() + } +} diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs new file mode 100644 index 00000000000000..7c9b54c2242515 --- /dev/null +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -0,0 +1,690 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! UAT Page Table management +//! +//! AGX GPUs use an MMU called the UAT, which is largely compatible with the ARM64 page table +//! format. This module manages the actual page tables by allocating raw memory pages from +//! the kernel page allocator. + +use core::fmt::Debug; +use core::mem::size_of; +use core::ops::Range; +use core::sync::atomic::{ + AtomicU64, + Ordering, // +}; + +use kernel::{ + addr::PhysicalAddr, + error::Result, + page::Page, + prelude::*, // +}; +#[cfg(CONFIG_DEV_COREDUMP)] +use kernel::{ + types::Owned, + uapi::{ + PF_R, + PF_W, + PF_X, // + }, +}; + +use crate::debug::*; +use crate::util::align; + +const DEBUG_CLASS: DebugFlags = DebugFlags::PgTable; + +/// Number of bits in a page offset. +pub(crate) const UAT_PGBIT: usize = 14; +/// UAT page size. +pub(crate) const UAT_PGSZ: usize = 1 << UAT_PGBIT; +/// UAT page offset mask. +pub(crate) const UAT_PGMSK: usize = UAT_PGSZ - 1; + +type Pte = AtomicU64; + +const PTE_BIT: usize = 3; // log2(sizeof(Pte)) +const PTE_SIZE: usize = 1 << PTE_BIT; + +/// Number of PTEs per page. +const UAT_NPTE: usize = UAT_PGSZ / size_of::(); + +/// Number of address bits to address a level +const UAT_LVBIT: usize = UAT_PGBIT - PTE_BIT; +/// Number of entries per level +const UAT_LVSZ: usize = UAT_NPTE; +/// Mask of level bits +const UAT_LVMSK: u64 = (UAT_LVSZ - 1) as u64; + +const UAT_LEVELS: usize = 3; + +/// UAT input address space +pub(crate) const UAT_IAS: usize = 39; +const UAT_IASMSK: u64 = (1u64 << UAT_IAS) - 1; + +const PTE_TYPE_BITS: u64 = 3; +const PTE_TYPE_LEAF_TABLE: u64 = 3; + +const UAT_NON_GLOBAL: u64 = 1 << 11; +const UAT_AP_SHIFT: u32 = 6; +const UAT_AP_BITS: u64 = 3 << UAT_AP_SHIFT; +const UAT_HIGH_BITS_SHIFT: u32 = 52; +const UAT_HIGH_BITS: u64 = 0xfff << UAT_HIGH_BITS_SHIFT; +const UAT_MEMATTR_SHIFT: u32 = 2; +const UAT_MEMATTR_BITS: u64 = 7 << UAT_MEMATTR_SHIFT; + +const UAT_PROT_BITS: u64 = UAT_AP_BITS | UAT_MEMATTR_BITS | UAT_HIGH_BITS; + +const UAT_AF: u64 = 1 << 10; + +const MEMATTR_CACHED: u8 = 0; +const MEMATTR_DEV: u8 = 1; +const MEMATTR_UNCACHED: u8 = 2; + +const AP_FW_GPU: u8 = 0; +const AP_FW: u8 = 1; +const AP_GPU: u8 = 2; + +const HIGH_BITS_PXN: u16 = 1 << 1; +const HIGH_BITS_UXN: u16 = 1 << 2; +const HIGH_BITS_GPU_ACCESS: u16 = 1 << 3; + +#[cfg(CONFIG_DEV_COREDUMP)] +pub(crate) const PTE_ADDR_BITS: u64 = (!UAT_PGMSK as u64) & (!UAT_HIGH_BITS); + +#[derive(Debug, Copy, Clone)] +pub(crate) struct Prot { + memattr: u8, + ap: u8, + high_bits: u16, +} + +// Firmware + GPU access +const PROT_FW_GPU_NA: Prot = Prot::from_bits(AP_FW_GPU, 0, 0); +const _PROT_FW_GPU_RO: Prot = Prot::from_bits(AP_FW_GPU, 0, 1); +const _PROT_FW_GPU_WO: Prot = Prot::from_bits(AP_FW_GPU, 1, 0); +const PROT_FW_GPU_RW: Prot = Prot::from_bits(AP_FW_GPU, 1, 1); + +// Firmware only access +const PROT_FW_RO: Prot = Prot::from_bits(AP_FW, 0, 0); +const _PROT_FW_NA: Prot = Prot::from_bits(AP_FW, 0, 1); +const PROT_FW_RW: Prot = Prot::from_bits(AP_FW, 1, 0); +const PROT_FW_RW_GPU_RO: Prot = Prot::from_bits(AP_FW, 1, 1); + +// GPU only access +const PROT_GPU_RO: Prot = Prot::from_bits(AP_GPU, 0, 0); +const PROT_GPU_WO: Prot = Prot::from_bits(AP_GPU, 0, 1); +const PROT_GPU_RW: Prot = Prot::from_bits(AP_GPU, 1, 0); +const _PROT_GPU_NA: Prot = Prot::from_bits(AP_GPU, 1, 1); + +#[cfg(CONFIG_DEV_COREDUMP)] +const PF_RW: u32 = PF_R | PF_W; +#[cfg(CONFIG_DEV_COREDUMP)] +const PF_RX: u32 = PF_R | PF_X; + +// For crash dumps +#[cfg(CONFIG_DEV_COREDUMP)] +const PROT_TO_PERMS_FW: [[u32; 4]; 4] = [ + [0, 0, 0, PF_RW], + [0, PF_RW, 0, PF_RW], + [PF_RX, PF_RX, 0, PF_R], + [PF_RX, PF_RW, 0, PF_R], +]; +#[cfg(CONFIG_DEV_COREDUMP)] +const PROT_TO_PERMS_OS: [[u32; 4]; 4] = [ + [0, PF_R, PF_W, PF_RW], + [PF_R, 0, PF_RW, PF_RW], + [0, 0, 0, 0], + [0, 0, 0, 0], +]; + +pub(crate) mod prot { + pub(crate) use super::Prot; + use super::*; + + /// Firmware MMIO R/W + pub(crate) const PROT_FW_MMIO_RW: Prot = PROT_FW_RW.memattr(MEMATTR_DEV); + /// Firmware MMIO R/O + pub(crate) const PROT_FW_MMIO_RO: Prot = PROT_FW_RO.memattr(MEMATTR_DEV); + /// Firmware shared (uncached) RW + pub(crate) const PROT_FW_SHARED_RW: Prot = PROT_FW_RW.memattr(MEMATTR_UNCACHED); + /// Firmware shared (uncached) RO + pub(crate) const PROT_FW_SHARED_RO: Prot = PROT_FW_RO.memattr(MEMATTR_UNCACHED); + /// Firmware private (cached) RW + pub(crate) const PROT_FW_PRIV_RW: Prot = PROT_FW_RW.memattr(MEMATTR_CACHED); + /// Firmware/GPU shared (uncached) RW + pub(crate) const PROT_GPU_FW_SHARED_RW: Prot = PROT_FW_GPU_RW.memattr(MEMATTR_UNCACHED); + /// Firmware/GPU shared (private) RW + pub(crate) const PROT_GPU_FW_PRIV_RW: Prot = PROT_FW_GPU_RW.memattr(MEMATTR_CACHED); + /// Firmware-RW/GPU-RO shared (private) RW + pub(crate) const PROT_GPU_RO_FW_PRIV_RW: Prot = PROT_FW_RW_GPU_RO.memattr(MEMATTR_CACHED); + /// GPU shared/coherent RW + pub(crate) const PROT_GPU_SHARED_RW: Prot = PROT_GPU_RW.memattr(MEMATTR_UNCACHED); + /// GPU shared/coherent RO + pub(crate) const PROT_GPU_SHARED_RO: Prot = PROT_GPU_RO.memattr(MEMATTR_UNCACHED); + /// GPU shared/coherent WO + pub(crate) const PROT_GPU_SHARED_WO: Prot = PROT_GPU_WO.memattr(MEMATTR_UNCACHED); +} + +impl Prot { + const fn from_bits(ap: u8, uxn: u16, pxn: u16) -> Self { + assert!(uxn <= 1); + assert!(pxn <= 1); + assert!(ap <= 3); + + Prot { + high_bits: HIGH_BITS_GPU_ACCESS | (pxn * HIGH_BITS_PXN) | (uxn * HIGH_BITS_UXN), + memattr: 0, + ap, + } + } + + #[cfg(CONFIG_DEV_COREDUMP)] + pub(crate) const fn from_pte(pte: u64) -> Self { + Prot { + high_bits: (pte >> UAT_HIGH_BITS_SHIFT) as u16, + ap: ((pte & UAT_AP_BITS) >> UAT_AP_SHIFT) as u8, + memattr: ((pte & UAT_MEMATTR_BITS) >> UAT_MEMATTR_SHIFT) as u8, + } + } + + #[cfg(CONFIG_DEV_COREDUMP)] + pub(crate) const fn elf_flags(&self) -> u32 { + let ap = (self.ap & 3) as usize; + let uxn = if self.high_bits & HIGH_BITS_UXN != 0 { + 1 + } else { + 0 + }; + let pxn = if self.high_bits & HIGH_BITS_PXN != 0 { + 1 + } else { + 0 + }; + let gpu = self.high_bits & HIGH_BITS_GPU_ACCESS != 0; + + // Format: + // [12 top bits of PTE] [12 bottom bits of PTE] [5 bits pad] [ELF RWX] + let mut perms = if gpu { + PROT_TO_PERMS_OS[ap][(uxn << 1) | pxn] + } else { + PROT_TO_PERMS_FW[ap][(uxn << 1) | pxn] + }; + + perms |= ((self.as_pte() >> 52) << 20) as u32; + perms |= ((self.as_pte() & 0xfff) << 8) as u32; + + perms + } + + const fn memattr(&self, memattr: u8) -> Self { + Self { memattr, ..*self } + } + + const fn as_pte(&self) -> u64 { + (self.ap as u64) << UAT_AP_SHIFT + | (self.high_bits as u64) << UAT_HIGH_BITS_SHIFT + | (self.memattr as u64) << UAT_MEMATTR_SHIFT + | UAT_AF + } + + pub(crate) const fn is_cached_noncoherent(&self) -> bool { + self.ap != AP_GPU && self.memattr == MEMATTR_CACHED + } + + pub(crate) const fn as_uncached(&self) -> Self { + self.memattr(MEMATTR_UNCACHED) + } +} + +impl Default for Prot { + fn default() -> Self { + PROT_FW_GPU_NA + } +} + +#[cfg(CONFIG_DEV_COREDUMP)] +pub(crate) struct DumpedPage { + pub(crate) iova: u64, + pub(crate) pte: u64, + pub(crate) data: Option>, +} + +pub(crate) struct UatPageTable { + ttb: PhysicalAddr, + ttb_owned: bool, + va_range: Range, + oas_mask: u64, +} + +impl UatPageTable { + pub(crate) fn new(oas: u32) -> Result { + mod_pr_debug!("UATPageTable::new: oas={}\n", oas); + let ttb_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; + let ttb = Page::into_phys(ttb_page); + Ok(UatPageTable { + ttb, + ttb_owned: true, + va_range: 0..(1u64 << UAT_IAS), + oas_mask: (1u64 << oas) - 1, + }) + } + + pub(crate) fn new_with_ttb(ttb: PhysicalAddr, va_range: Range, oas: u32) -> Result { + mod_pr_debug!( + "UATPageTable::new_with_ttb: ttb={:#x} range={:#x?} oas={}\n", + ttb, + va_range, + oas + ); + if ttb & (UAT_PGMSK as PhysicalAddr) != 0 { + return Err(EINVAL); + } + if (va_range.start | va_range.end) & (UAT_PGMSK as u64) != 0 { + return Err(EINVAL); + } + // SAFETY: The TTB is should remain valid (if properly mapped), as it is bootloader-managed. + if unsafe { Page::borrow_phys(&ttb) }.is_none() { + pr_err!( + "UATPageTable::new_with_ttb: ttb at {:#x} is not mapped (DT using no-map?)\n", + ttb + ); + return Err(EIO); + } + + Ok(UatPageTable { + ttb, + ttb_owned: false, + va_range, + oas_mask: (1u64 << oas) - 1, + }) + } + + pub(crate) fn ttb(&self) -> PhysicalAddr { + self.ttb + } + + fn with_pages( + &mut self, + iova_range: Range, + alloc: bool, + free: bool, + mut cb: F, + ) -> Result + where + F: FnMut(u64, &[Pte]) -> Result, + { + mod_pr_debug!( + "UATPageTable::with_pages: {:#x?} alloc={} free={}\n", + iova_range, + alloc, + free + ); + if (iova_range.start | iova_range.end) & (UAT_PGMSK as u64) != 0 { + pr_err!( + "UATPageTable::with_pages: iova range not aligned: {:#x?}\n", + iova_range + ); + return Err(EINVAL); + } + + if iova_range.is_empty() { + return Ok(()); + } + + let mut iova = iova_range.start & UAT_IASMSK; + let mut last_iova = iova; + // Handle the case where iova_range.end is just at the top boundary of the IAS + let end = ((iova_range.end - 1) & UAT_IASMSK) + 1; + + let mut pt_addr: [Option; UAT_LEVELS] = Default::default(); + pt_addr[UAT_LEVELS - 1] = Some(self.ttb); + + 'outer: while iova < end { + mod_pr_debug!("UATPageTable::with_pages: iova={:#x}\n", iova); + let addr_diff = last_iova ^ iova; + for level in (0..UAT_LEVELS - 1).rev() { + // If the iova has changed at this level or above, invalidate the physaddr + if addr_diff & !((1 << (UAT_PGBIT + (level + 1) * UAT_LVBIT)) - 1) != 0 { + if let Some(phys) = pt_addr[level].take() { + if free { + mod_pr_debug!( + "UATPageTable::with_pages: free level {} {:#x?}\n", + level, + phys + ); + // SAFETY: Page tables for our VA ranges always come from Page::into_phys(). + unsafe { Page::from_phys(phys) }; + } + mod_pr_debug!("UATPageTable::with_pages: invalidate level {}\n", level); + } + } + } + last_iova = iova; + for level in (0..UAT_LEVELS - 1).rev() { + // Fetch the page table base address for this level + if pt_addr[level].is_none() { + let phys = pt_addr[level + 1].unwrap(); + mod_pr_debug!( + "UATPageTable::with_pages: need level {}, parent phys {:#x}\n", + level, + phys + ); + let upidx = ((iova >> (UAT_PGBIT + (level + 1) * UAT_LVBIT) as u64) & UAT_LVMSK) + as usize; + // SAFETY: Page table addresses are either allocated by us, or + // firmware-managed and safe to borrow a struct page from. + let upt = unsafe { Page::borrow_phys_unchecked(&phys) }; + mod_pr_debug!("UATPageTable::with_pages: borrowed phys {:#x}\n", phys); + pt_addr[level] = + upt.with_pointer_into_page(upidx * PTE_SIZE, PTE_SIZE, |p| { + let uptep = p as *const _ as *const Pte; + // SAFETY: with_pointer_into_page() ensures the pointer is valid, + // and our index is aligned so it is safe to deref as an AtomicU64. + let upte = unsafe { &*uptep }; + let mut upte_val = upte.load(Ordering::Relaxed); + // Allocate if requested + if upte_val == 0 && alloc { + let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; + mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); + let pt_paddr = Page::into_phys(pt_page); + upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; + upte.store(upte_val, Ordering::Relaxed); + } + if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { + Ok(Some(upte_val & self.oas_mask & (!UAT_PGMSK as u64))) + } else if upte_val == 0 || (!alloc && !free) { + mod_pr_debug!("UATPageTable::with_pages: no level {}\n", level); + Ok(None) + } else { + pr_err!("UATPageTable::with_pages: Unexpected Table PTE value {:#x} at iova {:#x} index {} phys {:#x}\n", upte_val, + iova, level + 1, phys + ((upidx * PTE_SIZE) as PhysicalAddr)); + Ok(None) + } + })?; + mod_pr_debug!( + "UATPageTable::with_pages: level {} PT {:#x?}\n", + level, + pt_addr[level] + ); + } + // If we don't have a page table, skip this entire level + if pt_addr[level].is_none() { + let block = 1 << (UAT_PGBIT + UAT_LVBIT * (level + 1)); + let old = iova; + iova = align(iova + 1, block); + mod_pr_debug!( + "UATPageTable::with_pages: skip {:#x} {:#x} -> {:#x}\n", + block, + old, + iova + ); + continue 'outer; + } + } + + let idx = ((iova >> UAT_PGBIT as u64) & UAT_LVMSK) as usize; + let max_count = UAT_NPTE - idx; + let count = (((end - iova) >> UAT_PGBIT) as usize).min(max_count); + let phys = pt_addr[0].unwrap(); + mod_pr_debug!( + "UATPageTable::with_pages: leaf PT at {:#x} idx {:#x} count {:#x} iova {:#x}\n", + phys, + idx, + count, + iova + ); + // SAFETY: Page table addresses are either allocated by us, or + // firmware-managed and safe to borrow a struct page from. + let pt = unsafe { Page::borrow_phys_unchecked(&phys) }; + pt.with_pointer_into_page(idx * PTE_SIZE, count * PTE_SIZE, |p| { + let ptep = p as *const _ as *const Pte; + // SAFETY: We know this is a valid pointer to PTEs and the range is valid and + // checked by with_pointer_into_page(). + let ptes = unsafe { core::slice::from_raw_parts(ptep, count) }; + cb(iova, ptes)?; + Ok(()) + })?; + + let block = 1 << (UAT_PGBIT + UAT_LVBIT); + iova = align(iova + 1, block); + } + + if free { + for level in (0..UAT_LEVELS - 1).rev() { + if let Some(phys) = pt_addr[level] { + mod_pr_debug!( + "UATPageTable::with_pages: free level {} {:#x?}\n", + level, + phys + ); + // SAFETY: Page tables for our VA ranges always come from Page::into_phys(). + unsafe { Page::from_phys(phys) }; + } + } + } + + Ok(()) + } + + pub(crate) fn alloc_pages(&mut self, iova_range: Range) -> Result { + mod_pr_debug!("UATPageTable::alloc_pages: {:#x?}\n", iova_range); + self.with_pages(iova_range, true, false, |_, _| Ok(())) + } + + fn pte_bits(&self) -> u64 { + if self.ttb_owned { + // Owned page tables are userspace, so non-global + PTE_TYPE_LEAF_TABLE | UAT_NON_GLOBAL + } else { + // The sole non-owned page table is kernelspace, so global + PTE_TYPE_LEAF_TABLE + } + } + + pub(crate) fn map_pages( + &mut self, + iova_range: Range, + mut phys: PhysicalAddr, + prot: Prot, + one_page: bool, + ) -> Result { + mod_pr_debug!( + "UATPageTable::map_pages: {:#x?} {:#x?} {:?}\n", + iova_range, + phys, + prot + ); + if phys & (UAT_PGMSK as PhysicalAddr) != 0 { + pr_err!("UATPageTable::map_pages: phys not aligned: {:#x?}\n", phys); + return Err(EINVAL); + } + + let pte_bits = self.pte_bits(); + + self.with_pages(iova_range, true, false, |iova, ptes| { + for (idx, pte) in ptes.iter().enumerate() { + let ptev = pte.load(Ordering::Relaxed); + if ptev != 0 { + pr_err!( + "UATPageTable::map_pages: Page at IOVA {:#x} is mapped (PTE: {:#x})\n", + iova + (idx * UAT_PGSZ) as u64, + ptev + ); + } + pte.store(phys | prot.as_pte() | pte_bits, Ordering::Relaxed); + if !one_page { + phys += UAT_PGSZ as PhysicalAddr; + } + } + Ok(()) + }) + } + + pub(crate) fn reprot_pages(&mut self, iova_range: Range, prot: Prot) -> Result { + mod_pr_debug!( + "UATPageTable::reprot_pages: {:#x?} {:?}\n", + iova_range, + prot + ); + self.with_pages(iova_range, true, false, |iova, ptes| { + for (idx, pte) in ptes.iter().enumerate() { + let ptev = pte.load(Ordering::Relaxed); + if ptev & PTE_TYPE_BITS != PTE_TYPE_LEAF_TABLE { + pr_err!( + "UATPageTable::reprot_pages: Page at IOVA {:#x} is unmapped (PTE: {:#x})\n", + iova + (idx * UAT_PGSZ) as u64, + ptev + ); + continue; + } + pte.store((ptev & !UAT_PROT_BITS) | prot.as_pte(), Ordering::Relaxed); + } + Ok(()) + }) + } + + pub(crate) fn unmap_pages(&mut self, iova_range: Range) -> Result { + mod_pr_debug!("UATPageTable::unmap_pages: {:#x?}\n", iova_range); + self.with_pages(iova_range, false, false, |iova, ptes| { + for (idx, pte) in ptes.iter().enumerate() { + if pte.load(Ordering::Relaxed) & PTE_TYPE_LEAF_TABLE == 0 { + pr_err!( + "UATPageTable::unmap_pages: Page at IOVA {:#x} already unmapped\n", + iova + (idx * UAT_PGSZ) as u64 + ); + } + pte.store(0, Ordering::Relaxed); + } + Ok(()) + }) + } + + #[cfg(CONFIG_DEV_COREDUMP)] + pub(crate) fn dump_pages(&mut self, iova_range: Range) -> Result> { + let mut pages = KVVec::new(); + let oas_mask = self.oas_mask; + let iova_base = self.va_range.start & !UAT_IASMSK; + self.with_pages(iova_range, false, false, |iova, ptes| { + let iova = iova | iova_base; + for (idx, ppte) in ptes.iter().enumerate() { + let pte = ppte.load(Ordering::Relaxed); + if (pte & PTE_TYPE_LEAF_TABLE) != PTE_TYPE_LEAF_TABLE { + continue; + } + let memattr = ((pte & UAT_MEMATTR_BITS) >> UAT_MEMATTR_SHIFT) as u8; + + if !(memattr == MEMATTR_CACHED || memattr == MEMATTR_UNCACHED) { + pages.push( + DumpedPage { + iova: iova + (idx * UAT_PGSZ) as u64, + pte, + data: None, + }, + GFP_KERNEL, + )?; + continue; + } + let phys = pte & oas_mask & (!UAT_PGMSK as u64); + // SAFETY: GPU pages are either firmware/preallocated pages + // (which the kernel isn't concerned with and are either in + // the page map or not, and if they aren't, borrow_phys() + // will fail), or GPU page table pages (which we own), + // or GEM buffer pages (which are locked while they are + // mapped in the page table), so they should be safe to + // borrow. + // + // This does trust the firmware not to have any weird + // mappings in its own internal page tables, but since + // those are managed by the uPPL which is privileged anyway, + // this trust does not actually extend any trust boundary. + let src_page = match unsafe { Page::borrow_phys(&phys) } { + Some(page) => page, + None => { + pages.push( + DumpedPage { + iova: iova + (idx * UAT_PGSZ) as u64, + pte, + data: None, + }, + GFP_KERNEL, + )?; + continue; + } + }; + let dst_page = Page::alloc_page(GFP_KERNEL)?; + src_page.with_page_mapped(|psrc| -> Result { + // SAFETY: This could technically still have a data race with the firmware + // or other driver code (or even userspace with timestamp buffers), but while + // the Rust language technically says this is UB, in the real world, using + // atomic reads for this is guaranteed to never cause any harmful effects + // other than possibly reading torn/unreliable data. At least on ARM64 anyway. + // + // (Yes, I checked with Rust people about this. ~~ Lina) + // + let src_items = unsafe { + core::slice::from_raw_parts( + psrc as *const AtomicU64, + UAT_PGSZ / core::mem::size_of::(), + ) + }; + dst_page.with_page_mapped(|pdst| -> Result { + // SAFETY: We own the destination page, so it is safe to view its contents + // as a u64 slice. + let dst_items = unsafe { + core::slice::from_raw_parts_mut( + pdst as *mut u64, + UAT_PGSZ / core::mem::size_of::(), + ) + }; + for (si, di) in src_items.iter().zip(dst_items.iter_mut()) { + *di = si.load(Ordering::Relaxed); + } + Ok(()) + })?; + Ok(()) + })?; + pages.push( + DumpedPage { + iova: iova + (idx * UAT_PGSZ) as u64, + pte, + data: Some(dst_page), + }, + GFP_KERNEL, + )?; + } + Ok(()) + })?; + Ok(pages) + } +} + +impl Drop for UatPageTable { + fn drop(&mut self) { + mod_pr_debug!("UATPageTable::drop range: {:#x?}\n", &self.va_range); + if self + .with_pages(self.va_range.clone(), false, true, |iova, ptes| { + for (idx, pte) in ptes.iter().enumerate() { + if pte.load(Ordering::Relaxed) != 0 { + pr_err!( + "UATPageTable::drop: Leaked page at IOVA {:#x}\n", + iova + (idx * UAT_PGSZ) as u64 + ); + } + } + Ok(()) + }) + .is_err() + { + pr_err!("UATPageTable::drop failed to free page tables\n",); + } + if self.ttb_owned { + mod_pr_debug!("UATPageTable::drop: Free TTB {:#x}\n", self.ttb); + // SAFETY: If we own the ttb, it was allocated with Page::into_phys(). + unsafe { + Page::from_phys(self.ttb); + } + } + } +} diff --git a/drivers/gpu/drm/asahi/queue/common.rs b/drivers/gpu/drm/asahi/queue/common.rs new file mode 100644 index 00000000000000..a68352828cfbc3 --- /dev/null +++ b/drivers/gpu/drm/asahi/queue/common.rs @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Common queue functionality. +//! +//! Shared helpers used by the submission logic for multiple command types. + +use crate::file; +use crate::fw::job::UserTimestamp; + +use kernel::prelude::*; +use kernel::uapi; +use kernel::xarray; + +pub(super) fn get_timestamp_object( + objects: Pin<&xarray::XArray>>, + timestamp: uapi::drm_asahi_timestamp, +) -> Result> { + if timestamp.handle == 0 { + return Ok(None); + } + + let guard = objects.lock(); + let object = guard + .get(timestamp.handle.try_into()?) + .ok_or(ENOENT)? + .clone(); + core::mem::drop(guard); + + #[allow(irrefutable_let_patterns)] + if let file::Object::TimestampBuffer(mapping) = object { + let offset = timestamp.offset; + if (offset.checked_add(8).ok_or(EINVAL)?) as usize > mapping.size() { + return Err(ERANGE); + } + Ok(Some(UserTimestamp { + mapping: mapping.clone(), + offset: offset as usize, + })) + } else { + Err(EINVAL) + } +} diff --git a/drivers/gpu/drm/asahi/queue/compute.rs b/drivers/gpu/drm/asahi/queue/compute.rs new file mode 100644 index 00000000000000..62afc561806703 --- /dev/null +++ b/drivers/gpu/drm/asahi/queue/compute.rs @@ -0,0 +1,385 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![allow(clippy::unusual_byte_groupings)] + +//! Compute work queue. +//! +//! A compute queue consists of one underlying WorkQueue. +//! This module is in charge of creating all of the firmware structures required to submit compute +//! work to the GPU, based on the userspace command buffer. + +use super::common; +use crate::alloc::Allocator; +use crate::debug::*; +use crate::fw::types::*; +use crate::gpu::GpuManager; +use crate::{ + file, + fw, + gpu, + microseq, // +}; +use crate::{ + inner_ptr, + inner_weak_ptr, // +}; +use core::sync::atomic::Ordering; +use kernel::dma_fence::RawDmaFence; +use kernel::drm::sched::Job; +use kernel::prelude::*; +use kernel::sync::Arc; +use kernel::uapi; +use kernel::xarray; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Compute; + +#[versions(AGX)] +impl super::QueueInner::ver { + /// Submit work to a compute queue. + pub(super) fn submit_compute( + &self, + job: &mut Job, + cmdbuf: &uapi::drm_asahi_cmd_compute, + attachments: µseq::Attachments, + objects: Pin<&xarray::XArray>>, + id: u64, + flush_stamps: bool, + ) -> Result { + let gpu = match (*self.dev) + .gpu + .as_any() + .downcast_ref::() + { + Some(gpu) => gpu, + None => { + dev_crit!(self.dev.as_ref(), "GpuManager mismatched with Queue!\n"); + return Err(EIO); + } + }; + + let mut alloc = gpu.alloc(); + let kalloc = &mut *alloc; + + mod_dev_dbg!(self.dev, "[Submission {}] Compute!\n", id); + + if cmdbuf.flags != 0 { + return Err(EINVAL); + } + + let mut user_timestamps: fw::job::UserTimestamps = Default::default(); + user_timestamps.start = common::get_timestamp_object(objects, cmdbuf.ts.start)?; + user_timestamps.end = common::get_timestamp_object(objects, cmdbuf.ts.end)?; + + // This sequence number increases per new client/VM? assigned to some slot, + // but it's unclear *which* slot... + let slot_client_seq: u8 = (self.id & 0xff) as u8; + + let vm_bind = job.vm_bind.clone(); + + mod_dev_dbg!( + self.dev, + "[Submission {}] VM slot = {}\n", + id, + vm_bind.slot() + ); + + let notifier = self.notifier.clone(); + + let fence = job.fence.clone(); + let comp_job = job.get_comp()?; + let ev_comp = comp_job.event_info(); + + let preempt2_off = gpu.get_cfg().compute_preempt1_size; + let preempt3_off = preempt2_off + 8; + let preempt4_off = preempt3_off + 8; + let preempt5_off = preempt4_off + 8; + let preempt_size = preempt5_off + 8; + + let preempt_buf = self + .ualloc + .lock() + .array_empty_tagged(preempt_size, b"CPMT")?; + + mod_dev_dbg!( + self.dev, + "[Submission {}] Event #{} {:#x?} -> {:#x?}\n", + id, + ev_comp.slot, + ev_comp.value, + ev_comp.value.next(), + ); + + let timestamps = Arc::new( + kalloc.shared.new_default::()?, + GFP_KERNEL, + )?; + + let uuid = 0; + mod_dev_dbg!(self.dev, "[Submission {}] UUID = {:#x?}\n", id, uuid); + + // TODO: check + #[ver(V >= V13_0B4)] + let count = self.counter.fetch_add(1, Ordering::Relaxed); + + let comp = GpuObject::new_init_prealloc( + kalloc.gpu_ro.alloc_object()?, + |ptr: GpuWeakPointer| { + let notifier = notifier.clone(); + let vm_bind = vm_bind.clone(); + try_init!(fw::compute::RunCompute::ver { + preempt_buf: preempt_buf, + micro_seq: { + let mut builder = microseq::Builder::new(); + + let stats = gpu.initdata.runtime_pointers.stats.comp.weak_pointer(); + + let start_comp = builder.add(microseq::StartCompute::ver { + header: microseq::op::StartCompute::HEADER, + unk_pointer: inner_weak_ptr!(ptr, unk_pointee), + #[ver(G < G14X)] + job_params1: Some(inner_weak_ptr!(ptr, job_params1)), + #[ver(G >= G14X)] + job_params1: None, + #[ver(G >= G14X)] + registers: inner_weak_ptr!(ptr, registers), + stats, + work_queue: ev_comp.info_ptr, + vm_slot: vm_bind.slot(), + unk_28: 0x1, + event_generation: self.id as u32, + event_seq: U64(ev_comp.event_seq), + unk_38: 0x0, + job_params2: inner_weak_ptr!(ptr, job_params2), + unk_44: 0x0, + uuid, + attachments: *attachments, + padding: Default::default(), + #[ver(V >= V13_0B4)] + unk_flag: inner_weak_ptr!(ptr, unk_flag), + #[ver(V >= V13_0B4)] + counter: U64(count), + #[ver(V >= V13_0B4)] + notifier_buf: inner_weak_ptr!(notifier.weak_pointer(), state.unk_buf), + })?; + + if user_timestamps.any() { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(true), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), + work_queue: ev_comp.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, context_store_req), + uuid, + unk_30_padding: 0, + })?; + } + + #[ver(G < G14X)] + builder.add(microseq::WaitForIdle { + header: microseq::op::WaitForIdle::new(microseq::Pipe::Compute), + })?; + #[ver(G >= G14X)] + builder.add(microseq::WaitForIdle2 { + header: microseq::op::WaitForIdle2::HEADER, + })?; + + if user_timestamps.any() { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(false), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), + work_queue: ev_comp.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, context_store_req), + uuid, + unk_30_padding: 0, + })?; + } + + let off = builder.offset_to(start_comp); + builder.add(microseq::FinalizeCompute::ver { + header: microseq::op::FinalizeCompute::HEADER, + stats, + work_queue: ev_comp.info_ptr, + vm_slot: vm_bind.slot(), + #[ver(V < V13_0B4)] + unk_18: 0, + job_params2: inner_weak_ptr!(ptr, job_params2), + unk_24: 0, + uuid, + fw_stamp: ev_comp.fw_stamp_pointer, + stamp_value: ev_comp.value.next(), + unk_38: 0, + unk_3c: 0, + unk_40: 0, + unk_44: 0, + unk_48: 0, + unk_4c: 0, + unk_50: 0, + unk_54: 0, + unk_58: 0, + #[ver(G == G14 && V < V13_0B4)] + unk_5c_g14: U64(0), + restart_branch_offset: off, + has_attachments: (attachments.count > 0) as u32, + #[ver(V >= V13_0B4)] + unk_64: Default::default(), + #[ver(V >= V13_0B4)] + unk_flag: inner_weak_ptr!(ptr, unk_flag), + #[ver(V >= V13_0B4)] + unk_79: Default::default(), + })?; + + builder.add(microseq::RetireStamp { + header: microseq::op::RetireStamp::HEADER, + })?; + builder.build(&mut kalloc.private)? + }, + notifier, + vm_bind, + timestamps, + user_timestamps, + }) + }, + |inner, _ptr| { + let vm_slot = vm_bind.slot(); + try_init!(fw::compute::raw::RunCompute::ver { + tag: fw::workqueue::CommandType::RunCompute, + #[ver(V >= V13_0B4)] + counter: U64(count), + unk_4: 0, + vm_slot, + notifier: inner.notifier.gpu_pointer(), + unk_pointee: Default::default(), + #[ver(G < G14X)] + __pad0: Default::default(), + #[ver(G < G14X)] + job_params1 <- try_init!(fw::compute::raw::JobParameters1 { + preempt_buf1: inner.preempt_buf.gpu_pointer(), + cdm_ctrl_stream_base: U64(cmdbuf.cdm_ctrl_stream_base), + // buf2-5 Only if internal program is used + preempt_buf2: inner.preempt_buf.gpu_offset_pointer(preempt2_off), + preempt_buf3: inner.preempt_buf.gpu_offset_pointer(preempt3_off), + preempt_buf4: inner.preempt_buf.gpu_offset_pointer(preempt4_off), + preempt_buf5: inner.preempt_buf.gpu_offset_pointer(preempt5_off), + usc_exec_base_cp: U64(self.usc_exec_base), + unk_38: U64(0x8c60), + helper_program: cmdbuf.helper.binary, // Internal program addr | 1 + unk_44: 0, + helper_arg: U64(cmdbuf.helper.data), // Only if internal program used + helper_cfg: cmdbuf.helper.cfg, // 0x40 if internal program used + unk_54: 0, + unk_58: 1, + unk_5c: 0, + iogpu_unk_40: 0, // 0x1c if internal program used + __pad: Default::default(), + }), + #[ver(G >= G14X)] + registers: fw::job::raw::RegisterArray::new( + inner_weak_ptr!(_ptr, registers.registers), + |r| { + r.add(0x1a510, inner.preempt_buf.gpu_pointer().into()); + r.add(0x1a420, cmdbuf.cdm_ctrl_stream_base); + // buf2-5 Only if internal program is used + r.add(0x1a4d0, inner.preempt_buf.gpu_offset_pointer(preempt2_off).into()); + r.add(0x1a4d8, inner.preempt_buf.gpu_offset_pointer(preempt3_off).into()); + r.add(0x1a4e0, inner.preempt_buf.gpu_offset_pointer(preempt4_off).into()); + r.add(0x1a4e8, inner.preempt_buf.gpu_offset_pointer(preempt5_off).into()); + r.add(0x10071, self.usc_exec_base); // USC_EXEC_BASE_CP + r.add(0x11841, cmdbuf.helper.binary.into()); + r.add(0x11849, cmdbuf.helper.data); + r.add(0x11f81, cmdbuf.helper.cfg.into()); + r.add(0x1a440, 0x24201); + r.add(0x12091, 0 /* iogpu_unk_40 */); + /* + r.add(0x10201, 0x100); // Some kind of counter?? Does this matter? + r.add(0x10428, 0x100); // Some kind of counter?? Does this matter? + */ + } + ), + __pad1: Default::default(), + microsequence: inner.micro_seq.gpu_pointer(), + microsequence_size: inner.micro_seq.len() as u32, + job_params2 <- try_init!(fw::compute::raw::JobParameters2::ver { + #[ver(V >= V13_0B4)] + unk_0_0: 0, + unk_0: Default::default(), + preempt_buf1: inner.preempt_buf.gpu_pointer(), + cdm_ctrl_stream_end: U64(cmdbuf.cdm_ctrl_stream_end), + unk_34: Default::default(), + #[ver(G < G14X)] + unk_g14x: 0, + #[ver(G >= G14X)] + unk_g14x: 0x24201, + unk_58: 0, + #[ver(V < V13_0B4)] + unk_5c: 0, + }), + encoder_params <- try_init!(fw::job::raw::EncoderParams { + unk_8: 0x0, // fixed + sync_grow: 0x0, // check! + unk_10: 0x0, // fixed + encoder_id: 0, + unk_18: 0x0, // fixed + unk_mask: 0xffffffff, + sampler_array: U64(cmdbuf.sampler_heap), + sampler_count: cmdbuf.sampler_count as u32, + sampler_max: (cmdbuf.sampler_count as u32) + 1, + }), + meta <- try_init!(fw::job::raw::JobMeta { + unk_0: 0, + unk_2: 0, + no_preemption: 0, + stamp: ev_comp.stamp_pointer, + fw_stamp: ev_comp.fw_stamp_pointer, + stamp_value: ev_comp.value.next(), + stamp_slot: ev_comp.slot, + evctl_index: 0, // fixed + flush_stamps: flush_stamps as u32, + uuid, + event_seq: ev_comp.event_seq as u32, + }), + command_time: U64(0), + timestamp_pointers <- try_init!(fw::job::raw::TimestampPointers { + start_addr: Some(inner_ptr!(inner.timestamps.gpu_pointer(), start)), + end_addr: Some(inner_ptr!(inner.timestamps.gpu_pointer(), end)), + }), + user_timestamp_pointers: inner.user_timestamps.pointers()?, + client_sequence: slot_client_seq, + pad_2d1: Default::default(), + unk_2d4: 0, + unk_2d8: 0, + #[ver(V >= V13_0B4)] + context_store_req: U64(0), + #[ver(V >= V13_0B4)] + context_store_compl: U64(0), + #[ver(V >= V13_0B4)] + unk_2e9: Default::default(), + #[ver(V >= V13_0B4)] + unk_flag: U32(0), + #[ver(V >= V13_0B4)] + unk_pad: Default::default(), + }) + }, + )?; + + core::mem::drop(alloc); + + fence.add_command(); + comp_job.add_cb(comp, vm_bind.slot(), move |error| { + if let Some(err) = error { + fence.set_error(err.into()) + } + + fence.command_complete(); + })?; + + comp_job.next_seq(); + + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs new file mode 100644 index 00000000000000..caae3e19f994c6 --- /dev/null +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -0,0 +1,937 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Submission queue management +//! +//! This module implements the userspace view of submission queues and the logic to map userspace +//! submissions to firmware queues. + +use kernel::dma_fence::*; +use kernel::prelude::*; +use kernel::{ + c_str, + dma_fence, + drm::sched, + macros::versions, + sync::{ + Arc, + LockClassKey, + Mutex, // + }, + uapi, + xarray, // +}; + +use crate::alloc::Allocator; +use crate::debug::*; +use crate::driver::{AsahiDevRef, AsahiDevice}; +use crate::file::MAX_COMMANDS_PER_SUBMISSION; +use crate::fw::types::*; +use crate::gpu::GpuManager; +use crate::inner_weak_ptr; +use crate::microseq; +use crate::module_parameters; +use crate::util::{ + AnyBitPattern, + Reader, // +}; +use crate::{ + alloc, + buffer, + channel, + event, + file, + fw, + gpu, + mmu, + workqueue, // +}; + +use core::sync::atomic::{ + AtomicU64, + Ordering, // +}; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Queue; + +const WQ_SIZE: u32 = 0x500; + +mod common; +mod compute; +mod render; + +/// Trait implemented by all versioned queues. +pub(crate) trait Queue: Send + Sync { + fn submit( + &mut self, + id: u64, + syncs: KVec, + in_sync_count: usize, + cmdbuf_raw: &[u8], + objects: Pin<&xarray::XArray>>, + ) -> Result; +} + +#[versions(AGX)] +struct SubQueue { + wq: Arc, +} + +#[versions(AGX)] +impl SubQueue::ver { + fn new_job(&mut self, fence: dma_fence::Fence) -> SubQueueJob::ver { + SubQueueJob::ver { + wq: self.wq.clone(), + fence: Some(fence), + job: None, + } + } +} + +#[versions(AGX)] +struct SubQueueJob { + wq: Arc, + job: Option, + fence: Option, +} + +#[versions(AGX)] +impl SubQueueJob::ver { + fn get(&mut self) -> Result<&mut workqueue::Job::ver> { + if self.job.is_none() { + mod_pr_debug!("SubQueueJob: Creating {:?} job\n", self.wq.pipe_type()); + self.job + .replace(self.wq.new_job(self.fence.take().unwrap())?); + } + Ok(self.job.as_mut().expect("expected a Job")) + } + + fn commit(&mut self) -> Result { + match self.job.as_mut() { + Some(job) => job.commit(), + None => Ok(()), + } + } + + fn can_submit(&self) -> Option { + self.job.as_ref().and_then(|job| job.can_submit()) + } +} + +#[versions(AGX)] +pub(crate) struct Queue { + dev: AsahiDevRef, + _sched: sched::Scheduler, + entity: sched::Entity, + vm: mmu::Vm, + q_vtx: Option, + q_frag: Option, + q_comp: Option, + fence_ctx: FenceContexts, + inner: QueueInner::ver, +} + +#[versions(AGX)] +pub(crate) struct QueueInner { + dev: AsahiDevRef, + ualloc: Arc>, + buffer: buffer::Buffer::ver, + gpu_context: Arc, + notifier_list: Arc>, + notifier: Arc>, + usc_exec_base: u64, + id: u64, + #[ver(V >= V13_0B4)] + counter: AtomicU64, +} + +#[versions(AGX)] +#[derive(Default)] +pub(crate) struct JobFence { + id: u64, + pending: AtomicU64, +} + +#[versions(AGX)] +impl JobFence::ver { + fn add_command(self: &FenceObject) { + self.pending.fetch_add(1, Ordering::Relaxed); + } + + fn command_complete(self: &FenceObject) { + let remain = self.pending.fetch_sub(1, Ordering::Relaxed) - 1; + mod_pr_debug!( + "JobFence[{}]: Command complete (remain: {})\n", + self.id, + remain + ); + if remain == 0 { + mod_pr_debug!("JobFence[{}]: Signaling\n", self.id); + if self.signal().is_err() { + pr_err!("JobFence[{}]: Fence signal failed\n", self.id); + } + } + } +} + +#[versions(AGX)] +#[vtable] +impl dma_fence::FenceOps for JobFence::ver { + fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { + c_str!("asahi") + } + fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { + c_str!("queue") + } +} + +#[versions(AGX)] +pub(crate) struct QueueJob { + dev: AsahiDevRef, + vm_bind: mmu::VmBind, + op_guard: Option, + sj_vtx: Option, + sj_frag: Option, + sj_comp: Option, + fence: UserFence, + notifier: Arc>, + notification_count: u32, + did_run: bool, + id: u64, +} + +#[versions(AGX)] +impl QueueJob::ver { + fn get_vtx(&mut self) -> Result<&mut workqueue::Job::ver> { + self.sj_vtx + .as_mut() + .ok_or_else(|| { + cls_pr_debug!(Errors, "No vertex queue\n"); + EINVAL + })? + .get() + } + fn get_frag(&mut self) -> Result<&mut workqueue::Job::ver> { + self.sj_frag + .as_mut() + .ok_or_else(|| { + cls_pr_debug!(Errors, "No fragment queue\n"); + EINVAL + })? + .get() + } + fn get_comp(&mut self) -> Result<&mut workqueue::Job::ver> { + self.sj_comp + .as_mut() + .ok_or_else(|| { + cls_pr_debug!(Errors, "No compute queue\n"); + EINVAL + })? + .get() + } + + fn commit(&mut self) -> Result { + mod_dev_dbg!(self.dev, "QueueJob {}: Committing\n", self.id); + + self.sj_vtx.as_mut().map(|a| a.commit()).unwrap_or(Ok(()))?; + self.sj_frag + .as_mut() + .map(|a| a.commit()) + .unwrap_or(Ok(()))?; + self.sj_comp.as_mut().map(|a| a.commit()).unwrap_or(Ok(())) + } +} + +#[versions(AGX)] +impl sched::JobImpl for QueueJob::ver { + fn prepare(job: &mut sched::Job) -> Option { + mod_dev_dbg!(job.dev, "QueueJob {}: Checking runnability\n", job.id); + + if let Some(sj) = job.sj_vtx.as_ref() { + if let Some(fence) = sj.can_submit() { + mod_dev_dbg!( + job.dev, + "QueueJob {}: Blocking due to vertex queue full\n", + job.id + ); + return Some(fence); + } + } + if let Some(sj) = job.sj_frag.as_ref() { + if let Some(fence) = sj.can_submit() { + mod_dev_dbg!( + job.dev, + "QueueJob {}: Blocking due to fragment queue full\n", + job.id + ); + return Some(fence); + } + } + if let Some(sj) = job.sj_comp.as_ref() { + if let Some(fence) = sj.can_submit() { + mod_dev_dbg!( + job.dev, + "QueueJob {}: Blocking due to compute queue full\n", + job.id + ); + return Some(fence); + } + } + None + } + + #[allow(unused_assignments)] + fn run(job: &mut sched::Job) -> Result> { + mod_dev_dbg!(job.dev, "QueueJob {}: Running Job\n", job.id); + + // We can only increase the notifier threshold here, now that we are + // actually running the job. We cannot increase it while queueing the + // job without introducing subtle race conditions. Suppose we did, as + // early versions of drm/asahi did: + // + // 1. When processing the ioctl submit, a job is queued to drm_sched. + // Incorrectly, the notifier threshold is increased, gating firmware + // events. + // 2. When DRM schedules an event, the hardware is kicked. + // 3. When the number of processed jobs equals the threshold, the + // firmware signals the complete event to the kernel + // 4. When the kernel gets a complete event, we signal the out-syncs. + // + // Does that work? There are a few scenarios. + // + // 1. There is nothing else ioctl submitted before the job completes. + // The job is scheduled, completes, and signals immediately. + // Everything works. + // 2. There is nontrivial sync across different queues. Since each queue + // has a separate own notifier threshold, submitting one does not + // block scheduling of the other. Everything works the way you'd + // expect. drm/sched handles the wait/signal ordering. + // 3. Two ioctls are submitted back-to-back. The first signals a fence + // that the second waits on. Due to the notifier threshold increment, + // the first job's completion event is deferred. But in good + // conditions, drm/sched will schedule the second submit anyway + // because it kills the pointless intra-queue sync. Then both + // commands execute and are signalled together. + // 4. Two ioctls are submitted back-to-back as above, but conditions are + // bad. Reporting completion of the first job is still masked by the + // notifier threshold, but the intra-queue fences are not optimized + // out in drm/sched... drm/sched doesn't schedule the second job + // until the first is signalled, but the first isn't signalled until + // the second is completed, but the second can't complete until it's + // scheduled. We hang! + // + // In good conditions, everything works properly and/or we win the race + // to mask the issue. So the issue here is challenging to hit. + // Nevertheless, we do need to get it right. + // + // The intention with drm/sched is that jobs that are not yet scheduled + // are "invisible" to the firmware. Incrementing the notifier threshold + // earlier than this violates that which leads to circles like the + // above. Deferring the increment to submit solves the race. + job.notifier.threshold.with(|raw, _inner| { + raw.increase(job.notification_count); + }); + + let gpu = match (*job.dev) + .gpu + .clone() + .arc_as_any() + .downcast::() + { + Ok(gpu) => gpu, + Err(_) => { + dev_crit!(job.dev.as_ref(), "GpuManager mismatched with QueueJob!\n"); + return Err(EIO); + } + }; + + if job.op_guard.is_none() { + job.op_guard = Some(gpu.start_op()?); + } + + // First submit all the commands for each queue. This can fail. + + let mut frag_job = None; + let mut frag_sub = None; + if let Some(sj) = job.sj_frag.as_mut() { + frag_job = sj.job.take(); + if let Some(wqjob) = frag_job.as_mut() { + mod_dev_dbg!(job.dev, "QueueJob {}: Submit fragment\n", job.id); + frag_sub = Some(wqjob.submit()?); + } + } + + let mut vtx_job = None; + let mut vtx_sub = None; + if let Some(sj) = job.sj_vtx.as_mut() { + vtx_job = sj.job.take(); + if let Some(wqjob) = vtx_job.as_mut() { + mod_dev_dbg!(job.dev, "QueueJob {}: Submit vertex\n", job.id); + vtx_sub = Some(wqjob.submit()?); + } + } + + let mut comp_job = None; + let mut comp_sub = None; + if let Some(sj) = job.sj_comp.as_mut() { + comp_job = sj.job.take(); + if let Some(wqjob) = comp_job.as_mut() { + mod_dev_dbg!(job.dev, "QueueJob {}: Submit compute\n", job.id); + comp_sub = Some(wqjob.submit()?); + } + } + + // Now we fully commit to running the job + mod_dev_dbg!(job.dev, "QueueJob {}: Run fragment\n", job.id); + frag_sub.map(|a| gpu.run_job(a)).transpose()?; + + mod_dev_dbg!(job.dev, "QueueJob {}: Run vertex\n", job.id); + vtx_sub.map(|a| gpu.run_job(a)).transpose()?; + + mod_dev_dbg!(job.dev, "QueueJob {}: Run compute\n", job.id); + comp_sub.map(|a| gpu.run_job(a)).transpose()?; + + mod_dev_dbg!(job.dev, "QueueJob {}: Drop compute job\n", job.id); + core::mem::drop(comp_job); + mod_dev_dbg!(job.dev, "QueueJob {}: Drop vertex job\n", job.id); + core::mem::drop(vtx_job); + mod_dev_dbg!(job.dev, "QueueJob {}: Drop fragment job\n", job.id); + core::mem::drop(frag_job); + + job.did_run = true; + + Ok(Some(Fence::from_fence(&job.fence))) + } + + fn timed_out(job: &mut sched::Job) -> sched::Status { + // FIXME: Handle timeouts properly + dev_err!( + job.dev.as_ref(), + "QueueJob {}: Job timed out on the DRM scheduler, things will probably break (ran: {})\n", + job.id, job.did_run + ); + sched::Status::NoDevice + } + + fn cancel(job: &mut sched::Job) { + dev_info!( + job.dev.as_ref(), + "QueueJob {}: Job canceled on DRM scheduler teardown\n", + job.id + ); + } +} + +#[versions(AGX)] +impl Drop for QueueJob::ver { + fn drop(&mut self) { + mod_dev_dbg!(self.dev, "QueueJob {}: Dropping\n", self.id); + } +} + +static QUEUE_NAME: &CStr = c_str!("asahi_fence"); +static QUEUE_CLASS_KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); + +#[versions(AGX)] +impl Queue::ver { + /// Create a new user queue. + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + dev: &AsahiDevice, + vm: mmu::Vm, + alloc: &mut gpu::KernelAllocators, + ualloc: Arc>, + ualloc_priv: Arc>, + event_manager: Arc, + mgr: &buffer::BufferManager::ver, + id: u64, + priority: u32, + usc_exec_base: u64, + ) -> Result { + mod_dev_dbg!(dev, "[Queue {}] Creating queue\n", id); + + // Must be shared, no cache management on this one! + let mut notifier_list = alloc.shared.new_default::()?; + + let self_ptr = notifier_list.weak_pointer(); + notifier_list.with_mut(|raw, _inner| { + raw.list_head.next = Some(inner_weak_ptr!(self_ptr, list_head)); + }); + + let threshold = alloc.shared.new_default::()?; + + let notifier: Arc> = Arc::new( + alloc.private.new_init( + /*try_*/ init!(fw::event::Notifier::ver { threshold }), + |inner, _p| { + try_init!(fw::event::raw::Notifier::ver { + threshold: inner.threshold.gpu_pointer(), + generation: AtomicU32::new(id as u32), + cur_count: AtomicU32::new(0), + unk_10: AtomicU32::new(0x50), + state: Default::default() + }) + }, + )?, + GFP_KERNEL, + )?; + + // Priorities are handled by the AGX scheduler, there is no meaning within a + // per-queue scheduler. Use a single run queue wth Kernel priority. + let sched = + sched::Scheduler::new(dev.as_ref(), 1, WQ_SIZE, 0, 100000, c_str!("asahi_sched"))?; + let entity = sched::Entity::new(&sched, sched::Priority::Kernel)?; + + let buffer = + buffer::Buffer::ver::new(&*(*dev).gpu, alloc, ualloc.clone(), ualloc_priv, mgr)?; + + let mut ret = Queue::ver { + dev: dev.into(), + _sched: sched, + entity, + vm, + q_vtx: None, + q_frag: None, + q_comp: None, + fence_ctx: FenceContexts::new(1, QUEUE_NAME, QUEUE_CLASS_KEY)?, + inner: QueueInner::ver { + dev: dev.into(), + ualloc, + gpu_context: Arc::new( + workqueue::GpuContext::new(dev, alloc, buffer.any_ref())?, + GFP_KERNEL, + )?, + + buffer, + notifier_list: Arc::new(notifier_list, GFP_KERNEL)?, + notifier, + usc_exec_base, + id, + #[ver(V >= V13_0B4)] + counter: AtomicU64::new(0), + }, + }; + + // Rendering structures + let tvb_blocks = *module_parameters::initial_tvb_size.value(); + + ret.inner.buffer.ensure_blocks(tvb_blocks)?; + + ret.q_vtx = Some(SubQueue::ver { + wq: workqueue::WorkQueue::ver::new( + dev, + alloc, + event_manager.clone(), + ret.inner.gpu_context.clone(), + ret.inner.notifier_list.clone(), + channel::PipeType::Vertex, + id, + priority, + WQ_SIZE, + )?, + }); + + ret.q_frag = Some(SubQueue::ver { + wq: workqueue::WorkQueue::ver::new( + dev, + alloc, + event_manager.clone(), + ret.inner.gpu_context.clone(), + ret.inner.notifier_list.clone(), + channel::PipeType::Fragment, + id, + priority, + WQ_SIZE, + )?, + }); + + // Compute structures + ret.q_comp = Some(SubQueue::ver { + wq: workqueue::WorkQueue::ver::new( + dev, + alloc, + event_manager, + ret.inner.gpu_context.clone(), + ret.inner.notifier_list.clone(), + channel::PipeType::Compute, + id, + priority, + WQ_SIZE, + )?, + }); + + mod_dev_dbg!(dev, "[Queue {}] Queue created\n", id); + Ok(ret) + } +} + +const SQ_RENDER: usize = 0; +const SQ_COMPUTE: usize = 1; +const SQ_COUNT: usize = 2; + +// SAFETY: All bit patterns are valid by construction. +unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_header {} +unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_render {} +unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_compute {} +unsafe impl AnyBitPattern for uapi::drm_asahi_attachment {} + +fn build_attachments(reader: &mut Reader<'_>, size: usize) -> Result { + const STRIDE: usize = core::mem::size_of::(); + let count = size / STRIDE; + + if count > microseq::MAX_ATTACHMENTS { + return Err(EINVAL); + } + + let mut attachments: microseq::Attachments = Default::default(); + attachments.count = count as u32; + + for i in 0..count { + let att: uapi::drm_asahi_attachment = reader.read()?; + + if att.flags != 0 || att.pad != 0 { + return Err(EINVAL); + } + + // Some kind of power-of-2 exponent related to attachment size, in + // bounds [1, 6]? We don't know what this is exactly yet. + let unk_e = 1; + + let cache_lines = (att.size + 127) >> 7; + attachments.list[i as usize] = microseq::Attachment { + address: U64(att.pointer), + size: cache_lines.try_into()?, + unk_c: 0x17, + unk_e: unk_e as u16, + }; + } + + Ok(attachments) +} + +#[versions(AGX)] +impl Queue for Queue::ver { + fn submit( + &mut self, + id: u64, + mut syncs: KVec, + in_sync_count: usize, + cmdbuf_raw: &[u8], + objects: Pin<&xarray::XArray>>, + ) -> Result { + let gpu = match (*self.dev) + .gpu + .clone() + .arc_as_any() + .downcast::() + { + Ok(gpu) => gpu, + Err(_) => { + dev_crit!(self.dev.as_ref(), "GpuManager mismatched with JobImpl!\n"); + return Err(EIO); + } + }; + + mod_dev_dbg!(self.dev, "[Submission {}] Submit job\n", id); + + if gpu.is_crashed() { + dev_err!( + self.dev.as_ref(), + "[Submission {}] GPU is crashed, cannot submit\n", + id + ); + return Err(ENODEV); + } + + let op_guard = if in_sync_count > 0 { + Some(gpu.start_op()?) + } else { + None + }; + + let mut events: [KVec>; SQ_COUNT] = + Default::default(); + + events[SQ_RENDER].push( + self.q_frag.as_ref().and_then(|a| a.wq.event_info()), + GFP_KERNEL, + )?; + events[SQ_COMPUTE].push( + self.q_comp.as_ref().and_then(|a| a.wq.event_info()), + GFP_KERNEL, + )?; + + let vm_bind = gpu.bind_vm(&self.vm)?; + let vm_slot = vm_bind.slot(); + + mod_dev_dbg!(self.dev, "[Submission {}] Creating job\n", id); + + // FIXME: I think this can violate the fence seqno ordering contract. + // If we have e.g. a render submission with no barriers and then a compute submission + // with no barriers, it's possible for the compute submission to complete first, and + // therefore its fence. Maybe we should have separate fence contexts for render + // and compute, and then do a ? (Vert+frag should be fine since there is no vert + // without frag, and frag always serializes.) + let fence: UserFence = self + .fence_ctx + .new_fence::( + 0, + JobFence::ver { + id, + pending: Default::default(), + }, + )? + .into(); + + let mut cmdbuf = Reader::new(cmdbuf_raw); + + // First, parse the headers to determine the number of compute/render + // commands. This will be used to determine when to flush stamps. + // + // We also use it to determine how many notifications the job will + // generate. We could calculate that in the second pass since we don't + // need until much later, but it's convenient to gather everything at + // the same time. + let mut nr_commands = 0; + let mut last_compute = 0; + let mut last_render = 0; + let mut nr_render = 0; + let mut nr_compute = 0; + + while !cmdbuf.is_empty() { + let header: uapi::drm_asahi_cmd_header = cmdbuf.read()?; + cmdbuf.skip(header.size as usize); + nr_commands += 1; + + match header.cmd_type as u32 { + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { + last_compute = nr_commands; + nr_render += 1; + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { + last_render = nr_commands; + nr_compute += 1; + } + _ => {} + } + } + + let mut job = self.entity.new_job( + 1, + QueueJob::ver { + dev: self.dev.clone(), + vm_bind, + op_guard, + sj_vtx: self + .q_vtx + .as_mut() + .map(|a| a.new_job(Fence::from_fence(&fence))), + sj_frag: self + .q_frag + .as_mut() + .map(|a| a.new_job(Fence::from_fence(&fence))), + sj_comp: self + .q_comp + .as_mut() + .map(|a| a.new_job(Fence::from_fence(&fence))), + fence, + notifier: self.inner.notifier.clone(), + + // Each render command generates 2 notifications: 1 for the + // vertex part, 1 for the fragment part. Each compute command + // generates 1 notification. Sum up to calculate the total + // notification count for the job. + notification_count: (2 * nr_render) + nr_compute, + + did_run: false, + id, + }, + )?; + + mod_dev_dbg!( + self.dev, + "[Submission {}] Adding {} in_syncs\n", + id, + in_sync_count + ); + for sync in syncs.drain(0..in_sync_count) { + if let Some(fence) = sync.fence { + job.add_dependency(fence)?; + } + } + + // Validate the number of hardware commands, ignoring software commands + let nr_hw_commands = nr_render + nr_compute; + if nr_hw_commands == 0 || nr_hw_commands > MAX_COMMANDS_PER_SUBMISSION { + cls_pr_debug!( + Errors, + "submit: Command count {} out of valid range [1, {}]\n", + nr_hw_commands, + MAX_COMMANDS_PER_SUBMISSION - 1 + ); + return Err(EINVAL); + } + + cmdbuf.rewind(); + + let mut command_index = 0; + let mut vertex_attachments: microseq::Attachments = Default::default(); + let mut fragment_attachments: microseq::Attachments = Default::default(); + let mut compute_attachments: microseq::Attachments = Default::default(); + + // Parse the full command buffer submitting as we go + while !cmdbuf.is_empty() { + let header: uapi::drm_asahi_cmd_header = cmdbuf.read()?; + let header_size = header.size as usize; + + // Pre-increment command index to match last_compute/last_render + command_index += 1; + + for (queue_idx, index) in [header.vdm_barrier, header.cdm_barrier].iter().enumerate() { + if *index == uapi::DRM_ASAHI_BARRIER_NONE as u16 { + continue; + } + if let Some(event) = events[queue_idx].get(*index as usize).ok_or_else(|| { + cls_pr_debug!(Errors, "Invalid barrier #{}: {}\n", queue_idx, index); + EINVAL + })? { + let mut alloc = gpu.alloc(); + let queue_job = match header.cmd_type as u32 { + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => job.get_vtx()?, + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => job.get_comp()?, + _ => return Err(EINVAL), + }; + mod_dev_dbg!(self.dev, "[Submission {}] Create Explicit Barrier\n", id); + let barrier = alloc.private.new_init( + pin_init::zeroed::(), + |_inner, _p| { + let queue_job = &queue_job; + try_init!(fw::workqueue::raw::Barrier { + tag: fw::workqueue::CommandType::Barrier, + wait_stamp: event.fw_stamp_pointer, + wait_value: event.value, + wait_slot: event.slot, + stamp_self: queue_job.event_info().value.next(), + uuid: 0xffffbbbb, + external_barrier: 0, + internal_barrier_type: 1, + padding: Default::default(), + }) + }, + )?; + mod_dev_dbg!(self.dev, "[Submission {}] Add Explicit Barrier\n", id); + queue_job.add(barrier, vm_slot)?; + } else { + assert!(*index == 0); + } + } + + match header.cmd_type as u32 { + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { + let render: uapi::drm_asahi_cmd_render = cmdbuf.read_up_to(header_size)?; + + self.inner.submit_render( + &mut job, + &render, + &vertex_attachments, + &fragment_attachments, + objects, + id, + command_index == last_render, + )?; + events[SQ_RENDER].push( + Some( + job.sj_frag + .as_ref() + .expect("No frag queue?") + .job + .as_ref() + .expect("No frag job?") + .event_info(), + ), + GFP_KERNEL, + )?; + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { + let compute: uapi::drm_asahi_cmd_compute = cmdbuf.read_up_to(header_size)?; + + self.inner.submit_compute( + &mut job, + &compute, + &compute_attachments, + objects, + id, + command_index == last_compute, + )?; + events[SQ_COMPUTE].push( + Some( + job.sj_comp + .as_ref() + .expect("No comp queue?") + .job + .as_ref() + .expect("No comp job?") + .event_info(), + ), + GFP_KERNEL, + )?; + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_VERTEX_ATTACHMENTS => { + vertex_attachments = build_attachments(&mut cmdbuf, header_size)?; + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_FRAGMENT_ATTACHMENTS => { + fragment_attachments = build_attachments(&mut cmdbuf, header_size)?; + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_COMPUTE_ATTACHMENTS => { + compute_attachments = build_attachments(&mut cmdbuf, header_size)?; + } + _ => { + cls_pr_debug!(Errors, "Unknown command type {}\n", header.cmd_type); + return Err(EINVAL); + } + } + } + + mod_dev_dbg!( + self.dev, + "Queue {}: Committing job {}\n", + self.inner.id, + job.id + ); + job.commit()?; + + mod_dev_dbg!(self.dev, "Queue {}: Arming job {}\n", self.inner.id, job.id); + let mut job = job.arm(); + let out_fence = job.fences().finished(); + mod_dev_dbg!( + self.dev, + "Queue {}: Pushing job {}\n", + self.inner.id, + job.id + ); + job.push(); + + mod_dev_dbg!( + self.dev, + "Queue {}: Adding {} out_syncs\n", + self.inner.id, + syncs.len() + ); + for mut sync in syncs { + if let Some(chain) = sync.chain_fence.take() { + sync.syncobj + .add_point(chain, &out_fence, sync.timeline_value); + } else { + sync.syncobj.replace_fence(Some(&out_fence)); + } + } + + Ok(()) + } +} + +#[versions(AGX)] +impl Drop for Queue::ver { + fn drop(&mut self) { + mod_dev_dbg!(self.dev, "[Queue {}] Dropping queue\n", self.inner.id); + } +} diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs new file mode 100644 index 00000000000000..32273b2975b505 --- /dev/null +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -0,0 +1,1400 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![allow(clippy::unusual_byte_groupings)] + +//! Render work queue. +//! +//! A render queue consists of two underlying WorkQueues, one for vertex and one for fragment work. +//! This module is in charge of creating all of the firmware structures required to submit 3D +//! rendering work to the GPU, based on the userspace command buffer. + +use super::common; +use crate::alloc::Allocator; +use crate::debug::*; +use crate::fw::types::*; +use crate::gpu::GpuManager; +use crate::util::*; +use crate::{ + buffer, + file, + fw, + gpu, + microseq, // +}; +use crate::{ + inner_ptr, + inner_weak_ptr, // +}; +use core::sync::atomic::Ordering; +use kernel::dma_fence::RawDmaFence; +use kernel::drm::sched::Job; +use kernel::prelude::*; +use kernel::sync::Arc; +use kernel::uapi; +use kernel::xarray; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Render; + +/// Tiling/Vertex control bit to disable using more than one GPU cluster. This results in decreased +/// throughput but also less latency, which is probably desirable for light vertex loads where the +/// overhead of clustering/merging would exceed the time it takes to just run the job on one +/// cluster. +const TILECTL_DISABLE_CLUSTERING: u32 = 1u32 << 0; + +#[versions(AGX)] +impl super::QueueInner::ver { + /// Get the appropriate tiling parameters for a given userspace command buffer. + fn get_tiling_params( + cmdbuf: &uapi::drm_asahi_cmd_render, + num_clusters: u32, + ) -> Result { + let width: u32 = cmdbuf.width_px as u32; + let height: u32 = cmdbuf.height_px as u32; + let layers: u32 = cmdbuf.layers as u32; + + if layers == 0 || layers > 2048 { + cls_pr_debug!(Errors, "Layer count invalid ({})\n", layers); + return Err(EINVAL); + } + + // This is overflow safe: all these calculations are done in u32. + // At 64Kx64K max dimensions above, this is 2**32 pixels max. + // In terms of tiles that are always larger than one pixel, + // this can never overflow. Note that real actual dimensions + // are limited to 16K * 16K below anyway. + // + // Once we multiply by the layer count, then we need to check + // for overflow or use u64. + + let tile_width = 32u32; + let tile_height = 32u32; + + let utile_width = cmdbuf.utile_width_px as u32; + let utile_height = cmdbuf.utile_height_px as u32; + + match (utile_width, utile_height) { + (32, 32) | (32, 16) | (16, 16) => (), + _ => { + cls_pr_debug!( + Errors, + "uTile size invalid ({} x {})\n", + utile_width, + utile_height + ); + return Err(EINVAL); + } + }; + + let utiles_per_tile_x = tile_width / utile_width; + let utiles_per_tile_y = tile_height / utile_height; + + let utiles_per_tile = utiles_per_tile_x * utiles_per_tile_y; + + let tiles_x = width.div_ceil(tile_width); + let tiles_y = height.div_ceil(tile_height); + let tiles = tiles_x * tiles_y; + + let mtiles_x = 4u32; + let mtiles_y = 4u32; + let mtiles = mtiles_x * mtiles_y; + + let tiles_per_mtile_x = align(tiles_x.div_ceil(mtiles_x), 4); + let tiles_per_mtile_y = align(tiles_y.div_ceil(mtiles_y), 4); + let tiles_per_mtile = tiles_per_mtile_x * tiles_per_mtile_y; + + let mtile_x1 = tiles_per_mtile_x; + let mtile_x2 = 2 * tiles_per_mtile_x; + let mtile_x3 = 3 * tiles_per_mtile_x; + + let mtile_y1 = tiles_per_mtile_y; + let mtile_y2 = 2 * tiles_per_mtile_y; + let mtile_y3 = 3 * tiles_per_mtile_y; + + let rgn_entry_size = 5; + // Macrotile stride in 32-bit words + let rgn_size = align(rgn_entry_size * tiles_per_mtile * utiles_per_tile, 4) / 4; + let tilemap_size = (4 * rgn_size * mtiles) as usize * layers as usize; + + let tpc_entry_size = 8; + // TPC stride in 32-bit words + let tpc_mtile_stride = tpc_entry_size * utiles_per_tile * tiles_per_mtile / 4; + let tpc_size = + (4 * tpc_mtile_stride * mtiles) as usize * layers as usize * num_clusters as usize; + + // No idea where this comes from, but it fits what macOS does... + // GUESS: Number of 32K heap blocks to fit a 5-byte region header/pointer per tile? + // That would make a ton of sense... + let meta1_layer_stride = if num_clusters > 1 { + (align(tiles_x, 2) * align(tiles_y, 4) * utiles_per_tile).div_ceil(0x1980) + } else { + 0 + }; + + let mut min_tvb_blocks = align((tiles_x * tiles_y).div_ceil(128), 8); + + if num_clusters > 1 { + min_tvb_blocks = min_tvb_blocks.max(7 + 2 * layers); + } + + Ok(buffer::TileInfo { + tiles_x, + tiles_y, + tiles, + utile_width, + utile_height, + //mtiles_x, + //mtiles_y, + tiles_per_mtile_x, + tiles_per_mtile_y, + //tiles_per_mtile, + utiles_per_mtile_x: tiles_per_mtile_x * utiles_per_tile_x, + utiles_per_mtile_y: tiles_per_mtile_y * utiles_per_tile_y, + //utiles_per_mtile: tiles_per_mtile * utiles_per_tile, + tilemap_size, + tpc_size, + meta1_layer_stride, + #[ver(G < G14X)] + meta1_blocks: meta1_layer_stride * (cmdbuf.layers as u32), + #[ver(G >= G14X)] + meta1_blocks: meta1_layer_stride, + layermeta_size: if layers > 1 { 0x100 } else { 0 }, + min_tvb_blocks: min_tvb_blocks as usize, + params: fw::vertex::raw::TilingParameters { + rgn_size, + unk_4: 0x88, + ppp_ctrl: cmdbuf.ppp_ctrl, + x_max: (width - 1) as u16, + y_max: (height - 1) as u16, + te_screen: ((tiles_y - 1) << 12) | (tiles_x - 1), + te_mtile1: mtile_x3 | (mtile_x2 << 9) | (mtile_x1 << 18), + te_mtile2: mtile_y3 | (mtile_y2 << 9) | (mtile_y1 << 18), + tiles_per_mtile, + tpc_stride: tpc_mtile_stride, + unk_24: 0x100, + unk_28: if layers > 1 { + 0xe000 | (layers - 1) + } else { + 0x8000 + }, + helper_cfg: cmdbuf.vertex_helper.cfg, + __pad: Default::default(), + }, + }) + } + + /// Submit work to a render queue. + pub(super) fn submit_render( + &self, + job: &mut Job, + cmdbuf: &uapi::drm_asahi_cmd_render, + vertex_attachments: µseq::Attachments, + fragment_attachments: µseq::Attachments, + objects: Pin<&xarray::XArray>>, + id: u64, + flush_stamps: bool, + ) -> Result { + mod_dev_dbg!(self.dev, "[Submission {}] Render!\n", id); + + if cmdbuf.flags + & !(uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_VERTEX_SCRATCH + | uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES + | uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_NO_VERTEX_CLUSTERING + | uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT) as u32 + != 0 + { + cls_pr_debug!(Errors, "Invalid flags ({:#x})\n", cmdbuf.flags); + return Err(EINVAL); + } + + if cmdbuf.width_px == 0 + || cmdbuf.height_px == 0 + || cmdbuf.width_px > 16384 + || cmdbuf.height_px > 16384 + { + cls_pr_debug!( + Errors, + "Invalid dimensions ({}x{})\n", + cmdbuf.width_px, + cmdbuf.height_px + ); + return Err(EINVAL); + } + + let mut vtx_user_timestamps: fw::job::UserTimestamps = Default::default(); + let mut frg_user_timestamps: fw::job::UserTimestamps = Default::default(); + + vtx_user_timestamps.start = common::get_timestamp_object(objects, cmdbuf.ts_vtx.start)?; + vtx_user_timestamps.end = common::get_timestamp_object(objects, cmdbuf.ts_vtx.end)?; + frg_user_timestamps.start = common::get_timestamp_object(objects, cmdbuf.ts_frag.start)?; + frg_user_timestamps.end = common::get_timestamp_object(objects, cmdbuf.ts_frag.end)?; + + let gpu = match (*self.dev) + .gpu + .as_any() + .downcast_ref::() + { + Some(gpu) => gpu, + None => { + dev_crit!(self.dev.as_ref(), "GpuManager mismatched with Queue!\n"); + return Err(EIO); + } + }; + + let nclusters = gpu.get_dyncfg().id.num_clusters; + + // Can be set to false to disable clustering (for simpler jobs), but then the + // core masks below should be adjusted to cover a single rolling cluster. + let mut clustering = nclusters > 1; + + if debug_enabled(debug::DebugFlags::DisableClustering) + || cmdbuf.flags + & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_NO_VERTEX_CLUSTERING as u32 + != 0 + { + clustering = false; + } + + #[ver(G != G14)] + let tiling_control = { + let render_cfg = gpu.get_cfg().render; + let mut tiling_control = render_cfg.tiling_control; + + if !clustering { + tiling_control |= TILECTL_DISABLE_CLUSTERING; + } + tiling_control + }; + + let mut alloc = gpu.alloc(); + let kalloc = &mut *alloc; + + // This sequence number increases per new client/VM? assigned to some slot, + // but it's unclear *which* slot... + let slot_client_seq: u8 = (self.id & 0xff) as u8; + + let tile_info = Self::get_tiling_params(&cmdbuf, if clustering { nclusters } else { 1 })?; + + let buffer = &self.buffer; + let notifier = self.notifier.clone(); + + let tvb_autogrown = buffer.auto_grow()?; + if tvb_autogrown { + let new_size = buffer.block_count() as usize; + cls_dev_dbg!( + TVBStats, + &self.dev, + "[Submission {}] TVB grew to {} bytes ({} blocks) due to overflows\n", + id, + new_size * buffer::BLOCK_SIZE, + new_size, + ); + } + + let tvb_grown = buffer.ensure_blocks(tile_info.min_tvb_blocks)?; + if tvb_grown { + cls_dev_dbg!( + TVBStats, + &self.dev, + "[Submission {}] TVB grew to {} bytes ({} blocks) due to dimensions ({}x{})\n", + id, + tile_info.min_tvb_blocks * buffer::BLOCK_SIZE, + tile_info.min_tvb_blocks, + cmdbuf.width_px, + cmdbuf.height_px + ); + } + + let scene = Arc::new(buffer.new_scene(kalloc, &tile_info)?, GFP_KERNEL)?; + + let vm_bind = job.vm_bind.clone(); + + mod_dev_dbg!( + self.dev, + "[Submission {}] VM slot = {}\n", + id, + vm_bind.slot() + ); + + let ev_vtx = job.get_vtx()?.event_info(); + let ev_frag = job.get_frag()?.event_info(); + + mod_dev_dbg!( + self.dev, + "[Submission {}] Vert event #{} -> {:#x?}\n", + id, + ev_vtx.slot, + ev_vtx.value.next(), + ); + mod_dev_dbg!( + self.dev, + "[Submission {}] Frag event #{} -> {:#x?}\n", + id, + ev_frag.slot, + ev_frag.value.next(), + ); + + let uuid_3d = 0; + let uuid_ta = 0; + + mod_dev_dbg!( + self.dev, + "[Submission {}] Vert UUID = {:#x?}\n", + id, + uuid_ta + ); + mod_dev_dbg!( + self.dev, + "[Submission {}] Frag UUID = {:#x?}\n", + id, + uuid_3d + ); + + let fence = job.fence.clone(); + let frag_job = job.get_frag()?; + + mod_dev_dbg!(self.dev, "[Submission {}] Create Barrier\n", id); + let barrier = kalloc.private.new_init( + pin_init::zeroed::(), + |_inner, _p| { + try_init!(fw::workqueue::raw::Barrier { + tag: fw::workqueue::CommandType::Barrier, + wait_stamp: ev_vtx.fw_stamp_pointer, + wait_value: ev_vtx.value.next(), + wait_slot: ev_vtx.slot, + stamp_self: ev_frag.value.next(), + uuid: uuid_3d, + external_barrier: 0, + internal_barrier_type: 0, + padding: Default::default(), + }) + }, + )?; + + mod_dev_dbg!(self.dev, "[Submission {}] Add Barrier\n", id); + frag_job.add(barrier, vm_bind.slot())?; + + let timestamps = Arc::new( + kalloc.shared.new_default::()?, + GFP_KERNEL, + )?; + + let unk1 = false; + + let mut tile_config: u64 = 0; + if !unk1 { + tile_config |= 0x280; + } + if cmdbuf.layers > 1 { + tile_config |= 1; + } + if cmdbuf.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES as u32 + != 0 + { + tile_config |= 0x10000; + } + + let samples_log2 = match cmdbuf.samples { + 1 => 0, + 2 => 1, + 4 => 2, + _ => { + cls_pr_debug!(Errors, "Invalid sample count {}\n", cmdbuf.samples); + return Err(EINVAL); + } + }; + + let utile_config = ((tile_info.utile_width / 16) << 12) + | ((tile_info.utile_height / 16) << 14) + | samples_log2; + + // Calculate the number of 2KiB blocks to allocate per utile. This is + // just a bit of dimensional analysis. + let pixels_per_utile: u32 = + (cmdbuf.utile_width_px as u32) * (cmdbuf.utile_height_px as u32); + let samples_per_utile: u32 = pixels_per_utile << samples_log2; + let utile_size_bytes: u32 = (cmdbuf.sample_size_B as u32) * samples_per_utile; + let block_size_bytes: u32 = 2048; + let blocks_per_utile: u32 = utile_size_bytes.div_ceil(block_size_bytes); + + #[ver(G >= G14X)] + let frg_tilecfg = 0x0000000_00036011 + | (((tile_info.tiles_x - 1) as u64) << 44) + | (((tile_info.tiles_y - 1) as u64) << 53) + | (if unk1 { 0 } else { 0x20_00000000 }) + | (if cmdbuf.layers > 1 { 0x1_00000000 } else { 0 }) + | ((utile_config as u64 & 0xf000) << 28); + + // TODO: check + #[ver(V >= V13_0B4)] + let count_frag = self.counter.fetch_add(2, Ordering::Relaxed); + #[ver(V >= V13_0B4)] + let count_vtx = count_frag + 1; + + // Unknowns handling + + #[ver(G >= G14)] + let g14_unk = 0x4040404; + #[ver(G < G14)] + let g14_unk = 0; + #[ver(G < G14X)] + let frg_unk_140 = 0x8c60; + let frg_unk_158 = 0x1c; + #[ver(G >= G14)] + let load_bgobjvals = cmdbuf.isp_bgobjvals as u64; + #[ver(G < G14)] + let load_bgobjvals = cmdbuf.isp_bgobjvals as u64 | 0x400; + let reload_zlsctrl = cmdbuf.zls_ctrl; + let iogpu_unk54 = 0x3a0012006b0003; + let iogpu_unk56 = 1; + #[ver(G < G14)] + let tiling_control_2 = 0; + #[ver(G >= G14X)] + let tiling_control_2 = 4; + #[ver(G >= G14X)] + let vtx_unk_f0 = 0x1c; + #[ver(G < G14)] + let vtx_unk_f0 = 0x1c + (align(tile_info.meta1_blocks, 4) as u64); + let vtx_unk_118 = 0x1c; + + // DRM_ASAHI_RENDER_DBIAS_IS_INT chosen to match hardware bit. + let isp_ctl = 0xc000u32 + | (cmdbuf.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT as u32); + + // Always allow preemption at the UAPI level + let no_preemption = false; + + mod_dev_dbg!(self.dev, "[Submission {}] Create Frag\n", id); + let frag = GpuObject::new_init_prealloc( + kalloc.gpu_ro.alloc_object()?, + |ptr: GpuWeakPointer| { + let scene = scene.clone(); + let notifier = notifier.clone(); + let vm_bind = vm_bind.clone(); + let timestamps = timestamps.clone(); + let private = &mut kalloc.private; + try_init!(fw::fragment::RunFragment::ver { + micro_seq: { + let mut builder = microseq::Builder::new(); + + let stats = inner_weak_ptr!( + gpu.initdata.runtime_pointers.stats.frag.weak_pointer(), + stats + ); + + let start_frag = builder.add(microseq::StartFragment::ver { + header: microseq::op::StartFragment::HEADER, + #[ver(G < G14X)] + job_params2: Some(inner_weak_ptr!(ptr, job_params2)), + #[ver(G < G14X)] + job_params1: Some(inner_weak_ptr!(ptr, job_params1)), + #[ver(G >= G14X)] + job_params1: None, + #[ver(G >= G14X)] + job_params2: None, + #[ver(G >= G14X)] + registers: inner_weak_ptr!(ptr, registers), + scene: scene.gpu_pointer(), + stats, + busy_flag: inner_weak_ptr!(ptr, busy_flag), + tvb_overflow_count: inner_weak_ptr!(ptr, tvb_overflow_count), + unk_pointer: inner_weak_ptr!(ptr, unk_pointee), + work_queue: ev_frag.info_ptr, + work_item: ptr, + vm_slot: vm_bind.slot(), + unk_50: 0x1, // fixed + event_generation: self.id as u32, + buffer_slot: scene.slot(), + sync_grow: 0, + event_seq: U64(ev_frag.event_seq), + unk_68: 0, + unk_758_flag: inner_weak_ptr!(ptr, unk_758_flag), + unk_job_buf: inner_weak_ptr!(ptr, unk_buf_0), + #[ver(V >= V13_3)] + unk_7c_0: U64(0), + unk_7c: 0, + unk_80: 0, + unk_84: unk1.into(), + uuid: uuid_3d, + attachments: *fragment_attachments, + padding: 0, + #[ver(V >= V13_0B4)] + counter: U64(count_frag), + #[ver(V >= V13_0B4)] + notifier_buf: inner_weak_ptr!(notifier.weak_pointer(), state.unk_buf), + })?; + + if frg_user_timestamps.any() { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(true), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), + work_queue: ev_frag.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_3d, + unk_30_padding: 0, + })?; + } + + #[ver(G < G14X)] + builder.add(microseq::WaitForIdle { + header: microseq::op::WaitForIdle::new(microseq::Pipe::Fragment), + })?; + #[ver(G >= G14X)] + builder.add(microseq::WaitForIdle2 { + header: microseq::op::WaitForIdle2::HEADER, + })?; + + if frg_user_timestamps.any() { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(false), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), + work_queue: ev_frag.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_3d, + unk_30_padding: 0, + })?; + } + + let off = builder.offset_to(start_frag); + builder.add(microseq::FinalizeFragment::ver { + header: microseq::op::FinalizeFragment::HEADER, + uuid: uuid_3d, + unk_8: 0, + fw_stamp: ev_frag.fw_stamp_pointer, + stamp_value: ev_frag.value.next(), + unk_18: 0, + scene: scene.weak_pointer(), + buffer: scene.weak_buffer_pointer(), + unk_2c: U64(1), + stats, + unk_pointer: inner_weak_ptr!(ptr, unk_pointee), + busy_flag: inner_weak_ptr!(ptr, busy_flag), + work_queue: ev_frag.info_ptr, + work_item: ptr, + vm_slot: vm_bind.slot(), + unk_60: 0, + unk_758_flag: inner_weak_ptr!(ptr, unk_758_flag), + #[ver(V >= V13_3)] + unk_6c_0: U64(0), + unk_6c: U64(0), + unk_74: U64(0), + unk_7c: U64(0), + unk_84: U64(0), + unk_8c: U64(0), + #[ver(G == G14 && V < V13_0B4)] + unk_8c_g14: U64(0), + restart_branch_offset: off, + has_attachments: (fragment_attachments.count > 0) as u32, + #[ver(V >= V13_0B4)] + unk_9c: Default::default(), + })?; + + builder.add(microseq::RetireStamp { + header: microseq::op::RetireStamp::HEADER, + })?; + + builder.build(private)? + }, + notifier, + scene, + vm_bind, + aux_fb: self.ualloc.lock().array_empty_tagged(0x8000, b"AXFB")?, + timestamps, + user_timestamps: frg_user_timestamps, + }) + }, + |inner, _ptr| { + let vm_slot = vm_bind.slot(); + let aux_fb_info = fw::fragment::raw::AuxFBInfo::ver { + isp_ctl: isp_ctl, + unk2: 0, + width: cmdbuf.width_px as u32, + height: cmdbuf.height_px as u32, + #[ver(V >= V13_0B4)] + unk3: U64(0x100000), + }; + + try_init!(fw::fragment::raw::RunFragment::ver { + tag: fw::workqueue::CommandType::RunFragment, + #[ver(V >= V13_0B4)] + counter: U64(count_frag), + vm_slot, + unk_8: 0, + microsequence: inner.micro_seq.gpu_pointer(), + microsequence_size: inner.micro_seq.len() as u32, + notifier: inner.notifier.gpu_pointer(), + buffer: inner.scene.buffer_pointer(), + scene: inner.scene.gpu_pointer(), + unk_buffer_buf: inner.scene.kernel_buffer_pointer(), + tvb_tilemap: inner.scene.tvb_tilemap_pointer(), + ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), + samples: cmdbuf.samples as u32, + tiles_per_mtile_y: tile_info.tiles_per_mtile_y as u16, + tiles_per_mtile_x: tile_info.tiles_per_mtile_x as u16, + unk_50: U64(0), + unk_58: U64(0), + isp_merge_upper_x: F32::from_bits(cmdbuf.isp_merge_upper_x), + isp_merge_upper_y: F32::from_bits(cmdbuf.isp_merge_upper_y), + unk_68: U64(0), + tile_count: U64(tile_info.tiles as u64), + #[ver(G < G14X)] + job_params1 <- try_init!(fw::fragment::raw::JobParameters1::ver { + utile_config, + unk_4: 0, + bg: fw::fragment::raw::BackgroundProgram { + rsrc_spec: U64(cmdbuf.bg.rsrc_spec as u64), + address: U64(cmdbuf.bg.usc as u64), + }, + ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), + isp_scissor_base: U64(cmdbuf.isp_scissor_base), + isp_dbias_base: U64(cmdbuf.isp_dbias_base), + isp_oclqry_base: U64(cmdbuf.isp_oclqry_base), + aux_fb_info, + isp_zls_pixels: U64(cmdbuf.isp_zls_pixels as u64), + zls_ctrl: U64(cmdbuf.zls_ctrl), + #[ver(G >= G14)] + unk_58_g14_0: U64(g14_unk), + #[ver(G >= G14)] + unk_58_g14_8: U64(0), + z_load: U64(cmdbuf.depth.base), + z_store: U64(cmdbuf.depth.base), + s_load: U64(cmdbuf.stencil.base), + s_store: U64(cmdbuf.stencil.base), + #[ver(G >= G14)] + unk_68_g14_0: Default::default(), + z_load_stride: U64(cmdbuf.depth.stride as u64), + z_store_stride: U64(cmdbuf.depth.stride as u64), + s_load_stride: U64(cmdbuf.stencil.stride as u64), + s_store_stride: U64(cmdbuf.stencil.stride as u64), + z_load_comp: U64(cmdbuf.depth.comp_base), + z_load_comp_stride: U64(cmdbuf.depth.comp_stride as u64), + z_store_comp: U64(cmdbuf.depth.comp_base), + z_store_comp_stride: U64(cmdbuf.depth.comp_stride as u64), + s_load_comp: U64(cmdbuf.stencil.comp_base), + s_load_comp_stride: U64(cmdbuf.stencil.comp_stride as u64), + s_store_comp: U64(cmdbuf.stencil.comp_base), + s_store_comp_stride: U64(cmdbuf.stencil.comp_stride as u64), + tvb_tilemap: inner.scene.tvb_tilemap_pointer(), + tvb_layermeta: inner.scene.tvb_layermeta_pointer(), + mtile_stride_dwords: U64((4 * tile_info.params.rgn_size as u64) << 24), + tvb_heapmeta: inner.scene.tvb_heapmeta_pointer(), + tile_config: U64(tile_config), + aux_fb: inner.aux_fb.gpu_pointer(), + unk_108: Default::default(), + usc_exec_base_isp: U64(self.usc_exec_base), + unk_140: U64(frg_unk_140), + helper_program: cmdbuf.fragment_helper.binary, + unk_14c: 0, + helper_arg: U64(cmdbuf.fragment_helper.data), + unk_158: U64(frg_unk_158), + unk_160: U64(0), + __pad: Default::default(), + #[ver(V < V13_0B4)] + __pad1: Default::default(), + }), + #[ver(G < G14X)] + job_params2 <- try_init!(fw::fragment::raw::JobParameters2 { + eot_rsrc_spec: cmdbuf.eot.rsrc_spec, + eot_usc: cmdbuf.eot.usc, + unk_8: 0x0, + unk_c: 0x0, + isp_merge_upper_x: F32::from_bits(cmdbuf.isp_merge_upper_x), + isp_merge_upper_y: F32::from_bits(cmdbuf.isp_merge_upper_y), + unk_18: U64(0x0), + utiles_per_mtile_y: tile_info.utiles_per_mtile_y as u16, + utiles_per_mtile_x: tile_info.utiles_per_mtile_x as u16, + unk_24: 0x0, + tile_counts: ((tile_info.tiles_y - 1) << 12) | (tile_info.tiles_x - 1), + tib_blocks: blocks_per_utile, + isp_bgobjdepth: cmdbuf.isp_bgobjdepth, + // TODO: does this flag need to be exposed to userspace? + isp_bgobjvals: load_bgobjvals as u32, + unk_38: 0x0, + unk_3c: 0x1, + helper_cfg: cmdbuf.fragment_helper.cfg, + __pad: Default::default(), + }), + #[ver(G >= G14X)] + registers: fw::job::raw::RegisterArray::new( + inner_weak_ptr!(_ptr, registers.registers), + |r| { + r.add(0x1739, 1); + r.add(0x10009, utile_config.into()); + r.add(0x15379, cmdbuf.eot.rsrc_spec.into()); + r.add(0x15381, cmdbuf.eot.usc.into()); + r.add(0x15369, cmdbuf.bg.rsrc_spec.into()); + r.add(0x15371, cmdbuf.bg.usc.into()); + r.add(0x15131, cmdbuf.isp_merge_upper_x.into()); + r.add(0x15139, cmdbuf.isp_merge_upper_y.into()); + r.add(0x100a1, 0); + r.add(0x15069, 0); + r.add(0x15071, 0); // pointer + r.add(0x16058, 0); + r.add(0x10019, cmdbuf.ppp_multisamplectl); + let isp_mtile_size = (tile_info.utiles_per_mtile_y + | (tile_info.utiles_per_mtile_x << 16)) + .into(); + r.add(0x100b1, isp_mtile_size); // ISP_MTILE_SIZE + r.add(0x16030, isp_mtile_size); // ISP_MTILE_SIZE + r.add( + 0x100d9, + (((tile_info.tiles_y - 1) << 12) | (tile_info.tiles_x - 1)).into(), + ); // TE_SCREEN + r.add(0x16098, inner.scene.tvb_heapmeta_pointer().into()); + r.add(0x15109, cmdbuf.isp_scissor_base); // ISP_SCISSOR_BASE + r.add(0x15101, cmdbuf.isp_dbias_base); // ISP_DBIAS_BASE + r.add(0x15021, isp_ctl.into()); // aux_fb_info.unk_1 + r.add( + 0x15211, + ((cmdbuf.height_px as u64) << 32) | cmdbuf.width_px as u64, + ); // aux_fb_info.{width, heigh + r.add(0x15049, 0x100000); // s2.aux_fb_info.unk3 + r.add(0x10051, blocks_per_utile.into()); // s1.unk_2c + r.add(0x15321, cmdbuf.isp_zls_pixels.into()); // ISP_ZLS_PIXELS + r.add(0x15301, cmdbuf.isp_bgobjdepth.into()); // ISP_BGOBJDEPTH + r.add(0x15309, load_bgobjvals); // ISP_BGOBJVALS + r.add(0x15311, cmdbuf.isp_oclqry_base); // ISP_OCLQRY_BASE + r.add(0x15319, cmdbuf.zls_ctrl); // ISP_ZLSCTL + r.add(0x15349, g14_unk); // s2.unk_58_g14_0 + r.add(0x15351, 0); // s2.unk_58_g14_8 + r.add(0x15329, cmdbuf.depth.base); // ISP_ZLOAD_BASE + r.add(0x15331, cmdbuf.depth.base); // ISP_ZSTORE_BASE + r.add(0x15339, cmdbuf.stencil.base); // ISP_STENCIL_LOAD_BASE + r.add(0x15341, cmdbuf.stencil.base); // ISP_STENCIL_STORE_BASE + r.add(0x15231, 0); + r.add(0x15221, 0); + r.add(0x15239, 0); + r.add(0x15229, 0); + r.add(0x15401, cmdbuf.depth.stride as u64); // load + r.add(0x15421, cmdbuf.depth.stride as u64); // store + r.add(0x15409, cmdbuf.stencil.stride as u64); // load + r.add(0x15429, cmdbuf.stencil.stride as u64); + r.add(0x153c1, cmdbuf.depth.comp_base); // load + r.add(0x15411, cmdbuf.depth.comp_stride as u64); // load + r.add(0x153c9, cmdbuf.depth.comp_base); // store + r.add(0x15431, cmdbuf.depth.comp_stride as u64); // store + r.add(0x153d1, cmdbuf.stencil.comp_base); // load + r.add(0x15419, cmdbuf.stencil.comp_stride as u64); // load + r.add(0x153d9, cmdbuf.stencil.comp_base); // store + r.add(0x15439, cmdbuf.stencil.comp_stride as u64); // store + r.add(0x16429, inner.scene.tvb_tilemap_pointer().into()); + r.add(0x16060, inner.scene.tvb_layermeta_pointer().into()); + r.add(0x16431, (4 * tile_info.params.rgn_size as u64) << 24); // ISP_RGN? + r.add(0x10039, tile_config); // tile_config ISP_CTL? + r.add(0x16451, 0x0); // ISP_RENDER_ORIGIN + r.add(0x11821, cmdbuf.fragment_helper.binary.into()); + r.add(0x11829, cmdbuf.fragment_helper.data); + r.add(0x11f79, cmdbuf.fragment_helper.cfg.into()); + r.add(0x15359, 0); + r.add(0x10069, self.usc_exec_base); // frag; USC_EXEC_BASE_ISP + r.add(0x16020, 0); + r.add(0x16461, inner.aux_fb.gpu_pointer().into()); + r.add(0x16090, inner.aux_fb.gpu_pointer().into()); + r.add(0x120a1, frg_unk_158); + r.add(0x160a8, 0); + r.add(0x16068, frg_tilecfg); + r.add(0x160b8, 0x0); + /* + r.add(0x10201, 0x100); // Some kind of counter?? Does this matter? + r.add(0x10428, 0x100); // Some kind of counter?? Does this matter? + r.add(0x1c838, 1); // ? + r.add(0x1ca28, 0x1502960f00); // ?? + r.add(0x1731, 0x1); // ?? + */ + } + ), + job_params3 <- try_init!(fw::fragment::raw::JobParameters3::ver { + isp_dbias_base: fw::fragment::raw::ArrayAddr { + ptr: U64(cmdbuf.isp_dbias_base), + unk_padding: U64(0), + }, + isp_scissor_base: fw::fragment::raw::ArrayAddr { + ptr: U64(cmdbuf.isp_scissor_base), + unk_padding: U64(0), + }, + isp_oclqry_base: U64(cmdbuf.isp_oclqry_base), + unk_118: U64(0x0), + unk_120: Default::default(), + unk_partial_bg: fw::fragment::raw::BackgroundProgram { + rsrc_spec: U64(cmdbuf.partial_bg.rsrc_spec as u64), + address: U64(cmdbuf.partial_bg.usc as u64), + }, + unk_258: U64(0), + unk_260: U64(0), + unk_268: U64(0), + unk_270: U64(0), + partial_bg: fw::fragment::raw::BackgroundProgram { + rsrc_spec: U64(cmdbuf.partial_bg.rsrc_spec as u64), + address: U64(cmdbuf.partial_bg.usc as u64), + }, + zls_ctrl: U64(reload_zlsctrl), + unk_290: U64(g14_unk), + z_load: U64(cmdbuf.depth.base), + z_partial_stride: U64(cmdbuf.depth.stride as u64), + z_partial_comp_stride: U64(cmdbuf.depth.comp_stride as u64), + z_store: U64(cmdbuf.depth.base), + z_partial: U64(cmdbuf.depth.base), + z_partial_comp: U64(cmdbuf.depth.comp_base), + s_load: U64(cmdbuf.stencil.base), + s_partial_stride: U64(cmdbuf.stencil.stride as u64), + s_partial_comp_stride: U64(cmdbuf.stencil.comp_stride as u64), + s_store: U64(cmdbuf.stencil.base), + s_partial: U64(cmdbuf.stencil.base), + s_partial_comp: U64(cmdbuf.stencil.comp_base), + unk_2f8: Default::default(), + tib_blocks: blocks_per_utile, + unk_30c: 0x0, + aux_fb_info, + tile_config: U64(tile_config), + unk_328_padding: Default::default(), + unk_partial_eot: fw::fragment::raw::EotProgram::new( + cmdbuf.partial_eot.rsrc_spec, + cmdbuf.partial_eot.usc + ), + partial_eot: fw::fragment::raw::EotProgram::new( + cmdbuf.partial_eot.rsrc_spec, + cmdbuf.partial_eot.usc + ), + isp_bgobjdepth: cmdbuf.isp_bgobjdepth, + isp_bgobjvals: cmdbuf.isp_bgobjvals, + sample_size: cmdbuf.sample_size_B as u32, + unk_37c: 0x0, + unk_380: U64(0x0), + unk_388: U64(0x0), + #[ver(V >= V13_0B4)] + unk_390_0: U64(0x0), + isp_zls_pixels: U64(cmdbuf.isp_zls_pixels as u64), + }), + unk_758_flag: 0, + unk_75c_flag: 0, + unk_buf: Default::default(), + busy_flag: 0, + tvb_overflow_count: 0, + unk_878: 0, + encoder_params <- try_init!(fw::job::raw::EncoderParams { + // Maybe set when reloading z/s? + unk_8: 0, + sync_grow: 0, + unk_10: 0x0, // fixed + encoder_id: 0, + unk_18: 0x0, // fixed + unk_mask: 0xffffffffu32, + sampler_array: U64(cmdbuf.sampler_heap), + sampler_count: cmdbuf.sampler_count as u32, + sampler_max: (cmdbuf.sampler_count as u32) + 1, + }), + process_empty_tiles: (cmdbuf.flags + & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES as u32 + != 0) as u32, + // TODO: needs to be investigated + no_clear_pipeline_textures: 1, + // TODO: needs to be investigated + msaa_zs: 0, + unk_pointee: 0, + #[ver(V >= V13_3)] + unk_v13_3: 0, + meta <- try_init!(fw::job::raw::JobMeta { + unk_0: 0, + unk_2: 0, + no_preemption: no_preemption as u8, + stamp: ev_frag.stamp_pointer, + fw_stamp: ev_frag.fw_stamp_pointer, + stamp_value: ev_frag.value.next(), + stamp_slot: ev_frag.slot, + evctl_index: 0, // fixed + flush_stamps: flush_stamps as u32, + uuid: uuid_3d, + event_seq: ev_frag.event_seq as u32, + }), + unk_after_meta: unk1.into(), + unk_buf_0: U64(0), + unk_buf_8: U64(0), + #[ver(G < G14X)] + unk_buf_10: U64(1), + #[ver(G >= G14X)] + unk_buf_10: U64(0), + command_time: U64(0), + timestamp_pointers <- try_init!(fw::job::raw::TimestampPointers { + start_addr: Some(inner_ptr!(inner.timestamps.gpu_pointer(), frag.start)), + end_addr: Some(inner_ptr!(inner.timestamps.gpu_pointer(), frag.end)), + }), + user_timestamp_pointers: inner.user_timestamps.pointers()?, + client_sequence: slot_client_seq, + pad_925: Default::default(), + unk_928: 0, + unk_92c: 0, + #[ver(V >= V13_0B4)] + unk_ts: U64(0), + #[ver(V >= V13_0B4)] + unk_92d_8: Default::default(), + }) + }, + )?; + + mod_dev_dbg!(self.dev, "[Submission {}] Add Frag\n", id); + fence.add_command(); + + frag_job.add_cb(frag, vm_bind.slot(), move |error| { + if let Some(err) = error { + fence.set_error(err.into()); + } + + fence.command_complete(); + })?; + + let fence = job.fence.clone(); + let vtx_job = job.get_vtx()?; + + if scene.rebind() || tvb_grown || tvb_autogrown { + mod_dev_dbg!(self.dev, "[Submission {}] Create Bind Buffer\n", id); + let bind_buffer = kalloc.private.new_init( + { + let scene = scene.clone(); + try_init!(fw::buffer::InitBuffer::ver { scene }) + }, + |inner, _ptr| { + let vm_slot = vm_bind.slot(); + try_init!(fw::buffer::raw::InitBuffer::ver { + tag: fw::workqueue::CommandType::InitBuffer, + vm_slot, + buffer_slot: inner.scene.slot(), + unk_c: 0, + block_count: buffer.block_count(), + buffer: inner.scene.buffer_pointer(), + stamp_value: ev_vtx.value.next(), + }) + }, + )?; + + mod_dev_dbg!(self.dev, "[Submission {}] Add Bind Buffer\n", id); + vtx_job.add(bind_buffer, vm_bind.slot())?; + } + + mod_dev_dbg!(self.dev, "[Submission {}] Create Vertex\n", id); + let vtx = GpuObject::new_init_prealloc( + kalloc.gpu_ro.alloc_object()?, + |ptr: GpuWeakPointer| { + let scene = scene.clone(); + let vm_bind = vm_bind.clone(); + let timestamps = timestamps.clone(); + let private = &mut kalloc.private; + try_init!(fw::vertex::RunVertex::ver { + micro_seq: { + let mut builder = microseq::Builder::new(); + + let stats = inner_weak_ptr!( + gpu.initdata.runtime_pointers.stats.vtx.weak_pointer(), + stats + ); + + let start_vtx = builder.add(microseq::StartVertex::ver { + header: microseq::op::StartVertex::HEADER, + #[ver(G < G14X)] + tiling_params: Some(inner_weak_ptr!(ptr, tiling_params)), + #[ver(G < G14X)] + job_params1: Some(inner_weak_ptr!(ptr, job_params1)), + #[ver(G >= G14X)] + tiling_params: None, + #[ver(G >= G14X)] + job_params1: None, + #[ver(G >= G14X)] + registers: inner_weak_ptr!(ptr, registers), + buffer: scene.weak_buffer_pointer(), + scene: scene.weak_pointer(), + stats, + work_queue: ev_vtx.info_ptr, + vm_slot: vm_bind.slot(), + unk_38: 1, // fixed + event_generation: self.id as u32, + buffer_slot: scene.slot(), + unk_44: 0, + event_seq: U64(ev_vtx.event_seq), + unk_50: 0, + unk_pointer: inner_weak_ptr!(ptr, unk_pointee), + unk_job_buf: inner_weak_ptr!(ptr, unk_buf_0), + unk_64: 0x0, // fixed + unk_68: unk1.into(), + uuid: uuid_ta, + attachments: *vertex_attachments, + padding: 0, + #[ver(V >= V13_0B4)] + counter: U64(count_vtx), + #[ver(V >= V13_0B4)] + notifier_buf: inner_weak_ptr!(notifier.weak_pointer(), state.unk_buf), + #[ver(V < V13_0B4)] + unk_178: 0x0, // padding? + #[ver(V >= V13_0B4)] + unk_178: (!clustering) as u32, + })?; + + if vtx_user_timestamps.any() { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(true), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), + work_queue: ev_vtx.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_ta, + unk_30_padding: 0, + })?; + } + + #[ver(G < G14X)] + builder.add(microseq::WaitForIdle { + header: microseq::op::WaitForIdle::new(microseq::Pipe::Vertex), + })?; + #[ver(G >= G14X)] + builder.add(microseq::WaitForIdle2 { + header: microseq::op::WaitForIdle2::HEADER, + })?; + + if vtx_user_timestamps.any() { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(false), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), + work_queue: ev_vtx.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_ta, + unk_30_padding: 0, + })?; + } + + let off = builder.offset_to(start_vtx); + builder.add(microseq::FinalizeVertex::ver { + header: microseq::op::FinalizeVertex::HEADER, + scene: scene.weak_pointer(), + buffer: scene.weak_buffer_pointer(), + stats, + work_queue: ev_vtx.info_ptr, + vm_slot: vm_bind.slot(), + unk_28: 0x0, // fixed + unk_pointer: inner_weak_ptr!(ptr, unk_pointee), + unk_34: 0x0, // fixed + uuid: uuid_ta, + fw_stamp: ev_vtx.fw_stamp_pointer, + stamp_value: ev_vtx.value.next(), + unk_48: U64(0x0), // fixed + unk_50: 0x0, // fixed + unk_54: 0x0, // fixed + unk_58: U64(0x0), // fixed + unk_60: 0x0, // fixed + unk_64: 0x0, // fixed + unk_68: 0x0, // fixed + #[ver(G >= G14 && V < V13_0B4)] + unk_68_g14: U64(0), + restart_branch_offset: off, + has_attachments: (vertex_attachments.count > 0) as u32, + #[ver(V >= V13_0B4)] + unk_74: Default::default(), // Ventura + })?; + + builder.add(microseq::RetireStamp { + header: microseq::op::RetireStamp::HEADER, + })?; + builder.build(private)? + }, + notifier, + scene, + vm_bind, + timestamps, + user_timestamps: vtx_user_timestamps, + }) + }, + |inner, _ptr| { + let vm_slot = vm_bind.slot(); + #[ver(G < G14)] + let core_masks = gpu.core_masks_packed(); + + try_init!(fw::vertex::raw::RunVertex::ver { + tag: fw::workqueue::CommandType::RunVertex, + #[ver(V >= V13_0B4)] + counter: U64(count_vtx), + vm_slot, + unk_8: 0, + notifier: inner.notifier.gpu_pointer(), + buffer_slot: inner.scene.slot(), + unk_1c: 0, + buffer: inner.scene.buffer_pointer(), + scene: inner.scene.gpu_pointer(), + unk_buffer_buf: inner.scene.kernel_buffer_pointer(), + unk_34: 0, + #[ver(G < G14X)] + job_params1 <- try_init!(fw::vertex::raw::JobParameters1::ver { + unk_0: U64(if unk1 { 0 } else { 0x200 }), // sometimes 0 + unk_8: f32!(1e-20), // fixed + unk_c: f32!(1e-20), // fixed + tvb_tilemap: inner.scene.tvb_tilemap_pointer(), + #[ver(G < G14)] + tvb_cluster_tilemaps: inner.scene.cluster_tilemaps_pointer(), + tpc: inner.scene.tpc_pointer(), + tvb_heapmeta: inner.scene.tvb_heapmeta_pointer().or(0x8000_0000_0000_0000), + iogpu_unk_54: U64(iogpu_unk54), // fixed + iogpu_unk_56: U64(iogpu_unk56), // fixed + #[ver(G < G14)] + tvb_cluster_meta1: inner + .scene + .meta_1_pointer() + .map(|x| x.or((tile_info.meta1_layer_stride as u64) << 50)), + utile_config, + unk_4c: 0, + ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), // fixed + tvb_layermeta: inner.scene.tvb_layermeta_pointer(), + #[ver(G < G14)] + tvb_cluster_layermeta: inner.scene.tvb_cluster_layermeta_pointer(), + #[ver(G < G14)] + core_mask: Array::new([ + *core_masks.first().unwrap_or(&0), + *core_masks.get(1).unwrap_or(&0), + ]), + preempt_buf1: inner.scene.preempt_buf_1_pointer(), + preempt_buf2: inner.scene.preempt_buf_2_pointer(), + unk_80: U64(0x1), // fixed + preempt_buf3: inner.scene.preempt_buf_3_pointer().or(0x4_0000_0000_0000), // check + vdm_ctrl_stream_base: U64(cmdbuf.vdm_ctrl_stream_base), + #[ver(G < G14)] + tvb_cluster_meta2: inner.scene.meta_2_pointer(), + #[ver(G < G14)] + tvb_cluster_meta3: inner.scene.meta_3_pointer(), + #[ver(G < G14)] + tiling_control, + #[ver(G < G14)] + unk_ac: tiling_control_2 as u32, // fixed + unk_b0: Default::default(), // fixed + usc_exec_base_ta: U64(self.usc_exec_base), + #[ver(G < G14)] + tvb_cluster_meta4: inner + .scene + .meta_4_pointer() + .map(|x| x.or(0x3000_0000_0000_0000)), + #[ver(G < G14)] + unk_f0: U64(vtx_unk_f0), + unk_f8: U64(0x8c60), // fixed + helper_program: cmdbuf.vertex_helper.binary, + unk_104: 0, + helper_arg: U64(cmdbuf.vertex_helper.data), + unk_110: Default::default(), // fixed + unk_118: vtx_unk_118 as u32, // fixed + __pad: Default::default(), + }), + #[ver(G < G14X)] + tiling_params: tile_info.params, + #[ver(G >= G14X)] + registers: fw::job::raw::RegisterArray::new( + inner_weak_ptr!(_ptr, registers.registers), + |r| { + r.add(0x10141, if unk1 { 0 } else { 0x200 }); // s2.unk_0 + r.add(0x1c039, inner.scene.tvb_tilemap_pointer().into()); + r.add(0x1c9c8, inner.scene.tvb_tilemap_pointer().into()); + + let cl_tilemaps_ptr = inner + .scene + .cluster_tilemaps_pointer() + .map_or(0, |a| a.into()); + r.add(0x1c041, cl_tilemaps_ptr); + r.add(0x1c9d0, cl_tilemaps_ptr); + r.add(0x1c0a1, inner.scene.tpc_pointer().into()); // TE_TPC_ADDR + + let tvb_heapmeta_ptr = inner + .scene + .tvb_heapmeta_pointer() + .or(0x8000_0000_0000_0000) + .into(); + r.add(0x1c031, tvb_heapmeta_ptr); + r.add(0x1c9c0, tvb_heapmeta_ptr); + r.add(0x1c051, iogpu_unk54); // iogpu_unk_54/55 + r.add(0x1c061, iogpu_unk56); // iogpu_unk_56 + r.add(0x10149, utile_config.into()); // s2.unk_48 utile_config + r.add(0x10139, cmdbuf.ppp_multisamplectl); // PPP_MULTISAMPLECTL + r.add(0x10111, inner.scene.preempt_buf_1_pointer().into()); + r.add(0x1c9b0, inner.scene.preempt_buf_1_pointer().into()); + r.add(0x10119, inner.scene.preempt_buf_2_pointer().into()); + r.add(0x1c9b8, inner.scene.preempt_buf_2_pointer().into()); + r.add(0x1c958, 1); // s2.unk_80 + r.add( + 0x1c950, + inner + .scene + .preempt_buf_3_pointer() + .or(0x4_0000_0000_0000) + .into(), + ); + r.add(0x1c930, 0); // VCE related addr, lsb to enable + r.add(0x1c880, cmdbuf.vdm_ctrl_stream_base); // VDM_CTRL_STREAM_BASE + r.add(0x1c898, 0x0); // if lsb set, faults in UL1C0, possibly missing addr. + r.add( + 0x1c948, + inner.scene.meta_2_pointer().map_or(0, |a| a.into()), + ); // tvb_cluster_meta2 + r.add( + 0x1c888, + inner.scene.meta_3_pointer().map_or(0, |a| a.into()), + ); // tvb_cluster_meta3 + r.add(0x1c890, tiling_control.into()); // tvb_tiling_control + r.add(0x1c918, tiling_control_2); + r.add(0x1c079, inner.scene.tvb_layermeta_pointer().into()); + r.add(0x1c9d8, inner.scene.tvb_layermeta_pointer().into()); + let cl_layermeta_pointer = + inner.scene.tvb_cluster_layermeta_pointer().map_or(0, |a| a.into()); + r.add(0x1c089, cl_layermeta_pointer); + r.add(0x1c9e0, cl_layermeta_pointer); + let cl_meta_4_pointer = + inner.scene.meta_4_pointer().map_or(0, |a| a.into()); + r.add(0x16c41, cl_meta_4_pointer); // tvb_cluster_meta4 + r.add(0x1ca40, cl_meta_4_pointer); // tvb_cluster_meta4 + r.add(0x1c9a8, vtx_unk_f0); // + meta1_blocks? min_free_tvb_pages? + r.add( + 0x1c920, + inner.scene.meta_1_pointer().map_or(0, |a| a.into()), + ); // ??? | meta1_blocks? + r.add(0x10151, 0); + r.add(0x1c199, 0); + r.add(0x1c1a1, 0); + r.add(0x1c1a9, 0); // 0x10151 bit 1 enables + r.add(0x1c1b1, 0); + r.add(0x1c1b9, 0); + r.add(0x10061, self.usc_exec_base); // USC_EXEC_BASE_TA + r.add(0x11801, cmdbuf.vertex_helper.binary.into()); + r.add(0x11809, cmdbuf.vertex_helper.data); + r.add(0x11f71, cmdbuf.vertex_helper.cfg.into()); + r.add(0x1c0b1, tile_info.params.rgn_size.into()); // TE_PSG + r.add(0x1c850, tile_info.params.rgn_size.into()); + r.add(0x10131, tile_info.params.unk_4.into()); + r.add(0x10121, tile_info.params.ppp_ctrl.into()); // PPP_CTRL + r.add( + 0x10129, + tile_info.params.x_max as u64 + | ((tile_info.params.y_max as u64) << 16), + ); // PPP_SCREEN + r.add(0x101b9, tile_info.params.te_screen.into()); // TE_SCREEN + r.add(0x1c069, tile_info.params.te_mtile1.into()); // TE_MTILE1 + r.add(0x1c071, tile_info.params.te_mtile2.into()); // TE_MTILE2 + r.add(0x1c081, tile_info.params.tiles_per_mtile.into()); // TE_MTILE + r.add(0x1c0a9, tile_info.params.tpc_stride.into()); // TE_TPC + r.add(0x10171, tile_info.params.unk_24.into()); + r.add(0x10169, tile_info.params.unk_28.into()); // TA_RENDER_TARGET_MAX + r.add(0x12099, vtx_unk_118); + r.add(0x1c9e8, (tile_info.params.unk_28 & 0x4fff).into()); + /* + r.add(0x10209, 0x100); // Some kind of counter?? Does this matter? + r.add(0x1c9f0, 0x100); // Some kind of counter?? Does this matter? + r.add(0x1c830, 1); // ? + r.add(0x1ca30, 0x1502960e60); // ? + r.add(0x16c39, 0x1502960e60); // ? + r.add(0x1c910, 0xa0000b011d); // ? + r.add(0x1c8e0, 0xff); // cluster mask + r.add(0x1c8e8, 0); // ? + */ + } + ), + tpc: inner.scene.tpc_pointer(), + tpc_size: U64(tile_info.tpc_size as u64), + microsequence: inner.micro_seq.gpu_pointer(), + microsequence_size: inner.micro_seq.len() as u32, + fragment_stamp_slot: ev_frag.slot, + fragment_stamp_value: ev_frag.value.next(), + unk_pointee: 0, + unk_pad: 0, + job_params2 <- try_init!(fw::vertex::raw::JobParameters2 { + unk_480: Default::default(), // fixed + unk_498: U64(0x0), // fixed + unk_4a0: 0x0, // fixed + preempt_buf1: inner.scene.preempt_buf_1_pointer(), + unk_4ac: 0x0, // fixed + unk_4b0: U64(0x0), // fixed + unk_4b8: 0x0, // fixed + unk_4bc: U64(0x0), // fixed + unk_4c4_padding: Default::default(), + unk_50c: 0x0, // fixed + unk_510: U64(0x0), // fixed + unk_518: U64(0x0), // fixed + unk_520: U64(0x0), // fixed + }), + encoder_params <- try_init!(fw::job::raw::EncoderParams { + unk_8: 0x0, // fixed + sync_grow: 0x0, // fixed + unk_10: 0x0, // fixed + encoder_id: 0, + unk_18: 0x0, // fixed + unk_mask: 0xffffffffu32, + sampler_array: U64(cmdbuf.sampler_heap), + sampler_count: cmdbuf.sampler_count as u32, + sampler_max: (cmdbuf.sampler_count as u32) + 1, + }), + unk_55c: 0, + unk_560: 0, + sync_grow: 0, + unk_568: 0, + uses_scratch: (cmdbuf.flags + & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_VERTEX_SCRATCH as u32 + != 0) as u32, + meta <- try_init!(fw::job::raw::JobMeta { + unk_0: 0, + unk_2: 0, + no_preemption: no_preemption as u8, + stamp: ev_vtx.stamp_pointer, + fw_stamp: ev_vtx.fw_stamp_pointer, + stamp_value: ev_vtx.value.next(), + stamp_slot: ev_vtx.slot, + evctl_index: 0, // fixed + flush_stamps: flush_stamps as u32, + uuid: uuid_ta, + event_seq: ev_vtx.event_seq as u32, + }), + unk_after_meta: unk1.into(), + unk_buf_0: U64(0), + unk_buf_8: U64(0), + unk_buf_10: U64(0), + command_time: U64(0), + timestamp_pointers <- try_init!(fw::job::raw::TimestampPointers { + start_addr: Some(inner_ptr!(inner.timestamps.gpu_pointer(), vtx.start)), + end_addr: Some(inner_ptr!(inner.timestamps.gpu_pointer(), vtx.end)), + }), + user_timestamp_pointers: inner.user_timestamps.pointers()?, + client_sequence: slot_client_seq, + pad_5d5: Default::default(), + unk_5d8: 0, + unk_5dc: 0, + #[ver(V >= V13_0B4)] + unk_ts: U64(0), + #[ver(V >= V13_0B4)] + unk_5dd_8: Default::default(), + }) + }, + )?; + + core::mem::drop(alloc); + + mod_dev_dbg!(self.dev, "[Submission {}] Add Vertex\n", id); + fence.add_command(); + vtx_job.add_cb(vtx, vm_bind.slot(), move |error| { + if let Some(err) = error { + fence.set_error(err.into()) + } + + fence.command_complete(); + })?; + + mod_dev_dbg!(self.dev, "[Submission {}] Increment counters\n", id); + + // TODO: handle rollbacks, move to job submit? + buffer.increment(); + + job.get_vtx()?.next_seq(); + job.get_frag()?.next_seq(); + + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs new file mode 100644 index 00000000000000..6ebbaa56f48c81 --- /dev/null +++ b/drivers/gpu/drm/asahi/regs.rs @@ -0,0 +1,491 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU MMIO register abstraction +//! +//! Since the vast majority of the interactions with the GPU are brokered through the firmware, +//! there is very little need to interact directly with GPU MMIO register. This module abstracts +//! the few operations that require that, mainly reading the MMU fault status, reading GPU ID +//! information, and starting the GPU firmware coprocessor. + +use crate::hw; +use kernel::{ + c_str, + device::Core, + devres::Devres, + io::mem::IoMem, + platform, + prelude::*, + types::ARef, // +}; + +/// Size of the ASC control MMIO region. +pub(crate) const ASC_CTL_SIZE: usize = 0x4000; + +/// Size of the SGX MMIO region. +pub(crate) const SGX_SIZE: usize = 0x1000000; + +const CPU_CONTROL: usize = 0x44; +const CPU_RUN: u32 = 0x1 << 4; // BIT(4) + +const FAULT_INFO: usize = 0x17030; + +const ID_VERSION: usize = 0xd04000; +const ID_UNK08: usize = 0xd04008; +const ID_COUNTS_1: usize = 0xd04010; +const ID_COUNTS_2: usize = 0xd04014; +const ID_UNK18: usize = 0xd04018; +const ID_CLUSTERS: usize = 0xd0401c; + +const CORE_MASK_0: usize = 0xd01500; +const CORE_MASK_1: usize = 0xd01514; + +const CORE_MASKS_G14X: usize = 0xe01500; +const FAULT_INFO_G14X: usize = 0xd8c0; +const FAULT_ADDR_G14X: usize = 0xd8c8; + +/// Enum representing the unit that caused an MMU fault. +#[allow(non_camel_case_types)] +#[allow(clippy::upper_case_acronyms)] +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub(crate) enum FaultUnit { + /// Decompress / pixel fetch + DCMP(u8), + /// USC L1 Cache (device loads/stores) + UL1C(u8), + /// Compress / pixel store + CMP(u8), + GSL1(u8), + IAP(u8), + VCE(u8), + /// Tiling Engine + TE(u8), + RAS(u8), + /// Vertex Data Master + VDM(u8), + PPP(u8), + /// ISP Parameter Fetch + IPF(u8), + IPF_CPF(u8), + VF(u8), + VF_CPF(u8), + /// Depth/Stencil load/store + ZLS(u8), + + /// Parameter Management + dPM, + /// Compute Data Master + dCDM_KS(u8), + dIPP, + dIPP_CS, + // Vertex Data Master + dVDM_CSD, + dVDM_SSD, + dVDM_ILF, + dVDM_ILD, + dRDE(u8), + FC, + GSL2, + + /// Graphics L2 Cache Control? + GL2CC_META(u8), + GL2CC_MB, + + /// Parameter Management + gPM_SP(u8), + /// Vertex Data Master - CSD + gVDM_CSD_SP(u8), + gVDM_SSD_SP(u8), + gVDM_ILF_SP(u8), + gVDM_TFP_SP(u8), + gVDM_MMB_SP(u8), + /// Compute Data Master + gCDM_CS_KS0_SP(u8), + gCDM_CS_KS1_SP(u8), + gCDM_CS_KS2_SP(u8), + gCDM_KS0_SP(u8), + gCDM_KS1_SP(u8), + gCDM_KS2_SP(u8), + gIPP_SP(u8), + gIPP_CS_SP(u8), + gRDE0_SP(u8), + gRDE1_SP(u8), + + gCDM_CS, + gCDM_ID, + gCDM_CSR, + gCDM_CSW, + gCDM_CTXR, + gCDM_CTXW, + gIPP, + gIPP_CS, + gKSM_RCE, + + Unknown(u8), +} + +/// Reason for an MMU fault. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub(crate) enum FaultReason { + Unmapped, + AfFault, + WriteOnly, + ReadOnly, + NoAccess, + Unknown(u8), +} + +/// Collection of information about an MMU fault. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub(crate) struct FaultInfo { + pub(crate) address: u64, + pub(crate) sideband: u8, + pub(crate) vm_slot: u32, + pub(crate) unit_code: u8, + pub(crate) unit: FaultUnit, + pub(crate) level: u8, + pub(crate) unk_5: u8, + pub(crate) read: bool, + pub(crate) reason: FaultReason, +} + +/// Device resources for this GPU instance. +pub(crate) struct Resources { + dev: ARef, + sgx: Pin>>>, +} + +impl Resources { + /// Map the required resources given our platform device. + pub(crate) fn new(pdev: &platform::Device) -> Result { + let sgx_req = pdev.io_request_by_name(c_str!("sgx")).ok_or(EINVAL)?; + let sgx_iomem = KBox::pin_init(sgx_req.iomap_sized::(), GFP_KERNEL)?; + + Ok(Resources { + // SAFETY: This device does DMA via the UAT IOMMU. + dev: pdev.into(), + sgx: sgx_iomem, + }) + } + + fn sgx_read32(&self) -> u32 { + if let Some(sgx) = self.sgx.try_access() { + sgx.read32_relaxed(OFF) + } else { + 0 + } + } + + /* Not yet used + fn sgx_write32(&self, val: u32) { + if let Some(sgx) = self.sgx.try_access() { + sgx.write32_relaxed(val, OFF) + } + } + */ + + fn sgx_read64(&self) -> u64 { + if let Some(sgx) = self.sgx.try_access() { + sgx.read64_relaxed(OFF) + } else { + 0 + } + } + + /* Not yet used + fn sgx_write64(&self, val: u64) { + if let Some(sgx) = self.sgx.try_access() { + sgx.write64_relaxed(val, OFF) + } + } + */ + + /// Initialize the MMIO registers for the GPU. + pub(crate) fn init_mmio(&self) -> Result { + // Nothing to do for now... + + Ok(()) + } + + /// Start the ASC coprocessor CPU. + pub(crate) fn start_cpu(pdev: &platform::Device) -> Result { + let asc_req = pdev.io_request_by_name(c_str!("asc")).ok_or(EINVAL)?; + let asc_iomem = KBox::pin_init(asc_req.iomap_sized::(), GFP_KERNEL)?; + let res = asc_iomem.access(pdev.as_ref())?; + + let val = res.read32_relaxed(CPU_CONTROL); + res.write32_relaxed(val | CPU_RUN, CPU_CONTROL); + Ok(()) + } + + /// Get the GPU identification info from registers. + /// + /// See [`hw::GpuIdConfig`] for the result. + pub(crate) fn get_gpu_id(&self) -> Result { + let id_version = self.sgx_read32::(); + let id_unk08 = self.sgx_read32::(); + let id_counts_1 = self.sgx_read32::(); + let id_counts_2 = self.sgx_read32::(); + let id_unk18 = self.sgx_read32::(); + let id_clusters = self.sgx_read32::(); + + dev_info!( + self.dev.as_ref(), + "GPU ID registers: {:#x} {:#x} {:#x} {:#x} {:#x} {:#x}\n", + id_version, + id_unk08, + id_counts_1, + id_counts_2, + id_unk18, + id_clusters + ); + + let gpu_gen = (id_version >> 24) & 0xff; + + let mut core_mask_regs = KVec::new(); + + let num_clusters = match gpu_gen { + 4 | 5 => { + // G13 | G14G + core_mask_regs.push(self.sgx_read32::(), GFP_KERNEL)?; + core_mask_regs.push(self.sgx_read32::(), GFP_KERNEL)?; + (id_clusters >> 12) & 0xff + } + 6 => { + // G14X + core_mask_regs.push(self.sgx_read32::(), GFP_KERNEL)?; + core_mask_regs.push(self.sgx_read32::<{ CORE_MASKS_G14X + 4 }>(), GFP_KERNEL)?; + core_mask_regs.push(self.sgx_read32::<{ CORE_MASKS_G14X + 8 }>(), GFP_KERNEL)?; + // Clusters per die * num dies + ((id_counts_1 >> 8) & 0xff) * ((id_counts_1 >> 16) & 0xf) + } + a => { + dev_err!(self.dev.as_ref(), "Unknown GPU generation {}\n", a); + return Err(ENODEV); + } + }; + + let mut core_masks_packed = KVec::new(); + core_masks_packed.extend_from_slice(&core_mask_regs, GFP_KERNEL)?; + + dev_info!(self.dev.as_ref(), "Core masks: {:#x?}\n", core_masks_packed); + + let num_cores = id_counts_1 & 0xff; + + if num_cores > 32 { + dev_err!( + self.dev.as_ref(), + "Too many cores per cluster ({} > 32)\n", + num_cores + ); + return Err(ENODEV); + } + + if num_cores * num_clusters > (core_mask_regs.len() * 32) as u32 { + dev_err!( + self.dev.as_ref(), + "Too many total cores ({} x {} > {})\n", + num_clusters, + num_cores, + core_mask_regs.len() * 32 + ); + return Err(ENODEV); + } + + let mut core_masks = KVec::new(); + let mut total_active_cores: u32 = 0; + + let max_core_mask = ((1u64 << num_cores) - 1) as u32; + for _ in 0..num_clusters { + let mask = core_mask_regs[0] & max_core_mask; + core_masks.push(mask, GFP_KERNEL)?; + for i in 0..core_mask_regs.len() { + core_mask_regs[i] >>= num_cores; + if i < (core_mask_regs.len() - 1) { + core_mask_regs[i] |= core_mask_regs[i + 1] << (32 - num_cores); + } + } + total_active_cores += mask.count_ones(); + } + + if core_mask_regs.iter().any(|a| *a != 0) { + dev_err!( + self.dev.as_ref(), + "Leftover core mask: {:#x?}\n", + core_mask_regs + ); + return Err(EIO); + } + + let (gpu_rev, gpu_rev_id) = match (id_version >> 8) & 0xff { + 0x00 => (hw::GpuRevision::A0, hw::GpuRevisionID::A0), + 0x01 => (hw::GpuRevision::A1, hw::GpuRevisionID::A1), + 0x10 => (hw::GpuRevision::B0, hw::GpuRevisionID::B0), + 0x11 => (hw::GpuRevision::B1, hw::GpuRevisionID::B1), + 0x20 => (hw::GpuRevision::C0, hw::GpuRevisionID::C0), + 0x21 => (hw::GpuRevision::C1, hw::GpuRevisionID::C1), + a => { + dev_err!(self.dev.as_ref(), "Unknown GPU revision {}\n", a); + return Err(ENODEV); + } + }; + + Ok(hw::GpuIdConfig { + gpu_gen: match (id_version >> 24) & 0xff { + 4 => hw::GpuGen::G13, + 5 => hw::GpuGen::G14, + 6 => hw::GpuGen::G14, // G14X has a separate ID + a => { + dev_err!(self.dev.as_ref(), "Unknown GPU generation {}\n", a); + return Err(ENODEV); + } + }, + gpu_variant: match (id_version >> 16) & 0xff { + 1 => hw::GpuVariant::P, // Guess + 2 => hw::GpuVariant::G, + 3 => hw::GpuVariant::S, + 4 => { + if num_clusters > 4 { + hw::GpuVariant::D + } else { + hw::GpuVariant::C + } + } + a => { + dev_err!(self.dev.as_ref(), "Unknown GPU variant {}\n", a); + return Err(ENODEV); + } + }, + gpu_rev, + gpu_rev_id, + num_clusters, + num_cores, + num_frags: num_cores, // Used to be id_counts_1[15:8] but does not work for G14X + num_gps: (id_counts_2 >> 16) & 0xff, + total_active_cores, + core_masks, + core_masks_packed, + }) + } + + /// Get the fault information from the MMU status register, if one occurred. + pub(crate) fn get_fault_info(&self, cfg: &'static hw::HwConfig) -> Option { + let g14x = cfg.gpu_core as u32 >= hw::GpuCore::G14S as u32; + + let fault_info = if g14x { + self.sgx_read64::() + } else { + self.sgx_read64::() + }; + + if fault_info & 1 == 0 { + return None; + } + + let fault_addr = if g14x { + self.sgx_read64::() + } else { + fault_info >> 30 + }; + + let unit_code = ((fault_info >> 9) & 0xff) as u8; + let unit = match unit_code { + 0x00..=0x9f => match unit_code & 0xf { + 0x0 => FaultUnit::DCMP(unit_code >> 4), + 0x1 => FaultUnit::UL1C(unit_code >> 4), + 0x2 => FaultUnit::CMP(unit_code >> 4), + 0x3 => FaultUnit::GSL1(unit_code >> 4), + 0x4 => FaultUnit::IAP(unit_code >> 4), + 0x5 => FaultUnit::VCE(unit_code >> 4), + 0x6 => FaultUnit::TE(unit_code >> 4), + 0x7 => FaultUnit::RAS(unit_code >> 4), + 0x8 => FaultUnit::VDM(unit_code >> 4), + 0x9 => FaultUnit::PPP(unit_code >> 4), + 0xa => FaultUnit::IPF(unit_code >> 4), + 0xb => FaultUnit::IPF_CPF(unit_code >> 4), + 0xc => FaultUnit::VF(unit_code >> 4), + 0xd => FaultUnit::VF_CPF(unit_code >> 4), + 0xe => FaultUnit::ZLS(unit_code >> 4), + _ => FaultUnit::Unknown(unit_code), + }, + 0xa1 => FaultUnit::dPM, + 0xa2 => FaultUnit::dCDM_KS(0), + 0xa3 => FaultUnit::dCDM_KS(1), + 0xa4 => FaultUnit::dCDM_KS(2), + 0xa5 => FaultUnit::dIPP, + 0xa6 => FaultUnit::dIPP_CS, + 0xa7 => FaultUnit::dVDM_CSD, + 0xa8 => FaultUnit::dVDM_SSD, + 0xa9 => FaultUnit::dVDM_ILF, + 0xaa => FaultUnit::dVDM_ILD, + 0xab => FaultUnit::dRDE(0), + 0xac => FaultUnit::dRDE(1), + 0xad => FaultUnit::FC, + 0xae => FaultUnit::GSL2, + 0xb0..=0xb7 => FaultUnit::GL2CC_META(unit_code & 0xf), + 0xb8 => FaultUnit::GL2CC_MB, + 0xd0..=0xdf if g14x => match unit_code & 0xf { + 0x0 => FaultUnit::gCDM_CS, + 0x1 => FaultUnit::gCDM_ID, + 0x2 => FaultUnit::gCDM_CSR, + 0x3 => FaultUnit::gCDM_CSW, + 0x4 => FaultUnit::gCDM_CTXR, + 0x5 => FaultUnit::gCDM_CTXW, + 0x6 => FaultUnit::gIPP, + 0x7 => FaultUnit::gIPP_CS, + 0x8 => FaultUnit::gKSM_RCE, + _ => FaultUnit::Unknown(unit_code), + }, + 0xe0..=0xff if g14x => match unit_code & 0xf { + 0x0 => FaultUnit::gPM_SP((unit_code >> 4) & 1), + 0x1 => FaultUnit::gVDM_CSD_SP((unit_code >> 4) & 1), + 0x2 => FaultUnit::gVDM_SSD_SP((unit_code >> 4) & 1), + 0x3 => FaultUnit::gVDM_ILF_SP((unit_code >> 4) & 1), + 0x4 => FaultUnit::gVDM_TFP_SP((unit_code >> 4) & 1), + 0x5 => FaultUnit::gVDM_MMB_SP((unit_code >> 4) & 1), + 0x6 => FaultUnit::gRDE0_SP((unit_code >> 4) & 1), + _ => FaultUnit::Unknown(unit_code), + }, + 0xe0..=0xff if !g14x => match unit_code & 0xf { + 0x0 => FaultUnit::gPM_SP((unit_code >> 4) & 1), + 0x1 => FaultUnit::gVDM_CSD_SP((unit_code >> 4) & 1), + 0x2 => FaultUnit::gVDM_SSD_SP((unit_code >> 4) & 1), + 0x3 => FaultUnit::gVDM_ILF_SP((unit_code >> 4) & 1), + 0x4 => FaultUnit::gVDM_TFP_SP((unit_code >> 4) & 1), + 0x5 => FaultUnit::gVDM_MMB_SP((unit_code >> 4) & 1), + 0x6 => FaultUnit::gCDM_CS_KS0_SP((unit_code >> 4) & 1), + 0x7 => FaultUnit::gCDM_CS_KS1_SP((unit_code >> 4) & 1), + 0x8 => FaultUnit::gCDM_CS_KS2_SP((unit_code >> 4) & 1), + 0x9 => FaultUnit::gCDM_KS0_SP((unit_code >> 4) & 1), + 0xa => FaultUnit::gCDM_KS1_SP((unit_code >> 4) & 1), + 0xb => FaultUnit::gCDM_KS2_SP((unit_code >> 4) & 1), + 0xc => FaultUnit::gIPP_SP((unit_code >> 4) & 1), + 0xd => FaultUnit::gIPP_CS_SP((unit_code >> 4) & 1), + 0xe => FaultUnit::gRDE0_SP((unit_code >> 4) & 1), + 0xf => FaultUnit::gRDE1_SP((unit_code >> 4) & 1), + _ => FaultUnit::Unknown(unit_code), + }, + _ => FaultUnit::Unknown(unit_code), + }; + + let reason = match (fault_info >> 1) & 0x7 { + 0 => FaultReason::Unmapped, + 1 => FaultReason::AfFault, + 2 => FaultReason::WriteOnly, + 3 => FaultReason::ReadOnly, + 4 => FaultReason::NoAccess, + a => FaultReason::Unknown(a as u8), + }; + + Some(FaultInfo { + address: fault_addr << 6, + sideband: ((fault_info >> 23) & 0x7f) as u8, + vm_slot: ((fault_info >> 17) & 0x3f) as u32, + unit_code, + unit, + level: ((fault_info >> 7) & 3) as u8, + unk_5: ((fault_info >> 5) & 3) as u8, + read: (fault_info & (1 << 4)) != 0, + reason, + }) + } +} diff --git a/drivers/gpu/drm/asahi/slotalloc.rs b/drivers/gpu/drm/asahi/slotalloc.rs new file mode 100644 index 00000000000000..fde7470fe57791 --- /dev/null +++ b/drivers/gpu/drm/asahi/slotalloc.rs @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Generic slot allocator +//! +//! This is a simple allocator to manage fixed-size pools of GPU resources that are transiently +//! required during command execution. Each item resides in a "slot" at a given index. Users borrow +//! and return free items from the available pool. +//! +//! Allocations are "sticky", and return a token that callers can use to request the same slot +//! again later. This allows slots to be lazily invalidated, so that multiple uses by the same user +//! avoid any actual cleanup work. +//! +//! The allocation policy is currently a simple LRU mechanism, doing a full linear scan over the +//! slots when no token was previously provided. This is probably good enough, since in the absence +//! of serious system contention most allocation requests will be immediately fulfilled from the +//! previous slot without doing an LRU scan. + +use core::num::NonZeroUsize; +use core::ops::{ + Deref, + DerefMut, // +}; +use kernel::{ + error::{ + code::*, + Result, // + }, + prelude::*, + str::CStr, + sync::{ + Arc, + CondVar, + LockClassKey, + Mutex, // + }, +}; + +/// Trait representing a single item within a slot. +pub(crate) trait SlotItem { + /// Arbitrary user data associated with the SlotAllocator. + type Data; + + /// Called eagerly when this item is released back into the available pool. + fn release(&mut self, _data: &mut Self::Data, _slot: u32) {} +} + +/// Trivial implementation for users which do not require any slot data nor any allocator data. +impl SlotItem for () { + type Data = (); +} + +/// Represents a current or previous allocation of an item from a slot. Users keep `SlotToken`s +/// around across allocations to request that, if possible, the same slot be reused. +#[derive(Copy, Clone, Debug)] +pub(crate) struct SlotToken { + time: u64, + slot: u32, +} + +impl SlotToken { + /// Returns the slot index that this token represents a past assignment to. + pub(crate) fn last_slot(&self) -> u32 { + self.slot + } +} + +/// A guard representing active ownership of a slot. +pub(crate) struct Guard { + item: Option, + changed: bool, + token: SlotToken, + alloc: Arc>, +} + +impl Guard { + /// Returns the active slot owned by this `Guard`. + pub(crate) fn slot(&self) -> u32 { + self.token.slot + } + + /// Returns `true` if the slot changed since the last allocation (or no `SlotToken` was + /// provided), or `false` if the previously allocated slot was successfully re-acquired with + /// no other users in the interim. + pub(crate) fn changed(&self) -> bool { + self.changed + } + + /// Returns a `SlotToken` that can be used to re-request the same slot at a later time, after + /// this `Guard` is dropped. + pub(crate) fn token(&self) -> SlotToken { + self.token + } +} + +impl Deref for Guard { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.item.as_ref().expect("SlotItem Guard lost our item!") + } +} + +impl DerefMut for Guard { + fn deref_mut(&mut self) -> &mut Self::Target { + self.item.as_mut().expect("SlotItem Guard lost our item!") + } +} + +/// A slot item that is currently free. +struct Entry { + item: T, + get_time: u64, + drop_time: u64, +} + +/// Inner data for the `SlotAllocator`, protected by a `Mutex`. +struct SlotAllocatorInner { + data: T::Data, + slots: KVec>>, + get_count: u64, + drop_count: u64, + slot_limit: usize, +} + +/// A single slot allocator instance. +#[pin_data] +struct SlotAllocatorOuter { + #[pin] + inner: Mutex>, + #[pin] + cond: CondVar, +} + +/// A shared reference to a slot allocator instance. +pub(crate) struct SlotAllocator(Arc>); + +impl SlotAllocator { + /// Creates a new `SlotAllocator`, with a fixed number of slots and arbitrary associated data. + /// + /// The caller provides a constructor callback which takes a reference to the `T::Data` and + /// creates a single slot. This is called during construction to create all the initial + /// items, which then live the lifetime of the `SlotAllocator`. + pub(crate) fn new( + num_slots: u32, + mut data: T::Data, + mut constructor: impl FnMut(&mut T::Data, u32) -> Option, + name: &'static CStr, + lock_key1: Pin<&'static LockClassKey>, + lock_key2: Pin<&'static LockClassKey>, + ) -> Result> { + let mut slots = KVec::with_capacity(num_slots as usize, GFP_KERNEL)?; + + for i in 0..num_slots { + slots + .push( + constructor(&mut data, i).map(|item| Entry { + item, + get_time: 0, + drop_time: 0, + }), + GFP_KERNEL, + ) + .expect("try_push() failed after reservation"); + } + + let inner = SlotAllocatorInner { + data, + slots, + get_count: 0, + drop_count: 0, + slot_limit: usize::MAX, + }; + + let alloc = Arc::pin_init( + pin_init!(SlotAllocatorOuter { + // SAFETY: `mutex_init!` is called below. + inner <- Mutex::new(inner, name, lock_key1), + // SAFETY: `condvar_init!` is called below. + cond <- CondVar::new(name, lock_key2), + }), + GFP_KERNEL, + )?; + + Ok(SlotAllocator(alloc)) + } + + /// Calls a callback on the inner data associated with this allocator, taking the lock. + pub(crate) fn with_inner(&self, cb: impl FnOnce(&mut T::Data) -> RetVal) -> RetVal { + let mut inner = self.0.inner.lock(); + cb(&mut inner.data) + } + + /// Set the slot limit for this allocator. New bindings will not use slots above + /// this threshold. + pub(crate) fn set_limit(&self, limit: Option) { + let mut inner = self.0.inner.lock(); + inner.slot_limit = limit.unwrap_or(NonZeroUsize::MAX).get(); + } + + /// Gets a fresh slot, optionally reusing a previous allocation if a `SlotToken` is provided. + /// + /// Blocks if no slots are free. + pub(crate) fn get(&self, token: Option) -> Result> { + self.get_inner(token, |_a, _b| Ok(())) + } + + /// Gets a fresh slot, optionally reusing a previous allocation if a `SlotToken` is provided. + /// + /// Blocks if no slots are free. + /// + /// This version allows the caller to pass in a callback that gets a mutable reference to the + /// user data for the allocator and the freshly acquired slot, which is called before the + /// allocator lock is released. This can be used to perform bookkeeping associated with + /// specific slots (such as tracking their current owner). + pub(crate) fn get_inner( + &self, + token: Option, + cb: impl FnOnce(&mut T::Data, &mut Guard) -> Result<()>, + ) -> Result> { + let mut inner = self.0.inner.lock(); + + if let Some(token) = token { + if (token.slot as usize) < inner.slot_limit { + let slot = &mut inner.slots[token.slot as usize]; + if slot.is_some() { + let count = slot.as_ref().unwrap().get_time; + if count == token.time { + let mut guard = Guard { + item: Some(slot.take().unwrap().item), + token, + changed: false, + alloc: self.0.clone(), + }; + cb(&mut inner.data, &mut guard)?; + return Ok(guard); + } + } + } + } + + let mut first = true; + let slot = loop { + let mut oldest_time = u64::MAX; + let mut oldest_slot = 0u32; + + for (i, slot) in inner.slots.iter().enumerate() { + if i >= inner.slot_limit { + break; + } + if let Some(slot) = slot.as_ref() { + if slot.drop_time < oldest_time { + oldest_slot = i as u32; + oldest_time = slot.drop_time; + } + } + } + + if oldest_time == u64::MAX { + if first && inner.slot_limit == usize::MAX { + pr_warn!( + "{}: out of slots, blocking\n", + core::any::type_name::() + ); + } + first = false; + if self.0.cond.wait_interruptible(&mut inner) { + return Err(ERESTARTSYS); + } + } else { + break oldest_slot; + } + }; + + inner.get_count += 1; + + let item = inner.slots[slot as usize] + .take() + .expect("Someone stole our slot?") + .item; + + let mut guard = Guard { + item: Some(item), + changed: true, + token: SlotToken { + time: inner.get_count, + slot, + }, + alloc: self.0.clone(), + }; + + cb(&mut inner.data, &mut guard)?; + Ok(guard) + } +} + +impl Clone for SlotAllocator { + fn clone(&self) -> Self { + SlotAllocator(self.0.clone()) + } +} + +impl Drop for Guard { + fn drop(&mut self) { + let mut inner = self.alloc.inner.lock(); + if inner.slots[self.token.slot as usize].is_some() { + pr_crit!( + "{}: tried to return an item into a full slot ({})\n", + core::any::type_name::(), + self.token.slot + ); + } else { + inner.drop_count += 1; + let mut item = self.item.take().expect("Guard lost its item"); + item.release(&mut inner.data, self.token.slot); + inner.slots[self.token.slot as usize] = Some(Entry { + item, + get_time: self.token.time, + drop_time: inner.drop_count, + }); + self.alloc.cond.notify_one(); + } + } +} diff --git a/drivers/gpu/drm/asahi/util.rs b/drivers/gpu/drm/asahi/util.rs new file mode 100644 index 00000000000000..1a41d8f16d4432 --- /dev/null +++ b/drivers/gpu/drm/asahi/util.rs @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Miscellaneous utility functions + +use core::ops::{ + Add, + BitAnd, + Div, + Not, + Sub, // +}; +use kernel::prelude::*; + +/// Aligns an integer type to a power of two. +pub(crate) fn align(a: T, b: T) -> T +where + T: Copy + + Default + + BitAnd + + Not + + Add + + Sub + + Div + + core::cmp::PartialEq, +{ + let def: T = Default::default(); + #[allow(clippy::eq_op)] + let one: T = !def / !def; + + assert!((b & (b - one)) == def); + + (a + b - one) & !(b - one) +} + +/// Aligns an integer type down to a power of two. +pub(crate) fn align_down(a: T, b: T) -> T +where + T: Copy + + Default + + BitAnd + + Not + + Sub + + Div + + core::cmp::PartialEq, +{ + let def: T = Default::default(); + #[allow(clippy::eq_op)] + let one: T = !def / !def; + + assert!((b & (b - one)) == def); + + a & !(b - one) +} + +pub(crate) trait RangeExt { + fn overlaps(&self, other: Self) -> bool; + fn is_superset(&self, other: Self) -> bool; + // fn len(&self) -> usize; + fn range(&self) -> T; +} + +impl + Default + Copy + Sub> RangeExt for core::ops::Range +where + usize: core::convert::TryFrom, + >::Error: core::fmt::Debug, +{ + fn overlaps(&self, other: Self) -> bool { + !(self.is_empty() || other.is_empty() || self.end <= other.start || other.end <= self.start) + } + fn is_superset(&self, other: Self) -> bool { + !self.is_empty() + && (other.is_empty() || (other.start >= self.start && other.end <= self.end)) + } + fn range(&self) -> T { + if self.is_empty() { + Default::default() + } else { + self.end - self.start + } + } + // fn len(&self) -> usize { + // self.range().try_into().unwrap() + // } +} + +pub(crate) fn gcd(in_n: u64, in_m: u64) -> u64 { + let mut n = in_n; + let mut m = in_m; + + while n != 0 { + let remainder = m % n; + m = n; + n = remainder; + } + + m +} + +pub(crate) unsafe trait AnyBitPattern: Default + Sized + Copy + 'static {} + +pub(crate) struct Reader<'a> { + buffer: &'a [u8], + offset: usize, +} + +impl<'a> Reader<'a> { + pub(crate) fn new(buffer: &'a [u8]) -> Self { + Reader { buffer, offset: 0 } + } + + pub(crate) fn read_up_to(&mut self, max_size: usize) -> Result { + let mut obj: T = Default::default(); + let size: usize = core::mem::size_of::().min(max_size); + let range = self.offset..self.offset + size; + let src = self.buffer.get(range).ok_or(EINVAL)?; + + // SAFETY: The output pointer is valid, and the size does not exceed + // the type size, and all bit patterns are valid. + let dst = unsafe { core::slice::from_raw_parts_mut(&mut obj as *mut _ as *mut u8, size) }; + + dst.copy_from_slice(src); + self.offset += size; + Ok(obj) + } + + pub(crate) fn read(&mut self) -> Result { + self.read_up_to(!0) + } + + pub(crate) fn is_empty(&self) -> bool { + self.offset >= self.buffer.len() + } + + pub(crate) fn skip(&mut self, size: usize) { + self.offset += size + } + + pub(crate) fn rewind(&mut self) { + self.offset = 0 + } +} diff --git a/drivers/gpu/drm/asahi/vm/mod.rs b/drivers/gpu/drm/asahi/vm/mod.rs new file mode 100644 index 00000000000000..63cf8a76cd5ce4 --- /dev/null +++ b/drivers/gpu/drm/asahi/vm/mod.rs @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Virtual address space management + +mod range; diff --git a/drivers/gpu/drm/asahi/vm/range.rs b/drivers/gpu/drm/asahi/vm/range.rs new file mode 100644 index 00000000000000..727d29a5cccf02 --- /dev/null +++ b/drivers/gpu/drm/asahi/vm/range.rs @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT + +// Copied from tyr's mmu/vm/range.rs + +//! Range allocator. +//! +//! This module allows you to search for unused ranges to store GEM objects. + +use kernel::alloc::Flags; +use kernel::maple_tree::MapleTreeAlloc; +use kernel::prelude::*; +use kernel::sync::Arc; + +use core::ops::Range; + +/// The actual storage for the ranges. +/// +/// All ranges must fit within the `range` field. +/// +/// The implementation is different on 32-bit and 64-bit cpus. On 64-bit, the 64-bit addresses are +/// stored directly in the maple tree, but on 32-bit, the maple tree stores the ranges translated +/// in the range zero until `range.end-range.start`. This is done because the maple tree uses +/// unsigned long as its address type, which is too small to store the 64-bit address directly on +/// 32-bit machines. +#[pin_data] +struct RangeAllocInner { + #[pin] + maple: MapleTreeAlloc<()>, + range: Range, +} + +/// This object allows you to allocate ranges on the inner maple tree. +pub(crate) struct RangeAlloc { + inner: Arc, +} + +/// Represents a live range in the maple tree. +/// +/// The destructor removes the range from the maple tree, allowing others to allocate it in the +/// future. +pub(crate) struct LiveRange { + inner: Arc, + offset: u64, + size: usize, +} + +impl RangeAlloc { + pub(crate) fn new(start: u64, end: u64, gfp: Flags) -> Result { + if end < start { + return Err(EINVAL); + } + + let inner = Arc::pin_init( + try_pin_init!(RangeAllocInner { + maple <- MapleTreeAlloc::new(), + range: start..end, + }), + gfp, + )?; + + Ok(RangeAlloc { inner }) + } + + pub(crate) fn allocate(&self, size: usize, gfp: Flags) -> Result { + let maple_start = self.inner.range.start as usize; + let maple_end = self.inner.range.end as usize; + + let offset = self + .inner + .maple + .alloc_range(size, (), maple_start..maple_end, gfp)?; + + Ok(LiveRange { + inner: self.inner.clone(), + offset: offset as u64, + size, + }) + } + + pub(crate) fn insert(&self, start: u64, end: u64, gfp: Flags) -> Result { + if end < start { + return Err(EINVAL); + } + if start < self.inner.range.start { + return Err(EINVAL); + } + if end > self.inner.range.end { + return Err(EINVAL); + } + + self.inner + .maple + .insert_range(start as usize..end as usize, (), gfp)?; + + Ok(LiveRange { + inner: self.inner.clone(), + offset: start, + size: (end - start) as usize, + }) + } +} + +impl LiveRange { + pub(crate) fn size(&self) -> usize { + self.size + } + + pub(crate) fn start(&self) -> u64 { + self.offset + } + + pub(crate) fn end(&self) -> u64 { + self.offset + self.size as u64 + } + + pub(crate) fn range(&self) -> Range { + self.start()..self.end() + } +} + +impl Drop for LiveRange { + fn drop(&mut self) { + self.inner.maple.erase(self.offset as usize); + } +} diff --git a/drivers/gpu/drm/asahi/workqueue.rs b/drivers/gpu/drm/asahi/workqueue.rs new file mode 100644 index 00000000000000..55b6058d50224f --- /dev/null +++ b/drivers/gpu/drm/asahi/workqueue.rs @@ -0,0 +1,1015 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU command execution queues +//! +//! The AGX GPU firmware schedules GPU work commands out of work queues, which are ring buffers of +//! pointers to work commands. There can be an arbitrary number of work queues. Work queues have an +//! associated type (vertex, fragment, or compute) and may only contain generic commands or commands +//! specific to that type. +//! +//! This module manages queueing work commands into a work queue and submitting them for execution +//! by the firmware. An active work queue needs an event to signal completion of its work, which is +//! owned by what we call a batch. This event then notifies the work queue when work is completed, +//! and that triggers freeing of all resources associated with that work. An idle work queue gives +//! up its associated event. + +use crate::debug::*; +use crate::fw::channels::{ + ChannelErrorType, + PipeType, // +}; +use crate::fw::types::*; +use crate::fw::workqueue::*; +use crate::no_debug; +use crate::object::OpaqueGpuObject; +use crate::{ + channel, + driver, + event, + fw, + gpu, + regs, // +}; +use core::any::Any; +use core::num::NonZeroU64; +use core::sync::atomic::Ordering; +use kernel::{ + dma_fence, + error::code::*, + new_mutex, + prelude::*, + sync::{ + lock::{ + mutex::MutexBackend, + Guard, // + }, + Arc, + Mutex, // + }, + workqueue::{ + self, + impl_has_work, + new_work, + Work, + WorkItem, // + }, // +}; + +pub(crate) trait OpaqueCommandObject: OpaqueGpuObject {} + +impl OpaqueCommandObject for GpuObject where T: Command {} + +const DEBUG_CLASS: DebugFlags = DebugFlags::WorkQueue; + +const MAX_JOB_SLOTS: u32 = 127; + +/// An enum of possible errors that might cause a piece of work to fail execution. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum WorkError { + /// GPU timeout (command execution took too long). + Timeout, + /// GPU MMU fault (invalid access). + Fault(regs::FaultInfo), + /// Work failed due to an error caused by other concurrent GPU work. + Killed, + /// Channel error + ChannelError(ChannelErrorType), + /// The GPU crashed. + NoDevice, + /// Unknown reason. + Unknown, +} + +impl From for kernel::error::Error { + fn from(err: WorkError) -> Self { + match err { + WorkError::Timeout => ETIMEDOUT, + // Not EFAULT because that's for userspace faults + WorkError::Fault(_) => EIO, + WorkError::Unknown => ENODATA, + WorkError::Killed => ECANCELED, + WorkError::NoDevice => ENODEV, + WorkError::ChannelError(_) => EIO, + } + } +} + +/// A GPU context tracking structure, which must be explicitly invalidated when dropped. +pub(crate) struct GpuContext { + dev: driver::AsahiDevRef, + data: Option>>, +} +no_debug!(GpuContext); + +impl GpuContext { + /// Allocate a new GPU context. + pub(crate) fn new( + dev: &driver::AsahiDevice, + alloc: &mut gpu::KernelAllocators, + buffer: Arc, + ) -> Result { + Ok(GpuContext { + dev: dev.into(), + data: Some(KBox::new( + alloc.shared.new_object( + fw::workqueue::GpuContextData { _buffer: buffer }, + |_inner| Default::default(), + )?, + GFP_KERNEL, + )?), + }) + } + + /// Returns the GPU pointer to the inner GPU context data structure. + pub(crate) fn gpu_pointer(&self) -> GpuPointer<'_, fw::workqueue::GpuContextData> { + self.data.as_ref().unwrap().gpu_pointer() + } +} + +impl Drop for GpuContext { + fn drop(&mut self) { + mod_dev_dbg!(self.dev, "GpuContext: Freeing GPU context\n"); + let data = self.data.take().unwrap(); + (*self.dev).gpu.free_context(data); + } +} + +struct SubmittedWork +where + O: OpaqueCommandObject, + C: FnOnce(Option) + Send + Sync + 'static, +{ + object: O, + value: EventValue, + error: Option, + wptr: u32, + vm_slot: u32, + callback: Option, + fence: dma_fence::Fence, +} + +pub(crate) trait GenSubmittedWork: Send + Sync { + fn gpu_va(&self) -> NonZeroU64; + fn value(&self) -> event::EventValue; + fn wptr(&self) -> u32; + fn set_wptr(&mut self, wptr: u32); + fn mark_error(&mut self, error: WorkError); + fn complete(&mut self); + fn get_fence(&self) -> dma_fence::Fence; +} + +#[pin_data] +struct SubmittedWorkContainer { + #[pin] + work: Work, + inner: KBox, +} + +impl_has_work! { + impl HasWork for SubmittedWorkContainer { self.work } +} + +impl WorkItem for SubmittedWorkContainer { + type Pointer = Pin>; + + fn run(this: Pin>) { + mod_pr_debug!("WorkQueue: Freeing command @ {:?}\n", this.inner.gpu_va()); + } +} + +impl SubmittedWorkContainer { + fn inner_mut(self: Pin<&mut Self>) -> &mut KBox { + // SAFETY: inner does not require structural pinning. + unsafe { &mut self.get_unchecked_mut().inner } + } +} + +impl) + Send + Sync> GenSubmittedWork + for SubmittedWork +{ + fn gpu_va(&self) -> NonZeroU64 { + self.object.gpu_va() + } + + fn value(&self) -> event::EventValue { + self.value + } + + fn wptr(&self) -> u32 { + self.wptr + } + + fn set_wptr(&mut self, wptr: u32) { + self.wptr = wptr; + } + + fn complete(&mut self) { + if let Some(cb) = self.callback.take() { + cb(self.error); + } + } + + fn mark_error(&mut self, error: WorkError) { + mod_pr_debug!("WorkQueue: Command at value {:#x?} failed\n", self.value); + self.error = Some(match error { + WorkError::Fault(info) if info.vm_slot != self.vm_slot => WorkError::Killed, + err => err, + }); + } + + fn get_fence(&self) -> dma_fence::Fence { + self.fence.clone() + } +} + +/// Inner data for managing a single work queue. +#[versions(AGX)] +struct WorkQueueInner { + dev: driver::AsahiDevRef, + event_manager: Arc, + info: GpuObject, + new: bool, + pipe_type: PipeType, + size: u32, + wptr: u32, + pending: KVec>>, + last_token: Option, + pending_jobs: usize, + last_submitted: Option, + last_completed: Option, + event: Option<(event::Event, event::EventValue)>, + priority: u32, + commit_seq: u64, + submit_seq: u64, + event_seq: u64, +} + +/// An instance of a work queue. +#[versions(AGX)] +#[pin_data] +pub(crate) struct WorkQueue { + info_pointer: GpuWeakPointer, + #[pin] + inner: Mutex, +} + +#[versions(AGX)] +impl WorkQueueInner::ver { + /// Return the GPU done pointer, representing how many work items have been completed by the + /// GPU. + fn doneptr(&self) -> u32 { + self.info + .state + .with(|raw, _inner| raw.gpu_doneptr.load(Ordering::Acquire)) + } +} + +#[versions(AGX)] +#[derive(Copy, Clone)] +pub(crate) struct QueueEventInfo { + pub(crate) stamp_pointer: GpuWeakPointer, + pub(crate) fw_stamp_pointer: GpuWeakPointer, + pub(crate) slot: u32, + pub(crate) value: event::EventValue, + pub(crate) cmd_seq: u64, + pub(crate) event_seq: u64, + pub(crate) info_ptr: GpuWeakPointer, +} + +#[versions(AGX)] +pub(crate) struct Job { + wq: Arc, + event_info: QueueEventInfo::ver, + start_value: EventValue, + pending: KVec>>, + committed: bool, + submitted: bool, + event_count: usize, + fence: dma_fence::Fence, +} + +#[versions(AGX)] +pub(crate) struct JobSubmission<'a> { + inner: Option>, + wptr: u32, + event_count: usize, + command_count: usize, +} + +#[versions(AGX)] +impl Job::ver { + pub(crate) fn event_info(&self) -> QueueEventInfo::ver { + let mut info = self.event_info; + info.cmd_seq += self.pending.len() as u64; + info.event_seq += self.event_count as u64; + + info + } + + pub(crate) fn next_seq(&mut self) { + self.event_count += 1; + self.event_info.value.increment(); + } + + pub(crate) fn add( + &mut self, + command: O, + vm_slot: u32, + ) -> Result { + self.add_cb(command, vm_slot, |_| {}) + } + + pub(crate) fn add_cb( + &mut self, + command: O, + vm_slot: u32, + callback: impl FnOnce(Option) + Sync + Send + 'static, + ) -> Result { + if self.committed { + pr_err!("WorkQueue: Tried to mutate committed Job\n"); + return Err(EINVAL); + } + + let fence = self.fence.clone(); + let value = self.event_info.value.next(); + + self.pending.push( + KBox::try_pin_init( + try_pin_init!(SubmittedWorkContainer { + work <- new_work!("SubmittedWorkWrapper::work"), + inner: KBox::new(SubmittedWork::<_, _> { + object: command, + value, + error: None, + callback: Some(callback), + wptr: 0, + vm_slot, + fence, + }, GFP_KERNEL)? + }), + GFP_KERNEL, + )?, + GFP_KERNEL, + )?; + + Ok(()) + } + + pub(crate) fn commit(&mut self) -> Result { + if self.committed { + pr_err!("WorkQueue: Tried to commit committed Job\n"); + return Err(EINVAL); + } + + if self.pending.is_empty() { + pr_err!("WorkQueue: Job::commit() with no commands\n"); + return Err(EINVAL); + } + + let mut inner = self.wq.inner.lock(); + + let ev = inner.event.as_mut().expect("WorkQueue: Job lost its event"); + + if ev.1 != self.start_value { + pr_err!( + "WorkQueue: Job::commit() out of order (event slot {} {:?} != {:?}\n", + ev.0.slot(), + ev.1, + self.start_value + ); + return Err(EINVAL); + } + + ev.1 = self.event_info.value; + inner.commit_seq += self.pending.len() as u64; + inner.event_seq += self.event_count as u64; + self.committed = true; + + Ok(()) + } + + pub(crate) fn can_submit(&self) -> Option { + let inner = self.wq.inner.lock(); + if inner.free_slots() > self.event_count && inner.free_space() > self.pending.len() { + None + } else if let Some(work) = inner.pending.first() { + Some(work.inner.get_fence()) + } else { + pr_err!( + "WorkQueue: Cannot submit, but queue is empty? {} > {}, {} > {} (pend={} ls={:#x?} lc={:#x?}) ev={:#x?} cur={:#x?} slot {:?}\n", + inner.free_slots(), + self.event_count, + inner.free_space(), + self.pending.len(), + inner.pending.len(), + inner.last_submitted, + inner.last_completed, + inner.event.as_ref().map(|a| a.1), + inner.event.as_ref().map(|a| a.0.current()), + inner.event.as_ref().map(|a| a.0.slot()), + ); + None + } + } + + pub(crate) fn submit(&mut self) -> Result> { + if !self.committed { + pr_err!("WorkQueue: Tried to submit uncommitted Job\n"); + return Err(EINVAL); + } + + if self.submitted { + pr_err!("WorkQueue: Tried to submit Job twice\n"); + return Err(EINVAL); + } + + if self.pending.is_empty() { + pr_err!("WorkQueue: Job::submit() with no commands\n"); + return Err(EINVAL); + } + + let mut inner = self.wq.inner.lock(); + + if inner.submit_seq != self.event_info.cmd_seq { + pr_err!( + "WorkQueue: Job::submit() out of order (submit_seq {} != {})\n", + inner.submit_seq, + self.event_info.cmd_seq + ); + return Err(EINVAL); + } + + if inner.commit_seq < (self.event_info.cmd_seq + self.pending.len() as u64) { + pr_err!( + "WorkQueue: Job::submit() out of order (commit_seq {} != {})\n", + inner.commit_seq, + (self.event_info.cmd_seq + self.pending.len() as u64) + ); + return Err(EINVAL); + } + + let mut wptr = inner.wptr; + let command_count = self.pending.len(); + + if inner.free_space() <= command_count { + pr_err!("WorkQueue: Job does not fit in ring buffer\n"); + return Err(EBUSY); + } + + inner.pending.reserve(command_count, GFP_KERNEL)?; + + inner.last_submitted = Some(self.event_info.value); + mod_dev_dbg!( + inner.dev, + "WorkQueue: submitting {} cmds at {:#x?}, lc {:#x?}, cur {:#x?}, pending {}, events {}\n", + self.pending.len(), + inner.last_submitted, + inner.last_completed, + inner.event.as_ref().map(|a| a.0.current()), + inner.pending.len(), + self.event_count, + ); + + for mut command in self.pending.drain(..) { + command.as_mut().inner_mut().set_wptr(wptr); + + let next_wptr = (wptr + 1) % inner.size; + assert!(inner.doneptr() != next_wptr); + inner.info.ring[wptr as usize] = command.inner.gpu_va().get(); + wptr = next_wptr; + + // Cannot fail, since we did a reserve(1) above + inner + .pending + .push(command, GFP_KERNEL) + .expect("push() failed after reserve()"); + } + + self.submitted = true; + + Ok(JobSubmission::ver { + inner: Some(inner), + wptr, + command_count, + event_count: self.event_count, + }) + } +} + +#[versions(AGX)] +impl<'a> JobSubmission::ver<'a> { + pub(crate) fn run(mut self, channel: &mut channel::PipeChannel::ver) { + let command_count = self.command_count; + let mut inner = self.inner.take().expect("No inner?"); + let wptr = self.wptr; + core::mem::forget(self); + + inner + .info + .state + .with(|raw, _inner| raw.cpu_wptr.store(wptr, Ordering::Release)); + + inner.wptr = wptr; + + let event = inner.event.as_mut().expect("JobSubmission lost its event"); + + let event_slot = event.0.slot(); + + let msg = fw::channels::RunWorkQueueMsg::ver { + pipe_type: inner.pipe_type, + work_queue: Some(inner.info.weak_pointer()), + wptr: inner.wptr, + event_slot, + is_new: inner.new, + __pad: Default::default(), + }; + channel.send(&msg); + inner.new = false; + + inner.submit_seq += command_count as u64; + } + + pub(crate) fn pipe_type(&self) -> PipeType { + self.inner.as_ref().expect("No inner?").pipe_type + } + + pub(crate) fn priority(&self) -> u32 { + self.inner.as_ref().expect("No inner?").priority + } +} + +#[versions(AGX)] +impl Drop for Job::ver { + fn drop(&mut self) { + mod_pr_debug!("WorkQueue: Dropping Job\n"); + let mut inner = self.wq.inner.lock(); + + if !self.committed { + pr_info!( + "WorkQueue: Dropping uncommitted job with {} events\n", + self.event_count + ); + } + + if self.committed && !self.submitted { + let pipe_type = inner.pipe_type; + let event = inner.event.as_mut().expect("Job lost its event"); + pr_info!( + "WorkQueue({:?}): Roll back {} events (slot {} val {:#x?}) and {} commands\n", + pipe_type, + self.event_count, + event.0.slot(), + event.1, + self.pending.len() + ); + event.1.sub(self.event_count as u32); + inner.commit_seq -= self.pending.len() as u64; + inner.event_seq -= self.event_count as u64; + } + + inner.pending_jobs -= 1; + + if inner.pending.is_empty() && inner.pending_jobs == 0 { + mod_pr_debug!("WorkQueue({:?}): Dropping event\n", inner.pipe_type); + inner.event = None; + inner.last_submitted = None; + inner.last_completed = None; + } + mod_pr_debug!("WorkQueue({:?}): Dropped Job\n", inner.pipe_type); + } +} + +#[versions(AGX)] +impl<'a> Drop for JobSubmission::ver<'a> { + fn drop(&mut self) { + let inner = self.inner.as_mut().expect("No inner?"); + mod_pr_debug!("WorkQueue({:?}): Dropping JobSubmission\n", inner.pipe_type); + + let new_len = inner.pending.len() - self.command_count; + inner.pending.truncate(new_len); + + let pipe_type = inner.pipe_type; + let event = inner.event.as_mut().expect("JobSubmission lost its event"); + pr_info!( + "WorkQueue({:?}): JobSubmission: Roll back {} events (slot {} val {:#x?}) and {} commands\n", + pipe_type, + self.event_count, + event.0.slot(), + event.1, + self.command_count + ); + event.1.sub(self.event_count as u32); + let val = event.1; + inner.commit_seq -= self.command_count as u64; + inner.event_seq -= self.event_count as u64; + inner.last_submitted = Some(val); + mod_pr_debug!("WorkQueue({:?}): Dropped JobSubmission\n", inner.pipe_type); + } +} + +#[versions(AGX)] +impl WorkQueueInner::ver { + /// Return the number of free entries in the workqueue + pub(crate) fn free_space(&self) -> usize { + self.size as usize - self.pending.len() - 1 + } + + pub(crate) fn free_slots(&self) -> usize { + let busy_slots = if let Some(ls) = self.last_submitted { + let lc = self + .last_completed + .expect("last_submitted but not completed?"); + ls.delta(&lc) + } else { + 0 + }; + + ((MAX_JOB_SLOTS as i32) - busy_slots).max(0) as usize + } +} + +#[versions(AGX)] +impl WorkQueue::ver { + /// Create a new WorkQueue of a given type and priority. + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + dev: &driver::AsahiDevice, + alloc: &mut gpu::KernelAllocators, + event_manager: Arc, + gpu_context: Arc, + notifier_list: Arc>, + pipe_type: PipeType, + id: u64, + priority: u32, + size: u32, + ) -> Result> { + let gpu_buf = alloc.private.array_empty_tagged(0x2c18, b"GPBF")?; + let mut state = alloc.shared.new_default::()?; + let ring = alloc.shared.array_empty(size as usize)?; + let mut prio = *raw::PRIORITY.get(priority as usize).ok_or(EINVAL)?; + + if pipe_type == PipeType::Compute && !debug_enabled(DebugFlags::Debug0) { + // Hack to disable compute preemption until we fix it + prio.0 = 0; + prio.5 = 1; + } + + let inner = WorkQueueInner::ver { + dev: dev.into(), + event_manager, + // Use shared (coherent) state with verbose faults so we can dump state correctly + info: if debug_enabled(DebugFlags::VerboseFaults) { + &mut alloc.shared + } else { + &mut alloc.private + } + .new_init( + try_init!(QueueInfo::ver { + state: { + state.with_mut(|raw, _inner| { + raw.rb_size = size; + }); + state + }, + ring, + gpu_buf, + notifier_list: notifier_list, + gpu_context: gpu_context, + }), + |inner, _p| { + try_init!(raw::QueueInfo::ver { + state: inner.state.gpu_pointer(), + ring: inner.ring.gpu_pointer(), + notifier_list: inner.notifier_list.gpu_pointer(), + gpu_buf: inner.gpu_buf.gpu_pointer(), + gpu_rptr1: Default::default(), + gpu_rptr2: Default::default(), + gpu_rptr3: Default::default(), + event_id: AtomicI32::new(-1), + priority: prio, + unk_4c: -1, + uuid: id as u32, + unk_54: -1, + unk_58: Default::default(), + busy: Default::default(), + __pad: Default::default(), + #[ver(V >= V13_2 && G < G14X)] + unk_84_0: 0, + unk_84_state: Default::default(), + error_count: Default::default(), + unk_8c: 0, + unk_90: 0, + unk_94: 0, + pending: Default::default(), + unk_9c: 0, + gpu_context: inner.gpu_context.gpu_pointer(), + unk_a8: Default::default(), + #[ver(V >= V13_2 && G < G14X)] + unk_b0: 0, + }) + }, + )?, + new: true, + pipe_type, + size, + wptr: 0, + pending: KVec::new(), + last_token: None, + event: None, + priority, + pending_jobs: 0, + commit_seq: 0, + submit_seq: 0, + event_seq: 0, + last_completed: None, + last_submitted: None, + }; + + let info_pointer = inner.info.weak_pointer(); + + Arc::pin_init( + pin_init!(Self { + info_pointer, + inner <- match pipe_type { + PipeType::Vertex => new_mutex!(inner, "WorkQueue::inner (Vertex)"), + PipeType::Fragment => new_mutex!(inner, "WorkQueue::inner (Fragment)"), + PipeType::Compute => new_mutex!(inner, "WorkQueue::inner (Compute)"), + }, + }), + GFP_KERNEL, + ) + } + + pub(crate) fn event_info(&self) -> Option { + let inner = self.inner.lock(); + + inner.event.as_ref().map(|ev| QueueEventInfo::ver { + stamp_pointer: ev.0.stamp_pointer(), + fw_stamp_pointer: ev.0.fw_stamp_pointer(), + slot: ev.0.slot(), + value: ev.1, + cmd_seq: inner.commit_seq, + event_seq: inner.event_seq, + info_ptr: self.info_pointer, + }) + } + + pub(crate) fn new_job(self: &Arc, fence: dma_fence::Fence) -> Result { + let mut inner = self.inner.lock(); + + if inner.event.is_none() { + mod_pr_debug!("WorkQueue({:?}): Grabbing event\n", inner.pipe_type); + let event = inner.event_manager.get(inner.last_token, self.clone())?; + let cur = event.current(); + inner.last_token = Some(event.token()); + mod_pr_debug!( + "WorkQueue({:?}): Grabbed event slot {}: {:#x?}\n", + inner.pipe_type, + event.slot(), + cur + ); + inner.event = Some((event, cur)); + inner.last_submitted = Some(cur); + inner.last_completed = Some(cur); + } + + inner.pending_jobs += 1; + + let ev = &inner.event.as_ref().unwrap(); + + mod_pr_debug!( + "WorkQueue({:?}): New job at value {:#x?} slot {}\n", + inner.pipe_type, + ev.1, + ev.0.slot() + ); + Ok(Job::ver { + wq: self.clone(), + event_info: QueueEventInfo::ver { + stamp_pointer: ev.0.stamp_pointer(), + fw_stamp_pointer: ev.0.fw_stamp_pointer(), + slot: ev.0.slot(), + value: ev.1, + cmd_seq: inner.commit_seq, + event_seq: inner.event_seq, + info_ptr: self.info_pointer, + }, + start_value: ev.1, + pending: KVec::new(), + event_count: 0, + committed: false, + submitted: false, + fence, + }) + } + + pub(crate) fn pipe_type(&self) -> PipeType { + self.inner.lock().pipe_type + } + + pub(crate) fn dump_info(&self) { + pr_info!("WorkQueue @ {:?}:", self.info_pointer); + self.inner.lock().info.with(|raw, _inner| { + pr_info!(" GPU rptr1: {:#x}", raw.gpu_rptr1.load(Ordering::Relaxed)); + pr_info!(" GPU rptr1: {:#x}", raw.gpu_rptr2.load(Ordering::Relaxed)); + pr_info!(" GPU rptr1: {:#x}", raw.gpu_rptr3.load(Ordering::Relaxed)); + pr_info!(" Event ID: {:#x}", raw.event_id.load(Ordering::Relaxed)); + pr_info!(" Busy: {:#x}", raw.busy.load(Ordering::Relaxed)); + pr_info!(" Unk 84: {:#x}", raw.unk_84_state.load(Ordering::Relaxed)); + pr_info!( + " Error count: {:#x}", + raw.error_count.load(Ordering::Relaxed) + ); + pr_info!(" Pending: {:#x}", raw.pending.load(Ordering::Relaxed)); + }); + } + + pub(crate) fn info_pointer(&self) -> GpuWeakPointer { + self.info_pointer + } +} + +/// Trait used to erase the version-specific type of WorkQueues, to avoid leaking +/// version-specificity into the event module. +pub(crate) trait WorkQueue { + /// Cast as an Any type. + fn as_any(&self) -> &dyn Any; + + fn signal(&self) -> bool; + fn mark_error(&self, value: event::EventValue, error: WorkError); + fn fail_all(&self, error: WorkError); +} + +#[versions(AGX)] +impl WorkQueue for WorkQueue::ver { + fn as_any(&self) -> &dyn Any { + self + } + + /// Signal a workqueue that some work was completed. + /// + /// This will check the event stamp value to find out exactly how many commands were processed. + fn signal(&self) -> bool { + let mut inner = self.inner.lock(); + let event = inner.event.as_ref(); + let value = match event { + None => { + mod_pr_debug!("WorkQueue: signal() called but no event?\n"); + + if inner.pending_jobs > 0 || !inner.pending.is_empty() { + pr_crit!("WorkQueue: signal() called with no event and pending jobs.\n"); + } + return true; + } + Some(event) => event.0.current(), + }; + + if let Some(lc) = inner.last_completed { + if value < lc { + pr_err!( + "WorkQueue: event rolled back? cur {:#x?}, lc {:#x?}, ls {:#x?}", + value, + inner.last_completed, + inner.last_submitted + ); + } + } else { + pr_crit!("WorkQueue: signal() called with no last_completed.\n"); + } + inner.last_completed = Some(value); + + mod_pr_debug!( + "WorkQueue({:?}): Signaling event {:?} value {:#x?}\n", + inner.pipe_type, + inner.last_token, + value + ); + + let mut completed_commands: usize = 0; + + for cmd in inner.pending.iter() { + if cmd.inner.value() <= value { + mod_pr_debug!( + "WorkQueue({:?}): Command at value {:#x?} complete\n", + inner.pipe_type, + cmd.inner.value() + ); + completed_commands += 1; + } else { + break; + } + } + + if completed_commands == 0 { + return inner.pending.is_empty(); + } + + let last_wptr = inner.pending[completed_commands - 1].inner.wptr(); + let pipe_type = inner.pipe_type; + + for mut cmd in inner.pending.drain(..completed_commands) { + mod_pr_debug!( + "WorkQueue({:?}): Queueing command @ {:?} for cleanup\n", + pipe_type, + cmd.inner.gpu_va() + ); + cmd.as_mut().inner_mut().complete(); + workqueue::system().enqueue(cmd); + } + + mod_pr_debug!( + "WorkQueue({:?}): Completed {} commands, left pending {}, ls {:#x?}, lc {:#x?}\n", + inner.pipe_type, + completed_commands, + inner.pending.len(), + inner.last_submitted, + inner.last_completed, + ); + + inner + .info + .state + .with(|raw, _inner| raw.cpu_freeptr.store(last_wptr, Ordering::Release)); + + let empty = inner.pending.is_empty(); + if empty && inner.pending_jobs == 0 { + inner.event = None; + inner.last_submitted = None; + inner.last_completed = None; + } + + empty + } + + /// Mark this queue's work up to a certain stamp value as having failed. + fn mark_error(&self, value: event::EventValue, error: WorkError) { + // If anything is marked completed, we can consider it successful + // at this point, even if we didn't get the signal event yet. + self.signal(); + + let mut inner = self.inner.lock(); + + if inner.event.is_none() { + mod_pr_debug!("WorkQueue: signal_fault() called but no event?\n"); + + if inner.pending_jobs > 0 || !inner.pending.is_empty() { + pr_crit!("WorkQueue: signal_fault() called with no event and pending jobs.\n"); + } + return; + } + + mod_pr_debug!( + "WorkQueue({:?}): Signaling fault for event {:?} at value {:#x?}\n", + inner.pipe_type, + inner.last_token, + value + ); + + for cmd in inner.pending.iter_mut() { + if cmd.inner.value() <= value { + cmd.as_mut().inner_mut().mark_error(error); + } else { + break; + } + } + } + + /// Mark all of this queue's work as having failed, and complete it. + fn fail_all(&self, error: WorkError) { + // If anything is marked completed, we can consider it successful + // at this point, even if we didn't get the signal event yet. + self.signal(); + + let mut inner = self.inner.lock(); + + if inner.event.is_none() { + mod_pr_debug!("WorkQueue: fail_all() called but no event?\n"); + + if inner.pending_jobs > 0 || !inner.pending.is_empty() { + pr_crit!("WorkQueue: fail_all() called with no event and pending jobs.\n"); + } + return; + } + + mod_pr_debug!( + "WorkQueue({:?}): Failing all jobs {:?}\n", + inner.pipe_type, + error + ); + + let mut cmds = KVec::new(); + + core::mem::swap(&mut inner.pending, &mut cmds); + + if inner.pending_jobs == 0 { + inner.event = None; + } + + core::mem::drop(inner); + + for mut cmd in cmds { + cmd.as_mut().inner_mut().mark_error(error); + cmd.as_mut().inner_mut().complete(); + } + } +} diff --git a/drivers/gpu/drm/display/drm_dp_mst_topology.c b/drivers/gpu/drm/display/drm_dp_mst_topology.c index 64e5c176d5cce9..be749dcad3b585 100644 --- a/drivers/gpu/drm/display/drm_dp_mst_topology.c +++ b/drivers/gpu/drm/display/drm_dp_mst_topology.c @@ -4572,7 +4572,8 @@ int drm_dp_atomic_release_time_slots(struct drm_atomic_state *state, if (!payload->delete) { payload->pbn = 0; payload->delete = true; - topology_state->payload_mask &= ~BIT(payload->vcpi - 1); + if (payload->vcpi > 0) + topology_state->payload_mask &= ~BIT(payload->vcpi - 1); } return 0; diff --git a/drivers/gpu/drm/drm_atomic.c b/drivers/gpu/drm/drm_atomic.c index 67e095e398a34d..06f0205664fc5c 100644 --- a/drivers/gpu/drm/drm_atomic.c +++ b/drivers/gpu/drm/drm_atomic.c @@ -641,6 +641,38 @@ drm_atomic_get_colorop_state(struct drm_atomic_state *state, } EXPORT_SYMBOL(drm_atomic_get_colorop_state); +/** + * drm_atomic_get_old_colorop_state - get colorop state, if it exists + * @state: global atomic state object + * @colorop: colorop to grab + * + * This function returns the old colorop state for the given colorop, or + * NULL if the colorop is not part of the global atomic state. + */ +struct drm_colorop_state * +drm_atomic_get_old_colorop_state(struct drm_atomic_state *state, + struct drm_colorop *colorop) +{ + return state->colorops[drm_colorop_index(colorop)].old_state; +} +EXPORT_SYMBOL(drm_atomic_get_old_colorop_state); + +/** + * drm_atomic_get_new_colorop_state - get colorop state, if it exists + * @state: global atomic state object + * @colorop: colorop to grab + * + * This function returns the new colorop state for the given colorop, or + * NULL if the colorop is not part of the global atomic state. + */ +struct drm_colorop_state * +drm_atomic_get_new_colorop_state(struct drm_atomic_state *state, + struct drm_colorop *colorop) +{ + return state->colorops[drm_colorop_index(colorop)].new_state; +} +EXPORT_SYMBOL(drm_atomic_get_new_colorop_state); + static bool plane_switching_crtc(const struct drm_plane_state *old_plane_state, const struct drm_plane_state *new_plane_state) diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c index 5beea645035f20..cc1f0c102414f1 100644 --- a/drivers/gpu/drm/drm_atomic_helper.c +++ b/drivers/gpu/drm/drm_atomic_helper.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/gpu/drm/drm_buddy.c b/drivers/gpu/drm/drm_buddy.c index 2f279b46bd2cf8..8308116058cc1b 100644 --- a/drivers/gpu/drm/drm_buddy.c +++ b/drivers/gpu/drm/drm_buddy.c @@ -420,6 +420,7 @@ void drm_buddy_fini(struct drm_buddy *mm) for_each_free_tree(i) kfree(mm->free_trees[i]); + kfree(mm->free_trees); kfree(mm->roots); } EXPORT_SYMBOL(drm_buddy_fini); diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c index 0de47e83d84df1..c19cfb00f1fcec 100644 --- a/drivers/gpu/drm/drm_gpuvm.c +++ b/drivers/gpu/drm/drm_gpuvm.c @@ -2352,6 +2352,7 @@ op_map_cb(const struct drm_gpuvm_ops *fn, void *priv, op.map.va.range = req->map.va.range; op.map.gem.obj = req->map.gem.obj; op.map.gem.offset = req->map.gem.offset; + op.map.flags = req->map.flags; return fn->sm_step_map(&op, priv); } @@ -2390,6 +2391,97 @@ op_unmap_cb(const struct drm_gpuvm_ops *fn, void *priv, return fn->sm_step_unmap(&op, priv); } +static bool can_merge_flags(struct drm_gpuvm *gpuvm, enum drm_gpuva_flags a, + enum drm_gpuva_flags b) +{ + if (gpuvm->ops->sm_can_merge_flags) + return gpuvm->ops->sm_can_merge_flags(a, b); + return a == b; +} + +static bool __can_merge(struct drm_gpuvm *gpuvm, const struct drm_gpuva_op_map *a, + const struct drm_gpuva_op_map *b) +{ + /* Only GEM-based mappings can be merged, and they must point to + * the same GEM object. + */ + if (a->gem.obj != b->gem.obj || !a->gem.obj) + return false; + + if (can_merge_flags(gpuvm, a->flags, b->flags)) + return false; + + /* Order VAs for the rest of the checks. */ + if (a->va.addr > b->va.addr) + swap(a, b); + + /* We assume the caller already checked that VAs overlap or are + * contiguous. + */ + if (drm_WARN_ON(gpuvm->drm, b->va.addr > a->va.addr + a->va.range)) + return false; + + if (a->flags & DRM_GPUVA_REPEAT) { + u64 va_diff = b->va.addr - a->va.addr; + + /* If this is a repeated mapping, both the GEM range + * and offset must match. + */ + if (a->gem.range != b->gem.range || + a->gem.offset != b->gem.offset) + return false; + + /* The difference between the VA addresses must be a + * multiple of the repeated range, otherwise there's + * a shift. + */ + if (do_div(va_diff, a->gem.range)) + return false; + + return true; + } + + /* We intentionally ignore u64 underflows because all we care about + * here is whether the VA diff matches the GEM offset diff. + */ + return b->va.addr - a->va.addr == b->gem.offset - a->gem.offset; +} + +static bool can_merge(struct drm_gpuvm *gpuvm, const struct drm_gpuva *a, + const struct drm_gpuva_op_map *b) +{ + struct drm_gpuva_op_map tmp = { + .va.addr = a->va.addr, + .va.range = a->va.range, + .gem.offset = a->gem.offset, + .gem.obj = a->gem.obj, + .flags = a->flags, + }; + + return __can_merge(gpuvm, &tmp, b); +} + +static int validate_map_request(struct drm_gpuvm *gpuvm, + const struct drm_gpuva_op_map *req) +{ + if (unlikely(!drm_gpuvm_range_valid(gpuvm, req->va.addr, req->va.range))) + return -EINVAL; + + if (req->flags & DRM_GPUVA_REPEAT) { + u64 va_range = req->va.range; + + /* For a repeated mapping, GEM range must be > 0 + * and a multiple of the VA range. + */ + if (unlikely(!req->gem.range || + va_range < req->gem.range || + do_div(va_range, req->gem.range))) + return -EINVAL; + } + + return 0; +} + static int __drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, const struct drm_gpuvm_ops *ops, void *priv, @@ -2405,7 +2497,8 @@ __drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, u64 req_end = req_addr + req_range; int ret; - if (unlikely(!drm_gpuvm_range_valid(gpuvm, req_addr, req_range))) + ret = validate_map_request(gpuvm, &req->map); + if (unlikely(ret)) return -EINVAL; drm_gpuvm_for_each_va_range_safe(va, next, gpuvm, req_addr, req_end) { @@ -2414,7 +2507,7 @@ __drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, u64 addr = va->va.addr; u64 range = va->va.range; u64 end = addr + range; - bool merge = !!va->gem.obj; + bool merge = can_merge(gpuvm, va, &req->map); if (madvise && obj) continue; @@ -2442,7 +2535,10 @@ __drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, .va.addr = req_end, .va.range = range - req_range, .gem.obj = obj, - .gem.offset = offset + req_range, + .gem.range = va->gem.range, + .gem.offset = offset + + (va->flags & DRM_GPUVA_REPEAT ? 0 : req_range), + .flags = va->flags, }; struct drm_gpuva_op_unmap u = { .va = va, @@ -2463,7 +2559,9 @@ __drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, .va.addr = addr, .va.range = ls_range, .gem.obj = obj, + .gem.range = va->gem.range, .gem.offset = offset, + .flags = va->flags, }; struct drm_gpuva_op_unmap u = { .va = va }; @@ -2505,8 +2603,10 @@ __drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, .va.addr = req_end, .va.range = end - req_end, .gem.obj = obj, - .gem.offset = offset + ls_range + - req_range, + .gem.range = va->gem.range, + .gem.offset = offset + + (va->flags & DRM_GPUVA_REPEAT ? 0 : ls_range + req_range), + .flags = va->flags, }; ret = op_remap_cb(ops, priv, &p, &n, &u); @@ -2543,7 +2643,10 @@ __drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, .va.addr = req_end, .va.range = end - req_end, .gem.obj = obj, - .gem.offset = offset + req_end - addr, + .gem.range = va->gem.range, + .gem.offset = offset + + (va->flags & DRM_GPUVA_REPEAT ? 0 : req_end - addr), + .flags = va->flags, }; struct drm_gpuva_op_unmap u = { .va = va, @@ -2594,7 +2697,9 @@ __drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, prev.va.addr = addr; prev.va.range = req_addr - addr; prev.gem.obj = obj; + prev.gem.range = va->gem.range; prev.gem.offset = offset; + prev.flags = va->flags; prev_split = true; } @@ -2603,7 +2708,10 @@ __drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, next.va.addr = req_end; next.va.range = end - req_end; next.gem.obj = obj; - next.gem.offset = offset + (req_end - addr); + prev.gem.range = va->gem.range; + next.gem.offset = offset + + (va->flags & DRM_GPUVA_REPEAT ? 0 : req_end - addr); + next.flags = va->flags; next_split = true; } @@ -3146,6 +3254,55 @@ drm_gpuvm_prefetch_ops_create(struct drm_gpuvm *gpuvm, } EXPORT_SYMBOL_GPL(drm_gpuvm_prefetch_ops_create); +/** + * drm_gpuvm_bo_unmap() - unmaps a GEM + * @vm_bo: the &drm_gpuvm_bo abstraction + * + * This function calls the unmap callback for every GPUVA attached to a GEM. + * + * It is the callers responsibility to protect the GEMs GPUVA list against + * concurrent access using the GEMs dma_resv lock. + * + * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure + */ +int +drm_gpuvm_bo_unmap(struct drm_gpuvm_bo *vm_bo, void *priv) +{ + struct drm_gpuva_ops *ops; + struct drm_gpuva_op *op; + int ret; + + if (unlikely(!vm_bo->vm)) + return -EINVAL; + + const struct drm_gpuvm_ops *vm_ops = vm_bo->vm->ops; + + if (unlikely(!(vm_ops && vm_ops->sm_step_unmap))) + return -EINVAL; + + if (drm_gpuvm_immediate_mode(vm_bo->vm)) { + guard(mutex)(&vm_bo->obj->gpuva.lock); + ops = drm_gpuvm_bo_unmap_ops_create(vm_bo); + } else { + ops = drm_gpuvm_bo_unmap_ops_create(vm_bo); + } + if (IS_ERR(ops)) + return PTR_ERR(ops); + + drm_gpuva_for_each_op(op, ops) { + drm_WARN_ON(vm_bo->vm->drm, op->op != DRM_GPUVA_OP_UNMAP); + + ret = op_unmap_cb(vm_ops, priv, op->unmap.va, false, false); + if (ret) + goto cleanup; + } + +cleanup: + drm_gpuva_ops_free(vm_bo->vm, ops); + return ret; +} +EXPORT_SYMBOL_GPL(drm_gpuvm_bo_unmap); + /** * drm_gpuvm_bo_unmap_ops_create() - creates the &drm_gpuva_ops to unmap a GEM * @vm_bo: the &drm_gpuvm_bo abstraction diff --git a/drivers/gpu/drm/drm_plane.c b/drivers/gpu/drm/drm_plane.c index b143589717e648..bed2562bf911b2 100644 --- a/drivers/gpu/drm/drm_plane.c +++ b/drivers/gpu/drm/drm_plane.c @@ -1867,9 +1867,9 @@ int drm_plane_create_color_pipeline_property(struct drm_plane *plane, prop = drm_property_create_enum(plane->dev, DRM_MODE_PROP_ATOMIC, "COLOR_PIPELINE", all_pipelines, len); - if (IS_ERR(prop)) { + if (!prop) { kfree(all_pipelines); - return PTR_ERR(prop); + return -ENOMEM; } drm_object_attach_property(&plane->base, prop, 0); diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.h b/drivers/gpu/drm/exynos/exynos_drm_drv.h index 23646e55f142c3..06c29ff2aac0e1 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_drv.h +++ b/drivers/gpu/drm/exynos/exynos_drm_drv.h @@ -199,6 +199,7 @@ struct drm_exynos_file_private { struct exynos_drm_private { struct device *g2d_dev; struct device *dma_dev; + struct device *vidi_dev; void *mapping; /* for atomic commit */ diff --git a/drivers/gpu/drm/exynos/exynos_drm_vidi.c b/drivers/gpu/drm/exynos/exynos_drm_vidi.c index 64c69dd2966ec9..67bbf9b8bc0ef8 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_vidi.c +++ b/drivers/gpu/drm/exynos/exynos_drm_vidi.c @@ -187,29 +187,37 @@ static ssize_t vidi_store_connection(struct device *dev, const char *buf, size_t len) { struct vidi_context *ctx = dev_get_drvdata(dev); - int ret; + int ret, new_connected; - ret = kstrtoint(buf, 0, &ctx->connected); + ret = kstrtoint(buf, 0, &new_connected); if (ret) return ret; - - if (ctx->connected > 1) + if (new_connected > 1) return -EINVAL; + mutex_lock(&ctx->lock); + /* * Use fake edid data for test. If raw_edid is set then it can't be * tested. */ if (ctx->raw_edid) { DRM_DEV_DEBUG_KMS(dev, "edid data is not fake data.\n"); - return -EINVAL; + ret = -EINVAL; + goto fail; } + ctx->connected = new_connected; + mutex_unlock(&ctx->lock); + DRM_DEV_DEBUG_KMS(dev, "requested connection.\n"); drm_helper_hpd_irq_event(ctx->drm_dev); return len; +fail: + mutex_unlock(&ctx->lock); + return ret; } static DEVICE_ATTR(connection, 0644, vidi_show_connection, @@ -224,9 +232,14 @@ ATTRIBUTE_GROUPS(vidi); int vidi_connection_ioctl(struct drm_device *drm_dev, void *data, struct drm_file *file_priv) { - struct vidi_context *ctx = dev_get_drvdata(drm_dev->dev); + struct exynos_drm_private *priv = drm_dev->dev_private; + struct device *dev = priv ? priv->vidi_dev : NULL; + struct vidi_context *ctx = dev ? dev_get_drvdata(dev) : NULL; struct drm_exynos_vidi_connection *vidi = data; + if (!ctx) + return -ENODEV; + if (!vidi) { DRM_DEV_DEBUG_KMS(ctx->dev, "user data for vidi is null.\n"); @@ -239,21 +252,38 @@ int vidi_connection_ioctl(struct drm_device *drm_dev, void *data, return -EINVAL; } + mutex_lock(&ctx->lock); if (ctx->connected == vidi->connection) { + mutex_unlock(&ctx->lock); DRM_DEV_DEBUG_KMS(ctx->dev, "same connection request.\n"); return -EINVAL; } + mutex_unlock(&ctx->lock); if (vidi->connection) { const struct drm_edid *drm_edid; - const struct edid *raw_edid; + const void __user *edid_userptr = u64_to_user_ptr(vidi->edid); + void *edid_buf; + struct edid hdr; size_t size; - raw_edid = (const struct edid *)(unsigned long)vidi->edid; - size = (raw_edid->extensions + 1) * EDID_LENGTH; + if (copy_from_user(&hdr, edid_userptr, sizeof(hdr))) + return -EFAULT; + + size = (hdr.extensions + 1) * EDID_LENGTH; + + edid_buf = kmalloc(size, GFP_KERNEL); + if (!edid_buf) + return -ENOMEM; - drm_edid = drm_edid_alloc(raw_edid, size); + if (copy_from_user(edid_buf, edid_userptr, size)) { + kfree(edid_buf); + return -EFAULT; + } + + drm_edid = drm_edid_alloc(edid_buf, size); + kfree(edid_buf); if (!drm_edid) return -ENOMEM; @@ -263,14 +293,21 @@ int vidi_connection_ioctl(struct drm_device *drm_dev, void *data, "edid data is invalid.\n"); return -EINVAL; } + mutex_lock(&ctx->lock); ctx->raw_edid = drm_edid; + mutex_unlock(&ctx->lock); } else { /* with connection = 0, free raw_edid */ + mutex_lock(&ctx->lock); drm_edid_free(ctx->raw_edid); ctx->raw_edid = NULL; + mutex_unlock(&ctx->lock); } + mutex_lock(&ctx->lock); ctx->connected = vidi->connection; + mutex_unlock(&ctx->lock); + drm_helper_hpd_irq_event(ctx->drm_dev); return 0; @@ -285,7 +322,7 @@ static enum drm_connector_status vidi_detect(struct drm_connector *connector, * connection request would come from user side * to do hotplug through specific ioctl. */ - return ctx->connected ? connector_status_connected : + return READ_ONCE(ctx->connected) ? connector_status_connected : connector_status_disconnected; } @@ -308,11 +345,15 @@ static int vidi_get_modes(struct drm_connector *connector) const struct drm_edid *drm_edid; int count; + mutex_lock(&ctx->lock); + if (ctx->raw_edid) drm_edid = drm_edid_dup(ctx->raw_edid); else drm_edid = drm_edid_alloc(fake_edid_info, sizeof(fake_edid_info)); + mutex_unlock(&ctx->lock); + drm_edid_connector_update(connector, drm_edid); count = drm_edid_connector_add_modes(connector); @@ -372,6 +413,7 @@ static int vidi_bind(struct device *dev, struct device *master, void *data) { struct vidi_context *ctx = dev_get_drvdata(dev); struct drm_device *drm_dev = data; + struct exynos_drm_private *priv = drm_dev->dev_private; struct drm_encoder *encoder = &ctx->encoder; struct exynos_drm_plane *exynos_plane; struct exynos_drm_plane_config plane_config = { 0 }; @@ -379,6 +421,8 @@ static int vidi_bind(struct device *dev, struct device *master, void *data) int ret; ctx->drm_dev = drm_dev; + if (priv) + priv->vidi_dev = dev; plane_config.pixel_formats = formats; plane_config.num_pixel_formats = ARRAY_SIZE(formats); @@ -424,8 +468,12 @@ static int vidi_bind(struct device *dev, struct device *master, void *data) static void vidi_unbind(struct device *dev, struct device *master, void *data) { struct vidi_context *ctx = dev_get_drvdata(dev); + struct drm_device *drm_dev = data; + struct exynos_drm_private *priv = drm_dev->dev_private; timer_delete_sync(&ctx->timer); + if (priv) + priv->vidi_dev = NULL; } static const struct component_ops vidi_component_ops = { @@ -457,9 +505,13 @@ static void vidi_remove(struct platform_device *pdev) { struct vidi_context *ctx = platform_get_drvdata(pdev); + mutex_lock(&ctx->lock); + drm_edid_free(ctx->raw_edid); ctx->raw_edid = NULL; + mutex_unlock(&ctx->lock); + component_del(&pdev->dev, &vidi_component_ops); } diff --git a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_comm.h b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_comm.h index 4add05c7f161a0..f9ee7ebfec55c3 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_comm.h +++ b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_comm.h @@ -40,6 +40,10 @@ struct hibmc_dp_dev { struct mutex lock; /* protects concurrent RW in hibmc_dp_reg_write_field() */ struct hibmc_dp_link link; u8 dpcd[DP_RECEIVER_CAP_SIZE]; + u8 downstream_ports[DP_MAX_DOWNSTREAM_PORTS]; + struct drm_dp_desc desc; + bool is_branch; + int hpd_status; void __iomem *serdes_base; }; diff --git a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_config.h b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_config.h index 08f9e1caf7fcbb..efb30a7584758c 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_config.h +++ b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_config.h @@ -17,5 +17,7 @@ #define HIBMC_DP_LINK_RATE_CAL 27 #define HIBMC_DP_SYNC_DELAY(lanes) ((lanes) == 0x2 ? 86 : 46) #define HIBMC_DP_INT_ENABLE 0xc +/* HIBMC_DP_LINK_RATE_CAL * 10000 * 80% = 216000 */ +#define DP_MODE_VALI_CAL 216000 #endif diff --git a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.c b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.c index 8f0daec7d17490..d5bd3c45649b2f 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.c +++ b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.c @@ -2,6 +2,7 @@ // Copyright (c) 2024 Hisilicon Limited. #include +#include #include #include "dp_config.h" #include "dp_comm.h" @@ -176,13 +177,18 @@ int hibmc_dp_hw_init(struct hibmc_dp *dp) dp_dev->link.cap.lanes = 0x2; dp_dev->link.cap.link_rate = DP_LINK_BW_8_1; - /* hdcp data */ - writel(HIBMC_DP_HDCP, dp_dev->base + HIBMC_DP_HDCP_CFG); /* int init */ writel(0, dp_dev->base + HIBMC_DP_INTR_ENABLE); writel(HIBMC_DP_INT_RST, dp_dev->base + HIBMC_DP_INTR_ORIGINAL_STATUS); + /* clr colorbar */ + writel(0, dp_dev->base + HIBMC_DP_COLOR_BAR_CTRL); /* rst */ + writel(0, dp_dev->base + HIBMC_DP_DPTX_RST_CTRL); + usleep_range(30, 50); + /* de-rst */ writel(HIBMC_DP_DPTX_RST, dp_dev->base + HIBMC_DP_DPTX_RST_CTRL); + /* hdcp data */ + writel(HIBMC_DP_HDCP, dp_dev->base + HIBMC_DP_HDCP_CFG); /* clock enable */ writel(HIBMC_DP_CLK_EN, dp_dev->base + HIBMC_DP_DPTX_CLK_CTRL); @@ -263,6 +269,16 @@ void hibmc_dp_reset_link(struct hibmc_dp *dp) dp->dp_dev->link.status.channel_equalized = false; } +u8 hibmc_dp_get_link_rate(struct hibmc_dp *dp) +{ + return dp->dp_dev->link.cap.link_rate; +} + +u8 hibmc_dp_get_lanes(struct hibmc_dp *dp) +{ + return dp->dp_dev->link.cap.lanes; +} + static const struct hibmc_dp_color_raw g_rgb_raw[] = { {CBAR_COLOR_BAR, 0x000, 0x000, 0x000}, {CBAR_WHITE, 0xfff, 0xfff, 0xfff}, @@ -305,3 +321,21 @@ void hibmc_dp_set_cbar(struct hibmc_dp *dp, const struct hibmc_dp_cbar_cfg *cfg) hibmc_dp_reg_write_field(dp_dev, HIBMC_DP_COLOR_BAR_CTRL, BIT(0), cfg->enable); writel(HIBMC_DP_SYNC_EN_MASK, dp_dev->base + HIBMC_DP_TIMING_SYNC_CTRL); } + +bool hibmc_dp_check_hpd_status(struct hibmc_dp *dp, int exp_status) +{ + u32 status; + int ret; + + ret = readl_poll_timeout(dp->dp_dev->base + HIBMC_DP_HPD_STATUS, status, + FIELD_GET(HIBMC_DP_HPD_CUR_STATE, status) == exp_status, + 1000, 100000); /* DP spec says 100ms */ + if (ret) { + drm_dbg_dp(dp->drm_dev, "wait hpd status timeout"); + return false; + } + + dp->dp_dev->hpd_status = exp_status; + + return true; +} diff --git a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.h b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.h index 665f5b166dfb5c..31316fe1ea8dbf 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.h +++ b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.h @@ -14,6 +14,11 @@ struct hibmc_dp_dev; +enum hibmc_hpd_status { + HIBMC_HPD_OUT, + HIBMC_HPD_IN, +}; + enum hibmc_dp_cbar_pattern { CBAR_COLOR_BAR, CBAR_WHITE, @@ -60,5 +65,8 @@ void hibmc_dp_reset_link(struct hibmc_dp *dp); void hibmc_dp_hpd_cfg(struct hibmc_dp *dp); void hibmc_dp_enable_int(struct hibmc_dp *dp); void hibmc_dp_disable_int(struct hibmc_dp *dp); +bool hibmc_dp_check_hpd_status(struct hibmc_dp *dp, int exp_status); +u8 hibmc_dp_get_link_rate(struct hibmc_dp *dp); +u8 hibmc_dp_get_lanes(struct hibmc_dp *dp); #endif diff --git a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_reg.h b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_reg.h index 394b1e933c3ae7..64306abcd98666 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_reg.h +++ b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_reg.h @@ -24,6 +24,9 @@ #define HIBMC_DP_CFG_AUX_READY_DATA_BYTE GENMASK(16, 12) #define HIBMC_DP_CFG_AUX GENMASK(24, 17) +#define HIBMC_DP_HPD_STATUS 0x98 +#define HIBMC_DP_HPD_CUR_STATE GENMASK(7, 4) + #define HIBMC_DP_PHYIF_CTRL0 0xa0 #define HIBMC_DP_CFG_SCRAMBLE_EN BIT(0) #define HIBMC_DP_CFG_PAT_SEL GENMASK(7, 4) diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_dp.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_dp.c index d06832e62e966e..616821e3c933bc 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_dp.c +++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_dp.c @@ -12,6 +12,8 @@ #include "hibmc_drm_drv.h" #include "dp/dp_hw.h" +#include "dp/dp_comm.h" +#include "dp/dp_config.h" #define DP_MASKED_SINK_HPD_PLUG_INT BIT(2) @@ -31,17 +33,76 @@ static int hibmc_dp_connector_get_modes(struct drm_connector *connector) return count; } +static bool hibmc_dp_get_dpcd(struct hibmc_dp_dev *dp_dev) +{ + int ret; + + ret = drm_dp_read_dpcd_caps(dp_dev->aux, dp_dev->dpcd); + if (ret) + return false; + + dp_dev->is_branch = drm_dp_is_branch(dp_dev->dpcd); + + ret = drm_dp_read_desc(dp_dev->aux, &dp_dev->desc, dp_dev->is_branch); + if (ret) + return false; + + ret = drm_dp_read_downstream_info(dp_dev->aux, dp_dev->dpcd, dp_dev->downstream_ports); + if (ret) + return false; + + return true; +} + static int hibmc_dp_detect(struct drm_connector *connector, struct drm_modeset_acquire_ctx *ctx, bool force) { - mdelay(200); + struct hibmc_dp *dp = to_hibmc_dp(connector); + struct hibmc_dp_dev *dp_dev = dp->dp_dev; + int ret; + + if (dp->irq_status) { + if (dp_dev->hpd_status != HIBMC_HPD_IN) + return connector_status_disconnected; + } + + if (!hibmc_dp_get_dpcd(dp_dev)) + return connector_status_disconnected; + + if (!dp_dev->is_branch) + return connector_status_connected; + + if (drm_dp_read_sink_count_cap(connector, dp_dev->dpcd, &dp_dev->desc) && + dp_dev->downstream_ports[0] & DP_DS_PORT_HPD) { + ret = drm_dp_read_sink_count(dp_dev->aux); + if (ret > 0) + return connector_status_connected; + } + + return connector_status_disconnected; +} + +static int hibmc_dp_mode_valid(struct drm_connector *connector, + const struct drm_display_mode *mode, + struct drm_modeset_acquire_ctx *ctx, + enum drm_mode_status *status) +{ + struct hibmc_dp *dp = to_hibmc_dp(connector); + u64 cur_val, max_val; - return drm_connector_helper_detect_from_ddc(connector, ctx, force); + /* check DP link BW */ + cur_val = (u64)mode->clock * HIBMC_DP_BPP; + max_val = (u64)hibmc_dp_get_link_rate(dp) * DP_MODE_VALI_CAL * hibmc_dp_get_lanes(dp); + + *status = cur_val > max_val ? MODE_CLOCK_HIGH : MODE_OK; + + return 0; } static const struct drm_connector_helper_funcs hibmc_dp_conn_helper_funcs = { .get_modes = hibmc_dp_connector_get_modes, .detect_ctx = hibmc_dp_detect, + .mode_valid_ctx = hibmc_dp_mode_valid, }; static int hibmc_dp_late_register(struct drm_connector *connector) @@ -115,7 +176,7 @@ irqreturn_t hibmc_dp_hpd_isr(int irq, void *arg) { struct drm_device *dev = (struct drm_device *)arg; struct hibmc_drm_private *priv = to_hibmc_drm_private(dev); - int idx; + int idx, exp_status; if (!drm_dev_enter(dev, &idx)) return -ENODEV; @@ -123,12 +184,14 @@ irqreturn_t hibmc_dp_hpd_isr(int irq, void *arg) if (priv->dp.irq_status & DP_MASKED_SINK_HPD_PLUG_INT) { drm_dbg_dp(&priv->dev, "HPD IN isr occur!\n"); hibmc_dp_hpd_cfg(&priv->dp); + exp_status = HIBMC_HPD_IN; } else { drm_dbg_dp(&priv->dev, "HPD OUT isr occur!\n"); hibmc_dp_reset_link(&priv->dp); + exp_status = HIBMC_HPD_OUT; } - if (dev->registered) + if (hibmc_dp_check_hpd_status(&priv->dp, exp_status)) drm_connector_helper_hpd_irq_event(&priv->dp.connector); drm_dev_exit(idx); diff --git a/drivers/gpu/drm/i915/display/intel_acpi.c b/drivers/gpu/drm/i915/display/intel_acpi.c index 68c01932f7b4f6..e06f324027bec9 100644 --- a/drivers/gpu/drm/i915/display/intel_acpi.c +++ b/drivers/gpu/drm/i915/display/intel_acpi.c @@ -96,6 +96,7 @@ static void intel_dsm_platform_mux_info(acpi_handle dhandle) if (!pkg->package.count) { DRM_DEBUG_DRIVER("no connection in _DSM\n"); + ACPI_FREE(pkg); return; } diff --git a/drivers/gpu/drm/i915/display/intel_colorop.c b/drivers/gpu/drm/i915/display/intel_colorop.c index f2fc0d8780ceea..1d84933f05aa9f 100644 --- a/drivers/gpu/drm/i915/display/intel_colorop.c +++ b/drivers/gpu/drm/i915/display/intel_colorop.c @@ -2,7 +2,9 @@ /* * Copyright © 2025 Intel Corporation */ + #include "intel_colorop.h" +#include "intel_display_types.h" struct intel_colorop *to_intel_colorop(struct drm_colorop *colorop) { diff --git a/drivers/gpu/drm/i915/display/intel_colorop.h b/drivers/gpu/drm/i915/display/intel_colorop.h index 21d58eb9f3d0f6..9276eee6e75a3e 100644 --- a/drivers/gpu/drm/i915/display/intel_colorop.h +++ b/drivers/gpu/drm/i915/display/intel_colorop.h @@ -6,7 +6,9 @@ #ifndef __INTEL_COLOROP_H__ #define __INTEL_COLOROP_H__ -#include "intel_display_types.h" +enum intel_color_block; +struct drm_colorop; +struct intel_colorop; struct intel_colorop *to_intel_colorop(struct drm_colorop *colorop); struct intel_colorop *intel_colorop_alloc(void); diff --git a/drivers/gpu/drm/i915/display/intel_display_device.h b/drivers/gpu/drm/i915/display/intel_display_device.h index b559ef43d54704..6944d081f0ad3b 100644 --- a/drivers/gpu/drm/i915/display/intel_display_device.h +++ b/drivers/gpu/drm/i915/display/intel_display_device.h @@ -189,6 +189,7 @@ struct intel_display_platforms { #define HAS_MSO(__display) (DISPLAY_VER(__display) >= 12) #define HAS_OVERLAY(__display) (DISPLAY_INFO(__display)->has_overlay) #define HAS_PIPEDMC(__display) (DISPLAY_VER(__display) >= 12) +#define HAS_PIXEL_NORMALIZER(__display) (DISPLAY_VER(__display) >= 35) #define HAS_PSR(__display) (DISPLAY_INFO(__display)->has_psr) #define HAS_PSR_HW_TRACKING(__display) (DISPLAY_INFO(__display)->has_psr_hw_tracking) #define HAS_PSR2_SEL_FETCH(__display) (DISPLAY_VER(__display) >= 12) diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h index 06bf8f7c0989b0..6e26751e8d0e30 100644 --- a/drivers/gpu/drm/i915/display/intel_display_types.h +++ b/drivers/gpu/drm/i915/display/intel_display_types.h @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/gpu/drm/i915/display/intel_fbc.c b/drivers/gpu/drm/i915/display/intel_fbc.c index 437d2fda20a7e7..8d387709f25cc5 100644 --- a/drivers/gpu/drm/i915/display/intel_fbc.c +++ b/drivers/gpu/drm/i915/display/intel_fbc.c @@ -1120,13 +1120,15 @@ static bool xe3p_lpd_fbc_pixel_format_is_valid(const struct intel_plane_state *p } } -bool -intel_fbc_is_enable_pixel_normalizer(const struct intel_plane_state *plane_state) +bool intel_fbc_need_pixel_normalizer(const struct intel_plane_state *plane_state) { struct intel_display *display = to_intel_display(plane_state); - return DISPLAY_VER(display) >= 35 && - xe3p_lpd_fbc_fp16_format_is_valid(plane_state); + if (HAS_PIXEL_NORMALIZER(display) && + xe3p_lpd_fbc_fp16_format_is_valid(plane_state)) + return true; + + return false; } static bool pixel_format_is_valid(const struct intel_plane_state *plane_state) diff --git a/drivers/gpu/drm/i915/display/intel_fbc.h b/drivers/gpu/drm/i915/display/intel_fbc.h index 91424563206a31..7e2416b29a0ea2 100644 --- a/drivers/gpu/drm/i915/display/intel_fbc.h +++ b/drivers/gpu/drm/i915/display/intel_fbc.h @@ -53,7 +53,6 @@ void intel_fbc_prepare_dirty_rect(struct intel_atomic_state *state, struct intel_crtc *crtc); void intel_fbc_dirty_rect_update_noarm(struct intel_dsb *dsb, struct intel_plane *plane); -bool -intel_fbc_is_enable_pixel_normalizer(const struct intel_plane_state *plane_state); +bool intel_fbc_need_pixel_normalizer(const struct intel_plane_state *plane_state); #endif /* __INTEL_FBC_H__ */ diff --git a/drivers/gpu/drm/i915/display/skl_universal_plane.c b/drivers/gpu/drm/i915/display/skl_universal_plane.c index ee8e24497d2cfd..ed14b9ea2ad2d2 100644 --- a/drivers/gpu/drm/i915/display/skl_universal_plane.c +++ b/drivers/gpu/drm/i915/display/skl_universal_plane.c @@ -892,23 +892,20 @@ static void icl_plane_disable_sel_fetch_arm(struct intel_dsb *dsb, intel_de_write_dsb(display, dsb, SEL_FETCH_PLANE_CTL(pipe, plane->id), 0); } -static void x3p_lpd_plane_update_pixel_normalizer(struct intel_dsb *dsb, - struct intel_plane *plane, - bool enable) +static bool plane_has_normalizer(struct intel_plane *plane) { struct intel_display *display = to_intel_display(plane); - enum intel_fbc_id fbc_id = skl_fbc_id_for_pipe(plane->pipe); - u32 val; - /* Only HDR planes have pixel normalizer and don't matter if no FBC */ - if (!skl_plane_has_fbc(display, fbc_id, plane->id)) - return; + return HAS_PIXEL_NORMALIZER(display) && icl_is_hdr_plane(display, plane->id); +} - val = enable ? PLANE_PIXEL_NORMALIZE_NORM_FACTOR(PLANE_PIXEL_NORMALIZE_NORM_FACTOR_1_0) | - PLANE_PIXEL_NORMALIZE_ENABLE : 0; +static u32 pixel_normalizer_value(const struct intel_plane_state *plane_state) +{ + if (!intel_fbc_need_pixel_normalizer(plane_state)) + return 0; - intel_de_write_dsb(display, dsb, - PLANE_PIXEL_NORMALIZE(plane->pipe, plane->id), val); + return PLANE_PIXEL_NORMALIZE_ENABLE | + PLANE_PIXEL_NORMALIZE_NORM_FACTOR(PLANE_PIXEL_NORMALIZE_NORM_FACTOR_1_0); } static void @@ -927,8 +924,9 @@ icl_plane_disable_arm(struct intel_dsb *dsb, icl_plane_disable_sel_fetch_arm(dsb, plane, crtc_state); - if (DISPLAY_VER(display) >= 35) - x3p_lpd_plane_update_pixel_normalizer(dsb, plane, false); + if (plane_has_normalizer(plane)) + intel_de_write_dsb(display, dsb, + PLANE_PIXEL_NORMALIZE(plane->pipe, plane->id), 0); intel_de_write_dsb(display, dsb, PLANE_CTL(pipe, plane_id), 0); intel_de_write_dsb(display, dsb, PLANE_SURF(pipe, plane_id), 0); @@ -1677,11 +1675,13 @@ icl_plane_update_arm(struct intel_dsb *dsb, /* * In order to have FBC for fp16 formats pixel normalizer block must be - * active. Check if pixel normalizer block need to be enabled for FBC. - * If needed, use normalization factor as 1.0 and enable the block. + * active. For FP16 formats, use normalization factor as 1.0 and enable + * the block. */ - if (intel_fbc_is_enable_pixel_normalizer(plane_state)) - x3p_lpd_plane_update_pixel_normalizer(dsb, plane, true); + if (plane_has_normalizer(plane)) + intel_de_write_dsb(display, dsb, + PLANE_PIXEL_NORMALIZE(plane->pipe, plane->id), + pixel_normalizer_value(plane_state)); /* * The control register self-arms if the plane was previously diff --git a/drivers/gpu/drm/msm/adreno/a2xx_gpu.c b/drivers/gpu/drm/msm/adreno/a2xx_gpu.c index 1b1ee14b65cf03..f6e48bb8cc69ee 100644 --- a/drivers/gpu/drm/msm/adreno/a2xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a2xx_gpu.c @@ -77,7 +77,10 @@ static bool a2xx_me_init(struct msm_gpu *gpu) /* Vertex and Pixel Shader Start Addresses in instructions * (3 DWORDS per instruction) */ - OUT_RING(ring, 0x80000180); + if (adreno_is_a225(adreno_gpu)) + OUT_RING(ring, 0x80000300); + else + OUT_RING(ring, 0x80000180); /* Maximum Contexts */ OUT_RING(ring, 0x00000001); /* Write Confirm Interval and The CP will wait the diff --git a/drivers/gpu/drm/msm/adreno/a6xx_catalog.c b/drivers/gpu/drm/msm/adreno/a6xx_catalog.c index 4c042133261c96..550a53a7865eb0 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_catalog.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_catalog.c @@ -1689,7 +1689,7 @@ static const struct adreno_reglist_pipe x285_nonctxt_regs[] = { { REG_A7XX_SP_READ_SEL, 0x0001ff00, BIT(PIPE_NONE) }, { REG_A6XX_TPL1_DBG_ECO_CNTL, 0x10000000, BIT(PIPE_NONE) }, /* BIT(26): Disable final clamp for bicubic filtering */ - { REG_A6XX_TPL1_DBG_ECO_CNTL1, 0x00000720, BIT(PIPE_NONE) }, + { REG_A6XX_TPL1_DBG_ECO_CNTL1, 0x04000720, BIT(PIPE_NONE) }, { REG_A6XX_UCHE_MODE_CNTL, 0x80080000, BIT(PIPE_NONE) }, { REG_A8XX_UCHE_CCHE_MODE_CNTL, 0x00001000, BIT(PIPE_NONE) }, { REG_A8XX_UCHE_CCHE_CACHE_WAYS, 0x00000800, BIT(PIPE_NONE) }, diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.c b/drivers/gpu/drm/msm/adreno/adreno_gpu.c index 1c80909e63cab4..d5fe6f6f0decc4 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_gpu.c +++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.c @@ -376,8 +376,7 @@ int adreno_get_param(struct msm_gpu *gpu, struct msm_context *ctx, *value = adreno_gpu->info->gmem; return 0; case MSM_PARAM_GMEM_BASE: - if (adreno_is_a650_family(adreno_gpu) || - adreno_is_a740_family(adreno_gpu)) + if (adreno_gpu->info->family >= ADRENO_6XX_GEN4) *value = 0; else *value = 0x100000; diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_0_msm8998.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_0_msm8998.h index f91220496082bd..b1b03d8b30fa0c 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_0_msm8998.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_0_msm8998.h @@ -42,24 +42,19 @@ static const struct dpu_ctl_cfg msm8998_ctl[] = { .name = "ctl_0", .id = CTL_0, .base = 0x1000, .len = 0x94, .features = BIT(DPU_CTL_SPLIT_DISPLAY), - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9), }, { .name = "ctl_1", .id = CTL_1, .base = 0x1200, .len = 0x94, - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10), }, { .name = "ctl_2", .id = CTL_2, .base = 0x1400, .len = 0x94, .features = BIT(DPU_CTL_SPLIT_DISPLAY), - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11), }, { .name = "ctl_3", .id = CTL_3, .base = 0x1600, .len = 0x94, - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12), }, { .name = "ctl_4", .id = CTL_4, .base = 0x1800, .len = 0x94, - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 13), }, }; diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_2_sdm660.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_2_sdm660.h index 8f9a097147c02b..64df4e80ea43de 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_2_sdm660.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_2_sdm660.h @@ -37,24 +37,19 @@ static const struct dpu_ctl_cfg sdm660_ctl[] = { .name = "ctl_0", .id = CTL_0, .base = 0x1000, .len = 0x94, .features = BIT(DPU_CTL_SPLIT_DISPLAY), - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9), }, { .name = "ctl_1", .id = CTL_1, .base = 0x1200, .len = 0x94, - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10), }, { .name = "ctl_2", .id = CTL_2, .base = 0x1400, .len = 0x94, .features = BIT(DPU_CTL_SPLIT_DISPLAY), - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11), }, { .name = "ctl_3", .id = CTL_3, .base = 0x1600, .len = 0x94, - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12), }, { .name = "ctl_4", .id = CTL_4, .base = 0x1800, .len = 0x94, - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 13), }, }; diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_3_sdm630.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_3_sdm630.h index 0ad18bd273ff8c..b409af89991820 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_3_sdm630.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_3_3_sdm630.h @@ -36,24 +36,19 @@ static const struct dpu_ctl_cfg sdm630_ctl[] = { .name = "ctl_0", .id = CTL_0, .base = 0x1000, .len = 0x94, .features = BIT(DPU_CTL_SPLIT_DISPLAY), - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9), }, { .name = "ctl_1", .id = CTL_1, .base = 0x1200, .len = 0x94, - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10), }, { .name = "ctl_2", .id = CTL_2, .base = 0x1400, .len = 0x94, .features = BIT(DPU_CTL_SPLIT_DISPLAY), - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11), }, { .name = "ctl_3", .id = CTL_3, .base = 0x1600, .len = 0x94, - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12), }, { .name = "ctl_4", .id = CTL_4, .base = 0x1800, .len = 0x94, - .intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 13), }, }; diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_7_2_sc7280.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_7_2_sc7280.h index 8f978b9c345202..2f8688224f3430 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_7_2_sc7280.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_7_2_sc7280.h @@ -13,6 +13,7 @@ static const struct dpu_caps sc7280_dpu_caps = { .has_dim_layer = true, .has_idle_pc = true, .max_linewidth = 2400, + .has_3d_merge = true, .pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE, }; @@ -134,17 +135,24 @@ static const struct dpu_pingpong_cfg sc7280_pp[] = { .name = "pingpong_2", .id = PINGPONG_2, .base = 0x6b000, .len = 0, .sblk = &sc7280_pp_sblk, - .merge_3d = 0, + .merge_3d = MERGE_3D_1, .intr_done = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 10), }, { .name = "pingpong_3", .id = PINGPONG_3, .base = 0x6c000, .len = 0, .sblk = &sc7280_pp_sblk, - .merge_3d = 0, + .merge_3d = MERGE_3D_1, .intr_done = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 11), }, }; +static const struct dpu_merge_3d_cfg sc7280_merge_3d[] = { + { + .name = "merge_3d_1", .id = MERGE_3D_1, + .base = 0x4f000, .len = 0x8, + }, +}; + /* NOTE: sc7280 only has one DSC hard slice encoder */ static const struct dpu_dsc_cfg sc7280_dsc[] = { { @@ -247,6 +255,8 @@ const struct dpu_mdss_cfg dpu_sc7280_cfg = { .mixer = sc7280_lm, .pingpong_count = ARRAY_SIZE(sc7280_pp), .pingpong = sc7280_pp, + .merge_3d_count = ARRAY_SIZE(sc7280_merge_3d), + .merge_3d = sc7280_merge_3d, .dsc_count = ARRAY_SIZE(sc7280_dsc), .dsc = sc7280_dsc, .wb_count = ARRAY_SIZE(sc7280_wb), diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c index 9f3957f24c6a3d..eba1d52211f68f 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c @@ -785,24 +785,24 @@ static void _dpu_encoder_update_vsync_source(struct dpu_encoder_virt *dpu_enc, return; } + vsync_cfg.vsync_source = disp_info->vsync_source; + vsync_cfg.frame_rate = drm_mode_vrefresh(&dpu_enc->base.crtc->state->adjusted_mode); + if (hw_mdptop->ops.setup_vsync_source) { for (i = 0; i < dpu_enc->num_phys_encs; i++) vsync_cfg.ppnumber[i] = dpu_enc->hw_pp[i]->idx; vsync_cfg.pp_count = dpu_enc->num_phys_encs; - vsync_cfg.frame_rate = drm_mode_vrefresh(&dpu_enc->base.crtc->state->adjusted_mode); - - vsync_cfg.vsync_source = disp_info->vsync_source; hw_mdptop->ops.setup_vsync_source(hw_mdptop, &vsync_cfg); + } - for (i = 0; i < dpu_enc->num_phys_encs; i++) { - phys_enc = dpu_enc->phys_encs[i]; + for (i = 0; i < dpu_enc->num_phys_encs; i++) { + phys_enc = dpu_enc->phys_encs[i]; - if (phys_enc->has_intf_te && phys_enc->hw_intf->ops.vsync_sel) - phys_enc->hw_intf->ops.vsync_sel(phys_enc->hw_intf, - vsync_cfg.vsync_source); - } + if (phys_enc->has_intf_te && phys_enc->hw_intf->ops.vsync_sel) + phys_enc->hw_intf->ops.vsync_sel(phys_enc->hw_intf, + &vsync_cfg); } } diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c index 0ec6d67c7c70b1..93db1484f60698 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c @@ -681,10 +681,11 @@ static int dpu_encoder_phys_cmd_wait_for_commit_done( if (!dpu_encoder_phys_cmd_is_master(phys_enc)) return 0; - if (phys_enc->hw_ctl->ops.is_started(phys_enc->hw_ctl)) - return dpu_encoder_phys_cmd_wait_for_tx_complete(phys_enc); + if (phys_enc->irq[INTR_IDX_CTL_START] && + !phys_enc->hw_ctl->ops.is_started(phys_enc->hw_ctl)) + return _dpu_encoder_phys_cmd_wait_for_ctl_start(phys_enc); - return _dpu_encoder_phys_cmd_wait_for_ctl_start(phys_enc); + return dpu_encoder_phys_cmd_wait_for_tx_complete(phys_enc); } static void dpu_encoder_phys_cmd_handle_post_kickoff( diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.c index a80ac82a96255d..7e620f59098499 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.c @@ -67,6 +67,10 @@ #define INTF_MISR_CTRL 0x180 #define INTF_MISR_SIGNATURE 0x184 +#define INTF_WD_TIMER_0_CTL 0x230 +#define INTF_WD_TIMER_0_CTL2 0x234 +#define INTF_WD_TIMER_0_LOAD_VALUE 0x238 + #define INTF_MUX 0x25C #define INTF_STATUS 0x26C #define INTF_AVR_CONTROL 0x270 @@ -475,7 +479,20 @@ static int dpu_hw_intf_get_vsync_info(struct dpu_hw_intf *intf, } static void dpu_hw_intf_vsync_sel(struct dpu_hw_intf *intf, - enum dpu_vsync_source vsync_source) + struct dpu_vsync_source_cfg *cfg) +{ + struct dpu_hw_blk_reg_map *c; + + if (!intf) + return; + + c = &intf->hw; + + DPU_REG_WRITE(c, INTF_TEAR_MDP_VSYNC_SEL, (cfg->vsync_source & 0xf)); +} + +static void dpu_hw_intf_vsync_sel_v8(struct dpu_hw_intf *intf, + struct dpu_vsync_source_cfg *cfg) { struct dpu_hw_blk_reg_map *c; @@ -484,7 +501,30 @@ static void dpu_hw_intf_vsync_sel(struct dpu_hw_intf *intf, c = &intf->hw; - DPU_REG_WRITE(c, INTF_TEAR_MDP_VSYNC_SEL, (vsync_source & 0xf)); + if (cfg->vsync_source >= DPU_VSYNC_SOURCE_WD_TIMER_4 && + cfg->vsync_source <= DPU_VSYNC_SOURCE_WD_TIMER_1) { + pr_warn_once("DPU 8.x supports only GPIOs and timer0 as TE sources\n"); + return; + } + + if (cfg->vsync_source == DPU_VSYNC_SOURCE_WD_TIMER_0) { + u32 reg; + + DPU_REG_WRITE(c, INTF_WD_TIMER_0_LOAD_VALUE, + CALCULATE_WD_LOAD_VALUE(cfg->frame_rate)); + + DPU_REG_WRITE(c, INTF_WD_TIMER_0_CTL, BIT(0)); /* clear timer */ + + reg = BIT(8); /* enable heartbeat timer */ + reg |= BIT(0); /* enable WD timer */ + reg |= BIT(1); /* select default 16 clock ticks */ + DPU_REG_WRITE(c, INTF_WD_TIMER_0_CTL2, reg); + + /* make sure that timers are enabled/disabled for vsync state */ + wmb(); + } + + dpu_hw_intf_vsync_sel(intf, cfg); } static void dpu_hw_intf_disable_autorefresh(struct dpu_hw_intf *intf, @@ -598,7 +638,10 @@ struct dpu_hw_intf *dpu_hw_intf_init(struct drm_device *dev, c->ops.enable_tearcheck = dpu_hw_intf_enable_te; c->ops.disable_tearcheck = dpu_hw_intf_disable_te; c->ops.connect_external_te = dpu_hw_intf_connect_external_te; - c->ops.vsync_sel = dpu_hw_intf_vsync_sel; + if (mdss_rev->core_major_ver >= 8) + c->ops.vsync_sel = dpu_hw_intf_vsync_sel_v8; + else + c->ops.vsync_sel = dpu_hw_intf_vsync_sel; c->ops.disable_autorefresh = dpu_hw_intf_disable_autorefresh; } diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.h index 5a19cd74fa9471..f6ef2c21b66d42 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.h @@ -12,6 +12,7 @@ #include "dpu_hw_util.h" struct dpu_hw_intf; +struct dpu_vsync_source_cfg; /* intf timing settings */ struct dpu_hw_intf_timing_params { @@ -104,7 +105,7 @@ struct dpu_hw_intf_ops { int (*connect_external_te)(struct dpu_hw_intf *intf, bool enable_external_te); - void (*vsync_sel)(struct dpu_hw_intf *intf, enum dpu_vsync_source vsync_source); + void (*vsync_sel)(struct dpu_hw_intf *intf, struct dpu_vsync_source_cfg *cfg); void (*disable_autorefresh)(struct dpu_hw_intf *intf, uint32_t encoder_id, u16 vdisplay); diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c index 6f1fc790ad6d81..f275b14da4ffd0 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c @@ -72,6 +72,8 @@ #define SSPP_EXCL_REC_XY_REC1 0x188 #define SSPP_EXCL_REC_SIZE 0x1B4 #define SSPP_EXCL_REC_XY 0x1B8 +#define SSPP_UBWC_STATIC_CTRL_REC1 0x1c0 +#define SSPP_UBWC_ERROR_STATUS_REC1 0x1c8 #define SSPP_CLK_CTRL 0x330 /* SSPP_SRC_OP_MODE & OP_MODE_REC1 */ @@ -215,7 +217,7 @@ static void dpu_hw_sspp_setup_format(struct dpu_sw_pipe *pipe, u32 chroma_samp, unpack, src_format; u32 opmode = 0; u32 fast_clear = 0; - u32 op_mode_off, unpack_pat_off, format_off; + u32 op_mode_off, unpack_pat_off, format_off, ubwc_ctrl_off, ubwc_error_off; if (!ctx || !fmt) return; @@ -225,10 +227,21 @@ static void dpu_hw_sspp_setup_format(struct dpu_sw_pipe *pipe, op_mode_off = SSPP_SRC_OP_MODE; unpack_pat_off = SSPP_SRC_UNPACK_PATTERN; format_off = SSPP_SRC_FORMAT; + ubwc_ctrl_off = SSPP_UBWC_STATIC_CTRL; + ubwc_error_off = SSPP_UBWC_ERROR_STATUS; } else { op_mode_off = SSPP_SRC_OP_MODE_REC1; unpack_pat_off = SSPP_SRC_UNPACK_PATTERN_REC1; format_off = SSPP_SRC_FORMAT_REC1; + + /* reg wasn't present before DPU 8.0 */ + if (ctx->mdss_ver->core_major_ver >= 8) { + ubwc_ctrl_off = SSPP_UBWC_STATIC_CTRL_REC1; + ubwc_error_off = SSPP_UBWC_ERROR_STATUS_REC1; + } else { + ubwc_ctrl_off = SSPP_UBWC_STATIC_CTRL; + ubwc_error_off = SSPP_UBWC_ERROR_STATUS; + } } c = &ctx->hw; @@ -270,36 +283,41 @@ static void dpu_hw_sspp_setup_format(struct dpu_sw_pipe *pipe, ((fmt->bpp - 1) << 9); if (fmt->fetch_mode != MDP_FETCH_LINEAR) { + u32 hbb = ctx->ubwc->highest_bank_bit - 13; + u32 ctrl_val; + if (MSM_FORMAT_IS_UBWC(fmt)) opmode |= MDSS_MDP_OP_BWC_EN; src_format |= (fmt->fetch_mode & 3) << 30; /*FRAME_FORMAT */ DPU_REG_WRITE(c, SSPP_FETCH_CONFIG, DPU_FETCH_CONFIG_RESET_VALUE | - ctx->ubwc->highest_bank_bit << 18); - switch (ctx->ubwc->ubwc_enc_version) { - case UBWC_1_0: + hbb << 18); + + if (ctx->ubwc->ubwc_enc_version == UBWC_1_0) { fast_clear = fmt->alpha_enable ? BIT(31) : 0; - DPU_REG_WRITE(c, SSPP_UBWC_STATIC_CTRL, - fast_clear | (ctx->ubwc->ubwc_swizzle & 0x1) | - BIT(8) | - (ctx->ubwc->highest_bank_bit << 4)); - break; - case UBWC_2_0: + ctrl_val = fast_clear | (ctx->ubwc->ubwc_swizzle & 0x1) | + BIT(8) | (hbb << 4); + } else if (ctx->ubwc->ubwc_enc_version == UBWC_2_0) { fast_clear = fmt->alpha_enable ? BIT(31) : 0; - DPU_REG_WRITE(c, SSPP_UBWC_STATIC_CTRL, - fast_clear | (ctx->ubwc->ubwc_swizzle) | - (ctx->ubwc->highest_bank_bit << 4)); - break; - case UBWC_3_0: - DPU_REG_WRITE(c, SSPP_UBWC_STATIC_CTRL, - BIT(30) | (ctx->ubwc->ubwc_swizzle) | - (ctx->ubwc->highest_bank_bit << 4)); - break; - case UBWC_4_0: - DPU_REG_WRITE(c, SSPP_UBWC_STATIC_CTRL, - MSM_FORMAT_IS_YUV(fmt) ? 0 : BIT(30)); - break; + ctrl_val = fast_clear | ctx->ubwc->ubwc_swizzle | (hbb << 4); + } else if (ctx->ubwc->ubwc_enc_version == UBWC_3_0) { + ctrl_val = BIT(30) | (ctx->ubwc->ubwc_swizzle) | (hbb << 4); + } else if (ctx->ubwc->ubwc_enc_version == UBWC_4_0) { + ctrl_val = MSM_FORMAT_IS_YUV(fmt) ? 0 : BIT(30); + } else if (ctx->ubwc->ubwc_enc_version <= UBWC_6_0) { + if (MSM_FORMAT_IS_YUV(fmt)) + ctrl_val = 0; + else if (MSM_FORMAT_IS_DX(fmt)) /* or FP16, but it's unsupported */ + ctrl_val = BIT(30); + else + ctrl_val = BIT(30) | BIT(31); + /* SDE also sets bits for lossy formats, but we don't support them yet */ + } else { + DRM_WARN_ONCE("Unsupported UBWC version %x\n", ctx->ubwc->ubwc_enc_version); + ctrl_val = 0; } + + DPU_REG_WRITE(c, ubwc_ctrl_off, ctrl_val); } opmode |= MDSS_MDP_OP_PE_OVERRIDE; @@ -325,7 +343,7 @@ static void dpu_hw_sspp_setup_format(struct dpu_sw_pipe *pipe, DPU_REG_WRITE(c, op_mode_off, opmode); /* clear previous UBWC error */ - DPU_REG_WRITE(c, SSPP_UBWC_ERROR_STATUS, BIT(31)); + DPU_REG_WRITE(c, ubwc_error_off, BIT(31)); } static void dpu_hw_sspp_setup_pe_config(struct dpu_hw_sspp *ctx, diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_top.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_top.c index 96dc10589bee6c..1ebd75d4f9be8d 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_top.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_top.c @@ -22,13 +22,6 @@ #define TRAFFIC_SHAPER_WR_CLIENT(num) (0x060 + (num * 4)) #define TRAFFIC_SHAPER_FIXPOINT_FACTOR 4 -#define MDP_TICK_COUNT 16 -#define XO_CLK_RATE 19200 -#define MS_TICKS_IN_SEC 1000 - -#define CALCULATE_WD_LOAD_VALUE(fps) \ - ((uint32_t)((MS_TICKS_IN_SEC * XO_CLK_RATE)/(MDP_TICK_COUNT * fps))) - static void dpu_hw_setup_split_pipe(struct dpu_hw_mdp *mdp, struct split_pipe_cfg *cfg) { diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h index 67b08e99335dcf..6fe65bc3bff4e8 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h @@ -21,6 +21,13 @@ #define TO_S15D16(_x_)((_x_) << 7) +#define MDP_TICK_COUNT 16 +#define XO_CLK_RATE 19200 +#define MS_TICKS_IN_SEC 1000 + +#define CALCULATE_WD_LOAD_VALUE(fps) \ + ((uint32_t)((MS_TICKS_IN_SEC * XO_CLK_RATE)/(MDP_TICK_COUNT * fps))) + extern const struct dpu_csc_cfg dpu_csc_YUV2RGB_601L; extern const struct dpu_csc_cfg dpu_csc10_YUV2RGB_601L; extern const struct dpu_csc_cfg dpu_csc10_rgb2yuv_601l; diff --git a/drivers/gpu/drm/msm/disp/mdp_format.c b/drivers/gpu/drm/msm/disp/mdp_format.c index 426782d50cb49d..eebedb1a2636e7 100644 --- a/drivers/gpu/drm/msm/disp/mdp_format.c +++ b/drivers/gpu/drm/msm/disp/mdp_format.c @@ -479,25 +479,25 @@ static const struct msm_format mdp_formats[] = { 0, BPC8, BPC8, BPC8, C2_R_Cr, C0_G_Y, C1_B_Cb, C0_G_Y, false, CHROMA_H2V1, 4, 2, MSM_FORMAT_FLAG_YUV, - MDP_FETCH_LINEAR, 2), + MDP_FETCH_LINEAR, 1), INTERLEAVED_YUV_FMT(UYVY, 0, BPC8, BPC8, BPC8, C1_B_Cb, C0_G_Y, C2_R_Cr, C0_G_Y, false, CHROMA_H2V1, 4, 2, MSM_FORMAT_FLAG_YUV, - MDP_FETCH_LINEAR, 2), + MDP_FETCH_LINEAR, 1), INTERLEAVED_YUV_FMT(YUYV, 0, BPC8, BPC8, BPC8, C0_G_Y, C1_B_Cb, C0_G_Y, C2_R_Cr, false, CHROMA_H2V1, 4, 2, MSM_FORMAT_FLAG_YUV, - MDP_FETCH_LINEAR, 2), + MDP_FETCH_LINEAR, 1), INTERLEAVED_YUV_FMT(YVYU, 0, BPC8, BPC8, BPC8, C0_G_Y, C2_R_Cr, C0_G_Y, C1_B_Cb, false, CHROMA_H2V1, 4, 2, MSM_FORMAT_FLAG_YUV, - MDP_FETCH_LINEAR, 2), + MDP_FETCH_LINEAR, 1), /* 3 plane YUV */ PLANAR_YUV_FMT(YUV420, diff --git a/drivers/gpu/drm/msm/dp/dp_ctrl.c b/drivers/gpu/drm/msm/dp/dp_ctrl.c index cbcc7c2f0ffc46..94411870a5e0a5 100644 --- a/drivers/gpu/drm/msm/dp/dp_ctrl.c +++ b/drivers/gpu/drm/msm/dp/dp_ctrl.c @@ -2395,20 +2395,32 @@ static void msm_dp_ctrl_config_msa(struct msm_dp_ctrl_private *ctrl, bool is_ycbcr_420) { u32 pixel_m, pixel_n; - u32 mvid, nvid, pixel_div = 0, dispcc_input_rate; + u32 mvid, nvid, pixel_div, dispcc_input_rate; u32 const nvid_fixed = DP_LINK_CONSTANT_N_VALUE; u32 const link_rate_hbr2 = 540000; u32 const link_rate_hbr3 = 810000; unsigned long den, num; - if (rate == link_rate_hbr3) + switch (rate) { + case link_rate_hbr3: pixel_div = 6; - else if (rate == 162000 || rate == 270000) - pixel_div = 2; - else if (rate == link_rate_hbr2) + break; + case link_rate_hbr2: pixel_div = 4; - else + break; + case 162000: + case 270000: + pixel_div = 2; + break; + default: + /* + * This cannot be reached but the compiler is not able to know + * that statically so return early to avoid a possibly invalid + * division. + */ DRM_ERROR("Invalid pixel mux divider\n"); + return; + } dispcc_input_rate = (rate * 10) / pixel_div; diff --git a/drivers/gpu/drm/msm/dp/dp_display.c b/drivers/gpu/drm/msm/dp/dp_display.c index 9bd9cd5c1e03cc..a082f4d3ebe27d 100644 --- a/drivers/gpu/drm/msm/dp/dp_display.c +++ b/drivers/gpu/drm/msm/dp/dp_display.c @@ -141,8 +141,8 @@ static const struct msm_dp_desc msm_dp_desc_glymur[] = { static const struct msm_dp_desc msm_dp_desc_sa8775p[] = { { .io_start = 0x0af54000, .id = MSM_DP_CONTROLLER_0, .wide_bus_supported = true }, { .io_start = 0x0af5c000, .id = MSM_DP_CONTROLLER_1, .wide_bus_supported = true }, - { .io_start = 0x22154000, .id = MSM_DP_CONTROLLER_2, .wide_bus_supported = true }, - { .io_start = 0x2215c000, .id = MSM_DP_CONTROLLER_3, .wide_bus_supported = true }, + { .io_start = 0x22154000, .id = MSM_DP_CONTROLLER_0, .wide_bus_supported = true }, + { .io_start = 0x2215c000, .id = MSM_DP_CONTROLLER_1, .wide_bus_supported = true }, {} }; diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c index fdefcbd9c2848a..a156c7e7cea832 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c @@ -628,12 +628,7 @@ static int dsi_pll_14nm_postdiv_determine_rate(struct clk_hw *hw, DBG("DSI%d PLL parent rate=%lu", pll_14nm->phy->id, req->rate); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - NULL, - postdiv->width, - postdiv->flags); - - return 0; + return divider_determine_rate(hw, req, NULL, postdiv->width, postdiv->flags); } static int dsi_pll_14nm_postdiv_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/gpu/drm/msm/msm_mdss.c b/drivers/gpu/drm/msm/msm_mdss.c index bf9a33e925ac83..910242f5a71f24 100644 --- a/drivers/gpu/drm/msm/msm_mdss.c +++ b/drivers/gpu/drm/msm/msm_mdss.c @@ -229,7 +229,7 @@ static void msm_mdss_setup_ubwc_dec_50(struct msm_mdss *msm_mdss) { const struct qcom_ubwc_cfg_data *data = msm_mdss->mdss_data; u32 value = MDSS_UBWC_STATIC_UBWC_SWIZZLE(data->ubwc_swizzle) | - MDSS_UBWC_STATIC_HIGHEST_BANK_BIT(data->highest_bank_bit); + MDSS_UBWC_STATIC_HIGHEST_BANK_BIT(data->highest_bank_bit - 13); if (data->ubwc_bank_spread) value |= MDSS_UBWC_STATIC_UBWC_BANK_SPREAD; diff --git a/drivers/gpu/drm/nova/driver.rs b/drivers/gpu/drm/nova/driver.rs index 2246d8e104e083..b01954773255b1 100644 --- a/drivers/gpu/drm/nova/driver.rs +++ b/drivers/gpu/drm/nova/driver.rs @@ -63,6 +63,8 @@ impl drm::Driver for NovaDriver { const INFO: drm::DriverInfo = INFO; + const FEATURES: u32 = drm::driver::FEAT_GEM; + kernel::declare_drm_ioctls! { (NOVA_GETPARAM, drm_nova_getparam, ioctl::RENDER_ALLOW, File::get_param), (NOVA_GEM_CREATE, drm_nova_gem_create, ioctl::AUTH | ioctl::RENDER_ALLOW, File::gem_create), diff --git a/drivers/gpu/drm/nova/file.rs b/drivers/gpu/drm/nova/file.rs index a3b7bd36792c1b..a8288bc014c250 100644 --- a/drivers/gpu/drm/nova/file.rs +++ b/drivers/gpu/drm/nova/file.rs @@ -4,6 +4,7 @@ use crate::driver::{NovaDevice, NovaDriver}; use crate::gem::NovaObject; use kernel::{ alloc::flags::*, + bindings, drm::{self, gem::BaseObject}, pci, prelude::*, @@ -18,6 +19,10 @@ impl drm::file::DriverFile for File { fn open(_dev: &NovaDevice) -> Result>> { Ok(KBox::new(Self, GFP_KERNEL)?.into()) } + + fn as_raw(&self) -> *mut bindings::drm_file { + todo!() + } } impl File { diff --git a/drivers/gpu/drm/nova/gem.rs b/drivers/gpu/drm/nova/gem.rs index 2760ba4f3450be..680f91919e8421 100644 --- a/drivers/gpu/drm/nova/gem.rs +++ b/drivers/gpu/drm/nova/gem.rs @@ -16,10 +16,12 @@ use crate::{ #[pin_data] pub(crate) struct NovaObject {} +#[vtable] impl gem::DriverObject for NovaObject { type Driver = NovaDriver; + type Args = (); - fn new(_dev: &NovaDevice, _size: usize) -> impl PinInit { + fn new(_dev: &NovaDevice, _size: usize, _args: Self::Args) -> impl PinInit { try_pin_init!(NovaObject {}) } } @@ -33,7 +35,7 @@ impl NovaObject { return Err(EINVAL); } - gem::Object::new(dev, aligned_size) + gem::Object::new(dev, aligned_size, ()) } /// Look up a GEM object handle for a `File` and return an `ObjectRef` for it. diff --git a/drivers/gpu/drm/panel/panel-lg-sw43408.c b/drivers/gpu/drm/panel/panel-lg-sw43408.c index 46a56ea92ad9ff..6e307fba658f78 100644 --- a/drivers/gpu/drm/panel/panel-lg-sw43408.c +++ b/drivers/gpu/drm/panel/panel-lg-sw43408.c @@ -294,10 +294,6 @@ static void sw43408_remove(struct mipi_dsi_device *dsi) struct sw43408_panel *ctx = mipi_dsi_get_drvdata(dsi); int ret; - ret = sw43408_unprepare(&ctx->base); - if (ret < 0) - dev_err(&dsi->dev, "failed to unprepare panel: %d\n", ret); - ret = mipi_dsi_detach(dsi); if (ret < 0) dev_err(&dsi->dev, "failed to detach from DSI host: %d\n", ret); diff --git a/drivers/gpu/drm/panthor/panthor_fw.c b/drivers/gpu/drm/panthor/panthor_fw.c index 1a5e3c1a27fbc0..9533b1a31820e6 100644 --- a/drivers/gpu/drm/panthor/panthor_fw.c +++ b/drivers/gpu/drm/panthor/panthor_fw.c @@ -1187,7 +1187,6 @@ void panthor_fw_pre_reset(struct panthor_device *ptdev, bool on_hang) else ptdev->reset.fast = true; } - panthor_fw_stop(ptdev); panthor_job_irq_suspend(&ptdev->fw->irq); panthor_fw_stop(ptdev); @@ -1261,10 +1260,6 @@ void panthor_fw_unplug(struct panthor_device *ptdev) if (ptdev->fw->irq.irq) panthor_job_irq_suspend(&ptdev->fw->irq); - panthor_fw_halt_mcu(ptdev); - if (!panthor_fw_wait_mcu_halted(ptdev)) - drm_warn(&ptdev->base, "Failed to halt MCU on unplug"); - panthor_fw_stop(ptdev); } diff --git a/drivers/gpu/drm/panthor/panthor_gpu.c b/drivers/gpu/drm/panthor/panthor_gpu.c index 06b231b2460ab5..ff5231269518ef 100644 --- a/drivers/gpu/drm/panthor/panthor_gpu.c +++ b/drivers/gpu/drm/panthor/panthor_gpu.c @@ -51,7 +51,7 @@ struct panthor_gpu { static void panthor_gpu_coherency_set(struct panthor_device *ptdev) { gpu_write(ptdev, GPU_COHERENCY_PROTOCOL, - ptdev->coherent ? GPU_COHERENCY_PROT_BIT(ACE_LITE) : GPU_COHERENCY_NONE); + ptdev->coherent ? GPU_COHERENCY_ACE_LITE : GPU_COHERENCY_NONE); } static void panthor_gpu_l2_config_set(struct panthor_device *ptdev) @@ -289,38 +289,42 @@ int panthor_gpu_l2_power_on(struct panthor_device *ptdev) int panthor_gpu_flush_caches(struct panthor_device *ptdev, u32 l2, u32 lsc, u32 other) { - bool timedout = false; unsigned long flags; + int ret = 0; /* Serialize cache flush operations. */ guard(mutex)(&ptdev->gpu->cache_flush_lock); spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); - if (!drm_WARN_ON(&ptdev->base, - ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED)) { + if (!(ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED)) { ptdev->gpu->pending_reqs |= GPU_IRQ_CLEAN_CACHES_COMPLETED; gpu_write(ptdev, GPU_CMD, GPU_FLUSH_CACHES(l2, lsc, other)); + } else { + ret = -EIO; } spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); + if (ret) + return ret; + if (!wait_event_timeout(ptdev->gpu->reqs_acked, !(ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED), msecs_to_jiffies(100))) { spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); if ((ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED) != 0 && !(gpu_read(ptdev, GPU_INT_RAWSTAT) & GPU_IRQ_CLEAN_CACHES_COMPLETED)) - timedout = true; + ret = -ETIMEDOUT; else ptdev->gpu->pending_reqs &= ~GPU_IRQ_CLEAN_CACHES_COMPLETED; spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); } - if (timedout) { + if (ret) { + panthor_device_schedule_reset(ptdev); drm_err(&ptdev->base, "Flush caches timeout"); - return -ETIMEDOUT; } - return 0; + return ret; } /** @@ -360,6 +364,7 @@ int panthor_gpu_soft_reset(struct panthor_device *ptdev) return -ETIMEDOUT; } + ptdev->gpu->pending_reqs = 0; return 0; } diff --git a/drivers/gpu/drm/panthor/panthor_mmu.c b/drivers/gpu/drm/panthor/panthor_mmu.c index f6339963e4960d..9194bad4b6196c 100644 --- a/drivers/gpu/drm/panthor/panthor_mmu.c +++ b/drivers/gpu/drm/panthor/panthor_mmu.c @@ -1503,6 +1503,10 @@ static void panthor_vm_destroy(struct panthor_vm *vm) vm->destroyed = true; + /* Tell scheduler to stop all GPU work related to this VM */ + if (refcount_read(&vm->as.active_cnt) > 0) + panthor_sched_prepare_for_vm_destruction(vm->ptdev); + mutex_lock(&vm->heaps.lock); panthor_heap_pool_destroy(vm->heaps.pool); vm->heaps.pool = NULL; diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c index a6b8024e1a3cd5..bd397d773d72b5 100644 --- a/drivers/gpu/drm/panthor/panthor_sched.c +++ b/drivers/gpu/drm/panthor/panthor_sched.c @@ -1072,18 +1072,6 @@ group_is_idle(struct panthor_group *group) return hweight32(inactive_queues) == group->queue_count; } -static void -queue_reset_timeout_locked(struct panthor_queue *queue) -{ - lockdep_assert_held(&queue->fence_ctx.lock); - - if (queue->timeout.remaining != MAX_SCHEDULE_TIMEOUT) { - mod_delayed_work(queue->scheduler.timeout_wq, - &queue->timeout.work, - msecs_to_jiffies(JOB_TIMEOUT_MS)); - } -} - static bool group_can_run(struct panthor_group *group) { @@ -1100,6 +1088,18 @@ queue_timeout_is_suspended(struct panthor_queue *queue) return queue->timeout.remaining != MAX_SCHEDULE_TIMEOUT; } +static void +queue_reset_timeout_locked(struct panthor_queue *queue) +{ + lockdep_assert_held(&queue->fence_ctx.lock); + + if (!queue_timeout_is_suspended(queue)) { + mod_delayed_work(queue->scheduler.timeout_wq, + &queue->timeout.work, + msecs_to_jiffies(JOB_TIMEOUT_MS)); + } +} + static void queue_suspend_timeout_locked(struct panthor_queue *queue) { @@ -2020,10 +2020,10 @@ struct panthor_sched_tick_ctx { struct list_head groups[PANTHOR_CSG_PRIORITY_COUNT]; u32 idle_group_count; u32 group_count; - enum panthor_csg_priority min_priority; struct panthor_vm *vms[MAX_CS_PER_CSG]; u32 as_count; bool immediate_tick; + bool stop_tick; u32 csg_upd_failed_mask; }; @@ -2066,17 +2066,21 @@ tick_ctx_pick_groups_from_list(const struct panthor_scheduler *sched, if (!owned_by_tick_ctx) group_get(group); - list_move_tail(&group->run_node, &ctx->groups[group->priority]); ctx->group_count++; + + /* If we have more than one active group with the same priority, + * we need to keep ticking to rotate the CSG priority. + */ if (group_is_idle(group)) ctx->idle_group_count++; + else if (!list_empty(&ctx->groups[group->priority])) + ctx->stop_tick = false; + + list_move_tail(&group->run_node, &ctx->groups[group->priority]); if (i == ctx->as_count) ctx->vms[ctx->as_count++] = group->vm; - if (ctx->min_priority > group->priority) - ctx->min_priority = group->priority; - if (tick_ctx_is_full(sched, ctx)) return; } @@ -2085,31 +2089,22 @@ tick_ctx_pick_groups_from_list(const struct panthor_scheduler *sched, static void tick_ctx_insert_old_group(struct panthor_scheduler *sched, struct panthor_sched_tick_ctx *ctx, - struct panthor_group *group, - bool full_tick) + struct panthor_group *group) { struct panthor_csg_slot *csg_slot = &sched->csg_slots[group->csg_id]; struct panthor_group *other_group; - if (!full_tick) { - list_add_tail(&group->run_node, &ctx->old_groups[group->priority]); - return; - } - - /* Rotate to make sure groups with lower CSG slot - * priorities have a chance to get a higher CSG slot - * priority next time they get picked. This priority - * has an impact on resource request ordering, so it's - * important to make sure we don't let one group starve - * all other groups with the same group priority. - */ + /* Class groups in descending priority order so we can easily rotate. */ list_for_each_entry(other_group, &ctx->old_groups[csg_slot->group->priority], run_node) { struct panthor_csg_slot *other_csg_slot = &sched->csg_slots[other_group->csg_id]; - if (other_csg_slot->priority > csg_slot->priority) { - list_add_tail(&csg_slot->group->run_node, &other_group->run_node); + /* Our group has a higher prio than the one we're testing against, + * place it just before. + */ + if (csg_slot->priority > other_csg_slot->priority) { + list_add_tail(&group->run_node, &other_group->run_node); return; } } @@ -2119,8 +2114,7 @@ tick_ctx_insert_old_group(struct panthor_scheduler *sched, static void tick_ctx_init(struct panthor_scheduler *sched, - struct panthor_sched_tick_ctx *ctx, - bool full_tick) + struct panthor_sched_tick_ctx *ctx) { struct panthor_device *ptdev = sched->ptdev; struct panthor_csg_slots_upd_ctx upd_ctx; @@ -2130,7 +2124,7 @@ tick_ctx_init(struct panthor_scheduler *sched, memset(ctx, 0, sizeof(*ctx)); csgs_upd_ctx_init(&upd_ctx); - ctx->min_priority = PANTHOR_CSG_PRIORITY_COUNT; + ctx->stop_tick = true; for (i = 0; i < ARRAY_SIZE(ctx->groups); i++) { INIT_LIST_HEAD(&ctx->groups[i]); INIT_LIST_HEAD(&ctx->old_groups[i]); @@ -2158,7 +2152,7 @@ tick_ctx_init(struct panthor_scheduler *sched, group->fatal_queues |= GENMASK(group->queue_count - 1, 0); } - tick_ctx_insert_old_group(sched, ctx, group, full_tick); + tick_ctx_insert_old_group(sched, ctx, group); csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, i, csg_iface->output->ack ^ CSG_STATUS_UPDATE, CSG_STATUS_UPDATE); @@ -2442,32 +2436,18 @@ static u64 tick_ctx_update_resched_target(struct panthor_scheduler *sched, const struct panthor_sched_tick_ctx *ctx) { - /* We had space left, no need to reschedule until some external event happens. */ - if (!tick_ctx_is_full(sched, ctx)) - goto no_tick; - - /* If idle groups were scheduled, no need to wake up until some external - * event happens (group unblocked, new job submitted, ...). - */ - if (ctx->idle_group_count) - goto no_tick; + u64 resched_target; - if (drm_WARN_ON(&sched->ptdev->base, ctx->min_priority >= PANTHOR_CSG_PRIORITY_COUNT)) + if (ctx->stop_tick) goto no_tick; - /* If there are groups of the same priority waiting, we need to - * keep the scheduler ticking, otherwise, we'll just wait for - * new groups with higher priority to be queued. - */ - if (!list_empty(&sched->groups.runnable[ctx->min_priority])) { - u64 resched_target = sched->last_tick + sched->tick_period; + resched_target = sched->last_tick + sched->tick_period; - if (time_before64(sched->resched_target, sched->last_tick) || - time_before64(resched_target, sched->resched_target)) - sched->resched_target = resched_target; + if (time_before64(sched->resched_target, sched->last_tick) || + time_before64(resched_target, sched->resched_target)) + sched->resched_target = resched_target; - return sched->resched_target - sched->last_tick; - } + return sched->resched_target - sched->last_tick; no_tick: sched->resched_target = U64_MAX; @@ -2480,9 +2460,11 @@ static void tick_work(struct work_struct *work) tick_work.work); struct panthor_device *ptdev = sched->ptdev; struct panthor_sched_tick_ctx ctx; + u64 resched_target = sched->resched_target; u64 remaining_jiffies = 0, resched_delay; u64 now = get_jiffies_64(); int prio, ret, cookie; + bool full_tick; if (!drm_dev_enter(&ptdev->base, &cookie)) return; @@ -2491,18 +2473,24 @@ static void tick_work(struct work_struct *work) if (drm_WARN_ON(&ptdev->base, ret)) goto out_dev_exit; - if (time_before64(now, sched->resched_target)) - remaining_jiffies = sched->resched_target - now; + /* If the tick is stopped, calculate when the next tick would be */ + if (resched_target == U64_MAX) + resched_target = sched->last_tick + sched->tick_period; + + if (time_before64(now, resched_target)) + remaining_jiffies = resched_target - now; + + full_tick = remaining_jiffies == 0; mutex_lock(&sched->lock); if (panthor_device_reset_is_pending(sched->ptdev)) goto out_unlock; - tick_ctx_init(sched, &ctx, remaining_jiffies != 0); + tick_ctx_init(sched, &ctx); if (ctx.csg_upd_failed_mask) goto out_cleanup_ctx; - if (remaining_jiffies) { + if (!full_tick) { /* Scheduling forced in the middle of a tick. Only RT groups * can preempt non-RT ones. Currently running RT groups can't be * preempted. @@ -2524,9 +2512,29 @@ static void tick_work(struct work_struct *work) for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0 && !tick_ctx_is_full(sched, &ctx); prio--) { + struct panthor_group *old_highest_prio_group = + list_first_entry_or_null(&ctx.old_groups[prio], + struct panthor_group, run_node); + + /* Pull out the group with the highest prio for rotation. */ + if (old_highest_prio_group) + list_del(&old_highest_prio_group->run_node); + + /* Re-insert old active groups so they get a chance to run with higher prio. */ + tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], true, true); + + /* Fill the remaining slots with runnable groups. */ tick_ctx_pick_groups_from_list(sched, &ctx, &sched->groups.runnable[prio], true, false); - tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], true, true); + + /* Re-insert the old group with the highest prio, and give it a chance to be + * scheduled again (but with a lower prio) if there's room left. + */ + if (old_highest_prio_group) { + list_add_tail(&old_highest_prio_group->run_node, &ctx.old_groups[prio]); + tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], + true, true); + } } /* If we have free CSG slots left, pick idle groups */ @@ -2651,14 +2659,33 @@ static void sync_upd_work(struct work_struct *work) sched_queue_delayed_work(sched, tick, 0); } +static void sched_resume_tick(struct panthor_device *ptdev) +{ + struct panthor_scheduler *sched = ptdev->scheduler; + u64 delay_jiffies, now; + + drm_WARN_ON(&ptdev->base, sched->resched_target != U64_MAX); + + /* Scheduler tick was off, recalculate the resched_target based on the + * last tick event, and queue the scheduler work. + */ + now = get_jiffies_64(); + sched->resched_target = sched->last_tick + sched->tick_period; + if (sched->used_csg_slot_count == sched->csg_slot_count && + time_before64(now, sched->resched_target)) + delay_jiffies = min_t(unsigned long, sched->resched_target - now, ULONG_MAX); + else + delay_jiffies = 0; + + sched_queue_delayed_work(sched, tick, delay_jiffies); +} + static void group_schedule_locked(struct panthor_group *group, u32 queue_mask) { struct panthor_device *ptdev = group->ptdev; struct panthor_scheduler *sched = ptdev->scheduler; struct list_head *queue = &sched->groups.runnable[group->priority]; - u64 delay_jiffies = 0; bool was_idle; - u64 now; if (!group_can_run(group)) return; @@ -2703,13 +2730,7 @@ static void group_schedule_locked(struct panthor_group *group, u32 queue_mask) /* Scheduler tick was off, recalculate the resched_target based on the * last tick event, and queue the scheduler work. */ - now = get_jiffies_64(); - sched->resched_target = sched->last_tick + sched->tick_period; - if (sched->used_csg_slot_count == sched->csg_slot_count && - time_before64(now, sched->resched_target)) - delay_jiffies = min_t(unsigned long, sched->resched_target - now, ULONG_MAX); - - sched_queue_delayed_work(sched, tick, delay_jiffies); + sched_resume_tick(ptdev); } static void queue_stop(struct panthor_queue *queue, @@ -2784,6 +2805,20 @@ void panthor_sched_report_mmu_fault(struct panthor_device *ptdev) panthor_sched_immediate_tick(ptdev); } +void panthor_sched_prepare_for_vm_destruction(struct panthor_device *ptdev) +{ + /* FW can write out internal state, like the heap context, during CSG + * suspend. It is therefore important that the scheduler has fully + * evicted any pending and related groups before VM destruction can + * safely continue. Failure to do so can lead to GPU page faults. + * A controlled termination of a Panthor instance involves destroying + * the group(s) before the VM. This means any relevant group eviction + * has already been initiated by this point, and we just need to + * ensure that any pending tick_work() has been completed. + */ + flush_work(&ptdev->scheduler->tick_work.work); +} + void panthor_sched_resume(struct panthor_device *ptdev) { /* Force a tick to re-evaluate after a resume. */ @@ -3343,6 +3378,18 @@ queue_run_job(struct drm_sched_job *sched_job) if (group->csg_id < 0) { group_schedule_locked(group, BIT(job->queue_idx)); } else { + u32 queue_mask = BIT(job->queue_idx); + bool resume_tick = group_is_idle(group) && + (group->idle_queues & queue_mask) && + !(group->blocked_queues & queue_mask) && + sched->resched_target == U64_MAX; + + /* We just added something to the queue, so it's no longer idle. */ + group->idle_queues &= ~queue_mask; + + if (resume_tick) + sched_resume_tick(ptdev); + gpu_write(ptdev, CSF_DOORBELL(queue->doorbell_id), 1); if (!sched->pm.has_ref && !(group->blocked_queues & BIT(job->queue_idx))) { diff --git a/drivers/gpu/drm/panthor/panthor_sched.h b/drivers/gpu/drm/panthor/panthor_sched.h index f4a475aa34c0aa..9a8692de8adedb 100644 --- a/drivers/gpu/drm/panthor/panthor_sched.h +++ b/drivers/gpu/drm/panthor/panthor_sched.h @@ -50,6 +50,7 @@ void panthor_sched_suspend(struct panthor_device *ptdev); void panthor_sched_resume(struct panthor_device *ptdev); void panthor_sched_report_mmu_fault(struct panthor_device *ptdev); +void panthor_sched_prepare_for_vm_destruction(struct panthor_device *ptdev); void panthor_sched_report_fw_events(struct panthor_device *ptdev, u32 events); void panthor_fdinfo_gather_group_samples(struct panthor_file *pfile); diff --git a/drivers/gpu/drm/rockchip/dw_hdmi_qp-rockchip.c b/drivers/gpu/drm/rockchip/dw_hdmi_qp-rockchip.c index 8604342f994327..c7158b1b8c59e8 100644 --- a/drivers/gpu/drm/rockchip/dw_hdmi_qp-rockchip.c +++ b/drivers/gpu/drm/rockchip/dw_hdmi_qp-rockchip.c @@ -280,12 +280,7 @@ static irqreturn_t dw_hdmi_qp_rk3576_hardirq(int irq, void *dev_id) static irqreturn_t dw_hdmi_qp_rk3576_irq(int irq, void *dev_id) { struct rockchip_hdmi_qp *hdmi = dev_id; - u32 intr_stat, val; - - regmap_read(hdmi->regmap, RK3576_IOC_HDMI_HPD_STATUS, &intr_stat); - - if (!intr_stat) - return IRQ_NONE; + u32 val; val = FIELD_PREP_WM16(RK3576_HDMI_HPD_INT_CLR, 1); regmap_write(hdmi->regmap, RK3576_IOC_MISC_CON0, val); diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c index fe174a4857be7f..e1920f3f920aee 100644 --- a/drivers/gpu/drm/scheduler/sched_entity.c +++ b/drivers/gpu/drm/scheduler/sched_entity.c @@ -420,7 +420,12 @@ static bool drm_sched_entity_add_dependency_cb(struct drm_sched_entity *entity, /* * Fence is from the same scheduler, only need to wait for - * it to be scheduled + * it to be scheduled. + * + * Note: s_fence->sched could have been freed and reallocated + * as another scheduler. This false positive case is okay, as if + * the old scheduler was freed all of its jobs must have + * signaled their completion fences. */ fence = dma_fence_get(&s_fence->scheduled); dma_fence_put(entity->dependency); diff --git a/drivers/gpu/drm/scheduler/sched_fence.c b/drivers/gpu/drm/scheduler/sched_fence.c index 9391d6f0dc01d7..d05ab041a581a6 100644 --- a/drivers/gpu/drm/scheduler/sched_fence.c +++ b/drivers/gpu/drm/scheduler/sched_fence.c @@ -92,7 +92,7 @@ static const char *drm_sched_fence_get_driver_name(struct dma_fence *fence) static const char *drm_sched_fence_get_timeline_name(struct dma_fence *f) { struct drm_sched_fence *fence = to_drm_sched_fence(f); - return (const char *)fence->sched->name; + return (const char *)fence->sched_name; } static void drm_sched_fence_free_rcu(struct rcu_head *rcu) @@ -228,6 +228,8 @@ void drm_sched_fence_init(struct drm_sched_fence *fence, unsigned seq; fence->sched = entity->rq->sched; + strscpy(fence->sched_name, entity->rq->sched->name, + sizeof(fence->sched_name)); seq = atomic_inc_return(&entity->fence_seq); dma_fence_init(&fence->scheduled, &drm_sched_fence_ops_scheduled, &fence->lock, entity->fence_context, seq); diff --git a/drivers/gpu/drm/sysfb/simpledrm.c b/drivers/gpu/drm/sysfb/simpledrm.c index 7a95d2dacd9d27..2ab6d96b4ec9e7 100644 --- a/drivers/gpu/drm/sysfb/simpledrm.c +++ b/drivers/gpu/drm/sysfb/simpledrm.c @@ -841,6 +841,12 @@ static int simpledrm_probe(struct platform_device *pdev) struct drm_device *dev; int ret; + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (ret) + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (ret) + return dev_err_probe(&pdev->dev, ret, "Failed to set dma mask\n"); + sdev = simpledrm_device_create(&simpledrm_driver, pdev); if (IS_ERR(sdev)) return PTR_ERR(sdev); diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index 0389c558c03675..c333d4186c8b89 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -186,6 +186,8 @@ impl drm::Driver for TyrDriver { const INFO: drm::DriverInfo = INFO; + const FEATURES: u32 = drm::driver::FEAT_GEM; + kernel::declare_drm_ioctls! { (PANTHOR_DEV_QUERY, drm_panthor_dev_query, ioctl::RENDER_ALLOW, File::dev_query), } diff --git a/drivers/gpu/drm/tyr/file.rs b/drivers/gpu/drm/tyr/file.rs index 0ef432947b73d5..8fc26e03a9034b 100644 --- a/drivers/gpu/drm/tyr/file.rs +++ b/drivers/gpu/drm/tyr/file.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 or MIT +use kernel::bindings; use kernel::drm; use kernel::prelude::*; use kernel::uaccess::UserSlice; @@ -20,6 +21,10 @@ impl drm::file::DriverFile for File { fn open(_dev: &drm::Device) -> Result>> { KBox::try_pin_init(try_pin_init!(Self {}), GFP_KERNEL) } + + fn as_raw(&self) -> *mut bindings::drm_file { + todo!() + } } impl File { diff --git a/drivers/gpu/drm/tyr/gem.rs b/drivers/gpu/drm/tyr/gem.rs index 1273bf89dbd5d7..83493904a13f5e 100644 --- a/drivers/gpu/drm/tyr/gem.rs +++ b/drivers/gpu/drm/tyr/gem.rs @@ -9,10 +9,12 @@ use kernel::prelude::*; #[pin_data] pub(crate) struct TyrObject {} +#[vtable] impl gem::DriverObject for TyrObject { type Driver = TyrDriver; + type Args = (); - fn new(_dev: &TyrDevice, _size: usize) -> impl PinInit { + fn new(_dev: &TyrDevice, _size: usize, _args: ()) -> impl PinInit { try_pin_init!(TyrObject {}) } } diff --git a/drivers/gpu/drm/vkms/vkms_composer.c b/drivers/gpu/drm/vkms/vkms_composer.c index 3cf3f26e0d8eae..cd85de4ffd03d2 100644 --- a/drivers/gpu/drm/vkms/vkms_composer.c +++ b/drivers/gpu/drm/vkms/vkms_composer.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/gpu/drm/vkms/vkms_drv.c b/drivers/gpu/drm/vkms/vkms_drv.c index dd1402f4377369..434c295f44ba65 100644 --- a/drivers/gpu/drm/vkms/vkms_drv.c +++ b/drivers/gpu/drm/vkms/vkms_drv.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index 71acd45aa33b00..579e98dec7492f 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -1942,7 +1942,7 @@ static vm_fault_t xe_bo_cpu_fault(struct vm_fault *vmf) int err = 0; int idx; - if (!drm_dev_enter(&xe->drm, &idx)) + if (xe_device_wedged(xe) || !drm_dev_enter(&xe->drm, &idx)) return ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot); ret = xe_bo_cpu_fault_fastpath(vmf, xe, bo, needs_rpm); diff --git a/drivers/gpu/drm/xe/xe_configfs.h b/drivers/gpu/drm/xe/xe_configfs.h index fed57be0b90e14..f3683bc7eb90cc 100644 --- a/drivers/gpu/drm/xe/xe_configfs.h +++ b/drivers/gpu/drm/xe/xe_configfs.h @@ -21,9 +21,11 @@ bool xe_configfs_primary_gt_allowed(struct pci_dev *pdev); bool xe_configfs_media_gt_allowed(struct pci_dev *pdev); u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev); bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev); -u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev, enum xe_engine_class, +u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev, + enum xe_engine_class class, const u32 **cs); -u32 xe_configfs_get_ctx_restore_post_bb(struct pci_dev *pdev, enum xe_engine_class, +u32 xe_configfs_get_ctx_restore_post_bb(struct pci_dev *pdev, + enum xe_engine_class class, const u32 **cs); #ifdef CONFIG_PCI_IOV unsigned int xe_configfs_get_max_vfs(struct pci_dev *pdev); @@ -37,9 +39,11 @@ static inline bool xe_configfs_primary_gt_allowed(struct pci_dev *pdev) { return static inline bool xe_configfs_media_gt_allowed(struct pci_dev *pdev) { return true; } static inline u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev) { return U64_MAX; } static inline bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev) { return false; } -static inline u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev, enum xe_engine_class, +static inline u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev, + enum xe_engine_class class, const u32 **cs) { return 0; } -static inline u32 xe_configfs_get_ctx_restore_post_bb(struct pci_dev *pdev, enum xe_engine_class, +static inline u32 xe_configfs_get_ctx_restore_post_bb(struct pci_dev *pdev, + enum xe_engine_class class, const u32 **cs) { return 0; } static inline unsigned int xe_configfs_get_max_vfs(struct pci_dev *pdev) { return UINT_MAX; } #endif diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index 9a6d49fcd8e49b..ef5acef32f4167 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -976,6 +976,7 @@ int xe_device_probe(struct xe_device *xe) err_unregister_display: xe_display_unregister(xe); + drm_dev_unregister(&xe->drm); return err; } diff --git a/drivers/gpu/drm/xe/xe_guc_pc.c b/drivers/gpu/drm/xe/xe_guc_pc.c index 951a49fb1d3e49..7f09cf0e495fee 100644 --- a/drivers/gpu/drm/xe/xe_guc_pc.c +++ b/drivers/gpu/drm/xe/xe_guc_pc.c @@ -1214,6 +1214,36 @@ int xe_guc_pc_set_power_profile(struct xe_guc_pc *pc, const char *buf) return ret; } +static int pc_action_set_dcc(struct xe_guc_pc *pc, bool enable) +{ + int ret; + + ret = pc_action_set_param(pc, + SLPC_PARAM_TASK_ENABLE_DCC, + enable); + if (!ret) + return pc_action_set_param(pc, + SLPC_PARAM_TASK_DISABLE_DCC, + !enable); + else + return ret; +} + +static int pc_modify_defaults(struct xe_guc_pc *pc) +{ + struct xe_device *xe = pc_to_xe(pc); + struct xe_gt *gt = pc_to_gt(pc); + int ret = 0; + + if (xe->info.platform == XE_PANTHERLAKE) { + ret = pc_action_set_dcc(pc, false); + if (unlikely(ret)) + xe_gt_err(gt, "Failed to modify DCC default: %pe\n", ERR_PTR(ret)); + } + + return ret; +} + /** * xe_guc_pc_start - Start GuC's Power Conservation component * @pc: Xe_GuC_PC instance @@ -1271,6 +1301,10 @@ int xe_guc_pc_start(struct xe_guc_pc *pc) ktime_ms_delta(ktime_get(), earlier)); } + ret = pc_modify_defaults(pc); + if (ret) + return ret; + ret = pc_init_freqs(pc); if (ret) goto out; diff --git a/drivers/gpu/drm/xe/xe_mmio.c b/drivers/gpu/drm/xe/xe_mmio.c index 350dca1f092596..3d59440ec44dd7 100644 --- a/drivers/gpu/drm/xe/xe_mmio.c +++ b/drivers/gpu/drm/xe/xe_mmio.c @@ -260,11 +260,11 @@ u64 xe_mmio_read64_2x32(struct xe_mmio *mmio, struct xe_reg reg) struct xe_reg reg_udw = { .addr = reg.addr + 0x4 }; u32 ldw, udw, oldudw, retries; - reg.addr = xe_mmio_adjusted_addr(mmio, reg.addr); - reg_udw.addr = xe_mmio_adjusted_addr(mmio, reg_udw.addr); - - /* we shouldn't adjust just one register address */ - xe_tile_assert(mmio->tile, reg_udw.addr == reg.addr + 0x4); + /* + * The two dwords of a 64-bit register can never straddle the offset + * adjustment cutoff. + */ + xe_tile_assert(mmio->tile, !in_range(mmio->adj_limit, reg.addr + 1, 7)); oldudw = xe_mmio_read32(mmio, reg_udw); for (retries = 5; retries; --retries) { diff --git a/drivers/gpu/drm/xe/xe_module.h b/drivers/gpu/drm/xe/xe_module.h index 5a3bfea8b7b4c4..b6684953927016 100644 --- a/drivers/gpu/drm/xe/xe_module.h +++ b/drivers/gpu/drm/xe/xe_module.h @@ -12,7 +12,7 @@ struct xe_modparam { bool force_execlist; bool probe_display; - u32 force_vram_bar_size; + int force_vram_bar_size; int guc_log_level; char *guc_firmware_path; char *huc_firmware_path; diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index 2aa883f5ef797e..38ea9d7dad091e 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -533,6 +533,12 @@ static int read_gmdid(struct xe_device *xe, enum xe_gmdid_type type, u32 *ver, u struct xe_gt *gt __free(kfree) = NULL; int err; + /* Don't try to read media ver if media GT is not allowed */ + if (type == GMDID_MEDIA && !xe_configfs_media_gt_allowed(to_pci_dev(xe->drm.dev))) { + *ver = *revid = 0; + return 0; + } + gt = kzalloc(sizeof(*gt), GFP_KERNEL); if (!gt) return -ENOMEM; diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c b/drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c index c0b767ac735cf2..d1c1f6c2956641 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c +++ b/drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c @@ -349,18 +349,33 @@ static const struct attribute_group *xe_sriov_vf_attr_groups[] = { /* no user serviceable parts below */ -static struct kobject *create_xe_sriov_kobj(struct xe_device *xe, unsigned int vfid) +static void action_put_kobject(void *arg) +{ + struct kobject *kobj = arg; + + kobject_put(kobj); +} + +static struct kobject *create_xe_sriov_kobj(struct xe_device *xe, unsigned int vfid, + const struct kobj_type *ktype) { struct xe_sriov_kobj *vkobj; + int err; xe_sriov_pf_assert_vfid(xe, vfid); vkobj = kzalloc(sizeof(*vkobj), GFP_KERNEL); if (!vkobj) - return NULL; + return ERR_PTR(-ENOMEM); vkobj->xe = xe; vkobj->vfid = vfid; + kobject_init(&vkobj->base, ktype); + + err = devm_add_action_or_reset(xe->drm.dev, action_put_kobject, &vkobj->base); + if (err) + return ERR_PTR(err); + return &vkobj->base; } @@ -471,28 +486,17 @@ static void pf_sysfs_note(struct xe_device *xe, int err, const char *what) xe_sriov_dbg(xe, "Failed to setup sysfs %s (%pe)\n", what, ERR_PTR(err)); } -static void action_put_kobject(void *arg) -{ - struct kobject *kobj = arg; - - kobject_put(kobj); -} - static int pf_setup_root(struct xe_device *xe) { struct kobject *parent = &xe->drm.dev->kobj; struct kobject *root; int err; - root = create_xe_sriov_kobj(xe, PFID); - if (!root) - return pf_sysfs_error(xe, -ENOMEM, "root obj"); - - err = devm_add_action_or_reset(xe->drm.dev, action_put_kobject, root); - if (err) - return pf_sysfs_error(xe, err, "root action"); + root = create_xe_sriov_kobj(xe, PFID, &xe_sriov_dev_ktype); + if (IS_ERR(root)) + return pf_sysfs_error(xe, PTR_ERR(root), "root obj"); - err = kobject_init_and_add(root, &xe_sriov_dev_ktype, parent, "sriov_admin"); + err = kobject_add(root, parent, "sriov_admin"); if (err) return pf_sysfs_error(xe, err, "root init"); @@ -513,20 +517,14 @@ static int pf_setup_tree(struct xe_device *xe) root = xe->sriov.pf.sysfs.root; for (n = 0; n <= totalvfs; n++) { - kobj = create_xe_sriov_kobj(xe, VFID(n)); - if (!kobj) - return pf_sysfs_error(xe, -ENOMEM, "tree obj"); - - err = devm_add_action_or_reset(xe->drm.dev, action_put_kobject, root); - if (err) - return pf_sysfs_error(xe, err, "tree action"); + kobj = create_xe_sriov_kobj(xe, VFID(n), &xe_sriov_vf_ktype); + if (IS_ERR(kobj)) + return pf_sysfs_error(xe, PTR_ERR(kobj), "tree obj"); if (n) - err = kobject_init_and_add(kobj, &xe_sriov_vf_ktype, - root, "vf%u", n); + err = kobject_add(kobj, root, "vf%u", n); else - err = kobject_init_and_add(kobj, &xe_sriov_vf_ktype, - root, "pf"); + err = kobject_add(kobj, root, "pf"); if (err) return pf_sysfs_error(xe, err, "tree init"); diff --git a/drivers/gpu/drm/xe/xe_wa.c b/drivers/gpu/drm/xe/xe_wa.c index e32dd2fde6f1c5..c7eab0c4af7a88 100644 --- a/drivers/gpu/drm/xe/xe_wa.c +++ b/drivers/gpu/drm/xe/xe_wa.c @@ -567,16 +567,6 @@ static const struct xe_rtp_entry_sr engine_was[] = { FUNC(xe_rtp_match_first_render_or_compute)), XE_RTP_ACTIONS(SET(ROW_CHICKEN, EARLY_EOT_DIS)) }, - { XE_RTP_NAME("14019988906"), - XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), - FUNC(xe_rtp_match_first_render_or_compute)), - XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FLSH_IGNORES_PSD)) - }, - { XE_RTP_NAME("14019877138"), - XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), - FUNC(xe_rtp_match_first_render_or_compute)), - XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT)) - }, { XE_RTP_NAME("14020338487"), XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), FUNC(xe_rtp_match_first_render_or_compute)), @@ -873,6 +863,14 @@ static const struct xe_rtp_entry_sr lrc_was[] = { XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)), XE_RTP_ACTIONS(SET(WM_CHICKEN3, HIZ_PLANE_COMPRESSION_DIS)) }, + { XE_RTP_NAME("14019988906"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FLSH_IGNORES_PSD)) + }, + { XE_RTP_NAME("14019877138"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT)) + }, { XE_RTP_NAME("14021490052"), XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)), XE_RTP_ACTIONS(SET(FF_MODE, diff --git a/drivers/gpu/nova-core/falcon.rs b/drivers/gpu/nova-core/falcon.rs index 82c661aef594ff..3ab33ea36d9c8c 100644 --- a/drivers/gpu/nova-core/falcon.rs +++ b/drivers/gpu/nova-core/falcon.rs @@ -8,7 +8,10 @@ use hal::FalconHal; use kernel::{ device, - dma::DmaAddress, + dma::{ + DmaAddress, + DmaMask, // + }, io::poll::read_poll_timeout, prelude::*, sync::aref::ARef, @@ -472,6 +475,12 @@ impl Falcon { return Err(EINVAL); } + // The DMATRFBASE/1 register pair only supports a 49-bit address. + if dma_start > DmaMask::new::<49>().value() { + dev_err!(self.dev, "DMA address {:#x} exceeds 49 bits\n", dma_start); + return Err(ERANGE); + } + // DMA transfers can only be done in units of 256 bytes. Compute how many such transfers we // need to perform. let num_transfers = load_offsets.len.div_ceil(DMA_LEN); diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig index 920a64b66b25b3..5eb94e84bcd9c8 100644 --- a/drivers/hid/Kconfig +++ b/drivers/hid/Kconfig @@ -731,7 +731,8 @@ config HID_MAGICMOUSE Support for the Apple Magic Mouse/Trackpad multi-touch. Say Y here if you want support for the multi-touch features of the - Apple Wireless "Magic" Mouse and the Apple Wireless "Magic" Trackpad. + Apple Wireless "Magic" Mouse, the Apple Wireless "Magic" Trackpad and + force touch Trackpads in Macbooks starting from 2015. config HID_MALTRON tristate "Maltron L90 keyboard" @@ -1448,4 +1449,8 @@ endif # HID source "drivers/hid/usbhid/Kconfig" +source "drivers/hid/spi-hid/Kconfig" + +source "drivers/hid/dockchannel-hid/Kconfig" + endif # HID_SUPPORT diff --git a/drivers/hid/Makefile b/drivers/hid/Makefile index 361a7daedeb854..2ed4942821539f 100644 --- a/drivers/hid/Makefile +++ b/drivers/hid/Makefile @@ -173,6 +173,12 @@ obj-$(CONFIG_INTEL_ISH_HID) += intel-ish-hid/ obj-$(CONFIG_AMD_SFH_HID) += amd-sfh-hid/ +obj-$(CONFIG_HID_DOCKCHANNEL) += dockchannel-hid/ + +obj-$(CONFIG_SPI_HID_APPLE_CORE) += spi-hid/ + +obj-$(CONFIG_HID_DOCKCHANNEL) += dockchannel-hid/ + obj-$(CONFIG_SURFACE_HID_CORE) += surface-hid/ obj-$(CONFIG_INTEL_THC_HID) += intel-thc-hid/ diff --git a/drivers/hid/dockchannel-hid/Kconfig b/drivers/hid/dockchannel-hid/Kconfig new file mode 100644 index 00000000000000..254961ad15e19c --- /dev/null +++ b/drivers/hid/dockchannel-hid/Kconfig @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +menu "DockChannel HID support" + depends on APPLE_DOCKCHANNEL + +config HID_DOCKCHANNEL + tristate "HID over DockChannel transport layer for Apple Silicon SoCs" + depends on APPLE_DOCKCHANNEL && INPUT && OF && HID + help + Say Y here if you use an M2 or later Apple Silicon based laptop. + The keyboard and touchpad are HID based devices connected via the + proprietary DockChannel interface. + +endmenu diff --git a/drivers/hid/dockchannel-hid/Makefile b/drivers/hid/dockchannel-hid/Makefile new file mode 100644 index 00000000000000..7dba766b047fcc --- /dev/null +++ b/drivers/hid/dockchannel-hid/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# +# Makefile for DockChannel HID transport drivers +# + +obj-$(CONFIG_HID_DOCKCHANNEL) += dockchannel-hid.o diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c new file mode 100644 index 00000000000000..a712a724ded30b --- /dev/null +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -0,0 +1,1213 @@ +/* + * SPDX-License-Identifier: GPL-2.0 OR MIT + * + * Apple DockChannel HID transport driver + * + * Copyright The Asahi Linux Contributors + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../hid-ids.h" + +#define COMMAND_TIMEOUT_MS 1000 +#define START_TIMEOUT_MS 2000 + +#define MAX_INTERFACES 16 + +/* Data + checksum */ +#define MAX_PKT_SIZE (0xffff + 4) + +#define DCHID_CHANNEL_CMD 0x11 +#define DCHID_CHANNEL_REPORT 0x12 + +struct dchid_hdr { + u8 hdr_len; + u8 channel; + u16 length; + u8 seq; + u8 iface; + u16 pad; +} __packed; + +#define IFACE_COMM 0 + +#define FLAGS_GROUP GENMASK(7, 6) +#define FLAGS_REQ GENMASK(5, 0) + +#define REQ_SET_REPORT 0 +#define REQ_GET_REPORT 1 + +struct dchid_subhdr { + u8 flags; + u8 unk; + u16 length; + u32 retcode; +} __packed; + +#define EVENT_GPIO_CMD 0xa0 +#define EVENT_INIT 0xf0 +#define EVENT_READY 0xf1 + +struct dchid_init_hdr { + u8 type; + u8 unk1; + u8 unk2; + u8 iface; + char name[16]; + u8 more_packets; + u8 unkpad; +} __packed; + +#define INIT_HID_DESCRIPTOR 0 +#define INIT_GPIO_REQUEST 1 +#define INIT_TERMINATOR 2 +#define INIT_PRODUCT_NAME 7 + +#define CMD_RESET_INTERFACE 0x40 +#define CMD_SEND_FIRMWARE 0x95 +#define CMD_ENABLE_INTERFACE 0xb4 +#define CMD_ACK_GPIO_CMD 0xa1 + +struct dchid_init_block_hdr { + u16 type; + u16 length; +} __packed; + +#define MAX_GPIO_NAME 32 + +struct dchid_gpio_request { + u16 unk; + u16 id; + char name[MAX_GPIO_NAME]; +} __packed; + +struct dchid_gpio_cmd { + u8 type; + u8 iface; + u8 gpio; + u8 unk; + u8 cmd; +} __packed; + +struct dchid_gpio_ack { + u8 type; + u32 retcode; + u8 cmd[]; +} __packed; + +#define STM_REPORT_ID 0x10 +#define STM_REPORT_SERIAL 0x11 +#define STM_REPORT_KEYBTYPE 0x14 + +struct dchid_stm_id { + u8 unk; + u16 vendor_id; + u16 product_id; + u16 version_number; + u8 unk2; + u8 unk3; + u8 keyboard_type; + u8 serial_length; + /* Serial follows, but we grab it with a different report. */ +} __packed; + +#define FW_MAGIC 0x46444948 +#define FW_VER 1 + +struct fw_header { + u32 magic; + u32 version; + u32 hdr_length; + u32 data_length; + u32 iface_offset; +} __packed; + +struct dchid_work { + struct work_struct work; + struct dchid_iface *iface; + + struct dchid_hdr hdr; + u8 data[]; +}; + +struct dchid_iface { + struct dockchannel_hid *dchid; + struct hid_device *hid; + struct workqueue_struct *wq; + + bool creating; + struct work_struct create_work; + + int index; + const char *name; + const struct device_node *of_node; + + uint8_t tx_seq; + bool deferred; + bool starting; + bool open; + struct completion ready; + + void *hid_desc; + size_t hid_desc_len; + + struct gpio_desc *gpio; + char gpio_name[MAX_GPIO_NAME]; + int gpio_id; + + struct mutex out_mutex; + u32 out_flags; + int out_report; + u32 retcode; + void *resp_buf; + size_t resp_size; + struct completion out_complete; + + u32 keyboard_layout_id; +}; + +struct dockchannel_hid { + struct device *dev; + struct dockchannel *dc; + struct device_link *helper_link; + + bool id_ready; + struct dchid_stm_id device_id; + char serial[64]; + + struct dchid_iface *comm; + struct dchid_iface *ifaces[MAX_INTERFACES]; + + u8 pkt_buf[MAX_PKT_SIZE]; + + /* Workqueue to asynchronously create HID devices */ + struct workqueue_struct *new_iface_wq; +}; + +static ssize_t apple_layout_id_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct hid_device *hdev = to_hid_device(dev); + struct dchid_iface *iface = hdev->driver_data; + + return scnprintf(buf, PAGE_SIZE, "%d\n", iface->keyboard_layout_id); +} + +static DEVICE_ATTR_RO(apple_layout_id); + +static struct dchid_iface * +dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) +{ + struct dchid_iface *iface; + + if (index >= MAX_INTERFACES) { + dev_err(dchid->dev, "Interface index %d out of range\n", index); + return NULL; + } + + if (dchid->ifaces[index]) + return dchid->ifaces[index]; + + iface = devm_kzalloc(dchid->dev, sizeof(struct dchid_iface), GFP_KERNEL); + if (!iface) + return NULL; + + iface->index = index; + iface->name = devm_kstrdup(dchid->dev, name, GFP_KERNEL); + iface->dchid = dchid; + iface->out_report= -1; + init_completion(&iface->out_complete); + init_completion(&iface->ready); + mutex_init(&iface->out_mutex); + iface->wq = alloc_ordered_workqueue("dchid-%s", WQ_MEM_RECLAIM, iface->name); + if (!iface->wq) + return NULL; + + /* Comm is not a HID subdevice */ + if (!strcmp(name, "comm")) { + dchid->ifaces[index] = iface; + return iface; + } + + iface->of_node = of_get_child_by_name(dchid->dev->of_node, name); + if (!iface->of_node) { + dev_warn(dchid->dev, "No OF node for subdevice %s, ignoring.", name); + return NULL; + } + + dchid->ifaces[index] = iface; + return iface; +} + +static u32 dchid_checksum(void *p, size_t length) +{ + u32 sum = 0; + + while (length >= 4) { + sum += get_unaligned_le32(p); + p += 4; + length -= 4; + } + + WARN_ON_ONCE(length); + return sum; +} + +static int dchid_send(struct dchid_iface *iface, u32 flags, void *msg, size_t size) +{ + u32 checksum = 0xffffffff; + size_t wsize = round_down(size, 4); + size_t tsize = size - wsize; + int ret; + struct { + struct dchid_hdr hdr; + struct dchid_subhdr sub; + } __packed h; + + memset(&h, 0, sizeof(h)); + h.hdr.hdr_len = sizeof(h.hdr); + h.hdr.channel = DCHID_CHANNEL_CMD; + h.hdr.length = round_up(size, 4) + sizeof(h.sub); + h.hdr.seq = iface->tx_seq; + h.hdr.iface = iface->index; + h.sub.flags = flags; + h.sub.length = size; + + ret = dockchannel_send(iface->dchid->dc, &h, sizeof(h)); + if (ret < 0) + return ret; + checksum -= dchid_checksum(&h, sizeof(h)); + + ret = dockchannel_send(iface->dchid->dc, msg, wsize); + if (ret < 0) + return ret; + checksum -= dchid_checksum(msg, wsize); + + if (tsize) { + u8 tail[4] = {0, 0, 0, 0}; + + memcpy(tail, msg + wsize, tsize); + ret = dockchannel_send(iface->dchid->dc, tail, sizeof(tail)); + if (ret < 0) + return ret; + checksum -= dchid_checksum(tail, sizeof(tail)); + } + + ret = dockchannel_send(iface->dchid->dc, &checksum, sizeof(checksum)); + if (ret < 0) + return ret; + + return 0; +} + +static int dchid_cmd(struct dchid_iface *iface, u32 type, u32 req, + void *data, size_t size, void *resp_buf, size_t resp_size) +{ + int ret; + int report_id = *(u8*)data; + + mutex_lock(&iface->out_mutex); + + WARN_ON(iface->out_report != -1); + iface->out_report = report_id; + iface->out_flags = FIELD_PREP(FLAGS_GROUP, type) | FIELD_PREP(FLAGS_REQ, req); + iface->resp_buf = resp_buf; + iface->resp_size = resp_size; + reinit_completion(&iface->out_complete); + + ret = dchid_send(iface, iface->out_flags, data, size); + if (ret < 0) + goto done; + + if (!wait_for_completion_timeout(&iface->out_complete, msecs_to_jiffies(COMMAND_TIMEOUT_MS))) { + dev_err(iface->dchid->dev, "output report 0x%x to iface %d (%s) timed out\n", + report_id, iface->index, iface->name); + ret = -ETIMEDOUT; + goto done; + } + + ret = iface->resp_size; + if (iface->retcode) { + dev_err(iface->dchid->dev, + "output report 0x%x to iface %d (%s) failed with err 0x%x\n", + report_id, iface->index, iface->name, iface->retcode); + ret = -EIO; + } + +done: + iface->tx_seq++; + iface->out_report = -1; + iface->out_flags = 0; + iface->resp_buf = NULL; + iface->resp_size = 0; + mutex_unlock(&iface->out_mutex); + return ret; +} + +static int dchid_comm_cmd(struct dockchannel_hid *dchid, void *cmd, size_t size) +{ + return dchid_cmd(dchid->comm, HID_FEATURE_REPORT, REQ_SET_REPORT, cmd, size, NULL, 0); +} + +static int dchid_enable_interface(struct dchid_iface *iface) +{ + u8 msg[] = { CMD_ENABLE_INTERFACE, iface->index }; + + return dchid_comm_cmd(iface->dchid, msg, sizeof(msg)); +} + +static int dchid_reset_interface(struct dchid_iface *iface, int state) +{ + u8 msg[] = { CMD_RESET_INTERFACE, 1, iface->index, state }; + + return dchid_comm_cmd(iface->dchid, msg, sizeof(msg)); +} + +static int dchid_send_firmware(struct dchid_iface *iface, void *firmware, size_t size) +{ + struct { + u8 cmd; + u8 unk1; + u8 unk2; + u8 iface; + u64 addr; + u32 size; + } __packed msg = { + .cmd = CMD_SEND_FIRMWARE, + .unk1 = 2, + .unk2 = 0, + .iface = iface->index, + .size = size, + }; + dma_addr_t addr; + void *buf = dmam_alloc_coherent(iface->dchid->dev, size, &addr, GFP_KERNEL); + + if (IS_ERR_OR_NULL(buf)) + return buf ? PTR_ERR(buf) : -ENOMEM; + + msg.addr = addr; + memcpy(buf, firmware, size); + wmb(); + + return dchid_comm_cmd(iface->dchid, &msg, sizeof(msg)); +} + +static int dchid_get_firmware(struct dchid_iface *iface, void **firmware, size_t *size) +{ + int ret; + const char *fw_name; + const struct firmware *fw; + struct fw_header *hdr; + u8 *fw_data; + + ret = of_property_read_string(iface->of_node, "firmware-name", &fw_name); + if (ret) { + /* Firmware is only for some devices */ + *firmware = NULL; + *size = 0; + return 0; + } + + ret = request_firmware(&fw, fw_name, iface->dchid->dev); + if (ret) + return ret; + + hdr = (struct fw_header *)fw->data; + + if (hdr->magic != FW_MAGIC || hdr->version != FW_VER || + hdr->hdr_length < sizeof(*hdr) || hdr->hdr_length > fw->size || + (hdr->hdr_length + (size_t)hdr->data_length) > fw->size || + hdr->iface_offset >= hdr->data_length) { + dev_warn(iface->dchid->dev, "%s: invalid firmware header\n", + fw_name); + ret = -EINVAL; + goto done; + } + + fw_data = devm_kmemdup(iface->dchid->dev, fw->data + hdr->hdr_length, + hdr->data_length, GFP_KERNEL); + if (!fw_data) { + ret = -ENOMEM; + goto done; + } + + if (hdr->iface_offset) + fw_data[hdr->iface_offset] = iface->index; + + *firmware = fw_data; + *size = hdr->data_length; + +done: + release_firmware(fw); + return ret; +} + +static int dchid_request_gpio(struct dchid_iface *iface) +{ + char prop_name[MAX_GPIO_NAME + 16]; + + if (iface->gpio) + return 0; + + dev_info(iface->dchid->dev, "Requesting GPIO %s#%d: %s\n", + iface->name, iface->gpio_id, iface->gpio_name); + + snprintf(prop_name, sizeof(prop_name), "apple,%s", iface->gpio_name); + + iface->gpio = devm_gpiod_get_index(iface->dchid->dev, prop_name, 0, GPIOD_OUT_LOW); + + if (IS_ERR_OR_NULL(iface->gpio)) { + dev_err(iface->dchid->dev, "Failed to request GPIO %s-gpios\n", prop_name); + iface->gpio = NULL; + return -1; + } + + return 0; +} + +static int dchid_start_interface(struct dchid_iface *iface) +{ + void *fw; + size_t size; + int ret; + + if (iface->starting) { + dev_warn(iface->dchid->dev, "Interface %s is already starting", iface->name); + return -EINPROGRESS; + } + + dev_info(iface->dchid->dev, "Starting interface %s\n", iface->name); + + iface->starting = true; + + /* Look to see if we need firmware */ + ret = dchid_get_firmware(iface, &fw, &size); + if (ret < 0) + goto err; + + /* If we need a GPIO, make sure we have it. */ + if (iface->gpio_id) { + ret = dchid_request_gpio(iface); + if (ret < 0) + goto err; + } + + /* Only multi-touch has firmware */ + if (fw && size) { + + /* Send firmware to the device */ + dev_info(iface->dchid->dev, "Sending firmware for %s\n", iface->name); + ret = dchid_send_firmware(iface, fw, size); + if (ret < 0) { + dev_err(iface->dchid->dev, "Failed to send %s firmwareS", iface->name); + goto err; + } + + /* After loading firmware, multi-touch needs a reset */ + dev_info(iface->dchid->dev, "Resetting %s\n", iface->name); + dchid_reset_interface(iface, 0); + dchid_reset_interface(iface, 2); + } + + return 0; + +err: + iface->starting = false; + return ret; +} + +static int dchid_start(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + + if (iface->keyboard_layout_id) { + int ret = device_create_file(&hdev->dev, &dev_attr_apple_layout_id); + if (ret) { + dev_warn(iface->dchid->dev, "Failed to create apple_layout_id: %d", ret); + iface->keyboard_layout_id = 0; + } + } + + return 0; +}; + +static void dchid_stop(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + + if (iface->keyboard_layout_id) + device_remove_file(&hdev->dev, &dev_attr_apple_layout_id); +} + +static int dchid_open(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + int ret; + + if (!completion_done(&iface->ready)) { + ret = dchid_start_interface(iface); + if (ret < 0) + return ret; + + if (!wait_for_completion_timeout(&iface->ready, msecs_to_jiffies(START_TIMEOUT_MS))) { + dev_err(iface->dchid->dev, "iface %s start timed out\n", iface->name); + return -ETIMEDOUT; + } + } + + iface->open = true; + return 0; +} + +static void dchid_close(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + + iface->open = false; +} + +static int dchid_parse(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + + return hid_parse_report(hdev, iface->hid_desc, iface->hid_desc_len); +} + +/* Note: buf excludes report number! For ease of fetching strings/etc. */ +static int dchid_get_report_cmd(struct dchid_iface *iface, u8 reportnum, void *buf, size_t len) +{ + int ret = dchid_cmd(iface, HID_FEATURE_REPORT, REQ_GET_REPORT, &reportnum, 1, buf, len); + + return ret <= 0 ? ret : ret - 1; +} + +/* Note: buf includes report number! */ +static int dchid_set_report(struct dchid_iface *iface, void *buf, size_t len) +{ + return dchid_cmd(iface, HID_OUTPUT_REPORT, REQ_SET_REPORT, buf, len, NULL, 0); +} + +static int dchid_raw_request(struct hid_device *hdev, + unsigned char reportnum, __u8 *buf, size_t len, + unsigned char rtype, int reqtype) +{ + struct dchid_iface *iface = hdev->driver_data; + + switch (reqtype) { + case HID_REQ_GET_REPORT: + buf[0] = reportnum; + return dchid_cmd(iface, rtype, REQ_GET_REPORT, &reportnum, 1, buf + 1, len - 1); + case HID_REQ_SET_REPORT: + return dchid_set_report(iface, buf, len); + default: + return -EIO; + } + + return 0; +} + +static struct hid_ll_driver dchid_ll = { + .start = &dchid_start, + .stop = &dchid_stop, + .open = &dchid_open, + .close = &dchid_close, + .parse = &dchid_parse, + .raw_request = &dchid_raw_request, +}; + +static void dchid_create_interface_work(struct work_struct *ws) +{ + struct dchid_iface *iface = container_of(ws, struct dchid_iface, create_work); + struct dockchannel_hid *dchid = iface->dchid; + struct hid_device *hid; + int ret; + + if (iface->hid) { + dev_warn(dchid->dev, "Interface %s already created!\n", + iface->name); + return; + } + + dev_info(dchid->dev, "New interface %s\n", iface->name); + + /* Start the interface. This is not the entire init process, as firmware is loaded later on device open. */ + ret = dchid_enable_interface(iface); + if (ret < 0) { + dev_warn(dchid->dev, "Failed to enable %s: %d\n", iface->name, ret); + return; + } + + iface->deferred = false; + + hid = hid_allocate_device(); + if (IS_ERR(hid)) + return; + + snprintf(hid->name, sizeof(hid->name), "Apple MTP %s", iface->name); + snprintf(hid->phys, sizeof(hid->phys), "%s.%d (%s)", + dev_name(dchid->dev), iface->index, iface->name); + strscpy(hid->uniq, dchid->serial, sizeof(hid->uniq)); + + hid->ll_driver = &dchid_ll; + hid->bus = BUS_HOST; + hid->vendor = dchid->device_id.vendor_id; + hid->product = dchid->device_id.product_id; + hid->version = dchid->device_id.version_number; + hid->type = HID_TYPE_OTHER; + if (!strcmp(iface->name, "multi-touch")) { + hid->type = HID_TYPE_SPI_MOUSE; + } else if (!strcmp(iface->name, "keyboard")) { + u32 country_code = 0; + + hid->type = HID_TYPE_SPI_KEYBOARD; + + /* + * We have to get the country code from the device tree, since the + * device provides no reliable way to get this info. + */ + if (!of_property_read_u32(iface->of_node, "hid-country-code", &country_code)) + hid->country = country_code; + + of_property_read_u32(iface->of_node, "apple,keyboard-layout-id", + &iface->keyboard_layout_id); + } + + hid->dev.parent = iface->dchid->dev; + hid->driver_data = iface; + + iface->hid = hid; + + ret = hid_add_device(hid); + if (ret < 0) { + iface->hid = NULL; + hid_destroy_device(hid); + dev_warn(iface->dchid->dev, "Failed to register hid device %s", iface->name); + } +} + +static int dchid_create_interface(struct dchid_iface *iface) +{ + if (iface->creating) + return -EBUSY; + + iface->creating = true; + INIT_WORK(&iface->create_work, dchid_create_interface_work); + return queue_work(iface->dchid->new_iface_wq, &iface->create_work); +} + +static void dchid_handle_descriptor(struct dchid_iface *iface, void *hid_desc, size_t desc_len) +{ + if (iface->hid) { + dev_warn(iface->dchid->dev, "Tried to initialize already started interface %s!\n", + iface->name); + return; + } + + iface->hid_desc = devm_kmemdup(iface->dchid->dev, hid_desc, desc_len, GFP_KERNEL); + if (!iface->hid_desc) + return; + + iface->hid_desc_len = desc_len; +} + +static void dchid_handle_ready(struct dockchannel_hid *dchid, void *data, size_t length) +{ + struct dchid_iface *iface; + u8 *pkt = data; + u8 index; + int i, ret; + + if (length < 2) { + dev_err(dchid->dev, "Bad length for ready message: %zu\n", length); + return; + } + + index = pkt[1]; + + if (index >= MAX_INTERFACES) { + dev_err(dchid->dev, "Got ready notification for bad iface %d\n", index); + return; + } + + iface = dchid->ifaces[index]; + if (!iface) { + dev_err(dchid->dev, "Got ready notification for unknown iface %d\n", index); + return; + } + + dev_info(dchid->dev, "Interface %s is now ready\n", iface->name); + complete_all(&iface->ready); + + /* When STM is ready, grab global device info */ + if (!strcmp(iface->name, "stm")) { + ret = dchid_get_report_cmd(iface, STM_REPORT_ID, &dchid->device_id, + sizeof(dchid->device_id)); + if (ret < sizeof(dchid->device_id)) { + dev_warn(iface->dchid->dev, "Failed to get device ID from STM!\n"); + /* Fake it and keep going. Things might still work... */ + memset(&dchid->device_id, 0, sizeof(dchid->device_id)); + dchid->device_id.vendor_id = HOST_VENDOR_ID_APPLE; + } + ret = dchid_get_report_cmd(iface, STM_REPORT_SERIAL, dchid->serial, + sizeof(dchid->serial) - 1); + if (ret < 0) { + dev_warn(iface->dchid->dev, "Failed to get serial from STM!\n"); + dchid->serial[0] = 0; + } + + dchid->id_ready = true; + for (i = 0; i < MAX_INTERFACES; i++) { + if (!dchid->ifaces[i] || !dchid->ifaces[i]->deferred) + continue; + dchid_create_interface(dchid->ifaces[i]); + } + } +} + +static void dchid_handle_init(struct dockchannel_hid *dchid, void *data, size_t length) +{ + struct dchid_init_hdr *hdr = data; + struct dchid_iface *iface; + struct dchid_init_block_hdr *blk; + + if (length < sizeof(*hdr)) + return; + + iface = dchid_get_interface(dchid, hdr->iface, hdr->name); + if (!iface) + return; + + data += sizeof(*hdr); + length -= sizeof(*hdr); + + while (length >= sizeof(*blk)) { + blk = data; + data += sizeof(*blk); + length -= sizeof(*blk); + + if (blk->length > length) + break; + + switch (blk->type) { + case INIT_HID_DESCRIPTOR: + dchid_handle_descriptor(iface, data, blk->length); + break; + + case INIT_GPIO_REQUEST: { + struct dchid_gpio_request *req = data; + + if (sizeof(*req) > length) + break; + + if (iface->gpio_id) { + dev_err(dchid->dev, + "Cannot request more than one GPIO per interface!\n"); + break; + } + + strscpy(iface->gpio_name, req->name, MAX_GPIO_NAME); + iface->gpio_id = req->id; + break; + } + + case INIT_TERMINATOR: + break; + + case INIT_PRODUCT_NAME: { + char *product = data; + + if (product[blk->length - 1] != 0) { + dev_warn(dchid->dev, "Unterminated product name for %s\n", + iface->name); + } else { + dev_info(dchid->dev, "Product name for %s: %s\n", + iface->name, product); + } + break; + } + + default: + dev_warn(dchid->dev, "Unknown init packet %d for %s\n", + blk->type, iface->name); + break; + } + + data += blk->length; + length -= blk->length; + + if (blk->type == INIT_TERMINATOR) + break; + } + + if (hdr->more_packets) + return; + + /* We need to enable STM first, since it'll give us the device IDs */ + if (iface->dchid->id_ready || !strcmp(iface->name, "stm")) { + dchid_create_interface(iface); + } else { + iface->deferred = true; + } +} + +static void dchid_handle_gpio(struct dockchannel_hid *dchid, void *data, size_t length) +{ + struct dchid_gpio_cmd *cmd = data; + struct dchid_iface *iface; + u32 retcode = 0xe000f00d; /* Give it a random Apple-style error code */ + struct dchid_gpio_ack *ack; + + if (length < sizeof(*cmd)) + return; + + if (cmd->iface >= MAX_INTERFACES || !(iface = dchid->ifaces[cmd->iface])) { + dev_err(dchid->dev, "Got GPIO command for bad inteface %d\n", cmd->iface); + goto err; + } + + if (dchid_request_gpio(iface) < 0) + goto err; + + if (!iface->gpio || cmd->gpio != iface->gpio_id) { + dev_err(dchid->dev, "Got GPIO command for bad GPIO %s#%d\n", + iface->name, cmd->gpio); + goto err; + } + + dev_info(dchid->dev, "GPIO command: %s#%d: %d\n", iface->name, cmd->gpio, cmd->cmd); + + switch (cmd->cmd) { + case 3: + /* Pulse. */ + gpiod_set_value_cansleep(iface->gpio, 1); + msleep(10); /* Random guess... */ + gpiod_set_value_cansleep(iface->gpio, 0); + retcode = 0; + break; + default: + dev_err(dchid->dev, "Unknown GPIO command %d\n", cmd->cmd ); + break; + } + +err: + /* Ack it */ + ack = kzalloc(sizeof(*ack) + length, GFP_KERNEL); + if (!ack) + return; + + ack->type = CMD_ACK_GPIO_CMD; + ack->retcode = retcode; + memcpy(ack->cmd, data, length); + + if (dchid_comm_cmd(dchid, ack, sizeof(*ack) + length) < 0) + dev_err(dchid->dev, "Failed to ACK GPIO command\n"); + + kfree(ack); +} + +static void dchid_handle_event(struct dockchannel_hid *dchid, void *data, size_t length) +{ + u8 *p = data; + switch (*p) { + case EVENT_INIT: + dchid_handle_init(dchid, data, length); + break; + case EVENT_READY: + dchid_handle_ready(dchid, data, length); + break; + case EVENT_GPIO_CMD: + dchid_handle_gpio(dchid, data, length); + break; + } +} + +static void dchid_handle_report(struct dchid_iface *iface, void *data, size_t length) +{ + struct dockchannel_hid *dchid = iface->dchid; + + if (!iface->hid) { + dev_warn(dchid->dev, "Report received but %s is not initialized!\n", iface->name); + return; + } + + if (!iface->open) + return; + + hid_input_report(iface->hid, HID_INPUT_REPORT, data, length, 1); +} + +static void dchid_packet_work(struct work_struct *ws) +{ + struct dchid_work *work = container_of(ws, struct dchid_work, work); + struct dchid_subhdr *shdr = (void *)work->data; + struct dockchannel_hid *dchid = work->iface->dchid; + int type = FIELD_GET(FLAGS_GROUP, shdr->flags); + u8 *payload = work->data + sizeof(*shdr); + + if (shdr->length + sizeof(*shdr) > work->hdr.length) { + dev_err(dchid->dev, "Bad sub header length (%d > %zu)\n", + shdr->length, work->hdr.length - sizeof(*shdr)); + return; + } + + switch (type) { + case HID_INPUT_REPORT: + if (work->hdr.iface == IFACE_COMM) + dchid_handle_event(dchid, payload, shdr->length); + else + dchid_handle_report(work->iface, payload, shdr->length); + break; + default: + dev_err(dchid->dev, "Received unknown packet type %d\n", type); + break; + } + + kfree(work); +} + +static void dchid_handle_ack(struct dchid_iface *iface, struct dchid_hdr *hdr, void *data) +{ + struct dchid_subhdr *shdr = (void *)data; + u8 *payload = data + sizeof(*shdr); + + if (shdr->length + sizeof(*shdr) > hdr->length) { + dev_err(iface->dchid->dev, "Bad sub header length (%d > %ld)\n", + shdr->length, hdr->length - sizeof(*shdr)); + return; + } + if (shdr->flags != iface->out_flags) { + dev_err(iface->dchid->dev, + "Received unexpected flags 0x%x on ACK channel (expFected 0x%x)\n", + shdr->flags, iface->out_flags); + return; + } + + if (shdr->length < 1) { + dev_err(iface->dchid->dev, "Received length 0 output report ack\n"); + return; + } + if (iface->tx_seq != hdr->seq) { + dev_err(iface->dchid->dev, "Received ACK with bad seq (expected %d, got %d)\n", + iface->tx_seq, hdr->seq); + return; + } + if (iface->out_report != payload[0]) { + dev_err(iface->dchid->dev, "Received ACK with bad report (expected %d, got %d\n", + iface->out_report, payload[0]); + return; + } + + if (iface->resp_buf && iface->resp_size) + memcpy(iface->resp_buf, payload + 1, min((size_t)shdr->length - 1, iface->resp_size)); + + iface->resp_size = shdr->length; + iface->out_report = -1; + iface->retcode = shdr->retcode; + complete(&iface->out_complete); +} + +static void dchid_handle_packet(void *cookie, size_t avail) +{ + struct dockchannel_hid *dchid = cookie; + struct dchid_hdr hdr; + struct dchid_work *work; + struct dchid_iface *iface; + u32 checksum; + + if (dockchannel_recv(dchid->dc, &hdr, sizeof(hdr)) != sizeof(hdr)) { + dev_err(dchid->dev, "Read failed (header)\n"); + return; + } + + if (hdr.hdr_len != sizeof(hdr)) { + dev_err(dchid->dev, "Bad header length %d\n", hdr.hdr_len); + goto done; + } + + if (dockchannel_recv(dchid->dc, dchid->pkt_buf, hdr.length + 4) != (hdr.length + 4)) { + dev_err(dchid->dev, "Read failed (body)\n"); + goto done; + } + + checksum = dchid_checksum(&hdr, sizeof(hdr)); + checksum += dchid_checksum(dchid->pkt_buf, hdr.length + 4); + + if (checksum != 0xffffffff) { + dev_err(dchid->dev, "Checksum mismatch (iface %d): 0x%08x != 0xffffffff\n", + hdr.iface, checksum); + goto done; + } + + + if (hdr.iface >= MAX_INTERFACES) { + dev_err(dchid->dev, "Bad iface %d\n", hdr.iface); + } + + iface = dchid->ifaces[hdr.iface]; + + if (!iface) { + dev_err(dchid->dev, "Received packet for uninitialized iface %d\n", hdr.iface); + goto done; + } + + switch (hdr.channel) { + case DCHID_CHANNEL_CMD: + dchid_handle_ack(iface, &hdr, dchid->pkt_buf); + goto done; + case DCHID_CHANNEL_REPORT: + break; + default: + dev_warn(dchid->dev, "Unknown channel 0x%x, treating as report...\n", + hdr.channel); + break; + } + + work = kzalloc(sizeof(*work) + hdr.length, GFP_KERNEL); + if (!work) + return; + + work->hdr = hdr; + work->iface = iface; + memcpy(work->data, dchid->pkt_buf, hdr.length); + INIT_WORK(&work->work, dchid_packet_work); + + queue_work(iface->wq, &work->work); + +done: + dockchannel_await(dchid->dc, dchid_handle_packet, dchid, sizeof(struct dchid_hdr)); +} + +static int dockchannel_hid_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dockchannel_hid *dchid; + struct device_node *child, *helper; + struct platform_device *helper_pdev; + struct property *prop; + int ret; + + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (ret) + return ret; + + dchid = devm_kzalloc(dev, sizeof(*dchid), GFP_KERNEL); + if (!dchid) { + return -ENOMEM; + } + + dchid->dev = dev; + + /* + * First make sure all the GPIOs are available, in cased we need to defer. + * This is necessary because MTP will request them by name later, and by then + * it's too late to defer the probe. + */ + + for_each_child_of_node(dev->of_node, child) { + for_each_property_of_node(child, prop) { + size_t len = strlen(prop->name); + struct gpio_desc *gpio; + + if (len < 12 || strncmp("apple,", prop->name, 6) || + strcmp("-gpios", prop->name + len - 6)) + continue; + + gpio = fwnode_gpiod_get_index(&child->fwnode, prop->name, 0, GPIOD_ASIS, + prop->name); + if (IS_ERR_OR_NULL(gpio)) { + if (PTR_ERR(gpio) == -EPROBE_DEFER) { + of_node_put(child); + return -EPROBE_DEFER; + } + } else { + gpiod_put(gpio); + } + } + } + + /* + * Make sure we also have the MTP coprocessor available, and + * defer probe if the helper hasn't probed yet. + */ + helper = of_parse_phandle(dev->of_node, "apple,helper-cpu", 0); + if (!helper) { + dev_err(dev, "Missing apple,helper-cpu property"); + return -EINVAL; + } + + helper_pdev = of_find_device_by_node(helper); + of_node_put(helper); + if (!helper_pdev) { + dev_err(dev, "Failed to find helper device"); + return -EINVAL; + } + + dchid->helper_link = device_link_add(dev, &helper_pdev->dev, + DL_FLAG_AUTOREMOVE_CONSUMER); + put_device(&helper_pdev->dev); + if (!dchid->helper_link) { + dev_err(dev, "Failed to link to helper device"); + return -EINVAL; + } + + if (dchid->helper_link->supplier->links.status != DL_DEV_DRIVER_BOUND) + return -EPROBE_DEFER; + + /* Now it is safe to begin initializing */ + dchid->dc = dockchannel_init(pdev); + if (IS_ERR_OR_NULL(dchid->dc)) { + return PTR_ERR(dchid->dc); + } + dchid->new_iface_wq = alloc_workqueue("dchid-new", WQ_MEM_RECLAIM, 0); + if (!dchid->new_iface_wq) + return -ENOMEM; + + dchid->comm = dchid_get_interface(dchid, IFACE_COMM, "comm"); + if (!dchid->comm) { + dev_err(dchid->dev, "Failed to initialize comm interface"); + return -EIO; + } + + dev_info(dchid->dev, "Initialized, awaiting packets\n"); + dockchannel_await(dchid->dc, dchid_handle_packet, dchid, sizeof(struct dchid_hdr)); + + return 0; +} + +static void dockchannel_hid_remove(struct platform_device *pdev) +{ + BUG_ON(1); +} + +static const struct of_device_id dockchannel_hid_of_match[] = { + { .compatible = "apple,dockchannel-hid" }, + {}, +}; +MODULE_DEVICE_TABLE(of, dockchannel_hid_of_match); +MODULE_FIRMWARE("apple/tpmtfw-*.bin"); + +static struct platform_driver dockchannel_hid_driver = { + .driver = { + .name = "dockchannel-hid", + .of_match_table = dockchannel_hid_of_match, + }, + .probe = dockchannel_hid_probe, + .remove = dockchannel_hid_remove, +}; +module_platform_driver(dockchannel_hid_driver); + +MODULE_DESCRIPTION("Apple DockChannel HID transport driver"); +MODULE_AUTHOR("Hector Martin "); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/hid/hid-apple.c b/drivers/hid/hid-apple.c index 57da4f86a9fa7f..6d80a7e3cf185c 100644 --- a/drivers/hid/hid-apple.c +++ b/drivers/hid/hid-apple.c @@ -54,10 +54,16 @@ #define APPLE_MAGIC_REPORT_ID_POWER 3 #define APPLE_MAGIC_REPORT_ID_BRIGHTNESS 1 +// DO NOT UPSTREAM: +// temporary Fn key mode until xkeyboard-config has keyboard layouts with media +// key mappings. At that point auto mode can drop function key mappings and this +// mode can be dropped. +#define FKEYS_IGNORE 5 + static unsigned int fnmode = 3; module_param(fnmode, uint, 0644); MODULE_PARM_DESC(fnmode, "Mode of fn key on Apple keyboards (0 = disabled, " - "1 = fkeyslast, 2 = fkeysfirst, [3] = auto, 4 = fkeysdisabled)"); + "1 = fkeyslast, 2 = fkeysfirst, [3] = auto, 4 = fkeysdisabled, 5 = fkeysignore))"); static int iso_layout = -1; module_param(iso_layout, int, 0644); @@ -277,6 +283,16 @@ static const struct apple_key_translation apple_fn_keys[] = { { } }; +static const struct apple_key_translation apple_fn_keys_minimal[] = { + { KEY_BACKSPACE, KEY_DELETE }, + { KEY_ENTER, KEY_INSERT }, + { KEY_UP, KEY_PAGEUP }, + { KEY_DOWN, KEY_PAGEDOWN }, + { KEY_LEFT, KEY_HOME }, + { KEY_RIGHT, KEY_END }, + { } +}; + static const struct apple_key_translation powerbook_fn_keys[] = { { KEY_BACKSPACE, KEY_DELETE }, { KEY_F1, KEY_BRIGHTNESSDOWN, APPLE_FLAG_FKEY }, @@ -433,6 +449,8 @@ static int hidinput_apple_event(struct hid_device *hid, struct input_dev *input, real_fnmode = 2; else real_fnmode = 1; + } else if (fnmode == FKEYS_IGNORE) { + real_fnmode = 2; } else { real_fnmode = fnmode; } @@ -473,6 +491,23 @@ static int hidinput_apple_event(struct hid_device *hid, struct input_dev *input, asc->fn_on = !!value; if (real_fnmode) { + switch (hid->bus) { + case BUS_HOST: + case BUS_SPI: + switch (hid->product) { + case SPI_DEVICE_ID_APPLE_MACBOOK_PRO13_2020: + case HOST_DEVICE_ID_APPLE_MACBOOK_PRO13_2022: + table = macbookpro_dedicated_esc_fn_keys; + break; + default: + if (fnmode == FKEYS_IGNORE) + table = apple_fn_keys_minimal; + else + table = magic_keyboard_2021_and_2024_fn_keys; + break; + } + break; + default: switch (hid->product) { case USB_DEVICE_ID_APPLE_ALU_WIRELESS_ANSI: case USB_DEVICE_ID_APPLE_ALU_WIRELESS_ISO: @@ -521,6 +556,7 @@ static int hidinput_apple_event(struct hid_device *hid, struct input_dev *input, else table = apple_fn_keys; } + } trans = apple_find_translation(table, code); @@ -704,6 +740,7 @@ static void apple_setup_input(struct input_dev *input) /* Enable all needed keys */ apple_setup_key_translation(input, apple_fn_keys); + apple_setup_key_translation(input, apple_fn_keys_minimal); apple_setup_key_translation(input, powerbook_fn_keys); apple_setup_key_translation(input, powerbook_numlock_keys); apple_setup_key_translation(input, apple_iso_keyboard); @@ -938,6 +975,15 @@ static int apple_probe(struct hid_device *hdev, struct apple_sc *asc; int ret; + if ((id->bus == BUS_SPI || id->bus == BUS_HOST) && id->vendor == SPI_VENDOR_ID_APPLE && + hdev->type != HID_TYPE_SPI_KEYBOARD) + return -ENODEV; + + // key remapping will happen in xkeyboard-config so ignore + // APPLE_ISO_TILDE_QUIRK + if ((id->bus == BUS_SPI || id->bus == BUS_HOST) && fnmode == FKEYS_IGNORE) + quirks &= ~APPLE_ISO_TILDE_QUIRK; + asc = devm_kzalloc(&hdev->dev, sizeof(*asc), GFP_KERNEL); if (asc == NULL) { hid_err(hdev, "can't alloc apple descriptor\n"); @@ -1192,6 +1238,8 @@ static const struct hid_device_id apple_devices[] = { .driver_data = APPLE_NUMLOCK_EMULATION | APPLE_HAS_FN }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER1_TP_ONLY), .driver_data = APPLE_NUMLOCK_EMULATION | APPLE_HAS_FN }, + { HID_SPI_DEVICE(SPI_VENDOR_ID_APPLE, HID_ANY_ID), + .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, // TODO: remove APPLE_ISO_TILDE_QUIRK { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_2021), .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_RDESC_BATTERY }, { HID_BLUETOOTH_DEVICE(BT_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_2021), @@ -1204,6 +1252,8 @@ static const struct hid_device_id apple_devices[] = { .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_RDESC_BATTERY }, { HID_BLUETOOTH_DEVICE(BT_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_NUMPAD_2021), .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, + { HID_DEVICE(BUS_HOST, HID_GROUP_ANY, HOST_VENDOR_ID_APPLE, HID_ANY_ID), + .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, // TODO: remove APPLE_ISO_TILDE_QUIRK { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_2024), .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_RDESC_BATTERY }, { HID_BLUETOOTH_DEVICE(BT_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_2024), diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c index a5b3a8ca2fcbc8..3b1b999e7e5150 100644 --- a/drivers/hid/hid-core.c +++ b/drivers/hid/hid-core.c @@ -468,7 +468,10 @@ static int hid_parser_global(struct hid_parser *parser, struct hid_item *item) case HID_GLOBAL_ITEM_TAG_REPORT_SIZE: parser->global.report_size = item_udata(item); - if (parser->global.report_size > 256) { + /* Arbitrary maximum. Some Apple devices have 16384 here. + * This * HID_MAX_USAGES must fit in a signed integer. + */ + if (parser->global.report_size > 16384) { hid_err(parser->device, "invalid report_size %d\n", parser->global.report_size); return -1; @@ -2319,6 +2322,12 @@ int hid_connect(struct hid_device *hdev, unsigned int connect_mask) case BUS_SDW: bus = "SOUNDWIRE"; break; + case BUS_SPI: + bus = "SPI"; + break; + case BUS_HOST: + bus = "HOST"; + break; case BUS_VIRTUAL: bus = "VIRTUAL"; break; diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h index 9c2bf584d9f6f2..adad71cc8bf02d 100644 --- a/drivers/hid/hid-ids.h +++ b/drivers/hid/hid-ids.h @@ -93,6 +93,8 @@ #define USB_VENDOR_ID_APPLE 0x05ac #define BT_VENDOR_ID_APPLE 0x004c +#define SPI_VENDOR_ID_APPLE 0x05ac +#define HOST_VENDOR_ID_APPLE 0x05ac #define USB_DEVICE_ID_APPLE_MIGHTYMOUSE 0x0304 #define USB_DEVICE_ID_APPLE_MAGICMOUSE 0x030d #define USB_DEVICE_ID_APPLE_MAGICMOUSE2 0x0269 @@ -197,6 +199,14 @@ #define USB_DEVICE_ID_APPLE_IRCONTROL5 0x8243 #define USB_DEVICE_ID_APPLE_TOUCHBAR_BACKLIGHT 0x8102 #define USB_DEVICE_ID_APPLE_TOUCHBAR_DISPLAY 0x8302 +#define SPI_DEVICE_ID_APPLE_MACBOOK_AIR_2020 0x0281 +#define SPI_DEVICE_ID_APPLE_MACBOOK_PRO13_2020 0x0341 +#define SPI_DEVICE_ID_APPLE_MACBOOK_PRO14_2021 0x0342 +#define SPI_DEVICE_ID_APPLE_MACBOOK_PRO16_2021 0x0343 +#define HOST_DEVICE_ID_APPLE_MACBOOK_AIR13_2022 0x0351 +#define HOST_DEVICE_ID_APPLE_MACBOOK_PRO14_2023 0x0352 +#define HOST_DEVICE_ID_APPLE_MACBOOK_PRO16_2023 0x0353 +#define HOST_DEVICE_ID_APPLE_MACBOOK_PRO13_2022 0x0354 #define USB_VENDOR_ID_ASETEK 0x2433 #define USB_DEVICE_ID_ASETEK_INVICTA 0xf300 diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 7d4a25c6de0eb7..ae6f48071e1f32 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -60,8 +60,14 @@ MODULE_PARM_DESC(report_undeciphered, "Report undeciphered multi-touch state fie #define MOUSE_REPORT_ID 0x29 #define MOUSE2_REPORT_ID 0x12 #define DOUBLE_REPORT_ID 0xf7 +#define SPI_REPORT_ID 0x02 +#define SPI_RESET_REPORT_ID 0x60 +#define MTP_REPORT_ID 0x75 +#define SENSOR_DIMENSIONS_REPORT_ID 0xd9 #define USB_BATTERY_TIMEOUT_SEC 60 +#define MAX_CONTACTS 16 + /* These definitions are not precise, but they're close enough. (Bits * 0x03 seem to indicate the aspect ratio of the touch, bits 0x70 seem * to be some kind of bit mask -- 0x20 may be a near-field reading, @@ -112,30 +118,53 @@ MODULE_PARM_DESC(report_undeciphered, "Report undeciphered multi-touch state fie #define TRACKPAD2_RES_Y \ ((TRACKPAD2_MAX_Y - TRACKPAD2_MIN_Y) / (TRACKPAD2_DIMENSION_Y / 100)) +/* These are fallback values, since the real values will be queried from the device. */ +#define J314_TP_DIMENSION_X (float)13000 +#define J314_TP_MIN_X -5900 +#define J314_TP_MAX_X 6500 +#define J314_TP_RES_X \ + ((J314_TP_MAX_X - J314_TP_MIN_X) / (J314_TP_DIMENSION_X / 100)) +#define J314_TP_DIMENSION_Y (float)8100 +#define J314_TP_MIN_Y -200 +#define J314_TP_MAX_Y 7400 +#define J314_TP_RES_Y \ + ((J314_TP_MAX_Y - J314_TP_MIN_Y) / (J314_TP_DIMENSION_Y / 100)) + +#define J314_TP_MAX_FINGER_ORIENTATION 16384 + +struct magicmouse_input_ops { + int (*raw_event)(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size); + int (*setup_input)(struct input_dev *input, struct hid_device *hdev); +}; + /** * struct magicmouse_sc - Tracks Magic Mouse-specific data. * @input: Input device through which we report events. * @quirks: Currently unused. + * @query_dimensions: Whether to query and update dimensions on first open * @ntouches: Number of touches in most recent touch report. * @scroll_accel: Number of consecutive scroll motions. * @scroll_jiffies: Time of last scroll motion. + * @pos: multi touch position data of the last report. * @touches: Most recent data for a touch, indexed by tracking ID. * @tracking_ids: Mapping of current touch input data to @touches. * @hdev: Pointer to the underlying HID device. * @work: Workqueue to handle initialization retry for quirky devices. * @battery_timer: Timer for obtaining battery level information. + * @input_ops: Input ops based on device type. */ struct magicmouse_sc { struct input_dev *input; unsigned long quirks; + bool query_dimensions; int ntouches; int scroll_accel; unsigned long scroll_jiffies; + struct input_mt_pos pos[MAX_CONTACTS]; struct { - short x; - short y; short scroll_x; short scroll_y; short scroll_x_hr; @@ -143,14 +172,112 @@ struct magicmouse_sc { u8 size; bool scroll_x_active; bool scroll_y_active; - } touches[16]; - int tracking_ids[16]; + } touches[MAX_CONTACTS]; + int tracking_ids[MAX_CONTACTS]; struct hid_device *hdev; struct delayed_work work; struct timer_list battery_timer; + struct magicmouse_input_ops input_ops; }; +static int magicmouse_enable_multitouch(struct hid_device *hdev); + +static inline int le16_to_int(__le16 x) +{ + return (signed short)le16_to_cpu(x); +} + +static int magicmouse_open(struct input_dev *dev) +{ + struct hid_device *hdev = input_get_drvdata(dev); + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + int ret; + + ret = hid_hw_open(hdev); + if (ret) + return ret; + + /* + * Some devices repond with 'invalid report id' when feature + * report switching it into multitouch mode is sent to it. + * + * This results in -EIO from the _raw low-level transport callback, + * but there seems to be no other way of switching the mode. + * Thus the super-ugly hacky success check below. + * + * MTP devices do not need this. + */ + if (hdev->bus != BUS_HOST) { + ret = magicmouse_enable_multitouch(hdev); + if (ret != -EIO && ret < 0) { + hid_err(hdev, "unable to request touch data (%d)\n", ret); + return ret; + } + if (ret == -EIO && (hdev->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2 || + hdev->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2_USBC)) { + schedule_delayed_work(&msc->work, msecs_to_jiffies(500)); + } + } + + /* + * For Apple Silicon trackpads, we want to query the dimensions on + * device open. This is because doing so requires the firmware, but + * we don't want to force a firmware load until the device is opened + * for the first time. So do that here and update the input properties + * just in time before userspace queries them. + */ + if (msc->query_dimensions) { + struct input_dev *input = msc->input; + u8 buf[32]; + struct { + __le32 width; + __le32 height; + __le16 min_x; + __le16 min_y; + __le16 max_x; + __le16 max_y; + } dim; + uint32_t x_span, y_span; + + ret = hid_hw_raw_request(hdev, SENSOR_DIMENSIONS_REPORT_ID, buf, sizeof(buf), HID_FEATURE_REPORT, HID_REQ_GET_REPORT); + if (ret < (int)(1 + sizeof(dim))) { + hid_err(hdev, "unable to request dimensions (%d)\n", ret); + return ret; + } + + memcpy(&dim, buf + 1, sizeof(dim)); + + /* finger position */ + input_set_abs_params(input, ABS_MT_POSITION_X, + le16_to_int(dim.min_x), le16_to_int(dim.max_x), 0, 0); + /* Y axis is inverted */ + input_set_abs_params(input, ABS_MT_POSITION_Y, + -le16_to_int(dim.max_y), -le16_to_int(dim.min_y), 0, 0); + x_span = le16_to_int(dim.max_x) - le16_to_int(dim.min_x); + y_span = le16_to_int(dim.max_y) - le16_to_int(dim.min_y); + + /* X/Y resolution */ + input_abs_set_res(input, ABS_MT_POSITION_X, 100 * x_span / le32_to_cpu(dim.width) ); + input_abs_set_res(input, ABS_MT_POSITION_Y, 100 * y_span / le32_to_cpu(dim.height) ); + + /* copy info, as input_mt_init_slots() does */ + dev->absinfo[ABS_X] = dev->absinfo[ABS_MT_POSITION_X]; + dev->absinfo[ABS_Y] = dev->absinfo[ABS_MT_POSITION_Y]; + + msc->query_dimensions = false; + } + + return 0; +} + +static void magicmouse_close(struct input_dev *dev) +{ + struct hid_device *hdev = input_get_drvdata(dev); + + hid_hw_close(hdev); +} + static int magicmouse_firm_touch(struct magicmouse_sc *msc) { int touch = -1; @@ -192,7 +319,7 @@ static void magicmouse_emit_buttons(struct magicmouse_sc *msc, int state) } else if (last_state != 0) { state = last_state; } else if ((id = magicmouse_firm_touch(msc)) >= 0) { - int x = msc->touches[id].x; + int x = msc->pos[id].x; if (x < middle_button_start) state = 1; else if (x > middle_button_stop) @@ -256,8 +383,8 @@ static void magicmouse_emit_touch(struct magicmouse_sc *msc, int raw_id, u8 *tda /* Store tracking ID and other fields. */ msc->tracking_ids[raw_id] = id; - msc->touches[id].x = x; - msc->touches[id].y = y; + msc->pos[id].x = x; + msc->pos[id].y = y; msc->touches[id].size = size; /* If requested, emulate a scroll wheel by detecting small @@ -387,6 +514,14 @@ static int magicmouse_raw_event(struct hid_device *hdev, struct hid_report *report, u8 *data, int size) { struct magicmouse_sc *msc = hid_get_drvdata(hdev); + + return msc->input_ops.raw_event(hdev, report, data, size); +} + +static int magicmouse_raw_event_usb(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); struct input_dev *input = msc->input; int x = 0, y = 0, ii, clicks = 0, npoints; @@ -518,6 +653,174 @@ static int magicmouse_raw_event(struct hid_device *hdev, return 1; } +/** + * struct tp_finger - single trackpad finger structure, le16-aligned + * + * @unknown1: unknown + * @unknown2: unknown + * @abs_x: absolute x coordinate + * @abs_y: absolute y coordinate + * @rel_x: relative x coordinate + * @rel_y: relative y coordinate + * @tool_major: tool area, major axis + * @tool_minor: tool area, minor axis + * @orientation: 16384 when point, else 15 bit angle + * @touch_major: touch area, major axis + * @touch_minor: touch area, minor axis + * @unused: zeros + * @pressure: pressure on forcetouch touchpad + * @multi: one finger: varies, more fingers: constant + */ +struct tp_finger { + __le16 unknown1; + __le16 unknown2; + __le16 abs_x; + __le16 abs_y; + __le16 rel_x; + __le16 rel_y; + __le16 tool_major; + __le16 tool_minor; + __le16 orientation; + __le16 touch_major; + __le16 touch_minor; + __le16 unused[2]; + __le16 pressure; + __le16 multi; +} __attribute__((packed, aligned(2))); + +/** + * vendor trackpad report + * + * @num_fingers: the number of fingers being reported in @fingers + * @buttons: same as HID buttons + */ +struct tp_header { + // HID vendor part, up to 1751 bytes + u8 unknown[22]; + u8 num_fingers; + u8 buttons; + u8 unknown3[14]; +}; + +/** + * standard HID mouse report + * + * @report_id: reportid + * @buttons: HID Usage Buttons 3 1-bit reports + */ +struct tp_mouse_report { + // HID mouse report + u8 report_id; + u8 buttons; + u8 rel_x; + u8 rel_y; + u8 padding[4]; +}; + +static void report_finger_data(struct input_dev *input, int slot, + const struct input_mt_pos *pos, + const struct tp_finger *f) +{ + input_mt_slot(input, slot); + input_mt_report_slot_state(input, MT_TOOL_FINGER, true); + + input_report_abs(input, ABS_MT_TOUCH_MAJOR, + le16_to_int(f->touch_major) << 1); + input_report_abs(input, ABS_MT_TOUCH_MINOR, + le16_to_int(f->touch_minor) << 1); + input_report_abs(input, ABS_MT_WIDTH_MAJOR, + le16_to_int(f->tool_major) << 1); + input_report_abs(input, ABS_MT_WIDTH_MINOR, + le16_to_int(f->tool_minor) << 1); + input_report_abs(input, ABS_MT_ORIENTATION, + J314_TP_MAX_FINGER_ORIENTATION - le16_to_int(f->orientation)); + input_report_abs(input, ABS_MT_PRESSURE, le16_to_int(f->pressure)); + input_report_abs(input, ABS_MT_POSITION_X, pos->x); + input_report_abs(input, ABS_MT_POSITION_Y, pos->y); +} + +static int magicmouse_raw_event_mtp(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + struct input_dev *input = msc->input; + struct tp_header *tp_hdr; + struct tp_finger *f; + int i, n; + u32 npoints; + const size_t hdr_sz = sizeof(struct tp_header); + const size_t touch_sz = sizeof(struct tp_finger); + u8 map_contacs[MAX_CONTACTS]; + + // hid_warn(hdev, "%s\n", __func__); + // print_hex_dump_debug("appleft ev: ", DUMP_PREFIX_OFFSET, 16, 1, data, + // size, false); + + /* Expect 46 bytes of prefix, and N * 30 bytes of touch data. */ + if (size < hdr_sz || ((size - hdr_sz) % touch_sz) != 0) + return 0; + + tp_hdr = (struct tp_header *)data; + + npoints = (size - hdr_sz) / touch_sz; + if (npoints < tp_hdr->num_fingers || npoints > MAX_CONTACTS) { + hid_warn(hdev, + "unexpected number of touches (%u) for " + "report\n", + npoints); + return 0; + } + + n = 0; + for (i = 0; i < tp_hdr->num_fingers; i++) { + f = (struct tp_finger *)(data + hdr_sz + i * touch_sz); + if (le16_to_int(f->touch_major) == 0) + continue; + + hid_dbg(hdev, "ev x:%04x y:%04x\n", le16_to_int(f->abs_x), + le16_to_int(f->abs_y)); + msc->pos[n].x = le16_to_int(f->abs_x); + msc->pos[n].y = -le16_to_int(f->abs_y); + map_contacs[n] = i; + n++; + } + + input_mt_assign_slots(input, msc->tracking_ids, msc->pos, n, 0); + + for (i = 0; i < n; i++) { + int idx = map_contacs[i]; + f = (struct tp_finger *)(data + hdr_sz + idx * touch_sz); + report_finger_data(input, msc->tracking_ids[i], &msc->pos[i], f); + } + + input_mt_sync_frame(input); + input_report_key(input, BTN_MOUSE, tp_hdr->buttons & 1); + + input_sync(input); + return 1; +} + +static int magicmouse_raw_event_spi(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + const size_t hdr_sz = sizeof(struct tp_mouse_report); + + if (!size) + return 0; + + if (data[0] == SPI_RESET_REPORT_ID) { + hid_info(hdev, "Touch controller was reset, re-enabling touch mode\n"); + schedule_delayed_work(&msc->work, msecs_to_jiffies(10)); + return 1; + } + + if (data[0] != SPI_REPORT_ID || size < hdr_sz) + return 0; + + return magicmouse_raw_event_mtp(hdev, report, data + hdr_sz, size - hdr_sz); +} + static int magicmouse_event(struct hid_device *hdev, struct hid_field *field, struct hid_usage *usage, __s32 value) { @@ -536,7 +839,17 @@ static int magicmouse_event(struct hid_device *hdev, struct hid_field *field, return 0; } -static int magicmouse_setup_input(struct input_dev *input, struct hid_device *hdev) + +static int magicmouse_setup_input(struct input_dev *input, + struct hid_device *hdev) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + + return msc->input_ops.setup_input(input, hdev); +} + +static int magicmouse_setup_input_usb(struct input_dev *input, + struct hid_device *hdev) { int error; int mt_flags = 0; @@ -615,7 +928,7 @@ static int magicmouse_setup_input(struct input_dev *input, struct hid_device *hd __set_bit(EV_ABS, input->evbit); - error = input_mt_init_slots(input, 16, mt_flags); + error = input_mt_init_slots(input, MAX_CONTACTS, mt_flags); if (error) return error; input_set_abs_params(input, ABS_MT_TOUCH_MAJOR, 0, 255 << 2, @@ -695,6 +1008,109 @@ static int magicmouse_setup_input(struct input_dev *input, struct hid_device *hd */ __clear_bit(EV_REP, input->evbit); + /* + * This isn't strictly speaking needed for USB, but enabling MT on + * device open is probably more robust than only doing it once on probe + * even if USB devices are not known to suffer from the SPI reset issue. + */ + input->open = magicmouse_open; + input->close = magicmouse_close; + return 0; +} + +static int magicmouse_setup_input_mtp(struct input_dev *input, + struct hid_device *hdev) +{ + int error; + int mt_flags = 0; + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + + __set_bit(INPUT_PROP_BUTTONPAD, input->propbit); + __clear_bit(BTN_0, input->keybit); + __clear_bit(BTN_RIGHT, input->keybit); + __clear_bit(BTN_MIDDLE, input->keybit); + __clear_bit(EV_REL, input->evbit); + __clear_bit(REL_X, input->relbit); + __clear_bit(REL_Y, input->relbit); + + mt_flags = INPUT_MT_POINTER | INPUT_MT_DROP_UNUSED | INPUT_MT_TRACK; + + /* finger touch area */ + input_set_abs_params(input, ABS_MT_TOUCH_MAJOR, 0, 5000, 0, 0); + input_set_abs_params(input, ABS_MT_TOUCH_MINOR, 0, 5000, 0, 0); + + /* finger approach area */ + input_set_abs_params(input, ABS_MT_WIDTH_MAJOR, 0, 5000, 0, 0); + input_set_abs_params(input, ABS_MT_WIDTH_MINOR, 0, 5000, 0, 0); + + /* Note: Touch Y position from the device is inverted relative + * to how pointer motion is reported (and relative to how USB + * HID recommends the coordinates work). This driver keeps + * the origin at the same position, and just uses the additive + * inverse of the reported Y. + */ + + input_set_abs_params(input, ABS_MT_PRESSURE, 0, 6000, 0, 0); + + /* + * This makes libinput recognize this as a PressurePad and + * stop trying to use pressure for touch size. Pressure unit + * seems to be ~grams on these touchpads. + */ + input_abs_set_res(input, ABS_MT_PRESSURE, 1); + + /* finger orientation */ + input_set_abs_params(input, ABS_MT_ORIENTATION, -J314_TP_MAX_FINGER_ORIENTATION, + J314_TP_MAX_FINGER_ORIENTATION, 0, 0); + + /* finger position */ + input_set_abs_params(input, ABS_MT_POSITION_X, J314_TP_MIN_X, J314_TP_MAX_X, + 0, 0); + /* Y axis is inverted */ + input_set_abs_params(input, ABS_MT_POSITION_Y, -J314_TP_MAX_Y, -J314_TP_MIN_Y, + 0, 0); + + /* X/Y resolution */ + input_abs_set_res(input, ABS_MT_POSITION_X, J314_TP_RES_X); + input_abs_set_res(input, ABS_MT_POSITION_Y, J314_TP_RES_Y); + + input_set_events_per_packet(input, 60); + + /* touchpad button */ + input_set_capability(input, EV_KEY, BTN_MOUSE); + + /* + * hid-input may mark device as using autorepeat, but the trackpad does + * not actually want it. + */ + __clear_bit(EV_REP, input->evbit); + + error = input_mt_init_slots(input, MAX_CONTACTS, mt_flags); + if (error) + return error; + + /* + * Override the default input->open function to send the MT + * enable every time the device is opened. This ensures it works + * even if we missed a reset event due to the device being closed. + * input->close is overridden for symmetry. + * + * This also takes care of the dimensions query. + */ + input->open = magicmouse_open; + input->close = magicmouse_close; + msc->query_dimensions = true; + + return 0; +} + +static int magicmouse_setup_input_spi(struct input_dev *input, + struct hid_device *hdev) +{ + int ret = magicmouse_setup_input_mtp(input, hdev); + if (ret) + return ret; + return 0; } @@ -748,6 +1164,10 @@ static int magicmouse_enable_multitouch(struct hid_device *hdev) int feature_size; switch (hdev->product) { + case SPI_DEVICE_ID_APPLE_MACBOOK_AIR_2020: + case SPI_DEVICE_ID_APPLE_MACBOOK_PRO13_2020: + case SPI_DEVICE_ID_APPLE_MACBOOK_PRO14_2021: + case SPI_DEVICE_ID_APPLE_MACBOOK_PRO16_2021: case USB_DEVICE_ID_APPLE_MAGICTRACKPAD2: case USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC: switch (hdev->vendor) { @@ -755,7 +1175,7 @@ static int magicmouse_enable_multitouch(struct hid_device *hdev) feature_size = sizeof(feature_mt_trackpad2_bt); feature = feature_mt_trackpad2_bt; break; - default: /* USB_VENDOR_ID_APPLE */ + default: /* USB_VENDOR_ID_APPLE || SPI_VENDOR_ID_APPLE */ feature_size = sizeof(feature_mt_trackpad2_usb); feature = feature_mt_trackpad2_usb; } @@ -852,12 +1272,29 @@ static int magicmouse_probe(struct hid_device *hdev, struct hid_report *report; int ret; + if ((id->bus == BUS_SPI || id->bus == BUS_HOST) && id->vendor == SPI_VENDOR_ID_APPLE && + hdev->type != HID_TYPE_SPI_MOUSE) + return -ENODEV; + msc = devm_kzalloc(&hdev->dev, sizeof(*msc), GFP_KERNEL); if (msc == NULL) { hid_err(hdev, "can't alloc magicmouse descriptor\n"); return -ENOMEM; } + // internal trackpad use a data format use input ops to avoid + // conflicts with the report ID. + if (id->bus == BUS_HOST) { + msc->input_ops.raw_event = magicmouse_raw_event_mtp; + msc->input_ops.setup_input = magicmouse_setup_input_mtp; + } else if (id->bus == BUS_SPI) { + msc->input_ops.raw_event = magicmouse_raw_event_spi; + msc->input_ops.setup_input = magicmouse_setup_input_spi; + } else { + msc->input_ops.raw_event = magicmouse_raw_event_usb; + msc->input_ops.setup_input = magicmouse_setup_input_usb; + } + msc->scroll_accel = SCROLL_ACCEL_DEFAULT; msc->hdev = hdev; INIT_DEFERRABLE_WORK(&msc->work, magicmouse_enable_mt_work); @@ -916,11 +1353,20 @@ static int magicmouse_probe(struct hid_device *hdev, TRACKPAD2_USB_REPORT_ID, 0); } break; - default: /* USB_DEVICE_ID_APPLE_MAGICTRACKPAD */ - report = hid_register_report(hdev, HID_INPUT_REPORT, - TRACKPAD_REPORT_ID, 0); - report = hid_register_report(hdev, HID_INPUT_REPORT, - DOUBLE_REPORT_ID, 0); + default: + switch (id->bus) { + case BUS_HOST: + report = hid_register_report(hdev, HID_INPUT_REPORT, MTP_REPORT_ID, 0); + break; + case BUS_SPI: + report = hid_register_report(hdev, HID_INPUT_REPORT, SPI_REPORT_ID, 0); + break; + default: /* USB_DEVICE_ID_APPLE_MAGICTRACKPAD */ + report = hid_register_report(hdev, HID_INPUT_REPORT, + TRACKPAD_REPORT_ID, 0); + report = hid_register_report(hdev, HID_INPUT_REPORT, + DOUBLE_REPORT_ID, 0); + } } if (!report) { @@ -930,22 +1376,14 @@ static int magicmouse_probe(struct hid_device *hdev, } report->size = 6; - /* - * Some devices repond with 'invalid report id' when feature - * report switching it into multitouch mode is sent to it. - * - * This results in -EIO from the _raw low-level transport callback, - * but there seems to be no other way of switching the mode. - * Thus the super-ugly hacky success check below. - */ - ret = magicmouse_enable_multitouch(hdev); - if (ret != -EIO && ret < 0) { - hid_err(hdev, "unable to request touch data (%d)\n", ret); - goto err_stop_hw; - } - if (ret == -EIO && (id->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2 || - id->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2_USBC)) { - schedule_delayed_work(&msc->work, msecs_to_jiffies(500)); + /* MTP devices do not need the MT enable, this is handled by the MTP driver */ + if (id->bus == BUS_HOST) + return 0; + + /* SPI devices need to watch for reset events to re-send the MT enable */ + if (id->bus == BUS_SPI) { + report = hid_register_report(hdev, HID_INPUT_REPORT, SPI_RESET_REPORT_ID, 0); + report->size = 2; } return 0; @@ -1023,10 +1461,24 @@ static const struct hid_device_id magic_mice[] = { USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC), .driver_data = 0 }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC), .driver_data = 0 }, + { HID_SPI_DEVICE(SPI_VENDOR_ID_APPLE, HID_ANY_ID), + .driver_data = 0 }, + { HID_DEVICE(BUS_HOST, HID_GROUP_ANY, HOST_VENDOR_ID_APPLE, + HID_ANY_ID), .driver_data = 0 }, { } }; MODULE_DEVICE_TABLE(hid, magic_mice); +#ifdef CONFIG_PM +static int magicmouse_reset_resume(struct hid_device *hdev) +{ + if (hdev->bus == BUS_SPI) + return magicmouse_enable_multitouch(hdev); + + return 0; +} +#endif + static struct hid_driver magicmouse_driver = { .name = "magicmouse", .id_table = magic_mice, @@ -1037,6 +1489,10 @@ static struct hid_driver magicmouse_driver = { .event = magicmouse_event, .input_mapping = magicmouse_input_mapping, .input_configured = magicmouse_input_configured, +#ifdef CONFIG_PM + .reset_resume = magicmouse_reset_resume, +#endif + }; module_hid_driver(magicmouse_driver); diff --git a/drivers/hid/hid-playstation.c b/drivers/hid/hid-playstation.c index e4dfcf26b04e74..2ec6d4445e84ba 100644 --- a/drivers/hid/hid-playstation.c +++ b/drivers/hid/hid-playstation.c @@ -774,7 +774,9 @@ ps_gamepad_create(struct hid_device *hdev, #if IS_ENABLED(CONFIG_PLAYSTATION_FF) if (play_effect) { input_set_capability(gamepad, EV_FF, FF_RUMBLE); - input_ff_create_memless(gamepad, NULL, play_effect); + ret = input_ff_create_memless(gamepad, NULL, play_effect); + if (ret) + return ERR_PTR(ret); } #endif diff --git a/drivers/hid/intel-ish-hid/ishtp/bus.c b/drivers/hid/intel-ish-hid/ishtp/bus.c index c3915f3a060ead..b890fbf97a75cd 100644 --- a/drivers/hid/intel-ish-hid/ishtp/bus.c +++ b/drivers/hid/intel-ish-hid/ishtp/bus.c @@ -730,7 +730,7 @@ void ishtp_bus_remove_all_clients(struct ishtp_device *ishtp_dev, spin_lock_irqsave(&ishtp_dev->cl_list_lock, flags); list_for_each_entry(cl, &ishtp_dev->cl_list, link) { cl->state = ISHTP_CL_DISCONNECTED; - if (warm_reset && cl->device->reference_count) + if (warm_reset && cl->device && cl->device->reference_count) continue; /* diff --git a/drivers/hid/intel-thc-hid/intel-thc/intel-thc-dev.c b/drivers/hid/intel-thc-hid/intel-thc/intel-thc-dev.c index 7e220a4c5ded7b..d8e195189e4bfd 100644 --- a/drivers/hid/intel-thc-hid/intel-thc/intel-thc-dev.c +++ b/drivers/hid/intel-thc-hid/intel-thc/intel-thc-dev.c @@ -1597,6 +1597,7 @@ int thc_i2c_set_rx_max_size(struct thc_device *dev, u32 max_rx_size) if (ret) return ret; + val = val & ~THC_M_PRT_SPI_ICRRD_OPCODE_I2C_MAX_SIZE; val |= FIELD_PREP(THC_M_PRT_SPI_ICRRD_OPCODE_I2C_MAX_SIZE, max_rx_size); ret = regmap_write(dev->thc_regmap, THC_M_PRT_SPI_ICRRD_OPCODE_OFFSET, val); @@ -1667,6 +1668,7 @@ int thc_i2c_set_rx_int_delay(struct thc_device *dev, u32 delay_us) return ret; /* THC hardware counts at 10us unit */ + val = val & ~THC_M_PRT_SPI_ICRRD_OPCODE_I2C_INTERVAL; val |= FIELD_PREP(THC_M_PRT_SPI_ICRRD_OPCODE_I2C_INTERVAL, DIV_ROUND_UP(delay_us, 10)); ret = regmap_write(dev->thc_regmap, THC_M_PRT_SPI_ICRRD_OPCODE_OFFSET, val); diff --git a/drivers/hid/spi-hid/Kconfig b/drivers/hid/spi-hid/Kconfig new file mode 100644 index 00000000000000..59076c6ebeed9b --- /dev/null +++ b/drivers/hid/spi-hid/Kconfig @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: GPL-2.0-only +menu "SPI HID support" + depends on SPI + +config SPI_HID_APPLE_OF + tristate "HID over SPI transport layer for Apple Silicon SoCs" + depends on INPUT && OF + select SPI_HID_APPLE_CORE + help + Say Y here if you use Apple Silicon based laptop. The keyboard and + touchpad are HID based devices connected via SPI. + + If unsure, say N. + + This support is also available as a module. If so, the module + will be called spi-hid-apple-of. It will also build/depend on the + module spi-hid-apple. + +endmenu + +config SPI_HID_APPLE_CORE + tristate + select HID + select CRC16 diff --git a/drivers/hid/spi-hid/Makefile b/drivers/hid/spi-hid/Makefile new file mode 100644 index 00000000000000..f276ee12cb94fc --- /dev/null +++ b/drivers/hid/spi-hid/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for SPI HID tarnsport drivers +# + +obj-$(CONFIG_SPI_HID_APPLE_CORE) += spi-hid-apple.o + +spi-hid-apple-objs = spi-hid-apple-core.o + +obj-$(CONFIG_SPI_HID_APPLE_OF) += spi-hid-apple-of.o diff --git a/drivers/hid/spi-hid/spi-hid-apple-core.c b/drivers/hid/spi-hid/spi-hid-apple-core.c new file mode 100644 index 00000000000000..2ed909895391c8 --- /dev/null +++ b/drivers/hid/spi-hid/spi-hid-apple-core.c @@ -0,0 +1,1194 @@ +/* + * SPDX-License-Identifier: GPL-2.0 + * + * Apple SPI HID transport driver + * + * Copyright (C) The Asahi Linux Contributors + * + * Based on: drivers/input/applespi.c + * + * MacBook (Pro) SPI keyboard and touchpad driver + * + * Copyright (c) 2015-2018 Federico Lorenzi + * Copyright (c) 2017-2018 Ronald Tschalär + * + */ + +//#define DEBUG 2 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "spi-hid-apple.h" + +#define SPIHID_DEF_WAIT msecs_to_jiffies(1000) + +#define SPIHID_MAX_INPUT_REPORT_SIZE 0x800 + +/* support only keyboard, trackpad and management dev for now */ +#define SPIHID_MAX_DEVICES 3 + +#define SPIHID_DEVICE_ID_MNGT 0x0 +#define SPIHID_DEVICE_ID_KBD 0x1 +#define SPIHID_DEVICE_ID_TP 0x2 +#define SPIHID_DEVICE_ID_INFO 0xd0 + +#define SPIHID_READ_PACKET 0x20 +#define SPIHID_WRITE_PACKET 0x40 + +#define SPIHID_DESC_MAX 512 + +#define SPIHID_SET_LEDS 0x0151 /* caps lock */ + +#define SPI_RW_CHG_DELAY_US 200 /* 'Inter Stage Us'? */ + +static const u8 spi_hid_apple_booted[4] = { 0xa0, 0x80, 0x00, 0x00 }; +static const u8 spi_hid_apple_status_ok[4] = { 0xac, 0x27, 0x68, 0xd5 }; + +struct spihid_interface { + struct hid_device *hid; + u8 *hid_desc; + u32 hid_desc_len; + u32 id; + unsigned country; + u32 max_control_report_len; + u32 max_input_report_len; + u32 max_output_report_len; + u8 name[32]; + u8 reply_buf[SPIHID_DESC_MAX]; + u32 reply_len; + bool ready; +}; + +struct spihid_input_report { + u8 *buf; + u32 length; + u32 offset; + u8 device; + u8 flags; +}; + +struct spihid_apple { + struct spi_device *spidev; + + struct spihid_apple_ops *ops; + + struct spihid_interface mngt; + struct spihid_interface kbd; + struct spihid_interface tp; + + wait_queue_head_t wait; + struct mutex tx_lock; //< protects against concurrent SPI writes + + struct spi_message rx_msg; + struct spi_message tx_msg; + struct spi_transfer rx_transfer; + struct spi_transfer tx_transfer; + struct spi_transfer status_transfer; + + u8 *rx_buf; + u8 *tx_buf; + u8 *status_buf; + + u8 vendor[32]; + u8 product[64]; + u8 serial[32]; + + u32 num_devices; + + u32 vendor_id; + u32 product_id; + u32 version_number; + + u8 msg_id; + + /* fragmented HID report */ + struct spihid_input_report report; + + /* state tracking flags */ + bool status_booted; + +#ifdef IRQ_WAKE_SUPPORT + bool irq_wake_enabled; +#endif +}; + +/** + * struct spihid_msg_hdr - common header of protocol messages. + * + * Each message begins with fixed header, followed by a message-type specific + * payload, and ends with a 16-bit crc. Because of the varying lengths of the + * payload, the crc is defined at the end of each payload struct, rather than + * in this struct. + * + * @unknown0: request type? output, input (0x10), feature, protocol + * @unknown1: maybe report id? + * @unknown2: mostly zero, in info request maybe device num + * @id: incremented on each message, rolls over after 255; there is a + * separate counter for each message type. + * @rsplen: response length (the exact nature of this field is quite + * speculative). On a request/write this is often the same as + * @length, though in some cases it has been seen to be much larger + * (e.g. 0x400); on a response/read this the same as on the + * request; for reads that are not responses it is 0. + * @length: length of the remainder of the data in the whole message + * structure (after re-assembly in case of being split over + * multiple spi-packets), minus the trailing crc. The total size + * of a message is therefore @length + 10. + */ + +struct spihid_msg_hdr { + u8 unknown0; + u8 unknown1; + u8 unknown2; + u8 id; + __le16 rsplen; + __le16 length; +}; + +/** + * struct spihid_transfer_packet - a complete spi packet; always 256 bytes. This carries + * the (parts of the) message in the data. But note that this does not + * necessarily contain a complete message, as in some cases (e.g. many + * fingers pressed) the message is split over multiple packets (see the + * @offset, @remain, and @length fields). In general the data parts in + * spihid_transfer_packet's are concatenated until @remaining is 0, and the + * result is an message. + * + * @flags: 0x40 = write (to device), 0x20 = read (from device); note that + * the response to a write still has 0x40. + * @device: 1 = keyboard, 2 = touchpad + * @offset: specifies the offset of this packet's data in the complete + * message; i.e. > 0 indicates this is a continuation packet (in + * the second packet for a message split over multiple packets + * this would then be the same as the @length in the first packet) + * @remain: number of message bytes remaining in subsequents packets (in + * the first packet of a message split over two packets this would + * then be the same as the @length in the second packet) + * @length: length of the valid data in the @data in this packet + * @data: all or part of a message + * @crc16: crc over this whole structure minus this @crc16 field. This + * covers just this packet, even on multi-packet messages (in + * contrast to the crc in the message). + */ +struct spihid_transfer_packet { + u8 flags; + u8 device; + __le16 offset; + __le16 remain; + __le16 length; + u8 data[246]; + __le16 crc16; +}; + +/* + * how HID is mapped onto the protocol is not fully clear. This are the known + * reports/request: + * + * pkt.flags pkt.dev? msg.u0 msg.u1 msg.u2 + * info 0x40 0xd0 0x20 0x01 0xd0 + * + * info mngt: 0x40 0xd0 0x20 0x10 0x00 + * info kbd: 0x40 0xd0 0x20 0x10 0x01 + * info tp: 0x40 0xd0 0x20 0x10 0x02 + * + * desc kbd: 0x40 0xd0 0x20 0x10 0x01 + * desc trackpad: 0x40 0xd0 0x20 0x10 0x02 + * + * mt mode: 0x40 0x02 0x52 0x02 0x00 set protocol? + * capslock led 0x40 0x01 0x51 0x01 0x00 output report + * + * report kbd: 0x20 0x01 0x10 0x01 0x00 input report + * report tp: 0x20 0x02 0x10 0x02 0x00 input report + * + */ + + +static int spihid_apple_request(struct spihid_apple *spihid, u8 target, u8 unk0, + u8 unk1, u8 unk2, u16 resp_len, u8 *buf, + size_t len) +{ + struct spihid_transfer_packet *pkt; + struct spihid_msg_hdr *hdr; + u16 crc; + int err; + + /* know reports are small enoug to fit in a single packet */ + if (len > sizeof(pkt->data) - sizeof(*hdr) - sizeof(__le16)) + return -EINVAL; + + err = mutex_lock_interruptible(&spihid->tx_lock); + if (err < 0) + return err; + + pkt = (struct spihid_transfer_packet *)spihid->tx_buf; + + memset(pkt, 0, sizeof(*pkt)); + pkt->flags = SPIHID_WRITE_PACKET; + pkt->device = target; + pkt->length = cpu_to_le16(sizeof(*hdr) + len + sizeof(__le16)); + + hdr = (struct spihid_msg_hdr *)&pkt->data[0]; + hdr->unknown0 = unk0; + hdr->unknown1 = unk1; + hdr->unknown2 = unk2; + hdr->id = spihid->msg_id++; + hdr->rsplen = cpu_to_le16(resp_len); + hdr->length = cpu_to_le16(len); + + if (len) + memcpy(pkt->data + sizeof(*hdr), buf, len); + crc = crc16(0, &pkt->data[0], sizeof(*hdr) + len); + put_unaligned_le16(crc, pkt->data + sizeof(*hdr) + len); + + pkt->crc16 = cpu_to_le16(crc16(0, spihid->tx_buf, + offsetof(struct spihid_transfer_packet, crc16))); + + memset(spihid->status_buf, 0, sizeof(spi_hid_apple_status_ok)); + + err = spi_sync(spihid->spidev, &spihid->tx_msg); + + if (memcmp(spihid->status_buf, spi_hid_apple_status_ok, + sizeof(spi_hid_apple_status_ok))) { + u8 *b = spihid->status_buf; + dev_warn_ratelimited(&spihid->spidev->dev, "status message " + "mismatch: %02x %02x %02x %02x\n", + b[0], b[1], b[2], b[3]); + } + mutex_unlock(&spihid->tx_lock); + if (err < 0) + return err; + + return (int)len; +} + +static struct spihid_apple *spihid_get_data(struct spihid_interface *idev) +{ + switch (idev->id) { + case SPIHID_DEVICE_ID_KBD: + return container_of(idev, struct spihid_apple, kbd); + case SPIHID_DEVICE_ID_TP: + return container_of(idev, struct spihid_apple, tp); + default: + return NULL; + } +} + +static int apple_ll_start(struct hid_device *hdev) +{ + /* no-op SPI transport is already setup */ + return 0; +}; + +static void apple_ll_stop(struct hid_device *hdev) +{ + /* no-op, devices will be desstroyed on driver destruction */ +} + +static int apple_ll_open(struct hid_device *hdev) +{ + struct spihid_apple *spihid; + struct spihid_interface *idev = hdev->driver_data; + + if (idev->hid_desc_len == 0) { + spihid = spihid_get_data(idev); + dev_warn(&spihid->spidev->dev, + "HID descriptor missing for dev %u", idev->id); + } else + idev->ready = true; + + return 0; +} + +static void apple_ll_close(struct hid_device *hdev) +{ + struct spihid_interface *idev = hdev->driver_data; + idev->ready = false; +} + +static int apple_ll_parse(struct hid_device *hdev) +{ + struct spihid_interface *idev = hdev->driver_data; + + return hid_parse_report(hdev, idev->hid_desc, idev->hid_desc_len); +} + +static int apple_ll_raw_request(struct hid_device *hdev, + unsigned char reportnum, __u8 *buf, size_t len, + unsigned char rtype, int reqtype) +{ + struct spihid_interface *idev = hdev->driver_data; + struct spihid_apple *spihid = spihid_get_data(idev); + int ret; + + dev_dbg(&spihid->spidev->dev, + "apple_ll_raw_request: device:%u reportnum:%hhu rtype:%hhu", + idev->id, reportnum, rtype); + + switch (reqtype) { + case HID_REQ_GET_REPORT: + if (rtype != HID_FEATURE_REPORT) + return -EINVAL; + + idev->reply_len = 0; + ret = spihid_apple_request(spihid, idev->id, 0x32, reportnum, 0x00, len, NULL, 0); + if (ret < 0) + return ret; + + ret = wait_event_interruptible_timeout(spihid->wait, idev->reply_len, + SPIHID_DEF_WAIT); + if (ret == 0) + ret = -ETIMEDOUT; + if (ret < 0) { + dev_err(&spihid->spidev->dev, "waiting for get report failed: %d", ret); + return ret; + } + memcpy(buf, idev->reply_buf, max_t(size_t, len, idev->reply_len)); + return idev->reply_len; + + case HID_REQ_SET_REPORT: + if (buf[0] != reportnum) + return -EINVAL; + if (reportnum != idev->id) { + dev_warn(&spihid->spidev->dev, + "device:%u reportnum:" + "%hhu mismatch", + idev->id, reportnum); + return -EINVAL; + } + return spihid_apple_request(spihid, idev->id, 0x52, reportnum, 0x00, 2, buf, len); + default: + return -EIO; + } +} + +static int apple_ll_output_report(struct hid_device *hdev, __u8 *buf, + size_t len) +{ + struct spihid_interface *idev = hdev->driver_data; + struct spihid_apple *spihid = spihid_get_data(idev); + if (!spihid) + return -1; + + dev_dbg(&spihid->spidev->dev, + "apple_ll_output_report: device:%u len:%zu:", + idev->id, len); + // second idev->id should maybe be buf[0]? + return spihid_apple_request(spihid, idev->id, 0x51, idev->id, 0x00, 0, buf, len); +} + +static struct hid_ll_driver apple_hid_ll = { + .start = &apple_ll_start, + .stop = &apple_ll_stop, + .open = &apple_ll_open, + .close = &apple_ll_close, + .parse = &apple_ll_parse, + .raw_request = &apple_ll_raw_request, + .output_report = &apple_ll_output_report, + .max_buffer_size = SPIHID_MAX_INPUT_REPORT_SIZE, +}; + +static struct spihid_interface *spihid_get_iface(struct spihid_apple *spihid, + u32 iface) +{ + switch (iface) { + case SPIHID_DEVICE_ID_MNGT: + return &spihid->mngt; + case SPIHID_DEVICE_ID_KBD: + return &spihid->kbd; + case SPIHID_DEVICE_ID_TP: + return &spihid->tp; + default: + return NULL; + } +} + +static int spihid_verify_msg(struct spihid_apple *spihid, u8 *buf, size_t len) +{ + u16 msg_crc, crc; + struct device *dev = &spihid->spidev->dev; + + crc = crc16(0, buf, len - sizeof(__le16)); + msg_crc = get_unaligned_le16(buf + len - sizeof(__le16)); + if (crc != msg_crc) { + dev_warn_ratelimited(dev, "Read message crc mismatch\n"); + return 0; + } + return 1; +} + +static bool spihid_status_report(struct spihid_apple *spihid, u8 *pl, + size_t len) +{ + struct device *dev = &spihid->spidev->dev; + dev_dbg(dev, "%s: len: %zu", __func__, len); + if (len == 5 && pl[0] == 0xe0) + return true; + + return false; +} + +static bool spihid_process_input_report(struct spihid_apple *spihid, u32 device, + struct spihid_msg_hdr *hdr, u8 *payload, + size_t len) +{ + //dev_dbg(&spihid>spidev->dev, "input report: req:%hx iface:%u ", hdr->unknown0, device); + if (hdr->unknown0 != 0x10) + return false; + + /* HID device as well but Vendor usage only, handle it internally for now */ + if (device == 0) { + if (hdr->unknown1 == 0xe0) { + return spihid_status_report(spihid, payload, len); + } + } else if (device < SPIHID_MAX_DEVICES) { + struct spihid_interface *iface = + spihid_get_iface(spihid, device); + if (iface && iface->hid && iface->ready) { + hid_input_report(iface->hid, HID_INPUT_REPORT, payload, + len, 1); + return true; + } + } else + dev_dbg(&spihid->spidev->dev, + "unexpected iface:%u for input report", device); + + return false; +} + +struct spihid_device_info { + __le16 u0[2]; + __le16 num_devices; + __le16 vendor_id; + __le16 product_id; + __le16 version_number; + __le16 vendor_str[2]; //< offset and string length + __le16 product_str[2]; //< offset and string length + __le16 serial_str[2]; //< offset and string length +}; + +static bool spihid_process_device_info(struct spihid_apple *spihid, u32 iface, + u8 *payload, size_t len) +{ + struct device *dev = &spihid->spidev->dev; + + if (iface != SPIHID_DEVICE_ID_INFO) + return false; + + if (spihid->vendor_id == 0 && + len >= sizeof(struct spihid_device_info)) { + struct spihid_device_info *info = + (struct spihid_device_info *)payload; + u16 voff, vlen, poff, plen, soff, slen; + u32 num_devices; + + num_devices = __le16_to_cpu(info->num_devices); + + if (num_devices < SPIHID_MAX_DEVICES) { + dev_err(dev, + "Device info reports %u devices, expecting at least 3", + num_devices); + return false; + } + spihid->num_devices = num_devices; + + if (spihid->num_devices > SPIHID_MAX_DEVICES) { + dev_info( + dev, + "limiting the number of devices to mngt, kbd and mouse"); + spihid->num_devices = SPIHID_MAX_DEVICES; + } + + spihid->vendor_id = __le16_to_cpu(info->vendor_id); + spihid->product_id = __le16_to_cpu(info->product_id); + spihid->version_number = __le16_to_cpu(info->version_number); + + voff = __le16_to_cpu(info->vendor_str[0]); + vlen = __le16_to_cpu(info->vendor_str[1]); + + if (voff < len && vlen <= len - voff && + vlen < sizeof(spihid->vendor)) { + memcpy(spihid->vendor, payload + voff, vlen); + spihid->vendor[vlen] = '\0'; + } + + poff = __le16_to_cpu(info->product_str[0]); + plen = __le16_to_cpu(info->product_str[1]); + + if (poff < len && plen <= len - poff && + plen < sizeof(spihid->product)) { + memcpy(spihid->product, payload + poff, plen); + spihid->product[plen] = '\0'; + } + + soff = __le16_to_cpu(info->serial_str[0]); + slen = __le16_to_cpu(info->serial_str[1]); + + if (soff < len && slen <= len - soff && + slen < sizeof(spihid->serial)) { + memcpy(spihid->vendor, payload + soff, slen); + spihid->serial[slen] = '\0'; + } + + wake_up_interruptible(&spihid->wait); + } + return true; +} + +struct spihid_iface_info { + u8 u_0; + u8 interface_num; + u8 u_2; + u8 u_3; + u8 u_4; + u8 country_code; + __le16 max_input_report_len; + __le16 max_output_report_len; + __le16 max_control_report_len; + __le16 name_offset; + __le16 name_length; +}; + +static bool spihid_process_iface_info(struct spihid_apple *spihid, u32 num, + u8 *payload, size_t len) +{ + struct spihid_iface_info *info; + struct spihid_interface *iface = spihid_get_iface(spihid, num); + u32 name_off, name_len; + + if (!iface) + return false; + + if (!iface->max_input_report_len) { + if (len < sizeof(*info)) + return false; + + info = (struct spihid_iface_info *)payload; + + iface->max_input_report_len = + le16_to_cpu(info->max_input_report_len); + iface->max_output_report_len = + le16_to_cpu(info->max_output_report_len); + iface->max_control_report_len = + le16_to_cpu(info->max_control_report_len); + iface->country = info->country_code; + + name_off = le16_to_cpu(info->name_offset); + name_len = le16_to_cpu(info->name_length); + + if (name_off < len && name_len <= len - name_off && + name_len < sizeof(iface->name)) { + memcpy(iface->name, payload + name_off, name_len); + iface->name[name_len] = '\0'; + } + + dev_dbg(&spihid->spidev->dev, "Info for %s, country code: 0x%x", + iface->name, iface->country); + + wake_up_interruptible(&spihid->wait); + } + + return true; +} + +static int spihid_register_hid_device(struct spihid_apple *spihid, + struct spihid_interface *idev, u8 device); + +static bool spihid_process_iface_hid_report_desc(struct spihid_apple *spihid, + u32 num, u8 *payload, + size_t len) +{ + struct spihid_interface *iface = spihid_get_iface(spihid, num); + + if (!iface) + return false; + + if (iface->hid_desc_len == 0) { + if (len > SPIHID_DESC_MAX) + return false; + memcpy(iface->hid_desc, payload, len); + iface->hid_desc_len = len; + + /* do not register the mngt iface as HID device */ + if (num > 0) + spihid_register_hid_device(spihid, iface, num); + + wake_up_interruptible(&spihid->wait); + } + return true; +} + +static bool spihid_process_iface_get_report(struct spihid_apple *spihid, + u32 device, u8 report, + u8 *payload, size_t len) +{ + struct spihid_interface *iface = spihid_get_iface(spihid, device); + + if (!iface) + return false; + + if (len > sizeof(iface->reply_buf) || len < 1) + return false; + + memcpy(iface->reply_buf, payload, len); + iface->reply_len = len; + + wake_up_interruptible(&spihid->wait); + + return true; +} + +static bool spihid_process_response(struct spihid_apple *spihid, u32 device, + struct spihid_msg_hdr *hdr, u8 *payload, + size_t len) +{ + if (hdr->unknown0 == 0x20) { + switch (hdr->unknown1) { + case 0x01: + return spihid_process_device_info(spihid, hdr->unknown2, + payload, len); + case 0x02: + return spihid_process_iface_info(spihid, hdr->unknown2, + payload, len); + case 0x10: + return spihid_process_iface_hid_report_desc( + spihid, hdr->unknown2, payload, len); + default: + break; + } + } + + if (hdr->unknown0 == 0x32) { + return spihid_process_iface_get_report(spihid, device, hdr->unknown1, payload, len); + } + + return false; +} + +static void spihid_process_message(struct spihid_apple *spihid, u8 *data, + size_t length, u8 device, u8 flags) +{ + struct device *dev = &spihid->spidev->dev; + struct spihid_msg_hdr *hdr; + bool handled = false; + size_t payload_len; + u8 *payload; + + if (!spihid_verify_msg(spihid, data, length)) + return; + + hdr = (struct spihid_msg_hdr *)data; + payload_len = le16_to_cpu(hdr->length); + + if (payload_len == 0 || + (payload_len + sizeof(struct spihid_msg_hdr) + 2) > length) + return; + + payload = data + sizeof(struct spihid_msg_hdr); + + switch (flags) { + case SPIHID_READ_PACKET: + handled = spihid_process_input_report(spihid, device, hdr, + payload, payload_len); + break; + case SPIHID_WRITE_PACKET: + handled = spihid_process_response(spihid, device, hdr, payload, + payload_len); + break; + default: + break; + } + +#if defined(DEBUG) && DEBUG > 1 + { + dev_dbg(dev, + "R msg: req:%02hhx rep:%02hhx dev:%02hhx id:%hu len:%hu\n", + hdr->unknown0, hdr->unknown1, hdr->unknown2, hdr->id, + hdr->length); + print_hex_dump_debug("spihid msg: ", DUMP_PREFIX_OFFSET, 16, 1, + payload, le16_to_cpu(hdr->length), true); + } +#else + if (!handled) { + dev_dbg(dev, + "R unhandled msg: req:%02hhx rep:%02hhx dev:%02hhx id:%hu len:%hu\n", + hdr->unknown0, hdr->unknown1, hdr->unknown2, hdr->id, + hdr->length); + print_hex_dump_debug("spihid msg: ", DUMP_PREFIX_OFFSET, 16, 1, + payload, le16_to_cpu(hdr->length), true); + } +#endif +} + +static void spihid_assemble_message(struct spihid_apple *spihid, + struct spihid_transfer_packet *pkt) +{ + size_t length, offset, remain; + struct device *dev = &spihid->spidev->dev; + struct spihid_input_report *rep = &spihid->report; + + length = le16_to_cpu(pkt->length); + remain = le16_to_cpu(pkt->remain); + offset = le16_to_cpu(pkt->offset); + + if (offset + length + remain > U16_MAX) { + return; + } + + if (pkt->device != rep->device || pkt->flags != rep->flags || + offset != rep->offset) { + rep->device = 0; + rep->flags = 0; + rep->offset = 0; + rep->length = 0; + } + + if (offset == 0) { + if (rep->offset != 0) { + dev_warn(dev, "incomplete report off:%u len:%u", + rep->offset, rep->length); + } + memcpy(rep->buf, pkt->data, length); + rep->offset = length; + rep->length = length + remain; + rep->device = pkt->device; + rep->flags = pkt->flags; + } else if (offset == rep->offset) { + if (offset + length + remain != rep->length) { + dev_warn(dev, "incomplete report off:%u len:%u", + rep->offset, rep->length); + return; + } + memcpy(rep->buf + offset, pkt->data, length); + rep->offset += length; + + if (rep->offset == rep->length) { + spihid_process_message(spihid, rep->buf, rep->length, + rep->device, rep->flags); + rep->device = 0; + rep->flags = 0; + rep->offset = 0; + rep->length = 0; + } + } +} + +static void spihid_process_read(struct spihid_apple *spihid) +{ + u16 crc; + size_t length; + struct device *dev = &spihid->spidev->dev; + struct spihid_transfer_packet *pkt; + + pkt = (struct spihid_transfer_packet *)spihid->rx_buf; + + /* check transfer packet crc */ + crc = crc16(0, spihid->rx_buf, + offsetof(struct spihid_transfer_packet, crc16)); + if (crc != le16_to_cpu(pkt->crc16)) { + dev_warn_ratelimited(dev, "Read package crc mismatch\n"); + return; + } + + length = le16_to_cpu(pkt->length); + + if (length < sizeof(struct spihid_msg_hdr) + 2) { + if (length == sizeof(spi_hid_apple_booted) && + !memcmp(pkt->data, spi_hid_apple_booted, length)) { + if (!spihid->status_booted) { + spihid->status_booted = true; + wake_up_interruptible(&spihid->wait); + } + } else { + dev_info(dev, "R short packet: len:%zu\n", length); + print_hex_dump(KERN_INFO, "spihid pkt:", + DUMP_PREFIX_OFFSET, 16, 1, pkt->data, + length, false); + } + return; + } + +#if defined(DEBUG) && DEBUG > 1 + dev_dbg(dev, + "R pkt: flags:%02hhx dev:%02hhx off:%hu remain:%hu, len:%zu\n", + pkt->flags, pkt->device, pkt->offset, pkt->remain, length); +#if defined(DEBUG) && DEBUG > 2 + print_hex_dump_debug("spihid pkt: ", DUMP_PREFIX_OFFSET, 16, 1, + spihid->rx_buf, + sizeof(struct spihid_transfer_packet), true); +#endif +#endif + + if (length > sizeof(pkt->data)) { + dev_warn_ratelimited(dev, "Invalid pkt len:%zu", length); + return; + } + + /* short message */ + if (pkt->offset == 0 && pkt->remain == 0) { + spihid_process_message(spihid, pkt->data, length, pkt->device, + pkt->flags); + } else { + spihid_assemble_message(spihid, pkt); + } +} + +static void spihid_read_packet_sync(struct spihid_apple *spihid) +{ + int err; + + err = spi_sync(spihid->spidev, &spihid->rx_msg); + if (!err) { + spihid_process_read(spihid); + } else { + dev_warn(&spihid->spidev->dev, "RX failed: %d\n", err); + } +} + +irqreturn_t spihid_apple_core_irq(int irq, void *data) +{ + struct spi_device *spi = data; + struct spihid_apple *spihid = spi_get_drvdata(spi); + + spihid_read_packet_sync(spihid); + + return IRQ_HANDLED; +} +EXPORT_SYMBOL_GPL(spihid_apple_core_irq); + +static void spihid_apple_setup_spi_msgs(struct spihid_apple *spihid) +{ + memset(&spihid->rx_transfer, 0, sizeof(spihid->rx_transfer)); + + spihid->rx_transfer.rx_buf = spihid->rx_buf; + spihid->rx_transfer.len = sizeof(struct spihid_transfer_packet); + + spi_message_init(&spihid->rx_msg); + spi_message_add_tail(&spihid->rx_transfer, &spihid->rx_msg); + + memset(&spihid->tx_transfer, 0, sizeof(spihid->rx_transfer)); + memset(&spihid->status_transfer, 0, sizeof(spihid->status_transfer)); + + spihid->tx_transfer.tx_buf = spihid->tx_buf; + spihid->tx_transfer.len = sizeof(struct spihid_transfer_packet); + spihid->tx_transfer.delay.unit = SPI_DELAY_UNIT_USECS; + spihid->tx_transfer.delay.value = SPI_RW_CHG_DELAY_US; + + spihid->status_transfer.rx_buf = spihid->status_buf; + spihid->status_transfer.len = sizeof(spi_hid_apple_status_ok); + + spi_message_init(&spihid->tx_msg); + spi_message_add_tail(&spihid->tx_transfer, &spihid->tx_msg); + spi_message_add_tail(&spihid->status_transfer, &spihid->tx_msg); +} + +static int spihid_apple_setup_spi(struct spihid_apple *spihid) +{ + spihid_apple_setup_spi_msgs(spihid); + + return spihid->ops->power_on(spihid->ops); +} + +static int spihid_register_hid_device(struct spihid_apple *spihid, + struct spihid_interface *iface, u8 device) +{ + int ret; + char *suffix; + struct hid_device *hid; + + iface->id = device; + + hid = hid_allocate_device(); + if (IS_ERR(hid)) + return PTR_ERR(hid); + + /* + * Use 'Apple SPI Keyboard' and 'Apple SPI Trackpad' as input device + * names. The device names need to be distinct since at least Kwin uses + * the tripple Vendor ID, Product ID, Name to identify devices. + */ + snprintf(hid->name, sizeof(hid->name), "Apple SPI %s", iface->name); + // strip ' / Boot' suffix from the name + suffix = strstr(hid->name, " / Boot"); + if (suffix) + suffix[0] = '\0'; + snprintf(hid->phys, sizeof(hid->phys), "%s (%hhx)", + dev_name(&spihid->spidev->dev), device); + strscpy(hid->uniq, spihid->serial, sizeof(hid->uniq)); + + hid->ll_driver = &apple_hid_ll; + hid->bus = BUS_SPI; + hid->vendor = spihid->vendor_id; + hid->product = spihid->product_id; + hid->version = spihid->version_number; + + if (device == SPIHID_DEVICE_ID_KBD) + hid->type = HID_TYPE_SPI_KEYBOARD; + else if (device == SPIHID_DEVICE_ID_TP) + hid->type = HID_TYPE_SPI_MOUSE; + + hid->country = iface->country; + hid->dev.parent = &spihid->spidev->dev; + hid->driver_data = iface; + + ret = hid_add_device(hid); + if (ret < 0) { + hid_destroy_device(hid); + dev_warn(&spihid->spidev->dev, + "Failed to register hid device %hhu", device); + return ret; + } + + iface->hid = hid; + + return 0; +} + +static void spihid_destroy_hid_device(struct spihid_interface *iface) +{ + if (iface->hid) { + hid_destroy_device(iface->hid); + iface->hid = NULL; + } + iface->ready = false; +} + +int spihid_apple_core_probe(struct spi_device *spi, struct spihid_apple_ops *ops) +{ + struct device *dev = &spi->dev; + struct spihid_apple *spihid; + int err, i; + + if (!ops || !ops->power_on || !ops->power_off || !ops->enable_irq || !ops->disable_irq) + return -EINVAL; + + spihid = devm_kzalloc(dev, sizeof(*spihid), GFP_KERNEL); + if (!spihid) + return -ENOMEM; + + spihid->ops = ops; + spihid->spidev = spi; + + // init spi + spi_set_drvdata(spi, spihid); + + /* + * allocate SPI buffers + * Overallocate the receice buffer since it passed directly into + * hid_input_report / hid_report_raw_event. The later expects the buffer + * to be HID_MAX_BUFFER_SIZE (16k) or hid_ll_driver.max_buffer_size if + * set. + */ + spihid->rx_buf = devm_kmalloc( + &spi->dev, SPIHID_MAX_INPUT_REPORT_SIZE, GFP_KERNEL); + spihid->tx_buf = devm_kmalloc( + &spi->dev, sizeof(struct spihid_transfer_packet), GFP_KERNEL); + spihid->status_buf = devm_kmalloc( + &spi->dev, sizeof(spi_hid_apple_status_ok), GFP_KERNEL); + + if (!spihid->rx_buf || !spihid->tx_buf || !spihid->status_buf) + return -ENOMEM; + + spihid->report.buf = + devm_kmalloc(dev, SPIHID_MAX_INPUT_REPORT_SIZE, GFP_KERNEL); + + spihid->kbd.hid_desc = devm_kmalloc(dev, SPIHID_DESC_MAX, GFP_KERNEL); + spihid->tp.hid_desc = devm_kmalloc(dev, SPIHID_DESC_MAX, GFP_KERNEL); + + if (!spihid->report.buf || !spihid->kbd.hid_desc || + !spihid->tp.hid_desc) + return -ENOMEM; + + init_waitqueue_head(&spihid->wait); + + mutex_init(&spihid->tx_lock); + + /* Init spi transfer buffers and power device on */ + err = spihid_apple_setup_spi(spihid); + if (err < 0) + goto error; + + /* enable HID irq */ + spihid->ops->enable_irq(spihid->ops); + + // wait for boot message + err = wait_event_interruptible_timeout(spihid->wait, + spihid->status_booted, + msecs_to_jiffies(1000)); + if (err == 0) + err = -ENODEV; + if (err < 0) { + dev_err(dev, "waiting for device boot failed: %d", err); + goto error; + } + + /* request device information */ + dev_dbg(dev, "request device info"); + spihid_apple_request(spihid, 0xd0, 0x20, 0x01, 0xd0, 0, NULL, 0); + err = wait_event_interruptible_timeout(spihid->wait, spihid->vendor_id, + SPIHID_DEF_WAIT); + if (err == 0) + err = -ENODEV; + if (err < 0) { + dev_err(dev, "waiting for device info failed: %d", err); + goto error; + } + + /* request interface information */ + for (i = 0; i < spihid->num_devices; i++) { + struct spihid_interface *iface = spihid_get_iface(spihid, i); + if (!iface) + continue; + dev_dbg(dev, "request interface info 0x%02x", i); + spihid_apple_request(spihid, 0xd0, 0x20, 0x02, i, + SPIHID_DESC_MAX, NULL, 0); + err = wait_event_interruptible_timeout( + spihid->wait, iface->max_input_report_len, + SPIHID_DEF_WAIT); + } + + /* request HID report descriptors */ + for (i = 1; i < spihid->num_devices; i++) { + struct spihid_interface *iface = spihid_get_iface(spihid, i); + if (!iface) + continue; + dev_dbg(dev, "request hid report desc 0x%02x", i); + spihid_apple_request(spihid, 0xd0, 0x20, 0x10, i, + SPIHID_DESC_MAX, NULL, 0); + wait_event_interruptible_timeout( + spihid->wait, iface->hid_desc_len, SPIHID_DEF_WAIT); + } + + return 0; +error: + return err; +} +EXPORT_SYMBOL_GPL(spihid_apple_core_probe); + +void spihid_apple_core_remove(struct spi_device *spi) +{ + struct spihid_apple *spihid = spi_get_drvdata(spi); + + /* destroy input devices */ + + spihid_destroy_hid_device(&spihid->tp); + spihid_destroy_hid_device(&spihid->kbd); + + /* disable irq */ + spihid->ops->disable_irq(spihid->ops); + + /* power SPI device down */ + spihid->ops->power_off(spihid->ops); +} +EXPORT_SYMBOL_GPL(spihid_apple_core_remove); + +void spihid_apple_core_shutdown(struct spi_device *spi) +{ + struct spihid_apple *spihid = spi_get_drvdata(spi); + + /* disable irq */ + spihid->ops->disable_irq(spihid->ops); + + /* power SPI device down */ + spihid->ops->power_off(spihid->ops); +} +EXPORT_SYMBOL_GPL(spihid_apple_core_shutdown); + +#ifdef CONFIG_PM_SLEEP +static int spihid_apple_core_suspend(struct device *dev) +{ + int ret; +#ifdef IRQ_WAKE_SUPPORT + int wake_status; +#endif + struct spihid_apple *spihid = spi_get_drvdata(to_spi_device(dev)); + + if (spihid->tp.hid) { + ret = hid_driver_suspend(spihid->tp.hid, PMSG_SUSPEND); + if (ret < 0) + return ret; + } + + if (spihid->kbd.hid) { + ret = hid_driver_suspend(spihid->kbd.hid, PMSG_SUSPEND); + if (ret < 0) { + if (spihid->tp.hid) + hid_driver_resume(spihid->tp.hid); + return ret; + } + } + + /* Save some power */ + spihid->ops->disable_irq(spihid->ops); + +#ifdef IRQ_WAKE_SUPPORT + if (device_may_wakeup(dev)) { + wake_status = spihid->ops->enable_irq_wake(spihid->ops); + if (!wake_status) + spihid->irq_wake_enabled = true; + else + dev_warn(dev, "Failed to enable irq wake: %d\n", + wake_status); + } else { + spihid->ops->power_off(spihid->ops); + } +#else + spihid->ops->power_off(spihid->ops); +#endif + + return 0; +} + +static int spihid_apple_core_resume(struct device *dev) +{ + int ret_tp = 0, ret_kbd = 0; + struct spihid_apple *spihid = spi_get_drvdata(to_spi_device(dev)); +#ifdef IRQ_WAKE_SUPPORT + int wake_status; + + if (!device_may_wakeup(dev)) { + spihid->ops->power_on(spihid->ops); + } else if (spihid->irq_wake_enabled) { + wake_status = spihid->ops->disable_irq_wake(spihid->ops); + if (!wake_status) + spihid->irq_wake_enabled = false; + else + dev_warn(dev, "Failed to disable irq wake: %d\n", + wake_status); + } +#endif + + spihid->ops->enable_irq(spihid->ops); + spihid->ops->power_on(spihid->ops); + + if (spihid->tp.hid) + ret_tp = hid_driver_reset_resume(spihid->tp.hid); + if (spihid->kbd.hid) + ret_kbd = hid_driver_reset_resume(spihid->kbd.hid); + + if (ret_tp < 0) + return ret_tp; + + return ret_kbd; +} +#endif + +const struct dev_pm_ops spihid_apple_core_pm = { + SET_SYSTEM_SLEEP_PM_OPS(spihid_apple_core_suspend, + spihid_apple_core_resume) +}; +EXPORT_SYMBOL_GPL(spihid_apple_core_pm); + +MODULE_DESCRIPTION("Apple SPI HID transport driver"); +MODULE_AUTHOR("Janne Grunau "); +MODULE_LICENSE("GPL"); diff --git a/drivers/hid/spi-hid/spi-hid-apple-of.c b/drivers/hid/spi-hid/spi-hid-apple-of.c new file mode 100644 index 00000000000000..b631212b836d30 --- /dev/null +++ b/drivers/hid/spi-hid/spi-hid-apple-of.c @@ -0,0 +1,153 @@ +/* + * SPDX-License-Identifier: GPL-2.0 + * + * Apple SPI HID transport driver - Open Firmware + * + * Copyright (C) The Asahi Linux Contributors + */ + +#include +#include +#include +#include + +#include "spi-hid-apple.h" + + +struct spihid_apple_of { + struct spihid_apple_ops ops; + + struct gpio_desc *enable_gpio; + int irq; +}; + +static int spihid_apple_of_power_on(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + /* reset the controller on boot */ + gpiod_direction_output(sh_of->enable_gpio, 1); + msleep(5); + gpiod_direction_output(sh_of->enable_gpio, 0); + msleep(5); + /* turn SPI device on */ + gpiod_direction_output(sh_of->enable_gpio, 1); + msleep(50); + + return 0; +} + +static int spihid_apple_of_power_off(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + /* turn SPI device off */ + gpiod_direction_output(sh_of->enable_gpio, 0); + + return 0; +} + +static int spihid_apple_of_enable_irq(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + enable_irq(sh_of->irq); + + return 0; +} + +static int spihid_apple_of_disable_irq(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + disable_irq(sh_of->irq); + + return 0; +} + +static int spihid_apple_of_enable_irq_wake(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + return enable_irq_wake(sh_of->irq); +} + +static int spihid_apple_of_disable_irq_wake(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + return disable_irq_wake(sh_of->irq); +} + +static int spihid_apple_of_probe(struct spi_device *spi) +{ + struct device *dev = &spi->dev; + struct spihid_apple_of *spihid_of; + int err; + + spihid_of = devm_kzalloc(dev, sizeof(*spihid_of), GFP_KERNEL); + if (!spihid_of) + return -ENOMEM; + + spihid_of->ops.power_on = spihid_apple_of_power_on; + spihid_of->ops.power_off = spihid_apple_of_power_off; + spihid_of->ops.enable_irq = spihid_apple_of_enable_irq; + spihid_of->ops.disable_irq = spihid_apple_of_disable_irq; + spihid_of->ops.enable_irq_wake = spihid_apple_of_enable_irq_wake; + spihid_of->ops.disable_irq_wake = spihid_apple_of_disable_irq_wake; + + spihid_of->enable_gpio = devm_gpiod_get_index(dev, "spien", 0, 0); + if (IS_ERR(spihid_of->enable_gpio)) { + err = PTR_ERR(spihid_of->enable_gpio); + dev_err(dev, "failed to get 'spien' gpio pin: %d", err); + return err; + } + + spihid_of->irq = of_irq_get(dev->of_node, 0); + if (spihid_of->irq < 0) { + err = spihid_of->irq; + dev_err(dev, "failed to get 'extended-irq': %d", err); + return err; + } + err = devm_request_threaded_irq(dev, spihid_of->irq, NULL, + spihid_apple_core_irq, IRQF_ONESHOT | IRQF_NO_AUTOEN, + "spi-hid-apple-irq", spi); + if (err < 0) { + dev_err(dev, "failed to request extended-irq %d: %d", + spihid_of->irq, err); + return err; + } + + return spihid_apple_core_probe(spi, &spihid_of->ops); +} + +static const struct of_device_id spihid_apple_of_match[] = { + { .compatible = "apple,spi-hid-transport" }, + {}, +}; +MODULE_DEVICE_TABLE(of, spihid_apple_of_match); + +static struct spi_device_id spihid_apple_of_id[] = { + { "spi-hid-transport", 0 }, + {} +}; +MODULE_DEVICE_TABLE(spi, spihid_apple_of_id); + +static struct spi_driver spihid_apple_of_driver = { + .driver = { + .name = "spi-hid-apple-of", + .pm = &spihid_apple_core_pm, + .of_match_table = of_match_ptr(spihid_apple_of_match), + }, + + .id_table = spihid_apple_of_id, + .probe = spihid_apple_of_probe, + .remove = spihid_apple_core_remove, + .shutdown = spihid_apple_core_shutdown, +}; + +module_spi_driver(spihid_apple_of_driver); + +MODULE_DESCRIPTION("Apple SPI HID transport driver for OpenFirmware systems"); +MODULE_AUTHOR("Janne Grunau "); +MODULE_LICENSE("GPL"); diff --git a/drivers/hid/spi-hid/spi-hid-apple.h b/drivers/hid/spi-hid/spi-hid-apple.h new file mode 100644 index 00000000000000..9abecd1ba78028 --- /dev/null +++ b/drivers/hid/spi-hid/spi-hid-apple.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ + +#ifndef SPI_HID_APPLE_H +#define SPI_HID_APPLE_H + +#include +#include + +/** + * struct spihid_apple_ops - Ops to control the device from the core driver. + * + * @power_on: reset and power the device on. + * @power_off: power the device off. + * @enable_irq: enable irq or ACPI gpe. + * @disable_irq: disable irq or ACPI gpe. + */ + +struct spihid_apple_ops { + int (*power_on)(struct spihid_apple_ops *ops); + int (*power_off)(struct spihid_apple_ops *ops); + int (*enable_irq)(struct spihid_apple_ops *ops); + int (*disable_irq)(struct spihid_apple_ops *ops); + int (*enable_irq_wake)(struct spihid_apple_ops *ops); + int (*disable_irq_wake)(struct spihid_apple_ops *ops); +}; + +irqreturn_t spihid_apple_core_irq(int irq, void *data); + +int spihid_apple_core_probe(struct spi_device *spi, struct spihid_apple_ops *ops); +void spihid_apple_core_remove(struct spi_device *spi); +void spihid_apple_core_shutdown(struct spi_device *spi); + +extern const struct dev_pm_ops spihid_apple_core_pm; + +#endif /* SPI_HID_APPLE_H */ diff --git a/drivers/hv/mshv_eventfd.c b/drivers/hv/mshv_eventfd.c index 0b75ff1edb7359..6d176ed8ae5163 100644 --- a/drivers/hv/mshv_eventfd.c +++ b/drivers/hv/mshv_eventfd.c @@ -87,8 +87,9 @@ static void mshv_irqfd_resampler_ack(struct mshv_irq_ack_notifier *mian) idx = srcu_read_lock(&partition->pt_irq_srcu); - hlist_for_each_entry_rcu(irqfd, &resampler->rsmplr_irqfd_list, - irqfd_resampler_hnode) { + hlist_for_each_entry_srcu(irqfd, &resampler->rsmplr_irqfd_list, + irqfd_resampler_hnode, + srcu_read_lock_held(&partition->pt_irq_srcu)) { if (hv_should_clear_interrupt(irqfd->irqfd_lapic_irq.lapic_control.interrupt_type)) hv_call_clear_virtual_interrupt(partition->pt_id); diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index a53af6fe81a657..1d5cba142828e7 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -1350,7 +1351,7 @@ static void vmbus_message_sched(struct hv_per_cpu_context *hv_cpu, void *message } } -void vmbus_isr(void) +static void __vmbus_isr(void) { struct hv_per_cpu_context *hv_cpu = this_cpu_ptr(hv_context.cpu_context); @@ -1363,6 +1364,53 @@ void vmbus_isr(void) add_interrupt_randomness(vmbus_interrupt); } + +static DEFINE_PER_CPU(bool, vmbus_irq_pending); +static DEFINE_PER_CPU(struct task_struct *, vmbus_irqd); + +static void vmbus_irqd_wake(void) +{ + struct task_struct *tsk = __this_cpu_read(vmbus_irqd); + + __this_cpu_write(vmbus_irq_pending, true); + wake_up_process(tsk); +} + +static void vmbus_irqd_setup(unsigned int cpu) +{ + sched_set_fifo(current); +} + +static int vmbus_irqd_should_run(unsigned int cpu) +{ + return __this_cpu_read(vmbus_irq_pending); +} + +static void run_vmbus_irqd(unsigned int cpu) +{ + __this_cpu_write(vmbus_irq_pending, false); + __vmbus_isr(); +} + +static bool vmbus_irq_initialized; + +static struct smp_hotplug_thread vmbus_irq_threads = { + .store = &vmbus_irqd, + .setup = vmbus_irqd_setup, + .thread_should_run = vmbus_irqd_should_run, + .thread_fn = run_vmbus_irqd, + .thread_comm = "vmbus_irq/%u", +}; + +void vmbus_isr(void) +{ + if (IS_ENABLED(CONFIG_PREEMPT_RT)) { + vmbus_irqd_wake(); + } else { + lockdep_hardirq_threaded(); + __vmbus_isr(); + } +} EXPORT_SYMBOL_FOR_MODULES(vmbus_isr, "mshv_vtl"); static irqreturn_t vmbus_percpu_isr(int irq, void *dev_id) @@ -1462,6 +1510,13 @@ static int vmbus_bus_init(void) * the VMbus interrupt handler. */ + if (IS_ENABLED(CONFIG_PREEMPT_RT) && !vmbus_irq_initialized) { + ret = smpboot_register_percpu_thread(&vmbus_irq_threads); + if (ret) + goto err_kthread; + vmbus_irq_initialized = true; + } + if (vmbus_irq == -1) { hv_setup_vmbus_handler(vmbus_isr); } else { @@ -1507,6 +1562,11 @@ static int vmbus_bus_init(void) free_percpu(vmbus_evt); } err_setup: + if (IS_ENABLED(CONFIG_PREEMPT_RT) && vmbus_irq_initialized) { + smpboot_unregister_percpu_thread(&vmbus_irq_threads); + vmbus_irq_initialized = false; + } +err_kthread: bus_unregister(&hv_bus); return ret; } @@ -2976,6 +3036,10 @@ static void __exit vmbus_exit(void) free_percpu_irq(vmbus_irq, vmbus_evt); free_percpu(vmbus_evt); } + if (IS_ENABLED(CONFIG_PREEMPT_RT) && vmbus_irq_initialized) { + smpboot_unregister_percpu_thread(&vmbus_irq_threads); + vmbus_irq_initialized = false; + } for_each_online_cpu(cpu) { struct hv_per_cpu_context *hv_cpu = per_cpu_ptr(hv_context.cpu_context, cpu); diff --git a/drivers/hwmon/ibmpex.c b/drivers/hwmon/ibmpex.c index 129f3a9e8fe965..228c5f6c6f3836 100644 --- a/drivers/hwmon/ibmpex.c +++ b/drivers/hwmon/ibmpex.c @@ -277,9 +277,6 @@ static ssize_t ibmpex_high_low_store(struct device *dev, { struct ibmpex_bmc_data *data = dev_get_drvdata(dev); - if (!data) - return -ENODEV; - ibmpex_reset_high_low_data(data); return count; @@ -511,9 +508,6 @@ static void ibmpex_bmc_delete(struct ibmpex_bmc_data *data) { int i, j; - hwmon_device_unregister(data->hwmon_dev); - dev_set_drvdata(data->bmc_device, NULL); - device_remove_file(data->bmc_device, &sensor_dev_attr_reset_high_low.dev_attr); device_remove_file(data->bmc_device, &dev_attr_name.attr); @@ -527,7 +521,8 @@ static void ibmpex_bmc_delete(struct ibmpex_bmc_data *data) } list_del(&data->list); - + dev_set_drvdata(data->bmc_device, NULL); + hwmon_device_unregister(data->hwmon_dev); ipmi_destroy_user(data->user); kfree(data->sensors); kfree(data); diff --git a/drivers/hwmon/pmbus/mpq8785.c b/drivers/hwmon/pmbus/mpq8785.c index 1f56aaf4dde807..87bd039c77b9b3 100644 --- a/drivers/hwmon/pmbus/mpq8785.c +++ b/drivers/hwmon/pmbus/mpq8785.c @@ -47,6 +47,33 @@ static int mpq8785_identify(struct i2c_client *client, return 0; }; +static int mpq8785_read_byte_data(struct i2c_client *client, int page, int reg) +{ + int ret; + + switch (reg) { + case PMBUS_VOUT_MODE: + ret = pmbus_read_byte_data(client, page, reg); + if (ret < 0) + return ret; + + if ((ret >> 5) == 1) { + /* + * The MPQ8785 chip reports VOUT_MODE as VID mode, but the driver + * treats VID as direct mode. Without this, identification would fail + * due to mode mismatch. + * This override ensures the reported mode matches the driver + * configuration, allowing successful initialization. + */ + return PB_VOUT_MODE_DIRECT; + } + + return ret; + default: + return -ENODATA; + } +} + static int mpm82504_read_word_data(struct i2c_client *client, int page, int phase, int reg) { @@ -129,6 +156,7 @@ static int mpq8785_probe(struct i2c_client *client) break; case mpq8785: info->identify = mpq8785_identify; + info->read_byte_data = mpq8785_read_byte_data; break; default: return -ENODEV; diff --git a/drivers/hwspinlock/omap_hwspinlock.c b/drivers/hwspinlock/omap_hwspinlock.c index 27b47b8623c09c..2d8de835bc2429 100644 --- a/drivers/hwspinlock/omap_hwspinlock.c +++ b/drivers/hwspinlock/omap_hwspinlock.c @@ -88,7 +88,9 @@ static int omap_hwspinlock_probe(struct platform_device *pdev) * make sure the module is enabled and clocked before reading * the module SYSSTATUS register */ - devm_pm_runtime_enable(&pdev->dev); + ret = devm_pm_runtime_enable(&pdev->dev); + if (ret) + return ret; ret = pm_runtime_resume_and_get(&pdev->dev); if (ret < 0) return ret; diff --git a/drivers/hwtracing/coresight/coresight-etm3x-core.c b/drivers/hwtracing/coresight/coresight-etm3x-core.c index a5e809589d3e38..0c011b7041696a 100644 --- a/drivers/hwtracing/coresight/coresight-etm3x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm3x-core.c @@ -795,16 +795,16 @@ static int __init etm_hp_setup(void) { int ret; - ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ARM_CORESIGHT_STARTING, - "arm/coresight:starting", - etm_starting_cpu, etm_dying_cpu); + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING, + "arm/coresight:starting", + etm_starting_cpu, etm_dying_cpu); if (ret) return ret; - ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN, - "arm/coresight:online", - etm_online_cpu, NULL); + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, + "arm/coresight:online", + etm_online_cpu, NULL); /* HP dyn state ID returned in ret on success */ if (ret > 0) { diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c index e0d83ee01b77a5..fc0a946053ddef 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etr.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c @@ -1306,6 +1306,19 @@ static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev) raw_spin_lock_irqsave(&drvdata->spinlock, flags); + /* + * Since the sysfs buffer allocation and the hardware enablement is not + * in the same critical region, it's possible to race with the perf. + */ + if (coresight_get_mode(csdev) == CS_MODE_PERF) { + drvdata->sysfs_buf = NULL; + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + + /* Free allocated memory out side of the spinlock */ + tmc_etr_free_sysfs_buf(sysfs_buf); + return -EBUSY; + } + /* * In sysFS mode we can have multiple writers per sink. Since this * sink is already enabled no memory is needed and the HW need not be diff --git a/drivers/hwtracing/stm/Kconfig b/drivers/hwtracing/stm/Kconfig index eda6b11d40a1f9..cd7f0b0f3fbebc 100644 --- a/drivers/hwtracing/stm/Kconfig +++ b/drivers/hwtracing/stm/Kconfig @@ -13,7 +13,7 @@ if STM config STM_PROTO_BASIC tristate "Basic STM framing protocol driver" - default CONFIG_STM + default STM help This is a simple framing protocol for sending data over STM devices. This was the protocol that the STM framework used @@ -28,7 +28,7 @@ config STM_PROTO_BASIC config STM_PROTO_SYS_T tristate "MIPI SyS-T STM framing protocol driver" - default CONFIG_STM + default STM help This is an implementation of MIPI SyS-T protocol to be used over the STP transport. In addition to the data payload, it diff --git a/drivers/i3c/master.c b/drivers/i3c/master.c index 7f606c8716480c..5408332861a1bf 100644 --- a/drivers/i3c/master.c +++ b/drivers/i3c/master.c @@ -618,7 +618,8 @@ static int i3c_set_hotjoin(struct i3c_master_controller *master, bool enable) else ret = master->ops->disable_hotjoin(master); - master->hotjoin = enable; + if (!ret) + master->hotjoin = enable; i3c_bus_normaluse_unlock(&master->bus); @@ -2881,7 +2882,6 @@ int i3c_master_register(struct i3c_master_controller *master, INIT_LIST_HEAD(&master->boardinfo.i3c); device_initialize(&master->dev); - dev_set_name(&master->dev, "i3c-%d", i3cbus->id); master->dev.dma_mask = parent->dma_mask; master->dev.coherent_dma_mask = parent->coherent_dma_mask; @@ -2891,6 +2891,8 @@ int i3c_master_register(struct i3c_master_controller *master, if (ret) goto err_put_dev; + dev_set_name(&master->dev, "i3c-%d", i3cbus->id); + ret = of_populate_i3c_bus(master); if (ret) goto err_put_dev; diff --git a/drivers/i3c/master/dw-i3c-master.c b/drivers/i3c/master/dw-i3c-master.c index 889e2ed5bc8303..4033bc16677ff1 100644 --- a/drivers/i3c/master/dw-i3c-master.c +++ b/drivers/i3c/master/dw-i3c-master.c @@ -1099,6 +1099,7 @@ static int dw_i3c_master_i2c_xfers(struct i2c_dev_desc *dev, dev_err(master->dev, "<%s> cannot resume i3c bus master, err: %d\n", __func__, ret); + dw_i3c_master_free_xfer(xfer); return ret; } @@ -1570,6 +1571,8 @@ int dw_i3c_common_probe(struct dw_i3c_master *master, spin_lock_init(&master->xferqueue.lock); INIT_LIST_HEAD(&master->xferqueue.list); + spin_lock_init(&master->devs_lock); + writel(INTR_ALL, master->regs + INTR_STATUS); irq = platform_get_irq(pdev, 0); ret = devm_request_irq(&pdev->dev, irq, diff --git a/drivers/iio/accel/sca3000.c b/drivers/iio/accel/sca3000.c index bfa8a3f5a92f43..9ef4d6e2746697 100644 --- a/drivers/iio/accel/sca3000.c +++ b/drivers/iio/accel/sca3000.c @@ -1489,7 +1489,11 @@ static int sca3000_probe(struct spi_device *spi) if (ret) goto error_free_irq; - return iio_device_register(indio_dev); + ret = iio_device_register(indio_dev); + if (ret) + goto error_free_irq; + + return 0; error_free_irq: if (spi->irq) diff --git a/drivers/iio/common/Kconfig b/drivers/iio/common/Kconfig index 1ccb5ccf370660..e3818ef567822b 100644 --- a/drivers/iio/common/Kconfig +++ b/drivers/iio/common/Kconfig @@ -3,6 +3,7 @@ # IIO common modules # +source "drivers/iio/common/aop_sensors/Kconfig" source "drivers/iio/common/cros_ec_sensors/Kconfig" source "drivers/iio/common/hid-sensors/Kconfig" source "drivers/iio/common/inv_sensors/Kconfig" diff --git a/drivers/iio/common/Makefile b/drivers/iio/common/Makefile index d3e952239a6219..5f99a429725d66 100644 --- a/drivers/iio/common/Makefile +++ b/drivers/iio/common/Makefile @@ -8,6 +8,7 @@ # # When adding new entries keep the list in alphabetical order +obj-y += aop_sensors/ obj-y += cros_ec_sensors/ obj-y += hid-sensors/ obj-y += inv_sensors/ diff --git a/drivers/iio/common/aop_sensors/Kconfig b/drivers/iio/common/aop_sensors/Kconfig new file mode 100644 index 00000000000000..6f6949bdf6a5d8 --- /dev/null +++ b/drivers/iio/common/aop_sensors/Kconfig @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT + +config IIO_AOP_SENSOR_LAS + tristate "AOP Lid angle sensor" + depends on ARCH_APPLE || COMPILE_TEST + depends on RUST + depends on SYSFS + select APPLE_AOP + help + Module to handle the lid angle sensor attached to the AOP + coprocessor on Apple laptops. + +config IIO_AOP_SENSOR_ALS + tristate "AOP Ambient light sensor" + depends on ARCH_APPLE || COMPILE_TEST + depends on RUST + depends on SYSFS + depends on RUST_FW_LOADER_ABSTRACTIONS + select APPLE_AOP + help + Module to handle the ambient light sensor attached to the AOP + coprocessor on Apple laptops. diff --git a/drivers/iio/common/aop_sensors/Makefile b/drivers/iio/common/aop_sensors/Makefile new file mode 100644 index 00000000000000..8da5a19efe0f0c --- /dev/null +++ b/drivers/iio/common/aop_sensors/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT + +obj-$(CONFIG_IIO_AOP_SENSOR_LAS) += aop_las.o +obj-$(CONFIG_IIO_AOP_SENSOR_ALS) += aop_als.o diff --git a/drivers/iio/common/aop_sensors/aop_als.rs b/drivers/iio/common/aop_sensors/aop_als.rs new file mode 100644 index 00000000000000..87a4686f3ca03a --- /dev/null +++ b/drivers/iio/common/aop_sensors/aop_als.rs @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Apple AOP ambient light sensor driver +//! +//! Copyright (C) The Asahi Linux Contributors + +use kernel::{ + bindings, c_str, + device::Core, + firmware::Firmware, + iio::common::aop_sensors::{AopSensorData, IIORegistration, MessageProcessor}, + module_platform_driver, of, platform, + prelude::*, + soc::apple::aop::{EPICService, AOP}, + sync::Arc, + types::ForeignOwnable, +}; + +const EPIC_SUBTYPE_GET_AOP_PROPERTY: u16 = 0xa; +const EPIC_SUBTYPE_SET_ALS_PROPERTY: u16 = 0x4; +const LUX_OFFSET_CT720: usize = 0x1d; +const LUX_OFFSET_VD6286: usize = 0x28; + +fn get_lux_offset(aop: &dyn AOP, dev: &platform::Device, svc: &EPICService) -> Result { + let name = get_aop_property(aop, svc, 0xf, 16)?.1; + match name.as_slice() { + b"Redbird\0" => Ok(LUX_OFFSET_VD6286), + b"FireFish2\0" => Ok(LUX_OFFSET_CT720), + _ => { + dev_warn!( + dev.as_ref(), + "Unknown sensor type {:?}", + core::str::from_utf8(&name) + ); + Err(EIO) + } + } +} + +fn enable_als(aop: &dyn AOP, dev: &platform::Device, svc: &EPICService) -> Result<()> { + let fw = Firmware::request(c_str!("apple/aop-als-cal.bin"), dev.as_ref())?; + set_als_property(aop, svc, 0xb, fw.data())?; + set_als_property(aop, svc, 0, &200000u32.to_le_bytes())?; + + Ok(()) +} + +fn get_aop_property( + aop: &dyn AOP, + svc: &EPICService, + tag: u32, + data_len: usize, +) -> Result<(u32, KVec)> { + let mut buf = KVec::new(); + buf.resize(8, 0, GFP_KERNEL)?; + buf[4..8].copy_from_slice(&tag.to_le_bytes()); + aop.epic_call_ret(svc, EPIC_SUBTYPE_GET_AOP_PROPERTY, &buf, data_len) +} + +fn set_als_property(aop: &dyn AOP, svc: &EPICService, tag: u32, data: &[u8]) -> Result { + let mut buf = KVec::new(); + buf.resize(data.len() + 8, 0, GFP_KERNEL)?; + buf[8..].copy_from_slice(data); + buf[4..8].copy_from_slice(&tag.to_le_bytes()); + aop.epic_call(svc, EPIC_SUBTYPE_SET_ALS_PROPERTY, &buf) +} + +fn f32_to_u32(f: u32) -> u32 { + if f & 0x80000000 != 0 { + return 0; + } + let exp = ((f & 0x7f800000) >> 23) as i32 - 127; + if exp < 0 { + return 0; + } + if exp == 128 && f & 0x7fffff != 0 { + return 0; + } + let mant = f & 0x7fffff | 0x800000; + if exp <= 23 { + return mant >> (23 - exp); + } + if exp >= 32 { + return u32::MAX; + } + mant << (exp - 23) +} + +struct MsgProc(usize); + +impl MessageProcessor for MsgProc { + fn process(&self, message: &[u8]) -> u32 { + let offset = self.0; + let raw = u32::from_le_bytes(message[offset..offset + 4].try_into().unwrap()); + f32_to_u32(raw) + } +} + +#[repr(transparent)] +struct IIOAopAlsDriver(IIORegistration); + +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + (), + [(of::DeviceId::new(c_str!("apple,aop-als")), ())] +); + +impl platform::Driver for IIOAopAlsDriver { + type IdInfo = (); + + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + fn probe(pdev: &platform::Device, _info: Option<&()>) -> impl PinInit { + let dev = pdev.as_ref(); + let parent = dev.parent().unwrap(); + // SAFETY: our parent is AOP, and AopDriver is repr(transparent) for Arc + let adata_ptr = unsafe { Pin::>>::borrow(parent.get_drvdata()) }; + let adata = (&*adata_ptr).clone(); + // SAFETY: AOP sets the platform data correctly + let service = unsafe { *((*dev.as_raw()).platform_data as *const EPICService) }; + let ty = bindings::BINDINGS_IIO_LIGHT; + let offset = get_lux_offset(adata.as_ref(), pdev, &service)?; + let data = AopSensorData::new(dev.into(), ty, MsgProc(offset))?; + adata.add_fakehid_listener(service, data.clone())?; + enable_als(adata.as_ref(), pdev, &service)?; + let info_mask = 1 << bindings::BINDINGS_IIO_CHAN_INFO_PROCESSED; + Ok(IIOAopAlsDriver(IIORegistration::::new( + data, + c"aop-sensors-als", + ty, + info_mask, + &THIS_MODULE, + )?)) + } +} + +module_platform_driver! { + type: IIOAopAlsDriver, + name: "iio_aop_als", + description: "AOP ambient light sensor driver", + license: "Dual MIT/GPL", + alias: ["platform:iio_aop_als"], + firmware: ["apple/aop-als-cal.bin"], +} diff --git a/drivers/iio/common/aop_sensors/aop_las.rs b/drivers/iio/common/aop_sensors/aop_las.rs new file mode 100644 index 00000000000000..9256d61aed9d83 --- /dev/null +++ b/drivers/iio/common/aop_sensors/aop_las.rs @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Apple AOP lid angle sensor driver +//! +//! Copyright (C) The Asahi Linux Contributors + +use kernel::{ + bindings, c_str, + device::Core, + iio::common::aop_sensors::{AopSensorData, IIORegistration, MessageProcessor}, + module_platform_driver, of, platform, + prelude::*, + soc::apple::aop::{EPICService, AOP}, + sync::Arc, + types::ForeignOwnable, +}; + +struct MsgProc; + +impl MessageProcessor for MsgProc { + fn process(&self, message: &[u8]) -> u32 { + message[1] as u32 + } +} + +#[repr(transparent)] +struct IIOAopLasDriver(IIORegistration); + +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + (), + [(of::DeviceId::new(c_str!("apple,aop-las")), ())] +); + +impl platform::Driver for IIOAopLasDriver { + type IdInfo = (); + + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + fn probe(pdev: &platform::Device, _info: Option<&()>) -> impl PinInit { + let dev = pdev.as_ref(); + let parent = dev.parent().unwrap(); + // SAFETY: our parent is AOP, and AopDriver is repr(transparent) for Arc + let adata_ptr = unsafe { Pin::>>::borrow(parent.get_drvdata()) }; + let adata = (&*adata_ptr).clone(); + // SAFETY: AOP sets the platform data correctly + let service = unsafe { *((*dev.as_raw()).platform_data as *const EPICService) }; + + let ty = bindings::BINDINGS_IIO_ANGL; + let data = AopSensorData::new(dev.into(), ty, MsgProc)?; + adata.add_fakehid_listener(service, data.clone())?; + let info_mask = 1 << bindings::BINDINGS_IIO_CHAN_INFO_RAW; + Ok(IIOAopLasDriver(IIORegistration::::new( + data, + c"aop-sensors-las", + ty, + info_mask, + &THIS_MODULE, + )?)) + } +} + +module_platform_driver! { + type: IIOAopLasDriver, + name: "iio_aop_las", + description: "AOP lid angle sensor driver", + license: "Dual MIT/GPL", + alias: ["platform:iio_aop_las"], +} diff --git a/drivers/iio/gyro/mpu3050-core.c b/drivers/iio/gyro/mpu3050-core.c index 67ae7d1012bc27..ee2fcd20545dee 100644 --- a/drivers/iio/gyro/mpu3050-core.c +++ b/drivers/iio/gyro/mpu3050-core.c @@ -1162,10 +1162,8 @@ int mpu3050_common_probe(struct device *dev, mpu3050->regs[1].supply = mpu3050_reg_vlogic; ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(mpu3050->regs), mpu3050->regs); - if (ret) { - dev_err(dev, "Cannot get regulators\n"); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, "Cannot get regulators\n"); ret = mpu3050_power_up(mpu3050); if (ret) diff --git a/drivers/iio/pressure/mprls0025pa.c b/drivers/iio/pressure/mprls0025pa.c index 2336f2760eaeb7..d4133fef91fac4 100644 --- a/drivers/iio/pressure/mprls0025pa.c +++ b/drivers/iio/pressure/mprls0025pa.c @@ -59,7 +59,7 @@ * * Values given to the userspace in sysfs interface: * * raw - press_cnt - * * offset - (-1 * outputmin) - pmin / scale + * * offset - (-1 * outputmin) + pmin / scale * note: With all sensors from the datasheet pmin = 0 * which reduces the offset to (-1 * outputmin) */ @@ -160,8 +160,8 @@ static const struct iio_chan_spec mpr_channels[] = { BIT(IIO_CHAN_INFO_OFFSET), .scan_index = 0, .scan_type = { - .sign = 's', - .realbits = 32, + .sign = 'u', + .realbits = 24, .storagebits = 32, .endianness = IIO_CPU, }, @@ -313,8 +313,7 @@ static int mpr_read_raw(struct iio_dev *indio_dev, return IIO_VAL_INT_PLUS_NANO; case IIO_CHAN_INFO_OFFSET: *val = data->offset; - *val2 = data->offset2; - return IIO_VAL_INT_PLUS_NANO; + return IIO_VAL_INT; default: return -EINVAL; } @@ -330,8 +329,9 @@ int mpr_common_probe(struct device *dev, const struct mpr_ops *ops, int irq) struct mpr_data *data; struct iio_dev *indio_dev; const char *triplet; - s64 scale, offset; + s64 odelta, pdelta; u32 func; + s32 tmp; indio_dev = devm_iio_device_alloc(dev, sizeof(*data)); if (!indio_dev) @@ -405,23 +405,17 @@ int mpr_common_probe(struct device *dev, const struct mpr_ops *ops, int irq) data->outmin = mpr_func_spec[data->function].output_min; data->outmax = mpr_func_spec[data->function].output_max; - /* use 64 bit calculation for preserving a reasonable precision */ - scale = div_s64(((s64)(data->pmax - data->pmin)) * NANO, - data->outmax - data->outmin); - data->scale = div_s64_rem(scale, NANO, &data->scale2); - /* - * multiply with NANO before dividing by scale and later divide by NANO - * again. - */ - offset = ((-1LL) * (s64)data->outmin) * NANO - - div_s64(div_s64((s64)data->pmin * NANO, scale), NANO); - data->offset = div_s64_rem(offset, NANO, &data->offset2); + odelta = data->outmax - data->outmin; + pdelta = data->pmax - data->pmin; + + data->scale = div_s64_rem(div_s64(pdelta * NANO, odelta), NANO, &tmp); + data->scale2 = tmp; + + data->offset = div_s64(odelta * data->pmin, pdelta) - data->outmin; if (data->irq > 0) { - ret = devm_request_irq(dev, data->irq, mpr_eoc_handler, - IRQF_TRIGGER_RISING, - dev_name(dev), - data); + ret = devm_request_irq(dev, data->irq, mpr_eoc_handler, 0, + dev_name(dev), data); if (ret) return dev_err_probe(dev, ret, "request irq %d failed\n", data->irq); diff --git a/drivers/iio/pressure/mprls0025pa.h b/drivers/iio/pressure/mprls0025pa.h index d62a018eaff32b..b6944b30512677 100644 --- a/drivers/iio/pressure/mprls0025pa.h +++ b/drivers/iio/pressure/mprls0025pa.h @@ -53,7 +53,6 @@ enum mpr_func_id { * @scale: pressure scale * @scale2: pressure scale, decimal number * @offset: pressure offset - * @offset2: pressure offset, decimal number * @gpiod_reset: reset * @irq: end of conversion irq. used to distinguish between irq mode and * reading in a loop until data is ready @@ -75,7 +74,6 @@ struct mpr_data { int scale; int scale2; int offset; - int offset2; struct gpio_desc *gpiod_reset; int irq; struct completion completion; diff --git a/drivers/iio/pressure/mprls0025pa_spi.c b/drivers/iio/pressure/mprls0025pa_spi.c index d04102f8a4a035..cf17eb2e720830 100644 --- a/drivers/iio/pressure/mprls0025pa_spi.c +++ b/drivers/iio/pressure/mprls0025pa_spi.c @@ -8,6 +8,7 @@ * https://prod-edam.honeywell.com/content/dam/honeywell-edam/sps/siot/en-us/products/sensors/pressure-sensors/board-mount-pressure-sensors/micropressure-mpr-series/documents/sps-siot-mpr-series-datasheet-32332628-ciid-172626.pdf */ +#include #include #include #include @@ -40,17 +41,25 @@ static int mpr_spi_xfer(struct mpr_data *data, const u8 cmd, const u8 pkt_len) { struct spi_device *spi = to_spi_device(data->dev); struct mpr_spi_buf *buf = spi_get_drvdata(spi); - struct spi_transfer xfer; + struct spi_transfer xfers[2] = { }; if (pkt_len > MPR_MEASUREMENT_RD_SIZE) return -EOVERFLOW; buf->tx[0] = cmd; - xfer.tx_buf = buf->tx; - xfer.rx_buf = data->buffer; - xfer.len = pkt_len; - return spi_sync_transfer(spi, &xfer, 1); + /* + * Dummy transfer with no data, just cause a 2.5us+ delay between the CS assert + * and the first clock edge as per the datasheet tHDSS timing requirement. + */ + xfers[0].delay.value = 2500; + xfers[0].delay.unit = SPI_DELAY_UNIT_NSECS; + + xfers[1].tx_buf = buf->tx; + xfers[1].rx_buf = data->buffer; + xfers[1].len = pkt_len; + + return spi_sync_transfer(spi, xfers, ARRAY_SIZE(xfers)); } static const struct mpr_ops mpr_spi_ops = { diff --git a/drivers/iio/test/Kconfig b/drivers/iio/test/Kconfig index 6e65e929791ca2..4fc17dd0dcd770 100644 --- a/drivers/iio/test/Kconfig +++ b/drivers/iio/test/Kconfig @@ -8,7 +8,6 @@ config IIO_GTS_KUNIT_TEST tristate "Test IIO gain-time-scale helpers" if !KUNIT_ALL_TESTS depends on KUNIT select IIO_GTS_HELPER - select TEST_KUNIT_DEVICE_HELPERS default KUNIT_ALL_TESTS help build unit tests for the IIO light sensor gain-time-scale helpers. diff --git a/drivers/infiniband/core/cache.c b/drivers/infiniband/core/cache.c index 81cf3c902e8195..0fc1c5bce2f0d3 100644 --- a/drivers/infiniband/core/cache.c +++ b/drivers/infiniband/core/cache.c @@ -1537,7 +1537,8 @@ static void ib_cache_event_task(struct work_struct *_work) * the cache. */ ret = ib_cache_update(work->event.device, work->event.element.port_num, - work->event.event == IB_EVENT_GID_CHANGE, + work->event.event == IB_EVENT_GID_CHANGE || + work->event.event == IB_EVENT_CLIENT_REREGISTER, work->event.event == IB_EVENT_PKEY_CHANGE, work->enforce_security); diff --git a/drivers/infiniband/core/iwcm.c b/drivers/infiniband/core/iwcm.c index 62410578dec37d..eb942ab9c40559 100644 --- a/drivers/infiniband/core/iwcm.c +++ b/drivers/infiniband/core/iwcm.c @@ -95,7 +95,6 @@ static struct workqueue_struct *iwcm_wq; struct iwcm_work { struct work_struct work; struct iwcm_id_private *cm_id; - struct list_head list; struct iw_cm_event event; struct list_head free_list; }; @@ -178,7 +177,6 @@ static int alloc_work_entries(struct iwcm_id_private *cm_id_priv, int count) return -ENOMEM; } work->cm_id = cm_id_priv; - INIT_LIST_HEAD(&work->list); put_work(work); } return 0; @@ -213,7 +211,6 @@ static void free_cm_id(struct iwcm_id_private *cm_id_priv) static bool iwcm_deref_id(struct iwcm_id_private *cm_id_priv) { if (refcount_dec_and_test(&cm_id_priv->refcount)) { - BUG_ON(!list_empty(&cm_id_priv->work_list)); free_cm_id(cm_id_priv); return true; } @@ -260,7 +257,6 @@ struct iw_cm_id *iw_create_cm_id(struct ib_device *device, refcount_set(&cm_id_priv->refcount, 1); init_waitqueue_head(&cm_id_priv->connect_wait); init_completion(&cm_id_priv->destroy_comp); - INIT_LIST_HEAD(&cm_id_priv->work_list); INIT_LIST_HEAD(&cm_id_priv->work_free_list); return &cm_id_priv->id; @@ -1007,13 +1003,13 @@ static int process_event(struct iwcm_id_private *cm_id_priv, } /* - * Process events on the work_list for the cm_id. If the callback - * function requests that the cm_id be deleted, a flag is set in the - * cm_id flags to indicate that when the last reference is - * removed, the cm_id is to be destroyed. This is necessary to - * distinguish between an object that will be destroyed by the app - * thread asleep on the destroy_comp list vs. an object destroyed - * here synchronously when the last reference is removed. + * Process events for the cm_id. If the callback function requests + * that the cm_id be deleted, a flag is set in the cm_id flags to + * indicate that when the last reference is removed, the cm_id is + * to be destroyed. This is necessary to distinguish between an + * object that will be destroyed by the app thread asleep on the + * destroy_comp list vs. an object destroyed here synchronously + * when the last reference is removed. */ static void cm_work_handler(struct work_struct *_work) { @@ -1024,35 +1020,26 @@ static void cm_work_handler(struct work_struct *_work) int ret = 0; spin_lock_irqsave(&cm_id_priv->lock, flags); - while (!list_empty(&cm_id_priv->work_list)) { - work = list_first_entry(&cm_id_priv->work_list, - struct iwcm_work, list); - list_del_init(&work->list); - levent = work->event; - put_work(work); - spin_unlock_irqrestore(&cm_id_priv->lock, flags); - - if (!test_bit(IWCM_F_DROP_EVENTS, &cm_id_priv->flags)) { - ret = process_event(cm_id_priv, &levent); - if (ret) { - destroy_cm_id(&cm_id_priv->id); - WARN_ON_ONCE(iwcm_deref_id(cm_id_priv)); - } - } else - pr_debug("dropping event %d\n", levent.event); - if (iwcm_deref_id(cm_id_priv)) - return; - spin_lock_irqsave(&cm_id_priv->lock, flags); - } + levent = work->event; + put_work(work); spin_unlock_irqrestore(&cm_id_priv->lock, flags); + + if (!test_bit(IWCM_F_DROP_EVENTS, &cm_id_priv->flags)) { + ret = process_event(cm_id_priv, &levent); + if (ret) { + destroy_cm_id(&cm_id_priv->id); + WARN_ON_ONCE(iwcm_deref_id(cm_id_priv)); + } + } else + pr_debug("dropping event %d\n", levent.event); + if (iwcm_deref_id(cm_id_priv)) + return; } /* * This function is called on interrupt context. Schedule events on * the iwcm_wq thread to allow callback functions to downcall into - * the CM and/or block. Events are queued to a per-CM_ID - * work_list. If this is the first event on the work_list, the work - * element is also queued on the iwcm_wq thread. + * the CM and/or block. * * Each event holds a reference on the cm_id. Until the last posted * event has been delivered and processed, the cm_id cannot be @@ -1094,7 +1081,6 @@ static int cm_event_handler(struct iw_cm_id *cm_id, } refcount_inc(&cm_id_priv->refcount); - list_add_tail(&work->list, &cm_id_priv->work_list); queue_work(iwcm_wq, &work->work); out: spin_unlock_irqrestore(&cm_id_priv->lock, flags); diff --git a/drivers/infiniband/core/iwcm.h b/drivers/infiniband/core/iwcm.h index bf74639be1287c..b56fb12edece40 100644 --- a/drivers/infiniband/core/iwcm.h +++ b/drivers/infiniband/core/iwcm.h @@ -50,7 +50,6 @@ struct iwcm_id_private { struct ib_qp *qp; struct completion destroy_comp; wait_queue_head_t connect_wait; - struct list_head work_list; spinlock_t lock; refcount_t refcount; struct list_head work_free_list; diff --git a/drivers/infiniband/core/rw.c b/drivers/infiniband/core/rw.c index 6354ddf2a274ce..2522ff1cc462c7 100644 --- a/drivers/infiniband/core/rw.c +++ b/drivers/infiniband/core/rw.c @@ -651,34 +651,57 @@ unsigned int rdma_rw_mr_factor(struct ib_device *device, u32 port_num, } EXPORT_SYMBOL(rdma_rw_mr_factor); +/** + * rdma_rw_max_send_wr - compute max Send WRs needed for RDMA R/W contexts + * @dev: RDMA device + * @port_num: port number + * @max_rdma_ctxs: number of rdma_rw_ctx structures + * @create_flags: QP create flags (pass IB_QP_CREATE_INTEGRITY_EN if + * data integrity will be enabled on the QP) + * + * Returns the total number of Send Queue entries needed for + * @max_rdma_ctxs. The result accounts for memory registration and + * invalidation work requests when the device requires them. + * + * ULPs use this to size Send Queues and Send CQs before creating a + * Queue Pair. + */ +unsigned int rdma_rw_max_send_wr(struct ib_device *dev, u32 port_num, + unsigned int max_rdma_ctxs, u32 create_flags) +{ + unsigned int factor = 1; + unsigned int result; + + if (create_flags & IB_QP_CREATE_INTEGRITY_EN || + rdma_rw_can_use_mr(dev, port_num)) + factor += 2; /* reg + inv */ + + if (check_mul_overflow(factor, max_rdma_ctxs, &result)) + return UINT_MAX; + return result; +} +EXPORT_SYMBOL(rdma_rw_max_send_wr); + void rdma_rw_init_qp(struct ib_device *dev, struct ib_qp_init_attr *attr) { - u32 factor; + unsigned int factor = 1; WARN_ON_ONCE(attr->port_num == 0); /* - * Each context needs at least one RDMA READ or WRITE WR. - * - * For some hardware we might need more, eventually we should ask the - * HCA driver for a multiplier here. - */ - factor = 1; - - /* - * If the device needs MRs to perform RDMA READ or WRITE operations, - * we'll need two additional MRs for the registrations and the - * invalidation. + * If the device uses MRs to perform RDMA READ or WRITE operations, + * or if data integrity is enabled, account for registration and + * invalidation work requests. */ if (attr->create_flags & IB_QP_CREATE_INTEGRITY_EN || rdma_rw_can_use_mr(dev, attr->port_num)) - factor += 2; /* inv + reg */ + factor += 2; /* reg + inv */ attr->cap.max_send_wr += factor * attr->cap.max_rdma_ctxs; /* - * But maybe we were just too high in the sky and the device doesn't - * even support all we need, and we'll have to live with what we get.. + * The device might not support all we need, and we'll have to + * live with what we get. */ attr->cap.max_send_wr = min_t(u32, attr->cap.max_send_wr, dev->attrs.max_qp_wr); diff --git a/drivers/infiniband/core/user_mad.c b/drivers/infiniband/core/user_mad.c index fd67fc9fe85a46..2f7e3c4483fc57 100644 --- a/drivers/infiniband/core/user_mad.c +++ b/drivers/infiniband/core/user_mad.c @@ -514,7 +514,8 @@ static ssize_t ib_umad_write(struct file *filp, const char __user *buf, struct rdma_ah_attr ah_attr; struct ib_ah *ah; __be64 *tid; - int ret, data_len, hdr_len, copy_offset, rmpp_active; + int ret, hdr_len, copy_offset, rmpp_active; + size_t data_len; u8 base_version; if (count < hdr_size(file) + IB_MGMT_RMPP_HDR) @@ -588,7 +589,10 @@ static ssize_t ib_umad_write(struct file *filp, const char __user *buf, } base_version = ((struct ib_mad_hdr *)&packet->mad.data)->base_version; - data_len = count - hdr_size(file) - hdr_len; + if (check_sub_overflow(count, hdr_size(file) + hdr_len, &data_len)) { + ret = -EINVAL; + goto err_ah; + } packet->msg = ib_create_send_mad(agent, be32_to_cpu(packet->mad.hdr.qpn), packet->mad.hdr.pkey_index, rmpp_active, diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index ce16404cdfb8cc..f4616deeca5453 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -2049,7 +2049,10 @@ static int ib_uverbs_post_send(struct uverbs_attr_bundle *attrs) if (ret) return ret; - user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL); + if (cmd.wqe_size < sizeof(struct ib_uverbs_send_wr)) + return -EINVAL; + + user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL | __GFP_NOWARN); if (!user_wr) return -ENOMEM; @@ -2239,7 +2242,7 @@ ib_uverbs_unmarshall_recv(struct uverbs_req_iter *iter, u32 wr_count, if (ret) return ERR_PTR(ret); - user_wr = kmalloc(wqe_size, GFP_KERNEL); + user_wr = kmalloc(wqe_size, GFP_KERNEL | __GFP_NOWARN); if (!user_wr) return ERR_PTR(-ENOMEM); diff --git a/drivers/infiniband/hw/hns/hns_roce_ah.c b/drivers/infiniband/hw/hns/hns_roce_ah.c index 0c1c32d23c8848..8a605da8a93c97 100644 --- a/drivers/infiniband/hw/hns/hns_roce_ah.c +++ b/drivers/infiniband/hw/hns/hns_roce_ah.c @@ -60,7 +60,7 @@ int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, u8 tclass = get_tclass(grh); u8 priority = 0; u8 tc_mode = 0; - int ret; + int ret = 0; if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 && udata) { ret = -EOPNOTSUPP; @@ -77,19 +77,18 @@ int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, ah->av.flowlabel = grh->flow_label; ah->av.udp_sport = get_ah_udp_sport(ah_attr); ah->av.tclass = tclass; + ah->av.sl = rdma_ah_get_sl(ah_attr); - ret = hr_dev->hw->get_dscp(hr_dev, tclass, &tc_mode, &priority); - if (ret == -EOPNOTSUPP) - ret = 0; - - if (ret && grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) - goto err_out; + if (grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) { + ret = hr_dev->hw->get_dscp(hr_dev, tclass, &tc_mode, &priority); + if (ret == -EOPNOTSUPP) + ret = 0; + else if (ret) + goto err_out; - if (tc_mode == HNAE3_TC_MAP_MODE_DSCP && - grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) - ah->av.sl = priority; - else - ah->av.sl = rdma_ah_get_sl(ah_attr); + if (tc_mode == HNAE3_TC_MAP_MODE_DSCP) + ah->av.sl = priority; + } if (!check_sl_valid(hr_dev, ah->av.sl)) { ret = -EINVAL; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 2d6ae89e525b8a..a2ae4f33e459f3 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -3739,6 +3739,23 @@ static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev, HNS_ROCE_V2_CQ_DEFAULT_INTERVAL); } +static bool left_sw_wc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) +{ + struct hns_roce_qp *hr_qp; + + list_for_each_entry(hr_qp, &hr_cq->sq_list, sq_node) { + if (hr_qp->sq.head != hr_qp->sq.tail) + return true; + } + + list_for_each_entry(hr_qp, &hr_cq->rq_list, rq_node) { + if (hr_qp->rq.head != hr_qp->rq.tail) + return true; + } + + return false; +} + static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) { @@ -3747,6 +3764,12 @@ static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq, struct hns_roce_v2_db cq_db = {}; u32 notify_flag; + if (hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN) { + if ((flags & IB_CQ_REPORT_MISSED_EVENTS) && + left_sw_wc(hr_dev, hr_cq)) + return 1; + return 0; + } /* * flags = 0, then notify_flag : next * flags = 1, then notify flag : solocited @@ -5053,20 +5076,22 @@ static int hns_roce_set_sl(struct ib_qp *ibqp, struct ib_device *ibdev = &hr_dev->ib_dev; int ret; - ret = hns_roce_hw_v2_get_dscp(hr_dev, get_tclass(&attr->ah_attr.grh), - &hr_qp->tc_mode, &hr_qp->priority); - if (ret && ret != -EOPNOTSUPP && - grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) { - ibdev_err_ratelimited(ibdev, - "failed to get dscp, ret = %d.\n", ret); - return ret; - } + hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr); - if (hr_qp->tc_mode == HNAE3_TC_MAP_MODE_DSCP && - grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) - hr_qp->sl = hr_qp->priority; - else - hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr); + if (grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) { + ret = hns_roce_hw_v2_get_dscp(hr_dev, + get_tclass(&attr->ah_attr.grh), + &hr_qp->tc_mode, &hr_qp->priority); + if (ret && ret != -EOPNOTSUPP) { + ibdev_err_ratelimited(ibdev, + "failed to get dscp, ret = %d.\n", + ret); + return ret; + } + + if (hr_qp->tc_mode == HNAE3_TC_MAP_MODE_DSCP) + hr_qp->sl = hr_qp->priority; + } if (!check_sl_valid(hr_dev, hr_qp->sl)) return -EINVAL; @@ -6956,7 +6981,8 @@ static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev) INIT_WORK(&hr_dev->ecc_work, fmea_ram_ecc_work); - hr_dev->irq_workq = alloc_ordered_workqueue("hns_roce_irq_workq", 0); + hr_dev->irq_workq = alloc_ordered_workqueue("hns_roce_irq_workq", + WQ_MEM_RECLAIM); if (!hr_dev->irq_workq) { dev_err(dev, "failed to create irq workqueue.\n"); ret = -ENOMEM; diff --git a/drivers/infiniband/hw/hns/hns_roce_restrack.c b/drivers/infiniband/hw/hns/hns_roce_restrack.c index 230187dda6a07b..085791cc617c4f 100644 --- a/drivers/infiniband/hw/hns/hns_roce_restrack.c +++ b/drivers/infiniband/hw/hns/hns_roce_restrack.c @@ -51,7 +51,7 @@ int hns_roce_fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ib_cq) ret = hr_dev->hw->query_cqc(hr_dev, hr_cq->cqn, &context); if (ret) - return -EINVAL; + return ret; ret = nla_put(msg, RDMA_NLDEV_ATTR_RES_RAW, sizeof(context), &context); @@ -177,7 +177,7 @@ int hns_roce_fill_res_mr_entry_raw(struct sk_buff *msg, struct ib_mr *ib_mr) ret = hr_dev->hw->query_mpt(hr_dev, hr_mr->key, &context); if (ret) - return -EINVAL; + return ret; ret = nla_put(msg, RDMA_NLDEV_ATTR_RES_RAW, sizeof(context), &context); diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 40284bbb45d6dc..99974794100054 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -561,12 +561,20 @@ static int mlx5_query_port_roce(struct ib_device *device, u32 port_num, * of an error it will still be zeroed out. * Use native port in case of reps */ - if (dev->is_rep) - err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN, - 1, 0); - else - err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN, - mdev_port_num, 0); + if (dev->is_rep) { + struct mlx5_eswitch_rep *rep; + + rep = dev->port[port_num - 1].rep; + if (rep) { + mdev = mlx5_eswitch_get_core_dev(rep->esw); + WARN_ON(!mdev); + } + mdev_port_num = 1; + } + + err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN, + mdev_port_num, 0); + if (err) goto out; ext = !!MLX5_GET_ETH_PROTO(ptys_reg, out, true, eth_proto_capability); @@ -2878,7 +2886,6 @@ static void mlx5_ib_handle_event(struct work_struct *_work) container_of(_work, struct mlx5_ib_event_work, work); struct mlx5_ib_dev *ibdev; struct ib_event ibev; - bool fatal = false; if (work->is_slave) { ibdev = mlx5_ib_get_ibdev_from_mpi(work->mpi); @@ -2889,12 +2896,6 @@ static void mlx5_ib_handle_event(struct work_struct *_work) } switch (work->event) { - case MLX5_DEV_EVENT_SYS_ERROR: - ibev.event = IB_EVENT_DEVICE_FATAL; - mlx5_ib_handle_internal_error(ibdev); - ibev.element.port_num = (u8)(unsigned long)work->param; - fatal = true; - break; case MLX5_EVENT_TYPE_PORT_CHANGE: if (handle_port_change(ibdev, work->param, &ibev)) goto out; @@ -2916,8 +2917,6 @@ static void mlx5_ib_handle_event(struct work_struct *_work) if (ibdev->ib_active) ib_dispatch_event(&ibev); - if (fatal) - ibdev->ib_active = false; out: kfree(work); } @@ -2961,6 +2960,66 @@ static int mlx5_ib_event_slave_port(struct notifier_block *nb, return NOTIFY_OK; } +static void mlx5_ib_handle_sys_error_event(struct work_struct *_work) +{ + struct mlx5_ib_event_work *work = + container_of(_work, struct mlx5_ib_event_work, work); + struct mlx5_ib_dev *ibdev = work->dev; + struct ib_event ibev; + + ibev.event = IB_EVENT_DEVICE_FATAL; + mlx5_ib_handle_internal_error(ibdev); + ibev.element.port_num = (u8)(unsigned long)work->param; + ibev.device = &ibdev->ib_dev; + + if (!rdma_is_port_valid(&ibdev->ib_dev, ibev.element.port_num)) { + mlx5_ib_warn(ibdev, "warning: event on port %d\n", ibev.element.port_num); + goto out; + } + + if (ibdev->ib_active) + ib_dispatch_event(&ibev); + + ibdev->ib_active = false; +out: + kfree(work); +} + +static int mlx5_ib_sys_error_event(struct notifier_block *nb, + unsigned long event, void *param) +{ + struct mlx5_ib_event_work *work; + + if (event != MLX5_DEV_EVENT_SYS_ERROR) + return NOTIFY_DONE; + + work = kmalloc(sizeof(*work), GFP_ATOMIC); + if (!work) + return NOTIFY_DONE; + + INIT_WORK(&work->work, mlx5_ib_handle_sys_error_event); + work->dev = container_of(nb, struct mlx5_ib_dev, sys_error_events); + work->is_slave = false; + work->param = param; + work->event = event; + + queue_work(mlx5_ib_event_wq, &work->work); + + return NOTIFY_OK; +} + +static int mlx5_ib_stage_sys_error_notifier_init(struct mlx5_ib_dev *dev) +{ + dev->sys_error_events.notifier_call = mlx5_ib_sys_error_event; + mlx5_notifier_register(dev->mdev, &dev->sys_error_events); + return 0; +} + +static void mlx5_ib_stage_sys_error_notifier_cleanup(struct mlx5_ib_dev *dev) +{ + mlx5_notifier_unregister(dev->mdev, &dev->sys_error_events); +} + static int mlx5_ib_get_plane_num(struct mlx5_core_dev *mdev, u8 *num_plane) { struct mlx5_hca_vport_context vport_ctx; @@ -4466,12 +4525,16 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev) MLX5_HCA_CAP_2_GENERAL_OBJECT_TYPES_RDMA_CTRL) { err = mlx5_ib_init_ucaps(dev); if (err) - return err; + goto err_ucaps; } dev->ib_dev.use_cq_dim = true; return 0; + +err_ucaps: + bitmap_free(dev->var_table.bitmap); + return err; } static const struct ib_device_ops mlx5_ib_dev_port_ops = { @@ -4807,6 +4870,9 @@ static const struct mlx5_ib_profile pf_profile = { STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID, mlx5_ib_devx_init, mlx5_ib_devx_cleanup), + STAGE_CREATE(MLX5_IB_STAGE_SYS_ERROR_NOTIFIER, + mlx5_ib_stage_sys_error_notifier_init, + mlx5_ib_stage_sys_error_notifier_cleanup), STAGE_CREATE(MLX5_IB_STAGE_IB_REG, mlx5_ib_stage_ib_reg_init, mlx5_ib_stage_ib_reg_cleanup), @@ -4864,6 +4930,9 @@ const struct mlx5_ib_profile raw_eth_profile = { STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID, mlx5_ib_devx_init, mlx5_ib_devx_cleanup), + STAGE_CREATE(MLX5_IB_STAGE_SYS_ERROR_NOTIFIER, + mlx5_ib_stage_sys_error_notifier_init, + mlx5_ib_stage_sys_error_notifier_cleanup), STAGE_CREATE(MLX5_IB_STAGE_IB_REG, mlx5_ib_stage_ib_reg_init, mlx5_ib_stage_ib_reg_cleanup), diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 09d82d5f95e354..fbccb0362590bb 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -1007,6 +1007,7 @@ enum mlx5_ib_stages { MLX5_IB_STAGE_BFREG, MLX5_IB_STAGE_PRE_IB_REG_UMR, MLX5_IB_STAGE_WHITELIST_UID, + MLX5_IB_STAGE_SYS_ERROR_NOTIFIER, MLX5_IB_STAGE_IB_REG, MLX5_IB_STAGE_DEVICE_NOTIFIER, MLX5_IB_STAGE_POST_IB_REG_UMR, @@ -1165,6 +1166,7 @@ struct mlx5_ib_dev { /* protect accessing data_direct_dev */ struct mutex data_direct_lock; struct notifier_block mdev_events; + struct notifier_block sys_error_events; struct notifier_block lag_events; int num_ports; /* serialize update of capability mask diff --git a/drivers/infiniband/hw/mlx5/std_types.c b/drivers/infiniband/hw/mlx5/std_types.c index 2fcf553044e15e..1ee31611b4b3f1 100644 --- a/drivers/infiniband/hw/mlx5/std_types.c +++ b/drivers/infiniband/hw/mlx5/std_types.c @@ -195,7 +195,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_GET_DATA_DIRECT_SYSFS_PATH)( int out_len = uverbs_attr_get_len(attrs, MLX5_IB_ATTR_GET_DATA_DIRECT_SYSFS_PATH); u32 dev_path_len; - char *dev_path; + char *dev_path = NULL; int ret; c = to_mucontext(ib_uverbs_get_ucontext(attrs)); @@ -223,9 +223,9 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_GET_DATA_DIRECT_SYSFS_PATH)( ret = uverbs_copy_to(attrs, MLX5_IB_ATTR_GET_DATA_DIRECT_SYSFS_PATH, dev_path, dev_path_len); - kfree(dev_path); end: + kfree(dev_path); mutex_unlock(&dev->data_direct_lock); return ret; } diff --git a/drivers/infiniband/sw/rxe/rxe_comp.c b/drivers/infiniband/sw/rxe/rxe_comp.c index a5b2b62f596b05..1390e861bd1d74 100644 --- a/drivers/infiniband/sw/rxe/rxe_comp.c +++ b/drivers/infiniband/sw/rxe/rxe_comp.c @@ -119,12 +119,15 @@ void retransmit_timer(struct timer_list *t) rxe_dbg_qp(qp, "retransmit timer fired\n"); + if (!rxe_get(qp)) + return; spin_lock_irqsave(&qp->state_lock, flags); if (qp->valid) { qp->comp.timeout = 1; rxe_sched_task(&qp->send_task); } spin_unlock_irqrestore(&qp->state_lock, flags); + rxe_put(qp); } void rxe_comp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb) diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c index b1df0523884881..8116cf0fa6da35 100644 --- a/drivers/infiniband/sw/rxe/rxe_mr.c +++ b/drivers/infiniband/sw/rxe/rxe_mr.c @@ -72,14 +72,46 @@ void rxe_mr_init_dma(int access, struct rxe_mr *mr) mr->ibmr.type = IB_MR_TYPE_DMA; } +/* + * Convert iova to page_info index. The page_info stores pages of size + * PAGE_SIZE, but MRs can have different page sizes. This function + * handles the conversion for all cases: + * + * 1. mr->page_size > PAGE_SIZE: + * The MR's iova may not be aligned to mr->page_size. We use the + * aligned base (iova & page_mask) as reference, then calculate + * which PAGE_SIZE sub-page the iova falls into. + * + * 2. mr->page_size <= PAGE_SIZE: + * Use simple shift arithmetic since each page_info entry corresponds + * to one or more MR pages. + */ static unsigned long rxe_mr_iova_to_index(struct rxe_mr *mr, u64 iova) { - return (iova >> mr->page_shift) - (mr->ibmr.iova >> mr->page_shift); + int idx; + + if (mr_page_size(mr) > PAGE_SIZE) + idx = (iova - (mr->ibmr.iova & mr->page_mask)) >> PAGE_SHIFT; + else + idx = (iova >> mr->page_shift) - + (mr->ibmr.iova >> mr->page_shift); + + WARN_ON(idx >= mr->nbuf); + return idx; } +/* + * Convert iova to offset within the page_info entry. + * + * For mr_page_size > PAGE_SIZE, the offset is within the system page. + * For mr_page_size <= PAGE_SIZE, the offset is within the MR page size. + */ static unsigned long rxe_mr_iova_to_page_offset(struct rxe_mr *mr, u64 iova) { - return iova & (mr_page_size(mr) - 1); + if (mr_page_size(mr) > PAGE_SIZE) + return iova & (PAGE_SIZE - 1); + else + return iova & (mr_page_size(mr) - 1); } static bool is_pmem_page(struct page *pg) @@ -93,37 +125,69 @@ static bool is_pmem_page(struct page *pg) static int rxe_mr_fill_pages_from_sgt(struct rxe_mr *mr, struct sg_table *sgt) { - XA_STATE(xas, &mr->page_list, 0); struct sg_page_iter sg_iter; struct page *page; bool persistent = !!(mr->access & IB_ACCESS_FLUSH_PERSISTENT); + WARN_ON(mr_page_size(mr) != PAGE_SIZE); + __sg_page_iter_start(&sg_iter, sgt->sgl, sgt->orig_nents, 0); if (!__sg_page_iter_next(&sg_iter)) return 0; - do { - xas_lock(&xas); - while (true) { - page = sg_page_iter_page(&sg_iter); - - if (persistent && !is_pmem_page(page)) { - rxe_dbg_mr(mr, "Page can't be persistent\n"); - xas_set_err(&xas, -EINVAL); - break; - } + while (true) { + page = sg_page_iter_page(&sg_iter); - xas_store(&xas, page); - if (xas_error(&xas)) - break; - xas_next(&xas); - if (!__sg_page_iter_next(&sg_iter)) - break; + if (persistent && !is_pmem_page(page)) { + rxe_dbg_mr(mr, "Page can't be persistent\n"); + return -EINVAL; } - xas_unlock(&xas); - } while (xas_nomem(&xas, GFP_KERNEL)); - return xas_error(&xas); + mr->page_info[mr->nbuf].page = page; + mr->page_info[mr->nbuf].offset = 0; + mr->nbuf++; + + if (!__sg_page_iter_next(&sg_iter)) + break; + } + + return 0; +} + +static int __alloc_mr_page_info(struct rxe_mr *mr, int num_pages) +{ + mr->page_info = kcalloc(num_pages, sizeof(struct rxe_mr_page), + GFP_KERNEL); + if (!mr->page_info) + return -ENOMEM; + + mr->max_allowed_buf = num_pages; + mr->nbuf = 0; + + return 0; +} + +static int alloc_mr_page_info(struct rxe_mr *mr, int num_pages) +{ + int ret; + + WARN_ON(mr->num_buf); + ret = __alloc_mr_page_info(mr, num_pages); + if (ret) + return ret; + + mr->num_buf = num_pages; + + return 0; +} + +static void free_mr_page_info(struct rxe_mr *mr) +{ + if (!mr->page_info) + return; + + kfree(mr->page_info); + mr->page_info = NULL; } int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, @@ -134,8 +198,6 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, rxe_mr_init(access, mr); - xa_init(&mr->page_list); - umem = ib_umem_get(&rxe->ib_dev, start, length, access); if (IS_ERR(umem)) { rxe_dbg_mr(mr, "Unable to pin memory region err = %d\n", @@ -143,46 +205,24 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, return PTR_ERR(umem); } + err = alloc_mr_page_info(mr, ib_umem_num_pages(umem)); + if (err) + goto err2; + err = rxe_mr_fill_pages_from_sgt(mr, &umem->sgt_append.sgt); - if (err) { - ib_umem_release(umem); - return err; - } + if (err) + goto err1; mr->umem = umem; mr->ibmr.type = IB_MR_TYPE_USER; mr->state = RXE_MR_STATE_VALID; return 0; -} - -static int rxe_mr_alloc(struct rxe_mr *mr, int num_buf) -{ - XA_STATE(xas, &mr->page_list, 0); - int i = 0; - int err; - - xa_init(&mr->page_list); - - do { - xas_lock(&xas); - while (i != num_buf) { - xas_store(&xas, XA_ZERO_ENTRY); - if (xas_error(&xas)) - break; - xas_next(&xas); - i++; - } - xas_unlock(&xas); - } while (xas_nomem(&xas, GFP_KERNEL)); - - err = xas_error(&xas); - if (err) - return err; - - mr->num_buf = num_buf; - - return 0; +err1: + free_mr_page_info(mr); +err2: + ib_umem_release(umem); + return err; } int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr) @@ -192,7 +232,7 @@ int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr) /* always allow remote access for FMRs */ rxe_mr_init(RXE_ACCESS_REMOTE, mr); - err = rxe_mr_alloc(mr, max_pages); + err = alloc_mr_page_info(mr, max_pages); if (err) goto err1; @@ -205,26 +245,43 @@ int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr) return err; } +/* + * I) MRs with page_size >= PAGE_SIZE, + * Split a large MR page (mr->page_size) into multiple PAGE_SIZE + * sub-pages and store them in page_info, offset is always 0. + * + * Called when mr->page_size > PAGE_SIZE. Each call to rxe_set_page() + * represents one mr->page_size region, which we must split into + * (mr->page_size >> PAGE_SHIFT) individual pages. + * + * II) MRs with page_size < PAGE_SIZE, + * Save each PAGE_SIZE page and its offset within the system page in page_info. + */ static int rxe_set_page(struct ib_mr *ibmr, u64 dma_addr) { struct rxe_mr *mr = to_rmr(ibmr); - struct page *page = ib_virt_dma_to_page(dma_addr); bool persistent = !!(mr->access & IB_ACCESS_FLUSH_PERSISTENT); - int err; + u32 i, pages_per_mr = mr_page_size(mr) >> PAGE_SHIFT; - if (persistent && !is_pmem_page(page)) { - rxe_dbg_mr(mr, "Page cannot be persistent\n"); - return -EINVAL; - } + pages_per_mr = MAX(1, pages_per_mr); - if (unlikely(mr->nbuf == mr->num_buf)) - return -ENOMEM; + for (i = 0; i < pages_per_mr; i++) { + u64 addr = dma_addr + i * PAGE_SIZE; + struct page *sub_page = ib_virt_dma_to_page(addr); - err = xa_err(xa_store(&mr->page_list, mr->nbuf, page, GFP_KERNEL)); - if (err) - return err; + if (unlikely(mr->nbuf >= mr->max_allowed_buf)) + return -ENOMEM; + + if (persistent && !is_pmem_page(sub_page)) { + rxe_dbg_mr(mr, "Page cannot be persistent\n"); + return -EINVAL; + } + + mr->page_info[mr->nbuf].page = sub_page; + mr->page_info[mr->nbuf].offset = addr & (PAGE_SIZE - 1); + mr->nbuf++; + } - mr->nbuf++; return 0; } @@ -234,6 +291,31 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sgl, struct rxe_mr *mr = to_rmr(ibmr); unsigned int page_size = mr_page_size(mr); + /* + * Ensure page_size and PAGE_SIZE are compatible for mapping. + * We require one to be a multiple of the other for correct + * iova-to-page conversion. + */ + if (!IS_ALIGNED(page_size, PAGE_SIZE) && + !IS_ALIGNED(PAGE_SIZE, page_size)) { + rxe_dbg_mr(mr, "MR page size %u must be compatible with PAGE_SIZE %lu\n", + page_size, PAGE_SIZE); + return -EINVAL; + } + + if (mr_page_size(mr) > PAGE_SIZE) { + /* resize page_info if needed */ + u32 map_mr_pages = (page_size >> PAGE_SHIFT) * mr->num_buf; + + if (map_mr_pages > mr->max_allowed_buf) { + rxe_dbg_mr(mr, "requested pages %u exceed max %u\n", + map_mr_pages, mr->max_allowed_buf); + free_mr_page_info(mr); + if (__alloc_mr_page_info(mr, map_mr_pages)) + return -ENOMEM; + } + } + mr->nbuf = 0; mr->page_shift = ilog2(page_size); mr->page_mask = ~((u64)page_size - 1); @@ -245,30 +327,30 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sgl, static int rxe_mr_copy_xarray(struct rxe_mr *mr, u64 iova, void *addr, unsigned int length, enum rxe_mr_copy_dir dir) { - unsigned int page_offset = rxe_mr_iova_to_page_offset(mr, iova); - unsigned long index = rxe_mr_iova_to_index(mr, iova); unsigned int bytes; - struct page *page; - void *va; + u8 *va; while (length) { - page = xa_load(&mr->page_list, index); - if (!page) + unsigned long index = rxe_mr_iova_to_index(mr, iova); + struct rxe_mr_page *info = &mr->page_info[index]; + unsigned int page_offset = rxe_mr_iova_to_page_offset(mr, iova); + + if (!info->page) return -EFAULT; - bytes = min_t(unsigned int, length, - mr_page_size(mr) - page_offset); - va = kmap_local_page(page); + page_offset += info->offset; + bytes = min_t(unsigned int, length, PAGE_SIZE - page_offset); + va = kmap_local_page(info->page); + if (dir == RXE_FROM_MR_OBJ) memcpy(addr, va + page_offset, bytes); else memcpy(va + page_offset, addr, bytes); kunmap_local(va); - page_offset = 0; addr += bytes; + iova += bytes; length -= bytes; - index++; } return 0; @@ -426,9 +508,6 @@ int copy_data( static int rxe_mr_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length) { - unsigned int page_offset; - unsigned long index; - struct page *page; unsigned int bytes; int err; u8 *va; @@ -438,15 +517,17 @@ static int rxe_mr_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int leng return err; while (length > 0) { - index = rxe_mr_iova_to_index(mr, iova); - page = xa_load(&mr->page_list, index); - page_offset = rxe_mr_iova_to_page_offset(mr, iova); - if (!page) + unsigned long index = rxe_mr_iova_to_index(mr, iova); + struct rxe_mr_page *info = &mr->page_info[index]; + unsigned int page_offset = rxe_mr_iova_to_page_offset(mr, iova); + + if (!info->page) return -EFAULT; - bytes = min_t(unsigned int, length, - mr_page_size(mr) - page_offset); - va = kmap_local_page(page); + page_offset += info->offset; + bytes = min_t(unsigned int, length, PAGE_SIZE - page_offset); + + va = kmap_local_page(info->page); arch_wb_cache_pmem(va + page_offset, bytes); kunmap_local(va); @@ -501,6 +582,7 @@ enum resp_states rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, } else { unsigned long index; int err; + struct rxe_mr_page *info; err = mr_check_range(mr, iova, sizeof(value)); if (err) { @@ -509,9 +591,12 @@ enum resp_states rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, } page_offset = rxe_mr_iova_to_page_offset(mr, iova); index = rxe_mr_iova_to_index(mr, iova); - page = xa_load(&mr->page_list, index); - if (!page) + info = &mr->page_info[index]; + if (!info->page) return RESPST_ERR_RKEY_VIOLATION; + + page_offset += info->offset; + page = info->page; } if (unlikely(page_offset & 0x7)) { @@ -550,6 +635,7 @@ enum resp_states rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value) } else { unsigned long index; int err; + struct rxe_mr_page *info; /* See IBA oA19-28 */ err = mr_check_range(mr, iova, sizeof(value)); @@ -559,9 +645,12 @@ enum resp_states rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value) } page_offset = rxe_mr_iova_to_page_offset(mr, iova); index = rxe_mr_iova_to_index(mr, iova); - page = xa_load(&mr->page_list, index); - if (!page) + info = &mr->page_info[index]; + if (!info->page) return RESPST_ERR_RKEY_VIOLATION; + + page_offset += info->offset; + page = info->page; } /* See IBA A19.4.2 */ @@ -725,5 +814,5 @@ void rxe_mr_cleanup(struct rxe_pool_elem *elem) ib_umem_release(mr->umem); if (mr->ibmr.type != IB_MR_TYPE_DMA) - xa_destroy(&mr->page_list); + free_mr_page_info(mr); } diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c index 373b03f223bebc..12d03f390b0978 100644 --- a/drivers/infiniband/sw/rxe/rxe_req.c +++ b/drivers/infiniband/sw/rxe/rxe_req.c @@ -102,6 +102,8 @@ void rnr_nak_timer(struct timer_list *t) rxe_dbg_qp(qp, "nak timer fired\n"); + if (!rxe_get(qp)) + return; spin_lock_irqsave(&qp->state_lock, flags); if (qp->valid) { /* request a send queue retry */ @@ -110,6 +112,7 @@ void rnr_nak_timer(struct timer_list *t) rxe_sched_task(&qp->send_task); } spin_unlock_irqrestore(&qp->state_lock, flags); + rxe_put(qp); } static void req_check_sq_drain_done(struct rxe_qp *qp) diff --git a/drivers/infiniband/sw/rxe/rxe_srq.c b/drivers/infiniband/sw/rxe/rxe_srq.c index 2a234f26ac1044..c9a7cd38953d31 100644 --- a/drivers/infiniband/sw/rxe/rxe_srq.c +++ b/drivers/infiniband/sw/rxe/rxe_srq.c @@ -77,9 +77,6 @@ int rxe_srq_from_init(struct rxe_dev *rxe, struct rxe_srq *srq, goto err_free; } - srq->rq.queue = q; - init->attr.max_wr = srq->rq.max_wr; - if (uresp) { if (copy_to_user(&uresp->srq_num, &srq->srq_num, sizeof(uresp->srq_num))) { @@ -88,6 +85,9 @@ int rxe_srq_from_init(struct rxe_dev *rxe, struct rxe_srq *srq, } } + srq->rq.queue = q; + init->attr.max_wr = srq->rq.max_wr; + return 0; err_free: diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h index fd48075810dd10..1b8ed1031bd57f 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.h +++ b/drivers/infiniband/sw/rxe/rxe_verbs.h @@ -335,6 +335,11 @@ static inline int rkey_is_mw(u32 rkey) return (index >= RXE_MIN_MW_INDEX) && (index <= RXE_MAX_MW_INDEX); } +struct rxe_mr_page { + struct page *page; + unsigned int offset; /* offset in system page */ +}; + struct rxe_mr { struct rxe_pool_elem elem; struct ib_mr ibmr; @@ -351,10 +356,13 @@ struct rxe_mr { unsigned int page_shift; u64 page_mask; + /* size of page_info when mr allocated */ u32 num_buf; + /* real size of page_info */ + u32 max_allowed_buf; u32 nbuf; - struct xarray page_list; + struct rxe_mr_page *page_info; }; static inline unsigned int mr_page_size(struct rxe_mr *mr) diff --git a/drivers/infiniband/sw/siw/siw_qp_rx.c b/drivers/infiniband/sw/siw/siw_qp_rx.c index a10820e3388782..e8a88b378d51d4 100644 --- a/drivers/infiniband/sw/siw/siw_qp_rx.c +++ b/drivers/infiniband/sw/siw/siw_qp_rx.c @@ -1435,7 +1435,8 @@ int siw_tcp_rx_data(read_descriptor_t *rd_desc, struct sk_buff *skb, } if (unlikely(rv != 0 && rv != -EAGAIN)) { if ((srx->state > SIW_GET_HDR || - qp->rx_fpdu->more_ddp_segs) && run_completion) + (qp->rx_fpdu && qp->rx_fpdu->more_ddp_segs)) && + run_completion) siw_rdmap_complete(qp, rv); siw_dbg_qp(qp, "rx error %d, rx state %d\n", rv, diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.c b/drivers/infiniband/ulp/rtrs/rtrs-srv.c index 9ecc6343455d6c..adb798e2a54ae1 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-srv.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.c @@ -208,7 +208,6 @@ static int rdma_write_sg(struct rtrs_srv_op *id) size_t sg_cnt; int err, offset; bool need_inval; - u32 rkey = 0; struct ib_reg_wr rwr; struct ib_sge *plist; struct ib_sge list; @@ -240,11 +239,6 @@ static int rdma_write_sg(struct rtrs_srv_op *id) wr->wr.num_sge = 1; wr->remote_addr = le64_to_cpu(id->rd_msg->desc[0].addr); wr->rkey = le32_to_cpu(id->rd_msg->desc[0].key); - if (rkey == 0) - rkey = wr->rkey; - else - /* Only one key is actually used */ - WARN_ON_ONCE(rkey != wr->rkey); wr->wr.opcode = IB_WR_RDMA_WRITE; wr->wr.wr_cqe = &io_comp_cqe; @@ -277,7 +271,7 @@ static int rdma_write_sg(struct rtrs_srv_op *id) inv_wr.opcode = IB_WR_SEND_WITH_INV; inv_wr.wr_cqe = &io_comp_cqe; inv_wr.send_flags = 0; - inv_wr.ex.invalidate_rkey = rkey; + inv_wr.ex.invalidate_rkey = wr->rkey; } imm_wr.wr.next = NULL; @@ -601,7 +595,7 @@ static int map_cont_bufs(struct rtrs_srv_path *srv_path) srv_path->mrs_num++) { struct rtrs_srv_mr *srv_mr = &srv_path->mrs[srv_path->mrs_num]; struct scatterlist *s; - int nr, nr_sgt, chunks; + int nr, nr_sgt, chunks, ind; sgt = &srv_mr->sgt; chunks = chunks_per_mr * srv_path->mrs_num; @@ -631,7 +625,7 @@ static int map_cont_bufs(struct rtrs_srv_path *srv_path) } nr = ib_map_mr_sg(mr, sgt->sgl, nr_sgt, NULL, max_chunk_size); - if (nr != nr_sgt) { + if (nr < nr_sgt) { err = nr < 0 ? nr : -EINVAL; goto dereg_mr; } @@ -647,9 +641,24 @@ static int map_cont_bufs(struct rtrs_srv_path *srv_path) goto dereg_mr; } } - /* Eventually dma addr for each chunk can be cached */ - for_each_sg(sgt->sgl, s, nr_sgt, i) - srv_path->dma_addr[chunks + i] = sg_dma_address(s); + + /* + * Cache DMA addresses by traversing sg entries. If + * regions were merged, an inner loop is required to + * populate the DMA address array by traversing larger + * regions. + */ + ind = chunks; + for_each_sg(sgt->sgl, s, nr_sgt, i) { + unsigned int dma_len = sg_dma_len(s); + u64 dma_addr = sg_dma_address(s); + u64 dma_addr_end = dma_addr + dma_len; + + do { + srv_path->dma_addr[ind++] = dma_addr; + dma_addr += max_chunk_size; + } while (dma_addr < dma_addr_end); + } ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey)); srv_mr->mr = mr; diff --git a/drivers/input/misc/Kconfig b/drivers/input/misc/Kconfig index 94a753fcb64fa5..c327a21b418ce5 100644 --- a/drivers/input/misc/Kconfig +++ b/drivers/input/misc/Kconfig @@ -1003,4 +1003,15 @@ config INPUT_STPMIC1_ONKEY To compile this driver as a module, choose M here: the module will be called stpmic1_onkey. +config INPUT_MACSMC_INPUT + tristate "Apple Mac SMC lid/buttons" + depends on MFD_MACSMC + help + Say Y here if you want to use the input events delivered via the + SMC controller on Apple Mac machines using the macsmc driver. + This includes lid open/close and the power button. + + To compile this driver as a module, choose M here: the + module will be called macsmc-input. + endif diff --git a/drivers/input/misc/Makefile b/drivers/input/misc/Makefile index 415fc4e2918be8..aa148f6a0f3233 100644 --- a/drivers/input/misc/Makefile +++ b/drivers/input/misc/Makefile @@ -52,6 +52,7 @@ obj-$(CONFIG_INPUT_IQS7222) += iqs7222.o obj-$(CONFIG_INPUT_KEYSPAN_REMOTE) += keyspan_remote.o obj-$(CONFIG_INPUT_KXTJ9) += kxtj9.o obj-$(CONFIG_INPUT_M68K_BEEP) += m68kspkr.o +obj-$(CONFIG_INPUT_MACSMC_INPUT) += macsmc-input.o obj-$(CONFIG_INPUT_MAX7360_ROTARY) += max7360-rotary.o obj-$(CONFIG_INPUT_MAX77650_ONKEY) += max77650-onkey.o obj-$(CONFIG_INPUT_MAX77693_HAPTIC) += max77693-haptic.o diff --git a/drivers/input/misc/macsmc-input.c b/drivers/input/misc/macsmc-input.c new file mode 100644 index 00000000000000..2cead3b7f45fed --- /dev/null +++ b/drivers/input/misc/macsmc-input.c @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SMC input event driver + * Copyright The Asahi Linux Contributors + * + * This driver exposes HID events from the SMC as an input device. + * This includes the lid open/close and power button notifications. + */ + +#include +#include +#include +#include +#include +#include + +/** + * struct macsmc_input + * @dev: Underlying struct device for the input sub-device + * @smc: Pointer to apple_smc struct of the mfd parent + * @input: Allocated input_dev; devres managed + * @nb: Notifier block used for incoming events from SMC (e.g. button pressed down) + * @wakeup_mode: Set to true when system is suspended and power button events should wake it + */ +struct macsmc_input { + struct device *dev; + struct apple_smc *smc; + struct input_dev *input; + struct notifier_block nb; + bool wakeup_mode; +}; + +#define SMC_EV_BTN 0x7201 +#define SMC_EV_LID 0x7203 + +#define BTN_POWER 0x01 /* power button on e.g. Mac Mini chasis pressed */ +#define BTN_TOUCHID 0x06 /* combined TouchID / power button on MacBooks pressed */ +#define BTN_POWER_HELD_SHORT 0xfe /* power button briefly held down */ +#define BTN_POWER_HELD_LONG 0x00 /* power button held down; sent just before forced poweroff */ + +static void macsmc_input_event_button(struct macsmc_input *smcin, unsigned long event) +{ + u8 button = (event >> 8) & 0xff; + u8 state = !!(event & 0xff); + + switch (button) { + case BTN_POWER: + case BTN_TOUCHID: + pm_wakeup_dev_event(smcin->dev, 0, (smcin->wakeup_mode && state)); + /* + * Suppress KEY_POWER reports when suspended to avoid powering down + * immediately after waking from s2idle. + * */ + if (smcin->wakeup_mode) + return; + + input_report_key(smcin->input, KEY_POWER, state); + input_sync(smcin->input); + break; + case BTN_POWER_HELD_SHORT: /* power button held down; ignore */ + break; + case BTN_POWER_HELD_LONG: + /* + * If we get here the power button has been held down for a while and + * we have about 4 seconds before forced power-off is triggered by SMC. + * Try to do an emergency shutdown to make sure the NVMe cache is + * flushed. macOS actually does this by panicing (!)... + */ + if (state) { + dev_crit(smcin->dev, "Triggering forced shutdown!\n"); + if (kernel_can_power_off()) + kernel_power_off(); + else /* Missing macsmc-reboot driver? */ + kernel_restart("SMC power button triggered restart"); + } + break; + default: + dev_warn(smcin->dev, "Unknown SMC button event: %04lx\n", event & 0xffff); + } +} + +static void macsmc_input_event_lid(struct macsmc_input *smcin, unsigned long event) +{ + u8 lid_state = !!((event >> 8) & 0xff); + + pm_wakeup_dev_event(smcin->dev, 0, (smcin->wakeup_mode && !lid_state)); + input_report_switch(smcin->input, SW_LID, lid_state); + input_sync(smcin->input); +} + +static int macsmc_input_event(struct notifier_block *nb, unsigned long event, void *data) +{ + struct macsmc_input *smcin = container_of(nb, struct macsmc_input, nb); + u16 type = event >> 16; + + switch (type) { + case SMC_EV_BTN: + macsmc_input_event_button(smcin, event); + return NOTIFY_OK; + case SMC_EV_LID: + macsmc_input_event_lid(smcin, event); + return NOTIFY_OK; + default: + /* SMC event meant for another driver */ + return NOTIFY_DONE; + } +} + +static int macsmc_input_probe(struct platform_device *pdev) +{ + struct apple_smc *smc = dev_get_drvdata(pdev->dev.parent); + struct macsmc_input *smcin; + bool have_lid, have_power; + int error; + + /* Bail early if this SMC neither supports power button nor lid events */ + have_lid = apple_smc_key_exists(smc, SMC_KEY(MSLD)); + have_power = apple_smc_key_exists(smc, SMC_KEY(bHLD)); + if (!have_lid && !have_power) + return -ENODEV; + + smcin = devm_kzalloc(&pdev->dev, sizeof(*smcin), GFP_KERNEL); + if (!smcin) + return -ENOMEM; + + smcin->dev = &pdev->dev; + smcin->smc = smc; + platform_set_drvdata(pdev, smcin); + + smcin->input = devm_input_allocate_device(&pdev->dev); + if (!smcin->input) + return -ENOMEM; + + smcin->input->phys = "macsmc-input (0)"; + smcin->input->name = "Apple SMC power/lid events"; + + if (have_lid) + input_set_capability(smcin->input, EV_SW, SW_LID); + if (have_power) + input_set_capability(smcin->input, EV_KEY, KEY_POWER); + + if (have_lid) { + u8 val; + + error = apple_smc_read_u8(smc, SMC_KEY(MSLD), &val); + if (error < 0) + dev_warn(&pdev->dev, "Failed to read initial lid state\n"); + else + input_report_switch(smcin->input, SW_LID, val); + } + + if (have_power) { + u32 val; + + error = apple_smc_read_u32(smc, SMC_KEY(bHLD), &val); + if (error < 0) + dev_warn(&pdev->dev, "Failed to read initial power button state\n"); + else + input_report_key(smcin->input, KEY_POWER, val & 1); + } + + error = input_register_device(smcin->input); + if (error) { + dev_err(&pdev->dev, "Failed to register input device: %d\n", error); + return error; + } + + input_sync(smcin->input); + + smcin->nb.notifier_call = macsmc_input_event; + blocking_notifier_chain_register(&smc->event_handlers, &smcin->nb); + + device_init_wakeup(&pdev->dev, true); + + return 0; +} + +static int macsmc_input_pm_prepare(struct device *dev) +{ + struct macsmc_input *smcin = dev_get_drvdata(dev); + + smcin->wakeup_mode = true; + return 0; +} + +static void macsmc_input_pm_complete(struct device *dev) +{ + struct macsmc_input *smcin = dev_get_drvdata(dev); + + smcin->wakeup_mode = false; +} + +static const struct dev_pm_ops macsmc_input_pm_ops = { + .prepare = macsmc_input_pm_prepare, + .complete = macsmc_input_pm_complete, +}; + +static struct platform_driver macsmc_input_driver = { + .driver = { + .name = "macsmc-input", + .pm = &macsmc_input_pm_ops, + }, + .probe = macsmc_input_probe, +}; +module_platform_driver(macsmc_input_driver); + +MODULE_AUTHOR("Hector Martin "); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple SMC input driver"); +MODULE_ALIAS("platform:macsmc-input"); diff --git a/drivers/interconnect/mediatek/icc-emi.c b/drivers/interconnect/mediatek/icc-emi.c index 7da740b5fa8d62..dfa3a9cd939983 100644 --- a/drivers/interconnect/mediatek/icc-emi.c +++ b/drivers/interconnect/mediatek/icc-emi.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -22,7 +23,9 @@ static int mtk_emi_icc_aggregate(struct icc_node *node, u32 tag, u32 avg_bw, { struct mtk_icc_node *in = node->data; - *agg_avg += avg_bw; + if (check_add_overflow(*agg_avg, avg_bw, agg_avg)) + *agg_avg = U32_MAX; + *agg_peak = max_t(u32, *agg_peak, peak_bw); in->sum_avg = *agg_avg; @@ -40,7 +43,7 @@ static int mtk_emi_icc_set(struct icc_node *src, struct icc_node *dst) if (unlikely(!src->provider)) return -EINVAL; - dev = src->provider->dev; + dev = src->provider->dev->parent; switch (node->ep) { case 0: @@ -97,7 +100,7 @@ int mtk_emi_icc_probe(struct platform_device *pdev) if (!data) return -ENOMEM; - provider->dev = pdev->dev.parent; + provider->dev = dev; provider->set = mtk_emi_icc_set; provider->aggregate = mtk_emi_icc_aggregate; provider->xlate = of_icc_xlate_onecell; diff --git a/drivers/interconnect/qcom/qcs8300.c b/drivers/interconnect/qcom/qcs8300.c index 70a377bbcf2930..bc403a9bf68c65 100644 --- a/drivers/interconnect/qcom/qcs8300.c +++ b/drivers/interconnect/qcom/qcs8300.c @@ -629,7 +629,7 @@ static struct qcom_icc_node qxm_nsp = { .name = "qxm_nsp", .channels = 2, .buswidth = 32, - .num_links = 1, + .num_links = 2, .link_nodes = { &qns_hcp, &qns_nsp_gemnoc }, }; diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index b742ef1adb352b..df1c238dc88855 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -15,7 +15,6 @@ irqreturn_t amd_iommu_int_thread(int irq, void *data); irqreturn_t amd_iommu_int_thread_evtlog(int irq, void *data); irqreturn_t amd_iommu_int_thread_pprlog(int irq, void *data); irqreturn_t amd_iommu_int_thread_galog(int irq, void *data); -irqreturn_t amd_iommu_int_handler(int irq, void *data); void amd_iommu_restart_log(struct amd_iommu *iommu, const char *evt_type, u8 cntrl_intr, u8 cntrl_log, u32 status_run_mask, u32 status_overflow_mask); diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 384c90b4f90a0a..62a7a718acf8fb 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -2356,12 +2356,8 @@ static int iommu_setup_msi(struct amd_iommu *iommu) if (r) return r; - r = request_threaded_irq(iommu->dev->irq, - amd_iommu_int_handler, - amd_iommu_int_thread, - 0, "AMD-Vi", - iommu); - + r = request_threaded_irq(iommu->dev->irq, NULL, amd_iommu_int_thread, + IRQF_ONESHOT, "AMD-Vi", iommu); if (r) { pci_disable_msi(iommu->dev); return r; @@ -2535,8 +2531,8 @@ static int __iommu_setup_intcapxt(struct amd_iommu *iommu, const char *devname, return irq; } - ret = request_threaded_irq(irq, amd_iommu_int_handler, - thread_fn, 0, devname, iommu); + ret = request_threaded_irq(irq, NULL, thread_fn, IRQF_ONESHOT, devname, + iommu); if (ret) { irq_domain_free_irqs(irq, 1); irq_domain_remove(domain); diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 7c12be1b247f43..0f9045ce93af18 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1151,11 +1151,6 @@ irqreturn_t amd_iommu_int_thread(int irq, void *data) return IRQ_HANDLED; } -irqreturn_t amd_iommu_int_handler(int irq, void *data) -{ - return IRQ_WAKE_THREAD; -} - /**************************************************************************** * * IOMMU command queuing functions diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 83a5aabcd15d97..1663ce5279a8fe 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -28,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -201,6 +203,7 @@ struct apple_dart_hw { * @lock: lock for hardware operations involving this dart * @pgsize: pagesize supported by this DART * @supports_bypass: indicates if this DART supports bypass mode + * @locked: indicates if this DART is locked * @sid2group: maps stream ids to iommu_groups * @iommu: iommu core device */ @@ -222,12 +225,18 @@ struct apple_dart { u32 num_streams; u32 supports_bypass : 1; u32 four_level : 1; + u32 locked : 1; + + dma_addr_t dma_min; + dma_addr_t dma_max; struct iommu_group *sid2group[DART_MAX_STREAMS]; struct iommu_device iommu; u32 save_tcr[DART_MAX_STREAMS]; u32 save_ttbr[DART_MAX_STREAMS][DART_MAX_TTBR]; + + u64 *locked_ttbr[DART_MAX_STREAMS][DART_MAX_TTBR]; }; /* @@ -267,6 +276,7 @@ struct apple_dart_domain { struct io_pgtable_ops *pgtbl_ops; bool finalized; + u64 mask; struct mutex init_lock; struct apple_dart_atomic_stream_map stream_maps[MAX_DARTS_PER_DEVICE]; @@ -284,6 +294,7 @@ struct apple_dart_domain { struct apple_dart_master_cfg { /* Intersection of DART capabilitles */ u32 supports_bypass : 1; + u32 locked : 1; struct apple_dart_stream_map stream_maps[MAX_DARTS_PER_DEVICE]; }; @@ -378,6 +389,82 @@ apple_dart_hw_clear_all_ttbrs(struct apple_dart_stream_map *stream_map) apple_dart_hw_clear_ttbr(stream_map, i); } +static int +apple_dart_hw_map_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) +{ + struct apple_dart *dart = stream_map->dart; + int sid; + + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { + u32 ttbr; + phys_addr_t phys; + u64 *l1_tbl; + + ttbr = readl(dart->regs + DART_TTBR(dart, sid, idx)); + + if (!(ttbr & dart->hw->ttbr_valid)) { + dev_err(dart->dev, "Invalid ttbr[%u] for locked dart\n", + idx); + return -EIO; + } + + ttbr &= ~dart->hw->ttbr_valid; + + if (dart->hw->ttbr_addr_field_shift) + ttbr >>= dart->hw->ttbr_addr_field_shift; + phys = ((phys_addr_t) ttbr) << dart->hw->ttbr_shift; + + l1_tbl = devm_memremap(dart->dev, phys, dart->pgsize, + MEMREMAP_WB); + if (!l1_tbl) + return -ENOMEM; + + dart->locked_ttbr[sid][idx] = l1_tbl; + } + + return 0; +} + +static int +apple_dart_hw_unmap_locked_ttbr(struct apple_dart_stream_map *stream_map, + u8 idx) +{ + struct apple_dart *dart = stream_map->dart; + int sid; + + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { + /* TODO: locked L1 table might need to be restored to boot state */ + if (dart->locked_ttbr[sid][idx]) { + memset(dart->locked_ttbr[sid][idx], 0, dart->pgsize); + devm_memunmap(dart->dev, dart->locked_ttbr[sid][idx]); + } + dart->locked_ttbr[sid][idx] = NULL; + } + + return 0; +} + +static int +apple_dart_hw_sync_locked(struct io_pgtable_cfg *cfg, + struct apple_dart_stream_map *stream_map) +{ + struct apple_dart *dart = stream_map->dart; + int sid; + + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { + for (int idx = 0; idx < dart->hw->ttbr_count; idx++) { + u64 *ttbrep = dart->locked_ttbr[sid][idx]; + u64 *ptep = cfg->apple_dart_cfg.ttbr[idx]; + if (!ttbrep || !ptep) + continue; + for (int entry = 0; entry < dart->pgsize / sizeof(*ptep); entry++) + ttbrep[entry] = ptep[entry]; + } + } + + return 0; +} + static int apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map *stream_map, u32 command) @@ -464,17 +551,9 @@ apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) static int apple_dart_hw_reset(struct apple_dart *dart) { - u32 config; struct apple_dart_stream_map stream_map; int i; - config = readl(dart->regs + dart->hw->lock); - if (config & dart->hw->lock_bit) { - dev_err(dart->dev, "DART is locked down until reboot: %08x\n", - config); - return -EINVAL; - } - stream_map.dart = dart; bitmap_zero(stream_map.sidmap, DART_MAX_STREAMS); bitmap_set(stream_map.sidmap, 0, dart->num_streams); @@ -499,6 +578,8 @@ static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain) int i, j; struct apple_dart_atomic_stream_map *domain_stream_map; struct apple_dart_stream_map stream_map; + struct io_pgtable_cfg *pgtbl_cfg = + &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg; for_each_stream_map(i, domain, domain_stream_map) { stream_map.dart = domain_stream_map->dart; @@ -506,7 +587,13 @@ static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain) for (j = 0; j < BITS_TO_LONGS(stream_map.dart->num_streams); j++) stream_map.sidmap[j] = atomic_long_read(&domain_stream_map->sidmap[j]); + WARN_ON(pm_runtime_get_sync(stream_map.dart->dev) < 0); + + if (stream_map.dart->locked) + apple_dart_hw_sync_locked(pgtbl_cfg, &stream_map); + stream_map.dart->hw->invalidate_tlb(&stream_map); + pm_runtime_put(stream_map.dart->dev); } } @@ -537,7 +624,7 @@ static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain, if (!ops) return 0; - return ops->iova_to_phys(ops, iova); + return ops->iova_to_phys(ops, iova & dart_domain->mask); } static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, @@ -551,8 +638,8 @@ static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, if (!ops) return -ENODEV; - return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, - mapped); + return ops->map_pages(ops, iova & dart_domain->mask, paddr, pgsize, + pgcount, prot, gfp, mapped); } static size_t apple_dart_unmap_pages(struct iommu_domain *domain, @@ -563,7 +650,8 @@ static size_t apple_dart_unmap_pages(struct iommu_domain *domain, struct apple_dart_domain *dart_domain = to_dart_domain(domain); struct io_pgtable_ops *ops = dart_domain->pgtbl_ops; - return ops->unmap_pages(ops, iova, pgsize, pgcount, gather); + return ops->unmap_pages(ops, iova & dart_domain->mask, pgsize, pgcount, + gather); } static void @@ -574,9 +662,10 @@ apple_dart_setup_translation(struct apple_dart_domain *domain, struct io_pgtable_cfg *pgtbl_cfg = &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg; - for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i) - apple_dart_hw_set_ttbr(stream_map, i, - pgtbl_cfg->apple_dart_cfg.ttbr[i]); + for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i) { + u64 ttbr = virt_to_phys(pgtbl_cfg->apple_dart_cfg.ttbr[i]); + apple_dart_hw_set_ttbr(stream_map, i, ttbr); + } for (; i < stream_map->dart->hw->ttbr_count; ++i) apple_dart_hw_clear_ttbr(stream_map, i); @@ -585,11 +674,31 @@ apple_dart_setup_translation(struct apple_dart_domain *domain, stream_map->dart->hw->invalidate_tlb(stream_map); } +static void +apple_dart_setup_translation_locked(struct apple_dart_domain *domain, + struct apple_dart_stream_map *stream_map) +{ + int i; + struct io_pgtable_cfg *pgtbl_cfg = + &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg; + + /* Locked DARTs are set up by the bootloader. */ + for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i) + apple_dart_hw_map_locked_ttbr(stream_map, i); + for (; i < stream_map->dart->hw->ttbr_count; ++i) + apple_dart_hw_unmap_locked_ttbr(stream_map, i); + + apple_dart_hw_sync_locked(pgtbl_cfg, stream_map); + stream_map->dart->hw->invalidate_tlb(stream_map); +} + static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, struct apple_dart_master_cfg *cfg) { struct apple_dart *dart = cfg->stream_maps[0].dart; struct io_pgtable_cfg pgtbl_cfg; + dma_addr_t dma_max = dart->dma_max; + u32 ias = min_t(u32, dart->ias, fls64(dma_max)); int ret = 0; int i, j; @@ -610,12 +719,48 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, pgtbl_cfg = (struct io_pgtable_cfg){ .pgsize_bitmap = dart->pgsize, - .ias = dart->ias, + .ias = ias, .oas = dart->oas, .coherent_walk = 1, .iommu_dev = dart->dev, }; + if (dart->locked) { + unsigned long *sidmap; + int sid; + u32 ttbr; + + /* Locked DARTs can only have a single stream bound */ + sidmap = cfg->stream_maps[0].sidmap; + sid = find_first_bit(sidmap, dart->num_streams); + + WARN_ON((sid < 0) || bitmap_weight(sidmap, dart->num_streams) > 1); + ttbr = readl(dart->regs + DART_TTBR(dart, sid, 0)); + + WARN_ON(!(ttbr & dart->hw->ttbr_valid)); + + /* If the DART is locked, we need to keep the translation level count. */ + if (dart->hw->tcr_4level && dart->ias > 36) { + if (readl(dart->regs + DART_TCR(dart, sid)) & dart->hw->tcr_4level) { + if (ias < 37) { + dev_info(dart->dev, "Expanded to ias=37 due to lock\n"); + pgtbl_cfg.ias = 37; + } + } else if (ias > 36) { + dev_info(dart->dev, "Limited to ias=36 due to lock\n"); + pgtbl_cfg.ias = 36; + if (dart->dma_min == 0 && dma_max == DMA_BIT_MASK(dart->ias)) { + dma_max = DMA_BIT_MASK(pgtbl_cfg.ias); + } else if ((dart->dma_min ^ dma_max) & ~DMA_BIT_MASK(36)) { + dev_err(dart->dev, + "Invalid DMA range for locked 3-level PT\n"); + ret = -ENOMEM; + goto done; + } + } + } + } + dart_domain->pgtbl_ops = alloc_io_pgtable_ops(dart->hw->fmt, &pgtbl_cfg, &dart_domain->domain); if (!dart_domain->pgtbl_ops) { @@ -623,10 +768,16 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, goto done; } + if (pgtbl_cfg.pgsize_bitmap == SZ_4K) + dart_domain->mask = DMA_BIT_MASK(min_t(u32, dart->ias, 32)); + else if (pgtbl_cfg.apple_dart_cfg.n_levels == 3) + dart_domain->mask = DMA_BIT_MASK(min_t(u32, dart->ias, 36)); + else if (pgtbl_cfg.apple_dart_cfg.n_levels == 4) + dart_domain->mask = DMA_BIT_MASK(min_t(u32, dart->ias, 47)); + dart_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; - dart_domain->domain.geometry.aperture_start = 0; - dart_domain->domain.geometry.aperture_end = - (dma_addr_t)DMA_BIT_MASK(pgtbl_cfg.ias); + dart_domain->domain.geometry.aperture_start = dart->dma_min; + dart_domain->domain.geometry.aperture_end = dma_max; dart_domain->domain.geometry.force_aperture = true; dart_domain->finalized = true; @@ -680,17 +831,29 @@ static int apple_dart_attach_dev_paging(struct iommu_domain *domain, struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); struct apple_dart_domain *dart_domain = to_dart_domain(domain); + for_each_stream_map(i, cfg, stream_map) + WARN_ON(pm_runtime_get_sync(stream_map->dart->dev) < 0); + ret = apple_dart_finalize_domain(dart_domain, cfg); if (ret) - return ret; + goto err; ret = apple_dart_domain_add_streams(dart_domain, cfg); if (ret) - return ret; + goto err; + + for_each_stream_map(i, cfg, stream_map) { + if (!stream_map->dart->locked) + apple_dart_setup_translation(dart_domain, stream_map); + else + apple_dart_setup_translation_locked(dart_domain, + stream_map); + } +err: for_each_stream_map(i, cfg, stream_map) - apple_dart_setup_translation(dart_domain, stream_map); - return 0; + pm_runtime_put(stream_map->dart->dev); + return ret; } static int apple_dart_attach_dev_identity(struct iommu_domain *domain, @@ -704,8 +867,17 @@ static int apple_dart_attach_dev_identity(struct iommu_domain *domain, if (!cfg->supports_bypass) return -EINVAL; + if (cfg->locked) + return -EINVAL; + + for_each_stream_map(i, cfg, stream_map) + WARN_ON(pm_runtime_get_sync(stream_map->dart->dev) < 0); + for_each_stream_map(i, cfg, stream_map) apple_dart_hw_enable_bypass(stream_map); + + for_each_stream_map(i, cfg, stream_map) + pm_runtime_put(stream_map->dart->dev); return 0; } @@ -726,8 +898,17 @@ static int apple_dart_attach_dev_blocked(struct iommu_domain *domain, struct apple_dart_stream_map *stream_map; int i; + if (cfg->locked) + return -EINVAL; + + for_each_stream_map(i, cfg, stream_map) + WARN_ON(pm_runtime_get_sync(stream_map->dart->dev) < 0); + for_each_stream_map(i, cfg, stream_map) apple_dart_hw_disable_dma(stream_map); + + for_each_stream_map(i, cfg, stream_map) + pm_runtime_put(stream_map->dart->dev); return 0; } @@ -746,21 +927,29 @@ static struct iommu_device *apple_dart_probe_device(struct device *dev) struct apple_dart_stream_map *stream_map; int i; - if (!cfg) + if (!dev_iommu_fwspec_get(dev) || !cfg) return ERR_PTR(-ENODEV); for_each_stream_map(i, cfg, stream_map) - device_link_add( - dev, stream_map->dart->dev, - DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER); + device_link_add(dev, stream_map->dart->dev, + DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER | + DL_FLAG_RPM_ACTIVE); return &cfg->stream_maps[0].dart->iommu; } static void apple_dart_release_device(struct device *dev) { + int i, j; + struct apple_dart_stream_map *stream_map; struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); + for_each_stream_map(j, cfg, stream_map) { + if (stream_map->dart->locked) + for (i = 0; i < stream_map->dart->hw->ttbr_count; ++i) + apple_dart_hw_unmap_locked_ttbr(stream_map, i); + } + kfree(cfg); } @@ -817,6 +1006,8 @@ static int apple_dart_of_xlate(struct device *dev, return -ENOMEM; /* Will be ANDed with DART capabilities */ cfg->supports_bypass = true; + /* Will be ORed with DART capabilities*/ + cfg->locked = false; } dev_iommu_priv_set(dev, cfg); @@ -829,6 +1020,7 @@ static int apple_dart_of_xlate(struct device *dev, } cfg->supports_bypass &= dart->supports_bypass; + cfg->locked |= dart->locked; for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { if (cfg->stream_maps[i].dart == dart) { @@ -971,6 +1163,8 @@ static int apple_dart_def_domain_type(struct device *dev) return IOMMU_DOMAIN_IDENTITY; if (!cfg->supports_bypass) return IOMMU_DOMAIN_DMA; + if (cfg->locked) + return IOMMU_DOMAIN_DMA; return 0; } @@ -1003,12 +1197,12 @@ static void apple_dart_get_resv_regions(struct device *dev, static const struct iommu_ops apple_dart_iommu_ops = { .identity_domain = &apple_dart_identity_domain, .blocked_domain = &apple_dart_blocked_domain, + .def_domain_type = apple_dart_def_domain_type, .domain_alloc_paging = apple_dart_domain_alloc_paging, .probe_device = apple_dart_probe_device, .release_device = apple_dart_release_device, .device_group = apple_dart_device_group, .of_xlate = apple_dart_of_xlate, - .def_domain_type = apple_dart_def_domain_type, .get_resv_regions = apple_dart_get_resv_regions, .owner = THIS_MODULE, .default_domain_ops = &(const struct iommu_domain_ops) { @@ -1102,6 +1296,22 @@ static irqreturn_t apple_dart_t8110_irq(int irq, void *dev) return IRQ_HANDLED; } +static irqreturn_t apple_dart_irq(int irq, void *dev) +{ + irqreturn_t ret; + struct apple_dart *dart = dev; + + WARN_ON(pm_runtime_get_sync(dart->dev) < 0); + ret = dart->hw->irq_handler(irq, dev); + pm_runtime_put(dart->dev); + return ret; +} + +static bool apple_dart_is_locked(struct apple_dart *dart) +{ + return !!(readl(dart->regs + dart->hw->lock) & dart->hw->lock_bit); +} + static int apple_dart_probe(struct platform_device *pdev) { int ret; @@ -1109,6 +1319,7 @@ static int apple_dart_probe(struct platform_device *pdev) struct resource *res; struct apple_dart *dart; struct device *dev = &pdev->dev; + u64 dma_range[2]; dart = devm_kzalloc(dev, sizeof(*dart), GFP_KERNEL); if (!dart) @@ -1140,6 +1351,14 @@ static int apple_dart_probe(struct platform_device *pdev) if (ret) return ret; + pm_runtime_get_noresume(dev); + pm_runtime_set_active(dev); + pm_runtime_irq_safe(dev); + + ret = devm_pm_runtime_enable(dev); + if (ret) + goto err_clk_disable; + dart_params[0] = readl(dart->regs + DART_PARAMS1); dart_params[1] = readl(dart->regs + DART_PARAMS2); dart->pgsize = 1 << FIELD_GET(DART_PARAMS1_PAGE_SHIFT, dart_params[0]); @@ -1163,6 +1382,26 @@ static int apple_dart_probe(struct platform_device *pdev) break; } + dart->dma_min = 0; + dart->dma_max = DMA_BIT_MASK(dart->ias); + + ret = of_property_read_u64_array(dev->of_node, "apple,dma-range", dma_range, 2); + if (ret == -EINVAL) { + ret = 0; + } else if (ret) { + goto err_clk_disable; + } else { + dart->dma_min = dma_range[0]; + dart->dma_max = dma_range[0] + dma_range[1] - 1; + if ((dart->dma_min ^ dart->dma_max) & ~DMA_BIT_MASK(dart->ias)) { + dev_err(&pdev->dev, "Invalid DMA range for ias=%d\n", + dart->ias); + goto err_clk_disable; + } + dev_info(&pdev->dev, "Limiting DMA range to %pad..%pad\n", + &dart->dma_min, &dart->dma_max); + } + if (dart->num_streams > DART_MAX_STREAMS) { dev_err(&pdev->dev, "Too many streams (%d > %d)\n", dart->num_streams, DART_MAX_STREAMS); @@ -1170,11 +1409,14 @@ static int apple_dart_probe(struct platform_device *pdev) goto err_clk_disable; } - ret = apple_dart_hw_reset(dart); - if (ret) - goto err_clk_disable; + dart->locked = apple_dart_is_locked(dart); + if (!dart->locked) { + ret = apple_dart_hw_reset(dart); + if (ret) + goto err_clk_disable; + } - ret = request_irq(dart->irq, dart->hw->irq_handler, IRQF_SHARED, + ret = request_irq(dart->irq, apple_dart_irq, IRQF_SHARED, "apple-dart fault handler", dart); if (ret) goto err_clk_disable; @@ -1190,11 +1432,13 @@ static int apple_dart_probe(struct platform_device *pdev) if (ret) goto err_sysfs_remove; + pm_runtime_put(dev); + dev_info( &pdev->dev, - "DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d, AS %d -> %d] initialized\n", + "DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d, locked: %d, AS %d -> %d] initialized\n", dart->pgsize, dart->num_streams, dart->supports_bypass, - dart->pgsize > PAGE_SIZE, dart->ias, dart->oas); + dart->pgsize > PAGE_SIZE, dart->locked, dart->ias, dart->oas); return 0; err_sysfs_remove: @@ -1202,6 +1446,7 @@ static int apple_dart_probe(struct platform_device *pdev) err_free_irq: free_irq(dart->irq, dart); err_clk_disable: + pm_runtime_put(dev); clk_bulk_disable_unprepare(dart->num_clks, dart->clks); return ret; @@ -1211,7 +1456,9 @@ static void apple_dart_remove(struct platform_device *pdev) { struct apple_dart *dart = platform_get_drvdata(pdev); - apple_dart_hw_reset(dart); + if (!dart->locked) + apple_dart_hw_reset(dart); + free_irq(dart->irq, dart); iommu_device_unregister(&dart->iommu); @@ -1329,6 +1576,10 @@ static __maybe_unused int apple_dart_suspend(struct device *dev) struct apple_dart *dart = dev_get_drvdata(dev); unsigned int sid, idx; + /* Locked DARTs can't be restored so skip saving their registers. */ + if (dart->locked) + return 0; + for (sid = 0; sid < dart->num_streams; sid++) { dart->save_tcr[sid] = readl(dart->regs + DART_TCR(dart, sid)); for (idx = 0; idx < dart->hw->ttbr_count; idx++) @@ -1345,6 +1596,10 @@ static __maybe_unused int apple_dart_resume(struct device *dev) unsigned int sid, idx; int ret; + /* Locked DARTs can't be restored, and they should not need it */ + if (dart->locked) + return 0; + ret = apple_dart_hw_reset(dart); if (ret) { dev_err(dev, "Failed to reset DART on resume\n"); @@ -1361,7 +1616,7 @@ static __maybe_unused int apple_dart_resume(struct device *dev) return 0; } -static DEFINE_SIMPLE_DEV_PM_OPS(apple_dart_pm_ops, apple_dart_suspend, apple_dart_resume); +static DEFINE_RUNTIME_DEV_PM_OPS(apple_dart_pm_ops, apple_dart_suspend, apple_dart_resume, NULL); static const struct of_device_id apple_dart_of_match[] = { { .compatible = "apple,t8103-dart", .data = &apple_dart_hw_t8103 }, @@ -1377,7 +1632,7 @@ static struct platform_driver apple_dart_driver = { .name = "apple-dart", .of_match_table = apple_dart_of_match, .suppress_bind_attrs = true, - .pm = pm_sleep_ptr(&apple_dart_pm_ops), + .pm = pm_ptr(&apple_dart_pm_ops), }, .probe = apple_dart_probe, .remove = apple_dart_remove, diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-impl.c b/drivers/iommu/arm/arm-smmu/arm-smmu-impl.c index db9b9a8e139c87..4565a58bb213f4 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-impl.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-impl.c @@ -228,3 +228,17 @@ struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu) return smmu; } + +int __init arm_smmu_impl_module_init(void) +{ + if (IS_ENABLED(CONFIG_ARM_SMMU_QCOM)) + return qcom_smmu_module_init(); + + return 0; +} + +void __exit arm_smmu_impl_module_exit(void) +{ + if (IS_ENABLED(CONFIG_ARM_SMMU_QCOM)) + qcom_smmu_module_exit(); +} diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index 573085349df34f..22906d2c9a2db2 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -774,10 +774,6 @@ struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu) { const struct device_node *np = smmu->dev->of_node; const struct of_device_id *match; - static u8 tbu_registered; - - if (!tbu_registered++) - platform_driver_register(&qcom_smmu_tbu_driver); #ifdef CONFIG_ACPI if (np == NULL) { @@ -802,3 +798,13 @@ struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu) return smmu; } + +int __init qcom_smmu_module_init(void) +{ + return platform_driver_register(&qcom_smmu_tbu_driver); +} + +void __exit qcom_smmu_module_exit(void) +{ + platform_driver_unregister(&qcom_smmu_tbu_driver); +} diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c index 5e690cf85ec969..1e218fbea35a02 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c @@ -2365,7 +2365,29 @@ static struct platform_driver arm_smmu_driver = { .remove = arm_smmu_device_remove, .shutdown = arm_smmu_device_shutdown, }; -module_platform_driver(arm_smmu_driver); + +static int __init arm_smmu_init(void) +{ + int ret; + + ret = platform_driver_register(&arm_smmu_driver); + if (ret) + return ret; + + ret = arm_smmu_impl_module_init(); + if (ret) + platform_driver_unregister(&arm_smmu_driver); + + return ret; +} +module_init(arm_smmu_init); + +static void __exit arm_smmu_exit(void) +{ + arm_smmu_impl_module_exit(); + platform_driver_unregister(&arm_smmu_driver); +} +module_exit(arm_smmu_exit); MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations"); MODULE_AUTHOR("Will Deacon "); diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.h b/drivers/iommu/arm/arm-smmu/arm-smmu.h index 2dbf3243b5ad2d..26d2e33cd328b8 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.h +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.h @@ -540,6 +540,11 @@ struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu); struct arm_smmu_device *nvidia_smmu_impl_init(struct arm_smmu_device *smmu); struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu); +int __init arm_smmu_impl_module_init(void); +void __exit arm_smmu_impl_module_exit(void); +int __init qcom_smmu_module_init(void); +void __exit qcom_smmu_module_exit(void); + void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx); int arm_mmu500_reset(struct arm_smmu_device *smmu); diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index c92088855450ad..8ef083185fc726 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -572,8 +572,13 @@ static int iova_reserve_iommu_regions(struct device *dev, if (region->type == IOMMU_RESV_SW_MSI) continue; - lo = iova_pfn(iovad, region->start); - hi = iova_pfn(iovad, region->start + region->length - 1); + if (region->type == IOMMU_RESV_TRANSLATED) { + lo = iova_pfn(iovad, region->dva); + hi = iova_pfn(iovad, region->dva + region->length - 1); + } else { + lo = iova_pfn(iovad, region->start); + hi = iova_pfn(iovad, region->start + region->length - 1); + } reserve_iova(iovad, lo, hi); if (region->type == IOMMU_RESV_MSI) diff --git a/drivers/iommu/generic_pt/fmt/amdv1.h b/drivers/iommu/generic_pt/fmt/amdv1.h index aa8e1a8ec95fde..3b2c41d9654d72 100644 --- a/drivers/iommu/generic_pt/fmt/amdv1.h +++ b/drivers/iommu/generic_pt/fmt/amdv1.h @@ -354,7 +354,8 @@ static inline int amdv1pt_iommu_set_prot(struct pt_common *common, * Ideally we'd have an IOMMU_ENCRYPTED flag set by higher levels to * control this. For now if the tables use sme_set then so do the ptes. */ - if (pt_feature(common, PT_FEAT_AMDV1_ENCRYPT_TABLES)) + if (pt_feature(common, PT_FEAT_AMDV1_ENCRYPT_TABLES) && + !(iommu_prot & IOMMU_MMIO)) pte = __sme_set(pte); attrs->descriptor_bits = pte; diff --git a/drivers/iommu/generic_pt/fmt/x86_64.h b/drivers/iommu/generic_pt/fmt/x86_64.h index 210748d9d6e8aa..ed9a47cbb6e022 100644 --- a/drivers/iommu/generic_pt/fmt/x86_64.h +++ b/drivers/iommu/generic_pt/fmt/x86_64.h @@ -227,7 +227,8 @@ static inline int x86_64_pt_iommu_set_prot(struct pt_common *common, * Ideally we'd have an IOMMU_ENCRYPTED flag set by higher levels to * control this. For now if the tables use sme_set then so do the ptes. */ - if (pt_feature(common, PT_FEAT_X86_64_AMD_ENCRYPT_TABLES)) + if (pt_feature(common, PT_FEAT_X86_64_AMD_ENCRYPT_TABLES) && + !(iommu_prot & IOMMU_MMIO)) pte = __sme_set(pte); attrs->descriptor_bits = pte; diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 134302fbcd9263..705828b06e329c 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -1240,22 +1240,22 @@ static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8 } did = context_domain_id(context); - context_clear_entry(context); + context_clear_present(context); __iommu_flush_cache(iommu, context, sizeof(*context)); spin_unlock(&iommu->lock); intel_context_flush_no_pasid(info, context, did); + context_clear_entry(context); + __iommu_flush_cache(iommu, context, sizeof(*context)); } int __domain_setup_first_level(struct intel_iommu *iommu, struct device *dev, ioasid_t pasid, u16 did, phys_addr_t fsptptr, int flags, struct iommu_domain *old) { - if (!old) - return intel_pasid_setup_first_level(iommu, dev, fsptptr, pasid, - did, flags); - return intel_pasid_replace_first_level(iommu, dev, fsptptr, pasid, did, - iommu_domain_did(old, iommu), - flags); + if (old) + intel_pasid_tear_down_entry(iommu, dev, pasid, false); + + return intel_pasid_setup_first_level(iommu, dev, fsptptr, pasid, did, flags); } static int domain_setup_second_level(struct intel_iommu *iommu, @@ -1263,23 +1263,20 @@ static int domain_setup_second_level(struct intel_iommu *iommu, struct device *dev, ioasid_t pasid, struct iommu_domain *old) { - if (!old) - return intel_pasid_setup_second_level(iommu, domain, - dev, pasid); - return intel_pasid_replace_second_level(iommu, domain, dev, - iommu_domain_did(old, iommu), - pasid); + if (old) + intel_pasid_tear_down_entry(iommu, dev, pasid, false); + + return intel_pasid_setup_second_level(iommu, domain, dev, pasid); } static int domain_setup_passthrough(struct intel_iommu *iommu, struct device *dev, ioasid_t pasid, struct iommu_domain *old) { - if (!old) - return intel_pasid_setup_pass_through(iommu, dev, pasid); - return intel_pasid_replace_pass_through(iommu, dev, - iommu_domain_did(old, iommu), - pasid); + if (old) + intel_pasid_tear_down_entry(iommu, dev, pasid, false); + + return intel_pasid_setup_pass_through(iommu, dev, pasid); } static int domain_setup_first_level(struct intel_iommu *iommu, diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index 25c5e22096d44e..599913fb65d59e 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -900,7 +900,26 @@ static inline int pfn_level_offset(u64 pfn, int level) static inline void context_set_present(struct context_entry *context) { - context->lo |= 1; + u64 val; + + dma_wmb(); + val = READ_ONCE(context->lo) | 1; + WRITE_ONCE(context->lo, val); +} + +/* + * Clear the Present (P) bit (bit 0) of a context table entry. This initiates + * the transition of the entry's ownership from hardware to software. The + * caller is responsible for fulfilling the invalidation handshake recommended + * by the VT-d spec, Section 6.5.3.3 (Guidance to Software for Invalidations). + */ +static inline void context_clear_present(struct context_entry *context) +{ + u64 val; + + val = READ_ONCE(context->lo) & GENMASK_ULL(63, 1); + WRITE_ONCE(context->lo, val); + dma_wmb(); } static inline void context_set_fault_enable(struct context_entry *context) diff --git a/drivers/iommu/intel/nested.c b/drivers/iommu/intel/nested.c index a3fb8c193ca647..e9a440e9c960b2 100644 --- a/drivers/iommu/intel/nested.c +++ b/drivers/iommu/intel/nested.c @@ -136,11 +136,10 @@ static int domain_setup_nested(struct intel_iommu *iommu, struct device *dev, ioasid_t pasid, struct iommu_domain *old) { - if (!old) - return intel_pasid_setup_nested(iommu, dev, pasid, domain); - return intel_pasid_replace_nested(iommu, dev, pasid, - iommu_domain_did(old, iommu), - domain); + if (old) + intel_pasid_tear_down_entry(iommu, dev, pasid, false); + + return intel_pasid_setup_nested(iommu, dev, pasid, domain); } static int intel_nested_set_dev_pasid(struct iommu_domain *domain, diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index 3e2255057079c5..34b209b88be2aa 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -153,6 +153,9 @@ static struct pasid_entry *intel_pasid_get_entry(struct device *dev, u32 pasid) if (!entries) return NULL; + if (!ecap_coherent(info->iommu->ecap)) + clflush_cache_range(entries, VTD_PAGE_SIZE); + /* * The pasid directory table entry won't be freed after * allocation. No worry about the race with free and @@ -165,10 +168,8 @@ static struct pasid_entry *intel_pasid_get_entry(struct device *dev, u32 pasid) iommu_free_pages(entries); goto retry; } - if (!ecap_coherent(info->iommu->ecap)) { - clflush_cache_range(entries, VTD_PAGE_SIZE); + if (!ecap_coherent(info->iommu->ecap)) clflush_cache_range(&dir[dir_index].val, sizeof(*dir)); - } } return &entries[index]; @@ -272,7 +273,7 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev, did = pasid_get_domain_id(pte); pgtt = pasid_pte_get_pgtt(pte); - intel_pasid_clear_entry(dev, pasid, fault_ignore); + pasid_clear_present(pte); spin_unlock(&iommu->lock); if (!ecap_coherent(iommu->ecap)) @@ -286,6 +287,10 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev, iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH); devtlb_invalidation_with_pasid(iommu, dev, pasid); + intel_pasid_clear_entry(dev, pasid, fault_ignore); + if (!ecap_coherent(iommu->ecap)) + clflush_cache_range(pte, sizeof(*pte)); + if (!fault_ignore) intel_iommu_drain_pasid_prq(dev, pasid); } @@ -412,50 +417,6 @@ int intel_pasid_setup_first_level(struct intel_iommu *iommu, struct device *dev, return 0; } -int intel_pasid_replace_first_level(struct intel_iommu *iommu, - struct device *dev, phys_addr_t fsptptr, - u32 pasid, u16 did, u16 old_did, - int flags) -{ - struct pasid_entry *pte, new_pte; - - if (!ecap_flts(iommu->ecap)) { - pr_err("No first level translation support on %s\n", - iommu->name); - return -EINVAL; - } - - if ((flags & PASID_FLAG_FL5LP) && !cap_fl5lp_support(iommu->cap)) { - pr_err("No 5-level paging support for first-level on %s\n", - iommu->name); - return -EINVAL; - } - - pasid_pte_config_first_level(iommu, &new_pte, fsptptr, did, flags); - - spin_lock(&iommu->lock); - pte = intel_pasid_get_entry(dev, pasid); - if (!pte) { - spin_unlock(&iommu->lock); - return -ENODEV; - } - - if (!pasid_pte_is_present(pte)) { - spin_unlock(&iommu->lock); - return -EINVAL; - } - - WARN_ON(old_did != pasid_get_domain_id(pte)); - - *pte = new_pte; - spin_unlock(&iommu->lock); - - intel_pasid_flush_present(iommu, dev, pasid, old_did, pte); - intel_iommu_drain_pasid_prq(dev, pasid); - - return 0; -} - /* * Set up the scalable mode pasid entry for second only translation type. */ @@ -522,51 +483,6 @@ int intel_pasid_setup_second_level(struct intel_iommu *iommu, return 0; } -int intel_pasid_replace_second_level(struct intel_iommu *iommu, - struct dmar_domain *domain, - struct device *dev, u16 old_did, - u32 pasid) -{ - struct pasid_entry *pte, new_pte; - u16 did; - - /* - * If hardware advertises no support for second level - * translation, return directly. - */ - if (!ecap_slts(iommu->ecap)) { - pr_err("No second level translation support on %s\n", - iommu->name); - return -EINVAL; - } - - did = domain_id_iommu(domain, iommu); - - pasid_pte_config_second_level(iommu, &new_pte, domain, did); - - spin_lock(&iommu->lock); - pte = intel_pasid_get_entry(dev, pasid); - if (!pte) { - spin_unlock(&iommu->lock); - return -ENODEV; - } - - if (!pasid_pte_is_present(pte)) { - spin_unlock(&iommu->lock); - return -EINVAL; - } - - WARN_ON(old_did != pasid_get_domain_id(pte)); - - *pte = new_pte; - spin_unlock(&iommu->lock); - - intel_pasid_flush_present(iommu, dev, pasid, old_did, pte); - intel_iommu_drain_pasid_prq(dev, pasid); - - return 0; -} - /* * Set up dirty tracking on a second only or nested translation type. */ @@ -679,38 +595,6 @@ int intel_pasid_setup_pass_through(struct intel_iommu *iommu, return 0; } -int intel_pasid_replace_pass_through(struct intel_iommu *iommu, - struct device *dev, u16 old_did, - u32 pasid) -{ - struct pasid_entry *pte, new_pte; - u16 did = FLPT_DEFAULT_DID; - - pasid_pte_config_pass_through(iommu, &new_pte, did); - - spin_lock(&iommu->lock); - pte = intel_pasid_get_entry(dev, pasid); - if (!pte) { - spin_unlock(&iommu->lock); - return -ENODEV; - } - - if (!pasid_pte_is_present(pte)) { - spin_unlock(&iommu->lock); - return -EINVAL; - } - - WARN_ON(old_did != pasid_get_domain_id(pte)); - - *pte = new_pte; - spin_unlock(&iommu->lock); - - intel_pasid_flush_present(iommu, dev, pasid, old_did, pte); - intel_iommu_drain_pasid_prq(dev, pasid); - - return 0; -} - /* * Set the page snoop control for a pasid entry which has been set up. */ @@ -844,69 +728,6 @@ int intel_pasid_setup_nested(struct intel_iommu *iommu, struct device *dev, return 0; } -int intel_pasid_replace_nested(struct intel_iommu *iommu, - struct device *dev, u32 pasid, - u16 old_did, struct dmar_domain *domain) -{ - struct iommu_hwpt_vtd_s1 *s1_cfg = &domain->s1_cfg; - struct dmar_domain *s2_domain = domain->s2_domain; - u16 did = domain_id_iommu(domain, iommu); - struct pasid_entry *pte, new_pte; - - /* Address width should match the address width supported by hardware */ - switch (s1_cfg->addr_width) { - case ADDR_WIDTH_4LEVEL: - break; - case ADDR_WIDTH_5LEVEL: - if (!cap_fl5lp_support(iommu->cap)) { - dev_err_ratelimited(dev, - "5-level paging not supported\n"); - return -EINVAL; - } - break; - default: - dev_err_ratelimited(dev, "Invalid stage-1 address width %d\n", - s1_cfg->addr_width); - return -EINVAL; - } - - if ((s1_cfg->flags & IOMMU_VTD_S1_SRE) && !ecap_srs(iommu->ecap)) { - pr_err_ratelimited("No supervisor request support on %s\n", - iommu->name); - return -EINVAL; - } - - if ((s1_cfg->flags & IOMMU_VTD_S1_EAFE) && !ecap_eafs(iommu->ecap)) { - pr_err_ratelimited("No extended access flag support on %s\n", - iommu->name); - return -EINVAL; - } - - pasid_pte_config_nestd(iommu, &new_pte, s1_cfg, s2_domain, did); - - spin_lock(&iommu->lock); - pte = intel_pasid_get_entry(dev, pasid); - if (!pte) { - spin_unlock(&iommu->lock); - return -ENODEV; - } - - if (!pasid_pte_is_present(pte)) { - spin_unlock(&iommu->lock); - return -EINVAL; - } - - WARN_ON(old_did != pasid_get_domain_id(pte)); - - *pte = new_pte; - spin_unlock(&iommu->lock); - - intel_pasid_flush_present(iommu, dev, pasid, old_did, pte); - intel_iommu_drain_pasid_prq(dev, pasid); - - return 0; -} - /* * Interfaces to setup or teardown a pasid table to the scalable-mode * context table entry: @@ -1019,7 +840,7 @@ static int device_pasid_table_setup(struct device *dev, u8 bus, u8 devfn) } if (context_copied(iommu, bus, devfn)) { - context_clear_entry(context); + context_clear_present(context); __iommu_flush_cache(iommu, context, sizeof(*context)); /* @@ -1039,6 +860,9 @@ static int device_pasid_table_setup(struct device *dev, u8 bus, u8 devfn) iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH); devtlb_invalidation_with_pasid(iommu, dev, IOMMU_NO_PASID); + context_clear_entry(context); + __iommu_flush_cache(iommu, context, sizeof(*context)); + /* * At this point, the device is supposed to finish reset at * its driver probe stage, so no in-flight DMA will exist, diff --git a/drivers/iommu/intel/pasid.h b/drivers/iommu/intel/pasid.h index 3809793e0259f1..48d3bb6b68dea0 100644 --- a/drivers/iommu/intel/pasid.h +++ b/drivers/iommu/intel/pasid.h @@ -234,9 +234,23 @@ static inline void pasid_set_wpe(struct pasid_entry *pe) */ static inline void pasid_set_present(struct pasid_entry *pe) { + dma_wmb(); pasid_set_bits(&pe->val[0], 1 << 0, 1); } +/* + * Clear the Present (P) bit (bit 0) of a scalable-mode PASID table entry. + * This initiates the transition of the entry's ownership from hardware + * to software. The caller is responsible for fulfilling the invalidation + * handshake recommended by the VT-d spec, Section 6.5.3.3 (Guidance to + * Software for Invalidations). + */ +static inline void pasid_clear_present(struct pasid_entry *pe) +{ + pasid_set_bits(&pe->val[0], 1 << 0, 0); + dma_wmb(); +} + /* * Setup Page Walk Snoop bit (Bit 87) of a scalable mode PASID * entry. @@ -302,20 +316,6 @@ int intel_pasid_setup_pass_through(struct intel_iommu *iommu, struct device *dev, u32 pasid); int intel_pasid_setup_nested(struct intel_iommu *iommu, struct device *dev, u32 pasid, struct dmar_domain *domain); -int intel_pasid_replace_first_level(struct intel_iommu *iommu, - struct device *dev, phys_addr_t fsptptr, - u32 pasid, u16 did, u16 old_did, int flags); -int intel_pasid_replace_second_level(struct intel_iommu *iommu, - struct dmar_domain *domain, - struct device *dev, u16 old_did, - u32 pasid); -int intel_pasid_replace_pass_through(struct intel_iommu *iommu, - struct device *dev, u16 old_did, - u32 pasid); -int intel_pasid_replace_nested(struct intel_iommu *iommu, - struct device *dev, u32 pasid, - u16 old_did, struct dmar_domain *domain); - void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev, u32 pasid, bool fault_ignore); diff --git a/drivers/iommu/io-pgtable-dart.c b/drivers/iommu/io-pgtable-dart.c index 54d287cc0dd1b8..2334a2ea04bb3a 100644 --- a/drivers/iommu/io-pgtable-dart.c +++ b/drivers/iommu/io-pgtable-dart.c @@ -435,7 +435,7 @@ apple_dart_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) iommu_alloc_pages_sz(GFP_KERNEL, DART_GRANULE(data)); if (!data->pgd[i]) goto out_free_data; - cfg->apple_dart_cfg.ttbr[i] = virt_to_phys(data->pgd[i]); + cfg->apple_dart_cfg.ttbr[i] = data->pgd[i]; } return &data->iop; diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 2ca990dfbb884f..d30b22ad74f808 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -90,6 +90,7 @@ static const char * const iommu_group_resv_type_string[] = { [IOMMU_RESV_RESERVED] = "reserved", [IOMMU_RESV_MSI] = "msi", [IOMMU_RESV_SW_MSI] = "msi", + [IOMMU_RESV_TRANSLATED] = "translated", }; #define IOMMU_CMD_LINE_DMA_API BIT(0) @@ -133,8 +134,8 @@ static void __iommu_group_set_domain_nofail(struct iommu_group *group, static int iommu_setup_default_domain(struct iommu_group *group, int target_type); -static int iommu_create_device_direct_mappings(struct iommu_domain *domain, - struct device *dev); +static int iommu_create_device_fw_mappings(struct iommu_domain *domain, + struct device *dev); static ssize_t iommu_group_store_type(struct iommu_group *group, const char *buf, size_t count); static struct group_device *iommu_group_alloc_device(struct iommu_group *group, @@ -640,7 +641,7 @@ static int __iommu_probe_device(struct device *dev, struct list_head *group_list list_add_tail(&gdev->list, &group->devices); WARN_ON(group->default_domain && !group->domain); if (group->default_domain) - iommu_create_device_direct_mappings(group->default_domain, dev); + iommu_create_device_fw_mappings(group->default_domain, dev); if (group->domain) { ret = __iommu_device_set_domain(group, dev, group->domain, NULL, 0); @@ -1169,8 +1170,8 @@ int iommu_group_set_name(struct iommu_group *group, const char *name) } EXPORT_SYMBOL_GPL(iommu_group_set_name); -static int iommu_create_device_direct_mappings(struct iommu_domain *domain, - struct device *dev) +static int iommu_create_device_fw_mappings(struct iommu_domain *domain, + struct device *dev) { struct iommu_resv_region *entry; struct list_head mappings; @@ -1187,27 +1188,35 @@ static int iommu_create_device_direct_mappings(struct iommu_domain *domain, /* We need to consider overlapping regions for different devices */ list_for_each_entry(entry, &mappings, list) { - dma_addr_t start, end, addr; + dma_addr_t start, end, addr, iova; size_t map_size = 0; if (entry->type == IOMMU_RESV_DIRECT) dev->iommu->require_direct = 1; + if (entry->type == IOMMU_RESV_TRANSLATED) + dev->iommu->require_translated = 1; if ((entry->type != IOMMU_RESV_DIRECT && - entry->type != IOMMU_RESV_DIRECT_RELAXABLE) || + entry->type != IOMMU_RESV_DIRECT_RELAXABLE && + entry->type != IOMMU_RESV_TRANSLATED) || !iommu_is_dma_domain(domain)) continue; start = ALIGN(entry->start, pg_size); end = ALIGN(entry->start + entry->length, pg_size); - for (addr = start; addr <= end; addr += pg_size) { + if (entry->type == IOMMU_RESV_TRANSLATED) + iova = ALIGN(entry->dva, pg_size); + else + iova = start; + + for (addr = start; addr <= end; addr += pg_size, iova += pg_size) { phys_addr_t phys_addr; if (addr == end) goto map_end; - phys_addr = iommu_iova_to_phys(domain, addr); + phys_addr = iommu_iova_to_phys(domain, iova); if (!phys_addr) { map_size += pg_size; continue; @@ -1215,7 +1224,7 @@ static int iommu_create_device_direct_mappings(struct iommu_domain *domain, map_end: if (map_size) { - ret = iommu_map(domain, addr - map_size, + ret = iommu_map(domain, iova - map_size, addr - map_size, map_size, entry->prot, GFP_KERNEL); if (ret) @@ -2318,6 +2327,19 @@ static int __iommu_device_set_domain(struct iommu_group *group, "Firmware has requested this device have a 1:1 IOMMU mapping, rejecting configuring the device without a 1:1 mapping. Contact your platform vendor.\n"); return -EINVAL; } + /* + * If the device requires IOMMU_RESV_TRANSLATED then we cannot allow + * the identy or blocking domain to be attached as it does not contain + * the required translated mapping. + */ + if (dev->iommu->require_translated && + (new_domain->type == IOMMU_DOMAIN_IDENTITY || + new_domain->type == IOMMU_DOMAIN_BLOCKED || + new_domain == group->blocking_domain)) { + dev_warn(dev, + "Firmware has requested this device have a translated IOMMU mapping, rejecting configuring the device without a translated mapping. Contact your platform vendor.\n"); + return -EINVAL; + } if (dev->iommu->attach_deferred) { if (new_domain == group->default_domain) @@ -2859,10 +2881,11 @@ void iommu_put_resv_regions(struct device *dev, struct list_head *list) } EXPORT_SYMBOL(iommu_put_resv_regions); -struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start, - size_t length, int prot, - enum iommu_resv_type type, - gfp_t gfp) +struct iommu_resv_region *iommu_alloc_resv_region_tr(phys_addr_t start, + dma_addr_t dva_start, + size_t length, int prot, + enum iommu_resv_type type, + gfp_t gfp) { struct iommu_resv_region *region; @@ -2872,11 +2895,25 @@ struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start, INIT_LIST_HEAD(®ion->list); region->start = start; + if (type == IOMMU_RESV_TRANSLATED) + region->dva = dva_start; region->length = length; region->prot = prot; region->type = type; return region; } +EXPORT_SYMBOL_GPL(iommu_alloc_resv_region_tr); + +struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start, + size_t length, int prot, + enum iommu_resv_type type, + gfp_t gfp) +{ + if (type == IOMMU_RESV_TRANSLATED) + return NULL; + + return iommu_alloc_resv_region_tr(start, 0, length, prot, type, gfp); +} EXPORT_SYMBOL_GPL(iommu_alloc_resv_region); void iommu_set_default_passthrough(bool cmd_line) @@ -3001,7 +3038,7 @@ static int iommu_setup_default_domain(struct iommu_group *group, struct iommu_domain *old_dom = group->default_domain; struct group_device *gdev; struct iommu_domain *dom; - bool direct_failed; + bool fw_failed; int req_type; int ret; @@ -3031,10 +3068,10 @@ static int iommu_setup_default_domain(struct iommu_group *group, * mapped before their device is attached, in order to guarantee * continuity with any FW activity */ - direct_failed = false; + fw_failed = false; for_each_group_device(group, gdev) { - if (iommu_create_device_direct_mappings(dom, gdev->dev)) { - direct_failed = true; + if (iommu_create_device_fw_mappings(dom, gdev->dev)) { + fw_failed = true; dev_warn_once( gdev->dev->iommu->iommu_dev->dev, "IOMMU driver was not able to establish FW requested direct mapping."); @@ -3066,9 +3103,9 @@ static int iommu_setup_default_domain(struct iommu_group *group, * trying again after attaching. If this happens it means the device * will not continuously have the IOMMU_RESV_DIRECT map. */ - if (direct_failed) { + if (fw_failed) { for_each_group_device(group, gdev) { - ret = iommu_create_device_direct_mappings(dom, gdev->dev); + ret = iommu_create_device_fw_mappings(dom, gdev->dev); if (ret) goto err_restore_domain; } diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c index 6b989a62def20e..69377addd6cebb 100644 --- a/drivers/iommu/of_iommu.c +++ b/drivers/iommu/of_iommu.c @@ -147,6 +147,8 @@ int of_iommu_configure(struct device *dev, struct device_node *master_np, of_pci_check_device_ats(dev, master_np); } else { err = of_iommu_configure_device(master_np, dev, id); + if (err == -EPROBE_DEFER) + iommu_fwspec_free(dev); } if (err && dev_iommu_present) @@ -187,9 +189,7 @@ iommu_resv_region_get_type(struct device *dev, if (start == phys->start && end == phys->end) return IOMMU_RESV_DIRECT; - dev_warn(dev, "treating non-direct mapping [%pr] -> [%pap-%pap] as reservation\n", phys, - &start, &end); - return IOMMU_RESV_RESERVED; + return IOMMU_RESV_TRANSLATED; } /** @@ -260,8 +260,13 @@ void of_iommu_get_resv_regions(struct device *dev, struct list_head *list) } type = iommu_resv_region_get_type(dev, &phys, iova, length); - region = iommu_alloc_resv_region(iova, length, prot, type, + if (type == IOMMU_RESV_TRANSLATED) + region = iommu_alloc_resv_region_tr(phys.start, iova, length, prot, type, + GFP_KERNEL); + else + region = iommu_alloc_resv_region(iova, length, prot, type, GFP_KERNEL); + if (region) list_add_tail(®ion->list, list); } diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 3c70364e7cddd6..4607d28f23887f 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -134,8 +134,12 @@ #define AIC2_IRQ_CFG 0x2000 +/* AIC v3 registers (MMIO) */ +#define AIC3_IRQ_CFG 0x10000 + /* * AIC2 registers are laid out like this, starting at AIC2_IRQ_CFG: + * AIC3 registers use the same layout but start at AIC3_IRQ_CFG: * * Repeat for each die: * IRQ_CFG: u32 * MAX_IRQS @@ -293,6 +297,15 @@ static const struct aic_info aic2_info __initconst = { .local_fast_ipi = true, }; +static const struct aic_info aic3_info __initconst = { + .version = 3, + + .irq_cfg = AIC3_IRQ_CFG, + + .fast_ipi = true, + .local_fast_ipi = true, +}; + static const struct of_device_id aic_info_match[] = { { .compatible = "apple,t8103-aic", @@ -310,6 +323,10 @@ static const struct of_device_id aic_info_match[] = { .compatible = "apple,aic2", .data = &aic2_info, }, + { + .compatible = "apple,t8122-aic3", + .data = &aic3_info, + }, {} }; @@ -620,7 +637,7 @@ static int aic_irq_domain_map(struct irq_domain *id, unsigned int irq, u32 type = FIELD_GET(AIC_EVENT_TYPE, hw); struct irq_chip *chip = &aic_chip; - if (ic->info.version == 2) + if (ic->info.version == 2 || ic->info.version == 3) chip = &aic2_chip; if (type == AIC_EVENT_TYPE_IRQ) { @@ -991,7 +1008,7 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p break; } - case 2: { + case 2 ... 3: { u32 info1, info3; info1 = aic_ic_read(irqc, AIC2_INFO1); @@ -1065,7 +1082,7 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p off += irqc->info.die_stride; } - if (irqc->info.version == 2) { + if (irqc->info.version == 2 || irqc->info.version == 3) { u32 config = aic_ic_read(irqc, AIC2_CONFIG); config |= AIC2_CONFIG_ENABLE; @@ -1116,3 +1133,4 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p IRQCHIP_DECLARE(apple_aic, "apple,aic", aic_of_ic_init); IRQCHIP_DECLARE(apple_aic2, "apple,aic2", aic_of_ic_init); +IRQCHIP_DECLARE(apple_aic3, "apple,t8122-aic3", aic_of_ic_init); diff --git a/drivers/irqchip/irq-sifive-plic.c b/drivers/irqchip/irq-sifive-plic.c index 210a5795963772..60fd8f91762b18 100644 --- a/drivers/irqchip/irq-sifive-plic.c +++ b/drivers/irqchip/irq-sifive-plic.c @@ -68,15 +68,17 @@ #define PLIC_QUIRK_CP100_CLAIM_REGISTER_ERRATUM 1 struct plic_priv { - struct fwnode_handle *fwnode; - struct cpumask lmask; - struct irq_domain *irqdomain; - void __iomem *regs; - unsigned long plic_quirks; - unsigned int nr_irqs; - unsigned long *prio_save; - u32 gsi_base; - int acpi_plic_id; + struct fwnode_handle *fwnode; + struct cpumask lmask; + struct irq_domain *irqdomain; + void __iomem *regs; + unsigned long plic_quirks; + /* device interrupts + 1 to compensate for the reserved hwirq 0 */ + unsigned int __private total_irqs; + unsigned int irq_groups; + unsigned long *prio_save; + u32 gsi_base; + int acpi_plic_id; }; struct plic_handler { @@ -91,6 +93,12 @@ struct plic_handler { u32 *enable_save; struct plic_priv *priv; }; + +/* + * Macro to deal with the insanity of hardware interrupt 0 being reserved */ +#define for_each_device_irq(iter, priv) \ + for (unsigned int iter = 1; iter < ACCESS_PRIVATE(priv, total_irqs); iter++) + static int plic_parent_irq __ro_after_init; static bool plic_global_setup_done __ro_after_init; static DEFINE_PER_CPU(struct plic_handler, plic_handlers); @@ -257,14 +265,11 @@ static int plic_irq_set_type(struct irq_data *d, unsigned int type) static int plic_irq_suspend(void *data) { - struct plic_priv *priv; - - priv = per_cpu_ptr(&plic_handlers, smp_processor_id())->priv; + struct plic_priv *priv = this_cpu_ptr(&plic_handlers)->priv; - /* irq ID 0 is reserved */ - for (unsigned int i = 1; i < priv->nr_irqs; i++) { - __assign_bit(i, priv->prio_save, - readl(priv->regs + PRIORITY_BASE + i * PRIORITY_PER_ID)); + for_each_device_irq(irq, priv) { + __assign_bit(irq, priv->prio_save, + readl(priv->regs + PRIORITY_BASE + irq * PRIORITY_PER_ID)); } return 0; @@ -272,18 +277,15 @@ static int plic_irq_suspend(void *data) static void plic_irq_resume(void *data) { - unsigned int i, index, cpu; + struct plic_priv *priv = this_cpu_ptr(&plic_handlers)->priv; + unsigned int index, cpu; unsigned long flags; u32 __iomem *reg; - struct plic_priv *priv; - - priv = per_cpu_ptr(&plic_handlers, smp_processor_id())->priv; - /* irq ID 0 is reserved */ - for (i = 1; i < priv->nr_irqs; i++) { - index = BIT_WORD(i); - writel((priv->prio_save[index] & BIT_MASK(i)) ? 1 : 0, - priv->regs + PRIORITY_BASE + i * PRIORITY_PER_ID); + for_each_device_irq(irq, priv) { + index = BIT_WORD(irq); + writel((priv->prio_save[index] & BIT_MASK(irq)) ? 1 : 0, + priv->regs + PRIORITY_BASE + irq * PRIORITY_PER_ID); } for_each_present_cpu(cpu) { @@ -293,7 +295,7 @@ static void plic_irq_resume(void *data) continue; raw_spin_lock_irqsave(&handler->enable_lock, flags); - for (i = 0; i < DIV_ROUND_UP(priv->nr_irqs, 32); i++) { + for (unsigned int i = 0; i < priv->irq_groups; i++) { reg = handler->enable_base + i * sizeof(u32); writel(handler->enable_save[i], reg); } @@ -431,7 +433,7 @@ static u32 cp100_isolate_pending_irq(int nr_irq_groups, struct plic_handler *han static irq_hw_number_t cp100_get_hwirq(struct plic_handler *handler, void __iomem *claim) { - int nr_irq_groups = DIV_ROUND_UP(handler->priv->nr_irqs, 32); + int nr_irq_groups = handler->priv->irq_groups; u32 __iomem *enable = handler->enable_base; irq_hw_number_t hwirq = 0; u32 iso_mask; @@ -614,7 +616,6 @@ static int plic_probe(struct fwnode_handle *fwnode) struct plic_handler *handler; u32 nr_irqs, parent_hwirq; struct plic_priv *priv; - irq_hw_number_t hwirq; void __iomem *regs; int id, context_id; u32 gsi_base; @@ -647,7 +648,16 @@ static int plic_probe(struct fwnode_handle *fwnode) priv->fwnode = fwnode; priv->plic_quirks = plic_quirks; - priv->nr_irqs = nr_irqs; + /* + * The firmware provides the number of device interrupts. As + * hardware interrupt 0 is reserved, the number of total interrupts + * is nr_irqs + 1. + */ + nr_irqs++; + ACCESS_PRIVATE(priv, total_irqs) = nr_irqs; + /* Precalculate the number of register groups */ + priv->irq_groups = DIV_ROUND_UP(nr_irqs, 32); + priv->regs = regs; priv->gsi_base = gsi_base; priv->acpi_plic_id = id; @@ -686,7 +696,7 @@ static int plic_probe(struct fwnode_handle *fwnode) u32 __iomem *enable_base = priv->regs + CONTEXT_ENABLE_BASE + i * CONTEXT_ENABLE_SIZE; - for (int j = 0; j <= nr_irqs / 32; j++) + for (int j = 0; j < priv->irq_groups; j++) writel(0, enable_base + j); } continue; @@ -718,23 +728,21 @@ static int plic_probe(struct fwnode_handle *fwnode) context_id * CONTEXT_ENABLE_SIZE; handler->priv = priv; - handler->enable_save = kcalloc(DIV_ROUND_UP(nr_irqs, 32), - sizeof(*handler->enable_save), GFP_KERNEL); + handler->enable_save = kcalloc(priv->irq_groups, sizeof(*handler->enable_save), + GFP_KERNEL); if (!handler->enable_save) { error = -ENOMEM; goto fail_cleanup_contexts; } done: - for (hwirq = 1; hwirq <= nr_irqs; hwirq++) { + for_each_device_irq(hwirq, priv) { plic_toggle(handler, hwirq, 0); - writel(1, priv->regs + PRIORITY_BASE + - hwirq * PRIORITY_PER_ID); + writel(1, priv->regs + PRIORITY_BASE + hwirq * PRIORITY_PER_ID); } nr_handlers++; } - priv->irqdomain = irq_domain_create_linear(fwnode, nr_irqs + 1, - &plic_irqdomain_ops, priv); + priv->irqdomain = irq_domain_create_linear(fwnode, nr_irqs, &plic_irqdomain_ops, priv); if (WARN_ON(!priv->irqdomain)) { error = -ENOMEM; goto fail_cleanup_contexts; diff --git a/drivers/leds/leds-expresswire.c b/drivers/leds/leds-expresswire.c index bb69be228a6d3c..25c6b159a6ee93 100644 --- a/drivers/leds/leds-expresswire.c +++ b/drivers/leds/leds-expresswire.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -16,37 +17,41 @@ void expresswire_power_off(struct expresswire_common_props *props) { gpiod_set_value_cansleep(props->ctrl_gpio, 0); - usleep_range(props->timing.poweroff_us, props->timing.poweroff_us * 2); + fsleep(props->timing.poweroff_us); } EXPORT_SYMBOL_NS_GPL(expresswire_power_off, "EXPRESSWIRE"); void expresswire_enable(struct expresswire_common_props *props) { + unsigned long flags; + + local_irq_save(flags); + gpiod_set_value(props->ctrl_gpio, 1); udelay(props->timing.detect_delay_us); gpiod_set_value(props->ctrl_gpio, 0); udelay(props->timing.detect_us); gpiod_set_value(props->ctrl_gpio, 1); + + local_irq_restore(flags); } EXPORT_SYMBOL_NS_GPL(expresswire_enable, "EXPRESSWIRE"); -void expresswire_start(struct expresswire_common_props *props) +static void expresswire_start(struct expresswire_common_props *props) { gpiod_set_value(props->ctrl_gpio, 1); udelay(props->timing.data_start_us); } -EXPORT_SYMBOL_NS_GPL(expresswire_start, "EXPRESSWIRE"); -void expresswire_end(struct expresswire_common_props *props) +static void expresswire_end(struct expresswire_common_props *props) { gpiod_set_value(props->ctrl_gpio, 0); udelay(props->timing.end_of_data_low_us); gpiod_set_value(props->ctrl_gpio, 1); udelay(props->timing.end_of_data_high_us); } -EXPORT_SYMBOL_NS_GPL(expresswire_end, "EXPRESSWIRE"); -void expresswire_set_bit(struct expresswire_common_props *props, bool bit) +static void expresswire_set_bit(struct expresswire_common_props *props, bool bit) { if (bit) { gpiod_set_value(props->ctrl_gpio, 0); @@ -60,13 +65,18 @@ void expresswire_set_bit(struct expresswire_common_props *props, bool bit) udelay(props->timing.short_bitset_us); } } -EXPORT_SYMBOL_NS_GPL(expresswire_set_bit, "EXPRESSWIRE"); void expresswire_write_u8(struct expresswire_common_props *props, u8 val) { + unsigned long flags; + + local_irq_save(flags); + expresswire_start(props); for (int i = 7; i >= 0; i--) expresswire_set_bit(props, val & BIT(i)); expresswire_end(props); + + local_irq_restore(flags); } EXPORT_SYMBOL_NS_GPL(expresswire_write_u8, "EXPRESSWIRE"); diff --git a/drivers/leds/rgb/leds-qcom-lpg.c b/drivers/leds/rgb/leds-qcom-lpg.c index 72da0bf469ad8b..f54851dfb42fce 100644 --- a/drivers/leds/rgb/leds-qcom-lpg.c +++ b/drivers/leds/rgb/leds-qcom-lpg.c @@ -369,7 +369,7 @@ static int lpg_lut_store(struct lpg *lpg, struct led_pattern *pattern, { unsigned int idx; u16 val; - int i; + int i, ret; idx = bitmap_find_next_zero_area(lpg->lut_bitmap, lpg->lut_size, 0, len, 0); @@ -379,8 +379,10 @@ static int lpg_lut_store(struct lpg *lpg, struct led_pattern *pattern, for (i = 0; i < len; i++) { val = pattern[i].brightness; - regmap_bulk_write(lpg->map, lpg->lut_base + LPG_LUT_REG(idx + i), - &val, sizeof(val)); + ret = regmap_bulk_write(lpg->map, lpg->lut_base + LPG_LUT_REG(idx + i), + &val, sizeof(val)); + if (ret) + return ret; } bitmap_set(lpg->lut_bitmap, idx, len); diff --git a/drivers/mailbox/pcc.c b/drivers/mailbox/pcc.c index ff292b9e0be9ee..0e0a66359d4c38 100644 --- a/drivers/mailbox/pcc.c +++ b/drivers/mailbox/pcc.c @@ -305,22 +305,6 @@ static void pcc_chan_acknowledge(struct pcc_chan_info *pchan) pcc_chan_reg_read_modify_write(&pchan->db); } -static void *write_response(struct pcc_chan_info *pchan) -{ - struct pcc_header pcc_header; - void *buffer; - int data_len; - - memcpy_fromio(&pcc_header, pchan->chan.shmem, - sizeof(pcc_header)); - data_len = pcc_header.length - sizeof(u32) + sizeof(struct pcc_header); - - buffer = pchan->chan.rx_alloc(pchan->chan.mchan->cl, data_len); - if (buffer != NULL) - memcpy_fromio(buffer, pchan->chan.shmem, data_len); - return buffer; -} - /** * pcc_mbox_irq - PCC mailbox interrupt handler * @irq: interrupt number @@ -332,8 +316,6 @@ static irqreturn_t pcc_mbox_irq(int irq, void *p) { struct pcc_chan_info *pchan; struct mbox_chan *chan = p; - struct pcc_header *pcc_header = chan->active_req; - void *handle = NULL; pchan = chan->con_priv; @@ -357,17 +339,7 @@ static irqreturn_t pcc_mbox_irq(int irq, void *p) * required to avoid any possible race in updatation of this flag. */ pchan->chan_in_use = false; - - if (pchan->chan.rx_alloc) - handle = write_response(pchan); - - if (chan->active_req) { - pcc_header = chan->active_req; - if (pcc_header->flags & PCC_CMD_COMPLETION_NOTIFY) - mbox_chan_txdone(chan, 0); - } - - mbox_chan_received_data(chan, handle); + mbox_chan_received_data(chan, NULL); pcc_chan_acknowledge(pchan); @@ -411,24 +383,9 @@ pcc_mbox_request_channel(struct mbox_client *cl, int subspace_id) pcc_mchan = &pchan->chan; pcc_mchan->shmem = acpi_os_ioremap(pcc_mchan->shmem_base_addr, pcc_mchan->shmem_size); - if (!pcc_mchan->shmem) - goto err; - - pcc_mchan->manage_writes = false; - - /* This indicates that the channel is ready to accept messages. - * This needs to happen after the channel has registered - * its callback. There is no access point to do that in - * the mailbox API. That implies that the mailbox client must - * have set the allocate callback function prior to - * sending any messages. - */ - if (pchan->type == ACPI_PCCT_TYPE_EXT_PCC_SLAVE_SUBSPACE) - pcc_chan_reg_read_modify_write(&pchan->cmd_update); - - return pcc_mchan; + if (pcc_mchan->shmem) + return pcc_mchan; -err: mbox_free_channel(chan); return ERR_PTR(-ENXIO); } @@ -459,38 +416,8 @@ void pcc_mbox_free_channel(struct pcc_mbox_chan *pchan) } EXPORT_SYMBOL_GPL(pcc_mbox_free_channel); -static int pcc_write_to_buffer(struct mbox_chan *chan, void *data) -{ - struct pcc_chan_info *pchan = chan->con_priv; - struct pcc_mbox_chan *pcc_mbox_chan = &pchan->chan; - struct pcc_header *pcc_header = data; - - if (!pchan->chan.manage_writes) - return 0; - - /* The PCC header length includes the command field - * but not the other values from the header. - */ - int len = pcc_header->length - sizeof(u32) + sizeof(struct pcc_header); - u64 val; - - pcc_chan_reg_read(&pchan->cmd_complete, &val); - if (!val) { - pr_info("%s pchan->cmd_complete not set", __func__); - return -1; - } - memcpy_toio(pcc_mbox_chan->shmem, data, len); - return 0; -} - - /** - * pcc_send_data - Called from Mailbox Controller code. If - * pchan->chan.rx_alloc is set, then the command complete - * flag is checked and the data is written to the shared - * buffer io memory. - * - * If pchan->chan.rx_alloc is not set, then it is used + * pcc_send_data - Called from Mailbox Controller code. Used * here only to ring the channel doorbell. The PCC client * specific read/write is done in the client driver in * order to maintain atomicity over PCC channel once @@ -506,37 +433,17 @@ static int pcc_send_data(struct mbox_chan *chan, void *data) int ret; struct pcc_chan_info *pchan = chan->con_priv; - ret = pcc_write_to_buffer(chan, data); - if (ret) - return ret; - ret = pcc_chan_reg_read_modify_write(&pchan->cmd_update); if (ret) return ret; ret = pcc_chan_reg_read_modify_write(&pchan->db); - if (!ret && pchan->plat_irq > 0) pchan->chan_in_use = true; return ret; } - -static bool pcc_last_tx_done(struct mbox_chan *chan) -{ - struct pcc_chan_info *pchan = chan->con_priv; - u64 val; - - pcc_chan_reg_read(&pchan->cmd_complete, &val); - if (!val) - return false; - else - return true; -} - - - /** * pcc_startup - Called from Mailbox Controller code. Used here * to request the interrupt. @@ -582,7 +489,6 @@ static const struct mbox_chan_ops pcc_chan_ops = { .send_data = pcc_send_data, .startup = pcc_startup, .shutdown = pcc_shutdown, - .last_tx_done = pcc_last_tx_done, }; /** diff --git a/drivers/mcb/mcb-core.c b/drivers/mcb/mcb-core.c index c1367223e71a0f..3d487d75c483d7 100644 --- a/drivers/mcb/mcb-core.c +++ b/drivers/mcb/mcb-core.c @@ -85,7 +85,8 @@ static void mcb_remove(struct device *dev) struct mcb_device *mdev = to_mcb_device(dev); struct module *carrier_mod; - mdrv->remove(mdev); + if (mdrv->remove) + mdrv->remove(mdev); carrier_mod = mdev->dev.parent->driver->owner; module_put(carrier_mod); @@ -176,13 +177,13 @@ static const struct device_type mcb_carrier_device_type = { * @owner: The @mcb_driver's module * @mod_name: The name of the @mcb_driver's module * - * Register a @mcb_driver at the system. Perform some sanity checks, if - * the .probe and .remove methods are provided by the driver. + * Register a @mcb_driver at the system. Perform a sanity check, if + * .probe method is provided by the driver. */ int __mcb_register_driver(struct mcb_driver *drv, struct module *owner, const char *mod_name) { - if (!drv->probe || !drv->remove) + if (!drv->probe) return -EINVAL; drv->driver.owner = owner; diff --git a/drivers/md/dm-zone.c b/drivers/md/dm-zone.c index c95e417194b33c..f29acf64429a75 100644 --- a/drivers/md/dm-zone.c +++ b/drivers/md/dm-zone.c @@ -50,7 +50,7 @@ int dm_blk_report_zones(struct gendisk *disk, sector_t sector, { struct mapped_device *md = disk->private_data; struct dm_table *map; - struct dm_table *zone_revalidate_map = md->zone_revalidate_map; + struct dm_table *zone_revalidate_map = READ_ONCE(md->zone_revalidate_map); int srcu_idx, ret = -EIO; bool put_table = false; @@ -60,11 +60,13 @@ int dm_blk_report_zones(struct gendisk *disk, sector_t sector, * Zone revalidation during __bind() is in progress, but this * call is from a different process */ - if (dm_suspended_md(md)) - return -EAGAIN; - map = dm_get_live_table(md, &srcu_idx); put_table = true; + + if (dm_suspended_md(md)) { + ret = -EAGAIN; + goto do_put_table; + } } else { /* Zone revalidation during __bind() */ map = zone_revalidate_map; @@ -79,6 +81,7 @@ int dm_blk_report_zones(struct gendisk *disk, sector_t sector, ret = dm_blk_do_report_zones(md, map, nr_zones, &dm_args); } +do_put_table: if (put_table) dm_put_live_table(md, srcu_idx); diff --git a/drivers/md/dm.c b/drivers/md/dm.c index b6327920226012..8e029f6289c10c 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -1364,6 +1364,8 @@ void dm_submit_bio_remap(struct bio *clone, struct bio *tgt_clone) if (!tgt_clone) tgt_clone = clone; + bio_clone_blkg_association(tgt_clone, io->orig_bio); + /* * Account io->origin_bio to DM dev on behalf of target * that took ownership of IO with DM_MAPIO_SUBMITTED. diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index 9c1ade19b7741e..cd713a7dc27064 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -712,8 +712,10 @@ static int llbitmap_suspend_timeout(struct llbitmap *llbitmap, int page_idx) percpu_ref_kill(&pctl->active); if (!wait_event_timeout(pctl->wait, percpu_ref_is_zero(&pctl->active), - llbitmap->mddev->bitmap_info.daemon_sleep * HZ)) + llbitmap->mddev->bitmap_info.daemon_sleep * HZ)) { + percpu_ref_resurrect(&pctl->active); return -ETIMEDOUT; + } return 0; } diff --git a/drivers/md/md.h b/drivers/md/md.h index 6985f2829bbd6a..3bfbee595156d4 100644 --- a/drivers/md/md.h +++ b/drivers/md/md.h @@ -737,8 +737,8 @@ static inline int mddev_trylock(struct mddev *mddev) int ret; ret = mutex_trylock(&mddev->reconfig_mutex); - if (!ret && test_bit(MD_DELETED, &mddev->flags)) { - ret = -ENODEV; + if (ret && test_bit(MD_DELETED, &mddev->flags)) { + ret = 0; mutex_unlock(&mddev->reconfig_mutex); } return ret; diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index 57d50465eed1b7..cc9914bd15c197 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -3254,6 +3254,7 @@ static int raid1_run(struct mddev *mddev) if (!mddev_is_dm(mddev)) { ret = raid1_set_limits(mddev); if (ret) { + md_unregister_thread(mddev, &conf->thread); if (!mddev->private) raid1_free(mddev, conf); return ret; diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index 84be4cc7e87399..3a591e60a14497 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -3402,7 +3402,6 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, !test_bit(In_sync, &rdev->flags)) continue; /* This is where we read from */ - any_working = 1; sector = r10_bio->devs[j].addr; if (is_badblock(rdev, sector, max_sync, @@ -3417,6 +3416,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, continue; } } + any_working = 1; bio = r10_bio->devs[0].bio; bio->bi_next = biolist; biolist = bio; diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 8dc98f545969ff..bdf248db1330ae 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -3751,9 +3751,14 @@ static int need_this_block(struct stripe_head *sh, struct stripe_head_state *s, struct r5dev *dev = &sh->dev[disk_idx]; struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]], &sh->dev[s->failed_num[1]] }; + struct mddev *mddev = sh->raid_conf->mddev; + bool force_rcw = false; int i; - bool force_rcw = (sh->raid_conf->rmw_level == PARITY_DISABLE_RMW); + if (sh->raid_conf->rmw_level == PARITY_DISABLE_RMW || + (mddev->bitmap_ops && mddev->bitmap_ops->blocks_synced && + !mddev->bitmap_ops->blocks_synced(mddev, sh->sector))) + force_rcw = true; if (test_bit(R5_LOCKED, &dev->flags) || test_bit(R5_UPTODATE, &dev->flags)) @@ -8057,7 +8062,8 @@ static int raid5_run(struct mddev *mddev) goto abort; } - if (log_init(conf, journal_dev, raid5_has_ppl(conf))) + ret = log_init(conf, journal_dev, raid5_has_ppl(conf)); + if (ret) goto abort; return 0; diff --git a/drivers/media/common/videobuf2/videobuf2-dma-sg.c b/drivers/media/common/videobuf2/videobuf2-dma-sg.c index b3bf2173c14e1b..7c30731cb9a57b 100644 --- a/drivers/media/common/videobuf2/videobuf2-dma-sg.c +++ b/drivers/media/common/videobuf2/videobuf2-dma-sg.c @@ -345,6 +345,7 @@ static int vb2_dma_sg_mmap(void *buf_priv, struct vm_area_struct *vma) return err; } + vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP); /* * Use common vm_area operations to track buffer refcount. */ diff --git a/drivers/media/i2c/ccs/ccs-core.c b/drivers/media/i2c/ccs/ccs-core.c index f8523140784c71..43f7d515bab670 100644 --- a/drivers/media/i2c/ccs/ccs-core.c +++ b/drivers/media/i2c/ccs/ccs-core.c @@ -3425,7 +3425,21 @@ static int ccs_probe(struct i2c_client *client) sensor->scale_m = CCS_LIM(sensor, SCALER_N_MIN); /* prepare PLL configuration input values */ - sensor->pll.bus_type = CCS_PLL_BUS_TYPE_CSI2_DPHY; + switch (sensor->hwcfg.csi_signalling_mode) { + case CCS_CSI_SIGNALING_MODE_CSI_2_CPHY: + sensor->pll.bus_type = CCS_PLL_BUS_TYPE_CSI2_CPHY; + break; + case CCS_CSI_SIGNALING_MODE_CSI_2_DPHY: + case SMIAPP_CSI_SIGNALLING_MODE_CCP2_DATA_CLOCK: + case SMIAPP_CSI_SIGNALLING_MODE_CCP2_DATA_STROBE: + sensor->pll.bus_type = CCS_PLL_BUS_TYPE_CSI2_DPHY; + break; + default: + dev_err(&client->dev, "unsupported signalling mode %u\n", + sensor->hwcfg.csi_signalling_mode); + rval = -EINVAL; + goto out_cleanup; + } sensor->pll.csi2.lanes = sensor->hwcfg.lanes; if (CCS_LIM(sensor, CLOCK_CALCULATION) & CCS_CLOCK_CALCULATION_LANE_SPEED) { diff --git a/drivers/media/pci/mgb4/mgb4_trigger.c b/drivers/media/pci/mgb4/mgb4_trigger.c index 4f9a35904b4186..70cad324df608d 100644 --- a/drivers/media/pci/mgb4/mgb4_trigger.c +++ b/drivers/media/pci/mgb4/mgb4_trigger.c @@ -115,7 +115,7 @@ static int probe_trigger(struct iio_dev *indio_dev, int irq) if (!st->trig) return -ENOMEM; - ret = request_irq(irq, &iio_trigger_generic_data_rdy_poll, 0, + ret = request_irq(irq, &iio_trigger_generic_data_rdy_poll, IRQF_NO_THREAD, "mgb4-trigger", st->trig); if (ret) goto error_free_trig; diff --git a/drivers/media/platform/Kconfig b/drivers/media/platform/Kconfig index 3f0b7bb68cc940..8b31f087e7da20 100644 --- a/drivers/media/platform/Kconfig +++ b/drivers/media/platform/Kconfig @@ -65,6 +65,7 @@ config VIDEO_MUX source "drivers/media/platform/allegro-dvt/Kconfig" source "drivers/media/platform/amlogic/Kconfig" source "drivers/media/platform/amphion/Kconfig" +source "drivers/media/platform/apple/Kconfig" source "drivers/media/platform/arm/Kconfig" source "drivers/media/platform/aspeed/Kconfig" source "drivers/media/platform/atmel/Kconfig" diff --git a/drivers/media/platform/Makefile b/drivers/media/platform/Makefile index 6d5f79ddfcc37a..0e93f7d860caf4 100644 --- a/drivers/media/platform/Makefile +++ b/drivers/media/platform/Makefile @@ -8,6 +8,7 @@ obj-y += allegro-dvt/ obj-y += amlogic/ obj-y += amphion/ +obj-y += apple/ obj-y += arm/ obj-y += aspeed/ obj-y += atmel/ diff --git a/drivers/media/platform/apple/Kconfig b/drivers/media/platform/apple/Kconfig new file mode 100644 index 00000000000000..f16508bff5242a --- /dev/null +++ b/drivers/media/platform/apple/Kconfig @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0-only + +comment "Apple media platform drivers" + +source "drivers/media/platform/apple/isp/Kconfig" diff --git a/drivers/media/platform/apple/Makefile b/drivers/media/platform/apple/Makefile new file mode 100644 index 00000000000000..d8fe985b0e6c37 --- /dev/null +++ b/drivers/media/platform/apple/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-y += isp/ diff --git a/drivers/media/platform/apple/isp/.gitignore b/drivers/media/platform/apple/isp/.gitignore new file mode 100644 index 00000000000000..bd7fab40e0d98a --- /dev/null +++ b/drivers/media/platform/apple/isp/.gitignore @@ -0,0 +1 @@ +.clang-format diff --git a/drivers/media/platform/apple/isp/Kconfig b/drivers/media/platform/apple/isp/Kconfig new file mode 100644 index 00000000000000..5695bef44adf5b --- /dev/null +++ b/drivers/media/platform/apple/isp/Kconfig @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config VIDEO_APPLE_ISP + tristate "Apple Silicon Image Signal Processor driver" + select VIDEOBUF2_CORE + select VIDEOBUF2_V4L2 + select VIDEOBUF2_DMA_SG + depends on ARCH_APPLE || COMPILE_TEST + depends on OF_ADDRESS + depends on V4L_PLATFORM_DRIVERS + depends on VIDEO_DEV diff --git a/drivers/media/platform/apple/isp/Makefile b/drivers/media/platform/apple/isp/Makefile new file mode 100644 index 00000000000000..4649f32987f025 --- /dev/null +++ b/drivers/media/platform/apple/isp/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +apple-isp-y := isp-cam.o isp-cmd.o isp-drv.o isp-fw.o isp-iommu.o isp-ipc.o isp-v4l2.o +obj-$(CONFIG_VIDEO_APPLE_ISP) += apple-isp.o diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c new file mode 100644 index 00000000000000..c889173bd348f3 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -0,0 +1,498 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#include + +#include "isp-cam.h" +#include "isp-cmd.h" +#include "isp-fw.h" +#include "isp-iommu.h" + +#define ISP_MAX_PRESETS 32 + +struct isp_setfile { + u32 version; + u32 magic; + const char *path; + size_t size; +}; + +// clang-format off +static const struct isp_setfile isp_setfiles[] = { + [ISP_IMX248_1820_01] = {0x248, 0x18200103, "apple/isp_1820_01XX.dat", 0x442c}, + [ISP_IMX248_1822_02] = {0x248, 0x18220201, "apple/isp_1822_02XX.dat", 0x442c}, + [ISP_IMX343_5221_02] = {0x343, 0x52210211, "apple/isp_5221_02XX.dat", 0x4870}, + [ISP_IMX354_9251_02] = {0x354, 0x92510208, "apple/isp_9251_02XX.dat", 0xa5ec}, + [ISP_IMX356_4820_01] = {0x356, 0x48200107, "apple/isp_4820_01XX.dat", 0x9324}, + [ISP_IMX356_4820_02] = {0x356, 0x48200206, "apple/isp_4820_02XX.dat", 0x9324}, + [ISP_IMX364_8720_01] = {0x364, 0x87200103, "apple/isp_8720_01XX.dat", 0x36ac}, + [ISP_IMX364_8723_01] = {0x364, 0x87230101, "apple/isp_8723_01XX.dat", 0x361c}, + [ISP_IMX372_3820_01] = {0x372, 0x38200108, "apple/isp_3820_01XX.dat", 0xfdb0}, + [ISP_IMX372_3820_02] = {0x372, 0x38200205, "apple/isp_3820_02XX.dat", 0xfdb0}, + [ISP_IMX372_3820_11] = {0x372, 0x38201104, "apple/isp_3820_11XX.dat", 0xfdb0}, + [ISP_IMX372_3820_12] = {0x372, 0x38201204, "apple/isp_3820_12XX.dat", 0xfdb0}, + [ISP_IMX405_9720_01] = {0x405, 0x97200102, "apple/isp_9720_01XX.dat", 0x92c8}, + [ISP_IMX405_9721_01] = {0x405, 0x97210102, "apple/isp_9721_01XX.dat", 0x9818}, + [ISP_IMX405_9723_01] = {0x405, 0x97230101, "apple/isp_9723_01XX.dat", 0x92c8}, + [ISP_IMX414_2520_01] = {0x414, 0x25200102, "apple/isp_2520_01XX.dat", 0xa444}, + [ISP_IMX503_7820_01] = {0x503, 0x78200109, "apple/isp_7820_01XX.dat", 0xb268}, + [ISP_IMX503_7820_02] = {0x503, 0x78200206, "apple/isp_7820_02XX.dat", 0xb268}, + [ISP_IMX505_3921_01] = {0x505, 0x39210102, "apple/isp_3921_01XX.dat", 0x89b0}, + [ISP_IMX514_2820_01] = {0x514, 0x28200108, "apple/isp_2820_01XX.dat", 0xa198}, + [ISP_IMX514_2820_02] = {0x514, 0x28200205, "apple/isp_2820_02XX.dat", 0xa198}, + [ISP_IMX514_2820_03] = {0x514, 0x28200305, "apple/isp_2820_03XX.dat", 0xa198}, + [ISP_IMX514_2820_04] = {0x514, 0x28200405, "apple/isp_2820_04XX.dat", 0xa198}, + [ISP_IMX558_1921_01] = {0x558, 0x19210106, "apple/isp_1921_01XX.dat", 0xad40}, + [ISP_IMX558_1922_02] = {0x558, 0x19220201, "apple/isp_1922_02XX.dat", 0xad40}, + [ISP_IMX603_7920_01] = {0x603, 0x79200109, "apple/isp_7920_01XX.dat", 0xad2c}, + [ISP_IMX603_7920_02] = {0x603, 0x79200205, "apple/isp_7920_02XX.dat", 0xad2c}, + [ISP_IMX603_7921_01] = {0x603, 0x79210104, "apple/isp_7921_01XX.dat", 0xad90}, + [ISP_IMX613_4920_01] = {0x613, 0x49200108, "apple/isp_4920_01XX.dat", 0x9324}, + [ISP_IMX613_4920_02] = {0x613, 0x49200204, "apple/isp_4920_02XX.dat", 0x9324}, + [ISP_IMX614_2921_01] = {0x614, 0x29210107, "apple/isp_2921_01XX.dat", 0xed6c}, + [ISP_IMX614_2921_02] = {0x614, 0x29210202, "apple/isp_2921_02XX.dat", 0xed6c}, + [ISP_IMX614_2922_02] = {0x614, 0x29220201, "apple/isp_2922_02XX.dat", 0xed6c}, + [ISP_IMX633_3622_01] = {0x633, 0x36220111, "apple/isp_3622_01XX.dat", 0x100d4}, + [ISP_IMX703_7721_01] = {0x703, 0x77210106, "apple/isp_7721_01XX.dat", 0x936c}, + [ISP_IMX703_7722_01] = {0x703, 0x77220106, "apple/isp_7722_01XX.dat", 0xac20}, + [ISP_IMX713_4721_01] = {0x713, 0x47210107, "apple/isp_4721_01XX.dat", 0x936c}, + [ISP_IMX713_4722_01] = {0x713, 0x47220109, "apple/isp_4722_01XX.dat", 0x9218}, + [ISP_IMX714_2022_01] = {0x714, 0x20220107, "apple/isp_2022_01XX.dat", 0xa198}, + [ISP_IMX772_3721_01] = {0x772, 0x37210106, "apple/isp_3721_01XX.dat", 0xfdf8}, + [ISP_IMX772_3721_11] = {0x772, 0x37211106, "apple/isp_3721_11XX.dat", 0xfe14}, + [ISP_IMX772_3722_01] = {0x772, 0x37220104, "apple/isp_3722_01XX.dat", 0xfca4}, + [ISP_IMX772_3723_01] = {0x772, 0x37230106, "apple/isp_3723_01XX.dat", 0xfca4}, + [ISP_IMX814_2123_01] = {0x814, 0x21230101, "apple/isp_2123_01XX.dat", 0xed54}, + [ISP_IMX853_7622_01] = {0x853, 0x76220112, "apple/isp_7622_01XX.dat", 0x247f8}, + [ISP_IMX913_7523_01] = {0x913, 0x75230107, "apple/isp_7523_01XX.dat", 0x247f8}, + [ISP_VD56G0_6221_01] = {0xd56, 0x62210102, "apple/isp_6221_01XX.dat", 0x1b80}, + [ISP_VD56G0_6222_01] = {0xd56, 0x62220102, "apple/isp_6222_01XX.dat", 0x1b80}, +}; +// clang-format on + +static int isp_ch_get_sensor_id(struct apple_isp *isp, u32 ch) +{ + struct isp_format *fmt = isp_get_format(isp, ch); + enum isp_sensor_id id; + int err = 0; + + /* TODO need more datapoints to figure out the sub-versions + * Defaulting to 1st release for now, the calib files aren't too different. + */ + switch (fmt->version) { + case 0x248: + id = ISP_IMX248_1820_01; + break; + case 0x343: + id = ISP_IMX343_5221_02; + break; + case 0x354: + id = ISP_IMX354_9251_02; + break; + case 0x356: + id = ISP_IMX356_4820_01; + break; + case 0x364: + id = ISP_IMX364_8720_01; + break; + case 0x372: + id = ISP_IMX372_3820_01; + break; + case 0x405: + id = ISP_IMX405_9720_01; + break; + case 0x414: + id = ISP_IMX414_2520_01; + break; + case 0x503: + id = ISP_IMX503_7820_01; + break; + case 0x505: + id = ISP_IMX505_3921_01; + break; + case 0x514: + id = ISP_IMX514_2820_01; + break; + case 0x558: + id = ISP_IMX558_1921_01; + break; + case 0x603: + id = ISP_IMX603_7920_01; + break; + case 0x613: + id = ISP_IMX613_4920_01; + break; + case 0x614: + id = ISP_IMX614_2921_01; + break; + case 0x633: + id = ISP_IMX633_3622_01; + break; + case 0x703: + id = ISP_IMX703_7721_01; + break; + case 0x713: + id = ISP_IMX713_4721_01; + break; + case 0x714: + id = ISP_IMX714_2022_01; + break; + case 0x772: + id = ISP_IMX772_3721_01; + break; + case 0x814: + id = ISP_IMX814_2123_01; + break; + case 0x853: + id = ISP_IMX853_7622_01; + break; + case 0x913: + id = ISP_IMX913_7523_01; + break; + case 0xd56: + id = ISP_VD56G0_6221_01; + break; + default: + err = -EINVAL; + break; + } + + if (err) + dev_err(isp->dev, "invalid sensor version: 0x%x\n", + fmt->version); + else + fmt->id = id; + + return err; +} + +static int isp_ch_get_camera_preset(struct apple_isp *isp, u32 ch, u32 ps) +{ + int err = 0; + + struct cmd_ch_camera_config *args; /* Too big to allocate on stack */ + args = kzalloc(sizeof(*args), GFP_KERNEL); + if (!args) + return -ENOMEM; + + err = isp_cmd_ch_camera_config_get(isp, ch, ps, args); + if (err) + goto exit; + + pr_info("apple-isp: ps: CISP_CMD_CH_CAMERA_CONFIG_GET: %d\n", ps); + print_hex_dump(KERN_INFO, "apple-isp: ps: ", DUMP_PREFIX_NONE, 32, 4, + args, sizeof(*args), false); + +exit: + kfree(args); + + return err; +} + +static int isp_ch_cache_sensor_info(struct apple_isp *isp, u32 ch) +{ + struct isp_format *fmt = isp_get_format(isp, ch); + int err = 0; + + struct cmd_ch_info *args; /* Too big to allocate on stack */ + args = kzalloc(sizeof(*args), GFP_KERNEL); + if (!args) + return -ENOMEM; + + err = isp_cmd_ch_info_get(isp, ch, args); + if (err) + goto exit; + + dev_info(isp->dev, "found sensor %x %s on ch %d\n", args->version, + args->module_sn, ch); + + fmt->version = args->version; + + pr_info("apple-isp: ch: CISP_CMD_CH_INFO_GET: %d\n", ch); + print_hex_dump(KERN_INFO, "apple-isp: ch: ", DUMP_PREFIX_NONE, 32, 4, + args, sizeof(*args), false); + + for (u32 ps = 0; ps < args->num_presets; ps++) { + isp_ch_get_camera_preset(isp, ch, ps); + } + + err = isp_ch_get_sensor_id(isp, ch); + if (err || + (fmt->id != ISP_IMX248_1820_01 && fmt->id != ISP_IMX558_1921_01 && + fmt->id != ISP_IMX364_8720_01)) { + dev_err(isp->dev, + "ch %d: unsupported sensor. Please file a bug report with hardware info & dmesg trace.\n", + ch); + return -ENODEV; + } + +exit: + kfree(args); + + return err; +} + +static int isp_detect_camera(struct apple_isp *isp) +{ + int err; + + struct cmd_config_get args; + memset(&args, 0, sizeof(args)); + + err = isp_cmd_config_get(isp, &args); + if (err) + return err; + + pr_info("apple-isp: CISP_CMD_CONFIG_GET: \n"); + print_hex_dump(KERN_INFO, "apple-isp: ", DUMP_PREFIX_NONE, 32, 4, &args, + sizeof(args), false); + + if (!args.num_channels) { + dev_err(isp->dev, "did not detect any channels\n"); + return -ENODEV; + } + + if (args.num_channels > ISP_MAX_CHANNELS) { + dev_warn(isp->dev, "found %d channels when maximum is %d\n", + args.num_channels, ISP_MAX_CHANNELS); + args.num_channels = ISP_MAX_CHANNELS; + } + + if (args.num_channels > 1) { + dev_warn( + isp->dev, + "warning: driver doesn't support multiple channels. Please file a bug report with hardware info & dmesg trace.\n"); + } + + isp->num_channels = args.num_channels; + isp->current_ch = 0; + + err = isp_ch_cache_sensor_info(isp, isp->current_ch); + if (err) { + dev_err(isp->dev, "failed to cache sensor info\n"); + return err; + } + + return 0; +} + +int apple_isp_detect_camera(struct apple_isp *isp) +{ + int err; + + /* RPM must be enabled prior to calling this */ + err = apple_isp_firmware_boot(isp); + if (err) { + dev_err(isp->dev, + "failed to boot firmware for initial sensor detection: %d\n", + err); + return -EPROBE_DEFER; + } + + err = isp_detect_camera(isp); + + isp_cmd_flicker_sensor_set(isp, 0); + + isp_cmd_ch_stop(isp, 0); + isp_cmd_ch_buffer_return(isp, isp->current_ch); + + apple_isp_firmware_shutdown(isp); + + return err; +} + +static int isp_ch_load_setfile(struct apple_isp *isp, u32 ch) +{ + struct isp_format *fmt = isp_get_format(isp, ch); + const struct isp_setfile *setfile = &isp_setfiles[fmt->id]; + const struct firmware *fw; + u32 magic; + int err; + + err = request_firmware(&fw, setfile->path, isp->dev); + if (err) { + dev_err(isp->dev, "failed to request setfile '%s': %d\n", + setfile->path, err); + return err; + } + + if (fw->size < setfile->size) { + dev_err(isp->dev, "setfile too small (0x%lx/0x%zx)\n", fw->size, + setfile->size); + release_firmware(fw); + return -EINVAL; + } + + magic = be32_to_cpup((__be32 *)fw->data); + if (magic != setfile->magic) { + dev_err(isp->dev, "setfile '%s' corrupted?\n", setfile->path); + release_firmware(fw); + return -EINVAL; + } + + memcpy(isp->data_surf->virt, (void *)fw->data, setfile->size); + release_firmware(fw); + + return isp_cmd_ch_set_file_load(isp, ch, isp->data_surf->iova, + setfile->size); +} + +static int isp_ch_configure_capture(struct apple_isp *isp, u32 ch) +{ + struct isp_format *fmt = isp_get_format(isp, ch); + int err; + + isp_cmd_flicker_sensor_set(isp, 0); + + /* The setfile isn't requisite but then we don't get calibration */ + err = isp_ch_load_setfile(isp, ch); + if (err) { + dev_err(isp->dev, "warning: calibration data not loaded: %d\n", + err); + + /* If this failed due to a signal, propagate */ + if (err == -EINTR) + return err; + } + + if (isp->hw->lpdp) { + err = isp_cmd_ch_lpdp_hs_receiver_tuning_set(isp, ch, 1, 15); + if (err) + return err; + } + + err = isp_cmd_ch_sbs_enable(isp, ch, 1); + if (err) + return err; + + err = isp_cmd_ch_camera_config_select(isp, ch, fmt->preset->index); + if (err) + return err; + + err = isp_cmd_ch_buffer_recycle_mode_set( + isp, ch, CISP_BUFFER_RECYCLE_MODE_EMPTY_ONLY); + if (err) + return err; + + err = isp_cmd_ch_buffer_recycle_start(isp, ch); + if (err) + return err; + + err = isp_cmd_ch_crop_set(isp, ch, fmt->preset->crop_offset.x, + fmt->preset->crop_offset.y, + fmt->preset->crop_size.x, + fmt->preset->crop_size.y); + if (err) + return err; + + err = isp_cmd_ch_output_config_set(isp, ch, fmt->preset->output_dim.x, + fmt->preset->output_dim.y, + fmt->strides, CISP_COLORSPACE_REC709, + CISP_OUTPUT_FORMAT_YUV_2PLANE); + if (err) + return err; + + err = isp_cmd_ch_preview_stream_set(isp, ch, 1); + if (err) + return err; + + err = isp_cmd_ch_cnr_start(isp, ch); + if (err) + return err; + + err = isp_cmd_ch_mbnr_enable(isp, ch, 0, ISP_MBNR_MODE_ENABLE, 1); + if (err) + return err; + + err = isp_cmd_apple_ch_ae_fd_scene_metering_config_set(isp, ch); + if (err) + return err; + + err = isp_cmd_apple_ch_ae_metering_mode_set(isp, ch, 3); + if (err) + return err; + + err = isp_cmd_ch_ae_stability_set(isp, ch, 32); + if (err) + return err; + + err = isp_cmd_ch_ae_stability_to_stable_set(isp, ch, 20); + if (err) + return err; + + err = isp_cmd_ch_sif_pixel_format_set(isp, ch); + if (err) + return err; + + err = isp_cmd_ch_ae_frame_rate_max_set(isp, ch, ISP_FRAME_RATE_DEN); + if (err) + return err; + + err = isp_cmd_ch_ae_frame_rate_min_set(isp, ch, ISP_FRAME_RATE_DEN2); + if (err) + return err; + + err = isp_cmd_apple_ch_temporal_filter_start(isp, ch, isp->temporal_filter); + if (err) + return err; + + err = isp_cmd_apple_ch_motion_history_start(isp, ch); + if (err) + return err; + + err = isp_cmd_apple_ch_temporal_filter_enable(isp, ch); + if (err) + return err; + + err = isp_cmd_ch_buffer_pool_config_set(isp, ch, CISP_POOL_TYPE_META); + if (err) + return err; + + err = isp_cmd_ch_buffer_pool_config_set(isp, ch, + CISP_POOL_TYPE_META_CAPTURE); + if (err) + return err; + + return 0; +} + +static int isp_configure_capture(struct apple_isp *isp) +{ + return isp_ch_configure_capture(isp, isp->current_ch); +} + +int apple_isp_start_camera(struct apple_isp *isp) +{ + int err; + + err = apple_isp_firmware_boot(isp); + if (err < 0) { + dev_err(isp->dev, "failed to boot firmware: %d\n", err); + return err; + } + + err = isp_configure_capture(isp); + if (err) { + dev_err(isp->dev, "failed to configure capture: %d\n", err); + apple_isp_firmware_shutdown(isp); + return err; + } + + return 0; +} + +void apple_isp_stop_camera(struct apple_isp *isp) +{ + apple_isp_firmware_shutdown(isp); +} + +int apple_isp_start_capture(struct apple_isp *isp) +{ + return isp_cmd_ch_start(isp, 0); // TODO channel mask +} + +void apple_isp_stop_capture(struct apple_isp *isp) +{ + isp_cmd_ch_stop(isp, 0); // TODO channel mask + isp_cmd_ch_buffer_return(isp, isp->current_ch); +} diff --git a/drivers/media/platform/apple/isp/isp-cam.h b/drivers/media/platform/apple/isp/isp-cam.h new file mode 100644 index 00000000000000..f4fa4224c7a934 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-cam.h @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#ifndef __ISP_CAM_H__ +#define __ISP_CAM_H__ + +#include "isp-drv.h" + +#define ISP_FRAME_RATE_NUM 256 +#define ISP_FRAME_RATE_DEN 7680 +#define ISP_FRAME_RATE_DEN2 3840 + +int apple_isp_detect_camera(struct apple_isp *isp); + +int apple_isp_start_camera(struct apple_isp *isp); +void apple_isp_stop_camera(struct apple_isp *isp); + +int apple_isp_start_capture(struct apple_isp *isp); +void apple_isp_stop_capture(struct apple_isp *isp); + +#endif /* __ISP_CAM_H__ */ diff --git a/drivers/media/platform/apple/isp/isp-cmd.c b/drivers/media/platform/apple/isp/isp-cmd.c new file mode 100644 index 00000000000000..ee491d2cb42c5b --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-cmd.c @@ -0,0 +1,634 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#include "isp-cmd.h" +#include "isp-drv.h" +#include "isp-iommu.h" +#include "isp-ipc.h" + +#define CISP_OPCODE_SHIFT 32UL +#define CISP_OPCODE(x) (((u64)(x)) << CISP_OPCODE_SHIFT) +#define CISP_OPCODE_GET(x) (((u64)(x)) >> CISP_OPCODE_SHIFT) + +#define CISP_TIMEOUT msecs_to_jiffies(3000) +#define CISP_SEND_IN(x, a) (cisp_send((x), &(a), sizeof(a), 0, CISP_TIMEOUT)) +#define CISP_SEND_INOUT(x, a) (cisp_send((x), &(a), sizeof(a), sizeof(a), CISP_TIMEOUT)) +#define CISP_SEND_OUT(x, a) (cisp_send_read((x), (a), sizeof(*a), sizeof(*a))) +#define CISP_POST_IN(x, a) (cisp_send((x), &(a), sizeof(a), 0, 0)) +#define CISP_POST_INOUT(x, a) (cisp_send((x), &(a), sizeof(a), sizeof(a), 0)) + +static int cisp_send(struct apple_isp *isp, void *args, u32 insize, u32 outsize, int timeout) +{ + struct isp_channel *chan = isp->chan_io; + struct isp_message *req = &chan->req; + int err; + + req->arg0 = isp->cmd_iova; + req->arg1 = insize; + req->arg2 = outsize; + + memcpy(isp->cmd_virt, args, insize); + err = ipc_chan_send(isp, chan, timeout); + if (err) { + u64 opcode; + memcpy(&opcode, args, sizeof(opcode)); + dev_err(isp->dev, + "%s: failed to send OPCODE 0x%04llx: [0x%llx, 0x%llx, 0x%llx]\n", + chan->name, CISP_OPCODE_GET(opcode), req->arg0, + req->arg1, req->arg2); + } + + return err; +} + +static int cisp_send_read(struct apple_isp *isp, void *args, u32 insize, + u32 outsize) +{ + /* TODO do I need to lock the iova space? */ + int err = cisp_send(isp, args, insize, outsize, CISP_TIMEOUT); + if (err) + return err; + + memcpy(args, isp->cmd_virt, outsize); + return 0; +} + +int isp_cmd_start(struct apple_isp *isp, u32 mode) +{ + struct cmd_start args = { + .opcode = CISP_OPCODE(CISP_CMD_START), + .mode = mode, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_stop(struct apple_isp *isp, u32 mode) +{ + struct cmd_stop args = { + .opcode = CISP_OPCODE(CISP_CMD_STOP), + .mode = mode, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_power_down(struct apple_isp *isp) +{ + struct cmd_power_down args = { + .opcode = CISP_OPCODE(CISP_CMD_POWER_DOWN), + }; + return CISP_POST_INOUT(isp, args); +} + +int isp_cmd_suspend(struct apple_isp *isp) +{ + struct cmd_suspend args = { + .opcode = CISP_OPCODE(CISP_CMD_SUSPEND), + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_print_enable(struct apple_isp *isp, u32 enable) +{ + struct cmd_print_enable args = { + .opcode = CISP_OPCODE(CISP_CMD_PRINT_ENABLE), + .enable = enable, + }; + return CISP_SEND_INOUT(isp, args); +} + +int isp_cmd_trace_enable(struct apple_isp *isp, u32 enable) +{ + struct cmd_trace_enable args = { + .opcode = CISP_OPCODE(CISP_CMD_TRACE_ENABLE), + .enable = enable, + }; + return CISP_SEND_INOUT(isp, args); +} + +int isp_cmd_config_get(struct apple_isp *isp, struct cmd_config_get *args) +{ + args->opcode = CISP_OPCODE(CISP_CMD_CONFIG_GET); + return CISP_SEND_OUT(isp, args); +} + +int isp_cmd_set_isp_pmu_base(struct apple_isp *isp, u64 pmu_base) +{ + struct cmd_set_isp_pmu_base args = { + .opcode = CISP_OPCODE(CISP_CMD_SET_ISP_PMU_BASE), + .pmu_base = pmu_base, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_set_dsid_clr_req_base2(struct apple_isp *isp, u64 dsid_clr_base0, + u64 dsid_clr_base1, u64 dsid_clr_base2, + u64 dsid_clr_base3, u32 dsid_clr_range0, + u32 dsid_clr_range1, u32 dsid_clr_range2, + u32 dsid_clr_range3) +{ + struct cmd_set_dsid_clr_req_base2 args = { + .opcode = CISP_OPCODE(CISP_CMD_SET_DSID_CLR_REG_BASE2), + .dsid_clr_base0 = dsid_clr_base0, + .dsid_clr_base1 = dsid_clr_base1, + .dsid_clr_base2 = dsid_clr_base2, + .dsid_clr_base3 = dsid_clr_base3, + .dsid_clr_range0 = dsid_clr_range0, + .dsid_clr_range1 = dsid_clr_range1, + .dsid_clr_range2 = dsid_clr_range2, + .dsid_clr_range3 = dsid_clr_range3, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_set_dsid_clr_req_base(struct apple_isp *isp, u64 dsid_clr_base, + u32 dsid_clr_range) +{ + struct cmd_set_dsid_clr_req_base args = { + .opcode = CISP_OPCODE(CISP_CMD_SET_DSID_CLR_REG_BASE), + .dsid_clr_base = dsid_clr_base, + .dsid_clr_range = dsid_clr_range, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_pmp_ctrl_set(struct apple_isp *isp, u64 clock_scratch, + u64 clock_base, u8 clock_bit, u8 clock_size, + u64 bandwidth_scratch, u64 bandwidth_base, + u8 bandwidth_bit, u8 bandwidth_size) +{ + struct cmd_pmp_ctrl_set args = { + .opcode = CISP_OPCODE(CISP_CMD_PMP_CTRL_SET), + .clock_scratch = clock_scratch, + .clock_base = clock_base, + .clock_bit = clock_bit, + .clock_size = clock_size, + .clock_pad = 0, + .bandwidth_scratch = bandwidth_scratch, + .bandwidth_base = bandwidth_base, + .bandwidth_bit = bandwidth_bit, + .bandwidth_size = bandwidth_size, + .bandwidth_pad = 0, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_fid_enter(struct apple_isp *isp) +{ + struct cmd_fid_enter args = { + .opcode = CISP_OPCODE(CISP_CMD_FID_ENTER), + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_fid_exit(struct apple_isp *isp) +{ + struct cmd_fid_exit args = { + .opcode = CISP_OPCODE(CISP_CMD_FID_EXIT), + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_start(struct apple_isp *isp, u32 chan) +{ + struct cmd_ch_start args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_START), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_stop(struct apple_isp *isp, u32 chan) +{ + struct cmd_ch_stop args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_STOP), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_flicker_sensor_set(struct apple_isp *isp, u32 mode) +{ + struct cmd_flicker_sensor_set args = { + .opcode = CISP_OPCODE(CISP_CMD_FLICKER_SENSOR_SET), + .mode = mode, + }; + return CISP_SEND_INOUT(isp, args); +} + +int isp_cmd_ch_info_get(struct apple_isp *isp, u32 chan, + struct cmd_ch_info *args) +{ + args->opcode = CISP_OPCODE(CISP_CMD_CH_INFO_GET); + args->chan = chan; + return CISP_SEND_OUT(isp, args); +} + +int isp_cmd_ch_camera_config_get(struct apple_isp *isp, u32 chan, u32 preset, + struct cmd_ch_camera_config *args) +{ + args->opcode = CISP_OPCODE(CISP_CMD_CH_CAMERA_CONFIG_GET); + args->preset = preset; + args->chan = chan; + return CISP_SEND_OUT(isp, args); +} + +int isp_cmd_ch_camera_config_current_get(struct apple_isp *isp, u32 chan, + struct cmd_ch_camera_config *args) +{ + args->opcode = CISP_OPCODE(CISP_CMD_CH_CAMERA_CONFIG_CURRENT_GET); + args->chan = chan; + return CISP_SEND_OUT(isp, args); +} + +int isp_cmd_ch_camera_config_select(struct apple_isp *isp, u32 chan, u32 preset) +{ + struct cmd_ch_camera_config_select args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_CAMERA_CONFIG_SELECT), + .chan = chan, + .preset = preset, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_buffer_return(struct apple_isp *isp, u32 chan) +{ + struct cmd_ch_buffer_return args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_BUFFER_RETURN), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_set_file_load(struct apple_isp *isp, u32 chan, u64 addr, + u32 size) +{ + if (isp->fw_compat >= ISP_FIRMWARE_V_13_5) { + struct cmd_ch_set_file_load64 args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_SET_FILE_LOAD), + .chan = chan, + .addr = addr, + .size = size, + }; + return CISP_SEND_IN(isp, args); + } else { + struct cmd_ch_set_file_load args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_SET_FILE_LOAD), + .chan = chan, + .addr = addr, + .size = size, + }; + return CISP_SEND_IN(isp, args); + } +} + +int isp_cmd_ch_sbs_enable(struct apple_isp *isp, u32 chan, u32 enable) +{ + struct cmd_ch_sbs_enable args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_SBS_ENABLE), + .chan = chan, + .enable = enable, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_crop_set(struct apple_isp *isp, u32 chan, u32 x1, u32 y1, u32 x2, + u32 y2) +{ + struct cmd_ch_crop_set args = { + .opcode = CISP_OPCODE(isp->hw->scl1 ? CISP_CMD_CH_CROP_SCL1_SET + : CISP_CMD_CH_CROP_SET), + .chan = chan, + .x1 = x1, + .y1 = y1, + .x2 = x2, + .y2 = y2, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_output_config_set(struct apple_isp *isp, u32 chan, u32 width, + u32 height, u32 strides[3], u32 colorspace, u32 format) +{ + struct cmd_ch_output_config_set args = { + .opcode = CISP_OPCODE(isp->hw->scl1 ? CISP_CMD_CH_OUTPUT_CONFIG_SCL1_SET + : CISP_CMD_CH_OUTPUT_CONFIG_SET), + .chan = chan, + .width = width, + .height = height, + .colorspace = colorspace, + .format = format, + .padding_rows = 0, + .unk_h0 = height, + .compress = 0, + .unk_w2 = width, + }; + memcpy(args.strides, strides, sizeof(args.strides)); + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_preview_stream_set(struct apple_isp *isp, u32 chan, u32 stream) +{ + struct cmd_ch_preview_stream_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_PREVIEW_STREAM_SET), + .chan = chan, + .stream = stream, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_als_disable(struct apple_isp *isp, u32 chan) +{ + struct cmd_ch_als_disable args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_ALS_DISABLE), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_cnr_start(struct apple_isp *isp, u32 chan) +{ + struct cmd_ch_cnr_start args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_CNR_START), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_mbnr_enable(struct apple_isp *isp, u32 chan, u32 use_case, + u32 mode, u32 enable_chroma) +{ + struct cmd_ch_mbnr_enable args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_MBNR_ENABLE), + .chan = chan, + .use_case = use_case, + .mode = mode, + .enable_chroma = enable_chroma, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_sif_pixel_format_set(struct apple_isp *isp, u32 chan) +{ + struct cmd_ch_sif_pixel_format_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_SIF_PIXEL_FORMAT_SET), + .chan = chan, + .format = 3, + .type = 1, + .compress = 0, + .unk_10 = 0, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_buffer_recycle_mode_set(struct apple_isp *isp, u32 chan, + u32 mode) +{ + struct cmd_ch_buffer_recycle_mode_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_BUFFER_RECYCLE_MODE_SET), + .chan = chan, + .mode = mode, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_buffer_recycle_start(struct apple_isp *isp, u32 chan) +{ + struct cmd_ch_buffer_recycle_start args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_BUFFER_RECYCLE_START), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_buffer_pool_config_set(struct apple_isp *isp, u32 chan, u16 type) +{ + struct cmd_ch_buffer_pool_config_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_BUFFER_POOL_CONFIG_SET), + .chan = chan, + .type = type, + .count = ISP_MAX_BUFFERS, + .meta_size0 = isp->hw->meta_size, + .meta_size1 = isp->hw->meta_size, + .unk0 = 0, + .unk1 = 0, + .unk2 = 0, + .data_blocks = 1, + .compress = 0, + }; + return CISP_SEND_INOUT(isp, args); +} + +int isp_cmd_ch_buffer_pool_return(struct apple_isp *isp, u32 chan) +{ + struct cmd_ch_buffer_pool_return args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_BUFFER_POOL_RETURN), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_temporal_filter_start(struct apple_isp *isp, u32 chan, u32 arg) +{ + struct cmd_apple_ch_temporal_filter_start args = { + .opcode = CISP_OPCODE(CISP_CMD_APPLE_CH_TEMPORAL_FILTER_START), + .chan = chan, + .unk_c = 1, + .unk_10 = arg, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_temporal_filter_stop(struct apple_isp *isp, u32 chan) +{ + struct cmd_apple_ch_temporal_filter_stop args = { + .opcode = CISP_OPCODE(CISP_CMD_APPLE_CH_TEMPORAL_FILTER_STOP), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_motion_history_start(struct apple_isp *isp, u32 chan) +{ + struct cmd_apple_ch_motion_history_start args = { + .opcode = CISP_OPCODE(CISP_CMD_APPLE_CH_MOTION_HISTORY_START), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_motion_history_stop(struct apple_isp *isp, u32 chan) +{ + struct cmd_apple_ch_motion_history_stop args = { + .opcode = CISP_OPCODE(CISP_CMD_APPLE_CH_MOTION_HISTORY_STOP), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_temporal_filter_enable(struct apple_isp *isp, u32 chan) +{ + struct cmd_apple_ch_temporal_filter_enable args = { + .opcode = CISP_OPCODE(CISP_CMD_APPLE_CH_TEMPORAL_FILTER_ENABLE), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_temporal_filter_disable(struct apple_isp *isp, u32 chan) +{ + struct cmd_apple_ch_temporal_filter_disable args = { + .opcode = + CISP_OPCODE(CISP_CMD_APPLE_CH_TEMPORAL_FILTER_DISABLE), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_ae_stability_set(struct apple_isp *isp, u32 chan, u32 stability) +{ + struct cmd_ch_ae_stability_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_AE_STABILITY_SET), + .chan = chan, + .stability = stability, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_ae_stability_to_stable_set(struct apple_isp *isp, u32 chan, + u32 stability) +{ + struct cmd_ch_ae_stability_to_stable_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_AE_STABILITY_TO_STABLE_SET), + .chan = chan, + .stability = stability, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_ae_frame_rate_max_get(struct apple_isp *isp, u32 chan, + struct cmd_ch_ae_frame_rate_max_get *args) +{ + args->opcode = CISP_OPCODE(CISP_CMD_CH_AE_FRAME_RATE_MAX_GET); + args->chan = chan; + return CISP_SEND_OUT(isp, args); +} + +int isp_cmd_ch_ae_frame_rate_max_set(struct apple_isp *isp, u32 chan, + u32 framerate) +{ + struct cmd_ch_ae_frame_rate_max_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_AE_FRAME_RATE_MAX_SET), + .chan = chan, + .framerate = framerate, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_ae_frame_rate_min_set(struct apple_isp *isp, u32 chan, + u32 framerate) +{ + struct cmd_ch_ae_frame_rate_min_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_AE_FRAME_RATE_MIN_SET), + .chan = chan, + .framerate = framerate, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_ae_fd_scene_metering_config_set(struct apple_isp *isp, + u32 chan) +{ + struct cmd_apple_ch_ae_fd_scene_metering_config_set args = { + .opcode = CISP_OPCODE( + CISP_CMD_APPLE_CH_AE_FD_SCENE_METERING_CONFIG_SET), + .chan = chan, + .unk_c = 0xb8, + .unk_10 = 0x2000200, + .unk_14 = 0x280800, + .unk_18 = 0xe10028, + .unk_1c = 0xa0399, + .unk_20 = 0x3cc02cc, + }; + return CISP_SEND_INOUT(isp, args); +} + +int isp_cmd_apple_ch_ae_metering_mode_set(struct apple_isp *isp, u32 chan, + u32 mode) +{ + struct cmd_apple_ch_ae_metering_mode_set args = { + .opcode = CISP_OPCODE(CISP_CMD_APPLE_CH_AE_METERING_MODE_SET), + .chan = chan, + .mode = mode, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_ae_flicker_freq_update_current_set(struct apple_isp *isp, + u32 chan, u32 freq) +{ + struct cmd_apple_ch_ae_flicker_freq_update_current_set args = { + .opcode = CISP_OPCODE( + CISP_CMD_APPLE_CH_AE_FLICKER_FREQ_UPDATE_CURRENT_SET), + .chan = chan, + .freq = freq, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_semantic_video_enable(struct apple_isp *isp, u32 chan, + u32 enable) +{ + struct cmd_ch_semantic_video_enable args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_SEMANTIC_VIDEO_ENABLE), + .chan = chan, + .enable = enable, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_semantic_awb_enable(struct apple_isp *isp, u32 chan, u32 enable) +{ + struct cmd_ch_semantic_awb_enable args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_SEMANTIC_AWB_ENABLE), + .chan = chan, + .enable = enable, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_lpdp_hs_receiver_tuning_set(struct apple_isp *isp, u32 chan, u32 unk1, u32 unk2) +{ + struct cmd_ch_lpdp_hs_receiver_tuning_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_LPDP_HS_RECEIVER_TUNING_SET), + .chan = chan, + .unk1 = unk1, + .unk2 = unk2, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_property_write(struct apple_isp *isp, u32 chan, u32 prop, u32 val) +{ + struct cmd_ch_property_write args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_PROPERTY_WRITE), + .chan = chan, + .prop = prop, + .val = val, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_property_read(struct apple_isp *isp, u32 chan, u32 prop, u32 *val) +{ + struct cmd_ch_property_write args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_PROPERTY_READ), + .chan = chan, + .prop = prop, + .val = 0xdeadbeef, + }; + int ret = CISP_SEND_OUT(isp, &args); + + *val = args.val; + + return ret; +} diff --git a/drivers/media/platform/apple/isp/isp-cmd.h b/drivers/media/platform/apple/isp/isp-cmd.h new file mode 100644 index 00000000000000..5a3c8cd9177e48 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-cmd.h @@ -0,0 +1,691 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#ifndef __ISP_CMD_H__ +#define __ISP_CMD_H__ + +#include "isp-drv.h" + +#define CISP_CMD_START 0x0000 +#define CISP_CMD_STOP 0x0001 +#define CISP_CMD_CONFIG_GET 0x0003 +#define CISP_CMD_PRINT_ENABLE 0x0004 +#define CISP_CMD_BUILDINFO 0x0006 +#define CISP_CMD_GET_BES_PARAM 0x000f +#define CISP_CMD_POWER_DOWN 0x0010 +#define CISP_CMD_SET_ISP_PMU_BASE 0x0011 +#define CISP_CMD_PMP_CTRL_SET 0x001c +#define CISP_CMD_TRACE_ENABLE 0x001d +#define CISP_CMD_SUSPEND 0x0021 +#define CISP_CMD_FID_ENTER 0x0022 +#define CISP_CMD_FID_EXIT 0x0023 +#define CISP_CMD_FLICKER_SENSOR_SET 0x0024 +#define CISP_CMD_CH_START 0x0100 +#define CISP_CMD_CH_STOP 0x0101 +#define CISP_CMD_CH_BUFFER_RETURN 0x0104 +#define CISP_CMD_CH_CAMERA_CONFIG_CURRENT_GET 0x0105 +#define CISP_CMD_CH_CAMERA_CONFIG_GET 0x0106 +#define CISP_CMD_CH_CAMERA_CONFIG_SELECT 0x0107 +#define CISP_CMD_CH_INFO_GET 0x010d +#define CISP_CMD_CH_BUFFER_RECYCLE_MODE_SET 0x010e +#define CISP_CMD_CH_BUFFER_RECYCLE_START 0x010f +#define CISP_CMD_CH_BUFFER_RECYCLE_STOP 0x0110 +#define CISP_CMD_CH_SET_FILE_LOAD 0x0111 +#define CISP_CMD_CH_SIF_PIXEL_FORMAT_SET 0x0115 +#define CISP_CMD_CH_BUFFER_POOL_CONFIG_GET 0x0116 +#define CISP_CMD_CH_BUFFER_POOL_CONFIG_SET 0x0117 +#define CISP_CMD_CH_CAMERA_MIPI_FREQUENCY_GET 0x011a +#define CISP_CMD_CH_CAMERA_PIX_FREQUENCY_GET 0x011f +#define CISP_CMD_CH_PROPERTY_WRITE 0x0122 +#define CISP_CMD_CH_PROPERTY_READ 0x0123 +#define CISP_CMD_CH_LOCAL_RAW_BUFFER_ENABLE 0x0125 +#define CISP_CMD_CH_META_DATA_ENABLE 0x0126 +#define CISP_CMD_CH_CAMERA_MIPI_FREQUENCY_TOTAL_GET 0x0133 +#define CISP_CMD_CH_SBS_ENABLE 0x013b +#define CISP_CMD_CH_LSC_POLYNOMIAL_COEFF_GET 0x0142 +#define CISP_CMD_CH_SET_META_DATA_REQUIRED 0x014f +#define CISP_CMD_CH_BUFFER_POOL_RETURN 0x015b +#define CISP_CMD_CH_CAMERA_AGILE_FREQ_ARRAY_CURRENT_GET 0x015e +#define CISP_CMD_CH_AE_START 0x0200 +#define CISP_CMD_CH_AE_STOP 0x0201 +#define CISP_CMD_CH_AE_FRAME_RATE_MAX_GET 0x0207 +#define CISP_CMD_CH_AE_FRAME_RATE_MAX_SET 0x0208 +#define CISP_CMD_CH_AE_FRAME_RATE_MIN_GET 0x0209 +#define CISP_CMD_CH_AE_FRAME_RATE_MIN_SET 0x020a +#define CISP_CMD_CH_AE_STABILITY_SET 0x021a +#define CISP_CMD_CH_AE_STABILITY_TO_STABLE_SET 0x0229 +#define CISP_CMD_CH_SENSOR_NVM_GET 0x0501 +#define CISP_CMD_CH_SENSOR_PERMODULE_LSC_INFO_GET 0x0507 +#define CISP_CMD_CH_SENSOR_PERMODULE_LSC_GRID_GET 0x0511 +#define CISP_CMD_CH_LPDP_HS_RECEIVER_TUNING_SET 0x051b +#define CISP_CMD_CH_FOCUS_LIMITS_GET 0x0701 +#define CISP_CMD_CH_CROP_GET 0x0800 +#define CISP_CMD_CH_CROP_SET 0x0801 +#define CISP_CMD_CH_SCALER_CROP_SET 0x080a +#define CISP_CMD_CH_CROP_SCL1_GET 0x080b +#define CISP_CMD_CH_CROP_SCL1_SET 0x080c +#define CISP_CMD_CH_SCALER_CROP_SCL1_SET 0x080d +#define CISP_CMD_CH_ALS_ENABLE 0x0a1c +#define CISP_CMD_CH_ALS_DISABLE 0x0a1d +#define CISP_CMD_CH_CNR_START 0x0a2f +#define CISP_CMD_CH_MBNR_ENABLE 0x0a3a +#define CISP_CMD_CH_OUTPUT_CONFIG_SET 0x0b01 +#define CISP_CMD_CH_OUTPUT_CONFIG_SCL1_SET 0x0b09 +#define CISP_CMD_CH_PREVIEW_STREAM_SET 0x0b0d +#define CISP_CMD_CH_SEMANTIC_VIDEO_ENABLE 0x0b17 +#define CISP_CMD_CH_SEMANTIC_AWB_ENABLE 0x0b18 +#define CISP_CMD_CH_FACE_DETECTION_START 0x0d00 +#define CISP_CMD_CH_FACE_DETECTION_STOP 0x0d01 +#define CISP_CMD_CH_FACE_DETECTION_CONFIG_GET 0x0d02 +#define CISP_CMD_CH_FACE_DETECTION_CONFIG_SET 0x0d03 +#define CISP_CMD_CH_FACE_DETECTION_DISABLE 0x0d04 +#define CISP_CMD_CH_FACE_DETECTION_ENABLE 0x0d05 +#define CISP_CMD_CH_FID_START 0x3000 +#define CISP_CMD_CH_FID_STOP 0x3001 +#define CISP_CMD_IPC_ENDPOINT_SET2 0x300c +#define CISP_CMD_IPC_ENDPOINT_UNSET2 0x300d +#define CISP_CMD_SET_DSID_CLR_REG_BASE2 0x3204 +#define CISP_CMD_SET_DSID_CLR_REG_BASE 0x3205 +#define CISP_CMD_APPLE_CH_AE_METERING_MODE_SET 0x8206 +#define CISP_CMD_APPLE_CH_AE_FD_SCENE_METERING_CONFIG_SET 0x820e +#define CISP_CMD_APPLE_CH_AE_FLICKER_FREQ_UPDATE_CURRENT_SET 0x8212 +#define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_START 0xc100 +#define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_STOP 0xc101 +#define CISP_CMD_APPLE_CH_MOTION_HISTORY_START 0xc102 +#define CISP_CMD_APPLE_CH_MOTION_HISTORY_STOP 0xc103 +#define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_ENABLE 0xc113 +#define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_DISABLE 0xc114 + +#define CISP_POOL_TYPE_META 0x0 +#define CISP_POOL_TYPE_RENDERED 0x1 +#define CISP_POOL_TYPE_FD 0x2 +#define CISP_POOL_TYPE_RAW 0x3 +#define CISP_POOL_TYPE_STAT 0x4 +#define CISP_POOL_TYPE_RAW_AUX 0x5 +#define CISP_POOL_TYPE_YCC 0x6 +#define CISP_POOL_TYPE_CAPTURE_FULL_RES 0x7 +#define CISP_POOL_TYPE_META_CAPTURE 0x8 +#define CISP_POOL_TYPE_RENDERED_SCL1 0x9 +#define CISP_POOL_TYPE_STAT_PIXELOUTPUT 0x11 +#define CISP_POOL_TYPE_FSCL 0x12 +#define CISP_POOL_TYPE_CAPTURE_FULL_RES_YCC 0x13 +#define CISP_POOL_TYPE_RENDERED_RAW 0x14 +#define CISP_POOL_TYPE_CAPTURE_PDC_RAW 0x16 +#define CISP_POOL_TYPE_FPC_DATA 0x17 +#define CISP_POOL_TYPE_AICAM_SEG 0x19 +#define CISP_POOL_TYPE_SPD 0x1a +#define CISP_POOL_TYPE_META_DEPTH 0x1c +#define CISP_POOL_TYPE_JASPER_DEPTH 0x1d +#define CISP_POOL_TYPE_RAW_SIFR 0x1f +#define CISP_POOL_TYPE_FEP_THUMBNAIL_DYNAMIC_POOL_RAW 0x21 + +#define CISP_COLORSPACE_REC709 0x1 +#define CISP_OUTPUT_FORMAT_YUV_2PLANE 0x0 +#define CISP_OUTPUT_FORMAT_YUV_1PLANE 0x1 +#define CISP_OUTPUT_FORMAT_RGB 0x2 +#define CISP_BUFFER_RECYCLE_MODE_EMPTY_ONLY 0x1 + +struct cmd_start { + u64 opcode; + u32 mode; +} __packed; +static_assert(sizeof(struct cmd_start) == 0xc); + +struct cmd_stop { + u64 opcode; + u32 mode; +} __packed; +static_assert(sizeof(struct cmd_stop) == 0xc); + +struct cmd_power_down { + u64 opcode; +} __packed; +static_assert(sizeof(struct cmd_power_down) == 0x8); + +struct cmd_suspend { + u64 opcode; +} __packed; +static_assert(sizeof(struct cmd_suspend) == 0x8); + +struct cmd_print_enable { + u64 opcode; + u32 enable; +} __packed; +static_assert(sizeof(struct cmd_print_enable) == 0xc); + +struct cmd_trace_enable { + u64 opcode; + u32 enable; +} __packed; +static_assert(sizeof(struct cmd_trace_enable) == 0xc); + +struct cmd_config_get { + u64 opcode; + u32 timestamp_freq; + u32 num_channels; + u32 unk_10; + u32 unk_14; + u32 unk_18; +} __packed; +static_assert(sizeof(struct cmd_config_get) == 0x1c); + +struct cmd_set_isp_pmu_base { + u64 opcode; + u64 pmu_base; +} __packed; +static_assert(sizeof(struct cmd_set_isp_pmu_base) == 0x10); + +struct cmd_set_dsid_clr_req_base2 { + u64 opcode; + u64 dsid_clr_base0; + u64 dsid_clr_base1; + u64 dsid_clr_base2; + u64 dsid_clr_base3; + u32 dsid_clr_range0; + u32 dsid_clr_range1; + u32 dsid_clr_range2; + u32 dsid_clr_range3; +} __packed; +static_assert(sizeof(struct cmd_set_dsid_clr_req_base2) == 0x38); + +struct cmd_set_dsid_clr_req_base { + u64 opcode; + u64 dsid_clr_base; + u32 dsid_clr_range; +} __packed; +static_assert(sizeof(struct cmd_set_dsid_clr_req_base) == 0x14); + +struct cmd_pmp_ctrl_set { + u64 opcode; + u64 clock_scratch; + u64 clock_base; + u8 clock_bit; + u8 clock_size; + u16 clock_pad; + u64 bandwidth_scratch; + u64 bandwidth_base; + u8 bandwidth_bit; + u8 bandwidth_size; + u16 bandwidth_pad; +} __packed; +static_assert(sizeof(struct cmd_pmp_ctrl_set) == 0x30); + +struct cmd_fid_enter { + u64 opcode; +} __packed; +static_assert(sizeof(struct cmd_fid_enter) == 0x8); + +struct cmd_fid_exit { + u64 opcode; +} __packed; +static_assert(sizeof(struct cmd_fid_exit) == 0x8); + +struct cmd_ipc_endpoint_set2 { + u64 opcode; + u32 unk; + u64 addr1; + u32 size1; + u64 addr2; + u32 size2; + u64 regs; + u32 unk2; +} __packed; +static_assert(sizeof(struct cmd_ipc_endpoint_set2) == 0x30); + +struct cmd_flicker_sensor_set { + u64 opcode; + u32 mode; +} __packed; +static_assert(sizeof(struct cmd_flicker_sensor_set) == 0xc); + +int isp_cmd_start(struct apple_isp *isp, u32 mode); +int isp_cmd_stop(struct apple_isp *isp, u32 mode); +int isp_cmd_power_down(struct apple_isp *isp); +int isp_cmd_suspend(struct apple_isp *isp); +int isp_cmd_print_enable(struct apple_isp *isp, u32 enable); +int isp_cmd_trace_enable(struct apple_isp *isp, u32 enable); +int isp_cmd_config_get(struct apple_isp *isp, struct cmd_config_get *args); +int isp_cmd_set_isp_pmu_base(struct apple_isp *isp, u64 pmu_base); +int isp_cmd_set_dsid_clr_req_base(struct apple_isp *isp, u64 dsid_clr_base, + u32 dsid_clr_range); +int isp_cmd_set_dsid_clr_req_base2(struct apple_isp *isp, u64 dsid_clr_base0, + u64 dsid_clr_base1, u64 dsid_clr_base2, + u64 dsid_clr_base3, u32 dsid_clr_range0, + u32 dsid_clr_range1, u32 dsid_clr_range2, + u32 dsid_clr_range3); +int isp_cmd_pmp_ctrl_set(struct apple_isp *isp, u64 clock_scratch, + u64 clock_base, u8 clock_bit, u8 clock_size, + u64 bandwidth_scratch, u64 bandwidth_base, + u8 bandwidth_bit, u8 bandwidth_size); +int isp_cmd_fid_enter(struct apple_isp *isp); +int isp_cmd_fid_exit(struct apple_isp *isp); +int isp_cmd_flicker_sensor_set(struct apple_isp *isp, u32 mode); + +struct cmd_ch_start { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_ch_start) == 0xc); + +struct cmd_ch_stop { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_ch_stop) == 0xc); + +struct cmd_ch_info { + u64 opcode; + u32 chan; + u32 unk_c; // 0x7da0001, 0x7db0001 + u32 unk_10; // 0x300ac, 0x5006d + u32 unk_14; // 0x40007, 0x10007 + u32 unk_18; // 0x5, 0x2 + u32 unk_1c; // 0x1, 0x1 + u32 version; + u32 unk_24; // 0x7, 0x9 + u32 unk_28; // 0x1, 0x1410 + u32 unk_2c; // 0x7, 0x2 + u32 pad_30[7]; + u32 unk_4c; // 0x10000, 0x50000 + u32 unk_50; // 0x1, 0x1 + u32 unk_54; // 0x0, 0x0 + u32 unk_58; // 0x4, 0x4 + u32 unk_5c; // 0x10, 0x20 + u32 num_presets; + u32 unk_64; // 0x0, 0x0 + u32 unk_68; // 0x44c0, 0x4680 + u32 unk_6c; // 0x40, 0x40 + u32 unk_70; // 0x1, 0x1 + u32 unk_74; // 0x2, 0x2 + u32 unk_78; // 0x4000, 0x4000 + u32 unk_7c; // 0x40, 0x40 + u32 unk_80; // 0x1, 0x1 + u32 pad_84[2]; + u32 unk_8c; // 0x36, 0x36 + u32 pad_90[2]; + u32 timestamp_freq; + u16 pad_9c; + char module_sn[20]; + u16 pad_b0; + u32 unk_b4; // 0x8, 0x8 + u32 pad_b8[2]; + u32 unk_c0; // 0x4, 0x1 + u32 unk_c4; // 0x0, 0x0 + u32 unk_c8; // 0x0, 0x100 + u32 pad_cc[4]; + u32 unk_dc; // 0xff0000, 0xff0000 + u32 unk_e0; // 0xc00, 0xc00 + u32 unk_e4; // 0x0, 0x0 + u32 unk_e8; // 0x1c, 0x1c + u32 unk_ec; // 0x640, 0x680 + u32 unk_f0; // 0x4, 0x4 + u32 unk_f4; // 0x4, 0x4 + u32 pad_f8[6]; + u32 unk_110; // 0x0, 0x7800000 + u32 unk_114; // 0x0, 0x780 +} __packed; +static_assert(sizeof(struct cmd_ch_info) == 0x118); + +struct cmd_ch_camera_config { + u64 opcode; + u32 chan; + u32 preset; + u16 in_width; + u16 in_height; + u16 out_width; + u16 out_height; + u32 unk_28; + u32 unk_2c; + u32 unk_30[16]; + u32 sensor_clk; + u32 unk_64[4]; + u32 timestamp_freq; + u32 unk_78[2]; + u32 unk_80[16]; + u32 in_width2; // repeated in u32?? + u32 in_height2; + u32 unk_c8[3]; + u32 out_width2; + u32 out_height2; +} __packed; +static_assert(sizeof(struct cmd_ch_camera_config) == 0xdc); + +struct cmd_ch_camera_config_select { + u64 opcode; + u32 chan; + u32 preset; +} __packed; +static_assert(sizeof(struct cmd_ch_camera_config_select) == 0x10); + +struct cmd_ch_set_file_load { + u64 opcode; + u32 chan; + u32 addr; + u32 size; +} __packed; +static_assert(sizeof(struct cmd_ch_set_file_load) == 0x14); + +struct cmd_ch_set_file_load64 { + u64 opcode; + u32 chan; + u64 addr; + u32 size; +} __packed; +static_assert(sizeof(struct cmd_ch_set_file_load64) == 0x18); + +struct cmd_ch_buffer_return { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_ch_buffer_return) == 0xc); + +struct cmd_ch_sbs_enable { + u64 opcode; + u32 chan; + u32 enable; +} __packed; +static_assert(sizeof(struct cmd_ch_sbs_enable) == 0x10); + +struct cmd_ch_crop_set { + u64 opcode; + u32 chan; + u32 x1; + u32 y1; + u32 x2; + u32 y2; +} __packed; +static_assert(sizeof(struct cmd_ch_crop_set) == 0x1c); + +struct cmd_ch_output_config_set { + u64 opcode; + u32 chan; + u32 width; + u32 height; + u32 colorspace; + u32 format; + u32 strides[3]; + u32 padding_rows; + u32 unk_h0; + u32 compress; + u32 unk_w2; +} __packed; +static_assert(sizeof(struct cmd_ch_output_config_set) == 0x38); + +struct cmd_ch_preview_stream_set { + u64 opcode; + u32 chan; + u32 stream; +} __packed; +static_assert(sizeof(struct cmd_ch_preview_stream_set) == 0x10); + +struct cmd_ch_als_disable { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_ch_als_disable) == 0xc); + +struct cmd_ch_cnr_start { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_ch_cnr_start) == 0xc); + +struct cmd_ch_mbnr_enable { + u64 opcode; + u32 chan; + u32 use_case; + u32 mode; + u32 enable_chroma; +} __packed; +static_assert(sizeof(struct cmd_ch_mbnr_enable) == 0x18); + +struct cmd_ch_sif_pixel_format_set { + u64 opcode; + u32 chan; + u8 format; + u8 type; + u16 compress; + u32 unk_10; +} __packed; +static_assert(sizeof(struct cmd_ch_sif_pixel_format_set) == 0x14); + +struct cmd_ch_lpdp_hs_receiver_tuning_set { + u64 opcode; + u32 chan; + u32 unk1; + u32 unk2; +} __packed; +static_assert(sizeof(struct cmd_ch_lpdp_hs_receiver_tuning_set) == 0x14); + +struct cmd_ch_property_write { + u64 opcode; + u32 chan; + u32 prop; + u32 val; + u32 unk1; + u32 unk2; +} __packed; +static_assert(sizeof(struct cmd_ch_property_write) == 0x1c); + +int isp_cmd_ch_start(struct apple_isp *isp, u32 chan); +int isp_cmd_ch_stop(struct apple_isp *isp, u32 chan); +int isp_cmd_ch_info_get(struct apple_isp *isp, u32 chan, + struct cmd_ch_info *args); +int isp_cmd_ch_camera_config_get(struct apple_isp *isp, u32 chan, u32 preset, + struct cmd_ch_camera_config *args); +int isp_cmd_ch_camera_config_current_get(struct apple_isp *isp, u32 chan, + struct cmd_ch_camera_config *args); +int isp_cmd_ch_camera_config_select(struct apple_isp *isp, u32 chan, + u32 preset); +int isp_cmd_ch_set_file_load(struct apple_isp *isp, u32 chan, u64 addr, + u32 size); +int isp_cmd_ch_buffer_return(struct apple_isp *isp, u32 chan); +int isp_cmd_ch_sbs_enable(struct apple_isp *isp, u32 chan, u32 enable); +int isp_cmd_ch_crop_set(struct apple_isp *isp, u32 chan, u32 x1, u32 y1, u32 x2, + u32 y2); +int isp_cmd_ch_output_config_set(struct apple_isp *isp, u32 chan, u32 width, + u32 height, u32 strides[3], u32 colorspace, u32 format); +int isp_cmd_ch_preview_stream_set(struct apple_isp *isp, u32 chan, u32 stream); +int isp_cmd_ch_als_disable(struct apple_isp *isp, u32 chan); +int isp_cmd_ch_cnr_start(struct apple_isp *isp, u32 chan); +int isp_cmd_ch_mbnr_enable(struct apple_isp *isp, u32 chan, u32 use_case, + u32 mode, u32 enable_chroma); +int isp_cmd_ch_sif_pixel_format_set(struct apple_isp *isp, u32 chan); +int isp_cmd_ch_lpdp_hs_receiver_tuning_set(struct apple_isp *isp, u32 chan, u32 unk1, u32 unk2); + +int isp_cmd_ch_property_read(struct apple_isp *isp, u32 chan, u32 prop, u32 *val); +int isp_cmd_ch_property_write(struct apple_isp *isp, u32 chan, u32 prop, u32 val); + +enum isp_mbnr_mode { + ISP_MBNR_MODE_DISABLE = 0, + ISP_MBNR_MODE_ENABLE = 1, + ISP_MBNR_MODE_BYPASS = 2, +}; + +struct cmd_ch_buffer_recycle_mode_set { + u64 opcode; + u32 chan; + u32 mode; +} __packed; +static_assert(sizeof(struct cmd_ch_buffer_recycle_mode_set) == 0x10); + +struct cmd_ch_buffer_recycle_start { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_ch_buffer_recycle_start) == 0xc); + +struct cmd_ch_buffer_pool_config_set { + u64 opcode; + u32 chan; + u16 type; + u16 count; + u32 meta_size0; + u32 meta_size1; + u64 unk0; + u64 unk1; + u64 unk2; + u32 zero[0x19]; + u32 data_blocks; + u32 compress; +} __packed; +static_assert(sizeof(struct cmd_ch_buffer_pool_config_set) == 0x9c); + +struct cmd_ch_buffer_pool_return { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_ch_buffer_pool_return) == 0xc); + +int isp_cmd_ch_buffer_recycle_mode_set(struct apple_isp *isp, u32 chan, + u32 mode); +int isp_cmd_ch_buffer_recycle_start(struct apple_isp *isp, u32 chan); +int isp_cmd_ch_buffer_pool_config_set(struct apple_isp *isp, u32 chan, + u16 type); +int isp_cmd_ch_buffer_pool_config_get(struct apple_isp *isp, u32 chan, + u16 type); +int isp_cmd_ch_buffer_pool_return(struct apple_isp *isp, u32 chan); + +struct cmd_apple_ch_temporal_filter_start { + u64 opcode; + u32 chan; + u32 unk_c; + u32 unk_10; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_temporal_filter_start) == 0x14); + +struct cmd_apple_ch_temporal_filter_stop { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_temporal_filter_stop) == 0xc); + +struct cmd_apple_ch_motion_history_start { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_motion_history_start) == 0xc); + +struct cmd_apple_ch_motion_history_stop { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_motion_history_stop) == 0xc); + +struct cmd_apple_ch_temporal_filter_enable { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_temporal_filter_enable) == 0xc); + +struct cmd_apple_ch_temporal_filter_disable { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_temporal_filter_disable) == 0xc); + +int isp_cmd_apple_ch_temporal_filter_start(struct apple_isp *isp, u32 chan, u32 arg); +int isp_cmd_apple_ch_temporal_filter_stop(struct apple_isp *isp, u32 chan); +int isp_cmd_apple_ch_motion_history_start(struct apple_isp *isp, u32 chan); +int isp_cmd_apple_ch_motion_history_stop(struct apple_isp *isp, u32 chan); +int isp_cmd_apple_ch_temporal_filter_enable(struct apple_isp *isp, u32 chan); +int isp_cmd_apple_ch_temporal_filter_disable(struct apple_isp *isp, u32 chan); + +struct cmd_ch_ae_stability_set { + u64 opcode; + u32 chan; + u32 stability; +} __packed; +static_assert(sizeof(struct cmd_ch_ae_stability_set) == 0x10); + +struct cmd_ch_ae_stability_to_stable_set { + u64 opcode; + u32 chan; + u32 stability; +} __packed; +static_assert(sizeof(struct cmd_ch_ae_stability_to_stable_set) == 0x10); + +struct cmd_ch_ae_frame_rate_max_get { + u64 opcode; + u32 chan; + u32 framerate; +} __packed; +static_assert(sizeof(struct cmd_ch_ae_frame_rate_max_get) == 0x10); + +struct cmd_ch_ae_frame_rate_max_set { + u64 opcode; + u32 chan; + u32 framerate; +} __packed; +static_assert(sizeof(struct cmd_ch_ae_frame_rate_max_set) == 0x10); + +struct cmd_ch_ae_frame_rate_min_set { + u64 opcode; + u32 chan; + u32 framerate; +} __packed; +static_assert(sizeof(struct cmd_ch_ae_frame_rate_min_set) == 0x10); + +struct cmd_apple_ch_ae_fd_scene_metering_config_set { + u64 opcode; + u32 chan; + u32 unk_c; + u32 unk_10; + u32 unk_14; + u32 unk_18; + u32 unk_1c; + u32 unk_20; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_ae_fd_scene_metering_config_set) == + 0x24); + +struct cmd_apple_ch_ae_metering_mode_set { + u64 opcode; + u32 chan; + u32 mode; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_ae_metering_mode_set) == 0x10); + +struct cmd_apple_ch_ae_flicker_freq_update_current_set { + u64 opcode; + u32 chan; + u32 freq; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_ae_flicker_freq_update_current_set) == + 0x10); + +int isp_cmd_ch_ae_stability_set(struct apple_isp *isp, u32 chan, u32 stability); +int isp_cmd_ch_ae_stability_to_stable_set(struct apple_isp *isp, u32 chan, + u32 stability); +int isp_cmd_ch_ae_frame_rate_max_get(struct apple_isp *isp, u32 chan, + struct cmd_ch_ae_frame_rate_max_get *args); +int isp_cmd_ch_ae_frame_rate_max_set(struct apple_isp *isp, u32 chan, + u32 framerate); +int isp_cmd_ch_ae_frame_rate_min_set(struct apple_isp *isp, u32 chan, + u32 framerate); +int isp_cmd_apple_ch_ae_fd_scene_metering_config_set(struct apple_isp *isp, + u32 chan); +int isp_cmd_apple_ch_ae_metering_mode_set(struct apple_isp *isp, u32 chan, + u32 mode); +int isp_cmd_apple_ch_ae_flicker_freq_update_current_set(struct apple_isp *isp, + u32 chan, u32 freq); + +struct cmd_ch_semantic_video_enable { + u64 opcode; + u32 chan; + u32 enable; +} __packed; +static_assert(sizeof(struct cmd_ch_semantic_video_enable) == 0x10); + +struct cmd_ch_semantic_awb_enable { + u64 opcode; + u32 chan; + u32 enable; +} __packed; +static_assert(sizeof(struct cmd_ch_semantic_awb_enable) == 0x10); + +int isp_cmd_ch_semantic_video_enable(struct apple_isp *isp, u32 chan, + u32 enable); +int isp_cmd_ch_semantic_awb_enable(struct apple_isp *isp, u32 chan, u32 enable); + +#endif /* __ISP_CMD_H__ */ diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c new file mode 100644 index 00000000000000..848f7abd535a7f --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -0,0 +1,594 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Apple Image Signal Processor driver + * + * Copyright (C) 2023 The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "isp-cam.h" +#include "isp-fw.h" +#include "isp-iommu.h" +#include "isp-v4l2.h" + +static void apple_isp_detach_genpd(struct apple_isp *isp) +{ + if (isp->pd_count <= 1) + return; + + for (int i = isp->pd_count - 1; i >= 0; i--) { + if (isp->pd_link[i]) + device_link_del(isp->pd_link[i]); + if (!IS_ERR_OR_NULL(isp->pd_dev[i])) + dev_pm_domain_detach(isp->pd_dev[i], true); + } + + return; +} + +static int apple_isp_attach_genpd(struct apple_isp *isp) +{ + struct device *dev = isp->dev; + + isp->pd_count = of_count_phandle_with_args( + dev->of_node, "power-domains", "#power-domain-cells"); + if (isp->pd_count <= 1) + return 0; + + isp->pd_dev = devm_kcalloc(dev, isp->pd_count, sizeof(*isp->pd_dev), + GFP_KERNEL); + if (!isp->pd_dev) + return -ENOMEM; + + isp->pd_link = devm_kcalloc(dev, isp->pd_count, sizeof(*isp->pd_link), + GFP_KERNEL); + if (!isp->pd_link) + return -ENOMEM; + + for (int i = 0; i < isp->pd_count; i++) { + int flags = DL_FLAG_STATELESS; + + /* Primary power domain uses RPM integration */ + if (i == 0) + flags |= DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE; + + isp->pd_dev[i] = dev_pm_domain_attach_by_id(dev, i); + if (IS_ERR(isp->pd_dev[i])) { + apple_isp_detach_genpd(isp); + return PTR_ERR(isp->pd_dev[i]); + } + + isp->pd_link[i] = + device_link_add(dev, isp->pd_dev[i], flags); + + if (!isp->pd_link[i]) { + apple_isp_detach_genpd(isp); + return -EINVAL; + } + } + + return 0; +} + +static int apple_isp_init_iommu(struct apple_isp *isp) +{ + struct device *dev = isp->dev; + phys_addr_t heap_base; + size_t heap_size; + u64 vm_size; + int err; + int idx; + int size; + struct device_node *mem_node; + const __be32 *maps, *end; + + isp->domain = iommu_get_domain_for_dev(isp->dev); + if (!isp->domain) + return -ENODEV; + isp->shift = __ffs(isp->domain->pgsize_bitmap); + + idx = of_property_match_string(dev->of_node, "memory-region-names", + "heap"); + mem_node = of_parse_phandle(dev->of_node, "memory-region", idx); + if (!mem_node) { + dev_err(dev, "No memory-region found for heap\n"); + return -ENODEV; + } + + maps = of_get_property(mem_node, "iommu-addresses", &size); + if (!maps || !size) { + dev_err(dev, "No valid iommu-addresses found for heap\n"); + return -ENODEV; + } + + end = maps + size / sizeof(__be32); + + while (maps < end) { + maps++; + maps = of_translate_dma_region(dev->of_node, maps, &heap_base, + &heap_size); + } + + isp->fw.heap_top = heap_base + heap_size; + + err = of_property_read_u64(dev->of_node, "apple,dart-vm-size", + &vm_size); + if (err) { + dev_err(dev, "failed to read 'apple,dart-vm-size': %d\n", err); + return err; + } + + // FIXME: refactor this, maybe use regular iova stuff? + drm_mm_init(&isp->iovad, isp->fw.heap_top, + vm_size - (heap_base & 0xffffffff)); + + return 0; +} + +static void apple_isp_free_iommu(struct apple_isp *isp) +{ + drm_mm_takedown(&isp->iovad); +} + +static int isp_of_read_coord(struct device *dev, struct device_node *np, + const char *prop, struct coord *val) +{ + u32 xy[2]; + int ret; + + ret = of_property_read_u32_array(np, prop, xy, 2); + if (ret) { + dev_err(dev, "failed to read '%s' property\n", prop); + return ret; + } + + val->x = xy[0]; + val->y = xy[1]; + return 0; +} + +static int apple_isp_init_presets(struct apple_isp *isp) +{ + struct device *dev = isp->dev; + struct isp_preset *preset; + int err = 0; + + struct device_node *np __free(device_node) = + of_get_child_by_name(dev->of_node, "sensor-presets"); + if (!np) { + dev_err(dev, "failed to get DT node 'presets'\n"); + return -EINVAL; + } + + isp->num_presets = of_get_child_count(np); + if (!isp->num_presets) { + dev_err(dev, "no sensor presets found\n"); + return -EINVAL; + } + + isp->presets = devm_kzalloc( + dev, sizeof(*isp->presets) * isp->num_presets, GFP_KERNEL); + if (!isp->presets) + return -ENOMEM; + + preset = isp->presets; + for_each_child_of_node_scoped(np, child) { + u32 xywh[4]; + + err = of_property_read_u32(child, "apple,config-index", + &preset->index); + if (err) { + dev_err(dev, "no apple,config-index property\n"); + return err; + } + + err = isp_of_read_coord(dev, child, "apple,input-size", + &preset->input_dim); + if (err) + return err; + err = isp_of_read_coord(dev, child, "apple,output-size", + &preset->output_dim); + if (err) + return err; + + err = of_property_read_u32_array(child, "apple,crop", xywh, 4); + if (err) { + dev_err(dev, "failed to read 'apple,crop' property\n"); + return err; + } + preset->crop_offset.x = xywh[0]; + preset->crop_offset.y = xywh[1]; + preset->crop_size.x = xywh[2]; + preset->crop_size.y = xywh[3]; + + preset++; + } + + return 0; +} + +static const char * isp_fw2str(enum isp_firmware_version version) +{ + switch (version) { + case ISP_FIRMWARE_V_12_3: + return "12.3"; + case ISP_FIRMWARE_V_12_4: + return "12.4"; + case ISP_FIRMWARE_V_13_5: + return "13.5"; + default: + return "unknown"; + } +} + +#define ISP_FW_VERSION_MIN_LEN 3 +#define ISP_FW_VERSION_MAX_LEN 5 + +static enum isp_firmware_version isp_read_fw_version(struct device *dev, + const char *name) +{ + u32 ver[ISP_FW_VERSION_MAX_LEN]; + int len = of_property_read_variable_u32_array(dev->of_node, name, ver, + ISP_FW_VERSION_MIN_LEN, + ISP_FW_VERSION_MAX_LEN); + + switch (len) { + case 3: + if (ver[0] == 12 && ver[1] == 3 && ver[2] <= 1) + return ISP_FIRMWARE_V_12_3; + else if (ver[0] == 12 && ver[1] == 4 && ver[2] == 0) + return ISP_FIRMWARE_V_12_4; + else if (ver[0] == 13 && ver[1] == 5 && ver[2] == 0) + return ISP_FIRMWARE_V_13_5; + + dev_warn(dev, "unknown %s: %d.%d.%d\n", name, ver[0], ver[1], ver[2]); + break; + case 4: + dev_warn(dev, "unknown %s: %d.%d.%d.%d\n", name, ver[0], ver[1], + ver[2], ver[3]); + break; + case 5: + dev_warn(dev, "unknown %s: %d.%d.%d.%d.%d\n", name, ver[0], + ver[1], ver[2], ver[3], ver[4]); + break; + default: + dev_warn(dev, "could not parse %s: %d\n", name, len); + break; + } + + return ISP_FIRMWARE_V_UNKNOWN; +} + +static enum isp_firmware_version isp_check_firmware_version(struct device *dev) +{ + enum isp_firmware_version version, compat; + + /* firmware version is just informative */ + version = isp_read_fw_version(dev, "apple,firmware-version"); + compat = isp_read_fw_version(dev, "apple,firmware-compat"); + + dev_info(dev, "ISP firmware-compat: %s (FW: %s)\n", isp_fw2str(compat), + isp_fw2str(version)); + + return compat; +} + +static int apple_isp_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_isp *isp; + int err; + + err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(42)); + if (err) + return err; + + isp = devm_kzalloc(dev, sizeof(*isp), GFP_KERNEL); + if (!isp) + return -ENOMEM; + + isp->dev = dev; + isp->hw = of_device_get_match_data(dev); + platform_set_drvdata(pdev, isp); + dev_set_drvdata(dev, isp); + + /* Differences between firmware versions are rather minor so try to work + * with unknown firmware. + */ + isp->fw_compat = isp_check_firmware_version(dev); + + err = of_property_read_u32(dev->of_node, "apple,platform-id", + &isp->platform_id); + if (err) { + dev_err(dev, "failed to get 'apple,platform-id' property: %d\n", + err); + return err; + } + + err = of_property_read_u32(dev->of_node, "apple,temporal-filter", + &isp->temporal_filter); + if (err) + isp->temporal_filter = 0; + + err = apple_isp_init_presets(isp); + if (err) { + dev_err(dev, "failed to initialize presets\n"); + return err; + } + + err = apple_isp_attach_genpd(isp); + if (err) { + dev_err(dev, "failed to attatch power domains\n"); + return err; + } + + isp->coproc = devm_platform_ioremap_resource_byname(pdev, "coproc"); + if (IS_ERR(isp->coproc)) { + err = PTR_ERR(isp->coproc); + goto detach_genpd; + } + + isp->mbox = devm_platform_ioremap_resource_byname(pdev, "mbox"); + if (IS_ERR(isp->mbox)) { + err = PTR_ERR(isp->mbox); + goto detach_genpd; + } + + isp->gpio = devm_platform_ioremap_resource_byname(pdev, "gpio"); + if (IS_ERR(isp->gpio)) { + err = PTR_ERR(isp->gpio); + goto detach_genpd; + } + + isp->mbox2 = devm_platform_ioremap_resource_byname(pdev, "mbox2"); + if (IS_ERR(isp->mbox2)) { + err = PTR_ERR(isp->mbox2); + goto detach_genpd; + } + + isp->irq = platform_get_irq(pdev, 0); + if (isp->irq < 0) { + err = isp->irq; + goto detach_genpd; + } + if (!isp->irq) { + err = -ENODEV; + goto detach_genpd; + } + + mutex_init(&isp->iovad_lock); + mutex_init(&isp->video_lock); + spin_lock_init(&isp->buf_lock); + init_waitqueue_head(&isp->wait); + INIT_LIST_HEAD(&isp->gc); + INIT_LIST_HEAD(&isp->bufs_pending); + INIT_LIST_HEAD(&isp->bufs_submitted); + isp->wq = alloc_workqueue("apple-isp-wq", WQ_UNBOUND, 0); + if (!isp->wq) { + dev_err(dev, "failed to create workqueue\n"); + err = -ENOMEM; + goto detach_genpd; + } + + err = apple_isp_init_iommu(isp); + if (err) { + dev_err(dev, "failed to init iommu: %d\n", err); + goto destroy_wq; + } + + err = apple_isp_alloc_firmware_surface(isp); + if (err) { + dev_err(dev, "failed to alloc firmware surface: %d\n", err); + goto free_iommu; + } + + pm_runtime_enable(dev); + + err = apple_isp_detect_camera(isp); + if (err) { + dev_err(dev, "failed to detect camera: %d\n", err); + goto free_surface; + } + + err = apple_isp_setup_video(isp); + if (err) { + dev_err(dev, "failed to register video device: %d\n", err); + goto free_surface; + } + + dev_info(dev, "apple-isp probe!\n"); + + return 0; + +free_surface: + pm_runtime_disable(dev); + apple_isp_free_firmware_surface(isp); +free_iommu: + apple_isp_free_iommu(isp); +destroy_wq: + destroy_workqueue(isp->wq); +detach_genpd: + apple_isp_detach_genpd(isp); + return err; +} + +static void apple_isp_remove(struct platform_device *pdev) +{ + struct apple_isp *isp = platform_get_drvdata(pdev); + + apple_isp_remove_video(isp); + pm_runtime_disable(isp->dev); + apple_isp_free_firmware_surface(isp); + apple_isp_free_iommu(isp); + destroy_workqueue(isp->wq); + apple_isp_detach_genpd(isp); +} + +static const struct apple_isp_hw apple_isp_hw_t8103 = { + .gen = ISP_GEN_T8103, + .pmu_base = 0x23b704000, + + .dsid_count = 4, + .dsid_clr_base0 = 0x200014000, + .dsid_clr_base1 = 0x200054000, + .dsid_clr_base2 = 0x200094000, + .dsid_clr_base3 = 0x2000d4000, + .dsid_clr_range0 = 0x1000, + .dsid_clr_range1 = 0x1000, + .dsid_clr_range2 = 0x1000, + .dsid_clr_range3 = 0x1000, + + .clock_scratch = 0x23b738010, + .clock_base = 0x23bc3c000, + .clock_bit = 0x1, + .clock_size = 0x4, + .bandwidth_scratch = 0x23b73800c, + .bandwidth_base = 0x23bc3c000, + .bandwidth_bit = 0x0, + .bandwidth_size = 0x4, + + .scl1 = false, + .lpdp = false, + .meta_size = ISP_META_SIZE_T8103, +}; + +static const struct apple_isp_hw apple_isp_hw_t6000 = { + .gen = ISP_GEN_T8103, + .pmu_base = 0x28e584000, + + .dsid_count = 1, + .dsid_clr_base0 = 0x200014000, + .dsid_clr_base1 = 0x200054000, + .dsid_clr_base2 = 0x200094000, + .dsid_clr_base3 = 0x2000d4000, + .dsid_clr_range0 = 0x1000, + .dsid_clr_range1 = 0x1000, + .dsid_clr_range2 = 0x1000, + .dsid_clr_range3 = 0x1000, + + .clock_scratch = 0x28e3d0868, + .clock_base = 0x0, + .clock_bit = 0x0, + .clock_size = 0x8, + .bandwidth_scratch = 0x28e3d0980, + .bandwidth_base = 0x0, + .bandwidth_bit = 0x0, + .bandwidth_size = 0x8, + + .scl1 = false, + .lpdp = false, + .meta_size = ISP_META_SIZE_T8103, +}; + +static const struct apple_isp_hw apple_isp_hw_t8112 = { + .gen = ISP_GEN_T8112, + .pmu_base = 0x23b704000, + + .dsid_count = 1, + .dsid_clr_base0 = 0x200f14000, + .dsid_clr_range0 = 0x1000, + + .clock_scratch = 0x23b3d0560, + .clock_base = 0x0, + .clock_bit = 0x0, + .clock_size = 0x8, + .bandwidth_scratch = 0x23b3d05d0, + .bandwidth_base = 0x0, + .bandwidth_bit = 0x0, + .bandwidth_size = 0x8, + + .scl1 = false, + .lpdp = false, + .meta_size = ISP_META_SIZE_T8112, +}; + +static const struct apple_isp_hw apple_isp_hw_t6020 = { + .gen = ISP_GEN_T8112, + .pmu_base = 0x290284000, + + .dsid_count = 1, + .dsid_clr_base0 = 0x200f14000, + .dsid_clr_range0 = 0x1000, + + .clock_scratch = 0x28e3d10a8, + .clock_base = 0x0, + .clock_bit = 0x0, + .clock_size = 0x8, + .bandwidth_scratch = 0x28e3d1200, + .bandwidth_base = 0x0, + .bandwidth_bit = 0x0, + .bandwidth_size = 0x8, + + .scl1 = true, + .lpdp = true, + .meta_size = ISP_META_SIZE_T8112, +}; + +static const struct of_device_id apple_isp_of_match[] = { + { .compatible = "apple,t8103-isp", .data = &apple_isp_hw_t8103 }, + { .compatible = "apple,t8112-isp", .data = &apple_isp_hw_t8112 }, + { .compatible = "apple,t6000-isp", .data = &apple_isp_hw_t6000 }, + { .compatible = "apple,t6020-isp", .data = &apple_isp_hw_t6020 }, + {}, +}; +MODULE_DEVICE_TABLE(of, apple_isp_of_match); + +static __maybe_unused int apple_isp_runtime_suspend(struct device *dev) +{ + /* RPM sleep is called when the V4L2 file handle is closed */ + return 0; +} + +static __maybe_unused int apple_isp_runtime_resume(struct device *dev) +{ + return 0; +} + +static __maybe_unused int apple_isp_suspend(struct device *dev) +{ + struct apple_isp *isp = dev_get_drvdata(dev); + + /* We must restore V4L2 context on system resume. If we were streaming + * before, we (essentially) stop streaming and start streaming again. + */ + apple_isp_video_suspend(isp); + + return 0; +} + +static __maybe_unused int apple_isp_resume(struct device *dev) +{ + struct apple_isp *isp = dev_get_drvdata(dev); + + apple_isp_video_resume(isp); + + return 0; +} + +static const struct dev_pm_ops apple_isp_pm_ops = { + SYSTEM_SLEEP_PM_OPS(apple_isp_suspend, apple_isp_resume) + RUNTIME_PM_OPS(apple_isp_runtime_suspend, apple_isp_runtime_resume, NULL) +}; + +static struct platform_driver apple_isp_driver = { + .driver = { + .name = "apple-isp", + .of_match_table = apple_isp_of_match, + .pm = pm_ptr(&apple_isp_pm_ops), + }, + .probe = apple_isp_probe, + .remove = apple_isp_remove, +}; +module_platform_driver(apple_isp_driver); + +MODULE_AUTHOR("Eileen Yoon "); +MODULE_DESCRIPTION("Apple ISP driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h new file mode 100644 index 00000000000000..96a1d0b39f860d --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#ifndef __ISP_DRV_H__ +#define __ISP_DRV_H__ + +#include +#include +#include +#include + +#include +#include +#include +#include + +/* #define APPLE_ISP_DEBUG */ +#define APPLE_ISP_DEVICE_NAME "apple-isp" +#define APPLE_ISP_CARD_NAME "FaceTime HD Camera" + +#define ISP_MAX_CHANNELS 6 +#define ISP_IPC_MESSAGE_SIZE 64 +#define ISP_IPC_FLAG_ACK 0x1 +#define ISP_META_SIZE_T8103 0x4640 +#define ISP_META_SIZE_T8112 0x4840 + +/* used to limit the user space buffers to the buffer_pool_config */ +#define ISP_MAX_BUFFERS 16 + +enum isp_generation { + ISP_GEN_T8103, + ISP_GEN_T8112, +}; + +enum isp_firmware_version { + ISP_FIRMWARE_V_UNKNOWN, + ISP_FIRMWARE_V_12_3, + ISP_FIRMWARE_V_12_4, + ISP_FIRMWARE_V_13_5, +}; + +struct isp_surf { + struct drm_mm_node *mm; + struct list_head head; + u64 size; + u64 type; + u32 num_pages; + struct page **pages; + struct sg_table sgt; + dma_addr_t iova; + void *virt; + refcount_t refcount; + bool gc; + bool submitted; +}; + +struct isp_message { + u64 arg0; + u64 arg1; + u64 arg2; + u64 arg3; + u64 arg4; + u64 arg5; + u64 arg6; + u64 arg7; +} __packed; +static_assert(sizeof(struct isp_message) == ISP_IPC_MESSAGE_SIZE); + +struct isp_channel { + char *name; + u32 type; + u32 src; + u32 num; + u64 size; + dma_addr_t iova; + void *virt; + u32 doorbell; + u32 cursor; + struct mutex lock; + struct isp_message req; + struct isp_message rsp; + const struct isp_chan_ops *ops; +}; + +struct coord { + u32 x; + u32 y; +}; + +struct isp_preset { + u32 index; + struct coord input_dim; + struct coord output_dim; + struct coord crop_offset; + struct coord crop_size; +}; + +struct apple_isp_hw { + enum isp_generation gen; + u64 pmu_base; + + int dsid_count; + u64 dsid_clr_base0; + u64 dsid_clr_base1; + u64 dsid_clr_base2; + u64 dsid_clr_base3; + u32 dsid_clr_range0; + u32 dsid_clr_range1; + u32 dsid_clr_range2; + u32 dsid_clr_range3; + + u64 clock_scratch; + u64 clock_base; + u8 clock_bit; + u8 clock_size; + u64 bandwidth_scratch; + u64 bandwidth_base; + u8 bandwidth_bit; + u8 bandwidth_size; + + u32 meta_size; + bool scl1; + bool lpdp; +}; + +enum isp_sensor_id { + ISP_IMX248_1820_01, + ISP_IMX248_1822_02, + ISP_IMX343_5221_02, + ISP_IMX354_9251_02, + ISP_IMX356_4820_01, + ISP_IMX356_4820_02, + ISP_IMX364_8720_01, + ISP_IMX364_8723_01, + ISP_IMX372_3820_01, + ISP_IMX372_3820_02, + ISP_IMX372_3820_11, + ISP_IMX372_3820_12, + ISP_IMX405_9720_01, + ISP_IMX405_9721_01, + ISP_IMX405_9723_01, + ISP_IMX414_2520_01, + ISP_IMX503_7820_01, + ISP_IMX503_7820_02, + ISP_IMX505_3921_01, + ISP_IMX514_2820_01, + ISP_IMX514_2820_02, + ISP_IMX514_2820_03, + ISP_IMX514_2820_04, + ISP_IMX558_1921_01, + ISP_IMX558_1922_02, + ISP_IMX603_7920_01, + ISP_IMX603_7920_02, + ISP_IMX603_7921_01, + ISP_IMX613_4920_01, + ISP_IMX613_4920_02, + ISP_IMX614_2921_01, + ISP_IMX614_2921_02, + ISP_IMX614_2922_02, + ISP_IMX633_3622_01, + ISP_IMX703_7721_01, + ISP_IMX703_7722_01, + ISP_IMX713_4721_01, + ISP_IMX713_4722_01, + ISP_IMX714_2022_01, + ISP_IMX772_3721_01, + ISP_IMX772_3721_11, + ISP_IMX772_3722_01, + ISP_IMX772_3723_01, + ISP_IMX814_2123_01, + ISP_IMX853_7622_01, + ISP_IMX913_7523_01, + ISP_VD56G0_6221_01, + ISP_VD56G0_6222_01, +}; + +struct isp_format { + enum isp_sensor_id id; + u32 version; + struct isp_preset *preset; + unsigned int num_planes; + u32 strides[VB2_MAX_PLANES]; + size_t plane_size[VB2_MAX_PLANES]; + size_t total_size; +}; + +struct apple_isp { + struct device *dev; + const struct apple_isp_hw *hw; + enum isp_firmware_version fw_compat; + u32 platform_id; + u32 temporal_filter; + struct isp_preset *presets; + int num_presets; + + int num_channels; + struct isp_format fmts[ISP_MAX_CHANNELS]; + unsigned int current_ch; + + struct video_device vdev; + struct media_device mdev; + struct v4l2_device v4l2_dev; + struct vb2_queue vbq; + struct mutex video_lock; + unsigned int sequence; + bool multiplanar; + + int pd_count; + struct device **pd_dev; + struct device_link **pd_link; + bool pds_active; + + int irq; + + void __iomem *coproc; + void __iomem *mbox; + void __iomem *gpio; + void __iomem *mbox2; + + struct iommu_domain *domain; + unsigned long shift; + struct drm_mm iovad; /* TODO iova.c can't allocate bottom-up */ + struct mutex iovad_lock; + + struct isp_firmware { + u64 heap_top; + } fw; + + struct isp_surf *ipc_surf; + struct isp_surf *extra_surf; + struct isp_surf *data_surf; + struct isp_surf *log_surf; + struct isp_surf *bt_surf; + struct isp_surf *meta_surfs[ISP_MAX_BUFFERS]; + struct list_head gc; + struct workqueue_struct *wq; + + int num_ipc_chans; + struct isp_channel **ipc_chans; + struct isp_channel *chan_tm; /* TERMINAL */ + struct isp_channel *chan_io; /* IO */ + struct isp_channel *chan_dg; /* DEBUG */ + struct isp_channel *chan_bh; /* BUF_H2T */ + struct isp_channel *chan_bt; /* BUF_T2H */ + struct isp_channel *chan_sm; /* SHAREDMALLOC */ + struct isp_channel *chan_it; /* IO_T2H */ + + wait_queue_head_t wait; + dma_addr_t cmd_iova; + void *cmd_virt; + + unsigned long state; + spinlock_t buf_lock; + struct list_head bufs_pending; + struct list_head bufs_submitted; +}; + +struct isp_chan_ops { + int (*handle)(struct apple_isp *isp, struct isp_channel *chan); +}; + +struct isp_buffer { + struct vb2_v4l2_buffer vb; + struct list_head link; + struct isp_surf surfs[VB2_MAX_PLANES]; +}; + +#define to_isp_buffer(x) container_of((x), struct isp_buffer, vb) + +enum { + ISP_STATE_STREAMING, + ISP_STATE_LOGGING, + ISP_STATE_SLEEPING, +}; + +#ifdef APPLE_ISP_DEBUG +#define isp_dbg(isp, fmt, ...) \ + dev_info((isp)->dev, "[%s] " fmt, __func__, ##__VA_ARGS__) +#else +#define isp_dbg(isp, fmt, ...) \ + dev_dbg((isp)->dev, "[%s] " fmt, __func__, ##__VA_ARGS__) +#endif + +#define isp_err(isp, fmt, ...) \ + dev_err((isp)->dev, "[%s] " fmt, __func__, ##__VA_ARGS__) + +#define isp_get_format(isp, ch) (&(isp)->fmts[(ch)]) +#define isp_get_current_format(isp) (isp_get_format(isp, isp->current_ch)) + +#endif /* __ISP_DRV_H__ */ diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c new file mode 100644 index 00000000000000..a39f5fb4445fa7 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -0,0 +1,788 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#include "isp-fw.h" + +#include +#include +#include +#include + +#include "isp-cmd.h" +#include "isp-iommu.h" +#include "isp-ipc.h" +#include "isp-regs.h" +#include "isp-v4l2.h" + +#define ISP_FIRMWARE_MDELAY 1 +#define ISP_FIRMWARE_MAX_TRIES 1000 + +#define ISP_FIRMWARE_IPC_SIZE 0x1c000 +#define ISP_FIRMWARE_DATA_SIZE 0x28000 + +#define ISP_COPROC_IN_WFI 0x3 + +static inline u32 isp_coproc_read32(struct apple_isp *isp, u32 reg) +{ + return readl(isp->coproc + reg); +} + +static inline void isp_coproc_write32(struct apple_isp *isp, u32 reg, u32 val) +{ + writel(val, isp->coproc + reg); +} + +static inline u32 isp_gpio_read32(struct apple_isp *isp, u32 reg) +{ + return readl(isp->gpio + reg); +} + +static inline void isp_gpio_write32(struct apple_isp *isp, u32 reg, u32 val) +{ + writel(val, isp->gpio + reg); +} + +static int apple_isp_power_up_domains(struct apple_isp *isp) +{ + int ret; + + if (isp->pds_active) + return 0; + + for (int i = 1; i < isp->pd_count; i++) { + ret = pm_runtime_get_sync(isp->pd_dev[i]); + if (ret < 0) { + dev_err(isp->dev, + "Failed to power up power domain %d: %d\n", i, ret); + while (--i != 1) + pm_runtime_put_sync(isp->pd_dev[i]); + return ret; + } + } + + isp->pds_active = true; + + return 0; +} + +static void apple_isp_power_down_domains(struct apple_isp *isp) +{ + int ret; + + if (!isp->pds_active) + return; + + for (int i = isp->pd_count - 1; i >= 1; i--) { + ret = pm_runtime_put_sync(isp->pd_dev[i]); + if (ret < 0) + dev_err(isp->dev, + "Failed to power up power domain %d: %d\n", i, ret); + } + + isp->pds_active = false; +} + +void *apple_isp_translate(struct apple_isp *isp, struct isp_surf *surf, + dma_addr_t iova, size_t size) +{ + dma_addr_t end = iova + size; + if (!surf) { + dev_err(isp->dev, + "Failed to translate IPC iova 0x%llx (0x%zx): No surface\n", + (long long)iova, size); + return NULL; + } + + if (end < iova || iova < surf->iova || + end > (surf->iova + surf->size)) { + dev_err(isp->dev, + "Failed to translate IPC iova 0x%llx (0x%zx): Out of bounds\n", + (long long)iova, size); + return NULL; + } + + if (!surf->virt) { + dev_err(isp->dev, + "Failed to translate IPC iova 0x%llx (0x%zx): No VMap\n", + (long long)iova, size); + return NULL; + } + + return surf->virt + (iova - surf->iova); +} + +struct isp_firmware_bootargs { + u32 pad_0[2]; + u64 ipc_iova; + u64 shared_base; + u64 shared_size; + u64 extra_iova; + u64 extra_size; + u32 platform_id; + u32 pad_40; + u64 logbuf_addr; + u64 logbuf_size; + u64 logbuf_entsize; + u32 ipc_size; + u32 pad_60[5]; + u32 unk5; + u32 pad_7c[13]; + u32 pad_b0; + u32 unk7; + u32 pad_b8[5]; + u32 unk_iova1; + u32 pad_c0[47]; + u32 unk9; +} __packed; +static_assert(sizeof(struct isp_firmware_bootargs) == 0x180); + +struct isp_chan_desc { + char name[64]; + u32 type; + u32 src; + u32 num; + u32 pad; + u64 iova; + u32 padding[0x2a]; +} __packed; +static_assert(sizeof(struct isp_chan_desc) == 0x100); + +static const struct isp_chan_ops tm_ops = { + .handle = ipc_tm_handle, +}; + +static const struct isp_chan_ops sm_ops = { + .handle = ipc_sm_handle, +}; + +static const struct isp_chan_ops bt_ops = { + .handle = ipc_bt_handle, +}; + +static irqreturn_t apple_isp_isr(int irq, void *dev) +{ + struct apple_isp *isp = dev; + + isp_mbox2_write32(isp, ISP_MBOX2_IRQ_ACK, + isp_mbox_read32(isp, ISP_MBOX_IRQ_INTERRUPT)); + + return IRQ_WAKE_THREAD; +} + +static irqreturn_t apple_isp_isr_thread(int irq, void *dev) +{ + struct apple_isp *isp = dev; + + wake_up_all(&isp->wait); + + ipc_chan_handle(isp, isp->chan_sm); + wake_up_all(&isp->wait); /* Some commands depend on sm */ + + ipc_chan_handle(isp, isp->chan_tm); + + ipc_chan_handle(isp, isp->chan_bt); + wake_up_all(&isp->wait); + + return IRQ_HANDLED; +} + +static void isp_disable_irq(struct apple_isp *isp) +{ + isp_mbox_write32(isp, ISP_MBOX_IRQ_ENABLE, 0x0); + free_irq(isp->irq, isp); + isp_gpio_write32(isp, ISP_GPIO_1, 0xfeedbabe); /* real funny */ +} + +static int isp_enable_irq(struct apple_isp *isp) +{ + int err; + + err = request_threaded_irq(isp->irq, apple_isp_isr, + apple_isp_isr_thread, 0, "apple-isp", isp); + if (err < 0) { + isp_err(isp, "failed to request IRQ#%u (%d)\n", isp->irq, err); + return err; + } + + isp_dbg(isp, "about to enable interrupts...\n"); + + isp_mbox_write32(isp, ISP_MBOX_IRQ_ENABLE, 0xf); + + return 0; +} + +static int isp_reset_coproc(struct apple_isp *isp) +{ + int retries; + u32 status; + u32 val; + + isp_coproc_write32(isp, ISP_COPROC_EDPRCR, 0x2); + + isp_coproc_write32(isp, ISP_COPROC_FABRIC_0, 0xff00ff); + isp_coproc_write32(isp, ISP_COPROC_FABRIC_1, 0xff00ff); + isp_coproc_write32(isp, ISP_COPROC_FABRIC_2, 0xff00ff); + isp_coproc_write32(isp, ISP_COPROC_FABRIC_3, 0xff00ff); + + isp_coproc_write32(isp, ISP_COPROC_IRQ_MASK_0, 0xffffffff); + isp_coproc_write32(isp, ISP_COPROC_IRQ_MASK_1, 0xffffffff); + isp_coproc_write32(isp, ISP_COPROC_IRQ_MASK_2, 0xffffffff); + isp_coproc_write32(isp, ISP_COPROC_IRQ_MASK_3, 0xffffffff); + isp_coproc_write32(isp, ISP_COPROC_IRQ_MASK_4, 0xffffffff); + isp_coproc_write32(isp, ISP_COPROC_IRQ_MASK_5, 0xffffffff); + + for (retries = 0; retries < 128; retries++) { + val = isp_coproc_read32(isp, 0x818); + if (val == 0) + break; + } + + for (retries = 0; retries < 128; retries++) { + val = isp_coproc_read32(isp, 0x81c); + if (val == 0) + break; + } + + for (retries = 0; retries < ISP_FIRMWARE_MAX_TRIES; retries++) { + status = isp_coproc_read32(isp, ISP_COPROC_STATUS); + if (status & ISP_COPROC_IN_WFI) { + isp_dbg(isp, "%d: coproc in WFI (status: 0x%x)\n", + retries, status); + break; + } + mdelay(ISP_FIRMWARE_MDELAY); + } + if (retries >= ISP_FIRMWARE_MAX_TRIES) { + isp_err(isp, "coproc NOT in WFI (status: 0x%x)\n", status); + return -ENODEV; + } + + return 0; +} + +static void isp_firmware_shutdown_stage1(struct apple_isp *isp) +{ + isp_coproc_write32(isp, ISP_COPROC_CONTROL, 0x0); + + apple_isp_power_down_domains(isp); +} + +static int isp_firmware_boot_stage1(struct apple_isp *isp) +{ + int err, retries; + // u32 val; + + err = apple_isp_power_up_domains(isp); + if (err < 0) + return err; + + + isp_gpio_write32(isp, ISP_GPIO_CLOCK_EN, 0x1); + +#if 0 + /* This doesn't work well with system sleep */ + val = isp_gpio_read32(isp, ISP_GPIO_1); + if (val == 0xfeedbabe) { + err = isp_reset_coproc(isp); + if (err < 0) + return err; + } +#endif + + err = isp_reset_coproc(isp); + if (err < 0) + return err; + + isp_gpio_write32(isp, ISP_GPIO_0, 0x0); + isp_gpio_write32(isp, ISP_GPIO_1, 0x0); + isp_gpio_write32(isp, ISP_GPIO_2, 0x0); + isp_gpio_write32(isp, ISP_GPIO_3, 0x0); + isp_gpio_write32(isp, ISP_GPIO_4, 0x0); + isp_gpio_write32(isp, ISP_GPIO_5, 0x0); + isp_gpio_write32(isp, ISP_GPIO_6, 0x0); + isp_gpio_write32(isp, ISP_GPIO_7, 0x0); + + isp_mbox_write32(isp, ISP_MBOX_IRQ_ENABLE, 0x0); + + isp_coproc_write32(isp, ISP_COPROC_CONTROL, 0x0); + isp_coproc_write32(isp, ISP_COPROC_CONTROL, 0x10); + + /* Wait for ISP_GPIO_7 to 0x0 -> 0x8042006 */ + for (retries = 0; retries < ISP_FIRMWARE_MAX_TRIES; retries++) { + u32 val = isp_gpio_read32(isp, ISP_GPIO_7); + if (val == 0x8042006) { + isp_dbg(isp, + "got first magic number (0x%x) from firmware\n", + val); + break; + } + mdelay(ISP_FIRMWARE_MDELAY); + } + if (retries >= ISP_FIRMWARE_MAX_TRIES) { + isp_err(isp, + "never received first magic number from firmware\n"); + return -ENODEV; + } + + return 0; +} + +int apple_isp_alloc_firmware_surface(struct apple_isp *isp) +{ + /* These are static, so let's do it once and for all */ + isp->ipc_surf = isp_alloc_surface_vmap(isp, ISP_FIRMWARE_IPC_SIZE); + if (!isp->ipc_surf) { + isp_err(isp, "failed to alloc shared surface for ipc\n"); + return -ENOMEM; + } + dev_info(isp->dev, "IPC surface iova: 0x%llx\n", + (long long)isp->ipc_surf->iova); + + isp->data_surf = isp_alloc_surface_vmap(isp, ISP_FIRMWARE_DATA_SIZE); + if (!isp->data_surf) { + isp_err(isp, "failed to alloc shared surface for data files\n"); + isp_free_surface(isp, isp->ipc_surf); + return -ENOMEM; + } + dev_info(isp->dev, "Data surface iova: 0x%llx\n", + (long long)isp->data_surf->iova); + + return 0; +} + +void apple_isp_free_firmware_surface(struct apple_isp *isp) +{ + isp_free_surface(isp, isp->data_surf); + isp_free_surface(isp, isp->ipc_surf); +} + +static void isp_firmware_shutdown_stage2(struct apple_isp *isp) +{ + isp_free_surface(isp, isp->extra_surf); +} + +static int isp_firmware_boot_stage2(struct apple_isp *isp) +{ + struct isp_firmware_bootargs args; + dma_addr_t args_iova; + void *args_virt; + int err, retries; + + u32 num_ipc_chans = isp_gpio_read32(isp, ISP_GPIO_0); + u32 args_offset = isp_gpio_read32(isp, ISP_GPIO_1); + u32 extra_size = isp_gpio_read32(isp, ISP_GPIO_3); + isp->num_ipc_chans = num_ipc_chans; + + if (!isp->num_ipc_chans) { + dev_err(isp->dev, "No IPC channels found\n"); + return -ENODEV; + } + + if (isp->num_ipc_chans != 7) + dev_warn(isp->dev, "unexpected channel count (%d)\n", + num_ipc_chans); + + isp->extra_surf = isp_alloc_surface_vmap(isp, extra_size); + if (!isp->extra_surf) { + isp_err(isp, "failed to alloc surface for extra heap\n"); + return -ENOMEM; + } + + args_iova = isp->ipc_surf->iova + args_offset + 0x40; + args_virt = isp->ipc_surf->virt + args_offset + 0x40; + isp->cmd_iova = args_iova + sizeof(args) + 0x40; + isp->cmd_virt = args_virt + sizeof(args) + 0x40; + + memset(&args, 0, sizeof(args)); + args.ipc_iova = isp->ipc_surf->iova; + args.ipc_size = isp->ipc_surf->size; + args.shared_base = isp->fw.heap_top & 0xffffffff; + args.shared_size = 0x10000000UL - args.shared_base; + args.extra_iova = isp->extra_surf->iova; + args.extra_size = isp->extra_surf->size; + args.platform_id = isp->platform_id; + args.unk5 = 0x40; + args.unk7 = 0x1; // 0? + args.unk_iova1 = args_iova + sizeof(args) - 0xc; + args.unk9 = 0x3; + memcpy(args_virt, &args, sizeof(args)); + + isp_gpio_write32(isp, ISP_GPIO_0, args_iova); + isp_gpio_write32(isp, ISP_GPIO_1, args_iova >> 32); + dma_wmb(); + + /* Wait for ISP_GPIO_7 to 0xf7fbdff9 -> 0x8042006 */ + isp_gpio_write32(isp, ISP_GPIO_7, 0xf7fbdff9); + + for (retries = 0; retries < ISP_FIRMWARE_MAX_TRIES; retries++) { + u32 val = isp_gpio_read32(isp, ISP_GPIO_7); + if (val == 0x8042006) { + isp_dbg(isp, + "got second magic number (0x%x) from firmware\n", + val); + break; + } + mdelay(ISP_FIRMWARE_MDELAY); + } + if (retries >= ISP_FIRMWARE_MAX_TRIES) { + isp_err(isp, + "never received second magic number from firmware\n"); + err = -ENODEV; + goto free_extra; + } + + return 0; + +free_extra: + isp_free_surface(isp, isp->extra_surf); + return err; +} + +static inline struct isp_channel *isp_get_chan_index(struct apple_isp *isp, + const char *name) +{ + for (int i = 0; i < isp->num_ipc_chans; i++) { + if (!strcasecmp(isp->ipc_chans[i]->name, name)) + return isp->ipc_chans[i]; + } + return NULL; +} + +static void isp_free_channel_info(struct apple_isp *isp) +{ + for (int i = 0; i < isp->num_ipc_chans; i++) { + struct isp_channel *chan = isp->ipc_chans[i]; + if (!chan) + continue; + kfree(chan->name); + kfree(chan); + isp->ipc_chans[i] = NULL; + } + kfree(isp->ipc_chans); + isp->ipc_chans = NULL; +} + +static int isp_fill_channel_info(struct apple_isp *isp) +{ + u64 table_iova = isp_gpio_read32(isp, ISP_GPIO_0) | + ((u64)isp_gpio_read32(isp, ISP_GPIO_1)) << 32; + void *table_virt = apple_isp_ipc_translate( + isp, table_iova, + sizeof(struct isp_chan_desc) * isp->num_ipc_chans); + + if (!table_virt) { + dev_err(isp->dev, "Failed to find channel table\n"); + return -EIO; + } + + isp->ipc_chans = kcalloc(isp->num_ipc_chans, + sizeof(struct isp_channel *), GFP_KERNEL); + if (!isp->ipc_chans) + goto out; + + for (int i = 0; i < isp->num_ipc_chans; i++) { + struct isp_chan_desc desc; + void *desc_virt = table_virt + (i * sizeof(desc)); + struct isp_channel *chan = + kzalloc(sizeof(struct isp_channel), GFP_KERNEL); + if (!chan) + goto out; + isp->ipc_chans[i] = chan; + + memcpy(&desc, desc_virt, sizeof(desc)); + chan->name = kstrdup(desc.name, GFP_KERNEL); + chan->type = desc.type; + chan->src = desc.src; + chan->doorbell = 1 << chan->src; + chan->num = desc.num; + chan->size = desc.num * ISP_IPC_MESSAGE_SIZE; + chan->iova = desc.iova; + chan->virt = + apple_isp_ipc_translate(isp, desc.iova, chan->size); + chan->cursor = 0; + mutex_init(&chan->lock); + + if (!chan->virt) { + dev_err(isp->dev, "Failed to find channel buffer\n"); + goto out; + } + + if ((chan->type != ISP_IPC_CHAN_TYPE_COMMAND) && + (chan->type != ISP_IPC_CHAN_TYPE_REPLY) && + (chan->type != ISP_IPC_CHAN_TYPE_REPORT)) { + isp_err(isp, "invalid ipc chan type (%d)\n", + chan->type); + goto out; + } + + isp_dbg(isp, "chan: %s type: %d src: %d num: %d iova: 0x%llx\n", + chan->name, chan->type, chan->src, chan->num, + chan->iova); + } + + isp->chan_tm = isp_get_chan_index(isp, "TERMINAL"); + isp->chan_io = isp_get_chan_index(isp, "IO"); + isp->chan_dg = isp_get_chan_index(isp, "DEBUG"); + isp->chan_bh = isp_get_chan_index(isp, "BUF_H2T"); + isp->chan_bt = isp_get_chan_index(isp, "BUF_T2H"); + isp->chan_sm = isp_get_chan_index(isp, "SHAREDMALLOC"); + isp->chan_it = isp_get_chan_index(isp, "IO_T2H"); + + if (!isp->chan_tm || !isp->chan_io || !isp->chan_dg || !isp->chan_bh || + !isp->chan_bt || !isp->chan_sm || !isp->chan_it) { + isp_err(isp, "did not find all of the required ipc chans\n"); + goto out; + } + + isp->chan_tm->ops = &tm_ops; + isp->chan_sm->ops = &sm_ops; + isp->chan_bt->ops = &bt_ops; + + return 0; +out: + isp_free_channel_info(isp); + return -ENOMEM; +} + +static void isp_firmware_shutdown_stage3(struct apple_isp *isp) +{ + isp_free_channel_info(isp); +} + +static int isp_firmware_boot_stage3(struct apple_isp *isp) +{ + int err, retries; + + err = isp_fill_channel_info(isp); + if (err < 0) + return err; + + /* Mask the command channels to prepare for submission */ + for (int i = 0; i < isp->num_ipc_chans; i++) { + struct isp_channel *chan = isp->ipc_chans[i]; + if (chan->type != ISP_IPC_CHAN_TYPE_COMMAND) + continue; + for (int j = 0; j < chan->num; j++) { + struct isp_message msg; + void *msg_virt = chan->virt + (j * sizeof(msg)); + + memset(&msg, 0, sizeof(msg)); + msg.arg0 = ISP_IPC_FLAG_ACK; + memcpy(msg_virt, &msg, sizeof(msg)); + } + } + dma_wmb(); + + /* Wait for ISP_GPIO_3 to 0x8042006 -> 0x0 */ + isp_gpio_write32(isp, ISP_GPIO_3, 0x8042006); + + for (retries = 0; retries < ISP_FIRMWARE_MAX_TRIES; retries++) { + u32 val = isp_gpio_read32(isp, ISP_GPIO_3); + if (val == 0x0) { + isp_dbg(isp, + "got third magic number (0x%x) from firmware\n", + val); + break; + } + mdelay(ISP_FIRMWARE_MDELAY); + } + if (retries >= ISP_FIRMWARE_MAX_TRIES) { + isp_err(isp, + "never received third magic number from firmware\n"); + isp_free_channel_info(isp); + return -ENODEV; + } + + isp_dbg(isp, "firmware booted!\n"); + + return 0; +} + +static int isp_stop_command_processor(struct apple_isp *isp) +{ + int retries; + +#if 0 + int res = isp_cmd_stop(isp, 0); + if (res) { + isp_err(isp, "isp_cmd_stop() failed\n"); + return res; + } + + /* Wait for ISP_GPIO_0 to 0xf7fbdff9 -> 0x8042006 */ + isp_gpio_write32(isp, ISP_GPIO_0, 0xf7fbdff9); + + isp_cmd_power_down(isp); +#else + isp_gpio_write32(isp, ISP_GPIO_0, 0xf7fbdff9); + + int res = isp_cmd_suspend(isp); + if (res) { + isp_err(isp, "isp_cmd_suspend() failed\n"); + return res; + } +#endif + + for (retries = 0; retries < ISP_FIRMWARE_MAX_TRIES; retries++) { + u32 val = isp_gpio_read32(isp, ISP_GPIO_0); + if (val == 0x8042006) { + isp_dbg(isp, "got magic number (0x%x) from firmware\n", + val); + break; + } + mdelay(ISP_FIRMWARE_MDELAY); + } + if (retries >= ISP_FIRMWARE_MAX_TRIES) { + isp_err(isp, "never received magic number from firmware\n"); + return -ENODEV; + } + + return 0; +} + +static int isp_start_command_processor(struct apple_isp *isp) +{ + int err; + + err = isp_cmd_print_enable(isp, 1); + if (err) + return err; + + err = isp_cmd_set_isp_pmu_base(isp, isp->hw->pmu_base); + if (err) + return err; + + if (isp->hw->dsid_count == 1) { + err = isp_cmd_set_dsid_clr_req_base( + isp, isp->hw->dsid_clr_base0, isp->hw->dsid_clr_range0); + if (err) + return err; + } else { + err = isp_cmd_set_dsid_clr_req_base2( + isp, isp->hw->dsid_clr_base0, isp->hw->dsid_clr_base1, + isp->hw->dsid_clr_base2, isp->hw->dsid_clr_base3, + isp->hw->dsid_clr_range0, isp->hw->dsid_clr_range1, + isp->hw->dsid_clr_range2, isp->hw->dsid_clr_range3); + if (err) + return err; + } + + err = isp_cmd_pmp_ctrl_set( + isp, isp->hw->clock_scratch, isp->hw->clock_base, + isp->hw->clock_bit, isp->hw->clock_size, + isp->hw->bandwidth_scratch, isp->hw->bandwidth_base, + isp->hw->bandwidth_bit, isp->hw->bandwidth_size); + if (err) + return err; + + err = isp_cmd_start(isp, 0); + if (err) + return err; + + /* Now we can access CISP_CMD_CH_* commands */ + + return 0; +} + +static void isp_collect_gc_surface(struct apple_isp *isp) +{ + struct isp_surf *tmp, *surf; + + isp->log_surf = NULL; + isp->bt_surf = NULL; + + list_for_each_entry_safe_reverse(surf, tmp, &isp->gc, head) { + isp_dbg(isp, "freeing iova: 0x%llx size: 0x%llx virt: %pS\n", + surf->iova, surf->size, (void *)surf->virt); + isp_free_surface(isp, surf); + } +} + +static int isp_firmware_boot(struct apple_isp *isp) +{ + int err; + + err = isp_firmware_boot_stage1(isp); + if (err < 0) { + isp_err(isp, "failed firmware boot stage 1: %d\n", err); + goto garbage_collect; + } + + err = isp_firmware_boot_stage2(isp); + if (err < 0) { + isp_err(isp, "failed firmware boot stage 2: %d\n", err); + goto shutdown_stage1; + } + + err = isp_firmware_boot_stage3(isp); + if (err < 0) { + isp_err(isp, "failed firmware boot stage 3: %d\n", err); + goto shutdown_stage2; + } + + err = isp_enable_irq(isp); + if (err < 0) { + isp_err(isp, "failed to enable interrupts: %d\n", err); + goto shutdown_stage3; + } + + err = isp_start_command_processor(isp); + if (err < 0) { + isp_err(isp, "failed to start command processor: %d\n", err); + goto disable_irqs; + } + + flush_workqueue(isp->wq); + + return 0; + +disable_irqs: + isp_disable_irq(isp); +shutdown_stage3: + isp_firmware_shutdown_stage3(isp); +shutdown_stage2: + isp_firmware_shutdown_stage2(isp); +shutdown_stage1: + isp_firmware_shutdown_stage1(isp); +garbage_collect: + isp_collect_gc_surface(isp); + return err; +} + +static void isp_firmware_shutdown(struct apple_isp *isp) +{ + flush_workqueue(isp->wq); + isp_stop_command_processor(isp); + isp_disable_irq(isp); + isp_firmware_shutdown_stage3(isp); + isp_firmware_shutdown_stage2(isp); + isp_firmware_shutdown_stage1(isp); + isp_collect_gc_surface(isp); +} + +int apple_isp_firmware_boot(struct apple_isp *isp) +{ + int err; + + /* Needs to be power cycled for IOMMU to behave correctly */ + err = pm_runtime_resume_and_get(isp->dev); + if (err < 0) { + dev_err(isp->dev, "failed to enable power: %d\n", err); + return err; + } + + err = isp_firmware_boot(isp); + if (err) { + dev_err(isp->dev, "failed to boot firmware: %d\n", err); + pm_runtime_put_sync(isp->dev); + return err; + } + + return 0; +} + +void apple_isp_firmware_shutdown(struct apple_isp *isp) +{ + isp_firmware_shutdown(isp); + pm_runtime_put_sync(isp->dev); +} diff --git a/drivers/media/platform/apple/isp/isp-fw.h b/drivers/media/platform/apple/isp/isp-fw.h new file mode 100644 index 00000000000000..974216f0989f91 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-fw.h @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#ifndef __ISP_FW_H__ +#define __ISP_FW_H__ + +#include "isp-drv.h" + +int apple_isp_alloc_firmware_surface(struct apple_isp *isp); +void apple_isp_free_firmware_surface(struct apple_isp *isp); + +int apple_isp_firmware_boot(struct apple_isp *isp); +void apple_isp_firmware_shutdown(struct apple_isp *isp); + +void *apple_isp_translate(struct apple_isp *isp, struct isp_surf *surf, + dma_addr_t iova, size_t size); + +static inline void *apple_isp_ipc_translate(struct apple_isp *isp, + dma_addr_t iova, size_t size) +{ + return apple_isp_translate(isp, isp->ipc_surf, iova, size); +} + +#endif /* __ISP_FW_H__ */ diff --git a/drivers/media/platform/apple/isp/isp-iommu.c b/drivers/media/platform/apple/isp/isp-iommu.c new file mode 100644 index 00000000000000..1ddd089d77355a --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-iommu.c @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#include +#include + +#include "isp-iommu.h" + +static void isp_surf_free_pages(struct isp_surf *surf) +{ + for (u32 i = 0; i < surf->num_pages && surf->pages[i] != NULL; i++) { + __free_page(surf->pages[i]); + } + kvfree(surf->pages); +} + +static int isp_surf_alloc_pages(struct isp_surf *surf) +{ + surf->pages = kvmalloc_array(surf->num_pages, sizeof(*surf->pages), + GFP_KERNEL); + if (!surf->pages) + return -ENOMEM; + + for (u32 i = 0; i < surf->num_pages; i++) { + surf->pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (surf->pages[i] == NULL) + goto free_pages; + } + + return 0; + +free_pages: + isp_surf_free_pages(surf); + return -ENOMEM; +} + +int isp_surf_vmap(struct apple_isp *isp, struct isp_surf *surf) +{ + surf->virt = vmap(surf->pages, surf->num_pages, VM_MAP, + pgprot_writecombine(PAGE_KERNEL)); + if (surf->virt == NULL) { + dev_err(isp->dev, "failed to vmap size 0x%llx\n", surf->size); + return -EINVAL; + } + + return 0; +} + +static void isp_surf_vunmap(struct apple_isp *isp, struct isp_surf *surf) +{ + if (surf->virt) + vunmap(surf->virt); + surf->virt = NULL; +} + +static void isp_surf_unreserve_iova(struct apple_isp *isp, + struct isp_surf *surf) +{ + if (surf->mm) { + mutex_lock(&isp->iovad_lock); + drm_mm_remove_node(surf->mm); + mutex_unlock(&isp->iovad_lock); + kfree(surf->mm); + } + surf->mm = NULL; +} + +static int isp_surf_reserve_iova(struct apple_isp *isp, struct isp_surf *surf) +{ + int err; + + surf->mm = kzalloc(sizeof(*surf->mm), GFP_KERNEL); + if (!surf->mm) + return -ENOMEM; + + mutex_lock(&isp->iovad_lock); + err = drm_mm_insert_node_generic(&isp->iovad, surf->mm, + ALIGN(surf->size, 1UL << isp->shift), + 1UL << isp->shift, 0, 0); + mutex_unlock(&isp->iovad_lock); + if (err < 0) { + dev_err(isp->dev, "failed to reserve 0x%llx of iova space\n", + surf->size); + goto mm_free; + } + + surf->iova = surf->mm->start; + + return 0; +mm_free: + kfree(surf->mm); + surf->mm = NULL; + return err; +} + +static void isp_surf_iommu_unmap(struct apple_isp *isp, struct isp_surf *surf) +{ + iommu_unmap(isp->domain, surf->iova, surf->size); + sg_free_table(&surf->sgt); +} + +static int isp_surf_iommu_map(struct apple_isp *isp, struct isp_surf *surf) +{ + unsigned long size; + int err; + + err = sg_alloc_table_from_pages(&surf->sgt, surf->pages, + surf->num_pages, 0, surf->size, + GFP_KERNEL); + if (err < 0) { + dev_err(isp->dev, "failed to alloc sgt from pages\n"); + return err; + } + + size = iommu_map_sgtable(isp->domain, surf->iova, &surf->sgt, + IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE); + if (size < surf->size) { + dev_err(isp->dev, "failed to iommu_map sgt to iova 0x%llx\n", + surf->iova); + sg_free_table(&surf->sgt); + return -ENXIO; + } + + return 0; +} + +static void __isp_surf_init(struct apple_isp *isp, struct isp_surf *surf, + u64 size, bool gc) +{ + surf->mm = NULL; + surf->virt = NULL; + surf->size = ALIGN(size, 1UL << isp->shift); + surf->num_pages = surf->size >> isp->shift; + surf->gc = gc; +} + +struct isp_surf *__isp_alloc_surface(struct apple_isp *isp, u64 size, bool gc) +{ + int err; + + struct isp_surf *surf = kzalloc(sizeof(struct isp_surf), GFP_KERNEL); + if (!surf) + return NULL; + + __isp_surf_init(isp, surf, size, gc); + + err = isp_surf_alloc_pages(surf); + if (err < 0) { + dev_err(isp->dev, "failed to allocate %d pages\n", + surf->num_pages); + goto free_surf; + } + + err = isp_surf_reserve_iova(isp, surf); + if (err < 0) { + dev_err(isp->dev, "failed to reserve 0x%llx of iova space\n", + surf->size); + goto free_pages; + } + + err = isp_surf_iommu_map(isp, surf); + if (err < 0) { + dev_err(isp->dev, + "failed to iommu_map size 0x%llx to iova 0x%llx\n", + surf->size, surf->iova); + goto unreserve_iova; + } + + refcount_set(&surf->refcount, 1); + if (surf->gc) + list_add_tail(&surf->head, &isp->gc); + + return surf; + +unreserve_iova: + isp_surf_unreserve_iova(isp, surf); +free_pages: + isp_surf_free_pages(surf); +free_surf: + kfree(surf); + return NULL; +} + +struct isp_surf *isp_alloc_surface_vmap(struct apple_isp *isp, u64 size) +{ + int err; + + struct isp_surf *surf = __isp_alloc_surface(isp, size, false); + if (!surf) + return NULL; + + err = isp_surf_vmap(isp, surf); + if (err < 0) { + dev_err(isp->dev, "failed to vmap iova 0x%llx - 0x%llx\n", + surf->iova, surf->iova + surf->size); + isp_free_surface(isp, surf); + return NULL; + } + + return surf; +} + +void isp_free_surface(struct apple_isp *isp, struct isp_surf *surf) +{ + if (refcount_dec_and_test(&surf->refcount)) { + isp_surf_vunmap(isp, surf); + isp_surf_iommu_unmap(isp, surf); + isp_surf_unreserve_iova(isp, surf); + isp_surf_free_pages(surf); + if (surf->gc) + list_del(&surf->head); + kfree(surf); + } +} + +int apple_isp_iommu_map_sgt(struct apple_isp *isp, struct isp_surf *surf, + struct sg_table *sgt, u64 size) +{ + int err; + ssize_t mapped; + + // TODO userptr sends unaligned sizes + surf->mm = NULL; + surf->size = size; + + err = isp_surf_reserve_iova(isp, surf); + if (err < 0) { + dev_err(isp->dev, "failed to reserve 0x%llx of iova space\n", + surf->size); + return err; + } + + mapped = iommu_map_sgtable(isp->domain, surf->iova, sgt, + IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE); + if (mapped < surf->size) { + dev_err(isp->dev, "failed to iommu_map sgt to iova 0x%llx\n", + surf->iova); + isp_surf_unreserve_iova(isp, surf); + return -ENXIO; + } + surf->size = mapped; + + return 0; +} + +void apple_isp_iommu_unmap_sgt(struct apple_isp *isp, struct isp_surf *surf) +{ + iommu_unmap(isp->domain, surf->iova, surf->size); + isp_surf_unreserve_iova(isp, surf); +} diff --git a/drivers/media/platform/apple/isp/isp-iommu.h b/drivers/media/platform/apple/isp/isp-iommu.h new file mode 100644 index 00000000000000..b99a182e284b72 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-iommu.h @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#ifndef __ISP_IOMMU_H__ +#define __ISP_IOMMU_H__ + +#include "isp-drv.h" + +struct isp_surf *__isp_alloc_surface(struct apple_isp *isp, u64 size, bool gc); +#define isp_alloc_surface(isp, size) (__isp_alloc_surface(isp, size, false)) +#define isp_alloc_surface_gc(isp, size) (__isp_alloc_surface(isp, size, true)) +struct isp_surf *isp_alloc_surface_vmap(struct apple_isp *isp, u64 size); +int isp_surf_vmap(struct apple_isp *isp, struct isp_surf *surf); +void isp_free_surface(struct apple_isp *isp, struct isp_surf *surf); + +int apple_isp_iommu_map_sgt(struct apple_isp *isp, struct isp_surf *surf, + struct sg_table *sgt, u64 size); +void apple_isp_iommu_unmap_sgt(struct apple_isp *isp, struct isp_surf *surf); + +#endif /* __ISP_IOMMU_H__ */ diff --git a/drivers/media/platform/apple/isp/isp-ipc.c b/drivers/media/platform/apple/isp/isp-ipc.c new file mode 100644 index 00000000000000..a1948717a31968 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-ipc.c @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#include "isp-iommu.h" +#include "isp-ipc.h" +#include "isp-regs.h" +#include "isp-fw.h" + +#define ISP_IPC_FLAG_TERMINAL_ACK 0x3 +#define ISP_IPC_BUFEXC_STAT_META_OFFSET 0x10 + +struct isp_sm_deferred_work { + struct work_struct work; + struct apple_isp *isp; + struct isp_surf *surf; +}; + +struct isp_bufexc_stat { + u64 unk_0; // 2 + u64 unk_8; // 2 + + u64 meta_iova; + u64 pad_20[3]; + u64 meta_size; // 0x4640 + u64 unk_38; + + u32 unk_40; // 1 + u32 unk_44; + u64 unk_48; + + u64 iova0; + u64 iova1; + u64 iova2; + u64 iova3; + u32 pad_70[4]; + + u32 unk_80; // 2 + u32 unk_84; // 1 + u32 unk_88; // 0x10 || 0x13 + u32 unk_8c; + u32 pad_90[96]; + + u32 unk_210; // 0x28 + u32 unk_214; + u32 index; + u16 bes_width; // 1296, 0x510 + u16 bes_height; // 736, 0x2e0 + + u32 unk_220; // 0x0 || 0x1 + u32 pad_224[3]; + u32 unk_230; // 0xf7ed38 + u32 unk_234; // 3 + u32 pad_238[2]; + u32 pad_240[16]; +} __packed; +static_assert(sizeof(struct isp_bufexc_stat) == ISP_IPC_BUFEXC_STAT_SIZE); + +static inline void *chan_msg_virt(struct isp_channel *chan, u32 index) +{ + return chan->virt + (index * ISP_IPC_MESSAGE_SIZE); +} + +static inline void chan_read_msg_index(struct apple_isp *isp, + struct isp_channel *chan, + struct isp_message *msg, u32 index) +{ + memcpy(msg, chan_msg_virt(chan, index), sizeof(*msg)); +} + +static inline void chan_read_msg(struct apple_isp *isp, + struct isp_channel *chan, + struct isp_message *msg) +{ + chan_read_msg_index(isp, chan, msg, chan->cursor); +} + +static inline void chan_write_msg_index(struct apple_isp *isp, + struct isp_channel *chan, + struct isp_message *msg, u32 index) +{ + u64 *p0 = chan_msg_virt(chan, index); + memcpy(p0 + 1, &msg->arg1, sizeof(*msg) - 8); + + /* Make sure we write arg0 last, since that indicates message validity. */ + + dma_wmb(); + *p0 = msg->arg0; + dma_wmb(); +} + +static inline void chan_write_msg(struct apple_isp *isp, + struct isp_channel *chan, + struct isp_message *msg) +{ + chan_write_msg_index(isp, chan, msg, chan->cursor); +} + +static inline void chan_update_cursor(struct isp_channel *chan) +{ + if (chan->cursor >= (chan->num - 1)) { + chan->cursor = 0; + } else { + chan->cursor += 1; + } +} + +static int chan_handle_once(struct apple_isp *isp, struct isp_channel *chan) +{ + int err; + + lockdep_assert_held(&chan->lock); + + err = chan->ops->handle(isp, chan); + if (err < 0) { + dev_err(isp->dev, "%s: handler failed: %d)\n", chan->name, err); + return err; + } + + chan_write_msg(isp, chan, &chan->rsp); + + isp_mbox2_write32(isp, ISP_MBOX2_IRQ_DOORBELL, chan->doorbell); + + chan_update_cursor(chan); + + return 0; +} + +static inline bool chan_rx_done(struct apple_isp *isp, struct isp_channel *chan) +{ + if (((chan->req.arg0 & 0xf) == ISP_IPC_FLAG_ACK) || + ((chan->req.arg0 & 0xf) == ISP_IPC_FLAG_TERMINAL_ACK)) { + return true; + } + return false; +} + +int ipc_chan_handle(struct apple_isp *isp, struct isp_channel *chan) +{ + int err = 0; + + mutex_lock(&chan->lock); + while (1) { + chan_read_msg(isp, chan, &chan->req); + if (chan_rx_done(isp, chan)) { + err = 0; + break; + } + err = chan_handle_once(isp, chan); + if (err < 0) { + break; + } + } + mutex_unlock(&chan->lock); + + return err; +} + +static inline bool chan_tx_done(struct apple_isp *isp, struct isp_channel *chan) +{ + dma_rmb(); + + chan_read_msg(isp, chan, &chan->rsp); + if ((chan->rsp.arg0) == (chan->req.arg0 | ISP_IPC_FLAG_ACK)) { + chan_update_cursor(chan); + return true; + } + return false; +} + +int ipc_chan_send(struct apple_isp *isp, struct isp_channel *chan, + unsigned long timeout) +{ + long t; + + chan_write_msg(isp, chan, &chan->req); + dma_wmb(); + + isp_mbox2_write32(isp, ISP_MBOX2_IRQ_DOORBELL, chan->doorbell); + + if (!timeout) + return 0; + + t = wait_event_timeout(isp->wait, chan_tx_done(isp, chan), timeout); + if (t == 0) { + dev_err(isp->dev, + "%s: timed out on request [0x%llx, 0x%llx, 0x%llx]\n", + chan->name, chan->req.arg0, chan->req.arg1, + chan->req.arg2); + return -ETIME; + } + + isp_dbg(isp, "%s: request success (%ld)\n", chan->name, t); + + return 0; +} + +int ipc_tm_handle(struct apple_isp *isp, struct isp_channel *chan) +{ + struct isp_message *rsp = &chan->rsp; + +#ifdef APPLE_ISP_DEBUG + struct isp_message *req = &chan->req; + char buf[512]; + dma_addr_t iova = req->arg0 & ~ISP_IPC_FLAG_TERMINAL_ACK; + u32 size = req->arg1; + if (iova && size && size < sizeof(buf) && + isp->log_surf) { + void *p = apple_isp_translate(isp, isp->log_surf, iova, size); + if (p) { + size = min_t(u32, size, 512); + memcpy(buf, p, size); + isp_dbg(isp, "ISPASC: %.*s", size, buf); + } + } +#endif + + rsp->arg0 = ISP_IPC_FLAG_ACK; + rsp->arg1 = 0x0; + rsp->arg2 = 0x0; + + return 0; +} + +int ipc_sm_handle(struct apple_isp *isp, struct isp_channel *chan) +{ + struct isp_message *req = &chan->req, *rsp = &chan->rsp; + int err; + + if (req->arg0 == 0x0) { + struct isp_surf *surf; + + surf = isp_alloc_surface_gc(isp, req->arg1); + if (!surf) { + isp_err(isp, "failed to alloc requested size 0x%llx\n", + req->arg1); + return -ENOMEM; + } + surf->type = req->arg2; + + rsp->arg0 = surf->iova | ISP_IPC_FLAG_ACK; + rsp->arg1 = 0x0; + rsp->arg2 = 0x0; /* macOS uses this to index surfaces */ + + switch (surf->type) { + case 0x4c4f47: /* "LOG" */ + isp->log_surf = surf; + break; + case 0x4d495343: /* "MISC" */ + /* Hacky... maybe there's a better way to identify this surface? */ + if (surf->size == 0xc000) + isp->bt_surf = surf; + break; + default: + // skip vmap + return 0; + } + + err = isp_surf_vmap(isp, surf); + if (err < 0) { + isp_err(isp, "failed to vmap iova=0x%llx size=0x%llx\n", + surf->iova, surf->size); + } + } else { + /* This should be the shared surface free request, but + * 1) The fw doesn't request to free all of what it requested + * 2) The fw continues to access the surface after + * So we link it to the gc, which runs after fw shutdown + */ + rsp->arg0 = req->arg0 | ISP_IPC_FLAG_ACK; + rsp->arg1 = 0x0; + rsp->arg2 = 0x0; + } + + return 0; +} diff --git a/drivers/media/platform/apple/isp/isp-ipc.h b/drivers/media/platform/apple/isp/isp-ipc.h new file mode 100644 index 00000000000000..0c1d681835c72f --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-ipc.h @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#ifndef __ISP_IPC_H__ +#define __ISP_IPC_H__ + +#include "isp-drv.h" + +#define ISP_IPC_CHAN_TYPE_COMMAND 0 +#define ISP_IPC_CHAN_TYPE_REPLY 1 +#define ISP_IPC_CHAN_TYPE_REPORT 2 + +#define ISP_IPC_BUFEXC_STAT_SIZE 0x280 +#define ISP_IPC_BUFEXC_FLAG_RENDER 0x10000000 +#define ISP_IPC_BUFEXC_FLAG_COMMAND 0x30000000 +#define ISP_IPC_BUFEXC_FLAG_ACK 0x80000000 + +int ipc_chan_handle(struct apple_isp *isp, struct isp_channel *chan); +int ipc_chan_send(struct apple_isp *isp, struct isp_channel *chan, + unsigned long timeout); + +int ipc_tm_handle(struct apple_isp *isp, struct isp_channel *chan); +int ipc_sm_handle(struct apple_isp *isp, struct isp_channel *chan); + +#endif /* __ISP_IPC_H__ */ diff --git a/drivers/media/platform/apple/isp/isp-regs.h b/drivers/media/platform/apple/isp/isp-regs.h new file mode 100644 index 00000000000000..7357fa10fa5483 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-regs.h @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#ifndef __ISP_REGS_H__ +#define __ISP_REGS_H__ + +#include "isp-drv.h" + +#define ISP_COPROC_FABRIC_0 0x738 +#define ISP_COPROC_FABRIC_1 0x798 +#define ISP_COPROC_FABRIC_2 0x7f8 +#define ISP_COPROC_FABRIC_3 0x858 + +#define ISP_COPROC_RVBAR 0x1050000 +#define ISP_COPROC_EDPRCR 0x1010310 +#define ISP_COPROC_CONTROL 0x1400044 +#define ISP_COPROC_STATUS 0x1400048 + +#define ISP_COPROC_IRQ_MASK_0 0x1400a00 +#define ISP_COPROC_IRQ_MASK_1 0x1400a04 +#define ISP_COPROC_IRQ_MASK_2 0x1400a08 +#define ISP_COPROC_IRQ_MASK_3 0x1400a0c +#define ISP_COPROC_IRQ_MASK_4 0x1400a10 +#define ISP_COPROC_IRQ_MASK_5 0x1400a14 + +#define ISP_MBOX_IRQ_INTERRUPT 0x00 +#define ISP_MBOX_IRQ_ENABLE 0x04 +#define ISP_MBOX2_IRQ_DOORBELL 0x00 +#define ISP_MBOX2_IRQ_ACK 0x0c + +#define ISP_GPIO_0 0x00 +#define ISP_GPIO_1 0x04 +#define ISP_GPIO_2 0x08 +#define ISP_GPIO_3 0x0c +#define ISP_GPIO_4 0x10 +#define ISP_GPIO_5 0x14 +#define ISP_GPIO_6 0x18 +#define ISP_GPIO_7 0x1c +#define ISP_GPIO_CLOCK_EN 0x20 + +static inline u32 isp_mbox_read32(struct apple_isp *isp, u32 reg) +{ + return readl(isp->mbox + reg); +} + +static inline void isp_mbox_write32(struct apple_isp *isp, u32 reg, u32 val) +{ + writel(val, isp->mbox + reg); +} + +static inline void isp_mbox2_write32(struct apple_isp *isp, u32 reg, u32 val) +{ + writel(val, isp->mbox2 + reg); +} + +#endif /* __ISP_REGS_H__ */ diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c new file mode 100644 index 00000000000000..0561653ea7becd --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -0,0 +1,910 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#include + +#include +#include +#include +#include +#include + +#include "isp-cam.h" +#include "isp-cmd.h" +#include "isp-iommu.h" +#include "isp-ipc.h" +#include "isp-fw.h" +#include "isp-v4l2.h" + +#define ISP_MIN_FRAMES 2 +#define ISP_MAX_PLANES 4 +#define ISP_MAX_PIX_FORMATS 2 +#define ISP_BUFFER_TIMEOUT msecs_to_jiffies(1500) +#define ISP_STRIDE_ALIGNMENT 64 + +static bool multiplanar = false; +module_param(multiplanar, bool, 0644); +MODULE_PARM_DESC(multiplanar, "Enable multiplanar API"); + +struct isp_buflist_buffer { + u64 iovas[ISP_MAX_PLANES]; + u32 flags[ISP_MAX_PLANES]; + u32 num_planes; + u32 pool_type; + u32 tag; + u32 pad; +} __packed; +static_assert(sizeof(struct isp_buflist_buffer) == 0x40); + +struct isp_buflist { + u64 type; + u64 num_buffers; + struct isp_buflist_buffer buffers[]; +}; + +int ipc_bt_handle(struct apple_isp *isp, struct isp_channel *chan) +{ + struct isp_message *req = &chan->req, *rsp = &chan->rsp; + struct isp_buffer *tmp, *buf; + struct isp_buflist *bl; + u32 count; + int err = 0; + + /* printk("H2T: 0x%llx 0x%llx 0x%llx\n", (long long)req->arg0, + (long long)req->arg1, (long long)req->arg2); */ + + if (req->arg1 < sizeof(struct isp_buflist)) { + dev_err(isp->dev, "%s: Bad length 0x%llx\n", chan->name, + req->arg1); + return -EIO; + } + + bl = apple_isp_translate(isp, isp->bt_surf, req->arg0, req->arg1); + + count = bl->num_buffers; + if (count > (req->arg1 - sizeof(struct isp_buffer)) / + sizeof(struct isp_buflist_buffer)) { + dev_err(isp->dev, "%s: Bad length 0x%llx\n", chan->name, + req->arg1); + return -EIO; + } + + spin_lock(&isp->buf_lock); + for (int i = 0; i < count; i++) { + struct isp_buflist_buffer *bufd = &bl->buffers[i]; + + /* printk("Return: 0x%llx (%d)\n", bufd->iovas[0], + bufd->pool_type); */ + + if (bufd->pool_type == 0) { + for (int j = 0; j < ARRAY_SIZE(isp->meta_surfs); j++) { + struct isp_surf *meta = isp->meta_surfs[j]; + if ((u32)bufd->iovas[0] == (u32)meta->iova) { + WARN_ON(!meta->submitted); + meta->submitted = false; + } + } + } else { + list_for_each_entry_safe_reverse( + buf, tmp, &isp->bufs_submitted, link) { + if ((u32)buf->surfs[0].iova == + (u32)bufd->iovas[0]) { + enum vb2_buffer_state state = + VB2_BUF_STATE_ERROR; + + buf->vb.vb2_buf.timestamp = + ktime_get_ns(); + buf->vb.sequence = isp->sequence++; + buf->vb.field = V4L2_FIELD_NONE; + if (req->arg2 == + ISP_IPC_BUFEXC_FLAG_RENDER) + state = VB2_BUF_STATE_DONE; + vb2_buffer_done(&buf->vb.vb2_buf, + state); + list_del(&buf->link); + } + } + } + } + spin_unlock(&isp->buf_lock); + + rsp->arg0 = req->arg0 | ISP_IPC_FLAG_ACK; + rsp->arg1 = 0x0; + rsp->arg2 = ISP_IPC_BUFEXC_FLAG_ACK; + + return err; +} + +static int isp_submit_buffers(struct apple_isp *isp) +{ + struct isp_format *fmt = isp_get_current_format(isp); + struct isp_channel *chan = isp->chan_bh; + struct isp_message *req = &chan->req; + struct isp_buffer *buf, *tmp; + unsigned long flags; + size_t offset; + int err; + + struct isp_buflist *bl = isp->cmd_virt; + struct isp_buflist_buffer *bufd = &bl->buffers[0]; + + bl->type = 1; + bl->num_buffers = 0; + + spin_lock_irqsave(&isp->buf_lock, flags); + for (int i = 0; i < ARRAY_SIZE(isp->meta_surfs); i++) { + struct isp_surf *meta = isp->meta_surfs[i]; + + if (meta->submitted) + continue; + + /* printk("Submit: 0x%llx .. 0x%llx (meta)\n", meta->iova, + meta->iova + meta->size); */ + + bufd->num_planes = 1; + bufd->pool_type = 0; + bufd->iovas[0] = meta->iova; + bufd->flags[0] = 0x40000000; + bufd++; + bl->num_buffers++; + + meta->submitted = true; + } + + while ((buf = list_first_entry_or_null(&isp->bufs_pending, + struct isp_buffer, link))) { + memset(bufd, 0, sizeof(*bufd)); + + bufd->num_planes = fmt->num_planes; + bufd->pool_type = isp->hw->scl1 ? CISP_POOL_TYPE_RENDERED_SCL1 : + CISP_POOL_TYPE_RENDERED; + offset = 0; + for (int j = 0; j < fmt->num_planes; j++) { + bufd->iovas[j] = buf->surfs[0].iova + offset; + bufd->flags[j] = 0x40000000; + offset += fmt->plane_size[j]; + } + + /* printk("Submit: 0x%llx .. 0x%llx (render)\n", + buf->surfs[0].iova, + buf->surfs[0].iova + buf->surfs[0].size); */ + bufd++; + bl->num_buffers++; + + /* + * Queue the buffer as submitted and release the lock for now. + * We need to do this before actually submitting to avoid a + * race with the buffer return codepath. + */ + list_move_tail(&buf->link, &isp->bufs_submitted); + } + + spin_unlock_irqrestore(&isp->buf_lock, flags); + + req->arg0 = isp->cmd_iova; + req->arg1 = max_t(u64, ISP_IPC_BUFEXC_STAT_SIZE, + ((uintptr_t)bufd - (uintptr_t)bl)); + req->arg2 = ISP_IPC_BUFEXC_FLAG_COMMAND; + + err = ipc_chan_send(isp, chan, ISP_BUFFER_TIMEOUT); + if (err) { + /* If we fail, consider the buffer not submitted. */ + dev_err(isp->dev, + "%s: failed to send bufs: [0x%llx, 0x%llx, 0x%llx]\n", + chan->name, req->arg0, req->arg1, req->arg2); + + /* + * Try to find the buffer in the list, and if it's + * still there, move it back to the pending list. + */ + spin_lock_irqsave(&isp->buf_lock, flags); + + bufd = &bl->buffers[0]; + for (int i = 0; i < bl->num_buffers; i++, bufd++) { + list_for_each_entry_safe_reverse( + buf, tmp, &isp->bufs_submitted, link) { + if (bufd->iovas[0] == buf->surfs[0].iova) { + list_move_tail(&buf->link, + &isp->bufs_pending); + } + } + for (int j = 0; j < ARRAY_SIZE(isp->meta_surfs); j++) { + struct isp_surf *meta = isp->meta_surfs[j]; + if (bufd->iovas[0] == meta->iova) { + meta->submitted = false; + } + } + } + + spin_unlock_irqrestore(&isp->buf_lock, flags); + } + + return err; +} + +/* + * Videobuf2 section + */ +static int isp_vb2_queue_setup(struct vb2_queue *vq, unsigned int *nbuffers, + unsigned int *num_planes, unsigned int sizes[], + struct device *alloc_devs[]) +{ + struct apple_isp *isp = vb2_get_drv_priv(vq); + struct isp_format *fmt = isp_get_current_format(isp); + + /* This is not strictly neccessary but makes it easy to enforce that + * at most 16 buffers are submitted at once. ISP on t6001 (FW 12.3) + * times out if more buffers are submitted than set in the buffer pool + * config before streaming is started. + */ + *nbuffers = min_t(unsigned int, *nbuffers, ISP_MAX_BUFFERS); + + if (*num_planes) { + if (sizes[0] < fmt->total_size) + return -EINVAL; + + return 0; + } + + *num_planes = 1; + sizes[0] = fmt->total_size; + + return 0; +} + +static void __isp_vb2_buf_cleanup(struct vb2_buffer *vb, unsigned int i) +{ + struct apple_isp *isp = vb2_get_drv_priv(vb->vb2_queue); + struct isp_buffer *buf = + container_of(vb, struct isp_buffer, vb.vb2_buf); + + while (i--) + apple_isp_iommu_unmap_sgt(isp, &buf->surfs[i]); +} + +static void isp_vb2_buf_cleanup(struct vb2_buffer *vb) +{ + __isp_vb2_buf_cleanup(vb, vb->num_planes); +} + +static int isp_vb2_buf_init(struct vb2_buffer *vb) +{ + struct apple_isp *isp = vb2_get_drv_priv(vb->vb2_queue); + struct isp_buffer *buf = + container_of(vb, struct isp_buffer, vb.vb2_buf); + unsigned int i; + int err; + + for (i = 0; i < vb->num_planes; i++) { + struct sg_table *sgt = vb2_dma_sg_plane_desc(vb, i); + err = apple_isp_iommu_map_sgt(isp, &buf->surfs[i], sgt, + vb2_plane_size(vb, i)); + if (err) + goto cleanup; + } + + return 0; + +cleanup: + __isp_vb2_buf_cleanup(vb, i); + return err; +} + +static int isp_vb2_buf_prepare(struct vb2_buffer *vb) +{ + struct apple_isp *isp = vb2_get_drv_priv(vb->vb2_queue); + struct isp_format *fmt = isp_get_current_format(isp); + + if (vb2_plane_size(vb, 0) < fmt->total_size) + return -EINVAL; + + vb2_set_plane_payload(vb, 0, fmt->total_size); + + return 0; +} + +static void isp_vb2_release_buffers(struct apple_isp *isp, + enum vb2_buffer_state state) +{ + struct isp_buffer *buf; + unsigned long flags; + + spin_lock_irqsave(&isp->buf_lock, flags); + list_for_each_entry(buf, &isp->bufs_submitted, link) + vb2_buffer_done(&buf->vb.vb2_buf, state); + INIT_LIST_HEAD(&isp->bufs_submitted); + list_for_each_entry(buf, &isp->bufs_pending, link) + vb2_buffer_done(&buf->vb.vb2_buf, state); + INIT_LIST_HEAD(&isp->bufs_pending); + spin_unlock_irqrestore(&isp->buf_lock, flags); +} + +static void isp_vb2_buf_queue(struct vb2_buffer *vb) +{ + struct apple_isp *isp = vb2_get_drv_priv(vb->vb2_queue); + struct isp_buffer *buf = + container_of(vb, struct isp_buffer, vb.vb2_buf); + unsigned long flags; + bool empty; + + spin_lock_irqsave(&isp->buf_lock, flags); + empty = list_empty(&isp->bufs_pending) && + list_empty(&isp->bufs_submitted); + list_add_tail(&buf->link, &isp->bufs_pending); + spin_unlock_irqrestore(&isp->buf_lock, flags); + + if (test_bit(ISP_STATE_STREAMING, &isp->state) && !empty) + isp_submit_buffers(isp); +} + +static int apple_isp_start_streaming(struct apple_isp *isp) +{ + int err; + + err = apple_isp_start_camera(isp); + if (err) { + dev_err(isp->dev, "failed to start camera: %d\n", err); + goto release_buffers; + } + + err = isp_submit_buffers(isp); + if (err) { + dev_err(isp->dev, "failed to send initial batch: %d\n", err); + goto stop_camera; + } + + err = apple_isp_start_capture(isp); + if (err) { + dev_err(isp->dev, "failed to start capture: %d\n", err); + goto stop_camera; + } + + set_bit(ISP_STATE_STREAMING, &isp->state); + + return 0; + +stop_camera: + apple_isp_stop_camera(isp); +release_buffers: + isp_vb2_release_buffers(isp, VB2_BUF_STATE_QUEUED); + return err; +} + +static void apple_isp_stop_streaming(struct apple_isp *isp) +{ + clear_bit(ISP_STATE_STREAMING, &isp->state); + apple_isp_stop_capture(isp); + apple_isp_stop_camera(isp); +} + +static int isp_vb2_start_streaming(struct vb2_queue *q, unsigned int count) +{ + struct apple_isp *isp = vb2_get_drv_priv(q); + + isp->sequence = 0; + + return apple_isp_start_streaming(isp); +} + +static void isp_vb2_stop_streaming(struct vb2_queue *q) +{ + struct apple_isp *isp = vb2_get_drv_priv(q); + + apple_isp_stop_streaming(isp); + isp_vb2_release_buffers(isp, VB2_BUF_STATE_ERROR); +} + +int apple_isp_video_suspend(struct apple_isp *isp) +{ + /* Swap into STATE_SLEEPING as isp_vb2_buf_queue() submits on + * STATE_STREAMING. + */ + if (test_bit(ISP_STATE_STREAMING, &isp->state)) { + /* Signal buffers to be recycled for clean shutdown */ + isp_vb2_release_buffers(isp, VB2_BUF_STATE_QUEUED); + apple_isp_stop_streaming(isp); + set_bit(ISP_STATE_SLEEPING, &isp->state); + } + + return 0; +} + +int apple_isp_video_resume(struct apple_isp *isp) +{ + if (test_bit(ISP_STATE_SLEEPING, &isp->state)) { + clear_bit(ISP_STATE_SLEEPING, &isp->state); + apple_isp_start_streaming(isp); + } + + return 0; +} + +static const struct vb2_ops isp_vb2_ops = { + .queue_setup = isp_vb2_queue_setup, + .buf_init = isp_vb2_buf_init, + .buf_cleanup = isp_vb2_buf_cleanup, + .buf_prepare = isp_vb2_buf_prepare, + .buf_queue = isp_vb2_buf_queue, + .start_streaming = isp_vb2_start_streaming, + .stop_streaming = isp_vb2_stop_streaming, + .wait_prepare = vb2_ops_wait_prepare, + .wait_finish = vb2_ops_wait_finish, +}; + +static int isp_set_preset(struct apple_isp *isp, struct isp_format *fmt, + struct isp_preset *preset) +{ + int i; + size_t total_size; + + fmt->preset = preset; + + /* I really fucking hope they all use NV12. */ + fmt->num_planes = 2; + fmt->strides[0] = ALIGN(preset->output_dim.x, ISP_STRIDE_ALIGNMENT); + /* UV subsampled interleaved */ + fmt->strides[1] = ALIGN(preset->output_dim.x, ISP_STRIDE_ALIGNMENT); + fmt->plane_size[0] = fmt->strides[0] * preset->output_dim.y; + fmt->plane_size[1] = fmt->strides[1] * preset->output_dim.y / 2; + + total_size = 0; + for (i = 0; i < fmt->num_planes; i++) + total_size += fmt->plane_size[i]; + fmt->total_size = total_size; + + return 0; +} + +static struct isp_preset *isp_select_preset(struct apple_isp *isp, u32 width, + u32 height) +{ + struct isp_preset *preset, *best = &isp->presets[0]; + int i, score, best_score = INT_MAX; + + /* Default if no dimensions */ + if (width == 0 || height == 0) + return &isp->presets[0]; + + for (i = 0; i < isp->num_presets; i++) { + preset = &isp->presets[i]; + score = abs((int)preset->output_dim.x - (int)width) + + abs((int)preset->output_dim.y - (int)height); + if (score < best_score) { + best = preset; + best_score = score; + } + } + + return best; +} + +/* + * V4L2 ioctl section + */ +static int isp_vidioc_querycap(struct file *file, void *priv, + struct v4l2_capability *cap) +{ + strscpy(cap->card, APPLE_ISP_CARD_NAME, sizeof(cap->card)); + strscpy(cap->driver, APPLE_ISP_DEVICE_NAME, sizeof(cap->driver)); + + return 0; +} + +static int isp_vidioc_enum_format(struct file *file, void *fh, + struct v4l2_fmtdesc *f) +{ + struct apple_isp *isp = video_drvdata(file); + + if (f->index >= ISP_MAX_PIX_FORMATS) + return -EINVAL; + + switch (f->index) { + case 0: + f->pixelformat = V4L2_PIX_FMT_NV12; + break; + case 1: + if (!isp->multiplanar) + return -EINVAL; + f->pixelformat = V4L2_PIX_FMT_NV12M; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int isp_vidioc_enum_framesizes(struct file *file, void *fh, + struct v4l2_frmsizeenum *f) +{ + struct apple_isp *isp = video_drvdata(file); + + if (f->index >= isp->num_presets) + return -EINVAL; + + if ((f->pixel_format != V4L2_PIX_FMT_NV12) && + (f->pixel_format != V4L2_PIX_FMT_NV12M)) + return -EINVAL; + + f->discrete.width = isp->presets[f->index].output_dim.x; + f->discrete.height = isp->presets[f->index].output_dim.y; + f->type = V4L2_FRMSIZE_TYPE_DISCRETE; + + return 0; +} + +static int isp_vidioc_enum_frameintervals(struct file *filp, void *priv, + struct v4l2_frmivalenum *interval) +{ + if (interval->index != 0) + return -EINVAL; + + interval->type = V4L2_FRMIVAL_TYPE_DISCRETE; + interval->discrete.numerator = 1; + interval->discrete.denominator = 30; + return 0; +} + +static inline void isp_get_sp_pix_format(struct apple_isp *isp, + struct v4l2_format *f, + struct isp_format *fmt) +{ + f->fmt.pix.width = fmt->preset->output_dim.x; + f->fmt.pix.height = fmt->preset->output_dim.y; + f->fmt.pix.bytesperline = fmt->strides[0]; + f->fmt.pix.sizeimage = fmt->total_size; + + f->fmt.pix.field = V4L2_FIELD_NONE; + f->fmt.pix.pixelformat = V4L2_PIX_FMT_NV12; + f->fmt.pix.colorspace = V4L2_COLORSPACE_REC709; + f->fmt.pix.ycbcr_enc = V4L2_YCBCR_ENC_709; + f->fmt.pix.xfer_func = V4L2_XFER_FUNC_709; +} + +static inline void isp_get_mp_pix_format(struct apple_isp *isp, + struct v4l2_format *f, + struct isp_format *fmt) +{ + f->fmt.pix_mp.width = fmt->preset->output_dim.x; + f->fmt.pix_mp.height = fmt->preset->output_dim.y; + f->fmt.pix_mp.num_planes = fmt->num_planes; + for (int i = 0; i < fmt->num_planes; i++) { + f->fmt.pix_mp.plane_fmt[i].sizeimage = fmt->plane_size[i]; + f->fmt.pix_mp.plane_fmt[i].bytesperline = fmt->strides[i]; + } + + f->fmt.pix_mp.field = V4L2_FIELD_NONE; + f->fmt.pix_mp.pixelformat = V4L2_PIX_FMT_NV12M; + f->fmt.pix_mp.colorspace = V4L2_COLORSPACE_REC709; + f->fmt.pix_mp.ycbcr_enc = V4L2_YCBCR_ENC_709; + f->fmt.pix_mp.xfer_func = V4L2_XFER_FUNC_709; +} + +static int isp_vidioc_get_format(struct file *file, void *fh, + struct v4l2_format *f) +{ + struct apple_isp *isp = video_drvdata(file); + struct isp_format *fmt = isp_get_current_format(isp); + + isp_get_sp_pix_format(isp, f, fmt); + + return 0; +} + +static int isp_vidioc_set_format(struct file *file, void *fh, + struct v4l2_format *f) +{ + struct apple_isp *isp = video_drvdata(file); + struct isp_format *fmt = isp_get_current_format(isp); + struct isp_preset *preset; + int err; + + preset = isp_select_preset(isp, f->fmt.pix.width, f->fmt.pix.height); + err = isp_set_preset(isp, fmt, preset); + if (err) + return err; + + isp_get_sp_pix_format(isp, f, fmt); + + isp->vbq.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + + return 0; +} + +static int isp_vidioc_try_format(struct file *file, void *fh, + struct v4l2_format *f) +{ + struct apple_isp *isp = video_drvdata(file); + struct isp_format fmt = *isp_get_current_format(isp); + struct isp_preset *preset; + int err; + + preset = isp_select_preset(isp, f->fmt.pix.width, f->fmt.pix.height); + err = isp_set_preset(isp, &fmt, preset); + if (err) + return err; + + isp_get_sp_pix_format(isp, f, &fmt); + + return 0; +} + +static int isp_vidioc_get_format_mplane(struct file *file, void *fh, + struct v4l2_format *f) +{ + struct apple_isp *isp = video_drvdata(file); + struct isp_format *fmt = isp_get_current_format(isp); + + if (!isp->multiplanar) + return -ENOTTY; + + isp_get_mp_pix_format(isp, f, fmt); + + return 0; +} + +static int isp_vidioc_set_format_mplane(struct file *file, void *fh, + struct v4l2_format *f) +{ + struct apple_isp *isp = video_drvdata(file); + struct isp_format *fmt = isp_get_current_format(isp); + struct isp_preset *preset; + int err; + + if (!isp->multiplanar) + return -ENOTTY; + + preset = isp_select_preset(isp, f->fmt.pix_mp.width, + f->fmt.pix_mp.height); + err = isp_set_preset(isp, fmt, preset); + if (err) + return err; + + isp_get_mp_pix_format(isp, f, fmt); + + isp->vbq.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; + + return 0; +} + +static int isp_vidioc_try_format_mplane(struct file *file, void *fh, + struct v4l2_format *f) +{ + struct apple_isp *isp = video_drvdata(file); + struct isp_format fmt = *isp_get_current_format(isp); + struct isp_preset *preset; + int err; + + if (!isp->multiplanar) + return -ENOTTY; + + preset = isp_select_preset(isp, f->fmt.pix_mp.width, + f->fmt.pix_mp.height); + err = isp_set_preset(isp, &fmt, preset); + if (err) + return err; + + isp_get_mp_pix_format(isp, f, &fmt); + + return 0; +} + +static int isp_vidioc_enum_input(struct file *file, void *fh, + struct v4l2_input *inp) +{ + if (inp->index) + return -EINVAL; + + strscpy(inp->name, APPLE_ISP_DEVICE_NAME, sizeof(inp->name)); + inp->type = V4L2_INPUT_TYPE_CAMERA; + + return 0; +} + +static int isp_vidioc_get_input(struct file *file, void *fh, unsigned int *i) +{ + *i = 0; + + return 0; +} + +static int isp_vidioc_set_input(struct file *file, void *fh, unsigned int i) +{ + if (i) + return -EINVAL; + + return 0; +} + +static int isp_vidioc_get_param(struct file *file, void *fh, + struct v4l2_streamparm *a) +{ + struct apple_isp *isp = video_drvdata(file); + + if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE && + (!isp->multiplanar || + a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)) + return -EINVAL; + + a->parm.capture.capability = V4L2_CAP_TIMEPERFRAME; + a->parm.capture.readbuffers = ISP_MIN_FRAMES; + a->parm.capture.timeperframe.numerator = ISP_FRAME_RATE_NUM; + a->parm.capture.timeperframe.denominator = ISP_FRAME_RATE_DEN; + + return 0; +} + +static int isp_vidioc_set_param(struct file *file, void *fh, + struct v4l2_streamparm *a) +{ + struct apple_isp *isp = video_drvdata(file); + + if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE && + (!isp->multiplanar || + a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)) + return -EINVAL; + + /* Not supporting frame rate sets. No use. Plus floats. */ + a->parm.capture.capability = V4L2_CAP_TIMEPERFRAME; + a->parm.capture.readbuffers = ISP_MIN_FRAMES; + a->parm.capture.timeperframe.numerator = ISP_FRAME_RATE_NUM; + a->parm.capture.timeperframe.denominator = ISP_FRAME_RATE_DEN; + + return 0; +} + +static const struct v4l2_ioctl_ops isp_v4l2_ioctl_ops = { + .vidioc_querycap = isp_vidioc_querycap, + + .vidioc_enum_fmt_vid_cap = isp_vidioc_enum_format, + .vidioc_g_fmt_vid_cap = isp_vidioc_get_format, + .vidioc_s_fmt_vid_cap = isp_vidioc_set_format, + .vidioc_try_fmt_vid_cap = isp_vidioc_try_format, + .vidioc_g_fmt_vid_cap_mplane = isp_vidioc_get_format_mplane, + .vidioc_s_fmt_vid_cap_mplane = isp_vidioc_set_format_mplane, + .vidioc_try_fmt_vid_cap_mplane = isp_vidioc_try_format_mplane, + + .vidioc_enum_framesizes = isp_vidioc_enum_framesizes, + .vidioc_enum_frameintervals = isp_vidioc_enum_frameintervals, + .vidioc_enum_input = isp_vidioc_enum_input, + .vidioc_g_input = isp_vidioc_get_input, + .vidioc_s_input = isp_vidioc_set_input, + .vidioc_g_parm = isp_vidioc_get_param, + .vidioc_s_parm = isp_vidioc_set_param, + + .vidioc_reqbufs = vb2_ioctl_reqbufs, + .vidioc_querybuf = vb2_ioctl_querybuf, + .vidioc_create_bufs = vb2_ioctl_create_bufs, + .vidioc_qbuf = vb2_ioctl_qbuf, + .vidioc_expbuf = vb2_ioctl_expbuf, + .vidioc_dqbuf = vb2_ioctl_dqbuf, + .vidioc_prepare_buf = vb2_ioctl_prepare_buf, + .vidioc_streamon = vb2_ioctl_streamon, + .vidioc_streamoff = vb2_ioctl_streamoff, +}; + +static const struct v4l2_file_operations isp_v4l2_fops = { + .owner = THIS_MODULE, + .open = v4l2_fh_open, + .release = vb2_fop_release, + .read = vb2_fop_read, + .poll = vb2_fop_poll, + .mmap = vb2_fop_mmap, + .unlocked_ioctl = video_ioctl2, +}; + +static const struct media_device_ops isp_media_device_ops = { + .link_notify = v4l2_pipeline_link_notify, +}; + +int apple_isp_setup_video(struct apple_isp *isp) +{ + struct video_device *vdev = &isp->vdev; + struct vb2_queue *vbq = &isp->vbq; + struct isp_format *fmt = isp_get_current_format(isp); + int err; + + err = isp_set_preset(isp, fmt, &isp->presets[0]); + if (err) { + dev_err(isp->dev, "failed to set default preset: %d\n", err); + return err; + } + + for (int i = 0; i < ARRAY_SIZE(isp->meta_surfs); i++) { + isp->meta_surfs[i] = + isp_alloc_surface_vmap(isp, isp->hw->meta_size); + if (!isp->meta_surfs[i]) { + isp_err(isp, "failed to alloc meta surface\n"); + err = -ENOMEM; + goto surf_cleanup; + } + } + + media_device_init(&isp->mdev); + isp->v4l2_dev.mdev = &isp->mdev; + isp->mdev.ops = &isp_media_device_ops; + isp->mdev.dev = isp->dev; + strscpy(isp->mdev.model, APPLE_ISP_DEVICE_NAME, + sizeof(isp->mdev.model)); + + err = media_device_register(&isp->mdev); + if (err) { + dev_err(isp->dev, "failed to register media device: %d\n", err); + goto media_cleanup; + } + + isp->multiplanar = multiplanar; + + err = v4l2_device_register(isp->dev, &isp->v4l2_dev); + if (err) { + dev_err(isp->dev, "failed to register v4l2 device: %d\n", err); + goto media_unregister; + } + + vbq->drv_priv = isp; + vbq->type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + vbq->io_modes = VB2_MMAP; + vbq->dev = isp->dev; + vbq->ops = &isp_vb2_ops; + vbq->mem_ops = &vb2_dma_sg_memops; + vbq->buf_struct_size = sizeof(struct isp_buffer); + vbq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; + vbq->min_queued_buffers = ISP_MIN_FRAMES; + vbq->lock = &isp->video_lock; + + err = vb2_queue_init(vbq); + if (err) { + dev_err(isp->dev, "failed to init vb2 queue: %d\n", err); + goto v4l2_unregister; + } + + vdev->queue = vbq; + vdev->fops = &isp_v4l2_fops; + vdev->ioctl_ops = &isp_v4l2_ioctl_ops; + vdev->device_caps = V4L2_BUF_TYPE_VIDEO_CAPTURE | V4L2_CAP_STREAMING; + if (isp->multiplanar) + vdev->device_caps |= V4L2_CAP_VIDEO_CAPTURE_MPLANE; + vdev->v4l2_dev = &isp->v4l2_dev; + vdev->vfl_type = VFL_TYPE_VIDEO; + vdev->vfl_dir = VFL_DIR_RX; + vdev->release = video_device_release_empty; + vdev->lock = &isp->video_lock; + strscpy(vdev->name, APPLE_ISP_DEVICE_NAME, sizeof(vdev->name)); + video_set_drvdata(vdev, isp); + + err = video_register_device(vdev, VFL_TYPE_VIDEO, 0); + if (err) { + dev_err(isp->dev, "failed to register video device: %d\n", err); + goto v4l2_unregister; + } + + return 0; + +v4l2_unregister: + v4l2_device_unregister(&isp->v4l2_dev); +media_unregister: + media_device_unregister(&isp->mdev); +media_cleanup: + media_device_cleanup(&isp->mdev); +surf_cleanup: + for (int i = 0; i < ARRAY_SIZE(isp->meta_surfs); i++) { + if (isp->meta_surfs[i]) + isp_free_surface(isp, isp->meta_surfs[i]); + isp->meta_surfs[i] = NULL; + } + + return err; +} + +void apple_isp_remove_video(struct apple_isp *isp) +{ + vb2_video_unregister_device(&isp->vdev); + v4l2_device_unregister(&isp->v4l2_dev); + media_device_unregister(&isp->mdev); + media_device_cleanup(&isp->mdev); + for (int i = 0; i < ARRAY_SIZE(isp->meta_surfs); i++) { + if (isp->meta_surfs[i]) + isp_free_surface(isp, isp->meta_surfs[i]); + isp->meta_surfs[i] = NULL; + } +} diff --git a/drivers/media/platform/apple/isp/isp-v4l2.h b/drivers/media/platform/apple/isp/isp-v4l2.h new file mode 100644 index 00000000000000..4d47deeb83b055 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-v4l2.h @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#ifndef __ISP_V4L2_H__ +#define __ISP_V4L2_H__ + +#include "isp-drv.h" + +int apple_isp_setup_video(struct apple_isp *isp); +void apple_isp_remove_video(struct apple_isp *isp); +int ipc_bt_handle(struct apple_isp *isp, struct isp_channel *chan); + +int apple_isp_video_suspend(struct apple_isp *isp); +int apple_isp_video_resume(struct apple_isp *isp); + +#endif /* __ISP_V4L2_H__ */ diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c index e3038c18ca3621..a4387ed58cac36 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c @@ -1753,8 +1753,10 @@ static int wave5_vpu_open_dec(struct file *filp) spin_lock_init(&inst->state_spinlock); inst->codec_info = kzalloc(sizeof(*inst->codec_info), GFP_KERNEL); - if (!inst->codec_info) + if (!inst->codec_info) { + kfree(inst); return -ENOMEM; + } v4l2_fh_init(&inst->v4l2_fh, vdev); v4l2_fh_add(&inst->v4l2_fh, filp); diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c index 9bfaa9fb3ceb3e..94fb5d7c87021a 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c @@ -1578,8 +1578,10 @@ static int wave5_vpu_open_enc(struct file *filp) inst->ops = &wave5_vpu_enc_inst_ops; inst->codec_info = kzalloc(sizeof(*inst->codec_info), GFP_KERNEL); - if (!inst->codec_info) + if (!inst->codec_info) { + kfree(inst); return -ENOMEM; + } v4l2_fh_init(&inst->v4l2_fh, vdev); v4l2_fh_add(&inst->v4l2_fh, filp); diff --git a/drivers/media/usb/uvc/uvc_video.c b/drivers/media/usb/uvc/uvc_video.c index 2094e059d7d39a..ec76595f3c4be0 100644 --- a/drivers/media/usb/uvc/uvc_video.c +++ b/drivers/media/usb/uvc/uvc_video.c @@ -1812,7 +1812,7 @@ static int uvc_alloc_urb_buffers(struct uvc_streaming *stream, npackets = UVC_MAX_PACKETS; /* Retry allocations until one succeed. */ - for (; npackets > 1; npackets /= 2) { + for (; npackets > 0; npackets /= 2) { stream->urb_size = psize * npackets; for (i = 0; i < UVC_URBS; ++i) { @@ -1837,6 +1837,7 @@ static int uvc_alloc_urb_buffers(struct uvc_streaming *stream, uvc_dbg(stream->dev, VIDEO, "Failed to allocate URB buffers (%u bytes per packet)\n", psize); + stream->urb_size = 0; return 0; } diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig index aace5766b38aa5..f7f12a0428aa25 100644 --- a/drivers/mfd/Kconfig +++ b/drivers/mfd/Kconfig @@ -407,6 +407,17 @@ config MFD_CS47L92 help Support for Cirrus Logic CS42L92, CS47L92 and CS47L93 Smart Codecs +config MFD_TN48M_CPLD + tristate "Delta Networks TN48M switch CPLD driver" + depends on I2C + depends on ARCH_MVEBU || COMPILE_TEST + select MFD_SIMPLE_MFD_I2C + help + Select this option to enable support for Delta Networks TN48M switch + CPLD. It consists of reset and GPIO drivers. CPLD provides GPIOS-s + for the SFP slots as well as power supply related information. + SFP support depends on the GPIO driver being selected. + config PMIC_DA903X bool "Dialog Semiconductor DA9030/DA9034 PMIC Support" depends on I2C=y diff --git a/drivers/mfd/arizona-core.c b/drivers/mfd/arizona-core.c index 85ff8717d85047..91975536d14d21 100644 --- a/drivers/mfd/arizona-core.c +++ b/drivers/mfd/arizona-core.c @@ -1100,7 +1100,7 @@ int arizona_dev_init(struct arizona *arizona) } else if (val & 0x01) { ret = wm5102_clear_write_sequencer(arizona); if (ret) - return ret; + goto err_reset; } break; default: diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index e3893e255ce5e4..358feec2d088fc 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -45,8 +45,12 @@ #define SMC_TIMEOUT_MS 500 static const struct mfd_cell apple_smc_devs[] = { + MFD_CELL_NAME("macsmc-input"), + MFD_CELL_NAME("macsmc-power"), MFD_CELL_OF("macsmc-gpio", NULL, NULL, 0, 0, "apple,smc-gpio"), + MFD_CELL_OF("macsmc-hwmon", NULL, NULL, 0, 0, "apple,smc-hwmon"), MFD_CELL_OF("macsmc-reboot", NULL, NULL, 0, 0, "apple,smc-reboot"), + MFD_CELL_OF("macsmc-rtc", NULL, NULL, 0, 0, "apple,smc-rtc"), }; static int apple_smc_cmd_locked(struct apple_smc *smc, u64 cmd, u64 arg, @@ -413,6 +417,7 @@ static int apple_smc_probe(struct platform_device *pdev) if (!smc) return -ENOMEM; + mutex_init(&smc->mutex); smc->dev = &pdev->dev; smc->sram_base = devm_platform_get_and_ioremap_resource(pdev, 1, &smc->sram); if (IS_ERR(smc->sram_base)) diff --git a/drivers/mfd/sec-irq.c b/drivers/mfd/sec-irq.c index 74ac70002d1fc5..ff2671186e89f2 100644 --- a/drivers/mfd/sec-irq.c +++ b/drivers/mfd/sec-irq.c @@ -198,6 +198,7 @@ static const struct regmap_irq_chip s2mpg10_irq_chip = { static const struct regmap_irq_chip s2mpg10_irq_chip_pmic = { .name = "s2mpg10-pmic", + .domain_suffix = "pmic", .status_base = S2MPG10_PMIC_INT1, .mask_base = S2MPG10_PMIC_INT1M, .num_regs = 6, diff --git a/drivers/mfd/simple-mfd-i2c.c b/drivers/mfd/simple-mfd-i2c.c index 8b751d8e3b5ae4..7315fad618e445 100644 --- a/drivers/mfd/simple-mfd-i2c.c +++ b/drivers/mfd/simple-mfd-i2c.c @@ -116,6 +116,7 @@ static const struct simple_mfd_data spacemit_p1 = { }; static const struct of_device_id simple_mfd_i2c_of_match[] = { + { .compatible = "delta,tn48m-cpld" }, { .compatible = "fsl,ls1028aqds-fpga" }, { .compatible = "fsl,lx2160aqds-fpga" }, { .compatible = "fsl,lx2160ardb-fpga" }, diff --git a/drivers/mmc/host/sdhci-pci-core.c b/drivers/mmc/host/sdhci-pci-core.c index 47a0a738862b58..93122133202367 100644 --- a/drivers/mmc/host/sdhci-pci-core.c +++ b/drivers/mmc/host/sdhci-pci-core.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -2129,6 +2130,7 @@ static struct sdhci_pci_slot *sdhci_pci_probe_slot( struct sdhci_host *host; int ret, bar = first_bar + slotno; size_t priv_size = chip->fixes ? chip->fixes->priv_size : 0; + u32 cd_debounce_delay_ms; if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM)) { dev_err(&pdev->dev, "BAR %d is not iomem. Aborting.\n", bar); @@ -2195,6 +2197,10 @@ static struct sdhci_pci_slot *sdhci_pci_probe_slot( if (host->mmc->caps & MMC_CAP_CD_WAKE) device_init_wakeup(&pdev->dev, true); + if (device_property_read_u32(&pdev->dev, "cd-debounce-delay-ms", + &cd_debounce_delay_ms)) + cd_debounce_delay_ms = 200; + if (slot->cd_idx >= 0) { struct gpiod_lookup_table *cd_gpio_lookup_table; @@ -2213,7 +2219,7 @@ static struct sdhci_pci_slot *sdhci_pci_probe_slot( ret = mmc_gpiod_request_cd(host->mmc, NULL, slot->cd_idx, slot->cd_override_level, - 0); + cd_debounce_delay_ms * 1000); if (ret == -EPROBE_DEFER) goto remove; @@ -2221,6 +2227,16 @@ static struct sdhci_pci_slot *sdhci_pci_probe_slot( dev_warn(&pdev->dev, "failed to setup card detect gpio\n"); slot->cd_idx = -1; } + } else if (is_of_node(pdev->dev.fwnode)) { + /* Allow all OF systems to use a CD GPIO if provided */ + + ret = mmc_gpiod_request_cd(host->mmc, "cd", 0, + slot->cd_override_level, + cd_debounce_delay_ms * 1000); + if (ret == -EPROBE_DEFER) + goto remove; + else if (ret == 0) + slot->cd_idx = 0; } if (chip->fixes && chip->fixes->add_host) diff --git a/drivers/mtd/devices/mtd_intel_dg.c b/drivers/mtd/devices/mtd_intel_dg.c index 2bab30dcd35fd0..7f751c48a76d4c 100644 --- a/drivers/mtd/devices/mtd_intel_dg.c +++ b/drivers/mtd/devices/mtd_intel_dg.c @@ -770,6 +770,7 @@ static int intel_dg_mtd_probe(struct auxiliary_device *aux_dev, kref_init(&nvm->refcnt); mutex_init(&nvm->lock); + nvm->nregions = nregions; for (n = 0, i = 0; i < INTEL_DG_NVM_REGIONS; i++) { if (!invm->regions[i].name) @@ -777,13 +778,15 @@ static int intel_dg_mtd_probe(struct auxiliary_device *aux_dev, char *name = kasprintf(GFP_KERNEL, "%s.%s", dev_name(&aux_dev->dev), invm->regions[i].name); - if (!name) - continue; + if (!name) { + ret = -ENOMEM; + goto err; + } + nvm->regions[n].name = name; nvm->regions[n].id = i; n++; } - nvm->nregions = n; /* in case where kasprintf fail */ ret = devm_pm_runtime_enable(device); if (ret < 0) { diff --git a/drivers/mtd/nand/raw/cadence-nand-controller.c b/drivers/mtd/nand/raw/cadence-nand-controller.c index 5f037753f78c82..99135ec230105e 100644 --- a/drivers/mtd/nand/raw/cadence-nand-controller.c +++ b/drivers/mtd/nand/raw/cadence-nand-controller.c @@ -1066,7 +1066,7 @@ static int cadence_nand_cdma_send(struct cdns_nand_ctrl *cdns_ctrl, } /* Send SDMA command and wait for finish. */ -static u32 +static int cadence_nand_cdma_send_and_wait(struct cdns_nand_ctrl *cdns_ctrl, u8 thread) { diff --git a/drivers/mtd/parsers/ofpart_core.c b/drivers/mtd/parsers/ofpart_core.c index abfa687989182f..09961c6f39496e 100644 --- a/drivers/mtd/parsers/ofpart_core.c +++ b/drivers/mtd/parsers/ofpart_core.c @@ -77,6 +77,7 @@ static int parse_fixed_partitions(struct mtd_info *master, of_id = of_match_node(parse_ofpart_match_table, ofpart_node); if (dedicated && !of_id) { /* The 'partitions' subnode might be used by another parser */ + of_node_put(ofpart_node); return 0; } @@ -91,12 +92,18 @@ static int parse_fixed_partitions(struct mtd_info *master, nr_parts++; } - if (nr_parts == 0) + if (nr_parts == 0) { + if (dedicated) + of_node_put(ofpart_node); return 0; + } parts = kcalloc(nr_parts, sizeof(*parts), GFP_KERNEL); - if (!parts) + if (!parts) { + if (dedicated) + of_node_put(ofpart_node); return -ENOMEM; + } i = 0; for_each_child_of_node(ofpart_node, pp) { @@ -175,6 +182,9 @@ static int parse_fixed_partitions(struct mtd_info *master, if (quirks && quirks->post_parse) quirks->post_parse(master, parts, nr_parts); + if (dedicated) + of_node_put(ofpart_node); + *pparts = parts; return nr_parts; @@ -183,6 +193,8 @@ static int parse_fixed_partitions(struct mtd_info *master, master->name, pp, mtd_node); ret = -EINVAL; ofpart_none: + if (dedicated) + of_node_put(ofpart_node); of_node_put(pp); kfree(parts); return ret; diff --git a/drivers/mtd/parsers/tplink_safeloader.c b/drivers/mtd/parsers/tplink_safeloader.c index e358a029dc70cf..4fcaf92d22e4fe 100644 --- a/drivers/mtd/parsers/tplink_safeloader.c +++ b/drivers/mtd/parsers/tplink_safeloader.c @@ -116,6 +116,7 @@ static int mtd_parser_tplink_safeloader_parse(struct mtd_info *mtd, return idx; err_free: + kfree(buf); for (idx -= 1; idx >= 0; idx--) kfree(parts[idx].name); err_free_parts: diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index 45bd2bb102ffdf..4c58d1dafcacba 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -791,26 +791,29 @@ static int bond_update_speed_duplex(struct slave *slave) struct ethtool_link_ksettings ecmd; int res; - slave->speed = SPEED_UNKNOWN; - slave->duplex = DUPLEX_UNKNOWN; - res = __ethtool_get_link_ksettings(slave_dev, &ecmd); if (res < 0) - return 1; + goto speed_duplex_unknown; if (ecmd.base.speed == 0 || ecmd.base.speed == ((__u32)-1)) - return 1; + goto speed_duplex_unknown; switch (ecmd.base.duplex) { case DUPLEX_FULL: case DUPLEX_HALF: break; default: - return 1; + goto speed_duplex_unknown; } slave->speed = ecmd.base.speed; slave->duplex = ecmd.base.duplex; return 0; + +speed_duplex_unknown: + slave->speed = SPEED_UNKNOWN; + slave->duplex = DUPLEX_UNKNOWN; + + return 1; } const char *bond_slave_link_status(s8 link) @@ -4311,9 +4314,13 @@ static int bond_close(struct net_device *bond_dev) bond_work_cancel_all(bond); bond->send_peer_notif = 0; + WRITE_ONCE(bond->recv_probe, NULL); + + /* Wait for any in-flight RX handlers */ + synchronize_net(); + if (bond_is_lb(bond)) bond_alb_deinitialize(bond); - bond->recv_probe = NULL; if (BOND_MODE(bond) == BOND_MODE_8023AD && bond->params.broadcast_neighbor) diff --git a/drivers/net/caif/caif_serial.c b/drivers/net/caif/caif_serial.c index c398ac42eae905..b90890030751f1 100644 --- a/drivers/net/caif/caif_serial.c +++ b/drivers/net/caif/caif_serial.c @@ -284,6 +284,7 @@ static void ser_release(struct work_struct *work) { struct list_head list; struct ser_device *ser, *tmp; + struct tty_struct *tty; spin_lock(&ser_lock); list_replace_init(&ser_release_list, &list); @@ -292,9 +293,11 @@ static void ser_release(struct work_struct *work) if (!list_empty(&list)) { rtnl_lock(); list_for_each_entry_safe(ser, tmp, &list, node) { + tty = ser->tty; dev_close(ser->dev); unregister_netdevice(ser->dev); debugfs_deinit(ser); + tty_kref_put(tty); } rtnl_unlock(); } @@ -355,8 +358,6 @@ static void ldisc_close(struct tty_struct *tty) { struct ser_device *ser = tty->disc_data; - tty_kref_put(ser->tty); - spin_lock(&ser_lock); list_move(&ser->node, &ser_release_list); spin_unlock(&ser_lock); diff --git a/drivers/net/ethernet/amd/Kconfig b/drivers/net/ethernet/amd/Kconfig index d54dca3074eb66..45e8d698781c15 100644 --- a/drivers/net/ethernet/amd/Kconfig +++ b/drivers/net/ethernet/amd/Kconfig @@ -165,7 +165,7 @@ config AMD_XGBE select CRC32 select PHYLIB select AMD_XGBE_HAVE_ECC if X86 - select NET_SELFTESTS + imply NET_SELFTESTS help This driver supports the AMD 10GbE Ethernet device found on an AMD SoC. diff --git a/drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.c b/drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.c index 198f49b40dbf00..2994f10446a63c 100644 --- a/drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.c +++ b/drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.c @@ -442,7 +442,7 @@ __bnge_hwrm_reserve_pf_rings(struct bnge_dev *bd, struct bnge_hw_rings *hwr) struct hwrm_func_cfg_input *req; u32 enables = 0; - if (bnge_hwrm_req_init(bd, req, HWRM_FUNC_QCFG)) + if (bnge_hwrm_req_init(bd, req, HWRM_FUNC_CFG)) return NULL; req->fid = cpu_to_le16(0xffff); diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c index 7a9573dcab741b..e879b04e21b0cd 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c @@ -1048,13 +1048,13 @@ static void hns3_init_tx_spare_buffer(struct hns3_enet_ring *ring) int order; if (!alloc_size) - return; + goto not_init; order = get_order(alloc_size); if (order > MAX_PAGE_ORDER) { if (net_ratelimit()) dev_warn(ring_to_dev(ring), "failed to allocate tx spare buffer, exceed to max order\n"); - return; + goto not_init; } tx_spare = devm_kzalloc(ring_to_dev(ring), sizeof(*tx_spare), @@ -1092,6 +1092,13 @@ static void hns3_init_tx_spare_buffer(struct hns3_enet_ring *ring) devm_kfree(ring_to_dev(ring), tx_spare); devm_kzalloc_error: ring->tqp->handle->kinfo.tx_spare_buf_size = 0; +not_init: + /* When driver init or reset_init, the ring->tx_spare is always NULL; + * but when called from hns3_set_ringparam, it's usually not NULL, and + * will be restored if hns3_init_all_ring() failed. So it's safe to set + * ring->tx_spare to NULL here. + */ + ring->tx_spare = NULL; } /* Use hns3_tx_spare_space() to make sure there is enough buffer diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_cn9k_pf.c b/drivers/net/ethernet/marvell/octeon_ep/octep_cn9k_pf.c index b5805969404fa1..01e82d0b6b2cdd 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_cn9k_pf.c +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_cn9k_pf.c @@ -307,7 +307,7 @@ static void octep_setup_iq_regs_cn93_pf(struct octep_device *oct, int iq_no) } /* Setup registers for a hardware Rx Queue */ -static void octep_setup_oq_regs_cn93_pf(struct octep_device *oct, int oq_no) +static int octep_setup_oq_regs_cn93_pf(struct octep_device *oct, int oq_no) { u64 reg_val; u64 oq_ctl = 0ULL; @@ -355,6 +355,7 @@ static void octep_setup_oq_regs_cn93_pf(struct octep_device *oct, int oq_no) reg_val = ((u64)time_threshold << 32) | CFG_GET_OQ_INTR_PKT(oct->conf); octep_write_csr64(oct, CN93_SDP_R_OUT_INT_LEVELS(oq_no), reg_val); + return 0; } /* Setup registers for a PF mailbox */ @@ -696,14 +697,26 @@ static void octep_enable_interrupts_cn93_pf(struct octep_device *oct) /* Disable all interrupts */ static void octep_disable_interrupts_cn93_pf(struct octep_device *oct) { - u64 intr_mask = 0ULL; + u64 reg_val, intr_mask = 0ULL; int srn, num_rings, i; srn = CFG_GET_PORTS_PF_SRN(oct->conf); num_rings = CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); - for (i = 0; i < num_rings; i++) - intr_mask |= (0x1ULL << (srn + i)); + for (i = 0; i < num_rings; i++) { + intr_mask |= BIT_ULL(srn + i); + reg_val = octep_read_csr64(oct, + CN93_SDP_R_IN_INT_LEVELS(srn + i)); + reg_val &= ~CN93_INT_ENA_BIT; + octep_write_csr64(oct, + CN93_SDP_R_IN_INT_LEVELS(srn + i), reg_val); + + reg_val = octep_read_csr64(oct, + CN93_SDP_R_OUT_INT_LEVELS(srn + i)); + reg_val &= ~CN93_INT_ENA_BIT; + octep_write_csr64(oct, + CN93_SDP_R_OUT_INT_LEVELS(srn + i), reg_val); + } octep_write_csr64(oct, CN93_SDP_EPF_IRERR_RINT_ENA_W1C, intr_mask); octep_write_csr64(oct, CN93_SDP_EPF_ORERR_RINT_ENA_W1C, intr_mask); diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_cnxk_pf.c b/drivers/net/ethernet/marvell/octeon_ep/octep_cnxk_pf.c index 5de0b5ecbc5fd1..09a3f1d0645b85 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_cnxk_pf.c +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_cnxk_pf.c @@ -8,6 +8,7 @@ #include #include #include +#include #include "octep_config.h" #include "octep_main.h" @@ -327,12 +328,14 @@ static void octep_setup_iq_regs_cnxk_pf(struct octep_device *oct, int iq_no) } /* Setup registers for a hardware Rx Queue */ -static void octep_setup_oq_regs_cnxk_pf(struct octep_device *oct, int oq_no) +static int octep_setup_oq_regs_cnxk_pf(struct octep_device *oct, int oq_no) { - u64 reg_val; - u64 oq_ctl = 0ULL; - u32 time_threshold = 0; struct octep_oq *oq = oct->oq[oq_no]; + unsigned long t_out_jiffies; + u32 time_threshold = 0; + u64 oq_ctl = 0ULL; + u64 reg_ba_val; + u64 reg_val; oq_no += CFG_GET_PORTS_PF_SRN(oct->conf); reg_val = octep_read_csr64(oct, CNXK_SDP_R_OUT_CONTROL(oq_no)); @@ -343,6 +346,36 @@ static void octep_setup_oq_regs_cnxk_pf(struct octep_device *oct, int oq_no) reg_val = octep_read_csr64(oct, CNXK_SDP_R_OUT_CONTROL(oq_no)); } while (!(reg_val & CNXK_R_OUT_CTL_IDLE)); } + octep_write_csr64(oct, CNXK_SDP_R_OUT_WMARK(oq_no), oq->max_count); + /* Wait for WMARK to get applied */ + usleep_range(10, 15); + + octep_write_csr64(oct, CNXK_SDP_R_OUT_SLIST_BADDR(oq_no), + oq->desc_ring_dma); + octep_write_csr64(oct, CNXK_SDP_R_OUT_SLIST_RSIZE(oq_no), + oq->max_count); + reg_ba_val = octep_read_csr64(oct, CNXK_SDP_R_OUT_SLIST_BADDR(oq_no)); + + if (reg_ba_val != oq->desc_ring_dma) { + t_out_jiffies = jiffies + 10 * HZ; + do { + if (reg_ba_val == ULLONG_MAX) + return -EFAULT; + octep_write_csr64(oct, + CNXK_SDP_R_OUT_SLIST_BADDR(oq_no), + oq->desc_ring_dma); + octep_write_csr64(oct, + CNXK_SDP_R_OUT_SLIST_RSIZE(oq_no), + oq->max_count); + reg_ba_val = + octep_read_csr64(oct, + CNXK_SDP_R_OUT_SLIST_BADDR(oq_no)); + } while ((reg_ba_val != oq->desc_ring_dma) && + time_before(jiffies, t_out_jiffies)); + + if (reg_ba_val != oq->desc_ring_dma) + return -EAGAIN; + } reg_val &= ~(CNXK_R_OUT_CTL_IMODE); reg_val &= ~(CNXK_R_OUT_CTL_ROR_P); @@ -356,10 +389,6 @@ static void octep_setup_oq_regs_cnxk_pf(struct octep_device *oct, int oq_no) reg_val |= (CNXK_R_OUT_CTL_ES_P); octep_write_csr64(oct, CNXK_SDP_R_OUT_CONTROL(oq_no), reg_val); - octep_write_csr64(oct, CNXK_SDP_R_OUT_SLIST_BADDR(oq_no), - oq->desc_ring_dma); - octep_write_csr64(oct, CNXK_SDP_R_OUT_SLIST_RSIZE(oq_no), - oq->max_count); oq_ctl = octep_read_csr64(oct, CNXK_SDP_R_OUT_CONTROL(oq_no)); @@ -385,6 +414,7 @@ static void octep_setup_oq_regs_cnxk_pf(struct octep_device *oct, int oq_no) reg_val &= ~0xFFFFFFFFULL; reg_val |= CFG_GET_OQ_WMARK(oct->conf); octep_write_csr64(oct, CNXK_SDP_R_OUT_WMARK(oq_no), reg_val); + return 0; } /* Setup registers for a PF mailbox */ @@ -720,14 +750,26 @@ static void octep_enable_interrupts_cnxk_pf(struct octep_device *oct) /* Disable all interrupts */ static void octep_disable_interrupts_cnxk_pf(struct octep_device *oct) { - u64 intr_mask = 0ULL; + u64 reg_val, intr_mask = 0ULL; int srn, num_rings, i; srn = CFG_GET_PORTS_PF_SRN(oct->conf); num_rings = CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); - for (i = 0; i < num_rings; i++) - intr_mask |= (0x1ULL << (srn + i)); + for (i = 0; i < num_rings; i++) { + intr_mask |= BIT_ULL(srn + i); + reg_val = octep_read_csr64(oct, + CNXK_SDP_R_IN_INT_LEVELS(srn + i)); + reg_val &= ~CNXK_INT_ENA_BIT; + octep_write_csr64(oct, + CNXK_SDP_R_IN_INT_LEVELS(srn + i), reg_val); + + reg_val = octep_read_csr64(oct, + CNXK_SDP_R_OUT_INT_LEVELS(srn + i)); + reg_val &= ~CNXK_INT_ENA_BIT; + octep_write_csr64(oct, + CNXK_SDP_R_OUT_INT_LEVELS(srn + i), reg_val); + } octep_write_csr64(oct, CNXK_SDP_EPF_IRERR_RINT_ENA_W1C, intr_mask); octep_write_csr64(oct, CNXK_SDP_EPF_ORERR_RINT_ENA_W1C, intr_mask); diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_main.h b/drivers/net/ethernet/marvell/octeon_ep/octep_main.h index 81ac4267811c81..35d0ff289a70df 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_main.h +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_main.h @@ -77,7 +77,7 @@ struct octep_pci_win_regs { struct octep_hw_ops { void (*setup_iq_regs)(struct octep_device *oct, int q); - void (*setup_oq_regs)(struct octep_device *oct, int q); + int (*setup_oq_regs)(struct octep_device *oct, int q); void (*setup_mbox_regs)(struct octep_device *oct, int mbox); irqreturn_t (*mbox_intr_handler)(void *ioq_vector); diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cn9k_pf.h b/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cn9k_pf.h index ca473502d7a02a..95f1dfff90cce4 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cn9k_pf.h +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cn9k_pf.h @@ -386,5 +386,6 @@ #define CN93_PEM_BAR4_INDEX 7 #define CN93_PEM_BAR4_INDEX_SIZE 0x400000ULL #define CN93_PEM_BAR4_INDEX_OFFSET (CN93_PEM_BAR4_INDEX * CN93_PEM_BAR4_INDEX_SIZE) +#define CN93_INT_ENA_BIT BIT_ULL(62) #endif /* _OCTEP_REGS_CN9K_PF_H_ */ diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cnxk_pf.h b/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cnxk_pf.h index e637d7c8224d49..4d172a552f80c5 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cnxk_pf.h +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cnxk_pf.h @@ -412,5 +412,6 @@ #define CNXK_PEM_BAR4_INDEX 7 #define CNXK_PEM_BAR4_INDEX_SIZE 0x400000ULL #define CNXK_PEM_BAR4_INDEX_OFFSET (CNXK_PEM_BAR4_INDEX * CNXK_PEM_BAR4_INDEX_SIZE) +#define CNXK_INT_ENA_BIT BIT_ULL(62) #endif /* _OCTEP_REGS_CNXK_PF_H_ */ diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c b/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c index 82b6b19e76b47a..f2a7c6a76c742a 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c @@ -12,6 +12,8 @@ #include "octep_config.h" #include "octep_main.h" +static void octep_oq_free_ring_buffers(struct octep_oq *oq); + static void octep_oq_reset_indices(struct octep_oq *oq) { oq->host_read_idx = 0; @@ -170,11 +172,15 @@ static int octep_setup_oq(struct octep_device *oct, int q_no) goto oq_fill_buff_err; octep_oq_reset_indices(oq); - oct->hw_ops.setup_oq_regs(oct, q_no); + if (oct->hw_ops.setup_oq_regs(oct, q_no)) + goto oq_setup_err; + oct->num_oqs++; return 0; +oq_setup_err: + octep_oq_free_ring_buffers(oq); oq_fill_buff_err: vfree(oq->buff_info); oq->buff_info = NULL; diff --git a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cn9k.c b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cn9k.c index 88937fce75f140..4c769b27c27892 100644 --- a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cn9k.c +++ b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cn9k.c @@ -196,7 +196,7 @@ static void octep_vf_setup_iq_regs_cn93(struct octep_vf_device *oct, int iq_no) } /* Setup registers for a hardware Rx Queue */ -static void octep_vf_setup_oq_regs_cn93(struct octep_vf_device *oct, int oq_no) +static int octep_vf_setup_oq_regs_cn93(struct octep_vf_device *oct, int oq_no) { struct octep_vf_oq *oq = oct->oq[oq_no]; u32 time_threshold = 0; @@ -239,6 +239,7 @@ static void octep_vf_setup_oq_regs_cn93(struct octep_vf_device *oct, int oq_no) time_threshold = CFG_GET_OQ_INTR_TIME(oct->conf); reg_val = ((u64)time_threshold << 32) | CFG_GET_OQ_INTR_PKT(oct->conf); octep_vf_write_csr64(oct, CN93_VF_SDP_R_OUT_INT_LEVELS(oq_no), reg_val); + return 0; } /* Setup registers for a VF mailbox */ diff --git a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cnxk.c b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cnxk.c index 1f79dfad42c626..a968b93a67943b 100644 --- a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cnxk.c +++ b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cnxk.c @@ -199,11 +199,13 @@ static void octep_vf_setup_iq_regs_cnxk(struct octep_vf_device *oct, int iq_no) } /* Setup registers for a hardware Rx Queue */ -static void octep_vf_setup_oq_regs_cnxk(struct octep_vf_device *oct, int oq_no) +static int octep_vf_setup_oq_regs_cnxk(struct octep_vf_device *oct, int oq_no) { struct octep_vf_oq *oq = oct->oq[oq_no]; + unsigned long t_out_jiffies; u32 time_threshold = 0; u64 oq_ctl = ULL(0); + u64 reg_ba_val; u64 reg_val; reg_val = octep_vf_read_csr64(oct, CNXK_VF_SDP_R_OUT_CONTROL(oq_no)); @@ -214,6 +216,38 @@ static void octep_vf_setup_oq_regs_cnxk(struct octep_vf_device *oct, int oq_no) reg_val = octep_vf_read_csr64(oct, CNXK_VF_SDP_R_OUT_CONTROL(oq_no)); } while (!(reg_val & CNXK_VF_R_OUT_CTL_IDLE)); } + octep_vf_write_csr64(oct, CNXK_VF_SDP_R_OUT_WMARK(oq_no), + oq->max_count); + /* Wait for WMARK to get applied */ + usleep_range(10, 15); + + octep_vf_write_csr64(oct, CNXK_VF_SDP_R_OUT_SLIST_BADDR(oq_no), + oq->desc_ring_dma); + octep_vf_write_csr64(oct, CNXK_VF_SDP_R_OUT_SLIST_RSIZE(oq_no), + oq->max_count); + reg_ba_val = octep_vf_read_csr64(oct, + CNXK_VF_SDP_R_OUT_SLIST_BADDR(oq_no)); + if (reg_ba_val != oq->desc_ring_dma) { + t_out_jiffies = jiffies + 10 * HZ; + do { + if (reg_ba_val == ULLONG_MAX) + return -EFAULT; + octep_vf_write_csr64(oct, + CNXK_VF_SDP_R_OUT_SLIST_BADDR + (oq_no), oq->desc_ring_dma); + octep_vf_write_csr64(oct, + CNXK_VF_SDP_R_OUT_SLIST_RSIZE + (oq_no), oq->max_count); + reg_ba_val = + octep_vf_read_csr64(oct, + CNXK_VF_SDP_R_OUT_SLIST_BADDR + (oq_no)); + } while ((reg_ba_val != oq->desc_ring_dma) && + time_before(jiffies, t_out_jiffies)); + + if (reg_ba_val != oq->desc_ring_dma) + return -EAGAIN; + } reg_val &= ~(CNXK_VF_R_OUT_CTL_IMODE); reg_val &= ~(CNXK_VF_R_OUT_CTL_ROR_P); @@ -227,8 +261,6 @@ static void octep_vf_setup_oq_regs_cnxk(struct octep_vf_device *oct, int oq_no) reg_val |= (CNXK_VF_R_OUT_CTL_ES_P); octep_vf_write_csr64(oct, CNXK_VF_SDP_R_OUT_CONTROL(oq_no), reg_val); - octep_vf_write_csr64(oct, CNXK_VF_SDP_R_OUT_SLIST_BADDR(oq_no), oq->desc_ring_dma); - octep_vf_write_csr64(oct, CNXK_VF_SDP_R_OUT_SLIST_RSIZE(oq_no), oq->max_count); oq_ctl = octep_vf_read_csr64(oct, CNXK_VF_SDP_R_OUT_CONTROL(oq_no)); /* Clear the ISIZE and BSIZE (22-0) */ @@ -250,6 +282,7 @@ static void octep_vf_setup_oq_regs_cnxk(struct octep_vf_device *oct, int oq_no) reg_val &= ~GENMASK_ULL(31, 0); reg_val |= CFG_GET_OQ_WMARK(oct->conf); octep_vf_write_csr64(oct, CNXK_VF_SDP_R_OUT_WMARK(oq_no), reg_val); + return 0; } /* Setup registers for a VF mailbox */ diff --git a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_main.h b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_main.h index b9f13506f46205..c74cd2369e90d4 100644 --- a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_main.h +++ b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_main.h @@ -55,7 +55,7 @@ struct octep_vf_mmio { struct octep_vf_hw_ops { void (*setup_iq_regs)(struct octep_vf_device *oct, int q); - void (*setup_oq_regs)(struct octep_vf_device *oct, int q); + int (*setup_oq_regs)(struct octep_vf_device *oct, int q); void (*setup_mbox_regs)(struct octep_vf_device *oct, int mbox); irqreturn_t (*non_ioq_intr_handler)(void *ioq_vector); diff --git a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_rx.c b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_rx.c index d70c8be3cfc40b..6f865dbbba6c67 100644 --- a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_rx.c +++ b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_rx.c @@ -12,6 +12,8 @@ #include "octep_vf_config.h" #include "octep_vf_main.h" +static void octep_vf_oq_free_ring_buffers(struct octep_vf_oq *oq); + static void octep_vf_oq_reset_indices(struct octep_vf_oq *oq) { oq->host_read_idx = 0; @@ -171,11 +173,15 @@ static int octep_vf_setup_oq(struct octep_vf_device *oct, int q_no) goto oq_fill_buff_err; octep_vf_oq_reset_indices(oq); - oct->hw_ops.setup_oq_regs(oct, q_no); + if (oct->hw_ops.setup_oq_regs(oct, q_no)) + goto oq_setup_err; + oct->num_oqs++; return 0; +oq_setup_err: + octep_vf_oq_free_ring_buffers(oq); oq_fill_buff_err: vfree(oq->buff_info); oq->buff_info = NULL; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c index 747fbdf2a908f1..8530df8b3fdaf7 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c @@ -3632,11 +3632,22 @@ static void rvu_remove(struct pci_dev *pdev) devm_kfree(&pdev->dev, rvu); } +static void rvu_shutdown(struct pci_dev *pdev) +{ + struct rvu *rvu = pci_get_drvdata(pdev); + + if (!rvu) + return; + + rvu_clear_rvum_blk_revid(rvu); +} + static struct pci_driver rvu_driver = { .name = DRV_NAME, .id_table = rvu_id_table, .probe = rvu_probe, .remove = rvu_remove, + .shutdown = rvu_shutdown, }; static int __init rvu_init_module(void) diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c index c7c70429eb6c10..8658cb2143dfc7 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c @@ -1042,32 +1042,35 @@ void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, rvu_write64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_ACTION(index, bank), *(u64 *)&action); - /* update the VF flow rule action with the VF default entry action */ - if (mcam_index < 0) - npc_update_vf_flow_entry(rvu, mcam, blkaddr, pcifunc, - *(u64 *)&action); - /* update the action change in default rule */ pfvf = rvu_get_pfvf(rvu, pcifunc); if (pfvf->def_ucast_rule) pfvf->def_ucast_rule->rx_action = action; - index = npc_get_nixlf_mcam_index(mcam, pcifunc, - nixlf, NIXLF_PROMISC_ENTRY); + if (mcam_index < 0) { + /* update the VF flow rule action with the VF default + * entry action + */ + npc_update_vf_flow_entry(rvu, mcam, blkaddr, pcifunc, + *(u64 *)&action); - /* If PF's promiscuous entry is enabled, - * Set RSS action for that entry as well - */ - npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index, blkaddr, - alg_idx); + index = npc_get_nixlf_mcam_index(mcam, pcifunc, + nixlf, NIXLF_PROMISC_ENTRY); - index = npc_get_nixlf_mcam_index(mcam, pcifunc, - nixlf, NIXLF_ALLMULTI_ENTRY); - /* If PF's allmulti entry is enabled, - * Set RSS action for that entry as well - */ - npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index, blkaddr, - alg_idx); + /* If PF's promiscuous entry is enabled, + * Set RSS action for that entry as well + */ + npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index, + blkaddr, alg_idx); + + index = npc_get_nixlf_mcam_index(mcam, pcifunc, + nixlf, NIXLF_ALLMULTI_ENTRY); + /* If PF's allmulti entry is enabled, + * Set RSS action for that entry as well + */ + npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index, + blkaddr, alg_idx); + } } void npc_enadis_default_mce_entry(struct rvu *rvu, u16 pcifunc, diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c index 6b2d8559f0eb1b..444bb67494ab70 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c @@ -3315,6 +3315,7 @@ static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id) err_sriov_cleannup: otx2_sriov_vfcfg_cleanup(pf); err_pf_sriov_init: + otx2_unregister_dl(pf); otx2_shutdown_tc(pf); err_mcam_flow_del: otx2_mcam_flow_del(pf); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index ff4ab4691baf0b..a06d08576fd4b8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -179,7 +179,8 @@ static inline u16 mlx5_min_rx_wqes(int wq_type, u32 wq_size) } /* Use this function to get max num channels (rxqs/txqs) only to create netdev */ -static inline int mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev) +static inline unsigned int +mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev) { return is_kdump_kernel() ? MLX5E_MIN_NUM_CHANNELS : diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c index 424f8a2728a3ef..74660e7fe6748b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c @@ -457,22 +457,8 @@ static void mlx5e_ptpsq_unhealthy_work(struct work_struct *work) { struct mlx5e_ptpsq *ptpsq = container_of(work, struct mlx5e_ptpsq, report_unhealthy_work); - struct mlx5e_txqsq *sq = &ptpsq->txqsq; - - /* Recovering the PTP SQ means re-enabling NAPI, which requires the - * netdev instance lock. However, SQ closing has to wait for this work - * task to finish while also holding the same lock. So either get the - * lock or find that the SQ is no longer enabled and thus this work is - * not relevant anymore. - */ - while (!netdev_trylock(sq->netdev)) { - if (!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)) - return; - msleep(20); - } mlx5e_reporter_tx_ptpsq_unhealthy(ptpsq); - netdev_unlock(sq->netdev); } static int mlx5e_ptp_open_txqsq(struct mlx5e_ptp *c, u32 tisn, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c index 0686fbdd5a0599..6efb626b55062b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright (c) 2019 Mellanox Technologies. +#include + #include "health.h" #include "params.h" #include "txrx.h" @@ -177,6 +179,16 @@ static int mlx5e_rx_reporter_timeout_recover(void *ctx) rq = ctx; priv = rq->priv; + /* Acquire netdev instance lock to synchronize with channel close and + * reopen flows. Either successfully obtain the lock, or detect that + * channels are closing for another reason, making this work no longer + * necessary. + */ + while (!netdev_trylock(rq->netdev)) { + if (!test_bit(MLX5E_STATE_CHANNELS_ACTIVE, &rq->priv->state)) + return 0; + msleep(20); + } mutex_lock(&priv->state_lock); eq = rq->cq.mcq.eq; @@ -186,6 +198,7 @@ static int mlx5e_rx_reporter_timeout_recover(void *ctx) clear_bit(MLX5E_SQ_STATE_ENABLED, &rq->icosq->state); mutex_unlock(&priv->state_lock); + netdev_unlock(rq->netdev); return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c index 9e2cf191ed3086..9f6454102cf799 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c @@ -1,6 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (c) 2019 Mellanox Technologies. */ +#include + #include "health.h" #include "en/ptp.h" #include "en/devlink.h" @@ -78,6 +80,18 @@ static int mlx5e_tx_reporter_err_cqe_recover(void *ctx) if (!test_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state)) return 0; + /* Recovering queues means re-enabling NAPI, which requires the netdev + * instance lock. However, SQ closing flows have to wait for work tasks + * to finish while also holding the netdev instance lock. So either get + * the lock or find that the SQ is no longer enabled and thus this work + * is not relevant anymore. + */ + while (!netdev_trylock(dev)) { + if (!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)) + return 0; + msleep(20); + } + err = mlx5_core_query_sq_state(mdev, sq->sqn, &state); if (err) { netdev_err(dev, "Failed to query SQ 0x%x state. err = %d\n", @@ -113,9 +127,11 @@ static int mlx5e_tx_reporter_err_cqe_recover(void *ctx) else mlx5e_trigger_napi_sched(sq->cq.napi); + netdev_unlock(dev); return 0; out: clear_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state); + netdev_unlock(dev); return err; } @@ -136,10 +152,24 @@ static int mlx5e_tx_reporter_timeout_recover(void *ctx) sq = to_ctx->sq; eq = sq->cq.mcq.eq; priv = sq->priv; + + /* Recovering the TX queues implies re-enabling NAPI, which requires + * the netdev instance lock. + * However, channel closing flows have to wait for this work to finish + * while holding the same lock. So either get the lock or find that + * channels are being closed for other reason and this work is not + * relevant anymore. + */ + while (!netdev_trylock(sq->netdev)) { + if (!test_bit(MLX5E_STATE_CHANNELS_ACTIVE, &priv->state)) + return 0; + msleep(20); + } + err = mlx5e_health_channel_eq_recover(sq->netdev, eq, sq->cq.ch_stats); if (!err) { to_ctx->status = 0; /* this sq recovered */ - return err; + goto out; } mutex_lock(&priv->state_lock); @@ -147,7 +177,7 @@ static int mlx5e_tx_reporter_timeout_recover(void *ctx) mutex_unlock(&priv->state_lock); if (!err) { to_ctx->status = 1; /* all channels recovered */ - return err; + goto out; } to_ctx->status = err; @@ -155,7 +185,8 @@ static int mlx5e_tx_reporter_timeout_recover(void *ctx) netdev_err(priv->netdev, "mlx5e_safe_reopen_channels failed recovering from a tx_timeout, err(%d).\n", err); - +out: + netdev_unlock(sq->netdev); return err; } @@ -172,10 +203,22 @@ static int mlx5e_tx_reporter_ptpsq_unhealthy_recover(void *ctx) return 0; priv = ptpsq->txqsq.priv; + netdev = priv->netdev; + + /* Recovering the PTP SQ means re-enabling NAPI, which requires the + * netdev instance lock. However, SQ closing has to wait for this work + * task to finish while also holding the same lock. So either get the + * lock or find that the SQ is no longer enabled and thus this work is + * not relevant anymore. + */ + while (!netdev_trylock(netdev)) { + if (!test_bit(MLX5E_SQ_STATE_ENABLED, &ptpsq->txqsq.state)) + return 0; + msleep(20); + } mutex_lock(&priv->state_lock); chs = &priv->channels; - netdev = priv->netdev; carrier_ok = netif_carrier_ok(netdev); netif_carrier_off(netdev); @@ -192,6 +235,7 @@ static int mlx5e_tx_reporter_ptpsq_unhealthy_recover(void *ctx) netif_carrier_on(netdev); mutex_unlock(&priv->state_lock); + netdev_unlock(netdev); return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c index 7819fb2972802f..d5d9146efca673 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB // Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +#include #include #include "lib/aso.h" #include "en/tc/post_act.h" @@ -115,7 +116,6 @@ mlx5e_tc_meter_modify(struct mlx5_core_dev *mdev, struct mlx5e_flow_meters *flow_meters; u8 cir_man, cir_exp, cbs_man, cbs_exp; struct mlx5_aso_wqe *aso_wqe; - unsigned long expires; struct mlx5_aso *aso; u64 rate, burst; u8 ds_cnt; @@ -187,12 +187,8 @@ mlx5e_tc_meter_modify(struct mlx5_core_dev *mdev, mlx5_aso_post_wqe(aso, true, &aso_wqe->ctrl); /* With newer FW, the wait for the first ASO WQE is more than 2us, put the wait 10ms. */ - expires = jiffies + msecs_to_jiffies(10); - do { - err = mlx5_aso_poll_cq(aso, true); - if (err) - usleep_range(2, 10); - } while (err && time_is_after_jiffies(expires)); + read_poll_timeout(mlx5_aso_poll_cq, err, !err, 10, 10 * USEC_PER_MSEC, + false, aso, true); mutex_unlock(&flow_meters->aso_lock); return err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c index 528b04d4de416b..90b3bc5f9166fd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c @@ -5,6 +5,7 @@ #include #include #include +#include #include "en.h" #include "lib/aso.h" @@ -1385,7 +1386,8 @@ static int macsec_aso_set_arm_event(struct mlx5_core_dev *mdev, struct mlx5e_mac MLX5_ACCESS_ASO_OPC_MOD_MACSEC); macsec_aso_build_ctrl(aso, &aso_wqe->aso_ctrl, in); mlx5_aso_post_wqe(maso, false, &aso_wqe->ctrl); - err = mlx5_aso_poll_cq(maso, false); + read_poll_timeout(mlx5_aso_poll_cq, err, !err, 10, 10 * USEC_PER_MSEC, + false, maso, false); mutex_unlock(&aso->aso_lock); return err; @@ -1397,7 +1399,6 @@ static int macsec_aso_query(struct mlx5_core_dev *mdev, struct mlx5e_macsec *mac struct mlx5e_macsec_aso *aso; struct mlx5_aso_wqe *aso_wqe; struct mlx5_aso *maso; - unsigned long expires; int err; aso = &macsec->aso; @@ -1411,12 +1412,8 @@ static int macsec_aso_query(struct mlx5_core_dev *mdev, struct mlx5e_macsec *mac macsec_aso_build_wqe_ctrl_seg(aso, &aso_wqe->aso_ctrl, NULL); mlx5_aso_post_wqe(maso, false, &aso_wqe->ctrl); - expires = jiffies + msecs_to_jiffies(10); - do { - err = mlx5_aso_poll_cq(maso, false); - if (err) - usleep_range(2, 10); - } while (err && time_is_after_jiffies(expires)); + read_poll_timeout(mlx5_aso_poll_cq, err, !err, 10, 10 * USEC_PER_MSEC, + false, maso, false); if (err) goto err_out; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 4b2963bbe7ff45..e15e6fb4cd8ead 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -688,19 +688,7 @@ static void mlx5e_rq_timeout_work(struct work_struct *timeout_work) struct mlx5e_rq, rx_timeout_work); - /* Acquire netdev instance lock to synchronize with channel close and - * reopen flows. Either successfully obtain the lock, or detect that - * channels are closing for another reason, making this work no longer - * necessary. - */ - while (!netdev_trylock(rq->netdev)) { - if (!test_bit(MLX5E_STATE_CHANNELS_ACTIVE, &rq->priv->state)) - return; - msleep(20); - } - mlx5e_reporter_rx_timeout(rq); - netdev_unlock(rq->netdev); } static int mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq *rq) @@ -1997,20 +1985,7 @@ void mlx5e_tx_err_cqe_work(struct work_struct *recover_work) struct mlx5e_txqsq *sq = container_of(recover_work, struct mlx5e_txqsq, recover_work); - /* Recovering queues means re-enabling NAPI, which requires the netdev - * instance lock. However, SQ closing flows have to wait for work tasks - * to finish while also holding the netdev instance lock. So either get - * the lock or find that the SQ is no longer enabled and thus this work - * is not relevant anymore. - */ - while (!netdev_trylock(sq->netdev)) { - if (!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)) - return; - msleep(20); - } - mlx5e_reporter_tx_err_cqe(sq); - netdev_unlock(sq->netdev); } static struct dim_cq_moder mlx5e_get_def_tx_moderation(u8 cq_period_mode) @@ -5121,19 +5096,6 @@ static void mlx5e_tx_timeout_work(struct work_struct *work) struct net_device *netdev = priv->netdev; int i; - /* Recovering the TX queues implies re-enabling NAPI, which requires - * the netdev instance lock. - * However, channel closing flows have to wait for this work to finish - * while holding the same lock. So either get the lock or find that - * channels are being closed for other reason and this work is not - * relevant anymore. - */ - while (!netdev_trylock(netdev)) { - if (!test_bit(MLX5E_STATE_CHANNELS_ACTIVE, &priv->state)) - return; - msleep(20); - } - for (i = 0; i < netdev->real_num_tx_queues; i++) { struct netdev_queue *dev_queue = netdev_get_tx_queue(netdev, i); @@ -5146,8 +5108,6 @@ static void mlx5e_tx_timeout_work(struct work_struct *work) /* break if tried to reopened channels */ break; } - - netdev_unlock(netdev); } static void mlx5e_tx_timeout(struct net_device *dev, unsigned int txqueue) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/wc.c b/drivers/net/ethernet/mellanox/mlx5/core/wc.c index 815a7c97d6b09e..04d03be1bb775d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/wc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/wc.c @@ -2,6 +2,7 @@ // Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. #include +#include #include #include "lib/clock.h" #include "mlx5_core.h" @@ -15,7 +16,7 @@ #define TEST_WC_NUM_WQES 255 #define TEST_WC_LOG_CQ_SZ (order_base_2(TEST_WC_NUM_WQES)) #define TEST_WC_SQ_LOG_WQ_SZ TEST_WC_LOG_CQ_SZ -#define TEST_WC_POLLING_MAX_TIME_JIFFIES msecs_to_jiffies(100) +#define TEST_WC_POLLING_MAX_TIME_USEC (100 * USEC_PER_MSEC) struct mlx5_wc_cq { /* data path - accessed per cqe */ @@ -359,7 +360,6 @@ static int mlx5_wc_poll_cq(struct mlx5_wc_sq *sq) static void mlx5_core_test_wc(struct mlx5_core_dev *mdev) { unsigned int offset = 0; - unsigned long expires; struct mlx5_wc_sq *sq; int i, err; @@ -389,13 +389,9 @@ static void mlx5_core_test_wc(struct mlx5_core_dev *mdev) mlx5_wc_post_nop(sq, &offset, true); - expires = jiffies + TEST_WC_POLLING_MAX_TIME_JIFFIES; - do { - err = mlx5_wc_poll_cq(sq); - if (err) - usleep_range(2, 10); - } while (mdev->wc_state == MLX5_WC_STATE_UNINITIALIZED && - time_is_after_jiffies(expires)); + poll_timeout_us(mlx5_wc_poll_cq(sq), + mdev->wc_state != MLX5_WC_STATE_UNINITIALIZED, 10, + TEST_WC_POLLING_MAX_TIME_USEC, false); mlx5_wc_destroy_sq(sq); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c index 693ebdf3870552..5edc28ba295531 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c @@ -1142,6 +1142,9 @@ static int fbnic_set_cls_rule_ins(struct fbnic_net *fbn, return -EINVAL; } + dest |= FIELD_PREP(FBNIC_RPC_ACT_TBL0_DMA_HINT, + FBNIC_RCD_HDR_AL_DMA_HINT_L4); + /* Write action table values */ act_tcam->dest = dest; act_tcam->rss_en_mask = fbnic_flow_hash_2_rss_en_mask(fbn, hash_idx); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw_log.c b/drivers/net/ethernet/meta/fbnic/fbnic_fw_log.c index 85a883dba385fe..d8a9a7d7c2375f 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_fw_log.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw_log.c @@ -51,8 +51,6 @@ int fbnic_fw_log_init(struct fbnic_dev *fbd) log->data_start = data; log->data_end = data + FBNIC_FW_LOG_SIZE; - fbnic_fw_log_enable(fbd, true); - return 0; } @@ -63,7 +61,6 @@ void fbnic_fw_log_free(struct fbnic_dev *fbd) if (!fbnic_fw_log_ready(fbd)) return; - fbnic_fw_log_disable(fbd); INIT_LIST_HEAD(&log->entries); log->size = 0; vfree(log->data_start); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c index 81c9d5c9a4b2c0..b4b396ca9bce3c 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c @@ -262,6 +262,23 @@ static int fbnic_set_mac(struct net_device *netdev, void *p) return 0; } +static int fbnic_change_mtu(struct net_device *dev, int new_mtu) +{ + struct fbnic_net *fbn = netdev_priv(dev); + + if (fbnic_check_split_frames(fbn->xdp_prog, new_mtu, fbn->hds_thresh)) { + dev_err(&dev->dev, + "MTU %d is larger than HDS threshold %d in XDP mode\n", + new_mtu, fbn->hds_thresh); + + return -EINVAL; + } + + WRITE_ONCE(dev->mtu, new_mtu); + + return 0; +} + void fbnic_clear_rx_mode(struct fbnic_dev *fbd) { struct net_device *netdev = fbd->netdev; @@ -533,6 +550,7 @@ static const struct net_device_ops fbnic_netdev_ops = { .ndo_start_xmit = fbnic_xmit_frame, .ndo_features_check = fbnic_features_check, .ndo_set_mac_address = fbnic_set_mac, + .ndo_change_mtu = fbnic_change_mtu, .ndo_set_rx_mode = fbnic_set_rx_mode, .ndo_get_stats64 = fbnic_get_stats64, .ndo_bpf = fbnic_bpf, @@ -787,6 +805,8 @@ struct net_device *fbnic_netdev_alloc(struct fbnic_dev *fbd) netdev->hw_enc_features |= netdev->features; netdev->features |= NETIF_F_NTUPLE; + netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_RX_SG; + netdev->min_mtu = IPV6_MIN_MTU; netdev->max_mtu = FBNIC_MAX_JUMBO_FRAME_SIZE - ETH_HLEN; diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c index 9240673c7533d2..e92187bc1c0fa4 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c @@ -307,11 +307,17 @@ static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto free_irqs; } + err = fbnic_fw_log_init(fbd); + if (err) + dev_warn(fbd->dev, + "Unable to initialize firmware log buffer: %d\n", + err); + err = fbnic_fw_request_mbx(fbd); if (err) { dev_err(&pdev->dev, "Firmware mailbox initialization failure\n"); - goto free_irqs; + goto free_fw_log; } /* Send the request to enable the FW logging to host. Note if this @@ -319,11 +325,7 @@ static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) * possible the FW is just too old to support the logging and needs * to be updated. */ - err = fbnic_fw_log_init(fbd); - if (err) - dev_warn(fbd->dev, - "Unable to initialize firmware log buffer: %d\n", - err); + fbnic_fw_log_enable(fbd, true); fbnic_devlink_register(fbd); fbnic_devlink_otp_check(fbd, "error detected during probe"); @@ -370,6 +372,8 @@ static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) * firmware updates for fixes. */ return 0; +free_fw_log: + fbnic_fw_log_free(fbd); free_irqs: fbnic_free_irqs(fbd); err_destroy_health: @@ -404,8 +408,9 @@ static void fbnic_remove(struct pci_dev *pdev) fbnic_hwmon_unregister(fbd); fbnic_dbg_fbd_exit(fbd); fbnic_devlink_unregister(fbd); - fbnic_fw_log_free(fbd); + fbnic_fw_log_disable(fbd); fbnic_fw_free_mbx(fbd); + fbnic_fw_log_free(fbd); fbnic_free_irqs(fbd); fbnic_devlink_health_destroy(fbd); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c b/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c index 7f31e890031c06..42a186db43ea93 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c @@ -338,9 +338,8 @@ void fbnic_rss_reinit(struct fbnic_dev *fbd, struct fbnic_net *fbn) else if (tstamp_mask & (1u << flow_type)) dest |= FBNIC_RPC_ACT_TBL0_TS_ENA; - if (act1_value[flow_type] & FBNIC_RPC_TCAM_ACT1_L4_VALID) - dest |= FIELD_PREP(FBNIC_RPC_ACT_TBL0_DMA_HINT, - FBNIC_RCD_HDR_AL_DMA_HINT_L4); + dest |= FIELD_PREP(FBNIC_RPC_ACT_TBL0_DMA_HINT, + FBNIC_RCD_HDR_AL_DMA_HINT_L4); rss_en_mask = fbnic_flow_hash_2_rss_en_mask(fbn, flow_type); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c index 13d508ce637f10..e119526fce14c3 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c @@ -2575,7 +2575,8 @@ static void fbnic_enable_bdq(struct fbnic_ring *hpq, struct fbnic_ring *ppq) } static void fbnic_config_drop_mode_rcq(struct fbnic_napi_vector *nv, - struct fbnic_ring *rcq, bool tx_pause) + struct fbnic_ring *rcq, bool tx_pause, + bool hdr_split) { struct fbnic_net *fbn = netdev_priv(nv->napi.dev); u32 drop_mode, rcq_ctl; @@ -2588,22 +2589,26 @@ static void fbnic_config_drop_mode_rcq(struct fbnic_napi_vector *nv, /* Specify packet layout */ rcq_ctl = FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_DROP_MODE_MASK, drop_mode) | FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_MIN_HROOM_MASK, FBNIC_RX_HROOM) | - FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_MIN_TROOM_MASK, FBNIC_RX_TROOM); + FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_MIN_TROOM_MASK, FBNIC_RX_TROOM) | + FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_EN_HDR_SPLIT, hdr_split); fbnic_ring_wr32(rcq, FBNIC_QUEUE_RDE_CTL0, rcq_ctl); } -void fbnic_config_drop_mode(struct fbnic_net *fbn, bool tx_pause) +void fbnic_config_drop_mode(struct fbnic_net *fbn, bool txp) { + bool hds; int i, t; + hds = fbn->hds_thresh < FBNIC_HDR_BYTES_MIN; + for (i = 0; i < fbn->num_napi; i++) { struct fbnic_napi_vector *nv = fbn->napi[i]; for (t = 0; t < nv->rxt_count; t++) { struct fbnic_q_triad *qt = &nv->qt[nv->txt_count + t]; - fbnic_config_drop_mode_rcq(nv, &qt->cmpl, tx_pause); + fbnic_config_drop_mode_rcq(nv, &qt->cmpl, txp, hds); } } } @@ -2654,20 +2659,18 @@ static void fbnic_enable_rcq(struct fbnic_napi_vector *nv, { struct fbnic_net *fbn = netdev_priv(nv->napi.dev); u32 log_size = fls(rcq->size_mask); - u32 hds_thresh = fbn->hds_thresh; u32 rcq_ctl = 0; - - fbnic_config_drop_mode_rcq(nv, rcq, fbn->tx_pause); + bool hdr_split; + u32 hds_thresh; /* Force lower bound on MAX_HEADER_BYTES. Below this, all frames should * be split at L4. It would also result in the frames being split at * L2/L3 depending on the frame size. */ - if (fbn->hds_thresh < FBNIC_HDR_BYTES_MIN) { - rcq_ctl = FBNIC_QUEUE_RDE_CTL0_EN_HDR_SPLIT; - hds_thresh = FBNIC_HDR_BYTES_MIN; - } + hdr_split = fbn->hds_thresh < FBNIC_HDR_BYTES_MIN; + fbnic_config_drop_mode_rcq(nv, rcq, fbn->tx_pause, hdr_split); + hds_thresh = max(fbn->hds_thresh, FBNIC_HDR_BYTES_MIN); rcq_ctl |= FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_PADLEN_MASK, FBNIC_RX_PAD) | FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_MAX_HDR_MASK, hds_thresh) | FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_PAYLD_OFF_MASK, diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h index 27776e844e29bf..51a98f27d5d912 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h @@ -66,7 +66,7 @@ struct fbnic_net; (4096 - FBNIC_RX_HROOM - FBNIC_RX_TROOM - FBNIC_RX_PAD) #define FBNIC_HDS_THRESH_DEFAULT \ (1536 - FBNIC_RX_PAD) -#define FBNIC_HDR_BYTES_MIN 128 +#define FBNIC_HDR_BYTES_MIN 256 struct fbnic_pkt_buff { struct xdp_buff buff; diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c b/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c index 2f168700f63c11..8b2e07821a9505 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c @@ -576,7 +576,7 @@ static int sparx5_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) static struct ptp_clock_info sparx5_ptp_clock_info = { .owner = THIS_MODULE, .name = "sparx5 ptp", - .max_adj = 200000, + .max_adj = 10000000, .gettime64 = sparx5_ptp_gettime64, .settime64 = sparx5_ptp_settime64, .adjtime = sparx5_ptp_adjtime, diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_qos.h b/drivers/net/ethernet/microchip/sparx5/sparx5_qos.h index 1231a80335d7b1..04f76f1e23f60a 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_qos.h +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_qos.h @@ -35,7 +35,7 @@ #define SPX5_SE_BURST_UNIT 4096 /* Dwrr */ -#define SPX5_DWRR_COST_MAX 63 +#define SPX5_DWRR_COST_MAX 31 struct sparx5_shaper { u32 mode; diff --git a/drivers/net/ethernet/mscc/ocelot_net.c b/drivers/net/ethernet/mscc/ocelot_net.c index 469784d3a1a67b..1b82693204640d 100644 --- a/drivers/net/ethernet/mscc/ocelot_net.c +++ b/drivers/net/ethernet/mscc/ocelot_net.c @@ -551,44 +551,81 @@ static int ocelot_port_stop(struct net_device *dev) return 0; } -static netdev_tx_t ocelot_port_xmit(struct sk_buff *skb, struct net_device *dev) +static bool ocelot_xmit_timestamp(struct ocelot *ocelot, int port, + struct sk_buff *skb, u32 *rew_op) { - struct ocelot_port_private *priv = netdev_priv(dev); - struct ocelot_port *ocelot_port = &priv->port; - struct ocelot *ocelot = ocelot_port->ocelot; - int port = priv->port.index; - u32 rew_op = 0; - - if (!static_branch_unlikely(&ocelot_fdma_enabled) && - !ocelot_can_inject(ocelot, 0)) - return NETDEV_TX_BUSY; - - /* Check if timestamping is needed */ if (ocelot->ptp && (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { struct sk_buff *clone = NULL; if (ocelot_port_txtstamp_request(ocelot, port, skb, &clone)) { kfree_skb(skb); - return NETDEV_TX_OK; + return false; } if (clone) OCELOT_SKB_CB(skb)->clone = clone; - rew_op = ocelot_ptp_rew_op(skb); + *rew_op = ocelot_ptp_rew_op(skb); } - if (static_branch_unlikely(&ocelot_fdma_enabled)) { - ocelot_fdma_inject_frame(ocelot, port, rew_op, skb, dev); - } else { - ocelot_port_inject_frame(ocelot, port, 0, rew_op, skb); + return true; +} + +static netdev_tx_t ocelot_port_xmit_fdma(struct sk_buff *skb, + struct net_device *dev) +{ + struct ocelot_port_private *priv = netdev_priv(dev); + struct ocelot_port *ocelot_port = &priv->port; + struct ocelot *ocelot = ocelot_port->ocelot; + int port = priv->port.index; + u32 rew_op = 0; + + if (!ocelot_xmit_timestamp(ocelot, port, skb, &rew_op)) + return NETDEV_TX_OK; + + ocelot_fdma_inject_frame(ocelot, port, rew_op, skb, dev); + + return NETDEV_TX_OK; +} - consume_skb(skb); +static netdev_tx_t ocelot_port_xmit_inj(struct sk_buff *skb, + struct net_device *dev) +{ + struct ocelot_port_private *priv = netdev_priv(dev); + struct ocelot_port *ocelot_port = &priv->port; + struct ocelot *ocelot = ocelot_port->ocelot; + int port = priv->port.index; + u32 rew_op = 0; + + ocelot_lock_inj_grp(ocelot, 0); + + if (!ocelot_can_inject(ocelot, 0)) { + ocelot_unlock_inj_grp(ocelot, 0); + return NETDEV_TX_BUSY; } + if (!ocelot_xmit_timestamp(ocelot, port, skb, &rew_op)) { + ocelot_unlock_inj_grp(ocelot, 0); + return NETDEV_TX_OK; + } + + ocelot_port_inject_frame(ocelot, port, 0, rew_op, skb); + + ocelot_unlock_inj_grp(ocelot, 0); + + consume_skb(skb); + return NETDEV_TX_OK; } +static netdev_tx_t ocelot_port_xmit(struct sk_buff *skb, struct net_device *dev) +{ + if (static_branch_unlikely(&ocelot_fdma_enabled)) + return ocelot_port_xmit_fdma(skb, dev); + + return ocelot_port_xmit_inj(skb, dev); +} + enum ocelot_action_type { OCELOT_MACT_LEARN, OCELOT_MACT_FORGET, diff --git a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c index 2d9efadb5d2ae1..347b0aff100b9f 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c @@ -263,9 +263,10 @@ static int ionic_get_link_ksettings(struct net_device *netdev, /* This means there's no module plugged in */ break; default: - dev_info(lif->ionic->dev, "unknown xcvr type pid=%d / 0x%x\n", - idev->port_info->status.xcvr.pid, - idev->port_info->status.xcvr.pid); + dev_dbg_ratelimited(lif->ionic->dev, + "unknown xcvr type pid=%d / 0x%x\n", + idev->port_info->status.xcvr.pid, + idev->port_info->status.xcvr.pid); break; } diff --git a/drivers/net/ethernet/renesas/rswitch_l2.c b/drivers/net/ethernet/renesas/rswitch_l2.c index 4a69ec77d69c60..9433cd8adced97 100644 --- a/drivers/net/ethernet/renesas/rswitch_l2.c +++ b/drivers/net/ethernet/renesas/rswitch_l2.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Renesas Ethernet Switch device driver * - * Copyright (C) 2025 Renesas Electronics Corporation + * Copyright (C) 2025 - 2026 Renesas Electronics Corporation */ #include @@ -60,6 +60,7 @@ static void rswitch_update_l2_hw_learning(struct rswitch_private *priv) static void rswitch_update_l2_hw_forwarding(struct rswitch_private *priv) { struct rswitch_device *rdev; + bool new_forwarding_offload; unsigned int fwd_mask; /* calculate fwd_mask with zeroes in bits corresponding to ports that @@ -73,8 +74,9 @@ static void rswitch_update_l2_hw_forwarding(struct rswitch_private *priv) } rswitch_for_all_ports(priv, rdev) { - if ((rdev_for_l2_offload(rdev) && rdev->forwarding_requested) || - rdev->forwarding_offloaded) { + new_forwarding_offload = (rdev_for_l2_offload(rdev) && rdev->forwarding_requested); + + if (new_forwarding_offload || rdev->forwarding_offloaded) { /* Update allowed offload destinations even for ports * with L2 offload enabled earlier. * @@ -84,13 +86,10 @@ static void rswitch_update_l2_hw_forwarding(struct rswitch_private *priv) priv->addr + FWPC2(rdev->port)); } - if (rdev_for_l2_offload(rdev) && - rdev->forwarding_requested && - !rdev->forwarding_offloaded) { + if (new_forwarding_offload && !rdev->forwarding_offloaded) rswitch_change_l2_hw_offloading(rdev, true, false); - } else if (rdev->forwarding_offloaded) { + else if (!new_forwarding_offload && rdev->forwarding_offloaded) rswitch_change_l2_hw_offloading(rdev, false, false); - } } } diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c index 107a7c84ace803..c05e3e7a539cfe 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c @@ -91,8 +91,8 @@ static void loongson_default_data(struct pci_dev *pdev, /* Get bus_id, this can be overwritten later */ plat->bus_id = pci_dev_id(pdev); - /* clk_csr_i = 20-35MHz & MDC = clk_csr_i/16 */ - plat->clk_csr = STMMAC_CSR_20_35M; + /* clk_csr_i = 100-150MHz & MDC = clk_csr_i/62 */ + plat->clk_csr = STMMAC_CSR_100_150M; plat->core_type = DWMAC_CORE_GMAC; plat->force_sf_dma_mode = 1; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index a379221b96a348..f98fd254315f6d 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -5023,13 +5023,27 @@ static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv, if (!priv->sph_active) return 0; - /* Not last descriptor */ - if (status & rx_not_ls) + /* For GMAC4, when split header is enabled, in some rare cases, the + * hardware does not fill buf2 of the first descriptor with payload. + * Thus we cannot assume buf2 is always fully filled if it is not + * the last descriptor. Otherwise, the length of buf2 of the second + * descriptor will be calculated wrong and cause an oops. + * + * If this is the last descriptor, 'plen' is the length of the + * received packet that was transferred to system memory. + * Otherwise, it is the accumulated number of bytes that have been + * transferred for the current packet. + * + * Thus 'plen - len' always gives the correct length of buf2. + */ + + /* Not GMAC4 and not last descriptor */ + if (priv->plat->core_type != DWMAC_CORE_GMAC4 && (status & rx_not_ls)) return priv->dma_conf.dma_buf_sz; + /* GMAC4 or last descriptor */ plen = stmmac_get_rx_frame_len(priv, p, coe); - /* Last descriptor */ return plen - len; } diff --git a/drivers/net/ethernet/sun/sunhme.c b/drivers/net/ethernet/sun/sunhme.c index 48f0a96c0e9e3a..6669980829980d 100644 --- a/drivers/net/ethernet/sun/sunhme.c +++ b/drivers/net/ethernet/sun/sunhme.c @@ -2551,6 +2551,9 @@ static int happy_meal_sbus_probe_one(struct platform_device *op, int is_qfe) goto err_out_clear_quattro; } + /* BIGMAC may have bogus sizes */ + if ((op->resource[3].end - op->resource[3].start) >= BMAC_REG_SIZE) + op->resource[3].end = op->resource[3].start + BMAC_REG_SIZE - 1; hp->bigmacregs = devm_platform_ioremap_resource(op, 3); if (IS_ERR(hp->bigmacregs)) { dev_err(&op->dev, "Cannot map BIGMAC registers.\n"); diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c index c509228be84d1b..4433b8e95b6acc 100644 --- a/drivers/net/macvlan.c +++ b/drivers/net/macvlan.c @@ -1572,6 +1572,11 @@ int macvlan_common_newlink(struct net_device *dev, if (create) macvlan_port_destroy(port->dev); } + /* @dev might have been made visible before an error was detected. + * Make sure to observe an RCU grace period before our caller + * (rtnl_newlink()) frees it. + */ + synchronize_net(); return err; } EXPORT_SYMBOL_GPL(macvlan_common_newlink); diff --git a/drivers/net/mctp/mctp-i2c.c b/drivers/net/mctp/mctp-i2c.c index f782d93f826efc..8043b57bdf2509 100644 --- a/drivers/net/mctp/mctp-i2c.c +++ b/drivers/net/mctp/mctp-i2c.c @@ -242,6 +242,12 @@ static int mctp_i2c_slave_cb(struct i2c_client *client, return 0; switch (event) { + case I2C_SLAVE_READ_REQUESTED: + case I2C_SLAVE_READ_PROCESSED: + /* MCTP I2C transport only uses writes */ + midev->rx_pos = 0; + *val = 0xff; + break; case I2C_SLAVE_WRITE_RECEIVED: if (midev->rx_pos < MCTP_I2C_BUFSZ) { midev->rx_buffer[midev->rx_pos] = *val; @@ -279,6 +285,9 @@ static int mctp_i2c_recv(struct mctp_i2c_dev *midev) size_t recvlen; int status; + if (midev->rx_pos == 0) + return 0; + /* + 1 for the PEC */ if (midev->rx_pos < MCTP_I2C_MINLEN + 1) { ndev->stats.rx_length_errors++; diff --git a/drivers/net/ovpn/io.c b/drivers/net/ovpn/io.c index 3e9e7f8444b344..955c9a37e1f8db 100644 --- a/drivers/net/ovpn/io.c +++ b/drivers/net/ovpn/io.c @@ -355,6 +355,7 @@ netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev) struct ovpn_priv *ovpn = netdev_priv(dev); struct sk_buff *segments, *curr, *next; struct sk_buff_head skb_list; + unsigned int tx_bytes = 0; struct ovpn_peer *peer; __be16 proto; int ret; @@ -365,7 +366,27 @@ netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev) /* verify IP header size in network packet */ proto = ovpn_ip_check_protocol(skb); if (unlikely(!proto || skb->protocol != proto)) - goto drop; + goto drop_no_peer; + + /* retrieve peer serving the destination IP of this packet */ + peer = ovpn_peer_get_by_dst(ovpn, skb); + if (unlikely(!peer)) { + switch (skb->protocol) { + case htons(ETH_P_IP): + net_dbg_ratelimited("%s: no peer to send data to dst=%pI4\n", + netdev_name(ovpn->dev), + &ip_hdr(skb)->daddr); + break; + case htons(ETH_P_IPV6): + net_dbg_ratelimited("%s: no peer to send data to dst=%pI6c\n", + netdev_name(ovpn->dev), + &ipv6_hdr(skb)->daddr); + break; + } + goto drop_no_peer; + } + /* dst was needed for peer selection - it can now be dropped */ + skb_dst_drop(skb); if (skb_is_gso(skb)) { segments = skb_gso_segment(skb, 0); @@ -394,36 +415,28 @@ netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev) continue; } + /* only count what we actually send */ + tx_bytes += curr->len; __skb_queue_tail(&skb_list, curr); } - skb_list.prev->next = NULL; - /* retrieve peer serving the destination IP of this packet */ - peer = ovpn_peer_get_by_dst(ovpn, skb); - if (unlikely(!peer)) { - switch (skb->protocol) { - case htons(ETH_P_IP): - net_dbg_ratelimited("%s: no peer to send data to dst=%pI4\n", - netdev_name(ovpn->dev), - &ip_hdr(skb)->daddr); - break; - case htons(ETH_P_IPV6): - net_dbg_ratelimited("%s: no peer to send data to dst=%pI6c\n", - netdev_name(ovpn->dev), - &ipv6_hdr(skb)->daddr); - break; - } - goto drop; + /* no segments survived: don't jump to 'drop' because we already + * incremented the counter for each failure in the loop + */ + if (unlikely(skb_queue_empty(&skb_list))) { + ovpn_peer_put(peer); + return NETDEV_TX_OK; } - /* dst was needed for peer selection - it can now be dropped */ - skb_dst_drop(skb); + skb_list.prev->next = NULL; - ovpn_peer_stats_increment_tx(&peer->vpn_stats, skb->len); + ovpn_peer_stats_increment_tx(&peer->vpn_stats, tx_bytes); ovpn_send(ovpn, skb_list.next, peer); return NETDEV_TX_OK; drop: + ovpn_peer_put(peer); +drop_no_peer: dev_dstats_tx_dropped(ovpn->dev); skb_tx_error(skb); kfree_skb_list(skb); diff --git a/drivers/net/ovpn/socket.c b/drivers/net/ovpn/socket.c index 9750871ab65cee..448cee3b3f9fa2 100644 --- a/drivers/net/ovpn/socket.c +++ b/drivers/net/ovpn/socket.c @@ -200,6 +200,22 @@ struct ovpn_socket *ovpn_socket_new(struct socket *sock, struct ovpn_peer *peer) ovpn_sock->sk = sk; kref_init(&ovpn_sock->refcount); + /* TCP sockets are per-peer, therefore they are linked to their unique + * peer + */ + if (sk->sk_protocol == IPPROTO_TCP) { + INIT_WORK(&ovpn_sock->tcp_tx_work, ovpn_tcp_tx_work); + ovpn_sock->peer = peer; + ovpn_peer_hold(peer); + } else if (sk->sk_protocol == IPPROTO_UDP) { + /* in UDP we only link the ovpn instance since the socket is + * shared among multiple peers + */ + ovpn_sock->ovpn = peer->ovpn; + netdev_hold(peer->ovpn->dev, &ovpn_sock->dev_tracker, + GFP_KERNEL); + } + /* the newly created ovpn_socket is holding reference to sk, * therefore we increase its refcounter. * @@ -212,29 +228,16 @@ struct ovpn_socket *ovpn_socket_new(struct socket *sock, struct ovpn_peer *peer) ret = ovpn_socket_attach(ovpn_sock, sock, peer); if (ret < 0) { + if (sk->sk_protocol == IPPROTO_TCP) + ovpn_peer_put(peer); + else if (sk->sk_protocol == IPPROTO_UDP) + netdev_put(peer->ovpn->dev, &ovpn_sock->dev_tracker); + sock_put(sk); kfree(ovpn_sock); ovpn_sock = ERR_PTR(ret); - goto sock_release; - } - - /* TCP sockets are per-peer, therefore they are linked to their unique - * peer - */ - if (sk->sk_protocol == IPPROTO_TCP) { - INIT_WORK(&ovpn_sock->tcp_tx_work, ovpn_tcp_tx_work); - ovpn_sock->peer = peer; - ovpn_peer_hold(peer); - } else if (sk->sk_protocol == IPPROTO_UDP) { - /* in UDP we only link the ovpn instance since the socket is - * shared among multiple peers - */ - ovpn_sock->ovpn = peer->ovpn; - netdev_hold(peer->ovpn->dev, &ovpn_sock->dev_tracker, - GFP_KERNEL); } - rcu_assign_sk_user_data(sk, ovpn_sock); sock_release: release_sock(sk); return ovpn_sock; diff --git a/drivers/net/ovpn/tcp.c b/drivers/net/ovpn/tcp.c index 0d7f30360d8746..ec2bbc28c19666 100644 --- a/drivers/net/ovpn/tcp.c +++ b/drivers/net/ovpn/tcp.c @@ -199,7 +199,19 @@ void ovpn_tcp_socket_detach(struct ovpn_socket *ovpn_sock) sk->sk_data_ready = peer->tcp.sk_cb.sk_data_ready; sk->sk_write_space = peer->tcp.sk_cb.sk_write_space; sk->sk_prot = peer->tcp.sk_cb.prot; - sk->sk_socket->ops = peer->tcp.sk_cb.ops; + + /* tcp_close() may race this function and could set + * sk->sk_socket to NULL. It does so by invoking + * sock_orphan(), which holds sk_callback_lock before + * doing the assignment. + * + * For this reason we acquire the same lock to avoid + * sk_socket to disappear under our feet + */ + write_lock_bh(&sk->sk_callback_lock); + if (sk->sk_socket) + sk->sk_socket->ops = peer->tcp.sk_cb.ops; + write_unlock_bh(&sk->sk_callback_lock); rcu_assign_sk_user_data(sk, NULL); } @@ -487,6 +499,7 @@ int ovpn_tcp_socket_attach(struct ovpn_socket *ovpn_sock, /* make sure no pre-existing encapsulation handler exists */ if (ovpn_sock->sk->sk_user_data) return -EBUSY; + rcu_assign_sk_user_data(ovpn_sock->sk, ovpn_sock); /* only a fully connected socket is expected. Connection should be * handled in userspace @@ -495,13 +508,14 @@ int ovpn_tcp_socket_attach(struct ovpn_socket *ovpn_sock, net_err_ratelimited("%s: provided TCP socket is not in ESTABLISHED state: %d\n", netdev_name(peer->ovpn->dev), ovpn_sock->sk->sk_state); - return -EINVAL; + ret = -EINVAL; + goto err; } ret = strp_init(&peer->tcp.strp, ovpn_sock->sk, &cb); if (ret < 0) { DEBUG_NET_WARN_ON_ONCE(1); - return ret; + goto err; } INIT_WORK(&peer->tcp.defer_del_work, ovpn_tcp_peer_del_work); @@ -536,6 +550,9 @@ int ovpn_tcp_socket_attach(struct ovpn_socket *ovpn_sock, strp_check_rcv(&peer->tcp.strp); return 0; +err: + rcu_assign_sk_user_data(ovpn_sock->sk, NULL); + return ret; } static void ovpn_tcp_close(struct sock *sk, long timeout) diff --git a/drivers/net/ovpn/udp.c b/drivers/net/ovpn/udp.c index d6a0f7a0b75d74..272b535ecaad4c 100644 --- a/drivers/net/ovpn/udp.c +++ b/drivers/net/ovpn/udp.c @@ -386,6 +386,7 @@ int ovpn_udp_socket_attach(struct ovpn_socket *ovpn_sock, struct socket *sock, struct ovpn_priv *ovpn) { struct udp_tunnel_sock_cfg cfg = { + .sk_user_data = ovpn_sock, .encap_type = UDP_ENCAP_OVPNINUDP, .encap_rcv = ovpn_udp_encap_recv, .encap_destroy = ovpn_udp_encap_destroy, diff --git a/drivers/net/usb/catc.c b/drivers/net/usb/catc.c index 6759388692f8e9..3c824340ffb068 100644 --- a/drivers/net/usb/catc.c +++ b/drivers/net/usb/catc.c @@ -64,6 +64,16 @@ static const char driver_name[] = "catc"; #define CTRL_QUEUE 16 /* Max control requests in flight (power of two) */ #define RX_PKT_SZ 1600 /* Max size of receive packet for F5U011 */ +/* + * USB endpoints. + */ + +enum catc_usb_ep { + CATC_USB_EP_CONTROL = 0, + CATC_USB_EP_BULK = 1, + CATC_USB_EP_INT_IN = 2, +}; + /* * Control requests. */ @@ -772,6 +782,13 @@ static int catc_probe(struct usb_interface *intf, const struct usb_device_id *id u8 broadcast[ETH_ALEN]; u8 *macbuf; int pktsz, ret = -ENOMEM; + static const u8 bulk_ep_addr[] = { + CATC_USB_EP_BULK | USB_DIR_OUT, + CATC_USB_EP_BULK | USB_DIR_IN, + 0}; + static const u8 int_ep_addr[] = { + CATC_USB_EP_INT_IN | USB_DIR_IN, + 0}; macbuf = kmalloc(ETH_ALEN, GFP_KERNEL); if (!macbuf) @@ -784,6 +801,14 @@ static int catc_probe(struct usb_interface *intf, const struct usb_device_id *id goto fail_mem; } + /* Verify that all required endpoints are present */ + if (!usb_check_bulk_endpoints(intf, bulk_ep_addr) || + !usb_check_int_endpoints(intf, int_ep_addr)) { + dev_err(dev, "Missing or invalid endpoints\n"); + ret = -ENODEV; + goto fail_mem; + } + netdev = alloc_etherdev(sizeof(struct catc)); if (!netdev) goto fail_mem; @@ -828,14 +853,14 @@ static int catc_probe(struct usb_interface *intf, const struct usb_device_id *id usb_fill_control_urb(catc->ctrl_urb, usbdev, usb_sndctrlpipe(usbdev, 0), NULL, NULL, 0, catc_ctrl_done, catc); - usb_fill_bulk_urb(catc->tx_urb, usbdev, usb_sndbulkpipe(usbdev, 1), - NULL, 0, catc_tx_done, catc); + usb_fill_bulk_urb(catc->tx_urb, usbdev, usb_sndbulkpipe(usbdev, CATC_USB_EP_BULK), + NULL, 0, catc_tx_done, catc); - usb_fill_bulk_urb(catc->rx_urb, usbdev, usb_rcvbulkpipe(usbdev, 1), - catc->rx_buf, pktsz, catc_rx_done, catc); + usb_fill_bulk_urb(catc->rx_urb, usbdev, usb_rcvbulkpipe(usbdev, CATC_USB_EP_BULK), + catc->rx_buf, pktsz, catc_rx_done, catc); - usb_fill_int_urb(catc->irq_urb, usbdev, usb_rcvintpipe(usbdev, 2), - catc->irq_buf, 2, catc_irq_done, catc, 1); + usb_fill_int_urb(catc->irq_urb, usbdev, usb_rcvintpipe(usbdev, CATC_USB_EP_INT_IN), + catc->irq_buf, 2, catc_irq_done, catc, 1); if (!catc->is_f5u011) { u32 *buf; diff --git a/drivers/net/wireless/ath/ath10k/sdio.c b/drivers/net/wireless/ath/ath10k/sdio.c index c06d50db40b818..00d0556dafefdb 100644 --- a/drivers/net/wireless/ath/ath10k/sdio.c +++ b/drivers/net/wireless/ath/ath10k/sdio.c @@ -2487,7 +2487,11 @@ void ath10k_sdio_fw_crashed_dump(struct ath10k *ar) if (fast_dump) ath10k_bmi_start(ar); + mutex_lock(&ar->dump_mutex); + + spin_lock_bh(&ar->data_lock); ar->stats.fw_crash_counter++; + spin_unlock_bh(&ar->data_lock); ath10k_sdio_disable_intrs(ar); @@ -2505,6 +2509,8 @@ void ath10k_sdio_fw_crashed_dump(struct ath10k *ar) ath10k_sdio_enable_intrs(ar); + mutex_unlock(&ar->dump_mutex); + ath10k_core_start_recovery(ar); } diff --git a/drivers/net/wireless/ath/ath11k/core.c b/drivers/net/wireless/ath/ath11k/core.c index 812686173ac8a9..06b4df2370e95d 100644 --- a/drivers/net/wireless/ath/ath11k/core.c +++ b/drivers/net/wireless/ath/ath11k/core.c @@ -997,6 +997,33 @@ static const struct dmi_system_id ath11k_pm_quirk_table[] = { {} }; +static const struct __ath11k_core_usecase_firmware_table { + u32 hw_rev; + const char *compatible; + const char *firmware_name; +} ath11k_core_usecase_firmware_table[] = { + { ATH11K_HW_WCN6855_HW21, "qcom,lemans-evk", "nfa765"}, + { ATH11K_HW_WCN6855_HW21, "qcom,monaco-evk", "nfa765"}, + { ATH11K_HW_WCN6855_HW21, "qcom,hamoa-iot-evk", "nfa765"}, + { /* Sentinel */ } +}; + +const char *ath11k_core_get_usecase_firmware(struct ath11k_base *ab) +{ + const struct __ath11k_core_usecase_firmware_table *entry = NULL; + + entry = ath11k_core_usecase_firmware_table; + while (entry->compatible) { + if (ab->hw_rev == entry->hw_rev && + of_machine_is_compatible(entry->compatible)) + return entry->firmware_name; + entry++; + } + + return NULL; +} +EXPORT_SYMBOL(ath11k_core_get_usecase_firmware); + void ath11k_fw_stats_pdevs_free(struct list_head *head) { struct ath11k_fw_stats_pdev *i, *tmp; diff --git a/drivers/net/wireless/ath/ath11k/core.h b/drivers/net/wireless/ath/ath11k/core.h index e8780b05ce11e3..834988dad591c7 100644 --- a/drivers/net/wireless/ath/ath11k/core.h +++ b/drivers/net/wireless/ath/ath11k/core.h @@ -1275,6 +1275,7 @@ bool ath11k_core_coldboot_cal_support(struct ath11k_base *ab); const struct firmware *ath11k_core_firmware_request(struct ath11k_base *ab, const char *filename); +const char *ath11k_core_get_usecase_firmware(struct ath11k_base *ab); static inline const char *ath11k_scan_state_str(enum ath11k_scan_state state) { @@ -1329,6 +1330,9 @@ static inline void ath11k_core_create_firmware_path(struct ath11k_base *ab, of_property_read_string(ab->dev->of_node, "firmware-name", &fw_name); + if (!fw_name) + fw_name = ath11k_core_get_usecase_firmware(ab); + if (fw_name && strncmp(filename, "board", 5)) snprintf(buf, buf_len, "%s/%s/%s/%s", ATH11K_FW_DIR, ab->hw_params.fw.dir, fw_name, filename); diff --git a/drivers/net/wireless/ath/ath12k/debugfs_htt_stats.c b/drivers/net/wireless/ath/ath12k/debugfs_htt_stats.c index 48b010a1b7566d..4f749d473d0e1f 100644 --- a/drivers/net/wireless/ath/ath12k/debugfs_htt_stats.c +++ b/drivers/net/wireless/ath/ath12k/debugfs_htt_stats.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include @@ -29,8 +29,10 @@ print_array_to_buf_index(u8 *buf, u32 offset, const char *header, u32 stats_inde " %u:%u,", stats_index++, le32_to_cpu(array[i])); } /* To overwrite the last trailing comma */ - index--; - *(buf + offset + index) = '\0'; + if (array_len > 0) { + index--; + *(buf + offset + index) = '\0'; + } if (footer) { index += scnprintf(buf + offset + index, diff --git a/drivers/net/wireless/ath/ath12k/mac.c b/drivers/net/wireless/ath/ath12k/mac.c index e0e49f782bf8df..63684ff9332d4f 100644 --- a/drivers/net/wireless/ath/ath12k/mac.c +++ b/drivers/net/wireless/ath/ath12k/mac.c @@ -4175,8 +4175,10 @@ ath12k_mac_op_change_vif_links(struct ieee80211_hw *hw, if (WARN_ON(!arvif)) return -EINVAL; - if (!arvif->is_created) + if (!arvif->is_created) { + ath12k_mac_unassign_link_vif(arvif); continue; + } if (WARN_ON(!arvif->ar)) return -EINVAL; diff --git a/drivers/net/wireless/ath/ath12k/wow.c b/drivers/net/wireless/ath/ath12k/wow.c index e8481626f19404..c78aa95d497911 100644 --- a/drivers/net/wireless/ath/ath12k/wow.c +++ b/drivers/net/wireless/ath/ath12k/wow.c @@ -135,6 +135,9 @@ static int ath12k_wow_cleanup(struct ath12k *ar) lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); list_for_each_entry(arvif, &ar->arvifs, list) { + if (arvif != &arvif->ahvif->deflink) + continue; + ret = ath12k_wow_vif_cleanup(arvif); if (ret) { ath12k_warn(ar->ab, "failed to clean wow wakeups on vdev %i: %d\n", @@ -479,8 +482,12 @@ static int ath12k_wow_set_wakeups(struct ath12k *ar, lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); list_for_each_entry(arvif, &ar->arvifs, list) { + if (arvif != &arvif->ahvif->deflink) + continue; + if (ath12k_wow_is_p2p_vdev(arvif->ahvif)) continue; + ret = ath12k_wow_vif_set_wakeups(arvif, wowlan); if (ret) { ath12k_warn(ar->ab, "failed to set wow wakeups on vdev %i: %d\n", @@ -538,6 +545,9 @@ static int ath12k_wow_nlo_cleanup(struct ath12k *ar) lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); list_for_each_entry(arvif, &ar->arvifs, list) { + if (arvif != &arvif->ahvif->deflink) + continue; + if (ath12k_wow_is_p2p_vdev(arvif->ahvif)) continue; @@ -745,6 +755,9 @@ static int ath12k_wow_arp_ns_offload(struct ath12k *ar, bool enable) list_for_each_entry(arvif, &ar->arvifs, list) { ahvif = arvif->ahvif; + if (arvif != &ahvif->deflink) + continue; + if (ahvif->vdev_type != WMI_VDEV_TYPE_STA) continue; @@ -776,6 +789,9 @@ static int ath12k_gtk_rekey_offload(struct ath12k *ar, bool enable) lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy); list_for_each_entry(arvif, &ar->arvifs, list) { + if (arvif != &arvif->ahvif->deflink) + continue; + if (arvif->ahvif->vdev_type != WMI_VDEV_TYPE_STA || !arvif->is_up || !arvif->rekey_data.enable_offload) diff --git a/drivers/net/wireless/ath/ath9k/Kconfig b/drivers/net/wireless/ath/ath9k/Kconfig index 0c47be06c153be..47d570a5ca6a1c 100644 --- a/drivers/net/wireless/ath/ath9k/Kconfig +++ b/drivers/net/wireless/ath/ath9k/Kconfig @@ -47,7 +47,7 @@ config ATH9K_PCI config ATH9K_AHB bool "Atheros ath9k AHB bus support" - depends on ATH9K + depends on ATH9K && OF default n help This option enables the AHB bus support in ath9k. diff --git a/drivers/net/wireless/ath/ath9k/common-debug.h b/drivers/net/wireless/ath/ath9k/common-debug.h index 2938b5b96b0749..97948af97682b9 100644 --- a/drivers/net/wireless/ath/ath9k/common-debug.h +++ b/drivers/net/wireless/ath/ath9k/common-debug.h @@ -19,14 +19,14 @@ /** * struct ath_rx_stats - RX Statistics * @rx_pkts_all: No. of total frames received, including ones that - may have had errors. + * may have had errors. * @rx_bytes_all: No. of total bytes received, including ones that - may have had errors. + * may have had errors. * @crc_err: No. of frames with incorrect CRC value * @decrypt_crc_err: No. of frames whose CRC check failed after - decryption process completed + * decryption process completed * @phy_err: No. of frames whose reception failed because the PHY - encountered an error + * encountered an error * @mic_err: No. of frames with incorrect TKIP MIC verification failure * @pre_delim_crc_err: Pre-Frame delimiter CRC error detections * @post_delim_crc_err: Post-Frame delimiter CRC error detections diff --git a/drivers/net/wireless/ath/ath9k/debug.h b/drivers/net/wireless/ath/ath9k/debug.h index cb3e75969875af..804e2a0a0c20f8 100644 --- a/drivers/net/wireless/ath/ath9k/debug.h +++ b/drivers/net/wireless/ath/ath9k/debug.h @@ -142,11 +142,12 @@ struct ath_interrupt_stats { /** * struct ath_tx_stats - Statistics about TX * @tx_pkts_all: No. of total frames transmitted, including ones that - may have had errors. + * may have had errors. * @tx_bytes_all: No. of total bytes transmitted, including ones that - may have had errors. + * may have had errors. * @queued: Total MPDUs (non-aggr) queued * @completed: Total MPDUs (non-aggr) completed + * @xretries: Total MPDUs with xretries * @a_aggr: Total no. of aggregates queued * @a_queued_hw: Total AMPDUs queued to hardware * @a_completed: Total AMPDUs completed @@ -154,14 +155,14 @@ struct ath_interrupt_stats { * @a_xretries: No. of AMPDUs dropped due to xretries * @txerr_filtered: No. of frames with TXERR_FILT flag set. * @fifo_underrun: FIFO underrun occurrences - Valid only for: - - non-aggregate condition. - - first packet of aggregate. + * Valid only for: + * - non-aggregate condition. + * - first packet of aggregate. * @xtxop: No. of frames filtered because of TXOP limit * @timer_exp: Transmit timer expiry * @desc_cfg_err: Descriptor configuration errors - * @data_urn: TX data underrun errors - * @delim_urn: TX delimiter underrun errors + * @data_underrun: TX data underrun errors + * @delim_underrun: TX delimiter underrun errors * @puttxbuf: Number of times hardware was given txbuf to write. * @txstart: Number of times hardware was told to start tx. * @txprocdesc: Number of times tx descriptor was processed diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile index e5ca0f51182271..6fd805023500be 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile @@ -25,7 +25,11 @@ brcmfmac-objs += \ btcoex.o \ vendor.o \ pno.o \ - xtlv.o + join_param.o \ + scan_param.o \ + xtlv.o \ + interface_create.o + brcmfmac-$(CONFIG_BRCMFMAC_PROTO_BCDC) += \ bcdc.o \ fwsignal.o diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h index fe31051a9e11b1..5efd7f6d757a4c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h @@ -107,6 +107,7 @@ struct brcmf_bus_ops { void (*debugfs_create)(struct device *dev); int (*reset)(struct device *dev); void (*remove)(struct device *dev); + void (*d2h_mb_rx)(struct device *dev, u32 data); }; @@ -286,6 +287,15 @@ static inline void brcmf_bus_remove(struct brcmf_bus *bus) bus->ops->remove(bus->dev); } +static inline +void brcmf_bus_d2h_mb_rx(struct brcmf_bus *bus, u32 data) +{ + if (!bus->ops->d2h_mb_rx) + return; + + return bus->ops->d2h_mb_rx(bus->dev, data); +} + /* * interface functions from common layer */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index bb96b87b2a6e56..a87965c5d1226a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -32,7 +32,11 @@ #include "vendor.h" #include "bus.h" #include "common.h" +#include "feature.h" #include "fwvid.h" +#include "xtlv.h" +#include "ratespec.h" +#include "interface_create.h" #define BRCMF_SCAN_IE_LEN_MAX 2048 @@ -64,6 +68,8 @@ #define RSN_CAP_MFPR_MASK BIT(6) #define RSN_CAP_MFPC_MASK BIT(7) #define RSN_PMKID_COUNT_LEN 2 +#define DPP_AKM_SUITE_TYPE 2 +#define WLAN_AKM_SUITE_DPP SUITE(WLAN_OUI_WFA, DPP_AKM_SUITE_TYPE) #define VNDR_IE_CMD_LEN 4 /* length of the set command * string :"add", "del" (+ NUL) @@ -76,10 +82,6 @@ #define DOT11_BCN_PRB_FIXED_LEN 12 /* beacon/probe fixed length */ -#define BRCMF_SCAN_JOIN_ACTIVE_DWELL_TIME_MS 320 -#define BRCMF_SCAN_JOIN_PASSIVE_DWELL_TIME_MS 400 -#define BRCMF_SCAN_JOIN_PROBE_INTERVAL_MS 20 - #define BRCMF_SCAN_CHANNEL_TIME 40 #define BRCMF_SCAN_UNASSOC_TIME 40 #define BRCMF_SCAN_PASSIVE_TIME 120 @@ -98,9 +100,6 @@ #define PKT_TOKEN_IDX 15 #define IDLE_TOKEN_IDX 12 -#define BRCMF_ASSOC_PARAMS_FIXED_SIZE \ - (sizeof(struct brcmf_assoc_params_le) - sizeof(u16)) - #define BRCMF_MAX_CHANSPEC_LIST \ (BRCMF_DCMD_MEDLEN / sizeof(__le32) - 1) @@ -124,6 +123,13 @@ struct cca_msrmnt_query { u32 time_req; }; +/* algo bit vector */ +#define KEY_ALGO_MASK(_algo) (1 << (_algo)) + +/* start enum value for BSS properties */ +#define WL_WSEC_INFO_BSS_BASE 0x0100 +#define WL_WSEC_INFO_BSS_ALGOS (WL_WSEC_INFO_BSS_BASE + 6) + static bool check_vif_up(struct brcmf_cfg80211_vif *vif) { if (!test_bit(BRCMF_VIF_STATUS_READY, &vif->sme_state)) { @@ -178,6 +184,15 @@ static struct ieee80211_rate __wl_rates[] = { .max_power = 30, \ } +#define CHAN6G(_channel) { \ + .band = NL80211_BAND_6GHZ, \ + .center_freq = ((_channel == 2) ? 5935 : 5950 + (5 * (_channel))), \ + .hw_value = (_channel), \ + .max_antenna_gain = 0, \ + .max_power = 30, \ +} + + static struct ieee80211_channel __wl_2ghz_channels[] = { CHAN2G(1, 2412), CHAN2G(2, 2417), CHAN2G(3, 2422), CHAN2G(4, 2427), CHAN2G(5, 2432), CHAN2G(6, 2437), CHAN2G(7, 2442), CHAN2G(8, 2447), @@ -194,6 +209,23 @@ static struct ieee80211_channel __wl_5ghz_channels[] = { CHAN5G(153), CHAN5G(157), CHAN5G(161), CHAN5G(165) }; +static struct ieee80211_channel __wl_6ghz_channels[] = { + CHAN6G(1), CHAN6G(2), CHAN6G(5), CHAN6G(9), CHAN6G(13), + CHAN6G(17), CHAN6G(21), CHAN6G(25), CHAN6G(29), CHAN6G(33), + CHAN6G(37), CHAN6G(41), CHAN6G(45), CHAN6G(49), CHAN6G(53), + CHAN6G(57), CHAN6G(61), CHAN6G(65), CHAN6G(69), CHAN6G(73), + CHAN6G(77), CHAN6G(81), CHAN6G(85), CHAN6G(89), CHAN6G(93), + CHAN6G(97), CHAN6G(101), CHAN6G(105), CHAN6G(109), CHAN6G(113), + CHAN6G(117), CHAN6G(121), CHAN6G(125), CHAN6G(129), CHAN6G(133), + CHAN6G(137), CHAN6G(141), CHAN6G(145), CHAN6G(149), CHAN6G(153), + CHAN6G(157), CHAN6G(161), CHAN6G(165), CHAN6G(169), CHAN6G(173), + CHAN6G(177), CHAN6G(181), CHAN6G(185), CHAN6G(189), CHAN6G(193), + CHAN6G(197), CHAN6G(201), CHAN6G(205), CHAN6G(209), CHAN6G(213), + CHAN6G(217), CHAN6G(221), CHAN6G(225), CHAN6G(229), CHAN6G(233), +}; + +struct ieee80211_sband_iftype_data sdata[NUM_NL80211_BANDS]; + /* Band templates duplicated per wiphy. The channel info * above is added to the band during setup. */ @@ -209,6 +241,12 @@ static const struct ieee80211_supported_band __wl_band_5ghz = { .n_bitrates = wl_a_rates_size, }; +static const struct ieee80211_supported_band __wl_band_6ghz = { + .band = NL80211_BAND_6GHZ, + .bitrates = wl_a_rates, + .n_bitrates = wl_a_rates_size, +}; + /* This is to override regulatory domains defined in cfg80211 module (reg.c) * By default world regulatory domain defined in reg.c puts the flags * NL80211_RRF_NO_IR for 5GHz channels (for * 36..48 and 149..165). @@ -217,35 +255,43 @@ static const struct ieee80211_supported_band __wl_band_5ghz = { * domain are to be done here. */ static const struct ieee80211_regdomain brcmf_regdom = { - .n_reg_rules = 4, + .n_reg_rules = 5, .alpha2 = "99", .reg_rules = { /* IEEE 802.11b/g, channels 1..11 */ - REG_RULE(2412-10, 2472+10, 40, 6, 20, 0), + REG_RULE(2412 - 10, 2472 + 10, 40, 6, 20, 0), /* If any */ /* IEEE 802.11 channel 14 - Only JP enables * this and for 802.11b only */ - REG_RULE(2484-10, 2484+10, 20, 6, 20, 0), + REG_RULE(2484 - 10, 2484 + 10, 20, 6, 20, 0), /* IEEE 802.11a, channel 36..64 */ - REG_RULE(5150-10, 5350+10, 160, 6, 20, 0), + REG_RULE(5150 - 10, 5350 + 10, 160, 6, 20, 0), /* IEEE 802.11a, channel 100..165 */ - REG_RULE(5470-10, 5850+10, 160, 6, 20, 0), } + REG_RULE(5470 - 10, 5850 + 10, 160, 6, 20, 0), + /* IEEE 802.11ax, 6E */ + REG_RULE(5935 - 10, 7115 + 10, 160, 6, 20, 0), } }; /* Note: brcmf_cipher_suites is an array of int defining which cipher suites * are supported. A pointer to this array and the number of entries is passed * on to upper layers. AES_CMAC defines whether or not the driver supports MFP. - * So the cipher suite AES_CMAC has to be the last one in the array, and when - * device does not support MFP then the number of suites will be decreased by 1 + * MFP support includes a few other suites, so if MFP is not supported, + * then the number of suites will be decreased by 4 */ static const u32 brcmf_cipher_suites[] = { WLAN_CIPHER_SUITE_WEP40, WLAN_CIPHER_SUITE_WEP104, WLAN_CIPHER_SUITE_TKIP, WLAN_CIPHER_SUITE_CCMP, - /* Keep as last entry: */ - WLAN_CIPHER_SUITE_AES_CMAC + WLAN_CIPHER_SUITE_CCMP_256, + WLAN_CIPHER_SUITE_GCMP, + WLAN_CIPHER_SUITE_GCMP_256, + /* Keep as last 4 entries: */ + WLAN_CIPHER_SUITE_AES_CMAC, + WLAN_CIPHER_SUITE_BIP_CMAC_256, + WLAN_CIPHER_SUITE_BIP_GMAC_128, + WLAN_CIPHER_SUITE_BIP_GMAC_256 }; /* Vendor specific ie. id = 221, oui and type defines exact ie */ @@ -267,48 +313,6 @@ struct parsed_vndr_ies { struct parsed_vndr_ie_info ie_info[VNDR_IE_PARSE_LIMIT]; }; -#define WL_INTERFACE_CREATE_VER_1 1 -#define WL_INTERFACE_CREATE_VER_2 2 -#define WL_INTERFACE_CREATE_VER_3 3 -#define WL_INTERFACE_CREATE_VER_MAX WL_INTERFACE_CREATE_VER_3 - -#define WL_INTERFACE_MAC_DONT_USE 0x0 -#define WL_INTERFACE_MAC_USE 0x2 - -#define WL_INTERFACE_CREATE_STA 0x0 -#define WL_INTERFACE_CREATE_AP 0x1 - -struct wl_interface_create_v1 { - u16 ver; /* structure version */ - u32 flags; /* flags for operation */ - u8 mac_addr[ETH_ALEN]; /* MAC address */ - u32 wlc_index; /* optional for wlc index */ -}; - -struct wl_interface_create_v2 { - u16 ver; /* structure version */ - u8 pad1[2]; - u32 flags; /* flags for operation */ - u8 mac_addr[ETH_ALEN]; /* MAC address */ - u8 iftype; /* type of interface created */ - u8 pad2; - u32 wlc_index; /* optional for wlc index */ -}; - -struct wl_interface_create_v3 { - u16 ver; /* structure version */ - u16 len; /* length of structure + data */ - u16 fixed_len; /* length of structure */ - u8 iftype; /* type of interface created */ - u8 wlc_index; /* optional for wlc index */ - u32 flags; /* flags for operation */ - u8 mac_addr[ETH_ALEN]; /* MAC address */ - u8 bssid[ETH_ALEN]; /* optional for BSSID */ - u8 if_index; /* interface index request */ - u8 pad[3]; - u8 data[]; /* Optional for specific data */ -}; - static u8 nl80211_band_to_fwil(enum nl80211_band band) { switch (band) { @@ -316,6 +320,8 @@ static u8 nl80211_band_to_fwil(enum nl80211_band band) return WLC_BAND_2G; case NL80211_BAND_5GHZ: return WLC_BAND_5G; + case NL80211_BAND_6GHZ: + return WLC_BAND_6G; default: WARN_ON(1); break; @@ -323,8 +329,25 @@ static u8 nl80211_band_to_fwil(enum nl80211_band band) return 0; } -static u16 chandef_to_chanspec(struct brcmu_d11inf *d11inf, - struct cfg80211_chan_def *ch) +static int nl80211_band_to_chanspec_band(enum nl80211_band band) +{ + switch (band) { + case NL80211_BAND_2GHZ: + return BRCMU_CHAN_BAND_2G; + case NL80211_BAND_5GHZ: + return BRCMU_CHAN_BAND_5G; + case NL80211_BAND_6GHZ: + return BRCMU_CHAN_BAND_6G; + case NL80211_BAND_60GHZ: + default: + WARN_ON_ONCE(1); + // Choose a safe default + return BRCMU_CHAN_BAND_2G; + } +} + +u16 chandef_to_chanspec(struct brcmu_d11inf *d11inf, + struct cfg80211_chan_def *ch) { struct brcmu_chan ch_inf; s32 primary_offset; @@ -382,17 +405,7 @@ static u16 chandef_to_chanspec(struct brcmu_d11inf *d11inf, default: WARN_ON_ONCE(1); } - switch (ch->chan->band) { - case NL80211_BAND_2GHZ: - ch_inf.band = BRCMU_CHAN_BAND_2G; - break; - case NL80211_BAND_5GHZ: - ch_inf.band = BRCMU_CHAN_BAND_5G; - break; - case NL80211_BAND_60GHZ: - default: - WARN_ON_ONCE(1); - } + ch_inf.band = nl80211_band_to_chanspec_band(ch->chan->band); d11inf->encchspec(&ch_inf); brcmf_dbg(TRACE, "chanspec: 0x%x\n", ch_inf.chspec); @@ -404,6 +417,7 @@ u16 channel_to_chanspec(struct brcmu_d11inf *d11inf, { struct brcmu_chan ch_inf; + ch_inf.band = nl80211_band_to_chanspec_band(ch->band); ch_inf.chnum = ieee80211_frequency_to_channel(ch->center_freq); ch_inf.bw = BRCMU_CHAN_BW_20; d11inf->encchspec(&ch_inf); @@ -581,231 +595,6 @@ brcmf_cfg80211_update_proto_addr_mode(struct wireless_dev *wdev) ADDR_INDIRECT); } -static int brcmf_get_first_free_bsscfgidx(struct brcmf_pub *drvr) -{ - int bsscfgidx; - - for (bsscfgidx = 0; bsscfgidx < BRCMF_MAX_IFS; bsscfgidx++) { - /* bsscfgidx 1 is reserved for legacy P2P */ - if (bsscfgidx == 1) - continue; - if (!drvr->iflist[bsscfgidx]) - return bsscfgidx; - } - - return -ENOMEM; -} - -static void brcmf_set_vif_sta_macaddr(struct brcmf_if *ifp, u8 *mac_addr) -{ - u8 mac_idx = ifp->drvr->sta_mac_idx; - - /* set difference MAC address with locally administered bit */ - memcpy(mac_addr, ifp->mac_addr, ETH_ALEN); - mac_addr[0] |= 0x02; - mac_addr[3] ^= mac_idx ? 0xC0 : 0xA0; - mac_idx++; - mac_idx = mac_idx % 2; - ifp->drvr->sta_mac_idx = mac_idx; -} - -static int brcmf_cfg80211_request_sta_if(struct brcmf_if *ifp, u8 *macaddr) -{ - struct wl_interface_create_v1 iface_v1; - struct wl_interface_create_v2 iface_v2; - struct wl_interface_create_v3 iface_v3; - u32 iface_create_ver; - int err; - - /* interface_create version 1 */ - memset(&iface_v1, 0, sizeof(iface_v1)); - iface_v1.ver = WL_INTERFACE_CREATE_VER_1; - iface_v1.flags = WL_INTERFACE_CREATE_STA | - WL_INTERFACE_MAC_USE; - if (!is_zero_ether_addr(macaddr)) - memcpy(iface_v1.mac_addr, macaddr, ETH_ALEN); - else - brcmf_set_vif_sta_macaddr(ifp, iface_v1.mac_addr); - - err = brcmf_fil_iovar_data_get(ifp, "interface_create", - &iface_v1, - sizeof(iface_v1)); - if (err) { - brcmf_info("failed to create interface(v1), err=%d\n", - err); - } else { - brcmf_dbg(INFO, "interface created(v1)\n"); - return 0; - } - - /* interface_create version 2 */ - memset(&iface_v2, 0, sizeof(iface_v2)); - iface_v2.ver = WL_INTERFACE_CREATE_VER_2; - iface_v2.flags = WL_INTERFACE_MAC_USE; - iface_v2.iftype = WL_INTERFACE_CREATE_STA; - if (!is_zero_ether_addr(macaddr)) - memcpy(iface_v2.mac_addr, macaddr, ETH_ALEN); - else - brcmf_set_vif_sta_macaddr(ifp, iface_v2.mac_addr); - - err = brcmf_fil_iovar_data_get(ifp, "interface_create", - &iface_v2, - sizeof(iface_v2)); - if (err) { - brcmf_info("failed to create interface(v2), err=%d\n", - err); - } else { - brcmf_dbg(INFO, "interface created(v2)\n"); - return 0; - } - - /* interface_create version 3+ */ - /* get supported version from firmware side */ - iface_create_ver = 0; - err = brcmf_fil_bsscfg_int_query(ifp, "interface_create", - &iface_create_ver); - if (err) { - brcmf_err("fail to get supported version, err=%d\n", err); - return -EOPNOTSUPP; - } - - switch (iface_create_ver) { - case WL_INTERFACE_CREATE_VER_3: - memset(&iface_v3, 0, sizeof(iface_v3)); - iface_v3.ver = WL_INTERFACE_CREATE_VER_3; - iface_v3.flags = WL_INTERFACE_MAC_USE; - iface_v3.iftype = WL_INTERFACE_CREATE_STA; - if (!is_zero_ether_addr(macaddr)) - memcpy(iface_v3.mac_addr, macaddr, ETH_ALEN); - else - brcmf_set_vif_sta_macaddr(ifp, iface_v3.mac_addr); - - err = brcmf_fil_iovar_data_get(ifp, "interface_create", - &iface_v3, - sizeof(iface_v3)); - - if (!err) - brcmf_dbg(INFO, "interface created(v3)\n"); - break; - default: - brcmf_err("not support interface create(v%d)\n", - iface_create_ver); - err = -EOPNOTSUPP; - break; - } - - if (err) { - brcmf_info("station interface creation failed (%d)\n", - err); - return -EIO; - } - - return 0; -} - -static int brcmf_cfg80211_request_ap_if(struct brcmf_if *ifp) -{ - struct wl_interface_create_v1 iface_v1; - struct wl_interface_create_v2 iface_v2; - struct wl_interface_create_v3 iface_v3; - u32 iface_create_ver; - struct brcmf_pub *drvr = ifp->drvr; - struct brcmf_mbss_ssid_le mbss_ssid_le; - int bsscfgidx; - int err; - - /* interface_create version 1 */ - memset(&iface_v1, 0, sizeof(iface_v1)); - iface_v1.ver = WL_INTERFACE_CREATE_VER_1; - iface_v1.flags = WL_INTERFACE_CREATE_AP | - WL_INTERFACE_MAC_USE; - - brcmf_set_vif_sta_macaddr(ifp, iface_v1.mac_addr); - - err = brcmf_fil_iovar_data_get(ifp, "interface_create", - &iface_v1, - sizeof(iface_v1)); - if (err) { - brcmf_info("failed to create interface(v1), err=%d\n", - err); - } else { - brcmf_dbg(INFO, "interface created(v1)\n"); - return 0; - } - - /* interface_create version 2 */ - memset(&iface_v2, 0, sizeof(iface_v2)); - iface_v2.ver = WL_INTERFACE_CREATE_VER_2; - iface_v2.flags = WL_INTERFACE_MAC_USE; - iface_v2.iftype = WL_INTERFACE_CREATE_AP; - - brcmf_set_vif_sta_macaddr(ifp, iface_v2.mac_addr); - - err = brcmf_fil_iovar_data_get(ifp, "interface_create", - &iface_v2, - sizeof(iface_v2)); - if (err) { - brcmf_info("failed to create interface(v2), err=%d\n", - err); - } else { - brcmf_dbg(INFO, "interface created(v2)\n"); - return 0; - } - - /* interface_create version 3+ */ - /* get supported version from firmware side */ - iface_create_ver = 0; - err = brcmf_fil_bsscfg_int_query(ifp, "interface_create", - &iface_create_ver); - if (err) { - brcmf_err("fail to get supported version, err=%d\n", err); - return -EOPNOTSUPP; - } - - switch (iface_create_ver) { - case WL_INTERFACE_CREATE_VER_3: - memset(&iface_v3, 0, sizeof(iface_v3)); - iface_v3.ver = WL_INTERFACE_CREATE_VER_3; - iface_v3.flags = WL_INTERFACE_MAC_USE; - iface_v3.iftype = WL_INTERFACE_CREATE_AP; - brcmf_set_vif_sta_macaddr(ifp, iface_v3.mac_addr); - - err = brcmf_fil_iovar_data_get(ifp, "interface_create", - &iface_v3, - sizeof(iface_v3)); - - if (!err) - brcmf_dbg(INFO, "interface created(v3)\n"); - break; - default: - brcmf_err("not support interface create(v%d)\n", - iface_create_ver); - err = -EOPNOTSUPP; - break; - } - - if (err) { - brcmf_info("Does not support interface_create (%d)\n", - err); - memset(&mbss_ssid_le, 0, sizeof(mbss_ssid_le)); - bsscfgidx = brcmf_get_first_free_bsscfgidx(ifp->drvr); - if (bsscfgidx < 0) - return bsscfgidx; - - mbss_ssid_le.bsscfgidx = cpu_to_le32(bsscfgidx); - mbss_ssid_le.SSID_len = cpu_to_le32(5); - sprintf(mbss_ssid_le.SSID, "ssid%d", bsscfgidx); - - err = brcmf_fil_bsscfg_data_set(ifp, "bsscfg:ssid", &mbss_ssid_le, - sizeof(mbss_ssid_le)); - - if (err < 0) - bphy_err(drvr, "setting ssid failed %d\n", err); - } - - return err; -} - /** * brcmf_apsta_add_vif() - create a new AP or STA virtual interface * @@ -1058,134 +847,11 @@ bool brcmf_is_apmode_operating(struct wiphy *wiphy) return ret; } -static void brcmf_scan_params_v2_to_v1(struct brcmf_scan_params_v2_le *params_v2_le, - struct brcmf_scan_params_le *params_le) -{ - size_t params_size; - u32 ch; - int n_channels, n_ssids; - - memcpy(¶ms_le->ssid_le, ¶ms_v2_le->ssid_le, - sizeof(params_le->ssid_le)); - memcpy(¶ms_le->bssid, ¶ms_v2_le->bssid, - sizeof(params_le->bssid)); - - params_le->bss_type = params_v2_le->bss_type; - params_le->scan_type = le32_to_cpu(params_v2_le->scan_type); - params_le->nprobes = params_v2_le->nprobes; - params_le->active_time = params_v2_le->active_time; - params_le->passive_time = params_v2_le->passive_time; - params_le->home_time = params_v2_le->home_time; - params_le->channel_num = params_v2_le->channel_num; - - ch = le32_to_cpu(params_v2_le->channel_num); - n_channels = ch & BRCMF_SCAN_PARAMS_COUNT_MASK; - n_ssids = ch >> BRCMF_SCAN_PARAMS_NSSID_SHIFT; - - params_size = sizeof(u16) * n_channels; - if (n_ssids > 0) { - params_size = roundup(params_size, sizeof(u32)); - params_size += sizeof(struct brcmf_ssid_le) * n_ssids; - } - - memcpy(¶ms_le->channel_list[0], - ¶ms_v2_le->channel_list[0], params_size); -} - -static void brcmf_escan_prep(struct brcmf_cfg80211_info *cfg, - struct brcmf_scan_params_v2_le *params_le, - struct cfg80211_scan_request *request) -{ - u32 n_ssids; - u32 n_channels; - s32 i; - s32 offset; - u16 chanspec; - char *ptr; - int length; - struct brcmf_ssid_le ssid_le; - - eth_broadcast_addr(params_le->bssid); - - length = BRCMF_SCAN_PARAMS_V2_FIXED_SIZE; - - params_le->version = cpu_to_le16(BRCMF_SCAN_PARAMS_VERSION_V2); - params_le->bss_type = DOT11_BSSTYPE_ANY; - params_le->scan_type = cpu_to_le32(BRCMF_SCANTYPE_ACTIVE); - params_le->channel_num = 0; - params_le->nprobes = cpu_to_le32(-1); - params_le->active_time = cpu_to_le32(-1); - params_le->passive_time = cpu_to_le32(-1); - params_le->home_time = cpu_to_le32(-1); - memset(¶ms_le->ssid_le, 0, sizeof(params_le->ssid_le)); - - /* Scan abort */ - if (!request) { - length += sizeof(u16); - params_le->channel_num = cpu_to_le32(1); - params_le->channel_list[0] = cpu_to_le16(-1); - params_le->length = cpu_to_le16(length); - return; - } - - n_ssids = request->n_ssids; - n_channels = request->n_channels; - - /* Copy channel array if applicable */ - brcmf_dbg(SCAN, "### List of channelspecs to scan ### %d\n", - n_channels); - if (n_channels > 0) { - length += roundup(sizeof(u16) * n_channels, sizeof(u32)); - for (i = 0; i < n_channels; i++) { - chanspec = channel_to_chanspec(&cfg->d11inf, - request->channels[i]); - brcmf_dbg(SCAN, "Chan : %d, Channel spec: %x\n", - request->channels[i]->hw_value, chanspec); - params_le->channel_list[i] = cpu_to_le16(chanspec); - } - } else { - brcmf_dbg(SCAN, "Scanning all channels\n"); - } - - /* Copy ssid array if applicable */ - brcmf_dbg(SCAN, "### List of SSIDs to scan ### %d\n", n_ssids); - if (n_ssids > 0) { - offset = offsetof(struct brcmf_scan_params_v2_le, channel_list) + - n_channels * sizeof(u16); - offset = roundup(offset, sizeof(u32)); - length += sizeof(ssid_le) * n_ssids; - ptr = (char *)params_le + offset; - for (i = 0; i < n_ssids; i++) { - memset(&ssid_le, 0, sizeof(ssid_le)); - ssid_le.SSID_len = - cpu_to_le32(request->ssids[i].ssid_len); - memcpy(ssid_le.SSID, request->ssids[i].ssid, - request->ssids[i].ssid_len); - if (!ssid_le.SSID_len) - brcmf_dbg(SCAN, "%d: Broadcast scan\n", i); - else - brcmf_dbg(SCAN, "%d: scan for %.32s size=%d\n", - i, ssid_le.SSID, ssid_le.SSID_len); - memcpy(ptr, &ssid_le, sizeof(ssid_le)); - ptr += sizeof(ssid_le); - } - } else { - brcmf_dbg(SCAN, "Performing passive scan\n"); - params_le->scan_type = cpu_to_le32(BRCMF_SCANTYPE_PASSIVE); - } - params_le->length = cpu_to_le16(length); - /* Adding mask to channel numbers */ - params_le->channel_num = - cpu_to_le32((n_ssids << BRCMF_SCAN_PARAMS_NSSID_SHIFT) | - (n_channels & BRCMF_SCAN_PARAMS_COUNT_MASK)); -} - s32 brcmf_notify_escan_complete(struct brcmf_cfg80211_info *cfg, struct brcmf_if *ifp, bool aborted, bool fw_abort) { struct brcmf_pub *drvr = cfg->pub; - struct brcmf_scan_params_v2_le params_v2_le; struct cfg80211_scan_request *scan_request; u64 reqid; u32 bucket; @@ -1201,25 +867,16 @@ s32 brcmf_notify_escan_complete(struct brcmf_cfg80211_info *cfg, timer_delete_sync(&cfg->escan_timeout); if (fw_abort) { + u32 len; + void *data = drvr->scan_param_handler.get_struct_for_request(cfg, &len, NULL); + if (!data){ + bphy_err(drvr, "Scan abort failed to prepare abort struct\n"); + return 0; + } /* Do a scan abort to stop the driver's scan engine */ brcmf_dbg(SCAN, "ABORT scan in firmware\n"); - - brcmf_escan_prep(cfg, ¶ms_v2_le, NULL); - - /* E-Scan (or anyother type) can be aborted by SCAN */ - if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_SCAN_V2)) { - err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SCAN, - ¶ms_v2_le, - sizeof(params_v2_le)); - } else { - struct brcmf_scan_params_le params_le; - - brcmf_scan_params_v2_to_v1(¶ms_v2_le, ¶ms_le); - err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SCAN, - ¶ms_le, - sizeof(params_le)); - } - + err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SCAN, data, len); + kfree(data); if (err) bphy_err(drvr, "Scan abort failed\n"); } @@ -1443,19 +1100,24 @@ brcmf_run_escan(struct brcmf_cfg80211_info *cfg, struct brcmf_if *ifp, struct cfg80211_scan_request *request) { struct brcmf_pub *drvr = cfg->pub; - s32 params_size = BRCMF_SCAN_PARAMS_V2_FIXED_SIZE + - offsetof(struct brcmf_escan_params_le, params_v2_le); + u32 struct_size = 0; + void *prepped_params = NULL; + u32 params_size = 0; struct brcmf_escan_params_le *params; s32 err = 0; brcmf_dbg(SCAN, "E-SCAN START\n"); - if (request != NULL) { - /* Allocate space for populating ssids in struct */ - params_size += sizeof(u32) * ((request->n_channels + 1) / 2); - - /* Allocate space for populating ssids in struct */ - params_size += sizeof(struct brcmf_ssid_le) * request->n_ssids; + prepped_params = drvr->scan_param_handler.get_struct_for_request(cfg, &struct_size, request); + if (!prepped_params) { + err = -EINVAL; + goto exit; + } + params_size = struct_size + + offsetof(struct brcmf_escan_params_le, params_v4_le); + if (!params_size) { + err = -EINVAL; + goto exit; } params = kzalloc(params_size, GFP_KERNEL); @@ -1463,27 +1125,14 @@ brcmf_run_escan(struct brcmf_cfg80211_info *cfg, struct brcmf_if *ifp, err = -ENOMEM; goto exit; } - BUG_ON(params_size + sizeof("escan") >= BRCMF_DCMD_MEDLEN); - brcmf_escan_prep(cfg, ¶ms->params_v2_le, request); - - params->version = cpu_to_le32(BRCMF_ESCAN_REQ_VERSION_V2); - - if (!brcmf_feat_is_enabled(ifp, BRCMF_FEAT_SCAN_V2)) { - struct brcmf_escan_params_le *params_v1; - - params_size -= BRCMF_SCAN_PARAMS_V2_FIXED_SIZE; - params_size += BRCMF_SCAN_PARAMS_FIXED_SIZE; - params_v1 = kzalloc(params_size, GFP_KERNEL); - if (!params_v1) { - err = -ENOMEM; - goto exit_params; - } - params_v1->version = cpu_to_le32(BRCMF_ESCAN_REQ_VERSION); - brcmf_scan_params_v2_to_v1(¶ms->params_v2_le, ¶ms_v1->params_le); - kfree(params); - params = params_v1; - } + /* Copy into the largest part */ + unsafe_memcpy( + ¶ms->params_v4_le, prepped_params, struct_size, + /* A composite flex-array that is at least as large as the memcpy due to the allocation above */); + /* We can now free the original prepped parameters */ + kfree(prepped_params); + params->version = cpu_to_le32(drvr->scan_param_handler.version); params->action = cpu_to_le16(WL_ESCAN_ACTION_START); params->sync_id = cpu_to_le16(0x1234); @@ -1495,7 +1144,6 @@ brcmf_run_escan(struct brcmf_cfg80211_info *cfg, struct brcmf_if *ifp, bphy_err(drvr, "error (%d)\n", err); } -exit_params: kfree(params); exit: return err; @@ -1779,21 +1427,19 @@ static void brcmf_link_down(struct brcmf_cfg80211_vif *vif, u16 reason, brcmf_dbg(TRACE, "Exit\n"); } -static s32 -brcmf_cfg80211_join_ibss(struct wiphy *wiphy, struct net_device *ndev, - struct cfg80211_ibss_params *params) +static s32 brcmf_cfg80211_join_ibss(struct wiphy *wiphy, + struct net_device *ndev, + struct cfg80211_ibss_params *params) { struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); struct brcmf_if *ifp = netdev_priv(ndev); struct brcmf_cfg80211_profile *profile = &ifp->vif->profile; struct brcmf_pub *drvr = cfg->pub; - struct brcmf_join_params join_params; - size_t join_params_size = 0; - s32 err = 0; + void *join_params; + u32 join_params_size = 0; s32 wsec = 0; s32 bcnprd; - u16 chanspec; - u32 ssid_len; + s32 err = 0; brcmf_dbg(TRACE, "Enter\n"); if (!check_vif_up(ifp->vif)) @@ -1867,58 +1513,39 @@ brcmf_cfg80211_join_ibss(struct wiphy *wiphy, struct net_device *ndev, goto done; } - /* Configure required join parameter */ - memset(&join_params, 0, sizeof(struct brcmf_join_params)); - - /* SSID */ - ssid_len = min_t(u32, params->ssid_len, IEEE80211_MAX_SSID_LEN); - memcpy(join_params.ssid_le.SSID, params->ssid, ssid_len); - join_params.ssid_le.SSID_len = cpu_to_le32(ssid_len); - join_params_size = sizeof(join_params.ssid_le); - - /* BSSID */ if (params->bssid) { - memcpy(join_params.params_le.bssid, params->bssid, ETH_ALEN); - join_params_size += BRCMF_ASSOC_PARAMS_FIXED_SIZE; memcpy(profile->bssid, params->bssid, ETH_ALEN); } else { - eth_broadcast_addr(join_params.params_le.bssid); eth_zero_addr(profile->bssid); } - /* Channel */ + cfg->ibss_starter = false; + cfg->channel = 0; if (params->chandef.chan) { - u32 target_channel; + u16 chanspec; + cfg->channel = ieee80211_frequency_to_channel( + params->chandef.chan->center_freq); + /* adding chanspec */ + chanspec = chandef_to_chanspec(&cfg->d11inf, ¶ms->chandef); - cfg->channel = - ieee80211_frequency_to_channel( - params->chandef.chan->center_freq); - if (params->channel_fixed) { - /* adding chanspec */ - chanspec = chandef_to_chanspec(&cfg->d11inf, - ¶ms->chandef); - join_params.params_le.chanspec_list[0] = - cpu_to_le16(chanspec); - join_params.params_le.chanspec_num = cpu_to_le32(1); - join_params_size += sizeof(join_params.params_le); - } - - /* set channel for starter */ - target_channel = cfg->channel; - err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_CHANNEL, - target_channel); + /* set chanspec */ + err = brcmf_fil_iovar_int_set(ifp, "chanspec", chanspec); if (err) { - bphy_err(drvr, "WLC_SET_CHANNEL failed (%d)\n", err); + bphy_err(drvr, "Setting chanspec failed (%d)\n", err); goto done; } - } else - cfg->channel = 0; - - cfg->ibss_starter = false; - + } - err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SET_SSID, - &join_params, join_params_size); + join_params = drvr->join_param_handler.get_struct_for_ibss( + cfg, &join_params_size, params); + if (!join_params) { + bphy_err(drvr, "Converting join params failed\n"); + goto done; + } + err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SET_SSID, join_params, + join_params_size); + /* Free params no matter what */ + kfree(join_params); if (err) { bphy_err(drvr, "WLC_SET_SSID failed (%d)\n", err); goto done; @@ -1969,6 +1596,10 @@ static s32 brcmf_set_wpa_version(struct net_device *ndev, if (drvr->bus_if->fwvid == BRCMF_FWVENDOR_CYW && sme->crypto.akm_suites[0] == WLAN_AKM_SUITE_SAE) val = WPA3_AUTH_SAE_PSK; + else if (sme->crypto.akm_suites[0] == WLAN_AKM_SUITE_SAE) + val = WPA3_AUTH_SAE_PSK; + else if (sme->crypto.akm_suites[0] == WLAN_AKM_SUITE_OWE) + val = WPA3_AUTH_OWE; else val = WPA2_AUTH_PSK | WPA2_AUTH_UNSPECIFIED; } else if (sme->crypto.wpa_versions & NL80211_WPA_VERSION_3) { @@ -2026,6 +1657,48 @@ static s32 brcmf_set_auth_type(struct net_device *ndev, return err; } +static s32 brcmf_set_wsec_info_algos(struct brcmf_if *ifp, u32 algos, u32 mask) +{ + struct brcmf_pub *drvr = ifp->drvr; + s32 err = 0; + struct brcmf_wsec_info *wsec_info; + struct brcmf_xtlv *wsec_info_tlv; + u16 tlv_data_len; + u8 tlv_data[8]; + u32 param_len; + u8 *buf; + + brcmf_dbg(TRACE, "Enter\n"); + + buf = kzalloc(sizeof(struct brcmf_wsec_info) + sizeof(tlv_data), + GFP_KERNEL); + if (!buf) { + bphy_err(drvr, "unable to allocate.\n"); + return -ENOMEM; + } + wsec_info = (struct brcmf_wsec_info *)buf; + wsec_info->version = BRCMF_WSEC_INFO_VER; + wsec_info_tlv = + (struct brcmf_xtlv *)(buf + + offsetof(struct brcmf_wsec_info, tlvs)); + wsec_info->num_tlvs++; + tlv_data_len = sizeof(tlv_data); + memcpy(tlv_data, &algos, sizeof(algos)); + memcpy(tlv_data + sizeof(algos), &mask, sizeof(mask)); + brcmf_xtlv_pack_header(wsec_info_tlv, WL_WSEC_INFO_BSS_ALGOS, + tlv_data_len, tlv_data, 0); + + param_len = offsetof(struct brcmf_wsec_info, tlvs) + + offsetof(struct brcmf_wsec_info_tlv, data) + tlv_data_len; + + err = brcmf_fil_bsscfg_data_set(ifp, "wsec_info", buf, param_len); + if (err) + brcmf_err("set wsec_info_error:%d\n", err); + + kfree(buf); + return err; +} + static s32 brcmf_set_wsec_mode(struct net_device *ndev, struct cfg80211_connect_params *sme) @@ -2038,6 +1711,8 @@ brcmf_set_wsec_mode(struct net_device *ndev, s32 gval = 0; s32 wsec; s32 err = 0; + u32 algos = 0; + u32 mask = 0; if (sme->crypto.n_ciphers_pairwise) { switch (sme->crypto.ciphers_pairwise[0]) { @@ -2054,6 +1729,15 @@ brcmf_set_wsec_mode(struct net_device *ndev, case WLAN_CIPHER_SUITE_AES_CMAC: pval = AES_ENABLED; break; + case WLAN_CIPHER_SUITE_GCMP_256: + if (!brcmf_feat_is_enabled(ifp, BRCMF_FEAT_GCMP)) { + brcmf_err("This chip does not support GCMP\n"); + return -EOPNOTSUPP; + } + pval = AES_ENABLED; + algos = KEY_ALGO_MASK(CRYPTO_ALGO_AES_GCM256); + mask = algos | KEY_ALGO_MASK(CRYPTO_ALGO_AES_CCM); + break; default: bphy_err(drvr, "invalid cipher pairwise (%d)\n", sme->crypto.ciphers_pairwise[0]); @@ -2075,6 +1759,15 @@ brcmf_set_wsec_mode(struct net_device *ndev, case WLAN_CIPHER_SUITE_AES_CMAC: gval = AES_ENABLED; break; + case WLAN_CIPHER_SUITE_GCMP_256: + if (!brcmf_feat_is_enabled(ifp, BRCMF_FEAT_GCMP)) { + brcmf_err("This chip does not support GCMP\n"); + return -EOPNOTSUPP; + } + gval = AES_ENABLED; + algos = KEY_ALGO_MASK(CRYPTO_ALGO_AES_GCM256); + mask = algos | KEY_ALGO_MASK(CRYPTO_ALGO_AES_CCM); + break; default: bphy_err(drvr, "invalid cipher group (%d)\n", sme->crypto.cipher_group); @@ -2083,6 +1776,7 @@ brcmf_set_wsec_mode(struct net_device *ndev, } brcmf_dbg(CONN, "pval (%d) gval (%d)\n", pval, gval); + brcmf_dbg(CONN, "algos (0x%x) mask (0x%x)\n", algos, mask); /* In case of privacy, but no security and WPS then simulate */ /* setting AES. WPS-2.0 allows no security */ if (brcmf_find_wpsie(sme->ie, sme->ie_len) && !pval && !gval && @@ -2095,6 +1789,15 @@ brcmf_set_wsec_mode(struct net_device *ndev, bphy_err(drvr, "error (%d)\n", err); return err; } + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_GCMP)) { + brcmf_dbg(CONN, "set_wsec_info algos (0x%x) mask (0x%x)\n", + algos, mask); + err = brcmf_set_wsec_info_algos(ifp, algos, mask); + if (err) { + brcmf_err("set wsec_info error (%d)\n", err); + return err; + } + } sec = &profile->sec; sec->cipher_pairwise = sme->crypto.ciphers_pairwise[0]; @@ -2118,9 +1821,13 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) u16 rsn_cap; u32 mfp; u16 count; + s32 okc_enable; + u16 pmkid_count; + const u8 *group_mgmt_cs = NULL; profile->use_fwsup = BRCMF_PROFILE_FWSUP_NONE; profile->is_ft = false; + profile->is_okc = false; if (!sme->crypto.n_akm_suites) return 0; @@ -2137,13 +1844,15 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) val = WPA_AUTH_UNSPECIFIED; if (sme->want_1x) profile->use_fwsup = BRCMF_PROFILE_FWSUP_1X; + else + profile->use_fwsup = BRCMF_PROFILE_FWSUP_ROAM; break; case WLAN_AKM_SUITE_PSK: val = WPA_AUTH_PSK; break; default: - bphy_err(drvr, "invalid akm suite (%d)\n", - sme->crypto.akm_suites[0]); + bphy_err(drvr, "invalid cipher group (%d)\n", + sme->crypto.cipher_group); return -EINVAL; } } else if (val & (WPA2_AUTH_PSK | WPA2_AUTH_UNSPECIFIED)) { @@ -2152,11 +1861,15 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) val = WPA2_AUTH_UNSPECIFIED; if (sme->want_1x) profile->use_fwsup = BRCMF_PROFILE_FWSUP_1X; + else + profile->use_fwsup = BRCMF_PROFILE_FWSUP_ROAM; break; case WLAN_AKM_SUITE_8021X_SHA256: val = WPA2_AUTH_1X_SHA256; if (sme->want_1x) profile->use_fwsup = BRCMF_PROFILE_FWSUP_1X; + else + profile->use_fwsup = BRCMF_PROFILE_FWSUP_ROAM; break; case WLAN_AKM_SUITE_PSK_SHA256: val = WPA2_AUTH_PSK_SHA256; @@ -2169,14 +1882,35 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) profile->is_ft = true; if (sme->want_1x) profile->use_fwsup = BRCMF_PROFILE_FWSUP_1X; + else + profile->use_fwsup = BRCMF_PROFILE_FWSUP_ROAM; break; case WLAN_AKM_SUITE_FT_PSK: val = WPA2_AUTH_PSK | WPA2_AUTH_FT; profile->is_ft = true; + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_FWSUP)) + profile->use_fwsup = BRCMF_PROFILE_FWSUP_PSK; + else + profile->use_fwsup = BRCMF_PROFILE_FWSUP_ROAM; + break; + case WLAN_AKM_SUITE_DPP: + val = WFA_AUTH_DPP; + profile->use_fwsup = BRCMF_PROFILE_FWSUP_NONE; + break; + case WLAN_AKM_SUITE_OWE: + val = WPA3_AUTH_OWE; + profile->use_fwsup = BRCMF_PROFILE_FWSUP_ROAM; + break; + case WLAN_AKM_SUITE_8021X_SUITE_B_192: + val = WPA3_AUTH_1X_SUITE_B_SHA384; + if (sme->want_1x) + profile->use_fwsup = BRCMF_PROFILE_FWSUP_1X; + else + profile->use_fwsup = BRCMF_PROFILE_FWSUP_ROAM; break; default: - bphy_err(drvr, "invalid akm suite (%d)\n", - sme->crypto.akm_suites[0]); + bphy_err(drvr, "invalid cipher group (%d)\n", + sme->crypto.cipher_group); return -EINVAL; } } else if (val & WPA3_AUTH_SAE_PSK) { @@ -2189,17 +1923,38 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) profile->is_ft = true; break; default: - bphy_err(drvr, "invalid akm suite (%d)\n", - sme->crypto.akm_suites[0]); + bphy_err(drvr, "invalid cipher group (%d)\n", + sme->crypto.cipher_group); return -EINVAL; } if (sme->crypto.sae_pwd) { profile->use_fwsup = BRCMF_PROFILE_FWSUP_SAE; } } - - if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X) + if ((profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X) || + (profile->use_fwsup == BRCMF_PROFILE_FWSUP_ROAM)) { brcmf_dbg(INFO, "using 1X offload\n"); + err = brcmf_fil_bsscfg_int_get(netdev_priv(ndev), "okc_enable", + &okc_enable); + if (err) { + bphy_err(drvr, "get okc_enable failed (%d)\n", err); + } else { + brcmf_dbg(INFO, "get okc_enable (%d)\n", okc_enable); + profile->is_okc = okc_enable; + } + } else if (profile->use_fwsup != BRCMF_PROFILE_FWSUP_SAE && + (val == WPA3_AUTH_SAE_PSK)) { + brcmf_dbg(INFO, "not using SAE offload\n"); + err = brcmf_fil_bsscfg_int_get(netdev_priv(ndev), "okc_enable", + &okc_enable); + if (err) { + bphy_err(drvr, "get okc_enable failed (%d)\n", err); + } else { + brcmf_dbg(INFO, "get okc_enable (%d)\n", okc_enable); + profile->is_okc = okc_enable; + } + } + if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_SAE) brcmf_dbg(INFO, "using SAE offload\n"); @@ -2235,14 +1990,47 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) mfp = BRCMF_MFP_REQUIRED; else if (rsn_cap & RSN_CAP_MFPC_MASK) mfp = BRCMF_MFP_CAPABLE; + /* In case of dpp, very low tput is observed if MFPC is set in + * firmmare. Firmware needs to ensure that MFPC is not set when + * MFPR was requested from fmac. However since this change being + * specific to DPP, fmac needs to set wpa_auth prior to mfp, so + * that firmware can use this info to prevent MFPC being set in + * case of dpp. + */ + if (val == WFA_AUTH_DPP) { + brcmf_dbg(CONN, "setting wpa_auth to 0x%0x\n", val); + err = brcmf_fil_bsscfg_int_set(netdev_priv(ndev), "wpa_auth", + val); + if (err) { + bphy_err(drvr, "could not set wpa_auth (%d)\n", err); + return err; + } + } + brcmf_fil_bsscfg_int_set(netdev_priv(ndev), "mfp", mfp); + offset += RSN_CAP_LEN; + if (mfp && (ie_len - offset >= RSN_PMKID_COUNT_LEN)) { + pmkid_count = ie[offset] + (ie[offset + 1] << 8); + offset += RSN_PMKID_COUNT_LEN + (pmkid_count * WLAN_PMKID_LEN); + if (ie_len - offset >= WPA_IE_MIN_OUI_LEN) { + group_mgmt_cs = &ie[offset]; + if (memcmp(group_mgmt_cs, RSN_OUI, TLV_OUI_LEN) == 0) { + brcmf_fil_bsscfg_data_set(ifp, "bip", + (void *)group_mgmt_cs, + WPA_IE_MIN_OUI_LEN); + } + } + } skip_mfp_config: - brcmf_dbg(CONN, "setting wpa_auth to 0x%0x\n", val); - err = brcmf_fil_bsscfg_int_set(netdev_priv(ndev), "wpa_auth", val); - if (err) { - bphy_err(drvr, "could not set wpa_auth (%d)\n", err); - return err; + if (val != WFA_AUTH_DPP) { + brcmf_dbg(CONN, "setting wpa_auth to 0x%0x\n", val); + err = brcmf_fil_bsscfg_int_set(netdev_priv(ndev), "wpa_auth", + val); + if (err) { + bphy_err(drvr, "could not set wpa_auth (%d)\n", err); + return err; + } } return err; @@ -2375,52 +2163,51 @@ static void brcmf_set_join_pref(struct brcmf_if *ifp, static s32 brcmf_cfg80211_connect(struct wiphy *wiphy, struct net_device *ndev, - struct cfg80211_connect_params *sme) + struct cfg80211_connect_params *params) { struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); struct brcmf_if *ifp = netdev_priv(ndev); struct brcmf_cfg80211_profile *profile = &ifp->vif->profile; - struct ieee80211_channel *chan = sme->channel; + struct ieee80211_channel *chan = params->channel; struct brcmf_pub *drvr = ifp->drvr; - struct brcmf_join_params join_params; - size_t join_params_size; + void *join_params; + u32 join_params_size; + void *fallback_join_params; + u32 fallback_join_params_size; const struct brcmf_tlv *rsn_ie; const struct brcmf_vs_tlv *wpa_ie; const void *ie; u32 ie_len; - struct brcmf_ext_join_params_le *ext_join_params; - u16 chanspec; s32 err = 0; - u32 ssid_len; brcmf_dbg(TRACE, "Enter\n"); if (!check_vif_up(ifp->vif)) return -EIO; - if (!sme->ssid) { + if (!params->ssid) { bphy_err(drvr, "Invalid ssid\n"); return -EOPNOTSUPP; } - if (sme->channel_hint) - chan = sme->channel_hint; + if (params->channel_hint) + chan = params->channel_hint; - if (sme->bssid_hint) - sme->bssid = sme->bssid_hint; + if (params->bssid_hint) + params->bssid = params->bssid_hint; if (ifp->vif == cfg->p2p.bss_idx[P2PAPI_BSSCFG_PRIMARY].vif) { /* A normal (non P2P) connection request setup. */ ie = NULL; ie_len = 0; /* find the WPA_IE */ - wpa_ie = brcmf_find_wpaie((u8 *)sme->ie, sme->ie_len); + wpa_ie = brcmf_find_wpaie((u8 *)params->ie, params->ie_len); if (wpa_ie) { ie = wpa_ie; ie_len = wpa_ie->len + TLV_HDR_LEN; } else { /* find the RSN_IE */ - rsn_ie = brcmf_parse_tlvs((const u8 *)sme->ie, - sme->ie_len, + rsn_ie = brcmf_parse_tlvs((const u8 *)params->ie, + params->ie_len, WLAN_EID_RSN); if (rsn_ie) { ie = rsn_ie; @@ -2431,7 +2218,7 @@ brcmf_cfg80211_connect(struct wiphy *wiphy, struct net_device *ndev, } err = brcmf_vif_set_mgmt_ie(ifp->vif, BRCMF_VNDR_IE_ASSOCREQ_FLAG, - sme->ie, sme->ie_len); + params->ie, params->ie_len); if (err) bphy_err(drvr, "Set Assoc REQ IE Failed\n"); else @@ -2442,166 +2229,129 @@ brcmf_cfg80211_connect(struct wiphy *wiphy, struct net_device *ndev, if (chan) { cfg->channel = ieee80211_frequency_to_channel(chan->center_freq); - chanspec = channel_to_chanspec(&cfg->d11inf, chan); - brcmf_dbg(CONN, "channel=%d, center_req=%d, chanspec=0x%04x\n", - cfg->channel, chan->center_freq, chanspec); + brcmf_dbg(CONN, "channel=%d, center_req=%d\n", + cfg->channel, chan->center_freq); } else { cfg->channel = 0; - chanspec = 0; } - brcmf_dbg(INFO, "ie (%p), ie_len (%zd)\n", sme->ie, sme->ie_len); + brcmf_dbg(INFO, "ie (%p), ie_len (%zd)\n", params->ie, params->ie_len); - err = brcmf_set_wpa_version(ndev, sme); + err = brcmf_set_wpa_version(ndev, params); if (err) { bphy_err(drvr, "wl_set_wpa_version failed (%d)\n", err); goto done; } - sme->auth_type = brcmf_war_auth_type(ifp, sme->auth_type); - err = brcmf_set_auth_type(ndev, sme); + params->auth_type = brcmf_war_auth_type(ifp, params->auth_type); + err = brcmf_set_auth_type(ndev, params); if (err) { bphy_err(drvr, "wl_set_auth_type failed (%d)\n", err); goto done; } - err = brcmf_set_wsec_mode(ndev, sme); + err = brcmf_set_wsec_mode(ndev, params); if (err) { bphy_err(drvr, "wl_set_set_cipher failed (%d)\n", err); goto done; } - err = brcmf_set_key_mgmt(ndev, sme); + err = brcmf_set_key_mgmt(ndev, params); if (err) { bphy_err(drvr, "wl_set_key_mgmt failed (%d)\n", err); goto done; } - err = brcmf_set_sharedkey(ndev, sme); - if (err) { - bphy_err(drvr, "brcmf_set_sharedkey failed (%d)\n", err); - goto done; - } - - if (sme->crypto.psk && - profile->use_fwsup != BRCMF_PROFILE_FWSUP_SAE) { - if (WARN_ON(profile->use_fwsup != BRCMF_PROFILE_FWSUP_NONE)) { - err = -EINVAL; - goto done; - } - brcmf_dbg(INFO, "using PSK offload\n"); - profile->use_fwsup = BRCMF_PROFILE_FWSUP_PSK; - } - - if (profile->use_fwsup != BRCMF_PROFILE_FWSUP_NONE) { - /* enable firmware supplicant for this interface */ - err = brcmf_fil_iovar_int_set(ifp, "sup_wpa", 1); - if (err < 0) { - bphy_err(drvr, "failed to enable fw supplicant\n"); - goto done; - } - } - - if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_PSK) - err = brcmf_set_pmk(ifp, sme->crypto.psk, - BRCMF_WSEC_MAX_PSK_LEN); - else if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_SAE) { - /* clean up user-space RSNE */ - err = brcmf_fil_iovar_data_set(ifp, "wpaie", NULL, 0); - if (err) { - bphy_err(drvr, "failed to clean up user-space RSNE\n"); - goto done; - } - err = brcmf_fwvid_set_sae_password(ifp, &sme->crypto); - if (!err && sme->crypto.psk) - err = brcmf_set_pmk(ifp, sme->crypto.psk, - BRCMF_WSEC_MAX_PSK_LEN); - } - if (err) - goto done; - - /* Join with specific BSSID and cached SSID - * If SSID is zero join based on BSSID only - */ - join_params_size = offsetof(struct brcmf_ext_join_params_le, assoc_le) + - offsetof(struct brcmf_assoc_params_le, chanspec_list); - if (cfg->channel) - join_params_size += sizeof(u16); - ext_join_params = kzalloc(sizeof(*ext_join_params), GFP_KERNEL); - if (ext_join_params == NULL) { - err = -ENOMEM; + err = brcmf_set_sharedkey(ndev, params); + if (err) { + bphy_err(drvr, "brcmf_set_sharedkey failed (%d)\n", err); goto done; } - ssid_len = min_t(u32, sme->ssid_len, IEEE80211_MAX_SSID_LEN); - ext_join_params->ssid_le.SSID_len = cpu_to_le32(ssid_len); - memcpy(&ext_join_params->ssid_le.SSID, sme->ssid, ssid_len); - if (ssid_len < IEEE80211_MAX_SSID_LEN) - brcmf_dbg(CONN, "SSID \"%s\", len (%d)\n", - ext_join_params->ssid_le.SSID, ssid_len); + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_FWSUP)) { + if (params->crypto.psk) { + if ((profile->use_fwsup != BRCMF_PROFILE_FWSUP_SAE) && + (profile->use_fwsup != BRCMF_PROFILE_FWSUP_PSK)) { + if (WARN_ON(profile->use_fwsup != + BRCMF_PROFILE_FWSUP_NONE)) { + err = -EINVAL; + goto done; + } + brcmf_dbg(INFO, "using PSK offload\n"); + profile->use_fwsup = BRCMF_PROFILE_FWSUP_PSK; + } + } - /* Set up join scan parameters */ - ext_join_params->scan_le.scan_type = -1; - ext_join_params->scan_le.home_time = cpu_to_le32(-1); + if (profile->use_fwsup != BRCMF_PROFILE_FWSUP_NONE) { + /* enable firmware supplicant for this interface */ + err = brcmf_fil_iovar_int_set(ifp, "sup_wpa", 1); + if (err < 0) { + bphy_err(drvr, + "failed to enable fw supplicant\n"); + goto done; + } + } else { + err = brcmf_fil_iovar_int_set(ifp, "sup_wpa", 0); + } - if (sme->bssid) - memcpy(&ext_join_params->assoc_le.bssid, sme->bssid, ETH_ALEN); - else - eth_broadcast_addr(ext_join_params->assoc_le.bssid); + if ((profile->use_fwsup == BRCMF_PROFILE_FWSUP_PSK) && + params->crypto.psk) + err = brcmf_set_pmk(ifp, params->crypto.psk, + BRCMF_WSEC_MAX_PSK_LEN); + else if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_SAE) { + /* clean up user-space RSNE */ + if (brcmf_fil_iovar_data_set(ifp, "wpaie", NULL, 0)) { + bphy_err( + drvr, + "failed to clean up user-space RSNE\n"); + goto done; + } + err = brcmf_fwvid_set_sae_password(ifp, ¶ms->crypto); + if (!err && params->crypto.psk) + err = brcmf_set_pmk(ifp, params->crypto.psk, + BRCMF_WSEC_MAX_PSK_LEN); + } + if (err) + goto done; + } + brcmf_set_join_pref(ifp, ¶ms->bss_select); + if (params->ssid_len < IEEE80211_MAX_SSID_LEN) + brcmf_dbg(CONN, "SSID \"%s\", len (%zu)\n", params->ssid, + params->ssid_len); + join_params = drvr->join_param_handler.get_struct_for_connect( + cfg, &join_params_size, params); - if (cfg->channel) { - ext_join_params->assoc_le.chanspec_num = cpu_to_le32(1); + if (join_params) { + err = brcmf_fil_bsscfg_data_set(ifp, "join", join_params, + join_params_size); - ext_join_params->assoc_le.chanspec_list[0] = - cpu_to_le16(chanspec); - /* Increase dwell time to receive probe response or detect - * beacon from target AP at a noisy air only during connect - * command. - */ - ext_join_params->scan_le.active_time = - cpu_to_le32(BRCMF_SCAN_JOIN_ACTIVE_DWELL_TIME_MS); - ext_join_params->scan_le.passive_time = - cpu_to_le32(BRCMF_SCAN_JOIN_PASSIVE_DWELL_TIME_MS); - /* To sync with presence period of VSDB GO send probe request - * more frequently. Probe request will be stopped when it gets - * probe response from target AP/GO. - */ - ext_join_params->scan_le.nprobes = - cpu_to_le32(BRCMF_SCAN_JOIN_ACTIVE_DWELL_TIME_MS / - BRCMF_SCAN_JOIN_PROBE_INTERVAL_MS); - } else { - ext_join_params->scan_le.active_time = cpu_to_le32(-1); - ext_join_params->scan_le.passive_time = cpu_to_le32(-1); - ext_join_params->scan_le.nprobes = cpu_to_le32(-1); + /* We only free the join parameters if we were successful. + * Otherwise they are used to extract the fallback, below */ + if (!err) { + kfree(join_params); + /* This is it. join command worked, we are done */ + goto done; + } + /* For versions >= 1, this should have worked, so report the error */ + if (drvr->join_param_handler.version >= 1) { + bphy_err(drvr, "Failed to use join iovar to join: %d\n", + err); + } } - brcmf_set_join_pref(ifp, &sme->bss_select); - - err = brcmf_fil_bsscfg_data_set(ifp, "join", ext_join_params, - join_params_size); - kfree(ext_join_params); - if (!err) - /* This is it. join command worked, we are done */ + /* Fallback to using WLC_SET_SSID approach, which just uses join_params parts of the structure */ + fallback_join_params = drvr->join_param_handler.get_join_from_ext_join( + join_params, &fallback_join_params_size); + if (!fallback_join_params) { + bphy_err(drvr, "Unable to generate fallback join params\n"); + kfree(join_params); goto done; - - /* join command failed, fallback to set ssid */ - memset(&join_params, 0, sizeof(join_params)); - join_params_size = sizeof(join_params.ssid_le); - - memcpy(&join_params.ssid_le.SSID, sme->ssid, ssid_len); - join_params.ssid_le.SSID_len = cpu_to_le32(ssid_len); - - if (sme->bssid) - memcpy(join_params.params_le.bssid, sme->bssid, ETH_ALEN); - else - eth_broadcast_addr(join_params.params_le.bssid); - - if (cfg->channel) { - join_params.params_le.chanspec_list[0] = cpu_to_le16(chanspec); - join_params.params_le.chanspec_num = cpu_to_le32(1); - join_params_size += sizeof(join_params.params_le); } err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SET_SSID, - &join_params, join_params_size); + fallback_join_params, + fallback_join_params_size); + + kfree(join_params); + kfree(fallback_join_params); if (err) bphy_err(drvr, "BRCMF_C_SET_SSID failed (%d)\n", err); @@ -2807,6 +2557,8 @@ brcmf_cfg80211_add_key(struct wiphy *wiphy, struct net_device *ndev, s32 val; s32 wsec; s32 err; + u32 algos = 0; + u32 mask = 0; u8 keybuf[8]; bool ext_key; @@ -2890,6 +2642,30 @@ brcmf_cfg80211_add_key(struct wiphy *wiphy, struct net_device *ndev, val = AES_ENABLED; brcmf_dbg(CONN, "WLAN_CIPHER_SUITE_CCMP\n"); break; + case WLAN_CIPHER_SUITE_GCMP_256: + if (!brcmf_feat_is_enabled(ifp, BRCMF_FEAT_GCMP)) { + brcmf_err("the low layer not support GCMP\n"); + err = -EOPNOTSUPP; + goto done; + } + key->algo = CRYPTO_ALGO_AES_GCM256; + val = AES_ENABLED; + brcmf_dbg(CONN, "WLAN_CIPHER_SUITE_GCMP_256\n"); + algos = KEY_ALGO_MASK(CRYPTO_ALGO_AES_GCM256); + mask = algos | KEY_ALGO_MASK(CRYPTO_ALGO_AES_CCM); + break; + case WLAN_CIPHER_SUITE_BIP_GMAC_256: + if (!brcmf_feat_is_enabled(ifp, BRCMF_FEAT_GCMP)) { + brcmf_err("the low layer not support GCMP\n"); + err = -EOPNOTSUPP; + goto done; + } + key->algo = CRYPTO_ALGO_BIP_GMAC256; + val = AES_ENABLED; + algos = KEY_ALGO_MASK(CRYPTO_ALGO_BIP_GMAC256); + mask = algos | KEY_ALGO_MASK(CRYPTO_ALGO_AES_CCM); + brcmf_dbg(CONN, "WLAN_CIPHER_SUITE_BIP_GMAC_256\n"); + break; default: bphy_err(drvr, "Invalid cipher (0x%x)\n", params->cipher); err = -EINVAL; @@ -2911,6 +2687,17 @@ brcmf_cfg80211_add_key(struct wiphy *wiphy, struct net_device *ndev, bphy_err(drvr, "set wsec error (%d)\n", err); goto done; } + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_GCMP)) { + brcmf_dbg(CONN, + "set_wsdec_info algos (0x%x) mask (0x%x)\n", + algos, mask); + err = brcmf_set_wsec_info_algos(ifp, algos, mask); + if (err) { + brcmf_err("set wsec_info error (%d)\n", err); + return err; + } + } + done: brcmf_dbg(TRACE, "Exit\n"); @@ -3131,6 +2918,70 @@ brcmf_cfg80211_get_station_ibss(struct brcmf_if *ifp, return 0; } +static void brcmf_convert_ratespec_to_rateinfo(u32 ratespec, + struct rate_info *rateinfo) +{ + /* First extract the bandwidth info */ + switch (ratespec & BRCMF_RSPEC_BW_MASK) { + case BRCMF_RSPEC_BW_20MHZ: + rateinfo->bw = RATE_INFO_BW_20; + break; + case BRCMF_RSPEC_BW_40MHZ: + rateinfo->bw = RATE_INFO_BW_40; + break; + case BRCMF_RSPEC_BW_80MHZ: + rateinfo->bw = RATE_INFO_BW_80; + break; + case BRCMF_RSPEC_BW_160MHZ: + rateinfo->bw = RATE_INFO_BW_160; + break; + case BRCMF_RSPEC_BW_320MHZ: + rateinfo->bw = RATE_INFO_BW_320; + break; + default: + /* Fill in nothing */ + break; + } + if (BRCMF_RSPEC_ISHT(ratespec)) { + rateinfo->flags |= RATE_INFO_FLAGS_MCS; + rateinfo->mcs = ratespec & BRCMF_RSPEC_HT_MCS_MASK; + } else if (BRCMF_RSPEC_ISVHT(ratespec)) { + rateinfo->flags |= RATE_INFO_FLAGS_VHT_MCS; + rateinfo->mcs = ratespec & BRCMF_RSPEC_VHT_MCS_MASK; + rateinfo->nss = (ratespec & BRCMF_RSPEC_VHT_NSS_MASK) >> + BRCMF_RSPEC_VHT_NSS_SHIFT; + } else if (BRCMF_RSPEC_ISHE(ratespec)) { + u32 ltf_gi = BRCMF_RSPEC_HE_LTF_GI(ratespec); + + rateinfo->flags |= RATE_INFO_FLAGS_HE_MCS; + rateinfo->mcs = ratespec & BRCMF_RSPEC_HE_MCS_MASK; + rateinfo->nss = (ratespec & BRCMF_RSPEC_HE_NSS_MASK) >> + BRCMF_RSPEC_HE_NSS_SHIFT; + rateinfo->he_dcm = BRCMF_RSPEC_HE_DCM(ratespec); + if (HE_IS_GI_0_8us(ltf_gi)) { + rateinfo->he_gi = NL80211_RATE_INFO_HE_GI_0_8; + } else if (HE_IS_GI_1_6us(ltf_gi)) { + rateinfo->he_gi = NL80211_RATE_INFO_HE_GI_1_6; + } else if (HE_IS_GI_3_2us(ltf_gi)) { + rateinfo->he_gi = NL80211_RATE_INFO_HE_GI_3_2; + } + } else if (BRCMF_RSPEC_ISEHT(ratespec)) { + u32 ltf_gi = BRCMF_RSPEC_EHT_LTF_GI(ratespec); + + rateinfo->flags |= RATE_INFO_FLAGS_EHT_MCS; + rateinfo->mcs = ratespec & BRCMF_RSPEC_EHT_MCS_MASK; + rateinfo->nss = (ratespec & BRCMF_RSPEC_EHT_NSS_MASK) >> + BRCMF_RSPEC_EHT_NSS_SHIFT; + if (EHT_IS_GI_0_8us(ltf_gi)) { + rateinfo->eht_gi = NL80211_RATE_INFO_EHT_GI_0_8; + } else if (EHT_IS_GI_1_6us(ltf_gi)) { + rateinfo->eht_gi = NL80211_RATE_INFO_EHT_GI_1_6; + } else if (EHT_IS_GI_3_2us(ltf_gi)) { + rateinfo->eht_gi = NL80211_RATE_INFO_EHT_GI_3_2; + } + } +} + static s32 brcmf_cfg80211_get_station(struct wiphy *wiphy, struct net_device *ndev, const u8 *mac, struct station_info *sinfo) @@ -3148,6 +2999,8 @@ brcmf_cfg80211_get_station(struct wiphy *wiphy, struct net_device *ndev, s32 count_rssi = 0; int rssi; u32 i; + u16 struct_ver; + u16 info_len; brcmf_dbg(TRACE, "Enter, MAC %pM\n", mac); if (!check_vif_up(ifp->vif)) @@ -3171,7 +3024,9 @@ brcmf_cfg80211_get_station(struct wiphy *wiphy, struct net_device *ndev, goto done; } } - brcmf_dbg(TRACE, "version %d\n", le16_to_cpu(sta_info_le.ver)); + info_len = le16_to_cpu(sta_info_le.len); + struct_ver = le16_to_cpu(sta_info_le.ver); + brcmf_dbg(TRACE, "version %d\n", struct_ver); sinfo->filled = BIT_ULL(NL80211_STA_INFO_INACTIVE_TIME); sinfo->inactive_time = le32_to_cpu(sta_info_le.idle) * 1000; sta_flags = le32_to_cpu(sta_info_le.flags); @@ -3205,12 +3060,13 @@ brcmf_cfg80211_get_station(struct wiphy *wiphy, struct net_device *ndev, sinfo->rxrate.legacy = le32_to_cpu(sta_info_le.rx_rate) / 100; } - if (le16_to_cpu(sta_info_le.ver) >= 4) { + if (struct_ver >= 4) { sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BYTES); sinfo->tx_bytes = le64_to_cpu(sta_info_le.tx_tot_bytes); sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_BYTES); sinfo->rx_bytes = le64_to_cpu(sta_info_le.rx_tot_bytes); } + for (i = 0; i < BRCMF_ANT_MAX; i++) { if (sta_info_le.rssi[i] == 0 || sta_info_le.rx_lastpkt_rssi[i] == 0) @@ -3249,6 +3105,25 @@ brcmf_cfg80211_get_station(struct wiphy *wiphy, struct net_device *ndev, } } } + /* Some version 7 structs have ratespecs from the last packet. */ + if (struct_ver >= 7) { + if (info_len >= sizeof(sta_info_le)) { + brcmf_convert_ratespec_to_rateinfo( + le32_to_cpu(sta_info_le.v7.tx_rspec), + &sinfo->txrate); + brcmf_convert_ratespec_to_rateinfo( + le32_to_cpu(sta_info_le.v7.rx_rspec), + &sinfo->rxrate); + } else { + /* We didn't get the fields we were expecting, fallback to nrate */ + u32 nrate = 0; + err = brcmf_fil_iovar_int_get(ifp, "nrate", &nrate); + if (!err) { + brcmf_convert_ratespec_to_rateinfo( + nrate, &sinfo->txrate); + } + } + } done: brcmf_dbg(TRACE, "Exit\n"); return err; @@ -3349,6 +3224,7 @@ static s32 brcmf_inform_single_bss(struct brcmf_cfg80211_info *cfg, struct cfg80211_bss *bss; enum nl80211_band band; struct brcmu_chan ch; + u16 chanspec; u16 channel; u32 freq; u16 notify_capability; @@ -3362,20 +3238,41 @@ static s32 brcmf_inform_single_bss(struct brcmf_cfg80211_info *cfg, return -EINVAL; } + chanspec = le16_to_cpu(bi->chanspec); if (!bi->ctl_ch) { - ch.chspec = le16_to_cpu(bi->chanspec); + ch.chspec = chanspec; cfg->d11inf.decchspec(&ch); bi->ctl_ch = ch.control_ch_num; } channel = bi->ctl_ch; - if (channel <= CH_MAX_2G_CHANNEL) - band = NL80211_BAND_2GHZ; - else + if (CHSPEC_IS6G(chanspec)) + band = NL80211_BAND_6GHZ; + else if (CHSPEC_IS5G(chanspec)) band = NL80211_BAND_5GHZ; + else + band = NL80211_BAND_2GHZ; freq = ieee80211_channel_to_frequency(channel, band); + if (!freq) { + brcmf_err("Invalid frequency %d returned for channel %d, band %d. chanspec was %04x\n", + freq, channel, band, bi->chanspec); + + /* We ignore this BSS ID rather than try to continue on. + * Otherwise we will cause an OOPs because our frequency is 0. + * The main case this occurs is some new frequency band + * we have not seen before, and if we return an error, + * we will cause the scan to fail. It seems better to + * report the error, skip this BSS, and move on. + */ + return 0; + } bss_data.chan = ieee80211_get_channel(wiphy, freq); + if (!bss_data.chan) { + brcmf_err("Could not convert frequency into channel for channel %d, band %d, chanspec was %04x\n", + channel, band, bi->chanspec); + return 0; + } bss_data.boottime_ns = ktime_to_ns(ktime_get_boottime()); notify_capability = le16_to_cpu(bi->capability); @@ -3424,8 +3321,9 @@ static s32 brcmf_inform_bss(struct brcmf_cfg80211_info *cfg) bss_list = (struct brcmf_scan_results *)cfg->escan_info.escan_buf; if (bss_list->count != 0 && - bss_list->version != BRCMF_BSS_INFO_VERSION) { - bphy_err(drvr, "Version %d != WL_BSS_INFO_VERSION\n", + (bss_list->version < BRCMF_BSS_INFO_MIN_VERSION || + bss_list->version > BRCMF_BSS_INFO_MAX_VERSION)) { + bphy_err(drvr, "BSS info version %d unsupported\n", bss_list->version); return -EOPNOTSUPP; } @@ -3463,7 +3361,7 @@ static s32 brcmf_inform_ibss(struct brcmf_cfg80211_info *cfg, buf = kzalloc(WL_BSS_INFO_MAX, GFP_KERNEL); if (buf == NULL) { err = -ENOMEM; - goto CleanUp; + goto cleanup; } *(__le32 *)buf = cpu_to_le32(WL_BSS_INFO_MAX); @@ -3472,7 +3370,7 @@ static s32 brcmf_inform_ibss(struct brcmf_cfg80211_info *cfg, buf, WL_BSS_INFO_MAX); if (err) { bphy_err(drvr, "WLC_GET_BSS_INFO failed: %d\n", err); - goto CleanUp; + goto cleanup; } bi = (struct brcmf_bss_info_le *)(buf + 4); @@ -3482,10 +3380,18 @@ static s32 brcmf_inform_ibss(struct brcmf_cfg80211_info *cfg, if (ch.band == BRCMU_CHAN_BAND_2G) band = wiphy->bands[NL80211_BAND_2GHZ]; - else + else if (ch.band == BRCMU_CHAN_BAND_5G) band = wiphy->bands[NL80211_BAND_5GHZ]; + else + band = wiphy->bands[NL80211_BAND_6GHZ]; freq = ieee80211_channel_to_frequency(ch.control_ch_num, band->band); + if (freq == 0) { + brcmf_err("Invalid frequency %d returned for channel %d, band %d. chanspec was %04x\n", + freq, ch.control_ch_num, ch.band, bi->chanspec); + goto cleanup; + } + cfg->channel = freq; notify_channel = ieee80211_get_channel(wiphy, freq); @@ -3508,12 +3414,12 @@ static s32 brcmf_inform_ibss(struct brcmf_cfg80211_info *cfg, if (!bss) { err = -ENOMEM; - goto CleanUp; + goto cleanup; } cfg80211_put_bss(wiphy, bss); -CleanUp: +cleanup: kfree(buf); @@ -3765,17 +3671,11 @@ brcmf_alloc_internal_escan_request(struct wiphy *wiphy, u32 n_netinfo) { } static int brcmf_internal_escan_add_info(struct cfg80211_scan_request *req, - u8 *ssid, u8 ssid_len, u8 channel) + u8 *ssid, u8 ssid_len, u8 channel, enum nl80211_band band) { struct ieee80211_channel *chan; - enum nl80211_band band; int freq, i; - if (channel <= CH_MAX_2G_CHANNEL) - band = NL80211_BAND_2GHZ; - else - band = NL80211_BAND_5GHZ; - freq = ieee80211_channel_to_frequency(channel, band); if (!freq) return -EINVAL; @@ -3831,53 +3731,30 @@ static int brcmf_start_internal_escan(struct brcmf_if *ifp, u32 fwmap, return 0; } -static struct brcmf_pno_net_info_le * -brcmf_get_netinfo_array(struct brcmf_pno_scanresults_le *pfn_v1) -{ - struct brcmf_pno_scanresults_v2_le *pfn_v2; - struct brcmf_pno_net_info_le *netinfo; - - switch (pfn_v1->version) { - default: - WARN_ON(1); - fallthrough; - case cpu_to_le32(1): - netinfo = (struct brcmf_pno_net_info_le *)(pfn_v1 + 1); - break; - case cpu_to_le32(2): - pfn_v2 = (struct brcmf_pno_scanresults_v2_le *)pfn_v1; - netinfo = (struct brcmf_pno_net_info_le *)(pfn_v2 + 1); - break; - } - - return netinfo; -} - /* PFN result doesn't have all the info which are required by the supplicant * (For e.g IEs) Do a target Escan so that sched scan results are reported * via wl_inform_single_bss in the required format. Escan does require the * scan request in the form of cfg80211_scan_request. For timebeing, create * cfg80211_scan_request one out of the received PNO event. */ -static s32 -brcmf_notify_sched_scan_results(struct brcmf_if *ifp, - const struct brcmf_event_msg *e, void *data) +static s32 brcmf_notify_sched_scan_results(struct brcmf_if *ifp, + const struct brcmf_event_msg *e, + void *data) { struct brcmf_pub *drvr = ifp->drvr; struct brcmf_cfg80211_info *cfg = drvr->config; - struct brcmf_pno_net_info_le *netinfo, *netinfo_start; struct cfg80211_scan_request *request = NULL; struct wiphy *wiphy = cfg_to_wiphy(cfg); int i, err = 0; - struct brcmf_pno_scanresults_le *pfn_result; u32 bucket_map; u32 result_count; u32 status; - u32 datalen; + u32 min_data_len; brcmf_dbg(SCAN, "Enter\n"); + min_data_len = drvr->pno_handler.get_min_data_len(); - if (e->datalen < (sizeof(*pfn_result) + sizeof(*netinfo))) { + if (e->datalen < min_data_len) { brcmf_dbg(SCAN, "Event data too small. Ignore\n"); return 0; } @@ -3887,9 +3764,8 @@ brcmf_notify_sched_scan_results(struct brcmf_if *ifp, return 0; } - pfn_result = (struct brcmf_pno_scanresults_le *)data; - result_count = le32_to_cpu(pfn_result->count); - status = le32_to_cpu(pfn_result->status); + result_count = drvr->pno_handler.get_result_count(data); + status = drvr->pno_handler.get_result_status(data); /* PFN event is limited to fit 512 bytes so we may get * multiple NET_FOUND events. For now place a warning here. @@ -3900,38 +3776,33 @@ brcmf_notify_sched_scan_results(struct brcmf_if *ifp, bphy_err(drvr, "FALSE PNO Event. (pfn_count == 0)\n"); goto out_err; } - - netinfo_start = brcmf_get_netinfo_array(pfn_result); - datalen = e->datalen - ((void *)netinfo_start - (void *)pfn_result); - if (datalen < result_count * sizeof(*netinfo)) { - bphy_err(drvr, "insufficient event data\n"); + err = drvr->pno_handler.validate_pfn_results(data, e->datalen); + if (err) { + bphy_err(drvr, "Invalid escan results (%d)", err); goto out_err; } - - request = brcmf_alloc_internal_escan_request(wiphy, - result_count); + request = brcmf_alloc_internal_escan_request(wiphy, result_count); if (!request) { err = -ENOMEM; goto out_err; } - bucket_map = 0; for (i = 0; i < result_count; i++) { - netinfo = &netinfo_start[i]; - - if (netinfo->SSID_len > IEEE80211_MAX_SSID_LEN) - netinfo->SSID_len = IEEE80211_MAX_SSID_LEN; - brcmf_dbg(SCAN, "SSID:%.32s Channel:%d\n", - netinfo->SSID, netinfo->channel); - bucket_map |= brcmf_pno_get_bucket_map(cfg->pno, netinfo); - err = brcmf_internal_escan_add_info(request, - netinfo->SSID, - netinfo->SSID_len, - netinfo->channel); + u8 channel; + enum nl80211_band band; + u8 ssid[IEEE80211_MAX_SSID_LEN]; + u8 ssid_len; + + drvr->pno_handler.get_result_info(data, i, &ssid, &ssid_len, + &channel, &band); + brcmf_dbg(SCAN, "SSID:%.32s Channel:%d Band:%d\n", ssid, + channel, band); + bucket_map |= drvr->pno_handler.get_bucket_map(data, i, cfg->pno); + err = brcmf_internal_escan_add_info(request, ssid, ssid_len, + channel, band); if (err) goto out_err; } - if (!bucket_map) goto free_req; @@ -4034,48 +3905,50 @@ static s32 brcmf_config_wowl_pattern(struct brcmf_if *ifp, u8 cmd[4], return ret; } -static s32 -brcmf_wowl_nd_results(struct brcmf_if *ifp, const struct brcmf_event_msg *e, - void *data) +static s32 brcmf_wowl_nd_results(struct brcmf_if *ifp, + const struct brcmf_event_msg *e, void *data) { struct brcmf_pub *drvr = ifp->drvr; struct brcmf_cfg80211_info *cfg = drvr->config; - struct brcmf_pno_scanresults_le *pfn_result; - struct brcmf_pno_net_info_le *netinfo; + u32 min_data_len; + u8 channel; + enum nl80211_band band; + u8 ssid[IEEE80211_MAX_SSID_LEN]; + u8 ssid_len; + u32 result_count; brcmf_dbg(SCAN, "Enter\n"); - if (e->datalen < (sizeof(*pfn_result) + sizeof(*netinfo))) { + min_data_len = drvr->pno_handler.get_min_data_len(); + + if (e->datalen < min_data_len) { brcmf_dbg(SCAN, "Event data too small. Ignore\n"); return 0; } - pfn_result = (struct brcmf_pno_scanresults_le *)data; if (e->event_code == BRCMF_E_PFN_NET_LOST) { brcmf_dbg(SCAN, "PFN NET LOST event. Ignore\n"); return 0; } - if (le32_to_cpu(pfn_result->count) < 1) { + result_count = drvr->pno_handler.get_result_count(data); + if (result_count < 1) { bphy_err(drvr, "Invalid result count, expected 1 (%d)\n", - le32_to_cpu(pfn_result->count)); + result_count); return -EINVAL; } - netinfo = brcmf_get_netinfo_array(pfn_result); - if (netinfo->SSID_len > IEEE80211_MAX_SSID_LEN) - netinfo->SSID_len = IEEE80211_MAX_SSID_LEN; - memcpy(cfg->wowl.nd->ssid.ssid, netinfo->SSID, netinfo->SSID_len); - cfg->wowl.nd->ssid.ssid_len = netinfo->SSID_len; + drvr->pno_handler.get_result_info(data, 0, &ssid, &ssid_len, &channel, + &band); + memcpy(cfg->wowl.nd->ssid.ssid, ssid, ssid_len); + cfg->wowl.nd->ssid.ssid_len = ssid_len; cfg->wowl.nd->n_channels = 1; cfg->wowl.nd->channels[0] = - ieee80211_channel_to_frequency(netinfo->channel, - netinfo->channel <= CH_MAX_2G_CHANNEL ? - NL80211_BAND_2GHZ : NL80211_BAND_5GHZ); + ieee80211_channel_to_frequency(channel, band); + cfg->wowl.nd_info->n_matches = 1; cfg->wowl.nd_info->matches[0] = cfg->wowl.nd; - /* Inform (the resume task) that the net detect information was recvd */ cfg->wowl.nd_data_completed = true; wake_up(&cfg->wowl.nd_data_wait); @@ -5149,6 +5022,25 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, settings->inactivity_timeout); dev_role = ifp->vif->wdev.iftype; mbss = ifp->vif->mbss; + /* Bring firmware into correct state for AP mode*/ + if (dev_role == NL80211_IFTYPE_AP) { + brcmf_dbg(TRACE, "set AP mode\n"); + err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_AP, 1); + if (err < 0) { + bphy_err(drvr, "setting AP mode failed %d\n", + err); + goto exit; + } + + bss_enable.bsscfgidx = cpu_to_le32(ifp->bsscfgidx); + bss_enable.enable = cpu_to_le32(WLC_AP_IOV_OP_MANUAL_AP_BSSCFG_CREATE); + err = brcmf_fil_iovar_data_set(ifp, "bss", &bss_enable, + sizeof(bss_enable)); + if (err < 0) { + bphy_err(drvr, "AP role set error, %d\n", err); + goto exit; + } + } /* store current 11d setting */ if (brcmf_fil_cmd_int_get(ifp, BRCMF_C_GET_REGULATORY, @@ -5742,6 +5634,9 @@ static int brcmf_cfg80211_get_channel(struct wiphy *wiphy, case BRCMU_CHAN_BAND_5G: band = NL80211_BAND_5GHZ; break; + case BRCMU_CHAN_BAND_6G: + band = NL80211_BAND_6GHZ; + break; } switch (ch.bw) { @@ -5763,9 +5658,19 @@ static int brcmf_cfg80211_get_channel(struct wiphy *wiphy, } freq = ieee80211_channel_to_frequency(ch.control_ch_num, band); + if (freq == 0) { + brcmf_err("Invalid frequency %d returned for channel %d, band %d. chanspec was %04x\n", + freq, ch.control_ch_num, ch.band, chanspec); + return -EINVAL; + } chandef->chan = ieee80211_get_channel(wiphy, freq); chandef->width = width; chandef->center_freq1 = ieee80211_channel_to_frequency(ch.chnum, band); + if (chandef->center_freq1 == 0) { + brcmf_err("Invalid frequency %d returned for channel %d, band %d. chanspec was %04x\n", + freq, ch.chnum, ch.band, chanspec); + return -EINVAL; + } chandef->center_freq2 = 0; return 0; @@ -5930,17 +5835,29 @@ static int brcmf_cfg80211_set_pmk(struct wiphy *wiphy, struct net_device *dev, const struct cfg80211_pmk_conf *conf) { struct brcmf_if *ifp; - + struct brcmf_pub *drvr; + int ret; brcmf_dbg(TRACE, "enter\n"); /* expect using firmware supplicant for 1X */ ifp = netdev_priv(dev); - if (WARN_ON(ifp->vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_1X)) + drvr = ifp->drvr; + if (WARN_ON((ifp->vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_1X) && + (ifp->vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_ROAM) && + (ifp->vif->profile.is_ft != true) && + (ifp->vif->profile.is_okc != true))) return -EINVAL; if (conf->pmk_len > BRCMF_WSEC_MAX_PSK_LEN) return -ERANGE; + if (ifp->vif->profile.is_okc) { + ret = brcmf_fil_iovar_data_set(ifp, "okc_info_pmk", conf->pmk, + conf->pmk_len); + if (ret < 0) + bphy_err(drvr, "okc_info_pmk iovar failed: ret=%d\n", + ret); + } return brcmf_set_pmk(ifp, conf->pmk, conf->pmk_len); } @@ -6399,6 +6316,46 @@ static s32 brcmf_get_assoc_ies(struct brcmf_cfg80211_info *cfg, return err; } +static bool brcmf_has_pmkid(const u8 *parse, u32 len) +{ + const struct brcmf_tlv *rsn_ie; + const u8 *ie; + u32 ie_len; + u32 offset; + u16 count; + + rsn_ie = brcmf_parse_tlvs(parse, len, WLAN_EID_RSN); + if (!rsn_ie) + goto done; + ie = (const u8 *)rsn_ie; + ie_len = rsn_ie->len + TLV_HDR_LEN; + /* Skip group data cipher suite */ + offset = TLV_HDR_LEN + WPA_IE_VERSION_LEN + WPA_IE_MIN_OUI_LEN; + if (offset + WPA_IE_SUITE_COUNT_LEN >= ie_len) + goto done; + /* Skip pairwise cipher suite(s) */ + count = ie[offset] + (ie[offset + 1] << 8); + offset += WPA_IE_SUITE_COUNT_LEN + (count * WPA_IE_MIN_OUI_LEN); + if (offset + WPA_IE_SUITE_COUNT_LEN >= ie_len) + goto done; + /* Skip auth key management suite(s) */ + count = ie[offset] + (ie[offset + 1] << 8); + offset += WPA_IE_SUITE_COUNT_LEN + (count * WPA_IE_MIN_OUI_LEN); + if (offset + RSN_CAP_LEN >= ie_len) + goto done; + /* Skip rsn capabilities */ + offset += RSN_CAP_LEN; + if (offset + RSN_PMKID_COUNT_LEN > ie_len) + goto done; + /* Extract PMKID count */ + count = ie[offset] + (ie[offset + 1] << 8); + if (count) + return true; + +done: + return false; +} + static s32 brcmf_bss_roaming_done(struct brcmf_cfg80211_info *cfg, struct net_device *ndev, @@ -6443,10 +6400,17 @@ brcmf_bss_roaming_done(struct brcmf_cfg80211_info *cfg, if (ch.band == BRCMU_CHAN_BAND_2G) band = wiphy->bands[NL80211_BAND_2GHZ]; - else + else if (ch.band == BRCMU_CHAN_BAND_5G) band = wiphy->bands[NL80211_BAND_5GHZ]; + else + band = wiphy->bands[NL80211_BAND_6GHZ]; freq = ieee80211_channel_to_frequency(ch.control_ch_num, band->band); + if (freq == 0) { + brcmf_err("Invalid frequency %d returned for channel %d, band %d. chanspec was %04x\n", + freq, ch.control_ch_num, ch.band, bi->chanspec); + goto done; + } notify_channel = ieee80211_get_channel(wiphy, freq); done: @@ -6462,11 +6426,16 @@ brcmf_bss_roaming_done(struct brcmf_cfg80211_info *cfg, cfg80211_roamed(ndev, &roam_info, GFP_KERNEL); brcmf_dbg(CONN, "Report roaming result\n"); - if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X && profile->is_ft) { - cfg80211_port_authorized(ndev, profile->bssid, NULL, 0, GFP_KERNEL); + if (((profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X || + profile->use_fwsup == BRCMF_PROFILE_FWSUP_ROAM) && + (brcmf_has_pmkid(roam_info.req_ie, roam_info.req_ie_len) || + profile->is_ft || profile->is_okc))) { + cfg80211_port_authorized(ndev, profile->bssid, NULL, 0, + GFP_KERNEL); brcmf_dbg(CONN, "Report port authorized\n"); } + clear_bit(BRCMF_VIF_STATUS_CONNECTING, &ifp->vif->sme_state); set_bit(BRCMF_VIF_STATUS_CONNECTED, &ifp->vif->sme_state); brcmf_dbg(TRACE, "Exit\n"); return err; @@ -6925,8 +6894,6 @@ static s32 brcmf_dongle_roam(struct brcmf_if *ifp) if (err) bphy_err(drvr, "WLC_SET_ROAM_TRIGGER error (%d)\n", err); - roam_delta[0] = cpu_to_le32(WL_ROAM_DELTA); - roam_delta[1] = cpu_to_le32(BRCM_BAND_ALL); err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SET_ROAM_DELTA, (void *)roam_delta, sizeof(roam_delta)); if (err) @@ -7019,15 +6986,34 @@ static int brcmf_construct_chaninfo(struct brcmf_cfg80211_info *cfg, goto fail_pbuf; } + /* Changing regulatory domain may change power limits upwards. + * To ensure that we correctly set the new band info, copy the original + * info first. + */ band = wiphy->bands[NL80211_BAND_2GHZ]; - if (band) + if (band) { + memcpy(band->channels, &__wl_2ghz_channels, + sizeof(__wl_2ghz_channels)); + band->n_channels = ARRAY_SIZE(__wl_2ghz_channels); for (i = 0; i < band->n_channels; i++) band->channels[i].flags = IEEE80211_CHAN_DISABLED; + } band = wiphy->bands[NL80211_BAND_5GHZ]; - if (band) + if (band) { + memcpy(band->channels, &__wl_5ghz_channels, + sizeof(__wl_5ghz_channels)); + band->n_channels = ARRAY_SIZE(__wl_5ghz_channels); for (i = 0; i < band->n_channels; i++) band->channels[i].flags = IEEE80211_CHAN_DISABLED; - + } + band = wiphy->bands[NL80211_BAND_6GHZ]; + if (band) { + memcpy(band->channels, &__wl_6ghz_channels, + sizeof(__wl_6ghz_channels)); + band->n_channels = ARRAY_SIZE(__wl_6ghz_channels); + for (i = 0; i < band->n_channels; i++) + band->channels[i].flags = IEEE80211_CHAN_DISABLED; + } total = le32_to_cpu(list->count); if (total > BRCMF_MAX_CHANSPEC_LIST) { bphy_err(drvr, "Invalid count of channel Spec. (%u)\n", @@ -7044,6 +7030,8 @@ static int brcmf_construct_chaninfo(struct brcmf_cfg80211_info *cfg, band = wiphy->bands[NL80211_BAND_2GHZ]; } else if (ch.band == BRCMU_CHAN_BAND_5G) { band = wiphy->bands[NL80211_BAND_5GHZ]; + } else if (ch.band == BRCMU_CHAN_BAND_6G) { + band = wiphy->bands[NL80211_BAND_6GHZ]; } else { bphy_err(drvr, "Invalid channel Spec. 0x%x.\n", ch.chspec); @@ -7065,6 +7053,7 @@ static int brcmf_construct_chaninfo(struct brcmf_cfg80211_info *cfg, break; } } + if (!channel) { /* It seems firmware supports some channel we never * considered. Something new in IEEE standard? @@ -7137,17 +7126,25 @@ static int brcmf_enable_bw40_2g(struct brcmf_cfg80211_info *cfg) struct brcmu_chan ch; u32 num_chan; int i, j; + s32 updown; /* verify support for bw_cap command */ - val = WLC_BAND_5G; + val = WLC_BAND_2G; err = brcmf_fil_iovar_int_query(ifp, "bw_cap", &val); - + brcmf_dbg(INFO, "Check bw_cap support:%d\n", err); if (!err) { + /* Setting the bw_cap is DOWN restricted. */ + updown = 0; + brcmf_fil_cmd_data_set(ifp, BRCMF_C_DOWN, &updown, sizeof(s32)); /* only set 2G bandwidth using bw_cap command */ band_bwcap.band = cpu_to_le32(WLC_BAND_2G); band_bwcap.bw_cap = cpu_to_le32(WLC_BW_CAP_40MHZ); err = brcmf_fil_iovar_data_set(ifp, "bw_cap", &band_bwcap, sizeof(band_bwcap)); + brcmf_dbg(INFO, "set bw_cap support:%d\n", err); + brcmf_c_set_joinpref_default(ifp); + updown = 1; + brcmf_fil_cmd_data_set(ifp, BRCMF_C_UP, &updown, sizeof(s32)); } else { brcmf_dbg(INFO, "fallback to mimo_bw_cap\n"); val = WLC_N_BW_40ALL; @@ -7209,7 +7206,7 @@ static int brcmf_enable_bw40_2g(struct brcmf_cfg80211_info *cfg) return err; } -static void brcmf_get_bwcap(struct brcmf_if *ifp, u32 bw_cap[]) +static void brcmf_get_bwcap(struct brcmf_if *ifp, u32 bw_cap[4], bool has_6g) { struct brcmf_pub *drvr = ifp->drvr; u32 band, mimo_bwcap; @@ -7217,17 +7214,29 @@ static void brcmf_get_bwcap(struct brcmf_if *ifp, u32 bw_cap[]) band = WLC_BAND_2G; err = brcmf_fil_iovar_int_query(ifp, "bw_cap", &band); - if (!err) { - bw_cap[NL80211_BAND_2GHZ] = band; - band = WLC_BAND_5G; - err = brcmf_fil_iovar_int_query(ifp, "bw_cap", &band); - if (!err) { - bw_cap[NL80211_BAND_5GHZ] = band; - return; - } - WARN_ON(1); + if (err) + goto fallback; + bw_cap[NL80211_BAND_2GHZ] = band; + band = WLC_BAND_5G; + err |= brcmf_fil_iovar_int_query(ifp, "bw_cap", &band); + if (err) + goto fallback; + bw_cap[NL80211_BAND_5GHZ] = band; + if (!has_6g) return; - } + band = WLC_BAND_6G; + err |= brcmf_fil_iovar_int_query(ifp, "bw_cap", &band); + /* Prior to the introduction of 6g, this function only + * did fallback in the case of 2g and 5g -failing. + * As mimo_bwcap does not have 6g bwcap info anyway, + * we keep that behavior. + */ + if (err) + return; + bw_cap[NL80211_BAND_6GHZ] = band; + return; +fallback: + brcmf_dbg(INFO, "fallback to mimo_bw_cap info\n"); err = brcmf_fil_iovar_int_get(ifp, "mimo_bw_cap", &mimo_bwcap); if (err) @@ -7251,8 +7260,11 @@ static void brcmf_get_bwcap(struct brcmf_if *ifp, u32 bw_cap[]) } static void brcmf_update_ht_cap(struct ieee80211_supported_band *band, - u32 bw_cap[2], u32 nchain) + u32 bw_cap[4], u32 nrxchain) { + /* Not supported in 6G band */ + if (band->band == NL80211_BAND_6GHZ) + return; band->ht_cap.ht_supported = true; if (bw_cap[band->band] & WLC_BW_40MHZ_BIT) { band->ht_cap.cap |= IEEE80211_HT_CAP_SGI_40; @@ -7262,32 +7274,49 @@ static void brcmf_update_ht_cap(struct ieee80211_supported_band *band, band->ht_cap.cap |= IEEE80211_HT_CAP_DSSSCCK40; band->ht_cap.ampdu_factor = IEEE80211_HT_MAX_AMPDU_64K; band->ht_cap.ampdu_density = IEEE80211_HT_MPDU_DENSITY_16; - memset(band->ht_cap.mcs.rx_mask, 0xff, nchain); + memset(band->ht_cap.mcs.rx_mask, 0xff, nrxchain); band->ht_cap.mcs.tx_params = IEEE80211_HT_MCS_TX_DEFINED; } -static __le16 brcmf_get_mcs_map(u32 nchain, enum ieee80211_vht_mcs_support supp) +static __le16 brcmf_get_mcs_map(u32 nstreams, + enum ieee80211_vht_mcs_support supp) { u16 mcs_map; int i; - for (i = 0, mcs_map = 0xFFFF; i < nchain; i++) + for (i = 0, mcs_map = 0xFFFF; i < nstreams; i++) mcs_map = (mcs_map << 2) | supp; return cpu_to_le16(mcs_map); } static void brcmf_update_vht_cap(struct ieee80211_supported_band *band, - u32 bw_cap[2], u32 nchain, u32 txstreams, - u32 txbf_bfe_cap, u32 txbf_bfr_cap) + u32 bw_cap[4], u32 txstreams, u32 rxstreams, + u32 txbf_bfe_cap, u32 txbf_bfr_cap, + u32 ldpc_cap, u32 stbc_rx, u32 stbc_tx) { __le16 mcs_map; - /* not allowed in 2.4G band */ - if (band->band == NL80211_BAND_2GHZ) + /* not allowed in 2.4G or 6G band */ + if (band->band == NL80211_BAND_2GHZ || band->band == NL80211_BAND_6GHZ) return; band->vht_cap.vht_supported = true; + band->vht_cap.vht_mcs.tx_highest = cpu_to_le16(433 * txstreams); + band->vht_cap.vht_mcs.rx_highest = cpu_to_le16(433 * rxstreams); + + band->vht_cap.cap |= IEEE80211_VHT_CAP_RX_ANTENNA_PATTERN | + IEEE80211_VHT_CAP_TX_ANTENNA_PATTERN; + + if (ldpc_cap) + band->vht_cap.cap |= IEEE80211_VHT_CAP_RXLDPC; + if (stbc_tx) + band->vht_cap.cap |= IEEE80211_VHT_CAP_TXSTBC; + + if (stbc_rx) + band->vht_cap.cap |= + (stbc_rx << IEEE80211_VHT_CAP_RXSTBC_SHIFT); + /* 80MHz is mandatory */ band->vht_cap.cap |= IEEE80211_VHT_CAP_SHORT_GI_80; if (bw_cap[band->band] & WLC_BW_160MHZ_BIT) { @@ -7295,8 +7324,10 @@ static void brcmf_update_vht_cap(struct ieee80211_supported_band *band, band->vht_cap.cap |= IEEE80211_VHT_CAP_SHORT_GI_160; } /* all support 256-QAM */ - mcs_map = brcmf_get_mcs_map(nchain, IEEE80211_VHT_MCS_SUPPORT_0_9); + mcs_map = brcmf_get_mcs_map(rxstreams, IEEE80211_VHT_MCS_SUPPORT_0_9); band->vht_cap.vht_mcs.rx_mcs_map = mcs_map; + mcs_map = brcmf_get_mcs_map(txstreams, IEEE80211_VHT_MCS_SUPPORT_0_9); + band->vht_cap.vht_mcs.tx_mcs_map = mcs_map; /* Beamforming support information */ @@ -7312,11 +7343,129 @@ static void brcmf_update_vht_cap(struct ieee80211_supported_band *band, if ((txbf_bfe_cap || txbf_bfr_cap) && (txstreams > 1)) { band->vht_cap.cap |= (2 << IEEE80211_VHT_CAP_BEAMFORMEE_STS_SHIFT); - band->vht_cap.cap |= ((txstreams - 1) << - IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_SHIFT); + band->vht_cap.cap |= + ((txstreams - 1) + << IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_SHIFT); band->vht_cap.cap |= IEEE80211_VHT_CAP_VHT_LINK_ADAPTATION_VHT_MRQ_MFB; } + /* AMPDU length limit, support max 1MB (2 ^ (13 + 7)) */ + band->vht_cap.cap |= + (7 << IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_SHIFT); +} + +static void brcmf_update_he_cap(struct ieee80211_supported_band *band, + struct ieee80211_sband_iftype_data *data) +{ + int idx = 1; + struct ieee80211_sta_he_cap *he_cap = &data->he_cap; + struct ieee80211_he_cap_elem *he_cap_elem = &he_cap->he_cap_elem; + struct ieee80211_he_mcs_nss_supp *he_mcs = &he_cap->he_mcs_nss_supp; + struct ieee80211_he_6ghz_capa *he_6ghz_capa = &data->he_6ghz_capa; + + if (!data) { + brcmf_err("failed to allocate sdata\n"); + return; + } + + data->types_mask = BIT(NL80211_IFTYPE_STATION); + he_cap->has_he = true; + + /* HE MAC Capabilities Information */ + he_cap_elem->mac_cap_info[0] = IEEE80211_HE_MAC_CAP0_HTC_HE | + IEEE80211_HE_MAC_CAP0_TWT_REQ | + IEEE80211_HE_MAC_CAP0_TWT_RES; + + he_cap_elem->mac_cap_info[1] = + IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_8US | + IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US; + + he_cap_elem->mac_cap_info[2] = IEEE80211_HE_MAC_CAP2_BSR | + IEEE80211_HE_MAC_CAP2_BCAST_TWT; + + he_cap_elem->mac_cap_info[3] = + IEEE80211_HE_MAC_CAP3_OMI_CONTROL | + IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_1 | + IEEE80211_HE_MAC_CAP3_FLEX_TWT_SCHED; + + he_cap_elem->mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU; + + /* HE PHY Capabilities Information */ + he_cap_elem->phy_cap_info[0] = + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G | + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G | + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G; + ; + + he_cap_elem->phy_cap_info[1] = + IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD; + + he_cap_elem->phy_cap_info[2] = + IEEE80211_HE_PHY_CAP2_NDP_4x_LTF_AND_3_2US | + IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO | + IEEE80211_HE_PHY_CAP2_UL_MU_PARTIAL_MU_MIMO; + + he_cap_elem->phy_cap_info[3] = + IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_TX_QPSK | + IEEE80211_HE_PHY_CAP3_DCM_MAX_TX_NSS_2 | + IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_RX_16_QAM | + IEEE80211_HE_PHY_CAP3_SU_BEAMFORMER; + + he_cap_elem->phy_cap_info[4] = + IEEE80211_HE_PHY_CAP4_SU_BEAMFORMEE | + IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_UNDER_80MHZ_MASK | + IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_ABOVE_80MHZ_4 | + IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_UNDER_80MHZ_8; + + he_cap_elem->phy_cap_info[5] = + IEEE80211_HE_PHY_CAP5_NG16_SU_FEEDBACK | + IEEE80211_HE_PHY_CAP5_NG16_MU_FEEDBACK | + IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_2; + + he_cap_elem->phy_cap_info[6] = + IEEE80211_HE_PHY_CAP6_CODEBOOK_SIZE_42_SU | + IEEE80211_HE_PHY_CAP6_CODEBOOK_SIZE_75_MU | + IEEE80211_HE_PHY_CAP6_TRIG_SU_BEAMFORMING_FB | + IEEE80211_HE_PHY_CAP6_TRIG_MU_BEAMFORMING_PARTIAL_BW_FB | + IEEE80211_HE_PHY_CAP6_TRIG_CQI_FB | + IEEE80211_HE_PHY_CAP6_PARTIAL_BW_EXT_RANGE | + IEEE80211_HE_PHY_CAP6_PPE_THRESHOLD_PRESENT; + + he_cap_elem->phy_cap_info[7] = + IEEE80211_HE_PHY_CAP7_HE_SU_MU_PPDU_4XLTF_AND_08_US_GI | + IEEE80211_HE_PHY_CAP7_MAX_NC_1; + + he_cap_elem->phy_cap_info[8] = + IEEE80211_HE_PHY_CAP8_HE_ER_SU_PPDU_4XLTF_AND_08_US_GI | + IEEE80211_HE_PHY_CAP8_20MHZ_IN_40MHZ_HE_PPDU_IN_2G | + IEEE80211_HE_PHY_CAP8_20MHZ_IN_160MHZ_HE_PPDU | + IEEE80211_HE_PHY_CAP8_80MHZ_IN_160MHZ_HE_PPDU; + + he_cap_elem->phy_cap_info[9] = + IEEE80211_HE_PHY_CAP9_TX_1024_QAM_LESS_THAN_242_TONE_RU | + IEEE80211_HE_PHY_CAP9_RX_1024_QAM_LESS_THAN_242_TONE_RU | + IEEE80211_HE_PHY_CAP9_RX_FULL_BW_SU_USING_MU_WITH_COMP_SIGB | + IEEE80211_HE_PHY_CAP9_RX_FULL_BW_SU_USING_MU_WITH_NON_COMP_SIGB; + + /* HE Supported MCS and NSS Set */ + he_mcs->rx_mcs_80 = cpu_to_le16(0xfffa); + he_mcs->tx_mcs_80 = cpu_to_le16(0xfffa); + he_mcs->rx_mcs_160 = cpu_to_le16(0xfffa); + he_mcs->tx_mcs_160 = cpu_to_le16(0xfffa); + /* HE 6 GHz band capabilities */ + if (band->band == NL80211_BAND_6GHZ) { + u16 capa = 0; + + capa = FIELD_PREP(IEEE80211_HE_6GHZ_CAP_MIN_MPDU_START, + IEEE80211_HT_MPDU_DENSITY_8) | + FIELD_PREP(IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP, + IEEE80211_VHT_MAX_AMPDU_1024K) | + FIELD_PREP(IEEE80211_HE_6GHZ_CAP_MAX_MPDU_LEN, + IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454); + he_6ghz_capa->capa = cpu_to_le16(capa); + } + band->n_iftype_data = idx; + band->iftype_data = data; } static int brcmf_setup_wiphybands(struct brcmf_cfg80211_info *cfg) @@ -7326,26 +7475,49 @@ static int brcmf_setup_wiphybands(struct brcmf_cfg80211_info *cfg) struct wiphy *wiphy = cfg_to_wiphy(cfg); u32 nmode; u32 vhtmode = 0; - u32 bw_cap[2] = { WLC_BW_20MHZ_BIT, WLC_BW_20MHZ_BIT }; + /* 2GHZ, 5GHZ, 60GHZ, 6GHZ */ + u32 bw_cap[4] = { 0, 0, 0, 0 }; u32 rxchain; - u32 nchain; + u32 txchain; + u32 nrxchain; + u32 ntxchain; int err; s32 i; struct ieee80211_supported_band *band; u32 txstreams = 0; + u32 rxstreams = 0; u32 txbf_bfe_cap = 0; u32 txbf_bfr_cap = 0; + u8 he_enable; + struct brcmf_he_defcap he_cap; + u32 ldpc_cap = 0; + u32 stbc_rx = 0; + u32 stbc_tx = 0; (void)brcmf_fil_iovar_int_get(ifp, "vhtmode", &vhtmode); + (void)brcmf_fil_iovar_int_get(ifp, "ldpc_cap", &ldpc_cap); + (void)brcmf_fil_iovar_int_get(ifp, "stbc_rx", &stbc_rx); + (void)brcmf_fil_iovar_int_get(ifp, "stbc_tx", &stbc_tx); + err = brcmf_fil_xtlv_int8_get(ifp, "he", BRCMF_HE_CMD_ENABLE, + &he_enable); + if (!err && he_enable) { + brcmf_fil_xtlv_data_get(ifp, "he", BRCMF_HE_CMD_DEFCAP, &he_cap, + sizeof(he_cap)); + brcmf_dbg_hex_dump(BRCMF_INFO_ON(), he_cap.mac_cap, 6, + "default HE mac cap\n"); + brcmf_dbg_hex_dump(BRCMF_INFO_ON(), he_cap.phy_cap, 11, + "default HE phy cap\n"); + } err = brcmf_fil_iovar_int_get(ifp, "nmode", &nmode); if (err) { bphy_err(drvr, "nmode error (%d)\n", err); - } else { - brcmf_get_bwcap(ifp, bw_cap); } - brcmf_dbg(INFO, "nmode=%d, vhtmode=%d, bw_cap=(%d, %d)\n", + brcmf_get_bwcap(ifp, bw_cap, he_enable != 0); + brcmf_dbg(INFO, + "nmode=%d, vhtmode=%d, bw_cap=(%d, %d, %d), he_enable=%d\n", nmode, vhtmode, bw_cap[NL80211_BAND_2GHZ], - bw_cap[NL80211_BAND_5GHZ]); + bw_cap[NL80211_BAND_5GHZ], bw_cap[NL80211_BAND_6GHZ], + he_enable); err = brcmf_fil_iovar_int_get(ifp, "rxchain", &rxchain); if (err) { @@ -7355,12 +7527,31 @@ static int brcmf_setup_wiphybands(struct brcmf_cfg80211_info *cfg) else bphy_err(drvr, "rxchain error (%d)\n", err); - nchain = 1; + nrxchain = 1; + rxchain = 1; } else { - for (nchain = 0; rxchain; nchain++) + for (nrxchain = 0; rxchain; nrxchain++) rxchain = rxchain & (rxchain - 1); } - brcmf_dbg(INFO, "nchain=%d\n", nchain); + brcmf_dbg(INFO, "nrxchain=%d\n", nrxchain); + err = brcmf_fil_iovar_int_get(ifp, "txchain", &txchain); + if (err) { + /* rxchain unsupported by firmware of older chips */ + if (err == -EBADE) + bphy_info_once(drvr, "rxchain unsupported\n"); + else + bphy_err(drvr, "rxchain error (%d)\n", err); + + ntxchain = 1; + txchain = 1; + } else { + for (ntxchain = 0; txchain; ntxchain++) + txchain = txchain & (txchain - 1); + } + brcmf_dbg(INFO, "ntxchain=%d\n", ntxchain); + + wiphy->available_antennas_rx = nrxchain; + wiphy->available_antennas_tx = ntxchain; err = brcmf_construct_chaninfo(cfg, bw_cap); if (err) { @@ -7369,6 +7560,7 @@ static int brcmf_setup_wiphybands(struct brcmf_cfg80211_info *cfg) } if (vhtmode) { + (void)brcmf_fil_iovar_int_get(ifp, "rxstreams", &rxstreams); (void)brcmf_fil_iovar_int_get(ifp, "txstreams", &txstreams); (void)brcmf_fil_iovar_int_get(ifp, "txbf_bfe_cap", &txbf_bfe_cap); @@ -7382,10 +7574,13 @@ static int brcmf_setup_wiphybands(struct brcmf_cfg80211_info *cfg) continue; if (nmode) - brcmf_update_ht_cap(band, bw_cap, nchain); + brcmf_update_ht_cap(band, bw_cap, nrxchain); if (vhtmode) - brcmf_update_vht_cap(band, bw_cap, nchain, txstreams, - txbf_bfe_cap, txbf_bfr_cap); + brcmf_update_vht_cap(band, bw_cap, txstreams, rxstreams, + txbf_bfe_cap, txbf_bfr_cap, + ldpc_cap, stbc_rx, stbc_tx); + if (he_enable) + brcmf_update_he_cap(band, &sdata[band->band]); } return 0; @@ -7640,7 +7835,7 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) struct ieee80211_supported_band *band; u16 max_interfaces = 0; bool gscan; - __le32 bandlist[3]; + __le32 bandlist[16]; u32 n_bands; int err, i; @@ -7708,6 +7903,13 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) } if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_SAE_EXT)) wiphy->features |= NL80211_FEATURE_SAE; + + /* High accuracy and low power scans are always supported. */ + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_HIGH_ACCURACY_SCAN); + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_LOW_POWER_SCAN); + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_LOW_SPAN_SCAN); + wiphy->features |= NL80211_FEATURE_LOW_PRIORITY_SCAN; + wiphy->mgmt_stypes = brcmf_txrx_stypes; wiphy->max_remain_on_channel_duration = 5000; if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_PNO)) { @@ -7763,12 +7965,27 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) band->n_channels = ARRAY_SIZE(__wl_5ghz_channels); wiphy->bands[NL80211_BAND_5GHZ] = band; } - } + if (bandlist[i] == cpu_to_le32(WLC_BAND_6G)) { + band = kmemdup(&__wl_band_6ghz, sizeof(__wl_band_6ghz), + GFP_KERNEL); + if (!band) + return -ENOMEM; + + band->channels = kmemdup(&__wl_6ghz_channels, + sizeof(__wl_6ghz_channels), + GFP_KERNEL); + if (!band->channels) { + kfree(band); + return -ENOMEM; + } + band->n_channels = ARRAY_SIZE(__wl_6ghz_channels); + wiphy->bands[NL80211_BAND_6GHZ] = band; + } + } if (wiphy->bands[NL80211_BAND_5GHZ] && brcmf_feat_is_enabled(ifp, BRCMF_FEAT_DOT11H)) - wiphy_ext_feature_set(wiphy, - NL80211_EXT_FEATURE_DFS_OFFLOAD); + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_DFS_OFFLOAD); wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_CQM_RSSI_LIST); @@ -8267,9 +8484,17 @@ static void brcmf_cfg80211_reg_notifier(struct wiphy *wiphy, } err = brcmf_translate_country_code(ifp->drvr, req->alpha2, &ccreq); - if (err) - return; - + if (err) { + /* Because we ignore the default country code above, + * we will start out in our custom reg domain, but the chip + * may already be set to the right country. + * As such, we force the bands to be re-set the first + * time we try to set a country for real. + */ + if (err != -EAGAIN || !cfg->force_band_setup) + return; + } + cfg->force_band_setup = false; err = brcmf_fil_iovar_data_set(ifp, "country", &ccreq, sizeof(ccreq)); if (err) { bphy_err(drvr, "Firmware rejected country setting\n"); @@ -8298,6 +8523,10 @@ static void brcmf_free_wiphy(struct wiphy *wiphy) kfree(wiphy->bands[NL80211_BAND_5GHZ]->channels); kfree(wiphy->bands[NL80211_BAND_5GHZ]); } + if (wiphy->bands[NL80211_BAND_6GHZ]) { + kfree(wiphy->bands[NL80211_BAND_6GHZ]->channels); + kfree(wiphy->bands[NL80211_BAND_6GHZ]); + } #if IS_ENABLED(CONFIG_PM) if (wiphy->wowlan != &brcmf_wowlan_support) kfree(wiphy->wowlan); @@ -8332,6 +8561,7 @@ struct brcmf_cfg80211_info *brcmf_cfg80211_attach(struct brcmf_pub *drvr, cfg->pub = drvr; init_vif_event(&cfg->vif_event); INIT_LIST_HEAD(&cfg->vif_list); + cfg->force_band_setup = true; vif = brcmf_alloc_vif(cfg, NL80211_IFTYPE_STATION); if (IS_ERR(vif)) @@ -8389,18 +8619,21 @@ struct brcmf_cfg80211_info *brcmf_cfg80211_attach(struct brcmf_pub *drvr, if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_DUMP_OBSS)) ops->dump_survey = brcmf_cfg80211_dump_survey; - err = wiphy_register(wiphy); - if (err < 0) { - bphy_err(drvr, "Could not register wiphy device (%d)\n", err); - goto priv_out; - } - + /* We have to configure the bands before we register the wiphy device + * because it requires that band capabilities be correct. + */ err = brcmf_setup_wiphybands(cfg); if (err) { bphy_err(drvr, "Setting wiphy bands failed (%d)\n", err); goto wiphy_unreg_out; } + err = wiphy_register(wiphy); + if (err < 0) { + bphy_err(drvr, "Could not register wiphy device (%d)\n", err); + goto priv_out; + } + /* If cfg80211 didn't disable 40MHz HT CAP in wiphy_register(), * setup 40MHz in 2GHz band and enable OBSS scanning. */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index 273c80f2d483a3..11f651ab9f4228 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -8,6 +8,7 @@ /* for brcmu_d11inf */ #include +#include #include "core.h" #include "fwil_types.h" @@ -127,7 +128,8 @@ enum brcmf_profile_fwsup { BRCMF_PROFILE_FWSUP_NONE, BRCMF_PROFILE_FWSUP_PSK, BRCMF_PROFILE_FWSUP_1X, - BRCMF_PROFILE_FWSUP_SAE + BRCMF_PROFILE_FWSUP_SAE, + BRCMF_PROFILE_FWSUP_ROAM }; /** @@ -172,6 +174,7 @@ struct brcmf_cfg80211_profile { enum brcmf_profile_fwsup use_fwsup; u16 use_fwauth; bool is_ft; + bool is_okc; }; /** @@ -350,6 +353,7 @@ struct brcmf_cfg80211_wowl { * @dongle_up: indicate whether dongle up or not. * @roam_on: on/off switch for dongle self-roaming. * @scan_tried: indicates if first scan attempted. + * @force_band_setup: indicates if we should force band setup * @dcmd_buf: dcmd buffer. * @extra_buf: mainly to grab assoc information. * @debugfsdir: debugfs folder for this device. @@ -380,6 +384,7 @@ struct brcmf_cfg80211_info { bool pwr_save; bool dongle_up; bool scan_tried; + bool force_band_setup; u8 *dcmd_buf; u8 *extra_buf; struct dentry *debugfsdir; @@ -409,6 +414,22 @@ struct brcmf_tlv { u8 data[]; }; +static inline enum nl80211_band fwil_band_to_nl80211(u16 band) +{ + switch (band) { + case WLC_BAND_2G: + return NL80211_BAND_2GHZ; + case WLC_BAND_5G: + return NL80211_BAND_5GHZ; + case WLC_BAND_6G: + return NL80211_BAND_6GHZ; + default: + WARN_ON(1); + break; + } + return 0; +} + static inline struct wiphy *cfg_to_wiphy(struct brcmf_cfg80211_info *cfg) { return cfg->wiphy; @@ -476,6 +497,8 @@ s32 brcmf_vif_set_mgmt_ie(struct brcmf_cfg80211_vif *vif, s32 pktflag, s32 brcmf_vif_clear_mgmt_ies(struct brcmf_cfg80211_vif *vif); u16 channel_to_chanspec(struct brcmu_d11inf *d11inf, struct ieee80211_channel *ch); +u16 chandef_to_chanspec(struct brcmu_d11inf *d11inf, + struct cfg80211_chan_def *ch); bool brcmf_get_vif_state_any(struct brcmf_cfg80211_info *cfg, unsigned long state); void brcmf_cfg80211_arm_vif_event(struct brcmf_cfg80211_info *cfg, diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c index 4239f2b21e5423..75b6bfa28b6c29 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c @@ -162,6 +162,15 @@ struct sbconfig { #define SRCI_SRBSZ_SHIFT 0 #define SR_BSZ_BASE 14 +#define SYSMEM_SRCI_ROMNB_MASK 0x3e0 +#define SYSMEM_SRCI_ROMNB_SHIFT 5 +#define SYSMEM_SRCI_SRNB_MASK 0x1f +#define SYSMEM_SRCI_SRNB_SHIFT 0 +#define SYSMEM_SRCI_NEW_ROMNB_MASK 0xff000000 +#define SYSMEM_SRCI_NEW_ROMNB_SHIFT 24 +#define SYSMEM_SRCI_NEW_SRNB_MASK 0xff0000 +#define SYSMEM_SRCI_NEW_SRNB_SHIFT 16 + struct sbsocramregs { u32 coreinfo; u32 bwalloc; @@ -436,25 +445,11 @@ static void brcmf_chip_ai_resetcore(struct brcmf_core_priv *core, u32 prereset, { struct brcmf_chip_priv *ci; int count; - struct brcmf_core *d11core2 = NULL; - struct brcmf_core_priv *d11priv2 = NULL; ci = core->chip; - /* special handle two D11 cores reset */ - if (core->pub.id == BCMA_CORE_80211) { - d11core2 = brcmf_chip_get_d11core(&ci->pub, 1); - if (d11core2) { - brcmf_dbg(INFO, "found two d11 cores, reset both\n"); - d11priv2 = container_of(d11core2, - struct brcmf_core_priv, pub); - } - } - /* must disable first to work for arbitrary current core state */ brcmf_chip_ai_coredisable(core, prereset, reset); - if (d11priv2) - brcmf_chip_ai_coredisable(d11priv2, prereset, reset); count = 0; while (ci->ops->read32(ci->ctx, core->wrapbase + BCMA_RESET_CTL) & @@ -466,30 +461,9 @@ static void brcmf_chip_ai_resetcore(struct brcmf_core_priv *core, u32 prereset, usleep_range(40, 60); } - if (d11priv2) { - count = 0; - while (ci->ops->read32(ci->ctx, - d11priv2->wrapbase + BCMA_RESET_CTL) & - BCMA_RESET_CTL_RESET) { - ci->ops->write32(ci->ctx, - d11priv2->wrapbase + BCMA_RESET_CTL, - 0); - count++; - if (count > 50) - break; - usleep_range(40, 60); - } - } - ci->ops->write32(ci->ctx, core->wrapbase + BCMA_IOCTL, postreset | BCMA_IOCTL_CLK); ci->ops->read32(ci->ctx, core->wrapbase + BCMA_IOCTL); - - if (d11priv2) { - ci->ops->write32(ci->ctx, d11priv2->wrapbase + BCMA_IOCTL, - postreset | BCMA_IOCTL_CLK); - ci->ops->read32(ci->ctx, d11priv2->wrapbase + BCMA_IOCTL); - } } char *brcmf_chip_name(u32 id, u32 rev, char *buf, uint len) @@ -659,6 +633,7 @@ static u32 brcmf_chip_sysmem_ramsize(struct brcmf_core_priv *sysmem) u32 memsize = 0; u32 coreinfo; u32 idx; + u32 nrb; u32 nb; u32 banksize; @@ -666,10 +641,16 @@ static u32 brcmf_chip_sysmem_ramsize(struct brcmf_core_priv *sysmem) brcmf_chip_resetcore(&sysmem->pub, 0, 0, 0); coreinfo = brcmf_chip_core_read32(sysmem, SYSMEMREGOFFS(coreinfo)); - nb = (coreinfo & SRCI_SRNB_MASK) >> SRCI_SRNB_SHIFT; + if (sysmem->pub.rev >= 12) { + nrb = (coreinfo & SYSMEM_SRCI_NEW_ROMNB_MASK) >> SYSMEM_SRCI_NEW_ROMNB_SHIFT; + nb = (coreinfo & SYSMEM_SRCI_NEW_SRNB_MASK) >> SYSMEM_SRCI_NEW_SRNB_SHIFT; + } else { + nrb = (coreinfo & SYSMEM_SRCI_ROMNB_MASK) >> SYSMEM_SRCI_ROMNB_SHIFT; + nb = (coreinfo & SYSMEM_SRCI_SRNB_MASK) >> SYSMEM_SRCI_SRNB_SHIFT; + } for (idx = 0; idx < nb; idx++) { - brcmf_chip_socram_banksize(sysmem, idx, &banksize); + brcmf_chip_socram_banksize(sysmem, idx + nrb, &banksize); memsize += banksize; } @@ -731,6 +712,7 @@ static u32 brcmf_chip_tcm_rambase(struct brcmf_chip_priv *ci) case BRCM_CC_4366_CHIP_ID: case BRCM_CC_43664_CHIP_ID: case BRCM_CC_43666_CHIP_ID: + case BRCM_CC_4388_CHIP_ID: return 0x200000; case BRCM_CC_4355_CHIP_ID: case BRCM_CC_4359_CHIP_ID: @@ -1338,14 +1320,15 @@ static inline void brcmf_chip_ca7_set_passive(struct brcmf_chip_priv *chip) { struct brcmf_core *core; + int i; brcmf_chip_disable_arm(chip, BCMA_CORE_ARM_CA7); - core = brcmf_chip_get_core(&chip->pub, BCMA_CORE_80211); - brcmf_chip_resetcore(core, D11_BCMA_IOCTL_PHYRESET | - D11_BCMA_IOCTL_PHYCLOCKEN, - D11_BCMA_IOCTL_PHYCLOCKEN, - D11_BCMA_IOCTL_PHYCLOCKEN); + /* Disable the cores only and let the firmware enable them. */ + for (i = 0; (core = brcmf_chip_get_d11core(&chip->pub, i)); i++) + brcmf_chip_coredisable(core, D11_BCMA_IOCTL_PHYRESET | + D11_BCMA_IOCTL_PHYCLOCKEN, + D11_BCMA_IOCTL_PHYCLOCKEN); } static bool brcmf_chip_ca7_set_active(struct brcmf_chip_priv *chip, u32 rstvec) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index 688f16c513192d..b80e0b4ad422ab 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -13,6 +13,7 @@ #include "core.h" #include "bus.h" #include "debug.h" +#include "fweh.h" #include "fwil.h" #include "fwil_types.h" #include "tracepoint.h" @@ -266,7 +267,6 @@ static int brcmf_c_process_cal_blob(struct brcmf_if *ifp) int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) { struct brcmf_pub *drvr = ifp->drvr; - struct brcmf_fweh_info *fweh = drvr->fweh; u8 buf[BRCMF_DCMD_SMLEN]; struct brcmf_bus *bus; struct brcmf_rev_info_le revinfo; @@ -412,27 +412,6 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) brcmf_c_set_joinpref_default(ifp); - /* Setup event_msgs, enable E_IF */ - err = brcmf_fil_iovar_data_get(ifp, "event_msgs", fweh->event_mask, - fweh->event_mask_len); - if (err) { - bphy_err(drvr, "Get event_msgs error (%d)\n", err); - goto done; - } - /* - * BRCMF_E_IF can safely be used to set the appropriate bit - * in the event_mask as the firmware event code is guaranteed - * to match the value of BRCMF_E_IF because it is old cruft - * that all vendors have. - */ - setbit(fweh->event_mask, BRCMF_E_IF); - err = brcmf_fil_iovar_data_set(ifp, "event_msgs", fweh->event_mask, - fweh->event_mask_len); - if (err) { - bphy_err(drvr, "Set event_msgs error (%d)\n", err); - goto done; - } - /* Setup default scan channel time */ err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_SCAN_CHANNEL_TIME, BRCMF_DEFAULT_SCAN_CHANNEL_TIME); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index 862a0336a0b591..3ef91663863da8 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -1227,7 +1227,14 @@ static int brcmf_bus_started(struct brcmf_pub *drvr, struct cfg80211_ops *ops) if (ret < 0) goto fail; - brcmf_feat_attach(drvr); + ret = brcmf_feat_attach(drvr); + if (ret) + goto fail; + + /* Setup event_msgs, enable E_IF */ + ret = brcmf_fweh_init_events(ifp); + if (ret) + goto fail; ret = brcmf_proto_init_done(drvr); if (ret < 0) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h index 399b6810e394de..4b52a3aa855de8 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h @@ -97,6 +97,68 @@ struct brcmf_rev_info { u32 nvramrev; }; +struct brcmf_pno_info; +enum nl80211_band; +/** + * struct pno_struct_handler + */ +struct pno_struct_handler { + u8 version; + int (*pno_config)(struct brcmf_if *ifp, u32 scan_freq, u32 mscan, + u32 bestn); + u32 (*get_min_data_len)(void); + u32 (*get_result_count)(void *data); + u32 (*get_result_status)(void *data); + int (*validate_pfn_results)(void *data, u32 event_datalen); + u32 (*get_bucket_map)(void *data, int idx, struct brcmf_pno_info *pi); + int (*get_result_info)(void *data, int result_idx, + u8 (*ssid)[IEEE80211_MAX_SSID_LEN], u8 *ssid_len, + u8 *channel, enum nl80211_band *band); +}; + +struct cfg80211_scan_request; +struct scan_param_struct_handler { + u8 version; + void *(*get_struct_for_request)(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_scan_request *request); +}; + +struct cfg80211_ibss_params; +struct cfg80211_connect_params; + +/** + * struct join_param_struct_handler - Handler for different join parameter versions + * + * There are a number of different, incompatible structures and interface versions for join/extended join parameters + * We abstract away the actual structures used, so that code does not have to worry about filling in structs properly. + * + * This interface deliberately takes and returns opaque structures. + * + * @version - Interface version the firmware supports/uses + * @get_struct_for_ibss - Return a join parameter structure for a set of IBSS parameters. + * This structure can be used to join the passed BSS. + * @get_struct_for_connect - Return an extended join parameter structure for a set of connect + * parameters. This structure can be used to join the SSID specified in the parameters. + * @get_join_from_ext_join - When an extended join does not work, we fall back to a regular join. + * This function produces a join parameter struture from an extended join one. + */ +struct join_param_struct_handler { + u8 version; + /* This returns a join_param type struct */ + void *(*get_struct_for_ibss)(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_ibss_params *params); + /* This returns an ext_join_param type struct */ + void *(*get_struct_for_connect)(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_connect_params *params); + /* This returns the join param portion of an ext_join_param type struct. + * The memory returned is separately allocated from the passed-in struct. + */ + void *(*get_join_from_ext_join)(void *ext_join_param, u32 *struct_size); +}; + /* Common structure for module and instance linkage */ struct brcmf_pub { /* Linkage ponters */ @@ -145,6 +207,10 @@ struct brcmf_pub { u8 sta_mac_idx; const struct brcmf_fwvid_ops *vops; void *vdata; + u16 cnt_ver; + struct pno_struct_handler pno_handler; + struct scan_param_struct_handler scan_param_handler; + struct join_param_struct_handler join_param_handler; }; /* forward declarations */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h index 9bb5f709d41a27..432d93ae8fb854 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h @@ -85,6 +85,7 @@ do { \ #define BRCMF_FIL_ON() (brcmf_msg_level & BRCMF_FIL_VAL) #define BRCMF_FWCON_ON() (brcmf_msg_level & BRCMF_FWCON_VAL) #define BRCMF_SCAN_ON() (brcmf_msg_level & BRCMF_SCAN_VAL) +#define BRCMF_INFO_ON() (brcmf_msg_level & BRCMF_INFO_VAL) #else /* defined(DEBUG) || defined(CONFIG_BRCM_TRACING) */ @@ -104,6 +105,7 @@ do { \ #define BRCMF_FIL_ON() 0 #define BRCMF_FWCON_ON() 0 #define BRCMF_SCAN_ON() 0 +#define BRCMF_INFO_ON() 0 #endif /* defined(DEBUG) || defined(CONFIG_BRCM_TRACING) */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c index 488364ef8ff2a1..d823ced048454a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c @@ -16,9 +16,24 @@ #include "fwvid.h" #include "feature.h" #include "common.h" +#include "pno.h" +#include "scan_param.h" +#include "join_param.h" #define BRCMF_FW_UNSUPPORTED 23 +/* MIN branch version supporting join iovar versioning */ +#define MIN_JOINEXT_V1_FW_MAJOR 17u +/* Branch/es supporting join iovar versioning prior to + * MIN_JOINEXT_V1_FW_MAJOR + */ +#define MIN_JOINEXT_V1_BR2_FW_MAJOR 16 +#define MIN_JOINEXT_V1_BR2_FW_MINOR 1 + +#define MIN_JOINEXT_V1_BR1_FW_MAJOR 14 +#define MIN_JOINEXT_V1_BR1_FW_MINOR_2 2 +#define MIN_JOINEXT_V1_BR1_FW_MINOR_4 4 + /* * expand feature list to array of feature strings. */ @@ -45,6 +60,7 @@ static const struct brcmf_feat_fwcap brcmf_fwcap_map[] = { { BRCMF_FEAT_SAE, "sae " }, { BRCMF_FEAT_FWAUTH, "idauth" }, { BRCMF_FEAT_SAE_EXT, "sae_ext" }, + { BRCMF_FEAT_GCMP, "gcmp"} }; #ifdef DEBUG @@ -136,7 +152,7 @@ struct brcmf_feat_wlcfeat { static const struct brcmf_feat_wlcfeat brcmf_feat_wlcfeat_map[] = { { 12, 0, BIT(BRCMF_FEAT_PMKID_V2) }, - { 13, 0, BIT(BRCMF_FEAT_PMKID_V3) }, + { 13, 0, BIT(BRCMF_FEAT_PMKID_V3) } }; static void brcmf_feat_wlc_version_overrides(struct brcmf_pub *drv) @@ -286,9 +302,12 @@ static int brcmf_feat_fwcap_debugfs_read(struct seq_file *seq, void *data) return 0; } -void brcmf_feat_attach(struct brcmf_pub *drvr) +int brcmf_feat_attach(struct brcmf_pub *drvr) { struct brcmf_if *ifp = brcmf_get_ifp(drvr, 0); + struct brcmf_join_version_le join_ver; + struct brcmf_scan_version_le scan_ver; + struct brcmf_pno_param_v3_le pno_params; struct brcmf_pno_macaddr_le pfn_mac; struct brcmf_gscan_config gscan_cfg; u32 wowl_cap; @@ -331,6 +350,7 @@ void brcmf_feat_attach(struct brcmf_pub *drvr) brcmf_feat_iovar_int_get(ifp, BRCMF_FEAT_TDLS, "tdls_enable"); brcmf_feat_iovar_int_get(ifp, BRCMF_FEAT_MFP, "mfp"); brcmf_feat_iovar_int_get(ifp, BRCMF_FEAT_DUMP_OBSS, "dump_obss"); + brcmf_feat_iovar_int_get(ifp, BRCMF_FEAT_EVENT_MSGS_EXT, "event_msgs_ext"); pfn_mac.version = BRCMF_PFN_MACADDR_CFG_VER; err = brcmf_fil_iovar_data_get(ifp, "pfn_macaddr", &pfn_mac, @@ -339,13 +359,71 @@ void brcmf_feat_attach(struct brcmf_pub *drvr) ifp->drvr->feat_flags |= BIT(BRCMF_FEAT_SCAN_RANDOM_MAC); brcmf_feat_iovar_int_get(ifp, BRCMF_FEAT_FWSUP, "sup_wpa"); - brcmf_feat_iovar_int_get(ifp, BRCMF_FEAT_SCAN_V2, "scan_ver"); + + err = brcmf_fil_iovar_data_get(ifp, "join_ver", &join_ver, sizeof(join_ver)); + if (!err) { + u16 ver = le16_to_cpu(join_ver.join_ver_major); + err = brcmf_join_param_setup_for_version(drvr, ver); + } else { + /* Default to version 0, unless it is one of the firmware branches + * that doesn't have a join_ver iovar but are still version 1 */ + u8 version = 0; + struct brcmf_wlc_version_le ver; + err = brcmf_fil_iovar_data_get(ifp, "wlc_ver", &ver, + sizeof(ver)); + if (!err) { + u16 major = le16_to_cpu(ver.wlc_ver_major); + u16 minor = le16_to_cpu(ver.wlc_ver_minor); + if (((major == MIN_JOINEXT_V1_BR1_FW_MAJOR) && + ((minor == MIN_JOINEXT_V1_BR1_FW_MINOR_2) || + (minor == MIN_JOINEXT_V1_BR1_FW_MINOR_4))) || + ((major == MIN_JOINEXT_V1_BR2_FW_MAJOR) && + (minor >= MIN_JOINEXT_V1_BR2_FW_MINOR)) || + (major >= MIN_JOINEXT_V1_FW_MAJOR)) { + version = 1; + } + } + err = brcmf_join_param_setup_for_version(drvr, version); + } + if (err) { + bphy_err(drvr, "Error setting up join structure handler: %d\n", + err); + return err; + } + err = brcmf_fil_iovar_data_get(ifp, "scan_ver", &scan_ver, + sizeof(scan_ver)); + if (!err) { + u16 ver = le16_to_cpu(scan_ver.scan_ver_major); + err = brcmf_scan_param_setup_for_version(drvr, ver); + } else { + /* Default to version 1. */ + err = brcmf_scan_param_setup_for_version(drvr, 1); + } + if (err) { + bphy_err(drvr, "Error setting up scan structure handler: %d\n", + err); + return err; + } + /* See what version of PFN scan is supported*/ + err = brcmf_fil_iovar_data_get(ifp, "pno_set", &pno_params, + sizeof(pno_params)); + if (!err) { + err = brcmf_pno_setup_for_version( + drvr, le16_to_cpu(pno_params.version)); + } else { + /* Default to version 2, supported by all chips we support. */ + err = brcmf_pno_setup_for_version(drvr, 2); + } + if (err) { + bphy_err(drvr, "Error setting up escan structure handler: %d\n", + err); + return err; + } brcmf_feat_wlc_version_overrides(drvr); brcmf_feat_firmware_overrides(drvr); brcmf_fwvid_feat_attach(ifp); - if (drvr->settings->feature_disable) { brcmf_dbg(INFO, "Features: 0x%02x, disable: 0x%02x\n", ifp->drvr->feat_flags, @@ -365,6 +443,7 @@ void brcmf_feat_attach(struct brcmf_pub *drvr) /* no quirks */ break; } + return 0; } void brcmf_feat_debugfs_create(struct brcmf_pub *drvr) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h index 31f8695ca41765..be271ca0fca588 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h @@ -30,8 +30,12 @@ * SAE: simultaneous authentication of equals * FWAUTH: Firmware authenticator * DUMP_OBSS: Firmware has capable to dump obss info to support ACS - * SCAN_V2: Version 2 scan params * SAE_EXT: SAE authentication handled by user-space supplicant + * PMKID_V2: Version 2 PMKID + * PMKID_V3: Version 3 PMKID + * EVENT_MSGS_EXT: Event messages extension + * JOIN_V1: Version 1 join struct + * GCMP: GCMP Cipher suite support */ #define BRCMF_FEAT_LIST \ BRCMF_FEAT_DEF(MBSS) \ @@ -56,10 +60,11 @@ BRCMF_FEAT_DEF(SAE) \ BRCMF_FEAT_DEF(FWAUTH) \ BRCMF_FEAT_DEF(DUMP_OBSS) \ - BRCMF_FEAT_DEF(SCAN_V2) \ BRCMF_FEAT_DEF(PMKID_V2) \ BRCMF_FEAT_DEF(PMKID_V3) \ - BRCMF_FEAT_DEF(SAE_EXT) + BRCMF_FEAT_DEF(SAE_EXT) \ + BRCMF_FEAT_DEF(EVENT_MSGS_EXT) \ + BRCMF_FEAT_DEF(GCMP) /* * Quirks: @@ -97,8 +102,10 @@ enum brcmf_feat_quirk { * brcmf_feat_attach() - determine features and quirks. * * @drvr: driver instance. + * + * Return: 0 in case of success, error code otherwise. */ -void brcmf_feat_attach(struct brcmf_pub *drvr); +int brcmf_feat_attach(struct brcmf_pub *drvr); /** * brcmf_feat_debugfs_create() - create debugfs entries. diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c index c2d98ee6652f3a..09819d233af97e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c @@ -11,8 +11,10 @@ #include "core.h" #include "debug.h" #include "tracepoint.h" +#include "feature.h" #include "fweh.h" #include "fwil.h" +#include "fwil_types.h" #include "proto.h" #include "bus.h" #include "fwvid.h" @@ -425,6 +427,67 @@ void brcmf_fweh_unregister(struct brcmf_pub *drvr, drvr->fweh->evt_handler[evt_handler_idx] = NULL; } +/** + * brcmf_fweh_init_events() - initialize event handling. + * + * @ifp: primary interface object. + */ +int brcmf_fweh_init_events(struct brcmf_if *ifp) +{ + struct brcmf_pub *drvr = ifp->drvr; + struct brcmf_eventmsgs_ext_le *eventmsgs; + size_t size = sizeof(*eventmsgs) + drvr->fweh->event_mask_len; + int err; + + eventmsgs = kzalloc(size, GFP_KERNEL); + if(!eventmsgs) + return -ENOMEM; + + eventmsgs->version = EVENTMSGS_VER; + eventmsgs->command = EVENTMSGS_NONE; + eventmsgs->len = drvr->fweh->event_mask_len; + eventmsgs->maxgetsize = drvr->fweh->event_mask_len; + + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_EVENT_MSGS_EXT)) + err = brcmf_fil_iovar_data_get(ifp, "event_msgs_ext", + eventmsgs, size); + else + err = brcmf_fil_iovar_data_get(ifp, "event_msgs", + drvr->fweh->event_mask, + drvr->fweh->event_mask_len); + + if (err) { + bphy_err(drvr, "Get event_msgs error (%d)\n", err); + kfree(eventmsgs); + return err; + } + + brcmf_dbg(EVENT, "Event mask len: driver=%d fw=%d\n", + drvr->fweh->event_mask_len, eventmsgs->len); + + /* want to handle IF event as well */ + brcmf_dbg(EVENT, "enable event IF\n"); + setbit(eventmsgs->mask, BRCMF_E_IF); + + eventmsgs->version = EVENTMSGS_VER; + eventmsgs->command = EVENTMSGS_SET_MASK; + eventmsgs->len = drvr->fweh->event_mask_len; + + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_EVENT_MSGS_EXT)) + err = brcmf_fil_iovar_data_set(ifp, "event_msgs_ext", + eventmsgs, size); + else + err = brcmf_fil_iovar_data_set(ifp, "event_msgs", + drvr->fweh->event_mask, + drvr->fweh->event_mask_len); + + if (err) + bphy_err(drvr, "Set event_msgs error (%d)\n", err); + + kfree(eventmsgs); + return err; +} + /** * brcmf_fweh_activate_events() - enables firmware events registered. * @@ -432,32 +495,47 @@ void brcmf_fweh_unregister(struct brcmf_pub *drvr, */ int brcmf_fweh_activate_events(struct brcmf_if *ifp) { - struct brcmf_fweh_info *fweh = ifp->drvr->fweh; - enum brcmf_fweh_event_code code; + struct brcmf_pub *drvr = ifp->drvr; + struct brcmf_eventmsgs_ext_le *eventmsgs; + size_t size = sizeof(*eventmsgs) + drvr->fweh->event_mask_len; int i, err; - memset(fweh->event_mask, 0, fweh->event_mask_len); - for (i = 0; i < fweh->num_event_codes; i++) { - if (fweh->evt_handler[i]) { - brcmf_fweh_map_fwevt_code(fweh, i, &code); + eventmsgs = kzalloc(size, GFP_KERNEL); + if(!eventmsgs) + return -ENOMEM; + + for (i = 0; i < drvr->fweh->num_event_codes; i++) { + if (drvr->fweh->evt_handler[i]) { brcmf_dbg(EVENT, "enable event %s\n", - brcmf_fweh_event_name(code)); - setbit(fweh->event_mask, i); + brcmf_fweh_event_name(i)); + setbit(eventmsgs->mask, i); } } /* want to handle IF event as well */ brcmf_dbg(EVENT, "enable event IF\n"); - setbit(fweh->event_mask, BRCMF_E_IF); + setbit(eventmsgs->mask, BRCMF_E_IF); + + eventmsgs->version = EVENTMSGS_VER; + eventmsgs->command = EVENTMSGS_SET_MASK; + eventmsgs->len = drvr->fweh->event_mask_len; /* allow per-vendor method to activate firmware events */ if (!brcmf_fwvid_activate_events(ifp)) return 0; - err = brcmf_fil_iovar_data_set(ifp, "event_msgs", fweh->event_mask, - fweh->event_mask_len); + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_EVENT_MSGS_EXT)) + err = brcmf_fil_iovar_data_set(ifp, "event_msgs_ext", + eventmsgs, size); + else + err = brcmf_fil_iovar_data_set(ifp, "event_msgs", + drvr->fweh->event_mask, + drvr->fweh->event_mask_len); + if (err) - bphy_err(fweh->drvr, "Set event_msgs error (%d)\n", err); + bphy_err(drvr, "Set event_msgs error (%d)\n", err); + + kfree(eventmsgs); return err; } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h index e327dd58d29c95..53c4b58e6323cc 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h @@ -356,6 +356,7 @@ int brcmf_fweh_register(struct brcmf_pub *drvr, enum brcmf_fweh_event_code code, void *data)); void brcmf_fweh_unregister(struct brcmf_pub *drvr, enum brcmf_fweh_event_code code); +int brcmf_fweh_init_events(struct brcmf_if *ifp); int brcmf_fweh_activate_events(struct brcmf_if *ifp); void brcmf_fweh_process_event(struct brcmf_pub *drvr, struct brcmf_event *event_packet, diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h index e74a23e11830c1..7b8f809cdc412d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h @@ -18,7 +18,8 @@ #define BRCMF_ARP_OL_HOST_AUTO_REPLY 0x00000004 #define BRCMF_ARP_OL_PEER_AUTO_REPLY 0x00000008 -#define BRCMF_BSS_INFO_VERSION 109 /* curr ver of brcmf_bss_info_le struct */ +#define BRCMF_BSS_INFO_MIN_VERSION 109 /* min ver of brcmf_bss_info_le struct */ +#define BRCMF_BSS_INFO_MAX_VERSION 112 /* max ver of brcmf_bss_info_le struct */ #define BRCMF_BSS_RSSI_ON_CHANNEL 0x0004 #define BRCMF_STA_BRCM 0x00000001 /* Running a Broadcom driver */ @@ -46,12 +47,10 @@ #define BRCMF_STA_DWDS_CAP 0x01000000 /* DWDS CAP */ #define BRCMF_STA_DWDS 0x02000000 /* DWDS active */ -/* size of brcmf_scan_params not including variable length array */ -#define BRCMF_SCAN_PARAMS_FIXED_SIZE 64 -#define BRCMF_SCAN_PARAMS_V2_FIXED_SIZE 72 - /* version of brcmf_scan_params structure */ #define BRCMF_SCAN_PARAMS_VERSION_V2 2 +#define BRCMF_SCAN_PARAMS_VERSION_V3 3 +#define BRCMF_SCAN_PARAMS_VERSION_V4 4 /* masks for channel and ssid count */ #define BRCMF_SCAN_PARAMS_COUNT_MASK 0x0000ffff @@ -62,16 +61,26 @@ #define BRCMF_SCANTYPE_ACTIVE 0 #define BRCMF_SCANTYPE_PASSIVE 1 +/* Additional scanning flags */ +#define BRCMF_SCANFLAGS_LOW_PRIO 0x2 +#define BRCMF_SCANFLAGS_LOW_POWER 0x1000 +#define BRCMF_SCANFLAGS_HIGH_ACCURACY 0x2000 +#define BRCMF_SCANFLAGS_LOW_SPAN 0x4000 + +/* scan ssid_type flags */ +#define BRCMF_SCANSSID_INC_RNR 0x02 /* Include RNR channels*/ + #define BRCMF_WSEC_MAX_PSK_LEN 32 #define BRCMF_WSEC_PASSPHRASE BIT(0) -#define BRCMF_WSEC_MAX_SAE_PASSWORD_LEN 128 +#define BRCMF_WSEC_MAX_SAE_PASSWORD_LEN 256 /* primary (ie tx) key */ #define BRCMF_PRIMARY_KEY (1 << 1) #define DOT11_BSSTYPE_ANY 2 #define BRCMF_ESCAN_REQ_VERSION 1 #define BRCMF_ESCAN_REQ_VERSION_V2 2 +#define BRCMF_ESCAN_REQ_VERSION_V3 3 #define BRCMF_MAXRATES_IN_SET 16 /* max # of rates in rateset */ @@ -320,29 +329,57 @@ struct brcmf_bss_info_le { __le16 beacon_period; /* units are Kusec */ __le16 capability; /* Capability information */ u8 SSID_len; - u8 SSID[32]; + u8 SSID[IEEE80211_MAX_SSID_LEN]; + u8 bcnflags; /* additional flags w.r.t. beacon */ struct { __le32 count; /* # rates in this set */ u8 rates[16]; /* rates in 500kbps units w/hi bit set if basic */ } rateset; /* supported rates */ __le16 chanspec; /* chanspec for bss */ __le16 atim_window; /* units are Kusec */ - u8 dtim_period; /* DTIM period */ + u8 dtim_period; /* DTIM period */ + u8 accessnet; /* from beacon interwork IE (if bcnflags) */ __le16 RSSI; /* receive signal strength (in dBm) */ s8 phy_noise; /* noise (in dBm) */ u8 n_cap; /* BSS is 802.11N Capable */ + u8 he_cap; /* BSS is he capable */ + u8 load; /* BSS Load from QBSS load IE if available */ /* 802.11N BSS Capabilities (based on HT_CAP_*): */ __le32 nbss_cap; u8 ctl_ch; /* 802.11N BSS control channel number */ - __le32 reserved32[1]; /* Reserved for expansion of BSS properties */ + u8 reserved1[3]; /* Reserved for expansion of BSS properties */ + __le16 vht_rxmcsmap; /* VHT rx mcs map (802.11ac IE, VHT_CAP_MCS_MAP_*) */ + __le16 vht_txmcsmap; /* VHT tx mcs map (802.11ac IE, VHT_CAP_MCS_MAP_*) */ u8 flags; /* flags */ - u8 reserved[3]; /* Reserved for expansion of BSS properties */ + u8 vht_cap; /* BSS is vht capable */ + u8 reserved2[2]; /* Reserved for expansion of BSS properties */ u8 basic_mcs[BRCMF_MCSSET_LEN]; /* 802.11N BSS required MCS set */ __le16 ie_offset; /* offset at which IEs start, from beginning */ + u8 reserved3[2]; /* Reserved for expansion of BSS properties */ __le32 ie_length; /* byte length of Information Elements */ __le16 SNR; /* average SNR of during frame reception */ + __le16 vht_mcsmap; /**< STA's Associated vhtmcsmap */ + __le16 vht_mcsmap_prop; /**< STA's Associated prop vhtmcsmap */ + __le16 vht_txmcsmap_prop; /**< prop VHT tx mcs prop */ + __le32 he_mcsmap; /**< STA's Associated hemcsmap */ + __le32 he_rxmcsmap; /**< HE rx mcs map (802.11ax IE, HE_CAP_MCS_MAP_*) */ + __le32 he_txmcsmap; /**< HE tx mcs map (802.11ax IE, HE_CAP_MCS_MAP_*) */ + __le32 timestamp[2]; /* Beacon Timestamp for FAKEAP req */ + /* V112 fields follow */ + u8 eht_cap; /* BSS is EHT capable */ + u8 reserved4[3]; /* Reserved for expansion of BSS properties */ + /* by the spec. it is maximum 16 streams hence all mcs code for all nss may not fit + * in a 32 bit mcs nss map but since this field only reflects the common mcs nss map + * between that of the peer and our device so it's probably ok to make it 32 bit and + * allow only a limited number of nss e.g. upto 8 of them in the map given the fact + * that our device probably won't exceed 4 streams anyway... + */ + __le32 eht_mcsmap; /* STA's associated EHT mcs code map */ + /* FIXME: change the following mcs code map to uint32 if all mcs+nss can fit in */ + u8 eht_rxmcsmap[6]; /* EHT rx mcs code map */ + u8 eht_txmcsmap[6]; /* EHT tx mcs code map */ /* Add new fields here */ /* variable length Information Elements */ }; @@ -366,23 +403,23 @@ struct brcmf_ssid8_le { }; struct brcmf_scan_params_le { - struct brcmf_ssid_le ssid_le; /* default: {0, ""} */ - u8 bssid[ETH_ALEN]; /* default: bcast */ - s8 bss_type; /* default: any, + struct brcmf_ssid_le ssid_le; /* default: {0, ""} */ + u8 bssid[ETH_ALEN]; /* default: bcast */ + s8 bss_type; /* default: any, * DOT11_BSSTYPE_ANY/INFRASTRUCTURE/INDEPENDENT */ - u8 scan_type; /* flags, 0 use default */ - __le32 nprobes; /* -1 use default, number of probes per channel */ - __le32 active_time; /* -1 use default, dwell time per channel for + u8 scan_type; /* flags, 0 use default */ + __le32 nprobes; /* -1 use default, number of probes per channel */ + __le32 active_time; /* -1 use default, dwell time per channel for * active scanning */ - __le32 passive_time; /* -1 use default, dwell time per channel + __le32 passive_time; /* -1 use default, dwell time per channel * for passive scanning */ __le32 home_time; /* -1 use default, dwell time for the * home channel between channel scans */ - __le32 channel_num; /* count of channels and ssids that follow + __le32 channel_num; /* count of channels and ssids that follow * * low half is count of channels in * channel_list, 0 means default (use all @@ -398,56 +435,125 @@ struct brcmf_scan_params_le { * fixed parameter portion is assumed, otherwise * ssid in the fixed portion is ignored */ - union { - __le16 padding; /* Reserve space for at least 1 entry for abort - * which uses an on stack brcmf_scan_params_le - */ - DECLARE_FLEX_ARRAY(__le16, channel_list); /* chanspecs */ - }; + __le16 channel_list[]; /* chanspecs */ }; struct brcmf_scan_params_v2_le { - __le16 version; /* structure version */ - __le16 length; /* structure length */ - struct brcmf_ssid_le ssid_le; /* default: {0, ""} */ - u8 bssid[ETH_ALEN]; /* default: bcast */ - s8 bss_type; /* default: any, - * DOT11_BSSTYPE_ANY/INFRASTRUCTURE/INDEPENDENT - */ - u8 pad; - __le32 scan_type; /* flags, 0 use default */ - __le32 nprobes; /* -1 use default, number of probes per channel */ - __le32 active_time; /* -1 use default, dwell time per channel for - * active scanning - */ - __le32 passive_time; /* -1 use default, dwell time per channel - * for passive scanning - */ - __le32 home_time; /* -1 use default, dwell time for the - * home channel between channel scans - */ - __le32 channel_num; /* count of channels and ssids that follow - * - * low half is count of channels in - * channel_list, 0 means default (use all - * available channels) - * - * high half is entries in struct brcmf_ssid - * array that follows channel_list, aligned for - * s32 (4 bytes) meaning an odd channel count - * implies a 2-byte pad between end of - * channel_list and first ssid - * - * if ssid count is zero, single ssid in the - * fixed parameter portion is assumed, otherwise - * ssid in the fixed portion is ignored - */ - union { - __le16 padding; /* Reserve space for at least 1 entry for abort - * which uses an on stack brcmf_scan_params_v2_le - */ - DECLARE_FLEX_ARRAY(__le16, channel_list); /* chanspecs */ - }; + __le16 version; /* structure version */ + __le16 length; /* structure length */ + struct brcmf_ssid_le ssid_le; /* default: {0, ""} */ + u8 bssid[ETH_ALEN]; /* default: bcast */ + s8 bss_type; /* default: any, + * DOT11_BSSTYPE_ANY/INFRASTRUCTURE/INDEPENDENT + */ + u8 PAD; + __le32 scan_type; /* flags, 0 use default */ + __le32 nprobes; /* -1 use default, number of probes per channel */ + __le32 active_time; /* -1 use default, dwell time per channel for + * active scanning + */ + __le32 passive_time; /* -1 use default, dwell time per channel + * for passive scanning + */ + __le32 home_time; /* -1 use default, dwell time for the + * home channel between channel scans + */ + __le32 channel_num; /* count of channels and ssids that follow + * + * low half is count of channels in + * channel_list, 0 means default (use all + * available channels) + * + * high half is entries in struct brcmf_ssid + * array that follows channel_list, aligned for + * s32 (4 bytes) meaning an odd channel count + * implies a 2-byte pad between end of + * channel_list and first ssid + * + * if ssid count is zero, single ssid in the + * fixed parameter portion is assumed, otherwise + * ssid in the fixed portion is ignored + */ + __le16 channel_list[]; /* chanspecs */ +}; + +struct brcmf_scan_params_v3_le { + __le16 version; /* structure version */ + __le16 length; /* structure length */ + struct brcmf_ssid_le ssid_le; /* default: {0, ""} */ + u8 bssid[ETH_ALEN]; /* default: bcast */ + s8 bss_type; /* default: any, + * DOT11_BSSTYPE_ANY/INFRASTRUCTURE/INDEPENDENT + */ + u8 ssid_type; /* short vs regular SSID */ + __le32 scan_type; /* flags, 0 use default */ + __le32 nprobes; /* -1 use default, number of probes per channel */ + __le32 active_time; /* -1 use default, dwell time per channel for + * active scanning + */ + __le32 passive_time; /* -1 use default, dwell time per channel + * for passive scanning + */ + __le32 home_time; /* -1 use default, dwell time for the + * home channel between channel scans + */ + __le32 channel_num; /* count of channels and ssids that follow + * + * low half is count of channels in + * channel_list, 0 means default (use all + * available channels) + * + * high half is entries in struct brcmf_ssid + * array that follows channel_list, aligned for + * s32 (4 bytes) meaning an odd channel count + * implies a 2-byte pad between end of + * channel_list and first ssid + * + * if ssid count is zero, single ssid in the + * fixed parameter portion is assumed, otherwise + * ssid in the fixed portion is ignored + */ + __le16 channel_list[]; /* chanspecs */ +}; + +struct brcmf_scan_params_v4_le { + __le16 version; /* structure version */ + __le16 length; /* structure length */ + struct brcmf_ssid_le ssid_le; /* default: {0, ""} */ + u8 bssid[ETH_ALEN]; /* default: bcast */ + s8 bss_type; /* default: any, + * DOT11_BSSTYPE_ANY/INFRASTRUCTURE/INDEPENDENT + */ + u8 ssid_type; /* short vs regular SSID */ + __le32 scan_type; /* flags, 0 use default */ + __le32 scan_type_ext; /* ext flags, 0 use default */ + __le32 nprobes; /* -1 use default, number of probes per channel */ + __le32 active_time; /* -1 use default, dwell time per channel for + * active scanning + */ + __le32 passive_time; /* -1 use default, dwell time per channel + * for passive scanning + */ + __le32 home_time; /* -1 use default, dwell time for the + * home channel between channel scans + */ + __le32 channel_num; /* count of channels and ssids that follow + * + * low half is count of channels in + * channel_list, 0 means default (use all + * available channels) + * + * high half is entries in struct brcmf_ssid + * array that follows channel_list, aligned for + * s32 (4 bytes) meaning an odd channel count + * implies a 2-byte pad between end of + * channel_list and first ssid + * + * if ssid count is zero, single ssid in the + * fixed parameter portion is assumed, otherwise + * ssid in the fixed portion is ignored + */ + __le16 channel_list[]; /* chanspecs */ }; struct brcmf_scan_results { @@ -464,6 +570,8 @@ struct brcmf_escan_params_le { union { struct brcmf_scan_params_le params_le; struct brcmf_scan_params_v2_le params_v2_le; + struct brcmf_scan_params_v3_le params_v3_le; + struct brcmf_scan_params_v4_le params_v4_le; }; }; @@ -482,11 +590,67 @@ struct brcmf_escan_result_le { struct brcmf_assoc_params_le { /* 00:00:00:00:00:00: broadcast scan */ u8 bssid[ETH_ALEN]; + /* 0: use chanspec_num, and the single bssid, + * otherwise count of chanspecs in chanspec_list + * AND paired bssids following chanspec_list + * also, chanspec_num has to be set to zero + * for bssid list to be used + */ + __le16 bssid_cnt; + /* 0: all available channels, otherwise count of chanspecs in + * chanspec_list */ + __le32 chanspec_num; + /* list of chanspecs */ + __le16 chanspec_list[]; +}; + +struct brcmf_assoc_params_v1_le { + __le16 version; + __le16 flags; + /* 00:00:00:00:00:00: broadcast scan */ + u8 bssid[ETH_ALEN]; + /* 0: use chanspec_num, and the single bssid, + * otherwise count of chanspecs in chanspec_list + * AND paired bssids following chanspec_list + * also, chanspec_num has to be set to zero + * for bssid list to be used + */ + __le16 bssid_cnt; /* 0: all available channels, otherwise count of chanspecs in * chanspec_list */ __le32 chanspec_num; /* list of chanspecs */ - __le16 chanspec_list[1]; + __le16 chanspec_list[]; +}; + +/* ML assoc and scan params */ +struct brcmf_ml_assoc_scan_params_v1_le { + /* whether to follow strictly ordered assoc ? */ + u8 ml_assoc_mode; + /* to identify whether ml scan needs to be triggered */ + u8 ml_scan_mode; + u8 pad[2]; +}; + +struct brcmf_assoc_params_v2_le { + __le16 version; + __le16 flags; + /* 00:00:00:00:00:00: broadcast scan */ + u8 bssid[ETH_ALEN]; + /* 0: use chanspec_num, and the single bssid, + * otherwise count of chanspecs in chanspec_list + * AND paired bssids following chanspec_list + * also, chanspec_num has to be set to zero + * for bssid list to be used + */ + __le16 bssid_cnt; + /* Multilink association and scan params */ + struct brcmf_ml_assoc_scan_params_v1_le ml_assoc_scan_params; + /* 0: all available channels, otherwise count of chanspecs in + * chanspec_list */ + __le32 chanspec_num; + /* list of chanspecs */ + __le16 chanspec_list[]; }; /** @@ -511,9 +675,19 @@ struct brcmf_join_params { struct brcmf_assoc_params_le params_le; }; +struct brcmf_join_params_v1 { + struct brcmf_ssid_le ssid_le; + struct brcmf_assoc_params_v1_le params_le; +}; +struct brcmf_join_params_v2 { + struct brcmf_ssid_le ssid_le; + struct brcmf_assoc_params_v2_le params_le; +}; + /* scan params for extended join */ struct brcmf_join_scan_params_le { u8 scan_type; /* 0 use default, active or passive scan */ + u8 PAD[3]; __le32 nprobes; /* -1 use default, nr of probes per channel */ __le32 active_time; /* -1 use default, dwell time per channel for * active scanning @@ -526,6 +700,23 @@ struct brcmf_join_scan_params_le { */ }; +/* scan params for extended join */ +struct brcmf_join_scan_params_v1_le { + u8 scan_type; /* 0 use default, active or passive scan */ + u8 ml_scan_mode; /* 0 scan ML channels in RNR, 1 scan only provided channels */ + u8 PAD[2]; + __le32 nprobes; /* -1 use default, nr of probes per channel */ + __le32 active_time; /* -1 use default, dwell time per channel for + * active scanning + */ + __le32 passive_time; /* -1 use default, dwell time per channel + * for passive scanning + */ + __le32 home_time; /* -1 use default, dwell time for the home + * channel between channel scans + */ +}; + /* extended join params */ struct brcmf_ext_join_params_le { struct brcmf_ssid_le ssid_le; /* {0, ""}: wildcard scan */ @@ -533,6 +724,24 @@ struct brcmf_ext_join_params_le { struct brcmf_assoc_params_le assoc_le; }; +/* extended join params */ +struct brcmf_ext_join_params_v1_le { + __le16 version; + u16 pad; + struct brcmf_ssid_le ssid_le; /* {0, ""}: wildcard scan */ + struct brcmf_join_scan_params_le scan_le; + struct brcmf_assoc_params_v1_le assoc_le; +}; + +/* extended join params v2 */ +struct brcmf_ext_join_params_v2_le { + __le16 version; + u16 pad; + struct brcmf_ssid_le ssid_le; /* {0, ""}: wildcard scan */ + struct brcmf_join_scan_params_v1_le scan_le; + struct brcmf_assoc_params_v2_le assoc_le; +}; + struct brcmf_wsec_key { u32 index; /* key index */ u32 len; /* key length */ @@ -580,11 +789,15 @@ struct brcmf_wsec_key_le { * @key_len: number of octets in key material. * @flags: key handling qualifiers. * @key: PMK key material. + * @opt_len: optional field length + * @opt_tlvs: optional fields in TLV format */ struct brcmf_wsec_pmk_le { __le16 key_len; __le16 flags; u8 key[BRCMF_WSEC_MAX_SAE_PASSWORD_LEN]; + __le16 opt_len; + u8 opt_tlvs[]; }; /** @@ -611,13 +824,17 @@ struct brcmf_channel_info_le { __le32 scan_channel; }; +#define BRCMF_MAX_ASSOC_OUI_NUM 6 +#define BRCMF_ASSOC_OUI_LEN 3 struct brcmf_sta_info_le { __le16 ver; /* version of this struct */ __le16 len; /* length in bytes of this structure */ __le16 cap; /* sta's advertised capabilities */ + u16 PAD; __le32 flags; /* flags defined below */ __le32 idle; /* time since data pkt rx'd from sta */ u8 ea[ETH_ALEN]; /* Station address */ + u16 PAD2; __le32 count; /* # rates in this set */ u8 rates[BRCMF_MAXRATES_IN_SET]; /* rates in 500kbps units */ /* w/hi bit set if basic */ @@ -649,6 +866,7 @@ struct brcmf_sta_info_le { __le16 aid; /* association ID */ __le16 ht_capabilities; /* advertised ht caps */ __le16 vht_flags; /* converted vht flags */ + u16 PAD3; __le32 tx_pkts_retry_cnt; /* # of frames where a retry was * exhausted. */ @@ -701,6 +919,13 @@ struct brcmf_sta_info_le { __le32 tx_rspec; /* Rate of last successful tx frame */ __le32 rx_rspec; /* Rate of last successful rx frame */ __le32 wnm_cap; /* wnm capabilities */ + __le16 he_flags; /* converted he flags */ + u16 PAD; + struct { + u8 count; + u8 oui[BRCMF_MAX_ASSOC_OUI_NUM][BRCMF_ASSOC_OUI_LEN]; + } vendor_oui; + u8 link_bw; } v7; }; }; @@ -833,6 +1058,30 @@ struct brcmf_wlc_version_le { __le16 wlc_ver_minor; }; +/** + * struct brcmf_join_version_le - join interface version + */ +struct brcmf_join_version_le { + __le16 version; /**< version of the structure */ + __le16 length; /**< length of the entire structure */ + + /* join interface version numbers */ + __le16 join_ver_major; /**< join interface major version number */ + u8 pad[2]; +}; +#define BRCMF_JOIN_VERSION_VERSION 1 + +/** + * struct brcmf_scan_version_le - scan interface version + */ +struct brcmf_scan_version_le { + __le16 version; + __le16 length; + __le16 scan_ver_major; +}; + +#define BRCMF_SCAN_VERSION_VERSION 1 + /** * struct brcmf_assoclist_le - request assoc list. * @@ -1009,6 +1258,46 @@ struct brcmf_pno_param_le { __le32 slow_freq; }; +/** + * struct brcmf_pno_param_le - PNO scan configuration parameters + * + * @version: PNO parameters version. + * @length: Length of PNO structure + * @scan_freq: scan frequency. + * @lost_network_timeout: #sec. to declare discovered network as lost. + * @flags: Bit field to control features of PFN such as sort criteria auto + * enable switch and background scan. + * @rssi_margin: Margin to avoid jitter for choosing a PFN based on RSSI sort + * criteria. + * @bestn: number of best networks in each scan. + * @mscan: number of scans recorded. + * @repeat: minimum number of scan intervals before scan frequency changes + * in adaptive scan. + * @exp: exponent of 2 for maximum scan interval. + * @slow_freq: slow scan period. + * @min_bound: min bound for scan time randomization + * @max_bound: max bound for scan time randomization + * @pfn_lp_scan_disable: unused + * @pfn_lp_scan_cnt: allow interleaving lp scan with hp scan + */ +struct brcmf_pno_param_v3_le { + __le16 version; + __le16 length; + __le32 scan_freq; + __le32 lost_network_timeout; + __le16 flags; + __le16 rssi_margin; + u8 bestn; + u8 mscan; + u8 repeat; + u8 exp; + __le32 slow_freq; + u8 min_bound; + u8 max_bound; + u8 pfn_lp_scan_disable; + u8 pfn_lp_scan_cnt; +}; + /** * struct brcmf_pno_config_le - PNO channel configuration. * @@ -1062,6 +1351,28 @@ struct brcmf_pno_net_info_le { __le16 timestamp; }; +/** + * struct brcmf_pno_net_info_v3_le - information per found network. + * + * @bssid: BSS network identifier. + * @chanspec: channel spec. + * @SSID_len: length of ssid. + * @SSID: ssid characters. + * @flags: flags + * @RSSI: receive signal strength (in dBm). + * @timestamp: age in seconds. + */ +struct brcmf_pno_net_info_v3_le { + u8 bssid[6]; + u16 chanspec; + u8 SSID_len; + u8 padding; + u16 flags; + u8 SSID[32]; + __le16 RSSI; + __le16 timestamp; +}; + /** * struct brcmf_pno_scanresults_le - result returned in PNO NET FOUND event. * @@ -1082,6 +1393,14 @@ struct brcmf_pno_scanresults_v2_le { __le32 scan_ch_bucket; }; +/* V2 and V3 structs are the same */ +struct brcmf_pno_scanresults_v3_le { + __le32 version; + __le32 status; + __le32 count; + __le32 scan_ch_bucket; +}; + /** * struct brcmf_pno_macaddr_le - to configure PNO macaddr randomization. * @@ -1236,4 +1555,141 @@ struct brcmf_mkeep_alive_pkt_le { u8 data[]; } __packed; +enum event_msgs_ext_command { + EVENTMSGS_NONE = 0, + EVENTMSGS_SET_BIT = 1, + EVENTMSGS_RESET_BIT = 2, + EVENTMSGS_SET_MASK = 3 +}; + +#define EVENTMSGS_VER 1 + +/** + * struct brcmf_eventmsgs_ext_le - new event message mask commands + * + * @version: EVENTMSGS_VER + * @command: one of enum event_msgs_ext_command + * @len: for set, the mask size from the application to the firmware. + * for get, the actual firmware mask size. + * @maxgetsize: for get, the max size that the application can read from + * the firmware. + */ +struct brcmf_eventmsgs_ext_le { + u8 version; + u8 command; + u8 len; + u8 maxgetsize; + u8 mask[]; +}; + +/* version of the brcmf_wl_wsec_info structure */ +#define BRCMF_WSEC_INFO_VER 1 + +/* tlv used to return wl_wsec_info properties */ +struct brcmf_wsec_info_tlv { + u16 type; + u16 len; /* data length */ + u8 data[1]; /* data follows */ +}; + +/* input/output data type for wsec_info iovar */ +struct brcmf_wsec_info { + u8 version; /* structure version */ + u8 pad[2]; + u8 num_tlvs; + struct brcmf_wsec_info_tlv tlvs[1]; /* tlv data follows */ +}; + +/* HE top level command IDs */ +enum { + BRCMF_HE_CMD_ENABLE = 0, + BRCMF_HE_CMD_FEATURES = 1, + BRCMF_HE_CMD_SR = 2, + BRCMF_HE_CMD_TESTBED = 3, + BRCMF_HE_CMD_BSR_SUPPORT = 4, + BRCMF_HE_CMD_BSSCOLOR = 5, + BRCMF_HE_CMD_PARTIAL_BSSCOLOR = 6, + BRCMF_HE_CMD_CAP = 7, + BRCMF_HE_CMD_OMI = 8, + BRCMF_HE_CMD_RANGE_EXT = 9, + BRCMF_HE_CMD_RTSDURTHRESH = 10, + BRCMF_HE_CMD_PEDURATION = 11, + BRCMF_HE_CMD_MUEDCA = 12, + BRCMF_HE_CMD_DYNFRAG = 13, + BRCMF_HE_CMD_PPET = 14, + BRCMF_HE_CMD_HTC = 15, + BRCMF_HE_CMD_AXMODE = 16, + BRCMF_HE_CMD_FRAGTX = 17, + BRCMF_HE_CMD_DEFCAP = 18, +}; + +#define BRCMF_HE_VER_1 1 + +struct brcmf_he_bsscolor { + u8 color; /* 1..63, on get returns currently in use color */ + u8 disabled; /* 0/1, 0 means disabled is false, so coloring is enabled */ + u8 switch_count; /* 0, immediate programming, 1 .. 255 beacon count down */ + u8 PAD; +}; + +struct brcmf_he_omi { + u8 peer[ETH_ALEN]; /* leave it all 0s' for non-AP */ + u8 rx_nss; /* 0..7 */ + u8 channel_width; /* 0:20, 1:40, 2:80, 3:160 */ + u8 ul_mu_disable; /* 0|1 */ + u8 tx_nsts; /* 0..7 */ + u8 er_su_disable; /* 0|1 */ + u8 dl_mumimo_resound; /* 0|1 */ + u8 ul_mu_data_disable; /* 0|1 */ + u8 tx_override; /* 0, only used for testbed AP */ + u8 PAD[2]; +}; + +struct brcmf_he_edca_v1 { + u8 aci_aifsn; + u8 ecw_min_max; + u8 muedca_timer; + u8 PAD; +}; + +#define BRCMF_AC_COUNT 4 +struct brcmf_he_muedca_v1 { + /* structure control */ + __le16 version; /* structure version */ + __le16 length; /* data length (starting after this field) */ + struct brcmf_he_edca_v1 ac_param_ap[BRCMF_AC_COUNT]; + struct brcmf_he_edca_v1 ac_param_sta[BRCMF_AC_COUNT]; +}; + +#define BRCMF_HE_SR_VER_1 1 + +#define SRC_PSR_DIS 0x01 +#define SRC_NON_SRG_OBSS_PD_SR_DIS 0x02 +#define SRC_NON_SRG_OFFSET_PRESENT 0x04 +#define SRC_SRG_INFORMATION_PRESENT 0x08 +#define SRC_HESIGA_SPATIAL_REUSE_VALUE15_ALLOWED 0x10 + +#define HE_SR_SRG_INFO_LEN 18 + +struct brcmf_he_sr_v1 { + /* structure control */ + __le16 version; /* structure version */ + __le16 length; /* data length (starting after this field) */ + u8 enabled; + u8 src; /* SR control, see above defines. */ + u8 non_srg_offset; /* Non-SRG Offset */ + u8 srg[HE_SR_SRG_INFO_LEN]; /* SRG Information */ +}; + +#define BRCMF_HE_DEFCAP_VER_1 1 + +struct brcmf_he_defcap { + __le16 version; /* structure version */ + __le16 length; /* data length (starting after this field) */ + u8 bsscfg_type; + u8 bsscfg_subtype; + u8 mac_cap[6]; + u8 phy_cap[11]; +}; + #endif /* FWIL_TYPES_H_ */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c new file mode 100644 index 00000000000000..1f40ff8d632c25 --- /dev/null +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (c) 2023 Daniel Berlin + */ + +/* This file handles firmware-side interface creation */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include "cfg80211.h" +#include "debug.h" +#include "fwil.h" +#include "proto.h" +#include "bus.h" +#include "common.h" +#include "interface_create.h" + +#define BRCMF_INTERFACE_CREATE_VER_1 1 +#define BRCMF_INTERFACE_CREATE_VER_2 2 +#define BRCMF_INTERFACE_CREATE_VER_3 3 +#define BRCMF_INTERFACE_CREATE_VER_MAX BRCMF_INTERFACE_CREATE_VER_3 + +/* These sets of flags specify whether to use various fields in the interface create structures */ + +/* This is only used with version 0 or 1 */ +#define BRCMF_INTERFACE_CREATE_STA (0 << 0) +#define BRCMF_INTERFACE_CREATE_AP (1 << 0) + +#define BRCMF_INTERFACE_MAC_DONT_USE (0 << 1) +#define BRCMF_INTERFACE_MAC_USE (1 << 1) + +#define BRCMF_INTERFACE_WLC_INDEX_DONT_USE (0 << 2) +#define BRCMF_INTERFACE_WLC_INDEX_USE (1 << 2) + +#define BRCMF_INTERFACE_IF_INDEX_DONT_USE (0 << 3) +#define BRCMF_INTERFACE_IF_INDEX_USE (1 << 3) + +#define BRCMF_INTERFACE_BSSID_DONT_USE (0 << 4) +#define BRCMF_INTERFACE_BSSID_USE (1 << 4) + +/* + * From revision >= 2 Bit 0 of flags field will not be used for STA or AP interface creation. + * "iftype" field shall be used for identifying the interface type. + */ +enum brcmf_interface_type { + BRCMF_INTERFACE_TYPE_STA = 0, + BRCMF_INTERFACE_TYPE_AP = 1, + /* The missing number here is deliberate */ + BRCMF_INTERFACE_TYPE_NAN = 3, + BRCMF_INTERFACE_TYPE_P2P_GO = 4, + BRCMF_INTERFACE_TYPE_P2P_GC = 5, + BRCMF_INTERFACE_TYPE_P2P_DISC = 6, + BRCMF_INTERFACE_TYPE_IBSS = 7, + BRCMF_INTERFACE_TYPE_MESH = 8 +}; + + +/* All sources treat these structures as being host endian. + * However, firmware treats it as little endian, so we do as well */ + +struct brcmf_interface_create_v1 { + __le16 ver; /* structure version */ + u8 pad1[2]; + __le32 flags; /* flags for operation */ + u8 mac_addr[ETH_ALEN]; /* MAC address */ + u8 pad2[2]; + __le32 wlc_index; /* optional for wlc index */ +}; + +struct brcmf_interface_create_v2 { + __le16 ver; /* structure version */ + u8 pad1[2]; + __le32 flags; /* flags for operation */ + u8 mac_addr[ETH_ALEN]; /* MAC address */ + u8 iftype; /* type of interface created */ + u8 pad2; + u32 wlc_index; /* optional for wlc index */ +}; + +struct brcmf_interface_create_v3 { + __le16 ver; /* structure version */ + __le16 len; /* length of structure + data */ + __le16 fixed_len; /* length of structure */ + u8 iftype; /* type of interface created */ + u8 wlc_index; /* optional for wlc index */ + __le32 flags; /* flags for operation */ + u8 mac_addr[ETH_ALEN]; /* MAC address */ + u8 bssid[ETH_ALEN]; /* optional for BSSID */ + u8 if_index; /* interface index request */ + u8 pad[3]; + u8 data[]; /* Optional for specific data */ +}; + +static int brcmf_get_first_free_bsscfgidx(struct brcmf_pub *drvr) +{ + int bsscfgidx; + + for (bsscfgidx = 0; bsscfgidx < BRCMF_MAX_IFS; bsscfgidx++) { + /* bsscfgidx 1 is reserved for legacy P2P */ + if (bsscfgidx == 1) + continue; + if (!drvr->iflist[bsscfgidx]) + return bsscfgidx; + } + + return -ENOMEM; +} + +static void brcmf_set_vif_sta_macaddr(struct brcmf_if *ifp, u8 *mac_addr) +{ + u8 mac_idx = ifp->drvr->sta_mac_idx; + + /* set difference MAC address with locally administered bit */ + memcpy(mac_addr, ifp->mac_addr, ETH_ALEN); + mac_addr[0] |= 0x02; + mac_addr[3] ^= mac_idx ? 0xC0 : 0xA0; + mac_idx++; + mac_idx = mac_idx % 2; + ifp->drvr->sta_mac_idx = mac_idx; +} + +static int brcmf_cfg80211_request_if_internal(struct brcmf_if *ifp, u32 version, + enum brcmf_interface_type if_type, + u8 *macaddr) +{ + switch (version) { + case BRCMF_INTERFACE_CREATE_VER_1: { + struct brcmf_interface_create_v1 iface_v1 = {}; + u32 flags = if_type; + + iface_v1.ver = cpu_to_le16(BRCMF_INTERFACE_CREATE_VER_1); + if (macaddr) { + flags |= BRCMF_INTERFACE_MAC_USE; + if (!is_zero_ether_addr(macaddr)) + memcpy(iface_v1.mac_addr, macaddr, ETH_ALEN); + else + brcmf_set_vif_sta_macaddr(ifp, + iface_v1.mac_addr); + } + iface_v1.flags = cpu_to_le32(flags); + return brcmf_fil_iovar_data_get(ifp, "interface_create", + &iface_v1, sizeof(iface_v1)); + } + case BRCMF_INTERFACE_CREATE_VER_2: { + struct brcmf_interface_create_v2 iface_v2 = {}; + u32 flags = 0; + + iface_v2.ver = cpu_to_le16(BRCMF_INTERFACE_CREATE_VER_2); + iface_v2.iftype = if_type; + if (macaddr) { + flags = BRCMF_INTERFACE_MAC_USE; + if (!is_zero_ether_addr(macaddr)) + memcpy(iface_v2.mac_addr, macaddr, ETH_ALEN); + else + brcmf_set_vif_sta_macaddr(ifp, + iface_v2.mac_addr); + } + iface_v2.flags = cpu_to_le32(flags); + return brcmf_fil_iovar_data_get(ifp, "interface_create", + &iface_v2, sizeof(iface_v2)); + } + case BRCMF_INTERFACE_CREATE_VER_3: { + struct brcmf_interface_create_v3 iface_v3 = {}; + u32 flags = 0; + + iface_v3.ver = cpu_to_le16(BRCMF_INTERFACE_CREATE_VER_3); + iface_v3.iftype = if_type; + iface_v3.len = cpu_to_le16(sizeof(iface_v3)); + iface_v3.fixed_len = cpu_to_le16(sizeof(iface_v3)); + if (macaddr) { + flags = BRCMF_INTERFACE_MAC_USE; + if (!is_zero_ether_addr(macaddr)) + memcpy(iface_v3.mac_addr, macaddr, ETH_ALEN); + else + brcmf_set_vif_sta_macaddr(ifp, + iface_v3.mac_addr); + } + iface_v3.flags = cpu_to_le32(flags); + return brcmf_fil_iovar_data_get(ifp, "interface_create", + &iface_v3, sizeof(iface_v3)); + } + default: + bphy_err(ifp->drvr, "Unknown interface create version:%d\n", + version); + return -EINVAL; + } +} +static int brcmf_cfg80211_request_if(struct brcmf_if *ifp, + enum brcmf_interface_type if_type, + u8 *macaddr) +{ + s32 err; + u32 iface_create_ver; + + /* Query the creation version, see if the firmware knows */ + iface_create_ver = 0; + err = brcmf_fil_bsscfg_int_query(ifp, "interface_create", + &iface_create_ver); + if (!err) { + err = brcmf_cfg80211_request_if_internal(ifp, iface_create_ver, + if_type, macaddr); + if (!err) { + brcmf_info("interface created (version %d)\n", + iface_create_ver); + } else { + bphy_err(ifp->drvr, + "failed to create interface (version %d):%d\n", + iface_create_ver, err); + } + return err; + } + /* Either version one or version two */ + err = brcmf_cfg80211_request_if_internal( + ifp, if_type, BRCMF_INTERFACE_CREATE_VER_2, macaddr); + if (!err) { + brcmf_info("interface created (version 2)\n"); + return 0; + } + err = brcmf_cfg80211_request_if_internal( + ifp, if_type, BRCMF_INTERFACE_CREATE_VER_1, macaddr); + if (!err) { + brcmf_info("interface created (version 1)\n"); + return 0; + } + bphy_err(ifp->drvr, + "interface creation failed, tried query, v2, v1: %d\n", err); + return -EINVAL; +} + +int brcmf_cfg80211_request_sta_if(struct brcmf_if *ifp, u8 *macaddr) +{ + return brcmf_cfg80211_request_if(ifp, BRCMF_INTERFACE_TYPE_STA, + macaddr); +} + +int brcmf_cfg80211_request_ap_if(struct brcmf_if *ifp) +{ + int err; + + err = brcmf_cfg80211_request_if(ifp, BRCMF_INTERFACE_TYPE_AP, NULL); + if (err) { + struct brcmf_mbss_ssid_le mbss_ssid_le; + int bsscfgidx; + + brcmf_info("Does not support interface_create (%d)\n", err); + memset(&mbss_ssid_le, 0, sizeof(mbss_ssid_le)); + bsscfgidx = brcmf_get_first_free_bsscfgidx(ifp->drvr); + if (bsscfgidx < 0) + return bsscfgidx; + + mbss_ssid_le.bsscfgidx = cpu_to_le32(bsscfgidx); + mbss_ssid_le.SSID_len = cpu_to_le32(5); + sprintf(mbss_ssid_le.SSID, "ssid%d", bsscfgidx); + + err = brcmf_fil_bsscfg_data_set(ifp, "bsscfg:ssid", + &mbss_ssid_le, + sizeof(mbss_ssid_le)); + + if (err < 0) + bphy_err(ifp->drvr, "setting ssid failed %d\n", err); + } + return err; +} diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.h new file mode 100644 index 00000000000000..669fa1508b67f6 --- /dev/null +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.h @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (c) 2023 Daniel Berlin + */ + +#ifndef _BRCMF_INTERFACE_CREATE_H_ +#define _BRCMF_INTERFACE_CREATE_H_ +#include "core.h" + +int brcmf_cfg80211_request_sta_if(struct brcmf_if *ifp, u8 *macaddr); +int brcmf_cfg80211_request_ap_if(struct brcmf_if *ifp); + +#endif /* _BRCMF_INTERFACE_CREATE_H_ */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/join_param.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/join_param.c new file mode 100644 index 00000000000000..4f026571c7e7eb --- /dev/null +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/join_param.c @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (c) 2023 Daniel Berlin + */ +#include +#include + +#include "core.h" +#include "debug.h" +#include "fwil_types.h" +#include "cfg80211.h" +#include "join_param.h" + +/* These defaults are the same as found in the DHD drivers, and represent + * reasonable defaults for various scan dwell and probe times. */ +#define BRCMF_SCAN_JOIN_ACTIVE_DWELL_TIME_MS 320 +#define BRCMF_SCAN_JOIN_PASSIVE_DWELL_TIME_MS 400 +#define BRCMF_SCAN_JOIN_PROBE_INTERVAL_MS 20 + +/* Most of the actual structure fields we fill in are the same for various versions + * However, due to various incompatible changes and variants, the fields are not always + * in the same place. + * This makes for code duplication, so we try to commonize setting fields where it makes sense. + */ + +static void brcmf_joinscan_set_ssid(struct brcmf_ssid_le *ssid_le, + const u8 *ssid, u32 ssid_len) +{ + ssid_len = min_t(u32, ssid_len, IEEE80211_MAX_SSID_LEN); + ssid_le->SSID_len = cpu_to_le32(ssid_len); + memcpy(ssid_le->SSID, ssid, ssid_len); +} + +static void brcmf_joinscan_set_bssid(u8 out_bssid[6], const u8 *in_bssid) +{ + if (in_bssid) { + memcpy(out_bssid, in_bssid, ETH_ALEN); + } else { + eth_broadcast_addr(out_bssid); + } +} + +/* Create a single channel chanspec list from a wireless stack channel */ +static void brcmf_joinscan_set_single_chanspec_from_channel( + struct brcmf_cfg80211_info *cfg, struct ieee80211_channel *chan, + __le32 *chanspec_count, __le16 (*chanspec_list)[]) +{ + u16 chanspec = channel_to_chanspec(&cfg->d11inf, chan); + *chanspec_count = cpu_to_le32(1); + (*chanspec_list)[0] = cpu_to_le16(chanspec); +} + +/* Create a single channel chanspec list from a wireless stack chandef */ +static void brcmf_joinscan_set_single_chanspec_from_chandef( + struct brcmf_cfg80211_info *cfg, struct cfg80211_chan_def *chandef, + __le32 *chanspec_count, __le16 (*chanspec_list)[]) +{ + u16 chanspec = chandef_to_chanspec(&cfg->d11inf, chandef); + *chanspec_count = cpu_to_le32(1); + (*chanspec_list)[0] = cpu_to_le16(chanspec); +} + +static void *brcmf_get_struct_for_ibss_v0(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_ibss_params *params) +{ + struct brcmf_join_params *join_params; + + u32 join_params_size = struct_size(join_params, params_le.chanspec_list, + params->chandef.chan != NULL); + + *struct_size = join_params_size; + join_params = kzalloc(join_params_size, GFP_KERNEL); + if (!join_params) { + bphy_err(cfg, "Unable to allocate memory for join params\n"); + return NULL; + } + brcmf_joinscan_set_ssid(&join_params->ssid_le, params->ssid, + params->ssid_len); + brcmf_joinscan_set_bssid(join_params->params_le.bssid, params->bssid); + /* Channel */ + if (cfg->channel) { + brcmf_joinscan_set_single_chanspec_from_chandef( + cfg, ¶ms->chandef, + &join_params->params_le.chanspec_num, + &join_params->params_le.chanspec_list); + } + return join_params; +} + +static void * +brcmf_get_prepped_struct_for_ibss_v1(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_ibss_params *params) +{ + struct brcmf_join_params_v1 *join_params; + u32 join_params_size = struct_size(join_params, params_le.chanspec_list, + params->chandef.chan != NULL); + + *struct_size = join_params_size; + join_params = kzalloc(join_params_size, GFP_KERNEL); + if (!join_params) { + bphy_err(cfg, "Unable to allocate memory for join params\n"); + return NULL; + } + join_params->params_le.version = cpu_to_le16(1); + brcmf_joinscan_set_ssid(&join_params->ssid_le, params->ssid, + params->ssid_len); + brcmf_joinscan_set_bssid(join_params->params_le.bssid, params->bssid); + /* Channel */ + if (cfg->channel) { + brcmf_joinscan_set_single_chanspec_from_chandef( + cfg, ¶ms->chandef, + &join_params->params_le.chanspec_num, + &join_params->params_le.chanspec_list); + } + return join_params; +} + +static void +brcmf_joinscan_set_common_v0v1_params(struct brcmf_join_scan_params_le *scan_le, + bool have_channel) +{ + /* Set up join scan parameters */ + scan_le->scan_type = 0; + scan_le->home_time = cpu_to_le32(-1); + + if (have_channel) { + /* Increase dwell time to receive probe response or detect + * beacon from target AP at a noisy air only during connect + * command. + */ + scan_le->active_time = + cpu_to_le32(BRCMF_SCAN_JOIN_ACTIVE_DWELL_TIME_MS); + scan_le->passive_time = + cpu_to_le32(BRCMF_SCAN_JOIN_PASSIVE_DWELL_TIME_MS); + /* To sync with presence period of VSDB GO send probe request + * more frequently. Probe request will be stopped when it gets + * probe response from target AP/GO. + */ + scan_le->nprobes = + cpu_to_le32(BRCMF_SCAN_JOIN_ACTIVE_DWELL_TIME_MS / + BRCMF_SCAN_JOIN_PROBE_INTERVAL_MS); + } else { + scan_le->active_time = cpu_to_le32(-1); + scan_le->passive_time = cpu_to_le32(-1); + scan_le->nprobes = cpu_to_le32(-1); + } +} +static void * +brcmf_get_struct_for_connect_v0(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_connect_params *params) +{ + struct brcmf_ext_join_params_le *ext_v0; + u32 join_params_size = + struct_size(ext_v0, assoc_le.chanspec_list, cfg->channel != 0); + + *struct_size = join_params_size; + ext_v0 = kzalloc(join_params_size, GFP_KERNEL); + if (!ext_v0) { + bphy_err( + cfg, + "Could not allocate memory for extended join parameters\n"); + return NULL; + } + brcmf_joinscan_set_ssid(&ext_v0->ssid_le, params->ssid, + params->ssid_len); + brcmf_joinscan_set_common_v0v1_params(&ext_v0->scan_le, + cfg->channel != 0); + brcmf_joinscan_set_bssid(ext_v0->assoc_le.bssid, params->bssid); + if (cfg->channel) { + struct ieee80211_channel *chan = params->channel_hint ? + params->channel_hint : + params->channel; + brcmf_joinscan_set_single_chanspec_from_channel( + cfg, chan, &ext_v0->assoc_le.chanspec_num, + &ext_v0->assoc_le.chanspec_list); + } + return ext_v0; +} + +static void * +brcmf_get_struct_for_connect_v1(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_connect_params *params) +{ + struct brcmf_ext_join_params_v1_le *ext_v1; + u32 join_params_size = + struct_size(ext_v1, assoc_le.chanspec_list, cfg->channel != 0); + + *struct_size = join_params_size; + ext_v1 = kzalloc(join_params_size, GFP_KERNEL); + if (!ext_v1) { + bphy_err( + cfg, + "Could not allocate memory for extended join parameters\n"); + return NULL; + } + ext_v1->version = cpu_to_le16(1); + ext_v1->assoc_le.version = cpu_to_le16(1); + brcmf_joinscan_set_ssid(&ext_v1->ssid_le, params->ssid, + params->ssid_len); + brcmf_joinscan_set_common_v0v1_params(&ext_v1->scan_le, + cfg->channel != 0); + brcmf_joinscan_set_bssid(ext_v1->assoc_le.bssid, params->bssid); + if (cfg->channel) { + struct ieee80211_channel *chan = params->channel_hint ? + params->channel_hint : + params->channel; + brcmf_joinscan_set_single_chanspec_from_channel( + cfg, chan, &ext_v1->assoc_le.chanspec_num, + &ext_v1->assoc_le.chanspec_list); + } + return ext_v1; +} + +static void *brcmf_get_join_from_ext_join_v0(void *ext_join, u32 *struct_size) +{ + struct brcmf_ext_join_params_le *ext_join_v0 = + (struct brcmf_ext_join_params_le *)ext_join; + u32 chanspec_num = le32_to_cpu(ext_join_v0->assoc_le.chanspec_num); + struct brcmf_join_params *join_params; + u32 join_params_size = + struct_size(join_params, params_le.chanspec_list, chanspec_num); + u32 assoc_size = struct_size_t(struct brcmf_assoc_params_le, + chanspec_list, chanspec_num); + + *struct_size = join_params_size; + join_params = kzalloc(join_params_size, GFP_KERNEL); + if (!join_params) { + return NULL; + } + memcpy(&join_params->ssid_le, &ext_join_v0->ssid_le, + sizeof(ext_join_v0->ssid_le)); + memcpy(&join_params->params_le, &ext_join_v0->assoc_le, assoc_size); + + return join_params; +} + +static void *brcmf_get_join_from_ext_join_v1(void *ext_join, u32 *struct_size) +{ + struct brcmf_ext_join_params_v1_le *ext_join_v1 = + (struct brcmf_ext_join_params_v1_le *)ext_join; + u32 chanspec_num = le32_to_cpu(ext_join_v1->assoc_le.chanspec_num); + struct brcmf_join_params_v1 *join_params; + u32 join_params_size = + struct_size(join_params, params_le.chanspec_list, chanspec_num); + u32 assoc_size = struct_size_t(struct brcmf_assoc_params_le, + chanspec_list, chanspec_num); + + *struct_size = join_params_size; + join_params = kzalloc(join_params_size, GFP_KERNEL); + if (!join_params) { + return NULL; + } + memcpy(&join_params->ssid_le, &ext_join_v1->ssid_le, + sizeof(ext_join_v1->ssid_le)); + memcpy(&join_params->params_le, &ext_join_v1->assoc_le, assoc_size); + + return join_params; +} + +int brcmf_join_param_setup_for_version(struct brcmf_pub *drvr, u8 version) +{ + drvr->join_param_handler.version = version; + switch (version) { + case 0: + drvr->join_param_handler.get_struct_for_ibss = + brcmf_get_struct_for_ibss_v0; + drvr->join_param_handler.get_struct_for_connect = + brcmf_get_struct_for_connect_v0; + drvr->join_param_handler.get_join_from_ext_join = + brcmf_get_join_from_ext_join_v0; + break; + case 1: + drvr->join_param_handler.get_struct_for_ibss = + brcmf_get_prepped_struct_for_ibss_v1; + drvr->join_param_handler.get_struct_for_connect = + brcmf_get_struct_for_connect_v1; + drvr->join_param_handler.get_join_from_ext_join = + brcmf_get_join_from_ext_join_v1; + break; + default: + return -EINVAL; + } + return 0; +} diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/join_param.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/join_param.h new file mode 100644 index 00000000000000..f549fe2a740823 --- /dev/null +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/join_param.h @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (c) 2023 Daniel Berlin + */ + +#ifndef _BRCMF_JOIN_PARAM_H +#define _BRCMF_JOIN_PARAM_H + +struct brcmf_pub; + +/** + * brcmf_join_param_setup_for_version() - Setup the driver to handle join structures + * + * There are a number of different structures and interface versions for join/extended join parameters + * This sets up the driver to handle a particular interface version. + * + * @drvr Driver structure to setup + * @ver Interface version + * Return: %0 if okay, error code otherwise + */ +int brcmf_join_param_setup_for_version(struct brcmf_pub *drvr, u8 ver); +#endif /* _BRCMF_JOIN_PARAM_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index 45fbcbdc7d9e4b..0e41d618486d39 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -47,9 +47,35 @@ #define MSGBUF_TYPE_RX_CMPLT 0x12 #define MSGBUF_TYPE_LPBK_DMAXFER 0x13 #define MSGBUF_TYPE_LPBK_DMAXFER_CMPLT 0x14 +#define MSGBUF_TYPE_FLOW_RING_RESUME 0x15 +#define MSGBUF_TYPE_FLOW_RING_RESUME_CMPLT 0x16 +#define MSGBUF_TYPE_FLOW_RING_SUSPEND 0x17 +#define MSGBUF_TYPE_FLOW_RING_SUSPEND_CMPLT 0x18 +#define MSGBUF_TYPE_INFO_BUF_POST 0x19 +#define MSGBUF_TYPE_INFO_BUF_CMPLT 0x1A +#define MSGBUF_TYPE_H2D_RING_CREATE 0x1B +#define MSGBUF_TYPE_D2H_RING_CREATE 0x1C +#define MSGBUF_TYPE_H2D_RING_CREATE_CMPLT 0x1D +#define MSGBUF_TYPE_D2H_RING_CREATE_CMPLT 0x1E +#define MSGBUF_TYPE_H2D_RING_CONFIG 0x1F +#define MSGBUF_TYPE_D2H_RING_CONFIG 0x20 +#define MSGBUF_TYPE_H2D_RING_CONFIG_CMPLT 0x21 +#define MSGBUF_TYPE_D2H_RING_CONFIG_CMPLT 0x22 +#define MSGBUF_TYPE_H2D_MAILBOX_DATA 0x23 +#define MSGBUF_TYPE_D2H_MAILBOX_DATA 0x24 +#define MSGBUF_TYPE_TIMSTAMP_BUFPOST 0x25 +#define MSGBUF_TYPE_HOSTTIMSTAMP 0x26 +#define MSGBUF_TYPE_HOSTTIMSTAMP_CMPLT 0x27 +#define MSGBUF_TYPE_FIRMWARE_TIMESTAMP 0x28 +#define MSGBUF_TYPE_SNAPSHOT_UPLOAD 0x29 +#define MSGBUF_TYPE_SNAPSHOT_CMPLT 0x2A +#define MSGBUF_TYPE_H2D_RING_DELETE 0x2B +#define MSGBUF_TYPE_D2H_RING_DELETE 0x2C +#define MSGBUF_TYPE_H2D_RING_DELETE_CMPLT 0x2D +#define MSGBUF_TYPE_D2H_RING_DELETE_CMPLT 0x2E #define NR_TX_PKTIDS 2048 -#define NR_RX_PKTIDS 1024 +#define NR_RX_PKTIDS 2048 #define BRCMF_IOCTL_REQ_PKTID 0xFFFE @@ -218,6 +244,19 @@ struct msgbuf_flowring_flush_resp { __le32 rsvd0[3]; }; +struct msgbuf_h2d_mailbox_data { + struct msgbuf_common_hdr msg; + __le32 data; + __le32 rsvd0[7]; +}; + +struct msgbuf_d2h_mailbox_data { + struct msgbuf_common_hdr msg; + struct msgbuf_completion_hdr compl_hdr; + __le32 data; + __le32 rsvd0[2]; +}; + struct brcmf_msgbuf_work_item { struct list_head queue; u32 flowid; @@ -1285,6 +1324,16 @@ brcmf_msgbuf_process_flow_ring_delete_response(struct brcmf_msgbuf *msgbuf, } +static void brcmf_msgbuf_process_d2h_mailbox_data(struct brcmf_msgbuf *msgbuf, + void *buf) +{ + struct msgbuf_d2h_mailbox_data *d2h_mb_data = buf; + struct brcmf_pub *drvr = msgbuf->drvr; + + brcmf_bus_d2h_mb_rx(drvr->bus_if, le32_to_cpu(d2h_mb_data->data)); +} + + static void brcmf_msgbuf_process_msgtype(struct brcmf_msgbuf *msgbuf, void *buf) { struct brcmf_pub *drvr = msgbuf->drvr; @@ -1327,6 +1376,10 @@ static void brcmf_msgbuf_process_msgtype(struct brcmf_msgbuf *msgbuf, void *buf) brcmf_dbg(MSGBUF, "MSGBUF_TYPE_RX_CMPLT\n"); brcmf_msgbuf_process_rx_complete(msgbuf, buf); break; + case MSGBUF_TYPE_D2H_MAILBOX_DATA: + brcmf_dbg(MSGBUF, "MSGBUF_TYPE_D2H_MAILBOX_DATA\n"); + brcmf_msgbuf_process_d2h_mailbox_data(msgbuf, buf); + break; default: bphy_err(drvr, "Unsupported msgtype %d\n", msg->msgtype); break; @@ -1465,6 +1518,38 @@ void brcmf_msgbuf_delete_flowring(struct brcmf_pub *drvr, u16 flowid) } } + +int brcmf_msgbuf_h2d_mb_write(struct brcmf_pub *drvr, u32 data) +{ + struct brcmf_msgbuf *msgbuf = (struct brcmf_msgbuf *)drvr->proto->pd; + struct brcmf_commonring *commonring; + struct msgbuf_h2d_mailbox_data *request; + void *ret_ptr; + int err; + + commonring = msgbuf->commonrings[BRCMF_H2D_MSGRING_CONTROL_SUBMIT]; + brcmf_commonring_lock(commonring); + ret_ptr = brcmf_commonring_reserve_for_write(commonring); + if (!ret_ptr) { + bphy_err(drvr, "Failed to reserve space in commonring\n"); + brcmf_commonring_unlock(commonring); + return -ENOMEM; + } + + request = (struct msgbuf_h2d_mailbox_data *)ret_ptr; + request->msg.msgtype = MSGBUF_TYPE_H2D_MAILBOX_DATA; + request->msg.ifidx = -1; + request->msg.flags = 0; + request->msg.request_id = 0; + request->data = data; + + err = brcmf_commonring_write_complete(commonring); + brcmf_commonring_unlock(commonring); + + return err; +} + + #ifdef DEBUG static int brcmf_msgbuf_stats_read(struct seq_file *seq, void *data) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h index 6a849f4a94dd7f..0ed48cf13d93cf 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h @@ -8,10 +8,10 @@ #ifdef CONFIG_BRCMFMAC_PROTO_MSGBUF #define BRCMF_H2D_MSGRING_CONTROL_SUBMIT_MAX_ITEM 64 -#define BRCMF_H2D_MSGRING_RXPOST_SUBMIT_MAX_ITEM 1024 +#define BRCMF_H2D_MSGRING_RXPOST_SUBMIT_MAX_ITEM 2048 #define BRCMF_D2H_MSGRING_CONTROL_COMPLETE_MAX_ITEM 64 #define BRCMF_D2H_MSGRING_TX_COMPLETE_MAX_ITEM 1024 -#define BRCMF_D2H_MSGRING_RX_COMPLETE_MAX_ITEM 1024 +#define BRCMF_D2H_MSGRING_RX_COMPLETE_MAX_ITEM 2048 #define BRCMF_H2D_TXFLOWRING_MAX_ITEM 512 #define BRCMF_H2D_MSGRING_CONTROL_SUBMIT_ITEMSIZE 40 @@ -32,6 +32,7 @@ int brcmf_proto_msgbuf_rx_trigger(struct device *dev); void brcmf_msgbuf_delete_flowring(struct brcmf_pub *drvr, u16 flowid); int brcmf_proto_msgbuf_attach(struct brcmf_pub *drvr); void brcmf_proto_msgbuf_detach(struct brcmf_pub *drvr); +int brcmf_msgbuf_h2d_mb_write(struct brcmf_pub *drvr, u32 data); #else static inline int brcmf_proto_msgbuf_attach(struct brcmf_pub *drvr) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c index e1752a513c733d..06d2933162b26f 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c @@ -1784,8 +1784,8 @@ bool brcmf_p2p_send_action_frame(struct brcmf_if *ifp, /* do not configure anything. it will be */ /* sent with a default configuration */ } else { - bphy_err(drvr, "Unknown Frame: category 0x%x, action 0x%x\n", - category, action); + bphy_info_once(drvr, "Unknown Frame: category 0x%x, action 0x%x\n", + category, action); return false; } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index 6327f4eca50078..9f11d555ade453 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -71,6 +71,8 @@ BRCMF_FW_CLM_DEF(4377B3, "brcmfmac4377b3-pcie"); BRCMF_FW_CLM_DEF(4378B1, "brcmfmac4378b1-pcie"); BRCMF_FW_CLM_DEF(4378B3, "brcmfmac4378b3-pcie"); BRCMF_FW_CLM_DEF(4387C2, "brcmfmac4387c2-pcie"); +BRCMF_FW_CLM_DEF(4388B0, "brcmfmac4388b0-pcie"); +BRCMF_FW_CLM_DEF(4388C0, "brcmfmac4388c0-pcie"); BRCMF_FW_CLM_DEF(54591, "brcmfmac54591-pcie"); /* firmware config files */ @@ -112,6 +114,8 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { BRCMF_FW_ENTRY(BRCM_CC_4378_CHIP_ID, 0x0000000F, 4378B1), /* revision ID 3 */ BRCMF_FW_ENTRY(BRCM_CC_4378_CHIP_ID, 0xFFFFFFE0, 4378B3), /* revision ID 5 */ BRCMF_FW_ENTRY(BRCM_CC_4387_CHIP_ID, 0xFFFFFFFF, 4387C2), /* revision ID 7 */ + BRCMF_FW_ENTRY(BRCM_CC_4388_CHIP_ID, 0x0000000F, 4388B0), + BRCMF_FW_ENTRY(BRCM_CC_4388_CHIP_ID, 0xFFFFFFF0, 4388C0), /* revision ID 4 */ }; #define BRCMF_PCIE_FW_UP_TIMEOUT 5000 /* msec */ @@ -219,11 +223,64 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { #define BRCMF_PCIE_SHARED_VERSION_MASK 0x00FF #define BRCMF_PCIE_SHARED_DMA_INDEX 0x10000 #define BRCMF_PCIE_SHARED_DMA_2B_IDX 0x100000 +#define BRCMF_PCIE_SHARED_USE_MAILBOX 0x2000000 +#define BRCMF_PCIE_SHARED_TIMESTAMP_DB0 0x8000000 #define BRCMF_PCIE_SHARED_HOSTRDY_DB1 0x10000000 +#define BRCMF_PCIE_SHARED_NO_OOB_DW 0x20000000 +#define BRCMF_PCIE_SHARED_INBAND_DS 0x40000000 +#define BRCMF_PCIE_SHARED_DAR 0x80000000 + +#define BRCMF_PCIE_SHARED2_EXTENDED_TRAP_DATA 0x1 +#define BRCMF_PCIE_SHARED2_TXSTATUS_METADATA 0x2 +#define BRCMF_PCIE_SHARED2_BT_LOGGING 0x4 +#define BRCMF_PCIE_SHARED2_SNAPSHOT_UPLOAD 0x8 +#define BRCMF_PCIE_SHARED2_SUBMIT_COUNT_WAR 0x10 +#define BRCMF_PCIE_SHARED2_FAST_DELETE_RING 0x20 +#define BRCMF_PCIE_SHARED2_EVTBUF_MAX_MASK 0xC0 +#define BRCMF_PCIE_SHARED2_PKT_TX_STATUS 0x100 +#define BRCMF_PCIE_SHARED2_FW_SMALL_MEMDUMP 0x200 +#define BRCMF_PCIE_SHARED2_FW_HC_ON_TRAP 0x400 +#define BRCMF_PCIE_SHARED2_HSCB 0x800 +#define BRCMF_PCIE_SHARED2_EDL_RING 0x1000 +#define BRCMF_PCIE_SHARED2_DEBUG_BUF_DEST 0x2000 +#define BRCMF_PCIE_SHARED2_PCIE_ENUM_RESET_FLR 0x4000 +#define BRCMF_PCIE_SHARED2_PKT_TIMESTAMP 0x8000 +#define BRCMF_PCIE_SHARED2_HP2P 0x10000 +#define BRCMF_PCIE_SHARED2_HWA 0x20000 +#define BRCMF_PCIE_SHARED2_TRAP_ON_HOST_DB7 0x40000 +#define BRCMF_PCIE_SHARED2_DURATION_SCALE 0x100000 +#define BRCMF_PCIE_SHARED2_D2H_D11_TX_STATUS 0x40000000 +#define BRCMF_PCIE_SHARED2_H2D_D11_TX_STATUS 0x80000000 #define BRCMF_PCIE_FLAGS_HTOD_SPLIT 0x4000 #define BRCMF_PCIE_FLAGS_DTOH_SPLIT 0x8000 +#define BRCMF_HOSTCAP_PCIEAPI_VERSION_MASK 0x000000FF +#define BRCMF_HOSTCAP_H2D_VALID_PHASE 0x00000100 +#define BRCMF_HOSTCAP_H2D_ENABLE_TRAP_ON_BADPHASE 0x00000200 +#define BRCMF_HOSTCAP_H2D_ENABLE_HOSTRDY 0x400 +#define BRCMF_HOSTCAP_DB0_TIMESTAMP 0x800 +#define BRCMF_HOSTCAP_DS_NO_OOB_DW 0x1000 +#define BRCMF_HOSTCAP_DS_INBAND_DW 0x2000 +#define BRCMF_HOSTCAP_H2D_IDMA 0x4000 +#define BRCMF_HOSTCAP_H2D_IFRM 0x8000 +#define BRCMF_HOSTCAP_H2D_DAR 0x10000 +#define BRCMF_HOSTCAP_EXTENDED_TRAP_DATA 0x20000 +#define BRCMF_HOSTCAP_TXSTATUS_METADATA 0x40000 +#define BRCMF_HOSTCAP_BT_LOGGING 0x80000 +#define BRCMF_HOSTCAP_SNAPSHOT_UPLOAD 0x100000 +#define BRCMF_HOSTCAP_FAST_DELETE_RING 0x200000 +#define BRCMF_HOSTCAP_PKT_TXSTATUS 0x400000 +#define BRCMF_HOSTCAP_UR_FW_NO_TRAP 0x800000 +#define BRCMF_HOSTCAP_HSCB 0x2000000 +#define BRCMF_HOSTCAP_EXT_TRAP_DBGBUF 0x4000000 +#define BRCMF_HOSTCAP_EDL_RING 0x10000000 +#define BRCMF_HOSTCAP_PKT_TIMESTAMP 0x20000000 +#define BRCMF_HOSTCAP_PKT_HP2P 0x40000000 +#define BRCMF_HOSTCAP_HWA 0x80000000 +#define BRCMF_HOSTCAP2_DURATION_SCALE_MASK 0x3F + +#define BRCMF_SHARED_FLAGS_OFFSET 0 #define BRCMF_SHARED_MAX_RXBUFPOST_OFFSET 34 #define BRCMF_SHARED_RING_BASE_OFFSET 52 #define BRCMF_SHARED_RX_DATAOFFSET_OFFSET 36 @@ -235,6 +292,11 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { #define BRCMF_SHARED_DMA_SCRATCH_ADDR_OFFSET 56 #define BRCMF_SHARED_DMA_RINGUPD_LEN_OFFSET 64 #define BRCMF_SHARED_DMA_RINGUPD_ADDR_OFFSET 68 +#define BRCMF_SHARED_FLAGS2_OFFSET 80 +#define BRCMF_SHARED_HOST_CAP_OFFSET 84 +#define BRCMF_SHARED_FLAGS3_OFFSET 108 +#define BRCMF_SHARED_HOST_CAP2_OFFSET 112 +#define BRCMF_SHARED_HOST_CAP3_OFFSET 116 #define BRCMF_RING_H2D_RING_COUNT_OFFSET 0 #define BRCMF_RING_D2H_RING_COUNT_OFFSET 1 @@ -280,6 +342,7 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { #define BRCMF_PCIE_CFGREG_PML1_SUB_CTRL1 0x248 #define BRCMF_PCIE_CFGREG_REG_BAR2_CONFIG 0x4E0 #define BRCMF_PCIE_CFGREG_REG_BAR3_CONFIG 0x4F4 +#define BRCMF_PCIE_CFGREG_TLCNTRL_5 0x814 #define BRCMF_PCIE_LINK_STATUS_CTRL_ASPM_ENAB 3 /* Magic number at a magic location to find RAM size */ @@ -299,6 +362,8 @@ struct brcmf_pcie_console { struct brcmf_pcie_shared_info { u32 tcm_base_address; u32 flags; + u32 flags2; + u32 flags3; struct brcmf_pcie_ringbuf *commonrings[BRCMF_NROF_COMMON_MSGRINGS]; struct brcmf_pcie_ringbuf *flowrings; u16 max_rxbufpost; @@ -315,6 +380,7 @@ struct brcmf_pcie_shared_info { void *ringupd; dma_addr_t ringupd_dmahandle; u8 version; + bool mb_via_ctl; }; #define BRCMF_OTP_MAX_PARAM_LEN 16 @@ -331,6 +397,7 @@ struct brcmf_pciedev_info { bool in_irq; struct pci_dev *pdev; char fw_name[BRCMF_FW_NAME_LEN]; + char sig_name[BRCMF_FW_NAME_LEN]; char nvram_name[BRCMF_FW_NAME_LEN]; char clm_name[BRCMF_FW_NAME_LEN]; char txcap_name[BRCMF_FW_NAME_LEN]; @@ -339,14 +406,16 @@ struct brcmf_pciedev_info { const struct brcmf_pcie_reginfo *reginfo; void __iomem *regs; void __iomem *tcm; - u32 ram_base; - u32 ram_size; + u32 fw_size; + bool skip_reset_vector; struct brcmf_chip *ci; u32 coreid; struct brcmf_pcie_shared_info shared; wait_queue_head_t mbdata_resp_wait; bool mbdata_completed; bool irq_allocated; + bool irq_ready; + bool have_msi; bool wowl_enabled; u8 dma_idx_sz; void *idxbuf; @@ -433,8 +502,6 @@ struct brcmf_pcie_reginfo { u32 intmask; u32 mailboxint; u32 mailboxmask; - u32 h2d_mailbox_0; - u32 h2d_mailbox_1; u32 int_d2h_db; u32 int_fn0; }; @@ -443,8 +510,6 @@ static const struct brcmf_pcie_reginfo brcmf_reginfo_default = { .intmask = BRCMF_PCIE_PCIE2REG_INTMASK, .mailboxint = BRCMF_PCIE_PCIE2REG_MAILBOXINT, .mailboxmask = BRCMF_PCIE_PCIE2REG_MAILBOXMASK, - .h2d_mailbox_0 = BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_0, - .h2d_mailbox_1 = BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_1, .int_d2h_db = BRCMF_PCIE_MB_INT_D2H_DB, .int_fn0 = BRCMF_PCIE_MB_INT_FN0, }; @@ -453,8 +518,6 @@ static const struct brcmf_pcie_reginfo brcmf_reginfo_64 = { .intmask = BRCMF_PCIE_64_PCIE2REG_INTMASK, .mailboxint = BRCMF_PCIE_64_PCIE2REG_MAILBOXINT, .mailboxmask = BRCMF_PCIE_64_PCIE2REG_MAILBOXMASK, - .h2d_mailbox_0 = BRCMF_PCIE_64_PCIE2REG_H2D_MAILBOX_0, - .h2d_mailbox_1 = BRCMF_PCIE_64_PCIE2REG_H2D_MAILBOX_1, .int_d2h_db = BRCMF_PCIE_64_MB_INT_D2H_DB, .int_fn0 = 0, }; @@ -493,6 +556,19 @@ brcmf_pcie_write_reg32(struct brcmf_pciedev_info *devinfo, u32 reg_offset, iowrite32(value, address); } +static u32 +brcmf_pcie_read_pcie32(struct brcmf_pciedev_info *devinfo, u32 reg_offset) +{ + return brcmf_pcie_read_reg32(devinfo, 0x2000 + reg_offset); +} + + +static void +brcmf_pcie_write_pcie32(struct brcmf_pciedev_info *devinfo, u32 reg_offset, + u32 value) +{ + brcmf_pcie_write_reg32(devinfo, 0x2000 + reg_offset, value); +} static u8 brcmf_pcie_read_tcm8(struct brcmf_pciedev_info *devinfo, u32 mem_offset) @@ -686,8 +762,30 @@ static void brcmf_pcie_reset_device(struct brcmf_pciedev_info *devinfo) /* Watchdog reset */ brcmf_pcie_select_core(devinfo, BCMA_CORE_CHIPCOMMON); - WRITECC32(devinfo, watchdog, 4); - msleep(100); + core = brcmf_chip_get_chipcommon(devinfo->ci); + + if (core->rev >= 65) { + u32 mask = CC_WD_SSRESET_PCIE_F0_EN; + + core = brcmf_chip_get_core(devinfo->ci, BCMA_CORE_PCIE2); + if (core->rev < 66) + mask |= CC_WD_SSRESET_PCIE_ALL_FN_EN; + + val = READCC32(devinfo, watchdog); + val &= ~CC_WD_ENABLE_MASK; + val |= mask; + WRITECC32(devinfo, watchdog, val); + val &= ~CC_WD_COUNTER_MASK; + val |= 4; + WRITECC32(devinfo, watchdog, val); + msleep(10); + val = READCC32(devinfo, intstatus); + val |= mask; + WRITECC32(devinfo, intstatus, val); + } else { + WRITECC32(devinfo, watchdog, 4); + msleep(100); + } /* Restore ASPM */ brcmf_pcie_select_core(devinfo, BCMA_CORE_PCIE2); @@ -697,14 +795,14 @@ static void brcmf_pcie_reset_device(struct brcmf_pciedev_info *devinfo) core = brcmf_chip_get_core(devinfo->ci, BCMA_CORE_PCIE2); if (core->rev <= 13) { for (i = 0; i < ARRAY_SIZE(cfg_offset); i++) { - brcmf_pcie_write_reg32(devinfo, + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGADDR, cfg_offset[i]); - val = brcmf_pcie_read_reg32(devinfo, + val = brcmf_pcie_read_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGDATA); brcmf_dbg(PCIE, "config offset 0x%04x, value 0x%04x\n", cfg_offset[i], val); - brcmf_pcie_write_reg32(devinfo, + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGDATA, val); } @@ -718,9 +816,9 @@ static void brcmf_pcie_attach(struct brcmf_pciedev_info *devinfo) /* BAR1 window may not be sized properly */ brcmf_pcie_select_core(devinfo, BCMA_CORE_PCIE2); - brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGADDR, 0x4e0); - config = brcmf_pcie_read_reg32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGDATA); - brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGDATA, config); + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGADDR, 0x4e0); + config = brcmf_pcie_read_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGDATA); + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGDATA, config); device_wakeup_enable(&devinfo->pdev->dev); } @@ -739,6 +837,21 @@ static int brcmf_pcie_enter_download_state(struct brcmf_pciedev_info *devinfo) brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_ARMCR4REG_BANKPDA, 0); } + + /* Ensure all IRQs are masked so the firmware doesn't get + * a hostready notification too early. + */ + + brcmf_pcie_write_pcie32(devinfo, devinfo->reginfo->mailboxmask, 0); + brcmf_pcie_write_pcie32(devinfo, devinfo->reginfo->mailboxint, + 0xffffffff); + + pci_write_config_dword(devinfo->pdev, BRCMF_PCIE_REG_INTMASK, 0); + + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGADDR, + BRCMF_PCIE_CFGREG_TLCNTRL_5); + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGDATA, + 0xffffffff); return 0; } @@ -769,6 +882,19 @@ brcmf_pcie_send_mb_data(struct brcmf_pciedev_info *devinfo, u32 htod_mb_data) u32 i; shared = &devinfo->shared; + + if (shared->mb_via_ctl) { + struct pci_dev *pdev = devinfo->pdev; + struct brcmf_bus *bus = dev_get_drvdata(&pdev->dev); + int ret; + + ret = brcmf_msgbuf_h2d_mb_write(bus->drvr, htod_mb_data); + if (ret < 0) + brcmf_err(bus, "Failed to send H2D mailbox data (%d)\n", + ret); + return ret; + } + addr = shared->htod_mb_data_addr; cur_htod_mb_data = brcmf_pcie_read_tcm32(devinfo, addr); @@ -796,8 +922,29 @@ brcmf_pcie_send_mb_data(struct brcmf_pciedev_info *devinfo, u32 htod_mb_data) return 0; } +static void brcmf_pcie_handle_mb_data(struct brcmf_pciedev_info *devinfo, u32 data) +{ + brcmf_dbg(PCIE, "D2H_MB_DATA: 0x%04x\n", data); + if (data & BRCMF_D2H_DEV_DS_ENTER_REQ) { + brcmf_dbg(PCIE, "D2H_MB_DATA: DEEP SLEEP REQ\n"); + brcmf_pcie_send_mb_data(devinfo, BRCMF_H2D_HOST_DS_ACK); + brcmf_dbg(PCIE, "D2H_MB_DATA: sent DEEP SLEEP ACK\n"); + } + if (data & BRCMF_D2H_DEV_DS_EXIT_NOTE) + brcmf_dbg(PCIE, "D2H_MB_DATA: DEEP SLEEP EXIT\n"); + if (data & BRCMF_D2H_DEV_D3_ACK) { + brcmf_dbg(PCIE, "D2H_MB_DATA: D3 ACK\n"); + devinfo->mbdata_completed = true; + wake_up(&devinfo->mbdata_resp_wait); + } + if (data & BRCMF_D2H_DEV_FWHALT) { + brcmf_dbg(PCIE, "D2H_MB_DATA: FW HALT\n"); + brcmf_fw_crashed(&devinfo->pdev->dev); + } +} -static void brcmf_pcie_handle_mb_data(struct brcmf_pciedev_info *devinfo) + +static void brcmf_pcie_poll_mb_data(struct brcmf_pciedev_info *devinfo) { struct brcmf_pcie_shared_info *shared; u32 addr; @@ -812,23 +959,16 @@ static void brcmf_pcie_handle_mb_data(struct brcmf_pciedev_info *devinfo) brcmf_pcie_write_tcm32(devinfo, addr, 0); - brcmf_dbg(PCIE, "D2H_MB_DATA: 0x%04x\n", dtoh_mb_data); - if (dtoh_mb_data & BRCMF_D2H_DEV_DS_ENTER_REQ) { - brcmf_dbg(PCIE, "D2H_MB_DATA: DEEP SLEEP REQ\n"); - brcmf_pcie_send_mb_data(devinfo, BRCMF_H2D_HOST_DS_ACK); - brcmf_dbg(PCIE, "D2H_MB_DATA: sent DEEP SLEEP ACK\n"); - } - if (dtoh_mb_data & BRCMF_D2H_DEV_DS_EXIT_NOTE) - brcmf_dbg(PCIE, "D2H_MB_DATA: DEEP SLEEP EXIT\n"); - if (dtoh_mb_data & BRCMF_D2H_DEV_D3_ACK) { - brcmf_dbg(PCIE, "D2H_MB_DATA: D3 ACK\n"); - devinfo->mbdata_completed = true; - wake_up(&devinfo->mbdata_resp_wait); - } - if (dtoh_mb_data & BRCMF_D2H_DEV_FWHALT) { - brcmf_dbg(PCIE, "D2H_MB_DATA: FW HALT\n"); - brcmf_fw_crashed(&devinfo->pdev->dev); - } + brcmf_pcie_handle_mb_data(devinfo, dtoh_mb_data); +} + + +static void brcmf_pcie_d2h_mb_rx(struct device *dev, u32 data) +{ + struct brcmf_bus *bus_if = dev_get_drvdata(dev); + struct brcmf_pciedev *buspub = bus_if->bus_priv.pcie; + + brcmf_pcie_handle_mb_data(buspub->devinfo, data); } @@ -907,33 +1047,45 @@ static void brcmf_pcie_bus_console_read(struct brcmf_pciedev_info *devinfo, static void brcmf_pcie_intr_disable(struct brcmf_pciedev_info *devinfo) { - brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->mailboxmask, 0); + brcmf_pcie_write_pcie32(devinfo, devinfo->reginfo->mailboxmask, 0); + + devinfo->irq_ready = false; } static void brcmf_pcie_intr_enable(struct brcmf_pciedev_info *devinfo) { - brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->mailboxmask, + brcmf_pcie_write_pcie32(devinfo, devinfo->reginfo->mailboxmask, devinfo->reginfo->int_d2h_db | devinfo->reginfo->int_fn0); + + devinfo->irq_ready = true; } static void brcmf_pcie_hostready(struct brcmf_pciedev_info *devinfo) { - if (devinfo->shared.flags & BRCMF_PCIE_SHARED_HOSTRDY_DB1) - brcmf_pcie_write_reg32(devinfo, - devinfo->reginfo->h2d_mailbox_1, 1); + if (devinfo->shared.flags & BRCMF_PCIE_SHARED_HOSTRDY_DB1) { + if (devinfo->shared.flags & BRCMF_PCIE_SHARED_DAR) + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_64_PCIE2REG_H2D_MAILBOX_1, 1); + else + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_1, 1); + } } static irqreturn_t brcmf_pcie_quick_check_isr(int irq, void *arg) { struct brcmf_pciedev_info *devinfo = (struct brcmf_pciedev_info *)arg; - if (brcmf_pcie_read_reg32(devinfo, devinfo->reginfo->mailboxint)) { + if (brcmf_pcie_read_pcie32(devinfo, devinfo->reginfo->mailboxint)) { brcmf_pcie_intr_disable(devinfo); brcmf_dbg(PCIE, "Enter\n"); return IRQ_WAKE_THREAD; } + + /* mailboxint is cleared by the firmware in MSI mode */ + if (devinfo->have_msi) + return IRQ_WAKE_THREAD; + return IRQ_NONE; } @@ -944,19 +1096,19 @@ static irqreturn_t brcmf_pcie_isr_thread(int irq, void *arg) u32 status; devinfo->in_irq = true; - status = brcmf_pcie_read_reg32(devinfo, devinfo->reginfo->mailboxint); + status = brcmf_pcie_read_pcie32(devinfo, devinfo->reginfo->mailboxint); brcmf_dbg(PCIE, "Enter %x\n", status); if (status) { - brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->mailboxint, + brcmf_pcie_write_pcie32(devinfo, devinfo->reginfo->mailboxint, status); if (status & devinfo->reginfo->int_fn0) - brcmf_pcie_handle_mb_data(devinfo); - if (status & devinfo->reginfo->int_d2h_db) { - if (devinfo->state == BRCMFMAC_PCIE_STATE_UP) - brcmf_proto_msgbuf_rx_trigger( - &devinfo->pdev->dev); - } + brcmf_pcie_poll_mb_data(devinfo); } + if (devinfo->have_msi || status & devinfo->reginfo->int_d2h_db) { + if (devinfo->state == BRCMFMAC_PCIE_STATE_UP && devinfo->irq_ready) + brcmf_proto_msgbuf_rx_trigger(&devinfo->pdev->dev); + } + brcmf_pcie_bus_console_read(devinfo, false); if (devinfo->state == BRCMFMAC_PCIE_STATE_UP) brcmf_pcie_intr_enable(devinfo); @@ -974,7 +1126,10 @@ static int brcmf_pcie_request_irq(struct brcmf_pciedev_info *devinfo) brcmf_dbg(PCIE, "Enter\n"); - pci_enable_msi(pdev); + devinfo->have_msi = pci_enable_msi(pdev) >= 0; + if (devinfo->have_msi) + brcmf_dbg(PCIE, "MSI enabled\n"); + if (request_threaded_irq(pdev->irq, brcmf_pcie_quick_check_isr, brcmf_pcie_isr_thread, IRQF_SHARED, "brcmf_pcie_intr", devinfo)) { @@ -1010,8 +1165,8 @@ static void brcmf_pcie_release_irq(struct brcmf_pciedev_info *devinfo) if (devinfo->in_irq) brcmf_err(bus, "Still in IRQ (processing) !!!\n"); - status = brcmf_pcie_read_reg32(devinfo, devinfo->reginfo->mailboxint); - brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->mailboxint, status); + status = brcmf_pcie_read_pcie32(devinfo, devinfo->reginfo->mailboxint); + brcmf_pcie_write_pcie32(devinfo, devinfo->reginfo->mailboxint, status); devinfo->irq_allocated = false; } @@ -1063,7 +1218,10 @@ static int brcmf_pcie_ring_mb_ring_bell(void *ctx) brcmf_dbg(PCIE, "RING !\n"); /* Any arbitrary value will do, lets use 1 */ - brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->h2d_mailbox_0, 1); + if (devinfo->shared.flags & BRCMF_PCIE_SHARED_DAR) + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_64_PCIE2REG_H2D_MAILBOX_0, 1); + else + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_0, 1); return 0; } @@ -1589,6 +1747,7 @@ static const struct brcmf_bus_ops brcmf_pcie_bus_ops = { .get_blob = brcmf_pcie_get_blob, .reset = brcmf_pcie_reset, .debugfs_create = brcmf_pcie_debugfs_create, + .d2h_mb_rx = brcmf_pcie_d2h_mb_rx, }; @@ -1620,12 +1779,16 @@ brcmf_pcie_init_share_ram_info(struct brcmf_pciedev_info *devinfo, { struct brcmf_bus *bus = dev_get_drvdata(&devinfo->pdev->dev); struct brcmf_pcie_shared_info *shared; + u32 host_cap; + u32 host_cap2; u32 addr; shared = &devinfo->shared; shared->tcm_base_address = sharedram_addr; - shared->flags = brcmf_pcie_read_tcm32(devinfo, sharedram_addr); + shared->flags = brcmf_pcie_read_tcm32(devinfo, sharedram_addr + + BRCMF_SHARED_FLAGS_OFFSET); + shared->version = (u8)(shared->flags & BRCMF_PCIE_SHARED_VERSION_MASK); brcmf_dbg(PCIE, "PCIe protocol version %d\n", shared->version); if ((shared->version > BRCMF_PCIE_MAX_SHARED_VERSION) || @@ -1666,29 +1829,223 @@ brcmf_pcie_init_share_ram_info(struct brcmf_pciedev_info *devinfo, brcmf_pcie_bus_console_init(devinfo); brcmf_pcie_bus_console_read(devinfo, false); + /* Features added in revision 6 follow */ + if (shared->version < 6) + return 0; + + shared->flags2 = brcmf_pcie_read_tcm32(devinfo, sharedram_addr + + BRCMF_SHARED_FLAGS2_OFFSET); + shared->flags3 = brcmf_pcie_read_tcm32(devinfo, sharedram_addr + + BRCMF_SHARED_FLAGS3_OFFSET); + + /* Check which mailbox mechanism to use */ + if (!(shared->flags & BRCMF_PCIE_SHARED_USE_MAILBOX)) + shared->mb_via_ctl = true; + + /* Update host support flags */ + host_cap = shared->version; + host_cap2 = 0; + + if (shared->flags & BRCMF_PCIE_SHARED_HOSTRDY_DB1) + host_cap |= BRCMF_HOSTCAP_H2D_ENABLE_HOSTRDY; + + if (shared->flags & BRCMF_PCIE_SHARED_DAR) + host_cap |= BRCMF_HOSTCAP_H2D_DAR; + + /* Disable DS: this is not currently properly supported */ + host_cap |= BRCMF_HOSTCAP_DS_NO_OOB_DW; + + brcmf_pcie_write_tcm32(devinfo, sharedram_addr + + BRCMF_SHARED_HOST_CAP_OFFSET, host_cap); + brcmf_pcie_write_tcm32(devinfo, sharedram_addr + + BRCMF_SHARED_HOST_CAP2_OFFSET, host_cap2); + return 0; } -struct brcmf_random_seed_footer { +struct brcmf_rtlv_footer { __le32 length; __le32 magic; }; +/** struct brcmf_fw_memmap_region - start/end of memory regions for chip + */ +struct brcmf_fw_memmap_region { + u32 start; + u32 end; +}; + +/** struct brcmf_fw_memmap + * + * @reset_vec - Reset vector - read only + * @int_vec - copied from ram, jumps here on success + * @rom - bootloader at rom start + * @mmap - struct/memory map written by host + * @vstatus - verification status + * @fw - firmware + * @sig - firwmare signature + * @heap - region for heap allocations + * @stack - region for stack allocations + * @prng - PRNG data, may be 0 length + * @nvram - NVRAM data + */ +struct brcmf_fw_memmap { + struct brcmf_fw_memmap_region reset_vec; + struct brcmf_fw_memmap_region int_vec; + struct brcmf_fw_memmap_region rom; + struct brcmf_fw_memmap_region mmap; + struct brcmf_fw_memmap_region vstatus; + struct brcmf_fw_memmap_region fw; + struct brcmf_fw_memmap_region sig; + struct brcmf_fw_memmap_region heap; + struct brcmf_fw_memmap_region stack; + struct brcmf_fw_memmap_region prng; + struct brcmf_fw_memmap_region nvram; +}; + +#define BRCMF_BL_HEAP_START_GAP 0x1000 +#define BRCMF_BL_HEAP_SIZE 0x10000 #define BRCMF_RANDOM_SEED_MAGIC 0xfeedc0de #define BRCMF_RANDOM_SEED_LENGTH 0x100 +#define BRCMF_FW_SIG_MAGIC 0xfeedfe51 +#define BRCMF_NVRAM_SIG_MAGIC 0xfeedfe52 +#define BRCMF_MEMMAP_MAGIC 0xfeedfe53 +#define BRCMF_VSTATUS_MAGIC 0xfeedfe54 +#define BRCMF_VSTATUS_SIZE 0x28 +#define BRCMF_END_MAGIC 0xfeed0e2d -static noinline_for_stack void -brcmf_pcie_provide_random_bytes(struct brcmf_pciedev_info *devinfo, u32 address) +static int brcmf_alloc_rtlv(struct brcmf_pciedev_info *devinfo, u32 *address, u32 type, u32 length) { + struct brcmf_bus *bus = dev_get_drvdata(&devinfo->pdev->dev); + u32 fw_top = devinfo->ci->rambase + devinfo->fw_size; + u32 ram_start = ALIGN(fw_top + BRCMF_BL_HEAP_START_GAP, 4); + u32 ram_end = ram_start + BRCMF_BL_HEAP_SIZE; + u32 start_addr; + struct brcmf_rtlv_footer footer = { + .magic = type, + }; + + length = ALIGN(length, 4); + start_addr = *address - length - sizeof(struct brcmf_rtlv_footer); + + if (length > 0xffff || start_addr > *address || start_addr < ram_end) { + brcmf_err(bus, "failed to allocate 0x%x bytes for rTLV type 0x%x\n", + length, type); + return -ENOMEM; + } + + /* Random seed does not use the length check code */ + if (type == BRCMF_RANDOM_SEED_MAGIC) + footer.length = length; + else + footer.length = length | ((length ^ 0xffff) << 16); + + memcpy_toio(devinfo->tcm + *address - sizeof(struct brcmf_rtlv_footer), + &footer, sizeof(struct brcmf_rtlv_footer)); + + *address = start_addr; + + return 0; +} + +static noinline_for_stack int +brcmf_pcie_add_random_seed(struct brcmf_pciedev_info *devinfo, u32 *address) +{ + int err; u8 randbuf[BRCMF_RANDOM_SEED_LENGTH]; + err = brcmf_alloc_rtlv(devinfo, address, + BRCMF_RANDOM_SEED_MAGIC, BRCMF_RANDOM_SEED_LENGTH); + if (err) + return err; + + /* Some Apple chips/firmwares expect a buffer of random + * data to be present before NVRAM + */ + brcmf_dbg(PCIE, "Download random seed\n"); + get_random_bytes(randbuf, BRCMF_RANDOM_SEED_LENGTH); - memcpy_toio(devinfo->tcm + address, randbuf, BRCMF_RANDOM_SEED_LENGTH); + memcpy_toio(devinfo->tcm + *address, randbuf, BRCMF_RANDOM_SEED_LENGTH); + + return 0; +} + +static int brcmf_pcie_add_signature(struct brcmf_pciedev_info *devinfo, + u32 *address, const struct firmware *fwsig) +{ + int err; + struct brcmf_fw_memmap memmap; + + brcmf_dbg(PCIE, "Download firmware signature\n"); + + memset(&memmap, 0, sizeof(memmap)); + + memmap.sig.end = *address; + err = brcmf_alloc_rtlv(devinfo, address, BRCMF_FW_SIG_MAGIC, fwsig->size); + if (err) + return err; + memmap.sig.start = *address; + + memmap.vstatus.end = *address; + err = brcmf_alloc_rtlv(devinfo, address, BRCMF_VSTATUS_MAGIC, BRCMF_VSTATUS_SIZE); + if (err) + return err; + memmap.vstatus.start = *address; + + err = brcmf_alloc_rtlv(devinfo, address, BRCMF_MEMMAP_MAGIC, sizeof(memmap)); + if (err) + return err; + + memmap.fw.start = devinfo->ci->rambase; + memmap.fw.end = memmap.fw.start + devinfo->fw_size; + memmap.heap.start = ALIGN(memmap.fw.end + BRCMF_BL_HEAP_START_GAP, 4); + memmap.heap.end = memmap.heap.start + BRCMF_BL_HEAP_SIZE; + + if (memmap.heap.end > *address) + return -ENOMEM; + + memcpy_toio(devinfo->tcm + memmap.sig.start, fwsig->data, fwsig->size); + memset_io(devinfo->tcm + memmap.vstatus.start, 0, BRCMF_VSTATUS_SIZE); + memcpy_toio(devinfo->tcm + *address, &memmap, sizeof(memmap)); + + err = brcmf_alloc_rtlv(devinfo, address, BRCMF_END_MAGIC, 0); + if (err) + return err; + + devinfo->skip_reset_vector = true; + + return 0; +} + +static int brcmf_pcie_populate_footers(struct brcmf_pciedev_info *devinfo, + u32 *address, const struct firmware *fwsig) +{ + int err; + + /* We only do this for Apple firmwares. If any other + * production firmwares are found to need this, the condition + * needs to be adjusted. + */ + if (!devinfo->fwseed) + return 0; + + err = brcmf_pcie_add_random_seed(devinfo, address); + if (err) + return err; + + if (fwsig) { + err = brcmf_pcie_add_signature(devinfo, address, fwsig); + if (err) + return err; + } + + return 0; } static int brcmf_pcie_download_fw_nvram(struct brcmf_pciedev_info *devinfo, - const struct firmware *fw, void *nvram, - u32 nvram_len) + const struct firmware *fw, + const struct firmware *fwsig, + void *nvram, u32 nvram_len) { struct brcmf_bus *bus = dev_get_drvdata(&devinfo->pdev->dev); u32 sharedram_addr; @@ -1708,6 +2065,7 @@ static int brcmf_pcie_download_fw_nvram(struct brcmf_pciedev_info *devinfo, (void *)fw->data, fw->size); resetintr = get_unaligned_le32(fw->data); + devinfo->fw_size = fw->size; release_firmware(fw); /* reset last 4 bytes of RAM address. to be used for shared @@ -1715,37 +2073,31 @@ static int brcmf_pcie_download_fw_nvram(struct brcmf_pciedev_info *devinfo, */ brcmf_pcie_write_ram32(devinfo, devinfo->ci->ramsize - 4, 0); + address = devinfo->ci->rambase + devinfo->ci->ramsize; + if (nvram) { brcmf_dbg(PCIE, "Download NVRAM %s\n", devinfo->nvram_name); - address = devinfo->ci->rambase + devinfo->ci->ramsize - - nvram_len; + address -= nvram_len; memcpy_toio(devinfo->tcm + address, nvram, nvram_len); brcmf_fw_nvram_free(nvram); - if (devinfo->fwseed) { - size_t rand_len = BRCMF_RANDOM_SEED_LENGTH; - struct brcmf_random_seed_footer footer = { - .length = cpu_to_le32(rand_len), - .magic = cpu_to_le32(BRCMF_RANDOM_SEED_MAGIC), - }; - - /* Some chips/firmwares expect a buffer of random - * data to be present before NVRAM - */ - brcmf_dbg(PCIE, "Download random seed\n"); - - address -= sizeof(footer); - memcpy_toio(devinfo->tcm + address, &footer, - sizeof(footer)); - - address -= rand_len; - brcmf_pcie_provide_random_bytes(devinfo, address); - } + err = brcmf_pcie_populate_footers(devinfo, &address, fwsig); + if (err) + brcmf_err(bus, "failed to populate firmware footers err=%d\n", err); } else { brcmf_dbg(PCIE, "No matching NVRAM file found %s\n", devinfo->nvram_name); } + release_firmware(fwsig); + + /* Clear free TCM. This isn't really necessary, but it + * makes debugging memory dumps a lot easier since we + * don't get a bunch of junk filling up the free space. + */ + memset_io(devinfo->tcm + devinfo->ci->rambase + devinfo->fw_size, + 0, address - devinfo->fw_size - devinfo->ci->rambase); + sharedram_addr_written = brcmf_pcie_read_ram32(devinfo, devinfo->ci->ramsize - 4); @@ -1889,9 +2241,9 @@ static int brcmf_pcie_buscore_reset(void *ctx, struct brcmf_chip *chip) else reg = BRCMF_PCIE_PCIE2REG_MAILBOXINT; - val = brcmf_pcie_read_reg32(devinfo, reg); + val = brcmf_pcie_read_pcie32(devinfo, reg); if (val != 0xffffffff) - brcmf_pcie_write_reg32(devinfo, reg, val); + brcmf_pcie_write_pcie32(devinfo, reg, val); return 0; } @@ -1902,7 +2254,8 @@ static void brcmf_pcie_buscore_activate(void *ctx, struct brcmf_chip *chip, { struct brcmf_pciedev_info *devinfo = (struct brcmf_pciedev_info *)ctx; - brcmf_pcie_write_tcm32(devinfo, 0, rstvec); + if (!devinfo->skip_reset_vector) + brcmf_pcie_write_tcm32(devinfo, 0, rstvec); } @@ -2073,6 +2426,11 @@ static int brcmf_pcie_read_otp(struct brcmf_pciedev_info *devinfo) base = 0x113c; words = 0x170; break; + case BRCM_CC_4388_CHIP_ID: + coreid = BCMA_CORE_GCI; + base = 0x115c; + words = 0x150; + break; default: /* OTP not supported on this chip */ return 0; @@ -2131,11 +2489,12 @@ static int brcmf_pcie_read_otp(struct brcmf_pciedev_info *devinfo) #define BRCMF_PCIE_FW_NVRAM 1 #define BRCMF_PCIE_FW_CLM 2 #define BRCMF_PCIE_FW_TXCAP 3 +#define BRCMF_PCIE_FW_SIG 4 static void brcmf_pcie_setup(struct device *dev, int ret, struct brcmf_fw_request *fwreq) { - const struct firmware *fw; + const struct firmware *fw, *fwsig; void *nvram; struct brcmf_bus *bus; struct brcmf_pciedev *pcie_bus_dev; @@ -2154,6 +2513,7 @@ static void brcmf_pcie_setup(struct device *dev, int ret, brcmf_pcie_attach(devinfo); fw = fwreq->items[BRCMF_PCIE_FW_CODE].binary; + fwsig = fwreq->items[BRCMF_PCIE_FW_SIG].binary; nvram = fwreq->items[BRCMF_PCIE_FW_NVRAM].nv_data.data; nvram_len = fwreq->items[BRCMF_PCIE_FW_NVRAM].nv_data.len; devinfo->clm_fw = fwreq->items[BRCMF_PCIE_FW_CLM].binary; @@ -2164,6 +2524,7 @@ static void brcmf_pcie_setup(struct device *dev, int ret, if (ret) { brcmf_err(bus, "Failed to get RAM info\n"); release_firmware(fw); + release_firmware(fwsig); brcmf_fw_nvram_free(nvram); goto fail; } @@ -2175,7 +2536,15 @@ static void brcmf_pcie_setup(struct device *dev, int ret, */ brcmf_pcie_adjust_ramsize(devinfo, (u8 *)fw->data, fw->size); - ret = brcmf_pcie_download_fw_nvram(devinfo, fw, nvram, nvram_len); + /* Newer firmwares will signal firmware boot via MSI, so make sure we + * initialize that upfront. + */ + brcmf_pcie_select_core(devinfo, BCMA_CORE_PCIE2); + ret = brcmf_pcie_request_irq(devinfo); + if (ret) + goto fail; + + ret = brcmf_pcie_download_fw_nvram(devinfo, fw, fwsig, nvram, nvram_len); if (ret) goto fail; @@ -2190,9 +2559,6 @@ static void brcmf_pcie_setup(struct device *dev, int ret, goto fail; brcmf_pcie_select_core(devinfo, BCMA_CORE_PCIE2); - ret = brcmf_pcie_request_irq(devinfo); - if (ret) - goto fail; /* hook the commonrings in the bus structure. */ for (i = 0; i < BRCMF_NROF_COMMON_MSGRINGS; i++) @@ -2240,6 +2606,7 @@ brcmf_pcie_prepare_fw_request(struct brcmf_pciedev_info *devinfo) { ".txt", devinfo->nvram_name }, { ".clm_blob", devinfo->clm_name }, { ".txcap_blob", devinfo->txcap_name }, + { ".sig", devinfo->sig_name }, }; fwreq = brcmf_fw_alloc_request(devinfo->ci->chip, devinfo->ci->chiprev, @@ -2250,6 +2617,8 @@ brcmf_pcie_prepare_fw_request(struct brcmf_pciedev_info *devinfo) return NULL; fwreq->items[BRCMF_PCIE_FW_CODE].type = BRCMF_FW_TYPE_BINARY; + fwreq->items[BRCMF_PCIE_FW_SIG].type = BRCMF_FW_TYPE_BINARY; + fwreq->items[BRCMF_PCIE_FW_SIG].flags = BRCMF_FW_REQF_OPTIONAL; fwreq->items[BRCMF_PCIE_FW_NVRAM].type = BRCMF_FW_TYPE_NVRAM; fwreq->items[BRCMF_PCIE_FW_NVRAM].flags = BRCMF_FW_REQF_OPTIONAL; fwreq->items[BRCMF_PCIE_FW_CLM].type = BRCMF_FW_TYPE_BINARY; @@ -2668,12 +3037,13 @@ static int brcmf_pcie_pm_leave_D3(struct device *dev) brcmf_dbg(PCIE, "Enter, dev=%p, bus=%p\n", dev, bus); /* Check if device is still up and running, if so we are ready */ - if (brcmf_pcie_read_reg32(devinfo, devinfo->reginfo->intmask) != 0) { + if (brcmf_pcie_read_pcie32(devinfo, devinfo->reginfo->intmask) != 0) { brcmf_dbg(PCIE, "Try to wakeup device....\n"); + /* Set the device up, so we can write the MB data message in ring mode */ + devinfo->state = BRCMFMAC_PCIE_STATE_UP; if (brcmf_pcie_send_mb_data(devinfo, BRCMF_H2D_HOST_D0_INFORM)) goto cleanup; brcmf_dbg(PCIE, "Hot resume, continue....\n"); - devinfo->state = BRCMFMAC_PCIE_STATE_UP; brcmf_pcie_select_core(devinfo, BCMA_CORE_PCIE2); brcmf_bus_change_state(bus, BRCMF_BUS_UP); brcmf_pcie_intr_enable(devinfo); @@ -2683,6 +3053,7 @@ static int brcmf_pcie_pm_leave_D3(struct device *dev) } cleanup: + devinfo->state = BRCMFMAC_PCIE_STATE_DOWN; brcmf_chip_detach(devinfo->ci); devinfo->ci = NULL; pdev = devinfo->pdev; @@ -2750,6 +3121,7 @@ static const struct pci_device_id brcmf_pcie_devid_table[] = { BRCMF_PCIE_DEVICE(BRCM_PCIE_4377_DEVICE_ID, WCC_SEED), BRCMF_PCIE_DEVICE(BRCM_PCIE_4378_DEVICE_ID, WCC_SEED), BRCMF_PCIE_DEVICE(BRCM_PCIE_4387_DEVICE_ID, WCC_SEED), + BRCMF_PCIE_DEVICE(BRCM_PCIE_4388_DEVICE_ID, WCC_SEED), BRCMF_PCIE_DEVICE(BRCM_PCIE_43752_DEVICE_ID, WCC_SEED), BRCMF_PCIE_DEVICE(CY_PCIE_54591_DEVICE_ID, CYW), { /* end: all zeroes */ } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c index 05f66ab13bed6d..dbeeaef75b165a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c @@ -12,8 +12,10 @@ #include "fwil_types.h" #include "cfg80211.h" #include "pno.h" +#include "feature.h" -#define BRCMF_PNO_VERSION 2 +#define BRCMF_PNO_VERSION_2 2 +#define BRCMF_PNO_VERSION_3 3 #define BRCMF_PNO_REPEAT 4 #define BRCMF_PNO_FREQ_EXPO_MAX 3 #define BRCMF_PNO_IMMEDIATE_SCAN_BIT 3 @@ -99,8 +101,62 @@ static int brcmf_pno_channel_config(struct brcmf_if *ifp, return brcmf_fil_iovar_data_set(ifp, "pfn_cfg", cfg, sizeof(*cfg)); } -static int brcmf_pno_config(struct brcmf_if *ifp, u32 scan_freq, - u32 mscan, u32 bestn) +static int brcmf_pno_config_v3(struct brcmf_if *ifp, u32 scan_freq, u32 mscan, + u32 bestn) +{ + struct brcmf_pub *drvr = ifp->drvr; + struct brcmf_pno_param_v3_le pfn_param; + u16 flags; + u32 pfnmem; + s32 err; + + memset(&pfn_param, 0, sizeof(pfn_param)); + pfn_param.version = cpu_to_le16(BRCMF_PNO_VERSION_3); + pfn_param.length = cpu_to_le16(sizeof(struct brcmf_pno_param_v3_le)); + + /* set extra pno params */ + flags = BIT(BRCMF_PNO_IMMEDIATE_SCAN_BIT) | + BIT(BRCMF_PNO_ENABLE_ADAPTSCAN_BIT); + pfn_param.repeat = BRCMF_PNO_REPEAT; + pfn_param.exp = BRCMF_PNO_FREQ_EXPO_MAX; + + /* set up pno scan fr */ + pfn_param.scan_freq = cpu_to_le32(scan_freq); + + if (mscan) { + pfnmem = bestn; + + /* set bestn in firmware */ + err = brcmf_fil_iovar_int_set(ifp, "pfnmem", pfnmem); + if (err < 0) { + bphy_err(drvr, "failed to set pfnmem\n"); + goto exit; + } + /* get max mscan which the firmware supports */ + err = brcmf_fil_iovar_int_get(ifp, "pfnmem", &pfnmem); + if (err < 0) { + bphy_err(drvr, "failed to get pfnmem\n"); + goto exit; + } + mscan = min_t(u32, mscan, pfnmem); + pfn_param.mscan = mscan; + pfn_param.bestn = bestn; + flags |= BIT(BRCMF_PNO_ENABLE_BD_SCAN_BIT); + brcmf_dbg(INFO, "mscan=%d, bestn=%d\n", mscan, bestn); + } + + pfn_param.flags = cpu_to_le16(flags); + err = brcmf_fil_iovar_data_set(ifp, "pfn_set", &pfn_param, + sizeof(pfn_param)); + if (err) + bphy_err(drvr, "pfn_set failed, err=%d\n", err); + +exit: + return err; +} + +static int brcmf_pno_config_v2(struct brcmf_if *ifp, u32 scan_freq, u32 mscan, + u32 bestn) { struct brcmf_pub *drvr = ifp->drvr; struct brcmf_pno_param_le pfn_param; @@ -109,7 +165,7 @@ static int brcmf_pno_config(struct brcmf_if *ifp, u32 scan_freq, s32 err; memset(&pfn_param, 0, sizeof(pfn_param)); - pfn_param.version = cpu_to_le32(BRCMF_PNO_VERSION); + pfn_param.version = cpu_to_le32(BRCMF_PNO_VERSION_2); /* set extra pno params */ flags = BIT(BRCMF_PNO_IMMEDIATE_SCAN_BIT) | @@ -152,6 +208,12 @@ static int brcmf_pno_config(struct brcmf_if *ifp, u32 scan_freq, return err; } +static int brcmf_pno_config(struct brcmf_if *ifp, u32 scan_freq, u32 mscan, + u32 bestn) +{ + return ifp->drvr->pno_handler.pno_config(ifp, scan_freq, mscan, bestn); +} + static int brcmf_pno_set_random(struct brcmf_if *ifp, struct brcmf_pno_info *pi) { struct brcmf_pub *drvr = ifp->drvr; @@ -275,7 +337,7 @@ static int brcmf_pno_get_bucket_channels(struct cfg80211_sched_scan_request *r, { u32 n_chan = le32_to_cpu(pno_cfg->channel_num); u16 chan; - int i, err = 0; + int i, err; for (i = 0; i < r->n_channels; i++) { if (n_chan >= BRCMF_NUMCHANNELS) { @@ -562,9 +624,82 @@ u64 brcmf_pno_find_reqid_by_bucket(struct brcmf_pno_info *pi, u32 bucket) return reqid; } -u32 brcmf_pno_get_bucket_map(struct brcmf_pno_info *pi, - struct brcmf_pno_net_info_le *ni) + +static struct brcmf_pno_net_info_le * +brcmf_get_netinfo_array(void *pfn_v1_data) +{ + struct brcmf_pno_scanresults_le *pfn_v1 = + (struct brcmf_pno_scanresults_le *)pfn_v1_data; + struct brcmf_pno_scanresults_v2_le *pfn_v2; + struct brcmf_pno_net_info_le *netinfo = NULL; + + switch (pfn_v1->version) { + default: + WARN_ON(1); + fallthrough; + case cpu_to_le32(1): + netinfo = (struct brcmf_pno_net_info_le *)(pfn_v1 + 1); + break; + case cpu_to_le32(2): + pfn_v2 = (struct brcmf_pno_scanresults_v2_le *)pfn_v1; + netinfo = (struct brcmf_pno_net_info_le *)(pfn_v2 + 1); + break; + case cpu_to_le32(3): + brcmf_err("Need to use brcmf_get_netinfo_v3_array\n"); + break; + } + + return netinfo; +} + +static struct brcmf_pno_net_info_v3_le * +brcmf_get_netinfo_v3_array(void*pfn_v3_data) +{ + struct brcmf_pno_scanresults_v3_le *pfn_v3 = + (struct brcmf_pno_scanresults_v3_le *)pfn_v3_data; + return (struct brcmf_pno_net_info_v3_le *) (pfn_v3 + 1); +} + +static u32 brcmf_pno_get_bucket_map(void *data, int idx, struct brcmf_pno_info *pi) +{ + + struct brcmf_pno_net_info_le *netinfo_start = + brcmf_get_netinfo_array(data); + struct brcmf_pno_net_info_le *ni = &netinfo_start[idx]; + struct cfg80211_sched_scan_request *req; + struct cfg80211_match_set *ms; + u32 bucket_map = 0; + int i, j; + + mutex_lock(&pi->req_lock); + for (i = 0; i < pi->n_reqs; i++) { + req = pi->reqs[i]; + + if (!req->n_match_sets) + continue; + for (j = 0; j < req->n_match_sets; j++) { + ms = &req->match_sets[j]; + if (ms->ssid.ssid_len == ni->SSID_len && + !memcmp(ms->ssid.ssid, ni->SSID, ni->SSID_len)) { + bucket_map |= BIT(i); + break; + } + if (is_valid_ether_addr(ms->bssid) && + !memcmp(ms->bssid, ni->bssid, ETH_ALEN)) { + bucket_map |= BIT(i); + break; + } + } + } + mutex_unlock(&pi->req_lock); + return bucket_map; +} + +static u32 brcmf_pno_get_bucket_map_v3(void *data, int idx, struct brcmf_pno_info *pi) { + struct brcmf_pno_net_info_v3_le *netinfo_v3_start = + brcmf_get_netinfo_v3_array(data); + struct brcmf_pno_net_info_v3_le *ni = &netinfo_v3_start[idx]; struct cfg80211_sched_scan_request *req; struct cfg80211_match_set *ms; u32 bucket_map = 0; @@ -593,3 +728,148 @@ u32 brcmf_pno_get_bucket_map(struct brcmf_pno_info *pi, mutex_unlock(&pi->req_lock); return bucket_map; } + +static u32 brcmf_pno_min_data_len(void) +{ + return sizeof(struct brcmf_pno_scanresults_le) + + sizeof(struct brcmf_pno_net_info_le); +} +static u32 brcmf_pno_min_data_len_v3(void) +{ + return sizeof(struct brcmf_pno_scanresults_v3_le) + + sizeof(struct brcmf_pno_net_info_v3_le); +} + +static int brcmf_pno_validate_pfn_results_v3(void *data, u32 eventlen) +{ + struct brcmf_pno_scanresults_v3_le *scanresult = + (struct brcmf_pno_scanresults_v3_le *)data; + struct brcmf_pno_net_info_v3_le *netinfo_v3_start = + brcmf_get_netinfo_v3_array(scanresult); + u32 datalen; + + if (!netinfo_v3_start) { + brcmf_err("did not get netinfo_v3 data\n"); + return -EINVAL; + } + datalen = eventlen - ((void *)netinfo_v3_start - (void *)data); + if (datalen < le32_to_cpu(scanresult->count) * sizeof(struct brcmf_pno_net_info_v3_le)) { + brcmf_err("insufficient event data\n"); + return -EINVAL; + } + return 0; +} + +static int brcmf_pno_validate_pfn_results(void *data, u32 eventlen) +{ + struct brcmf_pno_scanresults_le *scanresult = + (struct brcmf_pno_scanresults_le *)data; + struct brcmf_pno_net_info_le *netinfo_start = + brcmf_get_netinfo_array(scanresult); + u32 datalen; + + if (!netinfo_start) { + brcmf_err("did not get netinfo data\n"); + return -EINVAL; + } + datalen = eventlen - ((void *)netinfo_start - (void *)data); + if (datalen < le32_to_cpu(scanresult->count) * sizeof(struct brcmf_pno_net_info_le)) { + brcmf_err("insufficient event data\n"); + return -EINVAL; + } + return 0; +} + +static int brcmf_pno_get_result_info(void *data, int result_idx, + u8 (*ssid)[IEEE80211_MAX_SSID_LEN], + u8 *ssid_len, u8 *channel, + enum nl80211_band *band) +{ + struct brcmf_pno_scanresults_le *scanresult = + (struct brcmf_pno_scanresults_le *)data; + struct brcmf_pno_net_info_le *netinfo_start = + brcmf_get_netinfo_array(scanresult); + struct brcmf_pno_net_info_le *netinfo = &netinfo_start[result_idx]; + + *channel = netinfo->channel; + *band = netinfo->channel <= CH_MAX_2G_CHANNEL ? NL80211_BAND_2GHZ : + NL80211_BAND_5GHZ; + *ssid_len = netinfo->SSID_len; + if (netinfo->SSID_len > IEEE80211_MAX_SSID_LEN) + *ssid_len = IEEE80211_MAX_SSID_LEN; + memcpy(ssid, netinfo->SSID, *ssid_len); + + return 0; +} + +static int brcmf_pno_get_result_info_v3(void *data, int result_idx, + u8 (*ssid)[IEEE80211_MAX_SSID_LEN], + u8 *ssid_len, u8 *channel, + enum nl80211_band *band) +{ + struct brcmf_pno_scanresults_v3_le *scanresult = + (struct brcmf_pno_scanresults_v3_le *)data; + struct brcmf_pno_net_info_v3_le *netinfo_v3_start = + brcmf_get_netinfo_v3_array(scanresult); + struct brcmf_pno_net_info_v3_le *netinfo_v3 = + &netinfo_v3_start[result_idx]; + + *channel = CHSPEC_CHANNEL(netinfo_v3->chanspec); + *band = fwil_band_to_nl80211(CHSPEC_BAND(netinfo_v3->chanspec)); + *ssid_len = netinfo_v3->SSID_len; + if (netinfo_v3->SSID_len > IEEE80211_MAX_SSID_LEN) + *ssid_len = IEEE80211_MAX_SSID_LEN; + memcpy(ssid, netinfo_v3->SSID, *ssid_len); + + return 0; +} + +/* The count and status fields are in the same place for v1/2/3 */ +static u32 brcmf_pno_get_result_count_v123(void *data) +{ + struct brcmf_pno_scanresults_le *results = + (struct brcmf_pno_scanresults_le *)data; + return le32_to_cpu(results->count); +} +static u32 brcmf_pno_get_result_status_v123(void *data) +{ + struct brcmf_pno_scanresults_le *results = + (struct brcmf_pno_scanresults_le *)data; + return le32_to_cpu(results->status); +} + +int brcmf_pno_setup_for_version(struct brcmf_pub *drvr, u8 vers) +{ + /* The first supported version by this driver was version 2. + * The v2 functions handle version one structures if handed to them, + * but the config was always set to interface version 2. */ + switch (vers) { + case BRCMF_PNO_VERSION_2: { + drvr->pno_handler.version = BRCMF_PNO_VERSION_2; + drvr->pno_handler.pno_config = brcmf_pno_config_v2; + drvr->pno_handler.get_result_count = brcmf_pno_get_result_count_v123; + drvr->pno_handler.get_result_status = brcmf_pno_get_result_status_v123; + drvr->pno_handler.get_bucket_map = brcmf_pno_get_bucket_map; + drvr->pno_handler.get_min_data_len = brcmf_pno_min_data_len; + drvr->pno_handler.get_result_info = brcmf_pno_get_result_info; + drvr->pno_handler.validate_pfn_results = + brcmf_pno_validate_pfn_results; + break; + } + case BRCMF_PNO_VERSION_3: { + drvr->pno_handler.version = BRCMF_PNO_VERSION_3; + drvr->pno_handler.pno_config = brcmf_pno_config_v3; + drvr->pno_handler.get_result_count = brcmf_pno_get_result_count_v123; + drvr->pno_handler.get_result_status = brcmf_pno_get_result_status_v123; + drvr->pno_handler.get_bucket_map = brcmf_pno_get_bucket_map_v3; + drvr->pno_handler.get_min_data_len = brcmf_pno_min_data_len_v3; + drvr->pno_handler.get_result_info = brcmf_pno_get_result_info_v3; + drvr->pno_handler.validate_pfn_results = + brcmf_pno_validate_pfn_results_v3; + break; + } + default: + return -EINVAL; + } + return 0; +} diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.h index 25d406019ac340..0163c762f5385a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.h @@ -61,12 +61,12 @@ void brcmf_pno_detach(struct brcmf_cfg80211_info *cfg); u64 brcmf_pno_find_reqid_by_bucket(struct brcmf_pno_info *pi, u32 bucket); /** - * brcmf_pno_get_bucket_map - determine bucket map for given netinfo. + * brcmf_pno_setup_for_version - setup our PNO handler for whatever version structures + * are supported by the chip * - * @pi: pno instance used. - * @netinfo: netinfo to compare with bucket configuration. + * @cfg: CFG to fill in. + * @vers: Version to use */ -u32 brcmf_pno_get_bucket_map(struct brcmf_pno_info *pi, - struct brcmf_pno_net_info_le *netinfo); +int brcmf_pno_setup_for_version(struct brcmf_pub *drvr, u8 vers); #endif /* _BRCMF_PNO_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/ratespec.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/ratespec.h new file mode 100644 index 00000000000000..37e722daab14d4 --- /dev/null +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/ratespec.h @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (c) 2023 Daniel Berlin + */ + +#ifndef BRCMFMAC_RATESPEC_H +#define BRCMFMAC_RATESPEC_H +/* Rate spec. definitions */ +/* for BRCMF_RSPEC_ENCODING field >= BRCMF_RSPEC_ENCODING_HE, backward compatible */ + +/**< Legacy rate or MCS or MCS + NSS */ +#define BRCMF_RSPEC_RATE_MASK 0x000000FFu +/**< Tx chain expansion beyond Nsts */ +#define BRCMF_RSPEC_TXEXP_MASK 0x00000300u +#define BRCMF_RSPEC_TXEXP_SHIFT 8u +/* EHT GI indices */ +#define BRCMF_RSPEC_EHT_GI_MASK 0x00000C00u +#define BRCMF_RSPEC_EHT_GI_SHIFT 10u +/* HE GI indices */ +#define BRCMF_RSPEC_HE_GI_MASK 0x00000C00u +#define BRCMF_RSPEC_HE_GI_SHIFT 10u +/**< Range extension mask */ +#define BRCMF_RSPEC_ER_MASK 0x0000C000u +#define BRCMF_RSPEC_ER_SHIFT 14u +/**< Range extension tone config */ +#define BRCMF_RSPEC_ER_TONE_MASK 0x00004000u +#define BRCMF_RSPEC_ER_TONE_SHIFT 14u +/**< Range extension enable */ +#define BRCMF_RSPEC_ER_ENAB_MASK 0x00008000u +#define BRCMF_RSPEC_ER_ENAB_SHIFT 15u +/**< Bandwidth */ +#define BRCMF_RSPEC_BW_MASK 0x00070000u +#define BRCMF_RSPEC_BW_SHIFT 16u +/**< Dual Carrier Modulation */ +#define BRCMF_RSPEC_DCM 0x00080000u +#define BRCMF_RSPEC_DCM_SHIFT 19u +/**< STBC expansion, Nsts = 2 * Nss */ +#define BRCMF_RSPEC_STBC 0x00100000u +#define BRCMF_RSPEC_TXBF 0x00200000u +#define BRCMF_RSPEC_LDPC 0x00400000u +/* HT/VHT SGI indication */ +#define BRCMF_RSPEC_SGI 0x00800000u +/**< DSSS short preable - Encoding 0 */ +#define BRCMF_RSPEC_SHORT_PREAMBLE 0x00800000u +/**< Encoding of RSPEC_RATE field */ +#define BRCMF_RSPEC_ENCODING_MASK 0x07000000u +#define BRCMF_RSPEC_ENCODING_SHIFT 24u +#define BRCMF_RSPEC_OVERRIDE_RATE 0x40000000u /**< override rate only */ +#define BRCMF_RSPEC_OVERRIDE_MODE 0x80000000u /**< override both rate & mode */ + +/* ======== RSPEC_EHT_GI|RSPEC_SGI fields for EHT ======== */ +/* 11be Draft 0.4 Table 36-35:Common field for non-OFDMA transmission. + * Table 36-32 Common field for OFDMA transmission + */ +#define BRCMF_RSPEC_EHT_LTF_GI(rspec) \ + (((rspec) & BRCMF_RSPEC_EHT_GI_MASK) >> BRCMF_RSPEC_EHT_GI_SHIFT) +#define BRCMF_RSPEC_EHT_2x_LTF_GI_0_8us (0x0u) +#define BRCMF_RSPEC_EHT_2x_LTF_GI_1_6us (0x1u) +#define BRCMF_RSPEC_EHT_4x_LTF_GI_0_8us (0x2u) +#define BRCMF_RSPEC_EHT_4x_LTF_GI_3_2us (0x3u) +#define WL_EHT_GI_TO_RSPEC(gi) \ + ((u32)(((gi) << BRCMF_RSPEC_EHT_GI_SHIFT) & \ + BRCMF_RSPEC_EHT_GI_MASK)) +#define WL_EHT_GI_TO_RSPEC_SET(rspec, gi) \ + ((rspec & (~BRCMF_RSPEC_EHT_GI_MASK)) | WL_EHT_GI_TO_RSPEC(gi)) + +/* Macros for EHT LTF and GI */ +#define EHT_IS_2X_LTF(gi) \ + (((gi) == BRCMF_RSPEC_EHT_2x_LTF_GI_0_8us) || \ + ((gi) == BRCMF_RSPEC_EHT_2x_LTF_GI_1_6us)) +#define EHT_IS_4X_LTF(gi) \ + (((gi) == BRCMF_RSPEC_EHT_4x_LTF_GI_0_8us) || \ + ((gi) == BRCMF_RSPEC_EHT_4x_LTF_GI_3_2us)) + +#define EHT_IS_GI_0_8us(gi) \ + (((gi) == BRCMF_RSPEC_EHT_2x_LTF_GI_0_8us) || \ + ((gi) == BRCMF_RSPEC_EHT_4x_LTF_GI_0_8us)) +#define EHT_IS_GI_1_6us(gi) ((gi) == BRCMF_RSPEC_EHT_2x_LTF_GI_1_6us) +#define EHT_IS_GI_3_2us(gi) ((gi) == BRCMF_RSPEC_EHT_4x_LTF_GI_3_2us) + +/* ======== RSPEC_HE_GI|RSPEC_SGI fields for HE ======== */ + +/* GI for HE */ +#define BRCMF_RSPEC_HE_LTF_GI(rspec) \ + (((rspec) & BRCMF_RSPEC_HE_GI_MASK) >> BRCMF_RSPEC_HE_GI_SHIFT) +#define BRCMF_RSPEC_HE_1x_LTF_GI_0_8us (0x0u) +#define BRCMF_RSPEC_HE_2x_LTF_GI_0_8us (0x1u) +#define BRCMF_RSPEC_HE_2x_LTF_GI_1_6us (0x2u) +#define BRCMF_RSPEC_HE_4x_LTF_GI_3_2us (0x3u) +#define BRCMF_RSPEC_ISHEGI(rspec) \ + (RSPEC_HE_LTF_GI(rspec) > BRCMF_RSPEC_HE_1x_LTF_GI_0_8us) +#define HE_GI_TO_RSPEC(gi) \ + (((u32)(gi) << BRCMF_RSPEC_HE_GI_SHIFT) & BRCMF_RSPEC_HE_GI_MASK) +#define HE_GI_TO_RSPEC_SET(rspec, gi) \ + ((rspec & (~BRCMF_RSPEC_HE_GI_MASK)) | HE_GI_TO_RSPEC(gi)) + +/* Macros for HE LTF and GI */ +#define HE_IS_1X_LTF(gi) ((gi) == BRCMF_RSPEC_HE_1x_LTF_GI_0_8us) +#define HE_IS_2X_LTF(gi) \ + (((gi) == BRCMF_RSPEC_HE_2x_LTF_GI_0_8us) || \ + ((gi) == BRCMF_RSPEC_HE_2x_LTF_GI_1_6us)) +#define HE_IS_4X_LTF(gi) ((gi) == BRCMF_RSPEC_HE_4x_LTF_GI_3_2us) + +#define HE_IS_GI_0_8us(gi) \ + (((gi) == BRCMF_RSPEC_HE_1x_LTF_GI_0_8us) || \ + ((gi) == BRCMF_RSPEC_HE_2x_LTF_GI_0_8us)) +#define HE_IS_GI_1_6us(gi) ((gi) == BRCMF_RSPEC_HE_2x_LTF_GI_1_6us) +#define HE_IS_GI_3_2us(gi) ((gi) == BRCMF_RSPEC_HE_4x_LTF_GI_3_2us) + +/* RSPEC Macros for extracting and using HE-ER and DCM */ +#define BRCMF_RSPEC_HE_DCM(rspec) \ + (((rspec) & BRCMF_RSPEC_DCM) >> BRCMF_RSPEC_DCM_SHIFT) +#define BRCMF_RSPEC_HE_ER(rspec) \ + (((rspec) & BRCMF_RSPEC_ER_MASK) >> BRCMF_RSPEC_ER_SHIFT) +#define BRCMF_RSPEC_HE_ER_ENAB(rspec) \ + (((rspec) & BRCMF_RSPEC_ER_ENAB_MASK) >> BRCMF_RSPEC_ER_ENAB_SHIFT) +#define BRCMF_RSPEC_HE_ER_TONE(rspec) \ + (((rspec) & BRCMF_RSPEC_ER_TONE_MASK) >> BRCMF_RSPEC_ER_TONE_SHIFT) +/* ======== RSPEC_RATE field ======== */ + +/* Encoding 0 - legacy rate */ +/* DSSS, CCK, and OFDM rates in [500kbps] units */ +#define BRCMF_RSPEC_LEGACY_RATE_MASK 0x0000007F +#define WLC_RATE_1M 2 +#define WLC_RATE_2M 4 +#define WLC_RATE_5M5 11 +#define WLC_RATE_11M 22 +#define WLC_RATE_6M 12 +#define WLC_RATE_9M 18 +#define WLC_RATE_12M 24 +#define WLC_RATE_18M 36 +#define WLC_RATE_24M 48 +#define WLC_RATE_36M 72 +#define WLC_RATE_48M 96 +#define WLC_RATE_54M 108 + +/* Encoding 1 - HT MCS */ +/**< HT MCS value mask in rspec */ +#define BRCMF_RSPEC_HT_MCS_MASK 0x0000007F + +/* Encoding >= 2 */ +/* NSS & MCS values mask in rspec */ +#define BRCMF_RSPEC_NSS_MCS_MASK 0x000000FF +/* mimo MCS value mask in rspec */ +#define BRCMF_RSPEC_MCS_MASK 0x0000000F +/* mimo NSS value mask in rspec */ +#define BRCMF_RSPEC_NSS_MASK 0x000000F0 +/* mimo NSS value shift in rspec */ +#define BRCMF_RSPEC_NSS_SHIFT 4 + +/* Encoding 2 - VHT MCS + NSS */ +/**< VHT MCS value mask in rspec */ +#define BRCMF_RSPEC_VHT_MCS_MASK BRCMF_RSPEC_MCS_MASK +/**< VHT Nss value mask in rspec */ +#define BRCMF_RSPEC_VHT_NSS_MASK BRCMF_RSPEC_NSS_MASK +/**< VHT Nss value shift in rspec */ +#define BRCMF_RSPEC_VHT_NSS_SHIFT BRCMF_RSPEC_NSS_SHIFT + +/* Encoding 3 - HE MCS + NSS */ +/**< HE MCS value mask in rspec */ +#define BRCMF_RSPEC_HE_MCS_MASK BRCMF_RSPEC_MCS_MASK +/**< HE Nss value mask in rspec */ +#define BRCMF_RSPEC_HE_NSS_MASK BRCMF_RSPEC_NSS_MASK +/**< HE Nss value shift in rpsec */ +#define BRCMF_RSPEC_HE_NSS_SHIFT BRCMF_RSPEC_NSS_SHIFT + +#define BRCMF_RSPEC_HE_NSS_UNSPECIFIED 0xf + +/* Encoding 4 - EHT MCS + NSS */ +/**< EHT MCS value mask in rspec */ +#define BRCMF_RSPEC_EHT_MCS_MASK BRCMF_RSPEC_MCS_MASK +/**< EHT Nss value mask in rspec */ +#define BRCMF_RSPEC_EHT_NSS_MASK BRCMF_RSPEC_NSS_MASK +/**< EHT Nss value shift in rpsec */ +#define BRCMF_RSPEC_EHT_NSS_SHIFT BRCMF_RSPEC_NSS_SHIFT + +/* ======== RSPEC_BW field ======== */ + +#define BRCMF_RSPEC_BW_UNSPECIFIED 0u +#define BRCMF_RSPEC_BW_20MHZ 0x00010000u +#define BRCMF_RSPEC_BW_40MHZ 0x00020000u +#define BRCMF_RSPEC_BW_80MHZ 0x00030000u +#define BRCMF_RSPEC_BW_160MHZ 0x00040000u +#define BRCMF_RSPEC_BW_320MHZ 0x00060000u + +/* ======== RSPEC_ENCODING field ======== */ + +/* NOTE: Assuming the rate field is always NSS+MCS starting from VHT encoding! + * Modify/fix RSPEC_ISNSSMCS() macro if above condition changes any time. + */ +/**< Legacy rate is stored in RSPEC_RATE */ +#define BRCMF_RSPEC_ENCODE_RATE 0x00000000u +/**< HT MCS is stored in RSPEC_RATE */ +#define BRCMF_RSPEC_ENCODE_HT 0x01000000u +/**< VHT MCS and NSS are stored in RSPEC_RATE */ +#define BRCMF_RSPEC_ENCODE_VHT 0x02000000u +/**< HE MCS and NSS are stored in RSPEC_RATE */ +#define BRCMF_RSPEC_ENCODE_HE 0x03000000u +/**< EHT MCS and NSS are stored in RSPEC_RATE */ +#define BRCMF_RSPEC_ENCODE_EHT 0x04000000u + +/** + * =============================== + * Handy macros to parse rate spec + * =============================== + */ +#define BRCMF_RSPEC_BW(rspec) ((rspec) & BRCMF_RSPEC_BW_MASK) +#define BRCMF_RSPEC_IS20MHZ(rspec) (RSPEC_BW(rspec) == BRCMF_RSPEC_BW_20MHZ) +#define BRCMF_RSPEC_IS40MHZ(rspec) (RSPEC_BW(rspec) == BRCMF_RSPEC_BW_40MHZ) +#define BRCMF_RSPEC_IS80MHZ(rspec) (RSPEC_BW(rspec) == BRCMF_RSPEC_BW_80MHZ) +#define BRCMF_RSPEC_IS160MHZ(rspec) (RSPEC_BW(rspec) == BRCMF_RSPEC_BW_160MHZ) +#if defined(WL_BW320MHZ) +#define BRCMF_RSPEC_IS320MHZ(rspec) (RSPEC_BW(rspec) == BRCMF_RSPEC_BW_320MHZ) +#else +#define BRCMF_RSPEC_IS320MHZ(rspec) (FALSE) +#endif /* WL_BW320MHZ */ + +#define BRCMF_RSPEC_BW_GE(rspec, rspec_bw) (RSPEC_BW(rspec) >= rspec_bw) +#define BRCMF_RSPEC_BW_LE(rspec, rspec_bw) (RSPEC_BW(rspec) <= rspec_bw) +#define BRCMF_RSPEC_BW_GT(rspec, rspec_bw) (!RSPEC_BW_LE(rspec, rspec_bw)) +#define BRCMF_RSPEC_BW_LT(rspec, rspec_bw) (!RSPEC_BW_GE(rspec, rspec_bw)) + +#define BRCMF_RSPEC_ISSGI(rspec) (((rspec) & BRCMF_RSPEC_SGI) != 0) +#define BRCMF_RSPEC_ISLDPC(rspec) (((rspec) & BRCMF_RSPEC_LDPC) != 0) +#define BRCMF_RSPEC_ISSTBC(rspec) (((rspec) & BRCMF_RSPEC_STBC) != 0) +#define BRCMF_RSPEC_ISTXBF(rspec) (((rspec) & BRCMF_RSPEC_TXBF) != 0) + +#define BRCMF_RSPEC_TXEXP(rspec) \ + (((rspec) & BRCMF_RSPEC_TXEXP_MASK) >> BRCMF_RSPEC_TXEXP_SHIFT) + +#define BRCMF_RSPEC_ENCODE(rspec) \ + (((rspec) & BRCMF_RSPEC_ENCODING_MASK) >> BRCMF_RSPEC_ENCODING_SHIFT) +#define BRCMF_RSPEC_ISLEGACY(rspec) \ + (((rspec) & BRCMF_RSPEC_ENCODING_MASK) == BRCMF_RSPEC_ENCODE_RATE) + +#define BRCMF_RSPEC_ISHT(rspec) \ + (((rspec) & BRCMF_RSPEC_ENCODING_MASK) == BRCMF_RSPEC_ENCODE_HT) +#define BRCMF_RSPEC_ISVHT(rspec) \ + (((rspec) & BRCMF_RSPEC_ENCODING_MASK) == BRCMF_RSPEC_ENCODE_VHT) +#define BRCMF_RSPEC_ISHE(rspec) \ + (((rspec) & BRCMF_RSPEC_ENCODING_MASK) == BRCMF_RSPEC_ENCODE_HE) +#define BRCMF_RSPEC_ISEHT(rspec) \ + (((rspec) & BRCMF_RSPEC_ENCODING_MASK) == BRCMF_RSPEC_ENCODE_EHT) + +/* fast check if rate field is NSS+MCS format (starting from VHT ratespec) */ +#define BRCMF_RSPEC_ISVHTEXT(rspec) \ + (((rspec) & BRCMF_RSPEC_ENCODING_MASK) >= BRCMF_RSPEC_ENCODE_VHT) +/* fast check if rate field is NSS+MCS format (starting from HE ratespec) */ +#define BRCMF_RSPEC_ISHEEXT(rspec) \ + (((rspec) & BRCMF_RSPEC_ENCODING_MASK) >= BRCMF_RSPEC_ENCODE_HE) + +#endif /* BRCMFMAC_RATESPEC_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/scan_param.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/scan_param.c new file mode 100644 index 00000000000000..4f634509d25256 --- /dev/null +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/scan_param.c @@ -0,0 +1,446 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (c) 2023 Daniel Berlin + */ +#include +#include + +#include "core.h" +#include "debug.h" +#include "fwil_types.h" +#include "cfg80211.h" +#include "scan_param.h" + +static void brcmf_scan_param_set_defaults(u8 (*bssid)[ETH_ALEN], s8 *bss_type, __le32 *channel_num, + __le32 *nprobes, __le32 *active_time, + __le32 *passive_time, + __le32 *home_time) +{ + eth_broadcast_addr(*bssid); + *bss_type = DOT11_BSSTYPE_ANY; + *channel_num = 0; + *nprobes = cpu_to_le32(-1); + *active_time = cpu_to_le32(-1); + *passive_time = cpu_to_le32(-1); + *home_time = cpu_to_le32(-1); +} + +static void brcmf_scan_param_copy_chanspecs( + struct brcmf_cfg80211_info *cfg, __le16 (*dest_channels)[], + struct ieee80211_channel **in_channels, u32 n_channels) +{ + int i; + for (i = 0; i < n_channels; i++) { + u32 chanspec = + channel_to_chanspec(&cfg->d11inf, in_channels[i]); + brcmf_dbg(SCAN, "Chan : %d, Channel spec: %x\n", + in_channels[i]->hw_value, chanspec); + (*dest_channels)[i] = cpu_to_le16(chanspec); + } +} + +static void brcmf_scan_param_copy_ssids(char *dest_ssids, + struct cfg80211_ssid *in_ssids, + u32 n_ssids) +{ + int i; + for (i = 0; i < n_ssids; i++) { + struct brcmf_ssid_le ssid_le; + memset(&ssid_le, 0, sizeof(ssid_le)); + ssid_le.SSID_len = cpu_to_le32(in_ssids[i].ssid_len); + memcpy(ssid_le.SSID, in_ssids[i].ssid, in_ssids[i].ssid_len); + if (!ssid_le.SSID_len) + brcmf_dbg(SCAN, "%d: Broadcast scan\n", i); + else + brcmf_dbg(SCAN, "%d: scan for %.32s size=%d\n", i, + ssid_le.SSID, ssid_le.SSID_len); + memcpy(dest_ssids, &ssid_le, sizeof(ssid_le)); + dest_ssids += sizeof(ssid_le); + } +} + +/* The scan parameter structures have an array of SSID's that appears at the end in some cases. + * In these cases, the chan list is really the lower half of a pair, the upper half is a ssid number, + * and then after all of that there is an array of SSIDs */ +static u32 +brcmf_scan_param_tail_size(const struct cfg80211_scan_request *request, + u32 params_size) +{ + if (request != NULL) { + /* Allocate space for populating ssid upper half in struct */ + params_size += sizeof(u32) * ((request->n_channels + 1) / 2); + /* Allocate space for populating ssids in struct */ + params_size += sizeof(struct brcmf_ssid_le) * request->n_ssids; + } else { + params_size += sizeof(u16); + } + return params_size; +} + +static u32 brcmf_nl80211_scan_flags_to_scan_flags(u32 nl80211_flags) +{ + u32 scan_flags = 0; + if (nl80211_flags & NL80211_SCAN_FLAG_LOW_SPAN) { + scan_flags |= BRCMF_SCANFLAGS_LOW_SPAN; + brcmf_dbg(SCAN, "requested low span scan\n"); + } + if (nl80211_flags & NL80211_SCAN_FLAG_HIGH_ACCURACY) { + scan_flags |= BRCMF_SCANFLAGS_HIGH_ACCURACY; + brcmf_dbg(SCAN, "requested high accuracy scan\n"); + } + if (nl80211_flags & NL80211_SCAN_FLAG_LOW_POWER) { + scan_flags |= BRCMF_SCANFLAGS_LOW_POWER; + brcmf_dbg(SCAN, "requested low power scan\n"); + } + if (nl80211_flags & NL80211_SCAN_FLAG_LOW_PRIORITY) { + scan_flags |= BRCMF_SCANFLAGS_LOW_PRIO; + brcmf_dbg(SCAN, "requested low priority scan\n"); + } + return scan_flags; +} + +static void * +brcmf_scan_param_get_prepped_struct_v1(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_scan_request *request) +{ + u32 n_ssids; + u32 n_channels; + u32 params_size = sizeof(struct brcmf_scan_params_le); + u32 length; + struct brcmf_scan_params_le *params_le = NULL; + u8 scan_type = BRCMF_SCANTYPE_ACTIVE; + + length = offsetof(struct brcmf_scan_params_le, channel_list); + params_size = brcmf_scan_param_tail_size(request, params_size); + params_le = kzalloc(params_size, GFP_KERNEL); + if (!params_le) { + bphy_err(cfg, "Could not allocate scan params\n"); + return NULL; + } + brcmf_scan_param_set_defaults(¶ms_le->bssid, + ¶ms_le->bss_type, ¶ms_le->channel_num, + ¶ms_le->nprobes, ¶ms_le->active_time, + ¶ms_le->passive_time, ¶ms_le->home_time); + + /* Scan abort */ + if (!request) { + params_le->channel_num = cpu_to_le32(1); + params_le->channel_list[0] = cpu_to_le16(-1); + goto done; + } + + n_ssids = request->n_ssids; + n_channels = request->n_channels; + + /* Copy channel array if applicable */ + brcmf_dbg(SCAN, "### List of channelspecs to scan ### %d\n", + n_channels); + if (n_channels > 0) { + length += roundup(sizeof(u16) * n_channels, sizeof(u32)); + brcmf_scan_param_copy_chanspecs(cfg, ¶ms_le->channel_list, + request->channels, n_channels); + } else { + brcmf_dbg(SCAN, "Scanning all channels\n"); + } + + /* Copy ssid array if applicable */ + brcmf_dbg(SCAN, "### List of SSIDs to scan ### %d\n", n_ssids); + if (n_ssids > 0) { + s32 offset; + char *ptr; + + offset = + offsetof(struct brcmf_scan_params_le, channel_list) + + n_channels * sizeof(u16); + offset = roundup(offset, sizeof(u32)); + length += sizeof(struct brcmf_ssid_le) * n_ssids; + ptr = (char *)params_le + offset; + brcmf_scan_param_copy_ssids(ptr, request->ssids, n_ssids); + } else { + brcmf_dbg(SCAN, "Performing passive scan\n"); + scan_type = BRCMF_SCANTYPE_PASSIVE; + } + scan_type |= brcmf_nl80211_scan_flags_to_scan_flags(request->flags); + params_le->scan_type =scan_type; + /* Adding mask to channel numbers */ + params_le->channel_num = + cpu_to_le32((n_ssids << BRCMF_SCAN_PARAMS_NSSID_SHIFT) | + (n_channels & BRCMF_SCAN_PARAMS_COUNT_MASK)); +done: + *struct_size = length; + return params_le; +} + +static void * +brcmf_scan_param_get_prepped_struct_v2(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_scan_request *request) +{ + u32 n_ssids; + u32 n_channels; + u32 params_size = sizeof(struct brcmf_scan_params_v2_le); + u32 length; + struct brcmf_scan_params_v2_le *params_le = NULL; + u32 scan_type = BRCMF_SCANTYPE_ACTIVE; + + length = offsetof(struct brcmf_scan_params_v2_le, channel_list); + params_size = brcmf_scan_param_tail_size(request, params_size); + params_le = kzalloc(params_size, GFP_KERNEL); + if (!params_le) { + bphy_err(cfg, "Could not allocate scan params\n"); + return NULL; + } + params_le->version = cpu_to_le16(BRCMF_SCAN_PARAMS_VERSION_V2); + brcmf_scan_param_set_defaults(¶ms_le->bssid, + ¶ms_le->bss_type, ¶ms_le->channel_num, + ¶ms_le->nprobes, ¶ms_le->active_time, + ¶ms_le->passive_time, ¶ms_le->home_time); + + /* Scan abort */ + if (!request) { + length += sizeof(u16); + params_le->channel_num = cpu_to_le32(1); + params_le->channel_list[0] = cpu_to_le16(-1); + params_le->length = cpu_to_le16(length); + goto done; + } + + n_ssids = request->n_ssids; + n_channels = request->n_channels; + + /* Copy channel array if applicable */ + brcmf_dbg(SCAN, "### List of channelspecs to scan ### %d\n", + n_channels); + if (n_channels > 0) { + length += roundup(sizeof(u16) * n_channels, sizeof(u32)); + brcmf_scan_param_copy_chanspecs(cfg, ¶ms_le->channel_list, + request->channels, n_channels); + } else { + brcmf_dbg(SCAN, "Scanning all channels\n"); + } + + /* Copy ssid array if applicable */ + brcmf_dbg(SCAN, "### List of SSIDs to scan ### %d\n", n_ssids); + if (n_ssids > 0) { + s32 offset; + char *ptr; + + offset = + offsetof(struct brcmf_scan_params_v2_le, channel_list) + + n_channels * sizeof(u16); + offset = roundup(offset, sizeof(u32)); + length += sizeof(struct brcmf_ssid_le) * n_ssids; + ptr = (char *)params_le + offset; + brcmf_scan_param_copy_ssids(ptr, request->ssids, n_ssids); + + } else { + brcmf_dbg(SCAN, "Performing passive scan\n"); + scan_type = BRCMF_SCANTYPE_PASSIVE; + } + scan_type |= brcmf_nl80211_scan_flags_to_scan_flags(request->flags); + params_le->scan_type = cpu_to_le32(scan_type); + params_le->length = cpu_to_le16(length); + /* Adding mask to channel numbers */ + params_le->channel_num = + cpu_to_le32((n_ssids << BRCMF_SCAN_PARAMS_NSSID_SHIFT) | + (n_channels & BRCMF_SCAN_PARAMS_COUNT_MASK)); +done: + *struct_size = length; + return params_le; +} + +static void * +brcmf_scan_param_get_prepped_struct_v3(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_scan_request *request) +{ + u32 n_ssids; + u32 n_channels; + u32 params_size = sizeof(struct brcmf_scan_params_v3_le); + u32 length; + struct brcmf_scan_params_v3_le *params_le = NULL; + u32 scan_type = BRCMF_SCANTYPE_ACTIVE; + + length = offsetof(struct brcmf_scan_params_v3_le, channel_list); + params_size = brcmf_scan_param_tail_size(request, params_size); + params_le = kzalloc(params_size, GFP_KERNEL); + if (!params_le) { + bphy_err(cfg, "Could not allocate scan params\n"); + return NULL; + } + + params_le->version = cpu_to_le16(BRCMF_SCAN_PARAMS_VERSION_V3); + params_le->ssid_type = 0; + brcmf_scan_param_set_defaults(¶ms_le->bssid, + ¶ms_le->bss_type, ¶ms_le->channel_num, + ¶ms_le->nprobes, ¶ms_le->active_time, + ¶ms_le->passive_time, ¶ms_le->home_time); + + /* Scan abort */ + if (!request) { + length += sizeof(u16); + params_le->channel_num = cpu_to_le32(1); + params_le->channel_list[0] = cpu_to_le16(-1); + params_le->length = cpu_to_le16(length); + goto done; + } + + n_ssids = request->n_ssids; + n_channels = request->n_channels; + + /* Copy channel array if applicable */ + brcmf_dbg(SCAN, "### List of channelspecs to scan ### %d\n", + n_channels); + if (n_channels > 0) { + length += roundup(sizeof(u16) * n_channels, sizeof(u32)); + brcmf_scan_param_copy_chanspecs(cfg, ¶ms_le->channel_list, + request->channels, n_channels); + + } else { + brcmf_dbg(SCAN, "Scanning all channels\n"); + } + + /* Copy ssid array if applicable */ + brcmf_dbg(SCAN, "### List of SSIDs to scan ### %d\n", n_ssids); + if (n_ssids > 0) { + s32 offset; + char *ptr; + + offset = + offsetof(struct brcmf_scan_params_v3_le, channel_list) + + n_channels * sizeof(u16); + offset = roundup(offset, sizeof(u32)); + length += sizeof(struct brcmf_ssid_le) * n_ssids; + ptr = (char *)params_le + offset; + brcmf_scan_param_copy_ssids(ptr, request->ssids, n_ssids); + + } else { + brcmf_dbg(SCAN, "Performing passive scan\n"); + scan_type = BRCMF_SCANTYPE_PASSIVE; + } + scan_type |= brcmf_nl80211_scan_flags_to_scan_flags(request->flags); + params_le->scan_type = cpu_to_le32(scan_type); + params_le->length = cpu_to_le16(length); + params_le->channel_num = + cpu_to_le32((n_ssids << BRCMF_SCAN_PARAMS_NSSID_SHIFT) | + (n_channels & BRCMF_SCAN_PARAMS_COUNT_MASK)); + + /* Include RNR results if requested */ + if (request->flags & NL80211_SCAN_FLAG_COLOCATED_6GHZ) { + params_le->ssid_type |= BRCMF_SCANSSID_INC_RNR; + } + /* Adding mask to channel numbers */ +done: + *struct_size = length; + return params_le; +} + +static void * +brcmf_scan_param_get_prepped_struct_v4(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_scan_request *request) +{ + u32 n_ssids; + u32 n_channels; + u32 params_size = sizeof(struct brcmf_scan_params_v4_le); + u32 length; + struct brcmf_scan_params_v4_le *params_le = NULL; + u32 scan_type = BRCMF_SCANTYPE_ACTIVE; + + length = offsetof(struct brcmf_scan_params_v4_le, channel_list); + params_size = brcmf_scan_param_tail_size(request, params_size); + params_le = kzalloc(params_size, GFP_KERNEL); + if (!params_le) { + bphy_err(cfg, "Could not allocate scan params\n"); + return NULL; + } + params_le->version = cpu_to_le16(BRCMF_SCAN_PARAMS_VERSION_V4); + params_le->ssid_type = 0; + brcmf_scan_param_set_defaults(¶ms_le->bssid, + ¶ms_le->bss_type, ¶ms_le->channel_num, + ¶ms_le->nprobes, ¶ms_le->active_time, + ¶ms_le->passive_time, ¶ms_le->home_time); + + /* Scan abort */ + if (!request) { + length += sizeof(u16); + params_le->channel_num = cpu_to_le32(1); + params_le->channel_list[0] = cpu_to_le16(-1); + params_le->length = cpu_to_le16(length); + goto done; + } + + n_ssids = request->n_ssids; + n_channels = request->n_channels; + + /* Copy channel array if applicable */ + brcmf_dbg(SCAN, "### List of channelspecs to scan ### %d\n", + n_channels); + if (n_channels > 0) { + length += roundup(sizeof(u16) * n_channels, sizeof(u32)); + brcmf_scan_param_copy_chanspecs(cfg, ¶ms_le->channel_list, + request->channels, n_channels); + } else { + brcmf_dbg(SCAN, "Scanning all channels\n"); + } + + /* Copy ssid array if applicable */ + brcmf_dbg(SCAN, "### List of SSIDs to scan ### %d\n", n_ssids); + if (n_ssids > 0) { + s32 offset; + char *ptr; + + offset = + offsetof(struct brcmf_scan_params_v4_le, channel_list) + + n_channels * sizeof(u16); + offset = roundup(offset, sizeof(u32)); + length += sizeof(struct brcmf_ssid_le) * n_ssids; + ptr = (char *)params_le + offset; + brcmf_scan_param_copy_ssids(ptr, request->ssids, n_ssids); + } else { + brcmf_dbg(SCAN, "Performing passive scan\n"); + scan_type = BRCMF_SCANTYPE_PASSIVE; + } + scan_type |= brcmf_nl80211_scan_flags_to_scan_flags(request->flags); + params_le->scan_type = cpu_to_le32(scan_type); + params_le->length = cpu_to_le16(length); + /* Adding mask to channel numbers */ + params_le->channel_num = + cpu_to_le32((n_ssids << BRCMF_SCAN_PARAMS_NSSID_SHIFT) | + (n_channels & BRCMF_SCAN_PARAMS_COUNT_MASK)); + /* Include RNR results if requested */ + if (request->flags & NL80211_SCAN_FLAG_COLOCATED_6GHZ) { + params_le->ssid_type |= BRCMF_SCANSSID_INC_RNR; + } +done: + *struct_size = length; + return params_le; +} + +int brcmf_scan_param_setup_for_version(struct brcmf_pub *drvr, u8 version) +{ + drvr->scan_param_handler.version = version; + switch (version) { + case 1: { + drvr->scan_param_handler.get_struct_for_request = + brcmf_scan_param_get_prepped_struct_v1; + } break; + case 2: { + drvr->scan_param_handler.get_struct_for_request = + brcmf_scan_param_get_prepped_struct_v2; + } break; + case 3: { + drvr->scan_param_handler.get_struct_for_request = + brcmf_scan_param_get_prepped_struct_v3; + } break; + case 4: { + drvr->scan_param_handler.get_struct_for_request = + brcmf_scan_param_get_prepped_struct_v4; + + } break; + default: + return -EINVAL; + } + return 0; +} diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/scan_param.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/scan_param.h new file mode 100644 index 00000000000000..577de083c6e3cd --- /dev/null +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/scan_param.h @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (c) 2023 Daniel Berlin + */ + +#ifndef _BRCMF_SCAN_PARAM_H +#define _BRCMF_SCAN_PARAM_H + +struct brcmf_pub; + +/** + * brcmf_scan_param_setup_for_version() - Setup the driver to handle join structures + * + * There are a number of different structures and interface versions for scanning info + * This sets up the driver to handle a particular interface version. + * + * @drvr Driver structure to setup + * @ver Interface version + * Return: %0 if okay, error code otherwise + */ +int brcmf_scan_param_setup_for_version(struct brcmf_pub *, u8 ver); +#endif /* _BRCMF_SCAN_PARAM_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c b/drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c index 1e2b1e487eb76e..faf7eeeeb2d57e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c @@ -87,10 +87,20 @@ static void brcmu_d11ac_encchspec(struct brcmu_chan *ch) 0, d11ac_bw(ch->bw)); ch->chspec &= ~BRCMU_CHSPEC_D11AC_BND_MASK; - if (ch->chnum <= CH_MAX_2G_CHANNEL) - ch->chspec |= BRCMU_CHSPEC_D11AC_BND_2G; - else + switch (ch->band) { + case BRCMU_CHAN_BAND_6G: + ch->chspec |= BRCMU_CHSPEC_D11AC_BND_6G; + break; + case BRCMU_CHAN_BAND_5G: ch->chspec |= BRCMU_CHSPEC_D11AC_BND_5G; + break; + case BRCMU_CHAN_BAND_2G: + ch->chspec |= BRCMU_CHSPEC_D11AC_BND_2G; + break; + default: + WARN_ONCE(1, "Invalid band 0x%04x\n", ch->band); + break; + } } static void brcmu_d11n_decchspec(struct brcmu_chan *ch) @@ -117,7 +127,9 @@ static void brcmu_d11n_decchspec(struct brcmu_chan *ch) } break; default: - WARN_ONCE(1, "Invalid chanspec 0x%04x\n", ch->chspec); + WARN_ONCE(1, + "Invalid chanspec - unknown 11n bandwidth 0x%04x\n", + ch->chspec); break; } @@ -129,7 +141,8 @@ static void brcmu_d11n_decchspec(struct brcmu_chan *ch) ch->band = BRCMU_CHAN_BAND_2G; break; default: - WARN_ONCE(1, "Invalid chanspec 0x%04x\n", ch->chspec); + WARN_ONCE(1, "Invalid chanspec - unknown 11n band 0x%04x\n", + ch->chspec); break; } } @@ -156,7 +169,9 @@ static void brcmu_d11ac_decchspec(struct brcmu_chan *ch) ch->sb = BRCMU_CHAN_SB_U; ch->control_ch_num += CH_10MHZ_APART; } else { - WARN_ONCE(1, "Invalid chanspec 0x%04x\n", ch->chspec); + WARN_ONCE(1, + "Invalid chanspec - unknown 11ac channel distance 0x%04x\n", + ch->chspec); } break; case BRCMU_CHSPEC_D11AC_BW_80: @@ -177,7 +192,9 @@ static void brcmu_d11ac_decchspec(struct brcmu_chan *ch) ch->control_ch_num += CH_30MHZ_APART; break; default: - WARN_ONCE(1, "Invalid chanspec 0x%04x\n", ch->chspec); + WARN_ONCE(1, + "Invalid chanspec - unknown 11ac channel distance 0x%04x\n", + ch->chspec); break; } break; @@ -211,17 +228,24 @@ static void brcmu_d11ac_decchspec(struct brcmu_chan *ch) ch->control_ch_num += CH_70MHZ_APART; break; default: - WARN_ONCE(1, "Invalid chanspec 0x%04x\n", ch->chspec); + WARN_ONCE(1, + "Invalid chanspec - unknown 11ac channel distance 0x%04x\n", + ch->chspec); break; } break; case BRCMU_CHSPEC_D11AC_BW_8080: default: - WARN_ONCE(1, "Invalid chanspec 0x%04x\n", ch->chspec); + WARN_ONCE(1, + "Invalid chanspec - unknown 11ac channel bandwidth 0x%04x\n", + ch->chspec); break; } switch (ch->chspec & BRCMU_CHSPEC_D11AC_BND_MASK) { + case BRCMU_CHSPEC_D11AC_BND_6G: + ch->band = BRCMU_CHAN_BAND_6G; + break; case BRCMU_CHSPEC_D11AC_BND_5G: ch->band = BRCMU_CHAN_BAND_5G; break; @@ -229,7 +253,9 @@ static void brcmu_d11ac_decchspec(struct brcmu_chan *ch) ch->band = BRCMU_CHAN_BAND_2G; break; default: - WARN_ONCE(1, "Invalid chanspec 0x%04x\n", ch->chspec); + WARN_ONCE(1, + "Invalid chanspec - unknown 11ac channel band 0x%04x\n", + ch->chspec); break; } } diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h index df3b67ba4db290..f749337a06942e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h @@ -57,6 +57,7 @@ #define BRCM_CC_4377_CHIP_ID 0x4377 #define BRCM_CC_4378_CHIP_ID 0x4378 #define BRCM_CC_4387_CHIP_ID 0x4387 +#define BRCM_CC_4388_CHIP_ID 0x4388 #define CY_CC_4373_CHIP_ID 0x4373 #define CY_CC_43012_CHIP_ID 43012 #define CY_CC_43439_CHIP_ID 43439 @@ -99,6 +100,7 @@ #define BRCM_PCIE_4377_DEVICE_ID 0x4488 #define BRCM_PCIE_4378_DEVICE_ID 0x4425 #define BRCM_PCIE_4387_DEVICE_ID 0x4433 +#define BRCM_PCIE_4388_DEVICE_ID 0x4434 #define CY_PCIE_54591_DEVICE_ID 0x4417 /* brcmsmac IDs */ diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_d11.h b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_d11.h index f6344023855c36..bb48b744206223 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_d11.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_d11.h @@ -69,24 +69,44 @@ #define BRCMU_CHSPEC_D11AC_SB_UU BRCMU_CHSPEC_D11AC_SB_LUU #define BRCMU_CHSPEC_D11AC_SB_L BRCMU_CHSPEC_D11AC_SB_LLL #define BRCMU_CHSPEC_D11AC_SB_U BRCMU_CHSPEC_D11AC_SB_LLU +/* channel sideband indication for frequency >= 240MHz */ +#define BRCMU_CHSPEC_D11AC_320_SB_MASK 0x0780 +#define BRCMU_CHSPEC_D11AC_320_SB_SHIFT 7 +#define BRCMU_CHSPEC_D11AC_SB_LLLL 0x0000 +#define BRCMU_CHSPEC_D11AC_SB_LLLU 0x0080 +#define BRCMU_CHSPEC_D11AC_SB_LLUL 0x0100 +#define BRCMU_CHSPEC_D11AC_SB_LLUU 0x0180 +#define BRCMU_CHSPEC_D11AC_SB_LULL 0x0200 +#define BRCMU_CHSPEC_D11AC_SB_LULU 0x0280 +#define BRCMU_CHSPEC_D11AC_SB_LUUL 0x0300 +#define BRCMU_CHSPEC_D11AC_SB_LUUU 0x0380 +#define BRCMU_CHSPEC_D11AC_SB_ULLL 0x0400 +#define BRCMU_CHSPEC_D11AC_SB_ULLU 0x0480 +#define BRCMU_CHSPEC_D11AC_SB_ULUL 0x0500 +#define BRCMU_CHSPEC_D11AC_SB_ULUU 0x0580 +#define BRCMU_CHSPEC_D11AC_SB_UULL 0x0600 +#define BRCMU_CHSPEC_D11AC_SB_UULU 0x0680 +#define BRCMU_CHSPEC_D11AC_SB_UUUL 0x0700 +#define BRCMU_CHSPEC_D11AC_SB_UUUU 0x0780 #define BRCMU_CHSPEC_D11AC_BW_MASK 0x3800 #define BRCMU_CHSPEC_D11AC_BW_SHIFT 11 -#define BRCMU_CHSPEC_D11AC_BW_5 0x0000 -#define BRCMU_CHSPEC_D11AC_BW_10 0x0800 -#define BRCMU_CHSPEC_D11AC_BW_20 0x1000 -#define BRCMU_CHSPEC_D11AC_BW_40 0x1800 -#define BRCMU_CHSPEC_D11AC_BW_80 0x2000 -#define BRCMU_CHSPEC_D11AC_BW_160 0x2800 -#define BRCMU_CHSPEC_D11AC_BW_8080 0x3000 -#define BRCMU_CHSPEC_D11AC_BND_MASK 0xc000 -#define BRCMU_CHSPEC_D11AC_BND_SHIFT 14 -#define BRCMU_CHSPEC_D11AC_BND_2G 0x0000 -#define BRCMU_CHSPEC_D11AC_BND_3G 0x4000 -#define BRCMU_CHSPEC_D11AC_BND_4G 0x8000 -#define BRCMU_CHSPEC_D11AC_BND_5G 0xc000 +#define BRCMU_CHSPEC_D11AC_BW_10 0x0800 +#define BRCMU_CHSPEC_D11AC_BW_20 0x1000 +#define BRCMU_CHSPEC_D11AC_BW_40 0x1800 +#define BRCMU_CHSPEC_D11AC_BW_80 0x2000 +#define BRCMU_CHSPEC_D11AC_BW_160 0x2800 +#define BRCMU_CHSPEC_D11AC_BW_320 0x0000 +#define BRCMU_CHSPEC_D11AC_BW_8080 0x3000 +#define BRCMU_CHSPEC_D11AC_BND_MASK 0xc000 +#define BRCMU_CHSPEC_D11AC_BND_SHIFT 14 +#define BRCMU_CHSPEC_D11AC_BND_2G 0x0000 +#define BRCMU_CHSPEC_D11AC_BND_4G 0x8000 +#define BRCMU_CHSPEC_D11AC_BND_5G 0xc000 +#define BRCMU_CHSPEC_D11AC_BND_6G 0x4000 #define BRCMU_CHAN_BAND_2G 0 #define BRCMU_CHAN_BAND_5G 1 +#define BRCMU_CHAN_BAND_6G 2 enum brcmu_chan_bw { BRCMU_CHAN_BW_20, diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h index 7552bdb91991ce..ef042beeb586f9 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h @@ -31,6 +31,7 @@ /* bandstate array indices */ #define BAND_2G_INDEX 0 /* wlc->bandstate[x] index */ #define BAND_5G_INDEX 1 /* wlc->bandstate[x] index */ +#define BAND_6G_INDEX 2 /* wlc->bandstate[x] index */ /* * max # supported channels. The max channel no is 216, this is that + 1 @@ -48,17 +49,22 @@ #define WL_CHANSPEC_CTL_SB_UPPER 0x0200 #define WL_CHANSPEC_CTL_SB_NONE 0x0300 -#define WL_CHANSPEC_BW_MASK 0x0C00 -#define WL_CHANSPEC_BW_SHIFT 10 +#define WL_CHANSPEC_BW_MASK 0x3800 +#define WL_CHANSPEC_BW_SHIFT 11 #define WL_CHANSPEC_BW_10 0x0400 #define WL_CHANSPEC_BW_20 0x0800 #define WL_CHANSPEC_BW_40 0x0C00 #define WL_CHANSPEC_BW_80 0x2000 - -#define WL_CHANSPEC_BAND_MASK 0xf000 -#define WL_CHANSPEC_BAND_SHIFT 12 -#define WL_CHANSPEC_BAND_5G 0x1000 -#define WL_CHANSPEC_BAND_2G 0x2000 +#define WL_CHANSPEC_BW_160 0x2800 +#define WL_CHANSPEC_BW_8080 0x3000 +#define WL_CHANSPEC_BW_320 0x0000 + +#define WL_CHANSPEC_BAND_MASK 0xc000 +#define WL_CHANSPEC_BAND_SHIFT 14 +#define WL_CHANSPEC_BAND_2G 0x0000 +#define WL_CHANSPEC_BAND_4G 0x8000 +#define WL_CHANSPEC_BAND_5G 0xc000 +#define WL_CHANSPEC_BAND_6G 0x4000 #define INVCHANSPEC 255 #define WL_CHAN_VALID_HW (1 << 0) /* valid with current HW */ @@ -93,6 +99,9 @@ #define WLC_BAND_5G 1 /* 5 Ghz */ #define WLC_BAND_2G 2 /* 2.4 Ghz */ #define WLC_BAND_ALL 3 /* all bands */ +#define WLC_BAND_6G 4 /* 6 Ghz */ + +#define WLC_AP_IOV_OP_MANUAL_AP_BSSCFG_CREATE 2 #define CHSPEC_CHANNEL(chspec) ((u8)((chspec) & WL_CHANSPEC_CHAN_MASK)) #define CHSPEC_BAND(chspec) ((chspec) & WL_CHANSPEC_BAND_MASK) @@ -112,6 +121,12 @@ #define CHSPEC_IS80(chspec) \ (((chspec) & WL_CHANSPEC_BW_MASK) == WL_CHANSPEC_BW_80) +#define CHSPEC_IS160(chspec) \ + (((chspec) & WL_CHANSPEC_BW_MASK) == WL_CHANSPEC_BW_160) + +#define CHSPEC_IS6G(chspec) \ + (((chspec) & WL_CHANSPEC_BAND_MASK) == WL_CHANSPEC_BAND_6G) + #define CHSPEC_IS5G(chspec) \ (((chspec) & WL_CHANSPEC_BAND_MASK) == WL_CHANSPEC_BAND_5G) @@ -200,6 +215,13 @@ static inline bool ac_bitmap_tst(u8 bitmap, int prec) #define CRYPTO_ALGO_AES_RESERVED1 5 #define CRYPTO_ALGO_AES_RESERVED2 6 #define CRYPTO_ALGO_NALG 7 +#define CRYPTO_ALGO_AES_GCM 14 /* 128 bit GCM */ +#define CRYPTO_ALGO_AES_CCM256 15 /* 256 bit CCM */ +#define CRYPTO_ALGO_AES_GCM256 16 /* 256 bit GCM */ +#define CRYPTO_ALGO_BIP_CMAC256 17 /* 256 bit BIP CMAC */ +#define CRYPTO_ALGO_BIP_GMAC 18 /* 128 bit BIP GMAC */ +#define CRYPTO_ALGO_BIP_GMAC256 19 /* 256 bit BIP GMAC */ + /* wireless security bitvec */ @@ -232,6 +254,13 @@ static inline bool ac_bitmap_tst(u8 bitmap, int prec) #define WPA2_AUTH_PSK_SHA256 0x8000 /* PSK with SHA256 key derivation */ #define WPA3_AUTH_SAE_PSK 0x40000 /* SAE with 4-way handshake */ +#define WPA3_AUTH_OWE 0x100000 /* OWE */ +#define WFA_AUTH_DPP 0x200000 /* WFA DPP AUTH */ +#define WPA3_AUTH_1X_SUITE_B_SHA384 0x400000 /* Suite B-192 SHA384 */ + + +#define WFA_OUI "\x50\x6F\x9A" /* WFA OUI */ +#define DPP_VER 0x1A /* WFA DPP v1.0 */ #define DOT11_DEFAULT_RTS_LEN 2347 #define DOT11_DEFAULT_FRAG_LEN 2346 diff --git a/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h b/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h index 0340bba968688f..5c3b8fb41194ae 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h @@ -302,6 +302,14 @@ struct chipcregs { #define PMU_RCTL_LOGIC_DISABLE_MASK (1 << 27) +/* watchdog */ +#define CC_WD_SSRESET_PCIE_F0_EN 0x10000000 +#define CC_WD_SSRESET_PCIE_F1_EN 0x20000000 +#define CC_WD_SSRESET_PCIE_F2_EN 0x40000000 +#define CC_WD_SSRESET_PCIE_ALL_FN_EN 0x80000000 +#define CC_WD_COUNTER_MASK 0x0fffffff +#define CC_WD_ENABLE_MASK 0xf0000000 + /* * Maximum delay for the PMU state transition in us. * This is an upper bound intended for spinwaits etc. diff --git a/drivers/net/wireless/realtek/rtl8xxxu/core.c b/drivers/net/wireless/realtek/rtl8xxxu/core.c index c06ad064f37c65..f9a527f6a17535 100644 --- a/drivers/net/wireless/realtek/rtl8xxxu/core.c +++ b/drivers/net/wireless/realtek/rtl8xxxu/core.c @@ -7826,6 +7826,7 @@ static int rtl8xxxu_probe(struct usb_interface *interface, goto err_set_intfdata; hw->vif_data_size = sizeof(struct rtl8xxxu_vif); + hw->sta_data_size = sizeof(struct rtl8xxxu_sta_info); hw->wiphy->max_scan_ssids = 1; hw->wiphy->max_scan_ie_len = IEEE80211_MAX_DATA_LEN; diff --git a/drivers/net/wireless/realtek/rtw88/main.c b/drivers/net/wireless/realtek/rtw88/main.c index fa0ed39cb1992a..d93d21656f26c8 100644 --- a/drivers/net/wireless/realtek/rtw88/main.c +++ b/drivers/net/wireless/realtek/rtw88/main.c @@ -2444,10 +2444,10 @@ void rtw_core_enable_beacon(struct rtw_dev *rtwdev, bool enable) if (enable) { rtw_write32_set(rtwdev, REG_BCN_CTRL, BIT_EN_BCN_FUNCTION); - rtw_write32_clr(rtwdev, REG_TXPAUSE, BIT_HIGH_QUEUE); + rtw_write8_clr(rtwdev, REG_TXPAUSE, BIT_HIGH_QUEUE); } else { rtw_write32_clr(rtwdev, REG_BCN_CTRL, BIT_EN_BCN_FUNCTION); - rtw_write32_set(rtwdev, REG_TXPAUSE, BIT_HIGH_QUEUE); + rtw_write8_set(rtwdev, REG_TXPAUSE, BIT_HIGH_QUEUE); } } diff --git a/drivers/net/wireless/realtek/rtw89/core.c b/drivers/net/wireless/realtek/rtw89/core.c index 0824940c91aee1..53d32f3137ebe9 100644 --- a/drivers/net/wireless/realtek/rtw89/core.c +++ b/drivers/net/wireless/realtek/rtw89/core.c @@ -1207,7 +1207,7 @@ rtw89_core_tx_update_desc_info(struct rtw89_dev *rtwdev, if (addr_cam->valid && desc_info->mlo) upd_wlan_hdr = true; - if (rtw89_is_tx_rpt_skb(rtwdev, tx_req->skb)) + if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS || tx_req->with_wait) rtw89_tx_rpt_init(rtwdev, tx_req); is_bmc = (is_broadcast_ether_addr(hdr->addr1) || @@ -1342,13 +1342,15 @@ static int rtw89_core_tx_write_link(struct rtw89_dev *rtwdev, tx_req.rtwvif_link = rtwvif_link; tx_req.rtwsta_link = rtwsta_link; tx_req.desc_info.sw_mld = sw_mld; - rcu_assign_pointer(skb_data->wait, wait); + tx_req.with_wait = !!wait; rtw89_traffic_stats_accu(rtwdev, rtwvif, skb, true, true); rtw89_wow_parse_akm(rtwdev, skb); rtw89_core_tx_update_desc_info(rtwdev, &tx_req); rtw89_core_tx_wake(rtwdev, &tx_req); + rcu_assign_pointer(skb_data->wait, wait); + ret = rtw89_hci_tx_write(rtwdev, &tx_req); if (ret) { rtw89_err(rtwdev, "failed to transmit skb to HCI\n"); diff --git a/drivers/net/wireless/realtek/rtw89/core.h b/drivers/net/wireless/realtek/rtw89/core.h index a9cb47ea0b9352..92636cfc5ca58d 100644 --- a/drivers/net/wireless/realtek/rtw89/core.h +++ b/drivers/net/wireless/realtek/rtw89/core.h @@ -1211,6 +1211,8 @@ struct rtw89_core_tx_request { struct rtw89_vif_link *rtwvif_link; struct rtw89_sta_link *rtwsta_link; struct rtw89_tx_desc_info desc_info; + + bool with_wait; }; struct rtw89_txq { diff --git a/drivers/net/wireless/realtek/rtw89/debug.c b/drivers/net/wireless/realtek/rtw89/debug.c index 1264c2f82600bf..987eef8170f2bd 100644 --- a/drivers/net/wireless/realtek/rtw89/debug.c +++ b/drivers/net/wireless/realtek/rtw89/debug.c @@ -825,10 +825,6 @@ static ssize_t __print_txpwr_map(struct rtw89_dev *rtwdev, char *buf, size_t buf s8 *bufp, tmp; int ret; - bufp = vzalloc(map->addr_to - map->addr_from + 4); - if (!bufp) - return -ENOMEM; - if (path_num == 1) max_valid_addr = map->addr_to_1ss; else @@ -837,6 +833,10 @@ static ssize_t __print_txpwr_map(struct rtw89_dev *rtwdev, char *buf, size_t buf if (max_valid_addr == 0) return -EOPNOTSUPP; + bufp = vzalloc(map->addr_to - map->addr_from + 4); + if (!bufp) + return -ENOMEM; + for (addr = map->addr_from; addr <= max_valid_addr; addr += 4) { ret = rtw89_mac_txpwr_read32(rtwdev, RTW89_PHY_0, addr, &val); if (ret) diff --git a/drivers/net/xen-netback/xenbus.c b/drivers/net/xen-netback/xenbus.c index a78a25b872409a..61b547aab286a2 100644 --- a/drivers/net/xen-netback/xenbus.c +++ b/drivers/net/xen-netback/xenbus.c @@ -735,10 +735,11 @@ static void connect(struct backend_info *be) */ requested_num_queues = xenbus_read_unsigned(dev->otherend, "multi-queue-num-queues", 1); - if (requested_num_queues > xenvif_max_queues) { + if (requested_num_queues > xenvif_max_queues || + requested_num_queues == 0) { /* buggy or malicious guest */ xenbus_dev_fatal(dev, -EINVAL, - "guest requested %u queues, exceeding the maximum of %u.", + "guest requested %u queues, but valid range is 1 - %u.", requested_num_queues, xenvif_max_queues); return; } diff --git a/drivers/nvdimm/nd_virtio.c b/drivers/nvdimm/nd_virtio.c index c3f07be4aa22ad..af82385be7c6aa 100644 --- a/drivers/nvdimm/nd_virtio.c +++ b/drivers/nvdimm/nd_virtio.c @@ -44,6 +44,8 @@ static int virtio_pmem_flush(struct nd_region *nd_region) unsigned long flags; int err, err1; + guard(mutex)(&vpmem->flush_lock); + /* * Don't bother to submit the request to the device if the device is * not activated. @@ -53,7 +55,6 @@ static int virtio_pmem_flush(struct nd_region *nd_region) return -EIO; } - might_sleep(); req_data = kmalloc(sizeof(*req_data), GFP_KERNEL); if (!req_data) return -ENOMEM; diff --git a/drivers/nvdimm/virtio_pmem.c b/drivers/nvdimm/virtio_pmem.c index 2396d19ce54969..77b1966619059c 100644 --- a/drivers/nvdimm/virtio_pmem.c +++ b/drivers/nvdimm/virtio_pmem.c @@ -64,6 +64,7 @@ static int virtio_pmem_probe(struct virtio_device *vdev) goto out_err; } + mutex_init(&vpmem->flush_lock); vpmem->vdev = vdev; vdev->priv = vpmem; err = init_vq(vpmem); diff --git a/drivers/nvdimm/virtio_pmem.h b/drivers/nvdimm/virtio_pmem.h index 0dddefe594c46a..f72cf17f9518fb 100644 --- a/drivers/nvdimm/virtio_pmem.h +++ b/drivers/nvdimm/virtio_pmem.h @@ -13,6 +13,7 @@ #include #include #include +#include #include struct virtio_pmem_request { @@ -35,6 +36,9 @@ struct virtio_pmem { /* Virtio pmem request queue */ struct virtqueue *req_vq; + /* Serialize flush requests to the device. */ + struct mutex flush_lock; + /* nvdimm bus registers virtio pmem device */ struct nvdimm_bus *nvdimm_bus; struct nvdimm_bus_descriptor nd_desc; diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index ed61b97fde59f7..e4085e6ac8236d 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -203,8 +203,20 @@ struct apple_nvme { int irq; spinlock_t lock; + + /* + * Delayed cache flush handling state + */ + struct nvme_ns *flush_ns; + unsigned long flush_interval; + unsigned long last_flush; + struct delayed_work flush_dwork; }; +unsigned int flush_interval = 1000; +module_param(flush_interval, uint, 0644); +MODULE_PARM_DESC(flush_interval, "Grace period in msecs between flushes"); + static_assert(sizeof(struct nvme_command) == 64); static_assert(sizeof(struct apple_nvmmu_tcb) == 128); @@ -762,6 +774,26 @@ static int apple_nvme_remove_sq(struct apple_nvme *anv) return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0); } +static bool apple_nvme_delayed_flush(struct apple_nvme *anv, struct nvme_ns *ns, + struct request *req) +{ + if (!anv->flush_interval || req_op(req) != REQ_OP_FLUSH) + return false; + if (delayed_work_pending(&anv->flush_dwork)) + return true; + if (time_before(jiffies, anv->last_flush + anv->flush_interval)) { + kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &anv->flush_dwork, + anv->flush_interval); + if (WARN_ON_ONCE(anv->flush_ns && anv->flush_ns != ns)) + goto out; + anv->flush_ns = ns; + return true; + } +out: + anv->last_flush = jiffies; + return false; +} + static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { @@ -798,6 +830,11 @@ static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx, nvme_start_request(req); + if (apple_nvme_delayed_flush(anv, ns, req)) { + blk_mq_complete_request(req); + return BLK_STS_OK; + } + if (anv->hw->has_lsq_nvmmu) apple_nvme_submit_cmd_t8103(q, cmnd); else @@ -1453,6 +1490,28 @@ static void devm_apple_nvme_mempool_destroy(void *data) mempool_destroy(data); } +static void apple_nvme_flush_work(struct work_struct *work) +{ + struct nvme_command c = { }; + struct apple_nvme *anv; + struct nvme_ns *ns; + int err; + + anv = container_of(work, struct apple_nvme, flush_dwork.work); + ns = anv->flush_ns; + if (WARN_ON_ONCE(!ns)) + return; + + c.common.opcode = nvme_cmd_flush; + c.common.nsid = cpu_to_le32(anv->flush_ns->head->ns_id); + err = nvme_submit_sync_cmd(ns->queue, &c, NULL, 0); + if (err) { + dev_err(anv->dev, "Deferred flush failed: %d\n", err); + } else { + anv->last_flush = jiffies; + } +} + static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1621,6 +1680,14 @@ static int apple_nvme_probe(struct platform_device *pdev) goto out_uninit_ctrl; } + if (flush_interval) { + anv->flush_interval = msecs_to_jiffies(flush_interval); + anv->flush_ns = NULL; + anv->last_flush = jiffies - anv->flush_interval; + } + + INIT_DELAYED_WORK(&anv->flush_dwork, apple_nvme_flush_work); + nvme_reset_ctrl(&anv->ctrl); async_schedule(apple_nvme_async_probe, anv); @@ -1658,6 +1725,7 @@ static void apple_nvme_shutdown(struct platform_device *pdev) { struct apple_nvme *anv = platform_get_drvdata(pdev); + flush_delayed_work(&anv->flush_dwork); apple_nvme_disable(anv, true); if (apple_rtkit_is_running(anv->rtk)) { apple_rtkit_shutdown(anv->rtk); diff --git a/drivers/nvmem/Kconfig b/drivers/nvmem/Kconfig index bf47a982cf6290..74ddbd0f79b0ec 100644 --- a/drivers/nvmem/Kconfig +++ b/drivers/nvmem/Kconfig @@ -30,7 +30,7 @@ source "drivers/nvmem/layouts/Kconfig" config NVMEM_AN8855_EFUSE tristate "Airoha AN8855 eFuse support" - depends on MFD_AIROHA_AN8855 || COMPILE_TEST + depends on COMPILE_TEST help Say y here to enable support for reading eFuses on Airoha AN8855 Switch. These are e.g. used to store factory programmed diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c index 387c88c5525954..19be16943ee66e 100644 --- a/drivers/nvmem/core.c +++ b/drivers/nvmem/core.c @@ -1618,12 +1618,14 @@ static void nvmem_shift_read_buffer_in_place(struct nvmem_cell_entry *cell, void *p = *b++ >> bit_offset; /* setup rest of the bytes if any */ - for (i = 1; i < cell->bytes; i++) { + for (i = 1; i < (cell->bytes - bytes_offset); i++) { /* Get bits from next byte and shift them towards msb */ *p++ |= *b << (BITS_PER_BYTE - bit_offset); *p = *b++ >> bit_offset; } + /* point to end of the buffer unused bits will be cleared */ + p = buf + cell->bytes - 1; } else if (p != b) { memmove(p, b, cell->bytes - bytes_offset); p += cell->bytes - 1; diff --git a/drivers/of/unittest.c b/drivers/of/unittest.c index 3b773aaf9d0503..9c184e93f50c67 100644 --- a/drivers/of/unittest.c +++ b/drivers/of/unittest.c @@ -804,11 +804,13 @@ static void __init of_unittest_property_copy(void) new = __of_prop_dup(&p1, GFP_KERNEL); unittest(new && propcmp(&p1, new), "empty property didn't copy correctly\n"); - __of_prop_free(new); + if (new) + __of_prop_free(new); new = __of_prop_dup(&p2, GFP_KERNEL); unittest(new && propcmp(&p2, new), "non-empty property didn't copy correctly\n"); - __of_prop_free(new); + if (new) + __of_prop_free(new); #endif } diff --git a/drivers/opp/core.c b/drivers/opp/core.c index dbebb8c829bcb4..ae43c656f108c2 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -241,7 +241,7 @@ unsigned int dev_pm_opp_get_level(struct dev_pm_opp *opp) { if (IS_ERR_OR_NULL(opp) || !opp->available) { pr_err("%s: Invalid parameters\n", __func__); - return 0; + return U32_MAX; } return opp->level; diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index c254d2b8bf17b0..316d80f743a5e2 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -45,6 +45,7 @@ config PCIE_APPLE depends on ARCH_APPLE || COMPILE_TEST depends on OF depends on PCI_MSI + depends on PAGE_SIZE_16KB || COMPILE_TEST select PCI_HOST_COMMON select IRQ_MSI_LIB help diff --git a/drivers/pci/controller/cadence/pcie-cadence.c b/drivers/pci/controller/cadence/pcie-cadence.c index e6f1a4ac0fb7ab..a1eada56edba76 100644 --- a/drivers/pci/controller/cadence/pcie-cadence.c +++ b/drivers/pci/controller/cadence/pcie-cadence.c @@ -13,13 +13,13 @@ u8 cdns_pcie_find_capability(struct cdns_pcie *pcie, u8 cap) { return PCI_FIND_NEXT_CAP(cdns_pcie_read_cfg, PCI_CAPABILITY_LIST, - cap, pcie); + cap, NULL, pcie); } EXPORT_SYMBOL_GPL(cdns_pcie_find_capability); u16 cdns_pcie_find_ext_capability(struct cdns_pcie *pcie, u8 cap) { - return PCI_FIND_NEXT_EXT_CAP(cdns_pcie_read_cfg, 0, cap, pcie); + return PCI_FIND_NEXT_EXT_CAP(cdns_pcie_read_cfg, 0, cap, NULL, pcie); } EXPORT_SYMBOL_GPL(cdns_pcie_find_ext_capability); diff --git a/drivers/pci/controller/dwc/pci-dra7xx.c b/drivers/pci/controller/dwc/pci-dra7xx.c index 01cfd9aeb0b818..d5d26229063f2a 100644 --- a/drivers/pci/controller/dwc/pci-dra7xx.c +++ b/drivers/pci/controller/dwc/pci-dra7xx.c @@ -424,6 +424,7 @@ static int dra7xx_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no, } static const struct pci_epc_features dra7xx_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, .linkup_notifier = true, .msi_capable = true, }; diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c index 4668fc9648bff6..f28e335bbbfaf9 100644 --- a/drivers/pci/controller/dwc/pci-imx6.c +++ b/drivers/pci/controller/dwc/pci-imx6.c @@ -1387,6 +1387,7 @@ static int imx_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, } static const struct pci_epc_features imx8m_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, .msi_capable = true, .bar[BAR_1] = { .type = BAR_RESERVED, }, .bar[BAR_3] = { .type = BAR_RESERVED, }, @@ -1396,6 +1397,7 @@ static const struct pci_epc_features imx8m_pcie_epc_features = { }; static const struct pci_epc_features imx8q_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, .msi_capable = true, .bar[BAR_1] = { .type = BAR_RESERVED, }, .bar[BAR_3] = { .type = BAR_RESERVED, }, @@ -1416,6 +1418,7 @@ static const struct pci_epc_features imx8q_pcie_epc_features = { * BAR5 | Enable | 32-bit | 64 KB | Programmable Size */ static const struct pci_epc_features imx95_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, .msi_capable = true, .bar[BAR_1] = { .type = BAR_FIXED, .fixed_size = SZ_64K, }, .align = SZ_4K, diff --git a/drivers/pci/controller/dwc/pci-keystone.c b/drivers/pci/controller/dwc/pci-keystone.c index f86d9111f863f1..20fa4dadb82afe 100644 --- a/drivers/pci/controller/dwc/pci-keystone.c +++ b/drivers/pci/controller/dwc/pci-keystone.c @@ -930,6 +930,7 @@ static int ks_pcie_am654_raise_irq(struct dw_pcie_ep *ep, u8 func_no, } static const struct pci_epc_features ks_pcie_am654_epc_features = { + DWC_EPC_COMMON_FEATURES, .msi_capable = true, .msix_capable = true, .bar[BAR_0] = { .type = BAR_RESERVED, }, diff --git a/drivers/pci/controller/dwc/pcie-artpec6.c b/drivers/pci/controller/dwc/pcie-artpec6.c index f4a136ee2daf38..e994b75986c34a 100644 --- a/drivers/pci/controller/dwc/pcie-artpec6.c +++ b/drivers/pci/controller/dwc/pcie-artpec6.c @@ -370,6 +370,7 @@ static int artpec6_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no, } static const struct pci_epc_features artpec6_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, .msi_capable = true, }; diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c index 19571ac2b96171..59fd6ebf01489d 100644 --- a/drivers/pci/controller/dwc/pcie-designware-ep.c +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c @@ -72,47 +72,15 @@ EXPORT_SYMBOL_GPL(dw_pcie_ep_reset_bar); static u8 dw_pcie_ep_find_capability(struct dw_pcie_ep *ep, u8 func_no, u8 cap) { return PCI_FIND_NEXT_CAP(dw_pcie_ep_read_cfg, PCI_CAPABILITY_LIST, - cap, ep, func_no); + cap, NULL, ep, func_no); } -/** - * dw_pcie_ep_hide_ext_capability - Hide a capability from the linked list - * @pci: DWC PCI device - * @prev_cap: Capability preceding the capability that should be hidden - * @cap: Capability that should be hidden - * - * Return: 0 if success, errno otherwise. - */ -int dw_pcie_ep_hide_ext_capability(struct dw_pcie *pci, u8 prev_cap, u8 cap) +static u16 dw_pcie_ep_find_ext_capability(struct dw_pcie_ep *ep, + u8 func_no, u8 cap) { - u16 prev_cap_offset, cap_offset; - u32 prev_cap_header, cap_header; - - prev_cap_offset = dw_pcie_find_ext_capability(pci, prev_cap); - if (!prev_cap_offset) - return -EINVAL; - - prev_cap_header = dw_pcie_readl_dbi(pci, prev_cap_offset); - cap_offset = PCI_EXT_CAP_NEXT(prev_cap_header); - cap_header = dw_pcie_readl_dbi(pci, cap_offset); - - /* cap must immediately follow prev_cap. */ - if (PCI_EXT_CAP_ID(cap_header) != cap) - return -EINVAL; - - /* Clear next ptr. */ - prev_cap_header &= ~GENMASK(31, 20); - - /* Set next ptr to next ptr of cap. */ - prev_cap_header |= cap_header & GENMASK(31, 20); - - dw_pcie_dbi_ro_wr_en(pci); - dw_pcie_writel_dbi(pci, prev_cap_offset, prev_cap_header); - dw_pcie_dbi_ro_wr_dis(pci); - - return 0; + return PCI_FIND_NEXT_EXT_CAP(dw_pcie_ep_read_cfg, 0, + cap, NULL, ep, func_no); } -EXPORT_SYMBOL_GPL(dw_pcie_ep_hide_ext_capability); static int dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no, u8 vfunc_no, struct pci_epf_header *hdr) @@ -139,18 +107,23 @@ static int dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no, u8 vfunc_no, return 0; } -static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, u8 func_no, int type, - dma_addr_t parent_bus_addr, enum pci_barno bar, - size_t size) +/* BAR Match Mode inbound iATU mapping */ +static int dw_pcie_ep_ib_atu_bar(struct dw_pcie_ep *ep, u8 func_no, int type, + dma_addr_t parent_bus_addr, enum pci_barno bar, + size_t size) { int ret; u32 free_win; struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); - if (!ep->bar_to_atu[bar]) + if (!ep_func) + return -EINVAL; + + if (!ep_func->bar_to_atu[bar]) free_win = find_first_zero_bit(ep->ib_window_map, pci->num_ib_windows); else - free_win = ep->bar_to_atu[bar] - 1; + free_win = ep_func->bar_to_atu[bar] - 1; if (free_win >= pci->num_ib_windows) { dev_err(pci->dev, "No free inbound window\n"); @@ -168,12 +141,190 @@ static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, u8 func_no, int type, * Always increment free_win before assignment, since value 0 is used to identify * unallocated mapping. */ - ep->bar_to_atu[bar] = free_win + 1; + ep_func->bar_to_atu[bar] = free_win + 1; set_bit(free_win, ep->ib_window_map); return 0; } +static void dw_pcie_ep_clear_ib_maps(struct dw_pcie_ep *ep, u8 func_no, enum pci_barno bar) +{ + struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct device *dev = pci->dev; + unsigned int i, num; + u32 atu_index; + u32 *indexes; + + if (!ep_func) + return; + + /* Tear down the BAR Match Mode mapping, if any. */ + if (ep_func->bar_to_atu[bar]) { + atu_index = ep_func->bar_to_atu[bar] - 1; + dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_IB, atu_index); + clear_bit(atu_index, ep->ib_window_map); + ep_func->bar_to_atu[bar] = 0; + } + + /* Tear down all Address Match Mode mappings, if any. */ + indexes = ep_func->ib_atu_indexes[bar]; + num = ep_func->num_ib_atu_indexes[bar]; + ep_func->ib_atu_indexes[bar] = NULL; + ep_func->num_ib_atu_indexes[bar] = 0; + if (!indexes) + return; + for (i = 0; i < num; i++) { + dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_IB, indexes[i]); + clear_bit(indexes[i], ep->ib_window_map); + } + devm_kfree(dev, indexes); +} + +static u64 dw_pcie_ep_read_bar_assigned(struct dw_pcie_ep *ep, u8 func_no, + enum pci_barno bar, int flags) +{ + u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar); + u32 lo, hi; + u64 addr; + + lo = dw_pcie_ep_readl_dbi(ep, func_no, reg); + + if (flags & PCI_BASE_ADDRESS_SPACE) + return lo & PCI_BASE_ADDRESS_IO_MASK; + + addr = lo & PCI_BASE_ADDRESS_MEM_MASK; + if (!(flags & PCI_BASE_ADDRESS_MEM_TYPE_64)) + return addr; + + hi = dw_pcie_ep_readl_dbi(ep, func_no, reg + 4); + return addr | ((u64)hi << 32); +} + +static int dw_pcie_ep_validate_submap(struct dw_pcie_ep *ep, + const struct pci_epf_bar_submap *submap, + unsigned int num_submap, size_t bar_size) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + u32 align = pci->region_align; + size_t off = 0; + unsigned int i; + size_t size; + + if (!align || !IS_ALIGNED(bar_size, align)) + return -EINVAL; + + /* + * The submap array order defines the BAR layout (submap[0] starts + * at offset 0 and each entry immediately follows the previous + * one). Here, validate that it forms a strict, gapless + * decomposition of the BAR: + * - each entry has a non-zero size + * - sizes, implicit offsets and phys_addr are aligned to + * pci->region_align + * - each entry lies within the BAR range + * - the entries exactly cover the whole BAR + * + * Note: dw_pcie_prog_inbound_atu() also checks alignment for the + * PCI address and the target phys_addr, but validating up-front + * avoids partially programming iATU windows in vain. + */ + for (i = 0; i < num_submap; i++) { + size = submap[i].size; + + if (!size) + return -EINVAL; + + if (!IS_ALIGNED(size, align) || !IS_ALIGNED(off, align)) + return -EINVAL; + + if (!IS_ALIGNED(submap[i].phys_addr, align)) + return -EINVAL; + + if (off > bar_size || size > bar_size - off) + return -EINVAL; + + off += size; + } + if (off != bar_size) + return -EINVAL; + + return 0; +} + +/* Address Match Mode inbound iATU mapping */ +static int dw_pcie_ep_ib_atu_addr(struct dw_pcie_ep *ep, u8 func_no, int type, + const struct pci_epf_bar *epf_bar) +{ + struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); + const struct pci_epf_bar_submap *submap = epf_bar->submap; + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + enum pci_barno bar = epf_bar->barno; + struct device *dev = pci->dev; + u64 pci_addr, parent_bus_addr; + u64 size, base, off = 0; + int free_win, ret; + unsigned int i; + u32 *indexes; + + if (!ep_func || !epf_bar->num_submap || !submap || !epf_bar->size) + return -EINVAL; + + ret = dw_pcie_ep_validate_submap(ep, submap, epf_bar->num_submap, + epf_bar->size); + if (ret) + return ret; + + base = dw_pcie_ep_read_bar_assigned(ep, func_no, bar, epf_bar->flags); + if (!base) { + dev_err(dev, + "BAR%u not assigned, cannot set up sub-range mappings\n", + bar); + return -EINVAL; + } + + indexes = devm_kcalloc(dev, epf_bar->num_submap, sizeof(*indexes), + GFP_KERNEL); + if (!indexes) + return -ENOMEM; + + ep_func->ib_atu_indexes[bar] = indexes; + ep_func->num_ib_atu_indexes[bar] = 0; + + for (i = 0; i < epf_bar->num_submap; i++) { + size = submap[i].size; + parent_bus_addr = submap[i].phys_addr; + + if (off > (~0ULL) - base) { + ret = -EINVAL; + goto err; + } + + pci_addr = base + off; + off += size; + + free_win = find_first_zero_bit(ep->ib_window_map, + pci->num_ib_windows); + if (free_win >= pci->num_ib_windows) { + ret = -ENOSPC; + goto err; + } + + ret = dw_pcie_prog_inbound_atu(pci, free_win, type, + parent_bus_addr, pci_addr, size); + if (ret) + goto err; + + set_bit(free_win, ep->ib_window_map); + indexes[i] = free_win; + ep_func->num_ib_atu_indexes[bar] = i + 1; + } + return 0; +err: + dw_pcie_ep_clear_ib_maps(ep, func_no, bar); + return ret; +} + static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, struct dw_pcie_ob_atu_cfg *atu) { @@ -204,35 +355,34 @@ static void dw_pcie_ep_clear_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, struct dw_pcie_ep *ep = epc_get_drvdata(epc); struct dw_pcie *pci = to_dw_pcie_from_ep(ep); enum pci_barno bar = epf_bar->barno; - u32 atu_index = ep->bar_to_atu[bar] - 1; + struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); - if (!ep->bar_to_atu[bar]) + if (!ep_func || !ep_func->epf_bar[bar]) return; __dw_pcie_ep_reset_bar(pci, func_no, bar, epf_bar->flags); - dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_IB, atu_index); - clear_bit(atu_index, ep->ib_window_map); - ep->epf_bar[bar] = NULL; - ep->bar_to_atu[bar] = 0; + dw_pcie_ep_clear_ib_maps(ep, func_no, bar); + + ep_func->epf_bar[bar] = NULL; } -static unsigned int dw_pcie_ep_get_rebar_offset(struct dw_pcie *pci, +static unsigned int dw_pcie_ep_get_rebar_offset(struct dw_pcie_ep *ep, u8 func_no, enum pci_barno bar) { u32 reg, bar_index; unsigned int offset, nbars; int i; - offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_REBAR); + offset = dw_pcie_ep_find_ext_capability(ep, func_no, PCI_EXT_CAP_ID_REBAR); if (!offset) return offset; - reg = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL); + reg = dw_pcie_ep_readl_dbi(ep, func_no, offset + PCI_REBAR_CTRL); nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, reg); for (i = 0; i < nbars; i++, offset += PCI_REBAR_CTRL) { - reg = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL); + reg = dw_pcie_ep_readl_dbi(ep, func_no, offset + PCI_REBAR_CTRL); bar_index = FIELD_GET(PCI_REBAR_CTRL_BAR_IDX, reg); if (bar_index == bar) return offset; @@ -253,7 +403,7 @@ static int dw_pcie_ep_set_bar_resizable(struct dw_pcie_ep *ep, u8 func_no, u32 rebar_cap, rebar_ctrl; int ret; - rebar_offset = dw_pcie_ep_get_rebar_offset(pci, bar); + rebar_offset = dw_pcie_ep_get_rebar_offset(ep, func_no, bar); if (!rebar_offset) return -EINVAL; @@ -283,16 +433,16 @@ static int dw_pcie_ep_set_bar_resizable(struct dw_pcie_ep *ep, u8 func_no, * 1 MB to 128 TB. Bits 31:16 in PCI_REBAR_CTRL define "supported sizes" * bits for sizes 256 TB to 8 EB. Disallow sizes 256 TB to 8 EB. */ - rebar_ctrl = dw_pcie_readl_dbi(pci, rebar_offset + PCI_REBAR_CTRL); + rebar_ctrl = dw_pcie_ep_readl_dbi(ep, func_no, rebar_offset + PCI_REBAR_CTRL); rebar_ctrl &= ~GENMASK(31, 16); - dw_pcie_writel_dbi(pci, rebar_offset + PCI_REBAR_CTRL, rebar_ctrl); + dw_pcie_ep_writel_dbi(ep, func_no, rebar_offset + PCI_REBAR_CTRL, rebar_ctrl); /* * The "selected size" (bits 13:8) in PCI_REBAR_CTRL are automatically * updated when writing PCI_REBAR_CAP, see "Figure 3-26 Resizable BAR * Example for 32-bit Memory BAR0" in DWC EP databook 5.96a. */ - dw_pcie_writel_dbi(pci, rebar_offset + PCI_REBAR_CAP, rebar_cap); + dw_pcie_ep_writel_dbi(ep, func_no, rebar_offset + PCI_REBAR_CAP, rebar_cap); dw_pcie_dbi_ro_wr_dis(pci); @@ -341,12 +491,16 @@ static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, { struct dw_pcie_ep *ep = epc_get_drvdata(epc); struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); enum pci_barno bar = epf_bar->barno; size_t size = epf_bar->size; enum pci_epc_bar_type bar_type; int flags = epf_bar->flags; int ret, type; + if (!ep_func) + return -EINVAL; + /* * DWC does not allow BAR pairs to overlap, e.g. you cannot combine BARs * 1 and 2 to form a 64-bit BAR. @@ -360,21 +514,46 @@ static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, * calling clear_bar() would clear the BAR's PCI address assigned by the * host). */ - if (ep->epf_bar[bar]) { + if (ep_func->epf_bar[bar]) { /* * We can only dynamically change a BAR if the new BAR size and * BAR flags do not differ from the existing configuration. + * + * Note: this safety check only works when the caller uses + * a new struct pci_epf_bar in the second set_bar() call. + * If the same instance is updated in place and passed in, + * we cannot reliably detect invalid barno/size/flags + * changes here. */ - if (ep->epf_bar[bar]->barno != bar || - ep->epf_bar[bar]->size != size || - ep->epf_bar[bar]->flags != flags) + if (ep_func->epf_bar[bar]->barno != bar || + ep_func->epf_bar[bar]->size != size || + ep_func->epf_bar[bar]->flags != flags) return -EINVAL; + /* + * When dynamically changing a BAR, tear down any existing + * mappings before re-programming. This is redundant when + * both the old and new mappings are BAR Match Mode, but + * required to handle in-place updates and match-mode + * changes reliably. + */ + dw_pcie_ep_clear_ib_maps(ep, func_no, bar); + /* * When dynamically changing a BAR, skip writing the BAR reg, as * that would clear the BAR's PCI address assigned by the host. */ goto config_atu; + } else { + /* + * Subrange mapping is an update-only operation. The BAR + * must have been configured once without submaps so that + * subsequent set_bar() calls can update inbound mappings + * without touching the BAR register (and clobbering the + * host-assigned address). + */ + if (epf_bar->num_submap) + return -EINVAL; } bar_type = dw_pcie_ep_get_bar_type(ep, bar); @@ -408,12 +587,16 @@ static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, else type = PCIE_ATU_TYPE_IO; - ret = dw_pcie_ep_inbound_atu(ep, func_no, type, epf_bar->phys_addr, bar, - size); + if (epf_bar->num_submap) + ret = dw_pcie_ep_ib_atu_addr(ep, func_no, type, epf_bar); + else + ret = dw_pcie_ep_ib_atu_bar(ep, func_no, type, + epf_bar->phys_addr, bar, size); + if (ret) return ret; - ep->epf_bar[bar] = epf_bar; + ep_func->epf_bar[bar] = epf_bar; return 0; } @@ -601,6 +784,16 @@ static void dw_pcie_ep_stop(struct pci_epc *epc) struct dw_pcie_ep *ep = epc_get_drvdata(epc); struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + /* + * Tear down the dedicated outbound window used for MSI + * generation. This avoids leaking an iATU window across + * endpoint stop/start cycles. + */ + if (ep->msi_iatu_mapped) { + dw_pcie_ep_unmap_addr(epc, 0, 0, ep->msi_mem_phys); + ep->msi_iatu_mapped = false; + } + dw_pcie_stop_link(pci); } @@ -702,14 +895,37 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, msg_addr = ((u64)msg_addr_upper) << 32 | msg_addr_lower; msg_addr = dw_pcie_ep_align_addr(epc, msg_addr, &map_size, &offset); - ret = dw_pcie_ep_map_addr(epc, func_no, 0, ep->msi_mem_phys, msg_addr, - map_size); - if (ret) - return ret; - writel(msg_data | (interrupt_num - 1), ep->msi_mem + offset); + /* + * Program the outbound iATU once and keep it enabled. + * + * The spec warns that updating iATU registers while there are + * operations in flight on the AXI bridge interface is not + * supported, so we avoid reprogramming the region on every MSI, + * specifically unmapping immediately after writel(). + */ + if (!ep->msi_iatu_mapped) { + ret = dw_pcie_ep_map_addr(epc, func_no, 0, + ep->msi_mem_phys, msg_addr, + map_size); + if (ret) + return ret; + + ep->msi_iatu_mapped = true; + ep->msi_msg_addr = msg_addr; + ep->msi_map_size = map_size; + } else if (WARN_ON_ONCE(ep->msi_msg_addr != msg_addr || + ep->msi_map_size != map_size)) { + /* + * The host changed the MSI target address or the required + * mapping size changed. Reprogramming the iATU at runtime is + * unsafe on this controller, so bail out instead of trying to + * update the existing region. + */ + return -EINVAL; + } - dw_pcie_ep_unmap_addr(epc, func_no, 0, ep->msi_mem_phys); + writel(msg_data | (interrupt_num - 1), ep->msi_mem + offset); return 0; } @@ -775,7 +991,7 @@ int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, bir = FIELD_GET(PCI_MSIX_TABLE_BIR, tbl_offset); tbl_offset &= PCI_MSIX_TABLE_OFFSET; - msix_tbl = ep->epf_bar[bir]->addr + tbl_offset; + msix_tbl = ep_func->epf_bar[bir]->addr + tbl_offset; msg_addr = msix_tbl[(interrupt_num - 1)].msg_addr; msg_data = msix_tbl[(interrupt_num - 1)].msg_data; vec_ctrl = msix_tbl[(interrupt_num - 1)].vector_ctrl; @@ -836,20 +1052,20 @@ void dw_pcie_ep_deinit(struct dw_pcie_ep *ep) } EXPORT_SYMBOL_GPL(dw_pcie_ep_deinit); -static void dw_pcie_ep_init_non_sticky_registers(struct dw_pcie *pci) +static void dw_pcie_ep_init_rebar_registers(struct dw_pcie_ep *ep, u8 func_no) { - struct dw_pcie_ep *ep = &pci->ep; - unsigned int offset; - unsigned int nbars; + struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); + unsigned int offset, nbars; enum pci_barno bar; u32 reg, i, val; - offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_REBAR); + if (!ep_func) + return; - dw_pcie_dbi_ro_wr_en(pci); + offset = dw_pcie_ep_find_ext_capability(ep, func_no, PCI_EXT_CAP_ID_REBAR); if (offset) { - reg = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL); + reg = dw_pcie_ep_readl_dbi(ep, func_no, offset + PCI_REBAR_CTRL); nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, reg); /* @@ -870,16 +1086,28 @@ static void dw_pcie_ep_init_non_sticky_registers(struct dw_pcie *pci) * the controller when RESBAR_CAP_REG is written, which * is why RESBAR_CAP_REG is written here. */ - val = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL); + val = dw_pcie_ep_readl_dbi(ep, func_no, offset + PCI_REBAR_CTRL); bar = FIELD_GET(PCI_REBAR_CTRL_BAR_IDX, val); - if (ep->epf_bar[bar]) - pci_epc_bar_size_to_rebar_cap(ep->epf_bar[bar]->size, &val); + if (ep_func->epf_bar[bar]) + pci_epc_bar_size_to_rebar_cap(ep_func->epf_bar[bar]->size, &val); else val = BIT(4); - dw_pcie_writel_dbi(pci, offset + PCI_REBAR_CAP, val); + dw_pcie_ep_writel_dbi(ep, func_no, offset + PCI_REBAR_CAP, val); } } +} + +static void dw_pcie_ep_init_non_sticky_registers(struct dw_pcie *pci) +{ + struct dw_pcie_ep *ep = &pci->ep; + u8 funcs = ep->epc->max_functions; + u8 func_no; + + dw_pcie_dbi_ro_wr_en(pci); + + for (func_no = 0; func_no < funcs; func_no++) + dw_pcie_ep_init_rebar_registers(ep, func_no); dw_pcie_setup(pci); dw_pcie_dbi_ro_wr_dis(pci); @@ -1087,6 +1315,9 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep) struct device *dev = pci->dev; INIT_LIST_HEAD(&ep->func_list); + ep->msi_iatu_mapped = false; + ep->msi_msg_addr = 0; + ep->msi_map_size = 0; epc = devm_pci_epc_create(dev, &epc_ops); if (IS_ERR(epc)) { diff --git a/drivers/pci/controller/dwc/pcie-designware-plat.c b/drivers/pci/controller/dwc/pcie-designware-plat.c index 12f41886c65d1e..8530746ec5cbb6 100644 --- a/drivers/pci/controller/dwc/pcie-designware-plat.c +++ b/drivers/pci/controller/dwc/pcie-designware-plat.c @@ -61,6 +61,7 @@ static int dw_plat_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, } static const struct pci_epc_features dw_plat_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, .msi_capable = true, .msix_capable = true, }; diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c index 75fc8b767fccfe..345365ea97c74c 100644 --- a/drivers/pci/controller/dwc/pcie-designware.c +++ b/drivers/pci/controller/dwc/pcie-designware.c @@ -226,16 +226,69 @@ void dw_pcie_version_detect(struct dw_pcie *pci) u8 dw_pcie_find_capability(struct dw_pcie *pci, u8 cap) { return PCI_FIND_NEXT_CAP(dw_pcie_read_cfg, PCI_CAPABILITY_LIST, cap, - pci); + NULL, pci); } EXPORT_SYMBOL_GPL(dw_pcie_find_capability); u16 dw_pcie_find_ext_capability(struct dw_pcie *pci, u8 cap) { - return PCI_FIND_NEXT_EXT_CAP(dw_pcie_read_cfg, 0, cap, pci); + return PCI_FIND_NEXT_EXT_CAP(dw_pcie_read_cfg, 0, cap, NULL, pci); } EXPORT_SYMBOL_GPL(dw_pcie_find_ext_capability); +void dw_pcie_remove_capability(struct dw_pcie *pci, u8 cap) +{ + u8 cap_pos, pre_pos, next_pos; + u16 reg; + + cap_pos = PCI_FIND_NEXT_CAP(dw_pcie_read_cfg, PCI_CAPABILITY_LIST, cap, + &pre_pos, pci); + if (!cap_pos) + return; + + reg = dw_pcie_readw_dbi(pci, cap_pos); + next_pos = (reg & 0xff00) >> 8; + + dw_pcie_dbi_ro_wr_en(pci); + if (pre_pos == PCI_CAPABILITY_LIST) + dw_pcie_writeb_dbi(pci, PCI_CAPABILITY_LIST, next_pos); + else + dw_pcie_writeb_dbi(pci, pre_pos + 1, next_pos); + dw_pcie_dbi_ro_wr_dis(pci); +} +EXPORT_SYMBOL_GPL(dw_pcie_remove_capability); + +void dw_pcie_remove_ext_capability(struct dw_pcie *pci, u8 cap) +{ + int cap_pos, next_pos, pre_pos; + u32 pre_header, header; + + cap_pos = PCI_FIND_NEXT_EXT_CAP(dw_pcie_read_cfg, 0, cap, &pre_pos, pci); + if (!cap_pos) + return; + + header = dw_pcie_readl_dbi(pci, cap_pos); + /* + * If the first cap at offset PCI_CFG_SPACE_SIZE is removed, + * only set it's capid to zero as it cannot be skipped. + */ + if (cap_pos == PCI_CFG_SPACE_SIZE) { + dw_pcie_dbi_ro_wr_en(pci); + dw_pcie_writel_dbi(pci, cap_pos, header & 0xffff0000); + dw_pcie_dbi_ro_wr_dis(pci); + return; + } + + pre_header = dw_pcie_readl_dbi(pci, pre_pos); + next_pos = PCI_EXT_CAP_NEXT(header); + + dw_pcie_dbi_ro_wr_en(pci); + dw_pcie_writel_dbi(pci, pre_pos, + (pre_header & 0xfffff) | (next_pos << 20)); + dw_pcie_dbi_ro_wr_dis(pci); +} +EXPORT_SYMBOL_GPL(dw_pcie_remove_ext_capability); + static u16 __dw_pcie_find_vsec_capability(struct dw_pcie *pci, u16 vendor_id, u16 vsec_id) { @@ -246,7 +299,7 @@ static u16 __dw_pcie_find_vsec_capability(struct dw_pcie *pci, u16 vendor_id, return 0; while ((vsec = PCI_FIND_NEXT_EXT_CAP(dw_pcie_read_cfg, vsec, - PCI_EXT_CAP_ID_VNDR, pci))) { + PCI_EXT_CAP_ID_VNDR, NULL, pci))) { header = dw_pcie_readl_dbi(pci, vsec + PCI_VNDR_HEADER); if (PCI_VNDR_HEADER_ID(header) == vsec_id) return vsec; diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h index 31685951a08045..5c429b62cb0860 100644 --- a/drivers/pci/controller/dwc/pcie-designware.h +++ b/drivers/pci/controller/dwc/pcie-designware.h @@ -305,6 +305,10 @@ /* Default eDMA LLP memory size */ #define DMA_LLP_MEM_SIZE PAGE_SIZE +/* Common struct pci_epc_feature bits among DWC EP glue drivers */ +#define DWC_EPC_COMMON_FEATURES .dynamic_inbound_mapping = true, \ + .subrange_mapping = true + struct dw_pcie; struct dw_pcie_rp; struct dw_pcie_ep; @@ -463,6 +467,12 @@ struct dw_pcie_ep_func { u8 func_no; u8 msi_cap; /* MSI capability offset */ u8 msix_cap; /* MSI-X capability offset */ + u8 bar_to_atu[PCI_STD_NUM_BARS]; + struct pci_epf_bar *epf_bar[PCI_STD_NUM_BARS]; + + /* Only for Address Match Mode inbound iATU */ + u32 *ib_atu_indexes[PCI_STD_NUM_BARS]; + unsigned int num_ib_atu_indexes[PCI_STD_NUM_BARS]; }; struct dw_pcie_ep { @@ -472,13 +482,16 @@ struct dw_pcie_ep { phys_addr_t phys_base; size_t addr_size; size_t page_size; - u8 bar_to_atu[PCI_STD_NUM_BARS]; phys_addr_t *outbound_addr; unsigned long *ib_window_map; unsigned long *ob_window_map; void __iomem *msi_mem; phys_addr_t msi_mem_phys; - struct pci_epf_bar *epf_bar[PCI_STD_NUM_BARS]; + + /* MSI outbound iATU state */ + bool msi_iatu_mapped; + u64 msi_msg_addr; + size_t msi_map_size; }; struct dw_pcie_ops { @@ -562,6 +575,8 @@ void dw_pcie_version_detect(struct dw_pcie *pci); u8 dw_pcie_find_capability(struct dw_pcie *pci, u8 cap); u16 dw_pcie_find_ext_capability(struct dw_pcie *pci, u8 cap); +void dw_pcie_remove_capability(struct dw_pcie *pci, u8 cap); +void dw_pcie_remove_ext_capability(struct dw_pcie *pci, u8 cap); u16 dw_pcie_find_rasdes_capability(struct dw_pcie *pci); u16 dw_pcie_find_ptm_capability(struct dw_pcie *pci); @@ -899,7 +914,6 @@ int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8 func_no, u16 interrupt_num); void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar); -int dw_pcie_ep_hide_ext_capability(struct dw_pcie *pci, u8 prev_cap, u8 cap); struct dw_pcie_ep_func * dw_pcie_ep_get_func_from_ep(struct dw_pcie_ep *ep, u8 func_no); #else @@ -957,12 +971,6 @@ static inline void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar) { } -static inline int dw_pcie_ep_hide_ext_capability(struct dw_pcie *pci, - u8 prev_cap, u8 cap) -{ - return 0; -} - static inline struct dw_pcie_ep_func * dw_pcie_ep_get_func_from_ep(struct dw_pcie_ep *ep, u8 func_no) { diff --git a/drivers/pci/controller/dwc/pcie-dw-rockchip.c b/drivers/pci/controller/dwc/pcie-dw-rockchip.c index f8605fe61a415a..bf8ec3ca6f689f 100644 --- a/drivers/pci/controller/dwc/pcie-dw-rockchip.c +++ b/drivers/pci/controller/dwc/pcie-dw-rockchip.c @@ -327,9 +327,7 @@ static void rockchip_pcie_ep_hide_broken_ats_cap_rk3588(struct dw_pcie_ep *ep) if (!of_device_is_compatible(dev->of_node, "rockchip,rk3588-pcie-ep")) return; - if (dw_pcie_ep_hide_ext_capability(pci, PCI_EXT_CAP_ID_SECPCI, - PCI_EXT_CAP_ID_ATS)) - dev_err(dev, "failed to hide ATS capability\n"); + dw_pcie_remove_ext_capability(pci, PCI_EXT_CAP_ID_ATS); } static void rockchip_pcie_ep_init(struct dw_pcie_ep *ep) @@ -364,6 +362,7 @@ static int rockchip_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no, } static const struct pci_epc_features rockchip_pcie_epc_features_rk3568 = { + DWC_EPC_COMMON_FEATURES, .linkup_notifier = true, .msi_capable = true, .msix_capable = true, @@ -384,6 +383,7 @@ static const struct pci_epc_features rockchip_pcie_epc_features_rk3568 = { * BARs) would be overwritten, resulting in (all other BARs) no longer working. */ static const struct pci_epc_features rockchip_pcie_epc_features_rk3588 = { + DWC_EPC_COMMON_FEATURES, .linkup_notifier = true, .msi_capable = true, .msix_capable = true, diff --git a/drivers/pci/controller/dwc/pcie-keembay.c b/drivers/pci/controller/dwc/pcie-keembay.c index 60e74ac782af32..2666a9c3d67e71 100644 --- a/drivers/pci/controller/dwc/pcie-keembay.c +++ b/drivers/pci/controller/dwc/pcie-keembay.c @@ -309,6 +309,7 @@ static int keembay_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, } static const struct pci_epc_features keembay_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, .msi_capable = true, .msix_capable = true, .bar[BAR_0] = { .only_64bit = true, }, diff --git a/drivers/pci/controller/dwc/pcie-nxp-s32g.c b/drivers/pci/controller/dwc/pcie-nxp-s32g.c index 47745749f75c36..b3ec38099fa386 100644 --- a/drivers/pci/controller/dwc/pcie-nxp-s32g.c +++ b/drivers/pci/controller/dwc/pcie-nxp-s32g.c @@ -282,12 +282,12 @@ static int s32g_pcie_parse_ports(struct device *dev, struct s32g_pcie *s32g_pp) ret = s32g_pcie_parse_port(s32g_pp, of_port); if (ret) - goto err_port; + break; } -err_port: - list_for_each_entry_safe(port, tmp, &s32g_pp->ports, list) - list_del(&port->list); + if (ret) + list_for_each_entry_safe(port, tmp, &s32g_pp->ports, list) + list_del(&port->list); return ret; } diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c index f1bc0ac81a928b..5e990c7a5879f4 100644 --- a/drivers/pci/controller/dwc/pcie-qcom-ep.c +++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c @@ -820,6 +820,7 @@ static void qcom_pcie_ep_init_debugfs(struct qcom_pcie_ep *pcie_ep) } static const struct pci_epc_features qcom_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, .linkup_notifier = true, .msi_capable = true, .align = SZ_4K, diff --git a/drivers/pci/controller/dwc/pcie-rcar-gen4.c b/drivers/pci/controller/dwc/pcie-rcar-gen4.c index 80778917d2ddd3..a6912e85e4ddc3 100644 --- a/drivers/pci/controller/dwc/pcie-rcar-gen4.c +++ b/drivers/pci/controller/dwc/pcie-rcar-gen4.c @@ -420,6 +420,7 @@ static int rcar_gen4_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, } static const struct pci_epc_features rcar_gen4_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, .msi_capable = true, .bar[BAR_1] = { .type = BAR_RESERVED, }, .bar[BAR_3] = { .type = BAR_RESERVED, }, diff --git a/drivers/pci/controller/dwc/pcie-sophgo.c b/drivers/pci/controller/dwc/pcie-sophgo.c index ad4baaa34ffa1d..044088898819e5 100644 --- a/drivers/pci/controller/dwc/pcie-sophgo.c +++ b/drivers/pci/controller/dwc/pcie-sophgo.c @@ -161,6 +161,22 @@ static void sophgo_pcie_msi_enable(struct dw_pcie_rp *pp) raw_spin_unlock_irqrestore(&pp->lock, flags); } +static void sophgo_pcie_disable_l0s_l1(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + u32 offset, val; + + offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + + dw_pcie_dbi_ro_wr_en(pci); + + val = dw_pcie_readl_dbi(pci, PCI_EXP_LNKCAP + offset); + val &= ~(PCI_EXP_LNKCAP_ASPM_L0S | PCI_EXP_LNKCAP_ASPM_L1); + dw_pcie_writel_dbi(pci, PCI_EXP_LNKCAP + offset, val); + + dw_pcie_dbi_ro_wr_dis(pci); +} + static int sophgo_pcie_host_init(struct dw_pcie_rp *pp) { int irq; @@ -171,6 +187,8 @@ static int sophgo_pcie_host_init(struct dw_pcie_rp *pp) irq_set_chained_handler_and_data(irq, sophgo_pcie_intx_handler, pp); + sophgo_pcie_disable_l0s_l1(pp); + sophgo_pcie_msi_enable(pp); return 0; diff --git a/drivers/pci/controller/dwc/pcie-stm32-ep.c b/drivers/pci/controller/dwc/pcie-stm32-ep.c index 2cecf32d2b0f36..c1944b40ce02f5 100644 --- a/drivers/pci/controller/dwc/pcie-stm32-ep.c +++ b/drivers/pci/controller/dwc/pcie-stm32-ep.c @@ -70,6 +70,7 @@ static int stm32_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no, } static const struct pci_epc_features stm32_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, .msi_capable = true, .align = SZ_64K, }; diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c index 0ddeef70726dda..06571d806ab31f 100644 --- a/drivers/pci/controller/dwc/pcie-tegra194.c +++ b/drivers/pci/controller/dwc/pcie-tegra194.c @@ -1988,6 +1988,7 @@ static int tegra_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, } static const struct pci_epc_features tegra_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, .linkup_notifier = true, .msi_capable = true, .bar[BAR_0] = { .type = BAR_FIXED, .fixed_size = SZ_1M, diff --git a/drivers/pci/controller/dwc/pcie-uniphier-ep.c b/drivers/pci/controller/dwc/pcie-uniphier-ep.c index d6e73811216e2e..d52753060970fa 100644 --- a/drivers/pci/controller/dwc/pcie-uniphier-ep.c +++ b/drivers/pci/controller/dwc/pcie-uniphier-ep.c @@ -420,6 +420,7 @@ static const struct uniphier_pcie_ep_soc_data uniphier_pro5_data = { .init = uniphier_pcie_pro5_init_ep, .wait = NULL, .features = { + DWC_EPC_COMMON_FEATURES, .linkup_notifier = false, .msi_capable = true, .msix_capable = false, @@ -438,6 +439,7 @@ static const struct uniphier_pcie_ep_soc_data uniphier_nx1_data = { .init = uniphier_pcie_nx1_init_ep, .wait = uniphier_pcie_nx1_wait_ep, .features = { + DWC_EPC_COMMON_FEATURES, .linkup_notifier = false, .msi_capable = true, .msix_capable = false, diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 2d92fc79f6ddfa..d962724750840d 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -33,6 +33,10 @@ #include "pci-host-common.h" +static int link_up_timeout = 500; +module_param(link_up_timeout, int, 0644); +MODULE_PARM_DESC(link_up_timeout, "PCIe link training timeout in milliseconds"); + /* T8103 (original M1) and related SoCs */ #define CORE_RC_PHYIF_CTL 0x00024 #define CORE_RC_PHYIF_CTL_RUN BIT(0) @@ -550,22 +554,105 @@ static u32 apple_pcie_rid2sid_write(struct apple_pcie_port *port, return readl_relaxed(port_rid2sid_addr(port, idx)); } +static int apple_pcie_setup_link(struct apple_pcie *pcie, + struct apple_pcie_port *port, + struct device_node *np) +{ +#define MAX_AUX_PERST 3 + struct gpio_desc *aux_reset[MAX_AUX_PERST] = { NULL }; + u32 num_aux_resets = 0; + struct gpio_desc *reset, *pwren = NULL; + u32 stat; + int ret; + + /* + * Assert PERST# and configure the pin as output. + * The Aquantia AQC113 10GB nic used desktop macs is sensitive to + * deasserting it without prior clock setup. + * Observed on M1 Max/Ultra Mac Studios under m1n1's hypervisor. + */ + reset = devm_fwnode_gpiod_get(pcie->dev, of_fwnode_handle(np), "reset", + GPIOD_OUT_HIGH, "PERST#"); + if (IS_ERR(reset)) + return PTR_ERR(reset); + // HACK: use additional "reset-gpios" until pci-pwrctrl gains PERST# support. + for (u32 idx = 0; idx < MAX_AUX_PERST; idx++) { + aux_reset[idx] = devm_fwnode_gpiod_get_index(pcie->dev, + of_fwnode_handle(np), + "reset", idx + 1, + GPIOD_OUT_HIGH, + "PERST#"); + if (IS_ERR(aux_reset[idx])) { + if (PTR_ERR(aux_reset[idx]) == -ENOENT) + break; + else + return PTR_ERR(aux_reset[idx]); + } + num_aux_resets++; + } + dev_info(pcie->dev, "Using %u auxiliary PERST#\n", num_aux_resets); + + pwren = devm_fwnode_gpiod_get(pcie->dev, of_fwnode_handle(np), "pwren", + GPIOD_ASIS, "PWREN"); + if (IS_ERR(pwren)) { + if (PTR_ERR(pwren) == -ENOENT) + pwren = NULL; + else + return PTR_ERR(pwren); + } + + rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); + + /* Assert PERST# before setting up the clock */ + gpiod_set_value_cansleep(reset, 1); + for (u32 idx = 0; idx < num_aux_resets; idx++) + gpiod_set_value_cansleep(aux_reset[idx], 1); + + /* Power on the device if required */ + gpiod_set_value_cansleep(pwren, 1); + + ret = apple_pcie_setup_refclk(pcie, port); + if (ret < 0) + return ret; + + /* + * The minimal Tperst-clk value is 100us (PCIe CEM r5.0, 2.9.2) + * If powering up, the minimal Tpvperl is 100ms + */ + if (pwren) + msleep(100); + else + usleep_range(100, 200); + + /* Deassert PERST# */ + rmw_set(PORT_PERST_OFF, port->base + pcie->hw->port_perst); + gpiod_set_value_cansleep(reset, 0); + for (u32 idx = 0; idx < num_aux_resets; idx++) + gpiod_set_value_cansleep(aux_reset[idx], 0); + + /* Wait for 100ms after PERST# deassertion (PCIe r5.0, 6.6.1) */ + msleep(100); + + ret = readl_relaxed_poll_timeout(port->base + PORT_STATUS, stat, + stat & PORT_STATUS_READY, 100, 250000); + if (ret < 0) { + dev_err(pcie->dev, "port %pOF ready wait timeout\n", np); + return ret; + } + + return 0; +} + static int apple_pcie_setup_port(struct apple_pcie *pcie, struct device_node *np) { struct platform_device *platform = to_platform_device(pcie->dev); struct apple_pcie_port *port; - struct gpio_desc *reset; struct resource *res; char name[16]; - u32 stat, idx; + u32 link_stat, idx; int ret, i; - reset = devm_fwnode_gpiod_get(pcie->dev, of_fwnode_handle(np), "reset", - GPIOD_OUT_LOW, "PERST#"); - if (IS_ERR(reset)) - return PTR_ERR(reset); - port = devm_kzalloc(pcie->dev, sizeof(*port), GFP_KERNEL); if (!port) return -ENOMEM; @@ -601,30 +688,12 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, else port->phy = pcie->base + CORE_PHY_DEFAULT_BASE(port->idx); - rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); - - /* Assert PERST# before setting up the clock */ - gpiod_set_value_cansleep(reset, 1); - - ret = apple_pcie_setup_refclk(pcie, port); - if (ret < 0) - return ret; - - /* The minimal Tperst-clk value is 100us (PCIe CEM r5.0, 2.9.2) */ - usleep_range(100, 200); - - /* Deassert PERST# */ - rmw_set(PORT_PERST_OFF, port->base + pcie->hw->port_perst); - gpiod_set_value_cansleep(reset, 0); - - /* Wait for 100ms after PERST# deassertion (PCIe r5.0, 6.6.1) */ - msleep(100); - - ret = readl_relaxed_poll_timeout(port->base + PORT_STATUS, stat, - stat & PORT_STATUS_READY, 100, 250000); - if (ret < 0) { - dev_err(pcie->dev, "port %pOF ready wait timeout\n", np); - return ret; + /* link might be already brought up by u-boot, skip setup then */ + link_stat = readl_relaxed(port->base + PORT_LINKSTS); + if (!(link_stat & PORT_LINKSTS_UP)) { + ret = apple_pcie_setup_link(pcie, port, np); + if (ret) + return ret; } if (pcie->hw->port_refclk) @@ -658,10 +727,21 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, ret = apple_pcie_port_register_irqs(port); WARN_ON(ret); - writel_relaxed(PORT_LTSSMCTL_START, port->base + PORT_LTSSMCTL); + link_stat = readl_relaxed(port->base + PORT_LINKSTS); + if (!(link_stat & PORT_LINKSTS_UP)) { + unsigned long timeout, left; + /* start link training */ + writel_relaxed(PORT_LTSSMCTL_START, port->base + PORT_LTSSMCTL); - if (!wait_for_completion_timeout(&pcie->event, HZ / 10)) - dev_warn(pcie->dev, "%pOF link didn't come up\n", np); + timeout = link_up_timeout * HZ / 1000; + left = wait_for_completion_timeout(&pcie->event, timeout); + if (!left) + dev_warn(pcie->dev, "%pOF link didn't come up\n", np); + else + dev_info(pcie->dev, "%pOF link up after %ldms\n", np, + (timeout - left) * 1000 / HZ); + + } return 0; } @@ -845,13 +925,47 @@ static const struct pci_ecam_ops apple_pcie_cfg_ecam_ops = { } }; +static int apple_pcie_probe_port(struct device_node *np) +{ + struct gpio_desc *gd; + + /* check whether the GPPIO pin exists but leave it as is */ + gd = fwnode_gpiod_get_index(of_fwnode_handle(np), "reset", 0, + GPIOD_ASIS, "PERST#"); + if (IS_ERR(gd)) + return PTR_ERR(gd); + + gpiod_put(gd); + + gd = fwnode_gpiod_get_index(of_fwnode_handle(np), "pwren", 0, + GPIOD_ASIS, "PWREN"); + if (IS_ERR(gd)) { + if (PTR_ERR(gd) != -ENOENT) + return PTR_ERR(gd); + } else { + gpiod_put(gd); + } + + return 0; +} + static int apple_pcie_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct pci_host_bridge *bridge; + struct device_node *of_port; struct apple_pcie *pcie; int ret; + /* Check for probe dependencies for all ports first */ + for_each_available_child_of_node(dev->of_node, of_port) { + ret = apple_pcie_probe_port(of_port); + if (ret) { + of_node_put(of_port); + return dev_err_probe(dev, ret, "Port %pOF probe fail\n", of_port); + } + } + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*pcie)); if (!bridge) return -ENOMEM; diff --git a/drivers/pci/controller/pcie-mediatek.c b/drivers/pci/controller/pcie-mediatek.c index 4b78b6528f9fd3..5defa5cc4c2bdc 100644 --- a/drivers/pci/controller/pcie-mediatek.c +++ b/drivers/pci/controller/pcie-mediatek.c @@ -585,8 +585,10 @@ static int mtk_pcie_init_irq_domain(struct mtk_pcie_port *port, if (IS_ENABLED(CONFIG_PCI_MSI)) { ret = mtk_pcie_allocate_msi_domains(port); - if (ret) + if (ret) { + irq_domain_remove(port->irq_domain); return ret; + } } return 0; diff --git a/drivers/pci/controller/pcie-rzg3s-host.c b/drivers/pci/controller/pcie-rzg3s-host.c index 83ec66a7082361..1bea8360f89444 100644 --- a/drivers/pci/controller/pcie-rzg3s-host.c +++ b/drivers/pci/controller/pcie-rzg3s-host.c @@ -439,28 +439,9 @@ static void __iomem *rzg3s_pcie_root_map_bus(struct pci_bus *bus, return host->pcie + where; } -/* Serialized by 'pci_lock' */ -static int rzg3s_pcie_root_write(struct pci_bus *bus, unsigned int devfn, - int where, int size, u32 val) -{ - struct rzg3s_pcie_host *host = bus->sysdata; - int ret; - - /* Enable access control to the CFGU */ - writel_relaxed(RZG3S_PCI_PERM_CFG_HWINIT_EN, - host->axi + RZG3S_PCI_PERM); - - ret = pci_generic_config_write(bus, devfn, where, size, val); - - /* Disable access control to the CFGU */ - writel_relaxed(0, host->axi + RZG3S_PCI_PERM); - - return ret; -} - static struct pci_ops rzg3s_pcie_root_ops = { .read = pci_generic_config_read, - .write = rzg3s_pcie_root_write, + .write = pci_generic_config_write, .map_bus = rzg3s_pcie_root_map_bus, }; @@ -1065,14 +1046,14 @@ static int rzg3s_pcie_config_init(struct rzg3s_pcie_host *host) writel_relaxed(0xffffffff, host->pcie + RZG3S_PCI_CFG_BARMSK00L); writel_relaxed(0xffffffff, host->pcie + RZG3S_PCI_CFG_BARMSK00U); + /* Disable access control to the CFGU */ + writel_relaxed(0, host->axi + RZG3S_PCI_PERM); + /* Update bus info */ writeb_relaxed(primary_bus, host->pcie + PCI_PRIMARY_BUS); writeb_relaxed(secondary_bus, host->pcie + PCI_SECONDARY_BUS); writeb_relaxed(subordinate_bus, host->pcie + PCI_SUBORDINATE_BUS); - /* Disable access control to the CFGU */ - writel_relaxed(0, host->axi + RZG3S_PCI_PERM); - return 0; } @@ -1162,7 +1143,8 @@ static int rzg3s_pcie_resets_prepare_and_get(struct rzg3s_pcie_host *host) static int rzg3s_pcie_host_parse_port(struct rzg3s_pcie_host *host) { - struct device_node *of_port = of_get_next_child(host->dev->of_node, NULL); + struct device_node *of_port __free(device_node) = + of_get_next_child(host->dev->of_node, NULL); struct rzg3s_pcie_port *port = &host->port; int ret; diff --git a/drivers/pci/controller/pcie-xilinx.c b/drivers/pci/controller/pcie-xilinx.c index 937ea6ae1ac486..4aa139abac16e8 100644 --- a/drivers/pci/controller/pcie-xilinx.c +++ b/drivers/pci/controller/pcie-xilinx.c @@ -302,9 +302,10 @@ static int xilinx_allocate_msi_domains(struct xilinx_pcie *pcie) return 0; } -static void xilinx_free_msi_domains(struct xilinx_pcie *pcie) +static void xilinx_free_irq_domains(struct xilinx_pcie *pcie) { irq_domain_remove(pcie->msi_domain); + irq_domain_remove(pcie->leg_domain); } /* INTx Functions */ @@ -480,8 +481,10 @@ static int xilinx_pcie_init_irq_domain(struct xilinx_pcie *pcie) phys_addr_t pa = ALIGN_DOWN(virt_to_phys(pcie), SZ_4K); ret = xilinx_allocate_msi_domains(pcie); - if (ret) + if (ret) { + irq_domain_remove(pcie->leg_domain); return ret; + } pcie_write(pcie, upper_32_bits(pa), XILINX_PCIE_REG_MSIBASE1); pcie_write(pcie, lower_32_bits(pa), XILINX_PCIE_REG_MSIBASE2); @@ -600,7 +603,7 @@ static int xilinx_pcie_probe(struct platform_device *pdev) err = pci_host_probe(bridge); if (err) - xilinx_free_msi_domains(pcie); + xilinx_free_irq_domains(pcie); return err; } diff --git a/drivers/pci/endpoint/functions/pci-epf-mhi.c b/drivers/pci/endpoint/functions/pci-epf-mhi.c index 6643a88c7a0ce3..27de533f05716a 100644 --- a/drivers/pci/endpoint/functions/pci-epf-mhi.c +++ b/drivers/pci/endpoint/functions/pci-epf-mhi.c @@ -686,7 +686,7 @@ static int pci_epf_mhi_dma_init(struct pci_epf_mhi *epf_mhi) goto err_release_tx; } - epf_mhi->dma_wq = alloc_workqueue("pci_epf_mhi_dma_wq", 0, 0); + epf_mhi->dma_wq = alloc_workqueue("pci_epf_mhi_dma_wq", WQ_PERCPU, 0); if (!epf_mhi->dma_wq) { ret = -ENOMEM; goto err_release_rx; diff --git a/drivers/pci/endpoint/functions/pci-epf-ntb.c b/drivers/pci/endpoint/functions/pci-epf-ntb.c index e01a98e74d2111..a3a588e522e715 100644 --- a/drivers/pci/endpoint/functions/pci-epf-ntb.c +++ b/drivers/pci/endpoint/functions/pci-epf-ntb.c @@ -2124,8 +2124,13 @@ static int __init epf_ntb_init(void) { int ret; - kpcintb_workqueue = alloc_workqueue("kpcintb", WQ_MEM_RECLAIM | - WQ_HIGHPRI, 0); + kpcintb_workqueue = alloc_workqueue("kpcintb", + WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_PERCPU, 0); + if (!kpcintb_workqueue) { + pr_err("Failed to allocate kpcintb workqueue\n"); + return -ENOMEM; + } + ret = pci_epf_register_driver(&epf_ntb_driver); if (ret) { destroy_workqueue(kpcintb_workqueue); diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c index debd235253c5ba..62804120cd79a7 100644 --- a/drivers/pci/endpoint/functions/pci-epf-test.c +++ b/drivers/pci/endpoint/functions/pci-epf-test.c @@ -1188,7 +1188,7 @@ static int __init pci_epf_test_init(void) int ret; kpcitest_workqueue = alloc_workqueue("kpcitest", - WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); + WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_PERCPU, 0); if (!kpcitest_workqueue) { pr_err("Failed to allocate the kpcitest work queue\n"); return -ENOMEM; diff --git a/drivers/pci/endpoint/functions/pci-epf-vntb.c b/drivers/pci/endpoint/functions/pci-epf-vntb.c index 3ecc5059f92b31..20a400e8343927 100644 --- a/drivers/pci/endpoint/functions/pci-epf-vntb.c +++ b/drivers/pci/endpoint/functions/pci-epf-vntb.c @@ -1651,8 +1651,13 @@ static int __init epf_ntb_init(void) { int ret; - kpcintb_workqueue = alloc_workqueue("kpcintb", WQ_MEM_RECLAIM | - WQ_HIGHPRI, 0); + kpcintb_workqueue = alloc_workqueue("kpcintb", + WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_PERCPU, 0); + if (!kpcintb_workqueue) { + pr_err("Failed to allocate kpcintb workqueue\n"); + return -ENOMEM; + } + ret = pci_epf_register_driver(&epf_ntb_driver); if (ret) { destroy_workqueue(kpcintb_workqueue); diff --git a/drivers/pci/endpoint/pci-ep-cfs.c b/drivers/pci/endpoint/pci-ep-cfs.c index ef50c82e647f4d..43feb6139fa36c 100644 --- a/drivers/pci/endpoint/pci-ep-cfs.c +++ b/drivers/pci/endpoint/pci-ep-cfs.c @@ -23,7 +23,6 @@ struct pci_epf_group { struct config_group group; struct config_group primary_epc_group; struct config_group secondary_epc_group; - struct delayed_work cfs_work; struct pci_epf *epf; int index; }; @@ -103,7 +102,7 @@ static struct config_group secondary_epc_group = &epf_group->secondary_epc_group; config_group_init_type_name(secondary_epc_group, "secondary", &pci_secondary_epc_type); - configfs_register_group(&epf_group->group, secondary_epc_group); + configfs_add_default_group(secondary_epc_group, &epf_group->group); return secondary_epc_group; } @@ -166,7 +165,7 @@ static struct config_group config_group_init_type_name(primary_epc_group, "primary", &pci_primary_epc_type); - configfs_register_group(&epf_group->group, primary_epc_group); + configfs_add_default_group(primary_epc_group, &epf_group->group); return primary_epc_group; } @@ -570,15 +569,13 @@ static void pci_ep_cfs_add_type_group(struct pci_epf_group *epf_group) return; } - configfs_register_group(&epf_group->group, group); + configfs_add_default_group(group, &epf_group->group); } -static void pci_epf_cfs_work(struct work_struct *work) +static void pci_epf_cfs_add_sub_groups(struct pci_epf_group *epf_group) { - struct pci_epf_group *epf_group; struct config_group *group; - epf_group = container_of(work, struct pci_epf_group, cfs_work.work); group = pci_ep_cfs_add_primary_group(epf_group); if (IS_ERR(group)) { pr_err("failed to create 'primary' EPC interface\n"); @@ -637,9 +634,7 @@ static struct config_group *pci_epf_make(struct config_group *group, kfree(epf_name); - INIT_DELAYED_WORK(&epf_group->cfs_work, pci_epf_cfs_work); - queue_delayed_work(system_wq, &epf_group->cfs_work, - msecs_to_jiffies(1)); + pci_epf_cfs_add_sub_groups(epf_group); return &epf_group->group; diff --git a/drivers/pci/endpoint/pci-epc-core.c b/drivers/pci/endpoint/pci-epc-core.c index ca7f19cc973a43..068155819c575a 100644 --- a/drivers/pci/endpoint/pci-epc-core.c +++ b/drivers/pci/endpoint/pci-epc-core.c @@ -596,6 +596,14 @@ int pci_epc_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, if (!epc_features) return -EINVAL; + if (epf_bar->num_submap && !epf_bar->submap) + return -EINVAL; + + if (epf_bar->num_submap && + !(epc_features->dynamic_inbound_mapping && + epc_features->subrange_mapping)) + return -EINVAL; + if (epc_features->bar[bar].type == BAR_RESIZABLE && (epf_bar->size < SZ_1M || (u64)epf_bar->size > (SZ_128G * 1024))) return -EINVAL; diff --git a/drivers/pci/hotplug/pnv_php.c b/drivers/pci/hotplug/pnv_php.c index c5345bff9a5538..35f1758126c68b 100644 --- a/drivers/pci/hotplug/pnv_php.c +++ b/drivers/pci/hotplug/pnv_php.c @@ -802,7 +802,7 @@ static struct pnv_php_slot *pnv_php_alloc_slot(struct device_node *dn) } /* Allocate workqueue for this slot's interrupt handling */ - php_slot->wq = alloc_workqueue("pciehp-%s", 0, 0, php_slot->name); + php_slot->wq = alloc_workqueue("pciehp-%s", WQ_PERCPU, 0, php_slot->name); if (!php_slot->wq) { SLOT_WARN(php_slot, "Cannot alloc workqueue\n"); kfree(php_slot->name); diff --git a/drivers/pci/hotplug/shpchp_core.c b/drivers/pci/hotplug/shpchp_core.c index 0c341453afc605..56308515ecbaf3 100644 --- a/drivers/pci/hotplug/shpchp_core.c +++ b/drivers/pci/hotplug/shpchp_core.c @@ -80,7 +80,8 @@ static int init_slots(struct controller *ctrl) slot->device = ctrl->slot_device_offset + i; slot->number = ctrl->first_slot + (ctrl->slot_num_inc * i); - slot->wq = alloc_workqueue("shpchp-%d", 0, 0, slot->number); + slot->wq = alloc_workqueue("shpchp-%d", WQ_PERCPU, 0, + slot->number); if (!slot->wq) { retval = -ENOMEM; goto error_slot; diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c index 4a2fc7ab42c349..79a414fd6623bd 100644 --- a/drivers/pci/p2pdma.c +++ b/drivers/pci/p2pdma.c @@ -147,11 +147,19 @@ static int p2pmem_alloc_mmap(struct file *filp, struct kobject *kobj, * we have just allocated the page no one else should be * using it. */ - VM_WARN_ON_ONCE_PAGE(!page_ref_count(page), page); + VM_WARN_ON_ONCE_PAGE(page_ref_count(page), page); set_page_count(page, 1); ret = vm_insert_page(vma, vaddr, page); if (ret) { gen_pool_free(p2pdma->pool, (uintptr_t)kaddr, len); + + /* + * Reset the page count. We don't use put_page() + * because we don't want to trigger the + * p2pdma_folio_free() path. + */ + set_page_count(page, 0); + percpu_ref_put(ref); return ret; } percpu_ref_get(ref); diff --git a/drivers/pci/pci-acpi.c b/drivers/pci/pci-acpi.c index 9369377725fa03..0162acfb57896b 100644 --- a/drivers/pci/pci-acpi.c +++ b/drivers/pci/pci-acpi.c @@ -271,21 +271,6 @@ static acpi_status decode_type1_hpx_record(union acpi_object *record, return AE_OK; } -static bool pcie_root_rcb_set(struct pci_dev *dev) -{ - struct pci_dev *rp = pcie_find_root_port(dev); - u16 lnkctl; - - if (!rp) - return false; - - pcie_capability_read_word(rp, PCI_EXP_LNKCTL, &lnkctl); - if (lnkctl & PCI_EXP_LNKCTL_RCB) - return true; - - return false; -} - /* _HPX PCI Express Setting Record (Type 2) */ struct hpx_type2 { u32 revision; @@ -311,6 +296,7 @@ static void program_hpx_type2(struct pci_dev *dev, struct hpx_type2 *hpx) { int pos; u32 reg32; + const struct pci_host_bridge *host; if (!hpx) return; @@ -318,6 +304,15 @@ static void program_hpx_type2(struct pci_dev *dev, struct hpx_type2 *hpx) if (!pci_is_pcie(dev)) return; + host = pci_find_host_bridge(dev->bus); + + /* + * Only do the _HPX Type 2 programming if OS owns PCIe native + * hotplug but not AER. + */ + if (!host->native_pcie_hotplug || host->native_aer) + return; + if (hpx->revision > 1) { pci_warn(dev, "PCIe settings rev %d not supported\n", hpx->revision); @@ -325,33 +320,27 @@ static void program_hpx_type2(struct pci_dev *dev, struct hpx_type2 *hpx) } /* - * Don't allow _HPX to change MPS or MRRS settings. We manage - * those to make sure they're consistent with the rest of the - * platform. + * We only allow _HPX to program DEVCTL bits related to AER, namely + * PCI_EXP_DEVCTL_CERE, PCI_EXP_DEVCTL_NFERE, PCI_EXP_DEVCTL_FERE, + * and PCI_EXP_DEVCTL_URRE. + * + * The rest of DEVCTL is managed by the OS to make sure it's + * consistent with the rest of the platform. */ - hpx->pci_exp_devctl_and |= PCI_EXP_DEVCTL_PAYLOAD | - PCI_EXP_DEVCTL_READRQ; - hpx->pci_exp_devctl_or &= ~(PCI_EXP_DEVCTL_PAYLOAD | - PCI_EXP_DEVCTL_READRQ); + hpx->pci_exp_devctl_and |= ~PCI_EXP_AER_FLAGS; + hpx->pci_exp_devctl_or &= PCI_EXP_AER_FLAGS; /* Initialize Device Control Register */ pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL, ~hpx->pci_exp_devctl_and, hpx->pci_exp_devctl_or); - /* Initialize Link Control Register */ + /* Log if _HPX attempts to modify Link Control Register */ if (pcie_cap_has_lnkctl(dev)) { - - /* - * If the Root Port supports Read Completion Boundary of - * 128, set RCB to 128. Otherwise, clear it. - */ - hpx->pci_exp_lnkctl_and |= PCI_EXP_LNKCTL_RCB; - hpx->pci_exp_lnkctl_or &= ~PCI_EXP_LNKCTL_RCB; - if (pcie_root_rcb_set(dev)) - hpx->pci_exp_lnkctl_or |= PCI_EXP_LNKCTL_RCB; - - pcie_capability_clear_and_set_word(dev, PCI_EXP_LNKCTL, - ~hpx->pci_exp_lnkctl_and, hpx->pci_exp_lnkctl_or); + if (hpx->pci_exp_lnkctl_and != 0xffff || + hpx->pci_exp_lnkctl_or != 0) + pci_info(dev, "_HPX attempts Link Control setting (AND %#06x OR %#06x)\n", + hpx->pci_exp_lnkctl_and, + hpx->pci_exp_lnkctl_or); } /* Find Advanced Error Reporting Enhanced Capability */ diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 13dbb405dc31f8..a05978f5cf2c77 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -426,7 +426,7 @@ static int pci_dev_str_match(struct pci_dev *dev, const char *p, static u8 __pci_find_next_cap(struct pci_bus *bus, unsigned int devfn, u8 pos, int cap) { - return PCI_FIND_NEXT_CAP(pci_bus_read_config, pos, cap, bus, devfn); + return PCI_FIND_NEXT_CAP(pci_bus_read_config, pos, cap, NULL, bus, devfn); } u8 pci_find_next_capability(struct pci_dev *dev, u8 pos, int cap) @@ -531,7 +531,7 @@ u16 pci_find_next_ext_capability(struct pci_dev *dev, u16 start, int cap) return 0; return PCI_FIND_NEXT_EXT_CAP(pci_bus_read_config, start, cap, - dev->bus, dev->devfn); + NULL, dev->bus, dev->devfn); } EXPORT_SYMBOL_GPL(pci_find_next_ext_capability); @@ -600,7 +600,7 @@ static u8 __pci_find_next_ht_cap(struct pci_dev *dev, u8 pos, int ht_cap) mask = HT_5BIT_CAP_MASK; pos = PCI_FIND_NEXT_CAP(pci_bus_read_config, pos, - PCI_CAP_ID_HT, dev->bus, dev->devfn); + PCI_CAP_ID_HT, NULL, dev->bus, dev->devfn); while (pos) { rc = pci_read_config_byte(dev, pos + 3, &cap); if (rc != PCIBIOS_SUCCESSFUL) @@ -611,7 +611,7 @@ static u8 __pci_find_next_ht_cap(struct pci_dev *dev, u8 pos, int ht_cap) pos = PCI_FIND_NEXT_CAP(pci_bus_read_config, pos + PCI_CAP_LIST_NEXT, - PCI_CAP_ID_HT, dev->bus, + PCI_CAP_ID_HT, NULL, dev->bus, dev->devfn); } @@ -1488,6 +1488,9 @@ static int pci_set_low_power_state(struct pci_dev *dev, pci_power_t state, bool || (state == PCI_D2 && !dev->d2_support)) return -EIO; + if (dev->current_state == state) + return 0; + pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); if (PCI_POSSIBLE_ERROR(pmcsr)) { pci_err(dev, "Unable to change power state from %s to %s, device inaccessible\n", @@ -6591,7 +6594,7 @@ static void of_pci_bus_release_domain_nr(struct device *parent, int domain_nr) return; /* Release domain from IDA where it was allocated. */ - if (of_get_pci_domain_nr(parent->of_node) == domain_nr) + if (parent && of_get_pci_domain_nr(parent->of_node) == domain_nr) ida_free(&pci_domain_nr_static_ida, domain_nr); else ida_free(&pci_domain_nr_dynamic_ida, domain_nr); diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 0e67014aa0013a..36f32b8af6ab3d 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -88,6 +88,9 @@ struct pcie_tlp_log; #define PCI_BUS_BRIDGE_MEM_WINDOW 1 #define PCI_BUS_BRIDGE_PREF_MEM_WINDOW 2 +#define PCI_EXP_AER_FLAGS (PCI_EXP_DEVCTL_CERE | PCI_EXP_DEVCTL_NFERE | \ + PCI_EXP_DEVCTL_FERE | PCI_EXP_DEVCTL_URRE) + extern const unsigned char pcie_link_speed[]; extern bool pci_early_dump; @@ -103,17 +106,21 @@ bool pcie_cap_has_rtctl(const struct pci_dev *dev); * @read_cfg: Function pointer for reading PCI config space * @start: Starting position to begin search * @cap: Capability ID to find + * @prev_ptr: Pointer to store position of preceding capability (optional) * @args: Arguments to pass to read_cfg function * - * Search the capability list in PCI config space to find @cap. + * Search the capability list in PCI config space to find @cap. If + * found, update *prev_ptr with the position of the preceding capability + * (if prev_ptr != NULL) * Implements TTL (time-to-live) protection against infinite loops. * * Return: Position of the capability if found, 0 otherwise. */ -#define PCI_FIND_NEXT_CAP(read_cfg, start, cap, args...) \ +#define PCI_FIND_NEXT_CAP(read_cfg, start, cap, prev_ptr, args...) \ ({ \ int __ttl = PCI_FIND_CAP_TTL; \ - u8 __id, __found_pos = 0; \ + u8 __id, __found_pos = 0; \ + u8 __prev_pos = (start); \ u8 __pos = (start); \ u16 __ent; \ \ @@ -132,9 +139,12 @@ bool pcie_cap_has_rtctl(const struct pci_dev *dev); \ if (__id == (cap)) { \ __found_pos = __pos; \ + if (prev_ptr != NULL) \ + *(u8 *)prev_ptr = __prev_pos; \ break; \ } \ \ + __prev_pos = __pos; \ __pos = FIELD_GET(PCI_CAP_LIST_NEXT_MASK, __ent); \ } \ __found_pos; \ @@ -146,21 +156,26 @@ bool pcie_cap_has_rtctl(const struct pci_dev *dev); * @read_cfg: Function pointer for reading PCI config space * @start: Starting position to begin search (0 for initial search) * @cap: Extended capability ID to find + * @prev_ptr: Pointer to store position of preceding capability (optional) * @args: Arguments to pass to read_cfg function * * Search the extended capability list in PCI config space to find @cap. + * If found, update *prev_ptr with the position of the preceding capability + * (if prev_ptr != NULL) * Implements TTL protection against infinite loops using a calculated * maximum search count. * * Return: Position of the capability if found, 0 otherwise. */ -#define PCI_FIND_NEXT_EXT_CAP(read_cfg, start, cap, args...) \ +#define PCI_FIND_NEXT_EXT_CAP(read_cfg, start, cap, prev_ptr, args...) \ ({ \ u16 __pos = (start) ?: PCI_CFG_SPACE_SIZE; \ u16 __found_pos = 0; \ + u16 __prev_pos; \ int __ttl, __ret; \ u32 __header; \ \ + __prev_pos = __pos; \ __ttl = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8; \ while (__ttl-- > 0 && __pos >= PCI_CFG_SPACE_SIZE) { \ __ret = read_cfg##_dword(args, __pos, &__header); \ @@ -172,9 +187,12 @@ bool pcie_cap_has_rtctl(const struct pci_dev *dev); \ if (PCI_EXT_CAP_ID(__header) == (cap) && __pos != start) {\ __found_pos = __pos; \ + if (prev_ptr != NULL) \ + *(u16 *)prev_ptr = __prev_pos; \ break; \ } \ \ + __prev_pos = __pos; \ __pos = PCI_EXT_CAP_NEXT(__header); \ } \ __found_pos; \ diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c index e0bcaa896803c9..9472d86cef5524 100644 --- a/drivers/pci/pcie/aer.c +++ b/drivers/pci/pcie/aer.c @@ -239,9 +239,6 @@ void pcie_ecrc_get_policy(char *str) } #endif /* CONFIG_PCIE_ECRC */ -#define PCI_EXP_AER_FLAGS (PCI_EXP_DEVCTL_CERE | PCI_EXP_DEVCTL_NFERE | \ - PCI_EXP_DEVCTL_FERE | PCI_EXP_DEVCTL_URRE) - int pcie_aer_is_native(struct pci_dev *dev) { struct pci_host_bridge *host = pci_find_host_bridge(dev->bus); diff --git a/drivers/pci/pcie/portdrv.c b/drivers/pci/pcie/portdrv.c index 38a41ccf79b9a3..a0991da4821363 100644 --- a/drivers/pci/pcie/portdrv.c +++ b/drivers/pci/pcie/portdrv.c @@ -557,10 +557,10 @@ static int pcie_port_remove_service(struct device *dev) pciedev = to_pcie_device(dev); driver = to_service_driver(dev->driver); - if (driver && driver->remove) { + if (driver && driver->remove) driver->remove(pciedev); - put_device(dev); - } + + put_device(dev); return 0; } diff --git a/drivers/pci/pcie/ptm.c b/drivers/pci/pcie/ptm.c index ed0f9691e7d16e..c7c61869bc9cf8 100644 --- a/drivers/pci/pcie/ptm.c +++ b/drivers/pci/pcie/ptm.c @@ -542,8 +542,10 @@ struct pci_ptm_debugfs *pcie_ptm_create_debugfs(struct device *dev, void *pdata, return NULL; dirname = devm_kasprintf(dev, GFP_KERNEL, "pcie_ptm_%s", dev_name(dev)); - if (!dirname) + if (!dirname) { + kfree(ptm_debugfs); return NULL; + } ptm_debugfs->debugfs = debugfs_create_dir(dirname, NULL); ptm_debugfs->pdata = pdata; @@ -574,6 +576,7 @@ void pcie_ptm_destroy_debugfs(struct pci_ptm_debugfs *ptm_debugfs) mutex_destroy(&ptm_debugfs->lock); debugfs_remove_recursive(ptm_debugfs->debugfs); + kfree(ptm_debugfs); } EXPORT_SYMBOL_GPL(pcie_ptm_destroy_debugfs); #endif diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index 41183aed8f5d94..c791bca2891f63 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -2270,7 +2270,8 @@ int pci_configure_extended_tags(struct pci_dev *dev, void *ign) u16 ctl; int ret; - if (!pci_is_pcie(dev)) + /* PCI_EXP_DEVCTL_EXT_TAG is RsvdP in VFs */ + if (!pci_is_pcie(dev) || dev->is_virtfn) return 0; ret = pcie_capability_read_dword(dev, PCI_EXP_DEVCAP, &cap); @@ -2410,6 +2411,37 @@ static void pci_configure_serr(struct pci_dev *dev) } } +static void pci_configure_rcb(struct pci_dev *dev) +{ + struct pci_dev *rp; + u16 rp_lnkctl; + + /* + * Per PCIe r7.0, sec 7.5.3.7, RCB is only meaningful in Root Ports + * (where it is read-only), Endpoints, and Bridges. It may only be + * set for Endpoints and Bridges if it is set in the Root Port. For + * Endpoints, it is 'RsvdP' for Virtual Functions. + */ + if (!pci_is_pcie(dev) || + pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT || + pci_pcie_type(dev) == PCI_EXP_TYPE_UPSTREAM || + pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM || + pci_pcie_type(dev) == PCI_EXP_TYPE_RC_EC || + dev->is_virtfn) + return; + + /* Root Port often not visible to virtualized guests */ + rp = pcie_find_root_port(dev); + if (!rp) + return; + + pcie_capability_read_word(rp, PCI_EXP_LNKCTL, &rp_lnkctl); + pcie_capability_clear_and_set_word(dev, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_RCB, + (rp_lnkctl & PCI_EXP_LNKCTL_RCB) ? + PCI_EXP_LNKCTL_RCB : 0); +} + static void pci_configure_device(struct pci_dev *dev) { pci_configure_mps(dev); @@ -2419,6 +2451,7 @@ static void pci_configure_device(struct pci_dev *dev) pci_configure_aspm_l1ss(dev); pci_configure_eetlp_prefix(dev); pci_configure_serr(dev); + pci_configure_rcb(dev); pci_acpi_program_hp_params(dev); } diff --git a/drivers/pci/pwrctrl/pci-pwrctrl-tc9563.c b/drivers/pci/pwrctrl/pci-pwrctrl-tc9563.c index ec423432ac6550..0a63add84d095b 100644 --- a/drivers/pci/pwrctrl/pci-pwrctrl-tc9563.c +++ b/drivers/pci/pwrctrl/pci-pwrctrl-tc9563.c @@ -533,7 +533,7 @@ static int tc9563_pwrctrl_probe(struct platform_device *pdev) ctx->client = i2c_new_dummy_device(ctx->adapter, addr); if (IS_ERR(ctx->client)) { dev_err(dev, "Failed to create I2C client\n"); - i2c_put_adapter(ctx->adapter); + put_device(&ctx->adapter->dev); return PTR_ERR(ctx->client); } @@ -613,7 +613,7 @@ static int tc9563_pwrctrl_probe(struct platform_device *pdev) tc9563_pwrctrl_power_off(ctx); remove_i2c: i2c_unregister_device(ctx->client); - i2c_put_adapter(ctx->adapter); + put_device(&ctx->adapter->dev); return ret; } @@ -623,7 +623,7 @@ static void tc9563_pwrctrl_remove(struct platform_device *pdev) tc9563_pwrctrl_power_off(ctx); i2c_unregister_device(ctx->client); - i2c_put_adapter(ctx->adapter); + put_device(&ctx->adapter->dev); } static const struct of_device_id tc9563_pwrctrl_of_match[] = { diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index 280cd50d693bd2..6df78efd7f6dc2 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -5581,6 +5581,7 @@ static void quirk_no_ext_tags(struct pci_dev *pdev) pci_walk_bus(bridge->bus, pci_configure_extended_tags, NULL); } DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_3WARE, 0x1004, quirk_no_ext_tags); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_3WARE, 0x1005, quirk_no_ext_tags); DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SERVERWORKS, 0x0132, quirk_no_ext_tags); DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SERVERWORKS, 0x0140, quirk_no_ext_tags); DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SERVERWORKS, 0x0141, quirk_no_ext_tags); @@ -6188,6 +6189,10 @@ DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_PERICOM, 0x2303, pci_fixup_pericom_acs_store_forward); DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_PERICOM, 0x2303, pci_fixup_pericom_acs_store_forward); +DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_PERICOM, 0xb404, + pci_fixup_pericom_acs_store_forward); +DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_PERICOM, 0xb404, + pci_fixup_pericom_acs_store_forward); static void nvidia_ion_ahci_fixup(struct pci_dev *pdev) { diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c index a61d38777cdc4a..09a28cfcd5b888 100644 --- a/drivers/pci/setup-bus.c +++ b/drivers/pci/setup-bus.c @@ -225,14 +225,21 @@ static struct resource *pbus_select_window_for_type(struct pci_bus *bus, switch (iores_type) { case IORESOURCE_IO: - return pci_bus_resource_n(bus, PCI_BUS_BRIDGE_IO_WINDOW); + win = pci_bus_resource_n(bus, PCI_BUS_BRIDGE_IO_WINDOW); + if (win && (win->flags & IORESOURCE_IO)) + return win; + return NULL; case IORESOURCE_MEM: mmio = pci_bus_resource_n(bus, PCI_BUS_BRIDGE_MEM_WINDOW); mmio_pref = pci_bus_resource_n(bus, PCI_BUS_BRIDGE_PREF_MEM_WINDOW); - if (!(type & IORESOURCE_PREFETCH) || - !(mmio_pref->flags & IORESOURCE_MEM)) + if (mmio && !(mmio->flags & IORESOURCE_MEM)) + mmio = NULL; + if (mmio_pref && !(mmio_pref->flags & IORESOURCE_MEM)) + mmio_pref = NULL; + + if (!(type & IORESOURCE_PREFETCH) || !mmio_pref) return mmio; if ((type & IORESOURCE_MEM_64) || @@ -1070,16 +1077,13 @@ static resource_size_t calculate_memsize(resource_size_t size, resource_size_t min_size, resource_size_t add_size, resource_size_t children_add_size, - resource_size_t old_size, resource_size_t align) { if (size < min_size) size = min_size; - if (old_size == 1) - old_size = 0; size = max(size, add_size) + children_add_size; - return ALIGN(max(size, old_size), align); + return ALIGN(size, align); } resource_size_t __weak pcibios_window_alignment(struct pci_bus *bus, @@ -1227,66 +1231,43 @@ static inline resource_size_t calculate_mem_align(resource_size_t *aligns, return min_align; } -/** - * pbus_upstream_space_available - Check no upstream resource limits allocation - * @bus: The bus - * @res: The resource to help select the correct bridge window - * @size: The size required from the bridge window - * @align: Required alignment for the resource - * - * Check that @size can fit inside the upstream bridge resources that are - * already assigned. Select the upstream bridge window based on the type of - * @res. - * - * Return: %true if enough space is available on all assigned upstream +/* + * Calculate bridge window head alignment that leaves no gaps in between * resources. */ -static bool pbus_upstream_space_available(struct pci_bus *bus, - struct resource *res, - resource_size_t size, - resource_size_t align) -{ - struct resource_constraint constraint = { - .max = RESOURCE_SIZE_MAX, - .align = align, - }; - struct pci_bus *downstream = bus; - - while ((bus = bus->parent)) { - if (pci_is_root_bus(bus)) - break; +static resource_size_t calculate_head_align(resource_size_t *aligns, + int max_order) +{ + resource_size_t head_align = 1; + resource_size_t remainder = 0; + int order; - res = pbus_select_window(bus, res); - if (!res) - return false; - if (!res->parent) - continue; + /* Take the largest alignment as the starting point. */ + head_align <<= max_order + __ffs(SZ_1M); - if (resource_size(res) >= size) { - struct resource gap = {}; + for (order = max_order - 1; order >= 0; order--) { + resource_size_t align1 = 1; - if (find_resource_space(res, &gap, size, &constraint) == 0) { - gap.flags = res->flags; - pci_dbg(bus->self, - "Assigned bridge window %pR to %pR free space at %pR\n", - res, &bus->busn_res, &gap); - return true; - } - } + align1 <<= order + __ffs(SZ_1M); - if (bus->self) { - pci_info(bus->self, - "Assigned bridge window %pR to %pR cannot fit 0x%llx required for %s bridging to %pR\n", - res, &bus->busn_res, - (unsigned long long)size, - pci_name(downstream->self), - &downstream->busn_res); - } + /* + * Account smaller resources with alignment < max_order that + * could be used to fill head room if alignment less than + * max_order is used. + */ + remainder += aligns[order]; - return false; + /* + * Test if head fill is enough to satisfy the alignment of + * the larger resources after reducing the alignment. + */ + while ((head_align > align1) && (remainder >= head_align / 2)) { + head_align /= 2; + remainder -= head_align; + } } - return true; + return head_align; } /** @@ -1314,14 +1295,12 @@ static void pbus_size_mem(struct pci_bus *bus, unsigned long type, { struct pci_dev *dev; resource_size_t min_align, win_align, align, size, size0, size1 = 0; - resource_size_t aligns[28]; /* Alignments from 1MB to 128TB */ + resource_size_t aligns[28] = {}; /* Alignments from 1MB to 128TB */ int order, max_order; struct resource *b_res = pbus_select_window_for_type(bus, type); resource_size_t children_add_size = 0; resource_size_t children_add_align = 0; resource_size_t add_align = 0; - resource_size_t relaxed_align; - resource_size_t old_size; if (!b_res) return; @@ -1330,7 +1309,6 @@ static void pbus_size_mem(struct pci_bus *bus, unsigned long type, if (b_res->parent) return; - memset(aligns, 0, sizeof(aligns)); max_order = 0; size = 0; @@ -1375,12 +1353,8 @@ static void pbus_size_mem(struct pci_bus *bus, unsigned long type, continue; } size += max(r_size, align); - /* - * Exclude ranges with size > align from calculation of - * the alignment. - */ - if (r_size <= align) - aligns[order] += align; + + aligns[order] += align; if (order > max_order) max_order = order; @@ -1392,44 +1366,20 @@ static void pbus_size_mem(struct pci_bus *bus, unsigned long type, } } - old_size = resource_size(b_res); win_align = window_alignment(bus, b_res->flags); - min_align = calculate_mem_align(aligns, max_order); + min_align = calculate_head_align(aligns, max_order); min_align = max(min_align, win_align); - size0 = calculate_memsize(size, min_size, 0, 0, old_size, min_align); + size0 = calculate_memsize(size, min_size, 0, 0, win_align); if (size0) { resource_set_range(b_res, min_align, size0); b_res->flags &= ~IORESOURCE_DISABLED; } - if (bus->self && size0 && - !pbus_upstream_space_available(bus, b_res, size0, min_align)) { - relaxed_align = 1ULL << (max_order + __ffs(SZ_1M)); - relaxed_align = max(relaxed_align, win_align); - min_align = min(min_align, relaxed_align); - size0 = calculate_memsize(size, min_size, 0, 0, old_size, win_align); - resource_set_range(b_res, min_align, size0); - pci_info(bus->self, "bridge window %pR to %pR requires relaxed alignment rules\n", - b_res, &bus->busn_res); - } - if (realloc_head && (add_size > 0 || children_add_size > 0)) { add_align = max(min_align, add_align); size1 = calculate_memsize(size, min_size, add_size, children_add_size, - old_size, add_align); - - if (bus->self && size1 && - !pbus_upstream_space_available(bus, b_res, size1, add_align)) { - relaxed_align = 1ULL << (max_order + __ffs(SZ_1M)); - relaxed_align = max(relaxed_align, win_align); - min_align = min(min_align, relaxed_align); - size1 = calculate_memsize(size, min_size, add_size, children_add_size, - old_size, win_align); - pci_info(bus->self, - "bridge window %pR to %pR requires relaxed alignment rules\n", - b_res, &bus->busn_res); - } + win_align); } if (!size0 && !size1) { diff --git a/drivers/perf/arm_spe_pmu.c b/drivers/perf/arm_spe_pmu.c index 4801115f2b5405..5410fb7428d0e4 100644 --- a/drivers/perf/arm_spe_pmu.c +++ b/drivers/perf/arm_spe_pmu.c @@ -106,6 +106,8 @@ struct arm_spe_pmu { /* Keep track of our dynamic hotplug state */ static enum cpuhp_state arm_spe_pmu_online; +static void arm_spe_pmu_stop(struct perf_event *event, int flags); + enum arm_spe_pmu_buf_fault_action { SPE_PMU_BUF_FAULT_ACT_SPURIOUS, SPE_PMU_BUF_FAULT_ACT_FATAL, @@ -607,8 +609,8 @@ static u64 arm_spe_pmu_next_off(struct perf_output_handle *handle) return limit; } -static void arm_spe_perf_aux_output_begin(struct perf_output_handle *handle, - struct perf_event *event) +static int arm_spe_perf_aux_output_begin(struct perf_output_handle *handle, + struct perf_event *event) { u64 base, limit; struct arm_spe_pmu_buf *buf; @@ -622,7 +624,6 @@ static void arm_spe_perf_aux_output_begin(struct perf_output_handle *handle, /* Start a new aux session */ buf = perf_aux_output_begin(handle, event); if (!buf) { - event->hw.state |= PERF_HES_STOPPED; /* * We still need to clear the limit pointer, since the * profiler might only be disabled by virtue of a fault. @@ -642,6 +643,7 @@ static void arm_spe_perf_aux_output_begin(struct perf_output_handle *handle, out_write_limit: write_sysreg_s(limit, SYS_PMBLIMITR_EL1); + return (limit & PMBLIMITR_EL1_E) ? 0 : -EIO; } static void arm_spe_perf_aux_output_end(struct perf_output_handle *handle) @@ -781,7 +783,10 @@ static irqreturn_t arm_spe_pmu_irq_handler(int irq, void *dev) * when we get to it. */ if (!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED)) { - arm_spe_perf_aux_output_begin(handle, event); + if (arm_spe_perf_aux_output_begin(handle, event)) { + arm_spe_pmu_stop(event, PERF_EF_UPDATE); + break; + } isb(); } break; @@ -880,9 +885,10 @@ static void arm_spe_pmu_start(struct perf_event *event, int flags) struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle); hwc->state = 0; - arm_spe_perf_aux_output_begin(handle, event); - if (hwc->state) + if (arm_spe_perf_aux_output_begin(handle, event)) { + arm_spe_pmu_stop(event, 0); return; + } reg = arm_spe_event_to_pmsfcr(event); write_sysreg_s(reg, SYS_PMSFCR_EL1); diff --git a/drivers/phy/Kconfig b/drivers/phy/Kconfig index 678dd0452f0aa0..2fb5e3b313effc 100644 --- a/drivers/phy/Kconfig +++ b/drivers/phy/Kconfig @@ -103,6 +103,7 @@ config PHY_NXP_PTN3222 source "drivers/phy/allwinner/Kconfig" source "drivers/phy/amlogic/Kconfig" +source "drivers/phy/apple/Kconfig" source "drivers/phy/broadcom/Kconfig" source "drivers/phy/cadence/Kconfig" source "drivers/phy/freescale/Kconfig" diff --git a/drivers/phy/Makefile b/drivers/phy/Makefile index bfb27fb5a49428..eb84c3328d1062 100644 --- a/drivers/phy/Makefile +++ b/drivers/phy/Makefile @@ -15,6 +15,7 @@ obj-$(CONFIG_PHY_AIROHA_PCIE) += phy-airoha-pcie.o obj-$(CONFIG_PHY_NXP_PTN3222) += phy-nxp-ptn3222.o obj-y += allwinner/ \ amlogic/ \ + apple/ \ broadcom/ \ cadence/ \ freescale/ \ diff --git a/drivers/phy/apple/Kconfig b/drivers/phy/apple/Kconfig new file mode 100644 index 00000000000000..8409b67f6d1ecb --- /dev/null +++ b/drivers/phy/apple/Kconfig @@ -0,0 +1,23 @@ +# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +config PHY_APPLE_ATC + tristate "Apple Type-C PHY" + depends on (ARM64 && ARCH_APPLE) || (COMPILE_TEST && !GENERIC_ATOMIC64) + depends on TYPEC + select GENERIC_PHY + select APPLE_TUNABLE + help + Enable this to add support for the Apple Type-C PHY found in + Apple Silicon M-series SoCs. This PHY supports USB2, + USB3, USB4, Thunderbolt, and DisplayPort. + + If M is selected the module will be called 'phy-apple-atc'. + +config PHY_APPLE_DPTX + tristate "Apple DPTX PHY" + depends on ARCH_APPLE || COMPILE_TEST + select GENERIC_PHY + help + Enable this to add support for the Apple DPTX PHY found on Apple SoCs + such as the M2. + This driver provides support for DisplayPort and is used on the + Mac mini (M2 and M2 Pro, 2023). diff --git a/drivers/phy/apple/Makefile b/drivers/phy/apple/Makefile new file mode 100644 index 00000000000000..b9e7bf3e4ac170 --- /dev/null +++ b/drivers/phy/apple/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause + +obj-$(CONFIG_PHY_APPLE_ATC) += phy-apple-atc.o +phy-apple-atc-y := atc.o + +obj-$(CONFIG_PHY_APPLE_DPTX) += phy-apple-dptx.o +phy-apple-dptx-y += dptx.o diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c new file mode 100644 index 00000000000000..64d0c3dba1cbb9 --- /dev/null +++ b/drivers/phy/apple/atc.c @@ -0,0 +1,2297 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Apple Type-C PHY driver + * + * The Apple Type-C PHY (ATCPHY) is a combined PHY for USB 2.0, USB 3.x, + * USB4/Thunderbolt, and DisplayPort connectivity via Type-C ports found in + * Apple Silicon SoCs. + * + * The PHY handles muxing between these different protocols and also provides the + * reset controller for the attached DWC3 USB controller. + * + * No documentation for this PHY is available and its operation has been + * reverse engineered by observing the XNU's MMIO access using a thin hypervisor + * and correlating register access to XNU's very verbose debug output. Most + * register names comes from this debug output as well. + * + * In order to correctly setup the high speed lanes for the various modes + * calibration values copied from Apple's firmware by our bootloader m1n1 are + * required. Without these only USB2 operation is possible. + * + * Copyright (C) The Asahi Linux Contributors + * Author: Sven Peter + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define AUSPLL_FSM_CTRL 0x1014 + +#define AUSPLL_APB_CMD_OVERRIDE 0x2000 +#define AUSPLL_APB_CMD_OVERRIDE_REQ BIT(0) +#define AUSPLL_APB_CMD_OVERRIDE_ACK BIT(1) +#define AUSPLL_APB_CMD_OVERRIDE_UNK28 BIT(28) +#define AUSPLL_APB_CMD_OVERRIDE_CMD GENMASK(27, 3) + +#define AUSPLL_FREQ_DESC_A 0x2080 +#define AUSPLL_FD_FREQ_COUNT_TARGET GENMASK(9, 0) +#define AUSPLL_FD_FBDIVN_HALF BIT(10) +#define AUSPLL_FD_REV_DIVN GENMASK(13, 11) +#define AUSPLL_FD_KI_MAN GENMASK(17, 14) +#define AUSPLL_FD_KI_EXP GENMASK(21, 18) +#define AUSPLL_FD_KP_MAN GENMASK(25, 22) +#define AUSPLL_FD_KP_EXP GENMASK(29, 26) +#define AUSPLL_FD_KPKI_SCALE_HBW GENMASK(31, 30) + +#define AUSPLL_FREQ_DESC_B 0x2084 +#define AUSPLL_FD_FBDIVN_FRAC_DEN GENMASK(13, 0) +#define AUSPLL_FD_FBDIVN_FRAC_NUM GENMASK(27, 14) + +#define AUSPLL_FREQ_DESC_C 0x2088 +#define AUSPLL_FD_SDM_SSC_STEP GENMASK(7, 0) +#define AUSPLL_FD_SDM_SSC_EN BIT(8) +#define AUSPLL_FD_PCLK_DIV_SEL GENMASK(13, 9) +#define AUSPLL_FD_LFSDM_DIV GENMASK(15, 14) +#define AUSPLL_FD_LFCLK_CTRL GENMASK(19, 16) +#define AUSPLL_FD_VCLK_OP_DIVN GENMASK(21, 20) +#define AUSPLL_FD_VCLK_PRE_DIVN BIT(22) + +#define AUSPLL_DCO_EFUSE_SPARE 0x222c +#define AUSPLL_RODCO_ENCAP_EFUSE GENMASK(10, 9) +#define AUSPLL_RODCO_BIAS_ADJUST_EFUSE GENMASK(14, 12) + +#define AUSPLL_FRACN_CAN 0x22a4 +#define AUSPLL_DLL_START_CAPCODE GENMASK(18, 17) + +#define AUSPLL_CLKOUT_MASTER 0x2200 +#define AUSPLL_CLKOUT_MASTER_PCLK_DRVR_EN BIT(2) +#define AUSPLL_CLKOUT_MASTER_PCLK2_DRVR_EN BIT(4) +#define AUSPLL_CLKOUT_MASTER_REFBUFCLK_DRVR_EN BIT(6) + +#define AUSPLL_CLKOUT_DIV 0x2208 +#define AUSPLL_CLKOUT_PLLA_REFBUFCLK_DI GENMASK(20, 16) + +#define AUSPLL_BGR 0x2214 +#define AUSPLL_BGR_CTRL_AVAIL BIT(0) + +#define AUSPLL_CLKOUT_DTC_VREG 0x2220 +#define AUSPLL_DTC_VREG_ADJUST GENMASK(16, 14) +#define AUSPLL_DTC_VREG_BYPASS BIT(7) + +#define AUSPLL_FREQ_CFG 0x2224 +#define AUSPLL_FREQ_REFCLK GENMASK(1, 0) + +#define AUS_COMMON_SHIM_BLK_VREG 0x0a04 +#define AUS_VREG_TRIM GENMASK(6, 2) + +#define AUS_UNK_A20 0x0a20 +#define AUS_UNK_A20_TX_CAL_CODE GENMASK(23, 20) + +#define ACIOPHY_CMN_SHM_STS_REG0 0x0a74 +#define ACIOPHY_CMN_SHM_STS_REG0_CMD_READY BIT(0) + +#define CIO3PLL_CLK_CTRL 0x2a00 +#define CIO3PLL_CLK_PCLK_EN BIT(1) +#define CIO3PLL_CLK_REFCLK_EN BIT(5) + +#define CIO3PLL_DCO_NCTRL 0x2a38 +#define CIO3PLL_DCO_COARSEBIN_EFUSE0 GENMASK(6, 0) +#define CIO3PLL_DCO_COARSEBIN_EFUSE1 GENMASK(23, 17) + +#define CIO3PLL_FRACN_CAN 0x2aa4 +#define CIO3PLL_DLL_CAL_START_CAPCODE GENMASK(18, 17) + +#define CIO3PLL_DTC_VREG 0x2a20 +#define CIO3PLL_DTC_VREG_ADJUST GENMASK(16, 14) + +#define ACIOPHY_CFG0 0x08 +#define ACIOPHY_CFG0_COMMON_BIG_OV BIT(1) +#define ACIOPHY_CFG0_COMMON_SMALL_OV BIT(3) +#define ACIOPHY_CFG0_COMMON_CLAMP_OV BIT(5) +#define ACIOPHY_CFG0_RX_SMALL_OV GENMASK(9, 8) +#define ACIOPHY_CFG0_RX_BIG_OV GENMASK(13, 12) +#define ACIOPHY_CFG0_RX_CLAMP_OV GENMASK(17, 16) + +#define ACIOPHY_CROSSBAR 0x4c +#define ACIOPHY_CROSSBAR_PROTOCOL GENMASK(4, 0) +#define ACIOPHY_CROSSBAR_PROTOCOL_USB4 0x0 +#define ACIOPHY_CROSSBAR_PROTOCOL_USB4_SWAPPED 0x1 +#define ACIOPHY_CROSSBAR_PROTOCOL_USB3 0xa +#define ACIOPHY_CROSSBAR_PROTOCOL_USB3_SWAPPED 0xb +#define ACIOPHY_CROSSBAR_PROTOCOL_USB3_DP 0x10 +#define ACIOPHY_CROSSBAR_PROTOCOL_USB3_DP_SWAPPED 0x11 +#define ACIOPHY_CROSSBAR_PROTOCOL_DP 0x14 +#define ACIOPHY_CROSSBAR_DP_SINGLE_PMA GENMASK(16, 5) +#define ACIOPHY_CROSSBAR_DP_SINGLE_PMA_NONE 0x0000 +#define ACIOPHY_CROSSBAR_DP_SINGLE_PMA_UNK100 0x100 +#define ACIOPHY_CROSSBAR_DP_SINGLE_PMA_UNK008 0x008 +#define ACIOPHY_CROSSBAR_DP_BOTH_PMA BIT(17) + +#define ACIOPHY_LANE_MODE 0x48 +#define ACIOPHY_LANE_MODE_RX0 GENMASK(2, 0) +#define ACIOPHY_LANE_MODE_TX0 GENMASK(5, 3) +#define ACIOPHY_LANE_MODE_RX1 GENMASK(8, 6) +#define ACIOPHY_LANE_MODE_TX1 GENMASK(11, 9) + +enum atcphy_lane_mode { + ACIOPHY_LANE_MODE_USB4 = 0, + ACIOPHY_LANE_MODE_USB3 = 1, + ACIOPHY_LANE_MODE_DP = 2, + ACIOPHY_LANE_MODE_OFF = 3, +}; + +#define ACIOPHY_TOP_BIST_CIOPHY_CFG1 0x84 +#define ACIOPHY_TOP_BIST_CIOPHY_CFG1_CLK_EN BIT(27) +#define ACIOPHY_TOP_BIST_CIOPHY_CFG1_BIST_EN BIT(28) + +#define ACIOPHY_TOP_BIST_OV_CFG 0x8c +#define ACIOPHY_TOP_BIST_OV_CFG_LN0_RESET_N_OV BIT(13) +#define ACIOPHY_TOP_BIST_OV_CFG_LN0_PWR_DOWN_OV BIT(25) + +#define ACIOPHY_TOP_BIST_READ_CTRL 0x90 +#define ACIOPHY_TOP_BIST_READ_CTRL_LN0_PHY_STATUS_RE BIT(2) + +#define ACIOPHY_TOP_PHY_STAT 0x9c +#define ACIOPHY_TOP_PHY_STAT_LN0_UNK0 BIT(0) +#define ACIOPHY_TOP_PHY_STAT_LN0_UNK23 BIT(23) + +#define ACIOPHY_TOP_BIST_PHY_CFG0 0xa8 +#define ACIOPHY_TOP_BIST_PHY_CFG0_LN0_RESET_N BIT(0) + +#define ACIOPHY_TOP_BIST_PHY_CFG1 0xac +#define ACIOPHY_TOP_BIST_PHY_CFG1_LN0_PWR_DOWN GENMASK(13, 10) + +#define ACIOPHY_SLEEP_CTRL 0x1b0 +#define ACIOPHY_SLEEP_CTRL_TX_BIG_OV GENMASK(3, 2) +#define ACIOPHY_SLEEP_CTRL_TX_SMALL_OV GENMASK(7, 6) +#define ACIOPHY_SLEEP_CTRL_TX_CLAMP_OV GENMASK(11, 10) + +#define ACIOPHY_PLL_PCTL_FSM_CTRL1 0x1014 +#define ACIOPHY_PLL_APB_REQ_OV_SEL GENMASK(21, 13) +#define ACIOPHY_PLL_COMMON_CTRL 0x1028 +#define ACIOPHY_PLL_WAIT_FOR_CMN_READY_BEFORE_RESET_EXIT BIT(24) + +#define ATCPHY_POWER_CTRL 0x20000 +#define ATCPHY_POWER_STAT 0x20004 +#define ATCPHY_POWER_SLEEP_SMALL BIT(0) +#define ATCPHY_POWER_SLEEP_BIG BIT(1) +#define ATCPHY_POWER_CLAMP_EN BIT(2) +#define ATCPHY_POWER_APB_RESET_N BIT(3) +#define ATCPHY_POWER_PHY_RESET_N BIT(4) + +#define ATCPHY_MISC 0x20008 +#define ATCPHY_MISC_RESET_N BIT(0) +#define ATCPHY_MISC_LANE_SWAP BIT(2) + +#define ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0 0x7000 +#define DP_PMA_BYTECLK_RESET BIT(0) +#define DP_MAC_DIV20_CLK_SEL BIT(1) +#define DPTXPHY_PMA_LANE_RESET_N BIT(2) +#define DPTXPHY_PMA_LANE_RESET_N_OV BIT(3) +#define DPTX_PCLK1_SELECT GENMASK(6, 4) +#define DPTX_PCLK2_SELECT GENMASK(9, 7) +#define DPRX_PCLK_SELECT GENMASK(12, 10) +#define DPTX_PCLK1_ENABLE BIT(13) +#define DPTX_PCLK2_ENABLE BIT(14) +#define DPRX_PCLK_ENABLE BIT(15) + +#define ACIOPHY_DP_PCLK_STAT 0x7044 +#define ACIOPHY_AUSPLL_LOCK BIT(3) + +#define LN0_AUSPMA_RX_TOP 0x9000 +#define LN0_AUSPMA_RX_EQ 0xA000 +#define LN0_AUSPMA_RX_SHM 0xB000 +#define LN0_AUSPMA_TX_TOP 0xC000 +#define LN0_AUSPMA_TX_SHM 0xD000 + +#define LN1_AUSPMA_RX_TOP 0x10000 +#define LN1_AUSPMA_RX_EQ 0x11000 +#define LN1_AUSPMA_RX_SHM 0x12000 +#define LN1_AUSPMA_TX_TOP 0x13000 +#define LN1_AUSPMA_TX_SHM 0x14000 + +#define LN_AUSPMA_RX_TOP_PMAFSM 0x0010 +#define LN_AUSPMA_RX_TOP_PMAFSM_PCS_OV BIT(0) +#define LN_AUSPMA_RX_TOP_PMAFSM_PCS_REQ BIT(9) + +#define LN_AUSPMA_RX_TOP_TJ_CFG_RX_TXMODE 0x00F0 +#define LN_RX_TXMODE BIT(0) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_CTLE_CTRL0 0x00 +#define LN_TX_CLK_EN BIT(20) +#define LN_TX_CLK_EN_OV BIT(21) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_AFE_CTRL1 0x04 +#define LN_RX_DIV20_RESET_N_OV BIT(29) +#define LN_RX_DIV20_RESET_N BIT(30) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL2 0x08 +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL3 0x0C +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL4 0x10 +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL5 0x14 +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL6 0x18 +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL7 0x1C +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL8 0x20 +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL9 0x24 +#define LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL10 0x28 +#define LN_DTVREG_ADJUST GENMASK(31, 27) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL11 0x2C +#define LN_DTVREG_BIG_EN BIT(23) +#define LN_DTVREG_BIG_EN_OV BIT(24) +#define LN_DTVREG_SML_EN BIT(25) +#define LN_DTVREG_SML_EN_OV BIT(26) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL12 0x30 +#define LN_TX_BYTECLK_RESET_SYNC_CLR BIT(22) +#define LN_TX_BYTECLK_RESET_SYNC_CLR_OV BIT(23) +#define LN_TX_BYTECLK_RESET_SYNC_EN BIT(24) +#define LN_TX_BYTECLK_RESET_SYNC_EN_OV BIT(25) +#define LN_TX_HRCLK_SEL BIT(28) +#define LN_TX_HRCLK_SEL_OV BIT(29) +#define LN_TX_PBIAS_EN BIT(30) +#define LN_TX_PBIAS_EN_OV BIT(31) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL13 0x34 +#define LN_TX_PRE_EN BIT(0) +#define LN_TX_PRE_EN_OV BIT(1) +#define LN_TX_PST1_EN BIT(2) +#define LN_TX_PST1_EN_OV BIT(3) +#define LN_DTVREG_ADJUST_OV BIT(15) + +#define LN_AUSPMA_RX_SHM_TJ_UNK_CTRL14A 0x38 +#define LN_AUSPMA_RX_SHM_TJ_UNK_CTRL14B 0x3C +#define LN_AUSPMA_RX_SHM_TJ_UNK_CTRL15A 0x40 +#define LN_AUSPMA_RX_SHM_TJ_UNK_CTRL15B 0x44 +#define LN_AUSPMA_RX_SHM_TJ_RXA_SAVOS_CTRL16 0x48 +#define LN_RXTERM_EN BIT(21) +#define LN_RXTERM_EN_OV BIT(22) +#define LN_RXTERM_PULLUP_LEAK_EN BIT(23) +#define LN_RXTERM_PULLUP_LEAK_EN_OV BIT(24) +#define LN_TX_CAL_CODE GENMASK(29, 25) +#define LN_TX_CAL_CODE_OV BIT(30) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL17 0x4C +#define LN_TX_MARGIN GENMASK(19, 15) +#define LN_TX_MARGIN_OV BIT(20) +#define LN_TX_MARGIN_LSB BIT(21) +#define LN_TX_MARGIN_LSB_OV BIT(22) +#define LN_TX_MARGIN_P1 GENMASK(26, 23) +#define LN_TX_MARGIN_P1_OV BIT(27) +#define LN_TX_MARGIN_P1_LSB GENMASK(29, 28) +#define LN_TX_MARGIN_P1_LSB_OV BIT(30) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18 0x50 +#define LN_TX_P1_CODE GENMASK(3, 0) +#define LN_TX_P1_CODE_OV BIT(4) +#define LN_TX_P1_LSB_CODE GENMASK(6, 5) +#define LN_TX_P1_LSB_CODE_OV BIT(7) +#define LN_TX_MARGIN_PRE GENMASK(10, 8) +#define LN_TX_MARGIN_PRE_OV BIT(11) +#define LN_TX_MARGIN_PRE_LSB GENMASK(13, 12) +#define LN_TX_MARGIN_PRE_LSB_OV BIT(14) +#define LN_TX_PRE_LSB_CODE GENMASK(16, 15) +#define LN_TX_PRE_LSB_CODE_OV BIT(17) +#define LN_TX_PRE_CODE GENMASK(21, 18) +#define LN_TX_PRE_CODE_OV BIT(22) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19 0x54 +#define LN_TX_TEST_EN BIT(21) +#define LN_TX_TEST_EN_OV BIT(22) +#define LN_TX_EN BIT(23) +#define LN_TX_EN_OV BIT(24) +#define LN_TX_CLK_DLY_CTRL_TAPGEN GENMASK(27, 25) +#define LN_TX_CLK_DIV2_EN BIT(28) +#define LN_TX_CLK_DIV2_EN_OV BIT(29) +#define LN_TX_CLK_DIV2_RST BIT(30) +#define LN_TX_CLK_DIV2_RST_OV BIT(31) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL20 0x58 +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL21 0x5C +#define LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22 0x60 +#define LN_VREF_ADJUST_GRAY GENMASK(11, 7) +#define LN_VREF_ADJUST_GRAY_OV BIT(12) +#define LN_VREF_BIAS_SEL GENMASK(14, 13) +#define LN_VREF_BIAS_SEL_OV BIT(15) +#define LN_VREF_BOOST_EN BIT(16) +#define LN_VREF_BOOST_EN_OV BIT(17) +#define LN_VREF_EN BIT(18) +#define LN_VREF_EN_OV BIT(19) +#define LN_VREF_LPBKIN_DATA GENMASK(29, 28) +#define LN_VREF_TEST_RXLPBKDT_EN BIT(30) +#define LN_VREF_TEST_RXLPBKDT_EN_OV BIT(31) + +#define LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG0 0x00 +#define LN_BYTECLK_RESET_SYNC_EN_OV BIT(2) +#define LN_BYTECLK_RESET_SYNC_EN BIT(3) +#define LN_BYTECLK_RESET_SYNC_CLR_OV BIT(4) +#define LN_BYTECLK_RESET_SYNC_CLR BIT(5) +#define LN_BYTECLK_RESET_SYNC_SEL_OV BIT(6) + +#define LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG1 0x04 +#define LN_TXA_DIV2_EN_OV BIT(8) +#define LN_TXA_DIV2_EN BIT(9) +#define LN_TXA_DIV2_RESET_OV BIT(10) +#define LN_TXA_DIV2_RESET BIT(11) +#define LN_TXA_CLK_EN_OV BIT(22) +#define LN_TXA_CLK_EN BIT(23) + +#define LN_AUSPMA_TX_SHM_TXA_IMP_REG0 0x08 +#define LN_TXA_CAL_CTRL_OV BIT(0) +#define LN_TXA_CAL_CTRL GENMASK(18, 1) +#define LN_TXA_CAL_CTRL_BASE_OV BIT(19) +#define LN_TXA_CAL_CTRL_BASE GENMASK(23, 20) +#define LN_TXA_HIZ_OV BIT(29) +#define LN_TXA_HIZ BIT(30) + +#define LN_AUSPMA_TX_SHM_TXA_IMP_REG1 0x0C +#define LN_AUSPMA_TX_SHM_TXA_IMP_REG2 0x10 +#define LN_TXA_MARGIN_OV BIT(0) +#define LN_TXA_MARGIN GENMASK(18, 1) +#define LN_TXA_MARGIN_2R_OV BIT(19) +#define LN_TXA_MARGIN_2R BIT(20) + +#define LN_AUSPMA_TX_SHM_TXA_IMP_REG3 0x14 +#define LN_TXA_MARGIN_POST_OV BIT(0) +#define LN_TXA_MARGIN_POST GENMASK(10, 1) +#define LN_TXA_MARGIN_POST_2R_OV BIT(11) +#define LN_TXA_MARGIN_POST_2R BIT(12) +#define LN_TXA_MARGIN_POST_4R_OV BIT(13) +#define LN_TXA_MARGIN_POST_4R BIT(14) +#define LN_TXA_MARGIN_PRE_OV BIT(15) +#define LN_TXA_MARGIN_PRE GENMASK(21, 16) +#define LN_TXA_MARGIN_PRE_2R_OV BIT(22) +#define LN_TXA_MARGIN_PRE_2R BIT(23) +#define LN_TXA_MARGIN_PRE_4R_OV BIT(24) +#define LN_TXA_MARGIN_PRE_4R BIT(25) + +#define LN_AUSPMA_TX_SHM_TXA_UNK_REG0 0x18 +#define LN_AUSPMA_TX_SHM_TXA_UNK_REG1 0x1C +#define LN_AUSPMA_TX_SHM_TXA_UNK_REG2 0x20 + +#define LN_AUSPMA_TX_SHM_TXA_LDOCLK 0x24 +#define LN_LDOCLK_BYPASS_SML_OV BIT(8) +#define LN_LDOCLK_BYPASS_SML BIT(9) +#define LN_LDOCLK_BYPASS_BIG_OV BIT(10) +#define LN_LDOCLK_BYPASS_BIG BIT(11) +#define LN_LDOCLK_EN_SML_OV BIT(12) +#define LN_LDOCLK_EN_SML BIT(13) +#define LN_LDOCLK_EN_BIG_OV BIT(14) +#define LN_LDOCLK_EN_BIG BIT(15) + +/* LPDPTX registers */ +#define LPDPTX_AUX_CFG_BLK_AUX_CTRL 0x0000 +#define LPDPTX_BLK_AUX_CTRL_PWRDN BIT(4) +#define LPDPTX_BLK_AUX_RXOFFSET GENMASK(25, 22) + +#define LPDPTX_AUX_CFG_BLK_AUX_LDO_CTRL 0x0008 + +#define LPDPTX_AUX_CFG_BLK_AUX_MARGIN 0x000c +#define LPDPTX_MARGIN_RCAL_RXOFFSET_EN BIT(5) +#define LPDPTX_AUX_MARGIN_RCAL_TXSWING GENMASK(10, 6) + +#define LPDPTX_AUX_SHM_CFG_BLK_AUX_CTRL_REG0 0x0204 +#define LPDPTX_CFG_PMA_AUX_SEL_LF_DATA BIT(15) + +#define LPDPTX_AUX_SHM_CFG_BLK_AUX_CTRL_REG1 0x0208 +#define LPDPTX_CFG_PMA_PHYS_ADJ GENMASK(22, 20) +#define LPDPTX_CFG_PMA_PHYS_ADJ_OV BIT(19) + +#define LPDPTX_AUX_CONTROL 0x4000 +#define LPDPTX_AUX_PWN_DOWN 0x10 +#define LPDPTX_AUX_CLAMP_EN 0x04 +#define LPDPTX_SLEEP_B_BIG_IN 0x02 +#define LPDPTX_SLEEP_B_SML_IN 0x01 +#define LPDPTX_TXTERM_CODEMSB 0x400 +#define LPDPTX_TXTERM_CODE GENMASK(9, 5) + +/* pipehandler registers */ +#define PIPEHANDLER_OVERRIDE 0x00 +#define PIPEHANDLER_OVERRIDE_RXVALID BIT(0) +#define PIPEHANDLER_OVERRIDE_RXDETECT BIT(2) + +#define PIPEHANDLER_OVERRIDE_VALUES 0x04 +#define PIPEHANDLER_OVERRIDE_VAL_RXDETECT0 BIT(1) +#define PIPEHANDLER_OVERRIDE_VAL_RXDETECT1 BIT(2) +#define PIPEHANDLER_OVERRIDE_VAL_PHY_STATUS BIT(4) + +#define PIPEHANDLER_MUX_CTRL 0x0c +#define PIPEHANDLER_MUX_CTRL_CLK GENMASK(5, 3) +#define PIPEHANDLER_MUX_CTRL_DATA GENMASK(2, 0) +#define PIPEHANDLER_MUX_CTRL_CLK_OFF 0 +#define PIPEHANDLER_MUX_CTRL_CLK_USB3 1 +#define PIPEHANDLER_MUX_CTRL_CLK_USB4 2 +#define PIPEHANDLER_MUX_CTRL_CLK_DUMMY 4 + +#define PIPEHANDLER_MUX_CTRL_DATA_USB3 0 +#define PIPEHANDLER_MUX_CTRL_DATA_USB4 1 +#define PIPEHANDLER_MUX_CTRL_DATA_DUMMY 2 + +#define PIPEHANDLER_LOCK_REQ 0x10 +#define PIPEHANDLER_LOCK_ACK 0x14 +#define PIPEHANDLER_LOCK_EN BIT(0) + +#define PIPEHANDLER_AON_GEN 0x1C +#define PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN BIT(4) +#define PIPEHANDLER_AON_GEN_DWC3_RESET_N BIT(0) + +#define PIPEHANDLER_NONSELECTED_OVERRIDE 0x20 +#define PIPEHANDLER_NATIVE_RESET BIT(12) +#define PIPEHANDLER_DUMMY_PHY_EN BIT(15) +#define PIPEHANDLER_NATIVE_POWER_DOWN GENMASK(3, 0) + +#define PIPEHANDLER_LOCK_ACK_TIMEOUT_US 1000 + +/* USB2 PHY regs */ +#define USB2PHY_USBCTL 0x00 +#define USB2PHY_USBCTL_RUN 2 +#define USB2PHY_USBCTL_ISOLATION 4 + +#define USB2PHY_CTL 0x04 +#define USB2PHY_CTL_RESET BIT(0) +#define USB2PHY_CTL_PORT_RESET BIT(1) +#define USB2PHY_CTL_APB_RESET_N BIT(2) +#define USB2PHY_CTL_SIDDQ BIT(3) + +#define USB2PHY_SIG 0x08 +#define USB2PHY_SIG_VBUSDET_FORCE_VAL BIT(0) +#define USB2PHY_SIG_VBUSDET_FORCE_EN BIT(1) +#define USB2PHY_SIG_VBUSVLDEXT_FORCE_VAL BIT(2) +#define USB2PHY_SIG_VBUSVLDEXT_FORCE_EN BIT(3) +#define USB2PHY_SIG_HOST (7 << 12) + +#define USB2PHY_MISCTUNE 0x1c +#define USB2PHY_MISCTUNE_APBCLK_GATE_OFF BIT(29) +#define USB2PHY_MISCTUNE_REFCLK_GATE_OFF BIT(30) + +enum atcphy_dp_link_rate { + ATCPHY_DP_LINK_RATE_RBR, + ATCPHY_DP_LINK_RATE_HBR, + ATCPHY_DP_LINK_RATE_HBR2, + ATCPHY_DP_LINK_RATE_HBR3, +}; + +/** + * enum atcphy_pipehandler_state - States of the PIPE mux interface ("pipehandler") + * @ATCPHY_PIPEHANDLER_STATE_DUMMY: "Dummy PHY" (disables USB3, USB2 only) + * @ATCPHY_PIPEHANDLER_STATE_USB3: USB3 directly connected to the Type-C port + * @ATCPHY_PIPEHANDLER_STATE_USB4: USB3 tunneled via USB4/Thunderbolt + * + * DWC3's USB3 PIPE interface is connected to a multiplexer inside this PHY + * which can switch between a dummy state (which effectively disables any USB3 + * support and falls back to USB2 only operation via the separate ULPI interface), + * a USB3 state (for regular USB3 or USB3+DisplayPort operation) and a USB4 state + * (for USB3 tunneled via USB4/Thunderbolt). + */ +enum atcphy_pipehandler_state { + ATCPHY_PIPEHANDLER_STATE_DUMMY, + ATCPHY_PIPEHANDLER_STATE_USB3, + ATCPHY_PIPEHANDLER_STATE_USB4, +}; + +/** + * enum atcphy_mode - Operating modes of the PHY + * @APPLE_ATCPHY_MODE_OFF: all PHYs powered off + * @APPLE_ATCPHY_MODE_USB2: Nothing on the four SS lanes (i.e. USB2 only on D-/+) + * @APPLE_ATCPHY_MODE_USB3: USB3 on two lanes, nothing on the other two + * @APPLE_ATCPHY_MODE_USB3_DP: USB3 on two lanes and DisplayPort on the other two + * @APPLE_ATCPHY_MODE_TBT: Thunderbolt on all lanes + * @APPLE_ATCPHY_MODE_USB4: USB4 on all lanes + * @APPLE_ATCPHY_MODE_DP: DisplayPort on all lanes + */ +enum atcphy_mode { + APPLE_ATCPHY_MODE_OFF, + APPLE_ATCPHY_MODE_USB2, + APPLE_ATCPHY_MODE_USB3, + APPLE_ATCPHY_MODE_USB3_DP, + APPLE_ATCPHY_MODE_TBT, + APPLE_ATCPHY_MODE_USB4, + APPLE_ATCPHY_MODE_DP, +}; + +enum atcphy_lane { + APPLE_ATCPHY_LANE_0, + APPLE_ATCPHY_LANE_1, +}; + +/* Link rate configuration, field names are taken from XNU debug output or register names */ +struct atcphy_dp_link_rate_configuration { + u16 freqinit_count_target; + u16 fbdivn_frac_den; + u16 fbdivn_frac_num; + u16 pclk_div_sel; + u8 lfclk_ctrl; + u8 vclk_op_divn; + bool plla_clkout_vreg_bypass; + bool txa_ldoclk_bypass; + bool txa_div2_en; +}; + +/* Crossbar and lane configuration */ +struct atcphy_mode_configuration { + u32 crossbar; + u32 crossbar_dp_single_pma; + bool crossbar_dp_both_pma; + enum atcphy_lane_mode lane_mode[2]; + bool dp_lane[2]; + bool set_swap; +}; + +/** + * struct apple_atcphy - Apple Type-C PHY device struct + * @np: Device node pointer + * @dev: Device pointer + * @tunables: Firmware-provided tunable parameters + * @tunables.axi2af: AXI to AF interface tunables + * @tunables.common: Common tunables for all lanes + * @tunables.lane_usb3: USB3 lane-specific tunables + * @tunables.lane_dp: DisplayPort lane-specific tunables + * @tunables.lane_usb4: USB4 lane-specific tunables + * @mode: Current PHY operating mode + * @swap_lanes: True if lanes must be swapped due to cable orientation + * @dp_link_rate: DisplayPort link rate + * @pipehandler_up: True if the PIPE mux ("pipehandler") is set to USB3 or USB4 mode + * @regs: Memory-mapped registers + * @regs.core: Core registers + * @regs.axi2af: AXI to Apple Fabric interface registers + * @regs.usb2phy: USB2 PHY registers + * @regs.pipehandler: USB3 PIPE interface ("pipehandler") registers + * @regs.lpdptx: DisplayPort registers + * @res: Resources for memory-mapped registers, used to verify that tunables aren't out of bounds + * @res.core: Core register resource + * @res.axi2af: AXI to Apple Fabric interface resource + * @phys: PHY instances + * @phys.usb2: USB2 PHY instance + * @phys.usb3: USB3 PHY instance + * @phys.dp: DisplayPort PHY instance + * @phy_provider: PHY provider instance + * @rcdev: Reset controller device + * @sw: Type-C switch instance + * @mux: Type-C mux instance + * @lock: Mutex for synchronizing register access across PHY, Type-C switch/mux and reset controller + */ +struct apple_atcphy { + struct device_node *np; + struct device *dev; + + struct { + struct apple_tunable *axi2af; + struct apple_tunable *common[2]; + struct apple_tunable *lane_usb3[2]; + struct apple_tunable *lane_dp[2]; + struct apple_tunable *lane_usb4[2]; + } tunables; + + enum atcphy_mode mode; + bool swap_lanes; + int dp_link_rate; + bool pipehandler_up; + + struct { + void __iomem *core; + void __iomem *axi2af; + void __iomem *usb2phy; + void __iomem *pipehandler; + void __iomem *lpdptx; + } regs; + + struct { + struct resource *core; + struct resource *axi2af; + } res; + + struct { + struct phy *usb2; + struct phy *usb3; + struct phy *dp; + } phys; + struct phy_provider *phy_provider; + + struct reset_controller_dev rcdev; + + struct typec_switch *sw; + struct typec_mux *mux; + + struct mutex lock; +}; + +static const struct { + const struct atcphy_mode_configuration normal; + const struct atcphy_mode_configuration swapped; + bool enable_dp_aux; + enum atcphy_pipehandler_state pipehandler_state; +} atcphy_modes[] = { + [APPLE_ATCPHY_MODE_OFF] = { + .normal = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB3, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_NONE, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_OFF, ACIOPHY_LANE_MODE_OFF}, + .dp_lane = {false, false}, + .set_swap = false, + }, + .swapped = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB3_SWAPPED, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_NONE, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_OFF, ACIOPHY_LANE_MODE_OFF}, + .dp_lane = {false, false}, + .set_swap = false, /* doesn't matter since the SS lanes are off */ + }, + .enable_dp_aux = false, + .pipehandler_state = ATCPHY_PIPEHANDLER_STATE_DUMMY, + }, + [APPLE_ATCPHY_MODE_USB2] = { + .normal = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB3, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_NONE, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_OFF, ACIOPHY_LANE_MODE_OFF}, + .dp_lane = {false, false}, + .set_swap = false, + }, + .swapped = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB3_SWAPPED, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_NONE, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_OFF, ACIOPHY_LANE_MODE_OFF}, + .dp_lane = {false, false}, + .set_swap = false, /* doesn't matter since the SS lanes are off */ + }, + .enable_dp_aux = false, + .pipehandler_state = ATCPHY_PIPEHANDLER_STATE_DUMMY, + }, + [APPLE_ATCPHY_MODE_USB3] = { + /* + * Setting up the lanes as DP/USB3 is intentional here, USB3/USB3 does not work + * and isn't required since this PHY does not support 20GBps mode anyway. + * The only difference to APPLE_ATCPHY_MODE_USB3_DP is that DP Aux is not enabled. + */ + .normal = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB3_DP, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_UNK008, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_USB3, ACIOPHY_LANE_MODE_DP}, + .dp_lane = {false, true}, + .set_swap = false, + }, + .swapped = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB3_DP_SWAPPED, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_UNK008, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_DP, ACIOPHY_LANE_MODE_USB3}, + .dp_lane = {true, false}, + .set_swap = true, + }, + .enable_dp_aux = false, + .pipehandler_state = ATCPHY_PIPEHANDLER_STATE_USB3, + }, + [APPLE_ATCPHY_MODE_USB3_DP] = { + .normal = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB3_DP, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_UNK008, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_USB3, ACIOPHY_LANE_MODE_DP}, + .dp_lane = {false, true}, + .set_swap = false, + }, + .swapped = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB3_DP_SWAPPED, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_UNK008, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_DP, ACIOPHY_LANE_MODE_USB3}, + .dp_lane = {true, false}, + .set_swap = true, + }, + .enable_dp_aux = true, + .pipehandler_state = ATCPHY_PIPEHANDLER_STATE_USB3, + }, + [APPLE_ATCPHY_MODE_TBT] = { + .normal = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB4, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_NONE, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_USB4, ACIOPHY_LANE_MODE_USB4}, + .dp_lane = {false, false}, + .set_swap = false, + }, + .swapped = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB4_SWAPPED, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_NONE, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_USB4, ACIOPHY_LANE_MODE_USB4}, + .dp_lane = {false, false}, + .set_swap = false, /* intentionally false */ + }, + .enable_dp_aux = false, + .pipehandler_state = ATCPHY_PIPEHANDLER_STATE_DUMMY, + }, + [APPLE_ATCPHY_MODE_USB4] = { + .normal = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB4, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_NONE, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_USB4, ACIOPHY_LANE_MODE_USB4}, + .dp_lane = {false, false}, + .set_swap = false, + }, + .swapped = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB4_SWAPPED, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_NONE, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_USB4, ACIOPHY_LANE_MODE_USB4}, + .dp_lane = {false, false}, + .set_swap = false, /* intentionally false */ + }, + .enable_dp_aux = false, + .pipehandler_state = ATCPHY_PIPEHANDLER_STATE_USB4, + }, + [APPLE_ATCPHY_MODE_DP] = { + .normal = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_DP, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_UNK100, + .crossbar_dp_both_pma = true, + .lane_mode = {ACIOPHY_LANE_MODE_DP, ACIOPHY_LANE_MODE_DP}, + .dp_lane = {true, true}, + .set_swap = false, + }, + .swapped = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_DP, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_UNK008, + .crossbar_dp_both_pma = false, /* intentionally false */ + .lane_mode = {ACIOPHY_LANE_MODE_DP, ACIOPHY_LANE_MODE_DP}, + .dp_lane = {true, true}, + .set_swap = false, /* intentionally false */ + }, + .enable_dp_aux = true, + .pipehandler_state = ATCPHY_PIPEHANDLER_STATE_DUMMY, + }, +}; + +static const struct atcphy_dp_link_rate_configuration dp_lr_config[] = { + [ATCPHY_DP_LINK_RATE_RBR] = { + .freqinit_count_target = 0x21c, + .fbdivn_frac_den = 0x0, + .fbdivn_frac_num = 0x0, + .pclk_div_sel = 0x13, + .lfclk_ctrl = 0x5, + .vclk_op_divn = 0x2, + .plla_clkout_vreg_bypass = true, + .txa_ldoclk_bypass = true, + .txa_div2_en = true, + }, + [ATCPHY_DP_LINK_RATE_HBR] = { + .freqinit_count_target = 0x1c2, + .fbdivn_frac_den = 0x3ffe, + .fbdivn_frac_num = 0x1fff, + .pclk_div_sel = 0x9, + .lfclk_ctrl = 0x5, + .vclk_op_divn = 0x2, + .plla_clkout_vreg_bypass = true, + .txa_ldoclk_bypass = true, + .txa_div2_en = false, + }, + [ATCPHY_DP_LINK_RATE_HBR2] = { + .freqinit_count_target = 0x1c2, + .fbdivn_frac_den = 0x3ffe, + .fbdivn_frac_num = 0x1fff, + .pclk_div_sel = 0x4, + .lfclk_ctrl = 0x5, + .vclk_op_divn = 0x0, + .plla_clkout_vreg_bypass = true, + .txa_ldoclk_bypass = true, + .txa_div2_en = false, + }, + [ATCPHY_DP_LINK_RATE_HBR3] = { + .freqinit_count_target = 0x2a3, + .fbdivn_frac_den = 0x3ffc, + .fbdivn_frac_num = 0x2ffd, + .pclk_div_sel = 0x4, + .lfclk_ctrl = 0x6, + .vclk_op_divn = 0x0, + .plla_clkout_vreg_bypass = false, + .txa_ldoclk_bypass = false, + .txa_div2_en = false, + }, +}; + +static inline void mask32(void __iomem *reg, u32 mask, u32 set) +{ + u32 value = readl(reg); + + value &= ~mask; + value |= set; + writel(value, reg); +} + +static inline void core_mask32(struct apple_atcphy *atcphy, u32 reg, u32 mask, u32 set) +{ + mask32(atcphy->regs.core + reg, mask, set); +} + +static inline void set32(void __iomem *reg, u32 set) +{ + mask32(reg, 0, set); +} + +static inline void core_set32(struct apple_atcphy *atcphy, u32 reg, u32 set) +{ + core_mask32(atcphy, reg, 0, set); +} + +static inline void clear32(void __iomem *reg, u32 clear) +{ + mask32(reg, clear, 0); +} + +static inline void core_clear32(struct apple_atcphy *atcphy, u32 reg, u32 clear) +{ + core_mask32(atcphy, reg, clear, 0); +} + +static const struct atcphy_mode_configuration *atcphy_get_mode_config(struct apple_atcphy *atcphy, + enum atcphy_mode mode) +{ + if (atcphy->swap_lanes) + return &atcphy_modes[mode].swapped; + else + return &atcphy_modes[mode].normal; +} + +static void atcphy_apply_tunables(struct apple_atcphy *atcphy, enum atcphy_mode mode) +{ + const int lane0 = atcphy->swap_lanes ? 1 : 0; + const int lane1 = atcphy->swap_lanes ? 0 : 1; + + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.common[0]); + apple_tunable_apply(atcphy->regs.axi2af, atcphy->tunables.axi2af); + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.common[1]); + + switch (mode) { + /* + * USB 3.2 Gen 2x2 / SuperSpeed 20Gbps is not supported by this hardware and applying USB3 + * tunables to both lanes does not result in a working PHY configuration. Thus, both + * USB3-only and USB3/DP get the same tunable setup here. + */ + case APPLE_ATCPHY_MODE_USB3: + case APPLE_ATCPHY_MODE_USB3_DP: + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.lane_usb3[lane0]); + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.lane_dp[lane1]); + break; + + case APPLE_ATCPHY_MODE_DP: + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.lane_dp[lane0]); + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.lane_dp[lane1]); + break; + + /* + * Even though the various Thunderbolt versions and USB4 are different protocols they need + * the same tunables. The actual protocol-specific setup happens inside the Thunderbolt/USB4 + * native host interface. + */ + case APPLE_ATCPHY_MODE_TBT: + case APPLE_ATCPHY_MODE_USB4: + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.lane_usb4[lane0]); + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.lane_usb4[lane1]); + break; + + case APPLE_ATCPHY_MODE_OFF: + case APPLE_ATCPHY_MODE_USB2: + break; + } +} + +static int atcphy_pipehandler_lock(struct apple_atcphy *atcphy) +{ + int ret; + u32 reg; + + if (readl(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ) & PIPEHANDLER_LOCK_EN) { + dev_warn(atcphy->dev, "Pipehandler already locked\n"); + return 0; + } + + set32(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ, PIPEHANDLER_LOCK_EN); + + ret = readl_poll_timeout(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_ACK, reg, + reg & PIPEHANDLER_LOCK_EN, 10, PIPEHANDLER_LOCK_ACK_TIMEOUT_US); + if (ret) { + clear32(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ, 1); + dev_warn(atcphy->dev, "Pipehandler lock not acked.\n"); + } + + return ret; +} + +static int atcphy_pipehandler_unlock(struct apple_atcphy *atcphy) +{ + int ret; + u32 reg; + + clear32(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ, PIPEHANDLER_LOCK_EN); + ret = readl_poll_timeout(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_ACK, reg, + !(reg & PIPEHANDLER_LOCK_EN), 10, PIPEHANDLER_LOCK_ACK_TIMEOUT_US); + if (ret) + dev_warn(atcphy->dev, "Pipehandler lock release not acked.\n"); + + return ret; +} + +static int atcphy_pipehandler_check(struct apple_atcphy *atcphy) +{ + int ret; + + lockdep_assert_held(&atcphy->lock); + + if (readl(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_ACK) & PIPEHANDLER_LOCK_EN) { + dev_warn(atcphy->dev, "Pipehandler already locked\n"); + + ret = atcphy_pipehandler_unlock(atcphy); + if (ret) { + dev_err(atcphy->dev, "Failed to unlock pipehandler\n"); + return ret; + } + } + + return 0; +} + +static int atcphy_configure_pipehandler_usb3(struct apple_atcphy *atcphy, bool host) +{ + int ret; + u32 reg; + + ret = atcphy_pipehandler_check(atcphy); + if (ret) + return ret; + + /* + * Only host mode requires this unknown BIST sequence to work correctly, possibly due to + * some hardware quirk. Guest mode breaks if we try to apply this sequence. + */ + if (host) { + /* Force disable link detection */ + clear32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE_VALUES, + PIPEHANDLER_OVERRIDE_VAL_RXDETECT0 | PIPEHANDLER_OVERRIDE_VAL_RXDETECT1); + set32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE, + PIPEHANDLER_OVERRIDE_RXVALID); + set32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE, + PIPEHANDLER_OVERRIDE_RXDETECT); + + ret = atcphy_pipehandler_lock(atcphy); + if (ret) { + dev_err(atcphy->dev, "Failed to lock pipehandler"); + return ret; + } + + /* BIST dance */ + core_set32(atcphy, ACIOPHY_TOP_BIST_PHY_CFG0, + ACIOPHY_TOP_BIST_PHY_CFG0_LN0_RESET_N); + core_set32(atcphy, ACIOPHY_TOP_BIST_OV_CFG, ACIOPHY_TOP_BIST_OV_CFG_LN0_RESET_N_OV); + ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_TOP_PHY_STAT, reg, + !(reg & ACIOPHY_TOP_PHY_STAT_LN0_UNK23), 10, 10000); + if (ret) + dev_warn(atcphy->dev, + "Timed out waiting for ACIOPHY_TOP_PHY_STAT_LN0_UNK23\n"); + + core_set32(atcphy, ACIOPHY_TOP_BIST_READ_CTRL, + ACIOPHY_TOP_BIST_READ_CTRL_LN0_PHY_STATUS_RE); + core_clear32(atcphy, ACIOPHY_TOP_BIST_READ_CTRL, + ACIOPHY_TOP_BIST_READ_CTRL_LN0_PHY_STATUS_RE); + + core_mask32(atcphy, ACIOPHY_TOP_BIST_PHY_CFG1, + ACIOPHY_TOP_BIST_PHY_CFG1_LN0_PWR_DOWN, + FIELD_PREP(ACIOPHY_TOP_BIST_PHY_CFG1_LN0_PWR_DOWN, 3)); + + core_set32(atcphy, ACIOPHY_TOP_BIST_OV_CFG, + ACIOPHY_TOP_BIST_OV_CFG_LN0_PWR_DOWN_OV); + core_set32(atcphy, ACIOPHY_TOP_BIST_CIOPHY_CFG1, + ACIOPHY_TOP_BIST_CIOPHY_CFG1_CLK_EN); + core_set32(atcphy, ACIOPHY_TOP_BIST_CIOPHY_CFG1, + ACIOPHY_TOP_BIST_CIOPHY_CFG1_BIST_EN); + writel(0, atcphy->regs.core + ACIOPHY_TOP_BIST_CIOPHY_CFG1); + + ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_TOP_PHY_STAT, reg, + (reg & ACIOPHY_TOP_PHY_STAT_LN0_UNK0), 10, 10000); + if (ret) + dev_warn(atcphy->dev, + "timed out waiting for ACIOPHY_TOP_PHY_STAT_LN0_UNK0\n"); + + ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_TOP_PHY_STAT, reg, + !(reg & ACIOPHY_TOP_PHY_STAT_LN0_UNK23), 10, 10000); + if (ret) + dev_warn(atcphy->dev, + "timed out waiting for ACIOPHY_TOP_PHY_STAT_LN0_UNK23\n"); + + /* Clear reset for non-selected USB3 PHY (?) */ + mask32(atcphy->regs.pipehandler + PIPEHANDLER_NONSELECTED_OVERRIDE, + PIPEHANDLER_NATIVE_POWER_DOWN, FIELD_PREP(PIPEHANDLER_NATIVE_POWER_DOWN, 3)); + clear32(atcphy->regs.pipehandler + PIPEHANDLER_NONSELECTED_OVERRIDE, + PIPEHANDLER_NATIVE_RESET); + + /* More BIST stuff (?) */ + writel(0, atcphy->regs.core + ACIOPHY_TOP_BIST_OV_CFG); + core_set32(atcphy, ACIOPHY_TOP_BIST_CIOPHY_CFG1, + ACIOPHY_TOP_BIST_CIOPHY_CFG1_CLK_EN); + core_set32(atcphy, ACIOPHY_TOP_BIST_CIOPHY_CFG1, + ACIOPHY_TOP_BIST_CIOPHY_CFG1_BIST_EN); + } + + /* Configure PIPE mux to USB3 PHY */ + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CTRL_CLK, + FIELD_PREP(PIPEHANDLER_MUX_CTRL_CLK, PIPEHANDLER_MUX_CTRL_CLK_OFF)); + udelay(10); + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CTRL_DATA, + FIELD_PREP(PIPEHANDLER_MUX_CTRL_DATA, PIPEHANDLER_MUX_CTRL_DATA_USB3)); + udelay(10); + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CTRL_CLK, + FIELD_PREP(PIPEHANDLER_MUX_CTRL_CLK, PIPEHANDLER_MUX_CTRL_CLK_USB3)); + udelay(10); + + /* Remove link detection override */ + clear32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE, PIPEHANDLER_OVERRIDE_RXVALID); + clear32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE, PIPEHANDLER_OVERRIDE_RXDETECT); + + /* Pipehandler was only locked when the BIST sequence was applied for host mode */ + if (host) { + ret = atcphy_pipehandler_unlock(atcphy); + if (ret) + dev_warn(atcphy->dev, "Failed to unlock pipehandler"); + } + + return 0; +} + +static int atcphy_configure_pipehandler_dummy(struct apple_atcphy *atcphy) +{ + int ret; + + ret = atcphy_pipehandler_check(atcphy); + if (ret) + return ret; + + /* Force disable link detection */ + clear32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE_VALUES, + PIPEHANDLER_OVERRIDE_VAL_RXDETECT0 | PIPEHANDLER_OVERRIDE_VAL_RXDETECT1); + set32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE, PIPEHANDLER_OVERRIDE_RXVALID); + set32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE, PIPEHANDLER_OVERRIDE_RXDETECT); + + ret = atcphy_pipehandler_lock(atcphy); + if (ret) + dev_warn(atcphy->dev, "Failed to lock pipehandler"); + + /* Switch to dummy PHY */ + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CTRL_CLK, + FIELD_PREP(PIPEHANDLER_MUX_CTRL_CLK, PIPEHANDLER_MUX_CTRL_CLK_OFF)); + udelay(10); + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CTRL_DATA, + FIELD_PREP(PIPEHANDLER_MUX_CTRL_DATA, PIPEHANDLER_MUX_CTRL_DATA_DUMMY)); + udelay(10); + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CTRL_CLK, + FIELD_PREP(PIPEHANDLER_MUX_CTRL_CLK, PIPEHANDLER_MUX_CTRL_CLK_DUMMY)); + udelay(10); + + ret = atcphy_pipehandler_unlock(atcphy); + if (ret) + dev_warn(atcphy->dev, "Failed to unlock pipehandler"); + + mask32(atcphy->regs.pipehandler + PIPEHANDLER_NONSELECTED_OVERRIDE, + PIPEHANDLER_NATIVE_POWER_DOWN, FIELD_PREP(PIPEHANDLER_NATIVE_POWER_DOWN, 2)); + set32(atcphy->regs.pipehandler + PIPEHANDLER_NONSELECTED_OVERRIDE, + PIPEHANDLER_NATIVE_RESET); + + return 0; +} + +static int atcphy_configure_pipehandler(struct apple_atcphy *atcphy, bool host) +{ + int ret; + + lockdep_assert_held(&atcphy->lock); + + switch (atcphy_modes[atcphy->mode].pipehandler_state) { + case ATCPHY_PIPEHANDLER_STATE_USB3: + ret = atcphy_configure_pipehandler_usb3(atcphy, host); + atcphy->pipehandler_up = true; + break; + case ATCPHY_PIPEHANDLER_STATE_USB4: + dev_warn(atcphy->dev, + "ATCPHY_PIPEHANDLER_STATE_USB4 not implemented; falling back to USB2\n"); + ret = atcphy_configure_pipehandler_dummy(atcphy); + atcphy->pipehandler_up = false; + break; + default: + ret = -EINVAL; + } + + return ret; +} + +static void atcphy_setup_pipehandler(struct apple_atcphy *atcphy) +{ + lockdep_assert_held(&atcphy->lock); + + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CTRL_CLK, + FIELD_PREP(PIPEHANDLER_MUX_CTRL_CLK, PIPEHANDLER_MUX_CTRL_CLK_OFF)); + udelay(10); + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CTRL_DATA, + FIELD_PREP(PIPEHANDLER_MUX_CTRL_DATA, PIPEHANDLER_MUX_CTRL_DATA_DUMMY)); + udelay(10); + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CTRL_CLK, + FIELD_PREP(PIPEHANDLER_MUX_CTRL_CLK, PIPEHANDLER_MUX_CTRL_CLK_DUMMY)); + udelay(10); +} + +static void atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode mode) +{ + const struct atcphy_mode_configuration *mode_cfg = atcphy_get_mode_config(atcphy, mode); + + core_mask32(atcphy, ACIOPHY_LANE_MODE, ACIOPHY_LANE_MODE_RX0, + FIELD_PREP(ACIOPHY_LANE_MODE_RX0, mode_cfg->lane_mode[0])); + core_mask32(atcphy, ACIOPHY_LANE_MODE, ACIOPHY_LANE_MODE_TX0, + FIELD_PREP(ACIOPHY_LANE_MODE_TX0, mode_cfg->lane_mode[0])); + core_mask32(atcphy, ACIOPHY_LANE_MODE, ACIOPHY_LANE_MODE_RX1, + FIELD_PREP(ACIOPHY_LANE_MODE_RX1, mode_cfg->lane_mode[1])); + core_mask32(atcphy, ACIOPHY_LANE_MODE, ACIOPHY_LANE_MODE_TX1, + FIELD_PREP(ACIOPHY_LANE_MODE_TX1, mode_cfg->lane_mode[1])); + core_mask32(atcphy, ACIOPHY_CROSSBAR, ACIOPHY_CROSSBAR_PROTOCOL, + FIELD_PREP(ACIOPHY_CROSSBAR_PROTOCOL, mode_cfg->crossbar)); + + if (mode_cfg->set_swap) + core_set32(atcphy, ATCPHY_MISC, ATCPHY_MISC_LANE_SWAP); + else + core_clear32(atcphy, ATCPHY_MISC, ATCPHY_MISC_LANE_SWAP); + + core_mask32(atcphy, ACIOPHY_CROSSBAR, ACIOPHY_CROSSBAR_DP_SINGLE_PMA, + FIELD_PREP(ACIOPHY_CROSSBAR_DP_SINGLE_PMA, mode_cfg->crossbar_dp_single_pma)); + if (mode_cfg->crossbar_dp_both_pma) + core_set32(atcphy, ACIOPHY_CROSSBAR, ACIOPHY_CROSSBAR_DP_BOTH_PMA); + else + core_clear32(atcphy, ACIOPHY_CROSSBAR, ACIOPHY_CROSSBAR_DP_BOTH_PMA); + + if (mode_cfg->dp_lane[0]) { + core_set32(atcphy, LN0_AUSPMA_RX_TOP + LN_AUSPMA_RX_TOP_PMAFSM, + LN_AUSPMA_RX_TOP_PMAFSM_PCS_OV); + udelay(10); + core_clear32(atcphy, LN0_AUSPMA_RX_TOP + LN_AUSPMA_RX_TOP_PMAFSM, + LN_AUSPMA_RX_TOP_PMAFSM_PCS_REQ); + } else { + core_clear32(atcphy, LN0_AUSPMA_RX_TOP + LN_AUSPMA_RX_TOP_PMAFSM, + LN_AUSPMA_RX_TOP_PMAFSM_PCS_OV); + udelay(10); + } + + if (mode_cfg->dp_lane[1]) { + core_set32(atcphy, LN1_AUSPMA_RX_TOP + LN_AUSPMA_RX_TOP_PMAFSM, + LN_AUSPMA_RX_TOP_PMAFSM_PCS_OV); + udelay(10); + core_clear32(atcphy, LN1_AUSPMA_RX_TOP + LN_AUSPMA_RX_TOP_PMAFSM, + LN_AUSPMA_RX_TOP_PMAFSM_PCS_REQ); + } else { + core_clear32(atcphy, LN1_AUSPMA_RX_TOP + LN_AUSPMA_RX_TOP_PMAFSM, + LN_AUSPMA_RX_TOP_PMAFSM_PCS_OV); + udelay(10); + } +} + +static void atcphy_enable_dp_aux(struct apple_atcphy *atcphy) +{ + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTXPHY_PMA_LANE_RESET_N); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTXPHY_PMA_LANE_RESET_N_OV); + + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPRX_PCLK_SELECT, + FIELD_PREP(DPRX_PCLK_SELECT, 1)); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPRX_PCLK_ENABLE); + + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_SELECT, + FIELD_PREP(DPTX_PCLK1_SELECT, 1)); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE); + + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK2_SELECT, + FIELD_PREP(DPTX_PCLK2_SELECT, 1)); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK2_ENABLE); + + core_set32(atcphy, ACIOPHY_PLL_COMMON_CTRL, + ACIOPHY_PLL_WAIT_FOR_CMN_READY_BEFORE_RESET_EXIT); + + set32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_AUX_CLAMP_EN); + set32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_SLEEP_B_SML_IN); + udelay(10); + set32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_SLEEP_B_BIG_IN); + udelay(10); + clear32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_AUX_CLAMP_EN); + clear32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_AUX_PWN_DOWN); + clear32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_TXTERM_CODEMSB); + mask32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_TXTERM_CODE, + FIELD_PREP(LPDPTX_TXTERM_CODE, 0x16)); + + set32(atcphy->regs.lpdptx + LPDPTX_AUX_CFG_BLK_AUX_LDO_CTRL, 0x1c00); + mask32(atcphy->regs.lpdptx + LPDPTX_AUX_SHM_CFG_BLK_AUX_CTRL_REG1, LPDPTX_CFG_PMA_PHYS_ADJ, + FIELD_PREP(LPDPTX_CFG_PMA_PHYS_ADJ, 5)); + set32(atcphy->regs.lpdptx + LPDPTX_AUX_SHM_CFG_BLK_AUX_CTRL_REG1, + LPDPTX_CFG_PMA_PHYS_ADJ_OV); + + clear32(atcphy->regs.lpdptx + LPDPTX_AUX_CFG_BLK_AUX_MARGIN, + LPDPTX_MARGIN_RCAL_RXOFFSET_EN); + + clear32(atcphy->regs.lpdptx + LPDPTX_AUX_CFG_BLK_AUX_CTRL, LPDPTX_BLK_AUX_CTRL_PWRDN); + set32(atcphy->regs.lpdptx + LPDPTX_AUX_SHM_CFG_BLK_AUX_CTRL_REG0, + LPDPTX_CFG_PMA_AUX_SEL_LF_DATA); + mask32(atcphy->regs.lpdptx + LPDPTX_AUX_CFG_BLK_AUX_CTRL, LPDPTX_BLK_AUX_RXOFFSET, + FIELD_PREP(LPDPTX_BLK_AUX_RXOFFSET, 3)); + + mask32(atcphy->regs.lpdptx + LPDPTX_AUX_CFG_BLK_AUX_MARGIN, LPDPTX_AUX_MARGIN_RCAL_TXSWING, + FIELD_PREP(LPDPTX_AUX_MARGIN_RCAL_TXSWING, 12)); + + atcphy->dp_link_rate = -1; +} + +static void atcphy_disable_dp_aux(struct apple_atcphy *atcphy) +{ + set32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_AUX_PWN_DOWN); + set32(atcphy->regs.lpdptx + LPDPTX_AUX_CFG_BLK_AUX_CTRL, LPDPTX_BLK_AUX_CTRL_PWRDN); + set32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_AUX_CLAMP_EN); + clear32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_SLEEP_B_SML_IN); + udelay(10); + clear32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_SLEEP_B_BIG_IN); + udelay(10); + + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTXPHY_PMA_LANE_RESET_N); + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPRX_PCLK_ENABLE); + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE); + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK2_ENABLE); +} + +static int atcphy_dp_configure_lane(struct apple_atcphy *atcphy, enum atcphy_lane lane, + const struct atcphy_dp_link_rate_configuration *cfg) +{ + void __iomem *tx_shm, *rx_shm, *rx_top; + unsigned int tx_cal_code; + + lockdep_assert_held(&atcphy->lock); + + switch (lane) { + case APPLE_ATCPHY_LANE_0: + tx_shm = atcphy->regs.core + LN0_AUSPMA_TX_SHM; + rx_shm = atcphy->regs.core + LN0_AUSPMA_RX_SHM; + rx_top = atcphy->regs.core + LN0_AUSPMA_RX_TOP; + break; + case APPLE_ATCPHY_LANE_1: + tx_shm = atcphy->regs.core + LN1_AUSPMA_TX_SHM; + rx_shm = atcphy->regs.core + LN1_AUSPMA_RX_SHM; + rx_top = atcphy->regs.core + LN1_AUSPMA_RX_TOP; + break; + default: + return -EINVAL; + } + + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, LN_LDOCLK_EN_SML); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, LN_LDOCLK_EN_SML_OV); + udelay(10); + + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, LN_LDOCLK_EN_BIG); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, LN_LDOCLK_EN_BIG_OV); + udelay(10); + + if (cfg->txa_ldoclk_bypass) { + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, LN_LDOCLK_BYPASS_SML); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, LN_LDOCLK_BYPASS_SML_OV); + udelay(10); + + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, LN_LDOCLK_BYPASS_BIG); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, LN_LDOCLK_BYPASS_BIG_OV); + udelay(10); + } else { + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, LN_LDOCLK_BYPASS_SML); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, LN_LDOCLK_BYPASS_SML_OV); + udelay(10); + + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, LN_LDOCLK_BYPASS_BIG); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, LN_LDOCLK_BYPASS_BIG_OV); + udelay(10); + } + + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG0, LN_BYTECLK_RESET_SYNC_SEL_OV); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG0, LN_BYTECLK_RESET_SYNC_EN); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG0, LN_BYTECLK_RESET_SYNC_EN_OV); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG0, LN_BYTECLK_RESET_SYNC_CLR); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG0, LN_BYTECLK_RESET_SYNC_CLR_OV); + + if (cfg->txa_div2_en) + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG1, LN_TXA_DIV2_EN); + else + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG1, LN_TXA_DIV2_EN); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG1, LN_TXA_DIV2_EN_OV); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG1, LN_TXA_CLK_EN); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG1, LN_TXA_CLK_EN_OV); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG1, LN_TXA_DIV2_RESET); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG1, LN_TXA_DIV2_RESET_OV); + + mask32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG0, LN_TXA_CAL_CTRL_BASE, + FIELD_PREP(LN_TXA_CAL_CTRL_BASE, 0xf)); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG0, LN_TXA_CAL_CTRL_BASE_OV); + + tx_cal_code = FIELD_GET(AUS_UNK_A20_TX_CAL_CODE, readl(atcphy->regs.core + AUS_UNK_A20)); + mask32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG0, LN_TXA_CAL_CTRL, + FIELD_PREP(LN_TXA_CAL_CTRL, (1 << tx_cal_code) - 1)); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG0, LN_TXA_CAL_CTRL_OV); + + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG2, LN_TXA_MARGIN); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG2, LN_TXA_MARGIN_OV); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG2, LN_TXA_MARGIN_2R); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG2, LN_TXA_MARGIN_2R_OV); + + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_POST); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_POST_OV); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_POST_2R); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_POST_2R_OV); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_POST_4R); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_POST_4R_OV); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_PRE); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_PRE_OV); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_PRE_2R); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_PRE_2R_OV); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_PRE_4R); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_PRE_4R_OV); + + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG0, LN_TXA_HIZ); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG0, LN_TXA_HIZ_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_AFE_CTRL1, LN_RX_DIV20_RESET_N); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_AFE_CTRL1, LN_RX_DIV20_RESET_N_OV); + udelay(10); + + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_AFE_CTRL1, LN_RX_DIV20_RESET_N); + + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL12, LN_TX_BYTECLK_RESET_SYNC_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL12, LN_TX_BYTECLK_RESET_SYNC_EN_OV); + + mask32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_SAVOS_CTRL16, LN_TX_CAL_CODE, + FIELD_PREP(LN_TX_CAL_CODE, tx_cal_code)); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_SAVOS_CTRL16, LN_TX_CAL_CODE_OV); + + mask32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, LN_TX_CLK_DLY_CTRL_TAPGEN, + FIELD_PREP(LN_TX_CLK_DLY_CTRL_TAPGEN, 3)); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL10, LN_DTVREG_ADJUST); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL13, LN_DTVREG_ADJUST_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_SAVOS_CTRL16, LN_RXTERM_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_SAVOS_CTRL16, LN_RXTERM_EN_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, LN_TX_TEST_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, LN_TX_TEST_EN_OV); + + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_TEST_RXLPBKDT_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_TEST_RXLPBKDT_EN_OV); + mask32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_LPBKIN_DATA, + FIELD_PREP(LN_VREF_LPBKIN_DATA, 3)); + mask32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_BIAS_SEL, + FIELD_PREP(LN_VREF_BIAS_SEL, 2)); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_BIAS_SEL_OV); + mask32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_ADJUST_GRAY, + FIELD_PREP(LN_VREF_ADJUST_GRAY, 0x18)); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_ADJUST_GRAY_OV); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_EN_OV); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_BOOST_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_BOOST_EN_OV); + udelay(10); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_BOOST_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_BOOST_EN_OV); + udelay(10); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL13, LN_TX_PRE_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL13, LN_TX_PRE_EN_OV); + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL13, LN_TX_PST1_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL13, LN_TX_PST1_EN_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL12, LN_TX_PBIAS_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL12, LN_TX_PBIAS_EN_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_SAVOS_CTRL16, LN_RXTERM_PULLUP_LEAK_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_SAVOS_CTRL16, LN_RXTERM_PULLUP_LEAK_EN_OV); + + set32(rx_top + LN_AUSPMA_RX_TOP_TJ_CFG_RX_TXMODE, LN_RX_TXMODE); + + if (cfg->txa_div2_en) + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, LN_TX_CLK_DIV2_EN); + else + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, LN_TX_CLK_DIV2_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, LN_TX_CLK_DIV2_EN_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, LN_TX_CLK_DIV2_RST); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, LN_TX_CLK_DIV2_RST_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL12, LN_TX_HRCLK_SEL); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL12, LN_TX_HRCLK_SEL_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL17, LN_TX_MARGIN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL17, LN_TX_MARGIN_OV); + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL17, LN_TX_MARGIN_LSB); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL17, LN_TX_MARGIN_LSB_OV); + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL17, LN_TX_MARGIN_P1); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL17, LN_TX_MARGIN_P1_OV); + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL17, LN_TX_MARGIN_P1_LSB); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL17, LN_TX_MARGIN_P1_LSB_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_P1_CODE); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_P1_CODE_OV); + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_P1_LSB_CODE); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_P1_LSB_CODE_OV); + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_MARGIN_PRE); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_MARGIN_PRE_OV); + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_MARGIN_PRE_LSB); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_MARGIN_PRE_LSB_OV); + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_PRE_LSB_CODE); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_PRE_LSB_CODE_OV); + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_PRE_CODE); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_PRE_CODE_OV); + + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL11, LN_DTVREG_SML_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL11, LN_DTVREG_SML_EN_OV); + udelay(10); + + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL11, LN_DTVREG_BIG_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL11, LN_DTVREG_BIG_EN_OV); + udelay(10); + + mask32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL10, LN_DTVREG_ADJUST, + FIELD_PREP(LN_DTVREG_ADJUST, 0xa)); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL13, LN_DTVREG_ADJUST_OV); + udelay(10); + + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, LN_TX_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, LN_TX_EN_OV); + udelay(10); + + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_CTLE_CTRL0, LN_TX_CLK_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_CTLE_CTRL0, LN_TX_CLK_EN_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL12, LN_TX_BYTECLK_RESET_SYNC_CLR); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL12, LN_TX_BYTECLK_RESET_SYNC_CLR_OV); + + return 0; +} + +static int atcphy_auspll_apb_command(struct apple_atcphy *atcphy, u32 command) +{ + int ret; + u32 reg; + + reg = readl(atcphy->regs.core + AUSPLL_APB_CMD_OVERRIDE); + reg &= ~AUSPLL_APB_CMD_OVERRIDE_CMD; + reg |= FIELD_PREP(AUSPLL_APB_CMD_OVERRIDE_CMD, command); + reg |= AUSPLL_APB_CMD_OVERRIDE_REQ; + reg |= AUSPLL_APB_CMD_OVERRIDE_UNK28; + writel(reg, atcphy->regs.core + AUSPLL_APB_CMD_OVERRIDE); + + ret = readl_poll_timeout(atcphy->regs.core + AUSPLL_APB_CMD_OVERRIDE, reg, + (reg & AUSPLL_APB_CMD_OVERRIDE_ACK), 10, 10000); + if (ret) + dev_warn(atcphy->dev, "AUSPLL APB command was not acked\n"); + + core_clear32(atcphy, AUSPLL_APB_CMD_OVERRIDE, AUSPLL_APB_CMD_OVERRIDE_REQ); + + return 0; +} + +static int atcphy_dp_configure(struct apple_atcphy *atcphy, enum atcphy_dp_link_rate lr) +{ + const struct atcphy_dp_link_rate_configuration *cfg; + const struct atcphy_mode_configuration *mode_cfg; + int ret; + u32 reg; + + guard(mutex)(&atcphy->lock); + mode_cfg = atcphy_get_mode_config(atcphy, atcphy->mode); + cfg = &dp_lr_config[lr]; + + if (atcphy->dp_link_rate == lr) + return 0; + + ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_CMN_SHM_STS_REG0, reg, + (reg & ACIOPHY_CMN_SHM_STS_REG0_CMD_READY), 10, 10000); + if (ret) { + dev_err(atcphy->dev, "ACIOPHY_CMN_SHM_STS_REG0_CMD_READY not set.\n"); + return ret; + } + + core_clear32(atcphy, AUSPLL_FREQ_CFG, AUSPLL_FREQ_REFCLK); + + core_mask32(atcphy, AUSPLL_FREQ_DESC_A, AUSPLL_FD_FREQ_COUNT_TARGET, + FIELD_PREP(AUSPLL_FD_FREQ_COUNT_TARGET, cfg->freqinit_count_target)); + core_clear32(atcphy, AUSPLL_FREQ_DESC_A, AUSPLL_FD_FBDIVN_HALF); + core_clear32(atcphy, AUSPLL_FREQ_DESC_A, AUSPLL_FD_REV_DIVN); + core_mask32(atcphy, AUSPLL_FREQ_DESC_A, AUSPLL_FD_KI_MAN, FIELD_PREP(AUSPLL_FD_KI_MAN, 8)); + core_mask32(atcphy, AUSPLL_FREQ_DESC_A, AUSPLL_FD_KI_EXP, FIELD_PREP(AUSPLL_FD_KI_EXP, 3)); + core_mask32(atcphy, AUSPLL_FREQ_DESC_A, AUSPLL_FD_KP_MAN, FIELD_PREP(AUSPLL_FD_KP_MAN, 8)); + core_mask32(atcphy, AUSPLL_FREQ_DESC_A, AUSPLL_FD_KP_EXP, FIELD_PREP(AUSPLL_FD_KP_EXP, 7)); + core_clear32(atcphy, AUSPLL_FREQ_DESC_A, AUSPLL_FD_KPKI_SCALE_HBW); + + core_mask32(atcphy, AUSPLL_FREQ_DESC_B, AUSPLL_FD_FBDIVN_FRAC_DEN, + FIELD_PREP(AUSPLL_FD_FBDIVN_FRAC_DEN, cfg->fbdivn_frac_den)); + core_mask32(atcphy, AUSPLL_FREQ_DESC_B, AUSPLL_FD_FBDIVN_FRAC_NUM, + FIELD_PREP(AUSPLL_FD_FBDIVN_FRAC_NUM, cfg->fbdivn_frac_num)); + + core_clear32(atcphy, AUSPLL_FREQ_DESC_C, AUSPLL_FD_SDM_SSC_STEP); + core_clear32(atcphy, AUSPLL_FREQ_DESC_C, AUSPLL_FD_SDM_SSC_EN); + core_mask32(atcphy, AUSPLL_FREQ_DESC_C, AUSPLL_FD_PCLK_DIV_SEL, + FIELD_PREP(AUSPLL_FD_PCLK_DIV_SEL, cfg->pclk_div_sel)); + core_mask32(atcphy, AUSPLL_FREQ_DESC_C, AUSPLL_FD_LFSDM_DIV, + FIELD_PREP(AUSPLL_FD_LFSDM_DIV, 1)); + core_mask32(atcphy, AUSPLL_FREQ_DESC_C, AUSPLL_FD_LFCLK_CTRL, + FIELD_PREP(AUSPLL_FD_LFCLK_CTRL, cfg->lfclk_ctrl)); + core_mask32(atcphy, AUSPLL_FREQ_DESC_C, AUSPLL_FD_VCLK_OP_DIVN, + FIELD_PREP(AUSPLL_FD_VCLK_OP_DIVN, cfg->vclk_op_divn)); + core_set32(atcphy, AUSPLL_FREQ_DESC_C, AUSPLL_FD_VCLK_PRE_DIVN); + + core_mask32(atcphy, AUSPLL_CLKOUT_DIV, AUSPLL_CLKOUT_PLLA_REFBUFCLK_DI, + FIELD_PREP(AUSPLL_CLKOUT_PLLA_REFBUFCLK_DI, 7)); + + if (cfg->plla_clkout_vreg_bypass) + core_set32(atcphy, AUSPLL_CLKOUT_DTC_VREG, AUSPLL_DTC_VREG_BYPASS); + else + core_clear32(atcphy, AUSPLL_CLKOUT_DTC_VREG, AUSPLL_DTC_VREG_BYPASS); + + core_set32(atcphy, AUSPLL_BGR, AUSPLL_BGR_CTRL_AVAIL); + + core_set32(atcphy, AUSPLL_CLKOUT_MASTER, AUSPLL_CLKOUT_MASTER_PCLK_DRVR_EN); + core_set32(atcphy, AUSPLL_CLKOUT_MASTER, AUSPLL_CLKOUT_MASTER_PCLK2_DRVR_EN); + core_set32(atcphy, AUSPLL_CLKOUT_MASTER, AUSPLL_CLKOUT_MASTER_REFBUFCLK_DRVR_EN); + + ret = atcphy_auspll_apb_command(atcphy, 0); + if (ret) + return ret; + + ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT, reg, + (reg & ACIOPHY_AUSPLL_LOCK), 10, 10000); + if (ret) { + dev_err(atcphy->dev, "ACIOPHY_DP_PCLK did not lock.\n"); + return ret; + } + + ret = atcphy_auspll_apb_command(atcphy, 0x2800); + if (ret) + return ret; + + if (mode_cfg->dp_lane[0]) { + ret = atcphy_dp_configure_lane(atcphy, APPLE_ATCPHY_LANE_0, cfg); + if (ret) + return ret; + } + + if (mode_cfg->dp_lane[1]) { + ret = atcphy_dp_configure_lane(atcphy, APPLE_ATCPHY_LANE_1, cfg); + if (ret) + return ret; + } + + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DP_PMA_BYTECLK_RESET); + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DP_MAC_DIV20_CLK_SEL); + + atcphy->dp_link_rate = lr; + return 0; +} + +static void atcphy_usb2_power_off(struct apple_atcphy *atcphy) +{ + /* Disable the PHY, this clears USB2PHY_USBCTL_RUN */ + writel(USB2PHY_USBCTL_ISOLATION, atcphy->regs.usb2phy + USB2PHY_USBCTL); + udelay(10); + + /* Switch the PHY to low power mode */ + set32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_SIDDQ); + udelay(10); + + /* Enable all resets */ + set32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_PORT_RESET); + udelay(10); + set32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_RESET); + udelay(10); + clear32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_APB_RESET_N); + udelay(10); + set32(atcphy->regs.usb2phy + USB2PHY_MISCTUNE, USB2PHY_MISCTUNE_APBCLK_GATE_OFF); + set32(atcphy->regs.usb2phy + USB2PHY_MISCTUNE, USB2PHY_MISCTUNE_REFCLK_GATE_OFF); +} + +static int atcphy_power_off(struct apple_atcphy *atcphy) +{ + u32 reg; + int ret; + + atcphy_disable_dp_aux(atcphy); + + /* Enable all reset lines */ + core_clear32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_PHY_RESET_N); + core_set32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_CLAMP_EN); + core_clear32(atcphy, ATCPHY_MISC, ATCPHY_MISC_RESET_N | ATCPHY_MISC_LANE_SWAP); + core_clear32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_APB_RESET_N); + + core_clear32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_SLEEP_BIG); + ret = readl_poll_timeout(atcphy->regs.core + ATCPHY_POWER_STAT, reg, + !(reg & ATCPHY_POWER_SLEEP_BIG), 10, 1000); + if (ret) { + dev_err(atcphy->dev, "Failed to sleep atcphy \"big\"\n"); + return ret; + } + + core_clear32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_SLEEP_SMALL); + ret = readl_poll_timeout(atcphy->regs.core + ATCPHY_POWER_STAT, reg, + !(reg & ATCPHY_POWER_SLEEP_SMALL), 10, 1000); + if (ret) { + dev_err(atcphy->dev, "Failed to sleep atcphy \"small\"\n"); + return ret; + } + + return 0; +} + +static void atcphy_usb2_power_on(struct apple_atcphy *atcphy) +{ + set32(atcphy->regs.usb2phy + USB2PHY_SIG, + USB2PHY_SIG_VBUSDET_FORCE_VAL | USB2PHY_SIG_VBUSDET_FORCE_EN | + USB2PHY_SIG_VBUSVLDEXT_FORCE_VAL | USB2PHY_SIG_VBUSVLDEXT_FORCE_EN); + udelay(10); + + /* Take the PHY out of its low power state */ + clear32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_SIDDQ); + udelay(10); + + /* Release reset */ + clear32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_RESET); + udelay(10); + clear32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_PORT_RESET); + udelay(10); + set32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_APB_RESET_N); + udelay(10); + clear32(atcphy->regs.usb2phy + USB2PHY_MISCTUNE, USB2PHY_MISCTUNE_APBCLK_GATE_OFF); + clear32(atcphy->regs.usb2phy + USB2PHY_MISCTUNE, USB2PHY_MISCTUNE_REFCLK_GATE_OFF); + + /* Enable the PHY */ + writel(USB2PHY_USBCTL_RUN, atcphy->regs.usb2phy + USB2PHY_USBCTL); +} + +static int atcphy_power_on(struct apple_atcphy *atcphy) +{ + u32 reg; + int ret; + + atcphy_usb2_power_on(atcphy); + + core_set32(atcphy, ATCPHY_MISC, ATCPHY_MISC_RESET_N); + + core_set32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_SLEEP_SMALL); + ret = readl_poll_timeout(atcphy->regs.core + ATCPHY_POWER_STAT, reg, + reg & ATCPHY_POWER_SLEEP_SMALL, 100, 100000); + if (ret) { + dev_err(atcphy->dev, "failed to wakeup atcphy \"small\"\n"); + return ret; + } + + core_set32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_SLEEP_BIG); + ret = readl_poll_timeout(atcphy->regs.core + ATCPHY_POWER_STAT, reg, + reg & ATCPHY_POWER_SLEEP_BIG, 100, 100000); + if (ret) { + dev_err(atcphy->dev, "failed to wakeup atcphy \"big\"\n"); + return ret; + } + + core_clear32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_CLAMP_EN); + core_set32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_APB_RESET_N); + + return 0; +} + +static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) +{ + int ret = 0; + + lockdep_assert_held(&atcphy->lock); + + if (mode == APPLE_ATCPHY_MODE_OFF) { + ret = atcphy_power_off(atcphy); + atcphy->mode = mode; + return ret; + } + + ret = atcphy_power_on(atcphy); + if (ret) + return ret; + + atcphy_apply_tunables(atcphy, mode); + + core_set32(atcphy, AUSPLL_FSM_CTRL, 0x1fe000); + core_set32(atcphy, AUSPLL_APB_CMD_OVERRIDE, AUSPLL_APB_CMD_OVERRIDE_UNK28); + + set32(atcphy->regs.core + ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_SMALL_OV); + udelay(10); + set32(atcphy->regs.core + ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_BIG_OV); + udelay(10); + set32(atcphy->regs.core + ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_CLAMP_OV); + udelay(10); + + mask32(atcphy->regs.core + ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_SMALL_OV, + FIELD_PREP(ACIOPHY_SLEEP_CTRL_TX_SMALL_OV, 3)); + udelay(10); + mask32(atcphy->regs.core + ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_BIG_OV, + FIELD_PREP(ACIOPHY_SLEEP_CTRL_TX_BIG_OV, 3)); + udelay(10); + mask32(atcphy->regs.core + ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_CLAMP_OV, + FIELD_PREP(ACIOPHY_SLEEP_CTRL_TX_CLAMP_OV, 3)); + udelay(10); + + mask32(atcphy->regs.core + ACIOPHY_CFG0, ACIOPHY_CFG0_RX_BIG_OV, + FIELD_PREP(ACIOPHY_CFG0_RX_BIG_OV, 3)); + udelay(10); + mask32(atcphy->regs.core + ACIOPHY_CFG0, ACIOPHY_CFG0_RX_SMALL_OV, + FIELD_PREP(ACIOPHY_CFG0_RX_SMALL_OV, 3)); + udelay(10); + mask32(atcphy->regs.core + ACIOPHY_CFG0, ACIOPHY_CFG0_RX_CLAMP_OV, + FIELD_PREP(ACIOPHY_CFG0_RX_CLAMP_OV, 3)); + udelay(10); + + /* Setup AUX channel if DP altmode is requested */ + if (atcphy_modes[mode].enable_dp_aux) + atcphy_enable_dp_aux(atcphy); + + /* Enable clocks and configure lanes */ + core_set32(atcphy, CIO3PLL_CLK_CTRL, CIO3PLL_CLK_PCLK_EN); + core_set32(atcphy, CIO3PLL_CLK_CTRL, CIO3PLL_CLK_REFCLK_EN); + atcphy_configure_lanes(atcphy, mode); + + /* Take the USB3 PHY out of reset */ + core_set32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_PHY_RESET_N); + + atcphy->mode = mode; + + return 0; +} + +static int atcphy_usb2_set_mode(struct phy *phy, enum phy_mode mode, int submode) +{ + struct apple_atcphy *atcphy = phy_get_drvdata(phy); + + guard(mutex)(&atcphy->lock); + + switch (mode) { + case PHY_MODE_USB_HOST: + set32(atcphy->regs.usb2phy + USB2PHY_SIG, USB2PHY_SIG_HOST); + break; + case PHY_MODE_USB_DEVICE: + clear32(atcphy->regs.usb2phy + USB2PHY_SIG, USB2PHY_SIG_HOST); + break; + default: + return -EINVAL; + } + + return 0; +} + +static const struct phy_ops apple_atc_usb2_phy_ops = { + .owner = THIS_MODULE, + .set_mode = atcphy_usb2_set_mode, +}; + +static int atcphy_usb3_power_off(struct phy *phy) +{ + struct apple_atcphy *atcphy = phy_get_drvdata(phy); + int ret; + + guard(mutex)(&atcphy->lock); + + ret = atcphy_configure_pipehandler_dummy(atcphy); + if (ret) + dev_warn(atcphy->dev, "Failed to switch pipe to dummy: %d", ret); + + atcphy->pipehandler_up = false; + + if (atcphy->mode != APPLE_ATCPHY_MODE_OFF) + atcphy_configure(atcphy, APPLE_ATCPHY_MODE_OFF); + + return 0; +} + +static int atcphy_usb3_set_mode(struct phy *phy, enum phy_mode mode, int submode) +{ + struct apple_atcphy *atcphy = phy_get_drvdata(phy); + + guard(mutex)(&atcphy->lock); + + /* + * We may get multiple calls to set_mode (for host mode e.g. at least one from the dwc3 glue + * driver and then another one from the generic xhci code) but must only configure the + * PIPE handler once. + */ + if (atcphy->pipehandler_up) + return 0; + + switch (mode) { + case PHY_MODE_USB_HOST: + return atcphy_configure_pipehandler(atcphy, true); + case PHY_MODE_USB_DEVICE: + return atcphy_configure_pipehandler(atcphy, false); + default: + return -EINVAL; + } +} + +static const struct phy_ops apple_atc_usb3_phy_ops = { + .owner = THIS_MODULE, + .power_off = atcphy_usb3_power_off, + .set_mode = atcphy_usb3_set_mode, +}; + +static int atcphy_dpphy_set_mode(struct phy *phy, enum phy_mode mode, int submode) +{ + /* Nothing to do here since the setup already happened in mux_set */ + if (mode == PHY_MODE_DP && submode == 0) + return 0; + return -EINVAL; +} + +static int atcphy_dpphy_validate(struct phy *phy, enum phy_mode mode, int submode, + union phy_configure_opts *opts_) +{ + struct phy_configure_opts_dp *opts = &opts_->dp; + struct apple_atcphy *atcphy = phy_get_drvdata(phy); + + if (mode != PHY_MODE_DP) + return -EINVAL; + if (submode != 0) + return -EINVAL; + + switch (atcphy->mode) { + case APPLE_ATCPHY_MODE_USB3_DP: + opts->lanes = 2; + break; + case APPLE_ATCPHY_MODE_DP: + opts->lanes = 4; + break; + default: + opts->lanes = 0; + } + + return 0; +} + +static int atcphy_dpphy_configure(struct phy *phy, union phy_configure_opts *opts_) +{ + struct phy_configure_opts_dp *opts = &opts_->dp; + struct apple_atcphy *atcphy = phy_get_drvdata(phy); + enum atcphy_dp_link_rate link_rate; + + if (opts->set_voltages) + return -EINVAL; + if (opts->set_lanes) + return -EINVAL; + + if (opts->set_rate) { + switch (opts->link_rate) { + case 1620: + link_rate = ATCPHY_DP_LINK_RATE_RBR; + break; + case 2700: + link_rate = ATCPHY_DP_LINK_RATE_HBR; + break; + case 5400: + link_rate = ATCPHY_DP_LINK_RATE_HBR2; + break; + case 8100: + link_rate = ATCPHY_DP_LINK_RATE_HBR3; + break; + case 0: + return 0; + default: + dev_err(atcphy->dev, "Unsupported link rate: %d\n", opts->link_rate); + return -EINVAL; + } + + return atcphy_dp_configure(atcphy, link_rate); + } + + return 0; +} + +static const struct phy_ops apple_atc_dp_phy_ops = { + .owner = THIS_MODULE, + .configure = atcphy_dpphy_configure, + .validate = atcphy_dpphy_validate, + .set_mode = atcphy_dpphy_set_mode, +}; + +static struct phy *atcphy_xlate(struct device *dev, const struct of_phandle_args *args) +{ + struct apple_atcphy *atcphy = dev_get_drvdata(dev); + + switch (args->args[0]) { + case PHY_TYPE_USB2: + return atcphy->phys.usb2; + case PHY_TYPE_USB3: + return atcphy->phys.usb3; + case PHY_TYPE_DP: + return atcphy->phys.dp; + } + return ERR_PTR(-ENODEV); +} + +static int atcphy_probe_phy(struct apple_atcphy *atcphy) +{ + struct { + struct phy **phy; + const struct phy_ops *ops; + } phys[] = { + { &atcphy->phys.usb2, &apple_atc_usb2_phy_ops }, + { &atcphy->phys.usb3, &apple_atc_usb3_phy_ops }, + { &atcphy->phys.dp, &apple_atc_dp_phy_ops }, + }; + + for (int i = 0; i < ARRAY_SIZE(phys); i++) { + *phys[i].phy = devm_phy_create(atcphy->dev, NULL, phys[i].ops); + if (IS_ERR(*phys[i].phy)) + return PTR_ERR(*phys[i].phy); + phy_set_drvdata(*phys[i].phy, atcphy); + } + + atcphy->phy_provider = devm_of_phy_provider_register(atcphy->dev, atcphy_xlate); + if (IS_ERR(atcphy->phy_provider)) + return PTR_ERR(atcphy->phy_provider); + return 0; +} + +static void _atcphy_dwc3_reset_assert(struct apple_atcphy *atcphy) +{ + lockdep_assert_held(&atcphy->lock); + + clear32(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN, PIPEHANDLER_AON_GEN_DWC3_RESET_N); + set32(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN, + PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN); +} + +static int atcphy_dwc3_reset_assert(struct reset_controller_dev *rcdev, unsigned long id) +{ + struct apple_atcphy *atcphy = container_of(rcdev, struct apple_atcphy, rcdev); + int ret; + + guard(mutex)(&atcphy->lock); + + _atcphy_dwc3_reset_assert(atcphy); + + if (atcphy->pipehandler_up) { + ret = atcphy_configure_pipehandler_dummy(atcphy); + if (ret) + dev_warn(atcphy->dev, "Failed to switch PIPE to dummy: %d\n", ret); + else + atcphy->pipehandler_up = false; + } + + atcphy_usb2_power_off(atcphy); + + return 0; +} + +static int atcphy_dwc3_reset_deassert(struct reset_controller_dev *rcdev, unsigned long id) +{ + struct apple_atcphy *atcphy = container_of(rcdev, struct apple_atcphy, rcdev); + + guard(mutex)(&atcphy->lock); + + clear32(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN, + PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN); + set32(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN, PIPEHANDLER_AON_GEN_DWC3_RESET_N); + + return 0; +} + +const struct reset_control_ops atcphy_dwc3_reset_ops = { + .assert = atcphy_dwc3_reset_assert, + .deassert = atcphy_dwc3_reset_deassert, +}; + +static int atcphy_reset_xlate(struct reset_controller_dev *rcdev, + const struct of_phandle_args *reset_spec) +{ + return 0; +} + +static int atcphy_probe_rcdev(struct apple_atcphy *atcphy) +{ + atcphy->rcdev.owner = THIS_MODULE; + atcphy->rcdev.nr_resets = 1; + atcphy->rcdev.ops = &atcphy_dwc3_reset_ops; + atcphy->rcdev.of_node = atcphy->dev->of_node; + atcphy->rcdev.of_reset_n_cells = 0; + atcphy->rcdev.of_xlate = atcphy_reset_xlate; + + return devm_reset_controller_register(atcphy->dev, &atcphy->rcdev); +} + +static int atcphy_sw_set(struct typec_switch_dev *sw, enum typec_orientation orientation) +{ + struct apple_atcphy *atcphy = typec_switch_get_drvdata(sw); + + guard(mutex)(&atcphy->lock); + + switch (orientation) { + case TYPEC_ORIENTATION_NONE: + break; + case TYPEC_ORIENTATION_NORMAL: + atcphy->swap_lanes = false; + break; + case TYPEC_ORIENTATION_REVERSE: + atcphy->swap_lanes = true; + break; + } + + return 0; +} + +static int atcphy_probe_switch(struct apple_atcphy *atcphy) +{ + struct typec_switch_desc sw_desc = { + .drvdata = atcphy, + .fwnode = atcphy->dev->fwnode, + .set = atcphy_sw_set, + }; + + return PTR_ERR_OR_ZERO(typec_switch_register(atcphy->dev, &sw_desc)); +} + +static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *state) +{ + struct apple_atcphy *atcphy = typec_mux_get_drvdata(mux); + enum atcphy_mode target_mode; + + guard(mutex)(&atcphy->lock); + + if (state->mode == TYPEC_STATE_SAFE) { + target_mode = APPLE_ATCPHY_MODE_OFF; + } else if (state->mode == TYPEC_STATE_USB) { + target_mode = APPLE_ATCPHY_MODE_USB3; + } else if (!state->alt && state->mode == TYPEC_MODE_USB4) { + struct enter_usb_data *data = state->data; + u32 eudo_usb_mode = FIELD_GET(EUDO_USB_MODE_MASK, data->eudo); + + switch (eudo_usb_mode) { + case EUDO_USB_MODE_USB2: + target_mode = APPLE_ATCPHY_MODE_USB2; + break; + case EUDO_USB_MODE_USB3: + target_mode = APPLE_ATCPHY_MODE_USB3; + break; + case EUDO_USB_MODE_USB4: + target_mode = APPLE_ATCPHY_MODE_USB4; + break; + default: + dev_warn(atcphy->dev, "Unsupported EUDO USB mode: 0x%x.\n", eudo_usb_mode); + target_mode = APPLE_ATCPHY_MODE_OFF; + } + } else if (state->alt && state->alt->svid == USB_TYPEC_TBT_SID) { + target_mode = APPLE_ATCPHY_MODE_TBT; + } else if (state->alt && state->alt->svid == USB_TYPEC_DP_SID) { + switch (state->mode) { + case TYPEC_DP_STATE_C: + case TYPEC_DP_STATE_E: + target_mode = APPLE_ATCPHY_MODE_DP; + break; + case TYPEC_DP_STATE_D: + target_mode = APPLE_ATCPHY_MODE_USB3_DP; + break; + default: + dev_err(atcphy->dev, + "Unsupported DP pin assignment: 0x%lx, your connected device will not work.\n", + state->mode); + target_mode = APPLE_ATCPHY_MODE_OFF; + } + } else if (state->alt) { + dev_err(atcphy->dev, + "Unknown alternate mode SVID: 0x%x, your connected device will not work.\n", + state->alt->svid); + target_mode = APPLE_ATCPHY_MODE_OFF; + } else { + dev_err(atcphy->dev, "Unknown mode: 0x%lx, your connected device will not work.\n", + state->mode); + target_mode = APPLE_ATCPHY_MODE_OFF; + } + + if (atcphy->mode == target_mode) + return 0; + + /* + * If the pipehandler is still/already up here there's a bug somewhere so make sure to + * complain loudly. We can still try to switch modes and hope for the best though, + * in the worst case the hardware will fall back to USB2-only. + */ + WARN_ON_ONCE(atcphy->pipehandler_up); + return atcphy_configure(atcphy, target_mode); +} + +static int atcphy_probe_mux(struct apple_atcphy *atcphy) +{ + struct typec_mux_desc mux_desc = { + .drvdata = atcphy, + .fwnode = atcphy->dev->fwnode, + .set = atcphy_mux_set, + }; + + return PTR_ERR_OR_ZERO(typec_mux_register(atcphy->dev, &mux_desc)); +} + +static int atcphy_load_tunables(struct apple_atcphy *atcphy) +{ + struct { + const char *dt_name; + struct apple_tunable **tunable; + struct resource *res; + } tunables[] = { + { "apple,tunable-axi2af", &atcphy->tunables.axi2af, atcphy->res.axi2af }, + { "apple,tunable-common-a", &atcphy->tunables.common[0], atcphy->res.core }, + { "apple,tunable-common-b", &atcphy->tunables.common[1], atcphy->res.core }, + { "apple,tunable-lane0-usb", &atcphy->tunables.lane_usb3[0], atcphy->res.core }, + { "apple,tunable-lane1-usb", &atcphy->tunables.lane_usb3[1], atcphy->res.core }, + { "apple,tunable-lane0-cio", &atcphy->tunables.lane_usb4[0], atcphy->res.core }, + { "apple,tunable-lane1-cio", &atcphy->tunables.lane_usb4[1], atcphy->res.core }, + { "apple,tunable-lane0-dp", &atcphy->tunables.lane_dp[0], atcphy->res.core }, + { "apple,tunable-lane1-dp", &atcphy->tunables.lane_dp[1], atcphy->res.core }, + }; + + for (int i = 0; i < ARRAY_SIZE(tunables); i++) { + *tunables[i].tunable = devm_apple_tunable_parse( + atcphy->dev, atcphy->np, tunables[i].dt_name, tunables[i].res); + if (IS_ERR(*tunables[i].tunable)) { + dev_err(atcphy->dev, "Failed to read tunable %s: %ld\n", + tunables[i].dt_name, PTR_ERR(*tunables[i].tunable)); + return PTR_ERR(*tunables[i].tunable); + } + } + + return 0; +} + +static int atcphy_map_resources(struct platform_device *pdev, struct apple_atcphy *atcphy) +{ + struct { + const char *name; + void __iomem **addr; + struct resource **res; + } resources[] = { + { "core", &atcphy->regs.core, &atcphy->res.core }, + { "lpdptx", &atcphy->regs.lpdptx, NULL }, + { "axi2af", &atcphy->regs.axi2af, &atcphy->res.axi2af }, + { "usb2phy", &atcphy->regs.usb2phy, NULL }, + { "pipehandler", &atcphy->regs.pipehandler, NULL }, + }; + struct resource *res; + void __iomem *addr; + + for (int i = 0; i < ARRAY_SIZE(resources); i++) { + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, resources[i].name); + addr = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(addr)) + return dev_err_probe(atcphy->dev, PTR_ERR(addr), + "Unable to map %s regs", resources[i].name); + + *resources[i].addr = addr; + if (resources[i].res) + *resources[i].res = res; + } + + return 0; +} + +static int atcphy_probe_finalize(struct apple_atcphy *atcphy) +{ + int ret; + + guard(mutex)(&atcphy->lock); + + /* Reset dwc3 on probe, let dwc3 (consumer) deassert it */ + _atcphy_dwc3_reset_assert(atcphy); + + /* Reset atcphy to clear any state potentially left by the bootloader */ + atcphy_usb2_power_off(atcphy); + atcphy_power_off(atcphy); + atcphy_setup_pipehandler(atcphy); + + ret = atcphy_probe_rcdev(atcphy); + if (ret) + return dev_err_probe(atcphy->dev, ret, "Probing rcdev failed"); + ret = atcphy_probe_mux(atcphy); + if (ret) + return dev_err_probe(atcphy->dev, ret, "Probing mux failed"); + ret = atcphy_probe_switch(atcphy); + if (ret) + return dev_err_probe(atcphy->dev, ret, "Probing switch failed"); + ret = atcphy_probe_phy(atcphy); + if (ret) + return dev_err_probe(atcphy->dev, ret, "Probing phy failed"); + + return 0; +} + +static int atcphy_probe(struct platform_device *pdev) +{ + struct apple_atcphy *atcphy; + struct device *dev = &pdev->dev; + int ret; + + atcphy = devm_kzalloc(&pdev->dev, sizeof(*atcphy), GFP_KERNEL); + if (!atcphy) + return -ENOMEM; + + atcphy->dev = dev; + atcphy->np = dev->of_node; + mutex_init(&atcphy->lock); + platform_set_drvdata(pdev, atcphy); + + ret = atcphy_map_resources(pdev, atcphy); + if (ret) + return ret; + ret = atcphy_load_tunables(atcphy); + if (ret) + return ret; + + atcphy->mode = APPLE_ATCPHY_MODE_OFF; + atcphy->pipehandler_up = false; + + return atcphy_probe_finalize(atcphy); +} + +static const struct of_device_id atcphy_match[] = { + { .compatible = "apple,t8103-atcphy" }, + {}, +}; +MODULE_DEVICE_TABLE(of, atcphy_match); + +static struct platform_driver atcphy_driver = { + .driver = { + .name = "phy-apple-atc", + .of_match_table = atcphy_match, + }, + .probe = atcphy_probe, +}; +module_platform_driver(atcphy_driver); + +MODULE_AUTHOR("Sven Peter "); +MODULE_DESCRIPTION("Apple Type-C PHY driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/phy/apple/dptx.c b/drivers/phy/apple/dptx.c new file mode 100644 index 00000000000000..f0df2d40a18023 --- /dev/null +++ b/drivers/phy/apple/dptx.c @@ -0,0 +1,690 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Apple dptx PHY driver + * + * Copyright (C) The Asahi Linux Contributors + * Author: Janne Grunau + * + * based on drivers/phy/apple/atc.c + * + * Copyright (C) The Asahi Linux Contributors + * Author: Sven Peter + */ + +#include "dptx.h" + +#include +#include "linux/of.h" +#include +#include +#include +#include +#include +#include +#include + +#define DPTX_MAX_LANES 4 +#define DPTX_LANE0_OFFSET 0x5000 +#define DPTX_LANE_STRIDE 0x1000 +#define DPTX_LANE_END (DPTX_LANE0_OFFSET + DPTX_MAX_LANES * DPTX_LANE_STRIDE) + +enum apple_dptx_type { + DPTX_PHY_T8112, + DPTX_PHY_T6020, +}; + +struct apple_dptx_phy_hw { + enum apple_dptx_type type; +}; + +struct apple_dptx_phy { + struct device *dev; + + struct apple_dptx_phy_hw hw; + + int dp_link_rate; + + struct { + void __iomem *core; + void __iomem *dptx; + } regs; + + struct phy *phy_dp; + struct phy_provider *phy_provider; + + struct mutex lock; + + // TODO: m1n1 port things to clean up + u32 active_lanes; +}; + + +static inline void mask32(void __iomem *reg, u32 mask, u32 set) +{ + u32 value = readl(reg); + value &= ~mask; + value |= set; + writel(value, reg); +} + +static inline void set32(void __iomem *reg, u32 set) +{ + mask32(reg, 0, set); +} + +static inline void clear32(void __iomem *reg, u32 clear) +{ + mask32(reg, clear, 0); +} + + +static int dptx_phy_set_active_lane_count(struct apple_dptx_phy *phy, u32 num_lanes) +{ + u32 l, ctrl; + + dev_dbg(phy->dev, "set_active_lane_count(%u)\n", num_lanes); + + if (num_lanes == 3 || num_lanes > DPTX_MAX_LANES) + return -1; + + ctrl = readl(phy->regs.dptx + 0x4000); + writel(ctrl, phy->regs.dptx + 0x4000); + + for (l = 0; l < num_lanes; l++) { + u64 offset = 0x5000 + 0x1000 * l; + readl(phy->regs.dptx + offset); + writel(0x100, phy->regs.dptx + offset); + } + for (; l < DPTX_MAX_LANES; l++) { + u64 offset = 0x5000 + 0x1000 * l; + readl(phy->regs.dptx + offset); + writel(0x300, phy->regs.dptx + offset); + } + for (l = 0; l < num_lanes; l++) { + u64 offset = 0x5000 + 0x1000 * l; + readl(phy->regs.dptx + offset); + writel(0x0, phy->regs.dptx + offset); + } + for (; l < DPTX_MAX_LANES; l++) { + u64 offset = 0x5000 + 0x1000 * l; + readl(phy->regs.dptx + offset); + writel(0x300, phy->regs.dptx + offset); + } + + if (num_lanes > 0) { + // clear32(phy->regs.dptx + 0x4000, 0x4000000); + ctrl = readl(phy->regs.dptx + 0x4000); + ctrl &= ~0x4000000; + writel(ctrl, phy->regs.dptx + 0x4000); + } + phy->active_lanes = num_lanes; + + return 0; +} + +static int dptx_phy_activate(struct apple_dptx_phy *phy, u32 dcp_index) +{ + u32 val_2014; + u32 val_4008; + u32 val_4408; + + dev_dbg(phy->dev, "activate(dcp:%u)\n", dcp_index); + + // MMIO: R.4 0x23c500010 (dptx-phy[1], offset 0x10) = 0x0 + // MMIO: W.4 0x23c500010 (dptx-phy[1], offset 0x10) = 0x0 + readl(phy->regs.core + 0x10); + writel(dcp_index, phy->regs.core + 0x10); + + // MMIO: R.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x444 + // MMIO: W.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x454 + set32(phy->regs.core + 0x48, 0x010); + // MMIO: R.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x454 + // MMIO: W.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x474 + set32(phy->regs.core + 0x48, 0x020); + // MMIO: R.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x474 + // MMIO: W.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x434 + clear32(phy->regs.core + 0x48, 0x040); + // MMIO: R.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x434 + // MMIO: W.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x534 + set32(phy->regs.core + 0x48, 0x100); + // MMIO: R.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x534 + // MMIO: W.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x734 + set32(phy->regs.core + 0x48, 0x200); + // MMIO: R.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x734 + // MMIO: W.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x334 + clear32(phy->regs.core + 0x48, 0x400); + // MMIO: R.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x334 + // MMIO: W.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x335 + set32(phy->regs.core + 0x48, 0x001); + // MMIO: R.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x335 + // MMIO: W.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x337 + set32(phy->regs.core + 0x48, 0x002); + // MMIO: R.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x337 + // MMIO: W.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x333 + clear32(phy->regs.core + 0x48, 0x004); + + // MMIO: R.4 0x23c542014 (dptx-phy[0], offset 0x2014) = 0x80a0c + val_2014 = readl(phy->regs.dptx + 0x2014); + // MMIO: W.4 0x23c542014 (dptx-phy[0], offset 0x2014) = 0x300a0c + writel((0x30 << 16) | (val_2014 & 0xffff), phy->regs.dptx + 0x2014); + + // MMIO: R.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x644800 + // MMIO: W.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x654800 + set32(phy->regs.dptx + 0x20b8, 0x010000); + + // MMIO: R.4 0x23c542220 (dptx-phy[0], offset 0x2220) = 0x11090a2 + // MMIO: W.4 0x23c542220 (dptx-phy[0], offset 0x2220) = 0x11090a0 + clear32(phy->regs.dptx + 0x2220, 0x0000002); + + // MMIO: R.4 0x23c54222c (dptx-phy[0], offset 0x222c) = 0x103003 + // MMIO: W.4 0x23c54222c (dptx-phy[0], offset 0x222c) = 0x103803 + set32(phy->regs.dptx + 0x222c, 0x000800); + // MMIO: R.4 0x23c54222c (dptx-phy[0], offset 0x222c) = 0x103803 + // MMIO: W.4 0x23c54222c (dptx-phy[0], offset 0x222c) = 0x103903 + set32(phy->regs.dptx + 0x222c, 0x000100); + + // MMIO: R.4 0x23c542230 (dptx-phy[0], offset 0x2230) = 0x2308804 + // MMIO: W.4 0x23c542230 (dptx-phy[0], offset 0x2230) = 0x2208804 + clear32(phy->regs.dptx + 0x2230, 0x0100000); + + // MMIO: R.4 0x23c542278 (dptx-phy[0], offset 0x2278) = 0x18300811 + // MMIO: W.4 0x23c542278 (dptx-phy[0], offset 0x2278) = 0x10300811 + clear32(phy->regs.dptx + 0x2278, 0x08000000); + + // MMIO: R.4 0x23c5422a4 (dptx-phy[0], offset 0x22a4) = 0x1044200 + // MMIO: W.4 0x23c5422a4 (dptx-phy[0], offset 0x22a4) = 0x1044201 + set32(phy->regs.dptx + 0x22a4, 0x0000001); + + // MMIO: R.4 0x23c544008 (dptx-phy[0], offset 0x4008) = 0x18030 + val_4008 = readl(phy->regs.dptx + 0x4008); + // MMIO: W.4 0x23c544008 (dptx-phy[0], offset 0x4008) = 0x30030 + writel((0x6 << 15) | (val_4008 & 0x7fff), phy->regs.dptx + 0x4008); + // MMIO: R.4 0x23c544008 (dptx-phy[0], offset 0x4008) = 0x30030 + // MMIO: W.4 0x23c544008 (dptx-phy[0], offset 0x4008) = 0x30010 + clear32(phy->regs.dptx + 0x4008, 0x00020); + + // MMIO: R.4 0x23c54420c (dptx-phy[0], offset 0x420c) = 0x88e3 + // MMIO: W.4 0x23c54420c (dptx-phy[0], offset 0x420c) = 0x88c3 + clear32(phy->regs.dptx + 0x420c, 0x0020); + + // MMIO: R.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x0 + // MMIO: W.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000000 + set32(phy->regs.dptx + 0x4600, 0x8000000); + + // MMIO: R.4 0x23c545040 (dptx-phy[0], offset 0x5040) = 0x21780 + // MMIO: W.4 0x23c545040 (dptx-phy[0], offset 0x5040) = 0x221780 + // MMIO: R.4 0x23c546040 (dptx-phy[0], offset 0x6040) = 0x21780 + // MMIO: W.4 0x23c546040 (dptx-phy[0], offset 0x6040) = 0x221780 + // MMIO: R.4 0x23c547040 (dptx-phy[0], offset 0x7040) = 0x21780 + // MMIO: W.4 0x23c547040 (dptx-phy[0], offset 0x7040) = 0x221780 + // MMIO: R.4 0x23c548040 (dptx-phy[0], offset 0x8040) = 0x21780 + // MMIO: W.4 0x23c548040 (dptx-phy[0], offset 0x8040) = 0x221780 + for (u32 loff = DPTX_LANE0_OFFSET; loff < DPTX_LANE_END; + loff += DPTX_LANE_STRIDE) + set32(phy->regs.dptx + loff + 0x40, 0x200000); + + // MMIO: R.4 0x23c545040 (dptx-phy[0], offset 0x5040) = 0x221780 + // MMIO: W.4 0x23c545040 (dptx-phy[0], offset 0x5040) = 0x2a1780 + // MMIO: R.4 0x23c546040 (dptx-phy[0], offset 0x6040) = 0x221780 + // MMIO: W.4 0x23c546040 (dptx-phy[0], offset 0x6040) = 0x2a1780 + // MMIO: R.4 0x23c547040 (dptx-phy[0], offset 0x7040) = 0x221780 + // MMIO: W.4 0x23c547040 (dptx-phy[0], offset 0x7040) = 0x2a1780 + // MMIO: R.4 0x23c548040 (dptx-phy[0], offset 0x8040) = 0x221780 + // MMIO: W.4 0x23c548040 (dptx-phy[0], offset 0x8040) = 0x2a1780 + for (u32 loff = DPTX_LANE0_OFFSET; loff < DPTX_LANE_END; + loff += DPTX_LANE_STRIDE) + set32(phy->regs.dptx + loff + 0x40, 0x080000); + + // MMIO: R.4 0x23c545244 (dptx-phy[0], offset 0x5244) = 0x18 + // MMIO: W.4 0x23c545244 (dptx-phy[0], offset 0x5244) = 0x8 + // MMIO: R.4 0x23c546244 (dptx-phy[0], offset 0x6244) = 0x18 + // MMIO: W.4 0x23c546244 (dptx-phy[0], offset 0x6244) = 0x8 + // MMIO: R.4 0x23c547244 (dptx-phy[0], offset 0x7244) = 0x18 + // MMIO: W.4 0x23c547244 (dptx-phy[0], offset 0x7244) = 0x8 + // MMIO: R.4 0x23c548244 (dptx-phy[0], offset 0x8244) = 0x18 + // MMIO: W.4 0x23c548244 (dptx-phy[0], offset 0x8244) = 0x8 + for (u32 loff = DPTX_LANE0_OFFSET; loff < DPTX_LANE_END; + loff += DPTX_LANE_STRIDE) + clear32(phy->regs.dptx + loff + 0x244, 0x10); + + // MMIO: R.4 0x23c542214 (dptx-phy[0], offset 0x2214) = 0x1e0 + // MMIO: W.4 0x23c542214 (dptx-phy[0], offset 0x2214) = 0x1e1 + set32(phy->regs.dptx + 0x2214, 0x001); + + // MMIO: R.4 0x23c542224 (dptx-phy[0], offset 0x2224) = 0x20086001 + // MMIO: W.4 0x23c542224 (dptx-phy[0], offset 0x2224) = 0x20086000 + clear32(phy->regs.dptx + 0x2224, 0x00000001); + + // MMIO: R.4 0x23c542200 (dptx-phy[0], offset 0x2200) = 0x2000 + // MMIO: W.4 0x23c542200 (dptx-phy[0], offset 0x2200) = 0x2002 + set32(phy->regs.dptx + 0x2200, 0x0002); + + // MMIO: R.4 0x23c541000 (dptx-phy[0], offset 0x1000) = 0xe0000003 + // MMIO: W.4 0x23c541000 (dptx-phy[0], offset 0x1000) = 0xe0000001 + clear32(phy->regs.dptx + 0x1000, 0x00000002); + + // MMIO: R.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x41 + // MMIO: W.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x49 + set32(phy->regs.dptx + 0x4004, 0x08); + + /* TODO: no idea what happens here, supposedly setting/clearing some bits */ + // MMIO: R.4 0x23c544404 (dptx-phy[0], offset 0x4404) = 0x555d444 + readl(phy->regs.dptx + 0x4404); + // MMIO: W.4 0x23c544404 (dptx-phy[0], offset 0x4404) = 0x555d444 + writel(0x555d444, phy->regs.dptx + 0x4404); + // MMIO: R.4 0x23c544404 (dptx-phy[0], offset 0x4404) = 0x555d444 + readl(phy->regs.dptx + 0x4404); + // MMIO: W.4 0x23c544404 (dptx-phy[0], offset 0x4404) = 0x555d444 + writel(0x555d444, phy->regs.dptx + 0x4404); + + dptx_phy_set_active_lane_count(phy, 0); + + // MMIO: R.4 0x23c544200 (dptx-phy[0], offset 0x4200) = 0x4002430 + // MMIO: W.4 0x23c544200 (dptx-phy[0], offset 0x4200) = 0x4002420 + clear32(phy->regs.dptx + 0x4200, 0x0000010); + + // MMIO: R.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000000 + // MMIO: W.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000000 + clear32(phy->regs.dptx + 0x4600, 0x0000001); + // MMIO: R.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000000 + // MMIO: W.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000001 + set32(phy->regs.dptx + 0x4600, 0x0000001); + // MMIO: R.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000001 + // MMIO: W.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000003 + set32(phy->regs.dptx + 0x4600, 0x0000002); + // MMIO: R.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000043 + // MMIO: R.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000043 + // MMIO: W.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000041 + /* TODO: read first to check if the previous set(...,0x2) sticked? */ + readl(phy->regs.dptx + 0x4600); + clear32(phy->regs.dptx + 0x4600, 0x0000001); + + // MMIO: R.4 0x23c544408 (dptx-phy[0], offset 0x4408) = 0x482 + // MMIO: W.4 0x23c544408 (dptx-phy[0], offset 0x4408) = 0x482 + /* TODO: probably a set32 of an already set bit */ + val_4408 = readl(phy->regs.dptx + 0x4408); + if (val_4408 != 0x482 && val_4408 != 0x483) + dev_warn( + phy->dev, + "unexpected initial value at regs.dptx offset 0x4408: 0x%03x\n", + val_4408); + writel(val_4408, phy->regs.dptx + 0x4408); + // MMIO: R.4 0x23c544408 (dptx-phy[0], offset 0x4408) = 0x482 + // MMIO: W.4 0x23c544408 (dptx-phy[0], offset 0x4408) = 0x483 + set32(phy->regs.dptx + 0x4408, 0x001); + + return 0; +} + +static int dptx_phy_deactivate(struct apple_dptx_phy *phy) +{ + return 0; +} + +static int dptx_phy_set_link_rate(struct apple_dptx_phy *phy, u32 link_rate) +{ + u32 sts_1008, sts_1014, val_100c, val_20b0, val_20b4; + + dev_dbg(phy->dev, "set_link_rate(%u)\n", link_rate); + + // MMIO: R.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x49 + // MMIO: W.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x49 + set32(phy->regs.dptx + 0x4004, 0x08); + + // MMIO: R.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x41021ac + // MMIO: W.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x41021ac + clear32(phy->regs.dptx + 0x4000, 0x0000040); + + // MMIO: R.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x49 + // MMIO: W.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x41 + clear32(phy->regs.dptx + 0x4004, 0x08); + + // MMIO: R.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x41021ac + // MMIO: W.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x41021ac + clear32(phy->regs.dptx + 0x4000, 0x2000000); + // MMIO: R.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x41021ac + // MMIO: W.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x41021ac + set32(phy->regs.dptx + 0x4000, 0x1000000); + + // MMIO: R.4 0x23c542200 (dptx-phy[0], offset 0x2200) = 0x2002 + // MMIO: R.4 0x23c542200 (dptx-phy[0], offset 0x2200) = 0x2002 + // MMIO: W.4 0x23c542200 (dptx-phy[0], offset 0x2200) = 0x2000 + /* TODO: what is this read checking for? */ + readl(phy->regs.dptx + 0x2200); + clear32(phy->regs.dptx + 0x2200, 0x0002); + + // MMIO: R.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf000 + // MMIO: W.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf000 + // MMIO: R.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf000 + // MMIO: W.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf008 + /* TODO: what is the setting/clearing? */ + val_100c = readl(phy->regs.dptx + 0x100c); + writel(val_100c, phy->regs.dptx + 0x100c); + set32(phy->regs.dptx + 0x100c, 0x0008); + + // MMIO: R.4 0x23c541014 (dptx-phy[0], offset 0x1014) = 0x1 + sts_1014 = readl(phy->regs.dptx + 0x1014); + if (sts_1014 != 0x1) + dev_dbg(phy->dev, "unexpected?: dptx[0x1014]: %02x\n", sts_1014); + + // MMIO: R.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf008 + // MMIO: W.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf000 + clear32(phy->regs.dptx + 0x100c, 0x0008); + + // MMIO: R.4 0x23c541008 (dptx-phy[0], offset 0x1008) = 0x1 + sts_1008 = readl(phy->regs.dptx + 0x1008); + if (sts_1008 != 0x1) + dev_dbg(phy->dev, "unexpected?: dptx[0x1008]: %02x\n", sts_1008); + + // MMIO: R.4 0x23c542220 (dptx-phy[0], offset 0x2220) = 0x11090a0 + // MMIO: W.4 0x23c542220 (dptx-phy[0], offset 0x2220) = 0x1109020 + clear32(phy->regs.dptx + 0x2220, 0x0000080); + + // MMIO: R.4 0x23c5420b0 (dptx-phy[0], offset 0x20b0) = 0x1e0e01c2 + // MMIO: W.4 0x23c5420b0 (dptx-phy[0], offset 0x20b0) = 0x1e0e01c2 + val_20b0 = readl(phy->regs.dptx + 0x20b0); + /* TODO: what happens on dptx-phy */ + if (phy->hw.type == DPTX_PHY_T6020) + val_20b0 = (val_20b0 & ~0x3ff) | 0x2a3; + writel(val_20b0, phy->regs.dptx + 0x20b0); + + // MMIO: R.4 0x23c5420b4 (dptx-phy[0], offset 0x20b4) = 0x7fffffe + // MMIO: W.4 0x23c5420b4 (dptx-phy[0], offset 0x20b4) = 0x7fffffe + val_20b4 = readl(phy->regs.dptx + 0x20b4); + /* TODO: what happens on dptx-phy */ + if (phy->hw.type == DPTX_PHY_T6020) + val_20b4 = (val_20b4 | 0x4000000) & ~0x0008000; + writel(val_20b4, phy->regs.dptx + 0x20b4); + + // MMIO: R.4 0x23c5420b4 (dptx-phy[0], offset 0x20b4) = 0x7fffffe + // MMIO: W.4 0x23c5420b4 (dptx-phy[0], offset 0x20b4) = 0x7fffffe + val_20b4 = readl(phy->regs.dptx + 0x20b4); + /* TODO: what happens on dptx-phy */ + if (phy->hw.type == DPTX_PHY_T6020) + val_20b4 = (val_20b4 | 0x0000001) & ~0x0000004; + writel(val_20b4, phy->regs.dptx + 0x20b4); + + // MMIO: R.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x654800 + // MMIO: W.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x654800 + /* TODO: unclear */ + set32(phy->regs.dptx + 0x20b8, 0); + // MMIO: R.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x654800 + // MMIO: W.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x654800 + /* TODO: unclear */ + set32(phy->regs.dptx + 0x20b8, 0); + // MMIO: R.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x654800 + // MMIO: W.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x654800 + /* TODO: unclear */ + if (phy->hw.type == DPTX_PHY_T6020) + set32(phy->regs.dptx + 0x20b8, 0x010000); + else + set32(phy->regs.dptx + 0x20b8, 0); + // MMIO: R.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x654800 + // MMIO: W.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x454800 + clear32(phy->regs.dptx + 0x20b8, 0x200000); + + // MMIO: R.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x454800 + // MMIO: W.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x454800 + /* TODO: unclear */ + set32(phy->regs.dptx + 0x20b8, 0); + + // MMIO: R.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x0 + // MMIO: W.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x8 + // MMIO: R.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x8 + // MMIO: W.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0xc + // MMIO: R.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0xc + // MMIO: W.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x4000c + // MMIO: R.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x4000c + // MMIO: W.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0xc + // MMIO: R.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0xc + // MMIO: W.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x8000c + // MMIO: R.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x8000c + // MMIO: W.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0xc + // MMIO: R.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0xc + // MMIO: W.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x8 + // MMIO: R.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x8 + // MMIO: W.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x0 + set32(phy->regs.core + 0xa0, 0x8); + set32(phy->regs.core + 0xa0, 0x4); + set32(phy->regs.core + 0xa0, 0x40000); + clear32(phy->regs.core + 0xa0, 0x40000); + set32(phy->regs.core + 0xa0, 0x80000); + clear32(phy->regs.core + 0xa0, 0x80000); + clear32(phy->regs.core + 0xa0, 0x4); + clear32(phy->regs.core + 0xa0, 0x8); + + // MMIO: R.4 0x23c542000 (dptx-phy[0], offset 0x2000) = 0x2 + // MMIO: W.4 0x23c542000 (dptx-phy[0], offset 0x2000) = 0x2 + /* TODO: unclear */ + set32(phy->regs.dptx + 0x2000, 0x0); + + // MMIO: R.4 0x23c542018 (dptx-phy[0], offset 0x2018) = 0x0 + // MMIO: W.4 0x23c542018 (dptx-phy[0], offset 0x2018) = 0x0 + clear32(phy->regs.dptx + 0x2018, 0x0); + + // MMIO: R.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf000 + // MMIO: W.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf007 + set32(phy->regs.dptx + 0x100c, 0x0007); + // MMIO: R.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf007 + // MMIO: W.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf00f + set32(phy->regs.dptx + 0x100c, 0x0008); + + // MMIO: R.4 0x23c541014 (dptx-phy[0], offset 0x1014) = 0x38f + sts_1014 = readl(phy->regs.dptx + 0x1014); + if (sts_1014 != 0x38f) + dev_dbg(phy->dev, "unexpected?: dptx[0x1014]: %02x\n", sts_1014); + + // MMIO: R.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf00f + // MMIO: W.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf007 + clear32(phy->regs.dptx + 0x100c, 0x0008); + + // MMIO: R.4 0x23c541008 (dptx-phy[0], offset 0x1008) = 0x9 + sts_1008 = readl(phy->regs.dptx + 0x1008); + if (sts_1008 != 0x9) + dev_dbg(phy->dev, "unexpected?: dptx[0x1008]: %02x\n", sts_1008); + + // MMIO: R.4 0x23c542200 (dptx-phy[0], offset 0x2200) = 0x2000 + // MMIO: W.4 0x23c542200 (dptx-phy[0], offset 0x2200) = 0x2002 + set32(phy->regs.dptx + 0x2200, 0x0002); + + // MMIO: R.4 0x23c545010 (dptx-phy[0], offset 0x5010) = 0x18003000 + // MMIO: W.4 0x23c545010 (dptx-phy[0], offset 0x5010) = 0x18003000 + // MMIO: R.4 0x23c546010 (dptx-phy[0], offset 0x6010) = 0x18003000 + // MMIO: W.4 0x23c546010 (dptx-phy[0], offset 0x6010) = 0x18003000 + // MMIO: R.4 0x23c547010 (dptx-phy[0], offset 0x7010) = 0x18003000 + // MMIO: W.4 0x23c547010 (dptx-phy[0], offset 0x7010) = 0x18003000 + // MMIO: R.4 0x23c548010 (dptx-phy[0], offset 0x8010) = 0x18003000 + // MMIO: W.4 0x23c548010 (dptx-phy[0], offset 0x8010) = 0x18003000 + writel(0x18003000, phy->regs.dptx + 0x8010); + for (u32 loff = DPTX_LANE0_OFFSET; loff < DPTX_LANE_END; loff += DPTX_LANE_STRIDE) { + u32 val_l010 = readl(phy->regs.dptx + loff + 0x10); + writel(val_l010, phy->regs.dptx + loff + 0x10); + } + + // MMIO: R.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x41021ac + // MMIO: W.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x51021ac + set32(phy->regs.dptx + 0x4000, 0x1000000); + // MMIO: R.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x51021ac + // MMIO: W.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x71021ac + set32(phy->regs.dptx + 0x4000, 0x2000000); + + // MMIO: R.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x41 + // MMIO: W.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x49 + set32(phy->regs.dptx + 0x4004, 0x08); + + // MMIO: R.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x71021ac + // MMIO: W.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x71021ec + set32(phy->regs.dptx + 0x4000, 0x0000040); + + // MMIO: R.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x49 + // MMIO: W.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x48 + clear32(phy->regs.dptx + 0x4004, 0x01); + + return 0; +} + +static int dptx_phy_set_mode(struct phy *phy, enum phy_mode mode, int submode) +{ + struct apple_dptx_phy *dptx_phy = phy_get_drvdata(phy); + + switch (mode) { + case PHY_MODE_INVALID: + return dptx_phy_deactivate(dptx_phy); + case PHY_MODE_DP: + if (submode < 0 || submode > 5) + return -EINVAL; + return dptx_phy_activate(dptx_phy, submode); + default: + break; + } + + return -EINVAL; +} + +static int dptx_phy_validate(struct phy *phy, enum phy_mode mode, int submode, + union phy_configure_opts *opts_) +{ + struct phy_configure_opts_dp *opts = &opts_->dp; + + if (mode == PHY_MODE_INVALID) { + memset(opts, 0, sizeof(*opts)); + return 0; + } + + if (mode != PHY_MODE_DP) + return -EINVAL; + if (submode < 0 || submode > 5) + return -EINVAL; + + opts->lanes = 4; + opts->link_rate = 8100; + + for (int i = 0; i < 4; ++i) { + opts->voltage[i] = 3; + opts->pre[i] = 3; + } + + return 0; +} + +static int dptx_phy_configure(struct phy *phy, union phy_configure_opts *opts_) +{ + struct phy_configure_opts_dp *opts = &opts_->dp; + struct apple_dptx_phy *dptx_phy = phy_get_drvdata(phy); + enum dptx_phy_link_rate link_rate; + int ret = 0; + + if (opts->set_lanes) { + mutex_lock(&dptx_phy->lock); + ret = dptx_phy_set_active_lane_count(dptx_phy, opts->lanes); + mutex_unlock(&dptx_phy->lock); + } + + if (opts->set_rate) { + switch (opts->link_rate) { + case 1620: + link_rate = DPTX_PHY_LINK_RATE_RBR; + break; + case 2700: + link_rate = DPTX_PHY_LINK_RATE_HBR; + break; + case 5400: + link_rate = DPTX_PHY_LINK_RATE_HBR2; + break; + case 8100: + link_rate = DPTX_PHY_LINK_RATE_HBR3; + break; + case 0: + // TODO: disable! + return 0; + break; + default: + dev_err(dptx_phy->dev, "Unsupported link rate: %d\n", + opts->link_rate); + return -EINVAL; + } + + mutex_lock(&dptx_phy->lock); + ret = dptx_phy_set_link_rate(dptx_phy, link_rate); + mutex_unlock(&dptx_phy->lock); + } + + return ret; +} + +static const struct phy_ops apple_atc_dp_phy_ops = { + .owner = THIS_MODULE, + .configure = dptx_phy_configure, + .validate = dptx_phy_validate, + .set_mode = dptx_phy_set_mode, +}; + +static int dptx_phy_probe(struct platform_device *pdev) +{ + struct apple_dptx_phy *dptx_phy; + struct device *dev = &pdev->dev; + + dptx_phy = devm_kzalloc(dev, sizeof(*dptx_phy), GFP_KERNEL); + if (!dptx_phy) + return -ENOMEM; + + dptx_phy->dev = dev; + dptx_phy->hw = + *(struct apple_dptx_phy_hw *)of_device_get_match_data(dev); + platform_set_drvdata(pdev, dptx_phy); + + mutex_init(&dptx_phy->lock); + + dptx_phy->regs.core = + devm_platform_ioremap_resource_byname(pdev, "core"); + if (IS_ERR(dptx_phy->regs.core)) + return PTR_ERR(dptx_phy->regs.core); + dptx_phy->regs.dptx = + devm_platform_ioremap_resource_byname(pdev, "dptx"); + if (IS_ERR(dptx_phy->regs.dptx)) + return PTR_ERR(dptx_phy->regs.dptx); + + /* create phy */ + dptx_phy->phy_dp = + devm_phy_create(dptx_phy->dev, NULL, &apple_atc_dp_phy_ops); + if (IS_ERR(dptx_phy->phy_dp)) + return PTR_ERR(dptx_phy->phy_dp); + phy_set_drvdata(dptx_phy->phy_dp, dptx_phy); + + dptx_phy->phy_provider = + devm_of_phy_provider_register(dev, of_phy_simple_xlate); + if (IS_ERR(dptx_phy->phy_provider)) + return PTR_ERR(dptx_phy->phy_provider); + + return 0; +} + +static const struct apple_dptx_phy_hw apple_dptx_hw_t6020 = { + .type = DPTX_PHY_T6020, +}; + +static const struct apple_dptx_phy_hw apple_dptx_hw_t8112 = { + .type = DPTX_PHY_T8112, +}; + +static const struct of_device_id dptx_phy_match[] = { + { .compatible = "apple,t6020-dptx-phy", .data = &apple_dptx_hw_t6020 }, + { .compatible = "apple,t8112-dptx-phy", .data = &apple_dptx_hw_t8112 }, + {}, +}; +MODULE_DEVICE_TABLE(of, dptx_phy_match); + +static struct platform_driver dptx_phy_driver = { + .driver = { + .name = "phy-apple-dptx", + .of_match_table = dptx_phy_match, + }, + .probe = dptx_phy_probe, +}; + +module_platform_driver(dptx_phy_driver); + +MODULE_AUTHOR("Janne Grunau "); +MODULE_DESCRIPTION("Apple DP TX PHY driver"); + +MODULE_LICENSE("GPL"); diff --git a/drivers/phy/apple/dptx.h b/drivers/phy/apple/dptx.h new file mode 100644 index 00000000000000..2dd36d753eb357 --- /dev/null +++ b/drivers/phy/apple/dptx.h @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Apple DP TX PHY driver + * + * Copyright (C) The Asahi Linux Contributors + * Author: Janne Grunau + */ + +#ifndef PHY_APPLE_DPTX_H +#define PHY_APPLE_DPTX_H + +enum dptx_phy_link_rate { + DPTX_PHY_LINK_RATE_RBR, + DPTX_PHY_LINK_RATE_HBR, + DPTX_PHY_LINK_RATE_HBR2, + DPTX_PHY_LINK_RATE_HBR3, +}; +#endif /* PHY_APPLE_DPTX_H */ diff --git a/drivers/phy/freescale/phy-fsl-imx8qm-hsio.c b/drivers/phy/freescale/phy-fsl-imx8qm-hsio.c index 977d21d753a59a..279b8ac7822df7 100644 --- a/drivers/phy/freescale/phy-fsl-imx8qm-hsio.c +++ b/drivers/phy/freescale/phy-fsl-imx8qm-hsio.c @@ -251,7 +251,7 @@ static void imx_hsio_configure_clk_pad(struct phy *phy) struct imx_hsio_lane *lane = phy_get_drvdata(phy); struct imx_hsio_priv *priv = lane->priv; - if (strncmp(priv->refclk_pad, "output", 6) == 0) { + if (priv->refclk_pad && strncmp(priv->refclk_pad, "output", 6) == 0) { pll = true; regmap_update_bits(priv->misc, HSIO_CTRL0, HSIO_IOB_A_0_TXOE | HSIO_IOB_A_0_M1M0_MASK, diff --git a/drivers/phy/rockchip/phy-rockchip-samsung-hdptx.c b/drivers/phy/rockchip/phy-rockchip-samsung-hdptx.c index 29de2f7bdae8a3..cafa618d70fdca 100644 --- a/drivers/phy/rockchip/phy-rockchip-samsung-hdptx.c +++ b/drivers/phy/rockchip/phy-rockchip-samsung-hdptx.c @@ -414,6 +414,8 @@ struct rk_hdptx_phy { static const struct ropll_config ropll_tmds_cfg[] = { { 594000000ULL, 124, 124, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 62, 1, 16, 5, 0, 1, 1, 0, 0x20, 0x0c, 1, 0x0e, 0, 0, }, + { 461101250ULL, 97, 97, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 71, 1, 53, 2, 6, + 35, 1, 0, 0x20, 0x0c, 1, 0x0e, 0, 0, }, { 371250000ULL, 155, 155, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 62, 1, 16, 5, 0, 1, 1, 0, 0x20, 0x0c, 1, 0x0e, 0, 0, }, { 297000000ULL, 124, 124, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 62, 1, 16, 5, 0, diff --git a/drivers/pinctrl/meson/pinctrl-amlogic-a4.c b/drivers/pinctrl/meson/pinctrl-amlogic-a4.c index d9e3a8d5932a82..f05d8261624a4e 100644 --- a/drivers/pinctrl/meson/pinctrl-amlogic-a4.c +++ b/drivers/pinctrl/meson/pinctrl-amlogic-a4.c @@ -725,8 +725,9 @@ static u32 aml_bank_pins(struct device_node *np) if (of_parse_phandle_with_fixed_args(np, "gpio-ranges", 3, 0, &of_args)) return 0; - else - return of_args.args[2]; + + of_node_put(of_args.np); + return of_args.args[2]; } static int aml_bank_number(struct device_node *np) @@ -736,8 +737,9 @@ static int aml_bank_number(struct device_node *np) if (of_parse_phandle_with_fixed_args(np, "gpio-ranges", 3, 0, &of_args)) return -EINVAL; - else - return of_args.args[1] >> 8; + + of_node_put(of_args.np); + return of_args.args[1] >> 8; } static unsigned int aml_count_pins(struct device_node *np) diff --git a/drivers/pinctrl/pinctrl-equilibrium.c b/drivers/pinctrl/pinctrl-equilibrium.c index 2d04829b29c997..48b55c5bf8d4fd 100644 --- a/drivers/pinctrl/pinctrl-equilibrium.c +++ b/drivers/pinctrl/pinctrl-equilibrium.c @@ -846,6 +846,7 @@ static int pinbank_init(struct device_node *np, bank->pin_base = spec.args[1]; bank->nr_pins = spec.args[2]; + of_node_put(spec.np); bank->aval_pinmap = readl(bank->membase + REG_AVAIL); bank->id = id; diff --git a/drivers/pinctrl/pinctrl-k230.c b/drivers/pinctrl/pinctrl-k230.c index d716f23d837f7a..20f7c0f70eb77c 100644 --- a/drivers/pinctrl/pinctrl-k230.c +++ b/drivers/pinctrl/pinctrl-k230.c @@ -65,6 +65,7 @@ struct k230_pmx_func { }; struct k230_pinctrl { + struct device *dev; struct pinctrl_desc pctl; struct pinctrl_dev *pctl_dev; struct regmap *regmap_base; @@ -470,7 +471,7 @@ static int k230_pinctrl_parse_groups(struct device_node *np, struct k230_pinctrl *info, unsigned int index) { - struct device *dev = info->pctl_dev->dev; + struct device *dev = info->dev; const __be32 *list; int size, i, ret; @@ -511,7 +512,7 @@ static int k230_pinctrl_parse_functions(struct device_node *np, struct k230_pinctrl *info, unsigned int index) { - struct device *dev = info->pctl_dev->dev; + struct device *dev = info->dev; struct k230_pmx_func *func; struct k230_pin_group *grp; static unsigned int idx, i; @@ -596,6 +597,8 @@ static int k230_pinctrl_probe(struct platform_device *pdev) if (!info) return -ENOMEM; + info->dev = dev; + pctl = &info->pctl; pctl->name = "k230-pinctrl"; diff --git a/drivers/pinctrl/pinctrl-single.c b/drivers/pinctrl/pinctrl-single.c index 998f23d6c3179e..d85e6c1f632186 100644 --- a/drivers/pinctrl/pinctrl-single.c +++ b/drivers/pinctrl/pinctrl-single.c @@ -1359,6 +1359,7 @@ static int pcs_add_gpio_func(struct device_node *node, struct pcs_device *pcs) } range = devm_kzalloc(pcs->dev, sizeof(*range), GFP_KERNEL); if (!range) { + of_node_put(gpiospec.np); ret = -ENOMEM; break; } @@ -1368,6 +1369,7 @@ static int pcs_add_gpio_func(struct device_node *node, struct pcs_device *pcs) mutex_lock(&pcs->mutex); list_add_tail(&range->node, &pcs->gpiofuncs); mutex_unlock(&pcs->mutex); + of_node_put(gpiospec.np); } return ret; } diff --git a/drivers/pinctrl/qcom/pinctrl-sm8250-lpass-lpi.c b/drivers/pinctrl/qcom/pinctrl-sm8250-lpass-lpi.c index 64494a86490e2f..c27452eece3e6b 100644 --- a/drivers/pinctrl/qcom/pinctrl-sm8250-lpass-lpi.c +++ b/drivers/pinctrl/qcom/pinctrl-sm8250-lpass-lpi.c @@ -73,7 +73,7 @@ static const char * const i2s1_ws_groups[] = { "gpio7" }; static const char * const i2s1_data_groups[] = { "gpio8", "gpio9" }; static const char * const wsa_swr_clk_groups[] = { "gpio10" }; static const char * const wsa_swr_data_groups[] = { "gpio11" }; -static const char * const i2s2_data_groups[] = { "gpio12", "gpio12" }; +static const char * const i2s2_data_groups[] = { "gpio12", "gpio13" }; static const struct lpi_pingroup sm8250_groups[] = { LPI_PINGROUP(0, 0, swr_tx_clk, qua_mi2s_sclk, _, _), diff --git a/drivers/platform/chrome/cros_ec_lightbar.c b/drivers/platform/chrome/cros_ec_lightbar.c index 8352e97327911f..3702baff5d4f17 100644 --- a/drivers/platform/chrome/cros_ec_lightbar.c +++ b/drivers/platform/chrome/cros_ec_lightbar.c @@ -126,7 +126,7 @@ static int get_lightbar_version(struct cros_ec_dev *ec, param = (struct ec_params_lightbar *)msg->data; param->cmd = LIGHTBAR_CMD_VERSION; msg->outsize = sizeof(param->cmd); - msg->result = sizeof(resp->version); + msg->insize = sizeof(resp->version); ret = cros_ec_cmd_xfer_status(ec->ec_dev, msg); if (ret < 0 && ret != -EINVAL) { ret = 0; diff --git a/drivers/platform/chrome/cros_typec_switch.c b/drivers/platform/chrome/cros_typec_switch.c index 8d7c34abb0a128..d8a28d4e51a85e 100644 --- a/drivers/platform/chrome/cros_typec_switch.c +++ b/drivers/platform/chrome/cros_typec_switch.c @@ -230,20 +230,20 @@ static int cros_typec_register_switches(struct cros_typec_switch_data *sdata) adev = to_acpi_device_node(fwnode); if (!adev) { - dev_err(fwnode->dev, "Couldn't get ACPI device handle\n"); + dev_err(dev, "Couldn't get ACPI device handle for %pfwP\n", fwnode); ret = -ENODEV; goto err_switch; } ret = acpi_evaluate_integer(adev->handle, "_ADR", NULL, &index); if (ACPI_FAILURE(ret)) { - dev_err(fwnode->dev, "_ADR wasn't evaluated\n"); + dev_err(dev, "_ADR wasn't evaluated for %pfwP\n", fwnode); ret = -ENODATA; goto err_switch; } if (index >= EC_USB_PD_MAX_PORTS) { - dev_err(fwnode->dev, "Invalid port index number: %llu\n", index); + dev_err(dev, "%pfwP: Invalid port index number: %llu\n", fwnode, index); ret = -EINVAL; goto err_switch; } diff --git a/drivers/platform/x86/amd/pmf/core.c b/drivers/platform/x86/amd/pmf/core.c index 8fc293c9c5380d..15c27edfb6d855 100644 --- a/drivers/platform/x86/amd/pmf/core.c +++ b/drivers/platform/x86/amd/pmf/core.c @@ -314,6 +314,61 @@ int amd_pmf_init_metrics_table(struct amd_pmf_dev *dev) return 0; } +static int amd_pmf_reinit_ta(struct amd_pmf_dev *pdev) +{ + bool status; + int ret, i; + + for (i = 0; i < ARRAY_SIZE(amd_pmf_ta_uuid); i++) { + ret = amd_pmf_tee_init(pdev, &amd_pmf_ta_uuid[i]); + if (ret) { + dev_err(pdev->dev, "TEE init failed for UUID[%d] ret: %d\n", i, ret); + return ret; + } + + ret = amd_pmf_start_policy_engine(pdev); + dev_dbg(pdev->dev, "start policy engine ret: %d (UUID idx: %d)\n", ret, i); + status = ret == TA_PMF_TYPE_SUCCESS; + if (status) + break; + amd_pmf_tee_deinit(pdev); + } + + return 0; +} + +static int amd_pmf_restore_handler(struct device *dev) +{ + struct amd_pmf_dev *pdev = dev_get_drvdata(dev); + int ret; + + if (pdev->buf) { + ret = amd_pmf_set_dram_addr(pdev, false); + if (ret) + return ret; + } + + if (pdev->smart_pc_enabled) + amd_pmf_reinit_ta(pdev); + + return 0; +} + +static int amd_pmf_freeze_handler(struct device *dev) +{ + struct amd_pmf_dev *pdev = dev_get_drvdata(dev); + + if (!pdev->smart_pc_enabled) + return 0; + + cancel_delayed_work_sync(&pdev->pb_work); + /* Clear all TEE resources */ + amd_pmf_tee_deinit(pdev); + pdev->session_id = 0; + + return 0; +} + static int amd_pmf_suspend_handler(struct device *dev) { struct amd_pmf_dev *pdev = dev_get_drvdata(dev); @@ -347,7 +402,12 @@ static int amd_pmf_resume_handler(struct device *dev) return 0; } -static DEFINE_SIMPLE_DEV_PM_OPS(amd_pmf_pm, amd_pmf_suspend_handler, amd_pmf_resume_handler); +static const struct dev_pm_ops amd_pmf_pm = { + .suspend = amd_pmf_suspend_handler, + .resume = amd_pmf_resume_handler, + .freeze = amd_pmf_freeze_handler, + .restore = amd_pmf_restore_handler, +}; static void amd_pmf_init_features(struct amd_pmf_dev *dev) { diff --git a/drivers/platform/x86/amd/pmf/pmf.h b/drivers/platform/x86/amd/pmf/pmf.h index 9144c8c3bbaf2f..513a6309ce1303 100644 --- a/drivers/platform/x86/amd/pmf/pmf.h +++ b/drivers/platform/x86/amd/pmf/pmf.h @@ -129,6 +129,12 @@ struct cookie_header { typedef void (*apmf_event_handler_t)(acpi_handle handle, u32 event, void *data); +static const uuid_t amd_pmf_ta_uuid[] __used = { UUID_INIT(0xd9b39bf2, 0x66bd, 0x4154, 0xaf, 0xb8, + 0x8a, 0xcc, 0x2b, 0x2b, 0x60, 0xd6), + UUID_INIT(0x6fd93b77, 0x3fb8, 0x524d, 0xb1, 0x2d, + 0xc5, 0x29, 0xb1, 0x3d, 0x85, 0x43), + }; + /* APTS PMF BIOS Interface */ struct amd_pmf_apts_output { u16 table_version; @@ -895,4 +901,8 @@ void amd_pmf_populate_ta_inputs(struct amd_pmf_dev *dev, struct ta_pmf_enact_tab void amd_pmf_dump_ta_inputs(struct amd_pmf_dev *dev, struct ta_pmf_enact_table *in); int amd_pmf_invoke_cmd_enact(struct amd_pmf_dev *dev); +int amd_pmf_tee_init(struct amd_pmf_dev *dev, const uuid_t *uuid); +void amd_pmf_tee_deinit(struct amd_pmf_dev *dev); +int amd_pmf_start_policy_engine(struct amd_pmf_dev *dev); + #endif /* PMF_H */ diff --git a/drivers/platform/x86/amd/pmf/tee-if.c b/drivers/platform/x86/amd/pmf/tee-if.c index 0abce76f89ffe5..95ceb906a5f39a 100644 --- a/drivers/platform/x86/amd/pmf/tee-if.c +++ b/drivers/platform/x86/amd/pmf/tee-if.c @@ -27,12 +27,6 @@ module_param(pb_side_load, bool, 0444); MODULE_PARM_DESC(pb_side_load, "Sideload policy binaries debug policy failures"); #endif -static const uuid_t amd_pmf_ta_uuid[] = { UUID_INIT(0xd9b39bf2, 0x66bd, 0x4154, 0xaf, 0xb8, 0x8a, - 0xcc, 0x2b, 0x2b, 0x60, 0xd6), - UUID_INIT(0x6fd93b77, 0x3fb8, 0x524d, 0xb1, 0x2d, 0xc5, - 0x29, 0xb1, 0x3d, 0x85, 0x43), - }; - static const char *amd_pmf_uevent_as_str(unsigned int state) { switch (state) { @@ -324,7 +318,7 @@ static void amd_pmf_invoke_cmd(struct work_struct *work) schedule_delayed_work(&dev->pb_work, msecs_to_jiffies(pb_actions_ms)); } -static int amd_pmf_start_policy_engine(struct amd_pmf_dev *dev) +int amd_pmf_start_policy_engine(struct amd_pmf_dev *dev) { struct cookie_header *header; int res; @@ -480,7 +474,7 @@ static int amd_pmf_register_input_device(struct amd_pmf_dev *dev) return 0; } -static int amd_pmf_tee_init(struct amd_pmf_dev *dev, const uuid_t *uuid) +int amd_pmf_tee_init(struct amd_pmf_dev *dev, const uuid_t *uuid) { u32 size; int ret; @@ -528,7 +522,7 @@ static int amd_pmf_tee_init(struct amd_pmf_dev *dev, const uuid_t *uuid) return ret; } -static void amd_pmf_tee_deinit(struct amd_pmf_dev *dev) +void amd_pmf_tee_deinit(struct amd_pmf_dev *dev) { if (!dev->tee_ctx) return; diff --git a/drivers/platform/x86/hp/hp-wmi.c b/drivers/platform/x86/hp/hp-wmi.c index f4ea1ea05997b6..24d065ddfc6ae4 100644 --- a/drivers/platform/x86/hp/hp-wmi.c +++ b/drivers/platform/x86/hp/hp-wmi.c @@ -53,6 +53,66 @@ MODULE_ALIAS("wmi:5FB7F034-2C63-45E9-BE91-3D44E2C707E4"); #define zero_if_sup(tmp) (zero_insize_support?0:sizeof(tmp)) // use when zero insize is required +enum hp_thermal_profile_omen_v0 { + HP_OMEN_V0_THERMAL_PROFILE_DEFAULT = 0x00, + HP_OMEN_V0_THERMAL_PROFILE_PERFORMANCE = 0x01, + HP_OMEN_V0_THERMAL_PROFILE_COOL = 0x02, +}; + +enum hp_thermal_profile_omen_v1 { + HP_OMEN_V1_THERMAL_PROFILE_DEFAULT = 0x30, + HP_OMEN_V1_THERMAL_PROFILE_PERFORMANCE = 0x31, + HP_OMEN_V1_THERMAL_PROFILE_COOL = 0x50, +}; + +enum hp_thermal_profile_omen_flags { + HP_OMEN_EC_FLAGS_TURBO = 0x04, + HP_OMEN_EC_FLAGS_NOTIMER = 0x02, + HP_OMEN_EC_FLAGS_JUSTSET = 0x01, +}; + +enum hp_thermal_profile_victus { + HP_VICTUS_THERMAL_PROFILE_DEFAULT = 0x00, + HP_VICTUS_THERMAL_PROFILE_PERFORMANCE = 0x01, + HP_VICTUS_THERMAL_PROFILE_QUIET = 0x03, +}; + +enum hp_thermal_profile_victus_s { + HP_VICTUS_S_THERMAL_PROFILE_DEFAULT = 0x00, + HP_VICTUS_S_THERMAL_PROFILE_PERFORMANCE = 0x01, +}; + +enum hp_thermal_profile { + HP_THERMAL_PROFILE_PERFORMANCE = 0x00, + HP_THERMAL_PROFILE_DEFAULT = 0x01, + HP_THERMAL_PROFILE_COOL = 0x02, + HP_THERMAL_PROFILE_QUIET = 0x03, +}; + +struct thermal_profile_params { + u8 performance; + u8 balanced; + u8 low_power; +}; + +static const struct thermal_profile_params victus_s_thermal_params = { + .performance = HP_VICTUS_S_THERMAL_PROFILE_PERFORMANCE, + .balanced = HP_VICTUS_S_THERMAL_PROFILE_DEFAULT, + .low_power = HP_VICTUS_S_THERMAL_PROFILE_DEFAULT, +}; + +static const struct thermal_profile_params omen_v1_thermal_params = { + .performance = HP_OMEN_V1_THERMAL_PROFILE_PERFORMANCE, + .balanced = HP_OMEN_V1_THERMAL_PROFILE_DEFAULT, + .low_power = HP_OMEN_V1_THERMAL_PROFILE_DEFAULT, +}; + +/* + * A generic pointer for the currently-active board's thermal profile + * parameters. + */ +static struct thermal_profile_params *active_thermal_profile_params; + /* DMI board names of devices that should use the omen specific path for * thermal profiles. * This was obtained by taking a look in the windows omen command center @@ -99,12 +159,40 @@ static const char * const victus_thermal_profile_boards[] = { }; /* DMI Board names of Victus 16-r and Victus 16-s laptops */ -static const char * const victus_s_thermal_profile_boards[] = { - "8BBE", "8BD4", "8BD5", - "8C78", "8C99", "8C9C", - "8D41", +static const struct dmi_system_id victus_s_thermal_profile_boards[] __initconst = { + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BBE") }, + .driver_data = (void *)&victus_s_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BD4") }, + .driver_data = (void *)&victus_s_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BD5") }, + .driver_data = (void *)&victus_s_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8C78") }, + .driver_data = (void *)&omen_v1_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8C99") }, + .driver_data = (void *)&victus_s_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8C9C") }, + .driver_data = (void *)&victus_s_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8D41") }, + .driver_data = (void *)&victus_s_thermal_params, + }, + {}, }; +static bool is_victus_s_board; + enum hp_wmi_radio { HPWMI_WIFI = 0x0, HPWMI_BLUETOOTH = 0x1, @@ -225,42 +313,6 @@ enum hp_wireless2_bits { HPWMI_POWER_FW_OR_HW = HPWMI_POWER_BIOS | HPWMI_POWER_HARD, }; -enum hp_thermal_profile_omen_v0 { - HP_OMEN_V0_THERMAL_PROFILE_DEFAULT = 0x00, - HP_OMEN_V0_THERMAL_PROFILE_PERFORMANCE = 0x01, - HP_OMEN_V0_THERMAL_PROFILE_COOL = 0x02, -}; - -enum hp_thermal_profile_omen_v1 { - HP_OMEN_V1_THERMAL_PROFILE_DEFAULT = 0x30, - HP_OMEN_V1_THERMAL_PROFILE_PERFORMANCE = 0x31, - HP_OMEN_V1_THERMAL_PROFILE_COOL = 0x50, -}; - -enum hp_thermal_profile_omen_flags { - HP_OMEN_EC_FLAGS_TURBO = 0x04, - HP_OMEN_EC_FLAGS_NOTIMER = 0x02, - HP_OMEN_EC_FLAGS_JUSTSET = 0x01, -}; - -enum hp_thermal_profile_victus { - HP_VICTUS_THERMAL_PROFILE_DEFAULT = 0x00, - HP_VICTUS_THERMAL_PROFILE_PERFORMANCE = 0x01, - HP_VICTUS_THERMAL_PROFILE_QUIET = 0x03, -}; - -enum hp_thermal_profile_victus_s { - HP_VICTUS_S_THERMAL_PROFILE_DEFAULT = 0x00, - HP_VICTUS_S_THERMAL_PROFILE_PERFORMANCE = 0x01, -}; - -enum hp_thermal_profile { - HP_THERMAL_PROFILE_PERFORMANCE = 0x00, - HP_THERMAL_PROFILE_DEFAULT = 0x01, - HP_THERMAL_PROFILE_COOL = 0x02, - HP_THERMAL_PROFILE_QUIET = 0x03, -}; - #define IS_HWBLOCKED(x) ((x & HPWMI_POWER_FW_OR_HW) != HPWMI_POWER_FW_OR_HW) #define IS_SWBLOCKED(x) !(x & HPWMI_POWER_SOFT) @@ -1581,15 +1633,8 @@ static int platform_profile_victus_set_ec(enum platform_profile_option profile) static bool is_victus_s_thermal_profile(void) { - const char *board_name; - - board_name = dmi_get_system_info(DMI_BOARD_NAME); - if (!board_name) - return false; - - return match_string(victus_s_thermal_profile_boards, - ARRAY_SIZE(victus_s_thermal_profile_boards), - board_name) >= 0; + /* Initialised in driver init, hence safe to use here */ + return is_victus_s_board; } static int victus_s_gpu_thermal_profile_get(bool *ctgp_enable, @@ -1672,25 +1717,30 @@ static int victus_s_set_cpu_pl1_pl2(u8 pl1, u8 pl2) static int platform_profile_victus_s_set_ec(enum platform_profile_option profile) { + struct thermal_profile_params *params; bool gpu_ctgp_enable, gpu_ppab_enable; u8 gpu_dstate; /* Test shows 1 = 100%, 2 = 50%, 3 = 25%, 4 = 12.5% */ int err, tp; + params = active_thermal_profile_params; + if (!params) + return -ENODEV; + switch (profile) { case PLATFORM_PROFILE_PERFORMANCE: - tp = HP_VICTUS_S_THERMAL_PROFILE_PERFORMANCE; + tp = params->performance; gpu_ctgp_enable = true; gpu_ppab_enable = true; gpu_dstate = 1; break; case PLATFORM_PROFILE_BALANCED: - tp = HP_VICTUS_S_THERMAL_PROFILE_DEFAULT; + tp = params->balanced; gpu_ctgp_enable = false; gpu_ppab_enable = true; gpu_dstate = 1; break; case PLATFORM_PROFILE_LOW_POWER: - tp = HP_VICTUS_S_THERMAL_PROFILE_DEFAULT; + tp = params->low_power; gpu_ctgp_enable = false; gpu_ppab_enable = false; gpu_dstate = 1; @@ -2227,6 +2277,26 @@ static int hp_wmi_hwmon_init(void) return 0; } +static void __init setup_active_thermal_profile_params(void) +{ + const struct dmi_system_id *id; + + /* + * Currently only victus_s devices use the + * active_thermal_profile_params + */ + id = dmi_first_match(victus_s_thermal_profile_boards); + if (id) { + /* + * Marking this boolean is required to ensure that + * is_victus_s_thermal_profile() behaves like a valid + * wrapper. + */ + is_victus_s_board = true; + active_thermal_profile_params = id->driver_data; + } +} + static int __init hp_wmi_init(void) { int event_capable = wmi_has_guid(HPWMI_EVENT_GUID); @@ -2254,6 +2324,11 @@ static int __init hp_wmi_init(void) goto err_destroy_input; } + /* + * Setup active board's thermal profile parameters before + * starting platform driver probe. + */ + setup_active_thermal_profile_params(); err = platform_driver_probe(&hp_wmi_driver, hp_wmi_bios_setup); if (err) goto err_unregister_device; diff --git a/drivers/platform/x86/intel/int0002_vgpio.c b/drivers/platform/x86/intel/int0002_vgpio.c index 6f5629dc3f8dbc..562e8802564367 100644 --- a/drivers/platform/x86/intel/int0002_vgpio.c +++ b/drivers/platform/x86/intel/int0002_vgpio.c @@ -206,8 +206,8 @@ static int int0002_probe(struct platform_device *pdev) * FIXME: augment this if we managed to pull handling of shared * IRQs into gpiolib. */ - ret = devm_request_irq(dev, irq, int0002_irq, - IRQF_ONESHOT | IRQF_SHARED, "INT0002", chip); + ret = devm_request_irq(dev, irq, int0002_irq, IRQF_SHARED, "INT0002", + chip); if (ret) { dev_err(dev, "Error requesting IRQ %d: %d\n", irq, ret); return ret; diff --git a/drivers/pmdomain/apple/Makefile b/drivers/pmdomain/apple/Makefile index 53665af630be22..006d4c0071ab89 100644 --- a/drivers/pmdomain/apple/Makefile +++ b/drivers/pmdomain/apple/Makefile @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_APPLE_PMGR_PWRSTATE) += pmgr-pwrstate.o +obj-$(CONFIG_APPLE_PMP) += pmp-report.o diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index 82c33cf727a825..a92db8cf88bb90 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -21,7 +21,8 @@ #define APPLE_PMGR_AUTO_ENABLE BIT(28) #define APPLE_PMGR_PS_AUTO GENMASK(27, 24) #define APPLE_PMGR_PS_MIN GENMASK(19, 16) -#define APPLE_PMGR_PARENT_OFF BIT(11) +#define APPLE_PMGR_PS_RESET BIT(12) +#define APPLE_PMGR_BUSY BIT(11) #define APPLE_PMGR_DEV_DISABLE BIT(10) #define APPLE_PMGR_WAS_CLKGATED BIT(9) #define APPLE_PMGR_WAS_PWRGATED BIT(8) @@ -44,6 +45,9 @@ struct apple_pmgr_ps { struct regmap *regmap; u32 offset; u32 min_state; + bool force_disable; + bool force_reset; + bool externally_clocked; }; #define genpd_to_apple_pmgr_ps(_genpd) container_of(_genpd, struct apple_pmgr_ps, genpd) @@ -53,7 +57,7 @@ static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool a { int ret; struct apple_pmgr_ps *ps = genpd_to_apple_pmgr_ps(genpd); - u32 reg; + u32 reg, cur; ret = regmap_read(ps->regmap, ps->offset, ®); if (ret < 0) @@ -64,24 +68,57 @@ static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool a dev_err(ps->dev, "PS %s: powering off with RESET active\n", genpd->name); - reg &= ~(APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS | APPLE_PMGR_PS_TARGET); + if (pstate != APPLE_PMGR_PS_ACTIVE && (ps->force_disable || ps->force_reset)) { + u32 reg_pre = reg & ~(APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS); + + if (ps->force_disable) + reg_pre |= APPLE_PMGR_DEV_DISABLE; + if (ps->force_reset) + reg_pre |= APPLE_PMGR_PS_RESET; + + regmap_write(ps->regmap, ps->offset, reg_pre); + + ret = regmap_read_poll_timeout_atomic( + ps->regmap, ps->offset, cur, + (cur & (APPLE_PMGR_DEV_DISABLE | APPLE_PMGR_PS_RESET)) == + (reg_pre & (APPLE_PMGR_DEV_DISABLE | APPLE_PMGR_PS_RESET)), 1, + APPLE_PMGR_PS_SET_TIMEOUT); + + if (ret < 0) + dev_err(ps->dev, "PS %s: Failed to set reset/disable bits (now: 0x%x)\n", + genpd->name, reg); + } + + reg &= ~(APPLE_PMGR_DEV_DISABLE | APPLE_PMGR_PS_RESET | + APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS | APPLE_PMGR_PS_TARGET); reg |= FIELD_PREP(APPLE_PMGR_PS_TARGET, pstate); dev_dbg(ps->dev, "PS %s: pwrstate = 0x%x: 0x%x\n", genpd->name, pstate, reg); regmap_write(ps->regmap, ps->offset, reg); - ret = regmap_read_poll_timeout_atomic( - ps->regmap, ps->offset, reg, - (FIELD_GET(APPLE_PMGR_PS_ACTUAL, reg) == pstate), 1, - APPLE_PMGR_PS_SET_TIMEOUT); + if (ps->externally_clocked && pstate == APPLE_PMGR_PS_ACTIVE) { + /* + * If this clock domain requires an external clock, then + * consider the "clock gated" state to be good enough. + */ + ret = regmap_read_poll_timeout_atomic( + ps->regmap, ps->offset, cur, + FIELD_GET(APPLE_PMGR_PS_ACTUAL, cur) >= APPLE_PMGR_PS_CLKGATE, 1, + APPLE_PMGR_PS_SET_TIMEOUT); + } else { + ret = regmap_read_poll_timeout_atomic( + ps->regmap, ps->offset, cur, + FIELD_GET(APPLE_PMGR_PS_ACTUAL, cur) == pstate, 1, + APPLE_PMGR_PS_SET_TIMEOUT); + } + if (ret < 0) dev_err(ps->dev, "PS %s: Failed to reach power state 0x%x (now: 0x%x)\n", genpd->name, pstate, reg); if (auto_enable) { /* Not all devices implement this; this is a no-op where not implemented. */ - reg &= ~APPLE_PMGR_FLAGS; reg |= APPLE_PMGR_AUTO_ENABLE; regmap_write(ps->regmap, ps->offset, reg); } @@ -234,6 +271,15 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) regmap_update_bits(regmap, ps->offset, APPLE_PMGR_FLAGS | APPLE_PMGR_PS_MIN, FIELD_PREP(APPLE_PMGR_PS_MIN, ps->min_state)); + if (of_property_read_bool(node, "apple,force-disable")) + ps->force_disable = true; + + if (of_property_read_bool(node, "apple,force-reset")) + ps->force_reset = true; + + if (of_property_read_bool(node, "apple,externally-clocked")) + ps->externally_clocked = true; + active = apple_pmgr_ps_is_active(ps); if (of_property_read_bool(node, "apple,always-on")) { ps->genpd.flags |= GENPD_FLAG_ALWAYS_ON; @@ -242,6 +288,8 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) /* Turn it on so pm_genpd_init does not fail */ active = apple_pmgr_ps_power_on(&ps->genpd) == 0; } + } else if (active) { + ps->genpd.flags |= GENPD_FLAG_DEFER_OFF | GENPD_FLAG_ACTIVE_WAKEUP; } /* Turn on auto-PM if the domain is already on */ diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c new file mode 100644 index 00000000000000..af894adb7829f4 --- /dev/null +++ b/drivers/pmdomain/apple/pmp-report.c @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SoC PMP power state reporting driver + * + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include + +#define PMP_REPORT_READY 0x1 + +struct apple_pmp_report_offsets { + u32 tgt_read; + u32 tgt_write; + u32 actual; + u32 status; +}; + +struct apple_pmp_report { + struct device *dev; + const struct apple_pmp_report_offsets *offsets; + void __iomem *base; + spinlock_t lock; +}; + +static int apple_pmp_report_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + struct apple_pmp_report *rep; + int ret; + + rep = devm_kzalloc(dev, sizeof(*rep), GFP_KERNEL); + if (!rep) + return -ENOMEM; + + rep->dev = dev; + rep->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(rep->base)) + return PTR_ERR(rep->base); + rep->offsets = of_device_get_match_data(dev); + dev_set_drvdata(dev, rep); + ret = of_platform_populate(np, NULL, NULL, dev); + if (ret) + return dev_err_probe(dev, ret, "failed to create child devices\n"); + + return 0; +} + +static const struct apple_pmp_report_offsets apple_pmp_offsets_t600x = { + .tgt_read = 0xf80, + .tgt_write = 0x107c0, + .actual = 0x1000, + .status = 0x10, +}; + +static const struct apple_pmp_report_offsets apple_pmp_offsets_t602x = { + .tgt_read = 0x2000, + .tgt_write = 0x11000, + .actual = 0x2080, + .status = 0x10, +}; + +static const struct apple_pmp_report_offsets apple_pmp_offsets_t8112 = { + .tgt_read = 0xa00, + .tgt_write = 0x10500, + .actual = 0xa40, + .status = 0x10, +}; + +static const struct of_device_id apple_pmp_report_of_match[] = { + { .compatible = "apple,t6000-pmp-v2-report", .data = &apple_pmp_offsets_t600x }, + { .compatible = "apple,t6020-pmp-v2-report", .data = &apple_pmp_offsets_t602x }, + { .compatible = "apple,t8112-pmp-v2-report", .data = &apple_pmp_offsets_t8112 }, + {} +}; + +static struct platform_driver apple_pmp_report_driver = { + .probe = apple_pmp_report_probe, + .driver = { + .name = "apple-pmp-report", + .of_match_table = apple_pmp_report_of_match, + }, +}; + +struct apple_pmp_report_entry { + struct device *dev; + struct generic_pm_domain genpd; + u32 id; +}; + +#define genpd_to_apple_pmp_report_entry(_genpd) \ + container_of(_genpd, struct apple_pmp_report_entry, genpd) + +static int apple_pmp_report_set_state(struct generic_pm_domain *genpd, bool enable) +{ + struct apple_pmp_report_entry *ent = genpd_to_apple_pmp_report_entry(genpd); + struct apple_pmp_report *rep = dev_get_drvdata(ent->dev->parent); + u64 bit_val = 1 << ent->id; + u64 val; + unsigned long flags; + + spin_lock_irqsave(&rep->lock, flags); + val = readq(rep->base + rep->offsets->tgt_read); + val &= ~bit_val; + if (enable) + val |= bit_val; + writeq(val, rep->base + rep->offsets->tgt_write); + spin_unlock_irqrestore(&rep->lock, flags); + val = readq(rep->base + rep->offsets->status); + if ((val & PMP_REPORT_READY) == 0) + return 0; + return readq_poll_timeout_atomic( + rep->base + rep->offsets->actual, + val, + !!(val & bit_val) == !!enable, + 100, + 50000); +} + +static int apple_pmp_report_entry_power_on(struct generic_pm_domain *genpd) +{ + return apple_pmp_report_set_state(genpd, true); +} + +static int apple_pmp_report_entry_power_off(struct generic_pm_domain *genpd) +{ + return apple_pmp_report_set_state(genpd, false); +} + +static int apple_pmp_report_entry_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *node = dev->of_node; + struct apple_pmp_report_entry *ent; + int ret; + const char *name; + struct of_phandle_iterator it; + + ent = devm_kzalloc(dev, sizeof(*ent), GFP_KERNEL); + if (!ent) + return -ENOMEM; + + ent->dev = dev; + + ret = of_property_read_u32(node, "reg", &ent->id); + if (ret) + return dev_err_probe(dev, ret, "missing reg property\n"); + + ret = of_property_read_string(node, "label", &name); + if (ret < 0) + return dev_err_probe(dev, ret, "missing label property\n"); + + if (of_property_read_bool(node, "apple,always-on")) + ent->genpd.flags |= GENPD_FLAG_ACTIVE_WAKEUP; + ent->genpd.name = name; + ent->genpd.power_on = apple_pmp_report_entry_power_on; + ent->genpd.power_off = apple_pmp_report_entry_power_off; + + ret = pm_genpd_init(&ent->genpd, NULL, true); + if (ret) + return dev_err_probe(dev, ret, "pm_genpd_init failed\n"); + + ret = of_genpd_add_provider_simple(node, &ent->genpd); + if (ret) + return dev_err_probe(dev, ret, "of_genpd_add_provider_simple failed\n"); + + of_for_each_phandle(&it, ret, node, "power-domains", "#power-domain-cells", -1) { + struct of_phandle_args parent, child; + + parent.np = it.node; + parent.args_count = of_phandle_iterator_args(&it, parent.args, MAX_PHANDLE_ARGS); + child.np = node; + child.args_count = 0; + ret = of_genpd_add_subdomain(&parent, &child); + + if (ret == -EPROBE_DEFER) { + of_node_put(parent.np); + goto err_remove; + } else if (ret < 0) { + dev_err(dev, "failed to add to parent domain: %d (%s -> %s)\n", + ret, it.node->name, node->name); + of_node_put(parent.np); + goto err_remove; + } + } + + pm_genpd_remove_device(dev); + + return 0; +err_remove: + of_genpd_del_provider(node); + pm_genpd_remove(&ent->genpd); + return ret; +} + +static const struct of_device_id apple_pmp_report_entry_of_match[] = { + { .compatible = "apple,t6000-pmp-v2-report-entry" }, + {} +}; + +static struct platform_driver apple_pmp_report_entry_driver = { + .probe = apple_pmp_report_entry_probe, + .driver = { + .name = "apple-pmp-report-entry", + .of_match_table = apple_pmp_report_entry_of_match, + }, +}; + +MODULE_DEVICE_TABLE(of, apple_pmp_report_of_match); +MODULE_DEVICE_TABLE(of, apple_pmp_report_entry_of_match); + +static int __init apple_pmp_report_init(void) +{ + platform_driver_register(&apple_pmp_report_entry_driver); + platform_driver_register(&apple_pmp_report_driver); + return 0; +} + +static void __exit apple_pmp_report_exit(void) +{ + platform_driver_unregister(&apple_pmp_report_entry_driver); + platform_driver_unregister(&apple_pmp_report_driver); +} + +module_init(apple_pmp_report_init); +module_exit(apple_pmp_report_exit); + +MODULE_DESCRIPTION("PMP power state reporting driver for Apple SoCs"); diff --git a/drivers/pmdomain/core.c b/drivers/pmdomain/core.c index bf82775f6a6738..1d3493c57559fd 100644 --- a/drivers/pmdomain/core.c +++ b/drivers/pmdomain/core.c @@ -7,6 +7,7 @@ #define pr_fmt(fmt) "PM: " fmt #include +#include #include #include #include @@ -188,6 +189,7 @@ static const struct genpd_lock_ops genpd_raw_spin_ops = { #define genpd_is_dev_name_fw(genpd) (genpd->flags & GENPD_FLAG_DEV_NAME_FW) #define genpd_is_no_sync_state(genpd) (genpd->flags & GENPD_FLAG_NO_SYNC_STATE) #define genpd_is_no_stay_on(genpd) (genpd->flags & GENPD_FLAG_NO_STAY_ON) +#define genpd_is_defer_off(genpd) (genpd->flags & GENPD_FLAG_DEFER_OFF) static inline bool irq_safe_dev_in_sleep_domain(struct device *dev, const struct generic_pm_domain *genpd) @@ -941,6 +943,27 @@ static void genpd_queue_power_off_work(struct generic_pm_domain *genpd) queue_work(pm_wq, &genpd->power_off_work); } +/** + * genpd_must_defer - Check whether the genpd cannot be safely powered off. + * @genpd: PM domain about to be powered down. + * @one_dev_probing: True if we are being called from RPM callbacks on a device that + * is probing, to allow poweroff if that device is the sole remaining consumer probing. + * + * Returns true if the @genpd has the GENPD_FLAG_DEFER_OFF flag and there + * are any consumer devices which either do not exist yet (only represented + * by fwlinks) or whose drivers have not probed yet. + */ +static bool genpd_must_defer(struct generic_pm_domain *genpd, bool one_dev_probing) +{ + if (genpd_is_defer_off(genpd) && genpd->has_provider) { + int absent = fw_devlink_count_absent_consumers(genpd->provider); + + if (absent > (one_dev_probing ? 1 : 0)) + return true; + } + return false; +} + /** * genpd_power_off - Remove power from a given PM domain. * @genpd: PM domain to power down. @@ -954,7 +977,7 @@ static void genpd_queue_power_off_work(struct generic_pm_domain *genpd) * have been powered down, remove power from @genpd. */ static void genpd_power_off(struct generic_pm_domain *genpd, bool one_dev_on, - unsigned int depth) + bool one_dev_probing, unsigned int depth) { struct pm_domain_data *pdd; struct gpd_link *link; @@ -1002,6 +1025,14 @@ static void genpd_power_off(struct generic_pm_domain *genpd, bool one_dev_on, if (not_suspended > 1 || (not_suspended == 1 && !one_dev_on)) return; + /* + * Do not allow PM domain to be powered off if it is marked + * as GENPD_FLAG_DEFER_OFF and there are consumer devices + * which have not probed yet. + */ + if (genpd_must_defer(genpd, one_dev_probing)) + return; + if (genpd->gov && genpd->gov->power_down_ok) { if (!genpd->gov->power_down_ok(&genpd->domain)) return; @@ -1027,7 +1058,7 @@ static void genpd_power_off(struct generic_pm_domain *genpd, bool one_dev_on, list_for_each_entry(link, &genpd->child_links, child_node) { genpd_sd_counter_dec(link->parent); genpd_lock_nested(link->parent, depth + 1); - genpd_power_off(link->parent, false, depth + 1); + genpd_power_off(link->parent, false, false, depth + 1); genpd_unlock(link->parent); } } @@ -1086,7 +1117,7 @@ static int genpd_power_on(struct generic_pm_domain *genpd, unsigned int depth) child_node) { genpd_sd_counter_dec(link->parent); genpd_lock_nested(link->parent, depth + 1); - genpd_power_off(link->parent, false, depth + 1); + genpd_power_off(link->parent, false, false, depth + 1); genpd_unlock(link->parent); } @@ -1153,7 +1184,7 @@ static void genpd_power_off_work_fn(struct work_struct *work) genpd = container_of(work, struct generic_pm_domain, power_off_work); genpd_lock(genpd); - genpd_power_off(genpd, false, 0); + genpd_power_off(genpd, false, false, 0); genpd_unlock(genpd); } @@ -1218,6 +1249,7 @@ static int genpd_runtime_suspend(struct device *dev) struct generic_pm_domain_data *gpd_data = dev_gpd_data(dev); struct gpd_timing_data *td = gpd_data->td; bool runtime_pm = pm_runtime_enabled(dev); + bool probing = dev->links.status != DL_DEV_DRIVER_BOUND; ktime_t time_start = 0; s64 elapsed_ns; int ret; @@ -1272,7 +1304,7 @@ static int genpd_runtime_suspend(struct device *dev) return 0; genpd_lock(genpd); - genpd_power_off(genpd, true, 0); + genpd_power_off(genpd, true, probing, 0); gpd_data->rpm_pstate = genpd_drop_performance_state(dev); genpd_unlock(genpd); @@ -1293,6 +1325,7 @@ static int genpd_runtime_resume(struct device *dev) struct generic_pm_domain_data *gpd_data = dev_gpd_data(dev); struct gpd_timing_data *td = gpd_data->td; bool timed = td && pm_runtime_enabled(dev); + bool probing = dev->links.status != DL_DEV_DRIVER_BOUND; ktime_t time_start = 0; s64 elapsed_ns; int ret; @@ -1350,7 +1383,7 @@ static int genpd_runtime_resume(struct device *dev) err_poweroff: if (!pm_runtime_is_irq_safe(dev) || genpd_is_irq_safe(genpd)) { genpd_lock(genpd); - genpd_power_off(genpd, true, 0); + genpd_power_off(genpd, true, probing, 0); gpd_data->rpm_pstate = genpd_drop_performance_state(dev); genpd_unlock(genpd); } @@ -1418,6 +1451,9 @@ static void genpd_sync_power_off(struct generic_pm_domain *genpd, bool use_lock, || atomic_read(&genpd->sd_count) > 0) return; + if (genpd_must_defer(genpd, false)) + return; + /* Check that the children are in their deepest (powered-off) state. */ list_for_each_entry(link, &genpd->parent_links, parent_node) { struct generic_pm_domain *child = link->child; @@ -2445,6 +2481,12 @@ int pm_genpd_init(struct generic_pm_domain *genpd, return -EINVAL; } + /* Deferred-off power domains should be powered on at initialization. */ + if (genpd_is_defer_off(genpd) && !genpd_status_on(genpd)) { + pr_warn("deferred-off PM domain %s is not on at init\n", genpd->name); + genpd->flags &= ~GENPD_FLAG_DEFER_OFF; + } + /* Multiple states but no governor doesn't make sense. */ if (!gov && genpd->state_count > 1) pr_warn("%s: no governor for states\n", genpd->name); @@ -3511,7 +3553,7 @@ void of_genpd_sync_state(struct device_node *np) if (genpd->provider == of_fwnode_handle(np)) { genpd_lock(genpd); genpd->stay_on = false; - genpd_power_off(genpd, false, 0); + genpd_power_off(genpd, false, false, 0); genpd_unlock(genpd); } } @@ -3539,7 +3581,7 @@ static void genpd_provider_sync_state(struct device *dev) case GENPD_SYNC_STATE_SIMPLE: genpd_lock(genpd); genpd->stay_on = false; - genpd_power_off(genpd, false, 0); + genpd_power_off(genpd, false, false, 0); genpd_unlock(genpd); break; diff --git a/drivers/power/reset/macsmc-reboot.c b/drivers/power/reset/macsmc-reboot.c index e9702acdd366b0..94fcbf12fe3b93 100644 --- a/drivers/power/reset/macsmc-reboot.c +++ b/drivers/power/reset/macsmc-reboot.c @@ -205,6 +205,14 @@ static int macsmc_reboot_probe(struct platform_device *pdev) struct macsmc_reboot *reboot; int ret, i; + /* + * MFD will probe this device even without a node in the device tree, + * thus bail out early if the SMC on the current machines does not + * support reboot and has no node in the device tree. + */ + if (!pdev->dev.of_node) + return -ENODEV; + reboot = devm_kzalloc(&pdev->dev, sizeof(*reboot), GFP_KERNEL); if (!reboot) return -ENOMEM; diff --git a/drivers/power/reset/nvmem-reboot-mode.c b/drivers/power/reset/nvmem-reboot-mode.c index 41530b70cfc48c..d260715fccf67f 100644 --- a/drivers/power/reset/nvmem-reboot-mode.c +++ b/drivers/power/reset/nvmem-reboot-mode.c @@ -10,6 +10,7 @@ #include #include #include +#include struct nvmem_reboot_mode { struct reboot_mode_driver reboot; @@ -19,12 +20,22 @@ struct nvmem_reboot_mode { static int nvmem_reboot_mode_write(struct reboot_mode_driver *reboot, unsigned int magic) { - int ret; struct nvmem_reboot_mode *nvmem_rbm; + size_t buf_len; + void *buf; + int ret; nvmem_rbm = container_of(reboot, struct nvmem_reboot_mode, reboot); - ret = nvmem_cell_write(nvmem_rbm->cell, &magic, sizeof(magic)); + buf = nvmem_cell_read(nvmem_rbm->cell, &buf_len); + if (IS_ERR(buf)) + return PTR_ERR(buf); + kfree(buf); + + if (buf_len > sizeof(magic)) + return -EINVAL; + + ret = nvmem_cell_write(nvmem_rbm->cell, &magic, buf_len); if (ret < 0) dev_err(reboot->dev, "update reboot mode bits failed\n"); diff --git a/drivers/power/supply/Kconfig b/drivers/power/supply/Kconfig index 92f9f7aae92f24..e1018b864d8cd2 100644 --- a/drivers/power/supply/Kconfig +++ b/drivers/power/supply/Kconfig @@ -1132,4 +1132,11 @@ config FUEL_GAUGE_MM8013 the state of charge, temperature, cycle count, actual and design capacity, etc. +config CHARGER_MACSMC + tristate "Apple SMC Charger / Battery support" + depends on MFD_MACSMC + help + Say Y here to enable support for the charger and battery controls on + Apple SMC controllers, as used on Apple Silicon Macs. + endif # POWER_SUPPLY diff --git a/drivers/power/supply/Makefile b/drivers/power/supply/Makefile index 4b79d5abc49a7f..7c09106ab23004 100644 --- a/drivers/power/supply/Makefile +++ b/drivers/power/supply/Makefile @@ -81,6 +81,7 @@ obj-$(CONFIG_CHARGER_GPIO) += gpio-charger.o obj-$(CONFIG_CHARGER_MANAGER) += charger-manager.o obj-$(CONFIG_CHARGER_LT3651) += lt3651-charger.o obj-$(CONFIG_CHARGER_LTC4162L) += ltc4162-l-charger.o +obj-$(CONFIG_CHARGER_MACSMC) += macsmc-power.o obj-$(CONFIG_CHARGER_MAX14577) += max14577_charger.o obj-$(CONFIG_CHARGER_DETECTOR_MAX14656) += max14656_charger_detector.o obj-$(CONFIG_CHARGER_MAX77650) += max77650-charger.o diff --git a/drivers/power/supply/ab8500_charger.c b/drivers/power/supply/ab8500_charger.c index 5f4537766e5b90..1813fbdfa1c1f6 100644 --- a/drivers/power/supply/ab8500_charger.c +++ b/drivers/power/supply/ab8500_charger.c @@ -3466,26 +3466,6 @@ static int ab8500_charger_probe(struct platform_device *pdev) return ret; } - /* Request interrupts */ - for (i = 0; i < ARRAY_SIZE(ab8500_charger_irq); i++) { - irq = platform_get_irq_byname(pdev, ab8500_charger_irq[i].name); - if (irq < 0) - return irq; - - ret = devm_request_threaded_irq(dev, - irq, NULL, ab8500_charger_irq[i].isr, - IRQF_SHARED | IRQF_NO_SUSPEND | IRQF_ONESHOT, - ab8500_charger_irq[i].name, di); - - if (ret != 0) { - dev_err(dev, "failed to request %s IRQ %d: %d\n" - , ab8500_charger_irq[i].name, irq, ret); - return ret; - } - dev_dbg(dev, "Requested %s IRQ %d: %d\n", - ab8500_charger_irq[i].name, irq, ret); - } - /* initialize lock */ spin_lock_init(&di->usb_state.usb_lock); mutex_init(&di->usb_ipt_crnt_lock); @@ -3614,6 +3594,26 @@ static int ab8500_charger_probe(struct platform_device *pdev) return PTR_ERR(di->usb_chg.psy); } + /* Request interrupts */ + for (i = 0; i < ARRAY_SIZE(ab8500_charger_irq); i++) { + irq = platform_get_irq_byname(pdev, ab8500_charger_irq[i].name); + if (irq < 0) + return irq; + + ret = devm_request_threaded_irq(dev, + irq, NULL, ab8500_charger_irq[i].isr, + IRQF_SHARED | IRQF_NO_SUSPEND | IRQF_ONESHOT, + ab8500_charger_irq[i].name, di); + + if (ret != 0) { + dev_err(dev, "failed to request %s IRQ %d: %d\n" + , ab8500_charger_irq[i].name, irq, ret); + return ret; + } + dev_dbg(dev, "Requested %s IRQ %d: %d\n", + ab8500_charger_irq[i].name, irq, ret); + } + /* * Check what battery we have, since we always have the USB * psy, use that as a handle. diff --git a/drivers/power/supply/act8945a_charger.c b/drivers/power/supply/act8945a_charger.c index 3901a02f326a55..9dec4486b1439a 100644 --- a/drivers/power/supply/act8945a_charger.c +++ b/drivers/power/supply/act8945a_charger.c @@ -597,14 +597,6 @@ static int act8945a_charger_probe(struct platform_device *pdev) return irq ?: -ENXIO; } - ret = devm_request_irq(&pdev->dev, irq, act8945a_status_changed, - IRQF_TRIGGER_FALLING, "act8945a_interrupt", - charger); - if (ret) { - dev_err(&pdev->dev, "failed to request nIRQ pin IRQ\n"); - return ret; - } - charger->desc.name = "act8945a-charger"; charger->desc.get_property = act8945a_charger_get_property; charger->desc.properties = act8945a_charger_props; @@ -625,6 +617,14 @@ static int act8945a_charger_probe(struct platform_device *pdev) return PTR_ERR(charger->psy); } + ret = devm_request_irq(&pdev->dev, irq, act8945a_status_changed, + IRQF_TRIGGER_FALLING, "act8945a_interrupt", + charger); + if (ret) { + dev_err(&pdev->dev, "failed to request nIRQ pin IRQ\n"); + return ret; + } + platform_set_drvdata(pdev, charger); INIT_WORK(&charger->work, act8945a_work); diff --git a/drivers/power/supply/bq256xx_charger.c b/drivers/power/supply/bq256xx_charger.c index ae14162f017a9e..d3de4f8b80db1e 100644 --- a/drivers/power/supply/bq256xx_charger.c +++ b/drivers/power/supply/bq256xx_charger.c @@ -1741,6 +1741,12 @@ static int bq256xx_probe(struct i2c_client *client) usb_register_notifier(bq->usb3_phy, &bq->usb_nb); } + ret = bq256xx_power_supply_init(bq, &psy_cfg, dev); + if (ret) { + dev_err(dev, "Failed to register power supply\n"); + return ret; + } + if (client->irq) { ret = devm_request_threaded_irq(dev, client->irq, NULL, bq256xx_irq_handler_thread, @@ -1753,12 +1759,6 @@ static int bq256xx_probe(struct i2c_client *client) } } - ret = bq256xx_power_supply_init(bq, &psy_cfg, dev); - if (ret) { - dev_err(dev, "Failed to register power supply\n"); - return ret; - } - ret = bq256xx_hw_init(bq); if (ret) { dev_err(dev, "Cannot initialize the chip.\n"); diff --git a/drivers/power/supply/bq25980_charger.c b/drivers/power/supply/bq25980_charger.c index 723858d62d1414..73f06f09f134cd 100644 --- a/drivers/power/supply/bq25980_charger.c +++ b/drivers/power/supply/bq25980_charger.c @@ -1241,6 +1241,12 @@ static int bq25980_probe(struct i2c_client *client) return ret; } + ret = bq25980_power_supply_init(bq, dev); + if (ret) { + dev_err(dev, "Failed to register power supply\n"); + return ret; + } + if (client->irq) { ret = devm_request_threaded_irq(dev, client->irq, NULL, bq25980_irq_handler_thread, @@ -1251,12 +1257,6 @@ static int bq25980_probe(struct i2c_client *client) return ret; } - ret = bq25980_power_supply_init(bq, dev); - if (ret) { - dev_err(dev, "Failed to register power supply\n"); - return ret; - } - ret = bq25980_hw_init(bq); if (ret) { dev_err(dev, "Cannot initialize the chip.\n"); diff --git a/drivers/power/supply/bq27xxx_battery.c b/drivers/power/supply/bq27xxx_battery.c index 19445e39651c71..45f0e39b8c2dd4 100644 --- a/drivers/power/supply/bq27xxx_battery.c +++ b/drivers/power/supply/bq27xxx_battery.c @@ -1172,7 +1172,7 @@ static inline int bq27xxx_write(struct bq27xxx_device_info *di, int reg_index, return -EINVAL; if (!di->bus.write) - return -EPERM; + return -EOPNOTSUPP; ret = di->bus.write(di, di->regs[reg_index], value, single); if (ret < 0) @@ -1191,7 +1191,7 @@ static inline int bq27xxx_read_block(struct bq27xxx_device_info *di, int reg_ind return -EINVAL; if (!di->bus.read_bulk) - return -EPERM; + return -EOPNOTSUPP; ret = di->bus.read_bulk(di, di->regs[reg_index], data, len); if (ret < 0) @@ -1210,7 +1210,7 @@ static inline int bq27xxx_write_block(struct bq27xxx_device_info *di, int reg_in return -EINVAL; if (!di->bus.write_bulk) - return -EPERM; + return -EOPNOTSUPP; ret = di->bus.write_bulk(di, di->regs[reg_index], data, len); if (ret < 0) diff --git a/drivers/power/supply/cpcap-battery.c b/drivers/power/supply/cpcap-battery.c index 8106d1edcbc26a..507fdc1c866d56 100644 --- a/drivers/power/supply/cpcap-battery.c +++ b/drivers/power/supply/cpcap-battery.c @@ -1122,10 +1122,6 @@ static int cpcap_battery_probe(struct platform_device *pdev) platform_set_drvdata(pdev, ddata); - error = cpcap_battery_init_interrupts(pdev, ddata); - if (error) - return error; - error = cpcap_battery_init_iio(ddata); if (error) return error; @@ -1142,6 +1138,10 @@ static int cpcap_battery_probe(struct platform_device *pdev) return error; } + error = cpcap_battery_init_interrupts(pdev, ddata); + if (error) + return error; + atomic_set(&ddata->active, 1); error = cpcap_battery_calibrate(ddata); diff --git a/drivers/power/supply/goldfish_battery.c b/drivers/power/supply/goldfish_battery.c index 479195e35d734a..5aa24e4dc4455d 100644 --- a/drivers/power/supply/goldfish_battery.c +++ b/drivers/power/supply/goldfish_battery.c @@ -224,12 +224,6 @@ static int goldfish_battery_probe(struct platform_device *pdev) if (data->irq < 0) return -ENODEV; - ret = devm_request_irq(&pdev->dev, data->irq, - goldfish_battery_interrupt, - IRQF_SHARED, pdev->name, data); - if (ret) - return ret; - psy_cfg.drv_data = data; data->ac = devm_power_supply_register(&pdev->dev, @@ -244,6 +238,12 @@ static int goldfish_battery_probe(struct platform_device *pdev) if (IS_ERR(data->battery)) return PTR_ERR(data->battery); + ret = devm_request_irq(&pdev->dev, data->irq, + goldfish_battery_interrupt, + IRQF_SHARED, pdev->name, data); + if (ret) + return ret; + GOLDFISH_BATTERY_WRITE(data, BATTERY_INT_ENABLE, BATTERY_INT_MASK); return 0; } diff --git a/drivers/power/supply/macsmc-power.c b/drivers/power/supply/macsmc-power.c new file mode 100644 index 00000000000000..dc3ec5ef2b81cf --- /dev/null +++ b/drivers/power/supply/macsmc-power.c @@ -0,0 +1,1053 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SMC Power/Battery Management + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MAX_STRING_LENGTH 256 + +/* + * This number is not reported anywhere by SMC, but seems to be a good + * conversion factor for charge to energy across machines. We need this + * to convert in the driver, since if we don't userspace will try to do + * the conversion with a randomly guessed voltage and get it wrong. + * + * Ideally there would be a power supply prop to inform userspace of this + * number, but there isn't, only min/max. + */ +#define MACSMC_NOMINAL_CELL_VOLTAGE_MV 3800 + +struct macsmc_power { + struct device *dev; + struct apple_smc *smc; + struct power_supply_desc ac_desc; + struct power_supply_desc batt_desc; + + struct power_supply *batt; + char model_name[MAX_STRING_LENGTH]; + char serial_number[MAX_STRING_LENGTH]; + char mfg_date[MAX_STRING_LENGTH]; + + bool has_chwa; + bool has_chls; + bool has_ch0i; + bool has_ch0c; + bool has_chte; + + u8 num_cells; + int nominal_voltage_mv; + + struct power_supply *ac; + + struct notifier_block nb; + + struct work_struct critical_work; + bool shutdown_started; + + struct delayed_work dbg_log_work; +}; + +static int macsmc_log_power_set(const char *val, const struct kernel_param *kp); + +static const struct kernel_param_ops macsmc_log_power_ops = { + .set = macsmc_log_power_set, + .get = param_get_bool, +}; + +static bool log_power = false; +module_param_cb(log_power, &macsmc_log_power_ops, &log_power, 0644); +MODULE_PARM_DESC(log_power, "Periodically log power consumption for debugging"); + +#define POWER_LOG_INTERVAL (HZ) + +static struct macsmc_power *g_power; + +#define CHNC_BATTERY_FULL BIT(0) +#define CHNC_NO_CHARGER BIT(7) +#define CHNC_NOCHG_CH0C BIT(14) +#define CHNC_NOCHG_CH0B_CH0K BIT(15) +#define CHNC_BATTERY_FULL_2 BIT(18) +#define CHNC_BMS_BUSY BIT(23) +#define CHNC_CHLS_LIMIT BIT(24) +#define CHNC_NOAC_CH0J BIT(53) +#define CHNC_NOAC_CH0I BIT(54) + +#define CH0R_LOWER_FLAGS GENMASK(15, 0) +#define CH0R_NOAC_CH0I BIT(0) +#define CH0R_NOAC_DISCONNECTED BIT(4) +#define CH0R_NOAC_CH0J BIT(5) +#define CH0R_BMS_BUSY BIT(8) +#define CH0R_NOAC_CH0K BIT(9) +#define CH0R_NOAC_CHWA BIT(11) + +#define CH0X_CH0C BIT(0) +#define CH0X_CH0B BIT(1) + +#define ACSt_CAN_BOOT_AP BIT(2) +#define ACSt_CAN_BOOT_IBOOT BIT(1) + +#define CHWA_CHLS_FIXED_START_OFFSET 5 +#define CHLS_MIN_END_THRESHOLD 10 +#define CHLS_FORCE_DISCHARGE 0x100 +#define CHWA_FIXED_END_THRESHOLD 80 +#define CHWA_PROP_WRITE_THRESHOLD 95 + +#define FLT_EXP_BIAS 127 +#define FLT_EXP_MASK GENMASK(30, 23) +#define FLT_MANT_BIAS 23 +#define FLT_MANT_MASK GENMASK(22, 0) +#define FLT_SIGN_MASK BIT(31) +/* + * Many sensors report their data as IEEE-754 floats. No other SMC function uses + * them. + */ +static int apple_smc_read_f32_scaled(struct apple_smc *smc, smc_key key, + int *p, int scale) +{ + u32 fval; + u64 val; + int ret, exp; + + BUILD_BUG_ON(scale <= 0); + + ret = apple_smc_read_u32(smc, key, &fval); + if (ret < 0) + return ret; + + val = ((u64)((fval & FLT_MANT_MASK) | BIT(23))); + exp = ((fval >> 23) & 0xff) - FLT_EXP_BIAS - FLT_MANT_BIAS; + val *= scale; + + if (exp > 63) + val = U64_MAX; + else if (exp < -63) + val = 0; + else if (exp < 0) + val >>= -exp; + else if (exp != 0 && (val & ~((1UL << (64 - exp)) - 1))) /* overflow */ + val = U64_MAX; + else + val <<= exp; + + if (fval & FLT_SIGN_MASK) { + if (val > (-(s64)INT_MIN)) + *p = INT_MIN; + else + *p = -val; + } else { + if (val > INT_MAX) + *p = INT_MAX; + else + *p = val; + } + + return 0; +} + +static void macsmc_do_dbg(struct macsmc_power *power) +{ + int p_in = 0, p_sys = 0, p_3v8 = 0, p_mpmu = 0, p_spmu = 0, p_clvr = 0, p_cpu = 0; + s32 p_bat = 0; + s16 t_full = 0, t_empty = 0; + u8 charge = 0; + + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PDTR), &p_in, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PSTR), &p_sys, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PMVR), &p_3v8, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PHPC), &p_cpu, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PSVR), &p_clvr, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PPMC), &p_mpmu, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PPSC), &p_spmu, 1000); + apple_smc_read_s32(power->smc, SMC_KEY(B0AP), &p_bat); + apple_smc_read_s16(power->smc, SMC_KEY(B0TE), &t_empty); + apple_smc_read_s16(power->smc, SMC_KEY(B0TF), &t_full); + apple_smc_read_u8(power->smc, SMC_KEY(BUIC), &charge); + +#define FD3(x) ((x) / 1000), abs((x) % 1000) + dev_info(power->dev, + "In %2d.%03dW Sys %2d.%03dW 3V8 %2d.%03dW MPMU %2d.%03dW SPMU %2d.%03dW " + "CLVR %2d.%03dW CPU %2d.%03dW Batt %2d.%03dW %d%% T%s %dm\n", + FD3(p_in), FD3(p_sys), FD3(p_3v8), FD3(p_mpmu), FD3(p_spmu), FD3(p_clvr), + FD3(p_cpu), FD3(p_bat), charge, + t_full >= 0 ? "full" : "empty", + t_full >= 0 ? t_full : t_empty); +#undef FD3 +} + +static int macsmc_battery_get_status(struct macsmc_power *power) +{ + u64 nocharge_flags; + u32 nopower_flags; + u16 ac_current; + int charge_limit = 0; + bool limited = false; + bool flag; + int ret; + + /* + * Note: there are fallbacks in case some of these SMC keys disappear in the future + * or are not present on some machines. We treat the absence of the CHCE/CHCC/BSFC/CHSC + * flags as an error, since they are quite fundamental and simple booleans. + */ + + /* + * If power input is inhibited, we are definitely discharging. + * However, if the only reason is the BMS is doing a balancing cycle, + * go ahead and ignore that one to avoid spooking users. + */ + ret = apple_smc_read_u32(power->smc, SMC_KEY(CH0R), &nopower_flags); + if (!ret && (nopower_flags & CH0R_LOWER_FLAGS & ~CH0R_BMS_BUSY)) + return POWER_SUPPLY_STATUS_DISCHARGING; + + /* If no charger is present, we are definitely discharging. */ + ret = apple_smc_read_flag(power->smc, SMC_KEY(CHCE), &flag); + if (ret < 0) + return ret; + if (!flag) + return POWER_SUPPLY_STATUS_DISCHARGING; + + /* If AC is not charge capable, we are definitely discharging. */ + ret = apple_smc_read_flag(power->smc, SMC_KEY(CHCC), &flag); + if (ret < 0) + return ret; + if (!flag) + return POWER_SUPPLY_STATUS_DISCHARGING; + + /* + * If the AC input current limit is tiny or 0, we are discharging no matter + * how much the BMS believes it can charge. + */ + ret = apple_smc_read_u16(power->smc, SMC_KEY(AC-i), &ac_current); + if (!ret && ac_current < 100) + return POWER_SUPPLY_STATUS_DISCHARGING; + + /* If the battery is full, report it as such. */ + ret = apple_smc_read_flag(power->smc, SMC_KEY(BSFC), &flag); + if (ret < 0) + return ret; + if (flag) + return POWER_SUPPLY_STATUS_FULL; + + /* + * If we have charge limits supported and enabled and the SoC is above + * the start threshold, that means we are not charging for that reason + * (if not charging). + */ + if (power->has_chls) { + u16 vu16; + + ret = apple_smc_read_u16(power->smc, SMC_KEY(CHLS), &vu16); + if (ret == sizeof(vu16) && (vu16 & 0xff) >= CHLS_MIN_END_THRESHOLD) + charge_limit = (vu16 & 0xff) - CHWA_CHLS_FIXED_START_OFFSET; + } else if (power->has_chwa) { + ret = apple_smc_read_flag(power->smc, SMC_KEY(CHWA), &flag); + if (ret == 0 && flag) + charge_limit = CHWA_FIXED_END_THRESHOLD - CHWA_CHLS_FIXED_START_OFFSET; + } + + if (charge_limit > 0) { + u8 buic = 0; + + if (apple_smc_read_u8(power->smc, SMC_KEY(BUIC), &buic) >= 0 && + buic >= charge_limit) + limited = true; + } + + /* If there are reasons we aren't charging... */ + ret = apple_smc_read_u64(power->smc, SMC_KEY(CHNC), &nocharge_flags); + if (!ret) { + /* Perhaps the battery is full after all */ + if (nocharge_flags & CHNC_BATTERY_FULL) + return POWER_SUPPLY_STATUS_FULL; + /* + * Or maybe the BMS is just busy doing something, if so call it charging anyway. + * But CHWA limits show up as this, so exclude those. + */ + else if (nocharge_flags == CHNC_BMS_BUSY && !limited) + return POWER_SUPPLY_STATUS_CHARGING; + /* If we have other reasons we aren't charging, say we aren't */ + else if (nocharge_flags) + return POWER_SUPPLY_STATUS_NOT_CHARGING; + /* Else we're either charging or about to charge */ + else + return POWER_SUPPLY_STATUS_CHARGING; + } + + /* As a fallback, use the system charging flag. */ + ret = apple_smc_read_flag(power->smc, SMC_KEY(CHSC), &flag); + if (ret < 0) + return ret; + if (!flag) + return POWER_SUPPLY_STATUS_NOT_CHARGING; + else + return POWER_SUPPLY_STATUS_CHARGING; +} + +static int macsmc_battery_get_charge_behaviour(struct macsmc_power *power) +{ + int ret; + u8 val8; + u8 chte_buf[4]; + + if (power->has_ch0i) { + /* CH0I returns a bitmask like the low byte of CH0R */ + ret = apple_smc_read_u8(power->smc, SMC_KEY(CH0I), &val8); + if (ret) + return ret; + if (val8 & CH0R_NOAC_CH0I) + return POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE; + } + + /* Prefer CHTE available in newer firmwares */ + if (power->has_chte) { + ret = apple_smc_read(power->smc, SMC_KEY(CHTE), chte_buf, 4); + if (ret < 0) + return ret; + + if (chte_buf[0] == 0x01) + return POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE; + + } else if (power->has_ch0c) { + /* CH0C returns a bitmask containing CH0B/CH0C flags */ + ret = apple_smc_read_u8(power->smc, SMC_KEY(CH0C), &val8); + if (ret) + return ret; + if (val8 & CH0X_CH0C) + return POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE; + } + + return POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO; +} + +static int macsmc_battery_set_charge_behaviour(struct macsmc_power *power, int val) +{ + int ret; + + /* + * apple_smc_write_u32 does weird things with endianess, + * so we write raw bytes to ensure correctness of CHTE + */ + u8 chte_inhibit[4] = {0x01, 0x00, 0x00, 0x00}; + u8 chte_auto[4] = {0x00, 0x00, 0x00, 0x00}; + + /* + * CH0I/CH0C/CHTE are "hard" controls that will allow the battery to run down to 0. + * CH0K/CH0B are "soft" controls that are reset to 0 when SOC drops below 50%; + * we don't expose these yet. + */ + + switch (val) { + case POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO: + if (power->has_ch0i) { + ret = apple_smc_write_u8(power->smc, SMC_KEY(CH0I), 0); + if (ret) + return ret; + } + + if (power->has_chte) { + ret = apple_smc_write(power->smc, SMC_KEY(CHTE), chte_auto, 4); + if (ret) + return ret; + } else if (power->has_ch0c) { + ret = apple_smc_write_u8(power->smc, SMC_KEY(CH0C), 0); + if (ret) + return ret; + } + break; + + case POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE: + if (power->has_ch0i) { + ret = apple_smc_write_u8(power->smc, SMC_KEY(CH0I), 0); + if (ret) + return ret; + } + + /* Prefer CHTE available in newer firmwares */ + if (power->has_chte) + return apple_smc_write(power->smc, SMC_KEY(CHTE), chte_inhibit, 4); + else if (power->has_ch0c) + return apple_smc_write_u8(power->smc, SMC_KEY(CH0C), 1); + else + return -EINVAL; + break; + + case POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE: + if (!power->has_ch0i) + return -EINVAL; + + /* Prefer CHTE available in newer firmwares */ + if (power->has_chte) { + ret = apple_smc_write(power->smc, SMC_KEY(CHTE), chte_auto, 4); + if (ret) + return ret; + } else if (power->has_ch0c) { + ret = apple_smc_write_u8(power->smc, SMC_KEY(CH0C), 0); + if (ret) + return ret; + } + + return apple_smc_write_u8(power->smc, SMC_KEY(CH0I), 1); + + default: + return -EINVAL; + } + + return 0; +} + +static int macsmc_battery_get_date(const char *s, int *out) +{ + if (!isdigit(s[0]) || !isdigit(s[1])) + return -ENOTSUPP; + + *out = (s[0] - '0') * 10 + s[1] - '0'; + return 0; +} + +static int macsmc_battery_get_capacity_level(struct macsmc_power *power) +{ + bool flag; + u32 val; + int ret; + + /* Check for emergency shutdown condition */ + if (apple_smc_read_u32(power->smc, SMC_KEY(BCF0), &val) >= 0 && val) + return POWER_SUPPLY_CAPACITY_LEVEL_CRITICAL; + + /* Check AC status for whether we could boot in this state */ + if (apple_smc_read_u32(power->smc, SMC_KEY(ACSt), &val) >= 0) { + if (!(val & ACSt_CAN_BOOT_IBOOT)) + return POWER_SUPPLY_CAPACITY_LEVEL_CRITICAL; + + if (!(val & ACSt_CAN_BOOT_AP)) + return POWER_SUPPLY_CAPACITY_LEVEL_LOW; + } + + /* Check battery full flag */ + ret = apple_smc_read_flag(power->smc, SMC_KEY(BSFC), &flag); + if (ret < 0) + return POWER_SUPPLY_CAPACITY_LEVEL_UNKNOWN; + if (flag) + return POWER_SUPPLY_CAPACITY_LEVEL_FULL; + else + return POWER_SUPPLY_CAPACITY_LEVEL_NORMAL; +} + +static int macsmc_battery_get_property(struct power_supply *psy, + enum power_supply_property psp, + union power_supply_propval *val) +{ + struct macsmc_power *power = power_supply_get_drvdata(psy); + int ret = 0; + u8 vu8; + u16 vu16; + s16 vs16; + s32 vs32; + s64 vs64; + bool flag; + + switch (psp) { + case POWER_SUPPLY_PROP_STATUS: + val->intval = macsmc_battery_get_status(power); + ret = val->intval < 0 ? val->intval : 0; + break; + case POWER_SUPPLY_PROP_PRESENT: + val->intval = 1; + break; + case POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR: + val->intval = macsmc_battery_get_charge_behaviour(power); + ret = val->intval < 0 ? val->intval : 0; + break; + case POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0TE), &vu16); + val->intval = vu16 == 0xffff ? 0 : vu16 * 60; + break; + case POWER_SUPPLY_PROP_TIME_TO_FULL_NOW: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0TF), &vu16); + val->intval = vu16 == 0xffff ? 0 : vu16 * 60; + break; + case POWER_SUPPLY_PROP_CAPACITY: + ret = apple_smc_read_u8(power->smc, SMC_KEY(BUIC), &vu8); + val->intval = vu8; + break; + case POWER_SUPPLY_PROP_CAPACITY_LEVEL: + val->intval = macsmc_battery_get_capacity_level(power); + ret = val->intval < 0 ? val->intval : 0; + break; + case POWER_SUPPLY_PROP_VOLTAGE_NOW: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0AV), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_CURRENT_NOW: + ret = apple_smc_read_s16(power->smc, SMC_KEY(B0AC), &vs16); + val->intval = vs16 * 1000; + break; + case POWER_SUPPLY_PROP_POWER_NOW: + ret = apple_smc_read_s32(power->smc, SMC_KEY(B0AP), &vs32); + val->intval = vs32 * 1000; + break; + case POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN: + ret = apple_smc_read_u16(power->smc, SMC_KEY(BITV), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_VOLTAGE_MAX_DESIGN: + /* + * Battery cell max voltage? BVV* seem to return per-cell voltages, + * BVV[NOP] are probably the max voltages for the 3 cells but we don't + * know what will happen if they ever change the number of cells. + * So go with BVVN and multiply by the cell count (BNCB). + * BVVL seems to be the per-cell limit adjusted dynamically. + * Guess: BVVL = Limit, BVVN = Nominal, and the other cells got filled + * in around nearby letters? + */ + ret = apple_smc_read_u16(power->smc, SMC_KEY(BVVN), &vu16); + val->intval = vu16 * 1000 * power->num_cells; + break; + case POWER_SUPPLY_PROP_VOLTAGE_MIN: + /* Lifetime min */ + ret = apple_smc_read_s16(power->smc, SMC_KEY(BLPM), &vs16); + val->intval = vs16 * 1000; + break; + case POWER_SUPPLY_PROP_VOLTAGE_MAX: + /* Lifetime max */ + ret = apple_smc_read_s16(power->smc, SMC_KEY(BLPX), &vs16); + val->intval = vs16 * 1000; + break; + case POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0RC), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0RI), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0RV), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0DC), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_CHARGE_FULL: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0FC), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_CHARGE_NOW: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0RM), &vu16); + val->intval = swab16(vu16) * 1000; + break; + case POWER_SUPPLY_PROP_ENERGY_FULL_DESIGN: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0DC), &vu16); + val->intval = vu16 * power->nominal_voltage_mv; + break; + case POWER_SUPPLY_PROP_ENERGY_FULL: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0FC), &vu16); + val->intval = vu16 * power->nominal_voltage_mv; + break; + case POWER_SUPPLY_PROP_ENERGY_NOW: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0RM), &vu16); + val->intval = swab16(vu16) * power->nominal_voltage_mv; + break; + case POWER_SUPPLY_PROP_TEMP: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0AT), &vu16); + val->intval = vu16 - 2732; + break; + case POWER_SUPPLY_PROP_CHARGE_COUNTER: + ret = apple_smc_read_s64(power->smc, SMC_KEY(BAAC), &vs64); + val->intval = vs64; + break; + case POWER_SUPPLY_PROP_CYCLE_COUNT: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0CT), &vu16); + val->intval = vu16; + break; + case POWER_SUPPLY_PROP_SCOPE: + val->intval = POWER_SUPPLY_SCOPE_SYSTEM; + break; + case POWER_SUPPLY_PROP_HEALTH: + flag = false; + ret = apple_smc_read_flag(power->smc, SMC_KEY(BBAD), &flag); + val->intval = flag ? POWER_SUPPLY_HEALTH_DEAD : POWER_SUPPLY_HEALTH_GOOD; + break; + case POWER_SUPPLY_PROP_MODEL_NAME: + val->strval = power->model_name; + break; + case POWER_SUPPLY_PROP_SERIAL_NUMBER: + val->strval = power->serial_number; + break; + case POWER_SUPPLY_PROP_MANUFACTURE_YEAR: + ret = macsmc_battery_get_date(&power->mfg_date[0], &val->intval); + val->intval += 2000 - 8; /* -8 is a fixup for a firmware bug... */ + break; + case POWER_SUPPLY_PROP_MANUFACTURE_MONTH: + ret = macsmc_battery_get_date(&power->mfg_date[2], &val->intval); + break; + case POWER_SUPPLY_PROP_MANUFACTURE_DAY: + ret = macsmc_battery_get_date(&power->mfg_date[4], &val->intval); + break; + case POWER_SUPPLY_PROP_CHARGE_CONTROL_START_THRESHOLD: + case POWER_SUPPLY_PROP_CHARGE_CONTROL_END_THRESHOLD: + if (power->has_chls) { + ret = apple_smc_read_u16(power->smc, SMC_KEY(CHLS), &vu16); + val->intval = vu16 & 0xff; + if (val->intval < CHLS_MIN_END_THRESHOLD || val->intval >= 100) + val->intval = 100; + } else if (power->has_chwa) { + flag = false; + ret = apple_smc_read_flag(power->smc, SMC_KEY(CHWA), &flag); + val->intval = flag ? CHWA_FIXED_END_THRESHOLD : 100; + } else { + return -EINVAL; + } + if (psp == POWER_SUPPLY_PROP_CHARGE_CONTROL_START_THRESHOLD && + ret >= 0 && val->intval < 100 && val->intval >= CHLS_MIN_END_THRESHOLD) + val->intval -= CHWA_CHLS_FIXED_START_OFFSET; + break; + default: + return -EINVAL; + } + + return ret; +} + +static int macsmc_battery_set_property(struct power_supply *psy, + enum power_supply_property psp, + const union power_supply_propval *val) +{ + struct macsmc_power *power = power_supply_get_drvdata(psy); + + switch (psp) { + case POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR: + return macsmc_battery_set_charge_behaviour(power, val->intval); + case POWER_SUPPLY_PROP_CHARGE_CONTROL_START_THRESHOLD: + /* + * Ignore, we allow writes so userspace isn't confused but this is + * not configurable independently, it always is end - 5 or 100 depending + * on the end_threshold setting. + */ + return 0; + case POWER_SUPPLY_PROP_CHARGE_CONTROL_END_THRESHOLD: + if (power->has_chls) { + u16 kval = 0; + /* TODO: Make CHLS_FORCE_DISCHARGE configurable */ + if (val->intval < CHLS_MIN_END_THRESHOLD) + kval = CHLS_FORCE_DISCHARGE | CHLS_MIN_END_THRESHOLD; + else if (val->intval < 100) + kval = CHLS_FORCE_DISCHARGE | (val->intval & 0xff); + return apple_smc_write_u16(power->smc, SMC_KEY(CHLS), kval); + } else if (power->has_chwa) { + return apple_smc_write_flag(power->smc, SMC_KEY(CHWA), + val->intval <= CHWA_PROP_WRITE_THRESHOLD); + } else { + return -EINVAL; + } + default: + return -EINVAL; + } +} + +static int macsmc_battery_property_is_writeable(struct power_supply *psy, + enum power_supply_property psp) +{ + struct macsmc_power *power = power_supply_get_drvdata(psy); + + switch (psp) { + case POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR: + return true; + case POWER_SUPPLY_PROP_CHARGE_CONTROL_START_THRESHOLD: + case POWER_SUPPLY_PROP_CHARGE_CONTROL_END_THRESHOLD: + return power->has_chwa || power->has_chls; + default: + return false; + } +} + +static const enum power_supply_property macsmc_battery_props[] = { + POWER_SUPPLY_PROP_STATUS, + POWER_SUPPLY_PROP_PRESENT, + POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR, + POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW, + POWER_SUPPLY_PROP_TIME_TO_FULL_NOW, + POWER_SUPPLY_PROP_CAPACITY, + POWER_SUPPLY_PROP_CAPACITY_LEVEL, + POWER_SUPPLY_PROP_VOLTAGE_NOW, + POWER_SUPPLY_PROP_CURRENT_NOW, + POWER_SUPPLY_PROP_POWER_NOW, + POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN, + POWER_SUPPLY_PROP_VOLTAGE_MAX_DESIGN, + POWER_SUPPLY_PROP_VOLTAGE_MIN, + POWER_SUPPLY_PROP_VOLTAGE_MAX, + POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT, + POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX, + POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE, + POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN, + POWER_SUPPLY_PROP_CHARGE_FULL, + POWER_SUPPLY_PROP_CHARGE_NOW, + POWER_SUPPLY_PROP_ENERGY_FULL_DESIGN, + POWER_SUPPLY_PROP_ENERGY_FULL, + POWER_SUPPLY_PROP_ENERGY_NOW, + POWER_SUPPLY_PROP_TEMP, + POWER_SUPPLY_PROP_CHARGE_COUNTER, + POWER_SUPPLY_PROP_CYCLE_COUNT, + POWER_SUPPLY_PROP_SCOPE, + POWER_SUPPLY_PROP_HEALTH, + POWER_SUPPLY_PROP_MODEL_NAME, + POWER_SUPPLY_PROP_SERIAL_NUMBER, + POWER_SUPPLY_PROP_MANUFACTURE_YEAR, + POWER_SUPPLY_PROP_MANUFACTURE_MONTH, + POWER_SUPPLY_PROP_MANUFACTURE_DAY, + POWER_SUPPLY_PROP_CHARGE_CONTROL_START_THRESHOLD, + POWER_SUPPLY_PROP_CHARGE_CONTROL_END_THRESHOLD +}; + +static const struct power_supply_desc macsmc_battery_desc = { + .name = "macsmc-battery", + .type = POWER_SUPPLY_TYPE_BATTERY, + .get_property = macsmc_battery_get_property, + .set_property = macsmc_battery_set_property, + .property_is_writeable = macsmc_battery_property_is_writeable, + .properties = macsmc_battery_props, + .num_properties = ARRAY_SIZE(macsmc_battery_props), + .charge_behaviours = BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO) + | BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE) + | BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE), +}; + +static int macsmc_ac_get_property(struct power_supply *psy, + enum power_supply_property psp, + union power_supply_propval *val) +{ + struct macsmc_power *power = power_supply_get_drvdata(psy); + int ret = 0; + u16 vu16; + u32 vu32; + + switch (psp) { + case POWER_SUPPLY_PROP_ONLINE: + ret = apple_smc_read_u32(power->smc, SMC_KEY(CHIS), &vu32); + val->intval = !!vu32; + break; + case POWER_SUPPLY_PROP_VOLTAGE_NOW: + ret = apple_smc_read_u16(power->smc, SMC_KEY(AC-n), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT: + ret = apple_smc_read_u16(power->smc, SMC_KEY(AC-i), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_INPUT_POWER_LIMIT: + ret = apple_smc_read_u32(power->smc, SMC_KEY(ACPW), &vu32); + val->intval = vu32 * 1000; + break; + default: + return -EINVAL; + } + + return ret; +} + +static enum power_supply_property macsmc_ac_props[] = { + POWER_SUPPLY_PROP_ONLINE, + POWER_SUPPLY_PROP_INPUT_POWER_LIMIT, + POWER_SUPPLY_PROP_VOLTAGE_NOW, + POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, +}; + +static const struct power_supply_desc macsmc_ac_desc = { + .name = "macsmc-ac", + .type = POWER_SUPPLY_TYPE_MAINS, + .get_property = macsmc_ac_get_property, + .properties = macsmc_ac_props, + .num_properties = ARRAY_SIZE(macsmc_ac_props), +}; + +static int macsmc_log_power_set(const char *val, const struct kernel_param *kp) +{ + int ret = param_set_bool(val, kp); + + if (ret < 0) + return ret; + + if (log_power && g_power) + schedule_delayed_work(&g_power->dbg_log_work, 0); + + return 0; +} + +static void macsmc_dbg_work(struct work_struct *wrk) +{ + struct macsmc_power *power = container_of(to_delayed_work(wrk), + struct macsmc_power, dbg_log_work); + + macsmc_do_dbg(power); + + if (log_power) + schedule_delayed_work(&power->dbg_log_work, POWER_LOG_INTERVAL); +} + +static void macsmc_power_critical_work(struct work_struct *wrk) +{ + struct macsmc_power *power = container_of(wrk, struct macsmc_power, critical_work); + int ret; + u32 bcf0; + u16 bitv, b0av; + + /* + * Check if the battery voltage is below the design voltage. If it is, + * we have a few seconds until the machine dies. Explicitly shut down, + * which at least gets the NVMe controller to flush its cache. + */ + if (apple_smc_read_u16(power->smc, SMC_KEY(BITV), &bitv) >= 0 && + apple_smc_read_u16(power->smc, SMC_KEY(B0AV), &b0av) >= 0 && + b0av < bitv) { + dev_crit(power->dev, "Emergency notification: Battery is critical\n"); + if (kernel_can_power_off()) + kernel_power_off(); + else /* Missing macsmc-reboot driver? In this state, this will not boot anyway. */ + kernel_restart("Battery is critical"); + } + + /* This spams once per second, so make sure we only trigger shutdown once. */ + if (power->shutdown_started) + return; + + /* Check for battery empty condition */ + ret = apple_smc_read_u32(power->smc, SMC_KEY(BCF0), &bcf0); + if (ret < 0) { + dev_err(power->dev, + "Emergency notification: Failed to read battery status\n"); + } else if (bcf0 == 0) { + dev_warn(power->dev, "Emergency notification: Battery status is OK?\n"); + return; + } else { + dev_warn(power->dev, "Emergency notification: Battery is empty\n"); + } + + power->shutdown_started = true; + + /* + * Attempt to trigger an orderly shutdown. At this point, we should have a few + * minutes of reserve capacity left, enough to do a clean shutdown. + */ + dev_warn(power->dev, "Shutting down in 10 seconds\n"); + ssleep(10); + + /* + * Don't force it; if this stalls or fails, the last-resort check above will + * trigger a hard shutdown when shutdown is truly imminent. + */ + orderly_poweroff(false); +} + +static int macsmc_power_event(struct notifier_block *nb, unsigned long event, void *data) +{ + struct macsmc_power *power = container_of(nb, struct macsmc_power, nb); + + if ((event & 0xffffff00) == 0x71010100) { + bool charging = (event & 0xff) != 0; + + dev_info(power->dev, "Charging: %d\n", charging); + power_supply_changed(power->batt); + power_supply_changed(power->ac); + + return NOTIFY_OK; + } else if (event == 0x71020000) { + schedule_work(&power->critical_work); + + return NOTIFY_OK; + } else if ((event & 0xffff0000) == 0x71060000) { + u8 changed_port = event >> 8; + u8 cur_port; + + /* Port charging state change? */ + if (apple_smc_read_u8(power->smc, SMC_KEY(AC-W), &cur_port) >= 0) { + dev_info(power->dev, "Port %d state change (charge port: %d)\n", + changed_port + 1, cur_port); + } + + power_supply_changed(power->batt); + power_supply_changed(power->ac); + + return NOTIFY_OK; + } else if ((event & 0xffff0000) == 0x71130000) { + u8 port_index = (event >> 8) & 0xff; + u8 status = event & 0xff; + + if (port_index == 0xff) + dev_info(power->dev, "Connector event: Disconnect (status 0x%02x)\n", + status); + else + dev_info(power->dev, "Connector event: Port %d (status 0x%02x)\n", + port_index + 1, status); + + power_supply_changed(power->batt); + power_supply_changed(power->ac); + + return NOTIFY_OK; + } else if ((event & 0xff000000) == 0x71000000) { + dev_info(power->dev, "Unknown charger event 0x%lx\n", event); + + return NOTIFY_OK; + } else if ((event & 0xffff0000) == 0x72010000) { + /* Button event handled by macsmc-hid, but let's do a debug print */ + if (log_power) + macsmc_do_dbg(power); + + return NOTIFY_OK; + } + + return NOTIFY_DONE; +} + +static int macsmc_power_probe(struct platform_device *pdev) +{ + struct apple_smc *smc = dev_get_drvdata(pdev->dev.parent); + struct power_supply_config psy_cfg = {}; + struct macsmc_power *power; + bool flag; + u8 val8; + u16 vu16; + u32 val32; + int ret; + + power = devm_kzalloc(&pdev->dev, sizeof(*power), GFP_KERNEL); + if (!power) + return -ENOMEM; + + power->dev = &pdev->dev; + power->smc = smc; + power->ac_desc = macsmc_ac_desc; + power->batt_desc = macsmc_battery_desc; + dev_set_drvdata(&pdev->dev, power); + + /* Ignore devices without a charger/battery */ + if (macsmc_battery_get_status(power) <= POWER_SUPPLY_STATUS_UNKNOWN) + return -ENODEV; + + /* Fetch string properties */ + apple_smc_read(smc, SMC_KEY(BMDN), power->model_name, sizeof(power->model_name) - 1); + apple_smc_read(smc, SMC_KEY(BMSN), power->serial_number, sizeof(power->serial_number) - 1); + apple_smc_read(smc, SMC_KEY(BMDT), power->mfg_date, sizeof(power->mfg_date) - 1); + + if (apple_smc_read_u32(power->smc, SMC_KEY(CHTE), &val32) >= 0) + power->has_chte = true; + + if (apple_smc_read_u8(power->smc, SMC_KEY(CH0C), &val8) >= 0) + power->has_ch0c = true; + + if (apple_smc_read_u8(power->smc, SMC_KEY(CH0I), &val8) >= 0) + power->has_ch0i = true; + + /* Turn off the "optimized battery charging" flags, in case macOS left them on */ + if (power->has_chte) + apple_smc_write_u32(power->smc, SMC_KEY(CHTE), 0); + else if (power->has_ch0c) + apple_smc_write_u8(power->smc, SMC_KEY(CH0C), 0); + + if (power->has_ch0i) + apple_smc_write_u8(power->smc, SMC_KEY(CH0I), 0); + + apple_smc_write_u8(power->smc, SMC_KEY(CH0K), 0); + apple_smc_write_u8(power->smc, SMC_KEY(CH0B), 0); + + power->batt_desc.charge_behaviours = BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO); + + /* Newer firmwares do not have force discharge, so check if it's supported */ + if (power->has_ch0i) + power->batt_desc.charge_behaviours |= BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE); + + /* Older firmware uses CH0C, and newer firmware uses CHTE, so check if at least one is present*/ + if (power->has_chte || power->has_ch0c) + power->batt_desc.charge_behaviours |= BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE); + + /* + * Prefer CHWA as the SMC firmware from iBoot-10151.1.1 is not compatible with + * this CHLS usage. + */ + if (apple_smc_read_flag(power->smc, SMC_KEY(CHWA), &flag) == 0) { + power->has_chwa = true; + } else if (apple_smc_read_u16(power->smc, SMC_KEY(CHLS), &vu16) >= 0) { + power->has_chls = true; + } else { + /* Remove the last 2 properties that control the charge threshold */ + power->batt_desc.num_properties -= 2; + } + + apple_smc_read_u8(power->smc, SMC_KEY(BNCB), &power->num_cells); + power->nominal_voltage_mv = MACSMC_NOMINAL_CELL_VOLTAGE_MV * power->num_cells; + + /* Doing one read of this flag enables critical shutdown notifications */ + apple_smc_read_u32(power->smc, SMC_KEY(BCF0), &val32); + + psy_cfg.drv_data = power; + power->batt = devm_power_supply_register(&pdev->dev, &power->batt_desc, &psy_cfg); + if (IS_ERR(power->batt)) { + dev_err(&pdev->dev, "Failed to register battery\n"); + ret = PTR_ERR(power->batt); + return ret; + } + + /* SMC firmware in macOS 15.4 dropped "AC-i" and "AC-n" (and all keys + * with lower case last letter) without obvious replacement. */ + if (apple_smc_read_u16(power->smc, SMC_KEY(AC-n), &vu16) < 0) + power->ac_desc.num_properties -= 2; + + power->ac = devm_power_supply_register(&pdev->dev, &power->ac_desc, &psy_cfg); + if (IS_ERR(power->ac)) { + dev_err(&pdev->dev, "Failed to register AC adapter\n"); + ret = PTR_ERR(power->ac); + return ret; + } + + power->nb.notifier_call = macsmc_power_event; + blocking_notifier_chain_register(&smc->event_handlers, &power->nb); + + INIT_WORK(&power->critical_work, macsmc_power_critical_work); + INIT_DELAYED_WORK(&power->dbg_log_work, macsmc_dbg_work); + + g_power = power; + + if (log_power) + schedule_delayed_work(&power->dbg_log_work, 0); + + return 0; +} + +static void macsmc_power_remove(struct platform_device *pdev) +{ + struct macsmc_power *power = dev_get_drvdata(&pdev->dev); + + cancel_work(&power->critical_work); + cancel_delayed_work(&power->dbg_log_work); + + g_power = NULL; + + blocking_notifier_chain_unregister(&power->smc->event_handlers, &power->nb); +} + +static struct platform_driver macsmc_power_driver = { + .driver = { + .name = "macsmc-power", + .owner = THIS_MODULE, + }, + .probe = macsmc_power_probe, + .remove = macsmc_power_remove, +}; +module_platform_driver(macsmc_power_driver); + +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple SMC battery and power management driver"); +MODULE_AUTHOR("Hector Martin "); +MODULE_ALIAS("platform:macsmc-power"); diff --git a/drivers/power/supply/pf1550-charger.c b/drivers/power/supply/pf1550-charger.c index 98f1ee8eca3bc8..a457862ef46108 100644 --- a/drivers/power/supply/pf1550-charger.c +++ b/drivers/power/supply/pf1550-charger.c @@ -584,22 +584,6 @@ static int pf1550_charger_probe(struct platform_device *pdev) return dev_err_probe(chg->dev, ret, "failed to add battery sense work\n"); - for (i = 0; i < PF1550_CHARGER_IRQ_NR; i++) { - irq = platform_get_irq(pdev, i); - if (irq < 0) - return irq; - - chg->virqs[i] = irq; - - ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, - pf1550_charger_irq_handler, - IRQF_NO_SUSPEND, - "pf1550-charger", chg); - if (ret) - return dev_err_probe(&pdev->dev, ret, - "failed irq request\n"); - } - psy_cfg.drv_data = chg; chg->charger = devm_power_supply_register(&pdev->dev, @@ -616,6 +600,22 @@ static int pf1550_charger_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, PTR_ERR(chg->battery), "failed: power supply register\n"); + for (i = 0; i < PF1550_CHARGER_IRQ_NR; i++) { + irq = platform_get_irq(pdev, i); + if (irq < 0) + return irq; + + chg->virqs[i] = irq; + + ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, + pf1550_charger_irq_handler, + IRQF_NO_SUSPEND, + "pf1550-charger", chg); + if (ret) + return dev_err_probe(&pdev->dev, ret, + "failed irq request\n"); + } + pf1550_dt_parse_dev_info(chg); return pf1550_reg_init(chg); diff --git a/drivers/power/supply/pm8916_bms_vm.c b/drivers/power/supply/pm8916_bms_vm.c index 5120be086e6ffc..de5d571c03e212 100644 --- a/drivers/power/supply/pm8916_bms_vm.c +++ b/drivers/power/supply/pm8916_bms_vm.c @@ -167,15 +167,6 @@ static int pm8916_bms_vm_battery_probe(struct platform_device *pdev) if (ret < 0) return -EINVAL; - irq = platform_get_irq_byname(pdev, "fifo"); - if (irq < 0) - return irq; - - ret = devm_request_threaded_irq(dev, irq, NULL, pm8916_bms_vm_fifo_update_done_irq, - IRQF_ONESHOT, "pm8916_vm_bms", bat); - if (ret) - return ret; - ret = regmap_bulk_read(bat->regmap, bat->reg + PM8916_PERPH_TYPE, &tmp, 2); if (ret) goto comm_error; @@ -220,6 +211,15 @@ static int pm8916_bms_vm_battery_probe(struct platform_device *pdev) if (ret) return dev_err_probe(dev, ret, "Unable to get battery info\n"); + irq = platform_get_irq_byname(pdev, "fifo"); + if (irq < 0) + return irq; + + ret = devm_request_threaded_irq(dev, irq, NULL, pm8916_bms_vm_fifo_update_done_irq, + IRQF_ONESHOT, "pm8916_vm_bms", bat); + if (ret) + return ret; + platform_set_drvdata(pdev, bat); return 0; diff --git a/drivers/power/supply/pm8916_lbc.c b/drivers/power/supply/pm8916_lbc.c index c74b75b1b2676c..6b631012a7959a 100644 --- a/drivers/power/supply/pm8916_lbc.c +++ b/drivers/power/supply/pm8916_lbc.c @@ -274,15 +274,6 @@ static int pm8916_lbc_charger_probe(struct platform_device *pdev) return dev_err_probe(dev, -EINVAL, "Wrong amount of reg values: %d (4 expected)\n", len); - irq = platform_get_irq_byname(pdev, "usb_vbus"); - if (irq < 0) - return irq; - - ret = devm_request_threaded_irq(dev, irq, NULL, pm8916_lbc_charger_state_changed_irq, - IRQF_ONESHOT, "pm8916_lbc", chg); - if (ret) - return ret; - ret = device_property_read_u32_array(dev, "reg", chg->reg, len); if (ret) return ret; @@ -332,6 +323,10 @@ static int pm8916_lbc_charger_probe(struct platform_device *pdev) if (ret) return dev_err_probe(dev, ret, "Unable to get battery info\n"); + irq = platform_get_irq_byname(pdev, "usb_vbus"); + if (irq < 0) + return irq; + chg->edev = devm_extcon_dev_allocate(dev, pm8916_lbc_charger_cable); if (IS_ERR(chg->edev)) return PTR_ERR(chg->edev); @@ -340,6 +335,11 @@ static int pm8916_lbc_charger_probe(struct platform_device *pdev) if (ret < 0) return dev_err_probe(dev, ret, "failed to register extcon device\n"); + ret = devm_request_threaded_irq(dev, irq, NULL, pm8916_lbc_charger_state_changed_irq, + IRQF_ONESHOT, "pm8916_lbc", chg); + if (ret) + return ret; + ret = regmap_read(chg->regmap, chg->reg[LBC_USB] + PM8916_INT_RT_STS, &tmp); if (ret) goto comm_error; diff --git a/drivers/power/supply/qcom_battmgr.c b/drivers/power/supply/qcom_battmgr.c index c8028606bba009..80572ee945b4f8 100644 --- a/drivers/power/supply/qcom_battmgr.c +++ b/drivers/power/supply/qcom_battmgr.c @@ -1240,7 +1240,8 @@ static unsigned int qcom_battmgr_sc8280xp_parse_technology(const char *chemistry if ((!strncmp(chemistry, "LIO", BATTMGR_CHEMISTRY_LEN)) || (!strncmp(chemistry, "OOI", BATTMGR_CHEMISTRY_LEN))) return POWER_SUPPLY_TECHNOLOGY_LION; - if (!strncmp(chemistry, "LIP", BATTMGR_CHEMISTRY_LEN)) + if (!strncmp(chemistry, "LIP", BATTMGR_CHEMISTRY_LEN) || + !strncmp(chemistry, "LiP", BATTMGR_CHEMISTRY_LEN)) return POWER_SUPPLY_TECHNOLOGY_LIPO; pr_err("Unknown battery technology '%s'\n", chemistry); diff --git a/drivers/power/supply/rt9455_charger.c b/drivers/power/supply/rt9455_charger.c index 1ffe7f02932f6d..5130d2395e88fa 100644 --- a/drivers/power/supply/rt9455_charger.c +++ b/drivers/power/supply/rt9455_charger.c @@ -1663,6 +1663,15 @@ static int rt9455_probe(struct i2c_client *client) rt9455_charger_config.supplied_to = rt9455_charger_supplied_to; rt9455_charger_config.num_supplicants = ARRAY_SIZE(rt9455_charger_supplied_to); + + info->charger = devm_power_supply_register(dev, &rt9455_charger_desc, + &rt9455_charger_config); + if (IS_ERR(info->charger)) { + dev_err(dev, "Failed to register charger\n"); + ret = PTR_ERR(info->charger); + goto put_usb_notifier; + } + ret = devm_request_threaded_irq(dev, client->irq, NULL, rt9455_irq_handler_thread, IRQF_TRIGGER_LOW | IRQF_ONESHOT, @@ -1678,14 +1687,6 @@ static int rt9455_probe(struct i2c_client *client) goto put_usb_notifier; } - info->charger = devm_power_supply_register(dev, &rt9455_charger_desc, - &rt9455_charger_config); - if (IS_ERR(info->charger)) { - dev_err(dev, "Failed to register charger\n"); - ret = PTR_ERR(info->charger); - goto put_usb_notifier; - } - return 0; put_usb_notifier: diff --git a/drivers/power/supply/sbs-battery.c b/drivers/power/supply/sbs-battery.c index 943c82ee978f40..43c48196c16741 100644 --- a/drivers/power/supply/sbs-battery.c +++ b/drivers/power/supply/sbs-battery.c @@ -1174,24 +1174,6 @@ static int sbs_probe(struct i2c_client *client) i2c_set_clientdata(client, chip); - if (!chip->gpio_detect) - goto skip_gpio; - - irq = gpiod_to_irq(chip->gpio_detect); - if (irq <= 0) { - dev_warn(&client->dev, "Failed to get gpio as irq: %d\n", irq); - goto skip_gpio; - } - - rc = devm_request_threaded_irq(&client->dev, irq, NULL, sbs_irq, - IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING | IRQF_ONESHOT, - dev_name(&client->dev), chip); - if (rc) { - dev_warn(&client->dev, "Failed to request irq: %d\n", rc); - goto skip_gpio; - } - -skip_gpio: /* * Before we register, we might need to make sure we can actually talk * to the battery. @@ -1217,6 +1199,24 @@ static int sbs_probe(struct i2c_client *client) return dev_err_probe(&client->dev, PTR_ERR(chip->power_supply), "Failed to register power supply\n"); + if (!chip->gpio_detect) + goto out; + + irq = gpiod_to_irq(chip->gpio_detect); + if (irq <= 0) { + dev_warn(&client->dev, "Failed to get gpio as irq: %d\n", irq); + goto out; + } + + rc = devm_request_threaded_irq(&client->dev, irq, NULL, sbs_irq, + IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING | IRQF_ONESHOT, + dev_name(&client->dev), chip); + if (rc) { + dev_warn(&client->dev, "Failed to request irq: %d\n", rc); + goto out; + } + +out: dev_info(&client->dev, "%s: battery gas gauge device registered\n", client->name); diff --git a/drivers/power/supply/wm97xx_battery.c b/drivers/power/supply/wm97xx_battery.c index b3b0c37a9dd2d5..f00722c88c6fea 100644 --- a/drivers/power/supply/wm97xx_battery.c +++ b/drivers/power/supply/wm97xx_battery.c @@ -178,12 +178,6 @@ static int wm97xx_bat_probe(struct platform_device *dev) "failed to get charge GPIO\n"); if (charge_gpiod) { gpiod_set_consumer_name(charge_gpiod, "BATT CHRG"); - ret = request_irq(gpiod_to_irq(charge_gpiod), - wm97xx_chrg_irq, 0, - "AC Detect", dev); - if (ret) - return dev_err_probe(&dev->dev, ret, - "failed to request GPIO irq\n"); props++; /* POWER_SUPPLY_PROP_STATUS */ } @@ -199,10 +193,8 @@ static int wm97xx_bat_probe(struct platform_device *dev) props++; /* POWER_SUPPLY_PROP_VOLTAGE_MIN */ prop = kcalloc(props, sizeof(*prop), GFP_KERNEL); - if (!prop) { - ret = -ENOMEM; - goto err3; - } + if (!prop) + return -ENOMEM; prop[i++] = POWER_SUPPLY_PROP_PRESENT; if (charge_gpiod) @@ -236,15 +228,27 @@ static int wm97xx_bat_probe(struct platform_device *dev) schedule_work(&bat_work); } else { ret = PTR_ERR(bat_psy); - goto err4; + goto free; + } + + if (charge_gpiod) { + ret = request_irq(gpiod_to_irq(charge_gpiod), wm97xx_chrg_irq, + 0, "AC Detect", dev); + if (ret) { + dev_err_probe(&dev->dev, ret, + "failed to request GPIO irq\n"); + goto unregister; + } } return 0; -err4: + +unregister: + power_supply_unregister(bat_psy); + +free: kfree(prop); -err3: - if (charge_gpiod) - free_irq(gpiod_to_irq(charge_gpiod), dev); + return ret; } diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index 3ff6da3bf4e630..3705d0608a0fba 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -254,7 +254,7 @@ static void rapl_init_domains(struct rapl_package *rp); static int rapl_read_data_raw(struct rapl_domain *rd, enum rapl_primitives prim, bool xlate, u64 *data, - bool atomic); + bool pmu_ctx); static int rapl_write_data_raw(struct rapl_domain *rd, enum rapl_primitives prim, unsigned long long value); @@ -832,7 +832,7 @@ prim_fixups(struct rapl_domain *rd, enum rapl_primitives prim) */ static int rapl_read_data_raw(struct rapl_domain *rd, enum rapl_primitives prim, bool xlate, u64 *data, - bool atomic) + bool pmu_ctx) { u64 value; enum rapl_primitives prim_fixed = prim_fixups(rd, prim); @@ -854,7 +854,7 @@ static int rapl_read_data_raw(struct rapl_domain *rd, ra.mask = rpi->mask; - if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra, atomic)) { + if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra, pmu_ctx)) { pr_debug("failed to read reg 0x%llx for %s:%s\n", ra.reg.val, rd->rp->name, rd->name); return -EIO; } diff --git a/drivers/powercap/intel_rapl_msr.c b/drivers/powercap/intel_rapl_msr.c index 9a7e150b3536b9..152893dca5653a 100644 --- a/drivers/powercap/intel_rapl_msr.c +++ b/drivers/powercap/intel_rapl_msr.c @@ -110,16 +110,14 @@ static int rapl_cpu_down_prep(unsigned int cpu) return 0; } -static int rapl_msr_read_raw(int cpu, struct reg_action *ra, bool atomic) +static int rapl_msr_read_raw(int cpu, struct reg_action *ra, bool pmu_ctx) { /* - * When called from atomic-context (eg PMU event handler) - * perform MSR read directly using rdmsrq(). + * When called from PMU context, perform MSR read directly using + * rdmsrq() without IPI overhead. Package-scoped MSRs are readable + * from any CPU in the package. */ - if (atomic) { - if (unlikely(smp_processor_id() != cpu)) - return -EIO; - + if (pmu_ctx) { rdmsrq(ra->reg.msr, ra->value); goto out; } diff --git a/drivers/powercap/intel_rapl_tpmi.c b/drivers/powercap/intel_rapl_tpmi.c index 0a0b85f4528b1b..0f8abdc592bc19 100644 --- a/drivers/powercap/intel_rapl_tpmi.c +++ b/drivers/powercap/intel_rapl_tpmi.c @@ -157,7 +157,7 @@ static int parse_one_domain(struct tpmi_rapl_package *trp, u32 offset) tpmi_domain_flags = tpmi_domain_header >> 32 & 0xffff; if (tpmi_domain_version == TPMI_VERSION_INVALID) { - pr_warn(FW_BUG "Invalid version\n"); + pr_debug("Invalid version, other instances may be valid\n"); return -ENODEV; } diff --git a/drivers/pwm/pwm-tiehrpwm.c b/drivers/pwm/pwm-tiehrpwm.c index 7a86cb090f76f1..2533c95b0ba9d3 100644 --- a/drivers/pwm/pwm-tiehrpwm.c +++ b/drivers/pwm/pwm-tiehrpwm.c @@ -237,8 +237,6 @@ static int ehrpwm_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, if (period_cycles < 1) period_cycles = 1; - pm_runtime_get_sync(pwmchip_parent(chip)); - /* Update clock prescaler values */ ehrpwm_modify(pc->mmio_base, TBCTL, TBCTL_CLKDIV_MASK, tb_divval); @@ -290,8 +288,6 @@ static int ehrpwm_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, if (!(duty_cycles > period_cycles)) ehrpwm_write(pc->mmio_base, cmp_reg, duty_cycles); - pm_runtime_put_sync(pwmchip_parent(chip)); - return 0; } @@ -378,6 +374,8 @@ static int ehrpwm_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, int err; bool enabled = pwm->state.enabled; + guard(pm_runtime_active)(pwmchip_parent(chip)); + if (state->polarity != pwm->state.polarity) { if (enabled) { ehrpwm_pwm_disable(chip, pwm); diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c index 4b6182cde859ad..8ee33b777f6ce1 100644 --- a/drivers/regulator/core.c +++ b/drivers/regulator/core.c @@ -1444,6 +1444,33 @@ static int set_machine_constraints(struct regulator_dev *rdev) int ret = 0; const struct regulator_ops *ops = rdev->desc->ops; + /* + * If there is no mechanism for controlling the regulator then + * flag it as always_on so we don't end up duplicating checks + * for this so much. Note that we could control the state of + * a supply to control the output on a regulator that has no + * direct control. + */ + if (!rdev->ena_pin && !ops->enable) { + if (rdev->supply_name && !rdev->supply) + return -EPROBE_DEFER; + + if (rdev->supply) + rdev->constraints->always_on = + rdev->supply->rdev->constraints->always_on; + else + rdev->constraints->always_on = true; + } + + /* + * If we want to enable this regulator, make sure that we know the + * supplying regulator. + */ + if (rdev->constraints->always_on || rdev->constraints->boot_on) { + if (rdev->supply_name && !rdev->supply) + return -EPROBE_DEFER; + } + ret = machine_constraints_voltage(rdev, rdev->constraints); if (ret != 0) return ret; @@ -1609,37 +1636,15 @@ static int set_machine_constraints(struct regulator_dev *rdev) } } - /* - * If there is no mechanism for controlling the regulator then - * flag it as always_on so we don't end up duplicating checks - * for this so much. Note that we could control the state of - * a supply to control the output on a regulator that has no - * direct control. - */ - if (!rdev->ena_pin && !ops->enable) { - if (rdev->supply_name && !rdev->supply) - return -EPROBE_DEFER; - - if (rdev->supply) - rdev->constraints->always_on = - rdev->supply->rdev->constraints->always_on; - else - rdev->constraints->always_on = true; - } - /* If the constraints say the regulator should be on at this point * and we have control then make sure it is enabled. */ if (rdev->constraints->always_on || rdev->constraints->boot_on) { bool supply_enabled = false; - /* If we want to enable this regulator, make sure that we know - * the supplying regulator. - */ - if (rdev->supply_name && !rdev->supply) - return -EPROBE_DEFER; - - /* If supplying regulator has already been enabled, + /* We have ensured a potential supply has been resolved above. + * + * If supplying regulator has already been enabled, * it's not intended to have use_count increment * when rdev is only boot-on. */ @@ -2268,10 +2273,21 @@ static int regulator_resolve_supply(struct regulator_dev *rdev) * under-voltage. */ ret = register_regulator_event_forwarding(rdev); - if (ret < 0) + if (ret < 0) { + struct regulator *supply; + rdev_warn(rdev, "Failed to register event forwarding: %pe\n", ERR_PTR(ret)); + supply = rdev->supply; + rdev->supply = NULL; + + regulator_unlock_two(rdev, supply->rdev, &ww_ctx); + + regulator_put(supply); + goto out; + } + regulator_unlock_two(rdev, r, &ww_ctx); /* rdev->supply was created in set_supply() */ @@ -2285,8 +2301,16 @@ static int regulator_resolve_supply(struct regulator_dev *rdev) if (rdev->use_count) { ret = regulator_enable(rdev->supply); if (ret < 0) { - _regulator_put(rdev->supply); + struct regulator *supply; + + regulator_lock_two(rdev, rdev->supply->rdev, &ww_ctx); + + supply = rdev->supply; rdev->supply = NULL; + + regulator_unlock_two(rdev, supply->rdev, &ww_ctx); + + regulator_put(supply); goto out; } } diff --git a/drivers/regulator/mt6363-regulator.c b/drivers/regulator/mt6363-regulator.c index e0fbf92e768517..03af5fa5360073 100644 --- a/drivers/regulator/mt6363-regulator.c +++ b/drivers/regulator/mt6363-regulator.c @@ -861,7 +861,7 @@ static int mt6363_regulator_probe(struct platform_device *pdev) struct irq_domain *domain; struct irq_fwspec fwspec; struct spmi_device *sdev; - int i, ret; + int i, ret, val; config.regmap = mt6363_spmi_register_regmap(dev); if (IS_ERR(config.regmap)) @@ -870,6 +870,13 @@ static int mt6363_regulator_probe(struct platform_device *pdev) config.dev = dev; sdev = to_spmi_device(dev->parent); + /* + * The first read may fail if the bootloader sets sleep mode: wake up + * this PMIC with W/R on the SPMI bus and ignore the first result. + * This matches the MT6373 driver behavior. + */ + regmap_read(config.regmap, MT6363_TOP_TRAP, &val); + interrupt_parent = of_irq_find_parent(dev->of_node); if (!interrupt_parent) return dev_err_probe(dev, -EINVAL, "Cannot find IRQ parent\n"); diff --git a/drivers/remoteproc/imx_dsp_rproc.c b/drivers/remoteproc/imx_dsp_rproc.c index 5130a35214c929..5a9a8fa031f6d7 100644 --- a/drivers/remoteproc/imx_dsp_rproc.c +++ b/drivers/remoteproc/imx_dsp_rproc.c @@ -644,6 +644,32 @@ static void imx_dsp_rproc_free_mbox(struct imx_dsp_rproc *priv) mbox_free_channel(priv->rxdb_ch); } +static int imx_dsp_rproc_mem_alloc(struct rproc *rproc, + struct rproc_mem_entry *mem) +{ + struct device *dev = rproc->dev.parent; + void *va; + + va = ioremap_wc(mem->dma, mem->len); + if (!va) { + dev_err(dev, "Unable to map memory region: %pa+%zx\n", + &mem->dma, mem->len); + return -ENOMEM; + } + + mem->va = va; + + return 0; +} + +static int imx_dsp_rproc_mem_release(struct rproc *rproc, + struct rproc_mem_entry *mem) +{ + iounmap(mem->va); + + return 0; +} + /** * imx_dsp_rproc_add_carveout() - request mailbox channels * @priv: private data pointer @@ -659,7 +685,6 @@ static int imx_dsp_rproc_add_carveout(struct imx_dsp_rproc *priv) struct device *dev = rproc->dev.parent; struct device_node *np = dev->of_node; struct rproc_mem_entry *mem; - void __iomem *cpu_addr; int a, i = 0; u64 da; @@ -673,15 +698,10 @@ static int imx_dsp_rproc_add_carveout(struct imx_dsp_rproc *priv) if (imx_dsp_rproc_sys_to_da(priv, att->sa, att->size, &da)) return -EINVAL; - cpu_addr = devm_ioremap_wc(dev, att->sa, att->size); - if (!cpu_addr) { - dev_err(dev, "failed to map memory %p\n", &att->sa); - return -ENOMEM; - } - /* Register memory region */ - mem = rproc_mem_entry_init(dev, (void __force *)cpu_addr, (dma_addr_t)att->sa, - att->size, da, NULL, NULL, "dsp_mem"); + mem = rproc_mem_entry_init(dev, NULL, (dma_addr_t)att->sa, + att->size, da, imx_dsp_rproc_mem_alloc, + imx_dsp_rproc_mem_release, "dsp_mem"); if (mem) rproc_coredump_add_segment(rproc, da, att->size); @@ -709,15 +729,11 @@ static int imx_dsp_rproc_add_carveout(struct imx_dsp_rproc *priv) if (imx_dsp_rproc_sys_to_da(priv, res.start, resource_size(&res), &da)) return -EINVAL; - cpu_addr = devm_ioremap_resource_wc(dev, &res); - if (IS_ERR(cpu_addr)) { - dev_err(dev, "failed to map memory %pR\n", &res); - return PTR_ERR(cpu_addr); - } - /* Register memory region */ - mem = rproc_mem_entry_init(dev, (void __force *)cpu_addr, (dma_addr_t)res.start, - resource_size(&res), da, NULL, NULL, + mem = rproc_mem_entry_init(dev, NULL, (dma_addr_t)res.start, + resource_size(&res), da, + imx_dsp_rproc_mem_alloc, + imx_dsp_rproc_mem_release, "%.*s", strchrnul(res.name, '@') - res.name, res.name); if (!mem) return -ENOMEM; @@ -984,9 +1000,11 @@ static int imx_dsp_rproc_load(struct rproc *rproc, const struct firmware *fw) * Clear buffers after pm rumtime for internal ocram is not * accessible if power and clock are not enabled. */ - list_for_each_entry(carveout, &rproc->carveouts, node) { - if (carveout->va) - memset(carveout->va, 0, carveout->len); + if (rproc->state == RPROC_OFFLINE) { + list_for_each_entry(carveout, &rproc->carveouts, node) { + if (carveout->va) + memset(carveout->va, 0, carveout->len); + } } ret = imx_dsp_rproc_elf_load_segments(rproc, fw); diff --git a/drivers/remoteproc/imx_rproc.c b/drivers/remoteproc/imx_rproc.c index 3be8790c14a2cc..33f21ab24c9211 100644 --- a/drivers/remoteproc/imx_rproc.c +++ b/drivers/remoteproc/imx_rproc.c @@ -694,7 +694,7 @@ static int imx_rproc_addr_init(struct imx_rproc *priv, } priv->mem[b].sys_addr = res.start; priv->mem[b].size = resource_size(&res); - if (!strcmp(res.name, "rsc-table")) + if (strstarts(res.name, "rsc-table")) priv->rsc_table = priv->mem[b].cpu_addr; b++; } diff --git a/drivers/reset/Kconfig b/drivers/reset/Kconfig index 6e5d6deffa7d36..52ee102621eef1 100644 --- a/drivers/reset/Kconfig +++ b/drivers/reset/Kconfig @@ -161,7 +161,7 @@ config RESET_K210 config RESET_K230 tristate "Reset controller driver for Canaan Kendryte K230 SoC" depends on ARCH_CANAAN || COMPILE_TEST - depends on OF + default ARCH_CANAAN help Support for the Canaan Kendryte K230 RISC-V SoC reset controller. Say Y if you want to control reset signals provided by this diff --git a/drivers/rtc/rtc-amlogic-a4.c b/drivers/rtc/rtc-amlogic-a4.c index 123fb372fc9fe0..50938c35af36a6 100644 --- a/drivers/rtc/rtc-amlogic-a4.c +++ b/drivers/rtc/rtc-amlogic-a4.c @@ -369,7 +369,7 @@ static int aml_rtc_probe(struct platform_device *pdev) return PTR_ERR(rtc->rtc_dev); ret = devm_request_irq(dev, rtc->irq, aml_rtc_handler, - IRQF_ONESHOT, "aml-rtc alarm", rtc); + 0, "aml-rtc alarm", rtc); if (ret) { dev_err_probe(dev, ret, "IRQ%d request failed, ret = %d\n", rtc->irq, ret); diff --git a/drivers/s390/cio/css.c b/drivers/s390/cio/css.c index 4c85df7a548ef1..ac24e019020e89 100644 --- a/drivers/s390/cio/css.c +++ b/drivers/s390/cio/css.c @@ -235,7 +235,7 @@ struct subchannel *css_alloc_subchannel(struct subchannel_id schid, return sch; err: - kfree(sch); + put_device(&sch->dev); return ERR_PTR(ret); } diff --git a/drivers/scsi/csiostor/csio_scsi.c b/drivers/scsi/csiostor/csio_scsi.c index 34bde6650fae0f..356a7c577ec3ef 100644 --- a/drivers/scsi/csiostor/csio_scsi.c +++ b/drivers/scsi/csiostor/csio_scsi.c @@ -2074,7 +2074,7 @@ csio_eh_lun_reset_handler(struct scsi_cmnd *cmnd) struct csio_scsi_level_data sld; if (!rn) - goto fail; + goto fail_ret; csio_dbg(hw, "Request to reset LUN:%llu (ssni:0x%x tgtid:%d)\n", cmnd->device->lun, rn->flowid, rn->scsi_id); @@ -2220,6 +2220,7 @@ csio_eh_lun_reset_handler(struct scsi_cmnd *cmnd) csio_put_scsi_ioreq_lock(hw, scsim, ioreq); fail: CSIO_INC_STATS(rn, n_lun_rst_fail); +fail_ret: return FAILED; } diff --git a/drivers/scsi/elx/efct/efct_driver.c b/drivers/scsi/elx/efct/efct_driver.c index 1bd42f7db1773d..528399f725d428 100644 --- a/drivers/scsi/elx/efct/efct_driver.c +++ b/drivers/scsi/elx/efct/efct_driver.c @@ -415,12 +415,6 @@ efct_intr_thread(int irq, void *handle) return IRQ_HANDLED; } -static irqreturn_t -efct_intr_msix(int irq, void *handle) -{ - return IRQ_WAKE_THREAD; -} - static int efct_setup_msix(struct efct *efct, u32 num_intrs) { @@ -450,7 +444,7 @@ efct_setup_msix(struct efct *efct, u32 num_intrs) intr_ctx->index = i; rc = request_threaded_irq(pci_irq_vector(efct->pci, i), - efct_intr_msix, efct_intr_thread, 0, + NULL, efct_intr_thread, IRQF_ONESHOT, EFCT_DRIVER_NAME, intr_ctx); if (rc) { dev_err(&efct->pci->dev, diff --git a/drivers/scsi/qla2xxx/qla_bsg.c b/drivers/scsi/qla2xxx/qla_bsg.c index ccfc2d26dd3725..0798bfd0372e47 100644 --- a/drivers/scsi/qla2xxx/qla_bsg.c +++ b/drivers/scsi/qla2xxx/qla_bsg.c @@ -1546,8 +1546,9 @@ qla2x00_update_optrom(struct bsg_job *bsg_job) ha->optrom_buffer = NULL; ha->optrom_state = QLA_SWAITING; mutex_unlock(&ha->optrom_mutex); - bsg_job_done(bsg_job, bsg_reply->result, - bsg_reply->reply_payload_rcv_len); + if (!rval) + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return rval; } @@ -2612,8 +2613,9 @@ qla2x00_manage_host_stats(struct bsg_job *bsg_job) sizeof(struct ql_vnd_mng_host_stats_resp)); bsg_reply->result = DID_OK; - bsg_job_done(bsg_job, bsg_reply->result, - bsg_reply->reply_payload_rcv_len); + if (!ret) + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return ret; } @@ -2702,8 +2704,9 @@ qla2x00_get_host_stats(struct bsg_job *bsg_job) bsg_job->reply_payload.sg_cnt, data, response_len); bsg_reply->result = DID_OK; - bsg_job_done(bsg_job, bsg_reply->result, - bsg_reply->reply_payload_rcv_len); + if (!ret) + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); kfree(data); host_stat_out: @@ -2802,8 +2805,9 @@ qla2x00_get_tgt_stats(struct bsg_job *bsg_job) bsg_job->reply_payload.sg_cnt, data, response_len); bsg_reply->result = DID_OK; - bsg_job_done(bsg_job, bsg_reply->result, - bsg_reply->reply_payload_rcv_len); + if (!ret) + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); tgt_stat_out: kfree(data); @@ -2864,8 +2868,9 @@ qla2x00_manage_host_port(struct bsg_job *bsg_job) bsg_job->reply_payload.sg_cnt, &rsp_data, sizeof(struct ql_vnd_mng_host_port_resp)); bsg_reply->result = DID_OK; - bsg_job_done(bsg_job, bsg_reply->result, - bsg_reply->reply_payload_rcv_len); + if (!ret) + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return ret; } @@ -3240,7 +3245,8 @@ int qla2x00_mailbox_passthru(struct bsg_job *bsg_job) bsg_job->reply_len = sizeof(*bsg_job->reply); bsg_reply->result = DID_OK << 16; - bsg_job_done(bsg_job, bsg_reply->result, bsg_reply->reply_payload_rcv_len); + if (!ret) + bsg_job_done(bsg_job, bsg_reply->result, bsg_reply->reply_payload_rcv_len); kfree(req_data); diff --git a/drivers/scsi/qla2xxx/qla_gs.c b/drivers/scsi/qla2xxx/qla_gs.c index 51c7cea71f9022..880cd73feaca45 100644 --- a/drivers/scsi/qla2xxx/qla_gs.c +++ b/drivers/scsi/qla2xxx/qla_gs.c @@ -3266,9 +3266,6 @@ void qla_fab_scan_finish(scsi_qla_host_t *vha, srb_t *sp) atomic_read(&fcport->state) == FCS_ONLINE) || do_delete) { if (fcport->loop_id != FC_NO_LOOP_ID) { - if (fcport->flags & FCF_FCP2_DEVICE) - continue; - ql_log(ql_log_warn, vha, 0x20f0, "%s %d %8phC post del sess\n", __func__, __LINE__, @@ -3535,8 +3532,8 @@ int qla_fab_async_scan(scsi_qla_host_t *vha, srb_t *sp) if (vha->scan.scan_flags & SF_SCANNING) { spin_unlock_irqrestore(&vha->work_lock, flags); ql_dbg(ql_dbg_disc + ql_dbg_verbose, vha, 0x2012, - "%s: scan active\n", __func__); - return rval; + "%s: scan active for sp:%p\n", __func__, sp); + goto done_free_sp; } vha->scan.scan_flags |= SF_SCANNING; if (!sp) @@ -3701,23 +3698,25 @@ int qla_fab_async_scan(scsi_qla_host_t *vha, srb_t *sp) return rval; done_free_sp: - if (sp->u.iocb_cmd.u.ctarg.req) { - dma_free_coherent(&vha->hw->pdev->dev, - sp->u.iocb_cmd.u.ctarg.req_allocated_size, - sp->u.iocb_cmd.u.ctarg.req, - sp->u.iocb_cmd.u.ctarg.req_dma); - sp->u.iocb_cmd.u.ctarg.req = NULL; - } - if (sp->u.iocb_cmd.u.ctarg.rsp) { - dma_free_coherent(&vha->hw->pdev->dev, - sp->u.iocb_cmd.u.ctarg.rsp_allocated_size, - sp->u.iocb_cmd.u.ctarg.rsp, - sp->u.iocb_cmd.u.ctarg.rsp_dma); - sp->u.iocb_cmd.u.ctarg.rsp = NULL; - } + if (sp) { + if (sp->u.iocb_cmd.u.ctarg.req) { + dma_free_coherent(&vha->hw->pdev->dev, + sp->u.iocb_cmd.u.ctarg.req_allocated_size, + sp->u.iocb_cmd.u.ctarg.req, + sp->u.iocb_cmd.u.ctarg.req_dma); + sp->u.iocb_cmd.u.ctarg.req = NULL; + } + if (sp->u.iocb_cmd.u.ctarg.rsp) { + dma_free_coherent(&vha->hw->pdev->dev, + sp->u.iocb_cmd.u.ctarg.rsp_allocated_size, + sp->u.iocb_cmd.u.ctarg.rsp, + sp->u.iocb_cmd.u.ctarg.rsp_dma); + sp->u.iocb_cmd.u.ctarg.rsp = NULL; + } - /* ref: INIT */ - kref_put(&sp->cmd_kref, qla2x00_sp_release); + /* ref: INIT */ + kref_put(&sp->cmd_kref, qla2x00_sp_release); + } spin_lock_irqsave(&vha->work_lock, flags); vha->scan.scan_flags &= ~SF_SCANNING; diff --git a/drivers/scsi/qla2xxx/qla_init.c b/drivers/scsi/qla2xxx/qla_init.c index d395cbfe6802e9..80d66ad324ba7e 100644 --- a/drivers/scsi/qla2xxx/qla_init.c +++ b/drivers/scsi/qla2xxx/qla_init.c @@ -1859,15 +1859,6 @@ void qla2x00_handle_rscn(scsi_qla_host_t *vha, struct event_arg *ea) case RSCN_PORT_ADDR: fcport = qla2x00_find_fcport_by_nportid(vha, &ea->id, 1); if (fcport) { - if (ql2xfc2target && - fcport->flags & FCF_FCP2_DEVICE && - atomic_read(&fcport->state) == FCS_ONLINE) { - ql_dbg(ql_dbg_disc, vha, 0x2115, - "Delaying session delete for FCP2 portid=%06x %8phC ", - fcport->d_id.b24, fcport->port_name); - return; - } - if (vha->hw->flags.edif_enabled && DBELL_ACTIVE(vha)) { /* * On ipsec start by remote port, Target port @@ -2471,8 +2462,23 @@ qla24xx_handle_plogi_done_event(struct scsi_qla_host *vha, struct event_arg *ea) ea->sp->gen1, fcport->rscn_gen, ea->data[0], ea->data[1], ea->iop[0], ea->iop[1]); - if ((fcport->fw_login_state == DSC_LS_PLOGI_PEND) || - (fcport->fw_login_state == DSC_LS_PRLI_PEND)) { + if (fcport->fw_login_state == DSC_LS_PLOGI_PEND) { + ql_dbg(ql_dbg_disc, vha, 0x20ea, + "%s %d %8phC Remote is trying to login\n", + __func__, __LINE__, fcport->port_name); + /* + * If we get here, there is port thats already logged in, + * but it's state has not moved ahead. Recheck with FW on + * what state it is in and proceed ahead + */ + if (!N2N_TOPO(vha->hw)) { + fcport->fw_login_state = DSC_LS_PRLI_COMP; + qla24xx_post_gpdb_work(vha, fcport, 0); + } + return; + } + + if (fcport->fw_login_state == DSC_LS_PRLI_PEND) { ql_dbg(ql_dbg_disc, vha, 0x20ea, "%s %d %8phC Remote is trying to login\n", __func__, __LINE__, fcport->port_name); diff --git a/drivers/scsi/qla2xxx/qla_isr.c b/drivers/scsi/qla2xxx/qla_isr.c index a04a5aa0d00572..608d2f36e7b4ff 100644 --- a/drivers/scsi/qla2xxx/qla_isr.c +++ b/drivers/scsi/qla2xxx/qla_isr.c @@ -1676,13 +1676,28 @@ qla2x00_async_event(scsi_qla_host_t *vha, struct rsp_que *rsp, uint16_t *mb) /* Port logout */ fcport = qla2x00_find_fcport_by_loopid(vha, mb[1]); - if (!fcport) + if (!fcport) { + ql_dbg(ql_dbg_async, vha, 0x5011, + "Could not find fcport:%04x %04x %04x\n", + mb[1], mb[2], mb[3]); break; - if (atomic_read(&fcport->state) != FCS_ONLINE) + } + + if (atomic_read(&fcport->state) != FCS_ONLINE) { + ql_dbg(ql_dbg_async, vha, 0x5012, + "Port state is not online State:0x%x \n", + atomic_read(&fcport->state)); + ql_dbg(ql_dbg_async, vha, 0x5012, + "Scheduling session for deletion \n"); + fcport->logout_on_delete = 0; + qlt_schedule_sess_for_deletion(fcport); break; + } + ql_dbg(ql_dbg_async, vha, 0x508a, "Marking port lost loopid=%04x portid=%06x.\n", fcport->loop_id, fcport->d_id.b24); + if (qla_ini_mode_enabled(vha)) { fcport->logout_on_delete = 0; qlt_schedule_sess_for_deletion(fcport); diff --git a/drivers/scsi/qla2xxx/qla_os.c b/drivers/scsi/qla2xxx/qla_os.c index e939bc88e15197..35fb3e5f4e9f32 100644 --- a/drivers/scsi/qla2xxx/qla_os.c +++ b/drivers/scsi/qla2xxx/qla_os.c @@ -1183,7 +1183,8 @@ qla2x00_wait_for_hba_ready(scsi_qla_host_t *vha) while ((qla2x00_reset_active(vha) || ha->dpc_active || ha->flags.mbox_busy) || test_bit(FX00_RESET_RECOVERY, &vha->dpc_flags) || - test_bit(FX00_TARGET_SCAN, &vha->dpc_flags)) { + test_bit(FX00_TARGET_SCAN, &vha->dpc_flags) || + (vha->scan.scan_flags & SF_SCANNING)) { if (test_bit(UNLOADING, &base_vha->dpc_flags)) break; msleep(1000); diff --git a/drivers/scsi/smartpqi/smartpqi_init.c b/drivers/scsi/smartpqi/smartpqi_init.c index fe549e2b7c9407..c829d9590558df 100644 --- a/drivers/scsi/smartpqi/smartpqi_init.c +++ b/drivers/scsi/smartpqi/smartpqi_init.c @@ -1241,7 +1241,8 @@ static inline int pqi_report_phys_luns(struct pqi_ctrl_info *ctrl_info, void **b dev_err(&ctrl_info->pci_dev->dev, "RPL returned unsupported data format %u\n", rpl_response_format); - return -EINVAL; + rc = -EINVAL; + goto out_free_rpl_list; } else { dev_warn(&ctrl_info->pci_dev->dev, "RPL returned extended format 2 instead of 4\n"); @@ -1253,8 +1254,10 @@ static inline int pqi_report_phys_luns(struct pqi_ctrl_info *ctrl_info, void **b rpl_16byte_wwid_list = kmalloc(struct_size(rpl_16byte_wwid_list, lun_entries, num_physicals), GFP_KERNEL); - if (!rpl_16byte_wwid_list) - return -ENOMEM; + if (!rpl_16byte_wwid_list) { + rc = -ENOMEM; + goto out_free_rpl_list; + } put_unaligned_be32(num_physicals * sizeof(struct report_phys_lun_16byte_wwid), &rpl_16byte_wwid_list->header.list_length); @@ -1275,6 +1278,10 @@ static inline int pqi_report_phys_luns(struct pqi_ctrl_info *ctrl_info, void **b *buffer = rpl_16byte_wwid_list; return 0; + +out_free_rpl_list: + kfree(rpl_list); + return rc; } static inline int pqi_report_logical_luns(struct pqi_ctrl_info *ctrl_info, void **buffer) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index ad67368892311b..f1de7dc11a6e25 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -4,6 +4,15 @@ if ARCH_APPLE || COMPILE_TEST menu "Apple SoC drivers" +config APPLE_DOCKCHANNEL + tristate "Apple DockChannel FIFO" + depends on ARCH_APPLE || COMPILE_TEST + help + DockChannel is a simple FIFO used on Apple SoCs for debug and inter-processor + communications. + + Say 'y' here if you have an Apple SoC. + config APPLE_MAILBOX tristate "Apple SoC mailboxes" depends on PM @@ -16,6 +25,18 @@ config APPLE_MAILBOX Say Y here if you have an Apple SoC. +config APPLE_PMGR_MISC + bool "Apple SoC PMGR miscellaneous support" + depends on PM + help + The PMGR block in Apple SoCs provides high-level power state + controls for SoC devices. This driver manages miscellaneous + power controls. + +config APPLE_TUNABLE + tristate + depends on ARCH_APPLE || COMPILE_TEST + config APPLE_RTKIT tristate "Apple RTKit co-processor IPC protocol" depends on APPLE_MAILBOX @@ -28,6 +49,19 @@ config APPLE_RTKIT Say 'y' here if you have an Apple SoC. +config APPLE_RTKIT_HELPER + tristate "Apple Generic RTKit helper co-processor" + depends on APPLE_RTKIT + depends on ARCH_APPLE || COMPILE_TEST + help + Apple SoCs such as the M1 come with various co-processors running + their proprietary RTKit operating system. This option enables support + for a generic co-processor that does not implement any additional + in-band communications. It can be used for testing purposes, or for + coprocessors such as MTP that communicate over a different interface. + + Say 'y' here if you have an Apple SoC. + config APPLE_SART tristate "Apple SART DMA address filter" depends on ARCH_APPLE || COMPILE_TEST @@ -38,6 +72,55 @@ config APPLE_SART Say 'y' here if you have an Apple SoC. +config RUST_APPLE_MAILBOX + bool + depends on PM + depends on RUST + select APPLE_MAILBOX + +config RUST_APPLE_RTKIT + bool + depends on PM + depends on RUST + select APPLE_RTKIT + +config APPLE_AOP + tristate "Apple \"Always-on\" Processor" + depends on ARCH_APPLE || COMPILE_TEST + depends on PM + depends on RUST + select RUST_APPLE_RTKIT + help + A co-processor persent on certain Apple SoCs controlling accelerometers, + gyros, ambient light sensors and microphones. Is not actually always on. + + Say 'y' here if you have an Apple laptop. + +config APPLE_SEP + tristate "Apple Secure Element Processor" + depends on ARCH_APPLE || COMPILE_TEST + depends on PM + depends on RUST + select RUST_APPLE_RTKIT + select RUST_APPLE_MAILBOX + help + A security co-processor persent on Apple SoCs, controlling transparent + disk encryption, secure boot, HDCP, biometric auth and probably more. + + Say 'y' here if you have an Apple SoC. + +config APPLE_PMP + tristate "Apple Power Management Processor" + depends on ARCH_APPLE || COMPILE_TEST + depends on RUST + select RUST_APPLE_RTKIT + default y if ARCH_APPLE + help + A co-processor present on Apple SoCs, controlling power states of + fabric and other uncore components. + + Say 'y' here if you have an Apple SoC. + endmenu endif diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index 4d9ab8f3037b71..f9ab2a1ff53f42 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -1,10 +1,27 @@ # SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_APPLE_DOCKCHANNEL) += apple-dockchannel.o +apple-dockchannel-y = dockchannel.o + obj-$(CONFIG_APPLE_MAILBOX) += apple-mailbox.o apple-mailbox-y = mailbox.o +obj-$(CONFIG_APPLE_PMGR_MISC) += apple-pmgr-misc.o + +obj-$(CONFIG_APPLE_TUNABLE) += apple-tunable.o +apple-tunable-y = tunable.o + obj-$(CONFIG_APPLE_RTKIT) += apple-rtkit.o apple-rtkit-y = rtkit.o rtkit-crashlog.o +obj-$(CONFIG_APPLE_RTKIT_HELPER) += apple-rtkit-helper.o +apple-rtkit-helper-y = rtkit-helper.o + obj-$(CONFIG_APPLE_SART) += apple-sart.o apple-sart-y = sart.o + +obj-$(CONFIG_APPLE_AOP) += aop.o + +obj-$(CONFIG_APPLE_SEP) += sep.o + +obj-$(CONFIG_APPLE_PMP) += pmp.o diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs new file mode 100644 index 00000000000000..02318c11bd13b7 --- /dev/null +++ b/drivers/soc/apple/aop.rs @@ -0,0 +1,998 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![recursion_limit = "2048"] + +//! Apple AOP driver +//! +//! Copyright (C) The Asahi Linux Contributors + +use core::{arch::asm, cmp, mem, ptr, slice}; + +use kernel::{ + bindings, c_str, device, + device::Core, + dma::{CoherentAllocation, Device, DmaMask}, + error::from_err_ptr, + io::mem::IoMem, + iosys_map::IoSysMapRef, + module_platform_driver, new_condvar, new_mutex, of, platform, + prelude::*, + soc::apple::aop::{from_fourcc, EPICService, FakehidListener, AOP}, + soc::apple::rtkit, + sync::{Arc, ArcBorrow, CondVar, Mutex}, + types::{ARef, ForeignOwnable}, + workqueue::{self, impl_has_work, new_work, Work, WorkItem}, +}; + +const AOP_MAX_CALLS: usize = 8; +const AOP_MMIO_SIZE: usize = 0x1e0000; +const ASC_MMIO_SIZE: usize = 0x4000; +const BOOTARGS_OFFSET: usize = 0x22c; +const BOOTARGS_SIZE: usize = 0x230; +const CPU_CONTROL: usize = 0x44; +const CPU_RUN: u32 = 0x1 << 4; +const AFK_ENDPOINT_START: u8 = 0x20; +const AFK_ENDPOINT_COUNT: u8 = 0xf; +const AFK_OPC_GET_BUF: u64 = 0x89; +const AFK_OPC_INIT: u64 = 0x80; +const AFK_OPC_INIT_RX: u64 = 0x8b; +const AFK_OPC_INIT_TX: u64 = 0x8a; +const AFK_OPC_INIT_UNK: u64 = 0x8c; +const AFK_OPC_SEND: u64 = 0xa2; +const AFK_OPC_START_ACK: u64 = 0x86; +const AFK_OPC_SHUTDOWN_ACK: u64 = 0xc1; +const AFK_OPC_RECV: u64 = 0x85; +const AFK_MSG_GET_BUF_ACK: u64 = 0xa1 << 48; +const AFK_MSG_INIT: u64 = AFK_OPC_INIT << 48; +const AFK_MSG_INIT_ACK: u64 = 0xa0 << 48; +const AFK_MSG_START: u64 = 0xa3 << 48; +const AFK_MSG_SHUTDOWN: u64 = 0xc0 << 48; +const AFK_RB_BLOCK_STEP: usize = 0x40; +const EPIC_TYPE_NOTIFY: u32 = 0; +const EPIC_CATEGORY_REPORT: u8 = 0x00; +const EPIC_CATEGORY_NOTIFY: u8 = 0x10; +const EPIC_CATEGORY_REPLY: u8 = 0x20; +const EPIC_SUBTYPE_STD_SERVICE: u16 = 0xc0; +const EPIC_SUBTYPE_FAKEHID_REPORT: u16 = 0xc4; +const EPIC_SUBTYPE_RETCODE: u16 = 0x84; +const EPIC_SUBTYPE_RETCODE_PAYLOAD: u16 = 0xa0; +const EPIC_SUBTYPE_STRING: u16 = 0x8a; +const QE_MAGIC1: u32 = from_fourcc(b" POI"); +const QE_MAGIC2: u32 = from_fourcc(b" POA"); + +fn align_up(v: usize, a: usize) -> usize { + (v + a - 1) & !(a - 1) +} + +#[inline(always)] +fn mem_sync() { + unsafe { + asm!("dsb sy"); + } +} + +#[repr(C, packed)] +#[derive(Clone, Copy, Default)] +struct QEHeader { + magic: u32, + size: u32, + channel: u32, + ty: u32, +} + +#[repr(C, packed)] +#[derive(Clone, Copy, Default)] +struct EPICHeader { + version: u8, + seq: u16, + _pad0: u8, + _unk0: u32, + timestamp: u64, + // Subheader + length: u32, + sub_version: u8, + category: u8, + subtype: u16, + tag: u16, + _unk1: u16, + _pad1: u64, + inline_len: u32, +} + +#[repr(C, packed)] +struct EPICServiceAnnounce { + name: [u8; 20], + _unk0: u32, + retcode: u32, + _unk1: u32, + channel: u32, + _unk2: u32, + _unk3: u32, +} + +#[pin_data] +struct FutureValue { + #[pin] + val: Mutex>, + #[pin] + completion: CondVar, +} + +impl FutureValue { + fn pin_init() -> impl PinInit> { + pin_init!( + FutureValue { + val <- new_mutex!(None), + completion <- new_condvar!() + } + ) + } + fn complete(&self, val: T) { + *self.val.lock() = Some(val); + self.completion.notify_all(); + } + fn wait(&self) -> T { + let mut ret_guard = self.val.lock(); + while ret_guard.is_none() { + self.completion.wait(&mut ret_guard); + } + ret_guard.take().unwrap() + } + fn reset(&self) { + *self.val.lock() = None; + } +} + +struct AFKRingBuffer { + offset: usize, + block_size: usize, + buf_size: usize, +} + +struct CallResult { + retcode: u32, + extra_data: Option>, +} + +struct AFKEndpoint { + index: u8, + iomem: Option>, + txbuf: Option, + rxbuf: Option, + seq: u16, + calls: [Option>>; AOP_MAX_CALLS], + call_returns: [Option>; AOP_MAX_CALLS], +} + +unsafe impl Send for AFKEndpoint {} + +impl AFKEndpoint { + fn new(index: u8) -> AFKEndpoint { + AFKEndpoint { + index, + iomem: None, + txbuf: None, + rxbuf: None, + seq: 0, + calls: [const { None }; AOP_MAX_CALLS], + call_returns: [const { None }; AOP_MAX_CALLS], + } + } + + fn start(&self, rtkit: Pin<&mut rtkit::RtKit>) -> Result<()> { + rtkit.send_message(self.index, AFK_MSG_INIT) + } + + fn stop(&self, rtkit: Pin<&mut rtkit::RtKit>) -> Result<()> { + rtkit.send_message(self.index, AFK_MSG_SHUTDOWN) + } + + fn recv_message( + &mut self, + client: ArcBorrow<'_, AopData>, + rtkit: Pin<&mut rtkit::RtKit>, + msg: u64, + ) -> Result<()> { + let opc = msg >> 48; + match opc { + AFK_OPC_INIT => { + rtkit.send_message(self.index, AFK_MSG_INIT_ACK)?; + } + AFK_OPC_GET_BUF => { + self.recv_get_buf(client.dev.clone(), rtkit, msg)?; + } + AFK_OPC_INIT_UNK => {} // no-op + AFK_OPC_START_ACK => {} + AFK_OPC_INIT_RX => { + if self.rxbuf.is_some() { + dev_err!( + client.dev, + "Got InitRX message with existing rxbuf at endpoint {}", + self.index + ); + return Err(EIO); + } + self.rxbuf = Some(self.parse_ring_buf(msg)?); + if self.txbuf.is_some() { + rtkit.send_message(self.index, AFK_MSG_START)?; + } + } + AFK_OPC_INIT_TX => { + if self.txbuf.is_some() { + dev_err!( + client.dev, + "Got InitTX message with existing txbuf at endpoint {}", + self.index + ); + return Err(EIO); + } + self.txbuf = Some(self.parse_ring_buf(msg)?); + if self.rxbuf.is_some() { + rtkit.send_message(self.index, AFK_MSG_START)?; + } + } + AFK_OPC_RECV => { + self.recv_rb(client)?; + } + AFK_OPC_SHUTDOWN_ACK => { + client.shutdown_complete(); + } + _ => dev_err!( + client.dev, + "AFK endpoint {} got unknown message {}", + self.index, + msg + ), + } + Ok(()) + } + + fn parse_ring_buf(&self, msg: u64) -> Result { + let msg = msg as usize; + let size = ((msg >> 16) & 0xFFFF) * AFK_RB_BLOCK_STEP; + let offset = ((msg >> 32) & 0xFFFF) * AFK_RB_BLOCK_STEP; + let buf_size = self.iomem_read32(offset)? as usize; + let block_size = (size - buf_size) / 3; + Ok(AFKRingBuffer { + offset, + block_size, + buf_size, + }) + } + fn iomem_write32(&mut self, off: usize, data: u32) -> Result<()> { + let size = core::mem::size_of::(); + let data = data.to_le_bytes(); + let buf = unsafe { self.iomem.as_mut().ok_or(ENXIO)?.as_slice_mut(off, size)? }; + buf.copy_from_slice(&data); + Ok(()) + } + + fn iomem_read32(&self, off: usize) -> Result { + let size = core::mem::size_of::(); + let buf = unsafe { self.iomem.as_ref().ok_or(ENXIO)?.as_slice(off, size)? }; + Ok(u32::from_le_bytes(buf.try_into().unwrap())) + } + + fn memcpy_from_iomem(&self, off: usize, target: &mut [u8]) -> Result<()> { + // SAFETY: + // as_slice() checks that off and target.len() are whithin iomem's limits. + unsafe { + let src = self + .iomem + .as_ref() + .ok_or(ENXIO)? + .as_slice(off, target.len())?; + target.copy_from_slice(src); + } + Ok(()) + } + + fn memcpy_to_iomem(&mut self, off: usize, src: &[u8]) -> Result<()> { + // SAFETY: + // as_slice_mut() checks that off and src.len() are whithin iomem's limits. + unsafe { + let target = self + .iomem + .as_mut() + .ok_or(ENXIO)? + .as_slice_mut(off, src.len())?; + target.copy_from_slice(src); + } + Ok(()) + } + + fn recv_get_buf( + &mut self, + dev: ARef, + rtkit: Pin<&mut rtkit::RtKit>, + msg: u64, + ) -> Result<()> { + let size = ((msg & 0xFFFF0000) >> 16) as usize * AFK_RB_BLOCK_STEP; + if self.iomem.is_some() { + dev_err!( + dev, + "Got GetBuf message with existing buffer on endpoint {}", + self.index + ); + return Err(EIO); + } + let iomem = dev.while_bound_with(|bound_dev| { + CoherentAllocation::::alloc_coherent(bound_dev, size, GFP_KERNEL) + })?; + rtkit.send_message(self.index, AFK_MSG_GET_BUF_ACK | iomem.dma_handle())?; + self.iomem = Some(iomem); + Ok(()) + } + + fn recv_rb(&mut self, client: ArcBorrow<'_, AopData>) -> Result<()> { + let (buf_offset, block_size, buf_size) = match self.rxbuf.as_ref() { + Some(b) => (b.offset, b.block_size, b.buf_size), + None => { + dev_err!( + client.dev, + "Got Recv message with no rxbuf at endpoint {}", + self.index + ); + return Err(EIO); + } + }; + let mut rptr = self.iomem_read32(buf_offset + block_size)? as usize; + let mut wptr = self.iomem_read32(buf_offset + block_size * 2)?; + mem_sync(); + let base = buf_offset + block_size * 3; + let mut msg_buf = KVec::new(); + const QEH_SIZE: usize = mem::size_of::(); + while wptr as usize != rptr { + let mut qeh_bytes = [0; QEH_SIZE]; + self.memcpy_from_iomem(base + rptr, &mut qeh_bytes)?; + let mut qeh = unsafe { &*(qeh_bytes.as_ptr() as *const QEHeader) }; + if qeh.magic != QE_MAGIC1 && qeh.magic != QE_MAGIC2 { + let magic = qeh.magic; + dev_err!( + client.dev, + "Invalid magic on ep {}, got {:x}", + self.index, + magic + ); + return Err(EIO); + } + if qeh.size as usize > (buf_size - rptr - QEH_SIZE) { + rptr = 0; + self.memcpy_from_iomem(base + rptr, &mut qeh_bytes)?; + qeh = unsafe { &*(qeh_bytes.as_ptr() as *const QEHeader) }; + + if qeh.magic != QE_MAGIC1 && qeh.magic != QE_MAGIC2 { + let magic = qeh.magic; + dev_err!( + client.dev, + "Invalid magic on ep {}, got {:x}", + self.index, + magic + ); + return Err(EIO); + } + } + msg_buf.resize(qeh.size as usize, 0, GFP_KERNEL)?; + self.memcpy_from_iomem(base + rptr + QEH_SIZE, &mut msg_buf)?; + let (hdr_bytes, msg) = msg_buf.split_at(mem::size_of::()); + let header = unsafe { &*(hdr_bytes.as_ptr() as *const EPICHeader) }; + self.handle_ipc(client, qeh, header, msg)?; + rptr = align_up(rptr + QEH_SIZE + qeh.size as usize, block_size) % buf_size; + mem_sync(); + self.iomem_write32(buf_offset + block_size, rptr as u32)?; + wptr = self.iomem_read32(buf_offset + block_size * 2)?; + mem_sync(); + } + Ok(()) + } + fn handle_ipc( + &mut self, + client: ArcBorrow<'_, AopData>, + qhdr: &QEHeader, + ehdr: &EPICHeader, + data: &[u8], + ) -> Result<()> { + let subtype = ehdr.subtype; + if ehdr.category == EPIC_CATEGORY_REPORT { + if subtype == EPIC_SUBTYPE_STD_SERVICE { + let announce = unsafe { &*(data.as_ptr() as *const EPICServiceAnnounce) }; + let chan = announce.channel; + let name_len = announce + .name + .iter() + .position(|x| *x == 0) + .unwrap_or(announce.name.len()); + return Into::>::into(client).register_service( + self, + chan, + &announce.name[..name_len], + ); + } else if subtype == EPIC_SUBTYPE_FAKEHID_REPORT { + return client.process_fakehid_report(self, qhdr.channel, data); + } else { + dev_err!( + client.dev, + "Unexpected EPIC report subtype {:x} on endpoint {}", + subtype, + self.index + ); + return Err(EIO); + } + } else if ehdr.category == EPIC_CATEGORY_REPLY { + if subtype == EPIC_SUBTYPE_RETCODE_PAYLOAD + || subtype == EPIC_SUBTYPE_RETCODE + || subtype == EPIC_SUBTYPE_STRING + { + if data.len() < mem::size_of::() { + dev_err!( + client.dev, + "Retcode data too short on endpoint {}", + self.index + ); + return Err(EIO); + } + let retcode = u32::from_ne_bytes(data[..4].try_into().unwrap()); + let tag = ehdr.tag as usize; + if tag == 0 || tag - 1 > self.calls.len() || self.calls[tag - 1].is_none() { + dev_err!( + client.dev, + "Got a retcode with invalid tag {:?} on endpoint {}", + tag, + self.index + ); + return Err(EIO); + } + let future = self.calls[tag - 1].take().unwrap(); + let extra_data = if let Some(mut ret) = self.call_returns[tag - 1].take() { + let len = cmp::min(data.len() - 4, ret.len()); + ret[..len].copy_from_slice(&data[4..(len + 4)]); + ret.truncate(len); + Some(ret) + } else { + None + }; + future.complete(CallResult { + retcode, + extra_data, + }); + + return Ok(()); + } else { + dev_err!( + client.dev, + "Unexpected EPIC reply subtype {:x} on endpoint {}", + subtype, + self.index + ); + return Err(EIO); + } + } + dev_err!( + client.dev, + "Unexpected EPIC category {:x} on endpoint {}", + ehdr.category, + self.index + ); + Err(EIO) + } + fn send_rb( + &mut self, + client: &AopData, + rtkit: Pin<&mut rtkit::RtKit>, + channel: u32, + ty: u32, + header: &[u8], + data: &[u8], + ) -> Result<()> { + let (buf_offset, block_size, buf_size) = match self.txbuf.as_ref() { + Some(b) => (b.offset, b.block_size, b.buf_size), + None => { + dev_err!( + client.dev, + "Attempting to send message with no txbuf at endpoint {}", + self.index + ); + return Err(EIO); + } + }; + let base = buf_offset + block_size * 3; + mem_sync(); + let rptr = self.iomem_read32(buf_offset + block_size)? as usize; + let mut wptr = self.iomem_read32(buf_offset + block_size * 2)? as usize; + const QEH_SIZE: usize = mem::size_of::(); + if wptr < rptr && wptr + QEH_SIZE >= rptr { + dev_err!(client.dev, "Tx buffer full at endpoint {}", self.index); + return Err(EIO); + } + let payload_len = header.len() + data.len(); + let qeh = QEHeader { + magic: QE_MAGIC1, + size: payload_len as u32, + channel, + ty, + }; + let qeh_bytes = unsafe { + slice::from_raw_parts( + &qeh as *const QEHeader as *const u8, + mem::size_of::(), + ) + }; + self.memcpy_to_iomem(base + wptr, qeh_bytes)?; + if payload_len > buf_size - wptr - QEH_SIZE { + wptr = 0; + self.memcpy_to_iomem(base + wptr, qeh_bytes)?; + } + self.memcpy_to_iomem(base + wptr + QEH_SIZE, header)?; + self.memcpy_to_iomem(base + wptr + QEH_SIZE + header.len(), data)?; + wptr = align_up(wptr + QEH_SIZE + payload_len, block_size) % buf_size; + self.iomem_write32(buf_offset + block_size * 2, wptr as u32)?; + let msg = wptr as u64 | (AFK_OPC_SEND << 48); + rtkit.send_message(self.index, msg) + } + fn epic_notify( + &mut self, + client: &AopData, + rtkit: Pin<&mut rtkit::RtKit>, + channel: u32, + subtype: u16, + data: &[u8], + ret: Option>, + ) -> Result>> { + let mut tag = 0; + for i in 0..self.calls.len() { + if self.calls[i].is_none() { + tag = i + 1; + break; + } + } + if tag == 0 { + dev_err!( + client.dev, + "Too many inflight calls on endpoint {}", + self.index + ); + return Err(EIO); + } + let call = Arc::pin_init(FutureValue::pin_init(), GFP_KERNEL)?; + let hdr = EPICHeader { + version: 2, + seq: self.seq, + length: data.len() as u32, + sub_version: 2, + category: EPIC_CATEGORY_NOTIFY, + subtype, + tag: tag as u16, + ..EPICHeader::default() + }; + self.call_returns[tag - 1] = ret; + self.send_rb( + client, + rtkit, + channel, + EPIC_TYPE_NOTIFY, + unsafe { + slice::from_raw_parts( + &hdr as *const EPICHeader as *const u8, + mem::size_of::(), + ) + }, + data, + )?; + self.seq = self.seq.wrapping_add(1); + self.calls[tag - 1] = Some(call.clone()); + Ok(call) + } +} + +struct ListenerEntry { + svc: EPICService, + listener: Arc, +} + +unsafe impl Send for ListenerEntry {} + +#[pin_data] +struct AopData { + dev: ARef, + #[pin] + rtkit: Mutex>>, + #[pin] + endpoints: [Mutex; AFK_ENDPOINT_COUNT as usize], + #[pin] + ep_shutdown: FutureValue<()>, + #[pin] + hid_listeners: Mutex>, + #[pin] + subdevices: Mutex>, +} + +unsafe impl Send for AopData {} +unsafe impl Sync for AopData {} + +#[pin_data] +struct AopServiceRegisterWork { + name: &'static CStr, + data: Arc, + service: EPICService, + #[pin] + work: Work, +} + +impl_has_work! { + impl HasWork for AopServiceRegisterWork { self.work } +} + +impl AopServiceRegisterWork { + fn new( + name: &'static CStr, + data: Arc, + service: EPICService, + ) -> Result>> { + KBox::pin_init( + pin_init!(AopServiceRegisterWork { + name, data, service, + work <- new_work!("AopServiceRegisterWork::work"), + }), + GFP_KERNEL, + ) + } +} + +impl WorkItem for AopServiceRegisterWork { + type Pointer = Pin>; + + fn run(this: Pin>) { + let fwnode = this + .data + .dev + .fwnode() + .and_then(|x| x.get_child_by_name(this.name)); + let info = bindings::platform_device_info { + parent: this.data.dev.as_raw(), + name: this.name.as_ptr() as *const _, + id: bindings::PLATFORM_DEVID_AUTO, + res: ptr::null_mut(), + num_res: 0, + data: &this.service as *const EPICService as *const _, + size_data: mem::size_of::(), + dma_mask: 0, + fwnode: fwnode.map(|x| x.as_raw()).unwrap_or(ptr::null_mut()), + properties: ptr::null_mut(), + of_node_reused: false, + }; + let pdev = unsafe { from_err_ptr(bindings::platform_device_register_full(&info)) }; + match pdev { + Err(e) => { + dev_err!( + this.data.dev, + "Failed to create device for service {:?}: {:?}", + this.name, + e + ); + } + Ok(pdev) => { + let res = this.data.subdevices.lock().push(pdev, GFP_KERNEL); + if res.is_err() { + dev_err!(this.data.dev, "Failed to store subdevice"); + } + } + } + } +} + +impl AopData { + fn new(dev: &platform::Device) -> Result> { + Arc::pin_init( + pin_init!( + AopData { + dev: dev.as_ref().into(), + rtkit <- new_mutex!(None), + endpoints <- pin_init::pin_init_array_from_fn(|i| { + new_mutex!(AFKEndpoint::new(AFK_ENDPOINT_START + i as u8)) + }), + ep_shutdown <- FutureValue::pin_init(), + hid_listeners <- new_mutex!(KVec::new()), + subdevices <- new_mutex!(KVec::new()), + } + ), + GFP_KERNEL, + ) + } + fn start(&self) -> Result<()> { + { + let mut guard = self.rtkit.lock(); + let mut rtk = guard.as_mut().as_pin_mut().unwrap(); + rtk.as_mut().wake()?; + } + for ep in 0..AFK_ENDPOINT_COUNT { + let rtk_ep_num = AFK_ENDPOINT_START + ep; + let mut guard = self.rtkit.lock(); + let mut rtk = guard.as_mut().as_pin_mut().unwrap(); + if !rtk.as_mut().has_endpoint(rtk_ep_num) { + continue; + } + rtk.as_mut().start_endpoint(rtk_ep_num)?; + let ep_guard = self.endpoints[ep as usize].lock(); + ep_guard.start(rtk.as_mut())?; + } + Ok(()) + } + fn register_service( + self: Arc, + ep: &mut AFKEndpoint, + channel: u32, + name: &[u8], + ) -> Result<()> { + let svc = EPICService { + channel, + endpoint: ep.index, + }; + let dev_name = match name { + b"aop-audio" => c_str!("audio"), + b"las" => c_str!("las"), + b"als" => c_str!("als"), + _ => { + return Ok(()); + } + }; + // probe can call back into us, run it with locks dropped. + let work = AopServiceRegisterWork::new(dev_name, self, svc)?; + workqueue::system().enqueue(work); + Ok(()) + } + + fn process_fakehid_report(&self, ep: &AFKEndpoint, ch: u32, data: &[u8]) -> Result<()> { + let guard = self.hid_listeners.lock(); + for entry in &*guard { + if entry.svc.endpoint == ep.index && entry.svc.channel == ch { + return entry.listener.process_fakehid_report(data); + } + } + Ok(()) + } + + fn shutdown_complete(&self) { + self.ep_shutdown.complete(()); + } + + fn stop(&self) -> Result<()> { + for ep in 0..AFK_ENDPOINT_COUNT { + { + let rtk_ep_num = AFK_ENDPOINT_START + ep; + let mut guard = self.rtkit.lock(); + let mut rtk = guard.as_mut().as_pin_mut().unwrap(); + if !rtk.as_mut().has_endpoint(rtk_ep_num) { + continue; + } + let ep_guard = self.endpoints[ep as usize].lock(); + ep_guard.stop(rtk.as_mut())?; + } + self.ep_shutdown.wait(); + self.ep_shutdown.reset(); + } + Ok(()) + } + + fn patch_bootargs( + &self, + aop_mmio: &IoMem, + patches: &[(u32, u64)], + ) -> Result<()> { + let offset = aop_mmio.read32_relaxed(BOOTARGS_OFFSET) as usize; + let size = aop_mmio.read32_relaxed(BOOTARGS_SIZE) as usize; + let mut arg_bytes = KVec::::from_elem(0, size, GFP_KERNEL)?; + aop_mmio.try_memcpy_fromio(&mut arg_bytes, offset)?; + let mut idx = 0; + while idx < size { + let key = u32::from_le_bytes(arg_bytes[idx..idx + 4].try_into().unwrap()); + let size = u32::from_le_bytes(arg_bytes[idx + 4..idx + 8].try_into().unwrap()) as usize; + idx += 8; + for (k, v) in patches.iter() { + if *k != key { + continue; + } + arg_bytes[idx..idx + size].copy_from_slice(&(*v as u64).to_le_bytes()[..size]); + break; + } + idx += size; + } + aop_mmio.try_memcpy_toio(offset, &arg_bytes) + } + + fn start_cpu(&self, asc_mmio: &IoMem) -> Result<()> { + let val = asc_mmio.read32_relaxed(CPU_CONTROL); + asc_mmio.write32_relaxed(val | CPU_RUN, CPU_CONTROL); + Ok(()) + } +} + +impl AOP for AopData { + fn epic_call(&self, svc: &EPICService, subtype: u16, msg_bytes: &[u8]) -> Result { + let ep_idx = svc.endpoint - AFK_ENDPOINT_START; + let call = { + let mut rtk_guard = self.rtkit.lock(); + let mut rtk = rtk_guard.as_mut().as_pin_mut().unwrap(); + let mut ep_guard = self.endpoints[ep_idx as usize].lock(); + ep_guard.epic_notify(self, rtk.as_mut(), svc.channel, subtype, msg_bytes, None)? + }; + Ok(call.wait().retcode) + } + fn epic_call_ret( + &self, + svc: &EPICService, + subtype: u16, + msg_bytes: &[u8], + ret_len: usize, + ) -> Result<(u32, KVec)> { + let ep_idx = svc.endpoint - AFK_ENDPOINT_START; + let call = { + let mut rtk_guard = self.rtkit.lock(); + let mut rtk = rtk_guard.as_mut().as_pin_mut().unwrap(); + let mut ep_guard = self.endpoints[ep_idx as usize].lock(); + let mut ret_buf = KVec::new(); + ret_buf.resize(ret_len, 0, GFP_KERNEL)?; + ep_guard.epic_notify( + self, + rtk.as_mut(), + svc.channel, + subtype, + msg_bytes, + Some(ret_buf), + )? + }; + let res = call.wait(); + Ok((res.retcode, res.extra_data.unwrap())) + } + fn add_fakehid_listener( + &self, + svc: EPICService, + listener: Arc, + ) -> Result<()> { + let mut guard = self.hid_listeners.lock(); + Ok(guard.push(ListenerEntry { svc, listener }, GFP_KERNEL)?) + } + fn remove_fakehid_listener(&self, svc: &EPICService) -> bool { + let mut guard = self.hid_listeners.lock(); + for i in 0..guard.len() { + if guard[i].svc == *svc { + guard.swap_remove(i); + return true; + } + } + false + } + fn remove(&self) { + if let Err(e) = self.stop() { + dev_err!(self.dev, "Failed to stop AOP {:?}", e); + } + *self.rtkit.lock() = None; + let guard = self.subdevices.lock(); + for pdev in &*guard { + unsafe { + bindings::platform_device_unregister(*pdev); + } + } + } +} + +struct NoBuffer; +impl rtkit::Buffer for NoBuffer { + fn iova(&self) -> Result { + unreachable!() + } + fn buf(&mut self) -> Result> { + unreachable!() + } +} + +#[vtable] +impl rtkit::Operations for AopData { + type Data = Arc; + type Buffer = NoBuffer; + + fn recv_message(data: ::Borrowed<'_>, ep: u8, msg: u64) { + let mut guard = data.rtkit.lock(); + let mut rtk = guard.as_mut().as_pin_mut().unwrap(); + let mut ep_guard = data.endpoints[(ep - AFK_ENDPOINT_START) as usize].lock(); + let ret = ep_guard.recv_message(data, rtk.as_mut(), msg); + if let Err(e) = ret { + dev_err!(data.dev, "Failed to handle rtkit message, error: {:?}", e); + } + } + + fn crashed(data: ::Borrowed<'_>, _crashlog: Option<&[u8]>) { + dev_err!(data.dev, "AOP firmware crashed"); + } +} + +#[repr(transparent)] +struct AopDriver(Arc); + +struct AopHwConfig { + ec0p: u64, + alig: u64, + aopt: u64, +} + +const HW_CFG_T8103: AopHwConfig = AopHwConfig { + ec0p: 0x020000, + aopt: 1, + alig: 128, +}; +const HW_CFG_T8112: AopHwConfig = AopHwConfig { + ec0p: 0x020000, + aopt: 0, + alig: 128, +}; +const HW_CFG_T6000: AopHwConfig = AopHwConfig { + ec0p: 0x020000, + aopt: 0, + alig: 64, +}; +const HW_CFG_T6020: AopHwConfig = AopHwConfig { + ec0p: 0x0100_00000000, + aopt: 0, + alig: 64, +}; + +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + ::IdInfo, + [ + (of::DeviceId::new(c_str!("apple,t8103-aop")), &HW_CFG_T8103), + (of::DeviceId::new(c_str!("apple,t8112-aop")), &HW_CFG_T8112), + (of::DeviceId::new(c_str!("apple,t6000-aop")), &HW_CFG_T6000), + (of::DeviceId::new(c_str!("apple,t6020-aop")), &HW_CFG_T6020), + ] +); + +impl platform::Driver for AopDriver { + type IdInfo = &'static AopHwConfig; + + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + fn probe( + pdev: &platform::Device, + info: Option<&Self::IdInfo>, + ) -> impl PinInit { + let cfg = info.ok_or(ENODEV)?; + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::new::<42>())? }; + let aop_req = pdev.io_request_by_index(0).ok_or(EINVAL)?; + let aop_mmio = KBox::pin_init(aop_req.iomap_sized::(), GFP_KERNEL)?; + let asc_req = pdev.io_request_by_index(1).ok_or(EINVAL)?; + let asc_mmio = KBox::pin_init(asc_req.iomap_sized::(), GFP_KERNEL)?; + let data = AopData::new(pdev)?; + let aop_mmio = aop_mmio.access(pdev.as_ref())?; + data.patch_bootargs( + aop_mmio, + &[ + (from_fourcc(b"EC0p"), cfg.ec0p), + (from_fourcc(b"nCal"), 0x0), + (from_fourcc(b"alig"), cfg.alig), + (from_fourcc(b"AOPt"), cfg.aopt), + ], + )?; + let rtkit = rtkit::RtKit::::new(pdev.as_ref(), None, 0, data.clone())?; + *data.rtkit.lock() = Some(rtkit); + let asc_mmio = asc_mmio.access(pdev.as_ref())?; + let _ = data.start_cpu(asc_mmio); + data.start()?; + let data = data as Arc; + Ok(Self(data)) + } +} + +impl Drop for AopDriver { + fn drop(&mut self) { + self.0.remove(); + } +} + +unsafe impl Send for AopDriver {} + +module_platform_driver! { + type: AopDriver, + name: "apple_aop", + description: "AOP driver", + license: "Dual MIT/GPL", +} diff --git a/drivers/soc/apple/apple-pmgr-misc.c b/drivers/soc/apple/apple-pmgr-misc.c new file mode 100644 index 00000000000000..e768f34aacc586 --- /dev/null +++ b/drivers/soc/apple/apple-pmgr-misc.c @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SoC PMGR device power state driver + * + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include + +#define APPLE_CLKGEN_PSTATE 0 +#define APPLE_CLKGEN_PSTATE_DESIRED GENMASK(3, 0) + +#define SYS_DEV_PSTATE_SUSPEND 1 + +enum sys_device { + DEV_FABRIC, + DEV_DCS, + DEV_MAX, +}; + +struct apple_pmgr_sys_device { + void __iomem *base; + u32 active_state; + u32 suspend_state; +}; + +struct apple_pmgr_misc { + struct device *dev; + struct apple_pmgr_sys_device devices[DEV_MAX]; +}; + +static void apple_pmgr_sys_dev_set_pstate(struct apple_pmgr_misc *misc, + enum sys_device dev, bool active) +{ + u32 pstate; + u32 val; + + if (!misc->devices[dev].base) + return; + + if (active) + pstate = misc->devices[dev].active_state; + else + pstate = misc->devices[dev].suspend_state; + + printk("set %d ps to pstate %d\n", dev, pstate); + + val = readl_relaxed(misc->devices[dev].base + APPLE_CLKGEN_PSTATE); + val &= ~APPLE_CLKGEN_PSTATE_DESIRED; + val |= FIELD_PREP(APPLE_CLKGEN_PSTATE_DESIRED, pstate); + writel_relaxed(val, misc->devices[dev].base); +} + +static int __maybe_unused apple_pmgr_misc_suspend_noirq(struct device *dev) +{ + struct apple_pmgr_misc *misc = dev_get_drvdata(dev); + int i; + + for (i = 0; i < DEV_MAX; i++) + apple_pmgr_sys_dev_set_pstate(misc, i, false); + + return 0; +} + +static int __maybe_unused apple_pmgr_misc_resume_noirq(struct device *dev) +{ + struct apple_pmgr_misc *misc = dev_get_drvdata(dev); + int i; + + for (i = 0; i < DEV_MAX; i++) + apple_pmgr_sys_dev_set_pstate(misc, i, true); + + return 0; +} + +static bool apple_pmgr_init_device(struct apple_pmgr_misc *misc, + enum sys_device dev, const char *device_name) +{ + void __iomem *base; + char name[32]; + u32 val; + + snprintf(name, sizeof(name), "%s-ps", device_name); + + base = devm_platform_ioremap_resource_byname( + to_platform_device(misc->dev), name); + if (!base) + return false; + + val = readl_relaxed(base + APPLE_CLKGEN_PSTATE); + + misc->devices[dev].base = base; + misc->devices[dev].active_state = + FIELD_GET(APPLE_CLKGEN_PSTATE_DESIRED, val); + misc->devices[dev].suspend_state = SYS_DEV_PSTATE_SUSPEND; + + snprintf(name, sizeof(name), "apple,%s-min-ps", device_name); + of_property_read_u32(misc->dev->of_node, name, + &misc->devices[dev].suspend_state); + + return true; +} + +static int apple_pmgr_misc_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_pmgr_misc *misc; + int ret = -ENODEV; + + misc = devm_kzalloc(dev, sizeof(*misc), GFP_KERNEL); + if (!misc) + return -ENOMEM; + + misc->dev = dev; + + if (apple_pmgr_init_device(misc, DEV_FABRIC, "fabric")) + ret = 0; + + if (apple_pmgr_init_device(misc, DEV_DCS, "dcs")) + ret = 0; + + platform_set_drvdata(pdev, misc); + + return ret; +} + +static const struct of_device_id apple_pmgr_misc_of_match[] = { + { .compatible = "apple,t6000-pmgr-misc" }, + {} +}; + +MODULE_DEVICE_TABLE(of, apple_pmgr_misc_of_match); + +static const struct dev_pm_ops apple_pmgr_misc_pm_ops = { + SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_pmgr_misc_suspend_noirq, + apple_pmgr_misc_resume_noirq) +}; + +static struct platform_driver apple_pmgr_misc_driver = { + .probe = apple_pmgr_misc_probe, + .driver = { + .name = "apple-pmgr-misc", + .of_match_table = apple_pmgr_misc_of_match, + .pm = pm_ptr(&apple_pmgr_misc_pm_ops), + }, +}; + +MODULE_AUTHOR("Hector Martin "); +MODULE_DESCRIPTION("PMGR misc driver for Apple SoCs"); +MODULE_LICENSE("GPL v2"); + +module_platform_driver(apple_pmgr_misc_driver); diff --git a/drivers/soc/apple/dockchannel.c b/drivers/soc/apple/dockchannel.c new file mode 100644 index 00000000000000..8f67c639da4d20 --- /dev/null +++ b/drivers/soc/apple/dockchannel.c @@ -0,0 +1,405 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple DockChannel FIFO driver + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DOCKCHANNEL_MAX_IRQ 32 + +#define DOCKCHANNEL_TX_TIMEOUT_MS 1000 +#define DOCKCHANNEL_RX_TIMEOUT_MS 1000 + +#define IRQ_MASK 0x0 +#define IRQ_FLAG 0x4 + +#define IRQ_TX BIT(0) +#define IRQ_RX BIT(1) + +#define CONFIG_TX_THRESH 0x0 +#define CONFIG_RX_THRESH 0x4 + +#define DATA_TX8 0x4 +#define DATA_TX16 0x8 +#define DATA_TX24 0xc +#define DATA_TX32 0x10 +#define DATA_TX_FREE 0x14 +#define DATA_RX8 0x1c +#define DATA_RX16 0x20 +#define DATA_RX24 0x24 +#define DATA_RX32 0x28 +#define DATA_RX_COUNT 0x2c + +struct dockchannel { + struct device *dev; + int tx_irq; + int rx_irq; + + void __iomem *config_base; + void __iomem *data_base; + + u32 fifo_size; + bool awaiting; + struct completion tx_comp; + struct completion rx_comp; + + void *cookie; + void (*data_available)(void *cookie, size_t avail); +}; + +struct dockchannel_common { + struct device *dev; + struct irq_domain *domain; + int irq; + + void __iomem *irq_base; +}; + +/* Dockchannel FIFO functions */ + +static irqreturn_t dockchannel_tx_irq(int irq, void *data) +{ + struct dockchannel *dockchannel = data; + + disable_irq_nosync(irq); + complete(&dockchannel->tx_comp); + + return IRQ_HANDLED; +} + +static irqreturn_t dockchannel_rx_irq(int irq, void *data) +{ + struct dockchannel *dockchannel = data; + + disable_irq_nosync(irq); + + if (dockchannel->awaiting) { + return IRQ_WAKE_THREAD; + } else { + complete(&dockchannel->rx_comp); + return IRQ_HANDLED; + } +} + +static irqreturn_t dockchannel_rx_irq_thread(int irq, void *data) +{ + struct dockchannel *dockchannel = data; + size_t avail = readl_relaxed(dockchannel->data_base + DATA_RX_COUNT); + + dockchannel->awaiting = false; + dockchannel->data_available(dockchannel->cookie, avail); + + return IRQ_HANDLED; +} + +int dockchannel_send(struct dockchannel *dockchannel, const void *buf, size_t count) +{ + size_t left = count; + const u8 *p = buf; + + while (left > 0) { + size_t avail = readl_relaxed(dockchannel->data_base + DATA_TX_FREE); + size_t block = min(left, avail); + + if (avail == 0) { + size_t threshold = min((size_t)(dockchannel->fifo_size / 2), left); + + writel_relaxed(threshold, dockchannel->config_base + CONFIG_TX_THRESH); + reinit_completion(&dockchannel->tx_comp); + enable_irq(dockchannel->tx_irq); + + if (!wait_for_completion_timeout(&dockchannel->tx_comp, + msecs_to_jiffies(DOCKCHANNEL_TX_TIMEOUT_MS))) { + disable_irq(dockchannel->tx_irq); + return -ETIMEDOUT; + } + + continue; + } + + while (block >= 4) { + writel_relaxed(get_unaligned_le32(p), dockchannel->data_base + DATA_TX32); + p += 4; + left -= 4; + block -= 4; + } + while (block > 0) { + writeb_relaxed(*p++, dockchannel->data_base + DATA_TX8); + left--; + block--; + } + } + + return count; +} +EXPORT_SYMBOL(dockchannel_send); + +int dockchannel_recv(struct dockchannel *dockchannel, void *buf, size_t count) +{ + size_t left = count; + u8 *p = buf; + + while (left > 0) { + size_t avail = readl_relaxed(dockchannel->data_base + DATA_RX_COUNT); + size_t block = min(left, avail); + + if (avail == 0) { + size_t threshold = min((size_t)(dockchannel->fifo_size / 2), left); + + writel_relaxed(threshold, dockchannel->config_base + CONFIG_RX_THRESH); + reinit_completion(&dockchannel->rx_comp); + enable_irq(dockchannel->rx_irq); + + if (!wait_for_completion_timeout(&dockchannel->rx_comp, + msecs_to_jiffies(DOCKCHANNEL_RX_TIMEOUT_MS))) { + disable_irq(dockchannel->rx_irq); + return -ETIMEDOUT; + } + + continue; + } + + while (block >= 4) { + put_unaligned_le32(readl_relaxed(dockchannel->data_base + DATA_RX32), p); + p += 4; + left -= 4; + block -= 4; + } + while (block > 0) { + *p++ = readl_relaxed(dockchannel->data_base + DATA_RX8) >> 8; + left--; + block--; + } + } + + return count; +} +EXPORT_SYMBOL(dockchannel_recv); + +int dockchannel_await(struct dockchannel *dockchannel, + void (*callback)(void *cookie, size_t avail), + void *cookie, size_t count) +{ + size_t threshold = min((size_t)dockchannel->fifo_size, count); + + if (!count) { + dockchannel->awaiting = false; + disable_irq(dockchannel->rx_irq); + return 0; + } + + dockchannel->data_available = callback; + dockchannel->cookie = cookie; + dockchannel->awaiting = true; + writel_relaxed(threshold, dockchannel->config_base + CONFIG_RX_THRESH); + enable_irq(dockchannel->rx_irq); + + return threshold; +} +EXPORT_SYMBOL(dockchannel_await); + +struct dockchannel *dockchannel_init(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dockchannel *dockchannel; + int ret; + + dockchannel = devm_kzalloc(dev, sizeof(*dockchannel), GFP_KERNEL); + if (!dockchannel) + return ERR_PTR(-ENOMEM); + + dockchannel->dev = dev; + dockchannel->config_base = devm_platform_ioremap_resource_byname(pdev, "config"); + if (IS_ERR(dockchannel->config_base)) + return (__force void *)dockchannel->config_base; + + dockchannel->data_base = devm_platform_ioremap_resource_byname(pdev, "data"); + if (IS_ERR(dockchannel->data_base)) + return (__force void *)dockchannel->data_base; + + ret = of_property_read_u32(dev->of_node, "apple,fifo-size", &dockchannel->fifo_size); + if (ret) + return ERR_PTR(dev_err_probe(dev, ret, "Missing apple,fifo-size property")); + + init_completion(&dockchannel->tx_comp); + init_completion(&dockchannel->rx_comp); + + dockchannel->tx_irq = platform_get_irq_byname(pdev, "tx"); + if (dockchannel->tx_irq <= 0) { + return ERR_PTR(dev_err_probe(dev, dockchannel->tx_irq, + "Failed to get TX IRQ")); + } + + dockchannel->rx_irq = platform_get_irq_byname(pdev, "rx"); + if (dockchannel->rx_irq <= 0) { + return ERR_PTR(dev_err_probe(dev, dockchannel->rx_irq, + "Failed to get RX IRQ")); + } + + ret = devm_request_irq(dev, dockchannel->tx_irq, dockchannel_tx_irq, IRQF_NO_AUTOEN, + "apple-dockchannel-tx", dockchannel); + if (ret) + return ERR_PTR(dev_err_probe(dev, ret, "Failed to request TX IRQ")); + + ret = devm_request_threaded_irq(dev, dockchannel->rx_irq, dockchannel_rx_irq, + dockchannel_rx_irq_thread, IRQF_NO_AUTOEN, + "apple-dockchannel-rx", dockchannel); + if (ret) + return ERR_PTR(dev_err_probe(dev, ret, "Failed to request RX IRQ")); + + return dockchannel; +} +EXPORT_SYMBOL(dockchannel_init); + + +/* Dockchannel IRQchip */ + +static void dockchannel_irq(struct irq_desc *desc) +{ + unsigned int irq = irq_desc_get_irq(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + struct dockchannel_common *dcc = irq_get_handler_data(irq); + unsigned long flags = readl_relaxed(dcc->irq_base + IRQ_FLAG); + int bit; + + chained_irq_enter(chip, desc); + + for_each_set_bit(bit, &flags, DOCKCHANNEL_MAX_IRQ) + generic_handle_domain_irq(dcc->domain, bit); + + chained_irq_exit(chip, desc); +} + +static void dockchannel_irq_ack(struct irq_data *data) +{ + struct dockchannel_common *dcc = irq_data_get_irq_chip_data(data); + unsigned int hwirq = data->hwirq; + + writel_relaxed(BIT(hwirq), dcc->irq_base + IRQ_FLAG); +} + +static void dockchannel_irq_mask(struct irq_data *data) +{ + struct dockchannel_common *dcc = irq_data_get_irq_chip_data(data); + unsigned int hwirq = data->hwirq; + u32 val = readl_relaxed(dcc->irq_base + IRQ_MASK); + + writel_relaxed(val & ~BIT(hwirq), dcc->irq_base + IRQ_MASK); +} + +static void dockchannel_irq_unmask(struct irq_data *data) +{ + struct dockchannel_common *dcc = irq_data_get_irq_chip_data(data); + unsigned int hwirq = data->hwirq; + u32 val = readl_relaxed(dcc->irq_base + IRQ_MASK); + + writel_relaxed(val | BIT(hwirq), dcc->irq_base + IRQ_MASK); +} + +static const struct irq_chip dockchannel_irqchip = { + .name = "dockchannel-irqc", + .irq_ack = dockchannel_irq_ack, + .irq_mask = dockchannel_irq_mask, + .irq_unmask = dockchannel_irq_unmask, +}; + +static int dockchannel_irq_domain_map(struct irq_domain *d, unsigned int virq, + irq_hw_number_t hw) +{ + irq_set_chip_data(virq, d->host_data); + irq_set_chip_and_handler(virq, &dockchannel_irqchip, handle_level_irq); + + return 0; +} + +static const struct irq_domain_ops dockchannel_irq_domain_ops = { + .xlate = irq_domain_xlate_twocell, + .map = dockchannel_irq_domain_map, +}; + +static int dockchannel_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dockchannel_common *dcc; + struct device_node *child; + + dcc = devm_kzalloc(dev, sizeof(*dcc), GFP_KERNEL); + if (!dcc) + return -ENOMEM; + + dcc->dev = dev; + platform_set_drvdata(pdev, dcc); + + dcc->irq_base = devm_platform_ioremap_resource_byname(pdev, "irq"); + if (IS_ERR(dcc->irq_base)) + return PTR_ERR(dcc->irq_base); + + writel_relaxed(0, dcc->irq_base + IRQ_MASK); + writel_relaxed(~0, dcc->irq_base + IRQ_FLAG); + + dcc->domain = irq_domain_add_linear(dev->of_node, DOCKCHANNEL_MAX_IRQ, + &dockchannel_irq_domain_ops, dcc); + if (!dcc->domain) + return -ENOMEM; + + dcc->irq = platform_get_irq(pdev, 0); + if (dcc->irq <= 0) + return dev_err_probe(dev, dcc->irq, "Failed to get IRQ"); + + irq_set_handler_data(dcc->irq, dcc); + irq_set_chained_handler(dcc->irq, dockchannel_irq); + + devm_of_platform_populate(dev); + + return 0; +} + +static void dockchannel_remove(struct platform_device *pdev) +{ + struct dockchannel_common *dcc = platform_get_drvdata(pdev); + int hwirq; + + device_for_each_child(&pdev->dev, NULL, of_platform_device_destroy); + + irq_set_chained_handler_and_data(dcc->irq, NULL, NULL); + + for (hwirq = 0; hwirq < DOCKCHANNEL_MAX_IRQ; hwirq++) + irq_dispose_mapping(irq_find_mapping(dcc->domain, hwirq)); + + irq_domain_remove(dcc->domain); + + writel_relaxed(0, dcc->irq_base + IRQ_MASK); + writel_relaxed(~0, dcc->irq_base + IRQ_FLAG); +} + +static const struct of_device_id dockchannel_of_match[] = { + { .compatible = "apple,dockchannel" }, + {}, +}; +MODULE_DEVICE_TABLE(of, dockchannel_of_match); + +static struct platform_driver dockchannel_driver = { + .driver = { + .name = "dockchannel", + .of_match_table = dockchannel_of_match, + }, + .probe = dockchannel_probe, + .remove = dockchannel_remove, +}; +module_platform_driver(dockchannel_driver); + +MODULE_AUTHOR("Hector Martin "); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple DockChannel driver"); diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 5c48455185c9ba..330015eea2ebe4 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -28,9 +28,9 @@ #include #include #include +#include #include #include -#include "mailbox.h" #define APPLE_ASC_MBOX_CONTROL_FULL BIT(16) #define APPLE_ASC_MBOX_CONTROL_EMPTY BIT(17) diff --git a/drivers/soc/apple/pmp.rs b/drivers/soc/apple/pmp.rs new file mode 100644 index 00000000000000..1742d8207b9461 --- /dev/null +++ b/drivers/soc/apple/pmp.rs @@ -0,0 +1,436 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![recursion_limit = "2048"] + +//! Apple PMP driver +//! +//! Copyright (C) The Asahi Linux Contributors + +use core::{ + mem, + slice, // +}; + +use kernel::{ + bindings, + device::{ + self, + Core, // + }, + devres::Devres, + dma::CoherentAllocation, + io::mem::IoMem, + iosys_map::IoSysMapRef, + module_platform_driver, + new_mutex, + of, + platform, + prelude::*, + soc::apple::rtkit, + sync::{ + Arc, + Mutex, // + }, + types::{ + ARef, + ForeignOwnable, // + }, // +}; + +const PMP_MMIO_SIZE: usize = 0x100000; +const ASC_MMIO_SIZE: usize = 0x4000; +const BOOTARGS_OFFSET: usize = 0x22c; +const BOOTARGS_SIZE: usize = 0x230; +const CPU_CONTROL: usize = 0x44; +const CPU_RUN: u32 = 0x1 << 4; +const PMP_ENDPOINT: u8 = 0x20; +const OPC_GET_IOVA_TABLE: u64 = 0x10; +const OPC_MALLOC: u64 = 0x12; +const OPC_FREE: u64 = 0x14; +const OPC_SET_BUF: u64 = 0x30; +const OPC_REGISTER_IOREG: u64 = 0x32; +const OPC_SET_IOREG: u64 = 0x34; +const OPC_ACK_MASK: u64 = 0x1; +const OPC_SHIFT: u32 = 48; +const MALLOC_SIZE_MASK: u64 = 0xFFFFFF; +const MSG_IOVA_MASK: u64 = 0xFFFFFFFFFFFF; +const SET_IOREG_INDEX_MASK: u64 = 0xFFFF; +const PIO_VM_BASE: u64 = 0xc0000000; +const PIO_GRANULARITY: u64 = 0x1000000; + +const fn from_fourcc(b: &[u8; 4]) -> u32 { + b[3] as u32 | (b[2] as u32) << 8 | (b[1] as u32) << 16 | (b[0] as u32) << 24 +} + +struct PmpAllocation { + addr: u64, + alloc: CoherentAllocation, +} + +struct PmpState { + iova_table: Option>, + allocs: KVec, + value_buf: Option, + ioreg_entries: KVec, +} + +impl PmpState { + fn new() -> Result { + Ok(PmpState { + iova_table: None, + allocs: KVec::with_capacity(10, GFP_KERNEL)?, + value_buf: None, + ioreg_entries: KVec::with_capacity(340, GFP_KERNEL)?, + }) + } + fn find_alloc(&self, addr: u64) -> Option { + // Due to how pmp manages memory, iterating in reverse will + // usually result in us getting the right one on the first try + for (i, e) in self.allocs.iter().enumerate().rev() { + if e.addr == addr { + return Some(i); + } + } + None + } + fn get_buf(&mut self, addr: u64) -> Option<&mut CoherentAllocation> { + let idx = self.find_alloc(addr)?; + Some(&mut self.allocs[idx].alloc) + } +} + +#[pin_data] +struct PmpData { + dev: ARef, + pmp_mmio: Pin>>>, + asc_mmio: Pin>>>, + #[pin] + rtkit: Mutex>>, + #[pin] + state: Mutex, +} + +impl PmpData { + fn new(dev: &platform::Device) -> Result> { + let pmp_req = dev.io_request_by_name(c"pmp").ok_or(EINVAL)?; + let pmp_mmio = KBox::pin_init(pmp_req.iomap_sized::(), GFP_KERNEL)?; + let asc_req = dev.io_request_by_name(c"asc").ok_or(EINVAL)?; + let asc_mmio = KBox::pin_init(asc_req.iomap_sized::(), GFP_KERNEL)?; + Arc::pin_init( + try_pin_init!( + PmpData { + dev: dev.as_ref().into(), + pmp_mmio, + asc_mmio, + rtkit <- new_mutex!(None), + state <- new_mutex!(PmpState::new()?) + } + ), + GFP_KERNEL, + ) + } + fn start_cpu(&self) -> Result<()> { + let asc_mmio = self.asc_mmio.try_access().ok_or(ENXIO)?; + let val = asc_mmio.read32_relaxed(CPU_CONTROL); + asc_mmio.write32_relaxed(val | CPU_RUN, CPU_CONTROL); + Ok(()) + } + fn start(&self) -> Result<()> { + let mut guard = self.rtkit.lock(); + let mut rtk = guard.as_mut().as_pin_mut().unwrap(); + rtk.as_mut().wake()?; + rtk.start_endpoint(PMP_ENDPOINT) + } + fn pmp_read32(&self, off: usize) -> u32 { + if let Some(pmp_mmio) = self.pmp_mmio.try_access() { + pmp_mmio.read32_relaxed(off) + } else { + 0 + } + } + fn patch_bootargs(&self, patches: &[(u32, u32)]) -> Result<()> { + let offset = self.pmp_read32(BOOTARGS_OFFSET) as usize; + let size = self.pmp_read32(BOOTARGS_SIZE) as usize; + let mut arg_bytes = KVec::with_capacity(size, GFP_KERNEL)?; + for _ in 0..size { + arg_bytes.push(0, GFP_KERNEL).unwrap(); + } + { + let pmp_mmio = self.pmp_mmio.try_access().ok_or(ENXIO)?; + pmp_mmio.try_memcpy_fromio(&mut arg_bytes, offset)?; + } + let mut idx = 0; + while idx < size { + let key = u32::from_le_bytes(arg_bytes[idx..idx + 4].try_into().unwrap()); + let size = u32::from_le_bytes(arg_bytes[idx + 4..idx + 8].try_into().unwrap()) as usize; + idx += 8; + for (k, v) in patches.iter() { + if *k != key { + continue; + } + arg_bytes[idx..idx + size].copy_from_slice(&(*v as u64).to_le_bytes()[..size]); + break; + } + idx += size; + } + { + let pmp_mmio = self.pmp_mmio.try_access().ok_or(ENXIO)?; + pmp_mmio.try_memcpy_toio(offset, &arg_bytes) + } + } + fn get_iova_table(&self) -> Result { + let mut state = self.state.lock(); + if state.iova_table.is_some() { + dev_err!(self.dev, "Asked for iova table with existing buffer"); + return Err(EIO); + } + let node = self.dev.fwnode().ok_or(EIO)?; + let mut pio_base = PIO_VM_BASE; + let prop_name = c"apple,pio-ranges"; + let n_entries = node.property_count_elem::(prop_name)? / 2; + let ranges = node + .property_read_array_vec::(prop_name, n_entries * 2)? + .required_by(&self.dev)?; + let mut table = self.dev.while_bound_with(|bound_dev| { + CoherentAllocation::alloc_coherent(bound_dev, 512, GFP_KERNEL) + })?; + for i in 0..table.count() { + unsafe { table.write(&[0], i)? }; + } + + let domain = unsafe { bindings::iommu_get_domain_for_dev(self.dev.as_raw()) }; + for i in 0..n_entries { + let host_addr = ranges[i * 2]; + let size = ranges[i * 2 + 1]; + unsafe { + let err = bindings::iommu_map( + domain, + pio_base as usize, + host_addr, + size as usize, + (bindings::IOMMU_READ | bindings::IOMMU_WRITE | bindings::IOMMU_MMIO) as i32, + bindings::GFP_KERNEL, + ); + if err != 0 { + return Err(Error::from_errno(err)); + } + } + unsafe { table.write(&[host_addr, pio_base, size], i * 3)? }; + pio_base += PIO_GRANULARITY; + } + let msg = (OPC_GET_IOVA_TABLE | OPC_ACK_MASK) << OPC_SHIFT | table.dma_handle(); + state.iova_table = Some(table); + Ok(msg) + } + fn malloc(&self, size: u64) -> Result { + let iomem = self.dev.while_bound_with(|bound_dev| { + CoherentAllocation::alloc_coherent(bound_dev, size as usize, GFP_KERNEL) + })?; + let mut state = self.state.lock(); + let addr = iomem.dma_handle(); + let msg = (OPC_MALLOC | OPC_ACK_MASK) << OPC_SHIFT | addr; + state.allocs.push( + PmpAllocation { + addr: addr, + alloc: iomem, + }, + GFP_KERNEL, + )?; + Ok(msg) + } + fn free(&self, addr: u64) -> Result { + let mut state = self.state.lock(); + if let Some(idx) = state.find_alloc(addr) { + state.allocs.swap_remove(idx); + } else { + dev_err!( + self.dev, + "Attempted to free memory that was not allocated {}", + addr + ); + return Err(EIO); + } + let msg = (OPC_FREE | OPC_ACK_MASK) << OPC_SHIFT; + Ok(msg) + } + fn set_buf(&self, addr: u64) -> Result { + let mut state = self.state.lock(); + if state.value_buf.is_some() { + dev_err!(self.dev, "Setting a buffer when one exists"); + return Err(EIO); + } + let ptr_buf = if let Some(s) = state.get_buf(addr) { + s + } else { + dev_err!(self.dev, "Unable to find buffer"); + return Err(EIO); + }; + if ptr_buf.count() < mem::size_of::() { + dev_err!(self.dev, "Buffer too small"); + return Err(EIO); + } + let ptr = unsafe { *(ptr_buf.start_ptr() as *const u64) }; + state.value_buf = Some(ptr); + let msg = (OPC_SET_BUF | OPC_ACK_MASK) << OPC_SHIFT; + Ok(msg) + } + fn register_ioreg(&self, addr: u64) -> Result { + let mut state = self.state.lock(); + let msg_buf = if let Some(s) = state.get_buf(addr) { + s + } else { + dev_err!(self.dev, "Unable to find buffer"); + return Err(EIO); + }; + if msg_buf.count() < 0x44 { + dev_err!(self.dev, "Buffer too small"); + return Err(EIO); + } + let mut size = unsafe { *(msg_buf.start_ptr().offset(0x40) as *const u32) }; + if size == 0 { + let mut name_vec = KVec::with_capacity(0x31, GFP_KERNEL)?; + name_vec + .extend_from_slice( + unsafe { slice::from_raw_parts(msg_buf.start_ptr(), 0x30) }, + GFP_KERNEL, + ) + .unwrap(); + name_vec.push(0, GFP_KERNEL).unwrap(); + let name_str = CStr::from_bytes_until_nul(&name_vec).unwrap(); + let node = self.dev.fwnode().ok_or(EIO)?; + let tunables = node.get_child_by_name(c"tunables").ok_or(EIO)?; + if state.value_buf.is_none() { + dev_err!(self.dev, "Value buf not set"); + return Err(EIO); + } + let val_buf_addr = state.value_buf.unwrap(); + let val_buf = if let Some(s) = state.get_buf(val_buf_addr) { + s + } else { + dev_err!(self.dev, "Unable to find value buffer"); + return Err(EIO); + }; + if tunables.property_present(name_str) { + let len = tunables.property_count_elem::(name_str)?; + let data = tunables + .property_read_array_vec::(name_str, len)? + .required_by(&self.dev)?; + unsafe { + slice::from_raw_parts_mut(val_buf.start_ptr_mut(), len).copy_from_slice(&data); + } + size = len as u32; + } else { + dev_info!(self.dev, "unknown property {:?}", name_str); + } + } + state.ioreg_entries.push(size, GFP_KERNEL)?; + let index = state.ioreg_entries.len() as u64; + let msg = (OPC_REGISTER_IOREG | OPC_ACK_MASK) << OPC_SHIFT | (index << 32) | size as u64; + Ok(msg) + } + fn set_ioreg(&self, index: u64) -> Result { + let len = *self + .state + .lock() + .ioreg_entries + .get(index as usize) + .ok_or(EIO)? as u64; + let msg = (OPC_SET_IOREG | OPC_ACK_MASK) << OPC_SHIFT | len; + Ok(msg) + } + fn recv_message(&self, msg: u64) -> Result<()> { + let opc = (msg >> OPC_SHIFT) & 0xFF; + let reply = match opc { + OPC_GET_IOVA_TABLE => self.get_iova_table()?, + OPC_MALLOC => self.malloc(msg & MALLOC_SIZE_MASK)?, + OPC_FREE => self.free(msg & MSG_IOVA_MASK)?, + OPC_SET_BUF => self.set_buf(msg & MSG_IOVA_MASK)?, + OPC_REGISTER_IOREG => self.register_ioreg(msg & MSG_IOVA_MASK)?, + OPC_SET_IOREG => self.set_ioreg(msg & SET_IOREG_INDEX_MASK)?, + _ => { + dev_err!(self.dev, "Got unknown message {}", msg); + return Err(EIO); + } + }; + let mut rtk_guard = self.rtkit.lock(); + let rtk = rtk_guard.as_mut().as_pin_mut().unwrap(); + rtk.send_message(PMP_ENDPOINT, reply)?; + Ok(()) + } +} + +unsafe impl Send for PmpData {} +unsafe impl Sync for PmpData {} + +struct NoBuffer; +impl rtkit::Buffer for NoBuffer { + fn iova(&self) -> Result { + unreachable!() + } + fn buf(&mut self) -> Result> { + unreachable!() + } +} + +#[vtable] +impl rtkit::Operations for PmpData { + type Data = Arc; + type Buffer = NoBuffer; + + fn recv_message(data: ::Borrowed<'_>, _ep: u8, msg: u64) { + let ret = data.recv_message(msg); + if let Err(e) = ret { + dev_err!(data.dev, "Failed to handle rtkit message, error: {:?}", e); + } + } + + fn crashed(data: ::Borrowed<'_>, _crashlog: Option<&[u8]>) { + dev_err!(data.dev, "PMP firmware crashed"); + } +} + +#[allow(dead_code)] +struct PmpDriver(Arc); + +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + (), + [(of::DeviceId::new(c"apple,t6000-pmp-v2"), ())] +); + +impl platform::Driver for PmpDriver { + type IdInfo = (); + + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + fn probe(pdev: &platform::Device, _info: Option<&()>) -> impl PinInit { + let dev: ARef = pdev.as_ref().into(); + let data = PmpData::new(pdev)?; + let node = dev.fwnode().ok_or(EIO)?; + let dvid = node + .property_read(c"apple,dram-vendor-id") + .required_by(&dev)?; + let dcap = node + .property_read(c"apple,dram-capacity") + .required_by(&dev)?; + let bdid = node + .property_read(c"apple,board-id") + .required_by(&dev)?; + data.patch_bootargs(&[ + (from_fourcc(b"BDID"), bdid), + (from_fourcc(b"DCAP"), dcap), + (from_fourcc(b"DVID"), dvid), + ])?; + let rtkit = rtkit::RtKit::::new(&dev, None, 0, data.clone())?; + *data.rtkit.lock() = Some(rtkit); + data.start_cpu()?; + data.start()?; + Ok(PmpDriver(data)) + } +} + +module_platform_driver! { + type: PmpDriver, + name: "apple_pmp", + license: "Dual MIT/GPL", +} diff --git a/drivers/soc/apple/rtkit-helper.c b/drivers/soc/apple/rtkit-helper.c new file mode 100644 index 00000000000000..080d083ed9bd2f --- /dev/null +++ b/drivers/soc/apple/rtkit-helper.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple Generic RTKit helper coprocessor + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define APPLE_ASC_CPU_CONTROL 0x44 +#define APPLE_ASC_CPU_CONTROL_RUN BIT(4) + +struct apple_rtkit_helper { + struct device *dev; + struct apple_rtkit *rtk; + + void __iomem *asc_base; + + struct resource *sram; + void __iomem *sram_base; +}; + +static int apple_rtkit_helper_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) +{ + struct apple_rtkit_helper *helper = cookie; + struct resource res = { + .start = bfr->iova, + .end = bfr->iova + bfr->size - 1, + .name = "rtkit_map", + }; + + if (!bfr->iova) { + bfr->buffer = dma_alloc_coherent(helper->dev, bfr->size, + &bfr->iova, GFP_KERNEL); + if (!bfr->buffer) + return -ENOMEM; + return 0; + } + + if (!helper->sram) { + dev_err(helper->dev, + "RTKit buffer request with no SRAM region: %pR", &res); + return -EFAULT; + } + + res.flags = helper->sram->flags; + + if (res.end < res.start || !resource_contains(helper->sram, &res)) { + dev_err(helper->dev, + "RTKit buffer request outside SRAM region: %pR", &res); + return -EFAULT; + } + + bfr->iomem = helper->sram_base + (res.start - helper->sram->start); + bfr->is_mapped = true; + + return 0; +} + +static void apple_rtkit_helper_shmem_destroy(void *cookie, struct apple_rtkit_shmem *bfr) +{ + // no-op +} + +static const struct apple_rtkit_ops apple_rtkit_helper_ops = { + .shmem_setup = apple_rtkit_helper_shmem_setup, + .shmem_destroy = apple_rtkit_helper_shmem_destroy, +}; + +static int apple_rtkit_helper_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_rtkit_helper *helper; + int ret; + + /* 44 bits for addresses in standard RTKit requests */ + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44)); + if (ret) + return ret; + + helper = devm_kzalloc(dev, sizeof(*helper), GFP_KERNEL); + if (!helper) + return -ENOMEM; + + helper->dev = dev; + platform_set_drvdata(pdev, helper); + + helper->asc_base = devm_platform_ioremap_resource_byname(pdev, "asc"); + if (IS_ERR(helper->asc_base)) + return PTR_ERR(helper->asc_base); + + helper->sram = platform_get_resource_byname(pdev, IORESOURCE_MEM, "sram"); + if (helper->sram) { + helper->sram_base = devm_ioremap_resource(dev, helper->sram); + if (IS_ERR(helper->sram_base)) + return dev_err_probe(dev, PTR_ERR(helper->sram_base), + "Failed to map SRAM region"); + } + + helper->rtk = + devm_apple_rtkit_init(dev, helper, NULL, 0, &apple_rtkit_helper_ops); + if (IS_ERR(helper->rtk)) + return dev_err_probe(dev, PTR_ERR(helper->rtk), + "Failed to intialize RTKit"); + + writel_relaxed(APPLE_ASC_CPU_CONTROL_RUN, + helper->asc_base + APPLE_ASC_CPU_CONTROL); + + /* Works for both wake and boot */ + ret = apple_rtkit_wake(helper->rtk); + if (ret != 0) + return dev_err_probe(dev, ret, "Failed to wake up coprocessor"); + + return 0; +} + +static void apple_rtkit_helper_remove(struct platform_device *pdev) +{ + struct apple_rtkit_helper *helper = platform_get_drvdata(pdev); + + if (apple_rtkit_is_running(helper->rtk)) + apple_rtkit_quiesce(helper->rtk); + + writel_relaxed(0, helper->asc_base + APPLE_ASC_CPU_CONTROL); +} + +static const struct of_device_id apple_rtkit_helper_of_match[] = { + { .compatible = "apple,rtk-helper-asc4" }, + {}, +}; +MODULE_DEVICE_TABLE(of, apple_rtkit_helper_of_match); + +static struct platform_driver apple_rtkit_helper_driver = { + .driver = { + .name = "rtkit-helper", + .of_match_table = apple_rtkit_helper_of_match, + }, + .probe = apple_rtkit_helper_probe, + .remove = apple_rtkit_helper_remove, +}; +module_platform_driver(apple_rtkit_helper_driver); + +MODULE_AUTHOR("Hector Martin "); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple RTKit helper driver"); diff --git a/drivers/soc/apple/rtkit-internal.h b/drivers/soc/apple/rtkit-internal.h index b8d5244678f010..c82065a8bf7b03 100644 --- a/drivers/soc/apple/rtkit-internal.h +++ b/drivers/soc/apple/rtkit-internal.h @@ -15,9 +15,9 @@ #include #include #include +#include #include #include -#include "mailbox.h" #define APPLE_RTKIT_APP_ENDPOINT_START 0x20 #define APPLE_RTKIT_MAX_ENDPOINTS 0x100 diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index b8d4da147d23f7..67a21b21a30d98 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -22,6 +22,7 @@ enum { APPLE_RTKIT_EP_DEBUG = 3, APPLE_RTKIT_EP_IOREPORT = 4, APPLE_RTKIT_EP_OSLOG = 8, + APPLE_RTKIT_EP_TRACEKIT = 0xa, }; #define APPLE_RTKIT_MGMT_TYPE GENMASK_ULL(59, 52) @@ -191,6 +192,7 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) case APPLE_RTKIT_EP_DEBUG: case APPLE_RTKIT_EP_IOREPORT: case APPLE_RTKIT_EP_OSLOG: + case APPLE_RTKIT_EP_TRACEKIT: dev_dbg(rtk->dev, "RTKit: Starting system endpoint 0x%02x\n", ep); apple_rtkit_start_ep(rtk, ep); @@ -361,7 +363,7 @@ static void apple_rtkit_memcpy(struct apple_rtkit *rtk, void *dst, static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) { u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg); - u8 *bfr; + u8 *bfr __free(kfree) = NULL; if (type != APPLE_RTKIT_CRASHLOG_CRASH) { dev_warn(rtk->dev, "RTKit: Unknown crashlog message: %llx\n", @@ -394,9 +396,7 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) rtk->crashed = true; if (rtk->ops->crashed) - rtk->ops->crashed(rtk->cookie, bfr, rtk->crashlog_buffer.size); - - kfree(bfr); + rtk->ops->crashed(rtk->cookie, bfr, bfr ? rtk->crashlog_buffer.size : 0); } static void apple_rtkit_ioreport_rx(struct apple_rtkit *rtk, u64 msg) @@ -639,6 +639,12 @@ int apple_rtkit_poll(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_poll); +bool apple_rtkit_has_endpoint(struct apple_rtkit *rtk, u8 ep) +{ + return test_bit(ep, rtk->endpoints); +} +EXPORT_SYMBOL_GPL(apple_rtkit_has_endpoint); + int apple_rtkit_start_ep(struct apple_rtkit *rtk, u8 endpoint) { u64 msg; @@ -958,6 +964,12 @@ struct apple_rtkit *devm_apple_rtkit_init(struct device *dev, void *cookie, } EXPORT_SYMBOL_GPL(devm_apple_rtkit_init); +void devm_apple_rtkit_free(struct device *dev, struct apple_rtkit *rtk) +{ + devm_release_action(dev, apple_rtkit_free_wrapper, rtk); +} +EXPORT_SYMBOL_GPL(devm_apple_rtkit_free); + MODULE_LICENSE("Dual MIT/GPL"); MODULE_AUTHOR("Sven Peter "); MODULE_DESCRIPTION("Apple RTKit driver"); diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs new file mode 100644 index 00000000000000..24d16dd80958ad --- /dev/null +++ b/drivers/soc/apple/sep.rs @@ -0,0 +1,353 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![recursion_limit = "2048"] + +//! Apple SEP driver +//! +//! Copyright (C) The Asahi Linux Contributors + +use core::sync::atomic::{AtomicBool, Ordering}; + +use kernel::{ + bindings, c_str, device, dma, module_platform_driver, new_mutex, of, platform, + prelude::*, + soc::apple::mailbox::{MailCallback, Mailbox, Message}, + sync::{Arc, Mutex}, + types::{ARef, ForeignOwnable}, + workqueue::{self, impl_has_work, new_work, Work, WorkItem}, +}; + +const SHMEM_SIZE: usize = 0x30000; +const MSG_BOOT_TZ0: u64 = 0x5; +const MSG_BOOT_IMG4: u64 = 0x6; +const MSG_SET_SHMEM: u64 = 0x18; +const MSG_BOOT_TZ0_ACK1: u64 = 0x69; +const MSG_BOOT_TZ0_ACK2: u64 = 0xD2; +const MSG_BOOT_IMG4_ACK: u64 = 0x6A; +const MSG_ADVERTISE_EP: u64 = 0; +const EP_DISCOVER: u64 = 0xFD; +const EP_SHMEM: u64 = 0xFE; +const EP_BOOT: u64 = 0xFF; + +const MSG_TYPE_SHIFT: u32 = 16; +const MSG_TYPE_MASK: u64 = 0xFF; +//const MSG_PARAM_SHIFT: u32 = 24; +//const MSG_PARAM_MASK: u64 = 0xFF; + +const MSG_EP_MASK: u64 = 0xFF; +const MSG_DATA_SHIFT: u32 = 32; + +const IOVA_SHIFT: u32 = 0xC; + +type ShMem = dma::CoherentAllocation; + +fn align_up(v: usize, a: usize) -> usize { + (v + a - 1) & !(a - 1) +} + +fn memcpy_to_iomem(iomem: &mut ShMem, off: usize, src: &[u8]) -> Result<()> { + // SAFETY: + // as_slice_mut() checks that off and src.len() are whithin iomem's limits. + // memcpy_to_iomem is only called from within probe() ansuring there are no + // concurrent read and write accesses to the same region while the slice is + // alive per as_slice_mut()'s requiremnts. + unsafe { + let target = iomem.as_slice_mut(off, src.len())?; + target.copy_from_slice(src); + } + Ok(()) +} + +fn build_shmem(dev: &platform::Device) -> Result { + let fwnode = dev.as_ref().fwnode().ok_or(EIO)?; + let mut iomem = + dma::CoherentAllocation::::alloc_coherent(dev.as_ref(), SHMEM_SIZE, GFP_KERNEL)?; + + let panic_offset = 0x4000; + let panic_size = 0x8000; + memcpy_to_iomem(&mut iomem, panic_offset, &1u32.to_le_bytes())?; + + let lpol_offset = panic_offset + panic_size; + let lpol_prop_name = c_str!("local-policy-manifest"); + let lpol_prop_size = fwnode.property_count_elem::(lpol_prop_name)?; + let lpol = fwnode + .property_read_array_vec(lpol_prop_name, lpol_prop_size)? + .required_by(dev.as_ref())?; + memcpy_to_iomem( + &mut iomem, + lpol_offset, + &(lpol_prop_size as u32).to_le_bytes(), + )?; + memcpy_to_iomem(&mut iomem, lpol_offset + 4, &lpol)?; + let lpol_size = align_up(lpol_prop_size + 4, 0x4000); + + let ibot_offset = lpol_offset + lpol_size; + let ibot_prop_name = c_str!("iboot-manifest"); + let ibot_prop_size = fwnode.property_count_elem::(ibot_prop_name)?; + let ibot = fwnode + .property_read_array_vec(ibot_prop_name, ibot_prop_size)? + .required_by(dev.as_ref())?; + memcpy_to_iomem( + &mut iomem, + ibot_offset, + &(ibot_prop_size as u32).to_le_bytes(), + )?; + memcpy_to_iomem(&mut iomem, ibot_offset + 4, &ibot)?; + let ibot_size = align_up(ibot_prop_size + 4, 0x4000); + + memcpy_to_iomem(&mut iomem, 0, b"CNIP")?; + memcpy_to_iomem(&mut iomem, 4, &(panic_size as u32).to_le_bytes())?; + memcpy_to_iomem(&mut iomem, 8, &(panic_offset as u32).to_le_bytes())?; + + memcpy_to_iomem(&mut iomem, 16, b"OPLA")?; + memcpy_to_iomem(&mut iomem, 16 + 4, &(lpol_size as u32).to_le_bytes())?; + memcpy_to_iomem(&mut iomem, 16 + 8, &(lpol_offset as u32).to_le_bytes())?; + + memcpy_to_iomem(&mut iomem, 32, b"IPIS")?; + memcpy_to_iomem(&mut iomem, 32 + 4, &(ibot_size as u32).to_le_bytes())?; + memcpy_to_iomem(&mut iomem, 32 + 8, &(ibot_offset as u32).to_le_bytes())?; + + memcpy_to_iomem(&mut iomem, 48, b"llun")?; + Ok(iomem) +} + +#[pin_data] +struct SepReceiveWork { + data: Arc, + msg: Message, + #[pin] + work: Work, +} + +impl_has_work! { + impl HasWork for SepReceiveWork { self.work } +} + +impl SepReceiveWork { + fn new(data: Arc, msg: Message) -> Result> { + Arc::pin_init( + pin_init!(SepReceiveWork { + data, + msg, + work <- new_work!("SepReceiveWork::work"), + }), + GFP_ATOMIC, + ) + } +} + +impl WorkItem for SepReceiveWork { + type Pointer = Arc; + + fn run(this: Arc) { + this.data.process_message(this.msg); + } +} + +struct FwRegionParams { + addr: u64, + size: usize, +} + +#[pin_data] +struct SepData { + dev: ARef, + #[pin] + mbox: Mutex>>, + shmem: ShMem, + region_params: FwRegionParams, + fw_mapped: AtomicBool, +} + +impl SepData { + fn new( + dev: &platform::Device, + region_params: FwRegionParams, + ) -> Result> { + Arc::pin_init( + try_pin_init!(SepData { + shmem: build_shmem(dev)?, + dev: ARef::::from(dev.as_ref()), + mbox <- new_mutex!(None), + region_params, + fw_mapped: AtomicBool::new(false), + }), + GFP_KERNEL, + ) + } + fn start(&self) -> Result<()> { + self.mbox.lock().as_ref().unwrap().send( + Message { + msg0: EP_BOOT | (MSG_BOOT_TZ0 << MSG_TYPE_SHIFT), + msg1: 0, + }, + false, + ) + } + fn load_fw_and_shmem(&self) -> Result<()> { + let fw_addr = unsafe { + let res = bindings::dma_map_resource( + self.dev.as_raw(), + self.region_params.addr, + self.region_params.size, + bindings::dma_data_direction_DMA_TO_DEVICE, + 0, + ); + if bindings::dma_mapping_error(self.dev.as_raw(), res) != 0 { + dev_err!(self.dev, "Failed to map firmware"); + return Err(ENOMEM); + } + self.fw_mapped.store(true, Ordering::Relaxed); + res >> IOVA_SHIFT + }; + let guard = self.mbox.lock(); + let mbox = guard.as_ref().unwrap(); + mbox.send( + Message { + msg0: EP_BOOT | (MSG_BOOT_IMG4 << MSG_TYPE_SHIFT) | (fw_addr << MSG_DATA_SHIFT), + msg1: 0, + }, + false, + )?; + let shm_addr = self.shmem.dma_handle() >> IOVA_SHIFT; + mbox.send( + Message { + msg0: EP_SHMEM | (MSG_SET_SHMEM << MSG_TYPE_SHIFT) | (shm_addr << MSG_DATA_SHIFT), + msg1: 0, + }, + false, + )?; + Ok(()) + } + fn process_boot_msg(&self, msg: Message) { + let ty = (msg.msg0 >> MSG_TYPE_SHIFT) & MSG_TYPE_MASK; + match ty { + MSG_BOOT_TZ0_ACK1 => {} + MSG_BOOT_TZ0_ACK2 => { + let res = self.load_fw_and_shmem(); + if let Err(e) = res { + dev_err!(self.dev, "Unable to load firmware: {:?}", e); + } + } + MSG_BOOT_IMG4_ACK => {} + _ => { + dev_err!(self.dev, "Unknown boot message type: {}", ty); + } + } + } + fn process_discover_msg(&self, msg: Message) { + let ty = (msg.msg0 >> MSG_TYPE_SHIFT) & MSG_TYPE_MASK; + //let data = (msg.msg0 >> MSG_DATA_SHIFT) as u32; + //let param = (msg.msg0 >> MSG_PARAM_SHIFT) & MSG_PARAM_MASK; + match ty { + MSG_ADVERTISE_EP => { + /*dev_info!( + self.dev, + "Got endpoint {:?} at {}", + core::str::from_utf8(&data.to_be_bytes()), + param + );*/ + } + _ => { + //dev_warn!(self.dev, "Unknown discovery message type: {}", ty); + } + } + } + fn process_message(&self, msg: Message) { + let ep = msg.msg0 & MSG_EP_MASK; + match ep { + EP_BOOT => self.process_boot_msg(msg), + EP_DISCOVER => self.process_discover_msg(msg), + _ => {} // dev_warn!(self.dev, "Message from unknown endpoint: {}", ep), + } + } + fn remove(&self) { + *self.mbox.lock() = None; + if self.fw_mapped.load(Ordering::Relaxed) { + unsafe { + bindings::dma_unmap_resource( + self.dev.as_raw(), + self.region_params.addr, + self.region_params.size, + bindings::dma_data_direction_DMA_TO_DEVICE, + 0, + ); + } + } + } +} + +impl MailCallback for SepData { + type Data = Arc; + fn recv_message(data: ::Borrowed<'_>, msg: Message) { + let work = SepReceiveWork::new(data.into(), msg); + if let Ok(work) = work { + let res = workqueue::system().enqueue(work); + if res.is_err() { + dev_err!( + data.dev, + "Unable to schedule work item for message {}", + msg.msg0 + ); + } + } else { + dev_err!( + data.dev, + "Unable to allocate work item for message {}", + msg.msg0 + ); + } + } +} + +unsafe impl Send for SepData {} +unsafe impl Sync for SepData {} + +struct SepDriver(Arc); + +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + (), + [(of::DeviceId::new(c_str!("apple,sep")), ())] +); + +impl platform::Driver for SepDriver { + type IdInfo = (); + + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + fn probe( + pdev: &platform::Device, + _info: Option<&()>, + ) -> impl PinInit { + let of = pdev.as_ref().of_node().ok_or(EIO)?; + let res = of.reserved_mem_region_to_resource_byname(c_str!("sepfw"))?; + let data = SepData::new( + pdev, + FwRegionParams { + addr: res.start(), + size: res.size().try_into()?, + }, + )?; + *data.mbox.lock() = Some(Mailbox::new_byname( + pdev.as_ref(), + c_str!("mbox"), + data.clone(), + )?); + data.start()?; + Ok(Self(data)) + } +} + +impl Drop for SepDriver { + fn drop(&mut self) { + self.0.remove(); + } +} + +module_platform_driver! { + type: SepDriver, + name: "apple_sep", + description: "Secure enclave processor stub driver", + license: "Dual MIT/GPL", +} diff --git a/drivers/soc/apple/tunable.c b/drivers/soc/apple/tunable.c new file mode 100644 index 00000000000000..6593238391715b --- /dev/null +++ b/drivers/soc/apple/tunable.c @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple Silicon hardware tunable support + * + * Each tunable is a list with each entry containing a offset into the MMIO + * region, a mask of bits to be cleared and a set of bits to be set. These + * tunables are passed along by the previous boot stages and vary from device + * to device such that they cannot be hardcoded in the individual drivers. + * + * Copyright (C) The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include + +struct apple_tunable *devm_apple_tunable_parse(struct device *dev, + struct device_node *np, + const char *name, + struct resource *res) +{ + struct apple_tunable *tunable; + struct property *prop; + const __be32 *p; + size_t sz; + int i; + + if (resource_size(res) < 4) + return ERR_PTR(-EINVAL); + + prop = of_find_property(np, name, NULL); + if (!prop) + return ERR_PTR(-ENOENT); + + if (prop->length % (3 * sizeof(u32))) + return ERR_PTR(-EINVAL); + sz = prop->length / (3 * sizeof(u32)); + + tunable = devm_kzalloc(dev, struct_size(tunable, values, sz), GFP_KERNEL); + if (!tunable) + return ERR_PTR(-ENOMEM); + tunable->sz = sz; + + for (i = 0, p = NULL; i < tunable->sz; ++i) { + p = of_prop_next_u32(prop, p, &tunable->values[i].offset); + p = of_prop_next_u32(prop, p, &tunable->values[i].mask); + p = of_prop_next_u32(prop, p, &tunable->values[i].value); + + /* Sanity checks to catch bugs in our bootloader */ + if (tunable->values[i].offset % 4) + return ERR_PTR(-EINVAL); + if (tunable->values[i].offset > (resource_size(res) - 4)) + return ERR_PTR(-EINVAL); + } + + return tunable; +} +EXPORT_SYMBOL(devm_apple_tunable_parse); + +void apple_tunable_apply(void __iomem *regs, struct apple_tunable *tunable) +{ + size_t i; + + for (i = 0; i < tunable->sz; ++i) { + u32 val, old_val; + + old_val = readl(regs + tunable->values[i].offset); + val = old_val & ~tunable->values[i].mask; + val |= tunable->values[i].value; + if (val != old_val) + writel(val, regs + tunable->values[i].offset); + } +} +EXPORT_SYMBOL(apple_tunable_apply); + +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_AUTHOR("Sven Peter "); +MODULE_DESCRIPTION("Apple Silicon hardware tunable support"); diff --git a/drivers/soc/mediatek/mtk-svs.c b/drivers/soc/mediatek/mtk-svs.c index f45537546553ec..99edecb204f254 100644 --- a/drivers/soc/mediatek/mtk-svs.c +++ b/drivers/soc/mediatek/mtk-svs.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -789,7 +790,7 @@ static ssize_t svs_enable_debug_write(struct file *filp, struct svs_bank *svsb = file_inode(filp)->i_private; struct svs_platform *svsp = dev_get_drvdata(svsb->dev); int enabled, ret; - char *buf = NULL; + char *buf __free(kfree) = NULL; if (count >= PAGE_SIZE) return -EINVAL; @@ -807,8 +808,6 @@ static ssize_t svs_enable_debug_write(struct file *filp, svsb->mode_support = SVSB_MODE_ALL_DISABLE; } - kfree(buf); - return count; } diff --git a/drivers/soc/qcom/cmd-db.c b/drivers/soc/qcom/cmd-db.c index ae66c2623d250d..84a75d8c4b7020 100644 --- a/drivers/soc/qcom/cmd-db.c +++ b/drivers/soc/qcom/cmd-db.c @@ -349,15 +349,16 @@ static int cmd_db_dev_probe(struct platform_device *pdev) return -EINVAL; } - cmd_db_header = memremap(rmem->base, rmem->size, MEMREMAP_WC); - if (!cmd_db_header) { - ret = -ENOMEM; + cmd_db_header = devm_memremap(&pdev->dev, rmem->base, rmem->size, MEMREMAP_WC); + if (IS_ERR(cmd_db_header)) { + ret = PTR_ERR(cmd_db_header); cmd_db_header = NULL; return ret; } if (!cmd_db_magic_matches(cmd_db_header)) { dev_err(&pdev->dev, "Invalid Command DB Magic\n"); + cmd_db_header = NULL; return -EINVAL; } diff --git a/drivers/soc/qcom/smem.c b/drivers/soc/qcom/smem.c index c18a0c946f7627..d5c94b47f431f1 100644 --- a/drivers/soc/qcom/smem.c +++ b/drivers/soc/qcom/smem.c @@ -1219,7 +1219,9 @@ static int qcom_smem_probe(struct platform_device *pdev) smem->item_count = qcom_smem_get_item_count(smem); break; case SMEM_GLOBAL_HEAP_VERSION: - qcom_smem_map_global(smem, size); + ret = qcom_smem_map_global(smem, size); + if (ret < 0) + return ret; smem->item_count = SMEM_ITEM_COUNT; break; default: diff --git a/drivers/soundwire/Kconfig b/drivers/soundwire/Kconfig index ad56393e4c93b8..196a7daaabdb88 100644 --- a/drivers/soundwire/Kconfig +++ b/drivers/soundwire/Kconfig @@ -40,6 +40,7 @@ config SOUNDWIRE_INTEL select AUXILIARY_BUS depends on ACPI && SND_SOC depends on SND_SOC_SOF_HDA_MLINK || !SND_SOC_SOF_HDA_MLINK + depends on SND_HDA_CORE || !SND_HDA_ALIGNED_MMIO help SoundWire Intel Master driver. If you have an Intel platform which has a SoundWire Master then diff --git a/drivers/spi/spi-cadence-quadspi.c b/drivers/spi/spi-cadence-quadspi.c index 965b4cea3388a3..b9a560c75c5cd5 100644 --- a/drivers/spi/spi-cadence-quadspi.c +++ b/drivers/spi/spi-cadence-quadspi.c @@ -2012,10 +2012,8 @@ static int cqspi_probe(struct platform_device *pdev) goto probe_setup_failed; } - if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) { - pm_runtime_mark_last_busy(dev); + if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) pm_runtime_put_autosuspend(dev); - } return 0; probe_setup_failed: diff --git a/drivers/spi/spi-microchip-core-spi.c b/drivers/spi/spi-microchip-core-spi.c index 89e40fc45d73aa..c8ebb58e0369af 100644 --- a/drivers/spi/spi-microchip-core-spi.c +++ b/drivers/spi/spi-microchip-core-spi.c @@ -161,7 +161,7 @@ static int mchp_corespi_setup(struct spi_device *spi) return -EOPNOTSUPP; } - if (spi->mode & SPI_MODE_X_MASK & ~spi->controller->mode_bits) { + if ((spi->mode ^ spi->controller->mode_bits) & SPI_MODE_X_MASK) { dev_err(&spi->dev, "incompatible CPOL/CPHA, must match controller's Motorola mode\n"); return -EINVAL; } diff --git a/drivers/spi/spi-wpcm-fiu.c b/drivers/spi/spi-wpcm-fiu.c index a9aee2a6c7dcbc..c47b56f0933f14 100644 --- a/drivers/spi/spi-wpcm-fiu.c +++ b/drivers/spi/spi-wpcm-fiu.c @@ -459,11 +459,11 @@ static int wpcm_fiu_probe(struct platform_device *pdev) res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "memory"); fiu->memory = devm_ioremap_resource(dev, res); - fiu->memory_size = min_t(size_t, resource_size(res), MAX_MEMORY_SIZE_TOTAL); if (IS_ERR(fiu->memory)) return dev_err_probe(dev, PTR_ERR(fiu->memory), "Failed to map flash memory window\n"); + fiu->memory_size = min_t(size_t, resource_size(res), MAX_MEMORY_SIZE_TOTAL); fiu->shm_regmap = syscon_regmap_lookup_by_phandle_optional(dev->of_node, "nuvoton,shm"); wpcm_fiu_hw_init(fiu); diff --git a/drivers/spmi/spmi-apple-controller.c b/drivers/spmi/spmi-apple-controller.c index 697b3e8bb02356..87e3ee9d4f2aa5 100644 --- a/drivers/spmi/spmi-apple-controller.c +++ b/drivers/spmi/spmi-apple-controller.c @@ -149,6 +149,7 @@ static int apple_spmi_probe(struct platform_device *pdev) } static const struct of_device_id apple_spmi_match_table[] = { + { .compatible = "apple,t8103-spmi", }, { .compatible = "apple,spmi", }, {} }; diff --git a/drivers/staging/greybus/light.c b/drivers/staging/greybus/light.c index e509fdc715dbbf..38c233a706c483 100644 --- a/drivers/staging/greybus/light.c +++ b/drivers/staging/greybus/light.c @@ -1008,14 +1008,18 @@ static int gb_lights_light_config(struct gb_lights *glights, u8 id) if (!strlen(conf.name)) return -EINVAL; - light->channels_count = conf.channel_count; light->name = kstrndup(conf.name, NAMES_MAX, GFP_KERNEL); if (!light->name) return -ENOMEM; - light->channels = kcalloc(light->channels_count, + light->channels = kcalloc(conf.channel_count, sizeof(struct gb_channel), GFP_KERNEL); if (!light->channels) return -ENOMEM; + /* + * Publish channels_count only after channels allocation so cleanup + * doesn't walk a NULL channels pointer on allocation failure. + */ + light->channels_count = conf.channel_count; /* First we collect all the configurations for all channels */ for (i = 0; i < light->channels_count; i++) { diff --git a/drivers/thermal/intel/x86_pkg_temp_thermal.c b/drivers/thermal/intel/x86_pkg_temp_thermal.c index 3fc679b6f11b17..aab5f9fca9c33c 100644 --- a/drivers/thermal/intel/x86_pkg_temp_thermal.c +++ b/drivers/thermal/intel/x86_pkg_temp_thermal.c @@ -128,6 +128,9 @@ sys_set_trip_temp(struct thermal_zone_device *tzd, u32 l, h, mask, shift, intr; int tj_max, val, ret; + if (temp == THERMAL_TEMP_INVALID) + temp = 0; + tj_max = intel_tcc_get_tjmax(zonedev->cpu); if (tj_max < 0) return tj_max; diff --git a/drivers/thermal/thermal_of.c b/drivers/thermal/thermal_of.c index 1a51a4d240ff60..b6d0c92f5522bd 100644 --- a/drivers/thermal/thermal_of.c +++ b/drivers/thermal/thermal_of.c @@ -280,10 +280,10 @@ static bool thermal_of_cm_lookup(struct device_node *cm_np, struct cooling_spec *c) { for_each_child_of_node_scoped(cm_np, child) { - struct device_node *tr_np; int count, i; - tr_np = of_parse_phandle(child, "trip", 0); + struct device_node *tr_np __free(device_node) = + of_parse_phandle(child, "trip", 0); if (tr_np != trip->priv) continue; diff --git a/drivers/tty/serial/Kconfig b/drivers/tty/serial/Kconfig index 59221cce0028f6..3d06c65c2f491d 100644 --- a/drivers/tty/serial/Kconfig +++ b/drivers/tty/serial/Kconfig @@ -486,14 +486,14 @@ config SERIAL_IMX can enable its onboard serial port by enabling this option. config SERIAL_IMX_CONSOLE - tristate "Console on IMX serial port" + bool "Console on IMX serial port" depends on SERIAL_IMX select SERIAL_CORE_CONSOLE help If you have enabled the serial port on the Freescale IMX - CPU you can make it the console by answering Y/M to this option. + CPU you can make it the console by answering Y to this option. - Even if you say Y/M here, the currently visible virtual console + Even if you say Y here, the currently visible virtual console (/dev/tty0) will still be used as the system console by default, but you can alter that using a kernel command line option such as "console=ttymxc0". (Try "man bootparam" or see the documentation of @@ -671,7 +671,7 @@ config SERIAL_SH_SCI_EARLYCON default ARCH_RENESAS config SERIAL_SH_SCI_DMA - bool "DMA support" if EXPERT + bool "Support for DMA on SuperH SCI(F)" if EXPERT depends on SERIAL_SH_SCI && DMA_ENGINE default ARCH_RENESAS diff --git a/drivers/tty/serial/samsung_tty.c b/drivers/tty/serial/samsung_tty.c index c1fabad6ba1faf..99d83271d8dfbe 100644 --- a/drivers/tty/serial/samsung_tty.c +++ b/drivers/tty/serial/samsung_tty.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -1298,30 +1299,49 @@ static int apple_s5l_serial_startup(struct uart_port *port) return ret; } +static int __maybe_unused s3c24xx_serial_runtime_suspend(struct device *dev) +{ + struct uart_port *port = dev_get_drvdata(dev); + struct s3c24xx_uart_port *ourport = to_ourport(port); + int timeout = 10000; + + while (--timeout && !s3c24xx_serial_txempty_nofifo(port)) + udelay(100); + + if (!IS_ERR(ourport->baudclk)) + clk_disable_unprepare(ourport->baudclk); + + clk_disable_unprepare(ourport->clk); + return 0; +}; + +static int __maybe_unused s3c24xx_serial_runtime_resume(struct device *dev) +{ + struct uart_port *port = dev_get_drvdata(dev); + struct s3c24xx_uart_port *ourport = to_ourport(port); + + clk_prepare_enable(ourport->clk); + + if (!IS_ERR(ourport->baudclk)) + clk_prepare_enable(ourport->baudclk); + return 0; +}; + static void s3c24xx_serial_pm(struct uart_port *port, unsigned int level, unsigned int old) { struct s3c24xx_uart_port *ourport = to_ourport(port); - int timeout = 10000; ourport->pm_level = level; switch (level) { - case 3: - while (--timeout && !s3c24xx_serial_txempty_nofifo(port)) - udelay(100); - - if (!IS_ERR(ourport->baudclk)) - clk_disable_unprepare(ourport->baudclk); - - clk_disable_unprepare(ourport->clk); + case UART_PM_STATE_OFF: + pm_runtime_mark_last_busy(port->dev); + pm_runtime_put_sync(port->dev); break; - case 0: - clk_prepare_enable(ourport->clk); - - if (!IS_ERR(ourport->baudclk)) - clk_prepare_enable(ourport->baudclk); + case UART_PM_STATE_ON: + pm_runtime_get_sync(port->dev); break; default: dev_err(port->dev, "s3c24xx_serial: unknown pm %d\n", level); @@ -2044,18 +2064,15 @@ static int s3c24xx_serial_probe(struct platform_device *pdev) } } + pm_runtime_get_noresume(&pdev->dev); + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); + dev_dbg(&pdev->dev, "%s: adding port\n", __func__); uart_add_one_port(&s3c24xx_uart_drv, &ourport->port); platform_set_drvdata(pdev, &ourport->port); - /* - * Deactivate the clock enabled in s3c24xx_serial_init_port here, - * so that a potential re-enablement through the pm-callback overlaps - * and keeps the clock enabled in this case. - */ - clk_disable_unprepare(ourport->clk); - if (!IS_ERR(ourport->baudclk)) - clk_disable_unprepare(ourport->baudclk); + pm_runtime_put_sync(&pdev->dev); probe_index++; @@ -2065,26 +2082,40 @@ static int s3c24xx_serial_probe(struct platform_device *pdev) static void s3c24xx_serial_remove(struct platform_device *dev) { struct uart_port *port = s3c24xx_dev_to_port(&dev->dev); + struct s3c24xx_uart_port *ourport = to_ourport(port); - if (port) + if (port) { + pm_runtime_get_sync(&dev->dev); uart_remove_one_port(&s3c24xx_uart_drv, port); + clk_disable_unprepare(ourport->clk); + if (!IS_ERR(ourport->baudclk)) + clk_disable_unprepare(ourport->baudclk); + + pm_runtime_disable(&dev->dev); + pm_runtime_set_suspended(&dev->dev); + pm_runtime_put_noidle(&dev->dev); + } + uart_unregister_driver(&s3c24xx_uart_drv); } /* UART power management code */ -#ifdef CONFIG_PM_SLEEP -static int s3c24xx_serial_suspend(struct device *dev) + +static int __maybe_unused s3c24xx_serial_suspend(struct device *dev) { struct uart_port *port = s3c24xx_dev_to_port(dev); + if (!console_suspend_enabled && uart_console(port)) + device_set_wakeup_path(dev); + if (port) uart_suspend_port(&s3c24xx_uart_drv, port); return 0; } -static int s3c24xx_serial_resume(struct device *dev) +static int __maybe_unused s3c24xx_serial_resume(struct device *dev) { struct uart_port *port = s3c24xx_dev_to_port(dev); struct s3c24xx_uart_port *ourport = to_ourport(port); @@ -2104,7 +2135,7 @@ static int s3c24xx_serial_resume(struct device *dev) return 0; } -static int s3c24xx_serial_resume_noirq(struct device *dev) +static int __maybe_unused s3c24xx_serial_resume_noirq(struct device *dev) { struct uart_port *port = s3c24xx_dev_to_port(dev); struct s3c24xx_uart_port *ourport = to_ourport(port); @@ -2178,13 +2209,9 @@ static int s3c24xx_serial_resume_noirq(struct device *dev) static const struct dev_pm_ops s3c24xx_serial_pm_ops = { SET_SYSTEM_SLEEP_PM_OPS(s3c24xx_serial_suspend, s3c24xx_serial_resume) SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(NULL, s3c24xx_serial_resume_noirq) + SET_RUNTIME_PM_OPS(s3c24xx_serial_runtime_suspend, + s3c24xx_serial_runtime_resume, NULL) }; -#define SERIAL_SAMSUNG_PM_OPS (&s3c24xx_serial_pm_ops) - -#else /* !CONFIG_PM_SLEEP */ - -#define SERIAL_SAMSUNG_PM_OPS NULL -#endif /* CONFIG_PM_SLEEP */ /* Console code */ @@ -2672,7 +2699,7 @@ static struct platform_driver samsung_serial_driver = { .id_table = s3c24xx_serial_driver_ids, .driver = { .name = "samsung-uart", - .pm = SERIAL_SAMSUNG_PM_OPS, + .pm = &s3c24xx_serial_pm_ops, .of_match_table = of_match_ptr(s3c24xx_uart_dt_match), }, }; diff --git a/drivers/ufs/host/Kconfig b/drivers/ufs/host/Kconfig index 7d5117b2dab4a7..964ae70e7390d0 100644 --- a/drivers/ufs/host/Kconfig +++ b/drivers/ufs/host/Kconfig @@ -72,6 +72,7 @@ config SCSI_UFS_QCOM config SCSI_UFS_MEDIATEK tristate "Mediatek specific hooks to UFS controller platform driver" depends on SCSI_UFSHCD_PLATFORM && ARCH_MEDIATEK + depends on PM depends on RESET_CONTROLLER select PHY_MTK_UFS select RESET_TI_SYSCON diff --git a/drivers/ufs/host/ufs-mediatek.c b/drivers/ufs/host/ufs-mediatek.c index 66b11cc0703bd2..b3daaa07e92521 100644 --- a/drivers/ufs/host/ufs-mediatek.c +++ b/drivers/ufs/host/ufs-mediatek.c @@ -2437,7 +2437,6 @@ static void ufs_mtk_remove(struct platform_device *pdev) ufshcd_pltfrm_remove(pdev); } -#ifdef CONFIG_PM_SLEEP static int ufs_mtk_system_suspend(struct device *dev) { struct ufs_hba *hba = dev_get_drvdata(dev); @@ -2484,9 +2483,7 @@ static int ufs_mtk_system_resume(struct device *dev) return ret; } -#endif -#ifdef CONFIG_PM static int ufs_mtk_runtime_suspend(struct device *dev) { struct ufs_hba *hba = dev_get_drvdata(dev); @@ -2525,13 +2522,10 @@ static int ufs_mtk_runtime_resume(struct device *dev) return ufshcd_runtime_resume(dev); } -#endif static const struct dev_pm_ops ufs_mtk_pm_ops = { - SET_SYSTEM_SLEEP_PM_OPS(ufs_mtk_system_suspend, - ufs_mtk_system_resume) - SET_RUNTIME_PM_OPS(ufs_mtk_runtime_suspend, - ufs_mtk_runtime_resume, NULL) + SYSTEM_SLEEP_PM_OPS(ufs_mtk_system_suspend, ufs_mtk_system_resume) + RUNTIME_PM_OPS(ufs_mtk_runtime_suspend, ufs_mtk_runtime_resume, NULL) .prepare = ufshcd_suspend_prepare, .complete = ufshcd_resume_complete, }; @@ -2541,7 +2535,7 @@ static struct platform_driver ufs_mtk_pltform = { .remove = ufs_mtk_remove, .driver = { .name = "ufshcd-mtk", - .pm = &ufs_mtk_pm_ops, + .pm = pm_ptr(&ufs_mtk_pm_ops), .of_match_table = ufs_mtk_of_match, }, }; diff --git a/drivers/usb/cdns3/core.c b/drivers/usb/cdns3/core.c index 1243a5cea91b52..f0e32227c0b791 100644 --- a/drivers/usb/cdns3/core.c +++ b/drivers/usb/cdns3/core.c @@ -551,7 +551,7 @@ int cdns_resume(struct cdns *cdns) } } - if (cdns->roles[cdns->role]->resume) + if (!role_changed && cdns->roles[cdns->role]->resume) cdns->roles[cdns->role]->resume(cdns, power_lost); return 0; diff --git a/drivers/usb/gadget/udc/bdc/bdc_core.c b/drivers/usb/gadget/udc/bdc/bdc_core.c index 5c3d8b64c0e76e..f47aac078f6be6 100644 --- a/drivers/usb/gadget/udc/bdc/bdc_core.c +++ b/drivers/usb/gadget/udc/bdc/bdc_core.c @@ -35,8 +35,8 @@ static int poll_oip(struct bdc *bdc, u32 usec) u32 status; int ret; - ret = readl_poll_timeout(bdc->regs + BDC_BDCSC, status, - (BDC_CSTS(status) != BDC_OIP), 10, usec); + ret = readl_poll_timeout_atomic(bdc->regs + BDC_BDCSC, status, + (BDC_CSTS(status) != BDC_OIP), 10, usec); if (ret) dev_err(bdc->dev, "operation timedout BDCSC: 0x%08x\n", status); else diff --git a/drivers/usb/host/Kconfig b/drivers/usb/host/Kconfig index c4f17ce5c77b15..5f3ecce8398b39 100644 --- a/drivers/usb/host/Kconfig +++ b/drivers/usb/host/Kconfig @@ -51,6 +51,15 @@ config USB_XHCI_PCI_RENESAS installed on your system for this device to work. If unsure, say 'N'. +config USB_XHCI_PCI_ASMEDIA + bool "Support firmware loading for ASMedia xHCI controllers" + default USB_XHCI_PCI if ARCH_APPLE + depends on USB_XHCI_PCI + help + Say 'Y' to enable support for ASMedia xHCI controllers with + host-supplied firmware. These are usually present on Apple devices. + If unsure, say 'N'. + config USB_XHCI_PLATFORM tristate "Generic xHCI driver for a platform device" help diff --git a/drivers/usb/host/Makefile b/drivers/usb/host/Makefile index 4df946c05ba0ee..a197e0979f89bd 100644 --- a/drivers/usb/host/Makefile +++ b/drivers/usb/host/Makefile @@ -72,6 +72,8 @@ obj-$(CONFIG_USB_UHCI_HCD) += uhci-hcd.o obj-$(CONFIG_USB_FHCI_HCD) += fhci.o obj-$(CONFIG_USB_XHCI_HCD) += xhci-hcd.o obj-$(CONFIG_USB_XHCI_PCI) += xhci-pci.o +xhci-pci-y += xhci-pci-core.o +xhci-pci-$(CONFIG_USB_XHCI_PCI_ASMEDIA) += xhci-pci-asmedia.o obj-$(CONFIG_USB_XHCI_PCI_RENESAS) += xhci-pci-renesas.o obj-$(CONFIG_USB_XHCI_PLATFORM) += xhci-plat-hcd.o obj-$(CONFIG_USB_XHCI_HISTB) += xhci-histb.o diff --git a/drivers/usb/host/xhci-pci-asmedia.c b/drivers/usb/host/xhci-pci-asmedia.c new file mode 100644 index 00000000000000..d6b12f5c540296 --- /dev/null +++ b/drivers/usb/host/xhci-pci-asmedia.c @@ -0,0 +1,414 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * ASMedia xHCI firmware loader + * Copyright (C) The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include + +#include "xhci.h" +#include "xhci-trace.h" +#include "xhci-pci.h" + +/* Configuration space registers */ +#define ASMT_CFG_CONTROL 0xe0 +#define ASMT_CFG_CONTROL_WRITE BIT(1) +#define ASMT_CFG_CONTROL_READ BIT(0) + +#define ASMT_CFG_SRAM_ADDR 0xe2 + +#define ASMT_CFG_SRAM_ACCESS 0xef +#define ASMT_CFG_SRAM_ACCESS_READ BIT(6) +#define ASMT_CFG_SRAM_ACCESS_ENABLE BIT(7) + +#define ASMT_CFG_DATA_READ0 0xf0 +#define ASMT_CFG_DATA_READ1 0xf4 + +#define ASMT_CFG_DATA_WRITE0 0xf8 +#define ASMT_CFG_DATA_WRITE1 0xfc + +#define ASMT_CMD_GET_FWVER 0x8000060840 +#define ASMT_FWVER_ROM 0x010250090816 + +/* BAR0 registers */ +#define ASMT_REG_ADDR 0x3000 + +#define ASMT_REG_WDATA 0x3004 +#define ASMT_REG_RDATA 0x3008 + +#define ASMT_REG_STATUS 0x3009 +#define ASMT_REG_STATUS_BUSY BIT(7) + +#define ASMT_REG_CODE_WDATA 0x3010 +#define ASMT_REG_CODE_RDATA 0x3018 + +#define ASMT_MMIO_CPU_MISC 0x500e +#define ASMT_MMIO_CPU_MISC_CODE_RAM_WR BIT(0) + +#define ASMT_MMIO_CPU_MODE_NEXT 0x5040 +#define ASMT_MMIO_CPU_MODE_CUR 0x5041 + +#define ASMT_MMIO_CPU_MODE_RAM BIT(0) +#define ASMT_MMIO_CPU_MODE_HALFSPEED BIT(1) + +#define ASMT_MMIO_CPU_EXEC_CTRL 0x5042 +#define ASMT_MMIO_CPU_EXEC_CTRL_RESET BIT(0) +#define ASMT_MMIO_CPU_EXEC_CTRL_HALT BIT(1) + +#define TIMEOUT_USEC 10000 +#define RESET_TIMEOUT_USEC 500000 + +static int asmedia_mbox_tx(struct pci_dev *pdev, u64 data) +{ + u8 op; + int ret, err; + + ret = read_poll_timeout(pci_read_config_byte, err, + err || !(op & ASMT_CFG_CONTROL_WRITE), + 1, TIMEOUT_USEC, false, pdev, ASMT_CFG_CONTROL, + &op); + if (ret) { + dev_err(&pdev->dev, + "Timed out on mailbox tx: 0x%llx\n", + data); + return ret; + } + if (err) + return err; + + pci_write_config_dword(pdev, ASMT_CFG_DATA_WRITE0, data); + pci_write_config_dword(pdev, ASMT_CFG_DATA_WRITE1, data >> 32); + pci_write_config_byte(pdev, ASMT_CFG_CONTROL, ASMT_CFG_CONTROL_WRITE); + + return 0; +} + +static int asmedia_mbox_rx(struct pci_dev *pdev, u64 *data) +{ + u8 op; + u32 low, high; + int ret, err; + + ret = read_poll_timeout(pci_read_config_byte, err, + err || (op & ASMT_CFG_CONTROL_READ), + 1, TIMEOUT_USEC, false, pdev, ASMT_CFG_CONTROL, + &op); + if (ret) { + dev_err(&pdev->dev, "Timed out on mailbox rx\n"); + return ret; + } + if (err) + return err; + + pci_read_config_dword(pdev, ASMT_CFG_DATA_READ0, &low); + pci_read_config_dword(pdev, ASMT_CFG_DATA_READ1, &high); + pci_write_config_byte(pdev, ASMT_CFG_CONTROL, ASMT_CFG_CONTROL_READ); + + *data = ((u64)high << 32) | low; + return 0; +} + +static int asmedia_get_fw_version(struct pci_dev *pdev, u64 *version) +{ + int err = 0; + u64 cmd; + + err = asmedia_mbox_tx(pdev, ASMT_CMD_GET_FWVER); + if (err) + return err; + err = asmedia_mbox_tx(pdev, 0); + if (err) + return err; + + err = asmedia_mbox_rx(pdev, &cmd); + if (err) + return err; + err = asmedia_mbox_rx(pdev, version); + if (err) + return err; + + if (cmd != ASMT_CMD_GET_FWVER) { + dev_err(&pdev->dev, "Unexpected reply command 0x%llx\n", cmd); + return -EIO; + } + + return 0; +} + +static bool asmedia_check_firmware(struct pci_dev *pdev) +{ + u64 fwver; + int ret; + + ret = asmedia_get_fw_version(pdev, &fwver); + if (ret) + return ret; + + dev_info(&pdev->dev, "Firmware version: 0x%llx\n", fwver); + + return fwver != ASMT_FWVER_ROM; +} + +static int asmedia_wait_reset(struct pci_dev *pdev) +{ + struct usb_hcd *hcd = dev_get_drvdata(&pdev->dev); + struct xhci_cap_regs __iomem *cap = hcd->regs; + struct xhci_op_regs __iomem *op; + u32 val; + int ret; + + op = hcd->regs + HC_LENGTH(readl(&cap->hc_capbase)); + + ret = readl_poll_timeout(&op->command, + val, !(val & CMD_RESET), + 1000, RESET_TIMEOUT_USEC); + + if (!ret) + return 0; + + dev_err(hcd->self.controller, "Reset timed out, trying to kick it\n"); + + pci_write_config_byte(pdev, ASMT_CFG_SRAM_ACCESS, + ASMT_CFG_SRAM_ACCESS_ENABLE); + + pci_write_config_byte(pdev, ASMT_CFG_SRAM_ACCESS, 0); + + ret = readl_poll_timeout(&op->command, + val, !(val & CMD_RESET), + 1000, RESET_TIMEOUT_USEC); + + if (ret) + dev_err(hcd->self.controller, "Reset timed out, giving up\n"); + + return ret; +} + +static int asmedia_read_reg(struct usb_hcd *hcd, u16 addr, u8 *val) { + void __iomem *regs = hcd->regs; + u8 status; + int ret; + + ret = readb_poll_timeout(regs + ASMT_REG_STATUS, status, + !(status & ASMT_REG_STATUS_BUSY), + 1000, TIMEOUT_USEC); + + if (ret) { + dev_err(hcd->self.controller, + "Read reg wait timed out ([%04x])\n", addr); + return ret; + } + + writew_relaxed(addr, regs + ASMT_REG_ADDR); + + ret = readb_poll_timeout(regs + ASMT_REG_STATUS, status, + !(status & ASMT_REG_STATUS_BUSY), + 1000, TIMEOUT_USEC); + + if (ret) { + dev_err(hcd->self.controller, + "Read reg addr timed out ([%04x])\n", addr); + return ret; + } + + *val = readb_relaxed(regs + ASMT_REG_RDATA); + return 0; +} + +static int asmedia_write_reg(struct usb_hcd *hcd, u16 addr, u8 data, bool wait) { + void __iomem *regs = hcd->regs; + u8 status, val; + int ret, err; + + writew_relaxed(addr, regs + ASMT_REG_ADDR); + + ret = readb_poll_timeout(regs + ASMT_REG_STATUS, status, + !(status & ASMT_REG_STATUS_BUSY), + 1000, TIMEOUT_USEC); + + if (ret) { + dev_err(hcd->self.controller, + "Write reg addr timed out ([%04x] = %02x)\n", + addr, data); + return ret; + } + + writeb_relaxed(data, regs + ASMT_REG_WDATA); + + ret = readb_poll_timeout(regs + ASMT_REG_STATUS, status, + !(status & ASMT_REG_STATUS_BUSY), + 1000, TIMEOUT_USEC); + + if (ret) { + dev_err(hcd->self.controller, + "Write reg data timed out ([%04x] = %02x)\n", + addr, data); + return ret; + } + + if (!wait) + return 0; + + ret = read_poll_timeout(asmedia_read_reg, err, err || val == data, + 0, TIMEOUT_USEC, false, hcd, addr, &val); + if (ret) { + dev_err(hcd->self.controller, + "Verify register timed out ([%04x] = %02x)\n", + addr, data); + return ret; + } + if (err) { + dev_err(hcd->self.controller, + "Verify register read error ([%04x] = %02x)\n", + addr, data); + return err; + } + + return 0; +} + +static int asmedia_load_fw(struct pci_dev *pdev, const struct firmware *fw) +{ + struct usb_hcd *hcd; + void __iomem *regs; + const u16 *fw_data = (const u16 *)fw->data; + u16 raddr; + u32 data; + size_t index = 0, addr = 0; + size_t words = fw->size >> 1; + int ret, err; + + hcd = dev_get_drvdata(&pdev->dev); + regs = hcd->regs; + + ret = asmedia_write_reg(hcd, ASMT_MMIO_CPU_MODE_NEXT, + ASMT_MMIO_CPU_MODE_HALFSPEED, false); + if (ret) + return ret; + + ret = asmedia_write_reg(hcd, ASMT_MMIO_CPU_EXEC_CTRL, + ASMT_MMIO_CPU_EXEC_CTRL_RESET, false); + if (ret) + return ret; + + ret = asmedia_wait_reset(pdev); + if (ret) { + dev_err(hcd->self.controller, "Failed pre-upload reset\n"); + return ret; + } + + ret = asmedia_write_reg(hcd, ASMT_MMIO_CPU_EXEC_CTRL, + ASMT_MMIO_CPU_EXEC_CTRL_HALT, false); + if (ret) + return ret; + + ret = asmedia_write_reg(hcd, ASMT_MMIO_CPU_MISC, + ASMT_MMIO_CPU_MISC_CODE_RAM_WR, true); + if (ret) + return ret; + + pci_write_config_byte(pdev, ASMT_CFG_SRAM_ACCESS, + ASMT_CFG_SRAM_ACCESS_ENABLE); + + /* The firmware upload is interleaved in 0x4000 word blocks */ + addr = index = 0; + while (index < words) { + data = fw_data[index]; + if ((index | 0x4000) < words) + data |= fw_data[index | 0x4000] << 16; + + pci_write_config_word(pdev, ASMT_CFG_SRAM_ADDR, + addr); + + writel_relaxed(data, regs + ASMT_REG_CODE_WDATA); + + ret = read_poll_timeout(pci_read_config_word, err, + err || (raddr != addr), + 1, TIMEOUT_USEC, false, pdev, + ASMT_CFG_SRAM_ADDR, &raddr); + if (ret) { + dev_err(hcd->self.controller, "Word write timed out\n"); + return ret; + } + if (err) + return err; + + if (++index & 0x4000) + index += 0x4000; + addr += 2; + } + + pci_write_config_byte(pdev, ASMT_CFG_SRAM_ACCESS, 0); + + ret = asmedia_write_reg(hcd, ASMT_MMIO_CPU_MISC, 0, true); + if (ret) + return ret; + + ret = asmedia_write_reg(hcd, ASMT_MMIO_CPU_MODE_NEXT, + ASMT_MMIO_CPU_MODE_RAM | + ASMT_MMIO_CPU_MODE_HALFSPEED, false); + if (ret) + return ret; + + ret = asmedia_write_reg(hcd, ASMT_MMIO_CPU_EXEC_CTRL, 0, false); + if (ret) + return ret; + + ret = asmedia_wait_reset(pdev); + if (ret) { + dev_err(hcd->self.controller, "Failed post-upload reset\n"); + return ret; + } + + return 0; +} + +int asmedia_xhci_check_request_fw(struct pci_dev *pdev, + const struct pci_device_id *id) +{ + struct xhci_driver_data *driver_data = + (struct xhci_driver_data *)id->driver_data; + const char *fw_name = driver_data->firmware; + const struct firmware *fw; + int ret; + + /* Check if device has firmware, if so skip everything */ + ret = asmedia_check_firmware(pdev); + if (ret < 0) + return ret; + else if (ret == 1) + return 0; + + pci_dev_get(pdev); + ret = request_firmware(&fw, fw_name, &pdev->dev); + pci_dev_put(pdev); + if (ret) { + dev_err(&pdev->dev, "Could not load firmware %s: %d\n", + fw_name, ret); + return ret; + } + + ret = asmedia_load_fw(pdev, fw); + if (ret) { + dev_err(&pdev->dev, "Firmware upload failed: %d\n", ret); + goto err; + } + + ret = asmedia_check_firmware(pdev); + if (ret < 0) { + goto err; + } else if (ret != 1) { + dev_err(&pdev->dev, "Firmware version is too old after upload\n"); + ret = -EIO; + } else { + ret = 0; + } + +err: + release_firmware(fw); + return ret; +} diff --git a/drivers/usb/host/xhci-pci.c b/drivers/usb/host/xhci-pci-core.c similarity index 97% rename from drivers/usb/host/xhci-pci.c rename to drivers/usb/host/xhci-pci-core.c index 585b2f3117b08a..c58cdcdb3cc862 100644 --- a/drivers/usb/host/xhci-pci.c +++ b/drivers/usb/host/xhci-pci-core.c @@ -569,6 +569,18 @@ static int xhci_pci_setup(struct usb_hcd *hcd) struct pci_dev *pdev = to_pci_dev(hcd->self.controller); int retval; u8 sbrn; + struct xhci_driver_data *driver_data; + const struct pci_device_id *id; + + id = pci_match_id(to_pci_driver(pdev->dev.driver)->id_table, pdev); + if (id && id->driver_data && usb_hcd_is_primary_hcd(hcd)) { + driver_data = (struct xhci_driver_data *)id->driver_data; + if (driver_data->quirks & XHCI_ASMEDIA_FW_QUIRK) { + retval = asmedia_xhci_check_request_fw(pdev, id); + if (retval < 0) + return retval; + } + } xhci = hcd_to_xhci(hcd); @@ -938,10 +950,19 @@ static void xhci_pci_shutdown(struct usb_hcd *hcd) pci_set_power_state(pdev, PCI_D3hot); } +#define ASMEDIA_APPLE_FW_NAME "asmedia/asm2214a-apple.bin" + /*-------------------------------------------------------------------------*/ +static const struct xhci_driver_data asmedia_data = { + .quirks = XHCI_ASMEDIA_FW_QUIRK, + .firmware = ASMEDIA_APPLE_FW_NAME, +}; /* PCI driver selection metadata; PCI hotplugging uses this */ static const struct pci_device_id pci_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_ASMEDIA, 0x2142), + .driver_data = (unsigned long)&asmedia_data, + }, /* handle any USB 3.0 xHCI controller */ { PCI_DEVICE_CLASS(PCI_CLASS_SERIAL_USB_XHCI, ~0), }, @@ -949,6 +970,10 @@ static const struct pci_device_id pci_ids[] = { }; MODULE_DEVICE_TABLE(pci, pci_ids); +#if IS_ENABLED(CONFIG_USB_XHCI_PCI_ASMEDIA) +MODULE_FIRMWARE(ASMEDIA_APPLE_FW_NAME); +#endif + /* pci driver glue; this is a "new style" PCI driver module */ static struct pci_driver xhci_pci_driver = { .name = hcd_name, diff --git a/drivers/usb/host/xhci-pci.h b/drivers/usb/host/xhci-pci.h index e87c7d9d76b8e2..452908d1c069ba 100644 --- a/drivers/usb/host/xhci-pci.h +++ b/drivers/usb/host/xhci-pci.h @@ -7,4 +7,22 @@ int xhci_pci_common_probe(struct pci_dev *dev, const struct pci_device_id *id); void xhci_pci_remove(struct pci_dev *dev); +struct xhci_driver_data { + u64 quirks; + const char *firmware; +}; + +#if IS_ENABLED(CONFIG_USB_XHCI_PCI_ASMEDIA) +int asmedia_xhci_check_request_fw(struct pci_dev *dev, + const struct pci_device_id *id); + +#else +static inline int asmedia_xhci_check_request_fw(struct pci_dev *dev, + const struct pci_device_id *id) +{ + return 0; +} + +#endif + #endif diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h index 2b0796f6d00eae..d262671a3803ee 100644 --- a/drivers/usb/host/xhci.h +++ b/drivers/usb/host/xhci.h @@ -1644,6 +1644,7 @@ struct xhci_hcd { #define XHCI_CDNS_SCTX_QUIRK BIT_ULL(48) #define XHCI_ETRON_HOST BIT_ULL(49) #define XHCI_LIMIT_ENDPOINT_INTERVAL_9 BIT_ULL(50) +#define XHCI_ASMEDIA_FW_QUIRK BIT_ULL(51) unsigned int num_active_eps; unsigned int limit_active_eps; diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c index 9f2cc5fb9f4562..d4505a4264460b 100644 --- a/drivers/usb/serial/option.c +++ b/drivers/usb/serial/option.c @@ -1401,12 +1401,16 @@ static const struct usb_device_id option_ids[] = { .driver_info = NCTRL(0) | RSVD(1) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a0, 0xff), /* Telit FN20C04 (rmnet) */ .driver_info = RSVD(0) | NCTRL(3) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a1, 0xff), /* Telit FN20C04 (RNDIS) */ + .driver_info = NCTRL(4) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a2, 0xff), /* Telit FN920C04 (MBIM) */ .driver_info = NCTRL(4) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a3, 0xff), /* Telit FN920C04 (ECM) */ .driver_info = NCTRL(4) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a4, 0xff), /* Telit FN20C04 (rmnet) */ .driver_info = RSVD(0) | NCTRL(3) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a6, 0xff), /* Telit FN920C04 (RNDIS) */ + .driver_info = NCTRL(4) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a7, 0xff), /* Telit FN920C04 (MBIM) */ .driver_info = NCTRL(4) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a8, 0xff), /* Telit FN920C04 (ECM) */ @@ -1415,6 +1419,8 @@ static const struct usb_device_id option_ids[] = { .driver_info = RSVD(0) | NCTRL(2) | RSVD(3) | RSVD(4) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10aa, 0xff), /* Telit FN920C04 (MBIM) */ .driver_info = NCTRL(3) | RSVD(4) | RSVD(5) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10ab, 0xff), /* Telit FN920C04 (RNDIS) */ + .driver_info = NCTRL(3) | RSVD(4) | RSVD(5) }, { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x10b0, 0xff, 0xff, 0x30), /* Telit FE990B (rmnet) */ .driver_info = NCTRL(5) }, { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x10b0, 0xff, 0xff, 0x40) }, diff --git a/drivers/usb/typec/tcpm/fusb302.c b/drivers/usb/typec/tcpm/fusb302.c index 870a71f953f6cd..19ff8217818e7e 100644 --- a/drivers/usb/typec/tcpm/fusb302.c +++ b/drivers/usb/typec/tcpm/fusb302.c @@ -1756,8 +1756,7 @@ static int fusb302_probe(struct i2c_client *client) } ret = request_irq(chip->gpio_int_n_irq, fusb302_irq_intn, - IRQF_ONESHOT | IRQF_TRIGGER_LOW, - "fsc_interrupt_int_n", chip); + IRQF_TRIGGER_LOW, "fsc_interrupt_int_n", chip); if (ret < 0) { dev_err(dev, "cannot request IRQ for GPIO Int_N, ret=%d", ret); goto tcpm_unregister_port; diff --git a/drivers/usb/typec/ucsi/Kconfig b/drivers/usb/typec/ucsi/Kconfig index b812be4d0e674e..87dd992a4b9e9a 100644 --- a/drivers/usb/typec/ucsi/Kconfig +++ b/drivers/usb/typec/ucsi/Kconfig @@ -73,7 +73,6 @@ config CROS_EC_UCSI tristate "UCSI Driver for ChromeOS EC" depends on MFD_CROS_EC_DEV depends on CROS_USBPD_NOTIFY - depends on !EXTCON_TCSS_CROS_EC default MFD_CROS_EC_DEV help This driver enables UCSI support for a ChromeOS EC. The EC is diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c index cf45f6370c3698..d1e8053640a98c 100644 --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c @@ -1188,9 +1188,32 @@ hisi_acc_vfio_pci_get_device_state(struct vfio_device *vdev, return 0; } +static void hisi_acc_vf_pci_reset_prepare(struct pci_dev *pdev) +{ + struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev); + struct hisi_qm *qm = hisi_acc_vdev->pf_qm; + struct device *dev = &qm->pdev->dev; + u32 delay = 0; + + /* All reset requests need to be queued for processing */ + while (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) { + msleep(1); + if (++delay > QM_RESET_WAIT_TIMEOUT) { + dev_err(dev, "reset prepare failed\n"); + return; + } + } + + hisi_acc_vdev->set_reset_flag = true; +} + static void hisi_acc_vf_pci_aer_reset_done(struct pci_dev *pdev) { struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev); + struct hisi_qm *qm = hisi_acc_vdev->pf_qm; + + if (hisi_acc_vdev->set_reset_flag) + clear_bit(QM_RESETTING, &qm->misc_ctl); if (hisi_acc_vdev->core_device.vdev.migration_flags != VFIO_MIGRATION_STOP_COPY) @@ -1734,6 +1757,7 @@ static const struct pci_device_id hisi_acc_vfio_pci_table[] = { MODULE_DEVICE_TABLE(pci, hisi_acc_vfio_pci_table); static const struct pci_error_handlers hisi_acc_vf_err_handlers = { + .reset_prepare = hisi_acc_vf_pci_reset_prepare, .reset_done = hisi_acc_vf_pci_aer_reset_done, .error_detected = vfio_pci_core_aer_err_detected, }; diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h index cd55eba64dfb23..a3d91a31e3d886 100644 --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h @@ -27,6 +27,7 @@ #define ERROR_CHECK_TIMEOUT 100 #define CHECK_DELAY_TIME 100 +#define QM_RESET_WAIT_TIMEOUT 60000 #define QM_SQC_VFT_BASE_SHIFT_V2 28 #define QM_SQC_VFT_BASE_MASK_V2 GENMASK(15, 0) @@ -128,6 +129,7 @@ struct hisi_acc_vf_migration_file { struct hisi_acc_vf_core_device { struct vfio_pci_core_device core_device; u8 match_done; + bool set_reset_flag; /* * io_base is only valid when dev_opened is true, * which is protected by open_mutex. diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index 3a11e6f450f701..72c33b399800eb 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -588,6 +588,7 @@ EXPORT_SYMBOL_GPL(vfio_pci_core_enable); void vfio_pci_core_disable(struct vfio_pci_core_device *vdev) { + struct pci_dev *bridge; struct pci_dev *pdev = vdev->pdev; struct vfio_pci_dummy_resource *dummy_res, *tmp; struct vfio_pci_ioeventfd *ioeventfd, *ioeventfd_tmp; @@ -694,12 +695,20 @@ void vfio_pci_core_disable(struct vfio_pci_core_device *vdev) * We can not use the "try" reset interface here, which will * overwrite the previously restored configuration information. */ - if (vdev->reset_works && pci_dev_trylock(pdev)) { - if (!__pci_reset_function_locked(pdev)) - vdev->needs_reset = false; - pci_dev_unlock(pdev); + if (vdev->reset_works) { + bridge = pci_upstream_bridge(pdev); + if (bridge && !pci_dev_trylock(bridge)) + goto out_restore_state; + if (pci_dev_trylock(pdev)) { + if (!__pci_reset_function_locked(pdev)) + vdev->needs_reset = false; + pci_dev_unlock(pdev); + } + if (bridge) + pci_dev_unlock(bridge); } +out_restore_state: pci_restore_state(pdev); out: pci_disable_device(pdev); diff --git a/drivers/video/backlight/aw99706.c b/drivers/video/backlight/aw99706.c index df5b23b2f75340..938f352aaab7f8 100644 --- a/drivers/video/backlight/aw99706.c +++ b/drivers/video/backlight/aw99706.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/video/backlight/qcom-wled.c b/drivers/video/backlight/qcom-wled.c index a63bb42c8f8b03..8054e4787725ef 100644 --- a/drivers/video/backlight/qcom-wled.c +++ b/drivers/video/backlight/qcom-wled.c @@ -1244,6 +1244,15 @@ static const struct wled_var_cfg wled4_ovp_cfg = { .size = ARRAY_SIZE(wled4_ovp_values), }; +static const u32 pmi8994_wled_ovp_values[] = { + 31000, 29500, 19400, 17800, +}; + +static const struct wled_var_cfg pmi8994_wled_ovp_cfg = { + .values = pmi8994_wled_ovp_values, + .size = ARRAY_SIZE(pmi8994_wled_ovp_values), +}; + static inline u32 wled5_ovp_values_fn(u32 idx) { /* @@ -1357,6 +1366,29 @@ static int wled_configure(struct wled *wled) }, }; + const struct wled_u32_opts pmi8994_wled_opts[] = { + { + .name = "qcom,current-boost-limit", + .val_ptr = &cfg->boost_i_limit, + .cfg = &wled4_boost_i_limit_cfg, + }, + { + .name = "qcom,current-limit-microamp", + .val_ptr = &cfg->string_i_limit, + .cfg = &wled4_string_i_limit_cfg, + }, + { + .name = "qcom,ovp-millivolt", + .val_ptr = &cfg->ovp, + .cfg = &pmi8994_wled_ovp_cfg, + }, + { + .name = "qcom,switching-freq", + .val_ptr = &cfg->switch_freq, + .cfg = &wled3_switch_freq_cfg, + }, + }; + const struct wled_u32_opts wled5_opts[] = { { .name = "qcom,current-boost-limit", @@ -1423,8 +1455,14 @@ static int wled_configure(struct wled *wled) break; case 4: - u32_opts = wled4_opts; - size = ARRAY_SIZE(wled4_opts); + if (of_device_is_compatible(dev->of_node, "qcom,pmi8950-wled") || + of_device_is_compatible(dev->of_node, "qcom,pmi8994-wled")) { + u32_opts = pmi8994_wled_opts; + size = ARRAY_SIZE(pmi8994_wled_opts); + } else { + u32_opts = wled4_opts; + size = ARRAY_SIZE(wled4_opts); + } *cfg = wled4_config_defaults; wled->wled_set_brightness = wled4_set_brightness; wled->wled_sync_toggle = wled3_sync_toggle; diff --git a/drivers/video/fbdev/au1200fb.c b/drivers/video/fbdev/au1200fb.c index ed770222660b51..685e629e7e1642 100644 --- a/drivers/video/fbdev/au1200fb.c +++ b/drivers/video/fbdev/au1200fb.c @@ -1724,8 +1724,10 @@ static int au1200fb_drv_probe(struct platform_device *dev) /* Now hook interrupt too */ irq = platform_get_irq(dev, 0); - if (irq < 0) - return irq; + if (irq < 0) { + ret = irq; + goto failed; + } ret = request_irq(irq, au1200fb_handle_irq, IRQF_SHARED, "lcd", (void *)dev); diff --git a/drivers/video/fbdev/riva/riva_hw.c b/drivers/video/fbdev/riva/riva_hw.c index 8b829b7200642f..f292079566cfca 100644 --- a/drivers/video/fbdev/riva/riva_hw.c +++ b/drivers/video/fbdev/riva/riva_hw.c @@ -436,6 +436,9 @@ static char nv3_arb(nv3_fifo_info * res_info, nv3_sim_state * state, nv3_arb_in vmisses = 2; eburst_size = state->memory_width * 1; mburst_size = 32; + if (!state->mclk_khz) + return (0); + gns = 1000000 * (gmisses*state->mem_page_miss + state->mem_latency)/state->mclk_khz; ainfo->by_gfacc = gns*ainfo->gdrain_rate/1000000; ainfo->wcmocc = 0; diff --git a/drivers/video/fbdev/smscufx.c b/drivers/video/fbdev/smscufx.c index 5f0dd01fd83495..891ce7b76d637e 100644 --- a/drivers/video/fbdev/smscufx.c +++ b/drivers/video/fbdev/smscufx.c @@ -932,7 +932,6 @@ static int ufx_ops_ioctl(struct fb_info *info, unsigned int cmd, unsigned long arg) { struct ufx_data *dev = info->par; - struct dloarea *area = NULL; if (!atomic_read(&dev->usb_active)) return 0; @@ -947,6 +946,10 @@ static int ufx_ops_ioctl(struct fb_info *info, unsigned int cmd, /* TODO: Help propose a standard fb.h ioctl to report mmap damage */ if (cmd == UFX_IOCTL_REPORT_DAMAGE) { + struct dloarea *area __free(kfree) = kmalloc(sizeof(*area), GFP_KERNEL); + if (!area) + return -ENOMEM; + /* If we have a damage-aware client, turn fb_defio "off" * To avoid perf imact of unnecessary page fault handling. * Done by resetting the delay for this fb_info to a very @@ -956,7 +959,8 @@ static int ufx_ops_ioctl(struct fb_info *info, unsigned int cmd, if (info->fbdefio) info->fbdefio->delay = UFX_DEFIO_WRITE_DISABLE; - area = (struct dloarea *)arg; + if (copy_from_user(area, (u8 __user *)arg, sizeof(*area))) + return -EFAULT; if (area->x < 0) area->x = 0; diff --git a/drivers/video/of_display_timing.c b/drivers/video/of_display_timing.c index bebd371c6b93ea..a4cd446ac5a59f 100644 --- a/drivers/video/of_display_timing.c +++ b/drivers/video/of_display_timing.c @@ -195,7 +195,7 @@ struct display_timings *of_get_display_timings(const struct device_node *np) disp->num_timings = 0; disp->native_mode = 0; - for_each_child_of_node(timings_np, entry) { + for_each_child_of_node_scoped(timings_np, child) { struct display_timing *dt; int r; @@ -206,7 +206,7 @@ struct display_timings *of_get_display_timings(const struct device_node *np) goto timingfail; } - r = of_parse_display_timing(entry, dt); + r = of_parse_display_timing(child, dt); if (r) { /* * to not encourage wrong devicetrees, fail in case of @@ -218,7 +218,7 @@ struct display_timings *of_get_display_timings(const struct device_node *np) goto timingfail; } - if (native_mode == entry) + if (native_mode == child) disp->native_mode = disp->num_timings; disp->timings[disp->num_timings] = dt; diff --git a/drivers/watchdog/apple_wdt.c b/drivers/watchdog/apple_wdt.c index 66a158f67a712b..6b9b0f9b05cedf 100644 --- a/drivers/watchdog/apple_wdt.c +++ b/drivers/watchdog/apple_wdt.c @@ -218,6 +218,7 @@ static int apple_wdt_suspend(struct device *dev) static DEFINE_SIMPLE_DEV_PM_OPS(apple_wdt_pm_ops, apple_wdt_suspend, apple_wdt_resume); static const struct of_device_id apple_wdt_of_match[] = { + { .compatible = "apple,t8103-wdt" }, { .compatible = "apple,wdt" }, {}, }; diff --git a/drivers/watchdog/starfive-wdt.c b/drivers/watchdog/starfive-wdt.c index ed71d3960a0f26..af55adc4a3c69b 100644 --- a/drivers/watchdog/starfive-wdt.c +++ b/drivers/watchdog/starfive-wdt.c @@ -446,7 +446,7 @@ static int starfive_wdt_probe(struct platform_device *pdev) platform_set_drvdata(pdev, wdt); pm_runtime_enable(&pdev->dev); if (pm_runtime_enabled(&pdev->dev)) { - ret = pm_runtime_get_sync(&pdev->dev); + ret = pm_runtime_resume_and_get(&pdev->dev); if (ret < 0) return ret; } else { diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c index 49c3f992639435..8c44a25a7d2b95 100644 --- a/drivers/xen/balloon.c +++ b/drivers/xen/balloon.c @@ -724,6 +724,7 @@ static int __init balloon_add_regions(void) static int __init balloon_init(void) { struct task_struct *task; + unsigned long current_pages; int rc; if (!xen_domain()) @@ -731,12 +732,18 @@ static int __init balloon_init(void) pr_info("Initialising balloon driver\n"); - if (xen_released_pages >= get_num_physpages()) { - WARN(1, "Released pages underflow current target"); - return -ERANGE; + if (xen_pv_domain()) { + if (xen_released_pages >= xen_start_info->nr_pages) + goto underflow; + current_pages = min(xen_start_info->nr_pages - + xen_released_pages, max_pfn); + } else { + if (xen_unpopulated_pages >= get_num_physpages()) + goto underflow; + current_pages = get_num_physpages() - xen_unpopulated_pages; } - balloon_stats.current_pages = get_num_physpages() - xen_released_pages; + balloon_stats.current_pages = current_pages; balloon_stats.target_pages = balloon_stats.current_pages; balloon_stats.balloon_low = 0; balloon_stats.balloon_high = 0; @@ -767,6 +774,10 @@ static int __init balloon_init(void) xen_balloon_init(); return 0; + + underflow: + WARN(1, "Released pages underflow current target"); + return -ERANGE; } subsys_initcall(balloon_init); diff --git a/drivers/xen/grant-dma-ops.c b/drivers/xen/grant-dma-ops.c index 14077d23f2a19e..c2603e70017863 100644 --- a/drivers/xen/grant-dma-ops.c +++ b/drivers/xen/grant-dma-ops.c @@ -366,7 +366,8 @@ static int xen_grant_init_backend_domid(struct device *dev, if (np) { ret = xen_dt_grant_init_backend_domid(dev, np, backend_domid); of_node_put(np); - } else if (IS_ENABLED(CONFIG_XEN_VIRTIO_FORCE_GRANT) || xen_pv_domain()) { + } else if (!xen_initial_domain() && + (IS_ENABLED(CONFIG_XEN_VIRTIO_FORCE_GRANT) || xen_pv_domain())) { dev_info(dev, "Using dom0 as backend\n"); *backend_domid = 0; ret = 0; diff --git a/drivers/xen/unpopulated-alloc.c b/drivers/xen/unpopulated-alloc.c index d6fc2aefe2646b..1dc0b495c8e597 100644 --- a/drivers/xen/unpopulated-alloc.c +++ b/drivers/xen/unpopulated-alloc.c @@ -18,6 +18,9 @@ static unsigned int list_count; static struct resource *target_resource; +/* Pages to subtract from the memory count when setting balloon target. */ +unsigned long xen_unpopulated_pages __initdata; + /* * If arch is not happy with system "iomem_resource" being used for * the region allocation it can provide it's own view by creating specific diff --git a/fs/btrfs/bio.c b/fs/btrfs/bio.c index fa1d321a2fb838..e4d382d3a7aeac 100644 --- a/fs/btrfs/bio.c +++ b/fs/btrfs/bio.c @@ -480,6 +480,8 @@ static void btrfs_clone_write_end_io(struct bio *bio) static void btrfs_submit_dev_bio(struct btrfs_device *dev, struct bio *bio) { + u64 physical = bio->bi_iter.bi_sector << SECTOR_SHIFT; + if (!dev || !dev->bdev || test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) || (btrfs_op(bio) == BTRFS_MAP_WRITE && @@ -494,12 +496,13 @@ static void btrfs_submit_dev_bio(struct btrfs_device *dev, struct bio *bio) * For zone append writing, bi_sector must point the beginning of the * zone */ - if (bio_op(bio) == REQ_OP_ZONE_APPEND) { - u64 physical = bio->bi_iter.bi_sector << SECTOR_SHIFT; + if (btrfs_bio(bio)->can_use_append && btrfs_dev_is_sequential(dev, physical)) { u64 zone_start = round_down(physical, dev->fs_info->zone_size); ASSERT(btrfs_dev_is_sequential(dev, physical)); bio->bi_iter.bi_sector = zone_start >> SECTOR_SHIFT; + bio->bi_opf &= ~REQ_OP_WRITE; + bio->bi_opf |= REQ_OP_ZONE_APPEND; } btrfs_debug(dev->fs_info, "%s: rw %d 0x%x, sector=%llu, dev=%lu (%s id %llu), size=%u", @@ -747,7 +750,6 @@ static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num) u64 logical = bio->bi_iter.bi_sector << SECTOR_SHIFT; u64 length = bio->bi_iter.bi_size; u64 map_length = length; - bool use_append = btrfs_use_zone_append(bbio); struct btrfs_io_context *bioc = NULL; struct btrfs_io_stripe smap; blk_status_t status; @@ -775,8 +777,10 @@ static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num) if (bio_op(bio) == REQ_OP_WRITE && is_data_bbio(bbio)) bbio->orig_logical = logical; + bbio->can_use_append = btrfs_use_zone_append(bbio); + map_length = min(map_length, length); - if (use_append) + if (bbio->can_use_append) map_length = btrfs_append_map_length(bbio, map_length); if (map_length < length) { @@ -805,11 +809,6 @@ static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num) } if (btrfs_op(bio) == BTRFS_MAP_WRITE) { - if (use_append) { - bio->bi_opf &= ~REQ_OP_WRITE; - bio->bi_opf |= REQ_OP_ZONE_APPEND; - } - if (is_data_bbio(bbio) && bioc && bioc->use_rst) { /* * No locking for the list update, as we only add to @@ -836,7 +835,7 @@ static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num) status = errno_to_blk_status(ret); if (status) goto fail; - } else if (use_append || + } else if (bbio->can_use_append || (btrfs_is_zoned(fs_info) && inode && inode->flags & BTRFS_INODE_NODATASUM)) { ret = btrfs_alloc_dummy_sum(bbio); diff --git a/fs/btrfs/bio.h b/fs/btrfs/bio.h index 1be74209f0b8db..246c7519dff39c 100644 --- a/fs/btrfs/bio.h +++ b/fs/btrfs/bio.h @@ -92,6 +92,9 @@ struct btrfs_bio { /* Whether the csum generation for data write is async. */ bool async_csum; + /* Whether the bio is written using zone append. */ + bool can_use_append; + /* * This member must come last, bio_alloc_bioset will allocate enough * bytes for entire btrfs_bio but relies on bio being last. diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 08b14449fabeba..c7be37bcbc48d0 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -3675,6 +3675,14 @@ int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans) return ret; } +static void btrfs_maybe_reset_size_class(struct btrfs_block_group *bg) +{ + lockdep_assert_held(&bg->lock); + if (btrfs_block_group_should_use_size_class(bg) && + bg->used == 0 && bg->reserved == 0) + bg->size_class = BTRFS_BG_SZ_NONE; +} + int btrfs_update_block_group(struct btrfs_trans_handle *trans, u64 bytenr, u64 num_bytes, bool alloc) { @@ -3739,6 +3747,7 @@ int btrfs_update_block_group(struct btrfs_trans_handle *trans, old_val -= num_bytes; cache->used = old_val; cache->pinned += num_bytes; + btrfs_maybe_reset_size_class(cache); btrfs_space_info_update_bytes_pinned(space_info, num_bytes); space_info->bytes_used -= num_bytes; space_info->disk_used -= num_bytes * factor; @@ -3867,6 +3876,7 @@ void btrfs_free_reserved_bytes(struct btrfs_block_group *cache, u64 num_bytes, spin_lock(&cache->lock); bg_ro = cache->ro; cache->reserved -= num_bytes; + btrfs_maybe_reset_size_class(cache); if (is_delalloc) cache->delalloc_bytes -= num_bytes; spin_unlock(&cache->lock); diff --git a/fs/btrfs/block-rsv.c b/fs/btrfs/block-rsv.c index 96cf7a1629870f..52fb7d9425917e 100644 --- a/fs/btrfs/block-rsv.c +++ b/fs/btrfs/block-rsv.c @@ -276,10 +276,11 @@ u64 btrfs_block_rsv_release(struct btrfs_fs_info *fs_info, struct btrfs_block_rsv *target = NULL; /* - * If we are a delayed block reserve then push to the global rsv, - * otherwise dump into the global delayed reserve if it is not full. + * If we are a delayed refs block reserve then push to the global + * reserve, otherwise dump into the global delayed refs reserve if it is + * not full. */ - if (block_rsv->type == BTRFS_BLOCK_RSV_DELOPS) + if (block_rsv->type == BTRFS_BLOCK_RSV_DELREFS) target = global_rsv; else if (block_rsv != global_rsv && !btrfs_block_rsv_full(delayed_rsv)) target = delayed_rsv; diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c index 206587820fec09..bed9d1c11c67a0 100644 --- a/fs/btrfs/qgroup.c +++ b/fs/btrfs/qgroup.c @@ -1137,11 +1137,14 @@ int btrfs_quota_enable(struct btrfs_fs_info *fs_info, } if (ret > 0) { /* - * Shouldn't happen, but in case it does we - * don't need to do the btrfs_next_item, just - * continue. + * Shouldn't happen because the key should still + * be there (return 0), but in case it does it + * means we have reached the end of the tree - + * there are no more leaves with items that have + * a key greater than or equals to @found_key, + * so just stop the search loop. */ - continue; + break; } } ret = btrfs_next_item(tree_root, path); @@ -1607,8 +1610,10 @@ static int __del_qgroup_relation(struct btrfs_trans_handle *trans, u64 src, if (ret < 0 && ret != -ENOENT) goto out; ret2 = del_qgroup_relation_item(trans, dst, src); - if (ret2 < 0 && ret2 != -ENOENT) + if (ret2 < 0 && ret2 != -ENOENT) { + ret = ret2; goto out; + } /* At least one deletion succeeded, return 0 */ if (!ret || !ret2) diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index bd03f465e2d3ee..d3e1ba257b9c07 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -726,7 +726,7 @@ start_transaction(struct btrfs_root *root, unsigned int num_items, h->type = type; INIT_LIST_HEAD(&h->new_bgs); - btrfs_init_metadata_block_rsv(fs_info, &h->delayed_rsv, BTRFS_BLOCK_RSV_DELOPS); + btrfs_init_metadata_block_rsv(fs_info, &h->delayed_rsv, BTRFS_BLOCK_RSV_DELREFS); smp_mb(); if (cur_trans->state >= TRANS_STATE_COMMIT_START && @@ -2500,13 +2500,6 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) list_add_tail(&fs_info->chunk_root->dirty_list, &cur_trans->switch_commits); - if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) { - btrfs_set_root_node(&fs_info->block_group_root->root_item, - fs_info->block_group_root->node); - list_add_tail(&fs_info->block_group_root->dirty_list, - &cur_trans->switch_commits); - } - switch_commit_roots(trans); ASSERT(list_empty(&cur_trans->dirty_bgs)); diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index 8a08412f3529a1..99e167a697ba8e 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -1505,30 +1505,158 @@ struct btrfs_device *btrfs_scan_one_device(const char *path, } /* - * Try to find a chunk that intersects [start, start + len] range and when one - * such is found, record the end of it in *start + * Find the first pending extent intersecting a range. + * + * @device: the device to search + * @start: start of the range to check + * @len: length of the range to check + * @pending_start: output pointer for the start of the found pending extent + * @pending_end: output pointer for the end of the found pending extent (inclusive) + * + * Search for a pending chunk allocation that intersects the half-open range + * [start, start + len). + * + * Return: true if a pending extent was found, false otherwise. + * If the return value is true, store the first pending extent in + * [*pending_start, *pending_end]. Otherwise, the two output variables + * may still be modified, to something outside the range and should not + * be used. */ -static bool contains_pending_extent(struct btrfs_device *device, u64 *start, - u64 len) +static bool first_pending_extent(struct btrfs_device *device, u64 start, u64 len, + u64 *pending_start, u64 *pending_end) { - u64 physical_start, physical_end; - lockdep_assert_held(&device->fs_info->chunk_mutex); - if (btrfs_find_first_extent_bit(&device->alloc_state, *start, - &physical_start, &physical_end, + if (btrfs_find_first_extent_bit(&device->alloc_state, start, + pending_start, pending_end, CHUNK_ALLOCATED, NULL)) { - if (in_range(physical_start, *start, len) || - in_range(*start, physical_start, - physical_end + 1 - physical_start)) { - *start = physical_end + 1; + if (in_range(*pending_start, start, len) || + in_range(start, *pending_start, *pending_end + 1 - *pending_start)) { return true; } } return false; } +/* + * Find the first real hole accounting for pending extents. + * + * @device: the device containing the candidate hole + * @start: input/output pointer for the hole start position + * @len: input/output pointer for the hole length + * @min_hole_size: the size of hole we are looking for + * + * Given a potential hole specified by [*start, *start + *len), check for pending + * chunk allocations within that range. If pending extents are found, the hole is + * adjusted to represent the first true free space that is large enough when + * accounting for pending chunks. + * + * Note that this function must handle various cases involving non consecutive + * pending extents. + * + * Returns: true if a suitable hole was found and false otherwise. + * If the return value is true, then *start and *len are set to represent the hole. + * If the return value is false, then *start is set to the largest hole we + * found and *len is set to its length. + * If there are no holes at all, then *start is set to the end of the range and + * *len is set to 0. + */ +static bool find_hole_in_pending_extents(struct btrfs_device *device, u64 *start, + u64 *len, u64 min_hole_size) +{ + u64 pending_start, pending_end; + u64 end; + u64 max_hole_start = 0; + u64 max_hole_len = 0; + + lockdep_assert_held(&device->fs_info->chunk_mutex); + + if (*len == 0) + return false; + + end = *start + *len - 1; + + /* + * Loop until we either see a large enough hole or check every pending + * extent overlapping the candidate hole. + * At every hole that we observe, record it if it is the new max. + * At the end of the iteration, set the output variables to the max hole. + */ + while (true) { + if (first_pending_extent(device, *start, *len, &pending_start, &pending_end)) { + /* + * Case 1: the pending extent overlaps the start of + * candidate hole. That means the true hole is after the + * pending extent, but we need to find the next pending + * extent to properly size the hole. In the next loop, + * we will reduce to case 2 or 3. + * e.g., + * + * |----pending A----| real hole |----pending B----| + * | candidate hole | + * *start end + */ + if (pending_start <= *start) { + *start = pending_end + 1; + goto next; + } + /* + * Case 2: The pending extent starts after *start (and overlaps + * [*start, end), so the first hole just goes up to the start + * of the pending extent. + * e.g., + * + * | real hole |----pending A----| + * | candidate hole | + * *start end + */ + *len = pending_start - *start; + if (*len > max_hole_len) { + max_hole_start = *start; + max_hole_len = *len; + } + if (*len >= min_hole_size) + break; + /* + * If the hole wasn't big enough, then we advance past + * the pending extent and keep looking. + */ + *start = pending_end + 1; + goto next; + } else { + /* + * Case 3: There is no pending extent overlapping the + * range [*start, *start + *len - 1], so the only remaining + * hole is the remaining range. + * e.g., + * + * | candidate hole | + * | real hole | + * *start end + */ + + if (*len > max_hole_len) { + max_hole_start = *start; + max_hole_len = *len; + } + break; + } +next: + if (*start > end) + break; + *len = end - *start + 1; + } + if (max_hole_len) { + *start = max_hole_start; + *len = max_hole_len; + } else { + *start = end + 1; + *len = 0; + } + return max_hole_len >= min_hole_size; +} + static u64 dev_extent_search_start(struct btrfs_device *device) { switch (device->fs_devices->chunk_alloc_policy) { @@ -1593,59 +1721,57 @@ static bool dev_extent_hole_check_zoned(struct btrfs_device *device, } /* - * Check if specified hole is suitable for allocation. + * Validate and adjust a hole for chunk allocation + * + * @device: the device containing the candidate hole + * @hole_start: input/output pointer for the hole start position + * @hole_size: input/output pointer for the hole size + * @num_bytes: minimum allocation size required * - * @device: the device which we have the hole - * @hole_start: starting position of the hole - * @hole_size: the size of the hole - * @num_bytes: the size of the free space that we need + * Check if the specified hole is suitable for allocation and adjust it if + * necessary. The hole may be modified to skip over pending chunk allocations + * and to satisfy stricter zoned requirements on zoned filesystems. * - * This function may modify @hole_start and @hole_size to reflect the suitable - * position for allocation. Returns 1 if hole position is updated, 0 otherwise. + * For regular (non-zoned) allocation, if the hole after adjustment is smaller + * than @num_bytes, the search continues past additional pending extents until + * either a sufficiently large hole is found or no more pending extents exist. + * + * Return: true if a suitable hole was found and false otherwise. + * If the return value is true, then *hole_start and *hole_size are set to + * represent the hole we found. + * If the return value is false, then *hole_start is set to the largest + * hole we found and *hole_size is set to its length. + * If there are no holes at all, then *hole_start is set to the end of the range + * and *hole_size is set to 0. */ static bool dev_extent_hole_check(struct btrfs_device *device, u64 *hole_start, u64 *hole_size, u64 num_bytes) { - bool changed = false; - u64 hole_end = *hole_start + *hole_size; + bool found = false; + const u64 hole_end = *hole_start + *hole_size - 1; - for (;;) { - /* - * Check before we set max_hole_start, otherwise we could end up - * sending back this offset anyway. - */ - if (contains_pending_extent(device, hole_start, *hole_size)) { - if (hole_end >= *hole_start) - *hole_size = hole_end - *hole_start; - else - *hole_size = 0; - changed = true; - } + ASSERT(*hole_size > 0); - switch (device->fs_devices->chunk_alloc_policy) { - default: - btrfs_warn_unknown_chunk_allocation(device->fs_devices->chunk_alloc_policy); - fallthrough; - case BTRFS_CHUNK_ALLOC_REGULAR: - /* No extra check */ - break; - case BTRFS_CHUNK_ALLOC_ZONED: - if (dev_extent_hole_check_zoned(device, hole_start, - hole_size, num_bytes)) { - changed = true; - /* - * The changed hole can contain pending extent. - * Loop again to check that. - */ - continue; - } - break; - } +again: + *hole_size = hole_end - *hole_start + 1; + found = find_hole_in_pending_extents(device, hole_start, hole_size, num_bytes); + if (!found) + return found; + ASSERT(*hole_size >= num_bytes); + switch (device->fs_devices->chunk_alloc_policy) { + default: + btrfs_warn_unknown_chunk_allocation(device->fs_devices->chunk_alloc_policy); + fallthrough; + case BTRFS_CHUNK_ALLOC_REGULAR: + return found; + case BTRFS_CHUNK_ALLOC_ZONED: + if (dev_extent_hole_check_zoned(device, hole_start, hole_size, num_bytes)) + goto again; break; } - return changed; + return found; } /* @@ -1704,7 +1830,7 @@ static int find_free_dev_extent(struct btrfs_device *device, u64 num_bytes, ret = -ENOMEM; goto out; } -again: + if (search_start >= search_end || test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) { ret = -ENOSPC; @@ -1791,11 +1917,7 @@ static int find_free_dev_extent(struct btrfs_device *device, u64 num_bytes, */ if (search_end > search_start) { hole_size = search_end - search_start; - if (dev_extent_hole_check(device, &search_start, &hole_size, - num_bytes)) { - btrfs_release_path(path); - goto again; - } + dev_extent_hole_check(device, &search_start, &hole_size, num_bytes); if (hole_size > max_hole_size) { max_hole_start = search_start; @@ -4844,6 +4966,7 @@ int btrfs_shrink_device(struct btrfs_device *device, u64 new_size) u64 diff; u64 start; u64 free_diff = 0; + u64 pending_start, pending_end; new_size = round_down(new_size, fs_info->sectorsize); start = new_size; @@ -4889,7 +5012,7 @@ int btrfs_shrink_device(struct btrfs_device *device, u64 new_size) * in-memory chunks are synced to disk so that the loop below sees them * and relocates them accordingly. */ - if (contains_pending_extent(device, &start, diff)) { + if (first_pending_extent(device, start, diff, &pending_start, &pending_end)) { mutex_unlock(&fs_info->chunk_mutex); ret = btrfs_commit_transaction(trans); if (ret) diff --git a/fs/erofs/fileio.c b/fs/erofs/fileio.c index 932e8b353ba11e..2a778a02681a08 100644 --- a/fs/erofs/fileio.c +++ b/fs/erofs/fileio.c @@ -10,6 +10,7 @@ struct erofs_fileio_rq { struct bio bio; struct kiocb iocb; struct super_block *sb; + refcount_t ref; }; struct erofs_fileio { @@ -24,31 +25,28 @@ static void erofs_fileio_ki_complete(struct kiocb *iocb, long ret) container_of(iocb, struct erofs_fileio_rq, iocb); struct folio_iter fi; - if (ret > 0) { - if (ret != rq->bio.bi_iter.bi_size) { - bio_advance(&rq->bio, ret); - zero_fill_bio(&rq->bio); - } - ret = 0; + if (ret >= 0 && ret != rq->bio.bi_iter.bi_size) { + bio_advance(&rq->bio, ret); + zero_fill_bio(&rq->bio); } - if (rq->bio.bi_end_io) { - if (ret < 0 && !rq->bio.bi_status) - rq->bio.bi_status = errno_to_blk_status(ret); - } else { + if (!rq->bio.bi_end_io) { bio_for_each_folio_all(fi, &rq->bio) { DBG_BUGON(folio_test_uptodate(fi.folio)); - erofs_onlinefolio_end(fi.folio, ret, false); + erofs_onlinefolio_end(fi.folio, ret < 0, false); } + } else if (ret < 0 && !rq->bio.bi_status) { + rq->bio.bi_status = errno_to_blk_status(ret); } bio_endio(&rq->bio); bio_uninit(&rq->bio); - kfree(rq); + if (refcount_dec_and_test(&rq->ref)) + kfree(rq); } static void erofs_fileio_rq_submit(struct erofs_fileio_rq *rq) { struct iov_iter iter; - int ret; + ssize_t ret; if (!rq) return; @@ -64,6 +62,8 @@ static void erofs_fileio_rq_submit(struct erofs_fileio_rq *rq) ret = vfs_iocb_iter_read(rq->iocb.ki_filp, &rq->iocb, &iter); if (ret != -EIOCBQUEUED) erofs_fileio_ki_complete(&rq->iocb, ret); + if (refcount_dec_and_test(&rq->ref)) + kfree(rq); } static struct erofs_fileio_rq *erofs_fileio_rq_alloc(struct erofs_map_dev *mdev) @@ -74,6 +74,7 @@ static struct erofs_fileio_rq *erofs_fileio_rq_alloc(struct erofs_map_dev *mdev) bio_init(&rq->bio, NULL, rq->bvecs, ARRAY_SIZE(rq->bvecs), REQ_OP_READ); rq->iocb.ki_filp = mdev->m_dif->file; rq->sb = mdev->m_sb; + refcount_set(&rq->ref, 2); return rq; } diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c index 3d31f7840ca04d..b71fcf4be484a2 100644 --- a/fs/erofs/zdata.c +++ b/fs/erofs/zdata.c @@ -805,14 +805,26 @@ static int z_erofs_pcluster_begin(struct z_erofs_frontend *fe) struct erofs_map_blocks *map = &fe->map; struct super_block *sb = fe->inode->i_sb; struct z_erofs_pcluster *pcl = NULL; - void *ptr; + void *ptr = NULL; int ret; DBG_BUGON(fe->pcl); /* must be Z_EROFS_PCLUSTER_TAIL or pointed to previous pcluster */ DBG_BUGON(!fe->head); - if (!(map->m_flags & EROFS_MAP_META)) { + if (map->m_flags & EROFS_MAP_META) { + ret = erofs_init_metabuf(&map->buf, sb, + erofs_inode_in_metabox(fe->inode)); + if (ret) + return ret; + ptr = erofs_bread(&map->buf, map->m_pa, false); + if (IS_ERR(ptr)) { + erofs_err(sb, "failed to read inline data %pe @ pa %llu of nid %llu", + ptr, map->m_pa, EROFS_I(fe->inode)->nid); + return PTR_ERR(ptr); + } + ptr = map->buf.page; + } else { while (1) { rcu_read_lock(); pcl = xa_load(&EROFS_SB(sb)->managed_pslots, map->m_pa); @@ -852,18 +864,8 @@ static int z_erofs_pcluster_begin(struct z_erofs_frontend *fe) /* bind cache first when cached decompression is preferred */ z_erofs_bind_cache(fe); } else { - ret = erofs_init_metabuf(&map->buf, sb, - erofs_inode_in_metabox(fe->inode)); - if (ret) - return ret; - ptr = erofs_bread(&map->buf, map->m_pa, false); - if (IS_ERR(ptr)) { - ret = PTR_ERR(ptr); - erofs_err(sb, "failed to get inline folio %d", ret); - return ret; - } - folio_get(page_folio(map->buf.page)); - WRITE_ONCE(fe->pcl->compressed_bvecs[0].page, map->buf.page); + folio_get(page_folio((struct page *)ptr)); + WRITE_ONCE(fe->pcl->compressed_bvecs[0].page, ptr); fe->pcl->pageofs_in = map->m_pa & ~PAGE_MASK; fe->mode = Z_EROFS_PCLUSTER_FOLLOWED_NOINPLACE; } @@ -1324,9 +1326,10 @@ static int z_erofs_decompress_pcluster(struct z_erofs_backend *be, bool eio) GFP_NOWAIT | __GFP_NORETRY }, be->pagepool); if (IS_ERR(reason)) { - erofs_err(be->sb, "failed to decompress (%s) %ld @ pa %llu size %u => %u", - alg->name, PTR_ERR(reason), pcl->pos, - pcl->pclustersize, pcl->length); + if (pcl->besteffort || reason != ERR_PTR(-ENOMEM)) + erofs_err(be->sb, "failed to decompress (%s) %pe @ pa %llu size %u => %u", + alg->name, reason, pcl->pos, + pcl->pclustersize, pcl->length); err = PTR_ERR(reason); } else if (unlikely(reason)) { erofs_err(be->sb, "failed to decompress (%s) %s @ pa %llu size %u => %u", diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index 56112f201cace7..1524276aeac792 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -1795,6 +1795,10 @@ struct ext4_sb_info { * Main fast commit lock. This lock protects accesses to the * following fields: * ei->i_fc_list, s_fc_dentry_q, s_fc_q, s_fc_bytes, s_fc_bh. + * + * s_fc_lock can be taken from reclaim context (inode eviction) and is + * thus reclaim unsafe. Use ext4_fc_lock()/ext4_fc_unlock() helpers + * when acquiring / releasing the lock. */ struct mutex s_fc_lock; struct buffer_head *s_fc_bh; @@ -1839,6 +1843,18 @@ static inline void ext4_writepages_up_write(struct super_block *sb, int ctx) percpu_up_write(&EXT4_SB(sb)->s_writepages_rwsem); } +static inline int ext4_fc_lock(struct super_block *sb) +{ + mutex_lock(&EXT4_SB(sb)->s_fc_lock); + return memalloc_nofs_save(); +} + +static inline void ext4_fc_unlock(struct super_block *sb, int ctx) +{ + memalloc_nofs_restore(ctx); + mutex_unlock(&EXT4_SB(sb)->s_fc_lock); +} + static inline int ext4_valid_inum(struct super_block *sb, unsigned long ino) { return ino == EXT4_ROOT_INO || diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c index 2cf5759ba68940..18b39eed752670 100644 --- a/fs/ext4/extents.c +++ b/fs/ext4/extents.c @@ -43,8 +43,13 @@ #define EXT4_EXT_MARK_UNWRIT1 0x2 /* mark first half unwritten */ #define EXT4_EXT_MARK_UNWRIT2 0x4 /* mark second half unwritten */ -#define EXT4_EXT_DATA_VALID1 0x8 /* first half contains valid data */ -#define EXT4_EXT_DATA_VALID2 0x10 /* second half contains valid data */ +/* first half contains valid data */ +#define EXT4_EXT_DATA_ENTIRE_VALID1 0x8 /* has entirely valid data */ +#define EXT4_EXT_DATA_PARTIAL_VALID1 0x10 /* has partially valid data */ +#define EXT4_EXT_DATA_VALID1 (EXT4_EXT_DATA_ENTIRE_VALID1 | \ + EXT4_EXT_DATA_PARTIAL_VALID1) + +#define EXT4_EXT_DATA_VALID2 0x20 /* second half contains valid data */ static __le32 ext4_extent_block_csum(struct inode *inode, struct ext4_extent_header *eh) @@ -3190,8 +3195,12 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle, unsigned int ee_len, depth; int err = 0; - BUG_ON((split_flag & (EXT4_EXT_DATA_VALID1 | EXT4_EXT_DATA_VALID2)) == - (EXT4_EXT_DATA_VALID1 | EXT4_EXT_DATA_VALID2)); + BUG_ON((split_flag & EXT4_EXT_DATA_VALID1) == EXT4_EXT_DATA_VALID1); + BUG_ON((split_flag & EXT4_EXT_DATA_VALID1) && + (split_flag & EXT4_EXT_DATA_VALID2)); + + /* Do not cache extents that are in the process of being modified. */ + flags |= EXT4_EX_NOCACHE; ext_debug(inode, "logical block %llu\n", (unsigned long long)split); @@ -3258,7 +3267,7 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle, err = PTR_ERR(path); if (err != -ENOSPC && err != -EDQUOT && err != -ENOMEM) - return path; + goto out_path; /* * Get a new path to try to zeroout or fix the extent length. @@ -3272,7 +3281,7 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle, if (IS_ERR(path)) { EXT4_ERROR_INODE(inode, "Failed split extent on %u, err %ld", split, PTR_ERR(path)); - return path; + goto out_path; } depth = ext_depth(inode); ex = path[depth].p_ext; @@ -3304,6 +3313,23 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle, } if (!err) { + /* + * The first half contains partially valid data, the + * splitting of this extent has not been completed, fix + * extent length and ext4_split_extent() split will the + * first half again. + */ + if (split_flag & EXT4_EXT_DATA_PARTIAL_VALID1) { + /* + * Drop extent cache to prevent stale unwritten + * extents remaining after zeroing out. + */ + ext4_es_remove_extent(inode, + le32_to_cpu(zero_ex.ee_block), + ext4_ext_get_actual_len(&zero_ex)); + goto fix_extent_len; + } + /* update the extent length and mark as initialized */ ex->ee_len = cpu_to_le16(ee_len); ext4_ext_try_to_merge(handle, inode, path, ex); @@ -3332,6 +3358,10 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle, ext4_free_ext_path(path); path = ERR_PTR(err); } +out_path: + if (IS_ERR(path)) + /* Remove all remaining potentially stale extents. */ + ext4_es_remove_extent(inode, ee_block, ee_len); ext4_ext_show_leaf(inode, path); return path; } @@ -3366,6 +3396,9 @@ static struct ext4_ext_path *ext4_split_extent(handle_t *handle, ee_len = ext4_ext_get_actual_len(ex); unwritten = ext4_ext_is_unwritten(ex); + /* Do not cache extents that are in the process of being modified. */ + flags |= EXT4_EX_NOCACHE; + if (map->m_lblk + map->m_len < ee_block + ee_len) { split_flag1 = split_flag & EXT4_EXT_MAY_ZEROOUT; flags1 = flags | EXT4_GET_BLOCKS_SPLIT_NOMERGE; @@ -3373,7 +3406,9 @@ static struct ext4_ext_path *ext4_split_extent(handle_t *handle, split_flag1 |= EXT4_EXT_MARK_UNWRIT1 | EXT4_EXT_MARK_UNWRIT2; if (split_flag & EXT4_EXT_DATA_VALID2) - split_flag1 |= EXT4_EXT_DATA_VALID1; + split_flag1 |= map->m_lblk > ee_block ? + EXT4_EXT_DATA_PARTIAL_VALID1 : + EXT4_EXT_DATA_ENTIRE_VALID1; path = ext4_split_extent_at(handle, inode, path, map->m_lblk + map->m_len, split_flag1, flags1); if (IS_ERR(path)) @@ -3728,16 +3763,20 @@ static struct ext4_ext_path *ext4_split_convert_extents(handle_t *handle, /* Convert to unwritten */ if (flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN) { - split_flag |= EXT4_EXT_DATA_VALID1; - /* Convert to initialized */ - } else if (flags & EXT4_GET_BLOCKS_CONVERT) { + split_flag |= EXT4_EXT_DATA_ENTIRE_VALID1; + /* Split the existing unwritten extent */ + } else if (flags & (EXT4_GET_BLOCKS_UNWRIT_EXT | + EXT4_GET_BLOCKS_CONVERT)) { /* * It is safe to convert extent to initialized via explicit * zeroout only if extent is fully inside i_size or new_size. */ split_flag |= ee_block + ee_len <= eof_block ? EXT4_EXT_MAY_ZEROOUT : 0; - split_flag |= (EXT4_EXT_MARK_UNWRIT2 | EXT4_EXT_DATA_VALID2); + split_flag |= EXT4_EXT_MARK_UNWRIT2; + /* Convert to initialized */ + if (flags & EXT4_GET_BLOCKS_CONVERT) + split_flag |= EXT4_EXT_DATA_VALID2; } flags |= EXT4_GET_BLOCKS_SPLIT_NOMERGE; return ext4_split_extent(handle, inode, path, map, split_flag, flags, @@ -3913,7 +3952,7 @@ ext4_ext_handle_unwritten_extents(handle_t *handle, struct inode *inode, /* get_block() before submitting IO, split the extent */ if (flags & EXT4_GET_BLOCKS_SPLIT_NOMERGE) { path = ext4_split_convert_extents(handle, inode, map, path, - flags | EXT4_GET_BLOCKS_CONVERT, allocated); + flags, allocated); if (IS_ERR(path)) return path; /* @@ -5375,7 +5414,8 @@ ext4_ext_shift_extents(struct inode *inode, handle_t *handle, if (!extent) { EXT4_ERROR_INODE(inode, "unexpected hole at %lu", (unsigned long) *iterator); - return -EFSCORRUPTED; + ret = -EFSCORRUPTED; + goto out; } if (SHIFT == SHIFT_LEFT && *iterator > le32_to_cpu(extent->ee_block)) { diff --git a/fs/ext4/fast_commit.c b/fs/ext4/fast_commit.c index fa66b08de9994e..5bd57d7f921b9a 100644 --- a/fs/ext4/fast_commit.c +++ b/fs/ext4/fast_commit.c @@ -231,16 +231,16 @@ static bool ext4_fc_disabled(struct super_block *sb) void ext4_fc_del(struct inode *inode) { struct ext4_inode_info *ei = EXT4_I(inode); - struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); struct ext4_fc_dentry_update *fc_dentry; wait_queue_head_t *wq; + int alloc_ctx; if (ext4_fc_disabled(inode->i_sb)) return; - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(inode->i_sb); if (list_empty(&ei->i_fc_list) && list_empty(&ei->i_fc_dilist)) { - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(inode->i_sb, alloc_ctx); return; } @@ -275,9 +275,9 @@ void ext4_fc_del(struct inode *inode) #endif prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE); if (ext4_test_inode_state(inode, EXT4_STATE_FC_FLUSHING_DATA)) { - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(inode->i_sb, alloc_ctx); schedule(); - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(inode->i_sb); } finish_wait(wq, &wait.wq_entry); } @@ -288,7 +288,7 @@ void ext4_fc_del(struct inode *inode) * dentry create references, since it is not needed to log it anyways. */ if (list_empty(&ei->i_fc_dilist)) { - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(inode->i_sb, alloc_ctx); return; } @@ -298,7 +298,7 @@ void ext4_fc_del(struct inode *inode) list_del_init(&fc_dentry->fcd_dilist); WARN_ON(!list_empty(&ei->i_fc_dilist)); - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(inode->i_sb, alloc_ctx); release_dentry_name_snapshot(&fc_dentry->fcd_name); kmem_cache_free(ext4_fc_dentry_cachep, fc_dentry); @@ -315,6 +315,7 @@ void ext4_fc_mark_ineligible(struct super_block *sb, int reason, handle_t *handl tid_t tid; bool has_transaction = true; bool is_ineligible; + int alloc_ctx; if (ext4_fc_disabled(sb)) return; @@ -329,12 +330,12 @@ void ext4_fc_mark_ineligible(struct super_block *sb, int reason, handle_t *handl has_transaction = false; read_unlock(&sbi->s_journal->j_state_lock); } - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(sb); is_ineligible = ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE); if (has_transaction && (!is_ineligible || tid_gt(tid, sbi->s_fc_ineligible_tid))) sbi->s_fc_ineligible_tid = tid; ext4_set_mount_flag(sb, EXT4_MF_FC_INELIGIBLE); - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(sb, alloc_ctx); WARN_ON(reason >= EXT4_FC_REASON_MAX); sbi->s_fc_stats.fc_ineligible_reason_count[reason]++; } @@ -358,6 +359,7 @@ static int ext4_fc_track_template( struct ext4_inode_info *ei = EXT4_I(inode); struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); tid_t tid = 0; + int alloc_ctx; int ret; tid = handle->h_transaction->t_tid; @@ -373,14 +375,14 @@ static int ext4_fc_track_template( if (!enqueue) return ret; - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(inode->i_sb); if (list_empty(&EXT4_I(inode)->i_fc_list)) list_add_tail(&EXT4_I(inode)->i_fc_list, (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING || sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING) ? &sbi->s_fc_q[FC_Q_STAGING] : &sbi->s_fc_q[FC_Q_MAIN]); - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(inode->i_sb, alloc_ctx); return ret; } @@ -402,6 +404,7 @@ static int __track_dentry_update(handle_t *handle, struct inode *inode, struct inode *dir = dentry->d_parent->d_inode; struct super_block *sb = inode->i_sb; struct ext4_sb_info *sbi = EXT4_SB(sb); + int alloc_ctx; spin_unlock(&ei->i_fc_lock); @@ -425,7 +428,7 @@ static int __track_dentry_update(handle_t *handle, struct inode *inode, take_dentry_name_snapshot(&node->fcd_name, dentry); INIT_LIST_HEAD(&node->fcd_dilist); INIT_LIST_HEAD(&node->fcd_list); - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(sb); if (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING || sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING) list_add_tail(&node->fcd_list, @@ -446,7 +449,7 @@ static int __track_dentry_update(handle_t *handle, struct inode *inode, WARN_ON(!list_empty(&ei->i_fc_dilist)); list_add_tail(&node->fcd_dilist, &ei->i_fc_dilist); } - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(sb, alloc_ctx); spin_lock(&ei->i_fc_lock); return 0; @@ -1046,18 +1049,19 @@ static int ext4_fc_perform_commit(journal_t *journal) struct blk_plug plug; int ret = 0; u32 crc = 0; + int alloc_ctx; /* * Step 1: Mark all inodes on s_fc_q[MAIN] with * EXT4_STATE_FC_FLUSHING_DATA. This prevents these inodes from being * freed until the data flush is over. */ - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(sb); list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { ext4_set_inode_state(&iter->vfs_inode, EXT4_STATE_FC_FLUSHING_DATA); } - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(sb, alloc_ctx); /* Step 2: Flush data for all the eligible inodes. */ ret = ext4_fc_flush_data(journal); @@ -1067,7 +1071,7 @@ static int ext4_fc_perform_commit(journal_t *journal) * any error from step 2. This ensures that waiters waiting on * EXT4_STATE_FC_FLUSHING_DATA can resume. */ - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(sb); list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { ext4_clear_inode_state(&iter->vfs_inode, EXT4_STATE_FC_FLUSHING_DATA); @@ -1084,7 +1088,7 @@ static int ext4_fc_perform_commit(journal_t *journal) * prepare_to_wait() in ext4_fc_del(). */ smp_mb(); - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(sb, alloc_ctx); /* * If we encountered error in Step 2, return it now after clearing @@ -1101,12 +1105,12 @@ static int ext4_fc_perform_commit(journal_t *journal) * previous handles are now drained. We now mark the inodes on the * commit queue as being committed. */ - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(sb); list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { ext4_set_inode_state(&iter->vfs_inode, EXT4_STATE_FC_COMMITTING); } - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(sb, alloc_ctx); jbd2_journal_unlock_updates(journal); /* @@ -1117,6 +1121,7 @@ static int ext4_fc_perform_commit(journal_t *journal) blkdev_issue_flush(journal->j_fs_dev); blk_start_plug(&plug); + alloc_ctx = ext4_fc_lock(sb); /* Step 6: Write fast commit blocks to disk. */ if (sbi->s_fc_bytes == 0) { /* @@ -1134,7 +1139,6 @@ static int ext4_fc_perform_commit(journal_t *journal) } /* Step 6.2: Now write all the dentry updates. */ - mutex_lock(&sbi->s_fc_lock); ret = ext4_fc_commit_dentry_updates(journal, &crc); if (ret) goto out; @@ -1156,7 +1160,7 @@ static int ext4_fc_perform_commit(journal_t *journal) ret = ext4_fc_write_tail(sb, crc); out: - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(sb, alloc_ctx); blk_finish_plug(&plug); return ret; } @@ -1290,6 +1294,7 @@ static void ext4_fc_cleanup(journal_t *journal, int full, tid_t tid) struct ext4_sb_info *sbi = EXT4_SB(sb); struct ext4_inode_info *ei; struct ext4_fc_dentry_update *fc_dentry; + int alloc_ctx; if (full && sbi->s_fc_bh) sbi->s_fc_bh = NULL; @@ -1297,7 +1302,7 @@ static void ext4_fc_cleanup(journal_t *journal, int full, tid_t tid) trace_ext4_fc_cleanup(journal, full, tid); jbd2_fc_release_bufs(journal); - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(sb); while (!list_empty(&sbi->s_fc_q[FC_Q_MAIN])) { ei = list_first_entry(&sbi->s_fc_q[FC_Q_MAIN], struct ext4_inode_info, @@ -1356,7 +1361,7 @@ static void ext4_fc_cleanup(journal_t *journal, int full, tid_t tid) if (full) sbi->s_fc_bytes = 0; - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(sb, alloc_ctx); trace_ext4_fc_stats(sb); } diff --git a/fs/ext4/mballoc-test.c b/fs/ext4/mballoc-test.c index a9416b20ff64c9..4abb40d4561ce3 100644 --- a/fs/ext4/mballoc-test.c +++ b/fs/ext4/mballoc-test.c @@ -567,7 +567,7 @@ test_mark_diskspace_used_range(struct kunit *test, bitmap = mbt_ctx_bitmap(sb, TEST_GOAL_GROUP); memset(bitmap, 0, sb->s_blocksize); - ret = ext4_mb_mark_diskspace_used(ac, NULL, 0); + ret = ext4_mb_mark_diskspace_used(ac, NULL); KUNIT_ASSERT_EQ(test, ret, 0); max = EXT4_CLUSTERS_PER_GROUP(sb); diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index 56d50fd3310b47..dbc82b65f810fe 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -892,6 +892,21 @@ mb_update_avg_fragment_size(struct super_block *sb, struct ext4_group_info *grp) } } +static ext4_group_t ext4_get_allocation_groups_count( + struct ext4_allocation_context *ac) +{ + ext4_group_t ngroups = ext4_get_groups_count(ac->ac_sb); + + /* non-extent files are limited to low blocks/groups */ + if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS))) + ngroups = EXT4_SB(ac->ac_sb)->s_blockfile_groups; + + /* Pairs with smp_wmb() in ext4_update_super() */ + smp_rmb(); + + return ngroups; +} + static int ext4_mb_scan_groups_xa_range(struct ext4_allocation_context *ac, struct xarray *xa, ext4_group_t start, ext4_group_t end) @@ -899,7 +914,7 @@ static int ext4_mb_scan_groups_xa_range(struct ext4_allocation_context *ac, struct super_block *sb = ac->ac_sb; struct ext4_sb_info *sbi = EXT4_SB(sb); enum criteria cr = ac->ac_criteria; - ext4_group_t ngroups = ext4_get_groups_count(sb); + ext4_group_t ngroups = ext4_get_allocation_groups_count(ac); unsigned long group = start; struct ext4_group_info *grp; @@ -951,7 +966,7 @@ static int ext4_mb_scan_groups_p2_aligned(struct ext4_allocation_context *ac, ext4_group_t start, end; start = group; - end = ext4_get_groups_count(ac->ac_sb); + end = ext4_get_allocation_groups_count(ac); wrap_around: for (i = ac->ac_2order; i < MB_NUM_ORDERS(ac->ac_sb); i++) { ret = ext4_mb_scan_groups_largest_free_order_range(ac, i, @@ -1001,7 +1016,7 @@ static int ext4_mb_scan_groups_goal_fast(struct ext4_allocation_context *ac, ext4_group_t start, end; start = group; - end = ext4_get_groups_count(ac->ac_sb); + end = ext4_get_allocation_groups_count(ac); wrap_around: i = mb_avg_fragment_size_order(ac->ac_sb, ac->ac_g_ex.fe_len); for (; i < MB_NUM_ORDERS(ac->ac_sb); i++) { @@ -1083,7 +1098,7 @@ static int ext4_mb_scan_groups_best_avail(struct ext4_allocation_context *ac, min_order = fls(ac->ac_o_ex.fe_len); start = group; - end = ext4_get_groups_count(ac->ac_sb); + end = ext4_get_allocation_groups_count(ac); wrap_around: for (i = order; i >= min_order; i--) { int frag_order; @@ -1133,8 +1148,6 @@ static inline int should_optimize_scan(struct ext4_allocation_context *ac) return 0; if (ac->ac_criteria >= CR_GOAL_LEN_SLOW) return 0; - if (!ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) - return 0; return 1; } @@ -1182,11 +1195,7 @@ static int ext4_mb_scan_groups(struct ext4_allocation_context *ac) int ret = 0; ext4_group_t start; struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); - ext4_group_t ngroups = ext4_get_groups_count(ac->ac_sb); - - /* non-extent files are limited to low blocks/groups */ - if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS))) - ngroups = sbi->s_blockfile_groups; + ext4_group_t ngroups = ext4_get_allocation_groups_count(ac); /* searching for the right group start from the goal value specified */ start = ac->ac_g_ex.fe_group; @@ -1706,16 +1715,17 @@ ext4_mb_load_buddy_gfp(struct super_block *sb, ext4_group_t group, /* Avoid locking the folio in the fast path ... */ folio = __filemap_get_folio(inode->i_mapping, pnum, FGP_ACCESSED, 0); - if (IS_ERR(folio) || !folio_test_uptodate(folio)) { + if (IS_ERR(folio) || !folio_test_uptodate(folio) || folio_test_locked(folio)) { + /* + * folio_test_locked is employed to detect ongoing folio + * migrations, since concurrent migrations can lead to + * bitmap inconsistency. And if we are not uptodate that + * implies somebody just created the folio but is yet to + * initialize it. We can drop the folio reference and + * try to get the folio with lock in both cases to avoid + * concurrency. + */ if (!IS_ERR(folio)) - /* - * drop the folio reference and try - * to get the folio with lock. If we - * are not uptodate that implies - * somebody just created the folio but - * is yet to initialize it. So - * wait for it to initialize. - */ folio_put(folio); folio = __filemap_get_folio(inode->i_mapping, pnum, FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp); @@ -1764,7 +1774,7 @@ ext4_mb_load_buddy_gfp(struct super_block *sb, ext4_group_t group, /* we need another folio for the buddy */ folio = __filemap_get_folio(inode->i_mapping, pnum, FGP_ACCESSED, 0); - if (IS_ERR(folio) || !folio_test_uptodate(folio)) { + if (IS_ERR(folio) || !folio_test_uptodate(folio) || folio_test_locked(folio)) { if (!IS_ERR(folio)) folio_put(folio); folio = __filemap_get_folio(inode->i_mapping, pnum, @@ -4185,8 +4195,7 @@ ext4_mb_mark_context(handle_t *handle, struct super_block *sb, bool state, * Returns 0 if success or error code */ static noinline_for_stack int -ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, - handle_t *handle, unsigned int reserv_clstrs) +ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, handle_t *handle) { struct ext4_group_desc *gdp; struct ext4_sb_info *sbi; @@ -4241,13 +4250,6 @@ ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, BUG_ON(changed != ac->ac_b_ex.fe_len); #endif percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len); - /* - * Now reduce the dirty block count also. Should not go negative - */ - if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED)) - /* release all the reserved blocks if non delalloc */ - percpu_counter_sub(&sbi->s_dirtyclusters_counter, - reserv_clstrs); return err; } @@ -6332,7 +6334,7 @@ ext4_fsblk_t ext4_mb_new_blocks(handle_t *handle, ext4_mb_pa_put_free(ac); } if (likely(ac->ac_status == AC_STATUS_FOUND)) { - *errp = ext4_mb_mark_diskspace_used(ac, handle, reserv_clstrs); + *errp = ext4_mb_mark_diskspace_used(ac, handle); if (*errp) { ext4_discard_allocated_blocks(ac); goto errout; @@ -6363,12 +6365,9 @@ ext4_fsblk_t ext4_mb_new_blocks(handle_t *handle, out: if (inquota && ar->len < inquota) dquot_free_block(ar->inode, EXT4_C2B(sbi, inquota - ar->len)); - if (!ar->len) { - if ((ar->flags & EXT4_MB_DELALLOC_RESERVED) == 0) - /* release all the reserved blocks if non delalloc */ - percpu_counter_sub(&sbi->s_dirtyclusters_counter, - reserv_clstrs); - } + /* release any reserved blocks */ + if (reserv_clstrs) + percpu_counter_sub(&sbi->s_dirtyclusters_counter, reserv_clstrs); trace_ext4_allocate_blocks(ar, (unsigned long long)block); diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index c30e69392a6236..86e00b9e0d1cfc 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -151,6 +151,12 @@ static void f2fs_finish_read_bio(struct bio *bio, bool in_task) } dec_page_count(F2FS_F_SB(folio), __read_io_type(folio)); + + if (F2FS_F_SB(folio)->node_inode && is_node_folio(folio) && + f2fs_sanity_check_node_footer(F2FS_F_SB(folio), + folio, folio->index, NODE_TYPE_REGULAR, true)) + bio->bi_status = BLK_STS_IOERR; + folio_end_read(folio, bio->bi_status == BLK_STS_OK); } @@ -352,18 +358,27 @@ static void f2fs_write_end_io(struct bio *bio) STOP_CP_REASON_WRITE_FAIL); } - f2fs_bug_on(sbi, is_node_folio(folio) && - folio->index != nid_of_node(folio)); + if (is_node_folio(folio)) { + f2fs_sanity_check_node_footer(sbi, folio, + folio->index, NODE_TYPE_REGULAR, true); + f2fs_bug_on(sbi, folio->index != nid_of_node(folio)); + } dec_page_count(sbi, type); + + /* + * we should access sbi before folio_end_writeback() to + * avoid racing w/ kill_f2fs_super() + */ + if (type == F2FS_WB_CP_DATA && !get_pages(sbi, type) && + wq_has_sleeper(&sbi->cp_wait)) + wake_up(&sbi->cp_wait); + if (f2fs_in_warm_node_list(sbi, folio)) f2fs_del_fsync_node_entry(sbi, folio); folio_clear_f2fs_gcing(folio); folio_end_writeback(folio); } - if (!get_pages(sbi, F2FS_WB_CP_DATA) && - wq_has_sleeper(&sbi->cp_wait)) - wake_up(&sbi->cp_wait); bio_put(bio); } @@ -1418,7 +1433,6 @@ static int __allocate_data_block(struct dnode_of_data *dn, int seg_type) static void f2fs_map_lock(struct f2fs_sb_info *sbi, int flag) { - f2fs_down_read(&sbi->cp_enable_rwsem); if (flag == F2FS_GET_BLOCK_PRE_AIO) f2fs_down_read(&sbi->node_change); else @@ -1431,7 +1445,6 @@ static void f2fs_map_unlock(struct f2fs_sb_info *sbi, int flag) f2fs_up_read(&sbi->node_change); else f2fs_unlock_op(sbi); - f2fs_up_read(&sbi->cp_enable_rwsem); } int f2fs_get_block_locked(struct dnode_of_data *dn, pgoff_t index) @@ -1793,7 +1806,8 @@ int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag) return err; } -bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len) +static bool __f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len, + bool check_first) { struct f2fs_map_blocks map; block_t last_lblk; @@ -1815,10 +1829,17 @@ bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len) if (err || map.m_len == 0) return false; map.m_lblk += map.m_len; + if (check_first) + break; } return true; } +bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len) +{ + return __f2fs_overwrite_io(inode, pos, len, false); +} + static int f2fs_xattr_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo) { @@ -3929,6 +3950,7 @@ static int check_swap_activate(struct swap_info_struct *sis, while (cur_lblock < last_lblock && cur_lblock < sis->max) { struct f2fs_map_blocks map; + bool last_extent = false; retry: cond_resched(); @@ -3954,11 +3976,10 @@ static int check_swap_activate(struct swap_info_struct *sis, pblock = map.m_pblk; nr_pblocks = map.m_len; - if ((pblock - SM_I(sbi)->main_blkaddr) % blks_per_sec || - nr_pblocks % blks_per_sec || - f2fs_is_sequential_zone_area(sbi, pblock)) { - bool last_extent = false; - + if (!last_extent && + ((pblock - SM_I(sbi)->main_blkaddr) % blks_per_sec || + nr_pblocks % blks_per_sec || + f2fs_is_sequential_zone_area(sbi, pblock))) { not_aligned++; nr_pblocks = roundup(nr_pblocks, blks_per_sec); @@ -3979,8 +4000,8 @@ static int check_swap_activate(struct swap_info_struct *sis, goto out; } - if (!last_extent) - goto retry; + /* lookup block mapping info after block migration */ + goto retry; } if (cur_lblock + nr_pblocks >= sis->max) @@ -4181,7 +4202,7 @@ static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, * f2fs_map_lock and f2fs_balance_fs are not necessary. */ if ((flags & IOMAP_WRITE) && - !f2fs_overwrite_io(inode, offset, length)) + !__f2fs_overwrite_io(inode, offset, length, true)) map.m_may_create = true; err = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_DIO); diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 20edbb99b814a7..d9a8465cb2f4de 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -287,7 +287,7 @@ enum { #define DEF_CP_INTERVAL 60 /* 60 secs */ #define DEF_IDLE_INTERVAL 5 /* 5 secs */ #define DEF_DISABLE_INTERVAL 5 /* 5 secs */ -#define DEF_ENABLE_INTERVAL 5 /* 5 secs */ +#define DEF_ENABLE_INTERVAL 16 /* 16 secs */ #define DEF_DISABLE_QUICK_INTERVAL 1 /* 1 secs */ #define DEF_UMOUNT_DISCARD_TIMEOUT 5 /* 5 secs */ @@ -521,13 +521,25 @@ struct fsync_inode_entry { #define nats_in_cursum(jnl) (le16_to_cpu((jnl)->n_nats)) #define sits_in_cursum(jnl) (le16_to_cpu((jnl)->n_sits)) -#define nat_in_journal(jnl, i) ((jnl)->nat_j.entries[i].ne) -#define nid_in_journal(jnl, i) ((jnl)->nat_j.entries[i].nid) -#define sit_in_journal(jnl, i) ((jnl)->sit_j.entries[i].se) -#define segno_in_journal(jnl, i) ((jnl)->sit_j.entries[i].segno) - -#define MAX_NAT_JENTRIES(jnl) (NAT_JOURNAL_ENTRIES - nats_in_cursum(jnl)) -#define MAX_SIT_JENTRIES(jnl) (SIT_JOURNAL_ENTRIES - sits_in_cursum(jnl)) +#define nat_in_journal(jnl, i) \ + (((struct nat_journal_entry *)(jnl)->nat_j.entries)[i].ne) +#define nid_in_journal(jnl, i) \ + (((struct nat_journal_entry *)(jnl)->nat_j.entries)[i].nid) +#define sit_in_journal(jnl, i) \ + (((struct sit_journal_entry *)(jnl)->sit_j.entries)[i].se) +#define segno_in_journal(jnl, i) \ + (((struct sit_journal_entry *)(jnl)->sit_j.entries)[i].segno) + +#define sum_entries(sum) ((struct f2fs_summary *)(sum)) +#define sum_journal(sbi, sum) \ + ((struct f2fs_journal *)((char *)(sum) + \ + ((sbi)->entries_in_sum * sizeof(struct f2fs_summary)))) +#define sum_footer(sbi, sum) \ + ((struct summary_footer *)((char *)(sum) + (sbi)->sum_blocksize - \ + sizeof(struct summary_footer))) + +#define MAX_NAT_JENTRIES(sbi, jnl) ((sbi)->nat_journal_entries - nats_in_cursum(jnl)) +#define MAX_SIT_JENTRIES(sbi, jnl) ((sbi)->sit_journal_entries - sits_in_cursum(jnl)) static inline int update_nats_in_cursum(struct f2fs_journal *journal, int i) { @@ -545,14 +557,6 @@ static inline int update_sits_in_cursum(struct f2fs_journal *journal, int i) return before; } -static inline bool __has_cursum_space(struct f2fs_journal *journal, - int size, int type) -{ - if (type == NAT_JOURNAL) - return size <= MAX_NAT_JENTRIES(journal); - return size <= MAX_SIT_JENTRIES(journal); -} - /* for inline stuff */ #define DEF_INLINE_RESERVED_SIZE 1 static inline int get_extra_isize(struct inode *inode); @@ -1525,6 +1529,15 @@ enum f2fs_lookup_mode { LOOKUP_AUTO, }; +/* For node type in __get_node_folio() */ +enum node_type { + NODE_TYPE_REGULAR, + NODE_TYPE_INODE, + NODE_TYPE_XATTR, + NODE_TYPE_NON_INODE, +}; + + static inline int f2fs_test_bit(unsigned int nr, char *addr); static inline void f2fs_set_bit(unsigned int nr, char *addr); static inline void f2fs_clear_bit(unsigned int nr, char *addr); @@ -1716,7 +1729,6 @@ struct f2fs_sb_info { long interval_time[MAX_TIME]; /* to store thresholds */ struct ckpt_req_control cprc_info; /* for checkpoint request control */ struct cp_stats cp_stats; /* for time stat of checkpoint */ - struct f2fs_rwsem cp_enable_rwsem; /* block cache/dio write */ struct inode_management im[MAX_INO_ENTRY]; /* manage inode cache */ @@ -1764,6 +1776,15 @@ struct f2fs_sb_info { bool readdir_ra; /* readahead inode in readdir */ u64 max_io_bytes; /* max io bytes to merge IOs */ + /* variable summary block units */ + unsigned int sum_blocksize; /* sum block size */ + unsigned int sums_per_block; /* sum block count per block */ + unsigned int entries_in_sum; /* entry count in sum block */ + unsigned int sum_entry_size; /* total entry size in sum block */ + unsigned int sum_journal_size; /* journal size in sum block */ + unsigned int nat_journal_entries; /* nat journal entry count in the journal */ + unsigned int sit_journal_entries; /* sit journal entry count in the journal */ + block_t user_block_count; /* # of user blocks */ block_t total_valid_block_count; /* # of valid blocks */ block_t discard_blks; /* discard command candidats */ @@ -2813,6 +2834,14 @@ static inline block_t __start_sum_addr(struct f2fs_sb_info *sbi) return le32_to_cpu(F2FS_CKPT(sbi)->cp_pack_start_sum); } +static inline bool __has_cursum_space(struct f2fs_sb_info *sbi, + struct f2fs_journal *journal, int size, int type) +{ + if (type == NAT_JOURNAL) + return size <= MAX_NAT_JENTRIES(sbi, journal); + return size <= MAX_SIT_JENTRIES(sbi, journal); +} + extern void f2fs_mark_inode_dirty_sync(struct inode *inode, bool sync); static inline int inc_valid_node_count(struct f2fs_sb_info *sbi, struct inode *inode, bool is_inode) @@ -3857,6 +3886,9 @@ struct folio *f2fs_new_node_folio(struct dnode_of_data *dn, unsigned int ofs); void f2fs_ra_node_page(struct f2fs_sb_info *sbi, nid_t nid); struct folio *f2fs_get_node_folio(struct f2fs_sb_info *sbi, pgoff_t nid, enum node_type node_type); +int f2fs_sanity_check_node_footer(struct f2fs_sb_info *sbi, + struct folio *folio, pgoff_t nid, + enum node_type ntype, bool in_irq); struct folio *f2fs_get_inode_folio(struct f2fs_sb_info *sbi, pgoff_t ino); struct folio *f2fs_get_xnode_folio(struct f2fs_sb_info *sbi, pgoff_t xnid); int f2fs_move_node_folio(struct folio *node_folio, int gc_type); @@ -3956,7 +3988,8 @@ void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr, block_t len); void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk); void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk); -int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type, +int f2fs_lookup_journal_in_cursum(struct f2fs_sb_info *sbi, + struct f2fs_journal *journal, int type, unsigned int val, int alloc); void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc); int f2fs_check_and_fix_write_pointer(struct f2fs_sb_info *sbi); diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c index 384fa7e2085bf9..fa20bcd700588f 100644 --- a/fs/f2fs/gc.c +++ b/fs/f2fs/gc.c @@ -1769,8 +1769,8 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, sanity_check_seg_type(sbi, get_seg_entry(sbi, segno)->type); - segno = rounddown(segno, SUMS_PER_BLOCK); - sum_blk_cnt = DIV_ROUND_UP(end_segno - segno, SUMS_PER_BLOCK); + segno = rounddown(segno, sbi->sums_per_block); + sum_blk_cnt = DIV_ROUND_UP(end_segno - segno, sbi->sums_per_block); /* readahead multi ssa blocks those have contiguous address */ if (__is_large_section(sbi)) f2fs_ra_meta_pages(sbi, GET_SUM_BLOCK(sbi, segno), @@ -1780,17 +1780,17 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, while (segno < end_segno) { struct folio *sum_folio = f2fs_get_sum_folio(sbi, segno); - segno += SUMS_PER_BLOCK; + segno += sbi->sums_per_block; if (IS_ERR(sum_folio)) { int err = PTR_ERR(sum_folio); - end_segno = segno - SUMS_PER_BLOCK; - segno = rounddown(start_segno, SUMS_PER_BLOCK); + end_segno = segno - sbi->sums_per_block; + segno = rounddown(start_segno, sbi->sums_per_block); while (segno < end_segno) { sum_folio = filemap_get_folio(META_MAPPING(sbi), GET_SUM_BLOCK(sbi, segno)); folio_put_refs(sum_folio, 2); - segno += SUMS_PER_BLOCK; + segno += sbi->sums_per_block; } return err; } @@ -1806,8 +1806,8 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, /* find segment summary of victim */ struct folio *sum_folio = filemap_get_folio(META_MAPPING(sbi), GET_SUM_BLOCK(sbi, segno)); - unsigned int block_end_segno = rounddown(segno, SUMS_PER_BLOCK) - + SUMS_PER_BLOCK; + unsigned int block_end_segno = rounddown(segno, sbi->sums_per_block) + + sbi->sums_per_block; if (block_end_segno > end_segno) block_end_segno = end_segno; @@ -1833,12 +1833,13 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, migrated >= sbi->migration_granularity) continue; - sum = SUM_BLK_PAGE_ADDR(sum_folio, cur_segno); - if (type != GET_SUM_TYPE((&sum->footer))) { + sum = SUM_BLK_PAGE_ADDR(sbi, sum_folio, cur_segno); + if (type != GET_SUM_TYPE(sum_footer(sbi, sum))) { f2fs_err(sbi, "Inconsistent segment (%u) type " "[%d, %d] in SSA and SIT", cur_segno, type, - GET_SUM_TYPE((&sum->footer))); + GET_SUM_TYPE( + sum_footer(sbi, sum))); f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_CORRUPTED_SUMMARY); continue; @@ -2096,6 +2097,7 @@ int f2fs_gc_range(struct f2fs_sb_info *sbi, if (unlikely(f2fs_cp_error(sbi))) return -EIO; + stat_inc_gc_call_count(sbi, FOREGROUND); for (segno = start_seg; segno <= end_seg; segno += SEGS_PER_SEC(sbi)) { struct gc_inode_list gc_list = { .ilist = LIST_HEAD_INIT(gc_list.ilist), diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index 482a362f262543..591fcdf3ba77b9 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -606,7 +606,7 @@ int f2fs_get_node_info(struct f2fs_sb_info *sbi, nid_t nid, goto retry; } - i = f2fs_lookup_journal_in_cursum(journal, NAT_JOURNAL, nid, 0); + i = f2fs_lookup_journal_in_cursum(sbi, journal, NAT_JOURNAL, nid, 0); if (i >= 0) { ne = nat_in_journal(journal, i); node_info_from_raw_nat(ni, &ne); @@ -1500,9 +1500,9 @@ void f2fs_ra_node_page(struct f2fs_sb_info *sbi, nid_t nid) f2fs_folio_put(afolio, err ? true : false); } -static int sanity_check_node_footer(struct f2fs_sb_info *sbi, +int f2fs_sanity_check_node_footer(struct f2fs_sb_info *sbi, struct folio *folio, pgoff_t nid, - enum node_type ntype) + enum node_type ntype, bool in_irq) { if (unlikely(nid != nid_of_node(folio))) goto out_err; @@ -1527,12 +1527,13 @@ static int sanity_check_node_footer(struct f2fs_sb_info *sbi, goto out_err; return 0; out_err: - f2fs_warn(sbi, "inconsistent node block, node_type:%d, nid:%lu, " - "node_footer[nid:%u,ino:%u,ofs:%u,cpver:%llu,blkaddr:%u]", - ntype, nid, nid_of_node(folio), ino_of_node(folio), - ofs_of_node(folio), cpver_of_node(folio), - next_blkaddr_of_node(folio)); set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_warn_ratelimited(sbi, "inconsistent node block, node_type:%d, nid:%lu, " + "node_footer[nid:%u,ino:%u,ofs:%u,cpver:%llu,blkaddr:%u]", + ntype, nid, nid_of_node(folio), ino_of_node(folio), + ofs_of_node(folio), cpver_of_node(folio), + next_blkaddr_of_node(folio)); + f2fs_handle_error(sbi, ERROR_INCONSISTENT_FOOTER); return -EFSCORRUPTED; } @@ -1578,7 +1579,7 @@ static struct folio *__get_node_folio(struct f2fs_sb_info *sbi, pgoff_t nid, goto out_err; } page_hit: - err = sanity_check_node_footer(sbi, folio, nid, ntype); + err = f2fs_sanity_check_node_footer(sbi, folio, nid, ntype, false); if (!err) return folio; out_err: @@ -1751,7 +1752,12 @@ static bool __write_node_folio(struct folio *folio, bool atomic, bool *submitted /* get old block addr of this node page */ nid = nid_of_node(folio); - f2fs_bug_on(sbi, folio->index != nid); + + if (f2fs_sanity_check_node_footer(sbi, folio, nid, + NODE_TYPE_REGULAR, false)) { + f2fs_handle_critical_error(sbi, STOP_CP_REASON_CORRUPTED_NID); + goto redirty_out; + } if (f2fs_get_node_info(sbi, nid, &ni, !do_balance)) goto redirty_out; @@ -1774,8 +1780,13 @@ static bool __write_node_folio(struct folio *folio, bool atomic, bool *submitted goto redirty_out; } - if (atomic && !test_opt(sbi, NOBARRIER)) - fio.op_flags |= REQ_PREFLUSH | REQ_FUA; + if (atomic) { + if (!test_opt(sbi, NOBARRIER)) + fio.op_flags |= REQ_PREFLUSH | REQ_FUA; + if (IS_INODE(folio)) + set_dentry_mark(folio, + f2fs_need_dentry_mark(sbi, ino_of_node(folio))); + } /* should add to global list before clearing PAGECACHE status */ if (f2fs_in_warm_node_list(sbi, folio)) { @@ -1916,8 +1927,9 @@ int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode, if (is_inode_flag_set(inode, FI_DIRTY_INODE)) f2fs_update_inode(inode, folio); - set_dentry_mark(folio, - f2fs_need_dentry_mark(sbi, ino)); + if (!atomic) + set_dentry_mark(folio, + f2fs_need_dentry_mark(sbi, ino)); } /* may be written by other thread */ if (!folio_test_dirty(folio)) @@ -2937,7 +2949,7 @@ int f2fs_restore_node_summary(struct f2fs_sb_info *sbi, /* scan the node segment */ last_offset = BLKS_PER_SEG(sbi); addr = START_BLOCK(sbi, segno); - sum_entry = &sum->entries[0]; + sum_entry = sum_entries(sum); for (i = 0; i < last_offset; i += nrpages, addr += nrpages) { nrpages = bio_max_segs(last_offset - i); @@ -3078,7 +3090,7 @@ static int __flush_nat_entry_set(struct f2fs_sb_info *sbi, * #2, flush nat entries to nat page. */ if (enabled_nat_bits(sbi, cpc) || - !__has_cursum_space(journal, set->entry_cnt, NAT_JOURNAL)) + !__has_cursum_space(sbi, journal, set->entry_cnt, NAT_JOURNAL)) to_journal = false; if (to_journal) { @@ -3101,7 +3113,7 @@ static int __flush_nat_entry_set(struct f2fs_sb_info *sbi, f2fs_bug_on(sbi, nat_get_blkaddr(ne) == NEW_ADDR); if (to_journal) { - offset = f2fs_lookup_journal_in_cursum(journal, + offset = f2fs_lookup_journal_in_cursum(sbi, journal, NAT_JOURNAL, nid, 1); f2fs_bug_on(sbi, offset < 0); raw_ne = &nat_in_journal(journal, offset); @@ -3172,7 +3184,7 @@ int f2fs_flush_nat_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) * into nat entry set. */ if (enabled_nat_bits(sbi, cpc) || - !__has_cursum_space(journal, + !__has_cursum_space(sbi, journal, nm_i->nat_cnt[DIRTY_NAT], NAT_JOURNAL)) remove_nats_in_journal(sbi); @@ -3183,7 +3195,7 @@ int f2fs_flush_nat_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) set_idx = setvec[found - 1]->set + 1; for (idx = 0; idx < found; idx++) __adjust_nat_entry_set(setvec[idx], &sets, - MAX_NAT_JENTRIES(journal)); + MAX_NAT_JENTRIES(sbi, journal)); } /* flush dirty nats in nat entry set */ diff --git a/fs/f2fs/node.h b/fs/f2fs/node.h index 9cb8dcf8d41760..824ac9f0e6e42a 100644 --- a/fs/f2fs/node.h +++ b/fs/f2fs/node.h @@ -52,14 +52,6 @@ enum { IS_PREALLOC, /* nat entry is preallocated */ }; -/* For node type in __get_node_folio() */ -enum node_type { - NODE_TYPE_REGULAR, - NODE_TYPE_INODE, - NODE_TYPE_XATTR, - NODE_TYPE_NON_INODE, -}; - /* * For node information */ diff --git a/fs/f2fs/recovery.c b/fs/f2fs/recovery.c index c3415ebb9f5053..a6bfc8e759cf40 100644 --- a/fs/f2fs/recovery.c +++ b/fs/f2fs/recovery.c @@ -514,7 +514,7 @@ static int check_index_in_prev_nodes(struct f2fs_sb_info *sbi, struct curseg_info *curseg = CURSEG_I(sbi, i); if (curseg->segno == segno) { - sum = curseg->sum_blk->entries[blkoff]; + sum = sum_entries(curseg->sum_blk)[blkoff]; goto got_it; } } @@ -522,8 +522,8 @@ static int check_index_in_prev_nodes(struct f2fs_sb_info *sbi, sum_folio = f2fs_get_sum_folio(sbi, segno); if (IS_ERR(sum_folio)) return PTR_ERR(sum_folio); - sum_node = SUM_BLK_PAGE_ADDR(sum_folio, segno); - sum = sum_node->entries[blkoff]; + sum_node = SUM_BLK_PAGE_ADDR(sbi, sum_folio, segno); + sum = sum_entries(sum_node)[blkoff]; f2fs_folio_put(sum_folio, true); got_it: /* Use the locked dnode page and inode */ diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index c26424f4768635..90d0bac9d29400 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -2685,12 +2685,12 @@ int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra) valid_sum_count += f2fs_curseg_valid_blocks(sbi, i); } - sum_in_page = (PAGE_SIZE - 2 * SUM_JOURNAL_SIZE - + sum_in_page = (sbi->blocksize - 2 * sbi->sum_journal_size - SUM_FOOTER_SIZE) / SUMMARY_SIZE; if (valid_sum_count <= sum_in_page) return 1; else if ((valid_sum_count - sum_in_page) <= - (PAGE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE) + (sbi->blocksize - SUM_FOOTER_SIZE) / SUMMARY_SIZE) return 2; return 3; } @@ -2710,7 +2710,7 @@ void f2fs_update_meta_page(struct f2fs_sb_info *sbi, { struct folio *folio; - if (SUMS_PER_BLOCK == 1) + if (!f2fs_sb_has_packed_ssa(sbi)) folio = f2fs_grab_meta_folio(sbi, blk_addr); else folio = f2fs_get_meta_folio_retry(sbi, blk_addr); @@ -2728,7 +2728,7 @@ static void write_sum_page(struct f2fs_sb_info *sbi, { struct folio *folio; - if (SUMS_PER_BLOCK == 1) + if (!f2fs_sb_has_packed_ssa(sbi)) return f2fs_update_meta_page(sbi, (void *)sum_blk, GET_SUM_BLOCK(sbi, segno)); @@ -2736,7 +2736,8 @@ static void write_sum_page(struct f2fs_sb_info *sbi, if (IS_ERR(folio)) return; - memcpy(SUM_BLK_PAGE_ADDR(folio, segno), sum_blk, sizeof(*sum_blk)); + memcpy(SUM_BLK_PAGE_ADDR(sbi, folio, segno), sum_blk, + sbi->sum_blocksize); folio_mark_dirty(folio); f2fs_folio_put(folio, true); } @@ -2755,11 +2756,11 @@ static void write_current_sum_page(struct f2fs_sb_info *sbi, mutex_lock(&curseg->curseg_mutex); down_read(&curseg->journal_rwsem); - memcpy(&dst->journal, curseg->journal, SUM_JOURNAL_SIZE); + memcpy(sum_journal(sbi, dst), curseg->journal, sbi->sum_journal_size); up_read(&curseg->journal_rwsem); - memcpy(dst->entries, src->entries, SUM_ENTRY_SIZE); - memcpy(&dst->footer, &src->footer, SUM_FOOTER_SIZE); + memcpy(sum_entries(dst), sum_entries(src), sbi->sum_entry_size); + memcpy(sum_footer(sbi, dst), sum_footer(sbi, src), SUM_FOOTER_SIZE); mutex_unlock(&curseg->curseg_mutex); @@ -2932,7 +2933,7 @@ static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified) curseg->next_blkoff = 0; curseg->next_segno = NULL_SEGNO; - sum_footer = &(curseg->sum_blk->footer); + sum_footer = sum_footer(sbi, curseg->sum_blk); memset(sum_footer, 0, sizeof(struct summary_footer)); sanity_check_seg_type(sbi, seg_type); @@ -3078,11 +3079,11 @@ static int change_curseg(struct f2fs_sb_info *sbi, int type) sum_folio = f2fs_get_sum_folio(sbi, new_segno); if (IS_ERR(sum_folio)) { /* GC won't be able to use stale summary pages by cp_error */ - memset(curseg->sum_blk, 0, SUM_ENTRY_SIZE); + memset(curseg->sum_blk, 0, sbi->sum_entry_size); return PTR_ERR(sum_folio); } - sum_node = SUM_BLK_PAGE_ADDR(sum_folio, new_segno); - memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE); + sum_node = SUM_BLK_PAGE_ADDR(sbi, sum_folio, new_segno); + memcpy(curseg->sum_blk, sum_node, sbi->sum_entry_size); f2fs_folio_put(sum_folio, true); return 0; } @@ -3814,7 +3815,7 @@ int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct folio *folio, f2fs_wait_discard_bio(sbi, *new_blkaddr); - curseg->sum_blk->entries[curseg->next_blkoff] = *sum; + sum_entries(curseg->sum_blk)[curseg->next_blkoff] = *sum; if (curseg->alloc_type == SSR) { curseg->next_blkoff = f2fs_find_next_ssr_block(sbi, curseg); } else { @@ -4183,7 +4184,7 @@ void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, } curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr); - curseg->sum_blk->entries[curseg->next_blkoff] = *sum; + sum_entries(curseg->sum_blk)[curseg->next_blkoff] = *sum; if (!recover_curseg || recover_newaddr) { if (!from_gc) @@ -4303,12 +4304,12 @@ static int read_compacted_summaries(struct f2fs_sb_info *sbi) /* Step 1: restore nat cache */ seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); - memcpy(seg_i->journal, kaddr, SUM_JOURNAL_SIZE); + memcpy(seg_i->journal, kaddr, sbi->sum_journal_size); /* Step 2: restore sit cache */ seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); - memcpy(seg_i->journal, kaddr + SUM_JOURNAL_SIZE, SUM_JOURNAL_SIZE); - offset = 2 * SUM_JOURNAL_SIZE; + memcpy(seg_i->journal, kaddr + sbi->sum_journal_size, sbi->sum_journal_size); + offset = 2 * sbi->sum_journal_size; /* Step 3: restore summary entries */ for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { @@ -4330,9 +4331,9 @@ static int read_compacted_summaries(struct f2fs_sb_info *sbi) struct f2fs_summary *s; s = (struct f2fs_summary *)(kaddr + offset); - seg_i->sum_blk->entries[j] = *s; + sum_entries(seg_i->sum_blk)[j] = *s; offset += SUMMARY_SIZE; - if (offset + SUMMARY_SIZE <= PAGE_SIZE - + if (offset + SUMMARY_SIZE <= sbi->blocksize - SUM_FOOTER_SIZE) continue; @@ -4388,7 +4389,7 @@ static int read_normal_summaries(struct f2fs_sb_info *sbi, int type) if (IS_NODESEG(type)) { if (__exist_node_summaries(sbi)) { - struct f2fs_summary *ns = &sum->entries[0]; + struct f2fs_summary *ns = sum_entries(sum); int i; for (i = 0; i < BLKS_PER_SEG(sbi); i++, ns++) { @@ -4408,11 +4409,13 @@ static int read_normal_summaries(struct f2fs_sb_info *sbi, int type) /* update journal info */ down_write(&curseg->journal_rwsem); - memcpy(curseg->journal, &sum->journal, SUM_JOURNAL_SIZE); + memcpy(curseg->journal, sum_journal(sbi, sum), sbi->sum_journal_size); up_write(&curseg->journal_rwsem); - memcpy(curseg->sum_blk->entries, sum->entries, SUM_ENTRY_SIZE); - memcpy(&curseg->sum_blk->footer, &sum->footer, SUM_FOOTER_SIZE); + memcpy(sum_entries(curseg->sum_blk), sum_entries(sum), + sbi->sum_entry_size); + memcpy(sum_footer(sbi, curseg->sum_blk), sum_footer(sbi, sum), + SUM_FOOTER_SIZE); curseg->next_segno = segno; reset_curseg(sbi, type, 0); curseg->alloc_type = ckpt->alloc_type[type]; @@ -4456,8 +4459,8 @@ static int restore_curseg_summaries(struct f2fs_sb_info *sbi) } /* sanity check for summary blocks */ - if (nats_in_cursum(nat_j) > NAT_JOURNAL_ENTRIES || - sits_in_cursum(sit_j) > SIT_JOURNAL_ENTRIES) { + if (nats_in_cursum(nat_j) > sbi->nat_journal_entries || + sits_in_cursum(sit_j) > sbi->sit_journal_entries) { f2fs_err(sbi, "invalid journal entries nats %u sits %u", nats_in_cursum(nat_j), sits_in_cursum(sit_j)); return -EINVAL; @@ -4481,13 +4484,13 @@ static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr) /* Step 1: write nat cache */ seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); - memcpy(kaddr, seg_i->journal, SUM_JOURNAL_SIZE); - written_size += SUM_JOURNAL_SIZE; + memcpy(kaddr, seg_i->journal, sbi->sum_journal_size); + written_size += sbi->sum_journal_size; /* Step 2: write sit cache */ seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); - memcpy(kaddr + written_size, seg_i->journal, SUM_JOURNAL_SIZE); - written_size += SUM_JOURNAL_SIZE; + memcpy(kaddr + written_size, seg_i->journal, sbi->sum_journal_size); + written_size += sbi->sum_journal_size; /* Step 3: write summary entries */ for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { @@ -4500,10 +4503,10 @@ static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr) written_size = 0; } summary = (struct f2fs_summary *)(kaddr + written_size); - *summary = seg_i->sum_blk->entries[j]; + *summary = sum_entries(seg_i->sum_blk)[j]; written_size += SUMMARY_SIZE; - if (written_size + SUMMARY_SIZE <= PAGE_SIZE - + if (written_size + SUMMARY_SIZE <= sbi->blocksize - SUM_FOOTER_SIZE) continue; @@ -4545,8 +4548,9 @@ void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk) write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE); } -int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type, - unsigned int val, int alloc) +int f2fs_lookup_journal_in_cursum(struct f2fs_sb_info *sbi, + struct f2fs_journal *journal, int type, + unsigned int val, int alloc) { int i; @@ -4555,13 +4559,13 @@ int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type, if (le32_to_cpu(nid_in_journal(journal, i)) == val) return i; } - if (alloc && __has_cursum_space(journal, 1, NAT_JOURNAL)) + if (alloc && __has_cursum_space(sbi, journal, 1, NAT_JOURNAL)) return update_nats_in_cursum(journal, 1); } else if (type == SIT_JOURNAL) { for (i = 0; i < sits_in_cursum(journal); i++) if (le32_to_cpu(segno_in_journal(journal, i)) == val) return i; - if (alloc && __has_cursum_space(journal, 1, SIT_JOURNAL)) + if (alloc && __has_cursum_space(sbi, journal, 1, SIT_JOURNAL)) return update_sits_in_cursum(journal, 1); } return -1; @@ -4709,8 +4713,8 @@ void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) * entries, remove all entries from journal and add and account * them in sit entry set. */ - if (!__has_cursum_space(journal, sit_i->dirty_sentries, SIT_JOURNAL) || - !to_journal) + if (!__has_cursum_space(sbi, journal, + sit_i->dirty_sentries, SIT_JOURNAL) || !to_journal) remove_sits_in_journal(sbi); /* @@ -4727,7 +4731,8 @@ void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) unsigned int segno = start_segno; if (to_journal && - !__has_cursum_space(journal, ses->entry_cnt, SIT_JOURNAL)) + !__has_cursum_space(sbi, journal, ses->entry_cnt, + SIT_JOURNAL)) to_journal = false; if (to_journal) { @@ -4755,7 +4760,7 @@ void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) } if (to_journal) { - offset = f2fs_lookup_journal_in_cursum(journal, + offset = f2fs_lookup_journal_in_cursum(sbi, journal, SIT_JOURNAL, segno, 1); f2fs_bug_on(sbi, offset < 0); segno_in_journal(journal, offset) = @@ -4962,12 +4967,13 @@ static int build_curseg(struct f2fs_sb_info *sbi) for (i = 0; i < NO_CHECK_TYPE; i++) { mutex_init(&array[i].curseg_mutex); - array[i].sum_blk = f2fs_kzalloc(sbi, PAGE_SIZE, GFP_KERNEL); + array[i].sum_blk = f2fs_kzalloc(sbi, sbi->sum_blocksize, + GFP_KERNEL); if (!array[i].sum_blk) return -ENOMEM; init_rwsem(&array[i].journal_rwsem); array[i].journal = f2fs_kzalloc(sbi, - sizeof(struct f2fs_journal), GFP_KERNEL); + sbi->sum_journal_size, GFP_KERNEL); if (!array[i].journal) return -ENOMEM; array[i].seg_type = log_type_to_seg_type(i); diff --git a/fs/f2fs/segment.h b/fs/f2fs/segment.h index 07dcbcbeb7c6d3..3094f2de37b636 100644 --- a/fs/f2fs/segment.h +++ b/fs/f2fs/segment.h @@ -90,12 +90,11 @@ static inline void sanity_check_seg_type(struct f2fs_sb_info *sbi, #define GET_ZONE_FROM_SEG(sbi, segno) \ GET_ZONE_FROM_SEC(sbi, GET_SEC_FROM_SEG(sbi, segno)) -#define SUMS_PER_BLOCK (F2FS_BLKSIZE / F2FS_SUM_BLKSIZE) #define GET_SUM_BLOCK(sbi, segno) \ - (SM_I(sbi)->ssa_blkaddr + (segno / SUMS_PER_BLOCK)) -#define GET_SUM_BLKOFF(segno) (segno % SUMS_PER_BLOCK) -#define SUM_BLK_PAGE_ADDR(folio, segno) \ - (folio_address(folio) + GET_SUM_BLKOFF(segno) * F2FS_SUM_BLKSIZE) + (SM_I(sbi)->ssa_blkaddr + (segno / (sbi)->sums_per_block)) +#define GET_SUM_BLKOFF(sbi, segno) (segno % (sbi)->sums_per_block) +#define SUM_BLK_PAGE_ADDR(sbi, folio, segno) \ + (folio_address(folio) + GET_SUM_BLKOFF(sbi, segno) * (sbi)->sum_blocksize) #define GET_SUM_TYPE(footer) ((footer)->entry_type) #define SET_SUM_TYPE(footer, type) ((footer)->entry_type = (type)) diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index c4c225e09dc470..6be6d7372badf5 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -2636,11 +2636,10 @@ static int f2fs_disable_checkpoint(struct f2fs_sb_info *sbi) static int f2fs_enable_checkpoint(struct f2fs_sb_info *sbi) { unsigned int nr_pages = get_pages(sbi, F2FS_DIRTY_DATA) / 16; - long long start, writeback, lock, sync_inode, end; + long long start, writeback, end; int ret; - f2fs_info(sbi, "%s start, meta: %lld, node: %lld, data: %lld", - __func__, + f2fs_info(sbi, "f2fs_enable_checkpoint() starts, meta: %lld, node: %lld, data: %lld", get_pages(sbi, F2FS_DIRTY_META), get_pages(sbi, F2FS_DIRTY_NODES), get_pages(sbi, F2FS_DIRTY_DATA)); @@ -2659,18 +2658,11 @@ static int f2fs_enable_checkpoint(struct f2fs_sb_info *sbi) } writeback = ktime_get(); - f2fs_down_write(&sbi->cp_enable_rwsem); - - lock = ktime_get(); - - if (get_pages(sbi, F2FS_DIRTY_DATA)) - sync_inodes_sb(sbi->sb); + sync_inodes_sb(sbi->sb); if (unlikely(get_pages(sbi, F2FS_DIRTY_DATA))) - f2fs_warn(sbi, "%s: has some unwritten data: %lld", - __func__, get_pages(sbi, F2FS_DIRTY_DATA)); - - sync_inode = ktime_get(); + f2fs_warn(sbi, "checkpoint=enable has some unwritten data: %lld", + get_pages(sbi, F2FS_DIRTY_DATA)); f2fs_down_write(&sbi->gc_lock); f2fs_dirty_to_prefree(sbi); @@ -2679,13 +2671,6 @@ static int f2fs_enable_checkpoint(struct f2fs_sb_info *sbi) set_sbi_flag(sbi, SBI_IS_DIRTY); f2fs_up_write(&sbi->gc_lock); - f2fs_info(sbi, "%s sync_fs, meta: %lld, imeta: %lld, node: %lld, dents: %lld, qdata: %lld", - __func__, - get_pages(sbi, F2FS_DIRTY_META), - get_pages(sbi, F2FS_DIRTY_IMETA), - get_pages(sbi, F2FS_DIRTY_NODES), - get_pages(sbi, F2FS_DIRTY_DENTS), - get_pages(sbi, F2FS_DIRTY_QDATA)); ret = f2fs_sync_fs(sbi->sb, 1); if (ret) f2fs_err(sbi, "%s sync_fs failed, ret: %d", __func__, ret); @@ -2693,17 +2678,11 @@ static int f2fs_enable_checkpoint(struct f2fs_sb_info *sbi) /* Let's ensure there's no pending checkpoint anymore */ f2fs_flush_ckpt_thread(sbi); - f2fs_up_write(&sbi->cp_enable_rwsem); - end = ktime_get(); - f2fs_info(sbi, "%s end, writeback:%llu, " - "lock:%llu, sync_inode:%llu, sync_fs:%llu", - __func__, - ktime_ms_delta(writeback, start), - ktime_ms_delta(lock, writeback), - ktime_ms_delta(sync_inode, lock), - ktime_ms_delta(end, sync_inode)); + f2fs_info(sbi, "f2fs_enable_checkpoint() finishes, writeback:%llu, sync:%llu", + ktime_ms_delta(writeback, start), + ktime_ms_delta(end, writeback)); return ret; } @@ -4080,20 +4059,6 @@ static int sanity_check_raw_super(struct f2fs_sb_info *sbi, if (sanity_check_area_boundary(sbi, folio, index)) return -EFSCORRUPTED; - /* - * Check for legacy summary layout on 16KB+ block devices. - * Modern f2fs-tools packs multiple 4KB summary areas into one block, - * whereas legacy versions used one block per summary, leading - * to a much larger SSA. - */ - if (SUMS_PER_BLOCK > 1 && - !(__F2FS_HAS_FEATURE(raw_super, F2FS_FEATURE_PACKED_SSA))) { - f2fs_info(sbi, "Error: Device formatted with a legacy version. " - "Please reformat with a tool supporting the packed ssa " - "feature for block sizes larger than 4kb."); - return -EOPNOTSUPP; - } - return 0; } @@ -4304,6 +4269,18 @@ static void init_sb_info(struct f2fs_sb_info *sbi) spin_lock_init(&sbi->gc_remaining_trials_lock); atomic64_set(&sbi->current_atomic_write, 0); + sbi->sum_blocksize = f2fs_sb_has_packed_ssa(sbi) ? + 4096 : sbi->blocksize; + sbi->sums_per_block = sbi->blocksize / sbi->sum_blocksize; + sbi->entries_in_sum = sbi->sum_blocksize / 8; + sbi->sum_entry_size = SUMMARY_SIZE * sbi->entries_in_sum; + sbi->sum_journal_size = sbi->sum_blocksize - SUM_FOOTER_SIZE - + sbi->sum_entry_size; + sbi->nat_journal_entries = (sbi->sum_journal_size - 2) / + sizeof(struct nat_journal_entry); + sbi->sit_journal_entries = (sbi->sum_journal_size - 2) / + sizeof(struct sit_journal_entry); + sbi->dir_level = DEF_DIR_LEVEL; sbi->interval_time[CP_TIME] = DEF_CP_INTERVAL; sbi->interval_time[REQ_TIME] = DEF_IDLE_INTERVAL; @@ -4906,7 +4883,6 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc) init_f2fs_rwsem(&sbi->node_change); spin_lock_init(&sbi->stat_lock); init_f2fs_rwsem(&sbi->cp_rwsem); - init_f2fs_rwsem(&sbi->cp_enable_rwsem); init_f2fs_rwsem(&sbi->quota_sem); init_waitqueue_head(&sbi->cp_wait); spin_lock_init(&sbi->error_lock); diff --git a/fs/f2fs/sysfs.c b/fs/f2fs/sysfs.c index c42f4f979d13f3..353bf47959f36a 100644 --- a/fs/f2fs/sysfs.c +++ b/fs/f2fs/sysfs.c @@ -58,6 +58,7 @@ struct f2fs_attr { const char *buf, size_t len); int struct_type; int offset; + int size; int id; }; @@ -344,11 +345,30 @@ static ssize_t main_blkaddr_show(struct f2fs_attr *a, (unsigned long long)MAIN_BLKADDR(sbi)); } +static ssize_t __sbi_show_value(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf, + unsigned char *value) +{ + switch (a->size) { + case 1: + return sysfs_emit(buf, "%u\n", *(u8 *)value); + case 2: + return sysfs_emit(buf, "%u\n", *(u16 *)value); + case 4: + return sysfs_emit(buf, "%u\n", *(u32 *)value); + case 8: + return sysfs_emit(buf, "%llu\n", *(u64 *)value); + default: + f2fs_bug_on(sbi, 1); + return sysfs_emit(buf, + "show sysfs node value with wrong type\n"); + } +} + static ssize_t f2fs_sbi_show(struct f2fs_attr *a, struct f2fs_sb_info *sbi, char *buf) { unsigned char *ptr = NULL; - unsigned int *ui; ptr = __struct_ptr(sbi, a->struct_type); if (!ptr) @@ -428,9 +448,30 @@ static ssize_t f2fs_sbi_show(struct f2fs_attr *a, atomic_read(&sbi->cp_call_count[BACKGROUND])); #endif - ui = (unsigned int *)(ptr + a->offset); + return __sbi_show_value(a, sbi, buf, ptr + a->offset); +} - return sysfs_emit(buf, "%u\n", *ui); +static void __sbi_store_value(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, + unsigned char *ui, unsigned long value) +{ + switch (a->size) { + case 1: + *(u8 *)ui = value; + break; + case 2: + *(u16 *)ui = value; + break; + case 4: + *(u32 *)ui = value; + break; + case 8: + *(u64 *)ui = value; + break; + default: + f2fs_bug_on(sbi, 1); + f2fs_err(sbi, "store sysfs node value with wrong type"); + } } static ssize_t __sbi_store(struct f2fs_attr *a, @@ -749,7 +790,7 @@ static ssize_t __sbi_store(struct f2fs_attr *a, return count; } - if (!strcmp(a->attr.name, "gc_pin_file_threshold")) { + if (!strcmp(a->attr.name, "gc_pin_file_thresh")) { if (t > MAX_GC_FAILED_PINNED_FILES) return -EINVAL; sbi->gc_pin_file_threshold = t; @@ -906,7 +947,7 @@ static ssize_t __sbi_store(struct f2fs_attr *a, return count; } - *ui = (unsigned int)t; + __sbi_store_value(a, sbi, ptr + a->offset, t); return count; } @@ -1053,24 +1094,27 @@ static struct f2fs_attr f2fs_attr_sb_##_name = { \ .id = F2FS_FEATURE_##_feat, \ } -#define F2FS_ATTR_OFFSET(_struct_type, _name, _mode, _show, _store, _offset) \ +#define F2FS_ATTR_OFFSET(_struct_type, _name, _mode, _show, _store, _offset, _size) \ static struct f2fs_attr f2fs_attr_##_name = { \ .attr = {.name = __stringify(_name), .mode = _mode }, \ .show = _show, \ .store = _store, \ .struct_type = _struct_type, \ - .offset = _offset \ + .offset = _offset, \ + .size = _size \ } #define F2FS_RO_ATTR(struct_type, struct_name, name, elname) \ F2FS_ATTR_OFFSET(struct_type, name, 0444, \ f2fs_sbi_show, NULL, \ - offsetof(struct struct_name, elname)) + offsetof(struct struct_name, elname), \ + sizeof_field(struct struct_name, elname)) #define F2FS_RW_ATTR(struct_type, struct_name, name, elname) \ F2FS_ATTR_OFFSET(struct_type, name, 0644, \ f2fs_sbi_show, f2fs_sbi_store, \ - offsetof(struct struct_name, elname)) + offsetof(struct struct_name, elname), \ + sizeof_field(struct struct_name, elname)) #define F2FS_GENERAL_RO_ATTR(name) \ static struct f2fs_attr f2fs_attr_##name = __ATTR(name, 0444, name##_show, NULL) diff --git a/fs/fat/namei_msdos.c b/fs/fat/namei_msdos.c index 0b920ee40a7f9f..262ec1b790b560 100644 --- a/fs/fat/namei_msdos.c +++ b/fs/fat/namei_msdos.c @@ -325,7 +325,12 @@ static int msdos_rmdir(struct inode *dir, struct dentry *dentry) err = fat_remove_entries(dir, &sinfo); /* and releases bh */ if (err) goto out; - drop_nlink(dir); + if (dir->i_nlink >= 3) + drop_nlink(dir); + else { + fat_fs_error(sb, "parent dir link count too low (%u)", + dir->i_nlink); + } clear_nlink(inode); fat_truncate_time(inode, NULL, S_CTIME); diff --git a/fs/fat/namei_vfat.c b/fs/fat/namei_vfat.c index 5dbc4cbb8fce3d..47ff083cfc7e66 100644 --- a/fs/fat/namei_vfat.c +++ b/fs/fat/namei_vfat.c @@ -803,7 +803,12 @@ static int vfat_rmdir(struct inode *dir, struct dentry *dentry) err = fat_remove_entries(dir, &sinfo); /* and releases bh */ if (err) goto out; - drop_nlink(dir); + if (dir->i_nlink >= 3) + drop_nlink(dir); + else { + fat_fs_error(sb, "parent dir link count too low (%u)", + dir->i_nlink); + } clear_nlink(inode); fat_truncate_time(inode, NULL, S_ATIME|S_MTIME); diff --git a/fs/file_table.c b/fs/file_table.c index cd4a3db4659ac4..34244fccf2edf0 100644 --- a/fs/file_table.c +++ b/fs/file_table.c @@ -176,6 +176,11 @@ static int init_file(struct file *f, int flags, const struct cred *cred) f->f_flags = flags; f->f_mode = OPEN_FMODE(flags); + /* + * Disable permission and pre-content events for all files by default. + * They may be enabled later by fsnotify_open_perm_and_set_mode(). + */ + file_set_fsnotify_mode(f, FMODE_NONOTIFY_PERM); f->f_op = NULL; f->f_mapping = NULL; @@ -197,11 +202,6 @@ static int init_file(struct file *f, int flags, const struct cred *cred) * refcount bumps we should reinitialize the reused file first. */ file_ref_init(&f->f_ref, 1); - /* - * Disable permission and pre-content events for all files by default. - * They may be enabled later by fsnotify_open_perm_and_set_mode(). - */ - file_set_fsnotify_mode(f, FMODE_NONOTIFY_PERM); return 0; } diff --git a/fs/fs_struct.c b/fs/fs_struct.c index b8c46c5a38a057..394875d06fd606 100644 --- a/fs/fs_struct.c +++ b/fs/fs_struct.c @@ -6,6 +6,7 @@ #include #include #include +#include #include "internal.h" /* diff --git a/fs/gfs2/bmap.c b/fs/gfs2/bmap.c index 131091520de6be..fdcac8e3f2ba26 100644 --- a/fs/gfs2/bmap.c +++ b/fs/gfs2/bmap.c @@ -1127,10 +1127,18 @@ static int gfs2_iomap_begin(struct inode *inode, loff_t pos, loff_t length, goto out_unlock; break; default: - goto out_unlock; + goto out; } ret = gfs2_iomap_begin_write(inode, pos, length, flags, iomap, &mp); + if (ret) + goto out_unlock; + +out: + if (iomap->type == IOMAP_INLINE) { + iomap->private = metapath_dibh(&mp); + get_bh(iomap->private); + } out_unlock: release_metapath(&mp); @@ -1144,6 +1152,9 @@ static int gfs2_iomap_end(struct inode *inode, loff_t pos, loff_t length, struct gfs2_inode *ip = GFS2_I(inode); struct gfs2_sbd *sdp = GFS2_SB(inode); + if (iomap->private) + brelse(iomap->private); + switch (flags & (IOMAP_WRITE | IOMAP_ZERO)) { case IOMAP_WRITE: if (flags & IOMAP_DIRECT) diff --git a/fs/gfs2/glock.c b/fs/gfs2/glock.c index 92e029104d8a42..289851d70130bd 100644 --- a/fs/gfs2/glock.c +++ b/fs/gfs2/glock.c @@ -1284,31 +1284,45 @@ static int glocks_pending(unsigned int num_gh, struct gfs2_holder *ghs) * gfs2_glock_async_wait - wait on multiple asynchronous glock acquisitions * @num_gh: the number of holders in the array * @ghs: the glock holder array + * @retries: number of retries attempted so far * * Returns: 0 on success, meaning all glocks have been granted and are held. * -ESTALE if the request timed out, meaning all glocks were released, * and the caller should retry the operation. */ -int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs) +int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs, + unsigned int retries) { struct gfs2_sbd *sdp = ghs[0].gh_gl->gl_name.ln_sbd; - int i, ret = 0, timeout = 0; unsigned long start_time = jiffies; + int i, ret = 0; + long timeout; might_sleep(); - /* - * Total up the (minimum hold time * 2) of all glocks and use that to - * determine the max amount of time we should wait. - */ - for (i = 0; i < num_gh; i++) - timeout += ghs[i].gh_gl->gl_hold_time << 1; - if (!wait_event_timeout(sdp->sd_async_glock_wait, + timeout = GL_GLOCK_MIN_HOLD; + if (retries) { + unsigned int max_shift; + long incr; + + /* Add a random delay and increase the timeout exponentially. */ + max_shift = BITS_PER_LONG - 2 - __fls(GL_GLOCK_HOLD_INCR); + incr = min(GL_GLOCK_HOLD_INCR << min(retries - 1, max_shift), + 10 * HZ - GL_GLOCK_MIN_HOLD); + schedule_timeout_interruptible(get_random_long() % (incr / 3)); + if (signal_pending(current)) + goto interrupted; + timeout += (incr / 3) + get_random_long() % (incr / 3); + } + + if (!wait_event_interruptible_timeout(sdp->sd_async_glock_wait, !glocks_pending(num_gh, ghs), timeout)) { ret = -ESTALE; /* request timed out. */ goto out; } + if (signal_pending(current)) + goto interrupted; for (i = 0; i < num_gh; i++) { struct gfs2_holder *gh = &ghs[i]; @@ -1332,6 +1346,10 @@ int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs) } } return ret; + +interrupted: + ret = -EINTR; + goto out; } /** diff --git a/fs/gfs2/glock.h b/fs/gfs2/glock.h index 55d5985f32a080..dccbf36b8cb108 100644 --- a/fs/gfs2/glock.h +++ b/fs/gfs2/glock.h @@ -204,7 +204,8 @@ int gfs2_glock_poll(struct gfs2_holder *gh); int gfs2_instantiate(struct gfs2_holder *gh); int gfs2_glock_holder_ready(struct gfs2_holder *gh); int gfs2_glock_wait(struct gfs2_holder *gh); -int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs); +int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs, + unsigned int retries); void gfs2_glock_dq(struct gfs2_holder *gh); void gfs2_glock_dq_wait(struct gfs2_holder *gh); void gfs2_glock_dq_uninit(struct gfs2_holder *gh); diff --git a/fs/gfs2/inode.c b/fs/gfs2/inode.c index 36618e35319965..b6ed069b348721 100644 --- a/fs/gfs2/inode.c +++ b/fs/gfs2/inode.c @@ -1495,7 +1495,7 @@ static int gfs2_rename(struct inode *odir, struct dentry *odentry, unsigned int num_gh; int dir_rename = 0; struct gfs2_diradd da = { .nr_blocks = 0, .save_loc = 0, }; - unsigned int x; + unsigned int retries = 0, x; int error; gfs2_holder_mark_uninitialized(&r_gh); @@ -1545,12 +1545,17 @@ static int gfs2_rename(struct inode *odir, struct dentry *odentry, num_gh++; } +again: for (x = 0; x < num_gh; x++) { error = gfs2_glock_nq(ghs + x); if (error) goto out_gunlock; } - error = gfs2_glock_async_wait(num_gh, ghs); + error = gfs2_glock_async_wait(num_gh, ghs, retries); + if (error == -ESTALE) { + retries++; + goto again; + } if (error) goto out_gunlock; @@ -1739,7 +1744,7 @@ static int gfs2_exchange(struct inode *odir, struct dentry *odentry, struct gfs2_sbd *sdp = GFS2_SB(odir); struct gfs2_holder ghs[4], r_gh; unsigned int num_gh; - unsigned int x; + unsigned int retries = 0, x; umode_t old_mode = oip->i_inode.i_mode; umode_t new_mode = nip->i_inode.i_mode; int error; @@ -1783,13 +1788,18 @@ static int gfs2_exchange(struct inode *odir, struct dentry *odentry, gfs2_holder_init(nip->i_gl, LM_ST_EXCLUSIVE, GL_ASYNC, ghs + num_gh); num_gh++; +again: for (x = 0; x < num_gh; x++) { error = gfs2_glock_nq(ghs + x); if (error) goto out_gunlock; } - error = gfs2_glock_async_wait(num_gh, ghs); + error = gfs2_glock_async_wait(num_gh, ghs, retries); + if (error == -ESTALE) { + retries++; + goto again; + } if (error) goto out_gunlock; diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c index 8312cd2cdae471..347df29d610e67 100644 --- a/fs/gfs2/log.c +++ b/fs/gfs2/log.c @@ -888,8 +888,9 @@ void gfs2_write_log_header(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd, sb->s_blocksize - LH_V1_SIZE - 4); lh->lh_crc = cpu_to_be32(crc); - gfs2_log_write(sdp, jd, page, sb->s_blocksize, 0, dblock); - gfs2_log_submit_bio(&jd->jd_log_bio, REQ_OP_WRITE | op_flags); + gfs2_log_write(sdp, jd, page, sb->s_blocksize, 0, dblock, + REQ_OP_WRITE | op_flags); + gfs2_log_submit_write(&jd->jd_log_bio); } /** @@ -1096,7 +1097,7 @@ void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, u32 flags) if (gfs2_withdrawn(sdp)) goto out_withdraw; if (sdp->sd_jdesc) - gfs2_log_submit_bio(&sdp->sd_jdesc->jd_log_bio, REQ_OP_WRITE); + gfs2_log_submit_write(&sdp->sd_jdesc->jd_log_bio); if (gfs2_withdrawn(sdp)) goto out_withdraw; diff --git a/fs/gfs2/lops.c b/fs/gfs2/lops.c index d27a0b1080a97f..c3317432a25b38 100644 --- a/fs/gfs2/lops.c +++ b/fs/gfs2/lops.c @@ -229,21 +229,19 @@ static void gfs2_end_log_write(struct bio *bio) } /** - * gfs2_log_submit_bio - Submit any pending log bio + * gfs2_log_submit_write - Submit a pending log write bio * @biop: Address of the bio pointer - * @opf: REQ_OP | op_flags * * Submit any pending part-built or full bio to the block device. If * there is no pending bio, then this is a no-op. */ -void gfs2_log_submit_bio(struct bio **biop, blk_opf_t opf) +void gfs2_log_submit_write(struct bio **biop) { struct bio *bio = *biop; if (bio) { struct gfs2_sbd *sdp = bio->bi_private; atomic_inc(&sdp->sd_log_in_flight); - bio->bi_opf = opf; submit_bio(bio); *biop = NULL; } @@ -254,6 +252,7 @@ void gfs2_log_submit_bio(struct bio **biop, blk_opf_t opf) * @sdp: The super block * @blkno: The device block number we want to write to * @end_io: The bi_end_io callback + * @opf: REQ_OP | op_flags * * Allocate a new bio, initialize it with the given parameters and return it. * @@ -261,10 +260,10 @@ void gfs2_log_submit_bio(struct bio **biop, blk_opf_t opf) */ static struct bio *gfs2_log_alloc_bio(struct gfs2_sbd *sdp, u64 blkno, - bio_end_io_t *end_io) + bio_end_io_t *end_io, blk_opf_t opf) { struct super_block *sb = sdp->sd_vfs; - struct bio *bio = bio_alloc(sb->s_bdev, BIO_MAX_VECS, 0, GFP_NOIO); + struct bio *bio = bio_alloc(sb->s_bdev, BIO_MAX_VECS, opf, GFP_NOIO); bio->bi_iter.bi_sector = blkno << sdp->sd_fsb2bb_shift; bio->bi_end_io = end_io; @@ -303,10 +302,10 @@ static struct bio *gfs2_log_get_bio(struct gfs2_sbd *sdp, u64 blkno, nblk >>= sdp->sd_fsb2bb_shift; if (blkno == nblk && !flush) return bio; - gfs2_log_submit_bio(biop, op); + gfs2_log_submit_write(biop); } - *biop = gfs2_log_alloc_bio(sdp, blkno, end_io); + *biop = gfs2_log_alloc_bio(sdp, blkno, end_io, op); return *biop; } @@ -318,6 +317,7 @@ static struct bio *gfs2_log_get_bio(struct gfs2_sbd *sdp, u64 blkno, * @size: the size of the data to write * @offset: the offset within the page * @blkno: block number of the log entry + * @opf: REQ_OP | op_flags * * Try and add the page segment to the current bio. If that fails, * submit the current bio to the device and create a new one, and @@ -326,17 +326,17 @@ static struct bio *gfs2_log_get_bio(struct gfs2_sbd *sdp, u64 blkno, void gfs2_log_write(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd, struct page *page, unsigned size, unsigned offset, - u64 blkno) + u64 blkno, blk_opf_t opf) { struct bio *bio; int ret; - bio = gfs2_log_get_bio(sdp, blkno, &jd->jd_log_bio, REQ_OP_WRITE, + bio = gfs2_log_get_bio(sdp, blkno, &jd->jd_log_bio, opf, gfs2_end_log_write, false); ret = bio_add_page(bio, page, size, offset); if (ret == 0) { bio = gfs2_log_get_bio(sdp, blkno, &jd->jd_log_bio, - REQ_OP_WRITE, gfs2_end_log_write, true); + opf, gfs2_end_log_write, true); ret = bio_add_page(bio, page, size, offset); WARN_ON(ret == 0); } @@ -359,7 +359,7 @@ static void gfs2_log_write_bh(struct gfs2_sbd *sdp, struct buffer_head *bh) dblock = gfs2_log_bmap(sdp->sd_jdesc, sdp->sd_log_flush_head); gfs2_log_incr_head(sdp); gfs2_log_write(sdp, sdp->sd_jdesc, folio_page(bh->b_folio, 0), - bh->b_size, bh_offset(bh), dblock); + bh->b_size, bh_offset(bh), dblock, REQ_OP_WRITE); } /** @@ -380,7 +380,8 @@ static void gfs2_log_write_page(struct gfs2_sbd *sdp, struct page *page) dblock = gfs2_log_bmap(sdp->sd_jdesc, sdp->sd_log_flush_head); gfs2_log_incr_head(sdp); - gfs2_log_write(sdp, sdp->sd_jdesc, page, sb->s_blocksize, 0, dblock); + gfs2_log_write(sdp, sdp->sd_jdesc, page, sb->s_blocksize, 0, dblock, + REQ_OP_WRITE); } /** @@ -477,11 +478,12 @@ static void gfs2_jhead_process_page(struct gfs2_jdesc *jd, unsigned long index, folio_put_refs(folio, 2); } -static struct bio *gfs2_chain_bio(struct bio *prev, unsigned int nr_iovecs) +static struct bio *gfs2_chain_bio(struct bio *prev, unsigned int nr_iovecs, + blk_opf_t opf) { struct bio *new; - new = bio_alloc(prev->bi_bdev, nr_iovecs, prev->bi_opf, GFP_NOIO); + new = bio_alloc(prev->bi_bdev, nr_iovecs, opf, GFP_NOIO); bio_clone_blkg_association(new, prev); new->bi_iter.bi_sector = bio_end_sector(prev); bio_chain(new, prev); @@ -546,7 +548,8 @@ int gfs2_find_jhead(struct gfs2_jdesc *jd, struct gfs2_log_header_host *head) unsigned int blocks = (PAGE_SIZE - off) >> bsize_shift; - bio = gfs2_chain_bio(bio, blocks); + bio = gfs2_chain_bio(bio, blocks, + REQ_OP_READ); goto add_block_to_new_bio; } } @@ -556,8 +559,8 @@ int gfs2_find_jhead(struct gfs2_jdesc *jd, struct gfs2_log_header_host *head) submit_bio(bio); } - bio = gfs2_log_alloc_bio(sdp, dblock, gfs2_end_log_read); - bio->bi_opf = REQ_OP_READ; + bio = gfs2_log_alloc_bio(sdp, dblock, gfs2_end_log_read, + REQ_OP_READ); add_block_to_new_bio: bio_add_folio_nofail(bio, folio, bsize, off); block_added: diff --git a/fs/gfs2/lops.h b/fs/gfs2/lops.h index be740bf3366646..772557b63b48b7 100644 --- a/fs/gfs2/lops.h +++ b/fs/gfs2/lops.h @@ -16,8 +16,8 @@ void gfs2_log_incr_head(struct gfs2_sbd *sdp); u64 gfs2_log_bmap(struct gfs2_jdesc *jd, unsigned int lbn); void gfs2_log_write(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd, struct page *page, unsigned size, unsigned offset, - u64 blkno); -void gfs2_log_submit_bio(struct bio **biop, blk_opf_t opf); + u64 blkno, blk_opf_t opf); +void gfs2_log_submit_write(struct bio **biop); void gfs2_pin(struct gfs2_sbd *sdp, struct buffer_head *bh); int gfs2_find_jhead(struct gfs2_jdesc *jd, struct gfs2_log_header_host *head); diff --git a/fs/gfs2/ops_fstype.c b/fs/gfs2/ops_fstype.c index e7a88b717991ae..c7d57de7c8f06b 100644 --- a/fs/gfs2/ops_fstype.c +++ b/fs/gfs2/ops_fstype.c @@ -1276,7 +1276,6 @@ static int gfs2_fill_super(struct super_block *sb, struct fs_context *fc) if (error) { gfs2_freeze_unlock(sdp); - gfs2_destroy_threads(sdp); fs_err(sdp, "can't make FS RW: %d\n", error); goto fail_per_node; } @@ -1286,6 +1285,7 @@ static int gfs2_fill_super(struct super_block *sb, struct fs_context *fc) fail_per_node: init_per_node(sdp, UNDO); + gfs2_destroy_threads(sdp); fail_inodes: init_inodes(sdp, UNDO); fail_sb: diff --git a/fs/gfs2/quota.c b/fs/gfs2/quota.c index b1692f12a602a6..2b499b554e876a 100644 --- a/fs/gfs2/quota.c +++ b/fs/gfs2/quota.c @@ -334,6 +334,7 @@ static void qd_put(struct gfs2_quota_data *qd) lockref_mark_dead(&qd->qd_lockref); spin_unlock(&qd->qd_lockref.lock); + list_lru_del_obj(&gfs2_qd_lru, &qd->qd_lru); gfs2_qd_dispose(qd); return; } diff --git a/fs/gfs2/super.c b/fs/gfs2/super.c index f6cd907b3ec6c4..d96160636161c8 100644 --- a/fs/gfs2/super.c +++ b/fs/gfs2/super.c @@ -147,8 +147,10 @@ int gfs2_make_fs_rw(struct gfs2_sbd *sdp) } error = gfs2_quota_init(sdp); - if (!error && gfs2_withdrawn(sdp)) + if (!error && gfs2_withdrawn(sdp)) { + gfs2_quota_cleanup(sdp); error = -EIO; + } if (!error) set_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags); return error; diff --git a/fs/hfs/mdb.c b/fs/hfs/mdb.c index 53f3fae6021797..f28cd24dee8425 100644 --- a/fs/hfs/mdb.c +++ b/fs/hfs/mdb.c @@ -92,7 +92,7 @@ int hfs_mdb_get(struct super_block *sb) /* See if this is an HFS filesystem */ bh = sb_bread512(sb, part_start + HFS_MDB_BLK, mdb); if (!bh) - goto out; + return -EIO; if (mdb->drSigWord == cpu_to_be16(HFS_SUPER_MAGIC)) break; @@ -102,13 +102,14 @@ int hfs_mdb_get(struct super_block *sb) * (should do this only for cdrom/loop though) */ if (hfs_part_find(sb, &part_start, &part_size)) - goto out; + return -EIO; } HFS_SB(sb)->alloc_blksz = size = be32_to_cpu(mdb->drAlBlkSiz); if (!size || (size & (HFS_SECTOR_SIZE - 1))) { pr_err("bad allocation block size %d\n", size); - goto out_bh; + brelse(bh); + return -EIO; } size = min(HFS_SB(sb)->alloc_blksz, (u32)PAGE_SIZE); @@ -125,14 +126,16 @@ int hfs_mdb_get(struct super_block *sb) brelse(bh); if (!sb_set_blocksize(sb, size)) { pr_err("unable to set blocksize to %u\n", size); - goto out; + return -EIO; } bh = sb_bread512(sb, part_start + HFS_MDB_BLK, mdb); if (!bh) - goto out; - if (mdb->drSigWord != cpu_to_be16(HFS_SUPER_MAGIC)) - goto out_bh; + return -EIO; + if (mdb->drSigWord != cpu_to_be16(HFS_SUPER_MAGIC)) { + brelse(bh); + return -EIO; + } HFS_SB(sb)->mdb_bh = bh; HFS_SB(sb)->mdb = mdb; @@ -174,7 +177,7 @@ int hfs_mdb_get(struct super_block *sb) HFS_SB(sb)->bitmap = kzalloc(8192, GFP_KERNEL); if (!HFS_SB(sb)->bitmap) - goto out; + return -EIO; /* read in the bitmap */ block = be16_to_cpu(mdb->drVBMSt) + part_start; @@ -185,7 +188,7 @@ int hfs_mdb_get(struct super_block *sb) bh = sb_bread(sb, off >> sb->s_blocksize_bits); if (!bh) { pr_err("unable to read volume bitmap\n"); - goto out; + return -EIO; } off2 = off & (sb->s_blocksize - 1); len = min((int)sb->s_blocksize - off2, size); @@ -199,12 +202,12 @@ int hfs_mdb_get(struct super_block *sb) HFS_SB(sb)->ext_tree = hfs_btree_open(sb, HFS_EXT_CNID, hfs_ext_keycmp); if (!HFS_SB(sb)->ext_tree) { pr_err("unable to open extent tree\n"); - goto out; + return -EIO; } HFS_SB(sb)->cat_tree = hfs_btree_open(sb, HFS_CAT_CNID, hfs_cat_keycmp); if (!HFS_SB(sb)->cat_tree) { pr_err("unable to open catalog tree\n"); - goto out; + return -EIO; } attrib = mdb->drAtrb; @@ -229,12 +232,6 @@ int hfs_mdb_get(struct super_block *sb) } return 0; - -out_bh: - brelse(bh); -out: - hfs_mdb_put(sb); - return -EIO; } /* @@ -359,8 +356,6 @@ void hfs_mdb_close(struct super_block *sb) * Release the resources associated with the in-core MDB. */ void hfs_mdb_put(struct super_block *sb) { - if (!HFS_SB(sb)) - return; /* free the B-trees */ hfs_btree_close(HFS_SB(sb)->ext_tree); hfs_btree_close(HFS_SB(sb)->cat_tree); @@ -373,6 +368,4 @@ void hfs_mdb_put(struct super_block *sb) unload_nls(HFS_SB(sb)->nls_disk); kfree(HFS_SB(sb)->bitmap); - kfree(HFS_SB(sb)); - sb->s_fs_info = NULL; } diff --git a/fs/hfs/super.c b/fs/hfs/super.c index 47f50fa555a457..df289cbdd4e85b 100644 --- a/fs/hfs/super.c +++ b/fs/hfs/super.c @@ -431,10 +431,18 @@ static int hfs_init_fs_context(struct fs_context *fc) return 0; } +static void hfs_kill_super(struct super_block *sb) +{ + struct hfs_sb_info *hsb = HFS_SB(sb); + + kill_block_super(sb); + kfree(hsb); +} + static struct file_system_type hfs_fs_type = { .owner = THIS_MODULE, .name = "hfs", - .kill_sb = kill_block_super, + .kill_sb = hfs_kill_super, .fs_flags = FS_REQUIRES_DEV, .init_fs_context = hfs_init_fs_context, }; diff --git a/fs/hfsplus/bnode.c b/fs/hfsplus/bnode.c index 191661af967789..250a226336ea7a 100644 --- a/fs/hfsplus/bnode.c +++ b/fs/hfsplus/bnode.c @@ -629,7 +629,7 @@ struct hfs_bnode *hfs_bnode_create(struct hfs_btree *tree, u32 num) if (node) { pr_crit("new node %u already hashed?\n", num); WARN_ON(1); - return node; + return ERR_PTR(-EEXIST); } node = __hfs_bnode_create(tree, num); if (!node) diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index 6beb876658c09b..e3bedcbb5f1ead 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -409,8 +409,6 @@ static void iomap_read_init(struct folio *folio) struct iomap_folio_state *ifs = folio->private; if (ifs) { - size_t len = folio_size(folio); - /* * ifs->read_bytes_pending is used to track how many bytes are * read in asynchronously by the IO helper. We need to track @@ -418,23 +416,19 @@ static void iomap_read_init(struct folio *folio) * reading in all the necessary ranges of the folio and can end * the read. * - * Increase ->read_bytes_pending by the folio size to start, and - * add a +1 bias. We'll subtract the bias and any uptodate / - * zeroed ranges that did not require IO in iomap_read_end() - * after we're done processing the folio. + * Increase ->read_bytes_pending by the folio size to start. + * We'll subtract any uptodate / zeroed ranges that did not + * require IO in iomap_read_end() after we're done processing + * the folio. * * We do this because otherwise, we would have to increment * ifs->read_bytes_pending every time a range in the folio needs * to be read in, which can get expensive since the spinlock * needs to be held whenever modifying ifs->read_bytes_pending. - * - * We add the bias to ensure the read has not been ended on the - * folio when iomap_read_end() is called, even if the IO helper - * has already finished reading in the entire folio. */ spin_lock_irq(&ifs->state_lock); WARN_ON_ONCE(ifs->read_bytes_pending != 0); - ifs->read_bytes_pending = len + 1; + ifs->read_bytes_pending = folio_size(folio); spin_unlock_irq(&ifs->state_lock); } } @@ -465,11 +459,9 @@ static void iomap_read_end(struct folio *folio, size_t bytes_submitted) /* * Subtract any bytes that were initially accounted to - * read_bytes_pending but skipped for IO. The +1 accounts for - * the bias we added in iomap_read_init(). + * read_bytes_pending but skipped for IO. */ - ifs->read_bytes_pending -= - (folio_size(folio) + 1 - bytes_submitted); + ifs->read_bytes_pending -= folio_size(folio) - bytes_submitted; /* * If !ifs->read_bytes_pending, this means all pending reads by @@ -483,14 +475,16 @@ static void iomap_read_end(struct folio *folio, size_t bytes_submitted) spin_unlock_irq(&ifs->state_lock); if (end_read) folio_end_read(folio, uptodate); - } else if (!bytes_submitted) { + } else { /* - * If there were no bytes submitted, this means we are - * responsible for unlocking the folio here, since no IO helper - * has taken ownership of it. If there were bytes submitted, - * then the IO helper will end the read via - * iomap_finish_folio_read(). + * If a folio without an ifs is submitted to the IO helper, the + * read must be on the entire folio and the IO helper takes + * ownership of the folio. This means we should only enter + * iomap_read_end() for the !ifs case if no bytes were submitted + * to the IO helper, in which case we are responsible for + * unlocking the folio here. */ + WARN_ON_ONCE(bytes_submitted); folio_unlock(folio); } } @@ -502,6 +496,7 @@ static int iomap_read_folio_iter(struct iomap_iter *iter, loff_t pos = iter->pos; loff_t length = iomap_length(iter); struct folio *folio = ctx->cur_folio; + size_t folio_len = folio_size(folio); size_t poff, plen; loff_t pos_diff; int ret; @@ -515,8 +510,7 @@ static int iomap_read_folio_iter(struct iomap_iter *iter, ifs_alloc(iter->inode, folio, iter->flags); - length = min_t(loff_t, length, - folio_size(folio) - offset_in_folio(folio, pos)); + length = min_t(loff_t, length, folio_len - offset_in_folio(folio, pos)); while (length) { iomap_adjust_read_range(iter->inode, folio, &pos, length, &poff, &plen); @@ -542,7 +536,15 @@ static int iomap_read_folio_iter(struct iomap_iter *iter, ret = ctx->ops->read_folio_range(iter, ctx, plen); if (ret) return ret; + *bytes_submitted += plen; + /* + * If the entire folio has been read in by the IO + * helper, then the helper owns the folio and will end + * the read on it. + */ + if (*bytes_submitted == folio_len) + ctx->cur_folio = NULL; } ret = iomap_iter_advance(iter, plen); @@ -575,7 +577,8 @@ void iomap_read_folio(const struct iomap_ops *ops, if (ctx->ops->submit_read) ctx->ops->submit_read(ctx); - iomap_read_end(folio, bytes_submitted); + if (ctx->cur_folio) + iomap_read_end(ctx->cur_folio, bytes_submitted); } EXPORT_SYMBOL_GPL(iomap_read_folio); diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index 8e273408453a9c..6ec4940e019c64 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -442,9 +442,13 @@ static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio) nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter, BIO_MAX_VECS); do { size_t n; - if (dio->error) { - iov_iter_revert(dio->submit.iter, copied); - copied = ret = 0; + + /* + * If completions already occurred and reported errors, give up now and + * don't bother submitting more bios. + */ + if (unlikely(data_race(dio->error))) { + ret = 0; goto out; } diff --git a/fs/jfs/jfs_dtree.c b/fs/jfs/jfs_dtree.c index 0ab83bb7bbdf9a..9ab3f2fc61d176 100644 --- a/fs/jfs/jfs_dtree.c +++ b/fs/jfs/jfs_dtree.c @@ -2903,7 +2903,7 @@ int jfs_readdir(struct file *file, struct dir_context *ctx) stbl = DT_GETSTBL(p); for (i = index; i < p->header.nextindex; i++) { - if (stbl[i] < 0 || stbl[i] >= DTPAGEMAXSLOT) { + if (stbl[i] < 0) { jfs_err("JFS: Invalid stbl[%d] = %d for inode %ld, block = %lld", i, stbl[i], (long)ip->i_ino, (long long)bn); free_page(dirent_buf); @@ -3108,7 +3108,7 @@ static int dtReadFirst(struct inode *ip, struct btstack * btstack) /* get the leftmost entry */ stbl = DT_GETSTBL(p); - if (stbl[0] < 0 || stbl[0] >= DTPAGEMAXSLOT) { + if (stbl[0] < 0) { DT_PUTPAGE(mp); jfs_error(ip->i_sb, "stbl[0] out of bound\n"); return -EIO; diff --git a/fs/netfs/write_retry.c b/fs/netfs/write_retry.c index fc9c3e0d34d813..29489a23a22093 100644 --- a/fs/netfs/write_retry.c +++ b/fs/netfs/write_retry.c @@ -98,7 +98,6 @@ static void netfs_retry_write_stream(struct netfs_io_request *wreq, subreq->start = start; subreq->len = len; __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); - subreq->retry_count++; trace_netfs_sreq(subreq, netfs_sreq_trace_retry); /* Renegotiate max_len (wsize) */ diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index e3654f4a1b9aab..571bb40880b2d5 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -73,7 +73,7 @@ const struct address_space_operations nfs_dir_aops = { .free_folio = nfs_readdir_clear_array, }; -#define NFS_INIT_DTSIZE PAGE_SIZE +#define NFS_INIT_DTSIZE SZ_64K static struct nfs_open_dir_context * alloc_nfs_open_dir_context(struct inode *dir) @@ -84,7 +84,7 @@ alloc_nfs_open_dir_context(struct inode *dir) ctx = kzalloc(sizeof(*ctx), GFP_KERNEL_ACCOUNT); if (ctx != NULL) { ctx->attr_gencount = nfsi->attr_gencount; - ctx->dtsize = NFS_INIT_DTSIZE; + ctx->dtsize = min(NFS_SERVER(dir)->dtsize, NFS_INIT_DTSIZE); spin_lock(&dir->i_lock); if (list_empty(&nfsi->open_files) && (nfsi->cache_validity & NFS_INO_DATA_INVAL_DEFER)) diff --git a/fs/nfs/localio.c b/fs/nfs/localio.c index 41fbcb3f9167e9..3bd4f28c8afc26 100644 --- a/fs/nfs/localio.c +++ b/fs/nfs/localio.c @@ -58,6 +58,11 @@ struct nfs_local_fsync_ctx { static bool localio_enabled __read_mostly = true; module_param(localio_enabled, bool, 0644); +static int nfs_local_do_read(struct nfs_local_kiocb *iocb, + const struct rpc_call_ops *call_ops); +static int nfs_local_do_write(struct nfs_local_kiocb *iocb, + const struct rpc_call_ops *call_ops); + static inline bool nfs_client_is_local(const struct nfs_client *clp) { return !!rcu_access_pointer(clp->cl_uuid.net); @@ -286,6 +291,18 @@ nfs_local_open_fh(struct nfs_client *clp, const struct cred *cred, } EXPORT_SYMBOL_GPL(nfs_local_open_fh); +/* + * Ensure all page cache allocations are done from GFP_NOFS context to + * prevent direct reclaim recursion back into NFS via nfs_writepages. + */ +static void +nfs_local_mapping_set_gfp_nofs_context(struct address_space *m) +{ + gfp_t gfp_mask = mapping_gfp_mask(m); + + mapping_set_gfp_mask(m, (gfp_mask & ~(__GFP_FS))); +} + static void nfs_local_iocb_free(struct nfs_local_kiocb *iocb) { @@ -310,6 +327,7 @@ nfs_local_iocb_alloc(struct nfs_pgio_header *hdr, return NULL; } + nfs_local_mapping_set_gfp_nofs_context(file->f_mapping); init_sync_kiocb(&iocb->kiocb, file); iocb->hdr = hdr; @@ -542,13 +560,50 @@ nfs_local_iocb_release(struct nfs_local_kiocb *iocb) nfs_local_iocb_free(iocb); } -static void -nfs_local_pgio_release(struct nfs_local_kiocb *iocb) +static void nfs_local_pgio_restart(struct nfs_local_kiocb *iocb, + struct nfs_pgio_header *hdr) +{ + int status = 0; + + iocb->kiocb.ki_pos = hdr->args.offset; + iocb->kiocb.ki_flags &= ~(IOCB_DSYNC | IOCB_SYNC | IOCB_DIRECT); + iocb->kiocb.ki_complete = NULL; + iocb->aio_complete_work = NULL; + iocb->end_iter_index = -1; + + switch (hdr->rw_mode) { + case FMODE_READ: + nfs_local_iters_init(iocb, ITER_DEST); + status = nfs_local_do_read(iocb, hdr->task.tk_ops); + break; + case FMODE_WRITE: + nfs_local_iters_init(iocb, ITER_SOURCE); + status = nfs_local_do_write(iocb, hdr->task.tk_ops); + break; + default: + status = -EOPNOTSUPP; + } + + if (status != 0) { + nfs_local_iocb_release(iocb); + hdr->task.tk_status = status; + nfs_local_hdr_release(hdr, hdr->task.tk_ops); + } +} + +static void nfs_local_pgio_release(struct nfs_local_kiocb *iocb) { struct nfs_pgio_header *hdr = iocb->hdr; + struct rpc_task *task = &hdr->task; + + task->tk_action = NULL; + task->tk_ops->rpc_call_done(task, hdr); - nfs_local_iocb_release(iocb); - nfs_local_hdr_release(hdr, hdr->task.tk_ops); + if (task->tk_action == NULL) { + nfs_local_iocb_release(iocb); + task->tk_ops->rpc_release(hdr); + } else + nfs_local_pgio_restart(iocb, hdr); } /* @@ -773,19 +828,7 @@ static void nfs_local_write_done(struct nfs_local_kiocb *iocb) pr_info_ratelimited("nfs: Unexpected direct I/O write alignment failure\n"); } - /* Handle short writes as if they are ENOSPC */ - status = hdr->res.count; - if (status > 0 && status < hdr->args.count) { - hdr->mds_offset += status; - hdr->args.offset += status; - hdr->args.pgbase += status; - hdr->args.count -= status; - nfs_set_pgio_error(hdr, -ENOSPC, hdr->args.offset); - status = -ENOSPC; - /* record -ENOSPC in terms of nfs_local_pgio_done */ - (void) nfs_local_pgio_done(iocb, status, true); - } - if (hdr->task.tk_status < 0) + if (status < 0) nfs_reset_boot_verifier(hdr->inode); } @@ -946,8 +989,6 @@ int nfs_local_doio(struct nfs_client *clp, struct nfsd_file *localio, } if (status != 0) { - if (status == -EAGAIN) - nfs_localio_disable_client(clp); nfs_local_iocb_release(iocb); hdr->task.tk_status = status; nfs_local_hdr_release(hdr, call_ops); @@ -974,6 +1015,8 @@ nfs_local_run_commit(struct file *filp, struct nfs_commit_data *data) end = LLONG_MAX; } + nfs_local_mapping_set_gfp_nofs_context(filp->f_mapping); + dprintk("%s: commit %llu - %llu\n", __func__, start, end); return vfs_fsync_range(filp, start, end, 0); } @@ -1015,17 +1058,22 @@ nfs_local_fsync_ctx_free(struct nfs_local_fsync_ctx *ctx) static void nfs_local_fsync_work(struct work_struct *work) { + unsigned long old_flags = current->flags; struct nfs_local_fsync_ctx *ctx; int status; ctx = container_of(work, struct nfs_local_fsync_ctx, work); + current->flags |= PF_LOCAL_THROTTLE | PF_MEMALLOC_NOIO; + status = nfs_local_run_commit(nfs_to->nfsd_file_file(ctx->localio), ctx->data); nfs_local_commit_done(ctx->data, status); if (ctx->done != NULL) complete(ctx->done); nfs_local_fsync_ctx_free(ctx); + + current->flags = old_flags; } static struct nfs_local_fsync_ctx * @@ -1049,7 +1097,7 @@ int nfs_local_commit(struct nfsd_file *localio, { struct nfs_local_fsync_ctx *ctx; - ctx = nfs_local_fsync_ctx_alloc(data, localio, GFP_KERNEL); + ctx = nfs_local_fsync_ctx_alloc(data, localio, GFP_NOIO); if (!ctx) { nfs_local_commit_done(data, -ENOMEM); nfs_local_release_commit_data(localio, data, call_ops); @@ -1061,10 +1109,10 @@ int nfs_local_commit(struct nfsd_file *localio, if (how & FLUSH_SYNC) { DECLARE_COMPLETION_ONSTACK(done); ctx->done = &done; - queue_work(nfsiod_workqueue, &ctx->work); + queue_work(nfslocaliod_workqueue, &ctx->work); wait_for_completion(&done); } else - queue_work(nfsiod_workqueue, &ctx->work); + queue_work(nfslocaliod_workqueue, &ctx->work); return 0; } diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index cff225721d1ce5..ff8483d3373a8b 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -463,7 +463,8 @@ pnfs_mark_layout_stateid_invalid(struct pnfs_layout_hdr *lo, }; struct pnfs_layout_segment *lseg, *next; - set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); + if (test_and_set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) + return !list_empty(&lo->plh_segs); clear_bit(NFS_INO_LAYOUTCOMMIT, &NFS_I(lo->plh_inode)->flags); list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) pnfs_clear_lseg_state(lseg, lseg_list); diff --git a/fs/nfsd/export.c b/fs/nfsd/export.c index 2a1499f2ad196a..09fe268fe2c766 100644 --- a/fs/nfsd/export.c +++ b/fs/nfsd/export.c @@ -427,7 +427,8 @@ static int check_export(const struct path *path, int *flags, unsigned char *uuid * either a device number (so FS_REQUIRES_DEV needed) * or an FSID number (so NFSEXP_FSID or ->uuid is needed). * 2: We must be able to find an inode from a filehandle. - * This means that s_export_op must be set. + * This means that s_export_op must be set and comply with + * the requirements for remote filesystem export. * 3: We must not currently be on an idmapped mount. */ if (!(inode->i_sb->s_type->fs_flags & FS_REQUIRES_DEV) && @@ -437,8 +438,9 @@ static int check_export(const struct path *path, int *flags, unsigned char *uuid return -EINVAL; } - if (!exportfs_can_decode_fh(inode->i_sb->s_export_op)) { - dprintk("exp_export: export of invalid fs type.\n"); + if (!exportfs_may_export(inode->i_sb->s_export_op)) { + dprintk("exp_export: export of invalid fs type (%s).\n", + inode->i_sb->s_type->name); return -EINVAL; } diff --git a/fs/nfsd/nfs2acl.c b/fs/nfsd/nfs2acl.c index 5fb202acb0fd00..0ac538c7618009 100644 --- a/fs/nfsd/nfs2acl.c +++ b/fs/nfsd/nfs2acl.c @@ -45,7 +45,7 @@ static __be32 nfsacld_proc_getacl(struct svc_rqst *rqstp) inode = d_inode(fh->fh_dentry); if (argp->mask & ~NFS_ACL_MASK) { - resp->status = nfserr_inval; + resp->status = nfserr_io; goto out; } resp->mask = argp->mask; diff --git a/fs/nfsd/nfs4idmap.c b/fs/nfsd/nfs4idmap.c index 8cca1329f3485c..b5b3d45979c9b3 100644 --- a/fs/nfsd/nfs4idmap.c +++ b/fs/nfsd/nfs4idmap.c @@ -643,13 +643,31 @@ static __be32 encode_name_from_id(struct xdr_stream *xdr, return idmap_id_to_name(xdr, rqstp, type, id); } -__be32 -nfsd_map_name_to_uid(struct svc_rqst *rqstp, const char *name, size_t namelen, - kuid_t *uid) +/** + * nfsd_map_name_to_uid - Map user@domain to local UID + * @rqstp: RPC execution context + * @name: user@domain name to be mapped + * @namelen: length of name, in bytes + * @uid: OUT: mapped local UID value + * + * Returns nfs_ok on success or an NFSv4 status code on failure. + */ +__be32 nfsd_map_name_to_uid(struct svc_rqst *rqstp, const char *name, + size_t namelen, kuid_t *uid) { __be32 status; u32 id = -1; + /* + * The idmap lookup below triggers an upcall that invokes + * cache_check(). RQ_USEDEFERRAL must be clear to prevent + * cache_check() from setting RQ_DROPME via svc_defer(). + * NFSv4 servers are not permitted to drop requests. Also + * RQ_DROPME will force NFSv4.1 session slot processing to + * be skipped. + */ + WARN_ON_ONCE(test_bit(RQ_USEDEFERRAL, &rqstp->rq_flags)); + if (name == NULL || namelen == 0) return nfserr_inval; @@ -660,13 +678,31 @@ nfsd_map_name_to_uid(struct svc_rqst *rqstp, const char *name, size_t namelen, return status; } -__be32 -nfsd_map_name_to_gid(struct svc_rqst *rqstp, const char *name, size_t namelen, - kgid_t *gid) +/** + * nfsd_map_name_to_gid - Map user@domain to local GID + * @rqstp: RPC execution context + * @name: user@domain name to be mapped + * @namelen: length of name, in bytes + * @gid: OUT: mapped local GID value + * + * Returns nfs_ok on success or an NFSv4 status code on failure. + */ +__be32 nfsd_map_name_to_gid(struct svc_rqst *rqstp, const char *name, + size_t namelen, kgid_t *gid) { __be32 status; u32 id = -1; + /* + * The idmap lookup below triggers an upcall that invokes + * cache_check(). RQ_USEDEFERRAL must be clear to prevent + * cache_check() from setting RQ_DROPME via svc_defer(). + * NFSv4 servers are not permitted to drop requests. Also + * RQ_DROPME will force NFSv4.1 session slot processing to + * be skipped. + */ + WARN_ON_ONCE(test_bit(RQ_USEDEFERRAL, &rqstp->rq_flags)); + if (name == NULL || namelen == 0) return nfserr_inval; diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index 9ec08dd4fe823b..f780024f9a0887 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -3011,8 +3011,6 @@ nfsd4_proc_compound(struct svc_rqst *rqstp) BUG_ON(cstate->replay_owner); out: cstate->status = status; - /* Reset deferral mechanism for RPC deferrals */ - set_bit(RQ_USEDEFERRAL, &rqstp->rq_flags); return rpc_success; } diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c index 51ef97c2545688..5065727204b958 100644 --- a/fs/nfsd/nfs4xdr.c +++ b/fs/nfsd/nfs4xdr.c @@ -6013,6 +6013,22 @@ nfs4svc_decode_compoundargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) args->ops = args->iops; args->rqstp = rqstp; + /* + * NFSv4 operation decoders can invoke svc cache lookups + * that trigger svc_defer() when RQ_USEDEFERRAL is set, + * setting RQ_DROPME. This creates two problems: + * + * 1. Non-idempotency: Compounds make it too hard to avoid + * problems if a request is deferred and replayed. + * + * 2. Session slot leakage (NFSv4.1+): If RQ_DROPME is set + * during decode but SEQUENCE executes successfully, the + * session slot will be marked INUSE. The request is then + * dropped before encoding, so the slot is never released, + * rendering it permanently unusable by the client. + */ + clear_bit(RQ_USEDEFERRAL, &rqstp->rq_flags); + return nfsd4_decode_compound(args); } diff --git a/fs/nfsd/nfsproc.c b/fs/nfsd/nfsproc.c index 481e789a769749..8873033d1e82f4 100644 --- a/fs/nfsd/nfsproc.c +++ b/fs/nfsd/nfsproc.c @@ -33,7 +33,7 @@ static __be32 nfsd_map_status(__be32 status) break; case nfserr_symlink: case nfserr_wrong_type: - status = nfserr_inval; + status = nfserr_io; break; } return status; diff --git a/fs/nilfs2/sufile.c b/fs/nilfs2/sufile.c index 83f93337c01b24..eceedca0269725 100644 --- a/fs/nilfs2/sufile.c +++ b/fs/nilfs2/sufile.c @@ -1093,6 +1093,9 @@ int nilfs_sufile_trim_fs(struct inode *sufile, struct fstrim_range *range) else end_block = start_block + len - 1; + if (end_block < nilfs->ns_first_data_block) + goto out; + segnum = nilfs_get_segnum_of_block(nilfs, start_block); segnum_end = nilfs_get_segnum_of_block(nilfs, end_block); @@ -1191,6 +1194,7 @@ int nilfs_sufile_trim_fs(struct inode *sufile, struct fstrim_range *range) out_sem: up_read(&NILFS_MDT(sufile)->mi_sem); +out: range->len = ndiscarded << nilfs->ns_blocksize_bits; return ret; } diff --git a/fs/ntfs3/file.c b/fs/ntfs3/file.c index 2e7b2e566ebe18..5120bd78516946 100644 --- a/fs/ntfs3/file.c +++ b/fs/ntfs3/file.c @@ -995,7 +995,7 @@ static int ntfs_get_frame_pages(struct address_space *mapping, pgoff_t index, folio = __filemap_get_folio(mapping, index, FGP_LOCK | FGP_ACCESSED | FGP_CREAT, - gfp_mask); + gfp_mask | __GFP_ZERO); if (IS_ERR(folio)) { while (npages--) { folio = page_folio(pages[npages]); @@ -1077,8 +1077,12 @@ static ssize_t ntfs_compress_write(struct kiocb *iocb, struct iov_iter *from) goto out; if (lcn == SPARSE_LCN) { - ni->i_valid = valid = - frame_vbo + ((u64)clen << sbi->cluster_bits); + valid = frame_vbo + ((u64)clen << sbi->cluster_bits); + if (ni->i_valid == valid) { + err = -EINVAL; + goto out; + } + ni->i_valid = valid; continue; } diff --git a/fs/ntfs3/frecord.c b/fs/ntfs3/frecord.c index 641ddaf8d4a07e..d5bbd47e1ee9db 100644 --- a/fs/ntfs3/frecord.c +++ b/fs/ntfs3/frecord.c @@ -2046,18 +2046,18 @@ static struct page *ntfs_lock_new_page(struct address_space *mapping, } /* - * ni_readpage_cmpr + * ni_read_folio_cmpr * * When decompressing, we typically obtain more than one page per reference. * We inject the additional pages into the page cache. */ -int ni_readpage_cmpr(struct ntfs_inode *ni, struct folio *folio) +int ni_read_folio_cmpr(struct ntfs_inode *ni, struct folio *folio) { int err; struct ntfs_sb_info *sbi = ni->mi.sbi; struct address_space *mapping = folio->mapping; - pgoff_t index = folio->index; - u64 frame_vbo, vbo = (u64)index << PAGE_SHIFT; + pgoff_t index; + u64 frame_vbo, vbo = folio_pos(folio); struct page **pages = NULL; /* Array of at most 16 pages. stack? */ u8 frame_bits; CLST frame; @@ -2107,7 +2107,9 @@ int ni_readpage_cmpr(struct ntfs_inode *ni, struct folio *folio) pages[i] = pg; } + ni_lock(ni); err = ni_read_frame(ni, frame_vbo, pages, pages_per_frame, 0); + ni_unlock(ni); out1: for (i = 0; i < pages_per_frame; i++) { diff --git a/fs/ntfs3/fslog.c b/fs/ntfs3/fslog.c index 38934e6978ecec..28bd611f580d9c 100644 --- a/fs/ntfs3/fslog.c +++ b/fs/ntfs3/fslog.c @@ -3429,6 +3429,9 @@ static int do_action(struct ntfs_log *log, struct OPEN_ATTR_ENRTY *oe, e1 = Add2Ptr(attr, le16_to_cpu(lrh->attr_off)); esize = le16_to_cpu(e1->size); + if (PtrOffset(e1, Add2Ptr(hdr, used)) < esize) + goto dirty_vol; + e2 = Add2Ptr(e1, esize); memmove(e1, e2, PtrOffset(e2, Add2Ptr(hdr, used))); diff --git a/fs/ntfs3/inode.c b/fs/ntfs3/inode.c index 0a9ac5efeb67c2..ec8e954f4426c0 100644 --- a/fs/ntfs3/inode.c +++ b/fs/ntfs3/inode.c @@ -735,9 +735,8 @@ static int ntfs_read_folio(struct file *file, struct folio *folio) } if (is_compressed(ni)) { - ni_lock(ni); - err = ni_readpage_cmpr(ni, folio); - ni_unlock(ni); + /* ni_lock is taken inside ni_read_folio_cmpr after page locks */ + err = ni_read_folio_cmpr(ni, folio); return err; } diff --git a/fs/ntfs3/ntfs_fs.h b/fs/ntfs3/ntfs_fs.h index a4559c9f64e686..7b619bb151ce9a 100644 --- a/fs/ntfs3/ntfs_fs.h +++ b/fs/ntfs3/ntfs_fs.h @@ -568,7 +568,7 @@ int ni_write_inode(struct inode *inode, int sync, const char *hint); #define _ni_write_inode(i, w) ni_write_inode(i, w, __func__) int ni_fiemap(struct ntfs_inode *ni, struct fiemap_extent_info *fieinfo, __u64 vbo, __u64 len); -int ni_readpage_cmpr(struct ntfs_inode *ni, struct folio *folio); +int ni_read_folio_cmpr(struct ntfs_inode *ni, struct folio *folio); int ni_decompress_file(struct ntfs_inode *ni); int ni_read_frame(struct ntfs_inode *ni, u64 frame_vbo, struct page **pages, u32 pages_per_frame, int copy); diff --git a/fs/ntfs3/super.c b/fs/ntfs3/super.c index 8b0cf0ed4f72cc..0567a3b224ed3d 100644 --- a/fs/ntfs3/super.c +++ b/fs/ntfs3/super.c @@ -705,9 +705,7 @@ static void ntfs_put_super(struct super_block *sb) ntfs_set_state(sbi, NTFS_DIRTY_CLEAR); if (sbi->options) { - unload_nls(sbi->options->nls); - kfree(sbi->options->nls_name); - kfree(sbi->options); + put_mount_options(sbi->options); sbi->options = NULL; } @@ -1253,7 +1251,6 @@ static int ntfs_fill_super(struct super_block *sb, struct fs_context *fc) } } sbi->options = options; - fc->fs_private = NULL; sb->s_flags |= SB_NODIRATIME; sb->s_magic = 0x7366746e; // "ntfs" sb->s_op = &ntfs_sops; @@ -1679,9 +1676,7 @@ static int ntfs_fill_super(struct super_block *sb, struct fs_context *fc) out: /* sbi->options == options */ if (options) { - unload_nls(options->nls); - kfree(options->nls_name); - kfree(options); + put_mount_options(sbi->options); sbi->options = NULL; } diff --git a/fs/overlayfs/readdir.c b/fs/overlayfs/readdir.c index 160960bb0ad0b0..724ec9d93fc829 100644 --- a/fs/overlayfs/readdir.c +++ b/fs/overlayfs/readdir.c @@ -755,7 +755,7 @@ static bool ovl_fill_real(struct dir_context *ctx, const char *name, struct dir_context *orig_ctx = rdt->orig_ctx; bool res; - if (rdt->parent_ino && strcmp(name, "..") == 0) { + if (rdt->parent_ino && namelen == 2 && !strncmp(name, "..", 2)) { ino = rdt->parent_ino; } else if (rdt->cache) { struct ovl_cache_entry *p; diff --git a/fs/pidfs.c b/fs/pidfs.c index 1e20e36e0ed559..d18c51513f6c51 100644 --- a/fs/pidfs.c +++ b/fs/pidfs.c @@ -329,7 +329,7 @@ static long pidfd_info(struct file *file, unsigned int cmd, unsigned long arg) * namespace hierarchy. */ if (!pid_in_current_pidns(pid)) - return -ESRCH; + return -EREMOTE; attr = READ_ONCE(pid->attr); if (mask & PIDFD_INFO_EXIT) { diff --git a/fs/proc/array.c b/fs/proc/array.c index 42932f88141a97..5571177e0435d0 100644 --- a/fs/proc/array.c +++ b/fs/proc/array.c @@ -528,7 +528,7 @@ static int do_task_stat(struct seq_file *m, struct pid_namespace *ns, } sid = task_session_nr_ns(task, ns); - ppid = task_tgid_nr_ns(task->real_parent, ns); + ppid = task_ppid_nr_ns(task, ns); pgid = task_pgrp_nr_ns(task, ns); unlock_task_sighand(task, &flags); diff --git a/fs/pstore/ram_core.c b/fs/pstore/ram_core.c index f1848cdd6d3485..c9eaacdec37e48 100644 --- a/fs/pstore/ram_core.c +++ b/fs/pstore/ram_core.c @@ -298,6 +298,17 @@ void persistent_ram_save_old(struct persistent_ram_zone *prz) if (!size) return; + /* + * If the existing buffer is differently sized, free it so a new + * one is allocated. This can happen when persistent_ram_save_old() + * is called early in boot and later for a timer-triggered + * survivable crash when the crash dumps don't match in size + * (which would be extremely unlikely given kmsg buffers usually + * exceed prz buffer sizes). + */ + if (prz->old_log && prz->old_log_size != size) + persistent_ram_free_old(prz); + if (!prz->old_log) { persistent_ram_ecc_old(prz); prz->old_log = kvzalloc(size, GFP_KERNEL); diff --git a/fs/quota/quota.c b/fs/quota/quota.c index 7c2b75a4448528..de4379a9c79208 100644 --- a/fs/quota/quota.c +++ b/fs/quota/quota.c @@ -899,6 +899,7 @@ static struct super_block *quotactl_block(const char __user *special, int cmd) sb_start_write(sb); sb_end_write(sb); put_super(sb); + cond_resched(); goto retry; } return sb; diff --git a/fs/smb/client/cached_dir.h b/fs/smb/client/cached_dir.h index 1e383db7c33743..5091bf45345e81 100644 --- a/fs/smb/client/cached_dir.h +++ b/fs/smb/client/cached_dir.h @@ -36,10 +36,10 @@ struct cached_fid { struct list_head entry; struct cached_fids *cfids; const char *path; - bool has_lease:1; - bool is_open:1; - bool on_list:1; - bool file_all_info_is_valid:1; + bool has_lease; + bool is_open; + bool on_list; + bool file_all_info_is_valid; unsigned long time; /* jiffies of when lease was taken */ unsigned long last_access_time; /* jiffies of when last accessed */ struct kref refcount; diff --git a/fs/smb/client/fs_context.c b/fs/smb/client/fs_context.c index d4291d3a9a4858..2527d2d29f190d 100644 --- a/fs/smb/client/fs_context.c +++ b/fs/smb/client/fs_context.c @@ -826,9 +826,7 @@ static int smb3_fs_context_parse_monolithic(struct fs_context *fc, if (ret < 0) break; } - ret = smb3_handle_conflicting_options(fc); - - return ret; + return ret ?: smb3_handle_conflicting_options(fc); } /* diff --git a/fs/smb/client/smb2file.c b/fs/smb/client/smb2file.c index 2dd08388ea8733..1f7f284a784496 100644 --- a/fs/smb/client/smb2file.c +++ b/fs/smb/client/smb2file.c @@ -179,6 +179,8 @@ int smb2_open_file(const unsigned int xid, struct cifs_open_parms *oparms, &err_buftype); if (rc == -EACCES && retry_without_read_attributes) { free_rsp_buf(err_buftype, err_iov.iov_base); + memset(&err_iov, 0, sizeof(err_iov)); + err_buftype = CIFS_NO_BUFFER; oparms->desired_access &= ~FILE_READ_ATTRIBUTES; rc = SMB2_open(xid, oparms, smb2_path, &smb2_oplock, smb2_data, NULL, &err_iov, &err_buftype); diff --git a/fs/smb/client/smbdirect.c b/fs/smb/client/smbdirect.c index 788a0670c4a8da..c8cef098d48065 100644 --- a/fs/smb/client/smbdirect.c +++ b/fs/smb/client/smbdirect.c @@ -35,6 +35,10 @@ static void enqueue_reassembly( static struct smbdirect_recv_io *_get_first_reassembly( struct smbdirect_socket *sc); +static int smbd_post_send(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch, + struct smbdirect_send_io *request); + static int smbd_post_recv( struct smbdirect_socket *sc, struct smbdirect_recv_io *response); @@ -97,8 +101,23 @@ int smbd_send_credit_target = 255; /* The maximum single message size can be sent to remote peer */ int smbd_max_send_size = 1364; -/* The maximum fragmented upper-layer payload receive size supported */ -int smbd_max_fragmented_recv_size = 1024 * 1024; +/* + * The maximum fragmented upper-layer payload receive size supported + * + * Assume max_payload_per_credit is + * smbd_max_receive_size - 24 = 1340 + * + * The maximum number would be + * smbd_receive_credit_max * max_payload_per_credit + * + * 1340 * 255 = 341700 (0x536C4) + * + * The minimum value from the spec is 131072 (0x20000) + * + * For now we use the logic we used in ksmbd before: + * (1364 * 255) / 2 = 173910 (0x2A756) + */ +int smbd_max_fragmented_recv_size = (1364 * 255) / 2; /* The maximum single-message size which can be received */ int smbd_max_receive_size = 1364; @@ -493,27 +512,103 @@ static inline void *smbdirect_recv_io_payload(struct smbdirect_recv_io *response return (void *)response->packet; } +static struct smbdirect_send_io *smbd_alloc_send_io(struct smbdirect_socket *sc) +{ + struct smbdirect_send_io *msg; + + msg = mempool_alloc(sc->send_io.mem.pool, GFP_KERNEL); + if (!msg) + return ERR_PTR(-ENOMEM); + msg->socket = sc; + INIT_LIST_HEAD(&msg->sibling_list); + msg->num_sge = 0; + + return msg; +} + +static void smbd_free_send_io(struct smbdirect_send_io *msg) +{ + struct smbdirect_socket *sc = msg->socket; + size_t i; + + /* + * The list needs to be empty! + * The caller should take care of it. + */ + WARN_ON_ONCE(!list_empty(&msg->sibling_list)); + + /* + * Note we call ib_dma_unmap_page(), even if some sges are mapped using + * ib_dma_map_single(). + * + * The difference between _single() and _page() only matters for the + * ib_dma_map_*() case. + * + * For the ib_dma_unmap_*() case it does not matter as both take the + * dma_addr_t and dma_unmap_single_attrs() is just an alias to + * dma_unmap_page_attrs(). + */ + for (i = 0; i < msg->num_sge; i++) + ib_dma_unmap_page(sc->ib.dev, + msg->sge[i].addr, + msg->sge[i].length, + DMA_TO_DEVICE); + + mempool_free(msg, sc->send_io.mem.pool); +} + /* Called when a RDMA send is done */ static void send_done(struct ib_cq *cq, struct ib_wc *wc) { - int i; struct smbdirect_send_io *request = container_of(wc->wr_cqe, struct smbdirect_send_io, cqe); struct smbdirect_socket *sc = request->socket; + struct smbdirect_send_io *sibling, *next; int lcredits = 0; log_rdma_send(INFO, "smbdirect_send_io 0x%p completed wc->status=%s\n", request, ib_wc_status_msg(wc->status)); - for (i = 0; i < request->num_sge; i++) - ib_dma_unmap_single(sc->ib.dev, - request->sge[i].addr, - request->sge[i].length, - DMA_TO_DEVICE); - mempool_free(request, sc->send_io.mem.pool); + if (unlikely(!(request->wr.send_flags & IB_SEND_SIGNALED))) { + /* + * This happens when smbdirect_send_io is a sibling + * before the final message, it is signaled on + * error anyway, so we need to skip + * smbdirect_connection_free_send_io here, + * otherwise is will destroy the memory + * of the siblings too, which will cause + * use after free problems for the others + * triggered from ib_drain_qp(). + */ + if (wc->status != IB_WC_SUCCESS) + goto skip_free; + + /* + * This should not happen! + * But we better just close the + * connection... + */ + log_rdma_send(ERR, + "unexpected send completion wc->status=%s (%d) wc->opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + smbd_disconnect_rdma_connection(sc); + return; + } + + /* + * Free possible siblings and then the main send_io + */ + list_for_each_entry_safe(sibling, next, &request->sibling_list, sibling_list) { + list_del_init(&sibling->sibling_list); + smbd_free_send_io(sibling); + lcredits += 1; + } + /* Note this frees wc->wr_cqe, but not wc */ + smbd_free_send_io(request); lcredits += 1; if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_SEND) { +skip_free: if (wc->status != IB_WC_WR_FLUSH_ERR) log_rdma_send(ERR, "wc->status=%s wc->opcode=%d\n", ib_wc_status_msg(wc->status), wc->opcode); @@ -608,6 +703,7 @@ static bool process_negotiation_response( sp->max_frmr_depth * PAGE_SIZE); sp->max_frmr_depth = sp->max_read_write_size / PAGE_SIZE; + atomic_set(&sc->send_io.bcredits.count, 1); sc->recv_io.expected = SMBDIRECT_EXPECT_DATA_TRANSFER; return true; } @@ -618,6 +714,7 @@ static void smbd_post_send_credits(struct work_struct *work) struct smbdirect_recv_io *response; struct smbdirect_socket *sc = container_of(work, struct smbdirect_socket, recv_io.posted.refill_work); + int posted = 0; if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { return; @@ -640,9 +737,21 @@ static void smbd_post_send_credits(struct work_struct *work) } atomic_inc(&sc->recv_io.posted.count); + posted += 1; } } + atomic_add(posted, &sc->recv_io.credits.available); + + /* + * If the last send credit is waiting for credits + * it can grant we need to wake it up + */ + if (posted && + atomic_read(&sc->send_io.bcredits.count) == 0 && + atomic_read(&sc->send_io.credits.count) == 0) + wake_up(&sc->send_io.credits.wait_queue); + /* Promptly send an immediate packet as defined in [MS-SMBD] 3.1.1.1 */ if (atomic_read(&sc->recv_io.credits.count) < sc->recv_io.credits.target - 1) { @@ -659,6 +768,7 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) container_of(wc->wr_cqe, struct smbdirect_recv_io, cqe); struct smbdirect_socket *sc = response->socket; struct smbdirect_socket_parameters *sp = &sc->parameters; + int current_recv_credits; u16 old_recv_credit_target; u32 data_offset = 0; u32 data_length = 0; @@ -743,7 +853,8 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) } atomic_dec(&sc->recv_io.posted.count); - atomic_dec(&sc->recv_io.credits.count); + current_recv_credits = atomic_dec_return(&sc->recv_io.credits.count); + old_recv_credit_target = sc->recv_io.credits.target; sc->recv_io.credits.target = le16_to_cpu(data_transfer->credits_requested); @@ -779,7 +890,8 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) * reassembly queue and wake up the reading thread */ if (data_length) { - if (sc->recv_io.credits.target > old_recv_credit_target) + if (current_recv_credits <= (sc->recv_io.credits.target / 4) || + sc->recv_io.credits.target > old_recv_credit_target) queue_work(sc->workqueue, &sc->recv_io.posted.refill_work); enqueue_reassembly(sc, response, data_length); @@ -955,16 +1067,13 @@ static int smbd_ia_open( static int smbd_post_send_negotiate_req(struct smbdirect_socket *sc) { struct smbdirect_socket_parameters *sp = &sc->parameters; - struct ib_send_wr send_wr; - int rc = -ENOMEM; + int rc; struct smbdirect_send_io *request; struct smbdirect_negotiate_req *packet; - request = mempool_alloc(sc->send_io.mem.pool, GFP_KERNEL); - if (!request) - return rc; - - request->socket = sc; + request = smbd_alloc_send_io(sc); + if (IS_ERR(request)) + return PTR_ERR(request); packet = smbdirect_send_io_payload(request); packet->min_version = cpu_to_le16(SMBDIRECT_V1); @@ -976,7 +1085,6 @@ static int smbd_post_send_negotiate_req(struct smbdirect_socket *sc) packet->max_fragmented_size = cpu_to_le32(sp->max_fragmented_recv_size); - request->num_sge = 1; request->sge[0].addr = ib_dma_map_single( sc->ib.dev, (void *)packet, sizeof(*packet), DMA_TO_DEVICE); @@ -984,42 +1092,20 @@ static int smbd_post_send_negotiate_req(struct smbdirect_socket *sc) rc = -EIO; goto dma_mapping_failed; } + request->num_sge = 1; request->sge[0].length = sizeof(*packet); request->sge[0].lkey = sc->ib.pd->local_dma_lkey; - ib_dma_sync_single_for_device( - sc->ib.dev, request->sge[0].addr, - request->sge[0].length, DMA_TO_DEVICE); - - request->cqe.done = send_done; - - send_wr.next = NULL; - send_wr.wr_cqe = &request->cqe; - send_wr.sg_list = request->sge; - send_wr.num_sge = request->num_sge; - send_wr.opcode = IB_WR_SEND; - send_wr.send_flags = IB_SEND_SIGNALED; - - log_rdma_send(INFO, "sge addr=0x%llx length=%u lkey=0x%x\n", - request->sge[0].addr, - request->sge[0].length, request->sge[0].lkey); - - atomic_inc(&sc->send_io.pending.count); - rc = ib_post_send(sc->ib.qp, &send_wr, NULL); + rc = smbd_post_send(sc, NULL, request); if (!rc) return 0; - /* if we reach here, post send failed */ - log_rdma_send(ERR, "ib_post_send failed rc=%d\n", rc); - atomic_dec(&sc->send_io.pending.count); - ib_dma_unmap_single(sc->ib.dev, request->sge[0].addr, - request->sge[0].length, DMA_TO_DEVICE); - - smbd_disconnect_rdma_connection(sc); + if (rc == -EAGAIN) + rc = -EIO; dma_mapping_failed: - mempool_free(request, sc->send_io.mem.pool); + smbd_free_send_io(request); return rc; } @@ -1033,19 +1119,38 @@ static int smbd_post_send_negotiate_req(struct smbdirect_socket *sc) */ static int manage_credits_prior_sending(struct smbdirect_socket *sc) { + int missing; + int available; int new_credits; if (atomic_read(&sc->recv_io.credits.count) >= sc->recv_io.credits.target) return 0; - new_credits = atomic_read(&sc->recv_io.posted.count); - if (new_credits == 0) + missing = (int)sc->recv_io.credits.target - atomic_read(&sc->recv_io.credits.count); + available = atomic_xchg(&sc->recv_io.credits.available, 0); + new_credits = (u16)min3(U16_MAX, missing, available); + if (new_credits <= 0) { + /* + * If credits are available, but not granted + * we need to re-add them again. + */ + if (available) + atomic_add(available, &sc->recv_io.credits.available); return 0; + } - new_credits -= atomic_read(&sc->recv_io.credits.count); - if (new_credits <= 0) - return 0; + if (new_credits < available) { + /* + * Readd the remaining available again. + */ + available -= new_credits; + atomic_add(available, &sc->recv_io.credits.available); + } + /* + * Remember we granted the credits + */ + atomic_add(new_credits, &sc->recv_io.credits.count); return new_credits; } @@ -1075,12 +1180,27 @@ static int manage_keep_alive_before_sending(struct smbdirect_socket *sc) return 0; } +static int smbd_ib_post_send(struct smbdirect_socket *sc, + struct ib_send_wr *wr) +{ + int ret; + + atomic_inc(&sc->send_io.pending.count); + ret = ib_post_send(sc->ib.qp, wr, NULL); + if (ret) { + pr_err("failed to post send: %d\n", ret); + smbd_disconnect_rdma_connection(sc); + ret = -EAGAIN; + } + return ret; +} + /* Post the send request */ static int smbd_post_send(struct smbdirect_socket *sc, - struct smbdirect_send_io *request) + struct smbdirect_send_batch *batch, + struct smbdirect_send_io *request) { - struct ib_send_wr send_wr; - int rc, i; + int i; for (i = 0; i < request->num_sge; i++) { log_rdma_send(INFO, @@ -1094,79 +1214,245 @@ static int smbd_post_send(struct smbdirect_socket *sc, } request->cqe.done = send_done; + request->wr.next = NULL; + request->wr.sg_list = request->sge; + request->wr.num_sge = request->num_sge; + request->wr.opcode = IB_WR_SEND; + + if (batch) { + request->wr.wr_cqe = NULL; + request->wr.send_flags = 0; + if (!list_empty(&batch->msg_list)) { + struct smbdirect_send_io *last; + + last = list_last_entry(&batch->msg_list, + struct smbdirect_send_io, + sibling_list); + last->wr.next = &request->wr; + } + list_add_tail(&request->sibling_list, &batch->msg_list); + batch->wr_cnt++; + return 0; + } - send_wr.next = NULL; - send_wr.wr_cqe = &request->cqe; - send_wr.sg_list = request->sge; - send_wr.num_sge = request->num_sge; - send_wr.opcode = IB_WR_SEND; - send_wr.send_flags = IB_SEND_SIGNALED; + request->wr.wr_cqe = &request->cqe; + request->wr.send_flags = IB_SEND_SIGNALED; + return smbd_ib_post_send(sc, &request->wr); +} - rc = ib_post_send(sc->ib.qp, &send_wr, NULL); - if (rc) { - log_rdma_send(ERR, "ib_post_send failed rc=%d\n", rc); - smbd_disconnect_rdma_connection(sc); - rc = -EAGAIN; +static void smbd_send_batch_init(struct smbdirect_send_batch *batch, + bool need_invalidate_rkey, + unsigned int remote_key) +{ + INIT_LIST_HEAD(&batch->msg_list); + batch->wr_cnt = 0; + batch->need_invalidate_rkey = need_invalidate_rkey; + batch->remote_key = remote_key; + batch->credit = 0; +} + +static int smbd_send_batch_flush(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch, + bool is_last) +{ + struct smbdirect_send_io *first, *last; + int ret = 0; + + if (list_empty(&batch->msg_list)) + goto release_credit; + + first = list_first_entry(&batch->msg_list, + struct smbdirect_send_io, + sibling_list); + last = list_last_entry(&batch->msg_list, + struct smbdirect_send_io, + sibling_list); + + if (batch->need_invalidate_rkey) { + first->wr.opcode = IB_WR_SEND_WITH_INV; + first->wr.ex.invalidate_rkey = batch->remote_key; + batch->need_invalidate_rkey = false; + batch->remote_key = 0; } - return rc; + last->wr.send_flags = IB_SEND_SIGNALED; + last->wr.wr_cqe = &last->cqe; + + /* + * Remove last from batch->msg_list + * and splice the rest of batch->msg_list + * to last->sibling_list. + * + * batch->msg_list is a valid empty list + * at the end. + */ + list_del_init(&last->sibling_list); + list_splice_tail_init(&batch->msg_list, &last->sibling_list); + batch->wr_cnt = 0; + + ret = smbd_ib_post_send(sc, &first->wr); + if (ret) { + struct smbdirect_send_io *sibling, *next; + + list_for_each_entry_safe(sibling, next, &last->sibling_list, sibling_list) { + list_del_init(&sibling->sibling_list); + smbd_free_send_io(sibling); + } + smbd_free_send_io(last); + } + +release_credit: + if (is_last && !ret && batch->credit) { + atomic_add(batch->credit, &sc->send_io.bcredits.count); + batch->credit = 0; + wake_up(&sc->send_io.bcredits.wait_queue); + } + + return ret; +} + +static int wait_for_credits(struct smbdirect_socket *sc, + wait_queue_head_t *waitq, atomic_t *total_credits, + int needed) +{ + int ret; + + do { + if (atomic_sub_return(needed, total_credits) >= 0) + return 0; + + atomic_add(needed, total_credits); + ret = wait_event_interruptible(*waitq, + atomic_read(total_credits) >= needed || + sc->status != SMBDIRECT_SOCKET_CONNECTED); + + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) + return -ENOTCONN; + else if (ret < 0) + return ret; + } while (true); +} + +static int wait_for_send_bcredit(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch) +{ + int ret; + + if (batch->credit) + return 0; + + ret = wait_for_credits(sc, + &sc->send_io.bcredits.wait_queue, + &sc->send_io.bcredits.count, + 1); + if (ret) + return ret; + + batch->credit = 1; + return 0; +} + +static int wait_for_send_lcredit(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch) +{ + if (batch && (atomic_read(&sc->send_io.lcredits.count) <= 1)) { + int ret; + + ret = smbd_send_batch_flush(sc, batch, false); + if (ret) + return ret; + } + + return wait_for_credits(sc, + &sc->send_io.lcredits.wait_queue, + &sc->send_io.lcredits.count, + 1); +} + +static int wait_for_send_credits(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch) +{ + if (batch && + (batch->wr_cnt >= 16 || atomic_read(&sc->send_io.credits.count) <= 1)) { + int ret; + + ret = smbd_send_batch_flush(sc, batch, false); + if (ret) + return ret; + } + + return wait_for_credits(sc, + &sc->send_io.credits.wait_queue, + &sc->send_io.credits.count, + 1); } static int smbd_post_send_iter(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch, struct iov_iter *iter, int *_remaining_data_length) { struct smbdirect_socket_parameters *sp = &sc->parameters; - int i, rc; + int rc; int header_length; int data_length; struct smbdirect_send_io *request; struct smbdirect_data_transfer *packet; int new_credits = 0; + struct smbdirect_send_batch _batch; -wait_lcredit: - /* Wait for local send credits */ - rc = wait_event_interruptible(sc->send_io.lcredits.wait_queue, - atomic_read(&sc->send_io.lcredits.count) > 0 || - sc->status != SMBDIRECT_SOCKET_CONNECTED); - if (rc) - goto err_wait_lcredit; + if (!batch) { + smbd_send_batch_init(&_batch, false, 0); + batch = &_batch; + } - if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { - log_outgoing(ERR, "disconnected not sending on wait_credit\n"); + rc = wait_for_send_bcredit(sc, batch); + if (rc) { + log_outgoing(ERR, "disconnected not sending on wait_bcredit\n"); rc = -EAGAIN; - goto err_wait_lcredit; - } - if (unlikely(atomic_dec_return(&sc->send_io.lcredits.count) < 0)) { - atomic_inc(&sc->send_io.lcredits.count); - goto wait_lcredit; + goto err_wait_bcredit; } -wait_credit: - /* Wait for send credits. A SMBD packet needs one credit */ - rc = wait_event_interruptible(sc->send_io.credits.wait_queue, - atomic_read(&sc->send_io.credits.count) > 0 || - sc->status != SMBDIRECT_SOCKET_CONNECTED); - if (rc) - goto err_wait_credit; + rc = wait_for_send_lcredit(sc, batch); + if (rc) { + log_outgoing(ERR, "disconnected not sending on wait_lcredit\n"); + rc = -EAGAIN; + goto err_wait_lcredit; + } - if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { + rc = wait_for_send_credits(sc, batch); + if (rc) { log_outgoing(ERR, "disconnected not sending on wait_credit\n"); rc = -EAGAIN; goto err_wait_credit; } - if (unlikely(atomic_dec_return(&sc->send_io.credits.count) < 0)) { - atomic_inc(&sc->send_io.credits.count); - goto wait_credit; + + new_credits = manage_credits_prior_sending(sc); + if (new_credits == 0 && + atomic_read(&sc->send_io.credits.count) == 0 && + atomic_read(&sc->recv_io.credits.count) == 0) { + queue_work(sc->workqueue, &sc->recv_io.posted.refill_work); + rc = wait_event_interruptible(sc->send_io.credits.wait_queue, + atomic_read(&sc->send_io.credits.count) >= 1 || + atomic_read(&sc->recv_io.credits.available) >= 1 || + sc->status != SMBDIRECT_SOCKET_CONNECTED); + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) + rc = -ENOTCONN; + if (rc < 0) { + log_outgoing(ERR, "disconnected not sending on last credit\n"); + rc = -EAGAIN; + goto err_wait_credit; + } + + new_credits = manage_credits_prior_sending(sc); } - request = mempool_alloc(sc->send_io.mem.pool, GFP_KERNEL); - if (!request) { - rc = -ENOMEM; + request = smbd_alloc_send_io(sc); + if (IS_ERR(request)) { + rc = PTR_ERR(request); goto err_alloc; } - request->socket = sc; memset(request->sge, 0, sizeof(request->sge)); /* Map the packet to DMA */ @@ -1215,9 +1501,6 @@ static int smbd_post_send_iter(struct smbdirect_socket *sc, /* Fill in the packet header */ packet->credits_requested = cpu_to_le16(sp->send_credit_target); - - new_credits = manage_credits_prior_sending(sc); - atomic_add(new_credits, &sc->recv_io.credits.count); packet->credits_granted = cpu_to_le16(new_credits); packet->flags = 0; @@ -1240,32 +1523,18 @@ static int smbd_post_send_iter(struct smbdirect_socket *sc, le32_to_cpu(packet->data_length), le32_to_cpu(packet->remaining_data_length)); - /* - * Now that we got a local and a remote credit - * we add us as pending - */ - atomic_inc(&sc->send_io.pending.count); - - rc = smbd_post_send(sc, request); - if (!rc) - return 0; - - if (atomic_dec_and_test(&sc->send_io.pending.count)) - wake_up(&sc->send_io.pending.zero_wait_queue); + rc = smbd_post_send(sc, batch, request); + if (!rc) { + if (batch != &_batch) + return 0; - wake_up(&sc->send_io.pending.dec_wait_queue); + rc = smbd_send_batch_flush(sc, batch, true); + if (!rc) + return 0; + } err_dma: - for (i = 0; i < request->num_sge; i++) - if (request->sge[i].addr) - ib_dma_unmap_single(sc->ib.dev, - request->sge[i].addr, - request->sge[i].length, - DMA_TO_DEVICE); - mempool_free(request, sc->send_io.mem.pool); - - /* roll back the granted receive credits */ - atomic_sub(new_credits, &sc->recv_io.credits.count); + smbd_free_send_io(request); err_alloc: atomic_inc(&sc->send_io.credits.count); @@ -1276,6 +1545,11 @@ static int smbd_post_send_iter(struct smbdirect_socket *sc, wake_up(&sc->send_io.lcredits.wait_queue); err_wait_lcredit: + atomic_add(batch->credit, &sc->send_io.bcredits.count); + batch->credit = 0; + wake_up(&sc->send_io.bcredits.wait_queue); + +err_wait_bcredit: return rc; } @@ -1289,10 +1563,11 @@ static int smbd_post_send_empty(struct smbdirect_socket *sc) int remaining_data_length = 0; sc->statistics.send_empty++; - return smbd_post_send_iter(sc, NULL, &remaining_data_length); + return smbd_post_send_iter(sc, NULL, NULL, &remaining_data_length); } static int smbd_post_send_full_iter(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch, struct iov_iter *iter, int *_remaining_data_length) { @@ -1305,7 +1580,7 @@ static int smbd_post_send_full_iter(struct smbdirect_socket *sc, */ while (iov_iter_count(iter) > 0) { - rc = smbd_post_send_iter(sc, iter, _remaining_data_length); + rc = smbd_post_send_iter(sc, batch, iter, _remaining_data_length); if (rc < 0) break; } @@ -2227,8 +2502,10 @@ int smbd_send(struct TCP_Server_Info *server, struct smbdirect_socket_parameters *sp = &sc->parameters; struct smb_rqst *rqst; struct iov_iter iter; + struct smbdirect_send_batch batch; unsigned int remaining_data_length, klen; int rc, i, rqst_idx; + int error = 0; if (sc->status != SMBDIRECT_SOCKET_CONNECTED) return -EAGAIN; @@ -2253,6 +2530,7 @@ int smbd_send(struct TCP_Server_Info *server, num_rqst, remaining_data_length); rqst_idx = 0; + smbd_send_batch_init(&batch, false, 0); do { rqst = &rqst_array[rqst_idx]; @@ -2271,20 +2549,28 @@ int smbd_send(struct TCP_Server_Info *server, klen += rqst->rq_iov[i].iov_len; iov_iter_kvec(&iter, ITER_SOURCE, rqst->rq_iov, rqst->rq_nvec, klen); - rc = smbd_post_send_full_iter(sc, &iter, &remaining_data_length); - if (rc < 0) + rc = smbd_post_send_full_iter(sc, &batch, &iter, &remaining_data_length); + if (rc < 0) { + error = rc; break; + } if (iov_iter_count(&rqst->rq_iter) > 0) { /* And then the data pages if there are any */ - rc = smbd_post_send_full_iter(sc, &rqst->rq_iter, + rc = smbd_post_send_full_iter(sc, &batch, &rqst->rq_iter, &remaining_data_length); - if (rc < 0) + if (rc < 0) { + error = rc; break; + } } } while (++rqst_idx < num_rqst); + rc = smbd_send_batch_flush(sc, &batch, true); + if (unlikely(!rc && error)) + rc = error; + /* * As an optimization, we don't wait for individual I/O to finish * before sending the next one. diff --git a/fs/smb/common/smbdirect/smbdirect_socket.h b/fs/smb/common/smbdirect/smbdirect_socket.h index ee4c2726771a34..95265192bb01b2 100644 --- a/fs/smb/common/smbdirect/smbdirect_socket.h +++ b/fs/smb/common/smbdirect/smbdirect_socket.h @@ -162,6 +162,17 @@ struct smbdirect_socket { mempool_t *pool; } mem; + /* + * This is a coordination for smbdirect_send_batch. + * + * There's only one possible credit, which means + * only one instance is running at a time. + */ + struct { + atomic_t count; + wait_queue_head_t wait_queue; + } bcredits; + /* * The local credit state for ib_post_send() */ @@ -239,6 +250,7 @@ struct smbdirect_socket { */ struct { u16 target; + atomic_t available; atomic_t count; } credits; @@ -370,6 +382,9 @@ static __always_inline void smbdirect_socket_init(struct smbdirect_socket *sc) INIT_DELAYED_WORK(&sc->idle.timer_work, __smbdirect_socket_disabled_work); disable_delayed_work_sync(&sc->idle.timer_work); + atomic_set(&sc->send_io.bcredits.count, 0); + init_waitqueue_head(&sc->send_io.bcredits.wait_queue); + atomic_set(&sc->send_io.lcredits.count, 0); init_waitqueue_head(&sc->send_io.lcredits.wait_queue); @@ -387,6 +402,7 @@ static __always_inline void smbdirect_socket_init(struct smbdirect_socket *sc) INIT_WORK(&sc->recv_io.posted.refill_work, __smbdirect_socket_disabled_work); disable_work_sync(&sc->recv_io.posted.refill_work); + atomic_set(&sc->recv_io.credits.available, 0); atomic_set(&sc->recv_io.credits.count, 0); INIT_LIST_HEAD(&sc->recv_io.reassembly.list); @@ -483,6 +499,8 @@ struct smbdirect_send_batch { */ bool need_invalidate_rkey; u32 remote_key; + + int credit; }; struct smbdirect_recv_io { diff --git a/fs/smb/server/mgmt/user_session.c b/fs/smb/server/mgmt/user_session.c index 7d880ff34402e0..26cb87625f1c6b 100644 --- a/fs/smb/server/mgmt/user_session.c +++ b/fs/smb/server/mgmt/user_session.c @@ -32,12 +32,14 @@ static void free_channel_list(struct ksmbd_session *sess) struct channel *chann; unsigned long index; + down_write(&sess->chann_lock); xa_for_each(&sess->ksmbd_chann_list, index, chann) { xa_erase(&sess->ksmbd_chann_list, index); kfree(chann); } xa_destroy(&sess->ksmbd_chann_list); + up_write(&sess->chann_lock); } static void __session_rpc_close(struct ksmbd_session *sess, @@ -220,7 +222,9 @@ static int ksmbd_chann_del(struct ksmbd_conn *conn, struct ksmbd_session *sess) { struct channel *chann; + down_write(&sess->chann_lock); chann = xa_erase(&sess->ksmbd_chann_list, (long)conn); + up_write(&sess->chann_lock); if (!chann) return -ENOENT; @@ -454,6 +458,7 @@ static struct ksmbd_session *__session_create(int protocol) rwlock_init(&sess->tree_conns_lock); atomic_set(&sess->refcnt, 2); init_rwsem(&sess->rpc_lock); + init_rwsem(&sess->chann_lock); ret = __init_smb2_session(sess); if (ret) diff --git a/fs/smb/server/mgmt/user_session.h b/fs/smb/server/mgmt/user_session.h index c5749d6ec7151c..cba7f688f6b577 100644 --- a/fs/smb/server/mgmt/user_session.h +++ b/fs/smb/server/mgmt/user_session.h @@ -49,6 +49,7 @@ struct ksmbd_session { char sess_key[CIFS_KEY_SIZE]; struct hlist_node hlist; + struct rw_semaphore chann_lock; struct xarray ksmbd_chann_list; struct xarray tree_conns; struct ida tree_conn_ida; diff --git a/fs/smb/server/server.c b/fs/smb/server/server.c index 554ae90df906db..d2410a3f163aec 100644 --- a/fs/smb/server/server.c +++ b/fs/smb/server/server.c @@ -126,21 +126,21 @@ static int __process_request(struct ksmbd_work *work, struct ksmbd_conn *conn, andx_again: if (command >= conn->max_cmds) { conn->ops->set_rsp_status(work, STATUS_INVALID_PARAMETER); - return SERVER_HANDLER_CONTINUE; + return SERVER_HANDLER_ABORT; } cmds = &conn->cmds[command]; if (!cmds->proc) { ksmbd_debug(SMB, "*** not implemented yet cmd = %x\n", command); conn->ops->set_rsp_status(work, STATUS_NOT_IMPLEMENTED); - return SERVER_HANDLER_CONTINUE; + return SERVER_HANDLER_ABORT; } if (work->sess && conn->ops->is_sign_req(work, command)) { ret = conn->ops->check_sign_req(work); if (!ret) { conn->ops->set_rsp_status(work, STATUS_ACCESS_DENIED); - return SERVER_HANDLER_CONTINUE; + return SERVER_HANDLER_ABORT; } } diff --git a/fs/smb/server/smb2pdu.c b/fs/smb/server/smb2pdu.c index 2fcd0d4d1fb0de..1022d794bd2327 100644 --- a/fs/smb/server/smb2pdu.c +++ b/fs/smb/server/smb2pdu.c @@ -79,7 +79,13 @@ static inline bool check_session_id(struct ksmbd_conn *conn, u64 id) struct channel *lookup_chann_list(struct ksmbd_session *sess, struct ksmbd_conn *conn) { - return xa_load(&sess->ksmbd_chann_list, (long)conn); + struct channel *chann; + + down_read(&sess->chann_lock); + chann = xa_load(&sess->ksmbd_chann_list, (long)conn); + up_read(&sess->chann_lock); + + return chann; } /** @@ -1558,8 +1564,10 @@ static int ntlm_authenticate(struct ksmbd_work *work, return -ENOMEM; chann->conn = conn; + down_write(&sess->chann_lock); old = xa_store(&sess->ksmbd_chann_list, (long)conn, chann, KSMBD_DEFAULT_GFP); + up_write(&sess->chann_lock); if (xa_is_err(old)) { kfree(chann); return xa_err(old); @@ -1651,8 +1659,10 @@ static int krb5_authenticate(struct ksmbd_work *work, return -ENOMEM; chann->conn = conn; + down_write(&sess->chann_lock); old = xa_store(&sess->ksmbd_chann_list, (long)conn, chann, KSMBD_DEFAULT_GFP); + up_write(&sess->chann_lock); if (xa_is_err(old)) { kfree(chann); return xa_err(old); @@ -6104,14 +6114,14 @@ static int smb2_create_link(struct ksmbd_work *work, rc = -EINVAL; ksmbd_debug(SMB, "cannot delete %s\n", link_name); - goto out; } } else { rc = -EEXIST; ksmbd_debug(SMB, "link already exists\n"); - goto out; } ksmbd_vfs_kern_path_end_removing(&path); + if (rc) + goto out; } rc = ksmbd_vfs_link(work, target_name, link_name); if (rc) diff --git a/fs/smb/server/transport_rdma.c b/fs/smb/server/transport_rdma.c index e4273932e7e497..c94068b78a1d2d 100644 --- a/fs/smb/server/transport_rdma.c +++ b/fs/smb/server/transport_rdma.c @@ -221,6 +221,7 @@ static void smb_direct_disconnect_wake_up_all(struct smbdirect_socket *sc) * in order to notice the broken connection. */ wake_up_all(&sc->status_wait); + wake_up_all(&sc->send_io.bcredits.wait_queue); wake_up_all(&sc->send_io.lcredits.wait_queue); wake_up_all(&sc->send_io.credits.wait_queue); wake_up_all(&sc->send_io.pending.zero_wait_queue); @@ -644,6 +645,7 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) struct smbdirect_data_transfer *data_transfer = (struct smbdirect_data_transfer *)recvmsg->packet; u32 remaining_data_length, data_offset, data_length; + int current_recv_credits; u16 old_recv_credit_target; if (wc->byte_len < @@ -682,7 +684,7 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) } atomic_dec(&sc->recv_io.posted.count); - atomic_dec(&sc->recv_io.credits.count); + current_recv_credits = atomic_dec_return(&sc->recv_io.credits.count); old_recv_credit_target = sc->recv_io.credits.target; sc->recv_io.credits.target = @@ -702,7 +704,8 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) wake_up(&sc->send_io.credits.wait_queue); if (data_length) { - if (sc->recv_io.credits.target > old_recv_credit_target) + if (current_recv_credits <= (sc->recv_io.credits.target / 4) || + sc->recv_io.credits.target > old_recv_credit_target) queue_work(sc->workqueue, &sc->recv_io.posted.refill_work); enqueue_reassembly(sc, recvmsg, (int)data_length); @@ -1028,6 +1031,17 @@ static void smb_direct_post_recv_credits(struct work_struct *work) } } + atomic_add(credits, &sc->recv_io.credits.available); + + /* + * If the last send credit is waiting for credits + * it can grant we need to wake it up + */ + if (credits && + atomic_read(&sc->send_io.bcredits.count) == 0 && + atomic_read(&sc->send_io.credits.count) == 0) + wake_up(&sc->send_io.credits.wait_queue); + if (credits) queue_work(sc->workqueue, &sc->idle.immediate_work); } @@ -1045,6 +1059,31 @@ static void send_done(struct ib_cq *cq, struct ib_wc *wc) ib_wc_status_msg(wc->status), wc->status, wc->opcode); + if (unlikely(!(sendmsg->wr.send_flags & IB_SEND_SIGNALED))) { + /* + * This happens when smbdirect_send_io is a sibling + * before the final message, it is signaled on + * error anyway, so we need to skip + * smbdirect_connection_free_send_io here, + * otherwise is will destroy the memory + * of the siblings too, which will cause + * use after free problems for the others + * triggered from ib_drain_qp(). + */ + if (wc->status != IB_WC_SUCCESS) + goto skip_free; + + /* + * This should not happen! + * But we better just close the + * connection... + */ + pr_err("unexpected send completion wc->status=%s (%d) wc->opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + smb_direct_disconnect_rdma_connection(sc); + return; + } + /* * Free possible siblings and then the main send_io */ @@ -1058,6 +1097,7 @@ static void send_done(struct ib_cq *cq, struct ib_wc *wc) lcredits += 1; if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_SEND) { +skip_free: pr_err("Send error. status='%s (%d)', opcode=%d\n", ib_wc_status_msg(wc->status), wc->status, wc->opcode); @@ -1074,19 +1114,37 @@ static void send_done(struct ib_cq *cq, struct ib_wc *wc) static int manage_credits_prior_sending(struct smbdirect_socket *sc) { + int missing; + int available; int new_credits; if (atomic_read(&sc->recv_io.credits.count) >= sc->recv_io.credits.target) return 0; - new_credits = atomic_read(&sc->recv_io.posted.count); - if (new_credits == 0) + missing = (int)sc->recv_io.credits.target - atomic_read(&sc->recv_io.credits.count); + available = atomic_xchg(&sc->recv_io.credits.available, 0); + new_credits = (u16)min3(U16_MAX, missing, available); + if (new_credits <= 0) { + /* + * If credits are available, but not granted + * we need to re-add them again. + */ + if (available) + atomic_add(available, &sc->recv_io.credits.available); return 0; + } - new_credits -= atomic_read(&sc->recv_io.credits.count); - if (new_credits <= 0) - return 0; + if (new_credits < available) { + /* + * Readd the remaining available again. + */ + available -= new_credits; + atomic_add(available, &sc->recv_io.credits.available); + } + /* + * Remember we granted the credits + */ atomic_add(new_credits, &sc->recv_io.credits.count); return new_credits; } @@ -1130,6 +1188,7 @@ static void smb_direct_send_ctx_init(struct smbdirect_send_batch *send_ctx, send_ctx->wr_cnt = 0; send_ctx->need_invalidate_rkey = need_invalidate_rkey; send_ctx->remote_key = remote_key; + send_ctx->credit = 0; } static int smb_direct_flush_send_list(struct smbdirect_socket *sc, @@ -1137,10 +1196,10 @@ static int smb_direct_flush_send_list(struct smbdirect_socket *sc, bool is_last) { struct smbdirect_send_io *first, *last; - int ret; + int ret = 0; if (list_empty(&send_ctx->msg_list)) - return 0; + goto release_credit; first = list_first_entry(&send_ctx->msg_list, struct smbdirect_send_io, @@ -1182,6 +1241,13 @@ static int smb_direct_flush_send_list(struct smbdirect_socket *sc, smb_direct_free_sendmsg(sc, last); } +release_credit: + if (is_last && !ret && send_ctx->credit) { + atomic_add(send_ctx->credit, &sc->send_io.bcredits.count); + send_ctx->credit = 0; + wake_up(&sc->send_io.bcredits.wait_queue); + } + return ret; } @@ -1207,6 +1273,25 @@ static int wait_for_credits(struct smbdirect_socket *sc, } while (true); } +static int wait_for_send_bcredit(struct smbdirect_socket *sc, + struct smbdirect_send_batch *send_ctx) +{ + int ret; + + if (send_ctx->credit) + return 0; + + ret = wait_for_credits(sc, + &sc->send_io.bcredits.wait_queue, + &sc->send_io.bcredits.count, + 1); + if (ret) + return ret; + + send_ctx->credit = 1; + return 0; +} + static int wait_for_send_lcredit(struct smbdirect_socket *sc, struct smbdirect_send_batch *send_ctx) { @@ -1256,6 +1341,7 @@ static int calc_rw_credits(struct smbdirect_socket *sc, static int smb_direct_create_header(struct smbdirect_socket *sc, int size, int remaining_data_length, + int new_credits, struct smbdirect_send_io **sendmsg_out) { struct smbdirect_socket_parameters *sp = &sc->parameters; @@ -1271,7 +1357,7 @@ static int smb_direct_create_header(struct smbdirect_socket *sc, /* Fill in the packet header */ packet = (struct smbdirect_data_transfer *)sendmsg->packet; packet->credits_requested = cpu_to_le16(sp->send_credit_target); - packet->credits_granted = cpu_to_le16(manage_credits_prior_sending(sc)); + packet->credits_granted = cpu_to_le16(new_credits); packet->flags = 0; if (manage_keep_alive_before_sending(sc)) @@ -1408,6 +1494,17 @@ static int smb_direct_post_send_data(struct smbdirect_socket *sc, struct smbdirect_send_io *msg; int data_length; struct scatterlist sg[SMBDIRECT_SEND_IO_MAX_SGE - 1]; + struct smbdirect_send_batch _send_ctx; + int new_credits; + + if (!send_ctx) { + smb_direct_send_ctx_init(&_send_ctx, false, 0); + send_ctx = &_send_ctx; + } + + ret = wait_for_send_bcredit(sc, send_ctx); + if (ret) + goto bcredit_failed; ret = wait_for_send_lcredit(sc, send_ctx); if (ret) @@ -1417,12 +1514,29 @@ static int smb_direct_post_send_data(struct smbdirect_socket *sc, if (ret) goto credit_failed; + new_credits = manage_credits_prior_sending(sc); + if (new_credits == 0 && + atomic_read(&sc->send_io.credits.count) == 0 && + atomic_read(&sc->recv_io.credits.count) == 0) { + queue_work(sc->workqueue, &sc->recv_io.posted.refill_work); + ret = wait_event_interruptible(sc->send_io.credits.wait_queue, + atomic_read(&sc->send_io.credits.count) >= 1 || + atomic_read(&sc->recv_io.credits.available) >= 1 || + sc->status != SMBDIRECT_SOCKET_CONNECTED); + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) + ret = -ENOTCONN; + if (ret < 0) + goto credit_failed; + + new_credits = manage_credits_prior_sending(sc); + } + data_length = 0; for (i = 0; i < niov; i++) data_length += iov[i].iov_len; ret = smb_direct_create_header(sc, data_length, remaining_data_length, - &msg); + new_credits, &msg); if (ret) goto header_failed; @@ -1460,6 +1574,13 @@ static int smb_direct_post_send_data(struct smbdirect_socket *sc, ret = post_sendmsg(sc, send_ctx, msg); if (ret) goto err; + + if (send_ctx == &_send_ctx) { + ret = smb_direct_flush_send_list(sc, send_ctx, true); + if (ret) + goto err; + } + return 0; err: smb_direct_free_sendmsg(sc, msg); @@ -1468,6 +1589,9 @@ static int smb_direct_post_send_data(struct smbdirect_socket *sc, credit_failed: atomic_inc(&sc->send_io.lcredits.count); lcredit_failed: + atomic_add(send_ctx->credit, &sc->send_io.bcredits.count); + send_ctx->credit = 0; +bcredit_failed: return ret; } @@ -1939,6 +2063,7 @@ static int smb_direct_send_negotiate_response(struct smbdirect_socket *sc, resp->max_fragmented_size = cpu_to_le32(sp->max_fragmented_recv_size); + atomic_set(&sc->send_io.bcredits.count, 1); sc->recv_io.expected = SMBDIRECT_EXPECT_DATA_TRANSFER; sc->status = SMBDIRECT_SOCKET_CONNECTED; } diff --git a/fs/smb/server/transport_tcp.c b/fs/smb/server/transport_tcp.c index 4bb07937d7efeb..2436dabada957a 100644 --- a/fs/smb/server/transport_tcp.c +++ b/fs/smb/server/transport_tcp.c @@ -40,6 +40,7 @@ static const struct ksmbd_transport_ops ksmbd_tcp_transport_ops; static void tcp_stop_kthread(struct task_struct *kthread); static struct interface *alloc_iface(char *ifname); +static void ksmbd_tcp_disconnect(struct ksmbd_transport *t); #define KSMBD_TRANS(t) (&(t)->transport) #define TCP_TRANS(t) ((struct tcp_transport *)container_of(t, \ @@ -202,7 +203,7 @@ static int ksmbd_tcp_new_connection(struct socket *client_sk) if (IS_ERR(handler)) { pr_err("cannot start conn thread\n"); rc = PTR_ERR(handler); - free_transport(t); + ksmbd_tcp_disconnect(KSMBD_TRANS(t)); } return rc; } diff --git a/fs/tests/exec_kunit.c b/fs/tests/exec_kunit.c index 7c77d039680bbe..f412d1a0f6bba9 100644 --- a/fs/tests/exec_kunit.c +++ b/fs/tests/exec_kunit.c @@ -87,9 +87,6 @@ static const struct bprm_stack_limits_result bprm_stack_limits_results[] = { .argc = 0, .envc = ARG_MAX / sizeof(void *) - 1 }, .expected_argmin = ULONG_MAX - sizeof(void *) }, /* Raising rlim_stack / 4 to _STK_LIM / 4 * 3 will see more space. */ - { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * (_STK_LIM / 4 * 3), - .argc = 0, .envc = 0 }, - .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * (_STK_LIM / 4 * 3), .argc = 0, .envc = 0 }, .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, @@ -103,9 +100,6 @@ static const struct bprm_stack_limits_result bprm_stack_limits_results[] = { { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * _STK_LIM, .argc = 0, .envc = 0 }, .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, - { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * _STK_LIM, - .argc = 0, .envc = 0 }, - .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, }; static void exec_test_bprm_stack_limits(struct kunit *test) diff --git a/fs/xfs/scrub/btree.c b/fs/xfs/scrub/btree.c index cd6f0ff382a7c8..acade92c5fce1a 100644 --- a/fs/xfs/scrub/btree.c +++ b/fs/xfs/scrub/btree.c @@ -370,12 +370,15 @@ xchk_btree_check_block_owner( { xfs_agnumber_t agno; xfs_agblock_t agbno; + bool is_bnobt, is_rmapbt; bool init_sa; int error = 0; if (!bs->cur) return 0; + is_bnobt = xfs_btree_is_bno(bs->cur->bc_ops); + is_rmapbt = xfs_btree_is_rmap(bs->cur->bc_ops); agno = xfs_daddr_to_agno(bs->cur->bc_mp, daddr); agbno = xfs_daddr_to_agbno(bs->cur->bc_mp, daddr); @@ -398,11 +401,11 @@ xchk_btree_check_block_owner( * have to nullify it (to shut down further block owner checks) if * self-xref encounters problems. */ - if (!bs->sc->sa.bno_cur && xfs_btree_is_bno(bs->cur->bc_ops)) + if (!bs->sc->sa.bno_cur && is_bnobt) bs->cur = NULL; xchk_xref_is_only_owned_by(bs->sc, agbno, 1, bs->oinfo); - if (!bs->sc->sa.rmap_cur && xfs_btree_is_rmap(bs->cur->bc_ops)) + if (!bs->sc->sa.rmap_cur && is_rmapbt) bs->cur = NULL; out_free: diff --git a/include/acpi/pcc.h b/include/acpi/pcc.h index 9af3b502f83952..840bfc95bae332 100644 --- a/include/acpi/pcc.h +++ b/include/acpi/pcc.h @@ -17,35 +17,6 @@ struct pcc_mbox_chan { u32 latency; u32 max_access_rate; u16 min_turnaround_time; - - /* Set to true to indicate that the mailbox should manage - * writing the dat to the shared buffer. This differs from - * the case where the drivesr are writing to the buffer and - * using send_data only to ring the doorbell. If this flag - * is set, then the void * data parameter of send_data must - * point to a kernel-memory buffer formatted in accordance with - * the PCC specification. - * - * The active buffer management will include reading the - * notify_on_completion flag, and will then - * call mbox_chan_txdone when the acknowledgment interrupt is - * received. - */ - bool manage_writes; - - /* Optional callback that allows the driver - * to allocate the memory used for receiving - * messages. The return value is the location - * inside the buffer where the mailbox should write the data. - */ - void *(*rx_alloc)(struct mbox_client *cl, int size); -}; - -struct pcc_header { - u32 signature; - u32 flags; - u32 length; - u32 command; }; /* Generic Communications Channel Shared Memory Region */ diff --git a/include/asm-generic/rqspinlock.h b/include/asm-generic/rqspinlock.h index 0f2dcbbfee2f0a..5c5cf2f7fc395a 100644 --- a/include/asm-generic/rqspinlock.h +++ b/include/asm-generic/rqspinlock.h @@ -191,7 +191,7 @@ static __always_inline int res_spin_lock(rqspinlock_t *lock) #else -#define res_spin_lock(lock) resilient_tas_spin_lock(lock) +#define res_spin_lock(lock) ({ grab_held_lock_entry(lock); resilient_tas_spin_lock(lock); }) #endif /* CONFIG_QUEUED_SPINLOCKS */ diff --git a/include/drm/drm_atomic.h b/include/drm/drm_atomic.h index 43783891d3594b..ff1fecd30478a9 100644 --- a/include/drm/drm_atomic.h +++ b/include/drm/drm_atomic.h @@ -30,7 +30,6 @@ #include #include -#include /** * struct drm_crtc_commit - track modeset commits on a CRTC @@ -707,6 +706,14 @@ drm_atomic_get_plane_state(struct drm_atomic_state *state, struct drm_colorop_state * drm_atomic_get_colorop_state(struct drm_atomic_state *state, struct drm_colorop *colorop); + +struct drm_colorop_state * +drm_atomic_get_old_colorop_state(struct drm_atomic_state *state, + struct drm_colorop *colorop); +struct drm_colorop_state * +drm_atomic_get_new_colorop_state(struct drm_atomic_state *state, + struct drm_colorop *colorop); + struct drm_connector_state * __must_check drm_atomic_get_connector_state(struct drm_atomic_state *state, struct drm_connector *connector); @@ -803,36 +810,6 @@ drm_atomic_get_new_plane_state(const struct drm_atomic_state *state, return state->planes[drm_plane_index(plane)].new_state; } -/** - * drm_atomic_get_old_colorop_state - get colorop state, if it exists - * @state: global atomic state object - * @colorop: colorop to grab - * - * This function returns the old colorop state for the given colorop, or - * NULL if the colorop is not part of the global atomic state. - */ -static inline struct drm_colorop_state * -drm_atomic_get_old_colorop_state(struct drm_atomic_state *state, - struct drm_colorop *colorop) -{ - return state->colorops[drm_colorop_index(colorop)].old_state; -} - -/** - * drm_atomic_get_new_colorop_state - get colorop state, if it exists - * @state: global atomic state object - * @colorop: colorop to grab - * - * This function returns the new colorop state for the given colorop, or - * NULL if the colorop is not part of the global atomic state. - */ -static inline struct drm_colorop_state * -drm_atomic_get_new_colorop_state(struct drm_atomic_state *state, - struct drm_colorop *colorop) -{ - return state->colorops[drm_colorop_index(colorop)].new_state; -} - /** * drm_atomic_get_old_connector_state - get connector state, if it exists * @state: global atomic state object diff --git a/include/drm/drm_gpuvm.h b/include/drm/drm_gpuvm.h index fdfc575b260360..75c48345677b0f 100644 --- a/include/drm/drm_gpuvm.h +++ b/include/drm/drm_gpuvm.h @@ -57,10 +57,19 @@ enum drm_gpuva_flags { */ DRM_GPUVA_SPARSE = (1 << 1), + /** + * @DRM_GPUVA_REPEAT: + * + * Flag indicating that the &drm_gpuva is a mapping of a GEM + * object with a certain range that is repeated multiple times to + * fill the virtual address range. + */ + DRM_GPUVA_REPEAT = (1 << 2), + /** * @DRM_GPUVA_USERBITS: user defined bits */ - DRM_GPUVA_USERBITS = (1 << 2), + DRM_GPUVA_USERBITS = (1 << 3), }; /** @@ -112,6 +121,18 @@ struct drm_gpuva { */ u64 offset; + /* + * @gem.range: the range of the GEM that is mapped + * + * When dealing with normal mappings, this must be zero. + * When flags has DRM_GPUVA_REPEAT set, this field must be + * smaller than va.range and va.range must be a multiple of + * gem.range. + * This is a u32 not a u64 because we expect repeated mappings + * to be pointing to relatively small portions of a GEM object. + */ + u32 range; + /** * @gem.obj: the mapped &drm_gem_object */ @@ -882,11 +903,27 @@ struct drm_gpuva_op_map { */ u64 offset; + /* + * @gem.range: the range of the GEM that is mapped + * + * When dealing with normal mappings, this must be zero. + * When flags has DRM_GPUVA_REPEAT set, it must be a multiple + * of va.range. This is a u32 not a u64 because we expect + * repeated mappings to be pointing to a relatively small + * portion of a GEM object. + */ + u32 range; + /** * @gem.obj: the &drm_gem_object to map */ struct drm_gem_object *obj; } gem; + + /** + * @flags: requested flags for the &drm_gpuva for this mapping + */ + enum drm_gpuva_flags flags; }; /** @@ -1123,6 +1160,7 @@ void drm_gpuva_ops_free(struct drm_gpuvm *gpuvm, static inline void drm_gpuva_init_from_op(struct drm_gpuva *va, struct drm_gpuva_op_map *op) { + va->flags = op->flags; va->va.addr = op->va.addr; va->va.range = op->va.range; va->gem.obj = op->gem.obj; @@ -1248,6 +1286,16 @@ struct drm_gpuvm_ops { * used. */ int (*sm_step_unmap)(struct drm_gpuva_op *op, void *priv); + + /** + * @sm_can_merge_flags: called during &drm_gpuvm_sm_map + * + * This callback is called to determine whether two va ranges can be merged, + * based on their flags. + * + * If NULL, va ranges can only be merged if their flags are equal. + */ + bool (*sm_can_merge_flags)(enum drm_gpuva_flags a, enum drm_gpuva_flags b); }; int drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, void *priv, @@ -1255,6 +1303,7 @@ int drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, void *priv, int drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, void *priv, u64 addr, u64 range); +int drm_gpuvm_bo_unmap(struct drm_gpuvm_bo *bo, void *priv); int drm_gpuvm_sm_map_exec_lock(struct drm_gpuvm *gpuvm, struct drm_exec *exec, unsigned int num_fences, diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h index fb88301b3c45c7..e2d646070cd316 100644 --- a/include/drm/gpu_scheduler.h +++ b/include/drm/gpu_scheduler.h @@ -301,6 +301,11 @@ struct drm_sched_fence { * @lock: the lock used by the scheduled and the finished fences. */ spinlock_t lock; + /** + * @sched_name: the name of the scheduler that owns this fence. We + * keep a copy here since fences can outlive their scheduler. + */ + char sched_name[16]; /** * @owner: job owner for debugging */ diff --git a/include/drm/intel/intel_lb_mei_interface.h b/include/drm/intel/intel_lb_mei_interface.h index d65be2cba2ab9f..0850738a30fc75 100644 --- a/include/drm/intel/intel_lb_mei_interface.h +++ b/include/drm/intel/intel_lb_mei_interface.h @@ -53,7 +53,8 @@ enum intel_lb_status { */ struct intel_lb_component_ops { /** - * push_payload - Sends a payload to the authentication firmware + * @push_payload: Sends a payload to the authentication firmware + * * @dev: Device struct corresponding to the mei device * @type: Payload type (see &enum intel_lb_type) * @flags: Payload flags bitmap (e.g. %INTEL_LB_FLAGS_IS_PERSISTENT) diff --git a/include/linux/ata.h b/include/linux/ata.h index 54b416e2699590..c9013e472aa3d5 100644 --- a/include/linux/ata.h +++ b/include/linux/ata.h @@ -29,7 +29,6 @@ enum { ATA_MAX_SECTORS_128 = 128, ATA_MAX_SECTORS = 256, ATA_MAX_SECTORS_1024 = 1024, - ATA_MAX_SECTORS_8191 = 8191, ATA_MAX_SECTORS_LBA48 = 65535,/* avoid count to be 0000h */ ATA_MAX_SECTORS_TAPE = 65535, ATA_MAX_TRIM_RNUM = 64, /* 512-byte payload / (6-byte LBA + 2-byte range per entry) */ diff --git a/include/linux/audit.h b/include/linux/audit.h index 536f8ee8da818c..b8d8029c6c480c 100644 --- a/include/linux/audit.h +++ b/include/linux/audit.h @@ -128,12 +128,6 @@ enum audit_nfcfgop { extern int __init audit_register_class(int class, unsigned *list); extern int audit_classify_syscall(int abi, unsigned syscall); extern int audit_classify_arch(int arch); -/* only for compat system calls */ -extern unsigned compat_write_class[]; -extern unsigned compat_read_class[]; -extern unsigned compat_dir_class[]; -extern unsigned compat_chattr_class[]; -extern unsigned compat_signal_class[]; /* audit_names->type values */ #define AUDIT_TYPE_UNKNOWN 0 /* we don't know yet */ diff --git a/include/linux/audit_arch.h b/include/linux/audit_arch.h index 0e34d673ef1712..2b8153791e6a5d 100644 --- a/include/linux/audit_arch.h +++ b/include/linux/audit_arch.h @@ -23,4 +23,11 @@ enum auditsc_class_t { extern int audit_classify_compat_syscall(int abi, unsigned syscall); +/* only for compat system calls */ +extern unsigned compat_write_class[]; +extern unsigned compat_read_class[]; +extern unsigned compat_dir_class[]; +extern unsigned compat_chattr_class[]; +extern unsigned compat_signal_class[]; + #endif diff --git a/include/linux/bpf.h b/include/linux/bpf.h index e5be698256d15a..7b2e51216e736a 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -3243,6 +3243,11 @@ static inline void bpf_prog_report_arena_violation(bool write, unsigned long add } #endif /* CONFIG_BPF_SYSCALL */ +static inline bool bpf_net_capable(void) +{ + return capable(CAP_NET_ADMIN) || capable(CAP_SYS_ADMIN); +} + static __always_inline int bpf_probe_read_kernel_common(void *dst, u32 size, const void *unsafe_ptr) { diff --git a/include/linux/bpf_mprog.h b/include/linux/bpf_mprog.h index 929225f7b09594..0b9f4caeeb0a32 100644 --- a/include/linux/bpf_mprog.h +++ b/include/linux/bpf_mprog.h @@ -340,4 +340,14 @@ static inline bool bpf_mprog_supported(enum bpf_prog_type type) return false; } } + +static inline bool bpf_mprog_detach_empty(enum bpf_prog_type type) +{ + switch (type) { + case BPF_PROG_TYPE_SCHED_CLS: + return bpf_net_capable(); + default: + return false; + } +} #endif /* __BPF_MPROG_H */ diff --git a/include/linux/capability.h b/include/linux/capability.h index 1fb08922552c79..37db92b3d6f89e 100644 --- a/include/linux/capability.h +++ b/include/linux/capability.h @@ -203,6 +203,12 @@ static inline bool checkpoint_restore_ns_capable(struct user_namespace *ns) ns_capable(ns, CAP_SYS_ADMIN); } +static inline bool checkpoint_restore_ns_capable_noaudit(struct user_namespace *ns) +{ + return ns_capable_noaudit(ns, CAP_CHECKPOINT_RESTORE) || + ns_capable_noaudit(ns, CAP_SYS_ADMIN); +} + /* audit system wants to get cap info from files as well */ int get_vfs_caps_from_disk(struct mnt_idmap *idmap, const struct dentry *dentry, diff --git a/include/linux/clk.h b/include/linux/clk.h index b607482ca77e98..64ff118ffb1a1d 100644 --- a/include/linux/clk.h +++ b/include/linux/clk.h @@ -228,6 +228,23 @@ int devm_clk_rate_exclusive_get(struct device *dev, struct clk *clk); */ void clk_rate_exclusive_put(struct clk *clk); +/** + * clk_save_context - save clock context for poweroff + * + * Saves the context of the clock register for powerstates in which the + * contents of the registers will be lost. Occurs deep within the suspend + * code so locking is not necessary. + */ +int clk_save_context(void); + +/** + * clk_restore_context - restore clock context after poweroff + * + * This occurs with all clocks enabled. Occurs deep within the resume code + * so locking is not necessary. + */ +void clk_restore_context(void); + #else static inline int clk_notifier_register(struct clk *clk, @@ -293,6 +310,13 @@ static inline int devm_clk_rate_exclusive_get(struct device *dev, struct clk *cl static inline void clk_rate_exclusive_put(struct clk *clk) {} +static inline int clk_save_context(void) +{ + return 0; +} + +static inline void clk_restore_context(void) {} + #endif #ifdef CONFIG_HAVE_CLK_PREPARE @@ -933,23 +957,6 @@ struct clk *clk_get_parent(struct clk *clk); */ struct clk *clk_get_sys(const char *dev_id, const char *con_id); -/** - * clk_save_context - save clock context for poweroff - * - * Saves the context of the clock register for powerstates in which the - * contents of the registers will be lost. Occurs deep within the suspend - * code so locking is not necessary. - */ -int clk_save_context(void); - -/** - * clk_restore_context - restore clock context after poweroff - * - * This occurs with all clocks enabled. Occurs deep within the resume code - * so locking is not necessary. - */ -void clk_restore_context(void); - #else /* !CONFIG_HAVE_CLK */ static inline struct clk *clk_get(struct device *dev, const char *id) @@ -1129,13 +1136,6 @@ static inline struct clk *clk_get_sys(const char *dev_id, const char *con_id) return NULL; } -static inline int clk_save_context(void) -{ - return 0; -} - -static inline void clk_restore_context(void) {} - #endif /* clk_prepare_enable helps cases using clk_enable in non-atomic context. */ diff --git a/include/linux/device_cgroup.h b/include/linux/device_cgroup.h index 0864773a57e8fe..822085bc2d202c 100644 --- a/include/linux/device_cgroup.h +++ b/include/linux/device_cgroup.h @@ -21,7 +21,7 @@ static inline int devcgroup_inode_permission(struct inode *inode, int mask) if (likely(!S_ISBLK(inode->i_mode) && !S_ISCHR(inode->i_mode))) return 0; - if (likely(!inode->i_rdev)) + if (!inode->i_rdev) return 0; if (S_ISBLK(inode->i_mode)) diff --git a/include/linux/exportfs.h b/include/linux/exportfs.h index f0cf2714ec52dd..9bd93d6bd9a49e 100644 --- a/include/linux/exportfs.h +++ b/include/linux/exportfs.h @@ -317,6 +317,15 @@ static inline bool exportfs_can_decode_fh(const struct export_operations *nop) return nop && nop->fh_to_dentry; } +static inline bool exportfs_may_export(const struct export_operations *nop) +{ + /* + * Do not allow nfs export for filesystems with custom ->open() or + * ->permission() ops, which nfsd does not respect (e.g. pidfs, nsfs). + */ + return exportfs_can_decode_fh(nop) && !nop->open && !nop->permission; +} + static inline bool exportfs_can_encode_fh(const struct export_operations *nop, int fh_flags) { diff --git a/include/linux/f2fs_fs.h b/include/linux/f2fs_fs.h index a7880787cad366..dc41722fcc9dde 100644 --- a/include/linux/f2fs_fs.h +++ b/include/linux/f2fs_fs.h @@ -17,7 +17,6 @@ #define F2FS_LOG_SECTORS_PER_BLOCK (PAGE_SHIFT - 9) /* log number for sector/blk */ #define F2FS_BLKSIZE PAGE_SIZE /* support only block == page */ #define F2FS_BLKSIZE_BITS PAGE_SHIFT /* bits for F2FS_BLKSIZE */ -#define F2FS_SUM_BLKSIZE 4096 /* only support 4096 byte sum block */ #define F2FS_MAX_EXTENSION 64 /* # of extension entries */ #define F2FS_EXTENSION_LEN 8 /* max size of extension */ @@ -442,10 +441,8 @@ struct f2fs_sit_block { * from node's page's beginning to get a data block address. * ex) data_blkaddr = (block_t)(nodepage_start_address + ofs_in_node) */ -#define ENTRIES_IN_SUM (F2FS_SUM_BLKSIZE / 8) #define SUMMARY_SIZE (7) /* sizeof(struct f2fs_summary) */ #define SUM_FOOTER_SIZE (5) /* sizeof(struct summary_footer) */ -#define SUM_ENTRY_SIZE (SUMMARY_SIZE * ENTRIES_IN_SUM) /* a summary entry for a block in a segment */ struct f2fs_summary { @@ -468,22 +465,6 @@ struct summary_footer { __le32 check_sum; /* summary checksum */ } __packed; -#define SUM_JOURNAL_SIZE (F2FS_SUM_BLKSIZE - SUM_FOOTER_SIZE -\ - SUM_ENTRY_SIZE) -#define NAT_JOURNAL_ENTRIES ((SUM_JOURNAL_SIZE - 2) /\ - sizeof(struct nat_journal_entry)) -#define NAT_JOURNAL_RESERVED ((SUM_JOURNAL_SIZE - 2) %\ - sizeof(struct nat_journal_entry)) -#define SIT_JOURNAL_ENTRIES ((SUM_JOURNAL_SIZE - 2) /\ - sizeof(struct sit_journal_entry)) -#define SIT_JOURNAL_RESERVED ((SUM_JOURNAL_SIZE - 2) %\ - sizeof(struct sit_journal_entry)) - -/* Reserved area should make size of f2fs_extra_info equals to - * that of nat_journal and sit_journal. - */ -#define EXTRA_INFO_RESERVED (SUM_JOURNAL_SIZE - 2 - 8) - /* * frequently updated NAT/SIT entries can be stored in the spare area in * summary blocks @@ -498,9 +479,16 @@ struct nat_journal_entry { struct f2fs_nat_entry ne; } __packed; +/* + * The nat_journal structure is a placeholder whose actual size varies depending + * on the use of packed_ssa. Therefore, it must always be accessed only through + * specific sets of macros and fields, and size calculations should use + * size-related macros instead of sizeof(). + * Relevant macros: sbi->nat_journal_entries, nat_in_journal(), + * nid_in_journal(), MAX_NAT_JENTRIES(). + */ struct nat_journal { - struct nat_journal_entry entries[NAT_JOURNAL_ENTRIES]; - __u8 reserved[NAT_JOURNAL_RESERVED]; + struct nat_journal_entry entries[0]; } __packed; struct sit_journal_entry { @@ -508,14 +496,21 @@ struct sit_journal_entry { struct f2fs_sit_entry se; } __packed; +/* + * The sit_journal structure is a placeholder whose actual size varies depending + * on the use of packed_ssa. Therefore, it must always be accessed only through + * specific sets of macros and fields, and size calculations should use + * size-related macros instead of sizeof(). + * Relevant macros: sbi->sit_journal_entries, sit_in_journal(), + * segno_in_journal(), MAX_SIT_JENTRIES(). + */ struct sit_journal { - struct sit_journal_entry entries[SIT_JOURNAL_ENTRIES]; - __u8 reserved[SIT_JOURNAL_RESERVED]; + struct sit_journal_entry entries[0]; } __packed; struct f2fs_extra_info { __le64 kbytes_written; - __u8 reserved[EXTRA_INFO_RESERVED]; + __u8 reserved[]; } __packed; struct f2fs_journal { @@ -531,11 +526,33 @@ struct f2fs_journal { }; } __packed; -/* Block-sized summary block structure */ +/* + * Block-sized summary block structure + * + * The f2fs_summary_block structure is a placeholder whose actual size varies + * depending on the use of packed_ssa. Therefore, it must always be accessed + * only through specific sets of macros and fields, and size calculations should + * use size-related macros instead of sizeof(). + * Relevant macros: sbi->sum_blocksize, sbi->entries_in_sum, + * sbi->sum_entry_size, sum_entries(), sum_journal(), sum_footer(). + * + * Summary Block Layout + * + * +-----------------------+ <--- Block Start + * | struct f2fs_summary | + * | entries[0] | + * | ... | + * | entries[N-1] | + * +-----------------------+ + * | struct f2fs_journal | + * +-----------------------+ + * | struct summary_footer | + * +-----------------------+ <--- Block End + */ struct f2fs_summary_block { - struct f2fs_summary entries[ENTRIES_IN_SUM]; - struct f2fs_journal journal; - struct summary_footer footer; + struct f2fs_summary entries[0]; + // struct f2fs_journal journal; + // struct summary_footer footer; } __packed; /* diff --git a/include/linux/filter.h b/include/linux/filter.h index fd54fed8f95f3b..7452817d707d19 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1375,24 +1375,13 @@ static inline bool bpf_jit_kallsyms_enabled(void) return false; } -int __bpf_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char *sym); +int bpf_address_lookup(unsigned long addr, unsigned long *size, + unsigned long *off, char *sym); bool is_bpf_text_address(unsigned long addr); int bpf_get_kallsym(unsigned int symnum, unsigned long *value, char *type, char *sym); struct bpf_prog *bpf_prog_ksym_find(unsigned long addr); -static inline int -bpf_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char **modname, char *sym) -{ - int ret = __bpf_address_lookup(addr, size, off, sym); - - if (ret && modname) - *modname = NULL; - return ret; -} - void bpf_prog_kallsyms_add(struct bpf_prog *fp); void bpf_prog_kallsyms_del(struct bpf_prog *fp); @@ -1431,8 +1420,8 @@ static inline bool bpf_jit_kallsyms_enabled(void) } static inline int -__bpf_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char *sym) +bpf_address_lookup(unsigned long addr, unsigned long *size, + unsigned long *off, char *sym) { return 0; } @@ -1453,13 +1442,6 @@ static inline struct bpf_prog *bpf_prog_ksym_find(unsigned long addr) return NULL; } -static inline int -bpf_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char **modname, char *sym) -{ - return 0; -} - static inline void bpf_prog_kallsyms_add(struct bpf_prog *fp) { } diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index a3a8989e3268da..fa74ae5cc9dae0 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -87,11 +87,13 @@ struct ftrace_hash; defined(CONFIG_DYNAMIC_FTRACE) int ftrace_mod_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char **modname, char *sym); + unsigned long *off, char **modname, + const unsigned char **modbuildid, char *sym); #else static inline int ftrace_mod_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char **modname, char *sym) + unsigned long *off, char **modname, + const unsigned char **modbuildid, char *sym) { return 0; } @@ -359,7 +361,6 @@ enum { FTRACE_OPS_FL_DIRECT = BIT(17), FTRACE_OPS_FL_SUBOP = BIT(18), FTRACE_OPS_FL_GRAPH = BIT(19), - FTRACE_OPS_FL_JMP = BIT(20), }; #ifndef CONFIG_DYNAMIC_FTRACE_WITH_ARGS diff --git a/include/linux/fwnode.h b/include/linux/fwnode.h index 097be89487bf5c..53e46648131d91 100644 --- a/include/linux/fwnode.h +++ b/include/linux/fwnode.h @@ -229,5 +229,6 @@ int fwnode_link_add(struct fwnode_handle *con, struct fwnode_handle *sup, void fwnode_links_purge(struct fwnode_handle *fwnode); void fw_devlink_purge_absent_suppliers(struct fwnode_handle *fwnode); bool fw_devlink_is_strict(void); +int fw_devlink_count_absent_consumers(struct fwnode_handle *fwnode); #endif diff --git a/include/linux/hid.h b/include/linux/hid.h index dce862cafbbd35..5952d95a53e8c5 100644 --- a/include/linux/hid.h +++ b/include/linux/hid.h @@ -625,7 +625,9 @@ struct hid_input { enum hid_type { HID_TYPE_OTHER = 0, HID_TYPE_USBMOUSE, - HID_TYPE_USBNONE + HID_TYPE_USBNONE, + HID_TYPE_SPI_KEYBOARD, + HID_TYPE_SPI_MOUSE, }; enum hid_battery_status { @@ -786,6 +788,8 @@ struct hid_descriptor { .bus = BUS_BLUETOOTH, .vendor = (ven), .product = (prod) #define HID_I2C_DEVICE(ven, prod) \ .bus = BUS_I2C, .vendor = (ven), .product = (prod) +#define HID_SPI_DEVICE(ven, prod) \ + .bus = BUS_SPI, .vendor = (ven), .product = (prod) #define HID_REPORT_ID(rep) \ .report_type = (rep) diff --git a/include/linux/hisi_acc_qm.h b/include/linux/hisi_acc_qm.h index ca1ec437a3ca3b..59f9858049586a 100644 --- a/include/linux/hisi_acc_qm.h +++ b/include/linux/hisi_acc_qm.h @@ -447,12 +447,16 @@ struct hisi_qp_ops { int (*fill_sqe)(void *sqe, void *q_parm, void *d_parm); }; +struct instance_backlog { + struct list_head list; + spinlock_t lock; +}; + struct hisi_qp { u32 qp_id; u16 sq_depth; u16 cq_depth; u8 alg_type; - u8 req_type; struct qm_dma qdma; void *sqe; @@ -462,7 +466,6 @@ struct hisi_qp { struct hisi_qp_status qp_status; struct hisi_qp_ops *hw_ops; - void *qp_ctx; void (*req_cb)(struct hisi_qp *qp, void *data); void (*event_cb)(struct hisi_qp *qp); @@ -471,6 +474,10 @@ struct hisi_qp { bool is_in_kernel; u16 pasid; struct uacce_queue *uacce_q; + + spinlock_t qp_lock; + struct instance_backlog backlog; + const void **msg; }; static inline int vfs_num_set(const char *val, const struct kernel_param *kp) @@ -575,7 +582,7 @@ struct hisi_acc_sgl_pool *hisi_acc_create_sgl_pool(struct device *dev, void hisi_acc_free_sgl_pool(struct device *dev, struct hisi_acc_sgl_pool *pool); int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num, - u8 alg_type, int node, struct hisi_qp **qps); + u8 *alg_type, int node, struct hisi_qp **qps); void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num); void hisi_qm_dev_shutdown(struct pci_dev *pdev); void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list); diff --git a/include/linux/hw_random.h b/include/linux/hw_random.h index b424555753b11f..b77bc55a4cf356 100644 --- a/include/linux/hw_random.h +++ b/include/linux/hw_random.h @@ -15,6 +15,7 @@ #include #include #include +#include /** * struct hwrng - Hardware Random Number Generator driver @@ -48,6 +49,7 @@ struct hwrng { /* internal. */ struct list_head list; struct kref ref; + struct work_struct cleanup_work; struct completion cleanup_done; struct completion dying; }; diff --git a/include/linux/input/adp5589.h b/include/linux/input/adp5589.h deleted file mode 100644 index 0e4742c8c81e3c..00000000000000 --- a/include/linux/input/adp5589.h +++ /dev/null @@ -1,180 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Analog Devices ADP5589/ADP5585 I/O Expander and QWERTY Keypad Controller - * - * Copyright 2010-2011 Analog Devices Inc. - */ - -#ifndef _ADP5589_H -#define _ADP5589_H - -/* - * ADP5589 specific GPI and Keymap defines - */ - -#define ADP5589_KEYMAPSIZE 88 - -#define ADP5589_GPI_PIN_ROW0 97 -#define ADP5589_GPI_PIN_ROW1 98 -#define ADP5589_GPI_PIN_ROW2 99 -#define ADP5589_GPI_PIN_ROW3 100 -#define ADP5589_GPI_PIN_ROW4 101 -#define ADP5589_GPI_PIN_ROW5 102 -#define ADP5589_GPI_PIN_ROW6 103 -#define ADP5589_GPI_PIN_ROW7 104 -#define ADP5589_GPI_PIN_COL0 105 -#define ADP5589_GPI_PIN_COL1 106 -#define ADP5589_GPI_PIN_COL2 107 -#define ADP5589_GPI_PIN_COL3 108 -#define ADP5589_GPI_PIN_COL4 109 -#define ADP5589_GPI_PIN_COL5 110 -#define ADP5589_GPI_PIN_COL6 111 -#define ADP5589_GPI_PIN_COL7 112 -#define ADP5589_GPI_PIN_COL8 113 -#define ADP5589_GPI_PIN_COL9 114 -#define ADP5589_GPI_PIN_COL10 115 -#define GPI_LOGIC1 116 -#define GPI_LOGIC2 117 - -#define ADP5589_GPI_PIN_ROW_BASE ADP5589_GPI_PIN_ROW0 -#define ADP5589_GPI_PIN_ROW_END ADP5589_GPI_PIN_ROW7 -#define ADP5589_GPI_PIN_COL_BASE ADP5589_GPI_PIN_COL0 -#define ADP5589_GPI_PIN_COL_END ADP5589_GPI_PIN_COL10 - -#define ADP5589_GPI_PIN_BASE ADP5589_GPI_PIN_ROW_BASE -#define ADP5589_GPI_PIN_END ADP5589_GPI_PIN_COL_END - -#define ADP5589_GPIMAPSIZE_MAX (ADP5589_GPI_PIN_END - ADP5589_GPI_PIN_BASE + 1) - -/* - * ADP5585 specific GPI and Keymap defines - */ - -#define ADP5585_KEYMAPSIZE 30 - -#define ADP5585_GPI_PIN_ROW0 37 -#define ADP5585_GPI_PIN_ROW1 38 -#define ADP5585_GPI_PIN_ROW2 39 -#define ADP5585_GPI_PIN_ROW3 40 -#define ADP5585_GPI_PIN_ROW4 41 -#define ADP5585_GPI_PIN_ROW5 42 -#define ADP5585_GPI_PIN_COL0 43 -#define ADP5585_GPI_PIN_COL1 44 -#define ADP5585_GPI_PIN_COL2 45 -#define ADP5585_GPI_PIN_COL3 46 -#define ADP5585_GPI_PIN_COL4 47 -#define GPI_LOGIC 48 - -#define ADP5585_GPI_PIN_ROW_BASE ADP5585_GPI_PIN_ROW0 -#define ADP5585_GPI_PIN_ROW_END ADP5585_GPI_PIN_ROW5 -#define ADP5585_GPI_PIN_COL_BASE ADP5585_GPI_PIN_COL0 -#define ADP5585_GPI_PIN_COL_END ADP5585_GPI_PIN_COL4 - -#define ADP5585_GPI_PIN_BASE ADP5585_GPI_PIN_ROW_BASE -#define ADP5585_GPI_PIN_END ADP5585_GPI_PIN_COL_END - -#define ADP5585_GPIMAPSIZE_MAX (ADP5585_GPI_PIN_END - ADP5585_GPI_PIN_BASE + 1) - -struct adp5589_gpi_map { - unsigned short pin; - unsigned short sw_evt; -}; - -/* scan_cycle_time */ -#define ADP5589_SCAN_CYCLE_10ms 0 -#define ADP5589_SCAN_CYCLE_20ms 1 -#define ADP5589_SCAN_CYCLE_30ms 2 -#define ADP5589_SCAN_CYCLE_40ms 3 - -/* RESET_CFG */ -#define RESET_PULSE_WIDTH_500us 0 -#define RESET_PULSE_WIDTH_1ms 1 -#define RESET_PULSE_WIDTH_2ms 2 -#define RESET_PULSE_WIDTH_10ms 3 - -#define RESET_TRIG_TIME_0ms (0 << 2) -#define RESET_TRIG_TIME_1000ms (1 << 2) -#define RESET_TRIG_TIME_1500ms (2 << 2) -#define RESET_TRIG_TIME_2000ms (3 << 2) -#define RESET_TRIG_TIME_2500ms (4 << 2) -#define RESET_TRIG_TIME_3000ms (5 << 2) -#define RESET_TRIG_TIME_3500ms (6 << 2) -#define RESET_TRIG_TIME_4000ms (7 << 2) - -#define RESET_PASSTHRU_EN (1 << 5) -#define RESET1_POL_HIGH (1 << 6) -#define RESET1_POL_LOW (0 << 6) -#define RESET2_POL_HIGH (1 << 7) -#define RESET2_POL_LOW (0 << 7) - -/* ADP5589 Mask Bits: - * C C C C C C C C C C C | R R R R R R R R - * 1 9 8 7 6 5 4 3 2 1 0 | 7 6 5 4 3 2 1 0 - * 0 - * ---------------- BIT ------------------ - * 1 1 1 1 1 1 1 1 1 0 0 | 0 0 0 0 0 0 0 0 - * 8 7 6 5 4 3 2 1 0 9 8 | 7 6 5 4 3 2 1 0 - */ - -#define ADP_ROW(x) (1 << (x)) -#define ADP_COL(x) (1 << (x + 8)) -#define ADP5589_ROW_MASK 0xFF -#define ADP5589_COL_MASK 0xFF -#define ADP5589_COL_SHIFT 8 -#define ADP5589_MAX_ROW_NUM 7 -#define ADP5589_MAX_COL_NUM 10 - -/* ADP5585 Mask Bits: - * C C C C C | R R R R R R - * 4 3 2 1 0 | 5 4 3 2 1 0 - * - * ---- BIT -- ----------- - * 1 0 0 0 0 | 0 0 0 0 0 0 - * 0 9 8 7 6 | 5 4 3 2 1 0 - */ - -#define ADP5585_ROW_MASK 0x3F -#define ADP5585_COL_MASK 0x1F -#define ADP5585_ROW_SHIFT 0 -#define ADP5585_COL_SHIFT 6 -#define ADP5585_MAX_ROW_NUM 5 -#define ADP5585_MAX_COL_NUM 4 - -#define ADP5585_ROW(x) (1 << ((x) & ADP5585_ROW_MASK)) -#define ADP5585_COL(x) (1 << (((x) & ADP5585_COL_MASK) + ADP5585_COL_SHIFT)) - -/* Put one of these structures in i2c_board_info platform_data */ - -struct adp5589_kpad_platform_data { - unsigned keypad_en_mask; /* Keypad (Rows/Columns) enable mask */ - const unsigned short *keymap; /* Pointer to keymap */ - unsigned short keymapsize; /* Keymap size */ - bool repeat; /* Enable key repeat */ - bool en_keylock; /* Enable key lock feature (ADP5589 only)*/ - unsigned char unlock_key1; /* Unlock Key 1 (ADP5589 only) */ - unsigned char unlock_key2; /* Unlock Key 2 (ADP5589 only) */ - unsigned char unlock_timer; /* Time in seconds [0..7] between the two unlock keys 0=disable (ADP5589 only) */ - unsigned char scan_cycle_time; /* Time between consecutive scan cycles */ - unsigned char reset_cfg; /* Reset config */ - unsigned short reset1_key_1; /* Reset Key 1 */ - unsigned short reset1_key_2; /* Reset Key 2 */ - unsigned short reset1_key_3; /* Reset Key 3 */ - unsigned short reset2_key_1; /* Reset Key 1 */ - unsigned short reset2_key_2; /* Reset Key 2 */ - unsigned debounce_dis_mask; /* Disable debounce mask */ - unsigned pull_dis_mask; /* Disable all pull resistors mask */ - unsigned pullup_en_100k; /* Pull-Up 100k Enable Mask */ - unsigned pullup_en_300k; /* Pull-Up 300k Enable Mask */ - unsigned pulldown_en_300k; /* Pull-Down 300k Enable Mask */ - const struct adp5589_gpi_map *gpimap; - unsigned short gpimapsize; - const struct adp5589_gpio_platform_data *gpio_data; -}; - -struct i2c_client; /* forward declaration */ - -struct adp5589_gpio_platform_data { - int gpio_start; /* GPIO Chip base # */ -}; - -#endif diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h index f479ef5b3341cf..fa1f328d671204 100644 --- a/include/linux/intel_rapl.h +++ b/include/linux/intel_rapl.h @@ -152,7 +152,7 @@ struct rapl_if_priv { union rapl_reg reg_unit; union rapl_reg regs[RAPL_DOMAIN_MAX][RAPL_DOMAIN_REG_MAX]; int limits[RAPL_DOMAIN_MAX]; - int (*read_raw)(int id, struct reg_action *ra, bool atomic); + int (*read_raw)(int id, struct reg_action *ra, bool pmu_ctx); int (*write_raw)(int id, struct reg_action *ra); void *defaults; void *rpi; diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index 266f2b39213a01..b2bb878abd113c 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h @@ -228,7 +228,7 @@ static inline int __must_check devm_request_irq(struct device *dev, unsigned int irq, irq_handler_t handler, unsigned long irqflags, const char *devname, void *dev_id) { - return devm_request_threaded_irq(dev, irq, handler, NULL, irqflags, + return devm_request_threaded_irq(dev, irq, handler, NULL, irqflags | IRQF_COND_ONESHOT, devname, dev_id); } diff --git a/include/linux/io-pgtable.h b/include/linux/io-pgtable.h index 7a1516011ccf72..5fc33ac318dec2 100644 --- a/include/linux/io-pgtable.h +++ b/include/linux/io-pgtable.h @@ -176,7 +176,7 @@ struct io_pgtable_cfg { } arm_mali_lpae_cfg; struct { - u64 ttbr[4]; + void *ttbr[4]; u32 n_ttbrs; u32 n_levels; } apple_dart_cfg; diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index a3e8ddc9b380f8..4c9770536eb5d9 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -444,6 +444,9 @@ struct io_ring_ctx { struct list_head defer_list; unsigned nr_drained; + /* protected by ->completion_lock */ + unsigned nr_req_allocated; + #ifdef CONFIG_NET_RX_BUSY_POLL struct list_head napi_list; /* track busy poll napi_id */ spinlock_t napi_lock; /* napi_list lock */ @@ -456,10 +459,6 @@ struct io_ring_ctx { DECLARE_HASHTABLE(napi_ht, 4); #endif - /* protected by ->completion_lock */ - unsigned evfd_last_cq_tail; - unsigned nr_req_allocated; - /* * Protection for resize vs mmap races - both the mmap and resize * side will need to grab this lock, to prevent either side from diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 8c66284a91a8b0..6e593b7dcc76c5 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -289,12 +289,18 @@ enum iommu_resv_type { IOMMU_RESV_MSI, /* Software-managed MSI translation window */ IOMMU_RESV_SW_MSI, + /* + * Memory regions which must be mapped with the specified mapping + * at all times. + */ + IOMMU_RESV_TRANSLATED, }; /** * struct iommu_resv_region - descriptor for a reserved memory region * @list: Linked list pointers * @start: System physical start address of the region + * @start: Device virtual start address of the region for IOMMU_RESV_TRANSLATED * @length: Length of the region in bytes * @prot: IOMMU Protection flags (READ/WRITE/...) * @type: Type of the reserved region @@ -303,6 +309,7 @@ enum iommu_resv_type { struct iommu_resv_region { struct list_head list; phys_addr_t start; + dma_addr_t dva; size_t length; int prot; enum iommu_resv_type type; @@ -838,6 +845,7 @@ struct iommu_fault_param { * @pci_32bit_workaround: Limit DMA allocations to 32-bit IOVAs * @require_direct: device requires IOMMU_RESV_DIRECT regions * @shadow_on_flush: IOTLB flushes are used to sync shadow tables + * @require_translated: device requires IOMMU_RESV_TRANSLATED regions * * TODO: migrate other per device data pointers under iommu_dev_data, e.g. * struct iommu_group *iommu_group; @@ -853,6 +861,7 @@ struct dev_iommu { u32 pci_32bit_workaround:1; u32 require_direct:1; u32 shadow_on_flush:1; + u32 require_translated:1; }; int iommu_device_register(struct iommu_device *iommu, @@ -937,6 +946,9 @@ extern bool iommu_default_passthrough(void); extern struct iommu_resv_region * iommu_alloc_resv_region(phys_addr_t start, size_t length, int prot, enum iommu_resv_type type, gfp_t gfp); +extern struct iommu_resv_region * +iommu_alloc_resv_region_tr(phys_addr_t start, dma_addr_t dva_start, size_t length, + int prot, enum iommu_resv_type type, gfp_t gfp); extern int iommu_get_group_resv_regions(struct iommu_group *group, struct list_head *head); diff --git a/include/linux/leds-expresswire.h b/include/linux/leds-expresswire.h index a422921f4159f2..7f8c4795f69fa0 100644 --- a/include/linux/leds-expresswire.h +++ b/include/linux/leds-expresswire.h @@ -30,9 +30,6 @@ struct expresswire_common_props { void expresswire_power_off(struct expresswire_common_props *props); void expresswire_enable(struct expresswire_common_props *props); -void expresswire_start(struct expresswire_common_props *props); -void expresswire_end(struct expresswire_common_props *props); -void expresswire_set_bit(struct expresswire_common_props *props, bool bit); void expresswire_write_u8(struct expresswire_common_props *props, u8 val); #endif /* _LEDS_EXPRESSWIRE_H */ diff --git a/include/linux/libata.h b/include/linux/libata.h index 39534fafa36ae1..a5c66eb708f466 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -75,7 +75,7 @@ enum ata_quirks { __ATA_QUIRK_NO_DMA_LOG, /* Do not use DMA for log read */ __ATA_QUIRK_NOTRIM, /* Do not use TRIM */ __ATA_QUIRK_MAX_SEC_1024, /* Limit max sects to 1024 */ - __ATA_QUIRK_MAX_SEC_8191, /* Limit max sects to 8191 */ + __ATA_QUIRK_MAX_SEC, /* Limit max sectors */ __ATA_QUIRK_MAX_TRIM_128M, /* Limit max trim size to 128M */ __ATA_QUIRK_NO_NCQ_ON_ATI, /* Disable NCQ on ATI chipset */ __ATA_QUIRK_NO_LPM_ON_ATI, /* Disable LPM on ATI chipset */ @@ -116,7 +116,7 @@ enum { ATA_QUIRK_NO_DMA_LOG = (1U << __ATA_QUIRK_NO_DMA_LOG), ATA_QUIRK_NOTRIM = (1U << __ATA_QUIRK_NOTRIM), ATA_QUIRK_MAX_SEC_1024 = (1U << __ATA_QUIRK_MAX_SEC_1024), - ATA_QUIRK_MAX_SEC_8191 = (1U << __ATA_QUIRK_MAX_SEC_8191), + ATA_QUIRK_MAX_SEC = (1U << __ATA_QUIRK_MAX_SEC), ATA_QUIRK_MAX_TRIM_128M = (1U << __ATA_QUIRK_MAX_TRIM_128M), ATA_QUIRK_NO_NCQ_ON_ATI = (1U << __ATA_QUIRK_NO_NCQ_ON_ATI), ATA_QUIRK_NO_LPM_ON_ATI = (1U << __ATA_QUIRK_NO_LPM_ON_ATI), @@ -903,6 +903,9 @@ struct ata_port { u64 qc_active; int nr_active_links; /* #links with active qcs */ + struct work_struct deferred_qc_work; + struct ata_queued_cmd *deferred_qc; + struct ata_link link; /* host default link */ struct ata_link *slave_link; /* see ata_slave_link_init() */ diff --git a/include/linux/memory_ordering_model.h b/include/linux/memory_ordering_model.h new file mode 100644 index 00000000000000..267a12ca66307e --- /dev/null +++ b/include/linux/memory_ordering_model.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_MEMORY_ORDERING_MODEL_H +#define __ASM_MEMORY_ORDERING_MODEL_H + +/* Arch hooks to implement the PR_{GET_SET}_MEM_MODEL prctls */ + +struct task_struct; +int arch_prctl_mem_model_get(struct task_struct *t); +int arch_prctl_mem_model_set(struct task_struct *t, unsigned long val); + +#endif diff --git a/include/linux/mfd/wm8350/core.h b/include/linux/mfd/wm8350/core.h index 5f70d3b5d1b1a4..097ef4dfcdac8c 100644 --- a/include/linux/mfd/wm8350/core.h +++ b/include/linux/mfd/wm8350/core.h @@ -667,7 +667,7 @@ static inline int wm8350_register_irq(struct wm8350 *wm8350, int irq, return -ENODEV; return request_threaded_irq(irq + wm8350->irq_base, NULL, - handler, flags, name, data); + handler, flags | IRQF_ONESHOT, name, data); } static inline void wm8350_free_irq(struct wm8350 *wm8350, int irq, void *data) diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 1c54aa6f74fbc6..1967d1c79139bb 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -1281,12 +1281,12 @@ static inline bool mlx5_rl_is_supported(struct mlx5_core_dev *dev) static inline int mlx5_core_is_mp_slave(struct mlx5_core_dev *dev) { return MLX5_CAP_GEN(dev, affiliate_nic_vport_criteria) && - MLX5_CAP_GEN(dev, num_vhca_ports) <= 1; + MLX5_CAP_GEN_MAX(dev, num_vhca_ports) <= 1; } static inline int mlx5_core_is_mp_master(struct mlx5_core_dev *dev) { - return MLX5_CAP_GEN(dev, num_vhca_ports) > 1; + return MLX5_CAP_GEN_MAX(dev, num_vhca_ports) > 1; } static inline int mlx5_core_mp_enabled(struct mlx5_core_dev *dev) diff --git a/include/linux/module.h b/include/linux/module.h index d80c3ea5747266..ac254525014cd6 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -748,6 +748,15 @@ static inline void __module_get(struct module *module) __mod ? __mod->name : "kernel"; \ }) +static inline const unsigned char *module_buildid(struct module *mod) +{ +#ifdef CONFIG_STACKTRACE_BUILD_ID + return mod->build_id; +#else + return NULL; +#endif +} + /* Dereference module function descriptor */ void *dereference_module_function_descriptor(struct module *mod, void *ptr); diff --git a/include/linux/mtd/spinand.h b/include/linux/mtd/spinand.h index ce76f5c632e179..049d55c38d5228 100644 --- a/include/linux/mtd/spinand.h +++ b/include/linux/mtd/spinand.h @@ -287,7 +287,7 @@ struct spinand_device; /** * struct spinand_id - SPI NAND id structure - * @data: buffer containing the id bytes. Currently 4 bytes large, but can + * @data: buffer containing the id bytes. Currently 5 bytes large, but can * be extended if required * @len: ID length */ diff --git a/include/linux/pci-epc.h b/include/linux/pci-epc.h index 4286bfdbfdfad2..c021c7af175fd0 100644 --- a/include/linux/pci-epc.h +++ b/include/linux/pci-epc.h @@ -223,6 +223,13 @@ struct pci_epc_bar_desc { /** * struct pci_epc_features - features supported by a EPC device per function * @linkup_notifier: indicate if the EPC device can notify EPF driver on link up + * @dynamic_inbound_mapping: indicate if the EPC device supports updating + * inbound mappings for an already configured BAR + * (i.e. allow calling pci_epc_set_bar() again + * without first calling pci_epc_clear_bar()) + * @subrange_mapping: indicate if the EPC device can map inbound subranges for a + * BAR. This feature depends on @dynamic_inbound_mapping + * feature. * @msi_capable: indicate if the endpoint function has MSI capability * @msix_capable: indicate if the endpoint function has MSI-X capability * @intx_capable: indicate if the endpoint can raise INTx interrupts @@ -231,6 +238,8 @@ struct pci_epc_bar_desc { */ struct pci_epc_features { unsigned int linkup_notifier : 1; + unsigned int dynamic_inbound_mapping : 1; + unsigned int subrange_mapping : 1; unsigned int msi_capable : 1; unsigned int msix_capable : 1; unsigned int intx_capable : 1; diff --git a/include/linux/pci-epf.h b/include/linux/pci-epf.h index 48f68c4dcfa5e2..7737a7c03260d9 100644 --- a/include/linux/pci-epf.h +++ b/include/linux/pci-epf.h @@ -110,6 +110,22 @@ struct pci_epf_driver { #define to_pci_epf_driver(drv) container_of_const((drv), struct pci_epf_driver, driver) +/** + * struct pci_epf_bar_submap - BAR subrange for inbound mapping + * @phys_addr: target physical/DMA address for this subrange + * @size: the size of the subrange to be mapped + * + * When pci_epf_bar.num_submap is >0, pci_epf_bar.submap describes the + * complete BAR layout. This allows an EPC driver to program multiple + * inbound translation windows for a single BAR when supported by the + * controller. The array order defines the BAR layout (submap[0] at offset + * 0, and each immediately follows the previous one). + */ +struct pci_epf_bar_submap { + dma_addr_t phys_addr; + size_t size; +}; + /** * struct pci_epf_bar - represents the BAR of EPF device * @phys_addr: physical address that should be mapped to the BAR @@ -119,6 +135,9 @@ struct pci_epf_driver { * requirement * @barno: BAR number * @flags: flags that are set for the BAR + * @num_submap: number of entries in @submap + * @submap: array of subrange descriptors allocated by the caller. See + * struct pci_epf_bar_submap for the semantics in detail. */ struct pci_epf_bar { dma_addr_t phys_addr; @@ -127,6 +146,10 @@ struct pci_epf_bar { size_t mem_size; enum pci_barno barno; int flags; + + /* Optional sub-range mapping */ + unsigned int num_submap; + struct pci_epf_bar_submap *submap; }; /** diff --git a/include/linux/pm_domain.h b/include/linux/pm_domain.h index 93ba0143ca476a..309385f9b4c8f9 100644 --- a/include/linux/pm_domain.h +++ b/include/linux/pm_domain.h @@ -121,6 +121,13 @@ struct dev_pm_domain_list { * powered-off until the ->sync_state() callback is * invoked. This flag informs genpd to allow a * power-off without waiting for ->sync_state(). + * GENPD_FLAG_DEFER_OFF: Defer powerdown if there are any consumer + * device fwlinks indicating that some consumer + * devices have not yet probed. This is useful + * for power domains which are active at boot and + * must not be shut down until all consumers + * complete their probe sequence. + */ #define GENPD_FLAG_PM_CLK (1U << 0) #define GENPD_FLAG_IRQ_SAFE (1U << 1) @@ -133,6 +140,7 @@ struct dev_pm_domain_list { #define GENPD_FLAG_DEV_NAME_FW (1U << 8) #define GENPD_FLAG_NO_SYNC_STATE (1U << 9) #define GENPD_FLAG_NO_STAY_ON (1U << 10) +#define GENPD_FLAG_DEFER_OFF (1U << 11) enum gpd_status { GENPD_STATE_ON = 0, /* PM domain is on */ diff --git a/include/linux/psp.h b/include/linux/psp.h index 92e60aeef21e13..b337dcce1e9916 100644 --- a/include/linux/psp.h +++ b/include/linux/psp.h @@ -18,6 +18,7 @@ * and should include an appropriate local definition in their source file. */ #define PSP_CMDRESP_STS GENMASK(15, 0) +#define PSP_TEE_STS_RING_BUSY 0x0000000d /* Ring already initialized */ #define PSP_CMDRESP_CMD GENMASK(23, 16) #define PSP_CMDRESP_RESERVED GENMASK(29, 24) #define PSP_CMDRESP_RECOVERY BIT(30) diff --git a/include/linux/seq_file.h b/include/linux/seq_file.h index d6ebf0596510c9..2fb266ea69fa4d 100644 --- a/include/linux/seq_file.h +++ b/include/linux/seq_file.h @@ -181,7 +181,6 @@ int seq_open_private(struct file *, const struct seq_operations *, int); int seq_release_private(struct inode *, struct file *); #ifdef CONFIG_BINARY_PRINTF -__printf(2, 0) void seq_bprintf(struct seq_file *m, const char *f, const u32 *binary); #endif diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h index 221123660e710e..f827cc3cb1460d 100644 --- a/include/linux/seqlock.h +++ b/include/linux/seqlock.h @@ -1303,15 +1303,14 @@ __scoped_seqlock_next(struct ss_tmp *sst, seqlock_t *lock, enum ss_state target) __scoped_seqlock_next(&_s, _seqlock, _target)) /** - * scoped_seqlock_read (lock, ss_state) - execute the read side critical - * section without manual sequence - * counter handling or calls to other - * helpers - * @lock: pointer to seqlock_t protecting the data - * @ss_state: one of {ss_lock, ss_lock_irqsave, ss_lockless} indicating - * the type of critical read section - * - * Example: + * scoped_seqlock_read() - execute the read-side critical section + * without manual sequence counter handling + * or calls to other helpers + * @_seqlock: pointer to seqlock_t protecting the data + * @_target: an enum ss_state: one of {ss_lock, ss_lock_irqsave, ss_lockless} + * indicating the type of critical read section + * + * Example:: * * scoped_seqlock_read (&lock, ss_lock) { * // read-side critical section diff --git a/include/linux/skmsg.h b/include/linux/skmsg.h index 49847888c287ab..829b281d6c9c27 100644 --- a/include/linux/skmsg.h +++ b/include/linux/skmsg.h @@ -97,6 +97,8 @@ struct sk_psock { struct sk_buff_head ingress_skb; struct list_head ingress_msg; spinlock_t ingress_lock; + /** @msg_tot_len: Total bytes queued in ingress_msg list. */ + u32 msg_tot_len; unsigned long state; struct list_head link; spinlock_t link_lock; @@ -141,6 +143,8 @@ int sk_msg_memcopy_from_iter(struct sock *sk, struct iov_iter *from, struct sk_msg *msg, u32 bytes); int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, int len, int flags); +int __sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, + int len, int flags, int *copied_from_self); bool sk_msg_is_readable(struct sock *sk); static inline void sk_msg_check_to_free(struct sk_msg *msg, u32 i, u32 bytes) @@ -319,6 +323,27 @@ static inline void sock_drop(struct sock *sk, struct sk_buff *skb) kfree_skb(skb); } +static inline u32 sk_psock_get_msg_len_nolock(struct sk_psock *psock) +{ + /* Used by ioctl to read msg_tot_len only; lock-free for performance */ + return READ_ONCE(psock->msg_tot_len); +} + +static inline void sk_psock_msg_len_add_locked(struct sk_psock *psock, int diff) +{ + /* Use WRITE_ONCE to ensure correct read in sk_psock_get_msg_len_nolock(). + * ingress_lock should be held to prevent concurrent updates to msg_tot_len + */ + WRITE_ONCE(psock->msg_tot_len, psock->msg_tot_len + diff); +} + +static inline void sk_psock_msg_len_add(struct sk_psock *psock, int diff) +{ + spin_lock_bh(&psock->ingress_lock); + sk_psock_msg_len_add_locked(psock, diff); + spin_unlock_bh(&psock->ingress_lock); +} + static inline bool sk_psock_queue_msg(struct sk_psock *psock, struct sk_msg *msg) { @@ -327,6 +352,7 @@ static inline bool sk_psock_queue_msg(struct sk_psock *psock, spin_lock_bh(&psock->ingress_lock); if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) { list_add_tail(&msg->list, &psock->ingress_msg); + sk_psock_msg_len_add_locked(psock, msg->sg.size); ret = true; } else { sk_msg_free(psock->sk, msg); @@ -343,18 +369,25 @@ static inline struct sk_msg *sk_psock_dequeue_msg(struct sk_psock *psock) spin_lock_bh(&psock->ingress_lock); msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list); - if (msg) + if (msg) { list_del(&msg->list); + sk_psock_msg_len_add_locked(psock, -msg->sg.size); + } spin_unlock_bh(&psock->ingress_lock); return msg; } +static inline struct sk_msg *sk_psock_peek_msg_locked(struct sk_psock *psock) +{ + return list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list); +} + static inline struct sk_msg *sk_psock_peek_msg(struct sk_psock *psock) { struct sk_msg *msg; spin_lock_bh(&psock->ingress_lock); - msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list); + msg = sk_psock_peek_msg_locked(psock); spin_unlock_bh(&psock->ingress_lock); return msg; } @@ -521,6 +554,39 @@ static inline bool sk_psock_strp_enabled(struct sk_psock *psock) return !!psock->saved_data_ready; } +/* for tcp only, sk is locked */ +static inline ssize_t sk_psock_msg_inq(struct sock *sk) +{ + struct sk_psock *psock; + ssize_t inq = 0; + + psock = sk_psock_get(sk); + if (likely(psock)) { + inq = sk_psock_get_msg_len_nolock(psock); + sk_psock_put(sk, psock); + } + return inq; +} + +/* for udp only, sk is not locked */ +static inline ssize_t sk_msg_first_len(struct sock *sk) +{ + struct sk_psock *psock; + struct sk_msg *msg; + ssize_t inq = 0; + + psock = sk_psock_get(sk); + if (likely(psock)) { + spin_lock_bh(&psock->ingress_lock); + msg = sk_psock_peek_msg_locked(psock); + if (msg) + inq = msg->sg.size; + spin_unlock_bh(&psock->ingress_lock); + sk_psock_put(sk, psock); + } + return inq; +} + #if IS_ENABLED(CONFIG_NET_SOCK_MSG) #define BPF_F_STRPARSER (1UL << 1) diff --git a/include/linux/soc/apple/dockchannel.h b/include/linux/soc/apple/dockchannel.h new file mode 100644 index 00000000000000..0b7093935ddf47 --- /dev/null +++ b/include/linux/soc/apple/dockchannel.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Apple Dockchannel devices + * Copyright (C) The Asahi Linux Contributors + */ +#ifndef _LINUX_APPLE_DOCKCHANNEL_H_ +#define _LINUX_APPLE_DOCKCHANNEL_H_ + +#include +#include +#include + +#if IS_ENABLED(CONFIG_APPLE_DOCKCHANNEL) + +struct dockchannel; + +struct dockchannel *dockchannel_init(struct platform_device *pdev); + +int dockchannel_send(struct dockchannel *dockchannel, const void *buf, size_t count); +int dockchannel_recv(struct dockchannel *dockchannel, void *buf, size_t count); +int dockchannel_await(struct dockchannel *dockchannel, + void (*callback)(void *cookie, size_t avail), + void *cookie, size_t count); + +#endif +#endif diff --git a/drivers/soc/apple/mailbox.h b/include/linux/soc/apple/mailbox.h similarity index 100% rename from drivers/soc/apple/mailbox.h rename to include/linux/soc/apple/mailbox.h diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 736f530180179b..22e1d3bb35ef0c 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -78,6 +78,13 @@ struct apple_rtkit; struct apple_rtkit *devm_apple_rtkit_init(struct device *dev, void *cookie, const char *mbox_name, int mbox_idx, const struct apple_rtkit_ops *ops); +/* + * Frees internal RTKit state allocated by devm_apple_rtkit_init(). + * + * @dev: Pointer to the device node this coprocessor is assocated with + * @rtk: Internal RTKit state initialized by devm_apple_rtkit_init() + */ +void devm_apple_rtkit_free(struct device *dev, struct apple_rtkit *rtk); /* * Non-devm version of devm_apple_rtkit_init. Must be freed with @@ -172,4 +179,12 @@ int apple_rtkit_send_message(struct apple_rtkit *rtk, u8 ep, u64 message, */ int apple_rtkit_poll(struct apple_rtkit *rtk); +/* + * Checks if an endpoint with a given index exists + * + * @rtk: RTKit reference + * @ep: endpoint to check for + */ +bool apple_rtkit_has_endpoint(struct apple_rtkit *rtk, u8 ep); + #endif /* _LINUX_APPLE_RTKIT_H_ */ diff --git a/include/linux/soc/apple/tunable.h b/include/linux/soc/apple/tunable.h new file mode 100644 index 00000000000000..531ca814cd0235 --- /dev/null +++ b/include/linux/soc/apple/tunable.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Apple Silicon hardware tunable support + * + * Each tunable is a list with each entry containing a offset into the MMIO + * region, a mask of bits to be cleared and a set of bits to be set. These + * tunables are passed along by the previous boot stages and vary from device + * to device such that they cannot be hardcoded in the individual drivers. + * + * Copyright (C) The Asahi Linux Contributors + */ + +#ifndef _LINUX_SOC_APPLE_TUNABLE_H_ +#define _LINUX_SOC_APPLE_TUNABLE_H_ + +#include +#include + +/** + * Struct to store an Apple Silicon hardware tunable. + * + * Each tunable is a list with each entry containing a offset into the MMIO + * region, a mask of bits to be cleared and a set of bits to be set. These + * tunables are passed along by the previous boot stages and vary from device + * to device such that they cannot be hardcoded in the individual drivers. + * + * @param sz Number of [offset, mask, value] tuples stored in values. + * @param values [offset, mask, value] array. + */ +struct apple_tunable { + size_t sz; + struct { + u32 offset; + u32 mask; + u32 value; + } values[] __counted_by(sz); +}; + +/** + * Parse an array of hardware tunables from the device tree. + * + * @dev: Device node used for devm_kzalloc internally. + * @np: Device node which contains the tunable array. + * @name: Name of the device tree property which contains the tunables. + * @res: Resource to which the tunables will be applied, used for bound checking + * + * @return: devres allocated struct on success or PTR_ERR on failure. + */ +struct apple_tunable *devm_apple_tunable_parse(struct device *dev, + struct device_node *np, + const char *name, + struct resource *res); + +/** + * Apply a previously loaded hardware tunable. + * + * @param regs: MMIO to which the tunable will be applied. + * @param tunable: Pointer to the tunable. + */ +void apple_tunable_apply(void __iomem *regs, struct apple_tunable *tunable); + +#endif diff --git a/include/linux/soc/qcom/ubwc.h b/include/linux/soc/qcom/ubwc.h index 0a4edfe3d96d4f..f052e241736c4b 100644 --- a/include/linux/soc/qcom/ubwc.h +++ b/include/linux/soc/qcom/ubwc.h @@ -8,6 +8,7 @@ #define __QCOM_UBWC_H__ #include +#include #include struct qcom_ubwc_cfg_data { diff --git a/include/linux/string.h b/include/linux/string.h index 1b564c36d721b9..b850bd91b3d884 100644 --- a/include/linux/string.h +++ b/include/linux/string.h @@ -336,8 +336,8 @@ int __sysfs_match_string(const char * const *array, size_t n, const char *s); #define sysfs_match_string(_a, _s) __sysfs_match_string(_a, ARRAY_SIZE(_a), _s) #ifdef CONFIG_BINARY_PRINTF -__printf(3, 0) int vbin_printf(u32 *bin_buf, size_t size, const char *fmt, va_list args); -__printf(3, 0) int bstr_printf(char *buf, size_t size, const char *fmt, const u32 *bin_buf); +int vbin_printf(u32 *bin_buf, size_t size, const char *fmt, va_list args); +int bstr_printf(char *buf, size_t size, const char *fmt, const u32 *bin_buf); #endif extern ssize_t memory_read_from_buffer(void *to, size_t count, loff_t *ppos, diff --git a/include/linux/sunrpc/xdrgen/_builtins.h b/include/linux/sunrpc/xdrgen/_builtins.h index 66ca3ece951ab9..a5ab75d2db0446 100644 --- a/include/linux/sunrpc/xdrgen/_builtins.h +++ b/include/linux/sunrpc/xdrgen/_builtins.h @@ -188,12 +188,10 @@ xdrgen_decode_string(struct xdr_stream *xdr, string *ptr, u32 maxlen) return false; if (unlikely(maxlen && len > maxlen)) return false; - if (len != 0) { - p = xdr_inline_decode(xdr, len); - if (unlikely(!p)) - return false; - ptr->data = (unsigned char *)p; - } + p = xdr_inline_decode(xdr, len); + if (unlikely(!p)) + return false; + ptr->data = (unsigned char *)p; ptr->len = len; return true; } @@ -219,12 +217,10 @@ xdrgen_decode_opaque(struct xdr_stream *xdr, opaque *ptr, u32 maxlen) return false; if (unlikely(maxlen && len > maxlen)) return false; - if (len != 0) { - p = xdr_inline_decode(xdr, len); - if (unlikely(!p)) - return false; - ptr->data = (u8 *)p; - } + p = xdr_inline_decode(xdr, len); + if (unlikely(!p)) + return false; + ptr->data = (u8 *)p; ptr->len = len; return true; } diff --git a/include/linux/u64_stats_sync.h b/include/linux/u64_stats_sync.h index 457879938fc198..3366090a86bd2f 100644 --- a/include/linux/u64_stats_sync.h +++ b/include/linux/u64_stats_sync.h @@ -89,6 +89,11 @@ static inline void u64_stats_add(u64_stats_t *p, unsigned long val) local64_add(val, &p->v); } +static inline void u64_stats_sub(u64_stats_t *p, s64 val) +{ + local64_sub(val, &p->v); +} + static inline void u64_stats_inc(u64_stats_t *p) { local64_inc(&p->v); @@ -130,6 +135,11 @@ static inline void u64_stats_add(u64_stats_t *p, unsigned long val) p->v += val; } +static inline void u64_stats_sub(u64_stats_t *p, s64 val) +{ + p->v -= val; +} + static inline void u64_stats_inc(u64_stats_t *p) { p->v++; diff --git a/include/linux/xarray.h b/include/linux/xarray.h index be850174e802e6..64f2a5e06cebcc 100644 --- a/include/linux/xarray.h +++ b/include/linux/xarray.h @@ -563,6 +563,8 @@ void *__xa_erase(struct xarray *, unsigned long index); void *__xa_store(struct xarray *, unsigned long index, void *entry, gfp_t); void *__xa_cmpxchg(struct xarray *, unsigned long index, void *old, void *entry, gfp_t); +void *__xa_cmpxchg_raw(struct xarray *, unsigned long index, void *old, + void *entry, gfp_t); int __must_check __xa_insert(struct xarray *, unsigned long index, void *entry, gfp_t); int __must_check __xa_alloc(struct xarray *, u32 *id, void *entry, diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h index 4263e71a23efb2..8aadf4cdead2bd 100644 --- a/include/net/bluetooth/hci_core.h +++ b/include/net/bluetooth/hci_core.h @@ -730,6 +730,8 @@ struct hci_conn { __u16 le_per_adv_data_offset; __u8 le_adv_phy; __u8 le_adv_sec_phy; + __u8 le_tx_def_phys; + __u8 le_rx_def_phys; __u8 le_tx_phy; __u8 le_rx_phy; __s8 rssi; diff --git a/include/net/inet_ecn.h b/include/net/inet_ecn.h index ea32393464a291..827b87a95dab3f 100644 --- a/include/net/inet_ecn.h +++ b/include/net/inet_ecn.h @@ -51,11 +51,25 @@ static inline __u8 INET_ECN_encapsulate(__u8 outer, __u8 inner) return outer; } +/* Apply either ECT(0) or ECT(1) */ +static inline void __INET_ECN_xmit(struct sock *sk, bool use_ect_1) +{ + __u8 ect = use_ect_1 ? INET_ECN_ECT_1 : INET_ECN_ECT_0; + + /* Mask the complete byte in case the connection alternates between + * ECT(0) and ECT(1). + */ + inet_sk(sk)->tos &= ~INET_ECN_MASK; + inet_sk(sk)->tos |= ect; + if (inet6_sk(sk)) { + inet6_sk(sk)->tclass &= ~INET_ECN_MASK; + inet6_sk(sk)->tclass |= ect; + } +} + static inline void INET_ECN_xmit(struct sock *sk) { - inet_sk(sk)->tos |= INET_ECN_ECT_0; - if (inet6_sk(sk) != NULL) - inet6_sk(sk)->tclass |= INET_ECN_ECT_0; + __INET_ECN_xmit(sk, false); } static inline void INET_ECN_dontxmit(struct sock *sk) diff --git a/include/net/ipv6.h b/include/net/ipv6.h index 74fbf1ad8065a6..6a933690e0ff5c 100644 --- a/include/net/ipv6.h +++ b/include/net/ipv6.h @@ -1280,12 +1280,15 @@ int ipv6_sock_mc_drop(struct sock *sk, int ifindex, static inline int ip6_sock_set_v6only(struct sock *sk) { - if (inet_sk(sk)->inet_num) - return -EINVAL; + int ret = 0; + lock_sock(sk); - sk->sk_ipv6only = true; + if (inet_sk(sk)->inet_num) + ret = -EINVAL; + else + sk->sk_ipv6only = true; release_sock(sk); - return 0; + return ret; } static inline void ip6_sock_set_recverr(struct sock *sk) diff --git a/include/net/netfilter/nf_conntrack_count.h b/include/net/netfilter/nf_conntrack_count.h index 52a06de41aa0f9..cf0166520cf338 100644 --- a/include/net/netfilter/nf_conntrack_count.h +++ b/include/net/netfilter/nf_conntrack_count.h @@ -13,6 +13,7 @@ struct nf_conncount_list { u32 last_gc; /* jiffies at most recent gc */ struct list_head head; /* connections with the same filtering key */ unsigned int count; /* length of list */ + unsigned int last_gc_count; /* length of list at most recent gc */ }; struct nf_conncount_data *nf_conncount_init(struct net *net, unsigned int keylen); diff --git a/include/net/netfilter/nf_queue.h b/include/net/netfilter/nf_queue.h index 4aeffddb75861e..45eb26b2e95b37 100644 --- a/include/net/netfilter/nf_queue.h +++ b/include/net/netfilter/nf_queue.h @@ -6,11 +6,13 @@ #include #include #include +#include #include /* Each queued (to userspace) skbuff has one of these. */ struct nf_queue_entry { struct list_head list; + struct rhash_head hash_node; struct sk_buff *skb; unsigned int id; unsigned int hook_index; /* index in hook_entries->hook[] */ @@ -19,7 +21,9 @@ struct nf_queue_entry { struct net_device *physout; #endif struct nf_hook_state state; + bool nf_ct_is_unconfirmed; u16 size; /* sizeof(entry) + saved route keys */ + u16 queue_num; /* extra space to store route keys */ }; diff --git a/include/net/netfilter/nf_tables.h b/include/net/netfilter/nf_tables.h index 0e266c2d0e7f07..05f57ba6224476 100644 --- a/include/net/netfilter/nf_tables.h +++ b/include/net/netfilter/nf_tables.h @@ -278,6 +278,8 @@ struct nft_userdata { unsigned char data[]; }; +#define NFT_SET_ELEM_INTERNAL_LAST 0x1 + /* placeholder structure for opaque set element backend representation. */ struct nft_elem_priv { }; @@ -287,6 +289,7 @@ struct nft_elem_priv { }; * @key: element key * @key_end: closing element key * @data: element data + * @flags: flags * @priv: element private data and extensions */ struct nft_set_elem { @@ -302,6 +305,7 @@ struct nft_set_elem { u32 buf[NFT_DATA_VALUE_MAXLEN / sizeof(u32)]; struct nft_data val; } data; + u32 flags; struct nft_elem_priv *priv; }; @@ -452,6 +456,7 @@ struct nft_set_ext; * @init: initialize private data of new set instance * @destroy: destroy private data of set instance * @gc_init: initialize garbage collection + * @abort_skip_removal: skip removal of elements from abort path * @elemsize: element private size * * Operations lookup, update and delete have simpler interfaces, are faster @@ -509,6 +514,7 @@ struct nft_set_ops { const struct nft_set *set); void (*gc_init)(const struct nft_set *set); + bool abort_skip_removal; unsigned int elemsize; }; diff --git a/include/net/netns/ipv4.h b/include/net/netns/ipv4.h index 2dbd46fc4734b7..8e971c7bf16464 100644 --- a/include/net/netns/ipv4.h +++ b/include/net/netns/ipv4.h @@ -88,6 +88,12 @@ struct netns_ipv4 { int sysctl_tcp_rcvbuf_low_rtt; __cacheline_group_end(netns_ipv4_read_rx); + /* ICMP rate limiter hot cache line. */ + __cacheline_group_begin_aligned(icmp); + atomic_t icmp_global_credit; + u32 icmp_global_stamp; + __cacheline_group_end_aligned(icmp); + struct inet_timewait_death_row tcp_death_row; struct udp_table *udp_table; @@ -141,8 +147,7 @@ struct netns_ipv4 { int sysctl_icmp_ratemask; int sysctl_icmp_msgs_per_sec; int sysctl_icmp_msgs_burst; - atomic_t icmp_global_credit; - u32 icmp_global_stamp; + u32 ip_rt_min_pmtu; int ip_rt_mtu_expires; int ip_rt_min_advmss; diff --git a/include/net/tcp.h b/include/net/tcp.h index 0deb5e9dd91146..e0a5cf2f78181c 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -1203,7 +1203,15 @@ enum tcp_ca_ack_event_flags { #define TCP_CONG_NON_RESTRICTED BIT(0) /* Requires ECN/ECT set on all packets */ #define TCP_CONG_NEEDS_ECN BIT(1) -#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN) +/* Require successfully negotiated AccECN capability */ +#define TCP_CONG_NEEDS_ACCECN BIT(2) +/* Use ECT(1) instead of ECT(0) while the CA is uninitialized */ +#define TCP_CONG_ECT_1_NEGOTIATION BIT(3) +/* Cannot fallback to RFC3168 during AccECN negotiation */ +#define TCP_CONG_NO_FALLBACK_RFC3168 BIT(4) +#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN | \ + TCP_CONG_NEEDS_ACCECN | TCP_CONG_ECT_1_NEGOTIATION | \ + TCP_CONG_NO_FALLBACK_RFC3168) union tcp_cc_info; @@ -1335,6 +1343,27 @@ static inline bool tcp_ca_needs_ecn(const struct sock *sk) return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN; } +static inline bool tcp_ca_needs_accecn(const struct sock *sk) +{ + const struct inet_connection_sock *icsk = inet_csk(sk); + + return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ACCECN; +} + +static inline bool tcp_ca_ect_1_negotiation(const struct sock *sk) +{ + const struct inet_connection_sock *icsk = inet_csk(sk); + + return icsk->icsk_ca_ops->flags & TCP_CONG_ECT_1_NEGOTIATION; +} + +static inline bool tcp_ca_no_fallback_rfc3168(const struct sock *sk) +{ + const struct inet_connection_sock *icsk = inet_csk(sk); + + return icsk->icsk_ca_ops->flags & TCP_CONG_NO_FALLBACK_RFC3168; +} + static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event) { const struct inet_connection_sock *icsk = inet_csk(sk); diff --git a/include/net/tcp_ecn.h b/include/net/tcp_ecn.h index f13e5cd2b1ac3c..a709fb1756eb75 100644 --- a/include/net/tcp_ecn.h +++ b/include/net/tcp_ecn.h @@ -31,6 +31,12 @@ enum tcp_accecn_option { TCP_ACCECN_OPTION_FULL = 2, }; +/* Apply either ECT(0) or ECT(1) based on TCP_CONG_ECT_1_NEGOTIATION flag */ +static inline void INET_ECN_xmit_ect_1_negotiation(struct sock *sk) +{ + __INET_ECN_xmit(sk, tcp_ca_ect_1_negotiation(sk)); +} + static inline void tcp_ecn_queue_cwr(struct tcp_sock *tp) { /* Do not set CWR if in AccECN mode! */ @@ -467,6 +473,26 @@ static inline u8 tcp_accecn_option_init(const struct sk_buff *skb, return TCP_ACCECN_OPT_COUNTER_SEEN; } +static inline void tcp_ecn_rcv_synack_accecn(struct sock *sk, + const struct sk_buff *skb, u8 dsf) +{ + struct tcp_sock *tp = tcp_sk(sk); + + tcp_ecn_mode_set(tp, TCP_ECN_MODE_ACCECN); + tp->syn_ect_rcv = dsf & INET_ECN_MASK; + /* Demand Accurate ECN option in response to the SYN on the SYN/ACK + * and the TCP server will try to send one more packet with an AccECN + * Option at a later point during the connection. + */ + if (tp->rx_opt.accecn && + tp->saw_accecn_opt < TCP_ACCECN_OPT_COUNTER_SEEN) { + u8 saw_opt = tcp_accecn_option_init(skb, tp->rx_opt.accecn); + + tcp_accecn_saw_opt_fail_recv(tp, saw_opt); + tp->accecn_opt_demand = 2; + } +} + /* See Table 2 of the AccECN draft */ static inline void tcp_ecn_rcv_synack(struct sock *sk, const struct sk_buff *skb, const struct tcphdr *th, u8 ip_dsfield) @@ -489,32 +515,32 @@ static inline void tcp_ecn_rcv_synack(struct sock *sk, const struct sk_buff *skb tcp_ecn_mode_set(tp, TCP_ECN_DISABLED); break; case 0x1: - case 0x5: /* +========+========+============+=============+ * | A | B | SYN/ACK | Feedback | * | | | B->A | Mode of A | * | | | AE CWR ECE | | * +========+========+============+=============+ - * | AccECN | Nonce | 1 0 1 | (Reserved) | * | AccECN | ECN | 0 0 1 | Classic ECN | * | Nonce | AccECN | 0 0 1 | Classic ECN | * | ECN | AccECN | 0 0 1 | Classic ECN | * +========+========+============+=============+ */ - if (tcp_ecn_mode_pending(tp)) - /* Downgrade from AccECN, or requested initially */ + if (tcp_ca_no_fallback_rfc3168(sk)) + tcp_ecn_mode_set(tp, TCP_ECN_DISABLED); + else tcp_ecn_mode_set(tp, TCP_ECN_MODE_RFC3168); break; - default: - tcp_ecn_mode_set(tp, TCP_ECN_MODE_ACCECN); - tp->syn_ect_rcv = ip_dsfield & INET_ECN_MASK; - if (tp->rx_opt.accecn && - tp->saw_accecn_opt < TCP_ACCECN_OPT_COUNTER_SEEN) { - u8 saw_opt = tcp_accecn_option_init(skb, tp->rx_opt.accecn); - - tcp_accecn_saw_opt_fail_recv(tp, saw_opt); - tp->accecn_opt_demand = 2; + case 0x5: + if (tcp_ecn_mode_pending(tp)) { + tcp_ecn_rcv_synack_accecn(sk, skb, ip_dsfield); + if (INET_ECN_is_ce(ip_dsfield)) { + tp->received_ce++; + tp->received_ce_pending++; + } } + break; + default: + tcp_ecn_rcv_synack_accecn(sk, skb, ip_dsfield); if (INET_ECN_is_ce(ip_dsfield) && tcp_accecn_validate_syn_feedback(sk, ace, tp->syn_ect_snt)) { @@ -525,9 +551,11 @@ static inline void tcp_ecn_rcv_synack(struct sock *sk, const struct sk_buff *skb } } -static inline void tcp_ecn_rcv_syn(struct tcp_sock *tp, const struct tcphdr *th, +static inline void tcp_ecn_rcv_syn(struct sock *sk, const struct tcphdr *th, const struct sk_buff *skb) { + struct tcp_sock *tp = tcp_sk(sk); + if (tcp_ecn_mode_pending(tp)) { if (!tcp_accecn_syn_requested(th)) { /* Downgrade to classic ECN feedback */ @@ -539,7 +567,8 @@ static inline void tcp_ecn_rcv_syn(struct tcp_sock *tp, const struct tcphdr *th, tcp_ecn_mode_set(tp, TCP_ECN_MODE_ACCECN); } } - if (tcp_ecn_mode_rfc3168(tp) && (!th->ece || !th->cwr)) + if (tcp_ecn_mode_rfc3168(tp) && + (!th->ece || !th->cwr || tcp_ca_no_fallback_rfc3168(sk))) tcp_ecn_mode_set(tp, TCP_ECN_DISABLED); } @@ -561,7 +590,7 @@ static inline void tcp_ecn_send_synack(struct sock *sk, struct sk_buff *skb) TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_ECE; else if (tcp_ca_needs_ecn(sk) || tcp_bpf_ca_needs_ecn(sk)) - INET_ECN_xmit(sk); + INET_ECN_xmit_ect_1_negotiation(sk); if (tp->ecn_flags & TCP_ECN_MODE_ACCECN) { TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_ACE; @@ -579,7 +608,8 @@ static inline void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb) bool use_ecn, use_accecn; u8 tcp_ecn = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_ecn); - use_accecn = tcp_ecn == TCP_ECN_IN_ACCECN_OUT_ACCECN; + use_accecn = tcp_ecn == TCP_ECN_IN_ACCECN_OUT_ACCECN || + tcp_ca_needs_accecn(sk); use_ecn = tcp_ecn == TCP_ECN_IN_ECN_OUT_ECN || tcp_ecn == TCP_ECN_IN_ACCECN_OUT_ECN || tcp_ca_needs_ecn(sk) || bpf_needs_ecn || use_accecn; @@ -595,7 +625,7 @@ static inline void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb) if (use_ecn) { if (tcp_ca_needs_ecn(sk) || bpf_needs_ecn) - INET_ECN_xmit(sk); + INET_ECN_xmit_ect_1_negotiation(sk); TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ECE | TCPHDR_CWR; if (use_accecn) { diff --git a/include/rdma/rw.h b/include/rdma/rw.h index d606cac482338b..9a8f4b76ce588d 100644 --- a/include/rdma/rw.h +++ b/include/rdma/rw.h @@ -66,6 +66,8 @@ int rdma_rw_ctx_post(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u32 port_num, unsigned int rdma_rw_mr_factor(struct ib_device *device, u32 port_num, unsigned int maxpages); +unsigned int rdma_rw_max_send_wr(struct ib_device *dev, u32 port_num, + unsigned int max_rdma_ctxs, u32 create_flags); void rdma_rw_init_qp(struct ib_device *dev, struct ib_qp_init_attr *attr); int rdma_rw_init_mrs(struct ib_qp *qp, struct ib_qp_init_attr *attr); void rdma_rw_cleanup_mrs(struct ib_qp *qp); diff --git a/include/sound/control.h b/include/sound/control.h index e07f6b960641ff..9be6546bf787de 100644 --- a/include/sound/control.h +++ b/include/sound/control.h @@ -14,9 +14,12 @@ #define snd_kcontrol_chip(kcontrol) ((kcontrol)->private_data) struct snd_kcontrol; +struct snd_ctl_file; typedef int (snd_kcontrol_info_t) (struct snd_kcontrol * kcontrol, struct snd_ctl_elem_info * uinfo); typedef int (snd_kcontrol_get_t) (struct snd_kcontrol * kcontrol, struct snd_ctl_elem_value * ucontrol); typedef int (snd_kcontrol_put_t) (struct snd_kcontrol * kcontrol, struct snd_ctl_elem_value * ucontrol); +typedef int (snd_kcontrol_lock_t) (struct snd_kcontrol * kcontrol, struct snd_ctl_file *owner); +typedef void (snd_kcontrol_unlock_t) (struct snd_kcontrol * kcontrol); typedef int (snd_kcontrol_tlv_rw_t)(struct snd_kcontrol *kcontrol, int op_flag, /* SNDRV_CTL_TLV_OP_XXX */ unsigned int size, @@ -55,6 +58,8 @@ struct snd_kcontrol_new { snd_kcontrol_info_t *info; snd_kcontrol_get_t *get; snd_kcontrol_put_t *put; + snd_kcontrol_lock_t *lock; + snd_kcontrol_unlock_t *unlock; union { snd_kcontrol_tlv_rw_t *c; const unsigned int *p; @@ -74,6 +79,8 @@ struct snd_kcontrol { snd_kcontrol_info_t *info; snd_kcontrol_get_t *get; snd_kcontrol_put_t *put; + snd_kcontrol_lock_t *lock; + snd_kcontrol_unlock_t *unlock; union { snd_kcontrol_tlv_rw_t *c; const unsigned int *p; diff --git a/include/sound/cs42l42.h b/include/sound/cs42l42.h index 1bd8eee54f6665..b3657965d49109 100644 --- a/include/sound/cs42l42.h +++ b/include/sound/cs42l42.h @@ -62,6 +62,10 @@ #define CS42L42_INTERNAL_FS_MASK (1 << CS42L42_INTERNAL_FS_SHIFT) #define CS42L42_SFTRAMP_RATE (CS42L42_PAGE_10 + 0x0A) +#define CS42L42_SFTRAMP_ASR_RATE_MASK GENMASK(7, 4) +#define CS42L42_SFTRAMP_ASR_RATE_SHIFT 4 +#define CS42L42_SFTRAMP_DSR_RATE_MASK GENMASK(3, 0) +#define CS42L42_SFTRAMP_DSR_RATE_SHIFT 0 #define CS42L42_SLOW_START_ENABLE (CS42L42_PAGE_10 + 0x0B) #define CS42L42_SLOW_START_EN_MASK GENMASK(6, 4) #define CS42L42_SLOW_START_EN_SHIFT 4 diff --git a/include/sound/sdca_jack.h b/include/sound/sdca_jack.h new file mode 100644 index 00000000000000..3ec22046d3ebc2 --- /dev/null +++ b/include/sound/sdca_jack.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * The MIPI SDCA specification is available for public downloads at + * https://www.mipi.org/mipi-sdca-v1-0-download + * + * Copyright (C) 2025 Cirrus Logic, Inc. and + * Cirrus Logic International Semiconductor Ltd. + */ + +#ifndef __SDCA_JACK_H__ +#define __SDCA_JACK_H__ + +struct sdca_interrupt; +struct snd_kcontrol; +struct snd_soc_jack; + +/** + * struct jack_state - Jack state structure to keep data between interrupts + * @kctl: Pointer to the ALSA control attached to this jack + * @jack: Pointer to the ASoC jack struct for this jack + */ +struct jack_state { + struct snd_kcontrol *kctl; + struct snd_soc_jack *jack; +}; + +int sdca_jack_alloc_state(struct sdca_interrupt *interrupt); +int sdca_jack_process(struct sdca_interrupt *interrupt); +int sdca_jack_set_jack(struct sdca_interrupt_info *info, struct snd_soc_jack *jack); +int sdca_jack_report(struct sdca_interrupt *interrupt); + +#endif // __SDCA_JACK_H__ diff --git a/include/sound/soc-card.h b/include/sound/soc-card.h index ecc02e955279fd..ef46cac97d9968 100644 --- a/include/sound/soc-card.h +++ b/include/sound/soc-card.h @@ -44,7 +44,7 @@ int snd_soc_card_resume_post(struct snd_soc_card *card); int snd_soc_card_probe(struct snd_soc_card *card); int snd_soc_card_late_probe(struct snd_soc_card *card); -void snd_soc_card_fixup_controls(struct snd_soc_card *card); +int snd_soc_card_fixup_controls(struct snd_soc_card *card); int snd_soc_card_remove(struct snd_soc_card *card); int snd_soc_card_set_bias_level(struct snd_soc_card *card, diff --git a/include/sound/soc.h b/include/sound/soc.h index aa0fe6b802934b..8e2f1b8b8886f5 100644 --- a/include/sound/soc.h +++ b/include/sound/soc.h @@ -575,8 +575,14 @@ int snd_soc_get_volsw_sx(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol); int snd_soc_put_volsw_sx(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol); +bool snd_soc_control_matches(struct snd_kcontrol *kcontrol, + const char *pattern); int snd_soc_limit_volume(struct snd_soc_card *card, const char *name, int max); +int snd_soc_deactivate_kctl(struct snd_soc_card *card, + const char *name, int active); +int snd_soc_set_enum_kctl(struct snd_soc_card *card, + const char *name, const char *strval); int snd_soc_bytes_info(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_info *uinfo); int snd_soc_bytes_get(struct snd_kcontrol *kcontrol, @@ -1003,7 +1009,7 @@ struct snd_soc_card { int (*probe)(struct snd_soc_card *card); int (*late_probe)(struct snd_soc_card *card); - void (*fixup_controls)(struct snd_soc_card *card); + int (*fixup_controls)(struct snd_soc_card *card); int (*remove)(struct snd_soc_card *card); /* the pre and post PM functions are used to do any PM work before and diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h index f80aa4c9d88fdc..ba828108662975 100644 --- a/include/uapi/drm/amdgpu_drm.h +++ b/include/uapi/drm/amdgpu_drm.h @@ -105,8 +105,6 @@ extern "C" { * * %AMDGPU_GEM_DOMAIN_DOORBELL Doorbell. It is an MMIO region for * signalling user mode queues. - * - * %AMDGPU_GEM_DOMAIN_MMIO_REMAP MMIO remap page (special mapping for HDP flushing). */ #define AMDGPU_GEM_DOMAIN_CPU 0x1 #define AMDGPU_GEM_DOMAIN_GTT 0x2 @@ -115,15 +113,13 @@ extern "C" { #define AMDGPU_GEM_DOMAIN_GWS 0x10 #define AMDGPU_GEM_DOMAIN_OA 0x20 #define AMDGPU_GEM_DOMAIN_DOORBELL 0x40 -#define AMDGPU_GEM_DOMAIN_MMIO_REMAP 0x80 #define AMDGPU_GEM_DOMAIN_MASK (AMDGPU_GEM_DOMAIN_CPU | \ AMDGPU_GEM_DOMAIN_GTT | \ AMDGPU_GEM_DOMAIN_VRAM | \ AMDGPU_GEM_DOMAIN_GDS | \ AMDGPU_GEM_DOMAIN_GWS | \ AMDGPU_GEM_DOMAIN_OA | \ - AMDGPU_GEM_DOMAIN_DOORBELL | \ - AMDGPU_GEM_DOMAIN_MMIO_REMAP) + AMDGPU_GEM_DOMAIN_DOORBELL) /* Flag that CPU access will be required for the case of VRAM domain */ #define AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED (1 << 0) diff --git a/include/uapi/linux/nfs.h b/include/uapi/linux/nfs.h index 71c7196d32817d..e629c495353456 100644 --- a/include/uapi/linux/nfs.h +++ b/include/uapi/linux/nfs.h @@ -55,7 +55,7 @@ NFSERR_NODEV = 19, /* v2 v3 v4 */ NFSERR_NOTDIR = 20, /* v2 v3 v4 */ NFSERR_ISDIR = 21, /* v2 v3 v4 */ - NFSERR_INVAL = 22, /* v2 v3 v4 */ + NFSERR_INVAL = 22, /* v3 v4 */ NFSERR_FBIG = 27, /* v2 v3 v4 */ NFSERR_NOSPC = 28, /* v2 v3 v4 */ NFSERR_ROFS = 30, /* v2 v3 v4 */ diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h index 51c4e8c82b1e98..d58fce1f4ffafb 100644 --- a/include/uapi/linux/prctl.h +++ b/include/uapi/linux/prctl.h @@ -6,6 +6,11 @@ /* Values to pass as first argument to prctl() */ +#define PR_GET_MEM_MODEL 0x6d4d444c +#define PR_SET_MEM_MODEL 0x4d4d444c +# define PR_SET_MEM_MODEL_DEFAULT 0 +# define PR_SET_MEM_MODEL_TSO 1 + #define PR_SET_PDEATHSIG 1 /* Second arg is a signal */ #define PR_GET_PDEATHSIG 2 /* Second arg is a ptr to return the signal */ diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index 19154228780b22..8e15c3a70ed413 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -1342,17 +1342,13 @@ static inline void *ufshcd_get_variant(struct ufs_hba *hba) return hba->priv; } -#ifdef CONFIG_PM extern int ufshcd_runtime_suspend(struct device *dev); extern int ufshcd_runtime_resume(struct device *dev); -#endif -#ifdef CONFIG_PM_SLEEP extern int ufshcd_system_suspend(struct device *dev); extern int ufshcd_system_resume(struct device *dev); extern int ufshcd_system_freeze(struct device *dev); extern int ufshcd_system_thaw(struct device *dev); extern int ufshcd_system_restore(struct device *dev); -#endif extern int ufshcd_dme_reset(struct ufs_hba *hba); extern int ufshcd_dme_enable(struct ufs_hba *hba); diff --git a/include/xen/xen.h b/include/xen/xen.h index 61854e3f283776..f280c5dcf92369 100644 --- a/include/xen/xen.h +++ b/include/xen/xen.h @@ -69,11 +69,13 @@ extern u64 xen_saved_max_mem_size; #endif #ifdef CONFIG_XEN_UNPOPULATED_ALLOC +extern unsigned long xen_unpopulated_pages; int xen_alloc_unpopulated_pages(unsigned int nr_pages, struct page **pages); void xen_free_unpopulated_pages(unsigned int nr_pages, struct page **pages); #include int arch_xen_unpopulated_init(struct resource **res); #else +#define xen_unpopulated_pages 0UL #include static inline int xen_alloc_unpopulated_pages(unsigned int nr_pages, struct page **pages) diff --git a/io_uring/cancel.h b/io_uring/cancel.h index 6783961ede1bfc..1b201a0943030e 100644 --- a/io_uring/cancel.h +++ b/io_uring/cancel.h @@ -6,10 +6,8 @@ struct io_cancel_data { struct io_ring_ctx *ctx; - union { - u64 data; - struct file *file; - }; + u64 data; + struct file *file; u8 opcode; u32 flags; int seq; diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c index 2fa7d3601edb0c..f42d573b5b1734 100644 --- a/io_uring/io-wq.c +++ b/io_uring/io-wq.c @@ -34,6 +34,7 @@ enum { enum { IO_WQ_BIT_EXIT = 0, /* wq exiting */ + IO_WQ_BIT_EXIT_ON_IDLE = 1, /* allow all workers to exit on idle */ }; enum { @@ -706,9 +707,13 @@ static int io_wq_worker(void *data) raw_spin_lock(&acct->workers_lock); /* * Last sleep timed out. Exit if we're not the last worker, - * or if someone modified our affinity. + * or if someone modified our affinity. If wq is marked + * idle-exit, drop the worker as well. This is used to avoid + * keeping io-wq workers around for tasks that no longer have + * any active io_uring instances. */ - if (last_timeout && (exit_mask || acct->nr_workers > 1)) { + if ((last_timeout && (exit_mask || acct->nr_workers > 1)) || + test_bit(IO_WQ_BIT_EXIT_ON_IDLE, &wq->state)) { acct->nr_workers--; raw_spin_unlock(&acct->workers_lock); __set_current_state(TASK_RUNNING); @@ -963,6 +968,24 @@ static bool io_wq_worker_wake(struct io_worker *worker, void *data) return false; } +void io_wq_set_exit_on_idle(struct io_wq *wq, bool enable) +{ + if (!wq->task) + return; + + if (!enable) { + clear_bit(IO_WQ_BIT_EXIT_ON_IDLE, &wq->state); + return; + } + + if (test_and_set_bit(IO_WQ_BIT_EXIT_ON_IDLE, &wq->state)) + return; + + rcu_read_lock(); + io_wq_for_each_worker(wq, io_wq_worker_wake, NULL); + rcu_read_unlock(); +} + static void io_run_cancel(struct io_wq_work *work, struct io_wq *wq) { do { diff --git a/io_uring/io-wq.h b/io_uring/io-wq.h index 774abab54732ef..94b14742b70328 100644 --- a/io_uring/io-wq.h +++ b/io_uring/io-wq.h @@ -41,6 +41,7 @@ struct io_wq_data { struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data); void io_wq_exit_start(struct io_wq *wq); void io_wq_put_and_exit(struct io_wq *wq); +void io_wq_set_exit_on_idle(struct io_wq *wq, bool enable); void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work); void io_wq_hash_work(struct io_wq_work *work, void *val); diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index b7a077c11c21a9..63efd60829f376 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -138,7 +138,7 @@ static void io_queue_sqe(struct io_kiocb *req, unsigned int extra_flags); static void __io_req_caches_free(struct io_ring_ctx *ctx); -static __read_mostly DEFINE_STATIC_KEY_FALSE(io_key_has_sqarray); +static __read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(io_key_has_sqarray, HZ); struct kmem_cache *req_cachep; static struct workqueue_struct *iou_wq __ro_after_init; @@ -2375,7 +2375,7 @@ static bool io_get_sqe(struct io_ring_ctx *ctx, const struct io_uring_sqe **sqe) unsigned mask = ctx->sq_entries - 1; unsigned head = ctx->cached_sq_head++ & mask; - if (static_branch_unlikely(&io_key_has_sqarray) && + if (static_branch_unlikely(&io_key_has_sqarray.key) && (!(ctx->flags & IORING_SETUP_NO_SQARRAY))) { head = READ_ONCE(ctx->sq_array[head]); if (unlikely(head >= ctx->sq_entries)) { @@ -2867,7 +2867,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) io_rings_free(ctx); if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) - static_branch_dec(&io_key_has_sqarray); + static_branch_slow_dec_deferred(&io_key_has_sqarray); percpu_ref_exit(&ctx->refs); free_uid(ctx->user); @@ -3256,7 +3256,11 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, ctx = file->private_data; ret = -EBADFD; - if (unlikely(ctx->flags & IORING_SETUP_R_DISABLED)) + /* + * Keep IORING_SETUP_R_DISABLED check before submitter_task load + * in io_uring_add_tctx_node() -> __io_uring_add_tctx_node_from_submit() + */ + if (unlikely(smp_load_acquire(&ctx->flags) & IORING_SETUP_R_DISABLED)) goto out; /* @@ -3596,7 +3600,7 @@ static __cold int io_uring_create(struct io_ctx_config *config) ctx->clock_offset = 0; if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) - static_branch_inc(&io_key_has_sqarray); + static_branch_deferred_inc(&io_key_has_sqarray); if ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) && !(ctx->flags & IORING_SETUP_IOPOLL) && diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c index 796d131107ddb9..67d4fe576473ab 100644 --- a/io_uring/kbuf.c +++ b/io_uring/kbuf.c @@ -669,8 +669,9 @@ int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg) bl->buf_ring = br; if (reg.flags & IOU_PBUF_RING_INC) bl->flags |= IOBL_INC; - io_buffer_add_list(ctx, bl, reg.bgid); - return 0; + ret = io_buffer_add_list(ctx, bl, reg.bgid); + if (!ret) + return 0; fail: io_free_region(ctx->user, &bl->region); kfree(bl); diff --git a/io_uring/msg_ring.c b/io_uring/msg_ring.c index 7063ea7964e754..87b4d306cf1b6a 100644 --- a/io_uring/msg_ring.c +++ b/io_uring/msg_ring.c @@ -125,7 +125,11 @@ static int __io_msg_ring_data(struct io_ring_ctx *target_ctx, return -EINVAL; if (!(msg->flags & IORING_MSG_RING_FLAGS_PASS) && msg->dst_fd) return -EINVAL; - if (target_ctx->flags & IORING_SETUP_R_DISABLED) + /* + * Keep IORING_SETUP_R_DISABLED check before submitter_task load + * in io_msg_data_remote() -> io_msg_remote_post() + */ + if (smp_load_acquire(&target_ctx->flags) & IORING_SETUP_R_DISABLED) return -EBADFD; if (io_msg_need_remote(target_ctx)) @@ -245,7 +249,11 @@ static int io_msg_send_fd(struct io_kiocb *req, unsigned int issue_flags) return -EINVAL; if (target_ctx == ctx) return -EINVAL; - if (target_ctx->flags & IORING_SETUP_R_DISABLED) + /* + * Keep IORING_SETUP_R_DISABLED check before submitter_task load + * in io_msg_fd_remote() + */ + if (smp_load_acquire(&target_ctx->flags) & IORING_SETUP_R_DISABLED) return -EBADFD; if (!msg->src_file) { int ret = io_msg_grab_file(req, issue_flags); diff --git a/io_uring/register.c b/io_uring/register.c index 3d3822ff3fd9e1..12318c276068e4 100644 --- a/io_uring/register.c +++ b/io_uring/register.c @@ -193,7 +193,8 @@ static int io_register_enable_rings(struct io_ring_ctx *ctx) if (ctx->restrictions.registered) ctx->restricted = 1; - ctx->flags &= ~IORING_SETUP_R_DISABLED; + /* Keep submitter_task store before clearing IORING_SETUP_R_DISABLED */ + smp_store_release(&ctx->flags, ctx->flags & ~IORING_SETUP_R_DISABLED); if (ctx->sq_data && wq_has_sleeper(&ctx->sq_data->wait)) wake_up(&ctx->sq_data->wait); return 0; diff --git a/io_uring/sync.c b/io_uring/sync.c index cea2d381ffd2a4..ab7fa1cd7dd639 100644 --- a/io_uring/sync.c +++ b/io_uring/sync.c @@ -62,6 +62,8 @@ int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -EINVAL; sync->off = READ_ONCE(sqe->off); + if (sync->off < 0) + return -EINVAL; sync->len = READ_ONCE(sqe->len); req->flags |= REQ_F_FORCE_ASYNC; return 0; diff --git a/io_uring/tctx.c b/io_uring/tctx.c index 6d6f44215ec807..91f4b830b77b9e 100644 --- a/io_uring/tctx.c +++ b/io_uring/tctx.c @@ -122,6 +122,14 @@ int __io_uring_add_tctx_node(struct io_ring_ctx *ctx) return ret; } } + + /* + * Re-activate io-wq keepalive on any new io_uring usage. The wq may have + * been marked for idle-exit when the task temporarily had no active + * io_uring instances. + */ + if (tctx->io_wq) + io_wq_set_exit_on_idle(tctx->io_wq, false); if (!xa_load(&tctx->xa, (unsigned long)ctx)) { node = kmalloc(sizeof(*node), GFP_KERNEL); if (!node) @@ -183,6 +191,9 @@ __cold void io_uring_del_tctx_node(unsigned long index) if (tctx->last == node->ctx) tctx->last = NULL; kfree(node); + + if (xa_empty(&tctx->xa) && tctx->io_wq) + io_wq_set_exit_on_idle(tctx->io_wq, true); } __cold void io_uring_clean_tctx(struct io_uring_task *tctx) diff --git a/ipc/ipc_sysctl.c b/ipc/ipc_sysctl.c index 15b17e86e198cf..9b087ebeb643b7 100644 --- a/ipc/ipc_sysctl.c +++ b/ipc/ipc_sysctl.c @@ -214,7 +214,7 @@ static int ipc_permissions(struct ctl_table_header *head, const struct ctl_table if (((table->data == &ns->ids[IPC_SEM_IDS].next_id) || (table->data == &ns->ids[IPC_MSG_IDS].next_id) || (table->data == &ns->ids[IPC_SHM_IDS].next_id)) && - checkpoint_restore_ns_capable(ns->user_ns)) + checkpoint_restore_ns_capable_noaudit(ns->user_ns)) mode = 0666; else #endif diff --git a/kernel/bpf/bpf_insn_array.c b/kernel/bpf/bpf_insn_array.c index c96630cb75bf7a..37b43102953eeb 100644 --- a/kernel/bpf/bpf_insn_array.c +++ b/kernel/bpf/bpf_insn_array.c @@ -126,7 +126,7 @@ static int insn_array_map_direct_value_addr(const struct bpf_map *map, u64 *imm, return -EINVAL; /* from BPF's point of view, this map is a jump table */ - *imm = (unsigned long)insn_array->ips + off; + *imm = (unsigned long)insn_array->ips; return 0; } diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 1b9b18e5b03cb0..85c0feaae0d3cd 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -713,8 +713,8 @@ static struct bpf_ksym *bpf_ksym_find(unsigned long addr) return n ? container_of(n, struct bpf_ksym, tnode) : NULL; } -int __bpf_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char *sym) +int bpf_address_lookup(unsigned long addr, unsigned long *size, + unsigned long *off, char *sym) { struct bpf_ksym *ksym; int ret = 0; diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index db72b96f9c8c85..f66284f8ec2cc9 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -1077,7 +1077,7 @@ const struct bpf_func_proto bpf_snprintf_proto = { .func = bpf_snprintf, .gpl_only = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_MEM_OR_NULL, + .arg1_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, .arg2_type = ARG_CONST_SIZE_OR_ZERO, .arg3_type = ARG_PTR_TO_CONST_STR, .arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, diff --git a/kernel/bpf/rqspinlock.c b/kernel/bpf/rqspinlock.c index f7d0c8d4644edb..2fdfa828e3d35a 100644 --- a/kernel/bpf/rqspinlock.c +++ b/kernel/bpf/rqspinlock.c @@ -265,10 +265,11 @@ int __lockfunc resilient_tas_spin_lock(rqspinlock_t *lock) RES_INIT_TIMEOUT(ts); /* - * The fast path is not invoked for the TAS fallback, so we must grab - * the deadlock detection entry here. + * We are either called directly from res_spin_lock after grabbing the + * deadlock detection entry when queued spinlocks are disabled, or from + * resilient_queued_spin_lock_slowpath after grabbing the deadlock + * detection entry. No need to obtain it here. */ - grab_held_lock_entry(lock); /* * Since the waiting loop's time is dependent on the amount of diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 4ff82144f8853d..ce7db2f3be6f63 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1366,11 +1366,6 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token, return ret; } -static bool bpf_net_capable(void) -{ - return capable(CAP_NET_ADMIN) || capable(CAP_SYS_ADMIN); -} - #define BPF_MAP_CREATE_LAST_FIELD excl_prog_hash_size /* called via syscall */ static int map_create(union bpf_attr *attr, bpfptr_t uattr) @@ -2820,6 +2815,13 @@ static int bpf_prog_verify_signature(struct bpf_prog *prog, union bpf_attr *attr void *sig; int err = 0; + /* + * Don't attempt to use kmalloc_large or vmalloc for signatures. + * Practical signature for BPF program should be below this limit. + */ + if (attr->signature_size > KMALLOC_MAX_CACHE_SIZE) + return -EINVAL; + if (system_keyring_id_check(attr->keyring_id) == 0) key = bpf_lookup_system_key(attr->keyring_id); else @@ -4565,6 +4567,8 @@ static int bpf_prog_detach(const union bpf_attr *attr) prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); if (IS_ERR(prog)) return PTR_ERR(prog); + } else if (!bpf_mprog_detach_empty(ptype)) { + return -EPERM; } } else if (is_cgroup_prog_type(ptype, 0, false)) { if (attr->attach_flags || attr->relative_fd) @@ -5310,6 +5314,9 @@ static int bpf_map_get_info_by_fd(struct file *file, if (info.hash_size != SHA256_DIGEST_SIZE) return -EINVAL; + if (!READ_ONCE(map->frozen)) + return -EPERM; + err = map->ops->map_get_hash(map, SHA256_DIGEST_SIZE, map->sha); if (err != 0) return err; @@ -6407,7 +6414,7 @@ static const struct bpf_func_proto bpf_kallsyms_lookup_name_proto = { .func = bpf_kallsyms_lookup_name, .gpl_only = false, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_MEM, + .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg2_type = ARG_CONST_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED, diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 976d89011b1570..b9a358d7a78f1a 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -214,10 +214,15 @@ static int modify_fentry(struct bpf_trampoline *tr, u32 orig_flags, int ret; if (tr->func.ftrace_managed) { + unsigned long addr = (unsigned long) new_addr; + + if (bpf_trampoline_use_jmp(tr->flags)) + addr = ftrace_jmp_set(addr); + if (lock_direct_mutex) - ret = modify_ftrace_direct(tr->fops, (long)new_addr); + ret = modify_ftrace_direct(tr->fops, addr); else - ret = modify_ftrace_direct_nolock(tr->fops, (long)new_addr); + ret = modify_ftrace_direct_nolock(tr->fops, addr); } else { ret = bpf_trampoline_update_fentry(tr, orig_flags, old_addr, new_addr); @@ -240,10 +245,15 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr) } if (tr->func.ftrace_managed) { + unsigned long addr = (unsigned long) new_addr; + + if (bpf_trampoline_use_jmp(tr->flags)) + addr = ftrace_jmp_set(addr); + ret = ftrace_set_filter_ip(tr->fops, (unsigned long)ip, 0, 1); if (ret) return ret; - ret = register_ftrace_direct(tr->fops, (long)new_addr); + ret = register_ftrace_direct(tr->fops, addr); } else { ret = bpf_trampoline_update_fentry(tr, 0, NULL, new_addr); } @@ -499,13 +509,6 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut if (err) goto out_free; -#ifdef CONFIG_DYNAMIC_FTRACE_WITH_JMP - if (bpf_trampoline_use_jmp(tr->flags)) - tr->fops->flags |= FTRACE_OPS_FL_JMP; - else - tr->fops->flags &= ~FTRACE_OPS_FL_JMP; -#endif - WARN_ON(tr->cur_image && total == 0); if (tr->cur_image) /* progs already running at this address */ @@ -533,15 +536,8 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut tr->cur_image = im; out: /* If any error happens, restore previous flags */ - if (err) { + if (err) tr->flags = orig_flags; -#ifdef CONFIG_DYNAMIC_FTRACE_WITH_JMP - if (bpf_trampoline_use_jmp(tr->flags)) - tr->fops->flags |= FTRACE_OPS_FL_JMP; - else - tr->fops->flags &= ~FTRACE_OPS_FL_JMP; -#endif - } kfree(tlinks); return err; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 3135643d569552..fe01edfcc34c6a 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -16827,6 +16827,7 @@ static void sync_linked_regs(struct bpf_verifier_state *vstate, struct bpf_reg_s } else { s32 saved_subreg_def = reg->subreg_def; s32 saved_off = reg->off; + u32 saved_id = reg->id; fake_reg.type = SCALAR_VALUE; __mark_reg_known(&fake_reg, (s32)reg->off - (s32)known_reg->off); @@ -16834,10 +16835,11 @@ static void sync_linked_regs(struct bpf_verifier_state *vstate, struct bpf_reg_s /* reg = known_reg; reg += delta */ copy_register_state(reg, known_reg); /* - * Must preserve off, id and add_const flag, + * Must preserve off, id and subreg_def flag, * otherwise another sync_linked_regs() will be incorrect. */ reg->off = saved_off; + reg->id = saved_id; reg->subreg_def = saved_subreg_def; scalar32_min_max_add(reg, &fake_reg); @@ -19835,8 +19837,10 @@ static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) } } if (bpf_calls_callback(env, insn_idx)) { - if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) + if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) { + loop = true; goto hit; + } goto skip_inf_loop_check; } /* attempt to detect infinite loop to avoid unnecessary doomed work */ @@ -20611,17 +20615,19 @@ static int do_check(struct bpf_verifier_env *env) * may skip a nospec patched-in after the jump. This can * currently never happen because nospec_result is only * used for the write-ops - * `*(size*)(dst_reg+off)=src_reg|imm32` which must - * never skip the following insn. Still, add a warning - * to document this in case nospec_result is used - * elsewhere in the future. + * `*(size*)(dst_reg+off)=src_reg|imm32` and helper + * calls. These must never skip the following insn + * (i.e., bpf_insn_successors()'s opcode_info.can_jump + * is false). Still, add a warning to document this in + * case nospec_result is used elsewhere in the future. * * All non-branch instructions have a single * fall-through edge. For these, nospec_result should * already work. */ - if (verifier_bug_if(BPF_CLASS(insn->code) == BPF_JMP || - BPF_CLASS(insn->code) == BPF_JMP32, env, + if (verifier_bug_if((BPF_CLASS(insn->code) == BPF_JMP || + BPF_CLASS(insn->code) == BPF_JMP32) && + BPF_OP(insn->code) != BPF_CALL, env, "speculation barrier after jump instruction may not have the desired effect")) return -EFAULT; process_bpf_exit: @@ -20679,29 +20685,29 @@ static int find_btf_percpu_datasec(struct btf *btf) } /* - * Add btf to the used_btfs array and return the index. (If the btf was - * already added, then just return the index.) Upon successful insertion - * increase btf refcnt, and, if present, also refcount the corresponding - * kernel module. + * Add btf to the env->used_btfs array. If needed, refcount the + * corresponding kernel module. To simplify caller's logic + * in case of error or if btf was added before the function + * decreases the btf refcount. */ static int __add_used_btf(struct bpf_verifier_env *env, struct btf *btf) { struct btf_mod_pair *btf_mod; + int ret = 0; int i; /* check whether we recorded this BTF (and maybe module) already */ for (i = 0; i < env->used_btf_cnt; i++) if (env->used_btfs[i].btf == btf) - return i; + goto ret_put; if (env->used_btf_cnt >= MAX_USED_BTFS) { verbose(env, "The total number of btfs per program has reached the limit of %u\n", MAX_USED_BTFS); - return -E2BIG; + ret = -E2BIG; + goto ret_put; } - btf_get(btf); - btf_mod = &env->used_btfs[env->used_btf_cnt]; btf_mod->btf = btf; btf_mod->module = NULL; @@ -20710,12 +20716,18 @@ static int __add_used_btf(struct bpf_verifier_env *env, struct btf *btf) if (btf_is_module(btf)) { btf_mod->module = btf_try_get_module(btf); if (!btf_mod->module) { - btf_put(btf); - return -ENXIO; + ret = -ENXIO; + goto ret_put; } } - return env->used_btf_cnt++; + env->used_btf_cnt++; + return 0; + +ret_put: + /* Either error or this BTF was already added */ + btf_put(btf); + return ret; } /* replace pseudo btf_id with kernel symbol address */ @@ -20812,9 +20824,7 @@ static int check_pseudo_btf_id(struct bpf_verifier_env *env, btf_fd = insn[1].imm; if (btf_fd) { - CLASS(fd, f)(btf_fd); - - btf = __btf_get_by_fd(f); + btf = btf_get_by_fd(btf_fd); if (IS_ERR(btf)) { verbose(env, "invalid module BTF object FD specified.\n"); return -EINVAL; @@ -20824,17 +20834,17 @@ static int check_pseudo_btf_id(struct bpf_verifier_env *env, verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"); return -EINVAL; } + btf_get(btf_vmlinux); btf = btf_vmlinux; } err = __check_pseudo_btf_id(env, insn, aux, btf); - if (err) + if (err) { + btf_put(btf); return err; + } - err = __add_used_btf(env, btf); - if (err < 0) - return err; - return 0; + return __add_used_btf(env, btf); } static bool is_tracing_prog_type(enum bpf_prog_type type) @@ -24650,10 +24660,8 @@ static int add_fd_from_fd_array(struct bpf_verifier_env *env, int fd) btf = __btf_get_by_fd(f); if (!IS_ERR(btf)) { - err = __add_used_btf(env, btf); - if (err < 0) - return err; - return 0; + btf_get(btf); + return __add_used_btf(env, btf); } verbose(env, "fd %d is not pointing to valid bpf_map or btf\n", fd); @@ -25076,15 +25084,18 @@ static int compute_scc(struct bpf_verifier_env *env) } /* * Assign SCC number only if component has two or more elements, - * or if component has a self reference. + * or if component has a self reference, or if instruction is a + * callback calling function (implicit loop). */ - assign_scc = stack[stack_sz - 1] != w; - for (j = 0; j < succ->cnt; ++j) { + assign_scc = stack[stack_sz - 1] != w; /* two or more elements? */ + for (j = 0; j < succ->cnt; ++j) { /* self reference? */ if (succ->items[j] == w) { assign_scc = true; break; } } + if (bpf_calls_callback(env, w)) /* implicit loop? */ + assign_scc = true; /* Pop component elements from stack */ do { t = stack[--stack_sz]; diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c index 049e296f586ccd..cdd6e025935d34 100644 --- a/kernel/kallsyms.c +++ b/kernel/kallsyms.c @@ -345,7 +345,7 @@ int kallsyms_lookup_size_offset(unsigned long addr, unsigned long *symbolsize, return 1; } return !!module_address_lookup(addr, symbolsize, offset, NULL, NULL, namebuf) || - !!__bpf_address_lookup(addr, symbolsize, offset, namebuf); + !!bpf_address_lookup(addr, symbolsize, offset, namebuf); } static int kallsyms_lookup_buildid(unsigned long addr, @@ -377,12 +377,11 @@ static int kallsyms_lookup_buildid(unsigned long addr, ret = module_address_lookup(addr, symbolsize, offset, modname, modbuildid, namebuf); if (!ret) - ret = bpf_address_lookup(addr, symbolsize, - offset, modname, namebuf); + ret = bpf_address_lookup(addr, symbolsize, offset, namebuf); if (!ret) - ret = ftrace_mod_address_lookup(addr, symbolsize, - offset, modname, namebuf); + ret = ftrace_mod_address_lookup(addr, symbolsize, offset, + modname, modbuildid, namebuf); return ret; } diff --git a/kernel/locking/lockdep_internals.h b/kernel/locking/lockdep_internals.h index 0e5e6ffe91a3fe..91a802fea0aa0f 100644 --- a/kernel/locking/lockdep_internals.h +++ b/kernel/locking/lockdep_internals.h @@ -121,7 +121,7 @@ enum { #define MAX_LOCKDEP_CHAINS (1UL << MAX_LOCKDEP_CHAINS_BITS) -#define AVG_LOCKDEP_CHAIN_DEPTH 5 +#define AVG_LOCKDEP_CHAIN_DEPTH 10 #define MAX_LOCKDEP_CHAIN_HLOCKS (MAX_LOCKDEP_CHAINS * AVG_LOCKDEP_CHAIN_DEPTH) extern struct lock_chain lock_chains[]; diff --git a/kernel/module/kallsyms.c b/kernel/module/kallsyms.c index 00a60796327c06..0fc11e45df9b91 100644 --- a/kernel/module/kallsyms.c +++ b/kernel/module/kallsyms.c @@ -334,13 +334,8 @@ int module_address_lookup(unsigned long addr, if (mod) { if (modname) *modname = mod->name; - if (modbuildid) { -#if IS_ENABLED(CONFIG_STACKTRACE_BUILD_ID) - *modbuildid = mod->build_id; -#else - *modbuildid = NULL; -#endif - } + if (modbuildid) + *modbuildid = module_buildid(mod); sym = find_kallsyms_symbol(mod, addr, size, offset); diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c index af8d07bafe02a7..d04eae18ea688c 100644 --- a/kernel/power/hibernate.c +++ b/kernel/power/hibernate.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include "power.h" @@ -110,7 +111,8 @@ bool hibernation_available(void) { return nohibernate == 0 && !security_locked_down(LOCKDOWN_HIBERNATION) && - !secretmem_active() && !cxl_mem_active(); + !secretmem_active() && !cxl_mem_active() && + !of_machine_is_compatible("apple,arm-platform"); } /** diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 07e51974b06bc6..83934402a287b5 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -1750,7 +1750,7 @@ rcu_torture_writer(void *arg) ulo[i] = cur_ops->get_comp_state(); gp_snap = cur_ops->start_gp_poll(); rcu_torture_writer_state = RTWS_POLL_WAIT; - if (cur_ops->exp_current && !torture_random(&rand) % 0xff) + if (cur_ops->exp_current && !(torture_random(&rand) & 0xff)) cur_ops->exp_current(); while (!cur_ops->poll_gp_state(gp_snap)) { gp_snap1 = cur_ops->get_gp_state(); @@ -1772,7 +1772,7 @@ rcu_torture_writer(void *arg) cur_ops->get_comp_state_full(&rgo[i]); cur_ops->start_gp_poll_full(&gp_snap_full); rcu_torture_writer_state = RTWS_POLL_WAIT_FULL; - if (cur_ops->exp_current && !torture_random(&rand) % 0xff) + if (cur_ops->exp_current && !(torture_random(&rand) & 0xff)) cur_ops->exp_current(); while (!cur_ops->poll_gp_state_full(&gp_snap_full)) { cur_ops->get_gp_state_full(&gp_snap1_full); diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index b8bbe7960cda7a..2265b9c2906e1e 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -203,7 +203,7 @@ struct rcu_data { /* during and after the last grace */ /* period it is aware of. */ struct irq_work defer_qs_iw; /* Obtain later scheduler attention. */ - int defer_qs_iw_pending; /* Scheduler attention pending? */ + int defer_qs_pending; /* irqwork or softirq pending? */ struct work_struct strict_work; /* Schedule readers for strict GPs. */ /* 2) batch handling */ diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index dbe2d02be824a0..95ad967adcf3cb 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -487,8 +487,8 @@ rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags) union rcu_special special; rdp = this_cpu_ptr(&rcu_data); - if (rdp->defer_qs_iw_pending == DEFER_QS_PENDING) - rdp->defer_qs_iw_pending = DEFER_QS_IDLE; + if (rdp->defer_qs_pending == DEFER_QS_PENDING) + rdp->defer_qs_pending = DEFER_QS_IDLE; /* * If RCU core is waiting for this CPU to exit its critical section, @@ -645,7 +645,7 @@ static void rcu_preempt_deferred_qs_handler(struct irq_work *iwp) * 5. Deferred QS reporting does not happen. */ if (rcu_preempt_depth() > 0) - WRITE_ONCE(rdp->defer_qs_iw_pending, DEFER_QS_IDLE); + WRITE_ONCE(rdp->defer_qs_pending, DEFER_QS_IDLE); } /* @@ -747,7 +747,10 @@ static void rcu_read_unlock_special(struct task_struct *t) // Using softirq, safe to awaken, and either the // wakeup is free or there is either an expedited // GP in flight or a potential need to deboost. - raise_softirq_irqoff(RCU_SOFTIRQ); + if (rdp->defer_qs_pending != DEFER_QS_PENDING) { + rdp->defer_qs_pending = DEFER_QS_PENDING; + raise_softirq_irqoff(RCU_SOFTIRQ); + } } else { // Enabling BH or preempt does reschedule, so... // Also if no expediting and no possible deboosting, @@ -755,11 +758,11 @@ static void rcu_read_unlock_special(struct task_struct *t) // tick enabled. set_need_resched_current(); if (IS_ENABLED(CONFIG_IRQ_WORK) && irqs_were_disabled && - needs_exp && rdp->defer_qs_iw_pending != DEFER_QS_PENDING && + needs_exp && rdp->defer_qs_pending != DEFER_QS_PENDING && cpu_online(rdp->cpu)) { // Get scheduler to re-evaluate and call hooks. // If !IRQ_WORK, FQS scan will eventually IPI. - rdp->defer_qs_iw_pending = DEFER_QS_PENDING; + rdp->defer_qs_pending = DEFER_QS_PENDING; irq_work_queue_on(&rdp->defer_qs_iw, rdp->cpu); } } diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 854984967fe295..dbf4e32a063f76 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -119,6 +119,9 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_cfs_tp); EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_se_tp); EXPORT_TRACEPOINT_SYMBOL_GPL(sched_update_nr_running_tp); EXPORT_TRACEPOINT_SYMBOL_GPL(sched_compute_energy_tp); +EXPORT_TRACEPOINT_SYMBOL_GPL(sched_entry_tp); +EXPORT_TRACEPOINT_SYMBOL_GPL(sched_exit_tp); +EXPORT_TRACEPOINT_SYMBOL_GPL(sched_set_need_resched_tp); DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); DEFINE_PER_CPU(struct rnd_state, sched_rnd_state); @@ -1136,6 +1139,7 @@ void __trace_set_need_resched(struct task_struct *curr, int tif) { trace_sched_set_need_resched_tp(curr, smp_processor_id(), tif); } +EXPORT_SYMBOL_GPL(__trace_set_need_resched); void resched_curr(struct rq *rq) { @@ -9111,6 +9115,7 @@ void sched_move_task(struct task_struct *tsk, bool for_autogroup) { unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE; bool resched = false; + bool queued = false; struct rq *rq; CLASS(task_rq_lock, rq_guard)(tsk); @@ -9122,10 +9127,13 @@ void sched_move_task(struct task_struct *tsk, bool for_autogroup) scx_cgroup_move_task(tsk); if (scope->running) resched = true; + queued = scope->queued; } if (resched) resched_curr(rq); + else if (queued) + wakeup_preempt(rq, tsk, 0); __balance_callbacks(rq, &rq_guard.rf); } @@ -10729,10 +10737,9 @@ void sched_mm_cid_exit(struct task_struct *t) return; /* * Mode change. The task has the CID unset - * already. The CPU CID is still valid and - * does not have MM_CID_TRANSIT set as the - * mode change has just taken effect under - * mm::mm_cid::lock. Drop it. + * already and dealt with an eventually set + * TRANSIT bit. If the CID is owned by the CPU + * then drop it. */ mm_drop_cid_on_cpu(mm, this_cpu_ptr(mm->mm_cid.pcpu)); } diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 7bcde7114f1b65..e3a6b8ed1d6db1 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -3656,6 +3656,9 @@ static void __dl_clear_params(struct sched_dl_entity *dl_se) dl_se->dl_non_contending = 0; dl_se->dl_overrun = 0; dl_se->dl_server = 0; + dl_se->dl_defer = 0; + dl_se->dl_defer_running = 0; + dl_se->dl_defer_armed = 0; #ifdef CONFIG_RT_MUTEXES dl_se->pi_se = dl_se; diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index f1867fe8e5c535..e0ff909050190b 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -2100,6 +2100,7 @@ static void push_rt_tasks(struct rq *rq) */ static int rto_next_cpu(struct root_domain *rd) { + int this_cpu = smp_processor_id(); int next; int cpu; @@ -2123,6 +2124,10 @@ static int rto_next_cpu(struct root_domain *rd) rd->rto_cpu = cpu; + /* Do not send IPI to self */ + if (cpu == this_cpu) + continue; + if (cpu < nr_cpu_ids) return cpu; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index bd350e40859d89..1b4283e9edc3bd 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -3758,8 +3758,10 @@ static __always_inline void mm_unset_cid_on_task(struct task_struct *t) static __always_inline void mm_drop_cid_on_cpu(struct mm_struct *mm, struct mm_cid_pcpu *pcp) { /* Clear the ONCPU bit, but do not set UNSET in the per CPU storage */ - pcp->cid = cpu_cid_to_cid(pcp->cid); - mm_drop_cid(mm, pcp->cid); + if (cid_on_cpu(pcp->cid)) { + pcp->cid = cpu_cid_to_cid(pcp->cid); + mm_drop_cid(mm, pcp->cid); + } } static inline unsigned int __mm_get_cid(struct mm_struct *mm, unsigned int max_cids) diff --git a/kernel/sys.c b/kernel/sys.c index 8b58eece4e580b..93e4d60b6fead8 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -45,6 +45,7 @@ #include #include #include +#include #include #include @@ -2515,6 +2516,16 @@ static int prctl_set_thp_disable(bool thp_disable, unsigned long flags, return 0; } +int __weak arch_prctl_mem_model_get(struct task_struct *t) +{ + return -EINVAL; +} + +int __weak arch_prctl_mem_model_set(struct task_struct *t, unsigned long val) +{ + return -EINVAL; +} + SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, unsigned long, arg4, unsigned long, arg5) { @@ -2528,6 +2539,16 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, error = 0; switch (option) { + case PR_GET_MEM_MODEL: + if (arg2 || arg3 || arg4 || arg5) + return -EINVAL; + error = arch_prctl_mem_model_get(me); + break; + case PR_SET_MEM_MODEL: + if (arg3 || arg4 || arg5) + return -EINVAL; + error = arch_prctl_mem_model_set(me, arg2); + break; case PR_SET_PDEATHSIG: if (!valid_signal(arg2)) { error = -EINVAL; diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 0e4bc1ca15ff10..84c8ab2a0cebf3 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -1742,7 +1742,7 @@ static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base, lockdep_assert_held(&cpu_base->lock); - debug_deactivate(timer); + debug_hrtimer_deactivate(timer); base->running = timer; /* diff --git a/kernel/time/sched_clock.c b/kernel/time/sched_clock.c index f39111830ca363..f3aaef695b8cdb 100644 --- a/kernel/time/sched_clock.c +++ b/kernel/time/sched_clock.c @@ -215,7 +215,7 @@ void sched_clock_register(u64 (*read)(void), int bits, unsigned long rate) update_clock_read_data(&rd); - if (sched_clock_timer.function != NULL) { + if (ACCESS_PRIVATE(&sched_clock_timer, function) != NULL) { /* update timeout for clock wrap */ hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL_HARD); diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c index d031c8d80be4fd..c4db5c2e710376 100644 --- a/kernel/trace/blktrace.c +++ b/kernel/trace/blktrace.c @@ -793,7 +793,7 @@ int blk_trace_setup(struct request_queue *q, char *name, dev_t dev, return PTR_ERR(bt); } blk_trace_setup_finalize(q, name, 1, bt, &buts2); - strcpy(buts.name, buts2.name); + strscpy(buts.name, buts2.name, BLKTRACE_BDEV_SIZE); mutex_unlock(&q->debugfs_mutex); if (copy_to_user(arg, &buts, sizeof(buts))) { diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index fe28d86f7c3576..59c2394981c721 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -1022,7 +1022,7 @@ const struct bpf_func_proto bpf_snprintf_btf_proto = { .func = bpf_snprintf_btf, .gpl_only = false, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_MEM, + .arg1_type = ARG_PTR_TO_MEM | MEM_WRITE, .arg2_type = ARG_CONST_SIZE, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg4_type = ARG_CONST_SIZE, @@ -1526,7 +1526,7 @@ static const struct bpf_func_proto bpf_read_branch_records_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM_OR_NULL, + .arg2_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, .arg3_type = ARG_CONST_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -1661,7 +1661,7 @@ static const struct bpf_func_proto bpf_get_stack_proto_raw_tp = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, + .arg2_type = ARG_PTR_TO_UNINIT_MEM, .arg3_type = ARG_CONST_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index aa758efc373149..e835d878038b21 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -6049,15 +6049,8 @@ int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr) if (ftrace_hash_empty(hash)) return -EINVAL; - /* This is a "raw" address, and this should never happen. */ - if (WARN_ON_ONCE(ftrace_is_jmp(addr))) - return -EINVAL; - mutex_lock(&direct_mutex); - if (ops->flags & FTRACE_OPS_FL_JMP) - addr = ftrace_jmp_set(addr); - /* Make sure requested entries are not already registered.. */ size = 1 << hash->size_bits; for (i = 0; i < size; i++) { @@ -6178,13 +6171,6 @@ __modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr) lockdep_assert_held_once(&direct_mutex); - /* This is a "raw" address, and this should never happen. */ - if (WARN_ON_ONCE(ftrace_is_jmp(addr))) - return -EINVAL; - - if (ops->flags & FTRACE_OPS_FL_JMP) - addr = ftrace_jmp_set(addr); - /* Enable the tmp_ops to have the same functions as the direct ops */ ftrace_ops_init(&tmp_ops); tmp_ops.func_hash = ops->func_hash; @@ -7753,7 +7739,8 @@ ftrace_func_address_lookup(struct ftrace_mod_map *mod_map, int ftrace_mod_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char **modname, char *sym) + unsigned long *off, char **modname, + const unsigned char **modbuildid, char *sym) { struct ftrace_mod_map *mod_map; int ret = 0; @@ -7765,6 +7752,8 @@ ftrace_mod_address_lookup(unsigned long addr, unsigned long *size, if (ret) { if (modname) *modname = mod_map->mod->name; + if (modbuildid) + *modbuildid = module_buildid(mod_map->mod); break; } } diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index 137b4d9bb116d9..2c6d3e33b9fb48 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -3963,11 +3963,6 @@ void trace_put_event_file(struct trace_event_file *file) EXPORT_SYMBOL_GPL(trace_put_event_file); #ifdef CONFIG_DYNAMIC_FTRACE - -/* Avoid typos */ -#define ENABLE_EVENT_STR "enable_event" -#define DISABLE_EVENT_STR "disable_event" - struct event_probe_data { struct trace_event_file *file; unsigned long count; diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index c97bb2fda5c02a..7e50df8b800b17 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -6911,7 +6911,7 @@ static int event_hist_trigger_parse(struct event_command *cmd_ops, remove_hist_vars(hist_data); - kfree(trigger_data); + trigger_data_free(trigger_data); destroy_hist_data(hist_data); goto out; diff --git a/kernel/ucount.c b/kernel/ucount.c index 586af49fc03e42..fc4a8f2d309651 100644 --- a/kernel/ucount.c +++ b/kernel/ucount.c @@ -47,7 +47,7 @@ static int set_permissions(struct ctl_table_header *head, int mode; /* Allow users with CAP_SYS_RESOURCE unrestrained access */ - if (ns_capable(user_ns, CAP_SYS_RESOURCE)) + if (ns_capable_noaudit(user_ns, CAP_SYS_RESOURCE)) mode = (table->mode & S_IRWXU) >> 6; else /* Allow all others at most read-only access */ diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 253311af47c6d3..2909c19540ed1e 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -117,6 +117,8 @@ enum wq_internal_consts { MAYDAY_INTERVAL = HZ / 10, /* and then every 100ms */ CREATE_COOLDOWN = HZ, /* time to breath after fail */ + RESCUER_BATCH = 16, /* process items per turn */ + /* * Rescue workers are used only on emergencies and shared by * all cpus. Give MIN_NICE. @@ -286,6 +288,7 @@ struct pool_workqueue { struct list_head pending_node; /* LN: node on wq_node_nr_active->pending_pwqs */ struct list_head pwqs_node; /* WR: node on wq->pwqs */ struct list_head mayday_node; /* MD: node on wq->maydays */ + struct work_struct mayday_cursor; /* L: cursor on pool->worklist */ u64 stats[PWQ_NR_STATS]; @@ -1120,6 +1123,12 @@ static struct worker *find_worker_executing_work(struct worker_pool *pool, return NULL; } +static void mayday_cursor_func(struct work_struct *work) +{ + /* should not be processed, only for marking position */ + BUG(); +} + /** * move_linked_works - move linked works to a list * @work: start of series of works to be scheduled @@ -1182,6 +1191,16 @@ static bool assign_work(struct work_struct *work, struct worker *worker, lockdep_assert_held(&pool->lock); + /* The cursor work should not be processed */ + if (unlikely(work->func == mayday_cursor_func)) { + /* only worker_thread() can possibly take this branch */ + WARN_ON_ONCE(worker->rescue_wq); + if (nextp) + *nextp = list_next_entry(work, entry); + list_del_init(&work->entry); + return false; + } + /* * A single work shouldn't be executed concurrently by multiple workers. * __queue_work() ensures that @work doesn't jump to a different pool @@ -3440,22 +3459,30 @@ static int worker_thread(void *__worker) static bool assign_rescuer_work(struct pool_workqueue *pwq, struct worker *rescuer) { struct worker_pool *pool = pwq->pool; + struct work_struct *cursor = &pwq->mayday_cursor; struct work_struct *work, *n; /* need rescue? */ if (!pwq->nr_active || !need_to_create_worker(pool)) return false; - /* - * Slurp in all works issued via this workqueue and - * process'em. - */ - list_for_each_entry_safe(work, n, &pool->worklist, entry) { - if (get_work_pwq(work) == pwq && assign_work(work, rescuer, &n)) + /* search from the start or cursor if available */ + if (list_empty(&cursor->entry)) + work = list_first_entry(&pool->worklist, struct work_struct, entry); + else + work = list_next_entry(cursor, entry); + + /* find the next work item to rescue */ + list_for_each_entry_safe_from(work, n, &pool->worklist, entry) { + if (get_work_pwq(work) == pwq && assign_work(work, rescuer, &n)) { pwq->stats[PWQ_STAT_RESCUED]++; + /* put the cursor for next search */ + list_move_tail(&cursor->entry, &n->entry); + return true; + } } - return !list_empty(&rescuer->scheduled); + return false; } /** @@ -3512,6 +3539,7 @@ static int rescuer_thread(void *__rescuer) struct pool_workqueue *pwq = list_first_entry(&wq->maydays, struct pool_workqueue, mayday_node); struct worker_pool *pool = pwq->pool; + unsigned int count = 0; __set_current_state(TASK_RUNNING); list_del_init(&pwq->mayday_node); @@ -3524,19 +3552,16 @@ static int rescuer_thread(void *__rescuer) WARN_ON_ONCE(!list_empty(&rescuer->scheduled)); - if (assign_rescuer_work(pwq, rescuer)) { + while (assign_rescuer_work(pwq, rescuer)) { process_scheduled_works(rescuer); /* - * The above execution of rescued work items could - * have created more to rescue through - * pwq_activate_first_inactive() or chained - * queueing. Let's put @pwq back on mayday list so - * that such back-to-back work items, which may be - * being used to relieve memory pressure, don't - * incur MAYDAY_INTERVAL delay inbetween. + * If the per-turn work item limit is reached and other + * PWQs are in mayday, requeue mayday for this PWQ and + * let the rescuer handle the other PWQs first. */ - if (pwq->nr_active && need_to_create_worker(pool)) { + if (++count > RESCUER_BATCH && !list_empty(&pwq->wq->maydays) && + pwq->nr_active && need_to_create_worker(pool)) { raw_spin_lock(&wq_mayday_lock); /* * Queue iff somebody else hasn't queued it already. @@ -3546,9 +3571,14 @@ static int rescuer_thread(void *__rescuer) list_add_tail(&pwq->mayday_node, &wq->maydays); } raw_spin_unlock(&wq_mayday_lock); + break; } } + /* The cursor can not be left behind without the rescuer watching it. */ + if (!list_empty(&pwq->mayday_cursor.entry) && list_empty(&pwq->mayday_node)) + list_del_init(&pwq->mayday_cursor.entry); + /* * Leave this pool. Notify regular workers; otherwise, we end up * with 0 concurrency and stalling the execution. @@ -5167,6 +5197,19 @@ static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq, INIT_LIST_HEAD(&pwq->pwqs_node); INIT_LIST_HEAD(&pwq->mayday_node); kthread_init_work(&pwq->release_work, pwq_release_workfn); + + /* + * Set the dummy cursor work with valid function and get_work_pwq(). + * + * The cursor work should only be in the pwq->pool->worklist, and + * should not be treated as a processable work item. + * + * WORK_STRUCT_PENDING and WORK_STRUCT_INACTIVE just make it less + * surprise for kernel debugging tools and reviewers. + */ + INIT_WORK(&pwq->mayday_cursor, mayday_cursor_func); + atomic_long_set(&pwq->mayday_cursor.data, (unsigned long)pwq | + WORK_STRUCT_PENDING | WORK_STRUCT_PWQ | WORK_STRUCT_INACTIVE); } /* sync @pwq with the current state of its associated wq and link it */ diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index ba36939fda79bf..cda3cf1fa302c3 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1273,7 +1273,7 @@ config BOOTPARAM_HUNG_TASK_PANIC high-availability systems that have uptime guarantees and where a hung tasks must be resolved ASAP. - Say N if unsure. + Say 0 if unsure. config DETECT_HUNG_TASK_BLOCKER bool "Dump Hung Tasks Blocker" diff --git a/lib/kstrtox.c b/lib/kstrtox.c index bdde40cd69d789..97be2a39f53710 100644 --- a/lib/kstrtox.c +++ b/lib/kstrtox.c @@ -340,8 +340,8 @@ EXPORT_SYMBOL(kstrtos8); * @s: input string * @res: result * - * This routine returns 0 iff the first character is one of 'YyTt1NnFf0', or - * [oO][NnFf] for "on" and "off". Otherwise it will return -EINVAL. Value + * This routine returns 0 iff the first character is one of 'EeYyTt1DdNnFf0', + * or [oO][NnFf] for "on" and "off". Otherwise it will return -EINVAL. Value * pointed to by res is updated upon finding a match. */ noinline diff --git a/lib/objpool.c b/lib/objpool.c index b998b720c7329d..d98fadf1de169f 100644 --- a/lib/objpool.c +++ b/lib/objpool.c @@ -142,7 +142,7 @@ int objpool_init(struct objpool_head *pool, int nr_objs, int object_size, pool->gfp = gfp & ~__GFP_ZERO; pool->context = context; pool->release = release; - slot_size = nr_cpu_ids * sizeof(struct objpool_slot); + slot_size = nr_cpu_ids * sizeof(struct objpool_slot *); pool->cpu_slots = kzalloc(slot_size, pool->gfp); if (!pool->cpu_slots) return -ENOMEM; diff --git a/lib/xarray.c b/lib/xarray.c index 9a8b4916540cf1..fe7f18d7194187 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -1738,9 +1738,6 @@ void *xa_store(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp) } EXPORT_SYMBOL(xa_store); -static inline void *__xa_cmpxchg_raw(struct xarray *xa, unsigned long index, - void *old, void *entry, gfp_t gfp); - /** * __xa_cmpxchg() - Conditionally replace an entry in the XArray. * @xa: XArray. @@ -1767,7 +1764,29 @@ void *__xa_cmpxchg(struct xarray *xa, unsigned long index, } EXPORT_SYMBOL(__xa_cmpxchg); -static inline void *__xa_cmpxchg_raw(struct xarray *xa, unsigned long index, +/** + * __xa_cmpxchg_raw() - Conditionally replace an entry in the XArray. + * @xa: XArray. + * @index: Index into array. + * @old: Old value to test against. + * @entry: New value to place in array. + * @gfp: Memory allocation flags. + * + * You must already be holding the xa_lock when calling this function. + * It will drop the lock if needed to allocate memory, and then reacquire + * it afterwards. + * + * If the entry at @index is the same as @old, replace it with @entry. + * If the return value is equal to @old, then the exchange was successful. + * + * This function is the same as __xa_cmpxchg() except that it does not coerce + * XA_ZERO_ENTRY to NULL on egress. + * + * Context: Any context. Expects xa_lock to be held on entry. May + * release and reacquire xa_lock if @gfp flags permit. + * Return: The old value at this index or xa_err() if an error happened. + */ +void *__xa_cmpxchg_raw(struct xarray *xa, unsigned long index, void *old, void *entry, gfp_t gfp) { XA_STATE(xas, xa, index); @@ -1787,6 +1806,7 @@ static inline void *__xa_cmpxchg_raw(struct xarray *xa, unsigned long index, return xas_result(&xas, curr); } +EXPORT_SYMBOL(__xa_cmpxchg_raw); /** * __xa_insert() - Store this entry in the XArray if no entry is present. diff --git a/mm/slub.c b/mm/slub.c index cdc1e652ec52fe..e1583757331e75 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -6265,11 +6265,29 @@ static void rcu_free_sheaf(struct rcu_head *head) free_empty_sheaf(s, sheaf); } +/* + * kvfree_call_rcu() can be called while holding a raw_spinlock_t. Since + * __kfree_rcu_sheaf() may acquire a spinlock_t (sleeping lock on PREEMPT_RT), + * this would violate lock nesting rules. Therefore, kvfree_call_rcu() avoids + * this problem by bypassing the sheaves layer entirely on PREEMPT_RT. + * + * However, lockdep still complains that it is invalid to acquire spinlock_t + * while holding raw_spinlock_t, even on !PREEMPT_RT where spinlock_t is a + * spinning lock. Tell lockdep that acquiring spinlock_t is valid here + * by temporarily raising the wait-type to LD_WAIT_CONFIG. + */ +static DEFINE_WAIT_OVERRIDE_MAP(kfree_rcu_sheaf_map, LD_WAIT_CONFIG); + bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) { struct slub_percpu_sheaves *pcs; struct slab_sheaf *rcu_sheaf; + if (WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_RT))) + return false; + + lock_map_acquire_try(&kfree_rcu_sheaf_map); + if (!local_trylock(&s->cpu_sheaves->lock)) goto fail; @@ -6346,10 +6364,12 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) local_unlock(&s->cpu_sheaves->lock); stat(s, FREE_RCU_SHEAF); + lock_map_release(&kfree_rcu_sheaf_map); return true; fail: stat(s, FREE_RCU_SHEAF_FAIL); + lock_map_release(&kfree_rcu_sheaf_map); return false; } diff --git a/net/atm/signaling.c b/net/atm/signaling.c index e70ae2c113f954..358fbe5e4d1d06 100644 --- a/net/atm/signaling.c +++ b/net/atm/signaling.c @@ -22,6 +22,36 @@ struct atm_vcc *sigd = NULL; +/* + * find_get_vcc - validate and get a reference to a vcc pointer + * @vcc: the vcc pointer to validate + * + * This function validates that @vcc points to a registered VCC in vcc_hash. + * If found, it increments the socket reference count and returns the vcc. + * The caller must call sock_put(sk_atm(vcc)) when done. + * + * Returns the vcc pointer if valid, NULL otherwise. + */ +static struct atm_vcc *find_get_vcc(struct atm_vcc *vcc) +{ + int i; + + read_lock(&vcc_sklist_lock); + for (i = 0; i < VCC_HTABLE_SIZE; i++) { + struct sock *s; + + sk_for_each(s, &vcc_hash[i]) { + if (atm_sk(s) == vcc) { + sock_hold(s); + read_unlock(&vcc_sklist_lock); + return vcc; + } + } + } + read_unlock(&vcc_sklist_lock); + return NULL; +} + static void sigd_put_skb(struct sk_buff *skb) { if (!sigd) { @@ -69,7 +99,14 @@ static int sigd_send(struct atm_vcc *vcc, struct sk_buff *skb) msg = (struct atmsvc_msg *) skb->data; WARN_ON(refcount_sub_and_test(skb->truesize, &sk_atm(vcc)->sk_wmem_alloc)); - vcc = *(struct atm_vcc **) &msg->vcc; + + vcc = find_get_vcc(*(struct atm_vcc **)&msg->vcc); + if (!vcc) { + pr_debug("invalid vcc pointer in msg\n"); + dev_kfree_skb(skb); + return -EINVAL; + } + pr_debug("%d (0x%lx)\n", (int)msg->type, (unsigned long)vcc); sk = sk_atm(vcc); @@ -100,7 +137,16 @@ static int sigd_send(struct atm_vcc *vcc, struct sk_buff *skb) clear_bit(ATM_VF_WAITING, &vcc->flags); break; case as_indicate: - vcc = *(struct atm_vcc **)&msg->listen_vcc; + /* Release the reference from msg->vcc, we'll use msg->listen_vcc instead */ + sock_put(sk); + + vcc = find_get_vcc(*(struct atm_vcc **)&msg->listen_vcc); + if (!vcc) { + pr_debug("invalid listen_vcc pointer in msg\n"); + dev_kfree_skb(skb); + return -EINVAL; + } + sk = sk_atm(vcc); pr_debug("as_indicate!!!\n"); lock_sock(sk); @@ -115,6 +161,8 @@ static int sigd_send(struct atm_vcc *vcc, struct sk_buff *skb) sk->sk_state_change(sk); as_indicate_complete: release_sock(sk); + /* Paired with find_get_vcc(msg->listen_vcc) above */ + sock_put(sk); return 0; case as_close: set_bit(ATM_VF_RELEASED, &vcc->flags); @@ -131,11 +179,15 @@ static int sigd_send(struct atm_vcc *vcc, struct sk_buff *skb) break; default: pr_alert("bad message type %d\n", (int)msg->type); + /* Paired with find_get_vcc(msg->vcc) above */ + sock_put(sk); return -EINVAL; } sk->sk_state_change(sk); out: dev_kfree_skb(skb); + /* Paired with find_get_vcc(msg->vcc) above */ + sock_put(sk); return 0; } diff --git a/net/bluetooth/hci_conn.c b/net/bluetooth/hci_conn.c index c3f7828bf9d54a..5a4374ccf8e840 100644 --- a/net/bluetooth/hci_conn.c +++ b/net/bluetooth/hci_conn.c @@ -1008,6 +1008,11 @@ static struct hci_conn *__hci_conn_add(struct hci_dev *hdev, int type, /* conn->src should reflect the local identity address */ hci_copy_identity_address(hdev, &conn->src, &conn->src_type); conn->mtu = hdev->le_mtu ? hdev->le_mtu : hdev->acl_mtu; + /* Use the controller supported PHYS as default until the + * remote features are resolved. + */ + conn->le_tx_def_phys = hdev->le_tx_def_phys; + conn->le_rx_def_phys = hdev->le_tx_def_phys; break; case CIS_LINK: /* conn->src should reflect the local identity address */ @@ -2928,22 +2933,22 @@ u32 hci_conn_get_phy(struct hci_conn *conn) break; case LE_LINK: - if (conn->le_tx_phy & HCI_LE_SET_PHY_1M) + if (conn->le_tx_def_phys & HCI_LE_SET_PHY_1M) phys |= BT_PHY_LE_1M_TX; - if (conn->le_rx_phy & HCI_LE_SET_PHY_1M) + if (conn->le_rx_def_phys & HCI_LE_SET_PHY_1M) phys |= BT_PHY_LE_1M_RX; - if (conn->le_tx_phy & HCI_LE_SET_PHY_2M) + if (conn->le_tx_def_phys & HCI_LE_SET_PHY_2M) phys |= BT_PHY_LE_2M_TX; - if (conn->le_rx_phy & HCI_LE_SET_PHY_2M) + if (conn->le_rx_def_phys & HCI_LE_SET_PHY_2M) phys |= BT_PHY_LE_2M_RX; - if (conn->le_tx_phy & HCI_LE_SET_PHY_CODED) + if (conn->le_tx_def_phys & HCI_LE_SET_PHY_CODED) phys |= BT_PHY_LE_CODED_TX; - if (conn->le_rx_phy & HCI_LE_SET_PHY_CODED) + if (conn->le_rx_def_phys & HCI_LE_SET_PHY_CODED) phys |= BT_PHY_LE_CODED_RX; break; diff --git a/net/bluetooth/hci_event.c b/net/bluetooth/hci_event.c index a9868f17ef40ff..58075bf7205540 100644 --- a/net/bluetooth/hci_event.c +++ b/net/bluetooth/hci_event.c @@ -6607,8 +6607,20 @@ static void hci_le_remote_feat_complete_evt(struct hci_dev *hdev, void *data, conn = hci_conn_hash_lookup_handle(hdev, __le16_to_cpu(ev->handle)); if (conn) { - if (!ev->status) - memcpy(conn->features[0], ev->features, 8); + if (!ev->status) { + memcpy(conn->le_features, ev->features, 8); + + /* Update supported PHYs */ + if (!(conn->le_features[1] & HCI_LE_PHY_2M)) { + conn->le_tx_def_phys &= ~HCI_LE_SET_PHY_2M; + conn->le_rx_def_phys &= ~HCI_LE_SET_PHY_2M; + } + + if (!(conn->le_features[1] & HCI_LE_PHY_CODED)) { + conn->le_tx_def_phys &= ~HCI_LE_SET_PHY_CODED; + conn->le_rx_def_phys &= ~HCI_LE_SET_PHY_CODED; + } + } if (conn->state == BT_CONFIG) { __u8 status; @@ -7221,9 +7233,21 @@ static void hci_le_read_all_remote_features_evt(struct hci_dev *hdev, if (!conn) goto unlock; - if (!ev->status) + if (!ev->status) { memcpy(conn->le_features, ev->features, 248); + /* Update supported PHYs */ + if (!(conn->le_features[1] & HCI_LE_PHY_2M)) { + conn->le_tx_def_phys &= ~HCI_LE_SET_PHY_2M; + conn->le_rx_def_phys &= ~HCI_LE_SET_PHY_2M; + } + + if (!(conn->le_features[1] & HCI_LE_PHY_CODED)) { + conn->le_tx_def_phys &= ~HCI_LE_SET_PHY_CODED; + conn->le_rx_def_phys &= ~HCI_LE_SET_PHY_CODED; + } + } + if (conn->state == BT_CONFIG) { __u8 status; diff --git a/net/bridge/br_multicast.c b/net/bridge/br_multicast.c index d55a4ab87837f7..e9a7e653040177 100644 --- a/net/bridge/br_multicast.c +++ b/net/bridge/br_multicast.c @@ -244,14 +244,11 @@ br_multicast_port_vid_to_port_ctx(struct net_bridge_port *port, u16 vid) lockdep_assert_held_once(&port->br->multicast_lock); - if (!br_opt_get(port->br, BROPT_MCAST_VLAN_SNOOPING_ENABLED)) - return NULL; - /* Take RCU to access the vlan. */ rcu_read_lock(); vlan = br_vlan_find(nbp_vlan_group_rcu(port), vid); - if (vlan && !br_multicast_port_ctx_vlan_disabled(&vlan->port_mcast_ctx)) + if (vlan) pmctx = &vlan->port_mcast_ctx; rcu_read_unlock(); @@ -701,7 +698,10 @@ br_multicast_port_ngroups_inc_one(struct net_bridge_mcast_port *pmctx, u32 max = READ_ONCE(pmctx->mdb_max_entries); u32 n = READ_ONCE(pmctx->mdb_n_entries); - if (max && n >= max) { + /* enforce the max limit when it's a port pmctx or a port-vlan pmctx + * with snooping enabled + */ + if (!br_multicast_port_ctx_vlan_disabled(pmctx) && max && n >= max) { NL_SET_ERR_MSG_FMT_MOD(extack, "%s is already in %u groups, and mcast_max_groups=%u", what, n, max); return -E2BIG; @@ -736,9 +736,7 @@ static int br_multicast_port_ngroups_inc(struct net_bridge_port *port, return err; } - /* Only count on the VLAN context if VID is given, and if snooping on - * that VLAN is enabled. - */ + /* Only count on the VLAN context if VID is given */ if (!group->vid) return 0; @@ -2011,6 +2009,18 @@ void br_multicast_port_ctx_init(struct net_bridge_port *port, timer_setup(&pmctx->ip6_own_query.timer, br_ip6_multicast_port_query_expired, 0); #endif + /* initialize mdb_n_entries if a new port vlan is being created */ + if (vlan) { + struct net_bridge_port_group *pg; + u32 n = 0; + + spin_lock_bh(&port->br->multicast_lock); + hlist_for_each_entry(pg, &port->mglist, mglist) + if (pg->key.addr.vid == vlan->vid) + n++; + WRITE_ONCE(pmctx->mdb_n_entries, n); + spin_unlock_bh(&port->br->multicast_lock); + } } void br_multicast_port_ctx_deinit(struct net_bridge_mcast_port *pmctx) @@ -2094,25 +2104,6 @@ static void __br_multicast_enable_port_ctx(struct net_bridge_mcast_port *pmctx) br_ip4_multicast_add_router(brmctx, pmctx); br_ip6_multicast_add_router(brmctx, pmctx); } - - if (br_multicast_port_ctx_is_vlan(pmctx)) { - struct net_bridge_port_group *pg; - u32 n = 0; - - /* The mcast_n_groups counter might be wrong. First, - * BR_VLFLAG_MCAST_ENABLED is toggled before temporary entries - * are flushed, thus mcast_n_groups after the toggle does not - * reflect the true values. And second, permanent entries added - * while BR_VLFLAG_MCAST_ENABLED was disabled, are not reflected - * either. Thus we have to refresh the counter. - */ - - hlist_for_each_entry(pg, &pmctx->port->mglist, mglist) { - if (pg->key.addr.vid == pmctx->vlan->vid) - n++; - } - WRITE_ONCE(pmctx->mdb_n_entries, n); - } } static void br_multicast_enable_port_ctx(struct net_bridge_mcast_port *pmctx) diff --git a/net/core/dev.c b/net/core/dev.c index ccef685023c299..f5e4040e08399c 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -738,7 +738,7 @@ static struct net_device_path *dev_fwd_path(struct net_device_path_stack *stack) { int k = stack->num_paths++; - if (WARN_ON_ONCE(k >= NET_DEVICE_PATH_STACK_MAX)) + if (k >= NET_DEVICE_PATH_STACK_MAX) return NULL; return &stack->path[k]; diff --git a/net/core/filter.c b/net/core/filter.c index 029e560e32ce3e..51318cb40f7788 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -4137,7 +4137,7 @@ static const struct bpf_func_proto bpf_xdp_store_bytes_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, - .arg3_type = ARG_PTR_TO_UNINIT_MEM, + .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg4_type = ARG_CONST_SIZE, }; @@ -6401,7 +6401,7 @@ static const struct bpf_func_proto bpf_xdp_fib_lookup_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, .arg3_type = ARG_CONST_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -6456,7 +6456,7 @@ static const struct bpf_func_proto bpf_skb_fib_lookup_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, .arg3_type = ARG_CONST_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -8010,9 +8010,9 @@ static const struct bpf_func_proto bpf_tcp_raw_gen_syncookie_ipv4_proto = { .gpl_only = true, /* __cookie_v4_init_sequence() is GPL */ .pkt_access = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM, + .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY, .arg1_size = sizeof(struct iphdr), - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE_OR_ZERO, }; @@ -8042,9 +8042,9 @@ static const struct bpf_func_proto bpf_tcp_raw_gen_syncookie_ipv6_proto = { .gpl_only = true, /* __cookie_v6_init_sequence() is GPL */ .pkt_access = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM, + .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY, .arg1_size = sizeof(struct ipv6hdr), - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE_OR_ZERO, }; @@ -8062,9 +8062,9 @@ static const struct bpf_func_proto bpf_tcp_raw_check_syncookie_ipv4_proto = { .gpl_only = true, /* __cookie_v4_check is GPL */ .pkt_access = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM, + .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY, .arg1_size = sizeof(struct iphdr), - .arg2_type = ARG_PTR_TO_FIXED_SIZE_MEM, + .arg2_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY, .arg2_size = sizeof(struct tcphdr), }; @@ -8086,9 +8086,9 @@ static const struct bpf_func_proto bpf_tcp_raw_check_syncookie_ipv6_proto = { .gpl_only = true, /* __cookie_v6_check is GPL */ .pkt_access = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM, + .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY, .arg1_size = sizeof(struct ipv6hdr), - .arg2_type = ARG_PTR_TO_FIXED_SIZE_MEM, + .arg2_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY, .arg2_size = sizeof(struct tcphdr), }; #endif /* CONFIG_SYN_COOKIES */ diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 61746c2b95f63e..fa6209f45de9cd 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -7231,10 +7231,15 @@ void skb_attempt_defer_free(struct sk_buff *skb) { struct skb_defer_node *sdn; unsigned long defer_count; - int cpu = skb->alloc_cpu; unsigned int defer_max; bool kick; + int cpu; + /* zero copy notifications should not be delayed. */ + if (skb_zcopy(skb)) + goto nodefer; + + cpu = skb->alloc_cpu; if (cpu == raw_smp_processor_id() || WARN_ON_ONCE(cpu >= nr_cpu_ids) || !cpu_online(cpu)) { diff --git a/net/core/skmsg.c b/net/core/skmsg.c index 2ac7731e1e0a74..ddde93dd8bc6d4 100644 --- a/net/core/skmsg.c +++ b/net/core/skmsg.c @@ -409,22 +409,26 @@ int sk_msg_memcopy_from_iter(struct sock *sk, struct iov_iter *from, } EXPORT_SYMBOL_GPL(sk_msg_memcopy_from_iter); -/* Receive sk_msg from psock->ingress_msg to @msg. */ -int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, - int len, int flags) +int __sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, + int len, int flags, int *copied_from_self) { struct iov_iter *iter = &msg->msg_iter; int peek = flags & MSG_PEEK; struct sk_msg *msg_rx; int i, copied = 0; + bool from_self; msg_rx = sk_psock_peek_msg(psock); + if (copied_from_self) + *copied_from_self = 0; + while (copied != len) { struct scatterlist *sge; if (unlikely(!msg_rx)) break; + from_self = msg_rx->sk == sk; i = msg_rx->sg.start; do { struct page *page; @@ -443,6 +447,9 @@ int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, } copied += copy; + if (from_self && copied_from_self) + *copied_from_self += copy; + if (likely(!peek)) { sge->offset += copy; sge->length -= copy; @@ -451,6 +458,7 @@ int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, atomic_sub(copy, &sk->sk_rmem_alloc); } msg_rx->sg.size -= copy; + sk_psock_msg_len_add(psock, -copy); if (!sge->length) { sk_msg_iter_var_next(i); @@ -487,6 +495,13 @@ int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, out: return copied; } + +/* Receive sk_msg from psock->ingress_msg to @msg. */ +int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, + int len, int flags) +{ + return __sk_msg_recvmsg(sk, psock, msg, len, flags, NULL); +} EXPORT_SYMBOL_GPL(sk_msg_recvmsg); bool sk_msg_is_readable(struct sock *sk) @@ -616,6 +631,12 @@ static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb if (unlikely(!msg)) return -EAGAIN; skb_set_owner_r(skb, sk); + + /* This is used in tcp_bpf_recvmsg_parser() to determine whether the + * data originates from the socket's own protocol stack. No need to + * refcount sk because msg's lifetime is bound to sk via the ingress_msg. + */ + msg->sk = sk; err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg, take_ref); if (err < 0) kfree(msg); @@ -801,9 +822,11 @@ static void __sk_psock_purge_ingress_msg(struct sk_psock *psock) list_del(&msg->list); if (!msg->skb) atomic_sub(msg->sg.size, &psock->sk->sk_rmem_alloc); + sk_psock_msg_len_add(psock, -msg->sg.size); sk_msg_free(psock->sk, msg); kfree(msg); } + WARN_ON_ONCE(psock->msg_tot_len); } static void __sk_psock_zap_ingress(struct sk_psock *psock) @@ -909,6 +932,7 @@ int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock, sk_msg_compute_data_pointers(msg); msg->sk = sk; ret = bpf_prog_run_pin_on_cpu(prog, msg); + msg->sk = NULL; ret = sk_psock_map_verd(ret, msg->sk_redir); psock->apply_bytes = msg->apply_bytes; if (ret == __SK_REDIRECT) { diff --git a/net/ipv4/icmp.c b/net/ipv4/icmp.c index 4abbec2f47ef58..471dd862f66394 100644 --- a/net/ipv4/icmp.c +++ b/net/ipv4/icmp.c @@ -248,7 +248,8 @@ bool icmp_global_allow(struct net *net) if (delta < HZ / 50) return false; - incr = READ_ONCE(net->ipv4.sysctl_icmp_msgs_per_sec) * delta / HZ; + incr = READ_ONCE(net->ipv4.sysctl_icmp_msgs_per_sec); + incr = div_u64((u64)incr * delta, HZ); if (!incr) return false; @@ -554,6 +555,21 @@ static struct rtable *icmp_route_lookup(struct net *net, struct flowi4 *fl4, /* steal dst entry from skb_in, don't drop refcnt */ skb_dstref_steal(skb_in); skb_dstref_restore(skb_in, orefdst); + + /* + * At this point, fl4_dec.daddr should NOT be local (we + * checked fl4_dec.saddr above). However, a race condition + * may occur if the address is added to the interface + * concurrently. In that case, ip_route_input() returns a + * LOCAL route with dst.output=ip_rt_bug, which must not + * be used for output. + */ + if (!err && rt2 && rt2->rt_type == RTN_LOCAL) { + net_warn_ratelimited("detected local route for %pI4 during ICMP sending, src %pI4\n", + &fl4_dec.daddr, &fl4_dec.saddr); + dst_release(&rt2->dst); + err = -EINVAL; + } } if (err) @@ -1031,16 +1047,22 @@ static void icmp_socket_deliver(struct sk_buff *skb, u32 info) /* Checkin full IP header plus 8 bytes of protocol to * avoid additional coding at protocol handlers. */ - if (!pskb_may_pull(skb, iph->ihl * 4 + 8)) { - __ICMP_INC_STATS(dev_net_rcu(skb->dev), ICMP_MIB_INERRORS); - return; - } + if (!pskb_may_pull(skb, iph->ihl * 4 + 8)) + goto out; + + /* IPPROTO_RAW sockets are not supposed to receive anything. */ + if (protocol == IPPROTO_RAW) + goto out; raw_icmp_error(skb, protocol, info); ipprot = rcu_dereference(inet_protos[protocol]); if (ipprot && ipprot->err_handler) ipprot->err_handler(skb, info); + return; + +out: + __ICMP_INC_STATS(dev_net_rcu(skb->dev), ICMP_MIB_INERRORS); } static bool icmp_tag_validation(int proto) diff --git a/net/ipv4/ping.c b/net/ipv4/ping.c index cfbd563498e85e..0fec4e56456671 100644 --- a/net/ipv4/ping.c +++ b/net/ipv4/ping.c @@ -148,7 +148,7 @@ void ping_unhash(struct sock *sk) pr_debug("ping_unhash(isk=%p,isk->num=%u)\n", isk, isk->inet_num); spin_lock(&ping_table.lock); if (sk_del_node_init_rcu(sk)) { - isk->inet_num = 0; + WRITE_ONCE(isk->inet_num, 0); isk->inet_sport = 0; sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); } @@ -181,31 +181,35 @@ static struct sock *ping_lookup(struct net *net, struct sk_buff *skb, u16 ident) } sk_for_each_rcu(sk, hslot) { + int bound_dev_if; + if (!net_eq(sock_net(sk), net)) continue; isk = inet_sk(sk); pr_debug("iterate\n"); - if (isk->inet_num != ident) + if (READ_ONCE(isk->inet_num) != ident) continue; + bound_dev_if = READ_ONCE(sk->sk_bound_dev_if); if (skb->protocol == htons(ETH_P_IP) && sk->sk_family == AF_INET) { + __be32 rcv_saddr = READ_ONCE(isk->inet_rcv_saddr); + pr_debug("found: %p: num=%d, daddr=%pI4, dif=%d\n", sk, - (int) isk->inet_num, &isk->inet_rcv_saddr, - sk->sk_bound_dev_if); + ident, &rcv_saddr, + bound_dev_if); - if (isk->inet_rcv_saddr && - isk->inet_rcv_saddr != ip_hdr(skb)->daddr) + if (rcv_saddr && rcv_saddr != ip_hdr(skb)->daddr) continue; #if IS_ENABLED(CONFIG_IPV6) } else if (skb->protocol == htons(ETH_P_IPV6) && sk->sk_family == AF_INET6) { pr_debug("found: %p: num=%d, daddr=%pI6c, dif=%d\n", sk, - (int) isk->inet_num, + ident, &sk->sk_v6_rcv_saddr, - sk->sk_bound_dev_if); + bound_dev_if); if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr) && !ipv6_addr_equal(&sk->sk_v6_rcv_saddr, @@ -216,8 +220,8 @@ static struct sock *ping_lookup(struct net *net, struct sk_buff *skb, u16 ident) continue; } - if (sk->sk_bound_dev_if && sk->sk_bound_dev_if != dif && - sk->sk_bound_dev_if != sdif) + if (bound_dev_if && bound_dev_if != dif && + bound_dev_if != sdif) continue; goto exit; @@ -392,7 +396,9 @@ static void ping_set_saddr(struct sock *sk, struct sockaddr_unsized *saddr) if (saddr->sa_family == AF_INET) { struct inet_sock *isk = inet_sk(sk); struct sockaddr_in *addr = (struct sockaddr_in *) saddr; - isk->inet_rcv_saddr = isk->inet_saddr = addr->sin_addr.s_addr; + + isk->inet_saddr = addr->sin_addr.s_addr; + WRITE_ONCE(isk->inet_rcv_saddr, addr->sin_addr.s_addr); #if IS_ENABLED(CONFIG_IPV6) } else if (saddr->sa_family == AF_INET6) { struct sockaddr_in6 *addr = (struct sockaddr_in6 *) saddr; @@ -849,7 +855,8 @@ int ping_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags, struct sk_buff *skb; int copied, err; - pr_debug("ping_recvmsg(sk=%p,sk->num=%u)\n", isk, isk->inet_num); + pr_debug("ping_recvmsg(sk=%p,sk->num=%u)\n", isk, + READ_ONCE(isk->inet_num)); err = -EOPNOTSUPP; if (flags & MSG_OOB) diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index d5319ebe24525e..81666571ecfb59 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -501,6 +501,9 @@ static void tcp_tx_timestamp(struct sock *sk, struct sockcm_cookie *sockc) struct sk_buff *skb = tcp_write_queue_tail(sk); u32 tsflags = sockc->tsflags; + if (unlikely(!skb)) + skb = skb_rb_last(&sk->tcp_rtx_queue); + if (tsflags && skb) { struct skb_shared_info *shinfo = skb_shinfo(skb); struct tcp_skb_cb *tcb = TCP_SKB_CB(skb); diff --git a/net/ipv4/tcp_bpf.c b/net/ipv4/tcp_bpf.c index a268e1595b22aa..ca8a5cb8e569d7 100644 --- a/net/ipv4/tcp_bpf.c +++ b/net/ipv4/tcp_bpf.c @@ -10,6 +10,7 @@ #include #include +#include void tcp_eat_skb(struct sock *sk, struct sk_buff *skb) { @@ -226,6 +227,7 @@ static int tcp_bpf_recvmsg_parser(struct sock *sk, int peek = flags & MSG_PEEK; struct sk_psock *psock; struct tcp_sock *tcp; + int copied_from_self = 0; int copied = 0; u32 seq; @@ -262,7 +264,7 @@ static int tcp_bpf_recvmsg_parser(struct sock *sk, } msg_bytes_ready: - copied = sk_msg_recvmsg(sk, psock, msg, len, flags); + copied = __sk_msg_recvmsg(sk, psock, msg, len, flags, &copied_from_self); /* The typical case for EFAULT is the socket was gracefully * shutdown with a FIN pkt. So check here the other case is * some error on copy_page_to_iter which would be unexpected. @@ -277,7 +279,7 @@ static int tcp_bpf_recvmsg_parser(struct sock *sk, goto out; } } - seq += copied; + seq += copied_from_self; if (!copied) { long timeo; int data; @@ -331,6 +333,24 @@ static int tcp_bpf_recvmsg_parser(struct sock *sk, return copied; } +static int tcp_bpf_ioctl(struct sock *sk, int cmd, int *karg) +{ + bool slow; + + if (cmd != SIOCINQ) + return tcp_ioctl(sk, cmd, karg); + + /* works similar as tcp_ioctl */ + if (sk->sk_state == TCP_LISTEN) + return -EINVAL; + + slow = lock_sock_fast(sk); + *karg = sk_psock_msg_inq(sk); + unlock_sock_fast(sk, slow); + + return 0; +} + static int tcp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags, int *addr_len) { @@ -609,6 +629,7 @@ static void tcp_bpf_rebuild_protos(struct proto prot[TCP_BPF_NUM_CFGS], prot[TCP_BPF_BASE].close = sock_map_close; prot[TCP_BPF_BASE].recvmsg = tcp_bpf_recvmsg; prot[TCP_BPF_BASE].sock_is_readable = sk_msg_is_readable; + prot[TCP_BPF_BASE].ioctl = tcp_bpf_ioctl; prot[TCP_BPF_TX] = prot[TCP_BPF_BASE]; prot[TCP_BPF_TX].sendmsg = tcp_bpf_sendmsg; diff --git a/net/ipv4/tcp_cong.c b/net/ipv4/tcp_cong.c index df758adbb445f2..e9f6c77e063163 100644 --- a/net/ipv4/tcp_cong.c +++ b/net/ipv4/tcp_cong.c @@ -16,6 +16,7 @@ #include #include #include +#include #include static DEFINE_SPINLOCK(tcp_cong_list_lock); @@ -227,7 +228,7 @@ void tcp_assign_congestion_control(struct sock *sk) memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv)); if (ca->flags & TCP_CONG_NEEDS_ECN) - INET_ECN_xmit(sk); + INET_ECN_xmit_ect_1_negotiation(sk); else INET_ECN_dontxmit(sk); } @@ -257,7 +258,7 @@ static void tcp_reinit_congestion_control(struct sock *sk, memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv)); if (ca->flags & TCP_CONG_NEEDS_ECN) - INET_ECN_xmit(sk); + INET_ECN_xmit_ect_1_negotiation(sk); else INET_ECN_dontxmit(sk); diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index 198f8a0d37be04..0d080a3e27d6f8 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -6843,7 +6843,7 @@ static int tcp_rcv_synsent_state_process(struct sock *sk, struct sk_buff *skb, tp->snd_wl1 = TCP_SKB_CB(skb)->seq; tp->max_window = tp->snd_wnd; - tcp_ecn_rcv_syn(tp, th, skb); + tcp_ecn_rcv_syn(sk, th, skb); tcp_mtup_init(sk); tcp_sync_mss(sk, icsk->icsk_pmtu_cookie); @@ -7248,7 +7248,8 @@ static void tcp_ecn_create_request(struct request_sock *req, u32 ecn_ok_dst; if (tcp_accecn_syn_requested(th) && - READ_ONCE(net->ipv4.sysctl_tcp_ecn) >= 3) { + (READ_ONCE(net->ipv4.sysctl_tcp_ecn) >= 3 || + tcp_ca_needs_accecn(listen_sk))) { inet_rsk(req)->ecn_ok = 1; tcp_rsk(req)->accecn_ok = 1; tcp_rsk(req)->syn_ect_rcv = TCP_SKB_CB(skb)->ip_dsfield & diff --git a/net/ipv4/tcp_minisocks.c b/net/ipv4/tcp_minisocks.c index bd5462154f970b..9776c921d1bb4d 100644 --- a/net/ipv4/tcp_minisocks.c +++ b/net/ipv4/tcp_minisocks.c @@ -485,9 +485,10 @@ static void tcp_ecn_openreq_child(struct sock *sk, tp->accecn_opt_demand = 1; tcp_ecn_received_counters_payload(sk, skb); } else { - tcp_ecn_mode_set(tp, inet_rsk(req)->ecn_ok ? - TCP_ECN_MODE_RFC3168 : - TCP_ECN_DISABLED); + if (inet_rsk(req)->ecn_ok && !tcp_ca_no_fallback_rfc3168(sk)) + tcp_ecn_mode_set(tp, TCP_ECN_MODE_RFC3168); + else + tcp_ecn_mode_set(tp, TCP_ECN_DISABLED); } } diff --git a/net/ipv4/udp_bpf.c b/net/ipv4/udp_bpf.c index 0735d820e413f3..91233e37cd97a2 100644 --- a/net/ipv4/udp_bpf.c +++ b/net/ipv4/udp_bpf.c @@ -5,6 +5,7 @@ #include #include #include +#include #include "udp_impl.h" @@ -111,12 +112,26 @@ enum { static DEFINE_SPINLOCK(udpv6_prot_lock); static struct proto udp_bpf_prots[UDP_BPF_NUM_PROTS]; +static int udp_bpf_ioctl(struct sock *sk, int cmd, int *karg) +{ + if (cmd != SIOCINQ) + return udp_ioctl(sk, cmd, karg); + + /* Since we don't hold a lock, sk_receive_queue may contain data. + * BPF might only be processing this data at the moment. We only + * care about the data in the ingress_msg here. + */ + *karg = sk_msg_first_len(sk); + return 0; +} + static void udp_bpf_rebuild_protos(struct proto *prot, const struct proto *base) { - *prot = *base; - prot->close = sock_map_close; - prot->recvmsg = udp_bpf_recvmsg; - prot->sock_is_readable = sk_msg_is_readable; + *prot = *base; + prot->close = sock_map_close; + prot->recvmsg = udp_bpf_recvmsg; + prot->sock_is_readable = sk_msg_is_readable; + prot->ioctl = udp_bpf_ioctl; } static void udp_bpf_check_v6_needs_rebuild(struct proto *ops) diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c index b705751eb73c6b..d3534bdb805da8 100644 --- a/net/ipv6/af_inet6.c +++ b/net/ipv6/af_inet6.c @@ -955,7 +955,7 @@ static int __net_init inet6_net_init(struct net *net) int err = 0; net->ipv6.sysctl.bindv6only = 0; - net->ipv6.sysctl.icmpv6_time = 1*HZ; + net->ipv6.sysctl.icmpv6_time = HZ / 10; net->ipv6.sysctl.icmpv6_echo_ignore_all = 0; net->ipv6.sysctl.icmpv6_echo_ignore_multicast = 0; net->ipv6.sysctl.icmpv6_echo_ignore_anycast = 0; diff --git a/net/ipv6/icmp.c b/net/ipv6/icmp.c index 9d37e7711bc2b8..55b1aa75ab8028 100644 --- a/net/ipv6/icmp.c +++ b/net/ipv6/icmp.c @@ -217,14 +217,9 @@ static bool icmpv6_xrlim_allow(struct sock *sk, u8 type, } else if (dev && (dev->flags & IFF_LOOPBACK)) { res = true; } else { - struct rt6_info *rt = dst_rt6_info(dst); - int tmo = net->ipv6.sysctl.icmpv6_time; + int tmo = READ_ONCE(net->ipv6.sysctl.icmpv6_time); struct inet_peer *peer; - /* Give more bandwidth to wider prefixes. */ - if (rt->rt6i_dst.plen < 128) - tmo >>= ((128 - rt->rt6i_dst.plen)>>5); - peer = inet_getpeer_v6(net->ipv6.peers, &fl6->daddr); res = inet_peer_xrlim_allow(peer, tmo); } @@ -1066,6 +1061,12 @@ enum skb_drop_reason icmpv6_notify(struct sk_buff *skb, u8 type, if (reason != SKB_NOT_DROPPED_YET) goto out; + if (nexthdr == IPPROTO_RAW) { + /* Add a more specific reason later ? */ + reason = SKB_DROP_REASON_NOT_SPECIFIED; + goto out; + } + /* BUGGG_FUTURE: we should try to parse exthdrs in this packet. Without this we will not able f.e. to make source routed pmtu discovery. diff --git a/net/ipv6/ip6_fib.c b/net/ipv6/ip6_fib.c index c6439e30e892af..cc149227b49f4a 100644 --- a/net/ipv6/ip6_fib.c +++ b/net/ipv6/ip6_fib.c @@ -1139,7 +1139,7 @@ static int fib6_add_rt2node(struct fib6_node *fn, struct fib6_info *rt, fib6_add_gc_list(iter); } if (!(rt->fib6_flags & (RTF_ADDRCONF | RTF_PREFIX_RT)) && - !iter->fib6_nh->fib_nh_gw_family) { + (iter->nh || !iter->fib6_nh->fib_nh_gw_family)) { iter->fib6_flags &= ~RTF_ADDRCONF; iter->fib6_flags &= ~RTF_PREFIX_RT; } diff --git a/net/mctp/device.c b/net/mctp/device.c index 4d404edd7446e1..04c5570bacff69 100644 --- a/net/mctp/device.c +++ b/net/mctp/device.c @@ -70,6 +70,7 @@ static int mctp_fill_addrinfo(struct sk_buff *skb, return -EMSGSIZE; hdr = nlmsg_data(nlh); + memset(hdr, 0, sizeof(*hdr)); hdr->ifa_family = AF_MCTP; hdr->ifa_prefixlen = 0; hdr->ifa_flags = 0; diff --git a/net/mctp/neigh.c b/net/mctp/neigh.c index 05b899f22d902b..fc85f0e6930143 100644 --- a/net/mctp/neigh.c +++ b/net/mctp/neigh.c @@ -218,6 +218,7 @@ static int mctp_fill_neigh(struct sk_buff *skb, u32 portid, u32 seq, int event, return -EMSGSIZE; hdr = nlmsg_data(nlh); + memset(hdr, 0, sizeof(*hdr)); hdr->ndm_family = AF_MCTP; hdr->ndm_ifindex = dev->ifindex; hdr->ndm_state = 0; // TODO other state bits? diff --git a/net/mctp/route.c b/net/mctp/route.c index 2ac4011a953fff..ecbbe4beb2133b 100644 --- a/net/mctp/route.c +++ b/net/mctp/route.c @@ -1643,6 +1643,7 @@ static int mctp_fill_rtinfo(struct sk_buff *skb, struct mctp_route *rt, return -EMSGSIZE; hdr = nlmsg_data(nlh); + memset(hdr, 0, sizeof(*hdr)); hdr->rtm_family = AF_MCTP; /* we use the _len fields as a number of EIDs, rather than diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c index 8d323366741810..bad9fc0f27d9cd 100644 --- a/net/mptcp/protocol.c +++ b/net/mptcp/protocol.c @@ -224,9 +224,6 @@ static bool mptcp_rcvbuf_grow(struct sock *sk, u32 newval) do_div(grow, oldval); rcvwin += grow << 1; - if (!RB_EMPTY_ROOT(&msk->out_of_order_queue)) - rcvwin += MPTCP_SKB_CB(msk->ooo_last_skb)->end_seq - msk->ack_seq; - cap = READ_ONCE(net->ipv4.sysctl_tcp_rmem[2]); rcvbuf = min_t(u32, mptcp_space_from_win(sk, rcvwin), cap); @@ -350,9 +347,6 @@ static void mptcp_data_queue_ofo(struct mptcp_sock *msk, struct sk_buff *skb) end: skb_condense(skb); skb_set_owner_r(skb, sk); - /* do not grow rcvbuf for not-yet-accepted or orphaned sockets. */ - if (sk->sk_socket) - mptcp_rcvbuf_grow(sk, msk->rcvq_space.space); } static void mptcp_init_skb(struct sock *ssk, struct sk_buff *skb, int offset, @@ -2088,8 +2082,8 @@ static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied) msk->rcvq_space.copied += copied; - mstamp = div_u64(tcp_clock_ns(), NSEC_PER_USEC); - time = tcp_stamp_us_delta(mstamp, msk->rcvq_space.time); + mstamp = mptcp_stamp(); + time = tcp_stamp_us_delta(mstamp, READ_ONCE(msk->rcvq_space.time)); rtt_us = msk->rcvq_space.rtt_us; if (rtt_us && time < (rtt_us >> 3)) @@ -3549,6 +3543,7 @@ struct sock *mptcp_sk_clone_init(const struct sock *sk, __mptcp_propagate_sndbuf(nsk, ssk); mptcp_rcv_space_init(msk, ssk); + msk->rcvq_space.time = mptcp_stamp(); if (mp_opt->suboptions & OPTION_MPTCP_MPC_ACK) __mptcp_subflow_fully_established(msk, subflow, mp_opt); @@ -3566,8 +3561,6 @@ void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk) msk->rcvq_space.copied = 0; msk->rcvq_space.rtt_us = 0; - msk->rcvq_space.time = tp->tcp_mstamp; - /* initial rcv_space offering made to peer */ msk->rcvq_space.space = min_t(u32, tp->rcv_wnd, TCP_INIT_CWND * tp->advmss); @@ -3763,6 +3756,7 @@ void mptcp_finish_connect(struct sock *ssk) * accessing the field below */ WRITE_ONCE(msk->local_key, subflow->local_key); + WRITE_ONCE(msk->rcvq_space.time, mptcp_stamp()); mptcp_pm_new_connection(msk, ssk, 0); } diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h index 66e9735007912e..39afd44e072f2c 100644 --- a/net/mptcp/protocol.h +++ b/net/mptcp/protocol.h @@ -915,6 +915,11 @@ static inline bool mptcp_is_fully_established(struct sock *sk) READ_ONCE(mptcp_sk(sk)->fully_established); } +static inline u64 mptcp_stamp(void) +{ + return div_u64(tcp_clock_ns(), NSEC_PER_USEC); +} + void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk); void mptcp_data_ready(struct sock *sk, struct sock *ssk); bool mptcp_finish_join(struct sock *sk); diff --git a/net/netfilter/ipvs/ip_vs_proto_sctp.c b/net/netfilter/ipvs/ip_vs_proto_sctp.c index 83e452916403d5..63c78a1f3918a7 100644 --- a/net/netfilter/ipvs/ip_vs_proto_sctp.c +++ b/net/netfilter/ipvs/ip_vs_proto_sctp.c @@ -10,7 +10,8 @@ #include static int -sctp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp); +sctp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp, + unsigned int sctphoff); static int sctp_conn_schedule(struct netns_ipvs *ipvs, int af, struct sk_buff *skb, @@ -108,7 +109,7 @@ sctp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, int ret; /* Some checks before mangling */ - if (!sctp_csum_check(cp->af, skb, pp)) + if (!sctp_csum_check(cp->af, skb, pp, sctphoff)) return 0; /* Call application helper if needed */ @@ -156,7 +157,7 @@ sctp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, int ret; /* Some checks before mangling */ - if (!sctp_csum_check(cp->af, skb, pp)) + if (!sctp_csum_check(cp->af, skb, pp, sctphoff)) return 0; /* Call application helper if needed */ @@ -185,19 +186,12 @@ sctp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, } static int -sctp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp) +sctp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp, + unsigned int sctphoff) { - unsigned int sctphoff; struct sctphdr *sh; __le32 cmp, val; -#ifdef CONFIG_IP_VS_IPV6 - if (af == AF_INET6) - sctphoff = sizeof(struct ipv6hdr); - else -#endif - sctphoff = ip_hdrlen(skb); - sh = (struct sctphdr *)(skb->data + sctphoff); cmp = sh->checksum; val = sctp_compute_cksum(skb, sctphoff); diff --git a/net/netfilter/ipvs/ip_vs_proto_tcp.c b/net/netfilter/ipvs/ip_vs_proto_tcp.c index f68a1533ee455e..8cc0a8ce624112 100644 --- a/net/netfilter/ipvs/ip_vs_proto_tcp.c +++ b/net/netfilter/ipvs/ip_vs_proto_tcp.c @@ -28,7 +28,8 @@ #include static int -tcp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp); +tcp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp, + unsigned int tcphoff); static int tcp_conn_schedule(struct netns_ipvs *ipvs, int af, struct sk_buff *skb, @@ -165,7 +166,7 @@ tcp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, int ret; /* Some checks before mangling */ - if (!tcp_csum_check(cp->af, skb, pp)) + if (!tcp_csum_check(cp->af, skb, pp, tcphoff)) return 0; /* Call application helper if needed */ @@ -243,7 +244,7 @@ tcp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, int ret; /* Some checks before mangling */ - if (!tcp_csum_check(cp->af, skb, pp)) + if (!tcp_csum_check(cp->af, skb, pp, tcphoff)) return 0; /* @@ -300,17 +301,9 @@ tcp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, static int -tcp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp) +tcp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp, + unsigned int tcphoff) { - unsigned int tcphoff; - -#ifdef CONFIG_IP_VS_IPV6 - if (af == AF_INET6) - tcphoff = sizeof(struct ipv6hdr); - else -#endif - tcphoff = ip_hdrlen(skb); - switch (skb->ip_summed) { case CHECKSUM_NONE: skb->csum = skb_checksum(skb, tcphoff, skb->len - tcphoff, 0); @@ -321,7 +314,7 @@ tcp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp) if (csum_ipv6_magic(&ipv6_hdr(skb)->saddr, &ipv6_hdr(skb)->daddr, skb->len - tcphoff, - ipv6_hdr(skb)->nexthdr, + IPPROTO_TCP, skb->csum)) { IP_VS_DBG_RL_PKT(0, af, pp, skb, 0, "Failed checksum for"); diff --git a/net/netfilter/ipvs/ip_vs_proto_udp.c b/net/netfilter/ipvs/ip_vs_proto_udp.c index 0f0107c80dd235..f9de632e38cdd6 100644 --- a/net/netfilter/ipvs/ip_vs_proto_udp.c +++ b/net/netfilter/ipvs/ip_vs_proto_udp.c @@ -24,7 +24,8 @@ #include static int -udp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp); +udp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp, + unsigned int udphoff); static int udp_conn_schedule(struct netns_ipvs *ipvs, int af, struct sk_buff *skb, @@ -154,7 +155,7 @@ udp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, int ret; /* Some checks before mangling */ - if (!udp_csum_check(cp->af, skb, pp)) + if (!udp_csum_check(cp->af, skb, pp, udphoff)) return 0; /* @@ -237,7 +238,7 @@ udp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, int ret; /* Some checks before mangling */ - if (!udp_csum_check(cp->af, skb, pp)) + if (!udp_csum_check(cp->af, skb, pp, udphoff)) return 0; /* @@ -296,17 +297,10 @@ udp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, static int -udp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp) +udp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp, + unsigned int udphoff) { struct udphdr _udph, *uh; - unsigned int udphoff; - -#ifdef CONFIG_IP_VS_IPV6 - if (af == AF_INET6) - udphoff = sizeof(struct ipv6hdr); - else -#endif - udphoff = ip_hdrlen(skb); uh = skb_header_pointer(skb, udphoff, sizeof(_udph), &_udph); if (uh == NULL) @@ -324,7 +318,7 @@ udp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp) if (csum_ipv6_magic(&ipv6_hdr(skb)->saddr, &ipv6_hdr(skb)->daddr, skb->len - udphoff, - ipv6_hdr(skb)->nexthdr, + IPPROTO_UDP, skb->csum)) { IP_VS_DBG_RL_PKT(0, af, pp, skb, 0, "Failed checksum for"); diff --git a/net/netfilter/ipvs/ip_vs_xmit.c b/net/netfilter/ipvs/ip_vs_xmit.c index 64c697212578ae..124f779424b0ff 100644 --- a/net/netfilter/ipvs/ip_vs_xmit.c +++ b/net/netfilter/ipvs/ip_vs_xmit.c @@ -294,6 +294,12 @@ static inline bool decrement_ttl(struct netns_ipvs *ipvs, return true; } +/* rt has device that is down */ +static bool rt_dev_is_down(const struct net_device *dev) +{ + return dev && !netif_running(dev); +} + /* Get route to destination or remote server */ static int __ip_vs_get_out_rt(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, @@ -309,9 +315,11 @@ __ip_vs_get_out_rt(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, if (dest) { dest_dst = __ip_vs_dst_check(dest); - if (likely(dest_dst)) + if (likely(dest_dst)) { rt = dst_rtable(dest_dst->dst_cache); - else { + if (ret_saddr) + *ret_saddr = dest_dst->dst_saddr.ip; + } else { dest_dst = ip_vs_dest_dst_alloc(); spin_lock_bh(&dest->dst_lock); if (!dest_dst) { @@ -327,14 +335,22 @@ __ip_vs_get_out_rt(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, ip_vs_dest_dst_free(dest_dst); goto err_unreach; } - __ip_vs_dst_set(dest, dest_dst, &rt->dst, 0); + /* It is forbidden to attach dest->dest_dst if + * device is going down. + */ + if (!rt_dev_is_down(dst_dev_rcu(&rt->dst))) + __ip_vs_dst_set(dest, dest_dst, &rt->dst, 0); + else + noref = 0; spin_unlock_bh(&dest->dst_lock); IP_VS_DBG(10, "new dst %pI4, src %pI4, refcnt=%d\n", &dest->addr.ip, &dest_dst->dst_saddr.ip, rcuref_read(&rt->dst.__rcuref)); + if (ret_saddr) + *ret_saddr = dest_dst->dst_saddr.ip; + if (!noref) + ip_vs_dest_dst_free(dest_dst); } - if (ret_saddr) - *ret_saddr = dest_dst->dst_saddr.ip; } else { noref = 0; @@ -471,9 +487,11 @@ __ip_vs_get_out_rt_v6(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, if (dest) { dest_dst = __ip_vs_dst_check(dest); - if (likely(dest_dst)) + if (likely(dest_dst)) { rt = dst_rt6_info(dest_dst->dst_cache); - else { + if (ret_saddr) + *ret_saddr = dest_dst->dst_saddr.in6; + } else { u32 cookie; dest_dst = ip_vs_dest_dst_alloc(); @@ -494,14 +512,22 @@ __ip_vs_get_out_rt_v6(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, } rt = dst_rt6_info(dst); cookie = rt6_get_cookie(rt); - __ip_vs_dst_set(dest, dest_dst, &rt->dst, cookie); + /* It is forbidden to attach dest->dest_dst if + * device is going down. + */ + if (!rt_dev_is_down(dst_dev_rcu(&rt->dst))) + __ip_vs_dst_set(dest, dest_dst, &rt->dst, cookie); + else + noref = 0; spin_unlock_bh(&dest->dst_lock); IP_VS_DBG(10, "new dst %pI6, src %pI6, refcnt=%d\n", &dest->addr.in6, &dest_dst->dst_saddr.in6, rcuref_read(&rt->dst.__rcuref)); + if (ret_saddr) + *ret_saddr = dest_dst->dst_saddr.in6; + if (!noref) + ip_vs_dest_dst_free(dest_dst); } - if (ret_saddr) - *ret_saddr = dest_dst->dst_saddr.in6; } else { noref = 0; dst = __ip_vs_route_output_v6(net, daddr, ret_saddr, do_xfrm, diff --git a/net/netfilter/nf_conncount.c b/net/netfilter/nf_conncount.c index 8487808c87614f..14e62b3263cd94 100644 --- a/net/netfilter/nf_conncount.c +++ b/net/netfilter/nf_conncount.c @@ -34,8 +34,9 @@ #define CONNCOUNT_SLOTS 256U -#define CONNCOUNT_GC_MAX_NODES 8 -#define MAX_KEYLEN 5 +#define CONNCOUNT_GC_MAX_NODES 8 +#define CONNCOUNT_GC_MAX_COLLECT 64 +#define MAX_KEYLEN 5 /* we will save the tuples of all connections we care about */ struct nf_conncount_tuple { @@ -178,16 +179,28 @@ static int __nf_conncount_add(struct net *net, return -ENOENT; if (ct && nf_ct_is_confirmed(ct)) { - err = -EEXIST; - goto out_put; + /* local connections are confirmed in postrouting so confirmation + * might have happened before hitting connlimit + */ + if (skb->skb_iif != LOOPBACK_IFINDEX) { + err = -EEXIST; + goto out_put; + } + + /* this is likely a local connection, skip optimization to avoid + * adding duplicates from a 'packet train' + */ + goto check_connections; } - if ((u32)jiffies == list->last_gc) + if ((u32)jiffies == list->last_gc && + (list->count - list->last_gc_count) < CONNCOUNT_GC_MAX_COLLECT) goto add_new_node; +check_connections: /* check the saved connections */ list_for_each_entry_safe(conn, conn_n, &list->head, node) { - if (collect > CONNCOUNT_GC_MAX_NODES) + if (collect > CONNCOUNT_GC_MAX_COLLECT) break; found = find_or_evict(net, list, conn); @@ -230,6 +243,7 @@ static int __nf_conncount_add(struct net *net, nf_ct_put(found_ct); } list->last_gc = (u32)jiffies; + list->last_gc_count = list->count; add_new_node: if (WARN_ON_ONCE(list->count > INT_MAX)) { @@ -277,6 +291,7 @@ void nf_conncount_list_init(struct nf_conncount_list *list) spin_lock_init(&list->list_lock); INIT_LIST_HEAD(&list->head); list->count = 0; + list->last_gc_count = 0; list->last_gc = (u32)jiffies; } EXPORT_SYMBOL_GPL(nf_conncount_list_init); @@ -316,13 +331,14 @@ static bool __nf_conncount_gc_list(struct net *net, } nf_ct_put(found_ct); - if (collected > CONNCOUNT_GC_MAX_NODES) + if (collected > CONNCOUNT_GC_MAX_COLLECT) break; } if (!list->count) ret = true; list->last_gc = (u32)jiffies; + list->last_gc_count = list->count; return ret; } diff --git a/net/netfilter/nf_conntrack_h323_main.c b/net/netfilter/nf_conntrack_h323_main.c index 14f73872f64778..e35814d68ce30b 100644 --- a/net/netfilter/nf_conntrack_h323_main.c +++ b/net/netfilter/nf_conntrack_h323_main.c @@ -1186,13 +1186,13 @@ static struct nf_conntrack_expect *find_expect(struct nf_conn *ct, { struct net *net = nf_ct_net(ct); struct nf_conntrack_expect *exp; - struct nf_conntrack_tuple tuple; + struct nf_conntrack_tuple tuple = { + .src.l3num = nf_ct_l3num(ct), + .dst.protonum = IPPROTO_TCP, + .dst.u.tcp.port = port, + }; - memset(&tuple.src.u3, 0, sizeof(tuple.src.u3)); - tuple.src.u.tcp.port = 0; memcpy(&tuple.dst.u3, addr, sizeof(tuple.dst.u3)); - tuple.dst.u.tcp.port = port; - tuple.dst.protonum = IPPROTO_TCP; exp = __nf_ct_expect_find(net, nf_ct_zone(ct), &tuple); if (exp && exp->master == ct) diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c index be92750e2af3ad..3b9c559ab1232a 100644 --- a/net/netfilter/nf_tables_api.c +++ b/net/netfilter/nf_tables_api.c @@ -2822,6 +2822,7 @@ static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 policy, err_register_hook: nft_chain_del(chain); + synchronize_rcu(); err_chain_add: nft_trans_destroy(trans); err_trans: @@ -3900,23 +3901,6 @@ static int nf_tables_dump_rules(struct sk_buff *skb, return skb->len; } -static int nf_tables_dumpreset_rules(struct sk_buff *skb, - struct netlink_callback *cb) -{ - struct nftables_pernet *nft_net = nft_pernet(sock_net(skb->sk)); - int ret; - - /* Mutex is held is to prevent that two concurrent dump-and-reset calls - * do not underrun counters and quotas. The commit_mutex is used for - * the lack a better lock, this is not transaction path. - */ - mutex_lock(&nft_net->commit_mutex); - ret = nf_tables_dump_rules(skb, cb); - mutex_unlock(&nft_net->commit_mutex); - - return ret; -} - static int nf_tables_dump_rules_start(struct netlink_callback *cb) { struct nft_rule_dump_ctx *ctx = (void *)cb->ctx; @@ -3936,16 +3920,10 @@ static int nf_tables_dump_rules_start(struct netlink_callback *cb) return -ENOMEM; } } - return 0; -} - -static int nf_tables_dumpreset_rules_start(struct netlink_callback *cb) -{ - struct nft_rule_dump_ctx *ctx = (void *)cb->ctx; - - ctx->reset = true; + if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETRULE_RESET) + ctx->reset = true; - return nf_tables_dump_rules_start(cb); + return 0; } static int nf_tables_dump_rules_done(struct netlink_callback *cb) @@ -4011,6 +3989,8 @@ static int nf_tables_getrule(struct sk_buff *skb, const struct nfnl_info *info, u32 portid = NETLINK_CB(skb).portid; struct net *net = info->net; struct sk_buff *skb2; + bool reset = false; + char *buf; if (info->nlh->nlmsg_flags & NLM_F_DUMP) { struct netlink_dump_control c = { @@ -4024,47 +4004,16 @@ static int nf_tables_getrule(struct sk_buff *skb, const struct nfnl_info *info, return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c); } - skb2 = nf_tables_getrule_single(portid, info, nla, false); - if (IS_ERR(skb2)) - return PTR_ERR(skb2); - - return nfnetlink_unicast(skb2, net, portid); -} - -static int nf_tables_getrule_reset(struct sk_buff *skb, - const struct nfnl_info *info, - const struct nlattr * const nla[]) -{ - struct nftables_pernet *nft_net = nft_pernet(info->net); - u32 portid = NETLINK_CB(skb).portid; - struct net *net = info->net; - struct sk_buff *skb2; - char *buf; - - if (info->nlh->nlmsg_flags & NLM_F_DUMP) { - struct netlink_dump_control c = { - .start= nf_tables_dumpreset_rules_start, - .dump = nf_tables_dumpreset_rules, - .done = nf_tables_dump_rules_done, - .module = THIS_MODULE, - .data = (void *)nla, - }; - - return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c); - } - - if (!try_module_get(THIS_MODULE)) - return -EINVAL; - rcu_read_unlock(); - mutex_lock(&nft_net->commit_mutex); - skb2 = nf_tables_getrule_single(portid, info, nla, true); - mutex_unlock(&nft_net->commit_mutex); - rcu_read_lock(); - module_put(THIS_MODULE); + if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETRULE_RESET) + reset = true; + skb2 = nf_tables_getrule_single(portid, info, nla, reset); if (IS_ERR(skb2)) return PTR_ERR(skb2); + if (!reset) + return nfnetlink_unicast(skb2, net, portid); + buf = kasprintf(GFP_ATOMIC, "%.*s:%u", nla_len(nla[NFTA_RULE_TABLE]), (char *)nla_data(nla[NFTA_RULE_TABLE]), @@ -6323,6 +6272,10 @@ static int nf_tables_dump_set(struct sk_buff *skb, struct netlink_callback *cb) nla_nest_end(skb, nest); nlmsg_end(skb, nlh); + if (dump_ctx->reset && args.iter.count > args.iter.skip) + audit_log_nft_set_reset(table, cb->seq, + args.iter.count - args.iter.skip); + rcu_read_unlock(); if (args.iter.err && args.iter.err != -EMSGSIZE) @@ -6338,26 +6291,6 @@ static int nf_tables_dump_set(struct sk_buff *skb, struct netlink_callback *cb) return -ENOSPC; } -static int nf_tables_dumpreset_set(struct sk_buff *skb, - struct netlink_callback *cb) -{ - struct nftables_pernet *nft_net = nft_pernet(sock_net(skb->sk)); - struct nft_set_dump_ctx *dump_ctx = cb->data; - int ret, skip = cb->args[0]; - - mutex_lock(&nft_net->commit_mutex); - - ret = nf_tables_dump_set(skb, cb); - - if (cb->args[0] > skip) - audit_log_nft_set_reset(dump_ctx->ctx.table, cb->seq, - cb->args[0] - skip); - - mutex_unlock(&nft_net->commit_mutex); - - return ret; -} - static int nf_tables_dump_set_start(struct netlink_callback *cb) { struct nft_set_dump_ctx *dump_ctx = cb->data; @@ -6601,8 +6534,13 @@ static int nf_tables_getsetelem(struct sk_buff *skb, { struct netlink_ext_ack *extack = info->extack; struct nft_set_dump_ctx dump_ctx; + int rem, err = 0, nelems = 0; + struct net *net = info->net; struct nlattr *attr; - int rem, err = 0; + bool reset = false; + + if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETSETELEM_RESET) + reset = true; if (info->nlh->nlmsg_flags & NLM_F_DUMP) { struct netlink_dump_control c = { @@ -6612,7 +6550,7 @@ static int nf_tables_getsetelem(struct sk_buff *skb, .module = THIS_MODULE, }; - err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, false); + err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, reset); if (err) return err; @@ -6623,75 +6561,21 @@ static int nf_tables_getsetelem(struct sk_buff *skb, if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS]) return -EINVAL; - err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, false); + err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, reset); if (err) return err; nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) { - err = nft_get_set_elem(&dump_ctx.ctx, dump_ctx.set, attr, false); - if (err < 0) { - NL_SET_BAD_ATTR(extack, attr); - break; - } - } - - return err; -} - -static int nf_tables_getsetelem_reset(struct sk_buff *skb, - const struct nfnl_info *info, - const struct nlattr * const nla[]) -{ - struct nftables_pernet *nft_net = nft_pernet(info->net); - struct netlink_ext_ack *extack = info->extack; - struct nft_set_dump_ctx dump_ctx; - int rem, err = 0, nelems = 0; - struct nlattr *attr; - - if (info->nlh->nlmsg_flags & NLM_F_DUMP) { - struct netlink_dump_control c = { - .start = nf_tables_dump_set_start, - .dump = nf_tables_dumpreset_set, - .done = nf_tables_dump_set_done, - .module = THIS_MODULE, - }; - - err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, true); - if (err) - return err; - - c.data = &dump_ctx; - return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c); - } - - if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS]) - return -EINVAL; - - if (!try_module_get(THIS_MODULE)) - return -EINVAL; - rcu_read_unlock(); - mutex_lock(&nft_net->commit_mutex); - rcu_read_lock(); - - err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, true); - if (err) - goto out_unlock; - - nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) { - err = nft_get_set_elem(&dump_ctx.ctx, dump_ctx.set, attr, true); + err = nft_get_set_elem(&dump_ctx.ctx, dump_ctx.set, attr, reset); if (err < 0) { NL_SET_BAD_ATTR(extack, attr); break; } nelems++; } - audit_log_nft_set_reset(dump_ctx.ctx.table, nft_base_seq(info->net), nelems); - -out_unlock: - rcu_read_unlock(); - mutex_unlock(&nft_net->commit_mutex); - rcu_read_lock(); - module_put(THIS_MODULE); + if (reset) + audit_log_nft_set_reset(dump_ctx.ctx.table, nft_base_seq(net), + nelems); return err; } @@ -7269,7 +7153,8 @@ static u32 nft_set_maxsize(const struct nft_set *set) } static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, - const struct nlattr *attr, u32 nlmsg_flags) + const struct nlattr *attr, u32 nlmsg_flags, + bool last) { struct nft_expr *expr_array[NFT_SET_EXPR_MAX] = {}; struct nlattr *nla[NFTA_SET_ELEM_MAX + 1]; @@ -7555,6 +7440,11 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, if (flags) *nft_set_ext_flags(ext) = flags; + if (last) + elem.flags = NFT_SET_ELEM_INTERNAL_LAST; + else + elem.flags = 0; + if (obj) *nft_set_ext_obj(ext) = obj; @@ -7635,6 +7525,11 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, * and an existing one. */ err = -EEXIST; + } else if (err == -ECANCELED) { + /* ECANCELED reports an existing nul-element in + * interval sets. + */ + err = 0; } goto err_element_clash; } @@ -7713,7 +7608,8 @@ static int nf_tables_newsetelem(struct sk_buff *skb, nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla); nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) { - err = nft_add_set_elem(&ctx, set, attr, info->nlh->nlmsg_flags); + err = nft_add_set_elem(&ctx, set, attr, info->nlh->nlmsg_flags, + nla_is_last(attr, rem)); if (err < 0) { NL_SET_BAD_ATTR(extack, attr); return err; @@ -7806,7 +7702,8 @@ static bool nft_trans_elems_new_abort(const struct nft_ctx *ctx, continue; } - if (!te->set->ops->abort || nft_setelem_is_catchall(te->set, te->elems[i].priv)) + if (!te->set->ops->abort_skip_removal || + nft_setelem_is_catchall(te->set, te->elems[i].priv)) nft_setelem_remove(ctx->net, te->set, te->elems[i].priv); if (!nft_setelem_is_catchall(te->set, te->elems[i].priv)) @@ -7836,7 +7733,7 @@ static void nft_trans_elems_destroy_abort(const struct nft_ctx *ctx, } static int nft_del_setelem(struct nft_ctx *ctx, struct nft_set *set, - const struct nlattr *attr) + const struct nlattr *attr, bool last) { struct nlattr *nla[NFTA_SET_ELEM_MAX + 1]; struct nft_set_ext_tmpl tmpl; @@ -7904,6 +7801,11 @@ static int nft_del_setelem(struct nft_ctx *ctx, struct nft_set *set, if (flags) *nft_set_ext_flags(ext) = flags; + if (last) + elem.flags = NFT_SET_ELEM_INTERNAL_LAST; + else + elem.flags = 0; + trans = nft_trans_elem_alloc(ctx, NFT_MSG_DELSETELEM, set); if (trans == NULL) goto fail_trans; @@ -8051,7 +7953,8 @@ static int nf_tables_delsetelem(struct sk_buff *skb, return nft_set_flush(&ctx, set, genmask); nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) { - err = nft_del_setelem(&ctx, set, attr); + err = nft_del_setelem(&ctx, set, attr, + nla_is_last(attr, rem)); if (err == -ENOENT && NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYSETELEM) continue; @@ -8544,19 +8447,6 @@ static int nf_tables_dump_obj(struct sk_buff *skb, struct netlink_callback *cb) return skb->len; } -static int nf_tables_dumpreset_obj(struct sk_buff *skb, - struct netlink_callback *cb) -{ - struct nftables_pernet *nft_net = nft_pernet(sock_net(skb->sk)); - int ret; - - mutex_lock(&nft_net->commit_mutex); - ret = nf_tables_dump_obj(skb, cb); - mutex_unlock(&nft_net->commit_mutex); - - return ret; -} - static int nf_tables_dump_obj_start(struct netlink_callback *cb) { struct nft_obj_dump_ctx *ctx = (void *)cb->ctx; @@ -8573,16 +8463,10 @@ static int nf_tables_dump_obj_start(struct netlink_callback *cb) if (nla[NFTA_OBJ_TYPE]) ctx->type = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE])); - return 0; -} - -static int nf_tables_dumpreset_obj_start(struct netlink_callback *cb) -{ - struct nft_obj_dump_ctx *ctx = (void *)cb->ctx; - - ctx->reset = true; + if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET) + ctx->reset = true; - return nf_tables_dump_obj_start(cb); + return 0; } static int nf_tables_dump_obj_done(struct netlink_callback *cb) @@ -8644,42 +8528,16 @@ nf_tables_getobj_single(u32 portid, const struct nfnl_info *info, static int nf_tables_getobj(struct sk_buff *skb, const struct nfnl_info *info, const struct nlattr * const nla[]) { - u32 portid = NETLINK_CB(skb).portid; - struct sk_buff *skb2; - - if (info->nlh->nlmsg_flags & NLM_F_DUMP) { - struct netlink_dump_control c = { - .start = nf_tables_dump_obj_start, - .dump = nf_tables_dump_obj, - .done = nf_tables_dump_obj_done, - .module = THIS_MODULE, - .data = (void *)nla, - }; - - return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c); - } - - skb2 = nf_tables_getobj_single(portid, info, nla, false); - if (IS_ERR(skb2)) - return PTR_ERR(skb2); - - return nfnetlink_unicast(skb2, info->net, portid); -} - -static int nf_tables_getobj_reset(struct sk_buff *skb, - const struct nfnl_info *info, - const struct nlattr * const nla[]) -{ - struct nftables_pernet *nft_net = nft_pernet(info->net); u32 portid = NETLINK_CB(skb).portid; struct net *net = info->net; struct sk_buff *skb2; + bool reset = false; char *buf; if (info->nlh->nlmsg_flags & NLM_F_DUMP) { struct netlink_dump_control c = { - .start = nf_tables_dumpreset_obj_start, - .dump = nf_tables_dumpreset_obj, + .start = nf_tables_dump_obj_start, + .dump = nf_tables_dump_obj, .done = nf_tables_dump_obj_done, .module = THIS_MODULE, .data = (void *)nla, @@ -8688,18 +8546,16 @@ static int nf_tables_getobj_reset(struct sk_buff *skb, return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c); } - if (!try_module_get(THIS_MODULE)) - return -EINVAL; - rcu_read_unlock(); - mutex_lock(&nft_net->commit_mutex); - skb2 = nf_tables_getobj_single(portid, info, nla, true); - mutex_unlock(&nft_net->commit_mutex); - rcu_read_lock(); - module_put(THIS_MODULE); + if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET) + reset = true; + skb2 = nf_tables_getobj_single(portid, info, nla, reset); if (IS_ERR(skb2)) return PTR_ERR(skb2); + if (!reset) + return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid); + buf = kasprintf(GFP_ATOMIC, "%.*s:%u", nla_len(nla[NFTA_OBJ_TABLE]), (char *)nla_data(nla[NFTA_OBJ_TABLE]), @@ -10017,7 +9873,7 @@ static const struct nfnl_callback nf_tables_cb[NFT_MSG_MAX] = { .policy = nft_rule_policy, }, [NFT_MSG_GETRULE_RESET] = { - .call = nf_tables_getrule_reset, + .call = nf_tables_getrule, .type = NFNL_CB_RCU, .attr_count = NFTA_RULE_MAX, .policy = nft_rule_policy, @@ -10071,7 +9927,7 @@ static const struct nfnl_callback nf_tables_cb[NFT_MSG_MAX] = { .policy = nft_set_elem_list_policy, }, [NFT_MSG_GETSETELEM_RESET] = { - .call = nf_tables_getsetelem_reset, + .call = nf_tables_getsetelem, .type = NFNL_CB_RCU, .attr_count = NFTA_SET_ELEM_LIST_MAX, .policy = nft_set_elem_list_policy, @@ -10117,7 +9973,7 @@ static const struct nfnl_callback nf_tables_cb[NFT_MSG_MAX] = { .policy = nft_obj_policy, }, [NFT_MSG_GETOBJ_RESET] = { - .call = nf_tables_getobj_reset, + .call = nf_tables_getobj, .type = NFNL_CB_RCU, .attr_count = NFTA_OBJ_MAX, .policy = nft_obj_policy, @@ -11536,6 +11392,13 @@ static int nf_tables_abort(struct net *net, struct sk_buff *skb, ret = __nf_tables_abort(net, action); nft_gc_seq_end(nft_net, gc_seq); + if (action == NFNL_ABORT_NONE) { + struct nft_table *table; + + list_for_each_entry(table, &nft_net->tables, list) + table->validate_state = NFT_VALIDATE_SKIP; + } + WARN_ON_ONCE(!list_empty(&nft_net->commit_list)); /* module autoload needs to happen after GC sequence update because it diff --git a/net/netfilter/nfnetlink_queue.c b/net/netfilter/nfnetlink_queue.c index 8b7b39d8a10913..34548213f2f14f 100644 --- a/net/netfilter/nfnetlink_queue.c +++ b/net/netfilter/nfnetlink_queue.c @@ -30,6 +30,8 @@ #include #include #include +#include +#include #include #include #include @@ -47,6 +49,8 @@ #endif #define NFQNL_QMAX_DEFAULT 1024 +#define NFQNL_HASH_MIN 1024 +#define NFQNL_HASH_MAX 1048576 /* We're using struct nlattr which has 16bit nla_len. Note that nla_len * includes the header length. Thus, the maximum packet length that we @@ -56,6 +60,26 @@ */ #define NFQNL_MAX_COPY_RANGE (0xffff - NLA_HDRLEN) +/* Composite key for packet lookup: (net, queue_num, packet_id) */ +struct nfqnl_packet_key { + possible_net_t net; + u32 packet_id; + u16 queue_num; +} __aligned(sizeof(u32)); /* jhash2 requires 32-bit alignment */ + +/* Global rhashtable - one for entire system, all netns */ +static struct rhashtable nfqnl_packet_map __read_mostly; + +/* Helper to initialize composite key */ +static inline void nfqnl_init_key(struct nfqnl_packet_key *key, + struct net *net, u32 packet_id, u16 queue_num) +{ + memset(key, 0, sizeof(*key)); + write_pnet(&key->net, net); + key->packet_id = packet_id; + key->queue_num = queue_num; +} + struct nfqnl_instance { struct hlist_node hlist; /* global list of queues */ struct rcu_head rcu; @@ -100,6 +124,39 @@ static inline u_int8_t instance_hashfn(u_int16_t queue_num) return ((queue_num >> 8) ^ queue_num) % INSTANCE_BUCKETS; } +/* Extract composite key from nf_queue_entry for hashing */ +static u32 nfqnl_packet_obj_hashfn(const void *data, u32 len, u32 seed) +{ + const struct nf_queue_entry *entry = data; + struct nfqnl_packet_key key; + + nfqnl_init_key(&key, entry->state.net, entry->id, entry->queue_num); + + return jhash2((u32 *)&key, sizeof(key) / sizeof(u32), seed); +} + +/* Compare stack-allocated key against entry */ +static int nfqnl_packet_obj_cmpfn(struct rhashtable_compare_arg *arg, + const void *obj) +{ + const struct nfqnl_packet_key *key = arg->key; + const struct nf_queue_entry *entry = obj; + + return !net_eq(entry->state.net, read_pnet(&key->net)) || + entry->queue_num != key->queue_num || + entry->id != key->packet_id; +} + +static const struct rhashtable_params nfqnl_rhashtable_params = { + .head_offset = offsetof(struct nf_queue_entry, hash_node), + .key_len = sizeof(struct nfqnl_packet_key), + .obj_hashfn = nfqnl_packet_obj_hashfn, + .obj_cmpfn = nfqnl_packet_obj_cmpfn, + .automatic_shrinking = true, + .min_size = NFQNL_HASH_MIN, + .max_size = NFQNL_HASH_MAX, +}; + static struct nfqnl_instance * instance_lookup(struct nfnl_queue_net *q, u_int16_t queue_num) { @@ -191,33 +248,45 @@ instance_destroy(struct nfnl_queue_net *q, struct nfqnl_instance *inst) spin_unlock(&q->instances_lock); } -static inline void +static int __enqueue_entry(struct nfqnl_instance *queue, struct nf_queue_entry *entry) { - list_add_tail(&entry->list, &queue->queue_list); - queue->queue_total++; + int err; + + entry->queue_num = queue->queue_num; + + err = rhashtable_insert_fast(&nfqnl_packet_map, &entry->hash_node, + nfqnl_rhashtable_params); + if (unlikely(err)) + return err; + + list_add_tail(&entry->list, &queue->queue_list); + queue->queue_total++; + + return 0; } static void __dequeue_entry(struct nfqnl_instance *queue, struct nf_queue_entry *entry) { + rhashtable_remove_fast(&nfqnl_packet_map, &entry->hash_node, + nfqnl_rhashtable_params); list_del(&entry->list); queue->queue_total--; } static struct nf_queue_entry * -find_dequeue_entry(struct nfqnl_instance *queue, unsigned int id) +find_dequeue_entry(struct nfqnl_instance *queue, unsigned int id, + struct net *net) { - struct nf_queue_entry *entry = NULL, *i; + struct nfqnl_packet_key key; + struct nf_queue_entry *entry; - spin_lock_bh(&queue->lock); + nfqnl_init_key(&key, net, id, queue->queue_num); - list_for_each_entry(i, &queue->queue_list, list) { - if (i->id == id) { - entry = i; - break; - } - } + spin_lock_bh(&queue->lock); + entry = rhashtable_lookup_fast(&nfqnl_packet_map, &key, + nfqnl_rhashtable_params); if (entry) __dequeue_entry(queue, entry); @@ -369,6 +438,34 @@ static void nf_reinject(struct nf_queue_entry *entry, unsigned int verdict) nf_queue_entry_free(entry); } +/* return true if the entry has an unconfirmed conntrack attached that isn't owned by us + * exclusively. + */ +static bool nf_ct_drop_unconfirmed(const struct nf_queue_entry *entry, bool *is_unconfirmed) +{ +#if IS_ENABLED(CONFIG_NF_CONNTRACK) + struct nf_conn *ct = (void *)skb_nfct(entry->skb); + + if (!ct || nf_ct_is_confirmed(ct)) + return false; + + if (is_unconfirmed) + *is_unconfirmed = true; + + /* in some cases skb_clone() can occur after initial conntrack + * pickup, but conntrack assumes exclusive skb->_nfct ownership for + * unconfirmed entries. + * + * This happens for br_netfilter and with ip multicast routing. + * This can't be solved with serialization here because one clone + * could have been queued for local delivery or could be transmitted + * in parallel on another CPU. + */ + return refcount_read(&ct->ct_general.use) > 1; +#endif + return false; +} + static void nfqnl_reinject(struct nf_queue_entry *entry, unsigned int verdict) { const struct nf_ct_hook *ct_hook; @@ -396,6 +493,24 @@ static void nfqnl_reinject(struct nf_queue_entry *entry, unsigned int verdict) break; } } + + if (verdict != NF_DROP && entry->nf_ct_is_unconfirmed) { + /* If first queued segment was already reinjected then + * there is a good chance the ct entry is now confirmed. + * + * Handle the rare cases: + * - out-of-order verdict + * - threaded userspace reinjecting in parallel + * - first segment was dropped + * + * In all of those cases we can't handle this packet + * because we can't be sure that another CPU won't modify + * nf_conn->ext in parallel which isn't allowed. + */ + if (nf_ct_drop_unconfirmed(entry, NULL)) + verdict = NF_DROP; + } + nf_reinject(entry, verdict); } @@ -407,8 +522,7 @@ nfqnl_flush(struct nfqnl_instance *queue, nfqnl_cmpfn cmpfn, unsigned long data) spin_lock_bh(&queue->lock); list_for_each_entry_safe(entry, next, &queue->queue_list, list) { if (!cmpfn || cmpfn(entry, data)) { - list_del(&entry->list); - queue->queue_total--; + __dequeue_entry(queue, entry); nfqnl_reinject(entry, NF_DROP); } } @@ -826,49 +940,6 @@ nfqnl_build_packet_message(struct net *net, struct nfqnl_instance *queue, return NULL; } -static bool nf_ct_drop_unconfirmed(const struct nf_queue_entry *entry) -{ -#if IS_ENABLED(CONFIG_NF_CONNTRACK) - static const unsigned long flags = IPS_CONFIRMED | IPS_DYING; - struct nf_conn *ct = (void *)skb_nfct(entry->skb); - unsigned long status; - unsigned int use; - - if (!ct) - return false; - - status = READ_ONCE(ct->status); - if ((status & flags) == IPS_DYING) - return true; - - if (status & IPS_CONFIRMED) - return false; - - /* in some cases skb_clone() can occur after initial conntrack - * pickup, but conntrack assumes exclusive skb->_nfct ownership for - * unconfirmed entries. - * - * This happens for br_netfilter and with ip multicast routing. - * We can't be solved with serialization here because one clone could - * have been queued for local delivery. - */ - use = refcount_read(&ct->ct_general.use); - if (likely(use == 1)) - return false; - - /* Can't decrement further? Exclusive ownership. */ - if (!refcount_dec_not_one(&ct->ct_general.use)) - return false; - - skb_set_nfct(entry->skb, 0); - /* No nf_ct_put(): we already decremented .use and it cannot - * drop down to 0. - */ - return true; -#endif - return false; -} - static int __nfqnl_enqueue_packet(struct net *net, struct nfqnl_instance *queue, struct nf_queue_entry *entry) @@ -885,26 +956,23 @@ __nfqnl_enqueue_packet(struct net *net, struct nfqnl_instance *queue, } spin_lock_bh(&queue->lock); - if (nf_ct_drop_unconfirmed(entry)) - goto err_out_free_nskb; + if (queue->queue_total >= queue->queue_maxlen) + goto err_out_queue_drop; - if (queue->queue_total >= queue->queue_maxlen) { - if (queue->flags & NFQA_CFG_F_FAIL_OPEN) { - failopen = 1; - err = 0; - } else { - queue->queue_dropped++; - net_warn_ratelimited("nf_queue: full at %d entries, dropping packets(s)\n", - queue->queue_total); - } - goto err_out_free_nskb; - } entry->id = ++queue->id_sequence; *packet_id_ptr = htonl(entry->id); + /* Insert into hash BEFORE unicast. If failure don't send to userspace. */ + err = __enqueue_entry(queue, entry); + if (unlikely(err)) + goto err_out_queue_drop; + /* nfnetlink_unicast will either free the nskb or add it to a socket */ err = nfnetlink_unicast(nskb, net, queue->peer_portid); if (err < 0) { + /* Unicast failed - remove entry we just inserted */ + __dequeue_entry(queue, entry); + if (queue->flags & NFQA_CFG_F_FAIL_OPEN) { failopen = 1; err = 0; @@ -914,12 +982,22 @@ __nfqnl_enqueue_packet(struct net *net, struct nfqnl_instance *queue, goto err_out_unlock; } - __enqueue_entry(queue, entry); - spin_unlock_bh(&queue->lock); return 0; -err_out_free_nskb: +err_out_queue_drop: + if (queue->flags & NFQA_CFG_F_FAIL_OPEN) { + failopen = 1; + err = 0; + } else { + queue->queue_dropped++; + + if (queue->queue_total >= queue->queue_maxlen) + net_warn_ratelimited("nf_queue: full at %d entries, dropping packets(s)\n", + queue->queue_total); + else + net_warn_ratelimited("nf_queue: hash insert failed: %d\n", err); + } kfree_skb(nskb); err_out_unlock: spin_unlock_bh(&queue->lock); @@ -998,9 +1076,10 @@ __nfqnl_enqueue_packet_gso(struct net *net, struct nfqnl_instance *queue, static int nfqnl_enqueue_packet(struct nf_queue_entry *entry, unsigned int queuenum) { - unsigned int queued; - struct nfqnl_instance *queue; struct sk_buff *skb, *segs, *nskb; + bool ct_is_unconfirmed = false; + struct nfqnl_instance *queue; + unsigned int queued; int err = -ENOBUFS; struct net *net = entry->state.net; struct nfnl_queue_net *q = nfnl_queue_pernet(net); @@ -1024,6 +1103,15 @@ nfqnl_enqueue_packet(struct nf_queue_entry *entry, unsigned int queuenum) break; } + /* Check if someone already holds another reference to + * unconfirmed ct. If so, we cannot queue the skb: + * concurrent modifications of nf_conn->ext are not + * allowed and we can't know if another CPU isn't + * processing the same nf_conn entry in parallel. + */ + if (nf_ct_drop_unconfirmed(entry, &ct_is_unconfirmed)) + return -EINVAL; + if (!skb_is_gso(skb) || ((queue->flags & NFQA_CFG_F_GSO) && !skb_is_gso_sctp(skb))) return __nfqnl_enqueue_packet(net, queue, entry); @@ -1037,7 +1125,23 @@ nfqnl_enqueue_packet(struct nf_queue_entry *entry, unsigned int queuenum) goto out_err; queued = 0; err = 0; + skb_list_walk_safe(segs, segs, nskb) { + if (ct_is_unconfirmed && queued > 0) { + /* skb_gso_segment() increments the ct refcount. + * This is a problem for unconfirmed (not in hash) + * entries, those can race when reinjections happen + * in parallel. + * + * Annotate this for all queued entries except the + * first one. + * + * As long as the first one is reinjected first it + * will do the confirmation for us. + */ + entry->nf_ct_is_unconfirmed = ct_is_unconfirmed; + } + if (err == 0) err = __nfqnl_enqueue_packet_gso(net, queue, segs, entry); @@ -1430,7 +1534,7 @@ static int nfqnl_recv_verdict(struct sk_buff *skb, const struct nfnl_info *info, verdict = ntohl(vhdr->verdict); - entry = find_dequeue_entry(queue, ntohl(vhdr->id)); + entry = find_dequeue_entry(queue, ntohl(vhdr->id), info->net); if (entry == NULL) return -ENOENT; @@ -1781,10 +1885,14 @@ static int __init nfnetlink_queue_init(void) { int status; + status = rhashtable_init(&nfqnl_packet_map, &nfqnl_rhashtable_params); + if (status < 0) + return status; + status = register_pernet_subsys(&nfnl_queue_net_ops); if (status < 0) { pr_err("failed to register pernet ops\n"); - goto out; + goto cleanup_rhashtable; } netlink_register_notifier(&nfqnl_rtnl_notifier); @@ -1809,7 +1917,8 @@ static int __init nfnetlink_queue_init(void) cleanup_netlink_notifier: netlink_unregister_notifier(&nfqnl_rtnl_notifier); unregister_pernet_subsys(&nfnl_queue_net_ops); -out: +cleanup_rhashtable: + rhashtable_destroy(&nfqnl_packet_map); return status; } @@ -1821,6 +1930,8 @@ static void __exit nfnetlink_queue_fini(void) netlink_unregister_notifier(&nfqnl_rtnl_notifier); unregister_pernet_subsys(&nfnl_queue_net_ops); + rhashtable_destroy(&nfqnl_packet_map); + rcu_barrier(); /* Wait for completion of call_rcu()'s */ } diff --git a/net/netfilter/nft_compat.c b/net/netfilter/nft_compat.c index 72711d62fddfa4..08f620311b03f1 100644 --- a/net/netfilter/nft_compat.c +++ b/net/netfilter/nft_compat.c @@ -134,7 +134,8 @@ static void nft_target_eval_bridge(const struct nft_expr *expr, } static const struct nla_policy nft_target_policy[NFTA_TARGET_MAX + 1] = { - [NFTA_TARGET_NAME] = { .type = NLA_NUL_STRING }, + [NFTA_TARGET_NAME] = { .type = NLA_NUL_STRING, + .len = XT_EXTENSION_MAXNAMELEN, }, [NFTA_TARGET_REV] = NLA_POLICY_MAX(NLA_BE32, 255), [NFTA_TARGET_INFO] = { .type = NLA_BINARY }, }; @@ -434,7 +435,8 @@ static void nft_match_eval(const struct nft_expr *expr, } static const struct nla_policy nft_match_policy[NFTA_MATCH_MAX + 1] = { - [NFTA_MATCH_NAME] = { .type = NLA_NUL_STRING }, + [NFTA_MATCH_NAME] = { .type = NLA_NUL_STRING, + .len = XT_EXTENSION_MAXNAMELEN }, [NFTA_MATCH_REV] = NLA_POLICY_MAX(NLA_BE32, 255), [NFTA_MATCH_INFO] = { .type = NLA_BINARY }, }; @@ -693,7 +695,12 @@ static int nfnl_compat_get_rcu(struct sk_buff *skb, name = nla_data(tb[NFTA_COMPAT_NAME]); rev = ntohl(nla_get_be32(tb[NFTA_COMPAT_REV])); - target = ntohl(nla_get_be32(tb[NFTA_COMPAT_TYPE])); + /* x_tables api checks for 'target == 1' to mean target, + * everything else means 'match'. + * In x_tables world, the number is set by kernel, not + * userspace. + */ + target = nla_get_be32(tb[NFTA_COMPAT_TYPE]) == htonl(1); switch(family) { case AF_INET: diff --git a/net/netfilter/nft_counter.c b/net/netfilter/nft_counter.c index cc732532949630..169ae93688bcc5 100644 --- a/net/netfilter/nft_counter.c +++ b/net/netfilter/nft_counter.c @@ -32,6 +32,9 @@ struct nft_counter_percpu_priv { static DEFINE_PER_CPU(struct u64_stats_sync, nft_counter_sync); +/* control plane only: sync fetch+reset */ +static DEFINE_SPINLOCK(nft_counter_lock); + static inline void nft_counter_do_eval(struct nft_counter_percpu_priv *priv, struct nft_regs *regs, const struct nft_pktinfo *pkt) @@ -117,8 +120,8 @@ static void nft_counter_reset(struct nft_counter_percpu_priv *priv, nft_sync = this_cpu_ptr(&nft_counter_sync); u64_stats_update_begin(nft_sync); - u64_stats_add(&this_cpu->packets, -total->packets); - u64_stats_add(&this_cpu->bytes, -total->bytes); + u64_stats_sub(&this_cpu->packets, total->packets); + u64_stats_sub(&this_cpu->bytes, total->bytes); u64_stats_update_end(nft_sync); local_bh_enable(); @@ -148,13 +151,25 @@ static void nft_counter_fetch(struct nft_counter_percpu_priv *priv, } } +static void nft_counter_fetch_and_reset(struct nft_counter_percpu_priv *priv, + struct nft_counter_tot *total) +{ + spin_lock(&nft_counter_lock); + nft_counter_fetch(priv, total); + nft_counter_reset(priv, total); + spin_unlock(&nft_counter_lock); +} + static int nft_counter_do_dump(struct sk_buff *skb, struct nft_counter_percpu_priv *priv, bool reset) { struct nft_counter_tot total; - nft_counter_fetch(priv, &total); + if (unlikely(reset)) + nft_counter_fetch_and_reset(priv, &total); + else + nft_counter_fetch(priv, &total); if (nla_put_be64(skb, NFTA_COUNTER_BYTES, cpu_to_be64(total.bytes), NFTA_COUNTER_PAD) || @@ -162,9 +177,6 @@ static int nft_counter_do_dump(struct sk_buff *skb, NFTA_COUNTER_PAD)) goto nla_put_failure; - if (reset) - nft_counter_reset(priv, &total); - return 0; nla_put_failure: diff --git a/net/netfilter/nft_quota.c b/net/netfilter/nft_quota.c index df0798da2329b9..cb6c0e04ff6755 100644 --- a/net/netfilter/nft_quota.c +++ b/net/netfilter/nft_quota.c @@ -140,11 +140,16 @@ static int nft_quota_do_dump(struct sk_buff *skb, struct nft_quota *priv, u64 consumed, consumed_cap, quota; u32 flags = priv->flags; - /* Since we inconditionally increment consumed quota for each packet + /* Since we unconditionally increment consumed quota for each packet * that we see, don't go over the quota boundary in what we send to * userspace. */ - consumed = atomic64_read(priv->consumed); + if (reset) { + consumed = atomic64_xchg(priv->consumed, 0); + clear_bit(NFT_QUOTA_DEPLETED_BIT, &priv->flags); + } else { + consumed = atomic64_read(priv->consumed); + } quota = atomic64_read(&priv->quota); if (consumed >= quota) { consumed_cap = quota; @@ -160,10 +165,6 @@ static int nft_quota_do_dump(struct sk_buff *skb, struct nft_quota *priv, nla_put_be32(skb, NFTA_QUOTA_FLAGS, htonl(flags))) goto nla_put_failure; - if (reset) { - atomic64_sub(consumed, priv->consumed); - clear_bit(NFT_QUOTA_DEPLETED_BIT, &priv->flags); - } return 0; nla_put_failure: diff --git a/net/netfilter/nft_set_hash.c b/net/netfilter/nft_set_hash.c index ba01ce75d6dea9..739b992bde591e 100644 --- a/net/netfilter/nft_set_hash.c +++ b/net/netfilter/nft_set_hash.c @@ -619,15 +619,20 @@ static struct nft_elem_priv * nft_hash_get(const struct net *net, const struct nft_set *set, const struct nft_set_elem *elem, unsigned int flags) { + const u32 *key = (const u32 *)&elem->key.val; struct nft_hash *priv = nft_set_priv(set); u8 genmask = nft_genmask_cur(net); struct nft_hash_elem *he; u32 hash; - hash = jhash(elem->key.val.data, set->klen, priv->seed); + if (set->klen == 4) + hash = jhash_1word(*key, priv->seed); + else + hash = jhash(key, set->klen, priv->seed); + hash = reciprocal_scale(hash, priv->buckets); hlist_for_each_entry_rcu(he, &priv->table[hash], node) { - if (!memcmp(nft_set_ext_key(&he->ext), elem->key.val.data, set->klen) && + if (!memcmp(nft_set_ext_key(&he->ext), key, set->klen) && nft_set_elem_active(&he->ext, genmask)) return &he->priv; } diff --git a/net/netfilter/nft_set_pipapo.c b/net/netfilter/nft_set_pipapo.c index 6d77a5f0088ad0..18e1903b1d3d0b 100644 --- a/net/netfilter/nft_set_pipapo.c +++ b/net/netfilter/nft_set_pipapo.c @@ -2370,6 +2370,7 @@ const struct nft_set_type nft_set_pipapo_type = { .gc_init = nft_pipapo_gc_init, .commit = nft_pipapo_commit, .abort = nft_pipapo_abort, + .abort_skip_removal = true, .elemsize = offsetof(struct nft_pipapo_elem, ext), }, }; @@ -2394,6 +2395,7 @@ const struct nft_set_type nft_set_pipapo_avx2_type = { .gc_init = nft_pipapo_gc_init, .commit = nft_pipapo_commit, .abort = nft_pipapo_abort, + .abort_skip_removal = true, .elemsize = offsetof(struct nft_pipapo_elem, ext), }, }; diff --git a/net/netfilter/nft_set_rbtree.c b/net/netfilter/nft_set_rbtree.c index ca594161b8402e..644d4b9167057c 100644 --- a/net/netfilter/nft_set_rbtree.c +++ b/net/netfilter/nft_set_rbtree.c @@ -10,21 +10,41 @@ #include #include #include +#include #include #include #include #include +struct nft_array_interval { + struct nft_set_ext *from; + struct nft_set_ext *to; +}; + +struct nft_array { + u32 max_intervals; + u32 num_intervals; + struct nft_array_interval *intervals; + struct rcu_head rcu_head; +}; + struct nft_rbtree { struct rb_root root; rwlock_t lock; - seqcount_rwlock_t count; + struct nft_array __rcu *array; + struct nft_array *array_next; + unsigned long start_rbe_cookie; unsigned long last_gc; + struct list_head expired; + u64 last_tstamp; }; struct nft_rbtree_elem { struct nft_elem_priv priv; - struct rb_node node; + union { + struct rb_node node; + struct list_head list; + }; struct nft_set_ext ext; }; @@ -39,6 +59,13 @@ static bool nft_rbtree_interval_start(const struct nft_rbtree_elem *rbe) return !nft_rbtree_interval_end(rbe); } +static bool nft_rbtree_interval_null(const struct nft_set *set, + const struct nft_rbtree_elem *rbe) +{ + return (!memchr_inv(nft_set_ext_key(&rbe->ext), 0, set->klen) && + nft_rbtree_interval_end(rbe)); +} + static int nft_rbtree_cmp(const struct nft_set *set, const struct nft_rbtree_elem *e1, const struct nft_rbtree_elem *e2) @@ -47,67 +74,33 @@ static int nft_rbtree_cmp(const struct nft_set *set, set->klen); } -static bool nft_rbtree_elem_expired(const struct nft_rbtree_elem *rbe) -{ - return nft_set_elem_expired(&rbe->ext); -} +struct nft_array_lookup_ctx { + const u32 *key; + u32 klen; +}; -static const struct nft_set_ext * -__nft_rbtree_lookup(const struct net *net, const struct nft_set *set, - const u32 *key, unsigned int seq) +static int nft_array_lookup_cmp(const void *pkey, const void *entry) { - struct nft_rbtree *priv = nft_set_priv(set); - const struct nft_rbtree_elem *rbe, *interval = NULL; - u8 genmask = nft_genmask_cur(net); - const struct rb_node *parent; - int d; + const struct nft_array_interval *interval = entry; + const struct nft_array_lookup_ctx *ctx = pkey; + int a, b; - parent = rcu_dereference_raw(priv->root.rb_node); - while (parent != NULL) { - if (read_seqcount_retry(&priv->count, seq)) - return NULL; - - rbe = rb_entry(parent, struct nft_rbtree_elem, node); + if (!interval->from) + return 1; - d = memcmp(nft_set_ext_key(&rbe->ext), key, set->klen); - if (d < 0) { - parent = rcu_dereference_raw(parent->rb_left); - if (interval && - !nft_rbtree_cmp(set, rbe, interval) && - nft_rbtree_interval_end(rbe) && - nft_rbtree_interval_start(interval)) - continue; - if (nft_set_elem_active(&rbe->ext, genmask) && - !nft_rbtree_elem_expired(rbe)) - interval = rbe; - } else if (d > 0) - parent = rcu_dereference_raw(parent->rb_right); - else { - if (!nft_set_elem_active(&rbe->ext, genmask)) { - parent = rcu_dereference_raw(parent->rb_left); - continue; - } - - if (nft_rbtree_elem_expired(rbe)) - return NULL; - - if (nft_rbtree_interval_end(rbe)) { - if (nft_set_is_anonymous(set)) - return NULL; - parent = rcu_dereference_raw(parent->rb_left); - interval = NULL; - continue; - } + a = memcmp(ctx->key, nft_set_ext_key(interval->from), ctx->klen); + if (!interval->to) + b = -1; + else + b = memcmp(ctx->key, nft_set_ext_key(interval->to), ctx->klen); - return &rbe->ext; - } - } + if (a >= 0 && b < 0) + return 0; - if (set->flags & NFT_SET_INTERVAL && interval != NULL && - nft_rbtree_interval_start(interval)) - return &interval->ext; + if (a < 0) + return -1; - return NULL; + return 1; } INDIRECT_CALLABLE_SCOPE @@ -116,83 +109,57 @@ nft_rbtree_lookup(const struct net *net, const struct nft_set *set, const u32 *key) { struct nft_rbtree *priv = nft_set_priv(set); - unsigned int seq = read_seqcount_begin(&priv->count); - const struct nft_set_ext *ext; - - ext = __nft_rbtree_lookup(net, set, key, seq); - if (ext || !read_seqcount_retry(&priv->count, seq)) - return ext; - - read_lock_bh(&priv->lock); - seq = read_seqcount_begin(&priv->count); - ext = __nft_rbtree_lookup(net, set, key, seq); - read_unlock_bh(&priv->lock); - - return ext; + struct nft_array *array = rcu_dereference(priv->array); + const struct nft_array_interval *interval; + struct nft_array_lookup_ctx ctx = { + .key = key, + .klen = set->klen, + }; + + if (!array) + return NULL; + + interval = bsearch(&ctx, array->intervals, array->num_intervals, + sizeof(struct nft_array_interval), + nft_array_lookup_cmp); + if (!interval || nft_set_elem_expired(interval->from)) + return NULL; + + return interval->from; } -static bool __nft_rbtree_get(const struct net *net, const struct nft_set *set, - const u32 *key, struct nft_rbtree_elem **elem, - unsigned int seq, unsigned int flags, u8 genmask) -{ - struct nft_rbtree_elem *rbe, *interval = NULL; - struct nft_rbtree *priv = nft_set_priv(set); - const struct rb_node *parent; - const void *this; - int d; - - parent = rcu_dereference_raw(priv->root.rb_node); - while (parent != NULL) { - if (read_seqcount_retry(&priv->count, seq)) - return false; - - rbe = rb_entry(parent, struct nft_rbtree_elem, node); - - this = nft_set_ext_key(&rbe->ext); - d = memcmp(this, key, set->klen); - if (d < 0) { - parent = rcu_dereference_raw(parent->rb_left); - if (!(flags & NFT_SET_ELEM_INTERVAL_END)) - interval = rbe; - } else if (d > 0) { - parent = rcu_dereference_raw(parent->rb_right); - if (flags & NFT_SET_ELEM_INTERVAL_END) - interval = rbe; - } else { - if (!nft_set_elem_active(&rbe->ext, genmask)) { - parent = rcu_dereference_raw(parent->rb_left); - continue; - } +struct nft_array_get_ctx { + const u32 *key; + unsigned int flags; + u32 klen; +}; - if (nft_set_elem_expired(&rbe->ext)) - return false; +static int nft_array_get_cmp(const void *pkey, const void *entry) +{ + const struct nft_array_interval *interval = entry; + const struct nft_array_get_ctx *ctx = pkey; + int a, b; - if (!nft_set_ext_exists(&rbe->ext, NFT_SET_EXT_FLAGS) || - (*nft_set_ext_flags(&rbe->ext) & NFT_SET_ELEM_INTERVAL_END) == - (flags & NFT_SET_ELEM_INTERVAL_END)) { - *elem = rbe; - return true; - } + if (!interval->from) + return 1; - if (nft_rbtree_interval_end(rbe)) - interval = NULL; + a = memcmp(ctx->key, nft_set_ext_key(interval->from), ctx->klen); + if (!interval->to) + b = -1; + else + b = memcmp(ctx->key, nft_set_ext_key(interval->to), ctx->klen); - parent = rcu_dereference_raw(parent->rb_left); - } + if (a >= 0) { + if (ctx->flags & NFT_SET_ELEM_INTERVAL_END && b <= 0) + return 0; + else if (b < 0) + return 0; } - if (set->flags & NFT_SET_INTERVAL && interval != NULL && - nft_set_elem_active(&interval->ext, genmask) && - !nft_set_elem_expired(&interval->ext) && - ((!nft_rbtree_interval_end(interval) && - !(flags & NFT_SET_ELEM_INTERVAL_END)) || - (nft_rbtree_interval_end(interval) && - (flags & NFT_SET_ELEM_INTERVAL_END)))) { - *elem = interval; - return true; - } + if (a < 0) + return -1; - return false; + return 1; } static struct nft_elem_priv * @@ -200,34 +167,41 @@ nft_rbtree_get(const struct net *net, const struct nft_set *set, const struct nft_set_elem *elem, unsigned int flags) { struct nft_rbtree *priv = nft_set_priv(set); - unsigned int seq = read_seqcount_begin(&priv->count); - struct nft_rbtree_elem *rbe = ERR_PTR(-ENOENT); - const u32 *key = (const u32 *)&elem->key.val; - u8 genmask = nft_genmask_cur(net); - bool ret; - - ret = __nft_rbtree_get(net, set, key, &rbe, seq, flags, genmask); - if (ret || !read_seqcount_retry(&priv->count, seq)) - return &rbe->priv; - - read_lock_bh(&priv->lock); - seq = read_seqcount_begin(&priv->count); - ret = __nft_rbtree_get(net, set, key, &rbe, seq, flags, genmask); - read_unlock_bh(&priv->lock); - - if (!ret) + struct nft_array *array = rcu_dereference(priv->array); + const struct nft_array_interval *interval; + struct nft_array_get_ctx ctx = { + .key = (const u32 *)&elem->key.val, + .flags = flags, + .klen = set->klen, + }; + struct nft_rbtree_elem *rbe; + + if (!array) return ERR_PTR(-ENOENT); + interval = bsearch(&ctx, array->intervals, array->num_intervals, + sizeof(struct nft_array_interval), nft_array_get_cmp); + if (!interval || nft_set_elem_expired(interval->from)) + return ERR_PTR(-ENOENT); + + if (flags & NFT_SET_ELEM_INTERVAL_END) + rbe = container_of(interval->to, struct nft_rbtree_elem, ext); + else + rbe = container_of(interval->from, struct nft_rbtree_elem, ext); + return &rbe->priv; } -static void nft_rbtree_gc_elem_remove(struct net *net, struct nft_set *set, - struct nft_rbtree *priv, - struct nft_rbtree_elem *rbe) +static void nft_rbtree_gc_elem_move(struct net *net, struct nft_set *set, + struct nft_rbtree *priv, + struct nft_rbtree_elem *rbe) { lockdep_assert_held_write(&priv->lock); nft_setelem_data_deactivate(net, set, &rbe->priv); rb_erase(&rbe->node, &priv->root); + + /* collected later on in commit callback */ + list_add(&rbe->list, &priv->expired); } static const struct nft_rbtree_elem * @@ -238,11 +212,6 @@ nft_rbtree_gc_elem(const struct nft_set *__set, struct nft_rbtree *priv, struct rb_node *prev = rb_prev(&rbe->node); struct net *net = read_pnet(&set->net); struct nft_rbtree_elem *rbe_prev; - struct nft_trans_gc *gc; - - gc = nft_trans_gc_alloc(set, 0, GFP_ATOMIC); - if (!gc) - return ERR_PTR(-ENOMEM); /* search for end interval coming before this element. * end intervals don't carry a timeout extension, they @@ -260,28 +229,10 @@ nft_rbtree_gc_elem(const struct nft_set *__set, struct nft_rbtree *priv, rbe_prev = NULL; if (prev) { rbe_prev = rb_entry(prev, struct nft_rbtree_elem, node); - nft_rbtree_gc_elem_remove(net, set, priv, rbe_prev); - - /* There is always room in this trans gc for this element, - * memory allocation never actually happens, hence, the warning - * splat in such case. No need to set NFT_SET_ELEM_DEAD_BIT, - * this is synchronous gc which never fails. - */ - gc = nft_trans_gc_queue_sync(gc, GFP_ATOMIC); - if (WARN_ON_ONCE(!gc)) - return ERR_PTR(-ENOMEM); - - nft_trans_gc_elem_add(gc, rbe_prev); + nft_rbtree_gc_elem_move(net, set, priv, rbe_prev); } - nft_rbtree_gc_elem_remove(net, set, priv, rbe); - gc = nft_trans_gc_queue_sync(gc, GFP_ATOMIC); - if (WARN_ON_ONCE(!gc)) - return ERR_PTR(-ENOMEM); - - nft_trans_gc_elem_add(gc, rbe); - - nft_trans_gc_queue_sync_done(gc); + nft_rbtree_gc_elem_move(net, set, priv, rbe); return rbe_prev; } @@ -302,16 +253,107 @@ static bool nft_rbtree_update_first(const struct nft_set *set, return false; } +/* Only for anonymous sets which do not allow updates, all element are active. */ +static struct nft_rbtree_elem *nft_rbtree_prev_active(struct nft_rbtree_elem *rbe) +{ + struct rb_node *node; + + node = rb_prev(&rbe->node); + if (!node) + return NULL; + + return rb_entry(node, struct nft_rbtree_elem, node); +} + +static struct nft_rbtree_elem * +__nft_rbtree_next_active(struct rb_node *node, u8 genmask) +{ + struct nft_rbtree_elem *next_rbe; + + while (node) { + next_rbe = rb_entry(node, struct nft_rbtree_elem, node); + if (!nft_set_elem_active(&next_rbe->ext, genmask)) { + node = rb_next(node); + continue; + } + + return next_rbe; + } + + return NULL; +} + +static struct nft_rbtree_elem * +nft_rbtree_next_active(struct nft_rbtree_elem *rbe, u8 genmask) +{ + return __nft_rbtree_next_active(rb_next(&rbe->node), genmask); +} + +static void nft_rbtree_maybe_reset_start_cookie(struct nft_rbtree *priv, + u64 tstamp) +{ + if (priv->last_tstamp != tstamp) { + priv->start_rbe_cookie = 0; + priv->last_tstamp = tstamp; + } +} + +static void nft_rbtree_set_start_cookie(struct nft_rbtree *priv, + const struct nft_rbtree_elem *rbe) +{ + priv->start_rbe_cookie = (unsigned long)rbe; +} + +static void nft_rbtree_set_start_cookie_open(struct nft_rbtree *priv, + const struct nft_rbtree_elem *rbe, + unsigned long open_interval) +{ + priv->start_rbe_cookie = (unsigned long)rbe | open_interval; +} + +#define NFT_RBTREE_OPEN_INTERVAL 1UL + +static bool nft_rbtree_cmp_start_cookie(struct nft_rbtree *priv, + const struct nft_rbtree_elem *rbe) +{ + return (priv->start_rbe_cookie & ~NFT_RBTREE_OPEN_INTERVAL) == (unsigned long)rbe; +} + +static bool nft_rbtree_insert_same_interval(const struct net *net, + struct nft_rbtree *priv, + struct nft_rbtree_elem *rbe) +{ + u8 genmask = nft_genmask_next(net); + struct nft_rbtree_elem *next_rbe; + + if (!priv->start_rbe_cookie) + return true; + + next_rbe = nft_rbtree_next_active(rbe, genmask); + if (next_rbe) { + /* Closest start element differs from last element added. */ + if (nft_rbtree_interval_start(next_rbe) && + nft_rbtree_cmp_start_cookie(priv, next_rbe)) { + priv->start_rbe_cookie = 0; + return true; + } + } + + priv->start_rbe_cookie = 0; + + return false; +} + static int __nft_rbtree_insert(const struct net *net, const struct nft_set *set, struct nft_rbtree_elem *new, - struct nft_elem_priv **elem_priv) + struct nft_elem_priv **elem_priv, u64 tstamp, bool last) { - struct nft_rbtree_elem *rbe, *rbe_le = NULL, *rbe_ge = NULL; + struct nft_rbtree_elem *rbe, *rbe_le = NULL, *rbe_ge = NULL, *rbe_prev; struct rb_node *node, *next, *parent, **p, *first = NULL; struct nft_rbtree *priv = nft_set_priv(set); u8 cur_genmask = nft_genmask_cur(net); u8 genmask = nft_genmask_next(net); - u64 tstamp = nft_net_tstamp(net); + unsigned long open_interval = 0; int d; /* Descend the tree to search for an existing element greater than the @@ -417,12 +459,46 @@ static int __nft_rbtree_insert(const struct net *net, const struct nft_set *set, } } + if (nft_rbtree_interval_null(set, new)) { + priv->start_rbe_cookie = 0; + } else if (nft_rbtree_interval_start(new) && priv->start_rbe_cookie) { + if (nft_set_is_anonymous(set)) { + priv->start_rbe_cookie = 0; + } else if (priv->start_rbe_cookie & NFT_RBTREE_OPEN_INTERVAL) { + /* Previous element is an open interval that partially + * overlaps with an existing non-open interval. + */ + return -ENOTEMPTY; + } + } + /* - new start element matching existing start element: full overlap * reported as -EEXIST, cleared by caller if NLM_F_EXCL is not given. */ if (rbe_ge && !nft_rbtree_cmp(set, new, rbe_ge) && nft_rbtree_interval_start(rbe_ge) == nft_rbtree_interval_start(new)) { *elem_priv = &rbe_ge->priv; + + /* - Corner case: new start element of open interval (which + * comes as last element in the batch) overlaps the start of + * an existing interval with an end element: partial overlap. + */ + node = rb_first(&priv->root); + rbe = __nft_rbtree_next_active(node, genmask); + if (rbe && nft_rbtree_interval_end(rbe)) { + rbe = nft_rbtree_next_active(rbe, genmask); + if (rbe && + nft_rbtree_interval_start(rbe) && + !nft_rbtree_cmp(set, new, rbe)) { + if (last) + return -ENOTEMPTY; + + /* Maybe open interval? */ + open_interval = NFT_RBTREE_OPEN_INTERVAL; + } + } + nft_rbtree_set_start_cookie_open(priv, rbe_ge, open_interval); + return -EEXIST; } @@ -431,18 +507,37 @@ static int __nft_rbtree_insert(const struct net *net, const struct nft_set *set, */ if (rbe_le && !nft_rbtree_cmp(set, new, rbe_le) && nft_rbtree_interval_end(rbe_le) == nft_rbtree_interval_end(new)) { + /* - ignore null interval, otherwise NLM_F_CREATE bogusly + * reports EEXIST. + */ + if (nft_rbtree_interval_null(set, new)) + return -ECANCELED; + *elem_priv = &rbe_le->priv; + + /* - start and end element belong to the same interval. */ + if (!nft_rbtree_insert_same_interval(net, priv, rbe_le)) + return -ENOTEMPTY; + return -EEXIST; } /* - new start element with existing closest, less or equal key value * being a start element: partial overlap, reported as -ENOTEMPTY. * Anonymous sets allow for two consecutive start element since they - * are constant, skip them to avoid bogus overlap reports. + * are constant, but validate that this new start element does not + * sit in between an existing start and end elements: partial overlap, + * reported as -ENOTEMPTY. */ - if (!nft_set_is_anonymous(set) && rbe_le && - nft_rbtree_interval_start(rbe_le) && nft_rbtree_interval_start(new)) - return -ENOTEMPTY; + if (rbe_le && + nft_rbtree_interval_start(rbe_le) && nft_rbtree_interval_start(new)) { + if (!nft_set_is_anonymous(set)) + return -ENOTEMPTY; + + rbe_prev = nft_rbtree_prev_active(rbe_le); + if (rbe_prev && nft_rbtree_interval_end(rbe_prev)) + return -ENOTEMPTY; + } /* - new end element with existing closest, less or equal key value * being a end element: partial overlap, reported as -ENOTEMPTY. @@ -458,6 +553,12 @@ static int __nft_rbtree_insert(const struct net *net, const struct nft_set *set, nft_rbtree_interval_end(rbe_ge) && nft_rbtree_interval_end(new)) return -ENOTEMPTY; + /* - start element overlaps an open interval but end element is new: + * partial overlap, reported as -ENOEMPTY. + */ + if (!rbe_ge && priv->start_rbe_cookie && nft_rbtree_interval_end(new)) + return -ENOTEMPTY; + /* Accepted element: pick insertion point depending on key value */ parent = NULL; p = &priv->root.rb_node; @@ -481,14 +582,102 @@ static int __nft_rbtree_insert(const struct net *net, const struct nft_set *set, return 0; } +static int nft_array_intervals_alloc(struct nft_array *array, u32 max_intervals) +{ + struct nft_array_interval *intervals; + + intervals = kvcalloc(max_intervals, sizeof(struct nft_array_interval), + GFP_KERNEL_ACCOUNT); + if (!intervals) + return -ENOMEM; + + if (array->intervals) + kvfree(array->intervals); + + array->intervals = intervals; + array->max_intervals = max_intervals; + + return 0; +} + +static struct nft_array *nft_array_alloc(u32 max_intervals) +{ + struct nft_array *array; + + array = kzalloc(sizeof(*array), GFP_KERNEL_ACCOUNT); + if (!array) + return NULL; + + if (nft_array_intervals_alloc(array, max_intervals) < 0) { + kfree(array); + return NULL; + } + + return array; +} + +#define NFT_ARRAY_EXTRA_SIZE 10240 + +/* Similar to nft_rbtree_{u,k}size to hide details to userspace, but consider + * packed representation coming from userspace for anonymous sets too. + */ +static u32 nft_array_elems(const struct nft_set *set) +{ + u32 nelems = atomic_read(&set->nelems); + + /* Adjacent intervals are represented with a single start element in + * anonymous sets, use the current element counter as is. + */ + if (nft_set_is_anonymous(set)) + return nelems; + + /* Add extra room for never matching interval at the beginning and open + * interval at the end which only use a single element to represent it. + * The conversion to array will compact intervals, this allows reduce + * memory consumption. + */ + return (nelems / 2) + 2; +} + +static int nft_array_may_resize(const struct nft_set *set) +{ + u32 nelems = nft_array_elems(set), new_max_intervals; + struct nft_rbtree *priv = nft_set_priv(set); + struct nft_array *array; + + if (!priv->array_next) { + array = nft_array_alloc(nelems + NFT_ARRAY_EXTRA_SIZE); + if (!array) + return -ENOMEM; + + priv->array_next = array; + } + + if (nelems < priv->array_next->max_intervals) + return 0; + + new_max_intervals = priv->array_next->max_intervals + NFT_ARRAY_EXTRA_SIZE; + if (nft_array_intervals_alloc(priv->array_next, new_max_intervals) < 0) + return -ENOMEM; + + return 0; +} + static int nft_rbtree_insert(const struct net *net, const struct nft_set *set, const struct nft_set_elem *elem, struct nft_elem_priv **elem_priv) { struct nft_rbtree_elem *rbe = nft_elem_priv_cast(elem->priv); + bool last = !!(elem->flags & NFT_SET_ELEM_INTERNAL_LAST); struct nft_rbtree *priv = nft_set_priv(set); + u64 tstamp = nft_net_tstamp(net); int err; + nft_rbtree_maybe_reset_start_cookie(priv, tstamp); + + if (nft_array_may_resize(set) < 0) + return -ENOMEM; + do { if (fatal_signal_pending(current)) return -EINTR; @@ -496,10 +685,12 @@ static int nft_rbtree_insert(const struct net *net, const struct nft_set *set, cond_resched(); write_lock_bh(&priv->lock); - write_seqcount_begin(&priv->count); - err = __nft_rbtree_insert(net, set, rbe, elem_priv); - write_seqcount_end(&priv->count); + err = __nft_rbtree_insert(net, set, rbe, elem_priv, tstamp, last); write_unlock_bh(&priv->lock); + + if (nft_rbtree_interval_end(rbe)) + priv->start_rbe_cookie = 0; + } while (err == -EAGAIN); return err; @@ -508,9 +699,7 @@ static int nft_rbtree_insert(const struct net *net, const struct nft_set *set, static void nft_rbtree_erase(struct nft_rbtree *priv, struct nft_rbtree_elem *rbe) { write_lock_bh(&priv->lock); - write_seqcount_begin(&priv->count); rb_erase(&rbe->node, &priv->root); - write_seqcount_end(&priv->count); write_unlock_bh(&priv->lock); } @@ -533,6 +722,48 @@ static void nft_rbtree_activate(const struct net *net, nft_clear(net, &rbe->ext); } +static struct nft_rbtree_elem * +nft_rbtree_next_inactive(struct nft_rbtree_elem *rbe, u8 genmask) +{ + struct nft_rbtree_elem *next_rbe; + struct rb_node *node; + + node = rb_next(&rbe->node); + if (node) { + next_rbe = rb_entry(node, struct nft_rbtree_elem, node); + if (nft_rbtree_interval_start(next_rbe) && + !nft_set_elem_active(&next_rbe->ext, genmask)) + return next_rbe; + } + + return NULL; +} + +static bool nft_rbtree_deactivate_same_interval(const struct net *net, + struct nft_rbtree *priv, + struct nft_rbtree_elem *rbe) +{ + u8 genmask = nft_genmask_next(net); + struct nft_rbtree_elem *next_rbe; + + if (!priv->start_rbe_cookie) + return true; + + next_rbe = nft_rbtree_next_inactive(rbe, genmask); + if (next_rbe) { + /* Closest start element differs from last element added. */ + if (nft_rbtree_interval_start(next_rbe) && + nft_rbtree_cmp_start_cookie(priv, next_rbe)) { + priv->start_rbe_cookie = 0; + return true; + } + } + + priv->start_rbe_cookie = 0; + + return false; +} + static void nft_rbtree_flush(const struct net *net, const struct nft_set *set, struct nft_elem_priv *elem_priv) @@ -547,12 +778,22 @@ nft_rbtree_deactivate(const struct net *net, const struct nft_set *set, const struct nft_set_elem *elem) { struct nft_rbtree_elem *rbe, *this = nft_elem_priv_cast(elem->priv); - const struct nft_rbtree *priv = nft_set_priv(set); + bool last = !!(elem->flags & NFT_SET_ELEM_INTERNAL_LAST); + struct nft_rbtree *priv = nft_set_priv(set); const struct rb_node *parent = priv->root.rb_node; u8 genmask = nft_genmask_next(net); u64 tstamp = nft_net_tstamp(net); int d; + nft_rbtree_maybe_reset_start_cookie(priv, tstamp); + + if (nft_rbtree_interval_start(this) || + nft_rbtree_interval_null(set, this)) + priv->start_rbe_cookie = 0; + + if (nft_array_may_resize(set) < 0) + return NULL; + while (parent != NULL) { rbe = rb_entry(parent, struct nft_rbtree_elem, node); @@ -577,6 +818,13 @@ nft_rbtree_deactivate(const struct net *net, const struct nft_set *set, parent = parent->rb_left; continue; } + + if (nft_rbtree_interval_start(rbe)) { + if (!last) + nft_rbtree_set_start_cookie(priv, rbe); + } else if (!nft_rbtree_deactivate_same_interval(net, priv, rbe)) + return NULL; + nft_rbtree_flush(net, set, &rbe->priv); return &rbe->priv; } @@ -615,6 +863,11 @@ static void nft_rbtree_walk(const struct nft_ctx *ctx, switch (iter->type) { case NFT_ITER_UPDATE: lockdep_assert_held(&nft_pernet(ctx->net)->commit_mutex); + + if (nft_array_may_resize(set) < 0) { + iter->err = -ENOMEM; + break; + } nft_rbtree_do_walk(ctx, set, iter); break; case NFT_ITER_READ: @@ -629,29 +882,13 @@ static void nft_rbtree_walk(const struct nft_ctx *ctx, } } -static void nft_rbtree_gc_remove(struct net *net, struct nft_set *set, - struct nft_rbtree *priv, - struct nft_rbtree_elem *rbe) -{ - nft_setelem_data_deactivate(net, set, &rbe->priv); - nft_rbtree_erase(priv, rbe); -} - -static void nft_rbtree_gc(struct nft_set *set) +static void nft_rbtree_gc_scan(struct nft_set *set) { struct nft_rbtree *priv = nft_set_priv(set); struct nft_rbtree_elem *rbe, *rbe_end = NULL; struct net *net = read_pnet(&set->net); u64 tstamp = nft_net_tstamp(net); struct rb_node *node, *next; - struct nft_trans_gc *gc; - - set = nft_set_container_of(priv); - net = read_pnet(&set->net); - - gc = nft_trans_gc_alloc(set, 0, GFP_KERNEL); - if (!gc) - return; for (node = rb_first(&priv->root); node ; node = next) { next = rb_next(node); @@ -669,34 +906,46 @@ static void nft_rbtree_gc(struct nft_set *set) if (!__nft_set_elem_expired(&rbe->ext, tstamp)) continue; - gc = nft_trans_gc_queue_sync(gc, GFP_KERNEL); - if (!gc) - goto try_later; - /* end element needs to be removed first, it has * no timeout extension. */ + write_lock_bh(&priv->lock); if (rbe_end) { - nft_rbtree_gc_remove(net, set, priv, rbe_end); - nft_trans_gc_elem_add(gc, rbe_end); + nft_rbtree_gc_elem_move(net, set, priv, rbe_end); rbe_end = NULL; } - gc = nft_trans_gc_queue_sync(gc, GFP_KERNEL); - if (!gc) - goto try_later; - - nft_rbtree_gc_remove(net, set, priv, rbe); - nft_trans_gc_elem_add(gc, rbe); + nft_rbtree_gc_elem_move(net, set, priv, rbe); + write_unlock_bh(&priv->lock); } -try_later: + priv->last_gc = jiffies; +} + +static void nft_rbtree_gc_queue(struct nft_set *set) +{ + struct nft_rbtree *priv = nft_set_priv(set); + struct nft_rbtree_elem *rbe, *rbe_end; + struct nft_trans_gc *gc; - if (gc) { - gc = nft_trans_gc_catchall_sync(gc); - nft_trans_gc_queue_sync_done(gc); - priv->last_gc = jiffies; + if (list_empty(&priv->expired)) + return; + + gc = nft_trans_gc_alloc(set, 0, GFP_KERNEL); + if (!gc) + return; + + list_for_each_entry_safe(rbe, rbe_end, &priv->expired, list) { + list_del(&rbe->list); + nft_trans_gc_elem_add(gc, rbe); + + gc = nft_trans_gc_queue_sync(gc, GFP_KERNEL); + if (!gc) + return; } + + gc = nft_trans_gc_catchall_sync(gc); + nft_trans_gc_queue_sync_done(gc); } static u64 nft_rbtree_privsize(const struct nlattr * const nla[], @@ -714,24 +963,45 @@ static int nft_rbtree_init(const struct nft_set *set, BUILD_BUG_ON(offsetof(struct nft_rbtree_elem, priv) != 0); rwlock_init(&priv->lock); - seqcount_rwlock_init(&priv->count, &priv->lock); priv->root = RB_ROOT; + INIT_LIST_HEAD(&priv->expired); + + priv->array = NULL; + priv->array_next = NULL; return 0; } +static void __nft_array_free(struct nft_array *array) +{ + kvfree(array->intervals); + kfree(array); +} + static void nft_rbtree_destroy(const struct nft_ctx *ctx, const struct nft_set *set) { struct nft_rbtree *priv = nft_set_priv(set); - struct nft_rbtree_elem *rbe; + struct nft_rbtree_elem *rbe, *next; + struct nft_array *array; struct rb_node *node; + list_for_each_entry_safe(rbe, next, &priv->expired, list) { + list_del(&rbe->list); + nf_tables_set_elem_destroy(ctx, set, &rbe->priv); + } + while ((node = priv->root.rb_node) != NULL) { rb_erase(node, &priv->root); rbe = rb_entry(node, struct nft_rbtree_elem, node); nf_tables_set_elem_destroy(ctx, set, &rbe->priv); } + + array = rcu_dereference_protected(priv->array, true); + if (array) + __nft_array_free(array); + if (priv->array_next) + __nft_array_free(priv->array_next); } static bool nft_rbtree_estimate(const struct nft_set_desc *desc, u32 features, @@ -752,12 +1022,105 @@ static bool nft_rbtree_estimate(const struct nft_set_desc *desc, u32 features, return true; } +static void nft_array_free_rcu(struct rcu_head *rcu_head) +{ + struct nft_array *array = container_of(rcu_head, struct nft_array, rcu_head); + + __nft_array_free(array); +} + static void nft_rbtree_commit(struct nft_set *set) { struct nft_rbtree *priv = nft_set_priv(set); + struct nft_rbtree_elem *rbe, *prev_rbe; + struct nft_array *old; + u32 num_intervals = 0; + struct rb_node *node; + /* No changes, skip, eg. elements updates only. */ + if (!priv->array_next) + return; + + /* GC can be performed if the binary search blob is going + * to be rebuilt. It has to be done in two phases: first + * scan tree and move all expired elements to the expired + * list. + * + * Then, after blob has been re-built and published to other + * CPUs, queue collected entries for freeing. + */ if (time_after_eq(jiffies, priv->last_gc + nft_set_gc_interval(set))) - nft_rbtree_gc(set); + nft_rbtree_gc_scan(set); + + /* Reverse walk to create an array from smaller to largest interval. */ + node = rb_last(&priv->root); + if (node) + prev_rbe = rb_entry(node, struct nft_rbtree_elem, node); + else + prev_rbe = NULL; + + while (prev_rbe) { + rbe = prev_rbe; + + if (nft_rbtree_interval_start(rbe)) + priv->array_next->intervals[num_intervals].from = &rbe->ext; + else if (nft_rbtree_interval_end(rbe)) + priv->array_next->intervals[num_intervals++].to = &rbe->ext; + + if (num_intervals >= priv->array_next->max_intervals) { + pr_warn_once("malformed interval set from userspace?"); + goto err_out; + } + + node = rb_prev(node); + if (!node) + break; + + prev_rbe = rb_entry(node, struct nft_rbtree_elem, node); + + /* For anonymous sets, when adjacent ranges are found, + * the end element is not added to the set to pack the set + * representation. Use next start element to complete this + * interval. + */ + if (nft_rbtree_interval_start(rbe) && + nft_rbtree_interval_start(prev_rbe) && + priv->array_next->intervals[num_intervals].from) + priv->array_next->intervals[num_intervals++].to = &prev_rbe->ext; + + if (num_intervals >= priv->array_next->max_intervals) { + pr_warn_once("malformed interval set from userspace?"); + goto err_out; + } + } + + if (priv->array_next->intervals[num_intervals].from) + num_intervals++; +err_out: + priv->array_next->num_intervals = num_intervals; + old = rcu_replace_pointer(priv->array, priv->array_next, + lockdep_is_held(&nft_pernet(read_pnet(&set->net))->commit_mutex)); + priv->array_next = NULL; + if (old) + call_rcu(&old->rcu_head, nft_array_free_rcu); + + /* New blob is public, queue collected entries for freeing. + * call_rcu ensures elements stay around until readers are done. + */ + nft_rbtree_gc_queue(set); +} + +static void nft_rbtree_abort(const struct nft_set *set) +{ + struct nft_rbtree *priv = nft_set_priv(set); + struct nft_array *array_next; + + if (!priv->array_next) + return; + + array_next = priv->array_next; + priv->array_next = NULL; + __nft_array_free(array_next); } static void nft_rbtree_gc_init(const struct nft_set *set) @@ -821,6 +1184,7 @@ const struct nft_set_type nft_set_rbtree_type = { .flush = nft_rbtree_flush, .activate = nft_rbtree_activate, .commit = nft_rbtree_commit, + .abort = nft_rbtree_abort, .gc_init = nft_rbtree_gc_init, .lookup = nft_rbtree_lookup, .walk = nft_rbtree_walk, diff --git a/net/nfc/hci/llc_shdlc.c b/net/nfc/hci/llc_shdlc.c index 4fc37894860c9e..08c8aa1530d8a0 100644 --- a/net/nfc/hci/llc_shdlc.c +++ b/net/nfc/hci/llc_shdlc.c @@ -762,6 +762,14 @@ static void llc_shdlc_deinit(struct nfc_llc *llc) { struct llc_shdlc *shdlc = nfc_llc_get_data(llc); + timer_shutdown_sync(&shdlc->connect_timer); + timer_shutdown_sync(&shdlc->t1_timer); + timer_shutdown_sync(&shdlc->t2_timer); + shdlc->t1_active = false; + shdlc->t2_active = false; + + cancel_work_sync(&shdlc->sm_work); + skb_queue_purge(&shdlc->rcv_q); skb_queue_purge(&shdlc->send_q); skb_queue_purge(&shdlc->ack_pending_q); diff --git a/net/psp/Kconfig b/net/psp/Kconfig index 371e8771f3bd38..84d6b0f2546067 100644 --- a/net/psp/Kconfig +++ b/net/psp/Kconfig @@ -6,6 +6,7 @@ config INET_PSP bool "PSP Security Protocol support" depends on INET select SKB_DECRYPTED + select SKB_EXTENSIONS select SOCK_VALIDATE_XMIT help Enable kernel support for the PSP Security Protocol (PSP). diff --git a/net/rds/send.c b/net/rds/send.c index 0b3d0ef2f008b6..071c5dca969a23 100644 --- a/net/rds/send.c +++ b/net/rds/send.c @@ -1382,9 +1382,11 @@ int rds_sendmsg(struct socket *sock, struct msghdr *msg, size_t payload_len) else queue_delayed_work(rds_wq, &cpath->cp_send_w, 1); rcu_read_unlock(); + + if (ret) + goto out; } - if (ret) - goto out; + rds_message_put(rm); for (ind = 0; ind < vct.indx; ind++) diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c index f97f77b041d974..d8201eb3ac5f3d 100644 --- a/net/smc/af_smc.c +++ b/net/smc/af_smc.c @@ -3357,11 +3357,10 @@ int smc_create_clcsk(struct net *net, struct sock *sk, int family) return 0; } -static int __smc_create(struct net *net, struct socket *sock, int protocol, - int kern, struct socket *clcsock) +static int smc_create(struct net *net, struct socket *sock, int protocol, + int kern) { int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET; - struct smc_sock *smc; struct sock *sk; int rc; @@ -3380,15 +3379,7 @@ static int __smc_create(struct net *net, struct socket *sock, int protocol, if (!sk) goto out; - /* create internal TCP socket for CLC handshake and fallback */ - smc = smc_sk(sk); - - rc = 0; - if (clcsock) - smc->clcsock = clcsock; - else - rc = smc_create_clcsk(net, sk, family); - + rc = smc_create_clcsk(net, sk, family); if (rc) { sk_common_release(sk); sock->sk = NULL; @@ -3397,76 +3388,12 @@ static int __smc_create(struct net *net, struct socket *sock, int protocol, return rc; } -static int smc_create(struct net *net, struct socket *sock, int protocol, - int kern) -{ - return __smc_create(net, sock, protocol, kern, NULL); -} - static const struct net_proto_family smc_sock_family_ops = { .family = PF_SMC, .owner = THIS_MODULE, .create = smc_create, }; -static int smc_ulp_init(struct sock *sk) -{ - struct socket *tcp = sk->sk_socket; - struct net *net = sock_net(sk); - struct socket *smcsock; - int protocol, ret; - - /* only TCP can be replaced */ - if (tcp->type != SOCK_STREAM || sk->sk_protocol != IPPROTO_TCP || - (sk->sk_family != AF_INET && sk->sk_family != AF_INET6)) - return -ESOCKTNOSUPPORT; - /* don't handle wq now */ - if (tcp->state != SS_UNCONNECTED || !tcp->file || tcp->wq.fasync_list) - return -ENOTCONN; - - if (sk->sk_family == AF_INET) - protocol = SMCPROTO_SMC; - else - protocol = SMCPROTO_SMC6; - - smcsock = sock_alloc(); - if (!smcsock) - return -ENFILE; - - smcsock->type = SOCK_STREAM; - __module_get(THIS_MODULE); /* tried in __tcp_ulp_find_autoload */ - ret = __smc_create(net, smcsock, protocol, 1, tcp); - if (ret) { - sock_release(smcsock); /* module_put() which ops won't be NULL */ - return ret; - } - - /* replace tcp socket to smc */ - smcsock->file = tcp->file; - smcsock->file->private_data = smcsock; - smcsock->file->f_inode = SOCK_INODE(smcsock); /* replace inode when sock_close */ - smcsock->file->f_path.dentry->d_inode = SOCK_INODE(smcsock); /* dput() in __fput */ - tcp->file = NULL; - - return ret; -} - -static void smc_ulp_clone(const struct request_sock *req, struct sock *newsk, - const gfp_t priority) -{ - struct inet_connection_sock *icsk = inet_csk(newsk); - - /* don't inherit ulp ops to child when listen */ - icsk->icsk_ulp_ops = NULL; -} - -static struct tcp_ulp_ops smc_ulp_ops __read_mostly = { - .name = "smc", - .owner = THIS_MODULE, - .init = smc_ulp_init, - .clone = smc_ulp_clone, -}; - unsigned int smc_net_id; static __net_init int smc_net_init(struct net *net) @@ -3589,16 +3516,10 @@ static int __init smc_init(void) pr_err("%s: ib_register fails with %d\n", __func__, rc); goto out_sock; } - - rc = tcp_register_ulp(&smc_ulp_ops); - if (rc) { - pr_err("%s: tcp_ulp_register fails with %d\n", __func__, rc); - goto out_ib; - } rc = smc_inet_init(); if (rc) { pr_err("%s: smc_inet_init fails with %d\n", __func__, rc); - goto out_ulp; + goto out_ib; } rc = bpf_smc_hs_ctrl_init(); if (rc) { @@ -3610,8 +3531,6 @@ static int __init smc_init(void) return 0; out_inet: smc_inet_exit(); -out_ulp: - tcp_unregister_ulp(&smc_ulp_ops); out_ib: smc_ib_unregister_client(); out_sock: @@ -3647,7 +3566,6 @@ static void __exit smc_exit(void) { static_branch_disable(&tcp_have_smc); smc_inet_exit(); - tcp_unregister_ulp(&smc_ulp_ops); sock_unregister(PF_SMC); smc_core_exit(); smc_ib_unregister_client(); @@ -3672,7 +3590,6 @@ MODULE_AUTHOR("Ursula Braun "); MODULE_DESCRIPTION("smc socket address family"); MODULE_LICENSE("GPL"); MODULE_ALIAS_NETPROTO(PF_SMC); -MODULE_ALIAS_TCP_ULP("smc"); /* 256 for IPPROTO_SMC and 1 for SOCK_STREAM */ MODULE_ALIAS_NET_PF_PROTO_TYPE(PF_INET, 256, 1); #if IS_ENABLED(CONFIG_IPV6) diff --git a/net/sunrpc/auth_gss/auth_gss.c b/net/sunrpc/auth_gss/auth_gss.c index 5c095cb8cb201e..bb3c3db2713b1f 100644 --- a/net/sunrpc/auth_gss/auth_gss.c +++ b/net/sunrpc/auth_gss/auth_gss.c @@ -39,6 +39,8 @@ static const struct rpc_authops authgss_ops; static const struct rpc_credops gss_credops; static const struct rpc_credops gss_nullops; +static void gss_free_callback(struct kref *kref); + #define GSS_RETRY_EXPIRED 5 static unsigned int gss_expired_cred_retry_delay = GSS_RETRY_EXPIRED; @@ -551,6 +553,7 @@ gss_alloc_msg(struct gss_auth *gss_auth, } return gss_msg; err_put_pipe_version: + kref_put(&gss_auth->kref, gss_free_callback); put_pipe_version(gss_auth->net); err_free_msg: kfree(gss_msg); diff --git a/net/sunrpc/auth_gss/gss_rpc_xdr.c b/net/sunrpc/auth_gss/gss_rpc_xdr.c index 7d2cdc2bd374e2..f320c0a8e60493 100644 --- a/net/sunrpc/auth_gss/gss_rpc_xdr.c +++ b/net/sunrpc/auth_gss/gss_rpc_xdr.c @@ -320,29 +320,47 @@ static int gssx_dec_status(struct xdr_stream *xdr, /* status->minor_status */ p = xdr_inline_decode(xdr, 8); - if (unlikely(p == NULL)) - return -ENOSPC; + if (unlikely(p == NULL)) { + err = -ENOSPC; + goto out_free_mech; + } p = xdr_decode_hyper(p, &status->minor_status); /* status->major_status_string */ err = gssx_dec_buffer(xdr, &status->major_status_string); if (err) - return err; + goto out_free_mech; /* status->minor_status_string */ err = gssx_dec_buffer(xdr, &status->minor_status_string); if (err) - return err; + goto out_free_major_status_string; /* status->server_ctx */ err = gssx_dec_buffer(xdr, &status->server_ctx); if (err) - return err; + goto out_free_minor_status_string; /* we assume we have no options for now, so simply consume them */ /* status->options */ err = dummy_dec_opt_array(xdr, &status->options); + if (err) + goto out_free_server_ctx; + return 0; + +out_free_server_ctx: + kfree(status->server_ctx.data); + status->server_ctx.data = NULL; +out_free_minor_status_string: + kfree(status->minor_status_string.data); + status->minor_status_string.data = NULL; +out_free_major_status_string: + kfree(status->major_status_string.data); + status->major_status_string.data = NULL; +out_free_mech: + kfree(status->mech.data); + status->mech.data = NULL; return err; } @@ -505,28 +523,35 @@ static int gssx_dec_name(struct xdr_stream *xdr, /* name->name_type */ err = gssx_dec_buffer(xdr, &dummy_netobj); if (err) - return err; + goto out_free_display_name; /* name->exported_name */ err = gssx_dec_buffer(xdr, &dummy_netobj); if (err) - return err; + goto out_free_display_name; /* name->exported_composite_name */ err = gssx_dec_buffer(xdr, &dummy_netobj); if (err) - return err; + goto out_free_display_name; /* we assume we have no attributes for now, so simply consume them */ /* name->name_attributes */ err = dummy_dec_nameattr_array(xdr, &dummy_name_attr_array); if (err) - return err; + goto out_free_display_name; /* we assume we have no options for now, so simply consume them */ /* name->extensions */ err = dummy_dec_opt_array(xdr, &dummy_option_array); + if (err) + goto out_free_display_name; + return 0; + +out_free_display_name: + kfree(name->display_name.data); + name->display_name.data = NULL; return err; } @@ -649,32 +674,34 @@ static int gssx_dec_ctx(struct xdr_stream *xdr, /* ctx->state */ err = gssx_dec_buffer(xdr, &ctx->state); if (err) - return err; + goto out_free_exported_context_token; /* ctx->need_release */ err = gssx_dec_bool(xdr, &ctx->need_release); if (err) - return err; + goto out_free_state; /* ctx->mech */ err = gssx_dec_buffer(xdr, &ctx->mech); if (err) - return err; + goto out_free_state; /* ctx->src_name */ err = gssx_dec_name(xdr, &ctx->src_name); if (err) - return err; + goto out_free_mech; /* ctx->targ_name */ err = gssx_dec_name(xdr, &ctx->targ_name); if (err) - return err; + goto out_free_src_name; /* ctx->lifetime */ p = xdr_inline_decode(xdr, 8+8); - if (unlikely(p == NULL)) - return -ENOSPC; + if (unlikely(p == NULL)) { + err = -ENOSPC; + goto out_free_targ_name; + } p = xdr_decode_hyper(p, &ctx->lifetime); /* ctx->ctx_flags */ @@ -683,17 +710,36 @@ static int gssx_dec_ctx(struct xdr_stream *xdr, /* ctx->locally_initiated */ err = gssx_dec_bool(xdr, &ctx->locally_initiated); if (err) - return err; + goto out_free_targ_name; /* ctx->open */ err = gssx_dec_bool(xdr, &ctx->open); if (err) - return err; + goto out_free_targ_name; /* we assume we have no options for now, so simply consume them */ /* ctx->options */ err = dummy_dec_opt_array(xdr, &ctx->options); + if (err) + goto out_free_targ_name; + + return 0; +out_free_targ_name: + kfree(ctx->targ_name.display_name.data); + ctx->targ_name.display_name.data = NULL; +out_free_src_name: + kfree(ctx->src_name.display_name.data); + ctx->src_name.display_name.data = NULL; +out_free_mech: + kfree(ctx->mech.data); + ctx->mech.data = NULL; +out_free_state: + kfree(ctx->state.data); + ctx->state.data = NULL; +out_free_exported_context_token: + kfree(ctx->exported_context_token.data); + ctx->exported_context_token.data = NULL; return err; } diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c index b7b318ad25c42c..9b623849723ed0 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_transport.c +++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c @@ -462,7 +462,10 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) newxprt->sc_max_bc_requests = 2; } - /* Arbitrary estimate of the needed number of rdma_rw contexts. + /* Estimate the needed number of rdma_rw contexts. The maximum + * Read and Write chunks have one segment each. Each request + * can involve one Read chunk and either a Write chunk or Reply + * chunk; thus a factor of three. */ maxpayload = min(xprt->xpt_server->sv_max_payload, RPCSVC_MAXPAYLOAD_RDMA); @@ -470,7 +473,8 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) rdma_rw_mr_factor(dev, newxprt->sc_port_num, maxpayload >> PAGE_SHIFT); - newxprt->sc_sq_depth = rq_depth + ctxts; + newxprt->sc_sq_depth = rq_depth + + rdma_rw_max_send_wr(dev, newxprt->sc_port_num, ctxts, 0); if (newxprt->sc_sq_depth > dev->attrs.max_qp_wr) newxprt->sc_sq_depth = dev->attrs.max_qp_wr; atomic_set(&newxprt->sc_sq_avail, newxprt->sc_sq_depth); diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index d0511225799ba3..f6d56e70c7a2c3 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -1650,10 +1650,9 @@ static int unix_stream_connect(struct socket *sock, struct sockaddr_unsized *uad timeo = sock_sndtimeo(sk, flags & O_NONBLOCK); - /* First of all allocate resources. - * If we will make it after state is locked, - * we will have to recheck all again in any case. - */ + err = prepare_peercred(&peercred); + if (err) + goto out; /* create new sock for complete connection */ newsk = unix_create1(net, NULL, 0, sock->type); @@ -1662,10 +1661,6 @@ static int unix_stream_connect(struct socket *sock, struct sockaddr_unsized *uad goto out; } - err = prepare_peercred(&peercred); - if (err) - goto out; - /* Allocate skb for sending to listening sock */ skb = sock_wmalloc(newsk, 1, 0, GFP_KERNEL); if (!skb) { diff --git a/net/wireless/core.c b/net/wireless/core.c index 9a420d627d3ce0..a04f96dc9a1d73 100644 --- a/net/wireless/core.c +++ b/net/wireless/core.c @@ -1415,8 +1415,10 @@ void cfg80211_leave(struct cfg80211_registered_device *rdev, cfg80211_leave_ocb(rdev, dev); break; case NL80211_IFTYPE_P2P_DEVICE: + cfg80211_stop_p2p_device(rdev, wdev); + break; case NL80211_IFTYPE_NAN: - /* cannot happen, has no netdev */ + cfg80211_stop_nan(rdev, wdev); break; case NL80211_IFTYPE_AP_VLAN: case NL80211_IFTYPE_MONITOR: diff --git a/net/wireless/scan.c b/net/wireless/scan.c index 7546647752fd8f..eb0e77813d466f 100644 --- a/net/wireless/scan.c +++ b/net/wireless/scan.c @@ -1959,7 +1959,7 @@ cfg80211_update_known_bss(struct cfg80211_registered_device *rdev, ether_addr_copy(known->parent_bssid, new->parent_bssid); known->pub.max_bssid_indicator = new->pub.max_bssid_indicator; known->pub.bssid_index = new->pub.bssid_index; - known->pub.use_for &= new->pub.use_for; + known->pub.use_for = new->pub.use_for; known->pub.cannot_use_reasons = new->pub.cannot_use_reasons; known->bss_source = new->bss_source; diff --git a/rust/bindgen_parameters b/rust/bindgen_parameters index fd2fd1c3cb9a51..eeb8c15e709c38 100644 --- a/rust/bindgen_parameters +++ b/rust/bindgen_parameters @@ -12,9 +12,15 @@ # Packed type cannot transitively contain a `#[repr(align)]` type. --opaque-type alt_instr +--opaque-type snd_codec_options +--opaque-type snd_codec +--opaque-type snd_compr_params --opaque-type x86_msi_data --opaque-type x86_msi_addr_lo +# Packed types cannot have larger alignment than the maximal natural aligment of menbers +--opaque-type snd_dec_flac + # `try` is a reserved keyword since Rust 2018; solved in `bindgen` v0.59.2, # commit 2aed6b021680 ("context: Escape the try keyword properly"). --opaque-type kunit_try_catch diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index a067038b4b422b..025dae9fc05534 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -31,9 +31,14 @@ #include #include #include +#include #include #include +#include +#include #include +#include +#include #include #include #include @@ -48,9 +53,13 @@ #include #include #include +#include #include #include +#include +#include #include +#include #include #include #include @@ -59,13 +68,20 @@ #include #include #include +#include +#include +#include #include +#include #include #include #include #include #include +#include +#include #include +#include #include #include #include @@ -80,12 +96,17 @@ #include #include #include +#include +#include #include #include #include #include #include #include +#include +#include +#include #include /* @@ -111,8 +132,16 @@ const gfp_t RUST_CONST_HELPER___GFP_ZERO = __GFP_ZERO; const gfp_t RUST_CONST_HELPER___GFP_HIGHMEM = ___GFP_HIGHMEM; const gfp_t RUST_CONST_HELPER___GFP_NOWARN = ___GFP_NOWARN; const blk_features_t RUST_CONST_HELPER_BLK_FEAT_ROTATIONAL = BLK_FEAT_ROTATIONAL; +const uint32_t RUST_CONST_HELPER_DRM_EXEC_INTERRUPTIBLE_WAIT = DRM_EXEC_INTERRUPTIBLE_WAIT; const fop_flags_t RUST_CONST_HELPER_FOP_UNSIGNED_OFFSET = FOP_UNSIGNED_OFFSET; +const u64 BINDINGS_SNDRV_PCM_FMTBIT_FLOAT_LE = SNDRV_PCM_FMTBIT_FLOAT_LE; + +const u32 BINDINGS_IIO_CHAN_INFO_RAW = IIO_CHAN_INFO_RAW; +const u32 BINDINGS_IIO_CHAN_INFO_PROCESSED = IIO_CHAN_INFO_PROCESSED; +const u32 BINDINGS_IIO_ANGL = IIO_ANGL; +const u32 BINDINGS_IIO_LIGHT = IIO_LIGHT; + const xa_mark_t RUST_CONST_HELPER_XA_PRESENT = XA_PRESENT; const gfp_t RUST_CONST_HELPER_XA_FLAGS_ALLOC = XA_FLAGS_ALLOC; diff --git a/rust/helpers/device.c b/rust/helpers/device.c index 9a4316bafedfbc..41c3f4df8909e8 100644 --- a/rust/helpers/device.c +++ b/rust/helpers/device.c @@ -25,3 +25,13 @@ void rust_helper_dev_set_drvdata(struct device *dev, void *data) { dev_set_drvdata(dev, data); } + +void rust_helper_device_lock(struct device *dev) +{ + device_lock(dev); +} + +void rust_helper_device_unlock(struct device *dev) +{ + device_unlock(dev); +} diff --git a/rust/helpers/dma-fence.c b/rust/helpers/dma-fence.c new file mode 100644 index 00000000000000..6491016262934b --- /dev/null +++ b/rust/helpers/dma-fence.c @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#ifdef CONFIG_DMA_SHARED_BUFFER + +void rust_helper_dma_fence_get(struct dma_fence *fence) +{ + dma_fence_get(fence); +} + +void rust_helper_dma_fence_put(struct dma_fence *fence) +{ + dma_fence_put(fence); +} + +struct dma_fence_chain *rust_helper_dma_fence_chain_alloc(void) +{ + return dma_fence_chain_alloc(); +} + +void rust_helper_dma_fence_chain_free(struct dma_fence_chain *chain) +{ + dma_fence_chain_free(chain); +} + +void rust_helper_dma_fence_set_error(struct dma_fence *fence, int error) +{ + dma_fence_set_error(fence, error); +} + +#endif diff --git a/rust/helpers/dma-mapping.c b/rust/helpers/dma-mapping.c new file mode 100644 index 00000000000000..0d795b1b0738dc --- /dev/null +++ b/rust/helpers/dma-mapping.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +int rust_helper_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) +{ + return dma_mapping_error(dev, dma_addr); +} diff --git a/rust/helpers/dma-resv.c b/rust/helpers/dma-resv.c new file mode 100644 index 00000000000000..05501cb814513b --- /dev/null +++ b/rust/helpers/dma-resv.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +int rust_helper_dma_resv_lock(struct dma_resv *obj, struct ww_acquire_ctx *ctx) +{ + return dma_resv_lock(obj, ctx); +} + +void rust_helper_dma_resv_unlock(struct dma_resv *obj) +{ + dma_resv_unlock(obj); +} diff --git a/rust/helpers/drm.c b/rust/helpers/drm.c index 450b406c6f2739..a4e997d0b47320 100644 --- a/rust/helpers/drm.c +++ b/rust/helpers/drm.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include #include #ifdef CONFIG_DRM @@ -20,4 +21,49 @@ __u64 rust_helper_drm_vma_node_offset_addr(struct drm_vma_offset_node *node) return drm_vma_node_offset_addr(node); } -#endif +#ifdef CONFIG_DRM_GEM_SHMEM_HELPER +void rust_helper_drm_gem_shmem_object_free(struct drm_gem_object *obj) +{ + return drm_gem_shmem_object_free(obj); +} + +void rust_helper_drm_gem_shmem_object_print_info(struct drm_printer *p, unsigned int indent, + const struct drm_gem_object *obj) +{ + drm_gem_shmem_object_print_info(p, indent, obj); +} + +int rust_helper_drm_gem_shmem_object_pin(struct drm_gem_object *obj) +{ + return drm_gem_shmem_object_pin(obj); +} + +void rust_helper_drm_gem_shmem_object_unpin(struct drm_gem_object *obj) +{ + drm_gem_shmem_object_unpin(obj); +} + +struct sg_table *rust_helper_drm_gem_shmem_object_get_sg_table(struct drm_gem_object *obj) +{ + return drm_gem_shmem_object_get_sg_table(obj); +} + +int rust_helper_drm_gem_shmem_object_vmap(struct drm_gem_object *obj, + struct iosys_map *map) +{ + return drm_gem_shmem_object_vmap(obj, map); +} + +void rust_helper_drm_gem_shmem_object_vunmap(struct drm_gem_object *obj, + struct iosys_map *map) +{ + drm_gem_shmem_object_vunmap(obj, map); +} + +int rust_helper_drm_gem_shmem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) +{ + return drm_gem_shmem_object_mmap(obj, vma); +} + +#endif /* CONFIG_DRM_GEM_SHMEM_HELPER */ +#endif /* CONFIG_DRM */ diff --git a/rust/helpers/drm_gpuvm.c b/rust/helpers/drm_gpuvm.c new file mode 100644 index 00000000000000..1ed5b5841f68c2 --- /dev/null +++ b/rust/helpers/drm_gpuvm.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#ifdef CONFIG_DRM +#ifdef CONFIG_DRM_GPUVM + +struct drm_gpuvm *rust_helper_drm_gpuvm_get(struct drm_gpuvm *obj) +{ + return drm_gpuvm_get(obj); +} + +void rust_helper_drm_gpuvm_exec_unlock(struct drm_gpuvm_exec *vm_exec) +{ + return drm_gpuvm_exec_unlock(vm_exec); +} + +void rust_helper_drm_gpuva_init_from_op(struct drm_gpuva *va, struct drm_gpuva_op_map *op) +{ + drm_gpuva_init_from_op(va, op); +} + +struct drm_gpuvm_bo *rust_helper_drm_gpuvm_bo_get(struct drm_gpuvm_bo *vm_bo) +{ + return drm_gpuvm_bo_get(vm_bo); +} + +bool rust_helper_drm_gpuvm_immediate_mode(struct drm_gpuvm *gpuvm) +{ + return drm_gpuvm_immediate_mode(gpuvm); +} + +bool rust_helper_drm_gpuvm_is_extobj(struct drm_gpuvm *gpuvm, struct drm_gem_object *obj) +{ + return drm_gpuvm_is_extobj(gpuvm, obj); +} + +#endif +#endif diff --git a/rust/helpers/drm_syncobj.c b/rust/helpers/drm_syncobj.c new file mode 100644 index 00000000000000..9e14c989edfd72 --- /dev/null +++ b/rust/helpers/drm_syncobj.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#ifdef CONFIG_DRM + +void rust_helper_drm_syncobj_get(struct drm_syncobj *obj) +{ + drm_syncobj_get(obj); +} + +void rust_helper_drm_syncobj_put(struct drm_syncobj *obj) +{ + drm_syncobj_put(obj); +} + +struct dma_fence *rust_helper_drm_syncobj_fence_get(struct drm_syncobj *syncobj) +{ + return drm_syncobj_fence_get(syncobj); +} + +#endif diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 79c72762ad9c4b..b663807262416a 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -25,11 +25,17 @@ #include "cred.c" #include "device.c" #include "dma.c" +#include "dma-fence.c" +#include "dma-mapping.c" +#include "dma-resv.c" #include "drm.c" +#include "drm_gpuvm.c" +#include "drm_syncobj.c" #include "err.c" #include "irq.c" #include "fs.c" #include "io.c" +#include "iosys_map.c" #include "jump_label.c" #include "kunit.c" #include "maple_tree.c" diff --git a/rust/helpers/io.c b/rust/helpers/io.c index c475913c69e647..6b1b05ab977b0c 100644 --- a/rust/helpers/io.c +++ b/rust/helpers/io.c @@ -18,6 +18,16 @@ void rust_helper_iounmap(void __iomem *addr) iounmap(addr); } +void rust_helper_memcpy_fromio(void *to, const void __iomem *from, long count) +{ + memcpy_fromio(to, from, count); +} + +void rust_helper_memcpy_toio(void __iomem *to, const void *from, size_t count) +{ + memcpy_toio(to, from, count); +} + u8 rust_helper_readb(const void __iomem *addr) { return readb(addr); diff --git a/rust/helpers/iosys_map.c b/rust/helpers/iosys_map.c new file mode 100644 index 00000000000000..f99598367147ff --- /dev/null +++ b/rust/helpers/iosys_map.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +void rust_helper_iosys_map_memcpy_to(struct iosys_map *dst, size_t dst_offset, + const void *src, size_t len) +{ + iosys_map_memcpy_to(dst, dst_offset, src, len); +} + +void rust_helper_iosys_map_memcpy_from(void *dst, const struct iosys_map *src, + size_t src_offset, size_t len) +{ + iosys_map_memcpy_from(dst, src, src_offset, len); +} + +void rust_helper_iosys_map_memset(struct iosys_map *dst, size_t offset, + int value, size_t len) +{ + iosys_map_memset(dst, offset, value, len); +} diff --git a/rust/helpers/of.c b/rust/helpers/of.c index 86b51167c913f9..8c3958795357c0 100644 --- a/rust/helpers/of.c +++ b/rust/helpers/of.c @@ -1,8 +1,29 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include bool rust_helper_is_of_node(const struct fwnode_handle *fwnode) { return is_of_node(fwnode); } + +const struct of_device_id *rust_helper_of_match_device( + const struct of_device_id *matches, const struct device *dev) +{ + return of_match_device(matches, dev); +} + +#ifdef CONFIG_OF +bool rust_helper_of_node_is_root(const struct device_node *np) +{ + return of_node_is_root(np); +} +#endif + +struct device_node *rust_helper_of_parse_phandle(const struct device_node *np, + const char *phandle_name, + int index) +{ + return of_parse_phandle(np, phandle_name, index); +} diff --git a/rust/helpers/page.c b/rust/helpers/page.c index 7144de5a61dbdb..1e15bb4d525b59 100644 --- a/rust/helpers/page.c +++ b/rust/helpers/page.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include #include @@ -25,3 +26,28 @@ int rust_helper_page_to_nid(const struct page *page) return page_to_nid(page); } #endif + +struct page *rust_helper_phys_to_page(phys_addr_t phys) +{ + return phys_to_page(phys); +} + +phys_addr_t rust_helper_page_to_phys(struct page *page) +{ + return page_to_phys(page); +} + +unsigned long rust_helper_phys_to_pfn(phys_addr_t phys) +{ + return __phys_to_pfn(phys); +} + +struct page *rust_helper_pfn_to_page(unsigned long pfn) +{ + return pfn_to_page(pfn); +} + +bool rust_helper_pfn_valid(unsigned long pfn) +{ + return pfn_valid(pfn); +} diff --git a/rust/helpers/xarray.c b/rust/helpers/xarray.c index 60b299f11451d2..b6c078e6a343c2 100644 --- a/rust/helpers/xarray.c +++ b/rust/helpers/xarray.c @@ -2,6 +2,11 @@ #include +void *rust_helper_xa_zero_entry(void) +{ + return XA_ZERO_ENTRY; +} + int rust_helper_xa_err(void *entry) { return xa_err(entry); diff --git a/rust/kernel/addr.rs b/rust/kernel/addr.rs new file mode 100644 index 00000000000000..06aff10a033235 --- /dev/null +++ b/rust/kernel/addr.rs @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Kernel core address types. + +use bindings; +use core::ffi; + +/// A physical memory address (which may be wider than the CPU pointer size) +pub type PhysicalAddr = bindings::phys_addr_t; +/// A DMA memory address (which may be narrower than `PhysicalAddr` on some systems) +pub type DmaAddr = bindings::dma_addr_t; +/// A physical resource size, typically the same width as `PhysicalAddr` +pub type ResourceSize = bindings::resource_size_t; +/// A raw page frame number, not to be confused with the C `pfn_t` which also encodes flags. +pub type Pfn = ffi::c_ulong; diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index e38720349dcf7e..719927971f5796 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -18,6 +18,8 @@ pub use self::kvec::KVec; pub use self::kvec::VVec; pub use self::kvec::Vec; +use crate::types::declare_flags_type; + /// Indicates an allocation error. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub struct AllocError; @@ -25,45 +27,13 @@ pub struct AllocError; use crate::error::{code::EINVAL, Result}; use core::{alloc::Layout, ptr::NonNull}; -/// Flags to be used when allocating memory. -/// -/// They can be combined with the operators `|`, `&`, and `!`. -/// -/// Values can be used from the [`flags`] module. -#[derive(Clone, Copy, PartialEq)] -pub struct Flags(u32); - -impl Flags { - /// Get the raw representation of this flag. - pub(crate) fn as_raw(self) -> u32 { - self.0 - } - - /// Check whether `flags` is contained in `self`. - pub fn contains(self, flags: Flags) -> bool { - (self & flags) == flags - } -} - -impl core::ops::BitOr for Flags { - type Output = Self; - fn bitor(self, rhs: Self) -> Self::Output { - Self(self.0 | rhs.0) - } -} - -impl core::ops::BitAnd for Flags { - type Output = Self; - fn bitand(self, rhs: Self) -> Self::Output { - Self(self.0 & rhs.0) - } -} - -impl core::ops::Not for Flags { - type Output = Self; - fn not(self) -> Self::Output { - Self(!self.0) - } +declare_flags_type! { + /// Flags to be used when allocating memory. + /// + /// They can be combined with the operators `|`, `&`, and `!`. + /// + /// Values can be used from the [`flags`] module. + pub struct Flags(u32); } /// Allocation flags. diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index 63bfb91b36712a..1a53412c458dd3 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -1,4 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +// FIXME +#![allow(clippy::undocumented_unsafe_blocks)] //! Allocator support. //! diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs index 622b3529edfcbc..2684598cde4cbe 100644 --- a/rust/kernel/alloc/kbox.rs +++ b/rust/kernel/alloc/kbox.rs @@ -682,6 +682,16 @@ where } } +impl AsRef for Box +where + T: ?Sized, + A: Allocator, +{ + fn as_ref(&self) -> &T { + self + } +} + /// # Examples /// /// ``` diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs index ac8d6f763ae81d..6cc48330515dd1 100644 --- a/rust/kernel/alloc/kvec.rs +++ b/rust/kernel/alloc/kvec.rs @@ -19,12 +19,15 @@ use core::{ ops::DerefMut, ops::Index, ops::IndexMut, + ops::{Range, RangeBounds}, ptr, ptr::NonNull, slice, slice::SliceIndex, }; +mod drain; +use self::drain::Drain; mod errors; pub use self::errors::{InsertError, PushError, RemoveError}; @@ -192,6 +195,19 @@ where self.len } + /// Forcefully sets `self.len` to `new_len`. + /// + /// # Safety + /// + /// - `new_len` must be less than or equal to [`Self::capacity`]. + /// - If `new_len` is greater than `self.len`, all elements within the interval + /// [`self.len`,`new_len`) must be initialized. + #[inline] + pub unsafe fn set_len(&mut self, new_len: usize) { + debug_assert!(new_len <= self.capacity()); + self.len = new_len; + } + /// Increments `self.len` by `additional`. /// /// # Safety @@ -733,6 +749,76 @@ where } self.truncate(num_kept); } + + /// Removes the specified range from the vector in bulk, returning all + /// removed elements as an iterator. If the iterator is dropped before + /// being fully consumed, it drops the remaining removed elements. + /// + /// The returned iterator keeps a mutable borrow on the vector to optimize + /// its implementation. + /// + /// # Panics + /// + /// Panics if the starting point is greater than the end point or if + /// the end point is greater than the length of the vector. + /// + /// # Leaking + /// + /// If the returned iterator goes out of scope without being dropped (due to + /// [`mem::forget`], for example), the vector may have lost and leaked + /// elements arbitrarily, including elements outside the range. + /// + /// # Examples + /// + /// ``` + /// let mut v = vec![1, 2, 3]; + /// let u: Vec<_> = v.drain(1..).collect(); + /// assert_eq!(v, &[1]); + /// assert_eq!(u, &[2, 3]); + /// + /// // A full range clears the vector, like `clear()` does + /// v.drain(..); + /// assert_eq!(v, &[]); + /// ``` + pub fn drain(&mut self, range: R) -> Drain<'_, T, A> + where + R: RangeBounds, + { + let len = self.len(); + let Range { start, end } = slice::range(range, ..len); + + unsafe { + // set self.vec length's to start, to be safe in case Drain is leaked + self.set_len(start); + let range_slice = slice::from_raw_parts(self.as_ptr().add(start), end - start); + Drain { + tail_start: end, + tail_len: len - end, + iter: range_slice.iter(), + vec: NonNull::from(self), + } + } + } + /// Removes an element from the vector and returns it. + /// + /// The removed element is replaced by the last element of the vector. + /// + /// This does not preserve ordering of the remaining elements, but is *O*(1). + /// If you need to preserve the element order, use [`remove`] instead. + pub fn swap_remove(&mut self, index: usize) -> T { + if index > self.len() { + panic!("Index out of range"); + } + // SAFETY: index is in range + // self.len() - 1 is in range since at last 1 element exists + unsafe { + let old = ptr::read(self.as_ptr().add(index)); + let last = ptr::read(self.as_ptr().add(self.len() - 1)); + ptr::write(self.as_mut_ptr().add(index), last); + self.dec_len(1); + old + } + } } impl Vec { @@ -1399,3 +1485,51 @@ mod tests { } } } + +// #[stable(feature = "array_try_from_vec", since = "1.48.0")] +impl TryFrom> for [T; N] { + type Error = Vec; + + /// Gets the entire contents of the `Vec` as an array, + /// if its size exactly matches that of the requested array. + /// + /// # Examples + /// + /// ``` + /// assert_eq!(vec![1, 2, 3].try_into(), Ok([1, 2, 3])); + /// assert_eq!(>::new().try_into(), Ok([])); + /// ``` + /// + /// If the length doesn't match, the input comes back in `Err`: + /// ``` + /// let r: Result<[i32; 4], _> = (0..10).collect::>().try_into(); + /// assert_eq!(r, Err(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9])); + /// ``` + /// + /// If you're fine with just getting a prefix of the `Vec`, + /// you can call [`.truncate(N)`](Vec::truncate) first. + /// ``` + /// let mut v = String::from("hello world").into_bytes(); + /// v.sort(); + /// v.truncate(2); + /// let [a, b]: [_; 2] = v.try_into().unwrap(); + /// assert_eq!(a, b' '); + /// assert_eq!(b, b'd'); + /// ``` + fn try_from(mut vec: Vec) -> Result<[T; N], Vec> { + if vec.len() != N { + return Err(vec); + } + + // SAFETY: `.set_len(0)` is always sound. + unsafe { vec.dec_len(vec.len()) }; + + // SAFETY: A `Vec`'s pointer is always aligned properly, and + // the alignment the array needs is the same as the items. + // We checked earlier that we have sufficient items. + // The items will not double-drop as the `set_len` + // tells the `Vec` not to also drop them. + let array = unsafe { ptr::read(vec.as_ptr() as *const [T; N]) }; + Ok(array) + } +} diff --git a/rust/kernel/alloc/kvec/drain.rs b/rust/kernel/alloc/kvec/drain.rs new file mode 100644 index 00000000000000..035878fd112843 --- /dev/null +++ b/rust/kernel/alloc/kvec/drain.rs @@ -0,0 +1,181 @@ +//! Rust standard library vendored code. +//! +//! The contents of this file come from the Rust standard library, hosted in +//! the repository, licensed under +//! "Apache-2.0 OR MIT" and adapted for kernel use. For copyright details, +//! see . +#![allow(clippy::undocumented_unsafe_blocks)] + +use core::fmt; +use core::iter::FusedIterator; +use core::mem::{self, SizedTypeProperties}; +use core::ptr::{self, NonNull}; +use core::slice::{self}; + +use super::{Allocator, Vec}; + +/// A draining iterator for `Vec`. +/// +/// This `struct` is created by [`Vec::drain`]. +/// See its documentation for more. +/// +/// # Example +/// +/// ``` +/// let mut v = vec![0, 1, 2]; +/// let iter: std::vec::Drain<'_, _> = v.drain(..); +/// ``` +// #[stable(feature = "drain", since = "1.6.0")] +pub struct Drain< + 'a, + T, + A: Allocator, + // #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator + 'a = Global, +> { + /// Index of tail to preserve + pub(super) tail_start: usize, + /// Length of tail + pub(super) tail_len: usize, + /// Current remaining range to remove + pub(super) iter: slice::Iter<'a, T>, + pub(super) vec: NonNull>, +} + +// #[stable(feature = "collection_debug", since = "1.17.0")] +impl fmt::Debug for Drain<'_, T, A> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("Drain").field(&self.iter.as_slice()).finish() + } +} + +impl<'a, T, A: Allocator> Drain<'a, T, A> { + /// Returns the remaining items of this iterator as a slice. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec!['a', 'b', 'c']; + /// let mut drain = vec.drain(..); + /// assert_eq!(drain.as_slice(), &['a', 'b', 'c']); + /// let _ = drain.next().unwrap(); + /// assert_eq!(drain.as_slice(), &['b', 'c']); + /// ``` + #[must_use] + // #[stable(feature = "vec_drain_as_slice", since = "1.46.0")] + pub fn as_slice(&self) -> &[T] { + self.iter.as_slice() + } +} + +// #[stable(feature = "vec_drain_as_slice", since = "1.46.0")] +impl<'a, T, A: Allocator> AsRef<[T]> for Drain<'a, T, A> { + fn as_ref(&self) -> &[T] { + self.as_slice() + } +} + +// #[stable(feature = "drain", since = "1.6.0")] +unsafe impl Sync for Drain<'_, T, A> {} +// #[stable(feature = "drain", since = "1.6.0")] +unsafe impl Send for Drain<'_, T, A> {} + +// #[stable(feature = "drain", since = "1.6.0")] +impl Iterator for Drain<'_, T, A> { + type Item = T; + + #[inline] + fn next(&mut self) -> Option { + self.iter + .next() + .map(|elt| unsafe { ptr::read(elt as *const _) }) + } + + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } +} + +// #[stable(feature = "drain", since = "1.6.0")] +impl DoubleEndedIterator for Drain<'_, T, A> { + #[inline] + fn next_back(&mut self) -> Option { + self.iter + .next_back() + .map(|elt| unsafe { ptr::read(elt as *const _) }) + } +} + +// #[stable(feature = "drain", since = "1.6.0")] +impl Drop for Drain<'_, T, A> { + fn drop(&mut self) { + /// Moves back the un-`Drain`ed elements to restore the original `Vec`. + struct DropGuard<'r, 'a, T, A: Allocator>(&'r mut Drain<'a, T, A>); + + impl<'r, 'a, T, A: Allocator> Drop for DropGuard<'r, 'a, T, A> { + fn drop(&mut self) { + if self.0.tail_len > 0 { + unsafe { + let source_vec = self.0.vec.as_mut(); + // memmove back untouched tail, update to new length + let start = source_vec.len(); + let tail = self.0.tail_start; + if tail != start { + let src = source_vec.as_ptr().add(tail); + let dst = source_vec.as_mut_ptr().add(start); + ptr::copy(src, dst, self.0.tail_len); + } + source_vec.set_len(start + self.0.tail_len); + } + } + } + } + + let iter = mem::take(&mut self.iter); + let drop_len = iter.len(); + + let mut vec = self.vec; + + if T::IS_ZST { + // ZSTs have no identity, so we don't need to move them around, we only need to drop the correct amount. + // this can be achieved by manipulating the Vec length instead of moving values out from `iter`. + unsafe { + let vec = vec.as_mut(); + let old_len = vec.len(); + vec.set_len(old_len + drop_len + self.tail_len); + vec.truncate(old_len + self.tail_len); + } + + return; + } + + // ensure elements are moved back into their appropriate places, even when drop_in_place panics + let _guard = DropGuard(self); + + if drop_len == 0 { + return; + } + + // as_slice() must only be called when iter.len() is > 0 because + // it also gets touched by vec::Splice which may turn it into a dangling pointer + // which would make it and the vec pointer point to different allocations which would + // lead to invalid pointer arithmetic below. + let drop_ptr = iter.as_slice().as_ptr(); + + unsafe { + // drop_ptr comes from a slice::Iter which only gives us a &[T] but for drop_in_place + // a pointer with mutable provenance is necessary. Therefore we must reconstruct + // it from the original vec but also avoid creating a &mut to the front since that could + // invalidate raw pointers to it which some unsafe code might rely on. + let vec_ptr = vec.as_mut().as_mut_ptr(); + #[cfg(not(version("1.87")))] + let drop_offset = drop_ptr.sub_ptr(vec_ptr); + #[cfg(version("1.87"))] + let drop_offset = drop_ptr.offset_from_unsigned(vec_ptr); + let to_drop = ptr::slice_from_raw_parts_mut(vec_ptr.add(drop_offset), drop_len); + ptr::drop_in_place(to_drop); + } + } +} + +// #[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for Drain<'_, T, A> {} diff --git a/rust/kernel/devcoredump.rs b/rust/kernel/devcoredump.rs new file mode 100644 index 00000000000000..a4a42d862f63b5 --- /dev/null +++ b/rust/kernel/devcoredump.rs @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Device coredump support. +//! +//! C header: [`include/linux/devcoredump.h`](../../../../include/linux/devcoredump.h) + +use crate::{ + alloc, bindings, device, error::from_result, prelude::Result, time::Jiffies, + types::ForeignOwnable, ThisModule, +}; + +use core::ops::Deref; + +/// The default timeout for device coredumps. +pub const DEFAULT_TIMEOUT: Jiffies = bindings::DEVCD_TIMEOUT as Jiffies; + +/// Trait to implement reading from a device coredump. +/// +/// Users must implement this trait to provide device coredump support. +pub trait DevCoreDump { + /// Returns the IOVA (virtual address) of the buffer from RTKit's point of view, or an error if + /// unavailable. + fn read(&self, buf: &mut [u8], offset: usize) -> Result; +} + +unsafe extern "C" fn read_callback< + 'a, + T: ForeignOwnable: Deref>, + D: DevCoreDump, +>( + buffer: *mut crate::ffi::c_char, + offset: bindings::loff_t, + count: usize, + data: *mut crate::ffi::c_void, + _datalen: usize, +) -> isize { + // SAFETY: This pointer came from into_foreign() below. + let coredump = unsafe { T::borrow(data.cast()) }; + // SAFETY: The caller guarantees `buffer` points to at least `count` bytes. + let buf = unsafe { core::slice::from_raw_parts_mut(buffer, count) }; + + from_result(|| Ok(coredump.read(buf, offset.try_into()?)?.try_into()?)) +} + +unsafe extern "C" fn free_callback< + 'a, + T: ForeignOwnable: Deref>, + D: DevCoreDump, +>( + data: *mut crate::ffi::c_void, +) { + // SAFETY: This pointer came from into_foreign() below. + unsafe { + T::from_foreign(data.cast()); + } +} + +/// Registers a coredump for the given device. +pub fn dev_coredump<'a, T: ForeignOwnable: Deref>, D: DevCoreDump>( + dev: &device::Device, + module: &'static ThisModule, + coredump: T, + gfp: alloc::Flags, + timeout: Jiffies, +) { + // SAFETY: Call upholds dev_coredumpm lifetime requirements. + unsafe { + bindings::dev_coredumpm_timeout( + dev.as_raw(), + module.0, + coredump.into_foreign() as *mut _, + 0, + gfp.as_raw(), + Some(read_callback::<'a, T, D>), + Some(free_callback::<'a, T, D>), + timeout, + ) + } +} diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index 031720bf5d8ca2..e49eac46891a7c 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -4,17 +4,28 @@ //! //! C header: [`include/linux/device.h`](srctree/include/linux/device.h) +use crate::of; use crate::{ - bindings, fmt, + bindings, + fmt, prelude::*, sync::aref::ARef, - types::{ForeignOwnable, Opaque}, + types::{ + ForeignOwnable, + Opaque, // + }, // +}; +use core::{ + any::TypeId, + marker::PhantomData, + ptr, // }; -use core::{any::TypeId, marker::PhantomData, ptr}; #[cfg(CONFIG_PRINTK)] use crate::c_str; +use crate::types::NotThreadSafe; + pub mod property; // Assert that we can `read()` / `write()` a `TypeId` instance from / into `struct driver_type`. @@ -330,13 +341,13 @@ impl Device { impl Device { /// Obtain the raw `struct device *`. - pub(crate) fn as_raw(&self) -> *mut bindings::device { + pub fn as_raw(&self) -> *mut bindings::device { self.0.get() } /// Returns a reference to the parent device, if any. #[cfg_attr(not(CONFIG_AUXILIARY_BUS), expect(dead_code))] - pub(crate) fn parent(&self) -> Option<&Device> { + pub fn parent(&self) -> Option<&Device> { // SAFETY: // - By the type invariant `self.as_raw()` is always valid. // - The parent device is only ever set at device creation. @@ -353,6 +364,13 @@ impl Device { } } + /// Returns the driver_data pointer. + pub fn get_drvdata(&self) -> *mut T { + // SAFETY: dev_get_drvdata returns a field of the device, + // pointer to which is valid by type invariant + unsafe { bindings::dev_get_drvdata(self.as_raw()) as *mut T } + } + /// Convert a raw C `struct device` pointer to a `&'a Device`. /// /// # Safety @@ -366,6 +384,13 @@ impl Device { unsafe { &*ptr.cast() } } + /// Gets the OpenFirmware node attached to this device + pub fn of_node(&self) -> Option { + let ptr = self.0.get(); + // SAFETY: This is safe as long as of_node is NULL or valid. + unsafe { of::Node::get_from_raw((*ptr).of_node) } + } + /// Prints an emergency-level message (level 0) prefixed with device information. /// /// More details are available from [`dev_emerg`]. @@ -484,6 +509,58 @@ impl Device { // defined as a `#[repr(transparent)]` wrapper around `fwnode_handle`. Some(unsafe { &*fwnode_handle.cast() }) } + + /// Locks the [`Device`] for exclusive access. + pub fn lock(&self) -> Guard<'_, Ctx> { + // SAFETY: `self` is always valid by the type invariant. + unsafe { bindings::device_lock(self.as_raw()) }; + + Guard { + dev: self, + _not_send: NotThreadSafe, + } + } + + /// ensure Device is bound + pub fn is_bound(&self) -> Option> { + let guard = self.lock(); + if !unsafe { bindings::device_is_bound(self.as_raw()) } { + return None; + } + Some(guard) + } + + /// excute closure while the device is bound + pub fn while_bound_with(&self, f: F) -> Result + where + F: FnOnce(&Device) -> Result, + { + let _guard = self.lock(); + if unsafe { !bindings::device_is_bound(self.as_raw()) } { + return Err(ENODEV); + } + let ptr: *const Self = self; + let ptr = ptr.cast::>(); + f(unsafe { &*ptr }) + } +} + +/// A lock guard. +/// +/// The lock is unlocked when the guard goes out of scope. +#[must_use = "the lock unlocks immediately when the guard is unused"] +pub struct Guard<'a, Ctx: DeviceContext = Normal> { + dev: &'a Device, + _not_send: NotThreadSafe, +} + +impl Drop for Guard<'_, Ctx> { + fn drop(&mut self) { + // SAFETY: + // - `self.xa.xa` is always valid by the type invariant. + // - The caller holds the lock, so it is safe to unlock it. + unsafe { bindings::device_unlock(self.dev.as_raw()) }; + } } // SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic @@ -596,6 +673,13 @@ impl DeviceContext for Core {} impl DeviceContext for CoreInternal {} impl DeviceContext for Normal {} +impl AsRef> for Device { + #[inline] + fn as_ref(&self) -> &Device { + self + } +} + /// Convert device references to bus device references. /// /// Bus devices can implement this trait to allow abstractions to provide the bus device in @@ -715,7 +799,7 @@ macro_rules! impl_device_context_into_aref { macro_rules! dev_printk { ($method:ident, $dev:expr, $($f:tt)*) => { { - ($dev).$method($crate::prelude::fmt!($($f)*)); + $crate::device::Device::$method($dev.as_ref(), $crate::prelude::fmt!($($f)*)) } } } diff --git a/rust/kernel/device/property.rs b/rust/kernel/device/property.rs index 3a332a8c53a9eb..71bbce339949ee 100644 --- a/rust/kernel/device/property.rs +++ b/rust/kernel/device/property.rs @@ -58,7 +58,7 @@ impl FwNode { } /// Obtain the raw `struct fwnode_handle *`. - pub(crate) fn as_raw(&self) -> *mut bindings::fwnode_handle { + pub fn as_raw(&self) -> *mut bindings::fwnode_handle { self.0.get() } diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs index 835d9c11948e7d..a9917ce71ded48 100644 --- a/rust/kernel/devres.rs +++ b/rust/kernel/devres.rs @@ -8,30 +8,24 @@ use crate::{ alloc::Flags, bindings, - device::{Bound, Device}, - error::{to_result, Error, Result}, - ffi::c_void, + device::{ + Bound, + Device, // + }, + error::to_result, prelude::*, - revocable::{Revocable, RevocableGuard}, - sync::{aref::ARef, rcu, Completion}, - types::{ForeignOwnable, Opaque, ScopeGuard}, + revocable::{ + Revocable, + RevocableGuard, // + }, + sync::{ + aref::ARef, + rcu, + Arc, // + }, + types::ForeignOwnable, }; -use pin_init::Wrapper; - -/// [`Devres`] inner data accessed from [`Devres::callback`]. -#[pin_data] -struct Inner { - #[pin] - data: Revocable, - /// Tracks whether [`Devres::callback`] has been completed. - #[pin] - devm: Completion, - /// Tracks whether revoking [`Self::data`] has been completed. - #[pin] - revoke: Completion, -} - /// This abstraction is meant to be used by subsystems to containerize [`Device`] bound resources to /// manage their lifetime. /// @@ -105,18 +99,13 @@ struct Inner { /// # fn no_run(dev: &Device) -> Result<(), Error> { /// // SAFETY: Invalid usage for example purposes. /// let iomem = unsafe { IoMem::<{ core::mem::size_of::() }>::new(0xBAAAAAAD)? }; -/// let devres = KBox::pin_init(Devres::new(dev, iomem), GFP_KERNEL)?; +/// let devres = Devres::new(dev, iomem)?; /// /// let res = devres.try_access().ok_or(ENXIO)?; /// res.write8(0x42, 0x0); /// # Ok(()) /// # } /// ``` -/// -/// # Invariants -/// -/// `Self::inner` is guaranteed to be initialized and is always accessed read-only. -#[pin_data(PinnedDrop)] pub struct Devres { dev: ARef, /// Pointer to [`Self::devres_callback`]. @@ -124,14 +113,7 @@ pub struct Devres { /// Has to be stored, since Rust does not guarantee to always return the same address for a /// function. However, the C API uses the address as a key. callback: unsafe extern "C" fn(*mut c_void), - /// Contains all the fields shared with [`Self::callback`]. - // TODO: Replace with `UnsafePinned`, once available. - // - // Subsequently, the `drop_in_place()` in `Devres::drop` and `Devres::new` as well as the - // explicit `Send` and `Sync' impls can be removed. - #[pin] - inner: Opaque>, - _add_action: (), + data: Arc>, } impl Devres { @@ -139,74 +121,48 @@ impl Devres { /// /// The `data` encapsulated within the returned `Devres` instance' `data` will be /// (revoked)[`Revocable`] once the device is detached. - pub fn new<'a, E>( - dev: &'a Device, - data: impl PinInit + 'a, - ) -> impl PinInit + 'a + pub fn new(dev: &Device, data: impl PinInit) -> Result where - T: 'a, Error: From, { - try_pin_init!(&this in Self { - dev: dev.into(), - callback: Self::devres_callback, - // INVARIANT: `inner` is properly initialized. - inner <- Opaque::pin_init(try_pin_init!(Inner { - devm <- Completion::new(), - revoke <- Completion::new(), - data <- Revocable::new(data), - })), - // TODO: Replace with "initializer code blocks" [1] once available. - // - // [1] https://github.com/Rust-for-Linux/pin-init/pull/69 - _add_action: { - // SAFETY: `this` is a valid pointer to uninitialized memory. - let inner = unsafe { &raw mut (*this.as_ptr()).inner }; + let callback = Self::devres_callback; + let data = Arc::pin_init(Revocable::new(data), GFP_KERNEL)?; + let devres_data = data.clone(); + + // SAFETY: + // - `dev.as_raw()` is a pointer to a valid bound device. + // - `data` is guaranteed to be a valid for the duration of the lifetime of `Self`. + // - `devm_add_action()` is guaranteed not to call `callback` for the entire lifetime of + // `dev`. + to_result(unsafe { + bindings::devm_add_action( + dev.as_raw(), + Some(callback), + Arc::as_ptr(&data).cast_mut().cast(), + ) + })?; - // SAFETY: - // - `dev.as_raw()` is a pointer to a valid bound device. - // - `inner` is guaranteed to be a valid for the duration of the lifetime of `Self`. - // - `devm_add_action()` is guaranteed not to call `callback` until `this` has been - // properly initialized, because we require `dev` (i.e. the *bound* device) to - // live at least as long as the returned `impl PinInit`. - to_result(unsafe { - bindings::devm_add_action(dev.as_raw(), Some(*callback), inner.cast()) - }).inspect_err(|_| { - let inner = Opaque::cast_into(inner); + // `devm_add_action()` was successful and has consumed the reference count. + core::mem::forget(devres_data); - // SAFETY: `inner` is a valid pointer to an `Inner` and valid for both reads - // and writes. - unsafe { core::ptr::drop_in_place(inner) }; - })?; - }, + Ok(Self { + dev: dev.into(), + callback, + data, }) } - fn inner(&self) -> &Inner { - // SAFETY: By the type invairants of `Self`, `inner` is properly initialized and always - // accessed read-only. - unsafe { &*self.inner.get() } - } - fn data(&self) -> &Revocable { - &self.inner().data + &self.data } #[allow(clippy::missing_safety_doc)] unsafe extern "C" fn devres_callback(ptr: *mut kernel::ffi::c_void) { - // SAFETY: In `Self::new` we've passed a valid pointer to `Inner` to `devm_add_action()`, - // hence `ptr` must be a valid pointer to `Inner`. - let inner = unsafe { &*ptr.cast::>() }; + // SAFETY: In `Self::new` we've passed a valid pointer of `Revocable` to + // `devm_add_action()`, hence `ptr` must be a valid pointer to `Revocable`. + let data = unsafe { Arc::from_raw(ptr.cast::>()) }; - // Ensure that `inner` can't be used anymore after we signal completion of this callback. - let inner = ScopeGuard::new_with_data(inner, |inner| inner.devm.complete_all()); - - if !inner.data.revoke() { - // If `revoke()` returns false, it means that `Devres::drop` already started revoking - // `data` for us. Hence we have to wait until `Devres::drop` signals that it - // completed revoking `data`. - inner.revoke.wait_for_completion(); - } + data.revoke(); } fn remove_action(&self) -> bool { @@ -218,7 +174,7 @@ impl Devres { bindings::devm_remove_action_nowarn( self.dev.as_raw(), Some(self.callback), - core::ptr::from_ref(self.inner()).cast_mut().cast(), + core::ptr::from_ref(self.data()).cast_mut().cast(), ) } == 0) } @@ -289,31 +245,19 @@ unsafe impl Send for Devres {} // SAFETY: `Devres` can be shared with any task, if `T: Sync`. unsafe impl Sync for Devres {} -#[pinned_drop] -impl PinnedDrop for Devres { - fn drop(self: Pin<&mut Self>) { +impl Drop for Devres { + fn drop(&mut self) { // SAFETY: When `drop` runs, it is guaranteed that nobody is accessing the revocable data // anymore, hence it is safe not to wait for the grace period to finish. if unsafe { self.data().revoke_nosync() } { // We revoked `self.data` before the devres action did, hence try to remove it. - if !self.remove_action() { - // We could not remove the devres action, which means that it now runs concurrently, - // hence signal that `self.data` has been revoked by us successfully. - self.inner().revoke.complete_all(); - - // Wait for `Self::devres_callback` to be done using this object. - self.inner().devm.wait_for_completion(); + if self.remove_action() { + // SAFETY: In `Self::new` we have taken an additional reference count of `self.data` + // for `devm_add_action()`. Since `remove_action()` was successful, we have to drop + // this additional reference count. + drop(unsafe { Arc::from_raw(Arc::as_ptr(&self.data)) }); } - } else { - // `Self::devres_callback` revokes `self.data` for us, hence wait for it to be done - // using this object. - self.inner().devm.wait_for_completion(); } - - // INVARIANT: At this point it is guaranteed that `inner` can't be accessed any more. - // - // SAFETY: `inner` is valid for dropping. - unsafe { core::ptr::drop_in_place(self.inner.get()) }; } } diff --git a/rust/kernel/dma_buf.rs b/rust/kernel/dma_buf.rs new file mode 100644 index 00000000000000..318518ff0b28f9 --- /dev/null +++ b/rust/kernel/dma_buf.rs @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! DMA buffer API +//! +//! C header: [`include/linux/dma-buf.h`](srctree/include/linux/dma-buf.h) + +use bindings; +use kernel::types::*; + +/// A DMA buffer object. +/// +/// # Invariants +/// +/// The data layout of this type is equivalent to that of `struct dma_buf`. +#[repr(transparent)] +pub struct DmaBuf(Opaque); + +// SAFETY: `struct dma_buf` is thread-safe +unsafe impl Send for DmaBuf {} +// SAFETY: `struct dma_buf` is thread-safe +unsafe impl Sync for DmaBuf {} + +impl DmaBuf { + /// Convert from a `*mut bindings::dma_buf` to a [`DmaBuf`]. + /// + /// # Safety + /// + /// The caller guarantees that `self_ptr` points to a valid initialized `struct dma_buf` for the + /// duration of the lifetime of `'a`, and promises to not violate rust's data aliasing rules + /// using the reference provided by this function. + pub(crate) unsafe fn as_ref<'a>(self_ptr: *mut bindings::dma_buf) -> &'a Self { + // SAFETY: Our data layout is equivalent to `dma_buf` . + unsafe { &*self_ptr.cast() } + } + + pub(crate) fn as_raw(&self) -> *mut bindings::dma_buf { + self.0.get() + } +} diff --git a/rust/kernel/dma_fence.rs b/rust/kernel/dma_fence.rs new file mode 100644 index 00000000000000..139451e75685f9 --- /dev/null +++ b/rust/kernel/dma_fence.rs @@ -0,0 +1,480 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! DMA fence abstraction. +//! +//! C header: [`include/linux/dma_fence.h`](../../include/linux/dma_fence.h) + +use crate::{ + bindings, + error::{to_result, Result}, + prelude::*, + sync::LockClassKey, + types::Opaque, +}; +use core::ops::{Deref, DerefMut}; +use core::ptr::addr_of_mut; +use core::sync::atomic::{AtomicU64, Ordering}; + +mod private { + /// Marker that a trait cannot be implemented outside of this mod + pub trait Sealed {} +} + +/// Any kind of DMA Fence Object +/// +/// # Invariants +/// raw() returns a valid pointer to a dma_fence and we own a reference to it. +pub trait RawDmaFence: private::Sealed { + /// Returns the raw `struct dma_fence` pointer. + fn raw(&self) -> *mut bindings::dma_fence; + + /// Returns the raw `struct dma_fence` pointer and consumes the object. + /// + /// The caller is responsible for dropping the reference. + fn into_raw(self) -> *mut bindings::dma_fence + where + Self: Sized, + { + let ptr = self.raw(); + core::mem::forget(self); + ptr + } + + /// Advances this fence to the chain node which will signal this sequence number. + /// If no sequence number is provided, this returns `self` again. + /// If the seqno has already been signaled, returns None. + fn chain_find_seqno(self, seqno: u64) -> Result> + where + Self: Sized, + { + let mut ptr = self.into_raw(); + + // SAFETY: This will safely fail if this DmaFence is not a chain. + // `ptr` is valid per the type invariant. + let ret = unsafe { bindings::dma_fence_chain_find_seqno(&mut ptr, seqno) }; + + if ret != 0 { + // SAFETY: This is either an owned reference or NULL, dma_fence_put can handle both. + unsafe { bindings::dma_fence_put(ptr) }; + Err(Error::from_errno(ret)) + } else if ptr.is_null() { + Ok(None) + } else { + // SAFETY: ptr is valid and non-NULL as checked above. + Ok(Some(unsafe { Fence::from_raw(ptr) })) + } + } + + /// Signal completion of this fence + fn signal(&self) -> Result { + // SAFETY: Safe to call on any valid dma_fence object + to_result(unsafe { bindings::dma_fence_signal(self.raw()) }) + } + + /// Set the error flag on this fence + fn set_error(&self, err: Error) { + // SAFETY: Safe to call on any valid dma_fence object + unsafe { bindings::dma_fence_set_error(self.raw(), err.to_errno()) }; + } +} + +/// A generic DMA Fence Object +/// +/// # Invariants +/// ptr is a valid pointer to a dma_fence and we own a reference to it. +pub struct Fence { + ptr: *mut bindings::dma_fence, +} + +impl Fence { + /// Create a new Fence object from a raw pointer to a dma_fence. + /// + /// # Safety + /// The caller must own a reference to the dma_fence, which is transferred to the new object. + pub(crate) unsafe fn from_raw(ptr: *mut bindings::dma_fence) -> Fence { + Fence { ptr } + } + + /// Create a new Fence object from a raw pointer to a dma_fence. + /// + /// # Safety + /// Takes a borrowed reference to the dma_fence, and increments the reference count. + pub(crate) unsafe fn get_raw(ptr: *mut bindings::dma_fence) -> Fence { + // SAFETY: Pointer is valid per the safety contract + unsafe { bindings::dma_fence_get(ptr) }; + Fence { ptr } + } + + /// Create a new Fence object from a RawDmaFence. + pub fn from_fence(fence: &dyn RawDmaFence) -> Fence { + // SAFETY: Pointer is valid per the RawDmaFence contract + unsafe { Self::get_raw(fence.raw()) } + } +} + +impl private::Sealed for Fence {} + +impl RawDmaFence for Fence { + fn raw(&self) -> *mut bindings::dma_fence { + self.ptr + } +} + +impl Drop for Fence { + fn drop(&mut self) { + // SAFETY: We own a reference to this syncobj. + unsafe { bindings::dma_fence_put(self.ptr) }; + } +} + +impl Clone for Fence { + fn clone(&self) -> Self { + // SAFETY: `ptr` is valid per the type invariant and we own a reference to it. + unsafe { + bindings::dma_fence_get(self.ptr); + Self::from_raw(self.ptr) + } + } +} + +// SAFETY: The API for these objects is thread safe +unsafe impl Sync for Fence {} +// SAFETY: The API for these objects is thread safe +unsafe impl Send for Fence {} + +/// Trait which must be implemented by driver-specific fence objects. +#[vtable] +pub trait FenceOps: Sized + Send + Sync { + /// Returns the driver name. This is a callback to allow drivers to compute the name at + /// runtime, without having it to store permanently for each fence, or build a cache of + /// some sort. + fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr; + + /// Return the name of the context this fence belongs to. This is a callback to allow drivers + /// to compute the name at runtime, without having it to store permanently for each fence, or + /// build a cache of some sort. + fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr; + + /// Enable software signaling of fence. + fn enable_signaling(self: &FenceObject) -> bool { + false + } + + /// Peek whether the fence is signaled, as a fastpath optimization for e.g. dma_fence_wait() or + /// dma_fence_add_callback(). + fn signaled(self: &FenceObject) -> bool { + false + } +} + +unsafe extern "C" fn get_driver_name_cb( + fence: *mut bindings::dma_fence, +) -> *const crate::ffi::c_char { + // SAFETY: All of our fences are FenceObject. + let p = unsafe { crate::container_of!(fence, FenceObject, fence) as *mut FenceObject }; + + // SAFETY: The caller is responsible for passing a valid dma_fence subtype + T::get_driver_name(unsafe { &mut *p }).as_char_ptr() +} + +unsafe extern "C" fn get_timeline_name_cb( + fence: *mut bindings::dma_fence, +) -> *const crate::ffi::c_char { + // SAFETY: All of our fences are FenceObject. + let p = unsafe { crate::container_of!(fence, FenceObject, fence) as *mut FenceObject }; + + // SAFETY: The caller is responsible for passing a valid dma_fence subtype + T::get_timeline_name(unsafe { &mut *p }).as_char_ptr() +} + +unsafe extern "C" fn enable_signaling_cb(fence: *mut bindings::dma_fence) -> bool { + // SAFETY: All of our fences are FenceObject. + let p = unsafe { crate::container_of!(fence, FenceObject, fence) as *mut FenceObject }; + + // SAFETY: The caller is responsible for passing a valid dma_fence subtype + T::enable_signaling(unsafe { &mut *p }) +} + +unsafe extern "C" fn signaled_cb(fence: *mut bindings::dma_fence) -> bool { + // SAFETY: All of our fences are FenceObject. + let p = unsafe { crate::container_of!(fence, FenceObject, fence) as *mut FenceObject }; + + // SAFETY: The caller is responsible for passing a valid dma_fence subtype + T::signaled(unsafe { &mut *p }) +} + +unsafe extern "C" fn release_cb(fence: *mut bindings::dma_fence) { + // SAFETY: All of our fences are FenceObject. + let p = unsafe { crate::container_of!(fence, FenceObject, fence) as *mut FenceObject }; + + // SAFETY: p is never used after this + unsafe { + core::ptr::drop_in_place(&mut (*p).inner); + } + + // SAFETY: All of our fences are allocated using kmalloc, so this is safe. + unsafe { bindings::dma_fence_free(fence) }; +} + +/// A driver-specific DMA Fence Object +/// +/// # Invariants +/// ptr is a valid pointer to a dma_fence and we own a reference to it. +#[repr(C)] +pub struct FenceObject { + fence: bindings::dma_fence, + lock: Opaque, + inner: T, +} + +impl FenceObject { + const SIZE: usize = core::mem::size_of::(); + + const VTABLE: bindings::dma_fence_ops = bindings::dma_fence_ops { + get_driver_name: Some(get_driver_name_cb::), + get_timeline_name: Some(get_timeline_name_cb::), + enable_signaling: if T::HAS_ENABLE_SIGNALING { + Some(enable_signaling_cb::) + } else { + None + }, + signaled: if T::HAS_SIGNALED { + Some(signaled_cb::) + } else { + None + }, + wait: None, // Deprecated + release: Some(release_cb::), + set_deadline: None, + }; +} + +impl Deref for FenceObject { + type Target = T; + + fn deref(&self) -> &T { + &self.inner + } +} + +impl DerefMut for FenceObject { + fn deref_mut(&mut self) -> &mut T { + &mut self.inner + } +} + +impl private::Sealed for FenceObject {} +impl RawDmaFence for FenceObject { + fn raw(&self) -> *mut bindings::dma_fence { + &self.fence as *const _ as *mut _ + } +} + +/// A unique reference to a driver-specific fence object +pub struct UniqueFence(*mut FenceObject); + +impl Deref for UniqueFence { + type Target = FenceObject; + + fn deref(&self) -> &FenceObject { + // SAFETY: The pointer is always valid for UniqueFence objects + unsafe { &*self.0 } + } +} + +impl DerefMut for UniqueFence { + fn deref_mut(&mut self) -> &mut FenceObject { + // SAFETY: The pointer is always valid for UniqueFence objects + unsafe { &mut *self.0 } + } +} + +impl private::Sealed for UniqueFence {} +impl RawDmaFence for UniqueFence { + fn raw(&self) -> *mut bindings::dma_fence { + // SAFETY: The pointer is always valid for UniqueFence objects + unsafe { addr_of_mut!((*self.0).fence) } + } +} + +impl From> for UserFence { + fn from(value: UniqueFence) -> Self { + let ptr = value.0; + core::mem::forget(value); + + UserFence(ptr) + } +} + +impl Drop for UniqueFence { + fn drop(&mut self) { + // SAFETY: We own a reference to this fence. + unsafe { bindings::dma_fence_put(self.raw()) }; + } +} + +// SAFETY: The API for these objects is thread safe +unsafe impl Sync for UniqueFence {} +// SAFETY: The API for these objects is thread safe +unsafe impl Send for UniqueFence {} + +/// A shared reference to a driver-specific fence object +pub struct UserFence(*mut FenceObject); + +impl Deref for UserFence { + type Target = FenceObject; + + fn deref(&self) -> &FenceObject { + // SAFETY: The pointer is always valid for UserFence objects + unsafe { &*self.0 } + } +} + +impl Clone for UserFence { + fn clone(&self) -> Self { + // SAFETY: `ptr` is valid per the type invariant and we own a reference to it. + unsafe { + bindings::dma_fence_get(self.raw()); + Self(self.0) + } + } +} + +impl private::Sealed for UserFence {} +impl RawDmaFence for UserFence { + fn raw(&self) -> *mut bindings::dma_fence { + // SAFETY: The pointer is always valid for UserFence objects + unsafe { addr_of_mut!((*self.0).fence) } + } +} + +impl Drop for UserFence { + fn drop(&mut self) { + // SAFETY: We own a reference to this fence. + unsafe { bindings::dma_fence_put(self.raw()) }; + } +} + +// SAFETY: The API for these objects is thread safe +unsafe impl Sync for UserFence {} +// SAFETY: The API for these objects is thread safe +unsafe impl Send for UserFence {} + +/// An array of fence contexts, out of which fences can be created. +pub struct FenceContexts { + start: u64, + count: u32, + seqnos: KVec, + lock_name: &'static CStr, + lock_key: Pin<&'static LockClassKey>, +} + +impl FenceContexts { + /// Create a new set of fence contexts. + pub fn new( + count: u32, + name: &'static CStr, + key: Pin<&'static LockClassKey>, + ) -> Result { + let mut seqnos: KVec = KVec::new(); + + seqnos.reserve(count as usize, GFP_KERNEL)?; + + for _ in 0..count { + seqnos.push(Default::default(), GFP_KERNEL)?; + } + + // SAFETY: This is always safe to call + let start = unsafe { bindings::dma_fence_context_alloc(count as crate::ffi::c_uint) }; + + Ok(FenceContexts { + start, + count, + seqnos, + lock_name: name, + lock_key: key, + }) + } + + /// Create a new fence in a given context index. + pub fn new_fence(&self, context: u32, inner: T) -> Result> { + if context > self.count { + return Err(EINVAL); + } + + // SAFETY: krealloc is always safe to call like this + let p = unsafe { + bindings::krealloc_node_align( + core::ptr::null_mut(), + FenceObject::::SIZE, + 1, + bindings::GFP_KERNEL | bindings::__GFP_ZERO, + bindings::NUMA_NO_NODE, + ) as *mut FenceObject + }; + + if p.is_null() { + return Err(ENOMEM); + } + + let seqno = self.seqnos[context as usize].fetch_add(1, Ordering::Relaxed); + + // SAFETY: The pointer is valid, so pointers to members are too. + // After this, all fields are initialized. + unsafe { + addr_of_mut!((*p).inner).write(inner); + bindings::__spin_lock_init( + addr_of_mut!((*p).lock) as *mut _, + self.lock_name.as_char_ptr(), + self.lock_key.as_ptr(), + ); + bindings::dma_fence_init64( + addr_of_mut!((*p).fence), + &FenceObject::::VTABLE, + addr_of_mut!((*p).lock) as *mut _, + self.start + context as u64, + seqno, + ); + }; + + Ok(UniqueFence(p)) + } +} + +/// A DMA Fence Chain Object +/// +/// # Invariants +/// ptr is a valid pointer to a dma_fence_chain which we own. +pub struct FenceChain { + ptr: *mut bindings::dma_fence_chain, +} + +impl FenceChain { + /// Create a new DmaFenceChain object. + pub fn new() -> Result { + // SAFETY: This function is safe to call and takes no arguments. + let ptr = unsafe { bindings::dma_fence_chain_alloc() }; + + if ptr.is_null() { + Err(ENOMEM) + } else { + Ok(FenceChain { ptr }) + } + } + + /// Convert the DmaFenceChain into the underlying raw pointer. + /// + /// This assumes the caller will take ownership of the object. + pub(crate) fn into_raw(self) -> *mut bindings::dma_fence_chain { + let ptr = self.ptr; + core::mem::forget(self); + ptr + } +} + +impl Drop for FenceChain { + fn drop(&mut self) { + // SAFETY: We own this dma_fence_chain. + unsafe { bindings::dma_fence_chain_free(self.ptr) }; + } +} diff --git a/rust/kernel/driver.rs b/rust/kernel/driver.rs index bee3ae21a27b2a..36de8098754d05 100644 --- a/rust/kernel/driver.rs +++ b/rust/kernel/driver.rs @@ -94,10 +94,14 @@ //! [`device_id`]: kernel::device_id //! [`module_driver`]: kernel::module_driver -use crate::error::{Error, Result}; -use crate::{acpi, device, of, str::CStr, try_pin_init, types::Opaque, ThisModule}; -use core::pin::Pin; -use pin_init::{pin_data, pinned_drop, PinInit}; +use crate::{ + acpi, + device, + of, + prelude::*, + types::Opaque, + ThisModule, // +}; /// Trait describing the layout of a specific device driver. /// diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 3ce8f62a005696..76625537f38023 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -86,7 +86,7 @@ impl Device { name: crate::str::as_char_ptr_in_const_context(T::INFO.name).cast_mut(), desc: crate::str::as_char_ptr_in_const_context(T::INFO.desc).cast_mut(), - driver_features: drm::driver::FEAT_GEM, + driver_features: T::FEATURES, ioctls: T::IOCTLS.as_ptr(), num_ioctls: T::IOCTLS.len() as i32, fops: &Self::GEM_FOPS, @@ -184,7 +184,10 @@ impl Device { // SAFETY: // - When `release` runs it is guaranteed that there is no further access to `this`. // - `this` is valid for dropping. - unsafe { core::ptr::drop_in_place(this) }; + // unsafe { core::ptr::drop_in_place(this) }; + // HACK: data might be uninitialized so leak the DRM device instead. The expected number + // of times the asahi device gets released is once at poweroff or reboot. + let _ = core::mem::ManuallyDrop::new(this); } } diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index f30ee4c6245cda..bb072b3a01817e 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -13,7 +13,16 @@ use crate::{ use macros::vtable; /// Driver use the GEM memory manager. This should be set for all modern drivers. -pub(crate) const FEAT_GEM: u32 = bindings::drm_driver_feature_DRIVER_GEM; +pub const FEAT_GEM: u32 = bindings::drm_driver_feature_DRIVER_GEM; +/// Driver supports dedicated render nodes. +pub const FEAT_RENDER: u32 = bindings::drm_driver_feature_DRIVER_RENDER; +/// Driver supports DRM sync objects for explicit synchronization of command submission. +pub const FEAT_SYNCOBJ: u32 = bindings::drm_driver_feature_DRIVER_SYNCOBJ; +/// Driver supports the timeline flavor of DRM sync objects for explicit synchronization of command +/// submission. +pub const FEAT_SYNCOBJ_TIMELINE: u32 = bindings::drm_driver_feature_DRIVER_SYNCOBJ_TIMELINE; +/// Driver supports user defined GPU VA bindings for GEM objects. +pub const FEAT_GEM_GPUVA: u32 = bindings::drm_driver_feature_DRIVER_GEM_GPUVA; /// Information data for a DRM Driver. pub struct DriverInfo { @@ -111,6 +120,9 @@ pub trait Driver { /// Driver metadata const INFO: DriverInfo; + /// Feature flags + const FEATURES: u32; + /// IOCTL list. See `kernel::drm::ioctl::declare_drm_ioctls!{}`. const IOCTLS: &'static [drm::ioctl::DrmIoctlDescriptor]; } diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index 8c46f8d519516a..9c7bcace70eef5 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -15,6 +15,9 @@ pub trait DriverFile { /// Open a new file (called when a client opens the DRM device). fn open(device: &drm::Device) -> Result>>; + + /// Get raw drm_file pointer + fn as_raw(&self) -> *mut bindings::drm_file; } /// An open DRM File. diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index a7f682e95c0184..381bd798f2199a 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -3,18 +3,58 @@ //! DRM GEM API //! //! C header: [`include/drm/drm_gem.h`](srctree/include/drm/drm_gem.h) +#[cfg(CONFIG_DRM_GEM_SHMEM_HELPER = "y")] +pub mod shmem; use crate::{ alloc::flags::*, - bindings, drm, + bindings, dma_buf, drm, drm::driver::{AllocImpl, AllocOps}, - error::{to_result, Result}, + error::{from_err_ptr, to_result, Result}, prelude::*, sync::aref::{ARef, AlwaysRefCounted}, types::Opaque, }; +use core::marker::PhantomData; use core::{ops::Deref, ptr::NonNull}; +/// A macro for implementing [`AlwaysRefCounted`] for any GEM object type. +/// +/// Since all GEM objects use the same refcounting scheme. +#[macro_export] +macro_rules! impl_aref_for_gem_obj { + ( + impl $( <$( $tparam_id:ident ),+> )? for $type:ty + $( + where + $( $bind_param:path : $bind_trait:path ),+ + )? + ) => { + // SAFETY: All gem objects are refcounted + unsafe impl $( <$( $tparam_id ),+> )? $crate::types::AlwaysRefCounted for $type + where + Self: IntoGEMObject, + $( $( $bind_param : $bind_trait ),+ )? + { + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the refcount is + // non-zero. + unsafe { bindings::drm_gem_object_get(self.as_raw()) }; + } + + unsafe fn dec_ref(obj: core::ptr::NonNull) { + // SAFETY: `obj` is a valid pointer to an `Object`. + let obj = unsafe { obj.as_ref() }.as_raw(); + + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::drm_gem_object_put(obj) }; + } + } + }; +} + +pub(crate) use impl_aref_for_gem_obj; + /// A type alias for retrieving a [`Driver`]s [`DriverFile`] implementation from its /// [`DriverObject`] implementation. /// @@ -23,12 +63,20 @@ use core::{ops::Deref, ptr::NonNull}; pub type DriverFile = drm::File<<::Driver as drm::Driver>::File>; /// GEM object functions, which must be implemented by drivers. +#[vtable] pub trait DriverObject: Sync + Send + Sized { /// Parent `Driver` for this object. type Driver: drm::Driver; + /// The data type to use for passing arguments to [`DriverObject::new`]. + type Args; + /// Create a new driver data object for a GEM object of a given size. - fn new(dev: &drm::Device, size: usize) -> impl PinInit; + fn new( + dev: &drm::Device, + size: usize, + args: Self::Args, + ) -> impl PinInit; /// Open a new handle to an existing object, associated with a File. fn open(_obj: &::Object, _file: &DriverFile) -> Result { @@ -37,6 +85,14 @@ pub trait DriverObject: Sync + Send + Sized { /// Close a handle to an existing object, associated with a File. fn close(_obj: &::Object, _file: &DriverFile) {} + + /// Optional handle for exporting a gem object. + fn export( + _obj: &::Object, + _flags: u32, + ) -> Result::Object>> { + unimplemented!() + } } /// Trait that represents a GEM object subtype @@ -86,6 +142,21 @@ extern "C" fn close_callback( T::close(obj, file); } +extern "C" fn export_callback( + raw_obj: *mut bindings::drm_gem_object, + flags: i32, +) -> *mut bindings::dma_buf { + // SAFETY: `export_callback` is specified in the AllocOps structure for `Object`, ensuring + // that `raw_obj` is contained within a `Object`. + let obj = unsafe { <::Object as IntoGEMObject>::from_raw(raw_obj) }; + + match T::export(obj, flags as _) { + // DRM takes a hold of the reference + Ok(buf) => buf.into_raw(), + Err(e) => e.to_ptr(), + } +} + impl IntoGEMObject for Object { fn as_raw(&self) -> *mut bindings::drm_gem_object { self.obj.get() @@ -150,6 +221,28 @@ pub trait BaseObject: IntoGEMObject { Ok(unsafe { ARef::from_raw(obj.into()) }) } + /// Export a [`DmaBuf`] for this GEM object using the DRM prime helper library. + /// + /// `flags` should be a set of flags from [`fs::file::flags`](kernel::fs::file::flags). + fn prime_export(&self, flags: u32) -> Result> { + // SAFETY: + // - `as_raw()` always returns a valid pointer to a `drm_gem_object`. + // - `drm_gem_prime_export()` returns either an error pointer, or a valid pointer to an + // initialized `dma_buf` on success. + let dma_ptr = + from_err_ptr(unsafe { bindings::drm_gem_prime_export(self.as_raw(), flags as _) })?; + + // SAFETY: + // - We checked that dma_ptr is not an error, so it must point to an initialized dma_buf + // - We used drm_gem_prime_export(), so `dma_ptr` will remain valid until a call to + // `drm_gem_prime_release()` which we don't call here. + let dma_buf = unsafe { dma_buf::DmaBuf::as_ref(dma_ptr) }; + + // INVARIANT: We used drm_gem_prime_export() to create this dma_buf, fulfilling the + // invariant that this dma_buf came from a GEM object of type `Self`. + Ok(DmaBuf(dma_buf.into(), PhantomData)) + } + /// Creates an mmap offset to map the object from userspace. fn create_mmap_offset(&self) -> Result { // SAFETY: The arguments are valid per the type invariant. @@ -158,10 +251,35 @@ pub trait BaseObject: IntoGEMObject { // SAFETY: The arguments are valid per the type invariant. Ok(unsafe { bindings::drm_vma_node_offset_addr(&raw mut (*self.as_raw()).vma_node) }) } + + /// Lock the gpuva lock + fn lock_gpuva(&self) { + unsafe { + bindings::mutex_lock(&raw mut (*self.as_raw()).gpuva.lock); + } + } + + /// Lock the gpuva lock + fn unlock_gpuva(&self) { + unsafe { + bindings::mutex_unlock(&raw mut (*self.as_raw()).gpuva.lock); + } + } } impl BaseObject for T {} +/// Crate-private base operations shared by all GEM object classes. +pub(crate) trait BaseObjectPrivate: IntoGEMObject { + /// Return a pointer to this object's dma_resv. + fn raw_dma_resv(&self) -> *mut bindings::dma_resv { + // SAFETY: `as_gem_obj()` always returns a valid pointer to the base DRM gem object + unsafe { (*self.as_raw()).resv } + } +} + +impl BaseObjectPrivate for T {} + /// A base GEM object. /// /// # Invariants @@ -181,7 +299,11 @@ impl Object { open: Some(open_callback::), close: Some(close_callback::), print_info: None, - export: None, + export: if T::HAS_EXPORT { + Some(export_callback::) + } else { + None + }, pin: None, unpin: None, get_sg_table: None, @@ -195,11 +317,11 @@ impl Object { }; /// Create a new GEM object. - pub fn new(dev: &drm::Device, size: usize) -> Result> { + pub fn new(dev: &drm::Device, size: usize, args: T::Args) -> Result> { let obj: Pin> = KBox::pin_init( try_pin_init!(Self { obj: Opaque::new(bindings::drm_gem_object::default()), - data <- T::new(dev, size), + data <- T::new(dev, size, args), }), GFP_KERNEL, )?; @@ -252,21 +374,7 @@ impl Object { } } -// SAFETY: Instances of `Object` are always reference-counted. -unsafe impl crate::types::AlwaysRefCounted for Object { - fn inc_ref(&self) { - // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. - unsafe { bindings::drm_gem_object_get(self.as_raw()) }; - } - - unsafe fn dec_ref(obj: NonNull) { - // SAFETY: `obj` is a valid pointer to an `Object`. - let obj = unsafe { obj.as_ref() }; - - // SAFETY: The safety requirements guarantee that the refcount is non-zero. - unsafe { bindings::drm_gem_object_put(obj.as_raw()) } - } -} +impl_aref_for_gem_obj!(impl for Object where T: DriverObject); impl super::private::Sealed for Object {} @@ -292,6 +400,49 @@ impl AllocImpl for Object { }; } +/// A [`dma_buf::DmaBuf`] which has been exported from a GEM object. +/// +/// The [`dma_buf::DmaBuf`] will be released when this type is dropped. +/// +/// # Invariants +/// +/// - `self.0` points to a valid initialized [`dma_buf::DmaBuf`] for the lifetime of this object. +/// - The GEM object from which this [`dma_buf::DmaBuf`] was exported from is guaranteed to be of +/// type `T`. +pub struct DmaBuf(NonNull, PhantomData); + +impl Deref for DmaBuf { + type Target = dma_buf::DmaBuf; + + #[inline] + fn deref(&self) -> &Self::Target { + // SAFETY: This pointer is guaranteed to be valid by our type invariants. + unsafe { self.0.as_ref() } + } +} + +impl Drop for DmaBuf { + #[inline] + fn drop(&mut self) { + // SAFETY: + // - `dma_buf::DmaBuf` is guaranteed to have an identical layout to `struct dma_buf` + // by its type invariants. + // - We hold the last reference to this `DmaBuf`, making it safe to destroy. + unsafe { bindings::drm_gem_dmabuf_release(self.0.cast().as_ptr()) } + } +} + +impl DmaBuf { + /// Leak the reference for this [`DmaBuf`] and return a raw pointer to it. + #[inline] + pub(crate) fn into_raw(self) -> *mut bindings::dma_buf { + let dma_ptr = self.as_raw(); + + core::mem::forget(self); + dma_ptr + } +} + pub(super) const fn create_fops() -> bindings::file_operations { // SAFETY: As by the type invariant, it is safe to initialize `bindings::file_operations` // zeroed. diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs new file mode 100644 index 00000000000000..bfb05e19cc4500 --- /dev/null +++ b/rust/kernel/drm/gem/shmem.rs @@ -0,0 +1,442 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! DRM GEM shmem helper objects +//! +//! C header: [`include/linux/drm/drm_gem_shmem_helper.h`](srctree/include/linux/drm/drm_gem_shmem_helper.h) + +// TODO: +// - There are a number of spots here that manually acquire/release the DMA reservation lock using +// dma_resv_(un)lock(). In the future we should add support for ww mutex, expose a method to +// acquire a reference to the WwMutex, and then use that directly instead of the C functions here. + +use crate::{ + container_of, + drm::{device, driver, gem, private::Sealed}, + error::{from_err_ptr, to_result}, + iosys_map::*, + prelude::*, + scatterlist, + transmute::*, + types::{ARef, Opaque}, +}; +use core::{ + mem::{self, MaybeUninit}, + ops::{Deref, DerefMut}, + ptr::NonNull, +}; +use gem::{BaseObject, BaseObjectPrivate, DriverObject, IntoGEMObject}; + +/// A struct for controlling the creation of shmem-backed GEM objects. +/// +/// This is used with [`Object::new()`] to control various properties that can only be set when +/// initially creating a shmem-backed GEM object. +#[derive(Default)] +pub struct ObjectConfig<'a, T: DriverObject> { + /// Whether to set the write-combine map flag. + pub map_wc: bool, + + /// Reuse the DMA reservation from another GEM object. + /// + /// The newly created [`Object`] will hold an owned refcount to `parent_resv_obj` if specified. + pub parent_resv_obj: Option<&'a Object>, +} + +/// A shmem-backed GEM object. +/// +/// # Invariants +/// +/// `obj` contains a valid initialized `struct drm_gem_shmem_object` for the lifetime of this +/// object. +#[repr(C)] +#[pin_data] +pub struct Object { + #[pin] + obj: Opaque, + // Parent object that owns this object's DMA reservation object + parent_resv_obj: Option>>, + #[pin] + inner: T, +} + +super::impl_aref_for_gem_obj!(impl for Object where T: DriverObject); + +// SAFETY: This object is thread-safe via our type invariants. +unsafe impl Send for Object {} +// SAFETY: This object is thread-safe via our type invariants. +unsafe impl Sync for Object {} + +impl Object { + /// `drm_gem_object_funcs` vtable suitable for GEM shmem objects. + const VTABLE: bindings::drm_gem_object_funcs = bindings::drm_gem_object_funcs { + free: Some(Self::free_callback), + open: Some(super::open_callback::), + close: Some(super::close_callback::), + print_info: Some(bindings::drm_gem_shmem_object_print_info), + export: if T::HAS_EXPORT { + Some(super::export_callback::) + } else { + None + }, + pin: Some(bindings::drm_gem_shmem_object_pin), + unpin: Some(bindings::drm_gem_shmem_object_unpin), + get_sg_table: Some(bindings::drm_gem_shmem_object_get_sg_table), + vmap: Some(bindings::drm_gem_shmem_object_vmap), + vunmap: Some(bindings::drm_gem_shmem_object_vunmap), + mmap: Some(bindings::drm_gem_shmem_object_mmap), + status: None, + rss: None, + // SAFETY: `drm_gem_shmem_vm_ops` is static const on the C side, so immutable references are + // safe here and such references shall be valid forever + vm_ops: unsafe { &bindings::drm_gem_shmem_vm_ops }, + evict: None, + }; + + /// Return a raw pointer to the embedded drm_gem_shmem_object. + fn as_shmem(&self) -> *mut bindings::drm_gem_shmem_object { + self.obj.get() + } + + /// Create a new shmem-backed DRM object of the given size. + /// + /// Additional config options can be specified using `config`. + pub fn new( + dev: &device::Device, + size: usize, + config: ObjectConfig<'_, T>, + args: T::Args, + ) -> Result> { + let new: Pin> = KBox::try_pin_init( + try_pin_init!(Self { + obj <- Opaque::init_zeroed(), + parent_resv_obj: config.parent_resv_obj.map(|p| p.into()), + inner <- T::new(dev, size, args), + }), + GFP_KERNEL, + )?; + + // SAFETY: `obj.as_raw()` is guaranteed to be valid by the initialization above. + unsafe { (*new.as_raw()).funcs = &Self::VTABLE }; + + // SAFETY: The arguments are all valid via the type invariants. + to_result(unsafe { bindings::drm_gem_shmem_init(dev.as_raw(), new.as_shmem(), size) })?; + + // SAFETY: We never move out of `self`. + let new = KBox::into_raw(unsafe { Pin::into_inner_unchecked(new) }); + + // SAFETY: We're taking over the owned refcount from `drm_gem_shmem_init`. + let obj = unsafe { ARef::from_raw(NonNull::new_unchecked(new)) }; + + // Start filling out values from `config` + if let Some(parent_resv) = config.parent_resv_obj { + // SAFETY: We have yet to expose the new gem object outside of this function, so it is + // safe to modify this field. + unsafe { (*obj.obj.get()).base.resv = parent_resv.raw_dma_resv() }; + } + + // SAFETY: We have yet to expose this object outside of this function, so we're guaranteed + // to have exclusive access - thus making this safe to hold a mutable reference to. + let shmem = unsafe { &mut *obj.as_shmem() }; + shmem.set_map_wc(config.map_wc); + + Ok(obj) + } + + /// Returns the `Device` that owns this GEM object. + pub fn dev(&self) -> &device::Device { + // SAFETY: `dev` will have been initialized in `Self::new()` by `drm_gem_shmem_init()`. + unsafe { device::Device::from_raw((*self.as_raw()).dev) } + } + + extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) { + // SAFETY: + // - DRM always passes a valid gem object here + // - We used drm_gem_shmem_create() in our create_gem_object callback, so we know that + // `obj` is contained within a drm_gem_shmem_object + let this = unsafe { container_of!(obj, bindings::drm_gem_shmem_object, base) }; + + // SAFETY: + // - We're in free_callback - so this function is safe to call. + // - We won't be using the gem resources on `this` after this call. + unsafe { bindings::drm_gem_shmem_release(this) }; + + // SAFETY: + // - We verified above that `obj` is valid, which makes `this` valid + // - This function is set in AllocOps, so we know that `this` is contained within a + // `Object` + let this = unsafe { container_of!(Opaque::cast_from(this), Self, obj) }.cast_mut(); + + // SAFETY: We're recovering the Kbox<> we created in gem_create_object() + let _ = unsafe { KBox::from_raw(this) }; + } + + /// Creates (if necessary) and returns an immutable reference to a scatter-gather table of DMA + /// pages for this object. + /// + /// This will pin the object in memory. + #[inline] + pub fn sg_table(&self) -> Result<&scatterlist::SGTable> { + // SAFETY: + // - drm_gem_shmem_get_pages_sgt is thread-safe. + // - drm_gem_shmem_get_pages_sgt returns either a valid pointer to a scatterlist, or an + // error pointer. + let sgt = from_err_ptr(unsafe { bindings::drm_gem_shmem_get_pages_sgt(self.as_shmem()) })?; + + // SAFETY: We checked above that `sgt` is not an error pointer, so it must be a valid + // pointer to a scatterlist + Ok(unsafe { scatterlist::SGTable::from_raw(sgt) }) + } + + /// Creates (if necessary) and returns an owned reference to a scatter-gather table of DMA pages + /// for this object. + /// + /// This is the same as [`sg_table`](Self::sg_table), except that it instead returns an + /// [`shmem::SGTable`] which holds a reference to the associated gem object, instead of a + /// reference to an [`scatterlist::SGTable`]. + /// + /// This will pin the object in memory. + /// + /// [`shmem::SGTable`]: SGTable + pub fn owned_sg_table(&self) -> Result> { + Ok(SGTable { + sgt: self.sg_table()?.into(), + // INVARIANT: We take an owned refcount to `self` here, ensuring that `sgt` remains + // valid for as long as this `SGTable`. + _owner: self.into(), + }) + } + + /// Attempt to create a [`RawIoSysMap`] from the gem object. + fn raw_vmap(&self) -> Result> { + build_assert!( + mem::size_of::() > 0, + "It doesn't make sense for the mapping type to be a ZST" + ); + + let mut map: MaybeUninit = MaybeUninit::uninit(); + + // SAFETY: drm_gem_shmem_vmap can be called with the DMA reservation lock held + to_result(unsafe { + // TODO: see top of file + bindings::dma_resv_lock(self.raw_dma_resv(), core::ptr::null_mut()); + let ret = bindings::drm_gem_shmem_vmap_locked(self.as_shmem(), map.as_mut_ptr()); + bindings::dma_resv_unlock(self.raw_dma_resv()); + ret + })?; + + // SAFETY: if drm_gem_shmem_vmap did not fail, map is initialized now + Ok(unsafe { RawIoSysMap::from_raw(map.assume_init()) }) + } + + /// Unmap a [`RawIoSysMap`] from the gem object. + /// + /// # Safety + /// + /// - The caller promises that `map` came from a prior call to [`Self::raw_vmap`] on this gem + /// object. + /// - The caller promises that the memory pointed to by `map` will no longer be accesed through + /// this instance. + unsafe fn raw_vunmap(&self, map: &mut RawIoSysMap) { + let resv = self.raw_dma_resv(); + + // SAFETY: + // - This function is safe to call with the DMA reservation lock held + // - Our `ARef` is proof that the underlying gem object here is initialized and thus safe to + // dereference. + unsafe { + // TODO: see top of file + bindings::dma_resv_lock(resv, core::ptr::null_mut()); + bindings::drm_gem_shmem_vunmap_locked(self.as_shmem(), map.as_raw_mut()); + bindings::dma_resv_unlock(resv); + } + } + + /// Creates and returns a virtual kernel memory mapping for this object. + pub fn vmap(&self) -> Result> { + let map = self.raw_vmap()?; + + Ok(VMapRef { + // SAFETY: + // - The size of the vmap is the same as the size of the gem + // - The vmap will remain alive until this object is dropped. + map: unsafe { IoSysMapRef::new(map, self.size()) }, + owner: self, + }) + } + + /// Creates and returns an owned reference to a virtual kernel memory mapping for this object. + pub fn owned_vmap(&self) -> Result> { + Ok(VMap { + map: self.raw_vmap()?, + owner: self.into(), + }) + } +} + +impl Deref for Object { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl DerefMut for Object { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +impl Sealed for Object {} + +impl gem::IntoGEMObject for Object { + fn as_raw(&self) -> *mut bindings::drm_gem_object { + // SAFETY: + // - Our immutable reference is proof that this is safe to dereference. + // - `obj` is always a valid drm_gem_shmem_object via our type invariants. + unsafe { &raw mut (*self.obj.get()).base } + } + + unsafe fn from_raw<'a>(obj: *mut bindings::drm_gem_object) -> &'a Object { + // SAFETY: The safety contract of from_gem_obj() guarantees that `obj` is contained within + // `Self` + unsafe { + let obj = Opaque::cast_from(container_of!(obj, bindings::drm_gem_shmem_object, base)); + + &*container_of!(obj, Object, obj) + } + } +} + +impl driver::AllocImpl for Object { + type Driver = T::Driver; + + const ALLOC_OPS: driver::AllocOps = driver::AllocOps { + gem_create_object: None, + prime_handle_to_fd: None, + prime_fd_to_handle: None, + gem_prime_import: None, + gem_prime_import_sg_table: Some(bindings::drm_gem_shmem_prime_import_sg_table), + dumb_create: Some(bindings::drm_gem_shmem_dumb_create), + dumb_map_offset: None, + }; +} + +/// A borrowed reference to a virtual mapping for a shmem-based GEM object in kernel address space. +pub struct VMapRef<'a, D: DriverObject, T: AsBytes + FromBytes> { + map: IoSysMapRef<'a, T>, + owner: &'a Object, +} + +impl<'a, D: DriverObject, T: AsBytes + FromBytes> Clone for VMapRef<'a, D, T> { + fn clone(&self) -> Self { + // SAFETY: We have a successful vmap already, so this can't fail + unsafe { self.owner.vmap().unwrap_unchecked() } + } +} + +impl<'a, D: DriverObject, T: AsBytes + FromBytes> Deref for VMapRef<'a, D, T> { + type Target = IoSysMapRef<'a, T>; + + fn deref(&self) -> &Self::Target { + &self.map + } +} + +impl<'a, D: DriverObject, T: AsBytes + FromBytes> DerefMut for VMapRef<'a, D, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.map + } +} + +impl<'a, D: DriverObject, T: AsBytes + FromBytes> Drop for VMapRef<'a, D, T> { + fn drop(&mut self) { + // SAFETY: Our existence is proof that this map was previously created using self.owner. + unsafe { self.owner.raw_vunmap(&mut self.map) }; + } +} + +/// An owned reference to a virtual mapping for a shmem-based GEM object in kernel address space. +/// +/// # Invariants +/// +/// - The memory pointed to by `map` is at least as large as `T`. +/// - The memory pointed to by `map` remains valid at least until this object is dropped. +pub struct VMap { + map: RawIoSysMap, + owner: ARef>, +} + +impl Clone for VMap { + fn clone(&self) -> Self { + // SAFETY: We have a successful vmap already, so this can't fail + unsafe { self.owner.owned_vmap().unwrap_unchecked() } + } +} + +impl<'a, D: DriverObject, T: AsBytes + FromBytes> From> for VMap { + fn from(value: VMapRef<'a, D, T>) -> Self { + let this = Self { + map: value.map.clone(), + owner: value.owner.into(), + }; + + mem::forget(value); + this + } +} + +impl VMap { + /// Return a reference to the iosys map for this `VMap`. + pub fn get(&self) -> IoSysMapRef<'_, T> { + // SAFETY: The size of the iosys_map is equivalent to the size of the gem object. + unsafe { IoSysMapRef::new(self.map.clone(), self.owner.size()) } + } + + /// Borrows a reference to the object that owns this virtual mapping. + pub fn owner(&self) -> &Object { + &self.owner + } +} + +impl Drop for VMap { + fn drop(&mut self) { + // SAFETY: Our existence is proof that this map was previously created using self.owner + unsafe { self.owner.raw_vunmap(&mut self.map) }; + } +} + +/// SAFETY: `iosys_map` objects are safe to send across threads. +unsafe impl Send for VMap {} +/// SAFETY: `iosys_map` objects are safe to send across threads. +unsafe impl Sync for VMap {} + +/// An owned reference to a scatter-gather table of DMA address spans for a GEM shmem object. +/// +/// This object holds an owned reference to the underlying GEM shmem object, ensuring that the +/// [`scatterlist::SGTable`] referenced by this type remains valid for the lifetime of this object. +/// +/// # Invariants +/// +/// - `sgt` is kept alive by `_owner`, ensuring it remains valid for as long as `Self`. +/// - `sgt` corresponds to the owned object in `_owner`. +/// - This object is only exposed in situations where we know the underlying `SGTable` will not be +/// modified for the lifetime of this object. Thus, it is safe to send/access this type across +/// threads. +pub struct SGTable { + sgt: NonNull, + _owner: ARef>, +} + +// SAFETY: This object is thread-safe via our type invariants. +unsafe impl Send for SGTable {} +// SAFETY: This object is thread-safe via our type invariants. +unsafe impl Sync for SGTable {} + +impl Deref for SGTable { + type Target = scatterlist::SGTable; + + fn deref(&self) -> &Self::Target { + // SAFETY: Creating an immutable reference to this is safe via our type invariants. + unsafe { self.sgt.as_ref() } + } +} diff --git a/rust/kernel/drm/gpuvm.rs b/rust/kernel/drm/gpuvm.rs new file mode 100644 index 00000000000000..04348b4e6d3933 --- /dev/null +++ b/rust/kernel/drm/gpuvm.rs @@ -0,0 +1,835 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM Sync Objects +//! +//! C header: [`include/drm/drm_gpuvm.h`](../../../../include/drm/drm_gpuvm.h) + +#![allow(missing_docs)] + +use crate::{ + bindings, drm, + drm::{ + device, + gem::{ + BaseObject, + IntoGEMObject, // + }, + }, + error::{ + code::{EINVAL, ENOMEM}, + from_result, to_result, Error, Result, + }, + prelude::*, + types::{ARef, AlwaysRefCounted, Opaque}, +}; + +use core::cell::UnsafeCell; +use core::marker::{PhantomData, PhantomPinned}; +use core::mem::ManuallyDrop; +use core::ops::{Deref, DerefMut, Range}; +use core::ptr::NonNull; +use pin_init; + +/// GpuVaFlags to be used for a GpuVa. +/// +/// They can be combined with the operators `|`, `&`, and `!`. +#[derive(Clone, Copy, PartialEq, Default)] +pub struct GpuVaFlags(u32); + +impl GpuVaFlags { + /// No GpuVaFlags (zero) + pub const NONE: GpuVaFlags = GpuVaFlags(0); + + /// The backing GEM is invalidated. + pub const INVALIDATED: GpuVaFlags = GpuVaFlags(bindings::drm_gpuva_flags_DRM_GPUVA_INVALIDATED); + + /// The GpuVa is a sparse mapping. + pub const SPARSE: GpuVaFlags = GpuVaFlags(bindings::drm_gpuva_flags_DRM_GPUVA_SPARSE); + + /// The GpuVa is a repeat mapping. + pub const REPEAT: GpuVaFlags = GpuVaFlags(bindings::drm_gpuva_flags_DRM_GPUVA_REPEAT); + + /// Construct a driver-specific GpuVaFlag. + /// + /// The argument must be a flag index in the range [0..28]. + pub const fn user_flag(index: u32) -> GpuVaFlags { + let flags = bindings::drm_gpuva_flags_DRM_GPUVA_USERBITS << index; + assert!(flags != 0); + GpuVaFlags(flags) + } + + /// Get the raw representation of this flag. + pub(crate) fn as_raw(self) -> u32 { + self.0 + } + + /// Check whether `flags` is contained in `self`. + pub fn contains(self, flags: GpuVaFlags) -> bool { + (self & flags) == flags + } +} + +impl core::ops::BitOr for GpuVaFlags { + type Output = Self; + fn bitor(self, rhs: Self) -> Self::Output { + Self(self.0 | rhs.0) + } +} + +impl core::ops::BitAnd for GpuVaFlags { + type Output = Self; + fn bitand(self, rhs: Self) -> Self::Output { + Self(self.0 & rhs.0) + } +} + +impl core::ops::Not for GpuVaFlags { + type Output = Self; + fn not(self) -> Self::Output { + Self(!self.0) + } +} + +/// Trait that must be implemented by DRM drivers to represent a DRM GpuVm (a GPU address space). +pub trait DriverGpuVm: Sized { + /// The parent `Driver` implementation for this `DriverGpuVm`. + type Driver: drm::Driver; + type GpuVa: DriverGpuVa = (); + type GpuVmBo: DriverGpuVmBo = (); + type StepContext = (); + + fn step_map( + self: &mut UpdatingGpuVm<'_, Self>, + op: &mut OpMap, + ctx: &mut Self::StepContext, + ) -> Result; + fn step_unmap( + self: &mut UpdatingGpuVm<'_, Self>, + op: &mut OpUnMap, + ctx: &mut Self::StepContext, + ) -> Result; + fn step_remap( + self: &mut UpdatingGpuVm<'_, Self>, + op: &mut OpReMap, + vm_bo: &GpuVmBo, + ctx: &mut Self::StepContext, + ) -> Result; +} + +struct StepContext<'a, T: DriverGpuVm> { + gpuvm: &'a GpuVm, + ctx: &'a mut T::StepContext, +} + +/// Trait that must be implemented by DRM drivers to represent a DRM GpuVa (a mapping in GPU address space). +pub trait DriverGpuVa: Sized {} + +impl DriverGpuVa for () {} + +/// Trait that must be implemented by DRM drivers to represent a DRM GpuVmBo (a connection between a BO and a VM). +pub trait DriverGpuVmBo: Sized { + fn new() -> impl PinInit; +} + +/// Provide a default implementation for trivial types +impl DriverGpuVmBo for T { + fn new() -> impl PinInit { + pin_init::default() + } +} + +/// A convenience type for the driver's GEM object. +type Object = <::Driver as drm::driver::Driver>::Object; + +#[repr(transparent)] +pub struct OpMap(bindings::drm_gpuva_op_map, PhantomData); +#[repr(transparent)] +pub struct OpUnMap(bindings::drm_gpuva_op_unmap, PhantomData); +#[repr(transparent)] +pub struct OpReMap(bindings::drm_gpuva_op_remap, PhantomData); + +impl OpMap { + pub fn addr(&self) -> u64 { + self.0.va.addr + } + pub fn range(&self) -> u64 { + self.0.va.range + } + pub fn offset(&self) -> u64 { + self.0.gem.offset + } + pub fn flags(&self) -> GpuVaFlags { + GpuVaFlags(self.0.flags) + } + pub fn object(&self) -> &Object { + let p = unsafe { as IntoGEMObject>::from_raw(self.0.gem.obj) }; + // SAFETY: The GEM object has an active reference for the lifetime of this op + &*p + } + pub fn map_and_link_va( + &mut self, + gpuvm: &mut UpdatingGpuVm<'_, T>, + gpuva: Pin>>, + gpuvmbo: &GpuVmBo, + ) -> Result<(), Pin>>> { + // SAFETY: We are handing off the GpuVa ownership and it will not be moved. + let p = KBox::leak(unsafe { Pin::into_inner_unchecked(gpuva) }); + // SAFETY: These C functions are called with the correct invariants + unsafe { + bindings::drm_gpuva_init_from_op(&mut p.gpuva, &mut self.0); + if bindings::drm_gpuva_insert(gpuvm.0.gpuvm() as *mut _, &mut p.gpuva) != 0 { + // EEXIST, return the GpuVa to the caller as an error + return Err(Pin::new_unchecked(KBox::from_raw(p))); + }; + // SAFETY: This takes a new reference to the gpuvmbo. + gpuvmbo.lock_gpuva(); + bindings::drm_gpuva_link(&mut p.gpuva, &gpuvmbo.bo as *const _ as *mut _); + gpuvmbo.unlock_gpuva(); + } + Ok(()) + } +} + +impl OpUnMap { + pub fn va(&self) -> Option<&GpuVa> { + if self.0.va.is_null() { + return None; + } + // SAFETY: Container invariant is guaranteed for ops structs created for our types. + let p = unsafe { crate::container_of!(self.0.va, GpuVa, gpuva) as *mut GpuVa }; + // SAFETY: The GpuVa object reference is valid per the op_unmap contract + Some(unsafe { &*p }) + } + pub fn unmap_and_unlink_va(&mut self) -> Option>>> { + self.do_unmap_and_unlink_va(false) + } + pub fn unmap_and_unlink_va_defer(&mut self) -> Option>>> { + self.do_unmap_and_unlink_va(true) + } + fn do_unmap_and_unlink_va(&mut self, defer: bool) -> Option>>> { + if self.0.va.is_null() { + return None; + } + // SAFETY: Container invariant is guaranteed for ops structs created for our types. + let p = unsafe { crate::container_of!(self.0.va, GpuVa, gpuva) as *mut GpuVa }; + + // SAFETY: The GpuVa object reference is valid per the op_unmap contract + unsafe { + bindings::drm_gpuva_unmap(&mut self.0); + if defer { + bindings::drm_gpuva_unlink_defer(self.0.va); + } else { + bindings::drm_gpuva_unlink(self.0.va); + } + } + + // Unlinking/unmapping relinquishes ownership of the GpuVa object, + // so clear the pointer + self.0.va = core::ptr::null_mut(); + // SAFETY: The GpuVa object reference is valid per the op_unmap contract + Some(unsafe { Pin::new_unchecked(KBox::from_raw(p)) }) + } +} + +impl OpReMap { + pub fn prev_map(&mut self) -> Option<&mut OpMap> { + // SAFETY: The prev pointer must be valid if not-NULL per the op_remap contract + unsafe { (self.0.prev as *mut OpMap).as_mut() } + } + pub fn next_map(&mut self) -> Option<&mut OpMap> { + // SAFETY: The next pointer must be valid if not-NULL per the op_remap contract + unsafe { (self.0.next as *mut OpMap).as_mut() } + } + pub fn unmap(&mut self) -> &mut OpUnMap { + // SAFETY: The unmap pointer is always valid per the op_remap contract + unsafe { (self.0.unmap as *mut OpUnMap).as_mut().unwrap() } + } +} + +/// A base GPU VA. +#[repr(C)] +#[pin_data] +pub struct GpuVa { + #[pin] + gpuva: bindings::drm_gpuva, + #[pin] + inner: T::GpuVa, + #[pin] + _p: PhantomPinned, +} + +impl GpuVa { + pub fn new(inner: impl PinInit) -> Result>>> + where + Error: From, + { + KBox::try_pin_init( + try_pin_init!(Self { + gpuva <- pin_init::init_zeroed(), + inner <- inner, + _p: PhantomPinned + }), + GFP_KERNEL, + ) + } + + pub fn addr(&self) -> u64 { + self.gpuva.va.addr + } + pub fn range(&self) -> u64 { + self.gpuva.va.range + } + pub fn offset(&self) -> u64 { + self.gpuva.gem.offset + } + pub fn flags(&self) -> GpuVaFlags { + GpuVaFlags(self.gpuva.flags) + } +} + +/// A base GpuVm BO. +#[repr(C)] +#[pin_data] +pub struct GpuVmBo { + #[pin] + bo: bindings::drm_gpuvm_bo, + #[pin] + inner: T::GpuVmBo, + #[pin] + _p: PhantomPinned, +} + +impl GpuVmBo { + /// Return a reference to the inner driver data for this GpuVmBo + pub fn inner(&self) -> &T::GpuVmBo { + &self.inner + } + /// Lock the GpuVmBo's gem boject gpuva lock + pub fn lock_gpuva(&self) { + unsafe { + let lock = &raw mut (*self.bo.obj).gpuva.lock; + bindings::mutex_lock(lock); + } + } + /// Unlock the GpuVmBo's gem boject gpuva lock + pub fn unlock_gpuva(&self) { + unsafe { + let lock = &raw mut (*self.bo.obj).gpuva.lock; + bindings::mutex_unlock(lock); + } + } +} + +// SAFETY: DRM GpuVmBo objects are always reference counted and the get/put functions +// satisfy the requirements. +unsafe impl AlwaysRefCounted for GpuVmBo { + fn inc_ref(&self) { + // SAFETY: The drm_gpuvm_get function satisfies the requirements for inc_ref(). + unsafe { bindings::drm_gpuvm_bo_get(&self.bo as *const _ as *mut _) }; + } + + unsafe fn dec_ref(mut obj: NonNull) { + // SAFETY: drm_gpuvm_bo_put() requires holding the gpuva lock, which is the dma_resv lock by default. + // The drm_gpuvm_put function satisfies the requirements for dec_ref(). + // (We do not support custom locks yet.) + unsafe { + let resv = (*obj.as_mut().bo.obj).resv; + bindings::dma_resv_lock(resv, core::ptr::null_mut()); + bindings::drm_gpuvm_bo_put(&mut obj.as_mut().bo); + bindings::dma_resv_unlock(resv); + } + } +} + +/// A base GPU VM. +#[repr(C)] +#[pin_data] +pub struct GpuVm { + #[pin] + gpuvm: Opaque, + #[pin] + inner: UnsafeCell, + #[pin] + _p: PhantomPinned, +} + +pub(super) unsafe extern "C" fn vm_free_callback( + raw_gpuvm: *mut bindings::drm_gpuvm, +) { + // SAFETY: Container invariant is guaranteed for objects using our callback. + let p = unsafe { + crate::container_of!( + raw_gpuvm as *mut Opaque, + GpuVm, + gpuvm + ) as *mut GpuVm + }; + + // SAFETY: p is guaranteed to be valid for drm_gpuvm objects using this callback. + unsafe { drop(KBox::from_raw(p)) }; +} + +pub(super) unsafe extern "C" fn vm_bo_alloc_callback() -> *mut bindings::drm_gpuvm_bo +{ + let obj: Result>>> = KBox::try_pin_init( + try_pin_init!(GpuVmBo:: { + bo <- pin_init::default(), + inner <- T::GpuVmBo::new(), + _p: PhantomPinned + }), + GFP_KERNEL, + ); + + match obj { + Ok(obj) => + // SAFETY: The DRM core will keep this object pinned + unsafe { + let p = KBox::leak(Pin::into_inner_unchecked(obj)); + &mut p.bo + }, + Err(_) => core::ptr::null_mut(), + } +} + +pub(super) unsafe extern "C" fn vm_bo_free_callback( + raw_vm_bo: *mut bindings::drm_gpuvm_bo, +) { + // SAFETY: Container invariant is guaranteed for objects using this callback. + let p = unsafe { crate::container_of!(raw_vm_bo, GpuVmBo, bo) as *mut GpuVmBo }; + + // SAFETY: p is guaranteed to be valid for drm_gpuvm_bo objects using this callback. + unsafe { drop(KBox::from_raw(p)) }; +} + +pub(super) unsafe extern "C" fn step_map_callback( + op: *mut bindings::drm_gpuva_op, + _priv: *mut core::ffi::c_void, +) -> core::ffi::c_int { + // SAFETY: We know this is a map op, and OpMap is a transparent wrapper. + let map = unsafe { &mut *((&mut (*op).__bindgen_anon_1.map) as *mut _ as *mut OpMap) }; + // SAFETY: This is a pointer to a StepContext created inline in sm_map(), which is + // guaranteed to outlive this function. + let ctx = unsafe { &mut *(_priv as *mut StepContext<'_, T>) }; + + from_result(|| { + UpdatingGpuVm(ctx.gpuvm).step_map(map, ctx.ctx)?; + Ok(0) + }) +} + +pub(super) unsafe extern "C" fn step_remap_callback( + op: *mut bindings::drm_gpuva_op, + _priv: *mut core::ffi::c_void, +) -> core::ffi::c_int { + // SAFETY: We know this is a map op, and OpReMap is a transparent wrapper. + let remap = unsafe { &mut *((&mut (*op).__bindgen_anon_1.remap) as *mut _ as *mut OpReMap) }; + // SAFETY: This is a pointer to a StepContext created inline in sm_map(), which is + // guaranteed to outlive this function. + let ctx = unsafe { &mut *(_priv as *mut StepContext<'_, T>) }; + + let p_vm_bo = remap.unmap().va().unwrap().gpuva.vm_bo; + + let res = { + // SAFETY: vm_bo pointer must be valid and non-null by the step_remap invariants. + // Since we grab a ref, this reference's lifetime is until the decref. + let vm_bo_ref = unsafe { + bindings::drm_gpuvm_bo_get(p_vm_bo); + &*(crate::container_of!(p_vm_bo, GpuVmBo, bo) as *mut GpuVmBo) + }; + + from_result(|| { + UpdatingGpuVm(ctx.gpuvm).step_remap(remap, vm_bo_ref, ctx.ctx)?; + Ok(0) + }) + }; + + // SAFETY: We incremented the refcount above, and the Rust reference we took is + // no longer in scope. + unsafe { bindings::drm_gpuvm_bo_put_deferred(p_vm_bo) }; + + res +} +pub(super) unsafe extern "C" fn step_unmap_callback( + op: *mut bindings::drm_gpuva_op, + _priv: *mut core::ffi::c_void, +) -> core::ffi::c_int { + // SAFETY: We know this is a map op, and OpUnMap is a transparent wrapper. + let unmap = unsafe { &mut *((&mut (*op).__bindgen_anon_1.unmap) as *mut _ as *mut OpUnMap) }; + // SAFETY: This is a pointer to a StepContext created inline in sm_map(), which is + // guaranteed to outlive this function. + let ctx = unsafe { &mut *(_priv as *mut StepContext<'_, T>) }; + + from_result(|| { + UpdatingGpuVm(ctx.gpuvm).step_unmap(unmap, ctx.ctx)?; + Ok(0) + }) +} + +pub(super) unsafe extern "C" fn exec_lock_gem_object( + vm_exec: *mut bindings::drm_gpuvm_exec, +) -> core::ffi::c_int { + // SAFETY: The gpuvm_exec object is valid and priv_ is a GEM object pointer + // when this callback is used + unsafe { bindings::drm_exec_lock_obj(&mut (*vm_exec).exec, (*vm_exec).extra.priv_ as *mut _) } +} + +impl GpuVm { + const OPS: bindings::drm_gpuvm_ops = bindings::drm_gpuvm_ops { + vm_free: Some(vm_free_callback::), + op_alloc: None, + op_free: None, + vm_bo_alloc: Some(vm_bo_alloc_callback::), + vm_bo_free: Some(vm_bo_free_callback::), + vm_bo_validate: None, + sm_step_map: Some(step_map_callback::), + sm_step_remap: Some(step_remap_callback::), + sm_step_unmap: Some(step_unmap_callback::), + sm_can_merge_flags: None, + }; + + fn gpuvm(&self) -> *const bindings::drm_gpuvm { + self.gpuvm.get() + } + + pub fn new( + name: &'static CStr, + flags: bindings::drm_gpuvm_flags, + dev: &device::Device, + r_obj: ARef>, + range: Range, + reserve_range: Range, + inner: impl PinInit, + ) -> Result>> + where + Error: From, + { + let obj: Pin> = KBox::try_pin_init( + try_pin_init!(Self { + // SAFETY: drm_gpuvm_init cannot fail and always initializes the member + gpuvm <- unsafe { + pin_init::pin_init_from_closure(move |slot: *mut Opaque | { + // Zero-init required by drm_gpuvm_init + *slot = Opaque::zeroed(); + bindings::drm_gpuvm_init( + Opaque::cast_into(slot), + name.as_char_ptr(), + flags, + dev.as_raw(), + r_obj.as_raw() as *const _ as *mut _, + range.start, + range.end - range.start, + reserve_range.start, + reserve_range.end - reserve_range.start, + &Self::OPS + ); + Ok(()) + }) + }, + // SAFETY: Just passing through to the initializer argument + inner <- unsafe { + pin_init::pin_init_from_closure(move |slot: *mut UnsafeCell | { + inner.__pinned_init(slot as *mut _) + }) + }, + _p: PhantomPinned + }), + GFP_KERNEL, + )?; + + // SAFETY: We never move out of the object + let vm_ref = unsafe { + ARef::from_raw(NonNull::new_unchecked(KBox::leak( + Pin::into_inner_unchecked(obj), + ))) + }; + + Ok(vm_ref) + } + + pub fn exec_lock<'a, 'b>( + &'a self, + obj: Option<&'b Object>, + interruptible: bool, + ) -> Result> { + // Do not try to lock the object if it is internal (since it is already locked). + let is_ext = obj.map(|a| self.is_extobj(a)).unwrap_or(false); + + let mut guard = ManuallyDrop::new(LockedGpuVm { + gpuvm: self, + // vm_exec needs to be pinned, so stick it in a Box. + vm_exec: KBox::init( + init!(bindings::drm_gpuvm_exec { + vm: self.gpuvm() as *mut _, + flags: if interruptible { + bindings::DRM_EXEC_INTERRUPTIBLE_WAIT + } else { + 0 + }, + exec: Default::default(), + extra: match (is_ext, obj) { + (true, Some(obj)) => bindings::drm_gpuvm_exec__bindgen_ty_1 { + fn_: Some(exec_lock_gem_object), + priv_: obj.as_raw() as *const _ as *mut _, + }, + _ => Default::default(), + }, + num_fences: 0, + }), + GFP_KERNEL, + )?, + obj, + }); + + // SAFETY: The object is valid and was initialized above + to_result(unsafe { bindings::drm_gpuvm_exec_lock(&mut *guard.vm_exec) })?; + + Ok(ManuallyDrop::into_inner(guard)) + } + + /// Returns true if the given object is external to the GPUVM + /// (that is, if it does not share the DMA reservation object of the GPUVM). + pub fn is_extobj(&self, obj: &impl IntoGEMObject) -> bool { + let gem = obj.as_raw() as *const _ as *mut _; + // SAFETY: This is safe to call as long as the arguments are valid pointers. + unsafe { bindings::drm_gpuvm_is_extobj(self.gpuvm() as *mut _, gem) } + } + + pub fn bo_deferred_cleanup(&self) { + unsafe { bindings::drm_gpuvm_bo_deferred_cleanup(self.gpuvm() as *mut _) } + } + + pub fn find_bo(&self, obj: &Object) -> Option>> { + obj.lock_gpuva(); + // SAFETY: drm_gem_object.gpuva.lock was just locked. + let p = unsafe { + bindings::drm_gpuvm_bo_find(self.gpuvm() as *mut _, obj.as_raw() as *const _ as *mut _) + }; + obj.unlock_gpuva(); + if p.is_null() { + None + } else { + // SAFETY: All the drm_gpuvm_bo objects in this GpuVm are always allocated by us as GpuVmBo. + let p = unsafe { crate::container_of!(p, GpuVmBo, bo) as *mut GpuVmBo }; + // SAFETY: We checked for NULL above, and the types ensure that + // this object was created by vm_bo_alloc_callback. + Some(unsafe { ARef::from_raw(NonNull::new_unchecked(p)) }) + } + } + + pub fn obtain_bo(&self, obj: &Object) -> Result>> { + obj.lock_gpuva(); + // SAFETY: drm_gem_object.gpuva.lock was just locked. + let p = unsafe { + bindings::drm_gpuvm_bo_obtain( + self.gpuvm() as *mut _, + obj.as_raw() as *const _ as *mut _, + ) + }; + obj.unlock_gpuva(); + if p.is_null() { + Err(ENOMEM) + } else { + // SAFETY: Container invariant is guaranteed for GpuVmBo objects for this GpuVm. + let p = unsafe { crate::container_of!(p, GpuVmBo, bo) as *mut GpuVmBo }; + // SAFETY: We checked for NULL above, and the types ensure that + // this object was created by vm_bo_alloc_callback. + Ok(unsafe { ARef::from_raw(NonNull::new_unchecked(p)) }) + } + } + + pub fn bo_unmap(&self, ctx: &mut T::StepContext, bo: &GpuVmBo) -> Result { + let mut ctx = StepContext { ctx, gpuvm: self }; + // SAFETY: LockedGpuVm implies the right locks are held. + to_result(unsafe { + bindings::drm_gpuvm_bo_unmap(&bo.bo as *const _ as *mut _, &mut ctx as *mut _ as *mut _) + }) + } +} + +// SAFETY: DRM GpuVm objects are always reference counted and the get/put functions +// satisfy the requirements. +unsafe impl AlwaysRefCounted for GpuVm { + fn inc_ref(&self) { + // SAFETY: The drm_gpuvm_get function satisfies the requirements for inc_ref(). + unsafe { bindings::drm_gpuvm_get(&self.gpuvm as *const _ as *mut _) }; + } + + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The drm_gpuvm_put function satisfies the requirements for dec_ref(). + unsafe { bindings::drm_gpuvm_put(Opaque::cast_into(&(*obj.as_ptr()).gpuvm)) }; + } +} + +pub struct LockedGpuVm<'a, 'b, T: DriverGpuVm> { + gpuvm: &'a GpuVm, + vm_exec: KBox, + obj: Option<&'b Object>, +} + +impl LockedGpuVm<'_, '_, T> { + pub fn find_bo(&mut self) -> Option>> { + let obj = self.obj?; + // SAFETY: LockedGpuVm implies the right locks are held. + let p = unsafe { + bindings::drm_gpuvm_bo_find( + self.gpuvm.gpuvm() as *mut _, + obj.as_raw() as *const _ as *mut _, + ) + }; + if p.is_null() { + None + } else { + // SAFETY: All the drm_gpuvm_bo objects in this GpuVm are always allocated by us as GpuVmBo. + let p = unsafe { crate::container_of!(p, GpuVmBo, bo) as *mut GpuVmBo }; + // SAFETY: We checked for NULL above, and the types ensure that + // this object was created by vm_bo_alloc_callback. + Some(unsafe { ARef::from_raw(NonNull::new_unchecked(p)) }) + } + } + + pub fn obtain_bo(&mut self) -> Result>> { + let obj = self.obj.ok_or(EINVAL)?; + // SAFETY: LockedGpuVm implies the right locks are held. + let p = unsafe { + bindings::drm_gpuvm_bo_obtain( + self.gpuvm.gpuvm() as *mut _, + obj.as_raw() as *const _ as *mut _, + ) + }; + if p.is_null() { + Err(ENOMEM) + } else { + // SAFETY: Container invariant is guaranteed for GpuVmBo objects for this GpuVm. + let p = unsafe { crate::container_of!(p, GpuVmBo, bo) as *mut GpuVmBo }; + // SAFETY: We checked for NULL above, and the types ensure that + // this object was created by vm_bo_alloc_callback. + Ok(unsafe { ARef::from_raw(NonNull::new_unchecked(p)) }) + } + } + + pub fn sm_map( + &mut self, + ctx: &mut T::StepContext, + req_addr: u64, + req_range: u64, + req_offset: u64, + req_gem_range: u32, + flags: GpuVaFlags, + ) -> Result { + let obj = self.obj.ok_or(EINVAL)?; + let mut ctx = StepContext { + ctx, + gpuvm: self.gpuvm, + }; + + let req = bindings::drm_gpuvm_map_req { + map: bindings::drm_gpuva_op_map { + va: bindings::drm_gpuva_op_map__bindgen_ty_1 { + addr: req_addr, + range: req_range, + }, + gem: bindings::drm_gpuva_op_map__bindgen_ty_2 { + offset: req_offset, + range: req_gem_range, + obj: obj.as_raw(), + }, + flags: flags.as_raw(), + }, + }; + + // SAFETY: LockedGpuVm implies the right locks are held. + to_result(unsafe { + bindings::drm_gpuvm_sm_map( + self.gpuvm.gpuvm() as *mut _, + &mut ctx as *mut _ as *mut _, + &raw const req, + ) + }) + } + + pub fn sm_unmap(&mut self, ctx: &mut T::StepContext, req_addr: u64, req_range: u64) -> Result { + let mut ctx = StepContext { + ctx, + gpuvm: self.gpuvm, + }; + // SAFETY: LockedGpuVm implies the right locks are held. + to_result(unsafe { + bindings::drm_gpuvm_sm_unmap( + self.gpuvm.gpuvm() as *mut _, + &mut ctx as *mut _ as *mut _, + req_addr, + req_range, + ) + }) + } + + pub fn bo_unmap(&mut self, ctx: &mut T::StepContext, bo: &GpuVmBo) -> Result { + let mut ctx = StepContext { + ctx, + gpuvm: self.gpuvm, + }; + // SAFETY: LockedGpuVm implies the right locks are held. + to_result(unsafe { + bindings::drm_gpuvm_bo_unmap(&bo.bo as *const _ as *mut _, &mut ctx as *mut _ as *mut _) + }) + } +} + +impl Deref for LockedGpuVm<'_, '_, T> { + type Target = T; + + fn deref(&self) -> &T { + // SAFETY: The existence of this LockedGpuVm implies the lock is held, + // so this is the only reference + unsafe { &*self.gpuvm.inner.get() } + } +} + +impl DerefMut for LockedGpuVm<'_, '_, T> { + fn deref_mut(&mut self) -> &mut T { + // SAFETY: The existence of this UpdatingGpuVm implies the lock is held, + // so this is the only reference + unsafe { &mut *self.gpuvm.inner.get() } + } +} + +impl Drop for LockedGpuVm<'_, '_, T> { + fn drop(&mut self) { + // SAFETY: We hold the lock, so it's safe to unlock + unsafe { + bindings::drm_gpuvm_exec_unlock(&mut *self.vm_exec); + } + } +} + +pub struct UpdatingGpuVm<'a, T: DriverGpuVm>(&'a GpuVm); + +impl UpdatingGpuVm<'_, T> {} + +impl Deref for UpdatingGpuVm<'_, T> { + type Target = T; + + fn deref(&self) -> &T { + // SAFETY: The existence of this UpdatingGpuVm implies the lock is held, + // so this is the only reference + unsafe { &*self.0.inner.get() } + } +} + +impl DerefMut for UpdatingGpuVm<'_, T> { + fn deref_mut(&mut self) -> &mut T { + // SAFETY: The existence of this UpdatingGpuVm implies the lock is held, + // so this is the only reference + unsafe { &mut *self.0.inner.get() } + } +} + +// SAFETY: All our trait methods take locks +unsafe impl Sync for GpuVm {} +// SAFETY: All our trait methods take locks +unsafe impl Send for GpuVm {} + +// SAFETY: All our trait methods take locks +unsafe impl Sync for GpuVmBo {} +// SAFETY: All our trait methods take locks +unsafe impl Send for GpuVmBo {} diff --git a/rust/kernel/drm/mm.rs b/rust/kernel/drm/mm.rs new file mode 100644 index 00000000000000..7b13cfd7d53095 --- /dev/null +++ b/rust/kernel/drm/mm.rs @@ -0,0 +1,310 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM MM range allocator +//! +//! C header: [`include/drm/drm_mm.h`](../../../../include/drm/drm_mm.h) + +use crate::{ + alloc::flags::*, + bindings, + error::{to_result, Result}, + sync::{new_mutex, Arc, Mutex, UniqueArc}, + types::Opaque, +}; + +use crate::init::InPlaceInit; +use crate::prelude::KBox; + +use core::{ + marker::{PhantomData, PhantomPinned}, + ops::Deref, + pin::Pin, +}; + +/// Type alias representing a DRM MM node. +pub type Node = Pin>>; + +/// Trait which must be implemented by the inner allocator state type provided by the user. +pub trait AllocInner { + /// Notification that a node was dropped from the allocator. + fn drop_object(&mut self, _start: u64, _size: u64, _color: usize, _object: &mut T) {} +} + +impl AllocInner for () {} + +/// Wrapper type for a `struct drm_mm` plus user AllocInner object. +/// +/// # Invariants +/// The `drm_mm` struct is valid and initialized. +struct MmInner, T>(Opaque, A, PhantomData); + +/// Represents a single allocated node in the MM allocator +pub struct NodeData, T> { + node: bindings::drm_mm_node, + mm: Arc>>, + valid: bool, + /// A drm_mm_node needs to be pinned because nodes reference each other in a linked list. + _pin: PhantomPinned, + inner: T, +} + +// SAFETY: Allocator ops take the mutex, and there are no mutable actions on the node. +unsafe impl, T: Send> Send for NodeData {} +// SAFETY: Allocator ops take the mutex, and there are no mutable actions on the node. +unsafe impl, T: Sync> Sync for NodeData {} + +/// Available MM node insertion modes +#[repr(u32)] +pub enum InsertMode { + /// Search for the smallest hole (within the search range) that fits the desired node. + /// + /// Allocates the node from the bottom of the found hole. + Best = bindings::drm_mm_insert_mode_DRM_MM_INSERT_BEST, + + /// Search for the lowest hole (address closest to 0, within the search range) that fits the + /// desired node. + /// + /// Allocates the node from the bottom of the found hole. + Low = bindings::drm_mm_insert_mode_DRM_MM_INSERT_LOW, + + /// Search for the highest hole (address closest to U64_MAX, within the search range) that fits + /// the desired node. + /// + /// Allocates the node from the top of the found hole. The specified alignment for the node is + /// applied to the base of the node (`Node.start()`). + High = bindings::drm_mm_insert_mode_DRM_MM_INSERT_HIGH, + + /// Search for the most recently evicted hole (within the search range) that fits the desired + /// node. This is appropriate for use immediately after performing an eviction scan and removing + /// the selected nodes to form a hole. + /// + /// Allocates the node from the bottom of the found hole. + Evict = bindings::drm_mm_insert_mode_DRM_MM_INSERT_EVICT, +} + +/// A clonable, interlocked reference to the allocator state. +/// +/// This is useful to perform actions on the user-supplied `AllocInner` type given just a Node, +/// without immediately taking the lock. +#[derive(Clone)] +pub struct InnerRef, T>(Arc>>); + +impl, T> InnerRef { + /// Operate on the user `AllocInner` implementation, taking the lock. + pub fn with(&self, cb: impl FnOnce(&mut A) -> RetVal) -> RetVal { + let mut l = self.0.lock(); + cb(&mut l.1) + } +} + +impl, T> NodeData { + /// Returns the color of the node (an opaque value) + pub fn color(&self) -> usize { + self.node.color as usize + } + + /// Returns the start address of the node + pub fn start(&self) -> u64 { + self.node.start + } + + /// Returns the size of the node in bytes + pub fn size(&self) -> u64 { + self.node.size + } + + /// Operate on the user `AllocInner` implementation associated with this node's allocator. + pub fn with_inner(&self, cb: impl FnOnce(&mut A) -> RetVal) -> RetVal { + let mut l = self.mm.lock(); + cb(&mut l.1) + } + + /// Return a clonable, detached reference to the allocator inner data. + pub fn alloc_ref(&self) -> InnerRef { + InnerRef(self.mm.clone()) + } + + /// Return a mutable reference to the inner data. + pub fn inner_mut(self: Pin<&mut Self>) -> &mut T { + // SAFETY: This is okay because inner is not structural + unsafe { &mut self.get_unchecked_mut().inner } + } +} + +impl, T> Deref for NodeData { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl, T> Drop for NodeData { + fn drop(&mut self) { + if self.valid { + let mut guard = self.mm.lock(); + + // Inform the user allocator that a node is being dropped. + guard + .1 + .drop_object(self.start(), self.size(), self.color(), &mut self.inner); + // SAFETY: The MM lock is still taken, so we can safely remove the node. + unsafe { bindings::drm_mm_remove_node(&mut self.node) }; + } + } +} + +/// An instance of a DRM MM range allocator. +pub struct Allocator, T> { + mm: Arc>>, + _p: PhantomData, +} + +impl, T> Allocator { + /// Create a new range allocator for the given start and size range of addresses. + /// + /// The user may optionally provide an inner object representing allocator state, which will + /// be protected by the same lock. If not required, `()` can be used. + #[track_caller] + pub fn new(start: u64, size: u64, inner: A) -> Result> { + // SAFETY: We call `Mutex::init_lock` below. + let mm = UniqueArc::pin_init( + new_mutex!(MmInner(Opaque::uninit(), inner, PhantomData)), + GFP_KERNEL, + )?; + + // SAFETY: The Opaque instance provides a valid pointer, and it is initialized after + // this call. + unsafe { + bindings::drm_mm_init(mm.lock().0.get(), start, size); + } + + Ok(Allocator { + mm: mm.into(), + _p: PhantomData, + }) + } + + /// Insert a new node into the allocator of a given size. + /// + /// `node` is the user `T` type data to store into the node. + pub fn insert_node(&mut self, node: T, size: u64) -> Result> { + self.insert_node_generic(node, size, 0, 0, InsertMode::Best) + } + + /// Insert a new node into the allocator of a given size, with configurable alignment, + /// color, and insertion mode. + /// + /// `node` is the user `T` type data to store into the node. + pub fn insert_node_generic( + &mut self, + node: T, + size: u64, + alignment: u64, + color: usize, + mode: InsertMode, + ) -> Result> { + self.insert_node_in_range(node, size, alignment, color, 0, u64::MAX, mode) + } + + /// Insert a new node into the allocator of a given size, with configurable alignment, + /// color, insertion mode, and sub-range to allocate from. + /// + /// `node` is the user `T` type data to store into the node. + #[allow(clippy::too_many_arguments)] + pub fn insert_node_in_range( + &mut self, + node: T, + size: u64, + alignment: u64, + color: usize, + start: u64, + end: u64, + mode: InsertMode, + ) -> Result> { + let mut mm_node = KBox::new( + NodeData { + // SAFETY: This C struct should be zero-initialized. + node: unsafe { core::mem::zeroed() }, + valid: false, + inner: node, + mm: self.mm.clone(), + _pin: PhantomPinned, + }, + GFP_KERNEL, + )?; + + let guard = self.mm.lock(); + // SAFETY: We hold the lock and all pointers are valid. + to_result(unsafe { + bindings::drm_mm_insert_node_in_range( + guard.0.get(), + &mut mm_node.node, + size, + alignment, + color, + start, + end, + mode as u32, + ) + })?; + + mm_node.valid = true; + + Ok(Pin::from(mm_node)) + } + + /// Insert a node into the allocator at a fixed start address. + /// + /// `node` is the user `T` type data to store into the node. + pub fn reserve_node( + &mut self, + node: T, + start: u64, + size: u64, + color: usize, + ) -> Result> { + let mut mm_node = KBox::new( + NodeData { + // SAFETY: This C struct should be zero-initialized. + node: unsafe { core::mem::zeroed() }, + valid: false, + inner: node, + mm: self.mm.clone(), + _pin: PhantomPinned, + }, + GFP_KERNEL, + )?; + + mm_node.node.start = start; + mm_node.node.size = size; + mm_node.node.color = color as crate::ffi::c_ulong; + + let guard = self.mm.lock(); + // SAFETY: We hold the lock and all pointers are valid. + to_result(unsafe { bindings::drm_mm_reserve_node(guard.0.get(), &mut mm_node.node) })?; + + mm_node.valid = true; + + Ok(Pin::from(mm_node)) + } + + /// Operate on the inner user type `A`, taking the allocator lock + pub fn with_inner(&self, cb: impl FnOnce(&mut A) -> RetVal) -> RetVal { + let mut guard = self.mm.lock(); + cb(&mut guard.1) + } +} + +impl, T> Drop for MmInner { + fn drop(&mut self) { + // SAFETY: If the MmInner is dropped then all nodes are gone (since they hold references), + // so it is safe to tear down the allocator. + unsafe { + bindings::drm_mm_takedown(self.0.get()); + } + } +} + +// SAFETY: MmInner is safely Send if the AllocInner user type is Send. +unsafe impl, T> Send for MmInner {} diff --git a/rust/kernel/drm/mod.rs b/rust/kernel/drm/mod.rs index 1b82b6945edf25..882841415aa414 100644 --- a/rust/kernel/drm/mod.rs +++ b/rust/kernel/drm/mod.rs @@ -6,7 +6,12 @@ pub mod device; pub mod driver; pub mod file; pub mod gem; +#[cfg(CONFIG_DRM_GPUVM = "y")] +pub mod gpuvm; pub mod ioctl; +pub mod mm; +pub mod sched; +pub mod syncobj; pub use self::device::Device; pub use self::driver::Driver; diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs new file mode 100644 index 00000000000000..e2f5cd96014f93 --- /dev/null +++ b/rust/kernel/drm/sched.rs @@ -0,0 +1,391 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM Scheduler +//! +//! C header: [`include/drm/gpu_scheduler.h`](../../../../include/drm/gpu_scheduler.h) + +use crate::{ + bindings, device, + dma_fence::*, + error::{to_result, Result}, + prelude::*, + sync::{Arc, UniqueArc}, + time::{self, msecs_to_jiffies}, +}; +use core::marker::PhantomData; +use core::mem::MaybeUninit; +use core::ops::{Deref, DerefMut}; +use core::ptr::{addr_of, addr_of_mut}; + +/// Scheduler status after timeout recovery +#[repr(u32)] +pub enum Status { + /// Device recovered from the timeout and can execute jobs again + Nominal = bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_RESET, + /// Device is no longer available + NoDevice = bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_ENODEV, +} + +/// Scheduler priorities +#[repr(u32)] +pub enum Priority { + /// Low userspace priority + Low = bindings::drm_sched_priority_DRM_SCHED_PRIORITY_LOW, + /// Normal userspace priority + Normal = bindings::drm_sched_priority_DRM_SCHED_PRIORITY_NORMAL, + /// High userspace priority + High = bindings::drm_sched_priority_DRM_SCHED_PRIORITY_HIGH, + /// Kernel priority (highest) + Kernel = bindings::drm_sched_priority_DRM_SCHED_PRIORITY_KERNEL, +} + +/// Trait to be implemented by driver job objects. +pub trait JobImpl: Sized { + /// Called when the scheduler is considering scheduling this job next, to get another Fence + /// for this job to block on. Once it returns None, run() may be called. + fn prepare(_job: &mut Job) -> Option { + None // Equivalent to NULL function pointer + } + + /// Called to execute the job once all of the dependencies have been resolved. This may be + /// called multiple times, if timed_out() has happened and drm_sched_job_recovery() decides + /// to try it again. + fn run(job: &mut Job) -> Result>; + + /// Called when a job has taken too long to execute, to trigger GPU recovery. + /// + /// This method is called in a workqueue context. + fn timed_out(job: &mut Job) -> Status; + + /// Called for remaining jobs in drm_sched_fini() to ensure the job's fences + /// get signalled before the scheduler is torn down. + fn cancel(job: &mut Job); +} + +unsafe extern "C" fn prepare_job_cb( + sched_job: *mut bindings::drm_sched_job, + _s_entity: *mut bindings::drm_sched_entity, +) -> *mut bindings::dma_fence { + // SAFETY: All of our jobs are Job. + let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; + + // SAFETY: All of our jobs are Job. + match T::prepare(unsafe { &mut *p }) { + None => core::ptr::null_mut(), + Some(fence) => fence.into_raw(), + } +} + +unsafe extern "C" fn run_job_cb( + sched_job: *mut bindings::drm_sched_job, +) -> *mut bindings::dma_fence { + // SAFETY: All of our jobs are Job. + let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; + + // SAFETY: All of our jobs are Job. + match T::run(unsafe { &mut *p }) { + Err(e) => e.to_ptr(), + Ok(None) => core::ptr::null_mut(), + Ok(Some(fence)) => fence.into_raw(), + } +} + +unsafe extern "C" fn timedout_job_cb( + sched_job: *mut bindings::drm_sched_job, +) -> bindings::drm_gpu_sched_stat { + // SAFETY: All of our jobs are Job. + let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; + + // SAFETY: All of our jobs are Job. + T::timed_out(unsafe { &mut *p }) as bindings::drm_gpu_sched_stat +} + +unsafe extern "C" fn free_job_cb(sched_job: *mut bindings::drm_sched_job) { + // SAFETY: All of our jobs are Job. + let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; + + // Convert the job back to a Box and drop it + // SAFETY: All of our Jobs are created inside a box. + unsafe { drop(KBox::from_raw(p)) }; +} + +unsafe extern "C" fn cancel_job_cb(sched_job: *mut bindings::drm_sched_job) { + // SAFETY: All of our jobs are Job. + let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; + + // SAFETY: All of our jobs are Job. + T::cancel(unsafe { &mut *p }); + + let fence = unsafe { Fence::get_raw(&mut (*(*sched_job).s_fence).finished) }; + fence.set_error(ECANCELED); + let _ = fence.signal(); +} + +/// A DRM scheduler job. +pub struct Job { + job: bindings::drm_sched_job, + inner: T, +} + +impl Deref for Job { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl DerefMut for Job { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +impl Drop for Job { + fn drop(&mut self) { + // SAFETY: At this point the job has either been submitted and this is being called from + // `free_job_cb` above, or it hasn't and it is safe to call `drm_sched_job_cleanup`. + unsafe { bindings::drm_sched_job_cleanup(&mut self.job) }; + } +} + +/// A pending DRM scheduler job (not yet armed) +pub struct PendingJob<'a, T: JobImpl>(KBox>, PhantomData<&'a T>); + +impl<'a, T: JobImpl> PendingJob<'a, T> { + /// Add a fence as a dependency to the job + pub fn add_dependency(&mut self, fence: Fence) -> Result { + // SAFETY: C call with correct arguments + to_result(unsafe { + bindings::drm_sched_job_add_dependency(&mut self.0.job, fence.into_raw()) + }) + } + + /// Arm the job to make it ready for execution + pub fn arm(mut self) -> ArmedJob<'a, T> { + // SAFETY: C call with correct arguments + unsafe { bindings::drm_sched_job_arm(&mut self.0.job) }; + ArmedJob(self.0, PhantomData) + } +} + +impl<'a, T: JobImpl> Deref for PendingJob<'a, T> { + type Target = Job; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl<'a, T: JobImpl> DerefMut for PendingJob<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +/// An armed DRM scheduler job (not yet submitted) +pub struct ArmedJob<'a, T: JobImpl>(KBox>, PhantomData<&'a T>); + +impl<'a, T: JobImpl> ArmedJob<'a, T> { + /// Returns the job fences + pub fn fences(&mut self) -> JobFences<'_> { + // SAFETY: s_fence is always a valid drm_sched_fence pointer + JobFences(unsafe { &mut *self.0.job.s_fence }) + } + + /// Push the job for execution into the scheduler + pub fn push(self) { + // After this point, the job is submitted and owned by the scheduler + let ptr = match self { + ArmedJob(job, _) => KBox::>::into_raw(job), + }; + + // SAFETY: We are passing in ownership of a valid Box raw pointer. + unsafe { bindings::drm_sched_entity_push_job(addr_of_mut!((*ptr).job)) }; + } +} +impl<'a, T: JobImpl> Deref for ArmedJob<'a, T> { + type Target = Job; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl<'a, T: JobImpl> DerefMut for ArmedJob<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +/// Reference to the bundle of fences attached to a DRM scheduler job +pub struct JobFences<'a>(&'a mut bindings::drm_sched_fence); + +impl<'a> JobFences<'a> { + /// Returns a new reference to the job scheduled fence. + pub fn scheduled(&mut self) -> Fence { + // SAFETY: self.0.scheduled is always a valid fence + unsafe { Fence::get_raw(&mut self.0.scheduled) } + } + + /// Returns a new reference to the job finished fence. + pub fn finished(&mut self) -> Fence { + // SAFETY: self.0.finished is always a valid fence + unsafe { Fence::get_raw(&mut self.0.finished) } + } +} + +struct EntityInner { + entity: bindings::drm_sched_entity, + // TODO: Allow users to share guilty flag between entities + sched: Arc>, + guilty: bindings::atomic_t, + _p: PhantomData, +} + +impl Drop for EntityInner { + fn drop(&mut self) { + // SAFETY: The EntityInner is initialized. This will cancel/free all jobs. + unsafe { bindings::drm_sched_entity_destroy(&mut self.entity) }; + } +} + +// SAFETY: TODO +unsafe impl Sync for EntityInner {} +// SAFETY: TODO +unsafe impl Send for EntityInner {} + +/// A DRM scheduler entity. +pub struct Entity(Pin>>); + +impl Entity { + /// Create a new scheduler entity. + pub fn new(sched: &Scheduler, priority: Priority) -> Result { + let mut entity: KBox>> = + KBox::new_uninit(GFP_KERNEL | __GFP_ZERO)?; + + let mut sched_ptr = &sched.0.sched as *const _ as *mut _; + + // SAFETY: The Box is allocated above and valid. + unsafe { + bindings::drm_sched_entity_init( + addr_of_mut!((*entity.as_mut_ptr()).entity), + priority as _, + &mut sched_ptr, + 1, + addr_of_mut!((*entity.as_mut_ptr()).guilty), + ) + }; + + // SAFETY: The Box is allocated above and valid. + unsafe { addr_of_mut!((*entity.as_mut_ptr()).sched).write(sched.0.clone()) }; + + // SAFETY: entity is now initialized. + Ok(Self(Pin::from(unsafe { entity.assume_init() }))) + } + + /// Create a new job on this entity. + /// + /// The entity must outlive the pending job until it transitions into the submitted state, + /// after which the scheduler owns it. Since jobs must be submitted in creation order, + /// this requires a mutable reference to the entity, ensuring that only one new job can be + /// in flight at once. + pub fn new_job(&mut self, credits: u32, inner: T) -> Result> { + let mut job: KBox>> = Box::new_uninit(GFP_KERNEL | __GFP_ZERO)?; + + // SAFETY: We hold a reference to the entity (which is a valid pointer), + // and the job object was just allocated above. + to_result(unsafe { + bindings::drm_sched_job_init( + addr_of_mut!((*job.as_mut_ptr()).job), + &self.0.as_ref().get_ref().entity as *const _ as *mut _, + credits, + core::ptr::null_mut(), + 0, + ) + })?; + + // SAFETY: The Box pointer is valid, and this initializes the inner member. + unsafe { addr_of_mut!((*job.as_mut_ptr()).inner).write(inner) }; + + // SAFETY: All fields of the Job are now initialized. + Ok(PendingJob(unsafe { job.assume_init() }, PhantomData)) + } +} + +/// DRM scheduler inner data +pub struct SchedulerInner { + sched: bindings::drm_gpu_scheduler, + _p: PhantomData, +} + +impl Drop for SchedulerInner { + fn drop(&mut self) { + // SAFETY: The scheduler is valid. This assumes drm_sched_fini() will take care of + // freeing all in-progress jobs. + unsafe { bindings::drm_sched_stop(&mut self.sched, core::ptr::null_mut()) }; + unsafe { bindings::drm_sched_fini(&mut self.sched) }; + } +} + +// SAFETY: TODO +unsafe impl Sync for SchedulerInner {} +// SAFETY: TODO +unsafe impl Send for SchedulerInner {} + +/// A DRM Scheduler +pub struct Scheduler(Arc>); + +impl Scheduler { + const OPS: bindings::drm_sched_backend_ops = bindings::drm_sched_backend_ops { + prepare_job: Some(prepare_job_cb::), + run_job: Some(run_job_cb::), + timedout_job: Some(timedout_job_cb::), + free_job: Some(free_job_cb::), + cancel_job: Some(cancel_job_cb::), + }; + /// Creates a new DRM Scheduler object + // TODO: Shared timeout workqueues & scores + pub fn new( + device: &device::Device, + num_rqs: u32, + credit_limit: u32, + hang_limit: u32, + timeout_ms: time::Msecs, + name: &'static CStr, + ) -> Result> { + let mut sched: UniqueArc>> = + UniqueArc::new_uninit(GFP_KERNEL)?; + + // SAFETY: zero sched->sched_rq as drm_sched_init() uses it to exit early withoput initialisation + // TODO: allocate sched zzeroed instead + unsafe { + (*sched.as_mut_ptr()).sched.sched_rq = core::ptr::null_mut(); + }; + + let init_ops = bindings::drm_sched_init_args { + ops: &Self::OPS, + submit_wq: core::ptr::null_mut(), + timeout_wq: core::ptr::null_mut(), + num_rqs, + credit_limit, + hang_limit, + timeout: msecs_to_jiffies(timeout_ms).try_into()?, + score: core::ptr::null_mut(), + name: name.as_char_ptr(), + dev: device.as_raw(), + }; + + // SAFETY: The drm_sched pointer is valid and pinned as it was just allocated above. + // `device` is valid by its type invarants + to_result(unsafe { + bindings::drm_sched_init( + addr_of_mut!((*sched.as_mut_ptr()).sched), + addr_of!(init_ops), + ) + })?; + + // SAFETY: All fields of SchedulerInner are now initialized. + Ok(Scheduler(unsafe { sched.assume_init() }.into())) + } +} diff --git a/rust/kernel/drm/syncobj.rs b/rust/kernel/drm/syncobj.rs new file mode 100644 index 00000000000000..a022e08223588b --- /dev/null +++ b/rust/kernel/drm/syncobj.rs @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM Sync Objects +//! +//! C header: [`include/drm/drm_syncobj.h`](../../../../include/drm/drm_syncobj.h) + +use crate::{bindings, dma_fence::*, drm, error::Result, prelude::*}; + +/// A DRM Sync Object +/// +/// # Invariants +/// ptr is a valid pointer to a drm_syncobj and we own a reference to it. +pub struct SyncObj { + ptr: *mut bindings::drm_syncobj, +} + +impl SyncObj { + /// Looks up a sync object by its handle for a given `File`. + pub fn lookup_handle( + file: &drm::File, + handle: u32, + ) -> Result { + // SAFETY: The arguments are all valid per the type invariants. + let ptr = unsafe { bindings::drm_syncobj_find(file.as_raw() as *mut _, handle) }; + + if ptr.is_null() { + Err(ENOENT) + } else { + Ok(SyncObj { ptr }) + } + } + + /// Returns the DMA fence associated with this sync object, if any. + pub fn fence_get(&self) -> Option { + // SAFETY: self.ptr is always valid + let fence = unsafe { bindings::drm_syncobj_fence_get(self.ptr) }; + if fence.is_null() { + None + } else { + // SAFETY: The pointer is non-NULL and drm_syncobj_fence_get acquired an + // additional reference. + Some(unsafe { Fence::from_raw(fence) }) + } + } + + /// Replaces the DMA fence with a new one, or removes it if fence is None. + pub fn replace_fence(&self, fence: Option<&Fence>) { + // SAFETY: All arguments should be valid per the respective type invariants. + unsafe { + bindings::drm_syncobj_replace_fence( + self.ptr, + fence.map_or(core::ptr::null_mut(), |a| a.raw()), + ) + }; + } + + /// Adds a new timeline point to the syncobj. + pub fn add_point(&self, chain: FenceChain, fence: &Fence, point: u64) { + // SAFETY: All arguments should be valid per the respective type invariants. + // This takes over the FenceChain ownership. + unsafe { bindings::drm_syncobj_add_point(self.ptr, chain.into_raw(), fence.raw(), point) }; + } +} + +impl Drop for SyncObj { + fn drop(&mut self) { + // SAFETY: We own a reference to this syncobj. + unsafe { bindings::drm_syncobj_put(self.ptr) }; + } +} + +impl Clone for SyncObj { + fn clone(&self) -> Self { + // SAFETY: `ptr` is valid per the type invariant and we own a reference to it. + unsafe { bindings::drm_syncobj_get(self.ptr) }; + SyncObj { ptr: self.ptr } + } +} + +// SAFETY: drm_syncobj operations are internally locked. +unsafe impl Sync for SyncObj {} +// SAFETY: drm_syncobj operations are internally locked. +unsafe impl Send for SyncObj {} diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index 258b12afdcba35..bc116561f6c09c 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -66,8 +66,11 @@ pub mod code { declare_err!(EPIPE, "Broken pipe."); declare_err!(EDOM, "Math argument out of domain of func."); declare_err!(ERANGE, "Math result not representable."); + declare_err!(ENOSYS, "Invalid system call number."); + declare_err!(ENODATA, "No data available."); declare_err!(EOVERFLOW, "Value too large for defined data type."); declare_err!(ETIMEDOUT, "Connection timed out."); + declare_err!(ECANCELED, "Operation Canceled."); declare_err!(ERESTARTSYS, "Restart the system call."); declare_err!(ERESTARTNOINTR, "System call was interrupted by a signal and will be restarted."); declare_err!(ERESTARTNOHAND, "Restart if no handler."); diff --git a/rust/kernel/iio/common/aop_sensors.rs b/rust/kernel/iio/common/aop_sensors.rs new file mode 100644 index 00000000000000..de0835784b66a8 --- /dev/null +++ b/rust/kernel/iio/common/aop_sensors.rs @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Apple AOP sensors common code +//! +//! Copyright (C) The Asahi Linux Contributors + +use core::marker::{PhantomData, PhantomPinned}; +use core::ptr; +use core::sync::atomic::{AtomicU32, Ordering}; + +use kernel::{ + bindings, device, + prelude::*, + soc::apple::aop::FakehidListener, + sync::Arc, + types::{ARef, ForeignOwnable}, + ThisModule, +}; + +/// TODO: add documentation +pub trait MessageProcessor { + /// TODO: add documentation + fn process(&self, message: &[u8]) -> u32; +} + +/// TODO: add documentation +pub struct AopSensorData { + dev: ARef, + ty: u32, + value: AtomicU32, + msg_proc: T, +} + +impl AopSensorData { + /// TODO: add documentation + pub fn new(dev: ARef, ty: u32, msg_proc: T) -> Result>> { + Ok(Arc::new( + AopSensorData { + dev, + ty, + value: AtomicU32::new(0), + msg_proc, + }, + GFP_KERNEL, + )?) + } +} + +impl FakehidListener for AopSensorData { + fn process_fakehid_report(&self, data: &[u8]) -> Result<()> { + self.value + .store(self.msg_proc.process(data), Ordering::Relaxed); + Ok(()) + } +} + +unsafe extern "C" fn aop_read_raw( + dev: *mut bindings::iio_dev, + chan: *const bindings::iio_chan_spec, + val: *mut i32, + _: *mut i32, + mask: isize, +) -> i32 { + let data = unsafe { Arc::>::borrow((*dev).priv_.cast()) }; + let ty = unsafe { (*chan).type_ }; + if mask != bindings::BINDINGS_IIO_CHAN_INFO_PROCESSED as isize + && mask != bindings::BINDINGS_IIO_CHAN_INFO_RAW as isize + { + return EINVAL.to_errno(); + } + if data.ty != ty { + return EINVAL.to_errno(); + } + let value = data.value.load(Ordering::Relaxed); + unsafe { + *val = value as i32; + } + bindings::IIO_VAL_INT as i32 +} + +struct IIOSpec { + spec: [bindings::iio_chan_spec; 1], + vtable: bindings::iio_info, + _p: PhantomPinned, +} + +/// TODO: add documentation +pub struct IIORegistration { + dev: *mut bindings::iio_dev, + spec: Pin>, + registered: bool, + _p: PhantomData>, +} + +impl IIORegistration { + /// TODO: add documentation + pub fn new( + data: Arc>, + name: &'static CStr, + ty: u32, + info_mask: usize, + module: &ThisModule, + ) -> Result { + let spec = KBox::pin( + IIOSpec { + spec: [bindings::iio_chan_spec { + type_: ty, + __bindgen_anon_1: bindings::iio_chan_spec__bindgen_ty_1 { + scan_type: bindings::iio_scan_type { + sign: b'u' as _, + realbits: 32, + storagebits: 32, + ..Default::default() + }, + }, + info_mask_separate: info_mask, + ..Default::default() + }], + vtable: bindings::iio_info { + read_raw: Some(aop_read_raw::), + ..Default::default() + }, + _p: PhantomPinned, + }, + GFP_KERNEL, + )?; + let mut this = IIORegistration { + dev: ptr::null_mut(), + spec, + registered: false, + _p: PhantomData, + }; + this.dev = unsafe { bindings::iio_device_alloc(data.dev.as_raw(), 0) }; + unsafe { + (*this.dev).priv_ = data.clone().into_foreign().cast(); + (*this.dev).name = name.as_ptr() as _; + // spec is now pinned + (*this.dev).channels = this.spec.spec.as_ptr(); + (*this.dev).num_channels = this.spec.spec.len() as i32; + (*this.dev).info = &this.spec.vtable; + } + let ret = unsafe { bindings::__iio_device_register(this.dev, module.as_ptr()) }; + if ret < 0 { + dev_err!(data.dev, "Unable to register iio sensor"); + return Err(Error::from_errno(ret)); + } + this.registered = true; + Ok(this) + } +} + +impl Drop for IIORegistration { + fn drop(&mut self) { + if self.dev != ptr::null_mut() { + unsafe { + if self.registered { + bindings::iio_device_unregister(self.dev); + } + Arc::>::from_foreign((*self.dev).priv_.cast()); + bindings::iio_device_free(self.dev); + } + } + } +} + +unsafe impl Send for IIORegistration {} +unsafe impl Sync for IIORegistration {} diff --git a/rust/kernel/iio/common/mod.rs b/rust/kernel/iio/common/mod.rs new file mode 100644 index 00000000000000..b789e9bf44c9bf --- /dev/null +++ b/rust/kernel/iio/common/mod.rs @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! IIO common modules + +#[cfg(any(CONFIG_IIO_AOP_SENSOR_LAS, CONFIG_IIO_AOP_SENSOR_ALS,))] +pub mod aop_sensors; diff --git a/rust/kernel/iio/mod.rs b/rust/kernel/iio/mod.rs new file mode 100644 index 00000000000000..b0cb308f0b454c --- /dev/null +++ b/rust/kernel/iio/mod.rs @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT + +//! Industrial IO drivers + +pub mod common; diff --git a/rust/kernel/impl_flags.rs b/rust/kernel/impl_flags.rs new file mode 100644 index 00000000000000..e2bd7639da12e0 --- /dev/null +++ b/rust/kernel/impl_flags.rs @@ -0,0 +1,272 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Bitflag type generator. + +/// Common helper for declaring bitflag and bitmask types. +/// +/// This macro takes as input: +/// - A struct declaration representing a bitmask type +/// (e.g., `pub struct Permissions(u32)`). +/// - An enumeration declaration representing individual bit flags +/// (e.g., `pub enum Permission { ... }`). +/// +/// And generates: +/// - The struct and enum types with appropriate `#[repr]` attributes. +/// - Implementations of common bitflag operators +/// ([`::core::ops::BitOr`], [`::core::ops::BitAnd`], etc.). +/// - Utility methods such as `.contains()` to check flags. +/// +/// # Examples +/// +/// ``` +/// use kernel::impl_flags; +/// +/// impl_flags!( +/// /// Represents multiple permissions. +/// #[derive(Debug, Clone, Default, Copy, PartialEq, Eq)] +/// pub struct Permissions(u32); +/// +/// /// Represents a single permission. +/// #[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// pub enum Permission { +/// /// Read permission. +/// Read = 1 << 0, +/// +/// /// Write permission. +/// Write = 1 << 1, +/// +/// /// Execute permission. +/// Execute = 1 << 2, +/// } +/// ); +/// +/// // Combine multiple permissions using the bitwise OR (`|`) operator. +/// let mut read_write: Permissions = Permission::Read | Permission::Write; +/// assert!(read_write.contains(Permission::Read)); +/// assert!(read_write.contains(Permission::Write)); +/// assert!(!read_write.contains(Permission::Execute)); +/// assert!(read_write.contains_any(Permission::Read | Permission::Execute)); +/// assert!(read_write.contains_all(Permission::Read | Permission::Write)); +/// +/// // Using the bitwise OR assignment (`|=`) operator. +/// read_write |= Permission::Execute; +/// assert!(read_write.contains(Permission::Execute)); +/// +/// // Masking a permission with the bitwise AND (`&`) operator. +/// let read_only: Permissions = read_write & Permission::Read; +/// assert!(read_only.contains(Permission::Read)); +/// assert!(!read_only.contains(Permission::Write)); +/// +/// // Toggling permissions with the bitwise XOR (`^`) operator. +/// let toggled: Permissions = read_only ^ Permission::Read; +/// assert!(!toggled.contains(Permission::Read)); +/// +/// // Inverting permissions with the bitwise NOT (`!`) operator. +/// let negated = !read_only; +/// assert!(negated.contains(Permission::Write)); +/// assert!(!negated.contains(Permission::Read)); +/// ``` +#[macro_export] +macro_rules! impl_flags { + ( + $(#[$outer_flags:meta])* + $vis_flags:vis struct $flags:ident($ty:ty); + + $(#[$outer_flag:meta])* + $vis_flag:vis enum $flag:ident { + $( + $(#[$inner_flag:meta])* + $name:ident = $value:expr + ),+ $( , )? + } + ) => { + $(#[$outer_flags])* + #[repr(transparent)] + $vis_flags struct $flags($ty); + + $(#[$outer_flag])* + #[repr($ty)] + $vis_flag enum $flag { + $( + $(#[$inner_flag])* + $name = $value + ),+ + } + + impl ::core::convert::From<$flag> for $flags { + #[inline] + fn from(value: $flag) -> Self { + Self(value as $ty) + } + } + + impl ::core::convert::From<$flags> for $ty { + #[inline] + fn from(value: $flags) -> Self { + value.0 + } + } + + impl ::core::ops::BitOr for $flags { + type Output = Self; + #[inline] + fn bitor(self, rhs: Self) -> Self::Output { + Self(self.0 | rhs.0) + } + } + + impl ::core::ops::BitOrAssign for $flags { + #[inline] + fn bitor_assign(&mut self, rhs: Self) { + *self = *self | rhs; + } + } + + impl ::core::ops::BitOr<$flag> for $flags { + type Output = Self; + #[inline] + fn bitor(self, rhs: $flag) -> Self::Output { + self | Self::from(rhs) + } + } + + impl ::core::ops::BitOrAssign<$flag> for $flags { + #[inline] + fn bitor_assign(&mut self, rhs: $flag) { + *self = *self | rhs; + } + } + + impl ::core::ops::BitAnd for $flags { + type Output = Self; + #[inline] + fn bitand(self, rhs: Self) -> Self::Output { + Self(self.0 & rhs.0) + } + } + + impl ::core::ops::BitAndAssign for $flags { + #[inline] + fn bitand_assign(&mut self, rhs: Self) { + *self = *self & rhs; + } + } + + impl ::core::ops::BitAnd<$flag> for $flags { + type Output = Self; + #[inline] + fn bitand(self, rhs: $flag) -> Self::Output { + self & Self::from(rhs) + } + } + + impl ::core::ops::BitAndAssign<$flag> for $flags { + #[inline] + fn bitand_assign(&mut self, rhs: $flag) { + *self = *self & rhs; + } + } + + impl ::core::ops::BitXor for $flags { + type Output = Self; + #[inline] + fn bitxor(self, rhs: Self) -> Self::Output { + Self((self.0 ^ rhs.0) & Self::all_bits()) + } + } + + impl ::core::ops::BitXorAssign for $flags { + #[inline] + fn bitxor_assign(&mut self, rhs: Self) { + *self = *self ^ rhs; + } + } + + impl ::core::ops::BitXor<$flag> for $flags { + type Output = Self; + #[inline] + fn bitxor(self, rhs: $flag) -> Self::Output { + self ^ Self::from(rhs) + } + } + + impl ::core::ops::BitXorAssign<$flag> for $flags { + #[inline] + fn bitxor_assign(&mut self, rhs: $flag) { + *self = *self ^ rhs; + } + } + + impl ::core::ops::Not for $flags { + type Output = Self; + #[inline] + fn not(self) -> Self::Output { + Self((!self.0) & Self::all_bits()) + } + } + + impl ::core::ops::BitOr for $flag { + type Output = $flags; + #[inline] + fn bitor(self, rhs: Self) -> Self::Output { + $flags(self as $ty | rhs as $ty) + } + } + + impl ::core::ops::BitAnd for $flag { + type Output = $flags; + #[inline] + fn bitand(self, rhs: Self) -> Self::Output { + $flags(self as $ty & rhs as $ty) + } + } + + impl ::core::ops::BitXor for $flag { + type Output = $flags; + #[inline] + fn bitxor(self, rhs: Self) -> Self::Output { + $flags((self as $ty ^ rhs as $ty) & $flags::all_bits()) + } + } + + impl ::core::ops::Not for $flag { + type Output = $flags; + #[inline] + fn not(self) -> Self::Output { + $flags((!(self as $ty)) & $flags::all_bits()) + } + } + + impl $flags { + /// Returns an empty instance where no flags are set. + #[inline] + pub const fn empty() -> Self { + Self(0) + } + + /// Returns a mask containing all valid flag bits. + #[inline] + pub const fn all_bits() -> $ty { + 0 $( | $value )+ + } + + /// Checks if a specific flag is set. + #[inline] + pub fn contains(self, flag: $flag) -> bool { + (self.0 & flag as $ty) == flag as $ty + } + + /// Checks if at least one of the provided flags is set. + #[inline] + pub fn contains_any(self, flags: $flags) -> bool { + (self.0 & flags.0) != 0 + } + + /// Checks if all of the provided flags are set. + #[inline] + pub fn contains_all(self, flags: $flags) -> bool { + (self.0 & flags.0) == flags.0 + } + } + }; +} diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index 899b9a96276235..1ee65c555a6dda 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -219,17 +219,17 @@ pub trait InPlaceInit: Sized { /// [`Error`]: crate::error::Error #[macro_export] macro_rules! try_init { - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + ($(&$this:ident in)? $t:ident $(::$p:ident)* $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }) => { - ::pin_init::try_init!($(&$this in)? $t $(::<$($generics),*>)? { + ::pin_init::try_init!($(&$this in)? $t $(::$p)* $(::<$($generics),*>)? { $($fields)* }? $crate::error::Error) }; - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + ($(&$this:ident in)? $t:ident $(::$p:ident)* $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }? $err:ty) => { - ::pin_init::try_init!($(&$this in)? $t $(::<$($generics),*>)? { + ::pin_init::try_init!($(&$this in)? $t $(::$p)* $(::<$($generics),*>)? { $($fields)* }? $err) }; @@ -279,17 +279,17 @@ macro_rules! try_init { /// [`Error`]: crate::error::Error #[macro_export] macro_rules! try_pin_init { - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + ($(&$this:ident in)? $t:ident $(::$p:ident)* $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }) => { - ::pin_init::try_pin_init!($(&$this in)? $t $(::<$($generics),*>)? { + ::pin_init::try_pin_init!($(&$this in)? $t $(::$p)* $(::<$($generics),*>)? { $($fields)* }? $crate::error::Error) }; - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + ($(&$this:ident in)? $t:ident $(::$p:ident)* $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }? $err:ty) => { - ::pin_init::try_pin_init!($(&$this in)? $t $(::<$($generics),*>)? { + ::pin_init::try_pin_init!($(&$this in)? $t $(::$p)* $(::<$($generics),*>)? { $($fields)* }? $err) }; diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index b64b11f75a353e..e58f7b18489f1e 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -230,6 +230,15 @@ impl Io { } } + #[inline] + const fn length_valid(offset: usize, length: usize, size: usize) -> bool { + if let Some(end) = offset.checked_add(length) { + end <= size + } else { + false + } + } + #[inline] fn io_addr(&self, offset: usize) -> Result { if !Self::offset_valid::(offset, self.maxsize()) { @@ -249,6 +258,67 @@ impl Io { self.addr() + offset } + /// Copy memory block from an i/o memory by filling the specified buffer with it. + /// + /// # Examples + /// ``` + /// use kernel::io::mem::IoMem; + /// use kernel::io::mem::Resource; + /// + /// fn test(device: &Device, res: Resource) -> Result { + /// // Create an i/o memory block of at least 100 bytes. + /// let devres_mem = IoMem::<100>::new(res, device)?; + /// // aquire access to memory block + /// let mem = devres_mem.try_access()?; + /// + /// let mut buffer: [u8; 32] = [0; 32]; + /// + /// // Memcpy 16 bytes from an offset 10 of i/o memory block into the buffer. + /// mem.try_memcpy_fromio(&mut buffer[..16], 10)?; + /// + /// Ok(()) + /// } + /// ``` + pub fn try_memcpy_fromio(&self, buffer: &mut [u8], offset: usize) -> Result { + if buffer.len() == 0 || !Self::length_valid(offset, buffer.len(), self.maxsize()) { + return Err(EINVAL); + } + let addr = self.io_addr::(offset)?; + + // SAFETY: + // - The type invariants guarantee that `adr` is a valid pointer. + // - The bounds of `buffer` are checked with a call to `length_valid`. + unsafe { + bindings::memcpy_fromio( + buffer.as_mut_ptr() as *mut _, + addr as *const _, + buffer.len() as _, + ) + }; + Ok(()) + } + + /// Copy memory block to i/o memory from the specified buffer. + pub fn try_memcpy_toio(&self, offset: usize, buffer: &[u8]) -> Result { + if buffer.len() == 0 || !Self::length_valid(offset, buffer.len(), self.maxsize()) { + return Err(EINVAL); + } + // no need to check since offset + buffer.len() - 1 is valid + let addr = self.io_addr::(offset)?; + + // SAFETY: + // - The type invariants guarantee that `adr` is a valid pointer. + // - The bounds of `buffer` are checked with a call to `length_valid`. + unsafe { + bindings::memcpy_toio( + addr as *mut _, + buffer.as_ptr() as *const _, + buffer.len() as _, + ) + }; + Ok(()) + } + define_read!(read8, try_read8, readb -> u8); define_read!(read16, try_read16, readw -> u16); define_read!(read32, try_read32, readl -> u32); diff --git a/rust/kernel/io/mem.rs b/rust/kernel/io/mem.rs index b03b82cd531b2d..40d9d193816ff7 100644 --- a/rust/kernel/io/mem.rs +++ b/rust/kernel/io/mem.rs @@ -3,6 +3,7 @@ //! Generic memory-mapped IO. use core::ops::Deref; +use core::ptr::NonNull; use crate::{ c_str, @@ -11,6 +12,7 @@ use crate::{ Device, // }, devres::Devres, + impl_flags, io::{ self, resource::{ @@ -285,3 +287,105 @@ impl Deref for IoMem { unsafe { Io::from_raw(&self.io) } } } + +impl_flags!( + /// Flags to be used when remapping memory. + #[derive(Debug, Clone, Default, Copy, PartialEq, Eq)] + pub struct MemFlags(usize); + + /// Enum mirroring the C MEMREMAP_* eum values + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum MemFlag { + /// Matches the default mapping for System RAM on the architecture. + /// + /// This is usually a read-allocate write-back cache. Moreover, if this flag is specified and + /// the requested remap region is RAM, memremap() will bypass establishing a new mapping and + /// instead return a pointer into the direct map. + WB = bindings::MEMREMAP_WB as usize, + + /// Establish a mapping whereby writes either bypass the cache or are written through to memory + /// and never exist in a cache-dirty state with respect to program visibility. + /// + /// Attempts to map System RAM with this mapping type will fail. + WT = bindings::MEMREMAP_WT as usize, + + /// Establish a writecombine mapping, whereby writes may be coalesced together (e.g. in the + /// CPU's write buffers), but is otherwise uncached. + /// + /// Attempts to map System RAM with this mapping type will fail. + WC = bindings::MEMREMAP_WC as usize, + // Note: Skipping MEMREMAP_ENC/DEC since they are under-documented and have zero + // users outside of arch/x86. + } +); + +/// Represents a non-MMIO memory block. This is like [`IoMem`], but for cases where it is known +/// that the resource being mapped does not have I/O side effects. +// Invariants: +// `ptr` is a non-null and valid address of at least `usize` bytes and returned by a `memremap` +// call. +// ``` +pub struct Mem { + ptr: NonNull, + size: usize, +} + +impl Mem { + /// Tries to create a new instance of a memory block from a Resource. + /// + /// The resource described by `res` is mapped into the CPU's address space so that it can be + /// accessed directly. It is also consumed by this function so that it can't be mapped again + /// to a different address. + /// + /// If multiple caching flags are specified, the different mapping types will be attempted in + /// the order [`MemFlag::WB`], [`MemFlag::WT`], [`MemFlag::WC`]. + /// + /// # Flags + /// + /// * [`MemFlag::WB`]: Matches the default mapping for System RAM on the architecture. + /// This is usually a read-allocate write-back cache. Moreover, if this flag is specified and + /// the requested remap region is RAM, memremap() will bypass establishing a new mapping and + /// instead return a pointer into the direct map. + /// + /// * [`MemFlag::WT`]: Establish a mapping whereby writes either bypass the cache or are written + /// through to memory and never exist in a cache-dirty state with respect to program visibility. + /// Attempts to map System RAM with this mapping type will fail. + /// * [`MemFlag::WC`]: Establish a writecombine mapping, whereby writes may be coalesced together + /// (e.g. in the CPU's write buffers), but is otherwise uncached. Attempts to map System RAM with + /// this mapping type will fail. + /// + /// # Safety + /// + /// Callers must ensure that either (a) the resulting interface cannot be used to initiate DMA + /// operations, or (b) that DMA operations initiated via the returned interface use DMA handles + /// allocated through the `dma` module. + pub unsafe fn try_new(res: Resource, flags: MemFlags) -> Result { + let size: usize = res.size().try_into()?; + + let addr = unsafe { bindings::memremap(res.start(), size, flags.into()) }; + let ptr = NonNull::new(addr).ok_or(ENOMEM)?; + // INVARIANT: `ptr` is non-null and was returned by `memremap`, so it is valid. + Ok(Self { ptr, size }) + } + + /// Returns the base address of the memory mapping as a raw pointer. + /// + /// It is up to the caller to use this pointer safely, depending on the requirements of the + /// hardware backing this memory block. + pub fn ptr(&self) -> *mut u8 { + self.ptr.cast().as_ptr() + } + + /// Returns the size of this mapped memory block. + pub fn size(&self) -> usize { + self.size + } +} + +impl Drop for Mem { + fn drop(&mut self) { + // SAFETY: By the type invariant, `self.ptr` is a value returned by a previous successful + // call to `memremap`. + unsafe { bindings::memunmap(self.ptr.as_ptr()) }; + } +} diff --git a/rust/kernel/io/resource.rs b/rust/kernel/io/resource.rs index b7ac9faf141d2e..2c7fe290b51440 100644 --- a/rust/kernel/io/resource.rs +++ b/rust/kernel/io/resource.rs @@ -79,6 +79,18 @@ unsafe impl Sync for Region {} pub struct Resource(Opaque); impl Resource { + /// Create a new zeroed [`Resource`] + pub(crate) fn zeroed() -> Self { + Resource { + 0: Opaque::::zeroed(), + } + } + + /// Gets the raw pointer to the wrapped `bindings::resource`. + pub(crate) fn as_raw(&self) -> *mut bindings::resource { + self.0.get() + } + /// Creates a reference to a [`Resource`] from a valid pointer. /// /// # Safety diff --git a/rust/kernel/iosys_map.rs b/rust/kernel/iosys_map.rs new file mode 100644 index 00000000000000..039137c051a2f1 --- /dev/null +++ b/rust/kernel/iosys_map.rs @@ -0,0 +1,647 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! IO-agnostic memory mapping interfaces. +//! +//! This crate provides bindings for the `struct iosys_map` type, which provides a common interface +//! for memory mappings which can reside within coherent memory, or within IO memory. +//! +//! C header: [`include/linux/iosys-map.h`](srctree/include/linux/pci.h) + +use crate::{ + prelude::*, + transmute::{AsBytes, FromBytes}, +}; +use bindings; +use core::{ + marker::PhantomData, + mem::{self, MaybeUninit}, + ops::{Deref, DerefMut, Range}, +}; + +/// Raw unsized representation of a `struct iosys_map`. +/// +/// This struct is a transparent wrapper around `struct iosys_map`. The C API does not provide the +/// size of the mapping by default, and thus this type also does not include the size of the +/// mapping. As such, it cannot be used for actually accessing the underlying data pointed to by the +/// mapping. +/// +/// With the exception of kernel crates which may provide their own wrappers around `RawIoSysMap`, +/// users will typically not interact with this type directly. +pub struct RawIoSysMap(bindings::iosys_map, PhantomData); + +impl RawIoSysMap { + /// Convert from a raw `bindings::iosys_map`. + #[allow(unused)] + #[inline] + pub(crate) fn from_raw(val: bindings::iosys_map) -> Self { + Self(val, PhantomData) + } + + /// Convert from a `RawIoSysMap` to a raw `bindings::iosys_map` ref. + #[inline] + pub(crate) fn as_raw(&self) -> &bindings::iosys_map { + &self.0 + } + + /// Convert from a `RawIoSysMap` to a raw mutable `bindings::iosys_map` ref. + #[inline] + pub(crate) fn as_raw_mut(&mut self) -> &mut bindings::iosys_map { + &mut self.0 + } + + /// Returns whether the mapping is within IO memory space or not. + #[inline] + pub fn is_iomem(&self) -> bool { + self.0.is_iomem + } + + /// Returns the size of a single item in this mapping. + pub const fn item_size(&self) -> usize { + mem::size_of::() + } + + /// Returns a mutable address to the memory pointed to by this iosys map. + /// + /// Note that this address is not guaranteed to reside in system memory, and may reside in IO + /// memory. + #[inline] + pub fn as_mut_ptr(&self) -> *mut T { + if self.is_iomem() { + // SAFETY: We confirmed above that this iosys map is contained within iomem, so it's + // safe to read vaddr_iomem + unsafe { self.0.__bindgen_anon_1.vaddr_iomem } + } else { + // SAFETY: We confirmed above that this iosys map is not contaned within iomem, so it's + // safe to read vaddr. + unsafe { self.0.__bindgen_anon_1.vaddr } + } + .cast() + } + + /// Returns an immutable address to the memory pointed to by this iosys map. + /// + /// Note that this address is not guaranteed to reside in system memory, and may reside in IO + /// memory. + #[inline] + pub fn as_ptr(&self) -> *const T { + self.as_mut_ptr().cast_const() + } +} + +// SAFETY: As we make no guarantees about the validity of the mapping, there's no issue with sending +// this type between threads. +unsafe impl Send for RawIoSysMap {} + +impl Clone for RawIoSysMap { + fn clone(&self) -> Self { + Self(self.0, PhantomData) + } +} + +/// A sized version of a [`RawIoSysMap`]. +/// +/// Since this type includes the size of the [`RawIoSysMap`], it can be used for accessing the +/// underlying data pointed to by it. +/// +/// # Invariants +/// +/// - The iosys mapping referenced by this type is guaranteed to be of at least `size` bytes in +/// size +/// - The iosys mapping referenced by this type is valid for the lifetime `'a`. +#[derive(Clone)] +pub struct IoSysMapRef<'a, T: AsBytes + FromBytes> { + map: RawIoSysMap, + size: usize, + _p: PhantomData<&'a T>, +} + +impl<'a, T: AsBytes + FromBytes> IoSysMapRef<'a, T> { + /// Create a new [`IoSysMapRef`] from a [`RawIoSysMap`]. + /// + /// # Safety + /// + /// - The caller guarantees that the mapping referenced by `map` is of at least `size` bytes in + /// size. + /// - The caller guarantees that the mapping referenced by `map` remains valid for the lifetime + /// of `'a`. + #[allow(unused)] + pub(crate) unsafe fn new(map: RawIoSysMap, size: usize) -> IoSysMapRef<'a, T> { + // INVARIANT: Our safety contract fulfills the type invariants of `IoSysMapRef`. + IoSysMapRef { + map, + size, + _p: PhantomData, + } + } + + /// Return the size of the `IoSysMapRef`. + #[inline] + pub fn size(&self) -> usize { + self.size + } + + /// Writes `src` to the region starting from `offset`. + /// + /// `offset` is in units of `T`, not the number of bytes. + /// + /// This function can return the following errors: + /// + /// * [`EOVERFLOW`] if calculating the length of the slice results in an overflow. + /// * [`EINVAL`] if the slice would go out of bounds of the memory region. + /// + /// # Examples + /// + /// ``` + /// use kernel::iosys_map::*; + /// + /// # fn test() -> Result { + /// # let mut map = tests::VecIoSysMap::new(&[0; 3])?; + /// # { + /// # let mut map = map.get(); + /// map.write(&[1, 2, 3], 0)?; // (now [1, 2, 3]) + /// map.write(&[4], 2)?; // (now [1, 2, 4]) + /// # } + /// # + /// # map.assert_eq(&[1, 2, 4]); + /// # + /// # Ok::<(), Error>(()) } + /// # assert!(test().is_ok()); + /// ``` + pub fn write(&mut self, src: &[T], offset: usize) -> Result { + let range = self.compute_range(offset, src.len())?; + + // SAFETY: + // - The address pointed to by this iosys_map is guaranteed to be valid via IoSysMapRef's + // type invariants. + // - We checked that this range of memory is within bounds above + unsafe { + bindings::iosys_map_memcpy_to( + self.as_raw_mut(), + range.start, + src.as_ptr().cast(), + range.len(), + ) + }; + + Ok(()) + } + + /// Memset the region starting from `offset`. + /// + /// `offset` and `len` are in units of `T`, not the number of bytes. + /// + /// This function can return the following errors: + /// + /// * [`EOVERFLOW`] if calculating the length of the slice results in an overflow. + /// * [`EINVAL`] if the slice would go out of bounds of the memory region. + /// + /// # Examples + /// + /// ``` + /// use kernel::iosys_map::*; + /// + /// # fn test() -> Result { + /// # let mut map = tests::VecIoSysMap::new(&[0u8; 3])?; + /// # { + /// # let mut map = map.get(); + /// map.memset(7)?; // (now [7, 7, 7]) + /// # } + /// # + /// # map.assert_eq(&[7, 7, 7]); + /// # + /// # Ok::<(), Error>(()) } + /// # assert!(test().is_ok()); + /// ``` + pub fn memset(&mut self, value: i32) { + // SAFETY: + // - The address pointed to by this iosys_map is guaranteed to be valid via IoSysMapRef's + // type invariants. + unsafe { bindings::iosys_map_memset(self.as_raw_mut(), 0, value, self.size()) }; + } + + /// Attempt to compute the offset of an item within the iosys map using its index. + /// + /// Returns an error if an overflow occurs. + /// + /// # Safety + /// + /// This function checks for overflows, but it explicitly does not check if the offset goes out + /// of bounds. It is the caller's responsibility to check for this before using the returned + /// offset with the iosys_map API. + unsafe fn item_from_index(&self, idx: usize) -> Result { + self.item_size().checked_mul(idx).ok_or(EOVERFLOW) + } + + /// Compute the range within this mapping a specific data type at a given offset would occupy. + /// + /// This function returns the computed range if it doesn't overflow, but does not check whether + /// or not the range is within the bounds of the allocated region pointed to by this iosys + /// mapping. + /// + /// On success, the range returned by this function is guaranteed: + /// + /// * To be a valid range of memory within the virtual mapping for this gem object. + /// * To be properly aligned to [`RawIoSysMap::item_size()`]. + fn compute_range(&self, offset: usize, count: usize) -> Result> { + // SAFETY: If the offset is out of bounds, we'll catch this via overflow checks or when + // checking range_end. + let offset = unsafe { self.item_from_index(offset)? }; + let range_size = count.checked_mul(self.item_size()).ok_or(EOVERFLOW)?; + let range_end = offset.checked_add(range_size).ok_or(EOVERFLOW)?; + + if range_end > self.size { + return Err(EINVAL); + } + + // INVARIANT: Since `offset` and `count` are both in units of `T`, we're guaranteed that the + // range returned here is properly aligned to `T`. + Ok(offset..range_end) + } + + /// Common helper to compute the memory address of an item within the iosys mapping. + /// + /// Public but hidden, since it should only be used from [`iosys_map_read`] and + /// [`iosys_map_write`]. + #[doc(hidden)] + pub fn ptr_from_index(&self, offset: usize) -> Result<*mut T> { + // SAFETY: We check if the resulting offset goes out of bounds below. + let offset = unsafe { self.item_from_index(offset)? }; + + if offset.checked_add(self.item_size()).ok_or(EOVERFLOW)? > self.size() { + return Err(EINVAL); + } + + // SAFETY: We confirmed that `offset` + the item size does not go out of bounds above. + Ok(unsafe { self.as_mut_ptr().byte_add(offset) }) + } + + // TODO: + // This function is currently needed for making the iosys_map_read!() and iosys_map_write!() + // macros work due to a combination of a few limitations: + // + // * The current C API for iosys_map requires that we use offsets for reading/writing + // iosys_maps. + // * Calculating the offset of a field within a struct requires that we either: + // * Use field projection for calculating the offset of the field. We don't have this yet. + // * Explicitly specify the type of the struct, which would be cumbersome to require in the + // read/write macros. + // * Provide a typed pointer (or other reference) to the struct in question, allowing the + // use of &raw const and &raw mut. + // * Keep in mind: we can't simply cast the offset of an item in the iosys map into a typed + // pointer to fulfill the third option. While having invalid memory addresses as pointers + // is ok, adding an offset to a pointer in rust requires that the resulting memory address + // is within the same allocation. Since an invalid pointer has no allocation, we can't + // make that guarantee. + // + // So, until we have field projection the way we workaround this: + // + // * Calculate the offset (self.item_from_index()) of the struct within the iosys map + // * Calculate the memory address of the struct using the offset from the last step + // (self.ptr_from_index()). + // * Use that memory address with &raw const/&raw mut in order to calculate the memory address + // of the desired field, ensuring it remains in the same allocation (happens within the + // macros). + // * Convert the address from the last step back into an offset within the iosys map + // (offset_from_ptr()). + // + // Once we do get field projection, this silly code should be removed. + // + /// Convert a pointer to an item within the iosys map back into an offset. + /// + /// # Safety + /// + /// `ptr` must be a valid pointer to data within the iosys map. + unsafe fn offset_from_ptr(&self, ptr: *const F) -> usize { + // SAFETY: `ptr` always points to data within the memory pointed to by the iosys map, + // meaning it is within the same memory allocation. + // + // Additionally, since `ptr` is within the iosys mapping, the offset here will always be + // positive and safe to cast to a usize. + // (TODO: replace this with byte_offset_from_unsigned once it's available in the kernel) + unsafe { ptr.byte_offset_from(self.as_ptr()) as usize } + } + + /// Reads the value of `field` and ensures that its type is [`FromBytes`]. + /// + /// # Safety + /// + /// This must be called from the [`iosys_map_read`] macro which ensures that the `field` + /// pointer is validated beforehand. + /// + /// Public but hidden since it should only be used from the [`iosys_map_read`] macro. + #[doc(hidden)] + pub unsafe fn field_read(&self, field: *const F) -> F { + let mut field_val = MaybeUninit::::uninit(); + + // SAFETY: `field` is guaranteed valid via our safety contract. + let offset = unsafe { self.offset_from_ptr(field) }; + + // SAFETY: Since we verified `field` is valid above, `offset_from_ptr` will always return a + // valid offset within the iosys map. + unsafe { + bindings::iosys_map_memcpy_from( + field_val.as_mut_ptr().cast(), + self.as_raw(), + offset, + mem::size_of::(), + ) + } + + // SAFETY: We just initialized `field_val` above. + unsafe { field_val.assume_init() } + } + + /// Writes the value of `field` and ensures that its type is [`AsBytes`]. + /// + /// # Safety + /// + /// This must be called from the [`iosys_map_write`] macro which ensures that the `field` + /// pointers validated beforehand. + /// + /// Public but hidden since it should only be used from the [`iosys_map_write`] macro. + #[doc(hidden)] + pub unsafe fn field_write(&mut self, field: *mut F, val: F) { + // SAFETY: `field` is guaranteed valid via our safety contract. + let offset = unsafe { self.offset_from_ptr(field) }; + + // SAFETY: `offset_from_ptr` always returns a valid offset within the iosys map. + unsafe { + bindings::iosys_map_memcpy_to( + self.as_raw_mut(), + offset, + core::ptr::from_ref(&val).cast(), + mem::size_of::(), + ) + } + } +} + +impl<'a, T: AsBytes + FromBytes> Deref for IoSysMapRef<'a, T> { + type Target = RawIoSysMap; + + fn deref(&self) -> &Self::Target { + &self.map + } +} + +impl<'a, T: AsBytes + FromBytes> DerefMut for IoSysMapRef<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.map + } +} + +/// Reads from a field of an item from an iosys map ref. +/// +/// # Examples +/// +/// ``` +/// use kernel::{iosys_map::*, transmute::*}; +/// +/// #[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// struct MyStruct { a: u32, b: u16 } +/// +/// // SAFETY: All bit patterns are acceptable values for `MyStruct`. +/// unsafe impl FromBytes for MyStruct {}; +/// // SAFETY: Instances of `MyStruct` have no uninitialized portions. +/// unsafe impl AsBytes for MyStruct {}; +/// +/// # fn test() -> Result { +/// # let mut map = tests::VecIoSysMap::new(&[MyStruct { a: 42, b: 2 }; 3])?; +/// # let map = map.get(); +/// let whole = kernel::iosys_map_read!(map[2])?; +/// assert_eq!(whole, MyStruct { a: 42, b: 2 }); +/// +/// let field = kernel::iosys_map_read!(map[1].b)?; +/// assert_eq!(field, 2); +/// # Ok::<(), Error>(()) } +/// # assert!(test().is_ok()); +/// ``` +#[macro_export] +macro_rules! iosys_map_read { + ($map:expr, $idx:expr, $($field:tt)*) => {{ + (|| -> ::core::result::Result<_, $crate::error::Error> { + let map = &$map; + let item = $crate::iosys_map::IoSysMapRef::ptr_from_index(map, $idx)?; + + // SAFETY: `ptr_from_index()` ensures that `item` is always a valid (although + // potentially not dereferenceable, which is fine here) pointer to within the iosys + // mapping. + unsafe { + let ptr_field = &raw const (*item) $($field)*; + ::core::result::Result::Ok( + $crate::iosys_map::IoSysMapRef::field_read(map, ptr_field) + ) + } + })() + }}; + ($map:ident [ $idx: expr ] $($field:tt)* ) => { + $crate::iosys_map_read!($map, $idx, $($field)*) + }; + ($($map:ident).* [ $idx:expr ] $($field:tt)* ) => { + $crate::iosys_map_read!($($map).*, $idx, $($field)*) + }; +} + +/// Writes to a field of an item from an iosys map ref. +/// +/// # Examples +/// +/// ``` +/// use kernel::{iosys_map::*, transmute::*}; +/// +/// #[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// struct MyStruct { a: u32, b: u16 }; +/// +/// // SAFETY: All bit patterns are acceptable values for `MyStruct`. +/// unsafe impl FromBytes for MyStruct {}; +/// // SAFETY: Instances of `MyStruct` have no uninitialized portions. +/// unsafe impl AsBytes for MyStruct {}; +/// +/// # fn test() -> Result { +/// # let mut map = tests::VecIoSysMap::new(&[MyStruct { a: 42, b: 2 }; 3])?; +/// # let mut map = map.get(); +/// kernel::iosys_map_write!(map[2].b = 1337)?; +/// # assert_eq!(kernel::iosys_map_read!(map[2].b)?, 1337); +/// +/// kernel::iosys_map_write!(map[1] = MyStruct { a: 10, b: 20 })?; +/// # assert_eq!(kernel::iosys_map_read!(map[1])?, MyStruct { a: 10, b: 20 }); +/// # Ok::<(), Error>(()) } +/// # assert!(test().is_ok()); +/// ``` +#[macro_export] +macro_rules! iosys_map_write { + ($map:ident [ $idx:expr ] $($field:tt)*) => {{ + $crate::iosys_map_write!($map, $idx, $($field)*) + }}; + ($($map:ident).* [ $idx:expr ] $($field:tt)* ) => {{ + $crate::iosys_map_write!($($map).*, $idx, $($field)*) + }}; + ($map:expr, $idx:expr, = $val:expr) => { + (|| -> ::core::result::Result<_, $crate::error::Error> { + // (expand these outside of the unsafe block (clippy::macro-metavars-in-unsafe) + let map = &mut $map; + let val = $val; + + let item = $crate::iosys_map::IoSysMapRef::ptr_from_index(map, $idx)?; + // SAFETY: `item_from_index` ensures that `item` is always a valid item. + unsafe { $crate::iosys_map::IoSysMapRef::field_write(map, item, val) }; + ::core::result::Result::Ok(()) + })() + }; + ($map:expr, $idx:expr, $(.$field:ident)* = $val:expr) => { + (|| -> ::core::result::Result<_, $crate::error::Error> { + // (expand these outside of the unsafe block (clippy::macro-metavars-in-unsafe) + let map = &mut $map; + let val = $val; + + let item = $crate::iosys_map::IoSysMapRef::ptr_from_index(map, $idx)?; + + // SAFETY: `ptr_from_index()` ensures that `item` is always a valid (although + // potentially not dereferenceable, which is fine here) pointer to within the iosys + // mapping. + unsafe { + let ptr_field = &raw mut (*item) $(.$field)*; + $crate::iosys_map::IoSysMapRef::field_write(map, ptr_field, val) + }; + ::core::result::Result::Ok(()) + })() + }; +} + +#[doc(hidden)] +#[kunit_tests(rust_iosys_map)] +pub mod tests { + use super::*; + + /// A helper struct for managed IoSysMapRef structs which point to a [`Vec`]. + pub struct VecIoSysMap { + map: RawIoSysMap, + vec: KVec, + } + + impl VecIoSysMap { + pub fn new(src: &[T]) -> Result { + let mut vec = KVec::::new(); + + vec.extend_from_slice(src, GFP_KERNEL)?; + + let map = RawIoSysMap( + bindings::iosys_map { + is_iomem: false, + __bindgen_anon_1: bindings::iosys_map__bindgen_ty_1 { + vaddr: vec.as_mut_ptr().cast(), + }, + }, + PhantomData, + ); + + Ok(Self { map, vec }) + } + + pub fn get(&mut self) -> IoSysMapRef<'_, T> { + // SAFETY: + // * `map` points to `vec`, so the size of `map` is the size of the `vec`. + unsafe { IoSysMapRef::new(self.map.clone(), self.vec.len() * self.map.item_size()) } + } + + /// Assert whether or not the contents of this struct match src. + pub fn assert_eq(&self, src: &[T]) { + assert_eq!(*self.vec.as_ref(), *src) + } + } + + #[test] + fn basic() -> Result { + let mut map = VecIoSysMap::new(&[0; 3])?; + + map.get().write(&[1, 2, 3], 0)?; + map.assert_eq(&[1, 2, 3]); + + map.get().write(&[42], 1)?; + map.assert_eq(&[1, 42, 3]); + + Ok(()) + } + + #[test] + fn oob_accesses() -> Result { + let mut map = VecIoSysMap::new(&[0; 3])?; + + assert!(map.get().write(&[1, 2, 3, 69], 0).is_err()); + assert!(map.get().write(&[1, 2, 3], 69).is_err()); + map.assert_eq(&[0; 3]); + + Ok(()) + } + + #[test] + fn overflows() -> Result { + let mut map = VecIoSysMap::new(&[0; 3])?; + + assert!(map.get().write(&[1], usize::MAX).is_err()); + map.assert_eq(&[0; 3]); + + Ok(()) + } + + #[derive(Copy, Clone, Debug, PartialEq, Eq)] + struct TestStruct { + a: u32, + b: u64, + } + + // SAFETY: All bit patterns are acceptable values for `TestStruct`. + unsafe impl FromBytes for TestStruct {} + // SAFETY: Instances of `TestStruct` have no uninitialized portions. + unsafe impl AsBytes for TestStruct {} + + #[test] + fn basic_macro() -> Result { + let mut expected = [TestStruct { a: 1, b: 2 }; 5]; + let mut map = VecIoSysMap::new(&expected)?; + + { + let mut map_ref = map.get(); + + iosys_map_write!(map_ref[3].a = u32::MAX)?; + expected[3].a = u32::MAX; + + assert_eq!(iosys_map_read!(map_ref[3].a)?, u32::MAX); + assert_eq!( + iosys_map_read!(map_ref[3])?, + TestStruct { a: u32::MAX, b: 2 } + ); + } + + // Compare the entire array, so that we catch any mis-sized writes. + map.assert_eq(&expected); + + Ok(()) + } + + #[test] + fn macro_oob_accesses() -> Result { + let mut map = VecIoSysMap::new(&[TestStruct { a: 1, b: 2 }; 3])?; + let mut map = map.get(); + + assert!(iosys_map_read!(map[5].b).is_err()); + assert!(iosys_map_read!(map[1000]).is_err()); + assert!(iosys_map_write!(map[6969].a = 999).is_err()); + assert!(iosys_map_write!(map[243] = TestStruct { a: 99, b: 22 }).is_err()); + + Ok(()) + } + + #[test] + fn macro_overflows() -> Result { + let mut map = VecIoSysMap::new(&[TestStruct { a: 1, b: 2 }; 3])?; + let mut map = map.get(); + + assert!(iosys_map_read!(map[usize::MAX]).is_err()); + assert!(iosys_map_read!(map[usize::MAX].b).is_err()); + assert!(iosys_map_write!(map[usize::MAX] = TestStruct { a: 1, b: 1 }).is_err()); + assert!(iosys_map_write!(map[usize::MAX].b = 1).is_err()); + + Ok(()) + } +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index f812cf12004286..a978d7f0065fae 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -16,6 +16,21 @@ // Please see https://github.com/Rust-for-Linux/linux/issues/2 for details on // the unstable features in use. // +// ============ start asahi downstream features =========== +#![feature(associated_type_defaults)] +// +#![feature(cfg_version)] +// +// Stable since Rust 1.87.0. +#![feature(ptr_sub_ptr)] +// +#![feature(sized_type_properties)] +// +#![feature(slice_range)] +// +#![cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, feature(pin_coerce_unsized_trait))] +// ============ end asahi dowanstream features ============ +// // Stable since Rust 1.79.0. #![feature(generic_nonzero)] #![feature(inline_const)] @@ -65,6 +80,7 @@ extern crate self as kernel; pub use ffi; pub mod acpi; +pub mod addr; pub mod alloc; #[cfg(CONFIG_AUXILIARY_BUS)] pub mod auxiliary; @@ -84,10 +100,15 @@ pub mod cpufreq; pub mod cpumask; pub mod cred; pub mod debugfs; +#[cfg(CONFIG_DEV_COREDUMP)] +pub mod devcoredump; pub mod device; pub mod device_id; pub mod devres; pub mod dma; +pub mod dma_buf; +#[cfg(CONFIG_DMA_SHARED_BUFFER)] +pub mod dma_fence; pub mod driver; #[cfg(CONFIG_DRM = "y")] pub mod drm; @@ -100,9 +121,14 @@ pub mod fs; #[cfg(CONFIG_I2C = "y")] pub mod i2c; pub mod id_pool; +#[cfg(CONFIG_IIO)] +pub mod iio; +#[doc(hidden)] +pub mod impl_flags; pub mod init; pub mod io; pub mod ioctl; +pub mod iosys_map; pub mod iov; pub mod irq; pub mod jump_label; @@ -138,6 +164,7 @@ pub mod security; pub mod seq_file; pub mod sizes; pub mod slice; +pub mod soc; mod static_assert; #[doc(hidden)] pub mod std_vendor; diff --git a/rust/kernel/of.rs b/rust/kernel/of.rs index 58b20c367f993f..80c604ee831452 100644 --- a/rust/kernel/of.rs +++ b/rust/kernel/of.rs @@ -8,6 +8,17 @@ use crate::{ prelude::*, }; +// Note: Most OF functions turn into inline dummies with CONFIG_OF(_*) disabled. +// We have to either add config conditionals to helpers.c or here; let's do it +// here for now. In the future, once bindgen can auto-generate static inline +// helpers, this can go away if desired. + +use core::marker::PhantomData; +use core::num::NonZeroU32; + +use crate::error::to_result; +use crate::io::resource::Resource; + /// IdTable type for OF drivers. pub type IdTable = &'static dyn kernel::device_id::IdTable; @@ -50,6 +61,518 @@ impl DeviceId { } } +/// Type alias for an OF phandle +pub type PHandle = bindings::phandle; + +/// An OF device tree node. +/// +/// # Invariants +/// +/// `raw_node` points to a valid OF node, and we hold a reference to it. +pub struct Node { + raw_node: *mut bindings::device_node, +} + +#[allow(dead_code)] +impl Node { + /// Creates a `Node` from a raw C pointer. The pointer must be owned (the caller + /// gives up its reference). If the pointer is NULL, returns None. + pub(crate) unsafe fn from_raw(raw_node: *mut bindings::device_node) -> Option { + if raw_node.is_null() { + None + } else { + // INVARIANT: `raw_node` is valid per the above contract, and non-null per the + // above check. + Some(Node { raw_node }) + } + } + + /// Creates a `Node` from a raw C pointer. The pointer must be borrowed (the caller + /// retains its reference, which must be valid for the duration of the call). If the + /// pointer is NULL, returns None. + pub(crate) unsafe fn get_from_raw(raw_node: *mut bindings::device_node) -> Option { + // SAFETY: `raw_node` is valid or NULL per the above contract. `of_node_get` can handle + // NULL. + unsafe { + #[cfg(CONFIG_OF_DYNAMIC)] + bindings::of_node_get(raw_node); + Node::from_raw(raw_node) + } + } + + /// Returns a reference to the underlying C `device_node` structure. + pub(crate) fn node(&self) -> &bindings::device_node { + // SAFETY: `raw_node` is valid per the type invariant. + unsafe { &*self.raw_node } + } + + /// Returns the name of the node. + pub(crate) fn name(&self) -> &CStr { + // SAFETY: The lifetime of the `CStr` is the same as the lifetime of this `Node`. + unsafe { CStr::from_char_ptr(self.node().name) } + } + + /// Returns the phandle for this node. + pub(crate) fn phandle(&self) -> PHandle { + self.node().phandle + } + + /// Returns the full name (with address) for this node. + pub(crate) fn full_name(&self) -> &CStr { + // SAFETY: The lifetime of the `CStr` is the same as the lifetime of this `Node`. + unsafe { CStr::from_char_ptr(self.node().full_name) } + } + + /// Returns `true` if the node is the root node. + pub(crate) fn is_root(&self) -> bool { + #[cfg(not(CONFIG_OF))] + { + false + } + #[cfg(CONFIG_OF)] + // SAFETY: `raw_node` is valid per the type invariant + unsafe { + bindings::of_node_is_root(self.raw_node) + } + } + + /// Returns the parent node, if any. + pub(crate) fn parent(&self) -> Option { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: `raw_node` is valid per the type invariant, and `of_get_parent()` takes a + // new reference to the parent (or returns NULL). + unsafe { + Node::from_raw(bindings::of_get_parent(self.raw_node)) + } + } + + /// Returns an iterator over the node's children. + // TODO: use type alias for return type once type_alias_impl_trait is stable + pub fn children( + &self, + ) -> NodeIterator<'_, impl Fn(*mut bindings::device_node) -> *mut bindings::device_node + '_> + { + #[cfg(not(CONFIG_OF))] + { + NodeIterator::new(|_prev| core::ptr::null_mut()) + } + #[cfg(CONFIG_OF)] + // SAFETY: `raw_node` is valid per the type invariant, and the lifetime of the `NodeIterator` + // does not exceed the lifetime of the `Node` so it can borrow its reference. + NodeIterator::new(|prev| unsafe { bindings::of_get_next_child(self.raw_node, prev) }) + } + + /// Find a child by its name and return it, or None if not found. + #[allow(unused_variables)] + pub(crate) fn get_child_by_name(&self, name: &CStr) -> Option { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: `raw_node` is valid per the type invariant. + unsafe { + Node::from_raw(bindings::of_get_child_by_name( + self.raw_node, + name.as_char_ptr(), + )) + } + } + + /// Checks whether the node is compatible with the given compatible string. + /// + /// Returns `None` if there is no match, or `Some` if there is, with the value + /// representing as match score (higher values for more specific compatible matches). + #[allow(unused_variables)] + pub(crate) fn is_compatible(&self, compatible: &CStr) -> Option { + #[cfg(not(CONFIG_OF))] + let ret = 0; + #[cfg(CONFIG_OF)] + let ret = + // SAFETY: `raw_node` is valid per the type invariant. + unsafe { bindings::of_device_is_compatible(self.raw_node, compatible.as_char_ptr()) }; + + NonZeroU32::new(ret.try_into().ok()?) + } + + /// Parse a phandle property and return the Node referenced at a given index, if any. + /// + /// Used only for phandle properties with no arguments. + #[allow(unused_variables)] + pub fn parse_phandle(&self, name: &CStr, index: usize) -> Option { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: `raw_node` is valid per the type invariant. `of_parse_phandle` returns an + // owned reference. + unsafe { + Node::from_raw(bindings::of_parse_phandle( + self.raw_node, + name.as_char_ptr(), + index.try_into().ok()?, + )) + } + } + + #[allow(unused_variables)] + /// Get a reserved memory region as a resource + pub fn reserved_mem_region_to_resource_byname(&self, name: &CStr) -> Result { + #[cfg(not(CONFIG_OF))] + { + Err(ENOENT) + } + #[cfg(CONFIG_OF)] + { + let res = Resource::zeroed(); + // SAFETY: This function is safe to call as long as the arguments are valid pointers. + let ret = unsafe { + bindings::of_reserved_mem_region_to_resource_byname( + self.raw_node, + name.as_char_ptr(), + res.as_raw(), + ) + }; + to_result(ret)?; + + Ok(res) + } + } + + #[allow(unused_variables)] + /// Look up a node property by name, returning a `Property` object if found. + pub(crate) fn find_property(&self, propname: &CStr) -> Option> { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: `raw_node` is valid per the type invariant. The property structure + // returned borrows the reference to the owning node, and so has the same + // lifetime. + unsafe { + Property::from_raw(bindings::of_find_property( + self.raw_node, + propname.as_char_ptr(), + core::ptr::null_mut(), + )) + } + } + + /// Look up a mandatory node property by name, and decode it into a value type. + /// + /// Returns `Err(ENOENT)` if the property is not found. + /// + /// The type `T` must implement `TryFrom>`. + pub fn get_property<'a, T: TryFrom>>(&'a self, propname: &CStr) -> Result + where + crate::error::Error: From<>>::Error>, + { + Ok(self.find_property(propname).ok_or(ENOENT)?.try_into()?) + } + + /// Look up an optional node property by name, and decode it into a value type. + /// + /// Returns `Ok(None)` if the property is not found. + /// + /// The type `T` must implement `TryFrom>`. + pub fn get_opt_property<'a, T: TryFrom>>( + &'a self, + propname: &CStr, + ) -> Result> + where + crate::error::Error: From<>>::Error>, + { + self.find_property(propname) + .map_or(Ok(None), |p| Ok(Some(p.try_into()?))) + } +} + +/// A property attached to a device tree `Node`. +/// +/// # Invariants +/// +/// `raw` must be valid and point to a property that outlives the lifetime of this object. +#[derive(Copy, Clone)] +pub struct Property<'a> { + raw: *mut bindings::property, + _p: PhantomData<&'a Node>, +} + +impl<'a> Property<'a> { + #[cfg(CONFIG_OF)] + /// Create a `Property` object from a raw C pointer. Returns `None` if NULL. + /// + /// The passed pointer must be valid and outlive the lifetime argument, or NULL. + unsafe fn from_raw(raw: *mut bindings::property) -> Option> { + if raw.is_null() { + None + } else { + Some(Property { + raw, + _p: PhantomData, + }) + } + } + + /// Returns the name of the property as a `CStr`. + pub fn name(&self) -> &CStr { + // SAFETY: `raw` is valid per the type invariant, and the lifetime of the `CStr` does not + // outlive it. + unsafe { CStr::from_char_ptr((*self.raw).name) } + } + + /// Returns the name of the property as a `&[u8]`. + pub fn value(&self) -> &[u8] { + // SAFETY: `raw` is valid per the type invariant, and the lifetime of the slice does not + // outlive it. + unsafe { core::slice::from_raw_parts((*self.raw).value as *const u8, self.len()) } + } + + /// Returns the length of the property in bytes. + pub fn len(&self) -> usize { + // SAFETY: `raw` is valid per the type invariant. + unsafe { (*self.raw).length.try_into().unwrap() } + } + + /// Returns true if the property is empty (zero-length), which typically represents boolean true. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +/// A trait that represents a value decodable from a property with a fixed unit size. +/// +/// This allows us to auto-derive property decode implementations for `Vec`. +pub trait PropertyUnit: Sized { + /// The size in bytes of a single data unit. + const UNIT_SIZE: usize; + + /// Decode this data unit from a byte slice. The passed slice will have a length of `UNIT_SIZE`. + fn from_bytes(data: &[u8]) -> Result; +} + +// This doesn't work... +// impl<'a, T: PropertyUnit> TryFrom> for T { +// type Error = Error; +// +// fn try_from(p: Property<'_>) -> core::result::Result { +// if p.value().len() != T::UNIT_SIZE { +// Err(EINVAL) +// } else { +// Ok(T::from_bytes(p.value())?) +// } +// } +// } + +impl<'a, T: PropertyUnit> TryFrom> for KVec { + type Error = Error; + + fn try_from(p: Property<'_>) -> core::result::Result, Self::Error> { + if p.len() % T::UNIT_SIZE != 0 { + return Err(EINVAL); + } + + let mut v = Vec::new(); + let val = p.value(); + for off in (0..p.len()).step_by(T::UNIT_SIZE) { + v.push(T::from_bytes(&val[off..off + T::UNIT_SIZE])?, GFP_KERNEL)?; + } + Ok(v) + } +} + +macro_rules! prop_int_type ( + ($type:ty) => { + impl<'a> TryFrom> for $type { + type Error = Error; + + fn try_from(p: Property<'_>) -> core::result::Result<$type, Self::Error> { + Ok(<$type>::from_be_bytes(p.value().try_into().or(Err(EINVAL))?)) + } + } + + impl PropertyUnit for $type { + const UNIT_SIZE: usize = <$type>::BITS as usize / 8; + + fn from_bytes(data: &[u8]) -> Result { + Ok(<$type>::from_be_bytes(data.try_into().or(Err(EINVAL))?)) + } + } + } +); + +prop_int_type!(u8); +prop_int_type!(u16); +prop_int_type!(u32); +prop_int_type!(u64); +prop_int_type!(i8); +prop_int_type!(i16); +prop_int_type!(i32); +prop_int_type!(i64); + +/// An iterator across a collection of Node objects. +/// +/// # Invariants +/// +/// `cur` must be NULL or a valid node owned reference. If NULL, it represents either the first +/// or last position of the iterator. +/// +/// If `done` is true, `cur` must be NULL. +/// +/// fn_next must be a callback that iterates from one node to the next, and it must not capture +/// values that exceed the lifetime of the iterator. It must return owned references and also +/// take owned references. +pub struct NodeIterator<'a, T> +where + T: Fn(*mut bindings::device_node) -> *mut bindings::device_node, +{ + cur: *mut bindings::device_node, + done: bool, + fn_next: T, + _p: PhantomData<&'a T>, +} + +impl<'a, T> NodeIterator<'a, T> +where + T: Fn(*mut bindings::device_node) -> *mut bindings::device_node, +{ + fn new(next: T) -> NodeIterator<'a, T> { + // INVARIANT: `cur` is initialized to NULL to represent the initial state. + NodeIterator { + cur: core::ptr::null_mut(), + done: false, + fn_next: next, + _p: PhantomData, + } + } +} + +impl<'a, T> Iterator for NodeIterator<'a, T> +where + T: Fn(*mut bindings::device_node) -> *mut bindings::device_node, +{ + type Item = Node; + + fn next(&mut self) -> Option { + if self.done { + None + } else { + // INVARIANT: if the new `cur` is NULL, then the iterator has reached its end and we + // set `done` to `true`. + self.cur = (self.fn_next)(self.cur); + self.done = self.cur.is_null(); + // SAFETY: `fn_next` must return an owned reference per the iterator contract. + // The iterator itself is considered to own this reference, so we take another one. + unsafe { Node::get_from_raw(self.cur) } + } + } +} + +// Drop impl to ensure we drop the current node being iterated on, if any. +impl<'a, T> Drop for NodeIterator<'a, T> +where + T: Fn(*mut bindings::device_node) -> *mut bindings::device_node, +{ + fn drop(&mut self) { + // SAFETY: `cur` is valid or NULL, and `of_node_put()` can handle NULL. + #[cfg(CONFIG_OF_DYNAMIC)] + unsafe { + bindings::of_node_put(self.cur) + }; + } +} + +/// Returns the root node of the OF device tree (if any). +pub fn root() -> Option { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: bindings::of_root is always valid or NULL + unsafe { + Node::get_from_raw(bindings::of_root) + } +} + +/// Returns the /chosen node of the OF device tree (if any). +pub fn chosen() -> Option { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: bindings::of_chosen is always valid or NULL + unsafe { + Node::get_from_raw(bindings::of_chosen) + } +} + +/// Returns the /aliases node of the OF device tree (if any). +pub fn aliases() -> Option { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: bindings::of_aliases is always valid or NULL + unsafe { + Node::get_from_raw(bindings::of_aliases) + } +} + +/// Returns the system stdout node of the OF device tree (if any). +pub fn stdout() -> Option { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: bindings::of_stdout is always valid or NULL + unsafe { + Node::get_from_raw(bindings::of_stdout) + } +} + +#[allow(unused_variables)] +/// Looks up a node in the device tree by phandle. +pub fn find_node_by_phandle(handle: PHandle) -> Option { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: bindings::of_find_node_by_phandle always returns a valid pointer or NULL + unsafe { + #[allow(dead_code)] + Node::from_raw(bindings::of_find_node_by_phandle(handle)) + } +} + +impl Clone for Node { + fn clone(&self) -> Node { + // SAFETY: `raw_node` is valid and non-NULL per the type invariant, + // so this can never return None. + unsafe { Node::get_from_raw(self.raw_node).unwrap() } + } +} + +impl Drop for Node { + fn drop(&mut self) { + #[cfg(CONFIG_OF_DYNAMIC)] + // SAFETY: `raw_node` is valid per the type invariant. + unsafe { + bindings::of_node_put(self.raw_node) + }; + } +} + /// Create an OF `IdTable` with an "alias" for modpost. #[macro_export] macro_rules! of_device_table { diff --git a/rust/kernel/page.rs b/rust/kernel/page.rs index 432fc0297d4a85..c57311c2a64fed 100644 --- a/rust/kernel/page.rs +++ b/rust/kernel/page.rs @@ -3,10 +3,12 @@ //! Kernel page allocation and management. use crate::{ + addr::*, alloc::{AllocError, Flags}, bindings, error::code::*, error::Result, + types::{Opaque, Ownable, Owned}, uaccess::UserSliceReader, }; use core::{ @@ -77,7 +79,7 @@ pub const fn page_align(addr: usize) -> usize { /// /// [`VBox`]: kernel::alloc::VBox /// [`Vmalloc`]: kernel::alloc::allocator::Vmalloc -pub struct BorrowedPage<'a>(ManuallyDrop, PhantomData<&'a Page>); +pub struct BorrowedPage<'a>(ManuallyDrop>, PhantomData<&'a Page>); impl<'a> BorrowedPage<'a> { /// Constructs a [`BorrowedPage`] from a raw pointer to a `struct page`. @@ -87,7 +89,7 @@ impl<'a> BorrowedPage<'a> { /// - `ptr` must point to a valid `bindings::page`. /// - `ptr` must remain valid for the entire lifetime `'a`. pub unsafe fn from_raw(ptr: NonNull) -> Self { - let page = Page { page: ptr }; + let page = unsafe { Page::from_phys(bindings::page_to_phys(ptr.as_ptr())) }; // INVARIANT: The safety requirements guarantee that `ptr` is valid for the entire lifetime // `'a`. @@ -120,8 +122,9 @@ pub trait AsPageIter { /// # Invariants /// /// The pointer is valid, and has ownership over the page. +#[repr(transparent)] pub struct Page { - page: NonNull, + page: Opaque, } // SAFETY: Pages have no logic that relies on them staying on a given thread, so moving them across @@ -155,19 +158,20 @@ impl Page { /// # Ok::<(), kernel::alloc::AllocError>(()) /// ``` #[inline] - pub fn alloc_page(flags: Flags) -> Result { + pub fn alloc_page(flags: Flags) -> Result, AllocError> { // SAFETY: Depending on the value of `gfp_flags`, this call may sleep. Other than that, it // is always safe to call this method. let page = unsafe { bindings::alloc_pages(flags.as_raw(), 0) }; let page = NonNull::new(page).ok_or(AllocError)?; - // INVARIANT: We just successfully allocated a page, so we now have ownership of the newly - // allocated page. We transfer that ownership to the new `Page` object. - Ok(Self { page }) + // SAFETY: We just successfully allocated a page, so we now have ownership of the newly + // allocated page. We transfer that ownership to the new `Owned` object. + // Since `Page` is transparent, we can cast the pointer directly. + Ok(unsafe { Owned::from_raw(page.cast()) }) } /// Returns a raw pointer to the page. pub fn as_ptr(&self) -> *mut bindings::page { - self.page.as_ptr() + Opaque::cast_into(&self.page) } /// Get the node id containing this page. @@ -192,7 +196,7 @@ impl Page { /// different addresses. However, even if the addresses are different, the underlying memory is /// still the same for these purposes (e.g., it's still a data race if they both write to the /// same underlying byte at the same time). - fn with_page_mapped(&self, f: impl FnOnce(*mut u8) -> T) -> T { + pub fn with_page_mapped(&self, f: impl FnOnce(*mut u8) -> T) -> T { // SAFETY: `page` is valid due to the type invariants on `Page`. let mapped_addr = unsafe { bindings::kmap_local_page(self.as_ptr()) }; @@ -233,7 +237,7 @@ impl Page { /// different addresses. However, even if the addresses are different, the underlying memory is /// still the same for these purposes (e.g., it's still a data race if they both write to the /// same underlying byte at the same time). - fn with_pointer_into_page( + pub fn with_pointer_into_page( &self, off: usize, len: usize, @@ -340,12 +344,77 @@ impl Page { reader.read_raw(unsafe { core::slice::from_raw_parts_mut(dst.cast(), len) }) }) } + + /// Returns the physical address of this page. + pub fn phys(&self) -> PhysicalAddr { + // SAFETY: `page` is valid due to the type invariants on `Page`. + unsafe { bindings::page_to_phys(self.as_ptr()) } + } + + /// Converts a Rust-owned Page into its physical address. + /// The caller is responsible for calling `from_phys()` to avoid + /// leaking memory. + pub fn into_phys(this: Owned) -> PhysicalAddr { + ManuallyDrop::new(this).phys() + } + + /// Converts a physical address to a Rust-owned Page. + /// + /// SAFETY: + /// The caller must ensure that the physical address was previously returned + /// by a call to `Page::into_phys()`, and that the physical address is no + /// longer used after this call, nor is `from_phys()` called again on it. + pub unsafe fn from_phys(phys: PhysicalAddr) -> Owned { + // SAFETY: By the safety requirements, the physical address must be valid and + // have come from `into_phys()`, so phys_to_page() cannot fail and + // must return the original struct page pointer. + unsafe { Owned::from_raw(NonNull::new_unchecked(bindings::phys_to_page(phys)).cast()) } + } + + /// Borrows a Page from a physical address, without taking over ownership. + /// + /// If the physical address does not have a `struct page` entry or is not + /// part of the System RAM region, returns None. + /// + /// SAFETY: + /// The caller must ensure that the physical address, if it is backed by a + /// `struct page`, remains available for the duration of the borrowed + /// lifetime. + pub unsafe fn borrow_phys(phys: &PhysicalAddr) -> Option<&Self> { + // SAFETY: This is always safe, as it is just arithmetic + let pfn = unsafe { bindings::phys_to_pfn(*phys) }; + // SAFETY: This function is safe to call with any pfn + if !unsafe { bindings::pfn_valid(pfn) && bindings::page_is_ram(pfn) != 0 } { + None + } else { + // SAFETY: We have just checked that the pfn is valid above, so it must + // have a corresponding struct page. By the safety requirements, we can + // return a borrowed reference to it. + Some(unsafe { &*(bindings::pfn_to_page(pfn) as *mut Self as *const Self) }) + } + } + + /// Borrows a Page from a physical address, without taking over ownership + /// nor checking for validity. + /// + /// SAFETY: + /// The caller must ensure that the physical address is backed by a + /// `struct page` and corresponds to System RAM. + pub unsafe fn borrow_phys_unchecked(phys: &PhysicalAddr) -> &Self { + // SAFETY: This is always safe, as it is just arithmetic + let pfn = unsafe { bindings::phys_to_pfn(*phys) }; + // SAFETY: The caller guarantees that the pfn is valid. By the safety + // requirements, we can return a borrowed reference to it. + unsafe { &*(bindings::pfn_to_page(pfn) as *mut Self as *const Self) } + } } -impl Drop for Page { +// SAFETY: See below. +unsafe impl Ownable for Page { #[inline] - fn drop(&mut self) { + unsafe fn release(this: NonNull) { // SAFETY: By the type invariants, we have ownership of the page and can free it. - unsafe { bindings::__free_pages(self.page.as_ptr(), 0) }; + // Since Page is transparent, we can cast the raw pointer directly. + unsafe { bindings::__free_pages(this.cast().as_ptr(), 0) }; } } diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs index 35a5813ffb33f1..54c7ede9e8d989 100644 --- a/rust/kernel/platform.rs +++ b/rust/kernel/platform.rs @@ -277,6 +277,20 @@ impl Device { // returned by `platform_get_resource`. Some(unsafe { Resource::from_raw(resource) }) } + + /// excute closure while the device is bound + pub fn while_bound_with(&self, f: F) -> Result + where + F: FnOnce(&Device) -> Result, + { + let _guard = self.as_ref().lock(); + if unsafe { !bindings::device_is_bound(self.as_ref().as_raw()) } { + return Err(ENODEV); + } + let ptr: *const Self = self; + let ptr = ptr.cast::>(); + f(unsafe { &*ptr }) + } } impl Device { diff --git a/rust/kernel/pwm.rs b/rust/kernel/pwm.rs index cb00f8a8765c8e..2ba9cfd02bfdbe 100644 --- a/rust/kernel/pwm.rs +++ b/rust/kernel/pwm.rs @@ -601,7 +601,11 @@ impl Chip { let drvdata_ptr = unsafe { bindings::pwmchip_get_drvdata(c_chip_ptr) }; // SAFETY: We construct the `T` object in-place in the allocated private memory. - unsafe { data.__pinned_init(drvdata_ptr.cast())? }; + unsafe { data.__pinned_init(drvdata_ptr.cast()) }.inspect_err(|_| { + // SAFETY: It is safe to call `pwmchip_put()` with a valid pointer obtained + // from `pwmchip_alloc()`. We will not use pointer after this. + unsafe { bindings::pwmchip_put(c_chip_ptr) } + })?; // SAFETY: `c_chip_ptr` points to a valid chip. unsafe { diff --git a/rust/kernel/soc/apple/aop.rs b/rust/kernel/soc/apple/aop.rs new file mode 100644 index 00000000000000..da46ce0bcb027d --- /dev/null +++ b/rust/kernel/soc/apple/aop.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Common code for AOP endpoint drivers + +use kernel::{prelude::*, sync::Arc}; + +/// Representation of an "EPIC" service. +#[derive(Clone, Copy, PartialEq, Eq)] +#[repr(C)] +pub struct EPICService { + /// Channel id + pub channel: u32, + /// RTKit endpoint + pub endpoint: u8, +} + +/// Listener for the "HID" events sent by aop +pub trait FakehidListener { + /// Process the event. + fn process_fakehid_report(&self, data: &[u8]) -> Result<()>; +} + +/// AOP communications manager. +pub trait AOP: Send + Sync { + /// Calls a method on a specified service + fn epic_call(&self, svc: &EPICService, subtype: u16, msg_bytes: &[u8]) -> Result; + /// Just like epic_call, but also returns a value + fn epic_call_ret( + &self, + svc: &EPICService, + subtype: u16, + msg_bytes: &[u8], + ret_len: usize, + ) -> Result<(u32, KVec)>; + + /// Adds the listener for the specified service + fn add_fakehid_listener( + &self, + svc: EPICService, + listener: Arc, + ) -> Result<()>; + /// Remove the listener for the specified service + fn remove_fakehid_listener(&self, svc: &EPICService) -> bool; + /// Internal method to detach the device. + fn remove(&self); +} + +/// Converts a text representation of a FourCC to u32 +pub const fn from_fourcc(b: &[u8]) -> u32 { + b[3] as u32 | (b[2] as u32) << 8 | (b[1] as u32) << 16 | (b[0] as u32) << 24 +} diff --git a/rust/kernel/soc/apple/mailbox.rs b/rust/kernel/soc/apple/mailbox.rs new file mode 100644 index 00000000000000..f477ddc0afb00e --- /dev/null +++ b/rust/kernel/soc/apple/mailbox.rs @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Support for Apple ASC Mailbox. +//! +//! C header: [`include/linux/soc/apple/mailbox.h`](../../../../include/linux/gpio/driver.h) + +use crate::{ + bindings, + device, + error::{ + from_err_ptr, + to_result, // + }, + prelude::*, + str::CStrExt, + types::{ + ForeignOwnable, + ScopeGuard, // + }, // +}; + +use core::marker::PhantomData; + +/// 96-bit message. What it means is up to the upper layer +pub type Message = bindings::apple_mbox_msg; + +/// Mailbox receive callback +pub trait MailCallback { + /// Callback context + type Data: ForeignOwnable + Send + Sync; + + /// The actual callback. Called in an interrupt context. + fn recv_message(data: ::Borrowed<'_>, msg: Message); +} + +/// Wrapper over `struct apple_mbox *` +#[repr(transparent)] +pub struct Mailbox { + mbox: *mut bindings::apple_mbox, + _p: PhantomData, +} + +extern "C" fn mailbox_rx_callback( + _mbox: *mut bindings::apple_mbox, + msg: Message, + cookie: *mut crate::ffi::c_void, +) { + // SAFETY: cookie came from a call to `into_foreign` + T::recv_message(unsafe { T::Data::borrow(cookie.cast()) }, msg); +} + +impl Mailbox { + /// Creates a mailbox for the specified name. + pub fn new_byname( + dev: &device::Device, + mbox_name: &'static CStr, + data: T::Data, + ) -> Result> { + let ptr: *mut crate::ffi::c_void = data.into_foreign().cast(); + let guard = ScopeGuard::new(|| { + // SAFETY: `ptr` came from a previous call to `into_foreign`. + unsafe { T::Data::from_foreign(ptr.cast()) }; + }); + // SAFETY: Just calling the c function, all values are valid. + let mbox = unsafe { + from_err_ptr(bindings::apple_mbox_get_byname( + dev.as_raw(), + mbox_name.as_char_ptr(), + ))? + }; + // SAFETY: mbox is a valid pointer + unsafe { + (*mbox).cookie = ptr; + (*mbox).rx = Some(mailbox_rx_callback::); + to_result(bindings::apple_mbox_start(mbox))?; + } + guard.dismiss(); + Ok(Mailbox { + mbox, + _p: PhantomData, + }) + } + /// Sends the specified message + pub fn send(&self, msg: Message, atomic: bool) -> Result<()> { + // SAFETY: Calling the c function, `mbox` is a valid pointer + to_result(unsafe { bindings::apple_mbox_send(self.mbox, msg, atomic) }) + } +} + +impl Drop for Mailbox { + fn drop(&mut self) { + // SAFETY: mbox is a valid pointer + unsafe { bindings::apple_mbox_stop(self.mbox) }; + // SAFETY: `cookie` came from `into_foreign` + unsafe { T::Data::from_foreign((*self.mbox).cookie.cast()) }; + } +} + +unsafe impl Sync for Mailbox +where + T: MailCallback, + T::Data: Sync, +{ +} + +unsafe impl Send for Mailbox +where + T: MailCallback, + T::Data: Send, +{ +} diff --git a/rust/kernel/soc/apple/mod.rs b/rust/kernel/soc/apple/mod.rs new file mode 100644 index 00000000000000..e77eba782a5867 --- /dev/null +++ b/rust/kernel/soc/apple/mod.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Apple SoC drivers + +#[cfg(CONFIG_RUST_APPLE_RTKIT = "y")] +pub mod rtkit; + +#[cfg(any(CONFIG_APPLE_AOP = "y", CONFIG_APPLE_AOP = "m"))] +pub mod aop; + +#[cfg(CONFIG_RUST_APPLE_MAILBOX = "y")] +pub mod mailbox; diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs new file mode 100644 index 00000000000000..a5bfcfb8fa7334 --- /dev/null +++ b/rust/kernel/soc/apple/rtkit.rs @@ -0,0 +1,293 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Support for Apple RTKit coprocessors. +//! +//! C header: [`include/linux/soc/apple/rtkit.h`](../../../../include/linux/gpio/driver.h) + +use crate::{ + alloc::flags::*, + bindings, + device, + error::{ + from_err_ptr, + from_result, + to_result, // + }, + iosys_map::IoSysMapRef, + prelude::*, + str::CStrExt, + types::{ + ForeignOwnable, + ScopeGuard, // + }, // +}; + +use core::marker::PhantomData; +use core::ptr; +use macros::vtable; + +/// Trait to represent allocatable buffers for the RTKit core. +/// +/// Users must implement this trait for their own representation of those allocations. +pub trait Buffer { + /// Returns the IOVA (virtual address) of the buffer from RTKit's point of view, or an error if + /// unavailable. + fn iova(&self) -> Result; + + /// Returns a mutable byte slice of the buffer contents, or an + /// error if unavailable. + fn buf(&mut self) -> Result>; +} + +/// Callback operations for an RTKit client. +#[vtable] +pub trait Operations { + /// Arbitrary user context type. + type Data: ForeignOwnable + Send + Sync; + + /// Type representing an allocated buffer for RTKit. + type Buffer: Buffer; + + /// Called when RTKit crashes. + fn crashed(_data: ::Borrowed<'_>, _crashlog: Option<&[u8]>) {} + + /// Called when a message was received on a non-system endpoint. Called in non-IRQ context. + fn recv_message( + _data: ::Borrowed<'_>, + _endpoint: u8, + _message: u64, + ) { + } + + /// Called in IRQ context when a message was received on a non-system endpoint. + /// + /// Must return `true` if the message is handled, or `false` to process it in + /// the handling thread. + fn recv_message_early( + _data: ::Borrowed<'_>, + _endpoint: u8, + _message: u64, + ) -> bool { + false + } + + /// Allocate a buffer for use by RTKit. + fn shmem_alloc( + _data: ::Borrowed<'_>, + _size: usize, + ) -> Result { + Err(EINVAL) + } + + /// Map an existing buffer used by RTKit at a device-specified virtual address. + fn shmem_map( + _data: ::Borrowed<'_>, + _iova: usize, + _size: usize, + ) -> Result { + Err(EINVAL) + } +} + +/// Represents `struct apple_rtkit *`. +/// +/// # Invariants +/// +/// The rtk pointer is valid. +/// The data pointer is a valid pointer from T::Data::into_foreign(). +pub struct RtKit { + rtk: *mut bindings::apple_rtkit, + data: *mut core::ffi::c_void, + _p: PhantomData, +} + +unsafe extern "C" fn crashed_callback( + cookie: *mut core::ffi::c_void, + crashlog: *const core::ffi::c_void, + crashlog_size: usize, +) { + let crashlog = if !crashlog.is_null() && crashlog_size > 0 { + // SAFETY: The crashlog is either missing or a byte buffer of the specified size + Some(unsafe { core::slice::from_raw_parts(crashlog as *const u8, crashlog_size) }) + } else { + None + }; + // SAFETY: cookie is always a T::Data in this API + T::crashed(unsafe { T::Data::borrow(cookie.cast()) }, crashlog); +} + +unsafe extern "C" fn recv_message_callback( + cookie: *mut core::ffi::c_void, + endpoint: u8, + message: u64, +) { + // SAFETY: cookie is always a T::Data in this API + T::recv_message(unsafe { T::Data::borrow(cookie.cast()) }, endpoint, message); +} + +unsafe extern "C" fn recv_message_early_callback( + cookie: *mut core::ffi::c_void, + endpoint: u8, + message: u64, +) -> bool { + // SAFETY: cookie is always a T::Data in this API + T::recv_message_early(unsafe { T::Data::borrow(cookie.cast()) }, endpoint, message) +} + +unsafe extern "C" fn shmem_setup_callback( + cookie: *mut core::ffi::c_void, + bfr: *mut bindings::apple_rtkit_shmem, +) -> core::ffi::c_int { + // SAFETY: `bfr` is a valid buffer + let bfr_mut = unsafe { &mut *bfr }; + + from_result(|| { + let mut buf = if bfr_mut.iova != 0 { + bfr_mut.is_mapped = true; + T::shmem_map( + // SAFETY: `cookie` came from a previous call to `into_foreign`. + unsafe { T::Data::borrow(cookie.cast()) }, + bfr_mut.iova as usize, + bfr_mut.size, + )? + } else { + bfr_mut.is_mapped = false; + // SAFETY: `cookie` came from a previous call to `into_foreign`. + T::shmem_alloc(unsafe { T::Data::borrow(cookie.cast()) }, bfr_mut.size)? + }; + + let iova = buf.iova()?; + let iosys_map = buf.buf()?; + + if iosys_map.size() < bfr_mut.size { + return Err(ENOMEM); + } + + bfr_mut.iova = iova as u64; + bfr_mut.buffer = iosys_map.as_mut_ptr() as *mut _; + + // Now box the returned buffer type and stash it in the private pointer of the + // `apple_rtkit_shmem` struct for safekeeping. + let boxed = KBox::new(buf, GFP_KERNEL)?; + bfr_mut.private = KBox::into_raw(boxed) as *mut _; + Ok(0) + }) +} + +unsafe extern "C" fn shmem_destroy_callback( + _cookie: *mut core::ffi::c_void, + bfr: *mut bindings::apple_rtkit_shmem, +) { + // SAFETY: `bfr` is a valid buffer + let bfr_mut = unsafe { &mut *bfr }; + if !bfr_mut.private.is_null() { + // SAFETY: Per shmem_setup_callback, this has to be a pointer to a Buffer if it is set. + unsafe { + core::mem::drop(KBox::from_raw(bfr_mut.private as *mut T::Buffer)); + } + bfr_mut.private = core::ptr::null_mut(); + } +} + +impl RtKit { + const VTABLE: bindings::apple_rtkit_ops = bindings::apple_rtkit_ops { + crashed: Some(crashed_callback::), + recv_message: Some(recv_message_callback::), + recv_message_early: Some(recv_message_early_callback::), + shmem_setup: if T::HAS_SHMEM_ALLOC || T::HAS_SHMEM_MAP { + Some(shmem_setup_callback::) + } else { + None + }, + shmem_destroy: if T::HAS_SHMEM_ALLOC || T::HAS_SHMEM_MAP { + Some(shmem_destroy_callback::) + } else { + None + }, + }; + + /// Creates a new RTKit client for a given device and optional mailbox name or index. + pub fn new( + dev: &device::Device, + mbox_name: Option<&'static CStr>, + mbox_idx: usize, + data: T::Data, + ) -> Result { + let ptr: *mut crate::ffi::c_void = data.into_foreign().cast(); + let guard = ScopeGuard::new(|| { + // SAFETY: `ptr` came from a previous call to `into_foreign`. + unsafe { T::Data::from_foreign(ptr.cast()) }; + }); + // SAFETY: `dev` is valid by its type invarants and otherwise his just + // calls the C init function. + let rtk = unsafe { + from_err_ptr(bindings::apple_rtkit_init( + dev.as_raw(), + ptr, + match mbox_name { + Some(s) => s.as_char_ptr(), + None => ptr::null(), + }, + mbox_idx.try_into()?, + &Self::VTABLE, + )) + }?; + + guard.dismiss(); + // INVARIANT: `rtk` and `data` are valid here. + Ok(Self { + rtk, + data: ptr, + _p: PhantomData, + }) + } + + /// Boots (wakes up) the RTKit coprocessor. + pub fn wake(self: Pin<&mut Self>) -> Result { + // SAFETY: `rtk` is valid per the type invariant. + to_result(unsafe { bindings::apple_rtkit_wake(self.rtk) }) + } + + /// Waits for the RTKit coprocessor to finish booting. + pub fn boot(self: Pin<&mut Self>) -> Result { + // SAFETY: `rtk` is valid per the type invariant. + to_result(unsafe { bindings::apple_rtkit_boot(self.rtk) }) + } + + /// Starts a non-system endpoint. + pub fn start_endpoint(self: Pin<&mut Self>, endpoint: u8) -> Result { + // SAFETY: `rtk` is valid per the type invariant. + to_result(unsafe { bindings::apple_rtkit_start_ep(self.rtk, endpoint) }) + } + + /// Sends a message to a given endpoint. + pub fn send_message(self: Pin<&mut Self>, endpoint: u8, message: u64) -> Result { + // SAFETY: `rtk` is valid per the type invariant. + to_result(unsafe { + bindings::apple_rtkit_send_message(self.rtk, endpoint, message, ptr::null_mut(), false) + }) + } + + /// Checks if an endpoint is present + pub fn has_endpoint(self: Pin<&mut Self>, endpoint: u8) -> bool { + unsafe { bindings::apple_rtkit_has_endpoint(self.rtk, endpoint) } + } +} + +// SAFETY: `RtKit` operations require a mutable reference +unsafe impl Sync for RtKit {} + +// SAFETY: `RtKit` operations require a mutable reference +unsafe impl Send for RtKit {} + +impl Drop for RtKit { + fn drop(&mut self) { + // SAFETY: The pointer is valid by the type invariant. + unsafe { bindings::apple_rtkit_free(self.rtk) }; + + // Free context data. + // + // SAFETY: This matches the call to `into_foreign` from `new` in the success case. + unsafe { T::Data::from_foreign(self.data.cast()) }; + } +} diff --git a/rust/kernel/soc/mod.rs b/rust/kernel/soc/mod.rs new file mode 100644 index 00000000000000..e3024042e74f0d --- /dev/null +++ b/rust/kernel/soc/mod.rs @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! SoC drivers + +pub mod apple; diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs index 46a57d1fc309d3..961e252f5e137b 100644 --- a/rust/kernel/sync/lock.rs +++ b/rust/kernel/sync/lock.rs @@ -8,6 +8,7 @@ use super::LockClassKey; use crate::{ str::{CStr, CStrExt as _}, + try_pin_init, types::{NotThreadSafe, Opaque, ScopeGuard}, }; use core::{cell::UnsafeCell, marker::PhantomPinned, pin::Pin}; @@ -143,6 +144,31 @@ impl Lock { }), }) } + + /// Constructs a new lock initialiser taking an initialiser. + pub fn pin_init( + t: impl PinInit, + name: &'static CStr, + key: &'static LockClassKey, + ) -> impl PinInit + where + E: core::convert::From, + { + try_pin_init!(Self { + // SAFETY: We are just forwarding the initialization across a + // cast away from UnsafeCell, so the pin_init_from_closure and + // __pinned_init() requirements are in sync. + data <- unsafe { pin_init::pin_init_from_closure(move |slot: *mut UnsafeCell| { + t.__pinned_init(slot as *mut T) + })}, + _pin: PhantomPinned, + // SAFETY: `slot` is valid while the closure is called and both `name` and `key` have + // static lifetimes so they live indefinitely. + state <- Opaque::ffi_init(|slot| unsafe { + B::init(slot, name.as_char_ptr(), key.as_ptr()) + }), + }? E) + } } impl Lock<(), B> { @@ -281,10 +307,7 @@ impl core::ops::Deref for Guard<'_, T, B> { } } -impl core::ops::DerefMut for Guard<'_, T, B> -where - T: Unpin, -{ +impl core::ops::DerefMut for Guard<'_, T, B> { fn deref_mut(&mut self) -> &mut Self::Target { // SAFETY: The caller owns the lock, so it is safe to deref the protected data. unsafe { &mut *self.lock.data.get() } diff --git a/rust/kernel/sync/lock/global.rs b/rust/kernel/sync/lock/global.rs index eab48108a4aebe..79d0ef7fda8676 100644 --- a/rust/kernel/sync/lock/global.rs +++ b/rust/kernel/sync/lock/global.rs @@ -106,10 +106,7 @@ impl core::ops::Deref for GlobalGuard { } } -impl core::ops::DerefMut for GlobalGuard -where - B::Item: Unpin, -{ +impl core::ops::DerefMut for GlobalGuard { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.inner } diff --git a/rust/kernel/sync/lock/mutex.rs b/rust/kernel/sync/lock/mutex.rs index 581cee7ab842ad..45a1c4c6483e90 100644 --- a/rust/kernel/sync/lock/mutex.rs +++ b/rust/kernel/sync/lock/mutex.rs @@ -17,6 +17,19 @@ macro_rules! new_mutex { } pub use new_mutex; +/// Creates a [`Mutex`] initialiser with the given name and a newly-created lock class, +/// given an initialiser for the inner type. +/// +/// It uses the name if one is given, otherwise it generates one based on the file name and line +/// number. +#[macro_export] +macro_rules! new_mutex_pinned { + ($inner:expr $(, $name:literal)? $(,)?) => { + $crate::sync::Mutex::pin_init( + $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} + /// A mutual exclusion primitive. /// /// Exposes the kernel's [`struct mutex`]. When multiple threads attempt to lock the same mutex, diff --git a/rust/kernel/task.rs b/rust/kernel/task.rs index 49fad6de06740a..cc907fb531bcee 100644 --- a/rust/kernel/task.rs +++ b/rust/kernel/task.rs @@ -204,18 +204,6 @@ impl Task { self.0.get() } - /// Returns the group leader of the given task. - pub fn group_leader(&self) -> &Task { - // SAFETY: The group leader of a task never changes after initialization, so reading this - // field is not a data race. - let ptr = unsafe { *ptr::addr_of!((*self.as_ptr()).group_leader) }; - - // SAFETY: The lifetime of the returned task reference is tied to the lifetime of `self`, - // and given that a task has a reference to its group leader, we know it must be valid for - // the lifetime of the returned task reference. - unsafe { &*ptr.cast() } - } - /// Returns the PID of the given task. pub fn pid(&self) -> Pid { // SAFETY: The pid of a task never changes after initialization, so reading this field is @@ -345,6 +333,18 @@ impl CurrentTask { // `release_task()` call. Some(unsafe { PidNamespace::from_ptr(active_ns) }) } + + /// Returns the group leader of the current task. + pub fn group_leader(&self) -> &Task { + // SAFETY: The group leader of a task never changes while the task is running, and `self` + // is the current task, which is guaranteed running. + let ptr = unsafe { (*self.as_ptr()).group_leader }; + + // SAFETY: `current->group_leader` stays valid for at least the duration in which `current` + // is running, and the signature of this function ensures that the returned `&Task` can + // only be used while `current` is still valid, thus still running. + unsafe { &*ptr.cast() } + } } // SAFETY: The type invariants guarantee that `Task` is always refcounted. diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index 9c5e7dbf16323f..4081f73f29f85c 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -6,8 +6,9 @@ use crate::ffi::c_void; use core::{ cell::UnsafeCell, marker::{PhantomData, PhantomPinned}, - mem::MaybeUninit, + mem::{ManuallyDrop, MaybeUninit}, ops::{Deref, DerefMut}, + ptr::NonNull, }; use pin_init::{PinInit, Wrapper, Zeroable}; @@ -422,6 +423,114 @@ impl Wrapper for Opaque { } } +/// Types that may be owned by Rust code or borrowed, but have a lifetime managed by C code. +/// +/// It allows such types to define their own custom destructor function to be called when +/// a Rust-owned reference is dropped. +/// +/// This is usually implemented by wrappers to existing structures on the C side of the code. +/// +/// # Safety +/// +/// Implementers must ensure that any objects borrowed directly stay alive for the duration +/// of the borrow lifetime, and that any objects deemed owned by Rust stay alive while +/// that owned reference exists, until the [`Ownable::release()`] function is called. +pub unsafe trait Ownable { + /// Releases the object (frees it or returns it to foreign ownership). + /// + /// # Safety + /// + /// Callers must ensure that the object is no longer referenced after this call. + unsafe fn release(this: NonNull); +} + +/// A subtrait of Ownable that asserts that an Owned Rust reference is not only unique +/// within Rust, but also follows the same rules in kernel C code. That is, the kernel +/// will never mutate the contents of the object while Rust owns it. +/// +/// When this type is implemented for an Ownable type, it allows Owned to be dereferenced +/// into a &mut T. + +/// # Safety +/// +/// Implementers must ensure that the kernel never mutates the underlying type while +/// Rust owns it. +pub unsafe trait OwnableMut: Ownable {} + +/// An owned reference to an ownable kernel object. +/// +/// The object is automatically freed or released when an instance of [`Owned`] is +/// dropped. +/// +/// # Invariants +/// +/// The pointer stored in `ptr` is non-null and valid for the lifetime of the [`Owned`] instance. +pub struct Owned { + ptr: NonNull, + _p: PhantomData, +} + +// SAFETY: It is safe to send `Owned` to another thread when the underlying `T` is `Send` because +// it effectively means sharing `&mut T` (which is safe because `T` is `Send`). +unsafe impl Send for Owned {} + +// SAFETY: It is safe to send `&Owned` to another thread when the underlying `T` is `Sync` +// because it effectively means sharing `&T` (which is safe because `T` is `Sync`). +unsafe impl Sync for Owned {} + +impl Owned { + /// Creates a new instance of [`Owned`]. + /// + /// It takes over ownership of the underlying object. + /// + /// # Safety + /// + /// Callers must ensure that the underlying object is acquired and can be considered owned by + /// Rust. + pub unsafe fn from_raw(ptr: NonNull) -> Self { + // INVARIANT: The safety requirements guarantee that the new instance now owns the + // reference. + Self { + ptr, + _p: PhantomData, + } + } + + /// Consumes the `Owned`, returning a raw pointer. + /// + /// This function does not actually relinquish ownership of the object. + /// After calling this function, the caller is responsible for ownership previously managed + /// by the `Owned`. + pub fn into_raw(me: Self) -> NonNull { + ManuallyDrop::new(me).ptr + } +} + +impl Deref for Owned { + type Target = T; + + fn deref(&self) -> &Self::Target { + // SAFETY: The type invariants guarantee that the object is valid. + unsafe { self.ptr.as_ref() } + } +} + +impl DerefMut for Owned { + fn deref_mut(&mut self) -> &mut Self::Target { + // SAFETY: The type invariants guarantee that the object is valid, + // and that we can safely return a mutable reference to it. + unsafe { self.ptr.as_mut() } + } +} + +impl Drop for Owned { + fn drop(&mut self) { + // SAFETY: The type invariants guarantee that the `Owned` owns the object we're about to + // release. + unsafe { T::release(self.ptr) }; + } +} + /// Zero-sized type to mark types not [`Send`]. /// /// Add this type as a field to your struct if your type should not be sent to a different task. @@ -442,3 +551,86 @@ pub type NotThreadSafe = PhantomData<*mut ()>; /// [`NotThreadSafe`]: type@NotThreadSafe #[allow(non_upper_case_globals)] pub const NotThreadSafe: NotThreadSafe = PhantomData; + +/// Helper macro to declare a bitfield style type. The type will automatically +/// gain boolean operator implementations, as well as the `as_raw()` and `contains()` +/// methods, Debug, Copy, Clone, and PartialEq implementations. +/// +/// Optionally, a default value can be specified with `= value` syntax, which +/// will add a Default trait implementation. +/// +/// # Examples +/// +/// ``` +/// declare_flags_type! { +/// /// Flags to be used for foo. +/// pub struct FooFlags(u32); +/// } +/// +/// declare_flags_type! { +/// /// Flags to be used for bar. +/// pub struct BarFlags(u32) = 0; +/// } +/// ``` +macro_rules! declare_flags_type ( + ( + $(#[$outer:meta])* + $v:vis struct $t:ident ( $base:ty ); + $($rest:tt)* + ) => { + $(#[$outer])* + #[derive(Debug, Clone, Copy, PartialEq)] + $v struct $t($base); + + impl $t { + /// Get the raw representation of this flag. + pub(crate) fn as_raw(self) -> $base { + self.0 + } + + /// Check whether `flags` is contained in `self`. + pub fn contains(self, flags: Self) -> bool { + (self & flags) == flags + } + } + + impl core::ops::BitOr for $t { + type Output = Self; + fn bitor(self, rhs: Self) -> Self::Output { + Self(self.0 | rhs.0) + } + } + + impl core::ops::BitAnd for $t { + type Output = Self; + fn bitand(self, rhs: Self) -> Self::Output { + Self(self.0 & rhs.0) + } + } + + impl core::ops::Not for $t { + type Output = Self; + fn not(self) -> Self::Output { + Self(!self.0) + } + } + }; + ( + $(#[$outer:meta])* + $v:vis struct $t:ident ( $base:ty ) = $default:expr; + $($rest:tt)* + ) => { + declare_flags_type! { + $(#[$outer])* + $v struct $t ($base); + $($rest)* + } + impl Default for $t { + fn default() -> Self { + Self($default) + } + } + }; +); + +pub(crate) use declare_flags_type; diff --git a/rust/kernel/xarray.rs b/rust/kernel/xarray.rs index a49d6db2884588..e1f7e2d9b629f0 100644 --- a/rust/kernel/xarray.rs +++ b/rust/kernel/xarray.rs @@ -5,17 +5,20 @@ //! C header: [`include/linux/xarray.h`](srctree/include/linux/xarray.h) use crate::{ - alloc, bindings, build_assert, - error::{Error, Result}, - ffi::c_void, - types::{ForeignOwnable, NotThreadSafe, Opaque}, + alloc, + prelude::*, + types::{ForeignOwnable, NotThreadSafe, Opaque, ScopeGuard}, +}; +use core::{ + fmt, iter, + marker::PhantomData, + mem, ops, + ptr::{null_mut, NonNull}, }; -use core::{iter, marker::PhantomData, pin::Pin, ptr::NonNull}; -use pin_init::{pin_data, pin_init, pinned_drop, PinInit}; /// An array which efficiently maps sparse integer indices to owned objects. /// -/// This is similar to a [`crate::alloc::kvec::Vec>`], but more efficient when there are +/// This is similar to a [`Vec>`], but more efficient when there are /// holes in the index space, and can be efficiently grown. /// /// # Invariants @@ -26,10 +29,11 @@ use pin_init::{pin_data, pin_init, pinned_drop, PinInit}; /// # Examples /// /// ```rust -/// use kernel::alloc::KBox; -/// use kernel::xarray::{AllocKind, XArray}; +/// # use kernel::alloc::KBox; +/// # use kernel::xarray::XArray; +/// # use pin_init::stack_pin_init; /// -/// let xa = KBox::pin_init(XArray::new(AllocKind::Alloc1), GFP_KERNEL)?; +/// stack_pin_init!(let xa = XArray::new(Default::default())); /// /// let dead = KBox::new(0xdead, GFP_KERNEL)?; /// let beef = KBox::new(0xbeef, GFP_KERNEL)?; @@ -77,8 +81,10 @@ impl PinnedDrop for XArray { } /// Flags passed to [`XArray::new`] to configure the array's allocation tracking behavior. +#[derive(Default)] pub enum AllocKind { /// Consider the first element to be at index 0. + #[default] Alloc, /// Consider the first element to be at index 1. Alloc1, @@ -105,19 +111,69 @@ impl XArray { fn iter(&self) -> impl Iterator> + '_ { let mut index = 0; - // SAFETY: `self.xa` is always valid by the type invariant. - iter::once(unsafe { - bindings::xa_find(self.xa.get(), &mut index, usize::MAX, bindings::XA_PRESENT) - }) - .chain(iter::from_fn(move || { + core::iter::Iterator::chain( // SAFETY: `self.xa` is always valid by the type invariant. - Some(unsafe { - bindings::xa_find_after(self.xa.get(), &mut index, usize::MAX, bindings::XA_PRESENT) - }) - })) + iter::once(unsafe { + bindings::xa_find(self.xa.get(), &mut index, usize::MAX, bindings::XA_PRESENT) + }), + iter::from_fn(move || { + // SAFETY: `self.xa` is always valid by the type invariant. + Some(unsafe { + bindings::xa_find_after( + self.xa.get(), + &mut index, + usize::MAX, + bindings::XA_PRESENT, + ) + }) + }), + ) .map_while(|ptr| NonNull::new(ptr.cast())) } + /// Looks up and returns a reference to the lowest entry in the array between index and max, + /// returning a tuple of its index and a `Guard` if one exists. + /// + /// This guard blocks all other actions on the `XArray`. Callers are expected to drop the + /// `Guard` eagerly to avoid blocking other users, such as by taking a clone of the value. + pub fn find(&self, index: usize, max: usize) -> Option<(usize, ValueGuard<'_, T>)> { + let mut index: usize = index; + + // SAFETY: `self.xa` is always valid by the type invariant. + unsafe { bindings::xa_lock(self.xa.get()) }; + + // SAFETY: `self.xa` is always valid by the type invariant. + let guard = ScopeGuard::new(|| unsafe { bindings::xa_unlock(self.xa.get()) }); + + // SAFETY: `self.xa` is always valid by the type invariant. + let p = unsafe { bindings::xa_find(self.xa.get(), &mut index, max, bindings::XA_PRESENT) }; + + NonNull::new(p as *mut T).map(|ptr| { + guard.dismiss(); + ( + index, + ValueGuard { + xa: self, + ptr, + _not_send: NotThreadSafe, + }, + ) + }) + } + + fn with_guard(&self, guard: Option<&mut Guard<'_, T>>, f: F) -> U + where + F: FnOnce(&mut Guard<'_, T>) -> U, + { + match guard { + None => f(&mut self.lock()), + Some(guard) => { + assert_eq!(guard.xa.xa.get(), self.xa.get()); + f(guard) + } + } + } + /// Attempts to lock the [`XArray`] for exclusive access. pub fn try_lock(&self) -> Option> { // SAFETY: `self.xa` is always valid by the type invariant. @@ -141,6 +197,12 @@ impl XArray { _not_send: NotThreadSafe, } } + + /// Removes and returns the element at the given index. + pub fn remove(&self, index: usize) -> Option { + let mut guard = self.lock(); + guard.remove(index) + } } /// A lock guard. @@ -161,9 +223,41 @@ impl Drop for Guard<'_, T> { } } +/// A lock guard. +/// +/// The lock is unlocked when the guard goes out of scope. +#[must_use = "the lock unlocks immediately when the guard is unused"] +pub struct ValueGuard<'a, T: ForeignOwnable> { + xa: &'a XArray, + ptr: NonNull, + _not_send: NotThreadSafe, +} + +impl<'a, T: ForeignOwnable> ValueGuard<'a, T> { + /// Borrow the underlying value wrapped by the `Guard`. + /// + /// Returns a `T::Borrowed` type for the owned `ForeignOwnable` type. + pub fn borrow(&self) -> T::Borrowed<'_> { + // SAFETY: The value is owned by the `XArray`, the lifetime it is borrowed for must not + // outlive the `XArray` itself, nor the Guard that holds the lock ensuring the value + // remains in the `XArray`. + unsafe { T::borrow(self.ptr.as_ptr() as _) } + } +} + +impl Drop for ValueGuard<'_, T> { + fn drop(&mut self) { + // SAFETY: + // - `self.xa.xa` is always valid by the type invariant. + // - The caller holds the lock, so it is safe to unlock it. + unsafe { bindings::xa_unlock(self.xa.xa.get()) }; + } +} + /// The error returned by [`store`](Guard::store). /// /// Contains the underlying error and the value that was not stored. +#[derive(Debug)] pub struct StoreError { /// The error that occurred. pub error: Error, @@ -177,6 +271,11 @@ impl From> for Error { } } +fn to_usize(i: u32) -> usize { + i.try_into() + .unwrap_or_else(|_| build_error!("cannot convert u32 to usize")) +} + impl<'a, T: ForeignOwnable> Guard<'a, T> { fn load(&self, index: usize, f: F) -> Option where @@ -211,7 +310,7 @@ impl<'a, T: ForeignOwnable> Guard<'a, T> { // - The caller holds the lock. let ptr = unsafe { bindings::__xa_erase(self.xa.xa.get(), index) }.cast(); // SAFETY: - // - `ptr` is either NULL or came from `T::into_foreign`. + // - `ptr` is either `NULL` or came from `T::into_foreign`. // - `&mut self` guarantees that the lifetimes of [`T::Borrowed`] and [`T::BorrowedMut`] // borrowed from `self` have ended. unsafe { T::try_from_foreign(ptr) } @@ -259,13 +358,272 @@ impl<'a, T: ForeignOwnable> Guard<'a, T> { }) } else { let old = old.cast(); - // SAFETY: `ptr` is either NULL or came from `T::into_foreign`. + // SAFETY: `ptr` is either `NULL` or came from `T::into_foreign`. // // NB: `XA_ZERO_ENTRY` is never returned by functions belonging to the Normal XArray // API; such entries present as `NULL`. Ok(unsafe { T::try_from_foreign(old) }) } } + + /// Stores an element at the given index if no entry is present. + /// + /// May drop the lock if needed to allocate memory, and then reacquire it afterwards. + /// + /// On failure, returns the element which was attempted to be stored. + pub fn insert( + &mut self, + index: usize, + value: T, + gfp: alloc::Flags, + ) -> Result<(), StoreError> { + build_assert!( + T::FOREIGN_ALIGN >= 4, + "pointers stored in XArray must be 4-byte aligned" + ); + let ptr = value.into_foreign(); + // SAFETY: `self.xa` is always valid by the type invariant. + // + // INVARIANT: `ptr` came from `T::into_foreign`. + match unsafe { bindings::__xa_insert(self.xa.xa.get(), index, ptr.cast(), gfp.as_raw()) } { + 0 => Ok(()), + errno => { + // SAFETY: `ptr` came from `T::into_foreign` and `__xa_insert` does not take + // ownership of the value on error. + let value = unsafe { T::from_foreign(ptr) }; + Err(StoreError { + value, + error: Error::from_errno(errno), + }) + } + } + } + + /// Wrapper around `__xa_alloc`. + /// + /// On success, takes ownership of pointers passed in `op`. + /// + /// On failure, ownership returns to the caller. + /// + /// # Safety + /// + /// `ptr` must be `NULL` or have come from a previous call to `T::into_foreign`. + unsafe fn alloc( + &mut self, + limit: impl ops::RangeBounds, + ptr: *mut c_void, + gfp: alloc::Flags, + ) -> Result { + // NB: `xa_limit::{max,min}` are inclusive. + let limit = bindings::xa_limit { + max: match limit.end_bound() { + ops::Bound::Included(&end) => end, + ops::Bound::Excluded(&end) => end - 1, + ops::Bound::Unbounded => u32::MAX, + }, + min: match limit.start_bound() { + ops::Bound::Included(&start) => start, + ops::Bound::Excluded(&start) => start + 1, + ops::Bound::Unbounded => 0, + }, + }; + + let mut index = u32::MAX; + + // SAFETY: + // - `self.xa` is always valid by the type invariant. + // - `self.xa` was initialized with `XA_FLAGS_ALLOC` or `XA_FLAGS_ALLOC1`. + // + // INVARIANT: `ptr` is either `NULL` or came from `T::into_foreign`. + match unsafe { + bindings::__xa_alloc( + self.xa.xa.get(), + &mut index, + ptr.cast(), + limit, + gfp.as_raw(), + ) + } { + 0 => Ok(to_usize(index)), + errno => Err(Error::from_errno(errno)), + } + } + + /// Allocates an entry somewhere in the array. + /// + /// On success, returns the index at which the entry was stored. + /// + /// On failure, returns the entry which was attempted to be stored. + pub fn insert_limit( + &mut self, + limit: impl ops::RangeBounds, + value: T, + gfp: alloc::Flags, + ) -> Result> { + build_assert!( + T::FOREIGN_ALIGN >= 4, + "pointers stored in XArray must be 4-byte aligned" + ); + let ptr = value.into_foreign(); + // SAFETY: `ptr` came from `T::into_foreign`. + unsafe { self.alloc(limit, ptr, gfp) }.map_err(|error| { + // SAFETY: `ptr` came from `T::into_foreign` and `self.alloc` does not take ownership of + // the value on error. + let value = unsafe { T::from_foreign(ptr) }; + StoreError { value, error } + }) + } + + /// Reserves an entry in the array. + pub fn reserve(&mut self, index: usize, gfp: alloc::Flags) -> Result> { + // NB: `__xa_insert` internally coerces `NULL` to `XA_ZERO_ENTRY` on ingress. + let ptr = null_mut(); + // SAFETY: `self.xa` is always valid by the type invariant. + // + // INVARIANT: `ptr` is `NULL`. + match unsafe { bindings::__xa_insert(self.xa.xa.get(), index, ptr, gfp.as_raw()) } { + 0 => Ok(Reservation { xa: self.xa, index }), + errno => Err(Error::from_errno(errno)), + } + } + + /// Reserves an entry somewhere in the array. + pub fn reserve_limit( + &mut self, + limit: impl ops::RangeBounds, + gfp: alloc::Flags, + ) -> Result> { + // NB: `__xa_alloc` internally coerces `NULL` to `XA_ZERO_ENTRY` on ingress. + let ptr = null_mut(); + // SAFETY: `ptr` is `NULL`. + unsafe { self.alloc(limit, ptr, gfp) }.map(|index| Reservation { xa: self.xa, index }) + } +} + +/// A reserved slot in an array. +/// +/// The slot is released when the reservation goes out of scope. +/// +/// Note that the array lock *must not* be held when the reservation is filled or dropped as this +/// will lead to deadlock. [`Reservation::fill_locked`] and [`Reservation::release_locked`] can be +/// used in context where the array lock is held. +#[must_use = "the reservation is released immediately when the reservation is unused"] +pub struct Reservation<'a, T: ForeignOwnable> { + xa: &'a XArray, + index: usize, +} + +impl fmt::Debug for Reservation<'_, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Reservation") + .field("index", &self.index()) + .finish() + } +} + +impl Reservation<'_, T> { + /// Returns the index of the reservation. + pub fn index(&self) -> usize { + self.index + } + + /// Replaces the reserved entry with the given entry. + /// + /// # Safety + /// + /// `ptr` must be `NULL` or have come from a previous call to `T::into_foreign`. + unsafe fn replace(guard: &mut Guard<'_, T>, index: usize, ptr: *mut c_void) -> Result { + // SAFETY: `xa_zero_entry` wraps `XA_ZERO_ENTRY` which is always safe to use. + let old = unsafe { bindings::xa_zero_entry() }; + + // NB: `__xa_cmpxchg_raw` is used over `__xa_cmpxchg` because the latter coerces + // `XA_ZERO_ENTRY` to `NULL` on egress, which would prevent us from determining whether a + // replacement was made. + // + // SAFETY: `self.xa` is always valid by the type invariant. + // + // INVARIANT: `ptr` is either `NULL` or came from `T::into_foreign` and `old` is + // `XA_ZERO_ENTRY`. + let ret = + unsafe { bindings::__xa_cmpxchg_raw(guard.xa.xa.get(), index, old, ptr.cast(), 0) }; + + // SAFETY: `__xa_cmpxchg_raw` returns the old entry at this index on success or `xa_err` if + // an error happened. + match unsafe { bindings::xa_err(ret) } { + 0 => { + if ret == old { + Ok(()) + } else { + Err(EBUSY) + } + } + errno => Err(Error::from_errno(errno)), + } + } + + fn fill_inner(&self, guard: Option<&mut Guard<'_, T>>, value: T) -> Result<(), StoreError> { + let Self { xa, index } = self; + let index = *index; + + let ptr = value.into_foreign(); + xa.with_guard(guard, |guard| { + // SAFETY: `ptr` came from `T::into_foreign`. + unsafe { Self::replace(guard, index, ptr) } + }) + .map_err(|error| { + // SAFETY: `ptr` came from `T::into_foreign` and `Self::replace` does not take ownership + // of the value on error. + let value = unsafe { T::from_foreign(ptr) }; + StoreError { value, error } + }) + } + + /// Fills the reservation. + pub fn fill(self, value: T) -> Result<(), StoreError> { + let result = self.fill_inner(None, value); + mem::forget(self); + result + } + + /// Fills the reservation without acquiring the array lock. + /// + /// # Panics + /// + /// Panics if the passed guard locks a different array. + pub fn fill_locked(self, guard: &mut Guard<'_, T>, value: T) -> Result<(), StoreError> { + let result = self.fill_inner(Some(guard), value); + mem::forget(self); + result + } + + fn release_inner(&self, guard: Option<&mut Guard<'_, T>>) -> Result { + let Self { xa, index } = self; + let index = *index; + + xa.with_guard(guard, |guard| { + let ptr = null_mut(); + // SAFETY: `ptr` is `NULL`. + unsafe { Self::replace(guard, index, ptr) } + }) + } + + /// Releases the reservation without acquiring the array lock. + /// + /// # Panics + /// + /// Panics if the passed guard locks a different array. + pub fn release_locked(self, guard: &mut Guard<'_, T>) -> Result { + let result = self.release_inner(Some(guard)); + mem::forget(self); + result + } +} + +impl Drop for Reservation<'_, T> { + fn drop(&mut self) { + // NB: Errors here are possible since `Guard::store` does not honor reservations. + let _: Result = self.release_inner(None); + } } // SAFETY: `XArray` has no shared mutable state so it is `Send` iff `T` is `Send`. @@ -274,3 +632,133 @@ unsafe impl Send for XArray {} // SAFETY: `XArray` serialises the interior mutability it provides so it is `Sync` iff `T` is // `Send`. unsafe impl Sync for XArray {} + +#[macros::kunit_tests(rust_xarray_kunit)] +mod tests { + use super::*; + use pin_init::stack_pin_init; + + fn new_kbox(value: T) -> Result> { + KBox::new(value, GFP_KERNEL).map_err(Into::into) + } + + #[test] + fn test_alloc_kind_alloc() -> Result { + test_alloc_kind(AllocKind::Alloc, 0) + } + + #[test] + fn test_alloc_kind_alloc1() -> Result { + test_alloc_kind(AllocKind::Alloc1, 1) + } + + fn test_alloc_kind(kind: AllocKind, expected_index: usize) -> Result { + stack_pin_init!(let xa = XArray::new(kind)); + let mut guard = xa.lock(); + + let reservation = guard.reserve_limit(.., GFP_KERNEL)?; + assert_eq!(reservation.index(), expected_index); + reservation.release_locked(&mut guard)?; + + let insertion = guard.insert_limit(.., new_kbox(0x1337)?, GFP_KERNEL); + assert!(insertion.is_ok()); + let insertion_index = insertion.unwrap(); + assert_eq!(insertion_index, expected_index); + + Ok(()) + } + + const IDX: usize = 0x1337; + + fn insert(guard: &mut Guard<'_, T>, value: T) -> Result<(), StoreError> { + guard.insert(IDX, value, GFP_KERNEL) + } + + fn reserve<'a, T: ForeignOwnable>(guard: &mut Guard<'a, T>) -> Result> { + guard.reserve(IDX, GFP_KERNEL) + } + + #[track_caller] + fn check_not_vacant<'a>(guard: &mut Guard<'a, KBox>) -> Result { + // Insertion fails. + { + let beef = new_kbox(0xbeef)?; + let ret = insert(guard, beef); + assert!(ret.is_err()); + let StoreError { error, value } = ret.unwrap_err(); + assert_eq!(error, EBUSY); + assert_eq!(*value, 0xbeef); + } + + // Reservation fails. + { + let ret = reserve(guard); + assert!(ret.is_err()); + assert_eq!(ret.unwrap_err(), EBUSY); + } + + Ok(()) + } + + #[test] + fn test_insert_and_reserve_interaction() -> Result { + stack_pin_init!(let xa = XArray::new(Default::default())); + let mut guard = xa.lock(); + + // Vacant. + assert_eq!(guard.get(IDX), None); + + // Reservation succeeds. + let reservation = { + let ret = reserve(&mut guard); + assert!(ret.is_ok()); + ret.unwrap() + }; + + // Reserved presents as vacant. + assert_eq!(guard.get(IDX), None); + + check_not_vacant(&mut guard)?; + + // Release reservation. + { + let ret = reservation.release_locked(&mut guard); + assert!(ret.is_ok()); + let () = ret.unwrap(); + } + + // Vacant again. + assert_eq!(guard.get(IDX), None); + + // Insert succeeds. + { + let dead = new_kbox(0xdead)?; + let ret = insert(&mut guard, dead); + assert!(ret.is_ok()); + let () = ret.unwrap(); + } + + check_not_vacant(&mut guard)?; + + // Remove. + assert_eq!(guard.remove(IDX).as_deref(), Some(&0xdead)); + + // Reserve and fill. + { + let beef = new_kbox(0xbeef)?; + let ret = reserve(&mut guard); + assert!(ret.is_ok()); + let reservation = ret.unwrap(); + let ret = reservation.fill_locked(&mut guard, beef); + assert!(ret.is_ok()); + let () = ret.unwrap(); + }; + + check_not_vacant(&mut guard)?; + + // Remove. + assert_eq!(guard.remove(IDX).as_deref(), Some(&0xbeef)); + + Ok(()) + } +} diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs index 33f66e86418ab8..a6fc27ce5b2ee4 100644 --- a/rust/macros/lib.rs +++ b/rust/macros/lib.rs @@ -20,6 +20,7 @@ mod helpers; mod kunit; mod module; mod paste; +mod versions; mod vtable; use proc_macro::TokenStream; @@ -135,6 +136,12 @@ pub fn module(ts: TokenStream) -> TokenStream { module::module(ts) } +/// Declares multiple variants of a structure or impl code +#[proc_macro_attribute] +pub fn versions(attr: TokenStream, item: TokenStream) -> TokenStream { + versions::versions(attr, item) +} + /// Declares or implements a vtable trait. /// /// Linux's use of pure vtables is very close to Rust traits, but they differ diff --git a/rust/macros/versions.rs b/rust/macros/versions.rs new file mode 100644 index 00000000000000..b13a5d55c0e17b --- /dev/null +++ b/rust/macros/versions.rs @@ -0,0 +1,341 @@ +use proc_macro::{Delimiter, Group, Ident, Punct, Spacing, Span, TokenStream, TokenTree}; + +//use crate::helpers::expect_punct; + +fn expect_group(it: &mut impl Iterator) -> Group { + if let Some(TokenTree::Group(group)) = it.next() { + group + } else { + panic!("Expected Group") + } +} + +fn expect_punct(it: &mut impl Iterator) -> String { + if let Some(TokenTree::Punct(punct)) = it.next() { + punct.to_string() + } else { + panic!("Expected Group") + } +} + +fn drop_until_punct(it: &mut impl Iterator, delimiter: &str, is_struct: bool) { + let mut depth: isize = 0; + let mut colons: isize = 0; + for token in it.by_ref() { + if let TokenTree::Punct(punct) = token { + match punct.as_char() { + ':' => { + colons += 1; + } + '<' => { + if depth > 0 || colons == 2 || is_struct { + depth += 1; + } + colons = 0; + } + '>' => { + if depth > 0 { + depth -= 1; + } + colons = 0; + } + _ => { + colons = 0; + if depth == 0 && delimiter.contains(&punct.to_string()) { + break; + } + } + } + } + } +} + +fn drop_until_braces(it: &mut impl Iterator) { + let mut depth: isize = 0; + let mut colons: isize = 0; + for token in it.by_ref() { + match token { + TokenTree::Punct(punct) => match punct.as_char() { + ':' => { + colons += 1; + } + '<' => { + if depth > 0 || colons == 2 { + depth += 1; + } + colons = 0; + } + '>' => { + if depth > 0 { + depth -= 1; + } + colons = 0; + } + _ => colons = 0, + }, + TokenTree::Group(group) if group.delimiter() == Delimiter::Brace => { + if depth == 0 { + break; + } + } + _ => (), + } + } +} + +struct VersionConfig { + fields: &'static [&'static str], + enums: &'static [&'static [&'static str]], + versions: &'static [&'static [&'static str]], +} + +static AGX_VERSIONS: VersionConfig = VersionConfig { + fields: &["G", "V"], + enums: &[ + &["G13", "G14", "G14X"], + &["V12_3", "V12_4", "V13_0B4", "V13_2", "V13_3", "V13_5"], + ], + versions: &[ + &["G13", "V12_3"], + &["G14", "V12_4"], + &["G13", "V13_5"], + &["G14", "V13_5"], + &["G14X", "V13_5"], + ], +}; + +fn check_version( + config: &VersionConfig, + ver: &[usize], + it: &mut impl Iterator, +) -> bool { + let first = it.next().unwrap(); + let val: bool = match &first { + TokenTree::Group(group) => check_version(config, ver, &mut group.stream().into_iter()), + TokenTree::Ident(ident) => { + let key = config + .fields + .iter() + .position(|&r| r == ident.to_string()) + .unwrap_or_else(|| panic!("Unknown field {}", ident)); + let mut operator = expect_punct(it); + let mut rhs_token = it.next().unwrap(); + if let TokenTree::Punct(punct) = &rhs_token { + operator.extend(std::iter::once(punct.as_char())); + rhs_token = it.next().unwrap(); + } + let rhs_name = if let TokenTree::Ident(ident) = &rhs_token { + ident.to_string() + } else { + panic!("Unexpected token {}", ident) + }; + + let rhs = config.enums[key] + .iter() + .position(|&r| r == rhs_name) + .unwrap_or_else(|| panic!("Unknown value for {}:{}", ident, rhs_name)); + let lhs = ver[key]; + + match operator.as_str() { + "==" => lhs == rhs, + "!=" => lhs != rhs, + ">" => lhs > rhs, + ">=" => lhs >= rhs, + "<" => lhs < rhs, + "<=" => lhs <= rhs, + _ => panic!("Unknown operator {}", operator), + } + } + _ => { + panic!("Unknown token {}", first) + } + }; + + let boolop = it.next(); + match boolop { + Some(TokenTree::Punct(punct)) => { + let right = expect_punct(it); + if right != punct.to_string() { + panic!("Unexpected op {}{}", punct, right); + } + match punct.as_char() { + '&' => val && check_version(config, ver, it), + '|' => val || check_version(config, ver, it), + _ => panic!("Unexpected op {}{}", right, right), + } + } + Some(a) => panic!("Unexpected op {}", a), + None => val, + } +} + +fn filter_versions( + config: &VersionConfig, + tag: &str, + ver: &[usize], + tree: impl IntoIterator, + is_struct: bool, +) -> Vec { + let mut out = Vec::::new(); + let mut it = tree.into_iter(); + + while let Some(token) = it.next() { + let mut tail: Option = None; + match &token { + TokenTree::Punct(punct) if punct.to_string() == "#" => { + let group = expect_group(&mut it); + let mut grp_it = group.stream().into_iter(); + let attr = grp_it.next().unwrap(); + match attr { + TokenTree::Ident(ident) if ident.to_string() == "ver" => { + if check_version(config, ver, &mut grp_it) { + } else if is_struct { + drop_until_punct(&mut it, ",", true); + } else { + let first = it.next().unwrap(); + match &first { + TokenTree::Ident(ident) + if ["while", "for", "loop", "if", "match", "unsafe", "fn"] + .contains(&ident.to_string().as_str()) => + { + drop_until_braces(&mut it); + } + TokenTree::Group(_) => (), + _ => { + drop_until_punct(&mut it, ",;", false); + } + } + } + } + _ => { + out.push(token.clone()); + out.push(TokenTree::Group(group.clone())); + } + } + continue; + } + TokenTree::Punct(punct) if punct.to_string() == ":" => { + let next = it.next(); + match next { + Some(TokenTree::Punct(punct)) if punct.to_string() == ":" => { + let next = it.next(); + match next { + Some(TokenTree::Ident(idtag)) if idtag.to_string() == "ver" => { + let ident = match out.pop() { + Some(TokenTree::Ident(ident)) => ident, + a => panic!("$ver not following ident: {:?}", a), + }; + let name = ident.to_string() + tag; + let new_ident = Ident::new(name.as_str(), ident.span()); + out.push(TokenTree::Ident(new_ident)); + continue; + } + Some(a) => { + out.push(token.clone()); + out.push(token.clone()); + tail = Some(a); + } + None => { + out.push(token.clone()); + out.push(token.clone()); + } + } + } + Some(a) => { + out.push(token.clone()); + tail = Some(a); + } + None => { + out.push(token.clone()); + continue; + } + } + } + _ => { + tail = Some(token); + } + } + match &tail { + Some(TokenTree::Group(group)) => { + let new_body = + filter_versions(config, tag, ver, group.stream().into_iter(), is_struct); + let mut stream = TokenStream::new(); + stream.extend(new_body); + let mut filtered_group = Group::new(group.delimiter(), stream); + filtered_group.set_span(group.span()); + out.push(TokenTree::Group(filtered_group)); + } + Some(token) => { + out.push(token.clone()); + } + None => {} + } + } + + out +} + +pub(crate) fn versions(attr: TokenStream, item: TokenStream) -> TokenStream { + let config = match attr.to_string().as_str() { + "AGX" => &AGX_VERSIONS, + _ => panic!("Unknown version group {}", attr), + }; + + let mut it = item.into_iter(); + let mut out = TokenStream::new(); + let mut body: Vec = Vec::new(); + let mut is_struct = false; + + while let Some(token) = it.next() { + match token { + TokenTree::Punct(punct) if punct.to_string() == "#" => { + body.push(TokenTree::Punct(punct)); + body.push(it.next().unwrap()); + } + TokenTree::Ident(ident) + if ["struct", "enum", "union", "const", "type"] + .contains(&ident.to_string().as_str()) => + { + is_struct = ident.to_string() != "const"; + body.push(TokenTree::Ident(ident)); + body.push(it.next().unwrap()); + // This isn't valid syntax in a struct definition, so add it for the user + body.push(TokenTree::Punct(Punct::new(':', Spacing::Joint))); + body.push(TokenTree::Punct(Punct::new(':', Spacing::Alone))); + body.push(TokenTree::Ident(Ident::new("ver", Span::call_site()))); + break; + } + TokenTree::Ident(ident) if ident.to_string() == "impl" => { + body.push(TokenTree::Ident(ident)); + break; + } + TokenTree::Ident(ident) if ident.to_string() == "fn" => { + body.push(TokenTree::Ident(ident)); + break; + } + _ => { + body.push(token); + } + } + } + + body.extend(it); + + for ver in config.versions { + let tag = ver.join(""); + let mut ver_num = Vec::::new(); + for (i, comp) in ver.iter().enumerate() { + let idx = config.enums[i].iter().position(|&r| r == *comp).unwrap(); + ver_num.push(idx); + } + out.extend(filter_versions( + config, + &tag, + &ver_num, + body.clone(), + is_struct, + )); + } + + out +} diff --git a/rust/pin-init/src/lib.rs b/rust/pin-init/src/lib.rs index 8dc9dd5ac6fd32..955a83e7041b62 100644 --- a/rust/pin-init/src/lib.rs +++ b/rust/pin-init/src/lib.rs @@ -780,10 +780,10 @@ macro_rules! stack_try_pin_init { // module `macros` inside of `macros.rs`. #[macro_export] macro_rules! pin_init { - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + ($(&$this:ident in)? $t:ident $(::$p:ident)* $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }) => { - $crate::try_pin_init!($(&$this in)? $t $(::<$($generics),*>)? { + $crate::try_pin_init!($(&$this in)? $t $(::$p)* $(::<$($generics),*>)? { $($fields)* }? ::core::convert::Infallible) }; @@ -831,12 +831,12 @@ macro_rules! pin_init { // module `macros` inside of `macros.rs`. #[macro_export] macro_rules! try_pin_init { - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + ($(&$this:ident in)? $t:ident $(::$p:ident)* $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }? $err:ty) => { $crate::__init_internal!( @this($($this)?), - @typ($t $(::<$($generics),*>)? ), + @typ($t $(::$p)* $(::<$($generics),*>)? ), @fields($($fields)*), @error($err), @data(PinData, use_data), @@ -887,10 +887,10 @@ macro_rules! try_pin_init { // module `macros` inside of `macros.rs`. #[macro_export] macro_rules! init { - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + ($(&$this:ident in)? $t:ident $(::$p:ident)* $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }) => { - $crate::try_init!($(&$this in)? $t $(::<$($generics),*>)? { + $crate::try_init!($(&$this in)? $t $(::$p)* $(::<$($generics),*>)? { $($fields)* }? ::core::convert::Infallible) } @@ -936,12 +936,12 @@ macro_rules! init { // module `macros` inside of `macros.rs`. #[macro_export] macro_rules! try_init { - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + ($(&$this:ident in)? $t:ident $(::$p:ident)* $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }? $err:ty) => { $crate::__init_internal!( @this($($this)?), - @typ($t $(::<$($generics),*>)?), + @typ($t $(::$p)* $(::<$($generics),*>)?), @fields($($fields)*), @error($err), @data(InitData, /*no use_data*/), @@ -1574,6 +1574,21 @@ pub unsafe trait PinnedDrop: __internal::HasPinData { fn drop(self: Pin<&mut Self>, only_call_from_drop: __internal::OnlyCallFromDrop); } +/// Create a new default T. +/// +/// The returned initializer will use Default::default to initialize the `slot`. +#[inline] +pub fn default() -> impl Init { + // SAFETY: Because `T: Default`, T cannot require pinning and + // we can just move the data into the slot. + unsafe { + init_from_closure(|slot: *mut T| { + *slot = Default::default(); + Ok(()) + }) + } +} + /// Marker trait for types that can be initialized by writing just zeroes. /// /// # Safety diff --git a/rust/uapi/uapi_helper.h b/rust/uapi/uapi_helper.h index 06d7d1a2e8daba..512ac0aea08fde 100644 --- a/rust/uapi/uapi_helper.h +++ b/rust/uapi/uapi_helper.h @@ -7,10 +7,13 @@ */ #include +#include #include #include #include #include +#include +#include #include #include #include diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 32e209bc7985cb..5d158072db95cb 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -321,6 +321,9 @@ $(obj)/%.lst: $(obj)/%.c FORCE # the unstable features in use. rust_allowed_features := asm_const,asm_goto,arbitrary_self_types,lint_reasons,offset_of_nested,raw_ref_op,used_with_arg +# additional rust features used by the downstream asahi kernel +rust_allowed_features := $(rust_allowed_features),ptr_sub_ptr + # `--out-dir` is required to avoid temporaries being created by `rustc` in the # current working directory, which may be not accessible in the out-of-tree # modules case. diff --git a/scripts/dtc/data.c b/scripts/dtc/data.c index 5b25aa06041613..ce449824c80a06 100644 --- a/scripts/dtc/data.c +++ b/scripts/dtc/data.c @@ -184,6 +184,33 @@ struct data data_append_integer(struct data d, uint64_t value, int bits) } } +struct data data_append_float(struct data d, double value, int bits) +{ + float f32; + uint32_t u32; + double f64; + uint64_t u64; + fdt32_t value_32; + fdt64_t value_64; + + switch (bits) { + case 32: + f32 = value; + memcpy(&u32, &f32, sizeof(u32)); + value_32 = cpu_to_fdt32(u32); + return data_append_data(d, &value_32, 4); + + case 64: + f64 = value; + memcpy(&u64, &f64, sizeof(u64)); + value_64 = cpu_to_fdt64(u64); + return data_append_data(d, &value_64, 8); + + default: + die("Invalid literal size (%d)\n", bits); + } +} + struct data data_append_re(struct data d, uint64_t address, uint64_t size) { struct fdt_reserve_entry re; diff --git a/scripts/dtc/dtc-lexer.l b/scripts/dtc/dtc-lexer.l index 15d585c8079802..bd750717aa3a54 100644 --- a/scripts/dtc/dtc-lexer.l +++ b/scripts/dtc/dtc-lexer.l @@ -151,6 +151,28 @@ static void PRINTF(1, 2) lexical_error(const char *fmt, ...); return DT_LABEL; } +[-+]?(([0-9]+\.[0-9]*)|([0-9]*\.[0-9]+))(e[-+]?[0-9]+)?f? { + char *e; + DPRINT("Floating-point Literal: '%s'\n", yytext); + + errno = 0; + yylval.floating = strtod(yytext, &e); + + if (*e && (*e != 'f' || e[1])) { + lexical_error("Bad floating-point literal '%s'", + yytext); + } + + if (errno == ERANGE) + lexical_error("Floating-point literal '%s' out of range", + yytext); + else + /* ERANGE is the only strtod error triggerable + * by strings matching the pattern */ + assert(errno == 0); + return DT_FP_LITERAL; + } + {LABEL} { /* Missed includes or macro definitions while * preprocessing can lead to unexpected identifiers in diff --git a/scripts/dtc/dtc-parser.y b/scripts/dtc/dtc-parser.y index 4d5eece5262434..225a6b41b14fcf 100644 --- a/scripts/dtc/dtc-parser.y +++ b/scripts/dtc/dtc-parser.y @@ -48,6 +48,7 @@ static bool is_ref_relative(const char *ref) struct node *nodelist; struct reserve_info *re; uint64_t integer; + double floating; unsigned int flags; } @@ -61,6 +62,7 @@ static bool is_ref_relative(const char *ref) %token DT_OMIT_NO_REF %token DT_PROPNODENAME %token DT_LITERAL +%token DT_FP_LITERAL %token DT_CHAR_LITERAL %token DT_BYTE %token DT_STRING @@ -86,6 +88,7 @@ static bool is_ref_relative(const char *ref) %type subnode %type subnodes +%type floating_prim %type integer_prim %type integer_unary %type integer_mul @@ -395,6 +398,15 @@ arrayprefix: $$.data = data_add_marker(empty_data, TYPE_UINT32, NULL); $$.bits = 32; } + | arrayprefix floating_prim + { + if ($1.bits < 32) { + ERROR(&@2, "Floating-point values must be" + " 32-bit or 64-bit"); + } + + $$.data = data_append_float($1.data, $2, $1.bits); + } | arrayprefix integer_prim { if ($1.bits < 64) { @@ -439,6 +451,10 @@ arrayprefix: } ; +floating_prim: + DT_FP_LITERAL + ; + integer_prim: DT_LITERAL | DT_CHAR_LITERAL diff --git a/scripts/dtc/dtc.h b/scripts/dtc/dtc.h index 3a220b9afc99f9..cad8be1440bd69 100644 --- a/scripts/dtc/dtc.h +++ b/scripts/dtc/dtc.h @@ -177,6 +177,7 @@ struct data data_insert_at_marker(struct data d, struct marker *m, struct data data_merge(struct data d1, struct data d2); struct data data_append_cell(struct data d, cell_t word); struct data data_append_integer(struct data d, uint64_t word, int bits); +struct data data_append_float(struct data d, double value, int bits); struct data data_append_re(struct data d, uint64_t address, uint64_t size); struct data data_append_addr(struct data d, uint64_t addr); struct data data_append_byte(struct data d, uint8_t byte); diff --git a/security/apparmor/apparmorfs.c b/security/apparmor/apparmorfs.c index 907bd2667e28c7..9252172d50682b 100644 --- a/security/apparmor/apparmorfs.c +++ b/security/apparmor/apparmorfs.c @@ -1644,6 +1644,15 @@ static const char *rawdata_get_link_base(struct dentry *dentry, label = aa_get_label_rcu(&proxy->label); profile = labels_profile(label); + + /* rawdata can be null when aa_g_export_binary is unset during + * runtime and a profile is replaced + */ + if (!profile->rawdata) { + aa_put_label(label); + return ERR_PTR(-ENOENT); + } + depth = profile_depth(profile); target = gen_symlink_name(depth, profile->rawdata->name, name); aa_put_label(label); diff --git a/security/apparmor/file.c b/security/apparmor/file.c index c758204028780f..7de23e85cd5d01 100644 --- a/security/apparmor/file.c +++ b/security/apparmor/file.c @@ -154,8 +154,12 @@ static int path_name(const char *op, const struct cred *subj_cred, const char *info = NULL; int error; - error = aa_path_name(path, flags, buffer, name, &info, - labels_profile(label)->disconnected); + /* don't reaudit files closed during inheritance */ + if (unlikely(path->dentry == aa_null.dentry)) + error = -EACCES; + else + error = aa_path_name(path, flags, buffer, name, &info, + labels_profile(label)->disconnected); if (error) { fn_for_each_confined(label, profile, aa_audit_file(subj_cred, @@ -578,6 +582,9 @@ static bool __unix_needs_revalidation(struct file *file, struct aa_label *label, return false; if (request & NET_PEER_MASK) return false; + /* sock and sock->sk can be NULL for sockets being set up or torn down */ + if (!sock || !sock->sk) + return false; if (sock->sk->sk_family == PF_UNIX) { struct aa_sk_ctx *ctx = aa_sock(sock->sk); @@ -613,6 +620,10 @@ int aa_file_perm(const char *op, const struct cred *subj_cred, AA_BUG(!label); AA_BUG(!file); + /* don't reaudit files closed during inheritance */ + if (unlikely(file->f_path.dentry == aa_null.dentry)) + return -EACCES; + fctx = file_ctx(file); rcu_read_lock(); diff --git a/security/apparmor/include/match.h b/security/apparmor/include/match.h index 1fbe82f5021b16..0dde8eda3d1a59 100644 --- a/security/apparmor/include/match.h +++ b/security/apparmor/include/match.h @@ -104,16 +104,18 @@ struct aa_dfa { struct table_header *tables[YYTD_ID_TSIZE]; }; -#define byte_to_byte(X) (X) - #define UNPACK_ARRAY(TABLE, BLOB, LEN, TTYPE, BTYPE, NTOHX) \ do { \ typeof(LEN) __i; \ TTYPE *__t = (TTYPE *) TABLE; \ BTYPE *__b = (BTYPE *) BLOB; \ - for (__i = 0; __i < LEN; __i++) { \ - __t[__i] = NTOHX(__b[__i]); \ - } \ + BUILD_BUG_ON(sizeof(TTYPE) != sizeof(BTYPE)); \ + if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) \ + memcpy(__t, __b, (LEN) * sizeof(BTYPE)); \ + else /* copy & convert from big-endian */ \ + for (__i = 0; __i < LEN; __i++) { \ + __t[__i] = NTOHX(&__b[__i]); \ + } \ } while (0) static inline size_t table_size(size_t len, size_t el_size) diff --git a/security/apparmor/label.c b/security/apparmor/label.c index 913678f199c358..dd6c58f595ba8d 100644 --- a/security/apparmor/label.c +++ b/security/apparmor/label.c @@ -1278,7 +1278,7 @@ static inline aa_state_t match_component(struct aa_profile *profile, * @request: permissions to request * @perms: perms struct to set * - * Returns: 0 on success else ERROR + * Returns: state match stopped at or DFA_NOMATCH if aborted early * * For the label A//&B//&C this does the perm match for A//&B//&C * @perms should be preinitialized with allperms OR a previous permission @@ -1305,7 +1305,7 @@ static int label_compound_match(struct aa_profile *profile, /* no component visible */ *perms = allperms; - return 0; + return state; next: label_for_each_cont(i, label, tp) { @@ -1317,15 +1317,11 @@ static int label_compound_match(struct aa_profile *profile, goto fail; } *perms = *aa_lookup_perms(rules->policy, state); - aa_apply_modes_to_perms(profile, perms); - if ((perms->allow & request) != request) - return -EACCES; - - return 0; + return state; fail: *perms = nullperms; - return state; + return DFA_NOMATCH; } /** @@ -1338,7 +1334,7 @@ static int label_compound_match(struct aa_profile *profile, * @request: permissions to request * @perms: an initialized perms struct to add accumulation to * - * Returns: 0 on success else ERROR + * Returns: the state the match finished in, may be the none matching state * * For the label A//&B//&C this does the perm match for each of A and B and C * @perms should be preinitialized with allperms OR a previous permission @@ -1366,11 +1362,10 @@ static int label_components_match(struct aa_profile *profile, } /* no subcomponents visible - no change in perms */ - return 0; + return state; next: tmp = *aa_lookup_perms(rules->policy, state); - aa_apply_modes_to_perms(profile, &tmp); aa_perms_accum(perms, &tmp); label_for_each_cont(i, label, tp) { if (!aa_ns_visible(profile->ns, tp->ns, subns)) @@ -1379,18 +1374,17 @@ static int label_components_match(struct aa_profile *profile, if (!state) goto fail; tmp = *aa_lookup_perms(rules->policy, state); - aa_apply_modes_to_perms(profile, &tmp); aa_perms_accum(perms, &tmp); } if ((perms->allow & request) != request) - return -EACCES; + return DFA_NOMATCH; - return 0; + return state; fail: *perms = nullperms; - return -EACCES; + return DFA_NOMATCH; } /** @@ -1409,11 +1403,12 @@ int aa_label_match(struct aa_profile *profile, struct aa_ruleset *rules, struct aa_label *label, aa_state_t state, bool subns, u32 request, struct aa_perms *perms) { - int error = label_compound_match(profile, rules, label, state, subns, - request, perms); - if (!error) - return error; + aa_state_t tmp = label_compound_match(profile, rules, label, state, subns, + request, perms); + if ((perms->allow & request) == request) + return tmp; + /* failed compound_match try component matches */ *perms = allperms; return label_components_match(profile, rules, label, state, subns, request, perms); diff --git a/security/apparmor/lsm.c b/security/apparmor/lsm.c index a87cd60ed20699..9175fd677ef3db 100644 --- a/security/apparmor/lsm.c +++ b/security/apparmor/lsm.c @@ -520,33 +520,26 @@ static void apparmor_file_free_security(struct file *file) aa_put_label(rcu_access_pointer(ctx->label)); } -static int common_file_perm(const char *op, struct file *file, u32 mask, - bool in_atomic) +static int common_file_perm(const char *op, struct file *file, u32 mask) { struct aa_label *label; int error = 0; - bool needput; - - /* don't reaudit files closed during inheritance */ - if (unlikely(file->f_path.dentry == aa_null.dentry)) - return -EACCES; - label = __begin_current_label_crit_section(&needput); - error = aa_file_perm(op, current_cred(), label, file, mask, in_atomic); - __end_current_label_crit_section(label, needput); + label = begin_current_label_crit_section(); + error = aa_file_perm(op, current_cred(), label, file, mask, false); + end_current_label_crit_section(label); return error; } static int apparmor_file_receive(struct file *file) { - return common_file_perm(OP_FRECEIVE, file, aa_map_file_to_perms(file), - false); + return common_file_perm(OP_FRECEIVE, file, aa_map_file_to_perms(file)); } static int apparmor_file_permission(struct file *file, int mask) { - return common_file_perm(OP_FPERM, file, mask, false); + return common_file_perm(OP_FPERM, file, mask); } static int apparmor_file_lock(struct file *file, unsigned int cmd) @@ -556,11 +549,11 @@ static int apparmor_file_lock(struct file *file, unsigned int cmd) if (cmd == F_WRLCK) mask |= MAY_WRITE; - return common_file_perm(OP_FLOCK, file, mask, false); + return common_file_perm(OP_FLOCK, file, mask); } static int common_mmap(const char *op, struct file *file, unsigned long prot, - unsigned long flags, bool in_atomic) + unsigned long flags) { int mask = 0; @@ -578,21 +571,20 @@ static int common_mmap(const char *op, struct file *file, unsigned long prot, if (prot & PROT_EXEC) mask |= AA_EXEC_MMAP; - return common_file_perm(op, file, mask, in_atomic); + return common_file_perm(op, file, mask); } static int apparmor_mmap_file(struct file *file, unsigned long reqprot, unsigned long prot, unsigned long flags) { - return common_mmap(OP_FMMAP, file, prot, flags, GFP_ATOMIC); + return common_mmap(OP_FMMAP, file, prot, flags); } static int apparmor_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot, unsigned long prot) { return common_mmap(OP_FMPROT, vma->vm_file, prot, - !(vma->vm_flags & VM_SHARED) ? MAP_PRIVATE : 0, - false); + !(vma->vm_flags & VM_SHARED) ? MAP_PRIVATE : 0); } #ifdef CONFIG_IO_URING @@ -2145,7 +2137,8 @@ char *aa_get_buffer(bool in_atomic) if (!list_empty(&cache->head)) { aa_buf = list_first_entry(&cache->head, union aa_buffer, list); list_del(&aa_buf->list); - cache->hold--; + if (cache->hold) + cache->hold--; cache->count--; put_cpu_ptr(&aa_local_buffers); return &aa_buf->buffer[0]; diff --git a/security/apparmor/match.c b/security/apparmor/match.c index c5a91600842a16..bbeb3be68572f1 100644 --- a/security/apparmor/match.c +++ b/security/apparmor/match.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "include/lib.h" #include "include/match.h" @@ -42,11 +43,11 @@ static struct table_header *unpack_table(char *blob, size_t bsize) /* loaded td_id's start at 1, subtract 1 now to avoid doing * it every time we use td_id as an index */ - th.td_id = be16_to_cpu(*(__be16 *) (blob)) - 1; + th.td_id = get_unaligned_be16(blob) - 1; if (th.td_id > YYTD_ID_MAX) goto out; - th.td_flags = be16_to_cpu(*(__be16 *) (blob + 2)); - th.td_lolen = be32_to_cpu(*(__be32 *) (blob + 8)); + th.td_flags = get_unaligned_be16(blob + 2); + th.td_lolen = get_unaligned_be32(blob + 8); blob += sizeof(struct table_header); if (!(th.td_flags == YYTD_DATA16 || th.td_flags == YYTD_DATA32 || @@ -66,14 +67,13 @@ static struct table_header *unpack_table(char *blob, size_t bsize) table->td_flags = th.td_flags; table->td_lolen = th.td_lolen; if (th.td_flags == YYTD_DATA8) - UNPACK_ARRAY(table->td_data, blob, th.td_lolen, - u8, u8, byte_to_byte); + memcpy(table->td_data, blob, th.td_lolen); else if (th.td_flags == YYTD_DATA16) UNPACK_ARRAY(table->td_data, blob, th.td_lolen, - u16, __be16, be16_to_cpu); + u16, __be16, get_unaligned_be16); else if (th.td_flags == YYTD_DATA32) UNPACK_ARRAY(table->td_data, blob, th.td_lolen, - u32, __be32, be32_to_cpu); + u32, __be32, get_unaligned_be32); else goto fail; /* if table was vmalloced make sure the page tables are synced @@ -313,14 +313,14 @@ struct aa_dfa *aa_dfa_unpack(void *blob, size_t size, int flags) if (size < sizeof(struct table_set_header)) goto fail; - if (ntohl(*(__be32 *) data) != YYTH_MAGIC) + if (get_unaligned_be32(data) != YYTH_MAGIC) goto fail; - hsize = ntohl(*(__be32 *) (data + 4)); + hsize = get_unaligned_be32(data + 4); if (size < hsize) goto fail; - dfa->flags = ntohs(*(__be16 *) (data + 12)); + dfa->flags = get_unaligned_be16(data + 12); if (dfa->flags & ~(YYTH_FLAGS)) goto fail; @@ -329,7 +329,7 @@ struct aa_dfa *aa_dfa_unpack(void *blob, size_t size, int flags) * if (dfa->flags & YYTH_FLAGS_OOB_TRANS) { * if (hsize < 16 + 4) * goto fail; - * dfa->max_oob = ntol(*(__be32 *) (data + 16)); + * dfa->max_oob = get_unaligned_be32(data + 16); * if (dfa->max <= MAX_OOB_SUPPORTED) { * pr_err("AppArmor DFA OOB greater than supported\n"); * goto fail; diff --git a/security/apparmor/net.c b/security/apparmor/net.c index 45cf25605c345b..44c04102062f3d 100644 --- a/security/apparmor/net.c +++ b/security/apparmor/net.c @@ -326,8 +326,10 @@ int aa_sock_file_perm(const struct cred *subj_cred, struct aa_label *label, struct socket *sock = (struct socket *) file->private_data; AA_BUG(!label); - AA_BUG(!sock); - AA_BUG(!sock->sk); + + /* sock && sock->sk can be NULL for sockets being set up or torn down */ + if (!sock || !sock->sk) + return 0; if (sock->sk->sk_family == PF_UNIX) return aa_unix_file_perm(subj_cred, label, op, request, file); diff --git a/security/apparmor/policy_unpack.c b/security/apparmor/policy_unpack.c index 7523971e37d9c0..dd602bd5fca998 100644 --- a/security/apparmor/policy_unpack.c +++ b/security/apparmor/policy_unpack.c @@ -687,8 +687,10 @@ static ssize_t unpack_perms_table(struct aa_ext *e, struct aa_perms **perms) if (!aa_unpack_array(e, NULL, &size)) goto fail_reset; *perms = kcalloc(size, sizeof(struct aa_perms), GFP_KERNEL); - if (!*perms) - goto fail_reset; + if (!*perms) { + e->pos = pos; + return -ENOMEM; + } for (i = 0; i < size; i++) { if (!unpack_perm(e, version, &(*perms)[i])) goto fail; diff --git a/security/apparmor/resource.c b/security/apparmor/resource.c index 8e80db3ae21c09..64212b39ba4bbc 100644 --- a/security/apparmor/resource.c +++ b/security/apparmor/resource.c @@ -196,6 +196,11 @@ void __aa_transition_rlimits(struct aa_label *old_l, struct aa_label *new_l) rules->rlimits.limits[j].rlim_max); /* soft limit should not exceed hard limit */ rlim->rlim_cur = min(rlim->rlim_cur, rlim->rlim_max); + if (j == RLIMIT_CPU && + rlim->rlim_cur != RLIM_INFINITY && + IS_ENABLED(CONFIG_POSIX_TIMERS)) + (void) update_rlimit_cpu(current->group_leader, + rlim->rlim_cur); } } } diff --git a/security/integrity/evm/evm_crypto.c b/security/integrity/evm/evm_crypto.c index a5e730ffda57fb..5a8cef45bacf01 100644 --- a/security/integrity/evm/evm_crypto.c +++ b/security/integrity/evm/evm_crypto.c @@ -401,6 +401,7 @@ int evm_init_hmac(struct inode *inode, const struct xattr *xattrs, { struct shash_desc *desc; const struct xattr *xattr; + struct xattr_list *xattr_entry; desc = init_desc(EVM_XATTR_HMAC, HASH_ALGO_SHA1); if (IS_ERR(desc)) { @@ -408,11 +409,16 @@ int evm_init_hmac(struct inode *inode, const struct xattr *xattrs, return PTR_ERR(desc); } - for (xattr = xattrs; xattr->name; xattr++) { - if (!evm_protected_xattr(xattr->name)) - continue; + list_for_each_entry_lockless(xattr_entry, &evm_config_xattrnames, + list) { + for (xattr = xattrs; xattr->name; xattr++) { + if (strcmp(xattr_entry->name + + XATTR_SECURITY_PREFIX_LEN, xattr->name) != 0) + continue; - crypto_shash_update(desc, xattr->value, xattr->value_len); + crypto_shash_update(desc, xattr->value, + xattr->value_len); + } } hmac_add_misc(desc, inode, EVM_XATTR_HMAC, hmac_val); diff --git a/security/integrity/ima/ima.h b/security/integrity/ima/ima.h index e3d71d8d56e38c..89ebe98ffc5e57 100644 --- a/security/integrity/ima/ima.h +++ b/security/integrity/ima/ima.h @@ -441,7 +441,8 @@ int ima_check_blacklist(struct ima_iint_cache *iint, int ima_appraise_measurement(enum ima_hooks func, struct ima_iint_cache *iint, struct file *file, const unsigned char *filename, struct evm_ima_xattr_data *xattr_value, - int xattr_len, const struct modsig *modsig); + int xattr_len, const struct modsig *modsig, + bool bprm_is_check); int ima_must_appraise(struct mnt_idmap *idmap, struct inode *inode, int mask, enum ima_hooks func); void ima_update_xattr(struct ima_iint_cache *iint, struct file *file); @@ -466,7 +467,8 @@ static inline int ima_appraise_measurement(enum ima_hooks func, const unsigned char *filename, struct evm_ima_xattr_data *xattr_value, int xattr_len, - const struct modsig *modsig) + const struct modsig *modsig, + bool bprm_is_check) { return INTEGRITY_UNKNOWN; } diff --git a/security/integrity/ima/ima_appraise.c b/security/integrity/ima/ima_appraise.c index 5149ff4fd50d24..16c20c578ea878 100644 --- a/security/integrity/ima/ima_appraise.c +++ b/security/integrity/ima/ima_appraise.c @@ -470,17 +470,6 @@ int ima_check_blacklist(struct ima_iint_cache *iint, return rc; } -static bool is_bprm_creds_for_exec(enum ima_hooks func, struct file *file) -{ - struct linux_binprm *bprm; - - if (func == BPRM_CHECK) { - bprm = container_of(&file, struct linux_binprm, file); - return bprm->is_check; - } - return false; -} - /* * ima_appraise_measurement - appraise file measurement * @@ -492,7 +481,8 @@ static bool is_bprm_creds_for_exec(enum ima_hooks func, struct file *file) int ima_appraise_measurement(enum ima_hooks func, struct ima_iint_cache *iint, struct file *file, const unsigned char *filename, struct evm_ima_xattr_data *xattr_value, - int xattr_len, const struct modsig *modsig) + int xattr_len, const struct modsig *modsig, + bool bprm_is_check) { static const char op[] = "appraise_data"; int audit_msgno = AUDIT_INTEGRITY_DATA; @@ -514,7 +504,7 @@ int ima_appraise_measurement(enum ima_hooks func, struct ima_iint_cache *iint, * of the script interpreter(userspace). Differentiate kernel and * userspace enforced integrity audit messages. */ - if (is_bprm_creds_for_exec(func, file)) + if (bprm_is_check) audit_msgno = AUDIT_INTEGRITY_USERSPACE; /* If reading the xattr failed and there's no modsig, error out. */ diff --git a/security/integrity/ima/ima_main.c b/security/integrity/ima/ima_main.c index 5770cf691912aa..1d6229b156fb17 100644 --- a/security/integrity/ima/ima_main.c +++ b/security/integrity/ima/ima_main.c @@ -236,7 +236,8 @@ static void ima_file_free(struct file *file) static int process_measurement(struct file *file, const struct cred *cred, struct lsm_prop *prop, char *buf, loff_t size, int mask, enum ima_hooks func, - enum kernel_read_file_id read_id) + enum kernel_read_file_id read_id, + bool bprm_is_check) { struct inode *real_inode, *inode = file_inode(file); struct ima_iint_cache *iint = NULL; @@ -426,7 +427,8 @@ static int process_measurement(struct file *file, const struct cred *cred, inode_lock(inode); rc = ima_appraise_measurement(func, iint, file, pathname, xattr_value, - xattr_len, modsig); + xattr_len, modsig, + bprm_is_check); inode_unlock(inode); } if (!rc) @@ -493,14 +495,15 @@ static int ima_file_mmap(struct file *file, unsigned long reqprot, if (reqprot & PROT_EXEC) { ret = process_measurement(file, current_cred(), &prop, NULL, - 0, MAY_EXEC, MMAP_CHECK_REQPROT, 0); + 0, MAY_EXEC, MMAP_CHECK_REQPROT, 0, + false); if (ret) return ret; } if (prot & PROT_EXEC) return process_measurement(file, current_cred(), &prop, NULL, - 0, MAY_EXEC, MMAP_CHECK, 0); + 0, MAY_EXEC, MMAP_CHECK, 0, false); return 0; } @@ -584,7 +587,8 @@ static int ima_bprm_check(struct linux_binprm *bprm) security_current_getlsmprop_subj(&prop); return process_measurement(bprm->file, current_cred(), - &prop, NULL, 0, MAY_EXEC, BPRM_CHECK, 0); + &prop, NULL, 0, MAY_EXEC, BPRM_CHECK, 0, + bprm->is_check); } /** @@ -614,7 +618,7 @@ static int ima_creds_check(struct linux_binprm *bprm, const struct file *file) security_current_getlsmprop_subj(&prop); return process_measurement((struct file *)file, bprm->cred, &prop, NULL, - 0, MAY_EXEC, CREDS_CHECK, 0); + 0, MAY_EXEC, CREDS_CHECK, 0, false); } /** @@ -662,7 +666,7 @@ static int ima_file_check(struct file *file, int mask) security_current_getlsmprop_subj(&prop); return process_measurement(file, current_cred(), &prop, NULL, 0, mask & (MAY_READ | MAY_WRITE | MAY_EXEC | - MAY_APPEND), FILE_CHECK, 0); + MAY_APPEND), FILE_CHECK, 0, false); } static int __ima_inode_hash(struct inode *inode, struct file *file, char *buf, @@ -881,7 +885,7 @@ static int ima_read_file(struct file *file, enum kernel_read_file_id read_id, func = read_idmap[read_id] ?: FILE_CHECK; security_current_getlsmprop_subj(&prop); return process_measurement(file, current_cred(), &prop, NULL, 0, - MAY_READ, func, 0); + MAY_READ, func, 0, false); } const int read_idmap[READING_MAX_ID] = { @@ -925,7 +929,7 @@ static int ima_post_read_file(struct file *file, char *buf, loff_t size, func = read_idmap[read_id] ?: FILE_CHECK; security_current_getlsmprop_subj(&prop); return process_measurement(file, current_cred(), &prop, buf, size, - MAY_READ, func, read_id); + MAY_READ, func, read_id, false); } /** diff --git a/security/smack/smackfs.c b/security/smack/smackfs.c index 2a9d3f2ebbe13c..8919e330d2f606 100644 --- a/security/smack/smackfs.c +++ b/security/smack/smackfs.c @@ -70,6 +70,7 @@ enum smk_inos { static DEFINE_MUTEX(smack_cipso_lock); static DEFINE_MUTEX(smack_ambient_lock); static DEFINE_MUTEX(smk_net4addr_lock); +static DEFINE_MUTEX(smk_cipso_doi_lock); #if IS_ENABLED(CONFIG_IPV6) static DEFINE_MUTEX(smk_net6addr_lock); #endif /* CONFIG_IPV6 */ @@ -141,7 +142,7 @@ struct smack_parsed_rule { int smk_access2; }; -static int smk_cipso_doi_value = SMACK_CIPSO_DOI_DEFAULT; +static u32 smk_cipso_doi_value = CIPSO_V4_DOI_UNKNOWN; /* * Values for parsing cipso rules @@ -663,43 +664,60 @@ static const struct file_operations smk_load_ops = { }; /** - * smk_cipso_doi - initialize the CIPSO domain + * smk_cipso_doi - set netlabel maps + * @ndoi: new value for our CIPSO DOI + * @gfp_flags: kmalloc allocation context */ -static void smk_cipso_doi(void) +static int +smk_cipso_doi(u32 ndoi, gfp_t gfp_flags) { - int rc; + int rc = 0; struct cipso_v4_doi *doip; struct netlbl_audit nai; - smk_netlabel_audit_set(&nai); + mutex_lock(&smk_cipso_doi_lock); - rc = netlbl_cfg_map_del(NULL, PF_INET, NULL, NULL, &nai); - if (rc != 0) - printk(KERN_WARNING "%s:%d remove rc = %d\n", - __func__, __LINE__, rc); + if (smk_cipso_doi_value == ndoi) + goto clr_doi_lock; + + smk_netlabel_audit_set(&nai); - doip = kmalloc(sizeof(struct cipso_v4_doi), GFP_KERNEL | __GFP_NOFAIL); + doip = kmalloc(sizeof(struct cipso_v4_doi), gfp_flags); + if (!doip) { + rc = -ENOMEM; + goto clr_doi_lock; + } doip->map.std = NULL; - doip->doi = smk_cipso_doi_value; + doip->doi = ndoi; doip->type = CIPSO_V4_MAP_PASS; doip->tags[0] = CIPSO_V4_TAG_RBITMAP; for (rc = 1; rc < CIPSO_V4_TAG_MAXCNT; rc++) doip->tags[rc] = CIPSO_V4_TAG_INVALID; rc = netlbl_cfg_cipsov4_add(doip, &nai); - if (rc != 0) { - printk(KERN_WARNING "%s:%d cipso add rc = %d\n", - __func__, __LINE__, rc); + if (rc) { kfree(doip); - return; + goto clr_doi_lock; } - rc = netlbl_cfg_cipsov4_map_add(doip->doi, NULL, NULL, NULL, &nai); - if (rc != 0) { - printk(KERN_WARNING "%s:%d map add rc = %d\n", - __func__, __LINE__, rc); - netlbl_cfg_cipsov4_del(doip->doi, &nai); - return; + + if (smk_cipso_doi_value != CIPSO_V4_DOI_UNKNOWN) { + rc = netlbl_cfg_map_del(NULL, PF_INET, NULL, NULL, &nai); + if (rc && rc != -ENOENT) + goto clr_ndoi_def; + + netlbl_cfg_cipsov4_del(smk_cipso_doi_value, &nai); } + + rc = netlbl_cfg_cipsov4_map_add(ndoi, NULL, NULL, NULL, &nai); + if (rc) { + smk_cipso_doi_value = CIPSO_V4_DOI_UNKNOWN; // no default map +clr_ndoi_def: netlbl_cfg_cipsov4_del(ndoi, &nai); + } else + smk_cipso_doi_value = ndoi; + +clr_doi_lock: + mutex_unlock(&smk_cipso_doi_lock); + return rc; } /** @@ -1562,7 +1580,7 @@ static ssize_t smk_read_doi(struct file *filp, char __user *buf, if (*ppos != 0) return 0; - sprintf(temp, "%d", smk_cipso_doi_value); + sprintf(temp, "%lu", (unsigned long)smk_cipso_doi_value); rc = simple_read_from_buffer(buf, count, ppos, temp, strlen(temp)); return rc; @@ -1581,7 +1599,7 @@ static ssize_t smk_write_doi(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { char temp[80]; - int i; + unsigned long u; if (!smack_privileged(CAP_MAC_ADMIN)) return -EPERM; @@ -1594,14 +1612,13 @@ static ssize_t smk_write_doi(struct file *file, const char __user *buf, temp[count] = '\0'; - if (sscanf(temp, "%d", &i) != 1) + if (kstrtoul(temp, 10, &u)) return -EINVAL; - smk_cipso_doi_value = i; - - smk_cipso_doi(); + if (u == CIPSO_V4_DOI_UNKNOWN || u > U32_MAX) + return -EINVAL; - return count; + return smk_cipso_doi(u, GFP_KERNEL) ? : count; } static const struct file_operations smk_doi_ops = { @@ -2982,6 +2999,7 @@ int __init init_smk_fs(void) { int err; int rc; + struct netlbl_audit nai; if (smack_enabled == 0) return 0; @@ -3000,7 +3018,10 @@ int __init init_smk_fs(void) } } - smk_cipso_doi(); + smk_netlabel_audit_set(&nai); + (void) netlbl_cfg_map_del(NULL, PF_INET, NULL, NULL, &nai); + (void) smk_cipso_doi(SMACK_CIPSO_DOI_DEFAULT, + GFP_KERNEL | __GFP_NOFAIL); smk_unlbl_ambient(NULL); rc = smack_populate_secattr(&smack_known_floor); diff --git a/sound/core/compress_offload.c b/sound/core/compress_offload.c index da514fef45bca0..ed2eeb914c6dbf 100644 --- a/sound/core/compress_offload.c +++ b/sound/core/compress_offload.c @@ -514,12 +514,12 @@ static int snd_compr_get_codec_caps(struct snd_compr_stream *stream, unsigned long arg) { int retval; - struct snd_compr_codec_caps *caps __free(kfree) = NULL; if (!stream->ops->get_codec_caps) return -ENXIO; - caps = kzalloc(sizeof(*caps), GFP_KERNEL); + struct snd_compr_codec_caps *caps __free(kfree) = + kzalloc(sizeof(*caps), GFP_KERNEL); if (!caps) return -ENOMEM; @@ -647,7 +647,6 @@ snd_compress_check_input(struct snd_compr_stream *stream, struct snd_compr_param static int snd_compr_set_params(struct snd_compr_stream *stream, unsigned long arg) { - struct snd_compr_params *params __free(kfree) = NULL; int retval; if (stream->runtime->state == SNDRV_PCM_STATE_OPEN || stream->next_track) { @@ -655,7 +654,9 @@ snd_compr_set_params(struct snd_compr_stream *stream, unsigned long arg) * we should allow parameter change only when stream has been * opened not in other cases */ - params = memdup_user((void __user *)arg, sizeof(*params)); + struct snd_compr_params *params __free(kfree) = + memdup_user((void __user *)arg, sizeof(*params)); + if (IS_ERR(params)) return PTR_ERR(params); @@ -687,13 +688,13 @@ snd_compr_set_params(struct snd_compr_stream *stream, unsigned long arg) static int snd_compr_get_params(struct snd_compr_stream *stream, unsigned long arg) { - struct snd_codec *params __free(kfree) = NULL; int retval; if (!stream->ops->get_params) return -EBADFD; - params = kzalloc(sizeof(*params), GFP_KERNEL); + struct snd_codec *params __free(kfree) = + kzalloc(sizeof(*params), GFP_KERNEL); if (!params) return -ENOMEM; retval = stream->ops->get_params(stream, params); @@ -1104,12 +1105,13 @@ static int snd_compr_task_new(struct snd_compr_stream *stream, struct snd_compr_ static int snd_compr_task_create(struct snd_compr_stream *stream, unsigned long arg) { - struct snd_compr_task *task __free(kfree) = NULL; int retval; if (stream->runtime->state != SNDRV_PCM_STATE_SETUP) return -EPERM; - task = memdup_user((void __user *)arg, sizeof(*task)); + + struct snd_compr_task *task __free(kfree) = + memdup_user((void __user *)arg, sizeof(*task)); if (IS_ERR(task)) return PTR_ERR(task); retval = snd_compr_task_new(stream, task); @@ -1165,12 +1167,13 @@ static int snd_compr_task_start(struct snd_compr_stream *stream, struct snd_comp static int snd_compr_task_start_ioctl(struct snd_compr_stream *stream, unsigned long arg) { - struct snd_compr_task *task __free(kfree) = NULL; int retval; if (stream->runtime->state != SNDRV_PCM_STATE_SETUP) return -EPERM; - task = memdup_user((void __user *)arg, sizeof(*task)); + + struct snd_compr_task *task __free(kfree) = + memdup_user((void __user *)arg, sizeof(*task)); if (IS_ERR(task)) return PTR_ERR(task); retval = snd_compr_task_start(stream, task); @@ -1256,12 +1259,13 @@ static int snd_compr_task_status(struct snd_compr_stream *stream, static int snd_compr_task_status_ioctl(struct snd_compr_stream *stream, unsigned long arg) { - struct snd_compr_task_status *status __free(kfree) = NULL; int retval; if (stream->runtime->state != SNDRV_PCM_STATE_SETUP) return -EPERM; - status = memdup_user((void __user *)arg, sizeof(*status)); + + struct snd_compr_task_status *status __free(kfree) = + memdup_user((void __user *)arg, sizeof(*status)); if (IS_ERR(status)) return PTR_ERR(status); retval = snd_compr_task_status(stream, status); diff --git a/sound/core/control.c b/sound/core/control.c index 9c3fd5113a6173..a5f025d9bfba83 100644 --- a/sound/core/control.c +++ b/sound/core/control.c @@ -123,10 +123,12 @@ static int snd_ctl_release(struct inode *inode, struct file *file) scoped_guard(rwsem_write, &card->controls_rwsem) { list_for_each_entry(control, &card->controls, list) for (idx = 0; idx < control->count; idx++) - if (control->vd[idx].owner == ctl) + if (control->vd[idx].owner == ctl) { control->vd[idx].owner = NULL; + if (control->unlock) + control->unlock(control); + } } - snd_fasync_free(ctl->fasync); snd_ctl_empty_read_queue(ctl); put_pid(ctl->pid); @@ -303,6 +305,8 @@ struct snd_kcontrol *snd_ctl_new1(const struct snd_kcontrol_new *ncontrol, kctl->info = ncontrol->info; kctl->get = ncontrol->get; kctl->put = ncontrol->put; + kctl->lock = ncontrol->lock; + kctl->unlock = ncontrol->unlock; kctl->tlv.p = ncontrol->tlv.p; kctl->private_value = ncontrol->private_value; @@ -867,9 +871,9 @@ EXPORT_SYMBOL(snd_ctl_find_id); static int snd_ctl_card_info(struct snd_card *card, struct snd_ctl_file * ctl, unsigned int cmd, void __user *arg) { - struct snd_ctl_card_info *info __free(kfree) = NULL; + struct snd_ctl_card_info *info __free(kfree) = + kzalloc(sizeof(*info), GFP_KERNEL); - info = kzalloc(sizeof(*info), GFP_KERNEL); if (! info) return -ENOMEM; scoped_guard(rwsem_read, &snd_ioctl_rwsem) { @@ -1244,10 +1248,10 @@ static int snd_ctl_elem_read(struct snd_card *card, static int snd_ctl_elem_read_user(struct snd_card *card, struct snd_ctl_elem_value __user *_control) { - struct snd_ctl_elem_value *control __free(kfree) = NULL; int result; + struct snd_ctl_elem_value *control __free(kfree) = + memdup_user(_control, sizeof(*control)); - control = memdup_user(_control, sizeof(*control)); if (IS_ERR(control)) return PTR_ERR(control); @@ -1320,11 +1324,11 @@ static int snd_ctl_elem_write(struct snd_card *card, struct snd_ctl_file *file, static int snd_ctl_elem_write_user(struct snd_ctl_file *file, struct snd_ctl_elem_value __user *_control) { - struct snd_ctl_elem_value *control __free(kfree) = NULL; struct snd_card *card; int result; + struct snd_ctl_elem_value *control __free(kfree) = + memdup_user(_control, sizeof(*control)); - control = memdup_user(_control, sizeof(*control)); if (IS_ERR(control)) return PTR_ERR(control); @@ -1359,6 +1363,12 @@ static int snd_ctl_elem_lock(struct snd_ctl_file *file, vd = &kctl->vd[snd_ctl_get_ioff(kctl, &id)]; if (vd->owner) return -EBUSY; + + if (kctl->lock) { + int err = kctl->lock(kctl, file); + if (err < 0) + return err; + } vd->owner = file; return 0; } @@ -1383,6 +1393,8 @@ static int snd_ctl_elem_unlock(struct snd_ctl_file *file, if (vd->owner != file) return -EPERM; vd->owner = NULL; + if (kctl->unlock) + kctl->unlock(kctl); return 0; } diff --git a/sound/core/control_compat.c b/sound/core/control_compat.c index 6459809ed36486..b8988a4bcd9b5b 100644 --- a/sound/core/control_compat.c +++ b/sound/core/control_compat.c @@ -80,10 +80,10 @@ static int snd_ctl_elem_info_compat(struct snd_ctl_file *ctl, struct snd_ctl_elem_info32 __user *data32) { struct snd_card *card = ctl->card; - struct snd_ctl_elem_info *data __free(kfree) = NULL; int err; + struct snd_ctl_elem_info *data __free(kfree) = + kzalloc(sizeof(*data), GFP_KERNEL); - data = kzalloc(sizeof(*data), GFP_KERNEL); if (! data) return -ENOMEM; @@ -169,14 +169,15 @@ static int get_ctl_type(struct snd_card *card, struct snd_ctl_elem_id *id, int *countp) { struct snd_kcontrol *kctl; - struct snd_ctl_elem_info *info __free(kfree) = NULL; int err; guard(rwsem_read)(&card->controls_rwsem); kctl = snd_ctl_find_id(card, id); if (!kctl) return -ENOENT; - info = kzalloc(sizeof(*info), GFP_KERNEL); + + struct snd_ctl_elem_info *info __free(kfree) = + kzalloc(sizeof(*info), GFP_KERNEL); if (info == NULL) return -ENOMEM; info->id = *id; @@ -280,10 +281,10 @@ static int copy_ctl_value_to_user(void __user *userdata, static int __ctl_elem_read_user(struct snd_card *card, void __user *userdata, void __user *valuep) { - struct snd_ctl_elem_value *data __free(kfree) = NULL; int err, type, count; + struct snd_ctl_elem_value *data __free(kfree) = + kzalloc(sizeof(*data), GFP_KERNEL); - data = kzalloc(sizeof(*data), GFP_KERNEL); if (data == NULL) return -ENOMEM; @@ -314,11 +315,11 @@ static int ctl_elem_read_user(struct snd_card *card, static int __ctl_elem_write_user(struct snd_ctl_file *file, void __user *userdata, void __user *valuep) { - struct snd_ctl_elem_value *data __free(kfree) = NULL; struct snd_card *card = file->card; int err, type, count; + struct snd_ctl_elem_value *data __free(kfree) = + kzalloc(sizeof(*data), GFP_KERNEL); - data = kzalloc(sizeof(*data), GFP_KERNEL); if (data == NULL) return -ENOMEM; @@ -378,9 +379,9 @@ static int snd_ctl_elem_add_compat(struct snd_ctl_file *file, struct snd_ctl_elem_info32 __user *data32, int replace) { - struct snd_ctl_elem_info *data __free(kfree) = NULL; + struct snd_ctl_elem_info *data __free(kfree) = + kzalloc(sizeof(*data), GFP_KERNEL); - data = kzalloc(sizeof(*data), GFP_KERNEL); if (! data) return -ENOMEM; diff --git a/sound/core/control_led.c b/sound/core/control_led.c index e33dfcf863cf13..c7641d5084e7db 100644 --- a/sound/core/control_led.c +++ b/sound/core/control_led.c @@ -245,12 +245,12 @@ DEFINE_FREE(snd_card_unref, struct snd_card *, if (_T) snd_card_unref(_T)) static int snd_ctl_led_set_id(int card_number, struct snd_ctl_elem_id *id, unsigned int group, bool set) { - struct snd_card *card __free(snd_card_unref) = NULL; struct snd_kcontrol *kctl; struct snd_kcontrol_volatile *vd; unsigned int ioff, access, new_access; + struct snd_card *card __free(snd_card_unref) = + snd_card_ref(card_number); - card = snd_card_ref(card_number); if (!card) return -ENXIO; guard(rwsem_write)(&card->controls_rwsem); @@ -302,13 +302,13 @@ static void snd_ctl_led_clean(struct snd_card *card) static int snd_ctl_led_reset(int card_number, unsigned int group) { - struct snd_card *card __free(snd_card_unref) = NULL; struct snd_ctl_led_ctl *lctl, *_lctl; struct snd_ctl_led *led; struct snd_kcontrol_volatile *vd; bool change = false; + struct snd_card *card __free(snd_card_unref) = + snd_card_ref(card_number); - card = snd_card_ref(card_number); if (!card) return -ENXIO; @@ -598,11 +598,11 @@ static ssize_t list_show(struct device *dev, struct device_attribute *attr, char *buf) { struct snd_ctl_led_card *led_card = container_of(dev, struct snd_ctl_led_card, dev); - struct snd_card *card __free(snd_card_unref) = NULL; struct snd_ctl_led_ctl *lctl; size_t l = 0; + struct snd_card *card __free(snd_card_unref) = + snd_card_ref(led_card->number); - card = snd_card_ref(led_card->number); if (!card) return -ENXIO; guard(rwsem_read)(&card->controls_rwsem); diff --git a/sound/core/oss/mixer_oss.c b/sound/core/oss/mixer_oss.c index e839a4bb93f819..8d2d46d03301bd 100644 --- a/sound/core/oss/mixer_oss.c +++ b/sound/core/oss/mixer_oss.c @@ -517,8 +517,6 @@ static void snd_mixer_oss_get_volume1_vol(struct snd_mixer_oss_file *fmixer, unsigned int numid, int *left, int *right) { - struct snd_ctl_elem_info *uinfo __free(kfree) = NULL; - struct snd_ctl_elem_value *uctl __free(kfree) = NULL; struct snd_kcontrol *kctl; struct snd_card *card = fmixer->card; @@ -528,8 +526,11 @@ static void snd_mixer_oss_get_volume1_vol(struct snd_mixer_oss_file *fmixer, kctl = snd_ctl_find_numid(card, numid); if (!kctl) return; - uinfo = kzalloc(sizeof(*uinfo), GFP_KERNEL); - uctl = kzalloc(sizeof(*uctl), GFP_KERNEL); + + struct snd_ctl_elem_info *uinfo __free(kfree) = + kzalloc(sizeof(*uinfo), GFP_KERNEL); + struct snd_ctl_elem_value *uctl __free(kfree) = + kzalloc(sizeof(*uctl), GFP_KERNEL); if (uinfo == NULL || uctl == NULL) return; if (kctl->info(kctl, uinfo)) @@ -550,8 +551,6 @@ static void snd_mixer_oss_get_volume1_sw(struct snd_mixer_oss_file *fmixer, int *left, int *right, int route) { - struct snd_ctl_elem_info *uinfo __free(kfree) = NULL; - struct snd_ctl_elem_value *uctl __free(kfree) = NULL; struct snd_kcontrol *kctl; struct snd_card *card = fmixer->card; @@ -561,8 +560,11 @@ static void snd_mixer_oss_get_volume1_sw(struct snd_mixer_oss_file *fmixer, kctl = snd_ctl_find_numid(card, numid); if (!kctl) return; - uinfo = kzalloc(sizeof(*uinfo), GFP_KERNEL); - uctl = kzalloc(sizeof(*uctl), GFP_KERNEL); + + struct snd_ctl_elem_info *uinfo __free(kfree) = + kzalloc(sizeof(*uinfo), GFP_KERNEL); + struct snd_ctl_elem_value *uctl __free(kfree) = + kzalloc(sizeof(*uctl), GFP_KERNEL); if (uinfo == NULL || uctl == NULL) return; if (kctl->info(kctl, uinfo)) @@ -609,8 +611,6 @@ static void snd_mixer_oss_put_volume1_vol(struct snd_mixer_oss_file *fmixer, unsigned int numid, int left, int right) { - struct snd_ctl_elem_info *uinfo __free(kfree) = NULL; - struct snd_ctl_elem_value *uctl __free(kfree) = NULL; struct snd_kcontrol *kctl; struct snd_card *card = fmixer->card; int res; @@ -621,8 +621,11 @@ static void snd_mixer_oss_put_volume1_vol(struct snd_mixer_oss_file *fmixer, kctl = snd_ctl_find_numid(card, numid); if (!kctl) return; - uinfo = kzalloc(sizeof(*uinfo), GFP_KERNEL); - uctl = kzalloc(sizeof(*uctl), GFP_KERNEL); + + struct snd_ctl_elem_info *uinfo __free(kfree) = + kzalloc(sizeof(*uinfo), GFP_KERNEL); + struct snd_ctl_elem_value *uctl __free(kfree) = + kzalloc(sizeof(*uctl), GFP_KERNEL); if (uinfo == NULL || uctl == NULL) return; if (kctl->info(kctl, uinfo)) @@ -646,8 +649,6 @@ static void snd_mixer_oss_put_volume1_sw(struct snd_mixer_oss_file *fmixer, int left, int right, int route) { - struct snd_ctl_elem_info *uinfo __free(kfree) = NULL; - struct snd_ctl_elem_value *uctl __free(kfree) = NULL; struct snd_kcontrol *kctl; struct snd_card *card = fmixer->card; int res; @@ -658,8 +659,11 @@ static void snd_mixer_oss_put_volume1_sw(struct snd_mixer_oss_file *fmixer, kctl = snd_ctl_find_numid(card, numid); if (!kctl) return; - uinfo = kzalloc(sizeof(*uinfo), GFP_KERNEL); - uctl = kzalloc(sizeof(*uctl), GFP_KERNEL); + + struct snd_ctl_elem_info *uinfo __free(kfree) = + kzalloc(sizeof(*uinfo), GFP_KERNEL); + struct snd_ctl_elem_value *uctl __free(kfree) = + kzalloc(sizeof(*uctl), GFP_KERNEL); if (uinfo == NULL || uctl == NULL) return; if (kctl->info(kctl, uinfo)) @@ -783,12 +787,12 @@ static int snd_mixer_oss_get_recsrc2(struct snd_mixer_oss_file *fmixer, unsigned struct snd_kcontrol *kctl; struct snd_mixer_oss_slot *pslot; struct slot *slot; - struct snd_ctl_elem_info *uinfo __free(kfree) = NULL; - struct snd_ctl_elem_value *uctl __free(kfree) = NULL; int err, idx; - uinfo = kzalloc(sizeof(*uinfo), GFP_KERNEL); - uctl = kzalloc(sizeof(*uctl), GFP_KERNEL); + struct snd_ctl_elem_info *uinfo __free(kfree) = + kzalloc(sizeof(*uinfo), GFP_KERNEL); + struct snd_ctl_elem_value *uctl __free(kfree) = + kzalloc(sizeof(*uctl), GFP_KERNEL); if (uinfo == NULL || uctl == NULL) return -ENOMEM; guard(rwsem_read)(&card->controls_rwsem); @@ -825,13 +829,13 @@ static int snd_mixer_oss_put_recsrc2(struct snd_mixer_oss_file *fmixer, unsigned struct snd_kcontrol *kctl; struct snd_mixer_oss_slot *pslot; struct slot *slot = NULL; - struct snd_ctl_elem_info *uinfo __free(kfree) = NULL; - struct snd_ctl_elem_value *uctl __free(kfree) = NULL; int err; unsigned int idx; - uinfo = kzalloc(sizeof(*uinfo), GFP_KERNEL); - uctl = kzalloc(sizeof(*uctl), GFP_KERNEL); + struct snd_ctl_elem_info *uinfo __free(kfree) = + kzalloc(sizeof(*uinfo), GFP_KERNEL); + struct snd_ctl_elem_value *uctl __free(kfree) = + kzalloc(sizeof(*uctl), GFP_KERNEL); if (uinfo == NULL || uctl == NULL) return -ENOMEM; guard(rwsem_read)(&card->controls_rwsem); @@ -872,18 +876,18 @@ struct snd_mixer_oss_assign_table { static int snd_mixer_oss_build_test(struct snd_mixer_oss *mixer, struct slot *slot, const char *name, int index, int item) { - struct snd_ctl_elem_info *info __free(kfree) = NULL; struct snd_kcontrol *kcontrol; struct snd_card *card = mixer->card; int err; + struct snd_ctl_elem_info *info __free(kfree) = + kmalloc(sizeof(*info), GFP_KERNEL); + if (!info) + return -ENOMEM; scoped_guard(rwsem_read, &card->controls_rwsem) { kcontrol = snd_mixer_oss_test_id(mixer, name, index); if (kcontrol == NULL) return 0; - info = kmalloc(sizeof(*info), GFP_KERNEL); - if (!info) - return -ENOMEM; err = kcontrol->info(kcontrol, info); if (err < 0) return err; @@ -1006,9 +1010,9 @@ static int snd_mixer_oss_build_input(struct snd_mixer_oss *mixer, if (!ptr->index) kctl = snd_mixer_oss_test_id(mixer, "Capture Source", 0); if (kctl) { - struct snd_ctl_elem_info *uinfo __free(kfree) = NULL; + struct snd_ctl_elem_info *uinfo __free(kfree) = + kzalloc(sizeof(*uinfo), GFP_KERNEL); - uinfo = kzalloc(sizeof(*uinfo), GFP_KERNEL); if (!uinfo) return -ENOMEM; diff --git a/sound/core/oss/pcm_oss.c b/sound/core/oss/pcm_oss.c index b12df5b5ddfc17..3bc94d34b35e76 100644 --- a/sound/core/oss/pcm_oss.c +++ b/sound/core/oss/pcm_oss.c @@ -377,7 +377,6 @@ static int snd_pcm_hw_param_near(struct snd_pcm_substream *pcm, snd_pcm_hw_param_t var, unsigned int best, int *dir) { - struct snd_pcm_hw_params *save __free(kfree) = NULL; int v; unsigned int saved_min; int last = 0; @@ -397,19 +396,22 @@ static int snd_pcm_hw_param_near(struct snd_pcm_substream *pcm, maxdir = 1; max--; } - save = kmalloc(sizeof(*save), GFP_KERNEL); + + struct snd_pcm_hw_params *save __free(kfree) = + kmalloc(sizeof(*save), GFP_KERNEL); if (save == NULL) return -ENOMEM; *save = *params; saved_min = min; min = snd_pcm_hw_param_min(pcm, params, var, min, &mindir); if (min >= 0) { - struct snd_pcm_hw_params *params1 __free(kfree) = NULL; if (max < 0) goto _end; if ((unsigned int)min == saved_min && mindir == valdir) goto _end; - params1 = kmalloc(sizeof(*params1), GFP_KERNEL); + + struct snd_pcm_hw_params *params1 __free(kfree) = + kmalloc(sizeof(*params1), GFP_KERNEL); if (params1 == NULL) return -ENOMEM; *params1 = *save; @@ -781,10 +783,10 @@ static int choose_rate(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params, unsigned int best_rate) { const struct snd_interval *it; - struct snd_pcm_hw_params *save __free(kfree) = NULL; unsigned int rate, prev; - save = kmalloc(sizeof(*save), GFP_KERNEL); + struct snd_pcm_hw_params *save __free(kfree) = + kmalloc(sizeof(*save), GFP_KERNEL); if (save == NULL) return -ENOMEM; *save = *params; @@ -1836,7 +1838,6 @@ static int snd_pcm_oss_get_formats(struct snd_pcm_oss_file *pcm_oss_file) struct snd_pcm_substream *substream; int err; int direct; - struct snd_pcm_hw_params *params __free(kfree) = NULL; unsigned int formats = 0; const struct snd_mask *format_mask; int fmt; @@ -1856,7 +1857,9 @@ static int snd_pcm_oss_get_formats(struct snd_pcm_oss_file *pcm_oss_file) AFMT_S32_LE | AFMT_S32_BE | AFMT_S24_LE | AFMT_S24_BE | AFMT_S24_PACKED; - params = kmalloc(sizeof(*params), GFP_KERNEL); + + struct snd_pcm_hw_params *params __free(kfree) = + kmalloc(sizeof(*params), GFP_KERNEL); if (!params) return -ENOMEM; _snd_pcm_hw_params_any(params); diff --git a/sound/core/pcm.c b/sound/core/pcm.c index 283aac441fa0a7..0b512085eb63f2 100644 --- a/sound/core/pcm.c +++ b/sound/core/pcm.c @@ -328,13 +328,13 @@ static const char *snd_pcm_oss_format_name(int format) static void snd_pcm_proc_info_read(struct snd_pcm_substream *substream, struct snd_info_buffer *buffer) { - struct snd_pcm_info *info __free(kfree) = NULL; int err; if (! substream) return; - info = kmalloc(sizeof(*info), GFP_KERNEL); + struct snd_pcm_info *info __free(kfree) = + kmalloc(sizeof(*info), GFP_KERNEL); if (!info) return; diff --git a/sound/core/pcm_compat.c b/sound/core/pcm_compat.c index 54eb9bd8eb2188..e86f68f1f23c1e 100644 --- a/sound/core/pcm_compat.c +++ b/sound/core/pcm_compat.c @@ -235,7 +235,6 @@ static int snd_pcm_ioctl_hw_params_compat(struct snd_pcm_substream *substream, int refine, struct snd_pcm_hw_params32 __user *data32) { - struct snd_pcm_hw_params *data __free(kfree) = NULL; struct snd_pcm_runtime *runtime; int err; @@ -243,7 +242,8 @@ static int snd_pcm_ioctl_hw_params_compat(struct snd_pcm_substream *substream, if (!runtime) return -ENOTTY; - data = kmalloc(sizeof(*data), GFP_KERNEL); + struct snd_pcm_hw_params *data __free(kfree) = + kmalloc(sizeof(*data), GFP_KERNEL); if (!data) return -ENOMEM; @@ -332,7 +332,6 @@ static int snd_pcm_ioctl_xfern_compat(struct snd_pcm_substream *substream, compat_caddr_t buf; compat_caddr_t __user *bufptr; u32 frames; - void __user **bufs __free(kfree) = NULL; int err, ch, i; if (! substream->runtime) @@ -349,7 +348,9 @@ static int snd_pcm_ioctl_xfern_compat(struct snd_pcm_substream *substream, get_user(frames, &data32->frames)) return -EFAULT; bufptr = compat_ptr(buf); - bufs = kmalloc_array(ch, sizeof(void __user *), GFP_KERNEL); + + void __user **bufs __free(kfree) = + kmalloc_array(ch, sizeof(void __user *), GFP_KERNEL); if (bufs == NULL) return -ENOMEM; for (i = 0; i < ch; i++) { diff --git a/sound/core/pcm_dmaengine.c b/sound/core/pcm_dmaengine.c index f0c17503df425d..805191876fc923 100644 --- a/sound/core/pcm_dmaengine.c +++ b/sound/core/pcm_dmaengine.c @@ -22,6 +22,8 @@ struct dmaengine_pcm_runtime_data { struct dma_chan *dma_chan; dma_cookie_t cookie; + struct work_struct complete_wq; /* for nonatomic PCM */ + struct snd_pcm_substream *substream; unsigned int pos; }; @@ -147,6 +149,21 @@ static void dmaengine_pcm_dma_complete(void *arg) snd_pcm_period_elapsed(substream); } +static void dmaengine_pcm_dma_complete_nonatomic(struct work_struct *wq) +{ + struct dmaengine_pcm_runtime_data *prtd = \ + container_of(wq, struct dmaengine_pcm_runtime_data, complete_wq); + struct snd_pcm_substream *substream = prtd->substream; + dmaengine_pcm_dma_complete(substream); +} + +static void dmaengine_pcm_dma_complete_nonatomic_callback(void *arg) +{ + struct snd_pcm_substream *substream = arg; + struct dmaengine_pcm_runtime_data *prtd = substream_to_prtd(substream); + schedule_work(&prtd->complete_wq); +} + static int dmaengine_pcm_prepare_and_submit(struct snd_pcm_substream *substream) { struct dmaengine_pcm_runtime_data *prtd = substream_to_prtd(substream); @@ -169,7 +186,11 @@ static int dmaengine_pcm_prepare_and_submit(struct snd_pcm_substream *substream) if (!desc) return -ENOMEM; - desc->callback = dmaengine_pcm_dma_complete; + if (substream->pcm->nonatomic) + desc->callback = dmaengine_pcm_dma_complete_nonatomic_callback; + else + desc->callback = dmaengine_pcm_dma_complete; + desc->callback_param = substream; prtd->cookie = dmaengine_submit(desc); @@ -322,6 +343,10 @@ int snd_dmaengine_pcm_open(struct snd_pcm_substream *substream, if (!prtd) return -ENOMEM; + if (substream->pcm->nonatomic) + INIT_WORK(&prtd->complete_wq, dmaengine_pcm_dma_complete_nonatomic); + + prtd->substream = substream; prtd->dma_chan = chan; substream->runtime->private_data = prtd; @@ -355,7 +380,14 @@ static void __snd_dmaengine_pcm_close(struct snd_pcm_substream *substream, if (status == DMA_PAUSED) dmaengine_terminate_async(prtd->dma_chan); + /* + * The PCM might have been closed while suspended, which would + * skip the STOP trigger. Make sure we terminate. + */ + dmaengine_terminate_async(prtd->dma_chan); dmaengine_synchronize(prtd->dma_chan); + if (substream->pcm->nonatomic) + flush_work(&prtd->complete_wq); if (release_channel) dma_release_channel(prtd->dma_chan); kfree(prtd); diff --git a/sound/core/pcm_native.c b/sound/core/pcm_native.c index 7698a3c1ca13c7..257fa9a1f6fae6 100644 --- a/sound/core/pcm_native.c +++ b/sound/core/pcm_native.c @@ -242,10 +242,10 @@ int snd_pcm_info(struct snd_pcm_substream *substream, struct snd_pcm_info *info) int snd_pcm_info_user(struct snd_pcm_substream *substream, struct snd_pcm_info __user * _info) { - struct snd_pcm_info *info __free(kfree) = NULL; int err; + struct snd_pcm_info *info __free(kfree) = + kmalloc(sizeof(*info), GFP_KERNEL); - info = kmalloc(sizeof(*info), GFP_KERNEL); if (! info) return -ENOMEM; err = snd_pcm_info(substream, info); @@ -364,7 +364,6 @@ static int constrain_params_by_rules(struct snd_pcm_substream *substream, struct snd_pcm_hw_constraints *constrs = &substream->runtime->hw_constraints; unsigned int k; - unsigned int *rstamps __free(kfree) = NULL; unsigned int vstamps[SNDRV_PCM_HW_PARAM_LAST_INTERVAL + 1]; unsigned int stamp; struct snd_pcm_hw_rule *r; @@ -380,7 +379,8 @@ static int constrain_params_by_rules(struct snd_pcm_substream *substream, * Each member of 'rstamps' array represents the sequence number of * recent application of corresponding rule. */ - rstamps = kcalloc(constrs->rules_num, sizeof(unsigned int), GFP_KERNEL); + unsigned int *rstamps __free(kfree) = + kcalloc(constrs->rules_num, sizeof(unsigned int), GFP_KERNEL); if (!rstamps) return -ENOMEM; @@ -583,10 +583,10 @@ EXPORT_SYMBOL(snd_pcm_hw_refine); static int snd_pcm_hw_refine_user(struct snd_pcm_substream *substream, struct snd_pcm_hw_params __user * _params) { - struct snd_pcm_hw_params *params __free(kfree) = NULL; int err; + struct snd_pcm_hw_params *params __free(kfree) = + memdup_user(_params, sizeof(*params)); - params = memdup_user(_params, sizeof(*params)); if (IS_ERR(params)) return PTR_ERR(params); @@ -889,10 +889,10 @@ static int snd_pcm_hw_params(struct snd_pcm_substream *substream, static int snd_pcm_hw_params_user(struct snd_pcm_substream *substream, struct snd_pcm_hw_params __user * _params) { - struct snd_pcm_hw_params *params __free(kfree) = NULL; int err; + struct snd_pcm_hw_params *params __free(kfree) = + memdup_user(_params, sizeof(*params)); - params = memdup_user(_params, sizeof(*params)); if (IS_ERR(params)) return PTR_ERR(params); @@ -944,8 +944,9 @@ static int snd_pcm_hw_free(struct snd_pcm_substream *substream) goto unlock; result = do_hw_free(substream); snd_pcm_set_state(substream, SNDRV_PCM_STATE_OPEN); - cpu_latency_qos_remove_request(&substream->latency_pm_qos_req); - unlock: + if (cpu_latency_qos_request_active(&substream->latency_pm_qos_req)) + cpu_latency_qos_remove_request(&substream->latency_pm_qos_req); +unlock: snd_pcm_buffer_access_unlock(runtime); return result; } @@ -2267,7 +2268,6 @@ static int snd_pcm_link(struct snd_pcm_substream *substream, int fd) { struct snd_pcm_file *pcm_file; struct snd_pcm_substream *substream1; - struct snd_pcm_group *group __free(kfree) = NULL; struct snd_pcm_group *target_group; bool nonatomic = substream->pcm->nonatomic; CLASS(fd, f)(fd); @@ -2283,7 +2283,8 @@ static int snd_pcm_link(struct snd_pcm_substream *substream, int fd) if (substream == substream1) return -EINVAL; - group = kzalloc(sizeof(*group), GFP_KERNEL); + struct snd_pcm_group *group __free(kfree) = + kzalloc(sizeof(*group), GFP_KERNEL); if (!group) return -ENOMEM; snd_pcm_group_init(group); @@ -3292,7 +3293,6 @@ static int snd_pcm_xfern_frames_ioctl(struct snd_pcm_substream *substream, { struct snd_xfern xfern; struct snd_pcm_runtime *runtime = substream->runtime; - void *bufs __free(kfree) = NULL; snd_pcm_sframes_t result; if (runtime->state == SNDRV_PCM_STATE_OPEN) @@ -3304,7 +3304,8 @@ static int snd_pcm_xfern_frames_ioctl(struct snd_pcm_substream *substream, if (copy_from_user(&xfern, _xfern, sizeof(xfern))) return -EFAULT; - bufs = memdup_array_user(xfern.bufs, runtime->channels, sizeof(void *)); + void *bufs __free(kfree) = + memdup_array_user(xfern.bufs, runtime->channels, sizeof(void *)); if (IS_ERR(bufs)) return PTR_ERR(bufs); if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) @@ -3578,7 +3579,6 @@ static ssize_t snd_pcm_readv(struct kiocb *iocb, struct iov_iter *to) struct snd_pcm_runtime *runtime; snd_pcm_sframes_t result; unsigned long i; - void __user **bufs __free(kfree) = NULL; snd_pcm_uframes_t frames; const struct iovec *iov = iter_iov(to); @@ -3597,7 +3597,9 @@ static ssize_t snd_pcm_readv(struct kiocb *iocb, struct iov_iter *to) if (!frame_aligned(runtime, iov->iov_len)) return -EINVAL; frames = bytes_to_samples(runtime, iov->iov_len); - bufs = kmalloc_array(to->nr_segs, sizeof(void *), GFP_KERNEL); + + void __user **bufs __free(kfree) = + kmalloc_array(to->nr_segs, sizeof(void *), GFP_KERNEL); if (bufs == NULL) return -ENOMEM; for (i = 0; i < to->nr_segs; ++i) { @@ -3617,7 +3619,6 @@ static ssize_t snd_pcm_writev(struct kiocb *iocb, struct iov_iter *from) struct snd_pcm_runtime *runtime; snd_pcm_sframes_t result; unsigned long i; - void __user **bufs __free(kfree) = NULL; snd_pcm_uframes_t frames; const struct iovec *iov = iter_iov(from); @@ -3635,7 +3636,9 @@ static ssize_t snd_pcm_writev(struct kiocb *iocb, struct iov_iter *from) !frame_aligned(runtime, iov->iov_len)) return -EINVAL; frames = bytes_to_samples(runtime, iov->iov_len); - bufs = kmalloc_array(from->nr_segs, sizeof(void *), GFP_KERNEL); + + void __user **bufs __free(kfree) = + kmalloc_array(from->nr_segs, sizeof(void *), GFP_KERNEL); if (bufs == NULL) return -ENOMEM; for (i = 0; i < from->nr_segs; ++i) { @@ -4107,15 +4110,15 @@ static void snd_pcm_hw_convert_to_old_params(struct snd_pcm_hw_params_old *opara static int snd_pcm_hw_refine_old_user(struct snd_pcm_substream *substream, struct snd_pcm_hw_params_old __user * _oparams) { - struct snd_pcm_hw_params *params __free(kfree) = NULL; - struct snd_pcm_hw_params_old *oparams __free(kfree) = NULL; int err; - params = kmalloc(sizeof(*params), GFP_KERNEL); + struct snd_pcm_hw_params *params __free(kfree) = + kmalloc(sizeof(*params), GFP_KERNEL); if (!params) return -ENOMEM; - oparams = memdup_user(_oparams, sizeof(*oparams)); + struct snd_pcm_hw_params_old *oparams __free(kfree) = + memdup_user(_oparams, sizeof(*oparams)); if (IS_ERR(oparams)) return PTR_ERR(oparams); snd_pcm_hw_convert_from_old_params(params, oparams); @@ -4136,15 +4139,15 @@ static int snd_pcm_hw_refine_old_user(struct snd_pcm_substream *substream, static int snd_pcm_hw_params_old_user(struct snd_pcm_substream *substream, struct snd_pcm_hw_params_old __user * _oparams) { - struct snd_pcm_hw_params *params __free(kfree) = NULL; - struct snd_pcm_hw_params_old *oparams __free(kfree) = NULL; int err; - params = kmalloc(sizeof(*params), GFP_KERNEL); + struct snd_pcm_hw_params *params __free(kfree) = + kmalloc(sizeof(*params), GFP_KERNEL); if (!params) return -ENOMEM; - oparams = memdup_user(_oparams, sizeof(*oparams)); + struct snd_pcm_hw_params_old *oparams __free(kfree) = + memdup_user(_oparams, sizeof(*oparams)); if (IS_ERR(oparams)) return PTR_ERR(oparams); diff --git a/sound/core/seq/oss/seq_oss_init.c b/sound/core/seq/oss/seq_oss_init.c index 973f057eb731f4..e0c368bd09cb66 100644 --- a/sound/core/seq/oss/seq_oss_init.c +++ b/sound/core/seq/oss/seq_oss_init.c @@ -63,10 +63,10 @@ int __init snd_seq_oss_create_client(void) { int rc; - struct snd_seq_port_info *port __free(kfree) = NULL; struct snd_seq_port_callback port_callback; + struct snd_seq_port_info *port __free(kfree) = + kzalloc(sizeof(*port), GFP_KERNEL); - port = kzalloc(sizeof(*port), GFP_KERNEL); if (!port) return -ENOMEM; diff --git a/sound/core/seq/oss/seq_oss_midi.c b/sound/core/seq/oss/seq_oss_midi.c index 023e5d0a4351da..2d48c25ff4df2f 100644 --- a/sound/core/seq/oss/seq_oss_midi.c +++ b/sound/core/seq/oss/seq_oss_midi.c @@ -65,11 +65,11 @@ static int send_midi_event(struct seq_oss_devinfo *dp, struct snd_seq_event *ev, int snd_seq_oss_midi_lookup_ports(int client) { - struct snd_seq_client_info *clinfo __free(kfree) = NULL; - struct snd_seq_port_info *pinfo __free(kfree) = NULL; + struct snd_seq_client_info *clinfo __free(kfree) = + kzalloc(sizeof(*clinfo), GFP_KERNEL); + struct snd_seq_port_info *pinfo __free(kfree) = + kzalloc(sizeof(*pinfo), GFP_KERNEL); - clinfo = kzalloc(sizeof(*clinfo), GFP_KERNEL); - pinfo = kzalloc(sizeof(*pinfo), GFP_KERNEL); if (!clinfo || !pinfo) return -ENOMEM; clinfo->client = -1; @@ -305,10 +305,10 @@ int snd_seq_oss_midi_open(struct seq_oss_devinfo *dp, int dev, int fmode) { int perm; - struct seq_oss_midi *mdev __free(seq_oss_midi) = NULL; struct snd_seq_port_subscribe subs; + struct seq_oss_midi *mdev __free(seq_oss_midi) = + get_mididev(dp, dev); - mdev = get_mididev(dp, dev); if (!mdev) return -ENODEV; @@ -364,10 +364,10 @@ snd_seq_oss_midi_open(struct seq_oss_devinfo *dp, int dev, int fmode) int snd_seq_oss_midi_close(struct seq_oss_devinfo *dp, int dev) { - struct seq_oss_midi *mdev __free(seq_oss_midi) = NULL; struct snd_seq_port_subscribe subs; + struct seq_oss_midi *mdev __free(seq_oss_midi) = + get_mididev(dp, dev); - mdev = get_mididev(dp, dev); if (!mdev) return -ENODEV; guard(mutex)(&mdev->open_mutex); @@ -399,10 +399,10 @@ snd_seq_oss_midi_close(struct seq_oss_devinfo *dp, int dev) int snd_seq_oss_midi_filemode(struct seq_oss_devinfo *dp, int dev) { - struct seq_oss_midi *mdev __free(seq_oss_midi) = NULL; int mode; + struct seq_oss_midi *mdev __free(seq_oss_midi) = + get_mididev(dp, dev); - mdev = get_mididev(dp, dev); if (!mdev) return 0; @@ -422,9 +422,9 @@ snd_seq_oss_midi_filemode(struct seq_oss_devinfo *dp, int dev) void snd_seq_oss_midi_reset(struct seq_oss_devinfo *dp, int dev) { - struct seq_oss_midi *mdev __free(seq_oss_midi) = NULL; + struct seq_oss_midi *mdev __free(seq_oss_midi) = + get_mididev(dp, dev); - mdev = get_mididev(dp, dev); if (!mdev) return; if (!mdev->opened) @@ -468,9 +468,9 @@ snd_seq_oss_midi_reset(struct seq_oss_devinfo *dp, int dev) void snd_seq_oss_midi_get_addr(struct seq_oss_devinfo *dp, int dev, struct snd_seq_addr *addr) { - struct seq_oss_midi *mdev __free(seq_oss_midi) = NULL; + struct seq_oss_midi *mdev __free(seq_oss_midi) = + get_mididev(dp, dev); - mdev = get_mididev(dp, dev); if (!mdev) return; addr->client = mdev->client; @@ -485,11 +485,11 @@ int snd_seq_oss_midi_input(struct snd_seq_event *ev, int direct, void *private_data) { struct seq_oss_devinfo *dp = (struct seq_oss_devinfo *)private_data; - struct seq_oss_midi *mdev __free(seq_oss_midi) = NULL; if (dp->readq == NULL) return 0; - mdev = find_slot(ev->source.client, ev->source.port); + struct seq_oss_midi *mdev __free(seq_oss_midi) = + find_slot(ev->source.client, ev->source.port); if (!mdev) return 0; if (!(mdev->opened & PERM_READ)) @@ -595,9 +595,9 @@ send_midi_event(struct seq_oss_devinfo *dp, struct snd_seq_event *ev, struct seq int snd_seq_oss_midi_putc(struct seq_oss_devinfo *dp, int dev, unsigned char c, struct snd_seq_event *ev) { - struct seq_oss_midi *mdev __free(seq_oss_midi) = NULL; + struct seq_oss_midi *mdev __free(seq_oss_midi) = + get_mididev(dp, dev); - mdev = get_mididev(dp, dev); if (!mdev) return -ENODEV; if (snd_midi_event_encode_byte(mdev->coder, c, ev)) { @@ -613,9 +613,9 @@ snd_seq_oss_midi_putc(struct seq_oss_devinfo *dp, int dev, unsigned char c, stru int snd_seq_oss_midi_make_info(struct seq_oss_devinfo *dp, int dev, struct midi_info *inf) { - struct seq_oss_midi *mdev __free(seq_oss_midi) = NULL; + struct seq_oss_midi *mdev __free(seq_oss_midi) = + get_mididev(dp, dev); - mdev = get_mididev(dp, dev); if (!mdev) return -ENXIO; inf->device = dev; @@ -651,10 +651,9 @@ snd_seq_oss_midi_info_read(struct snd_info_buffer *buf) snd_iprintf(buf, "\nNumber of MIDI devices: %d\n", max_midi_devs); for (i = 0; i < max_midi_devs; i++) { - struct seq_oss_midi *mdev __free(seq_oss_midi) = NULL; - snd_iprintf(buf, "\nmidi %d: ", i); - mdev = get_mdev(i); + struct seq_oss_midi *mdev __free(seq_oss_midi) = + get_mdev(i); if (mdev == NULL) { snd_iprintf(buf, "*empty*\n"); continue; diff --git a/sound/core/seq/oss/seq_oss_synth.c b/sound/core/seq/oss/seq_oss_synth.c index 8c4e5913c7e69a..beea37ed942cb1 100644 --- a/sound/core/seq/oss/seq_oss_synth.c +++ b/sound/core/seq/oss/seq_oss_synth.c @@ -368,7 +368,6 @@ reset_channels(struct seq_oss_synthinfo *info) void snd_seq_oss_synth_reset(struct seq_oss_devinfo *dp, int dev) { - struct seq_oss_synth *rec __free(seq_oss_synth) = NULL; struct seq_oss_synthinfo *info; info = get_synthinfo_nospec(dp, dev); @@ -391,7 +390,8 @@ snd_seq_oss_synth_reset(struct seq_oss_devinfo *dp, int dev) return; } - rec = get_sdev(dev); + struct seq_oss_synth *rec __free(seq_oss_synth) = + get_sdev(dev); if (rec == NULL) return; if (rec->oper.reset) { @@ -415,7 +415,6 @@ int snd_seq_oss_synth_load_patch(struct seq_oss_devinfo *dp, int dev, int fmt, const char __user *buf, int p, int c) { - struct seq_oss_synth *rec __free(seq_oss_synth) = NULL; struct seq_oss_synthinfo *info; info = get_synthinfo_nospec(dp, dev); @@ -424,7 +423,9 @@ snd_seq_oss_synth_load_patch(struct seq_oss_devinfo *dp, int dev, int fmt, if (info->is_midi) return 0; - rec = get_synthdev(dp, dev); + + struct seq_oss_synth *rec __free(seq_oss_synth) = + get_synthdev(dp, dev); if (!rec) return -ENXIO; @@ -440,9 +441,9 @@ snd_seq_oss_synth_load_patch(struct seq_oss_devinfo *dp, int dev, int fmt, struct seq_oss_synthinfo * snd_seq_oss_synth_info(struct seq_oss_devinfo *dp, int dev) { - struct seq_oss_synth *rec __free(seq_oss_synth) = NULL; + struct seq_oss_synth *rec __free(seq_oss_synth) = + get_synthdev(dp, dev); - rec = get_synthdev(dp, dev); if (rec) return get_synthinfo_nospec(dp, dev); return NULL; @@ -495,13 +496,14 @@ snd_seq_oss_synth_addr(struct seq_oss_devinfo *dp, int dev, struct snd_seq_event int snd_seq_oss_synth_ioctl(struct seq_oss_devinfo *dp, int dev, unsigned int cmd, unsigned long addr) { - struct seq_oss_synth *rec __free(seq_oss_synth) = NULL; struct seq_oss_synthinfo *info; info = get_synthinfo_nospec(dp, dev); if (!info || info->is_midi) return -ENXIO; - rec = get_synthdev(dp, dev); + + struct seq_oss_synth *rec __free(seq_oss_synth) = + get_synthdev(dp, dev); if (!rec) return -ENXIO; if (rec->oper.ioctl == NULL) @@ -575,10 +577,9 @@ snd_seq_oss_synth_info_read(struct snd_info_buffer *buf) snd_iprintf(buf, "\nNumber of synth devices: %d\n", max_synth_devs); for (i = 0; i < max_synth_devs; i++) { - struct seq_oss_synth *rec __free(seq_oss_synth) = NULL; - snd_iprintf(buf, "\nsynth %d: ", i); - rec = get_sdev(i); + struct seq_oss_synth *rec __free(seq_oss_synth) = + get_sdev(i); if (rec == NULL) { snd_iprintf(buf, "*empty*\n"); continue; diff --git a/sound/core/seq/seq_clientmgr.c b/sound/core/seq/seq_clientmgr.c index f9a6e497f997cc..75a7a2af9d8c96 100644 --- a/sound/core/seq/seq_clientmgr.c +++ b/sound/core/seq/seq_clientmgr.c @@ -494,9 +494,9 @@ static int check_port_perm(struct snd_seq_client_port *port, unsigned int flags) */ static struct snd_seq_client *get_event_dest_client(struct snd_seq_event *event) { - struct snd_seq_client *dest __free(snd_seq_client) = NULL; + struct snd_seq_client *dest __free(snd_seq_client) = + snd_seq_client_use_ptr(event->dest.client); - dest = snd_seq_client_use_ptr(event->dest.client); if (dest == NULL) return NULL; if (! dest->accept_input) @@ -565,9 +565,9 @@ static int bounce_error_event(struct snd_seq_client *client, static int update_timestamp_of_queue(struct snd_seq_event *event, int queue, int real_time) { - struct snd_seq_queue *q __free(snd_seq_queue) = NULL; + struct snd_seq_queue *q __free(snd_seq_queue) = + queueptr(queue); - q = queueptr(queue); if (! q) return 0; event->queue = queue; @@ -609,13 +609,13 @@ static int _snd_seq_deliver_single_event(struct snd_seq_client *client, struct snd_seq_event *event, int atomic, int hop) { - struct snd_seq_client *dest __free(snd_seq_client) = NULL; - struct snd_seq_client_port *dest_port __free(snd_seq_port) = NULL; - - dest = get_event_dest_client(event); + struct snd_seq_client *dest __free(snd_seq_client) = + get_event_dest_client(event); if (dest == NULL) return -ENOENT; - dest_port = snd_seq_port_use_ptr(dest, event->dest.port); + + struct snd_seq_client_port *dest_port __free(snd_seq_port) = + snd_seq_port_use_ptr(dest, event->dest.port); if (dest_port == NULL) return -ENOENT; @@ -672,7 +672,6 @@ static int __deliver_to_subscribers(struct snd_seq_client *client, struct snd_seq_event *event, int port, int atomic, int hop) { - struct snd_seq_client_port *src_port __free(snd_seq_port) = NULL; struct snd_seq_subscribers *subs; int err, result = 0, num_ev = 0; union __snd_seq_event event_saved; @@ -681,7 +680,9 @@ static int __deliver_to_subscribers(struct snd_seq_client *client, if (port < 0) return 0; - src_port = snd_seq_port_use_ptr(client, port); + + struct snd_seq_client_port *src_port __free(snd_seq_port) = + snd_seq_port_use_ptr(client, port); if (!src_port) return 0; @@ -801,13 +802,13 @@ static int snd_seq_deliver_event(struct snd_seq_client *client, struct snd_seq_e */ int snd_seq_dispatch_event(struct snd_seq_event_cell *cell, int atomic, int hop) { - struct snd_seq_client *client __free(snd_seq_client) = NULL; int result; if (snd_BUG_ON(!cell)) return -EINVAL; - client = snd_seq_client_use_ptr(cell->event.source.client); + struct snd_seq_client *client __free(snd_seq_client) = + snd_seq_client_use_ptr(cell->event.source.client); if (client == NULL) { snd_seq_cell_free(cell); /* release this cell */ return -EINVAL; @@ -1154,10 +1155,10 @@ static int snd_seq_ioctl_system_info(struct snd_seq_client *client, void *arg) static int snd_seq_ioctl_running_mode(struct snd_seq_client *client, void *arg) { struct snd_seq_running_info *info = arg; - struct snd_seq_client *cptr __free(snd_seq_client) = NULL; - /* requested client number */ - cptr = client_load_and_use_ptr(info->client); + struct snd_seq_client *cptr __free(snd_seq_client) = + client_load_and_use_ptr(info->client); + if (cptr == NULL) return -ENOENT; /* don't change !!! */ @@ -1207,10 +1208,10 @@ static int snd_seq_ioctl_get_client_info(struct snd_seq_client *client, void *arg) { struct snd_seq_client_info *client_info = arg; - struct snd_seq_client *cptr __free(snd_seq_client) = NULL; - /* requested client number */ - cptr = client_load_and_use_ptr(client_info->client); + struct snd_seq_client *cptr __free(snd_seq_client) = + client_load_and_use_ptr(client_info->client); + if (cptr == NULL) return -ENOENT; /* don't change !!! */ @@ -1344,14 +1345,14 @@ static int snd_seq_ioctl_delete_port(struct snd_seq_client *client, void *arg) static int snd_seq_ioctl_get_port_info(struct snd_seq_client *client, void *arg) { struct snd_seq_port_info *info = arg; - struct snd_seq_client *cptr __free(snd_seq_client) = NULL; - struct snd_seq_client_port *port __free(snd_seq_port) = NULL; - cptr = client_load_and_use_ptr(info->addr.client); + struct snd_seq_client *cptr __free(snd_seq_client) = + client_load_and_use_ptr(info->addr.client); if (cptr == NULL) return -ENXIO; - port = snd_seq_port_use_ptr(cptr, info->addr.port); + struct snd_seq_client_port *port __free(snd_seq_port) = + snd_seq_port_use_ptr(cptr, info->addr.port); if (port == NULL) return -ENOENT; /* don't change */ @@ -1367,11 +1368,12 @@ static int snd_seq_ioctl_get_port_info(struct snd_seq_client *client, void *arg) static int snd_seq_ioctl_set_port_info(struct snd_seq_client *client, void *arg) { struct snd_seq_port_info *info = arg; - struct snd_seq_client_port *port __free(snd_seq_port) = NULL; if (info->addr.client != client->number) /* only set our own ports ! */ return -EPERM; - port = snd_seq_port_use_ptr(client, info->addr.port); + + struct snd_seq_client_port *port __free(snd_seq_port) = + snd_seq_port_use_ptr(client, info->addr.port); if (port) { snd_seq_set_port_info(port, info); /* notify the change */ @@ -1444,22 +1446,22 @@ static int snd_seq_ioctl_subscribe_port(struct snd_seq_client *client, void *arg) { struct snd_seq_port_subscribe *subs = arg; - struct snd_seq_client *receiver __free(snd_seq_client) = NULL; - struct snd_seq_client *sender __free(snd_seq_client) = NULL; - struct snd_seq_client_port *sport __free(snd_seq_port) = NULL; - struct snd_seq_client_port *dport __free(snd_seq_port) = NULL; int result; - receiver = client_load_and_use_ptr(subs->dest.client); + struct snd_seq_client *receiver __free(snd_seq_client) = + client_load_and_use_ptr(subs->dest.client); if (!receiver) return -EINVAL; - sender = client_load_and_use_ptr(subs->sender.client); + struct snd_seq_client *sender __free(snd_seq_client) = + client_load_and_use_ptr(subs->sender.client); if (!sender) return -EINVAL; - sport = snd_seq_port_use_ptr(sender, subs->sender.port); + struct snd_seq_client_port *sport __free(snd_seq_port) = + snd_seq_port_use_ptr(sender, subs->sender.port); if (!sport) return -EINVAL; - dport = snd_seq_port_use_ptr(receiver, subs->dest.port); + struct snd_seq_client_port *dport __free(snd_seq_port) = + snd_seq_port_use_ptr(receiver, subs->dest.port); if (!dport) return -EINVAL; @@ -1483,22 +1485,22 @@ static int snd_seq_ioctl_unsubscribe_port(struct snd_seq_client *client, void *arg) { struct snd_seq_port_subscribe *subs = arg; - struct snd_seq_client *receiver __free(snd_seq_client) = NULL; - struct snd_seq_client *sender __free(snd_seq_client) = NULL; - struct snd_seq_client_port *sport __free(snd_seq_port) = NULL; - struct snd_seq_client_port *dport __free(snd_seq_port) = NULL; int result; - receiver = snd_seq_client_use_ptr(subs->dest.client); + struct snd_seq_client *receiver __free(snd_seq_client) = + snd_seq_client_use_ptr(subs->dest.client); if (!receiver) return -ENXIO; - sender = snd_seq_client_use_ptr(subs->sender.client); + struct snd_seq_client *sender __free(snd_seq_client) = + snd_seq_client_use_ptr(subs->sender.client); if (!sender) return -ENXIO; - sport = snd_seq_port_use_ptr(sender, subs->sender.port); + struct snd_seq_client_port *sport __free(snd_seq_port) = + snd_seq_port_use_ptr(sender, subs->sender.port); if (!sport) return -ENXIO; - dport = snd_seq_port_use_ptr(receiver, subs->dest.port); + struct snd_seq_client_port *dport __free(snd_seq_port) = + snd_seq_port_use_ptr(receiver, subs->dest.port); if (!dport) return -ENXIO; @@ -1518,9 +1520,9 @@ static int snd_seq_ioctl_unsubscribe_port(struct snd_seq_client *client, static int snd_seq_ioctl_create_queue(struct snd_seq_client *client, void *arg) { struct snd_seq_queue_info *info = arg; - struct snd_seq_queue *q __free(snd_seq_queue) = NULL; + struct snd_seq_queue *q __free(snd_seq_queue) = + snd_seq_queue_alloc(client->number, info->locked, info->flags); - q = snd_seq_queue_alloc(client->number, info->locked, info->flags); if (IS_ERR(q)) return PTR_ERR(q); @@ -1549,9 +1551,9 @@ static int snd_seq_ioctl_get_queue_info(struct snd_seq_client *client, void *arg) { struct snd_seq_queue_info *info = arg; - struct snd_seq_queue *q __free(snd_seq_queue) = NULL; + struct snd_seq_queue *q __free(snd_seq_queue) = + queueptr(info->queue); - q = queueptr(info->queue); if (q == NULL) return -EINVAL; @@ -1569,7 +1571,6 @@ static int snd_seq_ioctl_set_queue_info(struct snd_seq_client *client, void *arg) { struct snd_seq_queue_info *info = arg; - struct snd_seq_queue *q __free(snd_seq_queue) = NULL; if (info->owner != client->number) return -EINVAL; @@ -1584,7 +1585,8 @@ static int snd_seq_ioctl_set_queue_info(struct snd_seq_client *client, return -EPERM; } - q = queueptr(info->queue); + struct snd_seq_queue *q __free(snd_seq_queue) = + queueptr(info->queue); if (! q) return -EINVAL; if (q->owner != client->number) @@ -1599,9 +1601,9 @@ static int snd_seq_ioctl_get_named_queue(struct snd_seq_client *client, void *arg) { struct snd_seq_queue_info *info = arg; - struct snd_seq_queue *q __free(snd_seq_queue) = NULL; + struct snd_seq_queue *q __free(snd_seq_queue) = + snd_seq_queue_find_name(info->name); - q = snd_seq_queue_find_name(info->name); if (q == NULL) return -EINVAL; info->queue = q->queue; @@ -1616,10 +1618,10 @@ static int snd_seq_ioctl_get_queue_status(struct snd_seq_client *client, void *arg) { struct snd_seq_queue_status *status = arg; - struct snd_seq_queue *queue __free(snd_seq_queue) = NULL; struct snd_seq_timer *tmr; + struct snd_seq_queue *queue __free(snd_seq_queue) = + queueptr(status->queue); - queue = queueptr(status->queue); if (queue == NULL) return -EINVAL; memset(status, 0, sizeof(*status)); @@ -1644,10 +1646,10 @@ static int snd_seq_ioctl_get_queue_tempo(struct snd_seq_client *client, void *arg) { struct snd_seq_queue_tempo *tempo = arg; - struct snd_seq_queue *queue __free(snd_seq_queue) = NULL; struct snd_seq_timer *tmr; + struct snd_seq_queue *queue __free(snd_seq_queue) = + queueptr(tempo->queue); - queue = queueptr(tempo->queue); if (queue == NULL) return -EINVAL; memset(tempo, 0, sizeof(*tempo)); @@ -1693,10 +1695,10 @@ static int snd_seq_ioctl_get_queue_timer(struct snd_seq_client *client, void *arg) { struct snd_seq_queue_timer *timer = arg; - struct snd_seq_queue *queue __free(snd_seq_queue) = NULL; struct snd_seq_timer *tmr; + struct snd_seq_queue *queue __free(snd_seq_queue) = + queueptr(timer->queue); - queue = queueptr(timer->queue); if (queue == NULL) return -EINVAL; @@ -1726,10 +1728,10 @@ static int snd_seq_ioctl_set_queue_timer(struct snd_seq_client *client, return -EINVAL; if (snd_seq_queue_check_access(timer->queue, client->number)) { - struct snd_seq_queue *q __free(snd_seq_queue) = NULL; struct snd_seq_timer *tmr; + struct snd_seq_queue *q __free(snd_seq_queue) = + queueptr(timer->queue); - q = queueptr(timer->queue); if (q == NULL) return -ENXIO; guard(mutex)(&q->timer_mutex); @@ -1788,9 +1790,9 @@ static int snd_seq_ioctl_get_client_pool(struct snd_seq_client *client, void *arg) { struct snd_seq_client_pool *info = arg; - struct snd_seq_client *cptr __free(snd_seq_client) = NULL; + struct snd_seq_client *cptr __free(snd_seq_client) = + client_load_and_use_ptr(info->client); - cptr = client_load_and_use_ptr(info->client); if (cptr == NULL) return -ENOENT; memset(info, 0, sizeof(*info)); @@ -1888,13 +1890,13 @@ static int snd_seq_ioctl_get_subscription(struct snd_seq_client *client, void *arg) { struct snd_seq_port_subscribe *subs = arg; - struct snd_seq_client *sender __free(snd_seq_client) = NULL; - struct snd_seq_client_port *sport __free(snd_seq_port) = NULL; - sender = client_load_and_use_ptr(subs->sender.client); + struct snd_seq_client *sender __free(snd_seq_client) = + client_load_and_use_ptr(subs->sender.client); if (!sender) return -EINVAL; - sport = snd_seq_port_use_ptr(sender, subs->sender.port); + struct snd_seq_client_port *sport __free(snd_seq_port) = + snd_seq_port_use_ptr(sender, subs->sender.port); if (!sport) return -EINVAL; return snd_seq_port_get_subscription(&sport->c_src, &subs->dest, subs); @@ -1907,16 +1909,16 @@ static int snd_seq_ioctl_get_subscription(struct snd_seq_client *client, static int snd_seq_ioctl_query_subs(struct snd_seq_client *client, void *arg) { struct snd_seq_query_subs *subs = arg; - struct snd_seq_client *cptr __free(snd_seq_client) = NULL; - struct snd_seq_client_port *port __free(snd_seq_port) = NULL; struct snd_seq_port_subs_info *group; struct list_head *p; int i; - cptr = client_load_and_use_ptr(subs->root.client); + struct snd_seq_client *cptr __free(snd_seq_client) = + client_load_and_use_ptr(subs->root.client); if (!cptr) return -ENXIO; - port = snd_seq_port_use_ptr(cptr, subs->root.port); + struct snd_seq_client_port *port __free(snd_seq_port) = + snd_seq_port_use_ptr(cptr, subs->root.port); if (!port) return -ENXIO; @@ -1963,7 +1965,6 @@ static int snd_seq_ioctl_query_next_client(struct snd_seq_client *client, void *arg) { struct snd_seq_client_info *info = arg; - struct snd_seq_client *cptr __free(snd_seq_client) = NULL; /* search for next client */ if (info->client < INT_MAX) @@ -1971,7 +1972,8 @@ static int snd_seq_ioctl_query_next_client(struct snd_seq_client *client, if (info->client < 0) info->client = 0; for (; info->client < SNDRV_SEQ_MAX_CLIENTS; info->client++) { - cptr = client_load_and_use_ptr(info->client); + struct snd_seq_client *cptr __free(snd_seq_client) = + client_load_and_use_ptr(info->client); if (cptr) { get_client_info(cptr, info); return 0; /* found */ @@ -1987,16 +1989,16 @@ static int snd_seq_ioctl_query_next_port(struct snd_seq_client *client, void *arg) { struct snd_seq_port_info *info = arg; - struct snd_seq_client *cptr __free(snd_seq_client) = NULL; - struct snd_seq_client_port *port __free(snd_seq_port) = NULL; - cptr = client_load_and_use_ptr(info->addr.client); + struct snd_seq_client *cptr __free(snd_seq_client) = + client_load_and_use_ptr(info->addr.client); if (cptr == NULL) return -ENXIO; /* search for next port */ info->addr.port++; - port = snd_seq_port_query_nearest(cptr, info); + struct snd_seq_client_port *port __free(snd_seq_port) = + snd_seq_port_query_nearest(cptr, info); if (port == NULL) return -ENOENT; @@ -2067,7 +2069,6 @@ static int snd_seq_ioctl_client_ump_info(struct snd_seq_client *caller, { struct snd_seq_client_ump_info __user *argp = (struct snd_seq_client_ump_info __user *)arg; - struct snd_seq_client *cptr __free(snd_seq_client) = NULL; int client, type, err = 0; size_t size; void *p; @@ -2083,7 +2084,9 @@ static int snd_seq_ioctl_client_ump_info(struct snd_seq_client *caller, size = sizeof(struct snd_ump_endpoint_info); else size = sizeof(struct snd_ump_block_info); - cptr = client_load_and_use_ptr(client); + + struct snd_seq_client *cptr __free(snd_seq_client) = + client_load_and_use_ptr(client); if (!cptr) return -ENOENT; @@ -2342,8 +2345,6 @@ EXPORT_SYMBOL(snd_seq_delete_kernel_client); int snd_seq_kernel_client_enqueue(int client, struct snd_seq_event *ev, struct file *file, bool blocking) { - struct snd_seq_client *cptr __free(snd_seq_client) = NULL; - if (snd_BUG_ON(!ev)) return -EINVAL; @@ -2360,7 +2361,8 @@ int snd_seq_kernel_client_enqueue(int client, struct snd_seq_event *ev, if (check_event_type_and_length(ev)) return -EINVAL; - cptr = client_load_and_use_ptr(client); + struct snd_seq_client *cptr __free(snd_seq_client) = + client_load_and_use_ptr(client); if (cptr == NULL) return -EINVAL; @@ -2385,8 +2387,6 @@ EXPORT_SYMBOL(snd_seq_kernel_client_enqueue); int snd_seq_kernel_client_dispatch(int client, struct snd_seq_event * ev, int atomic, int hop) { - struct snd_seq_client *cptr __free(snd_seq_client) = NULL; - if (snd_BUG_ON(!ev)) return -EINVAL; @@ -2397,7 +2397,8 @@ int snd_seq_kernel_client_dispatch(int client, struct snd_seq_event * ev, if (check_event_type_and_length(ev)) return -EINVAL; - cptr = snd_seq_client_use_ptr(client); + struct snd_seq_client *cptr __free(snd_seq_client) = + snd_seq_client_use_ptr(client); if (cptr == NULL) return -EINVAL; @@ -2450,9 +2451,9 @@ EXPORT_SYMBOL(snd_seq_kernel_client_ctl); /* a similar like above but taking locks; used only from OSS sequencer layer */ int snd_seq_kernel_client_ioctl(int clientid, unsigned int cmd, void *arg) { - struct snd_seq_client *client __free(snd_seq_client) = NULL; + struct snd_seq_client *client __free(snd_seq_client) = + client_load_and_use_ptr(clientid); - client = client_load_and_use_ptr(clientid); if (!client) return -ENXIO; guard(mutex)(&client->ioctl_mutex); @@ -2597,9 +2598,9 @@ void snd_seq_info_clients_read(struct snd_info_entry *entry, /* list the client table */ for (c = 0; c < SNDRV_SEQ_MAX_CLIENTS; c++) { - struct snd_seq_client *client __free(snd_seq_client) = NULL; + struct snd_seq_client *client __free(snd_seq_client) = + client_load_and_use_ptr(c); - client = client_load_and_use_ptr(c); if (client == NULL) continue; if (client->type == NO_CLIENT) diff --git a/sound/core/seq/seq_compat.c b/sound/core/seq/seq_compat.c index 643af4c1e83866..260428747e337b 100644 --- a/sound/core/seq/seq_compat.c +++ b/sound/core/seq/seq_compat.c @@ -31,10 +31,10 @@ struct snd_seq_port_info32 { static int snd_seq_call_port_info_ioctl(struct snd_seq_client *client, unsigned int cmd, struct snd_seq_port_info32 __user *data32) { - struct snd_seq_port_info *data __free(kfree) = NULL; int err; + struct snd_seq_port_info *data __free(kfree) = + kmalloc(sizeof(*data), GFP_KERNEL); - data = kmalloc(sizeof(*data), GFP_KERNEL); if (!data) return -ENOMEM; diff --git a/sound/core/seq/seq_midi.c b/sound/core/seq/seq_midi.c index 581e138a311592..73dbef8fb2ac01 100644 --- a/sound/core/seq/seq_midi.c +++ b/sound/core/seq/seq_midi.c @@ -270,8 +270,6 @@ snd_seq_midisynth_probe(struct device *_dev) struct snd_seq_device *dev = to_seq_dev(_dev); struct seq_midisynth_client *client; struct seq_midisynth *msynth, *ms; - struct snd_seq_port_info *port __free(kfree) = NULL; - struct snd_rawmidi_info *info __free(kfree) = NULL; struct snd_rawmidi *rmidi = dev->private_data; int newclient = 0; unsigned int p, ports; @@ -282,7 +280,9 @@ snd_seq_midisynth_probe(struct device *_dev) if (snd_BUG_ON(!card || device < 0 || device >= SNDRV_RAWMIDI_DEVICES)) return -EINVAL; - info = kmalloc(sizeof(*info), GFP_KERNEL); + + struct snd_rawmidi_info *info __free(kfree) = + kmalloc(sizeof(*info), GFP_KERNEL); if (! info) return -ENOMEM; info->device = device; @@ -320,7 +320,9 @@ snd_seq_midisynth_probe(struct device *_dev) } msynth = kcalloc(ports, sizeof(struct seq_midisynth), GFP_KERNEL); - port = kmalloc(sizeof(*port), GFP_KERNEL); + + struct snd_seq_port_info *port __free(kfree) = + kmalloc(sizeof(*port), GFP_KERNEL); if (msynth == NULL || port == NULL) goto __nomem; diff --git a/sound/core/seq/seq_ports.c b/sound/core/seq/seq_ports.c index 40fa379847e573..bbec34bba4f998 100644 --- a/sound/core/seq/seq_ports.c +++ b/sound/core/seq/seq_ports.c @@ -211,14 +211,13 @@ static void clear_subscriber_list(struct snd_seq_client *client, list_for_each_safe(p, n, &grp->list_head) { struct snd_seq_subscribers *subs; - struct snd_seq_client *c __free(snd_seq_client) = NULL; - struct snd_seq_client_port *aport __free(snd_seq_port) = NULL; subs = get_subscriber(p, is_src); - if (is_src) - aport = get_client_port(&subs->info.dest, &c); - else - aport = get_client_port(&subs->info.sender, &c); + struct snd_seq_client *c __free(snd_seq_client) = NULL; + struct snd_seq_client_port *aport __free(snd_seq_port) = + is_src ? + get_client_port(&subs->info.dest, &c) : + get_client_port(&subs->info.sender, &c); delete_and_unsubscribe_port(client, port, subs, is_src, false); if (!aport) { diff --git a/sound/core/seq/seq_queue.c b/sound/core/seq/seq_queue.c index f5c0e401c8ae56..c0c5e1424c5a1c 100644 --- a/sound/core/seq/seq_queue.c +++ b/sound/core/seq/seq_queue.c @@ -211,8 +211,9 @@ struct snd_seq_queue *snd_seq_queue_find_name(char *name) int i; for (i = 0; i < SNDRV_SEQ_MAX_QUEUES; i++) { - struct snd_seq_queue *q __free(snd_seq_queue) = NULL; - q = queueptr(i); + struct snd_seq_queue *q __free(snd_seq_queue) = + queueptr(i); + if (q) { if (strncmp(q->name, name, sizeof(q->name)) == 0) return no_free_ptr(q); @@ -285,12 +286,13 @@ void snd_seq_check_queue(struct snd_seq_queue *q, int atomic, int hop) int snd_seq_enqueue_event(struct snd_seq_event_cell *cell, int atomic, int hop) { int dest, err; - struct snd_seq_queue *q __free(snd_seq_queue) = NULL; if (snd_BUG_ON(!cell)) return -EINVAL; dest = cell->event.queue; /* destination queue */ - q = queueptr(dest); + + struct snd_seq_queue *q __free(snd_seq_queue) = + queueptr(dest); if (q == NULL) return -EINVAL; /* handle relative time stamps, convert them into absolute */ @@ -403,10 +405,10 @@ int snd_seq_queue_set_owner(int queueid, int client, int locked) int snd_seq_queue_timer_open(int queueid) { int result = 0; - struct snd_seq_queue *queue __free(snd_seq_queue) = NULL; struct snd_seq_timer *tmr; + struct snd_seq_queue *queue __free(snd_seq_queue) = + queueptr(queueid); - queue = queueptr(queueid); if (queue == NULL) return -EINVAL; tmr = queue->timer; @@ -423,10 +425,10 @@ int snd_seq_queue_timer_open(int queueid) */ int snd_seq_queue_timer_close(int queueid) { - struct snd_seq_queue *queue __free(snd_seq_queue) = NULL; int result = 0; + struct snd_seq_queue *queue __free(snd_seq_queue) = + queueptr(queueid); - queue = queueptr(queueid); if (queue == NULL) return -EINVAL; snd_seq_timer_close(queue); @@ -479,9 +481,9 @@ static void queue_use(struct snd_seq_queue *queue, int client, int use) */ int snd_seq_queue_use(int queueid, int client, int use) { - struct snd_seq_queue *queue __free(snd_seq_queue) = NULL; + struct snd_seq_queue *queue __free(snd_seq_queue) = + queueptr(queueid); - queue = queueptr(queueid); if (queue == NULL) return -EINVAL; guard(mutex)(&queue->timer_mutex); @@ -496,9 +498,9 @@ int snd_seq_queue_use(int queueid, int client, int use) */ int snd_seq_queue_is_used(int queueid, int client) { - struct snd_seq_queue *q __free(snd_seq_queue) = NULL; + struct snd_seq_queue *q __free(snd_seq_queue) = + queueptr(queueid); - q = queueptr(queueid); if (q == NULL) return -EINVAL; /* invalid queue */ return test_bit(client, q->clients_bitmap) ? 1 : 0; @@ -642,11 +644,11 @@ static void snd_seq_queue_process_event(struct snd_seq_queue *q, */ int snd_seq_control_queue(struct snd_seq_event *ev, int atomic, int hop) { - struct snd_seq_queue *q __free(snd_seq_queue) = NULL; - if (snd_BUG_ON(!ev)) return -EINVAL; - q = queueptr(ev->data.queue.queue); + + struct snd_seq_queue *q __free(snd_seq_queue) = + queueptr(ev->data.queue.queue); if (q == NULL) return -EINVAL; diff --git a/sound/core/seq/seq_ump_client.c b/sound/core/seq/seq_ump_client.c index 27247babb16dee..14b82a31eb1952 100644 --- a/sound/core/seq/seq_ump_client.c +++ b/sound/core/seq/seq_ump_client.c @@ -214,13 +214,13 @@ static bool skip_group(struct seq_ump_client *client, struct snd_ump_group *grou static int seq_ump_group_init(struct seq_ump_client *client, int group_index) { struct snd_ump_group *group = &client->ump->groups[group_index]; - struct snd_seq_port_info *port __free(kfree) = NULL; struct snd_seq_port_callback pcallbacks; if (skip_group(client, group)) return 0; - port = kzalloc(sizeof(*port), GFP_KERNEL); + struct snd_seq_port_info *port __free(kfree) = + kzalloc(sizeof(*port), GFP_KERNEL); if (!port) return -ENOMEM; @@ -243,12 +243,12 @@ static int seq_ump_group_init(struct seq_ump_client *client, int group_index) /* update the sequencer ports; called from notify_fb_change callback */ static void update_port_infos(struct seq_ump_client *client) { - struct snd_seq_port_info *old __free(kfree) = NULL; - struct snd_seq_port_info *new __free(kfree) = NULL; int i, err; - old = kzalloc(sizeof(*old), GFP_KERNEL); - new = kzalloc(sizeof(*new), GFP_KERNEL); + struct snd_seq_port_info *old __free(kfree) = + kzalloc(sizeof(*old), GFP_KERNEL); + struct snd_seq_port_info *new __free(kfree) = + kzalloc(sizeof(*new), GFP_KERNEL); if (!old || !new) return; @@ -278,12 +278,12 @@ static void update_port_infos(struct seq_ump_client *client) /* create a UMP Endpoint port */ static int create_ump_endpoint_port(struct seq_ump_client *client) { - struct snd_seq_port_info *port __free(kfree) = NULL; struct snd_seq_port_callback pcallbacks; unsigned int rawmidi_info = client->ump->core.info_flags; int err; - port = kzalloc(sizeof(*port), GFP_KERNEL); + struct snd_seq_port_info *port __free(kfree) = + kzalloc(sizeof(*port), GFP_KERNEL); if (!port) return -ENOMEM; diff --git a/sound/core/seq/seq_virmidi.c b/sound/core/seq/seq_virmidi.c index 9e7fd4993a1085..574493fbd50d0e 100644 --- a/sound/core/seq/seq_virmidi.c +++ b/sound/core/seq/seq_virmidi.c @@ -361,13 +361,13 @@ static int snd_virmidi_dev_attach_seq(struct snd_virmidi_dev *rdev) { int client; struct snd_seq_port_callback pcallbacks; - struct snd_seq_port_info *pinfo __free(kfree) = NULL; int err; if (rdev->client >= 0) return 0; - pinfo = kzalloc(sizeof(*pinfo), GFP_KERNEL); + struct snd_seq_port_info *pinfo __free(kfree) = + kzalloc(sizeof(*pinfo), GFP_KERNEL); if (!pinfo) return -ENOMEM; diff --git a/sound/core/timer.c b/sound/core/timer.c index d9fff5c87613ef..9a4a1748ff80bb 100644 --- a/sound/core/timer.c +++ b/sound/core/timer.c @@ -1614,12 +1614,12 @@ static int snd_timer_user_next_device(struct snd_timer_id __user *_tid) static int snd_timer_user_ginfo(struct file *file, struct snd_timer_ginfo __user *_ginfo) { - struct snd_timer_ginfo *ginfo __free(kfree) = NULL; struct snd_timer_id tid; struct snd_timer *t; struct list_head *p; + struct snd_timer_ginfo *ginfo __free(kfree) = + memdup_user(_ginfo, sizeof(*ginfo)); - ginfo = memdup_user(_ginfo, sizeof(*ginfo)); if (IS_ERR(ginfo)) return PTR_ERR(ginfo); @@ -1756,7 +1756,6 @@ static int snd_timer_user_info(struct file *file, struct snd_timer_info __user *_info) { struct snd_timer_user *tu; - struct snd_timer_info *info __free(kfree) = NULL; struct snd_timer *t; tu = file->private_data; @@ -1766,7 +1765,8 @@ static int snd_timer_user_info(struct file *file, if (!t) return -EBADFD; - info = kzalloc(sizeof(*info), GFP_KERNEL); + struct snd_timer_info *info __free(kfree) = + kzalloc(sizeof(*info), GFP_KERNEL); if (! info) return -ENOMEM; info->card = t->card ? t->card->number : -1; @@ -2192,10 +2192,10 @@ static int snd_utimer_ioctl_create(struct file *file, struct snd_timer_uinfo __user *_utimer_info) { struct snd_utimer *utimer; - struct snd_timer_uinfo *utimer_info __free(kfree) = NULL; int err, timer_fd; + struct snd_timer_uinfo *utimer_info __free(kfree) = + memdup_user(_utimer_info, sizeof(*utimer_info)); - utimer_info = memdup_user(_utimer_info, sizeof(*utimer_info)); if (IS_ERR(utimer_info)) return PTR_ERR(utimer_info); diff --git a/sound/core/vmaster.c b/sound/core/vmaster.c index c657659b236c42..76cc64245f5df4 100644 --- a/sound/core/vmaster.c +++ b/sound/core/vmaster.c @@ -56,10 +56,10 @@ struct link_follower { static int follower_update(struct link_follower *follower) { - struct snd_ctl_elem_value *uctl __free(kfree) = NULL; int err, ch; + struct snd_ctl_elem_value *uctl __free(kfree) = + kzalloc(sizeof(*uctl), GFP_KERNEL); - uctl = kzalloc(sizeof(*uctl), GFP_KERNEL); if (!uctl) return -ENOMEM; uctl->id = follower->follower.id; @@ -74,7 +74,6 @@ static int follower_update(struct link_follower *follower) /* get the follower ctl info and save the initial values */ static int follower_init(struct link_follower *follower) { - struct snd_ctl_elem_info *uinfo __free(kfree) = NULL; int err; if (follower->info.count) { @@ -84,7 +83,8 @@ static int follower_init(struct link_follower *follower) return 0; } - uinfo = kmalloc(sizeof(*uinfo), GFP_KERNEL); + struct snd_ctl_elem_info *uinfo __free(kfree) = + kmalloc(sizeof(*uinfo), GFP_KERNEL); if (!uinfo) return -ENOMEM; uinfo->id = follower->follower.id; @@ -341,9 +341,9 @@ static int master_get(struct snd_kcontrol *kcontrol, static int sync_followers(struct link_master *master, int old_val, int new_val) { struct link_follower *follower; - struct snd_ctl_elem_value *uval __free(kfree) = NULL; + struct snd_ctl_elem_value *uval __free(kfree) = + kmalloc(sizeof(*uval), GFP_KERNEL); - uval = kmalloc(sizeof(*uval), GFP_KERNEL); if (!uval) return -ENOMEM; list_for_each_entry(follower, &master->followers, list) { diff --git a/sound/hda/codecs/conexant.c b/sound/hda/codecs/conexant.c index 5fcbc1312c6971..0c517378a6d28f 100644 --- a/sound/hda/codecs/conexant.c +++ b/sound/hda/codecs/conexant.c @@ -299,6 +299,7 @@ enum { CXT_PINCFG_SWS_JS201D, CXT_PINCFG_TOP_SPEAKER, CXT_FIXUP_HP_A_U, + CXT_FIXUP_ACER_SWIFT_HP, }; /* for hda_fixup_thinkpad_acpi() */ @@ -1024,6 +1025,14 @@ static const struct hda_fixup cxt_fixups[] = { .type = HDA_FIXUP_FUNC, .v.func = cxt_fixup_hp_a_u, }, + [CXT_FIXUP_ACER_SWIFT_HP] = { + .type = HDA_FIXUP_PINS, + .v.pins = (const struct hda_pintbl[]) { + { 0x16, 0x0321403f }, /* Headphone */ + { 0x19, 0x40f001f0 }, /* Mic */ + { } + }, + }, }; static const struct hda_quirk cxt5045_fixups[] = { @@ -1073,6 +1082,7 @@ static const struct hda_quirk cxt5066_fixups[] = { SND_PCI_QUIRK(0x1025, 0x0543, "Acer Aspire One 522", CXT_FIXUP_STEREO_DMIC), SND_PCI_QUIRK(0x1025, 0x054c, "Acer Aspire 3830TG", CXT_FIXUP_ASPIRE_DMIC), SND_PCI_QUIRK(0x1025, 0x054f, "Acer Aspire 4830T", CXT_FIXUP_ASPIRE_DMIC), + SND_PCI_QUIRK(0x1025, 0x136d, "Acer Swift SF314", CXT_FIXUP_ACER_SWIFT_HP), SND_PCI_QUIRK(0x103c, 0x8079, "HP EliteBook 840 G3", CXT_FIXUP_HP_DOCK), SND_PCI_QUIRK(0x103c, 0x807C, "HP EliteBook 820 G3", CXT_FIXUP_HP_DOCK), SND_PCI_QUIRK(0x103c, 0x80FD, "HP ProBook 640 G2", CXT_FIXUP_HP_DOCK), @@ -1081,6 +1091,7 @@ static const struct hda_quirk cxt5066_fixups[] = { SND_PCI_QUIRK(0x103c, 0x8174, "HP Spectre x360", CXT_FIXUP_HP_SPECTRE), SND_PCI_QUIRK(0x103c, 0x822e, "HP ProBook 440 G4", CXT_FIXUP_MUTE_LED_GPIO), SND_PCI_QUIRK(0x103c, 0x8231, "HP ProBook 450 G4", CXT_FIXUP_MUTE_LED_GPIO), + SND_PCI_QUIRK(0x103c, 0x826b, "HP ZBook Studio G4", CXT_FIXUP_MUTE_LED_GPIO), SND_PCI_QUIRK(0x103c, 0x828c, "HP EliteBook 840 G4", CXT_FIXUP_HP_DOCK), SND_PCI_QUIRK(0x103c, 0x8299, "HP 800 G3 SFF", CXT_FIXUP_HP_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x103c, 0x829a, "HP 800 G3 DM", CXT_FIXUP_HP_MIC_NO_PRESENCE), diff --git a/sound/hda/codecs/generic.c b/sound/hda/codecs/generic.c index 7bcf9aef8275f9..443500a3518f41 100644 --- a/sound/hda/codecs/generic.c +++ b/sound/hda/codecs/generic.c @@ -1984,15 +1984,15 @@ static int parse_output_paths(struct hda_codec *codec) { struct hda_gen_spec *spec = codec->spec; struct auto_pin_cfg *cfg = &spec->autocfg; - struct auto_pin_cfg *best_cfg __free(kfree) = NULL; unsigned int val; int best_badness = INT_MAX; int badness; bool fill_hardwired = true, fill_mio_first = true; bool best_wired = true, best_mio = true; bool hp_spk_swapped = false; + struct auto_pin_cfg *best_cfg __free(kfree) = + kmalloc(sizeof(*best_cfg), GFP_KERNEL); - best_cfg = kmalloc(sizeof(*best_cfg), GFP_KERNEL); if (!best_cfg) return -ENOMEM; *best_cfg = *cfg; diff --git a/sound/hda/codecs/realtek/alc269.c b/sound/hda/codecs/realtek/alc269.c index b66965a521076e..15203c5855eb55 100644 --- a/sound/hda/codecs/realtek/alc269.c +++ b/sound/hda/codecs/realtek/alc269.c @@ -2916,7 +2916,6 @@ static void find_cirrus_companion_amps(struct hda_codec *cdc) { struct device *dev = hda_codec_dev(cdc); struct acpi_device *adev; - struct fwnode_handle *fwnode __free(fwnode_handle) = NULL; const char *bus = NULL; static const struct { const char *hid; @@ -2946,7 +2945,8 @@ static void find_cirrus_companion_amps(struct hda_codec *cdc) bus = "spi"; } - fwnode = fwnode_handle_get(acpi_fwnode_handle(adev)); + struct fwnode_handle *fwnode __free(fwnode_handle) = + fwnode_handle_get(acpi_fwnode_handle(adev)); acpi_dev_put(adev); if (!bus) { @@ -7056,7 +7056,9 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x144d, 0xc872, "Samsung Galaxy Book2 Pro (NP950XEE)", ALC298_FIXUP_SAMSUNG_AMP_V2_2_AMPS), SND_PCI_QUIRK(0x144d, 0xc886, "Samsung Galaxy Book3 Pro (NP964XFG)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), SND_PCI_QUIRK(0x144d, 0xc1ca, "Samsung Galaxy Book3 Pro 360 (NP960QFG)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), + SND_PCI_QUIRK(0x144d, 0xc1cb, "Samsung Galaxy Book3 Pro 360 (NP965QFG)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), SND_PCI_QUIRK(0x144d, 0xc1cc, "Samsung Galaxy Book3 Ultra (NT960XFH)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), + SND_PCI_QUIRK(0x1458, 0x900e, "Gigabyte G5 KF5 (2023)", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1458, 0xfa53, "Gigabyte BXBT-2807", ALC283_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1462, 0xb120, "MSI Cubi MS-B120", ALC283_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1462, 0xb171, "Cubi N 8GL (MS-B171)", ALC283_FIXUP_HEADSET_MIC), diff --git a/sound/hda/codecs/realtek/realtek.c b/sound/hda/codecs/realtek/realtek.c index ca377a5adadb5b..efe20b45052900 100644 --- a/sound/hda/codecs/realtek/realtek.c +++ b/sound/hda/codecs/realtek/realtek.c @@ -215,12 +215,13 @@ void alc_update_knob_master(struct hda_codec *codec, { unsigned int val; struct snd_kcontrol *kctl; - struct snd_ctl_elem_value *uctl __free(kfree) = NULL; kctl = snd_hda_find_mixer_ctl(codec, "Master Playback Volume"); if (!kctl) return; - uctl = kzalloc(sizeof(*uctl), GFP_KERNEL); + + struct snd_ctl_elem_value *uctl __free(kfree) = + kzalloc(sizeof(*uctl), GFP_KERNEL); if (!uctl) return; val = snd_hda_codec_read(codec, jack->nid, 0, diff --git a/sound/hda/common/codec.c b/sound/hda/common/codec.c index c6d44168c7f9de..ffe7c69d5a32c9 100644 --- a/sound/hda/common/codec.c +++ b/sound/hda/common/codec.c @@ -1854,9 +1854,9 @@ static int check_follower_present(struct hda_codec *codec, /* call kctl->put with the given value(s) */ static int put_kctl_with_value(struct snd_kcontrol *kctl, int val) { - struct snd_ctl_elem_value *ucontrol __free(kfree) = NULL; + struct snd_ctl_elem_value *ucontrol __free(kfree) = + kzalloc(sizeof(*ucontrol), GFP_KERNEL); - ucontrol = kzalloc(sizeof(*ucontrol), GFP_KERNEL); if (!ucontrol) return -ENOMEM; ucontrol->value.integer.value[0] = val; diff --git a/sound/hda/common/hda_jack.h b/sound/hda/common/hda_jack.h index ff7d289c034bfe..e9b9970c59ed4a 100644 --- a/sound/hda/common/hda_jack.h +++ b/sound/hda/common/hda_jack.h @@ -82,10 +82,10 @@ snd_hda_jack_detect_enable_callback_mst(struct hda_codec *codec, hda_nid_t nid, int dev_id, hda_jack_callback_fn func); /** - * snd_hda_jack_detect_enable - enable the jack-detection + * snd_hda_jack_detect_enable_callback - enable the jack-detection * @codec: the HDA codec * @nid: pin NID to enable - * @func: callback function to register + * @cb: callback function to register * * In the case of error, the return value will be a pointer embedded with * errno. Check and handle the return value appropriately with standard diff --git a/sound/hda/common/hda_local.h b/sound/hda/common/hda_local.h index a7e53277a0fea6..ab423f1cef5492 100644 --- a/sound/hda/common/hda_local.h +++ b/sound/hda/common/hda_local.h @@ -424,7 +424,7 @@ int _snd_hda_set_pin_ctl(struct hda_codec *codec, hda_nid_t pin, unsigned int val, bool cached); /** - * _snd_hda_set_pin_ctl - Set a pin-control value safely + * snd_hda_set_pin_ctl - Set a pin-control value safely * @codec: the codec instance * @pin: the pin NID to set the control * @val: the pin-control value (AC_PINCTL_* bits) diff --git a/sound/hda/common/sysfs.c b/sound/hda/common/sysfs.c index f8c8483fd5e5f3..bedf10b308850b 100644 --- a/sound/hda/common/sysfs.c +++ b/sound/hda/common/sysfs.c @@ -299,7 +299,6 @@ static void remove_trail_spaces(char *str) static int parse_hints(struct hda_codec *codec, const char *buf) { - char *key __free(kfree) = NULL; char *val; struct hda_hint *hint; @@ -308,7 +307,9 @@ static int parse_hints(struct hda_codec *codec, const char *buf) return 0; if (*buf == '=') return -EINVAL; - key = kstrndup_noeol(buf, 1024); + + char *key __free(kfree) = + kstrndup_noeol(buf, 1024); if (!key) return -ENOMEM; /* extract key and val */ diff --git a/sound/soc/apple/Kconfig b/sound/soc/apple/Kconfig index d8dc2f1ccc83e0..d47b189e0d611d 100644 --- a/sound/soc/apple/Kconfig +++ b/sound/soc/apple/Kconfig @@ -1,5 +1,16 @@ menu "Apple" +config SND_SOC_APPLE_AOP_AUDIO + tristate "AOP audio driver" + depends on ARCH_APPLE || COMPILE_TEST + depends on RUST + select APPLE_AOP + select SND_DMAENGINE_PCM + help + This option enables an ASoC driver for sound devices connected to the AOP + co-processor on ARM Macs. This includes the built-in microphone on those + machines. + config SND_SOC_APPLE_MCA tristate "Apple Silicon MCA driver" depends on ARCH_APPLE || COMPILE_TEST @@ -8,4 +19,21 @@ config SND_SOC_APPLE_MCA This option enables an ASoC platform driver for MCA peripherals found on Apple Silicon SoCs. +config SND_SOC_APPLE_MACAUDIO + tristate "Sound support for Apple Silicon Macs" + depends on ARCH_APPLE || COMPILE_TEST + select SND_SOC_APPLE_MCA + select SND_SIMPLE_CARD_UTILS + select APPLE_ADMAC if DMADEVICES + select COMMON_CLK_APPLE_NCO if COMMON_CLK + select SND_SOC_TAS2764 if I2C + select SND_SOC_TAS2770 if I2C + select SND_SOC_CS42L83 if I2C + select SND_SOC_CS42L84 if I2C + select REGULATOR_FIXED_VOLTAGE if REGULATOR + help + This option enables an ASoC machine-level driver for Apple Silicon Macs + and it also enables the required SoC and codec drivers for overall + sound support on these machines. + endmenu diff --git a/sound/soc/apple/Makefile b/sound/soc/apple/Makefile index 1eb8fbef60c617..7d4901f407014d 100644 --- a/sound/soc/apple/Makefile +++ b/sound/soc/apple/Makefile @@ -1,3 +1,10 @@ +snd-soc-aop-y := aop_audio.o +obj-$(CONFIG_SND_SOC_APPLE_AOP_AUDIO) += snd-soc-aop.o + snd-soc-apple-mca-y := mca.o obj-$(CONFIG_SND_SOC_APPLE_MCA) += snd-soc-apple-mca.o + +snd-soc-macaudio-objs := macaudio.o + +obj-$(CONFIG_SND_SOC_APPLE_MACAUDIO) += snd-soc-macaudio.o diff --git a/sound/soc/apple/aop_audio.rs b/sound/soc/apple/aop_audio.rs new file mode 100644 index 00000000000000..0ab5df0085c3d6 --- /dev/null +++ b/sound/soc/apple/aop_audio.rs @@ -0,0 +1,744 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![recursion_limit = "2048"] + +//! Apple AOP audio driver +//! +//! Copyright (C) The Asahi Linux Contributors + +use core::sync::atomic::{AtomicU32, Ordering}; +use core::{mem, ptr, slice}; +use core::fmt; + +use kernel::{ + bindings, c_str, device, + device::property::FwNode, + device::Core, + error::from_err_ptr, + module_platform_driver, of, platform, + prelude::*, + soc::apple::aop::{from_fourcc, EPICService, AOP}, + str::CString, + sync::Arc, + types::{ARef, ForeignOwnable}, +}; + +use pin_init::Zeroable; + +/// An unaligned u32 type. +/// +/// This is useful to avoid having to pack firmware structures entirely, since that is incompatible +/// with `#[derive(Debug)]` and atomics. +#[derive(Copy, Clone, Default)] +#[repr(C, packed(1))] +pub(crate) struct U32(pub(crate) u32); + +// SAFETY: U32 is zeroable just like u32 +unsafe impl Zeroable for U32 {} + +impl fmt::Debug for U32 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let v = self.0; + f.write_fmt(format_args!("{:#x}", v)) + } +} + +const EPIC_SUBTYPE_WRAPPED_CALL: u16 = 0x20; +const CALLTYPE_AUDIO_ATTACH_DEVICE: u32 = 0xc3000002; +const CALLTYPE_AUDIO_SET_PROP: u32 = 0xc3000005; +const PDM_NUM_COEFFS: usize = 120; +const DECIMATION_RATIOS: [u8; 3] = [0xf, 5, 2]; +const COEFFICIENTS: [u8; PDM_NUM_COEFFS * mem::size_of::()] = [ + 0x88, 0x03, 0x00, 0x00, 0x82, 0x08, 0x00, 0x00, 0x51, 0x12, 0x00, 0x00, 0x0a, 0x23, 0x00, 0x00, + 0xce, 0x3d, 0x00, 0x00, 0x97, 0x66, 0x00, 0x00, 0x43, 0xa2, 0x00, 0x00, 0x9c, 0xf6, 0x00, 0x00, + 0x53, 0x6a, 0x01, 0x00, 0xe6, 0x04, 0x02, 0x00, 0x7e, 0xce, 0x02, 0x00, 0xae, 0xcf, 0x03, 0x00, + 0x2e, 0x11, 0x05, 0x00, 0x7d, 0x9b, 0x06, 0x00, 0x75, 0x76, 0x08, 0x00, 0xd8, 0xa8, 0x0a, 0x00, + 0xd2, 0x37, 0x0d, 0x00, 0x82, 0x26, 0x10, 0x00, 0x86, 0x75, 0x13, 0x00, 0x97, 0x22, 0x17, 0x00, + 0x39, 0x28, 0x1b, 0x00, 0x89, 0x7d, 0x1f, 0x00, 0x2e, 0x16, 0x24, 0x00, 0x69, 0xe2, 0x28, 0x00, + 0x56, 0xcf, 0x2d, 0x00, 0x51, 0xc7, 0x32, 0x00, 0x80, 0xb2, 0x37, 0x00, 0x87, 0x77, 0x3c, 0x00, + 0x4c, 0xfc, 0x40, 0x00, 0xd9, 0x26, 0x45, 0x00, 0x47, 0xde, 0x48, 0x00, 0xa0, 0x0b, 0x4c, 0x00, + 0xc1, 0x9a, 0x4e, 0x00, 0x1f, 0x7b, 0x50, 0x00, 0x68, 0xa0, 0x51, 0x00, 0x06, 0x03, 0x52, 0x00, + 0x4a, 0x25, 0x00, 0x00, 0x4c, 0xaf, 0x00, 0x00, 0xc0, 0x07, 0x02, 0x00, 0x45, 0x99, 0x04, 0x00, + 0x9a, 0x84, 0x08, 0x00, 0x7d, 0x38, 0x0d, 0x00, 0x5f, 0x1a, 0x11, 0x00, 0xd9, 0x81, 0x11, 0x00, + 0x80, 0x44, 0x0b, 0x00, 0x8e, 0xe5, 0xfb, 0xff, 0xca, 0x32, 0xe3, 0xff, 0x52, 0xc7, 0xc4, 0xff, + 0xa6, 0xbc, 0xa8, 0xff, 0x83, 0xe6, 0x9a, 0xff, 0xb8, 0x5b, 0xa8, 0xff, 0x6b, 0xae, 0xdb, 0xff, + 0xe7, 0xd8, 0x38, 0x00, 0x24, 0x42, 0xba, 0x00, 0x33, 0x20, 0x50, 0x01, 0x6e, 0xdc, 0xe2, 0x01, + 0x42, 0x23, 0x58, 0x02, 0x2c, 0x50, 0x99, 0x02, 0xcf, 0xfa, 0xff, 0xff, 0x53, 0x0a, 0xff, 0xff, + 0x66, 0x23, 0xfb, 0xff, 0xa0, 0x3e, 0xf4, 0xff, 0xe6, 0x68, 0xf0, 0xff, 0xb8, 0x35, 0xf7, 0xff, + 0x56, 0xec, 0x04, 0x00, 0x37, 0xa3, 0x09, 0x00, 0x00, 0xd4, 0xfe, 0xff, 0x78, 0xa3, 0xf5, 0xff, + 0x03, 0xbf, 0xfe, 0xff, 0x84, 0xd5, 0x0b, 0x00, 0xbe, 0x0b, 0x04, 0x00, 0x52, 0x54, 0xf2, 0xff, + 0x6d, 0x3f, 0xf8, 0xff, 0xc5, 0x7f, 0x0f, 0x00, 0xe6, 0x9e, 0x0c, 0x00, 0x79, 0x03, 0xef, 0xff, + 0xd5, 0x33, 0xed, 0xff, 0xec, 0xd1, 0x11, 0x00, 0x7d, 0x69, 0x1a, 0x00, 0xd6, 0x55, 0xee, 0xff, + 0x88, 0x66, 0xdc, 0xff, 0x57, 0x26, 0x10, 0x00, 0xc7, 0x8d, 0x2e, 0x00, 0x82, 0x2e, 0xf3, 0xff, + 0x63, 0x69, 0xc4, 0xff, 0xcd, 0x08, 0x07, 0x00, 0x35, 0x34, 0x4b, 0x00, 0xaf, 0x21, 0x02, 0x00, + 0x83, 0xb6, 0xa1, 0xff, 0xe2, 0xd5, 0xef, 0xff, 0x94, 0x9b, 0x76, 0x00, 0xf3, 0xd7, 0x25, 0x00, + 0xff, 0xfc, 0x67, 0xff, 0xe3, 0xac, 0xb6, 0xff, 0x52, 0x1b, 0xcc, 0x00, 0x3c, 0x8a, 0x8b, 0x00, + 0x9f, 0x0c, 0xcd, 0xfe, 0x5c, 0x68, 0xcc, 0xfe, 0x4d, 0xc5, 0x98, 0x02, 0x82, 0xcf, 0xfb, 0x06, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +]; +const FILTER_LENGTHS: u32 = 0x542c47; +const AUDIO_DEV_PDM0: u32 = from_fourcc(b"pdm0"); +const AUDIO_DEV_LPAI: u32 = from_fourcc(b"lpai"); +const AUDIO_DEV_HPAI: u32 = from_fourcc(b"hpai"); +const POWER_STATE_OFF: u32 = from_fourcc(b"idle"); +const POWER_STATE_IDLE: u32 = from_fourcc(b"pw1 "); +const POWER_STATE_ON: u32 = from_fourcc(b"pwrd"); + +#[repr(C, packed)] +#[derive(Clone, Copy, Default)] +struct AudioAttachDevice { + _zero0: u32, + unk0: u32, + calltype: u32, + _zero1: u64, + _zero2: u64, + _pad0: u32, + len: u64, + dev_id: u32, + _pad1: u32, +} + +impl AudioAttachDevice { + fn new(dev_id: u32) -> AudioAttachDevice { + AudioAttachDevice { + unk0: 0xFFFFFFFF, + calltype: CALLTYPE_AUDIO_ATTACH_DEVICE, + dev_id, + len: 0x2c, + ..AudioAttachDevice::default() + } + } +} + +#[repr(C, packed(4))] +#[derive(Clone, Copy, Default)] +struct LpaiChannelConfig { + unk1: u32, + unk2: u32, + unk3: u32, + unk4: u32, +} + +#[repr(C, packed(4))] +#[derive(Debug, Copy, Clone)] +struct PDMConfig { + bytes_per_sample: u32, + clock_source: u32, + pdm_frequency: u32, + pdmc_frequency: u32, + slow_clock_speed: u32, + fast_clock_speed: u32, + channel_polarity_select: u32, + channel_phase_select: u32, + unk1: u32, + unk2: u16, + ratio1: u8, + ratio2: u8, + ratio3: u8, + _pad0: u8, + filter_lengths: U32, + coeff_bulk: U32, + coeffs: [u8; PDM_NUM_COEFFS * mem::size_of::()], + unk3: U32, + mic_turn_on_time_ms: U32, + _zero0: [u8; 8], + _zero1: [u8; 8], + unk4: U32, + mic_settle_time_ms: U32, + _zero2: [u8; 69], // ????? + _pad_extra: u8, // extra padding to increase the struct size to multiple of mem::size_of::() +} +// PDMConfig is intended to use `#[repr(C, packed)]` but this +// conflicts ith pin_init. Instead just ensure that it has the same size as if +// it where packed. +static_assert!(mem::size_of::() == 36 + 14 + (120 * 4) + 32 + 69 + 1); + +unsafe impl Zeroable for PDMConfig {} + +#[repr(C, packed(4))] +#[derive(Debug, Copy, Clone)] +struct DecimatorConfig { + latency: u32, + ratio1: u8, + ratio2: u8, + ratio3: u8, + _pad0: u8, + filter_lengths: u32, + coeff_bulk: u32, + coeffs: [u8; PDM_NUM_COEFFS * mem::size_of::()], +} +// DecimatorConfig is intended to use `#[repr(C, packed)]` but this +// conflicts ith pin_init. Instead just ensure that it has the same size as if +// it where packed. +static_assert!(mem::size_of::() == 16 + (120 * 4)); + +unsafe impl Zeroable for DecimatorConfig {} + +#[repr(C, packed(4))] +#[derive(Clone, Copy, Default, Debug)] +struct PowerSetting { + dev_id: u32, + cookie: u32, + _unk0: u32, + _zero0: [u8; 8], + target_pstate: u32, + unk1: u32, + _zero1: [u8; 20], +} + +impl PowerSetting { + fn new(dev_id: u32, cookie: u32, target_pstate: u32, unk1: u32) -> PowerSetting { + PowerSetting { + dev_id, + cookie, + target_pstate, + unk1, + ..PowerSetting::default() + } + } +} + +#[repr(C)] +#[derive(Clone, Copy, Default, Debug)] +struct AudioSetDeviceProp { + _zero0: u32, + unk0: u32, + calltype: u32, + _zero1: [u8; 8], + _zero2: [u8; 8], + _pad0: u32, + len: u32, + _pad1: u32, + dev_id: u32, + modifier: u32, + len2: u32, + data: T, +} +// AudioSetDeviceProp is intended to use `#[repr(C, packed)]` but this +// conflicts ith pin_init. Instead just ensure that it has the same size as if +// it where packed. +static_assert!(mem::size_of::>() == 52 + mem::size_of::()); +static_assert!(mem::size_of::>() == 52 + mem::size_of::()); +static_assert!(mem::size_of::>() == 52 + mem::size_of::()); +static_assert!(mem::size_of::>() == 52 + mem::size_of::()); + +impl AudioSetDeviceProp { + fn new(dev_id: u32, modifier: u32, data: T) -> AudioSetDeviceProp { + AudioSetDeviceProp { + unk0: 0xFFFFFFFF, + calltype: CALLTYPE_AUDIO_SET_PROP, + dev_id, + modifier, + len: mem::size_of::() as u32 + 0x30, + len2: mem::size_of::() as u32, + data, + ..AudioSetDeviceProp::default() + } + } +} + +unsafe impl Zeroable for AudioSetDeviceProp {} + +impl AudioSetDeviceProp { + fn try_init( + dev_id: u32, + modifier: u32, + data: impl Init, + ) -> impl Init, Error> + where + Error: From, + { + try_init!( + AudioSetDeviceProp { + unk0: 0xFFFFFFFF, + calltype: CALLTYPE_AUDIO_SET_PROP, + dev_id, + modifier, + len: mem::size_of::() as u32 + 0x30, + len2: mem::size_of::() as u32, + data <- data, + ..Zeroable::init_zeroed() + } + ) + } +} + +struct SndSocAopData { + dev: ARef, + adata: Arc, + service: EPICService, + pstate_cookie: AtomicU32, + fwnode: ARef, +} + +impl SndSocAopData { + fn new( + dev: ARef, + adata: Arc, + service: EPICService, + fwnode: ARef, + ) -> Result> { + Ok(Arc::new( + SndSocAopData { + dev, + adata, + service, + fwnode, + pstate_cookie: AtomicU32::new(1), + }, + GFP_KERNEL, + )?) + } + fn set_pdm_config(&self) -> Result<()> { + let pdm_cfg = init!(PDMConfig { + bytes_per_sample: 2, + clock_source: 0x706c6c20, // 'pll ' + pdm_frequency: 2400000, + pdmc_frequency: 24000000, + slow_clock_speed: 24000000, + fast_clock_speed: 24000000, + channel_polarity_select: 256, + channel_phase_select: 0, + unk1: 0xf7600, + unk2: 0, + ratio1: DECIMATION_RATIOS[0], + ratio2: DECIMATION_RATIOS[1], + ratio3: DECIMATION_RATIOS[2], + filter_lengths: U32(FILTER_LENGTHS), + coeff_bulk: U32(PDM_NUM_COEFFS as u32), + coeffs: COEFFICIENTS, + unk3: U32(1), + mic_turn_on_time_ms: U32(20), + unk4: U32(1), + mic_settle_time_ms: U32(50), + ..Zeroable::init_zeroed() + }); + let set_prop = AudioSetDeviceProp::::try_init(AUDIO_DEV_PDM0, 200, pdm_cfg); + let msg = KBox::try_init(set_prop, GFP_KERNEL)?; + let ret = self.epic_wrapped_call(msg.as_ref())?; + if ret != 0 { + dev_err!(self.dev, "Unable to set pdm config, return code {}", ret); + return Err(EIO); + } else { + Ok(()) + } + } + fn set_decimator_config(&self) -> Result<()> { + let pdm_cfg = init!(DecimatorConfig { + latency: 15, + ratio1: DECIMATION_RATIOS[0], + ratio2: DECIMATION_RATIOS[1], + ratio3: DECIMATION_RATIOS[2], + filter_lengths: FILTER_LENGTHS, + coeff_bulk: PDM_NUM_COEFFS as u32, + coeffs: COEFFICIENTS, + ..Zeroable::init_zeroed() + }); + let set_prop = + AudioSetDeviceProp::::try_init(AUDIO_DEV_PDM0, 210, pdm_cfg); + let msg = KBox::try_init(set_prop, GFP_KERNEL)?; + let ret = self.epic_wrapped_call(msg.as_ref())?; + if ret != 0 { + dev_err!( + self.dev, + "Unable to set decimator config, return code {}", + ret + ); + return Err(EIO); + } else { + Ok(()) + } + } + fn set_lpai_channel_cfg(&self) -> Result<()> { + let cfg = LpaiChannelConfig { + unk1: 7, + unk2: 7, + unk3: 1, + unk4: 7, + }; + let msg = AudioSetDeviceProp::new(AUDIO_DEV_LPAI, 301, cfg); + let ret = self.epic_wrapped_call(&msg)?; + if ret != 0 { + dev_err!( + self.dev, + "Unable to set lpai channel config, return code {}", + ret + ); + return Err(EIO); + } else { + Ok(()) + } + } + fn audio_attach_device(&self, dev_id: u32) -> Result<()> { + let msg = AudioAttachDevice::new(dev_id); + let ret = self.epic_wrapped_call(&msg)?; + if ret != 0 { + dev_err!( + self.dev, + "Unable to attach device {:?}, return code {}", + dev_id, + ret + ); + return Err(EIO); + } else { + Ok(()) + } + } + fn set_audio_power(&self, pstate: u32, unk1: u32) -> Result<()> { + let set_pstate = PowerSetting::new( + AUDIO_DEV_HPAI, + self.pstate_cookie.fetch_add(1, Ordering::Relaxed), + pstate, + unk1, + ); + let msg = AudioSetDeviceProp::new(AUDIO_DEV_HPAI, 202, set_pstate); + let ret = self.epic_wrapped_call(&msg)?; + if ret != 0 { + dev_err!( + self.dev, + "Unable to set power state {:?}, return code {}", + pstate, + ret + ); + return Err(EIO); + } else { + Ok(()) + } + } + fn epic_wrapped_call(&self, data: &T) -> Result { + let msg_bytes = + unsafe { slice::from_raw_parts(data as *const T as *const u8, mem::size_of::()) }; + self.adata + .epic_call(&self.service, EPIC_SUBTYPE_WRAPPED_CALL, msg_bytes) + } + fn request_dma_channel(&self) -> Result<*mut bindings::dma_chan> { + let res = unsafe { + from_err_ptr(bindings::dma_request_chan( + self.dev.as_raw(), + c_str!("dma").as_ptr() as _, + )) + }; + if res.is_err() { + dev_err!(self.dev, "Unable to get dma channel"); + } + res + } +} + +#[repr(transparent)] +struct SndSocAopDriver(*mut bindings::snd_card); + +fn copy_str(target: &mut [u8], source: &[u8]) { + for i in 0..source.len() { + target[i] = source[i]; + } +} + +unsafe fn dmaengine_slave_config( + chan: *mut bindings::dma_chan, + config: *mut bindings::dma_slave_config, +) -> i32 { + unsafe { + match (*(*chan).device).device_config { + Some(dc) => dc(chan, config), + None => ENOSYS.to_errno(), + } + } +} + +unsafe extern "C" fn aop_hw_params( + substream: *mut bindings::snd_pcm_substream, + params: *mut bindings::snd_pcm_hw_params, +) -> i32 { + let chan = unsafe { bindings::snd_dmaengine_pcm_get_chan(substream) }; + let mut slave_config = bindings::dma_slave_config::default(); + let ret = + unsafe { bindings::snd_hwparams_to_dma_slave_config(substream, params, &mut slave_config) }; + if ret < 0 { + return ret; + } + slave_config.src_port_window_size = 4; + unsafe { dmaengine_slave_config(chan, &mut slave_config) } +} + +unsafe extern "C" fn aop_pcm_open(substream: *mut bindings::snd_pcm_substream) -> i32 { + let data = unsafe { Arc::::borrow((*substream).private_data.cast()) }; + if let Err(e) = data.set_audio_power(POWER_STATE_IDLE, 0) { + dev_err!(data.dev, "Unable to enter 'pw1 ' state"); + return e.to_errno(); + } + let mut hwparams = bindings::snd_pcm_hardware { + info: bindings::SNDRV_PCM_INFO_MMAP + | bindings::SNDRV_PCM_INFO_MMAP_VALID + | bindings::SNDRV_PCM_INFO_INTERLEAVED, + formats: bindings::BINDINGS_SNDRV_PCM_FMTBIT_FLOAT_LE, + subformats: 0, + rates: bindings::SNDRV_PCM_RATE_48000, + rate_min: 48000, + rate_max: 48000, + channels_min: 3, + channels_max: 3, + periods_min: 2, + buffer_bytes_max: usize::MAX, + period_bytes_max: 0x4000, + periods_max: u32::MAX, + period_bytes_min: 256, + fifo_size: 16, + }; + let dma_chan = match data.request_dma_channel() { + Ok(dc) => dc, + Err(e) => return e.to_errno(), + }; + + if unsafe { (*substream).dma_buffer.dev.type_ == bindings::SNDRV_DMA_TYPE_UNKNOWN as _ } { + let ret = unsafe { + bindings::snd_pcm_set_managed_buffer( + substream, + bindings::SNDRV_DMA_TYPE_DEV_IRAM as i32, + (*(*dma_chan).device).dev, + 0, + 0, + ) + }; + if ret < 0 { + dev_err!(data.dev, "Unable to allocate dma buffers"); + unsafe { + bindings::dma_release_channel(dma_chan); + } + return ret; + } + } + + let ret = unsafe { + let mut dai_data = bindings::snd_dmaengine_dai_dma_data::default(); + bindings::snd_dmaengine_pcm_refine_runtime_hwparams( + substream, + &mut dai_data, + &mut hwparams, + dma_chan, + ) + }; + if ret != 0 { + dev_err!(data.dev, "Unable to refine hwparams"); + return ret; + } + if let Err(e) = data.set_audio_power(POWER_STATE_ON, 1) { + dev_err!(data.dev, "Unable to power mic on"); + return e.to_errno(); + } + unsafe { + (*(*substream).runtime).hw = hwparams; + bindings::snd_dmaengine_pcm_open(substream, dma_chan) + } +} + +unsafe extern "C" fn aop_pcm_prepare(_: *mut bindings::snd_pcm_substream) -> i32 { + 0 +} + +unsafe extern "C" fn aop_pcm_close(substream: *mut bindings::snd_pcm_substream) -> i32 { + let data = unsafe { Arc::::borrow((*substream).private_data.cast()) }; + if let Err(e) = data.set_audio_power(POWER_STATE_IDLE, 1) { + dev_err!(data.dev, "Unable to power mic off"); + return e.to_errno(); + } + let ret = unsafe { bindings::snd_dmaengine_pcm_close_release_chan(substream) }; + if ret != 0 { + dev_err!(data.dev, "Unable to close channel"); + return ret; + } + if let Err(e) = data.set_audio_power(POWER_STATE_OFF, 0) { + dev_err!(data.dev, "Unable to enter 'idle' power state"); + return e.to_errno(); + } + 0 +} + +unsafe extern "C" fn aop_pcm_free_private(pcm: *mut bindings::snd_pcm) { + unsafe { + Arc::::from_foreign((*pcm).private_data.cast()); + } +} + +impl SndSocAopDriver { + const VTABLE: bindings::snd_pcm_ops = bindings::snd_pcm_ops { + open: Some(aop_pcm_open), + close: Some(aop_pcm_close), + prepare: Some(aop_pcm_prepare), + trigger: Some(bindings::snd_dmaengine_pcm_trigger), + pointer: Some(bindings::snd_dmaengine_pcm_pointer), + ioctl: None, + hw_params: Some(aop_hw_params), + hw_free: None, + sync_stop: None, + get_time_info: None, + fill_silence: None, + copy: None, + page: None, + mmap: None, + ack: None, + }; + fn new(data: Arc) -> Result { + let mut this = SndSocAopDriver(ptr::null_mut()); + let ret = unsafe { + bindings::snd_card_new( + data.dev.as_raw(), + -1, + ptr::null(), + THIS_MODULE.as_ptr(), + 0, + &mut this.0, + ) + }; + if ret < 0 { + dev_err!(data.dev, "Unable to allocate sound card"); + return Err(Error::from_errno(ret)); + } + let chassis = data + .fwnode + .property_read::(c_str!("apple,chassis-name")) + .required_by(&data.dev)?; + let machine_kind = data + .fwnode + .property_read::(c_str!("apple,machine-kind")) + .required_by(&data.dev)?; + unsafe { + let name = b"aop_audio\0"; + let target = (*this.0).driver.as_mut(); + copy_str(target, name.as_ref()); + } + unsafe { + let prefix = b"Apple"; + let target = (*this.0).id.as_mut(); + copy_str(target, prefix.as_ref()); + let mut ptr = prefix.len(); + copy_str(&mut target[ptr..], chassis.to_bytes_with_nul()); + ptr += chassis.count_bytes(); + let suffix = b"HPAI\0"; + copy_str(&mut target[ptr..], suffix); + } + let longname_suffix = b"High-Power Audio Interface\0"; + let mut machine_name = KVec::with_capacity( + chassis.count_bytes() + 2 + machine_kind.count_bytes() + longname_suffix.len(), + GFP_KERNEL, + )?; + machine_name.extend_from_slice(machine_kind.to_bytes_with_nul(), GFP_KERNEL)?; + let last_item = machine_name.len() - 1; + machine_name[last_item] = b' '; + machine_name.extend_from_slice(chassis.to_bytes_with_nul(), GFP_KERNEL)?; + let last_item = machine_name.len() - 1; + machine_name[last_item] = b' '; + unsafe { + let target = (*this.0).shortname.as_mut(); + copy_str(target, machine_name.as_ref()); + let ptr = machine_name.len(); + let suffix = b"HPAI\0"; + copy_str(&mut target[ptr..], suffix); + } + machine_name.extend_from_slice(longname_suffix, GFP_KERNEL)?; + unsafe { + let target = (*this.0).longname.as_mut(); + copy_str(target, machine_name.as_ref()); + } + + let mut pcm = ptr::null_mut(); + let ret = + unsafe { bindings::snd_pcm_new(this.0, machine_name.as_ptr() as _, 0, 0, 1, &mut pcm) }; + if ret < 0 { + dev_err!(data.dev, "Unable to allocate PCM device"); + return Err(Error::from_errno(ret)); + } + + unsafe { + bindings::snd_pcm_set_ops( + pcm, + bindings::SNDRV_PCM_STREAM_CAPTURE as i32, + &Self::VTABLE, + ); + } + + unsafe { + (*pcm).private_data = data.clone().into_foreign() as _; + (*pcm).private_free = Some(aop_pcm_free_private); + (*pcm).info_flags = 0; + let name = c"aop_audio"; + copy_str((*pcm).name.as_mut(), name.to_bytes()); + } + + let ret = unsafe { bindings::snd_card_register(this.0) }; + if ret < 0 { + dev_err!(data.dev, "Unable to register sound card"); + return Err(Error::from_errno(ret)); + } + Ok(this) + } +} + +impl Drop for SndSocAopDriver { + fn drop(&mut self) { + if self.0 != ptr::null_mut() { + unsafe { + bindings::snd_card_free(self.0); + } + } + } +} + +unsafe impl Send for SndSocAopDriver {} +unsafe impl Sync for SndSocAopDriver {} + +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + (), + [(of::DeviceId::new(c_str!("apple,aop-audio")), ())] +); + +impl platform::Driver for SndSocAopDriver { + type IdInfo = (); + + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + fn probe(pdev: &platform::Device, _info: Option<&()>) -> impl PinInit { + let dev = ARef::::from(pdev.as_ref()); + let parent = pdev.as_ref().parent().unwrap(); + // SAFETY: our parent is AOP, and AopDriver is repr(transparent) for Arc + let adata_ptr = unsafe { Pin::>>::borrow(parent.get_drvdata()) }; + let adata = (&*adata_ptr).clone(); + // SAFETY: AOP sets the platform data correctly + let svc = unsafe { *((*dev.as_raw()).platform_data as *const EPICService) }; + let parent_fwnode = parent.fwnode().ok_or(ENOENT)?; + let fwnode = parent_fwnode + .get_child_by_name(c_str!("audio")) + .ok_or(EIO)?; + let audio = *module_parameters::mic_check_123.value() != 0; + if !audio && parent_fwnode.property_present(c_str!("apple,no-beamforming")) { + return Err(ENODEV); + } + let data = SndSocAopData::new(dev, adata, svc, fwnode)?; + for dev in [AUDIO_DEV_PDM0, AUDIO_DEV_HPAI, AUDIO_DEV_LPAI] { + data.audio_attach_device(dev)?; + } + data.set_lpai_channel_cfg()?; + data.set_pdm_config()?; + data.set_decimator_config()?; + Ok(Self::new(data)?) + } +} + +module_platform_driver! { + type: SndSocAopDriver, + name: "snd_soc_apple_aop", + description: "AOP microphone capture driver", + license: "Dual MIT/GPL", + alias: ["platform:snd_soc_apple_aop"], + params: { + mic_check_123: u8 { + default: 0, + description: "Enable mics without user space handling", + }, + }, +} diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c new file mode 100644 index 00000000000000..1a32679f41ea90 --- /dev/null +++ b/sound/soc/apple/macaudio.c @@ -0,0 +1,1684 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * ASoC machine driver for Apple Silicon Macs + * + * Copyright (C) The Asahi Linux Contributors + * + * Based on sound/soc/qcom/{sc7180.c|common.c} + * Copyright (c) 2018, Linaro Limited. + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + * + * + * The platform driver has independent frontend and backend DAIs with the + * option of routing backends to any of the frontends. The platform + * driver configures the routing based on DPCM couplings in ASoC runtime + * structures, which in turn are determined from DAPM paths by ASoC. But the + * platform driver doesn't supply relevant DAPM paths and leaves that up for + * the machine driver to fill in. The filled-in virtual topology can be + * anything as long as any backend isn't connected to more than one frontend + * at any given time. (The limitation is due to the unsupported case of + * reparenting of live BEs.) + */ + +/* #define DEBUG */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DRIVER_NAME "snd-soc-macaudio" + +/* + * CPU side is bit and frame clock provider + * I2S has both clocks inverted + */ +#define MACAUDIO_DAI_FMT (SND_SOC_DAIFMT_I2S | \ + SND_SOC_DAIFMT_CBC_CFC | \ + SND_SOC_DAIFMT_GATED | \ + SND_SOC_DAIFMT_IB_IF) +#define MACAUDIO_JACK_MASK (SND_JACK_HEADSET | SND_JACK_HEADPHONE) +#define MACAUDIO_SLOTWIDTH 32 +/* + * Maximum BCLK frequency + * + * Codec maximums: + * CS42L42 26.0 MHz + * TAS2770 27.1 MHz + * TAS2764 24.576 MHz + */ +#define MACAUDIO_MAX_BCLK_FREQ 24576000 + +#define SPEAKER_MAGIC_VALUE (s32)0xdec1be15 +/* milliseconds */ +#define SPEAKER_LOCK_TIMEOUT 250 + +enum macaudio_amp_type { + AMP_NONE, + AMP_TAS5770, + AMP_SN012776, + AMP_SSM3515, +}; + +enum macaudio_spkr_config { + SPKR_NONE, /* No speakers */ + SPKR_1W, /* 1 woofer / ch */ + SPKR_2W, /* 2 woofers / ch */ + SPKR_1W1T, /* 1 woofer + 1 tweeter / ch */ + SPKR_2W1T, /* 2 woofers + 1 tweeter / ch */ +}; + +struct macaudio_platform_cfg { + bool enable_speakers; + enum macaudio_amp_type amp; + enum macaudio_spkr_config speakers; + bool stereo; + int amp_gain; + int safe_vol; +}; + +static const char *volume_control_names[] = { + [AMP_TAS5770] = "* Speaker Playback Volume", + [AMP_SN012776] = "* Speaker Volume", + [AMP_SSM3515] = "* DAC Playback Volume", +}; + +#define SN012776_0DB 201 +#define SN012776_DB(x) (SN012776_0DB + 2 * (x)) +/* Same as SN012776 */ +#define TAS5770_0DB SN012776_0DB +#define TAS5770_DB(x) SN012776_DB(x) + +#define SSM3515_0DB (255 - 64) /* +24dB max, steps of 3/8 dB */ +#define SSM3515_DB(x) (SSM3515_0DB + (8 * (x) / 3)) + +struct macaudio_snd_data { + struct snd_soc_card card; + struct snd_soc_jack jack; + int jack_plugin_state; + + const struct macaudio_platform_cfg *cfg; + bool has_speakers; + bool has_sense; + bool has_safety; + unsigned int max_channels; + + struct macaudio_link_props { + /* frontend props */ + unsigned int bclk_ratio; + bool is_sense; + + /* backend props */ + bool is_speakers; + bool is_headphones; + unsigned int tdm_mask; + } *link_props; + + int speaker_sample_rate; + struct snd_kcontrol *speaker_sample_rate_kctl; + + struct mutex volume_lock_mutex; + bool speaker_volume_unlocked; + bool speaker_volume_was_locked; + struct snd_kcontrol *speaker_lock_kctl; + struct snd_ctl_file *speaker_lock_owner; + u64 bes_active; + bool speaker_lock_timeout_enabled; + ktime_t speaker_lock_timeout; + ktime_t speaker_lock_remain; + struct delayed_work lock_timeout_work; + struct work_struct lock_update_work; + +}; + +static int please_blow_up_my_speakers; +module_param(please_blow_up_my_speakers, int, 0644); +MODULE_PARM_DESC(please_blow_up_my_speakers, "Allow unsafe or untested operating configurations"); + +SND_SOC_DAILINK_DEFS(primary, + DAILINK_COMP_ARRAY(COMP_CPU("mca-pcm-0")), // CPU + DAILINK_COMP_ARRAY(COMP_DUMMY()), // CODEC + DAILINK_COMP_ARRAY(COMP_EMPTY())); // platform (filled at runtime) + +SND_SOC_DAILINK_DEFS(secondary, + DAILINK_COMP_ARRAY(COMP_CPU("mca-pcm-1")), // CPU + DAILINK_COMP_ARRAY(COMP_DUMMY()), // CODEC + DAILINK_COMP_ARRAY(COMP_EMPTY())); + +SND_SOC_DAILINK_DEFS(sense, + DAILINK_COMP_ARRAY(COMP_CPU("mca-pcm-2")), // CPU + DAILINK_COMP_ARRAY(COMP_DUMMY()), // CODEC + DAILINK_COMP_ARRAY(COMP_EMPTY())); + +static struct snd_soc_dai_link macaudio_fe_links[] = { + { + .name = "Primary", + .stream_name = "Primary", + .dynamic = 1, + .dpcm_merged_rate = 1, + .dpcm_merged_chan = 1, + .dpcm_merged_format = 1, + .dai_fmt = MACAUDIO_DAI_FMT, + SND_SOC_DAILINK_REG(primary), + }, + { + .name = "Secondary", + .stream_name = "Secondary", + .dynamic = 1, + .dpcm_merged_rate = 1, + .dpcm_merged_chan = 1, + .dpcm_merged_format = 1, + .dai_fmt = MACAUDIO_DAI_FMT, + .playback_only = 1, + SND_SOC_DAILINK_REG(secondary), + }, + { + .name = "Speaker Sense", + .stream_name = "Speaker Sense", + .capture_only = 1, + .dynamic = 1, + .dai_fmt = (SND_SOC_DAIFMT_I2S | \ + SND_SOC_DAIFMT_CBP_CFP | \ + SND_SOC_DAIFMT_GATED | \ + SND_SOC_DAIFMT_IB_IF), + SND_SOC_DAILINK_REG(sense), + }, +}; + +static struct macaudio_link_props macaudio_fe_link_props[] = { + { + /* + * Primary FE + * + * The bclk ratio at 64 for the primary frontend is important + * to ensure that the headphones codec's idea of left and right + * in a stereo stream over I2S fits in nicely with everyone else's. + * (This is until the headphones codec's driver supports + * set_tdm_slot.) + * + * The low bclk ratio precludes transmitting more than two + * channels over I2S, but that's okay since there is the secondary + * FE for speaker arrays anyway. + */ + .bclk_ratio = 64, + }, + { + /* + * Secondary FE + * + * Here we want frames plenty long to be able to drive all + * those fancy speaker arrays. + */ + .bclk_ratio = 256, + }, + { + .is_sense = 1, + } +}; + +static void macaudio_vlimit_unlock(struct macaudio_snd_data *ma, bool unlock) +{ + int ret, max; + const char *name = volume_control_names[ma->cfg->amp]; + + if (!name) { + WARN_ON_ONCE(1); + return; + } + + switch (ma->cfg->amp) { + case AMP_NONE: + WARN_ON_ONCE(1); + return; + case AMP_TAS5770: + if (unlock) + max = TAS5770_0DB; + else + max = 1; //TAS5770_DB(ma->cfg->safe_vol); + break; + case AMP_SN012776: + if (unlock) + max = SN012776_0DB; + else + max = 1; //SN012776_DB(ma->cfg->safe_vol); + break; + case AMP_SSM3515: + if (unlock) + max = SSM3515_0DB; + else + max = SSM3515_DB(ma->cfg->safe_vol); + break; + } + + ret = snd_soc_limit_volume(&ma->card, name, max); + if (ret < 0) + dev_err(ma->card.dev, "Failed to %slock volume %s: %d\n", + unlock ? "un" : "", name, ret); +} + +static void macaudio_vlimit_update(struct macaudio_snd_data *ma) +{ + int i; + bool unlock = true; + struct snd_kcontrol *kctl; + const char *reason; + + /* Do nothing if there is no safety configured */ + if (!ma->has_safety) + return; + + /* Check that someone is holding the main lock */ + if (!ma->speaker_lock_owner) { + reason = "Main control not locked"; + unlock = false; + } + + /* Check that the control has been pinged within the timeout */ + if (ma->speaker_lock_remain <= 0) { + reason = "Lock timeout"; + unlock = false; + } + + /* Check that *every* limited control is locked by the same owner */ + list_for_each_entry(kctl, &ma->card.snd_card->controls, list) { + if(!snd_soc_control_matches(kctl, volume_control_names[ma->cfg->amp])) + continue; + + for (i = 0; i < kctl->count; i++) { + if (kctl->vd[i].owner != ma->speaker_lock_owner) { + reason = "Not all child controls locked by the same process"; + unlock = false; + } + } + } + + + if (unlock != ma->speaker_volume_unlocked) { + if (unlock) { + dev_info(ma->card.dev, "Speaker volumes unlocked\n"); + } else { + dev_info(ma->card.dev, "Speaker volumes locked: %s\n", reason); + ma->speaker_volume_was_locked = true; + } + + macaudio_vlimit_unlock(ma, unlock); + ma->speaker_volume_unlocked = unlock; + snd_ctl_notify(ma->card.snd_card, SNDRV_CTL_EVENT_MASK_VALUE, + &ma->speaker_lock_kctl->id); + } +} + +static void macaudio_vlimit_enable_timeout(struct macaudio_snd_data *ma) +{ + mutex_lock(&ma->volume_lock_mutex); + + if (ma->speaker_lock_timeout_enabled) { + mutex_unlock(&ma->volume_lock_mutex); + return; + } + + if (ma->speaker_lock_remain > 0) { + ma->speaker_lock_timeout = ktime_add(ktime_get(), ma->speaker_lock_remain); + schedule_delayed_work(&ma->lock_timeout_work, usecs_to_jiffies(ktime_to_us(ma->speaker_lock_remain))); + dev_dbg(ma->card.dev, "Enabling volume limit timeout: %ld us left\n", + (long)ktime_to_us(ma->speaker_lock_remain)); + } + + macaudio_vlimit_update(ma); + + ma->speaker_lock_timeout_enabled = true; + mutex_unlock(&ma->volume_lock_mutex); +} + +static void macaudio_vlimit_disable_timeout(struct macaudio_snd_data *ma) +{ + ktime_t now; + + mutex_lock(&ma->volume_lock_mutex); + + if (!ma->speaker_lock_timeout_enabled) { + mutex_unlock(&ma->volume_lock_mutex); + return; + } + + now = ktime_get(); + + cancel_delayed_work(&ma->lock_timeout_work); + + if (ktime_after(now, ma->speaker_lock_timeout)) + ma->speaker_lock_remain = 0; + else if (ma->speaker_lock_remain > 0) + ma->speaker_lock_remain = ktime_sub(ma->speaker_lock_timeout, now); + + dev_dbg(ma->card.dev, "Disabling volume limit timeout: %ld us left\n", + (long)ktime_to_us(ma->speaker_lock_remain)); + + macaudio_vlimit_update(ma); + + ma->speaker_lock_timeout_enabled = false; + + mutex_unlock(&ma->volume_lock_mutex); +} + +static void macaudio_vlimit_timeout_work(struct work_struct *wrk) +{ + struct macaudio_snd_data *ma = container_of(to_delayed_work(wrk), + struct macaudio_snd_data, lock_timeout_work); + + mutex_lock(&ma->volume_lock_mutex); + + ma->speaker_lock_remain = 0; + macaudio_vlimit_update(ma); + + mutex_unlock(&ma->volume_lock_mutex); +} + +static void macaudio_vlimit_update_work(struct work_struct *wrk) +{ + struct macaudio_snd_data *ma = container_of(wrk, + struct macaudio_snd_data, lock_update_work); + + if (ma->bes_active) + macaudio_vlimit_enable_timeout(ma); + else + macaudio_vlimit_disable_timeout(ma); +} + +static int macaudio_copy_link(struct device *dev, struct snd_soc_dai_link *target, + struct snd_soc_dai_link *source) +{ + memcpy(target, source, sizeof(struct snd_soc_dai_link)); + + target->cpus = devm_kmemdup(dev, target->cpus, + sizeof(*target->cpus) * target->num_cpus, + GFP_KERNEL); + target->codecs = devm_kmemdup(dev, target->codecs, + sizeof(*target->codecs) * target->num_codecs, + GFP_KERNEL); + target->platforms = devm_kmemdup(dev, target->platforms, + sizeof(*target->platforms) * target->num_platforms, + GFP_KERNEL); + + if (!target->cpus || !target->codecs || !target->platforms) + return -ENOMEM; + + return 0; +} + +static int macaudio_parse_of_component(struct device_node *node, int index, + struct snd_soc_dai_link_component *comp) +{ + struct of_phandle_args args; + int ret; + + ret = of_parse_phandle_with_args(node, "sound-dai", "#sound-dai-cells", + index, &args); + if (ret) + return ret; + comp->of_node = args.np; + return snd_soc_get_dai_name(&args, &comp->dai_name); +} + +/* + * Parse one DPCM backend from the devicetree. This means taking one + * of the CPU DAIs and combining it with one or more CODEC DAIs. + */ +static int macaudio_parse_of_be_dai_link(struct macaudio_snd_data *ma, + struct snd_soc_dai_link *link, + int be_index, int ncodecs_per_be, + struct device_node *cpu, + struct device_node *codec) +{ + struct snd_soc_dai_link_component *comp; + struct device *dev = ma->card.dev; + int codec_base = be_index * ncodecs_per_be; + int ret, i; + + link->no_pcm = 1; + + link->dai_fmt = MACAUDIO_DAI_FMT; + + link->num_codecs = ncodecs_per_be; + link->codecs = devm_kcalloc(dev, ncodecs_per_be, + sizeof(*comp), GFP_KERNEL); + link->num_cpus = 1; + link->cpus = devm_kzalloc(dev, sizeof(*comp), GFP_KERNEL); + + if (!link->codecs || !link->cpus) + return -ENOMEM; + + link->num_platforms = 0; + + for_each_link_codecs(link, i, comp) { + ret = macaudio_parse_of_component(codec, codec_base + i, comp); + if (ret) + return dev_err_probe(ma->card.dev, ret, "parsing CODEC DAI of link '%s' at %pOF\n", + link->name, codec); + } + + ret = macaudio_parse_of_component(cpu, be_index, link->cpus); + if (ret) + return dev_err_probe(ma->card.dev, ret, "parsing CPU DAI of link '%s' at %pOF\n", + link->name, codec); + + link->name = link->cpus[0].dai_name; + + return 0; +} + +static int macaudio_parse_of(struct macaudio_snd_data *ma) +{ + struct device_node *codec = NULL; + struct device_node *cpu = NULL; + struct device_node *np = NULL; + struct device_node *platform = NULL; + struct snd_soc_dai_link *link = NULL; + struct snd_soc_card *card = &ma->card; + struct device *dev = card->dev; + struct macaudio_link_props *link_props; + int ret, num_links, i; + + ret = snd_soc_of_parse_card_name(card, "model"); + if (ret) { + dev_err_probe(dev, ret, "parsing card name\n"); + return ret; + } + /* + * Set long_name to prevent snd_soc_set_dmi_name() from setting one from + * make believe data u-boot provides in its SMBIOS emulation. + */ + card->long_name = card->name; + + /* Populate links, start with the fixed number of FE links */ + num_links = ARRAY_SIZE(macaudio_fe_links); + + /* Now add together the (dynamic) number of BE links */ + for_each_available_child_of_node(dev->of_node, np) { + int num_cpus; + + cpu = of_get_child_by_name(np, "cpu"); + if (!cpu) { + ret = dev_err_probe(dev, -EINVAL, + "missing CPU DAI node at %pOF\n", np); + goto err_free; + } + + num_cpus = of_count_phandle_with_args(cpu, "sound-dai", + "#sound-dai-cells"); + + if (num_cpus <= 0) { + ret = dev_err_probe(card->dev, -EINVAL, + "missing sound-dai property at %pOF\n", cpu); + goto err_free; + } + of_node_put(cpu); + cpu = NULL; + + /* Each CPU specified counts as one BE link */ + num_links += num_cpus; + } + + /* Allocate the DAI link array */ + card->dai_link = devm_kcalloc(dev, num_links, sizeof(*link), GFP_KERNEL); + ma->link_props = devm_kcalloc(dev, num_links, sizeof(*ma->link_props), GFP_KERNEL); + if (!card->dai_link || !ma->link_props) + return -ENOMEM; + + link = card->dai_link; + link_props = ma->link_props; + + for (i = 0; i < ARRAY_SIZE(macaudio_fe_links); i++) { + ret = macaudio_copy_link(dev, link, &macaudio_fe_links[i]); + if (ret) + goto err_free; + + memcpy(link_props, &macaudio_fe_link_props[i], sizeof(struct macaudio_link_props)); + link++; link_props++; + } + + for (i = 0; i < num_links; i++) + card->dai_link[i].id = i; + + /* We might disable the speakers, so count again */ + num_links = ARRAY_SIZE(macaudio_fe_links); + + /* Fill in the BEs */ + for_each_available_child_of_node(dev->of_node, np) { + const char *link_name; + bool speakers; + int be_index, num_codecs, num_bes, ncodecs_per_cpu, nchannels; + unsigned int left_mask, right_mask; + + ret = of_property_read_string(np, "link-name", &link_name); + if (ret) { + dev_err_probe(card->dev, ret, "missing link name\n"); + goto err_free; + } + + dev_dbg(ma->card.dev, "parsing link '%s'\n", link_name); + + speakers = !strcmp(link_name, "Speaker") + || !strcmp(link_name, "Speakers"); + if (speakers) { + if (!ma->cfg->enable_speakers && !please_blow_up_my_speakers) { + dev_err(card->dev, "driver can't assure safety on this model, disabling speakers\n"); + continue; + } + ma->has_speakers = 1; + if (ma->cfg->amp != AMP_SSM3515 && ma->cfg->safe_vol != 0) + ma->has_sense = 1; + } + + cpu = of_get_child_by_name(np, "cpu"); + codec = of_get_child_by_name(np, "codec"); + + if (!codec || !cpu) { + ret = dev_err_probe(dev, -EINVAL, + "missing DAI specifications for '%s'\n", link_name); + goto err_free; + } + + num_bes = of_count_phandle_with_args(cpu, "sound-dai", + "#sound-dai-cells"); + if (num_bes <= 0) { + ret = dev_err_probe(card->dev, -EINVAL, + "missing sound-dai property at %pOF\n", cpu); + goto err_free; + } + + num_codecs = of_count_phandle_with_args(codec, "sound-dai", + "#sound-dai-cells"); + if (num_codecs <= 0) { + ret = dev_err_probe(card->dev, -EINVAL, + "missing sound-dai property at %pOF\n", codec); + goto err_free; + } + + dev_dbg(ma->card.dev, "link '%s': %d CPUs %d CODECs\n", + link_name, num_bes, num_codecs); + + if (num_codecs % num_bes != 0) { + ret = dev_err_probe(card->dev, -EINVAL, + "bad combination of CODEC (%d) and CPU (%d) number at %pOF\n", + num_codecs, num_bes, np); + goto err_free; + } + + /* + * Now parse the cpu/codec lists into a number of DPCM backend links. + * In each link there will be one DAI from the cpu list paired with + * an evenly distributed number of DAIs from the codec list. (As is + * the binding semantics.) + */ + ncodecs_per_cpu = num_codecs / num_bes; + nchannels = num_codecs * (speakers ? 1 : 2); + + /* Save the max number of channels on the platform */ + if (nchannels > ma->max_channels) + ma->max_channels = nchannels; + + /* + * If there is a single speaker, assign two channels to it, because + * it can do downmix. + */ + if (nchannels < 2) + nchannels = 2; + + left_mask = 0; + for (i = 0; i < nchannels; i += 2) + left_mask = left_mask << 2 | 1; + right_mask = left_mask << 1; + + for (be_index = 0; be_index < num_bes; be_index++) { + /* + * Set initial link name to be overwritten by a BE-specific + * name later so that we can use at least use the provisional + * name in error messages. + */ + link->name = link_name; + + ret = macaudio_parse_of_be_dai_link(ma, link, be_index, + ncodecs_per_cpu, cpu, codec); + if (ret) + goto err_free; + + link_props->is_speakers = speakers; + link_props->is_headphones = !speakers; + + if (num_bes == 2) + /* This sound peripheral is split between left and right BE */ + link_props->tdm_mask = be_index ? right_mask : left_mask; + else + /* One BE covers all of the peripheral */ + link_props->tdm_mask = left_mask | right_mask; + + /* Steal platform OF reference for use in FE links later */ + platform = link->cpus->of_node; + + link++; link_props++; + } + + of_node_put(codec); + of_node_put(cpu); + cpu = codec = NULL; + + num_links += num_bes; + } + + for (i = 0; i < ARRAY_SIZE(macaudio_fe_links); i++) + card->dai_link[i].platforms->of_node = platform; + + /* Skip the speaker sense PCM link if this amp has no sense (or no speakers) */ + if (!ma->has_sense) { + for (i = 0; i < ARRAY_SIZE(macaudio_fe_links); i++) { + if (ma->link_props[i].is_sense) { + memmove(&card->dai_link[i], &card->dai_link[i + 1], + (num_links - i - 1) * sizeof (struct snd_soc_dai_link)); + num_links--; + break; + } + } + } + + card->num_links = num_links; + + return 0; + +err_free: + of_node_put(codec); + of_node_put(cpu); + of_node_put(np); + + if (!card->dai_link) + return ret; + + for (i = 0; i < num_links; i++) { + /* + * TODO: If we don't go through this path are the references + * freed inside ASoC? + */ + snd_soc_of_put_dai_link_codecs(&card->dai_link[i]); + snd_soc_of_put_dai_link_cpus(&card->dai_link[i]); + } + + return ret; +} + +static int macaudio_get_runtime_bclk_ratio(struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(rtd->card); + struct snd_soc_dpcm *dpcm; + + /* + * If this is a FE, look it up in link_props directly. + * If this is a BE, look it up in the respective FE. + */ + if (!rtd->dai_link->no_pcm) + return ma->link_props[rtd->dai_link->id].bclk_ratio; + + for_each_dpcm_fe(rtd, substream->stream, dpcm) { + int fe_id = dpcm->fe->dai_link->id; + + return ma->link_props[fe_id].bclk_ratio; + } + + return 0; +} + +static int macaudio_dpcm_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params) +{ + struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(rtd->card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + struct snd_soc_dai *cpu_dai = snd_soc_rtd_to_cpu(rtd, 0); + struct snd_interval *rate = hw_param_interval(params, + SNDRV_PCM_HW_PARAM_RATE); + int bclk_ratio = macaudio_get_runtime_bclk_ratio(substream); + int i; + + if (props->is_sense) { + rate->min = rate->max = cpu_dai->symmetric_rate; + return 0; + } + + /* Speakers BE */ + if (props->is_speakers) { + if (substream->stream == SNDRV_PCM_STREAM_CAPTURE) { + /* Sense PCM: keep the existing BE rate (0 if not already running) */ + rate->min = rate->max = cpu_dai->symmetric_rate; + + return 0; + } else { + /* + * Set the sense PCM rate control to inform userspace of the + * new sample rate. + */ + ma->speaker_sample_rate = params_rate(params); + snd_ctl_notify(ma->card.snd_card, SNDRV_CTL_EVENT_MASK_VALUE, + &ma->speaker_sample_rate_kctl->id); + } + } + + if (bclk_ratio) { + struct snd_soc_dai *dai; + int mclk = params_rate(params) * bclk_ratio; + + for_each_rtd_codec_dais(rtd, i, dai) { + snd_soc_dai_set_sysclk(dai, 0, mclk, SND_SOC_CLOCK_IN); + snd_soc_dai_set_bclk_ratio(dai, bclk_ratio); + } + + snd_soc_dai_set_sysclk(cpu_dai, 0, mclk, SND_SOC_CLOCK_OUT); + snd_soc_dai_set_bclk_ratio(cpu_dai, bclk_ratio); + } + + return 0; +} + +static int macaudio_fe_startup(struct snd_pcm_substream *substream) +{ + + struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(rtd->card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + int max_rate, ret; + + if (props->is_sense) { + /* + * Sense stream will not return data while playback is inactive, + * so do not time out. + */ + substream->wait_time = MAX_SCHEDULE_TIMEOUT; + return 0; + } + + ret = snd_pcm_hw_constraint_minmax(substream->runtime, + SNDRV_PCM_HW_PARAM_CHANNELS, + 0, ma->max_channels); + if (ret < 0) + return ret; + + max_rate = MACAUDIO_MAX_BCLK_FREQ / props->bclk_ratio; + ret = snd_pcm_hw_constraint_minmax(substream->runtime, + SNDRV_PCM_HW_PARAM_RATE, + 0, max_rate); + if (ret < 0) + return ret; + + return 0; +} + +static int macaudio_fe_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params) +{ + struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); + struct snd_soc_pcm_runtime *be; + struct snd_soc_dpcm *dpcm; + + be = NULL; + for_each_dpcm_be(rtd, substream->stream, dpcm) { + be = dpcm->be; + break; + } + + if (!be) { + dev_err(rtd->dev, "opening PCM device '%s' with no audio route configured by the user\n", + rtd->dai_link->name); + return -EINVAL; + } + + return macaudio_dpcm_hw_params(substream, params); +} + + +static void macaudio_dpcm_shutdown(struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); + struct snd_soc_dai *cpu_dai = snd_soc_rtd_to_cpu(rtd, 0); + struct snd_soc_dai *dai; + int bclk_ratio = macaudio_get_runtime_bclk_ratio(substream); + int i; + + if (bclk_ratio) { + for_each_rtd_codec_dais(rtd, i, dai) + snd_soc_dai_set_sysclk(dai, 0, 0, SND_SOC_CLOCK_IN); + + snd_soc_dai_set_sysclk(cpu_dai, 0, 0, SND_SOC_CLOCK_OUT); + } +} + +static int macaudio_be_hw_free(struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(rtd->card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + struct snd_soc_dai *dai; + int i; + + if (props->is_speakers && substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { + /* + * Clear the DAI rates, so the next open can change the sample rate. + * This won't happen automatically if the sense PCM is open. + */ + for_each_rtd_dais(rtd, i, dai) { + dai->symmetric_rate = 0; + } + + /* Notify userspace that the speakers are closed */ + ma->speaker_sample_rate = 0; + snd_ctl_notify(ma->card.snd_card, SNDRV_CTL_EVENT_MASK_VALUE, + &ma->speaker_sample_rate_kctl->id); + } + + return 0; +} + +static int macaudio_be_trigger(struct snd_pcm_substream *substream, int cmd) +{ + struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(rtd->card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + + if (props->is_speakers && substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + case SNDRV_PCM_TRIGGER_RESUME: + case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: + ma->bes_active |= BIT(rtd->dai_link->id); + break; + case SNDRV_PCM_TRIGGER_SUSPEND: + case SNDRV_PCM_TRIGGER_PAUSE_PUSH: + case SNDRV_PCM_TRIGGER_STOP: + ma->bes_active &= ~BIT(rtd->dai_link->id); + break; + default: + return -EINVAL; + } + + schedule_work(&ma->lock_update_work); + } + + return 0; +} + +static const struct snd_soc_ops macaudio_fe_ops = { + .startup = macaudio_fe_startup, + .shutdown = macaudio_dpcm_shutdown, + .hw_params = macaudio_fe_hw_params, +}; + +static const struct snd_soc_ops macaudio_be_ops = { + .hw_free = macaudio_be_hw_free, + .shutdown = macaudio_dpcm_shutdown, + .hw_params = macaudio_dpcm_hw_params, + .trigger = macaudio_be_trigger, +}; + +static int macaudio_be_assign_tdm(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_card *card = rtd->card; + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + struct snd_soc_dai *dai; + unsigned int mask; + int nslots, ret, i; + + if (!props->tdm_mask) + return 0; + + mask = props->tdm_mask; + nslots = __fls(mask) + 1; + + if (rtd->dai_link->num_codecs == 1) { + ret = snd_soc_dai_set_tdm_slot(snd_soc_rtd_to_codec(rtd, 0), mask, + 0, nslots, MACAUDIO_SLOTWIDTH); + + /* + * Headphones get a pass on -ENOTSUPP (see the comment + * around bclk_ratio value for primary FE). + */ + if (ret == -ENOTSUPP && props->is_headphones) + return 0; + + return ret; + } + + for_each_rtd_codec_dais(rtd, i, dai) { + int slot = __ffs(mask); + + mask &= ~(1 << slot); + ret = snd_soc_dai_set_tdm_slot(dai, 1 << slot, 0, nslots, + MACAUDIO_SLOTWIDTH); + if (ret) + return ret; + } + + return 0; +} + +static int macaudio_be_init(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_card *card = rtd->card; + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + struct snd_soc_dai *dai; + int i, ret; + + ret = macaudio_be_assign_tdm(rtd); + if (ret < 0) + return ret; + + if (props->is_headphones) { + for_each_rtd_codec_dais(rtd, i, dai) + snd_soc_component_set_jack(dai->component, &ma->jack, NULL); + } + + return 0; +} + +static void macaudio_be_exit(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_card *card = rtd->card; + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + struct snd_soc_dai *dai; + int i; + + if (props->is_headphones) { + for_each_rtd_codec_dais(rtd, i, dai) + snd_soc_component_set_jack(dai->component, NULL, NULL); + } +} + +static int macaudio_fe_init(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_card *card = rtd->card; + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + int nslots = props->bclk_ratio / MACAUDIO_SLOTWIDTH; + + if (props->is_sense) + return snd_soc_dai_set_tdm_slot(snd_soc_rtd_to_cpu(rtd, 0), 0, 0xffff, 16, 16); + + return snd_soc_dai_set_tdm_slot(snd_soc_rtd_to_cpu(rtd, 0), (1 << nslots) - 1, + (1 << nslots) - 1, nslots, MACAUDIO_SLOTWIDTH); +} + +static struct snd_soc_jack_pin macaudio_jack_pins[] = { + { + .pin = "Headphone", + .mask = SND_JACK_HEADPHONE, + }, + { + .pin = "Headset Mic", + .mask = SND_JACK_MICROPHONE, + }, +}; + +static int macaudio_probe(struct snd_soc_card *card) +{ + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + int ret; + + dev_dbg(card->dev, "%s!\n", __func__); + + ret = snd_soc_card_jack_new_pins(card, "Headphone Jack", + SND_JACK_HEADSET | SND_JACK_HEADPHONE, + &ma->jack, macaudio_jack_pins, + ARRAY_SIZE(macaudio_jack_pins)); + if (ret < 0) { + dev_err(card->dev, "jack creation failed: %d\n", ret); + return ret; + } + + return ret; +} + +static int macaudio_add_backend_dai_route(struct snd_soc_card *card, struct snd_soc_dai *dai, + bool is_speakers) +{ + struct snd_soc_dapm_route routes[2]; + struct snd_soc_dapm_route *r; + int nroutes = 0; + int ret; + + memset(routes, 0, sizeof(routes)); + + dev_dbg(card->dev, "adding routes for '%s'\n", dai->name); + + r = &routes[nroutes++]; + if (is_speakers) + r->source = "Speaker Playback"; + else + r->source = "Headphone Playback"; + r->sink = dai->stream[SNDRV_PCM_STREAM_PLAYBACK].widget->name; + + /* If headphone jack, add capture path */ + if (!is_speakers) { + r = &routes[nroutes++]; + r->source = dai->stream[SNDRV_PCM_STREAM_CAPTURE].widget->name; + r->sink = "Headset Capture"; + } + + /* If speakers, add sense capture path */ + if (is_speakers) { + r = &routes[nroutes++]; + r->source = dai->stream[SNDRV_PCM_STREAM_CAPTURE].widget->name; + r->sink = "Speaker Sense Capture"; + } + + ret = snd_soc_dapm_add_routes(&card->dapm, routes, nroutes); + if (ret) + dev_err(card->dev, "failed adding dynamic DAPM routes for %s\n", + dai->name); + return ret; +} + +static int macaudio_add_pin_routes(struct snd_soc_card *card, struct snd_soc_component *component, + bool is_speakers) +{ + struct snd_soc_dapm_route routes[2]; + struct snd_soc_dapm_route *r; + int nroutes = 0; + char buf[32]; + int ret; + + memset(routes, 0, sizeof(routes)); + + /* Connect the far ends of CODECs to pins */ + if (is_speakers) { + r = &routes[nroutes++]; + r->source = "OUT"; + if (component->name_prefix) { + snprintf(buf, sizeof(buf) - 1, "%s OUT", component->name_prefix); + r->source = buf; + } + r->sink = "Speaker"; + } else { + r = &routes[nroutes++]; + r->source = "Jack HP"; + r->sink = "Headphone"; + r = &routes[nroutes++]; + r->source = "Headset Mic"; + r->sink = "Jack HS"; + } + + ret = snd_soc_dapm_add_routes(&card->dapm, routes, nroutes); + if (ret) + dev_err(card->dev, "failed adding dynamic DAPM routes for %s\n", + component->name); + return ret; +} + +static int macaudio_late_probe(struct snd_soc_card *card) +{ + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + struct snd_soc_pcm_runtime *rtd; + struct snd_soc_dai *dai; + int ret, i; + + /* Add the dynamic DAPM routes */ + for_each_card_rtds(card, rtd) { + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + + if (!rtd->dai_link->no_pcm) + continue; + + for_each_rtd_cpu_dais(rtd, i, dai) { + ret = macaudio_add_backend_dai_route(card, dai, props->is_speakers); + + if (ret) + return ret; + } + + for_each_rtd_codec_dais(rtd, i, dai) { + ret = macaudio_add_pin_routes(card, dai->component, + props->is_speakers); + + if (ret) + return ret; + } + } + + if (ma->has_speakers) + ma->speaker_sample_rate_kctl = snd_soc_card_get_kcontrol(card, + "Speaker Sample Rate"); + if (ma->has_safety) { + ma->speaker_lock_kctl = snd_soc_card_get_kcontrol(card, + "Speaker Volume Unlock"); + + mutex_lock(&ma->volume_lock_mutex); + macaudio_vlimit_unlock(ma, false); + mutex_unlock(&ma->volume_lock_mutex); + } + + return 0; +} + +#define CHECK(call, pattern, value, min) \ + { \ + int ret = call(card, pattern, value); \ + int err = (ret >= 0 && ret < min) ? -ERANGE : ret; \ + if (err < 0) { \ + dev_err(card->dev, "%s on '%s': %d\n", #call, pattern, \ + ret); \ + if (please_blow_up_my_speakers < 2) \ + return err; \ + } else { \ + dev_dbg(card->dev, "%s on '%s': %d hits\n", #call, \ + pattern, ret); \ + } \ + } + +#define CHECK_CONCAT(call, suffix, value) \ + { \ + snprintf(buf, sizeof(buf), "%s%s", prefix, suffix); \ + CHECK(call, buf, value, 1); \ + } + +static int macaudio_set_speaker(struct snd_soc_card *card, const char *prefix, bool tweeter) +{ + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + char buf[256]; + + if (!ma->has_speakers) + return 0; + + switch (ma->cfg->amp) { + case AMP_TAS5770: + if (ma->cfg->stereo) { + CHECK_CONCAT(snd_soc_set_enum_kctl, "ASI1 Sel", "Left"); + CHECK_CONCAT(snd_soc_deactivate_kctl, "ASI1 Sel", 0); + } + + CHECK_CONCAT(snd_soc_limit_volume, "Amp Gain Volume", ma->cfg->amp_gain); + break; + case AMP_SN012776: + if (ma->cfg->stereo) { + CHECK_CONCAT(snd_soc_set_enum_kctl, "ASI1 Sel", "Left"); + CHECK_CONCAT(snd_soc_deactivate_kctl, "ASI1 Sel", 0); + } + + CHECK_CONCAT(snd_soc_limit_volume, "Amp Gain Volume", ma->cfg->amp_gain); + CHECK_CONCAT(snd_soc_set_enum_kctl, "HPF Corner Frequency", + tweeter ? "800 Hz" : "2 Hz"); + + if (please_blow_up_my_speakers < 2) + CHECK_CONCAT(snd_soc_deactivate_kctl, "HPF Corner Frequency", 0); + + CHECK_CONCAT(snd_soc_set_enum_kctl, "OCE Handling", "Retry"); + CHECK_CONCAT(snd_soc_deactivate_kctl, "OCE Handling", 0); + break; + case AMP_SSM3515: + /* TODO: check */ + CHECK_CONCAT(snd_soc_set_enum_kctl, "DAC Analog Gain Select", "8.4 V Span"); + + if (please_blow_up_my_speakers < 2) + CHECK_CONCAT(snd_soc_deactivate_kctl, "DAC Analog Gain Select", 0); + + /* TODO: HPF, needs new call to set */ + break; + default: + return -EINVAL; + } + + return 0; +} + +static int macaudio_fixup_controls(struct snd_soc_card *card) +{ + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + const char *p; + + /* Set the card ID early to avoid races with udev */ + p = strrchr(card->name, ' '); + if (p) { + snprintf(card->snd_card->id, sizeof(card->snd_card->id), + "Apple%s", p + 1); + } + + if (!ma->has_speakers) + return 0; + + /* + * This needs some care to avoid matches against cs42l84's + * "Jack HPF Corner Frequency". + */ + switch(ma->cfg->speakers) { + case SPKR_NONE: + WARN_ON(please_blow_up_my_speakers < 2); + return please_blow_up_my_speakers >= 2 ? 0 : -EINVAL; + case SPKR_1W: + /* only 1W stereo system (J313) is uses cs42l83 */ + if (ma->cfg->stereo) { + CHECK(macaudio_set_speaker, "* ", false, 0); + } else { + CHECK(macaudio_set_speaker, "", false, 0); + } + break; + case SPKR_2W: + CHECK(macaudio_set_speaker, "* Front ", false, 0); + CHECK(macaudio_set_speaker, "* Rear ", false, 0); + break; + case SPKR_1W1T: + CHECK(macaudio_set_speaker, "* Tweeter ", true, 0); + CHECK(macaudio_set_speaker, "* Woofer ", false, 0); + break; + case SPKR_2W1T: + CHECK(macaudio_set_speaker, "* Tweeter ", true, 0); + CHECK(macaudio_set_speaker, "* Woofer 1 ", false, 0); + CHECK(macaudio_set_speaker, "* Woofer 2 ", false, 0); + break; + } + + return 0; +} + +static const char * const macaudio_spk_mux_texts[] = { + "Primary", + "Secondary" +}; + +SOC_ENUM_SINGLE_VIRT_DECL(macaudio_spk_mux_enum, macaudio_spk_mux_texts); + +static const struct snd_kcontrol_new macaudio_spk_mux = + SOC_DAPM_ENUM("Speaker Playback Mux", macaudio_spk_mux_enum); + +static const char * const macaudio_hp_mux_texts[] = { + "Primary", + "Secondary" +}; + +SOC_ENUM_SINGLE_VIRT_DECL(macaudio_hp_mux_enum, macaudio_hp_mux_texts); + +static const struct snd_kcontrol_new macaudio_hp_mux = + SOC_DAPM_ENUM("Headphones Playback Mux", macaudio_hp_mux_enum); + +static const struct snd_soc_dapm_widget macaudio_snd_widgets[] = { + SND_SOC_DAPM_SPK("Speaker", NULL), + SND_SOC_DAPM_SPK("Speaker (Static)", NULL), + SND_SOC_DAPM_HP("Headphone", NULL), + SND_SOC_DAPM_MIC("Headset Mic", NULL), + + SND_SOC_DAPM_MUX("Speaker Playback Mux", SND_SOC_NOPM, 0, 0, &macaudio_spk_mux), + SND_SOC_DAPM_MUX("Headphone Playback Mux", SND_SOC_NOPM, 0, 0, &macaudio_hp_mux), + + SND_SOC_DAPM_AIF_OUT("Speaker Playback", NULL, 0, SND_SOC_NOPM, 0, 0), + SND_SOC_DAPM_AIF_OUT("Headphone Playback", NULL, 0, SND_SOC_NOPM, 0, 0), + + SND_SOC_DAPM_AIF_IN("Headset Capture", NULL, 0, SND_SOC_NOPM, 0, 0), + SND_SOC_DAPM_AIF_IN("Speaker Sense Capture", NULL, 0, SND_SOC_NOPM, 0, 0), +}; + +static int macaudio_sss_info(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_info *uinfo) +{ + uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER; + uinfo->count = 1; + uinfo->value.integer.min = 0; + uinfo->value.integer.max = 192000; + + return 0; +} + +static int macaudio_sss_get(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *uvalue) +{ + struct snd_soc_card *card = snd_kcontrol_chip(kcontrol); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + + /* + * TODO: Check if any locking is in order here. I would + * assume there is some ALSA-level lock, but DAPM implementations + * of kcontrol ops do explicit locking, so look into it. + */ + uvalue->value.integer.value[0] = ma->speaker_sample_rate; + + return 0; +} + +static int macaudio_slk_info(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_info *uinfo) +{ + uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER; + uinfo->count = 1; + uinfo->value.integer.min = INT_MIN; + uinfo->value.integer.max = INT_MAX; + + return 0; +} + +static int macaudio_slk_put(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *uvalue) +{ + struct snd_soc_card *card = snd_kcontrol_chip(kcontrol); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + + if (!ma->speaker_lock_owner) + return -EPERM; + + if (uvalue->value.integer.value[0] != SPEAKER_MAGIC_VALUE) + return -EINVAL; + + /* Serves as a notification that the lock was lost at some point */ + if (ma->speaker_volume_was_locked) { + ma->speaker_volume_was_locked = false; + return -ETIMEDOUT; + } + + mutex_lock(&ma->volume_lock_mutex); + + cancel_delayed_work(&ma->lock_timeout_work); + + ma->speaker_lock_remain = ms_to_ktime(SPEAKER_LOCK_TIMEOUT); + ma->speaker_lock_timeout = ktime_add(ktime_get(), ma->speaker_lock_remain); + macaudio_vlimit_update(ma); + + if (ma->speaker_lock_timeout_enabled) { + dev_dbg(ma->card.dev, "Volume limit timeout ping: %ld us left\n", + (long)ktime_to_us(ma->speaker_lock_remain)); + schedule_delayed_work(&ma->lock_timeout_work, usecs_to_jiffies(ktime_to_us(ma->speaker_lock_remain))); + } + + mutex_unlock(&ma->volume_lock_mutex); + + return 0; +} + +static int macaudio_slk_get(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *uvalue) +{ + struct snd_soc_card *card = snd_kcontrol_chip(kcontrol); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + + uvalue->value.integer.value[0] = ma->speaker_volume_unlocked ? 1 : 0; + + return 0; +} + +static int macaudio_slk_lock(struct snd_kcontrol *kcontrol, struct snd_ctl_file *owner) +{ + struct snd_soc_card *card = snd_kcontrol_chip(kcontrol); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + + mutex_lock(&ma->volume_lock_mutex); + ma->speaker_lock_owner = owner; + macaudio_vlimit_update(ma); + + /* + * Reset the unintended lock flag when the control is first locked. + * At this point the state is locked and cannot be unlocked until + * userspace writes to this control, so this cannot spuriously become + * true again until that point. + */ + ma->speaker_volume_was_locked = false; + + mutex_unlock(&ma->volume_lock_mutex); + + return 0; +} + +static void macaudio_slk_unlock(struct snd_kcontrol *kcontrol) +{ + struct snd_soc_card *card = snd_kcontrol_chip(kcontrol); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + + ma->speaker_lock_owner = NULL; + ma->speaker_lock_timeout = 0; + macaudio_vlimit_update(ma); +} + +/* + * Speaker limit controls go last. We only drop the unlock control, + * leaving sample rate, since that can be useful for safety + * bring-up before the kernel-side caps are ready. + */ +#define MACAUDIO_NUM_SPEAKER_LIMIT_CONTROLS 1 +/* + * If there are no speakers configured at all, we can drop both + * controls. + */ +#define MACAUDIO_NUM_SPEAKER_CONTROLS 2 + +static const struct snd_kcontrol_new macaudio_controls[] = { + SOC_DAPM_PIN_SWITCH("Speaker"), + SOC_DAPM_PIN_SWITCH("Headphone"), + SOC_DAPM_PIN_SWITCH("Headset Mic"), + { + .iface = SNDRV_CTL_ELEM_IFACE_MIXER, + .access = SNDRV_CTL_ELEM_ACCESS_READ | + SNDRV_CTL_ELEM_ACCESS_VOLATILE, + .name = "Speaker Sample Rate", + .info = macaudio_sss_info, .get = macaudio_sss_get, + }, + { + .iface = SNDRV_CTL_ELEM_IFACE_MIXER, + .access = SNDRV_CTL_ELEM_ACCESS_READ | + SNDRV_CTL_ELEM_ACCESS_WRITE | + SNDRV_CTL_ELEM_ACCESS_VOLATILE, + .name = "Speaker Volume Unlock", + .info = macaudio_slk_info, + .put = macaudio_slk_put, .get = macaudio_slk_get, + .lock = macaudio_slk_lock, .unlock = macaudio_slk_unlock, + }, +}; + +static const struct snd_soc_dapm_route macaudio_dapm_routes[] = { + /* Playback paths */ + { "Speaker Playback Mux", "Primary", "PCM0 TX" }, + { "Speaker Playback Mux", "Secondary", "PCM1 TX" }, + { "Speaker Playback", NULL, "Speaker Playback Mux"}, + + { "Headphone Playback Mux", "Primary", "PCM0 TX" }, + { "Headphone Playback Mux", "Secondary", "PCM1 TX" }, + { "Headphone Playback", NULL, "Headphone Playback Mux"}, + /* + * Additional paths (to specific I2S ports) are added dynamically. + */ + + /* Capture paths */ + { "PCM0 RX", NULL, "Headset Capture" }, + + /* Sense paths */ + { "PCM2 RX", NULL, "Speaker Sense Capture" }, +}; + +/* enable amp speakers stereo gain safe_vol */ +struct macaudio_platform_cfg macaudio_j180_cfg = { + false, AMP_SN012776, SPKR_1W1T, false, 10, -20, +}; +struct macaudio_platform_cfg macaudio_j274_cfg = { + true, AMP_TAS5770, SPKR_1W, false, 20, -20, +}; + +struct macaudio_platform_cfg macaudio_j293_cfg = { + true, AMP_TAS5770, SPKR_2W, true, 15, -20, +}; + +struct macaudio_platform_cfg macaudio_j313_cfg = { + true, AMP_TAS5770, SPKR_1W, true, 10, -20, +}; + +struct macaudio_platform_cfg macaudio_j314_cfg = { + true, AMP_SN012776, SPKR_2W1T, true, 15, -20, +}; + +struct macaudio_platform_cfg macaudio_j316_cfg = { + true, AMP_SN012776, SPKR_2W1T, true, 15, -20, +}; + +struct macaudio_platform_cfg macaudio_j37x_j47x_cfg = { + true, AMP_SN012776, SPKR_1W, false, 20, -20, +}; + +struct macaudio_platform_cfg macaudio_j413_cfg = { + true, AMP_SN012776, SPKR_1W1T, true, 15, -20, +}; + +struct macaudio_platform_cfg macaudio_j415_cfg = { + true, AMP_SN012776, SPKR_2W1T, true, 15, -20, +}; + +struct macaudio_platform_cfg macaudio_j45x_cfg = { + false, AMP_SSM3515, SPKR_1W1T, true, 9, -20, /* TODO: gain?? */ +}; + +struct macaudio_platform_cfg macaudio_j493_cfg = { + true, AMP_SN012776, SPKR_2W, true, 15, -20, +}; + +struct macaudio_platform_cfg macaudio_fallback_cfg = { + false, AMP_NONE, SPKR_NONE, false, 0, 0, +}; + +/* + * DT compatible/ID table rules: + * + * 1. Machines with **identical** speaker configurations (amps, models, chassis) + * are allowed to declare compatibility with the first model (chronologically), + * and are not enumerated in this array. + * + * 2. Machines with identical amps and speakers (=identical speaker protection + * rules) but a different chassis must use different compatibles, but may share + * the private data structure here. They are explicitly enumerated. + * + * 3. Machines with different amps or speaker layouts must use separate + * data structures. + * + * 4. Machines with identical speaker layouts and amps (but possibly different + * speaker models/chassis) may share the data structure, since only userspace + * cares about that (assuming our general -20dB safe level standard holds). + */ +static const struct of_device_id macaudio_snd_device_id[] = { + /* Model ID Amp Gain Speakers */ + /* j180 AID19 sn012776 10 1× 1W+1T */ + { .compatible = "apple,j180-macaudio", .data = &macaudio_j180_cfg }, + /* j274 AID6 tas5770 20 1× 1W */ + { .compatible = "apple,j274-macaudio", .data = &macaudio_j274_cfg }, + /* j293 AID3 tas5770 15 2× 2W */ + { .compatible = "apple,j293-macaudio", .data = &macaudio_j293_cfg }, + /* j313 AID4 tas5770 10 2× 1W */ + { .compatible = "apple,j313-macaudio", .data = &macaudio_j313_cfg }, + /* j314 AID8 sn012776 15 2× 2W+1T */ + { .compatible = "apple,j314-macaudio", .data = &macaudio_j314_cfg }, + /* j316 AID9 sn012776 15 2× 2W+1T */ + { .compatible = "apple,j316-macaudio", .data = &macaudio_j316_cfg }, + /* j375 AID10 sn012776 15 1× 1W */ + { .compatible = "apple,j375-macaudio", .data = &macaudio_j37x_j47x_cfg }, + /* j413 AID13 sn012776 15 2× 1W+1T */ + { .compatible = "apple,j413-macaudio", .data = &macaudio_j413_cfg }, + /* j414 AID14 sn012776 15 2× 2W+1T Compat: apple,j314-macaudio */ + /* j415 AID27 sn012776 15 2× 2W+1T */ + { .compatible = "apple,j415-macaudio", .data = &macaudio_j415_cfg }, + /* j416 AID15 sn012776 15 2× 2W+1T Compat: apple,j316-macaudio */ + /* j456 AID5 ssm3515 15 2× 1W+1T */ + { .compatible = "apple,j456-macaudio", .data = &macaudio_j45x_cfg }, + /* j457 AID7 ssm3515 15 2× 1W+1T Compat: apple,j456-macaudio */ + /* j473 AID12 sn012776 20 1× 1W */ + { .compatible = "apple,j473-macaudio", .data = &macaudio_j37x_j47x_cfg }, + /* j474 AID26 sn012776 20 1× 1W Compat: apple,j473-macaudio */ + /* j475 AID25 sn012776 20 1× 1W Compat: apple,j375-macaudio */ + /* j493 AID18 sn012776 15 2× 2W */ + { .compatible = "apple,j493-macaudio", .data = &macaudio_j493_cfg }, + /* Fallback, jack only */ + { .compatible = "apple,macaudio"}, + { } +}; +MODULE_DEVICE_TABLE(of, macaudio_snd_device_id); + +static int macaudio_snd_platform_probe(struct platform_device *pdev) +{ + struct snd_soc_card *card; + struct macaudio_snd_data *data; + struct device *dev = &pdev->dev; + struct snd_soc_dai_link *link; + const struct of_device_id *of_id; + int ret; + int i; + + of_id = of_match_device(macaudio_snd_device_id, dev); + if (!of_id) + return -EINVAL; + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + card = &data->card; + snd_soc_card_set_drvdata(card, data); + dev_set_drvdata(&pdev->dev, data); + mutex_init(&data->volume_lock_mutex); + + card->owner = THIS_MODULE; + card->driver_name = "macaudio"; + card->dev = dev; + card->dapm_widgets = macaudio_snd_widgets; + card->num_dapm_widgets = ARRAY_SIZE(macaudio_snd_widgets); + card->dapm_routes = macaudio_dapm_routes; + card->num_dapm_routes = ARRAY_SIZE(macaudio_dapm_routes); + card->controls = macaudio_controls; + card->num_controls = ARRAY_SIZE(macaudio_controls); + card->probe = macaudio_probe; + card->late_probe = macaudio_late_probe; + card->component_chaining = true; + card->fully_routed = true; + + if (of_id->data) + data->cfg = of_id->data; + else + data->cfg = &macaudio_fallback_cfg; + + card->fixup_controls = macaudio_fixup_controls; + + ret = macaudio_parse_of(data); + if (ret) + return ret; + + /* Remove useless controls */ + if (!data->has_speakers) /* No speakers, remove both */ + card->num_controls -= MACAUDIO_NUM_SPEAKER_CONTROLS; + else if (!data->cfg->safe_vol) /* No safety, remove unlock */ + card->num_controls -= MACAUDIO_NUM_SPEAKER_LIMIT_CONTROLS; + else /* Speakers with safety, mark us as such */ + data->has_safety = true; + + for_each_card_prelinks(card, i, link) { + if (link->no_pcm) { + link->ops = &macaudio_be_ops; + link->init = macaudio_be_init; + link->exit = macaudio_be_exit; + } else { + link->ops = &macaudio_fe_ops; + link->init = macaudio_fe_init; + } + } + + INIT_WORK(&data->lock_update_work, macaudio_vlimit_update_work); + INIT_DELAYED_WORK(&data->lock_timeout_work, macaudio_vlimit_timeout_work); + + return devm_snd_soc_register_card(dev, card); +} + +static void macaudio_snd_platform_remove(struct platform_device *pdev) +{ + struct macaudio_snd_data *ma = dev_get_drvdata(&pdev->dev); + + cancel_delayed_work_sync(&ma->lock_timeout_work); +} + +static struct platform_driver macaudio_snd_driver = { + .probe = macaudio_snd_platform_probe, + .remove = macaudio_snd_platform_remove, + .driver = { + .name = DRIVER_NAME, + .of_match_table = macaudio_snd_device_id, + .pm = &snd_soc_pm_ops, + }, +}; +module_platform_driver(macaudio_snd_driver); + +MODULE_AUTHOR("Martin Povišer "); +MODULE_DESCRIPTION("Apple Silicon Macs machine-level sound driver"); +MODULE_LICENSE("GPL"); diff --git a/sound/soc/apple/mca.c b/sound/soc/apple/mca.c index c4dcb2b545912c..01dacd10bd39ce 100644 --- a/sound/soc/apple/mca.c +++ b/sound/soc/apple/mca.c @@ -133,12 +133,17 @@ struct mca_cluster { struct clk *clk_parent; struct dma_chan *dma_chans[SNDRV_PCM_STREAM_LAST + 1]; - bool port_started[SNDRV_PCM_STREAM_LAST + 1]; - int port_driver; /* The cluster driving this cluster's port */ + bool clk_provider; + + bool port_clk_started[SNDRV_PCM_STREAM_LAST + 1]; + int port_clk_driver; /* The cluster driving this cluster's port */ bool clocks_in_use[SNDRV_PCM_STREAM_LAST + 1]; struct device_link *pd_link; + /* In case of clock consumer FE */ + int syncgen_in_use; + unsigned int bclk_ratio; /* Masks etc. picked up via the set_tdm_slot method */ @@ -157,7 +162,7 @@ struct mca_data { struct reset_control *rstc; struct device_link *pd_link; - /* Mutex for accessing port_driver of foreign clusters */ + /* Mutex for accessing port_clk_driver of foreign clusters */ struct mutex port_mutex; int nclusters; @@ -211,15 +216,21 @@ static void mca_fe_early_trigger(struct snd_pcm_substream *substream, int cmd, SERDES_STATUS_RST); /* * Experiments suggest that it takes at most ~1 us - * for the bit to clear, so wait 2 us for good measure. + * for the bit to clear, so wait 5 us for good measure. */ - udelay(2); + udelay(50); WARN_ON(readl_relaxed(cl->base + serdes_unit + REG_SERDES_STATUS) & SERDES_STATUS_RST); mca_modify(cl, serdes_conf, SERDES_CONF_SYNC_SEL, FIELD_PREP(SERDES_CONF_SYNC_SEL, 0)); mca_modify(cl, serdes_conf, SERDES_CONF_SYNC_SEL, FIELD_PREP(SERDES_CONF_SYNC_SEL, cl->no + 1)); + /* + * ADMAC gets started right after this. This delay seems + * to be needed for that to be reliable, e.g. ensure the + * clock is stable? + */ + udelay(100); break; default: break; @@ -256,11 +267,28 @@ static int mca_fe_trigger(struct snd_pcm_substream *substream, int cmd, return 0; } +static int mca_fe_get_portmask(struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *fe = snd_soc_substream_to_rtd(substream); + struct snd_soc_dpcm *dpcm; + int mask = 0; + + for_each_dpcm_be(fe, substream->stream, dpcm) { + int no = mca_dai_to_cluster(snd_soc_rtd_to_cpu(dpcm->be, 0))->no; + mask |= 1 << no; + } + + return mask; +} + static int mca_fe_enable_clocks(struct mca_cluster *cl) { struct mca_data *mca = cl->host; int ret; + if (!cl->clk_provider) + return -EINVAL; + ret = clk_prepare_enable(cl->clk_parent); if (ret) { dev_err(mca->dev, @@ -274,6 +302,7 @@ static int mca_fe_enable_clocks(struct mca_cluster *cl) * the power state driver would error out on seeing the device * as clock-gated. */ + WARN_ON(cl->pd_link); cl->pd_link = device_link_add(mca->dev, cl->pd_dev, DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE); @@ -297,7 +326,11 @@ static void mca_fe_disable_clocks(struct mca_cluster *cl) mca_modify(cl, REG_SYNCGEN_STATUS, SYNCGEN_STATUS_EN, 0); mca_modify(cl, REG_STATUS, STATUS_MCLK_EN, 0); - device_link_del(cl->pd_link); + if (cl->pd_link) { + device_link_del(cl->pd_link); + cl->pd_link = NULL; + } + clk_disable_unprepare(cl->clk_parent); } @@ -311,7 +344,7 @@ static bool mca_fe_clocks_in_use(struct mca_cluster *cl) for (i = 0; i < mca->nclusters; i++) { be_cl = &mca->clusters[i]; - if (be_cl->port_driver != cl->no) + if (be_cl->port_clk_driver != cl->no) continue; for_each_pcm_streams(stream) { @@ -325,59 +358,55 @@ static bool mca_fe_clocks_in_use(struct mca_cluster *cl) return false; } -static int mca_be_prepare(struct snd_pcm_substream *substream, +static int mca_fe_prepare(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { struct mca_cluster *cl = mca_dai_to_cluster(dai); struct mca_data *mca = cl->host; - struct mca_cluster *fe_cl; - int ret; - if (cl->port_driver < 0) - return -EINVAL; + if (cl->clk_provider) + return 0; - fe_cl = &mca->clusters[cl->port_driver]; + if (!cl->syncgen_in_use) { + int port = ffs(mca_fe_get_portmask(substream)) - 1; - /* - * Typically the CODECs we are paired with will require clocks - * to be present at time of unmute with the 'mute_stream' op - * or at time of DAPM widget power-up. We need to enable clocks - * here at the latest (frontend prepare would be too late). - */ - if (!mca_fe_clocks_in_use(fe_cl)) { - ret = mca_fe_enable_clocks(fe_cl); - if (ret < 0) - return ret; - } + WARN_ON(cl->pd_link); + cl->pd_link = device_link_add(mca->dev, cl->pd_dev, + DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME | + DL_FLAG_RPM_ACTIVE); + if (!cl->pd_link) { + dev_err(mca->dev, + "cluster %d: unable to prop-up power domain\n", cl->no); + return -EINVAL; + } - cl->clocks_in_use[substream->stream] = true; + writel_relaxed(port + 6 + 1, + cl->base + REG_SYNCGEN_MCLK_SEL); + mca_modify(cl, REG_SYNCGEN_STATUS, SYNCGEN_STATUS_EN, + SYNCGEN_STATUS_EN); + } + cl->syncgen_in_use |= 1 << substream->stream; return 0; } -static int mca_be_hw_free(struct snd_pcm_substream *substream, +static int mca_fe_hw_free(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { struct mca_cluster *cl = mca_dai_to_cluster(dai); - struct mca_data *mca = cl->host; - struct mca_cluster *fe_cl; - - if (cl->port_driver < 0) - return -EINVAL; - /* - * We are operating on a foreign cluster here, but since we - * belong to the same PCM, accesses should have been - * synchronized at ASoC level. - */ - fe_cl = &mca->clusters[cl->port_driver]; - if (!mca_fe_clocks_in_use(fe_cl)) - return 0; /* Nothing to do */ + if (cl->clk_provider) + return 0; - cl->clocks_in_use[substream->stream] = false; + cl->syncgen_in_use &= ~(1 << substream->stream); + if (cl->syncgen_in_use) + return 0; - if (!mca_fe_clocks_in_use(fe_cl)) - mca_fe_disable_clocks(fe_cl); + mca_modify(cl, REG_SYNCGEN_STATUS, SYNCGEN_STATUS_EN, 0); + if (cl->pd_link) { + device_link_del(cl->pd_link); + cl->pd_link = NULL; + } return 0; } @@ -392,7 +421,7 @@ static unsigned int mca_crop_mask(unsigned int mask, int nchans) static int mca_configure_serdes(struct mca_cluster *cl, int serdes_unit, unsigned int mask, int slots, int nchans, - int slot_width, bool is_tx, int port) + int slot_width, bool is_tx, int portmask) { __iomem void *serdes_base = cl->base + serdes_unit; u32 serdes_conf, serdes_conf_mask; @@ -451,7 +480,7 @@ static int mca_configure_serdes(struct mca_cluster *cl, int serdes_unit, serdes_base + REG_RX_SERDES_SLOTMASK); writel_relaxed(~((u32)mca_crop_mask(mask, nchans)), serdes_base + REG_RX_SERDES_SLOTMASK + 0x4); - writel_relaxed(1 << port, + writel_relaxed(portmask, serdes_base + REG_RX_SERDES_PORT); } @@ -507,9 +536,18 @@ static int mca_fe_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) u32 serdes_conf = 0; u32 bitstart; - if ((fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) != - SND_SOC_DAIFMT_BP_FP) + switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { + case SND_SOC_DAIFMT_BP_FP: + cl->clk_provider = true; + break; + + case SND_SOC_DAIFMT_BC_FC: + cl->clk_provider = false; + break; + + default: goto err; + } switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) { case SND_SOC_DAIFMT_I2S: @@ -566,24 +604,6 @@ static int mca_set_bclk_ratio(struct snd_soc_dai *dai, unsigned int ratio) return 0; } -static int mca_fe_get_port(struct snd_pcm_substream *substream) -{ - struct snd_soc_pcm_runtime *fe = snd_soc_substream_to_rtd(substream); - struct snd_soc_pcm_runtime *be; - struct snd_soc_dpcm *dpcm; - - be = NULL; - for_each_dpcm_be(fe, substream->stream, dpcm) { - be = dpcm->be; - break; - } - - if (!be) - return -EINVAL; - - return mca_dai_to_cluster(snd_soc_rtd_to_cpu(be, 0))->no; -} - static int mca_fe_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params, struct snd_soc_dai *dai) @@ -597,7 +617,7 @@ static int mca_fe_hw_params(struct snd_pcm_substream *substream, unsigned long bclk_ratio; unsigned int tdm_slots, tdm_slot_width, tdm_mask; u32 regval, pad; - int ret, port, nchans_ceiled; + int ret, portmask, nchans_ceiled; if (!cl->tdm_slot_width) { /* @@ -646,13 +666,13 @@ static int mca_fe_hw_params(struct snd_pcm_substream *substream, tdm_mask = (1 << tdm_slots) - 1; } - port = mca_fe_get_port(substream); - if (port < 0) - return port; + portmask = mca_fe_get_portmask(substream); + if (!portmask) + return -EINVAL; ret = mca_configure_serdes(cl, is_tx ? CLUSTER_TX_OFF : CLUSTER_RX_OFF, tdm_mask, tdm_slots, params_channels(params), - tdm_slot_width, is_tx, port); + tdm_slot_width, is_tx, portmask); if (ret) return ret; @@ -708,67 +728,123 @@ static const struct snd_soc_dai_ops mca_fe_ops = { .set_tdm_slot = mca_fe_set_tdm_slot, .hw_params = mca_fe_hw_params, .trigger = mca_fe_trigger, + .prepare = mca_fe_prepare, + .hw_free = mca_fe_hw_free, }; -static bool mca_be_started(struct mca_cluster *cl) +/* + * Is there a FE attached which will be feeding this port's clocks? + */ +static bool mca_be_clk_started(struct mca_cluster *cl) { int stream; for_each_pcm_streams(stream) - if (cl->port_started[stream]) + if (cl->port_clk_started[stream]) return true; return false; } -static int mca_be_startup(struct snd_pcm_substream *substream, +static struct snd_soc_pcm_runtime *mca_be_get_fe(struct snd_soc_pcm_runtime *be, + int stream) +{ + struct snd_soc_pcm_runtime *fe = NULL; + struct snd_soc_dpcm *dpcm; + + for_each_dpcm_fe(be, stream, dpcm) { + if (fe && dpcm->fe != fe) { + dev_err(be->dev, "many FE per one BE unsupported\n"); + return NULL; + } + + fe = dpcm->fe; + } + + return fe; +} + +static int mca_be_prepare(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { struct snd_soc_pcm_runtime *be = snd_soc_substream_to_rtd(substream); - struct snd_soc_pcm_runtime *fe; + struct snd_soc_pcm_runtime *fe = mca_be_get_fe(be, substream->stream); struct mca_cluster *cl = mca_dai_to_cluster(dai); - struct mca_cluster *fe_cl; struct mca_data *mca = cl->host; - struct snd_soc_dpcm *dpcm; + struct mca_cluster *fe_cl, *fe_clk_cl; + int ret; - fe = NULL; + fe_cl = mca_dai_to_cluster(snd_soc_rtd_to_cpu(fe, 0)); - for_each_dpcm_fe(be, substream->stream, dpcm) { - if (fe && dpcm->fe != fe) { - dev_err(mca->dev, "many FE per one BE unsupported\n"); - return -EINVAL; - } + if (!fe_cl->clk_provider) + return 0; - fe = dpcm->fe; + if (cl->port_clk_driver < 0) + return 0; + + fe_clk_cl = &mca->clusters[cl->port_clk_driver]; + + /* + * Typically the CODECs we are paired with will require clocks + * to be present at time of unmute with the 'mute_stream' op + * or at time of DAPM widget power-up. We need to enable clocks + * here at the latest (frontend prepare would be too late). + */ + if (!mca_fe_clocks_in_use(fe_clk_cl)) { + ret = mca_fe_enable_clocks(fe_clk_cl); + if (ret < 0) + return ret; } + cl->clocks_in_use[substream->stream] = true; + + return 0; +} + +static int mca_be_startup(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct snd_soc_pcm_runtime *be = snd_soc_substream_to_rtd(substream); + struct snd_soc_pcm_runtime *fe = mca_be_get_fe(be, substream->stream); + struct mca_cluster *cl = mca_dai_to_cluster(dai); + struct mca_cluster *fe_cl; + struct mca_data *mca = cl->host; + if (!fe) return -EINVAL; - fe_cl = mca_dai_to_cluster(snd_soc_rtd_to_cpu(fe, 0)); - if (mca_be_started(cl)) { + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { + writel_relaxed(PORT_DATA_SEL_TXA(fe_cl->no), + cl->base + REG_PORT_DATA_SEL); + mca_modify(cl, REG_PORT_ENABLES, PORT_ENABLES_TX_DATA, + PORT_ENABLES_TX_DATA); + } + + if (!fe_cl->clk_provider) + return 0; + + if (mca_be_clk_started(cl)) { /* * Port is already started in the other direction. * Make sure there isn't a conflict with another cluster - * driving the port. + * driving the port clocks. */ - if (cl->port_driver != fe_cl->no) + if (cl->port_clk_driver != fe_cl->no) return -EINVAL; - cl->port_started[substream->stream] = true; + cl->port_clk_started[substream->stream] = true; return 0; } - writel_relaxed(PORT_ENABLES_CLOCKS | PORT_ENABLES_TX_DATA, - cl->base + REG_PORT_ENABLES); writel_relaxed(FIELD_PREP(PORT_CLOCK_SEL, fe_cl->no + 1), cl->base + REG_PORT_CLOCK_SEL); - writel_relaxed(PORT_DATA_SEL_TXA(fe_cl->no), - cl->base + REG_PORT_DATA_SEL); + mca_modify(cl, REG_PORT_ENABLES, PORT_ENABLES_CLOCKS, + PORT_ENABLES_CLOCKS); + mutex_lock(&mca->port_mutex); - cl->port_driver = fe_cl->no; + cl->port_clk_driver = fe_cl->no; mutex_unlock(&mca->port_mutex); - cl->port_started[substream->stream] = true; + cl->port_clk_started[substream->stream] = true; return 0; } @@ -776,27 +852,60 @@ static int mca_be_startup(struct snd_pcm_substream *substream, static void mca_be_shutdown(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { + struct snd_soc_pcm_runtime *be = snd_soc_substream_to_rtd(substream); + struct snd_soc_pcm_runtime *fe = mca_be_get_fe(be, substream->stream); struct mca_cluster *cl = mca_dai_to_cluster(dai); + struct mca_cluster *fe_cl; struct mca_data *mca = cl->host; - cl->port_started[substream->stream] = false; + if (cl->clocks_in_use[substream->stream] && + !WARN_ON(cl->port_clk_driver < 0)) { + struct mca_cluster *fe_cl = &mca->clusters[cl->port_clk_driver]; - if (!mca_be_started(cl)) { /* - * Were we the last direction to shutdown? - * Turn off the lights. + * Typically the CODECs we are paired with will require clocks + * to be present at time of mute with the 'mute_stream' op. + * We need to disable the clocks here at the earliest (hw_free + * would be too early). + * + * We are operating on a foreign cluster here, but since we + * belong to the same PCM, accesses should have been + * synchronized at ASoC level. */ - writel_relaxed(0, cl->base + REG_PORT_ENABLES); + cl->clocks_in_use[substream->stream] = false; + + if (!mca_fe_clocks_in_use(fe_cl)) + mca_fe_disable_clocks(fe_cl); + } + + if (!fe) + return; + fe_cl = mca_dai_to_cluster(snd_soc_rtd_to_cpu(fe, 0)); + + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { + mca_modify(cl, REG_PORT_ENABLES, PORT_ENABLES_TX_DATA, 0); writel_relaxed(0, cl->base + REG_PORT_DATA_SEL); + } + + if (!fe_cl->clk_provider) + return; + + cl->port_clk_started[substream->stream] = false; + if (!mca_be_clk_started(cl)) { + /* + * Were we the last direction to shutdown? + * Turn off the lights (clocks). + */ + mca_modify(cl, REG_PORT_ENABLES, PORT_ENABLES_CLOCKS, 0); + writel_relaxed(0, cl->base + REG_PORT_CLOCK_SEL); mutex_lock(&mca->port_mutex); - cl->port_driver = -1; + cl->port_clk_driver = -1; mutex_unlock(&mca->port_mutex); } } static const struct snd_soc_dai_ops mca_be_ops = { .prepare = mca_be_prepare, - .hw_free = mca_be_hw_free, .startup = mca_be_startup, .shutdown = mca_be_shutdown, }; @@ -1020,8 +1129,10 @@ static void apple_mca_release(struct mca_data *mca) dev_pm_domain_detach(cl->pd_dev, true); } - if (mca->pd_link) + if (mca->pd_link) { device_link_del(mca->pd_link); + mca->pd_link = NULL; + } if (!IS_ERR_OR_NULL(mca->pd_dev)) dev_pm_domain_detach(mca->pd_dev, true); @@ -1096,7 +1207,7 @@ static int apple_mca_probe(struct platform_device *pdev) cl->host = mca; cl->no = i; cl->base = base + CLUSTER_STRIDE * i; - cl->port_driver = -1; + cl->port_clk_driver = -1; cl->clk_parent = of_clk_get(pdev->dev.of_node, i); if (IS_ERR(cl->clk_parent)) { dev_err(&pdev->dev, "unable to obtain clock %d: %ld\n", diff --git a/sound/soc/codecs/aw88261.c b/sound/soc/codecs/aw88261.c index 8f37bfb974ae48..96b1a85c26aa72 100644 --- a/sound/soc/codecs/aw88261.c +++ b/sound/soc/codecs/aw88261.c @@ -423,9 +423,10 @@ static int aw88261_dev_reg_update(struct aw88261 *aw88261, if (ret) break; + /* keep all three bits from current hw status */ read_val &= (~AW88261_AMPPD_MASK) | (~AW88261_PWDN_MASK) | (~AW88261_HMUTE_MASK); - reg_val &= (AW88261_AMPPD_MASK | AW88261_PWDN_MASK | AW88261_HMUTE_MASK); + reg_val &= (AW88261_AMPPD_MASK & AW88261_PWDN_MASK & AW88261_HMUTE_MASK); reg_val |= read_val; /* enable uls hmute */ diff --git a/sound/soc/codecs/cs4271.c b/sound/soc/codecs/cs4271.c index 77dfc83a3c0131..d8cdd37e9112b2 100644 --- a/sound/soc/codecs/cs4271.c +++ b/sound/soc/codecs/cs4271.c @@ -528,7 +528,7 @@ static int cs4271_soc_resume(struct snd_soc_component *component) ret = clk_prepare_enable(cs4271->clk); if (ret) { dev_err(component->dev, "Failed to enable clk: %d\n", ret); - return ret; + goto err_disable_regulators; } /* Do a proper reset after power up */ @@ -537,15 +537,21 @@ static int cs4271_soc_resume(struct snd_soc_component *component) /* Restore codec state */ ret = regcache_sync(cs4271->regmap); if (ret < 0) - return ret; + goto err_disable_clk; /* then disable the power-down bit */ ret = regmap_update_bits(cs4271->regmap, CS4271_MODE2, CS4271_MODE2_PDN, 0); if (ret < 0) - return ret; + goto err_disable_clk; return 0; + +err_disable_clk: + clk_disable_unprepare(cs4271->clk); +err_disable_regulators: + regulator_bulk_disable(ARRAY_SIZE(cs4271->supplies), cs4271->supplies); + return ret; } #else #define cs4271_soc_suspend NULL diff --git a/sound/soc/codecs/cs42l42.c b/sound/soc/codecs/cs42l42.c index 2652a639a79ad2..1f2efce071912d 100644 --- a/sound/soc/codecs/cs42l42.c +++ b/sound/soc/codecs/cs42l42.c @@ -1148,7 +1148,6 @@ struct snd_soc_dai_driver cs42l42_dai = { .formats = CS42L42_FORMATS, }, .symmetric_rate = 1, - .symmetric_sample_bits = 1, .ops = &cs42l42_ops, }; EXPORT_SYMBOL_NS_GPL(cs42l42_dai, "SND_SOC_CS42L42_CORE"); @@ -1676,7 +1675,7 @@ irqreturn_t cs42l42_irq_thread(int irq, void *data) return IRQ_NONE; } - /* Read sticky registers to clear interurpt */ + /* Read sticky registers to clear interrupt */ for (i = 0; i < ARRAY_SIZE(stickies); i++) { regmap_read(cs42l42->regmap, irq_params_table[i].status_addr, &(stickies[i])); @@ -2419,6 +2418,16 @@ int cs42l42_init(struct cs42l42_private *cs42l42) (1 << CS42L42_ADC_PDN_SHIFT) | (0 << CS42L42_PDN_ALL_SHIFT)); + /* + * Configure a faster digital ramp time, to avoid an audible + * fade-in when streams start. + */ + regmap_update_bits(cs42l42->regmap, CS42L42_SFTRAMP_RATE, + CS42L42_SFTRAMP_ASR_RATE_MASK | + CS42L42_SFTRAMP_DSR_RATE_MASK, + (10 << CS42L42_SFTRAMP_ASR_RATE_SHIFT) | + (1 << CS42L42_SFTRAMP_DSR_RATE_SHIFT)); + ret = cs42l42_handle_device_data(cs42l42->dev, cs42l42); if (ret != 0) goto err_shutdown; diff --git a/sound/soc/codecs/nau8821.c b/sound/soc/codecs/nau8821.c index 3beb3c44dc2c03..58d2d5e77c8f44 100644 --- a/sound/soc/codecs/nau8821.c +++ b/sound/soc/codecs/nau8821.c @@ -1264,6 +1264,14 @@ static int nau8821_component_probe(struct snd_soc_component *component) return 0; } +static void nau8821_component_remove(struct snd_soc_component *component) +{ + struct nau8821 *nau8821 = snd_soc_component_get_drvdata(component); + + if (nau8821->jdet_active) + cancel_delayed_work_sync(&nau8821->jdet_work); +}; + /** * nau8821_calc_fll_param - Calculate FLL parameters. * @fll_in: external clock provided to codec. @@ -1597,6 +1605,10 @@ static int __maybe_unused nau8821_suspend(struct snd_soc_component *component) if (nau8821->irq) disable_irq(nau8821->irq); + + if (nau8821->jdet_active) + cancel_delayed_work_sync(&nau8821->jdet_work); + snd_soc_dapm_force_bias_level(nau8821->dapm, SND_SOC_BIAS_OFF); /* Power down codec power; don't support button wakeup */ snd_soc_dapm_disable_pin(nau8821->dapm, "MICBIAS"); @@ -1621,6 +1633,7 @@ static int __maybe_unused nau8821_resume(struct snd_soc_component *component) static const struct snd_soc_component_driver nau8821_component_driver = { .probe = nau8821_component_probe, + .remove = nau8821_component_remove, .set_sysclk = nau8821_set_sysclk, .set_pll = nau8821_set_fll, .set_bias_level = nau8821_set_bias_level, @@ -1655,8 +1668,13 @@ int nau8821_enable_jack_detect(struct snd_soc_component *component, int ret; nau8821->jack = jack; + + if (nau8821->jdet_active) + return 0; + /* Initiate jack detection work queue */ INIT_DELAYED_WORK(&nau8821->jdet_work, nau8821_jdet_work); + nau8821->jdet_active = true; ret = devm_request_threaded_irq(nau8821->dev, nau8821->irq, NULL, nau8821_interrupt, IRQF_TRIGGER_LOW | IRQF_ONESHOT, diff --git a/sound/soc/codecs/nau8821.h b/sound/soc/codecs/nau8821.h index 88602923780d85..f9d7cd8cbd2116 100644 --- a/sound/soc/codecs/nau8821.h +++ b/sound/soc/codecs/nau8821.h @@ -562,6 +562,7 @@ struct nau8821 { struct snd_soc_dapm_context *dapm; struct snd_soc_jack *jack; struct delayed_work jdet_work; + bool jdet_active; int irq; int clk_id; int micbias_voltage; diff --git a/sound/soc/codecs/tas2764.c b/sound/soc/codecs/tas2764.c index 36e25e48b35463..a458458128aa89 100644 --- a/sound/soc/codecs/tas2764.c +++ b/sound/soc/codecs/tas2764.c @@ -34,6 +34,7 @@ struct tas2764_priv { struct snd_soc_component *component; struct gpio_desc *reset_gpio; struct gpio_desc *sdz_gpio; + struct regulator *sdz_reg; struct regmap *regmap; struct device *dev; int irq; @@ -41,6 +42,7 @@ struct tas2764_priv { int v_sense_slot; int i_sense_slot; + u32 sdout_zero_mask; bool dac_powered; bool unmuted; @@ -152,6 +154,8 @@ static int tas2764_codec_suspend(struct snd_soc_component *component) if (tas2764->sdz_gpio) gpiod_set_value_cansleep(tas2764->sdz_gpio, 0); + regulator_disable(tas2764->sdz_reg); + regcache_cache_only(tas2764->regmap, true); regcache_mark_dirty(tas2764->regmap); @@ -165,19 +169,26 @@ static int tas2764_codec_resume(struct snd_soc_component *component) struct tas2764_priv *tas2764 = snd_soc_component_get_drvdata(component); int ret; + ret = regulator_enable(tas2764->sdz_reg); + + if (ret) { + dev_err(tas2764->dev, "Failed to enable regulator\n"); + return ret; + } + if (tas2764->sdz_gpio) { gpiod_set_value_cansleep(tas2764->sdz_gpio, 1); - usleep_range(1000, 2000); } - ret = tas2764_update_pwr_ctrl(tas2764); + usleep_range(1000, 2000); + regcache_cache_only(tas2764->regmap, false); + + ret = regcache_sync(tas2764->regmap); if (ret < 0) return ret; - regcache_cache_only(tas2764->regmap, false); - - return regcache_sync(tas2764->regmap); + return tas2764_update_pwr_ctrl(tas2764); } #else #define tas2764_codec_suspend NULL @@ -210,7 +221,7 @@ static const struct snd_soc_dapm_widget tas2764_dapm_widgets[] = { SND_SOC_DAPM_DAC("DAC", NULL, SND_SOC_NOPM, 0, 0), SND_SOC_DAPM_OUTPUT("OUT"), SND_SOC_DAPM_SIGGEN("VMON"), - SND_SOC_DAPM_SIGGEN("IMON") + SND_SOC_DAPM_SIGGEN("IMON"), }; static const struct snd_soc_dapm_route tas2764_audio_map[] = { @@ -261,7 +272,6 @@ static int tas2764_mute(struct snd_soc_dai *dai, int mute, int direction) static int tas2764_set_bitwidth(struct tas2764_priv *tas2764, int bitwidth) { struct snd_soc_component *component = tas2764->component; - int sense_en; int val; int ret; @@ -296,28 +306,6 @@ static int tas2764_set_bitwidth(struct tas2764_priv *tas2764, int bitwidth) if (val < 0) return val; - if (val & (1 << TAS2764_VSENSE_POWER_EN)) - sense_en = 0; - else - sense_en = TAS2764_TDM_CFG5_VSNS_ENABLE; - - ret = snd_soc_component_update_bits(tas2764->component, TAS2764_TDM_CFG5, - TAS2764_TDM_CFG5_VSNS_ENABLE, - sense_en); - if (ret < 0) - return ret; - - if (val & (1 << TAS2764_ISENSE_POWER_EN)) - sense_en = 0; - else - sense_en = TAS2764_TDM_CFG6_ISNS_ENABLE; - - ret = snd_soc_component_update_bits(tas2764->component, TAS2764_TDM_CFG6, - TAS2764_TDM_CFG6_ISNS_ENABLE, - sense_en); - if (ret < 0) - return ret; - return 0; } @@ -373,6 +361,44 @@ static int tas2764_hw_params(struct snd_pcm_substream *substream, return tas2764_set_samplerate(tas2764, params_rate(params)); } +static int tas2764_write_sdout_zero_mask(struct tas2764_priv *tas2764, int bclk_ratio) +{ + struct snd_soc_component *component = tas2764->component; + int nsense_slots = bclk_ratio / 8; + u32 cropped_mask; + int i, ret; + + if (!tas2764->sdout_zero_mask) + return 0; + + cropped_mask = tas2764->sdout_zero_mask & GENMASK(nsense_slots - 1, 0); + + for (i = 0; i < 4; i++) { + ret = snd_soc_component_write(component, TAS2764_SDOUT_HIZ_1 + i, + (cropped_mask >> (i * 8)) & 0xff); + + if (ret < 0) + return ret; + } + + ret = snd_soc_component_update_bits(component, TAS2764_SDOUT_HIZ_9, + TAS2764_SDOUT_HIZ_9_FORCE_0_EN, + TAS2764_SDOUT_HIZ_9_FORCE_0_EN); + + if (ret < 0) + return ret; + + return 0; +} + +static int tas2764_set_bclk_ratio(struct snd_soc_dai *dai, unsigned int ratio) +{ + struct snd_soc_component *component = dai->component; + struct tas2764_priv *tas2764 = snd_soc_component_get_drvdata(component); + + return tas2764_write_sdout_zero_mask(tas2764, ratio); +} + static int tas2764_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) { struct snd_soc_component *component = dai->component; @@ -447,7 +473,6 @@ static int tas2764_set_dai_tdm_slot(struct snd_soc_dai *dai, int slots, int slot_width) { struct snd_soc_component *component = dai->component; - struct tas2764_priv *tas2764 = snd_soc_component_get_drvdata(component); int left_slot, right_slot; int slots_cfg; int slot_size; @@ -494,15 +519,26 @@ static int tas2764_set_dai_tdm_slot(struct snd_soc_dai *dai, if (ret < 0) return ret; - ret = snd_soc_component_update_bits(component, TAS2764_TDM_CFG5, + return 0; +} + +static int tas2764_set_ivsense_transmit(struct tas2764_priv *tas2764, int i_slot, int v_slot) +{ + int ret; + + ret = snd_soc_component_update_bits(tas2764->component, TAS2764_TDM_CFG5, + TAS2764_TDM_CFG5_VSNS_ENABLE | TAS2764_TDM_CFG5_50_MASK, - tas2764->v_sense_slot); + TAS2764_TDM_CFG5_VSNS_ENABLE | + v_slot); if (ret < 0) return ret; - ret = snd_soc_component_update_bits(component, TAS2764_TDM_CFG6, + ret = snd_soc_component_update_bits(tas2764->component, TAS2764_TDM_CFG6, + TAS2764_TDM_CFG6_ISNS_ENABLE | TAS2764_TDM_CFG6_50_MASK, - tas2764->i_sense_slot); + TAS2764_TDM_CFG6_ISNS_ENABLE | + i_slot); if (ret < 0) return ret; @@ -512,6 +548,7 @@ static int tas2764_set_dai_tdm_slot(struct snd_soc_dai *dai, static const struct snd_soc_dai_ops tas2764_dai_ops = { .mute_stream = tas2764_mute, .hw_params = tas2764_hw_params, + .set_bclk_ratio = tas2764_set_bclk_ratio, .set_fmt = tas2764_set_fmt, .set_tdm_slot = tas2764_set_dai_tdm_slot, .no_capture_mute = 1, @@ -659,11 +696,18 @@ static int tas2764_codec_probe(struct snd_soc_component *component) tas2764->component = component; + ret = regulator_enable(tas2764->sdz_reg); + if (ret != 0) { + dev_err(tas2764->dev, "Failed to enable regulator: %d\n", ret); + return ret; + } + if (tas2764->sdz_gpio) { gpiod_set_value_cansleep(tas2764->sdz_gpio, 1); - usleep_range(1000, 2000); } + usleep_range(1000, 2000); + tas2764_reset(tas2764); regmap_reinit_cache(tas2764->regmap, &tas2764_i2c_regmap); @@ -695,18 +739,33 @@ static int tas2764_codec_probe(struct snd_soc_component *component) dev_warn(tas2764->dev, "failed to request IRQ: %d\n", ret); } - ret = snd_soc_component_update_bits(tas2764->component, TAS2764_TDM_CFG5, - TAS2764_TDM_CFG5_VSNS_ENABLE, 0); - if (ret < 0) - return ret; + if (tas2764->i_sense_slot != -1 && tas2764->v_sense_slot != -1) { + ret = tas2764_set_ivsense_transmit(tas2764, tas2764->i_sense_slot, + tas2764->v_sense_slot); - ret = snd_soc_component_update_bits(tas2764->component, TAS2764_TDM_CFG6, - TAS2764_TDM_CFG6_ISNS_ENABLE, 0); - if (ret < 0) - return ret; + if (ret < 0) + return ret; + } switch (tas2764->devid) { case DEVID_SN012776: + if (tas2764->sdout_zero_mask) { + for (i = 0; i < 4; i++) { + ret = snd_soc_component_write(component, TAS2764_SDOUT_HIZ_1 + i, + (tas2764->sdout_zero_mask >> (i * 8)) & 0xff); + + if (ret < 0) + return ret; + } + + ret = snd_soc_component_update_bits(component, TAS2764_SDOUT_HIZ_9, + TAS2764_SDOUT_HIZ_9_FORCE_0_EN, + TAS2764_SDOUT_HIZ_9_FORCE_0_EN); + + if (ret < 0) + return ret; + } + ret = snd_soc_component_update_bits(component, TAS2764_PWR_CTRL, TAS2764_PWR_CTRL_BOP_SRC, TAS2764_PWR_CTRL_BOP_SRC); @@ -736,6 +795,13 @@ static int tas2764_codec_probe(struct snd_soc_component *component) return 0; } +static void tas2764_codec_remove(struct snd_soc_component *component) +{ + struct tas2764_priv *tas2764 = snd_soc_component_get_drvdata(component); + + regulator_disable(tas2764->sdz_reg); +} + static DECLARE_TLV_DB_SCALE(tas2764_digital_tlv, 1100, 50, 0); static DECLARE_TLV_DB_SCALE(tas2764_playback_volume, -10050, 50, 1); @@ -767,6 +833,7 @@ static const struct snd_kcontrol_new tas2764_snd_controls[] = { static const struct snd_soc_component_driver soc_component_driver_tas2764 = { .probe = tas2764_codec_probe, + .remove = tas2764_codec_remove, .suspend = tas2764_codec_suspend, .resume = tas2764_codec_resume, .controls = tas2764_snd_controls, @@ -836,6 +903,11 @@ static int tas2764_parse_dt(struct device *dev, struct tas2764_priv *tas2764) { int ret = 0; + tas2764->sdz_reg = devm_regulator_get(dev, "SDZ"); + if (IS_ERR(tas2764->sdz_reg)) + return dev_err_probe(dev, PTR_ERR(tas2764->sdz_reg), + "Failed to get SDZ supply\n"); + tas2764->reset_gpio = devm_gpiod_get_optional(tas2764->dev, "reset", GPIOD_OUT_HIGH); if (IS_ERR(tas2764->reset_gpio)) { @@ -856,12 +928,17 @@ static int tas2764_parse_dt(struct device *dev, struct tas2764_priv *tas2764) ret = fwnode_property_read_u32(dev->fwnode, "ti,imon-slot-no", &tas2764->i_sense_slot); if (ret) - tas2764->i_sense_slot = 0; + tas2764->i_sense_slot = -1; ret = fwnode_property_read_u32(dev->fwnode, "ti,vmon-slot-no", &tas2764->v_sense_slot); if (ret) - tas2764->v_sense_slot = 2; + tas2764->v_sense_slot = -1; + + ret = fwnode_property_read_u32(dev->fwnode, "ti,sdout-force-zero-mask", + &tas2764->sdout_zero_mask); + if (ret) + tas2764->sdout_zero_mask = 0; return 0; } diff --git a/sound/soc/codecs/tas2764.h b/sound/soc/codecs/tas2764.h index 538290ed3d92ac..4a419c11d4b08e 100644 --- a/sound/soc/codecs/tas2764.h +++ b/sound/soc/codecs/tas2764.h @@ -126,4 +126,15 @@ #define TAS2764_BOP_CFG0 TAS2764_REG(0X0, 0x1d) +#define TAS2764_SDOUT_HIZ_1 TAS2764_REG(0x1, 0x3d) +#define TAS2764_SDOUT_HIZ_2 TAS2764_REG(0x1, 0x3e) +#define TAS2764_SDOUT_HIZ_3 TAS2764_REG(0x1, 0x3f) +#define TAS2764_SDOUT_HIZ_4 TAS2764_REG(0x1, 0x40) +#define TAS2764_SDOUT_HIZ_5 TAS2764_REG(0x1, 0x41) +#define TAS2764_SDOUT_HIZ_6 TAS2764_REG(0x1, 0x42) +#define TAS2764_SDOUT_HIZ_7 TAS2764_REG(0x1, 0x43) +#define TAS2764_SDOUT_HIZ_8 TAS2764_REG(0x1, 0x44) +#define TAS2764_SDOUT_HIZ_9 TAS2764_REG(0x1, 0x45) +#define TAS2764_SDOUT_HIZ_9_FORCE_0_EN BIT(7) + #endif /* __TAS2764__ */ diff --git a/sound/soc/codecs/tas2770.c b/sound/soc/codecs/tas2770.c index 6f878b01716f72..e72027cf340bfe 100644 --- a/sound/soc/codecs/tas2770.c +++ b/sound/soc/codecs/tas2770.c @@ -71,23 +71,21 @@ static int tas2770_codec_suspend(struct snd_soc_component *component) struct tas2770_priv *tas2770 = snd_soc_component_get_drvdata(component); int ret = 0; - regcache_cache_only(tas2770->regmap, true); - regcache_mark_dirty(tas2770->regmap); + ret = snd_soc_component_update_bits(component, TAS2770_PWR_CTRL, + TAS2770_PWR_CTRL_MASK, + TAS2770_PWR_CTRL_SHUTDOWN); + if (ret < 0) + return ret; - if (tas2770->sdz_gpio) { + if (tas2770->sdz_gpio) gpiod_set_value_cansleep(tas2770->sdz_gpio, 0); - } else { - ret = snd_soc_component_update_bits(component, TAS2770_PWR_CTRL, - TAS2770_PWR_CTRL_MASK, - TAS2770_PWR_CTRL_SHUTDOWN); - if (ret < 0) { - regcache_cache_only(tas2770->regmap, false); - regcache_sync(tas2770->regmap); - return ret; - } - ret = 0; - } + regulator_disable(tas2770->sdz_reg); + + regcache_cache_only(tas2770->regmap, true); + regcache_mark_dirty(tas2770->regmap); + + usleep_range(6000, 7000); return ret; } @@ -97,18 +95,26 @@ static int tas2770_codec_resume(struct snd_soc_component *component) struct tas2770_priv *tas2770 = snd_soc_component_get_drvdata(component); int ret; - if (tas2770->sdz_gpio) { - gpiod_set_value_cansleep(tas2770->sdz_gpio, 1); - usleep_range(1000, 2000); - } else { - ret = tas2770_update_pwr_ctrl(tas2770); - if (ret < 0) - return ret; + ret = regulator_enable(tas2770->sdz_reg); + + if (ret) { + dev_err(tas2770->dev, "Failed to enable regulator\n"); + return ret; } + if (tas2770->sdz_gpio) + gpiod_set_value_cansleep(tas2770->sdz_gpio, 1); + + + usleep_range(1000, 2000); + regcache_cache_only(tas2770->regmap, false); - return regcache_sync(tas2770->regmap); + ret = regcache_sync(tas2770->regmap); + if (ret < 0) + return ret; + + return tas2770_update_pwr_ctrl(tas2770); } #else #define tas2770_codec_suspend NULL @@ -623,11 +629,18 @@ static int tas2770_codec_probe(struct snd_soc_component *component) tas2770->component = component; + ret = regulator_enable(tas2770->sdz_reg); + if (ret != 0) { + dev_err(tas2770->dev, "Failed to enable regulator: %d\n", ret); + return ret; + } + if (tas2770->sdz_gpio) { gpiod_set_value_cansleep(tas2770->sdz_gpio, 1); - usleep_range(1000, 2000); } + usleep_range(1000, 2000); + tas2770_reset(tas2770); regmap_reinit_cache(tas2770->regmap, &tas2770_i2c_regmap); @@ -641,14 +654,34 @@ static int tas2770_codec_probe(struct snd_soc_component *component) if (tas2770->pdm_slot != -1) { ret = tas2770_set_pdm_transmit(tas2770, tas2770->pdm_slot); - if (ret < 0) return ret; } + ret = snd_soc_component_update_bits(component, TAS2770_TDM_CFG_REG4, + TAS2770_TDM_CFG_REG4_TX_FILL, + tas2770->sdout_zfill ? 0 : + TAS2770_TDM_CFG_REG4_TX_FILL); + if (ret < 0) + return ret; + + ret = snd_soc_component_update_bits(component, TAS2770_DIN_PD, + TAS2770_DIN_PD_SDOUT, + tas2770->sdout_pd ? + TAS2770_DIN_PD_SDOUT : 0); + if (ret < 0) + return ret; + return 0; } +static void tas2770_codec_remove(struct snd_soc_component *component) +{ + struct tas2770_priv *tas2770 = snd_soc_component_get_drvdata(component); + + regulator_disable(tas2770->sdz_reg); +} + static DECLARE_TLV_DB_SCALE(tas2770_digital_tlv, 1100, 50, 0); static DECLARE_TLV_DB_SCALE(tas2770_playback_volume, -10050, 50, 0); @@ -661,6 +694,7 @@ static const struct snd_kcontrol_new tas2770_snd_controls[] = { static const struct snd_soc_component_driver soc_component_driver_tas2770 = { .probe = tas2770_codec_probe, + .remove = tas2770_codec_remove, .suspend = tas2770_codec_suspend, .resume = tas2770_codec_resume, .controls = tas2770_snd_controls, @@ -790,6 +824,14 @@ static int tas2770_parse_dt(struct device *dev, struct tas2770_priv *tas2770) if (rc) tas2770->pdm_slot = -1; + tas2770->sdout_pd = fwnode_property_read_bool(dev->fwnode, "ti,sdout-pull-down"); + tas2770->sdout_zfill = fwnode_property_read_bool(dev->fwnode, "ti,sdout-zero-fill"); + + tas2770->sdz_reg = devm_regulator_get(dev, "SDZ"); + if (IS_ERR(tas2770->sdz_reg)) + return dev_err_probe(dev, PTR_ERR(tas2770->sdz_reg), + "Failed to get SDZ supply\n"); + tas2770->sdz_gpio = devm_gpiod_get_optional(dev, "shutdown", GPIOD_OUT_HIGH); if (IS_ERR(tas2770->sdz_gpio)) { if (PTR_ERR(tas2770->sdz_gpio) == -EPROBE_DEFER) diff --git a/sound/soc/codecs/tas2770.h b/sound/soc/codecs/tas2770.h index 3fd2e7003c50b6..b309d19c58e1da 100644 --- a/sound/soc/codecs/tas2770.h +++ b/sound/soc/codecs/tas2770.h @@ -67,6 +67,14 @@ #define TAS2770_TDM_CFG_REG3_RXS_SHIFT 0x4 #define TAS2770_TDM_CFG_REG3_30_MASK GENMASK(3, 0) #define TAS2770_TDM_CFG_REG3_30_SHIFT 0 + /* TDM Configuration Reg4 */ +#define TAS2770_TDM_CFG_REG4 TAS2770_REG(0X0, 0x0E) +#define TAS2770_TDM_CFG_REG4_TX_LSB_CFG BIT(7) +#define TAS2770_TDM_CFG_REG4_TX_KEEPER_CFG BIT(6) +#define TAS2770_TDM_CFG_REG4_TX_KEEPER BIT(5) +#define TAS2770_TDM_CFG_REG4_TX_FILL BIT(4) +#define TAS2770_TDM_CFG_REG4_TX_OFFSET_MASK GENMASK(3, 1) +#define TAS2770_TDM_CFG_REG4_TX_EDGE_FALLING BIT(0) /* TDM Configuration Reg5 */ #define TAS2770_TDM_CFG_REG5 TAS2770_REG(0X0, 0x0F) #define TAS2770_TDM_CFG_REG5_VSNS_MASK BIT(6) @@ -115,6 +123,9 @@ #define TAS2770_TEMP_LSB TAS2770_REG(0X0, 0x2A) /* Interrupt Configuration */ #define TAS2770_INT_CFG TAS2770_REG(0X0, 0x30) + /* Data In Pull-Down */ +#define TAS2770_DIN_PD TAS2770_REG(0X0, 0x31) +#define TAS2770_DIN_PD_SDOUT BIT(7) /* Misc IRQ */ #define TAS2770_MISC_IRQ TAS2770_REG(0X0, 0x32) /* Clock Configuration */ @@ -139,11 +150,14 @@ struct tas2770_priv { struct snd_soc_component *component; struct gpio_desc *reset_gpio; struct gpio_desc *sdz_gpio; + struct regulator *sdz_reg; struct regmap *regmap; struct device *dev; int v_sense_slot; int i_sense_slot; int pdm_slot; + bool sdout_pd; + bool sdout_zfill; bool dac_powered; bool unmuted; }; diff --git a/sound/soc/fsl/fsl_xcvr.c b/sound/soc/fsl/fsl_xcvr.c index 5de93f458b569d..a268fb81a2f868 100644 --- a/sound/soc/fsl/fsl_xcvr.c +++ b/sound/soc/fsl/fsl_xcvr.c @@ -223,13 +223,10 @@ static int fsl_xcvr_mode_put(struct snd_kcontrol *kcontrol, xcvr->mode = snd_soc_enum_item_to_val(e, item[0]); - down_read(&card->snd_card->controls_rwsem); fsl_xcvr_activate_ctl(dai, fsl_xcvr_arc_mode_kctl.name, (xcvr->mode == FSL_XCVR_MODE_ARC)); fsl_xcvr_activate_ctl(dai, fsl_xcvr_earc_capds_kctl.name, (xcvr->mode == FSL_XCVR_MODE_EARC)); - up_read(&card->snd_card->controls_rwsem); - /* Allow playback for SPDIF only */ rtd = snd_soc_get_pcm_runtime(card, card->dai_link); rtd->pcm->streams[SNDRV_PCM_STREAM_PLAYBACK].substream_count = diff --git a/sound/soc/mediatek/mt8188/mt8188-mt6359.c b/sound/soc/mediatek/mt8188/mt8188-mt6359.c index 55ebac0c3cef2a..5a69378545b861 100644 --- a/sound/soc/mediatek/mt8188/mt8188-mt6359.c +++ b/sound/soc/mediatek/mt8188/mt8188-mt6359.c @@ -1280,7 +1280,7 @@ static struct snd_soc_dai_link mt8188_mt6359_dai_links[] = { }, }; -static void mt8188_fixup_controls(struct snd_soc_card *card) +static int mt8188_fixup_controls(struct snd_soc_card *card) { struct mtk_soc_card_data *soc_card_data = snd_soc_card_get_drvdata(card); struct mtk_platform_card_data *card_data = soc_card_data->card_data; @@ -1302,6 +1302,8 @@ static void mt8188_fixup_controls(struct snd_soc_card *card) else dev_warn(card->dev, "Cannot find ctl : Headphone Switch\n"); } + + return 0; } static struct snd_soc_card mt8188_mt6359_soc_card = { diff --git a/sound/soc/rockchip/rockchip_i2s_tdm.c b/sound/soc/rockchip/rockchip_i2s_tdm.c index 770b9bfbb384ae..fc52149ed6ae3e 100644 --- a/sound/soc/rockchip/rockchip_i2s_tdm.c +++ b/sound/soc/rockchip/rockchip_i2s_tdm.c @@ -22,6 +22,7 @@ #define DRV_NAME "rockchip-i2s-tdm" +#define DEFAULT_MCLK_FS 256 #define CH_GRP_MAX 4 /* The max channel 8 / 2 */ #define MULTIPLEX_CH_MAX 10 @@ -665,6 +666,15 @@ static int rockchip_i2s_tdm_hw_params(struct snd_pcm_substream *substream, mclk_rate = i2s_tdm->mclk_rx_freq; } + /* + * When the dai/component driver doesn't need to set mclk-fs for a specific + * clock, it can skip the call to set_sysclk() for that clock. + * In that case, simply use the clock rate from the params and multiply it by + * the default mclk-fs value. + */ + if (!mclk_rate) + mclk_rate = DEFAULT_MCLK_FS * params_rate(params); + err = clk_set_rate(mclk, mclk_rate); if (err) return err; diff --git a/sound/soc/sdca/Makefile b/sound/soc/sdca/Makefile index f6b73275d96493..b3b0f5d94c8de8 100644 --- a/sound/soc/sdca/Makefile +++ b/sound/soc/sdca/Makefile @@ -3,7 +3,7 @@ snd-soc-sdca-y := sdca_functions.o sdca_device.o sdca_function_device.o \ sdca_regmap.o sdca_asoc.o sdca_ump.o snd-soc-sdca-$(CONFIG_SND_SOC_SDCA_HID) += sdca_hid.o -snd-soc-sdca-$(CONFIG_SND_SOC_SDCA_IRQ) += sdca_interrupts.o +snd-soc-sdca-$(CONFIG_SND_SOC_SDCA_IRQ) += sdca_interrupts.o sdca_jack.o snd-soc-sdca-$(CONFIG_SND_SOC_SDCA_FDL) += sdca_fdl.o snd-soc-sdca-class-y := sdca_class.o diff --git a/sound/soc/sdca/sdca_asoc.c b/sound/soc/sdca/sdca_asoc.c index 2d328bbb95b945..9685281529e9f7 100644 --- a/sound/soc/sdca/sdca_asoc.c +++ b/sound/soc/sdca/sdca_asoc.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -792,6 +793,48 @@ static int control_limit_kctl(struct device *dev, return 0; } +static int volatile_get_volsw(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *ucontrol) +{ + struct snd_soc_component *component = snd_kcontrol_chip(kcontrol); + struct device *dev = component->dev; + int ret; + + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) { + dev_err(dev, "failed to resume reading %s: %d\n", + kcontrol->id.name, ret); + return ret; + } + + ret = snd_soc_get_volsw(kcontrol, ucontrol); + + pm_runtime_put(dev); + + return ret; +} + +static int volatile_put_volsw(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *ucontrol) +{ + struct snd_soc_component *component = snd_kcontrol_chip(kcontrol); + struct device *dev = component->dev; + int ret; + + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) { + dev_err(dev, "failed to resume writing %s: %d\n", + kcontrol->id.name, ret); + return ret; + } + + ret = snd_soc_put_volsw(kcontrol, ucontrol); + + pm_runtime_put(dev); + + return ret; +} + static int populate_control(struct device *dev, struct sdca_function_data *function, struct sdca_entity *entity, @@ -849,8 +892,13 @@ static int populate_control(struct device *dev, (*kctl)->private_value = (unsigned long)mc; (*kctl)->iface = SNDRV_CTL_ELEM_IFACE_MIXER; (*kctl)->info = snd_soc_info_volsw; - (*kctl)->get = snd_soc_get_volsw; - (*kctl)->put = snd_soc_put_volsw; + if (control->is_volatile) { + (*kctl)->get = volatile_get_volsw; + (*kctl)->put = volatile_put_volsw; + } else { + (*kctl)->get = snd_soc_get_volsw; + (*kctl)->put = snd_soc_put_volsw; + } if (readonly_control(control)) (*kctl)->access = SNDRV_CTL_ELEM_ACCESS_READ; @@ -1478,7 +1526,7 @@ static int set_usage(struct device *dev, struct regmap *regmap, unsigned int rate = sdca_range(range, SDCA_USAGE_SAMPLE_RATE, i); unsigned int width = sdca_range(range, SDCA_USAGE_SAMPLE_WIDTH, i); - if ((!rate || rate == target_rate) && width == target_width) { + if ((!rate || rate == target_rate) && (!width || width == target_width)) { unsigned int usage = sdca_range(range, SDCA_USAGE_NUMBER, i); unsigned int reg = SDW_SDCA_CTL(function->desc->adr, entity->id, sel, 0); diff --git a/sound/soc/sdca/sdca_functions.c b/sound/soc/sdca/sdca_functions.c index 5a1f120487ef05..e86004c9dea031 100644 --- a/sound/soc/sdca/sdca_functions.c +++ b/sound/soc/sdca/sdca_functions.c @@ -911,10 +911,6 @@ static int find_sdca_control_value(struct device *dev, struct sdca_entity *entit return 0; } -/* - * TODO: Add support for -cn- properties, allowing different channels to have - * different defaults etc. - */ static int find_sdca_entity_control(struct device *dev, struct sdca_entity *entity, struct fwnode_handle *control_node, struct sdca_control *control) diff --git a/sound/soc/sdca/sdca_interrupts.c b/sound/soc/sdca/sdca_interrupts.c index 8f6a2adfb6fbea..ff3a7e405fdcb5 100644 --- a/sound/soc/sdca/sdca_interrupts.c +++ b/sound/soc/sdca/sdca_interrupts.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -155,14 +156,7 @@ static irqreturn_t detected_mode_handler(int irq, void *data) { struct sdca_interrupt *interrupt = data; struct device *dev = interrupt->dev; - struct snd_soc_component *component = interrupt->component; - struct snd_soc_card *card = component->card; - struct rw_semaphore *rwsem = &card->snd_card->controls_rwsem; - struct snd_kcontrol *kctl = interrupt->priv; - struct snd_ctl_elem_value *ucontrol __free(kfree) = NULL; - struct soc_enum *soc_enum; irqreturn_t irqret = IRQ_NONE; - unsigned int reg, val; int ret; ret = pm_runtime_get_sync(dev); @@ -171,76 +165,9 @@ static irqreturn_t detected_mode_handler(int irq, void *data) goto error; } - if (!kctl) { - const char *name __free(kfree) = kasprintf(GFP_KERNEL, "%s %s", - interrupt->entity->label, - SDCA_CTL_SELECTED_MODE_NAME); - - if (!name) - goto error; - - kctl = snd_soc_component_get_kcontrol(component, name); - if (!kctl) { - dev_dbg(dev, "control not found: %s\n", name); - goto error; - } - - interrupt->priv = kctl; - } - - soc_enum = (struct soc_enum *)kctl->private_value; - - reg = SDW_SDCA_CTL(interrupt->function->desc->adr, interrupt->entity->id, - interrupt->control->sel, 0); - - ret = regmap_read(interrupt->function_regmap, reg, &val); - if (ret < 0) { - dev_err(dev, "failed to read detected mode: %d\n", ret); - goto error; - } - - switch (val) { - case SDCA_DETECTED_MODE_DETECTION_IN_PROGRESS: - case SDCA_DETECTED_MODE_JACK_UNKNOWN: - reg = SDW_SDCA_CTL(interrupt->function->desc->adr, - interrupt->entity->id, - SDCA_CTL_GE_SELECTED_MODE, 0); - - /* - * Selected mode is not normally marked as volatile register - * (RW), but here force a read from the hardware. If the - * detected mode is unknown we need to see what the device - * selected as a "safe" option. - */ - regcache_drop_region(interrupt->function_regmap, reg, reg); - - ret = regmap_read(interrupt->function_regmap, reg, &val); - if (ret) { - dev_err(dev, "failed to re-check selected mode: %d\n", ret); - goto error; - } - break; - default: - break; - } - - dev_dbg(dev, "%s: %#x\n", interrupt->name, val); - - ucontrol = kzalloc(sizeof(*ucontrol), GFP_KERNEL); - if (!ucontrol) - goto error; - - ucontrol->value.enumerated.item[0] = snd_soc_enum_val_to_item(soc_enum, val); - - down_write(rwsem); - ret = kctl->put(kctl, ucontrol); - up_write(rwsem); - if (ret < 0) { - dev_err(dev, "failed to update selected mode: %d\n", ret); + ret = sdca_jack_process(interrupt); + if (ret) goto error; - } - - snd_ctl_notify(card->snd_card, SNDRV_CTL_EVENT_MASK_VALUE, &kctl->id); irqret = IRQ_HANDLED; error: @@ -536,6 +463,10 @@ int sdca_irq_populate(struct sdca_function_data *function, handler = function_status_handler; break; case SDCA_CTL_TYPE_S(GE, DETECTED_MODE): + ret = sdca_jack_alloc_state(interrupt); + if (ret) + return ret; + handler = detected_mode_handler; break; case SDCA_CTL_TYPE_S(XU, FDL_CURRENTOWNER): diff --git a/sound/soc/sdca/sdca_jack.c b/sound/soc/sdca/sdca_jack.c new file mode 100644 index 00000000000000..bfa621b744e1a2 --- /dev/null +++ b/sound/soc/sdca/sdca_jack.c @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2025 Cirrus Logic, Inc. and +// Cirrus Logic International Semiconductor Ltd. + +/* + * The MIPI SDCA specification is available for public downloads at + * https://www.mipi.org/mipi-sdca-v1-0-download + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * sdca_jack_process - Process an SDCA jack event + * @interrupt: SDCA interrupt structure + * + * Return: Zero on success or a negative error code. + */ +int sdca_jack_process(struct sdca_interrupt *interrupt) +{ + struct device *dev = interrupt->dev; + struct snd_soc_component *component = interrupt->component; + struct snd_soc_card *card = component->card; + struct rw_semaphore *rwsem = &card->snd_card->controls_rwsem; + struct jack_state *state = interrupt->priv; + struct snd_kcontrol *kctl = state->kctl; + struct snd_ctl_elem_value *ucontrol __free(kfree) = NULL; + unsigned int reg, val; + int ret; + + guard(rwsem_write)(rwsem); + + if (!kctl) { + const char *name __free(kfree) = kasprintf(GFP_KERNEL, "%s %s", + interrupt->entity->label, + SDCA_CTL_SELECTED_MODE_NAME); + + if (!name) + return -ENOMEM; + + kctl = snd_soc_component_get_kcontrol(component, name); + if (!kctl) + dev_dbg(dev, "control not found: %s\n", name); + else + state->kctl = kctl; + } + + reg = SDW_SDCA_CTL(interrupt->function->desc->adr, interrupt->entity->id, + interrupt->control->sel, 0); + + ret = regmap_read(interrupt->function_regmap, reg, &val); + if (ret < 0) { + dev_err(dev, "failed to read detected mode: %d\n", ret); + return ret; + } + + reg = SDW_SDCA_CTL(interrupt->function->desc->adr, interrupt->entity->id, + SDCA_CTL_GE_SELECTED_MODE, 0); + + switch (val) { + case SDCA_DETECTED_MODE_DETECTION_IN_PROGRESS: + case SDCA_DETECTED_MODE_JACK_UNKNOWN: + /* + * Selected mode is not normally marked as volatile register + * (RW), but here force a read from the hardware. If the + * detected mode is unknown we need to see what the device + * selected as a "safe" option. + */ + regcache_drop_region(interrupt->function_regmap, reg, reg); + + ret = regmap_read(interrupt->function_regmap, reg, &val); + if (ret) { + dev_err(dev, "failed to re-check selected mode: %d\n", ret); + return ret; + } + break; + default: + break; + } + + dev_dbg(dev, "%s: %#x\n", interrupt->name, val); + + if (kctl) { + struct soc_enum *soc_enum = (struct soc_enum *)kctl->private_value; + + ucontrol = kzalloc(sizeof(*ucontrol), GFP_KERNEL); + if (!ucontrol) + return -ENOMEM; + + ucontrol->value.enumerated.item[0] = snd_soc_enum_val_to_item(soc_enum, val); + + ret = kctl->put(kctl, ucontrol); + if (ret < 0) { + dev_err(dev, "failed to update selected mode: %d\n", ret); + return ret; + } + + snd_ctl_notify(card->snd_card, SNDRV_CTL_EVENT_MASK_VALUE, &kctl->id); + } else { + ret = regmap_write(interrupt->function_regmap, reg, val); + if (ret) { + dev_err(dev, "failed to write selected mode: %d\n", ret); + return ret; + } + } + + return sdca_jack_report(interrupt); +} +EXPORT_SYMBOL_NS_GPL(sdca_jack_process, "SND_SOC_SDCA"); + +/** + * sdca_jack_alloc_state - allocate state for a jack interrupt + * @interrupt: SDCA interrupt structure. + * + * Return: Zero on success or a negative error code. + */ +int sdca_jack_alloc_state(struct sdca_interrupt *interrupt) +{ + struct device *dev = interrupt->dev; + struct jack_state *jack_state; + + jack_state = devm_kzalloc(dev, sizeof(*jack_state), GFP_KERNEL); + if (!jack_state) + return -ENOMEM; + + interrupt->priv = jack_state; + + return 0; +} +EXPORT_SYMBOL_NS_GPL(sdca_jack_alloc_state, "SND_SOC_SDCA"); + +/** + * sdca_jack_set_jack - attach an ASoC jack to SDCA + * @info: SDCA interrupt information. + * @jack: ASoC jack to be attached. + * + * Return: Zero on success or a negative error code. + */ +int sdca_jack_set_jack(struct sdca_interrupt_info *info, struct snd_soc_jack *jack) +{ + int i, ret; + + guard(mutex)(&info->irq_lock); + + for (i = 0; i < SDCA_MAX_INTERRUPTS; i++) { + struct sdca_interrupt *interrupt = &info->irqs[i]; + struct sdca_control *control = interrupt->control; + struct sdca_entity *entity = interrupt->entity; + struct jack_state *jack_state; + + if (!interrupt->irq) + continue; + + switch (SDCA_CTL_TYPE(entity->type, control->sel)) { + case SDCA_CTL_TYPE_S(GE, DETECTED_MODE): + jack_state = interrupt->priv; + jack_state->jack = jack; + + /* Report initial state in case IRQ was already handled */ + ret = sdca_jack_report(interrupt); + if (ret) + return ret; + break; + default: + break; + } + } + + return 0; +} +EXPORT_SYMBOL_NS_GPL(sdca_jack_set_jack, "SND_SOC_SDCA"); + +int sdca_jack_report(struct sdca_interrupt *interrupt) +{ + struct jack_state *jack_state = interrupt->priv; + struct sdca_control_range *range; + enum sdca_terminal_type type; + unsigned int report = 0; + unsigned int reg, val; + int ret; + + reg = SDW_SDCA_CTL(interrupt->function->desc->adr, interrupt->entity->id, + SDCA_CTL_GE_SELECTED_MODE, 0); + + ret = regmap_read(interrupt->function_regmap, reg, &val); + if (ret) { + dev_err(interrupt->dev, "failed to read selected mode: %d\n", ret); + return ret; + } + + range = sdca_selector_find_range(interrupt->dev, interrupt->entity, + SDCA_CTL_GE_SELECTED_MODE, + SDCA_SELECTED_MODE_NCOLS, 0); + if (!range) + return -EINVAL; + + type = sdca_range_search(range, SDCA_SELECTED_MODE_INDEX, + val, SDCA_SELECTED_MODE_TERM_TYPE); + + switch (type) { + case SDCA_TERM_TYPE_LINEIN_STEREO: + case SDCA_TERM_TYPE_LINEIN_FRONT_LR: + case SDCA_TERM_TYPE_LINEIN_CENTER_LFE: + case SDCA_TERM_TYPE_LINEIN_SURROUND_LR: + case SDCA_TERM_TYPE_LINEIN_REAR_LR: + report = SND_JACK_LINEIN; + break; + case SDCA_TERM_TYPE_LINEOUT_STEREO: + case SDCA_TERM_TYPE_LINEOUT_FRONT_LR: + case SDCA_TERM_TYPE_LINEOUT_CENTER_LFE: + case SDCA_TERM_TYPE_LINEOUT_SURROUND_LR: + case SDCA_TERM_TYPE_LINEOUT_REAR_LR: + report = SND_JACK_LINEOUT; + break; + case SDCA_TERM_TYPE_MIC_JACK: + report = SND_JACK_MICROPHONE; + break; + case SDCA_TERM_TYPE_HEADPHONE_JACK: + report = SND_JACK_HEADPHONE; + break; + case SDCA_TERM_TYPE_HEADSET_JACK: + report = SND_JACK_HEADSET; + break; + default: + break; + } + + snd_soc_jack_report(jack_state->jack, report, 0xFFFF); + + return 0; +} +EXPORT_SYMBOL_NS_GPL(sdca_jack_report, "SND_SOC_SDCA"); diff --git a/sound/soc/soc-card.c b/sound/soc/soc-card.c index 235427d6906173..bc02c7b864e295 100644 --- a/sound/soc/soc-card.c +++ b/sound/soc/soc-card.c @@ -184,10 +184,16 @@ int snd_soc_card_late_probe(struct snd_soc_card *card) return 0; } -void snd_soc_card_fixup_controls(struct snd_soc_card *card) +int snd_soc_card_fixup_controls(struct snd_soc_card *card) { - if (card->fixup_controls) - card->fixup_controls(card); + if (card->fixup_controls) { + int ret = card->fixup_controls(card); + + if (ret < 0) + return soc_card_ret(card, ret); + } + + return 0; } int snd_soc_card_remove(struct snd_soc_card *card) diff --git a/sound/soc/soc-core.c b/sound/soc/soc-core.c index e4b21bf39e59f6..dde58268892064 100644 --- a/sound/soc/soc-core.c +++ b/sound/soc/soc-core.c @@ -2288,7 +2288,10 @@ static int snd_soc_bind_card(struct snd_soc_card *card) goto probe_end; snd_soc_dapm_new_widgets(card); - snd_soc_card_fixup_controls(card); + + ret = snd_soc_card_fixup_controls(card); + if (ret < 0) + goto probe_end; ret = snd_card_register(card->snd_card); if (ret < 0) { diff --git a/sound/soc/soc-dapm.c b/sound/soc/soc-dapm.c index 4d920a59da3c49..15de858c5533ee 100644 --- a/sound/soc/soc-dapm.c +++ b/sound/soc/soc-dapm.c @@ -2526,6 +2526,141 @@ static const struct file_operations dapm_bias_fops = { .llseek = default_llseek, }; +static ssize_t dapm_graph_read_file(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct snd_soc_card *card = file->private_data; + struct snd_soc_dapm_context *dapm; + struct snd_soc_dapm_path *p; + struct snd_soc_dapm_widget *w; + struct snd_soc_pcm_runtime *rtd; + struct snd_soc_dapm_widget *wdone[16]; + struct snd_soc_dai *dai; + int i, num_wdone = 0, cluster = 0; + char *buf; + ssize_t bufsize; + ssize_t ret = 0; + + bufsize = 1024 * card->num_dapm_widgets; + buf = kmalloc(bufsize, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + mutex_lock(&card->dapm_mutex); + +#define bufprintf(...) \ + ret += scnprintf(buf + ret, bufsize - ret, __VA_ARGS__) + + bufprintf("digraph dapm {\n"); + + /* + * Print the user-visible devices of the card. + */ + bufprintf("subgraph cluster_%d {\n", cluster++); + bufprintf("label=\"Devices\";style=filled;fillcolor=gray;\n"); + for_each_card_rtds(card, rtd) { + if (rtd->dai_link->no_pcm) + continue; + + bufprintf("w%pK [label=\"%d: %s\"];\n", rtd, + rtd->pcm->device, rtd->dai_link->name); + } + bufprintf("};\n"); + + /* + * Print the playback/capture widgets of DAIs just next to + * the user-visible devices. Keep the list of already printed + * widgets in 'wdone', so they will be skipped later. + */ + for_each_card_rtds(card, rtd) { + for_each_rtd_cpu_dais(rtd, i, dai) { + if (dai->stream[SNDRV_PCM_STREAM_PLAYBACK].widget) { + w = dai->stream[SNDRV_PCM_STREAM_PLAYBACK].widget; + bufprintf("w%pK [label=\"%s\"];\n", w, w->name); + if (!rtd->dai_link->no_pcm) + bufprintf("w%pK -> w%pK;\n", rtd, w); + if (num_wdone < ARRAY_SIZE(wdone)) { + wdone[num_wdone] = w; + num_wdone++; + } + } + + if (dai->stream[SNDRV_PCM_STREAM_CAPTURE].widget) { + w = dai->stream[SNDRV_PCM_STREAM_CAPTURE].widget; + bufprintf("w%pK [label=\"%s\"];\n", w, w->name); + if (!rtd->dai_link->no_pcm) + bufprintf("w%pK -> w%pK;\n", w, rtd); + if (num_wdone < ARRAY_SIZE(wdone)) { + wdone[num_wdone] = w; + num_wdone++; + } + } + } + } + + for_each_card_dapms(card, dapm) { + const char *prefix = soc_dapm_prefix(dapm); + + if (dapm != &card->dapm) { + bufprintf("subgraph cluster_%d {\n", cluster++); + if (prefix) + bufprintf("label=\"%s\";\n", prefix); + else if (dapm->component) + bufprintf("label=\"%s\";\n", + dapm->component->name); + } + + for_each_card_widgets(dapm->card, w) { + const char *name = w->name; + bool skip = false; + + if (w->dapm != dapm) + continue; + + if (list_empty(&w->edges[0]) && list_empty(&w->edges[1])) + continue; + + for (i = 0; i < num_wdone; i++) + if (wdone[i] == w) + skip = true; + if (skip) + continue; + + if (prefix && strlen(name) > strlen(prefix) + 1) + name += strlen(prefix) + 1; + + bufprintf("w%pK [label=\"%s\"];\n", w, name); + } + + if (dapm != &card->dapm) + bufprintf("}\n"); + } + + list_for_each_entry(p, &card->paths, list) { + if (p->name) + bufprintf("w%pK -> w%pK [label=\"%s\"];\n", + p->source, p->sink, p->name); + else + bufprintf("w%pK -> w%pK;\n", p->source, p->sink); + } + + bufprintf("}\n"); +#undef bufprintf + + mutex_unlock(&card->dapm_mutex); + + ret = simple_read_from_buffer(user_buf, count, ppos, buf, ret); + + kfree(buf); + return ret; +} + +static const struct file_operations dapm_graph_fops = { + .open = simple_open, + .read = dapm_graph_read_file, + .llseek = default_llseek, +}; + void snd_soc_dapm_debugfs_init(struct snd_soc_dapm_context *dapm, struct dentry *parent) { @@ -2536,6 +2671,10 @@ void snd_soc_dapm_debugfs_init(struct snd_soc_dapm_context *dapm, debugfs_create_file("bias_level", 0444, dapm->debugfs_dapm, dapm, &dapm_bias_fops); + + if (dapm == &dapm->card->dapm) + debugfs_create_file("graph.dot", 0444, dapm->debugfs_dapm, + dapm->card, &dapm_graph_fops); } static void dapm_debugfs_add_widget(struct snd_soc_dapm_widget *w) diff --git a/sound/soc/soc-ops.c b/sound/soc/soc-ops.c index ba42939d5f013c..15d53c1345fead 100644 --- a/sound/soc/soc-ops.c +++ b/sound/soc/soc-ops.c @@ -445,6 +445,30 @@ int snd_soc_put_volsw_sx(struct snd_kcontrol *kcontrol, } EXPORT_SYMBOL_GPL(snd_soc_put_volsw_sx); +bool snd_soc_control_matches(struct snd_kcontrol *kctl, + const char *pattern) +{ + const char *name = kctl->id.name; + + if (pattern[0] == '*') { + int namelen; + int patternlen; + + pattern++; + if (pattern[0] == ' ') + pattern++; + + namelen = strlen(name); + patternlen = strlen(pattern); + + if (namelen > patternlen) + name += namelen - patternlen; + } + + return !strcmp(name, pattern); +} +EXPORT_SYMBOL_GPL(snd_soc_control_matches); + static int snd_soc_clip_to_platform_max(struct snd_kcontrol *kctl) { struct soc_mixer_control *mc = (struct soc_mixer_control *)kctl->private_value; @@ -476,39 +500,165 @@ static int snd_soc_clip_to_platform_max(struct snd_kcontrol *kctl) return ret; } +static int soc_limit_volume(struct snd_kcontrol *kctl, int max) +{ + struct soc_mixer_control *mc = (struct soc_mixer_control *)kctl->private_value; + + if (max <= 0 || max > mc->max - mc->min) + return -EINVAL; + mc->platform_max = max; + return snd_soc_clip_to_platform_max(kctl); +} + /** - * snd_soc_limit_volume - Set new limit to an existing volume control. + * snd_soc_limit_volume - Set new limit to existing volume controls * * @card: where to look for the control - * @name: Name of the control + * @name: name pattern * @max: new maximum limit + * + * Finds controls matching the given name (which can be either a name + * verbatim, or a pattern starting with the wildcard '*') and sets + * a platform volume limit on them. * - * Return 0 for success, else error. + * Return number of matching controls on success, else error. At least + * one control needs to match the pattern. */ int snd_soc_limit_volume(struct snd_soc_card *card, const char *name, int max) { struct snd_kcontrol *kctl; - int ret = -EINVAL; + int hits = 0; + int ret; - /* Sanity check for name and max */ - if (unlikely(!name || max <= 0)) + /* Sanity check for name */ + if (unlikely(!name)) return -EINVAL; - kctl = snd_soc_card_get_kcontrol(card, name); - if (kctl) { - struct soc_mixer_control *mc = - (struct soc_mixer_control *)kctl->private_value; + list_for_each_entry(kctl, &card->snd_card->controls, list) { + if (!snd_soc_control_matches(kctl, name)) + continue; - if (max <= mc->max - mc->min) { - mc->platform_max = max; - ret = snd_soc_clip_to_platform_max(kctl); - } + ret = soc_limit_volume(kctl, max); + if (ret < 0) + return ret; + hits++; } - return ret; + if (!hits) + return -EINVAL; + + return hits; } EXPORT_SYMBOL_GPL(snd_soc_limit_volume); +/** + * snd_soc_deactivate_kctl - Activate/deactive controls matching a pattern + * + * @card: where to look for the controls + * @name: name pattern + * @active: non-zero to activate, zero to deactivate + * + * Return number of matching controls on success, else error. + * No controls need to match. + */ +int snd_soc_deactivate_kctl(struct snd_soc_card *card, + const char *name, int active) +{ + struct snd_kcontrol *kctl; + int hits = 0; + int ret; + + /* Sanity check for name */ + if (unlikely(!name)) + return -EINVAL; + + list_for_each_entry(kctl, &card->snd_card->controls, list) { + if (!snd_soc_control_matches(kctl, name)) + continue; + + ret = snd_ctl_activate_id(card->snd_card, &kctl->id, active); + if (ret < 0) + return ret; + hits++; + } + + if (!hits) + return -EINVAL; + + return hits; +} +EXPORT_SYMBOL_GPL(snd_soc_deactivate_kctl); + +static int soc_set_enum_kctl(struct snd_kcontrol *kctl, const char *strval) +{ + struct snd_ctl_elem_value value; + struct snd_ctl_elem_info info; + int sel, i, ret; + + ret = kctl->info(kctl, &info); + if (ret < 0) + return ret; + + if (info.type != SNDRV_CTL_ELEM_TYPE_ENUMERATED) + return -EINVAL; + + for (sel = 0; sel < info.value.enumerated.items; sel++) { + info.value.enumerated.item = sel; + ret = kctl->info(kctl, &info); + if (ret < 0) + return ret; + + if (!strcmp(strval, info.value.enumerated.name)) + break; + } + + if (sel == info.value.enumerated.items) + return -EINVAL; + + for (i = 0; i < info.count; i++) + value.value.enumerated.item[i] = sel; + + return kctl->put(kctl, &value); +} + +/** + * snd_soc_set_enum_kctl - Set enumerated controls matching a pattern + * + * @card: where to look for the controls + * @name: name pattern + * @value: string value to set the controls to + * + * Return number of matching and set controls on success, else error. + * No controls need to match. + */ +int snd_soc_set_enum_kctl(struct snd_soc_card *card, + const char *name, const char *value) +{ + struct snd_kcontrol *kctl; + int hits = 0; + int ret; + + /* Sanity check for name */ + if (unlikely(!name)) + return -EINVAL; + + list_for_each_entry(kctl, &card->snd_card->controls, list) { + if (!snd_soc_control_matches(kctl, name)) + continue; + + ret = soc_set_enum_kctl(kctl, value); + if (ret < 0) + return ret; + hits++; + } + + if (!hits) + return -EINVAL; + + return hits; +} +EXPORT_SYMBOL_GPL(snd_soc_set_enum_kctl); + int snd_soc_bytes_info(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_info *uinfo) { diff --git a/sound/soc/tegra/tegra210_ahub.c b/sound/soc/tegra/tegra210_ahub.c index e795907a3963af..fc5892056f832f 100644 --- a/sound/soc/tegra/tegra210_ahub.c +++ b/sound/soc/tegra/tegra210_ahub.c @@ -2049,6 +2049,61 @@ static const struct snd_soc_component_driver tegra264_ahub_component = { .num_dapm_routes = ARRAY_SIZE(tegra264_ahub_routes), }; +static bool tegra210_ahub_wr_reg(struct device *dev, unsigned int reg) +{ + int part; + + if (reg % TEGRA210_XBAR_RX_STRIDE) + return false; + + for (part = 0; part < TEGRA210_XBAR_UPDATE_MAX_REG; part++) { + switch (reg & ~(part * TEGRA210_XBAR_PART1_RX)) { + case TEGRA210_AXBAR_PART_0_ADMAIF_RX1_0 ... TEGRA210_AXBAR_PART_0_ADMAIF_RX10_0: + case TEGRA210_AXBAR_PART_0_I2S1_RX1_0 ... TEGRA210_AXBAR_PART_0_I2S5_RX1_0: + case TEGRA210_AXBAR_PART_0_SFC1_RX1_0 ... TEGRA210_AXBAR_PART_0_SFC4_RX1_0: + case TEGRA210_AXBAR_PART_0_MIXER1_RX1_0 ... TEGRA210_AXBAR_PART_0_MIXER1_RX10_0: + case TEGRA210_AXBAR_PART_0_SPDIF1_RX1_0 ... TEGRA210_AXBAR_PART_0_SPDIF1_RX2_0: + case TEGRA210_AXBAR_PART_0_AFC1_RX1_0 ... TEGRA210_AXBAR_PART_0_AFC6_RX1_0: + case TEGRA210_AXBAR_PART_0_OPE1_RX1_0 ... TEGRA210_AXBAR_PART_0_OPE2_RX1_0: + case TEGRA210_AXBAR_PART_0_SPKPROT1_RX1_0: + case TEGRA210_AXBAR_PART_0_MVC1_RX1_0 ... TEGRA210_AXBAR_PART_0_MVC2_RX1_0: + case TEGRA210_AXBAR_PART_0_AMX1_RX1_0 ... TEGRA210_AXBAR_PART_0_ADX2_RX1_0: + return true; + default: + break; + } + } + + return false; +} + +static bool tegra186_ahub_wr_reg(struct device *dev, unsigned int reg) +{ + int part; + + if (reg % TEGRA210_XBAR_RX_STRIDE) + return false; + + for (part = 0; part < TEGRA186_XBAR_UPDATE_MAX_REG; part++) { + switch (reg & ~(part * TEGRA210_XBAR_PART1_RX)) { + case TEGRA210_AXBAR_PART_0_ADMAIF_RX1_0 ... TEGRA186_AXBAR_PART_0_I2S6_RX1_0: + case TEGRA210_AXBAR_PART_0_SFC1_RX1_0 ... TEGRA210_AXBAR_PART_0_SFC4_RX1_0: + case TEGRA210_AXBAR_PART_0_MIXER1_RX1_0 ... TEGRA210_AXBAR_PART_0_MIXER1_RX10_0: + case TEGRA186_AXBAR_PART_0_DSPK1_RX1_0 ... TEGRA186_AXBAR_PART_0_DSPK2_RX1_0: + case TEGRA210_AXBAR_PART_0_AFC1_RX1_0 ... TEGRA210_AXBAR_PART_0_AFC6_RX1_0: + case TEGRA210_AXBAR_PART_0_OPE1_RX1_0: + case TEGRA186_AXBAR_PART_0_MVC1_RX1_0 ... TEGRA186_AXBAR_PART_0_MVC2_RX1_0: + case TEGRA186_AXBAR_PART_0_AMX1_RX1_0 ... TEGRA186_AXBAR_PART_0_AMX3_RX4_0: + case TEGRA210_AXBAR_PART_0_ADX1_RX1_0 ... TEGRA186_AXBAR_PART_0_ASRC1_RX7_0: + return true; + default: + break; + } + } + + return false; +} + static bool tegra264_ahub_wr_reg(struct device *dev, unsigned int reg) { int part; @@ -2076,6 +2131,7 @@ static const struct regmap_config tegra210_ahub_regmap_config = { .reg_bits = 32, .val_bits = 32, .reg_stride = 4, + .writeable_reg = tegra210_ahub_wr_reg, .max_register = TEGRA210_MAX_REGISTER_ADDR, .cache_type = REGCACHE_FLAT, }; @@ -2084,6 +2140,7 @@ static const struct regmap_config tegra186_ahub_regmap_config = { .reg_bits = 32, .val_bits = 32, .reg_stride = 4, + .writeable_reg = tegra186_ahub_wr_reg, .max_register = TEGRA186_MAX_REGISTER_ADDR, .cache_type = REGCACHE_FLAT, }; diff --git a/sound/soc/tegra/tegra210_ahub.h b/sound/soc/tegra/tegra210_ahub.h index f355b2cfd19b22..acbe640dd3b572 100644 --- a/sound/soc/tegra/tegra210_ahub.h +++ b/sound/soc/tegra/tegra210_ahub.h @@ -68,6 +68,36 @@ #define TEGRA210_MAX_REGISTER_ADDR (TEGRA210_XBAR_PART2_RX + \ (TEGRA210_XBAR_RX_STRIDE * (TEGRA210_XBAR_AUDIO_RX_COUNT - 1))) +/* AXBAR register offsets */ +#define TEGRA186_AXBAR_PART_0_AMX1_RX1_0 0x120 +#define TEGRA186_AXBAR_PART_0_AMX3_RX4_0 0x14c +#define TEGRA186_AXBAR_PART_0_ASRC1_RX7_0 0x1a8 +#define TEGRA186_AXBAR_PART_0_DSPK1_RX1_0 0xc0 +#define TEGRA186_AXBAR_PART_0_DSPK2_RX1_0 0xc4 +#define TEGRA186_AXBAR_PART_0_I2S6_RX1_0 0x54 +#define TEGRA186_AXBAR_PART_0_MVC1_RX1_0 0x110 +#define TEGRA186_AXBAR_PART_0_MVC2_RX1_0 0x114 +#define TEGRA210_AXBAR_PART_0_ADMAIF_RX10_0 0x24 +#define TEGRA210_AXBAR_PART_0_ADMAIF_RX1_0 0x0 +#define TEGRA210_AXBAR_PART_0_ADX1_RX1_0 0x160 +#define TEGRA210_AXBAR_PART_0_ADX2_RX1_0 0x164 +#define TEGRA210_AXBAR_PART_0_AFC1_RX1_0 0xd0 +#define TEGRA210_AXBAR_PART_0_AFC6_RX1_0 0xe4 +#define TEGRA210_AXBAR_PART_0_AMX1_RX1_0 0x140 +#define TEGRA210_AXBAR_PART_0_I2S1_RX1_0 0x40 +#define TEGRA210_AXBAR_PART_0_I2S5_RX1_0 0x50 +#define TEGRA210_AXBAR_PART_0_MIXER1_RX10_0 0xa4 +#define TEGRA210_AXBAR_PART_0_MIXER1_RX1_0 0x80 +#define TEGRA210_AXBAR_PART_0_MVC1_RX1_0 0x120 +#define TEGRA210_AXBAR_PART_0_MVC2_RX1_0 0x124 +#define TEGRA210_AXBAR_PART_0_OPE1_RX1_0 0x100 +#define TEGRA210_AXBAR_PART_0_OPE2_RX1_0 0x104 +#define TEGRA210_AXBAR_PART_0_SFC1_RX1_0 0x60 +#define TEGRA210_AXBAR_PART_0_SFC4_RX1_0 0x6c +#define TEGRA210_AXBAR_PART_0_SPDIF1_RX1_0 0xc0 +#define TEGRA210_AXBAR_PART_0_SPDIF1_RX2_0 0xc4 +#define TEGRA210_AXBAR_PART_0_SPKPROT1_RX1_0 0x110 + #define MUX_REG(id) (TEGRA210_XBAR_RX_STRIDE * (id)) #define MUX_VALUE(npart, nbit) (1 + (nbit) + (npart) * 32) diff --git a/sound/usb/fcp.c b/sound/usb/fcp.c index 11e9a96b46ffe5..1f4595d1e217f9 100644 --- a/sound/usb/fcp.c +++ b/sound/usb/fcp.c @@ -182,10 +182,6 @@ static int fcp_usb(struct usb_mixer_interface *mixer, u32 opcode, { struct fcp_data *private = mixer->private_data; struct usb_device *dev = mixer->chip->dev; - struct fcp_usb_packet *req __free(kfree) = NULL; - struct fcp_usb_packet *resp __free(kfree) = NULL; - size_t req_buf_size = struct_size(req, data, req_size); - size_t resp_buf_size = struct_size(resp, data, resp_size); int retries = 0; const int max_retries = 5; int err; @@ -193,10 +189,14 @@ static int fcp_usb(struct usb_mixer_interface *mixer, u32 opcode, if (!mixer->urb) return -ENODEV; + struct fcp_usb_packet *req __free(kfree) = NULL; + size_t req_buf_size = struct_size(req, data, req_size); req = kmalloc(req_buf_size, GFP_KERNEL); if (!req) return -ENOMEM; + struct fcp_usb_packet *resp __free(kfree) = NULL; + size_t resp_buf_size = struct_size(resp, data, resp_size); resp = kmalloc(resp_buf_size, GFP_KERNEL); if (!resp) return -ENOMEM; @@ -300,16 +300,17 @@ static int fcp_usb(struct usb_mixer_interface *mixer, u32 opcode, static int fcp_reinit(struct usb_mixer_interface *mixer) { struct fcp_data *private = mixer->private_data; - void *step0_resp __free(kfree) = NULL; - void *step2_resp __free(kfree) = NULL; if (mixer->urb) return 0; - step0_resp = kmalloc(private->step0_resp_size, GFP_KERNEL); + void *step0_resp __free(kfree) = + kmalloc(private->step0_resp_size, GFP_KERNEL); if (!step0_resp) return -ENOMEM; - step2_resp = kmalloc(private->step2_resp_size, GFP_KERNEL); + + void *step2_resp __free(kfree) = + kmalloc(private->step2_resp_size, GFP_KERNEL); if (!step2_resp) return -ENOMEM; @@ -464,7 +465,6 @@ static int fcp_ioctl_init(struct usb_mixer_interface *mixer, struct fcp_init init; struct usb_device *dev = mixer->chip->dev; struct fcp_data *private = mixer->private_data; - void *resp __free(kfree) = NULL; void *step2_resp; int err, buf_size; @@ -485,7 +485,8 @@ static int fcp_ioctl_init(struct usb_mixer_interface *mixer, /* Allocate response buffer */ buf_size = init.step0_resp_size + init.step2_resp_size; - resp = kmalloc(buf_size, GFP_KERNEL); + void *resp __free(kfree) = + kmalloc(buf_size, GFP_KERNEL); if (!resp) return -ENOMEM; @@ -619,7 +620,6 @@ static int fcp_ioctl_set_meter_map(struct usb_mixer_interface *mixer, { struct fcp_meter_map map; struct fcp_data *private = mixer->private_data; - s16 *tmp_map __free(kfree) = NULL; int err; if (copy_from_user(&map, arg, sizeof(map))) @@ -641,7 +641,8 @@ static int fcp_ioctl_set_meter_map(struct usb_mixer_interface *mixer, return -EINVAL; /* Allocate and copy the map data */ - tmp_map = memdup_array_user(arg->map, map.map_size, sizeof(s16)); + s16 *tmp_map __free(kfree) = + memdup_array_user(arg->map, map.map_size, sizeof(s16)); if (IS_ERR(tmp_map)) return PTR_ERR(tmp_map); @@ -651,17 +652,16 @@ static int fcp_ioctl_set_meter_map(struct usb_mixer_interface *mixer, /* If the control doesn't exist, create it */ if (!private->meter_ctl) { - s16 *new_map __free(kfree) = NULL; - __le32 *meter_levels __free(kfree) = NULL; - /* Allocate buffer for the map */ - new_map = kmalloc_array(map.map_size, sizeof(s16), GFP_KERNEL); + s16 *new_map __free(kfree) = + kmalloc_array(map.map_size, sizeof(s16), GFP_KERNEL); if (!new_map) return -ENOMEM; /* Allocate buffer for reading meter levels */ - meter_levels = kmalloc_array(map.meter_slots, sizeof(__le32), - GFP_KERNEL); + __le32 *meter_levels __free(kfree) = + kmalloc_array(map.meter_slots, sizeof(__le32), + GFP_KERNEL); if (!meter_levels) return -ENOMEM; diff --git a/sound/usb/mixer_scarlett2.c b/sound/usb/mixer_scarlett2.c index bef8c9e544dd3d..88b7e42d159e00 100644 --- a/sound/usb/mixer_scarlett2.c +++ b/sound/usb/mixer_scarlett2.c @@ -2377,18 +2377,18 @@ static int scarlett2_usb( { struct scarlett2_data *private = mixer->private_data; struct usb_device *dev = mixer->chip->dev; - struct scarlett2_usb_packet *req __free(kfree) = NULL; - struct scarlett2_usb_packet *resp __free(kfree) = NULL; - size_t req_buf_size = struct_size(req, data, req_size); - size_t resp_buf_size = struct_size(resp, data, resp_size); int retries = 0; const int max_retries = 5; int err; + struct scarlett2_usb_packet *req __free(kfree) = NULL; + size_t req_buf_size = struct_size(req, data, req_size); req = kmalloc(req_buf_size, GFP_KERNEL); if (!req) return -ENOMEM; + struct scarlett2_usb_packet *resp __free(kfree) = NULL; + size_t resp_buf_size = struct_size(resp, data, resp_size); resp = kmalloc(resp_buf_size, GFP_KERNEL); if (!resp) return -ENOMEM; @@ -3919,9 +3919,9 @@ static int scarlett2_input_select_ctl_info( struct scarlett2_data *private = mixer->private_data; int inputs = private->info->gain_input_count; int i, err; - char **values __free(kfree) = NULL; + char **values __free(kfree) = + kcalloc(inputs, sizeof(char *), GFP_KERNEL); - values = kcalloc(inputs, sizeof(char *), GFP_KERNEL); if (!values) return -ENOMEM; @@ -9083,8 +9083,6 @@ static long scarlett2_hwdep_read(struct snd_hwdep *hw, __le32 len; } __packed req; - u8 *resp __free(kfree) = NULL; - /* Flash segment must first be selected */ if (private->flash_write_state != SCARLETT2_FLASH_WRITE_STATE_SELECTED) return -EINVAL; @@ -9122,7 +9120,8 @@ static long scarlett2_hwdep_read(struct snd_hwdep *hw, req.offset = cpu_to_le32(*offset); req.len = cpu_to_le32(count); - resp = kzalloc(count, GFP_KERNEL); + u8 *resp __free(kfree) = + kzalloc(count, GFP_KERNEL); if (!resp) return -ENOMEM; @@ -9267,7 +9266,6 @@ static ssize_t scarlett2_devmap_read( loff_t pos) { struct usb_mixer_interface *mixer = entry->private_data; - u8 *resp_buf __free(kfree) = NULL; const size_t block_size = SCARLETT2_DEVMAP_BLOCK_SIZE; size_t copied = 0; @@ -9277,7 +9275,8 @@ static ssize_t scarlett2_devmap_read( if (pos + count > entry->size) count = entry->size - pos; - resp_buf = kmalloc(block_size, GFP_KERNEL); + u8 *resp_buf __free(kfree) = + kmalloc(block_size, GFP_KERNEL); if (!resp_buf) return -ENOMEM; diff --git a/sound/usb/quirks.c b/sound/usb/quirks.c index 2d9f28558874c0..4f9d19bf1ccacd 100644 --- a/sound/usb/quirks.c +++ b/sound/usb/quirks.c @@ -555,7 +555,6 @@ int snd_usb_create_quirk(struct snd_usb_audio *chip, static int snd_usb_extigy_boot_quirk(struct usb_device *dev, struct usb_interface *intf) { struct usb_host_config *config = dev->actconfig; - struct usb_device_descriptor *new_device_descriptor __free(kfree) = NULL; int err; if (le16_to_cpu(get_cfg_desc(config)->wTotalLength) == EXTIGY_FIRMWARE_SIZE_OLD || @@ -566,8 +565,8 @@ static int snd_usb_extigy_boot_quirk(struct usb_device *dev, struct usb_interfac 0x10, 0x43, 0x0001, 0x000a, NULL, 0); if (err < 0) dev_dbg(&dev->dev, "error sending boot message: %d\n", err); - - new_device_descriptor = kmalloc(sizeof(*new_device_descriptor), GFP_KERNEL); + struct usb_device_descriptor *new_device_descriptor __free(kfree) = + kmalloc(sizeof(*new_device_descriptor), GFP_KERNEL); if (!new_device_descriptor) return -ENOMEM; err = usb_get_descriptor(dev, USB_DT_DEVICE, 0, @@ -910,7 +909,6 @@ static void mbox2_setup_48_24_magic(struct usb_device *dev) static int snd_usb_mbox2_boot_quirk(struct usb_device *dev) { struct usb_host_config *config = dev->actconfig; - struct usb_device_descriptor *new_device_descriptor __free(kfree) = NULL; int err; u8 bootresponse[0x12]; int fwsize; @@ -945,7 +943,8 @@ static int snd_usb_mbox2_boot_quirk(struct usb_device *dev) dev_dbg(&dev->dev, "device initialised!\n"); - new_device_descriptor = kmalloc(sizeof(*new_device_descriptor), GFP_KERNEL); + struct usb_device_descriptor *new_device_descriptor __free(kfree) = + kmalloc(sizeof(*new_device_descriptor), GFP_KERNEL); if (!new_device_descriptor) return -ENOMEM; @@ -1267,7 +1266,6 @@ static void mbox3_setup_defaults(struct usb_device *dev) static int snd_usb_mbox3_boot_quirk(struct usb_device *dev) { struct usb_host_config *config = dev->actconfig; - struct usb_device_descriptor *new_device_descriptor __free(kfree) = NULL; int err; int descriptor_size; @@ -1280,7 +1278,8 @@ static int snd_usb_mbox3_boot_quirk(struct usb_device *dev) dev_dbg(&dev->dev, "MBOX3: device initialised!\n"); - new_device_descriptor = kmalloc(sizeof(*new_device_descriptor), GFP_KERNEL); + struct usb_device_descriptor *new_device_descriptor __free(kfree) = + kmalloc(sizeof(*new_device_descriptor), GFP_KERNEL); if (!new_device_descriptor) return -ENOMEM; diff --git a/sound/usb/usx2y/us144mkii.c b/sound/usb/usx2y/us144mkii.c index f6572a576c150e..bc71968df8e2c6 100644 --- a/sound/usb/usx2y/us144mkii.c +++ b/sound/usb/usx2y/us144mkii.c @@ -412,7 +412,6 @@ static int tascam_probe(struct usb_interface *intf, struct snd_card *card; struct tascam_card *tascam; int err; - char *handshake_buf __free(kfree) = NULL; if (dev->speed != USB_SPEED_HIGH) dev_info( @@ -439,7 +438,8 @@ static int tascam_probe(struct usb_interface *intf, return -ENOENT; } - handshake_buf = kmalloc(1, GFP_KERNEL); + char *handshake_buf __free(kfree) = + kmalloc(1, GFP_KERNEL); if (!handshake_buf) return -ENOMEM; diff --git a/sound/usb/usx2y/us144mkii_controls.c b/sound/usb/usx2y/us144mkii_controls.c index 5d69441ef414b9..62055fb8e7bace 100644 --- a/sound/usb/usx2y/us144mkii_controls.c +++ b/sound/usb/usx2y/us144mkii_controls.c @@ -373,7 +373,6 @@ static int tascam_samplerate_get(struct snd_kcontrol *kcontrol, { struct tascam_card *tascam = (struct tascam_card *)snd_kcontrol_chip(kcontrol); - u8 *buf __free(kfree) = NULL; int err; u32 rate = 0; @@ -384,7 +383,8 @@ static int tascam_samplerate_get(struct snd_kcontrol *kcontrol, } } - buf = kmalloc(3, GFP_KERNEL); + u8 *buf __free(kfree) = + kmalloc(3, GFP_KERNEL); if (!buf) return -ENOMEM; diff --git a/sound/usb/usx2y/us144mkii_pcm.c b/sound/usb/usx2y/us144mkii_pcm.c index 0c84304d462469..03dfb1f3880122 100644 --- a/sound/usb/usx2y/us144mkii_pcm.c +++ b/sound/usb/usx2y/us144mkii_pcm.c @@ -115,7 +115,6 @@ void process_capture_routing_us144mkii(struct tascam_card *tascam, int us144mkii_configure_device_for_rate(struct tascam_card *tascam, int rate) { struct usb_device *dev = tascam->dev; - u8 *rate_payload_buf __free(kfree) = NULL; u16 rate_vendor_wValue; int err = 0; const u8 *current_payload_src; @@ -148,7 +147,8 @@ int us144mkii_configure_device_for_rate(struct tascam_card *tascam, int rate) return -EINVAL; } - rate_payload_buf = kmemdup(current_payload_src, 3, GFP_KERNEL); + u8 *rate_payload_buf __free(kfree) = + kmemdup(current_payload_src, 3, GFP_KERNEL); if (!rate_payload_buf) return -ENOMEM; diff --git a/tools/bpf/bpftool/net.c b/tools/bpf/bpftool/net.c index cfc6f944f7c33a..1a06b0b5eef350 100644 --- a/tools/bpf/bpftool/net.c +++ b/tools/bpf/bpftool/net.c @@ -156,7 +156,7 @@ static int netlink_recv(int sock, __u32 nl_pid, __u32 seq, bool multipart = true; struct nlmsgerr *err; struct nlmsghdr *nh; - char buf[4096]; + char buf[8192]; int len, ret; while (multipart) { @@ -201,6 +201,9 @@ static int netlink_recv(int sock, __u32 nl_pid, __u32 seq, return ret; } } + + if (len) + p_err("Invalid message or trailing data in Netlink response: %d bytes left", len); } ret = 0; done: diff --git a/tools/docs/find-unused-docs.sh b/tools/docs/find-unused-docs.sh index 05552dbda5bcb2..ca4e607ec3f720 100755 --- a/tools/docs/find-unused-docs.sh +++ b/tools/docs/find-unused-docs.sh @@ -28,7 +28,7 @@ if ! [ -d "$1" ]; then fi cd "$( dirname "${BASH_SOURCE[0]}" )" -cd .. +cd ../.. cd Documentation/ diff --git a/tools/include/nolibc/Makefile b/tools/include/nolibc/Makefile index 8118e22844f199..afb6a20ac1336c 100644 --- a/tools/include/nolibc/Makefile +++ b/tools/include/nolibc/Makefile @@ -103,9 +103,12 @@ headers_standalone: headers $(Q)$(MAKE) -C $(srctree) headers $(Q)$(MAKE) -C $(srctree) headers_install INSTALL_HDR_PATH=$(OUTPUT)sysroot +CFLAGS_s390 := -m64 +CFLAGS := $(CFLAGS_$(ARCH)) + headers_check: headers_standalone $(Q)for header in $(filter-out crt.h std.h,$(all_files)); do \ - $(CC) $(CLANG_CROSS_FLAGS) -Wall -Werror -nostdinc -fsyntax-only -x c /dev/null \ + $(CC) $(CFLAGS) $(CLANG_CROSS_FLAGS) -Wall -Werror -nostdinc -fsyntax-only -x c /dev/null \ -I$(or $(objtree),$(srctree))/usr/include -include $$header -include $$header || exit 1; \ done diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c index 6388392f49a0b5..53c6624161d797 100644 --- a/tools/lib/bpf/btf_dump.c +++ b/tools/lib/bpf/btf_dump.c @@ -1762,9 +1762,18 @@ static int btf_dump_get_bitfield_value(struct btf_dump *d, __u16 left_shift_bits, right_shift_bits; const __u8 *bytes = data; __u8 nr_copy_bits; + __u8 start_bit, nr_bytes; __u64 num = 0; int i; + /* Calculate how many bytes cover the bitfield */ + start_bit = bits_offset % 8; + nr_bytes = (start_bit + bit_sz + 7) / 8; + + /* Bound check */ + if (data + nr_bytes > d->typed_dump->data_end) + return -E2BIG; + /* Maximum supported bitfield size is 64 bits */ if (t->size > 8) { pr_warn("unexpected bitfield size %d\n", t->size); diff --git a/tools/lib/bpf/linker.c b/tools/lib/bpf/linker.c index f4403e3cf99464..78f92c39290af5 100644 --- a/tools/lib/bpf/linker.c +++ b/tools/lib/bpf/linker.c @@ -581,7 +581,7 @@ int bpf_linker__add_buf(struct bpf_linker *linker, void *buf, size_t buf_sz, written = 0; while (written < buf_sz) { - ret = write(fd, buf, buf_sz); + ret = write(fd, buf + written, buf_sz - written); if (ret < 0) { ret = -errno; pr_warn("failed to write '%s': %s\n", filename, errstr(ret)); diff --git a/tools/lib/bpf/netlink.c b/tools/lib/bpf/netlink.c index c997e69d507fef..c9a78fb16f115a 100644 --- a/tools/lib/bpf/netlink.c +++ b/tools/lib/bpf/netlink.c @@ -143,7 +143,7 @@ static int libbpf_netlink_recv(int sock, __u32 nl_pid, int seq, struct nlmsghdr *nh; int len, ret; - ret = alloc_iov(&iov, 4096); + ret = alloc_iov(&iov, 8192); if (ret) goto done; @@ -212,6 +212,8 @@ static int libbpf_netlink_recv(int sock, __u32 nl_pid, int seq, } } } + if (len) + pr_warn("Invalid message or trailing data in Netlink response: %d bytes left\n", len); } ret = 0; done: diff --git a/tools/net/sunrpc/xdrgen/generators/__init__.py b/tools/net/sunrpc/xdrgen/generators/__init__.py index e22632cf38fbec..1d577a986c6cf8 100644 --- a/tools/net/sunrpc/xdrgen/generators/__init__.py +++ b/tools/net/sunrpc/xdrgen/generators/__init__.py @@ -6,7 +6,7 @@ from jinja2 import Environment, FileSystemLoader, Template from xdr_ast import _XdrAst, Specification, _RpcProgram, _XdrTypeSpecifier -from xdr_ast import public_apis, pass_by_reference, get_header_name +from xdr_ast import public_apis, pass_by_reference, structs, get_header_name from xdr_parse import get_xdr_annotate @@ -25,6 +25,7 @@ def create_jinja2_environment(language: str, xdr_type: str) -> Environment: environment.globals["annotate"] = get_xdr_annotate() environment.globals["public_apis"] = public_apis environment.globals["pass_by_reference"] = pass_by_reference + environment.globals["structs"] = structs return environment case _: raise NotImplementedError("Language not supported") diff --git a/tools/net/sunrpc/xdrgen/templates/C/program/decoder/argument.j2 b/tools/net/sunrpc/xdrgen/templates/C/program/decoder/argument.j2 index 0b1709cca0d4a4..19b219dd276d3e 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/program/decoder/argument.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/program/decoder/argument.j2 @@ -14,7 +14,11 @@ bool {{ program }}_svc_decode_{{ argument }}(struct svc_rqst *rqstp, struct xdr_ {% if argument == 'void' %} return xdrgen_decode_void(xdr); {% else %} +{% if argument in structs %} struct {{ argument }} *argp = rqstp->rq_argp; +{% else %} + {{ argument }} *argp = rqstp->rq_argp; +{% endif %} return xdrgen_decode_{{ argument }}(xdr, argp); {% endif %} diff --git a/tools/net/sunrpc/xdrgen/templates/C/program/encoder/result.j2 b/tools/net/sunrpc/xdrgen/templates/C/program/encoder/result.j2 index 6fc61a5d47b7f5..746592cfda5628 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/program/encoder/result.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/program/encoder/result.j2 @@ -14,8 +14,14 @@ bool {{ program }}_svc_encode_{{ result }}(struct svc_rqst *rqstp, struct xdr_st {% if result == 'void' %} return xdrgen_encode_void(xdr); {% else %} +{% if result in structs %} struct {{ result }} *resp = rqstp->rq_resp; return xdrgen_encode_{{ result }}(xdr, resp); +{% else %} + {{ result }} *resp = rqstp->rq_resp; + + return xdrgen_encode_{{ result }}(xdr, *resp); +{% endif %} {% endif %} } diff --git a/tools/net/sunrpc/xdrgen/templates/C/source_top/client.j2 b/tools/net/sunrpc/xdrgen/templates/C/source_top/client.j2 index c5518c519854a0..df3598c38b2c29 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/source_top/client.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/source_top/client.j2 @@ -8,6 +8,5 @@ #include #include #include -#include #include diff --git a/tools/objtool/Makefile b/tools/objtool/Makefile index a40f3023292910..6964175abdfdf5 100644 --- a/tools/objtool/Makefile +++ b/tools/objtool/Makefile @@ -29,6 +29,8 @@ srctree := $(patsubst %/,%,$(dir $(CURDIR))) srctree := $(patsubst %/,%,$(dir $(srctree))) endif +RM ?= rm -f + LIBSUBCMD_DIR = $(srctree)/tools/lib/subcmd/ ifneq ($(OUTPUT),) LIBSUBCMD_OUTPUT = $(abspath $(OUTPUT))/libsubcmd diff --git a/tools/power/x86/intel-speed-select/isst-config.c b/tools/power/x86/intel-speed-select/isst-config.c index 558138eea75e97..d00d15490a98c2 100644 --- a/tools/power/x86/intel-speed-select/isst-config.c +++ b/tools/power/x86/intel-speed-select/isst-config.c @@ -950,9 +950,11 @@ int isolate_cpus(struct isst_id *id, int mask_size, cpu_set_t *cpu_mask, int lev ret = write(fd, "member", strlen("member")); if (ret == -1) { printf("Can't update to member\n"); + close(fd); return ret; } + close(fd); return 0; } diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c index 5ad45c2ac5bd8f..1b26d94c373fbd 100644 --- a/tools/power/x86/turbostat/turbostat.c +++ b/tools/power/x86/turbostat/turbostat.c @@ -2135,7 +2135,7 @@ off_t idx_to_offset(int idx) switch (idx) { case IDX_PKG_ENERGY: - if (valid_rapl_msrs & RAPL_AMD_F17H) + if (platform->plat_rapl_msrs & RAPL_AMD_F17H) offset = MSR_PKG_ENERGY_STAT; else offset = MSR_PKG_ENERGY_STATUS; @@ -3001,22 +3001,30 @@ void print_header(char *delim) } /* - * pct() + * pct(numerator, denominator) * - * If absolute value is < 1.1, return percentage - * otherwise, return nan + * Return sanity checked percentage (100.0 * numerator/denominotor) * - * return value is appropriate for printing percentages with %f - * while flagging some obvious erroneous values. + * n < 0: nan + * d <= 0: nan + * n/d > 1.1: nan */ -double pct(double d) +double pct(double numerator, double denominator) { + double retval; - double abs = fabs(d); + if (numerator < 0) + return nan(""); - if (abs < 1.10) - return (100.0 * d); - return nan(""); + if (denominator <= 0) + return nan(""); + + retval = 100.0 * numerator / denominator; + + if (retval > 110.0) + return nan(""); + + return retval; } int dump_counters(PER_THREAD_PARAMS) @@ -3046,7 +3054,7 @@ int dump_counters(PER_THREAD_PARAMS) outp += sprintf(outp, "LLC refs: %lld", t->llc.references); outp += sprintf(outp, "LLC miss: %lld", t->llc.misses); - outp += sprintf(outp, "LLC Hit%%: %.2f", pct((t->llc.references - t->llc.misses) / t->llc.references)); + outp += sprintf(outp, "LLC Hit%%: %.2f", pct((t->llc.references - t->llc.misses), t->llc.references)); for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { outp += sprintf(outp, "tADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, t->counter[i], mp->sp->path); @@ -3261,7 +3269,7 @@ int format_counters(PER_THREAD_PARAMS) outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), 1.0 / units * t->aperf / interval_float); if (DO_BIC(BIC_Busy)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(t->mperf / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(t->mperf, tsc)); if (DO_BIC(BIC_Bzy_MHz)) { if (has_base_hz) @@ -3302,7 +3310,7 @@ int format_counters(PER_THREAD_PARAMS) outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), t->llc.references / interval_float / 1000); if (DO_BIC(BIC_LLC_HIT)) - outp += sprintf(outp, fmt8, (printed++ ? delim : ""), pct((t->llc.references - t->llc.misses) / t->llc.references)); + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), pct((t->llc.references - t->llc.misses), t->llc.references)); } /* Added Thread Counters */ @@ -3315,7 +3323,7 @@ int format_counters(PER_THREAD_PARAMS) if (mp->type == COUNTER_USEC) outp += print_float_value(&printed, delim, t->counter[i] / interval_float / 10000); else - outp += print_float_value(&printed, delim, pct(t->counter[i] / tsc)); + outp += print_float_value(&printed, delim, pct(t->counter[i], tsc)); } } @@ -3329,7 +3337,7 @@ int format_counters(PER_THREAD_PARAMS) if (pp->type == COUNTER_USEC) outp += print_float_value(&printed, delim, t->perf_counter[i] / interval_float / 10000); else - outp += print_float_value(&printed, delim, pct(t->perf_counter[i] / tsc)); + outp += print_float_value(&printed, delim, pct(t->perf_counter[i], tsc)); } } @@ -3343,34 +3351,34 @@ int format_counters(PER_THREAD_PARAMS) break; case PMT_TYPE_XTAL_TIME: - value_converted = pct(value_raw / crystal_hz / interval_float); + value_converted = pct(value_raw / crystal_hz, interval_float); outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), value_converted); break; case PMT_TYPE_TCORE_CLOCK: - value_converted = pct(value_raw / tcore_clock_freq_hz / interval_float); + value_converted = pct(value_raw / tcore_clock_freq_hz, interval_float); outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), value_converted); } } /* C1 */ if (DO_BIC(BIC_CPU_c1)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(t->c1 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(t->c1, tsc)); /* print per-core data only for 1st thread in core */ if (!is_cpu_first_thread_in_core(t, c)) goto done; if (DO_BIC(BIC_CPU_c3)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c3 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c3, tsc)); if (DO_BIC(BIC_CPU_c6)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c6 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c6, tsc)); if (DO_BIC(BIC_CPU_c7)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c7 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c7, tsc)); /* Mod%c6 */ if (DO_BIC(BIC_Mod_c6)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->mc6_us / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->mc6_us, tsc)); if (DO_BIC(BIC_CoreTmp)) outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), c->core_temp_c); @@ -3386,7 +3394,7 @@ int format_counters(PER_THREAD_PARAMS) else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE) outp += print_decimal_value(mp->width, &printed, delim, c->counter[i]); else if (mp->format == FORMAT_PERCENT) - outp += print_float_value(&printed, delim, pct(c->counter[i] / tsc)); + outp += print_float_value(&printed, delim, pct(c->counter[i], tsc)); } /* Added perf Core counters */ @@ -3396,7 +3404,7 @@ int format_counters(PER_THREAD_PARAMS) else if (pp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE) outp += print_decimal_value(pp->width, &printed, delim, c->perf_counter[i]); else if (pp->format == FORMAT_PERCENT) - outp += print_float_value(&printed, delim, pct(c->perf_counter[i] / tsc)); + outp += print_float_value(&printed, delim, pct(c->perf_counter[i], tsc)); } /* Added PMT Core counters */ @@ -3409,12 +3417,12 @@ int format_counters(PER_THREAD_PARAMS) break; case PMT_TYPE_XTAL_TIME: - value_converted = pct(value_raw / crystal_hz / interval_float); + value_converted = pct(value_raw / crystal_hz, interval_float); outp += print_float_value(&printed, delim, value_converted); break; case PMT_TYPE_TCORE_CLOCK: - value_converted = pct(value_raw / tcore_clock_freq_hz / interval_float); + value_converted = pct(value_raw / tcore_clock_freq_hz, interval_float); outp += print_float_value(&printed, delim, value_converted); } } @@ -3470,39 +3478,39 @@ int format_counters(PER_THREAD_PARAMS) if (DO_BIC(BIC_Totl_c0)) outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100 * p->pkg_wtd_core_c0 / tsc); /* can exceed 100% */ if (DO_BIC(BIC_Any_c0)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_any_core_c0 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_any_core_c0, tsc)); if (DO_BIC(BIC_GFX_c0)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_any_gfxe_c0 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_any_gfxe_c0, tsc)); if (DO_BIC(BIC_CPUGFX)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_both_core_gfxe_c0 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_both_core_gfxe_c0, tsc)); if (DO_BIC(BIC_Pkgpc2)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc2 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc2, tsc)); if (DO_BIC(BIC_Pkgpc3)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc3 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc3, tsc)); if (DO_BIC(BIC_Pkgpc6)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc6 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc6, tsc)); if (DO_BIC(BIC_Pkgpc7)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc7 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc7, tsc)); if (DO_BIC(BIC_Pkgpc8)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc8 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc8, tsc)); if (DO_BIC(BIC_Pkgpc9)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc9 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc9, tsc)); if (DO_BIC(BIC_Pkgpc10)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc10 / tsc)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc10, tsc)); if (DO_BIC(BIC_Diec6)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->die_c6 / crystal_hz / interval_float)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->die_c6 / crystal_hz, interval_float)); if (DO_BIC(BIC_CPU_LPI)) { if (p->cpu_lpi >= 0) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->cpu_lpi / 1000000.0 / interval_float)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->cpu_lpi / 1000000.0, interval_float)); else outp += sprintf(outp, "%s(neg)", (printed++ ? delim : "")); } if (DO_BIC(BIC_SYS_LPI)) { if (p->sys_lpi >= 0) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->sys_lpi / 1000000.0 / interval_float)); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->sys_lpi / 1000000.0, interval_float)); else outp += sprintf(outp, "%s(neg)", (printed++ ? delim : "")); } @@ -3542,7 +3550,7 @@ int format_counters(PER_THREAD_PARAMS) else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE) outp += print_decimal_value(mp->width, &printed, delim, p->counter[i]); else if (mp->format == FORMAT_PERCENT) - outp += print_float_value(&printed, delim, pct(p->counter[i] / tsc)); + outp += print_float_value(&printed, delim, pct(p->counter[i], tsc)); } /* Added perf Package Counters */ @@ -3554,7 +3562,7 @@ int format_counters(PER_THREAD_PARAMS) else if (pp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE) outp += print_decimal_value(pp->width, &printed, delim, p->perf_counter[i]); else if (pp->format == FORMAT_PERCENT) - outp += print_float_value(&printed, delim, pct(p->perf_counter[i] / tsc)); + outp += print_float_value(&printed, delim, pct(p->perf_counter[i], tsc)); } /* Added PMT Package Counters */ @@ -3567,12 +3575,12 @@ int format_counters(PER_THREAD_PARAMS) break; case PMT_TYPE_XTAL_TIME: - value_converted = pct(value_raw / crystal_hz / interval_float); + value_converted = pct(value_raw / crystal_hz, interval_float); outp += print_float_value(&printed, delim, value_converted); break; case PMT_TYPE_TCORE_CLOCK: - value_converted = pct(value_raw / tcore_clock_freq_hz / interval_float); + value_converted = pct(value_raw / tcore_clock_freq_hz, interval_float); outp += print_float_value(&printed, delim, value_converted); } } diff --git a/tools/spi/.gitignore b/tools/spi/.gitignore index 14ddba3d21957b..038261b34ed83c 100644 --- a/tools/spi/.gitignore +++ b/tools/spi/.gitignore @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only spidev_fdx spidev_test +include/ diff --git a/tools/testing/selftests/bpf/prog_tests/stacktrace_ips.c b/tools/testing/selftests/bpf/prog_tests/stacktrace_ips.c index c9efdd2a5b18a5..c93718dafd9b63 100644 --- a/tools/testing/selftests/bpf/prog_tests/stacktrace_ips.c +++ b/tools/testing/selftests/bpf/prog_tests/stacktrace_ips.c @@ -74,11 +74,20 @@ static void test_stacktrace_ips_kprobe_multi(bool retprobe) load_kallsyms(); - check_stacktrace_ips(bpf_map__fd(skel->maps.stackmap), skel->bss->stack_key, 4, - ksym_get_addr("bpf_testmod_stacktrace_test_3"), - ksym_get_addr("bpf_testmod_stacktrace_test_2"), - ksym_get_addr("bpf_testmod_stacktrace_test_1"), - ksym_get_addr("bpf_testmod_test_read")); + if (retprobe) { + check_stacktrace_ips(bpf_map__fd(skel->maps.stackmap), skel->bss->stack_key, 4, + ksym_get_addr("bpf_testmod_stacktrace_test_3"), + ksym_get_addr("bpf_testmod_stacktrace_test_2"), + ksym_get_addr("bpf_testmod_stacktrace_test_1"), + ksym_get_addr("bpf_testmod_test_read")); + } else { + check_stacktrace_ips(bpf_map__fd(skel->maps.stackmap), skel->bss->stack_key, 5, + ksym_get_addr("bpf_testmod_stacktrace_test"), + ksym_get_addr("bpf_testmod_stacktrace_test_3"), + ksym_get_addr("bpf_testmod_stacktrace_test_2"), + ksym_get_addr("bpf_testmod_stacktrace_test_1"), + ksym_get_addr("bpf_testmod_test_read")); + } cleanup: stacktrace_ips__destroy(skel); diff --git a/tools/testing/selftests/bpf/prog_tests/test_xsk.c b/tools/testing/selftests/bpf/prog_tests/test_xsk.c index 5af28f359cfda4..bab4a31621c751 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_xsk.c +++ b/tools/testing/selftests/bpf/prog_tests/test_xsk.c @@ -433,7 +433,7 @@ static u32 pkt_nb_frags(u32 frame_size, struct pkt_stream *pkt_stream, struct pk } /* Search for the end of the packet in verbatim mode */ - if (!pkt_continues(pkt->options)) + if (!pkt_continues(pkt->options) || !pkt->valid) return nb_frags; next_frag = pkt_stream->current_pkt_nb; @@ -1090,6 +1090,8 @@ static int __receive_pkts(struct test_spec *test, struct xsk_socket_info *xsk) xsk_ring_prod__cancel(&umem->fq, nb_frags); } frags_processed -= nb_frags; + pkt_stream_cancel(pkt_stream); + pkts_sent--; } if (ifobj->use_fill_ring) diff --git a/tools/testing/selftests/bpf/prog_tests/wq.c b/tools/testing/selftests/bpf/prog_tests/wq.c index 15c67d23128b24..84831eecc935bd 100644 --- a/tools/testing/selftests/bpf/prog_tests/wq.c +++ b/tools/testing/selftests/bpf/prog_tests/wq.c @@ -16,12 +16,12 @@ void serial_test_wq(void) /* re-run the success test to check if the timer was actually executed */ wq_skel = wq__open_and_load(); - if (!ASSERT_OK_PTR(wq_skel, "wq_skel_load")) + if (!ASSERT_OK_PTR(wq_skel, "wq__open_and_load")) return; err = wq__attach(wq_skel); if (!ASSERT_OK(err, "wq_attach")) - return; + goto clean_up; prog_fd = bpf_program__fd(wq_skel->progs.test_syscall_array_sleepable); err = bpf_prog_test_run_opts(prog_fd, &topts); @@ -31,6 +31,7 @@ void serial_test_wq(void) usleep(50); /* 10 usecs should be enough, but give it extra */ ASSERT_EQ(wq_skel->bss->ok_sleepable, (1 << 1), "ok_sleepable"); +clean_up: wq__destroy(wq_skel); } diff --git a/tools/testing/selftests/bpf/veristat.c b/tools/testing/selftests/bpf/veristat.c index e962f133250c7d..1be1e353d40a7b 100644 --- a/tools/testing/selftests/bpf/veristat.c +++ b/tools/testing/selftests/bpf/veristat.c @@ -2580,7 +2580,7 @@ static void output_stats(const struct verif_stats *s, enum resfmt fmt, bool last if (last && fmt == RESFMT_TABLE) { output_header_underlines(); printf("Done. Processed %d files, %d programs. Skipped %d files, %d programs.\n", - env.files_processed, env.files_skipped, env.progs_processed, env.progs_skipped); + env.files_processed, env.progs_processed, env.files_skipped, env.progs_skipped); } } diff --git a/tools/testing/selftests/drivers/net/lib/sh/lib_netcons.sh b/tools/testing/selftests/drivers/net/lib/sh/lib_netcons.sh index ae8abff4be409e..64d3941576d5db 100644 --- a/tools/testing/selftests/drivers/net/lib/sh/lib_netcons.sh +++ b/tools/testing/selftests/drivers/net/lib/sh/lib_netcons.sh @@ -247,8 +247,8 @@ function listen_port_and_save_to() { SOCAT_MODE="UDP6-LISTEN" fi - # Just wait for 2 seconds - timeout 2 ip netns exec "${NAMESPACE}" \ + # Just wait for 3 seconds + timeout 3 ip netns exec "${NAMESPACE}" \ socat "${SOCAT_MODE}":"${PORT}",fork "${OUTPUT}" 2> /dev/null } diff --git a/tools/testing/selftests/drivers/net/mlxsw/tc_restrictions.sh b/tools/testing/selftests/drivers/net/mlxsw/tc_restrictions.sh index 0441a18f098b10..aac8ef490feb80 100755 --- a/tools/testing/selftests/drivers/net/mlxsw/tc_restrictions.sh +++ b/tools/testing/selftests/drivers/net/mlxsw/tc_restrictions.sh @@ -317,7 +317,7 @@ police_limits_test() tc filter add dev $swp1 ingress pref 1 proto ip handle 101 \ flower skip_sw \ - action police rate 0.5kbit burst 1m conform-exceed drop/ok + action police rate 0.5kbit burst 2k conform-exceed drop/ok check_fail $? "Incorrect success to add police action with too low rate" tc filter add dev $swp1 ingress pref 1 proto ip handle 101 \ @@ -327,7 +327,7 @@ police_limits_test() tc filter add dev $swp1 ingress pref 1 proto ip handle 101 \ flower skip_sw \ - action police rate 1.5kbit burst 1m conform-exceed drop/ok + action police rate 1.5kbit burst 2k conform-exceed drop/ok check_err $? "Failed to add police action with low rate" tc filter del dev $swp1 ingress protocol ip pref 1 handle 101 flower diff --git a/tools/testing/selftests/memfd/memfd_test.c b/tools/testing/selftests/memfd/memfd_test.c index 5b993924cc3f5b..2ca07ea7202a5f 100644 --- a/tools/testing/selftests/memfd/memfd_test.c +++ b/tools/testing/selftests/memfd/memfd_test.c @@ -18,6 +18,9 @@ #include #include #include +#include +#include +#include #include #include @@ -39,6 +42,20 @@ F_SEAL_EXEC) #define MFD_NOEXEC_SEAL 0x0008U +union semun { + int val; + struct semid_ds *buf; + unsigned short int *array; + struct seminfo *__buf; +}; + +/* + * we use semaphores on nested wait tasks due the use of CLONE_NEWPID: the + * child will be PID 1 and can't send SIGSTOP to themselves due special + * treatment of the init task, so the SIGSTOP/SIGCONT synchronization + * approach can't be used here. + */ +#define SEM_KEY 0xdeadbeef /* * Default is not to test hugetlbfs @@ -1333,8 +1350,22 @@ static int sysctl_nested(void *arg) static int sysctl_nested_wait(void *arg) { - /* Wait for a SIGCONT. */ - kill(getpid(), SIGSTOP); + int sem = semget(SEM_KEY, 1, 0600); + struct sembuf sembuf; + + if (sem < 0) { + perror("semget:"); + abort(); + } + sembuf.sem_num = 0; + sembuf.sem_flg = 0; + sembuf.sem_op = 0; + + if (semop(sem, &sembuf, 1) < 0) { + perror("semop:"); + abort(); + } + return sysctl_nested(arg); } @@ -1355,7 +1386,9 @@ static void test_sysctl_sysctl2_failset(void) static int sysctl_nested_child(void *arg) { - int pid; + int pid, sem; + union semun semun; + struct sembuf sembuf; printf("%s nested sysctl 0\n", memfd_str); sysctl_assert_write("0"); @@ -1389,23 +1422,53 @@ static int sysctl_nested_child(void *arg) test_sysctl_sysctl2_failset); join_thread(pid); + sem = semget(SEM_KEY, 1, IPC_CREAT | 0600); + if (sem < 0) { + perror("semget:"); + return 1; + } + semun.val = 1; + sembuf.sem_op = -1; + sembuf.sem_flg = 0; + sembuf.sem_num = 0; + /* Verify that the rules are actually inherited after fork. */ printf("%s nested sysctl 0 -> 1 after fork\n", memfd_str); sysctl_assert_write("0"); + if (semctl(sem, 0, SETVAL, semun) < 0) { + perror("semctl:"); + return 1; + } + pid = spawn_thread(CLONE_NEWPID, sysctl_nested_wait, test_sysctl_sysctl1_failset); sysctl_assert_write("1"); - kill(pid, SIGCONT); + + /* Allow child to continue */ + if (semop(sem, &sembuf, 1) < 0) { + perror("semop:"); + return 1; + } join_thread(pid); printf("%s nested sysctl 0 -> 2 after fork\n", memfd_str); sysctl_assert_write("0"); + if (semctl(sem, 0, SETVAL, semun) < 0) { + perror("semctl:"); + return 1; + } + pid = spawn_thread(CLONE_NEWPID, sysctl_nested_wait, test_sysctl_sysctl2_failset); sysctl_assert_write("2"); - kill(pid, SIGCONT); + + /* Allow child to continue */ + if (semop(sem, &sembuf, 1) < 0) { + perror("semop:"); + return 1; + } join_thread(pid); /* @@ -1415,28 +1478,62 @@ static int sysctl_nested_child(void *arg) */ printf("%s nested sysctl 2 -> 1 after fork\n", memfd_str); sysctl_assert_write("2"); + + if (semctl(sem, 0, SETVAL, semun) < 0) { + perror("semctl:"); + return 1; + } + pid = spawn_thread(CLONE_NEWPID, sysctl_nested_wait, test_sysctl_sysctl2); sysctl_assert_write("1"); - kill(pid, SIGCONT); + + /* Allow child to continue */ + if (semop(sem, &sembuf, 1) < 0) { + perror("semop:"); + return 1; + } join_thread(pid); printf("%s nested sysctl 2 -> 0 after fork\n", memfd_str); sysctl_assert_write("2"); + + if (semctl(sem, 0, SETVAL, semun) < 0) { + perror("semctl:"); + return 1; + } + pid = spawn_thread(CLONE_NEWPID, sysctl_nested_wait, test_sysctl_sysctl2); sysctl_assert_write("0"); - kill(pid, SIGCONT); + + /* Allow child to continue */ + if (semop(sem, &sembuf, 1) < 0) { + perror("semop:"); + return 1; + } join_thread(pid); printf("%s nested sysctl 1 -> 0 after fork\n", memfd_str); sysctl_assert_write("1"); + + if (semctl(sem, 0, SETVAL, semun) < 0) { + perror("semctl:"); + return 1; + } + pid = spawn_thread(CLONE_NEWPID, sysctl_nested_wait, test_sysctl_sysctl1); sysctl_assert_write("0"); - kill(pid, SIGCONT); + /* Allow child to continue */ + if (semop(sem, &sembuf, 1) < 0) { + perror("semop:"); + return 1; + } join_thread(pid); + semctl(sem, 0, IPC_RMID); + return 0; } diff --git a/tools/testing/selftests/mm/cow.c b/tools/testing/selftests/mm/cow.c index accfd198dbda84..83b3563be26b65 100644 --- a/tools/testing/selftests/mm/cow.c +++ b/tools/testing/selftests/mm/cow.c @@ -1612,8 +1612,8 @@ static void run_with_huge_zeropage(non_anon_test_fn fn, const char *desc) * the first sub-page and test if we get another sub-page populated * automatically. */ - FORCE_READ(mem); - FORCE_READ(smem); + FORCE_READ(*mem); + FORCE_READ(*smem); if (!pagemap_is_populated(pagemap_fd, mem + pagesize) || !pagemap_is_populated(pagemap_fd, smem + pagesize)) { ksft_test_result_skip("Did not get THPs populated\n"); @@ -1663,8 +1663,8 @@ static void run_with_memfd(non_anon_test_fn fn, const char *desc) } /* Fault the page in. */ - FORCE_READ(mem); - FORCE_READ(smem); + FORCE_READ(*mem); + FORCE_READ(*smem); fn(mem, smem, pagesize); munmap: @@ -1719,8 +1719,8 @@ static void run_with_tmpfile(non_anon_test_fn fn, const char *desc) } /* Fault the page in. */ - FORCE_READ(mem); - FORCE_READ(smem); + FORCE_READ(*mem); + FORCE_READ(*smem); fn(mem, smem, pagesize); munmap: @@ -1773,8 +1773,8 @@ static void run_with_memfd_hugetlb(non_anon_test_fn fn, const char *desc, } /* Fault the page in. */ - FORCE_READ(mem); - FORCE_READ(smem); + FORCE_READ(*mem); + FORCE_READ(*smem); fn(mem, smem, hugetlbsize); munmap: diff --git a/tools/testing/selftests/net/forwarding/pedit_dsfield.sh b/tools/testing/selftests/net/forwarding/pedit_dsfield.sh index af008fbf2725eb..eb2d8034de9c74 100755 --- a/tools/testing/selftests/net/forwarding/pedit_dsfield.sh +++ b/tools/testing/selftests/net/forwarding/pedit_dsfield.sh @@ -98,12 +98,20 @@ setup_prepare() h1_create h2_create switch_create + + if [ -f /proc/sys/net/bridge/bridge-nf-call-iptables ]; then + sysctl_set net.bridge.bridge-nf-call-iptables 0 + fi } cleanup() { pre_cleanup + if [ -f /proc/sys/net/bridge/bridge-nf-call-iptables ]; then + sysctl_restore net.bridge.bridge-nf-call-iptables + fi + switch_destroy h2_destroy h1_destroy diff --git a/tools/testing/selftests/net/forwarding/pedit_ip.sh b/tools/testing/selftests/net/forwarding/pedit_ip.sh index d14efb2d23b2e4..9235674627abd5 100755 --- a/tools/testing/selftests/net/forwarding/pedit_ip.sh +++ b/tools/testing/selftests/net/forwarding/pedit_ip.sh @@ -91,12 +91,20 @@ setup_prepare() h1_create h2_create switch_create + + if [ -f /proc/sys/net/bridge/bridge-nf-call-iptables ]; then + sysctl_set net.bridge.bridge-nf-call-iptables 0 + fi } cleanup() { pre_cleanup + if [ -f /proc/sys/net/bridge/bridge-nf-call-iptables ]; then + sysctl_restore net.bridge.bridge-nf-call-iptables + fi + switch_destroy h2_destroy h1_destroy diff --git a/tools/testing/selftests/net/forwarding/tc_actions.sh b/tools/testing/selftests/net/forwarding/tc_actions.sh index ea89e558672db0..86edbc7e2489b3 100755 --- a/tools/testing/selftests/net/forwarding/tc_actions.sh +++ b/tools/testing/selftests/net/forwarding/tc_actions.sh @@ -223,7 +223,7 @@ mirred_egress_to_ingress_tcp_test() ip_proto icmp \ action drop - ip vrf exec v$h1 ncat --recv-only -w10 -l -p 12345 -o $mirred_e2i_tf2 & + ip vrf exec v$h1 ncat --recv-only -w10 -l -p 12345 > $mirred_e2i_tf2 & local rpid=$! ip vrf exec v$h1 ncat -w1 --send-only 192.0.2.2 12345 <$mirred_e2i_tf1 wait -n $rpid diff --git a/tools/testing/selftests/net/forwarding/vxlan_bridge_1d.sh b/tools/testing/selftests/net/forwarding/vxlan_bridge_1d.sh index b43816dd998cab..457f41d5e584b5 100755 --- a/tools/testing/selftests/net/forwarding/vxlan_bridge_1d.sh +++ b/tools/testing/selftests/net/forwarding/vxlan_bridge_1d.sh @@ -567,6 +567,21 @@ vxlan_encapped_ping_do() local inner_tos=$1; shift local outer_tos=$1; shift + local ipv4hdr=$(: + )"45:"$( : IP version + IHL + )"$inner_tos:"$( : IP TOS + )"00:54:"$( : IP total length + )"99:83:"$( : IP identification + )"40:00:"$( : IP flags + frag off + )"40:"$( : IP TTL + )"01:"$( : IP proto + )"CHECKSUM:"$( : IP header csum + )"c0:00:02:03:"$( : IP saddr: 192.0.2.3 + )"c0:00:02:01"$( : IP daddr: 192.0.2.1 + ) + local checksum=$(payload_template_calc_checksum "$ipv4hdr") + ipv4hdr=$(payload_template_expand_checksum "$ipv4hdr" $checksum) + $MZ $dev -c $count -d 100msec -q \ -b $next_hop_mac -B $dest_ip \ -t udp tos=$outer_tos,sp=23456,dp=$VXPORT,p=$(: @@ -577,16 +592,7 @@ vxlan_encapped_ping_do() )"$dest_mac:"$( : ETH daddr )"$(mac_get w2):"$( : ETH saddr )"08:00:"$( : ETH type - )"45:"$( : IP version + IHL - )"$inner_tos:"$( : IP TOS - )"00:54:"$( : IP total length - )"99:83:"$( : IP identification - )"40:00:"$( : IP flags + frag off - )"40:"$( : IP TTL - )"01:"$( : IP proto - )"00:00:"$( : IP header csum - )"c0:00:02:03:"$( : IP saddr: 192.0.2.3 - )"c0:00:02:01:"$( : IP daddr: 192.0.2.1 + )"$ipv4hdr:"$( : IPv4 header )"08:"$( : ICMP type )"00:"$( : ICMP code )"8b:f2:"$( : ICMP csum diff --git a/tools/testing/selftests/net/forwarding/vxlan_bridge_1d_ipv6.sh b/tools/testing/selftests/net/forwarding/vxlan_bridge_1d_ipv6.sh index a603f7b0a08f07..e642feeada0e7e 100755 --- a/tools/testing/selftests/net/forwarding/vxlan_bridge_1d_ipv6.sh +++ b/tools/testing/selftests/net/forwarding/vxlan_bridge_1d_ipv6.sh @@ -695,7 +695,7 @@ vxlan_encapped_ping_do() )"6"$( : IP version )"$inner_tos"$( : Traffic class )"0:00:00:"$( : Flow label - )"00:08:"$( : Payload length + )"00:03:"$( : Payload length )"3a:"$( : Next header )"04:"$( : Hop limit )"$saddr:"$( : IP saddr diff --git a/tools/testing/selftests/net/lib.sh b/tools/testing/selftests/net/lib.sh index 0ec131b339bc42..b40694573f4c7d 100644 --- a/tools/testing/selftests/net/lib.sh +++ b/tools/testing/selftests/net/lib.sh @@ -577,7 +577,7 @@ ip_link_has_flag() local flag=$1; shift local state=$(ip -j link show "$name" | - jq --arg flag "$flag" 'any(.[].flags.[]; . == $flag)') + jq --arg flag "$flag" 'any(.[].flags[]; . == $flag)') [[ $state == true ]] } diff --git a/tools/testing/selftests/resctrl/resctrlfs.c b/tools/testing/selftests/resctrl/resctrlfs.c index 195f04c4d15868..b9c1bfb6cc029a 100644 --- a/tools/testing/selftests/resctrl/resctrlfs.c +++ b/tools/testing/selftests/resctrl/resctrlfs.c @@ -243,6 +243,16 @@ int snc_nodes_per_l3_cache(void) } snc_mode = cache_cpus / node_cpus; + /* + * On some platforms (e.g. Hygon), + * cache_cpus < node_cpus, the calculated snc_mode is 0. + * + * Set snc_mode = 1 to indicate that SNC mode is not + * supported on the platform. + */ + if (!snc_mode) + snc_mode = 1; + if (snc_mode > 1) ksft_print_msg("SNC-%d mode discovered.\n", snc_mode); }